1
2#ifndef _LINUX_CGROUP_H
3#define _LINUX_CGROUP_H
4
5
6
7
8
9
10
11
12#include <linux/sched.h>
13#include <linux/cpumask.h>
14#include <linux/nodemask.h>
15#include <linux/rculist.h>
16#include <linux/cgroupstats.h>
17#include <linux/fs.h>
18#include <linux/seq_file.h>
19#include <linux/kernfs.h>
20#include <linux/jump_label.h>
21#include <linux/types.h>
22#include <linux/ns_common.h>
23#include <linux/nsproxy.h>
24#include <linux/user_namespace.h>
25#include <linux/refcount.h>
26#include <linux/kernel_stat.h>
27
28#include <linux/cgroup-defs.h>
29
30#ifdef CONFIG_CGROUPS
31
32
33
34
35
36
37#define CGROUP_WEIGHT_MIN 1
38#define CGROUP_WEIGHT_DFL 100
39#define CGROUP_WEIGHT_MAX 10000
40
41
42#define CSS_TASK_ITER_PROCS (1U << 0)
43
44#define CSS_TASK_ITER_THREADED (1U << 1)
45
46
47#define CSS_TASK_ITER_SKIPPED (1U << 16)
48
49
50struct css_task_iter {
51 struct cgroup_subsys *ss;
52 unsigned int flags;
53
54 struct list_head *cset_pos;
55 struct list_head *cset_head;
56
57 struct list_head *tcset_pos;
58 struct list_head *tcset_head;
59
60 struct list_head *task_pos;
61 struct list_head *tasks_head;
62 struct list_head *mg_tasks_head;
63 struct list_head *dying_tasks_head;
64
65 struct css_set *cur_cset;
66 struct css_set *cur_dcset;
67 struct task_struct *cur_task;
68 struct list_head iters_node;
69};
70
71extern struct cgroup_root cgrp_dfl_root;
72extern struct css_set init_css_set;
73
74#define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys;
75#include <linux/cgroup_subsys.h>
76#undef SUBSYS
77
78#define SUBSYS(_x) \
79 extern struct static_key_true _x ## _cgrp_subsys_enabled_key; \
80 extern struct static_key_true _x ## _cgrp_subsys_on_dfl_key;
81#include <linux/cgroup_subsys.h>
82#undef SUBSYS
83
84
85
86
87
88#define cgroup_subsys_enabled(ss) \
89 static_branch_likely(&ss ## _enabled_key)
90
91
92
93
94
95#define cgroup_subsys_on_dfl(ss) \
96 static_branch_likely(&ss ## _on_dfl_key)
97
98bool css_has_online_children(struct cgroup_subsys_state *css);
99struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss);
100struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgroup,
101 struct cgroup_subsys *ss);
102struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgroup,
103 struct cgroup_subsys *ss);
104struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
105 struct cgroup_subsys *ss);
106
107struct cgroup *cgroup_get_from_path(const char *path);
108struct cgroup *cgroup_get_from_fd(int fd);
109
110int cgroup_attach_task_all(struct task_struct *from, struct task_struct *);
111int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from);
112
113int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts);
114int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts);
115int cgroup_rm_cftypes(struct cftype *cfts);
116void cgroup_file_notify(struct cgroup_file *cfile);
117
118int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen);
119int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry);
120int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
121 struct pid *pid, struct task_struct *tsk);
122
123void cgroup_fork(struct task_struct *p);
124extern int cgroup_can_fork(struct task_struct *p);
125extern void cgroup_cancel_fork(struct task_struct *p);
126extern void cgroup_post_fork(struct task_struct *p);
127void cgroup_exit(struct task_struct *p);
128void cgroup_release(struct task_struct *p);
129void cgroup_free(struct task_struct *p);
130
131int cgroup_init_early(void);
132int cgroup_init(void);
133
134
135
136
137
138struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
139 struct cgroup_subsys_state *parent);
140struct cgroup_subsys_state *css_next_descendant_pre(struct cgroup_subsys_state *pos,
141 struct cgroup_subsys_state *css);
142struct cgroup_subsys_state *css_rightmost_descendant(struct cgroup_subsys_state *pos);
143struct cgroup_subsys_state *css_next_descendant_post(struct cgroup_subsys_state *pos,
144 struct cgroup_subsys_state *css);
145
146struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
147 struct cgroup_subsys_state **dst_cssp);
148struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
149 struct cgroup_subsys_state **dst_cssp);
150
151void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,
152 struct css_task_iter *it);
153struct task_struct *css_task_iter_next(struct css_task_iter *it);
154void css_task_iter_end(struct css_task_iter *it);
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174#define css_for_each_child(pos, parent) \
175 for ((pos) = css_next_child(NULL, (parent)); (pos); \
176 (pos) = css_next_child((pos), (parent)))
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234#define css_for_each_descendant_pre(pos, css) \
235 for ((pos) = css_next_descendant_pre(NULL, (css)); (pos); \
236 (pos) = css_next_descendant_pre((pos), (css)))
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257#define css_for_each_descendant_post(pos, css) \
258 for ((pos) = css_next_descendant_post(NULL, (css)); (pos); \
259 (pos) = css_next_descendant_post((pos), (css)))
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279#define cgroup_taskset_for_each(task, dst_css, tset) \
280 for ((task) = cgroup_taskset_first((tset), &(dst_css)); \
281 (task); \
282 (task) = cgroup_taskset_next((tset), &(dst_css)))
283
284
285
286
287
288
289
290
291
292
293#define cgroup_taskset_for_each_leader(leader, dst_css, tset) \
294 for ((leader) = cgroup_taskset_first((tset), &(dst_css)); \
295 (leader); \
296 (leader) = cgroup_taskset_next((tset), &(dst_css))) \
297 if ((leader) != (leader)->group_leader) \
298 ; \
299 else
300
301
302
303
304
305
306
307
308
309
310
311static inline void css_get(struct cgroup_subsys_state *css)
312{
313 if (!(css->flags & CSS_NO_REF))
314 percpu_ref_get(&css->refcnt);
315}
316
317
318
319
320
321
322
323
324static inline void css_get_many(struct cgroup_subsys_state *css, unsigned int n)
325{
326 if (!(css->flags & CSS_NO_REF))
327 percpu_ref_get_many(&css->refcnt, n);
328}
329
330
331
332
333
334
335
336
337
338
339
340
341static inline bool css_tryget(struct cgroup_subsys_state *css)
342{
343 if (!(css->flags & CSS_NO_REF))
344 return percpu_ref_tryget(&css->refcnt);
345 return true;
346}
347
348
349
350
351
352
353
354
355
356
357
358static inline bool css_tryget_online(struct cgroup_subsys_state *css)
359{
360 if (!(css->flags & CSS_NO_REF))
361 return percpu_ref_tryget_live(&css->refcnt);
362 return true;
363}
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380static inline bool css_is_dying(struct cgroup_subsys_state *css)
381{
382 return !(css->flags & CSS_NO_REF) && percpu_ref_is_dying(&css->refcnt);
383}
384
385
386
387
388
389
390
391static inline void css_put(struct cgroup_subsys_state *css)
392{
393 if (!(css->flags & CSS_NO_REF))
394 percpu_ref_put(&css->refcnt);
395}
396
397
398
399
400
401
402
403
404static inline void css_put_many(struct cgroup_subsys_state *css, unsigned int n)
405{
406 if (!(css->flags & CSS_NO_REF))
407 percpu_ref_put_many(&css->refcnt, n);
408}
409
410static inline void cgroup_get(struct cgroup *cgrp)
411{
412 css_get(&cgrp->self);
413}
414
415static inline bool cgroup_tryget(struct cgroup *cgrp)
416{
417 return css_tryget(&cgrp->self);
418}
419
420static inline void cgroup_put(struct cgroup *cgrp)
421{
422 css_put(&cgrp->self);
423}
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438#ifdef CONFIG_PROVE_RCU
439extern struct mutex cgroup_mutex;
440extern spinlock_t css_set_lock;
441#define task_css_set_check(task, __c) \
442 rcu_dereference_check((task)->cgroups, \
443 lockdep_is_held(&cgroup_mutex) || \
444 lockdep_is_held(&css_set_lock) || \
445 ((task)->flags & PF_EXITING) || (__c))
446#else
447#define task_css_set_check(task, __c) \
448 rcu_dereference((task)->cgroups)
449#endif
450
451
452
453
454
455
456
457
458
459
460#define task_css_check(task, subsys_id, __c) \
461 task_css_set_check((task), (__c))->subsys[(subsys_id)]
462
463
464
465
466
467
468
469static inline struct css_set *task_css_set(struct task_struct *task)
470{
471 return task_css_set_check(task, false);
472}
473
474
475
476
477
478
479
480
481static inline struct cgroup_subsys_state *task_css(struct task_struct *task,
482 int subsys_id)
483{
484 return task_css_check(task, subsys_id, false);
485}
486
487
488
489
490
491
492
493
494
495
496static inline struct cgroup_subsys_state *
497task_get_css(struct task_struct *task, int subsys_id)
498{
499 struct cgroup_subsys_state *css;
500
501 rcu_read_lock();
502 while (true) {
503 css = task_css(task, subsys_id);
504
505
506
507
508
509
510 if (likely(css_tryget(css)))
511 break;
512 cpu_relax();
513 }
514 rcu_read_unlock();
515 return css;
516}
517
518
519
520
521
522
523
524
525
526static inline bool task_css_is_root(struct task_struct *task, int subsys_id)
527{
528 return task_css_check(task, subsys_id, true) ==
529 init_css_set.subsys[subsys_id];
530}
531
532static inline struct cgroup *task_cgroup(struct task_struct *task,
533 int subsys_id)
534{
535 return task_css(task, subsys_id)->cgroup;
536}
537
538static inline struct cgroup *task_dfl_cgroup(struct task_struct *task)
539{
540 return task_css_set(task)->dfl_cgrp;
541}
542
543static inline struct cgroup *cgroup_parent(struct cgroup *cgrp)
544{
545 struct cgroup_subsys_state *parent_css = cgrp->self.parent;
546
547 if (parent_css)
548 return container_of(parent_css, struct cgroup, self);
549 return NULL;
550}
551
552
553
554
555
556
557
558
559
560
561static inline bool cgroup_is_descendant(struct cgroup *cgrp,
562 struct cgroup *ancestor)
563{
564 if (cgrp->root != ancestor->root || cgrp->level < ancestor->level)
565 return false;
566 return cgrp->ancestor_ids[ancestor->level] == ancestor->id;
567}
568
569
570
571
572
573
574
575
576
577
578
579
580static inline struct cgroup *cgroup_ancestor(struct cgroup *cgrp,
581 int ancestor_level)
582{
583 if (cgrp->level < ancestor_level)
584 return NULL;
585 while (cgrp && cgrp->level > ancestor_level)
586 cgrp = cgroup_parent(cgrp);
587 return cgrp;
588}
589
590
591
592
593
594
595
596
597
598
599static inline bool task_under_cgroup_hierarchy(struct task_struct *task,
600 struct cgroup *ancestor)
601{
602 struct css_set *cset = task_css_set(task);
603
604 return cgroup_is_descendant(cset->dfl_cgrp, ancestor);
605}
606
607
608static inline bool cgroup_is_populated(struct cgroup *cgrp)
609{
610 return cgrp->nr_populated_csets + cgrp->nr_populated_domain_children +
611 cgrp->nr_populated_threaded_children;
612}
613
614
615static inline ino_t cgroup_ino(struct cgroup *cgrp)
616{
617 return cgrp->kn->id.ino;
618}
619
620
621static inline struct cftype *of_cft(struct kernfs_open_file *of)
622{
623 return of->kn->priv;
624}
625
626struct cgroup_subsys_state *of_css(struct kernfs_open_file *of);
627
628
629static inline struct cftype *seq_cft(struct seq_file *seq)
630{
631 return of_cft(seq->private);
632}
633
634static inline struct cgroup_subsys_state *seq_css(struct seq_file *seq)
635{
636 return of_css(seq->private);
637}
638
639
640
641
642
643
644static inline int cgroup_name(struct cgroup *cgrp, char *buf, size_t buflen)
645{
646 return kernfs_name(cgrp->kn, buf, buflen);
647}
648
649static inline int cgroup_path(struct cgroup *cgrp, char *buf, size_t buflen)
650{
651 return kernfs_path(cgrp->kn, buf, buflen);
652}
653
654static inline void pr_cont_cgroup_name(struct cgroup *cgrp)
655{
656 pr_cont_kernfs_name(cgrp->kn);
657}
658
659static inline void pr_cont_cgroup_path(struct cgroup *cgrp)
660{
661 pr_cont_kernfs_path(cgrp->kn);
662}
663
664static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
665{
666 return &cgrp->psi;
667}
668
669static inline void cgroup_init_kthreadd(void)
670{
671
672
673
674
675
676 current->no_cgroup_migration = 1;
677}
678
679static inline void cgroup_kthread_ready(void)
680{
681
682
683
684
685 current->no_cgroup_migration = 0;
686}
687
688static inline union kernfs_node_id *cgroup_get_kernfs_id(struct cgroup *cgrp)
689{
690 return &cgrp->kn->id;
691}
692
693void cgroup_path_from_kernfs_id(const union kernfs_node_id *id,
694 char *buf, size_t buflen);
695#else
696
697struct cgroup_subsys_state;
698struct cgroup;
699
700static inline void css_put(struct cgroup_subsys_state *css) {}
701static inline int cgroup_attach_task_all(struct task_struct *from,
702 struct task_struct *t) { return 0; }
703static inline int cgroupstats_build(struct cgroupstats *stats,
704 struct dentry *dentry) { return -EINVAL; }
705
706static inline void cgroup_fork(struct task_struct *p) {}
707static inline int cgroup_can_fork(struct task_struct *p) { return 0; }
708static inline void cgroup_cancel_fork(struct task_struct *p) {}
709static inline void cgroup_post_fork(struct task_struct *p) {}
710static inline void cgroup_exit(struct task_struct *p) {}
711static inline void cgroup_release(struct task_struct *p) {}
712static inline void cgroup_free(struct task_struct *p) {}
713
714static inline int cgroup_init_early(void) { return 0; }
715static inline int cgroup_init(void) { return 0; }
716static inline void cgroup_init_kthreadd(void) {}
717static inline void cgroup_kthread_ready(void) {}
718static inline union kernfs_node_id *cgroup_get_kernfs_id(struct cgroup *cgrp)
719{
720 return NULL;
721}
722
723static inline struct cgroup *cgroup_parent(struct cgroup *cgrp)
724{
725 return NULL;
726}
727
728static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
729{
730 return NULL;
731}
732
733static inline bool task_under_cgroup_hierarchy(struct task_struct *task,
734 struct cgroup *ancestor)
735{
736 return true;
737}
738
739static inline void cgroup_path_from_kernfs_id(const union kernfs_node_id *id,
740 char *buf, size_t buflen) {}
741#endif
742
743#ifdef CONFIG_CGROUPS
744
745
746
747void cgroup_rstat_updated(struct cgroup *cgrp, int cpu);
748void cgroup_rstat_flush(struct cgroup *cgrp);
749void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp);
750void cgroup_rstat_flush_hold(struct cgroup *cgrp);
751void cgroup_rstat_flush_release(void);
752
753
754
755
756#ifdef CONFIG_CGROUP_CPUACCT
757void cpuacct_charge(struct task_struct *tsk, u64 cputime);
758void cpuacct_account_field(struct task_struct *tsk, int index, u64 val);
759#else
760static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
761static inline void cpuacct_account_field(struct task_struct *tsk, int index,
762 u64 val) {}
763#endif
764
765void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec);
766void __cgroup_account_cputime_field(struct cgroup *cgrp,
767 enum cpu_usage_stat index, u64 delta_exec);
768
769static inline void cgroup_account_cputime(struct task_struct *task,
770 u64 delta_exec)
771{
772 struct cgroup *cgrp;
773
774 cpuacct_charge(task, delta_exec);
775
776 rcu_read_lock();
777 cgrp = task_dfl_cgroup(task);
778 if (cgroup_parent(cgrp))
779 __cgroup_account_cputime(cgrp, delta_exec);
780 rcu_read_unlock();
781}
782
783static inline void cgroup_account_cputime_field(struct task_struct *task,
784 enum cpu_usage_stat index,
785 u64 delta_exec)
786{
787 struct cgroup *cgrp;
788
789 cpuacct_account_field(task, index, delta_exec);
790
791 rcu_read_lock();
792 cgrp = task_dfl_cgroup(task);
793 if (cgroup_parent(cgrp))
794 __cgroup_account_cputime_field(cgrp, index, delta_exec);
795 rcu_read_unlock();
796}
797
798#else
799
800static inline void cgroup_account_cputime(struct task_struct *task,
801 u64 delta_exec) {}
802static inline void cgroup_account_cputime_field(struct task_struct *task,
803 enum cpu_usage_stat index,
804 u64 delta_exec) {}
805
806#endif
807
808
809
810
811
812#ifdef CONFIG_SOCK_CGROUP_DATA
813
814#if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID)
815extern spinlock_t cgroup_sk_update_lock;
816#endif
817
818void cgroup_sk_alloc_disable(void);
819void cgroup_sk_alloc(struct sock_cgroup_data *skcd);
820void cgroup_sk_free(struct sock_cgroup_data *skcd);
821
822static inline struct cgroup *sock_cgroup_ptr(struct sock_cgroup_data *skcd)
823{
824#if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID)
825 unsigned long v;
826
827
828
829
830
831 v = READ_ONCE(skcd->val);
832
833 if (v & 1)
834 return &cgrp_dfl_root.cgrp;
835
836 return (struct cgroup *)(unsigned long)v ?: &cgrp_dfl_root.cgrp;
837#else
838 return (struct cgroup *)(unsigned long)skcd->val;
839#endif
840}
841
842#else
843
844static inline void cgroup_sk_alloc(struct sock_cgroup_data *skcd) {}
845static inline void cgroup_sk_free(struct sock_cgroup_data *skcd) {}
846
847#endif
848
849struct cgroup_namespace {
850 refcount_t count;
851 struct ns_common ns;
852 struct user_namespace *user_ns;
853 struct ucounts *ucounts;
854 struct css_set *root_cset;
855};
856
857extern struct cgroup_namespace init_cgroup_ns;
858
859#ifdef CONFIG_CGROUPS
860
861void free_cgroup_ns(struct cgroup_namespace *ns);
862
863struct cgroup_namespace *copy_cgroup_ns(unsigned long flags,
864 struct user_namespace *user_ns,
865 struct cgroup_namespace *old_ns);
866
867int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen,
868 struct cgroup_namespace *ns);
869
870#else
871
872static inline void free_cgroup_ns(struct cgroup_namespace *ns) { }
873static inline struct cgroup_namespace *
874copy_cgroup_ns(unsigned long flags, struct user_namespace *user_ns,
875 struct cgroup_namespace *old_ns)
876{
877 return old_ns;
878}
879
880#endif
881
882static inline void get_cgroup_ns(struct cgroup_namespace *ns)
883{
884 if (ns)
885 refcount_inc(&ns->count);
886}
887
888static inline void put_cgroup_ns(struct cgroup_namespace *ns)
889{
890 if (ns && refcount_dec_and_test(&ns->count))
891 free_cgroup_ns(ns);
892}
893
894#ifdef CONFIG_CGROUPS
895
896void cgroup_enter_frozen(void);
897void cgroup_leave_frozen(bool always_leave);
898void cgroup_update_frozen(struct cgroup *cgrp);
899void cgroup_freeze(struct cgroup *cgrp, bool freeze);
900void cgroup_freezer_migrate_task(struct task_struct *task, struct cgroup *src,
901 struct cgroup *dst);
902
903static inline bool cgroup_task_freeze(struct task_struct *task)
904{
905 bool ret;
906
907 if (task->flags & PF_KTHREAD)
908 return false;
909
910 rcu_read_lock();
911 ret = test_bit(CGRP_FREEZE, &task_dfl_cgroup(task)->flags);
912 rcu_read_unlock();
913
914 return ret;
915}
916
917static inline bool cgroup_task_frozen(struct task_struct *task)
918{
919 return task->frozen;
920}
921
922#else
923
924static inline void cgroup_enter_frozen(void) { }
925static inline void cgroup_leave_frozen(bool always_leave) { }
926static inline bool cgroup_task_freeze(struct task_struct *task)
927{
928 return false;
929}
930static inline bool cgroup_task_frozen(struct task_struct *task)
931{
932 return false;
933}
934
935#endif
936
937#endif
938