1
2
3
4
5
6
7
8
9
10
11#ifndef _LINUX_MEMCONTROL_H
12#define _LINUX_MEMCONTROL_H
13#include <linux/cgroup.h>
14#include <linux/vm_event_item.h>
15#include <linux/hardirq.h>
16#include <linux/jump_label.h>
17#include <linux/page_counter.h>
18#include <linux/vmpressure.h>
19#include <linux/eventfd.h>
20#include <linux/mm.h>
21#include <linux/vmstat.h>
22#include <linux/writeback.h>
23#include <linux/page-flags.h>
24
25struct mem_cgroup;
26struct obj_cgroup;
27struct page;
28struct mm_struct;
29struct kmem_cache;
30
31
32enum memcg_stat_item {
33 MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS,
34 MEMCG_SOCK,
35 MEMCG_PERCPU_B,
36 MEMCG_NR_STAT,
37};
38
39enum memcg_memory_event {
40 MEMCG_LOW,
41 MEMCG_HIGH,
42 MEMCG_MAX,
43 MEMCG_OOM,
44 MEMCG_OOM_KILL,
45 MEMCG_SWAP_HIGH,
46 MEMCG_SWAP_MAX,
47 MEMCG_SWAP_FAIL,
48 MEMCG_NR_MEMORY_EVENTS,
49};
50
51struct mem_cgroup_reclaim_cookie {
52 pg_data_t *pgdat;
53 unsigned int generation;
54};
55
56#ifdef CONFIG_MEMCG
57
58#define MEM_CGROUP_ID_SHIFT 16
59#define MEM_CGROUP_ID_MAX USHRT_MAX
60
61struct mem_cgroup_id {
62 int id;
63 refcount_t ref;
64};
65
66
67
68
69
70
71
72enum mem_cgroup_events_target {
73 MEM_CGROUP_TARGET_THRESH,
74 MEM_CGROUP_TARGET_SOFTLIMIT,
75 MEM_CGROUP_NTARGETS,
76};
77
78struct memcg_vmstats_percpu {
79
80 long state[MEMCG_NR_STAT];
81 unsigned long events[NR_VM_EVENT_ITEMS];
82
83
84 long state_prev[MEMCG_NR_STAT];
85 unsigned long events_prev[NR_VM_EVENT_ITEMS];
86
87
88 unsigned long nr_page_events;
89 unsigned long targets[MEM_CGROUP_NTARGETS];
90};
91
92struct memcg_vmstats {
93
94 long state[MEMCG_NR_STAT];
95 unsigned long events[NR_VM_EVENT_ITEMS];
96
97
98 long state_pending[MEMCG_NR_STAT];
99 unsigned long events_pending[NR_VM_EVENT_ITEMS];
100};
101
102struct mem_cgroup_reclaim_iter {
103 struct mem_cgroup *position;
104
105 unsigned int generation;
106};
107
108struct lruvec_stat {
109 long count[NR_VM_NODE_STAT_ITEMS];
110};
111
112struct batched_lruvec_stat {
113 s32 count[NR_VM_NODE_STAT_ITEMS];
114};
115
116
117
118
119
120struct shrinker_info {
121 struct rcu_head rcu;
122 atomic_long_t *nr_deferred;
123 unsigned long *map;
124};
125
126
127
128
129struct mem_cgroup_per_node {
130 struct lruvec lruvec;
131
132
133
134
135
136
137
138
139 struct lruvec_stat __percpu *lruvec_stat_local;
140
141
142 struct batched_lruvec_stat __percpu *lruvec_stat_cpu;
143 atomic_long_t lruvec_stat[NR_VM_NODE_STAT_ITEMS];
144
145 unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
146
147 struct mem_cgroup_reclaim_iter iter;
148
149 struct shrinker_info __rcu *shrinker_info;
150
151 struct rb_node tree_node;
152 unsigned long usage_in_excess;
153
154 bool on_tree;
155 struct mem_cgroup *memcg;
156
157};
158
159struct mem_cgroup_threshold {
160 struct eventfd_ctx *eventfd;
161 unsigned long threshold;
162};
163
164
165struct mem_cgroup_threshold_ary {
166
167 int current_threshold;
168
169 unsigned int size;
170
171 struct mem_cgroup_threshold entries[];
172};
173
174struct mem_cgroup_thresholds {
175
176 struct mem_cgroup_threshold_ary *primary;
177
178
179
180
181
182 struct mem_cgroup_threshold_ary *spare;
183};
184
185enum memcg_kmem_state {
186 KMEM_NONE,
187 KMEM_ALLOCATED,
188 KMEM_ONLINE,
189};
190
191#if defined(CONFIG_SMP)
192struct memcg_padding {
193 char x[0];
194} ____cacheline_internodealigned_in_smp;
195#define MEMCG_PADDING(name) struct memcg_padding name;
196#else
197#define MEMCG_PADDING(name)
198#endif
199
200
201
202
203
204
205
206
207
208#define MEMCG_CGWB_FRN_CNT 4
209
210struct memcg_cgwb_frn {
211 u64 bdi_id;
212 int memcg_id;
213 u64 at;
214 struct wb_completion done;
215};
216
217
218
219
220
221
222
223struct obj_cgroup {
224 struct percpu_ref refcnt;
225 struct mem_cgroup *memcg;
226 atomic_t nr_charged_bytes;
227 union {
228 struct list_head list;
229 struct rcu_head rcu;
230 };
231};
232
233
234
235
236
237
238
239struct mem_cgroup {
240 struct cgroup_subsys_state css;
241
242
243 struct mem_cgroup_id id;
244
245
246 struct page_counter memory;
247
248 union {
249 struct page_counter swap;
250 struct page_counter memsw;
251 };
252
253
254 struct page_counter kmem;
255 struct page_counter tcpmem;
256
257
258 struct work_struct high_work;
259
260 unsigned long soft_limit;
261
262
263 struct vmpressure vmpressure;
264
265
266
267
268 bool oom_group;
269
270
271 bool oom_lock;
272 int under_oom;
273
274 int swappiness;
275
276 int oom_kill_disable;
277
278
279 struct cgroup_file events_file;
280 struct cgroup_file events_local_file;
281
282
283 struct cgroup_file swap_events_file;
284
285
286 struct mutex thresholds_lock;
287
288
289 struct mem_cgroup_thresholds thresholds;
290
291
292 struct mem_cgroup_thresholds memsw_thresholds;
293
294
295 struct list_head oom_notify;
296
297
298
299
300
301 unsigned long move_charge_at_immigrate;
302
303 spinlock_t move_lock;
304 unsigned long move_lock_flags;
305
306 MEMCG_PADDING(_pad1_);
307
308
309 struct memcg_vmstats vmstats;
310
311
312 atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS];
313 atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS];
314
315 unsigned long socket_pressure;
316
317
318 bool tcpmem_active;
319 int tcpmem_pressure;
320
321#ifdef CONFIG_MEMCG_KMEM
322 int kmemcg_id;
323 enum memcg_kmem_state kmem_state;
324 struct obj_cgroup __rcu *objcg;
325 struct list_head objcg_list;
326#endif
327
328 MEMCG_PADDING(_pad2_);
329
330
331
332
333 atomic_t moving_account;
334 struct task_struct *move_lock_task;
335
336 struct memcg_vmstats_percpu __percpu *vmstats_percpu;
337
338#ifdef CONFIG_CGROUP_WRITEBACK
339 struct list_head cgwb_list;
340 struct wb_domain cgwb_domain;
341 struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
342#endif
343
344
345 struct list_head event_list;
346 spinlock_t event_list_lock;
347
348#ifdef CONFIG_TRANSPARENT_HUGEPAGE
349 struct deferred_split deferred_split_queue;
350#endif
351
352 struct mem_cgroup_per_node *nodeinfo[0];
353
354};
355
356
357
358
359
360#define MEMCG_CHARGE_BATCH 32U
361
362extern struct mem_cgroup *root_mem_cgroup;
363
364enum page_memcg_data_flags {
365
366 MEMCG_DATA_OBJCGS = (1UL << 0),
367
368 MEMCG_DATA_KMEM = (1UL << 1),
369
370 __NR_MEMCG_DATA_FLAGS = (1UL << 2),
371};
372
373#define MEMCG_DATA_FLAGS_MASK (__NR_MEMCG_DATA_FLAGS - 1)
374
375static inline bool PageMemcgKmem(struct page *page);
376
377
378
379
380
381
382
383
384static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
385{
386 return READ_ONCE(objcg->memcg);
387}
388
389
390
391
392
393
394
395
396
397
398
399static inline struct mem_cgroup *__page_memcg(struct page *page)
400{
401 unsigned long memcg_data = page->memcg_data;
402
403 VM_BUG_ON_PAGE(PageSlab(page), page);
404 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_OBJCGS, page);
405 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page);
406
407 return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
408}
409
410
411
412
413
414
415
416
417
418
419
420static inline struct obj_cgroup *__page_objcg(struct page *page)
421{
422 unsigned long memcg_data = page->memcg_data;
423
424 VM_BUG_ON_PAGE(PageSlab(page), page);
425 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_OBJCGS, page);
426 VM_BUG_ON_PAGE(!(memcg_data & MEMCG_DATA_KMEM), page);
427
428 return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
429}
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451static inline struct mem_cgroup *page_memcg(struct page *page)
452{
453 if (PageMemcgKmem(page))
454 return obj_cgroup_memcg(__page_objcg(page));
455 else
456 return __page_memcg(page);
457}
458
459
460
461
462
463
464
465
466
467
468static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
469{
470 unsigned long memcg_data = READ_ONCE(page->memcg_data);
471
472 VM_BUG_ON_PAGE(PageSlab(page), page);
473 WARN_ON_ONCE(!rcu_read_lock_held());
474
475 if (memcg_data & MEMCG_DATA_KMEM) {
476 struct obj_cgroup *objcg;
477
478 objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
479 return obj_cgroup_memcg(objcg);
480 }
481
482 return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
483}
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506static inline struct mem_cgroup *page_memcg_check(struct page *page)
507{
508
509
510
511
512 unsigned long memcg_data = READ_ONCE(page->memcg_data);
513
514 if (memcg_data & MEMCG_DATA_OBJCGS)
515 return NULL;
516
517 if (memcg_data & MEMCG_DATA_KMEM) {
518 struct obj_cgroup *objcg;
519
520 objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
521 return obj_cgroup_memcg(objcg);
522 }
523
524 return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
525}
526
527#ifdef CONFIG_MEMCG_KMEM
528
529
530
531
532
533
534
535
536static inline bool PageMemcgKmem(struct page *page)
537{
538 VM_BUG_ON_PAGE(page->memcg_data & MEMCG_DATA_OBJCGS, page);
539 return page->memcg_data & MEMCG_DATA_KMEM;
540}
541
542
543
544
545
546
547
548
549
550
551
552static inline struct obj_cgroup **page_objcgs(struct page *page)
553{
554 unsigned long memcg_data = READ_ONCE(page->memcg_data);
555
556 VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), page);
557 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page);
558
559 return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
560}
561
562
563
564
565
566
567
568
569
570static inline struct obj_cgroup **page_objcgs_check(struct page *page)
571{
572 unsigned long memcg_data = READ_ONCE(page->memcg_data);
573
574 if (!memcg_data || !(memcg_data & MEMCG_DATA_OBJCGS))
575 return NULL;
576
577 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page);
578
579 return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
580}
581
582#else
583static inline bool PageMemcgKmem(struct page *page)
584{
585 return false;
586}
587
588static inline struct obj_cgroup **page_objcgs(struct page *page)
589{
590 return NULL;
591}
592
593static inline struct obj_cgroup **page_objcgs_check(struct page *page)
594{
595 return NULL;
596}
597#endif
598
599static __always_inline bool memcg_stat_item_in_bytes(int idx)
600{
601 if (idx == MEMCG_PERCPU_B)
602 return true;
603 return vmstat_item_in_bytes(idx);
604}
605
606static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
607{
608 return (memcg == root_mem_cgroup);
609}
610
611static inline bool mem_cgroup_disabled(void)
612{
613 return !cgroup_subsys_enabled(memory_cgrp_subsys);
614}
615
616static inline unsigned long mem_cgroup_protection(struct mem_cgroup *root,
617 struct mem_cgroup *memcg,
618 bool in_low_reclaim)
619{
620 if (mem_cgroup_disabled())
621 return 0;
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656 if (root == memcg)
657 return 0;
658
659 if (in_low_reclaim)
660 return READ_ONCE(memcg->memory.emin);
661
662 return max(READ_ONCE(memcg->memory.emin),
663 READ_ONCE(memcg->memory.elow));
664}
665
666void mem_cgroup_calculate_protection(struct mem_cgroup *root,
667 struct mem_cgroup *memcg);
668
669static inline bool mem_cgroup_supports_protection(struct mem_cgroup *memcg)
670{
671
672
673
674
675 return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg);
676
677}
678
679static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg)
680{
681 if (!mem_cgroup_supports_protection(memcg))
682 return false;
683
684 return READ_ONCE(memcg->memory.elow) >=
685 page_counter_read(&memcg->memory);
686}
687
688static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
689{
690 if (!mem_cgroup_supports_protection(memcg))
691 return false;
692
693 return READ_ONCE(memcg->memory.emin) >=
694 page_counter_read(&memcg->memory);
695}
696
697int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);
698int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
699 gfp_t gfp, swp_entry_t entry);
700void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
701
702void mem_cgroup_uncharge(struct page *page);
703void mem_cgroup_uncharge_list(struct list_head *page_list);
704
705void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
706
707
708
709
710
711
712
713
714
715
716static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
717 struct pglist_data *pgdat)
718{
719 struct mem_cgroup_per_node *mz;
720 struct lruvec *lruvec;
721
722 if (mem_cgroup_disabled()) {
723 lruvec = &pgdat->__lruvec;
724 goto out;
725 }
726
727 if (!memcg)
728 memcg = root_mem_cgroup;
729
730 mz = memcg->nodeinfo[pgdat->node_id];
731 lruvec = &mz->lruvec;
732out:
733
734
735
736
737
738 if (unlikely(lruvec->pgdat != pgdat))
739 lruvec->pgdat = pgdat;
740 return lruvec;
741}
742
743
744
745
746
747
748
749
750static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
751 struct pglist_data *pgdat)
752{
753 struct mem_cgroup *memcg = page_memcg(page);
754
755 VM_WARN_ON_ONCE_PAGE(!memcg && !mem_cgroup_disabled(), page);
756 return mem_cgroup_lruvec(memcg, pgdat);
757}
758
759static inline bool lruvec_holds_page_lru_lock(struct page *page,
760 struct lruvec *lruvec)
761{
762 pg_data_t *pgdat = page_pgdat(page);
763 const struct mem_cgroup *memcg;
764 struct mem_cgroup_per_node *mz;
765
766 if (mem_cgroup_disabled())
767 return lruvec == &pgdat->__lruvec;
768
769 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
770 memcg = page_memcg(page) ? : root_mem_cgroup;
771
772 return lruvec->pgdat == pgdat && mz->memcg == memcg;
773}
774
775struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
776
777struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
778
779struct lruvec *lock_page_lruvec(struct page *page);
780struct lruvec *lock_page_lruvec_irq(struct page *page);
781struct lruvec *lock_page_lruvec_irqsave(struct page *page,
782 unsigned long *flags);
783
784#ifdef CONFIG_DEBUG_VM
785void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page);
786#else
787static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
788{
789}
790#endif
791
792static inline
793struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
794 return css ? container_of(css, struct mem_cgroup, css) : NULL;
795}
796
797static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg)
798{
799 return percpu_ref_tryget(&objcg->refcnt);
800}
801
802static inline void obj_cgroup_get(struct obj_cgroup *objcg)
803{
804 percpu_ref_get(&objcg->refcnt);
805}
806
807static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
808 unsigned long nr)
809{
810 percpu_ref_get_many(&objcg->refcnt, nr);
811}
812
813static inline void obj_cgroup_put(struct obj_cgroup *objcg)
814{
815 percpu_ref_put(&objcg->refcnt);
816}
817
818static inline void mem_cgroup_put(struct mem_cgroup *memcg)
819{
820 if (memcg)
821 css_put(&memcg->css);
822}
823
824#define mem_cgroup_from_counter(counter, member) \
825 container_of(counter, struct mem_cgroup, member)
826
827struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
828 struct mem_cgroup *,
829 struct mem_cgroup_reclaim_cookie *);
830void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
831int mem_cgroup_scan_tasks(struct mem_cgroup *,
832 int (*)(struct task_struct *, void *), void *);
833
834static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
835{
836 if (mem_cgroup_disabled())
837 return 0;
838
839 return memcg->id.id;
840}
841struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
842
843static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
844{
845 return mem_cgroup_from_css(seq_css(m));
846}
847
848static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
849{
850 struct mem_cgroup_per_node *mz;
851
852 if (mem_cgroup_disabled())
853 return NULL;
854
855 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
856 return mz->memcg;
857}
858
859
860
861
862
863
864
865
866static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
867{
868 if (!memcg->memory.parent)
869 return NULL;
870 return mem_cgroup_from_counter(memcg->memory.parent, memory);
871}
872
873static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
874 struct mem_cgroup *root)
875{
876 if (root == memcg)
877 return true;
878 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
879}
880
881static inline bool mm_match_cgroup(struct mm_struct *mm,
882 struct mem_cgroup *memcg)
883{
884 struct mem_cgroup *task_memcg;
885 bool match = false;
886
887 rcu_read_lock();
888 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
889 if (task_memcg)
890 match = mem_cgroup_is_descendant(task_memcg, memcg);
891 rcu_read_unlock();
892 return match;
893}
894
895struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
896ino_t page_cgroup_ino(struct page *page);
897
898static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
899{
900 if (mem_cgroup_disabled())
901 return true;
902 return !!(memcg->css.flags & CSS_ONLINE);
903}
904
905
906
907
908int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
909
910void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
911 int zid, int nr_pages);
912
913static inline
914unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
915 enum lru_list lru, int zone_idx)
916{
917 struct mem_cgroup_per_node *mz;
918
919 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
920 return READ_ONCE(mz->lru_zone_size[zone_idx][lru]);
921}
922
923void mem_cgroup_handle_over_high(void);
924
925unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
926
927unsigned long mem_cgroup_size(struct mem_cgroup *memcg);
928
929void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
930 struct task_struct *p);
931
932void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);
933
934static inline void mem_cgroup_enter_user_fault(void)
935{
936 WARN_ON(current->in_user_fault);
937 current->in_user_fault = 1;
938}
939
940static inline void mem_cgroup_exit_user_fault(void)
941{
942 WARN_ON(!current->in_user_fault);
943 current->in_user_fault = 0;
944}
945
946static inline bool task_in_memcg_oom(struct task_struct *p)
947{
948 return p->memcg_in_oom;
949}
950
951bool mem_cgroup_oom_synchronize(bool wait);
952struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
953 struct mem_cgroup *oom_domain);
954void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
955
956#ifdef CONFIG_MEMCG_SWAP
957extern bool cgroup_memory_noswap;
958#endif
959
960void lock_page_memcg(struct page *page);
961void unlock_page_memcg(struct page *page);
962
963void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
964
965
966static inline void mod_memcg_state(struct mem_cgroup *memcg,
967 int idx, int val)
968{
969 unsigned long flags;
970
971 local_irq_save(flags);
972 __mod_memcg_state(memcg, idx, val);
973 local_irq_restore(flags);
974}
975
976static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
977 enum node_stat_item idx)
978{
979 struct mem_cgroup_per_node *pn;
980 long x;
981
982 if (mem_cgroup_disabled())
983 return node_page_state(lruvec_pgdat(lruvec), idx);
984
985 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
986 x = atomic_long_read(&pn->lruvec_stat[idx]);
987#ifdef CONFIG_SMP
988 if (x < 0)
989 x = 0;
990#endif
991 return x;
992}
993
994static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
995 enum node_stat_item idx)
996{
997 struct mem_cgroup_per_node *pn;
998 long x = 0;
999 int cpu;
1000
1001 if (mem_cgroup_disabled())
1002 return node_page_state(lruvec_pgdat(lruvec), idx);
1003
1004 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1005 for_each_possible_cpu(cpu)
1006 x += per_cpu(pn->lruvec_stat_local->count[idx], cpu);
1007#ifdef CONFIG_SMP
1008 if (x < 0)
1009 x = 0;
1010#endif
1011 return x;
1012}
1013
1014void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
1015 int val);
1016void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val);
1017
1018static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1019 int val)
1020{
1021 unsigned long flags;
1022
1023 local_irq_save(flags);
1024 __mod_lruvec_kmem_state(p, idx, val);
1025 local_irq_restore(flags);
1026}
1027
1028static inline void mod_memcg_lruvec_state(struct lruvec *lruvec,
1029 enum node_stat_item idx, int val)
1030{
1031 unsigned long flags;
1032
1033 local_irq_save(flags);
1034 __mod_memcg_lruvec_state(lruvec, idx, val);
1035 local_irq_restore(flags);
1036}
1037
1038void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
1039 unsigned long count);
1040
1041static inline void count_memcg_events(struct mem_cgroup *memcg,
1042 enum vm_event_item idx,
1043 unsigned long count)
1044{
1045 unsigned long flags;
1046
1047 local_irq_save(flags);
1048 __count_memcg_events(memcg, idx, count);
1049 local_irq_restore(flags);
1050}
1051
1052static inline void count_memcg_page_event(struct page *page,
1053 enum vm_event_item idx)
1054{
1055 struct mem_cgroup *memcg = page_memcg(page);
1056
1057 if (memcg)
1058 count_memcg_events(memcg, idx, 1);
1059}
1060
1061static inline void count_memcg_event_mm(struct mm_struct *mm,
1062 enum vm_event_item idx)
1063{
1064 struct mem_cgroup *memcg;
1065
1066 if (mem_cgroup_disabled())
1067 return;
1068
1069 rcu_read_lock();
1070 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1071 if (likely(memcg))
1072 count_memcg_events(memcg, idx, 1);
1073 rcu_read_unlock();
1074}
1075
1076static inline void memcg_memory_event(struct mem_cgroup *memcg,
1077 enum memcg_memory_event event)
1078{
1079 bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX ||
1080 event == MEMCG_SWAP_FAIL;
1081
1082 atomic_long_inc(&memcg->memory_events_local[event]);
1083 if (!swap_event)
1084 cgroup_file_notify(&memcg->events_local_file);
1085
1086 do {
1087 atomic_long_inc(&memcg->memory_events[event]);
1088 if (swap_event)
1089 cgroup_file_notify(&memcg->swap_events_file);
1090 else
1091 cgroup_file_notify(&memcg->events_file);
1092
1093 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1094 break;
1095 if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS)
1096 break;
1097 } while ((memcg = parent_mem_cgroup(memcg)) &&
1098 !mem_cgroup_is_root(memcg));
1099}
1100
1101static inline void memcg_memory_event_mm(struct mm_struct *mm,
1102 enum memcg_memory_event event)
1103{
1104 struct mem_cgroup *memcg;
1105
1106 if (mem_cgroup_disabled())
1107 return;
1108
1109 rcu_read_lock();
1110 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1111 if (likely(memcg))
1112 memcg_memory_event(memcg, event);
1113 rcu_read_unlock();
1114}
1115
1116void split_page_memcg(struct page *head, unsigned int nr);
1117
1118unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1119 gfp_t gfp_mask,
1120 unsigned long *total_scanned);
1121
1122#else
1123
1124#define MEM_CGROUP_ID_SHIFT 0
1125#define MEM_CGROUP_ID_MAX 0
1126
1127static inline struct mem_cgroup *page_memcg(struct page *page)
1128{
1129 return NULL;
1130}
1131
1132static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
1133{
1134 WARN_ON_ONCE(!rcu_read_lock_held());
1135 return NULL;
1136}
1137
1138static inline struct mem_cgroup *page_memcg_check(struct page *page)
1139{
1140 return NULL;
1141}
1142
1143static inline bool PageMemcgKmem(struct page *page)
1144{
1145 return false;
1146}
1147
1148static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
1149{
1150 return true;
1151}
1152
1153static inline bool mem_cgroup_disabled(void)
1154{
1155 return true;
1156}
1157
1158static inline void memcg_memory_event(struct mem_cgroup *memcg,
1159 enum memcg_memory_event event)
1160{
1161}
1162
1163static inline void memcg_memory_event_mm(struct mm_struct *mm,
1164 enum memcg_memory_event event)
1165{
1166}
1167
1168static inline unsigned long mem_cgroup_protection(struct mem_cgroup *root,
1169 struct mem_cgroup *memcg,
1170 bool in_low_reclaim)
1171{
1172 return 0;
1173}
1174
1175static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root,
1176 struct mem_cgroup *memcg)
1177{
1178}
1179
1180static inline bool mem_cgroup_below_low(struct mem_cgroup *memcg)
1181{
1182 return false;
1183}
1184
1185static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
1186{
1187 return false;
1188}
1189
1190static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
1191 gfp_t gfp_mask)
1192{
1193 return 0;
1194}
1195
1196static inline int mem_cgroup_swapin_charge_page(struct page *page,
1197 struct mm_struct *mm, gfp_t gfp, swp_entry_t entry)
1198{
1199 return 0;
1200}
1201
1202static inline void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)
1203{
1204}
1205
1206static inline void mem_cgroup_uncharge(struct page *page)
1207{
1208}
1209
1210static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
1211{
1212}
1213
1214static inline void mem_cgroup_migrate(struct page *old, struct page *new)
1215{
1216}
1217
1218static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
1219 struct pglist_data *pgdat)
1220{
1221 return &pgdat->__lruvec;
1222}
1223
1224static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
1225 struct pglist_data *pgdat)
1226{
1227 return &pgdat->__lruvec;
1228}
1229
1230static inline bool lruvec_holds_page_lru_lock(struct page *page,
1231 struct lruvec *lruvec)
1232{
1233 pg_data_t *pgdat = page_pgdat(page);
1234
1235 return lruvec == &pgdat->__lruvec;
1236}
1237
1238static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
1239{
1240}
1241
1242static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
1243{
1244 return NULL;
1245}
1246
1247static inline bool mm_match_cgroup(struct mm_struct *mm,
1248 struct mem_cgroup *memcg)
1249{
1250 return true;
1251}
1252
1253static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
1254{
1255 return NULL;
1256}
1257
1258static inline void mem_cgroup_put(struct mem_cgroup *memcg)
1259{
1260}
1261
1262static inline struct lruvec *lock_page_lruvec(struct page *page)
1263{
1264 struct pglist_data *pgdat = page_pgdat(page);
1265
1266 spin_lock(&pgdat->__lruvec.lru_lock);
1267 return &pgdat->__lruvec;
1268}
1269
1270static inline struct lruvec *lock_page_lruvec_irq(struct page *page)
1271{
1272 struct pglist_data *pgdat = page_pgdat(page);
1273
1274 spin_lock_irq(&pgdat->__lruvec.lru_lock);
1275 return &pgdat->__lruvec;
1276}
1277
1278static inline struct lruvec *lock_page_lruvec_irqsave(struct page *page,
1279 unsigned long *flagsp)
1280{
1281 struct pglist_data *pgdat = page_pgdat(page);
1282
1283 spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp);
1284 return &pgdat->__lruvec;
1285}
1286
1287static inline struct mem_cgroup *
1288mem_cgroup_iter(struct mem_cgroup *root,
1289 struct mem_cgroup *prev,
1290 struct mem_cgroup_reclaim_cookie *reclaim)
1291{
1292 return NULL;
1293}
1294
1295static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
1296 struct mem_cgroup *prev)
1297{
1298}
1299
1300static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1301 int (*fn)(struct task_struct *, void *), void *arg)
1302{
1303 return 0;
1304}
1305
1306static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
1307{
1308 return 0;
1309}
1310
1311static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
1312{
1313 WARN_ON_ONCE(id);
1314
1315 return NULL;
1316}
1317
1318static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
1319{
1320 return NULL;
1321}
1322
1323static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
1324{
1325 return NULL;
1326}
1327
1328static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
1329{
1330 return true;
1331}
1332
1333static inline
1334unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
1335 enum lru_list lru, int zone_idx)
1336{
1337 return 0;
1338}
1339
1340static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1341{
1342 return 0;
1343}
1344
1345static inline unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1346{
1347 return 0;
1348}
1349
1350static inline void
1351mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1352{
1353}
1354
1355static inline void
1356mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1357{
1358}
1359
1360static inline void lock_page_memcg(struct page *page)
1361{
1362}
1363
1364static inline void unlock_page_memcg(struct page *page)
1365{
1366}
1367
1368static inline void mem_cgroup_handle_over_high(void)
1369{
1370}
1371
1372static inline void mem_cgroup_enter_user_fault(void)
1373{
1374}
1375
1376static inline void mem_cgroup_exit_user_fault(void)
1377{
1378}
1379
1380static inline bool task_in_memcg_oom(struct task_struct *p)
1381{
1382 return false;
1383}
1384
1385static inline bool mem_cgroup_oom_synchronize(bool wait)
1386{
1387 return false;
1388}
1389
1390static inline struct mem_cgroup *mem_cgroup_get_oom_group(
1391 struct task_struct *victim, struct mem_cgroup *oom_domain)
1392{
1393 return NULL;
1394}
1395
1396static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1397{
1398}
1399
1400static inline void __mod_memcg_state(struct mem_cgroup *memcg,
1401 int idx,
1402 int nr)
1403{
1404}
1405
1406static inline void mod_memcg_state(struct mem_cgroup *memcg,
1407 int idx,
1408 int nr)
1409{
1410}
1411
1412static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
1413 enum node_stat_item idx)
1414{
1415 return node_page_state(lruvec_pgdat(lruvec), idx);
1416}
1417
1418static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
1419 enum node_stat_item idx)
1420{
1421 return node_page_state(lruvec_pgdat(lruvec), idx);
1422}
1423
1424static inline void __mod_memcg_lruvec_state(struct lruvec *lruvec,
1425 enum node_stat_item idx, int val)
1426{
1427}
1428
1429static inline void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1430 int val)
1431{
1432 struct page *page = virt_to_head_page(p);
1433
1434 __mod_node_page_state(page_pgdat(page), idx, val);
1435}
1436
1437static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
1438 int val)
1439{
1440 struct page *page = virt_to_head_page(p);
1441
1442 mod_node_page_state(page_pgdat(page), idx, val);
1443}
1444
1445static inline void count_memcg_events(struct mem_cgroup *memcg,
1446 enum vm_event_item idx,
1447 unsigned long count)
1448{
1449}
1450
1451static inline void __count_memcg_events(struct mem_cgroup *memcg,
1452 enum vm_event_item idx,
1453 unsigned long count)
1454{
1455}
1456
1457static inline void count_memcg_page_event(struct page *page,
1458 int idx)
1459{
1460}
1461
1462static inline
1463void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
1464{
1465}
1466
1467static inline void split_page_memcg(struct page *head, unsigned int nr)
1468{
1469}
1470
1471static inline
1472unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1473 gfp_t gfp_mask,
1474 unsigned long *total_scanned)
1475{
1476 return 0;
1477}
1478#endif
1479
1480static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx)
1481{
1482 __mod_lruvec_kmem_state(p, idx, 1);
1483}
1484
1485static inline void __dec_lruvec_kmem_state(void *p, enum node_stat_item idx)
1486{
1487 __mod_lruvec_kmem_state(p, idx, -1);
1488}
1489
1490static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
1491{
1492 struct mem_cgroup *memcg;
1493
1494 memcg = lruvec_memcg(lruvec);
1495 if (!memcg)
1496 return NULL;
1497 memcg = parent_mem_cgroup(memcg);
1498 if (!memcg)
1499 return NULL;
1500 return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));
1501}
1502
1503static inline void unlock_page_lruvec(struct lruvec *lruvec)
1504{
1505 spin_unlock(&lruvec->lru_lock);
1506}
1507
1508static inline void unlock_page_lruvec_irq(struct lruvec *lruvec)
1509{
1510 spin_unlock_irq(&lruvec->lru_lock);
1511}
1512
1513static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec,
1514 unsigned long flags)
1515{
1516 spin_unlock_irqrestore(&lruvec->lru_lock, flags);
1517}
1518
1519
1520static inline struct lruvec *relock_page_lruvec_irq(struct page *page,
1521 struct lruvec *locked_lruvec)
1522{
1523 if (locked_lruvec) {
1524 if (lruvec_holds_page_lru_lock(page, locked_lruvec))
1525 return locked_lruvec;
1526
1527 unlock_page_lruvec_irq(locked_lruvec);
1528 }
1529
1530 return lock_page_lruvec_irq(page);
1531}
1532
1533
1534static inline struct lruvec *relock_page_lruvec_irqsave(struct page *page,
1535 struct lruvec *locked_lruvec, unsigned long *flags)
1536{
1537 if (locked_lruvec) {
1538 if (lruvec_holds_page_lru_lock(page, locked_lruvec))
1539 return locked_lruvec;
1540
1541 unlock_page_lruvec_irqrestore(locked_lruvec, *flags);
1542 }
1543
1544 return lock_page_lruvec_irqsave(page, flags);
1545}
1546
1547#ifdef CONFIG_CGROUP_WRITEBACK
1548
1549struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
1550void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
1551 unsigned long *pheadroom, unsigned long *pdirty,
1552 unsigned long *pwriteback);
1553
1554void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
1555 struct bdi_writeback *wb);
1556
1557static inline void mem_cgroup_track_foreign_dirty(struct page *page,
1558 struct bdi_writeback *wb)
1559{
1560 if (mem_cgroup_disabled())
1561 return;
1562
1563 if (unlikely(&page_memcg(page)->css != wb->memcg_css))
1564 mem_cgroup_track_foreign_dirty_slowpath(page, wb);
1565}
1566
1567void mem_cgroup_flush_foreign(struct bdi_writeback *wb);
1568
1569#else
1570
1571static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
1572{
1573 return NULL;
1574}
1575
1576static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
1577 unsigned long *pfilepages,
1578 unsigned long *pheadroom,
1579 unsigned long *pdirty,
1580 unsigned long *pwriteback)
1581{
1582}
1583
1584static inline void mem_cgroup_track_foreign_dirty(struct page *page,
1585 struct bdi_writeback *wb)
1586{
1587}
1588
1589static inline void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
1590{
1591}
1592
1593#endif
1594
1595struct sock;
1596bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1597void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1598#ifdef CONFIG_MEMCG
1599extern struct static_key_false memcg_sockets_enabled_key;
1600#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
1601void mem_cgroup_sk_alloc(struct sock *sk);
1602void mem_cgroup_sk_free(struct sock *sk);
1603static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1604{
1605 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
1606 return true;
1607 do {
1608 if (time_before(jiffies, memcg->socket_pressure))
1609 return true;
1610 } while ((memcg = parent_mem_cgroup(memcg)));
1611 return false;
1612}
1613
1614int alloc_shrinker_info(struct mem_cgroup *memcg);
1615void free_shrinker_info(struct mem_cgroup *memcg);
1616void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);
1617void reparent_shrinker_deferred(struct mem_cgroup *memcg);
1618#else
1619#define mem_cgroup_sockets_enabled 0
1620static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
1621static inline void mem_cgroup_sk_free(struct sock *sk) { };
1622static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1623{
1624 return false;
1625}
1626
1627static inline void set_shrinker_bit(struct mem_cgroup *memcg,
1628 int nid, int shrinker_id)
1629{
1630}
1631#endif
1632
1633#ifdef CONFIG_MEMCG_KMEM
1634int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order);
1635void __memcg_kmem_uncharge_page(struct page *page, int order);
1636
1637struct obj_cgroup *get_obj_cgroup_from_current(void);
1638
1639int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size);
1640void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size);
1641
1642extern struct static_key_false memcg_kmem_enabled_key;
1643
1644extern int memcg_nr_cache_ids;
1645void memcg_get_cache_ids(void);
1646void memcg_put_cache_ids(void);
1647
1648
1649
1650
1651
1652
1653#define for_each_memcg_cache_index(_idx) \
1654 for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
1655
1656static inline bool memcg_kmem_enabled(void)
1657{
1658 return static_branch_likely(&memcg_kmem_enabled_key);
1659}
1660
1661static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1662 int order)
1663{
1664 if (memcg_kmem_enabled())
1665 return __memcg_kmem_charge_page(page, gfp, order);
1666 return 0;
1667}
1668
1669static inline void memcg_kmem_uncharge_page(struct page *page, int order)
1670{
1671 if (memcg_kmem_enabled())
1672 __memcg_kmem_uncharge_page(page, order);
1673}
1674
1675
1676
1677
1678
1679static inline int memcg_cache_id(struct mem_cgroup *memcg)
1680{
1681 return memcg ? memcg->kmemcg_id : -1;
1682}
1683
1684struct mem_cgroup *mem_cgroup_from_obj(void *p);
1685
1686#else
1687
1688static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1689 int order)
1690{
1691 return 0;
1692}
1693
1694static inline void memcg_kmem_uncharge_page(struct page *page, int order)
1695{
1696}
1697
1698static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp,
1699 int order)
1700{
1701 return 0;
1702}
1703
1704static inline void __memcg_kmem_uncharge_page(struct page *page, int order)
1705{
1706}
1707
1708#define for_each_memcg_cache_index(_idx) \
1709 for (; NULL; )
1710
1711static inline bool memcg_kmem_enabled(void)
1712{
1713 return false;
1714}
1715
1716static inline int memcg_cache_id(struct mem_cgroup *memcg)
1717{
1718 return -1;
1719}
1720
1721static inline void memcg_get_cache_ids(void)
1722{
1723}
1724
1725static inline void memcg_put_cache_ids(void)
1726{
1727}
1728
1729static inline struct mem_cgroup *mem_cgroup_from_obj(void *p)
1730{
1731 return NULL;
1732}
1733
1734#endif
1735
1736#endif
1737