1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/page_counter.h>
29#include <linux/memcontrol.h>
30#include <linux/cgroup.h>
31#include <linux/pagewalk.h>
32#include <linux/sched/mm.h>
33#include <linux/shmem_fs.h>
34#include <linux/hugetlb.h>
35#include <linux/pagemap.h>
36#include <linux/vm_event_item.h>
37#include <linux/smp.h>
38#include <linux/page-flags.h>
39#include <linux/backing-dev.h>
40#include <linux/bit_spinlock.h>
41#include <linux/rcupdate.h>
42#include <linux/limits.h>
43#include <linux/export.h>
44#include <linux/mutex.h>
45#include <linux/rbtree.h>
46#include <linux/slab.h>
47#include <linux/swap.h>
48#include <linux/swapops.h>
49#include <linux/spinlock.h>
50#include <linux/eventfd.h>
51#include <linux/poll.h>
52#include <linux/sort.h>
53#include <linux/fs.h>
54#include <linux/seq_file.h>
55#include <linux/vmpressure.h>
56#include <linux/mm_inline.h>
57#include <linux/swap_cgroup.h>
58#include <linux/cpu.h>
59#include <linux/oom.h>
60#include <linux/lockdep.h>
61#include <linux/file.h>
62#include <linux/tracehook.h>
63#include <linux/psi.h>
64#include <linux/seq_buf.h>
65#include "internal.h"
66#include <net/sock.h>
67#include <net/ip.h>
68#include "slab.h"
69
70#include <linux/uaccess.h>
71
72#include <trace/events/vmscan.h>
73
74struct cgroup_subsys memory_cgrp_subsys __read_mostly;
75EXPORT_SYMBOL(memory_cgrp_subsys);
76
77struct mem_cgroup *root_mem_cgroup __read_mostly;
78
79
80DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
81EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);
82
83
84static bool cgroup_memory_nosocket __ro_after_init;
85
86
87bool cgroup_memory_nokmem __ro_after_init;
88
89
90#ifdef CONFIG_MEMCG_SWAP
91bool cgroup_memory_noswap __ro_after_init;
92#else
93#define cgroup_memory_noswap 1
94#endif
95
96#ifdef CONFIG_CGROUP_WRITEBACK
97static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
98#endif
99
100
101static bool do_memsw_account(void)
102{
103 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
104}
105
106#define THRESHOLDS_EVENTS_TARGET 128
107#define SOFTLIMIT_EVENTS_TARGET 1024
108
109
110
111
112
113
114struct mem_cgroup_tree_per_node {
115 struct rb_root rb_root;
116 struct rb_node *rb_rightmost;
117 spinlock_t lock;
118};
119
120struct mem_cgroup_tree {
121 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
122};
123
124static struct mem_cgroup_tree soft_limit_tree __read_mostly;
125
126
127struct mem_cgroup_eventfd_list {
128 struct list_head list;
129 struct eventfd_ctx *eventfd;
130};
131
132
133
134
135struct mem_cgroup_event {
136
137
138
139 struct mem_cgroup *memcg;
140
141
142
143 struct eventfd_ctx *eventfd;
144
145
146
147 struct list_head list;
148
149
150
151
152
153 int (*register_event)(struct mem_cgroup *memcg,
154 struct eventfd_ctx *eventfd, const char *args);
155
156
157
158
159
160 void (*unregister_event)(struct mem_cgroup *memcg,
161 struct eventfd_ctx *eventfd);
162
163
164
165
166 poll_table pt;
167 wait_queue_head_t *wqh;
168 wait_queue_entry_t wait;
169 struct work_struct remove;
170};
171
172static void mem_cgroup_threshold(struct mem_cgroup *memcg);
173static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
174
175
176
177
178
179#define MOVE_ANON 0x1U
180#define MOVE_FILE 0x2U
181#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
182
183
184static struct move_charge_struct {
185 spinlock_t lock;
186 struct mm_struct *mm;
187 struct mem_cgroup *from;
188 struct mem_cgroup *to;
189 unsigned long flags;
190 unsigned long precharge;
191 unsigned long moved_charge;
192 unsigned long moved_swap;
193 struct task_struct *moving_task;
194 wait_queue_head_t waitq;
195} mc = {
196 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
197 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
198};
199
200
201
202
203
204#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
205#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
206
207
208enum res_type {
209 _MEM,
210 _MEMSWAP,
211 _OOM_TYPE,
212 _KMEM,
213 _TCP,
214};
215
216#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
217#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
218#define MEMFILE_ATTR(val) ((val) & 0xffff)
219
220#define OOM_CONTROL (0)
221
222
223
224
225
226
227#define for_each_mem_cgroup_tree(iter, root) \
228 for (iter = mem_cgroup_iter(root, NULL, NULL); \
229 iter != NULL; \
230 iter = mem_cgroup_iter(root, iter, NULL))
231
232#define for_each_mem_cgroup(iter) \
233 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
234 iter != NULL; \
235 iter = mem_cgroup_iter(NULL, iter, NULL))
236
237static inline bool should_force_charge(void)
238{
239 return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
240 (current->flags & PF_EXITING);
241}
242
243
244struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
245{
246 if (!memcg)
247 memcg = root_mem_cgroup;
248 return &memcg->vmpressure;
249}
250
251struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
252{
253 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
254}
255
256#ifdef CONFIG_MEMCG_KMEM
257extern spinlock_t css_set_lock;
258
259bool mem_cgroup_kmem_disabled(void)
260{
261 return cgroup_memory_nokmem;
262}
263
264static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
265 unsigned int nr_pages);
266
267static void obj_cgroup_release(struct percpu_ref *ref)
268{
269 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
270 unsigned int nr_bytes;
271 unsigned int nr_pages;
272 unsigned long flags;
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294 nr_bytes = atomic_read(&objcg->nr_charged_bytes);
295 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
296 nr_pages = nr_bytes >> PAGE_SHIFT;
297
298 if (nr_pages)
299 obj_cgroup_uncharge_pages(objcg, nr_pages);
300
301 spin_lock_irqsave(&css_set_lock, flags);
302 list_del(&objcg->list);
303 spin_unlock_irqrestore(&css_set_lock, flags);
304
305 percpu_ref_exit(ref);
306 kfree_rcu(objcg, rcu);
307}
308
309static struct obj_cgroup *obj_cgroup_alloc(void)
310{
311 struct obj_cgroup *objcg;
312 int ret;
313
314 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
315 if (!objcg)
316 return NULL;
317
318 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
319 GFP_KERNEL);
320 if (ret) {
321 kfree(objcg);
322 return NULL;
323 }
324 INIT_LIST_HEAD(&objcg->list);
325 return objcg;
326}
327
328static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
329 struct mem_cgroup *parent)
330{
331 struct obj_cgroup *objcg, *iter;
332
333 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
334
335 spin_lock_irq(&css_set_lock);
336
337
338 list_add(&objcg->list, &memcg->objcg_list);
339
340 list_for_each_entry(iter, &memcg->objcg_list, list)
341 WRITE_ONCE(iter->memcg, parent);
342
343 list_splice(&memcg->objcg_list, &parent->objcg_list);
344
345 spin_unlock_irq(&css_set_lock);
346
347 percpu_ref_kill(&objcg->refcnt);
348}
349
350
351
352
353
354
355
356
357
358
359
360
361static DEFINE_IDA(memcg_cache_ida);
362int memcg_nr_cache_ids;
363
364
365static DECLARE_RWSEM(memcg_cache_ids_sem);
366
367void memcg_get_cache_ids(void)
368{
369 down_read(&memcg_cache_ids_sem);
370}
371
372void memcg_put_cache_ids(void)
373{
374 up_read(&memcg_cache_ids_sem);
375}
376
377
378
379
380
381
382
383
384
385
386
387
388
389#define MEMCG_CACHES_MIN_SIZE 4
390#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
391
392
393
394
395
396
397
398DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
399EXPORT_SYMBOL(memcg_kmem_enabled_key);
400#endif
401
402
403
404
405
406
407
408
409
410
411
412
413struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
414{
415 struct mem_cgroup *memcg;
416
417 memcg = page_memcg(page);
418
419 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
420 memcg = root_mem_cgroup;
421
422 return &memcg->css;
423}
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438ino_t page_cgroup_ino(struct page *page)
439{
440 struct mem_cgroup *memcg;
441 unsigned long ino = 0;
442
443 rcu_read_lock();
444 memcg = page_memcg_check(page);
445
446 while (memcg && !(memcg->css.flags & CSS_ONLINE))
447 memcg = parent_mem_cgroup(memcg);
448 if (memcg)
449 ino = cgroup_ino(memcg->css.cgroup);
450 rcu_read_unlock();
451 return ino;
452}
453
454static struct mem_cgroup_per_node *
455mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
456{
457 int nid = page_to_nid(page);
458
459 return memcg->nodeinfo[nid];
460}
461
462static struct mem_cgroup_tree_per_node *
463soft_limit_tree_node(int nid)
464{
465 return soft_limit_tree.rb_tree_per_node[nid];
466}
467
468static struct mem_cgroup_tree_per_node *
469soft_limit_tree_from_page(struct page *page)
470{
471 int nid = page_to_nid(page);
472
473 return soft_limit_tree.rb_tree_per_node[nid];
474}
475
476static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
477 struct mem_cgroup_tree_per_node *mctz,
478 unsigned long new_usage_in_excess)
479{
480 struct rb_node **p = &mctz->rb_root.rb_node;
481 struct rb_node *parent = NULL;
482 struct mem_cgroup_per_node *mz_node;
483 bool rightmost = true;
484
485 if (mz->on_tree)
486 return;
487
488 mz->usage_in_excess = new_usage_in_excess;
489 if (!mz->usage_in_excess)
490 return;
491 while (*p) {
492 parent = *p;
493 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
494 tree_node);
495 if (mz->usage_in_excess < mz_node->usage_in_excess) {
496 p = &(*p)->rb_left;
497 rightmost = false;
498 } else {
499 p = &(*p)->rb_right;
500 }
501 }
502
503 if (rightmost)
504 mctz->rb_rightmost = &mz->tree_node;
505
506 rb_link_node(&mz->tree_node, parent, p);
507 rb_insert_color(&mz->tree_node, &mctz->rb_root);
508 mz->on_tree = true;
509}
510
511static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
512 struct mem_cgroup_tree_per_node *mctz)
513{
514 if (!mz->on_tree)
515 return;
516
517 if (&mz->tree_node == mctz->rb_rightmost)
518 mctz->rb_rightmost = rb_prev(&mz->tree_node);
519
520 rb_erase(&mz->tree_node, &mctz->rb_root);
521 mz->on_tree = false;
522}
523
524static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
525 struct mem_cgroup_tree_per_node *mctz)
526{
527 unsigned long flags;
528
529 spin_lock_irqsave(&mctz->lock, flags);
530 __mem_cgroup_remove_exceeded(mz, mctz);
531 spin_unlock_irqrestore(&mctz->lock, flags);
532}
533
534static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
535{
536 unsigned long nr_pages = page_counter_read(&memcg->memory);
537 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
538 unsigned long excess = 0;
539
540 if (nr_pages > soft_limit)
541 excess = nr_pages - soft_limit;
542
543 return excess;
544}
545
546static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
547{
548 unsigned long excess;
549 struct mem_cgroup_per_node *mz;
550 struct mem_cgroup_tree_per_node *mctz;
551
552 mctz = soft_limit_tree_from_page(page);
553 if (!mctz)
554 return;
555
556
557
558
559 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
560 mz = mem_cgroup_page_nodeinfo(memcg, page);
561 excess = soft_limit_excess(memcg);
562
563
564
565
566 if (excess || mz->on_tree) {
567 unsigned long flags;
568
569 spin_lock_irqsave(&mctz->lock, flags);
570
571 if (mz->on_tree)
572 __mem_cgroup_remove_exceeded(mz, mctz);
573
574
575
576
577 __mem_cgroup_insert_exceeded(mz, mctz, excess);
578 spin_unlock_irqrestore(&mctz->lock, flags);
579 }
580 }
581}
582
583static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
584{
585 struct mem_cgroup_tree_per_node *mctz;
586 struct mem_cgroup_per_node *mz;
587 int nid;
588
589 for_each_node(nid) {
590 mz = memcg->nodeinfo[nid];
591 mctz = soft_limit_tree_node(nid);
592 if (mctz)
593 mem_cgroup_remove_exceeded(mz, mctz);
594 }
595}
596
597static struct mem_cgroup_per_node *
598__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
599{
600 struct mem_cgroup_per_node *mz;
601
602retry:
603 mz = NULL;
604 if (!mctz->rb_rightmost)
605 goto done;
606
607 mz = rb_entry(mctz->rb_rightmost,
608 struct mem_cgroup_per_node, tree_node);
609
610
611
612
613
614 __mem_cgroup_remove_exceeded(mz, mctz);
615 if (!soft_limit_excess(mz->memcg) ||
616 !css_tryget(&mz->memcg->css))
617 goto retry;
618done:
619 return mz;
620}
621
622static struct mem_cgroup_per_node *
623mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
624{
625 struct mem_cgroup_per_node *mz;
626
627 spin_lock_irq(&mctz->lock);
628 mz = __mem_cgroup_largest_soft_limit_node(mctz);
629 spin_unlock_irq(&mctz->lock);
630 return mz;
631}
632
633
634
635
636
637
638
639void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
640{
641 if (mem_cgroup_disabled())
642 return;
643
644 __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
645 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
646}
647
648
649static unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
650{
651 long x = READ_ONCE(memcg->vmstats.state[idx]);
652#ifdef CONFIG_SMP
653 if (x < 0)
654 x = 0;
655#endif
656 return x;
657}
658
659
660static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
661{
662 long x = 0;
663 int cpu;
664
665 for_each_possible_cpu(cpu)
666 x += per_cpu(memcg->vmstats_percpu->state[idx], cpu);
667#ifdef CONFIG_SMP
668 if (x < 0)
669 x = 0;
670#endif
671 return x;
672}
673
674static struct mem_cgroup_per_node *
675parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
676{
677 struct mem_cgroup *parent;
678
679 parent = parent_mem_cgroup(pn->memcg);
680 if (!parent)
681 return NULL;
682 return parent->nodeinfo[nid];
683}
684
685void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
686 int val)
687{
688 struct mem_cgroup_per_node *pn;
689 struct mem_cgroup *memcg;
690 long x, threshold = MEMCG_CHARGE_BATCH;
691
692 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
693 memcg = pn->memcg;
694
695
696 __mod_memcg_state(memcg, idx, val);
697
698
699 __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
700
701 if (vmstat_item_in_bytes(idx))
702 threshold <<= PAGE_SHIFT;
703
704 x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
705 if (unlikely(abs(x) > threshold)) {
706 pg_data_t *pgdat = lruvec_pgdat(lruvec);
707 struct mem_cgroup_per_node *pi;
708
709 for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
710 atomic_long_add(x, &pi->lruvec_stat[idx]);
711 x = 0;
712 }
713 __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
714}
715
716
717
718
719
720
721
722
723
724
725
726void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
727 int val)
728{
729
730 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
731
732
733 if (!mem_cgroup_disabled())
734 __mod_memcg_lruvec_state(lruvec, idx, val);
735}
736
737void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx,
738 int val)
739{
740 struct page *head = compound_head(page);
741 struct mem_cgroup *memcg;
742 pg_data_t *pgdat = page_pgdat(page);
743 struct lruvec *lruvec;
744
745 rcu_read_lock();
746 memcg = page_memcg(head);
747
748 if (!memcg) {
749 rcu_read_unlock();
750 __mod_node_page_state(pgdat, idx, val);
751 return;
752 }
753
754 lruvec = mem_cgroup_lruvec(memcg, pgdat);
755 __mod_lruvec_state(lruvec, idx, val);
756 rcu_read_unlock();
757}
758EXPORT_SYMBOL(__mod_lruvec_page_state);
759
760void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val)
761{
762 pg_data_t *pgdat = page_pgdat(virt_to_page(p));
763 struct mem_cgroup *memcg;
764 struct lruvec *lruvec;
765
766 rcu_read_lock();
767 memcg = mem_cgroup_from_obj(p);
768
769
770
771
772
773
774
775 if (!memcg) {
776 __mod_node_page_state(pgdat, idx, val);
777 } else {
778 lruvec = mem_cgroup_lruvec(memcg, pgdat);
779 __mod_lruvec_state(lruvec, idx, val);
780 }
781 rcu_read_unlock();
782}
783
784
785
786
787
788static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
789 struct pglist_data *pgdat,
790 enum node_stat_item idx, int nr)
791{
792 struct mem_cgroup *memcg;
793 struct lruvec *lruvec;
794
795 rcu_read_lock();
796 memcg = obj_cgroup_memcg(objcg);
797 lruvec = mem_cgroup_lruvec(memcg, pgdat);
798 mod_memcg_lruvec_state(lruvec, idx, nr);
799 rcu_read_unlock();
800}
801
802
803
804
805
806
807
808void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
809 unsigned long count)
810{
811 if (mem_cgroup_disabled())
812 return;
813
814 __this_cpu_add(memcg->vmstats_percpu->events[idx], count);
815 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
816}
817
818static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
819{
820 return READ_ONCE(memcg->vmstats.events[event]);
821}
822
823static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
824{
825 long x = 0;
826 int cpu;
827
828 for_each_possible_cpu(cpu)
829 x += per_cpu(memcg->vmstats_percpu->events[event], cpu);
830 return x;
831}
832
833static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
834 struct page *page,
835 int nr_pages)
836{
837
838 if (nr_pages > 0)
839 __count_memcg_events(memcg, PGPGIN, 1);
840 else {
841 __count_memcg_events(memcg, PGPGOUT, 1);
842 nr_pages = -nr_pages;
843 }
844
845 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
846}
847
848static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
849 enum mem_cgroup_events_target target)
850{
851 unsigned long val, next;
852
853 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
854 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
855
856 if ((long)(next - val) < 0) {
857 switch (target) {
858 case MEM_CGROUP_TARGET_THRESH:
859 next = val + THRESHOLDS_EVENTS_TARGET;
860 break;
861 case MEM_CGROUP_TARGET_SOFTLIMIT:
862 next = val + SOFTLIMIT_EVENTS_TARGET;
863 break;
864 default:
865 break;
866 }
867 __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
868 return true;
869 }
870 return false;
871}
872
873
874
875
876
877static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
878{
879
880 if (unlikely(mem_cgroup_event_ratelimit(memcg,
881 MEM_CGROUP_TARGET_THRESH))) {
882 bool do_softlimit;
883
884 do_softlimit = mem_cgroup_event_ratelimit(memcg,
885 MEM_CGROUP_TARGET_SOFTLIMIT);
886 mem_cgroup_threshold(memcg);
887 if (unlikely(do_softlimit))
888 mem_cgroup_update_tree(memcg, page);
889 }
890}
891
892struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
893{
894
895
896
897
898
899 if (unlikely(!p))
900 return NULL;
901
902 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
903}
904EXPORT_SYMBOL(mem_cgroup_from_task);
905
906static __always_inline struct mem_cgroup *active_memcg(void)
907{
908 if (in_interrupt())
909 return this_cpu_read(int_active_memcg);
910 else
911 return current->active_memcg;
912}
913
914
915
916
917
918
919
920
921
922
923
924
925struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
926{
927 struct mem_cgroup *memcg;
928
929 if (mem_cgroup_disabled())
930 return NULL;
931
932
933
934
935
936
937
938
939
940
941 if (unlikely(!mm)) {
942 memcg = active_memcg();
943 if (unlikely(memcg)) {
944
945 css_get(&memcg->css);
946 return memcg;
947 }
948 mm = current->mm;
949 if (unlikely(!mm))
950 return root_mem_cgroup;
951 }
952
953 rcu_read_lock();
954 do {
955 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
956 if (unlikely(!memcg))
957 memcg = root_mem_cgroup;
958 } while (!css_tryget(&memcg->css));
959 rcu_read_unlock();
960 return memcg;
961}
962EXPORT_SYMBOL(get_mem_cgroup_from_mm);
963
964static __always_inline bool memcg_kmem_bypass(void)
965{
966
967 if (unlikely(active_memcg()))
968 return false;
969
970
971 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
972 return true;
973
974 return false;
975}
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
995 struct mem_cgroup *prev,
996 struct mem_cgroup_reclaim_cookie *reclaim)
997{
998 struct mem_cgroup_reclaim_iter *iter;
999 struct cgroup_subsys_state *css = NULL;
1000 struct mem_cgroup *memcg = NULL;
1001 struct mem_cgroup *pos = NULL;
1002
1003 if (mem_cgroup_disabled())
1004 return NULL;
1005
1006 if (!root)
1007 root = root_mem_cgroup;
1008
1009 if (prev && !reclaim)
1010 pos = prev;
1011
1012 rcu_read_lock();
1013
1014 if (reclaim) {
1015 struct mem_cgroup_per_node *mz;
1016
1017 mz = root->nodeinfo[reclaim->pgdat->node_id];
1018 iter = &mz->iter;
1019
1020 if (prev && reclaim->generation != iter->generation)
1021 goto out_unlock;
1022
1023 while (1) {
1024 pos = READ_ONCE(iter->position);
1025 if (!pos || css_tryget(&pos->css))
1026 break;
1027
1028
1029
1030
1031
1032
1033
1034
1035 (void)cmpxchg(&iter->position, pos, NULL);
1036 }
1037 }
1038
1039 if (pos)
1040 css = &pos->css;
1041
1042 for (;;) {
1043 css = css_next_descendant_pre(css, &root->css);
1044 if (!css) {
1045
1046
1047
1048
1049
1050
1051 if (!prev)
1052 continue;
1053 break;
1054 }
1055
1056
1057
1058
1059
1060
1061 memcg = mem_cgroup_from_css(css);
1062
1063 if (css == &root->css)
1064 break;
1065
1066 if (css_tryget(css))
1067 break;
1068
1069 memcg = NULL;
1070 }
1071
1072 if (reclaim) {
1073
1074
1075
1076
1077
1078 (void)cmpxchg(&iter->position, pos, memcg);
1079
1080 if (pos)
1081 css_put(&pos->css);
1082
1083 if (!memcg)
1084 iter->generation++;
1085 else if (!prev)
1086 reclaim->generation = iter->generation;
1087 }
1088
1089out_unlock:
1090 rcu_read_unlock();
1091 if (prev && prev != root)
1092 css_put(&prev->css);
1093
1094 return memcg;
1095}
1096
1097
1098
1099
1100
1101
1102void mem_cgroup_iter_break(struct mem_cgroup *root,
1103 struct mem_cgroup *prev)
1104{
1105 if (!root)
1106 root = root_mem_cgroup;
1107 if (prev && prev != root)
1108 css_put(&prev->css);
1109}
1110
1111static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1112 struct mem_cgroup *dead_memcg)
1113{
1114 struct mem_cgroup_reclaim_iter *iter;
1115 struct mem_cgroup_per_node *mz;
1116 int nid;
1117
1118 for_each_node(nid) {
1119 mz = from->nodeinfo[nid];
1120 iter = &mz->iter;
1121 cmpxchg(&iter->position, dead_memcg, NULL);
1122 }
1123}
1124
1125static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1126{
1127 struct mem_cgroup *memcg = dead_memcg;
1128 struct mem_cgroup *last;
1129
1130 do {
1131 __invalidate_reclaim_iterators(memcg, dead_memcg);
1132 last = memcg;
1133 } while ((memcg = parent_mem_cgroup(memcg)));
1134
1135
1136
1137
1138
1139
1140
1141 if (last != root_mem_cgroup)
1142 __invalidate_reclaim_iterators(root_mem_cgroup,
1143 dead_memcg);
1144}
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1160 int (*fn)(struct task_struct *, void *), void *arg)
1161{
1162 struct mem_cgroup *iter;
1163 int ret = 0;
1164
1165 BUG_ON(memcg == root_mem_cgroup);
1166
1167 for_each_mem_cgroup_tree(iter, memcg) {
1168 struct css_task_iter it;
1169 struct task_struct *task;
1170
1171 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
1172 while (!ret && (task = css_task_iter_next(&it)))
1173 ret = fn(task, arg);
1174 css_task_iter_end(&it);
1175 if (ret) {
1176 mem_cgroup_iter_break(memcg, iter);
1177 break;
1178 }
1179 }
1180 return ret;
1181}
1182
1183#ifdef CONFIG_DEBUG_VM
1184void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
1185{
1186 struct mem_cgroup *memcg;
1187
1188 if (mem_cgroup_disabled())
1189 return;
1190
1191 memcg = page_memcg(page);
1192
1193 if (!memcg)
1194 VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != root_mem_cgroup, page);
1195 else
1196 VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != memcg, page);
1197}
1198#endif
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210struct lruvec *lock_page_lruvec(struct page *page)
1211{
1212 struct lruvec *lruvec;
1213
1214 lruvec = mem_cgroup_page_lruvec(page);
1215 spin_lock(&lruvec->lru_lock);
1216
1217 lruvec_memcg_debug(lruvec, page);
1218
1219 return lruvec;
1220}
1221
1222struct lruvec *lock_page_lruvec_irq(struct page *page)
1223{
1224 struct lruvec *lruvec;
1225
1226 lruvec = mem_cgroup_page_lruvec(page);
1227 spin_lock_irq(&lruvec->lru_lock);
1228
1229 lruvec_memcg_debug(lruvec, page);
1230
1231 return lruvec;
1232}
1233
1234struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
1235{
1236 struct lruvec *lruvec;
1237
1238 lruvec = mem_cgroup_page_lruvec(page);
1239 spin_lock_irqsave(&lruvec->lru_lock, *flags);
1240
1241 lruvec_memcg_debug(lruvec, page);
1242
1243 return lruvec;
1244}
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1258 int zid, int nr_pages)
1259{
1260 struct mem_cgroup_per_node *mz;
1261 unsigned long *lru_size;
1262 long size;
1263
1264 if (mem_cgroup_disabled())
1265 return;
1266
1267 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1268 lru_size = &mz->lru_zone_size[zid][lru];
1269
1270 if (nr_pages < 0)
1271 *lru_size += nr_pages;
1272
1273 size = *lru_size;
1274 if (WARN_ONCE(size < 0,
1275 "%s(%p, %d, %d): lru_size %ld\n",
1276 __func__, lruvec, lru, nr_pages, size)) {
1277 VM_BUG_ON(1);
1278 *lru_size = 0;
1279 }
1280
1281 if (nr_pages > 0)
1282 *lru_size += nr_pages;
1283}
1284
1285
1286
1287
1288
1289
1290
1291
1292static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1293{
1294 unsigned long margin = 0;
1295 unsigned long count;
1296 unsigned long limit;
1297
1298 count = page_counter_read(&memcg->memory);
1299 limit = READ_ONCE(memcg->memory.max);
1300 if (count < limit)
1301 margin = limit - count;
1302
1303 if (do_memsw_account()) {
1304 count = page_counter_read(&memcg->memsw);
1305 limit = READ_ONCE(memcg->memsw.max);
1306 if (count < limit)
1307 margin = min(margin, limit - count);
1308 else
1309 margin = 0;
1310 }
1311
1312 return margin;
1313}
1314
1315
1316
1317
1318
1319
1320
1321
1322static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1323{
1324 struct mem_cgroup *from;
1325 struct mem_cgroup *to;
1326 bool ret = false;
1327
1328
1329
1330
1331 spin_lock(&mc.lock);
1332 from = mc.from;
1333 to = mc.to;
1334 if (!from)
1335 goto unlock;
1336
1337 ret = mem_cgroup_is_descendant(from, memcg) ||
1338 mem_cgroup_is_descendant(to, memcg);
1339unlock:
1340 spin_unlock(&mc.lock);
1341 return ret;
1342}
1343
1344static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1345{
1346 if (mc.moving_task && current != mc.moving_task) {
1347 if (mem_cgroup_under_move(memcg)) {
1348 DEFINE_WAIT(wait);
1349 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1350
1351 if (mc.moving_task)
1352 schedule();
1353 finish_wait(&mc.waitq, &wait);
1354 return true;
1355 }
1356 }
1357 return false;
1358}
1359
1360struct memory_stat {
1361 const char *name;
1362 unsigned int idx;
1363};
1364
1365static const struct memory_stat memory_stats[] = {
1366 { "anon", NR_ANON_MAPPED },
1367 { "file", NR_FILE_PAGES },
1368 { "kernel_stack", NR_KERNEL_STACK_KB },
1369 { "pagetables", NR_PAGETABLE },
1370 { "percpu", MEMCG_PERCPU_B },
1371 { "sock", MEMCG_SOCK },
1372 { "shmem", NR_SHMEM },
1373 { "file_mapped", NR_FILE_MAPPED },
1374 { "file_dirty", NR_FILE_DIRTY },
1375 { "file_writeback", NR_WRITEBACK },
1376#ifdef CONFIG_SWAP
1377 { "swapcached", NR_SWAPCACHE },
1378#endif
1379#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1380 { "anon_thp", NR_ANON_THPS },
1381 { "file_thp", NR_FILE_THPS },
1382 { "shmem_thp", NR_SHMEM_THPS },
1383#endif
1384 { "inactive_anon", NR_INACTIVE_ANON },
1385 { "active_anon", NR_ACTIVE_ANON },
1386 { "inactive_file", NR_INACTIVE_FILE },
1387 { "active_file", NR_ACTIVE_FILE },
1388 { "unevictable", NR_UNEVICTABLE },
1389 { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B },
1390 { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B },
1391
1392
1393 { "workingset_refault_anon", WORKINGSET_REFAULT_ANON },
1394 { "workingset_refault_file", WORKINGSET_REFAULT_FILE },
1395 { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON },
1396 { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE },
1397 { "workingset_restore_anon", WORKINGSET_RESTORE_ANON },
1398 { "workingset_restore_file", WORKINGSET_RESTORE_FILE },
1399 { "workingset_nodereclaim", WORKINGSET_NODERECLAIM },
1400};
1401
1402
1403static int memcg_page_state_unit(int item)
1404{
1405 switch (item) {
1406 case MEMCG_PERCPU_B:
1407 case NR_SLAB_RECLAIMABLE_B:
1408 case NR_SLAB_UNRECLAIMABLE_B:
1409 case WORKINGSET_REFAULT_ANON:
1410 case WORKINGSET_REFAULT_FILE:
1411 case WORKINGSET_ACTIVATE_ANON:
1412 case WORKINGSET_ACTIVATE_FILE:
1413 case WORKINGSET_RESTORE_ANON:
1414 case WORKINGSET_RESTORE_FILE:
1415 case WORKINGSET_NODERECLAIM:
1416 return 1;
1417 case NR_KERNEL_STACK_KB:
1418 return SZ_1K;
1419 default:
1420 return PAGE_SIZE;
1421 }
1422}
1423
1424static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
1425 int item)
1426{
1427 return memcg_page_state(memcg, item) * memcg_page_state_unit(item);
1428}
1429
1430static char *memory_stat_format(struct mem_cgroup *memcg)
1431{
1432 struct seq_buf s;
1433 int i;
1434
1435 seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
1436 if (!s.buffer)
1437 return NULL;
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449 cgroup_rstat_flush(memcg->css.cgroup);
1450
1451 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
1452 u64 size;
1453
1454 size = memcg_page_state_output(memcg, memory_stats[i].idx);
1455 seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size);
1456
1457 if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
1458 size += memcg_page_state_output(memcg,
1459 NR_SLAB_RECLAIMABLE_B);
1460 seq_buf_printf(&s, "slab %llu\n", size);
1461 }
1462 }
1463
1464
1465
1466 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT),
1467 memcg_events(memcg, PGFAULT));
1468 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT),
1469 memcg_events(memcg, PGMAJFAULT));
1470 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGREFILL),
1471 memcg_events(memcg, PGREFILL));
1472 seq_buf_printf(&s, "pgscan %lu\n",
1473 memcg_events(memcg, PGSCAN_KSWAPD) +
1474 memcg_events(memcg, PGSCAN_DIRECT));
1475 seq_buf_printf(&s, "pgsteal %lu\n",
1476 memcg_events(memcg, PGSTEAL_KSWAPD) +
1477 memcg_events(memcg, PGSTEAL_DIRECT));
1478 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE),
1479 memcg_events(memcg, PGACTIVATE));
1480 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE),
1481 memcg_events(memcg, PGDEACTIVATE));
1482 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE),
1483 memcg_events(memcg, PGLAZYFREE));
1484 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED),
1485 memcg_events(memcg, PGLAZYFREED));
1486
1487#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1488 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC),
1489 memcg_events(memcg, THP_FAULT_ALLOC));
1490 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC),
1491 memcg_events(memcg, THP_COLLAPSE_ALLOC));
1492#endif
1493
1494
1495 WARN_ON_ONCE(seq_buf_has_overflowed(&s));
1496
1497 return s.buffer;
1498}
1499
1500#define K(x) ((x) << (PAGE_SHIFT-10))
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1511{
1512 rcu_read_lock();
1513
1514 if (memcg) {
1515 pr_cont(",oom_memcg=");
1516 pr_cont_cgroup_path(memcg->css.cgroup);
1517 } else
1518 pr_cont(",global_oom");
1519 if (p) {
1520 pr_cont(",task_memcg=");
1521 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1522 }
1523 rcu_read_unlock();
1524}
1525
1526
1527
1528
1529
1530
1531void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1532{
1533 char *buf;
1534
1535 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1536 K((u64)page_counter_read(&memcg->memory)),
1537 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
1538 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1539 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1540 K((u64)page_counter_read(&memcg->swap)),
1541 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
1542 else {
1543 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1544 K((u64)page_counter_read(&memcg->memsw)),
1545 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1546 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1547 K((u64)page_counter_read(&memcg->kmem)),
1548 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1549 }
1550
1551 pr_info("Memory cgroup stats for ");
1552 pr_cont_cgroup_path(memcg->css.cgroup);
1553 pr_cont(":");
1554 buf = memory_stat_format(memcg);
1555 if (!buf)
1556 return;
1557 pr_info("%s", buf);
1558 kfree(buf);
1559}
1560
1561
1562
1563
1564unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1565{
1566 unsigned long max = READ_ONCE(memcg->memory.max);
1567
1568 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
1569 if (mem_cgroup_swappiness(memcg))
1570 max += min(READ_ONCE(memcg->swap.max),
1571 (unsigned long)total_swap_pages);
1572 } else {
1573 if (mem_cgroup_swappiness(memcg)) {
1574
1575 unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
1576
1577 max += min(swap, (unsigned long)total_swap_pages);
1578 }
1579 }
1580 return max;
1581}
1582
1583unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1584{
1585 return page_counter_read(&memcg->memory);
1586}
1587
1588static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1589 int order)
1590{
1591 struct oom_control oc = {
1592 .zonelist = NULL,
1593 .nodemask = NULL,
1594 .memcg = memcg,
1595 .gfp_mask = gfp_mask,
1596 .order = order,
1597 };
1598 bool ret = true;
1599
1600 if (mutex_lock_killable(&oom_lock))
1601 return true;
1602
1603 if (mem_cgroup_margin(memcg) >= (1 << order))
1604 goto unlock;
1605
1606
1607
1608
1609
1610 ret = should_force_charge() || out_of_memory(&oc);
1611
1612unlock:
1613 mutex_unlock(&oom_lock);
1614 return ret;
1615}
1616
1617static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1618 pg_data_t *pgdat,
1619 gfp_t gfp_mask,
1620 unsigned long *total_scanned)
1621{
1622 struct mem_cgroup *victim = NULL;
1623 int total = 0;
1624 int loop = 0;
1625 unsigned long excess;
1626 unsigned long nr_scanned;
1627 struct mem_cgroup_reclaim_cookie reclaim = {
1628 .pgdat = pgdat,
1629 };
1630
1631 excess = soft_limit_excess(root_memcg);
1632
1633 while (1) {
1634 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1635 if (!victim) {
1636 loop++;
1637 if (loop >= 2) {
1638
1639
1640
1641
1642
1643 if (!total)
1644 break;
1645
1646
1647
1648
1649
1650
1651 if (total >= (excess >> 2) ||
1652 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1653 break;
1654 }
1655 continue;
1656 }
1657 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1658 pgdat, &nr_scanned);
1659 *total_scanned += nr_scanned;
1660 if (!soft_limit_excess(root_memcg))
1661 break;
1662 }
1663 mem_cgroup_iter_break(root_memcg, victim);
1664 return total;
1665}
1666
1667#ifdef CONFIG_LOCKDEP
1668static struct lockdep_map memcg_oom_lock_dep_map = {
1669 .name = "memcg_oom_lock",
1670};
1671#endif
1672
1673static DEFINE_SPINLOCK(memcg_oom_lock);
1674
1675
1676
1677
1678
1679static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1680{
1681 struct mem_cgroup *iter, *failed = NULL;
1682
1683 spin_lock(&memcg_oom_lock);
1684
1685 for_each_mem_cgroup_tree(iter, memcg) {
1686 if (iter->oom_lock) {
1687
1688
1689
1690
1691 failed = iter;
1692 mem_cgroup_iter_break(memcg, iter);
1693 break;
1694 } else
1695 iter->oom_lock = true;
1696 }
1697
1698 if (failed) {
1699
1700
1701
1702
1703 for_each_mem_cgroup_tree(iter, memcg) {
1704 if (iter == failed) {
1705 mem_cgroup_iter_break(memcg, iter);
1706 break;
1707 }
1708 iter->oom_lock = false;
1709 }
1710 } else
1711 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1712
1713 spin_unlock(&memcg_oom_lock);
1714
1715 return !failed;
1716}
1717
1718static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1719{
1720 struct mem_cgroup *iter;
1721
1722 spin_lock(&memcg_oom_lock);
1723 mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
1724 for_each_mem_cgroup_tree(iter, memcg)
1725 iter->oom_lock = false;
1726 spin_unlock(&memcg_oom_lock);
1727}
1728
1729static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1730{
1731 struct mem_cgroup *iter;
1732
1733 spin_lock(&memcg_oom_lock);
1734 for_each_mem_cgroup_tree(iter, memcg)
1735 iter->under_oom++;
1736 spin_unlock(&memcg_oom_lock);
1737}
1738
1739static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1740{
1741 struct mem_cgroup *iter;
1742
1743
1744
1745
1746
1747 spin_lock(&memcg_oom_lock);
1748 for_each_mem_cgroup_tree(iter, memcg)
1749 if (iter->under_oom > 0)
1750 iter->under_oom--;
1751 spin_unlock(&memcg_oom_lock);
1752}
1753
1754static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1755
1756struct oom_wait_info {
1757 struct mem_cgroup *memcg;
1758 wait_queue_entry_t wait;
1759};
1760
1761static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1762 unsigned mode, int sync, void *arg)
1763{
1764 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1765 struct mem_cgroup *oom_wait_memcg;
1766 struct oom_wait_info *oom_wait_info;
1767
1768 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1769 oom_wait_memcg = oom_wait_info->memcg;
1770
1771 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1772 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1773 return 0;
1774 return autoremove_wake_function(wait, mode, sync, arg);
1775}
1776
1777static void memcg_oom_recover(struct mem_cgroup *memcg)
1778{
1779
1780
1781
1782
1783
1784
1785
1786
1787 if (memcg && memcg->under_oom)
1788 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1789}
1790
1791enum oom_status {
1792 OOM_SUCCESS,
1793 OOM_FAILED,
1794 OOM_ASYNC,
1795 OOM_SKIPPED
1796};
1797
1798static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1799{
1800 enum oom_status ret;
1801 bool locked;
1802
1803 if (order > PAGE_ALLOC_COSTLY_ORDER)
1804 return OOM_SKIPPED;
1805
1806 memcg_memory_event(memcg, MEMCG_OOM);
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826 if (memcg->oom_kill_disable) {
1827 if (!current->in_user_fault)
1828 return OOM_SKIPPED;
1829 css_get(&memcg->css);
1830 current->memcg_in_oom = memcg;
1831 current->memcg_oom_gfp_mask = mask;
1832 current->memcg_oom_order = order;
1833
1834 return OOM_ASYNC;
1835 }
1836
1837 mem_cgroup_mark_under_oom(memcg);
1838
1839 locked = mem_cgroup_oom_trylock(memcg);
1840
1841 if (locked)
1842 mem_cgroup_oom_notify(memcg);
1843
1844 mem_cgroup_unmark_under_oom(memcg);
1845 if (mem_cgroup_out_of_memory(memcg, mask, order))
1846 ret = OOM_SUCCESS;
1847 else
1848 ret = OOM_FAILED;
1849
1850 if (locked)
1851 mem_cgroup_oom_unlock(memcg);
1852
1853 return ret;
1854}
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873bool mem_cgroup_oom_synchronize(bool handle)
1874{
1875 struct mem_cgroup *memcg = current->memcg_in_oom;
1876 struct oom_wait_info owait;
1877 bool locked;
1878
1879
1880 if (!memcg)
1881 return false;
1882
1883 if (!handle)
1884 goto cleanup;
1885
1886 owait.memcg = memcg;
1887 owait.wait.flags = 0;
1888 owait.wait.func = memcg_oom_wake_function;
1889 owait.wait.private = current;
1890 INIT_LIST_HEAD(&owait.wait.entry);
1891
1892 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1893 mem_cgroup_mark_under_oom(memcg);
1894
1895 locked = mem_cgroup_oom_trylock(memcg);
1896
1897 if (locked)
1898 mem_cgroup_oom_notify(memcg);
1899
1900 if (locked && !memcg->oom_kill_disable) {
1901 mem_cgroup_unmark_under_oom(memcg);
1902 finish_wait(&memcg_oom_waitq, &owait.wait);
1903 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1904 current->memcg_oom_order);
1905 } else {
1906 schedule();
1907 mem_cgroup_unmark_under_oom(memcg);
1908 finish_wait(&memcg_oom_waitq, &owait.wait);
1909 }
1910
1911 if (locked) {
1912 mem_cgroup_oom_unlock(memcg);
1913
1914
1915
1916
1917
1918 memcg_oom_recover(memcg);
1919 }
1920cleanup:
1921 current->memcg_in_oom = NULL;
1922 css_put(&memcg->css);
1923 return true;
1924}
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
1937 struct mem_cgroup *oom_domain)
1938{
1939 struct mem_cgroup *oom_group = NULL;
1940 struct mem_cgroup *memcg;
1941
1942 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1943 return NULL;
1944
1945 if (!oom_domain)
1946 oom_domain = root_mem_cgroup;
1947
1948 rcu_read_lock();
1949
1950 memcg = mem_cgroup_from_task(victim);
1951 if (memcg == root_mem_cgroup)
1952 goto out;
1953
1954
1955
1956
1957
1958
1959 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
1960 goto out;
1961
1962
1963
1964
1965
1966
1967 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1968 if (memcg->oom_group)
1969 oom_group = memcg;
1970
1971 if (memcg == oom_domain)
1972 break;
1973 }
1974
1975 if (oom_group)
1976 css_get(&oom_group->css);
1977out:
1978 rcu_read_unlock();
1979
1980 return oom_group;
1981}
1982
1983void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1984{
1985 pr_info("Tasks in ");
1986 pr_cont_cgroup_path(memcg->css.cgroup);
1987 pr_cont(" are going to be killed due to memory.oom.group set\n");
1988}
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000void lock_page_memcg(struct page *page)
2001{
2002 struct page *head = compound_head(page);
2003 struct mem_cgroup *memcg;
2004 unsigned long flags;
2005
2006
2007
2008
2009
2010
2011 rcu_read_lock();
2012
2013 if (mem_cgroup_disabled())
2014 return;
2015again:
2016 memcg = page_memcg(head);
2017 if (unlikely(!memcg))
2018 return;
2019
2020#ifdef CONFIG_PROVE_LOCKING
2021 local_irq_save(flags);
2022 might_lock(&memcg->move_lock);
2023 local_irq_restore(flags);
2024#endif
2025
2026 if (atomic_read(&memcg->moving_account) <= 0)
2027 return;
2028
2029 spin_lock_irqsave(&memcg->move_lock, flags);
2030 if (memcg != page_memcg(head)) {
2031 spin_unlock_irqrestore(&memcg->move_lock, flags);
2032 goto again;
2033 }
2034
2035
2036
2037
2038
2039
2040
2041 memcg->move_lock_task = current;
2042 memcg->move_lock_flags = flags;
2043}
2044EXPORT_SYMBOL(lock_page_memcg);
2045
2046static void __unlock_page_memcg(struct mem_cgroup *memcg)
2047{
2048 if (memcg && memcg->move_lock_task == current) {
2049 unsigned long flags = memcg->move_lock_flags;
2050
2051 memcg->move_lock_task = NULL;
2052 memcg->move_lock_flags = 0;
2053
2054 spin_unlock_irqrestore(&memcg->move_lock, flags);
2055 }
2056
2057 rcu_read_unlock();
2058}
2059
2060
2061
2062
2063
2064void unlock_page_memcg(struct page *page)
2065{
2066 struct page *head = compound_head(page);
2067
2068 __unlock_page_memcg(page_memcg(head));
2069}
2070EXPORT_SYMBOL(unlock_page_memcg);
2071
2072struct obj_stock {
2073#ifdef CONFIG_MEMCG_KMEM
2074 struct obj_cgroup *cached_objcg;
2075 struct pglist_data *cached_pgdat;
2076 unsigned int nr_bytes;
2077 int nr_slab_reclaimable_b;
2078 int nr_slab_unreclaimable_b;
2079#else
2080 int dummy[0];
2081#endif
2082};
2083
2084struct memcg_stock_pcp {
2085 struct mem_cgroup *cached;
2086 unsigned int nr_pages;
2087 struct obj_stock task_obj;
2088 struct obj_stock irq_obj;
2089
2090 struct work_struct work;
2091 unsigned long flags;
2092#define FLUSHING_CACHED_CHARGE 0
2093};
2094static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2095static DEFINE_MUTEX(percpu_charge_mutex);
2096
2097#ifdef CONFIG_MEMCG_KMEM
2098static void drain_obj_stock(struct obj_stock *stock);
2099static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2100 struct mem_cgroup *root_memcg);
2101
2102#else
2103static inline void drain_obj_stock(struct obj_stock *stock)
2104{
2105}
2106static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2107 struct mem_cgroup *root_memcg)
2108{
2109 return false;
2110}
2111#endif
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124static inline struct obj_stock *get_obj_stock(unsigned long *pflags)
2125{
2126 struct memcg_stock_pcp *stock;
2127
2128 if (likely(in_task())) {
2129 *pflags = 0UL;
2130 preempt_disable();
2131 stock = this_cpu_ptr(&memcg_stock);
2132 return &stock->task_obj;
2133 }
2134
2135 local_irq_save(*pflags);
2136 stock = this_cpu_ptr(&memcg_stock);
2137 return &stock->irq_obj;
2138}
2139
2140static inline void put_obj_stock(unsigned long flags)
2141{
2142 if (likely(in_task()))
2143 preempt_enable();
2144 else
2145 local_irq_restore(flags);
2146}
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2160{
2161 struct memcg_stock_pcp *stock;
2162 unsigned long flags;
2163 bool ret = false;
2164
2165 if (nr_pages > MEMCG_CHARGE_BATCH)
2166 return ret;
2167
2168 local_irq_save(flags);
2169
2170 stock = this_cpu_ptr(&memcg_stock);
2171 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
2172 stock->nr_pages -= nr_pages;
2173 ret = true;
2174 }
2175
2176 local_irq_restore(flags);
2177
2178 return ret;
2179}
2180
2181
2182
2183
2184static void drain_stock(struct memcg_stock_pcp *stock)
2185{
2186 struct mem_cgroup *old = stock->cached;
2187
2188 if (!old)
2189 return;
2190
2191 if (stock->nr_pages) {
2192 page_counter_uncharge(&old->memory, stock->nr_pages);
2193 if (do_memsw_account())
2194 page_counter_uncharge(&old->memsw, stock->nr_pages);
2195 stock->nr_pages = 0;
2196 }
2197
2198 css_put(&old->css);
2199 stock->cached = NULL;
2200}
2201
2202static void drain_local_stock(struct work_struct *dummy)
2203{
2204 struct memcg_stock_pcp *stock;
2205 unsigned long flags;
2206
2207
2208
2209
2210
2211 local_irq_save(flags);
2212
2213 stock = this_cpu_ptr(&memcg_stock);
2214 drain_obj_stock(&stock->irq_obj);
2215 if (in_task())
2216 drain_obj_stock(&stock->task_obj);
2217 drain_stock(stock);
2218 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2219
2220 local_irq_restore(flags);
2221}
2222
2223
2224
2225
2226
2227static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2228{
2229 struct memcg_stock_pcp *stock;
2230 unsigned long flags;
2231
2232 local_irq_save(flags);
2233
2234 stock = this_cpu_ptr(&memcg_stock);
2235 if (stock->cached != memcg) {
2236 drain_stock(stock);
2237 css_get(&memcg->css);
2238 stock->cached = memcg;
2239 }
2240 stock->nr_pages += nr_pages;
2241
2242 if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2243 drain_stock(stock);
2244
2245 local_irq_restore(flags);
2246}
2247
2248
2249
2250
2251
2252static void drain_all_stock(struct mem_cgroup *root_memcg)
2253{
2254 int cpu, curcpu;
2255
2256
2257 if (!mutex_trylock(&percpu_charge_mutex))
2258 return;
2259
2260
2261
2262
2263
2264
2265 curcpu = get_cpu();
2266 for_each_online_cpu(cpu) {
2267 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2268 struct mem_cgroup *memcg;
2269 bool flush = false;
2270
2271 rcu_read_lock();
2272 memcg = stock->cached;
2273 if (memcg && stock->nr_pages &&
2274 mem_cgroup_is_descendant(memcg, root_memcg))
2275 flush = true;
2276 if (obj_stock_flush_required(stock, root_memcg))
2277 flush = true;
2278 rcu_read_unlock();
2279
2280 if (flush &&
2281 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2282 if (cpu == curcpu)
2283 drain_local_stock(&stock->work);
2284 else
2285 schedule_work_on(cpu, &stock->work);
2286 }
2287 }
2288 put_cpu();
2289 mutex_unlock(&percpu_charge_mutex);
2290}
2291
2292static void memcg_flush_lruvec_page_state(struct mem_cgroup *memcg, int cpu)
2293{
2294 int nid;
2295
2296 for_each_node(nid) {
2297 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
2298 unsigned long stat[NR_VM_NODE_STAT_ITEMS];
2299 struct batched_lruvec_stat *lstatc;
2300 int i;
2301
2302 lstatc = per_cpu_ptr(pn->lruvec_stat_cpu, cpu);
2303 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
2304 stat[i] = lstatc->count[i];
2305 lstatc->count[i] = 0;
2306 }
2307
2308 do {
2309 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
2310 atomic_long_add(stat[i], &pn->lruvec_stat[i]);
2311 } while ((pn = parent_nodeinfo(pn, nid)));
2312 }
2313}
2314
2315static int memcg_hotplug_cpu_dead(unsigned int cpu)
2316{
2317 struct memcg_stock_pcp *stock;
2318 struct mem_cgroup *memcg;
2319
2320 stock = &per_cpu(memcg_stock, cpu);
2321 drain_stock(stock);
2322
2323 for_each_mem_cgroup(memcg)
2324 memcg_flush_lruvec_page_state(memcg, cpu);
2325
2326 return 0;
2327}
2328
2329static unsigned long reclaim_high(struct mem_cgroup *memcg,
2330 unsigned int nr_pages,
2331 gfp_t gfp_mask)
2332{
2333 unsigned long nr_reclaimed = 0;
2334
2335 do {
2336 unsigned long pflags;
2337
2338 if (page_counter_read(&memcg->memory) <=
2339 READ_ONCE(memcg->memory.high))
2340 continue;
2341
2342 memcg_memory_event(memcg, MEMCG_HIGH);
2343
2344 psi_memstall_enter(&pflags);
2345 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2346 gfp_mask, true);
2347 psi_memstall_leave(&pflags);
2348 } while ((memcg = parent_mem_cgroup(memcg)) &&
2349 !mem_cgroup_is_root(memcg));
2350
2351 return nr_reclaimed;
2352}
2353
2354static void high_work_func(struct work_struct *work)
2355{
2356 struct mem_cgroup *memcg;
2357
2358 memcg = container_of(work, struct mem_cgroup, high_work);
2359 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2360}
2361
2362
2363
2364
2365
2366
2367#define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412 #define MEMCG_DELAY_PRECISION_SHIFT 20
2413 #define MEMCG_DELAY_SCALING_SHIFT 14
2414
2415static u64 calculate_overage(unsigned long usage, unsigned long high)
2416{
2417 u64 overage;
2418
2419 if (usage <= high)
2420 return 0;
2421
2422
2423
2424
2425
2426 high = max(high, 1UL);
2427
2428 overage = usage - high;
2429 overage <<= MEMCG_DELAY_PRECISION_SHIFT;
2430 return div64_u64(overage, high);
2431}
2432
2433static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2434{
2435 u64 overage, max_overage = 0;
2436
2437 do {
2438 overage = calculate_overage(page_counter_read(&memcg->memory),
2439 READ_ONCE(memcg->memory.high));
2440 max_overage = max(overage, max_overage);
2441 } while ((memcg = parent_mem_cgroup(memcg)) &&
2442 !mem_cgroup_is_root(memcg));
2443
2444 return max_overage;
2445}
2446
2447static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2448{
2449 u64 overage, max_overage = 0;
2450
2451 do {
2452 overage = calculate_overage(page_counter_read(&memcg->swap),
2453 READ_ONCE(memcg->swap.high));
2454 if (overage)
2455 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2456 max_overage = max(overage, max_overage);
2457 } while ((memcg = parent_mem_cgroup(memcg)) &&
2458 !mem_cgroup_is_root(memcg));
2459
2460 return max_overage;
2461}
2462
2463
2464
2465
2466
2467static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2468 unsigned int nr_pages,
2469 u64 max_overage)
2470{
2471 unsigned long penalty_jiffies;
2472
2473 if (!max_overage)
2474 return 0;
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484 penalty_jiffies = max_overage * max_overage * HZ;
2485 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2486 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
2497}
2498
2499
2500
2501
2502
2503void mem_cgroup_handle_over_high(void)
2504{
2505 unsigned long penalty_jiffies;
2506 unsigned long pflags;
2507 unsigned long nr_reclaimed;
2508 unsigned int nr_pages = current->memcg_nr_pages_over_high;
2509 int nr_retries = MAX_RECLAIM_RETRIES;
2510 struct mem_cgroup *memcg;
2511 bool in_retry = false;
2512
2513 if (likely(!nr_pages))
2514 return;
2515
2516 memcg = get_mem_cgroup_from_mm(current->mm);
2517 current->memcg_nr_pages_over_high = 0;
2518
2519retry_reclaim:
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529 nr_reclaimed = reclaim_high(memcg,
2530 in_retry ? SWAP_CLUSTER_MAX : nr_pages,
2531 GFP_KERNEL);
2532
2533
2534
2535
2536
2537 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2538 mem_find_max_overage(memcg));
2539
2540 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2541 swap_find_max_overage(memcg));
2542
2543
2544
2545
2546
2547
2548 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2549
2550
2551
2552
2553
2554
2555
2556 if (penalty_jiffies <= HZ / 100)
2557 goto out;
2558
2559
2560
2561
2562
2563
2564 if (nr_reclaimed || nr_retries--) {
2565 in_retry = true;
2566 goto retry_reclaim;
2567 }
2568
2569
2570
2571
2572
2573
2574 psi_memstall_enter(&pflags);
2575 schedule_timeout_killable(penalty_jiffies);
2576 psi_memstall_leave(&pflags);
2577
2578out:
2579 css_put(&memcg->css);
2580}
2581
2582static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
2583 unsigned int nr_pages)
2584{
2585 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2586 int nr_retries = MAX_RECLAIM_RETRIES;
2587 struct mem_cgroup *mem_over_limit;
2588 struct page_counter *counter;
2589 enum oom_status oom_status;
2590 unsigned long nr_reclaimed;
2591 bool may_swap = true;
2592 bool drained = false;
2593 unsigned long pflags;
2594
2595retry:
2596 if (consume_stock(memcg, nr_pages))
2597 return 0;
2598
2599 if (!do_memsw_account() ||
2600 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2601 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2602 goto done_restock;
2603 if (do_memsw_account())
2604 page_counter_uncharge(&memcg->memsw, batch);
2605 mem_over_limit = mem_cgroup_from_counter(counter, memory);
2606 } else {
2607 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2608 may_swap = false;
2609 }
2610
2611 if (batch > nr_pages) {
2612 batch = nr_pages;
2613 goto retry;
2614 }
2615
2616
2617
2618
2619
2620
2621
2622 if (gfp_mask & __GFP_ATOMIC)
2623 goto force;
2624
2625
2626
2627
2628
2629
2630
2631 if (unlikely(should_force_charge()))
2632 goto force;
2633
2634
2635
2636
2637
2638
2639
2640 if (unlikely(current->flags & PF_MEMALLOC))
2641 goto force;
2642
2643 if (unlikely(task_in_memcg_oom(current)))
2644 goto nomem;
2645
2646 if (!gfpflags_allow_blocking(gfp_mask))
2647 goto nomem;
2648
2649 memcg_memory_event(mem_over_limit, MEMCG_MAX);
2650
2651 psi_memstall_enter(&pflags);
2652 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2653 gfp_mask, may_swap);
2654 psi_memstall_leave(&pflags);
2655
2656 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2657 goto retry;
2658
2659 if (!drained) {
2660 drain_all_stock(mem_over_limit);
2661 drained = true;
2662 goto retry;
2663 }
2664
2665 if (gfp_mask & __GFP_NORETRY)
2666 goto nomem;
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2677 goto retry;
2678
2679
2680
2681
2682 if (mem_cgroup_wait_acct_move(mem_over_limit))
2683 goto retry;
2684
2685 if (nr_retries--)
2686 goto retry;
2687
2688 if (gfp_mask & __GFP_RETRY_MAYFAIL)
2689 goto nomem;
2690
2691 if (fatal_signal_pending(current))
2692 goto force;
2693
2694
2695
2696
2697
2698
2699 oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
2700 get_order(nr_pages * PAGE_SIZE));
2701 switch (oom_status) {
2702 case OOM_SUCCESS:
2703 nr_retries = MAX_RECLAIM_RETRIES;
2704 goto retry;
2705 case OOM_FAILED:
2706 goto force;
2707 default:
2708 goto nomem;
2709 }
2710nomem:
2711 if (!(gfp_mask & __GFP_NOFAIL))
2712 return -ENOMEM;
2713force:
2714
2715
2716
2717
2718
2719 page_counter_charge(&memcg->memory, nr_pages);
2720 if (do_memsw_account())
2721 page_counter_charge(&memcg->memsw, nr_pages);
2722
2723 return 0;
2724
2725done_restock:
2726 if (batch > nr_pages)
2727 refill_stock(memcg, batch - nr_pages);
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738 do {
2739 bool mem_high, swap_high;
2740
2741 mem_high = page_counter_read(&memcg->memory) >
2742 READ_ONCE(memcg->memory.high);
2743 swap_high = page_counter_read(&memcg->swap) >
2744 READ_ONCE(memcg->swap.high);
2745
2746
2747 if (in_interrupt()) {
2748 if (mem_high) {
2749 schedule_work(&memcg->high_work);
2750 break;
2751 }
2752 continue;
2753 }
2754
2755 if (mem_high || swap_high) {
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765 current->memcg_nr_pages_over_high += batch;
2766 set_notify_resume(current);
2767 break;
2768 }
2769 } while ((memcg = parent_mem_cgroup(memcg)));
2770
2771 return 0;
2772}
2773
2774static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2775 unsigned int nr_pages)
2776{
2777 if (mem_cgroup_is_root(memcg))
2778 return 0;
2779
2780 return try_charge_memcg(memcg, gfp_mask, nr_pages);
2781}
2782
2783#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
2784static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2785{
2786 if (mem_cgroup_is_root(memcg))
2787 return;
2788
2789 page_counter_uncharge(&memcg->memory, nr_pages);
2790 if (do_memsw_account())
2791 page_counter_uncharge(&memcg->memsw, nr_pages);
2792}
2793#endif
2794
2795static void commit_charge(struct page *page, struct mem_cgroup *memcg)
2796{
2797 VM_BUG_ON_PAGE(page_memcg(page), page);
2798
2799
2800
2801
2802
2803
2804
2805
2806 page->memcg_data = (unsigned long)memcg;
2807}
2808
2809static struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)
2810{
2811 struct mem_cgroup *memcg;
2812
2813 rcu_read_lock();
2814retry:
2815 memcg = obj_cgroup_memcg(objcg);
2816 if (unlikely(!css_tryget(&memcg->css)))
2817 goto retry;
2818 rcu_read_unlock();
2819
2820 return memcg;
2821}
2822
2823#ifdef CONFIG_MEMCG_KMEM
2824
2825
2826
2827
2828
2829#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT)
2830
2831int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
2832 gfp_t gfp, bool new_page)
2833{
2834 unsigned int objects = objs_per_slab_page(s, page);
2835 unsigned long memcg_data;
2836 void *vec;
2837
2838 gfp &= ~OBJCGS_CLEAR_MASK;
2839 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
2840 page_to_nid(page));
2841 if (!vec)
2842 return -ENOMEM;
2843
2844 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
2845 if (new_page) {
2846
2847
2848
2849
2850
2851 page->memcg_data = memcg_data;
2852 } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
2853
2854
2855
2856
2857
2858 kfree(vec);
2859 return 0;
2860 }
2861
2862 kmemleak_not_leak(vec);
2863 return 0;
2864}
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878struct mem_cgroup *mem_cgroup_from_obj(void *p)
2879{
2880 struct page *page;
2881
2882 if (mem_cgroup_disabled())
2883 return NULL;
2884
2885 page = virt_to_head_page(p);
2886
2887
2888
2889
2890
2891
2892 if (page_objcgs_check(page)) {
2893 struct obj_cgroup *objcg;
2894 unsigned int off;
2895
2896 off = obj_to_index(page->slab_cache, page, p);
2897 objcg = page_objcgs(page)[off];
2898 if (objcg)
2899 return obj_cgroup_memcg(objcg);
2900
2901 return NULL;
2902 }
2903
2904
2905
2906
2907
2908
2909
2910
2911 return page_memcg_check(page);
2912}
2913
2914__always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
2915{
2916 struct obj_cgroup *objcg = NULL;
2917 struct mem_cgroup *memcg;
2918
2919 if (memcg_kmem_bypass())
2920 return NULL;
2921
2922 rcu_read_lock();
2923 if (unlikely(active_memcg()))
2924 memcg = active_memcg();
2925 else
2926 memcg = mem_cgroup_from_task(current);
2927
2928 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
2929 objcg = rcu_dereference(memcg->objcg);
2930 if (objcg && obj_cgroup_tryget(objcg))
2931 break;
2932 objcg = NULL;
2933 }
2934 rcu_read_unlock();
2935
2936 return objcg;
2937}
2938
2939static int memcg_alloc_cache_id(void)
2940{
2941 int id, size;
2942 int err;
2943
2944 id = ida_simple_get(&memcg_cache_ida,
2945 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2946 if (id < 0)
2947 return id;
2948
2949 if (id < memcg_nr_cache_ids)
2950 return id;
2951
2952
2953
2954
2955
2956 down_write(&memcg_cache_ids_sem);
2957
2958 size = 2 * (id + 1);
2959 if (size < MEMCG_CACHES_MIN_SIZE)
2960 size = MEMCG_CACHES_MIN_SIZE;
2961 else if (size > MEMCG_CACHES_MAX_SIZE)
2962 size = MEMCG_CACHES_MAX_SIZE;
2963
2964 err = memcg_update_all_list_lrus(size);
2965 if (!err)
2966 memcg_nr_cache_ids = size;
2967
2968 up_write(&memcg_cache_ids_sem);
2969
2970 if (err) {
2971 ida_simple_remove(&memcg_cache_ida, id);
2972 return err;
2973 }
2974 return id;
2975}
2976
2977static void memcg_free_cache_id(int id)
2978{
2979 ida_simple_remove(&memcg_cache_ida, id);
2980}
2981
2982
2983
2984
2985
2986
2987static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
2988 unsigned int nr_pages)
2989{
2990 struct mem_cgroup *memcg;
2991
2992 memcg = get_mem_cgroup_from_objcg(objcg);
2993
2994 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2995 page_counter_uncharge(&memcg->kmem, nr_pages);
2996 refill_stock(memcg, nr_pages);
2997
2998 css_put(&memcg->css);
2999}
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
3010 unsigned int nr_pages)
3011{
3012 struct page_counter *counter;
3013 struct mem_cgroup *memcg;
3014 int ret;
3015
3016 memcg = get_mem_cgroup_from_objcg(objcg);
3017
3018 ret = try_charge_memcg(memcg, gfp, nr_pages);
3019 if (ret)
3020 goto out;
3021
3022 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
3023 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
3024
3025
3026
3027
3028
3029
3030 if (gfp & __GFP_NOFAIL) {
3031 page_counter_charge(&memcg->kmem, nr_pages);
3032 goto out;
3033 }
3034 cancel_charge(memcg, nr_pages);
3035 ret = -ENOMEM;
3036 }
3037out:
3038 css_put(&memcg->css);
3039
3040 return ret;
3041}
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
3052{
3053 struct obj_cgroup *objcg;
3054 int ret = 0;
3055
3056 objcg = get_obj_cgroup_from_current();
3057 if (objcg) {
3058 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
3059 if (!ret) {
3060 page->memcg_data = (unsigned long)objcg |
3061 MEMCG_DATA_KMEM;
3062 return 0;
3063 }
3064 obj_cgroup_put(objcg);
3065 }
3066 return ret;
3067}
3068
3069
3070
3071
3072
3073
3074void __memcg_kmem_uncharge_page(struct page *page, int order)
3075{
3076 struct obj_cgroup *objcg;
3077 unsigned int nr_pages = 1 << order;
3078
3079 if (!PageMemcgKmem(page))
3080 return;
3081
3082 objcg = __page_objcg(page);
3083 obj_cgroup_uncharge_pages(objcg, nr_pages);
3084 page->memcg_data = 0;
3085 obj_cgroup_put(objcg);
3086}
3087
3088void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
3089 enum node_stat_item idx, int nr)
3090{
3091 unsigned long flags;
3092 struct obj_stock *stock = get_obj_stock(&flags);
3093 int *bytes;
3094
3095
3096
3097
3098
3099
3100 if (stock->cached_objcg != objcg) {
3101 drain_obj_stock(stock);
3102 obj_cgroup_get(objcg);
3103 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3104 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3105 stock->cached_objcg = objcg;
3106 stock->cached_pgdat = pgdat;
3107 } else if (stock->cached_pgdat != pgdat) {
3108
3109 struct pglist_data *oldpg = stock->cached_pgdat;
3110
3111 if (stock->nr_slab_reclaimable_b) {
3112 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B,
3113 stock->nr_slab_reclaimable_b);
3114 stock->nr_slab_reclaimable_b = 0;
3115 }
3116 if (stock->nr_slab_unreclaimable_b) {
3117 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B,
3118 stock->nr_slab_unreclaimable_b);
3119 stock->nr_slab_unreclaimable_b = 0;
3120 }
3121 stock->cached_pgdat = pgdat;
3122 }
3123
3124 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b
3125 : &stock->nr_slab_unreclaimable_b;
3126
3127
3128
3129
3130 if (!*bytes) {
3131 *bytes = nr;
3132 nr = 0;
3133 } else {
3134 *bytes += nr;
3135 if (abs(*bytes) > PAGE_SIZE) {
3136 nr = *bytes;
3137 *bytes = 0;
3138 } else {
3139 nr = 0;
3140 }
3141 }
3142 if (nr)
3143 mod_objcg_mlstate(objcg, pgdat, idx, nr);
3144
3145 put_obj_stock(flags);
3146}
3147
3148static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3149{
3150 unsigned long flags;
3151 struct obj_stock *stock = get_obj_stock(&flags);
3152 bool ret = false;
3153
3154 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
3155 stock->nr_bytes -= nr_bytes;
3156 ret = true;
3157 }
3158
3159 put_obj_stock(flags);
3160
3161 return ret;
3162}
3163
3164static void drain_obj_stock(struct obj_stock *stock)
3165{
3166 struct obj_cgroup *old = stock->cached_objcg;
3167
3168 if (!old)
3169 return;
3170
3171 if (stock->nr_bytes) {
3172 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3173 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
3174
3175 if (nr_pages)
3176 obj_cgroup_uncharge_pages(old, nr_pages);
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188 atomic_add(nr_bytes, &old->nr_charged_bytes);
3189 stock->nr_bytes = 0;
3190 }
3191
3192
3193
3194
3195 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) {
3196 if (stock->nr_slab_reclaimable_b) {
3197 mod_objcg_mlstate(old, stock->cached_pgdat,
3198 NR_SLAB_RECLAIMABLE_B,
3199 stock->nr_slab_reclaimable_b);
3200 stock->nr_slab_reclaimable_b = 0;
3201 }
3202 if (stock->nr_slab_unreclaimable_b) {
3203 mod_objcg_mlstate(old, stock->cached_pgdat,
3204 NR_SLAB_UNRECLAIMABLE_B,
3205 stock->nr_slab_unreclaimable_b);
3206 stock->nr_slab_unreclaimable_b = 0;
3207 }
3208 stock->cached_pgdat = NULL;
3209 }
3210
3211 obj_cgroup_put(old);
3212 stock->cached_objcg = NULL;
3213}
3214
3215static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3216 struct mem_cgroup *root_memcg)
3217{
3218 struct mem_cgroup *memcg;
3219
3220 if (in_task() && stock->task_obj.cached_objcg) {
3221 memcg = obj_cgroup_memcg(stock->task_obj.cached_objcg);
3222 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3223 return true;
3224 }
3225 if (stock->irq_obj.cached_objcg) {
3226 memcg = obj_cgroup_memcg(stock->irq_obj.cached_objcg);
3227 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3228 return true;
3229 }
3230
3231 return false;
3232}
3233
3234static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,
3235 bool allow_uncharge)
3236{
3237 unsigned long flags;
3238 struct obj_stock *stock = get_obj_stock(&flags);
3239 unsigned int nr_pages = 0;
3240
3241 if (stock->cached_objcg != objcg) {
3242 drain_obj_stock(stock);
3243 obj_cgroup_get(objcg);
3244 stock->cached_objcg = objcg;
3245 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3246 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3247 allow_uncharge = true;
3248 }
3249 stock->nr_bytes += nr_bytes;
3250
3251 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) {
3252 nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3253 stock->nr_bytes &= (PAGE_SIZE - 1);
3254 }
3255
3256 put_obj_stock(flags);
3257
3258 if (nr_pages)
3259 obj_cgroup_uncharge_pages(objcg, nr_pages);
3260}
3261
3262int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3263{
3264 unsigned int nr_pages, nr_bytes;
3265 int ret;
3266
3267 if (consume_obj_stock(objcg, size))
3268 return 0;
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293 nr_pages = size >> PAGE_SHIFT;
3294 nr_bytes = size & (PAGE_SIZE - 1);
3295
3296 if (nr_bytes)
3297 nr_pages += 1;
3298
3299 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);
3300 if (!ret && nr_bytes)
3301 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);
3302
3303 return ret;
3304}
3305
3306void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3307{
3308 refill_obj_stock(objcg, size, true);
3309}
3310
3311#endif
3312
3313
3314
3315
3316void split_page_memcg(struct page *head, unsigned int nr)
3317{
3318 struct mem_cgroup *memcg = page_memcg(head);
3319 int i;
3320
3321 if (mem_cgroup_disabled() || !memcg)
3322 return;
3323
3324 for (i = 1; i < nr; i++)
3325 head[i].memcg_data = head->memcg_data;
3326
3327 if (PageMemcgKmem(head))
3328 obj_cgroup_get_many(__page_objcg(head), nr - 1);
3329 else
3330 css_get_many(&memcg->css, nr - 1);
3331}
3332
3333#ifdef CONFIG_MEMCG_SWAP
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348static int mem_cgroup_move_swap_account(swp_entry_t entry,
3349 struct mem_cgroup *from, struct mem_cgroup *to)
3350{
3351 unsigned short old_id, new_id;
3352
3353 old_id = mem_cgroup_id(from);
3354 new_id = mem_cgroup_id(to);
3355
3356 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
3357 mod_memcg_state(from, MEMCG_SWAP, -1);
3358 mod_memcg_state(to, MEMCG_SWAP, 1);
3359 return 0;
3360 }
3361 return -EINVAL;
3362}
3363#else
3364static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
3365 struct mem_cgroup *from, struct mem_cgroup *to)
3366{
3367 return -EINVAL;
3368}
3369#endif
3370
3371static DEFINE_MUTEX(memcg_max_mutex);
3372
3373static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3374 unsigned long max, bool memsw)
3375{
3376 bool enlarge = false;
3377 bool drained = false;
3378 int ret;
3379 bool limits_invariant;
3380 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
3381
3382 do {
3383 if (signal_pending(current)) {
3384 ret = -EINTR;
3385 break;
3386 }
3387
3388 mutex_lock(&memcg_max_mutex);
3389
3390
3391
3392
3393 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
3394 max <= memcg->memsw.max;
3395 if (!limits_invariant) {
3396 mutex_unlock(&memcg_max_mutex);
3397 ret = -EINVAL;
3398 break;
3399 }
3400 if (max > counter->max)
3401 enlarge = true;
3402 ret = page_counter_set_max(counter, max);
3403 mutex_unlock(&memcg_max_mutex);
3404
3405 if (!ret)
3406 break;
3407
3408 if (!drained) {
3409 drain_all_stock(memcg);
3410 drained = true;
3411 continue;
3412 }
3413
3414 if (!try_to_free_mem_cgroup_pages(memcg, 1,
3415 GFP_KERNEL, !memsw)) {
3416 ret = -EBUSY;
3417 break;
3418 }
3419 } while (true);
3420
3421 if (!ret && enlarge)
3422 memcg_oom_recover(memcg);
3423
3424 return ret;
3425}
3426
3427unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
3428 gfp_t gfp_mask,
3429 unsigned long *total_scanned)
3430{
3431 unsigned long nr_reclaimed = 0;
3432 struct mem_cgroup_per_node *mz, *next_mz = NULL;
3433 unsigned long reclaimed;
3434 int loop = 0;
3435 struct mem_cgroup_tree_per_node *mctz;
3436 unsigned long excess;
3437 unsigned long nr_scanned;
3438
3439 if (order > 0)
3440 return 0;
3441
3442 mctz = soft_limit_tree_node(pgdat->node_id);
3443
3444
3445
3446
3447
3448
3449 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
3450 return 0;
3451
3452
3453
3454
3455
3456
3457 do {
3458 if (next_mz)
3459 mz = next_mz;
3460 else
3461 mz = mem_cgroup_largest_soft_limit_node(mctz);
3462 if (!mz)
3463 break;
3464
3465 nr_scanned = 0;
3466 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3467 gfp_mask, &nr_scanned);
3468 nr_reclaimed += reclaimed;
3469 *total_scanned += nr_scanned;
3470 spin_lock_irq(&mctz->lock);
3471 __mem_cgroup_remove_exceeded(mz, mctz);
3472
3473
3474
3475
3476
3477 next_mz = NULL;
3478 if (!reclaimed)
3479 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3480
3481 excess = soft_limit_excess(mz->memcg);
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491 __mem_cgroup_insert_exceeded(mz, mctz, excess);
3492 spin_unlock_irq(&mctz->lock);
3493 css_put(&mz->memcg->css);
3494 loop++;
3495
3496
3497
3498
3499
3500 if (!nr_reclaimed &&
3501 (next_mz == NULL ||
3502 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3503 break;
3504 } while (!nr_reclaimed);
3505 if (next_mz)
3506 css_put(&next_mz->memcg->css);
3507 return nr_reclaimed;
3508}
3509
3510
3511
3512
3513
3514
3515static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3516{
3517 int nr_retries = MAX_RECLAIM_RETRIES;
3518
3519
3520 lru_add_drain_all();
3521
3522 drain_all_stock(memcg);
3523
3524
3525 while (nr_retries && page_counter_read(&memcg->memory)) {
3526 int progress;
3527
3528 if (signal_pending(current))
3529 return -EINTR;
3530
3531 progress = try_to_free_mem_cgroup_pages(memcg, 1,
3532 GFP_KERNEL, true);
3533 if (!progress) {
3534 nr_retries--;
3535
3536 congestion_wait(BLK_RW_ASYNC, HZ/10);
3537 }
3538
3539 }
3540
3541 return 0;
3542}
3543
3544static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
3545 char *buf, size_t nbytes,
3546 loff_t off)
3547{
3548 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3549
3550 if (mem_cgroup_is_root(memcg))
3551 return -EINVAL;
3552 return mem_cgroup_force_empty(memcg) ?: nbytes;
3553}
3554
3555static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3556 struct cftype *cft)
3557{
3558 return 1;
3559}
3560
3561static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3562 struct cftype *cft, u64 val)
3563{
3564 if (val == 1)
3565 return 0;
3566
3567 pr_warn_once("Non-hierarchical mode is deprecated. "
3568 "Please report your usecase to linux-mm@kvack.org if you "
3569 "depend on this functionality.\n");
3570
3571 return -EINVAL;
3572}
3573
3574static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3575{
3576 unsigned long val;
3577
3578 if (mem_cgroup_is_root(memcg)) {
3579
3580 cgroup_rstat_flush_irqsafe(memcg->css.cgroup);
3581 val = memcg_page_state(memcg, NR_FILE_PAGES) +
3582 memcg_page_state(memcg, NR_ANON_MAPPED);
3583 if (swap)
3584 val += memcg_page_state(memcg, MEMCG_SWAP);
3585 } else {
3586 if (!swap)
3587 val = page_counter_read(&memcg->memory);
3588 else
3589 val = page_counter_read(&memcg->memsw);
3590 }
3591 return val;
3592}
3593
3594enum {
3595 RES_USAGE,
3596 RES_LIMIT,
3597 RES_MAX_USAGE,
3598 RES_FAILCNT,
3599 RES_SOFT_LIMIT,
3600};
3601
3602static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3603 struct cftype *cft)
3604{
3605 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3606 struct page_counter *counter;
3607
3608 switch (MEMFILE_TYPE(cft->private)) {
3609 case _MEM:
3610 counter = &memcg->memory;
3611 break;
3612 case _MEMSWAP:
3613 counter = &memcg->memsw;
3614 break;
3615 case _KMEM:
3616 counter = &memcg->kmem;
3617 break;
3618 case _TCP:
3619 counter = &memcg->tcpmem;
3620 break;
3621 default:
3622 BUG();
3623 }
3624
3625 switch (MEMFILE_ATTR(cft->private)) {
3626 case RES_USAGE:
3627 if (counter == &memcg->memory)
3628 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3629 if (counter == &memcg->memsw)
3630 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3631 return (u64)page_counter_read(counter) * PAGE_SIZE;
3632 case RES_LIMIT:
3633 return (u64)counter->max * PAGE_SIZE;
3634 case RES_MAX_USAGE:
3635 return (u64)counter->watermark * PAGE_SIZE;
3636 case RES_FAILCNT:
3637 return counter->failcnt;
3638 case RES_SOFT_LIMIT:
3639 return (u64)memcg->soft_limit * PAGE_SIZE;
3640 default:
3641 BUG();
3642 }
3643}
3644
3645#ifdef CONFIG_MEMCG_KMEM
3646static int memcg_online_kmem(struct mem_cgroup *memcg)
3647{
3648 struct obj_cgroup *objcg;
3649 int memcg_id;
3650
3651 if (cgroup_memory_nokmem)
3652 return 0;
3653
3654 BUG_ON(memcg->kmemcg_id >= 0);
3655 BUG_ON(memcg->kmem_state);
3656
3657 memcg_id = memcg_alloc_cache_id();
3658 if (memcg_id < 0)
3659 return memcg_id;
3660
3661 objcg = obj_cgroup_alloc();
3662 if (!objcg) {
3663 memcg_free_cache_id(memcg_id);
3664 return -ENOMEM;
3665 }
3666 objcg->memcg = memcg;
3667 rcu_assign_pointer(memcg->objcg, objcg);
3668
3669 static_branch_enable(&memcg_kmem_enabled_key);
3670
3671 memcg->kmemcg_id = memcg_id;
3672 memcg->kmem_state = KMEM_ONLINE;
3673
3674 return 0;
3675}
3676
3677static void memcg_offline_kmem(struct mem_cgroup *memcg)
3678{
3679 struct cgroup_subsys_state *css;
3680 struct mem_cgroup *parent, *child;
3681 int kmemcg_id;
3682
3683 if (memcg->kmem_state != KMEM_ONLINE)
3684 return;
3685
3686 memcg->kmem_state = KMEM_ALLOCATED;
3687
3688 parent = parent_mem_cgroup(memcg);
3689 if (!parent)
3690 parent = root_mem_cgroup;
3691
3692 memcg_reparent_objcgs(memcg, parent);
3693
3694 kmemcg_id = memcg->kmemcg_id;
3695 BUG_ON(kmemcg_id < 0);
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705 rcu_read_lock();
3706 css_for_each_descendant_pre(css, &memcg->css) {
3707 child = mem_cgroup_from_css(css);
3708 BUG_ON(child->kmemcg_id != kmemcg_id);
3709 child->kmemcg_id = parent->kmemcg_id;
3710 }
3711 rcu_read_unlock();
3712
3713 memcg_drain_all_list_lrus(kmemcg_id, parent);
3714
3715 memcg_free_cache_id(kmemcg_id);
3716}
3717
3718static void memcg_free_kmem(struct mem_cgroup *memcg)
3719{
3720
3721 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3722 memcg_offline_kmem(memcg);
3723}
3724#else
3725static int memcg_online_kmem(struct mem_cgroup *memcg)
3726{
3727 return 0;
3728}
3729static void memcg_offline_kmem(struct mem_cgroup *memcg)
3730{
3731}
3732static void memcg_free_kmem(struct mem_cgroup *memcg)
3733{
3734}
3735#endif
3736
3737static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3738 unsigned long max)
3739{
3740 int ret;
3741
3742 mutex_lock(&memcg_max_mutex);
3743 ret = page_counter_set_max(&memcg->kmem, max);
3744 mutex_unlock(&memcg_max_mutex);
3745 return ret;
3746}
3747
3748static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3749{
3750 int ret;
3751
3752 mutex_lock(&memcg_max_mutex);
3753
3754 ret = page_counter_set_max(&memcg->tcpmem, max);
3755 if (ret)
3756 goto out;
3757
3758 if (!memcg->tcpmem_active) {
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775 static_branch_inc(&memcg_sockets_enabled_key);
3776 memcg->tcpmem_active = true;
3777 }
3778out:
3779 mutex_unlock(&memcg_max_mutex);
3780 return ret;
3781}
3782
3783
3784
3785
3786
3787static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3788 char *buf, size_t nbytes, loff_t off)
3789{
3790 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3791 unsigned long nr_pages;
3792 int ret;
3793
3794 buf = strstrip(buf);
3795 ret = page_counter_memparse(buf, "-1", &nr_pages);
3796 if (ret)
3797 return ret;
3798
3799 switch (MEMFILE_ATTR(of_cft(of)->private)) {
3800 case RES_LIMIT:
3801 if (mem_cgroup_is_root(memcg)) {
3802 ret = -EINVAL;
3803 break;
3804 }
3805 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3806 case _MEM:
3807 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3808 break;
3809 case _MEMSWAP:
3810 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3811 break;
3812 case _KMEM:
3813 pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
3814 "Please report your usecase to linux-mm@kvack.org if you "
3815 "depend on this functionality.\n");
3816 ret = memcg_update_kmem_max(memcg, nr_pages);
3817 break;
3818 case _TCP:
3819 ret = memcg_update_tcp_max(memcg, nr_pages);
3820 break;
3821 }
3822 break;
3823 case RES_SOFT_LIMIT:
3824 memcg->soft_limit = nr_pages;
3825 ret = 0;
3826 break;
3827 }
3828 return ret ?: nbytes;
3829}
3830
3831static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3832 size_t nbytes, loff_t off)
3833{
3834 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3835 struct page_counter *counter;
3836
3837 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3838 case _MEM:
3839 counter = &memcg->memory;
3840 break;
3841 case _MEMSWAP:
3842 counter = &memcg->memsw;
3843 break;
3844 case _KMEM:
3845 counter = &memcg->kmem;
3846 break;
3847 case _TCP:
3848 counter = &memcg->tcpmem;
3849 break;
3850 default:
3851 BUG();
3852 }
3853
3854 switch (MEMFILE_ATTR(of_cft(of)->private)) {
3855 case RES_MAX_USAGE:
3856 page_counter_reset_watermark(counter);
3857 break;
3858 case RES_FAILCNT:
3859 counter->failcnt = 0;
3860 break;
3861 default:
3862 BUG();
3863 }
3864
3865 return nbytes;
3866}
3867
3868static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3869 struct cftype *cft)
3870{
3871 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3872}
3873
3874#ifdef CONFIG_MMU
3875static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3876 struct cftype *cft, u64 val)
3877{
3878 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3879
3880 if (val & ~MOVE_MASK)
3881 return -EINVAL;
3882
3883
3884
3885
3886
3887
3888
3889 memcg->move_charge_at_immigrate = val;
3890 return 0;
3891}
3892#else
3893static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3894 struct cftype *cft, u64 val)
3895{
3896 return -ENOSYS;
3897}
3898#endif
3899
3900#ifdef CONFIG_NUMA
3901
3902#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
3903#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
3904#define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
3905
3906static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
3907 int nid, unsigned int lru_mask, bool tree)
3908{
3909 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
3910 unsigned long nr = 0;
3911 enum lru_list lru;
3912
3913 VM_BUG_ON((unsigned)nid >= nr_node_ids);
3914
3915 for_each_lru(lru) {
3916 if (!(BIT(lru) & lru_mask))
3917 continue;
3918 if (tree)
3919 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
3920 else
3921 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
3922 }
3923 return nr;
3924}
3925
3926static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
3927 unsigned int lru_mask,
3928 bool tree)
3929{
3930 unsigned long nr = 0;
3931 enum lru_list lru;
3932
3933 for_each_lru(lru) {
3934 if (!(BIT(lru) & lru_mask))
3935 continue;
3936 if (tree)
3937 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
3938 else
3939 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
3940 }
3941 return nr;
3942}
3943
3944static int memcg_numa_stat_show(struct seq_file *m, void *v)
3945{
3946 struct numa_stat {
3947 const char *name;
3948 unsigned int lru_mask;
3949 };
3950
3951 static const struct numa_stat stats[] = {
3952 { "total", LRU_ALL },
3953 { "file", LRU_ALL_FILE },
3954 { "anon", LRU_ALL_ANON },
3955 { "unevictable", BIT(LRU_UNEVICTABLE) },
3956 };
3957 const struct numa_stat *stat;
3958 int nid;
3959 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
3960
3961 cgroup_rstat_flush(memcg->css.cgroup);
3962
3963 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3964 seq_printf(m, "%s=%lu", stat->name,
3965 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3966 false));
3967 for_each_node_state(nid, N_MEMORY)
3968 seq_printf(m, " N%d=%lu", nid,
3969 mem_cgroup_node_nr_lru_pages(memcg, nid,
3970 stat->lru_mask, false));
3971 seq_putc(m, '\n');
3972 }
3973
3974 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3975
3976 seq_printf(m, "hierarchical_%s=%lu", stat->name,
3977 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3978 true));
3979 for_each_node_state(nid, N_MEMORY)
3980 seq_printf(m, " N%d=%lu", nid,
3981 mem_cgroup_node_nr_lru_pages(memcg, nid,
3982 stat->lru_mask, true));
3983 seq_putc(m, '\n');
3984 }
3985
3986 return 0;
3987}
3988#endif
3989
3990static const unsigned int memcg1_stats[] = {
3991 NR_FILE_PAGES,
3992 NR_ANON_MAPPED,
3993#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3994 NR_ANON_THPS,
3995#endif
3996 NR_SHMEM,
3997 NR_FILE_MAPPED,
3998 NR_FILE_DIRTY,
3999 NR_WRITEBACK,
4000 MEMCG_SWAP,
4001};
4002
4003static const char *const memcg1_stat_names[] = {
4004 "cache",
4005 "rss",
4006#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4007 "rss_huge",
4008#endif
4009 "shmem",
4010 "mapped_file",
4011 "dirty",
4012 "writeback",
4013 "swap",
4014};
4015
4016
4017static const unsigned int memcg1_events[] = {
4018 PGPGIN,
4019 PGPGOUT,
4020 PGFAULT,
4021 PGMAJFAULT,
4022};
4023
4024static int memcg_stat_show(struct seq_file *m, void *v)
4025{
4026 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4027 unsigned long memory, memsw;
4028 struct mem_cgroup *mi;
4029 unsigned int i;
4030
4031 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
4032
4033 cgroup_rstat_flush(memcg->css.cgroup);
4034
4035 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
4036 unsigned long nr;
4037
4038 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
4039 continue;
4040 nr = memcg_page_state_local(memcg, memcg1_stats[i]);
4041 seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
4042 }
4043
4044 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4045 seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
4046 memcg_events_local(memcg, memcg1_events[i]));
4047
4048 for (i = 0; i < NR_LRU_LISTS; i++)
4049 seq_printf(m, "%s %lu\n", lru_list_name(i),
4050 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
4051 PAGE_SIZE);
4052
4053
4054 memory = memsw = PAGE_COUNTER_MAX;
4055 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
4056 memory = min(memory, READ_ONCE(mi->memory.max));
4057 memsw = min(memsw, READ_ONCE(mi->memsw.max));
4058 }
4059 seq_printf(m, "hierarchical_memory_limit %llu\n",
4060 (u64)memory * PAGE_SIZE);
4061 if (do_memsw_account())
4062 seq_printf(m, "hierarchical_memsw_limit %llu\n",
4063 (u64)memsw * PAGE_SIZE);
4064
4065 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
4066 unsigned long nr;
4067
4068 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
4069 continue;
4070 nr = memcg_page_state(memcg, memcg1_stats[i]);
4071 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
4072 (u64)nr * PAGE_SIZE);
4073 }
4074
4075 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4076 seq_printf(m, "total_%s %llu\n",
4077 vm_event_name(memcg1_events[i]),
4078 (u64)memcg_events(memcg, memcg1_events[i]));
4079
4080 for (i = 0; i < NR_LRU_LISTS; i++)
4081 seq_printf(m, "total_%s %llu\n", lru_list_name(i),
4082 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4083 PAGE_SIZE);
4084
4085#ifdef CONFIG_DEBUG_VM
4086 {
4087 pg_data_t *pgdat;
4088 struct mem_cgroup_per_node *mz;
4089 unsigned long anon_cost = 0;
4090 unsigned long file_cost = 0;
4091
4092 for_each_online_pgdat(pgdat) {
4093 mz = memcg->nodeinfo[pgdat->node_id];
4094
4095 anon_cost += mz->lruvec.anon_cost;
4096 file_cost += mz->lruvec.file_cost;
4097 }
4098 seq_printf(m, "anon_cost %lu\n", anon_cost);
4099 seq_printf(m, "file_cost %lu\n", file_cost);
4100 }
4101#endif
4102
4103 return 0;
4104}
4105
4106static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
4107 struct cftype *cft)
4108{
4109 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4110
4111 return mem_cgroup_swappiness(memcg);
4112}
4113
4114static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
4115 struct cftype *cft, u64 val)
4116{
4117 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4118
4119 if (val > 100)
4120 return -EINVAL;
4121
4122 if (!mem_cgroup_is_root(memcg))
4123 memcg->swappiness = val;
4124 else
4125 vm_swappiness = val;
4126
4127 return 0;
4128}
4129
4130static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4131{
4132 struct mem_cgroup_threshold_ary *t;
4133 unsigned long usage;
4134 int i;
4135
4136 rcu_read_lock();
4137 if (!swap)
4138 t = rcu_dereference(memcg->thresholds.primary);
4139 else
4140 t = rcu_dereference(memcg->memsw_thresholds.primary);
4141
4142 if (!t)
4143 goto unlock;
4144
4145 usage = mem_cgroup_usage(memcg, swap);
4146
4147
4148
4149
4150
4151
4152 i = t->current_threshold;
4153
4154
4155
4156
4157
4158
4159
4160 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
4161 eventfd_signal(t->entries[i].eventfd, 1);
4162
4163
4164 i++;
4165
4166
4167
4168
4169
4170
4171
4172 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
4173 eventfd_signal(t->entries[i].eventfd, 1);
4174
4175
4176 t->current_threshold = i - 1;
4177unlock:
4178 rcu_read_unlock();
4179}
4180
4181static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4182{
4183 while (memcg) {
4184 __mem_cgroup_threshold(memcg, false);
4185 if (do_memsw_account())
4186 __mem_cgroup_threshold(memcg, true);
4187
4188 memcg = parent_mem_cgroup(memcg);
4189 }
4190}
4191
4192static int compare_thresholds(const void *a, const void *b)
4193{
4194 const struct mem_cgroup_threshold *_a = a;
4195 const struct mem_cgroup_threshold *_b = b;
4196
4197 if (_a->threshold > _b->threshold)
4198 return 1;
4199
4200 if (_a->threshold < _b->threshold)
4201 return -1;
4202
4203 return 0;
4204}
4205
4206static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
4207{
4208 struct mem_cgroup_eventfd_list *ev;
4209
4210 spin_lock(&memcg_oom_lock);
4211
4212 list_for_each_entry(ev, &memcg->oom_notify, list)
4213 eventfd_signal(ev->eventfd, 1);
4214
4215 spin_unlock(&memcg_oom_lock);
4216 return 0;
4217}
4218
4219static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
4220{
4221 struct mem_cgroup *iter;
4222
4223 for_each_mem_cgroup_tree(iter, memcg)
4224 mem_cgroup_oom_notify_cb(iter);
4225}
4226
4227static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4228 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
4229{
4230 struct mem_cgroup_thresholds *thresholds;
4231 struct mem_cgroup_threshold_ary *new;
4232 unsigned long threshold;
4233 unsigned long usage;
4234 int i, size, ret;
4235
4236 ret = page_counter_memparse(args, "-1", &threshold);
4237 if (ret)
4238 return ret;
4239
4240 mutex_lock(&memcg->thresholds_lock);
4241
4242 if (type == _MEM) {
4243 thresholds = &memcg->thresholds;
4244 usage = mem_cgroup_usage(memcg, false);
4245 } else if (type == _MEMSWAP) {
4246 thresholds = &memcg->memsw_thresholds;
4247 usage = mem_cgroup_usage(memcg, true);
4248 } else
4249 BUG();
4250
4251
4252 if (thresholds->primary)
4253 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4254
4255 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
4256
4257
4258 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
4259 if (!new) {
4260 ret = -ENOMEM;
4261 goto unlock;
4262 }
4263 new->size = size;
4264
4265
4266 if (thresholds->primary)
4267 memcpy(new->entries, thresholds->primary->entries,
4268 flex_array_size(new, entries, size - 1));
4269
4270
4271 new->entries[size - 1].eventfd = eventfd;
4272 new->entries[size - 1].threshold = threshold;
4273
4274
4275 sort(new->entries, size, sizeof(*new->entries),
4276 compare_thresholds, NULL);
4277
4278
4279 new->current_threshold = -1;
4280 for (i = 0; i < size; i++) {
4281 if (new->entries[i].threshold <= usage) {
4282
4283
4284
4285
4286
4287 ++new->current_threshold;
4288 } else
4289 break;
4290 }
4291
4292
4293 kfree(thresholds->spare);
4294 thresholds->spare = thresholds->primary;
4295
4296 rcu_assign_pointer(thresholds->primary, new);
4297
4298
4299 synchronize_rcu();
4300
4301unlock:
4302 mutex_unlock(&memcg->thresholds_lock);
4303
4304 return ret;
4305}
4306
4307static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4308 struct eventfd_ctx *eventfd, const char *args)
4309{
4310 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
4311}
4312
4313static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4314 struct eventfd_ctx *eventfd, const char *args)
4315{
4316 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
4317}
4318
4319static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4320 struct eventfd_ctx *eventfd, enum res_type type)
4321{
4322 struct mem_cgroup_thresholds *thresholds;
4323 struct mem_cgroup_threshold_ary *new;
4324 unsigned long usage;
4325 int i, j, size, entries;
4326
4327 mutex_lock(&memcg->thresholds_lock);
4328
4329 if (type == _MEM) {
4330 thresholds = &memcg->thresholds;
4331 usage = mem_cgroup_usage(memcg, false);
4332 } else if (type == _MEMSWAP) {
4333 thresholds = &memcg->memsw_thresholds;
4334 usage = mem_cgroup_usage(memcg, true);
4335 } else
4336 BUG();
4337
4338 if (!thresholds->primary)
4339 goto unlock;
4340
4341
4342 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4343
4344
4345 size = entries = 0;
4346 for (i = 0; i < thresholds->primary->size; i++) {
4347 if (thresholds->primary->entries[i].eventfd != eventfd)
4348 size++;
4349 else
4350 entries++;
4351 }
4352
4353 new = thresholds->spare;
4354
4355
4356 if (!entries)
4357 goto unlock;
4358
4359
4360 if (!size) {
4361 kfree(new);
4362 new = NULL;
4363 goto swap_buffers;
4364 }
4365
4366 new->size = size;
4367
4368
4369 new->current_threshold = -1;
4370 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4371 if (thresholds->primary->entries[i].eventfd == eventfd)
4372 continue;
4373
4374 new->entries[j] = thresholds->primary->entries[i];
4375 if (new->entries[j].threshold <= usage) {
4376
4377
4378
4379
4380
4381 ++new->current_threshold;
4382 }
4383 j++;
4384 }
4385
4386swap_buffers:
4387
4388 thresholds->spare = thresholds->primary;
4389
4390 rcu_assign_pointer(thresholds->primary, new);
4391
4392
4393 synchronize_rcu();
4394
4395
4396 if (!new) {
4397 kfree(thresholds->spare);
4398 thresholds->spare = NULL;
4399 }
4400unlock:
4401 mutex_unlock(&memcg->thresholds_lock);
4402}
4403
4404static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4405 struct eventfd_ctx *eventfd)
4406{
4407 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
4408}
4409
4410static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4411 struct eventfd_ctx *eventfd)
4412{
4413 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
4414}
4415
4416static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4417 struct eventfd_ctx *eventfd, const char *args)
4418{
4419 struct mem_cgroup_eventfd_list *event;
4420
4421 event = kmalloc(sizeof(*event), GFP_KERNEL);
4422 if (!event)
4423 return -ENOMEM;
4424
4425 spin_lock(&memcg_oom_lock);
4426
4427 event->eventfd = eventfd;
4428 list_add(&event->list, &memcg->oom_notify);
4429
4430
4431 if (memcg->under_oom)
4432 eventfd_signal(eventfd, 1);
4433 spin_unlock(&memcg_oom_lock);
4434
4435 return 0;
4436}
4437
4438static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4439 struct eventfd_ctx *eventfd)
4440{
4441 struct mem_cgroup_eventfd_list *ev, *tmp;
4442
4443 spin_lock(&memcg_oom_lock);
4444
4445 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
4446 if (ev->eventfd == eventfd) {
4447 list_del(&ev->list);
4448 kfree(ev);
4449 }
4450 }
4451
4452 spin_unlock(&memcg_oom_lock);
4453}
4454
4455static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
4456{
4457 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4458
4459 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
4460 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
4461 seq_printf(sf, "oom_kill %lu\n",
4462 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
4463 return 0;
4464}
4465
4466static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
4467 struct cftype *cft, u64 val)
4468{
4469 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4470
4471
4472 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
4473 return -EINVAL;
4474
4475 memcg->oom_kill_disable = val;
4476 if (!val)
4477 memcg_oom_recover(memcg);
4478
4479 return 0;
4480}
4481
4482#ifdef CONFIG_CGROUP_WRITEBACK
4483
4484#include <trace/events/writeback.h>
4485
4486static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4487{
4488 return wb_domain_init(&memcg->cgwb_domain, gfp);
4489}
4490
4491static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4492{
4493 wb_domain_exit(&memcg->cgwb_domain);
4494}
4495
4496static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4497{
4498 wb_domain_size_changed(&memcg->cgwb_domain);
4499}
4500
4501struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
4502{
4503 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4504
4505 if (!memcg->css.parent)
4506 return NULL;
4507
4508 return &memcg->cgwb_domain;
4509}
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
4530 unsigned long *pheadroom, unsigned long *pdirty,
4531 unsigned long *pwriteback)
4532{
4533 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4534 struct mem_cgroup *parent;
4535
4536 cgroup_rstat_flush_irqsafe(memcg->css.cgroup);
4537
4538 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
4539 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
4540 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) +
4541 memcg_page_state(memcg, NR_ACTIVE_FILE);
4542
4543 *pheadroom = PAGE_COUNTER_MAX;
4544 while ((parent = parent_mem_cgroup(memcg))) {
4545 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
4546 READ_ONCE(memcg->memory.high));
4547 unsigned long used = page_counter_read(&memcg->memory);
4548
4549 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
4550 memcg = parent;
4551 }
4552}
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
4599 struct bdi_writeback *wb)
4600{
4601 struct mem_cgroup *memcg = page_memcg(page);
4602 struct memcg_cgwb_frn *frn;
4603 u64 now = get_jiffies_64();
4604 u64 oldest_at = now;
4605 int oldest = -1;
4606 int i;
4607
4608 trace_track_foreign_dirty(page, wb);
4609
4610
4611
4612
4613
4614
4615 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4616 frn = &memcg->cgwb_frn[i];
4617 if (frn->bdi_id == wb->bdi->id &&
4618 frn->memcg_id == wb->memcg_css->id)
4619 break;
4620 if (time_before64(frn->at, oldest_at) &&
4621 atomic_read(&frn->done.cnt) == 1) {
4622 oldest = i;
4623 oldest_at = frn->at;
4624 }
4625 }
4626
4627 if (i < MEMCG_CGWB_FRN_CNT) {
4628
4629
4630
4631
4632
4633
4634
4635 unsigned long update_intv =
4636 min_t(unsigned long, HZ,
4637 msecs_to_jiffies(dirty_expire_interval * 10) / 8);
4638
4639 if (time_before64(frn->at, now - update_intv))
4640 frn->at = now;
4641 } else if (oldest >= 0) {
4642
4643 frn = &memcg->cgwb_frn[oldest];
4644 frn->bdi_id = wb->bdi->id;
4645 frn->memcg_id = wb->memcg_css->id;
4646 frn->at = now;
4647 }
4648}
4649
4650
4651void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
4652{
4653 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4654 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
4655 u64 now = jiffies_64;
4656 int i;
4657
4658 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4659 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4660
4661
4662
4663
4664
4665
4666
4667 if (time_after64(frn->at, now - intv) &&
4668 atomic_read(&frn->done.cnt) == 1) {
4669 frn->at = 0;
4670 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
4671 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
4672 WB_REASON_FOREIGN_FLUSH,
4673 &frn->done);
4674 }
4675 }
4676}
4677
4678#else
4679
4680static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4681{
4682 return 0;
4683}
4684
4685static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4686{
4687}
4688
4689static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4690{
4691}
4692
4693#endif
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713static void memcg_event_remove(struct work_struct *work)
4714{
4715 struct mem_cgroup_event *event =
4716 container_of(work, struct mem_cgroup_event, remove);
4717 struct mem_cgroup *memcg = event->memcg;
4718
4719 remove_wait_queue(event->wqh, &event->wait);
4720
4721 event->unregister_event(memcg, event->eventfd);
4722
4723
4724 eventfd_signal(event->eventfd, 1);
4725
4726 eventfd_ctx_put(event->eventfd);
4727 kfree(event);
4728 css_put(&memcg->css);
4729}
4730
4731
4732
4733
4734
4735
4736static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
4737 int sync, void *key)
4738{
4739 struct mem_cgroup_event *event =
4740 container_of(wait, struct mem_cgroup_event, wait);
4741 struct mem_cgroup *memcg = event->memcg;
4742 __poll_t flags = key_to_poll(key);
4743
4744 if (flags & EPOLLHUP) {
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754 spin_lock(&memcg->event_list_lock);
4755 if (!list_empty(&event->list)) {
4756 list_del_init(&event->list);
4757
4758
4759
4760
4761 schedule_work(&event->remove);
4762 }
4763 spin_unlock(&memcg->event_list_lock);
4764 }
4765
4766 return 0;
4767}
4768
4769static void memcg_event_ptable_queue_proc(struct file *file,
4770 wait_queue_head_t *wqh, poll_table *pt)
4771{
4772 struct mem_cgroup_event *event =
4773 container_of(pt, struct mem_cgroup_event, pt);
4774
4775 event->wqh = wqh;
4776 add_wait_queue(wqh, &event->wait);
4777}
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4788 char *buf, size_t nbytes, loff_t off)
4789{
4790 struct cgroup_subsys_state *css = of_css(of);
4791 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4792 struct mem_cgroup_event *event;
4793 struct cgroup_subsys_state *cfile_css;
4794 unsigned int efd, cfd;
4795 struct fd efile;
4796 struct fd cfile;
4797 const char *name;
4798 char *endp;
4799 int ret;
4800
4801 buf = strstrip(buf);
4802
4803 efd = simple_strtoul(buf, &endp, 10);
4804 if (*endp != ' ')
4805 return -EINVAL;
4806 buf = endp + 1;
4807
4808 cfd = simple_strtoul(buf, &endp, 10);
4809 if ((*endp != ' ') && (*endp != '\0'))
4810 return -EINVAL;
4811 buf = endp + 1;
4812
4813 event = kzalloc(sizeof(*event), GFP_KERNEL);
4814 if (!event)
4815 return -ENOMEM;
4816
4817 event->memcg = memcg;
4818 INIT_LIST_HEAD(&event->list);
4819 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4820 init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4821 INIT_WORK(&event->remove, memcg_event_remove);
4822
4823 efile = fdget(efd);
4824 if (!efile.file) {
4825 ret = -EBADF;
4826 goto out_kfree;
4827 }
4828
4829 event->eventfd = eventfd_ctx_fileget(efile.file);
4830 if (IS_ERR(event->eventfd)) {
4831 ret = PTR_ERR(event->eventfd);
4832 goto out_put_efile;
4833 }
4834
4835 cfile = fdget(cfd);
4836 if (!cfile.file) {
4837 ret = -EBADF;
4838 goto out_put_eventfd;
4839 }
4840
4841
4842
4843 ret = file_permission(cfile.file, MAY_READ);
4844 if (ret < 0)
4845 goto out_put_cfile;
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855 name = cfile.file->f_path.dentry->d_name.name;
4856
4857 if (!strcmp(name, "memory.usage_in_bytes")) {
4858 event->register_event = mem_cgroup_usage_register_event;
4859 event->unregister_event = mem_cgroup_usage_unregister_event;
4860 } else if (!strcmp(name, "memory.oom_control")) {
4861 event->register_event = mem_cgroup_oom_register_event;
4862 event->unregister_event = mem_cgroup_oom_unregister_event;
4863 } else if (!strcmp(name, "memory.pressure_level")) {
4864 event->register_event = vmpressure_register_event;
4865 event->unregister_event = vmpressure_unregister_event;
4866 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
4867 event->register_event = memsw_cgroup_usage_register_event;
4868 event->unregister_event = memsw_cgroup_usage_unregister_event;
4869 } else {
4870 ret = -EINVAL;
4871 goto out_put_cfile;
4872 }
4873
4874
4875
4876
4877
4878
4879 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
4880 &memory_cgrp_subsys);
4881 ret = -EINVAL;
4882 if (IS_ERR(cfile_css))
4883 goto out_put_cfile;
4884 if (cfile_css != css) {
4885 css_put(cfile_css);
4886 goto out_put_cfile;
4887 }
4888
4889 ret = event->register_event(memcg, event->eventfd, buf);
4890 if (ret)
4891 goto out_put_css;
4892
4893 vfs_poll(efile.file, &event->pt);
4894
4895 spin_lock(&memcg->event_list_lock);
4896 list_add(&event->list, &memcg->event_list);
4897 spin_unlock(&memcg->event_list_lock);
4898
4899 fdput(cfile);
4900 fdput(efile);
4901
4902 return nbytes;
4903
4904out_put_css:
4905 css_put(css);
4906out_put_cfile:
4907 fdput(cfile);
4908out_put_eventfd:
4909 eventfd_ctx_put(event->eventfd);
4910out_put_efile:
4911 fdput(efile);
4912out_kfree:
4913 kfree(event);
4914
4915 return ret;
4916}
4917
4918static struct cftype mem_cgroup_legacy_files[] = {
4919 {
4920 .name = "usage_in_bytes",
4921 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4922 .read_u64 = mem_cgroup_read_u64,
4923 },
4924 {
4925 .name = "max_usage_in_bytes",
4926 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4927 .write = mem_cgroup_reset,
4928 .read_u64 = mem_cgroup_read_u64,
4929 },
4930 {
4931 .name = "limit_in_bytes",
4932 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4933 .write = mem_cgroup_write,
4934 .read_u64 = mem_cgroup_read_u64,
4935 },
4936 {
4937 .name = "soft_limit_in_bytes",
4938 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4939 .write = mem_cgroup_write,
4940 .read_u64 = mem_cgroup_read_u64,
4941 },
4942 {
4943 .name = "failcnt",
4944 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4945 .write = mem_cgroup_reset,
4946 .read_u64 = mem_cgroup_read_u64,
4947 },
4948 {
4949 .name = "stat",
4950 .seq_show = memcg_stat_show,
4951 },
4952 {
4953 .name = "force_empty",
4954 .write = mem_cgroup_force_empty_write,
4955 },
4956 {
4957 .name = "use_hierarchy",
4958 .write_u64 = mem_cgroup_hierarchy_write,
4959 .read_u64 = mem_cgroup_hierarchy_read,
4960 },
4961 {
4962 .name = "cgroup.event_control",
4963 .write = memcg_write_event_control,
4964 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
4965 },
4966 {
4967 .name = "swappiness",
4968 .read_u64 = mem_cgroup_swappiness_read,
4969 .write_u64 = mem_cgroup_swappiness_write,
4970 },
4971 {
4972 .name = "move_charge_at_immigrate",
4973 .read_u64 = mem_cgroup_move_charge_read,
4974 .write_u64 = mem_cgroup_move_charge_write,
4975 },
4976 {
4977 .name = "oom_control",
4978 .seq_show = mem_cgroup_oom_control_read,
4979 .write_u64 = mem_cgroup_oom_control_write,
4980 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4981 },
4982 {
4983 .name = "pressure_level",
4984 },
4985#ifdef CONFIG_NUMA
4986 {
4987 .name = "numa_stat",
4988 .seq_show = memcg_numa_stat_show,
4989 },
4990#endif
4991 {
4992 .name = "kmem.limit_in_bytes",
4993 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4994 .write = mem_cgroup_write,
4995 .read_u64 = mem_cgroup_read_u64,
4996 },
4997 {
4998 .name = "kmem.usage_in_bytes",
4999 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
5000 .read_u64 = mem_cgroup_read_u64,
5001 },
5002 {
5003 .name = "kmem.failcnt",
5004 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
5005 .write = mem_cgroup_reset,
5006 .read_u64 = mem_cgroup_read_u64,
5007 },
5008 {
5009 .name = "kmem.max_usage_in_bytes",
5010 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
5011 .write = mem_cgroup_reset,
5012 .read_u64 = mem_cgroup_read_u64,
5013 },
5014#if defined(CONFIG_MEMCG_KMEM) && \
5015 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
5016 {
5017 .name = "kmem.slabinfo",
5018 .seq_show = memcg_slab_show,
5019 },
5020#endif
5021 {
5022 .name = "kmem.tcp.limit_in_bytes",
5023 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
5024 .write = mem_cgroup_write,
5025 .read_u64 = mem_cgroup_read_u64,
5026 },
5027 {
5028 .name = "kmem.tcp.usage_in_bytes",
5029 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
5030 .read_u64 = mem_cgroup_read_u64,
5031 },
5032 {
5033 .name = "kmem.tcp.failcnt",
5034 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
5035 .write = mem_cgroup_reset,
5036 .read_u64 = mem_cgroup_read_u64,
5037 },
5038 {
5039 .name = "kmem.tcp.max_usage_in_bytes",
5040 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
5041 .write = mem_cgroup_reset,
5042 .read_u64 = mem_cgroup_read_u64,
5043 },
5044 { },
5045};
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071static DEFINE_IDR(mem_cgroup_idr);
5072
5073static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5074{
5075 if (memcg->id.id > 0) {
5076 idr_remove(&mem_cgroup_idr, memcg->id.id);
5077 memcg->id.id = 0;
5078 }
5079}
5080
5081static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5082 unsigned int n)
5083{
5084 refcount_add(n, &memcg->id.ref);
5085}
5086
5087static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
5088{
5089 if (refcount_sub_and_test(n, &memcg->id.ref)) {
5090 mem_cgroup_id_remove(memcg);
5091
5092
5093 css_put(&memcg->css);
5094 }
5095}
5096
5097static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5098{
5099 mem_cgroup_id_put_many(memcg, 1);
5100}
5101
5102
5103
5104
5105
5106
5107
5108struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
5109{
5110 WARN_ON_ONCE(!rcu_read_lock_held());
5111 return idr_find(&mem_cgroup_idr, id);
5112}
5113
5114static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5115{
5116 struct mem_cgroup_per_node *pn;
5117 int tmp = node;
5118
5119
5120
5121
5122
5123
5124
5125
5126 if (!node_state(node, N_NORMAL_MEMORY))
5127 tmp = -1;
5128 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
5129 if (!pn)
5130 return 1;
5131
5132 pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,
5133 GFP_KERNEL_ACCOUNT);
5134 if (!pn->lruvec_stat_local) {
5135 kfree(pn);
5136 return 1;
5137 }
5138
5139 pn->lruvec_stat_cpu = alloc_percpu_gfp(struct batched_lruvec_stat,
5140 GFP_KERNEL_ACCOUNT);
5141 if (!pn->lruvec_stat_cpu) {
5142 free_percpu(pn->lruvec_stat_local);
5143 kfree(pn);
5144 return 1;
5145 }
5146
5147 lruvec_init(&pn->lruvec);
5148 pn->usage_in_excess = 0;
5149 pn->on_tree = false;
5150 pn->memcg = memcg;
5151
5152 memcg->nodeinfo[node] = pn;
5153 return 0;
5154}
5155
5156static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5157{
5158 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5159
5160 if (!pn)
5161 return;
5162
5163 free_percpu(pn->lruvec_stat_cpu);
5164 free_percpu(pn->lruvec_stat_local);
5165 kfree(pn);
5166}
5167
5168static void __mem_cgroup_free(struct mem_cgroup *memcg)
5169{
5170 int node;
5171
5172 for_each_node(node)
5173 free_mem_cgroup_per_node_info(memcg, node);
5174 free_percpu(memcg->vmstats_percpu);
5175 kfree(memcg);
5176}
5177
5178static void mem_cgroup_free(struct mem_cgroup *memcg)
5179{
5180 int cpu;
5181
5182 memcg_wb_domain_exit(memcg);
5183
5184
5185
5186
5187 for_each_online_cpu(cpu)
5188 memcg_flush_lruvec_page_state(memcg, cpu);
5189 __mem_cgroup_free(memcg);
5190}
5191
5192static struct mem_cgroup *mem_cgroup_alloc(void)
5193{
5194 struct mem_cgroup *memcg;
5195 unsigned int size;
5196 int node;
5197 int __maybe_unused i;
5198 long error = -ENOMEM;
5199
5200 size = sizeof(struct mem_cgroup);
5201 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
5202
5203 memcg = kzalloc(size, GFP_KERNEL);
5204 if (!memcg)
5205 return ERR_PTR(error);
5206
5207 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5208 1, MEM_CGROUP_ID_MAX,
5209 GFP_KERNEL);
5210 if (memcg->id.id < 0) {
5211 error = memcg->id.id;
5212 goto fail;
5213 }
5214
5215 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5216 GFP_KERNEL_ACCOUNT);
5217 if (!memcg->vmstats_percpu)
5218 goto fail;
5219
5220 for_each_node(node)
5221 if (alloc_mem_cgroup_per_node_info(memcg, node))
5222 goto fail;
5223
5224 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5225 goto fail;
5226
5227 INIT_WORK(&memcg->high_work, high_work_func);
5228 INIT_LIST_HEAD(&memcg->oom_notify);
5229 mutex_init(&memcg->thresholds_lock);
5230 spin_lock_init(&memcg->move_lock);
5231 vmpressure_init(&memcg->vmpressure);
5232 INIT_LIST_HEAD(&memcg->event_list);
5233 spin_lock_init(&memcg->event_list_lock);
5234 memcg->socket_pressure = jiffies;
5235#ifdef CONFIG_MEMCG_KMEM
5236 memcg->kmemcg_id = -1;
5237 INIT_LIST_HEAD(&memcg->objcg_list);
5238#endif
5239#ifdef CONFIG_CGROUP_WRITEBACK
5240 INIT_LIST_HEAD(&memcg->cgwb_list);
5241 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5242 memcg->cgwb_frn[i].done =
5243 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
5244#endif
5245#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5246 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5247 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5248 memcg->deferred_split_queue.split_queue_len = 0;
5249#endif
5250 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
5251 return memcg;
5252fail:
5253 mem_cgroup_id_remove(memcg);
5254 __mem_cgroup_free(memcg);
5255 return ERR_PTR(error);
5256}
5257
5258static struct cgroup_subsys_state * __ref
5259mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
5260{
5261 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
5262 struct mem_cgroup *memcg, *old_memcg;
5263 long error = -ENOMEM;
5264
5265 old_memcg = set_active_memcg(parent);
5266 memcg = mem_cgroup_alloc();
5267 set_active_memcg(old_memcg);
5268 if (IS_ERR(memcg))
5269 return ERR_CAST(memcg);
5270
5271 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5272 memcg->soft_limit = PAGE_COUNTER_MAX;
5273 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5274 if (parent) {
5275 memcg->swappiness = mem_cgroup_swappiness(parent);
5276 memcg->oom_kill_disable = parent->oom_kill_disable;
5277
5278 page_counter_init(&memcg->memory, &parent->memory);
5279 page_counter_init(&memcg->swap, &parent->swap);
5280 page_counter_init(&memcg->kmem, &parent->kmem);
5281 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
5282 } else {
5283 page_counter_init(&memcg->memory, NULL);
5284 page_counter_init(&memcg->swap, NULL);
5285 page_counter_init(&memcg->kmem, NULL);
5286 page_counter_init(&memcg->tcpmem, NULL);
5287
5288 root_mem_cgroup = memcg;
5289 return &memcg->css;
5290 }
5291
5292
5293 error = memcg_online_kmem(memcg);
5294 if (error)
5295 goto fail;
5296
5297 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5298 static_branch_inc(&memcg_sockets_enabled_key);
5299
5300 return &memcg->css;
5301fail:
5302 mem_cgroup_id_remove(memcg);
5303 mem_cgroup_free(memcg);
5304 return ERR_PTR(error);
5305}
5306
5307static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
5308{
5309 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5310
5311
5312
5313
5314
5315
5316 if (alloc_shrinker_info(memcg)) {
5317 mem_cgroup_id_remove(memcg);
5318 return -ENOMEM;
5319 }
5320
5321
5322 refcount_set(&memcg->id.ref, 1);
5323 css_get(css);
5324 return 0;
5325}
5326
5327static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
5328{
5329 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5330 struct mem_cgroup_event *event, *tmp;
5331
5332
5333
5334
5335
5336
5337 spin_lock(&memcg->event_list_lock);
5338 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
5339 list_del_init(&event->list);
5340 schedule_work(&event->remove);
5341 }
5342 spin_unlock(&memcg->event_list_lock);
5343
5344 page_counter_set_min(&memcg->memory, 0);
5345 page_counter_set_low(&memcg->memory, 0);
5346
5347 memcg_offline_kmem(memcg);
5348 reparent_shrinker_deferred(memcg);
5349 wb_memcg_offline(memcg);
5350
5351 drain_all_stock(memcg);
5352
5353 mem_cgroup_id_put(memcg);
5354}
5355
5356static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
5357{
5358 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5359
5360 invalidate_reclaim_iterators(memcg);
5361}
5362
5363static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
5364{
5365 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5366 int __maybe_unused i;
5367
5368#ifdef CONFIG_CGROUP_WRITEBACK
5369 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5370 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5371#endif
5372 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5373 static_branch_dec(&memcg_sockets_enabled_key);
5374
5375 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
5376 static_branch_dec(&memcg_sockets_enabled_key);
5377
5378 vmpressure_cleanup(&memcg->vmpressure);
5379 cancel_work_sync(&memcg->high_work);
5380 mem_cgroup_remove_from_trees(memcg);
5381 free_shrinker_info(memcg);
5382 memcg_free_kmem(memcg);
5383 mem_cgroup_free(memcg);
5384}
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
5400{
5401 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5402
5403 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5404 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5405 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5406 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
5407 page_counter_set_min(&memcg->memory, 0);
5408 page_counter_set_low(&memcg->memory, 0);
5409 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5410 memcg->soft_limit = PAGE_COUNTER_MAX;
5411 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5412 memcg_wb_domain_size_changed(memcg);
5413}
5414
5415static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
5416{
5417 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5418 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5419 struct memcg_vmstats_percpu *statc;
5420 long delta, v;
5421 int i;
5422
5423 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
5424
5425 for (i = 0; i < MEMCG_NR_STAT; i++) {
5426
5427
5428
5429
5430
5431 delta = memcg->vmstats.state_pending[i];
5432 if (delta)
5433 memcg->vmstats.state_pending[i] = 0;
5434
5435
5436 v = READ_ONCE(statc->state[i]);
5437 if (v != statc->state_prev[i]) {
5438 delta += v - statc->state_prev[i];
5439 statc->state_prev[i] = v;
5440 }
5441
5442 if (!delta)
5443 continue;
5444
5445
5446 memcg->vmstats.state[i] += delta;
5447 if (parent)
5448 parent->vmstats.state_pending[i] += delta;
5449 }
5450
5451 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
5452 delta = memcg->vmstats.events_pending[i];
5453 if (delta)
5454 memcg->vmstats.events_pending[i] = 0;
5455
5456 v = READ_ONCE(statc->events[i]);
5457 if (v != statc->events_prev[i]) {
5458 delta += v - statc->events_prev[i];
5459 statc->events_prev[i] = v;
5460 }
5461
5462 if (!delta)
5463 continue;
5464
5465 memcg->vmstats.events[i] += delta;
5466 if (parent)
5467 parent->vmstats.events_pending[i] += delta;
5468 }
5469}
5470
5471#ifdef CONFIG_MMU
5472
5473static int mem_cgroup_do_precharge(unsigned long count)
5474{
5475 int ret;
5476
5477
5478 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
5479 if (!ret) {
5480 mc.precharge += count;
5481 return ret;
5482 }
5483
5484
5485 while (count--) {
5486 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
5487 if (ret)
5488 return ret;
5489 mc.precharge++;
5490 cond_resched();
5491 }
5492 return 0;
5493}
5494
5495union mc_target {
5496 struct page *page;
5497 swp_entry_t ent;
5498};
5499
5500enum mc_target_type {
5501 MC_TARGET_NONE = 0,
5502 MC_TARGET_PAGE,
5503 MC_TARGET_SWAP,
5504 MC_TARGET_DEVICE,
5505};
5506
5507static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
5508 unsigned long addr, pte_t ptent)
5509{
5510 struct page *page = vm_normal_page(vma, addr, ptent);
5511
5512 if (!page || !page_mapped(page))
5513 return NULL;
5514 if (PageAnon(page)) {
5515 if (!(mc.flags & MOVE_ANON))
5516 return NULL;
5517 } else {
5518 if (!(mc.flags & MOVE_FILE))
5519 return NULL;
5520 }
5521 if (!get_page_unless_zero(page))
5522 return NULL;
5523
5524 return page;
5525}
5526
5527#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
5528static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5529 pte_t ptent, swp_entry_t *entry)
5530{
5531 struct page *page = NULL;
5532 swp_entry_t ent = pte_to_swp_entry(ptent);
5533
5534 if (!(mc.flags & MOVE_ANON))
5535 return NULL;
5536
5537
5538
5539
5540
5541
5542 if (is_device_private_entry(ent)) {
5543 page = pfn_swap_entry_to_page(ent);
5544
5545
5546
5547
5548 if (!page_ref_add_unless(page, 1, 1))
5549 return NULL;
5550 return page;
5551 }
5552
5553 if (non_swap_entry(ent))
5554 return NULL;
5555
5556
5557
5558
5559
5560 page = find_get_page(swap_address_space(ent), swp_offset(ent));
5561 entry->val = ent.val;
5562
5563 return page;
5564}
5565#else
5566static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5567 pte_t ptent, swp_entry_t *entry)
5568{
5569 return NULL;
5570}
5571#endif
5572
5573static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
5574 unsigned long addr, pte_t ptent, swp_entry_t *entry)
5575{
5576 if (!vma->vm_file)
5577 return NULL;
5578 if (!(mc.flags & MOVE_FILE))
5579 return NULL;
5580
5581
5582
5583 return find_get_incore_page(vma->vm_file->f_mapping,
5584 linear_page_index(vma, addr));
5585}
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599static int mem_cgroup_move_account(struct page *page,
5600 bool compound,
5601 struct mem_cgroup *from,
5602 struct mem_cgroup *to)
5603{
5604 struct lruvec *from_vec, *to_vec;
5605 struct pglist_data *pgdat;
5606 unsigned int nr_pages = compound ? thp_nr_pages(page) : 1;
5607 int ret;
5608
5609 VM_BUG_ON(from == to);
5610 VM_BUG_ON_PAGE(PageLRU(page), page);
5611 VM_BUG_ON(compound && !PageTransHuge(page));
5612
5613
5614
5615
5616
5617 ret = -EBUSY;
5618 if (!trylock_page(page))
5619 goto out;
5620
5621 ret = -EINVAL;
5622 if (page_memcg(page) != from)
5623 goto out_unlock;
5624
5625 pgdat = page_pgdat(page);
5626 from_vec = mem_cgroup_lruvec(from, pgdat);
5627 to_vec = mem_cgroup_lruvec(to, pgdat);
5628
5629 lock_page_memcg(page);
5630
5631 if (PageAnon(page)) {
5632 if (page_mapped(page)) {
5633 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
5634 __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
5635 if (PageTransHuge(page)) {
5636 __mod_lruvec_state(from_vec, NR_ANON_THPS,
5637 -nr_pages);
5638 __mod_lruvec_state(to_vec, NR_ANON_THPS,
5639 nr_pages);
5640 }
5641 }
5642 } else {
5643 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
5644 __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
5645
5646 if (PageSwapBacked(page)) {
5647 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
5648 __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
5649 }
5650
5651 if (page_mapped(page)) {
5652 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
5653 __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
5654 }
5655
5656 if (PageDirty(page)) {
5657 struct address_space *mapping = page_mapping(page);
5658
5659 if (mapping_can_writeback(mapping)) {
5660 __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
5661 -nr_pages);
5662 __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
5663 nr_pages);
5664 }
5665 }
5666 }
5667
5668 if (PageWriteback(page)) {
5669 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
5670 __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
5671 }
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686 smp_mb();
5687
5688 css_get(&to->css);
5689 css_put(&from->css);
5690
5691 page->memcg_data = (unsigned long)to;
5692
5693 __unlock_page_memcg(from);
5694
5695 ret = 0;
5696
5697 local_irq_disable();
5698 mem_cgroup_charge_statistics(to, page, nr_pages);
5699 memcg_check_events(to, page);
5700 mem_cgroup_charge_statistics(from, page, -nr_pages);
5701 memcg_check_events(from, page);
5702 local_irq_enable();
5703out_unlock:
5704 unlock_page(page);
5705out:
5706 return ret;
5707}
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
5736 unsigned long addr, pte_t ptent, union mc_target *target)
5737{
5738 struct page *page = NULL;
5739 enum mc_target_type ret = MC_TARGET_NONE;
5740 swp_entry_t ent = { .val = 0 };
5741
5742 if (pte_present(ptent))
5743 page = mc_handle_present_pte(vma, addr, ptent);
5744 else if (is_swap_pte(ptent))
5745 page = mc_handle_swap_pte(vma, ptent, &ent);
5746 else if (pte_none(ptent))
5747 page = mc_handle_file_pte(vma, addr, ptent, &ent);
5748
5749 if (!page && !ent.val)
5750 return ret;
5751 if (page) {
5752
5753
5754
5755
5756
5757 if (page_memcg(page) == mc.from) {
5758 ret = MC_TARGET_PAGE;
5759 if (is_device_private_page(page))
5760 ret = MC_TARGET_DEVICE;
5761 if (target)
5762 target->page = page;
5763 }
5764 if (!ret || !target)
5765 put_page(page);
5766 }
5767
5768
5769
5770
5771 if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
5772 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
5773 ret = MC_TARGET_SWAP;
5774 if (target)
5775 target->ent = ent;
5776 }
5777 return ret;
5778}
5779
5780#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5781
5782
5783
5784
5785
5786static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5787 unsigned long addr, pmd_t pmd, union mc_target *target)
5788{
5789 struct page *page = NULL;
5790 enum mc_target_type ret = MC_TARGET_NONE;
5791
5792 if (unlikely(is_swap_pmd(pmd))) {
5793 VM_BUG_ON(thp_migration_supported() &&
5794 !is_pmd_migration_entry(pmd));
5795 return ret;
5796 }
5797 page = pmd_page(pmd);
5798 VM_BUG_ON_PAGE(!page || !PageHead(page), page);
5799 if (!(mc.flags & MOVE_ANON))
5800 return ret;
5801 if (page_memcg(page) == mc.from) {
5802 ret = MC_TARGET_PAGE;
5803 if (target) {
5804 get_page(page);
5805 target->page = page;
5806 }
5807 }
5808 return ret;
5809}
5810#else
5811static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5812 unsigned long addr, pmd_t pmd, union mc_target *target)
5813{
5814 return MC_TARGET_NONE;
5815}
5816#endif
5817
5818static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
5819 unsigned long addr, unsigned long end,
5820 struct mm_walk *walk)
5821{
5822 struct vm_area_struct *vma = walk->vma;
5823 pte_t *pte;
5824 spinlock_t *ptl;
5825
5826 ptl = pmd_trans_huge_lock(pmd, vma);
5827 if (ptl) {
5828
5829
5830
5831
5832
5833 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
5834 mc.precharge += HPAGE_PMD_NR;
5835 spin_unlock(ptl);
5836 return 0;
5837 }
5838
5839 if (pmd_trans_unstable(pmd))
5840 return 0;
5841 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5842 for (; addr != end; pte++, addr += PAGE_SIZE)
5843 if (get_mctgt_type(vma, addr, *pte, NULL))
5844 mc.precharge++;
5845 pte_unmap_unlock(pte - 1, ptl);
5846 cond_resched();
5847
5848 return 0;
5849}
5850
5851static const struct mm_walk_ops precharge_walk_ops = {
5852 .pmd_entry = mem_cgroup_count_precharge_pte_range,
5853};
5854
5855static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5856{
5857 unsigned long precharge;
5858
5859 mmap_read_lock(mm);
5860 walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
5861 mmap_read_unlock(mm);
5862
5863 precharge = mc.precharge;
5864 mc.precharge = 0;
5865
5866 return precharge;
5867}
5868
5869static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5870{
5871 unsigned long precharge = mem_cgroup_count_precharge(mm);
5872
5873 VM_BUG_ON(mc.moving_task);
5874 mc.moving_task = current;
5875 return mem_cgroup_do_precharge(precharge);
5876}
5877
5878
5879static void __mem_cgroup_clear_mc(void)
5880{
5881 struct mem_cgroup *from = mc.from;
5882 struct mem_cgroup *to = mc.to;
5883
5884
5885 if (mc.precharge) {
5886 cancel_charge(mc.to, mc.precharge);
5887 mc.precharge = 0;
5888 }
5889
5890
5891
5892
5893 if (mc.moved_charge) {
5894 cancel_charge(mc.from, mc.moved_charge);
5895 mc.moved_charge = 0;
5896 }
5897
5898 if (mc.moved_swap) {
5899
5900 if (!mem_cgroup_is_root(mc.from))
5901 page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
5902
5903 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5904
5905
5906
5907
5908
5909 if (!mem_cgroup_is_root(mc.to))
5910 page_counter_uncharge(&mc.to->memory, mc.moved_swap);
5911
5912 mc.moved_swap = 0;
5913 }
5914 memcg_oom_recover(from);
5915 memcg_oom_recover(to);
5916 wake_up_all(&mc.waitq);
5917}
5918
5919static void mem_cgroup_clear_mc(void)
5920{
5921 struct mm_struct *mm = mc.mm;
5922
5923
5924
5925
5926
5927 mc.moving_task = NULL;
5928 __mem_cgroup_clear_mc();
5929 spin_lock(&mc.lock);
5930 mc.from = NULL;
5931 mc.to = NULL;
5932 mc.mm = NULL;
5933 spin_unlock(&mc.lock);
5934
5935 mmput(mm);
5936}
5937
5938static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5939{
5940 struct cgroup_subsys_state *css;
5941 struct mem_cgroup *memcg = NULL;
5942 struct mem_cgroup *from;
5943 struct task_struct *leader, *p;
5944 struct mm_struct *mm;
5945 unsigned long move_flags;
5946 int ret = 0;
5947
5948
5949 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5950 return 0;
5951
5952
5953
5954
5955
5956
5957
5958 p = NULL;
5959 cgroup_taskset_for_each_leader(leader, css, tset) {
5960 WARN_ON_ONCE(p);
5961 p = leader;
5962 memcg = mem_cgroup_from_css(css);
5963 }
5964 if (!p)
5965 return 0;
5966
5967
5968
5969
5970
5971
5972 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
5973 if (!move_flags)
5974 return 0;
5975
5976 from = mem_cgroup_from_task(p);
5977
5978 VM_BUG_ON(from == memcg);
5979
5980 mm = get_task_mm(p);
5981 if (!mm)
5982 return 0;
5983
5984 if (mm->owner == p) {
5985 VM_BUG_ON(mc.from);
5986 VM_BUG_ON(mc.to);
5987 VM_BUG_ON(mc.precharge);
5988 VM_BUG_ON(mc.moved_charge);
5989 VM_BUG_ON(mc.moved_swap);
5990
5991 spin_lock(&mc.lock);
5992 mc.mm = mm;
5993 mc.from = from;
5994 mc.to = memcg;
5995 mc.flags = move_flags;
5996 spin_unlock(&mc.lock);
5997
5998
5999 ret = mem_cgroup_precharge_mc(mm);
6000 if (ret)
6001 mem_cgroup_clear_mc();
6002 } else {
6003 mmput(mm);
6004 }
6005 return ret;
6006}
6007
6008static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
6009{
6010 if (mc.to)
6011 mem_cgroup_clear_mc();
6012}
6013
6014static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
6015 unsigned long addr, unsigned long end,
6016 struct mm_walk *walk)
6017{
6018 int ret = 0;
6019 struct vm_area_struct *vma = walk->vma;
6020 pte_t *pte;
6021 spinlock_t *ptl;
6022 enum mc_target_type target_type;
6023 union mc_target target;
6024 struct page *page;
6025
6026 ptl = pmd_trans_huge_lock(pmd, vma);
6027 if (ptl) {
6028 if (mc.precharge < HPAGE_PMD_NR) {
6029 spin_unlock(ptl);
6030 return 0;
6031 }
6032 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
6033 if (target_type == MC_TARGET_PAGE) {
6034 page = target.page;
6035 if (!isolate_lru_page(page)) {
6036 if (!mem_cgroup_move_account(page, true,
6037 mc.from, mc.to)) {
6038 mc.precharge -= HPAGE_PMD_NR;
6039 mc.moved_charge += HPAGE_PMD_NR;
6040 }
6041 putback_lru_page(page);
6042 }
6043 put_page(page);
6044 } else if (target_type == MC_TARGET_DEVICE) {
6045 page = target.page;
6046 if (!mem_cgroup_move_account(page, true,
6047 mc.from, mc.to)) {
6048 mc.precharge -= HPAGE_PMD_NR;
6049 mc.moved_charge += HPAGE_PMD_NR;
6050 }
6051 put_page(page);
6052 }
6053 spin_unlock(ptl);
6054 return 0;
6055 }
6056
6057 if (pmd_trans_unstable(pmd))
6058 return 0;
6059retry:
6060 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6061 for (; addr != end; addr += PAGE_SIZE) {
6062 pte_t ptent = *(pte++);
6063 bool device = false;
6064 swp_entry_t ent;
6065
6066 if (!mc.precharge)
6067 break;
6068
6069 switch (get_mctgt_type(vma, addr, ptent, &target)) {
6070 case MC_TARGET_DEVICE:
6071 device = true;
6072 fallthrough;
6073 case MC_TARGET_PAGE:
6074 page = target.page;
6075
6076
6077
6078
6079
6080
6081 if (PageTransCompound(page))
6082 goto put;
6083 if (!device && isolate_lru_page(page))
6084 goto put;
6085 if (!mem_cgroup_move_account(page, false,
6086 mc.from, mc.to)) {
6087 mc.precharge--;
6088
6089 mc.moved_charge++;
6090 }
6091 if (!device)
6092 putback_lru_page(page);
6093put:
6094 put_page(page);
6095 break;
6096 case MC_TARGET_SWAP:
6097 ent = target.ent;
6098 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
6099 mc.precharge--;
6100 mem_cgroup_id_get_many(mc.to, 1);
6101
6102 mc.moved_swap++;
6103 }
6104 break;
6105 default:
6106 break;
6107 }
6108 }
6109 pte_unmap_unlock(pte - 1, ptl);
6110 cond_resched();
6111
6112 if (addr != end) {
6113
6114
6115
6116
6117
6118
6119 ret = mem_cgroup_do_precharge(1);
6120 if (!ret)
6121 goto retry;
6122 }
6123
6124 return ret;
6125}
6126
6127static const struct mm_walk_ops charge_walk_ops = {
6128 .pmd_entry = mem_cgroup_move_charge_pte_range,
6129};
6130
6131static void mem_cgroup_move_charge(void)
6132{
6133 lru_add_drain_all();
6134
6135
6136
6137
6138
6139 atomic_inc(&mc.from->moving_account);
6140 synchronize_rcu();
6141retry:
6142 if (unlikely(!mmap_read_trylock(mc.mm))) {
6143
6144
6145
6146
6147
6148
6149
6150 __mem_cgroup_clear_mc();
6151 cond_resched();
6152 goto retry;
6153 }
6154
6155
6156
6157
6158 walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
6159 NULL);
6160
6161 mmap_read_unlock(mc.mm);
6162 atomic_dec(&mc.from->moving_account);
6163}
6164
6165static void mem_cgroup_move_task(void)
6166{
6167 if (mc.to) {
6168 mem_cgroup_move_charge();
6169 mem_cgroup_clear_mc();
6170 }
6171}
6172#else
6173static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
6174{
6175 return 0;
6176}
6177static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
6178{
6179}
6180static void mem_cgroup_move_task(void)
6181{
6182}
6183#endif
6184
6185static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
6186{
6187 if (value == PAGE_COUNTER_MAX)
6188 seq_puts(m, "max\n");
6189 else
6190 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
6191
6192 return 0;
6193}
6194
6195static u64 memory_current_read(struct cgroup_subsys_state *css,
6196 struct cftype *cft)
6197{
6198 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6199
6200 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
6201}
6202
6203static int memory_min_show(struct seq_file *m, void *v)
6204{
6205 return seq_puts_memcg_tunable(m,
6206 READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
6207}
6208
6209static ssize_t memory_min_write(struct kernfs_open_file *of,
6210 char *buf, size_t nbytes, loff_t off)
6211{
6212 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6213 unsigned long min;
6214 int err;
6215
6216 buf = strstrip(buf);
6217 err = page_counter_memparse(buf, "max", &min);
6218 if (err)
6219 return err;
6220
6221 page_counter_set_min(&memcg->memory, min);
6222
6223 return nbytes;
6224}
6225
6226static int memory_low_show(struct seq_file *m, void *v)
6227{
6228 return seq_puts_memcg_tunable(m,
6229 READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
6230}
6231
6232static ssize_t memory_low_write(struct kernfs_open_file *of,
6233 char *buf, size_t nbytes, loff_t off)
6234{
6235 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6236 unsigned long low;
6237 int err;
6238
6239 buf = strstrip(buf);
6240 err = page_counter_memparse(buf, "max", &low);
6241 if (err)
6242 return err;
6243
6244 page_counter_set_low(&memcg->memory, low);
6245
6246 return nbytes;
6247}
6248
6249static int memory_high_show(struct seq_file *m, void *v)
6250{
6251 return seq_puts_memcg_tunable(m,
6252 READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
6253}
6254
6255static ssize_t memory_high_write(struct kernfs_open_file *of,
6256 char *buf, size_t nbytes, loff_t off)
6257{
6258 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6259 unsigned int nr_retries = MAX_RECLAIM_RETRIES;
6260 bool drained = false;
6261 unsigned long high;
6262 int err;
6263
6264 buf = strstrip(buf);
6265 err = page_counter_memparse(buf, "max", &high);
6266 if (err)
6267 return err;
6268
6269 page_counter_set_high(&memcg->memory, high);
6270
6271 for (;;) {
6272 unsigned long nr_pages = page_counter_read(&memcg->memory);
6273 unsigned long reclaimed;
6274
6275 if (nr_pages <= high)
6276 break;
6277
6278 if (signal_pending(current))
6279 break;
6280
6281 if (!drained) {
6282 drain_all_stock(memcg);
6283 drained = true;
6284 continue;
6285 }
6286
6287 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6288 GFP_KERNEL, true);
6289
6290 if (!reclaimed && !nr_retries--)
6291 break;
6292 }
6293
6294 memcg_wb_domain_size_changed(memcg);
6295 return nbytes;
6296}
6297
6298static int memory_max_show(struct seq_file *m, void *v)
6299{
6300 return seq_puts_memcg_tunable(m,
6301 READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
6302}
6303
6304static ssize_t memory_max_write(struct kernfs_open_file *of,
6305 char *buf, size_t nbytes, loff_t off)
6306{
6307 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6308 unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
6309 bool drained = false;
6310 unsigned long max;
6311 int err;
6312
6313 buf = strstrip(buf);
6314 err = page_counter_memparse(buf, "max", &max);
6315 if (err)
6316 return err;
6317
6318 xchg(&memcg->memory.max, max);
6319
6320 for (;;) {
6321 unsigned long nr_pages = page_counter_read(&memcg->memory);
6322
6323 if (nr_pages <= max)
6324 break;
6325
6326 if (signal_pending(current))
6327 break;
6328
6329 if (!drained) {
6330 drain_all_stock(memcg);
6331 drained = true;
6332 continue;
6333 }
6334
6335 if (nr_reclaims) {
6336 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6337 GFP_KERNEL, true))
6338 nr_reclaims--;
6339 continue;
6340 }
6341
6342 memcg_memory_event(memcg, MEMCG_OOM);
6343 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6344 break;
6345 }
6346
6347 memcg_wb_domain_size_changed(memcg);
6348 return nbytes;
6349}
6350
6351static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
6352{
6353 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
6354 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
6355 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
6356 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
6357 seq_printf(m, "oom_kill %lu\n",
6358 atomic_long_read(&events[MEMCG_OOM_KILL]));
6359}
6360
6361static int memory_events_show(struct seq_file *m, void *v)
6362{
6363 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6364
6365 __memory_events_show(m, memcg->memory_events);
6366 return 0;
6367}
6368
6369static int memory_events_local_show(struct seq_file *m, void *v)
6370{
6371 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6372
6373 __memory_events_show(m, memcg->memory_events_local);
6374 return 0;
6375}
6376
6377static int memory_stat_show(struct seq_file *m, void *v)
6378{
6379 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6380 char *buf;
6381
6382 buf = memory_stat_format(memcg);
6383 if (!buf)
6384 return -ENOMEM;
6385 seq_puts(m, buf);
6386 kfree(buf);
6387 return 0;
6388}
6389
6390#ifdef CONFIG_NUMA
6391static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec,
6392 int item)
6393{
6394 return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item);
6395}
6396
6397static int memory_numa_stat_show(struct seq_file *m, void *v)
6398{
6399 int i;
6400 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6401
6402 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
6403 int nid;
6404
6405 if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
6406 continue;
6407
6408 seq_printf(m, "%s", memory_stats[i].name);
6409 for_each_node_state(nid, N_MEMORY) {
6410 u64 size;
6411 struct lruvec *lruvec;
6412
6413 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
6414 size = lruvec_page_state_output(lruvec,
6415 memory_stats[i].idx);
6416 seq_printf(m, " N%d=%llu", nid, size);
6417 }
6418 seq_putc(m, '\n');
6419 }
6420
6421 return 0;
6422}
6423#endif
6424
6425static int memory_oom_group_show(struct seq_file *m, void *v)
6426{
6427 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6428
6429 seq_printf(m, "%d\n", memcg->oom_group);
6430
6431 return 0;
6432}
6433
6434static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
6435 char *buf, size_t nbytes, loff_t off)
6436{
6437 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6438 int ret, oom_group;
6439
6440 buf = strstrip(buf);
6441 if (!buf)
6442 return -EINVAL;
6443
6444 ret = kstrtoint(buf, 0, &oom_group);
6445 if (ret)
6446 return ret;
6447
6448 if (oom_group != 0 && oom_group != 1)
6449 return -EINVAL;
6450
6451 memcg->oom_group = oom_group;
6452
6453 return nbytes;
6454}
6455
6456static struct cftype memory_files[] = {
6457 {
6458 .name = "current",
6459 .flags = CFTYPE_NOT_ON_ROOT,
6460 .read_u64 = memory_current_read,
6461 },
6462 {
6463 .name = "min",
6464 .flags = CFTYPE_NOT_ON_ROOT,
6465 .seq_show = memory_min_show,
6466 .write = memory_min_write,
6467 },
6468 {
6469 .name = "low",
6470 .flags = CFTYPE_NOT_ON_ROOT,
6471 .seq_show = memory_low_show,
6472 .write = memory_low_write,
6473 },
6474 {
6475 .name = "high",
6476 .flags = CFTYPE_NOT_ON_ROOT,
6477 .seq_show = memory_high_show,
6478 .write = memory_high_write,
6479 },
6480 {
6481 .name = "max",
6482 .flags = CFTYPE_NOT_ON_ROOT,
6483 .seq_show = memory_max_show,
6484 .write = memory_max_write,
6485 },
6486 {
6487 .name = "events",
6488 .flags = CFTYPE_NOT_ON_ROOT,
6489 .file_offset = offsetof(struct mem_cgroup, events_file),
6490 .seq_show = memory_events_show,
6491 },
6492 {
6493 .name = "events.local",
6494 .flags = CFTYPE_NOT_ON_ROOT,
6495 .file_offset = offsetof(struct mem_cgroup, events_local_file),
6496 .seq_show = memory_events_local_show,
6497 },
6498 {
6499 .name = "stat",
6500 .seq_show = memory_stat_show,
6501 },
6502#ifdef CONFIG_NUMA
6503 {
6504 .name = "numa_stat",
6505 .seq_show = memory_numa_stat_show,
6506 },
6507#endif
6508 {
6509 .name = "oom.group",
6510 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
6511 .seq_show = memory_oom_group_show,
6512 .write = memory_oom_group_write,
6513 },
6514 { }
6515};
6516
6517struct cgroup_subsys memory_cgrp_subsys = {
6518 .css_alloc = mem_cgroup_css_alloc,
6519 .css_online = mem_cgroup_css_online,
6520 .css_offline = mem_cgroup_css_offline,
6521 .css_released = mem_cgroup_css_released,
6522 .css_free = mem_cgroup_css_free,
6523 .css_reset = mem_cgroup_css_reset,
6524 .css_rstat_flush = mem_cgroup_css_rstat_flush,
6525 .can_attach = mem_cgroup_can_attach,
6526 .cancel_attach = mem_cgroup_cancel_attach,
6527 .post_attach = mem_cgroup_move_task,
6528 .dfl_cftypes = memory_files,
6529 .legacy_cftypes = mem_cgroup_legacy_files,
6530 .early_init = 0,
6531};
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576static unsigned long effective_protection(unsigned long usage,
6577 unsigned long parent_usage,
6578 unsigned long setting,
6579 unsigned long parent_effective,
6580 unsigned long siblings_protected)
6581{
6582 unsigned long protected;
6583 unsigned long ep;
6584
6585 protected = min(usage, setting);
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596 if (siblings_protected > parent_effective)
6597 return protected * parent_effective / siblings_protected;
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614 ep = protected;
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632 if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
6633 return ep;
6634 if (parent_effective > siblings_protected &&
6635 parent_usage > siblings_protected &&
6636 usage > protected) {
6637 unsigned long unclaimed;
6638
6639 unclaimed = parent_effective - siblings_protected;
6640 unclaimed *= usage - protected;
6641 unclaimed /= parent_usage - siblings_protected;
6642
6643 ep += unclaimed;
6644 }
6645
6646 return ep;
6647}
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6658 struct mem_cgroup *memcg)
6659{
6660 unsigned long usage, parent_usage;
6661 struct mem_cgroup *parent;
6662
6663 if (mem_cgroup_disabled())
6664 return;
6665
6666 if (!root)
6667 root = root_mem_cgroup;
6668
6669
6670
6671
6672
6673
6674
6675
6676 if (memcg == root)
6677 return;
6678
6679 usage = page_counter_read(&memcg->memory);
6680 if (!usage)
6681 return;
6682
6683 parent = parent_mem_cgroup(memcg);
6684
6685 if (!parent)
6686 return;
6687
6688 if (parent == root) {
6689 memcg->memory.emin = READ_ONCE(memcg->memory.min);
6690 memcg->memory.elow = READ_ONCE(memcg->memory.low);
6691 return;
6692 }
6693
6694 parent_usage = page_counter_read(&parent->memory);
6695
6696 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
6697 READ_ONCE(memcg->memory.min),
6698 READ_ONCE(parent->memory.emin),
6699 atomic_long_read(&parent->memory.children_min_usage)));
6700
6701 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
6702 READ_ONCE(memcg->memory.low),
6703 READ_ONCE(parent->memory.elow),
6704 atomic_long_read(&parent->memory.children_low_usage)));
6705}
6706
6707static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
6708 gfp_t gfp)
6709{
6710 unsigned int nr_pages = thp_nr_pages(page);
6711 int ret;
6712
6713 ret = try_charge(memcg, gfp, nr_pages);
6714 if (ret)
6715 goto out;
6716
6717 css_get(&memcg->css);
6718 commit_charge(page, memcg);
6719
6720 local_irq_disable();
6721 mem_cgroup_charge_statistics(memcg, page, nr_pages);
6722 memcg_check_events(memcg, page);
6723 local_irq_enable();
6724out:
6725 return ret;
6726}
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
6743{
6744 struct mem_cgroup *memcg;
6745 int ret;
6746
6747 if (mem_cgroup_disabled())
6748 return 0;
6749
6750 memcg = get_mem_cgroup_from_mm(mm);
6751 ret = __mem_cgroup_charge(page, memcg, gfp_mask);
6752 css_put(&memcg->css);
6753
6754 return ret;
6755}
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
6770 gfp_t gfp, swp_entry_t entry)
6771{
6772 struct mem_cgroup *memcg;
6773 unsigned short id;
6774 int ret;
6775
6776 if (mem_cgroup_disabled())
6777 return 0;
6778
6779 id = lookup_swap_cgroup_id(entry);
6780 rcu_read_lock();
6781 memcg = mem_cgroup_from_id(id);
6782 if (!memcg || !css_tryget_online(&memcg->css))
6783 memcg = get_mem_cgroup_from_mm(mm);
6784 rcu_read_unlock();
6785
6786 ret = __mem_cgroup_charge(page, memcg, gfp);
6787
6788 css_put(&memcg->css);
6789 return ret;
6790}
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)
6802{
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815 if (!mem_cgroup_disabled() && do_memsw_account()) {
6816
6817
6818
6819
6820
6821 mem_cgroup_uncharge_swap(entry, 1);
6822 }
6823}
6824
6825struct uncharge_gather {
6826 struct mem_cgroup *memcg;
6827 unsigned long nr_memory;
6828 unsigned long pgpgout;
6829 unsigned long nr_kmem;
6830 struct page *dummy_page;
6831};
6832
6833static inline void uncharge_gather_clear(struct uncharge_gather *ug)
6834{
6835 memset(ug, 0, sizeof(*ug));
6836}
6837
6838static void uncharge_batch(const struct uncharge_gather *ug)
6839{
6840 unsigned long flags;
6841
6842 if (ug->nr_memory) {
6843 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory);
6844 if (do_memsw_account())
6845 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory);
6846 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
6847 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6848 memcg_oom_recover(ug->memcg);
6849 }
6850
6851 local_irq_save(flags);
6852 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
6853 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory);
6854 memcg_check_events(ug->memcg, ug->dummy_page);
6855 local_irq_restore(flags);
6856
6857
6858 css_put(&ug->memcg->css);
6859}
6860
6861static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6862{
6863 unsigned long nr_pages;
6864 struct mem_cgroup *memcg;
6865 struct obj_cgroup *objcg;
6866 bool use_objcg = PageMemcgKmem(page);
6867
6868 VM_BUG_ON_PAGE(PageLRU(page), page);
6869
6870
6871
6872
6873
6874
6875 if (use_objcg) {
6876 objcg = __page_objcg(page);
6877
6878
6879
6880
6881 memcg = get_mem_cgroup_from_objcg(objcg);
6882 } else {
6883 memcg = __page_memcg(page);
6884 }
6885
6886 if (!memcg)
6887 return;
6888
6889 if (ug->memcg != memcg) {
6890 if (ug->memcg) {
6891 uncharge_batch(ug);
6892 uncharge_gather_clear(ug);
6893 }
6894 ug->memcg = memcg;
6895 ug->dummy_page = page;
6896
6897
6898 css_get(&memcg->css);
6899 }
6900
6901 nr_pages = compound_nr(page);
6902
6903 if (use_objcg) {
6904 ug->nr_memory += nr_pages;
6905 ug->nr_kmem += nr_pages;
6906
6907 page->memcg_data = 0;
6908 obj_cgroup_put(objcg);
6909 } else {
6910
6911 if (!mem_cgroup_is_root(memcg))
6912 ug->nr_memory += nr_pages;
6913 ug->pgpgout++;
6914
6915 page->memcg_data = 0;
6916 }
6917
6918 css_put(&memcg->css);
6919}
6920
6921
6922
6923
6924
6925
6926
6927void mem_cgroup_uncharge(struct page *page)
6928{
6929 struct uncharge_gather ug;
6930
6931 if (mem_cgroup_disabled())
6932 return;
6933
6934
6935 if (!page_memcg(page))
6936 return;
6937
6938 uncharge_gather_clear(&ug);
6939 uncharge_page(page, &ug);
6940 uncharge_batch(&ug);
6941}
6942
6943
6944
6945
6946
6947
6948
6949
6950void mem_cgroup_uncharge_list(struct list_head *page_list)
6951{
6952 struct uncharge_gather ug;
6953 struct page *page;
6954
6955 if (mem_cgroup_disabled())
6956 return;
6957
6958 uncharge_gather_clear(&ug);
6959 list_for_each_entry(page, page_list, lru)
6960 uncharge_page(page, &ug);
6961 if (ug.memcg)
6962 uncharge_batch(&ug);
6963}
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
6976{
6977 struct mem_cgroup *memcg;
6978 unsigned int nr_pages;
6979 unsigned long flags;
6980
6981 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
6982 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
6983 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
6984 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
6985 newpage);
6986
6987 if (mem_cgroup_disabled())
6988 return;
6989
6990
6991 if (page_memcg(newpage))
6992 return;
6993
6994 memcg = page_memcg(oldpage);
6995 VM_WARN_ON_ONCE_PAGE(!memcg, oldpage);
6996 if (!memcg)
6997 return;
6998
6999
7000 nr_pages = thp_nr_pages(newpage);
7001
7002 if (!mem_cgroup_is_root(memcg)) {
7003 page_counter_charge(&memcg->memory, nr_pages);
7004 if (do_memsw_account())
7005 page_counter_charge(&memcg->memsw, nr_pages);
7006 }
7007
7008 css_get(&memcg->css);
7009 commit_charge(newpage, memcg);
7010
7011 local_irq_save(flags);
7012 mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
7013 memcg_check_events(memcg, newpage);
7014 local_irq_restore(flags);
7015}
7016
7017DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
7018EXPORT_SYMBOL(memcg_sockets_enabled_key);
7019
7020void mem_cgroup_sk_alloc(struct sock *sk)
7021{
7022 struct mem_cgroup *memcg;
7023
7024 if (!mem_cgroup_sockets_enabled)
7025 return;
7026
7027
7028 if (in_interrupt())
7029 return;
7030
7031 rcu_read_lock();
7032 memcg = mem_cgroup_from_task(current);
7033 if (memcg == root_mem_cgroup)
7034 goto out;
7035 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
7036 goto out;
7037 if (css_tryget(&memcg->css))
7038 sk->sk_memcg = memcg;
7039out:
7040 rcu_read_unlock();
7041}
7042
7043void mem_cgroup_sk_free(struct sock *sk)
7044{
7045 if (sk->sk_memcg)
7046 css_put(&sk->sk_memcg->css);
7047}
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7058{
7059 gfp_t gfp_mask = GFP_KERNEL;
7060
7061 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
7062 struct page_counter *fail;
7063
7064 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
7065 memcg->tcpmem_pressure = 0;
7066 return true;
7067 }
7068 page_counter_charge(&memcg->tcpmem, nr_pages);
7069 memcg->tcpmem_pressure = 1;
7070 return false;
7071 }
7072
7073
7074 if (in_softirq())
7075 gfp_mask = GFP_NOWAIT;
7076
7077 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
7078
7079 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
7080 return true;
7081
7082 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
7083 return false;
7084}
7085
7086
7087
7088
7089
7090
7091void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7092{
7093 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
7094 page_counter_uncharge(&memcg->tcpmem, nr_pages);
7095 return;
7096 }
7097
7098 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
7099
7100 refill_stock(memcg, nr_pages);
7101}
7102
7103static int __init cgroup_memory(char *s)
7104{
7105 char *token;
7106
7107 while ((token = strsep(&s, ",")) != NULL) {
7108 if (!*token)
7109 continue;
7110 if (!strcmp(token, "nosocket"))
7111 cgroup_memory_nosocket = true;
7112 if (!strcmp(token, "nokmem"))
7113 cgroup_memory_nokmem = true;
7114 }
7115 return 0;
7116}
7117__setup("cgroup.memory=", cgroup_memory);
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127static int __init mem_cgroup_init(void)
7128{
7129 int cpu, node;
7130
7131
7132
7133
7134
7135
7136
7137 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
7138
7139 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
7140 memcg_hotplug_cpu_dead);
7141
7142 for_each_possible_cpu(cpu)
7143 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
7144 drain_local_stock);
7145
7146 for_each_node(node) {
7147 struct mem_cgroup_tree_per_node *rtpn;
7148
7149 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
7150 node_online(node) ? node : NUMA_NO_NODE);
7151
7152 rtpn->rb_root = RB_ROOT;
7153 rtpn->rb_rightmost = NULL;
7154 spin_lock_init(&rtpn->lock);
7155 soft_limit_tree.rb_tree_per_node[node] = rtpn;
7156 }
7157
7158 return 0;
7159}
7160subsys_initcall(mem_cgroup_init);
7161
7162#ifdef CONFIG_MEMCG_SWAP
7163static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7164{
7165 while (!refcount_inc_not_zero(&memcg->id.ref)) {
7166
7167
7168
7169
7170 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
7171 VM_BUG_ON(1);
7172 break;
7173 }
7174 memcg = parent_mem_cgroup(memcg);
7175 if (!memcg)
7176 memcg = root_mem_cgroup;
7177 }
7178 return memcg;
7179}
7180
7181
7182
7183
7184
7185
7186
7187
7188void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
7189{
7190 struct mem_cgroup *memcg, *swap_memcg;
7191 unsigned int nr_entries;
7192 unsigned short oldid;
7193
7194 VM_BUG_ON_PAGE(PageLRU(page), page);
7195 VM_BUG_ON_PAGE(page_count(page), page);
7196
7197 if (mem_cgroup_disabled())
7198 return;
7199
7200 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
7201 return;
7202
7203 memcg = page_memcg(page);
7204
7205 VM_WARN_ON_ONCE_PAGE(!memcg, page);
7206 if (!memcg)
7207 return;
7208
7209
7210
7211
7212
7213
7214 swap_memcg = mem_cgroup_id_get_online(memcg);
7215 nr_entries = thp_nr_pages(page);
7216
7217 if (nr_entries > 1)
7218 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
7219 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
7220 nr_entries);
7221 VM_BUG_ON_PAGE(oldid, page);
7222 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
7223
7224 page->memcg_data = 0;
7225
7226 if (!mem_cgroup_is_root(memcg))
7227 page_counter_uncharge(&memcg->memory, nr_entries);
7228
7229 if (!cgroup_memory_noswap && memcg != swap_memcg) {
7230 if (!mem_cgroup_is_root(swap_memcg))
7231 page_counter_charge(&swap_memcg->memsw, nr_entries);
7232 page_counter_uncharge(&memcg->memsw, nr_entries);
7233 }
7234
7235
7236
7237
7238
7239
7240
7241 VM_BUG_ON(!irqs_disabled());
7242 mem_cgroup_charge_statistics(memcg, page, -nr_entries);
7243 memcg_check_events(memcg, page);
7244
7245 css_put(&memcg->css);
7246}
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
7258{
7259 unsigned int nr_pages = thp_nr_pages(page);
7260 struct page_counter *counter;
7261 struct mem_cgroup *memcg;
7262 unsigned short oldid;
7263
7264 if (mem_cgroup_disabled())
7265 return 0;
7266
7267 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7268 return 0;
7269
7270 memcg = page_memcg(page);
7271
7272 VM_WARN_ON_ONCE_PAGE(!memcg, page);
7273 if (!memcg)
7274 return 0;
7275
7276 if (!entry.val) {
7277 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7278 return 0;
7279 }
7280
7281 memcg = mem_cgroup_id_get_online(memcg);
7282
7283 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
7284 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
7285 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7286 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7287 mem_cgroup_id_put(memcg);
7288 return -ENOMEM;
7289 }
7290
7291
7292 if (nr_pages > 1)
7293 mem_cgroup_id_get_many(memcg, nr_pages - 1);
7294 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
7295 VM_BUG_ON_PAGE(oldid, page);
7296 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
7297
7298 return 0;
7299}
7300
7301
7302
7303
7304
7305
7306void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
7307{
7308 struct mem_cgroup *memcg;
7309 unsigned short id;
7310
7311 id = swap_cgroup_record(entry, 0, nr_pages);
7312 rcu_read_lock();
7313 memcg = mem_cgroup_from_id(id);
7314 if (memcg) {
7315 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
7316 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
7317 page_counter_uncharge(&memcg->swap, nr_pages);
7318 else
7319 page_counter_uncharge(&memcg->memsw, nr_pages);
7320 }
7321 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
7322 mem_cgroup_id_put_many(memcg, nr_pages);
7323 }
7324 rcu_read_unlock();
7325}
7326
7327long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7328{
7329 long nr_swap_pages = get_nr_swap_pages();
7330
7331 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
7332 return nr_swap_pages;
7333 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7334 nr_swap_pages = min_t(long, nr_swap_pages,
7335 READ_ONCE(memcg->swap.max) -
7336 page_counter_read(&memcg->swap));
7337 return nr_swap_pages;
7338}
7339
7340bool mem_cgroup_swap_full(struct page *page)
7341{
7342 struct mem_cgroup *memcg;
7343
7344 VM_BUG_ON_PAGE(!PageLocked(page), page);
7345
7346 if (vm_swap_full())
7347 return true;
7348 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
7349 return false;
7350
7351 memcg = page_memcg(page);
7352 if (!memcg)
7353 return false;
7354
7355 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
7356 unsigned long usage = page_counter_read(&memcg->swap);
7357
7358 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7359 usage * 2 >= READ_ONCE(memcg->swap.max))
7360 return true;
7361 }
7362
7363 return false;
7364}
7365
7366static int __init setup_swap_account(char *s)
7367{
7368 if (!strcmp(s, "1"))
7369 cgroup_memory_noswap = false;
7370 else if (!strcmp(s, "0"))
7371 cgroup_memory_noswap = true;
7372 return 1;
7373}
7374__setup("swapaccount=", setup_swap_account);
7375
7376static u64 swap_current_read(struct cgroup_subsys_state *css,
7377 struct cftype *cft)
7378{
7379 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7380
7381 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7382}
7383
7384static int swap_high_show(struct seq_file *m, void *v)
7385{
7386 return seq_puts_memcg_tunable(m,
7387 READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
7388}
7389
7390static ssize_t swap_high_write(struct kernfs_open_file *of,
7391 char *buf, size_t nbytes, loff_t off)
7392{
7393 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7394 unsigned long high;
7395 int err;
7396
7397 buf = strstrip(buf);
7398 err = page_counter_memparse(buf, "max", &high);
7399 if (err)
7400 return err;
7401
7402 page_counter_set_high(&memcg->swap, high);
7403
7404 return nbytes;
7405}
7406
7407static int swap_max_show(struct seq_file *m, void *v)
7408{
7409 return seq_puts_memcg_tunable(m,
7410 READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
7411}
7412
7413static ssize_t swap_max_write(struct kernfs_open_file *of,
7414 char *buf, size_t nbytes, loff_t off)
7415{
7416 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7417 unsigned long max;
7418 int err;
7419
7420 buf = strstrip(buf);
7421 err = page_counter_memparse(buf, "max", &max);
7422 if (err)
7423 return err;
7424
7425 xchg(&memcg->swap.max, max);
7426
7427 return nbytes;
7428}
7429
7430static int swap_events_show(struct seq_file *m, void *v)
7431{
7432 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7433
7434 seq_printf(m, "high %lu\n",
7435 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
7436 seq_printf(m, "max %lu\n",
7437 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7438 seq_printf(m, "fail %lu\n",
7439 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7440
7441 return 0;
7442}
7443
7444static struct cftype swap_files[] = {
7445 {
7446 .name = "swap.current",
7447 .flags = CFTYPE_NOT_ON_ROOT,
7448 .read_u64 = swap_current_read,
7449 },
7450 {
7451 .name = "swap.high",
7452 .flags = CFTYPE_NOT_ON_ROOT,
7453 .seq_show = swap_high_show,
7454 .write = swap_high_write,
7455 },
7456 {
7457 .name = "swap.max",
7458 .flags = CFTYPE_NOT_ON_ROOT,
7459 .seq_show = swap_max_show,
7460 .write = swap_max_write,
7461 },
7462 {
7463 .name = "swap.events",
7464 .flags = CFTYPE_NOT_ON_ROOT,
7465 .file_offset = offsetof(struct mem_cgroup, swap_events_file),
7466 .seq_show = swap_events_show,
7467 },
7468 { }
7469};
7470
7471static struct cftype memsw_files[] = {
7472 {
7473 .name = "memsw.usage_in_bytes",
7474 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
7475 .read_u64 = mem_cgroup_read_u64,
7476 },
7477 {
7478 .name = "memsw.max_usage_in_bytes",
7479 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
7480 .write = mem_cgroup_reset,
7481 .read_u64 = mem_cgroup_read_u64,
7482 },
7483 {
7484 .name = "memsw.limit_in_bytes",
7485 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
7486 .write = mem_cgroup_write,
7487 .read_u64 = mem_cgroup_read_u64,
7488 },
7489 {
7490 .name = "memsw.failcnt",
7491 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
7492 .write = mem_cgroup_reset,
7493 .read_u64 = mem_cgroup_read_u64,
7494 },
7495 { },
7496};
7497
7498
7499
7500
7501
7502
7503
7504
7505static int __init mem_cgroup_swap_init(void)
7506{
7507
7508 if (mem_cgroup_disabled())
7509 cgroup_memory_noswap = true;
7510
7511 if (cgroup_memory_noswap)
7512 return 0;
7513
7514 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
7515 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
7516
7517 return 0;
7518}
7519core_initcall(mem_cgroup_swap_init);
7520
7521#endif
7522