1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/page_counter.h>
35#include <linux/memcontrol.h>
36#include <linux/cgroup.h>
37#include <linux/mm.h>
38#include <linux/sched/mm.h>
39#include <linux/shmem_fs.h>
40#include <linux/hugetlb.h>
41#include <linux/pagemap.h>
42#include <linux/smp.h>
43#include <linux/page-flags.h>
44#include <linux/backing-dev.h>
45#include <linux/bit_spinlock.h>
46#include <linux/rcupdate.h>
47#include <linux/limits.h>
48#include <linux/export.h>
49#include <linux/mutex.h>
50#include <linux/rbtree.h>
51#include <linux/slab.h>
52#include <linux/swap.h>
53#include <linux/swapops.h>
54#include <linux/spinlock.h>
55#include <linux/eventfd.h>
56#include <linux/poll.h>
57#include <linux/sort.h>
58#include <linux/fs.h>
59#include <linux/seq_file.h>
60#include <linux/vmpressure.h>
61#include <linux/mm_inline.h>
62#include <linux/swap_cgroup.h>
63#include <linux/cpu.h>
64#include <linux/oom.h>
65#include <linux/lockdep.h>
66#include <linux/file.h>
67#include <linux/tracehook.h>
68#include "internal.h"
69#include <net/sock.h>
70#include <net/ip.h>
71#include "slab.h"
72
73#include <linux/uaccess.h>
74
75#include <trace/events/vmscan.h>
76
77struct cgroup_subsys memory_cgrp_subsys __read_mostly;
78EXPORT_SYMBOL(memory_cgrp_subsys);
79
80struct mem_cgroup *root_mem_cgroup __read_mostly;
81
82#define MEM_CGROUP_RECLAIM_RETRIES 5
83
84
85static bool cgroup_memory_nosocket;
86
87
88static bool cgroup_memory_nokmem;
89
90
91#ifdef CONFIG_MEMCG_SWAP
92int do_swap_account __read_mostly;
93#else
94#define do_swap_account 0
95#endif
96
97
98static bool do_memsw_account(void)
99{
100 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
101}
102
103static const char *const mem_cgroup_lru_names[] = {
104 "inactive_anon",
105 "active_anon",
106 "inactive_file",
107 "active_file",
108 "unevictable",
109};
110
111#define THRESHOLDS_EVENTS_TARGET 128
112#define SOFTLIMIT_EVENTS_TARGET 1024
113#define NUMAINFO_EVENTS_TARGET 1024
114
115
116
117
118
119
120struct mem_cgroup_tree_per_node {
121 struct rb_root rb_root;
122 struct rb_node *rb_rightmost;
123 spinlock_t lock;
124};
125
126struct mem_cgroup_tree {
127 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
128};
129
130static struct mem_cgroup_tree soft_limit_tree __read_mostly;
131
132
133struct mem_cgroup_eventfd_list {
134 struct list_head list;
135 struct eventfd_ctx *eventfd;
136};
137
138
139
140
141struct mem_cgroup_event {
142
143
144
145 struct mem_cgroup *memcg;
146
147
148
149 struct eventfd_ctx *eventfd;
150
151
152
153 struct list_head list;
154
155
156
157
158
159 int (*register_event)(struct mem_cgroup *memcg,
160 struct eventfd_ctx *eventfd, const char *args);
161
162
163
164
165
166 void (*unregister_event)(struct mem_cgroup *memcg,
167 struct eventfd_ctx *eventfd);
168
169
170
171
172 poll_table pt;
173 wait_queue_head_t *wqh;
174 wait_queue_entry_t wait;
175 struct work_struct remove;
176};
177
178static void mem_cgroup_threshold(struct mem_cgroup *memcg);
179static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
180
181
182
183
184
185#define MOVE_ANON 0x1U
186#define MOVE_FILE 0x2U
187#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
188
189
190static struct move_charge_struct {
191 spinlock_t lock;
192 struct mm_struct *mm;
193 struct mem_cgroup *from;
194 struct mem_cgroup *to;
195 unsigned long flags;
196 unsigned long precharge;
197 unsigned long moved_charge;
198 unsigned long moved_swap;
199 struct task_struct *moving_task;
200 wait_queue_head_t waitq;
201} mc = {
202 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
203 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
204};
205
206
207
208
209
210#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
211#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
212
213enum charge_type {
214 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
215 MEM_CGROUP_CHARGE_TYPE_ANON,
216 MEM_CGROUP_CHARGE_TYPE_SWAPOUT,
217 MEM_CGROUP_CHARGE_TYPE_DROP,
218 NR_CHARGE_TYPE,
219};
220
221
222enum res_type {
223 _MEM,
224 _MEMSWAP,
225 _OOM_TYPE,
226 _KMEM,
227 _TCP,
228};
229
230#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
231#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
232#define MEMFILE_ATTR(val) ((val) & 0xffff)
233
234#define OOM_CONTROL (0)
235
236
237struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
238{
239 if (!memcg)
240 memcg = root_mem_cgroup;
241 return &memcg->vmpressure;
242}
243
244struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
245{
246 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
247}
248
249static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
250{
251 return (memcg == root_mem_cgroup);
252}
253
254#ifndef CONFIG_SLOB
255
256
257
258
259
260
261
262
263
264
265
266static DEFINE_IDA(memcg_cache_ida);
267int memcg_nr_cache_ids;
268
269
270static DECLARE_RWSEM(memcg_cache_ids_sem);
271
272void memcg_get_cache_ids(void)
273{
274 down_read(&memcg_cache_ids_sem);
275}
276
277void memcg_put_cache_ids(void)
278{
279 up_read(&memcg_cache_ids_sem);
280}
281
282
283
284
285
286
287
288
289
290
291
292
293
294#define MEMCG_CACHES_MIN_SIZE 4
295#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
296
297
298
299
300
301
302
303DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
304EXPORT_SYMBOL(memcg_kmem_enabled_key);
305
306struct workqueue_struct *memcg_kmem_cache_wq;
307
308#endif
309
310
311
312
313
314
315
316
317
318
319
320
321struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
322{
323 struct mem_cgroup *memcg;
324
325 memcg = page->mem_cgroup;
326
327 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
328 memcg = root_mem_cgroup;
329
330 return &memcg->css;
331}
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346ino_t page_cgroup_ino(struct page *page)
347{
348 struct mem_cgroup *memcg;
349 unsigned long ino = 0;
350
351 rcu_read_lock();
352 memcg = READ_ONCE(page->mem_cgroup);
353 while (memcg && !(memcg->css.flags & CSS_ONLINE))
354 memcg = parent_mem_cgroup(memcg);
355 if (memcg)
356 ino = cgroup_ino(memcg->css.cgroup);
357 rcu_read_unlock();
358 return ino;
359}
360
361static struct mem_cgroup_per_node *
362mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
363{
364 int nid = page_to_nid(page);
365
366 return memcg->nodeinfo[nid];
367}
368
369static struct mem_cgroup_tree_per_node *
370soft_limit_tree_node(int nid)
371{
372 return soft_limit_tree.rb_tree_per_node[nid];
373}
374
375static struct mem_cgroup_tree_per_node *
376soft_limit_tree_from_page(struct page *page)
377{
378 int nid = page_to_nid(page);
379
380 return soft_limit_tree.rb_tree_per_node[nid];
381}
382
383static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
384 struct mem_cgroup_tree_per_node *mctz,
385 unsigned long new_usage_in_excess)
386{
387 struct rb_node **p = &mctz->rb_root.rb_node;
388 struct rb_node *parent = NULL;
389 struct mem_cgroup_per_node *mz_node;
390 bool rightmost = true;
391
392 if (mz->on_tree)
393 return;
394
395 mz->usage_in_excess = new_usage_in_excess;
396 if (!mz->usage_in_excess)
397 return;
398 while (*p) {
399 parent = *p;
400 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
401 tree_node);
402 if (mz->usage_in_excess < mz_node->usage_in_excess) {
403 p = &(*p)->rb_left;
404 rightmost = false;
405 }
406
407
408
409
410
411 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
412 p = &(*p)->rb_right;
413 }
414
415 if (rightmost)
416 mctz->rb_rightmost = &mz->tree_node;
417
418 rb_link_node(&mz->tree_node, parent, p);
419 rb_insert_color(&mz->tree_node, &mctz->rb_root);
420 mz->on_tree = true;
421}
422
423static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
424 struct mem_cgroup_tree_per_node *mctz)
425{
426 if (!mz->on_tree)
427 return;
428
429 if (&mz->tree_node == mctz->rb_rightmost)
430 mctz->rb_rightmost = rb_prev(&mz->tree_node);
431
432 rb_erase(&mz->tree_node, &mctz->rb_root);
433 mz->on_tree = false;
434}
435
436static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
437 struct mem_cgroup_tree_per_node *mctz)
438{
439 unsigned long flags;
440
441 spin_lock_irqsave(&mctz->lock, flags);
442 __mem_cgroup_remove_exceeded(mz, mctz);
443 spin_unlock_irqrestore(&mctz->lock, flags);
444}
445
446static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
447{
448 unsigned long nr_pages = page_counter_read(&memcg->memory);
449 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
450 unsigned long excess = 0;
451
452 if (nr_pages > soft_limit)
453 excess = nr_pages - soft_limit;
454
455 return excess;
456}
457
458static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
459{
460 unsigned long excess;
461 struct mem_cgroup_per_node *mz;
462 struct mem_cgroup_tree_per_node *mctz;
463
464 mctz = soft_limit_tree_from_page(page);
465 if (!mctz)
466 return;
467
468
469
470
471 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
472 mz = mem_cgroup_page_nodeinfo(memcg, page);
473 excess = soft_limit_excess(memcg);
474
475
476
477
478 if (excess || mz->on_tree) {
479 unsigned long flags;
480
481 spin_lock_irqsave(&mctz->lock, flags);
482
483 if (mz->on_tree)
484 __mem_cgroup_remove_exceeded(mz, mctz);
485
486
487
488
489 __mem_cgroup_insert_exceeded(mz, mctz, excess);
490 spin_unlock_irqrestore(&mctz->lock, flags);
491 }
492 }
493}
494
495static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
496{
497 struct mem_cgroup_tree_per_node *mctz;
498 struct mem_cgroup_per_node *mz;
499 int nid;
500
501 for_each_node(nid) {
502 mz = mem_cgroup_nodeinfo(memcg, nid);
503 mctz = soft_limit_tree_node(nid);
504 if (mctz)
505 mem_cgroup_remove_exceeded(mz, mctz);
506 }
507}
508
509static struct mem_cgroup_per_node *
510__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
511{
512 struct mem_cgroup_per_node *mz;
513
514retry:
515 mz = NULL;
516 if (!mctz->rb_rightmost)
517 goto done;
518
519 mz = rb_entry(mctz->rb_rightmost,
520 struct mem_cgroup_per_node, tree_node);
521
522
523
524
525
526 __mem_cgroup_remove_exceeded(mz, mctz);
527 if (!soft_limit_excess(mz->memcg) ||
528 !css_tryget_online(&mz->memcg->css))
529 goto retry;
530done:
531 return mz;
532}
533
534static struct mem_cgroup_per_node *
535mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
536{
537 struct mem_cgroup_per_node *mz;
538
539 spin_lock_irq(&mctz->lock);
540 mz = __mem_cgroup_largest_soft_limit_node(mctz);
541 spin_unlock_irq(&mctz->lock);
542 return mz;
543}
544
545static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
546 int event)
547{
548 return atomic_long_read(&memcg->events[event]);
549}
550
551static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
552 struct page *page,
553 bool compound, int nr_pages)
554{
555
556
557
558
559 if (PageAnon(page))
560 __mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
561 else {
562 __mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
563 if (PageSwapBacked(page))
564 __mod_memcg_state(memcg, NR_SHMEM, nr_pages);
565 }
566
567 if (compound) {
568 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
569 __mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
570 }
571
572
573 if (nr_pages > 0)
574 __count_memcg_events(memcg, PGPGIN, 1);
575 else {
576 __count_memcg_events(memcg, PGPGOUT, 1);
577 nr_pages = -nr_pages;
578 }
579
580 __this_cpu_add(memcg->stat_cpu->nr_page_events, nr_pages);
581}
582
583unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
584 int nid, unsigned int lru_mask)
585{
586 struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
587 unsigned long nr = 0;
588 enum lru_list lru;
589
590 VM_BUG_ON((unsigned)nid >= nr_node_ids);
591
592 for_each_lru(lru) {
593 if (!(BIT(lru) & lru_mask))
594 continue;
595 nr += mem_cgroup_get_lru_size(lruvec, lru);
596 }
597 return nr;
598}
599
600static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
601 unsigned int lru_mask)
602{
603 unsigned long nr = 0;
604 int nid;
605
606 for_each_node_state(nid, N_MEMORY)
607 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
608 return nr;
609}
610
611static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
612 enum mem_cgroup_events_target target)
613{
614 unsigned long val, next;
615
616 val = __this_cpu_read(memcg->stat_cpu->nr_page_events);
617 next = __this_cpu_read(memcg->stat_cpu->targets[target]);
618
619 if ((long)(next - val) < 0) {
620 switch (target) {
621 case MEM_CGROUP_TARGET_THRESH:
622 next = val + THRESHOLDS_EVENTS_TARGET;
623 break;
624 case MEM_CGROUP_TARGET_SOFTLIMIT:
625 next = val + SOFTLIMIT_EVENTS_TARGET;
626 break;
627 case MEM_CGROUP_TARGET_NUMAINFO:
628 next = val + NUMAINFO_EVENTS_TARGET;
629 break;
630 default:
631 break;
632 }
633 __this_cpu_write(memcg->stat_cpu->targets[target], next);
634 return true;
635 }
636 return false;
637}
638
639
640
641
642
643static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
644{
645
646 if (unlikely(mem_cgroup_event_ratelimit(memcg,
647 MEM_CGROUP_TARGET_THRESH))) {
648 bool do_softlimit;
649 bool do_numainfo __maybe_unused;
650
651 do_softlimit = mem_cgroup_event_ratelimit(memcg,
652 MEM_CGROUP_TARGET_SOFTLIMIT);
653#if MAX_NUMNODES > 1
654 do_numainfo = mem_cgroup_event_ratelimit(memcg,
655 MEM_CGROUP_TARGET_NUMAINFO);
656#endif
657 mem_cgroup_threshold(memcg);
658 if (unlikely(do_softlimit))
659 mem_cgroup_update_tree(memcg, page);
660#if MAX_NUMNODES > 1
661 if (unlikely(do_numainfo))
662 atomic_inc(&memcg->numainfo_events);
663#endif
664 }
665}
666
667struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
668{
669
670
671
672
673
674 if (unlikely(!p))
675 return NULL;
676
677 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
678}
679EXPORT_SYMBOL(mem_cgroup_from_task);
680
681static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
682{
683 struct mem_cgroup *memcg = NULL;
684
685 rcu_read_lock();
686 do {
687
688
689
690
691
692 if (unlikely(!mm))
693 memcg = root_mem_cgroup;
694 else {
695 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
696 if (unlikely(!memcg))
697 memcg = root_mem_cgroup;
698 }
699 } while (!css_tryget_online(&memcg->css));
700 rcu_read_unlock();
701 return memcg;
702}
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
722 struct mem_cgroup *prev,
723 struct mem_cgroup_reclaim_cookie *reclaim)
724{
725 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
726 struct cgroup_subsys_state *css = NULL;
727 struct mem_cgroup *memcg = NULL;
728 struct mem_cgroup *pos = NULL;
729
730 if (mem_cgroup_disabled())
731 return NULL;
732
733 if (!root)
734 root = root_mem_cgroup;
735
736 if (prev && !reclaim)
737 pos = prev;
738
739 if (!root->use_hierarchy && root != root_mem_cgroup) {
740 if (prev)
741 goto out;
742 return root;
743 }
744
745 rcu_read_lock();
746
747 if (reclaim) {
748 struct mem_cgroup_per_node *mz;
749
750 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
751 iter = &mz->iter[reclaim->priority];
752
753 if (prev && reclaim->generation != iter->generation)
754 goto out_unlock;
755
756 while (1) {
757 pos = READ_ONCE(iter->position);
758 if (!pos || css_tryget(&pos->css))
759 break;
760
761
762
763
764
765
766
767
768 (void)cmpxchg(&iter->position, pos, NULL);
769 }
770 }
771
772 if (pos)
773 css = &pos->css;
774
775 for (;;) {
776 css = css_next_descendant_pre(css, &root->css);
777 if (!css) {
778
779
780
781
782
783
784 if (!prev)
785 continue;
786 break;
787 }
788
789
790
791
792
793
794 memcg = mem_cgroup_from_css(css);
795
796 if (css == &root->css)
797 break;
798
799 if (css_tryget(css))
800 break;
801
802 memcg = NULL;
803 }
804
805 if (reclaim) {
806
807
808
809
810
811 (void)cmpxchg(&iter->position, pos, memcg);
812
813 if (pos)
814 css_put(&pos->css);
815
816 if (!memcg)
817 iter->generation++;
818 else if (!prev)
819 reclaim->generation = iter->generation;
820 }
821
822out_unlock:
823 rcu_read_unlock();
824out:
825 if (prev && prev != root)
826 css_put(&prev->css);
827
828 return memcg;
829}
830
831
832
833
834
835
836void mem_cgroup_iter_break(struct mem_cgroup *root,
837 struct mem_cgroup *prev)
838{
839 if (!root)
840 root = root_mem_cgroup;
841 if (prev && prev != root)
842 css_put(&prev->css);
843}
844
845static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
846{
847 struct mem_cgroup *memcg = dead_memcg;
848 struct mem_cgroup_reclaim_iter *iter;
849 struct mem_cgroup_per_node *mz;
850 int nid;
851 int i;
852
853 while ((memcg = parent_mem_cgroup(memcg))) {
854 for_each_node(nid) {
855 mz = mem_cgroup_nodeinfo(memcg, nid);
856 for (i = 0; i <= DEF_PRIORITY; i++) {
857 iter = &mz->iter[i];
858 cmpxchg(&iter->position,
859 dead_memcg, NULL);
860 }
861 }
862 }
863}
864
865
866
867
868
869
870#define for_each_mem_cgroup_tree(iter, root) \
871 for (iter = mem_cgroup_iter(root, NULL, NULL); \
872 iter != NULL; \
873 iter = mem_cgroup_iter(root, iter, NULL))
874
875#define for_each_mem_cgroup(iter) \
876 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
877 iter != NULL; \
878 iter = mem_cgroup_iter(NULL, iter, NULL))
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
894 int (*fn)(struct task_struct *, void *), void *arg)
895{
896 struct mem_cgroup *iter;
897 int ret = 0;
898
899 BUG_ON(memcg == root_mem_cgroup);
900
901 for_each_mem_cgroup_tree(iter, memcg) {
902 struct css_task_iter it;
903 struct task_struct *task;
904
905 css_task_iter_start(&iter->css, 0, &it);
906 while (!ret && (task = css_task_iter_next(&it)))
907 ret = fn(task, arg);
908 css_task_iter_end(&it);
909 if (ret) {
910 mem_cgroup_iter_break(memcg, iter);
911 break;
912 }
913 }
914 return ret;
915}
916
917
918
919
920
921
922
923
924
925
926struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
927{
928 struct mem_cgroup_per_node *mz;
929 struct mem_cgroup *memcg;
930 struct lruvec *lruvec;
931
932 if (mem_cgroup_disabled()) {
933 lruvec = &pgdat->lruvec;
934 goto out;
935 }
936
937 memcg = page->mem_cgroup;
938
939
940
941
942 if (!memcg)
943 memcg = root_mem_cgroup;
944
945 mz = mem_cgroup_page_nodeinfo(memcg, page);
946 lruvec = &mz->lruvec;
947out:
948
949
950
951
952
953 if (unlikely(lruvec->pgdat != pgdat))
954 lruvec->pgdat = pgdat;
955 return lruvec;
956}
957
958
959
960
961
962
963
964
965
966
967
968
969void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
970 int zid, int nr_pages)
971{
972 struct mem_cgroup_per_node *mz;
973 unsigned long *lru_size;
974 long size;
975
976 if (mem_cgroup_disabled())
977 return;
978
979 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
980 lru_size = &mz->lru_zone_size[zid][lru];
981
982 if (nr_pages < 0)
983 *lru_size += nr_pages;
984
985 size = *lru_size;
986 if (WARN_ONCE(size < 0,
987 "%s(%p, %d, %d): lru_size %ld\n",
988 __func__, lruvec, lru, nr_pages, size)) {
989 VM_BUG_ON(1);
990 *lru_size = 0;
991 }
992
993 if (nr_pages > 0)
994 *lru_size += nr_pages;
995}
996
997bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
998{
999 struct mem_cgroup *task_memcg;
1000 struct task_struct *p;
1001 bool ret;
1002
1003 p = find_lock_task_mm(task);
1004 if (p) {
1005 task_memcg = get_mem_cgroup_from_mm(p->mm);
1006 task_unlock(p);
1007 } else {
1008
1009
1010
1011
1012
1013 rcu_read_lock();
1014 task_memcg = mem_cgroup_from_task(task);
1015 css_get(&task_memcg->css);
1016 rcu_read_unlock();
1017 }
1018 ret = mem_cgroup_is_descendant(task_memcg, memcg);
1019 css_put(&task_memcg->css);
1020 return ret;
1021}
1022
1023
1024
1025
1026
1027
1028
1029
1030static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1031{
1032 unsigned long margin = 0;
1033 unsigned long count;
1034 unsigned long limit;
1035
1036 count = page_counter_read(&memcg->memory);
1037 limit = READ_ONCE(memcg->memory.limit);
1038 if (count < limit)
1039 margin = limit - count;
1040
1041 if (do_memsw_account()) {
1042 count = page_counter_read(&memcg->memsw);
1043 limit = READ_ONCE(memcg->memsw.limit);
1044 if (count <= limit)
1045 margin = min(margin, limit - count);
1046 else
1047 margin = 0;
1048 }
1049
1050 return margin;
1051}
1052
1053
1054
1055
1056
1057
1058
1059
1060static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1061{
1062 struct mem_cgroup *from;
1063 struct mem_cgroup *to;
1064 bool ret = false;
1065
1066
1067
1068
1069 spin_lock(&mc.lock);
1070 from = mc.from;
1071 to = mc.to;
1072 if (!from)
1073 goto unlock;
1074
1075 ret = mem_cgroup_is_descendant(from, memcg) ||
1076 mem_cgroup_is_descendant(to, memcg);
1077unlock:
1078 spin_unlock(&mc.lock);
1079 return ret;
1080}
1081
1082static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1083{
1084 if (mc.moving_task && current != mc.moving_task) {
1085 if (mem_cgroup_under_move(memcg)) {
1086 DEFINE_WAIT(wait);
1087 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1088
1089 if (mc.moving_task)
1090 schedule();
1091 finish_wait(&mc.waitq, &wait);
1092 return true;
1093 }
1094 }
1095 return false;
1096}
1097
1098static const unsigned int memcg1_stats[] = {
1099 MEMCG_CACHE,
1100 MEMCG_RSS,
1101 MEMCG_RSS_HUGE,
1102 NR_SHMEM,
1103 NR_FILE_MAPPED,
1104 NR_FILE_DIRTY,
1105 NR_WRITEBACK,
1106 MEMCG_SWAP,
1107};
1108
1109static const char *const memcg1_stat_names[] = {
1110 "cache",
1111 "rss",
1112 "rss_huge",
1113 "shmem",
1114 "mapped_file",
1115 "dirty",
1116 "writeback",
1117 "swap",
1118};
1119
1120#define K(x) ((x) << (PAGE_SHIFT-10))
1121
1122
1123
1124
1125
1126
1127
1128
1129void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1130{
1131 struct mem_cgroup *iter;
1132 unsigned int i;
1133
1134 rcu_read_lock();
1135
1136 if (p) {
1137 pr_info("Task in ");
1138 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1139 pr_cont(" killed as a result of limit of ");
1140 } else {
1141 pr_info("Memory limit reached of cgroup ");
1142 }
1143
1144 pr_cont_cgroup_path(memcg->css.cgroup);
1145 pr_cont("\n");
1146
1147 rcu_read_unlock();
1148
1149 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1150 K((u64)page_counter_read(&memcg->memory)),
1151 K((u64)memcg->memory.limit), memcg->memory.failcnt);
1152 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1153 K((u64)page_counter_read(&memcg->memsw)),
1154 K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1155 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1156 K((u64)page_counter_read(&memcg->kmem)),
1157 K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1158
1159 for_each_mem_cgroup_tree(iter, memcg) {
1160 pr_info("Memory cgroup stats for ");
1161 pr_cont_cgroup_path(iter->css.cgroup);
1162 pr_cont(":");
1163
1164 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
1165 if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1166 continue;
1167 pr_cont(" %s:%luKB", memcg1_stat_names[i],
1168 K(memcg_page_state(iter, memcg1_stats[i])));
1169 }
1170
1171 for (i = 0; i < NR_LRU_LISTS; i++)
1172 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1173 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1174
1175 pr_cont("\n");
1176 }
1177}
1178
1179
1180
1181
1182unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
1183{
1184 unsigned long limit;
1185
1186 limit = memcg->memory.limit;
1187 if (mem_cgroup_swappiness(memcg)) {
1188 unsigned long memsw_limit;
1189 unsigned long swap_limit;
1190
1191 memsw_limit = memcg->memsw.limit;
1192 swap_limit = memcg->swap.limit;
1193 swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
1194 limit = min(limit + swap_limit, memsw_limit);
1195 }
1196 return limit;
1197}
1198
1199static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1200 int order)
1201{
1202 struct oom_control oc = {
1203 .zonelist = NULL,
1204 .nodemask = NULL,
1205 .memcg = memcg,
1206 .gfp_mask = gfp_mask,
1207 .order = order,
1208 };
1209 bool ret;
1210
1211 mutex_lock(&oom_lock);
1212 ret = out_of_memory(&oc);
1213 mutex_unlock(&oom_lock);
1214 return ret;
1215}
1216
1217#if MAX_NUMNODES > 1
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1230 int nid, bool noswap)
1231{
1232 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1233 return true;
1234 if (noswap || !total_swap_pages)
1235 return false;
1236 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1237 return true;
1238 return false;
1239
1240}
1241
1242
1243
1244
1245
1246
1247
1248static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1249{
1250 int nid;
1251
1252
1253
1254
1255 if (!atomic_read(&memcg->numainfo_events))
1256 return;
1257 if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1258 return;
1259
1260
1261 memcg->scan_nodes = node_states[N_MEMORY];
1262
1263 for_each_node_mask(nid, node_states[N_MEMORY]) {
1264
1265 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1266 node_clear(nid, memcg->scan_nodes);
1267 }
1268
1269 atomic_set(&memcg->numainfo_events, 0);
1270 atomic_set(&memcg->numainfo_updating, 0);
1271}
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1286{
1287 int node;
1288
1289 mem_cgroup_may_update_nodemask(memcg);
1290 node = memcg->last_scanned_node;
1291
1292 node = next_node_in(node, memcg->scan_nodes);
1293
1294
1295
1296
1297
1298 if (unlikely(node == MAX_NUMNODES))
1299 node = numa_node_id();
1300
1301 memcg->last_scanned_node = node;
1302 return node;
1303}
1304#else
1305int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1306{
1307 return 0;
1308}
1309#endif
1310
1311static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1312 pg_data_t *pgdat,
1313 gfp_t gfp_mask,
1314 unsigned long *total_scanned)
1315{
1316 struct mem_cgroup *victim = NULL;
1317 int total = 0;
1318 int loop = 0;
1319 unsigned long excess;
1320 unsigned long nr_scanned;
1321 struct mem_cgroup_reclaim_cookie reclaim = {
1322 .pgdat = pgdat,
1323 .priority = 0,
1324 };
1325
1326 excess = soft_limit_excess(root_memcg);
1327
1328 while (1) {
1329 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1330 if (!victim) {
1331 loop++;
1332 if (loop >= 2) {
1333
1334
1335
1336
1337
1338 if (!total)
1339 break;
1340
1341
1342
1343
1344
1345
1346 if (total >= (excess >> 2) ||
1347 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1348 break;
1349 }
1350 continue;
1351 }
1352 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1353 pgdat, &nr_scanned);
1354 *total_scanned += nr_scanned;
1355 if (!soft_limit_excess(root_memcg))
1356 break;
1357 }
1358 mem_cgroup_iter_break(root_memcg, victim);
1359 return total;
1360}
1361
1362#ifdef CONFIG_LOCKDEP
1363static struct lockdep_map memcg_oom_lock_dep_map = {
1364 .name = "memcg_oom_lock",
1365};
1366#endif
1367
1368static DEFINE_SPINLOCK(memcg_oom_lock);
1369
1370
1371
1372
1373
1374static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1375{
1376 struct mem_cgroup *iter, *failed = NULL;
1377
1378 spin_lock(&memcg_oom_lock);
1379
1380 for_each_mem_cgroup_tree(iter, memcg) {
1381 if (iter->oom_lock) {
1382
1383
1384
1385
1386 failed = iter;
1387 mem_cgroup_iter_break(memcg, iter);
1388 break;
1389 } else
1390 iter->oom_lock = true;
1391 }
1392
1393 if (failed) {
1394
1395
1396
1397
1398 for_each_mem_cgroup_tree(iter, memcg) {
1399 if (iter == failed) {
1400 mem_cgroup_iter_break(memcg, iter);
1401 break;
1402 }
1403 iter->oom_lock = false;
1404 }
1405 } else
1406 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1407
1408 spin_unlock(&memcg_oom_lock);
1409
1410 return !failed;
1411}
1412
1413static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1414{
1415 struct mem_cgroup *iter;
1416
1417 spin_lock(&memcg_oom_lock);
1418 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1419 for_each_mem_cgroup_tree(iter, memcg)
1420 iter->oom_lock = false;
1421 spin_unlock(&memcg_oom_lock);
1422}
1423
1424static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1425{
1426 struct mem_cgroup *iter;
1427
1428 spin_lock(&memcg_oom_lock);
1429 for_each_mem_cgroup_tree(iter, memcg)
1430 iter->under_oom++;
1431 spin_unlock(&memcg_oom_lock);
1432}
1433
1434static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1435{
1436 struct mem_cgroup *iter;
1437
1438
1439
1440
1441
1442 spin_lock(&memcg_oom_lock);
1443 for_each_mem_cgroup_tree(iter, memcg)
1444 if (iter->under_oom > 0)
1445 iter->under_oom--;
1446 spin_unlock(&memcg_oom_lock);
1447}
1448
1449static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1450
1451struct oom_wait_info {
1452 struct mem_cgroup *memcg;
1453 wait_queue_entry_t wait;
1454};
1455
1456static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1457 unsigned mode, int sync, void *arg)
1458{
1459 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1460 struct mem_cgroup *oom_wait_memcg;
1461 struct oom_wait_info *oom_wait_info;
1462
1463 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1464 oom_wait_memcg = oom_wait_info->memcg;
1465
1466 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1467 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1468 return 0;
1469 return autoremove_wake_function(wait, mode, sync, arg);
1470}
1471
1472static void memcg_oom_recover(struct mem_cgroup *memcg)
1473{
1474
1475
1476
1477
1478
1479
1480
1481
1482 if (memcg && memcg->under_oom)
1483 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1484}
1485
1486static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1487{
1488 if (!current->memcg_may_oom || order > PAGE_ALLOC_COSTLY_ORDER)
1489 return;
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504 css_get(&memcg->css);
1505 current->memcg_in_oom = memcg;
1506 current->memcg_oom_gfp_mask = mask;
1507 current->memcg_oom_order = order;
1508}
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527bool mem_cgroup_oom_synchronize(bool handle)
1528{
1529 struct mem_cgroup *memcg = current->memcg_in_oom;
1530 struct oom_wait_info owait;
1531 bool locked;
1532
1533
1534 if (!memcg)
1535 return false;
1536
1537 if (!handle)
1538 goto cleanup;
1539
1540 owait.memcg = memcg;
1541 owait.wait.flags = 0;
1542 owait.wait.func = memcg_oom_wake_function;
1543 owait.wait.private = current;
1544 INIT_LIST_HEAD(&owait.wait.entry);
1545
1546 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1547 mem_cgroup_mark_under_oom(memcg);
1548
1549 locked = mem_cgroup_oom_trylock(memcg);
1550
1551 if (locked)
1552 mem_cgroup_oom_notify(memcg);
1553
1554 if (locked && !memcg->oom_kill_disable) {
1555 mem_cgroup_unmark_under_oom(memcg);
1556 finish_wait(&memcg_oom_waitq, &owait.wait);
1557 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1558 current->memcg_oom_order);
1559 } else {
1560 schedule();
1561 mem_cgroup_unmark_under_oom(memcg);
1562 finish_wait(&memcg_oom_waitq, &owait.wait);
1563 }
1564
1565 if (locked) {
1566 mem_cgroup_oom_unlock(memcg);
1567
1568
1569
1570
1571
1572 memcg_oom_recover(memcg);
1573 }
1574cleanup:
1575 current->memcg_in_oom = NULL;
1576 css_put(&memcg->css);
1577 return true;
1578}
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591struct mem_cgroup *lock_page_memcg(struct page *page)
1592{
1593 struct mem_cgroup *memcg;
1594 unsigned long flags;
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607 rcu_read_lock();
1608
1609 if (mem_cgroup_disabled())
1610 return NULL;
1611again:
1612 memcg = page->mem_cgroup;
1613 if (unlikely(!memcg))
1614 return NULL;
1615
1616 if (atomic_read(&memcg->moving_account) <= 0)
1617 return memcg;
1618
1619 spin_lock_irqsave(&memcg->move_lock, flags);
1620 if (memcg != page->mem_cgroup) {
1621 spin_unlock_irqrestore(&memcg->move_lock, flags);
1622 goto again;
1623 }
1624
1625
1626
1627
1628
1629
1630 memcg->move_lock_task = current;
1631 memcg->move_lock_flags = flags;
1632
1633 return memcg;
1634}
1635EXPORT_SYMBOL(lock_page_memcg);
1636
1637
1638
1639
1640
1641
1642
1643void __unlock_page_memcg(struct mem_cgroup *memcg)
1644{
1645 if (memcg && memcg->move_lock_task == current) {
1646 unsigned long flags = memcg->move_lock_flags;
1647
1648 memcg->move_lock_task = NULL;
1649 memcg->move_lock_flags = 0;
1650
1651 spin_unlock_irqrestore(&memcg->move_lock, flags);
1652 }
1653
1654 rcu_read_unlock();
1655}
1656
1657
1658
1659
1660
1661void unlock_page_memcg(struct page *page)
1662{
1663 __unlock_page_memcg(page->mem_cgroup);
1664}
1665EXPORT_SYMBOL(unlock_page_memcg);
1666
1667struct memcg_stock_pcp {
1668 struct mem_cgroup *cached;
1669 unsigned int nr_pages;
1670 struct work_struct work;
1671 unsigned long flags;
1672#define FLUSHING_CACHED_CHARGE 0
1673};
1674static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1675static DEFINE_MUTEX(percpu_charge_mutex);
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1689{
1690 struct memcg_stock_pcp *stock;
1691 unsigned long flags;
1692 bool ret = false;
1693
1694 if (nr_pages > MEMCG_CHARGE_BATCH)
1695 return ret;
1696
1697 local_irq_save(flags);
1698
1699 stock = this_cpu_ptr(&memcg_stock);
1700 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1701 stock->nr_pages -= nr_pages;
1702 ret = true;
1703 }
1704
1705 local_irq_restore(flags);
1706
1707 return ret;
1708}
1709
1710
1711
1712
1713static void drain_stock(struct memcg_stock_pcp *stock)
1714{
1715 struct mem_cgroup *old = stock->cached;
1716
1717 if (stock->nr_pages) {
1718 page_counter_uncharge(&old->memory, stock->nr_pages);
1719 if (do_memsw_account())
1720 page_counter_uncharge(&old->memsw, stock->nr_pages);
1721 css_put_many(&old->css, stock->nr_pages);
1722 stock->nr_pages = 0;
1723 }
1724 stock->cached = NULL;
1725}
1726
1727static void drain_local_stock(struct work_struct *dummy)
1728{
1729 struct memcg_stock_pcp *stock;
1730 unsigned long flags;
1731
1732
1733
1734
1735
1736 local_irq_save(flags);
1737
1738 stock = this_cpu_ptr(&memcg_stock);
1739 drain_stock(stock);
1740 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1741
1742 local_irq_restore(flags);
1743}
1744
1745
1746
1747
1748
1749static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1750{
1751 struct memcg_stock_pcp *stock;
1752 unsigned long flags;
1753
1754 local_irq_save(flags);
1755
1756 stock = this_cpu_ptr(&memcg_stock);
1757 if (stock->cached != memcg) {
1758 drain_stock(stock);
1759 stock->cached = memcg;
1760 }
1761 stock->nr_pages += nr_pages;
1762
1763 if (stock->nr_pages > MEMCG_CHARGE_BATCH)
1764 drain_stock(stock);
1765
1766 local_irq_restore(flags);
1767}
1768
1769
1770
1771
1772
1773static void drain_all_stock(struct mem_cgroup *root_memcg)
1774{
1775 int cpu, curcpu;
1776
1777
1778 if (!mutex_trylock(&percpu_charge_mutex))
1779 return;
1780
1781
1782
1783
1784
1785
1786 curcpu = get_cpu();
1787 for_each_online_cpu(cpu) {
1788 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1789 struct mem_cgroup *memcg;
1790
1791 memcg = stock->cached;
1792 if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
1793 continue;
1794 if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
1795 css_put(&memcg->css);
1796 continue;
1797 }
1798 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
1799 if (cpu == curcpu)
1800 drain_local_stock(&stock->work);
1801 else
1802 schedule_work_on(cpu, &stock->work);
1803 }
1804 css_put(&memcg->css);
1805 }
1806 put_cpu();
1807 mutex_unlock(&percpu_charge_mutex);
1808}
1809
1810static int memcg_hotplug_cpu_dead(unsigned int cpu)
1811{
1812 struct memcg_stock_pcp *stock;
1813 struct mem_cgroup *memcg;
1814
1815 stock = &per_cpu(memcg_stock, cpu);
1816 drain_stock(stock);
1817
1818 for_each_mem_cgroup(memcg) {
1819 int i;
1820
1821 for (i = 0; i < MEMCG_NR_STAT; i++) {
1822 int nid;
1823 long x;
1824
1825 x = this_cpu_xchg(memcg->stat_cpu->count[i], 0);
1826 if (x)
1827 atomic_long_add(x, &memcg->stat[i]);
1828
1829 if (i >= NR_VM_NODE_STAT_ITEMS)
1830 continue;
1831
1832 for_each_node(nid) {
1833 struct mem_cgroup_per_node *pn;
1834
1835 pn = mem_cgroup_nodeinfo(memcg, nid);
1836 x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
1837 if (x)
1838 atomic_long_add(x, &pn->lruvec_stat[i]);
1839 }
1840 }
1841
1842 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
1843 long x;
1844
1845 x = this_cpu_xchg(memcg->stat_cpu->events[i], 0);
1846 if (x)
1847 atomic_long_add(x, &memcg->events[i]);
1848 }
1849 }
1850
1851 return 0;
1852}
1853
1854static void reclaim_high(struct mem_cgroup *memcg,
1855 unsigned int nr_pages,
1856 gfp_t gfp_mask)
1857{
1858 do {
1859 if (page_counter_read(&memcg->memory) <= memcg->high)
1860 continue;
1861 memcg_memory_event(memcg, MEMCG_HIGH);
1862 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
1863 } while ((memcg = parent_mem_cgroup(memcg)));
1864}
1865
1866static void high_work_func(struct work_struct *work)
1867{
1868 struct mem_cgroup *memcg;
1869
1870 memcg = container_of(work, struct mem_cgroup, high_work);
1871 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
1872}
1873
1874
1875
1876
1877
1878void mem_cgroup_handle_over_high(void)
1879{
1880 unsigned int nr_pages = current->memcg_nr_pages_over_high;
1881 struct mem_cgroup *memcg;
1882
1883 if (likely(!nr_pages))
1884 return;
1885
1886 memcg = get_mem_cgroup_from_mm(current->mm);
1887 reclaim_high(memcg, nr_pages, GFP_KERNEL);
1888 css_put(&memcg->css);
1889 current->memcg_nr_pages_over_high = 0;
1890}
1891
1892static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
1893 unsigned int nr_pages)
1894{
1895 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
1896 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1897 struct mem_cgroup *mem_over_limit;
1898 struct page_counter *counter;
1899 unsigned long nr_reclaimed;
1900 bool may_swap = true;
1901 bool drained = false;
1902
1903 if (mem_cgroup_is_root(memcg))
1904 return 0;
1905retry:
1906 if (consume_stock(memcg, nr_pages))
1907 return 0;
1908
1909 if (!do_memsw_account() ||
1910 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
1911 if (page_counter_try_charge(&memcg->memory, batch, &counter))
1912 goto done_restock;
1913 if (do_memsw_account())
1914 page_counter_uncharge(&memcg->memsw, batch);
1915 mem_over_limit = mem_cgroup_from_counter(counter, memory);
1916 } else {
1917 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1918 may_swap = false;
1919 }
1920
1921 if (batch > nr_pages) {
1922 batch = nr_pages;
1923 goto retry;
1924 }
1925
1926
1927
1928
1929
1930
1931
1932 if (unlikely(tsk_is_oom_victim(current) ||
1933 fatal_signal_pending(current) ||
1934 current->flags & PF_EXITING))
1935 goto force;
1936
1937
1938
1939
1940
1941
1942
1943 if (unlikely(current->flags & PF_MEMALLOC))
1944 goto force;
1945
1946 if (unlikely(task_in_memcg_oom(current)))
1947 goto nomem;
1948
1949 if (!gfpflags_allow_blocking(gfp_mask))
1950 goto nomem;
1951
1952 memcg_memory_event(mem_over_limit, MEMCG_MAX);
1953
1954 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
1955 gfp_mask, may_swap);
1956
1957 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
1958 goto retry;
1959
1960 if (!drained) {
1961 drain_all_stock(mem_over_limit);
1962 drained = true;
1963 goto retry;
1964 }
1965
1966 if (gfp_mask & __GFP_NORETRY)
1967 goto nomem;
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
1978 goto retry;
1979
1980
1981
1982
1983 if (mem_cgroup_wait_acct_move(mem_over_limit))
1984 goto retry;
1985
1986 if (nr_retries--)
1987 goto retry;
1988
1989 if (gfp_mask & __GFP_NOFAIL)
1990 goto force;
1991
1992 if (fatal_signal_pending(current))
1993 goto force;
1994
1995 memcg_memory_event(mem_over_limit, MEMCG_OOM);
1996
1997 mem_cgroup_oom(mem_over_limit, gfp_mask,
1998 get_order(nr_pages * PAGE_SIZE));
1999nomem:
2000 if (!(gfp_mask & __GFP_NOFAIL))
2001 return -ENOMEM;
2002force:
2003
2004
2005
2006
2007
2008 page_counter_charge(&memcg->memory, nr_pages);
2009 if (do_memsw_account())
2010 page_counter_charge(&memcg->memsw, nr_pages);
2011 css_get_many(&memcg->css, nr_pages);
2012
2013 return 0;
2014
2015done_restock:
2016 css_get_many(&memcg->css, batch);
2017 if (batch > nr_pages)
2018 refill_stock(memcg, batch - nr_pages);
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029 do {
2030 if (page_counter_read(&memcg->memory) > memcg->high) {
2031
2032 if (in_interrupt()) {
2033 schedule_work(&memcg->high_work);
2034 break;
2035 }
2036 current->memcg_nr_pages_over_high += batch;
2037 set_notify_resume(current);
2038 break;
2039 }
2040 } while ((memcg = parent_mem_cgroup(memcg)));
2041
2042 return 0;
2043}
2044
2045static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2046{
2047 if (mem_cgroup_is_root(memcg))
2048 return;
2049
2050 page_counter_uncharge(&memcg->memory, nr_pages);
2051 if (do_memsw_account())
2052 page_counter_uncharge(&memcg->memsw, nr_pages);
2053
2054 css_put_many(&memcg->css, nr_pages);
2055}
2056
2057static void lock_page_lru(struct page *page, int *isolated)
2058{
2059 struct zone *zone = page_zone(page);
2060
2061 spin_lock_irq(zone_lru_lock(zone));
2062 if (PageLRU(page)) {
2063 struct lruvec *lruvec;
2064
2065 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2066 ClearPageLRU(page);
2067 del_page_from_lru_list(page, lruvec, page_lru(page));
2068 *isolated = 1;
2069 } else
2070 *isolated = 0;
2071}
2072
2073static void unlock_page_lru(struct page *page, int isolated)
2074{
2075 struct zone *zone = page_zone(page);
2076
2077 if (isolated) {
2078 struct lruvec *lruvec;
2079
2080 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2081 VM_BUG_ON_PAGE(PageLRU(page), page);
2082 SetPageLRU(page);
2083 add_page_to_lru_list(page, lruvec, page_lru(page));
2084 }
2085 spin_unlock_irq(zone_lru_lock(zone));
2086}
2087
2088static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2089 bool lrucare)
2090{
2091 int isolated;
2092
2093 VM_BUG_ON_PAGE(page->mem_cgroup, page);
2094
2095
2096
2097
2098
2099 if (lrucare)
2100 lock_page_lru(page, &isolated);
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116 page->mem_cgroup = memcg;
2117
2118 if (lrucare)
2119 unlock_page_lru(page, isolated);
2120}
2121
2122#ifndef CONFIG_SLOB
2123static int memcg_alloc_cache_id(void)
2124{
2125 int id, size;
2126 int err;
2127
2128 id = ida_simple_get(&memcg_cache_ida,
2129 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2130 if (id < 0)
2131 return id;
2132
2133 if (id < memcg_nr_cache_ids)
2134 return id;
2135
2136
2137
2138
2139
2140 down_write(&memcg_cache_ids_sem);
2141
2142 size = 2 * (id + 1);
2143 if (size < MEMCG_CACHES_MIN_SIZE)
2144 size = MEMCG_CACHES_MIN_SIZE;
2145 else if (size > MEMCG_CACHES_MAX_SIZE)
2146 size = MEMCG_CACHES_MAX_SIZE;
2147
2148 err = memcg_update_all_caches(size);
2149 if (!err)
2150 err = memcg_update_all_list_lrus(size);
2151 if (!err)
2152 memcg_nr_cache_ids = size;
2153
2154 up_write(&memcg_cache_ids_sem);
2155
2156 if (err) {
2157 ida_simple_remove(&memcg_cache_ida, id);
2158 return err;
2159 }
2160 return id;
2161}
2162
2163static void memcg_free_cache_id(int id)
2164{
2165 ida_simple_remove(&memcg_cache_ida, id);
2166}
2167
2168struct memcg_kmem_cache_create_work {
2169 struct mem_cgroup *memcg;
2170 struct kmem_cache *cachep;
2171 struct work_struct work;
2172};
2173
2174static void memcg_kmem_cache_create_func(struct work_struct *w)
2175{
2176 struct memcg_kmem_cache_create_work *cw =
2177 container_of(w, struct memcg_kmem_cache_create_work, work);
2178 struct mem_cgroup *memcg = cw->memcg;
2179 struct kmem_cache *cachep = cw->cachep;
2180
2181 memcg_create_kmem_cache(memcg, cachep);
2182
2183 css_put(&memcg->css);
2184 kfree(cw);
2185}
2186
2187
2188
2189
2190static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2191 struct kmem_cache *cachep)
2192{
2193 struct memcg_kmem_cache_create_work *cw;
2194
2195 cw = kmalloc(sizeof(*cw), GFP_NOWAIT | __GFP_NOWARN);
2196 if (!cw)
2197 return;
2198
2199 css_get(&memcg->css);
2200
2201 cw->memcg = memcg;
2202 cw->cachep = cachep;
2203 INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2204
2205 queue_work(memcg_kmem_cache_wq, &cw->work);
2206}
2207
2208static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2209 struct kmem_cache *cachep)
2210{
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222 current->memcg_kmem_skip_account = 1;
2223 __memcg_schedule_kmem_cache_create(memcg, cachep);
2224 current->memcg_kmem_skip_account = 0;
2225}
2226
2227static inline bool memcg_kmem_bypass(void)
2228{
2229 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2230 return true;
2231 return false;
2232}
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2251{
2252 struct mem_cgroup *memcg;
2253 struct kmem_cache *memcg_cachep;
2254 int kmemcg_id;
2255
2256 VM_BUG_ON(!is_root_cache(cachep));
2257
2258 if (memcg_kmem_bypass())
2259 return cachep;
2260
2261 if (current->memcg_kmem_skip_account)
2262 return cachep;
2263
2264 memcg = get_mem_cgroup_from_mm(current->mm);
2265 kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2266 if (kmemcg_id < 0)
2267 goto out;
2268
2269 memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2270 if (likely(memcg_cachep))
2271 return memcg_cachep;
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285 memcg_schedule_kmem_cache_create(memcg, cachep);
2286out:
2287 css_put(&memcg->css);
2288 return cachep;
2289}
2290
2291
2292
2293
2294
2295void memcg_kmem_put_cache(struct kmem_cache *cachep)
2296{
2297 if (!is_root_cache(cachep))
2298 css_put(&cachep->memcg_params.memcg->css);
2299}
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2311 struct mem_cgroup *memcg)
2312{
2313 unsigned int nr_pages = 1 << order;
2314 struct page_counter *counter;
2315 int ret;
2316
2317 ret = try_charge(memcg, gfp, nr_pages);
2318 if (ret)
2319 return ret;
2320
2321 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2322 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2323 cancel_charge(memcg, nr_pages);
2324 return -ENOMEM;
2325 }
2326
2327 page->mem_cgroup = memcg;
2328
2329 return 0;
2330}
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2341{
2342 struct mem_cgroup *memcg;
2343 int ret = 0;
2344
2345 if (memcg_kmem_bypass())
2346 return 0;
2347
2348 memcg = get_mem_cgroup_from_mm(current->mm);
2349 if (!mem_cgroup_is_root(memcg)) {
2350 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2351 if (!ret)
2352 __SetPageKmemcg(page);
2353 }
2354 css_put(&memcg->css);
2355 return ret;
2356}
2357
2358
2359
2360
2361
2362void memcg_kmem_uncharge(struct page *page, int order)
2363{
2364 struct mem_cgroup *memcg = page->mem_cgroup;
2365 unsigned int nr_pages = 1 << order;
2366
2367 if (!memcg)
2368 return;
2369
2370 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2371
2372 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2373 page_counter_uncharge(&memcg->kmem, nr_pages);
2374
2375 page_counter_uncharge(&memcg->memory, nr_pages);
2376 if (do_memsw_account())
2377 page_counter_uncharge(&memcg->memsw, nr_pages);
2378
2379 page->mem_cgroup = NULL;
2380
2381
2382 if (PageKmemcg(page))
2383 __ClearPageKmemcg(page);
2384
2385 css_put_many(&memcg->css, nr_pages);
2386}
2387#endif
2388
2389#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2390
2391
2392
2393
2394
2395void mem_cgroup_split_huge_fixup(struct page *head)
2396{
2397 int i;
2398
2399 if (mem_cgroup_disabled())
2400 return;
2401
2402 for (i = 1; i < HPAGE_PMD_NR; i++)
2403 head[i].mem_cgroup = head->mem_cgroup;
2404
2405 __mod_memcg_state(head->mem_cgroup, MEMCG_RSS_HUGE, -HPAGE_PMD_NR);
2406}
2407#endif
2408
2409#ifdef CONFIG_MEMCG_SWAP
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424static int mem_cgroup_move_swap_account(swp_entry_t entry,
2425 struct mem_cgroup *from, struct mem_cgroup *to)
2426{
2427 unsigned short old_id, new_id;
2428
2429 old_id = mem_cgroup_id(from);
2430 new_id = mem_cgroup_id(to);
2431
2432 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2433 mod_memcg_state(from, MEMCG_SWAP, -1);
2434 mod_memcg_state(to, MEMCG_SWAP, 1);
2435 return 0;
2436 }
2437 return -EINVAL;
2438}
2439#else
2440static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2441 struct mem_cgroup *from, struct mem_cgroup *to)
2442{
2443 return -EINVAL;
2444}
2445#endif
2446
2447static DEFINE_MUTEX(memcg_limit_mutex);
2448
2449static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2450 unsigned long limit, bool memsw)
2451{
2452 bool enlarge = false;
2453 int ret;
2454 bool limits_invariant;
2455 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
2456
2457 do {
2458 if (signal_pending(current)) {
2459 ret = -EINTR;
2460 break;
2461 }
2462
2463 mutex_lock(&memcg_limit_mutex);
2464
2465
2466
2467
2468 limits_invariant = memsw ? limit >= memcg->memory.limit :
2469 limit <= memcg->memsw.limit;
2470 if (!limits_invariant) {
2471 mutex_unlock(&memcg_limit_mutex);
2472 ret = -EINVAL;
2473 break;
2474 }
2475 if (limit > counter->limit)
2476 enlarge = true;
2477 ret = page_counter_limit(counter, limit);
2478 mutex_unlock(&memcg_limit_mutex);
2479
2480 if (!ret)
2481 break;
2482
2483 if (!try_to_free_mem_cgroup_pages(memcg, 1,
2484 GFP_KERNEL, !memsw)) {
2485 ret = -EBUSY;
2486 break;
2487 }
2488 } while (true);
2489
2490 if (!ret && enlarge)
2491 memcg_oom_recover(memcg);
2492
2493 return ret;
2494}
2495
2496unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2497 gfp_t gfp_mask,
2498 unsigned long *total_scanned)
2499{
2500 unsigned long nr_reclaimed = 0;
2501 struct mem_cgroup_per_node *mz, *next_mz = NULL;
2502 unsigned long reclaimed;
2503 int loop = 0;
2504 struct mem_cgroup_tree_per_node *mctz;
2505 unsigned long excess;
2506 unsigned long nr_scanned;
2507
2508 if (order > 0)
2509 return 0;
2510
2511 mctz = soft_limit_tree_node(pgdat->node_id);
2512
2513
2514
2515
2516
2517
2518 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2519 return 0;
2520
2521
2522
2523
2524
2525
2526 do {
2527 if (next_mz)
2528 mz = next_mz;
2529 else
2530 mz = mem_cgroup_largest_soft_limit_node(mctz);
2531 if (!mz)
2532 break;
2533
2534 nr_scanned = 0;
2535 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2536 gfp_mask, &nr_scanned);
2537 nr_reclaimed += reclaimed;
2538 *total_scanned += nr_scanned;
2539 spin_lock_irq(&mctz->lock);
2540 __mem_cgroup_remove_exceeded(mz, mctz);
2541
2542
2543
2544
2545
2546 next_mz = NULL;
2547 if (!reclaimed)
2548 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2549
2550 excess = soft_limit_excess(mz->memcg);
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560 __mem_cgroup_insert_exceeded(mz, mctz, excess);
2561 spin_unlock_irq(&mctz->lock);
2562 css_put(&mz->memcg->css);
2563 loop++;
2564
2565
2566
2567
2568
2569 if (!nr_reclaimed &&
2570 (next_mz == NULL ||
2571 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2572 break;
2573 } while (!nr_reclaimed);
2574 if (next_mz)
2575 css_put(&next_mz->memcg->css);
2576 return nr_reclaimed;
2577}
2578
2579
2580
2581
2582
2583
2584
2585static inline bool memcg_has_children(struct mem_cgroup *memcg)
2586{
2587 bool ret;
2588
2589 rcu_read_lock();
2590 ret = css_next_child(NULL, &memcg->css);
2591 rcu_read_unlock();
2592 return ret;
2593}
2594
2595
2596
2597
2598
2599
2600static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2601{
2602 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2603
2604
2605 lru_add_drain_all();
2606
2607 while (nr_retries && page_counter_read(&memcg->memory)) {
2608 int progress;
2609
2610 if (signal_pending(current))
2611 return -EINTR;
2612
2613 progress = try_to_free_mem_cgroup_pages(memcg, 1,
2614 GFP_KERNEL, true);
2615 if (!progress) {
2616 nr_retries--;
2617
2618 congestion_wait(BLK_RW_ASYNC, HZ/10);
2619 }
2620
2621 }
2622
2623 return 0;
2624}
2625
2626static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2627 char *buf, size_t nbytes,
2628 loff_t off)
2629{
2630 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2631
2632 if (mem_cgroup_is_root(memcg))
2633 return -EINVAL;
2634 return mem_cgroup_force_empty(memcg) ?: nbytes;
2635}
2636
2637static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2638 struct cftype *cft)
2639{
2640 return mem_cgroup_from_css(css)->use_hierarchy;
2641}
2642
2643static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2644 struct cftype *cft, u64 val)
2645{
2646 int retval = 0;
2647 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2648 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2649
2650 if (memcg->use_hierarchy == val)
2651 return 0;
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661 if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2662 (val == 1 || val == 0)) {
2663 if (!memcg_has_children(memcg))
2664 memcg->use_hierarchy = val;
2665 else
2666 retval = -EBUSY;
2667 } else
2668 retval = -EINVAL;
2669
2670 return retval;
2671}
2672
2673static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
2674{
2675 struct mem_cgroup *iter;
2676 int i;
2677
2678 memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
2679
2680 for_each_mem_cgroup_tree(iter, memcg) {
2681 for (i = 0; i < MEMCG_NR_STAT; i++)
2682 stat[i] += memcg_page_state(iter, i);
2683 }
2684}
2685
2686static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
2687{
2688 struct mem_cgroup *iter;
2689 int i;
2690
2691 memset(events, 0, sizeof(*events) * NR_VM_EVENT_ITEMS);
2692
2693 for_each_mem_cgroup_tree(iter, memcg) {
2694 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
2695 events[i] += memcg_sum_events(iter, i);
2696 }
2697}
2698
2699static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2700{
2701 unsigned long val = 0;
2702
2703 if (mem_cgroup_is_root(memcg)) {
2704 struct mem_cgroup *iter;
2705
2706 for_each_mem_cgroup_tree(iter, memcg) {
2707 val += memcg_page_state(iter, MEMCG_CACHE);
2708 val += memcg_page_state(iter, MEMCG_RSS);
2709 if (swap)
2710 val += memcg_page_state(iter, MEMCG_SWAP);
2711 }
2712 } else {
2713 if (!swap)
2714 val = page_counter_read(&memcg->memory);
2715 else
2716 val = page_counter_read(&memcg->memsw);
2717 }
2718 return val;
2719}
2720
2721enum {
2722 RES_USAGE,
2723 RES_LIMIT,
2724 RES_MAX_USAGE,
2725 RES_FAILCNT,
2726 RES_SOFT_LIMIT,
2727};
2728
2729static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2730 struct cftype *cft)
2731{
2732 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2733 struct page_counter *counter;
2734
2735 switch (MEMFILE_TYPE(cft->private)) {
2736 case _MEM:
2737 counter = &memcg->memory;
2738 break;
2739 case _MEMSWAP:
2740 counter = &memcg->memsw;
2741 break;
2742 case _KMEM:
2743 counter = &memcg->kmem;
2744 break;
2745 case _TCP:
2746 counter = &memcg->tcpmem;
2747 break;
2748 default:
2749 BUG();
2750 }
2751
2752 switch (MEMFILE_ATTR(cft->private)) {
2753 case RES_USAGE:
2754 if (counter == &memcg->memory)
2755 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2756 if (counter == &memcg->memsw)
2757 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2758 return (u64)page_counter_read(counter) * PAGE_SIZE;
2759 case RES_LIMIT:
2760 return (u64)counter->limit * PAGE_SIZE;
2761 case RES_MAX_USAGE:
2762 return (u64)counter->watermark * PAGE_SIZE;
2763 case RES_FAILCNT:
2764 return counter->failcnt;
2765 case RES_SOFT_LIMIT:
2766 return (u64)memcg->soft_limit * PAGE_SIZE;
2767 default:
2768 BUG();
2769 }
2770}
2771
2772#ifndef CONFIG_SLOB
2773static int memcg_online_kmem(struct mem_cgroup *memcg)
2774{
2775 int memcg_id;
2776
2777 if (cgroup_memory_nokmem)
2778 return 0;
2779
2780 BUG_ON(memcg->kmemcg_id >= 0);
2781 BUG_ON(memcg->kmem_state);
2782
2783 memcg_id = memcg_alloc_cache_id();
2784 if (memcg_id < 0)
2785 return memcg_id;
2786
2787 static_branch_inc(&memcg_kmem_enabled_key);
2788
2789
2790
2791
2792
2793
2794 memcg->kmemcg_id = memcg_id;
2795 memcg->kmem_state = KMEM_ONLINE;
2796 INIT_LIST_HEAD(&memcg->kmem_caches);
2797
2798 return 0;
2799}
2800
2801static void memcg_offline_kmem(struct mem_cgroup *memcg)
2802{
2803 struct cgroup_subsys_state *css;
2804 struct mem_cgroup *parent, *child;
2805 int kmemcg_id;
2806
2807 if (memcg->kmem_state != KMEM_ONLINE)
2808 return;
2809
2810
2811
2812
2813
2814
2815 memcg->kmem_state = KMEM_ALLOCATED;
2816
2817 memcg_deactivate_kmem_caches(memcg);
2818
2819 kmemcg_id = memcg->kmemcg_id;
2820 BUG_ON(kmemcg_id < 0);
2821
2822 parent = parent_mem_cgroup(memcg);
2823 if (!parent)
2824 parent = root_mem_cgroup;
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834 rcu_read_lock();
2835 css_for_each_descendant_pre(css, &memcg->css) {
2836 child = mem_cgroup_from_css(css);
2837 BUG_ON(child->kmemcg_id != kmemcg_id);
2838 child->kmemcg_id = parent->kmemcg_id;
2839 if (!memcg->use_hierarchy)
2840 break;
2841 }
2842 rcu_read_unlock();
2843
2844 memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
2845
2846 memcg_free_cache_id(kmemcg_id);
2847}
2848
2849static void memcg_free_kmem(struct mem_cgroup *memcg)
2850{
2851
2852 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
2853 memcg_offline_kmem(memcg);
2854
2855 if (memcg->kmem_state == KMEM_ALLOCATED) {
2856 memcg_destroy_kmem_caches(memcg);
2857 static_branch_dec(&memcg_kmem_enabled_key);
2858 WARN_ON(page_counter_read(&memcg->kmem));
2859 }
2860}
2861#else
2862static int memcg_online_kmem(struct mem_cgroup *memcg)
2863{
2864 return 0;
2865}
2866static void memcg_offline_kmem(struct mem_cgroup *memcg)
2867{
2868}
2869static void memcg_free_kmem(struct mem_cgroup *memcg)
2870{
2871}
2872#endif
2873
2874static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2875 unsigned long limit)
2876{
2877 int ret;
2878
2879 mutex_lock(&memcg_limit_mutex);
2880 ret = page_counter_limit(&memcg->kmem, limit);
2881 mutex_unlock(&memcg_limit_mutex);
2882 return ret;
2883}
2884
2885static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
2886{
2887 int ret;
2888
2889 mutex_lock(&memcg_limit_mutex);
2890
2891 ret = page_counter_limit(&memcg->tcpmem, limit);
2892 if (ret)
2893 goto out;
2894
2895 if (!memcg->tcpmem_active) {
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912 static_branch_inc(&memcg_sockets_enabled_key);
2913 memcg->tcpmem_active = true;
2914 }
2915out:
2916 mutex_unlock(&memcg_limit_mutex);
2917 return ret;
2918}
2919
2920
2921
2922
2923
2924static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
2925 char *buf, size_t nbytes, loff_t off)
2926{
2927 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2928 unsigned long nr_pages;
2929 int ret;
2930
2931 buf = strstrip(buf);
2932 ret = page_counter_memparse(buf, "-1", &nr_pages);
2933 if (ret)
2934 return ret;
2935
2936 switch (MEMFILE_ATTR(of_cft(of)->private)) {
2937 case RES_LIMIT:
2938 if (mem_cgroup_is_root(memcg)) {
2939 ret = -EINVAL;
2940 break;
2941 }
2942 switch (MEMFILE_TYPE(of_cft(of)->private)) {
2943 case _MEM:
2944 ret = mem_cgroup_resize_limit(memcg, nr_pages, false);
2945 break;
2946 case _MEMSWAP:
2947 ret = mem_cgroup_resize_limit(memcg, nr_pages, true);
2948 break;
2949 case _KMEM:
2950 ret = memcg_update_kmem_limit(memcg, nr_pages);
2951 break;
2952 case _TCP:
2953 ret = memcg_update_tcp_limit(memcg, nr_pages);
2954 break;
2955 }
2956 break;
2957 case RES_SOFT_LIMIT:
2958 memcg->soft_limit = nr_pages;
2959 ret = 0;
2960 break;
2961 }
2962 return ret ?: nbytes;
2963}
2964
2965static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
2966 size_t nbytes, loff_t off)
2967{
2968 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2969 struct page_counter *counter;
2970
2971 switch (MEMFILE_TYPE(of_cft(of)->private)) {
2972 case _MEM:
2973 counter = &memcg->memory;
2974 break;
2975 case _MEMSWAP:
2976 counter = &memcg->memsw;
2977 break;
2978 case _KMEM:
2979 counter = &memcg->kmem;
2980 break;
2981 case _TCP:
2982 counter = &memcg->tcpmem;
2983 break;
2984 default:
2985 BUG();
2986 }
2987
2988 switch (MEMFILE_ATTR(of_cft(of)->private)) {
2989 case RES_MAX_USAGE:
2990 page_counter_reset_watermark(counter);
2991 break;
2992 case RES_FAILCNT:
2993 counter->failcnt = 0;
2994 break;
2995 default:
2996 BUG();
2997 }
2998
2999 return nbytes;
3000}
3001
3002static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3003 struct cftype *cft)
3004{
3005 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3006}
3007
3008#ifdef CONFIG_MMU
3009static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3010 struct cftype *cft, u64 val)
3011{
3012 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3013
3014 if (val & ~MOVE_MASK)
3015 return -EINVAL;
3016
3017
3018
3019
3020
3021
3022
3023 memcg->move_charge_at_immigrate = val;
3024 return 0;
3025}
3026#else
3027static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3028 struct cftype *cft, u64 val)
3029{
3030 return -ENOSYS;
3031}
3032#endif
3033
3034#ifdef CONFIG_NUMA
3035static int memcg_numa_stat_show(struct seq_file *m, void *v)
3036{
3037 struct numa_stat {
3038 const char *name;
3039 unsigned int lru_mask;
3040 };
3041
3042 static const struct numa_stat stats[] = {
3043 { "total", LRU_ALL },
3044 { "file", LRU_ALL_FILE },
3045 { "anon", LRU_ALL_ANON },
3046 { "unevictable", BIT(LRU_UNEVICTABLE) },
3047 };
3048 const struct numa_stat *stat;
3049 int nid;
3050 unsigned long nr;
3051 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3052
3053 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3054 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3055 seq_printf(m, "%s=%lu", stat->name, nr);
3056 for_each_node_state(nid, N_MEMORY) {
3057 nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3058 stat->lru_mask);
3059 seq_printf(m, " N%d=%lu", nid, nr);
3060 }
3061 seq_putc(m, '\n');
3062 }
3063
3064 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3065 struct mem_cgroup *iter;
3066
3067 nr = 0;
3068 for_each_mem_cgroup_tree(iter, memcg)
3069 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3070 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3071 for_each_node_state(nid, N_MEMORY) {
3072 nr = 0;
3073 for_each_mem_cgroup_tree(iter, memcg)
3074 nr += mem_cgroup_node_nr_lru_pages(
3075 iter, nid, stat->lru_mask);
3076 seq_printf(m, " N%d=%lu", nid, nr);
3077 }
3078 seq_putc(m, '\n');
3079 }
3080
3081 return 0;
3082}
3083#endif
3084
3085
3086unsigned int memcg1_events[] = {
3087 PGPGIN,
3088 PGPGOUT,
3089 PGFAULT,
3090 PGMAJFAULT,
3091};
3092
3093static const char *const memcg1_event_names[] = {
3094 "pgpgin",
3095 "pgpgout",
3096 "pgfault",
3097 "pgmajfault",
3098};
3099
3100static int memcg_stat_show(struct seq_file *m, void *v)
3101{
3102 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3103 unsigned long memory, memsw;
3104 struct mem_cgroup *mi;
3105 unsigned int i;
3106
3107 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3108 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3109
3110 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3111 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3112 continue;
3113 seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3114 memcg_page_state(memcg, memcg1_stats[i]) *
3115 PAGE_SIZE);
3116 }
3117
3118 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3119 seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3120 memcg_sum_events(memcg, memcg1_events[i]));
3121
3122 for (i = 0; i < NR_LRU_LISTS; i++)
3123 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3124 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3125
3126
3127 memory = memsw = PAGE_COUNTER_MAX;
3128 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3129 memory = min(memory, mi->memory.limit);
3130 memsw = min(memsw, mi->memsw.limit);
3131 }
3132 seq_printf(m, "hierarchical_memory_limit %llu\n",
3133 (u64)memory * PAGE_SIZE);
3134 if (do_memsw_account())
3135 seq_printf(m, "hierarchical_memsw_limit %llu\n",
3136 (u64)memsw * PAGE_SIZE);
3137
3138 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3139 unsigned long long val = 0;
3140
3141 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3142 continue;
3143 for_each_mem_cgroup_tree(mi, memcg)
3144 val += memcg_page_state(mi, memcg1_stats[i]) *
3145 PAGE_SIZE;
3146 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], val);
3147 }
3148
3149 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) {
3150 unsigned long long val = 0;
3151
3152 for_each_mem_cgroup_tree(mi, memcg)
3153 val += memcg_sum_events(mi, memcg1_events[i]);
3154 seq_printf(m, "total_%s %llu\n", memcg1_event_names[i], val);
3155 }
3156
3157 for (i = 0; i < NR_LRU_LISTS; i++) {
3158 unsigned long long val = 0;
3159
3160 for_each_mem_cgroup_tree(mi, memcg)
3161 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3162 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3163 }
3164
3165#ifdef CONFIG_DEBUG_VM
3166 {
3167 pg_data_t *pgdat;
3168 struct mem_cgroup_per_node *mz;
3169 struct zone_reclaim_stat *rstat;
3170 unsigned long recent_rotated[2] = {0, 0};
3171 unsigned long recent_scanned[2] = {0, 0};
3172
3173 for_each_online_pgdat(pgdat) {
3174 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3175 rstat = &mz->lruvec.reclaim_stat;
3176
3177 recent_rotated[0] += rstat->recent_rotated[0];
3178 recent_rotated[1] += rstat->recent_rotated[1];
3179 recent_scanned[0] += rstat->recent_scanned[0];
3180 recent_scanned[1] += rstat->recent_scanned[1];
3181 }
3182 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3183 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3184 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3185 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3186 }
3187#endif
3188
3189 return 0;
3190}
3191
3192static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3193 struct cftype *cft)
3194{
3195 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3196
3197 return mem_cgroup_swappiness(memcg);
3198}
3199
3200static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3201 struct cftype *cft, u64 val)
3202{
3203 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3204
3205 if (val > 100)
3206 return -EINVAL;
3207
3208 if (css->parent)
3209 memcg->swappiness = val;
3210 else
3211 vm_swappiness = val;
3212
3213 return 0;
3214}
3215
3216static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3217{
3218 struct mem_cgroup_threshold_ary *t;
3219 unsigned long usage;
3220 int i;
3221
3222 rcu_read_lock();
3223 if (!swap)
3224 t = rcu_dereference(memcg->thresholds.primary);
3225 else
3226 t = rcu_dereference(memcg->memsw_thresholds.primary);
3227
3228 if (!t)
3229 goto unlock;
3230
3231 usage = mem_cgroup_usage(memcg, swap);
3232
3233
3234
3235
3236
3237
3238 i = t->current_threshold;
3239
3240
3241
3242
3243
3244
3245
3246 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3247 eventfd_signal(t->entries[i].eventfd, 1);
3248
3249
3250 i++;
3251
3252
3253
3254
3255
3256
3257
3258 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3259 eventfd_signal(t->entries[i].eventfd, 1);
3260
3261
3262 t->current_threshold = i - 1;
3263unlock:
3264 rcu_read_unlock();
3265}
3266
3267static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3268{
3269 while (memcg) {
3270 __mem_cgroup_threshold(memcg, false);
3271 if (do_memsw_account())
3272 __mem_cgroup_threshold(memcg, true);
3273
3274 memcg = parent_mem_cgroup(memcg);
3275 }
3276}
3277
3278static int compare_thresholds(const void *a, const void *b)
3279{
3280 const struct mem_cgroup_threshold *_a = a;
3281 const struct mem_cgroup_threshold *_b = b;
3282
3283 if (_a->threshold > _b->threshold)
3284 return 1;
3285
3286 if (_a->threshold < _b->threshold)
3287 return -1;
3288
3289 return 0;
3290}
3291
3292static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3293{
3294 struct mem_cgroup_eventfd_list *ev;
3295
3296 spin_lock(&memcg_oom_lock);
3297
3298 list_for_each_entry(ev, &memcg->oom_notify, list)
3299 eventfd_signal(ev->eventfd, 1);
3300
3301 spin_unlock(&memcg_oom_lock);
3302 return 0;
3303}
3304
3305static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3306{
3307 struct mem_cgroup *iter;
3308
3309 for_each_mem_cgroup_tree(iter, memcg)
3310 mem_cgroup_oom_notify_cb(iter);
3311}
3312
3313static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3314 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3315{
3316 struct mem_cgroup_thresholds *thresholds;
3317 struct mem_cgroup_threshold_ary *new;
3318 unsigned long threshold;
3319 unsigned long usage;
3320 int i, size, ret;
3321
3322 ret = page_counter_memparse(args, "-1", &threshold);
3323 if (ret)
3324 return ret;
3325
3326 mutex_lock(&memcg->thresholds_lock);
3327
3328 if (type == _MEM) {
3329 thresholds = &memcg->thresholds;
3330 usage = mem_cgroup_usage(memcg, false);
3331 } else if (type == _MEMSWAP) {
3332 thresholds = &memcg->memsw_thresholds;
3333 usage = mem_cgroup_usage(memcg, true);
3334 } else
3335 BUG();
3336
3337
3338 if (thresholds->primary)
3339 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3340
3341 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3342
3343
3344 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3345 GFP_KERNEL);
3346 if (!new) {
3347 ret = -ENOMEM;
3348 goto unlock;
3349 }
3350 new->size = size;
3351
3352
3353 if (thresholds->primary) {
3354 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3355 sizeof(struct mem_cgroup_threshold));
3356 }
3357
3358
3359 new->entries[size - 1].eventfd = eventfd;
3360 new->entries[size - 1].threshold = threshold;
3361
3362
3363 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3364 compare_thresholds, NULL);
3365
3366
3367 new->current_threshold = -1;
3368 for (i = 0; i < size; i++) {
3369 if (new->entries[i].threshold <= usage) {
3370
3371
3372
3373
3374
3375 ++new->current_threshold;
3376 } else
3377 break;
3378 }
3379
3380
3381 kfree(thresholds->spare);
3382 thresholds->spare = thresholds->primary;
3383
3384 rcu_assign_pointer(thresholds->primary, new);
3385
3386
3387 synchronize_rcu();
3388
3389unlock:
3390 mutex_unlock(&memcg->thresholds_lock);
3391
3392 return ret;
3393}
3394
3395static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3396 struct eventfd_ctx *eventfd, const char *args)
3397{
3398 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3399}
3400
3401static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3402 struct eventfd_ctx *eventfd, const char *args)
3403{
3404 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3405}
3406
3407static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3408 struct eventfd_ctx *eventfd, enum res_type type)
3409{
3410 struct mem_cgroup_thresholds *thresholds;
3411 struct mem_cgroup_threshold_ary *new;
3412 unsigned long usage;
3413 int i, j, size;
3414
3415 mutex_lock(&memcg->thresholds_lock);
3416
3417 if (type == _MEM) {
3418 thresholds = &memcg->thresholds;
3419 usage = mem_cgroup_usage(memcg, false);
3420 } else if (type == _MEMSWAP) {
3421 thresholds = &memcg->memsw_thresholds;
3422 usage = mem_cgroup_usage(memcg, true);
3423 } else
3424 BUG();
3425
3426 if (!thresholds->primary)
3427 goto unlock;
3428
3429
3430 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3431
3432
3433 size = 0;
3434 for (i = 0; i < thresholds->primary->size; i++) {
3435 if (thresholds->primary->entries[i].eventfd != eventfd)
3436 size++;
3437 }
3438
3439 new = thresholds->spare;
3440
3441
3442 if (!size) {
3443 kfree(new);
3444 new = NULL;
3445 goto swap_buffers;
3446 }
3447
3448 new->size = size;
3449
3450
3451 new->current_threshold = -1;
3452 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3453 if (thresholds->primary->entries[i].eventfd == eventfd)
3454 continue;
3455
3456 new->entries[j] = thresholds->primary->entries[i];
3457 if (new->entries[j].threshold <= usage) {
3458
3459
3460
3461
3462
3463 ++new->current_threshold;
3464 }
3465 j++;
3466 }
3467
3468swap_buffers:
3469
3470 thresholds->spare = thresholds->primary;
3471
3472 rcu_assign_pointer(thresholds->primary, new);
3473
3474
3475 synchronize_rcu();
3476
3477
3478 if (!new) {
3479 kfree(thresholds->spare);
3480 thresholds->spare = NULL;
3481 }
3482unlock:
3483 mutex_unlock(&memcg->thresholds_lock);
3484}
3485
3486static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3487 struct eventfd_ctx *eventfd)
3488{
3489 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3490}
3491
3492static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3493 struct eventfd_ctx *eventfd)
3494{
3495 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3496}
3497
3498static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3499 struct eventfd_ctx *eventfd, const char *args)
3500{
3501 struct mem_cgroup_eventfd_list *event;
3502
3503 event = kmalloc(sizeof(*event), GFP_KERNEL);
3504 if (!event)
3505 return -ENOMEM;
3506
3507 spin_lock(&memcg_oom_lock);
3508
3509 event->eventfd = eventfd;
3510 list_add(&event->list, &memcg->oom_notify);
3511
3512
3513 if (memcg->under_oom)
3514 eventfd_signal(eventfd, 1);
3515 spin_unlock(&memcg_oom_lock);
3516
3517 return 0;
3518}
3519
3520static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3521 struct eventfd_ctx *eventfd)
3522{
3523 struct mem_cgroup_eventfd_list *ev, *tmp;
3524
3525 spin_lock(&memcg_oom_lock);
3526
3527 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3528 if (ev->eventfd == eventfd) {
3529 list_del(&ev->list);
3530 kfree(ev);
3531 }
3532 }
3533
3534 spin_unlock(&memcg_oom_lock);
3535}
3536
3537static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3538{
3539 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3540
3541 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3542 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3543 seq_printf(sf, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL));
3544 return 0;
3545}
3546
3547static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3548 struct cftype *cft, u64 val)
3549{
3550 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3551
3552
3553 if (!css->parent || !((val == 0) || (val == 1)))
3554 return -EINVAL;
3555
3556 memcg->oom_kill_disable = val;
3557 if (!val)
3558 memcg_oom_recover(memcg);
3559
3560 return 0;
3561}
3562
3563#ifdef CONFIG_CGROUP_WRITEBACK
3564
3565struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
3566{
3567 return &memcg->cgwb_list;
3568}
3569
3570static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3571{
3572 return wb_domain_init(&memcg->cgwb_domain, gfp);
3573}
3574
3575static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3576{
3577 wb_domain_exit(&memcg->cgwb_domain);
3578}
3579
3580static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3581{
3582 wb_domain_size_changed(&memcg->cgwb_domain);
3583}
3584
3585struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3586{
3587 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3588
3589 if (!memcg->css.parent)
3590 return NULL;
3591
3592 return &memcg->cgwb_domain;
3593}
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3614 unsigned long *pheadroom, unsigned long *pdirty,
3615 unsigned long *pwriteback)
3616{
3617 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3618 struct mem_cgroup *parent;
3619
3620 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
3621
3622
3623 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
3624 *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3625 (1 << LRU_ACTIVE_FILE));
3626 *pheadroom = PAGE_COUNTER_MAX;
3627
3628 while ((parent = parent_mem_cgroup(memcg))) {
3629 unsigned long ceiling = min(memcg->memory.limit, memcg->high);
3630 unsigned long used = page_counter_read(&memcg->memory);
3631
3632 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3633 memcg = parent;
3634 }
3635}
3636
3637#else
3638
3639static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3640{
3641 return 0;
3642}
3643
3644static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3645{
3646}
3647
3648static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3649{
3650}
3651
3652#endif
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672static void memcg_event_remove(struct work_struct *work)
3673{
3674 struct mem_cgroup_event *event =
3675 container_of(work, struct mem_cgroup_event, remove);
3676 struct mem_cgroup *memcg = event->memcg;
3677
3678 remove_wait_queue(event->wqh, &event->wait);
3679
3680 event->unregister_event(memcg, event->eventfd);
3681
3682
3683 eventfd_signal(event->eventfd, 1);
3684
3685 eventfd_ctx_put(event->eventfd);
3686 kfree(event);
3687 css_put(&memcg->css);
3688}
3689
3690
3691
3692
3693
3694
3695static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
3696 int sync, void *key)
3697{
3698 struct mem_cgroup_event *event =
3699 container_of(wait, struct mem_cgroup_event, wait);
3700 struct mem_cgroup *memcg = event->memcg;
3701 __poll_t flags = key_to_poll(key);
3702
3703 if (flags & EPOLLHUP) {
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713 spin_lock(&memcg->event_list_lock);
3714 if (!list_empty(&event->list)) {
3715 list_del_init(&event->list);
3716
3717
3718
3719
3720 schedule_work(&event->remove);
3721 }
3722 spin_unlock(&memcg->event_list_lock);
3723 }
3724
3725 return 0;
3726}
3727
3728static void memcg_event_ptable_queue_proc(struct file *file,
3729 wait_queue_head_t *wqh, poll_table *pt)
3730{
3731 struct mem_cgroup_event *event =
3732 container_of(pt, struct mem_cgroup_event, pt);
3733
3734 event->wqh = wqh;
3735 add_wait_queue(wqh, &event->wait);
3736}
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
3747 char *buf, size_t nbytes, loff_t off)
3748{
3749 struct cgroup_subsys_state *css = of_css(of);
3750 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3751 struct mem_cgroup_event *event;
3752 struct cgroup_subsys_state *cfile_css;
3753 unsigned int efd, cfd;
3754 struct fd efile;
3755 struct fd cfile;
3756 const char *name;
3757 char *endp;
3758 int ret;
3759
3760 buf = strstrip(buf);
3761
3762 efd = simple_strtoul(buf, &endp, 10);
3763 if (*endp != ' ')
3764 return -EINVAL;
3765 buf = endp + 1;
3766
3767 cfd = simple_strtoul(buf, &endp, 10);
3768 if ((*endp != ' ') && (*endp != '\0'))
3769 return -EINVAL;
3770 buf = endp + 1;
3771
3772 event = kzalloc(sizeof(*event), GFP_KERNEL);
3773 if (!event)
3774 return -ENOMEM;
3775
3776 event->memcg = memcg;
3777 INIT_LIST_HEAD(&event->list);
3778 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
3779 init_waitqueue_func_entry(&event->wait, memcg_event_wake);
3780 INIT_WORK(&event->remove, memcg_event_remove);
3781
3782 efile = fdget(efd);
3783 if (!efile.file) {
3784 ret = -EBADF;
3785 goto out_kfree;
3786 }
3787
3788 event->eventfd = eventfd_ctx_fileget(efile.file);
3789 if (IS_ERR(event->eventfd)) {
3790 ret = PTR_ERR(event->eventfd);
3791 goto out_put_efile;
3792 }
3793
3794 cfile = fdget(cfd);
3795 if (!cfile.file) {
3796 ret = -EBADF;
3797 goto out_put_eventfd;
3798 }
3799
3800
3801
3802 ret = inode_permission(file_inode(cfile.file), MAY_READ);
3803 if (ret < 0)
3804 goto out_put_cfile;
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814 name = cfile.file->f_path.dentry->d_name.name;
3815
3816 if (!strcmp(name, "memory.usage_in_bytes")) {
3817 event->register_event = mem_cgroup_usage_register_event;
3818 event->unregister_event = mem_cgroup_usage_unregister_event;
3819 } else if (!strcmp(name, "memory.oom_control")) {
3820 event->register_event = mem_cgroup_oom_register_event;
3821 event->unregister_event = mem_cgroup_oom_unregister_event;
3822 } else if (!strcmp(name, "memory.pressure_level")) {
3823 event->register_event = vmpressure_register_event;
3824 event->unregister_event = vmpressure_unregister_event;
3825 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
3826 event->register_event = memsw_cgroup_usage_register_event;
3827 event->unregister_event = memsw_cgroup_usage_unregister_event;
3828 } else {
3829 ret = -EINVAL;
3830 goto out_put_cfile;
3831 }
3832
3833
3834
3835
3836
3837
3838 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3839 &memory_cgrp_subsys);
3840 ret = -EINVAL;
3841 if (IS_ERR(cfile_css))
3842 goto out_put_cfile;
3843 if (cfile_css != css) {
3844 css_put(cfile_css);
3845 goto out_put_cfile;
3846 }
3847
3848 ret = event->register_event(memcg, event->eventfd, buf);
3849 if (ret)
3850 goto out_put_css;
3851
3852 efile.file->f_op->poll(efile.file, &event->pt);
3853
3854 spin_lock(&memcg->event_list_lock);
3855 list_add(&event->list, &memcg->event_list);
3856 spin_unlock(&memcg->event_list_lock);
3857
3858 fdput(cfile);
3859 fdput(efile);
3860
3861 return nbytes;
3862
3863out_put_css:
3864 css_put(css);
3865out_put_cfile:
3866 fdput(cfile);
3867out_put_eventfd:
3868 eventfd_ctx_put(event->eventfd);
3869out_put_efile:
3870 fdput(efile);
3871out_kfree:
3872 kfree(event);
3873
3874 return ret;
3875}
3876
3877static struct cftype mem_cgroup_legacy_files[] = {
3878 {
3879 .name = "usage_in_bytes",
3880 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3881 .read_u64 = mem_cgroup_read_u64,
3882 },
3883 {
3884 .name = "max_usage_in_bytes",
3885 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3886 .write = mem_cgroup_reset,
3887 .read_u64 = mem_cgroup_read_u64,
3888 },
3889 {
3890 .name = "limit_in_bytes",
3891 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3892 .write = mem_cgroup_write,
3893 .read_u64 = mem_cgroup_read_u64,
3894 },
3895 {
3896 .name = "soft_limit_in_bytes",
3897 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3898 .write = mem_cgroup_write,
3899 .read_u64 = mem_cgroup_read_u64,
3900 },
3901 {
3902 .name = "failcnt",
3903 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3904 .write = mem_cgroup_reset,
3905 .read_u64 = mem_cgroup_read_u64,
3906 },
3907 {
3908 .name = "stat",
3909 .seq_show = memcg_stat_show,
3910 },
3911 {
3912 .name = "force_empty",
3913 .write = mem_cgroup_force_empty_write,
3914 },
3915 {
3916 .name = "use_hierarchy",
3917 .write_u64 = mem_cgroup_hierarchy_write,
3918 .read_u64 = mem_cgroup_hierarchy_read,
3919 },
3920 {
3921 .name = "cgroup.event_control",
3922 .write = memcg_write_event_control,
3923 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
3924 },
3925 {
3926 .name = "swappiness",
3927 .read_u64 = mem_cgroup_swappiness_read,
3928 .write_u64 = mem_cgroup_swappiness_write,
3929 },
3930 {
3931 .name = "move_charge_at_immigrate",
3932 .read_u64 = mem_cgroup_move_charge_read,
3933 .write_u64 = mem_cgroup_move_charge_write,
3934 },
3935 {
3936 .name = "oom_control",
3937 .seq_show = mem_cgroup_oom_control_read,
3938 .write_u64 = mem_cgroup_oom_control_write,
3939 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
3940 },
3941 {
3942 .name = "pressure_level",
3943 },
3944#ifdef CONFIG_NUMA
3945 {
3946 .name = "numa_stat",
3947 .seq_show = memcg_numa_stat_show,
3948 },
3949#endif
3950 {
3951 .name = "kmem.limit_in_bytes",
3952 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
3953 .write = mem_cgroup_write,
3954 .read_u64 = mem_cgroup_read_u64,
3955 },
3956 {
3957 .name = "kmem.usage_in_bytes",
3958 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
3959 .read_u64 = mem_cgroup_read_u64,
3960 },
3961 {
3962 .name = "kmem.failcnt",
3963 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
3964 .write = mem_cgroup_reset,
3965 .read_u64 = mem_cgroup_read_u64,
3966 },
3967 {
3968 .name = "kmem.max_usage_in_bytes",
3969 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
3970 .write = mem_cgroup_reset,
3971 .read_u64 = mem_cgroup_read_u64,
3972 },
3973#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
3974 {
3975 .name = "kmem.slabinfo",
3976 .seq_start = memcg_slab_start,
3977 .seq_next = memcg_slab_next,
3978 .seq_stop = memcg_slab_stop,
3979 .seq_show = memcg_slab_show,
3980 },
3981#endif
3982 {
3983 .name = "kmem.tcp.limit_in_bytes",
3984 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
3985 .write = mem_cgroup_write,
3986 .read_u64 = mem_cgroup_read_u64,
3987 },
3988 {
3989 .name = "kmem.tcp.usage_in_bytes",
3990 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
3991 .read_u64 = mem_cgroup_read_u64,
3992 },
3993 {
3994 .name = "kmem.tcp.failcnt",
3995 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
3996 .write = mem_cgroup_reset,
3997 .read_u64 = mem_cgroup_read_u64,
3998 },
3999 {
4000 .name = "kmem.tcp.max_usage_in_bytes",
4001 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4002 .write = mem_cgroup_reset,
4003 .read_u64 = mem_cgroup_read_u64,
4004 },
4005 { },
4006};
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032static DEFINE_IDR(mem_cgroup_idr);
4033
4034static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4035{
4036 VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
4037 atomic_add(n, &memcg->id.ref);
4038}
4039
4040static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4041{
4042 VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
4043 if (atomic_sub_and_test(n, &memcg->id.ref)) {
4044 idr_remove(&mem_cgroup_idr, memcg->id.id);
4045 memcg->id.id = 0;
4046
4047
4048 css_put(&memcg->css);
4049 }
4050}
4051
4052static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4053{
4054 mem_cgroup_id_get_many(memcg, 1);
4055}
4056
4057static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4058{
4059 mem_cgroup_id_put_many(memcg, 1);
4060}
4061
4062
4063
4064
4065
4066
4067
4068struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4069{
4070 WARN_ON_ONCE(!rcu_read_lock_held());
4071 return idr_find(&mem_cgroup_idr, id);
4072}
4073
4074static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4075{
4076 struct mem_cgroup_per_node *pn;
4077 int tmp = node;
4078
4079
4080
4081
4082
4083
4084
4085
4086 if (!node_state(node, N_NORMAL_MEMORY))
4087 tmp = -1;
4088 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4089 if (!pn)
4090 return 1;
4091
4092 pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
4093 if (!pn->lruvec_stat_cpu) {
4094 kfree(pn);
4095 return 1;
4096 }
4097
4098 lruvec_init(&pn->lruvec);
4099 pn->usage_in_excess = 0;
4100 pn->on_tree = false;
4101 pn->memcg = memcg;
4102
4103 memcg->nodeinfo[node] = pn;
4104 return 0;
4105}
4106
4107static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4108{
4109 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
4110
4111 if (!pn)
4112 return;
4113
4114 free_percpu(pn->lruvec_stat_cpu);
4115 kfree(pn);
4116}
4117
4118static void __mem_cgroup_free(struct mem_cgroup *memcg)
4119{
4120 int node;
4121
4122 for_each_node(node)
4123 free_mem_cgroup_per_node_info(memcg, node);
4124 free_percpu(memcg->stat_cpu);
4125 kfree(memcg);
4126}
4127
4128static void mem_cgroup_free(struct mem_cgroup *memcg)
4129{
4130 memcg_wb_domain_exit(memcg);
4131 __mem_cgroup_free(memcg);
4132}
4133
4134static struct mem_cgroup *mem_cgroup_alloc(void)
4135{
4136 struct mem_cgroup *memcg;
4137 size_t size;
4138 int node;
4139
4140 size = sizeof(struct mem_cgroup);
4141 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4142
4143 memcg = kzalloc(size, GFP_KERNEL);
4144 if (!memcg)
4145 return NULL;
4146
4147 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4148 1, MEM_CGROUP_ID_MAX,
4149 GFP_KERNEL);
4150 if (memcg->id.id < 0)
4151 goto fail;
4152
4153 memcg->stat_cpu = alloc_percpu(struct mem_cgroup_stat_cpu);
4154 if (!memcg->stat_cpu)
4155 goto fail;
4156
4157 for_each_node(node)
4158 if (alloc_mem_cgroup_per_node_info(memcg, node))
4159 goto fail;
4160
4161 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4162 goto fail;
4163
4164 INIT_WORK(&memcg->high_work, high_work_func);
4165 memcg->last_scanned_node = MAX_NUMNODES;
4166 INIT_LIST_HEAD(&memcg->oom_notify);
4167 mutex_init(&memcg->thresholds_lock);
4168 spin_lock_init(&memcg->move_lock);
4169 vmpressure_init(&memcg->vmpressure);
4170 INIT_LIST_HEAD(&memcg->event_list);
4171 spin_lock_init(&memcg->event_list_lock);
4172 memcg->socket_pressure = jiffies;
4173#ifndef CONFIG_SLOB
4174 memcg->kmemcg_id = -1;
4175#endif
4176#ifdef CONFIG_CGROUP_WRITEBACK
4177 INIT_LIST_HEAD(&memcg->cgwb_list);
4178#endif
4179 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4180 return memcg;
4181fail:
4182 if (memcg->id.id > 0)
4183 idr_remove(&mem_cgroup_idr, memcg->id.id);
4184 __mem_cgroup_free(memcg);
4185 return NULL;
4186}
4187
4188static struct cgroup_subsys_state * __ref
4189mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4190{
4191 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4192 struct mem_cgroup *memcg;
4193 long error = -ENOMEM;
4194
4195 memcg = mem_cgroup_alloc();
4196 if (!memcg)
4197 return ERR_PTR(error);
4198
4199 memcg->high = PAGE_COUNTER_MAX;
4200 memcg->soft_limit = PAGE_COUNTER_MAX;
4201 if (parent) {
4202 memcg->swappiness = mem_cgroup_swappiness(parent);
4203 memcg->oom_kill_disable = parent->oom_kill_disable;
4204 }
4205 if (parent && parent->use_hierarchy) {
4206 memcg->use_hierarchy = true;
4207 page_counter_init(&memcg->memory, &parent->memory);
4208 page_counter_init(&memcg->swap, &parent->swap);
4209 page_counter_init(&memcg->memsw, &parent->memsw);
4210 page_counter_init(&memcg->kmem, &parent->kmem);
4211 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4212 } else {
4213 page_counter_init(&memcg->memory, NULL);
4214 page_counter_init(&memcg->swap, NULL);
4215 page_counter_init(&memcg->memsw, NULL);
4216 page_counter_init(&memcg->kmem, NULL);
4217 page_counter_init(&memcg->tcpmem, NULL);
4218
4219
4220
4221
4222
4223 if (parent != root_mem_cgroup)
4224 memory_cgrp_subsys.broken_hierarchy = true;
4225 }
4226
4227
4228 if (!parent) {
4229 root_mem_cgroup = memcg;
4230 return &memcg->css;
4231 }
4232
4233 error = memcg_online_kmem(memcg);
4234 if (error)
4235 goto fail;
4236
4237 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4238 static_branch_inc(&memcg_sockets_enabled_key);
4239
4240 return &memcg->css;
4241fail:
4242 mem_cgroup_free(memcg);
4243 return ERR_PTR(-ENOMEM);
4244}
4245
4246static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4247{
4248 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4249
4250
4251 atomic_set(&memcg->id.ref, 1);
4252 css_get(css);
4253 return 0;
4254}
4255
4256static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4257{
4258 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4259 struct mem_cgroup_event *event, *tmp;
4260
4261
4262
4263
4264
4265
4266 spin_lock(&memcg->event_list_lock);
4267 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4268 list_del_init(&event->list);
4269 schedule_work(&event->remove);
4270 }
4271 spin_unlock(&memcg->event_list_lock);
4272
4273 memcg->low = 0;
4274
4275 memcg_offline_kmem(memcg);
4276 wb_memcg_offline(memcg);
4277
4278 mem_cgroup_id_put(memcg);
4279}
4280
4281static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4282{
4283 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4284
4285 invalidate_reclaim_iterators(memcg);
4286}
4287
4288static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4289{
4290 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4291
4292 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4293 static_branch_dec(&memcg_sockets_enabled_key);
4294
4295 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4296 static_branch_dec(&memcg_sockets_enabled_key);
4297
4298 vmpressure_cleanup(&memcg->vmpressure);
4299 cancel_work_sync(&memcg->high_work);
4300 mem_cgroup_remove_from_trees(memcg);
4301 memcg_free_kmem(memcg);
4302 mem_cgroup_free(memcg);
4303}
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4319{
4320 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4321
4322 page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4323 page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4324 page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4325 page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4326 page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4327 memcg->low = 0;
4328 memcg->high = PAGE_COUNTER_MAX;
4329 memcg->soft_limit = PAGE_COUNTER_MAX;
4330 memcg_wb_domain_size_changed(memcg);
4331}
4332
4333#ifdef CONFIG_MMU
4334
4335static int mem_cgroup_do_precharge(unsigned long count)
4336{
4337 int ret;
4338
4339
4340 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4341 if (!ret) {
4342 mc.precharge += count;
4343 return ret;
4344 }
4345
4346
4347 while (count--) {
4348 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4349 if (ret)
4350 return ret;
4351 mc.precharge++;
4352 cond_resched();
4353 }
4354 return 0;
4355}
4356
4357union mc_target {
4358 struct page *page;
4359 swp_entry_t ent;
4360};
4361
4362enum mc_target_type {
4363 MC_TARGET_NONE = 0,
4364 MC_TARGET_PAGE,
4365 MC_TARGET_SWAP,
4366 MC_TARGET_DEVICE,
4367};
4368
4369static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4370 unsigned long addr, pte_t ptent)
4371{
4372 struct page *page = _vm_normal_page(vma, addr, ptent, true);
4373
4374 if (!page || !page_mapped(page))
4375 return NULL;
4376 if (PageAnon(page)) {
4377 if (!(mc.flags & MOVE_ANON))
4378 return NULL;
4379 } else {
4380 if (!(mc.flags & MOVE_FILE))
4381 return NULL;
4382 }
4383 if (!get_page_unless_zero(page))
4384 return NULL;
4385
4386 return page;
4387}
4388
4389#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
4390static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4391 pte_t ptent, swp_entry_t *entry)
4392{
4393 struct page *page = NULL;
4394 swp_entry_t ent = pte_to_swp_entry(ptent);
4395
4396 if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4397 return NULL;
4398
4399
4400
4401
4402
4403
4404 if (is_device_private_entry(ent)) {
4405 page = device_private_entry_to_page(ent);
4406
4407
4408
4409
4410 if (!page_ref_add_unless(page, 1, 1))
4411 return NULL;
4412 return page;
4413 }
4414
4415
4416
4417
4418
4419 page = find_get_page(swap_address_space(ent), swp_offset(ent));
4420 if (do_memsw_account())
4421 entry->val = ent.val;
4422
4423 return page;
4424}
4425#else
4426static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4427 pte_t ptent, swp_entry_t *entry)
4428{
4429 return NULL;
4430}
4431#endif
4432
4433static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4434 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4435{
4436 struct page *page = NULL;
4437 struct address_space *mapping;
4438 pgoff_t pgoff;
4439
4440 if (!vma->vm_file)
4441 return NULL;
4442 if (!(mc.flags & MOVE_FILE))
4443 return NULL;
4444
4445 mapping = vma->vm_file->f_mapping;
4446 pgoff = linear_page_index(vma, addr);
4447
4448
4449#ifdef CONFIG_SWAP
4450
4451 if (shmem_mapping(mapping)) {
4452 page = find_get_entry(mapping, pgoff);
4453 if (radix_tree_exceptional_entry(page)) {
4454 swp_entry_t swp = radix_to_swp_entry(page);
4455 if (do_memsw_account())
4456 *entry = swp;
4457 page = find_get_page(swap_address_space(swp),
4458 swp_offset(swp));
4459 }
4460 } else
4461 page = find_get_page(mapping, pgoff);
4462#else
4463 page = find_get_page(mapping, pgoff);
4464#endif
4465 return page;
4466}
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480static int mem_cgroup_move_account(struct page *page,
4481 bool compound,
4482 struct mem_cgroup *from,
4483 struct mem_cgroup *to)
4484{
4485 unsigned long flags;
4486 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4487 int ret;
4488 bool anon;
4489
4490 VM_BUG_ON(from == to);
4491 VM_BUG_ON_PAGE(PageLRU(page), page);
4492 VM_BUG_ON(compound && !PageTransHuge(page));
4493
4494
4495
4496
4497
4498 ret = -EBUSY;
4499 if (!trylock_page(page))
4500 goto out;
4501
4502 ret = -EINVAL;
4503 if (page->mem_cgroup != from)
4504 goto out_unlock;
4505
4506 anon = PageAnon(page);
4507
4508 spin_lock_irqsave(&from->move_lock, flags);
4509
4510 if (!anon && page_mapped(page)) {
4511 __mod_memcg_state(from, NR_FILE_MAPPED, -nr_pages);
4512 __mod_memcg_state(to, NR_FILE_MAPPED, nr_pages);
4513 }
4514
4515
4516
4517
4518
4519
4520 if (!anon && PageDirty(page)) {
4521 struct address_space *mapping = page_mapping(page);
4522
4523 if (mapping_cap_account_dirty(mapping)) {
4524 __mod_memcg_state(from, NR_FILE_DIRTY, -nr_pages);
4525 __mod_memcg_state(to, NR_FILE_DIRTY, nr_pages);
4526 }
4527 }
4528
4529 if (PageWriteback(page)) {
4530 __mod_memcg_state(from, NR_WRITEBACK, -nr_pages);
4531 __mod_memcg_state(to, NR_WRITEBACK, nr_pages);
4532 }
4533
4534
4535
4536
4537
4538
4539
4540
4541 page->mem_cgroup = to;
4542 spin_unlock_irqrestore(&from->move_lock, flags);
4543
4544 ret = 0;
4545
4546 local_irq_disable();
4547 mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4548 memcg_check_events(to, page);
4549 mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4550 memcg_check_events(from, page);
4551 local_irq_enable();
4552out_unlock:
4553 unlock_page(page);
4554out:
4555 return ret;
4556}
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4585 unsigned long addr, pte_t ptent, union mc_target *target)
4586{
4587 struct page *page = NULL;
4588 enum mc_target_type ret = MC_TARGET_NONE;
4589 swp_entry_t ent = { .val = 0 };
4590
4591 if (pte_present(ptent))
4592 page = mc_handle_present_pte(vma, addr, ptent);
4593 else if (is_swap_pte(ptent))
4594 page = mc_handle_swap_pte(vma, ptent, &ent);
4595 else if (pte_none(ptent))
4596 page = mc_handle_file_pte(vma, addr, ptent, &ent);
4597
4598 if (!page && !ent.val)
4599 return ret;
4600 if (page) {
4601
4602
4603
4604
4605
4606 if (page->mem_cgroup == mc.from) {
4607 ret = MC_TARGET_PAGE;
4608 if (is_device_private_page(page) ||
4609 is_device_public_page(page))
4610 ret = MC_TARGET_DEVICE;
4611 if (target)
4612 target->page = page;
4613 }
4614 if (!ret || !target)
4615 put_page(page);
4616 }
4617
4618
4619
4620
4621 if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
4622 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4623 ret = MC_TARGET_SWAP;
4624 if (target)
4625 target->ent = ent;
4626 }
4627 return ret;
4628}
4629
4630#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4631
4632
4633
4634
4635
4636static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4637 unsigned long addr, pmd_t pmd, union mc_target *target)
4638{
4639 struct page *page = NULL;
4640 enum mc_target_type ret = MC_TARGET_NONE;
4641
4642 if (unlikely(is_swap_pmd(pmd))) {
4643 VM_BUG_ON(thp_migration_supported() &&
4644 !is_pmd_migration_entry(pmd));
4645 return ret;
4646 }
4647 page = pmd_page(pmd);
4648 VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4649 if (!(mc.flags & MOVE_ANON))
4650 return ret;
4651 if (page->mem_cgroup == mc.from) {
4652 ret = MC_TARGET_PAGE;
4653 if (target) {
4654 get_page(page);
4655 target->page = page;
4656 }
4657 }
4658 return ret;
4659}
4660#else
4661static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4662 unsigned long addr, pmd_t pmd, union mc_target *target)
4663{
4664 return MC_TARGET_NONE;
4665}
4666#endif
4667
4668static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4669 unsigned long addr, unsigned long end,
4670 struct mm_walk *walk)
4671{
4672 struct vm_area_struct *vma = walk->vma;
4673 pte_t *pte;
4674 spinlock_t *ptl;
4675
4676 ptl = pmd_trans_huge_lock(pmd, vma);
4677 if (ptl) {
4678
4679
4680
4681
4682
4683 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4684 mc.precharge += HPAGE_PMD_NR;
4685 spin_unlock(ptl);
4686 return 0;
4687 }
4688
4689 if (pmd_trans_unstable(pmd))
4690 return 0;
4691 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4692 for (; addr != end; pte++, addr += PAGE_SIZE)
4693 if (get_mctgt_type(vma, addr, *pte, NULL))
4694 mc.precharge++;
4695 pte_unmap_unlock(pte - 1, ptl);
4696 cond_resched();
4697
4698 return 0;
4699}
4700
4701static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4702{
4703 unsigned long precharge;
4704
4705 struct mm_walk mem_cgroup_count_precharge_walk = {
4706 .pmd_entry = mem_cgroup_count_precharge_pte_range,
4707 .mm = mm,
4708 };
4709 down_read(&mm->mmap_sem);
4710 walk_page_range(0, mm->highest_vm_end,
4711 &mem_cgroup_count_precharge_walk);
4712 up_read(&mm->mmap_sem);
4713
4714 precharge = mc.precharge;
4715 mc.precharge = 0;
4716
4717 return precharge;
4718}
4719
4720static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4721{
4722 unsigned long precharge = mem_cgroup_count_precharge(mm);
4723
4724 VM_BUG_ON(mc.moving_task);
4725 mc.moving_task = current;
4726 return mem_cgroup_do_precharge(precharge);
4727}
4728
4729
4730static void __mem_cgroup_clear_mc(void)
4731{
4732 struct mem_cgroup *from = mc.from;
4733 struct mem_cgroup *to = mc.to;
4734
4735
4736 if (mc.precharge) {
4737 cancel_charge(mc.to, mc.precharge);
4738 mc.precharge = 0;
4739 }
4740
4741
4742
4743
4744 if (mc.moved_charge) {
4745 cancel_charge(mc.from, mc.moved_charge);
4746 mc.moved_charge = 0;
4747 }
4748
4749 if (mc.moved_swap) {
4750
4751 if (!mem_cgroup_is_root(mc.from))
4752 page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4753
4754 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
4755
4756
4757
4758
4759
4760 if (!mem_cgroup_is_root(mc.to))
4761 page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4762
4763 mem_cgroup_id_get_many(mc.to, mc.moved_swap);
4764 css_put_many(&mc.to->css, mc.moved_swap);
4765
4766 mc.moved_swap = 0;
4767 }
4768 memcg_oom_recover(from);
4769 memcg_oom_recover(to);
4770 wake_up_all(&mc.waitq);
4771}
4772
4773static void mem_cgroup_clear_mc(void)
4774{
4775 struct mm_struct *mm = mc.mm;
4776
4777
4778
4779
4780
4781 mc.moving_task = NULL;
4782 __mem_cgroup_clear_mc();
4783 spin_lock(&mc.lock);
4784 mc.from = NULL;
4785 mc.to = NULL;
4786 mc.mm = NULL;
4787 spin_unlock(&mc.lock);
4788
4789 mmput(mm);
4790}
4791
4792static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4793{
4794 struct cgroup_subsys_state *css;
4795 struct mem_cgroup *memcg = NULL;
4796 struct mem_cgroup *from;
4797 struct task_struct *leader, *p;
4798 struct mm_struct *mm;
4799 unsigned long move_flags;
4800 int ret = 0;
4801
4802
4803 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4804 return 0;
4805
4806
4807
4808
4809
4810
4811
4812 p = NULL;
4813 cgroup_taskset_for_each_leader(leader, css, tset) {
4814 WARN_ON_ONCE(p);
4815 p = leader;
4816 memcg = mem_cgroup_from_css(css);
4817 }
4818 if (!p)
4819 return 0;
4820
4821
4822
4823
4824
4825
4826 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4827 if (!move_flags)
4828 return 0;
4829
4830 from = mem_cgroup_from_task(p);
4831
4832 VM_BUG_ON(from == memcg);
4833
4834 mm = get_task_mm(p);
4835 if (!mm)
4836 return 0;
4837
4838 if (mm->owner == p) {
4839 VM_BUG_ON(mc.from);
4840 VM_BUG_ON(mc.to);
4841 VM_BUG_ON(mc.precharge);
4842 VM_BUG_ON(mc.moved_charge);
4843 VM_BUG_ON(mc.moved_swap);
4844
4845 spin_lock(&mc.lock);
4846 mc.mm = mm;
4847 mc.from = from;
4848 mc.to = memcg;
4849 mc.flags = move_flags;
4850 spin_unlock(&mc.lock);
4851
4852
4853 ret = mem_cgroup_precharge_mc(mm);
4854 if (ret)
4855 mem_cgroup_clear_mc();
4856 } else {
4857 mmput(mm);
4858 }
4859 return ret;
4860}
4861
4862static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4863{
4864 if (mc.to)
4865 mem_cgroup_clear_mc();
4866}
4867
4868static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4869 unsigned long addr, unsigned long end,
4870 struct mm_walk *walk)
4871{
4872 int ret = 0;
4873 struct vm_area_struct *vma = walk->vma;
4874 pte_t *pte;
4875 spinlock_t *ptl;
4876 enum mc_target_type target_type;
4877 union mc_target target;
4878 struct page *page;
4879
4880 ptl = pmd_trans_huge_lock(pmd, vma);
4881 if (ptl) {
4882 if (mc.precharge < HPAGE_PMD_NR) {
4883 spin_unlock(ptl);
4884 return 0;
4885 }
4886 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
4887 if (target_type == MC_TARGET_PAGE) {
4888 page = target.page;
4889 if (!isolate_lru_page(page)) {
4890 if (!mem_cgroup_move_account(page, true,
4891 mc.from, mc.to)) {
4892 mc.precharge -= HPAGE_PMD_NR;
4893 mc.moved_charge += HPAGE_PMD_NR;
4894 }
4895 putback_lru_page(page);
4896 }
4897 put_page(page);
4898 } else if (target_type == MC_TARGET_DEVICE) {
4899 page = target.page;
4900 if (!mem_cgroup_move_account(page, true,
4901 mc.from, mc.to)) {
4902 mc.precharge -= HPAGE_PMD_NR;
4903 mc.moved_charge += HPAGE_PMD_NR;
4904 }
4905 put_page(page);
4906 }
4907 spin_unlock(ptl);
4908 return 0;
4909 }
4910
4911 if (pmd_trans_unstable(pmd))
4912 return 0;
4913retry:
4914 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4915 for (; addr != end; addr += PAGE_SIZE) {
4916 pte_t ptent = *(pte++);
4917 bool device = false;
4918 swp_entry_t ent;
4919
4920 if (!mc.precharge)
4921 break;
4922
4923 switch (get_mctgt_type(vma, addr, ptent, &target)) {
4924 case MC_TARGET_DEVICE:
4925 device = true;
4926
4927 case MC_TARGET_PAGE:
4928 page = target.page;
4929
4930
4931
4932
4933
4934
4935 if (PageTransCompound(page))
4936 goto put;
4937 if (!device && isolate_lru_page(page))
4938 goto put;
4939 if (!mem_cgroup_move_account(page, false,
4940 mc.from, mc.to)) {
4941 mc.precharge--;
4942
4943 mc.moved_charge++;
4944 }
4945 if (!device)
4946 putback_lru_page(page);
4947put:
4948 put_page(page);
4949 break;
4950 case MC_TARGET_SWAP:
4951 ent = target.ent;
4952 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
4953 mc.precharge--;
4954
4955 mc.moved_swap++;
4956 }
4957 break;
4958 default:
4959 break;
4960 }
4961 }
4962 pte_unmap_unlock(pte - 1, ptl);
4963 cond_resched();
4964
4965 if (addr != end) {
4966
4967
4968
4969
4970
4971
4972 ret = mem_cgroup_do_precharge(1);
4973 if (!ret)
4974 goto retry;
4975 }
4976
4977 return ret;
4978}
4979
4980static void mem_cgroup_move_charge(void)
4981{
4982 struct mm_walk mem_cgroup_move_charge_walk = {
4983 .pmd_entry = mem_cgroup_move_charge_pte_range,
4984 .mm = mc.mm,
4985 };
4986
4987 lru_add_drain_all();
4988
4989
4990
4991
4992
4993 atomic_inc(&mc.from->moving_account);
4994 synchronize_rcu();
4995retry:
4996 if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
4997
4998
4999
5000
5001
5002
5003
5004 __mem_cgroup_clear_mc();
5005 cond_resched();
5006 goto retry;
5007 }
5008
5009
5010
5011
5012 walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
5013
5014 up_read(&mc.mm->mmap_sem);
5015 atomic_dec(&mc.from->moving_account);
5016}
5017
5018static void mem_cgroup_move_task(void)
5019{
5020 if (mc.to) {
5021 mem_cgroup_move_charge();
5022 mem_cgroup_clear_mc();
5023 }
5024}
5025#else
5026static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5027{
5028 return 0;
5029}
5030static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5031{
5032}
5033static void mem_cgroup_move_task(void)
5034{
5035}
5036#endif
5037
5038
5039
5040
5041
5042
5043static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5044{
5045
5046
5047
5048
5049
5050 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5051 root_mem_cgroup->use_hierarchy = true;
5052 else
5053 root_mem_cgroup->use_hierarchy = false;
5054}
5055
5056static u64 memory_current_read(struct cgroup_subsys_state *css,
5057 struct cftype *cft)
5058{
5059 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5060
5061 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5062}
5063
5064static int memory_low_show(struct seq_file *m, void *v)
5065{
5066 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5067 unsigned long low = READ_ONCE(memcg->low);
5068
5069 if (low == PAGE_COUNTER_MAX)
5070 seq_puts(m, "max\n");
5071 else
5072 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5073
5074 return 0;
5075}
5076
5077static ssize_t memory_low_write(struct kernfs_open_file *of,
5078 char *buf, size_t nbytes, loff_t off)
5079{
5080 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5081 unsigned long low;
5082 int err;
5083
5084 buf = strstrip(buf);
5085 err = page_counter_memparse(buf, "max", &low);
5086 if (err)
5087 return err;
5088
5089 memcg->low = low;
5090
5091 return nbytes;
5092}
5093
5094static int memory_high_show(struct seq_file *m, void *v)
5095{
5096 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5097 unsigned long high = READ_ONCE(memcg->high);
5098
5099 if (high == PAGE_COUNTER_MAX)
5100 seq_puts(m, "max\n");
5101 else
5102 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5103
5104 return 0;
5105}
5106
5107static ssize_t memory_high_write(struct kernfs_open_file *of,
5108 char *buf, size_t nbytes, loff_t off)
5109{
5110 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5111 unsigned long nr_pages;
5112 unsigned long high;
5113 int err;
5114
5115 buf = strstrip(buf);
5116 err = page_counter_memparse(buf, "max", &high);
5117 if (err)
5118 return err;
5119
5120 memcg->high = high;
5121
5122 nr_pages = page_counter_read(&memcg->memory);
5123 if (nr_pages > high)
5124 try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5125 GFP_KERNEL, true);
5126
5127 memcg_wb_domain_size_changed(memcg);
5128 return nbytes;
5129}
5130
5131static int memory_max_show(struct seq_file *m, void *v)
5132{
5133 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5134 unsigned long max = READ_ONCE(memcg->memory.limit);
5135
5136 if (max == PAGE_COUNTER_MAX)
5137 seq_puts(m, "max\n");
5138 else
5139 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5140
5141 return 0;
5142}
5143
5144static ssize_t memory_max_write(struct kernfs_open_file *of,
5145 char *buf, size_t nbytes, loff_t off)
5146{
5147 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5148 unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5149 bool drained = false;
5150 unsigned long max;
5151 int err;
5152
5153 buf = strstrip(buf);
5154 err = page_counter_memparse(buf, "max", &max);
5155 if (err)
5156 return err;
5157
5158 xchg(&memcg->memory.limit, max);
5159
5160 for (;;) {
5161 unsigned long nr_pages = page_counter_read(&memcg->memory);
5162
5163 if (nr_pages <= max)
5164 break;
5165
5166 if (signal_pending(current)) {
5167 err = -EINTR;
5168 break;
5169 }
5170
5171 if (!drained) {
5172 drain_all_stock(memcg);
5173 drained = true;
5174 continue;
5175 }
5176
5177 if (nr_reclaims) {
5178 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5179 GFP_KERNEL, true))
5180 nr_reclaims--;
5181 continue;
5182 }
5183
5184 memcg_memory_event(memcg, MEMCG_OOM);
5185 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5186 break;
5187 }
5188
5189 memcg_wb_domain_size_changed(memcg);
5190 return nbytes;
5191}
5192
5193static int memory_events_show(struct seq_file *m, void *v)
5194{
5195 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5196
5197 seq_printf(m, "low %lu\n",
5198 atomic_long_read(&memcg->memory_events[MEMCG_LOW]));
5199 seq_printf(m, "high %lu\n",
5200 atomic_long_read(&memcg->memory_events[MEMCG_HIGH]));
5201 seq_printf(m, "max %lu\n",
5202 atomic_long_read(&memcg->memory_events[MEMCG_MAX]));
5203 seq_printf(m, "oom %lu\n",
5204 atomic_long_read(&memcg->memory_events[MEMCG_OOM]));
5205 seq_printf(m, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL));
5206
5207 return 0;
5208}
5209
5210static int memory_stat_show(struct seq_file *m, void *v)
5211{
5212 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5213 unsigned long stat[MEMCG_NR_STAT];
5214 unsigned long events[NR_VM_EVENT_ITEMS];
5215 int i;
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228 tree_stat(memcg, stat);
5229 tree_events(memcg, events);
5230
5231 seq_printf(m, "anon %llu\n",
5232 (u64)stat[MEMCG_RSS] * PAGE_SIZE);
5233 seq_printf(m, "file %llu\n",
5234 (u64)stat[MEMCG_CACHE] * PAGE_SIZE);
5235 seq_printf(m, "kernel_stack %llu\n",
5236 (u64)stat[MEMCG_KERNEL_STACK_KB] * 1024);
5237 seq_printf(m, "slab %llu\n",
5238 (u64)(stat[NR_SLAB_RECLAIMABLE] +
5239 stat[NR_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5240 seq_printf(m, "sock %llu\n",
5241 (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5242
5243 seq_printf(m, "shmem %llu\n",
5244 (u64)stat[NR_SHMEM] * PAGE_SIZE);
5245 seq_printf(m, "file_mapped %llu\n",
5246 (u64)stat[NR_FILE_MAPPED] * PAGE_SIZE);
5247 seq_printf(m, "file_dirty %llu\n",
5248 (u64)stat[NR_FILE_DIRTY] * PAGE_SIZE);
5249 seq_printf(m, "file_writeback %llu\n",
5250 (u64)stat[NR_WRITEBACK] * PAGE_SIZE);
5251
5252 for (i = 0; i < NR_LRU_LISTS; i++) {
5253 struct mem_cgroup *mi;
5254 unsigned long val = 0;
5255
5256 for_each_mem_cgroup_tree(mi, memcg)
5257 val += mem_cgroup_nr_lru_pages(mi, BIT(i));
5258 seq_printf(m, "%s %llu\n",
5259 mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5260 }
5261
5262 seq_printf(m, "slab_reclaimable %llu\n",
5263 (u64)stat[NR_SLAB_RECLAIMABLE] * PAGE_SIZE);
5264 seq_printf(m, "slab_unreclaimable %llu\n",
5265 (u64)stat[NR_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5266
5267
5268
5269 seq_printf(m, "pgfault %lu\n", events[PGFAULT]);
5270 seq_printf(m, "pgmajfault %lu\n", events[PGMAJFAULT]);
5271
5272 seq_printf(m, "pgrefill %lu\n", events[PGREFILL]);
5273 seq_printf(m, "pgscan %lu\n", events[PGSCAN_KSWAPD] +
5274 events[PGSCAN_DIRECT]);
5275 seq_printf(m, "pgsteal %lu\n", events[PGSTEAL_KSWAPD] +
5276 events[PGSTEAL_DIRECT]);
5277 seq_printf(m, "pgactivate %lu\n", events[PGACTIVATE]);
5278 seq_printf(m, "pgdeactivate %lu\n", events[PGDEACTIVATE]);
5279 seq_printf(m, "pglazyfree %lu\n", events[PGLAZYFREE]);
5280 seq_printf(m, "pglazyfreed %lu\n", events[PGLAZYFREED]);
5281
5282 seq_printf(m, "workingset_refault %lu\n",
5283 stat[WORKINGSET_REFAULT]);
5284 seq_printf(m, "workingset_activate %lu\n",
5285 stat[WORKINGSET_ACTIVATE]);
5286 seq_printf(m, "workingset_nodereclaim %lu\n",
5287 stat[WORKINGSET_NODERECLAIM]);
5288
5289 return 0;
5290}
5291
5292static struct cftype memory_files[] = {
5293 {
5294 .name = "current",
5295 .flags = CFTYPE_NOT_ON_ROOT,
5296 .read_u64 = memory_current_read,
5297 },
5298 {
5299 .name = "low",
5300 .flags = CFTYPE_NOT_ON_ROOT,
5301 .seq_show = memory_low_show,
5302 .write = memory_low_write,
5303 },
5304 {
5305 .name = "high",
5306 .flags = CFTYPE_NOT_ON_ROOT,
5307 .seq_show = memory_high_show,
5308 .write = memory_high_write,
5309 },
5310 {
5311 .name = "max",
5312 .flags = CFTYPE_NOT_ON_ROOT,
5313 .seq_show = memory_max_show,
5314 .write = memory_max_write,
5315 },
5316 {
5317 .name = "events",
5318 .flags = CFTYPE_NOT_ON_ROOT,
5319 .file_offset = offsetof(struct mem_cgroup, events_file),
5320 .seq_show = memory_events_show,
5321 },
5322 {
5323 .name = "stat",
5324 .flags = CFTYPE_NOT_ON_ROOT,
5325 .seq_show = memory_stat_show,
5326 },
5327 { }
5328};
5329
5330struct cgroup_subsys memory_cgrp_subsys = {
5331 .css_alloc = mem_cgroup_css_alloc,
5332 .css_online = mem_cgroup_css_online,
5333 .css_offline = mem_cgroup_css_offline,
5334 .css_released = mem_cgroup_css_released,
5335 .css_free = mem_cgroup_css_free,
5336 .css_reset = mem_cgroup_css_reset,
5337 .can_attach = mem_cgroup_can_attach,
5338 .cancel_attach = mem_cgroup_cancel_attach,
5339 .post_attach = mem_cgroup_move_task,
5340 .bind = mem_cgroup_bind,
5341 .dfl_cftypes = memory_files,
5342 .legacy_cftypes = mem_cgroup_legacy_files,
5343 .early_init = 0,
5344};
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5380{
5381 if (mem_cgroup_disabled())
5382 return false;
5383
5384 if (!root)
5385 root = root_mem_cgroup;
5386 if (memcg == root)
5387 return false;
5388
5389 for (; memcg != root; memcg = parent_mem_cgroup(memcg)) {
5390 if (page_counter_read(&memcg->memory) >= memcg->low)
5391 return false;
5392 }
5393
5394 return true;
5395}
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5416 gfp_t gfp_mask, struct mem_cgroup **memcgp,
5417 bool compound)
5418{
5419 struct mem_cgroup *memcg = NULL;
5420 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5421 int ret = 0;
5422
5423 if (mem_cgroup_disabled())
5424 goto out;
5425
5426 if (PageSwapCache(page)) {
5427
5428
5429
5430
5431
5432
5433
5434 VM_BUG_ON_PAGE(!PageLocked(page), page);
5435 if (compound_head(page)->mem_cgroup)
5436 goto out;
5437
5438 if (do_swap_account) {
5439 swp_entry_t ent = { .val = page_private(page), };
5440 unsigned short id = lookup_swap_cgroup_id(ent);
5441
5442 rcu_read_lock();
5443 memcg = mem_cgroup_from_id(id);
5444 if (memcg && !css_tryget_online(&memcg->css))
5445 memcg = NULL;
5446 rcu_read_unlock();
5447 }
5448 }
5449
5450 if (!memcg)
5451 memcg = get_mem_cgroup_from_mm(mm);
5452
5453 ret = try_charge(memcg, gfp_mask, nr_pages);
5454
5455 css_put(&memcg->css);
5456out:
5457 *memcgp = memcg;
5458 return ret;
5459}
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5479 bool lrucare, bool compound)
5480{
5481 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5482
5483 VM_BUG_ON_PAGE(!page->mapping, page);
5484 VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5485
5486 if (mem_cgroup_disabled())
5487 return;
5488
5489
5490
5491
5492
5493 if (!memcg)
5494 return;
5495
5496 commit_charge(page, memcg, lrucare);
5497
5498 local_irq_disable();
5499 mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5500 memcg_check_events(memcg, page);
5501 local_irq_enable();
5502
5503 if (do_memsw_account() && PageSwapCache(page)) {
5504 swp_entry_t entry = { .val = page_private(page) };
5505
5506
5507
5508
5509
5510 mem_cgroup_uncharge_swap(entry, nr_pages);
5511 }
5512}
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5523 bool compound)
5524{
5525 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5526
5527 if (mem_cgroup_disabled())
5528 return;
5529
5530
5531
5532
5533
5534 if (!memcg)
5535 return;
5536
5537 cancel_charge(memcg, nr_pages);
5538}
5539
5540struct uncharge_gather {
5541 struct mem_cgroup *memcg;
5542 unsigned long pgpgout;
5543 unsigned long nr_anon;
5544 unsigned long nr_file;
5545 unsigned long nr_kmem;
5546 unsigned long nr_huge;
5547 unsigned long nr_shmem;
5548 struct page *dummy_page;
5549};
5550
5551static inline void uncharge_gather_clear(struct uncharge_gather *ug)
5552{
5553 memset(ug, 0, sizeof(*ug));
5554}
5555
5556static void uncharge_batch(const struct uncharge_gather *ug)
5557{
5558 unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
5559 unsigned long flags;
5560
5561 if (!mem_cgroup_is_root(ug->memcg)) {
5562 page_counter_uncharge(&ug->memcg->memory, nr_pages);
5563 if (do_memsw_account())
5564 page_counter_uncharge(&ug->memcg->memsw, nr_pages);
5565 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
5566 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
5567 memcg_oom_recover(ug->memcg);
5568 }
5569
5570 local_irq_save(flags);
5571 __mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);
5572 __mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file);
5573 __mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
5574 __mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
5575 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
5576 __this_cpu_add(ug->memcg->stat_cpu->nr_page_events, nr_pages);
5577 memcg_check_events(ug->memcg, ug->dummy_page);
5578 local_irq_restore(flags);
5579
5580 if (!mem_cgroup_is_root(ug->memcg))
5581 css_put_many(&ug->memcg->css, nr_pages);
5582}
5583
5584static void uncharge_page(struct page *page, struct uncharge_gather *ug)
5585{
5586 VM_BUG_ON_PAGE(PageLRU(page), page);
5587 VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) &&
5588 !PageHWPoison(page) , page);
5589
5590 if (!page->mem_cgroup)
5591 return;
5592
5593
5594
5595
5596
5597
5598
5599 if (ug->memcg != page->mem_cgroup) {
5600 if (ug->memcg) {
5601 uncharge_batch(ug);
5602 uncharge_gather_clear(ug);
5603 }
5604 ug->memcg = page->mem_cgroup;
5605 }
5606
5607 if (!PageKmemcg(page)) {
5608 unsigned int nr_pages = 1;
5609
5610 if (PageTransHuge(page)) {
5611 nr_pages <<= compound_order(page);
5612 ug->nr_huge += nr_pages;
5613 }
5614 if (PageAnon(page))
5615 ug->nr_anon += nr_pages;
5616 else {
5617 ug->nr_file += nr_pages;
5618 if (PageSwapBacked(page))
5619 ug->nr_shmem += nr_pages;
5620 }
5621 ug->pgpgout++;
5622 } else {
5623 ug->nr_kmem += 1 << compound_order(page);
5624 __ClearPageKmemcg(page);
5625 }
5626
5627 ug->dummy_page = page;
5628 page->mem_cgroup = NULL;
5629}
5630
5631static void uncharge_list(struct list_head *page_list)
5632{
5633 struct uncharge_gather ug;
5634 struct list_head *next;
5635
5636 uncharge_gather_clear(&ug);
5637
5638
5639
5640
5641
5642 next = page_list->next;
5643 do {
5644 struct page *page;
5645
5646 page = list_entry(next, struct page, lru);
5647 next = page->lru.next;
5648
5649 uncharge_page(page, &ug);
5650 } while (next != page_list);
5651
5652 if (ug.memcg)
5653 uncharge_batch(&ug);
5654}
5655
5656
5657
5658
5659
5660
5661
5662
5663void mem_cgroup_uncharge(struct page *page)
5664{
5665 struct uncharge_gather ug;
5666
5667 if (mem_cgroup_disabled())
5668 return;
5669
5670
5671 if (!page->mem_cgroup)
5672 return;
5673
5674 uncharge_gather_clear(&ug);
5675 uncharge_page(page, &ug);
5676 uncharge_batch(&ug);
5677}
5678
5679
5680
5681
5682
5683
5684
5685
5686void mem_cgroup_uncharge_list(struct list_head *page_list)
5687{
5688 if (mem_cgroup_disabled())
5689 return;
5690
5691 if (!list_empty(page_list))
5692 uncharge_list(page_list);
5693}
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
5706{
5707 struct mem_cgroup *memcg;
5708 unsigned int nr_pages;
5709 bool compound;
5710 unsigned long flags;
5711
5712 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5713 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
5714 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5715 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5716 newpage);
5717
5718 if (mem_cgroup_disabled())
5719 return;
5720
5721
5722 if (newpage->mem_cgroup)
5723 return;
5724
5725
5726 memcg = oldpage->mem_cgroup;
5727 if (!memcg)
5728 return;
5729
5730
5731 compound = PageTransHuge(newpage);
5732 nr_pages = compound ? hpage_nr_pages(newpage) : 1;
5733
5734 page_counter_charge(&memcg->memory, nr_pages);
5735 if (do_memsw_account())
5736 page_counter_charge(&memcg->memsw, nr_pages);
5737 css_get_many(&memcg->css, nr_pages);
5738
5739 commit_charge(newpage, memcg, false);
5740
5741 local_irq_save(flags);
5742 mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
5743 memcg_check_events(memcg, newpage);
5744 local_irq_restore(flags);
5745}
5746
5747DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5748EXPORT_SYMBOL(memcg_sockets_enabled_key);
5749
5750void mem_cgroup_sk_alloc(struct sock *sk)
5751{
5752 struct mem_cgroup *memcg;
5753
5754 if (!mem_cgroup_sockets_enabled)
5755 return;
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766 if (sk->sk_memcg) {
5767 css_get(&sk->sk_memcg->css);
5768 return;
5769 }
5770
5771 rcu_read_lock();
5772 memcg = mem_cgroup_from_task(current);
5773 if (memcg == root_mem_cgroup)
5774 goto out;
5775 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
5776 goto out;
5777 if (css_tryget_online(&memcg->css))
5778 sk->sk_memcg = memcg;
5779out:
5780 rcu_read_unlock();
5781}
5782
5783void mem_cgroup_sk_free(struct sock *sk)
5784{
5785 if (sk->sk_memcg)
5786 css_put(&sk->sk_memcg->css);
5787}
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5798{
5799 gfp_t gfp_mask = GFP_KERNEL;
5800
5801 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5802 struct page_counter *fail;
5803
5804 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
5805 memcg->tcpmem_pressure = 0;
5806 return true;
5807 }
5808 page_counter_charge(&memcg->tcpmem, nr_pages);
5809 memcg->tcpmem_pressure = 1;
5810 return false;
5811 }
5812
5813
5814 if (in_softirq())
5815 gfp_mask = GFP_NOWAIT;
5816
5817 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
5818
5819 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
5820 return true;
5821
5822 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
5823 return false;
5824}
5825
5826
5827
5828
5829
5830
5831void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5832{
5833 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5834 page_counter_uncharge(&memcg->tcpmem, nr_pages);
5835 return;
5836 }
5837
5838 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
5839
5840 refill_stock(memcg, nr_pages);
5841}
5842
5843static int __init cgroup_memory(char *s)
5844{
5845 char *token;
5846
5847 while ((token = strsep(&s, ",")) != NULL) {
5848 if (!*token)
5849 continue;
5850 if (!strcmp(token, "nosocket"))
5851 cgroup_memory_nosocket = true;
5852 if (!strcmp(token, "nokmem"))
5853 cgroup_memory_nokmem = true;
5854 }
5855 return 0;
5856}
5857__setup("cgroup.memory=", cgroup_memory);
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867static int __init mem_cgroup_init(void)
5868{
5869 int cpu, node;
5870
5871#ifndef CONFIG_SLOB
5872
5873
5874
5875
5876
5877
5878 memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
5879 BUG_ON(!memcg_kmem_cache_wq);
5880#endif
5881
5882 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
5883 memcg_hotplug_cpu_dead);
5884
5885 for_each_possible_cpu(cpu)
5886 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5887 drain_local_stock);
5888
5889 for_each_node(node) {
5890 struct mem_cgroup_tree_per_node *rtpn;
5891
5892 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5893 node_online(node) ? node : NUMA_NO_NODE);
5894
5895 rtpn->rb_root = RB_ROOT;
5896 rtpn->rb_rightmost = NULL;
5897 spin_lock_init(&rtpn->lock);
5898 soft_limit_tree.rb_tree_per_node[node] = rtpn;
5899 }
5900
5901 return 0;
5902}
5903subsys_initcall(mem_cgroup_init);
5904
5905#ifdef CONFIG_MEMCG_SWAP
5906static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
5907{
5908 while (!atomic_inc_not_zero(&memcg->id.ref)) {
5909
5910
5911
5912
5913 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
5914 VM_BUG_ON(1);
5915 break;
5916 }
5917 memcg = parent_mem_cgroup(memcg);
5918 if (!memcg)
5919 memcg = root_mem_cgroup;
5920 }
5921 return memcg;
5922}
5923
5924
5925
5926
5927
5928
5929
5930
5931void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5932{
5933 struct mem_cgroup *memcg, *swap_memcg;
5934 unsigned int nr_entries;
5935 unsigned short oldid;
5936
5937 VM_BUG_ON_PAGE(PageLRU(page), page);
5938 VM_BUG_ON_PAGE(page_count(page), page);
5939
5940 if (!do_memsw_account())
5941 return;
5942
5943 memcg = page->mem_cgroup;
5944
5945
5946 if (!memcg)
5947 return;
5948
5949
5950
5951
5952
5953
5954 swap_memcg = mem_cgroup_id_get_online(memcg);
5955 nr_entries = hpage_nr_pages(page);
5956
5957 if (nr_entries > 1)
5958 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
5959 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
5960 nr_entries);
5961 VM_BUG_ON_PAGE(oldid, page);
5962 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
5963
5964 page->mem_cgroup = NULL;
5965
5966 if (!mem_cgroup_is_root(memcg))
5967 page_counter_uncharge(&memcg->memory, nr_entries);
5968
5969 if (memcg != swap_memcg) {
5970 if (!mem_cgroup_is_root(swap_memcg))
5971 page_counter_charge(&swap_memcg->memsw, nr_entries);
5972 page_counter_uncharge(&memcg->memsw, nr_entries);
5973 }
5974
5975
5976
5977
5978
5979
5980
5981 VM_BUG_ON(!irqs_disabled());
5982 mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
5983 -nr_entries);
5984 memcg_check_events(memcg, page);
5985
5986 if (!mem_cgroup_is_root(memcg))
5987 css_put_many(&memcg->css, nr_entries);
5988}
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
6000{
6001 unsigned int nr_pages = hpage_nr_pages(page);
6002 struct page_counter *counter;
6003 struct mem_cgroup *memcg;
6004 unsigned short oldid;
6005
6006 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
6007 return 0;
6008
6009 memcg = page->mem_cgroup;
6010
6011
6012 if (!memcg)
6013 return 0;
6014
6015 memcg = mem_cgroup_id_get_online(memcg);
6016
6017 if (!mem_cgroup_is_root(memcg) &&
6018 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
6019 mem_cgroup_id_put(memcg);
6020 return -ENOMEM;
6021 }
6022
6023
6024 if (nr_pages > 1)
6025 mem_cgroup_id_get_many(memcg, nr_pages - 1);
6026 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
6027 VM_BUG_ON_PAGE(oldid, page);
6028 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
6029
6030 return 0;
6031}
6032
6033
6034
6035
6036
6037
6038void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
6039{
6040 struct mem_cgroup *memcg;
6041 unsigned short id;
6042
6043 if (!do_swap_account)
6044 return;
6045
6046 id = swap_cgroup_record(entry, 0, nr_pages);
6047 rcu_read_lock();
6048 memcg = mem_cgroup_from_id(id);
6049 if (memcg) {
6050 if (!mem_cgroup_is_root(memcg)) {
6051 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6052 page_counter_uncharge(&memcg->swap, nr_pages);
6053 else
6054 page_counter_uncharge(&memcg->memsw, nr_pages);
6055 }
6056 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
6057 mem_cgroup_id_put_many(memcg, nr_pages);
6058 }
6059 rcu_read_unlock();
6060}
6061
6062long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
6063{
6064 long nr_swap_pages = get_nr_swap_pages();
6065
6066 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6067 return nr_swap_pages;
6068 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6069 nr_swap_pages = min_t(long, nr_swap_pages,
6070 READ_ONCE(memcg->swap.limit) -
6071 page_counter_read(&memcg->swap));
6072 return nr_swap_pages;
6073}
6074
6075bool mem_cgroup_swap_full(struct page *page)
6076{
6077 struct mem_cgroup *memcg;
6078
6079 VM_BUG_ON_PAGE(!PageLocked(page), page);
6080
6081 if (vm_swap_full())
6082 return true;
6083 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6084 return false;
6085
6086 memcg = page->mem_cgroup;
6087 if (!memcg)
6088 return false;
6089
6090 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6091 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
6092 return true;
6093
6094 return false;
6095}
6096
6097
6098#ifdef CONFIG_MEMCG_SWAP_ENABLED
6099static int really_do_swap_account __initdata = 1;
6100#else
6101static int really_do_swap_account __initdata;
6102#endif
6103
6104static int __init enable_swap_account(char *s)
6105{
6106 if (!strcmp(s, "1"))
6107 really_do_swap_account = 1;
6108 else if (!strcmp(s, "0"))
6109 really_do_swap_account = 0;
6110 return 1;
6111}
6112__setup("swapaccount=", enable_swap_account);
6113
6114static u64 swap_current_read(struct cgroup_subsys_state *css,
6115 struct cftype *cft)
6116{
6117 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6118
6119 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6120}
6121
6122static int swap_max_show(struct seq_file *m, void *v)
6123{
6124 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6125 unsigned long max = READ_ONCE(memcg->swap.limit);
6126
6127 if (max == PAGE_COUNTER_MAX)
6128 seq_puts(m, "max\n");
6129 else
6130 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6131
6132 return 0;
6133}
6134
6135static ssize_t swap_max_write(struct kernfs_open_file *of,
6136 char *buf, size_t nbytes, loff_t off)
6137{
6138 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6139 unsigned long max;
6140 int err;
6141
6142 buf = strstrip(buf);
6143 err = page_counter_memparse(buf, "max", &max);
6144 if (err)
6145 return err;
6146
6147 mutex_lock(&memcg_limit_mutex);
6148 err = page_counter_limit(&memcg->swap, max);
6149 mutex_unlock(&memcg_limit_mutex);
6150 if (err)
6151 return err;
6152
6153 return nbytes;
6154}
6155
6156static struct cftype swap_files[] = {
6157 {
6158 .name = "swap.current",
6159 .flags = CFTYPE_NOT_ON_ROOT,
6160 .read_u64 = swap_current_read,
6161 },
6162 {
6163 .name = "swap.max",
6164 .flags = CFTYPE_NOT_ON_ROOT,
6165 .seq_show = swap_max_show,
6166 .write = swap_max_write,
6167 },
6168 { }
6169};
6170
6171static struct cftype memsw_cgroup_files[] = {
6172 {
6173 .name = "memsw.usage_in_bytes",
6174 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6175 .read_u64 = mem_cgroup_read_u64,
6176 },
6177 {
6178 .name = "memsw.max_usage_in_bytes",
6179 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6180 .write = mem_cgroup_reset,
6181 .read_u64 = mem_cgroup_read_u64,
6182 },
6183 {
6184 .name = "memsw.limit_in_bytes",
6185 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6186 .write = mem_cgroup_write,
6187 .read_u64 = mem_cgroup_read_u64,
6188 },
6189 {
6190 .name = "memsw.failcnt",
6191 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6192 .write = mem_cgroup_reset,
6193 .read_u64 = mem_cgroup_read_u64,
6194 },
6195 { },
6196};
6197
6198static int __init mem_cgroup_swap_init(void)
6199{
6200 if (!mem_cgroup_disabled() && really_do_swap_account) {
6201 do_swap_account = 1;
6202 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6203 swap_files));
6204 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6205 memsw_cgroup_files));
6206 }
6207 return 0;
6208}
6209subsys_initcall(mem_cgroup_swap_init);
6210
6211#endif
6212