1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#ifndef _LINUX_MEMCONTROL_H
21#define _LINUX_MEMCONTROL_H
22#include <linux/cgroup.h>
23#include <linux/vm_event_item.h>
24#include <linux/hardirq.h>
25#include <linux/jump_label.h>
26#include <linux/page_counter.h>
27#include <linux/vmpressure.h>
28#include <linux/eventfd.h>
29#include <linux/mmzone.h>
30#include <linux/writeback.h>
31
32struct mem_cgroup;
33struct page;
34struct mm_struct;
35struct kmem_cache;
36
37
38
39
40
41enum mem_cgroup_stat_index {
42
43
44
45 MEM_CGROUP_STAT_CACHE,
46 MEM_CGROUP_STAT_RSS,
47 MEM_CGROUP_STAT_RSS_HUGE,
48 MEM_CGROUP_STAT_FILE_MAPPED,
49 MEM_CGROUP_STAT_DIRTY,
50 MEM_CGROUP_STAT_WRITEBACK,
51 MEM_CGROUP_STAT_SWAP,
52 MEM_CGROUP_STAT_NSTATS,
53};
54
55struct mem_cgroup_reclaim_cookie {
56 struct zone *zone;
57 int priority;
58 unsigned int generation;
59};
60
61enum mem_cgroup_events_index {
62 MEM_CGROUP_EVENTS_PGPGIN,
63 MEM_CGROUP_EVENTS_PGPGOUT,
64 MEM_CGROUP_EVENTS_PGFAULT,
65 MEM_CGROUP_EVENTS_PGMAJFAULT,
66 MEM_CGROUP_EVENTS_NSTATS,
67
68 MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS,
69 MEMCG_HIGH,
70 MEMCG_MAX,
71 MEMCG_OOM,
72 MEMCG_NR_EVENTS,
73};
74
75
76
77
78
79
80
81enum mem_cgroup_events_target {
82 MEM_CGROUP_TARGET_THRESH,
83 MEM_CGROUP_TARGET_SOFTLIMIT,
84 MEM_CGROUP_TARGET_NUMAINFO,
85 MEM_CGROUP_NTARGETS,
86};
87
88
89
90
91enum cg_proto_flags {
92
93 MEMCG_SOCK_ACTIVE,
94
95 MEMCG_SOCK_ACTIVATED,
96};
97
98struct cg_proto {
99 struct page_counter memory_allocated;
100 struct percpu_counter sockets_allocated;
101 int memory_pressure;
102 long sysctl_mem[3];
103 unsigned long flags;
104
105
106
107
108
109
110
111
112
113 struct mem_cgroup *memcg;
114};
115
116#ifdef CONFIG_MEMCG
117struct mem_cgroup_stat_cpu {
118 long count[MEM_CGROUP_STAT_NSTATS];
119 unsigned long events[MEMCG_NR_EVENTS];
120 unsigned long nr_page_events;
121 unsigned long targets[MEM_CGROUP_NTARGETS];
122};
123
124struct mem_cgroup_reclaim_iter {
125 struct mem_cgroup *position;
126
127 unsigned int generation;
128};
129
130
131
132
133struct mem_cgroup_per_zone {
134 struct lruvec lruvec;
135 unsigned long lru_size[NR_LRU_LISTS];
136
137 struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1];
138
139 struct rb_node tree_node;
140 unsigned long usage_in_excess;
141
142 bool on_tree;
143 struct mem_cgroup *memcg;
144
145};
146
147struct mem_cgroup_per_node {
148 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
149};
150
151struct mem_cgroup_threshold {
152 struct eventfd_ctx *eventfd;
153 unsigned long threshold;
154};
155
156
157struct mem_cgroup_threshold_ary {
158
159 int current_threshold;
160
161 unsigned int size;
162
163 struct mem_cgroup_threshold entries[0];
164};
165
166struct mem_cgroup_thresholds {
167
168 struct mem_cgroup_threshold_ary *primary;
169
170
171
172
173
174 struct mem_cgroup_threshold_ary *spare;
175};
176
177
178
179
180
181
182
183struct mem_cgroup {
184 struct cgroup_subsys_state css;
185
186
187 struct page_counter memory;
188 struct page_counter memsw;
189 struct page_counter kmem;
190
191
192 unsigned long low;
193 unsigned long high;
194
195 unsigned long soft_limit;
196
197
198 struct vmpressure vmpressure;
199
200
201 int initialized;
202
203
204
205
206 bool use_hierarchy;
207
208
209 bool oom_lock;
210 int under_oom;
211
212 int swappiness;
213
214 int oom_kill_disable;
215
216
217 struct cgroup_file events_file;
218
219
220 struct mutex thresholds_lock;
221
222
223 struct mem_cgroup_thresholds thresholds;
224
225
226 struct mem_cgroup_thresholds memsw_thresholds;
227
228
229 struct list_head oom_notify;
230
231
232
233
234
235 unsigned long move_charge_at_immigrate;
236
237
238
239 atomic_t moving_account;
240
241 spinlock_t move_lock;
242 struct task_struct *move_lock_task;
243 unsigned long move_lock_flags;
244
245
246
247 struct mem_cgroup_stat_cpu __percpu *stat;
248
249#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
250 struct cg_proto tcp_mem;
251#endif
252#if defined(CONFIG_MEMCG_KMEM)
253
254 int kmemcg_id;
255 bool kmem_acct_activated;
256 bool kmem_acct_active;
257#endif
258
259 int last_scanned_node;
260#if MAX_NUMNODES > 1
261 nodemask_t scan_nodes;
262 atomic_t numainfo_events;
263 atomic_t numainfo_updating;
264#endif
265
266#ifdef CONFIG_CGROUP_WRITEBACK
267 struct list_head cgwb_list;
268 struct wb_domain cgwb_domain;
269#endif
270
271
272 struct list_head event_list;
273 spinlock_t event_list_lock;
274
275 struct mem_cgroup_per_node *nodeinfo[0];
276
277};
278extern struct cgroup_subsys_state *mem_cgroup_root_css;
279
280
281
282
283
284
285
286static inline void mem_cgroup_events(struct mem_cgroup *memcg,
287 enum mem_cgroup_events_index idx,
288 unsigned int nr)
289{
290 this_cpu_add(memcg->stat->events[idx], nr);
291 cgroup_file_notify(&memcg->events_file);
292}
293
294bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);
295
296int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
297 gfp_t gfp_mask, struct mem_cgroup **memcgp);
298void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
299 bool lrucare);
300void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg);
301void mem_cgroup_uncharge(struct page *page);
302void mem_cgroup_uncharge_list(struct list_head *page_list);
303
304void mem_cgroup_replace_page(struct page *oldpage, struct page *newpage);
305
306struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
307struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *);
308
309bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
310struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
311struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg);
312
313static inline
314struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
315 return css ? container_of(css, struct mem_cgroup, css) : NULL;
316}
317
318struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
319 struct mem_cgroup *,
320 struct mem_cgroup_reclaim_cookie *);
321void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
322
323static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
324 struct mem_cgroup *root)
325{
326 if (root == memcg)
327 return true;
328 if (!root->use_hierarchy)
329 return false;
330 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
331}
332
333static inline bool mm_match_cgroup(struct mm_struct *mm,
334 struct mem_cgroup *memcg)
335{
336 struct mem_cgroup *task_memcg;
337 bool match = false;
338
339 rcu_read_lock();
340 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
341 if (task_memcg)
342 match = mem_cgroup_is_descendant(task_memcg, memcg);
343 rcu_read_unlock();
344 return match;
345}
346
347struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
348ino_t page_cgroup_ino(struct page *page);
349
350static inline bool mem_cgroup_disabled(void)
351{
352 return !cgroup_subsys_enabled(memory_cgrp_subsys);
353}
354
355
356
357
358int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
359
360void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
361 int nr_pages);
362
363static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec)
364{
365 struct mem_cgroup_per_zone *mz;
366 struct mem_cgroup *memcg;
367
368 if (mem_cgroup_disabled())
369 return true;
370
371 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
372 memcg = mz->memcg;
373
374 return !!(memcg->css.flags & CSS_ONLINE);
375}
376
377static inline
378unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
379{
380 struct mem_cgroup_per_zone *mz;
381
382 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
383 return mz->lru_size[lru];
384}
385
386static inline bool mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
387{
388 unsigned long inactive_ratio;
389 unsigned long inactive;
390 unsigned long active;
391 unsigned long gb;
392
393 inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
394 active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
395
396 gb = (inactive + active) >> (30 - PAGE_SHIFT);
397 if (gb)
398 inactive_ratio = int_sqrt(10 * gb);
399 else
400 inactive_ratio = 1;
401
402 return inactive * inactive_ratio < active;
403}
404
405void mem_cgroup_handle_over_high(void);
406
407void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
408 struct task_struct *p);
409
410static inline void mem_cgroup_oom_enable(void)
411{
412 WARN_ON(current->memcg_may_oom);
413 current->memcg_may_oom = 1;
414}
415
416static inline void mem_cgroup_oom_disable(void)
417{
418 WARN_ON(!current->memcg_may_oom);
419 current->memcg_may_oom = 0;
420}
421
422static inline bool task_in_memcg_oom(struct task_struct *p)
423{
424 return p->memcg_in_oom;
425}
426
427bool mem_cgroup_oom_synchronize(bool wait);
428
429#ifdef CONFIG_MEMCG_SWAP
430extern int do_swap_account;
431#endif
432
433struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page);
434void mem_cgroup_end_page_stat(struct mem_cgroup *memcg);
435
436
437
438
439
440
441
442
443
444static inline void mem_cgroup_update_page_stat(struct mem_cgroup *memcg,
445 enum mem_cgroup_stat_index idx, int val)
446{
447 VM_BUG_ON(!rcu_read_lock_held());
448
449 if (memcg)
450 this_cpu_add(memcg->stat->count[idx], val);
451}
452
453static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg,
454 enum mem_cgroup_stat_index idx)
455{
456 mem_cgroup_update_page_stat(memcg, idx, 1);
457}
458
459static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg,
460 enum mem_cgroup_stat_index idx)
461{
462 mem_cgroup_update_page_stat(memcg, idx, -1);
463}
464
465unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
466 gfp_t gfp_mask,
467 unsigned long *total_scanned);
468
469static inline void mem_cgroup_count_vm_event(struct mm_struct *mm,
470 enum vm_event_item idx)
471{
472 struct mem_cgroup *memcg;
473
474 if (mem_cgroup_disabled())
475 return;
476
477 rcu_read_lock();
478 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
479 if (unlikely(!memcg))
480 goto out;
481
482 switch (idx) {
483 case PGFAULT:
484 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
485 break;
486 case PGMAJFAULT:
487 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
488 break;
489 default:
490 BUG();
491 }
492out:
493 rcu_read_unlock();
494}
495#ifdef CONFIG_TRANSPARENT_HUGEPAGE
496void mem_cgroup_split_huge_fixup(struct page *head);
497#endif
498
499#else
500struct mem_cgroup;
501
502static inline void mem_cgroup_events(struct mem_cgroup *memcg,
503 enum mem_cgroup_events_index idx,
504 unsigned int nr)
505{
506}
507
508static inline bool mem_cgroup_low(struct mem_cgroup *root,
509 struct mem_cgroup *memcg)
510{
511 return false;
512}
513
514static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
515 gfp_t gfp_mask,
516 struct mem_cgroup **memcgp)
517{
518 *memcgp = NULL;
519 return 0;
520}
521
522static inline void mem_cgroup_commit_charge(struct page *page,
523 struct mem_cgroup *memcg,
524 bool lrucare)
525{
526}
527
528static inline void mem_cgroup_cancel_charge(struct page *page,
529 struct mem_cgroup *memcg)
530{
531}
532
533static inline void mem_cgroup_uncharge(struct page *page)
534{
535}
536
537static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
538{
539}
540
541static inline void mem_cgroup_replace_page(struct page *old, struct page *new)
542{
543}
544
545static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
546 struct mem_cgroup *memcg)
547{
548 return &zone->lruvec;
549}
550
551static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
552 struct zone *zone)
553{
554 return &zone->lruvec;
555}
556
557static inline bool mm_match_cgroup(struct mm_struct *mm,
558 struct mem_cgroup *memcg)
559{
560 return true;
561}
562
563static inline bool task_in_mem_cgroup(struct task_struct *task,
564 const struct mem_cgroup *memcg)
565{
566 return true;
567}
568
569static inline struct mem_cgroup *
570mem_cgroup_iter(struct mem_cgroup *root,
571 struct mem_cgroup *prev,
572 struct mem_cgroup_reclaim_cookie *reclaim)
573{
574 return NULL;
575}
576
577static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
578 struct mem_cgroup *prev)
579{
580}
581
582static inline bool mem_cgroup_disabled(void)
583{
584 return true;
585}
586
587static inline bool
588mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
589{
590 return true;
591}
592
593static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec)
594{
595 return true;
596}
597
598static inline unsigned long
599mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
600{
601 return 0;
602}
603
604static inline void
605mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
606 int increment)
607{
608}
609
610static inline void
611mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
612{
613}
614
615static inline struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page)
616{
617 return NULL;
618}
619
620static inline void mem_cgroup_end_page_stat(struct mem_cgroup *memcg)
621{
622}
623
624static inline void mem_cgroup_handle_over_high(void)
625{
626}
627
628static inline void mem_cgroup_oom_enable(void)
629{
630}
631
632static inline void mem_cgroup_oom_disable(void)
633{
634}
635
636static inline bool task_in_memcg_oom(struct task_struct *p)
637{
638 return false;
639}
640
641static inline bool mem_cgroup_oom_synchronize(bool wait)
642{
643 return false;
644}
645
646static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg,
647 enum mem_cgroup_stat_index idx)
648{
649}
650
651static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg,
652 enum mem_cgroup_stat_index idx)
653{
654}
655
656static inline
657unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
658 gfp_t gfp_mask,
659 unsigned long *total_scanned)
660{
661 return 0;
662}
663
664static inline void mem_cgroup_split_huge_fixup(struct page *head)
665{
666}
667
668static inline
669void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
670{
671}
672#endif
673
674enum {
675 UNDER_LIMIT,
676 SOFT_LIMIT,
677 OVER_LIMIT,
678};
679
680#ifdef CONFIG_CGROUP_WRITEBACK
681
682struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
683struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
684void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
685 unsigned long *pheadroom, unsigned long *pdirty,
686 unsigned long *pwriteback);
687
688#else
689
690static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
691{
692 return NULL;
693}
694
695static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
696 unsigned long *pfilepages,
697 unsigned long *pheadroom,
698 unsigned long *pdirty,
699 unsigned long *pwriteback)
700{
701}
702
703#endif
704
705struct sock;
706#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
707void sock_update_memcg(struct sock *sk);
708void sock_release_memcg(struct sock *sk);
709#else
710static inline void sock_update_memcg(struct sock *sk)
711{
712}
713static inline void sock_release_memcg(struct sock *sk)
714{
715}
716#endif
717
718#ifdef CONFIG_MEMCG_KMEM
719extern struct static_key memcg_kmem_enabled_key;
720
721extern int memcg_nr_cache_ids;
722void memcg_get_cache_ids(void);
723void memcg_put_cache_ids(void);
724
725
726
727
728
729
730#define for_each_memcg_cache_index(_idx) \
731 for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
732
733static inline bool memcg_kmem_enabled(void)
734{
735 return static_key_false(&memcg_kmem_enabled_key);
736}
737
738static inline bool memcg_kmem_is_active(struct mem_cgroup *memcg)
739{
740 return memcg->kmem_acct_active;
741}
742
743
744
745
746
747
748
749
750
751
752
753
754int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
755 struct mem_cgroup *memcg);
756int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
757void __memcg_kmem_uncharge(struct page *page, int order);
758
759
760
761
762
763
764static inline int memcg_cache_id(struct mem_cgroup *memcg)
765{
766 return memcg ? memcg->kmemcg_id : -1;
767}
768
769struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep);
770void __memcg_kmem_put_cache(struct kmem_cache *cachep);
771
772static inline bool __memcg_kmem_bypass(gfp_t gfp)
773{
774 if (!memcg_kmem_enabled())
775 return true;
776 if (gfp & __GFP_NOACCOUNT)
777 return true;
778 if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD))
779 return true;
780 return false;
781}
782
783
784
785
786
787
788
789
790
791static __always_inline int memcg_kmem_charge(struct page *page,
792 gfp_t gfp, int order)
793{
794 if (__memcg_kmem_bypass(gfp))
795 return 0;
796 return __memcg_kmem_charge(page, gfp, order);
797}
798
799
800
801
802
803
804static __always_inline void memcg_kmem_uncharge(struct page *page, int order)
805{
806 if (memcg_kmem_enabled())
807 __memcg_kmem_uncharge(page, order);
808}
809
810
811
812
813
814
815
816
817static __always_inline struct kmem_cache *
818memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
819{
820 if (__memcg_kmem_bypass(gfp))
821 return cachep;
822 return __memcg_kmem_get_cache(cachep);
823}
824
825static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
826{
827 if (memcg_kmem_enabled())
828 __memcg_kmem_put_cache(cachep);
829}
830#else
831#define for_each_memcg_cache_index(_idx) \
832 for (; NULL; )
833
834static inline bool memcg_kmem_enabled(void)
835{
836 return false;
837}
838
839static inline bool memcg_kmem_is_active(struct mem_cgroup *memcg)
840{
841 return false;
842}
843
844static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
845{
846 return 0;
847}
848
849static inline void memcg_kmem_uncharge(struct page *page, int order)
850{
851}
852
853static inline int memcg_cache_id(struct mem_cgroup *memcg)
854{
855 return -1;
856}
857
858static inline void memcg_get_cache_ids(void)
859{
860}
861
862static inline void memcg_put_cache_ids(void)
863{
864}
865
866static inline struct kmem_cache *
867memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
868{
869 return cachep;
870}
871
872static inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
873{
874}
875#endif
876#endif
877