1
2#ifndef _LINUX_VMSTAT_H
3#define _LINUX_VMSTAT_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7#include <linux/mmzone.h>
8#include <linux/vm_event_item.h>
9#include <linux/atomic.h>
10#include <linux/static_key.h>
11#include <linux/mmdebug.h>
12
13extern int sysctl_stat_interval;
14
15#ifdef CONFIG_NUMA
16#define ENABLE_NUMA_STAT 1
17#define DISABLE_NUMA_STAT 0
18extern int sysctl_vm_numa_stat;
19DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
20int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
21 void *buffer, size_t *length, loff_t *ppos);
22#endif
23
24struct reclaim_stat {
25 unsigned nr_dirty;
26 unsigned nr_unqueued_dirty;
27 unsigned nr_congested;
28 unsigned nr_writeback;
29 unsigned nr_immediate;
30 unsigned nr_pageout;
31 unsigned nr_activate[ANON_AND_FILE];
32 unsigned nr_ref_keep;
33 unsigned nr_unmap_fail;
34 unsigned nr_lazyfree_fail;
35};
36
37enum writeback_stat_item {
38 NR_DIRTY_THRESHOLD,
39 NR_DIRTY_BG_THRESHOLD,
40 NR_VM_WRITEBACK_STAT_ITEMS,
41};
42
43#ifdef CONFIG_VM_EVENT_COUNTERS
44
45
46
47
48
49
50
51
52
53
54struct vm_event_state {
55 unsigned long event[NR_VM_EVENT_ITEMS];
56};
57
58DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
59
60
61
62
63
64static inline void __count_vm_event(enum vm_event_item item)
65{
66 raw_cpu_inc(vm_event_states.event[item]);
67}
68
69static inline void count_vm_event(enum vm_event_item item)
70{
71 this_cpu_inc(vm_event_states.event[item]);
72}
73
74static inline void __count_vm_events(enum vm_event_item item, long delta)
75{
76 raw_cpu_add(vm_event_states.event[item], delta);
77}
78
79static inline void count_vm_events(enum vm_event_item item, long delta)
80{
81 this_cpu_add(vm_event_states.event[item], delta);
82}
83
84extern void all_vm_events(unsigned long *);
85
86extern void vm_events_fold_cpu(int cpu);
87
88#else
89
90
91static inline void count_vm_event(enum vm_event_item item)
92{
93}
94static inline void count_vm_events(enum vm_event_item item, long delta)
95{
96}
97static inline void __count_vm_event(enum vm_event_item item)
98{
99}
100static inline void __count_vm_events(enum vm_event_item item, long delta)
101{
102}
103static inline void all_vm_events(unsigned long *ret)
104{
105}
106static inline void vm_events_fold_cpu(int cpu)
107{
108}
109
110#endif
111
112#ifdef CONFIG_NUMA_BALANCING
113#define count_vm_numa_event(x) count_vm_event(x)
114#define count_vm_numa_events(x, y) count_vm_events(x, y)
115#else
116#define count_vm_numa_event(x) do {} while (0)
117#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
118#endif
119
120#ifdef CONFIG_DEBUG_TLBFLUSH
121#define count_vm_tlb_event(x) count_vm_event(x)
122#define count_vm_tlb_events(x, y) count_vm_events(x, y)
123#else
124#define count_vm_tlb_event(x) do {} while (0)
125#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
126#endif
127
128#ifdef CONFIG_DEBUG_VM_VMACACHE
129#define count_vm_vmacache_event(x) count_vm_event(x)
130#else
131#define count_vm_vmacache_event(x) do {} while (0)
132#endif
133
134#define __count_zid_vm_events(item, zid, delta) \
135 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
136
137
138
139
140extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
141extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
142extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
143
144#ifdef CONFIG_NUMA
145static inline void zone_numa_state_add(long x, struct zone *zone,
146 enum numa_stat_item item)
147{
148 atomic_long_add(x, &zone->vm_numa_stat[item]);
149 atomic_long_add(x, &vm_numa_stat[item]);
150}
151
152static inline unsigned long global_numa_state(enum numa_stat_item item)
153{
154 long x = atomic_long_read(&vm_numa_stat[item]);
155
156 return x;
157}
158
159static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
160 enum numa_stat_item item)
161{
162 long x = atomic_long_read(&zone->vm_numa_stat[item]);
163 int cpu;
164
165 for_each_online_cpu(cpu)
166 x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
167
168 return x;
169}
170#endif
171
172static inline void zone_page_state_add(long x, struct zone *zone,
173 enum zone_stat_item item)
174{
175 atomic_long_add(x, &zone->vm_stat[item]);
176 atomic_long_add(x, &vm_zone_stat[item]);
177}
178
179static inline void node_page_state_add(long x, struct pglist_data *pgdat,
180 enum node_stat_item item)
181{
182 atomic_long_add(x, &pgdat->vm_stat[item]);
183 atomic_long_add(x, &vm_node_stat[item]);
184}
185
186static inline unsigned long global_zone_page_state(enum zone_stat_item item)
187{
188 long x = atomic_long_read(&vm_zone_stat[item]);
189#ifdef CONFIG_SMP
190 if (x < 0)
191 x = 0;
192#endif
193 return x;
194}
195
196static inline
197unsigned long global_node_page_state_pages(enum node_stat_item item)
198{
199 long x = atomic_long_read(&vm_node_stat[item]);
200#ifdef CONFIG_SMP
201 if (x < 0)
202 x = 0;
203#endif
204 return x;
205}
206
207static inline unsigned long global_node_page_state(enum node_stat_item item)
208{
209 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
210
211 return global_node_page_state_pages(item);
212}
213
214static inline unsigned long zone_page_state(struct zone *zone,
215 enum zone_stat_item item)
216{
217 long x = atomic_long_read(&zone->vm_stat[item]);
218#ifdef CONFIG_SMP
219 if (x < 0)
220 x = 0;
221#endif
222 return x;
223}
224
225
226
227
228
229
230
231static inline unsigned long zone_page_state_snapshot(struct zone *zone,
232 enum zone_stat_item item)
233{
234 long x = atomic_long_read(&zone->vm_stat[item]);
235
236#ifdef CONFIG_SMP
237 int cpu;
238 for_each_online_cpu(cpu)
239 x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
240
241 if (x < 0)
242 x = 0;
243#endif
244 return x;
245}
246
247#ifdef CONFIG_NUMA
248extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
249extern unsigned long sum_zone_node_page_state(int node,
250 enum zone_stat_item item);
251extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
252extern unsigned long node_page_state(struct pglist_data *pgdat,
253 enum node_stat_item item);
254extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
255 enum node_stat_item item);
256#else
257#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
258#define node_page_state(node, item) global_node_page_state(item)
259#define node_page_state_pages(node, item) global_node_page_state_pages(item)
260#endif
261
262#ifdef CONFIG_SMP
263void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
264void __inc_zone_page_state(struct page *, enum zone_stat_item);
265void __dec_zone_page_state(struct page *, enum zone_stat_item);
266
267void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
268void __inc_node_page_state(struct page *, enum node_stat_item);
269void __dec_node_page_state(struct page *, enum node_stat_item);
270
271void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
272void inc_zone_page_state(struct page *, enum zone_stat_item);
273void dec_zone_page_state(struct page *, enum zone_stat_item);
274
275void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
276void inc_node_page_state(struct page *, enum node_stat_item);
277void dec_node_page_state(struct page *, enum node_stat_item);
278
279extern void inc_node_state(struct pglist_data *, enum node_stat_item);
280extern void __inc_zone_state(struct zone *, enum zone_stat_item);
281extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
282extern void dec_zone_state(struct zone *, enum zone_stat_item);
283extern void __dec_zone_state(struct zone *, enum zone_stat_item);
284extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
285
286void quiet_vmstat(void);
287void cpu_vm_stats_fold(int cpu);
288void refresh_zone_stat_thresholds(void);
289
290struct ctl_table;
291int vmstat_refresh(struct ctl_table *, int write, void *buffer, size_t *lenp,
292 loff_t *ppos);
293
294void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
295
296int calculate_pressure_threshold(struct zone *zone);
297int calculate_normal_threshold(struct zone *zone);
298void set_pgdat_percpu_threshold(pg_data_t *pgdat,
299 int (*calculate_pressure)(struct zone *));
300#else
301
302
303
304
305
306static inline void __mod_zone_page_state(struct zone *zone,
307 enum zone_stat_item item, long delta)
308{
309 zone_page_state_add(delta, zone, item);
310}
311
312static inline void __mod_node_page_state(struct pglist_data *pgdat,
313 enum node_stat_item item, int delta)
314{
315 if (vmstat_item_in_bytes(item)) {
316
317
318
319
320
321
322 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
323 delta >>= PAGE_SHIFT;
324 }
325
326 node_page_state_add(delta, pgdat, item);
327}
328
329static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
330{
331 atomic_long_inc(&zone->vm_stat[item]);
332 atomic_long_inc(&vm_zone_stat[item]);
333}
334
335static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
336{
337 atomic_long_inc(&pgdat->vm_stat[item]);
338 atomic_long_inc(&vm_node_stat[item]);
339}
340
341static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
342{
343 atomic_long_dec(&zone->vm_stat[item]);
344 atomic_long_dec(&vm_zone_stat[item]);
345}
346
347static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
348{
349 atomic_long_dec(&pgdat->vm_stat[item]);
350 atomic_long_dec(&vm_node_stat[item]);
351}
352
353static inline void __inc_zone_page_state(struct page *page,
354 enum zone_stat_item item)
355{
356 __inc_zone_state(page_zone(page), item);
357}
358
359static inline void __inc_node_page_state(struct page *page,
360 enum node_stat_item item)
361{
362 __inc_node_state(page_pgdat(page), item);
363}
364
365
366static inline void __dec_zone_page_state(struct page *page,
367 enum zone_stat_item item)
368{
369 __dec_zone_state(page_zone(page), item);
370}
371
372static inline void __dec_node_page_state(struct page *page,
373 enum node_stat_item item)
374{
375 __dec_node_state(page_pgdat(page), item);
376}
377
378
379
380
381
382
383#define inc_zone_page_state __inc_zone_page_state
384#define dec_zone_page_state __dec_zone_page_state
385#define mod_zone_page_state __mod_zone_page_state
386
387#define inc_node_page_state __inc_node_page_state
388#define dec_node_page_state __dec_node_page_state
389#define mod_node_page_state __mod_node_page_state
390
391#define inc_zone_state __inc_zone_state
392#define inc_node_state __inc_node_state
393#define dec_zone_state __dec_zone_state
394
395#define set_pgdat_percpu_threshold(pgdat, callback) { }
396
397static inline void refresh_zone_stat_thresholds(void) { }
398static inline void cpu_vm_stats_fold(int cpu) { }
399static inline void quiet_vmstat(void) { }
400
401static inline void drain_zonestat(struct zone *zone,
402 struct per_cpu_pageset *pset) { }
403#endif
404
405static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
406 int migratetype)
407{
408 __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
409 if (is_migrate_cma(migratetype))
410 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
411}
412
413extern const char * const vmstat_text[];
414
415static inline const char *zone_stat_name(enum zone_stat_item item)
416{
417 return vmstat_text[item];
418}
419
420#ifdef CONFIG_NUMA
421static inline const char *numa_stat_name(enum numa_stat_item item)
422{
423 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
424 item];
425}
426#endif
427
428static inline const char *node_stat_name(enum node_stat_item item)
429{
430 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
431 NR_VM_NUMA_STAT_ITEMS +
432 item];
433}
434
435static inline const char *lru_list_name(enum lru_list lru)
436{
437 return node_stat_name(NR_LRU_BASE + lru) + 3;
438}
439
440static inline const char *writeback_stat_name(enum writeback_stat_item item)
441{
442 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
443 NR_VM_NUMA_STAT_ITEMS +
444 NR_VM_NODE_STAT_ITEMS +
445 item];
446}
447
448#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
449static inline const char *vm_event_name(enum vm_event_item item)
450{
451 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
452 NR_VM_NUMA_STAT_ITEMS +
453 NR_VM_NODE_STAT_ITEMS +
454 NR_VM_WRITEBACK_STAT_ITEMS +
455 item];
456}
457#endif
458
459#ifdef CONFIG_MEMCG
460
461void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
462 int val);
463
464static inline void mod_lruvec_state(struct lruvec *lruvec,
465 enum node_stat_item idx, int val)
466{
467 unsigned long flags;
468
469 local_irq_save(flags);
470 __mod_lruvec_state(lruvec, idx, val);
471 local_irq_restore(flags);
472}
473
474void __mod_lruvec_page_state(struct page *page,
475 enum node_stat_item idx, int val);
476
477static inline void mod_lruvec_page_state(struct page *page,
478 enum node_stat_item idx, int val)
479{
480 unsigned long flags;
481
482 local_irq_save(flags);
483 __mod_lruvec_page_state(page, idx, val);
484 local_irq_restore(flags);
485}
486
487#else
488
489static inline void __mod_lruvec_state(struct lruvec *lruvec,
490 enum node_stat_item idx, int val)
491{
492 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
493}
494
495static inline void mod_lruvec_state(struct lruvec *lruvec,
496 enum node_stat_item idx, int val)
497{
498 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
499}
500
501static inline void __mod_lruvec_page_state(struct page *page,
502 enum node_stat_item idx, int val)
503{
504 __mod_node_page_state(page_pgdat(page), idx, val);
505}
506
507static inline void mod_lruvec_page_state(struct page *page,
508 enum node_stat_item idx, int val)
509{
510 mod_node_page_state(page_pgdat(page), idx, val);
511}
512
513#endif
514
515static inline void __inc_lruvec_state(struct lruvec *lruvec,
516 enum node_stat_item idx)
517{
518 __mod_lruvec_state(lruvec, idx, 1);
519}
520
521static inline void __dec_lruvec_state(struct lruvec *lruvec,
522 enum node_stat_item idx)
523{
524 __mod_lruvec_state(lruvec, idx, -1);
525}
526
527static inline void __inc_lruvec_page_state(struct page *page,
528 enum node_stat_item idx)
529{
530 __mod_lruvec_page_state(page, idx, 1);
531}
532
533static inline void __dec_lruvec_page_state(struct page *page,
534 enum node_stat_item idx)
535{
536 __mod_lruvec_page_state(page, idx, -1);
537}
538
539static inline void inc_lruvec_state(struct lruvec *lruvec,
540 enum node_stat_item idx)
541{
542 mod_lruvec_state(lruvec, idx, 1);
543}
544
545static inline void dec_lruvec_state(struct lruvec *lruvec,
546 enum node_stat_item idx)
547{
548 mod_lruvec_state(lruvec, idx, -1);
549}
550
551static inline void inc_lruvec_page_state(struct page *page,
552 enum node_stat_item idx)
553{
554 mod_lruvec_page_state(page, idx, 1);
555}
556
557static inline void dec_lruvec_page_state(struct page *page,
558 enum node_stat_item idx)
559{
560 mod_lruvec_page_state(page, idx, -1);
561}
562
563#endif
564