1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99enum pageflags {
100 PG_locked,
101 PG_referenced,
102 PG_uptodate,
103 PG_dirty,
104 PG_lru,
105 PG_active,
106 PG_workingset,
107 PG_waiters,
108 PG_error,
109 PG_slab,
110 PG_owner_priv_1,
111 PG_arch_1,
112 PG_reserved,
113 PG_private,
114 PG_private_2,
115 PG_writeback,
116 PG_head,
117 PG_mappedtodisk,
118 PG_reclaim,
119 PG_swapbacked,
120 PG_unevictable,
121#ifdef CONFIG_MMU
122 PG_mlocked,
123#endif
124#ifdef CONFIG_ARCH_USES_PG_UNCACHED
125 PG_uncached,
126#endif
127#ifdef CONFIG_MEMORY_FAILURE
128 PG_hwpoison,
129#endif
130#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
131 PG_young,
132 PG_idle,
133#endif
134 __NR_PAGEFLAGS,
135
136
137 PG_checked = PG_owner_priv_1,
138
139
140 PG_swapcache = PG_owner_priv_1,
141
142
143
144
145
146 PG_fscache = PG_private_2,
147
148
149
150 PG_pinned = PG_owner_priv_1,
151
152 PG_savepinned = PG_dirty,
153
154 PG_foreign = PG_owner_priv_1,
155
156
157 PG_slob_free = PG_private,
158
159
160 PG_double_map = PG_private_2,
161
162
163 PG_isolated = PG_reclaim,
164};
165
166#ifndef __GENERATING_BOUNDS_H
167
168struct page;
169
170static inline struct page *compound_head(struct page *page)
171{
172 unsigned long head = READ_ONCE(page->compound_head);
173
174 if (unlikely(head & 1))
175 return (struct page *) (head - 1);
176 return page;
177}
178
179static __always_inline int PageTail(struct page *page)
180{
181 return READ_ONCE(page->compound_head) & 1;
182}
183
184static __always_inline int PageCompound(struct page *page)
185{
186 return test_bit(PG_head, &page->flags) || PageTail(page);
187}
188
189#define PAGE_POISON_PATTERN -1l
190static inline int PagePoisoned(const struct page *page)
191{
192 return page->flags == PAGE_POISON_PATTERN;
193}
194
195#ifdef CONFIG_DEBUG_VM
196void page_init_poison(struct page *page, size_t size);
197#else
198static inline void page_init_poison(struct page *page, size_t size)
199{
200}
201#endif
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226#define PF_POISONED_CHECK(page) ({ \
227 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
228 page; })
229#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
230#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
231#define PF_ONLY_HEAD(page, enforce) ({ \
232 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
233 PF_POISONED_CHECK(page); })
234#define PF_NO_TAIL(page, enforce) ({ \
235 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
236 PF_POISONED_CHECK(compound_head(page)); })
237#define PF_NO_COMPOUND(page, enforce) ({ \
238 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
239 PF_POISONED_CHECK(page); })
240
241
242
243
244#define TESTPAGEFLAG(uname, lname, policy) \
245static __always_inline int Page##uname(struct page *page) \
246 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
247
248#define SETPAGEFLAG(uname, lname, policy) \
249static __always_inline void SetPage##uname(struct page *page) \
250 { set_bit(PG_##lname, &policy(page, 1)->flags); }
251
252#define CLEARPAGEFLAG(uname, lname, policy) \
253static __always_inline void ClearPage##uname(struct page *page) \
254 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
255
256#define __SETPAGEFLAG(uname, lname, policy) \
257static __always_inline void __SetPage##uname(struct page *page) \
258 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
259
260#define __CLEARPAGEFLAG(uname, lname, policy) \
261static __always_inline void __ClearPage##uname(struct page *page) \
262 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
263
264#define TESTSETFLAG(uname, lname, policy) \
265static __always_inline int TestSetPage##uname(struct page *page) \
266 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
267
268#define TESTCLEARFLAG(uname, lname, policy) \
269static __always_inline int TestClearPage##uname(struct page *page) \
270 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
271
272#define PAGEFLAG(uname, lname, policy) \
273 TESTPAGEFLAG(uname, lname, policy) \
274 SETPAGEFLAG(uname, lname, policy) \
275 CLEARPAGEFLAG(uname, lname, policy)
276
277#define __PAGEFLAG(uname, lname, policy) \
278 TESTPAGEFLAG(uname, lname, policy) \
279 __SETPAGEFLAG(uname, lname, policy) \
280 __CLEARPAGEFLAG(uname, lname, policy)
281
282#define TESTSCFLAG(uname, lname, policy) \
283 TESTSETFLAG(uname, lname, policy) \
284 TESTCLEARFLAG(uname, lname, policy)
285
286#define TESTPAGEFLAG_FALSE(uname) \
287static inline int Page##uname(const struct page *page) { return 0; }
288
289#define SETPAGEFLAG_NOOP(uname) \
290static inline void SetPage##uname(struct page *page) { }
291
292#define CLEARPAGEFLAG_NOOP(uname) \
293static inline void ClearPage##uname(struct page *page) { }
294
295#define __CLEARPAGEFLAG_NOOP(uname) \
296static inline void __ClearPage##uname(struct page *page) { }
297
298#define TESTSETFLAG_FALSE(uname) \
299static inline int TestSetPage##uname(struct page *page) { return 0; }
300
301#define TESTCLEARFLAG_FALSE(uname) \
302static inline int TestClearPage##uname(struct page *page) { return 0; }
303
304#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
305 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
306
307#define TESTSCFLAG_FALSE(uname) \
308 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
309
310__PAGEFLAG(Locked, locked, PF_NO_TAIL)
311PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
312PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
313PAGEFLAG(Referenced, referenced, PF_HEAD)
314 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
315 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
316PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
317 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
318PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
319PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
320 TESTCLEARFLAG(Active, active, PF_HEAD)
321PAGEFLAG(Workingset, workingset, PF_HEAD)
322 TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
323__PAGEFLAG(Slab, slab, PF_NO_TAIL)
324__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
325PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
326
327
328PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
329 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
330PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
331PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
332
333PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
334 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
335 __SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
336PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
337 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
338 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
339
340
341
342
343
344
345PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
346 __CLEARPAGEFLAG(Private, private, PF_ANY)
347PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
348PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
349 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
350
351
352
353
354
355TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
356 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
357PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
358
359
360PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
361 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
362PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
363 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
364
365#ifdef CONFIG_HIGHMEM
366
367
368
369
370#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
371#else
372PAGEFLAG_FALSE(HighMem)
373#endif
374
375#ifdef CONFIG_SWAP
376static __always_inline int PageSwapCache(struct page *page)
377{
378#ifdef CONFIG_THP_SWAP
379 page = compound_head(page);
380#endif
381 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
382
383}
384SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
385CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
386#else
387PAGEFLAG_FALSE(SwapCache)
388#endif
389
390PAGEFLAG(Unevictable, unevictable, PF_HEAD)
391 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
392 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
393
394#ifdef CONFIG_MMU
395PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
396 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
397 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
398#else
399PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
400 TESTSCFLAG_FALSE(Mlocked)
401#endif
402
403#ifdef CONFIG_ARCH_USES_PG_UNCACHED
404PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
405#else
406PAGEFLAG_FALSE(Uncached)
407#endif
408
409#ifdef CONFIG_MEMORY_FAILURE
410PAGEFLAG(HWPoison, hwpoison, PF_ANY)
411TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
412#define __PG_HWPOISON (1UL << PG_hwpoison)
413extern bool set_hwpoison_free_buddy_page(struct page *page);
414#else
415PAGEFLAG_FALSE(HWPoison)
416static inline bool set_hwpoison_free_buddy_page(struct page *page)
417{
418 return 0;
419}
420#define __PG_HWPOISON 0
421#endif
422
423#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
424TESTPAGEFLAG(Young, young, PF_ANY)
425SETPAGEFLAG(Young, young, PF_ANY)
426TESTCLEARFLAG(Young, young, PF_ANY)
427PAGEFLAG(Idle, idle, PF_ANY)
428#endif
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447#define PAGE_MAPPING_ANON 0x1
448#define PAGE_MAPPING_MOVABLE 0x2
449#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
450#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
451
452static __always_inline int PageMappingFlags(struct page *page)
453{
454 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
455}
456
457static __always_inline int PageAnon(struct page *page)
458{
459 page = compound_head(page);
460 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
461}
462
463static __always_inline int __PageMovable(struct page *page)
464{
465 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
466 PAGE_MAPPING_MOVABLE;
467}
468
469#ifdef CONFIG_KSM
470
471
472
473
474
475
476static __always_inline int PageKsm(struct page *page)
477{
478 page = compound_head(page);
479 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
480 PAGE_MAPPING_KSM;
481}
482#else
483TESTPAGEFLAG_FALSE(Ksm)
484#endif
485
486u64 stable_page_flags(struct page *page);
487
488static inline int PageUptodate(struct page *page)
489{
490 int ret;
491 page = compound_head(page);
492 ret = test_bit(PG_uptodate, &(page)->flags);
493
494
495
496
497
498
499
500
501 if (ret)
502 smp_rmb();
503
504 return ret;
505}
506
507static __always_inline void __SetPageUptodate(struct page *page)
508{
509 VM_BUG_ON_PAGE(PageTail(page), page);
510 smp_wmb();
511 __set_bit(PG_uptodate, &page->flags);
512}
513
514static __always_inline void SetPageUptodate(struct page *page)
515{
516 VM_BUG_ON_PAGE(PageTail(page), page);
517
518
519
520
521
522 smp_wmb();
523 set_bit(PG_uptodate, &page->flags);
524}
525
526CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
527
528int test_clear_page_writeback(struct page *page);
529int __test_set_page_writeback(struct page *page, bool keep_write);
530
531#define test_set_page_writeback(page) \
532 __test_set_page_writeback(page, false)
533#define test_set_page_writeback_keepwrite(page) \
534 __test_set_page_writeback(page, true)
535
536static inline void set_page_writeback(struct page *page)
537{
538 test_set_page_writeback(page);
539}
540
541static inline void set_page_writeback_keepwrite(struct page *page)
542{
543 test_set_page_writeback_keepwrite(page);
544}
545
546__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
547
548static __always_inline void set_compound_head(struct page *page, struct page *head)
549{
550 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
551}
552
553static __always_inline void clear_compound_head(struct page *page)
554{
555 WRITE_ONCE(page->compound_head, 0);
556}
557
558#ifdef CONFIG_TRANSPARENT_HUGEPAGE
559static inline void ClearPageCompound(struct page *page)
560{
561 BUG_ON(!PageHead(page));
562 ClearPageHead(page);
563}
564#endif
565
566#define PG_head_mask ((1UL << PG_head))
567
568#ifdef CONFIG_HUGETLB_PAGE
569int PageHuge(struct page *page);
570int PageHeadHuge(struct page *page);
571bool page_huge_active(struct page *page);
572#else
573TESTPAGEFLAG_FALSE(Huge)
574TESTPAGEFLAG_FALSE(HeadHuge)
575
576static inline bool page_huge_active(struct page *page)
577{
578 return 0;
579}
580#endif
581
582
583#ifdef CONFIG_TRANSPARENT_HUGEPAGE
584
585
586
587
588
589
590
591
592static inline int PageTransHuge(struct page *page)
593{
594 VM_BUG_ON_PAGE(PageTail(page), page);
595 return PageHead(page);
596}
597
598
599
600
601
602
603static inline int PageTransCompound(struct page *page)
604{
605 return PageCompound(page);
606}
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624static inline int PageTransCompoundMap(struct page *page)
625{
626 return PageTransCompound(page) && atomic_read(&page->_mapcount) < 0;
627}
628
629
630
631
632
633
634static inline int PageTransTail(struct page *page)
635{
636 return PageTail(page);
637}
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652static inline int PageDoubleMap(struct page *page)
653{
654 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
655}
656
657static inline void SetPageDoubleMap(struct page *page)
658{
659 VM_BUG_ON_PAGE(!PageHead(page), page);
660 set_bit(PG_double_map, &page[1].flags);
661}
662
663static inline void ClearPageDoubleMap(struct page *page)
664{
665 VM_BUG_ON_PAGE(!PageHead(page), page);
666 clear_bit(PG_double_map, &page[1].flags);
667}
668static inline int TestSetPageDoubleMap(struct page *page)
669{
670 VM_BUG_ON_PAGE(!PageHead(page), page);
671 return test_and_set_bit(PG_double_map, &page[1].flags);
672}
673
674static inline int TestClearPageDoubleMap(struct page *page)
675{
676 VM_BUG_ON_PAGE(!PageHead(page), page);
677 return test_and_clear_bit(PG_double_map, &page[1].flags);
678}
679
680#else
681TESTPAGEFLAG_FALSE(TransHuge)
682TESTPAGEFLAG_FALSE(TransCompound)
683TESTPAGEFLAG_FALSE(TransCompoundMap)
684TESTPAGEFLAG_FALSE(TransTail)
685PAGEFLAG_FALSE(DoubleMap)
686 TESTSETFLAG_FALSE(DoubleMap)
687 TESTCLEARFLAG_FALSE(DoubleMap)
688#endif
689
690
691
692
693
694
695
696
697
698
699#define PAGE_TYPE_BASE 0xf0000000
700
701#define PAGE_MAPCOUNT_RESERVE -128
702#define PG_buddy 0x00000080
703#define PG_offline 0x00000100
704#define PG_kmemcg 0x00000200
705#define PG_table 0x00000400
706
707#define PageType(page, flag) \
708 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
709
710static inline int page_has_type(struct page *page)
711{
712 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;
713}
714
715#define PAGE_TYPE_OPS(uname, lname) \
716static __always_inline int Page##uname(struct page *page) \
717{ \
718 return PageType(page, PG_##lname); \
719} \
720static __always_inline void __SetPage##uname(struct page *page) \
721{ \
722 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
723 page->page_type &= ~PG_##lname; \
724} \
725static __always_inline void __ClearPage##uname(struct page *page) \
726{ \
727 VM_BUG_ON_PAGE(!Page##uname(page), page); \
728 page->page_type |= PG_##lname; \
729}
730
731
732
733
734
735PAGE_TYPE_OPS(Buddy, buddy)
736
737
738
739
740
741
742
743
744PAGE_TYPE_OPS(Offline, offline)
745
746
747
748
749
750PAGE_TYPE_OPS(Kmemcg, kmemcg)
751
752
753
754
755PAGE_TYPE_OPS(Table, table)
756
757extern bool is_free_buddy_page(struct page *page);
758
759__PAGEFLAG(Isolated, isolated, PF_ANY);
760
761
762
763
764
765static inline int PageSlabPfmemalloc(struct page *page)
766{
767 VM_BUG_ON_PAGE(!PageSlab(page), page);
768 return PageActive(page);
769}
770
771static inline void SetPageSlabPfmemalloc(struct page *page)
772{
773 VM_BUG_ON_PAGE(!PageSlab(page), page);
774 SetPageActive(page);
775}
776
777static inline void __ClearPageSlabPfmemalloc(struct page *page)
778{
779 VM_BUG_ON_PAGE(!PageSlab(page), page);
780 __ClearPageActive(page);
781}
782
783static inline void ClearPageSlabPfmemalloc(struct page *page)
784{
785 VM_BUG_ON_PAGE(!PageSlab(page), page);
786 ClearPageActive(page);
787}
788
789#ifdef CONFIG_MMU
790#define __PG_MLOCKED (1UL << PG_mlocked)
791#else
792#define __PG_MLOCKED 0
793#endif
794
795
796
797
798
799#define PAGE_FLAGS_CHECK_AT_FREE \
800 (1UL << PG_lru | 1UL << PG_locked | \
801 1UL << PG_private | 1UL << PG_private_2 | \
802 1UL << PG_writeback | 1UL << PG_reserved | \
803 1UL << PG_slab | 1UL << PG_active | \
804 1UL << PG_unevictable | __PG_MLOCKED)
805
806
807
808
809
810
811
812
813
814#define PAGE_FLAGS_CHECK_AT_PREP \
815 (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
816
817#define PAGE_FLAGS_PRIVATE \
818 (1UL << PG_private | 1UL << PG_private_2)
819
820
821
822
823
824
825
826static inline int page_has_private(struct page *page)
827{
828 return !!(page->flags & PAGE_FLAGS_PRIVATE);
829}
830
831#undef PF_ANY
832#undef PF_HEAD
833#undef PF_ONLY_HEAD
834#undef PF_NO_TAIL
835#undef PF_NO_COMPOUND
836#endif
837
838#endif
839