1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103enum pageflags {
104 PG_locked,
105 PG_referenced,
106 PG_uptodate,
107 PG_dirty,
108 PG_lru,
109 PG_active,
110 PG_workingset,
111 PG_waiters,
112 PG_error,
113 PG_slab,
114 PG_owner_priv_1,
115 PG_arch_1,
116 PG_reserved,
117 PG_private,
118 PG_private_2,
119 PG_writeback,
120 PG_head,
121 PG_mappedtodisk,
122 PG_reclaim,
123 PG_swapbacked,
124 PG_unevictable,
125#ifdef CONFIG_MMU
126 PG_mlocked,
127#endif
128#ifdef CONFIG_ARCH_USES_PG_UNCACHED
129 PG_uncached,
130#endif
131#ifdef CONFIG_MEMORY_FAILURE
132 PG_hwpoison,
133#endif
134#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
135 PG_young,
136 PG_idle,
137#endif
138#ifdef CONFIG_64BIT
139 PG_arch_2,
140#endif
141#ifdef CONFIG_KASAN_HW_TAGS
142 PG_skip_kasan_poison,
143#endif
144 __NR_PAGEFLAGS,
145
146
147 PG_checked = PG_owner_priv_1,
148
149
150 PG_swapcache = PG_owner_priv_1,
151
152
153
154
155
156 PG_fscache = PG_private_2,
157
158
159
160 PG_pinned = PG_owner_priv_1,
161
162 PG_savepinned = PG_dirty,
163
164 PG_foreign = PG_owner_priv_1,
165
166 PG_xen_remapped = PG_owner_priv_1,
167
168
169 PG_slob_free = PG_private,
170
171
172 PG_double_map = PG_workingset,
173
174#ifdef CONFIG_MEMORY_FAILURE
175
176
177
178
179
180 PG_has_hwpoisoned = PG_mappedtodisk,
181#endif
182
183
184 PG_isolated = PG_reclaim,
185
186
187 PG_reported = PG_uptodate,
188};
189
190#define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1)
191
192#ifndef __GENERATING_BOUNDS_H
193
194static inline unsigned long _compound_head(const struct page *page)
195{
196 unsigned long head = READ_ONCE(page->compound_head);
197
198 if (unlikely(head & 1))
199 return head - 1;
200 return (unsigned long)page;
201}
202
203#define compound_head(page) ((typeof(page))_compound_head(page))
204
205static __always_inline int PageTail(struct page *page)
206{
207 return READ_ONCE(page->compound_head) & 1;
208}
209
210static __always_inline int PageCompound(struct page *page)
211{
212 return test_bit(PG_head, &page->flags) || PageTail(page);
213}
214
215#define PAGE_POISON_PATTERN -1l
216static inline int PagePoisoned(const struct page *page)
217{
218 return page->flags == PAGE_POISON_PATTERN;
219}
220
221#ifdef CONFIG_DEBUG_VM
222void page_init_poison(struct page *page, size_t size);
223#else
224static inline void page_init_poison(struct page *page, size_t size)
225{
226}
227#endif
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255#define PF_POISONED_CHECK(page) ({ \
256 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
257 page; })
258#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
259#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
260#define PF_ONLY_HEAD(page, enforce) ({ \
261 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
262 PF_POISONED_CHECK(page); })
263#define PF_NO_TAIL(page, enforce) ({ \
264 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
265 PF_POISONED_CHECK(compound_head(page)); })
266#define PF_NO_COMPOUND(page, enforce) ({ \
267 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
268 PF_POISONED_CHECK(page); })
269#define PF_SECOND(page, enforce) ({ \
270 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
271 PF_POISONED_CHECK(&page[1]); })
272
273
274
275
276#define TESTPAGEFLAG(uname, lname, policy) \
277static __always_inline int Page##uname(struct page *page) \
278 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
279
280#define SETPAGEFLAG(uname, lname, policy) \
281static __always_inline void SetPage##uname(struct page *page) \
282 { set_bit(PG_##lname, &policy(page, 1)->flags); }
283
284#define CLEARPAGEFLAG(uname, lname, policy) \
285static __always_inline void ClearPage##uname(struct page *page) \
286 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
287
288#define __SETPAGEFLAG(uname, lname, policy) \
289static __always_inline void __SetPage##uname(struct page *page) \
290 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
291
292#define __CLEARPAGEFLAG(uname, lname, policy) \
293static __always_inline void __ClearPage##uname(struct page *page) \
294 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
295
296#define TESTSETFLAG(uname, lname, policy) \
297static __always_inline int TestSetPage##uname(struct page *page) \
298 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
299
300#define TESTCLEARFLAG(uname, lname, policy) \
301static __always_inline int TestClearPage##uname(struct page *page) \
302 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
303
304#define PAGEFLAG(uname, lname, policy) \
305 TESTPAGEFLAG(uname, lname, policy) \
306 SETPAGEFLAG(uname, lname, policy) \
307 CLEARPAGEFLAG(uname, lname, policy)
308
309#define __PAGEFLAG(uname, lname, policy) \
310 TESTPAGEFLAG(uname, lname, policy) \
311 __SETPAGEFLAG(uname, lname, policy) \
312 __CLEARPAGEFLAG(uname, lname, policy)
313
314#define TESTSCFLAG(uname, lname, policy) \
315 TESTSETFLAG(uname, lname, policy) \
316 TESTCLEARFLAG(uname, lname, policy)
317
318#define TESTPAGEFLAG_FALSE(uname) \
319static inline int Page##uname(const struct page *page) { return 0; }
320
321#define SETPAGEFLAG_NOOP(uname) \
322static inline void SetPage##uname(struct page *page) { }
323
324#define CLEARPAGEFLAG_NOOP(uname) \
325static inline void ClearPage##uname(struct page *page) { }
326
327#define __CLEARPAGEFLAG_NOOP(uname) \
328static inline void __ClearPage##uname(struct page *page) { }
329
330#define TESTSETFLAG_FALSE(uname) \
331static inline int TestSetPage##uname(struct page *page) { return 0; }
332
333#define TESTCLEARFLAG_FALSE(uname) \
334static inline int TestClearPage##uname(struct page *page) { return 0; }
335
336#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
337 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
338
339#define TESTSCFLAG_FALSE(uname) \
340 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
341
342__PAGEFLAG(Locked, locked, PF_NO_TAIL)
343PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
344PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL)
345PAGEFLAG(Referenced, referenced, PF_HEAD)
346 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
347 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
348PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
349 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
350PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
351 TESTCLEARFLAG(LRU, lru, PF_HEAD)
352PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
353 TESTCLEARFLAG(Active, active, PF_HEAD)
354PAGEFLAG(Workingset, workingset, PF_HEAD)
355 TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
356__PAGEFLAG(Slab, slab, PF_NO_TAIL)
357__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
358PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
359
360
361PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
362 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
363PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
364PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
365PAGEFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
366 TESTCLEARFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
367
368PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
369 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
370 __SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
371PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
372 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
373 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
374
375
376
377
378
379
380PAGEFLAG(Private, private, PF_ANY)
381PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
382PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
383 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
384
385
386
387
388
389TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
390 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
391PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
392
393
394PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
395 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
396PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
397 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
398
399#ifdef CONFIG_HIGHMEM
400
401
402
403
404#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
405#else
406PAGEFLAG_FALSE(HighMem)
407#endif
408
409#ifdef CONFIG_SWAP
410static __always_inline int PageSwapCache(struct page *page)
411{
412#ifdef CONFIG_THP_SWAP
413 page = compound_head(page);
414#endif
415 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
416
417}
418SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
419CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
420#else
421PAGEFLAG_FALSE(SwapCache)
422#endif
423
424PAGEFLAG(Unevictable, unevictable, PF_HEAD)
425 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
426 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
427
428#ifdef CONFIG_MMU
429PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
430 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
431 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
432#else
433PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
434 TESTSCFLAG_FALSE(Mlocked)
435#endif
436
437#ifdef CONFIG_ARCH_USES_PG_UNCACHED
438PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
439#else
440PAGEFLAG_FALSE(Uncached)
441#endif
442
443#ifdef CONFIG_MEMORY_FAILURE
444PAGEFLAG(HWPoison, hwpoison, PF_ANY)
445TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
446#define __PG_HWPOISON (1UL << PG_hwpoison)
447extern bool take_page_off_buddy(struct page *page);
448#else
449PAGEFLAG_FALSE(HWPoison)
450#define __PG_HWPOISON 0
451#endif
452
453#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
454TESTPAGEFLAG(Young, young, PF_ANY)
455SETPAGEFLAG(Young, young, PF_ANY)
456TESTCLEARFLAG(Young, young, PF_ANY)
457PAGEFLAG(Idle, idle, PF_ANY)
458#endif
459
460#ifdef CONFIG_KASAN_HW_TAGS
461PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD)
462#else
463PAGEFLAG_FALSE(SkipKASanPoison)
464#endif
465
466
467
468
469
470
471
472__PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491#define PAGE_MAPPING_ANON 0x1
492#define PAGE_MAPPING_MOVABLE 0x2
493#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
494#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
495
496static __always_inline int PageMappingFlags(struct page *page)
497{
498 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
499}
500
501static __always_inline int PageAnon(struct page *page)
502{
503 page = compound_head(page);
504 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
505}
506
507static __always_inline int __PageMovable(struct page *page)
508{
509 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
510 PAGE_MAPPING_MOVABLE;
511}
512
513#ifdef CONFIG_KSM
514
515
516
517
518
519
520static __always_inline int PageKsm(struct page *page)
521{
522 page = compound_head(page);
523 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
524 PAGE_MAPPING_KSM;
525}
526#else
527TESTPAGEFLAG_FALSE(Ksm)
528#endif
529
530u64 stable_page_flags(struct page *page);
531
532static inline int PageUptodate(struct page *page)
533{
534 int ret;
535 page = compound_head(page);
536 ret = test_bit(PG_uptodate, &(page)->flags);
537
538
539
540
541
542
543
544
545 if (ret)
546 smp_rmb();
547
548 return ret;
549}
550
551static __always_inline void __SetPageUptodate(struct page *page)
552{
553 VM_BUG_ON_PAGE(PageTail(page), page);
554 smp_wmb();
555 __set_bit(PG_uptodate, &page->flags);
556}
557
558static __always_inline void SetPageUptodate(struct page *page)
559{
560 VM_BUG_ON_PAGE(PageTail(page), page);
561
562
563
564
565
566 smp_wmb();
567 set_bit(PG_uptodate, &page->flags);
568}
569
570CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
571
572int test_clear_page_writeback(struct page *page);
573int __test_set_page_writeback(struct page *page, bool keep_write);
574
575#define test_set_page_writeback(page) \
576 __test_set_page_writeback(page, false)
577#define test_set_page_writeback_keepwrite(page) \
578 __test_set_page_writeback(page, true)
579
580static inline void set_page_writeback(struct page *page)
581{
582 test_set_page_writeback(page);
583}
584
585static inline void set_page_writeback_keepwrite(struct page *page)
586{
587 test_set_page_writeback_keepwrite(page);
588}
589
590__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
591
592static __always_inline void set_compound_head(struct page *page, struct page *head)
593{
594 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
595}
596
597static __always_inline void clear_compound_head(struct page *page)
598{
599 WRITE_ONCE(page->compound_head, 0);
600}
601
602#ifdef CONFIG_TRANSPARENT_HUGEPAGE
603static inline void ClearPageCompound(struct page *page)
604{
605 BUG_ON(!PageHead(page));
606 ClearPageHead(page);
607}
608#endif
609
610#define PG_head_mask ((1UL << PG_head))
611
612#ifdef CONFIG_HUGETLB_PAGE
613int PageHuge(struct page *page);
614int PageHeadHuge(struct page *page);
615#else
616TESTPAGEFLAG_FALSE(Huge)
617TESTPAGEFLAG_FALSE(HeadHuge)
618#endif
619
620
621#ifdef CONFIG_TRANSPARENT_HUGEPAGE
622
623
624
625
626
627
628
629
630static inline int PageTransHuge(struct page *page)
631{
632 VM_BUG_ON_PAGE(PageTail(page), page);
633 return PageHead(page);
634}
635
636
637
638
639
640
641static inline int PageTransCompound(struct page *page)
642{
643 return PageCompound(page);
644}
645
646
647
648
649
650
651static inline int PageTransTail(struct page *page)
652{
653 return PageTail(page);
654}
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669PAGEFLAG(DoubleMap, double_map, PF_SECOND)
670 TESTSCFLAG(DoubleMap, double_map, PF_SECOND)
671#else
672TESTPAGEFLAG_FALSE(TransHuge)
673TESTPAGEFLAG_FALSE(TransCompound)
674TESTPAGEFLAG_FALSE(TransCompoundMap)
675TESTPAGEFLAG_FALSE(TransTail)
676PAGEFLAG_FALSE(DoubleMap)
677 TESTSCFLAG_FALSE(DoubleMap)
678#endif
679
680#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
681
682
683
684
685
686
687PAGEFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
688 TESTSCFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
689#else
690PAGEFLAG_FALSE(HasHWPoisoned)
691 TESTSCFLAG_FALSE(HasHWPoisoned)
692#endif
693
694
695
696
697
698
699static inline bool is_page_hwpoison(struct page *page)
700{
701 if (PageHWPoison(page))
702 return true;
703 return PageHuge(page) && PageHWPoison(compound_head(page));
704}
705
706
707
708
709
710
711
712
713
714
715#define PAGE_TYPE_BASE 0xf0000000
716
717#define PAGE_MAPCOUNT_RESERVE -128
718#define PG_buddy 0x00000080
719#define PG_offline 0x00000100
720#define PG_table 0x00000200
721#define PG_guard 0x00000400
722
723#define PageType(page, flag) \
724 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
725
726static inline int page_has_type(struct page *page)
727{
728 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;
729}
730
731#define PAGE_TYPE_OPS(uname, lname) \
732static __always_inline int Page##uname(struct page *page) \
733{ \
734 return PageType(page, PG_##lname); \
735} \
736static __always_inline void __SetPage##uname(struct page *page) \
737{ \
738 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
739 page->page_type &= ~PG_##lname; \
740} \
741static __always_inline void __ClearPage##uname(struct page *page) \
742{ \
743 VM_BUG_ON_PAGE(!Page##uname(page), page); \
744 page->page_type |= PG_##lname; \
745}
746
747
748
749
750
751PAGE_TYPE_OPS(Buddy, buddy)
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775PAGE_TYPE_OPS(Offline, offline)
776
777extern void page_offline_freeze(void);
778extern void page_offline_thaw(void);
779extern void page_offline_begin(void);
780extern void page_offline_end(void);
781
782
783
784
785PAGE_TYPE_OPS(Table, table)
786
787
788
789
790PAGE_TYPE_OPS(Guard, guard)
791
792extern bool is_free_buddy_page(struct page *page);
793
794__PAGEFLAG(Isolated, isolated, PF_ANY);
795
796
797
798
799
800static inline int PageSlabPfmemalloc(struct page *page)
801{
802 VM_BUG_ON_PAGE(!PageSlab(page), page);
803 return PageActive(page);
804}
805
806
807
808
809
810static inline int __PageSlabPfmemalloc(struct page *page)
811{
812 return PageActive(page);
813}
814
815static inline void SetPageSlabPfmemalloc(struct page *page)
816{
817 VM_BUG_ON_PAGE(!PageSlab(page), page);
818 SetPageActive(page);
819}
820
821static inline void __ClearPageSlabPfmemalloc(struct page *page)
822{
823 VM_BUG_ON_PAGE(!PageSlab(page), page);
824 __ClearPageActive(page);
825}
826
827static inline void ClearPageSlabPfmemalloc(struct page *page)
828{
829 VM_BUG_ON_PAGE(!PageSlab(page), page);
830 ClearPageActive(page);
831}
832
833#ifdef CONFIG_MMU
834#define __PG_MLOCKED (1UL << PG_mlocked)
835#else
836#define __PG_MLOCKED 0
837#endif
838
839
840
841
842
843#define PAGE_FLAGS_CHECK_AT_FREE \
844 (1UL << PG_lru | 1UL << PG_locked | \
845 1UL << PG_private | 1UL << PG_private_2 | \
846 1UL << PG_writeback | 1UL << PG_reserved | \
847 1UL << PG_slab | 1UL << PG_active | \
848 1UL << PG_unevictable | __PG_MLOCKED)
849
850
851
852
853
854
855
856
857
858#define PAGE_FLAGS_CHECK_AT_PREP \
859 (PAGEFLAGS_MASK & ~__PG_HWPOISON)
860
861#define PAGE_FLAGS_PRIVATE \
862 (1UL << PG_private | 1UL << PG_private_2)
863
864
865
866
867
868
869
870static inline int page_has_private(struct page *page)
871{
872 return !!(page->flags & PAGE_FLAGS_PRIVATE);
873}
874
875#undef PF_ANY
876#undef PF_HEAD
877#undef PF_ONLY_HEAD
878#undef PF_NO_TAIL
879#undef PF_NO_COMPOUND
880#undef PF_SECOND
881#endif
882
883#endif
884