1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104enum pageflags {
105 PG_locked,
106 PG_referenced,
107 PG_uptodate,
108 PG_dirty,
109 PG_lru,
110 PG_active,
111 PG_workingset,
112 PG_waiters,
113 PG_error,
114 PG_slab,
115 PG_owner_priv_1,
116 PG_arch_1,
117 PG_reserved,
118 PG_private,
119 PG_private_2,
120 PG_writeback,
121 PG_head,
122 PG_mappedtodisk,
123 PG_reclaim,
124 PG_swapbacked,
125 PG_unevictable,
126#ifdef CONFIG_MMU
127 PG_mlocked,
128#endif
129#ifdef CONFIG_ARCH_USES_PG_UNCACHED
130 PG_uncached,
131#endif
132#ifdef CONFIG_MEMORY_FAILURE
133 PG_hwpoison,
134#endif
135#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
136 PG_young,
137 PG_idle,
138#endif
139 __NR_PAGEFLAGS,
140
141
142 PG_checked = PG_owner_priv_1,
143
144
145 PG_swapcache = PG_owner_priv_1,
146
147
148
149
150
151 PG_fscache = PG_private_2,
152
153
154
155 PG_pinned = PG_owner_priv_1,
156
157 PG_savepinned = PG_dirty,
158
159 PG_foreign = PG_owner_priv_1,
160
161 PG_xen_remapped = PG_owner_priv_1,
162
163
164 PG_slob_free = PG_private,
165
166
167 PG_double_map = PG_private_2,
168
169
170 PG_isolated = PG_reclaim,
171
172
173 PG_reported = PG_uptodate,
174};
175
176#ifndef __GENERATING_BOUNDS_H
177
178struct page;
179
180static inline struct page *compound_head(struct page *page)
181{
182 unsigned long head = READ_ONCE(page->compound_head);
183
184 if (unlikely(head & 1))
185 return (struct page *) (head - 1);
186 return page;
187}
188
189static __always_inline int PageTail(struct page *page)
190{
191 return READ_ONCE(page->compound_head) & 1;
192}
193
194static __always_inline int PageCompound(struct page *page)
195{
196 return test_bit(PG_head, &page->flags) || PageTail(page);
197}
198
199#define PAGE_POISON_PATTERN -1l
200static inline int PagePoisoned(const struct page *page)
201{
202 return page->flags == PAGE_POISON_PATTERN;
203}
204
205#ifdef CONFIG_DEBUG_VM
206void page_init_poison(struct page *page, size_t size);
207#else
208static inline void page_init_poison(struct page *page, size_t size)
209{
210}
211#endif
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236#define PF_POISONED_CHECK(page) ({ \
237 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
238 page; })
239#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
240#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
241#define PF_ONLY_HEAD(page, enforce) ({ \
242 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
243 PF_POISONED_CHECK(page); })
244#define PF_NO_TAIL(page, enforce) ({ \
245 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
246 PF_POISONED_CHECK(compound_head(page)); })
247#define PF_NO_COMPOUND(page, enforce) ({ \
248 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
249 PF_POISONED_CHECK(page); })
250
251
252
253
254#define TESTPAGEFLAG(uname, lname, policy) \
255static __always_inline int Page##uname(struct page *page) \
256 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
257
258#define SETPAGEFLAG(uname, lname, policy) \
259static __always_inline void SetPage##uname(struct page *page) \
260 { set_bit(PG_##lname, &policy(page, 1)->flags); }
261
262#define CLEARPAGEFLAG(uname, lname, policy) \
263static __always_inline void ClearPage##uname(struct page *page) \
264 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
265
266#define __SETPAGEFLAG(uname, lname, policy) \
267static __always_inline void __SetPage##uname(struct page *page) \
268 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
269
270#define __CLEARPAGEFLAG(uname, lname, policy) \
271static __always_inline void __ClearPage##uname(struct page *page) \
272 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
273
274#define TESTSETFLAG(uname, lname, policy) \
275static __always_inline int TestSetPage##uname(struct page *page) \
276 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
277
278#define TESTCLEARFLAG(uname, lname, policy) \
279static __always_inline int TestClearPage##uname(struct page *page) \
280 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
281
282#define PAGEFLAG(uname, lname, policy) \
283 TESTPAGEFLAG(uname, lname, policy) \
284 SETPAGEFLAG(uname, lname, policy) \
285 CLEARPAGEFLAG(uname, lname, policy)
286
287#define __PAGEFLAG(uname, lname, policy) \
288 TESTPAGEFLAG(uname, lname, policy) \
289 __SETPAGEFLAG(uname, lname, policy) \
290 __CLEARPAGEFLAG(uname, lname, policy)
291
292#define TESTSCFLAG(uname, lname, policy) \
293 TESTSETFLAG(uname, lname, policy) \
294 TESTCLEARFLAG(uname, lname, policy)
295
296#define TESTPAGEFLAG_FALSE(uname) \
297static inline int Page##uname(const struct page *page) { return 0; }
298
299#define SETPAGEFLAG_NOOP(uname) \
300static inline void SetPage##uname(struct page *page) { }
301
302#define CLEARPAGEFLAG_NOOP(uname) \
303static inline void ClearPage##uname(struct page *page) { }
304
305#define __CLEARPAGEFLAG_NOOP(uname) \
306static inline void __ClearPage##uname(struct page *page) { }
307
308#define TESTSETFLAG_FALSE(uname) \
309static inline int TestSetPage##uname(struct page *page) { return 0; }
310
311#define TESTCLEARFLAG_FALSE(uname) \
312static inline int TestClearPage##uname(struct page *page) { return 0; }
313
314#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
315 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
316
317#define TESTSCFLAG_FALSE(uname) \
318 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
319
320__PAGEFLAG(Locked, locked, PF_NO_TAIL)
321PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
322PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL)
323PAGEFLAG(Referenced, referenced, PF_HEAD)
324 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
325 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
326PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
327 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
328PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
329PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
330 TESTCLEARFLAG(Active, active, PF_HEAD)
331PAGEFLAG(Workingset, workingset, PF_HEAD)
332 TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
333__PAGEFLAG(Slab, slab, PF_NO_TAIL)
334__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
335PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
336
337
338PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
339 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
340PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
341PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
342PAGEFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
343 TESTCLEARFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
344
345PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
346 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
347 __SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
348PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
349 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
350 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
351
352
353
354
355
356
357PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
358 __CLEARPAGEFLAG(Private, private, PF_ANY)
359PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
360PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
361 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
362
363
364
365
366
367TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
368 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
369PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
370
371
372PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
373 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
374PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
375 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
376
377#ifdef CONFIG_HIGHMEM
378
379
380
381
382#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
383#else
384PAGEFLAG_FALSE(HighMem)
385#endif
386
387#ifdef CONFIG_SWAP
388static __always_inline int PageSwapCache(struct page *page)
389{
390#ifdef CONFIG_THP_SWAP
391 page = compound_head(page);
392#endif
393 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
394
395}
396SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
397CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
398#else
399PAGEFLAG_FALSE(SwapCache)
400#endif
401
402PAGEFLAG(Unevictable, unevictable, PF_HEAD)
403 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
404 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
405
406#ifdef CONFIG_MMU
407PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
408 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
409 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
410#else
411PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
412 TESTSCFLAG_FALSE(Mlocked)
413#endif
414
415#ifdef CONFIG_ARCH_USES_PG_UNCACHED
416PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
417#else
418PAGEFLAG_FALSE(Uncached)
419#endif
420
421#ifdef CONFIG_MEMORY_FAILURE
422PAGEFLAG(HWPoison, hwpoison, PF_ANY)
423TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
424#define __PG_HWPOISON (1UL << PG_hwpoison)
425extern bool set_hwpoison_free_buddy_page(struct page *page);
426#else
427PAGEFLAG_FALSE(HWPoison)
428static inline bool set_hwpoison_free_buddy_page(struct page *page)
429{
430 return 0;
431}
432#define __PG_HWPOISON 0
433#endif
434
435#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
436TESTPAGEFLAG(Young, young, PF_ANY)
437SETPAGEFLAG(Young, young, PF_ANY)
438TESTCLEARFLAG(Young, young, PF_ANY)
439PAGEFLAG(Idle, idle, PF_ANY)
440#endif
441
442
443
444
445
446
447
448__PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467#define PAGE_MAPPING_ANON 0x1
468#define PAGE_MAPPING_MOVABLE 0x2
469#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
470#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
471
472static __always_inline int PageMappingFlags(struct page *page)
473{
474 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
475}
476
477static __always_inline int PageAnon(struct page *page)
478{
479 page = compound_head(page);
480 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
481}
482
483static __always_inline int __PageMovable(struct page *page)
484{
485 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
486 PAGE_MAPPING_MOVABLE;
487}
488
489#ifdef CONFIG_KSM
490
491
492
493
494
495
496static __always_inline int PageKsm(struct page *page)
497{
498 page = compound_head(page);
499 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
500 PAGE_MAPPING_KSM;
501}
502#else
503TESTPAGEFLAG_FALSE(Ksm)
504#endif
505
506u64 stable_page_flags(struct page *page);
507
508static inline int PageUptodate(struct page *page)
509{
510 int ret;
511 page = compound_head(page);
512 ret = test_bit(PG_uptodate, &(page)->flags);
513
514
515
516
517
518
519
520
521 if (ret)
522 smp_rmb();
523
524 return ret;
525}
526
527static __always_inline void __SetPageUptodate(struct page *page)
528{
529 VM_BUG_ON_PAGE(PageTail(page), page);
530 smp_wmb();
531 __set_bit(PG_uptodate, &page->flags);
532}
533
534static __always_inline void SetPageUptodate(struct page *page)
535{
536 VM_BUG_ON_PAGE(PageTail(page), page);
537
538
539
540
541
542 smp_wmb();
543 set_bit(PG_uptodate, &page->flags);
544}
545
546CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
547
548int test_clear_page_writeback(struct page *page);
549int __test_set_page_writeback(struct page *page, bool keep_write);
550
551#define test_set_page_writeback(page) \
552 __test_set_page_writeback(page, false)
553#define test_set_page_writeback_keepwrite(page) \
554 __test_set_page_writeback(page, true)
555
556static inline void set_page_writeback(struct page *page)
557{
558 test_set_page_writeback(page);
559}
560
561static inline void set_page_writeback_keepwrite(struct page *page)
562{
563 test_set_page_writeback_keepwrite(page);
564}
565
566__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
567
568static __always_inline void set_compound_head(struct page *page, struct page *head)
569{
570 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
571}
572
573static __always_inline void clear_compound_head(struct page *page)
574{
575 WRITE_ONCE(page->compound_head, 0);
576}
577
578#ifdef CONFIG_TRANSPARENT_HUGEPAGE
579static inline void ClearPageCompound(struct page *page)
580{
581 BUG_ON(!PageHead(page));
582 ClearPageHead(page);
583}
584#endif
585
586#define PG_head_mask ((1UL << PG_head))
587
588#ifdef CONFIG_HUGETLB_PAGE
589int PageHuge(struct page *page);
590int PageHeadHuge(struct page *page);
591bool page_huge_active(struct page *page);
592#else
593TESTPAGEFLAG_FALSE(Huge)
594TESTPAGEFLAG_FALSE(HeadHuge)
595
596static inline bool page_huge_active(struct page *page)
597{
598 return 0;
599}
600#endif
601
602
603#ifdef CONFIG_TRANSPARENT_HUGEPAGE
604
605
606
607
608
609
610
611
612static inline int PageTransHuge(struct page *page)
613{
614 VM_BUG_ON_PAGE(PageTail(page), page);
615 return PageHead(page);
616}
617
618
619
620
621
622
623static inline int PageTransCompound(struct page *page)
624{
625 return PageCompound(page);
626}
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649static inline int PageTransCompoundMap(struct page *page)
650{
651 struct page *head;
652
653 if (!PageTransCompound(page))
654 return 0;
655
656 if (PageAnon(page))
657 return atomic_read(&page->_mapcount) < 0;
658
659 head = compound_head(page);
660
661 return atomic_read(&page->_mapcount) ==
662 atomic_read(compound_mapcount_ptr(head));
663}
664
665
666
667
668
669
670static inline int PageTransTail(struct page *page)
671{
672 return PageTail(page);
673}
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688static inline int PageDoubleMap(struct page *page)
689{
690 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
691}
692
693static inline void SetPageDoubleMap(struct page *page)
694{
695 VM_BUG_ON_PAGE(!PageHead(page), page);
696 set_bit(PG_double_map, &page[1].flags);
697}
698
699static inline void ClearPageDoubleMap(struct page *page)
700{
701 VM_BUG_ON_PAGE(!PageHead(page), page);
702 clear_bit(PG_double_map, &page[1].flags);
703}
704static inline int TestSetPageDoubleMap(struct page *page)
705{
706 VM_BUG_ON_PAGE(!PageHead(page), page);
707 return test_and_set_bit(PG_double_map, &page[1].flags);
708}
709
710static inline int TestClearPageDoubleMap(struct page *page)
711{
712 VM_BUG_ON_PAGE(!PageHead(page), page);
713 return test_and_clear_bit(PG_double_map, &page[1].flags);
714}
715
716#else
717TESTPAGEFLAG_FALSE(TransHuge)
718TESTPAGEFLAG_FALSE(TransCompound)
719TESTPAGEFLAG_FALSE(TransCompoundMap)
720TESTPAGEFLAG_FALSE(TransTail)
721PAGEFLAG_FALSE(DoubleMap)
722 TESTSETFLAG_FALSE(DoubleMap)
723 TESTCLEARFLAG_FALSE(DoubleMap)
724#endif
725
726
727
728
729
730
731
732
733
734
735#define PAGE_TYPE_BASE 0xf0000000
736
737#define PAGE_MAPCOUNT_RESERVE -128
738#define PG_buddy 0x00000080
739#define PG_offline 0x00000100
740#define PG_kmemcg 0x00000200
741#define PG_table 0x00000400
742#define PG_guard 0x00000800
743
744#define PageType(page, flag) \
745 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
746
747static inline int page_has_type(struct page *page)
748{
749 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;
750}
751
752#define PAGE_TYPE_OPS(uname, lname) \
753static __always_inline int Page##uname(struct page *page) \
754{ \
755 return PageType(page, PG_##lname); \
756} \
757static __always_inline void __SetPage##uname(struct page *page) \
758{ \
759 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
760 page->page_type &= ~PG_##lname; \
761} \
762static __always_inline void __ClearPage##uname(struct page *page) \
763{ \
764 VM_BUG_ON_PAGE(!Page##uname(page), page); \
765 page->page_type |= PG_##lname; \
766}
767
768
769
770
771
772PAGE_TYPE_OPS(Buddy, buddy)
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791PAGE_TYPE_OPS(Offline, offline)
792
793
794
795
796
797PAGE_TYPE_OPS(Kmemcg, kmemcg)
798
799
800
801
802PAGE_TYPE_OPS(Table, table)
803
804
805
806
807PAGE_TYPE_OPS(Guard, guard)
808
809extern bool is_free_buddy_page(struct page *page);
810
811__PAGEFLAG(Isolated, isolated, PF_ANY);
812
813
814
815
816
817static inline int PageSlabPfmemalloc(struct page *page)
818{
819 VM_BUG_ON_PAGE(!PageSlab(page), page);
820 return PageActive(page);
821}
822
823static inline void SetPageSlabPfmemalloc(struct page *page)
824{
825 VM_BUG_ON_PAGE(!PageSlab(page), page);
826 SetPageActive(page);
827}
828
829static inline void __ClearPageSlabPfmemalloc(struct page *page)
830{
831 VM_BUG_ON_PAGE(!PageSlab(page), page);
832 __ClearPageActive(page);
833}
834
835static inline void ClearPageSlabPfmemalloc(struct page *page)
836{
837 VM_BUG_ON_PAGE(!PageSlab(page), page);
838 ClearPageActive(page);
839}
840
841#ifdef CONFIG_MMU
842#define __PG_MLOCKED (1UL << PG_mlocked)
843#else
844#define __PG_MLOCKED 0
845#endif
846
847
848
849
850
851#define PAGE_FLAGS_CHECK_AT_FREE \
852 (1UL << PG_lru | 1UL << PG_locked | \
853 1UL << PG_private | 1UL << PG_private_2 | \
854 1UL << PG_writeback | 1UL << PG_reserved | \
855 1UL << PG_slab | 1UL << PG_active | \
856 1UL << PG_unevictable | __PG_MLOCKED)
857
858
859
860
861
862
863
864
865
866#define PAGE_FLAGS_CHECK_AT_PREP \
867 (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
868
869#define PAGE_FLAGS_PRIVATE \
870 (1UL << PG_private | 1UL << PG_private_2)
871
872
873
874
875
876
877
878static inline int page_has_private(struct page *page)
879{
880 return !!(page->flags & PAGE_FLAGS_PRIVATE);
881}
882
883#undef PF_ANY
884#undef PF_HEAD
885#undef PF_ONLY_HEAD
886#undef PF_NO_TAIL
887#undef PF_NO_COMPOUND
888#endif
889
890#endif
891