1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75enum pageflags {
76 PG_locked,
77 PG_error,
78 PG_referenced,
79 PG_uptodate,
80 PG_dirty,
81 PG_lru,
82 PG_active,
83 PG_waiters,
84 PG_slab,
85 PG_owner_priv_1,
86 PG_arch_1,
87 PG_reserved,
88 PG_private,
89 PG_private_2,
90 PG_writeback,
91 PG_head,
92 PG_mappedtodisk,
93 PG_reclaim,
94 PG_swapbacked,
95 PG_unevictable,
96#ifdef CONFIG_MMU
97 PG_mlocked,
98#endif
99#ifdef CONFIG_ARCH_USES_PG_UNCACHED
100 PG_uncached,
101#endif
102#ifdef CONFIG_MEMORY_FAILURE
103 PG_hwpoison,
104#endif
105#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
106 PG_young,
107 PG_idle,
108#endif
109#ifndef __GENKSYMS__
110
111
112
113
114 PG_workingset,
115#endif
116 __NR_PAGEFLAGS,
117
118
119 PG_checked = PG_owner_priv_1,
120
121
122 PG_swapcache = PG_owner_priv_1,
123
124
125
126
127
128 PG_fscache = PG_private_2,
129
130
131
132 PG_pinned = PG_owner_priv_1,
133
134 PG_savepinned = PG_dirty,
135
136 PG_foreign = PG_owner_priv_1,
137
138 PG_xen_remapped = PG_owner_priv_1,
139
140
141 PG_slob_free = PG_private,
142
143
144 PG_double_map = PG_private_2,
145
146
147 PG_isolated = PG_reclaim,
148
149
150 PG_reported = PG_uptodate,
151};
152
153#ifndef __GENERATING_BOUNDS_H
154
155struct page;
156
157static inline struct page *compound_head(struct page *page)
158{
159 unsigned long head = READ_ONCE(page->compound_head);
160
161 if (unlikely(head & 1))
162 return (struct page *) (head - 1);
163 return page;
164}
165
166static __always_inline int PageTail(struct page *page)
167{
168 return READ_ONCE(page->compound_head) & 1;
169}
170
171static __always_inline int PageCompound(struct page *page)
172{
173 return test_bit(PG_head, &page->flags) || PageTail(page);
174}
175
176#define PAGE_POISON_PATTERN -1l
177static inline int PagePoisoned(const struct page *page)
178{
179 return page->flags == PAGE_POISON_PATTERN;
180}
181
182#ifdef CONFIG_DEBUG_VM
183void page_init_poison(struct page *page, size_t size);
184#else
185static inline void page_init_poison(struct page *page, size_t size)
186{
187}
188#endif
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213#define PF_POISONED_CHECK(page) ({ \
214 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
215 page; })
216#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
217#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
218#define PF_ONLY_HEAD(page, enforce) ({ \
219 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
220 PF_POISONED_CHECK(page); })
221#define PF_NO_TAIL(page, enforce) ({ \
222 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
223 PF_POISONED_CHECK(compound_head(page)); })
224#define PF_NO_COMPOUND(page, enforce) ({ \
225 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
226 PF_POISONED_CHECK(page); })
227
228
229
230
231#define TESTPAGEFLAG(uname, lname, policy) \
232static __always_inline int Page##uname(struct page *page) \
233 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
234
235#define SETPAGEFLAG(uname, lname, policy) \
236static __always_inline void SetPage##uname(struct page *page) \
237 { set_bit(PG_##lname, &policy(page, 1)->flags); }
238
239#define CLEARPAGEFLAG(uname, lname, policy) \
240static __always_inline void ClearPage##uname(struct page *page) \
241 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
242
243#define __SETPAGEFLAG(uname, lname, policy) \
244static __always_inline void __SetPage##uname(struct page *page) \
245 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
246
247#define __CLEARPAGEFLAG(uname, lname, policy) \
248static __always_inline void __ClearPage##uname(struct page *page) \
249 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
250
251#define TESTSETFLAG(uname, lname, policy) \
252static __always_inline int TestSetPage##uname(struct page *page) \
253 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
254
255#define TESTCLEARFLAG(uname, lname, policy) \
256static __always_inline int TestClearPage##uname(struct page *page) \
257 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
258
259#define PAGEFLAG(uname, lname, policy) \
260 TESTPAGEFLAG(uname, lname, policy) \
261 SETPAGEFLAG(uname, lname, policy) \
262 CLEARPAGEFLAG(uname, lname, policy)
263
264#define __PAGEFLAG(uname, lname, policy) \
265 TESTPAGEFLAG(uname, lname, policy) \
266 __SETPAGEFLAG(uname, lname, policy) \
267 __CLEARPAGEFLAG(uname, lname, policy)
268
269#define TESTSCFLAG(uname, lname, policy) \
270 TESTSETFLAG(uname, lname, policy) \
271 TESTCLEARFLAG(uname, lname, policy)
272
273#define TESTPAGEFLAG_FALSE(uname) \
274static inline int Page##uname(const struct page *page) { return 0; }
275
276#define SETPAGEFLAG_NOOP(uname) \
277static inline void SetPage##uname(struct page *page) { }
278
279#define CLEARPAGEFLAG_NOOP(uname) \
280static inline void ClearPage##uname(struct page *page) { }
281
282#define __CLEARPAGEFLAG_NOOP(uname) \
283static inline void __ClearPage##uname(struct page *page) { }
284
285#define TESTSETFLAG_FALSE(uname) \
286static inline int TestSetPage##uname(struct page *page) { return 0; }
287
288#define TESTCLEARFLAG_FALSE(uname) \
289static inline int TestClearPage##uname(struct page *page) { return 0; }
290
291#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
292 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
293
294#define TESTSCFLAG_FALSE(uname) \
295 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
296
297__PAGEFLAG(Locked, locked, PF_NO_TAIL)
298PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
299PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
300PAGEFLAG(Referenced, referenced, PF_HEAD)
301 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
302 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
303PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
304 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
305PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
306 TESTCLEARFLAG(LRU, lru, PF_HEAD)
307PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
308 TESTCLEARFLAG(Active, active, PF_HEAD)
309PAGEFLAG(Workingset, workingset, PF_HEAD)
310 TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
311__PAGEFLAG(Slab, slab, PF_NO_TAIL)
312__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
313PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
314
315
316PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
317 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
318PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
319PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
320PAGEFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
321 TESTCLEARFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
322
323PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
324 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
325 __SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
326PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
327 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
328 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
329
330
331
332
333
334
335PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
336 __CLEARPAGEFLAG(Private, private, PF_ANY)
337PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
338PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
339 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
340
341
342
343
344
345TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
346 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
347PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
348
349
350PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
351 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
352PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
353 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
354
355#ifdef CONFIG_HIGHMEM
356
357
358
359
360#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
361#else
362PAGEFLAG_FALSE(HighMem)
363#endif
364
365#ifdef CONFIG_SWAP
366static __always_inline int PageSwapCache(struct page *page)
367{
368#ifdef CONFIG_THP_SWAP
369 page = compound_head(page);
370#endif
371 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
372
373}
374SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
375CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
376#else
377PAGEFLAG_FALSE(SwapCache)
378#endif
379
380PAGEFLAG(Unevictable, unevictable, PF_HEAD)
381 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
382 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
383
384#ifdef CONFIG_MMU
385PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
386 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
387 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
388#else
389PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
390 TESTSCFLAG_FALSE(Mlocked)
391#endif
392
393#ifdef CONFIG_ARCH_USES_PG_UNCACHED
394PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
395#else
396PAGEFLAG_FALSE(Uncached)
397#endif
398
399#ifdef CONFIG_MEMORY_FAILURE
400PAGEFLAG(HWPoison, hwpoison, PF_ANY)
401TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
402#define __PG_HWPOISON (1UL << PG_hwpoison)
403extern bool take_page_off_buddy(struct page *page);
404#else
405PAGEFLAG_FALSE(HWPoison)
406#define __PG_HWPOISON 0
407#endif
408
409#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
410TESTPAGEFLAG(Young, young, PF_ANY)
411SETPAGEFLAG(Young, young, PF_ANY)
412TESTCLEARFLAG(Young, young, PF_ANY)
413PAGEFLAG(Idle, idle, PF_ANY)
414#endif
415
416
417
418
419
420
421
422__PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441#define PAGE_MAPPING_ANON 0x1
442#define PAGE_MAPPING_MOVABLE 0x2
443#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
444#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
445
446static __always_inline int PageMappingFlags(struct page *page)
447{
448 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
449}
450
451static __always_inline int PageAnon(struct page *page)
452{
453 page = compound_head(page);
454 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
455}
456
457static __always_inline int __PageMovable(struct page *page)
458{
459 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
460 PAGE_MAPPING_MOVABLE;
461}
462
463#ifdef CONFIG_KSM
464
465
466
467
468
469
470static __always_inline int PageKsm(struct page *page)
471{
472 page = compound_head(page);
473 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
474 PAGE_MAPPING_KSM;
475}
476#else
477TESTPAGEFLAG_FALSE(Ksm)
478#endif
479
480u64 stable_page_flags(struct page *page);
481
482static inline int PageUptodate(struct page *page)
483{
484 int ret;
485 page = compound_head(page);
486 ret = test_bit(PG_uptodate, &(page)->flags);
487
488
489
490
491
492
493
494
495 if (ret)
496 smp_rmb();
497
498 return ret;
499}
500
501static __always_inline void __SetPageUptodate(struct page *page)
502{
503 VM_BUG_ON_PAGE(PageTail(page), page);
504 smp_wmb();
505 __set_bit(PG_uptodate, &page->flags);
506}
507
508static __always_inline void SetPageUptodate(struct page *page)
509{
510 VM_BUG_ON_PAGE(PageTail(page), page);
511
512
513
514
515
516 smp_wmb();
517 set_bit(PG_uptodate, &page->flags);
518}
519
520CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
521
522int test_clear_page_writeback(struct page *page);
523int __test_set_page_writeback(struct page *page, bool keep_write);
524
525#define test_set_page_writeback(page) \
526 __test_set_page_writeback(page, false)
527#define test_set_page_writeback_keepwrite(page) \
528 __test_set_page_writeback(page, true)
529
530static inline void set_page_writeback(struct page *page)
531{
532 test_set_page_writeback(page);
533}
534
535static inline void set_page_writeback_keepwrite(struct page *page)
536{
537 test_set_page_writeback_keepwrite(page);
538}
539
540__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
541
542static __always_inline void set_compound_head(struct page *page, struct page *head)
543{
544 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
545}
546
547static __always_inline void clear_compound_head(struct page *page)
548{
549 WRITE_ONCE(page->compound_head, 0);
550}
551
552#ifdef CONFIG_TRANSPARENT_HUGEPAGE
553static inline void ClearPageCompound(struct page *page)
554{
555 BUG_ON(!PageHead(page));
556 ClearPageHead(page);
557}
558#endif
559
560#define PG_head_mask ((1UL << PG_head))
561
562#ifdef CONFIG_HUGETLB_PAGE
563int PageHuge(struct page *page);
564int PageHeadHuge(struct page *page);
565#else
566TESTPAGEFLAG_FALSE(Huge)
567TESTPAGEFLAG_FALSE(HeadHuge)
568#endif
569
570
571#ifdef CONFIG_TRANSPARENT_HUGEPAGE
572
573
574
575
576
577
578
579
580static inline int PageTransHuge(struct page *page)
581{
582 VM_BUG_ON_PAGE(PageTail(page), page);
583 return PageHead(page);
584}
585
586
587
588
589
590
591static inline int PageTransCompound(struct page *page)
592{
593 return PageCompound(page);
594}
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617static inline int PageTransCompoundMap(struct page *page)
618{
619 struct page *head;
620
621 if (!PageTransCompound(page))
622 return 0;
623
624 if (PageAnon(page))
625 return atomic_read(&page->_mapcount) < 0;
626
627 head = compound_head(page);
628
629 return atomic_read(&page->_mapcount) ==
630 atomic_read(compound_mapcount_ptr(head));
631}
632
633
634
635
636
637
638static inline int PageTransTail(struct page *page)
639{
640 return PageTail(page);
641}
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656static inline int PageDoubleMap(struct page *page)
657{
658 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
659}
660
661static inline void SetPageDoubleMap(struct page *page)
662{
663 VM_BUG_ON_PAGE(!PageHead(page), page);
664 set_bit(PG_double_map, &page[1].flags);
665}
666
667static inline void ClearPageDoubleMap(struct page *page)
668{
669 VM_BUG_ON_PAGE(!PageHead(page), page);
670 clear_bit(PG_double_map, &page[1].flags);
671}
672static inline int TestSetPageDoubleMap(struct page *page)
673{
674 VM_BUG_ON_PAGE(!PageHead(page), page);
675 return test_and_set_bit(PG_double_map, &page[1].flags);
676}
677
678static inline int TestClearPageDoubleMap(struct page *page)
679{
680 VM_BUG_ON_PAGE(!PageHead(page), page);
681 return test_and_clear_bit(PG_double_map, &page[1].flags);
682}
683
684#else
685TESTPAGEFLAG_FALSE(TransHuge)
686TESTPAGEFLAG_FALSE(TransCompound)
687TESTPAGEFLAG_FALSE(TransCompoundMap)
688TESTPAGEFLAG_FALSE(TransTail)
689PAGEFLAG_FALSE(DoubleMap)
690 TESTSETFLAG_FALSE(DoubleMap)
691 TESTCLEARFLAG_FALSE(DoubleMap)
692#endif
693
694
695
696
697
698
699
700
701
702
703#define PAGE_TYPE_BASE 0xf0000000
704
705#define PAGE_MAPCOUNT_RESERVE -128
706#define PG_buddy 0x00000080
707#define PG_offline 0x00000100
708#define PG_table 0x00000200
709#define PG_guard 0x00000400
710
711#define PageType(page, flag) \
712 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
713
714static inline int page_has_type(struct page *page)
715{
716 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;
717}
718
719#define PAGE_TYPE_OPS(uname, lname) \
720static __always_inline int Page##uname(struct page *page) \
721{ \
722 return PageType(page, PG_##lname); \
723} \
724static __always_inline void __SetPage##uname(struct page *page) \
725{ \
726 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
727 page->page_type &= ~PG_##lname; \
728} \
729static __always_inline void __ClearPage##uname(struct page *page) \
730{ \
731 VM_BUG_ON_PAGE(!Page##uname(page), page); \
732 page->page_type |= PG_##lname; \
733}
734
735
736
737
738
739PAGE_TYPE_OPS(Buddy, buddy)
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758PAGE_TYPE_OPS(Offline, offline)
759
760
761
762
763PAGE_TYPE_OPS(Table, table)
764
765
766
767
768PAGE_TYPE_OPS(Guard, guard)
769
770extern bool is_free_buddy_page(struct page *page);
771
772__PAGEFLAG(Isolated, isolated, PF_ANY);
773
774
775
776
777
778static inline int PageSlabPfmemalloc(struct page *page)
779{
780 VM_BUG_ON_PAGE(!PageSlab(page), page);
781 return PageActive(page);
782}
783
784
785
786
787
788static inline int __PageSlabPfmemalloc(struct page *page)
789{
790 return PageActive(page);
791}
792
793static inline void SetPageSlabPfmemalloc(struct page *page)
794{
795 VM_BUG_ON_PAGE(!PageSlab(page), page);
796 SetPageActive(page);
797}
798
799static inline void __ClearPageSlabPfmemalloc(struct page *page)
800{
801 VM_BUG_ON_PAGE(!PageSlab(page), page);
802 __ClearPageActive(page);
803}
804
805static inline void ClearPageSlabPfmemalloc(struct page *page)
806{
807 VM_BUG_ON_PAGE(!PageSlab(page), page);
808 ClearPageActive(page);
809}
810
811#ifdef CONFIG_MMU
812#define __PG_MLOCKED (1UL << PG_mlocked)
813#else
814#define __PG_MLOCKED 0
815#endif
816
817
818
819
820
821#define PAGE_FLAGS_CHECK_AT_FREE \
822 (1UL << PG_lru | 1UL << PG_locked | \
823 1UL << PG_private | 1UL << PG_private_2 | \
824 1UL << PG_writeback | 1UL << PG_reserved | \
825 1UL << PG_slab | 1UL << PG_active | \
826 1UL << PG_unevictable | __PG_MLOCKED)
827
828
829
830
831
832
833
834
835
836#define PAGE_FLAGS_CHECK_AT_PREP \
837 (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
838
839#define PAGE_FLAGS_PRIVATE \
840 (1UL << PG_private | 1UL << PG_private_2)
841
842
843
844
845
846
847
848static inline int page_has_private(struct page *page)
849{
850 return !!(page->flags & PAGE_FLAGS_PRIVATE);
851}
852
853#undef PF_ANY
854#undef PF_HEAD
855#undef PF_ONLY_HEAD
856#undef PF_NO_TAIL
857#undef PF_NO_COMPOUND
858#endif
859
860#endif
861