1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100enum pageflags {
101 PG_locked,
102 PG_referenced,
103 PG_uptodate,
104 PG_dirty,
105 PG_lru,
106 PG_active,
107 PG_workingset,
108 PG_waiters,
109 PG_error,
110 PG_slab,
111 PG_owner_priv_1,
112 PG_arch_1,
113 PG_reserved,
114 PG_private,
115 PG_private_2,
116 PG_writeback,
117 PG_head,
118 PG_mappedtodisk,
119 PG_reclaim,
120 PG_swapbacked,
121 PG_unevictable,
122#ifdef CONFIG_MMU
123 PG_mlocked,
124#endif
125#ifdef CONFIG_ARCH_USES_PG_UNCACHED
126 PG_uncached,
127#endif
128#ifdef CONFIG_MEMORY_FAILURE
129 PG_hwpoison,
130#endif
131#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
132 PG_young,
133 PG_idle,
134#endif
135#ifdef CONFIG_64BIT
136 PG_arch_2,
137#endif
138#ifdef CONFIG_KASAN_HW_TAGS
139 PG_skip_kasan_poison,
140#endif
141 __NR_PAGEFLAGS,
142
143 PG_readahead = PG_reclaim,
144
145
146
147
148
149
150
151
152 PG_anon_exclusive = PG_mappedtodisk,
153
154
155 PG_checked = PG_owner_priv_1,
156
157
158 PG_swapcache = PG_owner_priv_1,
159
160
161
162
163
164 PG_fscache = PG_private_2,
165
166
167
168 PG_pinned = PG_owner_priv_1,
169
170 PG_savepinned = PG_dirty,
171
172 PG_foreign = PG_owner_priv_1,
173
174 PG_xen_remapped = PG_owner_priv_1,
175
176
177 PG_slob_free = PG_private,
178
179
180 PG_double_map = PG_workingset,
181
182#ifdef CONFIG_MEMORY_FAILURE
183
184
185
186
187
188 PG_has_hwpoisoned = PG_error,
189#endif
190
191
192 PG_isolated = PG_reclaim,
193
194
195 PG_reported = PG_uptodate,
196};
197
198#define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1)
199
200#ifndef __GENERATING_BOUNDS_H
201
202#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
203DECLARE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON,
204 hugetlb_optimize_vmemmap_key);
205
206static __always_inline bool hugetlb_optimize_vmemmap_enabled(void)
207{
208 return static_branch_maybe(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON,
209 &hugetlb_optimize_vmemmap_key);
210}
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
229{
230 if (!hugetlb_optimize_vmemmap_enabled())
231 return page;
232
233
234
235
236
237
238
239 if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
240 test_bit(PG_head, &page->flags)) {
241
242
243
244
245
246 unsigned long head = READ_ONCE(page[1].compound_head);
247
248 if (likely(head & 1))
249 return (const struct page *)(head - 1);
250 }
251 return page;
252}
253#else
254static inline const struct page *page_fixed_fake_head(const struct page *page)
255{
256 return page;
257}
258
259static inline bool hugetlb_optimize_vmemmap_enabled(void)
260{
261 return false;
262}
263#endif
264
265static __always_inline int page_is_fake_head(struct page *page)
266{
267 return page_fixed_fake_head(page) != page;
268}
269
270static inline unsigned long _compound_head(const struct page *page)
271{
272 unsigned long head = READ_ONCE(page->compound_head);
273
274 if (unlikely(head & 1))
275 return head - 1;
276 return (unsigned long)page_fixed_fake_head(page);
277}
278
279#define compound_head(page) ((typeof(page))_compound_head(page))
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294#define page_folio(p) (_Generic((p), \
295 const struct page *: (const struct folio *)_compound_head(p), \
296 struct page *: (struct folio *)_compound_head(p)))
297
298
299
300
301
302
303
304
305
306
307#define folio_page(folio, n) nth_page(&(folio)->page, n)
308
309static __always_inline int PageTail(struct page *page)
310{
311 return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);
312}
313
314static __always_inline int PageCompound(struct page *page)
315{
316 return test_bit(PG_head, &page->flags) ||
317 READ_ONCE(page->compound_head) & 1;
318}
319
320#define PAGE_POISON_PATTERN -1l
321static inline int PagePoisoned(const struct page *page)
322{
323 return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;
324}
325
326#ifdef CONFIG_DEBUG_VM
327void page_init_poison(struct page *page, size_t size);
328#else
329static inline void page_init_poison(struct page *page, size_t size)
330{
331}
332#endif
333
334static unsigned long *folio_flags(struct folio *folio, unsigned n)
335{
336 struct page *page = &folio->page;
337
338 VM_BUG_ON_PGFLAGS(PageTail(page), page);
339 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);
340 return &page[n].flags;
341}
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369#define PF_POISONED_CHECK(page) ({ \
370 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
371 page; })
372#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
373#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
374#define PF_ONLY_HEAD(page, enforce) ({ \
375 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
376 PF_POISONED_CHECK(page); })
377#define PF_NO_TAIL(page, enforce) ({ \
378 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
379 PF_POISONED_CHECK(compound_head(page)); })
380#define PF_NO_COMPOUND(page, enforce) ({ \
381 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
382 PF_POISONED_CHECK(page); })
383#define PF_SECOND(page, enforce) ({ \
384 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
385 PF_POISONED_CHECK(&page[1]); })
386
387
388#define FOLIO_PF_ANY 0
389#define FOLIO_PF_HEAD 0
390#define FOLIO_PF_ONLY_HEAD 0
391#define FOLIO_PF_NO_TAIL 0
392#define FOLIO_PF_NO_COMPOUND 0
393#define FOLIO_PF_SECOND 1
394
395
396
397
398#define TESTPAGEFLAG(uname, lname, policy) \
399static __always_inline bool folio_test_##lname(struct folio *folio) \
400{ return test_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
401static __always_inline int Page##uname(struct page *page) \
402{ return test_bit(PG_##lname, &policy(page, 0)->flags); }
403
404#define SETPAGEFLAG(uname, lname, policy) \
405static __always_inline \
406void folio_set_##lname(struct folio *folio) \
407{ set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
408static __always_inline void SetPage##uname(struct page *page) \
409{ set_bit(PG_##lname, &policy(page, 1)->flags); }
410
411#define CLEARPAGEFLAG(uname, lname, policy) \
412static __always_inline \
413void folio_clear_##lname(struct folio *folio) \
414{ clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
415static __always_inline void ClearPage##uname(struct page *page) \
416{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
417
418#define __SETPAGEFLAG(uname, lname, policy) \
419static __always_inline \
420void __folio_set_##lname(struct folio *folio) \
421{ __set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
422static __always_inline void __SetPage##uname(struct page *page) \
423{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
424
425#define __CLEARPAGEFLAG(uname, lname, policy) \
426static __always_inline \
427void __folio_clear_##lname(struct folio *folio) \
428{ __clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
429static __always_inline void __ClearPage##uname(struct page *page) \
430{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
431
432#define TESTSETFLAG(uname, lname, policy) \
433static __always_inline \
434bool folio_test_set_##lname(struct folio *folio) \
435{ return test_and_set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
436static __always_inline int TestSetPage##uname(struct page *page) \
437{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
438
439#define TESTCLEARFLAG(uname, lname, policy) \
440static __always_inline \
441bool folio_test_clear_##lname(struct folio *folio) \
442{ return test_and_clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
443static __always_inline int TestClearPage##uname(struct page *page) \
444{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
445
446#define PAGEFLAG(uname, lname, policy) \
447 TESTPAGEFLAG(uname, lname, policy) \
448 SETPAGEFLAG(uname, lname, policy) \
449 CLEARPAGEFLAG(uname, lname, policy)
450
451#define __PAGEFLAG(uname, lname, policy) \
452 TESTPAGEFLAG(uname, lname, policy) \
453 __SETPAGEFLAG(uname, lname, policy) \
454 __CLEARPAGEFLAG(uname, lname, policy)
455
456#define TESTSCFLAG(uname, lname, policy) \
457 TESTSETFLAG(uname, lname, policy) \
458 TESTCLEARFLAG(uname, lname, policy)
459
460#define TESTPAGEFLAG_FALSE(uname, lname) \
461static inline bool folio_test_##lname(const struct folio *folio) { return false; } \
462static inline int Page##uname(const struct page *page) { return 0; }
463
464#define SETPAGEFLAG_NOOP(uname, lname) \
465static inline void folio_set_##lname(struct folio *folio) { } \
466static inline void SetPage##uname(struct page *page) { }
467
468#define CLEARPAGEFLAG_NOOP(uname, lname) \
469static inline void folio_clear_##lname(struct folio *folio) { } \
470static inline void ClearPage##uname(struct page *page) { }
471
472#define __CLEARPAGEFLAG_NOOP(uname, lname) \
473static inline void __folio_clear_##lname(struct folio *folio) { } \
474static inline void __ClearPage##uname(struct page *page) { }
475
476#define TESTSETFLAG_FALSE(uname, lname) \
477static inline bool folio_test_set_##lname(struct folio *folio) \
478{ return 0; } \
479static inline int TestSetPage##uname(struct page *page) { return 0; }
480
481#define TESTCLEARFLAG_FALSE(uname, lname) \
482static inline bool folio_test_clear_##lname(struct folio *folio) \
483{ return 0; } \
484static inline int TestClearPage##uname(struct page *page) { return 0; }
485
486#define PAGEFLAG_FALSE(uname, lname) TESTPAGEFLAG_FALSE(uname, lname) \
487 SETPAGEFLAG_NOOP(uname, lname) CLEARPAGEFLAG_NOOP(uname, lname)
488
489#define TESTSCFLAG_FALSE(uname, lname) \
490 TESTSETFLAG_FALSE(uname, lname) TESTCLEARFLAG_FALSE(uname, lname)
491
492__PAGEFLAG(Locked, locked, PF_NO_TAIL)
493PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
494PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL)
495PAGEFLAG(Referenced, referenced, PF_HEAD)
496 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
497 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
498PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
499 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
500PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
501 TESTCLEARFLAG(LRU, lru, PF_HEAD)
502PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
503 TESTCLEARFLAG(Active, active, PF_HEAD)
504PAGEFLAG(Workingset, workingset, PF_HEAD)
505 TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
506__PAGEFLAG(Slab, slab, PF_NO_TAIL)
507__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
508PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
509
510
511PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
512 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
513PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
514PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
515PAGEFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
516 TESTCLEARFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
517
518PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
519 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
520 __SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
521PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
522 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
523 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
524
525
526
527
528
529
530PAGEFLAG(Private, private, PF_ANY)
531PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
532PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
533 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
534
535
536
537
538
539TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
540 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
541PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
542
543
544PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
545 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
546PAGEFLAG(Readahead, readahead, PF_NO_COMPOUND)
547 TESTCLEARFLAG(Readahead, readahead, PF_NO_COMPOUND)
548
549#ifdef CONFIG_HIGHMEM
550
551
552
553
554#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
555#else
556PAGEFLAG_FALSE(HighMem, highmem)
557#endif
558
559#ifdef CONFIG_SWAP
560static __always_inline bool folio_test_swapcache(struct folio *folio)
561{
562 return folio_test_swapbacked(folio) &&
563 test_bit(PG_swapcache, folio_flags(folio, 0));
564}
565
566static __always_inline bool PageSwapCache(struct page *page)
567{
568 return folio_test_swapcache(page_folio(page));
569}
570
571SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
572CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
573#else
574PAGEFLAG_FALSE(SwapCache, swapcache)
575#endif
576
577PAGEFLAG(Unevictable, unevictable, PF_HEAD)
578 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
579 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
580
581#ifdef CONFIG_MMU
582PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
583 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
584 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
585#else
586PAGEFLAG_FALSE(Mlocked, mlocked) __CLEARPAGEFLAG_NOOP(Mlocked, mlocked)
587 TESTSCFLAG_FALSE(Mlocked, mlocked)
588#endif
589
590#ifdef CONFIG_ARCH_USES_PG_UNCACHED
591PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
592#else
593PAGEFLAG_FALSE(Uncached, uncached)
594#endif
595
596#ifdef CONFIG_MEMORY_FAILURE
597PAGEFLAG(HWPoison, hwpoison, PF_ANY)
598TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
599#define __PG_HWPOISON (1UL << PG_hwpoison)
600#define MAGIC_HWPOISON 0x48575053U
601extern void SetPageHWPoisonTakenOff(struct page *page);
602extern void ClearPageHWPoisonTakenOff(struct page *page);
603extern bool take_page_off_buddy(struct page *page);
604extern bool put_page_back_buddy(struct page *page);
605#else
606PAGEFLAG_FALSE(HWPoison, hwpoison)
607#define __PG_HWPOISON 0
608#endif
609
610#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
611TESTPAGEFLAG(Young, young, PF_ANY)
612SETPAGEFLAG(Young, young, PF_ANY)
613TESTCLEARFLAG(Young, young, PF_ANY)
614PAGEFLAG(Idle, idle, PF_ANY)
615#endif
616
617#ifdef CONFIG_KASAN_HW_TAGS
618PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD)
619#else
620PAGEFLAG_FALSE(SkipKASanPoison, skip_kasan_poison)
621#endif
622
623
624
625
626
627
628
629__PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648#define PAGE_MAPPING_ANON 0x1
649#define PAGE_MAPPING_MOVABLE 0x2
650#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
651#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
652
653static __always_inline bool folio_mapping_flags(struct folio *folio)
654{
655 return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0;
656}
657
658static __always_inline int PageMappingFlags(struct page *page)
659{
660 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
661}
662
663static __always_inline bool folio_test_anon(struct folio *folio)
664{
665 return ((unsigned long)folio->mapping & PAGE_MAPPING_ANON) != 0;
666}
667
668static __always_inline bool PageAnon(struct page *page)
669{
670 return folio_test_anon(page_folio(page));
671}
672
673static __always_inline int __PageMovable(struct page *page)
674{
675 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
676 PAGE_MAPPING_MOVABLE;
677}
678
679#ifdef CONFIG_KSM
680
681
682
683
684
685
686static __always_inline bool folio_test_ksm(struct folio *folio)
687{
688 return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==
689 PAGE_MAPPING_KSM;
690}
691
692static __always_inline bool PageKsm(struct page *page)
693{
694 return folio_test_ksm(page_folio(page));
695}
696#else
697TESTPAGEFLAG_FALSE(Ksm, ksm)
698#endif
699
700u64 stable_page_flags(struct page *page);
701
702
703
704
705
706
707
708
709
710
711
712static inline bool folio_test_uptodate(struct folio *folio)
713{
714 bool ret = test_bit(PG_uptodate, folio_flags(folio, 0));
715
716
717
718
719
720
721
722
723 if (ret)
724 smp_rmb();
725
726 return ret;
727}
728
729static inline int PageUptodate(struct page *page)
730{
731 return folio_test_uptodate(page_folio(page));
732}
733
734static __always_inline void __folio_mark_uptodate(struct folio *folio)
735{
736 smp_wmb();
737 __set_bit(PG_uptodate, folio_flags(folio, 0));
738}
739
740static __always_inline void folio_mark_uptodate(struct folio *folio)
741{
742
743
744
745
746
747 smp_wmb();
748 set_bit(PG_uptodate, folio_flags(folio, 0));
749}
750
751static __always_inline void __SetPageUptodate(struct page *page)
752{
753 __folio_mark_uptodate((struct folio *)page);
754}
755
756static __always_inline void SetPageUptodate(struct page *page)
757{
758 folio_mark_uptodate((struct folio *)page);
759}
760
761CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
762
763bool __folio_start_writeback(struct folio *folio, bool keep_write);
764bool set_page_writeback(struct page *page);
765
766#define folio_start_writeback(folio) \
767 __folio_start_writeback(folio, false)
768#define folio_start_writeback_keepwrite(folio) \
769 __folio_start_writeback(folio, true)
770
771static inline void set_page_writeback_keepwrite(struct page *page)
772{
773 folio_start_writeback_keepwrite(page_folio(page));
774}
775
776static inline bool test_set_page_writeback(struct page *page)
777{
778 return set_page_writeback(page);
779}
780
781static __always_inline bool folio_test_head(struct folio *folio)
782{
783 return test_bit(PG_head, folio_flags(folio, FOLIO_PF_ANY));
784}
785
786static __always_inline int PageHead(struct page *page)
787{
788 PF_POISONED_CHECK(page);
789 return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);
790}
791
792__SETPAGEFLAG(Head, head, PF_ANY)
793__CLEARPAGEFLAG(Head, head, PF_ANY)
794CLEARPAGEFLAG(Head, head, PF_ANY)
795
796
797
798
799
800
801
802static inline bool folio_test_large(struct folio *folio)
803{
804 return folio_test_head(folio);
805}
806
807static __always_inline void set_compound_head(struct page *page, struct page *head)
808{
809 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
810}
811
812static __always_inline void clear_compound_head(struct page *page)
813{
814 WRITE_ONCE(page->compound_head, 0);
815}
816
817#ifdef CONFIG_TRANSPARENT_HUGEPAGE
818static inline void ClearPageCompound(struct page *page)
819{
820 BUG_ON(!PageHead(page));
821 ClearPageHead(page);
822}
823#endif
824
825#define PG_head_mask ((1UL << PG_head))
826
827#ifdef CONFIG_HUGETLB_PAGE
828int PageHuge(struct page *page);
829int PageHeadHuge(struct page *page);
830static inline bool folio_test_hugetlb(struct folio *folio)
831{
832 return PageHeadHuge(&folio->page);
833}
834#else
835TESTPAGEFLAG_FALSE(Huge, hugetlb)
836TESTPAGEFLAG_FALSE(HeadHuge, headhuge)
837#endif
838
839#ifdef CONFIG_TRANSPARENT_HUGEPAGE
840
841
842
843
844
845
846
847
848static inline int PageTransHuge(struct page *page)
849{
850 VM_BUG_ON_PAGE(PageTail(page), page);
851 return PageHead(page);
852}
853
854static inline bool folio_test_transhuge(struct folio *folio)
855{
856 return folio_test_head(folio);
857}
858
859
860
861
862
863
864static inline int PageTransCompound(struct page *page)
865{
866 return PageCompound(page);
867}
868
869
870
871
872
873
874static inline int PageTransTail(struct page *page)
875{
876 return PageTail(page);
877}
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892PAGEFLAG(DoubleMap, double_map, PF_SECOND)
893 TESTSCFLAG(DoubleMap, double_map, PF_SECOND)
894#else
895TESTPAGEFLAG_FALSE(TransHuge, transhuge)
896TESTPAGEFLAG_FALSE(TransCompound, transcompound)
897TESTPAGEFLAG_FALSE(TransCompoundMap, transcompoundmap)
898TESTPAGEFLAG_FALSE(TransTail, transtail)
899PAGEFLAG_FALSE(DoubleMap, double_map)
900 TESTSCFLAG_FALSE(DoubleMap, double_map)
901#endif
902
903#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
904
905
906
907
908
909
910PAGEFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
911 TESTSCFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
912#else
913PAGEFLAG_FALSE(HasHWPoisoned, has_hwpoisoned)
914 TESTSCFLAG_FALSE(HasHWPoisoned, has_hwpoisoned)
915#endif
916
917
918
919
920
921
922static inline bool is_page_hwpoison(struct page *page)
923{
924 if (PageHWPoison(page))
925 return true;
926 return PageHuge(page) && PageHWPoison(compound_head(page));
927}
928
929
930
931
932
933
934
935
936
937
938#define PAGE_TYPE_BASE 0xf0000000
939
940#define PAGE_MAPCOUNT_RESERVE -128
941#define PG_buddy 0x00000080
942#define PG_offline 0x00000100
943#define PG_table 0x00000200
944#define PG_guard 0x00000400
945
946#define PageType(page, flag) \
947 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
948
949static inline int page_has_type(struct page *page)
950{
951 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;
952}
953
954#define PAGE_TYPE_OPS(uname, lname) \
955static __always_inline int Page##uname(struct page *page) \
956{ \
957 return PageType(page, PG_##lname); \
958} \
959static __always_inline void __SetPage##uname(struct page *page) \
960{ \
961 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
962 page->page_type &= ~PG_##lname; \
963} \
964static __always_inline void __ClearPage##uname(struct page *page) \
965{ \
966 VM_BUG_ON_PAGE(!Page##uname(page), page); \
967 page->page_type |= PG_##lname; \
968}
969
970
971
972
973
974PAGE_TYPE_OPS(Buddy, buddy)
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998PAGE_TYPE_OPS(Offline, offline)
999
1000extern void page_offline_freeze(void);
1001extern void page_offline_thaw(void);
1002extern void page_offline_begin(void);
1003extern void page_offline_end(void);
1004
1005
1006
1007
1008PAGE_TYPE_OPS(Table, table)
1009
1010
1011
1012
1013PAGE_TYPE_OPS(Guard, guard)
1014
1015extern bool is_free_buddy_page(struct page *page);
1016
1017PAGEFLAG(Isolated, isolated, PF_ANY);
1018
1019static __always_inline int PageAnonExclusive(struct page *page)
1020{
1021 VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
1022 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1023 return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1024}
1025
1026static __always_inline void SetPageAnonExclusive(struct page *page)
1027{
1028 VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);
1029 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1030 set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1031}
1032
1033static __always_inline void ClearPageAnonExclusive(struct page *page)
1034{
1035 VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);
1036 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1037 clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1038}
1039
1040static __always_inline void __ClearPageAnonExclusive(struct page *page)
1041{
1042 VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
1043 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1044 __clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1045}
1046
1047#ifdef CONFIG_MMU
1048#define __PG_MLOCKED (1UL << PG_mlocked)
1049#else
1050#define __PG_MLOCKED 0
1051#endif
1052
1053
1054
1055
1056
1057#define PAGE_FLAGS_CHECK_AT_FREE \
1058 (1UL << PG_lru | 1UL << PG_locked | \
1059 1UL << PG_private | 1UL << PG_private_2 | \
1060 1UL << PG_writeback | 1UL << PG_reserved | \
1061 1UL << PG_slab | 1UL << PG_active | \
1062 1UL << PG_unevictable | __PG_MLOCKED)
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072#define PAGE_FLAGS_CHECK_AT_PREP \
1073 (PAGEFLAGS_MASK & ~__PG_HWPOISON)
1074
1075#define PAGE_FLAGS_PRIVATE \
1076 (1UL << PG_private | 1UL << PG_private_2)
1077
1078
1079
1080
1081
1082
1083
1084static inline int page_has_private(struct page *page)
1085{
1086 return !!(page->flags & PAGE_FLAGS_PRIVATE);
1087}
1088
1089static inline bool folio_has_private(struct folio *folio)
1090{
1091 return page_has_private(&folio->page);
1092}
1093
1094#undef PF_ANY
1095#undef PF_HEAD
1096#undef PF_ONLY_HEAD
1097#undef PF_NO_TAIL
1098#undef PF_NO_COMPOUND
1099#undef PF_SECOND
1100#endif
1101
1102#endif
1103