1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70enum pageflags {
71 PG_locked,
72 PG_error,
73 PG_referenced,
74 PG_uptodate,
75 PG_dirty,
76 PG_lru,
77 PG_active,
78 PG_waiters,
79 PG_slab,
80 PG_owner_priv_1,
81 PG_arch_1,
82 PG_reserved,
83 PG_private,
84 PG_private_2,
85 PG_writeback,
86 PG_head,
87 PG_mappedtodisk,
88 PG_reclaim,
89 PG_swapbacked,
90 PG_unevictable,
91#ifdef CONFIG_MMU
92 PG_mlocked,
93#endif
94#ifdef CONFIG_ARCH_USES_PG_UNCACHED
95 PG_uncached,
96#endif
97#ifdef CONFIG_MEMORY_FAILURE
98 PG_hwpoison,
99#endif
100#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
101 PG_young,
102 PG_idle,
103#endif
104 __NR_PAGEFLAGS,
105
106
107 PG_checked = PG_owner_priv_1,
108
109
110 PG_swapcache = PG_owner_priv_1,
111
112
113
114
115
116 PG_fscache = PG_private_2,
117
118
119
120 PG_pinned = PG_owner_priv_1,
121
122 PG_savepinned = PG_dirty,
123
124 PG_foreign = PG_owner_priv_1,
125
126
127 PG_slob_free = PG_private,
128
129
130 PG_double_map = PG_private_2,
131
132
133 PG_isolated = PG_reclaim,
134};
135
136#ifndef __GENERATING_BOUNDS_H
137
138struct page;
139
140static inline struct page *compound_head(struct page *page)
141{
142 unsigned long head = READ_ONCE(page->compound_head);
143
144 if (unlikely(head & 1))
145 return (struct page *) (head - 1);
146 return page;
147}
148
149static __always_inline int PageTail(struct page *page)
150{
151 return READ_ONCE(page->compound_head) & 1;
152}
153
154static __always_inline int PageCompound(struct page *page)
155{
156 return test_bit(PG_head, &page->flags) || PageTail(page);
157}
158
159#define PAGE_POISON_PATTERN -1l
160static inline int PagePoisoned(const struct page *page)
161{
162 return page->flags == PAGE_POISON_PATTERN;
163}
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188#define PF_POISONED_CHECK(page) ({ \
189 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
190 page; })
191#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
192#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
193#define PF_ONLY_HEAD(page, enforce) ({ \
194 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
195 PF_POISONED_CHECK(page); })
196#define PF_NO_TAIL(page, enforce) ({ \
197 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
198 PF_POISONED_CHECK(compound_head(page)); })
199#define PF_NO_COMPOUND(page, enforce) ({ \
200 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
201 PF_POISONED_CHECK(page); })
202
203
204
205
206#define TESTPAGEFLAG(uname, lname, policy) \
207static __always_inline int Page##uname(struct page *page) \
208 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
209
210#define SETPAGEFLAG(uname, lname, policy) \
211static __always_inline void SetPage##uname(struct page *page) \
212 { set_bit(PG_##lname, &policy(page, 1)->flags); }
213
214#define CLEARPAGEFLAG(uname, lname, policy) \
215static __always_inline void ClearPage##uname(struct page *page) \
216 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
217
218#define __SETPAGEFLAG(uname, lname, policy) \
219static __always_inline void __SetPage##uname(struct page *page) \
220 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
221
222#define __CLEARPAGEFLAG(uname, lname, policy) \
223static __always_inline void __ClearPage##uname(struct page *page) \
224 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
225
226#define TESTSETFLAG(uname, lname, policy) \
227static __always_inline int TestSetPage##uname(struct page *page) \
228 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
229
230#define TESTCLEARFLAG(uname, lname, policy) \
231static __always_inline int TestClearPage##uname(struct page *page) \
232 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
233
234#define PAGEFLAG(uname, lname, policy) \
235 TESTPAGEFLAG(uname, lname, policy) \
236 SETPAGEFLAG(uname, lname, policy) \
237 CLEARPAGEFLAG(uname, lname, policy)
238
239#define __PAGEFLAG(uname, lname, policy) \
240 TESTPAGEFLAG(uname, lname, policy) \
241 __SETPAGEFLAG(uname, lname, policy) \
242 __CLEARPAGEFLAG(uname, lname, policy)
243
244#define TESTSCFLAG(uname, lname, policy) \
245 TESTSETFLAG(uname, lname, policy) \
246 TESTCLEARFLAG(uname, lname, policy)
247
248#define TESTPAGEFLAG_FALSE(uname) \
249static inline int Page##uname(const struct page *page) { return 0; }
250
251#define SETPAGEFLAG_NOOP(uname) \
252static inline void SetPage##uname(struct page *page) { }
253
254#define CLEARPAGEFLAG_NOOP(uname) \
255static inline void ClearPage##uname(struct page *page) { }
256
257#define __CLEARPAGEFLAG_NOOP(uname) \
258static inline void __ClearPage##uname(struct page *page) { }
259
260#define TESTSETFLAG_FALSE(uname) \
261static inline int TestSetPage##uname(struct page *page) { return 0; }
262
263#define TESTCLEARFLAG_FALSE(uname) \
264static inline int TestClearPage##uname(struct page *page) { return 0; }
265
266#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
267 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
268
269#define TESTSCFLAG_FALSE(uname) \
270 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
271
272__PAGEFLAG(Locked, locked, PF_NO_TAIL)
273PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
274PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
275PAGEFLAG(Referenced, referenced, PF_HEAD)
276 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
277 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
278PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
279 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
280PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
281PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
282 TESTCLEARFLAG(Active, active, PF_HEAD)
283__PAGEFLAG(Slab, slab, PF_NO_TAIL)
284__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
285PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
286
287
288PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
289 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
290PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
291PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
292
293PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
294 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
295PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
296 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
297 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
298
299
300
301
302
303
304PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
305 __CLEARPAGEFLAG(Private, private, PF_ANY)
306PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
307PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
308 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
309
310
311
312
313
314TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
315 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
316PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
317
318
319PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
320 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
321PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
322 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
323
324#ifdef CONFIG_HIGHMEM
325
326
327
328
329#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
330#else
331PAGEFLAG_FALSE(HighMem)
332#endif
333
334#ifdef CONFIG_SWAP
335static __always_inline int PageSwapCache(struct page *page)
336{
337#ifdef CONFIG_THP_SWAP
338 page = compound_head(page);
339#endif
340 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
341
342}
343SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
344CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
345#else
346PAGEFLAG_FALSE(SwapCache)
347#endif
348
349PAGEFLAG(Unevictable, unevictable, PF_HEAD)
350 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
351 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
352
353#ifdef CONFIG_MMU
354PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
355 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
356 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
357#else
358PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
359 TESTSCFLAG_FALSE(Mlocked)
360#endif
361
362#ifdef CONFIG_ARCH_USES_PG_UNCACHED
363PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
364#else
365PAGEFLAG_FALSE(Uncached)
366#endif
367
368#ifdef CONFIG_MEMORY_FAILURE
369PAGEFLAG(HWPoison, hwpoison, PF_ANY)
370TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
371#define __PG_HWPOISON (1UL << PG_hwpoison)
372extern bool set_hwpoison_free_buddy_page(struct page *page);
373#else
374PAGEFLAG_FALSE(HWPoison)
375static inline bool set_hwpoison_free_buddy_page(struct page *page)
376{
377 return 0;
378}
379#define __PG_HWPOISON 0
380#endif
381
382#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
383TESTPAGEFLAG(Young, young, PF_ANY)
384SETPAGEFLAG(Young, young, PF_ANY)
385TESTCLEARFLAG(Young, young, PF_ANY)
386PAGEFLAG(Idle, idle, PF_ANY)
387#endif
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406#define PAGE_MAPPING_ANON 0x1
407#define PAGE_MAPPING_MOVABLE 0x2
408#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
409#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
410
411static __always_inline int PageMappingFlags(struct page *page)
412{
413 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
414}
415
416static __always_inline int PageAnon(struct page *page)
417{
418 page = compound_head(page);
419 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
420}
421
422static __always_inline int __PageMovable(struct page *page)
423{
424 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
425 PAGE_MAPPING_MOVABLE;
426}
427
428#ifdef CONFIG_KSM
429
430
431
432
433
434
435static __always_inline int PageKsm(struct page *page)
436{
437 page = compound_head(page);
438 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
439 PAGE_MAPPING_KSM;
440}
441#else
442TESTPAGEFLAG_FALSE(Ksm)
443#endif
444
445u64 stable_page_flags(struct page *page);
446
447static inline int PageUptodate(struct page *page)
448{
449 int ret;
450 page = compound_head(page);
451 ret = test_bit(PG_uptodate, &(page)->flags);
452
453
454
455
456
457
458
459
460 if (ret)
461 smp_rmb();
462
463 return ret;
464}
465
466static __always_inline void __SetPageUptodate(struct page *page)
467{
468 VM_BUG_ON_PAGE(PageTail(page), page);
469 smp_wmb();
470 __set_bit(PG_uptodate, &page->flags);
471}
472
473static __always_inline void SetPageUptodate(struct page *page)
474{
475 VM_BUG_ON_PAGE(PageTail(page), page);
476
477
478
479
480
481 smp_wmb();
482 set_bit(PG_uptodate, &page->flags);
483}
484
485CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
486
487int test_clear_page_writeback(struct page *page);
488int __test_set_page_writeback(struct page *page, bool keep_write);
489
490#define test_set_page_writeback(page) \
491 __test_set_page_writeback(page, false)
492#define test_set_page_writeback_keepwrite(page) \
493 __test_set_page_writeback(page, true)
494
495static inline void set_page_writeback(struct page *page)
496{
497 test_set_page_writeback(page);
498}
499
500static inline void set_page_writeback_keepwrite(struct page *page)
501{
502 test_set_page_writeback_keepwrite(page);
503}
504
505__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
506
507static __always_inline void set_compound_head(struct page *page, struct page *head)
508{
509 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
510}
511
512static __always_inline void clear_compound_head(struct page *page)
513{
514 WRITE_ONCE(page->compound_head, 0);
515}
516
517#ifdef CONFIG_TRANSPARENT_HUGEPAGE
518static inline void ClearPageCompound(struct page *page)
519{
520 BUG_ON(!PageHead(page));
521 ClearPageHead(page);
522}
523#endif
524
525#define PG_head_mask ((1UL << PG_head))
526
527#ifdef CONFIG_HUGETLB_PAGE
528int PageHuge(struct page *page);
529int PageHeadHuge(struct page *page);
530bool page_huge_active(struct page *page);
531#else
532TESTPAGEFLAG_FALSE(Huge)
533TESTPAGEFLAG_FALSE(HeadHuge)
534
535static inline bool page_huge_active(struct page *page)
536{
537 return 0;
538}
539#endif
540
541
542#ifdef CONFIG_TRANSPARENT_HUGEPAGE
543
544
545
546
547
548
549
550
551static inline int PageTransHuge(struct page *page)
552{
553 VM_BUG_ON_PAGE(PageTail(page), page);
554 return PageHead(page);
555}
556
557
558
559
560
561
562static inline int PageTransCompound(struct page *page)
563{
564 return PageCompound(page);
565}
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583static inline int PageTransCompoundMap(struct page *page)
584{
585 return PageTransCompound(page) && atomic_read(&page->_mapcount) < 0;
586}
587
588
589
590
591
592
593static inline int PageTransTail(struct page *page)
594{
595 return PageTail(page);
596}
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611static inline int PageDoubleMap(struct page *page)
612{
613 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
614}
615
616static inline void SetPageDoubleMap(struct page *page)
617{
618 VM_BUG_ON_PAGE(!PageHead(page), page);
619 set_bit(PG_double_map, &page[1].flags);
620}
621
622static inline void ClearPageDoubleMap(struct page *page)
623{
624 VM_BUG_ON_PAGE(!PageHead(page), page);
625 clear_bit(PG_double_map, &page[1].flags);
626}
627static inline int TestSetPageDoubleMap(struct page *page)
628{
629 VM_BUG_ON_PAGE(!PageHead(page), page);
630 return test_and_set_bit(PG_double_map, &page[1].flags);
631}
632
633static inline int TestClearPageDoubleMap(struct page *page)
634{
635 VM_BUG_ON_PAGE(!PageHead(page), page);
636 return test_and_clear_bit(PG_double_map, &page[1].flags);
637}
638
639#else
640TESTPAGEFLAG_FALSE(TransHuge)
641TESTPAGEFLAG_FALSE(TransCompound)
642TESTPAGEFLAG_FALSE(TransCompoundMap)
643TESTPAGEFLAG_FALSE(TransTail)
644PAGEFLAG_FALSE(DoubleMap)
645 TESTSETFLAG_FALSE(DoubleMap)
646 TESTCLEARFLAG_FALSE(DoubleMap)
647#endif
648
649
650
651
652
653
654
655
656
657
658#define PAGE_TYPE_BASE 0xf0000000
659
660#define PG_buddy 0x00000080
661#define PG_balloon 0x00000100
662#define PG_kmemcg 0x00000200
663#define PG_table 0x00000400
664
665#define PageType(page, flag) \
666 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
667
668#define PAGE_TYPE_OPS(uname, lname) \
669static __always_inline int Page##uname(struct page *page) \
670{ \
671 return PageType(page, PG_##lname); \
672} \
673static __always_inline void __SetPage##uname(struct page *page) \
674{ \
675 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
676 page->page_type &= ~PG_##lname; \
677} \
678static __always_inline void __ClearPage##uname(struct page *page) \
679{ \
680 VM_BUG_ON_PAGE(!Page##uname(page), page); \
681 page->page_type |= PG_##lname; \
682}
683
684
685
686
687
688PAGE_TYPE_OPS(Buddy, buddy)
689
690
691
692
693
694PAGE_TYPE_OPS(Balloon, balloon)
695
696
697
698
699
700PAGE_TYPE_OPS(Kmemcg, kmemcg)
701
702
703
704
705PAGE_TYPE_OPS(Table, table)
706
707extern bool is_free_buddy_page(struct page *page);
708
709__PAGEFLAG(Isolated, isolated, PF_ANY);
710
711
712
713
714
715static inline int PageSlabPfmemalloc(struct page *page)
716{
717 VM_BUG_ON_PAGE(!PageSlab(page), page);
718 return PageActive(page);
719}
720
721static inline void SetPageSlabPfmemalloc(struct page *page)
722{
723 VM_BUG_ON_PAGE(!PageSlab(page), page);
724 SetPageActive(page);
725}
726
727static inline void __ClearPageSlabPfmemalloc(struct page *page)
728{
729 VM_BUG_ON_PAGE(!PageSlab(page), page);
730 __ClearPageActive(page);
731}
732
733static inline void ClearPageSlabPfmemalloc(struct page *page)
734{
735 VM_BUG_ON_PAGE(!PageSlab(page), page);
736 ClearPageActive(page);
737}
738
739#ifdef CONFIG_MMU
740#define __PG_MLOCKED (1UL << PG_mlocked)
741#else
742#define __PG_MLOCKED 0
743#endif
744
745
746
747
748
749#define PAGE_FLAGS_CHECK_AT_FREE \
750 (1UL << PG_lru | 1UL << PG_locked | \
751 1UL << PG_private | 1UL << PG_private_2 | \
752 1UL << PG_writeback | 1UL << PG_reserved | \
753 1UL << PG_slab | 1UL << PG_active | \
754 1UL << PG_unevictable | __PG_MLOCKED)
755
756
757
758
759
760
761
762
763
764#define PAGE_FLAGS_CHECK_AT_PREP \
765 (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
766
767#define PAGE_FLAGS_PRIVATE \
768 (1UL << PG_private | 1UL << PG_private_2)
769
770
771
772
773
774
775
776static inline int page_has_private(struct page *page)
777{
778 return !!(page->flags & PAGE_FLAGS_PRIVATE);
779}
780
781#undef PF_ANY
782#undef PF_HEAD
783#undef PF_ONLY_HEAD
784#undef PF_NO_TAIL
785#undef PF_NO_COMPOUND
786#endif
787
788#endif
789