1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75enum pageflags {
76 PG_locked,
77 PG_error,
78 PG_referenced,
79 PG_uptodate,
80 PG_dirty,
81 PG_lru,
82 PG_active,
83 PG_waiters,
84 PG_slab,
85 PG_owner_priv_1,
86 PG_arch_1,
87 PG_reserved,
88 PG_private,
89 PG_private_2,
90 PG_writeback,
91 PG_head,
92 PG_mappedtodisk,
93 PG_reclaim,
94 PG_swapbacked,
95 PG_unevictable,
96#ifdef CONFIG_MMU
97 PG_mlocked,
98#endif
99#ifdef CONFIG_ARCH_USES_PG_UNCACHED
100 PG_uncached,
101#endif
102#ifdef CONFIG_MEMORY_FAILURE
103 PG_hwpoison,
104#endif
105#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
106 PG_young,
107 PG_idle,
108#endif
109#ifndef __GENKSYMS__
110
111
112
113
114 PG_workingset,
115#endif
116 __NR_PAGEFLAGS,
117
118
119 PG_checked = PG_owner_priv_1,
120
121
122 PG_swapcache = PG_owner_priv_1,
123
124
125
126
127
128 PG_fscache = PG_private_2,
129
130
131
132 PG_pinned = PG_owner_priv_1,
133
134 PG_savepinned = PG_dirty,
135
136 PG_foreign = PG_owner_priv_1,
137
138
139 PG_slob_free = PG_private,
140
141
142 PG_double_map = PG_private_2,
143
144
145 PG_isolated = PG_reclaim,
146
147
148 PG_reported = PG_uptodate,
149};
150
151#ifndef __GENERATING_BOUNDS_H
152
153struct page;
154
155static inline struct page *compound_head(struct page *page)
156{
157 unsigned long head = READ_ONCE(page->compound_head);
158
159 if (unlikely(head & 1))
160 return (struct page *) (head - 1);
161 return page;
162}
163
164static __always_inline int PageTail(struct page *page)
165{
166 return READ_ONCE(page->compound_head) & 1;
167}
168
169static __always_inline int PageCompound(struct page *page)
170{
171 return test_bit(PG_head, &page->flags) || PageTail(page);
172}
173
174#define PAGE_POISON_PATTERN -1l
175static inline int PagePoisoned(const struct page *page)
176{
177 return page->flags == PAGE_POISON_PATTERN;
178}
179
180#ifdef CONFIG_DEBUG_VM
181void page_init_poison(struct page *page, size_t size);
182#else
183static inline void page_init_poison(struct page *page, size_t size)
184{
185}
186#endif
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211#define PF_POISONED_CHECK(page) ({ \
212 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
213 page; })
214#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
215#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
216#define PF_ONLY_HEAD(page, enforce) ({ \
217 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
218 PF_POISONED_CHECK(page); })
219#define PF_NO_TAIL(page, enforce) ({ \
220 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
221 PF_POISONED_CHECK(compound_head(page)); })
222#define PF_NO_COMPOUND(page, enforce) ({ \
223 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
224 PF_POISONED_CHECK(page); })
225
226
227
228
229#define TESTPAGEFLAG(uname, lname, policy) \
230static __always_inline int Page##uname(struct page *page) \
231 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
232
233#define SETPAGEFLAG(uname, lname, policy) \
234static __always_inline void SetPage##uname(struct page *page) \
235 { set_bit(PG_##lname, &policy(page, 1)->flags); }
236
237#define CLEARPAGEFLAG(uname, lname, policy) \
238static __always_inline void ClearPage##uname(struct page *page) \
239 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
240
241#define __SETPAGEFLAG(uname, lname, policy) \
242static __always_inline void __SetPage##uname(struct page *page) \
243 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
244
245#define __CLEARPAGEFLAG(uname, lname, policy) \
246static __always_inline void __ClearPage##uname(struct page *page) \
247 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
248
249#define TESTSETFLAG(uname, lname, policy) \
250static __always_inline int TestSetPage##uname(struct page *page) \
251 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
252
253#define TESTCLEARFLAG(uname, lname, policy) \
254static __always_inline int TestClearPage##uname(struct page *page) \
255 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
256
257#define PAGEFLAG(uname, lname, policy) \
258 TESTPAGEFLAG(uname, lname, policy) \
259 SETPAGEFLAG(uname, lname, policy) \
260 CLEARPAGEFLAG(uname, lname, policy)
261
262#define __PAGEFLAG(uname, lname, policy) \
263 TESTPAGEFLAG(uname, lname, policy) \
264 __SETPAGEFLAG(uname, lname, policy) \
265 __CLEARPAGEFLAG(uname, lname, policy)
266
267#define TESTSCFLAG(uname, lname, policy) \
268 TESTSETFLAG(uname, lname, policy) \
269 TESTCLEARFLAG(uname, lname, policy)
270
271#define TESTPAGEFLAG_FALSE(uname) \
272static inline int Page##uname(const struct page *page) { return 0; }
273
274#define SETPAGEFLAG_NOOP(uname) \
275static inline void SetPage##uname(struct page *page) { }
276
277#define CLEARPAGEFLAG_NOOP(uname) \
278static inline void ClearPage##uname(struct page *page) { }
279
280#define __CLEARPAGEFLAG_NOOP(uname) \
281static inline void __ClearPage##uname(struct page *page) { }
282
283#define TESTSETFLAG_FALSE(uname) \
284static inline int TestSetPage##uname(struct page *page) { return 0; }
285
286#define TESTCLEARFLAG_FALSE(uname) \
287static inline int TestClearPage##uname(struct page *page) { return 0; }
288
289#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
290 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
291
292#define TESTSCFLAG_FALSE(uname) \
293 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
294
295__PAGEFLAG(Locked, locked, PF_NO_TAIL)
296PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
297PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
298PAGEFLAG(Referenced, referenced, PF_HEAD)
299 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
300 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
301PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
302 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
303PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
304 TESTCLEARFLAG(LRU, lru, PF_HEAD)
305PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
306 TESTCLEARFLAG(Active, active, PF_HEAD)
307PAGEFLAG(Workingset, workingset, PF_HEAD)
308 TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
309__PAGEFLAG(Slab, slab, PF_NO_TAIL)
310__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
311PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
312
313
314PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
315 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
316PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
317PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
318
319PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
320 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
321 __SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
322PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
323 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
324 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
325
326
327
328
329
330
331PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
332 __CLEARPAGEFLAG(Private, private, PF_ANY)
333PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
334PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
335 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
336
337
338
339
340
341TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
342 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
343PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
344
345
346PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
347 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
348PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
349 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
350
351#ifdef CONFIG_HIGHMEM
352
353
354
355
356#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
357#else
358PAGEFLAG_FALSE(HighMem)
359#endif
360
361#ifdef CONFIG_SWAP
362static __always_inline int PageSwapCache(struct page *page)
363{
364#ifdef CONFIG_THP_SWAP
365 page = compound_head(page);
366#endif
367 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
368
369}
370SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
371CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
372#else
373PAGEFLAG_FALSE(SwapCache)
374#endif
375
376PAGEFLAG(Unevictable, unevictable, PF_HEAD)
377 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
378 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
379
380#ifdef CONFIG_MMU
381PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
382 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
383 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
384#else
385PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
386 TESTSCFLAG_FALSE(Mlocked)
387#endif
388
389#ifdef CONFIG_ARCH_USES_PG_UNCACHED
390PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
391#else
392PAGEFLAG_FALSE(Uncached)
393#endif
394
395#ifdef CONFIG_MEMORY_FAILURE
396PAGEFLAG(HWPoison, hwpoison, PF_ANY)
397TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
398#define __PG_HWPOISON (1UL << PG_hwpoison)
399extern bool take_page_off_buddy(struct page *page);
400#else
401PAGEFLAG_FALSE(HWPoison)
402#define __PG_HWPOISON 0
403#endif
404
405#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
406TESTPAGEFLAG(Young, young, PF_ANY)
407SETPAGEFLAG(Young, young, PF_ANY)
408TESTCLEARFLAG(Young, young, PF_ANY)
409PAGEFLAG(Idle, idle, PF_ANY)
410#endif
411
412
413
414
415
416
417
418__PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437#define PAGE_MAPPING_ANON 0x1
438#define PAGE_MAPPING_MOVABLE 0x2
439#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
440#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
441
442static __always_inline int PageMappingFlags(struct page *page)
443{
444 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
445}
446
447static __always_inline int PageAnon(struct page *page)
448{
449 page = compound_head(page);
450 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
451}
452
453static __always_inline int __PageMovable(struct page *page)
454{
455 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
456 PAGE_MAPPING_MOVABLE;
457}
458
459#ifdef CONFIG_KSM
460
461
462
463
464
465
466static __always_inline int PageKsm(struct page *page)
467{
468 page = compound_head(page);
469 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
470 PAGE_MAPPING_KSM;
471}
472#else
473TESTPAGEFLAG_FALSE(Ksm)
474#endif
475
476u64 stable_page_flags(struct page *page);
477
478static inline int PageUptodate(struct page *page)
479{
480 int ret;
481 page = compound_head(page);
482 ret = test_bit(PG_uptodate, &(page)->flags);
483
484
485
486
487
488
489
490
491 if (ret)
492 smp_rmb();
493
494 return ret;
495}
496
497static __always_inline void __SetPageUptodate(struct page *page)
498{
499 VM_BUG_ON_PAGE(PageTail(page), page);
500 smp_wmb();
501 __set_bit(PG_uptodate, &page->flags);
502}
503
504static __always_inline void SetPageUptodate(struct page *page)
505{
506 VM_BUG_ON_PAGE(PageTail(page), page);
507
508
509
510
511
512 smp_wmb();
513 set_bit(PG_uptodate, &page->flags);
514}
515
516CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
517
518int test_clear_page_writeback(struct page *page);
519int __test_set_page_writeback(struct page *page, bool keep_write);
520
521#define test_set_page_writeback(page) \
522 __test_set_page_writeback(page, false)
523#define test_set_page_writeback_keepwrite(page) \
524 __test_set_page_writeback(page, true)
525
526static inline void set_page_writeback(struct page *page)
527{
528 test_set_page_writeback(page);
529}
530
531static inline void set_page_writeback_keepwrite(struct page *page)
532{
533 test_set_page_writeback_keepwrite(page);
534}
535
536__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
537
538static __always_inline void set_compound_head(struct page *page, struct page *head)
539{
540 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
541}
542
543static __always_inline void clear_compound_head(struct page *page)
544{
545 WRITE_ONCE(page->compound_head, 0);
546}
547
548#ifdef CONFIG_TRANSPARENT_HUGEPAGE
549static inline void ClearPageCompound(struct page *page)
550{
551 BUG_ON(!PageHead(page));
552 ClearPageHead(page);
553}
554#endif
555
556#define PG_head_mask ((1UL << PG_head))
557
558#ifdef CONFIG_HUGETLB_PAGE
559int PageHuge(struct page *page);
560int PageHeadHuge(struct page *page);
561#else
562TESTPAGEFLAG_FALSE(Huge)
563TESTPAGEFLAG_FALSE(HeadHuge)
564#endif
565
566
567#ifdef CONFIG_TRANSPARENT_HUGEPAGE
568
569
570
571
572
573
574
575
576static inline int PageTransHuge(struct page *page)
577{
578 VM_BUG_ON_PAGE(PageTail(page), page);
579 return PageHead(page);
580}
581
582
583
584
585
586
587static inline int PageTransCompound(struct page *page)
588{
589 return PageCompound(page);
590}
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613static inline int PageTransCompoundMap(struct page *page)
614{
615 struct page *head;
616
617 if (!PageTransCompound(page))
618 return 0;
619
620 if (PageAnon(page))
621 return atomic_read(&page->_mapcount) < 0;
622
623 head = compound_head(page);
624
625 return atomic_read(&page->_mapcount) ==
626 atomic_read(compound_mapcount_ptr(head));
627}
628
629
630
631
632
633
634static inline int PageTransTail(struct page *page)
635{
636 return PageTail(page);
637}
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652static inline int PageDoubleMap(struct page *page)
653{
654 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
655}
656
657static inline void SetPageDoubleMap(struct page *page)
658{
659 VM_BUG_ON_PAGE(!PageHead(page), page);
660 set_bit(PG_double_map, &page[1].flags);
661}
662
663static inline void ClearPageDoubleMap(struct page *page)
664{
665 VM_BUG_ON_PAGE(!PageHead(page), page);
666 clear_bit(PG_double_map, &page[1].flags);
667}
668static inline int TestSetPageDoubleMap(struct page *page)
669{
670 VM_BUG_ON_PAGE(!PageHead(page), page);
671 return test_and_set_bit(PG_double_map, &page[1].flags);
672}
673
674static inline int TestClearPageDoubleMap(struct page *page)
675{
676 VM_BUG_ON_PAGE(!PageHead(page), page);
677 return test_and_clear_bit(PG_double_map, &page[1].flags);
678}
679
680#else
681TESTPAGEFLAG_FALSE(TransHuge)
682TESTPAGEFLAG_FALSE(TransCompound)
683TESTPAGEFLAG_FALSE(TransCompoundMap)
684TESTPAGEFLAG_FALSE(TransTail)
685PAGEFLAG_FALSE(DoubleMap)
686 TESTSETFLAG_FALSE(DoubleMap)
687 TESTCLEARFLAG_FALSE(DoubleMap)
688#endif
689
690
691
692
693
694
695
696
697
698
699#define PAGE_TYPE_BASE 0xf0000000
700
701#define PAGE_MAPCOUNT_RESERVE -128
702#define PG_buddy 0x00000080
703#define PG_offline 0x00000100
704#define PG_table 0x00000200
705#define PG_guard 0x00000400
706
707#define PageType(page, flag) \
708 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
709
710static inline int page_has_type(struct page *page)
711{
712 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;
713}
714
715#define PAGE_TYPE_OPS(uname, lname) \
716static __always_inline int Page##uname(struct page *page) \
717{ \
718 return PageType(page, PG_##lname); \
719} \
720static __always_inline void __SetPage##uname(struct page *page) \
721{ \
722 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
723 page->page_type &= ~PG_##lname; \
724} \
725static __always_inline void __ClearPage##uname(struct page *page) \
726{ \
727 VM_BUG_ON_PAGE(!Page##uname(page), page); \
728 page->page_type |= PG_##lname; \
729}
730
731
732
733
734
735PAGE_TYPE_OPS(Buddy, buddy)
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754PAGE_TYPE_OPS(Offline, offline)
755
756
757
758
759PAGE_TYPE_OPS(Table, table)
760
761
762
763
764PAGE_TYPE_OPS(Guard, guard)
765
766extern bool is_free_buddy_page(struct page *page);
767
768__PAGEFLAG(Isolated, isolated, PF_ANY);
769
770
771
772
773
774static inline int PageSlabPfmemalloc(struct page *page)
775{
776 VM_BUG_ON_PAGE(!PageSlab(page), page);
777 return PageActive(page);
778}
779
780static inline void SetPageSlabPfmemalloc(struct page *page)
781{
782 VM_BUG_ON_PAGE(!PageSlab(page), page);
783 SetPageActive(page);
784}
785
786static inline void __ClearPageSlabPfmemalloc(struct page *page)
787{
788 VM_BUG_ON_PAGE(!PageSlab(page), page);
789 __ClearPageActive(page);
790}
791
792static inline void ClearPageSlabPfmemalloc(struct page *page)
793{
794 VM_BUG_ON_PAGE(!PageSlab(page), page);
795 ClearPageActive(page);
796}
797
798#ifdef CONFIG_MMU
799#define __PG_MLOCKED (1UL << PG_mlocked)
800#else
801#define __PG_MLOCKED 0
802#endif
803
804
805
806
807
808#define PAGE_FLAGS_CHECK_AT_FREE \
809 (1UL << PG_lru | 1UL << PG_locked | \
810 1UL << PG_private | 1UL << PG_private_2 | \
811 1UL << PG_writeback | 1UL << PG_reserved | \
812 1UL << PG_slab | 1UL << PG_active | \
813 1UL << PG_unevictable | __PG_MLOCKED)
814
815
816
817
818
819
820
821
822
823#define PAGE_FLAGS_CHECK_AT_PREP \
824 (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
825
826#define PAGE_FLAGS_PRIVATE \
827 (1UL << PG_private | 1UL << PG_private_2)
828
829
830
831
832
833
834
835static inline int page_has_private(struct page *page)
836{
837 return !!(page->flags & PAGE_FLAGS_PRIVATE);
838}
839
840#undef PF_ANY
841#undef PF_HEAD
842#undef PF_ONLY_HEAD
843#undef PF_NO_TAIL
844#undef PF_NO_COMPOUND
845#endif
846
847#endif
848