1
2
3
4
5#ifndef PAGE_FLAGS_H
6#define PAGE_FLAGS_H
7
8#include <linux/types.h>
9#include <linux/bug.h>
10#include <linux/mmdebug.h>
11#ifndef __GENERATING_BOUNDS_H
12#include <linux/mm_types.h>
13#include <generated/bounds.h>
14#endif
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74enum pageflags {
75 PG_locked,
76 PG_error,
77 PG_referenced,
78 PG_uptodate,
79 PG_dirty,
80 PG_lru,
81 PG_active,
82 PG_slab,
83 PG_owner_priv_1,
84 PG_arch_1,
85 PG_reserved,
86 PG_private,
87 PG_private_2,
88 PG_writeback,
89 PG_head,
90 PG_swapcache,
91 PG_mappedtodisk,
92 PG_reclaim,
93 PG_swapbacked,
94 PG_unevictable,
95#ifdef CONFIG_MMU
96 PG_mlocked,
97#endif
98#ifdef CONFIG_ARCH_USES_PG_UNCACHED
99 PG_uncached,
100#endif
101#ifdef CONFIG_MEMORY_FAILURE
102 PG_hwpoison,
103#endif
104#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
105 PG_young,
106 PG_idle,
107#endif
108 __NR_PAGEFLAGS,
109
110
111 PG_checked = PG_owner_priv_1,
112
113
114
115
116
117 PG_fscache = PG_private_2,
118
119
120
121 PG_pinned = PG_owner_priv_1,
122
123 PG_savepinned = PG_dirty,
124
125 PG_foreign = PG_owner_priv_1,
126
127
128 PG_slob_free = PG_private,
129
130
131 PG_double_map = PG_private_2,
132};
133
134#ifndef __GENERATING_BOUNDS_H
135
136struct page;
137
138static inline struct page *compound_head(struct page *page)
139{
140 unsigned long head = READ_ONCE(page->compound_head);
141
142 if (unlikely(head & 1))
143 return (struct page *) (head - 1);
144 return page;
145}
146
147static __always_inline int PageTail(struct page *page)
148{
149 return READ_ONCE(page->compound_head) & 1;
150}
151
152static __always_inline int PageCompound(struct page *page)
153{
154 return test_bit(PG_head, &page->flags) || PageTail(page);
155}
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174#define PF_ANY(page, enforce) page
175#define PF_HEAD(page, enforce) compound_head(page)
176#define PF_NO_TAIL(page, enforce) ({ \
177 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
178 compound_head(page);})
179#define PF_NO_COMPOUND(page, enforce) ({ \
180 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
181 page;})
182
183
184
185
186#define TESTPAGEFLAG(uname, lname, policy) \
187static __always_inline int Page##uname(struct page *page) \
188 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
189
190#define SETPAGEFLAG(uname, lname, policy) \
191static __always_inline void SetPage##uname(struct page *page) \
192 { set_bit(PG_##lname, &policy(page, 1)->flags); }
193
194#define CLEARPAGEFLAG(uname, lname, policy) \
195static __always_inline void ClearPage##uname(struct page *page) \
196 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
197
198#define __SETPAGEFLAG(uname, lname, policy) \
199static __always_inline void __SetPage##uname(struct page *page) \
200 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
201
202#define __CLEARPAGEFLAG(uname, lname, policy) \
203static __always_inline void __ClearPage##uname(struct page *page) \
204 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
205
206#define TESTSETFLAG(uname, lname, policy) \
207static __always_inline int TestSetPage##uname(struct page *page) \
208 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
209
210#define TESTCLEARFLAG(uname, lname, policy) \
211static __always_inline int TestClearPage##uname(struct page *page) \
212 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
213
214#define PAGEFLAG(uname, lname, policy) \
215 TESTPAGEFLAG(uname, lname, policy) \
216 SETPAGEFLAG(uname, lname, policy) \
217 CLEARPAGEFLAG(uname, lname, policy)
218
219#define __PAGEFLAG(uname, lname, policy) \
220 TESTPAGEFLAG(uname, lname, policy) \
221 __SETPAGEFLAG(uname, lname, policy) \
222 __CLEARPAGEFLAG(uname, lname, policy)
223
224#define TESTSCFLAG(uname, lname, policy) \
225 TESTSETFLAG(uname, lname, policy) \
226 TESTCLEARFLAG(uname, lname, policy)
227
228#define TESTPAGEFLAG_FALSE(uname) \
229static inline int Page##uname(const struct page *page) { return 0; }
230
231#define SETPAGEFLAG_NOOP(uname) \
232static inline void SetPage##uname(struct page *page) { }
233
234#define CLEARPAGEFLAG_NOOP(uname) \
235static inline void ClearPage##uname(struct page *page) { }
236
237#define __CLEARPAGEFLAG_NOOP(uname) \
238static inline void __ClearPage##uname(struct page *page) { }
239
240#define TESTSETFLAG_FALSE(uname) \
241static inline int TestSetPage##uname(struct page *page) { return 0; }
242
243#define TESTCLEARFLAG_FALSE(uname) \
244static inline int TestClearPage##uname(struct page *page) { return 0; }
245
246#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
247 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
248
249#define TESTSCFLAG_FALSE(uname) \
250 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
251
252__PAGEFLAG(Locked, locked, PF_NO_TAIL)
253PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
254PAGEFLAG(Referenced, referenced, PF_HEAD)
255 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
256 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
257PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
258 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
259PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
260PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
261 TESTCLEARFLAG(Active, active, PF_HEAD)
262__PAGEFLAG(Slab, slab, PF_NO_TAIL)
263__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
264PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
265
266
267PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
268 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
269PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
270PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
271
272PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
273 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
274PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
275 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
276 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
277
278
279
280
281
282
283PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
284 __CLEARPAGEFLAG(Private, private, PF_ANY)
285PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
286PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
287 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
288
289
290
291
292
293TESTPAGEFLAG(Writeback, writeback, PF_NO_COMPOUND)
294 TESTSCFLAG(Writeback, writeback, PF_NO_COMPOUND)
295PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_COMPOUND)
296
297
298PAGEFLAG(Reclaim, reclaim, PF_NO_COMPOUND)
299 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_COMPOUND)
300PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
301 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
302
303#ifdef CONFIG_HIGHMEM
304
305
306
307
308#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
309#else
310PAGEFLAG_FALSE(HighMem)
311#endif
312
313#ifdef CONFIG_SWAP
314PAGEFLAG(SwapCache, swapcache, PF_NO_COMPOUND)
315#else
316PAGEFLAG_FALSE(SwapCache)
317#endif
318
319PAGEFLAG(Unevictable, unevictable, PF_HEAD)
320 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
321 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
322
323#ifdef CONFIG_MMU
324PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
325 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
326 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
327#else
328PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
329 TESTSCFLAG_FALSE(Mlocked)
330#endif
331
332#ifdef CONFIG_ARCH_USES_PG_UNCACHED
333PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
334#else
335PAGEFLAG_FALSE(Uncached)
336#endif
337
338#ifdef CONFIG_MEMORY_FAILURE
339PAGEFLAG(HWPoison, hwpoison, PF_ANY)
340TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
341#define __PG_HWPOISON (1UL << PG_hwpoison)
342#else
343PAGEFLAG_FALSE(HWPoison)
344#define __PG_HWPOISON 0
345#endif
346
347#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
348TESTPAGEFLAG(Young, young, PF_ANY)
349SETPAGEFLAG(Young, young, PF_ANY)
350TESTCLEARFLAG(Young, young, PF_ANY)
351PAGEFLAG(Idle, idle, PF_ANY)
352#endif
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370#define PAGE_MAPPING_ANON 1
371#define PAGE_MAPPING_KSM 2
372#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
373
374static __always_inline int PageAnon(struct page *page)
375{
376 page = compound_head(page);
377 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
378}
379
380#ifdef CONFIG_KSM
381
382
383
384
385
386
387static __always_inline int PageKsm(struct page *page)
388{
389 page = compound_head(page);
390 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
391 (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
392}
393#else
394TESTPAGEFLAG_FALSE(Ksm)
395#endif
396
397u64 stable_page_flags(struct page *page);
398
399static inline int PageUptodate(struct page *page)
400{
401 int ret;
402 page = compound_head(page);
403 ret = test_bit(PG_uptodate, &(page)->flags);
404
405
406
407
408
409
410
411
412 if (ret)
413 smp_rmb();
414
415 return ret;
416}
417
418static __always_inline void __SetPageUptodate(struct page *page)
419{
420 VM_BUG_ON_PAGE(PageTail(page), page);
421 smp_wmb();
422 __set_bit(PG_uptodate, &page->flags);
423}
424
425static __always_inline void SetPageUptodate(struct page *page)
426{
427 VM_BUG_ON_PAGE(PageTail(page), page);
428
429
430
431
432
433 smp_wmb();
434 set_bit(PG_uptodate, &page->flags);
435}
436
437CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
438
439int test_clear_page_writeback(struct page *page);
440int __test_set_page_writeback(struct page *page, bool keep_write);
441
442#define test_set_page_writeback(page) \
443 __test_set_page_writeback(page, false)
444#define test_set_page_writeback_keepwrite(page) \
445 __test_set_page_writeback(page, true)
446
447static inline void set_page_writeback(struct page *page)
448{
449 test_set_page_writeback(page);
450}
451
452static inline void set_page_writeback_keepwrite(struct page *page)
453{
454 test_set_page_writeback_keepwrite(page);
455}
456
457__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
458
459static __always_inline void set_compound_head(struct page *page, struct page *head)
460{
461 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
462}
463
464static __always_inline void clear_compound_head(struct page *page)
465{
466 WRITE_ONCE(page->compound_head, 0);
467}
468
469#ifdef CONFIG_TRANSPARENT_HUGEPAGE
470static inline void ClearPageCompound(struct page *page)
471{
472 BUG_ON(!PageHead(page));
473 ClearPageHead(page);
474}
475#endif
476
477#define PG_head_mask ((1L << PG_head))
478
479#ifdef CONFIG_HUGETLB_PAGE
480int PageHuge(struct page *page);
481int PageHeadHuge(struct page *page);
482bool page_huge_active(struct page *page);
483#else
484TESTPAGEFLAG_FALSE(Huge)
485TESTPAGEFLAG_FALSE(HeadHuge)
486
487static inline bool page_huge_active(struct page *page)
488{
489 return 0;
490}
491#endif
492
493
494#ifdef CONFIG_TRANSPARENT_HUGEPAGE
495
496
497
498
499
500
501
502
503static inline int PageTransHuge(struct page *page)
504{
505 VM_BUG_ON_PAGE(PageTail(page), page);
506 return PageHead(page);
507}
508
509
510
511
512
513
514static inline int PageTransCompound(struct page *page)
515{
516 return PageCompound(page);
517}
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535static inline int PageTransCompoundMap(struct page *page)
536{
537 return PageTransCompound(page) && atomic_read(&page->_mapcount) < 0;
538}
539
540
541
542
543
544
545static inline int PageTransTail(struct page *page)
546{
547 return PageTail(page);
548}
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563static inline int PageDoubleMap(struct page *page)
564{
565 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
566}
567
568static inline int TestSetPageDoubleMap(struct page *page)
569{
570 VM_BUG_ON_PAGE(!PageHead(page), page);
571 return test_and_set_bit(PG_double_map, &page[1].flags);
572}
573
574static inline int TestClearPageDoubleMap(struct page *page)
575{
576 VM_BUG_ON_PAGE(!PageHead(page), page);
577 return test_and_clear_bit(PG_double_map, &page[1].flags);
578}
579
580#else
581TESTPAGEFLAG_FALSE(TransHuge)
582TESTPAGEFLAG_FALSE(TransCompound)
583TESTPAGEFLAG_FALSE(TransCompoundMap)
584TESTPAGEFLAG_FALSE(TransTail)
585TESTPAGEFLAG_FALSE(DoubleMap)
586 TESTSETFLAG_FALSE(DoubleMap)
587 TESTCLEARFLAG_FALSE(DoubleMap)
588#endif
589
590
591
592
593
594
595
596
597
598
599#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
600
601static inline int PageBuddy(struct page *page)
602{
603 return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;
604}
605
606static inline void __SetPageBuddy(struct page *page)
607{
608 VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
609 atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
610}
611
612static inline void __ClearPageBuddy(struct page *page)
613{
614 VM_BUG_ON_PAGE(!PageBuddy(page), page);
615 atomic_set(&page->_mapcount, -1);
616}
617
618extern bool is_free_buddy_page(struct page *page);
619
620#define PAGE_BALLOON_MAPCOUNT_VALUE (-256)
621
622static inline int PageBalloon(struct page *page)
623{
624 return atomic_read(&page->_mapcount) == PAGE_BALLOON_MAPCOUNT_VALUE;
625}
626
627static inline void __SetPageBalloon(struct page *page)
628{
629 VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
630 atomic_set(&page->_mapcount, PAGE_BALLOON_MAPCOUNT_VALUE);
631}
632
633static inline void __ClearPageBalloon(struct page *page)
634{
635 VM_BUG_ON_PAGE(!PageBalloon(page), page);
636 atomic_set(&page->_mapcount, -1);
637}
638
639
640
641
642
643static inline int PageSlabPfmemalloc(struct page *page)
644{
645 VM_BUG_ON_PAGE(!PageSlab(page), page);
646 return PageActive(page);
647}
648
649static inline void SetPageSlabPfmemalloc(struct page *page)
650{
651 VM_BUG_ON_PAGE(!PageSlab(page), page);
652 SetPageActive(page);
653}
654
655static inline void __ClearPageSlabPfmemalloc(struct page *page)
656{
657 VM_BUG_ON_PAGE(!PageSlab(page), page);
658 __ClearPageActive(page);
659}
660
661static inline void ClearPageSlabPfmemalloc(struct page *page)
662{
663 VM_BUG_ON_PAGE(!PageSlab(page), page);
664 ClearPageActive(page);
665}
666
667#ifdef CONFIG_MMU
668#define __PG_MLOCKED (1 << PG_mlocked)
669#else
670#define __PG_MLOCKED 0
671#endif
672
673
674
675
676
677#define PAGE_FLAGS_CHECK_AT_FREE \
678 (1 << PG_lru | 1 << PG_locked | \
679 1 << PG_private | 1 << PG_private_2 | \
680 1 << PG_writeback | 1 << PG_reserved | \
681 1 << PG_slab | 1 << PG_swapcache | 1 << PG_active | \
682 1 << PG_unevictable | __PG_MLOCKED)
683
684
685
686
687
688
689
690
691
692#define PAGE_FLAGS_CHECK_AT_PREP \
693 (((1 << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
694
695#define PAGE_FLAGS_PRIVATE \
696 (1 << PG_private | 1 << PG_private_2)
697
698
699
700
701
702
703
704static inline int page_has_private(struct page *page)
705{
706 return !!(page->flags & PAGE_FLAGS_PRIVATE);
707}
708
709#undef PF_ANY
710#undef PF_HEAD
711#undef PF_NO_TAIL
712#undef PF_NO_COMPOUND
713#endif
714
715#endif
716