1
2
3
4
5#ifndef PAGE_FLAGS_H
6#define PAGE_FLAGS_H
7
8#include <linux/types.h>
9#include <linux/bug.h>
10#include <linux/mmdebug.h>
11#ifndef __GENERATING_BOUNDS_H
12#include <linux/mm_types.h>
13#include <generated/bounds.h>
14#endif
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74enum pageflags {
75 PG_locked,
76 PG_error,
77 PG_referenced,
78 PG_uptodate,
79 PG_dirty,
80 PG_lru,
81 PG_active,
82 PG_slab,
83 PG_owner_priv_1,
84 PG_arch_1,
85 PG_reserved,
86 PG_private,
87 PG_private_2,
88 PG_writeback,
89 PG_head,
90 PG_swapcache,
91 PG_mappedtodisk,
92 PG_reclaim,
93 PG_swapbacked,
94 PG_unevictable,
95#ifdef CONFIG_MMU
96 PG_mlocked,
97#endif
98#ifdef CONFIG_ARCH_USES_PG_UNCACHED
99 PG_uncached,
100#endif
101#ifdef CONFIG_MEMORY_FAILURE
102 PG_hwpoison,
103#endif
104#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
105 PG_young,
106 PG_idle,
107#endif
108 __NR_PAGEFLAGS,
109
110
111 PG_checked = PG_owner_priv_1,
112
113
114
115
116
117 PG_fscache = PG_private_2,
118
119
120
121 PG_pinned = PG_owner_priv_1,
122
123 PG_savepinned = PG_dirty,
124
125 PG_foreign = PG_owner_priv_1,
126
127
128 PG_slob_free = PG_private,
129
130
131 PG_double_map = PG_private_2,
132
133
134 PG_isolated = PG_reclaim,
135};
136
137#ifndef __GENERATING_BOUNDS_H
138
139struct page;
140
141static inline struct page *compound_head(struct page *page)
142{
143 unsigned long head = READ_ONCE(page->compound_head);
144
145 if (unlikely(head & 1))
146 return (struct page *) (head - 1);
147 return page;
148}
149
150static __always_inline int PageTail(struct page *page)
151{
152 return READ_ONCE(page->compound_head) & 1;
153}
154
155static __always_inline int PageCompound(struct page *page)
156{
157 return test_bit(PG_head, &page->flags) || PageTail(page);
158}
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177#define PF_ANY(page, enforce) page
178#define PF_HEAD(page, enforce) compound_head(page)
179#define PF_NO_TAIL(page, enforce) ({ \
180 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
181 compound_head(page);})
182#define PF_NO_COMPOUND(page, enforce) ({ \
183 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
184 page;})
185
186
187
188
189#define TESTPAGEFLAG(uname, lname, policy) \
190static __always_inline int Page##uname(struct page *page) \
191 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
192
193#define SETPAGEFLAG(uname, lname, policy) \
194static __always_inline void SetPage##uname(struct page *page) \
195 { set_bit(PG_##lname, &policy(page, 1)->flags); }
196
197#define CLEARPAGEFLAG(uname, lname, policy) \
198static __always_inline void ClearPage##uname(struct page *page) \
199 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
200
201#define __SETPAGEFLAG(uname, lname, policy) \
202static __always_inline void __SetPage##uname(struct page *page) \
203 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
204
205#define __CLEARPAGEFLAG(uname, lname, policy) \
206static __always_inline void __ClearPage##uname(struct page *page) \
207 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
208
209#define TESTSETFLAG(uname, lname, policy) \
210static __always_inline int TestSetPage##uname(struct page *page) \
211 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
212
213#define TESTCLEARFLAG(uname, lname, policy) \
214static __always_inline int TestClearPage##uname(struct page *page) \
215 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
216
217#define PAGEFLAG(uname, lname, policy) \
218 TESTPAGEFLAG(uname, lname, policy) \
219 SETPAGEFLAG(uname, lname, policy) \
220 CLEARPAGEFLAG(uname, lname, policy)
221
222#define __PAGEFLAG(uname, lname, policy) \
223 TESTPAGEFLAG(uname, lname, policy) \
224 __SETPAGEFLAG(uname, lname, policy) \
225 __CLEARPAGEFLAG(uname, lname, policy)
226
227#define TESTSCFLAG(uname, lname, policy) \
228 TESTSETFLAG(uname, lname, policy) \
229 TESTCLEARFLAG(uname, lname, policy)
230
231#define TESTPAGEFLAG_FALSE(uname) \
232static inline int Page##uname(const struct page *page) { return 0; }
233
234#define SETPAGEFLAG_NOOP(uname) \
235static inline void SetPage##uname(struct page *page) { }
236
237#define CLEARPAGEFLAG_NOOP(uname) \
238static inline void ClearPage##uname(struct page *page) { }
239
240#define __CLEARPAGEFLAG_NOOP(uname) \
241static inline void __ClearPage##uname(struct page *page) { }
242
243#define TESTSETFLAG_FALSE(uname) \
244static inline int TestSetPage##uname(struct page *page) { return 0; }
245
246#define TESTCLEARFLAG_FALSE(uname) \
247static inline int TestClearPage##uname(struct page *page) { return 0; }
248
249#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
250 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
251
252#define TESTSCFLAG_FALSE(uname) \
253 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
254
255__PAGEFLAG(Locked, locked, PF_NO_TAIL)
256PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
257PAGEFLAG(Referenced, referenced, PF_HEAD)
258 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
259 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
260PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
261 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
262PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
263PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
264 TESTCLEARFLAG(Active, active, PF_HEAD)
265__PAGEFLAG(Slab, slab, PF_NO_TAIL)
266__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
267PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
268
269
270PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
271 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
272PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
273PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
274
275PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
276 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
277PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
278 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
279 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
280
281
282
283
284
285
286PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
287 __CLEARPAGEFLAG(Private, private, PF_ANY)
288PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
289PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
290 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
291
292
293
294
295
296TESTPAGEFLAG(Writeback, writeback, PF_NO_COMPOUND)
297 TESTSCFLAG(Writeback, writeback, PF_NO_COMPOUND)
298PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
299
300
301PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
302 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
303PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
304 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
305
306#ifdef CONFIG_HIGHMEM
307
308
309
310
311#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
312#else
313PAGEFLAG_FALSE(HighMem)
314#endif
315
316#ifdef CONFIG_SWAP
317PAGEFLAG(SwapCache, swapcache, PF_NO_COMPOUND)
318#else
319PAGEFLAG_FALSE(SwapCache)
320#endif
321
322PAGEFLAG(Unevictable, unevictable, PF_HEAD)
323 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
324 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
325
326#ifdef CONFIG_MMU
327PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
328 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
329 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
330#else
331PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
332 TESTSCFLAG_FALSE(Mlocked)
333#endif
334
335#ifdef CONFIG_ARCH_USES_PG_UNCACHED
336PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
337#else
338PAGEFLAG_FALSE(Uncached)
339#endif
340
341#ifdef CONFIG_MEMORY_FAILURE
342PAGEFLAG(HWPoison, hwpoison, PF_ANY)
343TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
344#define __PG_HWPOISON (1UL << PG_hwpoison)
345#else
346PAGEFLAG_FALSE(HWPoison)
347#define __PG_HWPOISON 0
348#endif
349
350#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
351TESTPAGEFLAG(Young, young, PF_ANY)
352SETPAGEFLAG(Young, young, PF_ANY)
353TESTCLEARFLAG(Young, young, PF_ANY)
354PAGEFLAG(Idle, idle, PF_ANY)
355#endif
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374#define PAGE_MAPPING_ANON 0x1
375#define PAGE_MAPPING_MOVABLE 0x2
376#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
377#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
378
379static __always_inline int PageMappingFlags(struct page *page)
380{
381 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
382}
383
384static __always_inline int PageAnon(struct page *page)
385{
386 page = compound_head(page);
387 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
388}
389
390static __always_inline int __PageMovable(struct page *page)
391{
392 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
393 PAGE_MAPPING_MOVABLE;
394}
395
396#ifdef CONFIG_KSM
397
398
399
400
401
402
403static __always_inline int PageKsm(struct page *page)
404{
405 page = compound_head(page);
406 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
407 PAGE_MAPPING_KSM;
408}
409#else
410TESTPAGEFLAG_FALSE(Ksm)
411#endif
412
413u64 stable_page_flags(struct page *page);
414
415static inline int PageUptodate(struct page *page)
416{
417 int ret;
418 page = compound_head(page);
419 ret = test_bit(PG_uptodate, &(page)->flags);
420
421
422
423
424
425
426
427
428 if (ret)
429 smp_rmb();
430
431 return ret;
432}
433
434static __always_inline void __SetPageUptodate(struct page *page)
435{
436 VM_BUG_ON_PAGE(PageTail(page), page);
437 smp_wmb();
438 __set_bit(PG_uptodate, &page->flags);
439}
440
441static __always_inline void SetPageUptodate(struct page *page)
442{
443 VM_BUG_ON_PAGE(PageTail(page), page);
444
445
446
447
448
449 smp_wmb();
450 set_bit(PG_uptodate, &page->flags);
451}
452
453CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
454
455int test_clear_page_writeback(struct page *page);
456int __test_set_page_writeback(struct page *page, bool keep_write);
457
458#define test_set_page_writeback(page) \
459 __test_set_page_writeback(page, false)
460#define test_set_page_writeback_keepwrite(page) \
461 __test_set_page_writeback(page, true)
462
463static inline void set_page_writeback(struct page *page)
464{
465 test_set_page_writeback(page);
466}
467
468static inline void set_page_writeback_keepwrite(struct page *page)
469{
470 test_set_page_writeback_keepwrite(page);
471}
472
473__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
474
475static __always_inline void set_compound_head(struct page *page, struct page *head)
476{
477 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
478}
479
480static __always_inline void clear_compound_head(struct page *page)
481{
482 WRITE_ONCE(page->compound_head, 0);
483}
484
485#ifdef CONFIG_TRANSPARENT_HUGEPAGE
486static inline void ClearPageCompound(struct page *page)
487{
488 BUG_ON(!PageHead(page));
489 ClearPageHead(page);
490}
491#endif
492
493#define PG_head_mask ((1UL << PG_head))
494
495#ifdef CONFIG_HUGETLB_PAGE
496int PageHuge(struct page *page);
497int PageHeadHuge(struct page *page);
498bool page_huge_active(struct page *page);
499#else
500TESTPAGEFLAG_FALSE(Huge)
501TESTPAGEFLAG_FALSE(HeadHuge)
502
503static inline bool page_huge_active(struct page *page)
504{
505 return 0;
506}
507#endif
508
509
510#ifdef CONFIG_TRANSPARENT_HUGEPAGE
511
512
513
514
515
516
517
518
519static inline int PageTransHuge(struct page *page)
520{
521 VM_BUG_ON_PAGE(PageTail(page), page);
522 return PageHead(page);
523}
524
525
526
527
528
529
530static inline int PageTransCompound(struct page *page)
531{
532 return PageCompound(page);
533}
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551static inline int PageTransCompoundMap(struct page *page)
552{
553 return PageTransCompound(page) && atomic_read(&page->_mapcount) < 0;
554}
555
556
557
558
559
560
561static inline int PageTransTail(struct page *page)
562{
563 return PageTail(page);
564}
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579static inline int PageDoubleMap(struct page *page)
580{
581 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
582}
583
584static inline void SetPageDoubleMap(struct page *page)
585{
586 VM_BUG_ON_PAGE(!PageHead(page), page);
587 set_bit(PG_double_map, &page[1].flags);
588}
589
590static inline void ClearPageDoubleMap(struct page *page)
591{
592 VM_BUG_ON_PAGE(!PageHead(page), page);
593 clear_bit(PG_double_map, &page[1].flags);
594}
595static inline int TestSetPageDoubleMap(struct page *page)
596{
597 VM_BUG_ON_PAGE(!PageHead(page), page);
598 return test_and_set_bit(PG_double_map, &page[1].flags);
599}
600
601static inline int TestClearPageDoubleMap(struct page *page)
602{
603 VM_BUG_ON_PAGE(!PageHead(page), page);
604 return test_and_clear_bit(PG_double_map, &page[1].flags);
605}
606
607#else
608TESTPAGEFLAG_FALSE(TransHuge)
609TESTPAGEFLAG_FALSE(TransCompound)
610TESTPAGEFLAG_FALSE(TransCompoundMap)
611TESTPAGEFLAG_FALSE(TransTail)
612PAGEFLAG_FALSE(DoubleMap)
613 TESTSETFLAG_FALSE(DoubleMap)
614 TESTCLEARFLAG_FALSE(DoubleMap)
615#endif
616
617
618
619
620
621
622
623
624#define PAGE_MAPCOUNT_OPS(uname, lname) \
625static __always_inline int Page##uname(struct page *page) \
626{ \
627 return atomic_read(&page->_mapcount) == \
628 PAGE_##lname##_MAPCOUNT_VALUE; \
629} \
630static __always_inline void __SetPage##uname(struct page *page) \
631{ \
632 VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page); \
633 atomic_set(&page->_mapcount, PAGE_##lname##_MAPCOUNT_VALUE); \
634} \
635static __always_inline void __ClearPage##uname(struct page *page) \
636{ \
637 VM_BUG_ON_PAGE(!Page##uname(page), page); \
638 atomic_set(&page->_mapcount, -1); \
639}
640
641
642
643
644
645#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
646PAGE_MAPCOUNT_OPS(Buddy, BUDDY)
647
648
649
650
651
652#define PAGE_BALLOON_MAPCOUNT_VALUE (-256)
653PAGE_MAPCOUNT_OPS(Balloon, BALLOON)
654
655
656
657
658
659#define PAGE_KMEMCG_MAPCOUNT_VALUE (-512)
660PAGE_MAPCOUNT_OPS(Kmemcg, KMEMCG)
661
662extern bool is_free_buddy_page(struct page *page);
663
664__PAGEFLAG(Isolated, isolated, PF_ANY);
665
666
667
668
669
670static inline int PageSlabPfmemalloc(struct page *page)
671{
672 VM_BUG_ON_PAGE(!PageSlab(page), page);
673 return PageActive(page);
674}
675
676static inline void SetPageSlabPfmemalloc(struct page *page)
677{
678 VM_BUG_ON_PAGE(!PageSlab(page), page);
679 SetPageActive(page);
680}
681
682static inline void __ClearPageSlabPfmemalloc(struct page *page)
683{
684 VM_BUG_ON_PAGE(!PageSlab(page), page);
685 __ClearPageActive(page);
686}
687
688static inline void ClearPageSlabPfmemalloc(struct page *page)
689{
690 VM_BUG_ON_PAGE(!PageSlab(page), page);
691 ClearPageActive(page);
692}
693
694#ifdef CONFIG_MMU
695#define __PG_MLOCKED (1UL << PG_mlocked)
696#else
697#define __PG_MLOCKED 0
698#endif
699
700
701
702
703
704#define PAGE_FLAGS_CHECK_AT_FREE \
705 (1UL << PG_lru | 1UL << PG_locked | \
706 1UL << PG_private | 1UL << PG_private_2 | \
707 1UL << PG_writeback | 1UL << PG_reserved | \
708 1UL << PG_slab | 1UL << PG_swapcache | 1UL << PG_active | \
709 1UL << PG_unevictable | __PG_MLOCKED)
710
711
712
713
714
715
716
717
718
719#define PAGE_FLAGS_CHECK_AT_PREP \
720 (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
721
722#define PAGE_FLAGS_PRIVATE \
723 (1UL << PG_private | 1UL << PG_private_2)
724
725
726
727
728
729
730
731static inline int page_has_private(struct page *page)
732{
733 return !!(page->flags & PAGE_FLAGS_PRIVATE);
734}
735
736#undef PF_ANY
737#undef PF_HEAD
738#undef PF_NO_TAIL
739#undef PF_NO_COMPOUND
740#endif
741
742#endif
743