1
2
3
4
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75enum pageflags {
76 PG_locked,
77 PG_error,
78 PG_referenced,
79 PG_uptodate,
80 PG_dirty,
81 PG_lru,
82 PG_active,
83 PG_waiters,
84 PG_slab,
85 PG_owner_priv_1,
86 PG_arch_1,
87 PG_reserved,
88 PG_private,
89 PG_private_2,
90 PG_writeback,
91 PG_head,
92 PG_mappedtodisk,
93 PG_reclaim,
94 PG_swapbacked,
95 PG_unevictable,
96#ifdef CONFIG_MMU
97 PG_mlocked,
98#endif
99#ifdef CONFIG_ARCH_USES_PG_UNCACHED
100 PG_uncached,
101#endif
102#ifdef CONFIG_MEMORY_FAILURE
103 PG_hwpoison,
104#endif
105#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
106 PG_young,
107 PG_idle,
108#endif
109 __NR_PAGEFLAGS,
110
111
112 PG_checked = PG_owner_priv_1,
113
114
115 PG_swapcache = PG_owner_priv_1,
116
117
118
119
120
121 PG_fscache = PG_private_2,
122
123
124
125 PG_pinned = PG_owner_priv_1,
126
127 PG_savepinned = PG_dirty,
128
129 PG_foreign = PG_owner_priv_1,
130
131
132 PG_slob_free = PG_private,
133
134
135 PG_double_map = PG_private_2,
136
137
138 PG_isolated = PG_reclaim,
139};
140
141#ifndef __GENERATING_BOUNDS_H
142
143struct page;
144
145static inline struct page *compound_head(struct page *page)
146{
147 unsigned long head = READ_ONCE(page->compound_head);
148
149 if (unlikely(head & 1))
150 return (struct page *) (head - 1);
151 return page;
152}
153
154static __always_inline int PageTail(struct page *page)
155{
156 return READ_ONCE(page->compound_head) & 1;
157}
158
159static __always_inline int PageCompound(struct page *page)
160{
161 return test_bit(PG_head, &page->flags) || PageTail(page);
162}
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184#define PF_ANY(page, enforce) page
185#define PF_HEAD(page, enforce) compound_head(page)
186#define PF_ONLY_HEAD(page, enforce) ({ \
187 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
188 page;})
189#define PF_NO_TAIL(page, enforce) ({ \
190 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
191 compound_head(page);})
192#define PF_NO_COMPOUND(page, enforce) ({ \
193 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
194 page;})
195
196
197
198
199#define TESTPAGEFLAG(uname, lname, policy) \
200static __always_inline int Page##uname(struct page *page) \
201 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
202
203#define SETPAGEFLAG(uname, lname, policy) \
204static __always_inline void SetPage##uname(struct page *page) \
205 { set_bit(PG_##lname, &policy(page, 1)->flags); }
206
207#define CLEARPAGEFLAG(uname, lname, policy) \
208static __always_inline void ClearPage##uname(struct page *page) \
209 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
210
211#define __SETPAGEFLAG(uname, lname, policy) \
212static __always_inline void __SetPage##uname(struct page *page) \
213 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
214
215#define __CLEARPAGEFLAG(uname, lname, policy) \
216static __always_inline void __ClearPage##uname(struct page *page) \
217 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
218
219#define TESTSETFLAG(uname, lname, policy) \
220static __always_inline int TestSetPage##uname(struct page *page) \
221 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
222
223#define TESTCLEARFLAG(uname, lname, policy) \
224static __always_inline int TestClearPage##uname(struct page *page) \
225 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
226
227#define PAGEFLAG(uname, lname, policy) \
228 TESTPAGEFLAG(uname, lname, policy) \
229 SETPAGEFLAG(uname, lname, policy) \
230 CLEARPAGEFLAG(uname, lname, policy)
231
232#define __PAGEFLAG(uname, lname, policy) \
233 TESTPAGEFLAG(uname, lname, policy) \
234 __SETPAGEFLAG(uname, lname, policy) \
235 __CLEARPAGEFLAG(uname, lname, policy)
236
237#define TESTSCFLAG(uname, lname, policy) \
238 TESTSETFLAG(uname, lname, policy) \
239 TESTCLEARFLAG(uname, lname, policy)
240
241#define TESTPAGEFLAG_FALSE(uname) \
242static inline int Page##uname(const struct page *page) { return 0; }
243
244#define SETPAGEFLAG_NOOP(uname) \
245static inline void SetPage##uname(struct page *page) { }
246
247#define CLEARPAGEFLAG_NOOP(uname) \
248static inline void ClearPage##uname(struct page *page) { }
249
250#define __CLEARPAGEFLAG_NOOP(uname) \
251static inline void __ClearPage##uname(struct page *page) { }
252
253#define TESTSETFLAG_FALSE(uname) \
254static inline int TestSetPage##uname(struct page *page) { return 0; }
255
256#define TESTCLEARFLAG_FALSE(uname) \
257static inline int TestClearPage##uname(struct page *page) { return 0; }
258
259#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
260 SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
261
262#define TESTSCFLAG_FALSE(uname) \
263 TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
264
265__PAGEFLAG(Locked, locked, PF_NO_TAIL)
266PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
267PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
268PAGEFLAG(Referenced, referenced, PF_HEAD)
269 TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
270 __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
271PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
272 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
273PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
274PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
275 TESTCLEARFLAG(Active, active, PF_HEAD)
276__PAGEFLAG(Slab, slab, PF_NO_TAIL)
277__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
278PAGEFLAG(Checked, checked, PF_NO_COMPOUND)
279
280
281PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
282 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
283PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
284PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
285
286PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
287 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
288PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
289 __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
290 __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
291
292
293
294
295
296
297PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
298 __CLEARPAGEFLAG(Private, private, PF_ANY)
299PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
300PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
301 TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
302
303
304
305
306
307TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
308 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
309PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
310
311
312PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
313 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
314PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
315 TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
316
317#ifdef CONFIG_HIGHMEM
318
319
320
321
322#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
323#else
324PAGEFLAG_FALSE(HighMem)
325#endif
326
327#ifdef CONFIG_SWAP
328static __always_inline int PageSwapCache(struct page *page)
329{
330#ifdef CONFIG_THP_SWAP
331 page = compound_head(page);
332#endif
333 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
334
335}
336SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
337CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
338#else
339PAGEFLAG_FALSE(SwapCache)
340#endif
341
342PAGEFLAG(Unevictable, unevictable, PF_HEAD)
343 __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
344 TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
345
346#ifdef CONFIG_MMU
347PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
348 __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
349 TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
350#else
351PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
352 TESTSCFLAG_FALSE(Mlocked)
353#endif
354
355#ifdef CONFIG_ARCH_USES_PG_UNCACHED
356PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
357#else
358PAGEFLAG_FALSE(Uncached)
359#endif
360
361#ifdef CONFIG_MEMORY_FAILURE
362PAGEFLAG(HWPoison, hwpoison, PF_ANY)
363TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
364#define __PG_HWPOISON (1UL << PG_hwpoison)
365#else
366PAGEFLAG_FALSE(HWPoison)
367#define __PG_HWPOISON 0
368#endif
369
370#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
371TESTPAGEFLAG(Young, young, PF_ANY)
372SETPAGEFLAG(Young, young, PF_ANY)
373TESTCLEARFLAG(Young, young, PF_ANY)
374PAGEFLAG(Idle, idle, PF_ANY)
375#endif
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394#define PAGE_MAPPING_ANON 0x1
395#define PAGE_MAPPING_MOVABLE 0x2
396#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
397#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
398
399static __always_inline int PageMappingFlags(struct page *page)
400{
401 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
402}
403
404static __always_inline int PageAnon(struct page *page)
405{
406 page = compound_head(page);
407 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
408}
409
410static __always_inline int __PageMovable(struct page *page)
411{
412 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
413 PAGE_MAPPING_MOVABLE;
414}
415
416#ifdef CONFIG_KSM
417
418
419
420
421
422
423static __always_inline int PageKsm(struct page *page)
424{
425 page = compound_head(page);
426 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
427 PAGE_MAPPING_KSM;
428}
429#else
430TESTPAGEFLAG_FALSE(Ksm)
431#endif
432
433u64 stable_page_flags(struct page *page);
434
435static inline int PageUptodate(struct page *page)
436{
437 int ret;
438 page = compound_head(page);
439 ret = test_bit(PG_uptodate, &(page)->flags);
440
441
442
443
444
445
446
447
448 if (ret)
449 smp_rmb();
450
451 return ret;
452}
453
454static __always_inline void __SetPageUptodate(struct page *page)
455{
456 VM_BUG_ON_PAGE(PageTail(page), page);
457 smp_wmb();
458 __set_bit(PG_uptodate, &page->flags);
459}
460
461static __always_inline void SetPageUptodate(struct page *page)
462{
463 VM_BUG_ON_PAGE(PageTail(page), page);
464
465
466
467
468
469 smp_wmb();
470 set_bit(PG_uptodate, &page->flags);
471}
472
473CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
474
475int test_clear_page_writeback(struct page *page);
476int __test_set_page_writeback(struct page *page, bool keep_write);
477
478#define test_set_page_writeback(page) \
479 __test_set_page_writeback(page, false)
480#define test_set_page_writeback_keepwrite(page) \
481 __test_set_page_writeback(page, true)
482
483static inline void set_page_writeback(struct page *page)
484{
485 test_set_page_writeback(page);
486}
487
488static inline void set_page_writeback_keepwrite(struct page *page)
489{
490 test_set_page_writeback_keepwrite(page);
491}
492
493__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
494
495static __always_inline void set_compound_head(struct page *page, struct page *head)
496{
497 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
498}
499
500static __always_inline void clear_compound_head(struct page *page)
501{
502 WRITE_ONCE(page->compound_head, 0);
503}
504
505#ifdef CONFIG_TRANSPARENT_HUGEPAGE
506static inline void ClearPageCompound(struct page *page)
507{
508 BUG_ON(!PageHead(page));
509 ClearPageHead(page);
510}
511#endif
512
513#define PG_head_mask ((1UL << PG_head))
514
515#ifdef CONFIG_HUGETLB_PAGE
516int PageHuge(struct page *page);
517int PageHeadHuge(struct page *page);
518bool page_huge_active(struct page *page);
519#else
520TESTPAGEFLAG_FALSE(Huge)
521TESTPAGEFLAG_FALSE(HeadHuge)
522
523static inline bool page_huge_active(struct page *page)
524{
525 return 0;
526}
527#endif
528
529
530#ifdef CONFIG_TRANSPARENT_HUGEPAGE
531
532
533
534
535
536
537
538
539static inline int PageTransHuge(struct page *page)
540{
541 VM_BUG_ON_PAGE(PageTail(page), page);
542 return PageHead(page);
543}
544
545
546
547
548
549
550static inline int PageTransCompound(struct page *page)
551{
552 return PageCompound(page);
553}
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571static inline int PageTransCompoundMap(struct page *page)
572{
573 return PageTransCompound(page) && atomic_read(&page->_mapcount) < 0;
574}
575
576
577
578
579
580
581static inline int PageTransTail(struct page *page)
582{
583 return PageTail(page);
584}
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599static inline int PageDoubleMap(struct page *page)
600{
601 return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
602}
603
604static inline void SetPageDoubleMap(struct page *page)
605{
606 VM_BUG_ON_PAGE(!PageHead(page), page);
607 set_bit(PG_double_map, &page[1].flags);
608}
609
610static inline void ClearPageDoubleMap(struct page *page)
611{
612 VM_BUG_ON_PAGE(!PageHead(page), page);
613 clear_bit(PG_double_map, &page[1].flags);
614}
615static inline int TestSetPageDoubleMap(struct page *page)
616{
617 VM_BUG_ON_PAGE(!PageHead(page), page);
618 return test_and_set_bit(PG_double_map, &page[1].flags);
619}
620
621static inline int TestClearPageDoubleMap(struct page *page)
622{
623 VM_BUG_ON_PAGE(!PageHead(page), page);
624 return test_and_clear_bit(PG_double_map, &page[1].flags);
625}
626
627#else
628TESTPAGEFLAG_FALSE(TransHuge)
629TESTPAGEFLAG_FALSE(TransCompound)
630TESTPAGEFLAG_FALSE(TransCompoundMap)
631TESTPAGEFLAG_FALSE(TransTail)
632PAGEFLAG_FALSE(DoubleMap)
633 TESTSETFLAG_FALSE(DoubleMap)
634 TESTCLEARFLAG_FALSE(DoubleMap)
635#endif
636
637
638
639
640
641
642
643
644#define PAGE_MAPCOUNT_OPS(uname, lname) \
645static __always_inline int Page##uname(struct page *page) \
646{ \
647 return atomic_read(&page->_mapcount) == \
648 PAGE_##lname##_MAPCOUNT_VALUE; \
649} \
650static __always_inline void __SetPage##uname(struct page *page) \
651{ \
652 VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page); \
653 atomic_set(&page->_mapcount, PAGE_##lname##_MAPCOUNT_VALUE); \
654} \
655static __always_inline void __ClearPage##uname(struct page *page) \
656{ \
657 VM_BUG_ON_PAGE(!Page##uname(page), page); \
658 atomic_set(&page->_mapcount, -1); \
659}
660
661
662
663
664
665#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
666PAGE_MAPCOUNT_OPS(Buddy, BUDDY)
667
668
669
670
671
672#define PAGE_BALLOON_MAPCOUNT_VALUE (-256)
673PAGE_MAPCOUNT_OPS(Balloon, BALLOON)
674
675
676
677
678
679#define PAGE_KMEMCG_MAPCOUNT_VALUE (-512)
680PAGE_MAPCOUNT_OPS(Kmemcg, KMEMCG)
681
682extern bool is_free_buddy_page(struct page *page);
683
684__PAGEFLAG(Isolated, isolated, PF_ANY);
685
686
687
688
689
690static inline int PageSlabPfmemalloc(struct page *page)
691{
692 VM_BUG_ON_PAGE(!PageSlab(page), page);
693 return PageActive(page);
694}
695
696static inline void SetPageSlabPfmemalloc(struct page *page)
697{
698 VM_BUG_ON_PAGE(!PageSlab(page), page);
699 SetPageActive(page);
700}
701
702static inline void __ClearPageSlabPfmemalloc(struct page *page)
703{
704 VM_BUG_ON_PAGE(!PageSlab(page), page);
705 __ClearPageActive(page);
706}
707
708static inline void ClearPageSlabPfmemalloc(struct page *page)
709{
710 VM_BUG_ON_PAGE(!PageSlab(page), page);
711 ClearPageActive(page);
712}
713
714#ifdef CONFIG_MMU
715#define __PG_MLOCKED (1UL << PG_mlocked)
716#else
717#define __PG_MLOCKED 0
718#endif
719
720
721
722
723
724#define PAGE_FLAGS_CHECK_AT_FREE \
725 (1UL << PG_lru | 1UL << PG_locked | \
726 1UL << PG_private | 1UL << PG_private_2 | \
727 1UL << PG_writeback | 1UL << PG_reserved | \
728 1UL << PG_slab | 1UL << PG_active | \
729 1UL << PG_unevictable | __PG_MLOCKED)
730
731
732
733
734
735
736
737
738
739#define PAGE_FLAGS_CHECK_AT_PREP \
740 (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
741
742#define PAGE_FLAGS_PRIVATE \
743 (1UL << PG_private | 1UL << PG_private_2)
744
745
746
747
748
749
750
751static inline int page_has_private(struct page *page)
752{
753 return !!(page->flags & PAGE_FLAGS_PRIVATE);
754}
755
756#undef PF_ANY
757#undef PF_HEAD
758#undef PF_ONLY_HEAD
759#undef PF_NO_TAIL
760#undef PF_NO_COMPOUND
761#endif
762
763#endif
764