1
2#ifndef _LINUX_PGTABLE_H
3#define _LINUX_PGTABLE_H
4
5#include <linux/pfn.h>
6#include <asm/pgtable.h>
7
8#ifndef __ASSEMBLY__
9#ifdef CONFIG_MMU
10
11#include <linux/mm_types.h>
12#include <linux/bug.h>
13#include <linux/errno.h>
14#include <asm-generic/pgtable_uffd.h>
15
16#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \
17 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS
18#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED
19#endif
20
21
22
23
24
25
26
27#ifndef USER_PGTABLES_CEILING
28#define USER_PGTABLES_CEILING 0UL
29#endif
30
31
32
33
34
35
36#ifndef FIRST_USER_ADDRESS
37#define FIRST_USER_ADDRESS 0UL
38#endif
39
40
41
42
43
44
45#ifndef pmd_pgtable
46#define pmd_pgtable(pmd) pmd_page(pmd)
47#endif
48
49
50
51
52
53
54
55
56
57
58
59
60
61static inline unsigned long pte_index(unsigned long address)
62{
63 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
64}
65#define pte_index pte_index
66
67#ifndef pmd_index
68static inline unsigned long pmd_index(unsigned long address)
69{
70 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
71}
72#define pmd_index pmd_index
73#endif
74
75#ifndef pud_index
76static inline unsigned long pud_index(unsigned long address)
77{
78 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
79}
80#define pud_index pud_index
81#endif
82
83#ifndef pgd_index
84
85#define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
86#endif
87
88#ifndef pte_offset_kernel
89static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
90{
91 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
92}
93#define pte_offset_kernel pte_offset_kernel
94#endif
95
96#if defined(CONFIG_HIGHPTE)
97#define pte_offset_map(dir, address) \
98 ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \
99 pte_index((address)))
100#define pte_unmap(pte) kunmap_atomic((pte))
101#else
102#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
103#define pte_unmap(pte) ((void)(pte))
104#endif
105
106
107#ifndef pmd_offset
108static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
109{
110 return pud_pgtable(*pud) + pmd_index(address);
111}
112#define pmd_offset pmd_offset
113#endif
114
115#ifndef pud_offset
116static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
117{
118 return p4d_pgtable(*p4d) + pud_index(address);
119}
120#define pud_offset pud_offset
121#endif
122
123static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address)
124{
125 return (pgd + pgd_index(address));
126};
127
128
129
130
131#ifndef pgd_offset
132#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
133#endif
134
135
136
137
138
139#ifndef pgd_offset_k
140#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
141#endif
142
143
144
145
146
147
148
149
150static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va)
151{
152 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va);
153}
154
155static inline pmd_t *pmd_off_k(unsigned long va)
156{
157 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va);
158}
159
160static inline pte_t *virt_to_kpte(unsigned long vaddr)
161{
162 pmd_t *pmd = pmd_off_k(vaddr);
163
164 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr);
165}
166
167#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
168extern int ptep_set_access_flags(struct vm_area_struct *vma,
169 unsigned long address, pte_t *ptep,
170 pte_t entry, int dirty);
171#endif
172
173#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
174#ifdef CONFIG_TRANSPARENT_HUGEPAGE
175extern int pmdp_set_access_flags(struct vm_area_struct *vma,
176 unsigned long address, pmd_t *pmdp,
177 pmd_t entry, int dirty);
178extern int pudp_set_access_flags(struct vm_area_struct *vma,
179 unsigned long address, pud_t *pudp,
180 pud_t entry, int dirty);
181#else
182static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
183 unsigned long address, pmd_t *pmdp,
184 pmd_t entry, int dirty)
185{
186 BUILD_BUG();
187 return 0;
188}
189static inline int pudp_set_access_flags(struct vm_area_struct *vma,
190 unsigned long address, pud_t *pudp,
191 pud_t entry, int dirty)
192{
193 BUILD_BUG();
194 return 0;
195}
196#endif
197#endif
198
199#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
200static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
201 unsigned long address,
202 pte_t *ptep)
203{
204 pte_t pte = *ptep;
205 int r = 1;
206 if (!pte_young(pte))
207 r = 0;
208 else
209 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte));
210 return r;
211}
212#endif
213
214#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
215#ifdef CONFIG_TRANSPARENT_HUGEPAGE
216static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
217 unsigned long address,
218 pmd_t *pmdp)
219{
220 pmd_t pmd = *pmdp;
221 int r = 1;
222 if (!pmd_young(pmd))
223 r = 0;
224 else
225 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));
226 return r;
227}
228#else
229static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
230 unsigned long address,
231 pmd_t *pmdp)
232{
233 BUILD_BUG();
234 return 0;
235}
236#endif
237#endif
238
239#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
240int ptep_clear_flush_young(struct vm_area_struct *vma,
241 unsigned long address, pte_t *ptep);
242#endif
243
244#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
245#ifdef CONFIG_TRANSPARENT_HUGEPAGE
246extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
247 unsigned long address, pmd_t *pmdp);
248#else
249
250
251
252
253static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
254 unsigned long address, pmd_t *pmdp)
255{
256 BUILD_BUG();
257 return 0;
258}
259#endif
260#endif
261
262#ifndef __HAVE_ARCH_PTEP_CLEAR
263static inline void ptep_clear(struct mm_struct *mm, unsigned long addr,
264 pte_t *ptep)
265{
266 pte_clear(mm, addr, ptep);
267}
268#endif
269
270#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
271static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
272 unsigned long address,
273 pte_t *ptep)
274{
275 pte_t pte = *ptep;
276 pte_clear(mm, address, ptep);
277 return pte;
278}
279#endif
280
281#ifndef __HAVE_ARCH_PTEP_GET
282static inline pte_t ptep_get(pte_t *ptep)
283{
284 return READ_ONCE(*ptep);
285}
286#endif
287
288#ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320static inline pte_t ptep_get_lockless(pte_t *ptep)
321{
322 pte_t pte;
323
324 do {
325 pte.pte_low = ptep->pte_low;
326 smp_rmb();
327 pte.pte_high = ptep->pte_high;
328 smp_rmb();
329 } while (unlikely(pte.pte_low != ptep->pte_low));
330
331 return pte;
332}
333#else
334
335
336
337static inline pte_t ptep_get_lockless(pte_t *ptep)
338{
339 return ptep_get(ptep);
340}
341#endif
342
343#ifdef CONFIG_TRANSPARENT_HUGEPAGE
344#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
345static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
346 unsigned long address,
347 pmd_t *pmdp)
348{
349 pmd_t pmd = *pmdp;
350 pmd_clear(pmdp);
351 return pmd;
352}
353#endif
354#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
355static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
356 unsigned long address,
357 pud_t *pudp)
358{
359 pud_t pud = *pudp;
360
361 pud_clear(pudp);
362 return pud;
363}
364#endif
365#endif
366
367#ifdef CONFIG_TRANSPARENT_HUGEPAGE
368#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
369static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
370 unsigned long address, pmd_t *pmdp,
371 int full)
372{
373 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
374}
375#endif
376
377#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL
378static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm,
379 unsigned long address, pud_t *pudp,
380 int full)
381{
382 return pudp_huge_get_and_clear(mm, address, pudp);
383}
384#endif
385#endif
386
387#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
388static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
389 unsigned long address, pte_t *ptep,
390 int full)
391{
392 pte_t pte;
393 pte = ptep_get_and_clear(mm, address, ptep);
394 return pte;
395}
396#endif
397
398
399
400
401
402
403
404
405
406
407#ifndef __HAVE_ARCH_UPDATE_MMU_TLB
408static inline void update_mmu_tlb(struct vm_area_struct *vma,
409 unsigned long address, pte_t *ptep)
410{
411}
412#define __HAVE_ARCH_UPDATE_MMU_TLB
413#endif
414
415
416
417
418
419
420#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
421static inline void pte_clear_not_present_full(struct mm_struct *mm,
422 unsigned long address,
423 pte_t *ptep,
424 int full)
425{
426 pte_clear(mm, address, ptep);
427}
428#endif
429
430#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
431extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
432 unsigned long address,
433 pte_t *ptep);
434#endif
435
436#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
437extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
438 unsigned long address,
439 pmd_t *pmdp);
440extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma,
441 unsigned long address,
442 pud_t *pudp);
443#endif
444
445#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
446struct mm_struct;
447static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
448{
449 pte_t old_pte = *ptep;
450 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
451}
452#endif
453
454
455
456
457
458
459
460
461
462#ifndef pte_sw_mkyoung
463static inline pte_t pte_sw_mkyoung(pte_t pte)
464{
465 return pte;
466}
467#define pte_sw_mkyoung pte_sw_mkyoung
468#endif
469
470#ifndef pte_savedwrite
471#define pte_savedwrite pte_write
472#endif
473
474#ifndef pte_mk_savedwrite
475#define pte_mk_savedwrite pte_mkwrite
476#endif
477
478#ifndef pte_clear_savedwrite
479#define pte_clear_savedwrite pte_wrprotect
480#endif
481
482#ifndef pmd_savedwrite
483#define pmd_savedwrite pmd_write
484#endif
485
486#ifndef pmd_mk_savedwrite
487#define pmd_mk_savedwrite pmd_mkwrite
488#endif
489
490#ifndef pmd_clear_savedwrite
491#define pmd_clear_savedwrite pmd_wrprotect
492#endif
493
494#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT
495#ifdef CONFIG_TRANSPARENT_HUGEPAGE
496static inline void pmdp_set_wrprotect(struct mm_struct *mm,
497 unsigned long address, pmd_t *pmdp)
498{
499 pmd_t old_pmd = *pmdp;
500 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));
501}
502#else
503static inline void pmdp_set_wrprotect(struct mm_struct *mm,
504 unsigned long address, pmd_t *pmdp)
505{
506 BUILD_BUG();
507}
508#endif
509#endif
510#ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT
511#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
512static inline void pudp_set_wrprotect(struct mm_struct *mm,
513 unsigned long address, pud_t *pudp)
514{
515 pud_t old_pud = *pudp;
516
517 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud));
518}
519#else
520static inline void pudp_set_wrprotect(struct mm_struct *mm,
521 unsigned long address, pud_t *pudp)
522{
523 BUILD_BUG();
524}
525#endif
526#endif
527
528#ifndef pmdp_collapse_flush
529#ifdef CONFIG_TRANSPARENT_HUGEPAGE
530extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
531 unsigned long address, pmd_t *pmdp);
532#else
533static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
534 unsigned long address,
535 pmd_t *pmdp)
536{
537 BUILD_BUG();
538 return *pmdp;
539}
540#define pmdp_collapse_flush pmdp_collapse_flush
541#endif
542#endif
543
544#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
545extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
546 pgtable_t pgtable);
547#endif
548
549#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
550extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
551#endif
552
553#ifdef CONFIG_TRANSPARENT_HUGEPAGE
554
555
556
557
558
559static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma,
560 unsigned long address, pmd_t *pmdp, pmd_t pmd)
561{
562 pmd_t old_pmd = *pmdp;
563 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
564 return old_pmd;
565}
566#endif
567
568#ifndef __HAVE_ARCH_PMDP_INVALIDATE
569extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
570 pmd_t *pmdp);
571#endif
572
573#ifndef __HAVE_ARCH_PTE_SAME
574static inline int pte_same(pte_t pte_a, pte_t pte_b)
575{
576 return pte_val(pte_a) == pte_val(pte_b);
577}
578#endif
579
580#ifndef __HAVE_ARCH_PTE_UNUSED
581
582
583
584
585
586
587static inline int pte_unused(pte_t pte)
588{
589 return 0;
590}
591#endif
592
593#ifndef pte_access_permitted
594#define pte_access_permitted(pte, write) \
595 (pte_present(pte) && (!(write) || pte_write(pte)))
596#endif
597
598#ifndef pmd_access_permitted
599#define pmd_access_permitted(pmd, write) \
600 (pmd_present(pmd) && (!(write) || pmd_write(pmd)))
601#endif
602
603#ifndef pud_access_permitted
604#define pud_access_permitted(pud, write) \
605 (pud_present(pud) && (!(write) || pud_write(pud)))
606#endif
607
608#ifndef p4d_access_permitted
609#define p4d_access_permitted(p4d, write) \
610 (p4d_present(p4d) && (!(write) || p4d_write(p4d)))
611#endif
612
613#ifndef pgd_access_permitted
614#define pgd_access_permitted(pgd, write) \
615 (pgd_present(pgd) && (!(write) || pgd_write(pgd)))
616#endif
617
618#ifndef __HAVE_ARCH_PMD_SAME
619static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
620{
621 return pmd_val(pmd_a) == pmd_val(pmd_b);
622}
623
624static inline int pud_same(pud_t pud_a, pud_t pud_b)
625{
626 return pud_val(pud_a) == pud_val(pud_b);
627}
628#endif
629
630#ifndef __HAVE_ARCH_P4D_SAME
631static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b)
632{
633 return p4d_val(p4d_a) == p4d_val(p4d_b);
634}
635#endif
636
637#ifndef __HAVE_ARCH_PGD_SAME
638static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b)
639{
640 return pgd_val(pgd_a) == pgd_val(pgd_b);
641}
642#endif
643
644
645
646
647
648
649
650
651
652#define set_pte_safe(ptep, pte) \
653({ \
654 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \
655 set_pte(ptep, pte); \
656})
657
658#define set_pmd_safe(pmdp, pmd) \
659({ \
660 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
661 set_pmd(pmdp, pmd); \
662})
663
664#define set_pud_safe(pudp, pud) \
665({ \
666 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \
667 set_pud(pudp, pud); \
668})
669
670#define set_p4d_safe(p4dp, p4d) \
671({ \
672 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
673 set_p4d(p4dp, p4d); \
674})
675
676#define set_pgd_safe(pgdp, pgd) \
677({ \
678 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
679 set_pgd(pgdp, pgd); \
680})
681
682#ifndef __HAVE_ARCH_DO_SWAP_PAGE
683
684
685
686
687
688
689
690
691static inline void arch_do_swap_page(struct mm_struct *mm,
692 struct vm_area_struct *vma,
693 unsigned long addr,
694 pte_t pte, pte_t oldpte)
695{
696
697}
698#endif
699
700#ifndef __HAVE_ARCH_UNMAP_ONE
701
702
703
704
705
706
707
708
709static inline int arch_unmap_one(struct mm_struct *mm,
710 struct vm_area_struct *vma,
711 unsigned long addr,
712 pte_t orig_pte)
713{
714 return 0;
715}
716#endif
717
718
719
720
721
722
723#ifndef __HAVE_ARCH_PREPARE_TO_SWAP
724static inline int arch_prepare_to_swap(struct page *page)
725{
726 return 0;
727}
728#endif
729
730#ifndef __HAVE_ARCH_SWAP_INVALIDATE
731static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
732{
733}
734
735static inline void arch_swap_invalidate_area(int type)
736{
737}
738#endif
739
740#ifndef __HAVE_ARCH_SWAP_RESTORE
741static inline void arch_swap_restore(swp_entry_t entry, struct page *page)
742{
743}
744#endif
745
746#ifndef __HAVE_ARCH_PGD_OFFSET_GATE
747#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
748#endif
749
750#ifndef __HAVE_ARCH_MOVE_PTE
751#define move_pte(pte, prot, old_addr, new_addr) (pte)
752#endif
753
754#ifndef pte_accessible
755# define pte_accessible(mm, pte) ((void)(pte), 1)
756#endif
757
758#ifndef flush_tlb_fix_spurious_fault
759#define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address)
760#endif
761
762
763
764
765
766
767
768#define pgd_addr_end(addr, end) \
769({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
770 (__boundary - 1 < (end) - 1)? __boundary: (end); \
771})
772
773#ifndef p4d_addr_end
774#define p4d_addr_end(addr, end) \
775({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \
776 (__boundary - 1 < (end) - 1)? __boundary: (end); \
777})
778#endif
779
780#ifndef pud_addr_end
781#define pud_addr_end(addr, end) \
782({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
783 (__boundary - 1 < (end) - 1)? __boundary: (end); \
784})
785#endif
786
787#ifndef pmd_addr_end
788#define pmd_addr_end(addr, end) \
789({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
790 (__boundary - 1 < (end) - 1)? __boundary: (end); \
791})
792#endif
793
794
795
796
797
798
799void pgd_clear_bad(pgd_t *);
800
801#ifndef __PAGETABLE_P4D_FOLDED
802void p4d_clear_bad(p4d_t *);
803#else
804#define p4d_clear_bad(p4d) do { } while (0)
805#endif
806
807#ifndef __PAGETABLE_PUD_FOLDED
808void pud_clear_bad(pud_t *);
809#else
810#define pud_clear_bad(p4d) do { } while (0)
811#endif
812
813void pmd_clear_bad(pmd_t *);
814
815static inline int pgd_none_or_clear_bad(pgd_t *pgd)
816{
817 if (pgd_none(*pgd))
818 return 1;
819 if (unlikely(pgd_bad(*pgd))) {
820 pgd_clear_bad(pgd);
821 return 1;
822 }
823 return 0;
824}
825
826static inline int p4d_none_or_clear_bad(p4d_t *p4d)
827{
828 if (p4d_none(*p4d))
829 return 1;
830 if (unlikely(p4d_bad(*p4d))) {
831 p4d_clear_bad(p4d);
832 return 1;
833 }
834 return 0;
835}
836
837static inline int pud_none_or_clear_bad(pud_t *pud)
838{
839 if (pud_none(*pud))
840 return 1;
841 if (unlikely(pud_bad(*pud))) {
842 pud_clear_bad(pud);
843 return 1;
844 }
845 return 0;
846}
847
848static inline int pmd_none_or_clear_bad(pmd_t *pmd)
849{
850 if (pmd_none(*pmd))
851 return 1;
852 if (unlikely(pmd_bad(*pmd))) {
853 pmd_clear_bad(pmd);
854 return 1;
855 }
856 return 0;
857}
858
859static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma,
860 unsigned long addr,
861 pte_t *ptep)
862{
863
864
865
866
867
868 return ptep_get_and_clear(vma->vm_mm, addr, ptep);
869}
870
871static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma,
872 unsigned long addr,
873 pte_t *ptep, pte_t pte)
874{
875
876
877
878
879 set_pte_at(vma->vm_mm, addr, ptep, pte);
880}
881
882#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
898 unsigned long addr,
899 pte_t *ptep)
900{
901 return __ptep_modify_prot_start(vma, addr, ptep);
902}
903
904
905
906
907
908static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
909 unsigned long addr,
910 pte_t *ptep, pte_t old_pte, pte_t pte)
911{
912 __ptep_modify_prot_commit(vma, addr, ptep, pte);
913}
914#endif
915#endif
916
917
918
919
920
921
922#ifndef pgprot_nx
923#define pgprot_nx(prot) (prot)
924#endif
925
926#ifndef pgprot_noncached
927#define pgprot_noncached(prot) (prot)
928#endif
929
930#ifndef pgprot_writecombine
931#define pgprot_writecombine pgprot_noncached
932#endif
933
934#ifndef pgprot_writethrough
935#define pgprot_writethrough pgprot_noncached
936#endif
937
938#ifndef pgprot_device
939#define pgprot_device pgprot_noncached
940#endif
941
942#ifndef pgprot_mhp
943#define pgprot_mhp(prot) (prot)
944#endif
945
946#ifdef CONFIG_MMU
947#ifndef pgprot_modify
948#define pgprot_modify pgprot_modify
949static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
950{
951 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot)))
952 newprot = pgprot_noncached(newprot);
953 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot)))
954 newprot = pgprot_writecombine(newprot);
955 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot)))
956 newprot = pgprot_device(newprot);
957 return newprot;
958}
959#endif
960#endif
961
962#ifndef pgprot_encrypted
963#define pgprot_encrypted(prot) (prot)
964#endif
965
966#ifndef pgprot_decrypted
967#define pgprot_decrypted(prot) (prot)
968#endif
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
986#define arch_enter_lazy_mmu_mode() do {} while (0)
987#define arch_leave_lazy_mmu_mode() do {} while (0)
988#define arch_flush_lazy_mmu_mode() do {} while (0)
989#endif
990
991
992
993
994
995
996
997
998
999
1000
1001
1002#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH
1003#define arch_start_context_switch(prev) do {} while (0)
1004#endif
1005
1006#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1007#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION
1008static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1009{
1010 return pmd;
1011}
1012
1013static inline int pmd_swp_soft_dirty(pmd_t pmd)
1014{
1015 return 0;
1016}
1017
1018static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1019{
1020 return pmd;
1021}
1022#endif
1023#else
1024static inline int pte_soft_dirty(pte_t pte)
1025{
1026 return 0;
1027}
1028
1029static inline int pmd_soft_dirty(pmd_t pmd)
1030{
1031 return 0;
1032}
1033
1034static inline pte_t pte_mksoft_dirty(pte_t pte)
1035{
1036 return pte;
1037}
1038
1039static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
1040{
1041 return pmd;
1042}
1043
1044static inline pte_t pte_clear_soft_dirty(pte_t pte)
1045{
1046 return pte;
1047}
1048
1049static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
1050{
1051 return pmd;
1052}
1053
1054static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1055{
1056 return pte;
1057}
1058
1059static inline int pte_swp_soft_dirty(pte_t pte)
1060{
1061 return 0;
1062}
1063
1064static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1065{
1066 return pte;
1067}
1068
1069static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1070{
1071 return pmd;
1072}
1073
1074static inline int pmd_swp_soft_dirty(pmd_t pmd)
1075{
1076 return 0;
1077}
1078
1079static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1080{
1081 return pmd;
1082}
1083#endif
1084
1085#ifndef __HAVE_PFNMAP_TRACKING
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
1097 unsigned long pfn, unsigned long addr,
1098 unsigned long size)
1099{
1100 return 0;
1101}
1102
1103
1104
1105
1106
1107static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
1108 pfn_t pfn)
1109{
1110}
1111
1112
1113
1114
1115
1116static inline int track_pfn_copy(struct vm_area_struct *vma)
1117{
1118 return 0;
1119}
1120
1121
1122
1123
1124
1125
1126static inline void untrack_pfn(struct vm_area_struct *vma,
1127 unsigned long pfn, unsigned long size)
1128{
1129}
1130
1131
1132
1133
1134static inline void untrack_pfn_moved(struct vm_area_struct *vma)
1135{
1136}
1137#else
1138extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
1139 unsigned long pfn, unsigned long addr,
1140 unsigned long size);
1141extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
1142 pfn_t pfn);
1143extern int track_pfn_copy(struct vm_area_struct *vma);
1144extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
1145 unsigned long size);
1146extern void untrack_pfn_moved(struct vm_area_struct *vma);
1147#endif
1148
1149#ifdef CONFIG_MMU
1150#ifdef __HAVE_COLOR_ZERO_PAGE
1151static inline int is_zero_pfn(unsigned long pfn)
1152{
1153 extern unsigned long zero_pfn;
1154 unsigned long offset_from_zero_pfn = pfn - zero_pfn;
1155 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
1156}
1157
1158#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
1159
1160#else
1161static inline int is_zero_pfn(unsigned long pfn)
1162{
1163 extern unsigned long zero_pfn;
1164 return pfn == zero_pfn;
1165}
1166
1167static inline unsigned long my_zero_pfn(unsigned long addr)
1168{
1169 extern unsigned long zero_pfn;
1170 return zero_pfn;
1171}
1172#endif
1173#else
1174static inline int is_zero_pfn(unsigned long pfn)
1175{
1176 return 0;
1177}
1178
1179static inline unsigned long my_zero_pfn(unsigned long addr)
1180{
1181 return 0;
1182}
1183#endif
1184
1185#ifdef CONFIG_MMU
1186
1187#ifndef CONFIG_TRANSPARENT_HUGEPAGE
1188static inline int pmd_trans_huge(pmd_t pmd)
1189{
1190 return 0;
1191}
1192#ifndef pmd_write
1193static inline int pmd_write(pmd_t pmd)
1194{
1195 BUG();
1196 return 0;
1197}
1198#endif
1199#endif
1200
1201#ifndef pud_write
1202static inline int pud_write(pud_t pud)
1203{
1204 BUG();
1205 return 0;
1206}
1207#endif
1208
1209#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
1210static inline int pmd_devmap(pmd_t pmd)
1211{
1212 return 0;
1213}
1214static inline int pud_devmap(pud_t pud)
1215{
1216 return 0;
1217}
1218static inline int pgd_devmap(pgd_t pgd)
1219{
1220 return 0;
1221}
1222#endif
1223
1224#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \
1225 (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1226 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD))
1227static inline int pud_trans_huge(pud_t pud)
1228{
1229 return 0;
1230}
1231#endif
1232
1233
1234static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud)
1235{
1236 pud_t pudval = READ_ONCE(*pud);
1237
1238 if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval))
1239 return 1;
1240 if (unlikely(pud_bad(pudval))) {
1241 pud_clear_bad(pud);
1242 return 1;
1243 }
1244 return 0;
1245}
1246
1247
1248static inline int pud_trans_unstable(pud_t *pud)
1249{
1250#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1251 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1252 return pud_none_or_trans_huge_or_dev_or_clear_bad(pud);
1253#else
1254 return 0;
1255#endif
1256}
1257
1258#ifndef pmd_read_atomic
1259static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
1260{
1261
1262
1263
1264
1265
1266 return *pmdp;
1267}
1268#endif
1269
1270#ifndef arch_needs_pgtable_deposit
1271#define arch_needs_pgtable_deposit() (false)
1272#endif
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
1295{
1296 pmd_t pmdval = pmd_read_atomic(pmd);
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1312 barrier();
1313#endif
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329 if (pmd_none(pmdval) || pmd_trans_huge(pmdval) ||
1330 (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval)))
1331 return 1;
1332 if (unlikely(pmd_bad(pmdval))) {
1333 pmd_clear_bad(pmd);
1334 return 1;
1335 }
1336 return 0;
1337}
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351static inline int pmd_trans_unstable(pmd_t *pmd)
1352{
1353#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1354 return pmd_none_or_trans_huge_or_clear_bad(pmd);
1355#else
1356 return 0;
1357#endif
1358}
1359
1360
1361
1362
1363
1364
1365
1366static inline int pmd_devmap_trans_unstable(pmd_t *pmd)
1367{
1368 return pmd_devmap(*pmd) || pmd_trans_unstable(pmd);
1369}
1370
1371#ifndef CONFIG_NUMA_BALANCING
1372
1373
1374
1375
1376
1377
1378
1379
1380static inline int pte_protnone(pte_t pte)
1381{
1382 return 0;
1383}
1384
1385static inline int pmd_protnone(pmd_t pmd)
1386{
1387 return 0;
1388}
1389#endif
1390
1391#endif
1392
1393#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
1394
1395#ifndef __PAGETABLE_P4D_FOLDED
1396int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);
1397int p4d_clear_huge(p4d_t *p4d);
1398#else
1399static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
1400{
1401 return 0;
1402}
1403static inline int p4d_clear_huge(p4d_t *p4d)
1404{
1405 return 0;
1406}
1407#endif
1408
1409int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);
1410int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
1411int pud_clear_huge(pud_t *pud);
1412int pmd_clear_huge(pmd_t *pmd);
1413int p4d_free_pud_page(p4d_t *p4d, unsigned long addr);
1414int pud_free_pmd_page(pud_t *pud, unsigned long addr);
1415int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
1416#else
1417static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
1418{
1419 return 0;
1420}
1421static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1422{
1423 return 0;
1424}
1425static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1426{
1427 return 0;
1428}
1429static inline int p4d_clear_huge(p4d_t *p4d)
1430{
1431 return 0;
1432}
1433static inline int pud_clear_huge(pud_t *pud)
1434{
1435 return 0;
1436}
1437static inline int pmd_clear_huge(pmd_t *pmd)
1438{
1439 return 0;
1440}
1441static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr)
1442{
1443 return 0;
1444}
1445static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1446{
1447 return 0;
1448}
1449static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1450{
1451 return 0;
1452}
1453#endif
1454
1455#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
1456#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1467#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1468#else
1469#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()
1470#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG()
1471#endif
1472#endif
1473
1474struct file;
1475int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
1476 unsigned long size, pgprot_t *vma_prot);
1477
1478#ifndef CONFIG_X86_ESPFIX64
1479static inline void init_espfix_bsp(void) { }
1480#endif
1481
1482extern void __init pgtable_cache_init(void);
1483
1484#ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED
1485static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot)
1486{
1487 return true;
1488}
1489
1490static inline bool arch_has_pfn_modify_check(void)
1491{
1492 return false;
1493}
1494#endif
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506#ifndef PAGE_KERNEL_RO
1507# define PAGE_KERNEL_RO PAGE_KERNEL
1508#endif
1509
1510#ifndef PAGE_KERNEL_EXEC
1511# define PAGE_KERNEL_EXEC PAGE_KERNEL
1512#endif
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522#define __PGTBL_PGD_MODIFIED 0
1523#define __PGTBL_P4D_MODIFIED 1
1524#define __PGTBL_PUD_MODIFIED 2
1525#define __PGTBL_PMD_MODIFIED 3
1526#define __PGTBL_PTE_MODIFIED 4
1527
1528#define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED)
1529#define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED)
1530#define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED)
1531#define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED)
1532#define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED)
1533
1534
1535typedef unsigned int pgtbl_mod_mask;
1536
1537#endif
1538
1539#if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)
1540#ifdef CONFIG_PHYS_ADDR_T_64BIT
1541
1542
1543
1544
1545
1546#error Missing MAX_POSSIBLE_PHYSMEM_BITS definition
1547#else
1548#define MAX_POSSIBLE_PHYSMEM_BITS 32
1549#endif
1550#endif
1551
1552#ifndef has_transparent_hugepage
1553#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1554#define has_transparent_hugepage() 1
1555#else
1556#define has_transparent_hugepage() 0
1557#endif
1558#endif
1559
1560
1561
1562
1563
1564#ifndef mm_p4d_folded
1565#define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED)
1566#endif
1567
1568#ifndef mm_pud_folded
1569#define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED)
1570#endif
1571
1572#ifndef mm_pmd_folded
1573#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED)
1574#endif
1575
1576#ifndef p4d_offset_lockless
1577#define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address)
1578#endif
1579#ifndef pud_offset_lockless
1580#define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address)
1581#endif
1582#ifndef pmd_offset_lockless
1583#define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address)
1584#endif
1585
1586
1587
1588
1589
1590
1591
1592
1593#ifndef pgd_leaf
1594#define pgd_leaf(x) 0
1595#endif
1596#ifndef p4d_leaf
1597#define p4d_leaf(x) 0
1598#endif
1599#ifndef pud_leaf
1600#define pud_leaf(x) 0
1601#endif
1602#ifndef pmd_leaf
1603#define pmd_leaf(x) 0
1604#endif
1605
1606#ifndef pgd_leaf_size
1607#define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT)
1608#endif
1609#ifndef p4d_leaf_size
1610#define p4d_leaf_size(x) P4D_SIZE
1611#endif
1612#ifndef pud_leaf_size
1613#define pud_leaf_size(x) PUD_SIZE
1614#endif
1615#ifndef pmd_leaf_size
1616#define pmd_leaf_size(x) PMD_SIZE
1617#endif
1618#ifndef pte_leaf_size
1619#define pte_leaf_size(x) PAGE_SIZE
1620#endif
1621
1622
1623
1624
1625
1626
1627
1628#ifndef MAX_PTRS_PER_PTE
1629#define MAX_PTRS_PER_PTE PTRS_PER_PTE
1630#endif
1631
1632#ifndef MAX_PTRS_PER_PMD
1633#define MAX_PTRS_PER_PMD PTRS_PER_PMD
1634#endif
1635
1636#ifndef MAX_PTRS_PER_PUD
1637#define MAX_PTRS_PER_PUD PTRS_PER_PUD
1638#endif
1639
1640#ifndef MAX_PTRS_PER_P4D
1641#define MAX_PTRS_PER_P4D PTRS_PER_P4D
1642#endif
1643
1644#endif
1645