1
2#ifndef _LINUX_PGTABLE_H
3#define _LINUX_PGTABLE_H
4
5#include <linux/pfn.h>
6#include <asm/pgtable.h>
7
8#ifndef __ASSEMBLY__
9#ifdef CONFIG_MMU
10
11#include <linux/mm_types.h>
12#include <linux/bug.h>
13#include <linux/errno.h>
14#include <asm-generic/pgtable_uffd.h>
15
16#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \
17 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS
18#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED
19#endif
20
21
22
23
24
25
26
27#ifndef USER_PGTABLES_CEILING
28#define USER_PGTABLES_CEILING 0UL
29#endif
30
31
32
33
34
35
36#ifndef FIRST_USER_ADDRESS
37#define FIRST_USER_ADDRESS 0UL
38#endif
39
40
41
42
43
44
45#ifndef pmd_pgtable
46#define pmd_pgtable(pmd) pmd_page(pmd)
47#endif
48
49
50
51
52
53
54
55
56
57
58
59
60
61static inline unsigned long pte_index(unsigned long address)
62{
63 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
64}
65
66#ifndef pmd_index
67static inline unsigned long pmd_index(unsigned long address)
68{
69 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
70}
71#define pmd_index pmd_index
72#endif
73
74#ifndef pud_index
75static inline unsigned long pud_index(unsigned long address)
76{
77 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
78}
79#define pud_index pud_index
80#endif
81
82#ifndef pgd_index
83
84#define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
85#endif
86
87#ifndef pte_offset_kernel
88static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
89{
90 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
91}
92#define pte_offset_kernel pte_offset_kernel
93#endif
94
95#if defined(CONFIG_HIGHPTE)
96#define pte_offset_map(dir, address) \
97 ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \
98 pte_index((address)))
99#define pte_unmap(pte) kunmap_atomic((pte))
100#else
101#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
102#define pte_unmap(pte) ((void)(pte))
103#endif
104
105
106#ifndef pmd_offset
107static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
108{
109 return pud_pgtable(*pud) + pmd_index(address);
110}
111#define pmd_offset pmd_offset
112#endif
113
114#ifndef pud_offset
115static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
116{
117 return p4d_pgtable(*p4d) + pud_index(address);
118}
119#define pud_offset pud_offset
120#endif
121
122static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address)
123{
124 return (pgd + pgd_index(address));
125};
126
127
128
129
130#ifndef pgd_offset
131#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
132#endif
133
134
135
136
137
138#ifndef pgd_offset_k
139#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
140#endif
141
142
143
144
145
146
147
148
149static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va)
150{
151 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va);
152}
153
154static inline pmd_t *pmd_off_k(unsigned long va)
155{
156 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va);
157}
158
159static inline pte_t *virt_to_kpte(unsigned long vaddr)
160{
161 pmd_t *pmd = pmd_off_k(vaddr);
162
163 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr);
164}
165
166#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
167extern int ptep_set_access_flags(struct vm_area_struct *vma,
168 unsigned long address, pte_t *ptep,
169 pte_t entry, int dirty);
170#endif
171
172#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
173#ifdef CONFIG_TRANSPARENT_HUGEPAGE
174extern int pmdp_set_access_flags(struct vm_area_struct *vma,
175 unsigned long address, pmd_t *pmdp,
176 pmd_t entry, int dirty);
177extern int pudp_set_access_flags(struct vm_area_struct *vma,
178 unsigned long address, pud_t *pudp,
179 pud_t entry, int dirty);
180#else
181static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
182 unsigned long address, pmd_t *pmdp,
183 pmd_t entry, int dirty)
184{
185 BUILD_BUG();
186 return 0;
187}
188static inline int pudp_set_access_flags(struct vm_area_struct *vma,
189 unsigned long address, pud_t *pudp,
190 pud_t entry, int dirty)
191{
192 BUILD_BUG();
193 return 0;
194}
195#endif
196#endif
197
198#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
199static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
200 unsigned long address,
201 pte_t *ptep)
202{
203 pte_t pte = *ptep;
204 int r = 1;
205 if (!pte_young(pte))
206 r = 0;
207 else
208 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte));
209 return r;
210}
211#endif
212
213#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
214#ifdef CONFIG_TRANSPARENT_HUGEPAGE
215static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
216 unsigned long address,
217 pmd_t *pmdp)
218{
219 pmd_t pmd = *pmdp;
220 int r = 1;
221 if (!pmd_young(pmd))
222 r = 0;
223 else
224 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));
225 return r;
226}
227#else
228static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
229 unsigned long address,
230 pmd_t *pmdp)
231{
232 BUILD_BUG();
233 return 0;
234}
235#endif
236#endif
237
238#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
239int ptep_clear_flush_young(struct vm_area_struct *vma,
240 unsigned long address, pte_t *ptep);
241#endif
242
243#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
244#ifdef CONFIG_TRANSPARENT_HUGEPAGE
245extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
246 unsigned long address, pmd_t *pmdp);
247#else
248
249
250
251
252static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
253 unsigned long address, pmd_t *pmdp)
254{
255 BUILD_BUG();
256 return 0;
257}
258#endif
259#endif
260
261#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
262static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
263 unsigned long address,
264 pte_t *ptep)
265{
266 pte_t pte = *ptep;
267 pte_clear(mm, address, ptep);
268 return pte;
269}
270#endif
271
272#ifndef __HAVE_ARCH_PTEP_GET
273static inline pte_t ptep_get(pte_t *ptep)
274{
275 return READ_ONCE(*ptep);
276}
277#endif
278
279#ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311static inline pte_t ptep_get_lockless(pte_t *ptep)
312{
313 pte_t pte;
314
315 do {
316 pte.pte_low = ptep->pte_low;
317 smp_rmb();
318 pte.pte_high = ptep->pte_high;
319 smp_rmb();
320 } while (unlikely(pte.pte_low != ptep->pte_low));
321
322 return pte;
323}
324#else
325
326
327
328static inline pte_t ptep_get_lockless(pte_t *ptep)
329{
330 return ptep_get(ptep);
331}
332#endif
333
334#ifdef CONFIG_TRANSPARENT_HUGEPAGE
335#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
336static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
337 unsigned long address,
338 pmd_t *pmdp)
339{
340 pmd_t pmd = *pmdp;
341 pmd_clear(pmdp);
342 return pmd;
343}
344#endif
345#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
346static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
347 unsigned long address,
348 pud_t *pudp)
349{
350 pud_t pud = *pudp;
351
352 pud_clear(pudp);
353 return pud;
354}
355#endif
356#endif
357
358#ifdef CONFIG_TRANSPARENT_HUGEPAGE
359#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
360static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
361 unsigned long address, pmd_t *pmdp,
362 int full)
363{
364 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
365}
366#endif
367
368#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL
369static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm,
370 unsigned long address, pud_t *pudp,
371 int full)
372{
373 return pudp_huge_get_and_clear(mm, address, pudp);
374}
375#endif
376#endif
377
378#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
379static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
380 unsigned long address, pte_t *ptep,
381 int full)
382{
383 pte_t pte;
384 pte = ptep_get_and_clear(mm, address, ptep);
385 return pte;
386}
387#endif
388
389
390
391
392
393
394
395
396
397
398#ifndef __HAVE_ARCH_UPDATE_MMU_TLB
399static inline void update_mmu_tlb(struct vm_area_struct *vma,
400 unsigned long address, pte_t *ptep)
401{
402}
403#define __HAVE_ARCH_UPDATE_MMU_TLB
404#endif
405
406
407
408
409
410
411#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
412static inline void pte_clear_not_present_full(struct mm_struct *mm,
413 unsigned long address,
414 pte_t *ptep,
415 int full)
416{
417 pte_clear(mm, address, ptep);
418}
419#endif
420
421#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
422extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
423 unsigned long address,
424 pte_t *ptep);
425#endif
426
427#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
428extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
429 unsigned long address,
430 pmd_t *pmdp);
431extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma,
432 unsigned long address,
433 pud_t *pudp);
434#endif
435
436#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
437struct mm_struct;
438static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
439{
440 pte_t old_pte = *ptep;
441 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
442}
443#endif
444
445
446
447
448
449
450
451
452
453#ifndef pte_sw_mkyoung
454static inline pte_t pte_sw_mkyoung(pte_t pte)
455{
456 return pte;
457}
458#define pte_sw_mkyoung pte_sw_mkyoung
459#endif
460
461#ifndef pte_savedwrite
462#define pte_savedwrite pte_write
463#endif
464
465#ifndef pte_mk_savedwrite
466#define pte_mk_savedwrite pte_mkwrite
467#endif
468
469#ifndef pte_clear_savedwrite
470#define pte_clear_savedwrite pte_wrprotect
471#endif
472
473#ifndef pmd_savedwrite
474#define pmd_savedwrite pmd_write
475#endif
476
477#ifndef pmd_mk_savedwrite
478#define pmd_mk_savedwrite pmd_mkwrite
479#endif
480
481#ifndef pmd_clear_savedwrite
482#define pmd_clear_savedwrite pmd_wrprotect
483#endif
484
485#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT
486#ifdef CONFIG_TRANSPARENT_HUGEPAGE
487static inline void pmdp_set_wrprotect(struct mm_struct *mm,
488 unsigned long address, pmd_t *pmdp)
489{
490 pmd_t old_pmd = *pmdp;
491 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));
492}
493#else
494static inline void pmdp_set_wrprotect(struct mm_struct *mm,
495 unsigned long address, pmd_t *pmdp)
496{
497 BUILD_BUG();
498}
499#endif
500#endif
501#ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT
502#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
503static inline void pudp_set_wrprotect(struct mm_struct *mm,
504 unsigned long address, pud_t *pudp)
505{
506 pud_t old_pud = *pudp;
507
508 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud));
509}
510#else
511static inline void pudp_set_wrprotect(struct mm_struct *mm,
512 unsigned long address, pud_t *pudp)
513{
514 BUILD_BUG();
515}
516#endif
517#endif
518
519#ifndef pmdp_collapse_flush
520#ifdef CONFIG_TRANSPARENT_HUGEPAGE
521extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
522 unsigned long address, pmd_t *pmdp);
523#else
524static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
525 unsigned long address,
526 pmd_t *pmdp)
527{
528 BUILD_BUG();
529 return *pmdp;
530}
531#define pmdp_collapse_flush pmdp_collapse_flush
532#endif
533#endif
534
535#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
536extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
537 pgtable_t pgtable);
538#endif
539
540#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
541extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
542#endif
543
544#ifdef CONFIG_TRANSPARENT_HUGEPAGE
545
546
547
548
549
550static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma,
551 unsigned long address, pmd_t *pmdp, pmd_t pmd)
552{
553 pmd_t old_pmd = *pmdp;
554 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
555 return old_pmd;
556}
557#endif
558
559#ifndef __HAVE_ARCH_PMDP_INVALIDATE
560extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
561 pmd_t *pmdp);
562#endif
563
564#ifndef __HAVE_ARCH_PTE_SAME
565static inline int pte_same(pte_t pte_a, pte_t pte_b)
566{
567 return pte_val(pte_a) == pte_val(pte_b);
568}
569#endif
570
571#ifndef __HAVE_ARCH_PTE_UNUSED
572
573
574
575
576
577
578static inline int pte_unused(pte_t pte)
579{
580 return 0;
581}
582#endif
583
584#ifndef pte_access_permitted
585#define pte_access_permitted(pte, write) \
586 (pte_present(pte) && (!(write) || pte_write(pte)))
587#endif
588
589#ifndef pmd_access_permitted
590#define pmd_access_permitted(pmd, write) \
591 (pmd_present(pmd) && (!(write) || pmd_write(pmd)))
592#endif
593
594#ifndef pud_access_permitted
595#define pud_access_permitted(pud, write) \
596 (pud_present(pud) && (!(write) || pud_write(pud)))
597#endif
598
599#ifndef p4d_access_permitted
600#define p4d_access_permitted(p4d, write) \
601 (p4d_present(p4d) && (!(write) || p4d_write(p4d)))
602#endif
603
604#ifndef pgd_access_permitted
605#define pgd_access_permitted(pgd, write) \
606 (pgd_present(pgd) && (!(write) || pgd_write(pgd)))
607#endif
608
609#ifndef __HAVE_ARCH_PMD_SAME
610static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
611{
612 return pmd_val(pmd_a) == pmd_val(pmd_b);
613}
614
615static inline int pud_same(pud_t pud_a, pud_t pud_b)
616{
617 return pud_val(pud_a) == pud_val(pud_b);
618}
619#endif
620
621#ifndef __HAVE_ARCH_P4D_SAME
622static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b)
623{
624 return p4d_val(p4d_a) == p4d_val(p4d_b);
625}
626#endif
627
628#ifndef __HAVE_ARCH_PGD_SAME
629static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b)
630{
631 return pgd_val(pgd_a) == pgd_val(pgd_b);
632}
633#endif
634
635
636
637
638
639
640
641
642
643#define set_pte_safe(ptep, pte) \
644({ \
645 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \
646 set_pte(ptep, pte); \
647})
648
649#define set_pmd_safe(pmdp, pmd) \
650({ \
651 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
652 set_pmd(pmdp, pmd); \
653})
654
655#define set_pud_safe(pudp, pud) \
656({ \
657 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \
658 set_pud(pudp, pud); \
659})
660
661#define set_p4d_safe(p4dp, p4d) \
662({ \
663 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
664 set_p4d(p4dp, p4d); \
665})
666
667#define set_pgd_safe(pgdp, pgd) \
668({ \
669 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
670 set_pgd(pgdp, pgd); \
671})
672
673#ifndef __HAVE_ARCH_DO_SWAP_PAGE
674
675
676
677
678
679
680
681
682static inline void arch_do_swap_page(struct mm_struct *mm,
683 struct vm_area_struct *vma,
684 unsigned long addr,
685 pte_t pte, pte_t oldpte)
686{
687
688}
689#endif
690
691#ifndef __HAVE_ARCH_UNMAP_ONE
692
693
694
695
696
697
698
699
700static inline int arch_unmap_one(struct mm_struct *mm,
701 struct vm_area_struct *vma,
702 unsigned long addr,
703 pte_t orig_pte)
704{
705 return 0;
706}
707#endif
708
709
710
711
712
713
714#ifndef __HAVE_ARCH_PREPARE_TO_SWAP
715static inline int arch_prepare_to_swap(struct page *page)
716{
717 return 0;
718}
719#endif
720
721#ifndef __HAVE_ARCH_SWAP_INVALIDATE
722static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
723{
724}
725
726static inline void arch_swap_invalidate_area(int type)
727{
728}
729#endif
730
731#ifndef __HAVE_ARCH_SWAP_RESTORE
732static inline void arch_swap_restore(swp_entry_t entry, struct page *page)
733{
734}
735#endif
736
737#ifndef __HAVE_ARCH_PGD_OFFSET_GATE
738#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
739#endif
740
741#ifndef __HAVE_ARCH_MOVE_PTE
742#define move_pte(pte, prot, old_addr, new_addr) (pte)
743#endif
744
745#ifndef pte_accessible
746# define pte_accessible(mm, pte) ((void)(pte), 1)
747#endif
748
749#ifndef flush_tlb_fix_spurious_fault
750#define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address)
751#endif
752
753
754
755
756
757
758
759#define pgd_addr_end(addr, end) \
760({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
761 (__boundary - 1 < (end) - 1)? __boundary: (end); \
762})
763
764#ifndef p4d_addr_end
765#define p4d_addr_end(addr, end) \
766({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \
767 (__boundary - 1 < (end) - 1)? __boundary: (end); \
768})
769#endif
770
771#ifndef pud_addr_end
772#define pud_addr_end(addr, end) \
773({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
774 (__boundary - 1 < (end) - 1)? __boundary: (end); \
775})
776#endif
777
778#ifndef pmd_addr_end
779#define pmd_addr_end(addr, end) \
780({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
781 (__boundary - 1 < (end) - 1)? __boundary: (end); \
782})
783#endif
784
785
786
787
788
789
790void pgd_clear_bad(pgd_t *);
791
792#ifndef __PAGETABLE_P4D_FOLDED
793void p4d_clear_bad(p4d_t *);
794#else
795#define p4d_clear_bad(p4d) do { } while (0)
796#endif
797
798#ifndef __PAGETABLE_PUD_FOLDED
799void pud_clear_bad(pud_t *);
800#else
801#define pud_clear_bad(p4d) do { } while (0)
802#endif
803
804void pmd_clear_bad(pmd_t *);
805
806static inline int pgd_none_or_clear_bad(pgd_t *pgd)
807{
808 if (pgd_none(*pgd))
809 return 1;
810 if (unlikely(pgd_bad(*pgd))) {
811 pgd_clear_bad(pgd);
812 return 1;
813 }
814 return 0;
815}
816
817static inline int p4d_none_or_clear_bad(p4d_t *p4d)
818{
819 if (p4d_none(*p4d))
820 return 1;
821 if (unlikely(p4d_bad(*p4d))) {
822 p4d_clear_bad(p4d);
823 return 1;
824 }
825 return 0;
826}
827
828static inline int pud_none_or_clear_bad(pud_t *pud)
829{
830 if (pud_none(*pud))
831 return 1;
832 if (unlikely(pud_bad(*pud))) {
833 pud_clear_bad(pud);
834 return 1;
835 }
836 return 0;
837}
838
839static inline int pmd_none_or_clear_bad(pmd_t *pmd)
840{
841 if (pmd_none(*pmd))
842 return 1;
843 if (unlikely(pmd_bad(*pmd))) {
844 pmd_clear_bad(pmd);
845 return 1;
846 }
847 return 0;
848}
849
850static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma,
851 unsigned long addr,
852 pte_t *ptep)
853{
854
855
856
857
858
859 return ptep_get_and_clear(vma->vm_mm, addr, ptep);
860}
861
862static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma,
863 unsigned long addr,
864 pte_t *ptep, pte_t pte)
865{
866
867
868
869
870 set_pte_at(vma->vm_mm, addr, ptep, pte);
871}
872
873#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
889 unsigned long addr,
890 pte_t *ptep)
891{
892 return __ptep_modify_prot_start(vma, addr, ptep);
893}
894
895
896
897
898
899static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
900 unsigned long addr,
901 pte_t *ptep, pte_t old_pte, pte_t pte)
902{
903 __ptep_modify_prot_commit(vma, addr, ptep, pte);
904}
905#endif
906#endif
907
908
909
910
911
912
913#ifndef pgprot_nx
914#define pgprot_nx(prot) (prot)
915#endif
916
917#ifndef pgprot_noncached
918#define pgprot_noncached(prot) (prot)
919#endif
920
921#ifndef pgprot_writecombine
922#define pgprot_writecombine pgprot_noncached
923#endif
924
925#ifndef pgprot_writethrough
926#define pgprot_writethrough pgprot_noncached
927#endif
928
929#ifndef pgprot_device
930#define pgprot_device pgprot_noncached
931#endif
932
933#ifndef pgprot_mhp
934#define pgprot_mhp(prot) (prot)
935#endif
936
937#ifdef CONFIG_MMU
938#ifndef pgprot_modify
939#define pgprot_modify pgprot_modify
940static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
941{
942 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot)))
943 newprot = pgprot_noncached(newprot);
944 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot)))
945 newprot = pgprot_writecombine(newprot);
946 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot)))
947 newprot = pgprot_device(newprot);
948 return newprot;
949}
950#endif
951#endif
952
953#ifndef pgprot_encrypted
954#define pgprot_encrypted(prot) (prot)
955#endif
956
957#ifndef pgprot_decrypted
958#define pgprot_decrypted(prot) (prot)
959#endif
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
977#define arch_enter_lazy_mmu_mode() do {} while (0)
978#define arch_leave_lazy_mmu_mode() do {} while (0)
979#define arch_flush_lazy_mmu_mode() do {} while (0)
980#endif
981
982
983
984
985
986
987
988
989
990
991
992
993#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH
994#define arch_start_context_switch(prev) do {} while (0)
995#endif
996
997#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
998#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION
999static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1000{
1001 return pmd;
1002}
1003
1004static inline int pmd_swp_soft_dirty(pmd_t pmd)
1005{
1006 return 0;
1007}
1008
1009static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1010{
1011 return pmd;
1012}
1013#endif
1014#else
1015static inline int pte_soft_dirty(pte_t pte)
1016{
1017 return 0;
1018}
1019
1020static inline int pmd_soft_dirty(pmd_t pmd)
1021{
1022 return 0;
1023}
1024
1025static inline pte_t pte_mksoft_dirty(pte_t pte)
1026{
1027 return pte;
1028}
1029
1030static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
1031{
1032 return pmd;
1033}
1034
1035static inline pte_t pte_clear_soft_dirty(pte_t pte)
1036{
1037 return pte;
1038}
1039
1040static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
1041{
1042 return pmd;
1043}
1044
1045static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1046{
1047 return pte;
1048}
1049
1050static inline int pte_swp_soft_dirty(pte_t pte)
1051{
1052 return 0;
1053}
1054
1055static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1056{
1057 return pte;
1058}
1059
1060static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1061{
1062 return pmd;
1063}
1064
1065static inline int pmd_swp_soft_dirty(pmd_t pmd)
1066{
1067 return 0;
1068}
1069
1070static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1071{
1072 return pmd;
1073}
1074#endif
1075
1076#ifndef __HAVE_PFNMAP_TRACKING
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
1088 unsigned long pfn, unsigned long addr,
1089 unsigned long size)
1090{
1091 return 0;
1092}
1093
1094
1095
1096
1097
1098static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
1099 pfn_t pfn)
1100{
1101}
1102
1103
1104
1105
1106
1107static inline int track_pfn_copy(struct vm_area_struct *vma)
1108{
1109 return 0;
1110}
1111
1112
1113
1114
1115
1116
1117static inline void untrack_pfn(struct vm_area_struct *vma,
1118 unsigned long pfn, unsigned long size)
1119{
1120}
1121
1122
1123
1124
1125static inline void untrack_pfn_moved(struct vm_area_struct *vma)
1126{
1127}
1128#else
1129extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
1130 unsigned long pfn, unsigned long addr,
1131 unsigned long size);
1132extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
1133 pfn_t pfn);
1134extern int track_pfn_copy(struct vm_area_struct *vma);
1135extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
1136 unsigned long size);
1137extern void untrack_pfn_moved(struct vm_area_struct *vma);
1138#endif
1139
1140#ifdef CONFIG_MMU
1141#ifdef __HAVE_COLOR_ZERO_PAGE
1142static inline int is_zero_pfn(unsigned long pfn)
1143{
1144 extern unsigned long zero_pfn;
1145 unsigned long offset_from_zero_pfn = pfn - zero_pfn;
1146 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
1147}
1148
1149#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
1150
1151#else
1152static inline int is_zero_pfn(unsigned long pfn)
1153{
1154 extern unsigned long zero_pfn;
1155 return pfn == zero_pfn;
1156}
1157
1158static inline unsigned long my_zero_pfn(unsigned long addr)
1159{
1160 extern unsigned long zero_pfn;
1161 return zero_pfn;
1162}
1163#endif
1164#else
1165static inline int is_zero_pfn(unsigned long pfn)
1166{
1167 return 0;
1168}
1169
1170static inline unsigned long my_zero_pfn(unsigned long addr)
1171{
1172 return 0;
1173}
1174#endif
1175
1176#ifdef CONFIG_MMU
1177
1178#ifndef CONFIG_TRANSPARENT_HUGEPAGE
1179static inline int pmd_trans_huge(pmd_t pmd)
1180{
1181 return 0;
1182}
1183#ifndef pmd_write
1184static inline int pmd_write(pmd_t pmd)
1185{
1186 BUG();
1187 return 0;
1188}
1189#endif
1190#endif
1191
1192#ifndef pud_write
1193static inline int pud_write(pud_t pud)
1194{
1195 BUG();
1196 return 0;
1197}
1198#endif
1199
1200#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
1201static inline int pmd_devmap(pmd_t pmd)
1202{
1203 return 0;
1204}
1205static inline int pud_devmap(pud_t pud)
1206{
1207 return 0;
1208}
1209static inline int pgd_devmap(pgd_t pgd)
1210{
1211 return 0;
1212}
1213#endif
1214
1215#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \
1216 (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1217 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD))
1218static inline int pud_trans_huge(pud_t pud)
1219{
1220 return 0;
1221}
1222#endif
1223
1224
1225static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud)
1226{
1227 pud_t pudval = READ_ONCE(*pud);
1228
1229 if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval))
1230 return 1;
1231 if (unlikely(pud_bad(pudval))) {
1232 pud_clear_bad(pud);
1233 return 1;
1234 }
1235 return 0;
1236}
1237
1238
1239static inline int pud_trans_unstable(pud_t *pud)
1240{
1241#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1242 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1243 return pud_none_or_trans_huge_or_dev_or_clear_bad(pud);
1244#else
1245 return 0;
1246#endif
1247}
1248
1249#ifndef pmd_read_atomic
1250static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
1251{
1252
1253
1254
1255
1256
1257 return *pmdp;
1258}
1259#endif
1260
1261#ifndef arch_needs_pgtable_deposit
1262#define arch_needs_pgtable_deposit() (false)
1263#endif
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
1286{
1287 pmd_t pmdval = pmd_read_atomic(pmd);
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1303 barrier();
1304#endif
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320 if (pmd_none(pmdval) || pmd_trans_huge(pmdval) ||
1321 (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval)))
1322 return 1;
1323 if (unlikely(pmd_bad(pmdval))) {
1324 pmd_clear_bad(pmd);
1325 return 1;
1326 }
1327 return 0;
1328}
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342static inline int pmd_trans_unstable(pmd_t *pmd)
1343{
1344#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1345 return pmd_none_or_trans_huge_or_clear_bad(pmd);
1346#else
1347 return 0;
1348#endif
1349}
1350
1351
1352
1353
1354
1355
1356
1357static inline int pmd_devmap_trans_unstable(pmd_t *pmd)
1358{
1359 return pmd_devmap(*pmd) || pmd_trans_unstable(pmd);
1360}
1361
1362#ifndef CONFIG_NUMA_BALANCING
1363
1364
1365
1366
1367
1368
1369
1370
1371static inline int pte_protnone(pte_t pte)
1372{
1373 return 0;
1374}
1375
1376static inline int pmd_protnone(pmd_t pmd)
1377{
1378 return 0;
1379}
1380#endif
1381
1382#endif
1383
1384#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
1385
1386#ifndef __PAGETABLE_P4D_FOLDED
1387int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);
1388int p4d_clear_huge(p4d_t *p4d);
1389#else
1390static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
1391{
1392 return 0;
1393}
1394static inline int p4d_clear_huge(p4d_t *p4d)
1395{
1396 return 0;
1397}
1398#endif
1399
1400int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);
1401int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
1402int pud_clear_huge(pud_t *pud);
1403int pmd_clear_huge(pmd_t *pmd);
1404int p4d_free_pud_page(p4d_t *p4d, unsigned long addr);
1405int pud_free_pmd_page(pud_t *pud, unsigned long addr);
1406int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
1407#else
1408static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
1409{
1410 return 0;
1411}
1412static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1413{
1414 return 0;
1415}
1416static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1417{
1418 return 0;
1419}
1420static inline int p4d_clear_huge(p4d_t *p4d)
1421{
1422 return 0;
1423}
1424static inline int pud_clear_huge(pud_t *pud)
1425{
1426 return 0;
1427}
1428static inline int pmd_clear_huge(pmd_t *pmd)
1429{
1430 return 0;
1431}
1432static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr)
1433{
1434 return 0;
1435}
1436static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1437{
1438 return 0;
1439}
1440static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1441{
1442 return 0;
1443}
1444#endif
1445
1446#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
1447#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1458#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1459#else
1460#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()
1461#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG()
1462#endif
1463#endif
1464
1465struct file;
1466int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
1467 unsigned long size, pgprot_t *vma_prot);
1468
1469#ifndef CONFIG_X86_ESPFIX64
1470static inline void init_espfix_bsp(void) { }
1471#endif
1472
1473extern void __init pgtable_cache_init(void);
1474
1475#ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED
1476static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot)
1477{
1478 return true;
1479}
1480
1481static inline bool arch_has_pfn_modify_check(void)
1482{
1483 return false;
1484}
1485#endif
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497#ifndef PAGE_KERNEL_RO
1498# define PAGE_KERNEL_RO PAGE_KERNEL
1499#endif
1500
1501#ifndef PAGE_KERNEL_EXEC
1502# define PAGE_KERNEL_EXEC PAGE_KERNEL
1503#endif
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513#define __PGTBL_PGD_MODIFIED 0
1514#define __PGTBL_P4D_MODIFIED 1
1515#define __PGTBL_PUD_MODIFIED 2
1516#define __PGTBL_PMD_MODIFIED 3
1517#define __PGTBL_PTE_MODIFIED 4
1518
1519#define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED)
1520#define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED)
1521#define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED)
1522#define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED)
1523#define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED)
1524
1525
1526typedef unsigned int pgtbl_mod_mask;
1527
1528#endif
1529
1530#if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)
1531#ifdef CONFIG_PHYS_ADDR_T_64BIT
1532
1533
1534
1535
1536
1537#error Missing MAX_POSSIBLE_PHYSMEM_BITS definition
1538#else
1539#define MAX_POSSIBLE_PHYSMEM_BITS 32
1540#endif
1541#endif
1542
1543#ifndef has_transparent_hugepage
1544#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1545#define has_transparent_hugepage() 1
1546#else
1547#define has_transparent_hugepage() 0
1548#endif
1549#endif
1550
1551
1552
1553
1554
1555#ifndef mm_p4d_folded
1556#define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED)
1557#endif
1558
1559#ifndef mm_pud_folded
1560#define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED)
1561#endif
1562
1563#ifndef mm_pmd_folded
1564#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED)
1565#endif
1566
1567#ifndef p4d_offset_lockless
1568#define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address)
1569#endif
1570#ifndef pud_offset_lockless
1571#define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address)
1572#endif
1573#ifndef pmd_offset_lockless
1574#define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address)
1575#endif
1576
1577
1578
1579
1580
1581
1582
1583
1584#ifndef pgd_leaf
1585#define pgd_leaf(x) 0
1586#endif
1587#ifndef p4d_leaf
1588#define p4d_leaf(x) 0
1589#endif
1590#ifndef pud_leaf
1591#define pud_leaf(x) 0
1592#endif
1593#ifndef pmd_leaf
1594#define pmd_leaf(x) 0
1595#endif
1596
1597#ifndef pgd_leaf_size
1598#define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT)
1599#endif
1600#ifndef p4d_leaf_size
1601#define p4d_leaf_size(x) P4D_SIZE
1602#endif
1603#ifndef pud_leaf_size
1604#define pud_leaf_size(x) PUD_SIZE
1605#endif
1606#ifndef pmd_leaf_size
1607#define pmd_leaf_size(x) PMD_SIZE
1608#endif
1609#ifndef pte_leaf_size
1610#define pte_leaf_size(x) PAGE_SIZE
1611#endif
1612
1613
1614
1615
1616
1617
1618
1619#ifndef MAX_PTRS_PER_PTE
1620#define MAX_PTRS_PER_PTE PTRS_PER_PTE
1621#endif
1622
1623#ifndef MAX_PTRS_PER_PMD
1624#define MAX_PTRS_PER_PMD PTRS_PER_PMD
1625#endif
1626
1627#ifndef MAX_PTRS_PER_PUD
1628#define MAX_PTRS_PER_PUD PTRS_PER_PUD
1629#endif
1630
1631#ifndef MAX_PTRS_PER_P4D
1632#define MAX_PTRS_PER_P4D PTRS_PER_P4D
1633#endif
1634
1635#endif
1636