1
2#ifndef _ASM_X86_PGTABLE_H
3#define _ASM_X86_PGTABLE_H
4
5#include <linux/mem_encrypt.h>
6#include <asm/page.h>
7#include <asm/pgtable_types.h>
8
9
10
11
12#define pgprot_noncached(prot) \
13 ((boot_cpu_data.x86 > 3) \
14 ? (__pgprot(pgprot_val(prot) | \
15 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS))) \
16 : (prot))
17
18
19
20
21#define pgprot_encrypted(prot) __pgprot(__sme_set(pgprot_val(prot)))
22#define pgprot_decrypted(prot) __pgprot(__sme_clr(pgprot_val(prot)))
23
24#ifndef __ASSEMBLY__
25#include <asm/x86_init.h>
26#include <asm/fpu/xstate.h>
27#include <asm/fpu/api.h>
28
29extern pgd_t early_top_pgt[PTRS_PER_PGD];
30int __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
31
32void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
33void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user);
34void ptdump_walk_pgd_level_checkwx(void);
35void ptdump_walk_user_pgd_level_checkwx(void);
36
37#ifdef CONFIG_DEBUG_WX
38#define debug_checkwx() ptdump_walk_pgd_level_checkwx()
39#define debug_checkwx_user() ptdump_walk_user_pgd_level_checkwx()
40#else
41#define debug_checkwx() do { } while (0)
42#define debug_checkwx_user() do { } while (0)
43#endif
44
45
46
47
48
49extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
50 __visible;
51#define ZERO_PAGE(vaddr) ((void)(vaddr),virt_to_page(empty_zero_page))
52
53extern spinlock_t pgd_lock;
54extern struct list_head pgd_list;
55
56extern struct mm_struct *pgd_page_get_mm(struct page *page);
57
58extern pmdval_t early_pmd_flags;
59
60#ifdef CONFIG_PARAVIRT_XXL
61#include <asm/paravirt.h>
62#else
63#define set_pte(ptep, pte) native_set_pte(ptep, pte)
64#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
65
66#define set_pte_atomic(ptep, pte) \
67 native_set_pte_atomic(ptep, pte)
68
69#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
70
71#ifndef __PAGETABLE_P4D_FOLDED
72#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
73#define pgd_clear(pgd) (pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
74#endif
75
76#ifndef set_p4d
77# define set_p4d(p4dp, p4d) native_set_p4d(p4dp, p4d)
78#endif
79
80#ifndef __PAGETABLE_PUD_FOLDED
81#define p4d_clear(p4d) native_p4d_clear(p4d)
82#endif
83
84#ifndef set_pud
85# define set_pud(pudp, pud) native_set_pud(pudp, pud)
86#endif
87
88#ifndef __PAGETABLE_PUD_FOLDED
89#define pud_clear(pud) native_pud_clear(pud)
90#endif
91
92#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
93#define pmd_clear(pmd) native_pmd_clear(pmd)
94
95#define pgd_val(x) native_pgd_val(x)
96#define __pgd(x) native_make_pgd(x)
97
98#ifndef __PAGETABLE_P4D_FOLDED
99#define p4d_val(x) native_p4d_val(x)
100#define __p4d(x) native_make_p4d(x)
101#endif
102
103#ifndef __PAGETABLE_PUD_FOLDED
104#define pud_val(x) native_pud_val(x)
105#define __pud(x) native_make_pud(x)
106#endif
107
108#ifndef __PAGETABLE_PMD_FOLDED
109#define pmd_val(x) native_pmd_val(x)
110#define __pmd(x) native_make_pmd(x)
111#endif
112
113#define pte_val(x) native_pte_val(x)
114#define __pte(x) native_make_pte(x)
115
116#define arch_end_context_switch(prev) do {} while(0)
117#endif
118
119
120
121
122
123static inline int pte_dirty(pte_t pte)
124{
125 return pte_flags(pte) & _PAGE_DIRTY;
126}
127
128
129static inline u32 read_pkru(void)
130{
131 if (boot_cpu_has(X86_FEATURE_OSPKE))
132 return rdpkru();
133 return 0;
134}
135
136static inline void write_pkru(u32 pkru)
137{
138 struct pkru_state *pk;
139
140 if (!boot_cpu_has(X86_FEATURE_OSPKE))
141 return;
142
143 pk = get_xsave_addr(¤t->thread.fpu.state.xsave, XFEATURE_PKRU);
144
145
146
147
148
149
150 fpregs_lock();
151 if (pk)
152 pk->pkru = pkru;
153 __write_pkru(pkru);
154 fpregs_unlock();
155}
156
157static inline int pte_young(pte_t pte)
158{
159 return pte_flags(pte) & _PAGE_ACCESSED;
160}
161
162static inline int pmd_dirty(pmd_t pmd)
163{
164 return pmd_flags(pmd) & _PAGE_DIRTY;
165}
166
167static inline int pmd_young(pmd_t pmd)
168{
169 return pmd_flags(pmd) & _PAGE_ACCESSED;
170}
171
172static inline int pud_dirty(pud_t pud)
173{
174 return pud_flags(pud) & _PAGE_DIRTY;
175}
176
177static inline int pud_young(pud_t pud)
178{
179 return pud_flags(pud) & _PAGE_ACCESSED;
180}
181
182static inline int pte_write(pte_t pte)
183{
184 return pte_flags(pte) & _PAGE_RW;
185}
186
187static inline int pte_huge(pte_t pte)
188{
189 return pte_flags(pte) & _PAGE_PSE;
190}
191
192static inline int pte_global(pte_t pte)
193{
194 return pte_flags(pte) & _PAGE_GLOBAL;
195}
196
197static inline int pte_exec(pte_t pte)
198{
199 return !(pte_flags(pte) & _PAGE_NX);
200}
201
202static inline int pte_special(pte_t pte)
203{
204 return pte_flags(pte) & _PAGE_SPECIAL;
205}
206
207
208
209static inline u64 protnone_mask(u64 val);
210
211static inline unsigned long pte_pfn(pte_t pte)
212{
213 phys_addr_t pfn = pte_val(pte);
214 pfn ^= protnone_mask(pfn);
215 return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
216}
217
218static inline unsigned long pmd_pfn(pmd_t pmd)
219{
220 phys_addr_t pfn = pmd_val(pmd);
221 pfn ^= protnone_mask(pfn);
222 return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;
223}
224
225static inline unsigned long pud_pfn(pud_t pud)
226{
227 phys_addr_t pfn = pud_val(pud);
228 pfn ^= protnone_mask(pfn);
229 return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT;
230}
231
232static inline unsigned long p4d_pfn(p4d_t p4d)
233{
234 return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
235}
236
237static inline unsigned long pgd_pfn(pgd_t pgd)
238{
239 return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
240}
241
242static inline int p4d_large(p4d_t p4d)
243{
244
245 return 0;
246}
247
248#define pte_page(pte) pfn_to_page(pte_pfn(pte))
249
250static inline int pmd_large(pmd_t pte)
251{
252 return pmd_flags(pte) & _PAGE_PSE;
253}
254
255#ifdef CONFIG_TRANSPARENT_HUGEPAGE
256static inline int pmd_trans_huge(pmd_t pmd)
257{
258 return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
259}
260
261#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
262static inline int pud_trans_huge(pud_t pud)
263{
264 return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
265}
266#endif
267
268#define has_transparent_hugepage has_transparent_hugepage
269static inline int has_transparent_hugepage(void)
270{
271 return boot_cpu_has(X86_FEATURE_PSE);
272}
273
274#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
275static inline int pmd_devmap(pmd_t pmd)
276{
277 return !!(pmd_val(pmd) & _PAGE_DEVMAP);
278}
279
280#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
281static inline int pud_devmap(pud_t pud)
282{
283 return !!(pud_val(pud) & _PAGE_DEVMAP);
284}
285#else
286static inline int pud_devmap(pud_t pud)
287{
288 return 0;
289}
290#endif
291
292static inline int pgd_devmap(pgd_t pgd)
293{
294 return 0;
295}
296#endif
297#endif
298
299static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
300{
301 pteval_t v = native_pte_val(pte);
302
303 return native_make_pte(v | set);
304}
305
306static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
307{
308 pteval_t v = native_pte_val(pte);
309
310 return native_make_pte(v & ~clear);
311}
312
313static inline pte_t pte_mkclean(pte_t pte)
314{
315 return pte_clear_flags(pte, _PAGE_DIRTY);
316}
317
318static inline pte_t pte_mkold(pte_t pte)
319{
320 return pte_clear_flags(pte, _PAGE_ACCESSED);
321}
322
323static inline pte_t pte_wrprotect(pte_t pte)
324{
325 return pte_clear_flags(pte, _PAGE_RW);
326}
327
328static inline pte_t pte_mkexec(pte_t pte)
329{
330 return pte_clear_flags(pte, _PAGE_NX);
331}
332
333static inline pte_t pte_mkdirty(pte_t pte)
334{
335 return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
336}
337
338static inline pte_t pte_mkyoung(pte_t pte)
339{
340 return pte_set_flags(pte, _PAGE_ACCESSED);
341}
342
343static inline pte_t pte_mkwrite(pte_t pte)
344{
345 return pte_set_flags(pte, _PAGE_RW);
346}
347
348static inline pte_t pte_mkhuge(pte_t pte)
349{
350 return pte_set_flags(pte, _PAGE_PSE);
351}
352
353static inline pte_t pte_clrhuge(pte_t pte)
354{
355 return pte_clear_flags(pte, _PAGE_PSE);
356}
357
358static inline pte_t pte_mkglobal(pte_t pte)
359{
360 return pte_set_flags(pte, _PAGE_GLOBAL);
361}
362
363static inline pte_t pte_clrglobal(pte_t pte)
364{
365 return pte_clear_flags(pte, _PAGE_GLOBAL);
366}
367
368static inline pte_t pte_mkspecial(pte_t pte)
369{
370 return pte_set_flags(pte, _PAGE_SPECIAL);
371}
372
373static inline pte_t pte_mkdevmap(pte_t pte)
374{
375 return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
376}
377
378static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
379{
380 pmdval_t v = native_pmd_val(pmd);
381
382 return native_make_pmd(v | set);
383}
384
385static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
386{
387 pmdval_t v = native_pmd_val(pmd);
388
389 return native_make_pmd(v & ~clear);
390}
391
392static inline pmd_t pmd_mkold(pmd_t pmd)
393{
394 return pmd_clear_flags(pmd, _PAGE_ACCESSED);
395}
396
397static inline pmd_t pmd_mkclean(pmd_t pmd)
398{
399 return pmd_clear_flags(pmd, _PAGE_DIRTY);
400}
401
402static inline pmd_t pmd_wrprotect(pmd_t pmd)
403{
404 return pmd_clear_flags(pmd, _PAGE_RW);
405}
406
407static inline pmd_t pmd_mkdirty(pmd_t pmd)
408{
409 return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
410}
411
412static inline pmd_t pmd_mkdevmap(pmd_t pmd)
413{
414 return pmd_set_flags(pmd, _PAGE_DEVMAP);
415}
416
417static inline pmd_t pmd_mkhuge(pmd_t pmd)
418{
419 return pmd_set_flags(pmd, _PAGE_PSE);
420}
421
422static inline pmd_t pmd_mkyoung(pmd_t pmd)
423{
424 return pmd_set_flags(pmd, _PAGE_ACCESSED);
425}
426
427static inline pmd_t pmd_mkwrite(pmd_t pmd)
428{
429 return pmd_set_flags(pmd, _PAGE_RW);
430}
431
432static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
433{
434 pudval_t v = native_pud_val(pud);
435
436 return native_make_pud(v | set);
437}
438
439static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
440{
441 pudval_t v = native_pud_val(pud);
442
443 return native_make_pud(v & ~clear);
444}
445
446static inline pud_t pud_mkold(pud_t pud)
447{
448 return pud_clear_flags(pud, _PAGE_ACCESSED);
449}
450
451static inline pud_t pud_mkclean(pud_t pud)
452{
453 return pud_clear_flags(pud, _PAGE_DIRTY);
454}
455
456static inline pud_t pud_wrprotect(pud_t pud)
457{
458 return pud_clear_flags(pud, _PAGE_RW);
459}
460
461static inline pud_t pud_mkdirty(pud_t pud)
462{
463 return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
464}
465
466static inline pud_t pud_mkdevmap(pud_t pud)
467{
468 return pud_set_flags(pud, _PAGE_DEVMAP);
469}
470
471static inline pud_t pud_mkhuge(pud_t pud)
472{
473 return pud_set_flags(pud, _PAGE_PSE);
474}
475
476static inline pud_t pud_mkyoung(pud_t pud)
477{
478 return pud_set_flags(pud, _PAGE_ACCESSED);
479}
480
481static inline pud_t pud_mkwrite(pud_t pud)
482{
483 return pud_set_flags(pud, _PAGE_RW);
484}
485
486#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
487static inline int pte_soft_dirty(pte_t pte)
488{
489 return pte_flags(pte) & _PAGE_SOFT_DIRTY;
490}
491
492static inline int pmd_soft_dirty(pmd_t pmd)
493{
494 return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
495}
496
497static inline int pud_soft_dirty(pud_t pud)
498{
499 return pud_flags(pud) & _PAGE_SOFT_DIRTY;
500}
501
502static inline pte_t pte_mksoft_dirty(pte_t pte)
503{
504 return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
505}
506
507static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
508{
509 return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
510}
511
512static inline pud_t pud_mksoft_dirty(pud_t pud)
513{
514 return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
515}
516
517static inline pte_t pte_clear_soft_dirty(pte_t pte)
518{
519 return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
520}
521
522static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
523{
524 return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
525}
526
527static inline pud_t pud_clear_soft_dirty(pud_t pud)
528{
529 return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
530}
531
532#endif
533
534
535
536
537
538static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
539{
540 pgprotval_t protval = pgprot_val(pgprot);
541
542 if (protval & _PAGE_PRESENT)
543 protval &= __supported_pte_mask;
544
545 return protval;
546}
547
548static inline pgprotval_t check_pgprot(pgprot_t pgprot)
549{
550 pgprotval_t massaged_val = massage_pgprot(pgprot);
551
552
553#ifdef CONFIG_DEBUG_VM
554 WARN_ONCE(pgprot_val(pgprot) != massaged_val,
555 "attempted to set unsupported pgprot: %016llx "
556 "bits: %016llx supported: %016llx\n",
557 (u64)pgprot_val(pgprot),
558 (u64)pgprot_val(pgprot) ^ massaged_val,
559 (u64)__supported_pte_mask);
560#endif
561
562 return massaged_val;
563}
564
565static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
566{
567 phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
568 pfn ^= protnone_mask(pgprot_val(pgprot));
569 pfn &= PTE_PFN_MASK;
570 return __pte(pfn | check_pgprot(pgprot));
571}
572
573static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
574{
575 phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
576 pfn ^= protnone_mask(pgprot_val(pgprot));
577 pfn &= PHYSICAL_PMD_PAGE_MASK;
578 return __pmd(pfn | check_pgprot(pgprot));
579}
580
581static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
582{
583 phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
584 pfn ^= protnone_mask(pgprot_val(pgprot));
585 pfn &= PHYSICAL_PUD_PAGE_MASK;
586 return __pud(pfn | check_pgprot(pgprot));
587}
588
589static inline pmd_t pmd_mknotpresent(pmd_t pmd)
590{
591 return pfn_pmd(pmd_pfn(pmd),
592 __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
593}
594
595static inline pud_t pud_mknotpresent(pud_t pud)
596{
597 return pfn_pud(pud_pfn(pud),
598 __pgprot(pud_flags(pud) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
599}
600
601static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);
602
603static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
604{
605 pteval_t val = pte_val(pte), oldval = val;
606
607
608
609
610
611 val &= _PAGE_CHG_MASK;
612 val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
613 val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
614 return __pte(val);
615}
616
617static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
618{
619 pmdval_t val = pmd_val(pmd), oldval = val;
620
621 val &= _HPAGE_CHG_MASK;
622 val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
623 val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
624 return __pmd(val);
625}
626
627
628#define pgprot_modify pgprot_modify
629static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
630{
631 pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
632 pgprotval_t addbits = pgprot_val(newprot);
633 return __pgprot(preservebits | addbits);
634}
635
636#define pte_pgprot(x) __pgprot(pte_flags(x))
637#define pmd_pgprot(x) __pgprot(pmd_flags(x))
638#define pud_pgprot(x) __pgprot(pud_flags(x))
639#define p4d_pgprot(x) __pgprot(p4d_flags(x))
640
641#define canon_pgprot(p) __pgprot(massage_pgprot(p))
642
643static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
644{
645 return canon_pgprot(prot);
646}
647
648static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
649 enum page_cache_mode pcm,
650 enum page_cache_mode new_pcm)
651{
652
653
654
655 if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
656 return 1;
657
658
659
660
661
662
663
664
665
666 if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
667 new_pcm == _PAGE_CACHE_MODE_WB) ||
668 (pcm == _PAGE_CACHE_MODE_WC &&
669 new_pcm == _PAGE_CACHE_MODE_WB) ||
670 (pcm == _PAGE_CACHE_MODE_WT &&
671 new_pcm == _PAGE_CACHE_MODE_WB) ||
672 (pcm == _PAGE_CACHE_MODE_WT &&
673 new_pcm == _PAGE_CACHE_MODE_WC)) {
674 return 0;
675 }
676
677 return 1;
678}
679
680pmd_t *populate_extra_pmd(unsigned long vaddr);
681pte_t *populate_extra_pte(unsigned long vaddr);
682
683#ifdef CONFIG_PAGE_TABLE_ISOLATION
684pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
685
686
687
688
689
690
691static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
692{
693 if (!static_cpu_has(X86_FEATURE_PTI))
694 return pgd;
695 return __pti_set_user_pgtbl(pgdp, pgd);
696}
697#else
698static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
699{
700 return pgd;
701}
702#endif
703
704#endif
705
706
707#ifdef CONFIG_X86_32
708# include <asm/pgtable_32.h>
709#else
710# include <asm/pgtable_64.h>
711#endif
712
713#ifndef __ASSEMBLY__
714#include <linux/mm_types.h>
715#include <linux/mmdebug.h>
716#include <linux/log2.h>
717#include <asm/fixmap.h>
718
719static inline int pte_none(pte_t pte)
720{
721 return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK));
722}
723
724#define __HAVE_ARCH_PTE_SAME
725static inline int pte_same(pte_t a, pte_t b)
726{
727 return a.pte == b.pte;
728}
729
730static inline int pte_present(pte_t a)
731{
732 return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
733}
734
735#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
736static inline int pte_devmap(pte_t a)
737{
738 return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP;
739}
740#endif
741
742#define pte_accessible pte_accessible
743static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
744{
745 if (pte_flags(a) & _PAGE_PRESENT)
746 return true;
747
748 if ((pte_flags(a) & _PAGE_PROTNONE) &&
749 mm_tlb_flush_pending(mm))
750 return true;
751
752 return false;
753}
754
755static inline int pmd_present(pmd_t pmd)
756{
757
758
759
760
761
762
763 return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
764}
765
766#ifdef CONFIG_NUMA_BALANCING
767
768
769
770
771static inline int pte_protnone(pte_t pte)
772{
773 return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
774 == _PAGE_PROTNONE;
775}
776
777static inline int pmd_protnone(pmd_t pmd)
778{
779 return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
780 == _PAGE_PROTNONE;
781}
782#endif
783
784static inline int pmd_none(pmd_t pmd)
785{
786
787
788 unsigned long val = native_pmd_val(pmd);
789 return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0;
790}
791
792static inline unsigned long pmd_page_vaddr(pmd_t pmd)
793{
794 return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));
795}
796
797
798
799
800
801#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
802
803
804
805
806
807
808
809static inline unsigned long pmd_index(unsigned long address)
810{
811 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
812}
813
814
815
816
817
818
819
820
821#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
822
823
824
825
826
827
828
829static inline unsigned long pte_index(unsigned long address)
830{
831 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
832}
833
834static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
835{
836 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
837}
838
839static inline int pmd_bad(pmd_t pmd)
840{
841 return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
842}
843
844static inline unsigned long pages_to_mb(unsigned long npg)
845{
846 return npg >> (20 - PAGE_SHIFT);
847}
848
849#if CONFIG_PGTABLE_LEVELS > 2
850static inline int pud_none(pud_t pud)
851{
852 return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
853}
854
855static inline int pud_present(pud_t pud)
856{
857 return pud_flags(pud) & _PAGE_PRESENT;
858}
859
860static inline unsigned long pud_page_vaddr(pud_t pud)
861{
862 return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud));
863}
864
865
866
867
868
869#define pud_page(pud) pfn_to_page(pud_pfn(pud))
870
871
872static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
873{
874 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
875}
876
877static inline int pud_large(pud_t pud)
878{
879 return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
880 (_PAGE_PSE | _PAGE_PRESENT);
881}
882
883static inline int pud_bad(pud_t pud)
884{
885 return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
886}
887#else
888static inline int pud_large(pud_t pud)
889{
890 return 0;
891}
892#endif
893
894static inline unsigned long pud_index(unsigned long address)
895{
896 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
897}
898
899#if CONFIG_PGTABLE_LEVELS > 3
900static inline int p4d_none(p4d_t p4d)
901{
902 return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
903}
904
905static inline int p4d_present(p4d_t p4d)
906{
907 return p4d_flags(p4d) & _PAGE_PRESENT;
908}
909
910static inline unsigned long p4d_page_vaddr(p4d_t p4d)
911{
912 return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));
913}
914
915
916
917
918
919#define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d))
920
921
922static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
923{
924 return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address);
925}
926
927static inline int p4d_bad(p4d_t p4d)
928{
929 unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;
930
931 if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
932 ignore_flags |= _PAGE_NX;
933
934 return (p4d_flags(p4d) & ~ignore_flags) != 0;
935}
936#endif
937
938static inline unsigned long p4d_index(unsigned long address)
939{
940 return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
941}
942
943#if CONFIG_PGTABLE_LEVELS > 4
944static inline int pgd_present(pgd_t pgd)
945{
946 if (!pgtable_l5_enabled())
947 return 1;
948 return pgd_flags(pgd) & _PAGE_PRESENT;
949}
950
951static inline unsigned long pgd_page_vaddr(pgd_t pgd)
952{
953 return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
954}
955
956
957
958
959
960#define pgd_page(pgd) pfn_to_page(pgd_pfn(pgd))
961
962
963static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
964{
965 if (!pgtable_l5_enabled())
966 return (p4d_t *)pgd;
967 return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
968}
969
970static inline int pgd_bad(pgd_t pgd)
971{
972 unsigned long ignore_flags = _PAGE_USER;
973
974 if (!pgtable_l5_enabled())
975 return 0;
976
977 if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
978 ignore_flags |= _PAGE_NX;
979
980 return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
981}
982
983static inline int pgd_none(pgd_t pgd)
984{
985 if (!pgtable_l5_enabled())
986 return 0;
987
988
989
990
991
992
993 return !native_pgd_val(pgd);
994}
995#endif
996
997#endif
998
999
1000
1001
1002
1003
1004
1005#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
1006
1007
1008
1009
1010
1011#define pgd_offset_pgd(pgd, address) (pgd + pgd_index((address)))
1012
1013
1014
1015#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
1016
1017
1018
1019
1020#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
1021
1022
1023#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET)
1024#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
1025
1026#ifndef __ASSEMBLY__
1027
1028extern int direct_gbpages;
1029void init_mem_mapping(void);
1030void early_alloc_pgt_buf(void);
1031extern void memblock_find_dma_reserve(void);
1032
1033#ifdef CONFIG_X86_64
1034
1035extern pgd_t trampoline_pgd_entry;
1036static inline void __meminit init_trampoline_default(void)
1037{
1038
1039 trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
1040}
1041
1042void __init poking_init(void);
1043
1044# ifdef CONFIG_RANDOMIZE_MEMORY
1045void __meminit init_trampoline(void);
1046# else
1047# define init_trampoline init_trampoline_default
1048# endif
1049#else
1050static inline void init_trampoline(void) { }
1051#endif
1052
1053
1054static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
1055{
1056 pte_t res = *ptep;
1057
1058
1059 native_pte_clear(NULL, 0, ptep);
1060 return res;
1061}
1062
1063static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
1064{
1065 pmd_t res = *pmdp;
1066
1067 native_pmd_clear(pmdp);
1068 return res;
1069}
1070
1071static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
1072{
1073 pud_t res = *pudp;
1074
1075 native_pud_clear(pudp);
1076 return res;
1077}
1078
1079static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
1080 pte_t *ptep , pte_t pte)
1081{
1082 native_set_pte(ptep, pte);
1083}
1084
1085static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1086 pmd_t *pmdp, pmd_t pmd)
1087{
1088 set_pmd(pmdp, pmd);
1089}
1090
1091static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
1092 pud_t *pudp, pud_t pud)
1093{
1094 native_set_pud(pudp, pud);
1095}
1096
1097
1098
1099
1100
1101
1102
1103
1104struct vm_area_struct;
1105
1106#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1107extern int ptep_set_access_flags(struct vm_area_struct *vma,
1108 unsigned long address, pte_t *ptep,
1109 pte_t entry, int dirty);
1110
1111#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1112extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
1113 unsigned long addr, pte_t *ptep);
1114
1115#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1116extern int ptep_clear_flush_young(struct vm_area_struct *vma,
1117 unsigned long address, pte_t *ptep);
1118
1119#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1120static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
1121 pte_t *ptep)
1122{
1123 pte_t pte = native_ptep_get_and_clear(ptep);
1124 return pte;
1125}
1126
1127#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1128static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1129 unsigned long addr, pte_t *ptep,
1130 int full)
1131{
1132 pte_t pte;
1133 if (full) {
1134
1135
1136
1137
1138 pte = native_local_ptep_get_and_clear(ptep);
1139 } else {
1140 pte = ptep_get_and_clear(mm, addr, ptep);
1141 }
1142 return pte;
1143}
1144
1145#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1146static inline void ptep_set_wrprotect(struct mm_struct *mm,
1147 unsigned long addr, pte_t *ptep)
1148{
1149 clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
1150}
1151
1152#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
1153
1154#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
1155
1156#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1157extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1158 unsigned long address, pmd_t *pmdp,
1159 pmd_t entry, int dirty);
1160extern int pudp_set_access_flags(struct vm_area_struct *vma,
1161 unsigned long address, pud_t *pudp,
1162 pud_t entry, int dirty);
1163
1164#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1165extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1166 unsigned long addr, pmd_t *pmdp);
1167extern int pudp_test_and_clear_young(struct vm_area_struct *vma,
1168 unsigned long addr, pud_t *pudp);
1169
1170#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
1171extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
1172 unsigned long address, pmd_t *pmdp);
1173
1174
1175#define pmd_write pmd_write
1176static inline int pmd_write(pmd_t pmd)
1177{
1178 return pmd_flags(pmd) & _PAGE_RW;
1179}
1180
1181#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1182static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
1183 pmd_t *pmdp)
1184{
1185 return native_pmdp_get_and_clear(pmdp);
1186}
1187
1188#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
1189static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
1190 unsigned long addr, pud_t *pudp)
1191{
1192 return native_pudp_get_and_clear(pudp);
1193}
1194
1195#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1196static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1197 unsigned long addr, pmd_t *pmdp)
1198{
1199 clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
1200}
1201
1202#define pud_write pud_write
1203static inline int pud_write(pud_t pud)
1204{
1205 return pud_flags(pud) & _PAGE_RW;
1206}
1207
1208#ifndef pmdp_establish
1209#define pmdp_establish pmdp_establish
1210static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1211 unsigned long address, pmd_t *pmdp, pmd_t pmd)
1212{
1213 if (IS_ENABLED(CONFIG_SMP)) {
1214 return xchg(pmdp, pmd);
1215 } else {
1216 pmd_t old = *pmdp;
1217 WRITE_ONCE(*pmdp, pmd);
1218 return old;
1219 }
1220}
1221#endif
1222
1223
1224
1225
1226
1227
1228
1229static inline bool pgdp_maps_userspace(void *__ptr)
1230{
1231 unsigned long ptr = (unsigned long)__ptr;
1232
1233 return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
1234}
1235
1236static inline int pgd_large(pgd_t pgd) { return 0; }
1237
1238#ifdef CONFIG_PAGE_TABLE_ISOLATION
1239
1240
1241
1242
1243
1244
1245#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT
1246
1247
1248
1249
1250
1251static inline void *ptr_set_bit(void *ptr, int bit)
1252{
1253 unsigned long __ptr = (unsigned long)ptr;
1254
1255 __ptr |= BIT(bit);
1256 return (void *)__ptr;
1257}
1258static inline void *ptr_clear_bit(void *ptr, int bit)
1259{
1260 unsigned long __ptr = (unsigned long)ptr;
1261
1262 __ptr &= ~BIT(bit);
1263 return (void *)__ptr;
1264}
1265
1266static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
1267{
1268 return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1269}
1270
1271static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
1272{
1273 return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1274}
1275
1276static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
1277{
1278 return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1279}
1280
1281static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
1282{
1283 return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1284}
1285#endif
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
1298{
1299 memcpy(dst, src, count * sizeof(pgd_t));
1300#ifdef CONFIG_PAGE_TABLE_ISOLATION
1301 if (!static_cpu_has(X86_FEATURE_PTI))
1302 return;
1303
1304 memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
1305 count * sizeof(pgd_t));
1306#endif
1307}
1308
1309#define PTE_SHIFT ilog2(PTRS_PER_PTE)
1310static inline int page_level_shift(enum pg_level level)
1311{
1312 return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
1313}
1314static inline unsigned long page_level_size(enum pg_level level)
1315{
1316 return 1UL << page_level_shift(level);
1317}
1318static inline unsigned long page_level_mask(enum pg_level level)
1319{
1320 return ~(page_level_size(level) - 1);
1321}
1322
1323
1324
1325
1326
1327static inline void update_mmu_cache(struct vm_area_struct *vma,
1328 unsigned long addr, pte_t *ptep)
1329{
1330}
1331static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
1332 unsigned long addr, pmd_t *pmd)
1333{
1334}
1335static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
1336 unsigned long addr, pud_t *pud)
1337{
1338}
1339
1340#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1341static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1342{
1343 return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1344}
1345
1346static inline int pte_swp_soft_dirty(pte_t pte)
1347{
1348 return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
1349}
1350
1351static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1352{
1353 return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1354}
1355
1356#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1357static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1358{
1359 return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1360}
1361
1362static inline int pmd_swp_soft_dirty(pmd_t pmd)
1363{
1364 return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY;
1365}
1366
1367static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1368{
1369 return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1370}
1371#endif
1372#endif
1373
1374#define PKRU_AD_BIT 0x1
1375#define PKRU_WD_BIT 0x2
1376#define PKRU_BITS_PER_PKEY 2
1377
1378#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1379extern u32 init_pkru_value;
1380#else
1381#define init_pkru_value 0
1382#endif
1383
1384static inline bool __pkru_allows_read(u32 pkru, u16 pkey)
1385{
1386 int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1387 return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits));
1388}
1389
1390static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
1391{
1392 int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1393
1394
1395
1396
1397 return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits));
1398}
1399
1400static inline u16 pte_flags_pkey(unsigned long pte_flags)
1401{
1402#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1403
1404 return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
1405#else
1406 return 0;
1407#endif
1408}
1409
1410static inline bool __pkru_allows_pkey(u16 pkey, bool write)
1411{
1412 u32 pkru = read_pkru();
1413
1414 if (!__pkru_allows_read(pkru, pkey))
1415 return false;
1416 if (write && !__pkru_allows_write(pkru, pkey))
1417 return false;
1418
1419 return true;
1420}
1421
1422
1423
1424
1425
1426
1427static inline bool __pte_access_permitted(unsigned long pteval, bool write)
1428{
1429 unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
1430
1431 if (write)
1432 need_pte_bits |= _PAGE_RW;
1433
1434 if ((pteval & need_pte_bits) != need_pte_bits)
1435 return 0;
1436
1437 return __pkru_allows_pkey(pte_flags_pkey(pteval), write);
1438}
1439
1440#define pte_access_permitted pte_access_permitted
1441static inline bool pte_access_permitted(pte_t pte, bool write)
1442{
1443 return __pte_access_permitted(pte_val(pte), write);
1444}
1445
1446#define pmd_access_permitted pmd_access_permitted
1447static inline bool pmd_access_permitted(pmd_t pmd, bool write)
1448{
1449 return __pte_access_permitted(pmd_val(pmd), write);
1450}
1451
1452#define pud_access_permitted pud_access_permitted
1453static inline bool pud_access_permitted(pud_t pud, bool write)
1454{
1455 return __pte_access_permitted(pud_val(pud), write);
1456}
1457
1458#define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1
1459extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot);
1460
1461static inline bool arch_has_pfn_modify_check(void)
1462{
1463 return boot_cpu_has_bug(X86_BUG_L1TF);
1464}
1465
1466#define arch_faults_on_old_pte arch_faults_on_old_pte
1467static inline bool arch_faults_on_old_pte(void)
1468{
1469 return false;
1470}
1471
1472#include <asm-generic/pgtable.h>
1473#endif
1474
1475#endif
1476