1
2
3
4
5
6
7
8#ifndef __ARM64_S2_PGTABLE_H_
9#define __ARM64_S2_PGTABLE_H_
10
11#include <linux/hugetlb.h>
12#include <asm/pgtable.h>
13
14
15
16
17
18
19#define pt_levels_pgdir_shift(lvls) ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - (lvls))
20
21
22
23
24
25
26
27
28
29
30
31#define stage2_pgtable_levels(ipa) ARM64_HW_PGTABLE_LEVELS((ipa) - 4)
32#define kvm_stage2_levels(kvm) VTCR_EL2_LVLS(kvm->arch.vtcr)
33
34
35#define stage2_pgdir_shift(kvm) pt_levels_pgdir_shift(kvm_stage2_levels(kvm))
36#define stage2_pgdir_size(kvm) (1ULL << stage2_pgdir_shift(kvm))
37#define stage2_pgdir_mask(kvm) ~(stage2_pgdir_size(kvm) - 1)
38
39
40
41
42
43
44
45#define pgd_ptrs_shift(ipa, pgdir_shift) \
46 ((ipa) > (pgdir_shift) ? ((ipa) - (pgdir_shift)) : 0)
47#define __s2_pgd_ptrs(ipa, lvls) \
48 (1 << (pgd_ptrs_shift((ipa), pt_levels_pgdir_shift(lvls))))
49#define __s2_pgd_size(ipa, lvls) (__s2_pgd_ptrs((ipa), (lvls)) * sizeof(pgd_t))
50
51#define stage2_pgd_ptrs(kvm) __s2_pgd_ptrs(kvm_phys_shift(kvm), kvm_stage2_levels(kvm))
52#define stage2_pgd_size(kvm) __s2_pgd_size(kvm_phys_shift(kvm), kvm_stage2_levels(kvm))
53
54
55
56
57
58
59#define kvm_mmu_cache_min_pages(kvm) (kvm_stage2_levels(kvm) - 1)
60
61
62static inline bool kvm_stage2_has_pud(struct kvm *kvm)
63{
64 return (CONFIG_PGTABLE_LEVELS > 3) && (kvm_stage2_levels(kvm) > 3);
65}
66
67#define S2_PUD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(1)
68#define S2_PUD_SIZE (1UL << S2_PUD_SHIFT)
69#define S2_PUD_MASK (~(S2_PUD_SIZE - 1))
70
71static inline bool stage2_pgd_none(struct kvm *kvm, pgd_t pgd)
72{
73 if (kvm_stage2_has_pud(kvm))
74 return pgd_none(pgd);
75 else
76 return 0;
77}
78
79static inline void stage2_pgd_clear(struct kvm *kvm, pgd_t *pgdp)
80{
81 if (kvm_stage2_has_pud(kvm))
82 pgd_clear(pgdp);
83}
84
85static inline bool stage2_pgd_present(struct kvm *kvm, pgd_t pgd)
86{
87 if (kvm_stage2_has_pud(kvm))
88 return pgd_present(pgd);
89 else
90 return 1;
91}
92
93static inline void stage2_pgd_populate(struct kvm *kvm, pgd_t *pgd, pud_t *pud)
94{
95 if (kvm_stage2_has_pud(kvm))
96 pgd_populate(NULL, pgd, pud);
97}
98
99static inline pud_t *stage2_pud_offset(struct kvm *kvm,
100 pgd_t *pgd, unsigned long address)
101{
102 if (kvm_stage2_has_pud(kvm))
103 return pud_offset(pgd, address);
104 else
105 return (pud_t *)pgd;
106}
107
108static inline void stage2_pud_free(struct kvm *kvm, pud_t *pud)
109{
110 if (kvm_stage2_has_pud(kvm))
111 free_page((unsigned long)pud);
112}
113
114static inline bool stage2_pud_table_empty(struct kvm *kvm, pud_t *pudp)
115{
116 if (kvm_stage2_has_pud(kvm))
117 return kvm_page_empty(pudp);
118 else
119 return false;
120}
121
122static inline phys_addr_t
123stage2_pud_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
124{
125 if (kvm_stage2_has_pud(kvm)) {
126 phys_addr_t boundary = (addr + S2_PUD_SIZE) & S2_PUD_MASK;
127
128 return (boundary - 1 < end - 1) ? boundary : end;
129 } else {
130 return end;
131 }
132}
133
134
135static inline bool kvm_stage2_has_pmd(struct kvm *kvm)
136{
137 return (CONFIG_PGTABLE_LEVELS > 2) && (kvm_stage2_levels(kvm) > 2);
138}
139
140#define S2_PMD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(2)
141#define S2_PMD_SIZE (1UL << S2_PMD_SHIFT)
142#define S2_PMD_MASK (~(S2_PMD_SIZE - 1))
143
144static inline bool stage2_pud_none(struct kvm *kvm, pud_t pud)
145{
146 if (kvm_stage2_has_pmd(kvm))
147 return pud_none(pud);
148 else
149 return 0;
150}
151
152static inline void stage2_pud_clear(struct kvm *kvm, pud_t *pud)
153{
154 if (kvm_stage2_has_pmd(kvm))
155 pud_clear(pud);
156}
157
158static inline bool stage2_pud_present(struct kvm *kvm, pud_t pud)
159{
160 if (kvm_stage2_has_pmd(kvm))
161 return pud_present(pud);
162 else
163 return 1;
164}
165
166static inline void stage2_pud_populate(struct kvm *kvm, pud_t *pud, pmd_t *pmd)
167{
168 if (kvm_stage2_has_pmd(kvm))
169 pud_populate(NULL, pud, pmd);
170}
171
172static inline pmd_t *stage2_pmd_offset(struct kvm *kvm,
173 pud_t *pud, unsigned long address)
174{
175 if (kvm_stage2_has_pmd(kvm))
176 return pmd_offset(pud, address);
177 else
178 return (pmd_t *)pud;
179}
180
181static inline void stage2_pmd_free(struct kvm *kvm, pmd_t *pmd)
182{
183 if (kvm_stage2_has_pmd(kvm))
184 free_page((unsigned long)pmd);
185}
186
187static inline bool stage2_pud_huge(struct kvm *kvm, pud_t pud)
188{
189 if (kvm_stage2_has_pmd(kvm))
190 return pud_huge(pud);
191 else
192 return 0;
193}
194
195static inline bool stage2_pmd_table_empty(struct kvm *kvm, pmd_t *pmdp)
196{
197 if (kvm_stage2_has_pmd(kvm))
198 return kvm_page_empty(pmdp);
199 else
200 return 0;
201}
202
203static inline phys_addr_t
204stage2_pmd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
205{
206 if (kvm_stage2_has_pmd(kvm)) {
207 phys_addr_t boundary = (addr + S2_PMD_SIZE) & S2_PMD_MASK;
208
209 return (boundary - 1 < end - 1) ? boundary : end;
210 } else {
211 return end;
212 }
213}
214
215static inline bool stage2_pte_table_empty(struct kvm *kvm, pte_t *ptep)
216{
217 return kvm_page_empty(ptep);
218}
219
220static inline unsigned long stage2_pgd_index(struct kvm *kvm, phys_addr_t addr)
221{
222 return (((addr) >> stage2_pgdir_shift(kvm)) & (stage2_pgd_ptrs(kvm) - 1));
223}
224
225static inline phys_addr_t
226stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
227{
228 phys_addr_t boundary = (addr + stage2_pgdir_size(kvm)) & stage2_pgdir_mask(kvm);
229
230 return (boundary - 1 < end - 1) ? boundary : end;
231}
232
233#endif
234