1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <asm/trans_pgd.h>
21#include <asm/pgalloc.h>
22#include <asm/pgtable.h>
23#include <linux/suspend.h>
24#include <linux/bug.h>
25#include <linux/mm.h>
26#include <linux/mmzone.h>
27
28static void *trans_alloc(struct trans_pgd_info *info)
29{
30 return info->trans_alloc_page(info->trans_alloc_arg);
31}
32
33static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr)
34{
35 pte_t pte = READ_ONCE(*src_ptep);
36
37 if (pte_valid(pte)) {
38
39
40
41
42
43 set_pte(dst_ptep, pte_mkwrite(pte));
44 } else if (debug_pagealloc_enabled() && !pte_none(pte)) {
45
46
47
48
49
50
51
52
53
54 BUG_ON(!pfn_valid(pte_pfn(pte)));
55
56 set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte)));
57 }
58}
59
60static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp,
61 pmd_t *src_pmdp, unsigned long start, unsigned long end)
62{
63 pte_t *src_ptep;
64 pte_t *dst_ptep;
65 unsigned long addr = start;
66
67 dst_ptep = trans_alloc(info);
68 if (!dst_ptep)
69 return -ENOMEM;
70 pmd_populate_kernel(NULL, dst_pmdp, dst_ptep);
71 dst_ptep = pte_offset_kernel(dst_pmdp, start);
72
73 src_ptep = pte_offset_kernel(src_pmdp, start);
74 do {
75 _copy_pte(dst_ptep, src_ptep, addr);
76 } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end);
77
78 return 0;
79}
80
81static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp,
82 pud_t *src_pudp, unsigned long start, unsigned long end)
83{
84 pmd_t *src_pmdp;
85 pmd_t *dst_pmdp;
86 unsigned long next;
87 unsigned long addr = start;
88
89 if (pud_none(READ_ONCE(*dst_pudp))) {
90 dst_pmdp = trans_alloc(info);
91 if (!dst_pmdp)
92 return -ENOMEM;
93 pud_populate(NULL, dst_pudp, dst_pmdp);
94 }
95 dst_pmdp = pmd_offset(dst_pudp, start);
96
97 src_pmdp = pmd_offset(src_pudp, start);
98 do {
99 pmd_t pmd = READ_ONCE(*src_pmdp);
100
101 next = pmd_addr_end(addr, end);
102 if (pmd_none(pmd))
103 continue;
104 if (pmd_table(pmd)) {
105 if (copy_pte(info, dst_pmdp, src_pmdp, addr, next))
106 return -ENOMEM;
107 } else {
108 set_pmd(dst_pmdp,
109 __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY));
110 }
111 } while (dst_pmdp++, src_pmdp++, addr = next, addr != end);
112
113 return 0;
114}
115
116static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp,
117 p4d_t *src_p4dp, unsigned long start,
118 unsigned long end)
119{
120 pud_t *dst_pudp;
121 pud_t *src_pudp;
122 unsigned long next;
123 unsigned long addr = start;
124
125 if (p4d_none(READ_ONCE(*dst_p4dp))) {
126 dst_pudp = trans_alloc(info);
127 if (!dst_pudp)
128 return -ENOMEM;
129 p4d_populate(NULL, dst_p4dp, dst_pudp);
130 }
131 dst_pudp = pud_offset(dst_p4dp, start);
132
133 src_pudp = pud_offset(src_p4dp, start);
134 do {
135 pud_t pud = READ_ONCE(*src_pudp);
136
137 next = pud_addr_end(addr, end);
138 if (pud_none(pud))
139 continue;
140 if (pud_table(pud)) {
141 if (copy_pmd(info, dst_pudp, src_pudp, addr, next))
142 return -ENOMEM;
143 } else {
144 set_pud(dst_pudp,
145 __pud(pud_val(pud) & ~PUD_SECT_RDONLY));
146 }
147 } while (dst_pudp++, src_pudp++, addr = next, addr != end);
148
149 return 0;
150}
151
152static int copy_p4d(struct trans_pgd_info *info, pgd_t *dst_pgdp,
153 pgd_t *src_pgdp, unsigned long start,
154 unsigned long end)
155{
156 p4d_t *dst_p4dp;
157 p4d_t *src_p4dp;
158 unsigned long next;
159 unsigned long addr = start;
160
161 dst_p4dp = p4d_offset(dst_pgdp, start);
162 src_p4dp = p4d_offset(src_pgdp, start);
163 do {
164 next = p4d_addr_end(addr, end);
165 if (p4d_none(READ_ONCE(*src_p4dp)))
166 continue;
167 if (copy_pud(info, dst_p4dp, src_p4dp, addr, next))
168 return -ENOMEM;
169 } while (dst_p4dp++, src_p4dp++, addr = next, addr != end);
170
171 return 0;
172}
173
174static int copy_page_tables(struct trans_pgd_info *info, pgd_t *dst_pgdp,
175 unsigned long start, unsigned long end)
176{
177 unsigned long next;
178 unsigned long addr = start;
179 pgd_t *src_pgdp = pgd_offset_k(start);
180
181 dst_pgdp = pgd_offset_pgd(dst_pgdp, start);
182 do {
183 next = pgd_addr_end(addr, end);
184 if (pgd_none(READ_ONCE(*src_pgdp)))
185 continue;
186 if (copy_p4d(info, dst_pgdp, src_pgdp, addr, next))
187 return -ENOMEM;
188 } while (dst_pgdp++, src_pgdp++, addr = next, addr != end);
189
190 return 0;
191}
192
193
194
195
196
197
198
199
200
201
202int trans_pgd_create_copy(struct trans_pgd_info *info, pgd_t **dst_pgdp,
203 unsigned long start, unsigned long end)
204{
205 int rc;
206 pgd_t *trans_pgd = trans_alloc(info);
207
208 if (!trans_pgd) {
209 pr_err("Failed to allocate memory for temporary page tables.\n");
210 return -ENOMEM;
211 }
212
213 rc = copy_page_tables(info, trans_pgd, start, end);
214 if (!rc)
215 *dst_pgdp = trans_pgd;
216
217 return rc;
218}
219
220
221
222
223
224
225
226
227
228
229
230int trans_pgd_map_page(struct trans_pgd_info *info, pgd_t *trans_pgd,
231 void *page, unsigned long dst_addr, pgprot_t pgprot)
232{
233 pgd_t *pgdp;
234 p4d_t *p4dp;
235 pud_t *pudp;
236 pmd_t *pmdp;
237 pte_t *ptep;
238
239 pgdp = pgd_offset_pgd(trans_pgd, dst_addr);
240 if (pgd_none(READ_ONCE(*pgdp))) {
241 p4dp = trans_alloc(info);
242 if (!pgdp)
243 return -ENOMEM;
244 pgd_populate(NULL, pgdp, p4dp);
245 }
246
247 p4dp = p4d_offset(pgdp, dst_addr);
248 if (p4d_none(READ_ONCE(*p4dp))) {
249 pudp = trans_alloc(info);
250 if (!pudp)
251 return -ENOMEM;
252 p4d_populate(NULL, p4dp, pudp);
253 }
254
255 pudp = pud_offset(p4dp, dst_addr);
256 if (pud_none(READ_ONCE(*pudp))) {
257 pmdp = trans_alloc(info);
258 if (!pmdp)
259 return -ENOMEM;
260 pud_populate(NULL, pudp, pmdp);
261 }
262
263 pmdp = pmd_offset(pudp, dst_addr);
264 if (pmd_none(READ_ONCE(*pmdp))) {
265 ptep = trans_alloc(info);
266 if (!ptep)
267 return -ENOMEM;
268 pmd_populate_kernel(NULL, pmdp, ptep);
269 }
270
271 ptep = pte_offset_kernel(pmdp, dst_addr);
272 set_pte(ptep, pfn_pte(virt_to_pfn(page), pgprot));
273
274 return 0;
275}
276
277
278
279
280
281
282
283
284
285
286
287int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0,
288 unsigned long *t0sz, void *page)
289{
290 phys_addr_t dst_addr = virt_to_phys(page);
291 unsigned long pfn = __phys_to_pfn(dst_addr);
292 int max_msb = (dst_addr & GENMASK(52, 48)) ? 51 : 47;
293 int bits_mapped = PAGE_SHIFT - 4;
294 unsigned long level_mask, prev_level_entry, *levels[4];
295 int this_level, index, level_lsb, level_msb;
296
297 dst_addr &= PAGE_MASK;
298 prev_level_entry = pte_val(pfn_pte(pfn, PAGE_KERNEL_EXEC));
299
300 for (this_level = 3; this_level >= 0; this_level--) {
301 levels[this_level] = trans_alloc(info);
302 if (!levels[this_level])
303 return -ENOMEM;
304
305 level_lsb = ARM64_HW_PGTABLE_LEVEL_SHIFT(this_level);
306 level_msb = min(level_lsb + bits_mapped, max_msb);
307 level_mask = GENMASK_ULL(level_msb, level_lsb);
308
309 index = (dst_addr & level_mask) >> level_lsb;
310 *(levels[this_level] + index) = prev_level_entry;
311
312 pfn = virt_to_pfn(levels[this_level]);
313 prev_level_entry = pte_val(pfn_pte(pfn,
314 __pgprot(PMD_TYPE_TABLE)));
315
316 if (level_msb == max_msb)
317 break;
318 }
319
320 *trans_ttbr0 = phys_to_ttbr(__pfn_to_phys(pfn));
321 *t0sz = TCR_T0SZ(max_msb + 1);
322
323 return 0;
324}
325