1
2
3
4
5
6
7
8
9
10#include <linux/init.h>
11#include <linux/fs.h>
12#include <linux/mm.h>
13#include <linux/hugetlb.h>
14#include <linux/pagemap.h>
15#include <linux/err.h>
16#include <linux/sysctl.h>
17#include <asm/mman.h>
18#include <asm/tlb.h>
19#include <asm/tlbflush.h>
20#include <asm/pgalloc.h>
21
22#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
23bool arch_hugetlb_migration_supported(struct hstate *h)
24{
25 size_t pagesize = huge_page_size(h);
26
27 switch (pagesize) {
28#ifdef CONFIG_ARM64_4K_PAGES
29 case PUD_SIZE:
30#endif
31 case PMD_SIZE:
32 case CONT_PMD_SIZE:
33 case CONT_PTE_SIZE:
34 return true;
35 }
36 pr_warn("%s: unrecognized huge page size 0x%lx\n",
37 __func__, pagesize);
38 return false;
39}
40#endif
41
42int pmd_huge(pmd_t pmd)
43{
44 return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
45}
46
47int pud_huge(pud_t pud)
48{
49#ifndef __PAGETABLE_PMD_FOLDED
50 return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
51#else
52 return 0;
53#endif
54}
55
56
57
58
59static inline pgprot_t pte_pgprot(pte_t pte)
60{
61 unsigned long pfn = pte_pfn(pte);
62
63 return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
64}
65
66static int find_num_contig(struct mm_struct *mm, unsigned long addr,
67 pte_t *ptep, size_t *pgsize)
68{
69 pgd_t *pgdp = pgd_offset(mm, addr);
70 pud_t *pudp;
71 pmd_t *pmdp;
72
73 *pgsize = PAGE_SIZE;
74 pudp = pud_offset(pgdp, addr);
75 pmdp = pmd_offset(pudp, addr);
76 if ((pte_t *)pmdp == ptep) {
77 *pgsize = PMD_SIZE;
78 return CONT_PMDS;
79 }
80 return CONT_PTES;
81}
82
83static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
84{
85 int contig_ptes = 0;
86
87 *pgsize = size;
88
89 switch (size) {
90#ifdef CONFIG_ARM64_4K_PAGES
91 case PUD_SIZE:
92#endif
93 case PMD_SIZE:
94 contig_ptes = 1;
95 break;
96 case CONT_PMD_SIZE:
97 *pgsize = PMD_SIZE;
98 contig_ptes = CONT_PMDS;
99 break;
100 case CONT_PTE_SIZE:
101 *pgsize = PAGE_SIZE;
102 contig_ptes = CONT_PTES;
103 break;
104 }
105
106 return contig_ptes;
107}
108
109
110
111
112
113
114
115
116
117static pte_t get_clear_flush(struct mm_struct *mm,
118 unsigned long addr,
119 pte_t *ptep,
120 unsigned long pgsize,
121 unsigned long ncontig)
122{
123 pte_t orig_pte = huge_ptep_get(ptep);
124 bool valid = pte_valid(orig_pte);
125 unsigned long i, saddr = addr;
126
127 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
128 pte_t pte = ptep_get_and_clear(mm, addr, ptep);
129
130
131
132
133
134
135 if (pte_dirty(pte))
136 orig_pte = pte_mkdirty(orig_pte);
137
138 if (pte_young(pte))
139 orig_pte = pte_mkyoung(orig_pte);
140 }
141
142 if (valid) {
143 struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
144 flush_tlb_range(&vma, saddr, addr);
145 }
146 return orig_pte;
147}
148
149
150
151
152
153
154
155
156
157
158static void clear_flush(struct mm_struct *mm,
159 unsigned long addr,
160 pte_t *ptep,
161 unsigned long pgsize,
162 unsigned long ncontig)
163{
164 struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
165 unsigned long i, saddr = addr;
166
167 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
168 pte_clear(mm, addr, ptep);
169
170 flush_tlb_range(&vma, saddr, addr);
171}
172
173void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
174 pte_t *ptep, pte_t pte)
175{
176 size_t pgsize;
177 int i;
178 int ncontig;
179 unsigned long pfn, dpfn;
180 pgprot_t hugeprot;
181
182
183
184
185
186 WARN_ON(!pte_present(pte));
187
188 if (!pte_cont(pte)) {
189 set_pte_at(mm, addr, ptep, pte);
190 return;
191 }
192
193 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
194 pfn = pte_pfn(pte);
195 dpfn = pgsize >> PAGE_SHIFT;
196 hugeprot = pte_pgprot(pte);
197
198 clear_flush(mm, addr, ptep, pgsize, ncontig);
199
200 for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
201 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
202}
203
204void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
205 pte_t *ptep, pte_t pte, unsigned long sz)
206{
207 int i, ncontig;
208 size_t pgsize;
209
210 ncontig = num_contig_ptes(sz, &pgsize);
211
212 for (i = 0; i < ncontig; i++, ptep++)
213 set_pte(ptep, pte);
214}
215
216pte_t *huge_pte_alloc(struct mm_struct *mm,
217 unsigned long addr, unsigned long sz)
218{
219 pgd_t *pgdp;
220 pud_t *pudp;
221 pmd_t *pmdp;
222 pte_t *ptep = NULL;
223
224 pgdp = pgd_offset(mm, addr);
225 pudp = pud_alloc(mm, pgdp, addr);
226 if (!pudp)
227 return NULL;
228
229 if (sz == PUD_SIZE) {
230 ptep = (pte_t *)pudp;
231 } else if (sz == (CONT_PTE_SIZE)) {
232 pmdp = pmd_alloc(mm, pudp, addr);
233
234 WARN_ON(addr & (sz - 1));
235
236
237
238
239
240
241
242 ptep = pte_alloc_map(mm, pmdp, addr);
243 } else if (sz == PMD_SIZE) {
244 if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
245 pud_none(READ_ONCE(*pudp)))
246 ptep = huge_pmd_share(mm, addr, pudp);
247 else
248 ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
249 } else if (sz == (CONT_PMD_SIZE)) {
250 pmdp = pmd_alloc(mm, pudp, addr);
251 WARN_ON(addr & (sz - 1));
252 return (pte_t *)pmdp;
253 }
254
255 return ptep;
256}
257
258pte_t *huge_pte_offset(struct mm_struct *mm,
259 unsigned long addr, unsigned long sz)
260{
261 pgd_t *pgdp;
262 pud_t *pudp, pud;
263 pmd_t *pmdp, pmd;
264
265 pgdp = pgd_offset(mm, addr);
266 if (!pgd_present(READ_ONCE(*pgdp)))
267 return NULL;
268
269 pudp = pud_offset(pgdp, addr);
270 pud = READ_ONCE(*pudp);
271 if (sz != PUD_SIZE && pud_none(pud))
272 return NULL;
273
274 if (pud_huge(pud) || !pud_present(pud))
275 return (pte_t *)pudp;
276
277
278 if (sz == CONT_PMD_SIZE)
279 addr &= CONT_PMD_MASK;
280
281 pmdp = pmd_offset(pudp, addr);
282 pmd = READ_ONCE(*pmdp);
283 if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
284 pmd_none(pmd))
285 return NULL;
286 if (pmd_huge(pmd) || !pmd_present(pmd))
287 return (pte_t *)pmdp;
288
289 if (sz == CONT_PTE_SIZE)
290 return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK));
291
292 return NULL;
293}
294
295pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
296 struct page *page, int writable)
297{
298 size_t pagesize = huge_page_size(hstate_vma(vma));
299
300 if (pagesize == CONT_PTE_SIZE) {
301 entry = pte_mkcont(entry);
302 } else if (pagesize == CONT_PMD_SIZE) {
303 entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
304 } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
305 pr_warn("%s: unrecognized huge page size 0x%lx\n",
306 __func__, pagesize);
307 }
308 return entry;
309}
310
311void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
312 pte_t *ptep, unsigned long sz)
313{
314 int i, ncontig;
315 size_t pgsize;
316
317 ncontig = num_contig_ptes(sz, &pgsize);
318
319 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
320 pte_clear(mm, addr, ptep);
321}
322
323pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
324 unsigned long addr, pte_t *ptep)
325{
326 int ncontig;
327 size_t pgsize;
328 pte_t orig_pte = huge_ptep_get(ptep);
329
330 if (!pte_cont(orig_pte))
331 return ptep_get_and_clear(mm, addr, ptep);
332
333 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
334
335 return get_clear_flush(mm, addr, ptep, pgsize, ncontig);
336}
337
338
339
340
341
342
343
344
345
346
347static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
348{
349 int i;
350
351 if (pte_write(pte) != pte_write(huge_ptep_get(ptep)))
352 return 1;
353
354 for (i = 0; i < ncontig; i++) {
355 pte_t orig_pte = huge_ptep_get(ptep + i);
356
357 if (pte_dirty(pte) != pte_dirty(orig_pte))
358 return 1;
359
360 if (pte_young(pte) != pte_young(orig_pte))
361 return 1;
362 }
363
364 return 0;
365}
366
367int huge_ptep_set_access_flags(struct vm_area_struct *vma,
368 unsigned long addr, pte_t *ptep,
369 pte_t pte, int dirty)
370{
371 int ncontig, i;
372 size_t pgsize = 0;
373 unsigned long pfn = pte_pfn(pte), dpfn;
374 pgprot_t hugeprot;
375 pte_t orig_pte;
376
377 if (!pte_cont(pte))
378 return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
379
380 ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
381 dpfn = pgsize >> PAGE_SHIFT;
382
383 if (!__cont_access_flags_changed(ptep, pte, ncontig))
384 return 0;
385
386 orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
387
388
389 if (pte_dirty(orig_pte))
390 pte = pte_mkdirty(pte);
391
392 if (pte_young(orig_pte))
393 pte = pte_mkyoung(pte);
394
395 hugeprot = pte_pgprot(pte);
396 for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
397 set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));
398
399 return 1;
400}
401
402void huge_ptep_set_wrprotect(struct mm_struct *mm,
403 unsigned long addr, pte_t *ptep)
404{
405 unsigned long pfn, dpfn;
406 pgprot_t hugeprot;
407 int ncontig, i;
408 size_t pgsize;
409 pte_t pte;
410
411 if (!pte_cont(READ_ONCE(*ptep))) {
412 ptep_set_wrprotect(mm, addr, ptep);
413 return;
414 }
415
416 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
417 dpfn = pgsize >> PAGE_SHIFT;
418
419 pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig);
420 pte = pte_wrprotect(pte);
421
422 hugeprot = pte_pgprot(pte);
423 pfn = pte_pfn(pte);
424
425 for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
426 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
427}
428
429void huge_ptep_clear_flush(struct vm_area_struct *vma,
430 unsigned long addr, pte_t *ptep)
431{
432 size_t pgsize;
433 int ncontig;
434
435 if (!pte_cont(READ_ONCE(*ptep))) {
436 ptep_clear_flush(vma, addr, ptep);
437 return;
438 }
439
440 ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
441 clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
442}
443
444static void __init add_huge_page_size(unsigned long size)
445{
446 if (size_to_hstate(size))
447 return;
448
449 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
450}
451
452static int __init hugetlbpage_init(void)
453{
454#ifdef CONFIG_ARM64_4K_PAGES
455 add_huge_page_size(PUD_SIZE);
456#endif
457 add_huge_page_size(CONT_PMD_SIZE);
458 add_huge_page_size(PMD_SIZE);
459 add_huge_page_size(CONT_PTE_SIZE);
460
461 return 0;
462}
463arch_initcall(hugetlbpage_init);
464
465static __init int setup_hugepagesz(char *opt)
466{
467 unsigned long ps = memparse(opt, &opt);
468
469 switch (ps) {
470#ifdef CONFIG_ARM64_4K_PAGES
471 case PUD_SIZE:
472#endif
473 case CONT_PMD_SIZE:
474 case PMD_SIZE:
475 case CONT_PTE_SIZE:
476 add_huge_page_size(ps);
477 return 1;
478 }
479
480 hugetlb_bad_size();
481 pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
482 return 0;
483}
484__setup("hugepagesz=", setup_hugepagesz);
485