1
2
3
4
5
6
7
8
9
10
11
12#include <linux/sched.h>
13#include <linux/mm_types.h>
14#include <linux/mm.h>
15
16#include <asm/pgalloc.h>
17#include <asm/pgtable.h>
18#include <asm/sections.h>
19#include <asm/mmu.h>
20#include <asm/tlb.h>
21
22#include "mmu_decl.h"
23
24#define CREATE_TRACE_POINTS
25#include <trace/events/thp.h>
26
27#if H_PGTABLE_RANGE > (USER_VSID_RANGE * (TASK_SIZE_USER64 / TASK_CONTEXT_SIZE))
28#warning Limited user VSID range means pagetable space is wasted
29#endif
30
31#ifdef CONFIG_SPARSEMEM_VMEMMAP
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111int __meminit hash__vmemmap_create_mapping(unsigned long start,
112 unsigned long page_size,
113 unsigned long phys)
114{
115 int rc = htab_bolt_mapping(start, start + page_size, phys,
116 pgprot_val(PAGE_KERNEL),
117 mmu_vmemmap_psize, mmu_kernel_ssize);
118 if (rc < 0) {
119 int rc2 = htab_remove_mapping(start, start + page_size,
120 mmu_vmemmap_psize,
121 mmu_kernel_ssize);
122 BUG_ON(rc2 && (rc2 != -ENOENT));
123 }
124 return rc;
125}
126
127#ifdef CONFIG_MEMORY_HOTPLUG
128void hash__vmemmap_remove_mapping(unsigned long start,
129 unsigned long page_size)
130{
131 int rc = htab_remove_mapping(start, start + page_size,
132 mmu_vmemmap_psize,
133 mmu_kernel_ssize);
134 BUG_ON((rc < 0) && (rc != -ENOENT));
135 WARN_ON(rc == -ENOENT);
136}
137#endif
138#endif
139
140
141
142
143
144
145int hash__map_kernel_page(unsigned long ea, unsigned long pa, unsigned long flags)
146{
147 pgd_t *pgdp;
148 pud_t *pudp;
149 pmd_t *pmdp;
150 pte_t *ptep;
151
152 BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE);
153 if (slab_is_available()) {
154 pgdp = pgd_offset_k(ea);
155 pudp = pud_alloc(&init_mm, pgdp, ea);
156 if (!pudp)
157 return -ENOMEM;
158 pmdp = pmd_alloc(&init_mm, pudp, ea);
159 if (!pmdp)
160 return -ENOMEM;
161 ptep = pte_alloc_kernel(pmdp, ea);
162 if (!ptep)
163 return -ENOMEM;
164 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT,
165 __pgprot(flags)));
166 } else {
167
168
169
170
171
172
173 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags,
174 mmu_io_psize, mmu_kernel_ssize)) {
175 printk(KERN_ERR "Failed to do bolted mapping IO "
176 "memory at %016lx !\n", pa);
177 return -ENOMEM;
178 }
179 }
180
181 smp_wmb();
182 return 0;
183}
184
185#ifdef CONFIG_TRANSPARENT_HUGEPAGE
186
187unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
188 pmd_t *pmdp, unsigned long clr,
189 unsigned long set)
190{
191 __be64 old_be, tmp;
192 unsigned long old;
193
194#ifdef CONFIG_DEBUG_VM
195 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
196 assert_spin_locked(&mm->page_table_lock);
197#endif
198
199 __asm__ __volatile__(
200 "1: ldarx %0,0,%3\n\
201 and. %1,%0,%6\n\
202 bne- 1b \n\
203 andc %1,%0,%4 \n\
204 or %1,%1,%7\n\
205 stdcx. %1,0,%3 \n\
206 bne- 1b"
207 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
208 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
209 "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
210 : "cc" );
211
212 old = be64_to_cpu(old_be);
213
214 trace_hugepage_update(addr, old, clr, set);
215 if (old & H_PAGE_HASHPTE)
216 hpte_do_hugepage_flush(mm, addr, pmdp, old);
217 return old;
218}
219
220pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
221 pmd_t *pmdp)
222{
223 pmd_t pmd;
224
225 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
226 VM_BUG_ON(pmd_trans_huge(*pmdp));
227 VM_BUG_ON(pmd_devmap(*pmdp));
228
229 pmd = *pmdp;
230 pmd_clear(pmdp);
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246 serialize_against_pte_lookup(vma->vm_mm);
247
248
249
250
251
252
253
254
255
256 flush_tlb_pmd_range(vma->vm_mm, &pmd, address);
257 return pmd;
258}
259
260
261
262
263
264void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
265 pgtable_t pgtable)
266{
267 pgtable_t *pgtable_slot;
268 assert_spin_locked(&mm->page_table_lock);
269
270
271
272 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
273 *pgtable_slot = pgtable;
274
275
276
277
278
279
280 smp_wmb();
281}
282
283pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
284{
285 pgtable_t pgtable;
286 pgtable_t *pgtable_slot;
287
288 assert_spin_locked(&mm->page_table_lock);
289 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
290 pgtable = *pgtable_slot;
291
292
293
294 *pgtable_slot = NULL;
295
296
297
298
299 memset(pgtable, 0, PTE_FRAG_SIZE);
300 return pgtable;
301}
302
303
304
305
306
307void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
308 pmd_t *pmdp, unsigned long old_pmd)
309{
310 int ssize;
311 unsigned int psize;
312 unsigned long vsid;
313 unsigned long flags = 0;
314
315
316#ifdef CONFIG_DEBUG_VM
317 psize = get_slice_psize(mm, addr);
318 BUG_ON(psize == MMU_PAGE_16M);
319#endif
320 if (old_pmd & H_PAGE_COMBO)
321 psize = MMU_PAGE_4K;
322 else
323 psize = MMU_PAGE_64K;
324
325 if (!is_kernel_addr(addr)) {
326 ssize = user_segment_size(addr);
327 vsid = get_user_vsid(&mm->context, addr, ssize);
328 WARN_ON(vsid == 0);
329 } else {
330 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
331 ssize = mmu_kernel_ssize;
332 }
333
334 if (mm_is_thread_local(mm))
335 flags |= HPTE_LOCAL_UPDATE;
336
337 return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
338}
339
340pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
341 unsigned long addr, pmd_t *pmdp)
342{
343 pmd_t old_pmd;
344 pgtable_t pgtable;
345 unsigned long old;
346 pgtable_t *pgtable_slot;
347
348 old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
349 old_pmd = __pmd(old);
350
351
352
353
354
355 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
356 pgtable = *pgtable_slot;
357
358
359
360
361 memset(pgtable, 0, PTE_FRAG_SIZE);
362
363
364
365
366
367
368
369
370
371
372 serialize_against_pte_lookup(mm);
373 return old_pmd;
374}
375
376int hash__has_transparent_hugepage(void)
377{
378
379 if (!mmu_has_feature(MMU_FTR_16M_PAGE))
380 return 0;
381
382
383
384 if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
385 return 0;
386
387
388
389
390
391
392
393
394 if (mmu_psize_defs[MMU_PAGE_64K].shift &&
395 (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
396 return 0;
397
398
399
400 if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
401 return 0;
402
403 return 1;
404}
405#endif
406
407#ifdef CONFIG_STRICT_KERNEL_RWX
408static bool hash__change_memory_range(unsigned long start, unsigned long end,
409 unsigned long newpp)
410{
411 unsigned long idx;
412 unsigned int step, shift;
413
414 shift = mmu_psize_defs[mmu_linear_psize].shift;
415 step = 1 << shift;
416
417 start = ALIGN_DOWN(start, step);
418 end = ALIGN(end, step);
419
420 if (start >= end)
421 return false;
422
423 pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",
424 start, end, newpp, step);
425
426 for (idx = start; idx < end; idx += step)
427
428 mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,
429 mmu_kernel_ssize);
430
431 return true;
432}
433
434void hash__mark_rodata_ro(void)
435{
436 unsigned long start, end;
437
438 start = (unsigned long)_stext;
439 end = (unsigned long)__init_begin;
440
441 WARN_ON(!hash__change_memory_range(start, end, PP_RXXX));
442}
443
444void hash__mark_initmem_nx(void)
445{
446 unsigned long start, end, pp;
447
448 start = (unsigned long)__init_begin;
449 end = (unsigned long)__init_end;
450
451 pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL));
452
453 WARN_ON(!hash__change_memory_range(start, end, pp));
454}
455#endif
456