1
2
3
4
5
6
7
8
9
10
11
12#include <linux/sched.h>
13#include <linux/mm_types.h>
14#include <linux/mm.h>
15
16#include <asm/pgalloc.h>
17#include <asm/pgtable.h>
18#include <asm/sections.h>
19#include <asm/mmu.h>
20#include <asm/tlb.h>
21
22#include "mmu_decl.h"
23
24#define CREATE_TRACE_POINTS
25#include <trace/events/thp.h>
26
27#ifdef CONFIG_SPARSEMEM_VMEMMAP
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107int __meminit hash__vmemmap_create_mapping(unsigned long start,
108 unsigned long page_size,
109 unsigned long phys)
110{
111 int rc = htab_bolt_mapping(start, start + page_size, phys,
112 pgprot_val(PAGE_KERNEL),
113 mmu_vmemmap_psize, mmu_kernel_ssize);
114 if (rc < 0) {
115 int rc2 = htab_remove_mapping(start, start + page_size,
116 mmu_vmemmap_psize,
117 mmu_kernel_ssize);
118 BUG_ON(rc2 && (rc2 != -ENOENT));
119 }
120 return rc;
121}
122
123#ifdef CONFIG_MEMORY_HOTPLUG
124void hash__vmemmap_remove_mapping(unsigned long start,
125 unsigned long page_size)
126{
127 int rc = htab_remove_mapping(start, start + page_size,
128 mmu_vmemmap_psize,
129 mmu_kernel_ssize);
130 BUG_ON((rc < 0) && (rc != -ENOENT));
131 WARN_ON(rc == -ENOENT);
132}
133#endif
134#endif
135
136
137
138
139
140
141int hash__map_kernel_page(unsigned long ea, unsigned long pa, unsigned long flags)
142{
143 pgd_t *pgdp;
144 pud_t *pudp;
145 pmd_t *pmdp;
146 pte_t *ptep;
147
148 BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE);
149 if (slab_is_available()) {
150 pgdp = pgd_offset_k(ea);
151 pudp = pud_alloc(&init_mm, pgdp, ea);
152 if (!pudp)
153 return -ENOMEM;
154 pmdp = pmd_alloc(&init_mm, pudp, ea);
155 if (!pmdp)
156 return -ENOMEM;
157 ptep = pte_alloc_kernel(pmdp, ea);
158 if (!ptep)
159 return -ENOMEM;
160 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT,
161 __pgprot(flags)));
162 } else {
163
164
165
166
167
168
169 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags,
170 mmu_io_psize, mmu_kernel_ssize)) {
171 printk(KERN_ERR "Failed to do bolted mapping IO "
172 "memory at %016lx !\n", pa);
173 return -ENOMEM;
174 }
175 }
176
177 smp_wmb();
178 return 0;
179}
180
181#ifdef CONFIG_TRANSPARENT_HUGEPAGE
182
183unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
184 pmd_t *pmdp, unsigned long clr,
185 unsigned long set)
186{
187 __be64 old_be, tmp;
188 unsigned long old;
189
190#ifdef CONFIG_DEBUG_VM
191 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
192 assert_spin_locked(&mm->page_table_lock);
193#endif
194
195 __asm__ __volatile__(
196 "1: ldarx %0,0,%3\n\
197 and. %1,%0,%6\n\
198 bne- 1b \n\
199 andc %1,%0,%4 \n\
200 or %1,%1,%7\n\
201 stdcx. %1,0,%3 \n\
202 bne- 1b"
203 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
204 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
205 "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
206 : "cc" );
207
208 old = be64_to_cpu(old_be);
209
210 trace_hugepage_update(addr, old, clr, set);
211 if (old & H_PAGE_HASHPTE)
212 hpte_do_hugepage_flush(mm, addr, pmdp, old);
213 return old;
214}
215
216pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
217 pmd_t *pmdp)
218{
219 pmd_t pmd;
220
221 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
222 VM_BUG_ON(pmd_trans_huge(*pmdp));
223 VM_BUG_ON(pmd_devmap(*pmdp));
224
225 pmd = *pmdp;
226 pmd_clear(pmdp);
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242 serialize_against_pte_lookup(vma->vm_mm);
243
244
245
246
247
248
249
250
251
252 flush_tlb_pmd_range(vma->vm_mm, &pmd, address);
253 return pmd;
254}
255
256
257
258
259
260void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
261 pgtable_t pgtable)
262{
263 pgtable_t *pgtable_slot;
264 assert_spin_locked(&mm->page_table_lock);
265
266
267
268 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
269 *pgtable_slot = pgtable;
270
271
272
273
274
275
276 smp_wmb();
277}
278
279pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
280{
281 pgtable_t pgtable;
282 pgtable_t *pgtable_slot;
283
284 assert_spin_locked(&mm->page_table_lock);
285 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
286 pgtable = *pgtable_slot;
287
288
289
290 *pgtable_slot = NULL;
291
292
293
294
295 memset(pgtable, 0, PTE_FRAG_SIZE);
296 return pgtable;
297}
298
299void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
300 unsigned long address, pmd_t *pmdp)
301{
302 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
303 VM_BUG_ON(REGION_ID(address) != USER_REGION_ID);
304 VM_BUG_ON(pmd_devmap(*pmdp));
305
306
307
308
309
310
311
312
313
314
315
316
317
318 pmd_hugepage_update(vma->vm_mm, address, pmdp, 0, _PAGE_PRIVILEGED);
319}
320
321
322
323
324
325void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
326 pmd_t *pmdp, unsigned long old_pmd)
327{
328 int ssize;
329 unsigned int psize;
330 unsigned long vsid;
331 unsigned long flags = 0;
332
333
334#ifdef CONFIG_DEBUG_VM
335 psize = get_slice_psize(mm, addr);
336 BUG_ON(psize == MMU_PAGE_16M);
337#endif
338 if (old_pmd & H_PAGE_COMBO)
339 psize = MMU_PAGE_4K;
340 else
341 psize = MMU_PAGE_64K;
342
343 if (!is_kernel_addr(addr)) {
344 ssize = user_segment_size(addr);
345 vsid = get_vsid(mm->context.id, addr, ssize);
346 WARN_ON(vsid == 0);
347 } else {
348 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
349 ssize = mmu_kernel_ssize;
350 }
351
352 if (mm_is_thread_local(mm))
353 flags |= HPTE_LOCAL_UPDATE;
354
355 return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
356}
357
358pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
359 unsigned long addr, pmd_t *pmdp)
360{
361 pmd_t old_pmd;
362 pgtable_t pgtable;
363 unsigned long old;
364 pgtable_t *pgtable_slot;
365
366 old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
367 old_pmd = __pmd(old);
368
369
370
371
372
373 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
374 pgtable = *pgtable_slot;
375
376
377
378
379 memset(pgtable, 0, PTE_FRAG_SIZE);
380
381
382
383
384
385
386
387
388
389
390 serialize_against_pte_lookup(mm);
391 return old_pmd;
392}
393
394int hash__has_transparent_hugepage(void)
395{
396
397 if (!mmu_has_feature(MMU_FTR_16M_PAGE))
398 return 0;
399
400
401
402 if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
403 return 0;
404
405
406
407
408
409
410
411
412 if (mmu_psize_defs[MMU_PAGE_64K].shift &&
413 (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
414 return 0;
415
416
417
418 if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
419 return 0;
420
421 return 1;
422}
423#endif
424
425#ifdef CONFIG_STRICT_KERNEL_RWX
426static bool hash__change_memory_range(unsigned long start, unsigned long end,
427 unsigned long newpp)
428{
429 unsigned long idx;
430 unsigned int step, shift;
431
432 shift = mmu_psize_defs[mmu_linear_psize].shift;
433 step = 1 << shift;
434
435 start = ALIGN_DOWN(start, step);
436 end = ALIGN(end, step);
437
438 if (start >= end)
439 return false;
440
441 pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",
442 start, end, newpp, step);
443
444 for (idx = start; idx < end; idx += step)
445
446 mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,
447 mmu_kernel_ssize);
448
449 return true;
450}
451
452void hash__mark_rodata_ro(void)
453{
454 unsigned long start, end;
455
456 start = (unsigned long)_stext;
457 end = (unsigned long)__init_begin;
458
459 WARN_ON(!hash__change_memory_range(start, end, PP_RXXX));
460}
461
462void hash__mark_initmem_nx(void)
463{
464 unsigned long start, end, pp;
465
466 start = (unsigned long)__init_begin;
467 end = (unsigned long)__init_end;
468
469 pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL));
470
471 WARN_ON(!hash__change_memory_range(start, end, pp));
472}
473#endif
474