1
2
3
4
5
6
7
8
9
10
11
12#include <linux/sched.h>
13#include <linux/mm_types.h>
14#include <linux/mm.h>
15
16#include <asm/pgalloc.h>
17#include <asm/pgtable.h>
18#include <asm/sections.h>
19#include <asm/mmu.h>
20#include <asm/tlb.h>
21
22#include "mmu_decl.h"
23
24#define CREATE_TRACE_POINTS
25#include <trace/events/thp.h>
26
27#ifdef CONFIG_SPARSEMEM_VMEMMAP
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107int __meminit hash__vmemmap_create_mapping(unsigned long start,
108 unsigned long page_size,
109 unsigned long phys)
110{
111 int rc = htab_bolt_mapping(start, start + page_size, phys,
112 pgprot_val(PAGE_KERNEL),
113 mmu_vmemmap_psize, mmu_kernel_ssize);
114 if (rc < 0) {
115 int rc2 = htab_remove_mapping(start, start + page_size,
116 mmu_vmemmap_psize,
117 mmu_kernel_ssize);
118 BUG_ON(rc2 && (rc2 != -ENOENT));
119 }
120 return rc;
121}
122
123#ifdef CONFIG_MEMORY_HOTPLUG
124void hash__vmemmap_remove_mapping(unsigned long start,
125 unsigned long page_size)
126{
127 int rc = htab_remove_mapping(start, start + page_size,
128 mmu_vmemmap_psize,
129 mmu_kernel_ssize);
130 BUG_ON((rc < 0) && (rc != -ENOENT));
131 WARN_ON(rc == -ENOENT);
132}
133#endif
134#endif
135
136
137
138
139
140
141int hash__map_kernel_page(unsigned long ea, unsigned long pa, unsigned long flags)
142{
143 pgd_t *pgdp;
144 pud_t *pudp;
145 pmd_t *pmdp;
146 pte_t *ptep;
147
148 BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE);
149 if (slab_is_available()) {
150 pgdp = pgd_offset_k(ea);
151 pudp = pud_alloc(&init_mm, pgdp, ea);
152 if (!pudp)
153 return -ENOMEM;
154 pmdp = pmd_alloc(&init_mm, pudp, ea);
155 if (!pmdp)
156 return -ENOMEM;
157 ptep = pte_alloc_kernel(pmdp, ea);
158 if (!ptep)
159 return -ENOMEM;
160 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT,
161 __pgprot(flags)));
162 } else {
163
164
165
166
167
168
169 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags,
170 mmu_io_psize, mmu_kernel_ssize)) {
171 printk(KERN_ERR "Failed to do bolted mapping IO "
172 "memory at %016lx !\n", pa);
173 return -ENOMEM;
174 }
175 }
176
177 smp_wmb();
178 return 0;
179}
180
181#ifdef CONFIG_TRANSPARENT_HUGEPAGE
182
183unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
184 pmd_t *pmdp, unsigned long clr,
185 unsigned long set)
186{
187 __be64 old_be, tmp;
188 unsigned long old;
189
190#ifdef CONFIG_DEBUG_VM
191 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
192 assert_spin_locked(&mm->page_table_lock);
193#endif
194
195 __asm__ __volatile__(
196 "1: ldarx %0,0,%3\n\
197 and. %1,%0,%6\n\
198 bne- 1b \n\
199 andc %1,%0,%4 \n\
200 or %1,%1,%7\n\
201 stdcx. %1,0,%3 \n\
202 bne- 1b"
203 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
204 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
205 "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
206 : "cc" );
207
208 old = be64_to_cpu(old_be);
209
210 trace_hugepage_update(addr, old, clr, set);
211 if (old & H_PAGE_HASHPTE)
212 hpte_do_hugepage_flush(mm, addr, pmdp, old);
213 return old;
214}
215
216pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
217 pmd_t *pmdp)
218{
219 pmd_t pmd;
220
221 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
222 VM_BUG_ON(pmd_trans_huge(*pmdp));
223 VM_BUG_ON(pmd_devmap(*pmdp));
224
225 pmd = *pmdp;
226 pmd_clear(pmdp);
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242 serialize_against_pte_lookup(vma->vm_mm);
243
244
245
246
247
248
249
250
251
252 flush_tlb_pmd_range(vma->vm_mm, &pmd, address);
253 return pmd;
254}
255
256
257
258
259
260void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
261 pgtable_t pgtable)
262{
263 pgtable_t *pgtable_slot;
264 assert_spin_locked(&mm->page_table_lock);
265
266
267
268 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
269 *pgtable_slot = pgtable;
270
271
272
273
274
275
276 smp_wmb();
277}
278
279pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
280{
281 pgtable_t pgtable;
282 pgtable_t *pgtable_slot;
283
284 assert_spin_locked(&mm->page_table_lock);
285 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
286 pgtable = *pgtable_slot;
287
288
289
290 *pgtable_slot = NULL;
291
292
293
294
295 memset(pgtable, 0, PTE_FRAG_SIZE);
296 return pgtable;
297}
298
299
300
301
302
303void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
304 pmd_t *pmdp, unsigned long old_pmd)
305{
306 int ssize;
307 unsigned int psize;
308 unsigned long vsid;
309 unsigned long flags = 0;
310
311
312#ifdef CONFIG_DEBUG_VM
313 psize = get_slice_psize(mm, addr);
314 BUG_ON(psize == MMU_PAGE_16M);
315#endif
316 if (old_pmd & H_PAGE_COMBO)
317 psize = MMU_PAGE_4K;
318 else
319 psize = MMU_PAGE_64K;
320
321 if (!is_kernel_addr(addr)) {
322 ssize = user_segment_size(addr);
323 vsid = get_vsid(mm->context.id, addr, ssize);
324 WARN_ON(vsid == 0);
325 } else {
326 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
327 ssize = mmu_kernel_ssize;
328 }
329
330 if (mm_is_thread_local(mm))
331 flags |= HPTE_LOCAL_UPDATE;
332
333 return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
334}
335
336pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
337 unsigned long addr, pmd_t *pmdp)
338{
339 pmd_t old_pmd;
340 pgtable_t pgtable;
341 unsigned long old;
342 pgtable_t *pgtable_slot;
343
344 old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
345 old_pmd = __pmd(old);
346
347
348
349
350
351 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
352 pgtable = *pgtable_slot;
353
354
355
356
357 memset(pgtable, 0, PTE_FRAG_SIZE);
358
359
360
361
362
363
364
365
366
367
368 serialize_against_pte_lookup(mm);
369 return old_pmd;
370}
371
372int hash__has_transparent_hugepage(void)
373{
374
375 if (!mmu_has_feature(MMU_FTR_16M_PAGE))
376 return 0;
377
378
379
380 if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
381 return 0;
382
383
384
385
386
387
388
389
390 if (mmu_psize_defs[MMU_PAGE_64K].shift &&
391 (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
392 return 0;
393
394
395
396 if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
397 return 0;
398
399 return 1;
400}
401#endif
402
403#ifdef CONFIG_STRICT_KERNEL_RWX
404static bool hash__change_memory_range(unsigned long start, unsigned long end,
405 unsigned long newpp)
406{
407 unsigned long idx;
408 unsigned int step, shift;
409
410 shift = mmu_psize_defs[mmu_linear_psize].shift;
411 step = 1 << shift;
412
413 start = ALIGN_DOWN(start, step);
414 end = ALIGN(end, step);
415
416 if (start >= end)
417 return false;
418
419 pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",
420 start, end, newpp, step);
421
422 for (idx = start; idx < end; idx += step)
423
424 mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,
425 mmu_kernel_ssize);
426
427 return true;
428}
429
430void hash__mark_rodata_ro(void)
431{
432 unsigned long start, end;
433
434 start = (unsigned long)_stext;
435 end = (unsigned long)__init_begin;
436
437 WARN_ON(!hash__change_memory_range(start, end, PP_RXXX));
438}
439
440void hash__mark_initmem_nx(void)
441{
442 unsigned long start, end, pp;
443
444 start = (unsigned long)__init_begin;
445 end = (unsigned long)__init_end;
446
447 pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL));
448
449 WARN_ON(!hash__change_memory_range(start, end, pp));
450}
451#endif
452