1
2
3
4
5
6
7
8#include <asm/mman.h>
9#include <asm/mmu_context.h>
10#include <asm/mmu.h>
11#include <asm/setup.h>
12#include <linux/pkeys.h>
13#include <linux/of_device.h>
14
15DEFINE_STATIC_KEY_TRUE(pkey_disabled);
16int pkeys_total;
17u32 initial_allocation_mask;
18u32 reserved_allocation_mask;
19static bool pkey_execute_disable_supported;
20static bool pkeys_devtree_defined;
21static u64 pkey_amr_mask;
22static u64 pkey_iamr_mask;
23static u64 pkey_uamor_mask;
24static int execute_only_key = 2;
25
26#define AMR_BITS_PER_PKEY 2
27#define AMR_RD_BIT 0x1UL
28#define AMR_WR_BIT 0x2UL
29#define IAMR_EX_BIT 0x1UL
30#define PKEY_REG_BITS (sizeof(u64)*8)
31#define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
32
33static void scan_pkey_feature(void)
34{
35 u32 vals[2];
36 struct device_node *cpu;
37
38 cpu = of_find_node_by_type(NULL, "cpu");
39 if (!cpu)
40 return;
41
42 if (of_property_read_u32_array(cpu,
43 "ibm,processor-storage-keys", vals, 2))
44 return;
45
46
47
48
49
50 pkeys_total = vals[0];
51 pkeys_devtree_defined = true;
52}
53
54static inline bool pkey_mmu_enabled(void)
55{
56 if (firmware_has_feature(FW_FEATURE_LPAR))
57 return pkeys_total;
58 else
59 return cpu_has_feature(CPU_FTR_PKEY);
60}
61
62static int pkey_initialize(void)
63{
64 int os_reserved, i;
65
66
67
68
69
70
71 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
72 (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
73
74
75
76
77
78 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
79 __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
80 != (sizeof(u64) * BITS_PER_BYTE));
81
82
83 scan_pkey_feature();
84
85
86
87
88
89
90 if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) &&
91 cpu_has_feature(CPU_FTRS_POWER8))
92 pkeys_total = 32;
93
94
95
96
97
98 pkeys_total = min_t(int, pkeys_total,
99 ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)+1));
100
101 if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total)
102 static_branch_enable(&pkey_disabled);
103 else
104 static_branch_disable(&pkey_disabled);
105
106 if (static_branch_likely(&pkey_disabled))
107 return 0;
108
109
110
111
112
113 if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
114 pkey_execute_disable_supported = false;
115 else
116 pkey_execute_disable_supported = true;
117
118#ifdef CONFIG_PPC_4K_PAGES
119
120
121
122
123 os_reserved = pkeys_total - 8;
124#else
125 os_reserved = 0;
126#endif
127
128 reserved_allocation_mask = (0x1 << 1) | (0x1 << execute_only_key);
129
130
131 pkey_amr_mask = ~0x0ul;
132 pkey_amr_mask &= ~(0x3ul << pkeyshift(0));
133
134 pkey_iamr_mask = ~0x0ul;
135 pkey_iamr_mask &= ~(0x3ul << pkeyshift(0));
136 pkey_iamr_mask &= ~(0x3ul << pkeyshift(execute_only_key));
137
138 pkey_uamor_mask = ~0x0ul;
139 pkey_uamor_mask &= ~(0x3ul << pkeyshift(0));
140 pkey_uamor_mask &= ~(0x3ul << pkeyshift(execute_only_key));
141
142
143 for (i = (pkeys_total - os_reserved); i < pkeys_total; i++) {
144 reserved_allocation_mask |= (0x1 << i);
145 pkey_uamor_mask &= ~(0x3ul << pkeyshift(i));
146 }
147 initial_allocation_mask = reserved_allocation_mask | (0x1 << 0);
148
149 if (unlikely((pkeys_total - os_reserved) <= execute_only_key)) {
150
151
152
153
154
155
156
157 execute_only_key = -1;
158 }
159
160 return 0;
161}
162
163arch_initcall(pkey_initialize);
164
165void pkey_mm_init(struct mm_struct *mm)
166{
167 if (static_branch_likely(&pkey_disabled))
168 return;
169 mm_pkey_allocation_map(mm) = initial_allocation_mask;
170 mm->context.execute_only_pkey = execute_only_key;
171}
172
173static inline u64 read_amr(void)
174{
175 return mfspr(SPRN_AMR);
176}
177
178static inline void write_amr(u64 value)
179{
180 mtspr(SPRN_AMR, value);
181}
182
183static inline u64 read_iamr(void)
184{
185 if (!likely(pkey_execute_disable_supported))
186 return 0x0UL;
187
188 return mfspr(SPRN_IAMR);
189}
190
191static inline void write_iamr(u64 value)
192{
193 if (!likely(pkey_execute_disable_supported))
194 return;
195
196 mtspr(SPRN_IAMR, value);
197}
198
199static inline u64 read_uamor(void)
200{
201 return mfspr(SPRN_UAMOR);
202}
203
204static inline void write_uamor(u64 value)
205{
206 mtspr(SPRN_UAMOR, value);
207}
208
209static bool is_pkey_enabled(int pkey)
210{
211 u64 uamor = read_uamor();
212 u64 pkey_bits = 0x3ul << pkeyshift(pkey);
213 u64 uamor_pkey_bits = (uamor & pkey_bits);
214
215
216
217
218
219 WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits));
220 return !!(uamor_pkey_bits);
221}
222
223static inline void init_amr(int pkey, u8 init_bits)
224{
225 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
226 u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
227
228 write_amr(old_amr | new_amr_bits);
229}
230
231static inline void init_iamr(int pkey, u8 init_bits)
232{
233 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
234 u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
235
236 write_iamr(old_iamr | new_iamr_bits);
237}
238
239
240
241
242
243int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
244 unsigned long init_val)
245{
246 u64 new_amr_bits = 0x0ul;
247 u64 new_iamr_bits = 0x0ul;
248
249 if (!is_pkey_enabled(pkey))
250 return -EINVAL;
251
252 if (init_val & PKEY_DISABLE_EXECUTE) {
253 if (!pkey_execute_disable_supported)
254 return -EINVAL;
255 new_iamr_bits |= IAMR_EX_BIT;
256 }
257 init_iamr(pkey, new_iamr_bits);
258
259
260 if (init_val & PKEY_DISABLE_ACCESS)
261 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
262 else if (init_val & PKEY_DISABLE_WRITE)
263 new_amr_bits |= AMR_WR_BIT;
264
265 init_amr(pkey, new_amr_bits);
266 return 0;
267}
268
269void thread_pkey_regs_save(struct thread_struct *thread)
270{
271 if (static_branch_likely(&pkey_disabled))
272 return;
273
274
275
276
277 thread->amr = read_amr();
278 thread->iamr = read_iamr();
279 thread->uamor = read_uamor();
280}
281
282void thread_pkey_regs_restore(struct thread_struct *new_thread,
283 struct thread_struct *old_thread)
284{
285 if (static_branch_likely(&pkey_disabled))
286 return;
287
288 if (old_thread->amr != new_thread->amr)
289 write_amr(new_thread->amr);
290 if (old_thread->iamr != new_thread->iamr)
291 write_iamr(new_thread->iamr);
292 if (old_thread->uamor != new_thread->uamor)
293 write_uamor(new_thread->uamor);
294}
295
296void thread_pkey_regs_init(struct thread_struct *thread)
297{
298 if (static_branch_likely(&pkey_disabled))
299 return;
300
301 thread->amr = pkey_amr_mask;
302 thread->iamr = pkey_iamr_mask;
303 thread->uamor = pkey_uamor_mask;
304
305 write_uamor(pkey_uamor_mask);
306 write_amr(pkey_amr_mask);
307 write_iamr(pkey_iamr_mask);
308}
309
310static inline bool pkey_allows_readwrite(int pkey)
311{
312 int pkey_shift = pkeyshift(pkey);
313
314 if (!is_pkey_enabled(pkey))
315 return true;
316
317 return !(read_amr() & ((AMR_RD_BIT|AMR_WR_BIT) << pkey_shift));
318}
319
320int __execute_only_pkey(struct mm_struct *mm)
321{
322 return mm->context.execute_only_pkey;
323}
324
325static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma)
326{
327
328 if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC)
329 return false;
330
331 return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey);
332}
333
334
335
336
337int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot,
338 int pkey)
339{
340
341
342
343
344 if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC))
345 return 0;
346
347
348
349
350
351 if (prot == PROT_EXEC) {
352 pkey = execute_only_pkey(vma->vm_mm);
353 if (pkey > 0)
354 return pkey;
355 }
356
357
358 return vma_pkey(vma);
359}
360
361static bool pkey_access_permitted(int pkey, bool write, bool execute)
362{
363 int pkey_shift;
364 u64 amr;
365
366 if (!is_pkey_enabled(pkey))
367 return true;
368
369 pkey_shift = pkeyshift(pkey);
370 if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift)))
371 return true;
372
373 amr = read_amr();
374 return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) ||
375 (write && !(amr & (AMR_WR_BIT << pkey_shift))));
376}
377
378bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
379{
380 if (static_branch_likely(&pkey_disabled))
381 return true;
382
383 return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
384}
385
386
387
388
389
390
391
392
393
394static inline bool vma_is_foreign(struct vm_area_struct *vma)
395{
396 if (!current->mm)
397 return true;
398
399
400 if (current->mm != vma->vm_mm)
401 return true;
402
403 return false;
404}
405
406bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
407 bool execute, bool foreign)
408{
409 if (static_branch_likely(&pkey_disabled))
410 return true;
411
412
413
414 if (foreign || vma_is_foreign(vma))
415 return true;
416
417 return pkey_access_permitted(vma_pkey(vma), write, execute);
418}
419
420void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm)
421{
422 if (static_branch_likely(&pkey_disabled))
423 return;
424
425
426 mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm);
427 mm->context.execute_only_pkey = oldmm->context.execute_only_pkey;
428}
429