1
2#ifndef _ASM_POWERPC_MMU_H_
3#define _ASM_POWERPC_MMU_H_
4#ifdef __KERNEL__
5
6#include <linux/types.h>
7
8#include <asm/asm-const.h>
9
10
11
12
13
14
15
16
17#define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001)
18#define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002)
19#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004)
20#define MMU_FTR_TYPE_44x ASM_CONST(0x00000008)
21#define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010)
22#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020)
23
24
25#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040)
26
27
28
29
30
31
32
33
34#define MMU_FTR_68_BIT_VA ASM_CONST(0x00002000)
35
36
37
38
39#define MMU_FTR_KERNEL_RO ASM_CONST(0x00004000)
40
41
42
43
44
45#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000)
46
47
48#define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000)
49
50
51
52
53#define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000)
54
55
56
57
58
59#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000)
60
61
62
63#define MMU_FTR_USE_TLBILX ASM_CONST(0x00080000)
64
65
66
67
68
69#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000)
70
71
72
73
74
75#define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000)
76
77
78
79
80#define MMU_FTR_USE_TLBRSRV ASM_CONST(0x00800000)
81
82
83
84#define MMU_FTR_USE_PAIRED_MAS ASM_CONST(0x01000000)
85
86
87
88#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000)
89
90
91
92#define MMU_FTR_16M_PAGE ASM_CONST(0x04000000)
93
94
95
96#define MMU_FTR_TLBIEL ASM_CONST(0x08000000)
97
98
99
100#define MMU_FTR_LOCKLESS_TLBIE ASM_CONST(0x10000000)
101
102
103
104#define MMU_FTR_CI_LARGE_PAGE ASM_CONST(0x20000000)
105
106
107
108#define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000)
109
110
111
112
113#define MMU_FTR_RADIX_KUAP ASM_CONST(0x80000000)
114
115
116#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 \
117 MMU_FTR_HPTE_TABLE | MMU_FTR_PPCAS_ARCH_V2
118#define MMU_FTRS_POWER MMU_FTRS_DEFAULT_HPTE_ARCH_V2
119#define MMU_FTRS_PPC970 MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA
120#define MMU_FTRS_POWER5 MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE
121#define MMU_FTRS_POWER6 MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA
122#define MMU_FTRS_POWER7 MMU_FTRS_POWER6
123#define MMU_FTRS_POWER8 MMU_FTRS_POWER6
124#define MMU_FTRS_POWER9 MMU_FTRS_POWER6
125#define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
126 MMU_FTR_CI_LARGE_PAGE
127#define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
128 MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
129#ifndef __ASSEMBLY__
130#include <linux/bug.h>
131#include <asm/cputable.h>
132#include <asm/page.h>
133
134typedef pte_t *pgtable_t;
135
136#ifdef CONFIG_PPC_FSL_BOOK3E
137#include <asm/percpu.h>
138DECLARE_PER_CPU(int, next_tlbcam_idx);
139#endif
140
141enum {
142 MMU_FTRS_POSSIBLE =
143#ifdef CONFIG_PPC_BOOK3S
144 MMU_FTR_HPTE_TABLE |
145#endif
146#ifdef CONFIG_PPC_8xx
147 MMU_FTR_TYPE_8xx |
148#endif
149#ifdef CONFIG_40x
150 MMU_FTR_TYPE_40x |
151#endif
152#ifdef CONFIG_44x
153 MMU_FTR_TYPE_44x |
154#endif
155#if defined(CONFIG_E200) || defined(CONFIG_E500)
156 MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX |
157#endif
158#ifdef CONFIG_PPC_47x
159 MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL |
160#endif
161#ifdef CONFIG_PPC_BOOK3S_32
162 MMU_FTR_USE_HIGH_BATS | MMU_FTR_NEED_DTLB_SW_LRU |
163#endif
164#ifdef CONFIG_PPC_BOOK3E_64
165 MMU_FTR_USE_TLBRSRV | MMU_FTR_USE_PAIRED_MAS |
166#endif
167#ifdef CONFIG_PPC_BOOK3S_64
168 MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
169 MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
170 MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
171 MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA |
172#endif
173#ifdef CONFIG_PPC_RADIX_MMU
174 MMU_FTR_TYPE_RADIX |
175#ifdef CONFIG_PPC_KUAP
176 MMU_FTR_RADIX_KUAP |
177#endif
178#endif
179 0,
180};
181
182static inline bool early_mmu_has_feature(unsigned long feature)
183{
184 return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
185}
186
187#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
188#include <linux/jump_label.h>
189
190#define NUM_MMU_FTR_KEYS 32
191
192extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
193
194extern void mmu_feature_keys_init(void);
195
196static __always_inline bool mmu_has_feature(unsigned long feature)
197{
198 int i;
199
200#ifndef __clang__
201 BUILD_BUG_ON(!__builtin_constant_p(feature));
202#endif
203
204#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
205 if (!static_key_initialized) {
206 printk("Warning! mmu_has_feature() used prior to jump label init!\n");
207 dump_stack();
208 return early_mmu_has_feature(feature);
209 }
210#endif
211
212 if (!(MMU_FTRS_POSSIBLE & feature))
213 return false;
214
215 i = __builtin_ctzl(feature);
216 return static_branch_likely(&mmu_feature_keys[i]);
217}
218
219static inline void mmu_clear_feature(unsigned long feature)
220{
221 int i;
222
223 i = __builtin_ctzl(feature);
224 cur_cpu_spec->mmu_features &= ~feature;
225 static_branch_disable(&mmu_feature_keys[i]);
226}
227#else
228
229static inline void mmu_feature_keys_init(void)
230{
231
232}
233
234static inline bool mmu_has_feature(unsigned long feature)
235{
236 return early_mmu_has_feature(feature);
237}
238
239static inline void mmu_clear_feature(unsigned long feature)
240{
241 cur_cpu_spec->mmu_features &= ~feature;
242}
243#endif
244
245extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
246
247#ifdef CONFIG_PPC64
248
249
250
251extern u64 ppc64_rma_size;
252
253
254extern void mmu_cleanup_all(void);
255extern void radix__mmu_cleanup_all(void);
256
257
258extern void mmu_partition_table_init(void);
259extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
260 unsigned long dw1, bool flush);
261#endif
262
263struct mm_struct;
264#ifdef CONFIG_DEBUG_VM
265extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
266#else
267static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
268{
269}
270#endif
271
272#ifdef CONFIG_PPC_RADIX_MMU
273static inline bool radix_enabled(void)
274{
275 return mmu_has_feature(MMU_FTR_TYPE_RADIX);
276}
277
278static inline bool early_radix_enabled(void)
279{
280 return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
281}
282#else
283static inline bool radix_enabled(void)
284{
285 return false;
286}
287
288static inline bool early_radix_enabled(void)
289{
290 return false;
291}
292#endif
293
294#ifdef CONFIG_PPC_MEM_KEYS
295extern u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address);
296#else
297static inline u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address)
298{
299 return 0;
300}
301#endif
302
303#ifdef CONFIG_STRICT_KERNEL_RWX
304static inline bool strict_kernel_rwx_enabled(void)
305{
306 return rodata_enabled;
307}
308#else
309static inline bool strict_kernel_rwx_enabled(void)
310{
311 return false;
312}
313#endif
314#endif
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332#define MMU_PAGE_4K 0
333#define MMU_PAGE_16K 1
334#define MMU_PAGE_64K 2
335#define MMU_PAGE_64K_AP 3
336#define MMU_PAGE_256K 4
337#define MMU_PAGE_512K 5
338#define MMU_PAGE_1M 6
339#define MMU_PAGE_2M 7
340#define MMU_PAGE_4M 8
341#define MMU_PAGE_8M 9
342#define MMU_PAGE_16M 10
343#define MMU_PAGE_64M 11
344#define MMU_PAGE_256M 12
345#define MMU_PAGE_1G 13
346#define MMU_PAGE_16G 14
347#define MMU_PAGE_64G 15
348
349
350
351
352
353#define MMU_PAGE_COUNT 16
354
355#ifdef CONFIG_PPC_BOOK3S_64
356#include <asm/book3s/64/mmu.h>
357#else
358
359#ifndef __ASSEMBLY__
360
361extern void early_init_mmu(void);
362extern void early_init_mmu_secondary(void);
363extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
364 phys_addr_t first_memblock_size);
365static inline void mmu_early_init_devtree(void) { }
366
367extern void *abatron_pteptrs[2];
368#endif
369#endif
370
371#if defined(CONFIG_PPC_BOOK3S_32)
372
373#include <asm/book3s/32/mmu-hash.h>
374#elif defined(CONFIG_PPC_MMU_NOHASH)
375#include <asm/nohash/mmu.h>
376#endif
377
378#endif
379#endif
380