1
2
3
4
5
6
7
8#ifndef __ASM_MMU_CONTEXT_H
9#define __ASM_MMU_CONTEXT_H
10
11#ifndef __ASSEMBLY__
12
13#include <linux/compiler.h>
14#include <linux/sched.h>
15#include <linux/sched/hotplug.h>
16#include <linux/mm_types.h>
17#include <linux/pgtable.h>
18
19#include <asm/cacheflush.h>
20#include <asm/cpufeature.h>
21#include <asm/proc-fns.h>
22#include <asm-generic/mm_hooks.h>
23#include <asm/cputype.h>
24#include <asm/sysreg.h>
25#include <asm/tlbflush.h>
26
27extern bool rodata_full;
28
29static inline void contextidr_thread_switch(struct task_struct *next)
30{
31 if (!IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR))
32 return;
33
34 write_sysreg(task_pid_nr(next), contextidr_el1);
35 isb();
36}
37
38
39
40
41static inline void cpu_set_reserved_ttbr0(void)
42{
43 unsigned long ttbr = phys_to_ttbr(__pa_symbol(empty_zero_page));
44
45 write_sysreg(ttbr, ttbr0_el1);
46 isb();
47}
48
49void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
50
51static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm)
52{
53 BUG_ON(pgd == swapper_pg_dir);
54 cpu_set_reserved_ttbr0();
55 cpu_do_switch_mm(virt_to_phys(pgd),mm);
56}
57
58
59
60
61
62
63extern u64 idmap_t0sz;
64extern u64 idmap_ptrs_per_pgd;
65
66static inline bool __cpu_uses_extended_idmap(void)
67{
68 if (IS_ENABLED(CONFIG_ARM64_VA_BITS_52))
69 return false;
70
71 return unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS));
72}
73
74
75
76
77
78static inline bool __cpu_uses_extended_idmap_level(void)
79{
80 return ARM64_HW_PGTABLE_LEVELS(64 - idmap_t0sz) > CONFIG_PGTABLE_LEVELS;
81}
82
83
84
85
86static inline void __cpu_set_tcr_t0sz(unsigned long t0sz)
87{
88 unsigned long tcr;
89
90 if (!__cpu_uses_extended_idmap())
91 return;
92
93 tcr = read_sysreg(tcr_el1);
94 tcr &= ~TCR_T0SZ_MASK;
95 tcr |= t0sz << TCR_T0SZ_OFFSET;
96 write_sysreg(tcr, tcr_el1);
97 isb();
98}
99
100#define cpu_set_default_tcr_t0sz() __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual))
101#define cpu_set_idmap_tcr_t0sz() __cpu_set_tcr_t0sz(idmap_t0sz)
102
103
104
105
106
107
108
109
110
111
112
113
114
115static inline void cpu_uninstall_idmap(void)
116{
117 struct mm_struct *mm = current->active_mm;
118
119 cpu_set_reserved_ttbr0();
120 local_flush_tlb_all();
121 cpu_set_default_tcr_t0sz();
122
123 if (mm != &init_mm && !system_uses_ttbr0_pan())
124 cpu_switch_mm(mm->pgd, mm);
125}
126
127static inline void cpu_install_idmap(void)
128{
129 cpu_set_reserved_ttbr0();
130 local_flush_tlb_all();
131 cpu_set_idmap_tcr_t0sz();
132
133 cpu_switch_mm(lm_alias(idmap_pg_dir), &init_mm);
134}
135
136
137
138
139
140static inline void cpu_replace_ttbr1(pgd_t *pgdp)
141{
142 typedef void (ttbr_replace_func)(phys_addr_t);
143 extern ttbr_replace_func idmap_cpu_replace_ttbr1;
144 ttbr_replace_func *replace_phys;
145
146
147 phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));
148
149 if (system_supports_cnp() && !WARN_ON(pgdp != lm_alias(swapper_pg_dir))) {
150
151
152
153
154
155
156
157
158 ttbr1 |= TTBR_CNP_BIT;
159 }
160
161 replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
162
163 cpu_install_idmap();
164 replace_phys(ttbr1);
165 cpu_uninstall_idmap();
166}
167
168
169
170
171
172
173
174
175
176
177#define destroy_context(mm) do { } while(0)
178void check_and_switch_context(struct mm_struct *mm);
179
180static inline int
181init_new_context(struct task_struct *tsk, struct mm_struct *mm)
182{
183 atomic64_set(&mm->context.id, 0);
184 refcount_set(&mm->context.pinned, 0);
185 return 0;
186}
187
188#ifdef CONFIG_ARM64_SW_TTBR0_PAN
189static inline void update_saved_ttbr0(struct task_struct *tsk,
190 struct mm_struct *mm)
191{
192 u64 ttbr;
193
194 if (!system_uses_ttbr0_pan())
195 return;
196
197 if (mm == &init_mm)
198 ttbr = __pa_symbol(empty_zero_page);
199 else
200 ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48;
201
202 WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);
203}
204#else
205static inline void update_saved_ttbr0(struct task_struct *tsk,
206 struct mm_struct *mm)
207{
208}
209#endif
210
211static inline void
212enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
213{
214
215
216
217
218 update_saved_ttbr0(tsk, &init_mm);
219}
220
221static inline void __switch_mm(struct mm_struct *next)
222{
223
224
225
226
227 if (next == &init_mm) {
228 cpu_set_reserved_ttbr0();
229 return;
230 }
231
232 check_and_switch_context(next);
233}
234
235static inline void
236switch_mm(struct mm_struct *prev, struct mm_struct *next,
237 struct task_struct *tsk)
238{
239 if (prev != next)
240 __switch_mm(next);
241
242
243
244
245
246
247
248 update_saved_ttbr0(tsk, next);
249}
250
251#define deactivate_mm(tsk,mm) do { } while (0)
252#define activate_mm(prev,next) switch_mm(prev, next, current)
253
254void verify_cpu_asid_bits(void);
255void post_ttbr_update_workaround(void);
256
257unsigned long arm64_mm_context_get(struct mm_struct *mm);
258void arm64_mm_context_put(struct mm_struct *mm);
259
260#endif
261
262#endif
263