1
2
3
4
5
6
7
8#include <linux/bitops.h>
9#include <linux/cpumask.h>
10#include <linux/mm.h>
11#include <linux/percpu.h>
12#include <linux/slab.h>
13#include <linux/spinlock.h>
14#include <linux/static_key.h>
15#include <asm/tlbflush.h>
16#include <asm/cacheflush.h>
17#include <asm/mmu_context.h>
18
19#ifdef CONFIG_MMU
20
21DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
22
23static unsigned long asid_bits;
24static unsigned long num_asids;
25static unsigned long asid_mask;
26
27static atomic_long_t current_version;
28
29static DEFINE_RAW_SPINLOCK(context_lock);
30static cpumask_t context_tlb_flush_pending;
31static unsigned long *context_asid_map;
32
33static DEFINE_PER_CPU(atomic_long_t, active_context);
34static DEFINE_PER_CPU(unsigned long, reserved_context);
35
36static bool check_update_reserved_context(unsigned long cntx,
37 unsigned long newcntx)
38{
39 int cpu;
40 bool hit = false;
41
42
43
44
45
46
47
48
49
50
51 for_each_possible_cpu(cpu) {
52 if (per_cpu(reserved_context, cpu) == cntx) {
53 hit = true;
54 per_cpu(reserved_context, cpu) = newcntx;
55 }
56 }
57
58 return hit;
59}
60
61static void __flush_context(void)
62{
63 int i;
64 unsigned long cntx;
65
66
67 lockdep_assert_held(&context_lock);
68
69
70 bitmap_clear(context_asid_map, 0, num_asids);
71
72
73 for_each_possible_cpu(i) {
74 cntx = atomic_long_xchg_relaxed(&per_cpu(active_context, i), 0);
75
76
77
78
79
80
81 if (cntx == 0)
82 cntx = per_cpu(reserved_context, i);
83
84 __set_bit(cntx & asid_mask, context_asid_map);
85 per_cpu(reserved_context, i) = cntx;
86 }
87
88
89 __set_bit(0, context_asid_map);
90
91
92 cpumask_setall(&context_tlb_flush_pending);
93}
94
95static unsigned long __new_context(struct mm_struct *mm)
96{
97 static u32 cur_idx = 1;
98 unsigned long cntx = atomic_long_read(&mm->context.id);
99 unsigned long asid, ver = atomic_long_read(¤t_version);
100
101
102 lockdep_assert_held(&context_lock);
103
104 if (cntx != 0) {
105 unsigned long newcntx = ver | (cntx & asid_mask);
106
107
108
109
110
111 if (check_update_reserved_context(cntx, newcntx))
112 return newcntx;
113
114
115
116
117
118 if (!__test_and_set_bit(cntx & asid_mask, context_asid_map))
119 return newcntx;
120 }
121
122
123
124
125
126 asid = find_next_zero_bit(context_asid_map, num_asids, cur_idx);
127 if (asid != num_asids)
128 goto set_asid;
129
130
131 ver = atomic_long_add_return_relaxed(num_asids, ¤t_version);
132
133
134 __flush_context();
135
136
137 asid = find_next_zero_bit(context_asid_map, num_asids, 1);
138
139set_asid:
140 __set_bit(asid, context_asid_map);
141 cur_idx = asid;
142 return asid | ver;
143}
144
145static void set_mm_asid(struct mm_struct *mm, unsigned int cpu)
146{
147 unsigned long flags;
148 bool need_flush_tlb = false;
149 unsigned long cntx, old_active_cntx;
150
151 cntx = atomic_long_read(&mm->context.id);
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169 old_active_cntx = atomic_long_read(&per_cpu(active_context, cpu));
170 if (old_active_cntx &&
171 ((cntx & ~asid_mask) == atomic_long_read(¤t_version)) &&
172 atomic_long_cmpxchg_relaxed(&per_cpu(active_context, cpu),
173 old_active_cntx, cntx))
174 goto switch_mm_fast;
175
176 raw_spin_lock_irqsave(&context_lock, flags);
177
178
179 cntx = atomic_long_read(&mm->context.id);
180 if ((cntx & ~asid_mask) != atomic_long_read(¤t_version)) {
181 cntx = __new_context(mm);
182 atomic_long_set(&mm->context.id, cntx);
183 }
184
185 if (cpumask_test_and_clear_cpu(cpu, &context_tlb_flush_pending))
186 need_flush_tlb = true;
187
188 atomic_long_set(&per_cpu(active_context, cpu), cntx);
189
190 raw_spin_unlock_irqrestore(&context_lock, flags);
191
192switch_mm_fast:
193 csr_write(CSR_SATP, virt_to_pfn(mm->pgd) |
194 ((cntx & asid_mask) << SATP_ASID_SHIFT) |
195 SATP_MODE);
196
197 if (need_flush_tlb)
198 local_flush_tlb_all();
199}
200
201static void set_mm_noasid(struct mm_struct *mm)
202{
203
204 csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | SATP_MODE);
205 local_flush_tlb_all();
206}
207
208static inline void set_mm(struct mm_struct *mm, unsigned int cpu)
209{
210 if (static_branch_unlikely(&use_asid_allocator))
211 set_mm_asid(mm, cpu);
212 else
213 set_mm_noasid(mm);
214}
215
216static int __init asids_init(void)
217{
218 unsigned long old;
219
220
221 old = csr_read(CSR_SATP);
222 asid_bits = old | (SATP_ASID_MASK << SATP_ASID_SHIFT);
223 csr_write(CSR_SATP, asid_bits);
224 asid_bits = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT) & SATP_ASID_MASK;
225 asid_bits = fls_long(asid_bits);
226 csr_write(CSR_SATP, old);
227
228
229
230
231
232
233 local_flush_tlb_all();
234
235
236 num_asids = 1 << asid_bits;
237 asid_mask = num_asids - 1;
238
239
240
241
242
243 if (num_asids > (2 * num_possible_cpus())) {
244 atomic_long_set(¤t_version, num_asids);
245
246 context_asid_map = bitmap_zalloc(num_asids, GFP_KERNEL);
247 if (!context_asid_map)
248 panic("Failed to allocate bitmap for %lu ASIDs\n",
249 num_asids);
250
251 __set_bit(0, context_asid_map);
252
253 static_branch_enable(&use_asid_allocator);
254
255 pr_info("ASID allocator using %lu bits (%lu entries)\n",
256 asid_bits, num_asids);
257 } else {
258 pr_info("ASID allocator disabled\n");
259 }
260
261 return 0;
262}
263early_initcall(asids_init);
264#else
265static inline void set_mm(struct mm_struct *mm, unsigned int cpu)
266{
267
268}
269#endif
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285static inline void flush_icache_deferred(struct mm_struct *mm, unsigned int cpu)
286{
287#ifdef CONFIG_SMP
288 cpumask_t *mask = &mm->context.icache_stale_mask;
289
290 if (cpumask_test_cpu(cpu, mask)) {
291 cpumask_clear_cpu(cpu, mask);
292
293
294
295
296 smp_mb();
297 local_flush_icache_all();
298 }
299
300#endif
301}
302
303void switch_mm(struct mm_struct *prev, struct mm_struct *next,
304 struct task_struct *task)
305{
306 unsigned int cpu;
307
308 if (unlikely(prev == next))
309 return;
310
311
312
313
314
315
316 cpu = smp_processor_id();
317
318 cpumask_clear_cpu(cpu, mm_cpumask(prev));
319 cpumask_set_cpu(cpu, mm_cpumask(next));
320
321 set_mm(next, cpu);
322
323 flush_icache_deferred(next, cpu);
324}
325