1
2
3
4
5
6
7
8
9
10#ifndef _ASMARM_TLBFLUSH_H
11#define _ASMARM_TLBFLUSH_H
12
13#ifdef CONFIG_MMU
14
15#include <asm/glue.h>
16
17#define TLB_V3_PAGE (1 << 0)
18#define TLB_V4_U_PAGE (1 << 1)
19#define TLB_V4_D_PAGE (1 << 2)
20#define TLB_V4_I_PAGE (1 << 3)
21#define TLB_V6_U_PAGE (1 << 4)
22#define TLB_V6_D_PAGE (1 << 5)
23#define TLB_V6_I_PAGE (1 << 6)
24
25#define TLB_V3_FULL (1 << 8)
26#define TLB_V4_U_FULL (1 << 9)
27#define TLB_V4_D_FULL (1 << 10)
28#define TLB_V4_I_FULL (1 << 11)
29#define TLB_V6_U_FULL (1 << 12)
30#define TLB_V6_D_FULL (1 << 13)
31#define TLB_V6_I_FULL (1 << 14)
32
33#define TLB_V6_U_ASID (1 << 16)
34#define TLB_V6_D_ASID (1 << 17)
35#define TLB_V6_I_ASID (1 << 18)
36
37
38#define TLB_V7_UIS_PAGE (1 << 19)
39#define TLB_V7_UIS_FULL (1 << 20)
40#define TLB_V7_UIS_ASID (1 << 21)
41
42#define TLB_BARRIER (1 << 28)
43#define TLB_L2CLEAN_FR (1 << 29)
44#define TLB_DCLEAN (1 << 30)
45#define TLB_WB (1 << 31)
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61#undef _TLB
62#undef MULTI_TLB
63
64#ifdef CONFIG_SMP_ON_UP
65#define MULTI_TLB 1
66#endif
67
68#define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE)
69
70#ifdef CONFIG_CPU_TLB_V4WT
71# define v4_possible_flags v4_tlb_flags
72# define v4_always_flags v4_tlb_flags
73# ifdef _TLB
74# define MULTI_TLB 1
75# else
76# define _TLB v4
77# endif
78#else
79# define v4_possible_flags 0
80# define v4_always_flags (-1UL)
81#endif
82
83#define fa_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
84 TLB_V4_U_FULL | TLB_V4_U_PAGE)
85
86#ifdef CONFIG_CPU_TLB_FA
87# define fa_possible_flags fa_tlb_flags
88# define fa_always_flags fa_tlb_flags
89# ifdef _TLB
90# define MULTI_TLB 1
91# else
92# define _TLB fa
93# endif
94#else
95# define fa_possible_flags 0
96# define fa_always_flags (-1UL)
97#endif
98
99#define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \
100 TLB_V4_I_FULL | TLB_V4_D_FULL | \
101 TLB_V4_I_PAGE | TLB_V4_D_PAGE)
102
103#ifdef CONFIG_CPU_TLB_V4WBI
104# define v4wbi_possible_flags v4wbi_tlb_flags
105# define v4wbi_always_flags v4wbi_tlb_flags
106# ifdef _TLB
107# define MULTI_TLB 1
108# else
109# define _TLB v4wbi
110# endif
111#else
112# define v4wbi_possible_flags 0
113# define v4wbi_always_flags (-1UL)
114#endif
115
116#define fr_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \
117 TLB_V4_I_FULL | TLB_V4_D_FULL | \
118 TLB_V4_I_PAGE | TLB_V4_D_PAGE)
119
120#ifdef CONFIG_CPU_TLB_FEROCEON
121# define fr_possible_flags fr_tlb_flags
122# define fr_always_flags fr_tlb_flags
123# ifdef _TLB
124# define MULTI_TLB 1
125# else
126# define _TLB v4wbi
127# endif
128#else
129# define fr_possible_flags 0
130# define fr_always_flags (-1UL)
131#endif
132
133#define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \
134 TLB_V4_I_FULL | TLB_V4_D_FULL | \
135 TLB_V4_D_PAGE)
136
137#ifdef CONFIG_CPU_TLB_V4WB
138# define v4wb_possible_flags v4wb_tlb_flags
139# define v4wb_always_flags v4wb_tlb_flags
140# ifdef _TLB
141# define MULTI_TLB 1
142# else
143# define _TLB v4wb
144# endif
145#else
146# define v4wb_possible_flags 0
147# define v4wb_always_flags (-1UL)
148#endif
149
150#define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
151 TLB_V6_I_FULL | TLB_V6_D_FULL | \
152 TLB_V6_I_PAGE | TLB_V6_D_PAGE | \
153 TLB_V6_I_ASID | TLB_V6_D_ASID)
154
155#ifdef CONFIG_CPU_TLB_V6
156# define v6wbi_possible_flags v6wbi_tlb_flags
157# define v6wbi_always_flags v6wbi_tlb_flags
158# ifdef _TLB
159# define MULTI_TLB 1
160# else
161# define _TLB v6wbi
162# endif
163#else
164# define v6wbi_possible_flags 0
165# define v6wbi_always_flags (-1UL)
166#endif
167
168#define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
169 TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
170#define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
171 TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
172
173#ifdef CONFIG_CPU_TLB_V7
174
175# ifdef CONFIG_SMP_ON_UP
176# define v7wbi_possible_flags (v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up)
177# define v7wbi_always_flags (v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up)
178# elif defined(CONFIG_SMP)
179# define v7wbi_possible_flags v7wbi_tlb_flags_smp
180# define v7wbi_always_flags v7wbi_tlb_flags_smp
181# else
182# define v7wbi_possible_flags v7wbi_tlb_flags_up
183# define v7wbi_always_flags v7wbi_tlb_flags_up
184# endif
185# ifdef _TLB
186# define MULTI_TLB 1
187# else
188# define _TLB v7wbi
189# endif
190#else
191# define v7wbi_possible_flags 0
192# define v7wbi_always_flags (-1UL)
193#endif
194
195#ifndef _TLB
196#error Unknown TLB model
197#endif
198
199#ifndef __ASSEMBLY__
200
201#include <linux/sched.h>
202
203struct cpu_tlb_fns {
204 void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
205 void (*flush_kern_range)(unsigned long, unsigned long);
206 unsigned long tlb_flags;
207};
208
209
210
211
212#ifdef MULTI_TLB
213
214#define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range
215#define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range
216
217#else
218
219#define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range)
220#define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range)
221
222extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
223extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
224
225#endif
226
227extern struct cpu_tlb_fns cpu_tlb;
228
229#define __cpu_tlb_flags cpu_tlb.tlb_flags
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286#define possible_tlb_flags (v4_possible_flags | \
287 v4wbi_possible_flags | \
288 fr_possible_flags | \
289 v4wb_possible_flags | \
290 fa_possible_flags | \
291 v6wbi_possible_flags | \
292 v7wbi_possible_flags)
293
294#define always_tlb_flags (v4_always_flags & \
295 v4wbi_always_flags & \
296 fr_always_flags & \
297 v4wb_always_flags & \
298 fa_always_flags & \
299 v6wbi_always_flags & \
300 v7wbi_always_flags)
301
302#define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f)))
303
304#define __tlb_op(f, insnarg, arg) \
305 do { \
306 if (always_tlb_flags & (f)) \
307 asm("mcr " insnarg \
308 : : "r" (arg) : "cc"); \
309 else if (possible_tlb_flags & (f)) \
310 asm("tst %1, %2\n\t" \
311 "mcrne " insnarg \
312 : : "r" (arg), "r" (__tlb_flag), "Ir" (f) \
313 : "cc"); \
314 } while (0)
315
316#define tlb_op(f, regs, arg) __tlb_op(f, "p15, 0, %0, " regs, arg)
317#define tlb_l2_op(f, regs, arg) __tlb_op(f, "p15, 1, %0, " regs, arg)
318
319static inline void local_flush_tlb_all(void)
320{
321 const int zero = 0;
322 const unsigned int __tlb_flag = __cpu_tlb_flags;
323
324 if (tlb_flag(TLB_WB))
325 dsb();
326
327 tlb_op(TLB_V3_FULL, "c6, c0, 0", zero);
328 tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero);
329 tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero);
330 tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
331 tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
332
333 if (tlb_flag(TLB_BARRIER)) {
334 dsb();
335 isb();
336 }
337}
338
339static inline void local_flush_tlb_mm(struct mm_struct *mm)
340{
341 const int zero = 0;
342 const int asid = ASID(mm);
343 const unsigned int __tlb_flag = __cpu_tlb_flags;
344
345 if (tlb_flag(TLB_WB))
346 dsb();
347
348 if (possible_tlb_flags & (TLB_V3_FULL|TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) {
349 if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) {
350 tlb_op(TLB_V3_FULL, "c6, c0, 0", zero);
351 tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero);
352 tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero);
353 tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero);
354 }
355 put_cpu();
356 }
357
358 tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid);
359 tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid);
360 tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid);
361#ifdef CONFIG_ARM_ERRATA_720789
362 tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", zero);
363#else
364 tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", asid);
365#endif
366
367 if (tlb_flag(TLB_BARRIER))
368 dsb();
369}
370
371static inline void
372local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
373{
374 const int zero = 0;
375 const unsigned int __tlb_flag = __cpu_tlb_flags;
376
377 uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
378
379 if (tlb_flag(TLB_WB))
380 dsb();
381
382 if (possible_tlb_flags & (TLB_V3_PAGE|TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) &&
383 cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
384 tlb_op(TLB_V3_PAGE, "c6, c0, 0", uaddr);
385 tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr);
386 tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", uaddr);
387 tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", uaddr);
388 if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
389 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
390 }
391
392 tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr);
393 tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr);
394 tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr);
395#ifdef CONFIG_ARM_ERRATA_720789
396 tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK);
397#else
398 tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", uaddr);
399#endif
400
401 if (tlb_flag(TLB_BARRIER))
402 dsb();
403}
404
405static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
406{
407 const int zero = 0;
408 const unsigned int __tlb_flag = __cpu_tlb_flags;
409
410 kaddr &= PAGE_MASK;
411
412 if (tlb_flag(TLB_WB))
413 dsb();
414
415 tlb_op(TLB_V3_PAGE, "c6, c0, 0", kaddr);
416 tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr);
417 tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr);
418 tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr);
419 if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
420 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
421
422 tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr);
423 tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr);
424 tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr);
425 tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr);
426
427 if (tlb_flag(TLB_BARRIER)) {
428 dsb();
429 isb();
430 }
431}
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446static inline void flush_pmd_entry(void *pmd)
447{
448 const unsigned int __tlb_flag = __cpu_tlb_flags;
449
450 tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd);
451 tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd);
452
453 if (tlb_flag(TLB_WB))
454 dsb();
455}
456
457static inline void clean_pmd_entry(void *pmd)
458{
459 const unsigned int __tlb_flag = __cpu_tlb_flags;
460
461 tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd);
462 tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd);
463}
464
465#undef tlb_op
466#undef tlb_flag
467#undef always_tlb_flags
468#undef possible_tlb_flags
469
470
471
472
473#define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma)
474#define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e)
475
476#ifndef CONFIG_SMP
477#define flush_tlb_all local_flush_tlb_all
478#define flush_tlb_mm local_flush_tlb_mm
479#define flush_tlb_page local_flush_tlb_page
480#define flush_tlb_kernel_page local_flush_tlb_kernel_page
481#define flush_tlb_range local_flush_tlb_range
482#define flush_tlb_kernel_range local_flush_tlb_kernel_range
483#else
484extern void flush_tlb_all(void);
485extern void flush_tlb_mm(struct mm_struct *mm);
486extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr);
487extern void flush_tlb_kernel_page(unsigned long kaddr);
488extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
489extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
490#endif
491
492
493
494
495
496
497
498#if __LINUX_ARM_ARCH__ < 6
499extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
500 pte_t *ptep);
501#else
502static inline void update_mmu_cache(struct vm_area_struct *vma,
503 unsigned long addr, pte_t *ptep)
504{
505}
506#endif
507
508#endif
509
510#endif
511
512#endif
513