1
2
3
4
5
6
7
8
9
10#include <linux/linkage.h>
11#include <linux/init.h>
12#include <asm/assembler.h>
13#include <asm/errno.h>
14#include <asm/unwind.h>
15#include <asm/hardware/cache-b15-rac.h>
16
17#include "proc-macros.S"
18
19#ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND
20.globl icache_size
21 .data
22 .align 2
23icache_size:
24 .long 64
25 .text
26#endif
27
28
29
30
31
32
33
34
35
36
37
38
39ENTRY(v7_invalidate_l1)
40 mov r0,
41 mcr p15, 2, r0, c0, c0, 0 @ select L1 data cache in CSSELR
42 isb
43 mrc p15, 1, r0, c0, c0, 0 @ read cache geometry from CCSIDR
44
45 movw r3,
46 and r3, r3, r0, lsr
47 clz r1, r3 @ WayShift
48 mov r2,
49 mov r3, r3, lsl r1 @ NumWays-1 shifted into bits [31:...]
50 movs r1, r2, lsl r1 @
51 moveq r1,
52
53 and r2, r0,
54 add r2, r2,
55
561: movw ip,
57 and r0, ip, r0, lsr
58
592: mov ip, r0, lsl r2 @ NumSet << SetShift
60 orr ip, ip, r3 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift)
61 mcr p15, 0, ip, c7, c6, 2
62 subs r0, r0,
63 bpl 2b
64 subs r3, r3, r1 @ Way--
65 bcc 3f
66 mrc p15, 1, r0, c0, c0, 0 @ re-read cache geometry from CCSIDR
67 b 1b
683: dsb st
69 isb
70 ret lr
71ENDPROC(v7_invalidate_l1)
72
73
74
75
76
77
78
79
80
81ENTRY(v7_flush_icache_all)
82 mov r0,
83 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
84 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
85 ret lr
86ENDPROC(v7_flush_icache_all)
87
88
89
90
91
92
93
94
95
96ENTRY(v7_flush_dcache_louis)
97 dmb @ ensure ordering with previous memory accesses
98 mrc p15, 1, r0, c0, c0, 1 @ read clidr, r0 = clidr
99ALT_SMP(mov r3, r0, lsr
100ALT_UP( mov r3, r0, lsr
101 ands r3, r3,
102 bne start_flush_levels @ LoU != 0, start flushing
103#ifdef CONFIG_ARM_ERRATA_643719
104ALT_SMP(mrc p15, 0, r2, c0, c0, 0) @ read main ID register
105ALT_UP( ret lr) @ LoUU is zero, so nothing to do
106 movw r1,
107 movt r1,
108 teq r1, r2, lsr
109 moveq r3,
110 beq start_flush_levels @ start flushing cache levels
111#endif
112 ret lr
113ENDPROC(v7_flush_dcache_louis)
114
115
116
117
118
119
120
121
122
123
124ENTRY(v7_flush_dcache_all)
125 dmb @ ensure ordering with previous memory accesses
126 mrc p15, 1, r0, c0, c0, 1 @ read clidr
127 mov r3, r0, lsr
128 ands r3, r3,
129 beq finished @ if loc is 0, then no need to clean
130start_flush_levels:
131 mov r10,
132flush_levels:
133 add r2, r10, r10, lsr
134 mov r1, r0, lsr r2 @ extract cache type bits from clidr
135 and r1, r1,
136 cmp r1,
137 blt skip @ skip if no cache, or just i-cache
138#ifdef CONFIG_PREEMPTION
139 save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic
140#endif
141 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
142 isb @ isb to sych the new cssr&csidr
143 mrc p15, 1, r1, c0, c0, 0 @ read the new csidr
144#ifdef CONFIG_PREEMPTION
145 restore_irqs_notrace r9
146#endif
147 and r2, r1,
148 add r2, r2,
149 movw r4,
150 ands r4, r4, r1, lsr
151 clz r5, r4 @ find bit position of way size increment
152 movw r7,
153 ands r7, r7, r1, lsr
154loop1:
155 mov r9, r7 @ create working copy of max index
156loop2:
157 ARM( orr r11, r10, r4, lsl r5 ) @ factor way and cache number into r11
158 THUMB( lsl r6, r4, r5 )
159 THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11
160 ARM( orr r11, r11, r9, lsl r2 ) @ factor index number into r11
161 THUMB( lsl r6, r9, r2 )
162 THUMB( orr r11, r11, r6 ) @ factor index number into r11
163 mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way
164 subs r9, r9,
165 bge loop2
166 subs r4, r4,
167 bge loop1
168skip:
169 add r10, r10,
170 cmp r3, r10
171#ifdef CONFIG_ARM_ERRATA_814220
172 dsb
173#endif
174 bgt flush_levels
175finished:
176 mov r10,
177 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
178 dsb st
179 isb
180 ret lr
181ENDPROC(v7_flush_dcache_all)
182
183
184
185
186
187
188
189
190
191
192
193
194ENTRY(v7_flush_kern_cache_all)
195 ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} )
196 THUMB( stmfd sp!, {r4-r7, r9-r11, lr} )
197 bl v7_flush_dcache_all
198 mov r0,
199 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
200 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
201 ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} )
202 THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} )
203 ret lr
204ENDPROC(v7_flush_kern_cache_all)
205
206
207
208
209
210
211
212ENTRY(v7_flush_kern_cache_louis)
213 ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} )
214 THUMB( stmfd sp!, {r4-r7, r9-r11, lr} )
215 bl v7_flush_dcache_louis
216 mov r0,
217 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
218 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
219 ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} )
220 THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} )
221 ret lr
222ENDPROC(v7_flush_kern_cache_louis)
223
224
225
226
227
228
229
230
231ENTRY(v7_flush_user_cache_all)
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246ENTRY(v7_flush_user_cache_range)
247 ret lr
248ENDPROC(v7_flush_user_cache_all)
249ENDPROC(v7_flush_user_cache_range)
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264ENTRY(v7_coherent_kern_range)
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280ENTRY(v7_coherent_user_range)
281 UNWIND(.fnstart )
282 dcache_line_size r2, r3
283 sub r3, r2,
284 bic r12, r0, r3
285#ifdef CONFIG_ARM_ERRATA_764369
286 ALT_SMP(W(dsb))
287 ALT_UP(W(nop))
288#endif
2891:
290 USER( mcr p15, 0, r12, c7, c11, 1 ) @ clean D line to the point of unification
291 add r12, r12, r2
292 cmp r12, r1
293 blo 1b
294 dsb ishst
295#ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND
296 ldr r3, =icache_size
297 ldr r2, [r3,
298#else
299 icache_line_size r2, r3
300#endif
301 sub r3, r2,
302 bic r12, r0, r3
3032:
304 USER( mcr p15, 0, r12, c7, c5, 1 ) @ invalidate I line
305 add r12, r12, r2
306 cmp r12, r1
307 blo 2b
308 mov r0,
309 ALT_SMP(mcr p15, 0, r0, c7, c1, 6) @ invalidate BTB Inner Shareable
310 ALT_UP(mcr p15, 0, r0, c7, c5, 6) @ invalidate BTB
311 dsb ishst
312 isb
313 ret lr
314
315
316
317
318
3199001:
320#ifdef CONFIG_ARM_ERRATA_775420
321 dsb
322#endif
323 mov r0,
324 ret lr
325 UNWIND(.fnend )
326ENDPROC(v7_coherent_kern_range)
327ENDPROC(v7_coherent_user_range)
328
329
330
331
332
333
334
335
336
337
338ENTRY(v7_flush_kern_dcache_area)
339 dcache_line_size r2, r3
340 add r1, r0, r1
341 sub r3, r2,
342 bic r0, r0, r3
343#ifdef CONFIG_ARM_ERRATA_764369
344 ALT_SMP(W(dsb))
345 ALT_UP(W(nop))
346#endif
3471:
348 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line
349 add r0, r0, r2
350 cmp r0, r1
351 blo 1b
352 dsb st
353 ret lr
354ENDPROC(v7_flush_kern_dcache_area)
355
356
357
358
359
360
361
362
363
364
365
366v7_dma_inv_range:
367 dcache_line_size r2, r3
368 sub r3, r2,
369 tst r0, r3
370 bic r0, r0, r3
371#ifdef CONFIG_ARM_ERRATA_764369
372 ALT_SMP(W(dsb))
373 ALT_UP(W(nop))
374#endif
375 mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line
376 addne r0, r0, r2
377
378 tst r1, r3
379 bic r1, r1, r3
380 mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D / U line
381 cmp r0, r1
3821:
383 mcrlo p15, 0, r0, c7, c6, 1 @ invalidate D / U line
384 addlo r0, r0, r2
385 cmplo r0, r1
386 blo 1b
387 dsb st
388 ret lr
389ENDPROC(v7_dma_inv_range)
390
391
392
393
394
395
396v7_dma_clean_range:
397 dcache_line_size r2, r3
398 sub r3, r2,
399 bic r0, r0, r3
400#ifdef CONFIG_ARM_ERRATA_764369
401 ALT_SMP(W(dsb))
402 ALT_UP(W(nop))
403#endif
4041:
405 mcr p15, 0, r0, c7, c10, 1 @ clean D / U line
406 add r0, r0, r2
407 cmp r0, r1
408 blo 1b
409 dsb st
410 ret lr
411ENDPROC(v7_dma_clean_range)
412
413
414
415
416
417
418ENTRY(v7_dma_flush_range)
419 dcache_line_size r2, r3
420 sub r3, r2,
421 bic r0, r0, r3
422#ifdef CONFIG_ARM_ERRATA_764369
423 ALT_SMP(W(dsb))
424 ALT_UP(W(nop))
425#endif
4261:
427 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line
428 add r0, r0, r2
429 cmp r0, r1
430 blo 1b
431 dsb st
432 ret lr
433ENDPROC(v7_dma_flush_range)
434
435
436
437
438
439
440
441ENTRY(v7_dma_map_area)
442 add r1, r1, r0
443 teq r2,
444 beq v7_dma_inv_range
445 b v7_dma_clean_range
446ENDPROC(v7_dma_map_area)
447
448
449
450
451
452
453
454ENTRY(v7_dma_unmap_area)
455 add r1, r1, r0
456 teq r2,
457 bne v7_dma_inv_range
458 ret lr
459ENDPROC(v7_dma_unmap_area)
460
461 __INITDATA
462
463 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
464 define_cache_functions v7
465
466
467
468
469#ifndef CONFIG_CACHE_B15_RAC
470 globl_equ b15_flush_kern_cache_all, v7_flush_kern_cache_all
471#endif
472 globl_equ b15_flush_icache_all, v7_flush_icache_all
473 globl_equ b15_flush_kern_cache_louis, v7_flush_kern_cache_louis
474 globl_equ b15_flush_user_cache_all, v7_flush_user_cache_all
475 globl_equ b15_flush_user_cache_range, v7_flush_user_cache_range
476 globl_equ b15_coherent_kern_range, v7_coherent_kern_range
477 globl_equ b15_coherent_user_range, v7_coherent_user_range
478 globl_equ b15_flush_kern_dcache_area, v7_flush_kern_dcache_area
479
480 globl_equ b15_dma_map_area, v7_dma_map_area
481 globl_equ b15_dma_unmap_area, v7_dma_unmap_area
482 globl_equ b15_dma_flush_range, v7_dma_flush_range
483
484 define_cache_functions b15
485