1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#ifndef __ASSEMBLY__
20#error "Only include this from assembly code"
21#endif
22
23#ifndef __ASM_ASSEMBLER_H
24#define __ASM_ASSEMBLER_H
25
26#include <asm/asm-offsets.h>
27#include <asm/cpufeature.h>
28#include <asm/mmu_context.h>
29#include <asm/page.h>
30#include <asm/pgtable-hwdef.h>
31#include <asm/ptrace.h>
32#include <asm/thread_info.h>
33
34
35
36
37 .macro disable_irq
38 msr daifset, #2
39 .endm
40
41 .macro enable_irq
42 msr daifclr, #2
43 .endm
44
45 .macro save_and_disable_irq, flags
46 mrs \flags, daif
47 msr daifset, #2
48 .endm
49
50 .macro restore_irq, flags
51 msr daif, \flags
52 .endm
53
54
55
56
57 .macro disable_dbg
58 msr daifset, #8
59 .endm
60
61 .macro enable_dbg
62 msr daifclr, #8
63 .endm
64
65 .macro disable_step_tsk, flgs, tmp
66 tbz \flgs, #TIF_SINGLESTEP, 9990f
67 mrs \tmp, mdscr_el1
68 bic \tmp, \tmp, #1
69 msr mdscr_el1, \tmp
70 isb
719990:
72 .endm
73
74 .macro enable_step_tsk, flgs, tmp
75 tbz \flgs, #TIF_SINGLESTEP, 9990f
76 disable_dbg
77 mrs \tmp, mdscr_el1
78 orr \tmp, \tmp, #1
79 msr mdscr_el1, \tmp
809990:
81 .endm
82
83
84
85
86
87
88 .macro enable_dbg_and_irq
89 msr daifclr, #(8 | 2)
90 .endm
91
92
93
94
95 .macro smp_dmb, opt
96 dmb \opt
97 .endm
98
99
100
101
102 .macro nops, num
103 .rept \num
104 nop
105 .endr
106 .endm
107
108
109
110
111 .macro _asm_extable, from, to
112 .pushsection __ex_table, "a"
113 .align 3
114 .long (\from - .), (\to - .)
115 .popsection
116 .endm
117
118#define USER(l, x...) \
1199999: x; \
120 _asm_extable 9999b, l
121
122
123
124
125lr .req x30
126
127
128
129
130 .macro ventry label
131 .align 7
132 b \label
133 .endm
134
135
136
137
138#ifdef CONFIG_CPU_BIG_ENDIAN
139#define CPU_BE(code...) code
140#else
141#define CPU_BE(code...)
142#endif
143
144
145
146
147#ifdef CONFIG_CPU_BIG_ENDIAN
148#define CPU_LE(code...)
149#else
150#define CPU_LE(code...) code
151#endif
152
153
154
155
156
157
158#ifndef CONFIG_CPU_BIG_ENDIAN
159 .macro regs_to_64, rd, lbits, hbits
160#else
161 .macro regs_to_64, rd, hbits, lbits
162#endif
163 orr \rd, \lbits, \hbits, lsl #32
164 .endm
165
166
167
168
169
170
171
172
173
174
175
176
177 .macro adr_l, dst, sym
178#ifndef MODULE
179 adrp \dst, \sym
180 add \dst, \dst, :lo12:\sym
181#else
182 movz \dst, #:abs_g3:\sym
183 movk \dst, #:abs_g2_nc:\sym
184 movk \dst, #:abs_g1_nc:\sym
185 movk \dst, #:abs_g0_nc:\sym
186#endif
187 .endm
188
189
190
191
192
193
194
195
196 .macro ldr_l, dst, sym, tmp=
197#ifndef MODULE
198 .ifb \tmp
199 adrp \dst, \sym
200 ldr \dst, [\dst, :lo12:\sym]
201 .else
202 adrp \tmp, \sym
203 ldr \dst, [\tmp, :lo12:\sym]
204 .endif
205#else
206 .ifb \tmp
207 adr_l \dst, \sym
208 ldr \dst, [\dst]
209 .else
210 adr_l \tmp, \sym
211 ldr \dst, [\tmp]
212 .endif
213#endif
214 .endm
215
216
217
218
219
220
221
222 .macro str_l, src, sym, tmp
223#ifndef MODULE
224 adrp \tmp, \sym
225 str \src, [\tmp, :lo12:\sym]
226#else
227 adr_l \tmp, \sym
228 str \src, [\tmp]
229#endif
230 .endm
231
232
233
234
235
236
237
238 .macro adr_this_cpu, dst, sym, tmp
239#ifndef MODULE
240 adrp \tmp, \sym
241 add \dst, \tmp, #:lo12:\sym
242#else
243 adr_l \dst, \sym
244#endif
245 mrs \tmp, tpidr_el1
246 add \dst, \dst, \tmp
247 .endm
248
249
250
251
252
253
254 .macro ldr_this_cpu dst, sym, tmp
255 adr_l \dst, \sym
256 mrs \tmp, tpidr_el1
257 ldr \dst, [\dst, \tmp]
258 .endm
259
260
261
262
263 .macro vma_vm_mm, rd, rn
264 ldr \rd, [\rn, #VMA_VM_MM]
265 .endm
266
267
268
269
270 .macro mmid, rd, rn
271 ldr \rd, [\rn, #MM_CONTEXT_ID]
272 .endm
273
274
275
276
277
278 .macro read_ctr, reg
279alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE
280 mrs \reg, ctr_el0
281 nop
282alternative_else
283 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
284alternative_endif
285 .endm
286
287
288
289
290
291
292 .macro raw_dcache_line_size, reg, tmp
293 mrs \tmp, ctr_el0
294 ubfm \tmp, \tmp, #16, #19
295 mov \reg, #4
296 lsl \reg, \reg, \tmp
297 .endm
298
299
300
301
302 .macro dcache_line_size, reg, tmp
303 read_ctr \tmp
304 ubfm \tmp, \tmp, #16, #19
305 mov \reg, #4
306 lsl \reg, \reg, \tmp
307 .endm
308
309
310
311
312
313 .macro raw_icache_line_size, reg, tmp
314 mrs \tmp, ctr_el0
315 and \tmp, \tmp, #0xf
316 mov \reg, #4
317 lsl \reg, \reg, \tmp
318 .endm
319
320
321
322
323 .macro icache_line_size, reg, tmp
324 read_ctr \tmp
325 and \tmp, \tmp, #0xf
326 mov \reg, #4
327 lsl \reg, \reg, \tmp
328 .endm
329
330
331
332
333 .macro tcr_set_idmap_t0sz, valreg, tmpreg
334#ifndef CONFIG_ARM64_VA_BITS_48
335 ldr_l \tmpreg, idmap_t0sz
336 bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
337#endif
338 .endm
339
340
341
342
343
344
345
346
347
348
349
350 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
351 dcache_line_size \tmp1, \tmp2
352 add \size, \kaddr, \size
353 sub \tmp2, \tmp1, #1
354 bic \kaddr, \kaddr, \tmp2
3559998:
356 .if (\op == cvau || \op == cvac)
357alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
358 dc \op, \kaddr
359alternative_else
360 dc civac, \kaddr
361alternative_endif
362 .elseif (\op == cvap)
363alternative_if ARM64_HAS_DCPOP
364 sys 3, c7, c12, 1, \kaddr
365alternative_else
366 dc cvac, \kaddr
367alternative_endif
368 .else
369 dc \op, \kaddr
370 .endif
371 add \kaddr, \kaddr, \tmp1
372 cmp \kaddr, \size
373 b.lo 9998b
374 dsb \domain
375 .endm
376
377
378
379
380 .macro reset_pmuserenr_el0, tmpreg
381 mrs \tmpreg, id_aa64dfr0_el1
382 sbfx \tmpreg, \tmpreg, #8, #4
383 cmp \tmpreg, #1
384 b.lt 9000f
385 msr pmuserenr_el0, xzr
3869000:
387 .endm
388
389
390
391
392 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
3939998: ldp \t1, \t2, [\src]
394 ldp \t3, \t4, [\src, #16]
395 ldp \t5, \t6, [\src, #32]
396 ldp \t7, \t8, [\src, #48]
397 add \src, \src, #64
398 stnp \t1, \t2, [\dest]
399 stnp \t3, \t4, [\dest, #16]
400 stnp \t5, \t6, [\dest, #32]
401 stnp \t7, \t8, [\dest, #48]
402 add \dest, \dest, #64
403 tst \src, #(PAGE_SIZE - 1)
404 b.ne 9998b
405 .endm
406
407
408
409
410
411#define ENDPIPROC(x) \
412 .globl __pi_##x; \
413 .type __pi_##x, %function; \
414 .set __pi_##x, x; \
415 .size __pi_##x, . - x; \
416 ENDPROC(x)
417
418
419
420
421#ifdef CONFIG_KPROBES
422#define NOKPROBE(x) \
423 .pushsection "_kprobe_blacklist", "aw"; \
424 .quad x; \
425 .popsection;
426#else
427#define NOKPROBE(x)
428#endif
429
430
431
432
433
434
435 .macro le64sym, sym
436 .long \sym\()_lo32
437 .long \sym\()_hi32
438 .endm
439
440
441
442
443
444
445 .macro mov_q, reg, val
446 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
447 movz \reg, :abs_g1_s:\val
448 .else
449 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
450 movz \reg, :abs_g2_s:\val
451 .else
452 movz \reg, :abs_g3:\val
453 movk \reg, :abs_g2_nc:\val
454 .endif
455 movk \reg, :abs_g1_nc:\val
456 .endif
457 movk \reg, :abs_g0_nc:\val
458 .endm
459
460
461
462
463 .macro get_thread_info, rd
464 mrs \rd, sp_el0
465 .endm
466
467
468
469
470
471
472
473
474 .macro pre_ttbr0_update_workaround, val, tmp0, tmp1
475#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
476alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003
477 mrs \tmp0, ttbr0_el1
478 mov \tmp1, #FALKOR_RESERVED_ASID
479 bfi \tmp0, \tmp1, #48, #16
480 msr ttbr0_el1, \tmp0
481 isb
482 bfi \tmp0, \val, #0, #48
483 msr ttbr0_el1, \tmp0
484 isb
485alternative_else_nop_endif
486#endif
487 .endm
488
489
490
491
492 .macro post_ttbr0_update_workaround
493#ifdef CONFIG_CAVIUM_ERRATUM_27456
494alternative_if ARM64_WORKAROUND_CAVIUM_27456
495 ic iallu
496 dsb nsh
497 isb
498alternative_else_nop_endif
499#endif
500 .endm
501
502#endif
503