1
2
3
4
5
6
7
8#ifndef __ASSEMBLY__
9#error "Only include this from assembly code"
10#endif
11
12#ifndef __ASM_ASSEMBLER_H
13#define __ASM_ASSEMBLER_H
14
15#include <asm-generic/export.h>
16
17#include <asm/asm-offsets.h>
18#include <asm/cpufeature.h>
19#include <asm/cputype.h>
20#include <asm/debug-monitors.h>
21#include <asm/page.h>
22#include <asm/pgtable-hwdef.h>
23#include <asm/ptrace.h>
24#include <asm/thread_info.h>
25
26 .macro save_and_disable_daif, flags
27 mrs \flags, daif
28 msr daifset, #0xf
29 .endm
30
31 .macro disable_daif
32 msr daifset, #0xf
33 .endm
34
35 .macro enable_daif
36 msr daifclr, #0xf
37 .endm
38
39 .macro restore_daif, flags:req
40 msr daif, \flags
41 .endm
42
43
44 .macro inherit_daif, pstate:req, tmp:req
45 and \tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
46 msr daif, \tmp
47 .endm
48
49
50 .macro enable_da_f
51 msr daifclr, #(8 | 4 | 1)
52 .endm
53
54
55
56
57 .macro save_and_disable_irq, flags
58 mrs \flags, daif
59 msr daifset, #2
60 .endm
61
62 .macro restore_irq, flags
63 msr daif, \flags
64 .endm
65
66 .macro enable_dbg
67 msr daifclr, #8
68 .endm
69
70 .macro disable_step_tsk, flgs, tmp
71 tbz \flgs, #TIF_SINGLESTEP, 9990f
72 mrs \tmp, mdscr_el1
73 bic \tmp, \tmp, #DBG_MDSCR_SS
74 msr mdscr_el1, \tmp
75 isb
769990:
77 .endm
78
79
80 .macro enable_step_tsk, flgs, tmp
81 tbz \flgs, #TIF_SINGLESTEP, 9990f
82 mrs \tmp, mdscr_el1
83 orr \tmp, \tmp, #DBG_MDSCR_SS
84 msr mdscr_el1, \tmp
859990:
86 .endm
87
88
89
90
91 .macro smp_dmb, opt
92 dmb \opt
93 .endm
94
95
96
97
98 .macro esb
99#ifdef CONFIG_ARM64_RAS_EXTN
100 hint #16
101#else
102 nop
103#endif
104 .endm
105
106
107
108
109 .macro csdb
110 hint #20
111 .endm
112
113
114
115
116 .macro sb
117alternative_if_not ARM64_HAS_SB
118 dsb nsh
119 isb
120alternative_else
121 SB_BARRIER_INSN
122 nop
123alternative_endif
124 .endm
125
126
127
128
129
130 .macro mask_nospec64, idx, limit, tmp
131 sub \tmp, \idx, \limit
132 bic \tmp, \tmp, \idx
133 and \idx, \idx, \tmp, asr #63
134 csdb
135 .endm
136
137
138
139
140 .macro nops, num
141 .rept \num
142 nop
143 .endr
144 .endm
145
146
147
148
149 .macro _asm_extable, from, to
150 .pushsection __ex_table, "a"
151 .align 3
152 .long (\from - .), (\to - .)
153 .popsection
154 .endm
155
156#define USER(l, x...) \
1579999: x; \
158 _asm_extable 9999b, l
159
160
161
162
163lr .req x30
164
165
166
167
168 .macro ventry label
169 .align 7
170 b \label
171 .endm
172
173
174
175
176#ifdef CONFIG_CPU_BIG_ENDIAN
177#define CPU_BE(code...) code
178#else
179#define CPU_BE(code...)
180#endif
181
182
183
184
185#ifdef CONFIG_CPU_BIG_ENDIAN
186#define CPU_LE(code...)
187#else
188#define CPU_LE(code...) code
189#endif
190
191
192
193
194
195
196#ifndef CONFIG_CPU_BIG_ENDIAN
197 .macro regs_to_64, rd, lbits, hbits
198#else
199 .macro regs_to_64, rd, hbits, lbits
200#endif
201 orr \rd, \lbits, \hbits, lsl #32
202 .endm
203
204
205
206
207
208
209
210
211
212 .macro adr_l, dst, sym
213 adrp \dst, \sym
214 add \dst, \dst, :lo12:\sym
215 .endm
216
217
218
219
220
221
222
223
224 .macro ldr_l, dst, sym, tmp=
225 .ifb \tmp
226 adrp \dst, \sym
227 ldr \dst, [\dst, :lo12:\sym]
228 .else
229 adrp \tmp, \sym
230 ldr \dst, [\tmp, :lo12:\sym]
231 .endif
232 .endm
233
234
235
236
237
238
239
240 .macro str_l, src, sym, tmp
241 adrp \tmp, \sym
242 str \src, [\tmp, :lo12:\sym]
243 .endm
244
245
246
247
248
249
250 .macro adr_this_cpu, dst, sym, tmp
251 adrp \tmp, \sym
252 add \dst, \tmp, #:lo12:\sym
253alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
254 mrs \tmp, tpidr_el1
255alternative_else
256 mrs \tmp, tpidr_el2
257alternative_endif
258 add \dst, \dst, \tmp
259 .endm
260
261
262
263
264
265
266 .macro ldr_this_cpu dst, sym, tmp
267 adr_l \dst, \sym
268alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
269 mrs \tmp, tpidr_el1
270alternative_else
271 mrs \tmp, tpidr_el2
272alternative_endif
273 ldr \dst, [\dst, \tmp]
274 .endm
275
276
277
278
279 .macro vma_vm_mm, rd, rn
280 ldr \rd, [\rn, #VMA_VM_MM]
281 .endm
282
283
284
285
286 .macro mmid, rd, rn
287 ldr \rd, [\rn, #MM_CONTEXT_ID]
288 .endm
289
290
291
292
293 .macro read_ctr, reg
294alternative_if_not ARM64_MISMATCHED_CACHE_TYPE
295 mrs \reg, ctr_el0
296 nop
297alternative_else
298 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
299alternative_endif
300 .endm
301
302
303
304
305
306
307 .macro raw_dcache_line_size, reg, tmp
308 mrs \tmp, ctr_el0
309 ubfm \tmp, \tmp, #16, #19
310 mov \reg, #4
311 lsl \reg, \reg, \tmp
312 .endm
313
314
315
316
317 .macro dcache_line_size, reg, tmp
318 read_ctr \tmp
319 ubfm \tmp, \tmp, #16, #19
320 mov \reg, #4
321 lsl \reg, \reg, \tmp
322 .endm
323
324
325
326
327
328 .macro raw_icache_line_size, reg, tmp
329 mrs \tmp, ctr_el0
330 and \tmp, \tmp, #0xf
331 mov \reg, #4
332 lsl \reg, \reg, \tmp
333 .endm
334
335
336
337
338 .macro icache_line_size, reg, tmp
339 read_ctr \tmp
340 and \tmp, \tmp, #0xf
341 mov \reg, #4
342 lsl \reg, \reg, \tmp
343 .endm
344
345
346
347
348 .macro tcr_set_t0sz, valreg, t0sz
349 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
350 .endm
351
352
353
354
355
356
357
358
359
360 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1
361 mrs \tmp0, ID_AA64MMFR0_EL1
362
363 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
364 mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX
365 cmp \tmp0, \tmp1
366 csel \tmp0, \tmp1, \tmp0, hi
367 bfi \tcr, \tmp0, \pos, #3
368 .endm
369
370
371
372
373
374
375
376
377
378
379
380 .macro __dcache_op_workaround_clean_cache, op, kaddr
381alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
382 dc \op, \kaddr
383alternative_else
384 dc civac, \kaddr
385alternative_endif
386 .endm
387
388 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
389 dcache_line_size \tmp1, \tmp2
390 add \size, \kaddr, \size
391 sub \tmp2, \tmp1, #1
392 bic \kaddr, \kaddr, \tmp2
3939998:
394 .ifc \op, cvau
395 __dcache_op_workaround_clean_cache \op, \kaddr
396 .else
397 .ifc \op, cvac
398 __dcache_op_workaround_clean_cache \op, \kaddr
399 .else
400 .ifc \op, cvap
401 sys 3, c7, c12, 1, \kaddr
402 .else
403 .ifc \op, cvadp
404 sys 3, c7, c13, 1, \kaddr
405 .else
406 dc \op, \kaddr
407 .endif
408 .endif
409 .endif
410 .endif
411 add \kaddr, \kaddr, \tmp1
412 cmp \kaddr, \size
413 b.lo 9998b
414 dsb \domain
415 .endm
416
417
418
419
420
421
422
423
424
425 .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
426 icache_line_size \tmp1, \tmp2
427 sub \tmp2, \tmp1, #1
428 bic \tmp2, \start, \tmp2
4299997:
430USER(\label, ic ivau, \tmp2)
431 add \tmp2, \tmp2, \tmp1
432 cmp \tmp2, \end
433 b.lo 9997b
434 dsb ish
435 isb
436 .endm
437
438
439
440
441 .macro reset_pmuserenr_el0, tmpreg
442 mrs \tmpreg, id_aa64dfr0_el1
443 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4
444 cmp \tmpreg, #1
445 b.lt 9000f
446 msr pmuserenr_el0, xzr
4479000:
448 .endm
449
450
451
452
453 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
4549998: ldp \t1, \t2, [\src]
455 ldp \t3, \t4, [\src, #16]
456 ldp \t5, \t6, [\src, #32]
457 ldp \t7, \t8, [\src, #48]
458 add \src, \src, #64
459 stnp \t1, \t2, [\dest]
460 stnp \t3, \t4, [\dest, #16]
461 stnp \t5, \t6, [\dest, #32]
462 stnp \t7, \t8, [\dest, #48]
463 add \dest, \dest, #64
464 tst \src, #(PAGE_SIZE - 1)
465 b.ne 9998b
466 .endm
467
468
469
470
471
472#define ENDPIPROC(x) \
473 .globl __pi_##x; \
474 .type __pi_##x, %function; \
475 .set __pi_##x, x; \
476 .size __pi_##x, . - x; \
477 ENDPROC(x)
478
479
480
481
482#ifdef CONFIG_KPROBES
483#define NOKPROBE(x) \
484 .pushsection "_kprobe_blacklist", "aw"; \
485 .quad x; \
486 .popsection;
487#else
488#define NOKPROBE(x)
489#endif
490
491#ifdef CONFIG_KASAN
492#define EXPORT_SYMBOL_NOKASAN(name)
493#else
494#define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name)
495#endif
496
497
498
499
500
501
502
503 .macro le64sym, sym
504 .long \sym\()_lo32
505 .long \sym\()_hi32
506 .endm
507
508
509
510
511
512
513 .macro mov_q, reg, val
514 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
515 movz \reg, :abs_g1_s:\val
516 .else
517 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
518 movz \reg, :abs_g2_s:\val
519 .else
520 movz \reg, :abs_g3:\val
521 movk \reg, :abs_g2_nc:\val
522 .endif
523 movk \reg, :abs_g1_nc:\val
524 .endif
525 movk \reg, :abs_g0_nc:\val
526 .endm
527
528
529
530
531 .macro get_current_task, rd
532 mrs \rd, sp_el0
533 .endm
534
535
536
537
538
539
540
541 .macro offset_ttbr1, ttbr
542#ifdef CONFIG_ARM64_USER_VA_BITS_52
543 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
544#endif
545 .endm
546
547
548
549
550
551
552 .macro restore_ttbr1, ttbr
553#ifdef CONFIG_ARM64_USER_VA_BITS_52
554 bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
555#endif
556 .endm
557
558
559
560
561
562
563
564
565 .macro phys_to_ttbr, ttbr, phys
566#ifdef CONFIG_ARM64_PA_BITS_52
567 orr \ttbr, \phys, \phys, lsr #46
568 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52
569#else
570 mov \ttbr, \phys
571#endif
572 .endm
573
574 .macro phys_to_pte, pte, phys
575#ifdef CONFIG_ARM64_PA_BITS_52
576
577
578
579
580 orr \pte, \phys, \phys, lsr #36
581 and \pte, \pte, #PTE_ADDR_MASK
582#else
583 mov \pte, \phys
584#endif
585 .endm
586
587 .macro pte_to_phys, phys, pte
588#ifdef CONFIG_ARM64_PA_BITS_52
589 ubfiz \phys, \pte, #(48 - 16 - 12), #16
590 bfxil \phys, \pte, #16, #32
591 lsl \phys, \phys, #16
592#else
593 and \phys, \pte, #PTE_ADDR_MASK
594#endif
595 .endm
596
597
598
599
600 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2
601#ifdef CONFIG_FUJITSU_ERRATUM_010001
602 mrs \tmp1, midr_el1
603
604 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK
605 and \tmp1, \tmp1, \tmp2
606 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001
607 cmp \tmp1, \tmp2
608 b.ne 10f
609
610 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001
611 bic \tcr, \tcr, \tmp2
61210:
613#endif
614 .endm
615
616
617
618
619
620 .macro pre_disable_mmu_workaround
621#ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041
622 isb
623#endif
624 .endm
625
626
627
628
629
630
631
632 .macro frame_push, regcount:req, extra
633 __frame st, \regcount, \extra
634 .endm
635
636
637
638
639
640
641
642 .macro frame_pop
643 __frame ld
644 .endm
645
646 .macro __frame_regs, reg1, reg2, op, num
647 .if .Lframe_regcount == \num
648 \op\()r \reg1, [sp, #(\num + 1) * 8]
649 .elseif .Lframe_regcount > \num
650 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8]
651 .endif
652 .endm
653
654 .macro __frame, op, regcount, extra=0
655 .ifc \op, st
656 .if (\regcount) < 0 || (\regcount) > 10
657 .error "regcount should be in the range [0 ... 10]"
658 .endif
659 .if ((\extra) % 16) != 0
660 .error "extra should be a multiple of 16 bytes"
661 .endif
662 .ifdef .Lframe_regcount
663 .if .Lframe_regcount != -1
664 .error "frame_push/frame_pop may not be nested"
665 .endif
666 .endif
667 .set .Lframe_regcount, \regcount
668 .set .Lframe_extra, \extra
669 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16
670 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]!
671 mov x29, sp
672 .endif
673
674 __frame_regs x19, x20, \op, 1
675 __frame_regs x21, x22, \op, 3
676 __frame_regs x23, x24, \op, 5
677 __frame_regs x25, x26, \op, 7
678 __frame_regs x27, x28, \op, 9
679
680 .ifc \op, ld
681 .if .Lframe_regcount == -1
682 .error "frame_push/frame_pop may not be nested"
683 .endif
684 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra
685 .set .Lframe_regcount, -1
686 .endif
687 .endm
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724 .macro cond_yield_neon, lbl
725 if_will_cond_yield_neon
726 do_cond_yield_neon
727 endif_yield_neon \lbl
728 .endm
729
730 .macro if_will_cond_yield_neon
731#ifdef CONFIG_PREEMPT
732 get_current_task x0
733 ldr x0, [x0, #TSK_TI_PREEMPT]
734 sub x0, x0, #PREEMPT_DISABLE_OFFSET
735 cbz x0, .Lyield_\@
736
737 .subsection 1
738.Lyield_\@ :
739#else
740 .section ".discard.cond_yield_neon", "ax"
741#endif
742 .endm
743
744 .macro do_cond_yield_neon
745 bl kernel_neon_end
746 bl kernel_neon_begin
747 .endm
748
749 .macro endif_yield_neon, lbl
750 .ifnb \lbl
751 b \lbl
752 .else
753 b .Lyield_out_\@
754 .endif
755 .previous
756.Lyield_out_\@ :
757 .endm
758
759#endif
760