1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <asm/assembler.h>
24#include <asm/asm-offsets.h>
25#include <asm/hwcap.h>
26#include <asm/pgtable.h>
27#include <asm/pgtable-hwdef.h>
28#include <asm/cpufeature.h>
29#include <asm/alternative.h>
30
31#ifdef CONFIG_ARM64_64K_PAGES
32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
33
34#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
35#else
36#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
37#endif
38
39#define TCR_SMP_FLAGS TCR_SHARED
40
41
42#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
43
44#define MAIR(attr, mt) ((attr) << ((mt) * 8))
45
46
47
48
49
50
51ENTRY(cpu_do_idle)
52 dsb sy
53 wfi
54 ret
55ENDPROC(cpu_do_idle)
56
57#ifdef CONFIG_CPU_PM
58
59
60
61
62
63ENTRY(cpu_do_suspend)
64 mrs x2, tpidr_el0
65 mrs x3, tpidrro_el0
66 mrs x4, contextidr_el1
67 mrs x5, cpacr_el1
68 mrs x6, tcr_el1
69 mrs x7, vbar_el1
70 mrs x8, mdscr_el1
71 mrs x9, oslsr_el1
72 mrs x10, sctlr_el1
73 stp x2, x3, [x0]
74 stp x4, xzr, [x0,
75 stp x5, x6, [x0,
76 stp x7, x8, [x0,
77 stp x9, x10, [x0,
78 ret
79ENDPROC(cpu_do_suspend)
80
81
82
83
84
85
86ENTRY(cpu_do_resume)
87 ldp x2, x3, [x0]
88 ldp x4, x5, [x0,
89 ldp x6, x8, [x0,
90 ldp x9, x10, [x0,
91 ldp x11, x12, [x0,
92 msr tpidr_el0, x2
93 msr tpidrro_el0, x3
94 msr contextidr_el1, x4
95 msr cpacr_el1, x6
96
97
98 mrs x5, tcr_el1
99 bfi x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
100
101 msr tcr_el1, x8
102 msr vbar_el1, x9
103
104
105
106
107
108
109
110 disable_dbg
111 msr mdscr_el1, x10
112
113 msr sctlr_el1, x12
114
115
116
117 ubfx x11, x11,
118 msr oslar_el1, x11
119 reset_pmuserenr_el0 x0
120 isb
121 ret
122ENDPROC(cpu_do_resume)
123#endif
124
125
126
127
128
129
130
131
132ENTRY(cpu_do_switch_mm)
133 mmid x1, x1
134 bfi x0, x1,
135 msr ttbr0_el1, x0
136 isb
137alternative_if_not ARM64_WORKAROUND_CAVIUM_27456
138 ret
139 nop
140 nop
141 nop
142alternative_else
143 ic iallu
144 dsb nsh
145 isb
146 ret
147alternative_endif
148ENDPROC(cpu_do_switch_mm)
149
150 .pushsection ".idmap.text", "ax"
151
152
153
154
155
156
157ENTRY(idmap_cpu_replace_ttbr1)
158 mrs x2, daif
159 msr daifset,
160
161 adrp x1, empty_zero_page
162 msr ttbr1_el1, x1
163 isb
164
165 tlbi vmalle1
166 dsb nsh
167 isb
168
169 msr ttbr1_el1, x0
170 isb
171
172 msr daif, x2
173
174 ret
175ENDPROC(idmap_cpu_replace_ttbr1)
176 .popsection
177
178
179
180
181
182
183
184ENTRY(__cpu_setup)
185 tlbi vmalle1
186 dsb nsh
187
188 mov x0,
189 msr cpacr_el1, x0
190 mov x0,
191 msr mdscr_el1, x0
192 isb
193 enable_dbg
194 reset_pmuserenr_el0 x0
195
196
197
198
199
200
201
202
203
204
205
206
207 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
208 MAIR(0x04, MT_DEVICE_nGnRE) | \
209 MAIR(0x0c, MT_DEVICE_GRE) | \
210 MAIR(0x44, MT_NORMAL_NC) | \
211 MAIR(0xff, MT_NORMAL) | \
212 MAIR(0xbb, MT_NORMAL_WT)
213 msr mair_el1, x5
214
215
216
217 adr x5, crval
218 ldp w5, w6, [x5]
219 mrs x0, sctlr_el1
220 bic x0, x0, x5
221 orr x0, x0, x6
222
223
224
225
226 ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
227 TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
228 tcr_set_idmap_t0sz x10, x9
229
230
231
232
233
234 mrs x9, ID_AA64MMFR0_EL1
235 bfi x10, x9,
236#ifdef CONFIG_ARM64_HW_AFDBM
237
238
239
240 mrs x9, ID_AA64MMFR1_EL1
241 and x9, x9,
242 cbz x9, 2f
243 cmp x9,
244 b.lt 1f
245 orr x10, x10,
2461: orr x10, x10,
2472:
248#endif
249 msr tcr_el1, x10
250 ret
251ENDPROC(__cpu_setup)
252
253
254
255
256
257
258
259
260
261
262
263
264
265 .type crval,
266crval:
267 .word 0xfcffffff
268 .word 0x34d5d91d
269