1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/linkage.h>
20#include <linux/const.h>
21#include <asm/unified.h>
22#include <asm/page.h>
23#include <asm/ptrace.h>
24#include <asm/asm-offsets.h>
25#include <asm/kvm_asm.h>
26#include <asm/kvm_arm.h>
27#include <asm/vfpmacros.h>
28#include "interrupts_head.S"
29
30 .text
31
32__kvm_hyp_code_start:
33 .globl __kvm_hyp_code_start
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49ENTRY(__kvm_tlb_flush_vmid_ipa)
50 push {r2, r3}
51
52 add r0, r0,
53 ldrd r2, r3, [r0]
54 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
55 isb
56 mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
57 dsb
58 isb
59 mov r2,
60 mov r3,
61 mcrr p15, 6, r2, r3, c2 @ Back to VMID
62 isb @ Not necessary if followed by eret
63
64 pop {r2, r3}
65 bx lr
66ENDPROC(__kvm_tlb_flush_vmid_ipa)
67
68
69
70
71
72
73
74ENTRY(__kvm_flush_vm_context)
75 mov r0,
76
77
78 mcr p15, 4, r0, c8, c3, 4
79
80 mcr p15, 0, r0, c7, c1, 0
81 dsb
82 isb @ Not necessary if followed by eret
83
84 bx lr
85ENDPROC(__kvm_flush_vm_context)
86
87
88
89
90
91
92
93
94ENTRY(__kvm_vcpu_run)
95 @ Save the vcpu pointer
96 mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
97
98 save_host_regs
99
100 restore_vgic_state
101 restore_timer_state
102
103 @ Store hardware CP15 state and load guest state
104 read_cp15_state store_to_vcpu = 0
105 write_cp15_state read_from_vcpu = 1
106
107 @ If the host kernel has not been configured with VFPv3 support,
108 @ then it is safer if we deny guests from using it as well.
109#ifdef CONFIG_VFPv3
110 @ Set FPEXC_EN so the guest doesn't trap floating point instructions
111 VFPFMRX r2, FPEXC @ VMRS
112 push {r2}
113 orr r2, r2,
114 VFPFMXR FPEXC, r2 @ VMSR
115#endif
116
117 @ Configure Hyp-role
118 configure_hyp_role vmentry
119
120 @ Trap coprocessor CRx accesses
121 set_hstr vmentry
122 set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
123 set_hdcr vmentry
124
125 @ Write configured ID register into MIDR alias
126 ldr r1, [vcpu,
127 mcr p15, 4, r1, c0, c0, 0
128
129 @ Write guest view of MPIDR into VMPIDR
130 ldr r1, [vcpu,
131 mcr p15, 4, r1, c0, c0, 5
132
133 @ Set up guest memory translation
134 ldr r1, [vcpu,
135 add r1, r1,
136 ldrd r2, r3, [r1]
137 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
138
139 @ We're all done, just restore the GPRs and go to the guest
140 restore_guest_regs
141 clrex @ Clear exclusive monitor
142 eret
143
144__kvm_vcpu_return:
145
146
147
148
149
150
151 save_guest_regs
152
153 @ Set VMID == 0
154 mov r2,
155 mov r3,
156 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
157
158 @ Don't trap coprocessor accesses for host kernel
159 set_hstr vmexit
160 set_hdcr vmexit
161 set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
162
163#ifdef CONFIG_VFPv3
164 @ Save floating point registers we if let guest use them.
165 tst r2,
166 bne after_vfp_restore
167
168 @ Switch VFP/NEON hardware state to the host's
169 add r7, vcpu,
170 store_vfp_state r7
171 add r7, vcpu,
172 ldr r7, [r7]
173 restore_vfp_state r7
174
175after_vfp_restore:
176 @ Restore FPEXC_EN which we clobbered on entry
177 pop {r2}
178 VFPFMXR FPEXC, r2
179#endif
180
181 @ Reset Hyp-role
182 configure_hyp_role vmexit
183
184 @ Let host read hardware MIDR
185 mrc p15, 0, r2, c0, c0, 0
186 mcr p15, 4, r2, c0, c0, 0
187
188 @ Back to hardware MPIDR
189 mrc p15, 0, r2, c0, c0, 5
190 mcr p15, 4, r2, c0, c0, 5
191
192 @ Store guest CP15 state and restore host state
193 read_cp15_state store_to_vcpu = 1
194 write_cp15_state read_from_vcpu = 0
195
196 save_timer_state
197 save_vgic_state
198
199 restore_host_regs
200 clrex @ Clear exclusive monitor
201 mov r0, r1 @ Return the return code
202 mov r1,
203 bx lr @ return to IOCTL
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227ENTRY(kvm_call_hyp)
228 hvc
229 bx lr
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261.macro bad_exception exception_code, panic_str
262 push {r0-r2}
263 mrrc p15, 6, r0, r1, c2 @ Read VTTBR
264 lsr r1, r1,
265 ands r1, r1,
266 beq 99f
267
268 load_vcpu @ Load VCPU pointer
269 .if \exception_code == ARM_EXCEPTION_DATA_ABORT
270 mrc p15, 4, r2, c5, c2, 0 @ HSR
271 mrc p15, 4, r1, c6, c0, 0 @ HDFAR
272 str r2, [vcpu,
273 str r1, [vcpu,
274 .endif
275 .if \exception_code == ARM_EXCEPTION_PREF_ABORT
276 mrc p15, 4, r2, c5, c2, 0 @ HSR
277 mrc p15, 4, r1, c6, c0, 2 @ HIFAR
278 str r2, [vcpu,
279 str r1, [vcpu,
280 .endif
281 mov r1,
282 b __kvm_vcpu_return
283
284 @ We were in the host already. Let's craft a panic-ing return to SVC.
28599: mrs r2, cpsr
286 bic r2, r2,
287 orr r2, r2,
288THUMB( orr r2, r2,
289 msr spsr_cxsf, r2
290 mrs r1, ELR_hyp
291 ldr r2, =BSYM(panic)
292 msr ELR_hyp, r2
293 ldr r0, =\panic_str
294 eret
295.endm
296
297 .text
298
299 .align 5
300__kvm_hyp_vector:
301 .globl __kvm_hyp_vector
302
303 @ Hyp-mode exception vector
304 W(b) hyp_reset
305 W(b) hyp_undef
306 W(b) hyp_svc
307 W(b) hyp_pabt
308 W(b) hyp_dabt
309 W(b) hyp_hvc
310 W(b) hyp_irq
311 W(b) hyp_fiq
312
313 .align
314hyp_reset:
315 b hyp_reset
316
317 .align
318hyp_undef:
319 bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
320
321 .align
322hyp_svc:
323 bad_exception ARM_EXCEPTION_HVC, svc_die_str
324
325 .align
326hyp_pabt:
327 bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
328
329 .align
330hyp_dabt:
331 bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
332
333 .align
334hyp_hvc:
335
336
337
338
339 push {r0, r1, r2}
340
341 @ Check syndrome register
342 mrc p15, 4, r1, c5, c2, 0 @ HSR
343 lsr r0, r1,
344#ifdef CONFIG_VFPv3
345 cmp r0,
346 beq switch_to_guest_vfp
347#endif
348 cmp r0,
349 bne guest_trap @ Not HVC instr.
350
351
352
353
354
355 mrrc p15, 6, r0, r2, c2
356 lsr r2, r2,
357 and r2, r2,
358 cmp r2,
359 bne guest_trap @ Guest called HVC
360
361host_switch_to_hyp:
362 pop {r0, r1, r2}
363
364 push {lr}
365 mrs lr, SPSR
366 push {lr}
367
368 mov lr, r0
369 mov r0, r1
370 mov r1, r2
371 mov r2, r3
372
373THUMB( orr lr,
374 blx lr @ Call the HYP function
375
376 pop {lr}
377 msr SPSR_csxf, lr
378 pop {lr}
379 eret
380
381guest_trap:
382 load_vcpu @ Load VCPU pointer to r0
383 str r1, [vcpu,
384
385 @ Check if we need the fault information
386 lsr r1, r1,
387 cmp r1,
388 mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
389 beq 2f
390 cmp r1,
391 bne 1f
392 mrc p15, 4, r2, c6, c0, 0 @ HDFAR
393
3942: str r2, [vcpu,
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410 mrc p15, 4, r1, c5, c2, 0 @ HSR
411 and r0, r1,
412 cmp r0,
413 tsteq r1,
414 mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
415 bne 3f
416
417
418 mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
419 isb
420 mrrc p15, 0, r0, r1, c7 @ PAR
421 tst r0,
422 bne 4f @ Failed translation
423 ubfx r2, r0,
424 lsl r2, r2,
425 orr r2, r2, r1, lsl
426
4273: load_vcpu @ Load VCPU pointer to r0
428 str r2, [r0,
429
4301: mov r1,
431 b __kvm_vcpu_return
432
4334: pop {r0, r1, r2} @ Failed translation, return to guest
434 eret
435
436
437
438
439
440
441
442#ifdef CONFIG_VFPv3
443switch_to_guest_vfp:
444 load_vcpu @ Load VCPU pointer to r0
445 push {r3-r7}
446
447 @ NEON/VFP used. Turn on VFP access.
448 set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))
449
450 @ Switch VFP/NEON hardware state to the guest's
451 add r7, r0,
452 ldr r7, [r7]
453 store_vfp_state r7
454 add r7, r0,
455 restore_vfp_state r7
456
457 pop {r3-r7}
458 pop {r0-r2}
459 eret
460#endif
461
462 .align
463hyp_irq:
464 push {r0, r1, r2}
465 mov r1,
466 load_vcpu @ Load VCPU pointer to r0
467 b __kvm_vcpu_return
468
469 .align
470hyp_fiq:
471 b hyp_fiq
472
473 .ltorg
474
475__kvm_hyp_code_end:
476 .globl __kvm_hyp_code_end
477
478 .section ".rodata"
479
480und_die_str:
481 .ascii "unexpected undefined exception in Hyp mode at: %#08x"
482pabt_die_str:
483 .ascii "unexpected prefetch abort in Hyp mode at: %#08x"
484dabt_die_str:
485 .ascii "unexpected data abort in Hyp mode at: %#08x"
486svc_die_str:
487 .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x"
488