1
2
3
4
5
6
7#include <hyp/switch.h>
8
9#include <linux/arm-smccc.h>
10#include <linux/kvm_host.h>
11#include <linux/types.h>
12#include <linux/jump_label.h>
13#include <uapi/linux/psci.h>
14
15#include <kvm/arm_psci.h>
16
17#include <asm/barrier.h>
18#include <asm/cpufeature.h>
19#include <asm/kprobes.h>
20#include <asm/kvm_asm.h>
21#include <asm/kvm_emulate.h>
22#include <asm/kvm_hyp.h>
23#include <asm/kvm_mmu.h>
24#include <asm/fpsimd.h>
25#include <asm/debug-monitors.h>
26#include <asm/processor.h>
27#include <asm/thread_info.h>
28
29
30DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
31DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
32DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
33
34static void __activate_traps(struct kvm_vcpu *vcpu)
35{
36 u64 val;
37
38 ___activate_traps(vcpu);
39
40 val = read_sysreg(cpacr_el1);
41 val |= CPACR_EL1_TTA;
42 val &= ~CPACR_EL1_ZEN;
43
44
45
46
47
48
49
50
51
52
53 val |= CPTR_EL2_TAM;
54
55 if (update_fp_enabled(vcpu)) {
56 if (vcpu_has_sve(vcpu))
57 val |= CPACR_EL1_ZEN;
58 } else {
59 val &= ~CPACR_EL1_FPEN;
60 __activate_traps_fpsimd32(vcpu);
61 }
62
63 write_sysreg(val, cpacr_el1);
64
65 write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1);
66}
67NOKPROBE_SYMBOL(__activate_traps);
68
69static void __deactivate_traps(struct kvm_vcpu *vcpu)
70{
71 extern char vectors[];
72
73 ___deactivate_traps(vcpu);
74
75 write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
76
77
78
79
80
81
82 asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
83
84 write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
85 write_sysreg(vectors, vbar_el1);
86}
87NOKPROBE_SYMBOL(__deactivate_traps);
88
89void activate_traps_vhe_load(struct kvm_vcpu *vcpu)
90{
91 __activate_traps_common(vcpu);
92}
93
94void deactivate_traps_vhe_put(void)
95{
96 u64 mdcr_el2 = read_sysreg(mdcr_el2);
97
98 mdcr_el2 &= MDCR_EL2_HPMN_MASK |
99 MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
100 MDCR_EL2_TPMS;
101
102 write_sysreg(mdcr_el2, mdcr_el2);
103
104 __deactivate_traps_common();
105}
106
107
108static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
109{
110 struct kvm_cpu_context *host_ctxt;
111 struct kvm_cpu_context *guest_ctxt;
112 u64 exit_code;
113
114 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
115 host_ctxt->__hyp_running_vcpu = vcpu;
116 guest_ctxt = &vcpu->arch.ctxt;
117
118 sysreg_save_host_state_vhe(host_ctxt);
119
120
121
122
123
124
125
126
127
128
129
130
131 __load_guest_stage2(vcpu->arch.hw_mmu);
132 __activate_traps(vcpu);
133
134 __kvm_adjust_pc(vcpu);
135
136 sysreg_restore_guest_state_vhe(guest_ctxt);
137 __debug_switch_to_guest(vcpu);
138
139 do {
140
141 exit_code = __guest_enter(vcpu);
142
143
144 } while (fixup_guest_exit(vcpu, &exit_code));
145
146 sysreg_save_guest_state_vhe(guest_ctxt);
147
148 __deactivate_traps(vcpu);
149
150 sysreg_restore_host_state_vhe(host_ctxt);
151
152 if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
153 __fpsimd_save_fpexc32(vcpu);
154
155 __debug_switch_to_host(vcpu);
156
157 return exit_code;
158}
159NOKPROBE_SYMBOL(__kvm_vcpu_run_vhe);
160
161int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
162{
163 int ret;
164
165 local_daif_mask();
166
167
168
169
170
171
172
173
174
175
176 pmr_sync();
177
178 ret = __kvm_vcpu_run_vhe(vcpu);
179
180
181
182
183
184 local_daif_restore(DAIF_PROCCTX_NOIRQ);
185
186
187
188
189
190
191 isb();
192
193 return ret;
194}
195
196static void __hyp_call_panic(u64 spsr, u64 elr, u64 par)
197{
198 struct kvm_cpu_context *host_ctxt;
199 struct kvm_vcpu *vcpu;
200
201 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
202 vcpu = host_ctxt->__hyp_running_vcpu;
203
204 __deactivate_traps(vcpu);
205 sysreg_restore_host_state_vhe(host_ctxt);
206
207 panic("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n",
208 spsr, elr,
209 read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR),
210 read_sysreg(hpfar_el2), par, vcpu);
211}
212NOKPROBE_SYMBOL(__hyp_call_panic);
213
214void __noreturn hyp_panic(void)
215{
216 u64 spsr = read_sysreg_el2(SYS_SPSR);
217 u64 elr = read_sysreg_el2(SYS_ELR);
218 u64 par = read_sysreg_par();
219
220 __hyp_call_panic(spsr, elr, par);
221 unreachable();
222}
223
224asmlinkage void kvm_unexpected_el2_exception(void)
225{
226 return __kvm_unexpected_el2_exception();
227}
228