1
2#include <linux/linkage.h>
3#include <asm/asm.h>
4#include <asm/bitsperlong.h>
5#include <asm/kvm_vcpu_regs.h>
6#include <asm/nospec-branch.h>
7#include <asm/segment.h>
8
9#define WORD_SIZE (BITS_PER_LONG / 8)
10
11#define VCPU_RAX __VCPU_REGS_RAX * WORD_SIZE
12#define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE
13#define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE
14#define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE
15
16#define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE
17#define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE
18#define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE
19
20#ifdef CONFIG_X86_64
21#define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE
22#define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE
23#define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE
24#define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE
25#define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE
26#define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE
27#define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE
28#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
29#endif
30
31.section .noinstr.text, "ax"
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47SYM_FUNC_START_LOCAL(vmx_vmenter)
48
49 je 2f
50
511: vmresume
52 ret
53
542: vmlaunch
55 ret
56
573: cmpb $0, kvm_rebooting
58 je 4f
59 ret
604: ud2
61
62 _ASM_EXTABLE(1b, 3b)
63 _ASM_EXTABLE(2b, 3b)
64
65SYM_FUNC_END(vmx_vmenter)
66
67
68
69
70
71
72
73
74
75
76
77SYM_FUNC_START(vmx_vmexit)
78#ifdef CONFIG_RETPOLINE
79 ALTERNATIVE "jmp .Lvmexit_skip_rsb", "", X86_FEATURE_RETPOLINE
80
81 push %_ASM_AX
82
83
84 FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
85
86
87 or $1, %_ASM_AX
88
89 pop %_ASM_AX
90.Lvmexit_skip_rsb:
91#endif
92 ret
93SYM_FUNC_END(vmx_vmexit)
94
95
96
97
98
99
100
101
102
103
104SYM_FUNC_START(__vmx_vcpu_run)
105 push %_ASM_BP
106 mov %_ASM_SP, %_ASM_BP
107#ifdef CONFIG_X86_64
108 push %r15
109 push %r14
110 push %r13
111 push %r12
112#else
113 push %edi
114 push %esi
115#endif
116 push %_ASM_BX
117
118
119
120
121
122 push %_ASM_ARG2
123
124
125 mov %_ASM_ARG3B, %bl
126
127
128 lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2
129 call vmx_update_host_rsp
130
131
132 mov (%_ASM_SP), %_ASM_AX
133
134
135 testb %bl, %bl
136
137
138 mov VCPU_RCX(%_ASM_AX), %_ASM_CX
139 mov VCPU_RDX(%_ASM_AX), %_ASM_DX
140 mov VCPU_RBX(%_ASM_AX), %_ASM_BX
141 mov VCPU_RBP(%_ASM_AX), %_ASM_BP
142 mov VCPU_RSI(%_ASM_AX), %_ASM_SI
143 mov VCPU_RDI(%_ASM_AX), %_ASM_DI
144#ifdef CONFIG_X86_64
145 mov VCPU_R8 (%_ASM_AX), %r8
146 mov VCPU_R9 (%_ASM_AX), %r9
147 mov VCPU_R10(%_ASM_AX), %r10
148 mov VCPU_R11(%_ASM_AX), %r11
149 mov VCPU_R12(%_ASM_AX), %r12
150 mov VCPU_R13(%_ASM_AX), %r13
151 mov VCPU_R14(%_ASM_AX), %r14
152 mov VCPU_R15(%_ASM_AX), %r15
153#endif
154
155 mov VCPU_RAX(%_ASM_AX), %_ASM_AX
156
157
158 call vmx_vmenter
159
160
161 jbe 2f
162
163
164 push %_ASM_AX
165
166
167 mov WORD_SIZE(%_ASM_SP), %_ASM_AX
168
169
170 pop VCPU_RAX(%_ASM_AX)
171 mov %_ASM_CX, VCPU_RCX(%_ASM_AX)
172 mov %_ASM_DX, VCPU_RDX(%_ASM_AX)
173 mov %_ASM_BX, VCPU_RBX(%_ASM_AX)
174 mov %_ASM_BP, VCPU_RBP(%_ASM_AX)
175 mov %_ASM_SI, VCPU_RSI(%_ASM_AX)
176 mov %_ASM_DI, VCPU_RDI(%_ASM_AX)
177#ifdef CONFIG_X86_64
178 mov %r8, VCPU_R8 (%_ASM_AX)
179 mov %r9, VCPU_R9 (%_ASM_AX)
180 mov %r10, VCPU_R10(%_ASM_AX)
181 mov %r11, VCPU_R11(%_ASM_AX)
182 mov %r12, VCPU_R12(%_ASM_AX)
183 mov %r13, VCPU_R13(%_ASM_AX)
184 mov %r14, VCPU_R14(%_ASM_AX)
185 mov %r15, VCPU_R15(%_ASM_AX)
186#endif
187
188
189 xor %eax, %eax
190
191
192
193
194
195
196
197
198
199
2001: xor %ecx, %ecx
201 xor %edx, %edx
202 xor %ebx, %ebx
203 xor %ebp, %ebp
204 xor %esi, %esi
205 xor %edi, %edi
206#ifdef CONFIG_X86_64
207 xor %r8d, %r8d
208 xor %r9d, %r9d
209 xor %r10d, %r10d
210 xor %r11d, %r11d
211 xor %r12d, %r12d
212 xor %r13d, %r13d
213 xor %r14d, %r14d
214 xor %r15d, %r15d
215#endif
216
217
218 add $WORD_SIZE, %_ASM_SP
219 pop %_ASM_BX
220
221#ifdef CONFIG_X86_64
222 pop %r12
223 pop %r13
224 pop %r14
225 pop %r15
226#else
227 pop %esi
228 pop %edi
229#endif
230 pop %_ASM_BP
231 ret
232
233
2342: mov $1, %eax
235 jmp 1b
236SYM_FUNC_END(__vmx_vcpu_run)
237
238
239.section .text, "ax"
240
241
242
243
244
245
246
247
248
249SYM_FUNC_START(vmread_error_trampoline)
250 push %_ASM_BP
251 mov %_ASM_SP, %_ASM_BP
252
253 push %_ASM_AX
254 push %_ASM_CX
255 push %_ASM_DX
256#ifdef CONFIG_X86_64
257 push %rdi
258 push %rsi
259 push %r8
260 push %r9
261 push %r10
262 push %r11
263#endif
264#ifdef CONFIG_X86_64
265
266 mov 3*WORD_SIZE(%rbp), %_ASM_ARG2
267 mov 2*WORD_SIZE(%rbp), %_ASM_ARG1
268#else
269
270 push 3*WORD_SIZE(%ebp)
271 push 2*WORD_SIZE(%ebp)
272#endif
273
274 call vmread_error
275
276#ifndef CONFIG_X86_64
277 add $8, %esp
278#endif
279
280
281 _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP)
282
283#ifdef CONFIG_X86_64
284 pop %r11
285 pop %r10
286 pop %r9
287 pop %r8
288 pop %rsi
289 pop %rdi
290#endif
291 pop %_ASM_DX
292 pop %_ASM_CX
293 pop %_ASM_AX
294 pop %_ASM_BP
295
296 ret
297SYM_FUNC_END(vmread_error_trampoline)
298
299SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
300
301
302
303
304
305 push %_ASM_BP
306 mov %_ASM_SP, %_ASM_BP
307
308#ifdef CONFIG_X86_64
309
310
311
312
313 and $-16, %rsp
314 push $__KERNEL_DS
315 push %rbp
316#endif
317 pushf
318 push $__KERNEL_CS
319 CALL_NOSPEC _ASM_ARG1
320
321
322
323
324
325
326
327 mov %_ASM_BP, %_ASM_SP
328 pop %_ASM_BP
329 ret
330SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
331