LXR linux/arch/x86/kvm/vmx/vmenter.S

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#include <linux/linkage.h>
   3#include <asm/asm.h>
   4#include <asm/bitsperlong.h>
   5#include <asm/kvm_vcpu_regs.h>
   6#include <asm/nospec-branch.h>
   7#include <asm/segment.h>
   8
   9#define WORD_SIZE (BITS_PER_LONG / 8)
  10
  11#define VCPU_RAX        __VCPU_REGS_RAX * WORD_SIZE
  12#define VCPU_RCX        __VCPU_REGS_RCX * WORD_SIZE
  13#define VCPU_RDX        __VCPU_REGS_RDX * WORD_SIZE
  14#define VCPU_RBX        __VCPU_REGS_RBX * WORD_SIZE
  15/* Intentionally omit RSP as it's context switched by hardware */
  16#define VCPU_RBP        __VCPU_REGS_RBP * WORD_SIZE
  17#define VCPU_RSI        __VCPU_REGS_RSI * WORD_SIZE
  18#define VCPU_RDI        __VCPU_REGS_RDI * WORD_SIZE
  19
  20#ifdef CONFIG_X86_64
  21#define VCPU_R8         __VCPU_REGS_R8  * WORD_SIZE
  22#define VCPU_R9         __VCPU_REGS_R9  * WORD_SIZE
  23#define VCPU_R10        __VCPU_REGS_R10 * WORD_SIZE
  24#define VCPU_R11        __VCPU_REGS_R11 * WORD_SIZE
  25#define VCPU_R12        __VCPU_REGS_R12 * WORD_SIZE
  26#define VCPU_R13        __VCPU_REGS_R13 * WORD_SIZE
  27#define VCPU_R14        __VCPU_REGS_R14 * WORD_SIZE
  28#define VCPU_R15        __VCPU_REGS_R15 * WORD_SIZE
  29#endif
  30
  31.section .noinstr.text, "ax"
  32
  33/**
  34 * vmx_vmenter - VM-Enter the current loaded VMCS
  35 *
  36 * %RFLAGS.ZF:  !VMCS.LAUNCHED, i.e. controls VMLAUNCH vs. VMRESUME
  37 *
  38 * Returns:
  39 *      %RFLAGS.CF is set on VM-Fail Invalid
  40 *      %RFLAGS.ZF is set on VM-Fail Valid
  41 *      %RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
  42 *
  43 * Note that VMRESUME/VMLAUNCH fall-through and return directly if
  44 * they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump
  45 * to vmx_vmexit.
  46 */
  47SYM_FUNC_START_LOCAL(vmx_vmenter)
  48        /* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */
  49        je 2f
  50
  511:      vmresume
  52        ret
  53
  542:      vmlaunch
  55        ret
  56
  573:      cmpb $0, kvm_rebooting
  58        je 4f
  59        ret
  604:      ud2
  61
  62        _ASM_EXTABLE(1b, 3b)
  63        _ASM_EXTABLE(2b, 3b)
  64
  65SYM_FUNC_END(vmx_vmenter)
  66
  67/**
  68 * vmx_vmexit - Handle a VMX VM-Exit
  69 *
  70 * Returns:
  71 *      %RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
  72 *
  73 * This is vmx_vmenter's partner in crime.  On a VM-Exit, control will jump
  74 * here after hardware loads the host's state, i.e. this is the destination
  75 * referred to by VMCS.HOST_RIP.
  76 */
  77SYM_FUNC_START(vmx_vmexit)
  78#ifdef CONFIG_RETPOLINE
  79        ALTERNATIVE "jmp .Lvmexit_skip_rsb", "", X86_FEATURE_RETPOLINE
  80        /* Preserve guest's RAX, it's used to stuff the RSB. */
  81        push %_ASM_AX
  82
  83        /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
  84        FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
  85
  86        /* Clear RFLAGS.CF and RFLAGS.ZF to preserve VM-Exit, i.e. !VM-Fail. */
  87        or $1, %_ASM_AX
  88
  89        pop %_ASM_AX
  90.Lvmexit_skip_rsb:
  91#endif
  92        ret
  93SYM_FUNC_END(vmx_vmexit)
  94
  95/**
  96 * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
  97 * @vmx:        struct vcpu_vmx * (forwarded to vmx_update_host_rsp)
  98 * @regs:       unsigned long * (to guest registers)
  99 * @launched:   %true if the VMCS has been launched
 100 *
 101 * Returns:
 102 *      0 on VM-Exit, 1 on VM-Fail
 103 */
 104SYM_FUNC_START(__vmx_vcpu_run)
 105        push %_ASM_BP
 106        mov  %_ASM_SP, %_ASM_BP
 107#ifdef CONFIG_X86_64
 108        push %r15
 109        push %r14
 110        push %r13
 111        push %r12
 112#else
 113        push %edi
 114        push %esi
 115#endif
 116        push %_ASM_BX
 117
 118        /*
 119         * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
 120         * @regs is needed after VM-Exit to save the guest's register values.
 121         */
 122        push %_ASM_ARG2
 123
 124        /* Copy @launched to BL, _ASM_ARG3 is volatile. */
 125        mov %_ASM_ARG3B, %bl
 126
 127        /* Adjust RSP to account for the CALL to vmx_vmenter(). */
 128        lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2
 129        call vmx_update_host_rsp
 130
 131        /* Load @regs to RAX. */
 132        mov (%_ASM_SP), %_ASM_AX
 133
 134        /* Check if vmlaunch or vmresume is needed */
 135        testb %bl, %bl
 136
 137        /* Load guest registers.  Don't clobber flags. */
 138        mov VCPU_RCX(%_ASM_AX), %_ASM_CX
 139        mov VCPU_RDX(%_ASM_AX), %_ASM_DX
 140        mov VCPU_RBX(%_ASM_AX), %_ASM_BX
 141        mov VCPU_RBP(%_ASM_AX), %_ASM_BP
 142        mov VCPU_RSI(%_ASM_AX), %_ASM_SI
 143        mov VCPU_RDI(%_ASM_AX), %_ASM_DI
 144#ifdef CONFIG_X86_64
 145        mov VCPU_R8 (%_ASM_AX),  %r8
 146        mov VCPU_R9 (%_ASM_AX),  %r9
 147        mov VCPU_R10(%_ASM_AX), %r10
 148        mov VCPU_R11(%_ASM_AX), %r11
 149        mov VCPU_R12(%_ASM_AX), %r12
 150        mov VCPU_R13(%_ASM_AX), %r13
 151        mov VCPU_R14(%_ASM_AX), %r14
 152        mov VCPU_R15(%_ASM_AX), %r15
 153#endif
 154        /* Load guest RAX.  This kills the @regs pointer! */
 155        mov VCPU_RAX(%_ASM_AX), %_ASM_AX
 156
 157        /* Enter guest mode */
 158        call vmx_vmenter
 159
 160        /* Jump on VM-Fail. */
 161        jbe 2f
 162
 163        /* Temporarily save guest's RAX. */
 164        push %_ASM_AX
 165
 166        /* Reload @regs to RAX. */
 167        mov WORD_SIZE(%_ASM_SP), %_ASM_AX
 168
 169        /* Save all guest registers, including RAX from the stack */
 170        pop           VCPU_RAX(%_ASM_AX)
 171        mov %_ASM_CX, VCPU_RCX(%_ASM_AX)
 172        mov %_ASM_DX, VCPU_RDX(%_ASM_AX)
 173        mov %_ASM_BX, VCPU_RBX(%_ASM_AX)
 174        mov %_ASM_BP, VCPU_RBP(%_ASM_AX)
 175        mov %_ASM_SI, VCPU_RSI(%_ASM_AX)
 176        mov %_ASM_DI, VCPU_RDI(%_ASM_AX)
 177#ifdef CONFIG_X86_64
 178        mov %r8,  VCPU_R8 (%_ASM_AX)
 179        mov %r9,  VCPU_R9 (%_ASM_AX)
 180        mov %r10, VCPU_R10(%_ASM_AX)
 181        mov %r11, VCPU_R11(%_ASM_AX)
 182        mov %r12, VCPU_R12(%_ASM_AX)
 183        mov %r13, VCPU_R13(%_ASM_AX)
 184        mov %r14, VCPU_R14(%_ASM_AX)
 185        mov %r15, VCPU_R15(%_ASM_AX)
 186#endif
 187
 188        /* Clear RAX to indicate VM-Exit (as opposed to VM-Fail). */
 189        xor %eax, %eax
 190
 191        /*
 192         * Clear all general purpose registers except RSP and RAX to prevent
 193         * speculative use of the guest's values, even those that are reloaded
 194         * via the stack.  In theory, an L1 cache miss when restoring registers
 195         * could lead to speculative execution with the guest's values.
 196         * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
 197         * free.  RSP and RAX are exempt as RSP is restored by hardware during
 198         * VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail.
 199         */
 2001:      xor %ecx, %ecx
 201        xor %edx, %edx
 202        xor %ebx, %ebx
 203        xor %ebp, %ebp
 204        xor %esi, %esi
 205        xor %edi, %edi
 206#ifdef CONFIG_X86_64
 207        xor %r8d,  %r8d
 208        xor %r9d,  %r9d
 209        xor %r10d, %r10d
 210        xor %r11d, %r11d
 211        xor %r12d, %r12d
 212        xor %r13d, %r13d
 213        xor %r14d, %r14d
 214        xor %r15d, %r15d
 215#endif
 216
 217        /* "POP" @regs. */
 218        add $WORD_SIZE, %_ASM_SP
 219        pop %_ASM_BX
 220
 221#ifdef CONFIG_X86_64
 222        pop %r12
 223        pop %r13
 224        pop %r14
 225        pop %r15
 226#else
 227        pop %esi
 228        pop %edi
 229#endif
 230        pop %_ASM_BP
 231        ret
 232
 233        /* VM-Fail.  Out-of-line to avoid a taken Jcc after VM-Exit. */
 2342:      mov $1, %eax
 235        jmp 1b
 236SYM_FUNC_END(__vmx_vcpu_run)
 237
 238
 239.section .text, "ax"
 240
 241/**
 242 * vmread_error_trampoline - Trampoline from inline asm to vmread_error()
 243 * @field:      VMCS field encoding that failed
 244 * @fault:      %true if the VMREAD faulted, %false if it failed
 245
 246 * Save and restore volatile registers across a call to vmread_error().  Note,
 247 * all parameters are passed on the stack.
 248 */
 249SYM_FUNC_START(vmread_error_trampoline)
 250        push %_ASM_BP
 251        mov  %_ASM_SP, %_ASM_BP
 252
 253        push %_ASM_AX
 254        push %_ASM_CX
 255        push %_ASM_DX
 256#ifdef CONFIG_X86_64
 257        push %rdi
 258        push %rsi
 259        push %r8
 260        push %r9
 261        push %r10
 262        push %r11
 263#endif
 264#ifdef CONFIG_X86_64
 265        /* Load @field and @fault to arg1 and arg2 respectively. */
 266        mov 3*WORD_SIZE(%rbp), %_ASM_ARG2
 267        mov 2*WORD_SIZE(%rbp), %_ASM_ARG1
 268#else
 269        /* Parameters are passed on the stack for 32-bit (see asmlinkage). */
 270        push 3*WORD_SIZE(%ebp)
 271        push 2*WORD_SIZE(%ebp)
 272#endif
 273
 274        call vmread_error
 275
 276#ifndef CONFIG_X86_64
 277        add $8, %esp
 278#endif
 279
 280        /* Zero out @fault, which will be popped into the result register. */
 281        _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP)
 282
 283#ifdef CONFIG_X86_64
 284        pop %r11
 285        pop %r10
 286        pop %r9
 287        pop %r8
 288        pop %rsi
 289        pop %rdi
 290#endif
 291        pop %_ASM_DX
 292        pop %_ASM_CX
 293        pop %_ASM_AX
 294        pop %_ASM_BP
 295
 296        ret
 297SYM_FUNC_END(vmread_error_trampoline)
 298
 299SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
 300        /*
 301         * Unconditionally create a stack frame, getting the correct RSP on the
 302         * stack (for x86-64) would take two instructions anyways, and RBP can
 303         * be used to restore RSP to make objtool happy (see below).
 304         */
 305        push %_ASM_BP
 306        mov %_ASM_SP, %_ASM_BP
 307
 308#ifdef CONFIG_X86_64
 309        /*
 310         * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
 311         * creating the synthetic interrupt stack frame for the IRQ/NMI.
 312         */
 313        and  $-16, %rsp
 314        push $__KERNEL_DS
 315        push %rbp
 316#endif
 317        pushf
 318        push $__KERNEL_CS
 319        CALL_NOSPEC _ASM_ARG1
 320
 321        /*
 322         * "Restore" RSP from RBP, even though IRET has already unwound RSP to
 323         * the correct value.  objtool doesn't know the callee will IRET and,
 324         * without the explicit restore, thinks the stack is getting walloped.
 325         * Using an unwind hint is problematic due to x86-64's dynamic alignment.
 326         */
 327        mov %_ASM_BP, %_ASM_SP
 328        pop %_ASM_BP
 329        ret
 330SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
 331