LXR linux/arch/powerpc/kvm/book3s_64

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2#include <asm/asm-offsets.h>
   3#include <asm/cache.h>
   4#include <asm/code-patching-asm.h>
   5#include <asm/exception-64s.h>
   6#include <asm/export.h>
   7#include <asm/kvm_asm.h>
   8#include <asm/kvm_book3s_asm.h>
   9#include <asm/mmu.h>
  10#include <asm/ppc_asm.h>
  11#include <asm/ptrace.h>
  12#include <asm/reg.h>
  13#include <asm/ultravisor-api.h>
  14
  15/*
  16 * These are branched to from interrupt handlers in exception-64s.S which set
  17 * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero.
  18 */
  19
  20/*
  21 * This is a hcall, so register convention is as
  22 * Documentation/powerpc/papr_hcalls.rst.
  23 *
  24 * This may also be a syscall from PR-KVM userspace that is to be
  25 * reflected to the PR guest kernel, so registers may be set up for
  26 * a system call rather than hcall. We don't currently clobber
  27 * anything here, but the 0xc00 handler has already clobbered CTR
  28 * and CR0, so PR-KVM can not support a guest kernel that preserves
  29 * those registers across its system calls.
  30 *
  31 * The state of registers is as kvmppc_interrupt, except CFAR is not
  32 * saved, R13 is not in SCRATCH0, and R10 does not contain the trap.
  33 */
  34.global kvmppc_hcall
  35.balign IFETCH_ALIGN_BYTES
  36kvmppc_hcall:
  37#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
  38        lbz     r10,HSTATE_IN_GUEST(r13)
  39        cmpwi   r10,KVM_GUEST_MODE_HV_P9
  40        beq     kvmppc_p9_exit_hcall
  41#endif
  42        ld      r10,PACA_EXGEN+EX_R13(r13)
  43        SET_SCRATCH0(r10)
  44        li      r10,0xc00
  45        /* Now we look like kvmppc_interrupt */
  46        li      r11,PACA_EXGEN
  47        b       .Lgot_save_area
  48
  49/*
  50 * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that
  51 * call convention:
  52 *
  53 * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area
  54 * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts
  55 * guest R13 also saved in SCRATCH0
  56 * R13          = PACA
  57 * R11          = (H)SRR0
  58 * R12          = (H)SRR1
  59 * R9           = guest CR
  60 * PPR is set to medium
  61 *
  62 * With the addition for KVM:
  63 * R10          = trap vector
  64 */
  65.global kvmppc_interrupt
  66.balign IFETCH_ALIGN_BYTES
  67kvmppc_interrupt:
  68#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
  69        std     r10,HSTATE_SCRATCH0(r13)
  70        lbz     r10,HSTATE_IN_GUEST(r13)
  71        cmpwi   r10,KVM_GUEST_MODE_HV_P9
  72        beq     kvmppc_p9_exit_interrupt
  73        ld      r10,HSTATE_SCRATCH0(r13)
  74#endif
  75        li      r11,PACA_EXGEN
  76        cmpdi   r10,0x200
  77        bgt+    .Lgot_save_area
  78        li      r11,PACA_EXMC
  79        beq     .Lgot_save_area
  80        li      r11,PACA_EXNMI
  81.Lgot_save_area:
  82        add     r11,r11,r13
  83BEGIN_FTR_SECTION
  84        ld      r12,EX_CFAR(r11)
  85        std     r12,HSTATE_CFAR(r13)
  86END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  87        ld      r12,EX_CTR(r11)
  88        mtctr   r12
  89BEGIN_FTR_SECTION
  90        ld      r12,EX_PPR(r11)
  91        std     r12,HSTATE_PPR(r13)
  92END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  93        ld      r12,EX_R12(r11)
  94        std     r12,HSTATE_SCRATCH0(r13)
  95        sldi    r12,r9,32
  96        or      r12,r12,r10
  97        ld      r9,EX_R9(r11)
  98        ld      r10,EX_R10(r11)
  99        ld      r11,EX_R11(r11)
 100
 101        /*
 102         * Hcalls and other interrupts come here after normalising register
 103         * contents and save locations:
 104         *
 105         * R12          = (guest CR << 32) | interrupt vector
 106         * R13          = PACA
 107         * guest R12 saved in shadow HSTATE_SCRATCH0
 108         * guest R13 saved in SPRN_SCRATCH0
 109         */
 110        std     r9,HSTATE_SCRATCH2(r13)
 111        lbz     r9,HSTATE_IN_GUEST(r13)
 112        cmpwi   r9,KVM_GUEST_MODE_SKIP
 113        beq-    .Lmaybe_skip
 114.Lno_skip:
 115#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 116#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
 117        cmpwi   r9,KVM_GUEST_MODE_GUEST
 118        beq     kvmppc_interrupt_pr
 119#endif
 120        b       kvmppc_interrupt_hv
 121#else
 122        b       kvmppc_interrupt_pr
 123#endif
 124
 125/*
 126 * "Skip" interrupts are part of a trick KVM uses a with hash guests to load
 127 * the faulting instruction in guest memory from the the hypervisor without
 128 * walking page tables.
 129 *
 130 * When the guest takes a fault that requires the hypervisor to load the
 131 * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1
 132 * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets
 133 * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions
 134 * but loads and stores will access the guest context. This is used to load
 135 * the faulting instruction using the faulting guest effective address.
 136 *
 137 * However the guest context may not be able to translate, or it may cause a
 138 * machine check or other issue, which results in a fault in the host
 139 * (even with KVM-HV).
 140 *
 141 * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they
 142 * are (or are likely) caused by that load, the instruction is skipped by
 143 * just returning with the PC advanced +4, where it is noticed the load did
 144 * not execute and it goes to the slow path which walks the page tables to
 145 * read guest memory.
 146 */
 147.Lmaybe_skip:
 148        cmpwi   r12,BOOK3S_INTERRUPT_MACHINE_CHECK
 149        beq     1f
 150        cmpwi   r12,BOOK3S_INTERRUPT_DATA_STORAGE
 151        beq     1f
 152        cmpwi   r12,BOOK3S_INTERRUPT_DATA_SEGMENT
 153        beq     1f
 154#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 155        /* HSRR interrupts get 2 added to interrupt number */
 156        cmpwi   r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2
 157        beq     2f
 158#endif
 159        b       .Lno_skip
 1601:      mfspr   r9,SPRN_SRR0
 161        addi    r9,r9,4
 162        mtspr   SPRN_SRR0,r9
 163        ld      r12,HSTATE_SCRATCH0(r13)
 164        ld      r9,HSTATE_SCRATCH2(r13)
 165        GET_SCRATCH0(r13)
 166        RFI_TO_KERNEL
 167#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 1682:      mfspr   r9,SPRN_HSRR0
 169        addi    r9,r9,4
 170        mtspr   SPRN_HSRR0,r9
 171        ld      r12,HSTATE_SCRATCH0(r13)
 172        ld      r9,HSTATE_SCRATCH2(r13)
 173        GET_SCRATCH0(r13)
 174        HRFI_TO_KERNEL
 175#endif
 176
 177#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 178
 179/* Stack frame offsets for kvmppc_p9_enter_guest */
 180#define SFS                     (144 + STACK_FRAME_MIN_SIZE)
 181#define STACK_SLOT_NVGPRS       (SFS - 144)     /* 18 gprs */
 182
 183/*
 184 * void kvmppc_p9_enter_guest(struct vcpu *vcpu);
 185 *
 186 * Enter the guest on a ISAv3.0 or later system.
 187 */
 188.balign IFETCH_ALIGN_BYTES
 189_GLOBAL(kvmppc_p9_enter_guest)
 190EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest)
 191        mflr    r0
 192        std     r0,PPC_LR_STKOFF(r1)
 193        stdu    r1,-SFS(r1)
 194
 195        std     r1,HSTATE_HOST_R1(r13)
 196
 197        mfcr    r4
 198        stw     r4,SFS+8(r1)
 199
 200        reg = 14
 201        .rept   18
 202        std     reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
 203        reg = reg + 1
 204        .endr
 205
 206        ld      r4,VCPU_LR(r3)
 207        mtlr    r4
 208        ld      r4,VCPU_CTR(r3)
 209        mtctr   r4
 210        ld      r4,VCPU_XER(r3)
 211        mtspr   SPRN_XER,r4
 212
 213        ld      r1,VCPU_CR(r3)
 214
 215BEGIN_FTR_SECTION
 216        ld      r4,VCPU_CFAR(r3)
 217        mtspr   SPRN_CFAR,r4
 218END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 219BEGIN_FTR_SECTION
 220        ld      r4,VCPU_PPR(r3)
 221        mtspr   SPRN_PPR,r4
 222END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 223
 224        reg = 4
 225        .rept   28
 226        ld      reg,__VCPU_GPR(reg)(r3)
 227        reg = reg + 1
 228        .endr
 229
 230        ld      r4,VCPU_KVM(r3)
 231        lbz     r4,KVM_SECURE_GUEST(r4)
 232        cmpdi   r4,0
 233        ld      r4,VCPU_GPR(R4)(r3)
 234        bne     .Lret_to_ultra
 235
 236        mtcr    r1
 237
 238        ld      r0,VCPU_GPR(R0)(r3)
 239        ld      r1,VCPU_GPR(R1)(r3)
 240        ld      r2,VCPU_GPR(R2)(r3)
 241        ld      r3,VCPU_GPR(R3)(r3)
 242
 243        HRFI_TO_GUEST
 244        b       .
 245
 246        /*
 247         * Use UV_RETURN ultracall to return control back to the Ultravisor
 248         * after processing an hypercall or interrupt that was forwarded
 249         * (a.k.a. reflected) to the Hypervisor.
 250         *
 251         * All registers have already been reloaded except the ucall requires:
 252         *   R0 = hcall result
 253         *   R2 = SRR1, so UV can detect a synthesized interrupt (if any)
 254         *   R3 = UV_RETURN
 255         */
 256.Lret_to_ultra:
 257        mtcr    r1
 258        ld      r1,VCPU_GPR(R1)(r3)
 259
 260        ld      r0,VCPU_GPR(R3)(r3)
 261        mfspr   r2,SPRN_SRR1
 262        LOAD_REG_IMMEDIATE(r3, UV_RETURN)
 263        sc      2
 264
 265/*
 266 * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from
 267 * above if the interrupt was taken for a guest that was entered via
 268 * kvmppc_p9_enter_guest().
 269 *
 270 * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs
 271 * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re-
 272 * establishes the host stack and registers to return from the
 273 * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers
 274 * (SPRs and FP, VEC, etc).
 275 */
 276.balign IFETCH_ALIGN_BYTES
 277kvmppc_p9_exit_hcall:
 278        mfspr   r11,SPRN_SRR0
 279        mfspr   r12,SPRN_SRR1
 280        li      r10,0xc00
 281        std     r10,HSTATE_SCRATCH0(r13)
 282
 283.balign IFETCH_ALIGN_BYTES
 284kvmppc_p9_exit_interrupt:
 285        /*
 286         * If set to KVM_GUEST_MODE_HV_P9 but we're still in the
 287         * hypervisor, that means we can't return from the entry stack.
 288         */
 289        rldicl. r10,r12,64-MSR_HV_LG,63
 290        bne-    kvmppc_p9_bad_interrupt
 291
 292        std     r1,HSTATE_SCRATCH1(r13)
 293        std     r3,HSTATE_SCRATCH2(r13)
 294        ld      r1,HSTATE_HOST_R1(r13)
 295        ld      r3,HSTATE_KVM_VCPU(r13)
 296
 297        std     r9,VCPU_CR(r3)
 298
 2991:
 300        std     r11,VCPU_PC(r3)
 301        std     r12,VCPU_MSR(r3)
 302
 303        reg = 14
 304        .rept   18
 305        std     reg,__VCPU_GPR(reg)(r3)
 306        reg = reg + 1
 307        .endr
 308
 309        /* r1, r3, r9-r13 are saved to vcpu by C code */
 310        std     r0,VCPU_GPR(R0)(r3)
 311        std     r2,VCPU_GPR(R2)(r3)
 312        reg = 4
 313        .rept   5
 314        std     reg,__VCPU_GPR(reg)(r3)
 315        reg = reg + 1
 316        .endr
 317
 318        ld      r2,PACATOC(r13)
 319
 320        mflr    r4
 321        std     r4,VCPU_LR(r3)
 322        mfspr   r4,SPRN_XER
 323        std     r4,VCPU_XER(r3)
 324
 325        reg = 14
 326        .rept   18
 327        ld      reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
 328        reg = reg + 1
 329        .endr
 330
 331        lwz     r4,SFS+8(r1)
 332        mtcr    r4
 333
 334        /*
 335         * Flush the link stack here, before executing the first blr on the
 336         * way out of the guest.
 337         *
 338         * The link stack won't match coming out of the guest anyway so the
 339         * only cost is the flush itself. The call clobbers r0.
 340         */
 3411:      nop
 342        patch_site 1b patch__call_kvm_flush_link_stack_p9
 343
 344        addi    r1,r1,SFS
 345        ld      r0,PPC_LR_STKOFF(r1)
 346        mtlr    r0
 347        blr
 348
 349/*
 350 * Took an interrupt somewhere right before HRFID to guest, so registers are
 351 * in a bad way. Return things hopefully enough to run host virtual code and
 352 * run the Linux interrupt handler (SRESET or MCE) to print something useful.
 353 *
 354 * We could be really clever and save all host registers in known locations
 355 * before setting HSTATE_IN_GUEST, then restoring them all here, and setting
 356 * return address to a fixup that sets them up again. But that's a lot of
 357 * effort for a small bit of code. Lots of other things to do first.
 358 */
 359kvmppc_p9_bad_interrupt:
 360BEGIN_MMU_FTR_SECTION
 361        /*
 362         * Hash host doesn't try to recover MMU (requires host SLB reload)
 363         */
 364        b       .
 365END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
 366        /*
 367         * Clean up guest registers to give host a chance to run.
 368         */
 369        li      r10,0
 370        mtspr   SPRN_AMR,r10
 371        mtspr   SPRN_IAMR,r10
 372        mtspr   SPRN_CIABR,r10
 373        mtspr   SPRN_DAWRX0,r10
 374BEGIN_FTR_SECTION
 375        mtspr   SPRN_DAWRX1,r10
 376END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
 377        mtspr   SPRN_PID,r10
 378
 379        /*
 380         * Switch to host MMU mode
 381         */
 382        ld      r10, HSTATE_KVM_VCPU(r13)
 383        ld      r10, VCPU_KVM(r10)
 384        lwz     r10, KVM_HOST_LPID(r10)
 385        mtspr   SPRN_LPID,r10
 386
 387        ld      r10, HSTATE_KVM_VCPU(r13)
 388        ld      r10, VCPU_KVM(r10)
 389        ld      r10, KVM_HOST_LPCR(r10)
 390        mtspr   SPRN_LPCR,r10
 391
 392        /*
 393         * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear
 394         * MSR_RI in r12 ([H]SRR1) so the handler won't try to return.
 395         */
 396        li      r10,KVM_GUEST_MODE_NONE
 397        stb     r10,HSTATE_IN_GUEST(r13)
 398        li      r10,MSR_RI
 399        andc    r12,r12,r10
 400
 401        /*
 402         * Go back to interrupt handler. MCE and SRESET have their specific
 403         * PACA save area so they should be used directly. They set up their
 404         * own stack. The other handlers all use EXGEN. They will use the
 405         * guest r1 if it looks like a kernel stack, so just load the
 406         * emergency stack and go to program check for all other interrupts.
 407         */
 408        ld      r10,HSTATE_SCRATCH0(r13)
 409        cmpwi   r10,BOOK3S_INTERRUPT_MACHINE_CHECK
 410        beq     machine_check_common
 411
 412        cmpwi   r10,BOOK3S_INTERRUPT_SYSTEM_RESET
 413        beq     system_reset_common
 414
 415        b       .
 416#endif
 417