LXR linux/arch/x86/kernel/sev-shared.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * AMD Encrypted Register State Support
   4 *
   5 * Author: Joerg Roedel <jroedel@suse.de>
   6 *
   7 * This file is not compiled stand-alone. It contains code shared
   8 * between the pre-decompression boot code and the running Linux kernel
   9 * and is included directly into both code-bases.
  10 */
  11
  12#ifndef __BOOT_COMPRESSED
  13#define error(v)        pr_err(v)
  14#define has_cpuflag(f)  boot_cpu_has(f)
  15#endif
  16
  17static bool __init sev_es_check_cpu_features(void)
  18{
  19        if (!has_cpuflag(X86_FEATURE_RDRAND)) {
  20                error("RDRAND instruction not supported - no trusted source of randomness available\n");
  21                return false;
  22        }
  23
  24        return true;
  25}
  26
  27static void __noreturn sev_es_terminate(unsigned int reason)
  28{
  29        u64 val = GHCB_MSR_TERM_REQ;
  30
  31        /*
  32         * Tell the hypervisor what went wrong - only reason-set 0 is
  33         * currently supported.
  34         */
  35        val |= GHCB_SEV_TERM_REASON(0, reason);
  36
  37        /* Request Guest Termination from Hypvervisor */
  38        sev_es_wr_ghcb_msr(val);
  39        VMGEXIT();
  40
  41        while (true)
  42                asm volatile("hlt\n" : : : "memory");
  43}
  44
  45static bool sev_es_negotiate_protocol(void)
  46{
  47        u64 val;
  48
  49        /* Do the GHCB protocol version negotiation */
  50        sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
  51        VMGEXIT();
  52        val = sev_es_rd_ghcb_msr();
  53
  54        if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
  55                return false;
  56
  57        if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR ||
  58            GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR)
  59                return false;
  60
  61        return true;
  62}
  63
  64static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
  65{
  66        ghcb->save.sw_exit_code = 0;
  67        memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
  68}
  69
  70static bool vc_decoding_needed(unsigned long exit_code)
  71{
  72        /* Exceptions don't require to decode the instruction */
  73        return !(exit_code >= SVM_EXIT_EXCP_BASE &&
  74                 exit_code <= SVM_EXIT_LAST_EXCP);
  75}
  76
  77static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
  78                                      struct pt_regs *regs,
  79                                      unsigned long exit_code)
  80{
  81        enum es_result ret = ES_OK;
  82
  83        memset(ctxt, 0, sizeof(*ctxt));
  84        ctxt->regs = regs;
  85
  86        if (vc_decoding_needed(exit_code))
  87                ret = vc_decode_insn(ctxt);
  88
  89        return ret;
  90}
  91
  92static void vc_finish_insn(struct es_em_ctxt *ctxt)
  93{
  94        ctxt->regs->ip += ctxt->insn.length;
  95}
  96
  97static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
  98                                          struct es_em_ctxt *ctxt,
  99                                          u64 exit_code, u64 exit_info_1,
 100                                          u64 exit_info_2)
 101{
 102        enum es_result ret;
 103
 104        /* Fill in protocol and format specifiers */
 105        ghcb->protocol_version = GHCB_PROTOCOL_MAX;
 106        ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
 107
 108        ghcb_set_sw_exit_code(ghcb, exit_code);
 109        ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
 110        ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
 111
 112        sev_es_wr_ghcb_msr(__pa(ghcb));
 113        VMGEXIT();
 114
 115        if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
 116                u64 info = ghcb->save.sw_exit_info_2;
 117                unsigned long v;
 118
 119                info = ghcb->save.sw_exit_info_2;
 120                v = info & SVM_EVTINJ_VEC_MASK;
 121
 122                /* Check if exception information from hypervisor is sane. */
 123                if ((info & SVM_EVTINJ_VALID) &&
 124                    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
 125                    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
 126                        ctxt->fi.vector = v;
 127                        if (info & SVM_EVTINJ_VALID_ERR)
 128                                ctxt->fi.error_code = info >> 32;
 129                        ret = ES_EXCEPTION;
 130                } else {
 131                        ret = ES_VMM_ERROR;
 132                }
 133        } else if (ghcb->save.sw_exit_info_1 & 0xffffffff) {
 134                ret = ES_VMM_ERROR;
 135        } else {
 136                ret = ES_OK;
 137        }
 138
 139        return ret;
 140}
 141
 142/*
 143 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
 144 * page yet, so it only supports the MSR based communication with the
 145 * hypervisor and only the CPUID exit-code.
 146 */
 147void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
 148{
 149        unsigned int fn = lower_bits(regs->ax, 32);
 150        unsigned long val;
 151
 152        /* Only CPUID is supported via MSR protocol */
 153        if (exit_code != SVM_EXIT_CPUID)
 154                goto fail;
 155
 156        sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
 157        VMGEXIT();
 158        val = sev_es_rd_ghcb_msr();
 159        if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
 160                goto fail;
 161        regs->ax = val >> 32;
 162
 163        sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
 164        VMGEXIT();
 165        val = sev_es_rd_ghcb_msr();
 166        if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
 167                goto fail;
 168        regs->bx = val >> 32;
 169
 170        sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
 171        VMGEXIT();
 172        val = sev_es_rd_ghcb_msr();
 173        if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
 174                goto fail;
 175        regs->cx = val >> 32;
 176
 177        sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
 178        VMGEXIT();
 179        val = sev_es_rd_ghcb_msr();
 180        if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
 181                goto fail;
 182        regs->dx = val >> 32;
 183
 184        /*
 185         * This is a VC handler and the #VC is only raised when SEV-ES is
 186         * active, which means SEV must be active too. Do sanity checks on the
 187         * CPUID results to make sure the hypervisor does not trick the kernel
 188         * into the no-sev path. This could map sensitive data unencrypted and
 189         * make it accessible to the hypervisor.
 190         *
 191         * In particular, check for:
 192         *      - Availability of CPUID leaf 0x8000001f
 193         *      - SEV CPUID bit.
 194         *
 195         * The hypervisor might still report the wrong C-bit position, but this
 196         * can't be checked here.
 197         */
 198
 199        if (fn == 0x80000000 && (regs->ax < 0x8000001f))
 200                /* SEV leaf check */
 201                goto fail;
 202        else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
 203                /* SEV bit */
 204                goto fail;
 205
 206        /* Skip over the CPUID two-byte opcode */
 207        regs->ip += 2;
 208
 209        return;
 210
 211fail:
 212        /* Terminate the guest */
 213        sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST);
 214}
 215
 216static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
 217                                          void *src, char *buf,
 218                                          unsigned int data_size,
 219                                          unsigned int count,
 220                                          bool backwards)
 221{
 222        int i, b = backwards ? -1 : 1;
 223        enum es_result ret = ES_OK;
 224
 225        for (i = 0; i < count; i++) {
 226                void *s = src + (i * data_size * b);
 227                char *d = buf + (i * data_size);
 228
 229                ret = vc_read_mem(ctxt, s, d, data_size);
 230                if (ret != ES_OK)
 231                        break;
 232        }
 233
 234        return ret;
 235}
 236
 237static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
 238                                           void *dst, char *buf,
 239                                           unsigned int data_size,
 240                                           unsigned int count,
 241                                           bool backwards)
 242{
 243        int i, s = backwards ? -1 : 1;
 244        enum es_result ret = ES_OK;
 245
 246        for (i = 0; i < count; i++) {
 247                void *d = dst + (i * data_size * s);
 248                char *b = buf + (i * data_size);
 249
 250                ret = vc_write_mem(ctxt, d, b, data_size);
 251                if (ret != ES_OK)
 252                        break;
 253        }
 254
 255        return ret;
 256}
 257
 258#define IOIO_TYPE_STR  BIT(2)
 259#define IOIO_TYPE_IN   1
 260#define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
 261#define IOIO_TYPE_OUT  0
 262#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
 263
 264#define IOIO_REP       BIT(3)
 265
 266#define IOIO_ADDR_64   BIT(9)
 267#define IOIO_ADDR_32   BIT(8)
 268#define IOIO_ADDR_16   BIT(7)
 269
 270#define IOIO_DATA_32   BIT(6)
 271#define IOIO_DATA_16   BIT(5)
 272#define IOIO_DATA_8    BIT(4)
 273
 274#define IOIO_SEG_ES    (0 << 10)
 275#define IOIO_SEG_DS    (3 << 10)
 276
 277static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
 278{
 279        struct insn *insn = &ctxt->insn;
 280        *exitinfo = 0;
 281
 282        switch (insn->opcode.bytes[0]) {
 283        /* INS opcodes */
 284        case 0x6c:
 285        case 0x6d:
 286                *exitinfo |= IOIO_TYPE_INS;
 287                *exitinfo |= IOIO_SEG_ES;
 288                *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 289                break;
 290
 291        /* OUTS opcodes */
 292        case 0x6e:
 293        case 0x6f:
 294                *exitinfo |= IOIO_TYPE_OUTS;
 295                *exitinfo |= IOIO_SEG_DS;
 296                *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 297                break;
 298
 299        /* IN immediate opcodes */
 300        case 0xe4:
 301        case 0xe5:
 302                *exitinfo |= IOIO_TYPE_IN;
 303                *exitinfo |= (u8)insn->immediate.value << 16;
 304                break;
 305
 306        /* OUT immediate opcodes */
 307        case 0xe6:
 308        case 0xe7:
 309                *exitinfo |= IOIO_TYPE_OUT;
 310                *exitinfo |= (u8)insn->immediate.value << 16;
 311                break;
 312
 313        /* IN register opcodes */
 314        case 0xec:
 315        case 0xed:
 316                *exitinfo |= IOIO_TYPE_IN;
 317                *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 318                break;
 319
 320        /* OUT register opcodes */
 321        case 0xee:
 322        case 0xef:
 323                *exitinfo |= IOIO_TYPE_OUT;
 324                *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 325                break;
 326
 327        default:
 328                return ES_DECODE_FAILED;
 329        }
 330
 331        switch (insn->opcode.bytes[0]) {
 332        case 0x6c:
 333        case 0x6e:
 334        case 0xe4:
 335        case 0xe6:
 336        case 0xec:
 337        case 0xee:
 338                /* Single byte opcodes */
 339                *exitinfo |= IOIO_DATA_8;
 340                break;
 341        default:
 342                /* Length determined by instruction parsing */
 343                *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
 344                                                     : IOIO_DATA_32;
 345        }
 346        switch (insn->addr_bytes) {
 347        case 2:
 348                *exitinfo |= IOIO_ADDR_16;
 349                break;
 350        case 4:
 351                *exitinfo |= IOIO_ADDR_32;
 352                break;
 353        case 8:
 354                *exitinfo |= IOIO_ADDR_64;
 355                break;
 356        }
 357
 358        if (insn_has_rep_prefix(insn))
 359                *exitinfo |= IOIO_REP;
 360
 361        return ES_OK;
 362}
 363
 364static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 365{
 366        struct pt_regs *regs = ctxt->regs;
 367        u64 exit_info_1, exit_info_2;
 368        enum es_result ret;
 369
 370        ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
 371        if (ret != ES_OK)
 372                return ret;
 373
 374        if (exit_info_1 & IOIO_TYPE_STR) {
 375
 376                /* (REP) INS/OUTS */
 377
 378                bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
 379                unsigned int io_bytes, exit_bytes;
 380                unsigned int ghcb_count, op_count;
 381                unsigned long es_base;
 382                u64 sw_scratch;
 383
 384                /*
 385                 * For the string variants with rep prefix the amount of in/out
 386                 * operations per #VC exception is limited so that the kernel
 387                 * has a chance to take interrupts and re-schedule while the
 388                 * instruction is emulated.
 389                 */
 390                io_bytes   = (exit_info_1 >> 4) & 0x7;
 391                ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
 392
 393                op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
 394                exit_info_2 = min(op_count, ghcb_count);
 395                exit_bytes  = exit_info_2 * io_bytes;
 396
 397                es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
 398
 399                /* Read bytes of OUTS into the shared buffer */
 400                if (!(exit_info_1 & IOIO_TYPE_IN)) {
 401                        ret = vc_insn_string_read(ctxt,
 402                                               (void *)(es_base + regs->si),
 403                                               ghcb->shared_buffer, io_bytes,
 404                                               exit_info_2, df);
 405                        if (ret)
 406                                return ret;
 407                }
 408
 409                /*
 410                 * Issue an VMGEXIT to the HV to consume the bytes from the
 411                 * shared buffer or to have it write them into the shared buffer
 412                 * depending on the instruction: OUTS or INS.
 413                 */
 414                sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
 415                ghcb_set_sw_scratch(ghcb, sw_scratch);
 416                ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
 417                                          exit_info_1, exit_info_2);
 418                if (ret != ES_OK)
 419                        return ret;
 420
 421                /* Read bytes from shared buffer into the guest's destination. */
 422                if (exit_info_1 & IOIO_TYPE_IN) {
 423                        ret = vc_insn_string_write(ctxt,
 424                                                   (void *)(es_base + regs->di),
 425                                                   ghcb->shared_buffer, io_bytes,
 426                                                   exit_info_2, df);
 427                        if (ret)
 428                                return ret;
 429
 430                        if (df)
 431                                regs->di -= exit_bytes;
 432                        else
 433                                regs->di += exit_bytes;
 434                } else {
 435                        if (df)
 436                                regs->si -= exit_bytes;
 437                        else
 438                                regs->si += exit_bytes;
 439                }
 440
 441                if (exit_info_1 & IOIO_REP)
 442                        regs->cx -= exit_info_2;
 443
 444                ret = regs->cx ? ES_RETRY : ES_OK;
 445
 446        } else {
 447
 448                /* IN/OUT into/from rAX */
 449
 450                int bits = (exit_info_1 & 0x70) >> 1;
 451                u64 rax = 0;
 452
 453                if (!(exit_info_1 & IOIO_TYPE_IN))
 454                        rax = lower_bits(regs->ax, bits);
 455
 456                ghcb_set_rax(ghcb, rax);
 457
 458                ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
 459                if (ret != ES_OK)
 460                        return ret;
 461
 462                if (exit_info_1 & IOIO_TYPE_IN) {
 463                        if (!ghcb_rax_is_valid(ghcb))
 464                                return ES_VMM_ERROR;
 465                        regs->ax = lower_bits(ghcb->save.rax, bits);
 466                }
 467        }
 468
 469        return ret;
 470}
 471
 472static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
 473                                      struct es_em_ctxt *ctxt)
 474{
 475        struct pt_regs *regs = ctxt->regs;
 476        u32 cr4 = native_read_cr4();
 477        enum es_result ret;
 478
 479        ghcb_set_rax(ghcb, regs->ax);
 480        ghcb_set_rcx(ghcb, regs->cx);
 481
 482        if (cr4 & X86_CR4_OSXSAVE)
 483                /* Safe to read xcr0 */
 484                ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
 485        else
 486                /* xgetbv will cause #GP - use reset value for xcr0 */
 487                ghcb_set_xcr0(ghcb, 1);
 488
 489        ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
 490        if (ret != ES_OK)
 491                return ret;
 492
 493        if (!(ghcb_rax_is_valid(ghcb) &&
 494              ghcb_rbx_is_valid(ghcb) &&
 495              ghcb_rcx_is_valid(ghcb) &&
 496              ghcb_rdx_is_valid(ghcb)))
 497                return ES_VMM_ERROR;
 498
 499        regs->ax = ghcb->save.rax;
 500        regs->bx = ghcb->save.rbx;
 501        regs->cx = ghcb->save.rcx;
 502        regs->dx = ghcb->save.rdx;
 503
 504        return ES_OK;
 505}
 506
 507static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
 508                                      struct es_em_ctxt *ctxt,
 509                                      unsigned long exit_code)
 510{
 511        bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
 512        enum es_result ret;
 513
 514        ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
 515        if (ret != ES_OK)
 516                return ret;
 517
 518        if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
 519             (!rdtscp || ghcb_rcx_is_valid(ghcb))))
 520                return ES_VMM_ERROR;
 521
 522        ctxt->regs->ax = ghcb->save.rax;
 523        ctxt->regs->dx = ghcb->save.rdx;
 524        if (rdtscp)
 525                ctxt->regs->cx = ghcb->save.rcx;
 526
 527        return ES_OK;
 528}
 529