1 // SPDX-License-Identifier: GPL-2.0 2 3 #ifndef __BOOT_COMPRESSED 4 #define has_cpuflag(f) cpu_feature_enabled(f) 5 #endif 6 7 static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt, 8 unsigned long exit_code) 9 { 10 unsigned int opcode = (unsigned int)ctxt->insn.opcode.value; 11 u8 modrm = ctxt->insn.modrm.value; 12 13 switch (exit_code) { 14 15 case SVM_EXIT_IOIO: 16 case SVM_EXIT_NPF: 17 /* handled separately */ 18 return ES_OK; 19 20 case SVM_EXIT_CPUID: 21 if (opcode == 0xa20f) 22 return ES_OK; 23 break; 24 25 case SVM_EXIT_INVD: 26 if (opcode == 0x080f) 27 return ES_OK; 28 break; 29 30 case SVM_EXIT_MONITOR: 31 /* MONITOR and MONITORX instructions generate the same error code */ 32 if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa)) 33 return ES_OK; 34 break; 35 36 case SVM_EXIT_MWAIT: 37 /* MWAIT and MWAITX instructions generate the same error code */ 38 if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb)) 39 return ES_OK; 40 break; 41 42 case SVM_EXIT_MSR: 43 /* RDMSR */ 44 if (opcode == 0x320f || 45 /* WRMSR */ 46 opcode == 0x300f) 47 return ES_OK; 48 break; 49 50 case SVM_EXIT_RDPMC: 51 if (opcode == 0x330f) 52 return ES_OK; 53 break; 54 55 case SVM_EXIT_RDTSC: 56 if (opcode == 0x310f) 57 return ES_OK; 58 break; 59 60 case SVM_EXIT_RDTSCP: 61 if (opcode == 0x010f && modrm == 0xf9) 62 return ES_OK; 63 break; 64 65 case SVM_EXIT_READ_DR7: 66 if (opcode == 0x210f && 67 X86_MODRM_REG(ctxt->insn.modrm.value) == 7) 68 return ES_OK; 69 break; 70 71 case SVM_EXIT_VMMCALL: 72 if (opcode == 0x010f && modrm == 0xd9) 73 return ES_OK; 74 75 break; 76 77 case SVM_EXIT_WRITE_DR7: 78 if (opcode == 0x230f && 79 X86_MODRM_REG(ctxt->insn.modrm.value) == 7) 80 return ES_OK; 81 break; 82 83 case SVM_EXIT_WBINVD: 84 if (opcode == 0x90f) 85 return ES_OK; 86 break; 87 88 default: 89 break; 90 } 91 92 sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n", 93 opcode, exit_code, ctxt->regs->ip); 94 95 return ES_UNSUPPORTED; 96 } 97 98 static bool vc_decoding_needed(unsigned long exit_code) 99 { 100 /* Exceptions don't require to decode the instruction */ 101 return !(exit_code >= SVM_EXIT_EXCP_BASE && 102 exit_code <= SVM_EXIT_LAST_EXCP); 103 } 104 105 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 106 struct pt_regs *regs, 107 unsigned long exit_code) 108 { 109 enum es_result ret = ES_OK; 110 111 memset(ctxt, 0, sizeof(*ctxt)); 112 ctxt->regs = regs; 113 114 if (vc_decoding_needed(exit_code)) 115 ret = vc_decode_insn(ctxt); 116 117 return ret; 118 } 119 120 static void vc_finish_insn(struct es_em_ctxt *ctxt) 121 { 122 ctxt->regs->ip += ctxt->insn.length; 123 } 124 125 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt, 126 unsigned long address, 127 bool write) 128 { 129 if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) { 130 ctxt->fi.vector = X86_TRAP_PF; 131 ctxt->fi.error_code = X86_PF_USER; 132 ctxt->fi.cr2 = address; 133 if (write) 134 ctxt->fi.error_code |= X86_PF_WRITE; 135 136 return ES_EXCEPTION; 137 } 138 139 return ES_OK; 140 } 141 142 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 143 void *src, char *buf, 144 unsigned int data_size, 145 unsigned int count, 146 bool backwards) 147 { 148 int i, b = backwards ? -1 : 1; 149 unsigned long address = (unsigned long)src; 150 enum es_result ret; 151 152 ret = vc_insn_string_check(ctxt, address, false); 153 if (ret != ES_OK) 154 return ret; 155 156 for (i = 0; i < count; i++) { 157 void *s = src + (i * data_size * b); 158 char *d = buf + (i * data_size); 159 160 ret = vc_read_mem(ctxt, s, d, data_size); 161 if (ret != ES_OK) 162 break; 163 } 164 165 return ret; 166 } 167 168 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 169 void *dst, char *buf, 170 unsigned int data_size, 171 unsigned int count, 172 bool backwards) 173 { 174 int i, s = backwards ? -1 : 1; 175 unsigned long address = (unsigned long)dst; 176 enum es_result ret; 177 178 ret = vc_insn_string_check(ctxt, address, true); 179 if (ret != ES_OK) 180 return ret; 181 182 for (i = 0; i < count; i++) { 183 void *d = dst + (i * data_size * s); 184 char *b = buf + (i * data_size); 185 186 ret = vc_write_mem(ctxt, d, b, data_size); 187 if (ret != ES_OK) 188 break; 189 } 190 191 return ret; 192 } 193 194 #define IOIO_TYPE_STR BIT(2) 195 #define IOIO_TYPE_IN 1 196 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 197 #define IOIO_TYPE_OUT 0 198 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 199 200 #define IOIO_REP BIT(3) 201 202 #define IOIO_ADDR_64 BIT(9) 203 #define IOIO_ADDR_32 BIT(8) 204 #define IOIO_ADDR_16 BIT(7) 205 206 #define IOIO_DATA_32 BIT(6) 207 #define IOIO_DATA_16 BIT(5) 208 #define IOIO_DATA_8 BIT(4) 209 210 #define IOIO_SEG_ES (0 << 10) 211 #define IOIO_SEG_DS (3 << 10) 212 213 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 214 { 215 struct insn *insn = &ctxt->insn; 216 size_t size; 217 u64 port; 218 219 *exitinfo = 0; 220 221 switch (insn->opcode.bytes[0]) { 222 /* INS opcodes */ 223 case 0x6c: 224 case 0x6d: 225 *exitinfo |= IOIO_TYPE_INS; 226 *exitinfo |= IOIO_SEG_ES; 227 port = ctxt->regs->dx & 0xffff; 228 break; 229 230 /* OUTS opcodes */ 231 case 0x6e: 232 case 0x6f: 233 *exitinfo |= IOIO_TYPE_OUTS; 234 *exitinfo |= IOIO_SEG_DS; 235 port = ctxt->regs->dx & 0xffff; 236 break; 237 238 /* IN immediate opcodes */ 239 case 0xe4: 240 case 0xe5: 241 *exitinfo |= IOIO_TYPE_IN; 242 port = (u8)insn->immediate.value & 0xffff; 243 break; 244 245 /* OUT immediate opcodes */ 246 case 0xe6: 247 case 0xe7: 248 *exitinfo |= IOIO_TYPE_OUT; 249 port = (u8)insn->immediate.value & 0xffff; 250 break; 251 252 /* IN register opcodes */ 253 case 0xec: 254 case 0xed: 255 *exitinfo |= IOIO_TYPE_IN; 256 port = ctxt->regs->dx & 0xffff; 257 break; 258 259 /* OUT register opcodes */ 260 case 0xee: 261 case 0xef: 262 *exitinfo |= IOIO_TYPE_OUT; 263 port = ctxt->regs->dx & 0xffff; 264 break; 265 266 default: 267 return ES_DECODE_FAILED; 268 } 269 270 *exitinfo |= port << 16; 271 272 switch (insn->opcode.bytes[0]) { 273 case 0x6c: 274 case 0x6e: 275 case 0xe4: 276 case 0xe6: 277 case 0xec: 278 case 0xee: 279 /* Single byte opcodes */ 280 *exitinfo |= IOIO_DATA_8; 281 size = 1; 282 break; 283 default: 284 /* Length determined by instruction parsing */ 285 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 286 : IOIO_DATA_32; 287 size = (insn->opnd_bytes == 2) ? 2 : 4; 288 } 289 290 switch (insn->addr_bytes) { 291 case 2: 292 *exitinfo |= IOIO_ADDR_16; 293 break; 294 case 4: 295 *exitinfo |= IOIO_ADDR_32; 296 break; 297 case 8: 298 *exitinfo |= IOIO_ADDR_64; 299 break; 300 } 301 302 if (insn_has_rep_prefix(insn)) 303 *exitinfo |= IOIO_REP; 304 305 return vc_ioio_check(ctxt, (u16)port, size); 306 } 307 308 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 309 { 310 struct pt_regs *regs = ctxt->regs; 311 u64 exit_info_1, exit_info_2; 312 enum es_result ret; 313 314 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 315 if (ret != ES_OK) 316 return ret; 317 318 if (exit_info_1 & IOIO_TYPE_STR) { 319 320 /* (REP) INS/OUTS */ 321 322 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 323 unsigned int io_bytes, exit_bytes; 324 unsigned int ghcb_count, op_count; 325 unsigned long es_base; 326 u64 sw_scratch; 327 328 /* 329 * For the string variants with rep prefix the amount of in/out 330 * operations per #VC exception is limited so that the kernel 331 * has a chance to take interrupts and re-schedule while the 332 * instruction is emulated. 333 */ 334 io_bytes = (exit_info_1 >> 4) & 0x7; 335 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 336 337 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 338 exit_info_2 = min(op_count, ghcb_count); 339 exit_bytes = exit_info_2 * io_bytes; 340 341 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 342 343 /* Read bytes of OUTS into the shared buffer */ 344 if (!(exit_info_1 & IOIO_TYPE_IN)) { 345 ret = vc_insn_string_read(ctxt, 346 (void *)(es_base + regs->si), 347 ghcb->shared_buffer, io_bytes, 348 exit_info_2, df); 349 if (ret) 350 return ret; 351 } 352 353 /* 354 * Issue an VMGEXIT to the HV to consume the bytes from the 355 * shared buffer or to have it write them into the shared buffer 356 * depending on the instruction: OUTS or INS. 357 */ 358 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 359 ghcb_set_sw_scratch(ghcb, sw_scratch); 360 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 361 exit_info_1, exit_info_2); 362 if (ret != ES_OK) 363 return ret; 364 365 /* Read bytes from shared buffer into the guest's destination. */ 366 if (exit_info_1 & IOIO_TYPE_IN) { 367 ret = vc_insn_string_write(ctxt, 368 (void *)(es_base + regs->di), 369 ghcb->shared_buffer, io_bytes, 370 exit_info_2, df); 371 if (ret) 372 return ret; 373 374 if (df) 375 regs->di -= exit_bytes; 376 else 377 regs->di += exit_bytes; 378 } else { 379 if (df) 380 regs->si -= exit_bytes; 381 else 382 regs->si += exit_bytes; 383 } 384 385 if (exit_info_1 & IOIO_REP) 386 regs->cx -= exit_info_2; 387 388 ret = regs->cx ? ES_RETRY : ES_OK; 389 390 } else { 391 392 /* IN/OUT into/from rAX */ 393 394 int bits = (exit_info_1 & 0x70) >> 1; 395 u64 rax = 0; 396 397 if (!(exit_info_1 & IOIO_TYPE_IN)) 398 rax = lower_bits(regs->ax, bits); 399 400 ghcb_set_rax(ghcb, rax); 401 402 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 403 if (ret != ES_OK) 404 return ret; 405 406 if (exit_info_1 & IOIO_TYPE_IN) { 407 if (!ghcb_rax_is_valid(ghcb)) 408 return ES_VMM_ERROR; 409 regs->ax = lower_bits(ghcb->save.rax, bits); 410 } 411 } 412 413 return ret; 414 } 415 416 enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 417 { 418 u32 ret; 419 420 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 421 if (!ret) 422 return ES_OK; 423 424 if (ret == 1) { 425 u64 info = ghcb->save.sw_exit_info_2; 426 unsigned long v = info & SVM_EVTINJ_VEC_MASK; 427 428 /* Check if exception information from hypervisor is sane. */ 429 if ((info & SVM_EVTINJ_VALID) && 430 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 431 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 432 ctxt->fi.vector = v; 433 434 if (info & SVM_EVTINJ_VALID_ERR) 435 ctxt->fi.error_code = info >> 32; 436 437 return ES_EXCEPTION; 438 } 439 } 440 441 return ES_VMM_ERROR; 442 } 443 444 enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 445 struct es_em_ctxt *ctxt, 446 u64 exit_code, u64 exit_info_1, 447 u64 exit_info_2) 448 { 449 /* Fill in protocol and format specifiers */ 450 ghcb->protocol_version = ghcb_version; 451 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 452 453 ghcb_set_sw_exit_code(ghcb, exit_code); 454 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 455 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 456 457 sev_es_wr_ghcb_msr(__pa(ghcb)); 458 VMGEXIT(); 459 460 return verify_exception_info(ghcb, ctxt); 461 } 462 463 static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 464 { 465 u32 cr4 = native_read_cr4(); 466 int ret; 467 468 ghcb_set_rax(ghcb, leaf->fn); 469 ghcb_set_rcx(ghcb, leaf->subfn); 470 471 if (cr4 & X86_CR4_OSXSAVE) 472 /* Safe to read xcr0 */ 473 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 474 else 475 /* xgetbv will cause #UD - use reset value for xcr0 */ 476 ghcb_set_xcr0(ghcb, 1); 477 478 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 479 if (ret != ES_OK) 480 return ret; 481 482 if (!(ghcb_rax_is_valid(ghcb) && 483 ghcb_rbx_is_valid(ghcb) && 484 ghcb_rcx_is_valid(ghcb) && 485 ghcb_rdx_is_valid(ghcb))) 486 return ES_VMM_ERROR; 487 488 leaf->eax = ghcb->save.rax; 489 leaf->ebx = ghcb->save.rbx; 490 leaf->ecx = ghcb->save.rcx; 491 leaf->edx = ghcb->save.rdx; 492 493 return ES_OK; 494 } 495 496 struct cpuid_ctx { 497 struct ghcb *ghcb; 498 struct es_em_ctxt *ctxt; 499 }; 500 501 static void snp_cpuid_hv_ghcb(void *p, struct cpuid_leaf *leaf) 502 { 503 struct cpuid_ctx *ctx = p; 504 505 if (__sev_cpuid_hv_ghcb(ctx->ghcb, ctx->ctxt, leaf)) 506 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV); 507 } 508 509 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 510 { 511 struct cpuid_ctx ctx = { ghcb, ctxt }; 512 struct pt_regs *regs = ctxt->regs; 513 struct cpuid_leaf leaf; 514 int ret; 515 516 leaf.fn = regs->ax; 517 leaf.subfn = regs->cx; 518 ret = snp_cpuid(snp_cpuid_hv_ghcb, &ctx, &leaf); 519 if (!ret) { 520 regs->ax = leaf.eax; 521 regs->bx = leaf.ebx; 522 regs->cx = leaf.ecx; 523 regs->dx = leaf.edx; 524 } 525 526 return ret; 527 } 528 529 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 530 struct es_em_ctxt *ctxt) 531 { 532 struct pt_regs *regs = ctxt->regs; 533 u32 cr4 = native_read_cr4(); 534 enum es_result ret; 535 int snp_cpuid_ret; 536 537 snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt); 538 if (!snp_cpuid_ret) 539 return ES_OK; 540 if (snp_cpuid_ret != -EOPNOTSUPP) 541 return ES_VMM_ERROR; 542 543 ghcb_set_rax(ghcb, regs->ax); 544 ghcb_set_rcx(ghcb, regs->cx); 545 546 if (cr4 & X86_CR4_OSXSAVE) 547 /* Safe to read xcr0 */ 548 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 549 else 550 /* xgetbv will cause #GP - use reset value for xcr0 */ 551 ghcb_set_xcr0(ghcb, 1); 552 553 if (has_cpuflag(X86_FEATURE_SHSTK) && regs->ax == 0xd && regs->cx == 1) { 554 struct msr m; 555 556 raw_rdmsr(MSR_IA32_XSS, &m); 557 ghcb_set_xss(ghcb, m.q); 558 } 559 560 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 561 if (ret != ES_OK) 562 return ret; 563 564 if (!(ghcb_rax_is_valid(ghcb) && 565 ghcb_rbx_is_valid(ghcb) && 566 ghcb_rcx_is_valid(ghcb) && 567 ghcb_rdx_is_valid(ghcb))) 568 return ES_VMM_ERROR; 569 570 regs->ax = ghcb->save.rax; 571 regs->bx = ghcb->save.rbx; 572 regs->cx = ghcb->save.rcx; 573 regs->dx = ghcb->save.rdx; 574 575 return ES_OK; 576 } 577 578 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 579 struct es_em_ctxt *ctxt, 580 unsigned long exit_code) 581 { 582 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 583 enum es_result ret; 584 585 /* 586 * The hypervisor should not be intercepting RDTSC/RDTSCP when Secure 587 * TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP 588 * instructions are being intercepted. If this should occur and Secure 589 * TSC is enabled, guest execution should be terminated as the guest 590 * cannot rely on the TSC value provided by the hypervisor. 591 */ 592 if (sev_status & MSR_AMD64_SNP_SECURE_TSC) 593 return ES_VMM_ERROR; 594 595 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 596 if (ret != ES_OK) 597 return ret; 598 599 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 600 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 601 return ES_VMM_ERROR; 602 603 ctxt->regs->ax = ghcb->save.rax; 604 ctxt->regs->dx = ghcb->save.rdx; 605 if (rdtscp) 606 ctxt->regs->cx = ghcb->save.rcx; 607 608 return ES_OK; 609 } 610 611 void snp_register_ghcb_early(unsigned long paddr) 612 { 613 unsigned long pfn = paddr >> PAGE_SHIFT; 614 u64 val; 615 616 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 617 VMGEXIT(); 618 619 val = sev_es_rd_ghcb_msr(); 620 621 /* If the response GPA is not ours then abort the guest */ 622 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 623 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 624 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 625 } 626 627 bool __init sev_es_check_cpu_features(void) 628 { 629 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 630 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 631 return false; 632 } 633 634 return true; 635 } 636 637 bool sev_es_negotiate_protocol(void) 638 { 639 u64 val; 640 641 /* Do the GHCB protocol version negotiation */ 642 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 643 VMGEXIT(); 644 val = sev_es_rd_ghcb_msr(); 645 646 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 647 return false; 648 649 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 650 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 651 return false; 652 653 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 654 655 return true; 656 } 657