1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021-2022 Intel Corporation */ 3 4 #undef pr_fmt 5 #define pr_fmt(fmt) "tdx: " fmt 6 7 #include <linux/cpufeature.h> 8 #include <linux/export.h> 9 #include <linux/io.h> 10 #include <linux/kexec.h> 11 #include <asm/coco.h> 12 #include <asm/tdx.h> 13 #include <asm/vmx.h> 14 #include <asm/ia32.h> 15 #include <asm/insn.h> 16 #include <asm/insn-eval.h> 17 #include <asm/cpuid/api.h> 18 #include <asm/paravirt_types.h> 19 #include <asm/pgtable.h> 20 #include <asm/set_memory.h> 21 #include <asm/traps.h> 22 23 /* MMIO direction */ 24 #define EPT_READ 0 25 #define EPT_WRITE 1 26 27 /* Port I/O direction */ 28 #define PORT_READ 0 29 #define PORT_WRITE 1 30 31 /* See Exit Qualification for I/O Instructions in VMX documentation */ 32 #define VE_IS_IO_IN(e) ((e) & BIT(3)) 33 #define VE_GET_IO_SIZE(e) (((e) & GENMASK(2, 0)) + 1) 34 #define VE_GET_PORT_NUM(e) ((e) >> 16) 35 #define VE_IS_IO_STRING(e) ((e) & BIT(4)) 36 37 /* TDX Module call error codes */ 38 #define TDCALL_RETURN_CODE(a) ((a) >> 32) 39 #define TDCALL_INVALID_OPERAND 0xc0000100 40 #define TDCALL_OPERAND_BUSY 0x80000200 41 42 #define TDREPORT_SUBTYPE_0 0 43 44 static atomic_long_t nr_shared; 45 46 /* Called from __tdx_hypercall() for unrecoverable failure */ 47 noinstr void __noreturn __tdx_hypercall_failed(void) 48 { 49 instrumentation_begin(); 50 panic("TDVMCALL failed. TDX module bug?"); 51 } 52 53 #ifdef CONFIG_KVM_GUEST 54 long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2, 55 unsigned long p3, unsigned long p4) 56 { 57 struct tdx_module_args args = { 58 .r10 = nr, 59 .r11 = p1, 60 .r12 = p2, 61 .r13 = p3, 62 .r14 = p4, 63 }; 64 65 return __tdx_hypercall(&args); 66 } 67 EXPORT_SYMBOL_GPL(tdx_kvm_hypercall); 68 #endif 69 70 /* 71 * Used for TDX guests to make calls directly to the TD module. This 72 * should only be used for calls that have no legitimate reason to fail 73 * or where the kernel can not survive the call failing. 74 */ 75 static inline void tdcall(u64 fn, struct tdx_module_args *args) 76 { 77 if (__tdcall_ret(fn, args)) 78 panic("TDCALL %lld failed (Buggy TDX module!)\n", fn); 79 } 80 81 /* Read TD-scoped metadata */ 82 static inline u64 tdg_vm_rd(u64 field, u64 *value) 83 { 84 struct tdx_module_args args = { 85 .rdx = field, 86 }; 87 u64 ret; 88 89 ret = __tdcall_ret(TDG_VM_RD, &args); 90 *value = args.r8; 91 92 return ret; 93 } 94 95 /* Write TD-scoped metadata */ 96 static inline u64 tdg_vm_wr(u64 field, u64 value, u64 mask) 97 { 98 struct tdx_module_args args = { 99 .rdx = field, 100 .r8 = value, 101 .r9 = mask, 102 }; 103 104 return __tdcall(TDG_VM_WR, &args); 105 } 106 107 /** 108 * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT 109 * subtype 0) using TDG.MR.REPORT TDCALL. 110 * @reportdata: Address of the input buffer which contains user-defined 111 * REPORTDATA to be included into TDREPORT. 112 * @tdreport: Address of the output buffer to store TDREPORT. 113 * 114 * Refer to section titled "TDG.MR.REPORT leaf" in the TDX Module v1.0 115 * specification for more information on TDG.MR.REPORT TDCALL. 116 * 117 * It is used in the TDX guest driver module to get the TDREPORT0. 118 * 119 * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation, 120 * or -EIO on other TDCALL failures. 121 */ 122 int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport) 123 { 124 struct tdx_module_args args = { 125 .rcx = virt_to_phys(tdreport), 126 .rdx = virt_to_phys(reportdata), 127 .r8 = TDREPORT_SUBTYPE_0, 128 }; 129 u64 ret; 130 131 ret = __tdcall(TDG_MR_REPORT, &args); 132 if (ret) { 133 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND) 134 return -ENXIO; 135 else if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY) 136 return -EBUSY; 137 return -EIO; 138 } 139 140 return 0; 141 } 142 EXPORT_SYMBOL_GPL(tdx_mcall_get_report0); 143 144 /** 145 * tdx_mcall_extend_rtmr() - Wrapper to extend RTMR registers using 146 * TDG.MR.RTMR.EXTEND TDCALL. 147 * @index: Index of RTMR register to be extended. 148 * @data: Address of the input buffer with RTMR register extend data. 149 * 150 * Refer to section titled "TDG.MR.RTMR.EXTEND leaf" in the TDX Module v1.0 151 * specification for more information on TDG.MR.RTMR.EXTEND TDCALL. 152 * 153 * It is used in the TDX guest driver module to allow user to extend the RTMR 154 * registers. 155 * 156 * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation, 157 * or -EIO on other TDCALL failures. 158 */ 159 int tdx_mcall_extend_rtmr(u8 index, u8 *data) 160 { 161 struct tdx_module_args args = { 162 .rcx = virt_to_phys(data), 163 .rdx = index, 164 }; 165 u64 ret; 166 167 ret = __tdcall(TDG_MR_RTMR_EXTEND, &args); 168 if (ret) { 169 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND) 170 return -ENXIO; 171 if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY) 172 return -EBUSY; 173 return -EIO; 174 } 175 176 return 0; 177 } 178 EXPORT_SYMBOL_GPL(tdx_mcall_extend_rtmr); 179 180 /** 181 * tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote 182 * hypercall. 183 * @buf: Address of the directly mapped shared kernel buffer which 184 * contains TDREPORT. The same buffer will be used by VMM to 185 * store the generated TD Quote output. 186 * @size: size of the tdquote buffer (4KB-aligned). 187 * 188 * Refer to section titled "TDG.VP.VMCALL<GetQuote>" in the TDX GHCI 189 * v1.0 specification for more information on GetQuote hypercall. 190 * It is used in the TDX guest driver module to get the TD Quote. 191 * 192 * Return 0 on success or error code on failure. 193 */ 194 u64 tdx_hcall_get_quote(u8 *buf, size_t size) 195 { 196 /* Since buf is a shared memory, set the shared (decrypted) bits */ 197 return _tdx_hypercall(TDVMCALL_GET_QUOTE, cc_mkdec(virt_to_phys(buf)), size, 0, 0); 198 } 199 EXPORT_SYMBOL_GPL(tdx_hcall_get_quote); 200 201 static void __noreturn tdx_panic(const char *msg) 202 { 203 struct tdx_module_args args = { 204 .r10 = TDX_HYPERCALL_STANDARD, 205 .r11 = TDVMCALL_REPORT_FATAL_ERROR, 206 .r12 = 0, /* Error code: 0 is Panic */ 207 }; 208 union { 209 /* Define register order according to the GHCI */ 210 struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; }; 211 212 char bytes[64] __nonstring; 213 } message; 214 215 /* VMM assumes '\0' in byte 65, if the message took all 64 bytes */ 216 strtomem_pad(message.bytes, msg, '\0'); 217 218 args.r8 = message.r8; 219 args.r9 = message.r9; 220 args.r14 = message.r14; 221 args.r15 = message.r15; 222 args.rdi = message.rdi; 223 args.rsi = message.rsi; 224 args.rbx = message.rbx; 225 args.rdx = message.rdx; 226 227 /* 228 * This hypercall should never return and it is not safe 229 * to keep the guest running. Call it forever if it 230 * happens to return. 231 */ 232 while (1) 233 __tdx_hypercall(&args); 234 } 235 236 /* 237 * The kernel cannot handle #VEs when accessing normal kernel memory. Ensure 238 * that no #VE will be delivered for accesses to TD-private memory. 239 * 240 * TDX 1.0 does not allow the guest to disable SEPT #VE on its own. The VMM 241 * controls if the guest will receive such #VE with TD attribute 242 * TDX_TD_ATTR_SEPT_VE_DISABLE. 243 * 244 * Newer TDX modules allow the guest to control if it wants to receive SEPT 245 * violation #VEs. 246 * 247 * Check if the feature is available and disable SEPT #VE if possible. 248 * 249 * If the TD is allowed to disable/enable SEPT #VEs, the TDX_TD_ATTR_SEPT_VE_DISABLE 250 * attribute is no longer reliable. It reflects the initial state of the 251 * control for the TD, but it will not be updated if someone (e.g. bootloader) 252 * changes it before the kernel starts. Kernel must check TDCS_TD_CTLS bit to 253 * determine if SEPT #VEs are enabled or disabled. 254 */ 255 static void disable_sept_ve(u64 td_attr) 256 { 257 const char *msg = "TD misconfiguration: SEPT #VE has to be disabled"; 258 bool debug = td_attr & TDX_TD_ATTR_DEBUG; 259 u64 config, controls; 260 261 /* Is this TD allowed to disable SEPT #VE */ 262 tdg_vm_rd(TDCS_CONFIG_FLAGS, &config); 263 if (!(config & TDCS_CONFIG_FLEXIBLE_PENDING_VE)) { 264 /* No SEPT #VE controls for the guest: check the attribute */ 265 if (td_attr & TDX_TD_ATTR_SEPT_VE_DISABLE) 266 return; 267 268 /* Relax SEPT_VE_DISABLE check for debug TD for backtraces */ 269 if (debug) 270 pr_warn("%s\n", msg); 271 else 272 tdx_panic(msg); 273 return; 274 } 275 276 /* Check if SEPT #VE has been disabled before us */ 277 tdg_vm_rd(TDCS_TD_CTLS, &controls); 278 if (controls & TD_CTLS_PENDING_VE_DISABLE) 279 return; 280 281 /* Keep #VEs enabled for splats in debugging environments */ 282 if (debug) 283 return; 284 285 /* Disable SEPT #VEs */ 286 tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_PENDING_VE_DISABLE, 287 TD_CTLS_PENDING_VE_DISABLE); 288 } 289 290 /* 291 * TDX 1.0 generates a #VE when accessing topology-related CPUID leafs (0xB and 292 * 0x1F) and the X2APIC_APICID MSR. The kernel returns all zeros on CPUID #VEs. 293 * In practice, this means that the kernel can only boot with a plain topology. 294 * Any complications will cause problems. 295 * 296 * The ENUM_TOPOLOGY feature allows the VMM to provide topology information. 297 * Enabling the feature eliminates topology-related #VEs: the TDX module 298 * virtualizes accesses to the CPUID leafs and the MSR. 299 * 300 * Enable ENUM_TOPOLOGY if it is available. 301 */ 302 static void enable_cpu_topology_enumeration(void) 303 { 304 u64 configured; 305 306 /* Has the VMM provided a valid topology configuration? */ 307 tdg_vm_rd(TDCS_TOPOLOGY_ENUM_CONFIGURED, &configured); 308 if (!configured) { 309 pr_err("VMM did not configure X2APIC_IDs properly\n"); 310 return; 311 } 312 313 tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_ENUM_TOPOLOGY, TD_CTLS_ENUM_TOPOLOGY); 314 } 315 316 static void reduce_unnecessary_ve(void) 317 { 318 u64 err = tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_REDUCE_VE, TD_CTLS_REDUCE_VE); 319 320 if (err == TDX_SUCCESS) 321 return; 322 323 /* 324 * Enabling REDUCE_VE includes ENUM_TOPOLOGY. Only try to 325 * enable ENUM_TOPOLOGY if REDUCE_VE was not successful. 326 */ 327 enable_cpu_topology_enumeration(); 328 } 329 330 static void tdx_setup(u64 *cc_mask) 331 { 332 struct tdx_module_args args = {}; 333 unsigned int gpa_width; 334 u64 td_attr; 335 336 /* 337 * TDINFO TDX module call is used to get the TD execution environment 338 * information like GPA width, number of available vcpus, debug mode 339 * information, etc. More details about the ABI can be found in TDX 340 * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL 341 * [TDG.VP.INFO]. 342 */ 343 tdcall(TDG_VP_INFO, &args); 344 345 /* 346 * The highest bit of a guest physical address is the "sharing" bit. 347 * Set it for shared pages and clear it for private pages. 348 * 349 * The GPA width that comes out of this call is critical. TDX guests 350 * can not meaningfully run without it. 351 */ 352 gpa_width = args.rcx & GENMASK(5, 0); 353 *cc_mask = BIT_ULL(gpa_width - 1); 354 355 td_attr = args.rdx; 356 357 /* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */ 358 tdg_vm_wr(TDCS_NOTIFY_ENABLES, 0, -1ULL); 359 360 disable_sept_ve(td_attr); 361 362 reduce_unnecessary_ve(); 363 } 364 365 /* 366 * The TDX module spec states that #VE may be injected for a limited set of 367 * reasons: 368 * 369 * - Emulation of the architectural #VE injection on EPT violation; 370 * 371 * - As a result of guest TD execution of a disallowed instruction, 372 * a disallowed MSR access, or CPUID virtualization; 373 * 374 * - A notification to the guest TD about anomalous behavior; 375 * 376 * The last one is opt-in and is not used by the kernel. 377 * 378 * The Intel Software Developer's Manual describes cases when instruction 379 * length field can be used in section "Information for VM Exits Due to 380 * Instruction Execution". 381 * 382 * For TDX, it ultimately means GET_VEINFO provides reliable instruction length 383 * information if #VE occurred due to instruction execution, but not for EPT 384 * violations. 385 */ 386 static int ve_instr_len(struct ve_info *ve) 387 { 388 switch (ve->exit_reason) { 389 case EXIT_REASON_HLT: 390 case EXIT_REASON_MSR_READ: 391 case EXIT_REASON_MSR_WRITE: 392 case EXIT_REASON_CPUID: 393 case EXIT_REASON_IO_INSTRUCTION: 394 /* It is safe to use ve->instr_len for #VE due instructions */ 395 return ve->instr_len; 396 case EXIT_REASON_EPT_VIOLATION: 397 /* 398 * For EPT violations, ve->insn_len is not defined. For those, 399 * the kernel must decode instructions manually and should not 400 * be using this function. 401 */ 402 WARN_ONCE(1, "ve->instr_len is not defined for EPT violations"); 403 return 0; 404 default: 405 WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason); 406 return ve->instr_len; 407 } 408 } 409 410 static u64 __cpuidle __halt(const bool irq_disabled) 411 { 412 struct tdx_module_args args = { 413 .r10 = TDX_HYPERCALL_STANDARD, 414 .r11 = hcall_func(EXIT_REASON_HLT), 415 .r12 = irq_disabled, 416 }; 417 418 /* 419 * Emulate HLT operation via hypercall. More info about ABI 420 * can be found in TDX Guest-Host-Communication Interface 421 * (GHCI), section 3.8 TDG.VP.VMCALL<Instruction.HLT>. 422 * 423 * The VMM uses the "IRQ disabled" param to understand IRQ 424 * enabled status (RFLAGS.IF) of the TD guest and to determine 425 * whether or not it should schedule the halted vCPU if an 426 * IRQ becomes pending. E.g. if IRQs are disabled, the VMM 427 * can keep the vCPU in virtual HLT, even if an IRQ is 428 * pending, without hanging/breaking the guest. 429 */ 430 return __tdx_hypercall(&args); 431 } 432 433 static int handle_halt(struct ve_info *ve) 434 { 435 const bool irq_disabled = irqs_disabled(); 436 437 /* 438 * HLT with IRQs enabled is unsafe, as an IRQ that is intended to be a 439 * wake event may be consumed before requesting HLT emulation, leaving 440 * the vCPU blocking indefinitely. 441 */ 442 if (WARN_ONCE(!irq_disabled, "HLT emulation with IRQs enabled")) 443 return -EIO; 444 445 if (__halt(irq_disabled)) 446 return -EIO; 447 448 return ve_instr_len(ve); 449 } 450 451 void __cpuidle tdx_halt(void) 452 { 453 const bool irq_disabled = false; 454 455 /* 456 * Use WARN_ONCE() to report the failure. 457 */ 458 if (__halt(irq_disabled)) 459 WARN_ONCE(1, "HLT instruction emulation failed\n"); 460 } 461 462 static void __cpuidle tdx_safe_halt(void) 463 { 464 tdx_halt(); 465 /* 466 * "__cpuidle" section doesn't support instrumentation, so stick 467 * with raw_* variant that avoids tracing hooks. 468 */ 469 raw_local_irq_enable(); 470 } 471 472 static int read_msr(struct pt_regs *regs, struct ve_info *ve) 473 { 474 struct tdx_module_args args = { 475 .r10 = TDX_HYPERCALL_STANDARD, 476 .r11 = hcall_func(EXIT_REASON_MSR_READ), 477 .r12 = regs->cx, 478 }; 479 480 /* 481 * Emulate the MSR read via hypercall. More info about ABI 482 * can be found in TDX Guest-Host-Communication Interface 483 * (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>". 484 */ 485 if (__tdx_hypercall(&args)) 486 return -EIO; 487 488 regs->ax = lower_32_bits(args.r11); 489 regs->dx = upper_32_bits(args.r11); 490 return ve_instr_len(ve); 491 } 492 493 static int write_msr(struct pt_regs *regs, struct ve_info *ve) 494 { 495 struct tdx_module_args args = { 496 .r10 = TDX_HYPERCALL_STANDARD, 497 .r11 = hcall_func(EXIT_REASON_MSR_WRITE), 498 .r12 = regs->cx, 499 .r13 = (u64)regs->dx << 32 | regs->ax, 500 }; 501 502 /* 503 * Emulate the MSR write via hypercall. More info about ABI 504 * can be found in TDX Guest-Host-Communication Interface 505 * (GHCI) section titled "TDG.VP.VMCALL<Instruction.WRMSR>". 506 */ 507 if (__tdx_hypercall(&args)) 508 return -EIO; 509 510 return ve_instr_len(ve); 511 } 512 513 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve) 514 { 515 struct tdx_module_args args = { 516 .r10 = TDX_HYPERCALL_STANDARD, 517 .r11 = hcall_func(EXIT_REASON_CPUID), 518 .r12 = regs->ax, 519 .r13 = regs->cx, 520 }; 521 522 /* 523 * Only allow VMM to control range reserved for hypervisor 524 * communication. 525 * 526 * Return all-zeros for any CPUID outside the range. It matches CPU 527 * behaviour for non-supported leaf. 528 */ 529 if (regs->ax < 0x40000000 || regs->ax > 0x4FFFFFFF) { 530 regs->ax = regs->bx = regs->cx = regs->dx = 0; 531 return ve_instr_len(ve); 532 } 533 534 /* 535 * Emulate the CPUID instruction via a hypercall. More info about 536 * ABI can be found in TDX Guest-Host-Communication Interface 537 * (GHCI), section titled "VP.VMCALL<Instruction.CPUID>". 538 */ 539 if (__tdx_hypercall(&args)) 540 return -EIO; 541 542 /* 543 * As per TDX GHCI CPUID ABI, r12-r15 registers contain contents of 544 * EAX, EBX, ECX, EDX registers after the CPUID instruction execution. 545 * So copy the register contents back to pt_regs. 546 */ 547 regs->ax = args.r12; 548 regs->bx = args.r13; 549 regs->cx = args.r14; 550 regs->dx = args.r15; 551 552 return ve_instr_len(ve); 553 } 554 555 static bool mmio_read(int size, unsigned long addr, unsigned long *val) 556 { 557 struct tdx_module_args args = { 558 .r10 = TDX_HYPERCALL_STANDARD, 559 .r11 = hcall_func(EXIT_REASON_EPT_VIOLATION), 560 .r12 = size, 561 .r13 = EPT_READ, 562 .r14 = addr, 563 }; 564 565 if (__tdx_hypercall(&args)) 566 return false; 567 568 *val = args.r11; 569 return true; 570 } 571 572 static bool mmio_write(int size, unsigned long addr, unsigned long val) 573 { 574 return !_tdx_hypercall(hcall_func(EXIT_REASON_EPT_VIOLATION), size, 575 EPT_WRITE, addr, val); 576 } 577 578 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve) 579 { 580 unsigned long *reg, val, vaddr; 581 char buffer[MAX_INSN_SIZE]; 582 enum insn_mmio_type mmio; 583 struct insn insn = {}; 584 int size, extend_size; 585 u8 extend_val = 0; 586 587 /* Only in-kernel MMIO is supported */ 588 if (WARN_ON_ONCE(user_mode(regs))) 589 return -EFAULT; 590 591 if (copy_from_kernel_nofault(buffer, (void *)regs->ip, MAX_INSN_SIZE)) 592 return -EFAULT; 593 594 if (insn_decode(&insn, buffer, MAX_INSN_SIZE, INSN_MODE_64)) 595 return -EINVAL; 596 597 mmio = insn_decode_mmio(&insn, &size); 598 if (WARN_ON_ONCE(mmio == INSN_MMIO_DECODE_FAILED)) 599 return -EINVAL; 600 601 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { 602 reg = insn_get_modrm_reg_ptr(&insn, regs); 603 if (!reg) 604 return -EINVAL; 605 } 606 607 if (!fault_in_kernel_space(ve->gla)) { 608 WARN_ONCE(1, "Access to userspace address is not supported"); 609 return -EINVAL; 610 } 611 612 /* 613 * Reject EPT violation #VEs that split pages. 614 * 615 * MMIO accesses are supposed to be naturally aligned and therefore 616 * never cross page boundaries. Seeing split page accesses indicates 617 * a bug or a load_unaligned_zeropad() that stepped into an MMIO page. 618 * 619 * load_unaligned_zeropad() will recover using exception fixups. 620 */ 621 vaddr = (unsigned long)insn_get_addr_ref(&insn, regs); 622 if (vaddr / PAGE_SIZE != (vaddr + size - 1) / PAGE_SIZE) 623 return -EFAULT; 624 625 /* Handle writes first */ 626 switch (mmio) { 627 case INSN_MMIO_WRITE: 628 memcpy(&val, reg, size); 629 if (!mmio_write(size, ve->gpa, val)) 630 return -EIO; 631 return insn.length; 632 case INSN_MMIO_WRITE_IMM: 633 val = insn.immediate.value; 634 if (!mmio_write(size, ve->gpa, val)) 635 return -EIO; 636 return insn.length; 637 case INSN_MMIO_READ: 638 case INSN_MMIO_READ_ZERO_EXTEND: 639 case INSN_MMIO_READ_SIGN_EXTEND: 640 /* Reads are handled below */ 641 break; 642 case INSN_MMIO_MOVS: 643 case INSN_MMIO_DECODE_FAILED: 644 /* 645 * MMIO was accessed with an instruction that could not be 646 * decoded or handled properly. It was likely not using io.h 647 * helpers or accessed MMIO accidentally. 648 */ 649 return -EINVAL; 650 default: 651 WARN_ONCE(1, "Unknown insn_decode_mmio() decode value?"); 652 return -EINVAL; 653 } 654 655 /* Handle reads */ 656 if (!mmio_read(size, ve->gpa, &val)) 657 return -EIO; 658 659 switch (mmio) { 660 case INSN_MMIO_READ: 661 /* Zero-extend for 32-bit operation */ 662 extend_size = size == 4 ? sizeof(*reg) : 0; 663 break; 664 case INSN_MMIO_READ_ZERO_EXTEND: 665 /* Zero extend based on operand size */ 666 extend_size = insn.opnd_bytes; 667 break; 668 case INSN_MMIO_READ_SIGN_EXTEND: 669 /* Sign extend based on operand size */ 670 extend_size = insn.opnd_bytes; 671 if (size == 1 && val & BIT(7)) 672 extend_val = 0xFF; 673 else if (size > 1 && val & BIT(15)) 674 extend_val = 0xFF; 675 break; 676 default: 677 /* All other cases has to be covered with the first switch() */ 678 WARN_ON_ONCE(1); 679 return -EINVAL; 680 } 681 682 if (extend_size) 683 memset(reg, extend_val, extend_size); 684 memcpy(reg, &val, size); 685 return insn.length; 686 } 687 688 static bool handle_in(struct pt_regs *regs, int size, int port) 689 { 690 struct tdx_module_args args = { 691 .r10 = TDX_HYPERCALL_STANDARD, 692 .r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION), 693 .r12 = size, 694 .r13 = PORT_READ, 695 .r14 = port, 696 }; 697 u64 mask = GENMASK(BITS_PER_BYTE * size, 0); 698 bool success; 699 700 /* 701 * Emulate the I/O read via hypercall. More info about ABI can be found 702 * in TDX Guest-Host-Communication Interface (GHCI) section titled 703 * "TDG.VP.VMCALL<Instruction.IO>". 704 */ 705 success = !__tdx_hypercall(&args); 706 707 /* Update part of the register affected by the emulated instruction */ 708 regs->ax &= ~mask; 709 if (success) 710 regs->ax |= args.r11 & mask; 711 712 return success; 713 } 714 715 static bool handle_out(struct pt_regs *regs, int size, int port) 716 { 717 u64 mask = GENMASK(BITS_PER_BYTE * size, 0); 718 719 /* 720 * Emulate the I/O write via hypercall. More info about ABI can be found 721 * in TDX Guest-Host-Communication Interface (GHCI) section titled 722 * "TDG.VP.VMCALL<Instruction.IO>". 723 */ 724 return !_tdx_hypercall(hcall_func(EXIT_REASON_IO_INSTRUCTION), size, 725 PORT_WRITE, port, regs->ax & mask); 726 } 727 728 /* 729 * Emulate I/O using hypercall. 730 * 731 * Assumes the IO instruction was using ax, which is enforced 732 * by the standard io.h macros. 733 * 734 * Return True on success or False on failure. 735 */ 736 static int handle_io(struct pt_regs *regs, struct ve_info *ve) 737 { 738 u32 exit_qual = ve->exit_qual; 739 int size, port; 740 bool in, ret; 741 742 if (VE_IS_IO_STRING(exit_qual)) 743 return -EIO; 744 745 in = VE_IS_IO_IN(exit_qual); 746 size = VE_GET_IO_SIZE(exit_qual); 747 port = VE_GET_PORT_NUM(exit_qual); 748 749 750 if (in) 751 ret = handle_in(regs, size, port); 752 else 753 ret = handle_out(regs, size, port); 754 if (!ret) 755 return -EIO; 756 757 return ve_instr_len(ve); 758 } 759 760 /* 761 * Early #VE exception handler. Only handles a subset of port I/O. 762 * Intended only for earlyprintk. If failed, return false. 763 */ 764 __init bool tdx_early_handle_ve(struct pt_regs *regs) 765 { 766 struct ve_info ve; 767 int insn_len; 768 769 tdx_get_ve_info(&ve); 770 771 if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION) 772 return false; 773 774 insn_len = handle_io(regs, &ve); 775 if (insn_len < 0) 776 return false; 777 778 regs->ip += insn_len; 779 return true; 780 } 781 782 void tdx_get_ve_info(struct ve_info *ve) 783 { 784 struct tdx_module_args args = {}; 785 786 /* 787 * Called during #VE handling to retrieve the #VE info from the 788 * TDX module. 789 * 790 * This has to be called early in #VE handling. A "nested" #VE which 791 * occurs before this will raise a #DF and is not recoverable. 792 * 793 * The call retrieves the #VE info from the TDX module, which also 794 * clears the "#VE valid" flag. This must be done before anything else 795 * because any #VE that occurs while the valid flag is set will lead to 796 * #DF. 797 * 798 * Note, the TDX module treats virtual NMIs as inhibited if the #VE 799 * valid flag is set. It means that NMI=>#VE will not result in a #DF. 800 */ 801 tdcall(TDG_VP_VEINFO_GET, &args); 802 803 /* Transfer the output parameters */ 804 ve->exit_reason = args.rcx; 805 ve->exit_qual = args.rdx; 806 ve->gla = args.r8; 807 ve->gpa = args.r9; 808 ve->instr_len = lower_32_bits(args.r10); 809 ve->instr_info = upper_32_bits(args.r10); 810 } 811 812 /* 813 * Handle the user initiated #VE. 814 * 815 * On success, returns the number of bytes RIP should be incremented (>=0) 816 * or -errno on error. 817 */ 818 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve) 819 { 820 switch (ve->exit_reason) { 821 case EXIT_REASON_CPUID: 822 return handle_cpuid(regs, ve); 823 default: 824 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason); 825 return -EIO; 826 } 827 } 828 829 static inline bool is_private_gpa(u64 gpa) 830 { 831 return gpa == cc_mkenc(gpa); 832 } 833 834 /* 835 * Handle the kernel #VE. 836 * 837 * On success, returns the number of bytes RIP should be incremented (>=0) 838 * or -errno on error. 839 */ 840 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve) 841 { 842 switch (ve->exit_reason) { 843 case EXIT_REASON_HLT: 844 return handle_halt(ve); 845 case EXIT_REASON_MSR_READ: 846 return read_msr(regs, ve); 847 case EXIT_REASON_MSR_WRITE: 848 return write_msr(regs, ve); 849 case EXIT_REASON_CPUID: 850 return handle_cpuid(regs, ve); 851 case EXIT_REASON_EPT_VIOLATION: 852 if (is_private_gpa(ve->gpa)) 853 panic("Unexpected EPT-violation on private memory."); 854 return handle_mmio(regs, ve); 855 case EXIT_REASON_IO_INSTRUCTION: 856 return handle_io(regs, ve); 857 default: 858 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason); 859 return -EIO; 860 } 861 } 862 863 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve) 864 { 865 int insn_len; 866 867 if (user_mode(regs)) 868 insn_len = virt_exception_user(regs, ve); 869 else 870 insn_len = virt_exception_kernel(regs, ve); 871 if (insn_len < 0) 872 return false; 873 874 /* After successful #VE handling, move the IP */ 875 regs->ip += insn_len; 876 877 return true; 878 } 879 880 static bool tdx_tlb_flush_required(bool private) 881 { 882 /* 883 * TDX guest is responsible for flushing TLB on private->shared 884 * transition. VMM is responsible for flushing on shared->private. 885 * 886 * The VMM _can't_ flush private addresses as it can't generate PAs 887 * with the guest's HKID. Shared memory isn't subject to integrity 888 * checking, i.e. the VMM doesn't need to flush for its own protection. 889 * 890 * There's no need to flush when converting from shared to private, 891 * as flushing is the VMM's responsibility in this case, e.g. it must 892 * flush to avoid integrity failures in the face of a buggy or 893 * malicious guest. 894 */ 895 return !private; 896 } 897 898 static bool tdx_cache_flush_required(void) 899 { 900 /* 901 * AMD SME/SEV can avoid cache flushing if HW enforces cache coherence. 902 * TDX doesn't have such capability. 903 * 904 * Flush cache unconditionally. 905 */ 906 return true; 907 } 908 909 /* 910 * Notify the VMM about page mapping conversion. More info about ABI 911 * can be found in TDX Guest-Host-Communication Interface (GHCI), 912 * section "TDG.VP.VMCALL<MapGPA>". 913 */ 914 static bool tdx_map_gpa(phys_addr_t start, phys_addr_t end, bool enc) 915 { 916 /* Retrying the hypercall a second time should succeed; use 3 just in case */ 917 const int max_retries_per_page = 3; 918 int retry_count = 0; 919 920 if (!enc) { 921 /* Set the shared (decrypted) bits: */ 922 start |= cc_mkdec(0); 923 end |= cc_mkdec(0); 924 } 925 926 while (retry_count < max_retries_per_page) { 927 struct tdx_module_args args = { 928 .r10 = TDX_HYPERCALL_STANDARD, 929 .r11 = TDVMCALL_MAP_GPA, 930 .r12 = start, 931 .r13 = end - start }; 932 933 u64 map_fail_paddr; 934 u64 ret = __tdx_hypercall(&args); 935 936 if (ret != TDVMCALL_STATUS_RETRY) 937 return !ret; 938 /* 939 * The guest must retry the operation for the pages in the 940 * region starting at the GPA specified in R11. R11 comes 941 * from the untrusted VMM. Sanity check it. 942 */ 943 map_fail_paddr = args.r11; 944 if (map_fail_paddr < start || map_fail_paddr >= end) 945 return false; 946 947 /* "Consume" a retry without forward progress */ 948 if (map_fail_paddr == start) { 949 retry_count++; 950 continue; 951 } 952 953 start = map_fail_paddr; 954 retry_count = 0; 955 } 956 957 return false; 958 } 959 960 /* 961 * Inform the VMM of the guest's intent for this physical page: shared with 962 * the VMM or private to the guest. The VMM is expected to change its mapping 963 * of the page in response. 964 */ 965 static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc) 966 { 967 phys_addr_t start = __pa(vaddr); 968 phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE); 969 970 if (!tdx_map_gpa(start, end, enc)) 971 return false; 972 973 /* shared->private conversion requires memory to be accepted before use */ 974 if (enc) 975 return tdx_accept_memory(start, end); 976 977 return true; 978 } 979 980 static int tdx_enc_status_change_prepare(unsigned long vaddr, int numpages, 981 bool enc) 982 { 983 /* 984 * Only handle shared->private conversion here. 985 * See the comment in tdx_early_init(). 986 */ 987 if (enc && !tdx_enc_status_changed(vaddr, numpages, enc)) 988 return -EIO; 989 990 return 0; 991 } 992 993 static int tdx_enc_status_change_finish(unsigned long vaddr, int numpages, 994 bool enc) 995 { 996 /* 997 * Only handle private->shared conversion here. 998 * See the comment in tdx_early_init(). 999 */ 1000 if (!enc && !tdx_enc_status_changed(vaddr, numpages, enc)) 1001 return -EIO; 1002 1003 if (enc) 1004 atomic_long_sub(numpages, &nr_shared); 1005 else 1006 atomic_long_add(numpages, &nr_shared); 1007 1008 return 0; 1009 } 1010 1011 /* Stop new private<->shared conversions */ 1012 static void tdx_kexec_begin(void) 1013 { 1014 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1015 return; 1016 1017 /* 1018 * Crash kernel reaches here with interrupts disabled: can't wait for 1019 * conversions to finish. 1020 * 1021 * If race happened, just report and proceed. 1022 */ 1023 if (!set_memory_enc_stop_conversion()) 1024 pr_warn("Failed to stop shared<->private conversions\n"); 1025 } 1026 1027 /* Walk direct mapping and convert all shared memory back to private */ 1028 static void tdx_kexec_finish(void) 1029 { 1030 unsigned long addr, end; 1031 long found = 0, shared; 1032 1033 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1034 return; 1035 1036 lockdep_assert_irqs_disabled(); 1037 1038 addr = PAGE_OFFSET; 1039 end = PAGE_OFFSET + get_max_mapped(); 1040 1041 while (addr < end) { 1042 unsigned long size; 1043 unsigned int level; 1044 pte_t *pte; 1045 1046 pte = lookup_address(addr, &level); 1047 size = page_level_size(level); 1048 1049 if (pte && pte_decrypted(*pte)) { 1050 int pages = size / PAGE_SIZE; 1051 1052 /* 1053 * Touching memory with shared bit set triggers implicit 1054 * conversion to shared. 1055 * 1056 * Make sure nobody touches the shared range from 1057 * now on. 1058 */ 1059 set_pte(pte, __pte(0)); 1060 1061 /* 1062 * Memory encryption state persists across kexec. 1063 * If tdx_enc_status_changed() fails in the first 1064 * kernel, it leaves memory in an unknown state. 1065 * 1066 * If that memory remains shared, accessing it in the 1067 * *next* kernel through a private mapping will result 1068 * in an unrecoverable guest shutdown. 1069 * 1070 * The kdump kernel boot is not impacted as it uses 1071 * a pre-reserved memory range that is always private. 1072 * However, gathering crash information could lead to 1073 * a crash if it accesses unconverted memory through 1074 * a private mapping which is possible when accessing 1075 * that memory through /proc/vmcore, for example. 1076 * 1077 * In all cases, print error info in order to leave 1078 * enough bread crumbs for debugging. 1079 */ 1080 if (!tdx_enc_status_changed(addr, pages, true)) { 1081 pr_err("Failed to unshare range %#lx-%#lx\n", 1082 addr, addr + size); 1083 } 1084 1085 found += pages; 1086 } 1087 1088 addr += size; 1089 } 1090 1091 __flush_tlb_all(); 1092 1093 shared = atomic_long_read(&nr_shared); 1094 if (shared != found) { 1095 pr_err("shared page accounting is off\n"); 1096 pr_err("nr_shared = %ld, nr_found = %ld\n", shared, found); 1097 } 1098 } 1099 1100 static __init void tdx_announce(void) 1101 { 1102 struct tdx_module_args args = {}; 1103 u64 controls; 1104 1105 pr_info("Guest detected\n"); 1106 1107 tdcall(TDG_VP_INFO, &args); 1108 tdx_dump_attributes(args.rdx); 1109 1110 tdg_vm_rd(TDCS_TD_CTLS, &controls); 1111 tdx_dump_td_ctls(controls); 1112 } 1113 1114 void __init tdx_early_init(void) 1115 { 1116 u64 cc_mask; 1117 u32 eax, sig[3]; 1118 1119 cpuid_count(TDX_CPUID_LEAF_ID, 0, &eax, &sig[0], &sig[2], &sig[1]); 1120 1121 if (memcmp(TDX_IDENT, sig, sizeof(sig))) 1122 return; 1123 1124 setup_force_cpu_cap(X86_FEATURE_TDX_GUEST); 1125 1126 /* TSC is the only reliable clock in TDX guest */ 1127 setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); 1128 1129 cc_vendor = CC_VENDOR_INTEL; 1130 1131 /* Configure the TD */ 1132 tdx_setup(&cc_mask); 1133 1134 cc_set_mask(cc_mask); 1135 1136 /* 1137 * All bits above GPA width are reserved and kernel treats shared bit 1138 * as flag, not as part of physical address. 1139 * 1140 * Adjust physical mask to only cover valid GPA bits. 1141 */ 1142 physical_mask &= cc_mask - 1; 1143 1144 /* 1145 * The kernel mapping should match the TDX metadata for the page. 1146 * load_unaligned_zeropad() can touch memory *adjacent* to that which is 1147 * owned by the caller and can catch even _momentary_ mismatches. Bad 1148 * things happen on mismatch: 1149 * 1150 * - Private mapping => Shared Page == Guest shutdown 1151 * - Shared mapping => Private Page == Recoverable #VE 1152 * 1153 * guest.enc_status_change_prepare() converts the page from 1154 * shared=>private before the mapping becomes private. 1155 * 1156 * guest.enc_status_change_finish() converts the page from 1157 * private=>shared after the mapping becomes private. 1158 * 1159 * In both cases there is a temporary shared mapping to a private page, 1160 * which can result in a #VE. But, there is never a private mapping to 1161 * a shared page. 1162 */ 1163 x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; 1164 x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; 1165 1166 x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; 1167 x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required; 1168 1169 x86_platform.guest.enc_kexec_begin = tdx_kexec_begin; 1170 x86_platform.guest.enc_kexec_finish = tdx_kexec_finish; 1171 1172 /* 1173 * Avoid "sti;hlt" execution in TDX guests as HLT induces a #VE that 1174 * will enable interrupts before HLT TDCALL invocation if executed 1175 * in STI-shadow, possibly resulting in missed wakeup events. 1176 * 1177 * Modify all possible HLT execution paths to use TDX specific routines 1178 * that directly execute TDCALL and toggle the interrupt state as 1179 * needed after TDCALL completion. This also reduces HLT related #VEs 1180 * in addition to having a reliable halt logic execution. 1181 */ 1182 pv_ops.irq.safe_halt = tdx_safe_halt; 1183 pv_ops.irq.halt = tdx_halt; 1184 1185 /* 1186 * TDX intercepts the RDMSR to read the X2APIC ID in the parallel 1187 * bringup low level code. That raises #VE which cannot be handled 1188 * there. 1189 * 1190 * Intel-TDX has a secure RDMSR hypercall, but that needs to be 1191 * implemented separately in the low level startup ASM code. 1192 * Until that is in place, disable parallel bringup for TDX. 1193 */ 1194 x86_cpuinit.parallel_bringup = false; 1195 1196 tdx_announce(); 1197 } 1198