1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021-2022 Intel Corporation */ 3 4 #undef pr_fmt 5 #define pr_fmt(fmt) "tdx: " fmt 6 7 #include <linux/cpufeature.h> 8 #include <linux/export.h> 9 #include <linux/io.h> 10 #include <linux/kexec.h> 11 #include <asm/coco.h> 12 #include <asm/tdx.h> 13 #include <asm/vmx.h> 14 #include <asm/ia32.h> 15 #include <asm/insn.h> 16 #include <asm/insn-eval.h> 17 #include <asm/cpuid/api.h> 18 #include <asm/paravirt_types.h> 19 #include <asm/pgtable.h> 20 #include <asm/set_memory.h> 21 #include <asm/traps.h> 22 23 /* MMIO direction */ 24 #define EPT_READ 0 25 #define EPT_WRITE 1 26 27 /* Port I/O direction */ 28 #define PORT_READ 0 29 #define PORT_WRITE 1 30 31 /* See Exit Qualification for I/O Instructions in VMX documentation */ 32 #define VE_IS_IO_IN(e) ((e) & BIT(3)) 33 #define VE_GET_IO_SIZE(e) (((e) & GENMASK(2, 0)) + 1) 34 #define VE_GET_PORT_NUM(e) ((e) >> 16) 35 #define VE_IS_IO_STRING(e) ((e) & BIT(4)) 36 37 /* TDX Module call error codes */ 38 #define TDCALL_RETURN_CODE(a) ((a) >> 32) 39 #define TDCALL_INVALID_OPERAND 0xc0000100 40 #define TDCALL_OPERAND_BUSY 0x80000200 41 42 #define TDREPORT_SUBTYPE_0 0 43 44 static atomic_long_t nr_shared; 45 46 /* Called from __tdx_hypercall() for unrecoverable failure */ 47 noinstr void __noreturn __tdx_hypercall_failed(void) 48 { 49 instrumentation_begin(); 50 panic("TDVMCALL failed. TDX module bug?"); 51 } 52 53 #ifdef CONFIG_KVM_GUEST 54 long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2, 55 unsigned long p3, unsigned long p4) 56 { 57 struct tdx_module_args args = { 58 .r10 = nr, 59 .r11 = p1, 60 .r12 = p2, 61 .r13 = p3, 62 .r14 = p4, 63 }; 64 65 return __tdx_hypercall(&args); 66 } 67 EXPORT_SYMBOL_GPL(tdx_kvm_hypercall); 68 #endif 69 70 /* 71 * Used for TDX guests to make calls directly to the TD module. This 72 * should only be used for calls that have no legitimate reason to fail 73 * or where the kernel can not survive the call failing. 74 */ 75 static inline void tdcall(u64 fn, struct tdx_module_args *args) 76 { 77 if (__tdcall_ret(fn, args)) 78 panic("TDCALL %lld failed (Buggy TDX module!)\n", fn); 79 } 80 81 /* Read TD-scoped metadata */ 82 static inline u64 tdg_vm_rd(u64 field, u64 *value) 83 { 84 struct tdx_module_args args = { 85 .rdx = field, 86 }; 87 u64 ret; 88 89 ret = __tdcall_ret(TDG_VM_RD, &args); 90 *value = args.r8; 91 92 return ret; 93 } 94 95 /* Write TD-scoped metadata */ 96 static inline u64 tdg_vm_wr(u64 field, u64 value, u64 mask) 97 { 98 struct tdx_module_args args = { 99 .rdx = field, 100 .r8 = value, 101 .r9 = mask, 102 }; 103 104 return __tdcall(TDG_VM_WR, &args); 105 } 106 107 /** 108 * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT 109 * subtype 0) using TDG.MR.REPORT TDCALL. 110 * @reportdata: Address of the input buffer which contains user-defined 111 * REPORTDATA to be included into TDREPORT. 112 * @tdreport: Address of the output buffer to store TDREPORT. 113 * 114 * Refer to section titled "TDG.MR.REPORT leaf" in the TDX Module v1.0 115 * specification for more information on TDG.MR.REPORT TDCALL. 116 * 117 * It is used in the TDX guest driver module to get the TDREPORT0. 118 * 119 * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation, 120 * or -EIO on other TDCALL failures. 121 */ 122 int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport) 123 { 124 struct tdx_module_args args = { 125 .rcx = virt_to_phys(tdreport), 126 .rdx = virt_to_phys(reportdata), 127 .r8 = TDREPORT_SUBTYPE_0, 128 }; 129 u64 ret; 130 131 ret = __tdcall(TDG_MR_REPORT, &args); 132 if (ret) { 133 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND) 134 return -ENXIO; 135 else if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY) 136 return -EBUSY; 137 return -EIO; 138 } 139 140 return 0; 141 } 142 EXPORT_SYMBOL_GPL(tdx_mcall_get_report0); 143 144 /** 145 * tdx_mcall_extend_rtmr() - Wrapper to extend RTMR registers using 146 * TDG.MR.RTMR.EXTEND TDCALL. 147 * @index: Index of RTMR register to be extended. 148 * @data: Address of the input buffer with RTMR register extend data. 149 * 150 * Refer to section titled "TDG.MR.RTMR.EXTEND leaf" in the TDX Module v1.0 151 * specification for more information on TDG.MR.RTMR.EXTEND TDCALL. 152 * 153 * It is used in the TDX guest driver module to allow user to extend the RTMR 154 * registers. 155 * 156 * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation, 157 * or -EIO on other TDCALL failures. 158 */ 159 int tdx_mcall_extend_rtmr(u8 index, u8 *data) 160 { 161 struct tdx_module_args args = { 162 .rcx = virt_to_phys(data), 163 .rdx = index, 164 }; 165 u64 ret; 166 167 ret = __tdcall(TDG_MR_RTMR_EXTEND, &args); 168 if (ret) { 169 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND) 170 return -ENXIO; 171 if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY) 172 return -EBUSY; 173 return -EIO; 174 } 175 176 return 0; 177 } 178 EXPORT_SYMBOL_GPL(tdx_mcall_extend_rtmr); 179 180 /** 181 * tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote 182 * hypercall. 183 * @buf: Address of the directly mapped shared kernel buffer which 184 * contains TDREPORT. The same buffer will be used by VMM to 185 * store the generated TD Quote output. 186 * @size: size of the tdquote buffer (4KB-aligned). 187 * 188 * Refer to section titled "TDG.VP.VMCALL<GetQuote>" in the TDX GHCI 189 * v1.0 specification for more information on GetQuote hypercall. 190 * It is used in the TDX guest driver module to get the TD Quote. 191 * 192 * Return 0 on success or error code on failure. 193 */ 194 u64 tdx_hcall_get_quote(u8 *buf, size_t size) 195 { 196 /* Since buf is a shared memory, set the shared (decrypted) bits */ 197 return _tdx_hypercall(TDVMCALL_GET_QUOTE, cc_mkdec(virt_to_phys(buf)), size, 0, 0); 198 } 199 EXPORT_SYMBOL_GPL(tdx_hcall_get_quote); 200 201 static void __noreturn tdx_panic(const char *msg) 202 { 203 struct tdx_module_args args = { 204 .r10 = TDX_HYPERCALL_STANDARD, 205 .r11 = TDVMCALL_REPORT_FATAL_ERROR, 206 .r12 = 0, /* Error code: 0 is Panic */ 207 }; 208 union { 209 /* Define register order according to the GHCI */ 210 struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; }; 211 212 char bytes[64] __nonstring; 213 } message; 214 215 /* VMM assumes '\0' in byte 65, if the message took all 64 bytes */ 216 strtomem_pad(message.bytes, msg, '\0'); 217 218 args.r8 = message.r8; 219 args.r9 = message.r9; 220 args.r14 = message.r14; 221 args.r15 = message.r15; 222 args.rdi = message.rdi; 223 args.rsi = message.rsi; 224 args.rbx = message.rbx; 225 args.rdx = message.rdx; 226 227 /* 228 * This hypercall should never return and it is not safe 229 * to keep the guest running. Call it forever if it 230 * happens to return. 231 */ 232 while (1) 233 __tdx_hypercall(&args); 234 } 235 236 /* 237 * The kernel cannot handle #VEs when accessing normal kernel memory. Ensure 238 * that no #VE will be delivered for accesses to TD-private memory. 239 * 240 * TDX 1.0 does not allow the guest to disable SEPT #VE on its own. The VMM 241 * controls if the guest will receive such #VE with TD attribute 242 * TDX_TD_ATTR_SEPT_VE_DISABLE. 243 * 244 * Newer TDX modules allow the guest to control if it wants to receive SEPT 245 * violation #VEs. 246 * 247 * Check if the feature is available and disable SEPT #VE if possible. 248 * 249 * If the TD is allowed to disable/enable SEPT #VEs, the TDX_TD_ATTR_SEPT_VE_DISABLE 250 * attribute is no longer reliable. It reflects the initial state of the 251 * control for the TD, but it will not be updated if someone (e.g. bootloader) 252 * changes it before the kernel starts. Kernel must check TDCS_TD_CTLS bit to 253 * determine if SEPT #VEs are enabled or disabled. 254 */ 255 static void disable_sept_ve(u64 td_attr) 256 { 257 const char *msg = "TD misconfiguration: SEPT #VE has to be disabled"; 258 bool debug = td_attr & TDX_TD_ATTR_DEBUG; 259 u64 config, controls; 260 261 /* Is this TD allowed to disable SEPT #VE */ 262 tdg_vm_rd(TDCS_CONFIG_FLAGS, &config); 263 if (!(config & TDCS_CONFIG_FLEXIBLE_PENDING_VE)) { 264 /* No SEPT #VE controls for the guest: check the attribute */ 265 if (td_attr & TDX_TD_ATTR_SEPT_VE_DISABLE) 266 return; 267 268 /* Relax SEPT_VE_DISABLE check for debug TD for backtraces */ 269 if (debug) 270 pr_warn("%s\n", msg); 271 else 272 tdx_panic(msg); 273 return; 274 } 275 276 /* Check if SEPT #VE has been disabled before us */ 277 tdg_vm_rd(TDCS_TD_CTLS, &controls); 278 if (controls & TD_CTLS_PENDING_VE_DISABLE) 279 return; 280 281 /* Keep #VEs enabled for splats in debugging environments */ 282 if (debug) 283 return; 284 285 /* Disable SEPT #VEs */ 286 tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_PENDING_VE_DISABLE, 287 TD_CTLS_PENDING_VE_DISABLE); 288 } 289 290 /* 291 * TDX 1.0 generates a #VE when accessing topology-related CPUID leafs (0xB and 292 * 0x1F) and the X2APIC_APICID MSR. The kernel returns all zeros on CPUID #VEs. 293 * In practice, this means that the kernel can only boot with a plain topology. 294 * Any complications will cause problems. 295 * 296 * The ENUM_TOPOLOGY feature allows the VMM to provide topology information. 297 * Enabling the feature eliminates topology-related #VEs: the TDX module 298 * virtualizes accesses to the CPUID leafs and the MSR. 299 * 300 * Enable ENUM_TOPOLOGY if it is available. 301 */ 302 static void enable_cpu_topology_enumeration(void) 303 { 304 u64 configured; 305 306 /* Has the VMM provided a valid topology configuration? */ 307 tdg_vm_rd(TDCS_TOPOLOGY_ENUM_CONFIGURED, &configured); 308 if (!configured) { 309 pr_err("VMM did not configure X2APIC_IDs properly\n"); 310 return; 311 } 312 313 tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_ENUM_TOPOLOGY, TD_CTLS_ENUM_TOPOLOGY); 314 } 315 316 static void reduce_unnecessary_ve(void) 317 { 318 u64 err = tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_REDUCE_VE, TD_CTLS_REDUCE_VE); 319 320 if (err == TDX_SUCCESS) 321 return; 322 323 /* 324 * Enabling REDUCE_VE includes ENUM_TOPOLOGY. Only try to 325 * enable ENUM_TOPOLOGY if REDUCE_VE was not successful. 326 */ 327 enable_cpu_topology_enumeration(); 328 } 329 330 static void tdx_setup(u64 *cc_mask) 331 { 332 struct tdx_module_args args = {}; 333 unsigned int gpa_width; 334 u64 td_attr; 335 336 /* 337 * TDINFO TDX module call is used to get the TD execution environment 338 * information like GPA width, number of available vcpus, debug mode 339 * information, etc. More details about the ABI can be found in TDX 340 * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL 341 * [TDG.VP.INFO]. 342 */ 343 tdcall(TDG_VP_INFO, &args); 344 345 /* 346 * The highest bit of a guest physical address is the "sharing" bit. 347 * Set it for shared pages and clear it for private pages. 348 * 349 * The GPA width that comes out of this call is critical. TDX guests 350 * can not meaningfully run without it. 351 */ 352 gpa_width = args.rcx & GENMASK(5, 0); 353 *cc_mask = BIT_ULL(gpa_width - 1); 354 355 td_attr = args.rdx; 356 357 /* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */ 358 tdg_vm_wr(TDCS_NOTIFY_ENABLES, 0, -1ULL); 359 360 disable_sept_ve(td_attr); 361 362 reduce_unnecessary_ve(); 363 } 364 365 /* 366 * The TDX module spec states that #VE may be injected for a limited set of 367 * reasons: 368 * 369 * - Emulation of the architectural #VE injection on EPT violation; 370 * 371 * - As a result of guest TD execution of a disallowed instruction, 372 * a disallowed MSR access, or CPUID virtualization; 373 * 374 * - A notification to the guest TD about anomalous behavior; 375 * 376 * The last one is opt-in and is not used by the kernel. 377 * 378 * The Intel Software Developer's Manual describes cases when instruction 379 * length field can be used in section "Information for VM Exits Due to 380 * Instruction Execution". 381 * 382 * For TDX, it ultimately means GET_VEINFO provides reliable instruction length 383 * information if #VE occurred due to instruction execution, but not for EPT 384 * violations. 385 */ 386 static int ve_instr_len(struct ve_info *ve) 387 { 388 switch (ve->exit_reason) { 389 case EXIT_REASON_HLT: 390 case EXIT_REASON_MSR_READ: 391 case EXIT_REASON_MSR_WRITE: 392 case EXIT_REASON_CPUID: 393 case EXIT_REASON_IO_INSTRUCTION: 394 /* It is safe to use ve->instr_len for #VE due instructions */ 395 return ve->instr_len; 396 case EXIT_REASON_EPT_VIOLATION: 397 /* 398 * For EPT violations, ve->insn_len is not defined. For those, 399 * the kernel must decode instructions manually and should not 400 * be using this function. 401 */ 402 WARN_ONCE(1, "ve->instr_len is not defined for EPT violations"); 403 return 0; 404 default: 405 WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason); 406 return ve->instr_len; 407 } 408 } 409 410 static u64 __cpuidle __halt(const bool irq_disabled) 411 { 412 struct tdx_module_args args = { 413 .r10 = TDX_HYPERCALL_STANDARD, 414 .r11 = hcall_func(EXIT_REASON_HLT), 415 .r12 = irq_disabled, 416 }; 417 418 /* 419 * Emulate HLT operation via hypercall. More info about ABI 420 * can be found in TDX Guest-Host-Communication Interface 421 * (GHCI), section 3.8 TDG.VP.VMCALL<Instruction.HLT>. 422 * 423 * The VMM uses the "IRQ disabled" param to understand IRQ 424 * enabled status (RFLAGS.IF) of the TD guest and to determine 425 * whether or not it should schedule the halted vCPU if an 426 * IRQ becomes pending. E.g. if IRQs are disabled, the VMM 427 * can keep the vCPU in virtual HLT, even if an IRQ is 428 * pending, without hanging/breaking the guest. 429 */ 430 return __tdx_hypercall(&args); 431 } 432 433 static int handle_halt(struct ve_info *ve) 434 { 435 const bool irq_disabled = irqs_disabled(); 436 437 /* 438 * HLT with IRQs enabled is unsafe, as an IRQ that is intended to be a 439 * wake event may be consumed before requesting HLT emulation, leaving 440 * the vCPU blocking indefinitely. 441 */ 442 if (WARN_ONCE(!irq_disabled, "HLT emulation with IRQs enabled")) 443 return -EIO; 444 445 if (__halt(irq_disabled)) 446 return -EIO; 447 448 return ve_instr_len(ve); 449 } 450 451 void __cpuidle tdx_halt(void) 452 { 453 const bool irq_disabled = false; 454 455 /* 456 * Use WARN_ONCE() to report the failure. 457 */ 458 if (__halt(irq_disabled)) 459 WARN_ONCE(1, "HLT instruction emulation failed\n"); 460 } 461 462 static void __cpuidle tdx_safe_halt(void) 463 { 464 tdx_halt(); 465 /* 466 * "__cpuidle" section doesn't support instrumentation, so stick 467 * with raw_* variant that avoids tracing hooks. 468 */ 469 raw_local_irq_enable(); 470 } 471 472 static int read_msr(struct pt_regs *regs, struct ve_info *ve) 473 { 474 struct tdx_module_args args = { 475 .r10 = TDX_HYPERCALL_STANDARD, 476 .r11 = hcall_func(EXIT_REASON_MSR_READ), 477 .r12 = regs->cx, 478 }; 479 480 /* 481 * Emulate the MSR read via hypercall. More info about ABI 482 * can be found in TDX Guest-Host-Communication Interface 483 * (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>". 484 */ 485 if (__tdx_hypercall(&args)) 486 return -EIO; 487 488 regs->ax = lower_32_bits(args.r11); 489 regs->dx = upper_32_bits(args.r11); 490 return ve_instr_len(ve); 491 } 492 493 static int write_msr(struct pt_regs *regs, struct ve_info *ve) 494 { 495 struct tdx_module_args args = { 496 .r10 = TDX_HYPERCALL_STANDARD, 497 .r11 = hcall_func(EXIT_REASON_MSR_WRITE), 498 .r12 = regs->cx, 499 .r13 = (u64)regs->dx << 32 | regs->ax, 500 }; 501 502 /* 503 * Emulate the MSR write via hypercall. More info about ABI 504 * can be found in TDX Guest-Host-Communication Interface 505 * (GHCI) section titled "TDG.VP.VMCALL<Instruction.WRMSR>". 506 */ 507 if (__tdx_hypercall(&args)) 508 return -EIO; 509 510 return ve_instr_len(ve); 511 } 512 513 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve) 514 { 515 struct tdx_module_args args = { 516 .r10 = TDX_HYPERCALL_STANDARD, 517 .r11 = hcall_func(EXIT_REASON_CPUID), 518 .r12 = regs->ax, 519 .r13 = regs->cx, 520 }; 521 522 /* 523 * Only allow VMM to control range reserved for hypervisor 524 * communication. 525 * 526 * Return all-zeros for any CPUID outside the range. It matches CPU 527 * behaviour for non-supported leaf. 528 */ 529 if (regs->ax < 0x40000000 || regs->ax > 0x4FFFFFFF) { 530 regs->ax = regs->bx = regs->cx = regs->dx = 0; 531 return ve_instr_len(ve); 532 } 533 534 /* 535 * Emulate the CPUID instruction via a hypercall. More info about 536 * ABI can be found in TDX Guest-Host-Communication Interface 537 * (GHCI), section titled "VP.VMCALL<Instruction.CPUID>". 538 */ 539 if (__tdx_hypercall(&args)) 540 return -EIO; 541 542 /* 543 * As per TDX GHCI CPUID ABI, r12-r15 registers contain contents of 544 * EAX, EBX, ECX, EDX registers after the CPUID instruction execution. 545 * So copy the register contents back to pt_regs. 546 */ 547 regs->ax = args.r12; 548 regs->bx = args.r13; 549 regs->cx = args.r14; 550 regs->dx = args.r15; 551 552 return ve_instr_len(ve); 553 } 554 555 static bool mmio_read(int size, unsigned long addr, unsigned long *val) 556 { 557 struct tdx_module_args args = { 558 .r10 = TDX_HYPERCALL_STANDARD, 559 .r11 = hcall_func(EXIT_REASON_EPT_VIOLATION), 560 .r12 = size, 561 .r13 = EPT_READ, 562 .r14 = addr, 563 }; 564 565 if (__tdx_hypercall(&args)) 566 return false; 567 568 *val = args.r11; 569 return true; 570 } 571 572 static bool mmio_write(int size, unsigned long addr, unsigned long val) 573 { 574 return !_tdx_hypercall(hcall_func(EXIT_REASON_EPT_VIOLATION), size, 575 EPT_WRITE, addr, val); 576 } 577 578 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve) 579 { 580 unsigned long *reg, val, vaddr; 581 char buffer[MAX_INSN_SIZE]; 582 enum insn_mmio_type mmio; 583 struct insn insn = {}; 584 int size, extend_size; 585 u8 extend_val = 0; 586 587 /* Only in-kernel MMIO is supported */ 588 if (WARN_ON_ONCE(user_mode(regs))) 589 return -EFAULT; 590 591 if (copy_from_kernel_nofault(buffer, (void *)regs->ip, MAX_INSN_SIZE)) 592 return -EFAULT; 593 594 if (insn_decode(&insn, buffer, MAX_INSN_SIZE, INSN_MODE_64)) 595 return -EINVAL; 596 597 mmio = insn_decode_mmio(&insn, &size); 598 if (WARN_ON_ONCE(mmio == INSN_MMIO_DECODE_FAILED)) 599 return -EINVAL; 600 601 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { 602 reg = insn_get_modrm_reg_ptr(&insn, regs); 603 if (!reg) 604 return -EINVAL; 605 } 606 607 if (!fault_in_kernel_space(ve->gla)) { 608 WARN_ONCE(1, "Access to userspace address is not supported"); 609 return -EINVAL; 610 } 611 612 /* 613 * Reject EPT violation #VEs that split pages. 614 * 615 * MMIO accesses are supposed to be naturally aligned and therefore 616 * never cross page boundaries. Seeing split page accesses indicates 617 * a bug or a load_unaligned_zeropad() that stepped into an MMIO page. 618 * 619 * load_unaligned_zeropad() will recover using exception fixups. 620 */ 621 vaddr = (unsigned long)insn_get_addr_ref(&insn, regs); 622 if (vaddr / PAGE_SIZE != (vaddr + size - 1) / PAGE_SIZE) 623 return -EFAULT; 624 625 /* Handle writes first */ 626 switch (mmio) { 627 case INSN_MMIO_WRITE: 628 memcpy(&val, reg, size); 629 if (!mmio_write(size, ve->gpa, val)) 630 return -EIO; 631 return insn.length; 632 case INSN_MMIO_WRITE_IMM: 633 val = insn.immediate.value; 634 if (!mmio_write(size, ve->gpa, val)) 635 return -EIO; 636 return insn.length; 637 case INSN_MMIO_READ: 638 case INSN_MMIO_READ_ZERO_EXTEND: 639 case INSN_MMIO_READ_SIGN_EXTEND: 640 /* Reads are handled below */ 641 break; 642 case INSN_MMIO_MOVS: 643 case INSN_MMIO_DECODE_FAILED: 644 /* 645 * MMIO was accessed with an instruction that could not be 646 * decoded or handled properly. It was likely not using io.h 647 * helpers or accessed MMIO accidentally. 648 */ 649 return -EINVAL; 650 default: 651 WARN_ONCE(1, "Unknown insn_decode_mmio() decode value?"); 652 return -EINVAL; 653 } 654 655 /* Handle reads */ 656 if (!mmio_read(size, ve->gpa, &val)) 657 return -EIO; 658 659 switch (mmio) { 660 case INSN_MMIO_READ: 661 /* Zero-extend for 32-bit operation */ 662 extend_size = size == 4 ? sizeof(*reg) : 0; 663 break; 664 case INSN_MMIO_READ_ZERO_EXTEND: 665 /* Zero extend based on operand size */ 666 extend_size = insn.opnd_bytes; 667 break; 668 case INSN_MMIO_READ_SIGN_EXTEND: 669 /* Sign extend based on operand size */ 670 extend_size = insn.opnd_bytes; 671 if (size == 1 && val & BIT(7)) 672 extend_val = 0xFF; 673 else if (size > 1 && val & BIT(15)) 674 extend_val = 0xFF; 675 break; 676 default: 677 /* All other cases has to be covered with the first switch() */ 678 WARN_ON_ONCE(1); 679 return -EINVAL; 680 } 681 682 if (extend_size) 683 memset(reg, extend_val, extend_size); 684 memcpy(reg, &val, size); 685 return insn.length; 686 } 687 688 static bool handle_in(struct pt_regs *regs, int size, int port) 689 { 690 struct tdx_module_args args = { 691 .r10 = TDX_HYPERCALL_STANDARD, 692 .r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION), 693 .r12 = size, 694 .r13 = PORT_READ, 695 .r14 = port, 696 }; 697 bool success; 698 u64 val; 699 700 /* 701 * Emulate the I/O read via hypercall. More info about ABI can be found 702 * in TDX Guest-Host-Communication Interface (GHCI) section titled 703 * "TDG.VP.VMCALL<Instruction.IO>". 704 */ 705 success = !__tdx_hypercall(&args); 706 val = success ? args.r11 : 0; 707 708 insn_assign_reg(®s->ax, val, size); 709 710 return success; 711 } 712 713 static bool handle_out(struct pt_regs *regs, int size, int port) 714 { 715 u64 mask = GENMASK(BITS_PER_BYTE * size - 1, 0); 716 717 /* 718 * Emulate the I/O write via hypercall. More info about ABI can be found 719 * in TDX Guest-Host-Communication Interface (GHCI) section titled 720 * "TDG.VP.VMCALL<Instruction.IO>". 721 */ 722 return !_tdx_hypercall(hcall_func(EXIT_REASON_IO_INSTRUCTION), size, 723 PORT_WRITE, port, regs->ax & mask); 724 } 725 726 /* 727 * Emulate I/O using hypercall. 728 * 729 * Assumes the IO instruction was using ax, which is enforced 730 * by the standard io.h macros. 731 * 732 * Return True on success or False on failure. 733 */ 734 static int handle_io(struct pt_regs *regs, struct ve_info *ve) 735 { 736 u32 exit_qual = ve->exit_qual; 737 int size, port; 738 bool in, ret; 739 740 if (VE_IS_IO_STRING(exit_qual)) 741 return -EIO; 742 743 in = VE_IS_IO_IN(exit_qual); 744 size = VE_GET_IO_SIZE(exit_qual); 745 port = VE_GET_PORT_NUM(exit_qual); 746 747 748 if (in) 749 ret = handle_in(regs, size, port); 750 else 751 ret = handle_out(regs, size, port); 752 if (!ret) 753 return -EIO; 754 755 return ve_instr_len(ve); 756 } 757 758 /* 759 * Early #VE exception handler. Only handles a subset of port I/O. 760 * Intended only for earlyprintk. If failed, return false. 761 */ 762 __init bool tdx_early_handle_ve(struct pt_regs *regs) 763 { 764 struct ve_info ve; 765 int insn_len; 766 767 tdx_get_ve_info(&ve); 768 769 if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION) 770 return false; 771 772 insn_len = handle_io(regs, &ve); 773 if (insn_len < 0) 774 return false; 775 776 regs->ip += insn_len; 777 return true; 778 } 779 780 void tdx_get_ve_info(struct ve_info *ve) 781 { 782 struct tdx_module_args args = {}; 783 784 /* 785 * Called during #VE handling to retrieve the #VE info from the 786 * TDX module. 787 * 788 * This has to be called early in #VE handling. A "nested" #VE which 789 * occurs before this will raise a #DF and is not recoverable. 790 * 791 * The call retrieves the #VE info from the TDX module, which also 792 * clears the "#VE valid" flag. This must be done before anything else 793 * because any #VE that occurs while the valid flag is set will lead to 794 * #DF. 795 * 796 * Note, the TDX module treats virtual NMIs as inhibited if the #VE 797 * valid flag is set. It means that NMI=>#VE will not result in a #DF. 798 */ 799 tdcall(TDG_VP_VEINFO_GET, &args); 800 801 /* Transfer the output parameters */ 802 ve->exit_reason = args.rcx; 803 ve->exit_qual = args.rdx; 804 ve->gla = args.r8; 805 ve->gpa = args.r9; 806 ve->instr_len = lower_32_bits(args.r10); 807 ve->instr_info = upper_32_bits(args.r10); 808 } 809 810 /* 811 * Handle the user initiated #VE. 812 * 813 * On success, returns the number of bytes RIP should be incremented (>=0) 814 * or -errno on error. 815 */ 816 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve) 817 { 818 switch (ve->exit_reason) { 819 case EXIT_REASON_CPUID: 820 return handle_cpuid(regs, ve); 821 default: 822 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason); 823 return -EIO; 824 } 825 } 826 827 static inline bool is_private_gpa(u64 gpa) 828 { 829 return gpa == cc_mkenc(gpa); 830 } 831 832 /* 833 * Handle the kernel #VE. 834 * 835 * On success, returns the number of bytes RIP should be incremented (>=0) 836 * or -errno on error. 837 */ 838 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve) 839 { 840 switch (ve->exit_reason) { 841 case EXIT_REASON_HLT: 842 return handle_halt(ve); 843 case EXIT_REASON_MSR_READ: 844 return read_msr(regs, ve); 845 case EXIT_REASON_MSR_WRITE: 846 return write_msr(regs, ve); 847 case EXIT_REASON_CPUID: 848 return handle_cpuid(regs, ve); 849 case EXIT_REASON_EPT_VIOLATION: 850 if (is_private_gpa(ve->gpa)) 851 panic("Unexpected EPT-violation on private memory."); 852 return handle_mmio(regs, ve); 853 case EXIT_REASON_IO_INSTRUCTION: 854 return handle_io(regs, ve); 855 default: 856 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason); 857 return -EIO; 858 } 859 } 860 861 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve) 862 { 863 int insn_len; 864 865 if (user_mode(regs)) 866 insn_len = virt_exception_user(regs, ve); 867 else 868 insn_len = virt_exception_kernel(regs, ve); 869 if (insn_len < 0) 870 return false; 871 872 /* After successful #VE handling, move the IP */ 873 regs->ip += insn_len; 874 875 return true; 876 } 877 878 static bool tdx_tlb_flush_required(bool private) 879 { 880 /* 881 * TDX guest is responsible for flushing TLB on private->shared 882 * transition. VMM is responsible for flushing on shared->private. 883 * 884 * The VMM _can't_ flush private addresses as it can't generate PAs 885 * with the guest's HKID. Shared memory isn't subject to integrity 886 * checking, i.e. the VMM doesn't need to flush for its own protection. 887 * 888 * There's no need to flush when converting from shared to private, 889 * as flushing is the VMM's responsibility in this case, e.g. it must 890 * flush to avoid integrity failures in the face of a buggy or 891 * malicious guest. 892 */ 893 return !private; 894 } 895 896 static bool tdx_cache_flush_required(void) 897 { 898 /* 899 * AMD SME/SEV can avoid cache flushing if HW enforces cache coherence. 900 * TDX doesn't have such capability. 901 * 902 * Flush cache unconditionally. 903 */ 904 return true; 905 } 906 907 /* 908 * Notify the VMM about page mapping conversion. More info about ABI 909 * can be found in TDX Guest-Host-Communication Interface (GHCI), 910 * section "TDG.VP.VMCALL<MapGPA>". 911 */ 912 static bool tdx_map_gpa(phys_addr_t start, phys_addr_t end, bool enc) 913 { 914 /* Retrying the hypercall a second time should succeed; use 3 just in case */ 915 const int max_retries_per_page = 3; 916 int retry_count = 0; 917 918 if (!enc) { 919 /* Set the shared (decrypted) bits: */ 920 start |= cc_mkdec(0); 921 end |= cc_mkdec(0); 922 } 923 924 while (retry_count < max_retries_per_page) { 925 struct tdx_module_args args = { 926 .r10 = TDX_HYPERCALL_STANDARD, 927 .r11 = TDVMCALL_MAP_GPA, 928 .r12 = start, 929 .r13 = end - start }; 930 931 u64 map_fail_paddr; 932 u64 ret = __tdx_hypercall(&args); 933 934 if (ret != TDVMCALL_STATUS_RETRY) 935 return !ret; 936 /* 937 * The guest must retry the operation for the pages in the 938 * region starting at the GPA specified in R11. R11 comes 939 * from the untrusted VMM. Sanity check it. 940 */ 941 map_fail_paddr = args.r11; 942 if (map_fail_paddr < start || map_fail_paddr >= end) 943 return false; 944 945 /* "Consume" a retry without forward progress */ 946 if (map_fail_paddr == start) { 947 retry_count++; 948 continue; 949 } 950 951 start = map_fail_paddr; 952 retry_count = 0; 953 } 954 955 return false; 956 } 957 958 /* 959 * Inform the VMM of the guest's intent for this physical page: shared with 960 * the VMM or private to the guest. The VMM is expected to change its mapping 961 * of the page in response. 962 */ 963 static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc) 964 { 965 phys_addr_t start = __pa(vaddr); 966 phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE); 967 968 if (!tdx_map_gpa(start, end, enc)) 969 return false; 970 971 /* shared->private conversion requires memory to be accepted before use */ 972 if (enc) 973 return tdx_accept_memory(start, end); 974 975 return true; 976 } 977 978 static int tdx_enc_status_change_prepare(unsigned long vaddr, int numpages, 979 bool enc) 980 { 981 /* 982 * Only handle shared->private conversion here. 983 * See the comment in tdx_early_init(). 984 */ 985 if (enc && !tdx_enc_status_changed(vaddr, numpages, enc)) 986 return -EIO; 987 988 return 0; 989 } 990 991 static int tdx_enc_status_change_finish(unsigned long vaddr, int numpages, 992 bool enc) 993 { 994 /* 995 * Only handle private->shared conversion here. 996 * See the comment in tdx_early_init(). 997 */ 998 if (!enc && !tdx_enc_status_changed(vaddr, numpages, enc)) 999 return -EIO; 1000 1001 if (enc) 1002 atomic_long_sub(numpages, &nr_shared); 1003 else 1004 atomic_long_add(numpages, &nr_shared); 1005 1006 return 0; 1007 } 1008 1009 /* Stop new private<->shared conversions */ 1010 static void tdx_kexec_begin(void) 1011 { 1012 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1013 return; 1014 1015 /* 1016 * Crash kernel reaches here with interrupts disabled: can't wait for 1017 * conversions to finish. 1018 * 1019 * If race happened, just report and proceed. 1020 */ 1021 if (!set_memory_enc_stop_conversion()) 1022 pr_warn("Failed to stop shared<->private conversions\n"); 1023 } 1024 1025 /* Walk direct mapping and convert all shared memory back to private */ 1026 static void tdx_kexec_finish(void) 1027 { 1028 unsigned long addr, end; 1029 long found = 0, shared; 1030 1031 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1032 return; 1033 1034 lockdep_assert_irqs_disabled(); 1035 1036 addr = PAGE_OFFSET; 1037 end = PAGE_OFFSET + get_max_mapped(); 1038 1039 while (addr < end) { 1040 unsigned long size; 1041 unsigned int level; 1042 pte_t *pte; 1043 1044 pte = lookup_address(addr, &level); 1045 size = page_level_size(level); 1046 1047 if (pte && pte_decrypted(*pte)) { 1048 int pages = size / PAGE_SIZE; 1049 1050 /* 1051 * Touching memory with shared bit set triggers implicit 1052 * conversion to shared. 1053 * 1054 * Make sure nobody touches the shared range from 1055 * now on. 1056 */ 1057 set_pte(pte, __pte(0)); 1058 1059 /* 1060 * Memory encryption state persists across kexec. 1061 * If tdx_enc_status_changed() fails in the first 1062 * kernel, it leaves memory in an unknown state. 1063 * 1064 * If that memory remains shared, accessing it in the 1065 * *next* kernel through a private mapping will result 1066 * in an unrecoverable guest shutdown. 1067 * 1068 * The kdump kernel boot is not impacted as it uses 1069 * a pre-reserved memory range that is always private. 1070 * However, gathering crash information could lead to 1071 * a crash if it accesses unconverted memory through 1072 * a private mapping which is possible when accessing 1073 * that memory through /proc/vmcore, for example. 1074 * 1075 * In all cases, print error info in order to leave 1076 * enough bread crumbs for debugging. 1077 */ 1078 if (!tdx_enc_status_changed(addr, pages, true)) { 1079 pr_err("Failed to unshare range %#lx-%#lx\n", 1080 addr, addr + size); 1081 } 1082 1083 found += pages; 1084 } 1085 1086 addr += size; 1087 } 1088 1089 __flush_tlb_all(); 1090 1091 shared = atomic_long_read(&nr_shared); 1092 if (shared != found) { 1093 pr_err("shared page accounting is off\n"); 1094 pr_err("nr_shared = %ld, nr_found = %ld\n", shared, found); 1095 } 1096 } 1097 1098 static __init void tdx_announce(void) 1099 { 1100 struct tdx_module_args args = {}; 1101 u64 controls; 1102 1103 pr_info("Guest detected\n"); 1104 1105 tdcall(TDG_VP_INFO, &args); 1106 tdx_dump_attributes(args.rdx); 1107 1108 tdg_vm_rd(TDCS_TD_CTLS, &controls); 1109 tdx_dump_td_ctls(controls); 1110 } 1111 1112 void __init tdx_early_init(void) 1113 { 1114 u64 cc_mask; 1115 u32 eax, sig[3]; 1116 1117 cpuid_count(TDX_CPUID_LEAF_ID, 0, &eax, &sig[0], &sig[2], &sig[1]); 1118 1119 if (memcmp(TDX_IDENT, sig, sizeof(sig))) 1120 return; 1121 1122 setup_force_cpu_cap(X86_FEATURE_TDX_GUEST); 1123 1124 /* TSC is the only reliable clock in TDX guest */ 1125 setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); 1126 1127 cc_vendor = CC_VENDOR_INTEL; 1128 1129 /* Configure the TD */ 1130 tdx_setup(&cc_mask); 1131 1132 cc_set_mask(cc_mask); 1133 1134 /* 1135 * All bits above GPA width are reserved and kernel treats shared bit 1136 * as flag, not as part of physical address. 1137 * 1138 * Adjust physical mask to only cover valid GPA bits. 1139 */ 1140 physical_mask &= cc_mask - 1; 1141 1142 /* 1143 * The kernel mapping should match the TDX metadata for the page. 1144 * load_unaligned_zeropad() can touch memory *adjacent* to that which is 1145 * owned by the caller and can catch even _momentary_ mismatches. Bad 1146 * things happen on mismatch: 1147 * 1148 * - Private mapping => Shared Page == Guest shutdown 1149 * - Shared mapping => Private Page == Recoverable #VE 1150 * 1151 * guest.enc_status_change_prepare() converts the page from 1152 * shared=>private before the mapping becomes private. 1153 * 1154 * guest.enc_status_change_finish() converts the page from 1155 * private=>shared after the mapping becomes private. 1156 * 1157 * In both cases there is a temporary shared mapping to a private page, 1158 * which can result in a #VE. But, there is never a private mapping to 1159 * a shared page. 1160 */ 1161 x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; 1162 x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; 1163 1164 x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; 1165 x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required; 1166 1167 x86_platform.guest.enc_kexec_begin = tdx_kexec_begin; 1168 x86_platform.guest.enc_kexec_finish = tdx_kexec_finish; 1169 1170 /* 1171 * Avoid "sti;hlt" execution in TDX guests as HLT induces a #VE that 1172 * will enable interrupts before HLT TDCALL invocation if executed 1173 * in STI-shadow, possibly resulting in missed wakeup events. 1174 * 1175 * Modify all possible HLT execution paths to use TDX specific routines 1176 * that directly execute TDCALL and toggle the interrupt state as 1177 * needed after TDCALL completion. This also reduces HLT related #VEs 1178 * in addition to having a reliable halt logic execution. 1179 */ 1180 pv_ops.irq.safe_halt = tdx_safe_halt; 1181 pv_ops.irq.halt = tdx_halt; 1182 1183 /* 1184 * TDX intercepts the RDMSR to read the X2APIC ID in the parallel 1185 * bringup low level code. That raises #VE which cannot be handled 1186 * there. 1187 * 1188 * Intel-TDX has a secure RDMSR hypercall, but that needs to be 1189 * implemented separately in the low level startup ASM code. 1190 * Until that is in place, disable parallel bringup for TDX. 1191 */ 1192 x86_cpuinit.parallel_bringup = false; 1193 1194 tdx_announce(); 1195 } 1196