1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021-2022 Intel Corporation */
3
4 #undef pr_fmt
5 #define pr_fmt(fmt) "tdx: " fmt
6
7 #include <linux/cpufeature.h>
8 #include <linux/export.h>
9 #include <linux/io.h>
10 #include <linux/kexec.h>
11 #include <asm/coco.h>
12 #include <asm/tdx.h>
13 #include <asm/vmx.h>
14 #include <asm/ia32.h>
15 #include <asm/insn.h>
16 #include <asm/insn-eval.h>
17 #include <asm/cpuid/api.h>
18 #include <asm/paravirt_types.h>
19 #include <asm/pgtable.h>
20 #include <asm/set_memory.h>
21 #include <asm/traps.h>
22
23 /* MMIO direction */
24 #define EPT_READ 0
25 #define EPT_WRITE 1
26
27 /* Port I/O direction */
28 #define PORT_READ 0
29 #define PORT_WRITE 1
30
31 /* See Exit Qualification for I/O Instructions in VMX documentation */
32 #define VE_IS_IO_IN(e) ((e) & BIT(3))
33 #define VE_GET_IO_SIZE(e) (((e) & GENMASK(2, 0)) + 1)
34 #define VE_GET_PORT_NUM(e) ((e) >> 16)
35 #define VE_IS_IO_STRING(e) ((e) & BIT(4))
36
37 /* TDX Module call error codes */
38 #define TDCALL_RETURN_CODE(a) ((a) >> 32)
39 #define TDCALL_INVALID_OPERAND 0xc0000100
40 #define TDCALL_OPERAND_BUSY 0x80000200
41
42 #define TDREPORT_SUBTYPE_0 0
43
44 static atomic_long_t nr_shared;
45
46 /* Called from __tdx_hypercall() for unrecoverable failure */
__tdx_hypercall_failed(void)47 noinstr void __noreturn __tdx_hypercall_failed(void)
48 {
49 instrumentation_begin();
50 panic("TDVMCALL failed. TDX module bug?");
51 }
52
53 #ifdef CONFIG_KVM_GUEST
tdx_kvm_hypercall(unsigned int nr,unsigned long p1,unsigned long p2,unsigned long p3,unsigned long p4)54 long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2,
55 unsigned long p3, unsigned long p4)
56 {
57 struct tdx_module_args args = {
58 .r10 = nr,
59 .r11 = p1,
60 .r12 = p2,
61 .r13 = p3,
62 .r14 = p4,
63 };
64
65 return __tdx_hypercall(&args);
66 }
67 EXPORT_SYMBOL_GPL(tdx_kvm_hypercall);
68 #endif
69
70 /*
71 * Used for TDX guests to make calls directly to the TD module. This
72 * should only be used for calls that have no legitimate reason to fail
73 * or where the kernel can not survive the call failing.
74 */
tdcall(u64 fn,struct tdx_module_args * args)75 static inline void tdcall(u64 fn, struct tdx_module_args *args)
76 {
77 if (__tdcall_ret(fn, args))
78 panic("TDCALL %lld failed (Buggy TDX module!)\n", fn);
79 }
80
81 /* Read TD-scoped metadata */
tdg_vm_rd(u64 field,u64 * value)82 static inline u64 tdg_vm_rd(u64 field, u64 *value)
83 {
84 struct tdx_module_args args = {
85 .rdx = field,
86 };
87 u64 ret;
88
89 ret = __tdcall_ret(TDG_VM_RD, &args);
90 *value = args.r8;
91
92 return ret;
93 }
94
95 /* Write TD-scoped metadata */
tdg_vm_wr(u64 field,u64 value,u64 mask)96 static inline u64 tdg_vm_wr(u64 field, u64 value, u64 mask)
97 {
98 struct tdx_module_args args = {
99 .rdx = field,
100 .r8 = value,
101 .r9 = mask,
102 };
103
104 return __tdcall(TDG_VM_WR, &args);
105 }
106
107 /**
108 * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT
109 * subtype 0) using TDG.MR.REPORT TDCALL.
110 * @reportdata: Address of the input buffer which contains user-defined
111 * REPORTDATA to be included into TDREPORT.
112 * @tdreport: Address of the output buffer to store TDREPORT.
113 *
114 * Refer to section titled "TDG.MR.REPORT leaf" in the TDX Module v1.0
115 * specification for more information on TDG.MR.REPORT TDCALL.
116 *
117 * It is used in the TDX guest driver module to get the TDREPORT0.
118 *
119 * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation,
120 * or -EIO on other TDCALL failures.
121 */
tdx_mcall_get_report0(u8 * reportdata,u8 * tdreport)122 int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
123 {
124 struct tdx_module_args args = {
125 .rcx = virt_to_phys(tdreport),
126 .rdx = virt_to_phys(reportdata),
127 .r8 = TDREPORT_SUBTYPE_0,
128 };
129 u64 ret;
130
131 ret = __tdcall(TDG_MR_REPORT, &args);
132 if (ret) {
133 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND)
134 return -ENXIO;
135 else if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY)
136 return -EBUSY;
137 return -EIO;
138 }
139
140 return 0;
141 }
142 EXPORT_SYMBOL_FOR_MODULES(tdx_mcall_get_report0, "tdx-guest");
143
144 /**
145 * tdx_mcall_extend_rtmr() - Wrapper to extend RTMR registers using
146 * TDG.MR.RTMR.EXTEND TDCALL.
147 * @index: Index of RTMR register to be extended.
148 * @data: Address of the input buffer with RTMR register extend data.
149 *
150 * Refer to section titled "TDG.MR.RTMR.EXTEND leaf" in the TDX Module v1.0
151 * specification for more information on TDG.MR.RTMR.EXTEND TDCALL.
152 *
153 * It is used in the TDX guest driver module to allow user to extend the RTMR
154 * registers.
155 *
156 * Return 0 on success, -ENXIO for invalid operands, -EBUSY for busy operation,
157 * or -EIO on other TDCALL failures.
158 */
tdx_mcall_extend_rtmr(u8 index,u8 * data)159 int tdx_mcall_extend_rtmr(u8 index, u8 *data)
160 {
161 struct tdx_module_args args = {
162 .rcx = virt_to_phys(data),
163 .rdx = index,
164 };
165 u64 ret;
166
167 ret = __tdcall(TDG_MR_RTMR_EXTEND, &args);
168 if (ret) {
169 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND)
170 return -ENXIO;
171 if (TDCALL_RETURN_CODE(ret) == TDCALL_OPERAND_BUSY)
172 return -EBUSY;
173 return -EIO;
174 }
175
176 return 0;
177 }
178 EXPORT_SYMBOL_FOR_MODULES(tdx_mcall_extend_rtmr, "tdx-guest");
179
180 /**
181 * tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote
182 * hypercall.
183 * @buf: Address of the directly mapped shared kernel buffer which
184 * contains TDREPORT. The same buffer will be used by VMM to
185 * store the generated TD Quote output.
186 * @size: size of the tdquote buffer (4KB-aligned).
187 *
188 * Refer to section titled "TDG.VP.VMCALL<GetQuote>" in the TDX GHCI
189 * v1.0 specification for more information on GetQuote hypercall.
190 * It is used in the TDX guest driver module to get the TD Quote.
191 *
192 * Return 0 on success or error code on failure.
193 */
tdx_hcall_get_quote(void * buf,size_t size)194 u64 tdx_hcall_get_quote(void *buf, size_t size)
195 {
196 /* Since buf is a shared memory, set the shared (decrypted) bits */
197 return _tdx_hypercall(TDVMCALL_GET_QUOTE, cc_mkdec(virt_to_phys(buf)), size, 0, 0);
198 }
199 EXPORT_SYMBOL_FOR_MODULES(tdx_hcall_get_quote, "tdx-guest");
200
201 /*
202 * Ask the TDX module what the largest Quote on the host platform might be.
203 */
tdx_get_max_quote_size(u64 * max_quote_size)204 int tdx_get_max_quote_size(u64 *max_quote_size)
205 {
206 u64 err;
207
208 err = tdg_vm_rd(TDCS_QUOTE_MAX_SIZE, max_quote_size);
209
210 /* Old modules do not support this. Tell the caller. */
211 if (err)
212 return -EINVAL;
213
214 return 0;
215 }
216 EXPORT_SYMBOL_FOR_MODULES(tdx_get_max_quote_size, "tdx-guest");
217
218 /*
219 * The kernel cannot handle #VEs when accessing normal kernel memory. Ensure
220 * that no #VE will be delivered for accesses to TD-private memory.
221 *
222 * TDX 1.0 does not allow the guest to disable SEPT #VE on its own. The VMM
223 * controls if the guest will receive such #VE with TD attribute
224 * TDX_TD_ATTR_SEPT_VE_DISABLE.
225 *
226 * Newer TDX modules allow the guest to control if it wants to receive SEPT
227 * violation #VEs.
228 *
229 * Check if the feature is available and disable SEPT #VE if possible.
230 *
231 * If the TD is allowed to disable/enable SEPT #VEs, the TDX_TD_ATTR_SEPT_VE_DISABLE
232 * attribute is no longer reliable. It reflects the initial state of the
233 * control for the TD, but it will not be updated if someone (e.g. bootloader)
234 * changes it before the kernel starts. Kernel must check TDCS_TD_CTLS bit to
235 * determine if SEPT #VEs are enabled or disabled.
236 */
disable_sept_ve(u64 td_attr)237 static void disable_sept_ve(u64 td_attr)
238 {
239 const char *msg = "TD misconfiguration: SEPT #VE has to be disabled";
240 bool debug = td_attr & TDX_TD_ATTR_DEBUG;
241 u64 config, controls;
242
243 /* Is this TD allowed to disable SEPT #VE */
244 tdg_vm_rd(TDCS_CONFIG_FLAGS, &config);
245 if (!(config & TDCS_CONFIG_FLEXIBLE_PENDING_VE)) {
246 /* No SEPT #VE controls for the guest: check the attribute */
247 if (td_attr & TDX_TD_ATTR_SEPT_VE_DISABLE)
248 return;
249
250 /* Relax SEPT_VE_DISABLE check for debug TD for backtraces */
251 if (debug)
252 pr_warn("%s\n", msg);
253 else
254 tdx_panic(msg);
255 return;
256 }
257
258 /* Check if SEPT #VE has been disabled before us */
259 tdg_vm_rd(TDCS_TD_CTLS, &controls);
260 if (controls & TD_CTLS_PENDING_VE_DISABLE)
261 return;
262
263 /* Keep #VEs enabled for splats in debugging environments */
264 if (debug)
265 return;
266
267 /* Disable SEPT #VEs */
268 tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_PENDING_VE_DISABLE,
269 TD_CTLS_PENDING_VE_DISABLE);
270 }
271
272 /*
273 * TDX 1.0 generates a #VE when accessing topology-related CPUID leafs (0xB and
274 * 0x1F) and the X2APIC_APICID MSR. The kernel returns all zeros on CPUID #VEs.
275 * In practice, this means that the kernel can only boot with a plain topology.
276 * Any complications will cause problems.
277 *
278 * The ENUM_TOPOLOGY feature allows the VMM to provide topology information.
279 * Enabling the feature eliminates topology-related #VEs: the TDX module
280 * virtualizes accesses to the CPUID leafs and the MSR.
281 *
282 * Enable ENUM_TOPOLOGY if it is available.
283 */
enable_cpu_topology_enumeration(void)284 static void enable_cpu_topology_enumeration(void)
285 {
286 u64 configured;
287
288 /* Has the VMM provided a valid topology configuration? */
289 tdg_vm_rd(TDCS_TOPOLOGY_ENUM_CONFIGURED, &configured);
290 if (!configured) {
291 pr_err("VMM did not configure X2APIC_IDs properly\n");
292 return;
293 }
294
295 tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_ENUM_TOPOLOGY, TD_CTLS_ENUM_TOPOLOGY);
296 }
297
reduce_unnecessary_ve(void)298 static void reduce_unnecessary_ve(void)
299 {
300 u64 err = tdg_vm_wr(TDCS_TD_CTLS, TD_CTLS_REDUCE_VE, TD_CTLS_REDUCE_VE);
301
302 if (err == TDX_SUCCESS)
303 return;
304
305 /*
306 * Enabling REDUCE_VE includes ENUM_TOPOLOGY. Only try to
307 * enable ENUM_TOPOLOGY if REDUCE_VE was not successful.
308 */
309 enable_cpu_topology_enumeration();
310 }
311
tdx_setup(u64 * cc_mask)312 static void tdx_setup(u64 *cc_mask)
313 {
314 struct tdx_module_args args = {};
315 unsigned int gpa_width;
316 u64 td_attr;
317
318 /*
319 * TDINFO TDX module call is used to get the TD execution environment
320 * information like GPA width, number of available vcpus, debug mode
321 * information, etc. More details about the ABI can be found in TDX
322 * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL
323 * [TDG.VP.INFO].
324 */
325 tdcall(TDG_VP_INFO, &args);
326
327 /*
328 * The highest bit of a guest physical address is the "sharing" bit.
329 * Set it for shared pages and clear it for private pages.
330 *
331 * The GPA width that comes out of this call is critical. TDX guests
332 * can not meaningfully run without it.
333 */
334 gpa_width = args.rcx & GENMASK(5, 0);
335 *cc_mask = BIT_ULL(gpa_width - 1);
336
337 td_attr = args.rdx;
338
339 /* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */
340 tdg_vm_wr(TDCS_NOTIFY_ENABLES, 0, -1ULL);
341
342 disable_sept_ve(td_attr);
343
344 reduce_unnecessary_ve();
345 }
346
347 /*
348 * The TDX module spec states that #VE may be injected for a limited set of
349 * reasons:
350 *
351 * - Emulation of the architectural #VE injection on EPT violation;
352 *
353 * - As a result of guest TD execution of a disallowed instruction,
354 * a disallowed MSR access, or CPUID virtualization;
355 *
356 * - A notification to the guest TD about anomalous behavior;
357 *
358 * The last one is opt-in and is not used by the kernel.
359 *
360 * The Intel Software Developer's Manual describes cases when instruction
361 * length field can be used in section "Information for VM Exits Due to
362 * Instruction Execution".
363 *
364 * For TDX, it ultimately means GET_VEINFO provides reliable instruction length
365 * information if #VE occurred due to instruction execution, but not for EPT
366 * violations.
367 */
ve_instr_len(struct ve_info * ve)368 static int ve_instr_len(struct ve_info *ve)
369 {
370 switch (ve->exit_reason) {
371 case EXIT_REASON_HLT:
372 case EXIT_REASON_MSR_READ:
373 case EXIT_REASON_MSR_WRITE:
374 case EXIT_REASON_CPUID:
375 case EXIT_REASON_IO_INSTRUCTION:
376 /* It is safe to use ve->instr_len for #VE due instructions */
377 return ve->instr_len;
378 case EXIT_REASON_EPT_VIOLATION:
379 /*
380 * For EPT violations, ve->insn_len is not defined. For those,
381 * the kernel must decode instructions manually and should not
382 * be using this function.
383 */
384 WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");
385 return 0;
386 default:
387 WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);
388 return ve->instr_len;
389 }
390 }
391
__halt(const bool irq_disabled)392 static u64 __cpuidle __halt(const bool irq_disabled)
393 {
394 struct tdx_module_args args = {
395 .r10 = TDX_HYPERCALL_STANDARD,
396 .r11 = hcall_func(EXIT_REASON_HLT),
397 .r12 = irq_disabled,
398 };
399
400 /*
401 * Emulate HLT operation via hypercall. More info about ABI
402 * can be found in TDX Guest-Host-Communication Interface
403 * (GHCI), section 3.8 TDG.VP.VMCALL<Instruction.HLT>.
404 *
405 * The VMM uses the "IRQ disabled" param to understand IRQ
406 * enabled status (RFLAGS.IF) of the TD guest and to determine
407 * whether or not it should schedule the halted vCPU if an
408 * IRQ becomes pending. E.g. if IRQs are disabled, the VMM
409 * can keep the vCPU in virtual HLT, even if an IRQ is
410 * pending, without hanging/breaking the guest.
411 */
412 return __tdx_hypercall(&args);
413 }
414
handle_halt(struct ve_info * ve)415 static int handle_halt(struct ve_info *ve)
416 {
417 const bool irq_disabled = irqs_disabled();
418
419 /*
420 * HLT with IRQs enabled is unsafe, as an IRQ that is intended to be a
421 * wake event may be consumed before requesting HLT emulation, leaving
422 * the vCPU blocking indefinitely.
423 */
424 if (WARN_ONCE(!irq_disabled, "HLT emulation with IRQs enabled"))
425 return -EIO;
426
427 if (__halt(irq_disabled))
428 return -EIO;
429
430 return ve_instr_len(ve);
431 }
432
tdx_halt(void)433 void __cpuidle tdx_halt(void)
434 {
435 const bool irq_disabled = false;
436
437 /*
438 * Use WARN_ONCE() to report the failure.
439 */
440 if (__halt(irq_disabled))
441 WARN_ONCE(1, "HLT instruction emulation failed\n");
442 }
443
tdx_safe_halt(void)444 static void __cpuidle tdx_safe_halt(void)
445 {
446 tdx_halt();
447 /*
448 * "__cpuidle" section doesn't support instrumentation, so stick
449 * with raw_* variant that avoids tracing hooks.
450 */
451 raw_local_irq_enable();
452 }
453
read_msr(struct pt_regs * regs,struct ve_info * ve)454 static int read_msr(struct pt_regs *regs, struct ve_info *ve)
455 {
456 struct tdx_module_args args = {
457 .r10 = TDX_HYPERCALL_STANDARD,
458 .r11 = hcall_func(EXIT_REASON_MSR_READ),
459 .r12 = regs->cx,
460 };
461
462 /*
463 * Emulate the MSR read via hypercall. More info about ABI
464 * can be found in TDX Guest-Host-Communication Interface
465 * (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>".
466 */
467 if (__tdx_hypercall(&args))
468 return -EIO;
469
470 regs->ax = lower_32_bits(args.r11);
471 regs->dx = upper_32_bits(args.r11);
472 return ve_instr_len(ve);
473 }
474
write_msr(struct pt_regs * regs,struct ve_info * ve)475 static int write_msr(struct pt_regs *regs, struct ve_info *ve)
476 {
477 struct tdx_module_args args = {
478 .r10 = TDX_HYPERCALL_STANDARD,
479 .r11 = hcall_func(EXIT_REASON_MSR_WRITE),
480 .r12 = regs->cx,
481 .r13 = (u64)regs->dx << 32 | regs->ax,
482 };
483
484 /*
485 * Emulate the MSR write via hypercall. More info about ABI
486 * can be found in TDX Guest-Host-Communication Interface
487 * (GHCI) section titled "TDG.VP.VMCALL<Instruction.WRMSR>".
488 */
489 if (__tdx_hypercall(&args))
490 return -EIO;
491
492 return ve_instr_len(ve);
493 }
494
handle_cpuid(struct pt_regs * regs,struct ve_info * ve)495 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
496 {
497 struct tdx_module_args args = {
498 .r10 = TDX_HYPERCALL_STANDARD,
499 .r11 = hcall_func(EXIT_REASON_CPUID),
500 .r12 = regs->ax,
501 .r13 = regs->cx,
502 };
503
504 /*
505 * Only allow VMM to control range reserved for hypervisor
506 * communication.
507 *
508 * Return all-zeros for any CPUID outside the range. It matches CPU
509 * behaviour for non-supported leaf.
510 */
511 if (regs->ax < 0x40000000 || regs->ax > 0x4FFFFFFF) {
512 regs->ax = regs->bx = regs->cx = regs->dx = 0;
513 return ve_instr_len(ve);
514 }
515
516 /*
517 * Emulate the CPUID instruction via a hypercall. More info about
518 * ABI can be found in TDX Guest-Host-Communication Interface
519 * (GHCI), section titled "VP.VMCALL<Instruction.CPUID>".
520 */
521 if (__tdx_hypercall(&args))
522 return -EIO;
523
524 /*
525 * As per TDX GHCI CPUID ABI, r12-r15 registers contain contents of
526 * EAX, EBX, ECX, EDX registers after the CPUID instruction execution.
527 * So copy the register contents back to pt_regs.
528 */
529 regs->ax = args.r12;
530 regs->bx = args.r13;
531 regs->cx = args.r14;
532 regs->dx = args.r15;
533
534 return ve_instr_len(ve);
535 }
536
mmio_read(int size,unsigned long addr,unsigned long * val)537 static bool mmio_read(int size, unsigned long addr, unsigned long *val)
538 {
539 struct tdx_module_args args = {
540 .r10 = TDX_HYPERCALL_STANDARD,
541 .r11 = hcall_func(EXIT_REASON_EPT_VIOLATION),
542 .r12 = size,
543 .r13 = EPT_READ,
544 .r14 = addr,
545 };
546
547 if (__tdx_hypercall(&args))
548 return false;
549
550 *val = args.r11;
551 return true;
552 }
553
mmio_write(int size,unsigned long addr,unsigned long val)554 static bool mmio_write(int size, unsigned long addr, unsigned long val)
555 {
556 return !_tdx_hypercall(hcall_func(EXIT_REASON_EPT_VIOLATION), size,
557 EPT_WRITE, addr, val);
558 }
559
handle_mmio(struct pt_regs * regs,struct ve_info * ve)560 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)
561 {
562 unsigned long *reg, val, vaddr;
563 char buffer[MAX_INSN_SIZE];
564 enum insn_mmio_type mmio;
565 struct insn insn = {};
566 int size, extend_size;
567 u8 extend_val = 0;
568
569 /* Only in-kernel MMIO is supported */
570 if (WARN_ON_ONCE(user_mode(regs)))
571 return -EFAULT;
572
573 if (copy_from_kernel_nofault(buffer, (void *)regs->ip, MAX_INSN_SIZE))
574 return -EFAULT;
575
576 if (insn_decode(&insn, buffer, MAX_INSN_SIZE, INSN_MODE_64))
577 return -EINVAL;
578
579 mmio = insn_decode_mmio(&insn, &size);
580 if (WARN_ON_ONCE(mmio == INSN_MMIO_DECODE_FAILED))
581 return -EINVAL;
582
583 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
584 reg = insn_get_modrm_reg_ptr(&insn, regs);
585 if (!reg)
586 return -EINVAL;
587 }
588
589 if (!fault_in_kernel_space(ve->gla)) {
590 WARN_ONCE(1, "Access to userspace address is not supported");
591 return -EINVAL;
592 }
593
594 /*
595 * Reject EPT violation #VEs that split pages.
596 *
597 * MMIO accesses are supposed to be naturally aligned and therefore
598 * never cross page boundaries. Seeing split page accesses indicates
599 * a bug or a load_unaligned_zeropad() that stepped into an MMIO page.
600 *
601 * load_unaligned_zeropad() will recover using exception fixups.
602 */
603 vaddr = (unsigned long)insn_get_addr_ref(&insn, regs);
604 if (vaddr / PAGE_SIZE != (vaddr + size - 1) / PAGE_SIZE)
605 return -EFAULT;
606
607 /* Handle writes first */
608 switch (mmio) {
609 case INSN_MMIO_WRITE:
610 memcpy(&val, reg, size);
611 if (!mmio_write(size, ve->gpa, val))
612 return -EIO;
613 return insn.length;
614 case INSN_MMIO_WRITE_IMM:
615 val = insn.immediate.value;
616 if (!mmio_write(size, ve->gpa, val))
617 return -EIO;
618 return insn.length;
619 case INSN_MMIO_READ:
620 case INSN_MMIO_READ_ZERO_EXTEND:
621 case INSN_MMIO_READ_SIGN_EXTEND:
622 /* Reads are handled below */
623 break;
624 case INSN_MMIO_MOVS:
625 case INSN_MMIO_DECODE_FAILED:
626 /*
627 * MMIO was accessed with an instruction that could not be
628 * decoded or handled properly. It was likely not using io.h
629 * helpers or accessed MMIO accidentally.
630 */
631 return -EINVAL;
632 default:
633 WARN_ONCE(1, "Unknown insn_decode_mmio() decode value?");
634 return -EINVAL;
635 }
636
637 /* Handle reads */
638 if (!mmio_read(size, ve->gpa, &val))
639 return -EIO;
640
641 switch (mmio) {
642 case INSN_MMIO_READ:
643 /* Zero-extend for 32-bit operation */
644 extend_size = size == 4 ? sizeof(*reg) : 0;
645 break;
646 case INSN_MMIO_READ_ZERO_EXTEND:
647 /* Zero extend based on operand size */
648 extend_size = insn.opnd_bytes;
649 break;
650 case INSN_MMIO_READ_SIGN_EXTEND:
651 /* Sign extend based on operand size */
652 extend_size = insn.opnd_bytes;
653 if (size == 1 && val & BIT(7))
654 extend_val = 0xFF;
655 else if (size > 1 && val & BIT(15))
656 extend_val = 0xFF;
657 break;
658 default:
659 /* All other cases has to be covered with the first switch() */
660 WARN_ON_ONCE(1);
661 return -EINVAL;
662 }
663
664 if (extend_size)
665 memset(reg, extend_val, extend_size);
666 memcpy(reg, &val, size);
667 return insn.length;
668 }
669
handle_in(struct pt_regs * regs,int size,int port)670 static bool handle_in(struct pt_regs *regs, int size, int port)
671 {
672 struct tdx_module_args args = {
673 .r10 = TDX_HYPERCALL_STANDARD,
674 .r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
675 .r12 = size,
676 .r13 = PORT_READ,
677 .r14 = port,
678 };
679 bool success;
680 u64 val;
681
682 /*
683 * Emulate the I/O read via hypercall. More info about ABI can be found
684 * in TDX Guest-Host-Communication Interface (GHCI) section titled
685 * "TDG.VP.VMCALL<Instruction.IO>".
686 */
687 success = !__tdx_hypercall(&args);
688 val = success ? args.r11 : 0;
689
690 insn_assign_reg(®s->ax, val, size);
691
692 return success;
693 }
694
handle_out(struct pt_regs * regs,int size,int port)695 static bool handle_out(struct pt_regs *regs, int size, int port)
696 {
697 u64 mask = GENMASK(BITS_PER_BYTE * size - 1, 0);
698
699 /*
700 * Emulate the I/O write via hypercall. More info about ABI can be found
701 * in TDX Guest-Host-Communication Interface (GHCI) section titled
702 * "TDG.VP.VMCALL<Instruction.IO>".
703 */
704 return !_tdx_hypercall(hcall_func(EXIT_REASON_IO_INSTRUCTION), size,
705 PORT_WRITE, port, regs->ax & mask);
706 }
707
708 /*
709 * Emulate I/O using hypercall.
710 *
711 * Assumes the IO instruction was using ax, which is enforced
712 * by the standard io.h macros.
713 *
714 * Return True on success or False on failure.
715 */
handle_io(struct pt_regs * regs,struct ve_info * ve)716 static int handle_io(struct pt_regs *regs, struct ve_info *ve)
717 {
718 u32 exit_qual = ve->exit_qual;
719 int size, port;
720 bool in, ret;
721
722 if (VE_IS_IO_STRING(exit_qual))
723 return -EIO;
724
725 in = VE_IS_IO_IN(exit_qual);
726 size = VE_GET_IO_SIZE(exit_qual);
727 port = VE_GET_PORT_NUM(exit_qual);
728
729
730 if (in)
731 ret = handle_in(regs, size, port);
732 else
733 ret = handle_out(regs, size, port);
734 if (!ret)
735 return -EIO;
736
737 return ve_instr_len(ve);
738 }
739
740 /*
741 * Early #VE exception handler. Only handles a subset of port I/O.
742 * Intended only for earlyprintk. If failed, return false.
743 */
tdx_early_handle_ve(struct pt_regs * regs)744 __init bool tdx_early_handle_ve(struct pt_regs *regs)
745 {
746 struct ve_info ve;
747 int insn_len;
748
749 tdx_get_ve_info(&ve);
750
751 if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION)
752 return false;
753
754 insn_len = handle_io(regs, &ve);
755 if (insn_len < 0)
756 return false;
757
758 regs->ip += insn_len;
759 return true;
760 }
761
tdx_get_ve_info(struct ve_info * ve)762 void tdx_get_ve_info(struct ve_info *ve)
763 {
764 struct tdx_module_args args = {};
765
766 /*
767 * Called during #VE handling to retrieve the #VE info from the
768 * TDX module.
769 *
770 * This has to be called early in #VE handling. A "nested" #VE which
771 * occurs before this will raise a #DF and is not recoverable.
772 *
773 * The call retrieves the #VE info from the TDX module, which also
774 * clears the "#VE valid" flag. This must be done before anything else
775 * because any #VE that occurs while the valid flag is set will lead to
776 * #DF.
777 *
778 * Note, the TDX module treats virtual NMIs as inhibited if the #VE
779 * valid flag is set. It means that NMI=>#VE will not result in a #DF.
780 */
781 tdcall(TDG_VP_VEINFO_GET, &args);
782
783 /* Transfer the output parameters */
784 ve->exit_reason = args.rcx;
785 ve->exit_qual = args.rdx;
786 ve->gla = args.r8;
787 ve->gpa = args.r9;
788 ve->instr_len = lower_32_bits(args.r10);
789 ve->instr_info = upper_32_bits(args.r10);
790 }
791
792 /*
793 * Handle the user initiated #VE.
794 *
795 * On success, returns the number of bytes RIP should be incremented (>=0)
796 * or -errno on error.
797 */
virt_exception_user(struct pt_regs * regs,struct ve_info * ve)798 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve)
799 {
800 switch (ve->exit_reason) {
801 case EXIT_REASON_CPUID:
802 return handle_cpuid(regs, ve);
803 default:
804 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);
805 return -EIO;
806 }
807 }
808
is_private_gpa(u64 gpa)809 static inline bool is_private_gpa(u64 gpa)
810 {
811 return gpa == cc_mkenc(gpa);
812 }
813
814 /*
815 * Handle the kernel #VE.
816 *
817 * On success, returns the number of bytes RIP should be incremented (>=0)
818 * or -errno on error.
819 */
virt_exception_kernel(struct pt_regs * regs,struct ve_info * ve)820 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve)
821 {
822 switch (ve->exit_reason) {
823 case EXIT_REASON_HLT:
824 return handle_halt(ve);
825 case EXIT_REASON_MSR_READ:
826 return read_msr(regs, ve);
827 case EXIT_REASON_MSR_WRITE:
828 return write_msr(regs, ve);
829 case EXIT_REASON_CPUID:
830 return handle_cpuid(regs, ve);
831 case EXIT_REASON_EPT_VIOLATION:
832 if (is_private_gpa(ve->gpa))
833 panic("Unexpected EPT-violation on private memory.");
834 return handle_mmio(regs, ve);
835 case EXIT_REASON_IO_INSTRUCTION:
836 return handle_io(regs, ve);
837 default:
838 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);
839 return -EIO;
840 }
841 }
842
tdx_handle_virt_exception(struct pt_regs * regs,struct ve_info * ve)843 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve)
844 {
845 int insn_len;
846
847 if (user_mode(regs))
848 insn_len = virt_exception_user(regs, ve);
849 else
850 insn_len = virt_exception_kernel(regs, ve);
851 if (insn_len < 0)
852 return false;
853
854 /* After successful #VE handling, move the IP */
855 regs->ip += insn_len;
856
857 return true;
858 }
859
tdx_tlb_flush_required(bool private)860 static bool tdx_tlb_flush_required(bool private)
861 {
862 /*
863 * TDX guest is responsible for flushing TLB on private->shared
864 * transition. VMM is responsible for flushing on shared->private.
865 *
866 * The VMM _can't_ flush private addresses as it can't generate PAs
867 * with the guest's HKID. Shared memory isn't subject to integrity
868 * checking, i.e. the VMM doesn't need to flush for its own protection.
869 *
870 * There's no need to flush when converting from shared to private,
871 * as flushing is the VMM's responsibility in this case, e.g. it must
872 * flush to avoid integrity failures in the face of a buggy or
873 * malicious guest.
874 */
875 return !private;
876 }
877
tdx_cache_flush_required(void)878 static bool tdx_cache_flush_required(void)
879 {
880 /*
881 * AMD SME/SEV can avoid cache flushing if HW enforces cache coherence.
882 * TDX doesn't have such capability.
883 *
884 * Flush cache unconditionally.
885 */
886 return true;
887 }
888
889 /*
890 * Notify the VMM about page mapping conversion. More info about ABI
891 * can be found in TDX Guest-Host-Communication Interface (GHCI),
892 * section "TDG.VP.VMCALL<MapGPA>".
893 */
tdx_map_gpa(phys_addr_t start,phys_addr_t end,bool enc)894 static bool tdx_map_gpa(phys_addr_t start, phys_addr_t end, bool enc)
895 {
896 /* Retrying the hypercall a second time should succeed; use 3 just in case */
897 const int max_retries_per_page = 3;
898 int retry_count = 0;
899
900 if (!enc) {
901 /* Set the shared (decrypted) bits: */
902 start |= cc_mkdec(0);
903 end |= cc_mkdec(0);
904 }
905
906 while (retry_count < max_retries_per_page) {
907 struct tdx_module_args args = {
908 .r10 = TDX_HYPERCALL_STANDARD,
909 .r11 = TDVMCALL_MAP_GPA,
910 .r12 = start,
911 .r13 = end - start };
912
913 u64 map_fail_paddr;
914 u64 ret = __tdx_hypercall(&args);
915
916 if (ret != TDVMCALL_STATUS_RETRY)
917 return !ret;
918 /*
919 * The guest must retry the operation for the pages in the
920 * region starting at the GPA specified in R11. R11 comes
921 * from the untrusted VMM. Sanity check it.
922 */
923 map_fail_paddr = args.r11;
924 if (map_fail_paddr < start || map_fail_paddr >= end)
925 return false;
926
927 /* "Consume" a retry without forward progress */
928 if (map_fail_paddr == start) {
929 retry_count++;
930 continue;
931 }
932
933 start = map_fail_paddr;
934 retry_count = 0;
935 }
936
937 return false;
938 }
939
940 /*
941 * Inform the VMM of the guest's intent for this physical page: shared with
942 * the VMM or private to the guest. The VMM is expected to change its mapping
943 * of the page in response.
944 */
tdx_enc_status_changed(unsigned long vaddr,int numpages,bool enc)945 static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)
946 {
947 phys_addr_t start = __pa(vaddr);
948 phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE);
949
950 if (!tdx_map_gpa(start, end, enc))
951 return false;
952
953 /* shared->private conversion requires memory to be accepted before use */
954 if (enc)
955 return tdx_accept_memory(start, end);
956
957 return true;
958 }
959
tdx_enc_status_change_prepare(unsigned long vaddr,int numpages,bool enc)960 static int tdx_enc_status_change_prepare(unsigned long vaddr, int numpages,
961 bool enc)
962 {
963 /*
964 * Only handle shared->private conversion here.
965 * See the comment in tdx_early_init().
966 */
967 if (enc && !tdx_enc_status_changed(vaddr, numpages, enc))
968 return -EIO;
969
970 return 0;
971 }
972
tdx_enc_status_change_finish(unsigned long vaddr,int numpages,bool enc)973 static int tdx_enc_status_change_finish(unsigned long vaddr, int numpages,
974 bool enc)
975 {
976 /*
977 * Only handle private->shared conversion here.
978 * See the comment in tdx_early_init().
979 */
980 if (!enc && !tdx_enc_status_changed(vaddr, numpages, enc))
981 return -EIO;
982
983 if (enc)
984 atomic_long_sub(numpages, &nr_shared);
985 else
986 atomic_long_add(numpages, &nr_shared);
987
988 return 0;
989 }
990
991 /* Stop new private<->shared conversions */
tdx_kexec_begin(void)992 static void tdx_kexec_begin(void)
993 {
994 if (!IS_ENABLED(CONFIG_KEXEC_CORE))
995 return;
996
997 /*
998 * Crash kernel reaches here with interrupts disabled: can't wait for
999 * conversions to finish.
1000 *
1001 * If race happened, just report and proceed.
1002 */
1003 if (!set_memory_enc_stop_conversion())
1004 pr_warn("Failed to stop shared<->private conversions\n");
1005 }
1006
1007 /* Walk direct mapping and convert all shared memory back to private */
tdx_kexec_finish(void)1008 static void tdx_kexec_finish(void)
1009 {
1010 unsigned long addr, end;
1011 long found = 0, shared;
1012
1013 if (!IS_ENABLED(CONFIG_KEXEC_CORE))
1014 return;
1015
1016 lockdep_assert_irqs_disabled();
1017
1018 addr = PAGE_OFFSET;
1019 end = PAGE_OFFSET + get_max_mapped();
1020
1021 while (addr < end) {
1022 unsigned long size;
1023 unsigned int level;
1024 pte_t *pte;
1025
1026 pte = lookup_address(addr, &level);
1027 size = page_level_size(level);
1028
1029 if (pte && pte_decrypted(*pte)) {
1030 int pages = size / PAGE_SIZE;
1031
1032 /*
1033 * Touching memory with shared bit set triggers implicit
1034 * conversion to shared.
1035 *
1036 * Make sure nobody touches the shared range from
1037 * now on.
1038 */
1039 set_pte(pte, __pte(0));
1040
1041 /*
1042 * Memory encryption state persists across kexec.
1043 * If tdx_enc_status_changed() fails in the first
1044 * kernel, it leaves memory in an unknown state.
1045 *
1046 * If that memory remains shared, accessing it in the
1047 * *next* kernel through a private mapping will result
1048 * in an unrecoverable guest shutdown.
1049 *
1050 * The kdump kernel boot is not impacted as it uses
1051 * a pre-reserved memory range that is always private.
1052 * However, gathering crash information could lead to
1053 * a crash if it accesses unconverted memory through
1054 * a private mapping which is possible when accessing
1055 * that memory through /proc/vmcore, for example.
1056 *
1057 * In all cases, print error info in order to leave
1058 * enough bread crumbs for debugging.
1059 */
1060 if (!tdx_enc_status_changed(addr, pages, true)) {
1061 pr_err("Failed to unshare range %#lx-%#lx\n",
1062 addr, addr + size);
1063 }
1064
1065 found += pages;
1066 }
1067
1068 addr += size;
1069 }
1070
1071 __flush_tlb_all();
1072
1073 shared = atomic_long_read(&nr_shared);
1074 if (shared != found) {
1075 pr_err("shared page accounting is off\n");
1076 pr_err("nr_shared = %ld, nr_found = %ld\n", shared, found);
1077 }
1078 }
1079
tdx_announce(void)1080 static __init void tdx_announce(void)
1081 {
1082 struct tdx_module_args args = {};
1083 u64 controls;
1084
1085 pr_info("Guest detected\n");
1086
1087 tdcall(TDG_VP_INFO, &args);
1088 tdx_dump_attributes(args.rdx);
1089
1090 tdg_vm_rd(TDCS_TD_CTLS, &controls);
1091 tdx_dump_td_ctls(controls);
1092 }
1093
tdx_early_init(void)1094 void __init tdx_early_init(void)
1095 {
1096 u64 cc_mask;
1097 u32 eax, sig[3];
1098
1099 cpuid_count(TDX_CPUID_LEAF_ID, 0, &eax, &sig[0], &sig[2], &sig[1]);
1100
1101 if (memcmp(TDX_IDENT, sig, sizeof(sig)))
1102 return;
1103
1104 setup_force_cpu_cap(X86_FEATURE_TDX_GUEST);
1105
1106 /* TSC is the only reliable clock in TDX guest */
1107 setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
1108
1109 cc_vendor = CC_VENDOR_INTEL;
1110
1111 /* Configure the TD */
1112 tdx_setup(&cc_mask);
1113
1114 cc_set_mask(cc_mask);
1115
1116 /*
1117 * All bits above GPA width are reserved and kernel treats shared bit
1118 * as flag, not as part of physical address.
1119 *
1120 * Adjust physical mask to only cover valid GPA bits.
1121 */
1122 physical_mask &= cc_mask - 1;
1123
1124 /*
1125 * The kernel mapping should match the TDX metadata for the page.
1126 * load_unaligned_zeropad() can touch memory *adjacent* to that which is
1127 * owned by the caller and can catch even _momentary_ mismatches. Bad
1128 * things happen on mismatch:
1129 *
1130 * - Private mapping => Shared Page == Guest shutdown
1131 * - Shared mapping => Private Page == Recoverable #VE
1132 *
1133 * guest.enc_status_change_prepare() converts the page from
1134 * shared=>private before the mapping becomes private.
1135 *
1136 * guest.enc_status_change_finish() converts the page from
1137 * private=>shared after the mapping becomes private.
1138 *
1139 * In both cases there is a temporary shared mapping to a private page,
1140 * which can result in a #VE. But, there is never a private mapping to
1141 * a shared page.
1142 */
1143 x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare;
1144 x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish;
1145
1146 x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required;
1147 x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;
1148
1149 x86_platform.guest.enc_kexec_begin = tdx_kexec_begin;
1150 x86_platform.guest.enc_kexec_finish = tdx_kexec_finish;
1151
1152 /*
1153 * Avoid "sti;hlt" execution in TDX guests as HLT induces a #VE that
1154 * will enable interrupts before HLT TDCALL invocation if executed
1155 * in STI-shadow, possibly resulting in missed wakeup events.
1156 *
1157 * Modify all possible HLT execution paths to use TDX specific routines
1158 * that directly execute TDCALL and toggle the interrupt state as
1159 * needed after TDCALL completion. This also reduces HLT related #VEs
1160 * in addition to having a reliable halt logic execution.
1161 */
1162 pv_ops.irq.safe_halt = tdx_safe_halt;
1163 pv_ops.irq.halt = tdx_halt;
1164
1165 /*
1166 * TDX intercepts the RDMSR to read the X2APIC ID in the parallel
1167 * bringup low level code. That raises #VE which cannot be handled
1168 * there.
1169 *
1170 * Intel-TDX has a secure RDMSR hypercall, but that needs to be
1171 * implemented separately in the low level startup ASM code.
1172 * Until that is in place, disable parallel bringup for TDX.
1173 */
1174 x86_cpuinit.parallel_bringup = false;
1175
1176 tdx_announce();
1177 }
1178