1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Secure AVIC Support (SEV-SNP Guests) 4 * 5 * Copyright (C) 2024 Advanced Micro Devices, Inc. 6 * 7 * Author: Neeraj Upadhyay <Neeraj.Upadhyay@amd.com> 8 */ 9 10 #include <linux/cc_platform.h> 11 #include <linux/cpumask.h> 12 #include <linux/percpu-defs.h> 13 #include <linux/align.h> 14 15 #include <asm/apic.h> 16 #include <asm/sev.h> 17 18 #include "local.h" 19 20 struct secure_avic_page { 21 u8 regs[PAGE_SIZE]; 22 } __aligned(PAGE_SIZE); 23 24 static struct secure_avic_page __percpu *savic_page __ro_after_init; 25 26 static int savic_acpi_madt_oem_check(char *oem_id, char *oem_table_id) 27 { 28 return x2apic_enabled() && cc_platform_has(CC_ATTR_SNP_SECURE_AVIC); 29 } 30 31 static inline void *get_reg_bitmap(unsigned int cpu, unsigned int offset) 32 { 33 return &per_cpu_ptr(savic_page, cpu)->regs[offset]; 34 } 35 36 static inline void update_vector(unsigned int cpu, unsigned int offset, 37 unsigned int vector, bool set) 38 { 39 void *bitmap = get_reg_bitmap(cpu, offset); 40 41 if (set) 42 apic_set_vector(vector, bitmap); 43 else 44 apic_clear_vector(vector, bitmap); 45 } 46 47 #define SAVIC_ALLOWED_IRR 0x204 48 49 /* 50 * When Secure AVIC is enabled, RDMSR/WRMSR of the APIC registers 51 * result in #VC exception (for non-accelerated register accesses) 52 * with VMEXIT_AVIC_NOACCEL error code. The #VC exception handler 53 * can read/write the x2APIC register in the guest APIC backing page. 54 * 55 * Since doing this would increase the latency of accessing x2APIC 56 * registers, instead of doing RDMSR/WRMSR based accesses and 57 * handling the APIC register reads/writes in the #VC exception handler, 58 * the read() and write() callbacks directly read/write the APIC register 59 * from/to the vCPU's APIC backing page. 60 */ 61 static u32 savic_read(u32 reg) 62 { 63 void *ap = this_cpu_ptr(savic_page); 64 65 switch (reg) { 66 case APIC_LVTT: 67 case APIC_TMICT: 68 case APIC_TMCCT: 69 case APIC_TDCR: 70 case APIC_LVTTHMR: 71 case APIC_LVTPC: 72 case APIC_LVT0: 73 case APIC_LVT1: 74 case APIC_LVTERR: 75 return savic_ghcb_msr_read(reg); 76 case APIC_ID: 77 case APIC_LVR: 78 case APIC_TASKPRI: 79 case APIC_ARBPRI: 80 case APIC_PROCPRI: 81 case APIC_LDR: 82 case APIC_SPIV: 83 case APIC_ESR: 84 case APIC_EFEAT: 85 case APIC_ECTRL: 86 case APIC_SEOI: 87 case APIC_IER: 88 case APIC_EILVTn(0) ... APIC_EILVTn(3): 89 return apic_get_reg(ap, reg); 90 case APIC_ICR: 91 return (u32)apic_get_reg64(ap, reg); 92 case APIC_ISR ... APIC_ISR + 0x70: 93 case APIC_TMR ... APIC_TMR + 0x70: 94 if (WARN_ONCE(!IS_ALIGNED(reg, 16), 95 "APIC register read offset 0x%x not aligned at 16 bytes", reg)) 96 return 0; 97 return apic_get_reg(ap, reg); 98 /* IRR and ALLOWED_IRR offset range */ 99 case APIC_IRR ... APIC_IRR + 0x74: 100 /* 101 * Valid APIC_IRR/SAVIC_ALLOWED_IRR registers are at 16 bytes strides from 102 * their respective base offset. APIC_IRRs are in the range 103 * 104 * (0x200, 0x210, ..., 0x270) 105 * 106 * while the SAVIC_ALLOWED_IRR range starts 4 bytes later, in the range 107 * 108 * (0x204, 0x214, ..., 0x274). 109 * 110 * Filter out everything else. 111 */ 112 if (WARN_ONCE(!(IS_ALIGNED(reg, 16) || 113 IS_ALIGNED(reg - 4, 16)), 114 "Misaligned APIC_IRR/ALLOWED_IRR APIC register read offset 0x%x", reg)) 115 return 0; 116 return apic_get_reg(ap, reg); 117 default: 118 pr_err("Error reading unknown Secure AVIC reg offset 0x%x\n", reg); 119 return 0; 120 } 121 } 122 123 #define SAVIC_NMI_REQ 0x278 124 125 /* 126 * On WRMSR to APIC_SELF_IPI register by the guest, Secure AVIC hardware 127 * updates the APIC_IRR in the APIC backing page of the vCPU. In addition, 128 * hardware evaluates the new APIC_IRR update for interrupt injection to 129 * the vCPU. So, self IPIs are hardware-accelerated. 130 */ 131 static inline void self_ipi_reg_write(unsigned int vector) 132 { 133 native_apic_msr_write(APIC_SELF_IPI, vector); 134 } 135 136 static void send_ipi_dest(unsigned int cpu, unsigned int vector, bool nmi) 137 { 138 if (nmi) 139 apic_set_reg(per_cpu_ptr(savic_page, cpu), SAVIC_NMI_REQ, 1); 140 else 141 update_vector(cpu, APIC_IRR, vector, true); 142 } 143 144 static void send_ipi_allbut(unsigned int vector, bool nmi) 145 { 146 unsigned int cpu, src_cpu; 147 148 guard(irqsave)(); 149 150 src_cpu = raw_smp_processor_id(); 151 152 for_each_cpu(cpu, cpu_online_mask) { 153 if (cpu == src_cpu) 154 continue; 155 send_ipi_dest(cpu, vector, nmi); 156 } 157 } 158 159 static inline void self_ipi(unsigned int vector, bool nmi) 160 { 161 u32 icr_low = APIC_SELF_IPI | vector; 162 163 if (nmi) 164 icr_low |= APIC_DM_NMI; 165 166 native_x2apic_icr_write(icr_low, 0); 167 } 168 169 static void savic_icr_write(u32 icr_low, u32 icr_high) 170 { 171 unsigned int dsh, vector; 172 u64 icr_data; 173 bool nmi; 174 175 dsh = icr_low & APIC_DEST_ALLBUT; 176 vector = icr_low & APIC_VECTOR_MASK; 177 nmi = ((icr_low & APIC_DM_FIXED_MASK) == APIC_DM_NMI); 178 179 switch (dsh) { 180 case APIC_DEST_SELF: 181 self_ipi(vector, nmi); 182 break; 183 case APIC_DEST_ALLINC: 184 self_ipi(vector, nmi); 185 fallthrough; 186 case APIC_DEST_ALLBUT: 187 send_ipi_allbut(vector, nmi); 188 break; 189 default: 190 send_ipi_dest(icr_high, vector, nmi); 191 break; 192 } 193 194 icr_data = ((u64)icr_high) << 32 | icr_low; 195 if (dsh != APIC_DEST_SELF) 196 savic_ghcb_msr_write(APIC_ICR, icr_data); 197 apic_set_reg64(this_cpu_ptr(savic_page), APIC_ICR, icr_data); 198 } 199 200 static void savic_write(u32 reg, u32 data) 201 { 202 void *ap = this_cpu_ptr(savic_page); 203 204 switch (reg) { 205 case APIC_LVTT: 206 case APIC_TMICT: 207 case APIC_TDCR: 208 case APIC_LVT0: 209 case APIC_LVT1: 210 case APIC_LVTTHMR: 211 case APIC_LVTPC: 212 case APIC_LVTERR: 213 savic_ghcb_msr_write(reg, data); 214 break; 215 case APIC_TASKPRI: 216 case APIC_EOI: 217 case APIC_SPIV: 218 case SAVIC_NMI_REQ: 219 case APIC_ESR: 220 case APIC_ECTRL: 221 case APIC_SEOI: 222 case APIC_IER: 223 case APIC_EILVTn(0) ... APIC_EILVTn(3): 224 apic_set_reg(ap, reg, data); 225 break; 226 case APIC_ICR: 227 savic_icr_write(data, 0); 228 break; 229 case APIC_SELF_IPI: 230 self_ipi_reg_write(data); 231 break; 232 /* ALLOWED_IRR offsets are writable */ 233 case SAVIC_ALLOWED_IRR ... SAVIC_ALLOWED_IRR + 0x70: 234 if (IS_ALIGNED(reg - 4, 16)) { 235 apic_set_reg(ap, reg, data); 236 break; 237 } 238 fallthrough; 239 default: 240 pr_err("Error writing unknown Secure AVIC reg offset 0x%x\n", reg); 241 } 242 } 243 244 static void send_ipi(u32 dest, unsigned int vector, unsigned int dsh) 245 { 246 unsigned int icr_low; 247 248 icr_low = __prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL); 249 savic_icr_write(icr_low, dest); 250 } 251 252 static void savic_send_ipi(int cpu, int vector) 253 { 254 u32 dest = per_cpu(x86_cpu_to_apicid, cpu); 255 256 send_ipi(dest, vector, 0); 257 } 258 259 static void send_ipi_mask(const struct cpumask *mask, unsigned int vector, bool excl_self) 260 { 261 unsigned int cpu, this_cpu; 262 263 guard(irqsave)(); 264 265 this_cpu = raw_smp_processor_id(); 266 267 for_each_cpu(cpu, mask) { 268 if (excl_self && cpu == this_cpu) 269 continue; 270 send_ipi(per_cpu(x86_cpu_to_apicid, cpu), vector, 0); 271 } 272 } 273 274 static void savic_send_ipi_mask(const struct cpumask *mask, int vector) 275 { 276 send_ipi_mask(mask, vector, false); 277 } 278 279 static void savic_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) 280 { 281 send_ipi_mask(mask, vector, true); 282 } 283 284 static void savic_send_ipi_allbutself(int vector) 285 { 286 send_ipi(0, vector, APIC_DEST_ALLBUT); 287 } 288 289 static void savic_send_ipi_all(int vector) 290 { 291 send_ipi(0, vector, APIC_DEST_ALLINC); 292 } 293 294 static void savic_send_ipi_self(int vector) 295 { 296 self_ipi_reg_write(vector); 297 } 298 299 static void savic_update_vector(unsigned int cpu, unsigned int vector, bool set) 300 { 301 update_vector(cpu, SAVIC_ALLOWED_IRR, vector, set); 302 } 303 304 static void savic_eoi(void) 305 { 306 unsigned int cpu; 307 int vec; 308 309 cpu = raw_smp_processor_id(); 310 vec = apic_find_highest_vector(get_reg_bitmap(cpu, APIC_ISR)); 311 if (WARN_ONCE(vec == -1, "EOI write while no active interrupt in APIC_ISR")) 312 return; 313 314 /* Is level-triggered interrupt? */ 315 if (apic_test_vector(vec, get_reg_bitmap(cpu, APIC_TMR))) { 316 update_vector(cpu, APIC_ISR, vec, false); 317 /* 318 * Propagate the EOI write to the hypervisor for level-triggered 319 * interrupts. Return to the guest from GHCB protocol event takes 320 * care of re-evaluating interrupt state. 321 */ 322 savic_ghcb_msr_write(APIC_EOI, 0); 323 } else { 324 /* 325 * Hardware clears APIC_ISR and re-evaluates the interrupt state 326 * to determine if there is any pending interrupt which can be 327 * delivered to CPU. 328 */ 329 native_apic_msr_eoi(); 330 } 331 } 332 333 static void savic_teardown(void) 334 { 335 /* Disable Secure AVIC */ 336 native_wrmsrq(MSR_AMD64_SAVIC_CONTROL, 0); 337 savic_unregister_gpa(NULL); 338 } 339 340 static void savic_setup(void) 341 { 342 void *ap = this_cpu_ptr(savic_page); 343 enum es_result res; 344 unsigned long gpa; 345 346 /* 347 * Before Secure AVIC is enabled, APIC MSR reads are intercepted. 348 * APIC_ID MSR read returns the value from the hypervisor. 349 */ 350 apic_set_reg(ap, APIC_ID, native_apic_msr_read(APIC_ID)); 351 352 gpa = __pa(ap); 353 354 /* 355 * The NPT entry for a vCPU's APIC backing page must always be 356 * present when the vCPU is running in order for Secure AVIC to 357 * function. A VMEXIT_BUSY is returned on VMRUN and the vCPU cannot 358 * be resumed if the NPT entry for the APIC backing page is not 359 * present. Notify GPA of the vCPU's APIC backing page to the 360 * hypervisor by calling savic_register_gpa(). Before executing 361 * VMRUN, the hypervisor makes use of this information to make sure 362 * the APIC backing page is mapped in NPT. 363 */ 364 res = savic_register_gpa(gpa); 365 if (res != ES_OK) 366 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 367 368 native_wrmsrq(MSR_AMD64_SAVIC_CONTROL, 369 gpa | MSR_AMD64_SAVIC_EN | MSR_AMD64_SAVIC_ALLOWEDNMI); 370 } 371 372 static int savic_probe(void) 373 { 374 if (!cc_platform_has(CC_ATTR_SNP_SECURE_AVIC)) 375 return 0; 376 377 if (!x2apic_mode) { 378 pr_err("Secure AVIC enabled in non x2APIC mode\n"); 379 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 380 /* unreachable */ 381 } 382 383 savic_page = alloc_percpu(struct secure_avic_page); 384 if (!savic_page) 385 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 386 387 return 1; 388 } 389 390 static struct apic apic_x2apic_savic __ro_after_init = { 391 392 .name = "secure avic x2apic", 393 .probe = savic_probe, 394 .acpi_madt_oem_check = savic_acpi_madt_oem_check, 395 .setup = savic_setup, 396 .teardown = savic_teardown, 397 398 .dest_mode_logical = false, 399 400 .disable_esr = 0, 401 402 .cpu_present_to_apicid = default_cpu_present_to_apicid, 403 404 .max_apic_id = UINT_MAX, 405 .x2apic_set_max_apicid = true, 406 .get_apic_id = x2apic_get_apic_id, 407 408 .calc_dest_apicid = apic_default_calc_apicid, 409 410 .send_IPI = savic_send_ipi, 411 .send_IPI_mask = savic_send_ipi_mask, 412 .send_IPI_mask_allbutself = savic_send_ipi_mask_allbutself, 413 .send_IPI_allbutself = savic_send_ipi_allbutself, 414 .send_IPI_all = savic_send_ipi_all, 415 .send_IPI_self = savic_send_ipi_self, 416 417 .nmi_to_offline_cpu = true, 418 419 .read = savic_read, 420 .write = savic_write, 421 .eoi = savic_eoi, 422 .icr_read = native_x2apic_icr_read, 423 .icr_write = savic_icr_write, 424 425 .update_vector = savic_update_vector, 426 }; 427 428 apic_driver(apic_x2apic_savic); 429