1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) "SVM: " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/hashtable.h> 19 #include <linux/amd-iommu.h> 20 #include <linux/kvm_host.h> 21 22 #include <asm/irq_remapping.h> 23 24 #include "trace.h" 25 #include "lapic.h" 26 #include "x86.h" 27 #include "irq.h" 28 #include "svm.h" 29 30 /* AVIC GATAG is encoded using VM and VCPU IDs */ 31 #define AVIC_VCPU_ID_BITS 8 32 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) 33 34 #define AVIC_VM_ID_BITS 24 35 #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) 36 #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) 37 38 #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ 39 (y & AVIC_VCPU_ID_MASK)) 40 #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) 41 #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 42 43 static bool force_avic; 44 module_param_unsafe(force_avic, bool, 0444); 45 46 /* Note: 47 * This hash table is used to map VM_ID to a struct kvm_svm, 48 * when handling AMD IOMMU GALOG notification to schedule in 49 * a particular vCPU. 50 */ 51 #define SVM_VM_DATA_HASH_BITS 8 52 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); 53 static u32 next_vm_id = 0; 54 static bool next_vm_id_wrapped = 0; 55 static DEFINE_SPINLOCK(svm_vm_data_hash_lock); 56 enum avic_modes avic_mode; 57 58 /* 59 * This is a wrapper of struct amd_iommu_ir_data. 60 */ 61 struct amd_svm_iommu_ir { 62 struct list_head node; /* Used by SVM for per-vcpu ir_list */ 63 void *data; /* Storing pointer to struct amd_ir_data */ 64 }; 65 66 static void avic_activate_vmcb(struct vcpu_svm *svm) 67 { 68 struct vmcb *vmcb = svm->vmcb01.ptr; 69 70 vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); 71 vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; 72 73 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 74 75 /* Note: 76 * KVM can support hybrid-AVIC mode, where KVM emulates x2APIC 77 * MSR accesses, while interrupt injection to a running vCPU 78 * can be achieved using AVIC doorbell. The AVIC hardware still 79 * accelerate MMIO accesses, but this does not cause any harm 80 * as the guest is not supposed to access xAPIC mmio when uses x2APIC. 81 */ 82 if (apic_x2apic_mode(svm->vcpu.arch.apic) && 83 avic_mode == AVIC_MODE_X2) { 84 vmcb->control.int_ctl |= X2APIC_MODE_MASK; 85 vmcb->control.avic_physical_id |= X2AVIC_MAX_PHYSICAL_ID; 86 /* Disabling MSR intercept for x2APIC registers */ 87 svm_set_x2apic_msr_interception(svm, false); 88 } else { 89 /* For xAVIC and hybrid-xAVIC modes */ 90 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID; 91 /* Enabling MSR intercept for x2APIC registers */ 92 svm_set_x2apic_msr_interception(svm, true); 93 } 94 } 95 96 static void avic_deactivate_vmcb(struct vcpu_svm *svm) 97 { 98 struct vmcb *vmcb = svm->vmcb01.ptr; 99 100 vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); 101 vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; 102 103 /* 104 * If running nested and the guest uses its own MSR bitmap, there 105 * is no need to update L0's msr bitmap 106 */ 107 if (is_guest_mode(&svm->vcpu) && 108 vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)) 109 return; 110 111 /* Enabling MSR intercept for x2APIC registers */ 112 svm_set_x2apic_msr_interception(svm, true); 113 } 114 115 /* Note: 116 * This function is called from IOMMU driver to notify 117 * SVM to schedule in a particular vCPU of a particular VM. 118 */ 119 int avic_ga_log_notifier(u32 ga_tag) 120 { 121 unsigned long flags; 122 struct kvm_svm *kvm_svm; 123 struct kvm_vcpu *vcpu = NULL; 124 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); 125 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); 126 127 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); 128 trace_kvm_avic_ga_log(vm_id, vcpu_id); 129 130 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 131 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { 132 if (kvm_svm->avic_vm_id != vm_id) 133 continue; 134 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); 135 break; 136 } 137 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 138 139 /* Note: 140 * At this point, the IOMMU should have already set the pending 141 * bit in the vAPIC backing page. So, we just need to schedule 142 * in the vcpu. 143 */ 144 if (vcpu) 145 kvm_vcpu_wake_up(vcpu); 146 147 return 0; 148 } 149 150 void avic_vm_destroy(struct kvm *kvm) 151 { 152 unsigned long flags; 153 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 154 155 if (!enable_apicv) 156 return; 157 158 if (kvm_svm->avic_logical_id_table_page) 159 __free_page(kvm_svm->avic_logical_id_table_page); 160 if (kvm_svm->avic_physical_id_table_page) 161 __free_page(kvm_svm->avic_physical_id_table_page); 162 163 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 164 hash_del(&kvm_svm->hnode); 165 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 166 } 167 168 int avic_vm_init(struct kvm *kvm) 169 { 170 unsigned long flags; 171 int err = -ENOMEM; 172 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 173 struct kvm_svm *k2; 174 struct page *p_page; 175 struct page *l_page; 176 u32 vm_id; 177 178 if (!enable_apicv) 179 return 0; 180 181 /* Allocating physical APIC ID table (4KB) */ 182 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 183 if (!p_page) 184 goto free_avic; 185 186 kvm_svm->avic_physical_id_table_page = p_page; 187 188 /* Allocating logical APIC ID table (4KB) */ 189 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 190 if (!l_page) 191 goto free_avic; 192 193 kvm_svm->avic_logical_id_table_page = l_page; 194 195 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 196 again: 197 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; 198 if (vm_id == 0) { /* id is 1-based, zero is not okay */ 199 next_vm_id_wrapped = 1; 200 goto again; 201 } 202 /* Is it still in use? Only possible if wrapped at least once */ 203 if (next_vm_id_wrapped) { 204 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { 205 if (k2->avic_vm_id == vm_id) 206 goto again; 207 } 208 } 209 kvm_svm->avic_vm_id = vm_id; 210 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); 211 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 212 213 return 0; 214 215 free_avic: 216 avic_vm_destroy(kvm); 217 return err; 218 } 219 220 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb) 221 { 222 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); 223 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); 224 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); 225 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); 226 227 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; 228 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; 229 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; 230 vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK; 231 232 if (kvm_apicv_activated(svm->vcpu.kvm)) 233 avic_activate_vmcb(svm); 234 else 235 avic_deactivate_vmcb(svm); 236 } 237 238 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, 239 unsigned int index) 240 { 241 u64 *avic_physical_id_table; 242 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 243 244 if ((avic_mode == AVIC_MODE_X1 && index > AVIC_MAX_PHYSICAL_ID) || 245 (avic_mode == AVIC_MODE_X2 && index > X2AVIC_MAX_PHYSICAL_ID)) 246 return NULL; 247 248 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); 249 250 return &avic_physical_id_table[index]; 251 } 252 253 /* 254 * Note: 255 * AVIC hardware walks the nested page table to check permissions, 256 * but does not use the SPA address specified in the leaf page 257 * table entry since it uses address in the AVIC_BACKING_PAGE pointer 258 * field of the VMCB. Therefore, we set up the 259 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. 260 */ 261 static int avic_alloc_access_page(struct kvm *kvm) 262 { 263 void __user *ret; 264 int r = 0; 265 266 mutex_lock(&kvm->slots_lock); 267 268 if (kvm->arch.apic_access_memslot_enabled) 269 goto out; 270 271 ret = __x86_set_memory_region(kvm, 272 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 273 APIC_DEFAULT_PHYS_BASE, 274 PAGE_SIZE); 275 if (IS_ERR(ret)) { 276 r = PTR_ERR(ret); 277 goto out; 278 } 279 280 kvm->arch.apic_access_memslot_enabled = true; 281 out: 282 mutex_unlock(&kvm->slots_lock); 283 return r; 284 } 285 286 static int avic_init_backing_page(struct kvm_vcpu *vcpu) 287 { 288 u64 *entry, new_entry; 289 int id = vcpu->vcpu_id; 290 struct vcpu_svm *svm = to_svm(vcpu); 291 292 if ((avic_mode == AVIC_MODE_X1 && id > AVIC_MAX_PHYSICAL_ID) || 293 (avic_mode == AVIC_MODE_X2 && id > X2AVIC_MAX_PHYSICAL_ID)) 294 return -EINVAL; 295 296 if (!vcpu->arch.apic->regs) 297 return -EINVAL; 298 299 if (kvm_apicv_activated(vcpu->kvm)) { 300 int ret; 301 302 ret = avic_alloc_access_page(vcpu->kvm); 303 if (ret) 304 return ret; 305 } 306 307 svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs); 308 309 /* Setting AVIC backing page address in the phy APIC ID table */ 310 entry = avic_get_physical_id_entry(vcpu, id); 311 if (!entry) 312 return -EINVAL; 313 314 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & 315 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | 316 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); 317 WRITE_ONCE(*entry, new_entry); 318 319 svm->avic_physical_id_cache = entry; 320 321 return 0; 322 } 323 324 void avic_ring_doorbell(struct kvm_vcpu *vcpu) 325 { 326 /* 327 * Note, the vCPU could get migrated to a different pCPU at any point, 328 * which could result in signalling the wrong/previous pCPU. But if 329 * that happens the vCPU is guaranteed to do a VMRUN (after being 330 * migrated) and thus will process pending interrupts, i.e. a doorbell 331 * is not needed (and the spurious one is harmless). 332 */ 333 int cpu = READ_ONCE(vcpu->cpu); 334 335 if (cpu != get_cpu()) { 336 wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); 337 trace_kvm_avic_doorbell(vcpu->vcpu_id, kvm_cpu_get_apicid(cpu)); 338 } 339 put_cpu(); 340 } 341 342 /* 343 * A fast-path version of avic_kick_target_vcpus(), which attempts to match 344 * destination APIC ID to vCPU without looping through all vCPUs. 345 */ 346 static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source, 347 u32 icrl, u32 icrh, u32 index) 348 { 349 u32 l1_physical_id, dest; 350 struct kvm_vcpu *target_vcpu; 351 int dest_mode = icrl & APIC_DEST_MASK; 352 int shorthand = icrl & APIC_SHORT_MASK; 353 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 354 355 if (shorthand != APIC_DEST_NOSHORT) 356 return -EINVAL; 357 358 if (apic_x2apic_mode(source)) 359 dest = icrh; 360 else 361 dest = GET_XAPIC_DEST_FIELD(icrh); 362 363 if (dest_mode == APIC_DEST_PHYSICAL) { 364 /* broadcast destination, use slow path */ 365 if (apic_x2apic_mode(source) && dest == X2APIC_BROADCAST) 366 return -EINVAL; 367 if (!apic_x2apic_mode(source) && dest == APIC_BROADCAST) 368 return -EINVAL; 369 370 l1_physical_id = dest; 371 372 if (WARN_ON_ONCE(l1_physical_id != index)) 373 return -EINVAL; 374 375 } else { 376 u32 bitmap, cluster; 377 int logid_index; 378 379 if (apic_x2apic_mode(source)) { 380 /* 16 bit dest mask, 16 bit cluster id */ 381 bitmap = dest & 0xFFFF0000; 382 cluster = (dest >> 16) << 4; 383 } else if (kvm_lapic_get_reg(source, APIC_DFR) == APIC_DFR_FLAT) { 384 /* 8 bit dest mask*/ 385 bitmap = dest; 386 cluster = 0; 387 } else { 388 /* 4 bit desk mask, 4 bit cluster id */ 389 bitmap = dest & 0xF; 390 cluster = (dest >> 4) << 2; 391 } 392 393 if (unlikely(!bitmap)) 394 /* guest bug: nobody to send the logical interrupt to */ 395 return 0; 396 397 if (!is_power_of_2(bitmap)) 398 /* multiple logical destinations, use slow path */ 399 return -EINVAL; 400 401 logid_index = cluster + __ffs(bitmap); 402 403 if (!apic_x2apic_mode(source)) { 404 u32 *avic_logical_id_table = 405 page_address(kvm_svm->avic_logical_id_table_page); 406 407 u32 logid_entry = avic_logical_id_table[logid_index]; 408 409 if (WARN_ON_ONCE(index != logid_index)) 410 return -EINVAL; 411 412 /* guest bug: non existing/reserved logical destination */ 413 if (unlikely(!(logid_entry & AVIC_LOGICAL_ID_ENTRY_VALID_MASK))) 414 return 0; 415 416 l1_physical_id = logid_entry & 417 AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 418 } else { 419 /* 420 * For x2APIC logical mode, cannot leverage the index. 421 * Instead, calculate physical ID from logical ID in ICRH. 422 */ 423 int cluster = (icrh & 0xffff0000) >> 16; 424 int apic = ffs(icrh & 0xffff) - 1; 425 426 /* 427 * If the x2APIC logical ID sub-field (i.e. icrh[15:0]) 428 * contains anything but a single bit, we cannot use the 429 * fast path, because it is limited to a single vCPU. 430 */ 431 if (apic < 0 || icrh != (1 << apic)) 432 return -EINVAL; 433 434 l1_physical_id = (cluster << 4) + apic; 435 } 436 } 437 438 target_vcpu = kvm_get_vcpu_by_id(kvm, l1_physical_id); 439 if (unlikely(!target_vcpu)) 440 /* guest bug: non existing vCPU is a target of this IPI*/ 441 return 0; 442 443 target_vcpu->arch.apic->irr_pending = true; 444 svm_complete_interrupt_delivery(target_vcpu, 445 icrl & APIC_MODE_MASK, 446 icrl & APIC_INT_LEVELTRIG, 447 icrl & APIC_VECTOR_MASK); 448 return 0; 449 } 450 451 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, 452 u32 icrl, u32 icrh, u32 index) 453 { 454 unsigned long i; 455 struct kvm_vcpu *vcpu; 456 457 if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index)) 458 return; 459 460 trace_kvm_avic_kick_vcpu_slowpath(icrh, icrl, index); 461 462 /* 463 * Wake any target vCPUs that are blocking, i.e. waiting for a wake 464 * event. There's no need to signal doorbells, as hardware has handled 465 * vCPUs that were in guest at the time of the IPI, and vCPUs that have 466 * since entered the guest will have processed pending IRQs at VMRUN. 467 */ 468 kvm_for_each_vcpu(i, vcpu, kvm) { 469 u32 dest; 470 471 if (apic_x2apic_mode(vcpu->arch.apic)) 472 dest = icrh; 473 else 474 dest = GET_XAPIC_DEST_FIELD(icrh); 475 476 if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK, 477 dest, icrl & APIC_DEST_MASK)) { 478 vcpu->arch.apic->irr_pending = true; 479 svm_complete_interrupt_delivery(vcpu, 480 icrl & APIC_MODE_MASK, 481 icrl & APIC_INT_LEVELTRIG, 482 icrl & APIC_VECTOR_MASK); 483 } 484 } 485 } 486 487 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) 488 { 489 struct vcpu_svm *svm = to_svm(vcpu); 490 u32 icrh = svm->vmcb->control.exit_info_1 >> 32; 491 u32 icrl = svm->vmcb->control.exit_info_1; 492 u32 id = svm->vmcb->control.exit_info_2 >> 32; 493 u32 index = svm->vmcb->control.exit_info_2 & 0x1FF; 494 struct kvm_lapic *apic = vcpu->arch.apic; 495 496 trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index); 497 498 switch (id) { 499 case AVIC_IPI_FAILURE_INVALID_INT_TYPE: 500 /* 501 * Emulate IPIs that are not handled by AVIC hardware, which 502 * only virtualizes Fixed, Edge-Triggered INTRs. The exit is 503 * a trap, e.g. ICR holds the correct value and RIP has been 504 * advanced, KVM is responsible only for emulating the IPI. 505 * Sadly, hardware may sometimes leave the BUSY flag set, in 506 * which case KVM needs to emulate the ICR write as well in 507 * order to clear the BUSY flag. 508 */ 509 if (icrl & APIC_ICR_BUSY) 510 kvm_apic_write_nodecode(vcpu, APIC_ICR); 511 else 512 kvm_apic_send_ipi(apic, icrl, icrh); 513 break; 514 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: 515 /* 516 * At this point, we expect that the AVIC HW has already 517 * set the appropriate IRR bits on the valid target 518 * vcpus. So, we just need to kick the appropriate vcpu. 519 */ 520 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index); 521 break; 522 case AVIC_IPI_FAILURE_INVALID_TARGET: 523 break; 524 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: 525 WARN_ONCE(1, "Invalid backing page\n"); 526 break; 527 default: 528 pr_err("Unknown IPI interception\n"); 529 } 530 531 return 1; 532 } 533 534 unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu) 535 { 536 if (is_guest_mode(vcpu)) 537 return APICV_INHIBIT_REASON_NESTED; 538 return 0; 539 } 540 541 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) 542 { 543 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 544 int index; 545 u32 *logical_apic_id_table; 546 int dlid = GET_APIC_LOGICAL_ID(ldr); 547 548 if (!dlid) 549 return NULL; 550 551 if (flat) { /* flat */ 552 index = ffs(dlid) - 1; 553 if (index > 7) 554 return NULL; 555 } else { /* cluster */ 556 int cluster = (dlid & 0xf0) >> 4; 557 int apic = ffs(dlid & 0x0f) - 1; 558 559 if ((apic < 0) || (apic > 7) || 560 (cluster >= 0xf)) 561 return NULL; 562 index = (cluster << 2) + apic; 563 } 564 565 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); 566 567 return &logical_apic_id_table[index]; 568 } 569 570 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) 571 { 572 bool flat; 573 u32 *entry, new_entry; 574 575 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; 576 entry = avic_get_logical_id_entry(vcpu, ldr, flat); 577 if (!entry) 578 return -EINVAL; 579 580 new_entry = READ_ONCE(*entry); 581 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 582 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); 583 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; 584 WRITE_ONCE(*entry, new_entry); 585 586 return 0; 587 } 588 589 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) 590 { 591 struct vcpu_svm *svm = to_svm(vcpu); 592 bool flat = svm->dfr_reg == APIC_DFR_FLAT; 593 u32 *entry; 594 595 /* Note: x2AVIC does not use logical APIC ID table */ 596 if (apic_x2apic_mode(vcpu->arch.apic)) 597 return; 598 599 entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); 600 if (entry) 601 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); 602 } 603 604 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) 605 { 606 int ret = 0; 607 struct vcpu_svm *svm = to_svm(vcpu); 608 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); 609 u32 id = kvm_xapic_id(vcpu->arch.apic); 610 611 /* AVIC does not support LDR update for x2APIC */ 612 if (apic_x2apic_mode(vcpu->arch.apic)) 613 return 0; 614 615 if (ldr == svm->ldr_reg) 616 return 0; 617 618 avic_invalidate_logical_id_entry(vcpu); 619 620 if (ldr) 621 ret = avic_ldr_write(vcpu, id, ldr); 622 623 if (!ret) 624 svm->ldr_reg = ldr; 625 626 return ret; 627 } 628 629 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) 630 { 631 struct vcpu_svm *svm = to_svm(vcpu); 632 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); 633 634 if (svm->dfr_reg == dfr) 635 return; 636 637 avic_invalidate_logical_id_entry(vcpu); 638 svm->dfr_reg = dfr; 639 } 640 641 static int avic_unaccel_trap_write(struct kvm_vcpu *vcpu) 642 { 643 u32 offset = to_svm(vcpu)->vmcb->control.exit_info_1 & 644 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 645 646 switch (offset) { 647 case APIC_LDR: 648 if (avic_handle_ldr_update(vcpu)) 649 return 0; 650 break; 651 case APIC_DFR: 652 avic_handle_dfr_update(vcpu); 653 break; 654 default: 655 break; 656 } 657 658 kvm_apic_write_nodecode(vcpu, offset); 659 return 1; 660 } 661 662 static bool is_avic_unaccelerated_access_trap(u32 offset) 663 { 664 bool ret = false; 665 666 switch (offset) { 667 case APIC_ID: 668 case APIC_EOI: 669 case APIC_RRR: 670 case APIC_LDR: 671 case APIC_DFR: 672 case APIC_SPIV: 673 case APIC_ESR: 674 case APIC_ICR: 675 case APIC_LVTT: 676 case APIC_LVTTHMR: 677 case APIC_LVTPC: 678 case APIC_LVT0: 679 case APIC_LVT1: 680 case APIC_LVTERR: 681 case APIC_TMICT: 682 case APIC_TDCR: 683 ret = true; 684 break; 685 default: 686 break; 687 } 688 return ret; 689 } 690 691 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu) 692 { 693 struct vcpu_svm *svm = to_svm(vcpu); 694 int ret = 0; 695 u32 offset = svm->vmcb->control.exit_info_1 & 696 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 697 u32 vector = svm->vmcb->control.exit_info_2 & 698 AVIC_UNACCEL_ACCESS_VECTOR_MASK; 699 bool write = (svm->vmcb->control.exit_info_1 >> 32) & 700 AVIC_UNACCEL_ACCESS_WRITE_MASK; 701 bool trap = is_avic_unaccelerated_access_trap(offset); 702 703 trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset, 704 trap, write, vector); 705 if (trap) { 706 /* Handling Trap */ 707 WARN_ONCE(!write, "svm: Handling trap read.\n"); 708 ret = avic_unaccel_trap_write(vcpu); 709 } else { 710 /* Handling Fault */ 711 ret = kvm_emulate_instruction(vcpu, 0); 712 } 713 714 return ret; 715 } 716 717 int avic_init_vcpu(struct vcpu_svm *svm) 718 { 719 int ret; 720 struct kvm_vcpu *vcpu = &svm->vcpu; 721 722 if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm)) 723 return 0; 724 725 ret = avic_init_backing_page(vcpu); 726 if (ret) 727 return ret; 728 729 INIT_LIST_HEAD(&svm->ir_list); 730 spin_lock_init(&svm->ir_list_lock); 731 svm->dfr_reg = APIC_DFR_FLAT; 732 733 return ret; 734 } 735 736 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu) 737 { 738 avic_handle_dfr_update(vcpu); 739 avic_handle_ldr_update(vcpu); 740 } 741 742 void avic_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 743 { 744 if (!lapic_in_kernel(vcpu) || avic_mode == AVIC_MODE_NONE) 745 return; 746 747 if (kvm_get_apic_mode(vcpu) == LAPIC_MODE_INVALID) { 748 WARN_ONCE(true, "Invalid local APIC state (vcpu_id=%d)", vcpu->vcpu_id); 749 return; 750 } 751 avic_refresh_apicv_exec_ctrl(vcpu); 752 } 753 754 static int avic_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) 755 { 756 int ret = 0; 757 unsigned long flags; 758 struct amd_svm_iommu_ir *ir; 759 struct vcpu_svm *svm = to_svm(vcpu); 760 761 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 762 return 0; 763 764 /* 765 * Here, we go through the per-vcpu ir_list to update all existing 766 * interrupt remapping table entry targeting this vcpu. 767 */ 768 spin_lock_irqsave(&svm->ir_list_lock, flags); 769 770 if (list_empty(&svm->ir_list)) 771 goto out; 772 773 list_for_each_entry(ir, &svm->ir_list, node) { 774 if (activate) 775 ret = amd_iommu_activate_guest_mode(ir->data); 776 else 777 ret = amd_iommu_deactivate_guest_mode(ir->data); 778 if (ret) 779 break; 780 } 781 out: 782 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 783 return ret; 784 } 785 786 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 787 { 788 unsigned long flags; 789 struct amd_svm_iommu_ir *cur; 790 791 spin_lock_irqsave(&svm->ir_list_lock, flags); 792 list_for_each_entry(cur, &svm->ir_list, node) { 793 if (cur->data != pi->ir_data) 794 continue; 795 list_del(&cur->node); 796 kfree(cur); 797 break; 798 } 799 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 800 } 801 802 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 803 { 804 int ret = 0; 805 unsigned long flags; 806 struct amd_svm_iommu_ir *ir; 807 808 /** 809 * In some cases, the existing irte is updated and re-set, 810 * so we need to check here if it's already been * added 811 * to the ir_list. 812 */ 813 if (pi->ir_data && (pi->prev_ga_tag != 0)) { 814 struct kvm *kvm = svm->vcpu.kvm; 815 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); 816 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 817 struct vcpu_svm *prev_svm; 818 819 if (!prev_vcpu) { 820 ret = -EINVAL; 821 goto out; 822 } 823 824 prev_svm = to_svm(prev_vcpu); 825 svm_ir_list_del(prev_svm, pi); 826 } 827 828 /** 829 * Allocating new amd_iommu_pi_data, which will get 830 * add to the per-vcpu ir_list. 831 */ 832 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); 833 if (!ir) { 834 ret = -ENOMEM; 835 goto out; 836 } 837 ir->data = pi->ir_data; 838 839 spin_lock_irqsave(&svm->ir_list_lock, flags); 840 list_add(&ir->node, &svm->ir_list); 841 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 842 out: 843 return ret; 844 } 845 846 /* 847 * Note: 848 * The HW cannot support posting multicast/broadcast 849 * interrupts to a vCPU. So, we still use legacy interrupt 850 * remapping for these kind of interrupts. 851 * 852 * For lowest-priority interrupts, we only support 853 * those with single CPU as the destination, e.g. user 854 * configures the interrupts via /proc/irq or uses 855 * irqbalance to make the interrupts single-CPU. 856 */ 857 static int 858 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 859 struct vcpu_data *vcpu_info, struct vcpu_svm **svm) 860 { 861 struct kvm_lapic_irq irq; 862 struct kvm_vcpu *vcpu = NULL; 863 864 kvm_set_msi_irq(kvm, e, &irq); 865 866 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 867 !kvm_irq_is_postable(&irq)) { 868 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", 869 __func__, irq.vector); 870 return -1; 871 } 872 873 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, 874 irq.vector); 875 *svm = to_svm(vcpu); 876 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); 877 vcpu_info->vector = irq.vector; 878 879 return 0; 880 } 881 882 /* 883 * avic_pi_update_irte - set IRTE for Posted-Interrupts 884 * 885 * @kvm: kvm 886 * @host_irq: host irq of the interrupt 887 * @guest_irq: gsi of the interrupt 888 * @set: set or unset PI 889 * returns 0 on success, < 0 on failure 890 */ 891 int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq, 892 uint32_t guest_irq, bool set) 893 { 894 struct kvm_kernel_irq_routing_entry *e; 895 struct kvm_irq_routing_table *irq_rt; 896 int idx, ret = 0; 897 898 if (!kvm_arch_has_assigned_device(kvm) || 899 !irq_remapping_cap(IRQ_POSTING_CAP)) 900 return 0; 901 902 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", 903 __func__, host_irq, guest_irq, set); 904 905 idx = srcu_read_lock(&kvm->irq_srcu); 906 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 907 908 if (guest_irq >= irq_rt->nr_rt_entries || 909 hlist_empty(&irq_rt->map[guest_irq])) { 910 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", 911 guest_irq, irq_rt->nr_rt_entries); 912 goto out; 913 } 914 915 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 916 struct vcpu_data vcpu_info; 917 struct vcpu_svm *svm = NULL; 918 919 if (e->type != KVM_IRQ_ROUTING_MSI) 920 continue; 921 922 /** 923 * Here, we setup with legacy mode in the following cases: 924 * 1. When cannot target interrupt to a specific vcpu. 925 * 2. Unsetting posted interrupt. 926 * 3. APIC virtualization is disabled for the vcpu. 927 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) 928 */ 929 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && 930 kvm_vcpu_apicv_active(&svm->vcpu)) { 931 struct amd_iommu_pi_data pi; 932 933 /* Try to enable guest_mode in IRTE */ 934 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & 935 AVIC_HPA_MASK); 936 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, 937 svm->vcpu.vcpu_id); 938 pi.is_guest_mode = true; 939 pi.vcpu_data = &vcpu_info; 940 ret = irq_set_vcpu_affinity(host_irq, &pi); 941 942 /** 943 * Here, we successfully setting up vcpu affinity in 944 * IOMMU guest mode. Now, we need to store the posted 945 * interrupt information in a per-vcpu ir_list so that 946 * we can reference to them directly when we update vcpu 947 * scheduling information in IOMMU irte. 948 */ 949 if (!ret && pi.is_guest_mode) 950 svm_ir_list_add(svm, &pi); 951 } else { 952 /* Use legacy mode in IRTE */ 953 struct amd_iommu_pi_data pi; 954 955 /** 956 * Here, pi is used to: 957 * - Tell IOMMU to use legacy mode for this interrupt. 958 * - Retrieve ga_tag of prior interrupt remapping data. 959 */ 960 pi.prev_ga_tag = 0; 961 pi.is_guest_mode = false; 962 ret = irq_set_vcpu_affinity(host_irq, &pi); 963 964 /** 965 * Check if the posted interrupt was previously 966 * setup with the guest_mode by checking if the ga_tag 967 * was cached. If so, we need to clean up the per-vcpu 968 * ir_list. 969 */ 970 if (!ret && pi.prev_ga_tag) { 971 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); 972 struct kvm_vcpu *vcpu; 973 974 vcpu = kvm_get_vcpu_by_id(kvm, id); 975 if (vcpu) 976 svm_ir_list_del(to_svm(vcpu), &pi); 977 } 978 } 979 980 if (!ret && svm) { 981 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, 982 e->gsi, vcpu_info.vector, 983 vcpu_info.pi_desc_addr, set); 984 } 985 986 if (ret < 0) { 987 pr_err("%s: failed to update PI IRTE\n", __func__); 988 goto out; 989 } 990 } 991 992 ret = 0; 993 out: 994 srcu_read_unlock(&kvm->irq_srcu, idx); 995 return ret; 996 } 997 998 bool avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason) 999 { 1000 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 1001 BIT(APICV_INHIBIT_REASON_ABSENT) | 1002 BIT(APICV_INHIBIT_REASON_HYPERV) | 1003 BIT(APICV_INHIBIT_REASON_NESTED) | 1004 BIT(APICV_INHIBIT_REASON_IRQWIN) | 1005 BIT(APICV_INHIBIT_REASON_PIT_REINJ) | 1006 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | 1007 BIT(APICV_INHIBIT_REASON_SEV) | 1008 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | 1009 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED); 1010 1011 return supported & BIT(reason); 1012 } 1013 1014 1015 static inline int 1016 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) 1017 { 1018 int ret = 0; 1019 unsigned long flags; 1020 struct amd_svm_iommu_ir *ir; 1021 struct vcpu_svm *svm = to_svm(vcpu); 1022 1023 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 1024 return 0; 1025 1026 /* 1027 * Here, we go through the per-vcpu ir_list to update all existing 1028 * interrupt remapping table entry targeting this vcpu. 1029 */ 1030 spin_lock_irqsave(&svm->ir_list_lock, flags); 1031 1032 if (list_empty(&svm->ir_list)) 1033 goto out; 1034 1035 list_for_each_entry(ir, &svm->ir_list, node) { 1036 ret = amd_iommu_update_ga(cpu, r, ir->data); 1037 if (ret) 1038 break; 1039 } 1040 out: 1041 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 1042 return ret; 1043 } 1044 1045 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1046 { 1047 u64 entry; 1048 int h_physical_id = kvm_cpu_get_apicid(cpu); 1049 struct vcpu_svm *svm = to_svm(vcpu); 1050 1051 lockdep_assert_preemption_disabled(); 1052 1053 if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) 1054 return; 1055 1056 /* 1057 * No need to update anything if the vCPU is blocking, i.e. if the vCPU 1058 * is being scheduled in after being preempted. The CPU entries in the 1059 * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'. 1060 * If the vCPU was migrated, its new CPU value will be stuffed when the 1061 * vCPU unblocks. 1062 */ 1063 if (kvm_vcpu_is_blocking(vcpu)) 1064 return; 1065 1066 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 1067 1068 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; 1069 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); 1070 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 1071 1072 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 1073 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true); 1074 } 1075 1076 void avic_vcpu_put(struct kvm_vcpu *vcpu) 1077 { 1078 u64 entry; 1079 struct vcpu_svm *svm = to_svm(vcpu); 1080 1081 lockdep_assert_preemption_disabled(); 1082 1083 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 1084 1085 /* Nothing to do if IsRunning == '0' due to vCPU blocking. */ 1086 if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)) 1087 return; 1088 1089 avic_update_iommu_vcpu_affinity(vcpu, -1, 0); 1090 1091 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 1092 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 1093 } 1094 1095 1096 void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 1097 { 1098 struct vcpu_svm *svm = to_svm(vcpu); 1099 struct vmcb *vmcb = svm->vmcb01.ptr; 1100 bool activated = kvm_vcpu_apicv_active(vcpu); 1101 1102 if (!enable_apicv) 1103 return; 1104 1105 if (activated) { 1106 /** 1107 * During AVIC temporary deactivation, guest could update 1108 * APIC ID, DFR and LDR registers, which would not be trapped 1109 * by avic_unaccelerated_access_interception(). In this case, 1110 * we need to check and update the AVIC logical APIC ID table 1111 * accordingly before re-activating. 1112 */ 1113 avic_apicv_post_state_restore(vcpu); 1114 avic_activate_vmcb(svm); 1115 } else { 1116 avic_deactivate_vmcb(svm); 1117 } 1118 vmcb_mark_dirty(vmcb, VMCB_AVIC); 1119 1120 if (activated) 1121 avic_vcpu_load(vcpu, vcpu->cpu); 1122 else 1123 avic_vcpu_put(vcpu); 1124 1125 avic_set_pi_irte_mode(vcpu, activated); 1126 } 1127 1128 void avic_vcpu_blocking(struct kvm_vcpu *vcpu) 1129 { 1130 if (!kvm_vcpu_apicv_active(vcpu)) 1131 return; 1132 1133 /* 1134 * Unload the AVIC when the vCPU is about to block, _before_ 1135 * the vCPU actually blocks. 1136 * 1137 * Any IRQs that arrive before IsRunning=0 will not cause an 1138 * incomplete IPI vmexit on the source, therefore vIRR will also 1139 * be checked by kvm_vcpu_check_block() before blocking. The 1140 * memory barrier implicit in set_current_state orders writing 1141 * IsRunning=0 before reading the vIRR. The processor needs a 1142 * matching memory barrier on interrupt delivery between writing 1143 * IRR and reading IsRunning; the lack of this barrier might be 1144 * the cause of errata #1235). 1145 */ 1146 avic_vcpu_put(vcpu); 1147 } 1148 1149 void avic_vcpu_unblocking(struct kvm_vcpu *vcpu) 1150 { 1151 if (!kvm_vcpu_apicv_active(vcpu)) 1152 return; 1153 1154 avic_vcpu_load(vcpu, vcpu->cpu); 1155 } 1156 1157 /* 1158 * Note: 1159 * - The module param avic enable both xAPIC and x2APIC mode. 1160 * - Hypervisor can support both xAVIC and x2AVIC in the same guest. 1161 * - The mode can be switched at run-time. 1162 */ 1163 bool avic_hardware_setup(struct kvm_x86_ops *x86_ops) 1164 { 1165 if (!npt_enabled) 1166 return false; 1167 1168 if (boot_cpu_has(X86_FEATURE_AVIC)) { 1169 avic_mode = AVIC_MODE_X1; 1170 pr_info("AVIC enabled\n"); 1171 } else if (force_avic) { 1172 /* 1173 * Some older systems does not advertise AVIC support. 1174 * See Revision Guide for specific AMD processor for more detail. 1175 */ 1176 avic_mode = AVIC_MODE_X1; 1177 pr_warn("AVIC is not supported in CPUID but force enabled"); 1178 pr_warn("Your system might crash and burn"); 1179 } 1180 1181 /* AVIC is a prerequisite for x2AVIC. */ 1182 if (boot_cpu_has(X86_FEATURE_X2AVIC)) { 1183 if (avic_mode == AVIC_MODE_X1) { 1184 avic_mode = AVIC_MODE_X2; 1185 pr_info("x2AVIC enabled\n"); 1186 } else { 1187 pr_warn(FW_BUG "Cannot support x2AVIC due to AVIC is disabled"); 1188 pr_warn(FW_BUG "Try enable AVIC using force_avic option"); 1189 } 1190 } 1191 1192 if (avic_mode != AVIC_MODE_NONE) 1193 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); 1194 1195 return !!avic_mode; 1196 } 1197