1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Local APIC virtualization 5 * 6 * Copyright (C) 2006 Qumranet, Inc. 7 * Copyright (C) 2007 Novell 8 * Copyright (C) 2007 Intel 9 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Dor Laor <dor.laor@qumranet.com> 13 * Gregory Haskins <ghaskins@novell.com> 14 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 15 * 16 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 17 */ 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/kvm_host.h> 21 #include <linux/kvm.h> 22 #include <linux/mm.h> 23 #include <linux/highmem.h> 24 #include <linux/smp.h> 25 #include <linux/hrtimer.h> 26 #include <linux/io.h> 27 #include <linux/export.h> 28 #include <linux/math64.h> 29 #include <linux/slab.h> 30 #include <asm/processor.h> 31 #include <asm/mce.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "ioapic.h" 42 #include "trace.h" 43 #include "x86.h" 44 #include "cpuid.h" 45 #include "hyperv.h" 46 #include "smm.h" 47 48 #ifndef CONFIG_X86_64 49 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 50 #else 51 #define mod_64(x, y) ((x) % (y)) 52 #endif 53 54 /* 14 is the version for Xeon and Pentium 8.4.8*/ 55 #define APIC_VERSION 0x14UL 56 #define LAPIC_MMIO_LENGTH (1 << 12) 57 /* followed define is not in apicdef.h */ 58 #define MAX_APIC_VECTOR 256 59 #define APIC_VECTORS_PER_REG 32 60 61 static bool lapic_timer_advance_dynamic __read_mostly; 62 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN 100 /* clock cycles */ 63 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX 10000 /* clock cycles */ 64 #define LAPIC_TIMER_ADVANCE_NS_INIT 1000 65 #define LAPIC_TIMER_ADVANCE_NS_MAX 5000 66 /* step-by-step approximation to mitigate fluctuation */ 67 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8 68 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data); 69 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data); 70 71 static inline void __kvm_lapic_set_reg(char *regs, int reg_off, u32 val) 72 { 73 *((u32 *) (regs + reg_off)) = val; 74 } 75 76 static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) 77 { 78 __kvm_lapic_set_reg(apic->regs, reg_off, val); 79 } 80 81 static __always_inline u64 __kvm_lapic_get_reg64(char *regs, int reg) 82 { 83 BUILD_BUG_ON(reg != APIC_ICR); 84 return *((u64 *) (regs + reg)); 85 } 86 87 static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg) 88 { 89 return __kvm_lapic_get_reg64(apic->regs, reg); 90 } 91 92 static __always_inline void __kvm_lapic_set_reg64(char *regs, int reg, u64 val) 93 { 94 BUILD_BUG_ON(reg != APIC_ICR); 95 *((u64 *) (regs + reg)) = val; 96 } 97 98 static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, 99 int reg, u64 val) 100 { 101 __kvm_lapic_set_reg64(apic->regs, reg, val); 102 } 103 104 static inline int apic_test_vector(int vec, void *bitmap) 105 { 106 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 107 } 108 109 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 110 { 111 struct kvm_lapic *apic = vcpu->arch.apic; 112 113 return apic_test_vector(vector, apic->regs + APIC_ISR) || 114 apic_test_vector(vector, apic->regs + APIC_IRR); 115 } 116 117 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 118 { 119 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 120 } 121 122 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 123 { 124 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 125 } 126 127 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ); 128 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ); 129 130 static inline int apic_enabled(struct kvm_lapic *apic) 131 { 132 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 133 } 134 135 #define LVT_MASK \ 136 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 137 138 #define LINT_MASK \ 139 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 140 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 141 142 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 143 { 144 return apic->vcpu->vcpu_id; 145 } 146 147 static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu) 148 { 149 return pi_inject_timer && kvm_vcpu_apicv_active(vcpu) && 150 (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm)); 151 } 152 153 bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu) 154 { 155 return kvm_x86_ops.set_hv_timer 156 && !(kvm_mwait_in_guest(vcpu->kvm) || 157 kvm_can_post_timer_interrupt(vcpu)); 158 } 159 160 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu) 161 { 162 return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE; 163 } 164 165 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 166 { 167 return ((id >> 4) << 16) | (1 << (id & 0xf)); 168 } 169 170 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 171 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 172 switch (map->logical_mode) { 173 case KVM_APIC_MODE_SW_DISABLED: 174 /* Arbitrarily use the flat map so that @cluster isn't NULL. */ 175 *cluster = map->xapic_flat_map; 176 *mask = 0; 177 return true; 178 case KVM_APIC_MODE_X2APIC: { 179 u32 offset = (dest_id >> 16) * 16; 180 u32 max_apic_id = map->max_apic_id; 181 182 if (offset <= max_apic_id) { 183 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 184 185 offset = array_index_nospec(offset, map->max_apic_id + 1); 186 *cluster = &map->phys_map[offset]; 187 *mask = dest_id & (0xffff >> (16 - cluster_size)); 188 } else { 189 *mask = 0; 190 } 191 192 return true; 193 } 194 case KVM_APIC_MODE_XAPIC_FLAT: 195 *cluster = map->xapic_flat_map; 196 *mask = dest_id & 0xff; 197 return true; 198 case KVM_APIC_MODE_XAPIC_CLUSTER: 199 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 200 *mask = dest_id & 0xf; 201 return true; 202 case KVM_APIC_MODE_MAP_DISABLED: 203 return false; 204 default: 205 WARN_ON_ONCE(1); 206 return false; 207 } 208 } 209 210 static void kvm_apic_map_free(struct rcu_head *rcu) 211 { 212 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 213 214 kvfree(map); 215 } 216 217 static int kvm_recalculate_phys_map(struct kvm_apic_map *new, 218 struct kvm_vcpu *vcpu, 219 bool *xapic_id_mismatch) 220 { 221 struct kvm_lapic *apic = vcpu->arch.apic; 222 u32 x2apic_id = kvm_x2apic_id(apic); 223 u32 xapic_id = kvm_xapic_id(apic); 224 u32 physical_id; 225 226 /* 227 * For simplicity, KVM always allocates enough space for all possible 228 * xAPIC IDs. Yell, but don't kill the VM, as KVM can continue on 229 * without the optimized map. 230 */ 231 if (WARN_ON_ONCE(xapic_id > new->max_apic_id)) 232 return -EINVAL; 233 234 /* 235 * Bail if a vCPU was added and/or enabled its APIC between allocating 236 * the map and doing the actual calculations for the map. Note, KVM 237 * hardcodes the x2APIC ID to vcpu_id, i.e. there's no TOCTOU bug if 238 * the compiler decides to reload x2apic_id after this check. 239 */ 240 if (x2apic_id > new->max_apic_id) 241 return -E2BIG; 242 243 /* 244 * Deliberately truncate the vCPU ID when detecting a mismatched APIC 245 * ID to avoid false positives if the vCPU ID, i.e. x2APIC ID, is a 246 * 32-bit value. Any unwanted aliasing due to truncation results will 247 * be detected below. 248 */ 249 if (!apic_x2apic_mode(apic) && xapic_id != (u8)vcpu->vcpu_id) 250 *xapic_id_mismatch = true; 251 252 /* 253 * Apply KVM's hotplug hack if userspace has enable 32-bit APIC IDs. 254 * Allow sending events to vCPUs by their x2APIC ID even if the target 255 * vCPU is in legacy xAPIC mode, and silently ignore aliased xAPIC IDs 256 * (the x2APIC ID is truncated to 8 bits, causing IDs > 0xff to wrap 257 * and collide). 258 * 259 * Honor the architectural (and KVM's non-optimized) behavior if 260 * userspace has not enabled 32-bit x2APIC IDs. Each APIC is supposed 261 * to process messages independently. If multiple vCPUs have the same 262 * effective APIC ID, e.g. due to the x2APIC wrap or because the guest 263 * manually modified its xAPIC IDs, events targeting that ID are 264 * supposed to be recognized by all vCPUs with said ID. 265 */ 266 if (vcpu->kvm->arch.x2apic_format) { 267 /* See also kvm_apic_match_physical_addr(). */ 268 if (apic_x2apic_mode(apic) || x2apic_id > 0xff) 269 new->phys_map[x2apic_id] = apic; 270 271 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 272 new->phys_map[xapic_id] = apic; 273 } else { 274 /* 275 * Disable the optimized map if the physical APIC ID is already 276 * mapped, i.e. is aliased to multiple vCPUs. The optimized 277 * map requires a strict 1:1 mapping between IDs and vCPUs. 278 */ 279 if (apic_x2apic_mode(apic)) 280 physical_id = x2apic_id; 281 else 282 physical_id = xapic_id; 283 284 if (new->phys_map[physical_id]) 285 return -EINVAL; 286 287 new->phys_map[physical_id] = apic; 288 } 289 290 return 0; 291 } 292 293 static void kvm_recalculate_logical_map(struct kvm_apic_map *new, 294 struct kvm_vcpu *vcpu) 295 { 296 struct kvm_lapic *apic = vcpu->arch.apic; 297 enum kvm_apic_logical_mode logical_mode; 298 struct kvm_lapic **cluster; 299 u16 mask; 300 u32 ldr; 301 302 if (new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) 303 return; 304 305 if (!kvm_apic_sw_enabled(apic)) 306 return; 307 308 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 309 if (!ldr) 310 return; 311 312 if (apic_x2apic_mode(apic)) { 313 logical_mode = KVM_APIC_MODE_X2APIC; 314 } else { 315 ldr = GET_APIC_LOGICAL_ID(ldr); 316 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 317 logical_mode = KVM_APIC_MODE_XAPIC_FLAT; 318 else 319 logical_mode = KVM_APIC_MODE_XAPIC_CLUSTER; 320 } 321 322 /* 323 * To optimize logical mode delivery, all software-enabled APICs must 324 * be configured for the same mode. 325 */ 326 if (new->logical_mode == KVM_APIC_MODE_SW_DISABLED) { 327 new->logical_mode = logical_mode; 328 } else if (new->logical_mode != logical_mode) { 329 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 330 return; 331 } 332 333 /* 334 * In x2APIC mode, the LDR is read-only and derived directly from the 335 * x2APIC ID, thus is guaranteed to be addressable. KVM reuses 336 * kvm_apic_map.phys_map to optimize logical mode x2APIC interrupts by 337 * reversing the LDR calculation to get cluster of APICs, i.e. no 338 * additional work is required. 339 */ 340 if (apic_x2apic_mode(apic)) { 341 WARN_ON_ONCE(ldr != kvm_apic_calc_x2apic_ldr(kvm_x2apic_id(apic))); 342 return; 343 } 344 345 if (WARN_ON_ONCE(!kvm_apic_map_get_logical_dest(new, ldr, 346 &cluster, &mask))) { 347 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 348 return; 349 } 350 351 if (!mask) 352 return; 353 354 ldr = ffs(mask) - 1; 355 if (!is_power_of_2(mask) || cluster[ldr]) 356 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 357 else 358 cluster[ldr] = apic; 359 } 360 361 /* 362 * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock. 363 * 364 * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with 365 * apic_map_lock_held. 366 */ 367 enum { 368 CLEAN, 369 UPDATE_IN_PROGRESS, 370 DIRTY 371 }; 372 373 void kvm_recalculate_apic_map(struct kvm *kvm) 374 { 375 struct kvm_apic_map *new, *old = NULL; 376 struct kvm_vcpu *vcpu; 377 unsigned long i; 378 u32 max_id = 255; /* enough space for any xAPIC ID */ 379 bool xapic_id_mismatch = false; 380 381 /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map. */ 382 if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN) 383 return; 384 385 WARN_ONCE(!irqchip_in_kernel(kvm), 386 "Dirty APIC map without an in-kernel local APIC"); 387 388 mutex_lock(&kvm->arch.apic_map_lock); 389 /* 390 * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map 391 * (if clean) or the APIC registers (if dirty). 392 */ 393 if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty, 394 DIRTY, UPDATE_IN_PROGRESS) == CLEAN) { 395 /* Someone else has updated the map. */ 396 mutex_unlock(&kvm->arch.apic_map_lock); 397 return; 398 } 399 400 kvm_for_each_vcpu(i, vcpu, kvm) 401 if (kvm_apic_present(vcpu)) 402 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 403 404 new = kvzalloc(sizeof(struct kvm_apic_map) + 405 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), 406 GFP_KERNEL_ACCOUNT); 407 408 if (!new) 409 goto out; 410 411 new->max_apic_id = max_id; 412 new->logical_mode = KVM_APIC_MODE_SW_DISABLED; 413 414 kvm_for_each_vcpu(i, vcpu, kvm) { 415 if (!kvm_apic_present(vcpu)) 416 continue; 417 418 if (kvm_recalculate_phys_map(new, vcpu, &xapic_id_mismatch)) { 419 kvfree(new); 420 new = NULL; 421 goto out; 422 } 423 424 kvm_recalculate_logical_map(new, vcpu); 425 } 426 out: 427 /* 428 * The optimized map is effectively KVM's internal version of APICv, 429 * and all unwanted aliasing that results in disabling the optimized 430 * map also applies to APICv. 431 */ 432 if (!new) 433 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED); 434 else 435 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED); 436 437 if (!new || new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) 438 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED); 439 else 440 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED); 441 442 if (xapic_id_mismatch) 443 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED); 444 else 445 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED); 446 447 old = rcu_dereference_protected(kvm->arch.apic_map, 448 lockdep_is_held(&kvm->arch.apic_map_lock)); 449 rcu_assign_pointer(kvm->arch.apic_map, new); 450 /* 451 * Write kvm->arch.apic_map before clearing apic->apic_map_dirty. 452 * If another update has come in, leave it DIRTY. 453 */ 454 atomic_cmpxchg_release(&kvm->arch.apic_map_dirty, 455 UPDATE_IN_PROGRESS, CLEAN); 456 mutex_unlock(&kvm->arch.apic_map_lock); 457 458 if (old) 459 call_rcu(&old->rcu, kvm_apic_map_free); 460 461 kvm_make_scan_ioapic_request(kvm); 462 } 463 464 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 465 { 466 bool enabled = val & APIC_SPIV_APIC_ENABLED; 467 468 kvm_lapic_set_reg(apic, APIC_SPIV, val); 469 470 if (enabled != apic->sw_enabled) { 471 apic->sw_enabled = enabled; 472 if (enabled) 473 static_branch_slow_dec_deferred(&apic_sw_disabled); 474 else 475 static_branch_inc(&apic_sw_disabled.key); 476 477 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 478 } 479 480 /* Check if there are APF page ready requests pending */ 481 if (enabled) 482 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu); 483 } 484 485 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 486 { 487 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 488 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 489 } 490 491 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 492 { 493 kvm_lapic_set_reg(apic, APIC_LDR, id); 494 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 495 } 496 497 static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val) 498 { 499 kvm_lapic_set_reg(apic, APIC_DFR, val); 500 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 501 } 502 503 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 504 { 505 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 506 507 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 508 509 kvm_lapic_set_reg(apic, APIC_ID, id); 510 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 511 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 512 } 513 514 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 515 { 516 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 517 } 518 519 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 520 { 521 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 522 } 523 524 static inline int apic_lvtt_period(struct kvm_lapic *apic) 525 { 526 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 527 } 528 529 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 530 { 531 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 532 } 533 534 static inline int apic_lvt_nmi_mode(u32 lvt_val) 535 { 536 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 537 } 538 539 static inline bool kvm_lapic_lvt_supported(struct kvm_lapic *apic, int lvt_index) 540 { 541 return apic->nr_lvt_entries > lvt_index; 542 } 543 544 static inline int kvm_apic_calc_nr_lvt_entries(struct kvm_vcpu *vcpu) 545 { 546 return KVM_APIC_MAX_NR_LVT_ENTRIES - !(vcpu->arch.mcg_cap & MCG_CMCI_P); 547 } 548 549 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 550 { 551 struct kvm_lapic *apic = vcpu->arch.apic; 552 u32 v = 0; 553 554 if (!lapic_in_kernel(vcpu)) 555 return; 556 557 v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); 558 559 /* 560 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) 561 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with 562 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC 563 * version first and level-triggered interrupts never get EOIed in 564 * IOAPIC. 565 */ 566 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) && 567 !ioapic_in_kernel(vcpu->kvm)) 568 v |= APIC_LVR_DIRECTED_EOI; 569 kvm_lapic_set_reg(apic, APIC_LVR, v); 570 } 571 572 void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu) 573 { 574 int nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); 575 struct kvm_lapic *apic = vcpu->arch.apic; 576 int i; 577 578 if (!lapic_in_kernel(vcpu) || nr_lvt_entries == apic->nr_lvt_entries) 579 return; 580 581 /* Initialize/mask any "new" LVT entries. */ 582 for (i = apic->nr_lvt_entries; i < nr_lvt_entries; i++) 583 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); 584 585 apic->nr_lvt_entries = nr_lvt_entries; 586 587 /* The number of LVT entries is reflected in the version register. */ 588 kvm_apic_set_version(vcpu); 589 } 590 591 static const unsigned int apic_lvt_mask[KVM_APIC_MAX_NR_LVT_ENTRIES] = { 592 [LVT_TIMER] = LVT_MASK, /* timer mode mask added at runtime */ 593 [LVT_THERMAL_MONITOR] = LVT_MASK | APIC_MODE_MASK, 594 [LVT_PERFORMANCE_COUNTER] = LVT_MASK | APIC_MODE_MASK, 595 [LVT_LINT0] = LINT_MASK, 596 [LVT_LINT1] = LINT_MASK, 597 [LVT_ERROR] = LVT_MASK, 598 [LVT_CMCI] = LVT_MASK | APIC_MODE_MASK 599 }; 600 601 static int find_highest_vector(void *bitmap) 602 { 603 int vec; 604 u32 *reg; 605 606 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 607 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 608 reg = bitmap + REG_POS(vec); 609 if (*reg) 610 return __fls(*reg) + vec; 611 } 612 613 return -1; 614 } 615 616 static u8 count_vectors(void *bitmap) 617 { 618 int vec; 619 u32 *reg; 620 u8 count = 0; 621 622 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 623 reg = bitmap + REG_POS(vec); 624 count += hweight32(*reg); 625 } 626 627 return count; 628 } 629 630 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr) 631 { 632 u32 i, vec; 633 u32 pir_val, irr_val, prev_irr_val; 634 int max_updated_irr; 635 636 max_updated_irr = -1; 637 *max_irr = -1; 638 639 for (i = vec = 0; i <= 7; i++, vec += 32) { 640 u32 *p_irr = (u32 *)(regs + APIC_IRR + i * 0x10); 641 642 irr_val = *p_irr; 643 pir_val = READ_ONCE(pir[i]); 644 645 if (pir_val) { 646 pir_val = xchg(&pir[i], 0); 647 648 prev_irr_val = irr_val; 649 do { 650 irr_val = prev_irr_val | pir_val; 651 } while (prev_irr_val != irr_val && 652 !try_cmpxchg(p_irr, &prev_irr_val, irr_val)); 653 654 if (prev_irr_val != irr_val) 655 max_updated_irr = __fls(irr_val ^ prev_irr_val) + vec; 656 } 657 if (irr_val) 658 *max_irr = __fls(irr_val) + vec; 659 } 660 661 return ((max_updated_irr != -1) && 662 (max_updated_irr == *max_irr)); 663 } 664 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 665 666 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr) 667 { 668 struct kvm_lapic *apic = vcpu->arch.apic; 669 bool irr_updated = __kvm_apic_update_irr(pir, apic->regs, max_irr); 670 671 if (unlikely(!apic->apicv_active && irr_updated)) 672 apic->irr_pending = true; 673 return irr_updated; 674 } 675 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 676 677 static inline int apic_search_irr(struct kvm_lapic *apic) 678 { 679 return find_highest_vector(apic->regs + APIC_IRR); 680 } 681 682 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 683 { 684 int result; 685 686 /* 687 * Note that irr_pending is just a hint. It will be always 688 * true with virtual interrupt delivery enabled. 689 */ 690 if (!apic->irr_pending) 691 return -1; 692 693 result = apic_search_irr(apic); 694 ASSERT(result == -1 || result >= 16); 695 696 return result; 697 } 698 699 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 700 { 701 if (unlikely(apic->apicv_active)) { 702 /* need to update RVI */ 703 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 704 static_call_cond(kvm_x86_hwapic_irr_update)(apic->vcpu, 705 apic_find_highest_irr(apic)); 706 } else { 707 apic->irr_pending = false; 708 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 709 if (apic_search_irr(apic) != -1) 710 apic->irr_pending = true; 711 } 712 } 713 714 void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec) 715 { 716 apic_clear_irr(vec, vcpu->arch.apic); 717 } 718 EXPORT_SYMBOL_GPL(kvm_apic_clear_irr); 719 720 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 721 { 722 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 723 return; 724 725 /* 726 * With APIC virtualization enabled, all caching is disabled 727 * because the processor can modify ISR under the hood. Instead 728 * just set SVI. 729 */ 730 if (unlikely(apic->apicv_active)) 731 static_call_cond(kvm_x86_hwapic_isr_update)(vec); 732 else { 733 ++apic->isr_count; 734 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 735 /* 736 * ISR (in service register) bit is set when injecting an interrupt. 737 * The highest vector is injected. Thus the latest bit set matches 738 * the highest bit in ISR. 739 */ 740 apic->highest_isr_cache = vec; 741 } 742 } 743 744 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 745 { 746 int result; 747 748 /* 749 * Note that isr_count is always 1, and highest_isr_cache 750 * is always -1, with APIC virtualization enabled. 751 */ 752 if (!apic->isr_count) 753 return -1; 754 if (likely(apic->highest_isr_cache != -1)) 755 return apic->highest_isr_cache; 756 757 result = find_highest_vector(apic->regs + APIC_ISR); 758 ASSERT(result == -1 || result >= 16); 759 760 return result; 761 } 762 763 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 764 { 765 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 766 return; 767 768 /* 769 * We do get here for APIC virtualization enabled if the guest 770 * uses the Hyper-V APIC enlightenment. In this case we may need 771 * to trigger a new interrupt delivery by writing the SVI field; 772 * on the other hand isr_count and highest_isr_cache are unused 773 * and must be left alone. 774 */ 775 if (unlikely(apic->apicv_active)) 776 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); 777 else { 778 --apic->isr_count; 779 BUG_ON(apic->isr_count < 0); 780 apic->highest_isr_cache = -1; 781 } 782 } 783 784 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 785 { 786 /* This may race with setting of irr in __apic_accept_irq() and 787 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 788 * will cause vmexit immediately and the value will be recalculated 789 * on the next vmentry. 790 */ 791 return apic_find_highest_irr(vcpu->arch.apic); 792 } 793 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 794 795 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 796 int vector, int level, int trig_mode, 797 struct dest_map *dest_map); 798 799 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 800 struct dest_map *dest_map) 801 { 802 struct kvm_lapic *apic = vcpu->arch.apic; 803 804 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 805 irq->level, irq->trig_mode, dest_map); 806 } 807 808 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map, 809 struct kvm_lapic_irq *irq, u32 min) 810 { 811 int i, count = 0; 812 struct kvm_vcpu *vcpu; 813 814 if (min > map->max_apic_id) 815 return 0; 816 817 for_each_set_bit(i, ipi_bitmap, 818 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { 819 if (map->phys_map[min + i]) { 820 vcpu = map->phys_map[min + i]->vcpu; 821 count += kvm_apic_set_irq(vcpu, irq, NULL); 822 } 823 } 824 825 return count; 826 } 827 828 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, 829 unsigned long ipi_bitmap_high, u32 min, 830 unsigned long icr, int op_64_bit) 831 { 832 struct kvm_apic_map *map; 833 struct kvm_lapic_irq irq = {0}; 834 int cluster_size = op_64_bit ? 64 : 32; 835 int count; 836 837 if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK)) 838 return -KVM_EINVAL; 839 840 irq.vector = icr & APIC_VECTOR_MASK; 841 irq.delivery_mode = icr & APIC_MODE_MASK; 842 irq.level = (icr & APIC_INT_ASSERT) != 0; 843 irq.trig_mode = icr & APIC_INT_LEVELTRIG; 844 845 rcu_read_lock(); 846 map = rcu_dereference(kvm->arch.apic_map); 847 848 count = -EOPNOTSUPP; 849 if (likely(map)) { 850 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min); 851 min += cluster_size; 852 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min); 853 } 854 855 rcu_read_unlock(); 856 return count; 857 } 858 859 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 860 { 861 862 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 863 sizeof(val)); 864 } 865 866 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 867 { 868 869 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 870 sizeof(*val)); 871 } 872 873 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 874 { 875 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 876 } 877 878 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 879 { 880 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) 881 return; 882 883 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 884 } 885 886 static bool pv_eoi_test_and_clr_pending(struct kvm_vcpu *vcpu) 887 { 888 u8 val; 889 890 if (pv_eoi_get_user(vcpu, &val) < 0) 891 return false; 892 893 val &= KVM_PV_EOI_ENABLED; 894 895 if (val && pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) 896 return false; 897 898 /* 899 * Clear pending bit in any case: it will be set again on vmentry. 900 * While this might not be ideal from performance point of view, 901 * this makes sure pv eoi is only enabled when we know it's safe. 902 */ 903 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 904 905 return val; 906 } 907 908 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 909 { 910 int highest_irr; 911 if (kvm_x86_ops.sync_pir_to_irr) 912 highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu); 913 else 914 highest_irr = apic_find_highest_irr(apic); 915 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 916 return -1; 917 return highest_irr; 918 } 919 920 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 921 { 922 u32 tpr, isrv, ppr, old_ppr; 923 int isr; 924 925 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 926 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 927 isr = apic_find_highest_isr(apic); 928 isrv = (isr != -1) ? isr : 0; 929 930 if ((tpr & 0xf0) >= (isrv & 0xf0)) 931 ppr = tpr & 0xff; 932 else 933 ppr = isrv & 0xf0; 934 935 *new_ppr = ppr; 936 if (old_ppr != ppr) 937 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 938 939 return ppr < old_ppr; 940 } 941 942 static void apic_update_ppr(struct kvm_lapic *apic) 943 { 944 u32 ppr; 945 946 if (__apic_update_ppr(apic, &ppr) && 947 apic_has_interrupt_for_ppr(apic, ppr) != -1) 948 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 949 } 950 951 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 952 { 953 apic_update_ppr(vcpu->arch.apic); 954 } 955 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 956 957 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 958 { 959 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 960 apic_update_ppr(apic); 961 } 962 963 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 964 { 965 return mda == (apic_x2apic_mode(apic) ? 966 X2APIC_BROADCAST : APIC_BROADCAST); 967 } 968 969 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 970 { 971 if (kvm_apic_broadcast(apic, mda)) 972 return true; 973 974 /* 975 * Hotplug hack: Accept interrupts for vCPUs in xAPIC mode as if they 976 * were in x2APIC mode if the target APIC ID can't be encoded as an 977 * xAPIC ID. This allows unique addressing of hotplugged vCPUs (which 978 * start in xAPIC mode) with an APIC ID that is unaddressable in xAPIC 979 * mode. Match the x2APIC ID if and only if the target APIC ID can't 980 * be encoded in xAPIC to avoid spurious matches against a vCPU that 981 * changed its (addressable) xAPIC ID (which is writable). 982 */ 983 if (apic_x2apic_mode(apic) || mda > 0xff) 984 return mda == kvm_x2apic_id(apic); 985 986 return mda == kvm_xapic_id(apic); 987 } 988 989 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 990 { 991 u32 logical_id; 992 993 if (kvm_apic_broadcast(apic, mda)) 994 return true; 995 996 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 997 998 if (apic_x2apic_mode(apic)) 999 return ((logical_id >> 16) == (mda >> 16)) 1000 && (logical_id & mda & 0xffff) != 0; 1001 1002 logical_id = GET_APIC_LOGICAL_ID(logical_id); 1003 1004 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 1005 case APIC_DFR_FLAT: 1006 return (logical_id & mda) != 0; 1007 case APIC_DFR_CLUSTER: 1008 return ((logical_id >> 4) == (mda >> 4)) 1009 && (logical_id & mda & 0xf) != 0; 1010 default: 1011 return false; 1012 } 1013 } 1014 1015 /* The KVM local APIC implementation has two quirks: 1016 * 1017 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 1018 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 1019 * KVM doesn't do that aliasing. 1020 * 1021 * - in-kernel IOAPIC messages have to be delivered directly to 1022 * x2APIC, because the kernel does not support interrupt remapping. 1023 * In order to support broadcast without interrupt remapping, x2APIC 1024 * rewrites the destination of non-IPI messages from APIC_BROADCAST 1025 * to X2APIC_BROADCAST. 1026 * 1027 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 1028 * important when userspace wants to use x2APIC-format MSIs, because 1029 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 1030 */ 1031 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 1032 struct kvm_lapic *source, struct kvm_lapic *target) 1033 { 1034 bool ipi = source != NULL; 1035 1036 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 1037 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 1038 return X2APIC_BROADCAST; 1039 1040 return dest_id; 1041 } 1042 1043 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 1044 int shorthand, unsigned int dest, int dest_mode) 1045 { 1046 struct kvm_lapic *target = vcpu->arch.apic; 1047 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 1048 1049 ASSERT(target); 1050 switch (shorthand) { 1051 case APIC_DEST_NOSHORT: 1052 if (dest_mode == APIC_DEST_PHYSICAL) 1053 return kvm_apic_match_physical_addr(target, mda); 1054 else 1055 return kvm_apic_match_logical_addr(target, mda); 1056 case APIC_DEST_SELF: 1057 return target == source; 1058 case APIC_DEST_ALLINC: 1059 return true; 1060 case APIC_DEST_ALLBUT: 1061 return target != source; 1062 default: 1063 return false; 1064 } 1065 } 1066 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 1067 1068 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 1069 const unsigned long *bitmap, u32 bitmap_size) 1070 { 1071 u32 mod; 1072 int i, idx = -1; 1073 1074 mod = vector % dest_vcpus; 1075 1076 for (i = 0; i <= mod; i++) { 1077 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 1078 BUG_ON(idx == bitmap_size); 1079 } 1080 1081 return idx; 1082 } 1083 1084 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 1085 { 1086 if (!kvm->arch.disabled_lapic_found) { 1087 kvm->arch.disabled_lapic_found = true; 1088 pr_info("Disabled LAPIC found during irq injection\n"); 1089 } 1090 } 1091 1092 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 1093 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 1094 { 1095 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 1096 if ((irq->dest_id == APIC_BROADCAST && 1097 map->logical_mode != KVM_APIC_MODE_X2APIC)) 1098 return true; 1099 if (irq->dest_id == X2APIC_BROADCAST) 1100 return true; 1101 } else { 1102 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 1103 if (irq->dest_id == (x2apic_ipi ? 1104 X2APIC_BROADCAST : APIC_BROADCAST)) 1105 return true; 1106 } 1107 1108 return false; 1109 } 1110 1111 /* Return true if the interrupt can be handled by using *bitmap as index mask 1112 * for valid destinations in *dst array. 1113 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 1114 * Note: we may have zero kvm_lapic destinations when we return true, which 1115 * means that the interrupt should be dropped. In this case, *bitmap would be 1116 * zero and *dst undefined. 1117 */ 1118 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 1119 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 1120 struct kvm_apic_map *map, struct kvm_lapic ***dst, 1121 unsigned long *bitmap) 1122 { 1123 int i, lowest; 1124 1125 if (irq->shorthand == APIC_DEST_SELF && src) { 1126 *dst = src; 1127 *bitmap = 1; 1128 return true; 1129 } else if (irq->shorthand) 1130 return false; 1131 1132 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 1133 return false; 1134 1135 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 1136 if (irq->dest_id > map->max_apic_id) { 1137 *bitmap = 0; 1138 } else { 1139 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1); 1140 *dst = &map->phys_map[dest_id]; 1141 *bitmap = 1; 1142 } 1143 return true; 1144 } 1145 1146 *bitmap = 0; 1147 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 1148 (u16 *)bitmap)) 1149 return false; 1150 1151 if (!kvm_lowest_prio_delivery(irq)) 1152 return true; 1153 1154 if (!kvm_vector_hashing_enabled()) { 1155 lowest = -1; 1156 for_each_set_bit(i, bitmap, 16) { 1157 if (!(*dst)[i]) 1158 continue; 1159 if (lowest < 0) 1160 lowest = i; 1161 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 1162 (*dst)[lowest]->vcpu) < 0) 1163 lowest = i; 1164 } 1165 } else { 1166 if (!*bitmap) 1167 return true; 1168 1169 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 1170 bitmap, 16); 1171 1172 if (!(*dst)[lowest]) { 1173 kvm_apic_disabled_lapic_found(kvm); 1174 *bitmap = 0; 1175 return true; 1176 } 1177 } 1178 1179 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 1180 1181 return true; 1182 } 1183 1184 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 1185 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 1186 { 1187 struct kvm_apic_map *map; 1188 unsigned long bitmap; 1189 struct kvm_lapic **dst = NULL; 1190 int i; 1191 bool ret; 1192 1193 *r = -1; 1194 1195 if (irq->shorthand == APIC_DEST_SELF) { 1196 if (KVM_BUG_ON(!src, kvm)) { 1197 *r = 0; 1198 return true; 1199 } 1200 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 1201 return true; 1202 } 1203 1204 rcu_read_lock(); 1205 map = rcu_dereference(kvm->arch.apic_map); 1206 1207 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 1208 if (ret) { 1209 *r = 0; 1210 for_each_set_bit(i, &bitmap, 16) { 1211 if (!dst[i]) 1212 continue; 1213 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 1214 } 1215 } 1216 1217 rcu_read_unlock(); 1218 return ret; 1219 } 1220 1221 /* 1222 * This routine tries to handle interrupts in posted mode, here is how 1223 * it deals with different cases: 1224 * - For single-destination interrupts, handle it in posted mode 1225 * - Else if vector hashing is enabled and it is a lowest-priority 1226 * interrupt, handle it in posted mode and use the following mechanism 1227 * to find the destination vCPU. 1228 * 1. For lowest-priority interrupts, store all the possible 1229 * destination vCPUs in an array. 1230 * 2. Use "guest vector % max number of destination vCPUs" to find 1231 * the right destination vCPU in the array for the lowest-priority 1232 * interrupt. 1233 * - Otherwise, use remapped mode to inject the interrupt. 1234 */ 1235 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 1236 struct kvm_vcpu **dest_vcpu) 1237 { 1238 struct kvm_apic_map *map; 1239 unsigned long bitmap; 1240 struct kvm_lapic **dst = NULL; 1241 bool ret = false; 1242 1243 if (irq->shorthand) 1244 return false; 1245 1246 rcu_read_lock(); 1247 map = rcu_dereference(kvm->arch.apic_map); 1248 1249 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 1250 hweight16(bitmap) == 1) { 1251 unsigned long i = find_first_bit(&bitmap, 16); 1252 1253 if (dst[i]) { 1254 *dest_vcpu = dst[i]->vcpu; 1255 ret = true; 1256 } 1257 } 1258 1259 rcu_read_unlock(); 1260 return ret; 1261 } 1262 1263 /* 1264 * Add a pending IRQ into lapic. 1265 * Return 1 if successfully added and 0 if discarded. 1266 */ 1267 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 1268 int vector, int level, int trig_mode, 1269 struct dest_map *dest_map) 1270 { 1271 int result = 0; 1272 struct kvm_vcpu *vcpu = apic->vcpu; 1273 1274 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 1275 trig_mode, vector); 1276 switch (delivery_mode) { 1277 case APIC_DM_LOWEST: 1278 vcpu->arch.apic_arb_prio++; 1279 fallthrough; 1280 case APIC_DM_FIXED: 1281 if (unlikely(trig_mode && !level)) 1282 break; 1283 1284 /* FIXME add logic for vcpu on reset */ 1285 if (unlikely(!apic_enabled(apic))) 1286 break; 1287 1288 result = 1; 1289 1290 if (dest_map) { 1291 __set_bit(vcpu->vcpu_id, dest_map->map); 1292 dest_map->vectors[vcpu->vcpu_id] = vector; 1293 } 1294 1295 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 1296 if (trig_mode) 1297 kvm_lapic_set_vector(vector, 1298 apic->regs + APIC_TMR); 1299 else 1300 kvm_lapic_clear_vector(vector, 1301 apic->regs + APIC_TMR); 1302 } 1303 1304 static_call(kvm_x86_deliver_interrupt)(apic, delivery_mode, 1305 trig_mode, vector); 1306 break; 1307 1308 case APIC_DM_REMRD: 1309 result = 1; 1310 vcpu->arch.pv.pv_unhalted = 1; 1311 kvm_make_request(KVM_REQ_EVENT, vcpu); 1312 kvm_vcpu_kick(vcpu); 1313 break; 1314 1315 case APIC_DM_SMI: 1316 if (!kvm_inject_smi(vcpu)) { 1317 kvm_vcpu_kick(vcpu); 1318 result = 1; 1319 } 1320 break; 1321 1322 case APIC_DM_NMI: 1323 result = 1; 1324 kvm_inject_nmi(vcpu); 1325 kvm_vcpu_kick(vcpu); 1326 break; 1327 1328 case APIC_DM_INIT: 1329 if (!trig_mode || level) { 1330 result = 1; 1331 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1332 apic->pending_events = (1UL << KVM_APIC_INIT); 1333 kvm_make_request(KVM_REQ_EVENT, vcpu); 1334 kvm_vcpu_kick(vcpu); 1335 } 1336 break; 1337 1338 case APIC_DM_STARTUP: 1339 result = 1; 1340 apic->sipi_vector = vector; 1341 /* make sure sipi_vector is visible for the receiver */ 1342 smp_wmb(); 1343 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1344 kvm_make_request(KVM_REQ_EVENT, vcpu); 1345 kvm_vcpu_kick(vcpu); 1346 break; 1347 1348 case APIC_DM_EXTINT: 1349 /* 1350 * Should only be called by kvm_apic_local_deliver() with LVT0, 1351 * before NMI watchdog was enabled. Already handled by 1352 * kvm_apic_accept_pic_intr(). 1353 */ 1354 break; 1355 1356 default: 1357 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1358 delivery_mode); 1359 break; 1360 } 1361 return result; 1362 } 1363 1364 /* 1365 * This routine identifies the destination vcpus mask meant to receive the 1366 * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find 1367 * out the destination vcpus array and set the bitmap or it traverses to 1368 * each available vcpu to identify the same. 1369 */ 1370 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, 1371 unsigned long *vcpu_bitmap) 1372 { 1373 struct kvm_lapic **dest_vcpu = NULL; 1374 struct kvm_lapic *src = NULL; 1375 struct kvm_apic_map *map; 1376 struct kvm_vcpu *vcpu; 1377 unsigned long bitmap, i; 1378 int vcpu_idx; 1379 bool ret; 1380 1381 rcu_read_lock(); 1382 map = rcu_dereference(kvm->arch.apic_map); 1383 1384 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu, 1385 &bitmap); 1386 if (ret) { 1387 for_each_set_bit(i, &bitmap, 16) { 1388 if (!dest_vcpu[i]) 1389 continue; 1390 vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx; 1391 __set_bit(vcpu_idx, vcpu_bitmap); 1392 } 1393 } else { 1394 kvm_for_each_vcpu(i, vcpu, kvm) { 1395 if (!kvm_apic_present(vcpu)) 1396 continue; 1397 if (!kvm_apic_match_dest(vcpu, NULL, 1398 irq->shorthand, 1399 irq->dest_id, 1400 irq->dest_mode)) 1401 continue; 1402 __set_bit(i, vcpu_bitmap); 1403 } 1404 } 1405 rcu_read_unlock(); 1406 } 1407 1408 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1409 { 1410 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1411 } 1412 1413 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1414 { 1415 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1416 } 1417 1418 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1419 { 1420 int trigger_mode; 1421 1422 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1423 if (!kvm_ioapic_handles_vector(apic, vector)) 1424 return; 1425 1426 /* Request a KVM exit to inform the userspace IOAPIC. */ 1427 if (irqchip_split(apic->vcpu->kvm)) { 1428 apic->vcpu->arch.pending_ioapic_eoi = vector; 1429 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1430 return; 1431 } 1432 1433 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1434 trigger_mode = IOAPIC_LEVEL_TRIG; 1435 else 1436 trigger_mode = IOAPIC_EDGE_TRIG; 1437 1438 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1439 } 1440 1441 static int apic_set_eoi(struct kvm_lapic *apic) 1442 { 1443 int vector = apic_find_highest_isr(apic); 1444 1445 trace_kvm_eoi(apic, vector); 1446 1447 /* 1448 * Not every write EOI will has corresponding ISR, 1449 * one example is when Kernel check timer on setup_IO_APIC 1450 */ 1451 if (vector == -1) 1452 return vector; 1453 1454 apic_clear_isr(vector, apic); 1455 apic_update_ppr(apic); 1456 1457 if (to_hv_vcpu(apic->vcpu) && 1458 test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap)) 1459 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1460 1461 kvm_ioapic_send_eoi(apic, vector); 1462 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1463 return vector; 1464 } 1465 1466 /* 1467 * this interface assumes a trap-like exit, which has already finished 1468 * desired side effect including vISR and vPPR update. 1469 */ 1470 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1471 { 1472 struct kvm_lapic *apic = vcpu->arch.apic; 1473 1474 trace_kvm_eoi(apic, vector); 1475 1476 kvm_ioapic_send_eoi(apic, vector); 1477 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1478 } 1479 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1480 1481 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high) 1482 { 1483 struct kvm_lapic_irq irq; 1484 1485 /* KVM has no delay and should always clear the BUSY/PENDING flag. */ 1486 WARN_ON_ONCE(icr_low & APIC_ICR_BUSY); 1487 1488 irq.vector = icr_low & APIC_VECTOR_MASK; 1489 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1490 irq.dest_mode = icr_low & APIC_DEST_MASK; 1491 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1492 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1493 irq.shorthand = icr_low & APIC_SHORT_MASK; 1494 irq.msi_redir_hint = false; 1495 if (apic_x2apic_mode(apic)) 1496 irq.dest_id = icr_high; 1497 else 1498 irq.dest_id = GET_XAPIC_DEST_FIELD(icr_high); 1499 1500 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1501 1502 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1503 } 1504 EXPORT_SYMBOL_GPL(kvm_apic_send_ipi); 1505 1506 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1507 { 1508 ktime_t remaining, now; 1509 s64 ns; 1510 1511 ASSERT(apic != NULL); 1512 1513 /* if initial count is 0, current count should also be 0 */ 1514 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1515 apic->lapic_timer.period == 0) 1516 return 0; 1517 1518 now = ktime_get(); 1519 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1520 if (ktime_to_ns(remaining) < 0) 1521 remaining = 0; 1522 1523 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1524 return div64_u64(ns, (APIC_BUS_CYCLE_NS * apic->divide_count)); 1525 } 1526 1527 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1528 { 1529 struct kvm_vcpu *vcpu = apic->vcpu; 1530 struct kvm_run *run = vcpu->run; 1531 1532 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1533 run->tpr_access.rip = kvm_rip_read(vcpu); 1534 run->tpr_access.is_write = write; 1535 } 1536 1537 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1538 { 1539 if (apic->vcpu->arch.tpr_access_reporting) 1540 __report_tpr_access(apic, write); 1541 } 1542 1543 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1544 { 1545 u32 val = 0; 1546 1547 if (offset >= LAPIC_MMIO_LENGTH) 1548 return 0; 1549 1550 switch (offset) { 1551 case APIC_ARBPRI: 1552 break; 1553 1554 case APIC_TMCCT: /* Timer CCR */ 1555 if (apic_lvtt_tscdeadline(apic)) 1556 return 0; 1557 1558 val = apic_get_tmcct(apic); 1559 break; 1560 case APIC_PROCPRI: 1561 apic_update_ppr(apic); 1562 val = kvm_lapic_get_reg(apic, offset); 1563 break; 1564 case APIC_TASKPRI: 1565 report_tpr_access(apic, false); 1566 fallthrough; 1567 default: 1568 val = kvm_lapic_get_reg(apic, offset); 1569 break; 1570 } 1571 1572 return val; 1573 } 1574 1575 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1576 { 1577 return container_of(dev, struct kvm_lapic, dev); 1578 } 1579 1580 #define APIC_REG_MASK(reg) (1ull << ((reg) >> 4)) 1581 #define APIC_REGS_MASK(first, count) \ 1582 (APIC_REG_MASK(first) * ((1ull << (count)) - 1)) 1583 1584 u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic) 1585 { 1586 /* Leave bits '0' for reserved and write-only registers. */ 1587 u64 valid_reg_mask = 1588 APIC_REG_MASK(APIC_ID) | 1589 APIC_REG_MASK(APIC_LVR) | 1590 APIC_REG_MASK(APIC_TASKPRI) | 1591 APIC_REG_MASK(APIC_PROCPRI) | 1592 APIC_REG_MASK(APIC_LDR) | 1593 APIC_REG_MASK(APIC_SPIV) | 1594 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) | 1595 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) | 1596 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) | 1597 APIC_REG_MASK(APIC_ESR) | 1598 APIC_REG_MASK(APIC_ICR) | 1599 APIC_REG_MASK(APIC_LVTT) | 1600 APIC_REG_MASK(APIC_LVTTHMR) | 1601 APIC_REG_MASK(APIC_LVTPC) | 1602 APIC_REG_MASK(APIC_LVT0) | 1603 APIC_REG_MASK(APIC_LVT1) | 1604 APIC_REG_MASK(APIC_LVTERR) | 1605 APIC_REG_MASK(APIC_TMICT) | 1606 APIC_REG_MASK(APIC_TMCCT) | 1607 APIC_REG_MASK(APIC_TDCR); 1608 1609 if (kvm_lapic_lvt_supported(apic, LVT_CMCI)) 1610 valid_reg_mask |= APIC_REG_MASK(APIC_LVTCMCI); 1611 1612 /* ARBPRI, DFR, and ICR2 are not valid in x2APIC mode. */ 1613 if (!apic_x2apic_mode(apic)) 1614 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI) | 1615 APIC_REG_MASK(APIC_DFR) | 1616 APIC_REG_MASK(APIC_ICR2); 1617 1618 return valid_reg_mask; 1619 } 1620 EXPORT_SYMBOL_GPL(kvm_lapic_readable_reg_mask); 1621 1622 static int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1623 void *data) 1624 { 1625 unsigned char alignment = offset & 0xf; 1626 u32 result; 1627 1628 /* 1629 * WARN if KVM reads ICR in x2APIC mode, as it's an 8-byte register in 1630 * x2APIC and needs to be manually handled by the caller. 1631 */ 1632 WARN_ON_ONCE(apic_x2apic_mode(apic) && offset == APIC_ICR); 1633 1634 if (alignment + len > 4) 1635 return 1; 1636 1637 if (offset > 0x3f0 || 1638 !(kvm_lapic_readable_reg_mask(apic) & APIC_REG_MASK(offset))) 1639 return 1; 1640 1641 result = __apic_read(apic, offset & ~0xf); 1642 1643 trace_kvm_apic_read(offset, result); 1644 1645 switch (len) { 1646 case 1: 1647 case 2: 1648 case 4: 1649 memcpy(data, (char *)&result + alignment, len); 1650 break; 1651 default: 1652 printk(KERN_ERR "Local APIC read with len = %x, " 1653 "should be 1,2, or 4 instead\n", len); 1654 break; 1655 } 1656 return 0; 1657 } 1658 1659 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1660 { 1661 return addr >= apic->base_address && 1662 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1663 } 1664 1665 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1666 gpa_t address, int len, void *data) 1667 { 1668 struct kvm_lapic *apic = to_lapic(this); 1669 u32 offset = address - apic->base_address; 1670 1671 if (!apic_mmio_in_range(apic, address)) 1672 return -EOPNOTSUPP; 1673 1674 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 1675 if (!kvm_check_has_quirk(vcpu->kvm, 1676 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 1677 return -EOPNOTSUPP; 1678 1679 memset(data, 0xff, len); 1680 return 0; 1681 } 1682 1683 kvm_lapic_reg_read(apic, offset, len, data); 1684 1685 return 0; 1686 } 1687 1688 static void update_divide_count(struct kvm_lapic *apic) 1689 { 1690 u32 tmp1, tmp2, tdcr; 1691 1692 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1693 tmp1 = tdcr & 0xf; 1694 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1695 apic->divide_count = 0x1 << (tmp2 & 0x7); 1696 } 1697 1698 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1699 { 1700 /* 1701 * Do not allow the guest to program periodic timers with small 1702 * interval, since the hrtimers are not throttled by the host 1703 * scheduler. 1704 */ 1705 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1706 s64 min_period = min_timer_period_us * 1000LL; 1707 1708 if (apic->lapic_timer.period < min_period) { 1709 pr_info_ratelimited( 1710 "vcpu %i: requested %lld ns " 1711 "lapic timer period limited to %lld ns\n", 1712 apic->vcpu->vcpu_id, 1713 apic->lapic_timer.period, min_period); 1714 apic->lapic_timer.period = min_period; 1715 } 1716 } 1717 } 1718 1719 static void cancel_hv_timer(struct kvm_lapic *apic); 1720 1721 static void cancel_apic_timer(struct kvm_lapic *apic) 1722 { 1723 hrtimer_cancel(&apic->lapic_timer.timer); 1724 preempt_disable(); 1725 if (apic->lapic_timer.hv_timer_in_use) 1726 cancel_hv_timer(apic); 1727 preempt_enable(); 1728 atomic_set(&apic->lapic_timer.pending, 0); 1729 } 1730 1731 static void apic_update_lvtt(struct kvm_lapic *apic) 1732 { 1733 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1734 apic->lapic_timer.timer_mode_mask; 1735 1736 if (apic->lapic_timer.timer_mode != timer_mode) { 1737 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1738 APIC_LVT_TIMER_TSCDEADLINE)) { 1739 cancel_apic_timer(apic); 1740 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1741 apic->lapic_timer.period = 0; 1742 apic->lapic_timer.tscdeadline = 0; 1743 } 1744 apic->lapic_timer.timer_mode = timer_mode; 1745 limit_periodic_timer_frequency(apic); 1746 } 1747 } 1748 1749 /* 1750 * On APICv, this test will cause a busy wait 1751 * during a higher-priority task. 1752 */ 1753 1754 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1755 { 1756 struct kvm_lapic *apic = vcpu->arch.apic; 1757 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1758 1759 if (kvm_apic_hw_enabled(apic)) { 1760 int vec = reg & APIC_VECTOR_MASK; 1761 void *bitmap = apic->regs + APIC_ISR; 1762 1763 if (apic->apicv_active) 1764 bitmap = apic->regs + APIC_IRR; 1765 1766 if (apic_test_vector(vec, bitmap)) 1767 return true; 1768 } 1769 return false; 1770 } 1771 1772 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles) 1773 { 1774 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns; 1775 1776 /* 1777 * If the guest TSC is running at a different ratio than the host, then 1778 * convert the delay to nanoseconds to achieve an accurate delay. Note 1779 * that __delay() uses delay_tsc whenever the hardware has TSC, thus 1780 * always for VMX enabled hardware. 1781 */ 1782 if (vcpu->arch.tsc_scaling_ratio == kvm_caps.default_tsc_scaling_ratio) { 1783 __delay(min(guest_cycles, 1784 nsec_to_cycles(vcpu, timer_advance_ns))); 1785 } else { 1786 u64 delay_ns = guest_cycles * 1000000ULL; 1787 do_div(delay_ns, vcpu->arch.virtual_tsc_khz); 1788 ndelay(min_t(u32, delay_ns, timer_advance_ns)); 1789 } 1790 } 1791 1792 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu, 1793 s64 advance_expire_delta) 1794 { 1795 struct kvm_lapic *apic = vcpu->arch.apic; 1796 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns; 1797 u64 ns; 1798 1799 /* Do not adjust for tiny fluctuations or large random spikes. */ 1800 if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX || 1801 abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN) 1802 return; 1803 1804 /* too early */ 1805 if (advance_expire_delta < 0) { 1806 ns = -advance_expire_delta * 1000000ULL; 1807 do_div(ns, vcpu->arch.virtual_tsc_khz); 1808 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP; 1809 } else { 1810 /* too late */ 1811 ns = advance_expire_delta * 1000000ULL; 1812 do_div(ns, vcpu->arch.virtual_tsc_khz); 1813 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP; 1814 } 1815 1816 if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX)) 1817 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; 1818 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 1819 } 1820 1821 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1822 { 1823 struct kvm_lapic *apic = vcpu->arch.apic; 1824 u64 guest_tsc, tsc_deadline; 1825 1826 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1827 apic->lapic_timer.expired_tscdeadline = 0; 1828 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1829 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1830 1831 if (lapic_timer_advance_dynamic) { 1832 adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline); 1833 /* 1834 * If the timer fired early, reread the TSC to account for the 1835 * overhead of the above adjustment to avoid waiting longer 1836 * than is necessary. 1837 */ 1838 if (guest_tsc < tsc_deadline) 1839 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1840 } 1841 1842 if (guest_tsc < tsc_deadline) 1843 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc); 1844 } 1845 1846 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1847 { 1848 if (lapic_in_kernel(vcpu) && 1849 vcpu->arch.apic->lapic_timer.expired_tscdeadline && 1850 vcpu->arch.apic->lapic_timer.timer_advance_ns && 1851 lapic_timer_int_injected(vcpu)) 1852 __kvm_wait_lapic_expire(vcpu); 1853 } 1854 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire); 1855 1856 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic) 1857 { 1858 struct kvm_timer *ktimer = &apic->lapic_timer; 1859 1860 kvm_apic_local_deliver(apic, APIC_LVTT); 1861 if (apic_lvtt_tscdeadline(apic)) { 1862 ktimer->tscdeadline = 0; 1863 } else if (apic_lvtt_oneshot(apic)) { 1864 ktimer->tscdeadline = 0; 1865 ktimer->target_expiration = 0; 1866 } 1867 } 1868 1869 static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) 1870 { 1871 struct kvm_vcpu *vcpu = apic->vcpu; 1872 struct kvm_timer *ktimer = &apic->lapic_timer; 1873 1874 if (atomic_read(&apic->lapic_timer.pending)) 1875 return; 1876 1877 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) 1878 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1879 1880 if (!from_timer_fn && apic->apicv_active) { 1881 WARN_ON(kvm_get_running_vcpu() != vcpu); 1882 kvm_apic_inject_pending_timer_irqs(apic); 1883 return; 1884 } 1885 1886 if (kvm_use_posted_timer_interrupt(apic->vcpu)) { 1887 /* 1888 * Ensure the guest's timer has truly expired before posting an 1889 * interrupt. Open code the relevant checks to avoid querying 1890 * lapic_timer_int_injected(), which will be false since the 1891 * interrupt isn't yet injected. Waiting until after injecting 1892 * is not an option since that won't help a posted interrupt. 1893 */ 1894 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline && 1895 vcpu->arch.apic->lapic_timer.timer_advance_ns) 1896 __kvm_wait_lapic_expire(vcpu); 1897 kvm_apic_inject_pending_timer_irqs(apic); 1898 return; 1899 } 1900 1901 atomic_inc(&apic->lapic_timer.pending); 1902 kvm_make_request(KVM_REQ_UNBLOCK, vcpu); 1903 if (from_timer_fn) 1904 kvm_vcpu_kick(vcpu); 1905 } 1906 1907 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1908 { 1909 struct kvm_timer *ktimer = &apic->lapic_timer; 1910 u64 guest_tsc, tscdeadline = ktimer->tscdeadline; 1911 u64 ns = 0; 1912 ktime_t expire; 1913 struct kvm_vcpu *vcpu = apic->vcpu; 1914 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1915 unsigned long flags; 1916 ktime_t now; 1917 1918 if (unlikely(!tscdeadline || !this_tsc_khz)) 1919 return; 1920 1921 local_irq_save(flags); 1922 1923 now = ktime_get(); 1924 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1925 1926 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1927 do_div(ns, this_tsc_khz); 1928 1929 if (likely(tscdeadline > guest_tsc) && 1930 likely(ns > apic->lapic_timer.timer_advance_ns)) { 1931 expire = ktime_add_ns(now, ns); 1932 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns); 1933 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD); 1934 } else 1935 apic_timer_expired(apic, false); 1936 1937 local_irq_restore(flags); 1938 } 1939 1940 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict) 1941 { 1942 return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count; 1943 } 1944 1945 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1946 { 1947 ktime_t now, remaining; 1948 u64 ns_remaining_old, ns_remaining_new; 1949 1950 apic->lapic_timer.period = 1951 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); 1952 limit_periodic_timer_frequency(apic); 1953 1954 now = ktime_get(); 1955 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1956 if (ktime_to_ns(remaining) < 0) 1957 remaining = 0; 1958 1959 ns_remaining_old = ktime_to_ns(remaining); 1960 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1961 apic->divide_count, old_divisor); 1962 1963 apic->lapic_timer.tscdeadline += 1964 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1965 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1966 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1967 } 1968 1969 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg) 1970 { 1971 ktime_t now; 1972 u64 tscl = rdtsc(); 1973 s64 deadline; 1974 1975 now = ktime_get(); 1976 apic->lapic_timer.period = 1977 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); 1978 1979 if (!apic->lapic_timer.period) { 1980 apic->lapic_timer.tscdeadline = 0; 1981 return false; 1982 } 1983 1984 limit_periodic_timer_frequency(apic); 1985 deadline = apic->lapic_timer.period; 1986 1987 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { 1988 if (unlikely(count_reg != APIC_TMICT)) { 1989 deadline = tmict_to_ns(apic, 1990 kvm_lapic_get_reg(apic, count_reg)); 1991 if (unlikely(deadline <= 0)) { 1992 if (apic_lvtt_period(apic)) 1993 deadline = apic->lapic_timer.period; 1994 else 1995 deadline = 0; 1996 } 1997 else if (unlikely(deadline > apic->lapic_timer.period)) { 1998 pr_info_ratelimited( 1999 "vcpu %i: requested lapic timer restore with " 2000 "starting count register %#x=%u (%lld ns) > initial count (%lld ns). " 2001 "Using initial count to start timer.\n", 2002 apic->vcpu->vcpu_id, 2003 count_reg, 2004 kvm_lapic_get_reg(apic, count_reg), 2005 deadline, apic->lapic_timer.period); 2006 kvm_lapic_set_reg(apic, count_reg, 0); 2007 deadline = apic->lapic_timer.period; 2008 } 2009 } 2010 } 2011 2012 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 2013 nsec_to_cycles(apic->vcpu, deadline); 2014 apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline); 2015 2016 return true; 2017 } 2018 2019 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 2020 { 2021 ktime_t now = ktime_get(); 2022 u64 tscl = rdtsc(); 2023 ktime_t delta; 2024 2025 /* 2026 * Synchronize both deadlines to the same time source or 2027 * differences in the periods (caused by differences in the 2028 * underlying clocks or numerical approximation errors) will 2029 * cause the two to drift apart over time as the errors 2030 * accumulate. 2031 */ 2032 apic->lapic_timer.target_expiration = 2033 ktime_add_ns(apic->lapic_timer.target_expiration, 2034 apic->lapic_timer.period); 2035 delta = ktime_sub(apic->lapic_timer.target_expiration, now); 2036 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 2037 nsec_to_cycles(apic->vcpu, delta); 2038 } 2039 2040 static void start_sw_period(struct kvm_lapic *apic) 2041 { 2042 if (!apic->lapic_timer.period) 2043 return; 2044 2045 if (ktime_after(ktime_get(), 2046 apic->lapic_timer.target_expiration)) { 2047 apic_timer_expired(apic, false); 2048 2049 if (apic_lvtt_oneshot(apic)) 2050 return; 2051 2052 advance_periodic_target_expiration(apic); 2053 } 2054 2055 hrtimer_start(&apic->lapic_timer.timer, 2056 apic->lapic_timer.target_expiration, 2057 HRTIMER_MODE_ABS_HARD); 2058 } 2059 2060 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 2061 { 2062 if (!lapic_in_kernel(vcpu)) 2063 return false; 2064 2065 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 2066 } 2067 2068 static void cancel_hv_timer(struct kvm_lapic *apic) 2069 { 2070 WARN_ON(preemptible()); 2071 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 2072 static_call(kvm_x86_cancel_hv_timer)(apic->vcpu); 2073 apic->lapic_timer.hv_timer_in_use = false; 2074 } 2075 2076 static bool start_hv_timer(struct kvm_lapic *apic) 2077 { 2078 struct kvm_timer *ktimer = &apic->lapic_timer; 2079 struct kvm_vcpu *vcpu = apic->vcpu; 2080 bool expired; 2081 2082 WARN_ON(preemptible()); 2083 if (!kvm_can_use_hv_timer(vcpu)) 2084 return false; 2085 2086 if (!ktimer->tscdeadline) 2087 return false; 2088 2089 if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired)) 2090 return false; 2091 2092 ktimer->hv_timer_in_use = true; 2093 hrtimer_cancel(&ktimer->timer); 2094 2095 /* 2096 * To simplify handling the periodic timer, leave the hv timer running 2097 * even if the deadline timer has expired, i.e. rely on the resulting 2098 * VM-Exit to recompute the periodic timer's target expiration. 2099 */ 2100 if (!apic_lvtt_period(apic)) { 2101 /* 2102 * Cancel the hv timer if the sw timer fired while the hv timer 2103 * was being programmed, or if the hv timer itself expired. 2104 */ 2105 if (atomic_read(&ktimer->pending)) { 2106 cancel_hv_timer(apic); 2107 } else if (expired) { 2108 apic_timer_expired(apic, false); 2109 cancel_hv_timer(apic); 2110 } 2111 } 2112 2113 trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use); 2114 2115 return true; 2116 } 2117 2118 static void start_sw_timer(struct kvm_lapic *apic) 2119 { 2120 struct kvm_timer *ktimer = &apic->lapic_timer; 2121 2122 WARN_ON(preemptible()); 2123 if (apic->lapic_timer.hv_timer_in_use) 2124 cancel_hv_timer(apic); 2125 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 2126 return; 2127 2128 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 2129 start_sw_period(apic); 2130 else if (apic_lvtt_tscdeadline(apic)) 2131 start_sw_tscdeadline(apic); 2132 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 2133 } 2134 2135 static void restart_apic_timer(struct kvm_lapic *apic) 2136 { 2137 preempt_disable(); 2138 2139 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending)) 2140 goto out; 2141 2142 if (!start_hv_timer(apic)) 2143 start_sw_timer(apic); 2144 out: 2145 preempt_enable(); 2146 } 2147 2148 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 2149 { 2150 struct kvm_lapic *apic = vcpu->arch.apic; 2151 2152 preempt_disable(); 2153 /* If the preempt notifier has already run, it also called apic_timer_expired */ 2154 if (!apic->lapic_timer.hv_timer_in_use) 2155 goto out; 2156 WARN_ON(kvm_vcpu_is_blocking(vcpu)); 2157 apic_timer_expired(apic, false); 2158 cancel_hv_timer(apic); 2159 2160 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 2161 advance_periodic_target_expiration(apic); 2162 restart_apic_timer(apic); 2163 } 2164 out: 2165 preempt_enable(); 2166 } 2167 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 2168 2169 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 2170 { 2171 restart_apic_timer(vcpu->arch.apic); 2172 } 2173 2174 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 2175 { 2176 struct kvm_lapic *apic = vcpu->arch.apic; 2177 2178 preempt_disable(); 2179 /* Possibly the TSC deadline timer is not enabled yet */ 2180 if (apic->lapic_timer.hv_timer_in_use) 2181 start_sw_timer(apic); 2182 preempt_enable(); 2183 } 2184 2185 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 2186 { 2187 struct kvm_lapic *apic = vcpu->arch.apic; 2188 2189 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 2190 restart_apic_timer(apic); 2191 } 2192 2193 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg) 2194 { 2195 atomic_set(&apic->lapic_timer.pending, 0); 2196 2197 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 2198 && !set_target_expiration(apic, count_reg)) 2199 return; 2200 2201 restart_apic_timer(apic); 2202 } 2203 2204 static void start_apic_timer(struct kvm_lapic *apic) 2205 { 2206 __start_apic_timer(apic, APIC_TMICT); 2207 } 2208 2209 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 2210 { 2211 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 2212 2213 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 2214 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 2215 if (lvt0_in_nmi_mode) { 2216 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 2217 } else 2218 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 2219 } 2220 } 2221 2222 static int get_lvt_index(u32 reg) 2223 { 2224 if (reg == APIC_LVTCMCI) 2225 return LVT_CMCI; 2226 if (reg < APIC_LVTT || reg > APIC_LVTERR) 2227 return -1; 2228 return array_index_nospec( 2229 (reg - APIC_LVTT) >> 4, KVM_APIC_MAX_NR_LVT_ENTRIES); 2230 } 2231 2232 static int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 2233 { 2234 int ret = 0; 2235 2236 trace_kvm_apic_write(reg, val); 2237 2238 switch (reg) { 2239 case APIC_ID: /* Local APIC ID */ 2240 if (!apic_x2apic_mode(apic)) { 2241 kvm_apic_set_xapic_id(apic, val >> 24); 2242 } else { 2243 ret = 1; 2244 } 2245 break; 2246 2247 case APIC_TASKPRI: 2248 report_tpr_access(apic, true); 2249 apic_set_tpr(apic, val & 0xff); 2250 break; 2251 2252 case APIC_EOI: 2253 apic_set_eoi(apic); 2254 break; 2255 2256 case APIC_LDR: 2257 if (!apic_x2apic_mode(apic)) 2258 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 2259 else 2260 ret = 1; 2261 break; 2262 2263 case APIC_DFR: 2264 if (!apic_x2apic_mode(apic)) 2265 kvm_apic_set_dfr(apic, val | 0x0FFFFFFF); 2266 else 2267 ret = 1; 2268 break; 2269 2270 case APIC_SPIV: { 2271 u32 mask = 0x3ff; 2272 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 2273 mask |= APIC_SPIV_DIRECTED_EOI; 2274 apic_set_spiv(apic, val & mask); 2275 if (!(val & APIC_SPIV_APIC_ENABLED)) { 2276 int i; 2277 2278 for (i = 0; i < apic->nr_lvt_entries; i++) { 2279 kvm_lapic_set_reg(apic, APIC_LVTx(i), 2280 kvm_lapic_get_reg(apic, APIC_LVTx(i)) | APIC_LVT_MASKED); 2281 } 2282 apic_update_lvtt(apic); 2283 atomic_set(&apic->lapic_timer.pending, 0); 2284 2285 } 2286 break; 2287 } 2288 case APIC_ICR: 2289 WARN_ON_ONCE(apic_x2apic_mode(apic)); 2290 2291 /* No delay here, so we always clear the pending bit */ 2292 val &= ~APIC_ICR_BUSY; 2293 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2)); 2294 kvm_lapic_set_reg(apic, APIC_ICR, val); 2295 break; 2296 case APIC_ICR2: 2297 if (apic_x2apic_mode(apic)) 2298 ret = 1; 2299 else 2300 kvm_lapic_set_reg(apic, APIC_ICR2, val & 0xff000000); 2301 break; 2302 2303 case APIC_LVT0: 2304 apic_manage_nmi_watchdog(apic, val); 2305 fallthrough; 2306 case APIC_LVTTHMR: 2307 case APIC_LVTPC: 2308 case APIC_LVT1: 2309 case APIC_LVTERR: 2310 case APIC_LVTCMCI: { 2311 u32 index = get_lvt_index(reg); 2312 if (!kvm_lapic_lvt_supported(apic, index)) { 2313 ret = 1; 2314 break; 2315 } 2316 if (!kvm_apic_sw_enabled(apic)) 2317 val |= APIC_LVT_MASKED; 2318 val &= apic_lvt_mask[index]; 2319 kvm_lapic_set_reg(apic, reg, val); 2320 break; 2321 } 2322 2323 case APIC_LVTT: 2324 if (!kvm_apic_sw_enabled(apic)) 2325 val |= APIC_LVT_MASKED; 2326 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 2327 kvm_lapic_set_reg(apic, APIC_LVTT, val); 2328 apic_update_lvtt(apic); 2329 break; 2330 2331 case APIC_TMICT: 2332 if (apic_lvtt_tscdeadline(apic)) 2333 break; 2334 2335 cancel_apic_timer(apic); 2336 kvm_lapic_set_reg(apic, APIC_TMICT, val); 2337 start_apic_timer(apic); 2338 break; 2339 2340 case APIC_TDCR: { 2341 uint32_t old_divisor = apic->divide_count; 2342 2343 kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb); 2344 update_divide_count(apic); 2345 if (apic->divide_count != old_divisor && 2346 apic->lapic_timer.period) { 2347 hrtimer_cancel(&apic->lapic_timer.timer); 2348 update_target_expiration(apic, old_divisor); 2349 restart_apic_timer(apic); 2350 } 2351 break; 2352 } 2353 case APIC_ESR: 2354 if (apic_x2apic_mode(apic) && val != 0) 2355 ret = 1; 2356 break; 2357 2358 case APIC_SELF_IPI: 2359 /* 2360 * Self-IPI exists only when x2APIC is enabled. Bits 7:0 hold 2361 * the vector, everything else is reserved. 2362 */ 2363 if (!apic_x2apic_mode(apic) || (val & ~APIC_VECTOR_MASK)) 2364 ret = 1; 2365 else 2366 kvm_apic_send_ipi(apic, APIC_DEST_SELF | val, 0); 2367 break; 2368 default: 2369 ret = 1; 2370 break; 2371 } 2372 2373 /* 2374 * Recalculate APIC maps if necessary, e.g. if the software enable bit 2375 * was toggled, the APIC ID changed, etc... The maps are marked dirty 2376 * on relevant changes, i.e. this is a nop for most writes. 2377 */ 2378 kvm_recalculate_apic_map(apic->vcpu->kvm); 2379 2380 return ret; 2381 } 2382 2383 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 2384 gpa_t address, int len, const void *data) 2385 { 2386 struct kvm_lapic *apic = to_lapic(this); 2387 unsigned int offset = address - apic->base_address; 2388 u32 val; 2389 2390 if (!apic_mmio_in_range(apic, address)) 2391 return -EOPNOTSUPP; 2392 2393 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 2394 if (!kvm_check_has_quirk(vcpu->kvm, 2395 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 2396 return -EOPNOTSUPP; 2397 2398 return 0; 2399 } 2400 2401 /* 2402 * APIC register must be aligned on 128-bits boundary. 2403 * 32/64/128 bits registers must be accessed thru 32 bits. 2404 * Refer SDM 8.4.1 2405 */ 2406 if (len != 4 || (offset & 0xf)) 2407 return 0; 2408 2409 val = *(u32*)data; 2410 2411 kvm_lapic_reg_write(apic, offset & 0xff0, val); 2412 2413 return 0; 2414 } 2415 2416 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 2417 { 2418 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 2419 } 2420 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 2421 2422 /* emulate APIC access in a trap manner */ 2423 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 2424 { 2425 struct kvm_lapic *apic = vcpu->arch.apic; 2426 u64 val; 2427 2428 /* 2429 * ICR is a single 64-bit register when x2APIC is enabled. For legacy 2430 * xAPIC, ICR writes need to go down the common (slightly slower) path 2431 * to get the upper half from ICR2. 2432 */ 2433 if (apic_x2apic_mode(apic) && offset == APIC_ICR) { 2434 val = kvm_lapic_get_reg64(apic, APIC_ICR); 2435 kvm_apic_send_ipi(apic, (u32)val, (u32)(val >> 32)); 2436 trace_kvm_apic_write(APIC_ICR, val); 2437 } else { 2438 /* TODO: optimize to just emulate side effect w/o one more write */ 2439 val = kvm_lapic_get_reg(apic, offset); 2440 kvm_lapic_reg_write(apic, offset, (u32)val); 2441 } 2442 } 2443 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 2444 2445 void kvm_free_lapic(struct kvm_vcpu *vcpu) 2446 { 2447 struct kvm_lapic *apic = vcpu->arch.apic; 2448 2449 if (!vcpu->arch.apic) 2450 return; 2451 2452 hrtimer_cancel(&apic->lapic_timer.timer); 2453 2454 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 2455 static_branch_slow_dec_deferred(&apic_hw_disabled); 2456 2457 if (!apic->sw_enabled) 2458 static_branch_slow_dec_deferred(&apic_sw_disabled); 2459 2460 if (apic->regs) 2461 free_page((unsigned long)apic->regs); 2462 2463 kfree(apic); 2464 } 2465 2466 /* 2467 *---------------------------------------------------------------------- 2468 * LAPIC interface 2469 *---------------------------------------------------------------------- 2470 */ 2471 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 2472 { 2473 struct kvm_lapic *apic = vcpu->arch.apic; 2474 2475 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) 2476 return 0; 2477 2478 return apic->lapic_timer.tscdeadline; 2479 } 2480 2481 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 2482 { 2483 struct kvm_lapic *apic = vcpu->arch.apic; 2484 2485 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) 2486 return; 2487 2488 hrtimer_cancel(&apic->lapic_timer.timer); 2489 apic->lapic_timer.tscdeadline = data; 2490 start_apic_timer(apic); 2491 } 2492 2493 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 2494 { 2495 apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4); 2496 } 2497 2498 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 2499 { 2500 u64 tpr; 2501 2502 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 2503 2504 return (tpr & 0xf0) >> 4; 2505 } 2506 2507 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 2508 { 2509 u64 old_value = vcpu->arch.apic_base; 2510 struct kvm_lapic *apic = vcpu->arch.apic; 2511 2512 vcpu->arch.apic_base = value; 2513 2514 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 2515 kvm_update_cpuid_runtime(vcpu); 2516 2517 if (!apic) 2518 return; 2519 2520 /* update jump label if enable bit changes */ 2521 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 2522 if (value & MSR_IA32_APICBASE_ENABLE) { 2523 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2524 static_branch_slow_dec_deferred(&apic_hw_disabled); 2525 /* Check if there are APF page ready requests pending */ 2526 kvm_make_request(KVM_REQ_APF_READY, vcpu); 2527 } else { 2528 static_branch_inc(&apic_hw_disabled.key); 2529 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 2530 } 2531 } 2532 2533 if ((old_value ^ value) & X2APIC_ENABLE) { 2534 if (value & X2APIC_ENABLE) 2535 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 2536 else if (value & MSR_IA32_APICBASE_ENABLE) 2537 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2538 } 2539 2540 if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) { 2541 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 2542 static_call_cond(kvm_x86_set_virtual_apic_mode)(vcpu); 2543 } 2544 2545 apic->base_address = apic->vcpu->arch.apic_base & 2546 MSR_IA32_APICBASE_BASE; 2547 2548 if ((value & MSR_IA32_APICBASE_ENABLE) && 2549 apic->base_address != APIC_DEFAULT_PHYS_BASE) { 2550 kvm_set_apicv_inhibit(apic->vcpu->kvm, 2551 APICV_INHIBIT_REASON_APIC_BASE_MODIFIED); 2552 } 2553 } 2554 2555 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu) 2556 { 2557 struct kvm_lapic *apic = vcpu->arch.apic; 2558 2559 if (apic->apicv_active) { 2560 /* irr_pending is always true when apicv is activated. */ 2561 apic->irr_pending = true; 2562 apic->isr_count = 1; 2563 } else { 2564 /* 2565 * Don't clear irr_pending, searching the IRR can race with 2566 * updates from the CPU as APICv is still active from hardware's 2567 * perspective. The flag will be cleared as appropriate when 2568 * KVM injects the interrupt. 2569 */ 2570 apic->isr_count = count_vectors(apic->regs + APIC_ISR); 2571 } 2572 apic->highest_isr_cache = -1; 2573 } 2574 2575 int kvm_alloc_apic_access_page(struct kvm *kvm) 2576 { 2577 struct page *page; 2578 void __user *hva; 2579 int ret = 0; 2580 2581 mutex_lock(&kvm->slots_lock); 2582 if (kvm->arch.apic_access_memslot_enabled || 2583 kvm->arch.apic_access_memslot_inhibited) 2584 goto out; 2585 2586 hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 2587 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE); 2588 if (IS_ERR(hva)) { 2589 ret = PTR_ERR(hva); 2590 goto out; 2591 } 2592 2593 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 2594 if (is_error_page(page)) { 2595 ret = -EFAULT; 2596 goto out; 2597 } 2598 2599 /* 2600 * Do not pin the page in memory, so that memory hot-unplug 2601 * is able to migrate it. 2602 */ 2603 put_page(page); 2604 kvm->arch.apic_access_memslot_enabled = true; 2605 out: 2606 mutex_unlock(&kvm->slots_lock); 2607 return ret; 2608 } 2609 EXPORT_SYMBOL_GPL(kvm_alloc_apic_access_page); 2610 2611 void kvm_inhibit_apic_access_page(struct kvm_vcpu *vcpu) 2612 { 2613 struct kvm *kvm = vcpu->kvm; 2614 2615 if (!kvm->arch.apic_access_memslot_enabled) 2616 return; 2617 2618 kvm_vcpu_srcu_read_unlock(vcpu); 2619 2620 mutex_lock(&kvm->slots_lock); 2621 2622 if (kvm->arch.apic_access_memslot_enabled) { 2623 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0); 2624 /* 2625 * Clear "enabled" after the memslot is deleted so that a 2626 * different vCPU doesn't get a false negative when checking 2627 * the flag out of slots_lock. No additional memory barrier is 2628 * needed as modifying memslots requires waiting other vCPUs to 2629 * drop SRCU (see above), and false positives are ok as the 2630 * flag is rechecked after acquiring slots_lock. 2631 */ 2632 kvm->arch.apic_access_memslot_enabled = false; 2633 2634 /* 2635 * Mark the memslot as inhibited to prevent reallocating the 2636 * memslot during vCPU creation, e.g. if a vCPU is hotplugged. 2637 */ 2638 kvm->arch.apic_access_memslot_inhibited = true; 2639 } 2640 2641 mutex_unlock(&kvm->slots_lock); 2642 2643 kvm_vcpu_srcu_read_lock(vcpu); 2644 } 2645 2646 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 2647 { 2648 struct kvm_lapic *apic = vcpu->arch.apic; 2649 u64 msr_val; 2650 int i; 2651 2652 if (!init_event) { 2653 msr_val = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE; 2654 if (kvm_vcpu_is_reset_bsp(vcpu)) 2655 msr_val |= MSR_IA32_APICBASE_BSP; 2656 kvm_lapic_set_base(vcpu, msr_val); 2657 } 2658 2659 if (!apic) 2660 return; 2661 2662 /* Stop the timer in case it's a reset to an active apic */ 2663 hrtimer_cancel(&apic->lapic_timer.timer); 2664 2665 /* The xAPIC ID is set at RESET even if the APIC was already enabled. */ 2666 if (!init_event) 2667 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2668 kvm_apic_set_version(apic->vcpu); 2669 2670 for (i = 0; i < apic->nr_lvt_entries; i++) 2671 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); 2672 apic_update_lvtt(apic); 2673 if (kvm_vcpu_is_reset_bsp(vcpu) && 2674 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2675 kvm_lapic_set_reg(apic, APIC_LVT0, 2676 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2677 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2678 2679 kvm_apic_set_dfr(apic, 0xffffffffU); 2680 apic_set_spiv(apic, 0xff); 2681 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2682 if (!apic_x2apic_mode(apic)) 2683 kvm_apic_set_ldr(apic, 0); 2684 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2685 if (!apic_x2apic_mode(apic)) { 2686 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2687 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2688 } else { 2689 kvm_lapic_set_reg64(apic, APIC_ICR, 0); 2690 } 2691 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2692 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2693 for (i = 0; i < 8; i++) { 2694 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2695 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2696 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2697 } 2698 kvm_apic_update_apicv(vcpu); 2699 update_divide_count(apic); 2700 atomic_set(&apic->lapic_timer.pending, 0); 2701 2702 vcpu->arch.pv_eoi.msr_val = 0; 2703 apic_update_ppr(apic); 2704 if (apic->apicv_active) { 2705 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); 2706 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, -1); 2707 static_call_cond(kvm_x86_hwapic_isr_update)(-1); 2708 } 2709 2710 vcpu->arch.apic_arb_prio = 0; 2711 vcpu->arch.apic_attention = 0; 2712 2713 kvm_recalculate_apic_map(vcpu->kvm); 2714 } 2715 2716 /* 2717 *---------------------------------------------------------------------- 2718 * timer interface 2719 *---------------------------------------------------------------------- 2720 */ 2721 2722 static bool lapic_is_periodic(struct kvm_lapic *apic) 2723 { 2724 return apic_lvtt_period(apic); 2725 } 2726 2727 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2728 { 2729 struct kvm_lapic *apic = vcpu->arch.apic; 2730 2731 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2732 return atomic_read(&apic->lapic_timer.pending); 2733 2734 return 0; 2735 } 2736 2737 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2738 { 2739 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2740 int vector, mode, trig_mode; 2741 2742 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2743 vector = reg & APIC_VECTOR_MASK; 2744 mode = reg & APIC_MODE_MASK; 2745 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2746 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 2747 NULL); 2748 } 2749 return 0; 2750 } 2751 2752 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2753 { 2754 struct kvm_lapic *apic = vcpu->arch.apic; 2755 2756 if (apic) 2757 kvm_apic_local_deliver(apic, APIC_LVT0); 2758 } 2759 2760 static const struct kvm_io_device_ops apic_mmio_ops = { 2761 .read = apic_mmio_read, 2762 .write = apic_mmio_write, 2763 }; 2764 2765 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2766 { 2767 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2768 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2769 2770 apic_timer_expired(apic, true); 2771 2772 if (lapic_is_periodic(apic)) { 2773 advance_periodic_target_expiration(apic); 2774 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2775 return HRTIMER_RESTART; 2776 } else 2777 return HRTIMER_NORESTART; 2778 } 2779 2780 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns) 2781 { 2782 struct kvm_lapic *apic; 2783 2784 ASSERT(vcpu != NULL); 2785 2786 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT); 2787 if (!apic) 2788 goto nomem; 2789 2790 vcpu->arch.apic = apic; 2791 2792 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 2793 if (!apic->regs) { 2794 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2795 vcpu->vcpu_id); 2796 goto nomem_free_apic; 2797 } 2798 apic->vcpu = vcpu; 2799 2800 apic->nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); 2801 2802 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2803 HRTIMER_MODE_ABS_HARD); 2804 apic->lapic_timer.timer.function = apic_timer_fn; 2805 if (timer_advance_ns == -1) { 2806 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; 2807 lapic_timer_advance_dynamic = true; 2808 } else { 2809 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 2810 lapic_timer_advance_dynamic = false; 2811 } 2812 2813 /* 2814 * Stuff the APIC ENABLE bit in lieu of temporarily incrementing 2815 * apic_hw_disabled; the full RESET value is set by kvm_lapic_reset(). 2816 */ 2817 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2818 static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2819 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2820 2821 return 0; 2822 nomem_free_apic: 2823 kfree(apic); 2824 vcpu->arch.apic = NULL; 2825 nomem: 2826 return -ENOMEM; 2827 } 2828 2829 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2830 { 2831 struct kvm_lapic *apic = vcpu->arch.apic; 2832 u32 ppr; 2833 2834 if (!kvm_apic_present(vcpu)) 2835 return -1; 2836 2837 __apic_update_ppr(apic, &ppr); 2838 return apic_has_interrupt_for_ppr(apic, ppr); 2839 } 2840 EXPORT_SYMBOL_GPL(kvm_apic_has_interrupt); 2841 2842 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2843 { 2844 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2845 2846 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2847 return 1; 2848 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2849 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2850 return 1; 2851 return 0; 2852 } 2853 2854 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2855 { 2856 struct kvm_lapic *apic = vcpu->arch.apic; 2857 2858 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2859 kvm_apic_inject_pending_timer_irqs(apic); 2860 atomic_set(&apic->lapic_timer.pending, 0); 2861 } 2862 } 2863 2864 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2865 { 2866 int vector = kvm_apic_has_interrupt(vcpu); 2867 struct kvm_lapic *apic = vcpu->arch.apic; 2868 u32 ppr; 2869 2870 if (vector == -1) 2871 return -1; 2872 2873 /* 2874 * We get here even with APIC virtualization enabled, if doing 2875 * nested virtualization and L1 runs with the "acknowledge interrupt 2876 * on exit" mode. Then we cannot inject the interrupt via RVI, 2877 * because the process would deliver it through the IDT. 2878 */ 2879 2880 apic_clear_irr(vector, apic); 2881 if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) { 2882 /* 2883 * For auto-EOI interrupts, there might be another pending 2884 * interrupt above PPR, so check whether to raise another 2885 * KVM_REQ_EVENT. 2886 */ 2887 apic_update_ppr(apic); 2888 } else { 2889 /* 2890 * For normal interrupts, PPR has been raised and there cannot 2891 * be a higher-priority pending interrupt---except if there was 2892 * a concurrent interrupt injection, but that would have 2893 * triggered KVM_REQ_EVENT already. 2894 */ 2895 apic_set_isr(vector, apic); 2896 __apic_update_ppr(apic, &ppr); 2897 } 2898 2899 return vector; 2900 } 2901 2902 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2903 struct kvm_lapic_state *s, bool set) 2904 { 2905 if (apic_x2apic_mode(vcpu->arch.apic)) { 2906 u32 *id = (u32 *)(s->regs + APIC_ID); 2907 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2908 u64 icr; 2909 2910 if (vcpu->kvm->arch.x2apic_format) { 2911 if (*id != vcpu->vcpu_id) 2912 return -EINVAL; 2913 } else { 2914 if (set) 2915 *id >>= 24; 2916 else 2917 *id <<= 24; 2918 } 2919 2920 /* 2921 * In x2APIC mode, the LDR is fixed and based on the id. And 2922 * ICR is internally a single 64-bit register, but needs to be 2923 * split to ICR+ICR2 in userspace for backwards compatibility. 2924 */ 2925 if (set) { 2926 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2927 2928 icr = __kvm_lapic_get_reg(s->regs, APIC_ICR) | 2929 (u64)__kvm_lapic_get_reg(s->regs, APIC_ICR2) << 32; 2930 __kvm_lapic_set_reg64(s->regs, APIC_ICR, icr); 2931 } else { 2932 icr = __kvm_lapic_get_reg64(s->regs, APIC_ICR); 2933 __kvm_lapic_set_reg(s->regs, APIC_ICR2, icr >> 32); 2934 } 2935 } 2936 2937 return 0; 2938 } 2939 2940 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2941 { 2942 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2943 2944 /* 2945 * Get calculated timer current count for remaining timer period (if 2946 * any) and store it in the returned register set. 2947 */ 2948 __kvm_lapic_set_reg(s->regs, APIC_TMCCT, 2949 __apic_read(vcpu->arch.apic, APIC_TMCCT)); 2950 2951 return kvm_apic_state_fixup(vcpu, s, false); 2952 } 2953 2954 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2955 { 2956 struct kvm_lapic *apic = vcpu->arch.apic; 2957 int r; 2958 2959 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2960 /* set SPIV separately to get count of SW disabled APICs right */ 2961 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2962 2963 r = kvm_apic_state_fixup(vcpu, s, true); 2964 if (r) { 2965 kvm_recalculate_apic_map(vcpu->kvm); 2966 return r; 2967 } 2968 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); 2969 2970 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 2971 kvm_recalculate_apic_map(vcpu->kvm); 2972 kvm_apic_set_version(vcpu); 2973 2974 apic_update_ppr(apic); 2975 cancel_apic_timer(apic); 2976 apic->lapic_timer.expired_tscdeadline = 0; 2977 apic_update_lvtt(apic); 2978 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2979 update_divide_count(apic); 2980 __start_apic_timer(apic, APIC_TMCCT); 2981 kvm_lapic_set_reg(apic, APIC_TMCCT, 0); 2982 kvm_apic_update_apicv(vcpu); 2983 if (apic->apicv_active) { 2984 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); 2985 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, apic_find_highest_irr(apic)); 2986 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); 2987 } 2988 kvm_make_request(KVM_REQ_EVENT, vcpu); 2989 if (ioapic_in_kernel(vcpu->kvm)) 2990 kvm_rtc_eoi_tracking_restore_one(vcpu); 2991 2992 vcpu->arch.apic_arb_prio = 0; 2993 2994 return 0; 2995 } 2996 2997 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 2998 { 2999 struct hrtimer *timer; 3000 3001 if (!lapic_in_kernel(vcpu) || 3002 kvm_can_post_timer_interrupt(vcpu)) 3003 return; 3004 3005 timer = &vcpu->arch.apic->lapic_timer.timer; 3006 if (hrtimer_cancel(timer)) 3007 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD); 3008 } 3009 3010 /* 3011 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 3012 * 3013 * Detect whether guest triggered PV EOI since the 3014 * last entry. If yes, set EOI on guests's behalf. 3015 * Clear PV EOI in guest memory in any case. 3016 */ 3017 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 3018 struct kvm_lapic *apic) 3019 { 3020 int vector; 3021 /* 3022 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 3023 * and KVM_PV_EOI_ENABLED in guest memory as follows: 3024 * 3025 * KVM_APIC_PV_EOI_PENDING is unset: 3026 * -> host disabled PV EOI. 3027 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 3028 * -> host enabled PV EOI, guest did not execute EOI yet. 3029 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 3030 * -> host enabled PV EOI, guest executed EOI. 3031 */ 3032 BUG_ON(!pv_eoi_enabled(vcpu)); 3033 3034 if (pv_eoi_test_and_clr_pending(vcpu)) 3035 return; 3036 vector = apic_set_eoi(apic); 3037 trace_kvm_pv_eoi(apic, vector); 3038 } 3039 3040 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 3041 { 3042 u32 data; 3043 3044 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 3045 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 3046 3047 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 3048 return; 3049 3050 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 3051 sizeof(u32))) 3052 return; 3053 3054 apic_set_tpr(vcpu->arch.apic, data & 0xff); 3055 } 3056 3057 /* 3058 * apic_sync_pv_eoi_to_guest - called before vmentry 3059 * 3060 * Detect whether it's safe to enable PV EOI and 3061 * if yes do so. 3062 */ 3063 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 3064 struct kvm_lapic *apic) 3065 { 3066 if (!pv_eoi_enabled(vcpu) || 3067 /* IRR set or many bits in ISR: could be nested. */ 3068 apic->irr_pending || 3069 /* Cache not set: could be safe but we don't bother. */ 3070 apic->highest_isr_cache == -1 || 3071 /* Need EOI to update ioapic. */ 3072 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 3073 /* 3074 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 3075 * so we need not do anything here. 3076 */ 3077 return; 3078 } 3079 3080 pv_eoi_set_pending(apic->vcpu); 3081 } 3082 3083 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 3084 { 3085 u32 data, tpr; 3086 int max_irr, max_isr; 3087 struct kvm_lapic *apic = vcpu->arch.apic; 3088 3089 apic_sync_pv_eoi_to_guest(vcpu, apic); 3090 3091 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 3092 return; 3093 3094 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 3095 max_irr = apic_find_highest_irr(apic); 3096 if (max_irr < 0) 3097 max_irr = 0; 3098 max_isr = apic_find_highest_isr(apic); 3099 if (max_isr < 0) 3100 max_isr = 0; 3101 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 3102 3103 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 3104 sizeof(u32)); 3105 } 3106 3107 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 3108 { 3109 if (vapic_addr) { 3110 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 3111 &vcpu->arch.apic->vapic_cache, 3112 vapic_addr, sizeof(u32))) 3113 return -EINVAL; 3114 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 3115 } else { 3116 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 3117 } 3118 3119 vcpu->arch.apic->vapic_addr = vapic_addr; 3120 return 0; 3121 } 3122 3123 int kvm_x2apic_icr_write(struct kvm_lapic *apic, u64 data) 3124 { 3125 data &= ~APIC_ICR_BUSY; 3126 3127 kvm_apic_send_ipi(apic, (u32)data, (u32)(data >> 32)); 3128 kvm_lapic_set_reg64(apic, APIC_ICR, data); 3129 trace_kvm_apic_write(APIC_ICR, data); 3130 return 0; 3131 } 3132 3133 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data) 3134 { 3135 u32 low; 3136 3137 if (reg == APIC_ICR) { 3138 *data = kvm_lapic_get_reg64(apic, APIC_ICR); 3139 return 0; 3140 } 3141 3142 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 3143 return 1; 3144 3145 *data = low; 3146 3147 return 0; 3148 } 3149 3150 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data) 3151 { 3152 /* 3153 * ICR is a 64-bit register in x2APIC mode (and Hyper-V PV vAPIC) and 3154 * can be written as such, all other registers remain accessible only 3155 * through 32-bit reads/writes. 3156 */ 3157 if (reg == APIC_ICR) 3158 return kvm_x2apic_icr_write(apic, data); 3159 3160 /* Bits 63:32 are reserved in all other registers. */ 3161 if (data >> 32) 3162 return 1; 3163 3164 return kvm_lapic_reg_write(apic, reg, (u32)data); 3165 } 3166 3167 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 3168 { 3169 struct kvm_lapic *apic = vcpu->arch.apic; 3170 u32 reg = (msr - APIC_BASE_MSR) << 4; 3171 3172 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 3173 return 1; 3174 3175 return kvm_lapic_msr_write(apic, reg, data); 3176 } 3177 3178 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 3179 { 3180 struct kvm_lapic *apic = vcpu->arch.apic; 3181 u32 reg = (msr - APIC_BASE_MSR) << 4; 3182 3183 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 3184 return 1; 3185 3186 return kvm_lapic_msr_read(apic, reg, data); 3187 } 3188 3189 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 3190 { 3191 if (!lapic_in_kernel(vcpu)) 3192 return 1; 3193 3194 return kvm_lapic_msr_write(vcpu->arch.apic, reg, data); 3195 } 3196 3197 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 3198 { 3199 if (!lapic_in_kernel(vcpu)) 3200 return 1; 3201 3202 return kvm_lapic_msr_read(vcpu->arch.apic, reg, data); 3203 } 3204 3205 int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) 3206 { 3207 u64 addr = data & ~KVM_MSR_ENABLED; 3208 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; 3209 unsigned long new_len; 3210 int ret; 3211 3212 if (!IS_ALIGNED(addr, 4)) 3213 return 1; 3214 3215 if (data & KVM_MSR_ENABLED) { 3216 if (addr == ghc->gpa && len <= ghc->len) 3217 new_len = ghc->len; 3218 else 3219 new_len = len; 3220 3221 ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); 3222 if (ret) 3223 return ret; 3224 } 3225 3226 vcpu->arch.pv_eoi.msr_val = data; 3227 3228 return 0; 3229 } 3230 3231 int kvm_apic_accept_events(struct kvm_vcpu *vcpu) 3232 { 3233 struct kvm_lapic *apic = vcpu->arch.apic; 3234 u8 sipi_vector; 3235 int r; 3236 3237 if (!kvm_apic_has_pending_init_or_sipi(vcpu)) 3238 return 0; 3239 3240 if (is_guest_mode(vcpu)) { 3241 r = kvm_check_nested_events(vcpu); 3242 if (r < 0) 3243 return r == -EBUSY ? 0 : r; 3244 /* 3245 * Continue processing INIT/SIPI even if a nested VM-Exit 3246 * occurred, e.g. pending SIPIs should be dropped if INIT+SIPI 3247 * are blocked as a result of transitioning to VMX root mode. 3248 */ 3249 } 3250 3251 /* 3252 * INITs are blocked while CPU is in specific states (SMM, VMX root 3253 * mode, SVM with GIF=0), while SIPIs are dropped if the CPU isn't in 3254 * wait-for-SIPI (WFS). 3255 */ 3256 if (!kvm_apic_init_sipi_allowed(vcpu)) { 3257 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 3258 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 3259 return 0; 3260 } 3261 3262 if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) { 3263 kvm_vcpu_reset(vcpu, true); 3264 if (kvm_vcpu_is_bsp(apic->vcpu)) 3265 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3266 else 3267 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 3268 } 3269 if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) { 3270 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 3271 /* evaluate pending_events before reading the vector */ 3272 smp_rmb(); 3273 sipi_vector = apic->sipi_vector; 3274 static_call(kvm_x86_vcpu_deliver_sipi_vector)(vcpu, sipi_vector); 3275 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3276 } 3277 } 3278 return 0; 3279 } 3280 3281 void kvm_lapic_exit(void) 3282 { 3283 static_key_deferred_flush(&apic_hw_disabled); 3284 WARN_ON(static_branch_unlikely(&apic_hw_disabled.key)); 3285 static_key_deferred_flush(&apic_sw_disabled); 3286 WARN_ON(static_branch_unlikely(&apic_sw_disabled.key)); 3287 } 3288