1 2 /* 3 * Local APIC virtualization 4 * 5 * Copyright (C) 2006 Qumranet, Inc. 6 * Copyright (C) 2007 Novell 7 * Copyright (C) 2007 Intel 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Dor Laor <dor.laor@qumranet.com> 12 * Gregory Haskins <ghaskins@novell.com> 13 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 14 * 15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/mm.h> 24 #include <linux/highmem.h> 25 #include <linux/smp.h> 26 #include <linux/hrtimer.h> 27 #include <linux/io.h> 28 #include <linux/export.h> 29 #include <linux/math64.h> 30 #include <linux/slab.h> 31 #include <asm/processor.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "trace.h" 42 #include "x86.h" 43 #include "cpuid.h" 44 #include "hyperv.h" 45 46 #ifndef CONFIG_X86_64 47 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 48 #else 49 #define mod_64(x, y) ((x) % (y)) 50 #endif 51 52 #define PRId64 "d" 53 #define PRIx64 "llx" 54 #define PRIu64 "u" 55 #define PRIo64 "o" 56 57 /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ 58 #define apic_debug(fmt, arg...) do {} while (0) 59 60 /* 14 is the version for Xeon and Pentium 8.4.8*/ 61 #define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16)) 62 #define LAPIC_MMIO_LENGTH (1 << 12) 63 /* followed define is not in apicdef.h */ 64 #define APIC_SHORT_MASK 0xc0000 65 #define APIC_DEST_NOSHORT 0x0 66 #define APIC_DEST_MASK 0x800 67 #define MAX_APIC_VECTOR 256 68 #define APIC_VECTORS_PER_REG 32 69 70 #define APIC_BROADCAST 0xFF 71 #define X2APIC_BROADCAST 0xFFFFFFFFul 72 73 static bool lapic_timer_advance_adjust_done = false; 74 #define LAPIC_TIMER_ADVANCE_ADJUST_DONE 100 75 /* step-by-step approximation to mitigate fluctuation */ 76 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8 77 78 static inline int apic_test_vector(int vec, void *bitmap) 79 { 80 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 81 } 82 83 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 84 { 85 struct kvm_lapic *apic = vcpu->arch.apic; 86 87 return apic_test_vector(vector, apic->regs + APIC_ISR) || 88 apic_test_vector(vector, apic->regs + APIC_IRR); 89 } 90 91 static inline void apic_clear_vector(int vec, void *bitmap) 92 { 93 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 94 } 95 96 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 97 { 98 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 99 } 100 101 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 102 { 103 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 104 } 105 106 struct static_key_deferred apic_hw_disabled __read_mostly; 107 struct static_key_deferred apic_sw_disabled __read_mostly; 108 109 static inline int apic_enabled(struct kvm_lapic *apic) 110 { 111 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 112 } 113 114 #define LVT_MASK \ 115 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 116 117 #define LINT_MASK \ 118 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 119 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 120 121 static inline u8 kvm_xapic_id(struct kvm_lapic *apic) 122 { 123 return kvm_lapic_get_reg(apic, APIC_ID) >> 24; 124 } 125 126 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 127 { 128 return apic->vcpu->vcpu_id; 129 } 130 131 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 132 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 133 switch (map->mode) { 134 case KVM_APIC_MODE_X2APIC: { 135 u32 offset = (dest_id >> 16) * 16; 136 u32 max_apic_id = map->max_apic_id; 137 138 if (offset <= max_apic_id) { 139 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 140 141 *cluster = &map->phys_map[offset]; 142 *mask = dest_id & (0xffff >> (16 - cluster_size)); 143 } else { 144 *mask = 0; 145 } 146 147 return true; 148 } 149 case KVM_APIC_MODE_XAPIC_FLAT: 150 *cluster = map->xapic_flat_map; 151 *mask = dest_id & 0xff; 152 return true; 153 case KVM_APIC_MODE_XAPIC_CLUSTER: 154 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 155 *mask = dest_id & 0xf; 156 return true; 157 default: 158 /* Not optimized. */ 159 return false; 160 } 161 } 162 163 static void kvm_apic_map_free(struct rcu_head *rcu) 164 { 165 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 166 167 kvfree(map); 168 } 169 170 static void recalculate_apic_map(struct kvm *kvm) 171 { 172 struct kvm_apic_map *new, *old = NULL; 173 struct kvm_vcpu *vcpu; 174 int i; 175 u32 max_id = 255; /* enough space for any xAPIC ID */ 176 177 mutex_lock(&kvm->arch.apic_map_lock); 178 179 kvm_for_each_vcpu(i, vcpu, kvm) 180 if (kvm_apic_present(vcpu)) 181 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 182 183 new = kvzalloc(sizeof(struct kvm_apic_map) + 184 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), 185 GFP_KERNEL_ACCOUNT); 186 187 if (!new) 188 goto out; 189 190 new->max_apic_id = max_id; 191 192 kvm_for_each_vcpu(i, vcpu, kvm) { 193 struct kvm_lapic *apic = vcpu->arch.apic; 194 struct kvm_lapic **cluster; 195 u16 mask; 196 u32 ldr; 197 u8 xapic_id; 198 u32 x2apic_id; 199 200 if (!kvm_apic_present(vcpu)) 201 continue; 202 203 xapic_id = kvm_xapic_id(apic); 204 x2apic_id = kvm_x2apic_id(apic); 205 206 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */ 207 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) && 208 x2apic_id <= new->max_apic_id) 209 new->phys_map[x2apic_id] = apic; 210 /* 211 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around, 212 * prevent them from masking VCPUs with APIC ID <= 0xff. 213 */ 214 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 215 new->phys_map[xapic_id] = apic; 216 217 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 218 219 if (apic_x2apic_mode(apic)) { 220 new->mode |= KVM_APIC_MODE_X2APIC; 221 } else if (ldr) { 222 ldr = GET_APIC_LOGICAL_ID(ldr); 223 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 224 new->mode |= KVM_APIC_MODE_XAPIC_FLAT; 225 else 226 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER; 227 } 228 229 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask)) 230 continue; 231 232 if (mask) 233 cluster[ffs(mask) - 1] = apic; 234 } 235 out: 236 old = rcu_dereference_protected(kvm->arch.apic_map, 237 lockdep_is_held(&kvm->arch.apic_map_lock)); 238 rcu_assign_pointer(kvm->arch.apic_map, new); 239 mutex_unlock(&kvm->arch.apic_map_lock); 240 241 if (old) 242 call_rcu(&old->rcu, kvm_apic_map_free); 243 244 kvm_make_scan_ioapic_request(kvm); 245 } 246 247 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 248 { 249 bool enabled = val & APIC_SPIV_APIC_ENABLED; 250 251 kvm_lapic_set_reg(apic, APIC_SPIV, val); 252 253 if (enabled != apic->sw_enabled) { 254 apic->sw_enabled = enabled; 255 if (enabled) 256 static_key_slow_dec_deferred(&apic_sw_disabled); 257 else 258 static_key_slow_inc(&apic_sw_disabled.key); 259 } 260 } 261 262 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 263 { 264 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 265 recalculate_apic_map(apic->vcpu->kvm); 266 } 267 268 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 269 { 270 kvm_lapic_set_reg(apic, APIC_LDR, id); 271 recalculate_apic_map(apic->vcpu->kvm); 272 } 273 274 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 275 { 276 return ((id >> 4) << 16) | (1 << (id & 0xf)); 277 } 278 279 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 280 { 281 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 282 283 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 284 285 kvm_lapic_set_reg(apic, APIC_ID, id); 286 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 287 recalculate_apic_map(apic->vcpu->kvm); 288 } 289 290 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 291 { 292 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 293 } 294 295 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 296 { 297 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 298 } 299 300 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 301 { 302 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 303 } 304 305 static inline int apic_lvtt_period(struct kvm_lapic *apic) 306 { 307 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 308 } 309 310 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 311 { 312 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 313 } 314 315 static inline int apic_lvt_nmi_mode(u32 lvt_val) 316 { 317 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 318 } 319 320 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 321 { 322 struct kvm_lapic *apic = vcpu->arch.apic; 323 struct kvm_cpuid_entry2 *feat; 324 u32 v = APIC_VERSION; 325 326 if (!lapic_in_kernel(vcpu)) 327 return; 328 329 /* 330 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) 331 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with 332 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC 333 * version first and level-triggered interrupts never get EOIed in 334 * IOAPIC. 335 */ 336 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 337 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) && 338 !ioapic_in_kernel(vcpu->kvm)) 339 v |= APIC_LVR_DIRECTED_EOI; 340 kvm_lapic_set_reg(apic, APIC_LVR, v); 341 } 342 343 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = { 344 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 345 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 346 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 347 LINT_MASK, LINT_MASK, /* LVT0-1 */ 348 LVT_MASK /* LVTERR */ 349 }; 350 351 static int find_highest_vector(void *bitmap) 352 { 353 int vec; 354 u32 *reg; 355 356 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 357 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 358 reg = bitmap + REG_POS(vec); 359 if (*reg) 360 return __fls(*reg) + vec; 361 } 362 363 return -1; 364 } 365 366 static u8 count_vectors(void *bitmap) 367 { 368 int vec; 369 u32 *reg; 370 u8 count = 0; 371 372 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 373 reg = bitmap + REG_POS(vec); 374 count += hweight32(*reg); 375 } 376 377 return count; 378 } 379 380 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr) 381 { 382 u32 i, vec; 383 u32 pir_val, irr_val, prev_irr_val; 384 int max_updated_irr; 385 386 max_updated_irr = -1; 387 *max_irr = -1; 388 389 for (i = vec = 0; i <= 7; i++, vec += 32) { 390 pir_val = READ_ONCE(pir[i]); 391 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10)); 392 if (pir_val) { 393 prev_irr_val = irr_val; 394 irr_val |= xchg(&pir[i], 0); 395 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val; 396 if (prev_irr_val != irr_val) { 397 max_updated_irr = 398 __fls(irr_val ^ prev_irr_val) + vec; 399 } 400 } 401 if (irr_val) 402 *max_irr = __fls(irr_val) + vec; 403 } 404 405 return ((max_updated_irr != -1) && 406 (max_updated_irr == *max_irr)); 407 } 408 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 409 410 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr) 411 { 412 struct kvm_lapic *apic = vcpu->arch.apic; 413 414 return __kvm_apic_update_irr(pir, apic->regs, max_irr); 415 } 416 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 417 418 static inline int apic_search_irr(struct kvm_lapic *apic) 419 { 420 return find_highest_vector(apic->regs + APIC_IRR); 421 } 422 423 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 424 { 425 int result; 426 427 /* 428 * Note that irr_pending is just a hint. It will be always 429 * true with virtual interrupt delivery enabled. 430 */ 431 if (!apic->irr_pending) 432 return -1; 433 434 result = apic_search_irr(apic); 435 ASSERT(result == -1 || result >= 16); 436 437 return result; 438 } 439 440 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 441 { 442 struct kvm_vcpu *vcpu; 443 444 vcpu = apic->vcpu; 445 446 if (unlikely(vcpu->arch.apicv_active)) { 447 /* need to update RVI */ 448 apic_clear_vector(vec, apic->regs + APIC_IRR); 449 kvm_x86_ops->hwapic_irr_update(vcpu, 450 apic_find_highest_irr(apic)); 451 } else { 452 apic->irr_pending = false; 453 apic_clear_vector(vec, apic->regs + APIC_IRR); 454 if (apic_search_irr(apic) != -1) 455 apic->irr_pending = true; 456 } 457 } 458 459 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 460 { 461 struct kvm_vcpu *vcpu; 462 463 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 464 return; 465 466 vcpu = apic->vcpu; 467 468 /* 469 * With APIC virtualization enabled, all caching is disabled 470 * because the processor can modify ISR under the hood. Instead 471 * just set SVI. 472 */ 473 if (unlikely(vcpu->arch.apicv_active)) 474 kvm_x86_ops->hwapic_isr_update(vcpu, vec); 475 else { 476 ++apic->isr_count; 477 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 478 /* 479 * ISR (in service register) bit is set when injecting an interrupt. 480 * The highest vector is injected. Thus the latest bit set matches 481 * the highest bit in ISR. 482 */ 483 apic->highest_isr_cache = vec; 484 } 485 } 486 487 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 488 { 489 int result; 490 491 /* 492 * Note that isr_count is always 1, and highest_isr_cache 493 * is always -1, with APIC virtualization enabled. 494 */ 495 if (!apic->isr_count) 496 return -1; 497 if (likely(apic->highest_isr_cache != -1)) 498 return apic->highest_isr_cache; 499 500 result = find_highest_vector(apic->regs + APIC_ISR); 501 ASSERT(result == -1 || result >= 16); 502 503 return result; 504 } 505 506 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 507 { 508 struct kvm_vcpu *vcpu; 509 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 510 return; 511 512 vcpu = apic->vcpu; 513 514 /* 515 * We do get here for APIC virtualization enabled if the guest 516 * uses the Hyper-V APIC enlightenment. In this case we may need 517 * to trigger a new interrupt delivery by writing the SVI field; 518 * on the other hand isr_count and highest_isr_cache are unused 519 * and must be left alone. 520 */ 521 if (unlikely(vcpu->arch.apicv_active)) 522 kvm_x86_ops->hwapic_isr_update(vcpu, 523 apic_find_highest_isr(apic)); 524 else { 525 --apic->isr_count; 526 BUG_ON(apic->isr_count < 0); 527 apic->highest_isr_cache = -1; 528 } 529 } 530 531 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 532 { 533 /* This may race with setting of irr in __apic_accept_irq() and 534 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 535 * will cause vmexit immediately and the value will be recalculated 536 * on the next vmentry. 537 */ 538 return apic_find_highest_irr(vcpu->arch.apic); 539 } 540 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 541 542 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 543 int vector, int level, int trig_mode, 544 struct dest_map *dest_map); 545 546 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 547 struct dest_map *dest_map) 548 { 549 struct kvm_lapic *apic = vcpu->arch.apic; 550 551 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 552 irq->level, irq->trig_mode, dest_map); 553 } 554 555 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, 556 unsigned long ipi_bitmap_high, u32 min, 557 unsigned long icr, int op_64_bit) 558 { 559 int i; 560 struct kvm_apic_map *map; 561 struct kvm_vcpu *vcpu; 562 struct kvm_lapic_irq irq = {0}; 563 int cluster_size = op_64_bit ? 64 : 32; 564 int count = 0; 565 566 irq.vector = icr & APIC_VECTOR_MASK; 567 irq.delivery_mode = icr & APIC_MODE_MASK; 568 irq.level = (icr & APIC_INT_ASSERT) != 0; 569 irq.trig_mode = icr & APIC_INT_LEVELTRIG; 570 571 if (icr & APIC_DEST_MASK) 572 return -KVM_EINVAL; 573 if (icr & APIC_SHORT_MASK) 574 return -KVM_EINVAL; 575 576 rcu_read_lock(); 577 map = rcu_dereference(kvm->arch.apic_map); 578 579 if (unlikely(!map)) { 580 count = -EOPNOTSUPP; 581 goto out; 582 } 583 584 if (min > map->max_apic_id) 585 goto out; 586 /* Bits above cluster_size are masked in the caller. */ 587 for_each_set_bit(i, &ipi_bitmap_low, 588 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { 589 if (map->phys_map[min + i]) { 590 vcpu = map->phys_map[min + i]->vcpu; 591 count += kvm_apic_set_irq(vcpu, &irq, NULL); 592 } 593 } 594 595 min += cluster_size; 596 597 if (min > map->max_apic_id) 598 goto out; 599 600 for_each_set_bit(i, &ipi_bitmap_high, 601 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { 602 if (map->phys_map[min + i]) { 603 vcpu = map->phys_map[min + i]->vcpu; 604 count += kvm_apic_set_irq(vcpu, &irq, NULL); 605 } 606 } 607 608 out: 609 rcu_read_unlock(); 610 return count; 611 } 612 613 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 614 { 615 616 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 617 sizeof(val)); 618 } 619 620 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 621 { 622 623 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 624 sizeof(*val)); 625 } 626 627 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 628 { 629 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 630 } 631 632 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) 633 { 634 u8 val; 635 if (pv_eoi_get_user(vcpu, &val) < 0) 636 apic_debug("Can't read EOI MSR value: 0x%llx\n", 637 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 638 return val & 0x1; 639 } 640 641 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 642 { 643 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) { 644 apic_debug("Can't set EOI MSR value: 0x%llx\n", 645 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 646 return; 647 } 648 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 649 } 650 651 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) 652 { 653 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) { 654 apic_debug("Can't clear EOI MSR value: 0x%llx\n", 655 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 656 return; 657 } 658 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 659 } 660 661 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 662 { 663 int highest_irr; 664 if (apic->vcpu->arch.apicv_active) 665 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu); 666 else 667 highest_irr = apic_find_highest_irr(apic); 668 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 669 return -1; 670 return highest_irr; 671 } 672 673 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 674 { 675 u32 tpr, isrv, ppr, old_ppr; 676 int isr; 677 678 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 679 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 680 isr = apic_find_highest_isr(apic); 681 isrv = (isr != -1) ? isr : 0; 682 683 if ((tpr & 0xf0) >= (isrv & 0xf0)) 684 ppr = tpr & 0xff; 685 else 686 ppr = isrv & 0xf0; 687 688 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", 689 apic, ppr, isr, isrv); 690 691 *new_ppr = ppr; 692 if (old_ppr != ppr) 693 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 694 695 return ppr < old_ppr; 696 } 697 698 static void apic_update_ppr(struct kvm_lapic *apic) 699 { 700 u32 ppr; 701 702 if (__apic_update_ppr(apic, &ppr) && 703 apic_has_interrupt_for_ppr(apic, ppr) != -1) 704 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 705 } 706 707 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 708 { 709 apic_update_ppr(vcpu->arch.apic); 710 } 711 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 712 713 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 714 { 715 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 716 apic_update_ppr(apic); 717 } 718 719 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 720 { 721 return mda == (apic_x2apic_mode(apic) ? 722 X2APIC_BROADCAST : APIC_BROADCAST); 723 } 724 725 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 726 { 727 if (kvm_apic_broadcast(apic, mda)) 728 return true; 729 730 if (apic_x2apic_mode(apic)) 731 return mda == kvm_x2apic_id(apic); 732 733 /* 734 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if 735 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and 736 * this allows unique addressing of VCPUs with APIC ID over 0xff. 737 * The 0xff condition is needed because writeable xAPIC ID. 738 */ 739 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic)) 740 return true; 741 742 return mda == kvm_xapic_id(apic); 743 } 744 745 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 746 { 747 u32 logical_id; 748 749 if (kvm_apic_broadcast(apic, mda)) 750 return true; 751 752 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 753 754 if (apic_x2apic_mode(apic)) 755 return ((logical_id >> 16) == (mda >> 16)) 756 && (logical_id & mda & 0xffff) != 0; 757 758 logical_id = GET_APIC_LOGICAL_ID(logical_id); 759 760 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 761 case APIC_DFR_FLAT: 762 return (logical_id & mda) != 0; 763 case APIC_DFR_CLUSTER: 764 return ((logical_id >> 4) == (mda >> 4)) 765 && (logical_id & mda & 0xf) != 0; 766 default: 767 apic_debug("Bad DFR vcpu %d: %08x\n", 768 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR)); 769 return false; 770 } 771 } 772 773 /* The KVM local APIC implementation has two quirks: 774 * 775 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 776 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 777 * KVM doesn't do that aliasing. 778 * 779 * - in-kernel IOAPIC messages have to be delivered directly to 780 * x2APIC, because the kernel does not support interrupt remapping. 781 * In order to support broadcast without interrupt remapping, x2APIC 782 * rewrites the destination of non-IPI messages from APIC_BROADCAST 783 * to X2APIC_BROADCAST. 784 * 785 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 786 * important when userspace wants to use x2APIC-format MSIs, because 787 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 788 */ 789 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 790 struct kvm_lapic *source, struct kvm_lapic *target) 791 { 792 bool ipi = source != NULL; 793 794 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 795 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 796 return X2APIC_BROADCAST; 797 798 return dest_id; 799 } 800 801 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 802 int short_hand, unsigned int dest, int dest_mode) 803 { 804 struct kvm_lapic *target = vcpu->arch.apic; 805 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 806 807 apic_debug("target %p, source %p, dest 0x%x, " 808 "dest_mode 0x%x, short_hand 0x%x\n", 809 target, source, dest, dest_mode, short_hand); 810 811 ASSERT(target); 812 switch (short_hand) { 813 case APIC_DEST_NOSHORT: 814 if (dest_mode == APIC_DEST_PHYSICAL) 815 return kvm_apic_match_physical_addr(target, mda); 816 else 817 return kvm_apic_match_logical_addr(target, mda); 818 case APIC_DEST_SELF: 819 return target == source; 820 case APIC_DEST_ALLINC: 821 return true; 822 case APIC_DEST_ALLBUT: 823 return target != source; 824 default: 825 apic_debug("kvm: apic: Bad dest shorthand value %x\n", 826 short_hand); 827 return false; 828 } 829 } 830 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 831 832 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 833 const unsigned long *bitmap, u32 bitmap_size) 834 { 835 u32 mod; 836 int i, idx = -1; 837 838 mod = vector % dest_vcpus; 839 840 for (i = 0; i <= mod; i++) { 841 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 842 BUG_ON(idx == bitmap_size); 843 } 844 845 return idx; 846 } 847 848 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 849 { 850 if (!kvm->arch.disabled_lapic_found) { 851 kvm->arch.disabled_lapic_found = true; 852 printk(KERN_INFO 853 "Disabled LAPIC found during irq injection\n"); 854 } 855 } 856 857 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 858 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 859 { 860 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 861 if ((irq->dest_id == APIC_BROADCAST && 862 map->mode != KVM_APIC_MODE_X2APIC)) 863 return true; 864 if (irq->dest_id == X2APIC_BROADCAST) 865 return true; 866 } else { 867 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 868 if (irq->dest_id == (x2apic_ipi ? 869 X2APIC_BROADCAST : APIC_BROADCAST)) 870 return true; 871 } 872 873 return false; 874 } 875 876 /* Return true if the interrupt can be handled by using *bitmap as index mask 877 * for valid destinations in *dst array. 878 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 879 * Note: we may have zero kvm_lapic destinations when we return true, which 880 * means that the interrupt should be dropped. In this case, *bitmap would be 881 * zero and *dst undefined. 882 */ 883 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 884 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 885 struct kvm_apic_map *map, struct kvm_lapic ***dst, 886 unsigned long *bitmap) 887 { 888 int i, lowest; 889 890 if (irq->shorthand == APIC_DEST_SELF && src) { 891 *dst = src; 892 *bitmap = 1; 893 return true; 894 } else if (irq->shorthand) 895 return false; 896 897 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 898 return false; 899 900 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 901 if (irq->dest_id > map->max_apic_id) { 902 *bitmap = 0; 903 } else { 904 *dst = &map->phys_map[irq->dest_id]; 905 *bitmap = 1; 906 } 907 return true; 908 } 909 910 *bitmap = 0; 911 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 912 (u16 *)bitmap)) 913 return false; 914 915 if (!kvm_lowest_prio_delivery(irq)) 916 return true; 917 918 if (!kvm_vector_hashing_enabled()) { 919 lowest = -1; 920 for_each_set_bit(i, bitmap, 16) { 921 if (!(*dst)[i]) 922 continue; 923 if (lowest < 0) 924 lowest = i; 925 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 926 (*dst)[lowest]->vcpu) < 0) 927 lowest = i; 928 } 929 } else { 930 if (!*bitmap) 931 return true; 932 933 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 934 bitmap, 16); 935 936 if (!(*dst)[lowest]) { 937 kvm_apic_disabled_lapic_found(kvm); 938 *bitmap = 0; 939 return true; 940 } 941 } 942 943 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 944 945 return true; 946 } 947 948 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 949 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 950 { 951 struct kvm_apic_map *map; 952 unsigned long bitmap; 953 struct kvm_lapic **dst = NULL; 954 int i; 955 bool ret; 956 957 *r = -1; 958 959 if (irq->shorthand == APIC_DEST_SELF) { 960 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 961 return true; 962 } 963 964 rcu_read_lock(); 965 map = rcu_dereference(kvm->arch.apic_map); 966 967 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 968 if (ret) { 969 *r = 0; 970 for_each_set_bit(i, &bitmap, 16) { 971 if (!dst[i]) 972 continue; 973 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 974 } 975 } 976 977 rcu_read_unlock(); 978 return ret; 979 } 980 981 /* 982 * This routine tries to handler interrupts in posted mode, here is how 983 * it deals with different cases: 984 * - For single-destination interrupts, handle it in posted mode 985 * - Else if vector hashing is enabled and it is a lowest-priority 986 * interrupt, handle it in posted mode and use the following mechanism 987 * to find the destinaiton vCPU. 988 * 1. For lowest-priority interrupts, store all the possible 989 * destination vCPUs in an array. 990 * 2. Use "guest vector % max number of destination vCPUs" to find 991 * the right destination vCPU in the array for the lowest-priority 992 * interrupt. 993 * - Otherwise, use remapped mode to inject the interrupt. 994 */ 995 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 996 struct kvm_vcpu **dest_vcpu) 997 { 998 struct kvm_apic_map *map; 999 unsigned long bitmap; 1000 struct kvm_lapic **dst = NULL; 1001 bool ret = false; 1002 1003 if (irq->shorthand) 1004 return false; 1005 1006 rcu_read_lock(); 1007 map = rcu_dereference(kvm->arch.apic_map); 1008 1009 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 1010 hweight16(bitmap) == 1) { 1011 unsigned long i = find_first_bit(&bitmap, 16); 1012 1013 if (dst[i]) { 1014 *dest_vcpu = dst[i]->vcpu; 1015 ret = true; 1016 } 1017 } 1018 1019 rcu_read_unlock(); 1020 return ret; 1021 } 1022 1023 /* 1024 * Add a pending IRQ into lapic. 1025 * Return 1 if successfully added and 0 if discarded. 1026 */ 1027 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 1028 int vector, int level, int trig_mode, 1029 struct dest_map *dest_map) 1030 { 1031 int result = 0; 1032 struct kvm_vcpu *vcpu = apic->vcpu; 1033 1034 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 1035 trig_mode, vector); 1036 switch (delivery_mode) { 1037 case APIC_DM_LOWEST: 1038 vcpu->arch.apic_arb_prio++; 1039 /* fall through */ 1040 case APIC_DM_FIXED: 1041 if (unlikely(trig_mode && !level)) 1042 break; 1043 1044 /* FIXME add logic for vcpu on reset */ 1045 if (unlikely(!apic_enabled(apic))) 1046 break; 1047 1048 result = 1; 1049 1050 if (dest_map) { 1051 __set_bit(vcpu->vcpu_id, dest_map->map); 1052 dest_map->vectors[vcpu->vcpu_id] = vector; 1053 } 1054 1055 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 1056 if (trig_mode) 1057 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR); 1058 else 1059 apic_clear_vector(vector, apic->regs + APIC_TMR); 1060 } 1061 1062 if (vcpu->arch.apicv_active) 1063 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector); 1064 else { 1065 kvm_lapic_set_irr(vector, apic); 1066 1067 kvm_make_request(KVM_REQ_EVENT, vcpu); 1068 kvm_vcpu_kick(vcpu); 1069 } 1070 break; 1071 1072 case APIC_DM_REMRD: 1073 result = 1; 1074 vcpu->arch.pv.pv_unhalted = 1; 1075 kvm_make_request(KVM_REQ_EVENT, vcpu); 1076 kvm_vcpu_kick(vcpu); 1077 break; 1078 1079 case APIC_DM_SMI: 1080 result = 1; 1081 kvm_make_request(KVM_REQ_SMI, vcpu); 1082 kvm_vcpu_kick(vcpu); 1083 break; 1084 1085 case APIC_DM_NMI: 1086 result = 1; 1087 kvm_inject_nmi(vcpu); 1088 kvm_vcpu_kick(vcpu); 1089 break; 1090 1091 case APIC_DM_INIT: 1092 if (!trig_mode || level) { 1093 result = 1; 1094 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1095 apic->pending_events = (1UL << KVM_APIC_INIT); 1096 /* make sure pending_events is visible before sending 1097 * the request */ 1098 smp_wmb(); 1099 kvm_make_request(KVM_REQ_EVENT, vcpu); 1100 kvm_vcpu_kick(vcpu); 1101 } else { 1102 apic_debug("Ignoring de-assert INIT to vcpu %d\n", 1103 vcpu->vcpu_id); 1104 } 1105 break; 1106 1107 case APIC_DM_STARTUP: 1108 apic_debug("SIPI to vcpu %d vector 0x%02x\n", 1109 vcpu->vcpu_id, vector); 1110 result = 1; 1111 apic->sipi_vector = vector; 1112 /* make sure sipi_vector is visible for the receiver */ 1113 smp_wmb(); 1114 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1115 kvm_make_request(KVM_REQ_EVENT, vcpu); 1116 kvm_vcpu_kick(vcpu); 1117 break; 1118 1119 case APIC_DM_EXTINT: 1120 /* 1121 * Should only be called by kvm_apic_local_deliver() with LVT0, 1122 * before NMI watchdog was enabled. Already handled by 1123 * kvm_apic_accept_pic_intr(). 1124 */ 1125 break; 1126 1127 default: 1128 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1129 delivery_mode); 1130 break; 1131 } 1132 return result; 1133 } 1134 1135 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1136 { 1137 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1138 } 1139 1140 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1141 { 1142 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1143 } 1144 1145 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1146 { 1147 int trigger_mode; 1148 1149 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1150 if (!kvm_ioapic_handles_vector(apic, vector)) 1151 return; 1152 1153 /* Request a KVM exit to inform the userspace IOAPIC. */ 1154 if (irqchip_split(apic->vcpu->kvm)) { 1155 apic->vcpu->arch.pending_ioapic_eoi = vector; 1156 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1157 return; 1158 } 1159 1160 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1161 trigger_mode = IOAPIC_LEVEL_TRIG; 1162 else 1163 trigger_mode = IOAPIC_EDGE_TRIG; 1164 1165 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1166 } 1167 1168 static int apic_set_eoi(struct kvm_lapic *apic) 1169 { 1170 int vector = apic_find_highest_isr(apic); 1171 1172 trace_kvm_eoi(apic, vector); 1173 1174 /* 1175 * Not every write EOI will has corresponding ISR, 1176 * one example is when Kernel check timer on setup_IO_APIC 1177 */ 1178 if (vector == -1) 1179 return vector; 1180 1181 apic_clear_isr(vector, apic); 1182 apic_update_ppr(apic); 1183 1184 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap)) 1185 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1186 1187 kvm_ioapic_send_eoi(apic, vector); 1188 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1189 return vector; 1190 } 1191 1192 /* 1193 * this interface assumes a trap-like exit, which has already finished 1194 * desired side effect including vISR and vPPR update. 1195 */ 1196 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1197 { 1198 struct kvm_lapic *apic = vcpu->arch.apic; 1199 1200 trace_kvm_eoi(apic, vector); 1201 1202 kvm_ioapic_send_eoi(apic, vector); 1203 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1204 } 1205 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1206 1207 static void apic_send_ipi(struct kvm_lapic *apic) 1208 { 1209 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR); 1210 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2); 1211 struct kvm_lapic_irq irq; 1212 1213 irq.vector = icr_low & APIC_VECTOR_MASK; 1214 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1215 irq.dest_mode = icr_low & APIC_DEST_MASK; 1216 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1217 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1218 irq.shorthand = icr_low & APIC_SHORT_MASK; 1219 irq.msi_redir_hint = false; 1220 if (apic_x2apic_mode(apic)) 1221 irq.dest_id = icr_high; 1222 else 1223 irq.dest_id = GET_APIC_DEST_FIELD(icr_high); 1224 1225 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1226 1227 apic_debug("icr_high 0x%x, icr_low 0x%x, " 1228 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " 1229 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, " 1230 "msi_redir_hint 0x%x\n", 1231 icr_high, icr_low, irq.shorthand, irq.dest_id, 1232 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, 1233 irq.vector, irq.msi_redir_hint); 1234 1235 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1236 } 1237 1238 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1239 { 1240 ktime_t remaining, now; 1241 s64 ns; 1242 u32 tmcct; 1243 1244 ASSERT(apic != NULL); 1245 1246 /* if initial count is 0, current count should also be 0 */ 1247 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1248 apic->lapic_timer.period == 0) 1249 return 0; 1250 1251 now = ktime_get(); 1252 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1253 if (ktime_to_ns(remaining) < 0) 1254 remaining = 0; 1255 1256 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1257 tmcct = div64_u64(ns, 1258 (APIC_BUS_CYCLE_NS * apic->divide_count)); 1259 1260 return tmcct; 1261 } 1262 1263 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1264 { 1265 struct kvm_vcpu *vcpu = apic->vcpu; 1266 struct kvm_run *run = vcpu->run; 1267 1268 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1269 run->tpr_access.rip = kvm_rip_read(vcpu); 1270 run->tpr_access.is_write = write; 1271 } 1272 1273 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1274 { 1275 if (apic->vcpu->arch.tpr_access_reporting) 1276 __report_tpr_access(apic, write); 1277 } 1278 1279 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1280 { 1281 u32 val = 0; 1282 1283 if (offset >= LAPIC_MMIO_LENGTH) 1284 return 0; 1285 1286 switch (offset) { 1287 case APIC_ARBPRI: 1288 apic_debug("Access APIC ARBPRI register which is for P6\n"); 1289 break; 1290 1291 case APIC_TMCCT: /* Timer CCR */ 1292 if (apic_lvtt_tscdeadline(apic)) 1293 return 0; 1294 1295 val = apic_get_tmcct(apic); 1296 break; 1297 case APIC_PROCPRI: 1298 apic_update_ppr(apic); 1299 val = kvm_lapic_get_reg(apic, offset); 1300 break; 1301 case APIC_TASKPRI: 1302 report_tpr_access(apic, false); 1303 /* fall thru */ 1304 default: 1305 val = kvm_lapic_get_reg(apic, offset); 1306 break; 1307 } 1308 1309 return val; 1310 } 1311 1312 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1313 { 1314 return container_of(dev, struct kvm_lapic, dev); 1315 } 1316 1317 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1318 void *data) 1319 { 1320 unsigned char alignment = offset & 0xf; 1321 u32 result; 1322 /* this bitmask has a bit cleared for each reserved register */ 1323 static const u64 rmask = 0x43ff01ffffffe70cULL; 1324 1325 if ((alignment + len) > 4) { 1326 apic_debug("KVM_APIC_READ: alignment error %x %d\n", 1327 offset, len); 1328 return 1; 1329 } 1330 1331 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) { 1332 apic_debug("KVM_APIC_READ: read reserved register %x\n", 1333 offset); 1334 return 1; 1335 } 1336 1337 result = __apic_read(apic, offset & ~0xf); 1338 1339 trace_kvm_apic_read(offset, result); 1340 1341 switch (len) { 1342 case 1: 1343 case 2: 1344 case 4: 1345 memcpy(data, (char *)&result + alignment, len); 1346 break; 1347 default: 1348 printk(KERN_ERR "Local APIC read with len = %x, " 1349 "should be 1,2, or 4 instead\n", len); 1350 break; 1351 } 1352 return 0; 1353 } 1354 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read); 1355 1356 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1357 { 1358 return addr >= apic->base_address && 1359 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1360 } 1361 1362 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1363 gpa_t address, int len, void *data) 1364 { 1365 struct kvm_lapic *apic = to_lapic(this); 1366 u32 offset = address - apic->base_address; 1367 1368 if (!apic_mmio_in_range(apic, address)) 1369 return -EOPNOTSUPP; 1370 1371 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 1372 if (!kvm_check_has_quirk(vcpu->kvm, 1373 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 1374 return -EOPNOTSUPP; 1375 1376 memset(data, 0xff, len); 1377 return 0; 1378 } 1379 1380 kvm_lapic_reg_read(apic, offset, len, data); 1381 1382 return 0; 1383 } 1384 1385 static void update_divide_count(struct kvm_lapic *apic) 1386 { 1387 u32 tmp1, tmp2, tdcr; 1388 1389 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1390 tmp1 = tdcr & 0xf; 1391 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1392 apic->divide_count = 0x1 << (tmp2 & 0x7); 1393 1394 apic_debug("timer divide count is 0x%x\n", 1395 apic->divide_count); 1396 } 1397 1398 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1399 { 1400 /* 1401 * Do not allow the guest to program periodic timers with small 1402 * interval, since the hrtimers are not throttled by the host 1403 * scheduler. 1404 */ 1405 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1406 s64 min_period = min_timer_period_us * 1000LL; 1407 1408 if (apic->lapic_timer.period < min_period) { 1409 pr_info_ratelimited( 1410 "kvm: vcpu %i: requested %lld ns " 1411 "lapic timer period limited to %lld ns\n", 1412 apic->vcpu->vcpu_id, 1413 apic->lapic_timer.period, min_period); 1414 apic->lapic_timer.period = min_period; 1415 } 1416 } 1417 } 1418 1419 static void apic_update_lvtt(struct kvm_lapic *apic) 1420 { 1421 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1422 apic->lapic_timer.timer_mode_mask; 1423 1424 if (apic->lapic_timer.timer_mode != timer_mode) { 1425 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1426 APIC_LVT_TIMER_TSCDEADLINE)) { 1427 hrtimer_cancel(&apic->lapic_timer.timer); 1428 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1429 apic->lapic_timer.period = 0; 1430 apic->lapic_timer.tscdeadline = 0; 1431 } 1432 apic->lapic_timer.timer_mode = timer_mode; 1433 limit_periodic_timer_frequency(apic); 1434 } 1435 } 1436 1437 static void apic_timer_expired(struct kvm_lapic *apic) 1438 { 1439 struct kvm_vcpu *vcpu = apic->vcpu; 1440 struct swait_queue_head *q = &vcpu->wq; 1441 struct kvm_timer *ktimer = &apic->lapic_timer; 1442 1443 if (atomic_read(&apic->lapic_timer.pending)) 1444 return; 1445 1446 atomic_inc(&apic->lapic_timer.pending); 1447 kvm_set_pending_timer(vcpu); 1448 1449 /* 1450 * For x86, the atomic_inc() is serialized, thus 1451 * using swait_active() is safe. 1452 */ 1453 if (swait_active(q)) 1454 swake_up_one(q); 1455 1456 if (apic_lvtt_tscdeadline(apic)) 1457 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1458 } 1459 1460 /* 1461 * On APICv, this test will cause a busy wait 1462 * during a higher-priority task. 1463 */ 1464 1465 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1466 { 1467 struct kvm_lapic *apic = vcpu->arch.apic; 1468 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1469 1470 if (kvm_apic_hw_enabled(apic)) { 1471 int vec = reg & APIC_VECTOR_MASK; 1472 void *bitmap = apic->regs + APIC_ISR; 1473 1474 if (vcpu->arch.apicv_active) 1475 bitmap = apic->regs + APIC_IRR; 1476 1477 if (apic_test_vector(vec, bitmap)) 1478 return true; 1479 } 1480 return false; 1481 } 1482 1483 void wait_lapic_expire(struct kvm_vcpu *vcpu) 1484 { 1485 struct kvm_lapic *apic = vcpu->arch.apic; 1486 u64 guest_tsc, tsc_deadline, ns; 1487 1488 if (!lapic_in_kernel(vcpu)) 1489 return; 1490 1491 if (apic->lapic_timer.expired_tscdeadline == 0) 1492 return; 1493 1494 if (!lapic_timer_int_injected(vcpu)) 1495 return; 1496 1497 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1498 apic->lapic_timer.expired_tscdeadline = 0; 1499 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1500 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1501 1502 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */ 1503 if (guest_tsc < tsc_deadline) 1504 __delay(min(tsc_deadline - guest_tsc, 1505 nsec_to_cycles(vcpu, lapic_timer_advance_ns))); 1506 1507 if (!lapic_timer_advance_adjust_done) { 1508 /* too early */ 1509 if (guest_tsc < tsc_deadline) { 1510 ns = (tsc_deadline - guest_tsc) * 1000000ULL; 1511 do_div(ns, vcpu->arch.virtual_tsc_khz); 1512 lapic_timer_advance_ns -= min((unsigned int)ns, 1513 lapic_timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP); 1514 } else { 1515 /* too late */ 1516 ns = (guest_tsc - tsc_deadline) * 1000000ULL; 1517 do_div(ns, vcpu->arch.virtual_tsc_khz); 1518 lapic_timer_advance_ns += min((unsigned int)ns, 1519 lapic_timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP); 1520 } 1521 if (abs(guest_tsc - tsc_deadline) < LAPIC_TIMER_ADVANCE_ADJUST_DONE) 1522 lapic_timer_advance_adjust_done = true; 1523 } 1524 } 1525 1526 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1527 { 1528 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline; 1529 u64 ns = 0; 1530 ktime_t expire; 1531 struct kvm_vcpu *vcpu = apic->vcpu; 1532 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1533 unsigned long flags; 1534 ktime_t now; 1535 1536 if (unlikely(!tscdeadline || !this_tsc_khz)) 1537 return; 1538 1539 local_irq_save(flags); 1540 1541 now = ktime_get(); 1542 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1543 if (likely(tscdeadline > guest_tsc)) { 1544 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1545 do_div(ns, this_tsc_khz); 1546 expire = ktime_add_ns(now, ns); 1547 expire = ktime_sub_ns(expire, lapic_timer_advance_ns); 1548 hrtimer_start(&apic->lapic_timer.timer, 1549 expire, HRTIMER_MODE_ABS_PINNED); 1550 } else 1551 apic_timer_expired(apic); 1552 1553 local_irq_restore(flags); 1554 } 1555 1556 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1557 { 1558 ktime_t now, remaining; 1559 u64 ns_remaining_old, ns_remaining_new; 1560 1561 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1562 * APIC_BUS_CYCLE_NS * apic->divide_count; 1563 limit_periodic_timer_frequency(apic); 1564 1565 now = ktime_get(); 1566 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1567 if (ktime_to_ns(remaining) < 0) 1568 remaining = 0; 1569 1570 ns_remaining_old = ktime_to_ns(remaining); 1571 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1572 apic->divide_count, old_divisor); 1573 1574 apic->lapic_timer.tscdeadline += 1575 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1576 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1577 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1578 } 1579 1580 static bool set_target_expiration(struct kvm_lapic *apic) 1581 { 1582 ktime_t now; 1583 u64 tscl = rdtsc(); 1584 1585 now = ktime_get(); 1586 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1587 * APIC_BUS_CYCLE_NS * apic->divide_count; 1588 1589 if (!apic->lapic_timer.period) { 1590 apic->lapic_timer.tscdeadline = 0; 1591 return false; 1592 } 1593 1594 limit_periodic_timer_frequency(apic); 1595 1596 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" 1597 PRIx64 ", " 1598 "timer initial count 0x%x, period %lldns, " 1599 "expire @ 0x%016" PRIx64 ".\n", __func__, 1600 APIC_BUS_CYCLE_NS, ktime_to_ns(now), 1601 kvm_lapic_get_reg(apic, APIC_TMICT), 1602 apic->lapic_timer.period, 1603 ktime_to_ns(ktime_add_ns(now, 1604 apic->lapic_timer.period))); 1605 1606 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 1607 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1608 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); 1609 1610 return true; 1611 } 1612 1613 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 1614 { 1615 ktime_t now = ktime_get(); 1616 u64 tscl = rdtsc(); 1617 ktime_t delta; 1618 1619 /* 1620 * Synchronize both deadlines to the same time source or 1621 * differences in the periods (caused by differences in the 1622 * underlying clocks or numerical approximation errors) will 1623 * cause the two to drift apart over time as the errors 1624 * accumulate. 1625 */ 1626 apic->lapic_timer.target_expiration = 1627 ktime_add_ns(apic->lapic_timer.target_expiration, 1628 apic->lapic_timer.period); 1629 delta = ktime_sub(apic->lapic_timer.target_expiration, now); 1630 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 1631 nsec_to_cycles(apic->vcpu, delta); 1632 } 1633 1634 static void start_sw_period(struct kvm_lapic *apic) 1635 { 1636 if (!apic->lapic_timer.period) 1637 return; 1638 1639 if (ktime_after(ktime_get(), 1640 apic->lapic_timer.target_expiration)) { 1641 apic_timer_expired(apic); 1642 1643 if (apic_lvtt_oneshot(apic)) 1644 return; 1645 1646 advance_periodic_target_expiration(apic); 1647 } 1648 1649 hrtimer_start(&apic->lapic_timer.timer, 1650 apic->lapic_timer.target_expiration, 1651 HRTIMER_MODE_ABS_PINNED); 1652 } 1653 1654 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 1655 { 1656 if (!lapic_in_kernel(vcpu)) 1657 return false; 1658 1659 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 1660 } 1661 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use); 1662 1663 static void cancel_hv_timer(struct kvm_lapic *apic) 1664 { 1665 WARN_ON(preemptible()); 1666 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1667 kvm_x86_ops->cancel_hv_timer(apic->vcpu); 1668 apic->lapic_timer.hv_timer_in_use = false; 1669 } 1670 1671 static bool start_hv_timer(struct kvm_lapic *apic) 1672 { 1673 struct kvm_timer *ktimer = &apic->lapic_timer; 1674 int r; 1675 1676 WARN_ON(preemptible()); 1677 if (!kvm_x86_ops->set_hv_timer) 1678 return false; 1679 1680 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1681 return false; 1682 1683 if (!ktimer->tscdeadline) 1684 return false; 1685 1686 r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline); 1687 if (r < 0) 1688 return false; 1689 1690 ktimer->hv_timer_in_use = true; 1691 hrtimer_cancel(&ktimer->timer); 1692 1693 /* 1694 * Also recheck ktimer->pending, in case the sw timer triggered in 1695 * the window. For periodic timer, leave the hv timer running for 1696 * simplicity, and the deadline will be recomputed on the next vmexit. 1697 */ 1698 if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) { 1699 if (r) 1700 apic_timer_expired(apic); 1701 return false; 1702 } 1703 1704 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true); 1705 return true; 1706 } 1707 1708 static void start_sw_timer(struct kvm_lapic *apic) 1709 { 1710 struct kvm_timer *ktimer = &apic->lapic_timer; 1711 1712 WARN_ON(preemptible()); 1713 if (apic->lapic_timer.hv_timer_in_use) 1714 cancel_hv_timer(apic); 1715 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1716 return; 1717 1718 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1719 start_sw_period(apic); 1720 else if (apic_lvtt_tscdeadline(apic)) 1721 start_sw_tscdeadline(apic); 1722 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 1723 } 1724 1725 static void restart_apic_timer(struct kvm_lapic *apic) 1726 { 1727 preempt_disable(); 1728 if (!start_hv_timer(apic)) 1729 start_sw_timer(apic); 1730 preempt_enable(); 1731 } 1732 1733 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 1734 { 1735 struct kvm_lapic *apic = vcpu->arch.apic; 1736 1737 preempt_disable(); 1738 /* If the preempt notifier has already run, it also called apic_timer_expired */ 1739 if (!apic->lapic_timer.hv_timer_in_use) 1740 goto out; 1741 WARN_ON(swait_active(&vcpu->wq)); 1742 cancel_hv_timer(apic); 1743 apic_timer_expired(apic); 1744 1745 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1746 advance_periodic_target_expiration(apic); 1747 restart_apic_timer(apic); 1748 } 1749 out: 1750 preempt_enable(); 1751 } 1752 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 1753 1754 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 1755 { 1756 restart_apic_timer(vcpu->arch.apic); 1757 } 1758 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer); 1759 1760 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 1761 { 1762 struct kvm_lapic *apic = vcpu->arch.apic; 1763 1764 preempt_disable(); 1765 /* Possibly the TSC deadline timer is not enabled yet */ 1766 if (apic->lapic_timer.hv_timer_in_use) 1767 start_sw_timer(apic); 1768 preempt_enable(); 1769 } 1770 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer); 1771 1772 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 1773 { 1774 struct kvm_lapic *apic = vcpu->arch.apic; 1775 1776 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1777 restart_apic_timer(apic); 1778 } 1779 1780 static void start_apic_timer(struct kvm_lapic *apic) 1781 { 1782 atomic_set(&apic->lapic_timer.pending, 0); 1783 1784 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1785 && !set_target_expiration(apic)) 1786 return; 1787 1788 restart_apic_timer(apic); 1789 } 1790 1791 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1792 { 1793 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 1794 1795 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 1796 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 1797 if (lvt0_in_nmi_mode) { 1798 apic_debug("Receive NMI setting on APIC_LVT0 " 1799 "for cpu %d\n", apic->vcpu->vcpu_id); 1800 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1801 } else 1802 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1803 } 1804 } 1805 1806 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 1807 { 1808 int ret = 0; 1809 1810 trace_kvm_apic_write(reg, val); 1811 1812 switch (reg) { 1813 case APIC_ID: /* Local APIC ID */ 1814 if (!apic_x2apic_mode(apic)) 1815 kvm_apic_set_xapic_id(apic, val >> 24); 1816 else 1817 ret = 1; 1818 break; 1819 1820 case APIC_TASKPRI: 1821 report_tpr_access(apic, true); 1822 apic_set_tpr(apic, val & 0xff); 1823 break; 1824 1825 case APIC_EOI: 1826 apic_set_eoi(apic); 1827 break; 1828 1829 case APIC_LDR: 1830 if (!apic_x2apic_mode(apic)) 1831 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 1832 else 1833 ret = 1; 1834 break; 1835 1836 case APIC_DFR: 1837 if (!apic_x2apic_mode(apic)) { 1838 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1839 recalculate_apic_map(apic->vcpu->kvm); 1840 } else 1841 ret = 1; 1842 break; 1843 1844 case APIC_SPIV: { 1845 u32 mask = 0x3ff; 1846 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1847 mask |= APIC_SPIV_DIRECTED_EOI; 1848 apic_set_spiv(apic, val & mask); 1849 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1850 int i; 1851 u32 lvt_val; 1852 1853 for (i = 0; i < KVM_APIC_LVT_NUM; i++) { 1854 lvt_val = kvm_lapic_get_reg(apic, 1855 APIC_LVTT + 0x10 * i); 1856 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, 1857 lvt_val | APIC_LVT_MASKED); 1858 } 1859 apic_update_lvtt(apic); 1860 atomic_set(&apic->lapic_timer.pending, 0); 1861 1862 } 1863 break; 1864 } 1865 case APIC_ICR: 1866 /* No delay here, so we always clear the pending bit */ 1867 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); 1868 apic_send_ipi(apic); 1869 break; 1870 1871 case APIC_ICR2: 1872 if (!apic_x2apic_mode(apic)) 1873 val &= 0xff000000; 1874 kvm_lapic_set_reg(apic, APIC_ICR2, val); 1875 break; 1876 1877 case APIC_LVT0: 1878 apic_manage_nmi_watchdog(apic, val); 1879 /* fall through */ 1880 case APIC_LVTTHMR: 1881 case APIC_LVTPC: 1882 case APIC_LVT1: 1883 case APIC_LVTERR: 1884 /* TODO: Check vector */ 1885 if (!kvm_apic_sw_enabled(apic)) 1886 val |= APIC_LVT_MASKED; 1887 1888 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1889 kvm_lapic_set_reg(apic, reg, val); 1890 1891 break; 1892 1893 case APIC_LVTT: 1894 if (!kvm_apic_sw_enabled(apic)) 1895 val |= APIC_LVT_MASKED; 1896 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1897 kvm_lapic_set_reg(apic, APIC_LVTT, val); 1898 apic_update_lvtt(apic); 1899 break; 1900 1901 case APIC_TMICT: 1902 if (apic_lvtt_tscdeadline(apic)) 1903 break; 1904 1905 hrtimer_cancel(&apic->lapic_timer.timer); 1906 kvm_lapic_set_reg(apic, APIC_TMICT, val); 1907 start_apic_timer(apic); 1908 break; 1909 1910 case APIC_TDCR: { 1911 uint32_t old_divisor = apic->divide_count; 1912 1913 if (val & 4) 1914 apic_debug("KVM_WRITE:TDCR %x\n", val); 1915 kvm_lapic_set_reg(apic, APIC_TDCR, val); 1916 update_divide_count(apic); 1917 if (apic->divide_count != old_divisor && 1918 apic->lapic_timer.period) { 1919 hrtimer_cancel(&apic->lapic_timer.timer); 1920 update_target_expiration(apic, old_divisor); 1921 restart_apic_timer(apic); 1922 } 1923 break; 1924 } 1925 case APIC_ESR: 1926 if (apic_x2apic_mode(apic) && val != 0) { 1927 apic_debug("KVM_WRITE:ESR not zero %x\n", val); 1928 ret = 1; 1929 } 1930 break; 1931 1932 case APIC_SELF_IPI: 1933 if (apic_x2apic_mode(apic)) { 1934 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff)); 1935 } else 1936 ret = 1; 1937 break; 1938 default: 1939 ret = 1; 1940 break; 1941 } 1942 if (ret) 1943 apic_debug("Local APIC Write to read-only register %x\n", reg); 1944 return ret; 1945 } 1946 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write); 1947 1948 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1949 gpa_t address, int len, const void *data) 1950 { 1951 struct kvm_lapic *apic = to_lapic(this); 1952 unsigned int offset = address - apic->base_address; 1953 u32 val; 1954 1955 if (!apic_mmio_in_range(apic, address)) 1956 return -EOPNOTSUPP; 1957 1958 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 1959 if (!kvm_check_has_quirk(vcpu->kvm, 1960 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 1961 return -EOPNOTSUPP; 1962 1963 return 0; 1964 } 1965 1966 /* 1967 * APIC register must be aligned on 128-bits boundary. 1968 * 32/64/128 bits registers must be accessed thru 32 bits. 1969 * Refer SDM 8.4.1 1970 */ 1971 if (len != 4 || (offset & 0xf)) { 1972 /* Don't shout loud, $infamous_os would cause only noise. */ 1973 apic_debug("apic write: bad size=%d %lx\n", len, (long)address); 1974 return 0; 1975 } 1976 1977 val = *(u32*)data; 1978 1979 /* too common printing */ 1980 if (offset != APIC_EOI) 1981 apic_debug("%s: offset 0x%x with length 0x%x, and value is " 1982 "0x%x\n", __func__, offset, len, val); 1983 1984 kvm_lapic_reg_write(apic, offset & 0xff0, val); 1985 1986 return 0; 1987 } 1988 1989 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 1990 { 1991 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 1992 } 1993 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 1994 1995 /* emulate APIC access in a trap manner */ 1996 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 1997 { 1998 u32 val = 0; 1999 2000 /* hw has done the conditional check and inst decode */ 2001 offset &= 0xff0; 2002 2003 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val); 2004 2005 /* TODO: optimize to just emulate side effect w/o one more write */ 2006 kvm_lapic_reg_write(vcpu->arch.apic, offset, val); 2007 } 2008 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 2009 2010 void kvm_free_lapic(struct kvm_vcpu *vcpu) 2011 { 2012 struct kvm_lapic *apic = vcpu->arch.apic; 2013 2014 if (!vcpu->arch.apic) 2015 return; 2016 2017 hrtimer_cancel(&apic->lapic_timer.timer); 2018 2019 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 2020 static_key_slow_dec_deferred(&apic_hw_disabled); 2021 2022 if (!apic->sw_enabled) 2023 static_key_slow_dec_deferred(&apic_sw_disabled); 2024 2025 if (apic->regs) 2026 free_page((unsigned long)apic->regs); 2027 2028 kfree(apic); 2029 } 2030 2031 /* 2032 *---------------------------------------------------------------------- 2033 * LAPIC interface 2034 *---------------------------------------------------------------------- 2035 */ 2036 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 2037 { 2038 struct kvm_lapic *apic = vcpu->arch.apic; 2039 2040 if (!lapic_in_kernel(vcpu) || 2041 !apic_lvtt_tscdeadline(apic)) 2042 return 0; 2043 2044 return apic->lapic_timer.tscdeadline; 2045 } 2046 2047 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 2048 { 2049 struct kvm_lapic *apic = vcpu->arch.apic; 2050 2051 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) || 2052 apic_lvtt_period(apic)) 2053 return; 2054 2055 hrtimer_cancel(&apic->lapic_timer.timer); 2056 apic->lapic_timer.tscdeadline = data; 2057 start_apic_timer(apic); 2058 } 2059 2060 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 2061 { 2062 struct kvm_lapic *apic = vcpu->arch.apic; 2063 2064 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 2065 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4)); 2066 } 2067 2068 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 2069 { 2070 u64 tpr; 2071 2072 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 2073 2074 return (tpr & 0xf0) >> 4; 2075 } 2076 2077 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 2078 { 2079 u64 old_value = vcpu->arch.apic_base; 2080 struct kvm_lapic *apic = vcpu->arch.apic; 2081 2082 if (!apic) 2083 value |= MSR_IA32_APICBASE_BSP; 2084 2085 vcpu->arch.apic_base = value; 2086 2087 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 2088 kvm_update_cpuid(vcpu); 2089 2090 if (!apic) 2091 return; 2092 2093 /* update jump label if enable bit changes */ 2094 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 2095 if (value & MSR_IA32_APICBASE_ENABLE) { 2096 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2097 static_key_slow_dec_deferred(&apic_hw_disabled); 2098 } else { 2099 static_key_slow_inc(&apic_hw_disabled.key); 2100 recalculate_apic_map(vcpu->kvm); 2101 } 2102 } 2103 2104 if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE)) 2105 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 2106 2107 if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) 2108 kvm_x86_ops->set_virtual_apic_mode(vcpu); 2109 2110 apic->base_address = apic->vcpu->arch.apic_base & 2111 MSR_IA32_APICBASE_BASE; 2112 2113 if ((value & MSR_IA32_APICBASE_ENABLE) && 2114 apic->base_address != APIC_DEFAULT_PHYS_BASE) 2115 pr_warn_once("APIC base relocation is unsupported by KVM"); 2116 2117 /* with FSB delivery interrupt, we can restart APIC functionality */ 2118 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " 2119 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); 2120 2121 } 2122 2123 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 2124 { 2125 struct kvm_lapic *apic = vcpu->arch.apic; 2126 int i; 2127 2128 if (!apic) 2129 return; 2130 2131 apic_debug("%s\n", __func__); 2132 2133 /* Stop the timer in case it's a reset to an active apic */ 2134 hrtimer_cancel(&apic->lapic_timer.timer); 2135 2136 if (!init_event) { 2137 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE | 2138 MSR_IA32_APICBASE_ENABLE); 2139 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2140 } 2141 kvm_apic_set_version(apic->vcpu); 2142 2143 for (i = 0; i < KVM_APIC_LVT_NUM; i++) 2144 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); 2145 apic_update_lvtt(apic); 2146 if (kvm_vcpu_is_reset_bsp(vcpu) && 2147 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2148 kvm_lapic_set_reg(apic, APIC_LVT0, 2149 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2150 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2151 2152 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU); 2153 apic_set_spiv(apic, 0xff); 2154 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2155 if (!apic_x2apic_mode(apic)) 2156 kvm_apic_set_ldr(apic, 0); 2157 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2158 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2159 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2160 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2161 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2162 for (i = 0; i < 8; i++) { 2163 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2164 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2165 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2166 } 2167 apic->irr_pending = vcpu->arch.apicv_active; 2168 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0; 2169 apic->highest_isr_cache = -1; 2170 update_divide_count(apic); 2171 atomic_set(&apic->lapic_timer.pending, 0); 2172 if (kvm_vcpu_is_bsp(vcpu)) 2173 kvm_lapic_set_base(vcpu, 2174 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); 2175 vcpu->arch.pv_eoi.msr_val = 0; 2176 apic_update_ppr(apic); 2177 if (vcpu->arch.apicv_active) { 2178 kvm_x86_ops->apicv_post_state_restore(vcpu); 2179 kvm_x86_ops->hwapic_irr_update(vcpu, -1); 2180 kvm_x86_ops->hwapic_isr_update(vcpu, -1); 2181 } 2182 2183 vcpu->arch.apic_arb_prio = 0; 2184 vcpu->arch.apic_attention = 0; 2185 2186 apic_debug("%s: vcpu=%p, id=0x%x, base_msr=" 2187 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__, 2188 vcpu, kvm_lapic_get_reg(apic, APIC_ID), 2189 vcpu->arch.apic_base, apic->base_address); 2190 } 2191 2192 /* 2193 *---------------------------------------------------------------------- 2194 * timer interface 2195 *---------------------------------------------------------------------- 2196 */ 2197 2198 static bool lapic_is_periodic(struct kvm_lapic *apic) 2199 { 2200 return apic_lvtt_period(apic); 2201 } 2202 2203 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2204 { 2205 struct kvm_lapic *apic = vcpu->arch.apic; 2206 2207 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2208 return atomic_read(&apic->lapic_timer.pending); 2209 2210 return 0; 2211 } 2212 2213 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2214 { 2215 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2216 int vector, mode, trig_mode; 2217 2218 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2219 vector = reg & APIC_VECTOR_MASK; 2220 mode = reg & APIC_MODE_MASK; 2221 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2222 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 2223 NULL); 2224 } 2225 return 0; 2226 } 2227 2228 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2229 { 2230 struct kvm_lapic *apic = vcpu->arch.apic; 2231 2232 if (apic) 2233 kvm_apic_local_deliver(apic, APIC_LVT0); 2234 } 2235 2236 static const struct kvm_io_device_ops apic_mmio_ops = { 2237 .read = apic_mmio_read, 2238 .write = apic_mmio_write, 2239 }; 2240 2241 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2242 { 2243 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2244 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2245 2246 apic_timer_expired(apic); 2247 2248 if (lapic_is_periodic(apic)) { 2249 advance_periodic_target_expiration(apic); 2250 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2251 return HRTIMER_RESTART; 2252 } else 2253 return HRTIMER_NORESTART; 2254 } 2255 2256 int kvm_create_lapic(struct kvm_vcpu *vcpu) 2257 { 2258 struct kvm_lapic *apic; 2259 2260 ASSERT(vcpu != NULL); 2261 apic_debug("apic_init %d\n", vcpu->vcpu_id); 2262 2263 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT); 2264 if (!apic) 2265 goto nomem; 2266 2267 vcpu->arch.apic = apic; 2268 2269 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 2270 if (!apic->regs) { 2271 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2272 vcpu->vcpu_id); 2273 goto nomem_free_apic; 2274 } 2275 apic->vcpu = vcpu; 2276 2277 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2278 HRTIMER_MODE_ABS_PINNED); 2279 apic->lapic_timer.timer.function = apic_timer_fn; 2280 2281 /* 2282 * APIC is created enabled. This will prevent kvm_lapic_set_base from 2283 * thinking that APIC satet has changed. 2284 */ 2285 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2286 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2287 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2288 2289 return 0; 2290 nomem_free_apic: 2291 kfree(apic); 2292 nomem: 2293 return -ENOMEM; 2294 } 2295 2296 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2297 { 2298 struct kvm_lapic *apic = vcpu->arch.apic; 2299 u32 ppr; 2300 2301 if (!apic_enabled(apic)) 2302 return -1; 2303 2304 __apic_update_ppr(apic, &ppr); 2305 return apic_has_interrupt_for_ppr(apic, ppr); 2306 } 2307 2308 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2309 { 2310 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2311 int r = 0; 2312 2313 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2314 r = 1; 2315 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2316 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2317 r = 1; 2318 return r; 2319 } 2320 2321 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2322 { 2323 struct kvm_lapic *apic = vcpu->arch.apic; 2324 2325 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2326 kvm_apic_local_deliver(apic, APIC_LVTT); 2327 if (apic_lvtt_tscdeadline(apic)) 2328 apic->lapic_timer.tscdeadline = 0; 2329 if (apic_lvtt_oneshot(apic)) { 2330 apic->lapic_timer.tscdeadline = 0; 2331 apic->lapic_timer.target_expiration = 0; 2332 } 2333 atomic_set(&apic->lapic_timer.pending, 0); 2334 } 2335 } 2336 2337 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2338 { 2339 int vector = kvm_apic_has_interrupt(vcpu); 2340 struct kvm_lapic *apic = vcpu->arch.apic; 2341 u32 ppr; 2342 2343 if (vector == -1) 2344 return -1; 2345 2346 /* 2347 * We get here even with APIC virtualization enabled, if doing 2348 * nested virtualization and L1 runs with the "acknowledge interrupt 2349 * on exit" mode. Then we cannot inject the interrupt via RVI, 2350 * because the process would deliver it through the IDT. 2351 */ 2352 2353 apic_clear_irr(vector, apic); 2354 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) { 2355 /* 2356 * For auto-EOI interrupts, there might be another pending 2357 * interrupt above PPR, so check whether to raise another 2358 * KVM_REQ_EVENT. 2359 */ 2360 apic_update_ppr(apic); 2361 } else { 2362 /* 2363 * For normal interrupts, PPR has been raised and there cannot 2364 * be a higher-priority pending interrupt---except if there was 2365 * a concurrent interrupt injection, but that would have 2366 * triggered KVM_REQ_EVENT already. 2367 */ 2368 apic_set_isr(vector, apic); 2369 __apic_update_ppr(apic, &ppr); 2370 } 2371 2372 return vector; 2373 } 2374 2375 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2376 struct kvm_lapic_state *s, bool set) 2377 { 2378 if (apic_x2apic_mode(vcpu->arch.apic)) { 2379 u32 *id = (u32 *)(s->regs + APIC_ID); 2380 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2381 2382 if (vcpu->kvm->arch.x2apic_format) { 2383 if (*id != vcpu->vcpu_id) 2384 return -EINVAL; 2385 } else { 2386 if (set) 2387 *id >>= 24; 2388 else 2389 *id <<= 24; 2390 } 2391 2392 /* In x2APIC mode, the LDR is fixed and based on the id */ 2393 if (set) 2394 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2395 } 2396 2397 return 0; 2398 } 2399 2400 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2401 { 2402 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2403 return kvm_apic_state_fixup(vcpu, s, false); 2404 } 2405 2406 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2407 { 2408 struct kvm_lapic *apic = vcpu->arch.apic; 2409 int r; 2410 2411 2412 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2413 /* set SPIV separately to get count of SW disabled APICs right */ 2414 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2415 2416 r = kvm_apic_state_fixup(vcpu, s, true); 2417 if (r) 2418 return r; 2419 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); 2420 2421 recalculate_apic_map(vcpu->kvm); 2422 kvm_apic_set_version(vcpu); 2423 2424 apic_update_ppr(apic); 2425 hrtimer_cancel(&apic->lapic_timer.timer); 2426 apic_update_lvtt(apic); 2427 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2428 update_divide_count(apic); 2429 start_apic_timer(apic); 2430 apic->irr_pending = true; 2431 apic->isr_count = vcpu->arch.apicv_active ? 2432 1 : count_vectors(apic->regs + APIC_ISR); 2433 apic->highest_isr_cache = -1; 2434 if (vcpu->arch.apicv_active) { 2435 kvm_x86_ops->apicv_post_state_restore(vcpu); 2436 kvm_x86_ops->hwapic_irr_update(vcpu, 2437 apic_find_highest_irr(apic)); 2438 kvm_x86_ops->hwapic_isr_update(vcpu, 2439 apic_find_highest_isr(apic)); 2440 } 2441 kvm_make_request(KVM_REQ_EVENT, vcpu); 2442 if (ioapic_in_kernel(vcpu->kvm)) 2443 kvm_rtc_eoi_tracking_restore_one(vcpu); 2444 2445 vcpu->arch.apic_arb_prio = 0; 2446 2447 return 0; 2448 } 2449 2450 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 2451 { 2452 struct hrtimer *timer; 2453 2454 if (!lapic_in_kernel(vcpu)) 2455 return; 2456 2457 timer = &vcpu->arch.apic->lapic_timer.timer; 2458 if (hrtimer_cancel(timer)) 2459 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 2460 } 2461 2462 /* 2463 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 2464 * 2465 * Detect whether guest triggered PV EOI since the 2466 * last entry. If yes, set EOI on guests's behalf. 2467 * Clear PV EOI in guest memory in any case. 2468 */ 2469 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 2470 struct kvm_lapic *apic) 2471 { 2472 bool pending; 2473 int vector; 2474 /* 2475 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 2476 * and KVM_PV_EOI_ENABLED in guest memory as follows: 2477 * 2478 * KVM_APIC_PV_EOI_PENDING is unset: 2479 * -> host disabled PV EOI. 2480 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 2481 * -> host enabled PV EOI, guest did not execute EOI yet. 2482 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 2483 * -> host enabled PV EOI, guest executed EOI. 2484 */ 2485 BUG_ON(!pv_eoi_enabled(vcpu)); 2486 pending = pv_eoi_get_pending(vcpu); 2487 /* 2488 * Clear pending bit in any case: it will be set again on vmentry. 2489 * While this might not be ideal from performance point of view, 2490 * this makes sure pv eoi is only enabled when we know it's safe. 2491 */ 2492 pv_eoi_clr_pending(vcpu); 2493 if (pending) 2494 return; 2495 vector = apic_set_eoi(apic); 2496 trace_kvm_pv_eoi(apic, vector); 2497 } 2498 2499 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 2500 { 2501 u32 data; 2502 2503 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 2504 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 2505 2506 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2507 return; 2508 2509 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2510 sizeof(u32))) 2511 return; 2512 2513 apic_set_tpr(vcpu->arch.apic, data & 0xff); 2514 } 2515 2516 /* 2517 * apic_sync_pv_eoi_to_guest - called before vmentry 2518 * 2519 * Detect whether it's safe to enable PV EOI and 2520 * if yes do so. 2521 */ 2522 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 2523 struct kvm_lapic *apic) 2524 { 2525 if (!pv_eoi_enabled(vcpu) || 2526 /* IRR set or many bits in ISR: could be nested. */ 2527 apic->irr_pending || 2528 /* Cache not set: could be safe but we don't bother. */ 2529 apic->highest_isr_cache == -1 || 2530 /* Need EOI to update ioapic. */ 2531 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 2532 /* 2533 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 2534 * so we need not do anything here. 2535 */ 2536 return; 2537 } 2538 2539 pv_eoi_set_pending(apic->vcpu); 2540 } 2541 2542 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 2543 { 2544 u32 data, tpr; 2545 int max_irr, max_isr; 2546 struct kvm_lapic *apic = vcpu->arch.apic; 2547 2548 apic_sync_pv_eoi_to_guest(vcpu, apic); 2549 2550 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2551 return; 2552 2553 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 2554 max_irr = apic_find_highest_irr(apic); 2555 if (max_irr < 0) 2556 max_irr = 0; 2557 max_isr = apic_find_highest_isr(apic); 2558 if (max_isr < 0) 2559 max_isr = 0; 2560 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 2561 2562 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2563 sizeof(u32)); 2564 } 2565 2566 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 2567 { 2568 if (vapic_addr) { 2569 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 2570 &vcpu->arch.apic->vapic_cache, 2571 vapic_addr, sizeof(u32))) 2572 return -EINVAL; 2573 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2574 } else { 2575 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2576 } 2577 2578 vcpu->arch.apic->vapic_addr = vapic_addr; 2579 return 0; 2580 } 2581 2582 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 2583 { 2584 struct kvm_lapic *apic = vcpu->arch.apic; 2585 u32 reg = (msr - APIC_BASE_MSR) << 4; 2586 2587 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2588 return 1; 2589 2590 if (reg == APIC_ICR2) 2591 return 1; 2592 2593 /* if this is ICR write vector before command */ 2594 if (reg == APIC_ICR) 2595 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2596 return kvm_lapic_reg_write(apic, reg, (u32)data); 2597 } 2598 2599 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 2600 { 2601 struct kvm_lapic *apic = vcpu->arch.apic; 2602 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; 2603 2604 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2605 return 1; 2606 2607 if (reg == APIC_DFR || reg == APIC_ICR2) { 2608 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n", 2609 reg); 2610 return 1; 2611 } 2612 2613 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2614 return 1; 2615 if (reg == APIC_ICR) 2616 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2617 2618 *data = (((u64)high) << 32) | low; 2619 2620 return 0; 2621 } 2622 2623 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 2624 { 2625 struct kvm_lapic *apic = vcpu->arch.apic; 2626 2627 if (!lapic_in_kernel(vcpu)) 2628 return 1; 2629 2630 /* if this is ICR write vector before command */ 2631 if (reg == APIC_ICR) 2632 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2633 return kvm_lapic_reg_write(apic, reg, (u32)data); 2634 } 2635 2636 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 2637 { 2638 struct kvm_lapic *apic = vcpu->arch.apic; 2639 u32 low, high = 0; 2640 2641 if (!lapic_in_kernel(vcpu)) 2642 return 1; 2643 2644 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2645 return 1; 2646 if (reg == APIC_ICR) 2647 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2648 2649 *data = (((u64)high) << 32) | low; 2650 2651 return 0; 2652 } 2653 2654 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) 2655 { 2656 u64 addr = data & ~KVM_MSR_ENABLED; 2657 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; 2658 unsigned long new_len; 2659 2660 if (!IS_ALIGNED(addr, 4)) 2661 return 1; 2662 2663 vcpu->arch.pv_eoi.msr_val = data; 2664 if (!pv_eoi_enabled(vcpu)) 2665 return 0; 2666 2667 if (addr == ghc->gpa && len <= ghc->len) 2668 new_len = ghc->len; 2669 else 2670 new_len = len; 2671 2672 return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); 2673 } 2674 2675 void kvm_apic_accept_events(struct kvm_vcpu *vcpu) 2676 { 2677 struct kvm_lapic *apic = vcpu->arch.apic; 2678 u8 sipi_vector; 2679 unsigned long pe; 2680 2681 if (!lapic_in_kernel(vcpu) || !apic->pending_events) 2682 return; 2683 2684 /* 2685 * INITs are latched while in SMM. Because an SMM CPU cannot 2686 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs 2687 * and delay processing of INIT until the next RSM. 2688 */ 2689 if (is_smm(vcpu)) { 2690 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 2691 if (test_bit(KVM_APIC_SIPI, &apic->pending_events)) 2692 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 2693 return; 2694 } 2695 2696 pe = xchg(&apic->pending_events, 0); 2697 if (test_bit(KVM_APIC_INIT, &pe)) { 2698 kvm_vcpu_reset(vcpu, true); 2699 if (kvm_vcpu_is_bsp(apic->vcpu)) 2700 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2701 else 2702 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 2703 } 2704 if (test_bit(KVM_APIC_SIPI, &pe) && 2705 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 2706 /* evaluate pending_events before reading the vector */ 2707 smp_rmb(); 2708 sipi_vector = apic->sipi_vector; 2709 apic_debug("vcpu %d received sipi with vector # %x\n", 2710 vcpu->vcpu_id, sipi_vector); 2711 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector); 2712 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2713 } 2714 } 2715 2716 void kvm_lapic_init(void) 2717 { 2718 /* do not patch jump label more than once per second */ 2719 jump_label_rate_limit(&apic_hw_disabled, HZ); 2720 jump_label_rate_limit(&apic_sw_disabled, HZ); 2721 } 2722 2723 void kvm_lapic_exit(void) 2724 { 2725 static_key_deferred_flush(&apic_hw_disabled); 2726 static_key_deferred_flush(&apic_sw_disabled); 2727 } 2728