1 2 /* 3 * Local APIC virtualization 4 * 5 * Copyright (C) 2006 Qumranet, Inc. 6 * Copyright (C) 2007 Novell 7 * Copyright (C) 2007 Intel 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Dor Laor <dor.laor@qumranet.com> 12 * Gregory Haskins <ghaskins@novell.com> 13 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 14 * 15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/mm.h> 24 #include <linux/highmem.h> 25 #include <linux/smp.h> 26 #include <linux/hrtimer.h> 27 #include <linux/io.h> 28 #include <linux/export.h> 29 #include <linux/math64.h> 30 #include <linux/slab.h> 31 #include <asm/processor.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "trace.h" 42 #include "x86.h" 43 #include "cpuid.h" 44 #include "hyperv.h" 45 46 #ifndef CONFIG_X86_64 47 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 48 #else 49 #define mod_64(x, y) ((x) % (y)) 50 #endif 51 52 #define PRId64 "d" 53 #define PRIx64 "llx" 54 #define PRIu64 "u" 55 #define PRIo64 "o" 56 57 /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ 58 #define apic_debug(fmt, arg...) 59 60 /* 14 is the version for Xeon and Pentium 8.4.8*/ 61 #define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16)) 62 #define LAPIC_MMIO_LENGTH (1 << 12) 63 /* followed define is not in apicdef.h */ 64 #define APIC_SHORT_MASK 0xc0000 65 #define APIC_DEST_NOSHORT 0x0 66 #define APIC_DEST_MASK 0x800 67 #define MAX_APIC_VECTOR 256 68 #define APIC_VECTORS_PER_REG 32 69 70 #define APIC_BROADCAST 0xFF 71 #define X2APIC_BROADCAST 0xFFFFFFFFul 72 73 static inline int apic_test_vector(int vec, void *bitmap) 74 { 75 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 76 } 77 78 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 79 { 80 struct kvm_lapic *apic = vcpu->arch.apic; 81 82 return apic_test_vector(vector, apic->regs + APIC_ISR) || 83 apic_test_vector(vector, apic->regs + APIC_IRR); 84 } 85 86 static inline void apic_clear_vector(int vec, void *bitmap) 87 { 88 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 89 } 90 91 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 92 { 93 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 94 } 95 96 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 97 { 98 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 99 } 100 101 struct static_key_deferred apic_hw_disabled __read_mostly; 102 struct static_key_deferred apic_sw_disabled __read_mostly; 103 104 static inline int apic_enabled(struct kvm_lapic *apic) 105 { 106 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 107 } 108 109 #define LVT_MASK \ 110 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 111 112 #define LINT_MASK \ 113 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 114 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 115 116 static inline u8 kvm_xapic_id(struct kvm_lapic *apic) 117 { 118 return kvm_lapic_get_reg(apic, APIC_ID) >> 24; 119 } 120 121 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 122 { 123 return apic->vcpu->vcpu_id; 124 } 125 126 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 127 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 128 switch (map->mode) { 129 case KVM_APIC_MODE_X2APIC: { 130 u32 offset = (dest_id >> 16) * 16; 131 u32 max_apic_id = map->max_apic_id; 132 133 if (offset <= max_apic_id) { 134 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 135 136 *cluster = &map->phys_map[offset]; 137 *mask = dest_id & (0xffff >> (16 - cluster_size)); 138 } else { 139 *mask = 0; 140 } 141 142 return true; 143 } 144 case KVM_APIC_MODE_XAPIC_FLAT: 145 *cluster = map->xapic_flat_map; 146 *mask = dest_id & 0xff; 147 return true; 148 case KVM_APIC_MODE_XAPIC_CLUSTER: 149 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 150 *mask = dest_id & 0xf; 151 return true; 152 default: 153 /* Not optimized. */ 154 return false; 155 } 156 } 157 158 static void kvm_apic_map_free(struct rcu_head *rcu) 159 { 160 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 161 162 kvfree(map); 163 } 164 165 static void recalculate_apic_map(struct kvm *kvm) 166 { 167 struct kvm_apic_map *new, *old = NULL; 168 struct kvm_vcpu *vcpu; 169 int i; 170 u32 max_id = 255; /* enough space for any xAPIC ID */ 171 172 mutex_lock(&kvm->arch.apic_map_lock); 173 174 kvm_for_each_vcpu(i, vcpu, kvm) 175 if (kvm_apic_present(vcpu)) 176 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 177 178 new = kvzalloc(sizeof(struct kvm_apic_map) + 179 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL); 180 181 if (!new) 182 goto out; 183 184 new->max_apic_id = max_id; 185 186 kvm_for_each_vcpu(i, vcpu, kvm) { 187 struct kvm_lapic *apic = vcpu->arch.apic; 188 struct kvm_lapic **cluster; 189 u16 mask; 190 u32 ldr; 191 u8 xapic_id; 192 u32 x2apic_id; 193 194 if (!kvm_apic_present(vcpu)) 195 continue; 196 197 xapic_id = kvm_xapic_id(apic); 198 x2apic_id = kvm_x2apic_id(apic); 199 200 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */ 201 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) && 202 x2apic_id <= new->max_apic_id) 203 new->phys_map[x2apic_id] = apic; 204 /* 205 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around, 206 * prevent them from masking VCPUs with APIC ID <= 0xff. 207 */ 208 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 209 new->phys_map[xapic_id] = apic; 210 211 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 212 213 if (apic_x2apic_mode(apic)) { 214 new->mode |= KVM_APIC_MODE_X2APIC; 215 } else if (ldr) { 216 ldr = GET_APIC_LOGICAL_ID(ldr); 217 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 218 new->mode |= KVM_APIC_MODE_XAPIC_FLAT; 219 else 220 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER; 221 } 222 223 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask)) 224 continue; 225 226 if (mask) 227 cluster[ffs(mask) - 1] = apic; 228 } 229 out: 230 old = rcu_dereference_protected(kvm->arch.apic_map, 231 lockdep_is_held(&kvm->arch.apic_map_lock)); 232 rcu_assign_pointer(kvm->arch.apic_map, new); 233 mutex_unlock(&kvm->arch.apic_map_lock); 234 235 if (old) 236 call_rcu(&old->rcu, kvm_apic_map_free); 237 238 kvm_make_scan_ioapic_request(kvm); 239 } 240 241 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 242 { 243 bool enabled = val & APIC_SPIV_APIC_ENABLED; 244 245 kvm_lapic_set_reg(apic, APIC_SPIV, val); 246 247 if (enabled != apic->sw_enabled) { 248 apic->sw_enabled = enabled; 249 if (enabled) { 250 static_key_slow_dec_deferred(&apic_sw_disabled); 251 recalculate_apic_map(apic->vcpu->kvm); 252 } else 253 static_key_slow_inc(&apic_sw_disabled.key); 254 } 255 } 256 257 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 258 { 259 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 260 recalculate_apic_map(apic->vcpu->kvm); 261 } 262 263 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 264 { 265 kvm_lapic_set_reg(apic, APIC_LDR, id); 266 recalculate_apic_map(apic->vcpu->kvm); 267 } 268 269 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 270 { 271 return ((id >> 4) << 16) | (1 << (id & 0xf)); 272 } 273 274 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 275 { 276 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 277 278 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 279 280 kvm_lapic_set_reg(apic, APIC_ID, id); 281 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 282 recalculate_apic_map(apic->vcpu->kvm); 283 } 284 285 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 286 { 287 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 288 } 289 290 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 291 { 292 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 293 } 294 295 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 296 { 297 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 298 } 299 300 static inline int apic_lvtt_period(struct kvm_lapic *apic) 301 { 302 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 303 } 304 305 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 306 { 307 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 308 } 309 310 static inline int apic_lvt_nmi_mode(u32 lvt_val) 311 { 312 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 313 } 314 315 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 316 { 317 struct kvm_lapic *apic = vcpu->arch.apic; 318 struct kvm_cpuid_entry2 *feat; 319 u32 v = APIC_VERSION; 320 321 if (!lapic_in_kernel(vcpu)) 322 return; 323 324 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 325 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31)))) 326 v |= APIC_LVR_DIRECTED_EOI; 327 kvm_lapic_set_reg(apic, APIC_LVR, v); 328 } 329 330 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = { 331 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 332 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 333 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 334 LINT_MASK, LINT_MASK, /* LVT0-1 */ 335 LVT_MASK /* LVTERR */ 336 }; 337 338 static int find_highest_vector(void *bitmap) 339 { 340 int vec; 341 u32 *reg; 342 343 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 344 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 345 reg = bitmap + REG_POS(vec); 346 if (*reg) 347 return __fls(*reg) + vec; 348 } 349 350 return -1; 351 } 352 353 static u8 count_vectors(void *bitmap) 354 { 355 int vec; 356 u32 *reg; 357 u8 count = 0; 358 359 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 360 reg = bitmap + REG_POS(vec); 361 count += hweight32(*reg); 362 } 363 364 return count; 365 } 366 367 int __kvm_apic_update_irr(u32 *pir, void *regs) 368 { 369 u32 i, vec; 370 u32 pir_val, irr_val; 371 int max_irr = -1; 372 373 for (i = vec = 0; i <= 7; i++, vec += 32) { 374 pir_val = READ_ONCE(pir[i]); 375 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10)); 376 if (pir_val) { 377 irr_val |= xchg(&pir[i], 0); 378 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val; 379 } 380 if (irr_val) 381 max_irr = __fls(irr_val) + vec; 382 } 383 384 return max_irr; 385 } 386 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 387 388 int kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir) 389 { 390 struct kvm_lapic *apic = vcpu->arch.apic; 391 392 return __kvm_apic_update_irr(pir, apic->regs); 393 } 394 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 395 396 static inline int apic_search_irr(struct kvm_lapic *apic) 397 { 398 return find_highest_vector(apic->regs + APIC_IRR); 399 } 400 401 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 402 { 403 int result; 404 405 /* 406 * Note that irr_pending is just a hint. It will be always 407 * true with virtual interrupt delivery enabled. 408 */ 409 if (!apic->irr_pending) 410 return -1; 411 412 result = apic_search_irr(apic); 413 ASSERT(result == -1 || result >= 16); 414 415 return result; 416 } 417 418 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 419 { 420 struct kvm_vcpu *vcpu; 421 422 vcpu = apic->vcpu; 423 424 if (unlikely(vcpu->arch.apicv_active)) { 425 /* need to update RVI */ 426 apic_clear_vector(vec, apic->regs + APIC_IRR); 427 kvm_x86_ops->hwapic_irr_update(vcpu, 428 apic_find_highest_irr(apic)); 429 } else { 430 apic->irr_pending = false; 431 apic_clear_vector(vec, apic->regs + APIC_IRR); 432 if (apic_search_irr(apic) != -1) 433 apic->irr_pending = true; 434 } 435 } 436 437 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 438 { 439 struct kvm_vcpu *vcpu; 440 441 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 442 return; 443 444 vcpu = apic->vcpu; 445 446 /* 447 * With APIC virtualization enabled, all caching is disabled 448 * because the processor can modify ISR under the hood. Instead 449 * just set SVI. 450 */ 451 if (unlikely(vcpu->arch.apicv_active)) 452 kvm_x86_ops->hwapic_isr_update(vcpu, vec); 453 else { 454 ++apic->isr_count; 455 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 456 /* 457 * ISR (in service register) bit is set when injecting an interrupt. 458 * The highest vector is injected. Thus the latest bit set matches 459 * the highest bit in ISR. 460 */ 461 apic->highest_isr_cache = vec; 462 } 463 } 464 465 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 466 { 467 int result; 468 469 /* 470 * Note that isr_count is always 1, and highest_isr_cache 471 * is always -1, with APIC virtualization enabled. 472 */ 473 if (!apic->isr_count) 474 return -1; 475 if (likely(apic->highest_isr_cache != -1)) 476 return apic->highest_isr_cache; 477 478 result = find_highest_vector(apic->regs + APIC_ISR); 479 ASSERT(result == -1 || result >= 16); 480 481 return result; 482 } 483 484 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 485 { 486 struct kvm_vcpu *vcpu; 487 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 488 return; 489 490 vcpu = apic->vcpu; 491 492 /* 493 * We do get here for APIC virtualization enabled if the guest 494 * uses the Hyper-V APIC enlightenment. In this case we may need 495 * to trigger a new interrupt delivery by writing the SVI field; 496 * on the other hand isr_count and highest_isr_cache are unused 497 * and must be left alone. 498 */ 499 if (unlikely(vcpu->arch.apicv_active)) 500 kvm_x86_ops->hwapic_isr_update(vcpu, 501 apic_find_highest_isr(apic)); 502 else { 503 --apic->isr_count; 504 BUG_ON(apic->isr_count < 0); 505 apic->highest_isr_cache = -1; 506 } 507 } 508 509 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 510 { 511 /* This may race with setting of irr in __apic_accept_irq() and 512 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 513 * will cause vmexit immediately and the value will be recalculated 514 * on the next vmentry. 515 */ 516 return apic_find_highest_irr(vcpu->arch.apic); 517 } 518 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 519 520 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 521 int vector, int level, int trig_mode, 522 struct dest_map *dest_map); 523 524 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 525 struct dest_map *dest_map) 526 { 527 struct kvm_lapic *apic = vcpu->arch.apic; 528 529 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 530 irq->level, irq->trig_mode, dest_map); 531 } 532 533 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 534 { 535 536 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 537 sizeof(val)); 538 } 539 540 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 541 { 542 543 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 544 sizeof(*val)); 545 } 546 547 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 548 { 549 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 550 } 551 552 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) 553 { 554 u8 val; 555 if (pv_eoi_get_user(vcpu, &val) < 0) 556 apic_debug("Can't read EOI MSR value: 0x%llx\n", 557 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 558 return val & 0x1; 559 } 560 561 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 562 { 563 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) { 564 apic_debug("Can't set EOI MSR value: 0x%llx\n", 565 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 566 return; 567 } 568 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 569 } 570 571 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) 572 { 573 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) { 574 apic_debug("Can't clear EOI MSR value: 0x%llx\n", 575 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 576 return; 577 } 578 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 579 } 580 581 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 582 { 583 int highest_irr; 584 if (kvm_x86_ops->sync_pir_to_irr && apic->vcpu->arch.apicv_active) 585 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu); 586 else 587 highest_irr = apic_find_highest_irr(apic); 588 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 589 return -1; 590 return highest_irr; 591 } 592 593 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 594 { 595 u32 tpr, isrv, ppr, old_ppr; 596 int isr; 597 598 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 599 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 600 isr = apic_find_highest_isr(apic); 601 isrv = (isr != -1) ? isr : 0; 602 603 if ((tpr & 0xf0) >= (isrv & 0xf0)) 604 ppr = tpr & 0xff; 605 else 606 ppr = isrv & 0xf0; 607 608 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", 609 apic, ppr, isr, isrv); 610 611 *new_ppr = ppr; 612 if (old_ppr != ppr) 613 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 614 615 return ppr < old_ppr; 616 } 617 618 static void apic_update_ppr(struct kvm_lapic *apic) 619 { 620 u32 ppr; 621 622 if (__apic_update_ppr(apic, &ppr) && 623 apic_has_interrupt_for_ppr(apic, ppr) != -1) 624 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 625 } 626 627 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 628 { 629 apic_update_ppr(vcpu->arch.apic); 630 } 631 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 632 633 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 634 { 635 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 636 apic_update_ppr(apic); 637 } 638 639 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 640 { 641 return mda == (apic_x2apic_mode(apic) ? 642 X2APIC_BROADCAST : APIC_BROADCAST); 643 } 644 645 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 646 { 647 if (kvm_apic_broadcast(apic, mda)) 648 return true; 649 650 if (apic_x2apic_mode(apic)) 651 return mda == kvm_x2apic_id(apic); 652 653 /* 654 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if 655 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and 656 * this allows unique addressing of VCPUs with APIC ID over 0xff. 657 * The 0xff condition is needed because writeable xAPIC ID. 658 */ 659 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic)) 660 return true; 661 662 return mda == kvm_xapic_id(apic); 663 } 664 665 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 666 { 667 u32 logical_id; 668 669 if (kvm_apic_broadcast(apic, mda)) 670 return true; 671 672 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 673 674 if (apic_x2apic_mode(apic)) 675 return ((logical_id >> 16) == (mda >> 16)) 676 && (logical_id & mda & 0xffff) != 0; 677 678 logical_id = GET_APIC_LOGICAL_ID(logical_id); 679 680 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 681 case APIC_DFR_FLAT: 682 return (logical_id & mda) != 0; 683 case APIC_DFR_CLUSTER: 684 return ((logical_id >> 4) == (mda >> 4)) 685 && (logical_id & mda & 0xf) != 0; 686 default: 687 apic_debug("Bad DFR vcpu %d: %08x\n", 688 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR)); 689 return false; 690 } 691 } 692 693 /* The KVM local APIC implementation has two quirks: 694 * 695 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 696 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 697 * KVM doesn't do that aliasing. 698 * 699 * - in-kernel IOAPIC messages have to be delivered directly to 700 * x2APIC, because the kernel does not support interrupt remapping. 701 * In order to support broadcast without interrupt remapping, x2APIC 702 * rewrites the destination of non-IPI messages from APIC_BROADCAST 703 * to X2APIC_BROADCAST. 704 * 705 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 706 * important when userspace wants to use x2APIC-format MSIs, because 707 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 708 */ 709 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 710 struct kvm_lapic *source, struct kvm_lapic *target) 711 { 712 bool ipi = source != NULL; 713 714 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 715 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 716 return X2APIC_BROADCAST; 717 718 return dest_id; 719 } 720 721 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 722 int short_hand, unsigned int dest, int dest_mode) 723 { 724 struct kvm_lapic *target = vcpu->arch.apic; 725 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 726 727 apic_debug("target %p, source %p, dest 0x%x, " 728 "dest_mode 0x%x, short_hand 0x%x\n", 729 target, source, dest, dest_mode, short_hand); 730 731 ASSERT(target); 732 switch (short_hand) { 733 case APIC_DEST_NOSHORT: 734 if (dest_mode == APIC_DEST_PHYSICAL) 735 return kvm_apic_match_physical_addr(target, mda); 736 else 737 return kvm_apic_match_logical_addr(target, mda); 738 case APIC_DEST_SELF: 739 return target == source; 740 case APIC_DEST_ALLINC: 741 return true; 742 case APIC_DEST_ALLBUT: 743 return target != source; 744 default: 745 apic_debug("kvm: apic: Bad dest shorthand value %x\n", 746 short_hand); 747 return false; 748 } 749 } 750 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 751 752 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 753 const unsigned long *bitmap, u32 bitmap_size) 754 { 755 u32 mod; 756 int i, idx = -1; 757 758 mod = vector % dest_vcpus; 759 760 for (i = 0; i <= mod; i++) { 761 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 762 BUG_ON(idx == bitmap_size); 763 } 764 765 return idx; 766 } 767 768 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 769 { 770 if (!kvm->arch.disabled_lapic_found) { 771 kvm->arch.disabled_lapic_found = true; 772 printk(KERN_INFO 773 "Disabled LAPIC found during irq injection\n"); 774 } 775 } 776 777 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 778 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 779 { 780 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 781 if ((irq->dest_id == APIC_BROADCAST && 782 map->mode != KVM_APIC_MODE_X2APIC)) 783 return true; 784 if (irq->dest_id == X2APIC_BROADCAST) 785 return true; 786 } else { 787 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 788 if (irq->dest_id == (x2apic_ipi ? 789 X2APIC_BROADCAST : APIC_BROADCAST)) 790 return true; 791 } 792 793 return false; 794 } 795 796 /* Return true if the interrupt can be handled by using *bitmap as index mask 797 * for valid destinations in *dst array. 798 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 799 * Note: we may have zero kvm_lapic destinations when we return true, which 800 * means that the interrupt should be dropped. In this case, *bitmap would be 801 * zero and *dst undefined. 802 */ 803 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 804 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 805 struct kvm_apic_map *map, struct kvm_lapic ***dst, 806 unsigned long *bitmap) 807 { 808 int i, lowest; 809 810 if (irq->shorthand == APIC_DEST_SELF && src) { 811 *dst = src; 812 *bitmap = 1; 813 return true; 814 } else if (irq->shorthand) 815 return false; 816 817 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 818 return false; 819 820 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 821 if (irq->dest_id > map->max_apic_id) { 822 *bitmap = 0; 823 } else { 824 *dst = &map->phys_map[irq->dest_id]; 825 *bitmap = 1; 826 } 827 return true; 828 } 829 830 *bitmap = 0; 831 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 832 (u16 *)bitmap)) 833 return false; 834 835 if (!kvm_lowest_prio_delivery(irq)) 836 return true; 837 838 if (!kvm_vector_hashing_enabled()) { 839 lowest = -1; 840 for_each_set_bit(i, bitmap, 16) { 841 if (!(*dst)[i]) 842 continue; 843 if (lowest < 0) 844 lowest = i; 845 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 846 (*dst)[lowest]->vcpu) < 0) 847 lowest = i; 848 } 849 } else { 850 if (!*bitmap) 851 return true; 852 853 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 854 bitmap, 16); 855 856 if (!(*dst)[lowest]) { 857 kvm_apic_disabled_lapic_found(kvm); 858 *bitmap = 0; 859 return true; 860 } 861 } 862 863 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 864 865 return true; 866 } 867 868 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 869 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 870 { 871 struct kvm_apic_map *map; 872 unsigned long bitmap; 873 struct kvm_lapic **dst = NULL; 874 int i; 875 bool ret; 876 877 *r = -1; 878 879 if (irq->shorthand == APIC_DEST_SELF) { 880 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 881 return true; 882 } 883 884 rcu_read_lock(); 885 map = rcu_dereference(kvm->arch.apic_map); 886 887 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 888 if (ret) 889 for_each_set_bit(i, &bitmap, 16) { 890 if (!dst[i]) 891 continue; 892 if (*r < 0) 893 *r = 0; 894 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 895 } 896 897 rcu_read_unlock(); 898 return ret; 899 } 900 901 /* 902 * This routine tries to handler interrupts in posted mode, here is how 903 * it deals with different cases: 904 * - For single-destination interrupts, handle it in posted mode 905 * - Else if vector hashing is enabled and it is a lowest-priority 906 * interrupt, handle it in posted mode and use the following mechanism 907 * to find the destinaiton vCPU. 908 * 1. For lowest-priority interrupts, store all the possible 909 * destination vCPUs in an array. 910 * 2. Use "guest vector % max number of destination vCPUs" to find 911 * the right destination vCPU in the array for the lowest-priority 912 * interrupt. 913 * - Otherwise, use remapped mode to inject the interrupt. 914 */ 915 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 916 struct kvm_vcpu **dest_vcpu) 917 { 918 struct kvm_apic_map *map; 919 unsigned long bitmap; 920 struct kvm_lapic **dst = NULL; 921 bool ret = false; 922 923 if (irq->shorthand) 924 return false; 925 926 rcu_read_lock(); 927 map = rcu_dereference(kvm->arch.apic_map); 928 929 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 930 hweight16(bitmap) == 1) { 931 unsigned long i = find_first_bit(&bitmap, 16); 932 933 if (dst[i]) { 934 *dest_vcpu = dst[i]->vcpu; 935 ret = true; 936 } 937 } 938 939 rcu_read_unlock(); 940 return ret; 941 } 942 943 /* 944 * Add a pending IRQ into lapic. 945 * Return 1 if successfully added and 0 if discarded. 946 */ 947 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 948 int vector, int level, int trig_mode, 949 struct dest_map *dest_map) 950 { 951 int result = 0; 952 struct kvm_vcpu *vcpu = apic->vcpu; 953 954 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 955 trig_mode, vector); 956 switch (delivery_mode) { 957 case APIC_DM_LOWEST: 958 vcpu->arch.apic_arb_prio++; 959 case APIC_DM_FIXED: 960 if (unlikely(trig_mode && !level)) 961 break; 962 963 /* FIXME add logic for vcpu on reset */ 964 if (unlikely(!apic_enabled(apic))) 965 break; 966 967 result = 1; 968 969 if (dest_map) { 970 __set_bit(vcpu->vcpu_id, dest_map->map); 971 dest_map->vectors[vcpu->vcpu_id] = vector; 972 } 973 974 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 975 if (trig_mode) 976 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR); 977 else 978 apic_clear_vector(vector, apic->regs + APIC_TMR); 979 } 980 981 if (vcpu->arch.apicv_active) 982 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector); 983 else { 984 kvm_lapic_set_irr(vector, apic); 985 986 kvm_make_request(KVM_REQ_EVENT, vcpu); 987 kvm_vcpu_kick(vcpu); 988 } 989 break; 990 991 case APIC_DM_REMRD: 992 result = 1; 993 vcpu->arch.pv.pv_unhalted = 1; 994 kvm_make_request(KVM_REQ_EVENT, vcpu); 995 kvm_vcpu_kick(vcpu); 996 break; 997 998 case APIC_DM_SMI: 999 result = 1; 1000 kvm_make_request(KVM_REQ_SMI, vcpu); 1001 kvm_vcpu_kick(vcpu); 1002 break; 1003 1004 case APIC_DM_NMI: 1005 result = 1; 1006 kvm_inject_nmi(vcpu); 1007 kvm_vcpu_kick(vcpu); 1008 break; 1009 1010 case APIC_DM_INIT: 1011 if (!trig_mode || level) { 1012 result = 1; 1013 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1014 apic->pending_events = (1UL << KVM_APIC_INIT); 1015 /* make sure pending_events is visible before sending 1016 * the request */ 1017 smp_wmb(); 1018 kvm_make_request(KVM_REQ_EVENT, vcpu); 1019 kvm_vcpu_kick(vcpu); 1020 } else { 1021 apic_debug("Ignoring de-assert INIT to vcpu %d\n", 1022 vcpu->vcpu_id); 1023 } 1024 break; 1025 1026 case APIC_DM_STARTUP: 1027 apic_debug("SIPI to vcpu %d vector 0x%02x\n", 1028 vcpu->vcpu_id, vector); 1029 result = 1; 1030 apic->sipi_vector = vector; 1031 /* make sure sipi_vector is visible for the receiver */ 1032 smp_wmb(); 1033 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1034 kvm_make_request(KVM_REQ_EVENT, vcpu); 1035 kvm_vcpu_kick(vcpu); 1036 break; 1037 1038 case APIC_DM_EXTINT: 1039 /* 1040 * Should only be called by kvm_apic_local_deliver() with LVT0, 1041 * before NMI watchdog was enabled. Already handled by 1042 * kvm_apic_accept_pic_intr(). 1043 */ 1044 break; 1045 1046 default: 1047 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1048 delivery_mode); 1049 break; 1050 } 1051 return result; 1052 } 1053 1054 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1055 { 1056 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1057 } 1058 1059 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1060 { 1061 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1062 } 1063 1064 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1065 { 1066 int trigger_mode; 1067 1068 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1069 if (!kvm_ioapic_handles_vector(apic, vector)) 1070 return; 1071 1072 /* Request a KVM exit to inform the userspace IOAPIC. */ 1073 if (irqchip_split(apic->vcpu->kvm)) { 1074 apic->vcpu->arch.pending_ioapic_eoi = vector; 1075 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1076 return; 1077 } 1078 1079 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1080 trigger_mode = IOAPIC_LEVEL_TRIG; 1081 else 1082 trigger_mode = IOAPIC_EDGE_TRIG; 1083 1084 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1085 } 1086 1087 static int apic_set_eoi(struct kvm_lapic *apic) 1088 { 1089 int vector = apic_find_highest_isr(apic); 1090 1091 trace_kvm_eoi(apic, vector); 1092 1093 /* 1094 * Not every write EOI will has corresponding ISR, 1095 * one example is when Kernel check timer on setup_IO_APIC 1096 */ 1097 if (vector == -1) 1098 return vector; 1099 1100 apic_clear_isr(vector, apic); 1101 apic_update_ppr(apic); 1102 1103 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap)) 1104 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1105 1106 kvm_ioapic_send_eoi(apic, vector); 1107 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1108 return vector; 1109 } 1110 1111 /* 1112 * this interface assumes a trap-like exit, which has already finished 1113 * desired side effect including vISR and vPPR update. 1114 */ 1115 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1116 { 1117 struct kvm_lapic *apic = vcpu->arch.apic; 1118 1119 trace_kvm_eoi(apic, vector); 1120 1121 kvm_ioapic_send_eoi(apic, vector); 1122 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1123 } 1124 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1125 1126 static void apic_send_ipi(struct kvm_lapic *apic) 1127 { 1128 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR); 1129 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2); 1130 struct kvm_lapic_irq irq; 1131 1132 irq.vector = icr_low & APIC_VECTOR_MASK; 1133 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1134 irq.dest_mode = icr_low & APIC_DEST_MASK; 1135 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1136 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1137 irq.shorthand = icr_low & APIC_SHORT_MASK; 1138 irq.msi_redir_hint = false; 1139 if (apic_x2apic_mode(apic)) 1140 irq.dest_id = icr_high; 1141 else 1142 irq.dest_id = GET_APIC_DEST_FIELD(icr_high); 1143 1144 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1145 1146 apic_debug("icr_high 0x%x, icr_low 0x%x, " 1147 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " 1148 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, " 1149 "msi_redir_hint 0x%x\n", 1150 icr_high, icr_low, irq.shorthand, irq.dest_id, 1151 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, 1152 irq.vector, irq.msi_redir_hint); 1153 1154 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1155 } 1156 1157 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1158 { 1159 ktime_t remaining, now; 1160 s64 ns; 1161 u32 tmcct; 1162 1163 ASSERT(apic != NULL); 1164 1165 /* if initial count is 0, current count should also be 0 */ 1166 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1167 apic->lapic_timer.period == 0) 1168 return 0; 1169 1170 now = ktime_get(); 1171 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1172 if (ktime_to_ns(remaining) < 0) 1173 remaining = 0; 1174 1175 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1176 tmcct = div64_u64(ns, 1177 (APIC_BUS_CYCLE_NS * apic->divide_count)); 1178 1179 return tmcct; 1180 } 1181 1182 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1183 { 1184 struct kvm_vcpu *vcpu = apic->vcpu; 1185 struct kvm_run *run = vcpu->run; 1186 1187 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1188 run->tpr_access.rip = kvm_rip_read(vcpu); 1189 run->tpr_access.is_write = write; 1190 } 1191 1192 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1193 { 1194 if (apic->vcpu->arch.tpr_access_reporting) 1195 __report_tpr_access(apic, write); 1196 } 1197 1198 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1199 { 1200 u32 val = 0; 1201 1202 if (offset >= LAPIC_MMIO_LENGTH) 1203 return 0; 1204 1205 switch (offset) { 1206 case APIC_ARBPRI: 1207 apic_debug("Access APIC ARBPRI register which is for P6\n"); 1208 break; 1209 1210 case APIC_TMCCT: /* Timer CCR */ 1211 if (apic_lvtt_tscdeadline(apic)) 1212 return 0; 1213 1214 val = apic_get_tmcct(apic); 1215 break; 1216 case APIC_PROCPRI: 1217 apic_update_ppr(apic); 1218 val = kvm_lapic_get_reg(apic, offset); 1219 break; 1220 case APIC_TASKPRI: 1221 report_tpr_access(apic, false); 1222 /* fall thru */ 1223 default: 1224 val = kvm_lapic_get_reg(apic, offset); 1225 break; 1226 } 1227 1228 return val; 1229 } 1230 1231 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1232 { 1233 return container_of(dev, struct kvm_lapic, dev); 1234 } 1235 1236 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1237 void *data) 1238 { 1239 unsigned char alignment = offset & 0xf; 1240 u32 result; 1241 /* this bitmask has a bit cleared for each reserved register */ 1242 static const u64 rmask = 0x43ff01ffffffe70cULL; 1243 1244 if ((alignment + len) > 4) { 1245 apic_debug("KVM_APIC_READ: alignment error %x %d\n", 1246 offset, len); 1247 return 1; 1248 } 1249 1250 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) { 1251 apic_debug("KVM_APIC_READ: read reserved register %x\n", 1252 offset); 1253 return 1; 1254 } 1255 1256 result = __apic_read(apic, offset & ~0xf); 1257 1258 trace_kvm_apic_read(offset, result); 1259 1260 switch (len) { 1261 case 1: 1262 case 2: 1263 case 4: 1264 memcpy(data, (char *)&result + alignment, len); 1265 break; 1266 default: 1267 printk(KERN_ERR "Local APIC read with len = %x, " 1268 "should be 1,2, or 4 instead\n", len); 1269 break; 1270 } 1271 return 0; 1272 } 1273 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read); 1274 1275 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1276 { 1277 return kvm_apic_hw_enabled(apic) && 1278 addr >= apic->base_address && 1279 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1280 } 1281 1282 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1283 gpa_t address, int len, void *data) 1284 { 1285 struct kvm_lapic *apic = to_lapic(this); 1286 u32 offset = address - apic->base_address; 1287 1288 if (!apic_mmio_in_range(apic, address)) 1289 return -EOPNOTSUPP; 1290 1291 kvm_lapic_reg_read(apic, offset, len, data); 1292 1293 return 0; 1294 } 1295 1296 static void update_divide_count(struct kvm_lapic *apic) 1297 { 1298 u32 tmp1, tmp2, tdcr; 1299 1300 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1301 tmp1 = tdcr & 0xf; 1302 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1303 apic->divide_count = 0x1 << (tmp2 & 0x7); 1304 1305 apic_debug("timer divide count is 0x%x\n", 1306 apic->divide_count); 1307 } 1308 1309 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1310 { 1311 /* 1312 * Do not allow the guest to program periodic timers with small 1313 * interval, since the hrtimers are not throttled by the host 1314 * scheduler. 1315 */ 1316 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1317 s64 min_period = min_timer_period_us * 1000LL; 1318 1319 if (apic->lapic_timer.period < min_period) { 1320 pr_info_ratelimited( 1321 "kvm: vcpu %i: requested %lld ns " 1322 "lapic timer period limited to %lld ns\n", 1323 apic->vcpu->vcpu_id, 1324 apic->lapic_timer.period, min_period); 1325 apic->lapic_timer.period = min_period; 1326 } 1327 } 1328 } 1329 1330 static void apic_update_lvtt(struct kvm_lapic *apic) 1331 { 1332 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1333 apic->lapic_timer.timer_mode_mask; 1334 1335 if (apic->lapic_timer.timer_mode != timer_mode) { 1336 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1337 APIC_LVT_TIMER_TSCDEADLINE)) { 1338 hrtimer_cancel(&apic->lapic_timer.timer); 1339 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1340 apic->lapic_timer.period = 0; 1341 apic->lapic_timer.tscdeadline = 0; 1342 } 1343 apic->lapic_timer.timer_mode = timer_mode; 1344 limit_periodic_timer_frequency(apic); 1345 } 1346 } 1347 1348 static void apic_timer_expired(struct kvm_lapic *apic) 1349 { 1350 struct kvm_vcpu *vcpu = apic->vcpu; 1351 struct swait_queue_head *q = &vcpu->wq; 1352 struct kvm_timer *ktimer = &apic->lapic_timer; 1353 1354 if (atomic_read(&apic->lapic_timer.pending)) 1355 return; 1356 1357 atomic_inc(&apic->lapic_timer.pending); 1358 kvm_set_pending_timer(vcpu); 1359 1360 /* 1361 * For x86, the atomic_inc() is serialized, thus 1362 * using swait_active() is safe. 1363 */ 1364 if (swait_active(q)) 1365 swake_up(q); 1366 1367 if (apic_lvtt_tscdeadline(apic)) 1368 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1369 } 1370 1371 /* 1372 * On APICv, this test will cause a busy wait 1373 * during a higher-priority task. 1374 */ 1375 1376 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1377 { 1378 struct kvm_lapic *apic = vcpu->arch.apic; 1379 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1380 1381 if (kvm_apic_hw_enabled(apic)) { 1382 int vec = reg & APIC_VECTOR_MASK; 1383 void *bitmap = apic->regs + APIC_ISR; 1384 1385 if (vcpu->arch.apicv_active) 1386 bitmap = apic->regs + APIC_IRR; 1387 1388 if (apic_test_vector(vec, bitmap)) 1389 return true; 1390 } 1391 return false; 1392 } 1393 1394 void wait_lapic_expire(struct kvm_vcpu *vcpu) 1395 { 1396 struct kvm_lapic *apic = vcpu->arch.apic; 1397 u64 guest_tsc, tsc_deadline; 1398 1399 if (!lapic_in_kernel(vcpu)) 1400 return; 1401 1402 if (apic->lapic_timer.expired_tscdeadline == 0) 1403 return; 1404 1405 if (!lapic_timer_int_injected(vcpu)) 1406 return; 1407 1408 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1409 apic->lapic_timer.expired_tscdeadline = 0; 1410 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1411 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1412 1413 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */ 1414 if (guest_tsc < tsc_deadline) 1415 __delay(min(tsc_deadline - guest_tsc, 1416 nsec_to_cycles(vcpu, lapic_timer_advance_ns))); 1417 } 1418 1419 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1420 { 1421 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline; 1422 u64 ns = 0; 1423 ktime_t expire; 1424 struct kvm_vcpu *vcpu = apic->vcpu; 1425 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1426 unsigned long flags; 1427 ktime_t now; 1428 1429 if (unlikely(!tscdeadline || !this_tsc_khz)) 1430 return; 1431 1432 local_irq_save(flags); 1433 1434 now = ktime_get(); 1435 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1436 if (likely(tscdeadline > guest_tsc)) { 1437 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1438 do_div(ns, this_tsc_khz); 1439 expire = ktime_add_ns(now, ns); 1440 expire = ktime_sub_ns(expire, lapic_timer_advance_ns); 1441 hrtimer_start(&apic->lapic_timer.timer, 1442 expire, HRTIMER_MODE_ABS_PINNED); 1443 } else 1444 apic_timer_expired(apic); 1445 1446 local_irq_restore(flags); 1447 } 1448 1449 static void start_sw_period(struct kvm_lapic *apic) 1450 { 1451 if (!apic->lapic_timer.period) 1452 return; 1453 1454 if (apic_lvtt_oneshot(apic) && 1455 ktime_after(ktime_get(), 1456 apic->lapic_timer.target_expiration)) { 1457 apic_timer_expired(apic); 1458 return; 1459 } 1460 1461 hrtimer_start(&apic->lapic_timer.timer, 1462 apic->lapic_timer.target_expiration, 1463 HRTIMER_MODE_ABS_PINNED); 1464 } 1465 1466 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1467 { 1468 ktime_t now, remaining; 1469 u64 ns_remaining_old, ns_remaining_new; 1470 1471 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1472 * APIC_BUS_CYCLE_NS * apic->divide_count; 1473 limit_periodic_timer_frequency(apic); 1474 1475 now = ktime_get(); 1476 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1477 if (ktime_to_ns(remaining) < 0) 1478 remaining = 0; 1479 1480 ns_remaining_old = ktime_to_ns(remaining); 1481 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1482 apic->divide_count, old_divisor); 1483 1484 apic->lapic_timer.tscdeadline += 1485 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1486 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1487 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1488 } 1489 1490 static bool set_target_expiration(struct kvm_lapic *apic) 1491 { 1492 ktime_t now; 1493 u64 tscl = rdtsc(); 1494 1495 now = ktime_get(); 1496 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1497 * APIC_BUS_CYCLE_NS * apic->divide_count; 1498 1499 if (!apic->lapic_timer.period) { 1500 apic->lapic_timer.tscdeadline = 0; 1501 return false; 1502 } 1503 1504 limit_periodic_timer_frequency(apic); 1505 1506 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" 1507 PRIx64 ", " 1508 "timer initial count 0x%x, period %lldns, " 1509 "expire @ 0x%016" PRIx64 ".\n", __func__, 1510 APIC_BUS_CYCLE_NS, ktime_to_ns(now), 1511 kvm_lapic_get_reg(apic, APIC_TMICT), 1512 apic->lapic_timer.period, 1513 ktime_to_ns(ktime_add_ns(now, 1514 apic->lapic_timer.period))); 1515 1516 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 1517 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1518 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); 1519 1520 return true; 1521 } 1522 1523 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 1524 { 1525 apic->lapic_timer.tscdeadline += 1526 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1527 apic->lapic_timer.target_expiration = 1528 ktime_add_ns(apic->lapic_timer.target_expiration, 1529 apic->lapic_timer.period); 1530 } 1531 1532 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 1533 { 1534 if (!lapic_in_kernel(vcpu)) 1535 return false; 1536 1537 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 1538 } 1539 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use); 1540 1541 static void cancel_hv_timer(struct kvm_lapic *apic) 1542 { 1543 WARN_ON(preemptible()); 1544 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1545 kvm_x86_ops->cancel_hv_timer(apic->vcpu); 1546 apic->lapic_timer.hv_timer_in_use = false; 1547 } 1548 1549 static bool start_hv_timer(struct kvm_lapic *apic) 1550 { 1551 struct kvm_timer *ktimer = &apic->lapic_timer; 1552 int r; 1553 1554 WARN_ON(preemptible()); 1555 if (!kvm_x86_ops->set_hv_timer) 1556 return false; 1557 1558 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1559 return false; 1560 1561 if (!ktimer->tscdeadline) 1562 return false; 1563 1564 r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline); 1565 if (r < 0) 1566 return false; 1567 1568 ktimer->hv_timer_in_use = true; 1569 hrtimer_cancel(&ktimer->timer); 1570 1571 /* 1572 * Also recheck ktimer->pending, in case the sw timer triggered in 1573 * the window. For periodic timer, leave the hv timer running for 1574 * simplicity, and the deadline will be recomputed on the next vmexit. 1575 */ 1576 if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) { 1577 if (r) 1578 apic_timer_expired(apic); 1579 return false; 1580 } 1581 1582 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true); 1583 return true; 1584 } 1585 1586 static void start_sw_timer(struct kvm_lapic *apic) 1587 { 1588 struct kvm_timer *ktimer = &apic->lapic_timer; 1589 1590 WARN_ON(preemptible()); 1591 if (apic->lapic_timer.hv_timer_in_use) 1592 cancel_hv_timer(apic); 1593 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1594 return; 1595 1596 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1597 start_sw_period(apic); 1598 else if (apic_lvtt_tscdeadline(apic)) 1599 start_sw_tscdeadline(apic); 1600 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 1601 } 1602 1603 static void restart_apic_timer(struct kvm_lapic *apic) 1604 { 1605 preempt_disable(); 1606 if (!start_hv_timer(apic)) 1607 start_sw_timer(apic); 1608 preempt_enable(); 1609 } 1610 1611 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 1612 { 1613 struct kvm_lapic *apic = vcpu->arch.apic; 1614 1615 preempt_disable(); 1616 /* If the preempt notifier has already run, it also called apic_timer_expired */ 1617 if (!apic->lapic_timer.hv_timer_in_use) 1618 goto out; 1619 WARN_ON(swait_active(&vcpu->wq)); 1620 cancel_hv_timer(apic); 1621 apic_timer_expired(apic); 1622 1623 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1624 advance_periodic_target_expiration(apic); 1625 restart_apic_timer(apic); 1626 } 1627 out: 1628 preempt_enable(); 1629 } 1630 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 1631 1632 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 1633 { 1634 restart_apic_timer(vcpu->arch.apic); 1635 } 1636 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer); 1637 1638 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 1639 { 1640 struct kvm_lapic *apic = vcpu->arch.apic; 1641 1642 preempt_disable(); 1643 /* Possibly the TSC deadline timer is not enabled yet */ 1644 if (apic->lapic_timer.hv_timer_in_use) 1645 start_sw_timer(apic); 1646 preempt_enable(); 1647 } 1648 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer); 1649 1650 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 1651 { 1652 struct kvm_lapic *apic = vcpu->arch.apic; 1653 1654 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1655 restart_apic_timer(apic); 1656 } 1657 1658 static void start_apic_timer(struct kvm_lapic *apic) 1659 { 1660 atomic_set(&apic->lapic_timer.pending, 0); 1661 1662 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1663 && !set_target_expiration(apic)) 1664 return; 1665 1666 restart_apic_timer(apic); 1667 } 1668 1669 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1670 { 1671 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 1672 1673 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 1674 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 1675 if (lvt0_in_nmi_mode) { 1676 apic_debug("Receive NMI setting on APIC_LVT0 " 1677 "for cpu %d\n", apic->vcpu->vcpu_id); 1678 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1679 } else 1680 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1681 } 1682 } 1683 1684 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 1685 { 1686 int ret = 0; 1687 1688 trace_kvm_apic_write(reg, val); 1689 1690 switch (reg) { 1691 case APIC_ID: /* Local APIC ID */ 1692 if (!apic_x2apic_mode(apic)) 1693 kvm_apic_set_xapic_id(apic, val >> 24); 1694 else 1695 ret = 1; 1696 break; 1697 1698 case APIC_TASKPRI: 1699 report_tpr_access(apic, true); 1700 apic_set_tpr(apic, val & 0xff); 1701 break; 1702 1703 case APIC_EOI: 1704 apic_set_eoi(apic); 1705 break; 1706 1707 case APIC_LDR: 1708 if (!apic_x2apic_mode(apic)) 1709 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 1710 else 1711 ret = 1; 1712 break; 1713 1714 case APIC_DFR: 1715 if (!apic_x2apic_mode(apic)) { 1716 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1717 recalculate_apic_map(apic->vcpu->kvm); 1718 } else 1719 ret = 1; 1720 break; 1721 1722 case APIC_SPIV: { 1723 u32 mask = 0x3ff; 1724 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1725 mask |= APIC_SPIV_DIRECTED_EOI; 1726 apic_set_spiv(apic, val & mask); 1727 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1728 int i; 1729 u32 lvt_val; 1730 1731 for (i = 0; i < KVM_APIC_LVT_NUM; i++) { 1732 lvt_val = kvm_lapic_get_reg(apic, 1733 APIC_LVTT + 0x10 * i); 1734 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, 1735 lvt_val | APIC_LVT_MASKED); 1736 } 1737 apic_update_lvtt(apic); 1738 atomic_set(&apic->lapic_timer.pending, 0); 1739 1740 } 1741 break; 1742 } 1743 case APIC_ICR: 1744 /* No delay here, so we always clear the pending bit */ 1745 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); 1746 apic_send_ipi(apic); 1747 break; 1748 1749 case APIC_ICR2: 1750 if (!apic_x2apic_mode(apic)) 1751 val &= 0xff000000; 1752 kvm_lapic_set_reg(apic, APIC_ICR2, val); 1753 break; 1754 1755 case APIC_LVT0: 1756 apic_manage_nmi_watchdog(apic, val); 1757 case APIC_LVTTHMR: 1758 case APIC_LVTPC: 1759 case APIC_LVT1: 1760 case APIC_LVTERR: 1761 /* TODO: Check vector */ 1762 if (!kvm_apic_sw_enabled(apic)) 1763 val |= APIC_LVT_MASKED; 1764 1765 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1766 kvm_lapic_set_reg(apic, reg, val); 1767 1768 break; 1769 1770 case APIC_LVTT: 1771 if (!kvm_apic_sw_enabled(apic)) 1772 val |= APIC_LVT_MASKED; 1773 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1774 kvm_lapic_set_reg(apic, APIC_LVTT, val); 1775 apic_update_lvtt(apic); 1776 break; 1777 1778 case APIC_TMICT: 1779 if (apic_lvtt_tscdeadline(apic)) 1780 break; 1781 1782 hrtimer_cancel(&apic->lapic_timer.timer); 1783 kvm_lapic_set_reg(apic, APIC_TMICT, val); 1784 start_apic_timer(apic); 1785 break; 1786 1787 case APIC_TDCR: { 1788 uint32_t old_divisor = apic->divide_count; 1789 1790 if (val & 4) 1791 apic_debug("KVM_WRITE:TDCR %x\n", val); 1792 kvm_lapic_set_reg(apic, APIC_TDCR, val); 1793 update_divide_count(apic); 1794 if (apic->divide_count != old_divisor && 1795 apic->lapic_timer.period) { 1796 hrtimer_cancel(&apic->lapic_timer.timer); 1797 update_target_expiration(apic, old_divisor); 1798 restart_apic_timer(apic); 1799 } 1800 break; 1801 } 1802 case APIC_ESR: 1803 if (apic_x2apic_mode(apic) && val != 0) { 1804 apic_debug("KVM_WRITE:ESR not zero %x\n", val); 1805 ret = 1; 1806 } 1807 break; 1808 1809 case APIC_SELF_IPI: 1810 if (apic_x2apic_mode(apic)) { 1811 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff)); 1812 } else 1813 ret = 1; 1814 break; 1815 default: 1816 ret = 1; 1817 break; 1818 } 1819 if (ret) 1820 apic_debug("Local APIC Write to read-only register %x\n", reg); 1821 return ret; 1822 } 1823 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write); 1824 1825 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1826 gpa_t address, int len, const void *data) 1827 { 1828 struct kvm_lapic *apic = to_lapic(this); 1829 unsigned int offset = address - apic->base_address; 1830 u32 val; 1831 1832 if (!apic_mmio_in_range(apic, address)) 1833 return -EOPNOTSUPP; 1834 1835 /* 1836 * APIC register must be aligned on 128-bits boundary. 1837 * 32/64/128 bits registers must be accessed thru 32 bits. 1838 * Refer SDM 8.4.1 1839 */ 1840 if (len != 4 || (offset & 0xf)) { 1841 /* Don't shout loud, $infamous_os would cause only noise. */ 1842 apic_debug("apic write: bad size=%d %lx\n", len, (long)address); 1843 return 0; 1844 } 1845 1846 val = *(u32*)data; 1847 1848 /* too common printing */ 1849 if (offset != APIC_EOI) 1850 apic_debug("%s: offset 0x%x with length 0x%x, and value is " 1851 "0x%x\n", __func__, offset, len, val); 1852 1853 kvm_lapic_reg_write(apic, offset & 0xff0, val); 1854 1855 return 0; 1856 } 1857 1858 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 1859 { 1860 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 1861 } 1862 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 1863 1864 /* emulate APIC access in a trap manner */ 1865 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 1866 { 1867 u32 val = 0; 1868 1869 /* hw has done the conditional check and inst decode */ 1870 offset &= 0xff0; 1871 1872 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val); 1873 1874 /* TODO: optimize to just emulate side effect w/o one more write */ 1875 kvm_lapic_reg_write(vcpu->arch.apic, offset, val); 1876 } 1877 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 1878 1879 void kvm_free_lapic(struct kvm_vcpu *vcpu) 1880 { 1881 struct kvm_lapic *apic = vcpu->arch.apic; 1882 1883 if (!vcpu->arch.apic) 1884 return; 1885 1886 hrtimer_cancel(&apic->lapic_timer.timer); 1887 1888 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 1889 static_key_slow_dec_deferred(&apic_hw_disabled); 1890 1891 if (!apic->sw_enabled) 1892 static_key_slow_dec_deferred(&apic_sw_disabled); 1893 1894 if (apic->regs) 1895 free_page((unsigned long)apic->regs); 1896 1897 kfree(apic); 1898 } 1899 1900 /* 1901 *---------------------------------------------------------------------- 1902 * LAPIC interface 1903 *---------------------------------------------------------------------- 1904 */ 1905 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 1906 { 1907 struct kvm_lapic *apic = vcpu->arch.apic; 1908 1909 if (!lapic_in_kernel(vcpu) || 1910 !apic_lvtt_tscdeadline(apic)) 1911 return 0; 1912 1913 return apic->lapic_timer.tscdeadline; 1914 } 1915 1916 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 1917 { 1918 struct kvm_lapic *apic = vcpu->arch.apic; 1919 1920 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) || 1921 apic_lvtt_period(apic)) 1922 return; 1923 1924 hrtimer_cancel(&apic->lapic_timer.timer); 1925 apic->lapic_timer.tscdeadline = data; 1926 start_apic_timer(apic); 1927 } 1928 1929 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 1930 { 1931 struct kvm_lapic *apic = vcpu->arch.apic; 1932 1933 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 1934 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4)); 1935 } 1936 1937 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 1938 { 1939 u64 tpr; 1940 1941 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 1942 1943 return (tpr & 0xf0) >> 4; 1944 } 1945 1946 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 1947 { 1948 u64 old_value = vcpu->arch.apic_base; 1949 struct kvm_lapic *apic = vcpu->arch.apic; 1950 1951 if (!apic) 1952 value |= MSR_IA32_APICBASE_BSP; 1953 1954 vcpu->arch.apic_base = value; 1955 1956 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 1957 kvm_update_cpuid(vcpu); 1958 1959 if (!apic) 1960 return; 1961 1962 /* update jump label if enable bit changes */ 1963 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 1964 if (value & MSR_IA32_APICBASE_ENABLE) { 1965 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 1966 static_key_slow_dec_deferred(&apic_hw_disabled); 1967 } else { 1968 static_key_slow_inc(&apic_hw_disabled.key); 1969 recalculate_apic_map(vcpu->kvm); 1970 } 1971 } 1972 1973 if ((old_value ^ value) & X2APIC_ENABLE) { 1974 if (value & X2APIC_ENABLE) { 1975 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 1976 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true); 1977 } else 1978 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false); 1979 } 1980 1981 apic->base_address = apic->vcpu->arch.apic_base & 1982 MSR_IA32_APICBASE_BASE; 1983 1984 if ((value & MSR_IA32_APICBASE_ENABLE) && 1985 apic->base_address != APIC_DEFAULT_PHYS_BASE) 1986 pr_warn_once("APIC base relocation is unsupported by KVM"); 1987 1988 /* with FSB delivery interrupt, we can restart APIC functionality */ 1989 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " 1990 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); 1991 1992 } 1993 1994 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 1995 { 1996 struct kvm_lapic *apic; 1997 int i; 1998 1999 apic_debug("%s\n", __func__); 2000 2001 ASSERT(vcpu); 2002 apic = vcpu->arch.apic; 2003 ASSERT(apic != NULL); 2004 2005 /* Stop the timer in case it's a reset to an active apic */ 2006 hrtimer_cancel(&apic->lapic_timer.timer); 2007 2008 if (!init_event) { 2009 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE | 2010 MSR_IA32_APICBASE_ENABLE); 2011 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2012 } 2013 kvm_apic_set_version(apic->vcpu); 2014 2015 for (i = 0; i < KVM_APIC_LVT_NUM; i++) 2016 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); 2017 apic_update_lvtt(apic); 2018 if (kvm_vcpu_is_reset_bsp(vcpu) && 2019 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2020 kvm_lapic_set_reg(apic, APIC_LVT0, 2021 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2022 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2023 2024 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU); 2025 apic_set_spiv(apic, 0xff); 2026 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2027 if (!apic_x2apic_mode(apic)) 2028 kvm_apic_set_ldr(apic, 0); 2029 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2030 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2031 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2032 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2033 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2034 for (i = 0; i < 8; i++) { 2035 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2036 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2037 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2038 } 2039 apic->irr_pending = vcpu->arch.apicv_active; 2040 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0; 2041 apic->highest_isr_cache = -1; 2042 update_divide_count(apic); 2043 atomic_set(&apic->lapic_timer.pending, 0); 2044 if (kvm_vcpu_is_bsp(vcpu)) 2045 kvm_lapic_set_base(vcpu, 2046 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); 2047 vcpu->arch.pv_eoi.msr_val = 0; 2048 apic_update_ppr(apic); 2049 if (vcpu->arch.apicv_active) { 2050 kvm_x86_ops->apicv_post_state_restore(vcpu); 2051 kvm_x86_ops->hwapic_irr_update(vcpu, -1); 2052 kvm_x86_ops->hwapic_isr_update(vcpu, -1); 2053 } 2054 2055 vcpu->arch.apic_arb_prio = 0; 2056 vcpu->arch.apic_attention = 0; 2057 2058 apic_debug("%s: vcpu=%p, id=0x%x, base_msr=" 2059 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__, 2060 vcpu, kvm_lapic_get_reg(apic, APIC_ID), 2061 vcpu->arch.apic_base, apic->base_address); 2062 } 2063 2064 /* 2065 *---------------------------------------------------------------------- 2066 * timer interface 2067 *---------------------------------------------------------------------- 2068 */ 2069 2070 static bool lapic_is_periodic(struct kvm_lapic *apic) 2071 { 2072 return apic_lvtt_period(apic); 2073 } 2074 2075 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2076 { 2077 struct kvm_lapic *apic = vcpu->arch.apic; 2078 2079 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2080 return atomic_read(&apic->lapic_timer.pending); 2081 2082 return 0; 2083 } 2084 2085 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2086 { 2087 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2088 int vector, mode, trig_mode; 2089 2090 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2091 vector = reg & APIC_VECTOR_MASK; 2092 mode = reg & APIC_MODE_MASK; 2093 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2094 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 2095 NULL); 2096 } 2097 return 0; 2098 } 2099 2100 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2101 { 2102 struct kvm_lapic *apic = vcpu->arch.apic; 2103 2104 if (apic) 2105 kvm_apic_local_deliver(apic, APIC_LVT0); 2106 } 2107 2108 static const struct kvm_io_device_ops apic_mmio_ops = { 2109 .read = apic_mmio_read, 2110 .write = apic_mmio_write, 2111 }; 2112 2113 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2114 { 2115 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2116 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2117 2118 apic_timer_expired(apic); 2119 2120 if (lapic_is_periodic(apic)) { 2121 advance_periodic_target_expiration(apic); 2122 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2123 return HRTIMER_RESTART; 2124 } else 2125 return HRTIMER_NORESTART; 2126 } 2127 2128 int kvm_create_lapic(struct kvm_vcpu *vcpu) 2129 { 2130 struct kvm_lapic *apic; 2131 2132 ASSERT(vcpu != NULL); 2133 apic_debug("apic_init %d\n", vcpu->vcpu_id); 2134 2135 apic = kzalloc(sizeof(*apic), GFP_KERNEL); 2136 if (!apic) 2137 goto nomem; 2138 2139 vcpu->arch.apic = apic; 2140 2141 apic->regs = (void *)get_zeroed_page(GFP_KERNEL); 2142 if (!apic->regs) { 2143 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2144 vcpu->vcpu_id); 2145 goto nomem_free_apic; 2146 } 2147 apic->vcpu = vcpu; 2148 2149 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2150 HRTIMER_MODE_ABS_PINNED); 2151 apic->lapic_timer.timer.function = apic_timer_fn; 2152 2153 /* 2154 * APIC is created enabled. This will prevent kvm_lapic_set_base from 2155 * thinking that APIC satet has changed. 2156 */ 2157 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2158 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2159 kvm_lapic_reset(vcpu, false); 2160 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2161 2162 return 0; 2163 nomem_free_apic: 2164 kfree(apic); 2165 nomem: 2166 return -ENOMEM; 2167 } 2168 2169 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2170 { 2171 struct kvm_lapic *apic = vcpu->arch.apic; 2172 u32 ppr; 2173 2174 if (!apic_enabled(apic)) 2175 return -1; 2176 2177 __apic_update_ppr(apic, &ppr); 2178 return apic_has_interrupt_for_ppr(apic, ppr); 2179 } 2180 2181 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2182 { 2183 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2184 int r = 0; 2185 2186 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2187 r = 1; 2188 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2189 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2190 r = 1; 2191 return r; 2192 } 2193 2194 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2195 { 2196 struct kvm_lapic *apic = vcpu->arch.apic; 2197 2198 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2199 kvm_apic_local_deliver(apic, APIC_LVTT); 2200 if (apic_lvtt_tscdeadline(apic)) 2201 apic->lapic_timer.tscdeadline = 0; 2202 if (apic_lvtt_oneshot(apic)) { 2203 apic->lapic_timer.tscdeadline = 0; 2204 apic->lapic_timer.target_expiration = 0; 2205 } 2206 atomic_set(&apic->lapic_timer.pending, 0); 2207 } 2208 } 2209 2210 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2211 { 2212 int vector = kvm_apic_has_interrupt(vcpu); 2213 struct kvm_lapic *apic = vcpu->arch.apic; 2214 u32 ppr; 2215 2216 if (vector == -1) 2217 return -1; 2218 2219 /* 2220 * We get here even with APIC virtualization enabled, if doing 2221 * nested virtualization and L1 runs with the "acknowledge interrupt 2222 * on exit" mode. Then we cannot inject the interrupt via RVI, 2223 * because the process would deliver it through the IDT. 2224 */ 2225 2226 apic_clear_irr(vector, apic); 2227 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) { 2228 /* 2229 * For auto-EOI interrupts, there might be another pending 2230 * interrupt above PPR, so check whether to raise another 2231 * KVM_REQ_EVENT. 2232 */ 2233 apic_update_ppr(apic); 2234 } else { 2235 /* 2236 * For normal interrupts, PPR has been raised and there cannot 2237 * be a higher-priority pending interrupt---except if there was 2238 * a concurrent interrupt injection, but that would have 2239 * triggered KVM_REQ_EVENT already. 2240 */ 2241 apic_set_isr(vector, apic); 2242 __apic_update_ppr(apic, &ppr); 2243 } 2244 2245 return vector; 2246 } 2247 2248 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2249 struct kvm_lapic_state *s, bool set) 2250 { 2251 if (apic_x2apic_mode(vcpu->arch.apic)) { 2252 u32 *id = (u32 *)(s->regs + APIC_ID); 2253 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2254 2255 if (vcpu->kvm->arch.x2apic_format) { 2256 if (*id != vcpu->vcpu_id) 2257 return -EINVAL; 2258 } else { 2259 if (set) 2260 *id >>= 24; 2261 else 2262 *id <<= 24; 2263 } 2264 2265 /* In x2APIC mode, the LDR is fixed and based on the id */ 2266 if (set) 2267 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2268 } 2269 2270 return 0; 2271 } 2272 2273 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2274 { 2275 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2276 return kvm_apic_state_fixup(vcpu, s, false); 2277 } 2278 2279 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2280 { 2281 struct kvm_lapic *apic = vcpu->arch.apic; 2282 int r; 2283 2284 2285 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2286 /* set SPIV separately to get count of SW disabled APICs right */ 2287 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2288 2289 r = kvm_apic_state_fixup(vcpu, s, true); 2290 if (r) 2291 return r; 2292 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); 2293 2294 recalculate_apic_map(vcpu->kvm); 2295 kvm_apic_set_version(vcpu); 2296 2297 apic_update_ppr(apic); 2298 hrtimer_cancel(&apic->lapic_timer.timer); 2299 apic_update_lvtt(apic); 2300 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2301 update_divide_count(apic); 2302 start_apic_timer(apic); 2303 apic->irr_pending = true; 2304 apic->isr_count = vcpu->arch.apicv_active ? 2305 1 : count_vectors(apic->regs + APIC_ISR); 2306 apic->highest_isr_cache = -1; 2307 if (vcpu->arch.apicv_active) { 2308 kvm_x86_ops->apicv_post_state_restore(vcpu); 2309 kvm_x86_ops->hwapic_irr_update(vcpu, 2310 apic_find_highest_irr(apic)); 2311 kvm_x86_ops->hwapic_isr_update(vcpu, 2312 apic_find_highest_isr(apic)); 2313 } 2314 kvm_make_request(KVM_REQ_EVENT, vcpu); 2315 if (ioapic_in_kernel(vcpu->kvm)) 2316 kvm_rtc_eoi_tracking_restore_one(vcpu); 2317 2318 vcpu->arch.apic_arb_prio = 0; 2319 2320 return 0; 2321 } 2322 2323 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 2324 { 2325 struct hrtimer *timer; 2326 2327 if (!lapic_in_kernel(vcpu)) 2328 return; 2329 2330 timer = &vcpu->arch.apic->lapic_timer.timer; 2331 if (hrtimer_cancel(timer)) 2332 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 2333 } 2334 2335 /* 2336 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 2337 * 2338 * Detect whether guest triggered PV EOI since the 2339 * last entry. If yes, set EOI on guests's behalf. 2340 * Clear PV EOI in guest memory in any case. 2341 */ 2342 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 2343 struct kvm_lapic *apic) 2344 { 2345 bool pending; 2346 int vector; 2347 /* 2348 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 2349 * and KVM_PV_EOI_ENABLED in guest memory as follows: 2350 * 2351 * KVM_APIC_PV_EOI_PENDING is unset: 2352 * -> host disabled PV EOI. 2353 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 2354 * -> host enabled PV EOI, guest did not execute EOI yet. 2355 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 2356 * -> host enabled PV EOI, guest executed EOI. 2357 */ 2358 BUG_ON(!pv_eoi_enabled(vcpu)); 2359 pending = pv_eoi_get_pending(vcpu); 2360 /* 2361 * Clear pending bit in any case: it will be set again on vmentry. 2362 * While this might not be ideal from performance point of view, 2363 * this makes sure pv eoi is only enabled when we know it's safe. 2364 */ 2365 pv_eoi_clr_pending(vcpu); 2366 if (pending) 2367 return; 2368 vector = apic_set_eoi(apic); 2369 trace_kvm_pv_eoi(apic, vector); 2370 } 2371 2372 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 2373 { 2374 u32 data; 2375 2376 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 2377 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 2378 2379 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2380 return; 2381 2382 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2383 sizeof(u32))) 2384 return; 2385 2386 apic_set_tpr(vcpu->arch.apic, data & 0xff); 2387 } 2388 2389 /* 2390 * apic_sync_pv_eoi_to_guest - called before vmentry 2391 * 2392 * Detect whether it's safe to enable PV EOI and 2393 * if yes do so. 2394 */ 2395 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 2396 struct kvm_lapic *apic) 2397 { 2398 if (!pv_eoi_enabled(vcpu) || 2399 /* IRR set or many bits in ISR: could be nested. */ 2400 apic->irr_pending || 2401 /* Cache not set: could be safe but we don't bother. */ 2402 apic->highest_isr_cache == -1 || 2403 /* Need EOI to update ioapic. */ 2404 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 2405 /* 2406 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 2407 * so we need not do anything here. 2408 */ 2409 return; 2410 } 2411 2412 pv_eoi_set_pending(apic->vcpu); 2413 } 2414 2415 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 2416 { 2417 u32 data, tpr; 2418 int max_irr, max_isr; 2419 struct kvm_lapic *apic = vcpu->arch.apic; 2420 2421 apic_sync_pv_eoi_to_guest(vcpu, apic); 2422 2423 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2424 return; 2425 2426 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 2427 max_irr = apic_find_highest_irr(apic); 2428 if (max_irr < 0) 2429 max_irr = 0; 2430 max_isr = apic_find_highest_isr(apic); 2431 if (max_isr < 0) 2432 max_isr = 0; 2433 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 2434 2435 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2436 sizeof(u32)); 2437 } 2438 2439 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 2440 { 2441 if (vapic_addr) { 2442 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 2443 &vcpu->arch.apic->vapic_cache, 2444 vapic_addr, sizeof(u32))) 2445 return -EINVAL; 2446 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2447 } else { 2448 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2449 } 2450 2451 vcpu->arch.apic->vapic_addr = vapic_addr; 2452 return 0; 2453 } 2454 2455 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 2456 { 2457 struct kvm_lapic *apic = vcpu->arch.apic; 2458 u32 reg = (msr - APIC_BASE_MSR) << 4; 2459 2460 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2461 return 1; 2462 2463 if (reg == APIC_ICR2) 2464 return 1; 2465 2466 /* if this is ICR write vector before command */ 2467 if (reg == APIC_ICR) 2468 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2469 return kvm_lapic_reg_write(apic, reg, (u32)data); 2470 } 2471 2472 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 2473 { 2474 struct kvm_lapic *apic = vcpu->arch.apic; 2475 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; 2476 2477 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2478 return 1; 2479 2480 if (reg == APIC_DFR || reg == APIC_ICR2) { 2481 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n", 2482 reg); 2483 return 1; 2484 } 2485 2486 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2487 return 1; 2488 if (reg == APIC_ICR) 2489 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2490 2491 *data = (((u64)high) << 32) | low; 2492 2493 return 0; 2494 } 2495 2496 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 2497 { 2498 struct kvm_lapic *apic = vcpu->arch.apic; 2499 2500 if (!lapic_in_kernel(vcpu)) 2501 return 1; 2502 2503 /* if this is ICR write vector before command */ 2504 if (reg == APIC_ICR) 2505 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2506 return kvm_lapic_reg_write(apic, reg, (u32)data); 2507 } 2508 2509 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 2510 { 2511 struct kvm_lapic *apic = vcpu->arch.apic; 2512 u32 low, high = 0; 2513 2514 if (!lapic_in_kernel(vcpu)) 2515 return 1; 2516 2517 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2518 return 1; 2519 if (reg == APIC_ICR) 2520 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2521 2522 *data = (((u64)high) << 32) | low; 2523 2524 return 0; 2525 } 2526 2527 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data) 2528 { 2529 u64 addr = data & ~KVM_MSR_ENABLED; 2530 if (!IS_ALIGNED(addr, 4)) 2531 return 1; 2532 2533 vcpu->arch.pv_eoi.msr_val = data; 2534 if (!pv_eoi_enabled(vcpu)) 2535 return 0; 2536 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, 2537 addr, sizeof(u8)); 2538 } 2539 2540 void kvm_apic_accept_events(struct kvm_vcpu *vcpu) 2541 { 2542 struct kvm_lapic *apic = vcpu->arch.apic; 2543 u8 sipi_vector; 2544 unsigned long pe; 2545 2546 if (!lapic_in_kernel(vcpu) || !apic->pending_events) 2547 return; 2548 2549 /* 2550 * INITs are latched while in SMM. Because an SMM CPU cannot 2551 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs 2552 * and delay processing of INIT until the next RSM. 2553 */ 2554 if (is_smm(vcpu)) { 2555 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 2556 if (test_bit(KVM_APIC_SIPI, &apic->pending_events)) 2557 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 2558 return; 2559 } 2560 2561 pe = xchg(&apic->pending_events, 0); 2562 if (test_bit(KVM_APIC_INIT, &pe)) { 2563 kvm_lapic_reset(vcpu, true); 2564 kvm_vcpu_reset(vcpu, true); 2565 if (kvm_vcpu_is_bsp(apic->vcpu)) 2566 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2567 else 2568 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 2569 } 2570 if (test_bit(KVM_APIC_SIPI, &pe) && 2571 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 2572 /* evaluate pending_events before reading the vector */ 2573 smp_rmb(); 2574 sipi_vector = apic->sipi_vector; 2575 apic_debug("vcpu %d received sipi with vector # %x\n", 2576 vcpu->vcpu_id, sipi_vector); 2577 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector); 2578 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2579 } 2580 } 2581 2582 void kvm_lapic_init(void) 2583 { 2584 /* do not patch jump label more than once per second */ 2585 jump_label_rate_limit(&apic_hw_disabled, HZ); 2586 jump_label_rate_limit(&apic_sw_disabled, HZ); 2587 } 2588 2589 void kvm_lapic_exit(void) 2590 { 2591 static_key_deferred_flush(&apic_hw_disabled); 2592 static_key_deferred_flush(&apic_sw_disabled); 2593 } 2594