Lines Matching +full:abs +full:- +full:flat
1 // SPDX-License-Identifier: GPL-2.0-only
51 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
64 * VM-Exit (to inject the guest event) and the subsequent VM-Enter to resume
75 /* step-by-step approximation to mitigate fluctuation */
86 apic_set_reg(apic->regs, reg_off, val); in kvm_lapic_set_reg()
91 return apic_get_reg64(apic->regs, reg); in kvm_lapic_get_reg64()
97 apic_set_reg64(apic->regs, reg, val); in kvm_lapic_set_reg64()
102 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_pending_eoi()
104 return apic_test_vector(vector, apic->regs + APIC_ISR) || in kvm_apic_pending_eoi()
105 apic_test_vector(vector, apic->regs + APIC_IRR); in kvm_apic_pending_eoi()
128 return apic->vcpu->vcpu_id; in kvm_x2apic_id()
134 (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm)); in kvm_can_post_timer_interrupt()
140 && !(kvm_mwait_in_guest(vcpu->kvm) || in kvm_can_use_hv_timer()
146 return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE; in kvm_use_posted_timer_interrupt()
156 switch (map->logical_mode) { in kvm_apic_map_get_logical_dest()
158 /* Arbitrarily use the flat map so that @cluster isn't NULL. */ in kvm_apic_map_get_logical_dest()
159 *cluster = map->xapic_flat_map; in kvm_apic_map_get_logical_dest()
164 u32 max_apic_id = map->max_apic_id; in kvm_apic_map_get_logical_dest()
167 u8 cluster_size = min(max_apic_id - offset + 1, 16U); in kvm_apic_map_get_logical_dest()
169 offset = array_index_nospec(offset, map->max_apic_id + 1); in kvm_apic_map_get_logical_dest()
170 *cluster = &map->phys_map[offset]; in kvm_apic_map_get_logical_dest()
171 *mask = dest_id & (0xffff >> (16 - cluster_size)); in kvm_apic_map_get_logical_dest()
179 *cluster = map->xapic_flat_map; in kvm_apic_map_get_logical_dest()
183 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; in kvm_apic_map_get_logical_dest()
198 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_recalculate_phys_map()
208 if (WARN_ON_ONCE(xapic_id > new->max_apic_id)) in kvm_recalculate_phys_map()
209 return -EINVAL; in kvm_recalculate_phys_map()
217 if (x2apic_id > new->max_apic_id) in kvm_recalculate_phys_map()
218 return -E2BIG; in kvm_recalculate_phys_map()
223 * 32-bit value. Any unwanted aliasing due to truncation results will in kvm_recalculate_phys_map()
226 if (!apic_x2apic_mode(apic) && xapic_id != (u8)vcpu->vcpu_id) in kvm_recalculate_phys_map()
230 * Apply KVM's hotplug hack if userspace has enable 32-bit APIC IDs. in kvm_recalculate_phys_map()
236 * Honor the architectural (and KVM's non-optimized) behavior if in kvm_recalculate_phys_map()
237 * userspace has not enabled 32-bit x2APIC IDs. Each APIC is supposed in kvm_recalculate_phys_map()
243 if (vcpu->kvm->arch.x2apic_format) { in kvm_recalculate_phys_map()
246 new->phys_map[x2apic_id] = apic; in kvm_recalculate_phys_map()
248 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) in kvm_recalculate_phys_map()
249 new->phys_map[xapic_id] = apic; in kvm_recalculate_phys_map()
261 if (new->phys_map[physical_id]) in kvm_recalculate_phys_map()
262 return -EINVAL; in kvm_recalculate_phys_map()
264 new->phys_map[physical_id] = apic; in kvm_recalculate_phys_map()
273 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_recalculate_logical_map()
279 if (new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) in kvm_recalculate_logical_map()
300 * To optimize logical mode delivery, all software-enabled APICs must in kvm_recalculate_logical_map()
303 if (new->logical_mode == KVM_APIC_MODE_SW_DISABLED) { in kvm_recalculate_logical_map()
304 new->logical_mode = logical_mode; in kvm_recalculate_logical_map()
305 } else if (new->logical_mode != logical_mode) { in kvm_recalculate_logical_map()
306 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; in kvm_recalculate_logical_map()
311 * In x2APIC mode, the LDR is read-only and derived directly from the in kvm_recalculate_logical_map()
322 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; in kvm_recalculate_logical_map()
329 ldr = ffs(mask) - 1; in kvm_recalculate_logical_map()
331 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; in kvm_recalculate_logical_map()
337 * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock.
339 * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with
357 /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map. */ in kvm_recalculate_apic_map()
358 if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN) in kvm_recalculate_apic_map()
362 "Dirty APIC map without an in-kernel local APIC"); in kvm_recalculate_apic_map()
364 mutex_lock(&kvm->arch.apic_map_lock); in kvm_recalculate_apic_map()
368 * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map (if clean) in kvm_recalculate_apic_map()
371 * ID, i.e. the map may still show up as in-progress. In that case in kvm_recalculate_apic_map()
374 if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty, in kvm_recalculate_apic_map()
377 mutex_unlock(&kvm->arch.apic_map_lock); in kvm_recalculate_apic_map()
392 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); in kvm_recalculate_apic_map()
401 new->max_apic_id = max_id; in kvm_recalculate_apic_map()
402 new->logical_mode = KVM_APIC_MODE_SW_DISABLED; in kvm_recalculate_apic_map()
412 if (r == -E2BIG) { in kvm_recalculate_apic_map()
433 if (!new || new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) in kvm_recalculate_apic_map()
443 old = rcu_dereference_protected(kvm->arch.apic_map, in kvm_recalculate_apic_map()
444 lockdep_is_held(&kvm->arch.apic_map_lock)); in kvm_recalculate_apic_map()
445 rcu_assign_pointer(kvm->arch.apic_map, new); in kvm_recalculate_apic_map()
447 * Write kvm->arch.apic_map before clearing apic->apic_map_dirty. in kvm_recalculate_apic_map()
450 atomic_cmpxchg_release(&kvm->arch.apic_map_dirty, in kvm_recalculate_apic_map()
452 mutex_unlock(&kvm->arch.apic_map_lock); in kvm_recalculate_apic_map()
466 if (enabled != apic->sw_enabled) { in apic_set_spiv()
467 apic->sw_enabled = enabled; in apic_set_spiv()
473 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in apic_set_spiv()
478 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu); in apic_set_spiv()
479 kvm_xen_sw_enable_lapic(apic->vcpu); in apic_set_spiv()
486 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_xapic_id()
492 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_ldr()
498 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_dfr()
505 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); in kvm_apic_set_x2apic_id()
509 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_x2apic_id()
519 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; in apic_lvtt_oneshot()
524 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; in apic_lvtt_period()
529 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; in apic_lvtt_tscdeadline()
539 return apic->nr_lvt_entries > lvt_index; in kvm_lapic_lvt_supported()
544 return KVM_APIC_MAX_NR_LVT_ENTRIES - !(vcpu->arch.mcg_cap & MCG_CMCI_P); in kvm_apic_calc_nr_lvt_entries()
549 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_version()
555 v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); in kvm_apic_set_version()
558 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) in kvm_apic_set_version()
560 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC in kvm_apic_set_version()
561 * version first and level-triggered interrupts never get EOIed in in kvm_apic_set_version()
565 !ioapic_in_kernel(vcpu->kvm)) in kvm_apic_set_version()
573 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_after_set_mcg_cap()
576 if (!lapic_in_kernel(vcpu) || nr_lvt_entries == apic->nr_lvt_entries) in kvm_apic_after_set_mcg_cap()
580 for (i = apic->nr_lvt_entries; i < nr_lvt_entries; i++) in kvm_apic_after_set_mcg_cap()
583 apic->nr_lvt_entries = nr_lvt_entries; in kvm_apic_after_set_mcg_cap()
621 max_updated_irr = -1; in __kvm_apic_update_irr()
622 *max_irr = -1; in __kvm_apic_update_irr()
646 return ((max_updated_irr != -1) && in __kvm_apic_update_irr()
653 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_update_irr()
654 bool irr_updated = __kvm_apic_update_irr(pir, apic->regs, max_irr); in kvm_apic_update_irr()
656 if (unlikely(!apic->apicv_active && irr_updated)) in kvm_apic_update_irr()
657 apic->irr_pending = true; in kvm_apic_update_irr()
664 return apic_find_highest_vector(apic->regs + APIC_IRR); in apic_search_irr()
675 if (!apic->irr_pending) in apic_find_highest_irr()
676 return -1; in apic_find_highest_irr()
679 ASSERT(result == -1 || result >= 16); in apic_find_highest_irr()
686 if (unlikely(apic->apicv_active)) { in apic_clear_irr()
687 apic_clear_vector(vec, apic->regs + APIC_IRR); in apic_clear_irr()
689 apic->irr_pending = false; in apic_clear_irr()
690 apic_clear_vector(vec, apic->regs + APIC_IRR); in apic_clear_irr()
691 if (apic_search_irr(apic) != -1) in apic_clear_irr()
692 apic->irr_pending = true; in apic_clear_irr()
698 apic_clear_irr(vec, vcpu->arch.apic); in kvm_apic_clear_irr()
704 return apic->regs + APIC_ISR + APIC_VECTOR_TO_REG_OFFSET(vec); in apic_vector_to_isr()
718 if (unlikely(apic->apicv_active)) in apic_set_isr()
719 kvm_x86_call(hwapic_isr_update)(apic->vcpu, vec); in apic_set_isr()
721 ++apic->isr_count; in apic_set_isr()
722 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); in apic_set_isr()
728 apic->highest_isr_cache = vec; in apic_set_isr()
738 * is always -1, with APIC virtualization enabled. in apic_find_highest_isr()
740 if (!apic->isr_count) in apic_find_highest_isr()
741 return -1; in apic_find_highest_isr()
742 if (likely(apic->highest_isr_cache != -1)) in apic_find_highest_isr()
743 return apic->highest_isr_cache; in apic_find_highest_isr()
745 result = apic_find_highest_vector(apic->regs + APIC_ISR); in apic_find_highest_isr()
746 ASSERT(result == -1 || result >= 16); in apic_find_highest_isr()
759 * uses the Hyper-V APIC enlightenment. In this case we may need in apic_clear_isr()
764 if (unlikely(apic->apicv_active)) in apic_clear_isr()
765 kvm_x86_call(hwapic_isr_update)(apic->vcpu, apic_find_highest_isr(apic)); in apic_clear_isr()
767 --apic->isr_count; in apic_clear_isr()
768 BUG_ON(apic->isr_count < 0); in apic_clear_isr()
769 apic->highest_isr_cache = -1; in apic_clear_isr()
775 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_update_hwapic_isr()
777 if (WARN_ON_ONCE(!lapic_in_kernel(vcpu)) || !apic->apicv_active) in kvm_apic_update_hwapic_isr()
791 return apic_find_highest_irr(vcpu->arch.apic); in kvm_lapic_find_highest_irr()
802 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_irq()
804 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, in kvm_apic_set_irq()
805 irq->level, irq->trig_mode, dest_map); in kvm_apic_set_irq()
814 if (min > map->max_apic_id) in __pv_send_ipi()
817 min = array_index_nospec(min, map->max_apic_id + 1); in __pv_send_ipi()
820 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { in __pv_send_ipi()
821 if (map->phys_map[min + i]) { in __pv_send_ipi()
822 vcpu = map->phys_map[min + i]->vcpu; in __pv_send_ipi()
840 return -KVM_EINVAL; in kvm_pv_send_ipi()
848 map = rcu_dereference(kvm->arch.apic_map); in kvm_pv_send_ipi()
850 count = -EOPNOTSUPP; in kvm_pv_send_ipi()
864 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, in pv_eoi_put_user()
871 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, in pv_eoi_get_user()
877 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; in pv_eoi_enabled()
885 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); in pv_eoi_set_pending()
905 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); in pv_eoi_test_and_clr_pending()
914 highest_irr = kvm_x86_call(sync_pir_to_irr)(apic->vcpu); in apic_has_interrupt_for_ppr()
917 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) in apic_has_interrupt_for_ppr()
918 return -1; in apic_has_interrupt_for_ppr()
930 isrv = (isr != -1) ? isr : 0; in __apic_update_ppr()
949 apic_has_interrupt_for_ppr(apic, ppr) != -1) in apic_update_ppr()
950 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in apic_update_ppr()
955 apic_update_ppr(vcpu->arch.apic); in kvm_apic_update_ppr()
1019 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
1023 * - in-kernel IOAPIC messages have to be delivered directly to
1026 * rewrites the destination of non-IPI messages from APIC_BROADCAST
1030 * important when userspace wants to use x2APIC-format MSIs, because
1031 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
1038 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && in kvm_apic_mda()
1048 struct kvm_lapic *target = vcpu->arch.apic; in kvm_apic_match_dest()
1081 if (!kvm->arch.disabled_lapic_found) { in kvm_apic_disabled_lapic_found()
1082 kvm->arch.disabled_lapic_found = true; in kvm_apic_disabled_lapic_found()
1090 if (kvm->arch.x2apic_broadcast_quirk_disabled) { in kvm_apic_is_broadcast_dest()
1091 if ((irq->dest_id == APIC_BROADCAST && in kvm_apic_is_broadcast_dest()
1092 map->logical_mode != KVM_APIC_MODE_X2APIC)) in kvm_apic_is_broadcast_dest()
1094 if (irq->dest_id == X2APIC_BROADCAST) in kvm_apic_is_broadcast_dest()
1098 if (irq->dest_id == (x2apic_ipi ? in kvm_apic_is_broadcast_dest()
1108 return (irq->delivery_mode == APIC_DM_LOWEST || irq->msi_redir_hint); in kvm_lowest_prio_delivery()
1113 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; in kvm_apic_compare_prio()
1130 if (irq->shorthand == APIC_DEST_SELF && src) { in kvm_apic_map_get_dest_lapic()
1134 } else if (irq->shorthand) in kvm_apic_map_get_dest_lapic()
1140 if (irq->dest_mode == APIC_DEST_PHYSICAL) { in kvm_apic_map_get_dest_lapic()
1141 if (irq->dest_id > map->max_apic_id) { in kvm_apic_map_get_dest_lapic()
1144 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1); in kvm_apic_map_get_dest_lapic()
1145 *dst = &map->phys_map[dest_id]; in kvm_apic_map_get_dest_lapic()
1152 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, in kvm_apic_map_get_dest_lapic()
1160 lowest = -1; in kvm_apic_map_get_dest_lapic()
1166 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, in kvm_apic_map_get_dest_lapic()
1167 (*dst)[lowest]->vcpu) < 0) in kvm_apic_map_get_dest_lapic()
1174 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), in kvm_apic_map_get_dest_lapic()
1198 *r = -1; in kvm_irq_delivery_to_apic_fast()
1200 if (irq->shorthand == APIC_DEST_SELF) { in kvm_irq_delivery_to_apic_fast()
1205 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); in kvm_irq_delivery_to_apic_fast()
1210 map = rcu_dereference(kvm->arch.apic_map); in kvm_irq_delivery_to_apic_fast()
1218 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); in kvm_irq_delivery_to_apic_fast()
1229 * - For single-destination interrupts, handle it in posted mode
1230 * - Else if vector hashing is enabled and it is a lowest-priority
1233 * 1. For lowest-priority interrupts, store all the possible
1236 * the right destination vCPU in the array for the lowest-priority
1238 * - Otherwise, use remapped mode to inject the interrupt.
1249 if (irq->shorthand) in kvm_intr_is_single_vcpu_fast()
1253 map = rcu_dereference(kvm->arch.apic_map); in kvm_intr_is_single_vcpu_fast()
1260 *dest_vcpu = dst[i]->vcpu; in kvm_intr_is_single_vcpu_fast()
1283 if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand, in kvm_intr_is_single_vcpu()
1284 irq->dest_id, irq->dest_mode)) in kvm_intr_is_single_vcpu()
1300 int r = -1; in kvm_irq_delivery_to_apic()
1308 if (irq->dest_mode == APIC_DEST_PHYSICAL && in kvm_irq_delivery_to_apic()
1309 irq->dest_id == 0xff && kvm_lowest_prio_delivery(irq)) { in kvm_irq_delivery_to_apic()
1311 irq->delivery_mode = APIC_DM_FIXED; in kvm_irq_delivery_to_apic()
1320 if (!kvm_apic_match_dest(vcpu, src, irq->shorthand, in kvm_irq_delivery_to_apic()
1321 irq->dest_id, irq->dest_mode)) in kvm_irq_delivery_to_apic()
1328 } else if (kvm_apic_sw_enabled(vcpu->arch.apic)) { in kvm_irq_delivery_to_apic()
1342 int idx = kvm_vector_to_index(irq->vector, dest_vcpus, in kvm_irq_delivery_to_apic()
1363 struct kvm_vcpu *vcpu = apic->vcpu; in __apic_accept_irq()
1365 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, in __apic_accept_irq()
1369 vcpu->arch.apic_arb_prio++; in __apic_accept_irq()
1382 __set_bit(vcpu->vcpu_id, dest_map->map); in __apic_accept_irq()
1383 dest_map->vectors[vcpu->vcpu_id] = vector; in __apic_accept_irq()
1386 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { in __apic_accept_irq()
1388 apic_set_vector(vector, apic->regs + APIC_TMR); in __apic_accept_irq()
1390 apic_clear_vector(vector, apic->regs + APIC_TMR); in __apic_accept_irq()
1399 vcpu->arch.pv.pv_unhalted = 1; in __apic_accept_irq()
1421 apic->pending_events = (1UL << KVM_APIC_INIT); in __apic_accept_irq()
1429 apic->sipi_vector = vector; in __apic_accept_irq()
1432 set_bit(KVM_APIC_SIPI, &apic->pending_events); in __apic_accept_irq()
1471 map = rcu_dereference(kvm->arch.apic_map); in kvm_bitmap_or_dest_vcpus()
1479 vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx; in kvm_bitmap_or_dest_vcpus()
1487 irq->shorthand, in kvm_bitmap_or_dest_vcpus()
1488 irq->dest_id, in kvm_bitmap_or_dest_vcpus()
1489 irq->dest_mode)) in kvm_bitmap_or_dest_vcpus()
1499 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); in kvm_ioapic_handles_vector()
1512 * routing, then re-scan the I/O APIC routes as EOIs for the IRQ likely in kvm_ioapic_send_eoi()
1515 if (apic->vcpu->arch.highest_stale_pending_ioapic_eoi == vector) in kvm_ioapic_send_eoi()
1516 kvm_make_request(KVM_REQ_SCAN_IOAPIC, apic->vcpu); in kvm_ioapic_send_eoi()
1519 if (irqchip_split(apic->vcpu->kvm)) { in kvm_ioapic_send_eoi()
1520 apic->vcpu->arch.pending_ioapic_eoi = vector; in kvm_ioapic_send_eoi()
1521 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); in kvm_ioapic_send_eoi()
1526 if (apic_test_vector(vector, apic->regs + APIC_TMR)) in kvm_ioapic_send_eoi()
1531 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); in kvm_ioapic_send_eoi()
1545 if (vector == -1) in apic_set_eoi()
1551 if (kvm_hv_synic_has_vector(apic->vcpu, vector)) in apic_set_eoi()
1552 kvm_hv_synic_send_eoi(apic->vcpu, vector); in apic_set_eoi()
1555 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in apic_set_eoi()
1560 * this interface assumes a trap-like exit, which has already finished
1565 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_eoi_accelerated()
1570 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in kvm_apic_set_eoi_accelerated()
1580 irq->vector = icr_low & APIC_VECTOR_MASK; in kvm_icr_to_lapic_irq()
1581 irq->delivery_mode = icr_low & APIC_MODE_MASK; in kvm_icr_to_lapic_irq()
1582 irq->dest_mode = icr_low & APIC_DEST_MASK; in kvm_icr_to_lapic_irq()
1583 irq->level = (icr_low & APIC_INT_ASSERT) != 0; in kvm_icr_to_lapic_irq()
1584 irq->trig_mode = icr_low & APIC_INT_LEVELTRIG; in kvm_icr_to_lapic_irq()
1585 irq->shorthand = icr_low & APIC_SHORT_MASK; in kvm_icr_to_lapic_irq()
1586 irq->msi_redir_hint = false; in kvm_icr_to_lapic_irq()
1588 irq->dest_id = icr_high; in kvm_icr_to_lapic_irq()
1590 irq->dest_id = GET_XAPIC_DEST_FIELD(icr_high); in kvm_icr_to_lapic_irq()
1601 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); in kvm_apic_send_ipi()
1614 apic->lapic_timer.period == 0) in apic_get_tmcct()
1618 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); in apic_get_tmcct()
1622 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); in apic_get_tmcct()
1623 return div64_u64(ns, (apic->vcpu->kvm->arch.apic_bus_cycle_ns * in apic_get_tmcct()
1624 apic->divide_count)); in apic_get_tmcct()
1629 struct kvm_vcpu *vcpu = apic->vcpu; in __report_tpr_access()
1630 struct kvm_run *run = vcpu->run; in __report_tpr_access()
1633 run->tpr_access.rip = kvm_rip_read(vcpu); in __report_tpr_access()
1634 run->tpr_access.is_write = write; in __report_tpr_access()
1639 if (apic->vcpu->arch.tpr_access_reporting) in report_tpr_access()
1682 (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1686 /* Leave bits '0' for reserved and write-only registers. */ in kvm_lapic_readable_reg_mask()
1729 * WARN if KVM reads ICR in x2APIC mode, as it's an 8-byte register in in kvm_lapic_reg_read()
1761 return addr >= apic->base_address && in apic_mmio_in_range()
1762 addr < apic->base_address + LAPIC_MMIO_LENGTH; in apic_mmio_in_range()
1769 u32 offset = address - apic->base_address; in apic_mmio_read()
1772 return -EOPNOTSUPP; in apic_mmio_read()
1775 if (!kvm_check_has_quirk(vcpu->kvm, in apic_mmio_read()
1777 return -EOPNOTSUPP; in apic_mmio_read()
1795 apic->divide_count = 0x1 << (tmp2 & 0x7); in update_divide_count()
1805 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { in limit_periodic_timer_frequency()
1808 if (apic->lapic_timer.period < min_period) { in limit_periodic_timer_frequency()
1812 apic->vcpu->vcpu_id, in limit_periodic_timer_frequency()
1813 apic->lapic_timer.period, min_period); in limit_periodic_timer_frequency()
1814 apic->lapic_timer.period = min_period; in limit_periodic_timer_frequency()
1823 hrtimer_cancel(&apic->lapic_timer.timer); in cancel_apic_timer()
1825 if (apic->lapic_timer.hv_timer_in_use) in cancel_apic_timer()
1828 atomic_set(&apic->lapic_timer.pending, 0); in cancel_apic_timer()
1834 apic->lapic_timer.timer_mode_mask; in apic_update_lvtt()
1836 if (apic->lapic_timer.timer_mode != timer_mode) { in apic_update_lvtt()
1841 apic->lapic_timer.period = 0; in apic_update_lvtt()
1842 apic->lapic_timer.tscdeadline = 0; in apic_update_lvtt()
1844 apic->lapic_timer.timer_mode = timer_mode; in apic_update_lvtt()
1851 * during a higher-priority task.
1856 struct kvm_lapic *apic = vcpu->arch.apic; in lapic_timer_int_injected()
1864 if (apic->guest_apic_protected) in lapic_timer_int_injected()
1870 void *bitmap = apic->regs + APIC_ISR; in lapic_timer_int_injected()
1872 if (apic->apicv_active) in lapic_timer_int_injected()
1873 bitmap = apic->regs + APIC_IRR; in lapic_timer_int_injected()
1883 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns; in __wait_lapic_expire()
1891 if (vcpu->arch.tsc_scaling_ratio == kvm_caps.default_tsc_scaling_ratio) { in __wait_lapic_expire()
1896 do_div(delay_ns, vcpu->arch.virtual_tsc_khz); in __wait_lapic_expire()
1904 struct kvm_lapic *apic = vcpu->arch.apic; in adjust_lapic_timer_advance()
1905 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns; in adjust_lapic_timer_advance()
1909 if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX || in adjust_lapic_timer_advance()
1910 abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN) in adjust_lapic_timer_advance()
1915 ns = -advance_expire_delta * 1000000ULL; in adjust_lapic_timer_advance()
1916 do_div(ns, vcpu->arch.virtual_tsc_khz); in adjust_lapic_timer_advance()
1917 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP; in adjust_lapic_timer_advance()
1921 do_div(ns, vcpu->arch.virtual_tsc_khz); in adjust_lapic_timer_advance()
1927 apic->lapic_timer.timer_advance_ns = timer_advance_ns; in adjust_lapic_timer_advance()
1932 struct kvm_lapic *apic = vcpu->arch.apic; in __kvm_wait_lapic_expire()
1935 tsc_deadline = apic->lapic_timer.expired_tscdeadline; in __kvm_wait_lapic_expire()
1936 apic->lapic_timer.expired_tscdeadline = 0; in __kvm_wait_lapic_expire()
1938 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); in __kvm_wait_lapic_expire()
1940 adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline); in __kvm_wait_lapic_expire()
1950 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc); in __kvm_wait_lapic_expire()
1956 vcpu->arch.apic->lapic_timer.expired_tscdeadline && in kvm_wait_lapic_expire()
1957 vcpu->arch.apic->lapic_timer.timer_advance_ns && in kvm_wait_lapic_expire()
1965 struct kvm_timer *ktimer = &apic->lapic_timer; in kvm_apic_inject_pending_timer_irqs()
1969 ktimer->tscdeadline = 0; in kvm_apic_inject_pending_timer_irqs()
1971 ktimer->tscdeadline = 0; in kvm_apic_inject_pending_timer_irqs()
1972 ktimer->target_expiration = 0; in kvm_apic_inject_pending_timer_irqs()
1978 struct kvm_vcpu *vcpu = apic->vcpu; in apic_timer_expired()
1979 struct kvm_timer *ktimer = &apic->lapic_timer; in apic_timer_expired()
1981 if (atomic_read(&apic->lapic_timer.pending)) in apic_timer_expired()
1984 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) in apic_timer_expired()
1985 ktimer->expired_tscdeadline = ktimer->tscdeadline; in apic_timer_expired()
1987 if (!from_timer_fn && apic->apicv_active) { in apic_timer_expired()
1993 if (kvm_use_posted_timer_interrupt(apic->vcpu)) { in apic_timer_expired()
2001 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline && in apic_timer_expired()
2002 vcpu->arch.apic->lapic_timer.timer_advance_ns) in apic_timer_expired()
2008 atomic_inc(&apic->lapic_timer.pending); in apic_timer_expired()
2016 struct kvm_timer *ktimer = &apic->lapic_timer; in start_sw_tscdeadline()
2017 u64 guest_tsc, tscdeadline = ktimer->tscdeadline; in start_sw_tscdeadline()
2020 struct kvm_vcpu *vcpu = apic->vcpu; in start_sw_tscdeadline()
2021 u32 this_tsc_khz = vcpu->arch.virtual_tsc_khz; in start_sw_tscdeadline()
2033 ns = (tscdeadline - guest_tsc) * 1000000ULL; in start_sw_tscdeadline()
2037 likely(ns > apic->lapic_timer.timer_advance_ns)) { in start_sw_tscdeadline()
2039 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns); in start_sw_tscdeadline()
2040 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD); in start_sw_tscdeadline()
2049 return (u64)tmict * apic->vcpu->kvm->arch.apic_bus_cycle_ns * in tmict_to_ns()
2050 (u64)apic->divide_count; in tmict_to_ns()
2058 apic->lapic_timer.period = in update_target_expiration()
2063 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); in update_target_expiration()
2069 apic->divide_count, old_divisor); in update_target_expiration()
2071 apic->lapic_timer.tscdeadline += in update_target_expiration()
2072 nsec_to_cycles(apic->vcpu, ns_remaining_new) - in update_target_expiration()
2073 nsec_to_cycles(apic->vcpu, ns_remaining_old); in update_target_expiration()
2074 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); in update_target_expiration()
2084 apic->lapic_timer.period = in set_target_expiration()
2087 if (!apic->lapic_timer.period) { in set_target_expiration()
2088 apic->lapic_timer.tscdeadline = 0; in set_target_expiration()
2093 deadline = apic->lapic_timer.period; in set_target_expiration()
2101 deadline = apic->lapic_timer.period; in set_target_expiration()
2105 else if (unlikely(deadline > apic->lapic_timer.period)) { in set_target_expiration()
2110 apic->vcpu->vcpu_id, in set_target_expiration()
2113 deadline, apic->lapic_timer.period); in set_target_expiration()
2115 deadline = apic->lapic_timer.period; in set_target_expiration()
2120 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + in set_target_expiration()
2121 nsec_to_cycles(apic->vcpu, deadline); in set_target_expiration()
2122 apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline); in set_target_expiration()
2140 apic->lapic_timer.target_expiration = in advance_periodic_target_expiration()
2141 ktime_add_ns(apic->lapic_timer.target_expiration, in advance_periodic_target_expiration()
2142 apic->lapic_timer.period); in advance_periodic_target_expiration()
2143 delta = ktime_sub(apic->lapic_timer.target_expiration, now); in advance_periodic_target_expiration()
2144 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + in advance_periodic_target_expiration()
2145 nsec_to_cycles(apic->vcpu, delta); in advance_periodic_target_expiration()
2150 if (!apic->lapic_timer.period) in start_sw_period()
2154 apic->lapic_timer.target_expiration)) { in start_sw_period()
2163 hrtimer_start(&apic->lapic_timer.timer, in start_sw_period()
2164 apic->lapic_timer.target_expiration, in start_sw_period()
2173 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; in kvm_lapic_hv_timer_in_use()
2179 WARN_ON(!apic->lapic_timer.hv_timer_in_use); in cancel_hv_timer()
2180 kvm_x86_call(cancel_hv_timer)(apic->vcpu); in cancel_hv_timer()
2181 apic->lapic_timer.hv_timer_in_use = false; in cancel_hv_timer()
2186 struct kvm_timer *ktimer = &apic->lapic_timer; in start_hv_timer()
2187 struct kvm_vcpu *vcpu = apic->vcpu; in start_hv_timer()
2194 if (!ktimer->tscdeadline) in start_hv_timer()
2197 if (kvm_x86_call(set_hv_timer)(vcpu, ktimer->tscdeadline, &expired)) in start_hv_timer()
2200 ktimer->hv_timer_in_use = true; in start_hv_timer()
2201 hrtimer_cancel(&ktimer->timer); in start_hv_timer()
2206 * VM-Exit to recompute the periodic timer's target expiration. in start_hv_timer()
2213 if (atomic_read(&ktimer->pending)) { in start_hv_timer()
2221 trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use); in start_hv_timer()
2228 struct kvm_timer *ktimer = &apic->lapic_timer; in start_sw_timer()
2231 if (apic->lapic_timer.hv_timer_in_use) in start_sw_timer()
2233 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) in start_sw_timer()
2240 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); in start_sw_timer()
2247 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending)) in restart_apic_timer()
2258 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_expired_hv_timer()
2262 if (!apic->lapic_timer.hv_timer_in_use) in kvm_lapic_expired_hv_timer()
2268 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { in kvm_lapic_expired_hv_timer()
2279 restart_apic_timer(vcpu->arch.apic); in kvm_lapic_switch_to_hv_timer()
2284 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_switch_to_sw_timer()
2288 if (apic->lapic_timer.hv_timer_in_use) in kvm_lapic_switch_to_sw_timer()
2295 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_restart_hv_timer()
2297 WARN_ON(!apic->lapic_timer.hv_timer_in_use); in kvm_lapic_restart_hv_timer()
2303 atomic_set(&apic->lapic_timer.pending, 0); in __start_apic_timer()
2321 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { in apic_manage_nmi_watchdog()
2322 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; in apic_manage_nmi_watchdog()
2324 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); in apic_manage_nmi_watchdog()
2326 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); in apic_manage_nmi_watchdog()
2335 return -1; in get_lvt_index()
2337 (reg - APIC_LVTT) >> 4, KVM_APIC_MAX_NR_LVT_ENTRIES); in get_lvt_index()
2386 for (i = 0; i < apic->nr_lvt_entries; i++) { in kvm_lapic_reg_write()
2391 atomic_set(&apic->lapic_timer.pending, 0); in kvm_lapic_reg_write()
2434 val &= (apic_lvt_mask[LVT_TIMER] | apic->lapic_timer.timer_mode_mask); in kvm_lapic_reg_write()
2449 uint32_t old_divisor = apic->divide_count; in kvm_lapic_reg_write()
2453 if (apic->divide_count != old_divisor && in kvm_lapic_reg_write()
2454 apic->lapic_timer.period) { in kvm_lapic_reg_write()
2455 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_lapic_reg_write()
2468 * Self-IPI exists only when x2APIC is enabled. Bits 7:0 hold in kvm_lapic_reg_write()
2486 kvm_recalculate_apic_map(apic->vcpu->kvm); in kvm_lapic_reg_write()
2495 unsigned int offset = address - apic->base_address; in apic_mmio_write()
2499 return -EOPNOTSUPP; in apic_mmio_write()
2502 if (!kvm_check_has_quirk(vcpu->kvm, in apic_mmio_write()
2504 return -EOPNOTSUPP; in apic_mmio_write()
2510 * APIC register must be aligned on 128-bits boundary. in apic_mmio_write()
2526 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); in kvm_lapic_set_eoi()
2553 if (!kvm_irq_delivery_to_apic_fast(apic->vcpu->kvm, apic, &irq, in __kvm_x2apic_icr_write()
2555 return -EWOULDBLOCK; in __kvm_x2apic_icr_write()
2593 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_write_nodecode()
2596 * ICR is a single 64-bit register when x2APIC is enabled, all others in kvm_apic_write_nodecode()
2597 * registers hold 32-bit values. For legacy xAPIC, ICR writes need to in kvm_apic_write_nodecode()
2604 * maybe-unecessary write, and both are in the noise anyways. in kvm_apic_write_nodecode()
2615 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_free_lapic()
2617 if (!vcpu->arch.apic) { in kvm_free_lapic()
2622 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_free_lapic()
2624 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) in kvm_free_lapic()
2627 if (!apic->sw_enabled) in kvm_free_lapic()
2630 if (apic->regs) in kvm_free_lapic()
2631 free_page((unsigned long)apic->regs); in kvm_free_lapic()
2637 *----------------------------------------------------------------------
2639 *----------------------------------------------------------------------
2643 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_get_lapic_tscdeadline_msr()
2648 return apic->lapic_timer.tscdeadline; in kvm_get_lapic_tscdeadline_msr()
2653 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_set_lapic_tscdeadline_msr()
2658 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_set_lapic_tscdeadline_msr()
2659 apic->lapic_timer.tscdeadline = data; in kvm_set_lapic_tscdeadline_msr()
2665 apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4); in kvm_lapic_set_tpr()
2672 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); in kvm_lapic_get_cr8()
2679 u64 old_value = vcpu->arch.apic_base; in __kvm_apic_set_base()
2680 struct kvm_lapic *apic = vcpu->arch.apic; in __kvm_apic_set_base()
2682 vcpu->arch.apic_base = value; in __kvm_apic_set_base()
2685 vcpu->arch.cpuid_dynamic_bits_dirty = true; in __kvm_apic_set_base()
2693 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in __kvm_apic_set_base()
2699 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in __kvm_apic_set_base()
2705 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); in __kvm_apic_set_base()
2707 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in __kvm_apic_set_base()
2715 apic->base_address = apic->vcpu->arch.apic_base & in __kvm_apic_set_base()
2719 apic->base_address != APIC_DEFAULT_PHYS_BASE) { in __kvm_apic_set_base()
2720 kvm_set_apicv_inhibit(apic->vcpu->kvm, in __kvm_apic_set_base()
2730 if (vcpu->arch.apic_base == value) in kvm_apic_set_base()
2746 kvm_recalculate_apic_map(vcpu->kvm); in kvm_apic_set_base()
2753 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_update_apicv()
2768 apic->irr_pending = true; in kvm_apic_update_apicv()
2770 if (apic->apicv_active) in kvm_apic_update_apicv()
2771 apic->isr_count = 1; in kvm_apic_update_apicv()
2773 apic->isr_count = count_vectors(apic->regs + APIC_ISR); in kvm_apic_update_apicv()
2775 apic->highest_isr_cache = -1; in kvm_apic_update_apicv()
2782 guard(mutex)(&kvm->slots_lock); in kvm_alloc_apic_access_page()
2784 if (kvm->arch.apic_access_memslot_enabled || in kvm_alloc_apic_access_page()
2785 kvm->arch.apic_access_memslot_inhibited) in kvm_alloc_apic_access_page()
2793 kvm->arch.apic_access_memslot_enabled = true; in kvm_alloc_apic_access_page()
2801 struct kvm *kvm = vcpu->kvm; in kvm_inhibit_apic_access_page()
2803 if (!kvm->arch.apic_access_memslot_enabled) in kvm_inhibit_apic_access_page()
2808 mutex_lock(&kvm->slots_lock); in kvm_inhibit_apic_access_page()
2810 if (kvm->arch.apic_access_memslot_enabled) { in kvm_inhibit_apic_access_page()
2820 kvm->arch.apic_access_memslot_enabled = false; in kvm_inhibit_apic_access_page()
2826 kvm->arch.apic_access_memslot_inhibited = true; in kvm_inhibit_apic_access_page()
2829 mutex_unlock(&kvm->slots_lock); in kvm_inhibit_apic_access_page()
2836 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_reset()
2860 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_lapic_reset()
2864 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in kvm_lapic_reset()
2865 kvm_apic_set_version(apic->vcpu); in kvm_lapic_reset()
2867 for (i = 0; i < apic->nr_lvt_entries; i++) in kvm_lapic_reset()
2871 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) in kvm_lapic_reset()
2897 atomic_set(&apic->lapic_timer.pending, 0); in kvm_lapic_reset()
2899 vcpu->arch.pv_eoi.msr_val = 0; in kvm_lapic_reset()
2901 if (apic->apicv_active) { in kvm_lapic_reset()
2903 kvm_x86_call(hwapic_isr_update)(vcpu, -1); in kvm_lapic_reset()
2906 vcpu->arch.apic_arb_prio = 0; in kvm_lapic_reset()
2907 vcpu->arch.apic_attention = 0; in kvm_lapic_reset()
2909 kvm_recalculate_apic_map(vcpu->kvm); in kvm_lapic_reset()
2913 *----------------------------------------------------------------------
2915 *----------------------------------------------------------------------
2925 struct kvm_lapic *apic = vcpu->arch.apic; in apic_has_pending_timer()
2928 return atomic_read(&apic->lapic_timer.pending); in apic_has_pending_timer()
2946 guest_cpuid_is_intel_compatible(apic->vcpu)) in kvm_apic_local_deliver()
2955 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_nmi_wd_deliver()
2975 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); in apic_timer_fn()
2987 if (!irqchip_in_kernel(vcpu->kvm)) { in kvm_create_lapic()
2996 vcpu->arch.apic = apic; in kvm_create_lapic()
2999 apic->regs = kvm_x86_call(alloc_apic_backing_page)(vcpu); in kvm_create_lapic()
3001 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); in kvm_create_lapic()
3002 if (!apic->regs) { in kvm_create_lapic()
3004 vcpu->vcpu_id); in kvm_create_lapic()
3007 apic->vcpu = vcpu; in kvm_create_lapic()
3009 apic->nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); in kvm_create_lapic()
3011 hrtimer_setup(&apic->lapic_timer.timer, apic_timer_fn, CLOCK_MONOTONIC, in kvm_create_lapic()
3014 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; in kvm_create_lapic()
3020 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; in kvm_create_lapic()
3022 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); in kvm_create_lapic()
3030 * unlikely to have inhibits. Ignore the current per-VM APICv state so in kvm_create_lapic()
3032 * the request will ensure the vCPU gets the correct state before VM-Entry. in kvm_create_lapic()
3035 apic->apicv_active = true; in kvm_create_lapic()
3042 vcpu->arch.apic = NULL; in kvm_create_lapic()
3044 return -ENOMEM; in kvm_create_lapic()
3049 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_has_interrupt()
3053 return -1; in kvm_apic_has_interrupt()
3055 if (apic->guest_apic_protected) in kvm_apic_has_interrupt()
3056 return -1; in kvm_apic_has_interrupt()
3065 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); in kvm_apic_accept_pic_intr()
3067 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) in kvm_apic_accept_pic_intr()
3077 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_inject_apic_timer_irqs()
3079 if (atomic_read(&apic->lapic_timer.pending) > 0) { in kvm_inject_apic_timer_irqs()
3081 atomic_set(&apic->lapic_timer.pending, 0); in kvm_inject_apic_timer_irqs()
3087 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_ack_interrupt()
3103 * For auto-EOI interrupts, there might be another pending in kvm_apic_ack_interrupt()
3111 * be a higher-priority pending interrupt---except if there was in kvm_apic_ack_interrupt()
3125 if (apic_x2apic_mode(vcpu->arch.apic)) { in kvm_apic_state_fixup()
3126 u32 x2apic_id = kvm_x2apic_id(vcpu->arch.apic); in kvm_apic_state_fixup()
3127 u32 *id = (u32 *)(s->regs + APIC_ID); in kvm_apic_state_fixup()
3128 u32 *ldr = (u32 *)(s->regs + APIC_LDR); in kvm_apic_state_fixup()
3131 if (vcpu->kvm->arch.x2apic_format) { in kvm_apic_state_fixup()
3133 return -EINVAL; in kvm_apic_state_fixup()
3152 * if the ICR is _not_ split, ICR is internally a single 64-bit in kvm_apic_state_fixup()
3161 icr = apic_get_reg(s->regs, APIC_ICR) | in kvm_apic_state_fixup()
3162 (u64)apic_get_reg(s->regs, APIC_ICR2) << 32; in kvm_apic_state_fixup()
3163 apic_set_reg64(s->regs, APIC_ICR, icr); in kvm_apic_state_fixup()
3165 icr = apic_get_reg64(s->regs, APIC_ICR); in kvm_apic_state_fixup()
3166 apic_set_reg(s->regs, APIC_ICR2, icr >> 32); in kvm_apic_state_fixup()
3176 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); in kvm_apic_get_state()
3182 apic_set_reg(s->regs, APIC_TMCCT, __apic_read(vcpu->arch.apic, APIC_TMCCT)); in kvm_apic_get_state()
3189 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_state()
3195 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); in kvm_apic_set_state()
3199 kvm_recalculate_apic_map(vcpu->kvm); in kvm_apic_set_state()
3202 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); in kvm_apic_set_state()
3204 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_state()
3205 kvm_recalculate_apic_map(vcpu->kvm); in kvm_apic_set_state()
3210 apic->lapic_timer.expired_tscdeadline = 0; in kvm_apic_set_state()
3217 if (apic->apicv_active) { in kvm_apic_set_state()
3224 if (ioapic_in_kernel(vcpu->kvm)) in kvm_apic_set_state()
3228 vcpu->arch.apic_arb_prio = 0; in kvm_apic_set_state()
3241 timer = &vcpu->arch.apic->lapic_timer.timer; in __kvm_migrate_apic_timer()
3247 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
3262 * -> host disabled PV EOI. in apic_sync_pv_eoi_from_guest()
3264 * -> host enabled PV EOI, guest did not execute EOI yet. in apic_sync_pv_eoi_from_guest()
3266 * -> host enabled PV EOI, guest executed EOI. in apic_sync_pv_eoi_from_guest()
3280 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) in kvm_lapic_sync_from_vapic()
3281 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); in kvm_lapic_sync_from_vapic()
3283 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) in kvm_lapic_sync_from_vapic()
3286 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, in kvm_lapic_sync_from_vapic()
3290 apic_set_tpr(vcpu->arch.apic, data & 0xff); in kvm_lapic_sync_from_vapic()
3294 * apic_sync_pv_eoi_to_guest - called before vmentry
3304 apic->irr_pending || in apic_sync_pv_eoi_to_guest()
3306 apic->highest_isr_cache == -1 || in apic_sync_pv_eoi_to_guest()
3308 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { in apic_sync_pv_eoi_to_guest()
3316 pv_eoi_set_pending(apic->vcpu); in apic_sync_pv_eoi_to_guest()
3323 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_sync_to_vapic()
3327 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) in kvm_lapic_sync_to_vapic()
3339 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, in kvm_lapic_sync_to_vapic()
3346 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, in kvm_lapic_set_vapic_addr()
3347 &vcpu->arch.apic->vapic_cache, in kvm_lapic_set_vapic_addr()
3349 return -EINVAL; in kvm_lapic_set_vapic_addr()
3350 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); in kvm_lapic_set_vapic_addr()
3352 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); in kvm_lapic_set_vapic_addr()
3355 vcpu->arch.apic->vapic_addr = vapic_addr; in kvm_lapic_set_vapic_addr()
3379 * ICR is a 64-bit register in x2APIC mode (and Hyper-V PV vAPIC) and in kvm_lapic_msr_write()
3381 * through 32-bit reads/writes. in kvm_lapic_msr_write()
3395 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_x2apic_msr_write()
3396 u32 reg = (msr - APIC_BASE_MSR) << 4; in kvm_x2apic_msr_write()
3406 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_x2apic_msr_read()
3407 u32 reg = (msr - APIC_BASE_MSR) << 4; in kvm_x2apic_msr_read()
3420 return kvm_lapic_msr_write(vcpu->arch.apic, reg, data); in kvm_hv_vapic_msr_write()
3428 return kvm_lapic_msr_read(vcpu->arch.apic, reg, data); in kvm_hv_vapic_msr_read()
3434 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; in kvm_lapic_set_pv_eoi()
3442 if (addr == ghc->gpa && len <= ghc->len) in kvm_lapic_set_pv_eoi()
3443 new_len = ghc->len; in kvm_lapic_set_pv_eoi()
3447 ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); in kvm_lapic_set_pv_eoi()
3452 vcpu->arch.pv_eoi.msr_val = data; in kvm_lapic_set_pv_eoi()
3459 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_accept_events()
3469 return r == -EBUSY ? 0 : r; in kvm_apic_accept_events()
3471 * Continue processing INIT/SIPI even if a nested VM-Exit in kvm_apic_accept_events()
3480 * wait-for-SIPI (WFS). in kvm_apic_accept_events()
3483 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); in kvm_apic_accept_events()
3484 clear_bit(KVM_APIC_SIPI, &apic->pending_events); in kvm_apic_accept_events()
3488 if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) { in kvm_apic_accept_events()
3490 if (kvm_vcpu_is_bsp(apic->vcpu)) in kvm_apic_accept_events()
3495 if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) { in kvm_apic_accept_events()
3496 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { in kvm_apic_accept_events()
3499 sipi_vector = apic->sipi_vector; in kvm_apic_accept_events()