xref: /linux/arch/x86/kvm/vmx/pmu_intel.c (revision 4f372263ef92ed2af55a8c226750b72021ff8d0f)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * KVM PMU support for Intel CPUs
4  *
5  * Copyright 2011 Red Hat, Inc. and/or its affiliates.
6  *
7  * Authors:
8  *   Avi Kivity   <avi@redhat.com>
9  *   Gleb Natapov <gleb@redhat.com>
10  */
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 
13 #include <linux/types.h>
14 #include <linux/kvm_host.h>
15 #include <linux/perf_event.h>
16 #include <asm/msr.h>
17 #include <asm/perf_event.h>
18 #include "x86.h"
19 #include "cpuid.h"
20 #include "lapic.h"
21 #include "nested.h"
22 #include "pmu.h"
23 
24 /*
25  * Perf's "BASE" is wildly misleading, architectural PMUs use bits 31:16 of ECX
26  * to encode the "type" of counter to read, i.e. this is not a "base".  And to
27  * further confuse things, non-architectural PMUs use bit 31 as a flag for
28  * "fast" reads, whereas the "type" is an explicit value.
29  */
30 #define INTEL_RDPMC_GP		0
31 #define INTEL_RDPMC_FIXED	INTEL_PMC_FIXED_RDPMC_BASE
32 
33 #define INTEL_RDPMC_TYPE_MASK	GENMASK(31, 16)
34 #define INTEL_RDPMC_INDEX_MASK	GENMASK(15, 0)
35 
36 #define MSR_PMC_FULL_WIDTH_BIT      (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0)
37 
38 static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data)
39 {
40 	struct kvm_pmc *pmc;
41 	u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl;
42 	int i;
43 
44 	pmu->fixed_ctr_ctrl = data;
45 	for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
46 		u8 new_ctrl = fixed_ctrl_field(data, i);
47 		u8 old_ctrl = fixed_ctrl_field(old_fixed_ctr_ctrl, i);
48 
49 		if (old_ctrl == new_ctrl)
50 			continue;
51 
52 		pmc = get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + i);
53 
54 		__set_bit(KVM_FIXED_PMC_BASE_IDX + i, pmu->pmc_in_use);
55 		kvm_pmu_request_counter_reprogram(pmc);
56 	}
57 }
58 
59 static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu,
60 					    unsigned int idx, u64 *mask)
61 {
62 	unsigned int type = idx & INTEL_RDPMC_TYPE_MASK;
63 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
64 	struct kvm_pmc *counters;
65 	unsigned int num_counters;
66 	u64 bitmask;
67 
68 	/*
69 	 * The encoding of ECX for RDPMC is different for architectural versus
70 	 * non-architecturals PMUs (PMUs with version '0').  For architectural
71 	 * PMUs, bits 31:16 specify the PMC type and bits 15:0 specify the PMC
72 	 * index.  For non-architectural PMUs, bit 31 is a "fast" flag, and
73 	 * bits 30:0 specify the PMC index.
74 	 *
75 	 * Yell and reject attempts to read PMCs for a non-architectural PMU,
76 	 * as KVM doesn't support such PMUs.
77 	 */
78 	if (WARN_ON_ONCE(!pmu->version))
79 		return NULL;
80 
81 	/*
82 	 * General Purpose (GP) PMCs are supported on all PMUs, and fixed PMCs
83 	 * are supported on all architectural PMUs, i.e. on all virtual PMUs
84 	 * supported by KVM.  Note, KVM only emulates fixed PMCs for PMU v2+,
85 	 * but the type itself is still valid, i.e. let RDPMC fail due to
86 	 * accessing a non-existent counter.  Reject attempts to read all other
87 	 * types, which are unknown/unsupported.
88 	 */
89 	switch (type) {
90 	case INTEL_RDPMC_FIXED:
91 		counters = pmu->fixed_counters;
92 		num_counters = pmu->nr_arch_fixed_counters;
93 		bitmask = pmu->counter_bitmask[KVM_PMC_FIXED];
94 		break;
95 	case INTEL_RDPMC_GP:
96 		counters = pmu->gp_counters;
97 		num_counters = pmu->nr_arch_gp_counters;
98 		bitmask = pmu->counter_bitmask[KVM_PMC_GP];
99 		break;
100 	default:
101 		return NULL;
102 	}
103 
104 	idx &= INTEL_RDPMC_INDEX_MASK;
105 	if (idx >= num_counters)
106 		return NULL;
107 
108 	*mask &= bitmask;
109 	return &counters[array_index_nospec(idx, num_counters)];
110 }
111 
112 static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu)
113 {
114 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM))
115 		return 0;
116 
117 	return vcpu->arch.perf_capabilities;
118 }
119 
120 static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu)
121 {
122 	return (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_FW_WRITES) != 0;
123 }
124 
125 static inline struct kvm_pmc *get_fw_gp_pmc(struct kvm_pmu *pmu, u32 msr)
126 {
127 	if (!fw_writes_is_enabled(pmu_to_vcpu(pmu)))
128 		return NULL;
129 
130 	return get_gp_pmc(pmu, msr, MSR_IA32_PMC0);
131 }
132 
133 static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index)
134 {
135 	struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu);
136 	bool ret = false;
137 
138 	if (!intel_pmu_lbr_is_enabled(vcpu))
139 		return ret;
140 
141 	ret = (index == MSR_LBR_SELECT) || (index == MSR_LBR_TOS) ||
142 		(index >= records->from && index < records->from + records->nr) ||
143 		(index >= records->to && index < records->to + records->nr);
144 
145 	if (!ret && records->info)
146 		ret = (index >= records->info && index < records->info + records->nr);
147 
148 	return ret;
149 }
150 
151 static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
152 {
153 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
154 	u64 perf_capabilities;
155 	int ret;
156 
157 	switch (msr) {
158 	case MSR_CORE_PERF_FIXED_CTR_CTRL:
159 		return kvm_pmu_has_perf_global_ctrl(pmu);
160 	case MSR_IA32_PEBS_ENABLE:
161 		ret = vcpu_get_perf_capabilities(vcpu) & PERF_CAP_PEBS_FORMAT;
162 		break;
163 	case MSR_IA32_DS_AREA:
164 		ret = guest_cpu_cap_has(vcpu, X86_FEATURE_DS);
165 		break;
166 	case MSR_PEBS_DATA_CFG:
167 		perf_capabilities = vcpu_get_perf_capabilities(vcpu);
168 		ret = (perf_capabilities & PERF_CAP_PEBS_BASELINE) &&
169 			((perf_capabilities & PERF_CAP_PEBS_FORMAT) > 3);
170 		break;
171 	default:
172 		ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) ||
173 			get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) ||
174 			get_fixed_pmc(pmu, msr) || get_fw_gp_pmc(pmu, msr) ||
175 			intel_pmu_is_valid_lbr_msr(vcpu, msr);
176 		break;
177 	}
178 
179 	return ret;
180 }
181 
182 static struct kvm_pmc *intel_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr)
183 {
184 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
185 	struct kvm_pmc *pmc;
186 
187 	pmc = get_fixed_pmc(pmu, msr);
188 	pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0);
189 	pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0);
190 
191 	return pmc;
192 }
193 
194 static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu)
195 {
196 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
197 
198 	if (lbr_desc->event) {
199 		perf_event_release_kernel(lbr_desc->event);
200 		lbr_desc->event = NULL;
201 		vcpu_to_pmu(vcpu)->event_count--;
202 	}
203 }
204 
205 int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu)
206 {
207 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
208 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
209 	struct perf_event *event;
210 
211 	/*
212 	 * The perf_event_attr is constructed in the minimum efficient way:
213 	 * - set 'pinned = true' to make it task pinned so that if another
214 	 *   cpu pinned event reclaims LBR, the event->oncpu will be set to -1;
215 	 * - set '.exclude_host = true' to record guest branches behavior;
216 	 *
217 	 * - set '.config = INTEL_FIXED_VLBR_EVENT' to indicates host perf
218 	 *   schedule the event without a real HW counter but a fake one;
219 	 *   check is_guest_lbr_event() and __intel_get_event_constraints();
220 	 *
221 	 * - set 'sample_type = PERF_SAMPLE_BRANCH_STACK' and
222 	 *   'branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK |
223 	 *   PERF_SAMPLE_BRANCH_USER' to configure it as a LBR callstack
224 	 *   event, which helps KVM to save/restore guest LBR records
225 	 *   during host context switches and reduces quite a lot overhead,
226 	 *   check branch_user_callstack() and intel_pmu_lbr_sched_task();
227 	 */
228 	struct perf_event_attr attr = {
229 		.type = PERF_TYPE_RAW,
230 		.size = sizeof(attr),
231 		.config = INTEL_FIXED_VLBR_EVENT,
232 		.sample_type = PERF_SAMPLE_BRANCH_STACK,
233 		.pinned = true,
234 		.exclude_host = true,
235 		.branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK |
236 					PERF_SAMPLE_BRANCH_USER,
237 	};
238 
239 	if (unlikely(lbr_desc->event)) {
240 		__set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
241 		return 0;
242 	}
243 
244 	event = perf_event_create_kernel_counter(&attr, -1,
245 						current, NULL, NULL);
246 	if (IS_ERR(event)) {
247 		pr_debug_ratelimited("%s: failed %ld\n",
248 					__func__, PTR_ERR(event));
249 		return PTR_ERR(event);
250 	}
251 	lbr_desc->event = event;
252 	pmu->event_count++;
253 	__set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
254 	return 0;
255 }
256 
257 /*
258  * It's safe to access LBR msrs from guest when they have not
259  * been passthrough since the host would help restore or reset
260  * the LBR msrs records when the guest LBR event is scheduled in.
261  */
262 static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu,
263 				     struct msr_data *msr_info, bool read)
264 {
265 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
266 	u32 index = msr_info->index;
267 
268 	if (!intel_pmu_is_valid_lbr_msr(vcpu, index))
269 		return false;
270 
271 	if (!lbr_desc->event && intel_pmu_create_guest_lbr_event(vcpu) < 0)
272 		goto dummy;
273 
274 	/*
275 	 * Disable irq to ensure the LBR feature doesn't get reclaimed by the
276 	 * host at the time the value is read from the msr, and this avoids the
277 	 * host LBR value to be leaked to the guest. If LBR has been reclaimed,
278 	 * return 0 on guest reads.
279 	 */
280 	local_irq_disable();
281 	if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) {
282 		if (read)
283 			rdmsrq(index, msr_info->data);
284 		else
285 			wrmsrq(index, msr_info->data);
286 		__set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
287 		local_irq_enable();
288 		return true;
289 	}
290 	clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
291 	local_irq_enable();
292 
293 dummy:
294 	if (read)
295 		msr_info->data = 0;
296 	return true;
297 }
298 
299 static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
300 {
301 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
302 	struct kvm_pmc *pmc;
303 	u32 msr = msr_info->index;
304 
305 	switch (msr) {
306 	case MSR_CORE_PERF_FIXED_CTR_CTRL:
307 		msr_info->data = pmu->fixed_ctr_ctrl;
308 		break;
309 	case MSR_IA32_PEBS_ENABLE:
310 		msr_info->data = pmu->pebs_enable;
311 		break;
312 	case MSR_IA32_DS_AREA:
313 		msr_info->data = pmu->ds_area;
314 		break;
315 	case MSR_PEBS_DATA_CFG:
316 		msr_info->data = pmu->pebs_data_cfg;
317 		break;
318 	default:
319 		if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) ||
320 		    (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) {
321 			u64 val = pmc_read_counter(pmc);
322 			msr_info->data =
323 				val & pmu->counter_bitmask[KVM_PMC_GP];
324 			break;
325 		} else if ((pmc = get_fixed_pmc(pmu, msr))) {
326 			u64 val = pmc_read_counter(pmc);
327 			msr_info->data =
328 				val & pmu->counter_bitmask[KVM_PMC_FIXED];
329 			break;
330 		} else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
331 			msr_info->data = pmc->eventsel;
332 			break;
333 		} else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, true)) {
334 			break;
335 		}
336 		return 1;
337 	}
338 
339 	return 0;
340 }
341 
342 static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
343 {
344 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
345 	struct kvm_pmc *pmc;
346 	u32 msr = msr_info->index;
347 	u64 data = msr_info->data;
348 	u64 reserved_bits, diff;
349 
350 	switch (msr) {
351 	case MSR_CORE_PERF_FIXED_CTR_CTRL:
352 		if (data & pmu->fixed_ctr_ctrl_rsvd)
353 			return 1;
354 
355 		if (pmu->fixed_ctr_ctrl != data)
356 			reprogram_fixed_counters(pmu, data);
357 		break;
358 	case MSR_IA32_PEBS_ENABLE:
359 		if (data & pmu->pebs_enable_rsvd)
360 			return 1;
361 
362 		if (pmu->pebs_enable != data) {
363 			diff = pmu->pebs_enable ^ data;
364 			pmu->pebs_enable = data;
365 			reprogram_counters(pmu, diff);
366 		}
367 		break;
368 	case MSR_IA32_DS_AREA:
369 		if (is_noncanonical_msr_address(data, vcpu))
370 			return 1;
371 
372 		pmu->ds_area = data;
373 		break;
374 	case MSR_PEBS_DATA_CFG:
375 		if (data & pmu->pebs_data_cfg_rsvd)
376 			return 1;
377 
378 		pmu->pebs_data_cfg = data;
379 		break;
380 	default:
381 		if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) ||
382 		    (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) {
383 			if ((msr & MSR_PMC_FULL_WIDTH_BIT) &&
384 			    (data & ~pmu->counter_bitmask[KVM_PMC_GP]))
385 				return 1;
386 
387 			if (!msr_info->host_initiated &&
388 			    !(msr & MSR_PMC_FULL_WIDTH_BIT))
389 				data = (s64)(s32)data;
390 			pmc_write_counter(pmc, data);
391 			break;
392 		} else if ((pmc = get_fixed_pmc(pmu, msr))) {
393 			pmc_write_counter(pmc, data);
394 			break;
395 		} else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
396 			reserved_bits = pmu->reserved_bits;
397 			if ((pmc->idx == 2) &&
398 			    (pmu->raw_event_mask & HSW_IN_TX_CHECKPOINTED))
399 				reserved_bits ^= HSW_IN_TX_CHECKPOINTED;
400 			if (data & reserved_bits)
401 				return 1;
402 
403 			if (data != pmc->eventsel) {
404 				pmc->eventsel = data;
405 				kvm_pmu_request_counter_reprogram(pmc);
406 			}
407 			break;
408 		} else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, false)) {
409 			break;
410 		}
411 		/* Not a known PMU MSR. */
412 		return 1;
413 	}
414 
415 	return 0;
416 }
417 
418 /*
419  * Map fixed counter events to architectural general purpose event encodings.
420  * Perf doesn't provide APIs to allow KVM to directly program a fixed counter,
421  * and so KVM instead programs the architectural event to effectively request
422  * the fixed counter.  Perf isn't guaranteed to use a fixed counter and may
423  * instead program the encoding into a general purpose counter, e.g. if a
424  * different perf_event is already utilizing the requested counter, but the end
425  * result is the same (ignoring the fact that using a general purpose counter
426  * will likely exacerbate counter contention).
427  *
428  * Forcibly inlined to allow asserting on @index at build time, and there should
429  * never be more than one user.
430  */
431 static __always_inline u64 intel_get_fixed_pmc_eventsel(unsigned int index)
432 {
433 	const enum perf_hw_id fixed_pmc_perf_ids[] = {
434 		[0] = PERF_COUNT_HW_INSTRUCTIONS,
435 		[1] = PERF_COUNT_HW_CPU_CYCLES,
436 		[2] = PERF_COUNT_HW_REF_CPU_CYCLES,
437 	};
438 	u64 eventsel;
439 
440 	BUILD_BUG_ON(ARRAY_SIZE(fixed_pmc_perf_ids) != KVM_MAX_NR_INTEL_FIXED_COUTNERS);
441 	BUILD_BUG_ON(index >= KVM_MAX_NR_INTEL_FIXED_COUTNERS);
442 
443 	/*
444 	 * Yell if perf reports support for a fixed counter but perf doesn't
445 	 * have a known encoding for the associated general purpose event.
446 	 */
447 	eventsel = perf_get_hw_event_config(fixed_pmc_perf_ids[index]);
448 	WARN_ON_ONCE(!eventsel && index < kvm_pmu_cap.num_counters_fixed);
449 	return eventsel;
450 }
451 
452 static void intel_pmu_enable_fixed_counter_bits(struct kvm_pmu *pmu, u64 bits)
453 {
454 	int i;
455 
456 	for (i = 0; i < pmu->nr_arch_fixed_counters; i++)
457 		pmu->fixed_ctr_ctrl_rsvd &= ~intel_fixed_bits_by_idx(i, bits);
458 }
459 
460 static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
461 {
462 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
463 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
464 	struct kvm_cpuid_entry2 *entry;
465 	union cpuid10_eax eax;
466 	union cpuid10_edx edx;
467 	u64 perf_capabilities;
468 	u64 counter_rsvd;
469 
470 	memset(&lbr_desc->records, 0, sizeof(lbr_desc->records));
471 
472 	/*
473 	 * Setting passthrough of LBR MSRs is done only in the VM-Entry loop,
474 	 * and PMU refresh is disallowed after the vCPU has run, i.e. this code
475 	 * should never be reached while KVM is passing through MSRs.
476 	 */
477 	if (KVM_BUG_ON(lbr_desc->msr_passthrough, vcpu->kvm))
478 		return;
479 
480 	entry = kvm_find_cpuid_entry(vcpu, 0xa);
481 	if (!entry)
482 		return;
483 
484 	eax.full = entry->eax;
485 	edx.full = entry->edx;
486 
487 	pmu->version = eax.split.version_id;
488 	if (!pmu->version)
489 		return;
490 
491 	pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters,
492 					 kvm_pmu_cap.num_counters_gp);
493 	eax.split.bit_width = min_t(int, eax.split.bit_width,
494 				    kvm_pmu_cap.bit_width_gp);
495 	pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << eax.split.bit_width) - 1;
496 	eax.split.mask_length = min_t(int, eax.split.mask_length,
497 				      kvm_pmu_cap.events_mask_len);
498 	pmu->available_event_types = ~entry->ebx &
499 					((1ull << eax.split.mask_length) - 1);
500 
501 	if (pmu->version == 1) {
502 		pmu->nr_arch_fixed_counters = 0;
503 	} else {
504 		pmu->nr_arch_fixed_counters = min_t(int, edx.split.num_counters_fixed,
505 						    kvm_pmu_cap.num_counters_fixed);
506 		edx.split.bit_width_fixed = min_t(int, edx.split.bit_width_fixed,
507 						  kvm_pmu_cap.bit_width_fixed);
508 		pmu->counter_bitmask[KVM_PMC_FIXED] =
509 			((u64)1 << edx.split.bit_width_fixed) - 1;
510 	}
511 
512 	intel_pmu_enable_fixed_counter_bits(pmu, INTEL_FIXED_0_KERNEL |
513 						 INTEL_FIXED_0_USER |
514 						 INTEL_FIXED_0_ENABLE_PMI);
515 
516 	counter_rsvd = ~(((1ull << pmu->nr_arch_gp_counters) - 1) |
517 		(((1ull << pmu->nr_arch_fixed_counters) - 1) << KVM_FIXED_PMC_BASE_IDX));
518 	pmu->global_ctrl_rsvd = counter_rsvd;
519 
520 	/*
521 	 * GLOBAL_STATUS and GLOBAL_OVF_CONTROL (a.k.a. GLOBAL_STATUS_RESET)
522 	 * share reserved bit definitions.  The kernel just happens to use
523 	 * OVF_CTRL for the names.
524 	 */
525 	pmu->global_status_rsvd = pmu->global_ctrl_rsvd
526 			& ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF |
527 			    MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD);
528 	if (vmx_pt_mode_is_host_guest())
529 		pmu->global_status_rsvd &=
530 				~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI;
531 
532 	entry = kvm_find_cpuid_entry_index(vcpu, 7, 0);
533 	if (entry &&
534 	    (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) &&
535 	    (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) {
536 		pmu->reserved_bits ^= HSW_IN_TX;
537 		pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);
538 	}
539 
540 	bitmap_set(pmu->all_valid_pmc_idx,
541 		0, pmu->nr_arch_gp_counters);
542 	bitmap_set(pmu->all_valid_pmc_idx,
543 		INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters);
544 
545 	perf_capabilities = vcpu_get_perf_capabilities(vcpu);
546 	if (cpuid_model_is_consistent(vcpu) &&
547 	    (perf_capabilities & PMU_CAP_LBR_FMT))
548 		memcpy(&lbr_desc->records, &vmx_lbr_caps, sizeof(vmx_lbr_caps));
549 	else
550 		lbr_desc->records.nr = 0;
551 
552 	if (lbr_desc->records.nr)
553 		bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1);
554 
555 	if (perf_capabilities & PERF_CAP_PEBS_FORMAT) {
556 		if (perf_capabilities & PERF_CAP_PEBS_BASELINE) {
557 			pmu->pebs_enable_rsvd = counter_rsvd;
558 			pmu->reserved_bits &= ~ICL_EVENTSEL_ADAPTIVE;
559 			pmu->pebs_data_cfg_rsvd = ~0xff00000full;
560 			intel_pmu_enable_fixed_counter_bits(pmu, ICL_FIXED_0_ADAPTIVE);
561 		} else {
562 			pmu->pebs_enable_rsvd =
563 				~((1ull << pmu->nr_arch_gp_counters) - 1);
564 		}
565 	}
566 }
567 
568 static void intel_pmu_init(struct kvm_vcpu *vcpu)
569 {
570 	int i;
571 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
572 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
573 
574 	for (i = 0; i < KVM_MAX_NR_INTEL_GP_COUNTERS; i++) {
575 		pmu->gp_counters[i].type = KVM_PMC_GP;
576 		pmu->gp_counters[i].vcpu = vcpu;
577 		pmu->gp_counters[i].idx = i;
578 		pmu->gp_counters[i].current_config = 0;
579 	}
580 
581 	for (i = 0; i < KVM_MAX_NR_INTEL_FIXED_COUTNERS; i++) {
582 		pmu->fixed_counters[i].type = KVM_PMC_FIXED;
583 		pmu->fixed_counters[i].vcpu = vcpu;
584 		pmu->fixed_counters[i].idx = i + KVM_FIXED_PMC_BASE_IDX;
585 		pmu->fixed_counters[i].current_config = 0;
586 		pmu->fixed_counters[i].eventsel = intel_get_fixed_pmc_eventsel(i);
587 	}
588 
589 	lbr_desc->records.nr = 0;
590 	lbr_desc->event = NULL;
591 	lbr_desc->msr_passthrough = false;
592 }
593 
594 static void intel_pmu_reset(struct kvm_vcpu *vcpu)
595 {
596 	intel_pmu_release_guest_lbr_event(vcpu);
597 }
598 
599 /*
600  * Emulate LBR_On_PMI behavior for 1 < pmu.version < 4.
601  *
602  * If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and
603  * the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL.
604  *
605  * Guest needs to re-enable LBR to resume branches recording.
606  */
607 static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu)
608 {
609 	u64 data = vmcs_read64(GUEST_IA32_DEBUGCTL);
610 
611 	if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) {
612 		data &= ~DEBUGCTLMSR_LBR;
613 		vmcs_write64(GUEST_IA32_DEBUGCTL, data);
614 	}
615 }
616 
617 static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
618 {
619 	u8 version = vcpu_to_pmu(vcpu)->version;
620 
621 	if (!intel_pmu_lbr_is_enabled(vcpu))
622 		return;
623 
624 	if (version > 1 && version < 4)
625 		intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu);
626 }
627 
628 static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set)
629 {
630 	struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu);
631 	int i;
632 
633 	for (i = 0; i < lbr->nr; i++) {
634 		vmx_set_intercept_for_msr(vcpu, lbr->from + i, MSR_TYPE_RW, set);
635 		vmx_set_intercept_for_msr(vcpu, lbr->to + i, MSR_TYPE_RW, set);
636 		if (lbr->info)
637 			vmx_set_intercept_for_msr(vcpu, lbr->info + i, MSR_TYPE_RW, set);
638 	}
639 
640 	vmx_set_intercept_for_msr(vcpu, MSR_LBR_SELECT, MSR_TYPE_RW, set);
641 	vmx_set_intercept_for_msr(vcpu, MSR_LBR_TOS, MSR_TYPE_RW, set);
642 }
643 
644 static inline void vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
645 {
646 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
647 
648 	if (!lbr_desc->msr_passthrough)
649 		return;
650 
651 	vmx_update_intercept_for_lbr_msrs(vcpu, true);
652 	lbr_desc->msr_passthrough = false;
653 }
654 
655 static inline void vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
656 {
657 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
658 
659 	if (lbr_desc->msr_passthrough)
660 		return;
661 
662 	vmx_update_intercept_for_lbr_msrs(vcpu, false);
663 	lbr_desc->msr_passthrough = true;
664 }
665 
666 /*
667  * Higher priority host perf events (e.g. cpu pinned) could reclaim the
668  * pmu resources (e.g. LBR) that were assigned to the guest. This is
669  * usually done via ipi calls (more details in perf_install_in_context).
670  *
671  * Before entering the non-root mode (with irq disabled here), double
672  * confirm that the pmu features enabled to the guest are not reclaimed
673  * by higher priority host events. Otherwise, disallow vcpu's access to
674  * the reclaimed features.
675  */
676 void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu)
677 {
678 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
679 	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
680 
681 	if (!lbr_desc->event) {
682 		vmx_disable_lbr_msrs_passthrough(vcpu);
683 		if (vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR)
684 			goto warn;
685 		if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use))
686 			goto warn;
687 		return;
688 	}
689 
690 	if (lbr_desc->event->state < PERF_EVENT_STATE_ACTIVE) {
691 		vmx_disable_lbr_msrs_passthrough(vcpu);
692 		__clear_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
693 		goto warn;
694 	} else
695 		vmx_enable_lbr_msrs_passthrough(vcpu);
696 
697 	return;
698 
699 warn:
700 	pr_warn_ratelimited("vcpu-%d: fail to passthrough LBR.\n", vcpu->vcpu_id);
701 }
702 
703 static void intel_pmu_cleanup(struct kvm_vcpu *vcpu)
704 {
705 	if (!(vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR))
706 		intel_pmu_release_guest_lbr_event(vcpu);
707 }
708 
709 void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu)
710 {
711 	struct kvm_pmc *pmc = NULL;
712 	int bit, hw_idx;
713 
714 	kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) {
715 		if (!pmc_speculative_in_use(pmc) ||
716 		    !pmc_is_globally_enabled(pmc) || !pmc->perf_event)
717 			continue;
718 
719 		/*
720 		 * A negative index indicates the event isn't mapped to a
721 		 * physical counter in the host, e.g. due to contention.
722 		 */
723 		hw_idx = pmc->perf_event->hw.idx;
724 		if (hw_idx != pmc->idx && hw_idx > -1)
725 			pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx);
726 	}
727 }
728 
729 struct kvm_pmu_ops intel_pmu_ops __initdata = {
730 	.rdpmc_ecx_to_pmc = intel_rdpmc_ecx_to_pmc,
731 	.msr_idx_to_pmc = intel_msr_idx_to_pmc,
732 	.is_valid_msr = intel_is_valid_msr,
733 	.get_msr = intel_pmu_get_msr,
734 	.set_msr = intel_pmu_set_msr,
735 	.refresh = intel_pmu_refresh,
736 	.init = intel_pmu_init,
737 	.reset = intel_pmu_reset,
738 	.deliver_pmi = intel_pmu_deliver_pmi,
739 	.cleanup = intel_pmu_cleanup,
740 	.EVENTSEL_EVENT = ARCH_PERFMON_EVENTSEL_EVENT,
741 	.MAX_NR_GP_COUNTERS = KVM_MAX_NR_INTEL_GP_COUNTERS,
742 	.MIN_NR_GP_COUNTERS = 1,
743 };
744