1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 Linaro Ltd. 4 * Author: Shannon Zhao <shannon.zhao@linaro.org> 5 */ 6 7 #include <linux/cpu.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 #include <linux/list.h> 11 #include <linux/perf_event.h> 12 #include <linux/perf/arm_pmu.h> 13 #include <linux/uaccess.h> 14 #include <asm/kvm_emulate.h> 15 #include <kvm/arm_pmu.h> 16 #include <kvm/arm_vgic.h> 17 18 #define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0) 19 20 DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available); 21 22 static LIST_HEAD(arm_pmus); 23 static DEFINE_MUTEX(arm_pmus_lock); 24 25 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc); 26 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc); 27 28 static struct kvm_vcpu *kvm_pmc_to_vcpu(const struct kvm_pmc *pmc) 29 { 30 return container_of(pmc, struct kvm_vcpu, arch.pmu.pmc[pmc->idx]); 31 } 32 33 static struct kvm_pmc *kvm_vcpu_idx_to_pmc(struct kvm_vcpu *vcpu, int cnt_idx) 34 { 35 return &vcpu->arch.pmu.pmc[cnt_idx]; 36 } 37 38 static u32 __kvm_pmu_event_mask(unsigned int pmuver) 39 { 40 switch (pmuver) { 41 case ID_AA64DFR0_EL1_PMUVer_IMP: 42 return GENMASK(9, 0); 43 case ID_AA64DFR0_EL1_PMUVer_V3P1: 44 case ID_AA64DFR0_EL1_PMUVer_V3P4: 45 case ID_AA64DFR0_EL1_PMUVer_V3P5: 46 case ID_AA64DFR0_EL1_PMUVer_V3P7: 47 return GENMASK(15, 0); 48 default: /* Shouldn't be here, just for sanity */ 49 WARN_ONCE(1, "Unknown PMU version %d\n", pmuver); 50 return 0; 51 } 52 } 53 54 static u32 kvm_pmu_event_mask(struct kvm *kvm) 55 { 56 u64 dfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1); 57 u8 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0); 58 59 return __kvm_pmu_event_mask(pmuver); 60 } 61 62 u64 kvm_pmu_evtyper_mask(struct kvm *kvm) 63 { 64 u64 mask = ARMV8_PMU_EXCLUDE_EL1 | ARMV8_PMU_EXCLUDE_EL0 | 65 kvm_pmu_event_mask(kvm); 66 67 if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL2, IMP)) 68 mask |= ARMV8_PMU_INCLUDE_EL2; 69 70 if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP)) 71 mask |= ARMV8_PMU_EXCLUDE_NS_EL0 | 72 ARMV8_PMU_EXCLUDE_NS_EL1 | 73 ARMV8_PMU_EXCLUDE_EL3; 74 75 return mask; 76 } 77 78 /** 79 * kvm_pmc_is_64bit - determine if counter is 64bit 80 * @pmc: counter context 81 */ 82 static bool kvm_pmc_is_64bit(struct kvm_pmc *pmc) 83 { 84 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 85 86 return (pmc->idx == ARMV8_PMU_CYCLE_IDX || 87 kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, V3P5)); 88 } 89 90 static bool kvm_pmc_has_64bit_overflow(struct kvm_pmc *pmc) 91 { 92 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 93 u64 val = kvm_vcpu_read_pmcr(vcpu); 94 95 if (kvm_pmu_counter_is_hyp(vcpu, pmc->idx)) 96 return __vcpu_sys_reg(vcpu, MDCR_EL2) & MDCR_EL2_HLP; 97 98 return (pmc->idx < ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LP)) || 99 (pmc->idx == ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LC)); 100 } 101 102 static bool kvm_pmu_counter_can_chain(struct kvm_pmc *pmc) 103 { 104 return (!(pmc->idx & 1) && (pmc->idx + 1) < ARMV8_PMU_CYCLE_IDX && 105 !kvm_pmc_has_64bit_overflow(pmc)); 106 } 107 108 static u32 counter_index_to_reg(u64 idx) 109 { 110 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + idx; 111 } 112 113 static u32 counter_index_to_evtreg(u64 idx) 114 { 115 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + idx; 116 } 117 118 static u64 kvm_pmc_read_evtreg(const struct kvm_pmc *pmc) 119 { 120 return __vcpu_sys_reg(kvm_pmc_to_vcpu(pmc), counter_index_to_evtreg(pmc->idx)); 121 } 122 123 static u64 kvm_pmu_get_pmc_value(struct kvm_pmc *pmc) 124 { 125 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 126 u64 counter, reg, enabled, running; 127 128 reg = counter_index_to_reg(pmc->idx); 129 counter = __vcpu_sys_reg(vcpu, reg); 130 131 /* 132 * The real counter value is equal to the value of counter register plus 133 * the value perf event counts. 134 */ 135 if (pmc->perf_event) 136 counter += perf_event_read_value(pmc->perf_event, &enabled, 137 &running); 138 139 if (!kvm_pmc_is_64bit(pmc)) 140 counter = lower_32_bits(counter); 141 142 return counter; 143 } 144 145 /** 146 * kvm_pmu_get_counter_value - get PMU counter value 147 * @vcpu: The vcpu pointer 148 * @select_idx: The counter index 149 */ 150 u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx) 151 { 152 if (!kvm_vcpu_has_pmu(vcpu)) 153 return 0; 154 155 return kvm_pmu_get_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx)); 156 } 157 158 static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force) 159 { 160 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 161 u64 reg; 162 163 kvm_pmu_release_perf_event(pmc); 164 165 reg = counter_index_to_reg(pmc->idx); 166 167 if (vcpu_mode_is_32bit(vcpu) && pmc->idx != ARMV8_PMU_CYCLE_IDX && 168 !force) { 169 /* 170 * Even with PMUv3p5, AArch32 cannot write to the top 171 * 32bit of the counters. The only possible course of 172 * action is to use PMCR.P, which will reset them to 173 * 0 (the only use of the 'force' parameter). 174 */ 175 val = __vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32); 176 val |= lower_32_bits(val); 177 } 178 179 __vcpu_sys_reg(vcpu, reg) = val; 180 181 /* Recreate the perf event to reflect the updated sample_period */ 182 kvm_pmu_create_perf_event(pmc); 183 } 184 185 /** 186 * kvm_pmu_set_counter_value - set PMU counter value 187 * @vcpu: The vcpu pointer 188 * @select_idx: The counter index 189 * @val: The counter value 190 */ 191 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) 192 { 193 if (!kvm_vcpu_has_pmu(vcpu)) 194 return; 195 196 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx), val, false); 197 } 198 199 /** 200 * kvm_pmu_release_perf_event - remove the perf event 201 * @pmc: The PMU counter pointer 202 */ 203 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) 204 { 205 if (pmc->perf_event) { 206 perf_event_disable(pmc->perf_event); 207 perf_event_release_kernel(pmc->perf_event); 208 pmc->perf_event = NULL; 209 } 210 } 211 212 /** 213 * kvm_pmu_stop_counter - stop PMU counter 214 * @pmc: The PMU counter pointer 215 * 216 * If this counter has been configured to monitor some event, release it here. 217 */ 218 static void kvm_pmu_stop_counter(struct kvm_pmc *pmc) 219 { 220 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 221 u64 reg, val; 222 223 if (!pmc->perf_event) 224 return; 225 226 val = kvm_pmu_get_pmc_value(pmc); 227 228 reg = counter_index_to_reg(pmc->idx); 229 230 __vcpu_sys_reg(vcpu, reg) = val; 231 232 kvm_pmu_release_perf_event(pmc); 233 } 234 235 /** 236 * kvm_pmu_vcpu_init - assign pmu counter idx for cpu 237 * @vcpu: The vcpu pointer 238 * 239 */ 240 void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu) 241 { 242 int i; 243 struct kvm_pmu *pmu = &vcpu->arch.pmu; 244 245 for (i = 0; i < KVM_ARMV8_PMU_MAX_COUNTERS; i++) 246 pmu->pmc[i].idx = i; 247 } 248 249 /** 250 * kvm_pmu_vcpu_reset - reset pmu state for cpu 251 * @vcpu: The vcpu pointer 252 * 253 */ 254 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) 255 { 256 unsigned long mask = kvm_pmu_implemented_counter_mask(vcpu); 257 int i; 258 259 for_each_set_bit(i, &mask, 32) 260 kvm_pmu_stop_counter(kvm_vcpu_idx_to_pmc(vcpu, i)); 261 } 262 263 /** 264 * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu 265 * @vcpu: The vcpu pointer 266 * 267 */ 268 void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu) 269 { 270 int i; 271 272 for (i = 0; i < KVM_ARMV8_PMU_MAX_COUNTERS; i++) 273 kvm_pmu_release_perf_event(kvm_vcpu_idx_to_pmc(vcpu, i)); 274 irq_work_sync(&vcpu->arch.pmu.overflow_work); 275 } 276 277 bool kvm_pmu_counter_is_hyp(struct kvm_vcpu *vcpu, unsigned int idx) 278 { 279 unsigned int hpmn; 280 281 if (!vcpu_has_nv(vcpu) || idx == ARMV8_PMU_CYCLE_IDX) 282 return false; 283 284 /* 285 * Programming HPMN=0 is CONSTRAINED UNPREDICTABLE if FEAT_HPMN0 isn't 286 * implemented. Since KVM's ability to emulate HPMN=0 does not directly 287 * depend on hardware (all PMU registers are trapped), make the 288 * implementation choice that all counters are included in the second 289 * range reserved for EL2/EL3. 290 */ 291 hpmn = SYS_FIELD_GET(MDCR_EL2, HPMN, __vcpu_sys_reg(vcpu, MDCR_EL2)); 292 return idx >= hpmn; 293 } 294 295 u64 kvm_pmu_accessible_counter_mask(struct kvm_vcpu *vcpu) 296 { 297 u64 mask = kvm_pmu_implemented_counter_mask(vcpu); 298 u64 hpmn; 299 300 if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu)) 301 return mask; 302 303 hpmn = SYS_FIELD_GET(MDCR_EL2, HPMN, __vcpu_sys_reg(vcpu, MDCR_EL2)); 304 return mask & ~GENMASK(vcpu->kvm->arch.pmcr_n - 1, hpmn); 305 } 306 307 u64 kvm_pmu_implemented_counter_mask(struct kvm_vcpu *vcpu) 308 { 309 u64 val = FIELD_GET(ARMV8_PMU_PMCR_N, kvm_vcpu_read_pmcr(vcpu)); 310 311 if (val == 0) 312 return BIT(ARMV8_PMU_CYCLE_IDX); 313 else 314 return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX); 315 } 316 317 /** 318 * kvm_pmu_enable_counter_mask - enable selected PMU counters 319 * @vcpu: The vcpu pointer 320 * @val: the value guest writes to PMCNTENSET register 321 * 322 * Call perf_event_enable to start counting the perf event 323 */ 324 void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val) 325 { 326 int i; 327 if (!kvm_vcpu_has_pmu(vcpu)) 328 return; 329 330 if (!(kvm_vcpu_read_pmcr(vcpu) & ARMV8_PMU_PMCR_E) || !val) 331 return; 332 333 for (i = 0; i < KVM_ARMV8_PMU_MAX_COUNTERS; i++) { 334 struct kvm_pmc *pmc; 335 336 if (!(val & BIT(i))) 337 continue; 338 339 pmc = kvm_vcpu_idx_to_pmc(vcpu, i); 340 341 if (!pmc->perf_event) { 342 kvm_pmu_create_perf_event(pmc); 343 } else { 344 perf_event_enable(pmc->perf_event); 345 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE) 346 kvm_debug("fail to enable perf event\n"); 347 } 348 } 349 } 350 351 /** 352 * kvm_pmu_disable_counter_mask - disable selected PMU counters 353 * @vcpu: The vcpu pointer 354 * @val: the value guest writes to PMCNTENCLR register 355 * 356 * Call perf_event_disable to stop counting the perf event 357 */ 358 void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val) 359 { 360 int i; 361 362 if (!kvm_vcpu_has_pmu(vcpu) || !val) 363 return; 364 365 for (i = 0; i < KVM_ARMV8_PMU_MAX_COUNTERS; i++) { 366 struct kvm_pmc *pmc; 367 368 if (!(val & BIT(i))) 369 continue; 370 371 pmc = kvm_vcpu_idx_to_pmc(vcpu, i); 372 373 if (pmc->perf_event) 374 perf_event_disable(pmc->perf_event); 375 } 376 } 377 378 static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu) 379 { 380 u64 reg = 0; 381 382 if ((kvm_vcpu_read_pmcr(vcpu) & ARMV8_PMU_PMCR_E)) { 383 reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0); 384 reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); 385 reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1); 386 } 387 388 return reg; 389 } 390 391 static void kvm_pmu_update_state(struct kvm_vcpu *vcpu) 392 { 393 struct kvm_pmu *pmu = &vcpu->arch.pmu; 394 bool overflow; 395 396 if (!kvm_vcpu_has_pmu(vcpu)) 397 return; 398 399 overflow = !!kvm_pmu_overflow_status(vcpu); 400 if (pmu->irq_level == overflow) 401 return; 402 403 pmu->irq_level = overflow; 404 405 if (likely(irqchip_in_kernel(vcpu->kvm))) { 406 int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu, 407 pmu->irq_num, overflow, pmu); 408 WARN_ON(ret); 409 } 410 } 411 412 bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu) 413 { 414 struct kvm_pmu *pmu = &vcpu->arch.pmu; 415 struct kvm_sync_regs *sregs = &vcpu->run->s.regs; 416 bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU; 417 418 if (likely(irqchip_in_kernel(vcpu->kvm))) 419 return false; 420 421 return pmu->irq_level != run_level; 422 } 423 424 /* 425 * Reflect the PMU overflow interrupt output level into the kvm_run structure 426 */ 427 void kvm_pmu_update_run(struct kvm_vcpu *vcpu) 428 { 429 struct kvm_sync_regs *regs = &vcpu->run->s.regs; 430 431 /* Populate the timer bitmap for user space */ 432 regs->device_irq_level &= ~KVM_ARM_DEV_PMU; 433 if (vcpu->arch.pmu.irq_level) 434 regs->device_irq_level |= KVM_ARM_DEV_PMU; 435 } 436 437 /** 438 * kvm_pmu_flush_hwstate - flush pmu state to cpu 439 * @vcpu: The vcpu pointer 440 * 441 * Check if the PMU has overflowed while we were running in the host, and inject 442 * an interrupt if that was the case. 443 */ 444 void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu) 445 { 446 kvm_pmu_update_state(vcpu); 447 } 448 449 /** 450 * kvm_pmu_sync_hwstate - sync pmu state from cpu 451 * @vcpu: The vcpu pointer 452 * 453 * Check if the PMU has overflowed while we were running in the guest, and 454 * inject an interrupt if that was the case. 455 */ 456 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) 457 { 458 kvm_pmu_update_state(vcpu); 459 } 460 461 /* 462 * When perf interrupt is an NMI, we cannot safely notify the vcpu corresponding 463 * to the event. 464 * This is why we need a callback to do it once outside of the NMI context. 465 */ 466 static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work) 467 { 468 struct kvm_vcpu *vcpu; 469 470 vcpu = container_of(work, struct kvm_vcpu, arch.pmu.overflow_work); 471 kvm_vcpu_kick(vcpu); 472 } 473 474 /* 475 * Perform an increment on any of the counters described in @mask, 476 * generating the overflow if required, and propagate it as a chained 477 * event if possible. 478 */ 479 static void kvm_pmu_counter_increment(struct kvm_vcpu *vcpu, 480 unsigned long mask, u32 event) 481 { 482 int i; 483 484 if (!(kvm_vcpu_read_pmcr(vcpu) & ARMV8_PMU_PMCR_E)) 485 return; 486 487 /* Weed out disabled counters */ 488 mask &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); 489 490 for_each_set_bit(i, &mask, ARMV8_PMU_CYCLE_IDX) { 491 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i); 492 u64 type, reg; 493 494 /* Filter on event type */ 495 type = __vcpu_sys_reg(vcpu, counter_index_to_evtreg(i)); 496 type &= kvm_pmu_event_mask(vcpu->kvm); 497 if (type != event) 498 continue; 499 500 /* Increment this counter */ 501 reg = __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) + 1; 502 if (!kvm_pmc_is_64bit(pmc)) 503 reg = lower_32_bits(reg); 504 __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) = reg; 505 506 /* No overflow? move on */ 507 if (kvm_pmc_has_64bit_overflow(pmc) ? reg : lower_32_bits(reg)) 508 continue; 509 510 /* Mark overflow */ 511 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i); 512 513 if (kvm_pmu_counter_can_chain(pmc)) 514 kvm_pmu_counter_increment(vcpu, BIT(i + 1), 515 ARMV8_PMUV3_PERFCTR_CHAIN); 516 } 517 } 518 519 /* Compute the sample period for a given counter value */ 520 static u64 compute_period(struct kvm_pmc *pmc, u64 counter) 521 { 522 u64 val; 523 524 if (kvm_pmc_is_64bit(pmc) && kvm_pmc_has_64bit_overflow(pmc)) 525 val = (-counter) & GENMASK(63, 0); 526 else 527 val = (-counter) & GENMASK(31, 0); 528 529 return val; 530 } 531 532 /* 533 * When the perf event overflows, set the overflow status and inform the vcpu. 534 */ 535 static void kvm_pmu_perf_overflow(struct perf_event *perf_event, 536 struct perf_sample_data *data, 537 struct pt_regs *regs) 538 { 539 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 540 struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu); 541 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 542 int idx = pmc->idx; 543 u64 period; 544 545 cpu_pmu->pmu.stop(perf_event, PERF_EF_UPDATE); 546 547 /* 548 * Reset the sample period to the architectural limit, 549 * i.e. the point where the counter overflows. 550 */ 551 period = compute_period(pmc, local64_read(&perf_event->count)); 552 553 local64_set(&perf_event->hw.period_left, 0); 554 perf_event->attr.sample_period = period; 555 perf_event->hw.sample_period = period; 556 557 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx); 558 559 if (kvm_pmu_counter_can_chain(pmc)) 560 kvm_pmu_counter_increment(vcpu, BIT(idx + 1), 561 ARMV8_PMUV3_PERFCTR_CHAIN); 562 563 if (kvm_pmu_overflow_status(vcpu)) { 564 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); 565 566 if (!in_nmi()) 567 kvm_vcpu_kick(vcpu); 568 else 569 irq_work_queue(&vcpu->arch.pmu.overflow_work); 570 } 571 572 cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD); 573 } 574 575 /** 576 * kvm_pmu_software_increment - do software increment 577 * @vcpu: The vcpu pointer 578 * @val: the value guest writes to PMSWINC register 579 */ 580 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) 581 { 582 kvm_pmu_counter_increment(vcpu, val, ARMV8_PMUV3_PERFCTR_SW_INCR); 583 } 584 585 /** 586 * kvm_pmu_handle_pmcr - handle PMCR register 587 * @vcpu: The vcpu pointer 588 * @val: the value guest writes to PMCR register 589 */ 590 void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) 591 { 592 int i; 593 594 if (!kvm_vcpu_has_pmu(vcpu)) 595 return; 596 597 /* Fixup PMCR_EL0 to reconcile the PMU version and the LP bit */ 598 if (!kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, V3P5)) 599 val &= ~ARMV8_PMU_PMCR_LP; 600 601 /* The reset bits don't indicate any state, and shouldn't be saved. */ 602 __vcpu_sys_reg(vcpu, PMCR_EL0) = val & ~(ARMV8_PMU_PMCR_C | ARMV8_PMU_PMCR_P); 603 604 if (val & ARMV8_PMU_PMCR_E) { 605 kvm_pmu_enable_counter_mask(vcpu, 606 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0)); 607 } else { 608 kvm_pmu_disable_counter_mask(vcpu, 609 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0)); 610 } 611 612 if (val & ARMV8_PMU_PMCR_C) 613 kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0); 614 615 if (val & ARMV8_PMU_PMCR_P) { 616 unsigned long mask = kvm_pmu_accessible_counter_mask(vcpu); 617 mask &= ~BIT(ARMV8_PMU_CYCLE_IDX); 618 for_each_set_bit(i, &mask, 32) 619 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, i), 0, true); 620 } 621 kvm_vcpu_pmu_restore_guest(vcpu); 622 } 623 624 static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc) 625 { 626 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 627 unsigned int mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 628 629 if (!(__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(pmc->idx))) 630 return false; 631 632 if (kvm_pmu_counter_is_hyp(vcpu, pmc->idx)) 633 return mdcr & MDCR_EL2_HPME; 634 635 return kvm_vcpu_read_pmcr(vcpu) & ARMV8_PMU_PMCR_E; 636 } 637 638 static bool kvm_pmc_counts_at_el0(struct kvm_pmc *pmc) 639 { 640 u64 evtreg = kvm_pmc_read_evtreg(pmc); 641 bool nsu = evtreg & ARMV8_PMU_EXCLUDE_NS_EL0; 642 bool u = evtreg & ARMV8_PMU_EXCLUDE_EL0; 643 644 return u == nsu; 645 } 646 647 static bool kvm_pmc_counts_at_el1(struct kvm_pmc *pmc) 648 { 649 u64 evtreg = kvm_pmc_read_evtreg(pmc); 650 bool nsk = evtreg & ARMV8_PMU_EXCLUDE_NS_EL1; 651 bool p = evtreg & ARMV8_PMU_EXCLUDE_EL1; 652 653 return p == nsk; 654 } 655 656 static bool kvm_pmc_counts_at_el2(struct kvm_pmc *pmc) 657 { 658 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 659 u64 mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 660 661 if (!kvm_pmu_counter_is_hyp(vcpu, pmc->idx) && (mdcr & MDCR_EL2_HPMD)) 662 return false; 663 664 return kvm_pmc_read_evtreg(pmc) & ARMV8_PMU_INCLUDE_EL2; 665 } 666 667 /** 668 * kvm_pmu_create_perf_event - create a perf event for a counter 669 * @pmc: Counter context 670 */ 671 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc) 672 { 673 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 674 struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu; 675 struct perf_event *event; 676 struct perf_event_attr attr; 677 u64 eventsel, evtreg; 678 679 evtreg = kvm_pmc_read_evtreg(pmc); 680 681 kvm_pmu_stop_counter(pmc); 682 if (pmc->idx == ARMV8_PMU_CYCLE_IDX) 683 eventsel = ARMV8_PMUV3_PERFCTR_CPU_CYCLES; 684 else 685 eventsel = evtreg & kvm_pmu_event_mask(vcpu->kvm); 686 687 /* 688 * Neither SW increment nor chained events need to be backed 689 * by a perf event. 690 */ 691 if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR || 692 eventsel == ARMV8_PMUV3_PERFCTR_CHAIN) 693 return; 694 695 /* 696 * If we have a filter in place and that the event isn't allowed, do 697 * not install a perf event either. 698 */ 699 if (vcpu->kvm->arch.pmu_filter && 700 !test_bit(eventsel, vcpu->kvm->arch.pmu_filter)) 701 return; 702 703 memset(&attr, 0, sizeof(struct perf_event_attr)); 704 attr.type = arm_pmu->pmu.type; 705 attr.size = sizeof(attr); 706 attr.pinned = 1; 707 attr.disabled = !kvm_pmu_counter_is_enabled(pmc); 708 attr.exclude_user = !kvm_pmc_counts_at_el0(pmc); 709 attr.exclude_hv = 1; /* Don't count EL2 events */ 710 attr.exclude_host = 1; /* Don't count host events */ 711 attr.config = eventsel; 712 713 /* 714 * Filter events at EL1 (i.e. vEL2) when in a hyp context based on the 715 * guest's EL2 filter. 716 */ 717 if (unlikely(is_hyp_ctxt(vcpu))) 718 attr.exclude_kernel = !kvm_pmc_counts_at_el2(pmc); 719 else 720 attr.exclude_kernel = !kvm_pmc_counts_at_el1(pmc); 721 722 /* 723 * If counting with a 64bit counter, advertise it to the perf 724 * code, carefully dealing with the initial sample period 725 * which also depends on the overflow. 726 */ 727 if (kvm_pmc_is_64bit(pmc)) 728 attr.config1 |= PERF_ATTR_CFG1_COUNTER_64BIT; 729 730 attr.sample_period = compute_period(pmc, kvm_pmu_get_pmc_value(pmc)); 731 732 event = perf_event_create_kernel_counter(&attr, -1, current, 733 kvm_pmu_perf_overflow, pmc); 734 735 if (IS_ERR(event)) { 736 pr_err_once("kvm: pmu event creation failed %ld\n", 737 PTR_ERR(event)); 738 return; 739 } 740 741 pmc->perf_event = event; 742 } 743 744 /** 745 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event 746 * @vcpu: The vcpu pointer 747 * @data: The data guest writes to PMXEVTYPER_EL0 748 * @select_idx: The number of selected counter 749 * 750 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an 751 * event with given hardware event number. Here we call perf_event API to 752 * emulate this action and create a kernel perf event for it. 753 */ 754 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, 755 u64 select_idx) 756 { 757 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, select_idx); 758 u64 reg; 759 760 if (!kvm_vcpu_has_pmu(vcpu)) 761 return; 762 763 reg = counter_index_to_evtreg(pmc->idx); 764 __vcpu_sys_reg(vcpu, reg) = data & kvm_pmu_evtyper_mask(vcpu->kvm); 765 766 kvm_pmu_create_perf_event(pmc); 767 } 768 769 void kvm_host_pmu_init(struct arm_pmu *pmu) 770 { 771 struct arm_pmu_entry *entry; 772 773 /* 774 * Check the sanitised PMU version for the system, as KVM does not 775 * support implementations where PMUv3 exists on a subset of CPUs. 776 */ 777 if (!pmuv3_implemented(kvm_arm_pmu_get_pmuver_limit())) 778 return; 779 780 mutex_lock(&arm_pmus_lock); 781 782 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 783 if (!entry) 784 goto out_unlock; 785 786 entry->arm_pmu = pmu; 787 list_add_tail(&entry->entry, &arm_pmus); 788 789 if (list_is_singular(&arm_pmus)) 790 static_branch_enable(&kvm_arm_pmu_available); 791 792 out_unlock: 793 mutex_unlock(&arm_pmus_lock); 794 } 795 796 static struct arm_pmu *kvm_pmu_probe_armpmu(void) 797 { 798 struct arm_pmu *tmp, *pmu = NULL; 799 struct arm_pmu_entry *entry; 800 int cpu; 801 802 mutex_lock(&arm_pmus_lock); 803 804 /* 805 * It is safe to use a stale cpu to iterate the list of PMUs so long as 806 * the same value is used for the entirety of the loop. Given this, and 807 * the fact that no percpu data is used for the lookup there is no need 808 * to disable preemption. 809 * 810 * It is still necessary to get a valid cpu, though, to probe for the 811 * default PMU instance as userspace is not required to specify a PMU 812 * type. In order to uphold the preexisting behavior KVM selects the 813 * PMU instance for the core during vcpu init. A dependent use 814 * case would be a user with disdain of all things big.LITTLE that 815 * affines the VMM to a particular cluster of cores. 816 * 817 * In any case, userspace should just do the sane thing and use the UAPI 818 * to select a PMU type directly. But, be wary of the baggage being 819 * carried here. 820 */ 821 cpu = raw_smp_processor_id(); 822 list_for_each_entry(entry, &arm_pmus, entry) { 823 tmp = entry->arm_pmu; 824 825 if (cpumask_test_cpu(cpu, &tmp->supported_cpus)) { 826 pmu = tmp; 827 break; 828 } 829 } 830 831 mutex_unlock(&arm_pmus_lock); 832 833 return pmu; 834 } 835 836 u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) 837 { 838 unsigned long *bmap = vcpu->kvm->arch.pmu_filter; 839 u64 val, mask = 0; 840 int base, i, nr_events; 841 842 if (!kvm_vcpu_has_pmu(vcpu)) 843 return 0; 844 845 if (!pmceid1) { 846 val = read_sysreg(pmceid0_el0); 847 /* always support CHAIN */ 848 val |= BIT(ARMV8_PMUV3_PERFCTR_CHAIN); 849 base = 0; 850 } else { 851 val = read_sysreg(pmceid1_el0); 852 /* 853 * Don't advertise STALL_SLOT*, as PMMIR_EL0 is handled 854 * as RAZ 855 */ 856 val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) | 857 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) | 858 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32)); 859 base = 32; 860 } 861 862 if (!bmap) 863 return val; 864 865 nr_events = kvm_pmu_event_mask(vcpu->kvm) + 1; 866 867 for (i = 0; i < 32; i += 8) { 868 u64 byte; 869 870 byte = bitmap_get_value8(bmap, base + i); 871 mask |= byte << i; 872 if (nr_events >= (0x4000 + base + 32)) { 873 byte = bitmap_get_value8(bmap, 0x4000 + base + i); 874 mask |= byte << (32 + i); 875 } 876 } 877 878 return val & mask; 879 } 880 881 void kvm_vcpu_reload_pmu(struct kvm_vcpu *vcpu) 882 { 883 u64 mask = kvm_pmu_implemented_counter_mask(vcpu); 884 885 kvm_pmu_handle_pmcr(vcpu, kvm_vcpu_read_pmcr(vcpu)); 886 887 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= mask; 888 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= mask; 889 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= mask; 890 } 891 892 int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) 893 { 894 if (!kvm_vcpu_has_pmu(vcpu)) 895 return 0; 896 897 if (!vcpu->arch.pmu.created) 898 return -EINVAL; 899 900 /* 901 * A valid interrupt configuration for the PMU is either to have a 902 * properly configured interrupt number and using an in-kernel 903 * irqchip, or to not have an in-kernel GIC and not set an IRQ. 904 */ 905 if (irqchip_in_kernel(vcpu->kvm)) { 906 int irq = vcpu->arch.pmu.irq_num; 907 /* 908 * If we are using an in-kernel vgic, at this point we know 909 * the vgic will be initialized, so we can check the PMU irq 910 * number against the dimensions of the vgic and make sure 911 * it's valid. 912 */ 913 if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq)) 914 return -EINVAL; 915 } else if (kvm_arm_pmu_irq_initialized(vcpu)) { 916 return -EINVAL; 917 } 918 919 /* One-off reload of the PMU on first run */ 920 kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu); 921 922 return 0; 923 } 924 925 static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) 926 { 927 if (irqchip_in_kernel(vcpu->kvm)) { 928 int ret; 929 930 /* 931 * If using the PMU with an in-kernel virtual GIC 932 * implementation, we require the GIC to be already 933 * initialized when initializing the PMU. 934 */ 935 if (!vgic_initialized(vcpu->kvm)) 936 return -ENODEV; 937 938 if (!kvm_arm_pmu_irq_initialized(vcpu)) 939 return -ENXIO; 940 941 ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num, 942 &vcpu->arch.pmu); 943 if (ret) 944 return ret; 945 } 946 947 init_irq_work(&vcpu->arch.pmu.overflow_work, 948 kvm_pmu_perf_overflow_notify_vcpu); 949 950 vcpu->arch.pmu.created = true; 951 return 0; 952 } 953 954 /* 955 * For one VM the interrupt type must be same for each vcpu. 956 * As a PPI, the interrupt number is the same for all vcpus, 957 * while as an SPI it must be a separate number per vcpu. 958 */ 959 static bool pmu_irq_is_valid(struct kvm *kvm, int irq) 960 { 961 unsigned long i; 962 struct kvm_vcpu *vcpu; 963 964 kvm_for_each_vcpu(i, vcpu, kvm) { 965 if (!kvm_arm_pmu_irq_initialized(vcpu)) 966 continue; 967 968 if (irq_is_ppi(irq)) { 969 if (vcpu->arch.pmu.irq_num != irq) 970 return false; 971 } else { 972 if (vcpu->arch.pmu.irq_num == irq) 973 return false; 974 } 975 } 976 977 return true; 978 } 979 980 /** 981 * kvm_arm_pmu_get_max_counters - Return the max number of PMU counters. 982 * @kvm: The kvm pointer 983 */ 984 u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm) 985 { 986 struct arm_pmu *arm_pmu = kvm->arch.arm_pmu; 987 988 /* 989 * The arm_pmu->cntr_mask considers the fixed counter(s) as well. 990 * Ignore those and return only the general-purpose counters. 991 */ 992 return bitmap_weight(arm_pmu->cntr_mask, ARMV8_PMU_MAX_GENERAL_COUNTERS); 993 } 994 995 static void kvm_arm_set_pmu(struct kvm *kvm, struct arm_pmu *arm_pmu) 996 { 997 lockdep_assert_held(&kvm->arch.config_lock); 998 999 kvm->arch.arm_pmu = arm_pmu; 1000 kvm->arch.pmcr_n = kvm_arm_pmu_get_max_counters(kvm); 1001 } 1002 1003 /** 1004 * kvm_arm_set_default_pmu - No PMU set, get the default one. 1005 * @kvm: The kvm pointer 1006 * 1007 * The observant among you will notice that the supported_cpus 1008 * mask does not get updated for the default PMU even though it 1009 * is quite possible the selected instance supports only a 1010 * subset of cores in the system. This is intentional, and 1011 * upholds the preexisting behavior on heterogeneous systems 1012 * where vCPUs can be scheduled on any core but the guest 1013 * counters could stop working. 1014 */ 1015 int kvm_arm_set_default_pmu(struct kvm *kvm) 1016 { 1017 struct arm_pmu *arm_pmu = kvm_pmu_probe_armpmu(); 1018 1019 if (!arm_pmu) 1020 return -ENODEV; 1021 1022 kvm_arm_set_pmu(kvm, arm_pmu); 1023 return 0; 1024 } 1025 1026 static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) 1027 { 1028 struct kvm *kvm = vcpu->kvm; 1029 struct arm_pmu_entry *entry; 1030 struct arm_pmu *arm_pmu; 1031 int ret = -ENXIO; 1032 1033 lockdep_assert_held(&kvm->arch.config_lock); 1034 mutex_lock(&arm_pmus_lock); 1035 1036 list_for_each_entry(entry, &arm_pmus, entry) { 1037 arm_pmu = entry->arm_pmu; 1038 if (arm_pmu->pmu.type == pmu_id) { 1039 if (kvm_vm_has_ran_once(kvm) || 1040 (kvm->arch.pmu_filter && kvm->arch.arm_pmu != arm_pmu)) { 1041 ret = -EBUSY; 1042 break; 1043 } 1044 1045 kvm_arm_set_pmu(kvm, arm_pmu); 1046 cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus); 1047 ret = 0; 1048 break; 1049 } 1050 } 1051 1052 mutex_unlock(&arm_pmus_lock); 1053 return ret; 1054 } 1055 1056 int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1057 { 1058 struct kvm *kvm = vcpu->kvm; 1059 1060 lockdep_assert_held(&kvm->arch.config_lock); 1061 1062 if (!kvm_vcpu_has_pmu(vcpu)) 1063 return -ENODEV; 1064 1065 if (vcpu->arch.pmu.created) 1066 return -EBUSY; 1067 1068 switch (attr->attr) { 1069 case KVM_ARM_VCPU_PMU_V3_IRQ: { 1070 int __user *uaddr = (int __user *)(long)attr->addr; 1071 int irq; 1072 1073 if (!irqchip_in_kernel(kvm)) 1074 return -EINVAL; 1075 1076 if (get_user(irq, uaddr)) 1077 return -EFAULT; 1078 1079 /* The PMU overflow interrupt can be a PPI or a valid SPI. */ 1080 if (!(irq_is_ppi(irq) || irq_is_spi(irq))) 1081 return -EINVAL; 1082 1083 if (!pmu_irq_is_valid(kvm, irq)) 1084 return -EINVAL; 1085 1086 if (kvm_arm_pmu_irq_initialized(vcpu)) 1087 return -EBUSY; 1088 1089 kvm_debug("Set kvm ARM PMU irq: %d\n", irq); 1090 vcpu->arch.pmu.irq_num = irq; 1091 return 0; 1092 } 1093 case KVM_ARM_VCPU_PMU_V3_FILTER: { 1094 u8 pmuver = kvm_arm_pmu_get_pmuver_limit(); 1095 struct kvm_pmu_event_filter __user *uaddr; 1096 struct kvm_pmu_event_filter filter; 1097 int nr_events; 1098 1099 /* 1100 * Allow userspace to specify an event filter for the entire 1101 * event range supported by PMUVer of the hardware, rather 1102 * than the guest's PMUVer for KVM backward compatibility. 1103 */ 1104 nr_events = __kvm_pmu_event_mask(pmuver) + 1; 1105 1106 uaddr = (struct kvm_pmu_event_filter __user *)(long)attr->addr; 1107 1108 if (copy_from_user(&filter, uaddr, sizeof(filter))) 1109 return -EFAULT; 1110 1111 if (((u32)filter.base_event + filter.nevents) > nr_events || 1112 (filter.action != KVM_PMU_EVENT_ALLOW && 1113 filter.action != KVM_PMU_EVENT_DENY)) 1114 return -EINVAL; 1115 1116 if (kvm_vm_has_ran_once(kvm)) 1117 return -EBUSY; 1118 1119 if (!kvm->arch.pmu_filter) { 1120 kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT); 1121 if (!kvm->arch.pmu_filter) 1122 return -ENOMEM; 1123 1124 /* 1125 * The default depends on the first applied filter. 1126 * If it allows events, the default is to deny. 1127 * Conversely, if the first filter denies a set of 1128 * events, the default is to allow. 1129 */ 1130 if (filter.action == KVM_PMU_EVENT_ALLOW) 1131 bitmap_zero(kvm->arch.pmu_filter, nr_events); 1132 else 1133 bitmap_fill(kvm->arch.pmu_filter, nr_events); 1134 } 1135 1136 if (filter.action == KVM_PMU_EVENT_ALLOW) 1137 bitmap_set(kvm->arch.pmu_filter, filter.base_event, filter.nevents); 1138 else 1139 bitmap_clear(kvm->arch.pmu_filter, filter.base_event, filter.nevents); 1140 1141 return 0; 1142 } 1143 case KVM_ARM_VCPU_PMU_V3_SET_PMU: { 1144 int __user *uaddr = (int __user *)(long)attr->addr; 1145 int pmu_id; 1146 1147 if (get_user(pmu_id, uaddr)) 1148 return -EFAULT; 1149 1150 return kvm_arm_pmu_v3_set_pmu(vcpu, pmu_id); 1151 } 1152 case KVM_ARM_VCPU_PMU_V3_INIT: 1153 return kvm_arm_pmu_v3_init(vcpu); 1154 } 1155 1156 return -ENXIO; 1157 } 1158 1159 int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1160 { 1161 switch (attr->attr) { 1162 case KVM_ARM_VCPU_PMU_V3_IRQ: { 1163 int __user *uaddr = (int __user *)(long)attr->addr; 1164 int irq; 1165 1166 if (!irqchip_in_kernel(vcpu->kvm)) 1167 return -EINVAL; 1168 1169 if (!kvm_vcpu_has_pmu(vcpu)) 1170 return -ENODEV; 1171 1172 if (!kvm_arm_pmu_irq_initialized(vcpu)) 1173 return -ENXIO; 1174 1175 irq = vcpu->arch.pmu.irq_num; 1176 return put_user(irq, uaddr); 1177 } 1178 } 1179 1180 return -ENXIO; 1181 } 1182 1183 int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1184 { 1185 switch (attr->attr) { 1186 case KVM_ARM_VCPU_PMU_V3_IRQ: 1187 case KVM_ARM_VCPU_PMU_V3_INIT: 1188 case KVM_ARM_VCPU_PMU_V3_FILTER: 1189 case KVM_ARM_VCPU_PMU_V3_SET_PMU: 1190 if (kvm_vcpu_has_pmu(vcpu)) 1191 return 0; 1192 } 1193 1194 return -ENXIO; 1195 } 1196 1197 u8 kvm_arm_pmu_get_pmuver_limit(void) 1198 { 1199 u64 tmp; 1200 1201 tmp = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); 1202 tmp = cpuid_feature_cap_perfmon_field(tmp, 1203 ID_AA64DFR0_EL1_PMUVer_SHIFT, 1204 ID_AA64DFR0_EL1_PMUVer_V3P5); 1205 return FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), tmp); 1206 } 1207 1208 /** 1209 * kvm_vcpu_read_pmcr - Read PMCR_EL0 register for the vCPU 1210 * @vcpu: The vcpu pointer 1211 */ 1212 u64 kvm_vcpu_read_pmcr(struct kvm_vcpu *vcpu) 1213 { 1214 u64 pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0); 1215 1216 return u64_replace_bits(pmcr, vcpu->kvm->arch.pmcr_n, ARMV8_PMU_PMCR_N); 1217 } 1218 1219 void kvm_pmu_nested_transition(struct kvm_vcpu *vcpu) 1220 { 1221 bool reprogrammed = false; 1222 unsigned long mask; 1223 int i; 1224 1225 if (!kvm_vcpu_has_pmu(vcpu)) 1226 return; 1227 1228 mask = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); 1229 for_each_set_bit(i, &mask, 32) { 1230 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i); 1231 1232 /* 1233 * We only need to reconfigure events where the filter is 1234 * different at EL1 vs. EL2, as we're multiplexing the true EL1 1235 * event filter bit for nested. 1236 */ 1237 if (kvm_pmc_counts_at_el1(pmc) == kvm_pmc_counts_at_el2(pmc)) 1238 continue; 1239 1240 kvm_pmu_create_perf_event(pmc); 1241 reprogrammed = true; 1242 } 1243 1244 if (reprogrammed) 1245 kvm_vcpu_pmu_restore_guest(vcpu); 1246 } 1247