1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 Linaro Ltd. 4 * Author: Shannon Zhao <shannon.zhao@linaro.org> 5 */ 6 7 #include <linux/cpu.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 #include <linux/list.h> 11 #include <linux/perf_event.h> 12 #include <linux/perf/arm_pmu.h> 13 #include <linux/uaccess.h> 14 #include <asm/kvm_emulate.h> 15 #include <kvm/arm_pmu.h> 16 #include <kvm/arm_vgic.h> 17 18 #define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0) 19 20 static LIST_HEAD(arm_pmus); 21 static DEFINE_MUTEX(arm_pmus_lock); 22 23 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc); 24 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc); 25 static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc); 26 27 bool kvm_supports_guest_pmuv3(void) 28 { 29 guard(mutex)(&arm_pmus_lock); 30 return !list_empty(&arm_pmus); 31 } 32 33 static struct kvm_vcpu *kvm_pmc_to_vcpu(const struct kvm_pmc *pmc) 34 { 35 return container_of(pmc, struct kvm_vcpu, arch.pmu.pmc[pmc->idx]); 36 } 37 38 static struct kvm_pmc *kvm_vcpu_idx_to_pmc(struct kvm_vcpu *vcpu, int cnt_idx) 39 { 40 return &vcpu->arch.pmu.pmc[cnt_idx]; 41 } 42 43 static u32 __kvm_pmu_event_mask(unsigned int pmuver) 44 { 45 switch (pmuver) { 46 case ID_AA64DFR0_EL1_PMUVer_IMP: 47 return GENMASK(9, 0); 48 case ID_AA64DFR0_EL1_PMUVer_V3P1: 49 case ID_AA64DFR0_EL1_PMUVer_V3P4: 50 case ID_AA64DFR0_EL1_PMUVer_V3P5: 51 case ID_AA64DFR0_EL1_PMUVer_V3P7: 52 return GENMASK(15, 0); 53 default: /* Shouldn't be here, just for sanity */ 54 WARN_ONCE(1, "Unknown PMU version %d\n", pmuver); 55 return 0; 56 } 57 } 58 59 static u32 kvm_pmu_event_mask(struct kvm *kvm) 60 { 61 u64 dfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1); 62 u8 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0); 63 64 return __kvm_pmu_event_mask(pmuver); 65 } 66 67 u64 kvm_pmu_evtyper_mask(struct kvm *kvm) 68 { 69 u64 mask = ARMV8_PMU_EXCLUDE_EL1 | ARMV8_PMU_EXCLUDE_EL0 | 70 kvm_pmu_event_mask(kvm); 71 72 if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL2, IMP)) 73 mask |= ARMV8_PMU_INCLUDE_EL2; 74 75 if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP)) 76 mask |= ARMV8_PMU_EXCLUDE_NS_EL0 | 77 ARMV8_PMU_EXCLUDE_NS_EL1 | 78 ARMV8_PMU_EXCLUDE_EL3; 79 80 return mask; 81 } 82 83 /** 84 * kvm_pmc_is_64bit - determine if counter is 64bit 85 * @pmc: counter context 86 */ 87 static bool kvm_pmc_is_64bit(struct kvm_pmc *pmc) 88 { 89 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 90 91 return (pmc->idx == ARMV8_PMU_CYCLE_IDX || 92 kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, V3P5)); 93 } 94 95 static bool kvm_pmc_has_64bit_overflow(struct kvm_pmc *pmc) 96 { 97 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 98 u64 val = kvm_vcpu_read_pmcr(vcpu); 99 100 if (kvm_pmu_counter_is_hyp(vcpu, pmc->idx)) 101 return __vcpu_sys_reg(vcpu, MDCR_EL2) & MDCR_EL2_HLP; 102 103 return (pmc->idx < ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LP)) || 104 (pmc->idx == ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LC)); 105 } 106 107 static bool kvm_pmu_counter_can_chain(struct kvm_pmc *pmc) 108 { 109 return (!(pmc->idx & 1) && (pmc->idx + 1) < ARMV8_PMU_CYCLE_IDX && 110 !kvm_pmc_has_64bit_overflow(pmc)); 111 } 112 113 static u32 counter_index_to_reg(u64 idx) 114 { 115 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + idx; 116 } 117 118 static u32 counter_index_to_evtreg(u64 idx) 119 { 120 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + idx; 121 } 122 123 static u64 kvm_pmc_read_evtreg(const struct kvm_pmc *pmc) 124 { 125 return __vcpu_sys_reg(kvm_pmc_to_vcpu(pmc), counter_index_to_evtreg(pmc->idx)); 126 } 127 128 static u64 kvm_pmu_get_pmc_value(struct kvm_pmc *pmc) 129 { 130 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 131 u64 counter, reg, enabled, running; 132 133 reg = counter_index_to_reg(pmc->idx); 134 counter = __vcpu_sys_reg(vcpu, reg); 135 136 /* 137 * The real counter value is equal to the value of counter register plus 138 * the value perf event counts. 139 */ 140 if (pmc->perf_event) 141 counter += perf_event_read_value(pmc->perf_event, &enabled, 142 &running); 143 144 if (!kvm_pmc_is_64bit(pmc)) 145 counter = lower_32_bits(counter); 146 147 return counter; 148 } 149 150 /** 151 * kvm_pmu_get_counter_value - get PMU counter value 152 * @vcpu: The vcpu pointer 153 * @select_idx: The counter index 154 */ 155 u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx) 156 { 157 return kvm_pmu_get_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx)); 158 } 159 160 static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force) 161 { 162 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 163 u64 reg; 164 165 kvm_pmu_release_perf_event(pmc); 166 167 reg = counter_index_to_reg(pmc->idx); 168 169 if (vcpu_mode_is_32bit(vcpu) && pmc->idx != ARMV8_PMU_CYCLE_IDX && 170 !force) { 171 /* 172 * Even with PMUv3p5, AArch32 cannot write to the top 173 * 32bit of the counters. The only possible course of 174 * action is to use PMCR.P, which will reset them to 175 * 0 (the only use of the 'force' parameter). 176 */ 177 val = (__vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32)) | 178 lower_32_bits(val); 179 } 180 181 __vcpu_assign_sys_reg(vcpu, reg, val); 182 183 /* Recreate the perf event to reflect the updated sample_period */ 184 kvm_pmu_create_perf_event(pmc); 185 } 186 187 /** 188 * kvm_pmu_set_counter_value - set PMU counter value 189 * @vcpu: The vcpu pointer 190 * @select_idx: The counter index 191 * @val: The counter value 192 */ 193 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) 194 { 195 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx), val, false); 196 } 197 198 /** 199 * kvm_pmu_set_counter_value_user - set PMU counter value from user 200 * @vcpu: The vcpu pointer 201 * @select_idx: The counter index 202 * @val: The counter value 203 */ 204 void kvm_pmu_set_counter_value_user(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) 205 { 206 kvm_pmu_release_perf_event(kvm_vcpu_idx_to_pmc(vcpu, select_idx)); 207 __vcpu_assign_sys_reg(vcpu, counter_index_to_reg(select_idx), val); 208 kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu); 209 } 210 211 /** 212 * kvm_pmu_release_perf_event - remove the perf event 213 * @pmc: The PMU counter pointer 214 */ 215 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) 216 { 217 if (pmc->perf_event) { 218 perf_event_disable(pmc->perf_event); 219 perf_event_release_kernel(pmc->perf_event); 220 pmc->perf_event = NULL; 221 } 222 } 223 224 /** 225 * kvm_pmu_stop_counter - stop PMU counter 226 * @pmc: The PMU counter pointer 227 * 228 * If this counter has been configured to monitor some event, release it here. 229 */ 230 static void kvm_pmu_stop_counter(struct kvm_pmc *pmc) 231 { 232 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 233 u64 reg, val; 234 235 if (!pmc->perf_event) 236 return; 237 238 val = kvm_pmu_get_pmc_value(pmc); 239 240 reg = counter_index_to_reg(pmc->idx); 241 242 __vcpu_assign_sys_reg(vcpu, reg, val); 243 244 kvm_pmu_release_perf_event(pmc); 245 } 246 247 /** 248 * kvm_pmu_vcpu_init - assign pmu counter idx for cpu 249 * @vcpu: The vcpu pointer 250 * 251 */ 252 void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu) 253 { 254 int i; 255 struct kvm_pmu *pmu = &vcpu->arch.pmu; 256 257 for (i = 0; i < KVM_ARMV8_PMU_MAX_COUNTERS; i++) 258 pmu->pmc[i].idx = i; 259 } 260 261 /** 262 * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu 263 * @vcpu: The vcpu pointer 264 * 265 */ 266 void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu) 267 { 268 int i; 269 270 for (i = 0; i < KVM_ARMV8_PMU_MAX_COUNTERS; i++) 271 kvm_pmu_release_perf_event(kvm_vcpu_idx_to_pmc(vcpu, i)); 272 irq_work_sync(&vcpu->arch.pmu.overflow_work); 273 } 274 275 static u64 kvm_pmu_hyp_counter_mask(struct kvm_vcpu *vcpu) 276 { 277 unsigned int hpmn, n; 278 279 if (!vcpu_has_nv(vcpu)) 280 return 0; 281 282 hpmn = SYS_FIELD_GET(MDCR_EL2, HPMN, __vcpu_sys_reg(vcpu, MDCR_EL2)); 283 n = vcpu->kvm->arch.nr_pmu_counters; 284 285 /* 286 * Programming HPMN to a value greater than PMCR_EL0.N is 287 * CONSTRAINED UNPREDICTABLE. Make the implementation choice that an 288 * UNKNOWN number of counters (in our case, zero) are reserved for EL2. 289 */ 290 if (hpmn >= n) 291 return 0; 292 293 /* 294 * Programming HPMN=0 is CONSTRAINED UNPREDICTABLE if FEAT_HPMN0 isn't 295 * implemented. Since KVM's ability to emulate HPMN=0 does not directly 296 * depend on hardware (all PMU registers are trapped), make the 297 * implementation choice that all counters are included in the second 298 * range reserved for EL2/EL3. 299 */ 300 return GENMASK(n - 1, hpmn); 301 } 302 303 bool kvm_pmu_counter_is_hyp(struct kvm_vcpu *vcpu, unsigned int idx) 304 { 305 return kvm_pmu_hyp_counter_mask(vcpu) & BIT(idx); 306 } 307 308 u64 kvm_pmu_accessible_counter_mask(struct kvm_vcpu *vcpu) 309 { 310 u64 mask = kvm_pmu_implemented_counter_mask(vcpu); 311 312 if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu)) 313 return mask; 314 315 return mask & ~kvm_pmu_hyp_counter_mask(vcpu); 316 } 317 318 u64 kvm_pmu_implemented_counter_mask(struct kvm_vcpu *vcpu) 319 { 320 u64 val = FIELD_GET(ARMV8_PMU_PMCR_N, kvm_vcpu_read_pmcr(vcpu)); 321 322 if (val == 0) 323 return BIT(ARMV8_PMU_CYCLE_IDX); 324 else 325 return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX); 326 } 327 328 static void kvm_pmc_enable_perf_event(struct kvm_pmc *pmc) 329 { 330 if (!pmc->perf_event) { 331 kvm_pmu_create_perf_event(pmc); 332 return; 333 } 334 335 perf_event_enable(pmc->perf_event); 336 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE) 337 kvm_debug("fail to enable perf event\n"); 338 } 339 340 static void kvm_pmc_disable_perf_event(struct kvm_pmc *pmc) 341 { 342 if (pmc->perf_event) 343 perf_event_disable(pmc->perf_event); 344 } 345 346 void kvm_pmu_reprogram_counter_mask(struct kvm_vcpu *vcpu, u64 val) 347 { 348 int i; 349 350 if (!val) 351 return; 352 353 for (i = 0; i < KVM_ARMV8_PMU_MAX_COUNTERS; i++) { 354 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i); 355 356 if (!(val & BIT(i))) 357 continue; 358 359 if (kvm_pmu_counter_is_enabled(pmc)) 360 kvm_pmc_enable_perf_event(pmc); 361 else 362 kvm_pmc_disable_perf_event(pmc); 363 } 364 365 kvm_vcpu_pmu_restore_guest(vcpu); 366 } 367 368 /* 369 * Returns the PMU overflow state, which is true if there exists an event 370 * counter where the values of the global enable control, PMOVSSET_EL0[n], and 371 * PMINTENSET_EL1[n] are all 1. 372 */ 373 static bool kvm_pmu_overflow_status(struct kvm_vcpu *vcpu) 374 { 375 u64 reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0); 376 377 reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1); 378 379 /* 380 * PMCR_EL0.E is the global enable control for event counters available 381 * to EL0 and EL1. 382 */ 383 if (!(kvm_vcpu_read_pmcr(vcpu) & ARMV8_PMU_PMCR_E)) 384 reg &= kvm_pmu_hyp_counter_mask(vcpu); 385 386 /* 387 * Otherwise, MDCR_EL2.HPME is the global enable control for event 388 * counters reserved for EL2. 389 */ 390 if (!(vcpu_read_sys_reg(vcpu, MDCR_EL2) & MDCR_EL2_HPME)) 391 reg &= ~kvm_pmu_hyp_counter_mask(vcpu); 392 393 return reg; 394 } 395 396 static void kvm_pmu_update_state(struct kvm_vcpu *vcpu) 397 { 398 struct kvm_pmu *pmu = &vcpu->arch.pmu; 399 400 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) 401 return; 402 403 WARN_ON(kvm_vgic_inject_irq(vcpu->kvm, vcpu, pmu->irq_num, 404 kvm_pmu_overflow_status(vcpu), pmu)); 405 } 406 407 bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu) 408 { 409 struct kvm_sync_regs *sregs = &vcpu->run->s.regs; 410 bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU; 411 412 return kvm_pmu_overflow_status(vcpu) != run_level; 413 } 414 415 /* 416 * Reflect the PMU overflow interrupt output level into the kvm_run structure 417 */ 418 bool kvm_pmu_update_run(struct kvm_vcpu *vcpu) 419 { 420 bool update = kvm_pmu_should_notify_user(vcpu); 421 if (update) 422 vcpu->run->s.regs.device_irq_level ^= KVM_ARM_DEV_PMU; 423 return update; 424 } 425 426 /** 427 * kvm_pmu_flush_hwstate - flush pmu state to cpu 428 * @vcpu: The vcpu pointer 429 * 430 * Check if the PMU has overflowed while we were running in the host, and inject 431 * an interrupt if that was the case. 432 */ 433 void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu) 434 { 435 kvm_pmu_update_state(vcpu); 436 } 437 438 /** 439 * kvm_pmu_sync_hwstate - sync pmu state from cpu 440 * @vcpu: The vcpu pointer 441 * 442 * Check if the PMU has overflowed while we were running in the guest, and 443 * inject an interrupt if that was the case. 444 */ 445 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) 446 { 447 kvm_pmu_update_state(vcpu); 448 } 449 450 /* 451 * When perf interrupt is an NMI, we cannot safely notify the vcpu corresponding 452 * to the event. 453 * This is why we need a callback to do it once outside of the NMI context. 454 */ 455 static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work) 456 { 457 struct kvm_vcpu *vcpu; 458 459 vcpu = container_of(work, struct kvm_vcpu, arch.pmu.overflow_work); 460 kvm_vcpu_kick(vcpu); 461 } 462 463 /* 464 * Perform an increment on any of the counters described in @mask, 465 * generating the overflow if required, and propagate it as a chained 466 * event if possible. 467 */ 468 static void kvm_pmu_counter_increment(struct kvm_vcpu *vcpu, 469 unsigned long mask, u32 event) 470 { 471 int i; 472 473 if (!(kvm_vcpu_read_pmcr(vcpu) & ARMV8_PMU_PMCR_E)) 474 return; 475 476 /* Weed out disabled counters */ 477 mask &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); 478 479 for_each_set_bit(i, &mask, ARMV8_PMU_CYCLE_IDX) { 480 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i); 481 u64 type, reg; 482 483 /* Filter on event type */ 484 type = __vcpu_sys_reg(vcpu, counter_index_to_evtreg(i)); 485 type &= kvm_pmu_event_mask(vcpu->kvm); 486 if (type != event) 487 continue; 488 489 /* Increment this counter */ 490 reg = __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) + 1; 491 if (!kvm_pmc_is_64bit(pmc)) 492 reg = lower_32_bits(reg); 493 __vcpu_assign_sys_reg(vcpu, counter_index_to_reg(i), reg); 494 495 /* No overflow? move on */ 496 if (kvm_pmc_has_64bit_overflow(pmc) ? reg : lower_32_bits(reg)) 497 continue; 498 499 /* Mark overflow */ 500 __vcpu_rmw_sys_reg(vcpu, PMOVSSET_EL0, |=, BIT(i)); 501 502 if (kvm_pmu_counter_can_chain(pmc)) 503 kvm_pmu_counter_increment(vcpu, BIT(i + 1), 504 ARMV8_PMUV3_PERFCTR_CHAIN); 505 } 506 } 507 508 /* Compute the sample period for a given counter value */ 509 static u64 compute_period(struct kvm_pmc *pmc, u64 counter) 510 { 511 u64 val; 512 513 if (kvm_pmc_is_64bit(pmc) && kvm_pmc_has_64bit_overflow(pmc)) 514 val = (-counter) & GENMASK(63, 0); 515 else 516 val = (-counter) & GENMASK(31, 0); 517 518 return val; 519 } 520 521 /* 522 * When the perf event overflows, set the overflow status and inform the vcpu. 523 */ 524 static void kvm_pmu_perf_overflow(struct perf_event *perf_event, 525 struct perf_sample_data *data, 526 struct pt_regs *regs) 527 { 528 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 529 struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu); 530 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 531 int idx = pmc->idx; 532 u64 period; 533 534 cpu_pmu->pmu.stop(perf_event, PERF_EF_UPDATE); 535 536 /* 537 * Reset the sample period to the architectural limit, 538 * i.e. the point where the counter overflows. 539 */ 540 period = compute_period(pmc, local64_read(&perf_event->count)); 541 542 local64_set(&perf_event->hw.period_left, 0); 543 perf_event->attr.sample_period = period; 544 perf_event->hw.sample_period = period; 545 546 __vcpu_rmw_sys_reg(vcpu, PMOVSSET_EL0, |=, BIT(idx)); 547 548 if (kvm_pmu_counter_can_chain(pmc)) 549 kvm_pmu_counter_increment(vcpu, BIT(idx + 1), 550 ARMV8_PMUV3_PERFCTR_CHAIN); 551 552 if (kvm_pmu_overflow_status(vcpu)) { 553 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); 554 555 if (!in_nmi()) 556 kvm_vcpu_kick(vcpu); 557 else 558 irq_work_queue(&vcpu->arch.pmu.overflow_work); 559 } 560 561 cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD); 562 } 563 564 /** 565 * kvm_pmu_software_increment - do software increment 566 * @vcpu: The vcpu pointer 567 * @val: the value guest writes to PMSWINC register 568 */ 569 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) 570 { 571 kvm_pmu_counter_increment(vcpu, val, ARMV8_PMUV3_PERFCTR_SW_INCR); 572 } 573 574 /** 575 * kvm_pmu_handle_pmcr - handle PMCR register 576 * @vcpu: The vcpu pointer 577 * @val: the value guest writes to PMCR register 578 */ 579 void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) 580 { 581 int i; 582 583 /* Fixup PMCR_EL0 to reconcile the PMU version and the LP bit */ 584 if (!kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, V3P5)) 585 val &= ~ARMV8_PMU_PMCR_LP; 586 587 /* Request a reload of the PMU to enable/disable affected counters */ 588 if ((__vcpu_sys_reg(vcpu, PMCR_EL0) ^ val) & ARMV8_PMU_PMCR_E) 589 kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu); 590 591 /* The reset bits don't indicate any state, and shouldn't be saved. */ 592 __vcpu_assign_sys_reg(vcpu, PMCR_EL0, (val & ~(ARMV8_PMU_PMCR_C | ARMV8_PMU_PMCR_P))); 593 594 if (val & ARMV8_PMU_PMCR_C) 595 kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0); 596 597 if (val & ARMV8_PMU_PMCR_P) { 598 unsigned long mask = kvm_pmu_implemented_counter_mask(vcpu) & 599 ~BIT(ARMV8_PMU_CYCLE_IDX); 600 601 if (!vcpu_is_el2(vcpu)) 602 mask &= ~kvm_pmu_hyp_counter_mask(vcpu); 603 604 for_each_set_bit(i, &mask, 32) 605 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, i), 0, true); 606 } 607 } 608 609 static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc) 610 { 611 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 612 unsigned int mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 613 614 if (!(__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(pmc->idx))) 615 return false; 616 617 if (kvm_pmu_counter_is_hyp(vcpu, pmc->idx)) 618 return mdcr & MDCR_EL2_HPME; 619 620 return kvm_vcpu_read_pmcr(vcpu) & ARMV8_PMU_PMCR_E; 621 } 622 623 static bool kvm_pmc_counts_at_el0(struct kvm_pmc *pmc) 624 { 625 u64 evtreg = kvm_pmc_read_evtreg(pmc); 626 bool nsu = evtreg & ARMV8_PMU_EXCLUDE_NS_EL0; 627 bool u = evtreg & ARMV8_PMU_EXCLUDE_EL0; 628 629 return u == nsu; 630 } 631 632 static bool kvm_pmc_counts_at_el1(struct kvm_pmc *pmc) 633 { 634 u64 evtreg = kvm_pmc_read_evtreg(pmc); 635 bool nsk = evtreg & ARMV8_PMU_EXCLUDE_NS_EL1; 636 bool p = evtreg & ARMV8_PMU_EXCLUDE_EL1; 637 638 return p == nsk; 639 } 640 641 static bool kvm_pmc_counts_at_el2(struct kvm_pmc *pmc) 642 { 643 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 644 u64 mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 645 646 if (!kvm_pmu_counter_is_hyp(vcpu, pmc->idx) && (mdcr & MDCR_EL2_HPMD)) 647 return false; 648 649 return kvm_pmc_read_evtreg(pmc) & ARMV8_PMU_INCLUDE_EL2; 650 } 651 652 static int kvm_map_pmu_event(struct kvm *kvm, unsigned int eventsel) 653 { 654 struct arm_pmu *pmu = kvm->arch.arm_pmu; 655 656 /* 657 * The CPU PMU likely isn't PMUv3; let the driver provide a mapping 658 * for the guest's PMUv3 event ID. 659 */ 660 if (unlikely(pmu->map_pmuv3_event)) 661 return pmu->map_pmuv3_event(eventsel); 662 663 return eventsel; 664 } 665 666 /** 667 * kvm_pmu_create_perf_event - create a perf event for a counter 668 * @pmc: Counter context 669 */ 670 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc) 671 { 672 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); 673 struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu; 674 struct perf_event *event; 675 struct perf_event_attr attr; 676 int eventsel; 677 u64 evtreg; 678 679 evtreg = kvm_pmc_read_evtreg(pmc); 680 681 kvm_pmu_stop_counter(pmc); 682 if (pmc->idx == ARMV8_PMU_CYCLE_IDX) 683 eventsel = ARMV8_PMUV3_PERFCTR_CPU_CYCLES; 684 else 685 eventsel = evtreg & kvm_pmu_event_mask(vcpu->kvm); 686 687 /* 688 * Neither SW increment nor chained events need to be backed 689 * by a perf event. 690 */ 691 if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR || 692 eventsel == ARMV8_PMUV3_PERFCTR_CHAIN) 693 return; 694 695 /* 696 * If we have a filter in place and that the event isn't allowed, do 697 * not install a perf event either. 698 */ 699 if (vcpu->kvm->arch.pmu_filter && 700 !test_bit(eventsel, vcpu->kvm->arch.pmu_filter)) 701 return; 702 703 /* 704 * Don't create an event if we're running on hardware that requires 705 * PMUv3 event translation and we couldn't find a valid mapping. 706 */ 707 eventsel = kvm_map_pmu_event(vcpu->kvm, eventsel); 708 if (eventsel < 0) 709 return; 710 711 memset(&attr, 0, sizeof(struct perf_event_attr)); 712 attr.type = arm_pmu->pmu.type; 713 attr.size = sizeof(attr); 714 attr.pinned = 1; 715 attr.disabled = !kvm_pmu_counter_is_enabled(pmc); 716 attr.exclude_user = !kvm_pmc_counts_at_el0(pmc); 717 attr.exclude_hv = 1; /* Don't count EL2 events */ 718 attr.exclude_host = 1; /* Don't count host events */ 719 attr.config = eventsel; 720 721 /* 722 * Filter events at EL1 (i.e. vEL2) when in a hyp context based on the 723 * guest's EL2 filter. 724 */ 725 if (unlikely(is_hyp_ctxt(vcpu))) 726 attr.exclude_kernel = !kvm_pmc_counts_at_el2(pmc); 727 else 728 attr.exclude_kernel = !kvm_pmc_counts_at_el1(pmc); 729 730 /* 731 * If counting with a 64bit counter, advertise it to the perf 732 * code, carefully dealing with the initial sample period 733 * which also depends on the overflow. 734 */ 735 if (kvm_pmc_is_64bit(pmc)) 736 attr.config1 |= PERF_ATTR_CFG1_COUNTER_64BIT; 737 738 attr.sample_period = compute_period(pmc, kvm_pmu_get_pmc_value(pmc)); 739 740 event = perf_event_create_kernel_counter(&attr, -1, current, 741 kvm_pmu_perf_overflow, pmc); 742 743 if (IS_ERR(event)) { 744 pr_err_once("kvm: pmu event creation failed %ld\n", 745 PTR_ERR(event)); 746 return; 747 } 748 749 pmc->perf_event = event; 750 } 751 752 /** 753 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event 754 * @vcpu: The vcpu pointer 755 * @data: The data guest writes to PMXEVTYPER_EL0 756 * @select_idx: The number of selected counter 757 * 758 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an 759 * event with given hardware event number. Here we call perf_event API to 760 * emulate this action and create a kernel perf event for it. 761 */ 762 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, 763 u64 select_idx) 764 { 765 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, select_idx); 766 u64 reg; 767 768 reg = counter_index_to_evtreg(pmc->idx); 769 __vcpu_assign_sys_reg(vcpu, reg, (data & kvm_pmu_evtyper_mask(vcpu->kvm))); 770 771 kvm_pmu_create_perf_event(pmc); 772 } 773 774 void kvm_host_pmu_init(struct arm_pmu *pmu) 775 { 776 struct arm_pmu_entry *entry; 777 778 /* 779 * Check the sanitised PMU version for the system, as KVM does not 780 * support implementations where PMUv3 exists on a subset of CPUs. 781 */ 782 if (!pmuv3_implemented(kvm_arm_pmu_get_pmuver_limit())) 783 return; 784 785 guard(mutex)(&arm_pmus_lock); 786 787 entry = kmalloc_obj(*entry); 788 if (!entry) 789 return; 790 791 entry->arm_pmu = pmu; 792 list_add_tail(&entry->entry, &arm_pmus); 793 } 794 795 static struct arm_pmu *kvm_pmu_probe_armpmu(void) 796 { 797 struct arm_pmu_entry *entry; 798 struct arm_pmu *pmu; 799 int cpu; 800 801 guard(mutex)(&arm_pmus_lock); 802 803 /* 804 * It is safe to use a stale cpu to iterate the list of PMUs so long as 805 * the same value is used for the entirety of the loop. Given this, and 806 * the fact that no percpu data is used for the lookup there is no need 807 * to disable preemption. 808 * 809 * It is still necessary to get a valid cpu, though, to probe for the 810 * default PMU instance as userspace is not required to specify a PMU 811 * type. In order to uphold the preexisting behavior KVM selects the 812 * PMU instance for the core during vcpu init. A dependent use 813 * case would be a user with disdain of all things big.LITTLE that 814 * affines the VMM to a particular cluster of cores. 815 * 816 * In any case, userspace should just do the sane thing and use the UAPI 817 * to select a PMU type directly. But, be wary of the baggage being 818 * carried here. 819 */ 820 cpu = raw_smp_processor_id(); 821 list_for_each_entry(entry, &arm_pmus, entry) { 822 pmu = entry->arm_pmu; 823 824 if (cpumask_test_cpu(cpu, &pmu->supported_cpus)) 825 return pmu; 826 } 827 828 return NULL; 829 } 830 831 static u64 __compute_pmceid(struct arm_pmu *pmu, bool pmceid1) 832 { 833 u32 hi[2], lo[2]; 834 835 bitmap_to_arr32(lo, pmu->pmceid_bitmap, ARMV8_PMUV3_MAX_COMMON_EVENTS); 836 bitmap_to_arr32(hi, pmu->pmceid_ext_bitmap, ARMV8_PMUV3_MAX_COMMON_EVENTS); 837 838 return ((u64)hi[pmceid1] << 32) | lo[pmceid1]; 839 } 840 841 static u64 compute_pmceid0(struct kvm_vcpu *vcpu) 842 { 843 u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 0); 844 845 /* always support SW_INCR */ 846 val |= BIT(ARMV8_PMUV3_PERFCTR_SW_INCR); 847 /* always support CHAIN */ 848 val |= BIT(ARMV8_PMUV3_PERFCTR_CHAIN); 849 return val; 850 } 851 852 static u64 compute_pmceid1(struct kvm_vcpu *vcpu) 853 { 854 u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 1); 855 856 /* 857 * If KVM_ARM_VCPU_PMU_V3_STRICT is not set, PMMIR_EL1 is 858 * unconditionally RAZ, so don't advertise STALL_SLOT* events. 859 */ 860 if (!kvm_vcpu_has_pmuv3_strict(vcpu)) 861 val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) | 862 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) | 863 BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32)); 864 865 return val; 866 } 867 868 u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) 869 { 870 unsigned long *bmap = vcpu->kvm->arch.pmu_filter; 871 u64 val, mask = 0; 872 int base, i, nr_events; 873 874 if (!pmceid1) { 875 val = compute_pmceid0(vcpu); 876 base = 0; 877 } else { 878 val = compute_pmceid1(vcpu); 879 base = 32; 880 } 881 882 if (!bmap) 883 return val; 884 885 nr_events = kvm_pmu_event_mask(vcpu->kvm) + 1; 886 887 for (i = 0; i < 32; i += 8) { 888 u64 byte; 889 890 byte = bitmap_get_value8(bmap, base + i); 891 mask |= byte << i; 892 if (nr_events >= (0x4000 + base + 32)) { 893 byte = bitmap_get_value8(bmap, 0x4000 + base + i); 894 mask |= byte << (32 + i); 895 } 896 } 897 898 return val & mask; 899 } 900 901 void kvm_vcpu_reload_pmu(struct kvm_vcpu *vcpu) 902 { 903 u64 mask = kvm_pmu_implemented_counter_mask(vcpu); 904 905 __vcpu_rmw_sys_reg(vcpu, PMOVSSET_EL0, &=, mask); 906 __vcpu_rmw_sys_reg(vcpu, PMINTENSET_EL1, &=, mask); 907 __vcpu_rmw_sys_reg(vcpu, PMCNTENSET_EL0, &=, mask); 908 909 kvm_pmu_reprogram_counter_mask(vcpu, mask); 910 } 911 912 int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) 913 { 914 if (!vcpu->arch.pmu.created) 915 return -EINVAL; 916 917 /* 918 * A valid interrupt configuration for the PMU is either to have a 919 * properly configured interrupt number and using an in-kernel 920 * irqchip, or to not have an in-kernel GIC and not set an IRQ. 921 */ 922 if (irqchip_in_kernel(vcpu->kvm)) { 923 int irq = vcpu->arch.pmu.irq_num; 924 /* 925 * If we are using an in-kernel vgic, at this point we know 926 * the vgic will be initialized, so we can check the PMU irq 927 * number against the dimensions of the vgic and make sure 928 * it's valid. 929 */ 930 if (!irq_is_ppi(vcpu->kvm, irq) && 931 !vgic_valid_spi(vcpu->kvm, irq)) 932 return -EINVAL; 933 } else if (kvm_arm_pmu_irq_initialized(vcpu)) { 934 return -EINVAL; 935 } 936 937 return 0; 938 } 939 940 static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) 941 { 942 /* Only possible when using KVM_ARM_VCPU_PMU_V3_STRICT */ 943 if (!vcpu->kvm->arch.arm_pmu) 944 return -ENXIO; 945 946 if (irqchip_in_kernel(vcpu->kvm)) { 947 int ret; 948 949 /* 950 * If using the PMU with an in-kernel virtual GIC 951 * implementation, we require the GIC to be already 952 * initialized when initializing the PMU. 953 */ 954 if (!vgic_initialized(vcpu->kvm)) 955 return -ENODEV; 956 957 if (!kvm_arm_pmu_irq_initialized(vcpu)) { 958 if (!vgic_is_v5(vcpu->kvm)) 959 return -ENXIO; 960 961 /* Use the architected irq number for GICv5. */ 962 vcpu->arch.pmu.irq_num = KVM_ARMV8_PMU_GICV5_IRQ; 963 } 964 965 ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num, 966 &vcpu->arch.pmu); 967 if (ret) 968 return ret; 969 } 970 971 init_irq_work(&vcpu->arch.pmu.overflow_work, 972 kvm_pmu_perf_overflow_notify_vcpu); 973 974 vcpu->arch.pmu.created = true; 975 return 0; 976 } 977 978 /* 979 * For one VM the interrupt type must be same for each vcpu. 980 * As a PPI, the interrupt number is the same for all vcpus, 981 * while as an SPI it must be a separate number per vcpu. 982 */ 983 static bool pmu_irq_is_valid(struct kvm *kvm, int irq) 984 { 985 unsigned long i; 986 struct kvm_vcpu *vcpu; 987 988 /* On GICv5, the PMUIRQ is architecturally mandated to be PPI 23 */ 989 if (vgic_is_v5(kvm) && irq != KVM_ARMV8_PMU_GICV5_IRQ) 990 return false; 991 992 kvm_for_each_vcpu(i, vcpu, kvm) { 993 if (!kvm_arm_pmu_irq_initialized(vcpu)) 994 continue; 995 996 if (irq_is_ppi(vcpu->kvm, irq)) { 997 if (vcpu->arch.pmu.irq_num != irq) 998 return false; 999 } else { 1000 if (vcpu->arch.pmu.irq_num == irq) 1001 return false; 1002 } 1003 } 1004 1005 return true; 1006 } 1007 1008 /** 1009 * kvm_arm_pmu_get_max_counters - Return the max number of PMU counters. 1010 * @kvm: The kvm pointer 1011 */ 1012 u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm) 1013 { 1014 struct arm_pmu *arm_pmu = kvm->arch.arm_pmu; 1015 1016 /* 1017 * Under KVM_ARM_VCPU_PMU_V3_STRICT no PMU exists until userspace sets 1018 * one, so this can be reached before arm_pmu is set. Report no 1019 * counters in that case. 1020 */ 1021 if (!arm_pmu) 1022 return 0; 1023 1024 /* 1025 * PMUv3 requires that all event counters are capable of counting any 1026 * event, though the same may not be true of non-PMUv3 hardware. 1027 */ 1028 if (cpus_have_final_cap(ARM64_WORKAROUND_PMUV3_IMPDEF_TRAPS)) 1029 return 1; 1030 1031 /* 1032 * The arm_pmu->cntr_mask considers the fixed counter(s) as well. 1033 * Ignore those and return only the general-purpose counters. 1034 */ 1035 return bitmap_weight(arm_pmu->cntr_mask, ARMV8_PMU_MAX_GENERAL_COUNTERS); 1036 } 1037 1038 static void kvm_arm_set_nr_counters(struct kvm *kvm, unsigned int nr) 1039 { 1040 kvm->arch.nr_pmu_counters = nr; 1041 1042 /* Reset MDCR_EL2.HPMN behind the vcpus' back... */ 1043 if (test_bit(KVM_ARM_VCPU_HAS_EL2, kvm->arch.vcpu_features)) { 1044 struct kvm_vcpu *vcpu; 1045 unsigned long i; 1046 1047 kvm_for_each_vcpu(i, vcpu, kvm) { 1048 u64 val = __vcpu_sys_reg(vcpu, MDCR_EL2); 1049 val &= ~MDCR_EL2_HPMN; 1050 val |= FIELD_PREP(MDCR_EL2_HPMN, kvm->arch.nr_pmu_counters); 1051 __vcpu_assign_sys_reg(vcpu, MDCR_EL2, val); 1052 } 1053 } 1054 } 1055 1056 static void kvm_arm_set_pmu(struct kvm *kvm, struct arm_pmu *arm_pmu) 1057 { 1058 lockdep_assert_held(&kvm->arch.config_lock); 1059 1060 kvm->arch.arm_pmu = arm_pmu; 1061 kvm_arm_set_nr_counters(kvm, kvm_arm_pmu_get_max_counters(kvm)); 1062 } 1063 1064 /** 1065 * kvm_arm_set_default_pmu - No PMU set and KVM_ARM_VCPU_PMU_V3_STRICT not 1066 * set, get the default one. 1067 * @kvm: The kvm pointer 1068 * 1069 * The observant among you will notice that the supported_cpus 1070 * mask does not get updated for the default PMU even though it 1071 * is quite possible the selected instance supports only a 1072 * subset of cores in the system. This is intentional, and 1073 * upholds the preexisting behavior on heterogeneous systems 1074 * where vCPUs can be scheduled on any core but the guest 1075 * counters could stop working. 1076 */ 1077 int kvm_arm_set_default_pmu(struct kvm *kvm) 1078 { 1079 struct arm_pmu *arm_pmu = kvm_pmu_probe_armpmu(); 1080 1081 if (!arm_pmu) 1082 return -ENODEV; 1083 1084 kvm_arm_set_pmu(kvm, arm_pmu); 1085 return 0; 1086 } 1087 1088 static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) 1089 { 1090 struct kvm *kvm = vcpu->kvm; 1091 struct arm_pmu_entry *entry; 1092 struct arm_pmu *arm_pmu; 1093 int ret = -ENXIO; 1094 1095 lockdep_assert_held(&kvm->arch.config_lock); 1096 mutex_lock(&arm_pmus_lock); 1097 1098 list_for_each_entry(entry, &arm_pmus, entry) { 1099 arm_pmu = entry->arm_pmu; 1100 if (arm_pmu->pmu.type == pmu_id) { 1101 if (kvm_vm_has_ran_once(kvm) || 1102 (kvm->arch.pmu_filter && kvm->arch.arm_pmu != arm_pmu)) { 1103 ret = -EBUSY; 1104 break; 1105 } 1106 1107 kvm_arm_set_pmu(kvm, arm_pmu); 1108 cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus); 1109 1110 /* 1111 * Since a specific PMU is explicitly selected, 1112 * PMMIR_EL1.SLOTS is deterministic to the guest. 1113 * If KVM_ARM_VCPU_PMU_V3_STRICT is set, snapshot 1114 * the value to allow the guest to read it. 1115 */ 1116 if (kvm_vcpu_has_pmuv3_strict(vcpu)) 1117 kvm->arch.pmmir_slots = 1118 FIELD_GET(ARMV8_PMU_SLOTS, 1119 arm_pmu->reg_pmmir); 1120 ret = 0; 1121 break; 1122 } 1123 } 1124 1125 mutex_unlock(&arm_pmus_lock); 1126 return ret; 1127 } 1128 1129 static int kvm_arm_pmu_v3_set_nr_counters(struct kvm_vcpu *vcpu, unsigned int n) 1130 { 1131 struct kvm *kvm = vcpu->kvm; 1132 1133 if (!kvm->arch.arm_pmu) 1134 return -EINVAL; 1135 1136 if (n > kvm_arm_pmu_get_max_counters(kvm)) 1137 return -EINVAL; 1138 1139 kvm_arm_set_nr_counters(kvm, n); 1140 return 0; 1141 } 1142 1143 int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1144 { 1145 struct kvm *kvm = vcpu->kvm; 1146 1147 lockdep_assert_held(&kvm->arch.config_lock); 1148 1149 if (!kvm_vcpu_has_pmu(vcpu)) 1150 return -ENODEV; 1151 1152 if (vcpu->arch.pmu.created) 1153 return -EBUSY; 1154 1155 switch (attr->attr) { 1156 case KVM_ARM_VCPU_PMU_V3_IRQ: { 1157 int __user *uaddr = (int __user *)(long)attr->addr; 1158 int irq; 1159 1160 if (!irqchip_in_kernel(kvm)) 1161 return -EINVAL; 1162 1163 if (get_user(irq, uaddr)) 1164 return -EFAULT; 1165 1166 /* The PMU overflow interrupt can be a PPI or a valid SPI. */ 1167 if (!(irq_is_ppi(vcpu->kvm, irq) || irq_is_spi(vcpu->kvm, irq))) 1168 return -EINVAL; 1169 1170 if (!pmu_irq_is_valid(kvm, irq)) 1171 return -EINVAL; 1172 1173 if (kvm_arm_pmu_irq_initialized(vcpu)) 1174 return -EBUSY; 1175 1176 kvm_debug("Set kvm ARM PMU irq: %d\n", irq); 1177 vcpu->arch.pmu.irq_num = irq; 1178 return 0; 1179 } 1180 case KVM_ARM_VCPU_PMU_V3_FILTER: { 1181 u8 pmuver = kvm_arm_pmu_get_pmuver_limit(); 1182 struct kvm_pmu_event_filter __user *uaddr; 1183 struct kvm_pmu_event_filter filter; 1184 int nr_events; 1185 1186 /* 1187 * Allow userspace to specify an event filter for the entire 1188 * event range supported by PMUVer of the hardware, rather 1189 * than the guest's PMUVer for KVM backward compatibility. 1190 */ 1191 nr_events = __kvm_pmu_event_mask(pmuver) + 1; 1192 1193 uaddr = (struct kvm_pmu_event_filter __user *)(long)attr->addr; 1194 1195 if (copy_from_user(&filter, uaddr, sizeof(filter))) 1196 return -EFAULT; 1197 1198 if (((u32)filter.base_event + filter.nevents) > nr_events || 1199 (filter.action != KVM_PMU_EVENT_ALLOW && 1200 filter.action != KVM_PMU_EVENT_DENY)) 1201 return -EINVAL; 1202 1203 if (kvm_vm_has_ran_once(kvm)) 1204 return -EBUSY; 1205 1206 if (!kvm->arch.arm_pmu) 1207 return -ENXIO; 1208 1209 if (!kvm->arch.pmu_filter) { 1210 kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT); 1211 if (!kvm->arch.pmu_filter) 1212 return -ENOMEM; 1213 1214 /* 1215 * The default depends on the first applied filter. 1216 * If it allows events, the default is to deny. 1217 * Conversely, if the first filter denies a set of 1218 * events, the default is to allow. 1219 */ 1220 if (filter.action == KVM_PMU_EVENT_ALLOW) 1221 bitmap_zero(kvm->arch.pmu_filter, nr_events); 1222 else 1223 bitmap_fill(kvm->arch.pmu_filter, nr_events); 1224 } 1225 1226 if (filter.action == KVM_PMU_EVENT_ALLOW) 1227 bitmap_set(kvm->arch.pmu_filter, filter.base_event, filter.nevents); 1228 else 1229 bitmap_clear(kvm->arch.pmu_filter, filter.base_event, filter.nevents); 1230 1231 return 0; 1232 } 1233 case KVM_ARM_VCPU_PMU_V3_SET_PMU: { 1234 int __user *uaddr = (int __user *)(long)attr->addr; 1235 int pmu_id; 1236 1237 if (get_user(pmu_id, uaddr)) 1238 return -EFAULT; 1239 1240 return kvm_arm_pmu_v3_set_pmu(vcpu, pmu_id); 1241 } 1242 case KVM_ARM_VCPU_PMU_V3_SET_NR_COUNTERS: { 1243 unsigned int __user *uaddr = (unsigned int __user *)(long)attr->addr; 1244 unsigned int n; 1245 1246 if (get_user(n, uaddr)) 1247 return -EFAULT; 1248 1249 return kvm_arm_pmu_v3_set_nr_counters(vcpu, n); 1250 } 1251 case KVM_ARM_VCPU_PMU_V3_INIT: 1252 return kvm_arm_pmu_v3_init(vcpu); 1253 } 1254 1255 return -ENXIO; 1256 } 1257 1258 int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1259 { 1260 switch (attr->attr) { 1261 case KVM_ARM_VCPU_PMU_V3_IRQ: { 1262 int __user *uaddr = (int __user *)(long)attr->addr; 1263 int irq; 1264 1265 if (!irqchip_in_kernel(vcpu->kvm)) 1266 return -EINVAL; 1267 1268 if (!kvm_vcpu_has_pmu(vcpu)) 1269 return -ENODEV; 1270 1271 if (!kvm_arm_pmu_irq_initialized(vcpu)) 1272 return -ENXIO; 1273 1274 irq = vcpu->arch.pmu.irq_num; 1275 return put_user(irq, uaddr); 1276 } 1277 } 1278 1279 return -ENXIO; 1280 } 1281 1282 int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1283 { 1284 switch (attr->attr) { 1285 case KVM_ARM_VCPU_PMU_V3_IRQ: 1286 case KVM_ARM_VCPU_PMU_V3_INIT: 1287 case KVM_ARM_VCPU_PMU_V3_FILTER: 1288 case KVM_ARM_VCPU_PMU_V3_SET_PMU: 1289 case KVM_ARM_VCPU_PMU_V3_SET_NR_COUNTERS: 1290 if (kvm_vcpu_has_pmu(vcpu)) 1291 return 0; 1292 } 1293 1294 return -ENXIO; 1295 } 1296 1297 u8 kvm_arm_pmu_get_pmuver_limit(void) 1298 { 1299 unsigned int pmuver; 1300 1301 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, 1302 read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1)); 1303 1304 /* 1305 * Spoof a barebones PMUv3 implementation if the system supports IMPDEF 1306 * traps of the PMUv3 sysregs 1307 */ 1308 if (cpus_have_final_cap(ARM64_WORKAROUND_PMUV3_IMPDEF_TRAPS)) 1309 return ID_AA64DFR0_EL1_PMUVer_IMP; 1310 1311 /* 1312 * Otherwise, treat IMPLEMENTATION DEFINED functionality as 1313 * unimplemented 1314 */ 1315 if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) 1316 return 0; 1317 1318 return min(pmuver, ID_AA64DFR0_EL1_PMUVer_V3P5); 1319 } 1320 1321 /** 1322 * kvm_vcpu_read_pmcr - Read PMCR_EL0 register for the vCPU 1323 * @vcpu: The vcpu pointer 1324 */ 1325 u64 kvm_vcpu_read_pmcr(struct kvm_vcpu *vcpu) 1326 { 1327 u64 pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0); 1328 u64 n = vcpu->kvm->arch.nr_pmu_counters; 1329 1330 if (vcpu_has_nv(vcpu) && !vcpu_is_el2(vcpu)) 1331 n = FIELD_GET(MDCR_EL2_HPMN, __vcpu_sys_reg(vcpu, MDCR_EL2)); 1332 1333 return u64_replace_bits(pmcr, n, ARMV8_PMU_PMCR_N); 1334 } 1335 1336 void kvm_pmu_nested_transition(struct kvm_vcpu *vcpu) 1337 { 1338 bool reprogrammed = false; 1339 unsigned long mask; 1340 int i; 1341 1342 mask = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); 1343 for_each_set_bit(i, &mask, 32) { 1344 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i); 1345 1346 /* 1347 * We only need to reconfigure events where the filter is 1348 * different at EL1 vs. EL2, as we're multiplexing the true EL1 1349 * event filter bit for nested. 1350 */ 1351 if (kvm_pmc_counts_at_el1(pmc) == kvm_pmc_counts_at_el2(pmc)) 1352 continue; 1353 1354 kvm_pmu_create_perf_event(pmc); 1355 reprogrammed = true; 1356 } 1357 1358 if (reprogrammed) 1359 kvm_vcpu_pmu_restore_guest(vcpu); 1360 } 1361