1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2023 Rivos Inc 4 * 5 * Authors: 6 * Atish Patra <atishp@rivosinc.com> 7 */ 8 9 #define pr_fmt(fmt) "riscv-kvm-pmu: " fmt 10 #include <linux/bitops.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/kvm_host.h> 14 #include <linux/nospec.h> 15 #include <linux/overflow.h> 16 #include <linux/perf/riscv_pmu.h> 17 #include <linux/slab.h> 18 #include <asm/csr.h> 19 #include <asm/kvm_isa.h> 20 #include <asm/kvm_vcpu_sbi.h> 21 #include <asm/kvm_vcpu_pmu.h> 22 #include <asm/sbi.h> 23 24 #define kvm_pmu_num_counters(pmu) ((pmu)->num_hw_ctrs + (pmu)->num_fw_ctrs) 25 #define get_event_type(x) (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16) 26 #define get_event_code(x) ((x) & SBI_PMU_EVENT_IDX_CODE_MASK) 27 28 static enum perf_hw_id hw_event_perf_map[SBI_PMU_HW_GENERAL_MAX] = { 29 [SBI_PMU_HW_CPU_CYCLES] = PERF_COUNT_HW_CPU_CYCLES, 30 [SBI_PMU_HW_INSTRUCTIONS] = PERF_COUNT_HW_INSTRUCTIONS, 31 [SBI_PMU_HW_CACHE_REFERENCES] = PERF_COUNT_HW_CACHE_REFERENCES, 32 [SBI_PMU_HW_CACHE_MISSES] = PERF_COUNT_HW_CACHE_MISSES, 33 [SBI_PMU_HW_BRANCH_INSTRUCTIONS] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS, 34 [SBI_PMU_HW_BRANCH_MISSES] = PERF_COUNT_HW_BRANCH_MISSES, 35 [SBI_PMU_HW_BUS_CYCLES] = PERF_COUNT_HW_BUS_CYCLES, 36 [SBI_PMU_HW_STALLED_CYCLES_FRONTEND] = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, 37 [SBI_PMU_HW_STALLED_CYCLES_BACKEND] = PERF_COUNT_HW_STALLED_CYCLES_BACKEND, 38 [SBI_PMU_HW_REF_CPU_CYCLES] = PERF_COUNT_HW_REF_CPU_CYCLES, 39 }; 40 41 static u64 kvm_pmu_get_sample_period(struct kvm_pmc *pmc) 42 { 43 u64 counter_val_mask = GENMASK(pmc->cinfo.width, 0); 44 u64 sample_period; 45 46 if (!pmc->counter_val) 47 sample_period = counter_val_mask; 48 else 49 sample_period = (-pmc->counter_val) & counter_val_mask; 50 51 return sample_period; 52 } 53 54 static u32 kvm_pmu_get_perf_event_type(unsigned long eidx) 55 { 56 enum sbi_pmu_event_type etype = get_event_type(eidx); 57 u32 type = PERF_TYPE_MAX; 58 59 switch (etype) { 60 case SBI_PMU_EVENT_TYPE_HW: 61 type = PERF_TYPE_HARDWARE; 62 break; 63 case SBI_PMU_EVENT_TYPE_CACHE: 64 type = PERF_TYPE_HW_CACHE; 65 break; 66 case SBI_PMU_EVENT_TYPE_RAW: 67 case SBI_PMU_EVENT_TYPE_RAW_V2: 68 case SBI_PMU_EVENT_TYPE_FW: 69 type = PERF_TYPE_RAW; 70 break; 71 default: 72 break; 73 } 74 75 return type; 76 } 77 78 static bool kvm_pmu_is_fw_event(unsigned long eidx) 79 { 80 return get_event_type(eidx) == SBI_PMU_EVENT_TYPE_FW; 81 } 82 83 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) 84 { 85 if (pmc->perf_event) { 86 perf_event_disable(pmc->perf_event); 87 perf_event_release_kernel(pmc->perf_event); 88 pmc->perf_event = NULL; 89 } 90 } 91 92 static u64 kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code) 93 { 94 return hw_event_perf_map[array_index_nospec(sbi_event_code, 95 SBI_PMU_HW_GENERAL_MAX)]; 96 } 97 98 static u64 kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code) 99 { 100 u64 config = U64_MAX; 101 unsigned int cache_type, cache_op, cache_result; 102 103 /* All the cache event masks lie within 0xFF. No separate masking is necessary */ 104 cache_type = (sbi_event_code & SBI_PMU_EVENT_CACHE_ID_CODE_MASK) >> 105 SBI_PMU_EVENT_CACHE_ID_SHIFT; 106 cache_op = (sbi_event_code & SBI_PMU_EVENT_CACHE_OP_ID_CODE_MASK) >> 107 SBI_PMU_EVENT_CACHE_OP_SHIFT; 108 cache_result = sbi_event_code & SBI_PMU_EVENT_CACHE_RESULT_ID_CODE_MASK; 109 110 if (cache_type >= PERF_COUNT_HW_CACHE_MAX || 111 cache_op >= PERF_COUNT_HW_CACHE_OP_MAX || 112 cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 113 return config; 114 115 config = cache_type | (cache_op << 8) | (cache_result << 16); 116 117 return config; 118 } 119 120 static u64 kvm_pmu_get_perf_event_config(unsigned long eidx, uint64_t evt_data) 121 { 122 enum sbi_pmu_event_type etype = get_event_type(eidx); 123 u32 ecode = get_event_code(eidx); 124 u64 config = U64_MAX; 125 126 switch (etype) { 127 case SBI_PMU_EVENT_TYPE_HW: 128 if (ecode < SBI_PMU_HW_GENERAL_MAX) 129 config = kvm_pmu_get_perf_event_hw_config(ecode); 130 break; 131 case SBI_PMU_EVENT_TYPE_CACHE: 132 config = kvm_pmu_get_perf_event_cache_config(ecode); 133 break; 134 case SBI_PMU_EVENT_TYPE_RAW: 135 config = evt_data & RISCV_PMU_RAW_EVENT_MASK; 136 break; 137 case SBI_PMU_EVENT_TYPE_RAW_V2: 138 config = evt_data & RISCV_PMU_RAW_EVENT_V2_MASK; 139 break; 140 case SBI_PMU_EVENT_TYPE_FW: 141 if (ecode < SBI_PMU_FW_MAX) 142 config = (1ULL << 63) | ecode; 143 break; 144 default: 145 break; 146 } 147 148 return config; 149 } 150 151 static int kvm_pmu_get_fixed_pmc_index(unsigned long eidx) 152 { 153 u32 etype = kvm_pmu_get_perf_event_type(eidx); 154 u32 ecode = get_event_code(eidx); 155 156 if (etype != SBI_PMU_EVENT_TYPE_HW) 157 return -EINVAL; 158 159 if (ecode == SBI_PMU_HW_CPU_CYCLES) 160 return 0; 161 else if (ecode == SBI_PMU_HW_INSTRUCTIONS) 162 return 2; 163 else 164 return -EINVAL; 165 } 166 167 static int kvm_pmu_get_programmable_pmc_index(struct kvm_pmu *kvpmu, unsigned long eidx, 168 unsigned long cbase, unsigned long cmask) 169 { 170 int ctr_idx = -1; 171 int i, pmc_idx; 172 int min, max; 173 174 if (kvm_pmu_is_fw_event(eidx)) { 175 /* Firmware counters are mapped 1:1 starting from num_hw_ctrs for simplicity */ 176 min = kvpmu->num_hw_ctrs; 177 max = min + kvpmu->num_fw_ctrs; 178 } else { 179 /* First 3 counters are reserved for fixed counters */ 180 min = 3; 181 max = kvpmu->num_hw_ctrs; 182 } 183 184 for_each_set_bit(i, &cmask, BITS_PER_LONG) { 185 pmc_idx = i + cbase; 186 if ((pmc_idx >= min && pmc_idx < max) && 187 !test_bit(pmc_idx, kvpmu->pmc_in_use)) { 188 ctr_idx = pmc_idx; 189 break; 190 } 191 } 192 193 return ctr_idx; 194 } 195 196 static int pmu_get_pmc_index(struct kvm_pmu *pmu, unsigned long eidx, 197 unsigned long cbase, unsigned long cmask) 198 { 199 int ret; 200 201 /* Fixed counters need to be have fixed mapping as they have different width */ 202 ret = kvm_pmu_get_fixed_pmc_index(eidx); 203 if (ret >= 0) 204 return ret; 205 206 return kvm_pmu_get_programmable_pmc_index(pmu, eidx, cbase, cmask); 207 } 208 209 static int pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx, 210 unsigned long *out_val) 211 { 212 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 213 struct kvm_pmc *pmc; 214 int fevent_code; 215 216 if (!IS_ENABLED(CONFIG_32BIT)) { 217 pr_warn("%s: should be invoked for only RV32\n", __func__); 218 return -EINVAL; 219 } 220 221 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) { 222 pr_warn("Invalid counter id [%ld]during read\n", cidx); 223 return -EINVAL; 224 } 225 226 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); 227 pmc = &kvpmu->pmc[cidx]; 228 229 if (pmc->cinfo.type != SBI_PMU_CTR_TYPE_FW) 230 return -EINVAL; 231 232 if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID) 233 return -EINVAL; 234 235 fevent_code = get_event_code(pmc->event_idx); 236 if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX, 237 "Invalid firmware event code: %d\n", fevent_code)) 238 return -EINVAL; 239 240 pmc->counter_val = kvpmu->fw_event[fevent_code].value; 241 242 *out_val = pmc->counter_val >> 32; 243 244 return 0; 245 } 246 247 static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, 248 unsigned long *out_val) 249 { 250 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 251 struct kvm_pmc *pmc; 252 u64 enabled, running; 253 int fevent_code; 254 255 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) { 256 pr_warn("Invalid counter id [%ld] during read\n", cidx); 257 return -EINVAL; 258 } 259 260 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); 261 pmc = &kvpmu->pmc[cidx]; 262 263 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { 264 if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID) 265 return -EINVAL; 266 267 fevent_code = get_event_code(pmc->event_idx); 268 if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX, 269 "Invalid firmware event code: %d\n", fevent_code)) 270 return -EINVAL; 271 272 pmc->counter_val = kvpmu->fw_event[fevent_code].value; 273 } else if (pmc->perf_event) { 274 pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); 275 } else { 276 return -EINVAL; 277 } 278 *out_val = pmc->counter_val; 279 280 return 0; 281 } 282 283 static int kvm_pmu_validate_counter_mask(struct kvm_pmu *kvpmu, unsigned long ctr_base, 284 unsigned long ctr_mask) 285 { 286 unsigned long num_ctrs = kvm_pmu_num_counters(kvpmu); 287 288 /* Make sure we have a valid counter mask requested from the caller */ 289 if (!ctr_mask || ctr_base >= num_ctrs || (ctr_base + __fls(ctr_mask) >= num_ctrs)) 290 return -EINVAL; 291 292 return 0; 293 } 294 295 static void kvm_riscv_pmu_overflow(struct perf_event *perf_event, 296 struct perf_sample_data *data, 297 struct pt_regs *regs) 298 { 299 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 300 struct kvm_vcpu *vcpu = pmc->vcpu; 301 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 302 struct riscv_pmu *rpmu = to_riscv_pmu(perf_event->pmu); 303 u64 period; 304 305 /* 306 * Stop the event counting by directly accessing the perf_event. 307 * Otherwise, this needs to deferred via a workqueue. 308 * That will introduce skew in the counter value because the actual 309 * physical counter would start after returning from this function. 310 * It will be stopped again once the workqueue is scheduled 311 */ 312 rpmu->pmu.stop(perf_event, PERF_EF_UPDATE); 313 314 /* 315 * The hw counter would start automatically when this function returns. 316 * Thus, the host may continue to interrupt and inject it to the guest 317 * even without the guest configuring the next event. Depending on the hardware 318 * the host may have some sluggishness only if privilege mode filtering is not 319 * available. In an ideal world, where qemu is not the only capable hardware, 320 * this can be removed. 321 * FYI: ARM64 does this way while x86 doesn't do anything as such. 322 * TODO: Should we keep it for RISC-V ? 323 */ 324 period = -(local64_read(&perf_event->count)); 325 326 local64_set(&perf_event->hw.period_left, 0); 327 perf_event->attr.sample_period = period; 328 perf_event->hw.sample_period = period; 329 330 set_bit(pmc->idx, kvpmu->pmc_overflown); 331 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_PMU_OVF); 332 333 rpmu->pmu.start(perf_event, PERF_EF_RELOAD); 334 } 335 336 static long kvm_pmu_create_perf_event(struct kvm_pmc *pmc, struct perf_event_attr *attr, 337 unsigned long flags, unsigned long eidx, 338 unsigned long evtdata) 339 { 340 struct perf_event *event; 341 342 kvm_pmu_release_perf_event(pmc); 343 attr->config = kvm_pmu_get_perf_event_config(eidx, evtdata); 344 if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE) { 345 //TODO: Do we really want to clear the value in hardware counter 346 pmc->counter_val = 0; 347 } 348 349 /* 350 * Set the default sample_period for now. The guest specified value 351 * will be updated in the start call. 352 */ 353 attr->sample_period = kvm_pmu_get_sample_period(pmc); 354 355 event = perf_event_create_kernel_counter(attr, -1, current, kvm_riscv_pmu_overflow, pmc); 356 if (IS_ERR(event)) { 357 pr_debug("kvm pmu event creation failed for eidx %lx: %ld\n", eidx, PTR_ERR(event)); 358 return PTR_ERR(event); 359 } 360 361 pmc->perf_event = event; 362 if (flags & SBI_PMU_CFG_FLAG_AUTO_START) 363 perf_event_enable(pmc->perf_event); 364 365 return 0; 366 } 367 368 int kvm_riscv_vcpu_pmu_incr_fw(struct kvm_vcpu *vcpu, unsigned long fid) 369 { 370 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 371 struct kvm_fw_event *fevent; 372 373 if (!kvpmu || fid >= SBI_PMU_FW_MAX) 374 return -EINVAL; 375 376 fevent = &kvpmu->fw_event[fid]; 377 if (fevent->started) 378 fevent->value++; 379 380 return 0; 381 } 382 383 int kvm_riscv_vcpu_pmu_read_hpm(struct kvm_vcpu *vcpu, unsigned int csr_num, 384 unsigned long *val, unsigned long new_val, 385 unsigned long wr_mask) 386 { 387 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 388 int cidx, ret = KVM_INSN_CONTINUE_NEXT_SEPC; 389 390 if (!kvpmu || !kvpmu->init_done) { 391 /* 392 * In absence of sscofpmf in the platform, the guest OS may use 393 * the legacy PMU driver to read cycle/instret. In that case, 394 * just return 0 to avoid any illegal trap. However, any other 395 * hpmcounter access should result in illegal trap as they must 396 * be access through SBI PMU only. 397 */ 398 if (csr_num == CSR_CYCLE || csr_num == CSR_INSTRET) { 399 *val = 0; 400 return ret; 401 } else { 402 return KVM_INSN_ILLEGAL_TRAP; 403 } 404 } 405 406 /* The counter CSR are read only. Thus, any write should result in illegal traps */ 407 if (wr_mask) 408 return KVM_INSN_ILLEGAL_TRAP; 409 410 cidx = csr_num - CSR_CYCLE; 411 412 if (pmu_ctr_read(vcpu, cidx, val) < 0) 413 return KVM_INSN_ILLEGAL_TRAP; 414 415 return ret; 416 } 417 418 static void kvm_pmu_clear_snapshot_area(struct kvm_vcpu *vcpu) 419 { 420 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 421 422 kfree(kvpmu->sdata); 423 kvpmu->sdata = NULL; 424 kvpmu->snapshot_addr = INVALID_GPA; 425 } 426 427 int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long saddr_low, 428 unsigned long saddr_high, unsigned long flags, 429 struct kvm_vcpu_sbi_return *retdata) 430 { 431 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 432 int snapshot_area_size = sizeof(struct riscv_pmu_snapshot_data); 433 int sbiret = 0; 434 gpa_t saddr; 435 436 if (!kvpmu || flags) { 437 sbiret = SBI_ERR_INVALID_PARAM; 438 goto out; 439 } 440 441 if (saddr_low == SBI_SHMEM_DISABLE && saddr_high == SBI_SHMEM_DISABLE) { 442 kvm_pmu_clear_snapshot_area(vcpu); 443 return 0; 444 } 445 446 saddr = saddr_low; 447 448 if (saddr_high != 0) { 449 if (IS_ENABLED(CONFIG_32BIT)) { 450 saddr |= ((gpa_t)saddr_high << 32); 451 } else { 452 sbiret = SBI_ERR_INVALID_ADDRESS; 453 goto out; 454 } 455 } 456 457 kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC | __GFP_ACCOUNT); 458 if (!kvpmu->sdata) { 459 sbiret = SBI_ERR_FAILURE; 460 goto out; 461 } 462 463 /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */ 464 if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) { 465 kfree(kvpmu->sdata); 466 kvpmu->sdata = NULL; 467 sbiret = SBI_ERR_INVALID_ADDRESS; 468 goto out; 469 } 470 471 kvpmu->snapshot_addr = saddr; 472 473 out: 474 retdata->err_val = sbiret; 475 476 return 0; 477 } 478 479 int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low, 480 unsigned long saddr_high, unsigned long num_events, 481 unsigned long flags, struct kvm_vcpu_sbi_return *retdata) 482 { 483 struct riscv_pmu_event_info *einfo = NULL; 484 size_t shmem_size; 485 gpa_t shmem; 486 u32 eidx, etype; 487 u64 econfig; 488 int ret; 489 490 if (flags != 0 || (saddr_low & (SZ_16 - 1)) || num_events == 0 || 491 check_mul_overflow(num_events, sizeof(*einfo), &shmem_size)) { 492 ret = SBI_ERR_INVALID_PARAM; 493 goto out; 494 } 495 496 shmem = saddr_low; 497 if (saddr_high != 0) { 498 if (IS_ENABLED(CONFIG_32BIT)) { 499 shmem |= ((gpa_t)saddr_high << 32); 500 } else { 501 ret = SBI_ERR_INVALID_ADDRESS; 502 goto out; 503 } 504 } 505 506 einfo = kvzalloc_objs(*einfo, num_events, 507 GFP_KERNEL_ACCOUNT | __GFP_NOWARN); 508 if (!einfo) { 509 ret = SBI_ERR_FAILURE; 510 goto out; 511 } 512 513 ret = kvm_vcpu_read_guest(vcpu, shmem, einfo, shmem_size); 514 if (ret) { 515 ret = SBI_ERR_FAILURE; 516 goto free_mem; 517 } 518 519 for (unsigned long i = 0; i < num_events; i++) { 520 eidx = einfo[i].event_idx; 521 etype = kvm_pmu_get_perf_event_type(eidx); 522 econfig = kvm_pmu_get_perf_event_config(eidx, einfo[i].event_data); 523 ret = riscv_pmu_get_event_info(etype, econfig, NULL); 524 einfo[i].output = (ret > 0) ? 1 : 0; 525 } 526 527 ret = kvm_vcpu_write_guest(vcpu, shmem, einfo, shmem_size); 528 if (ret) 529 ret = SBI_ERR_INVALID_ADDRESS; 530 531 free_mem: 532 kvfree(einfo); 533 out: 534 retdata->err_val = ret; 535 536 return 0; 537 } 538 539 int kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu *vcpu, 540 struct kvm_vcpu_sbi_return *retdata) 541 { 542 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 543 544 retdata->out_val = kvm_pmu_num_counters(kvpmu); 545 546 return 0; 547 } 548 549 int kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu *vcpu, unsigned long cidx, 550 struct kvm_vcpu_sbi_return *retdata) 551 { 552 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 553 554 if (cidx >= RISCV_KVM_MAX_COUNTERS || cidx == 1) { 555 retdata->err_val = SBI_ERR_INVALID_PARAM; 556 return 0; 557 } 558 559 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); 560 retdata->out_val = kvpmu->pmc[cidx].cinfo.value; 561 562 return 0; 563 } 564 565 int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base, 566 unsigned long ctr_mask, unsigned long flags, u64 ival, 567 struct kvm_vcpu_sbi_return *retdata) 568 { 569 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 570 int i, pmc_index, sbiret = 0; 571 struct kvm_pmc *pmc; 572 int fevent_code; 573 bool snap_flag_set = flags & SBI_PMU_START_FLAG_INIT_SNAPSHOT; 574 575 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) { 576 sbiret = SBI_ERR_INVALID_PARAM; 577 goto out; 578 } 579 580 if (snap_flag_set) { 581 if (kvpmu->snapshot_addr == INVALID_GPA) { 582 sbiret = SBI_ERR_NO_SHMEM; 583 goto out; 584 } 585 if (kvm_vcpu_read_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, 586 sizeof(struct riscv_pmu_snapshot_data))) { 587 pr_warn("Unable to read snapshot shared memory while starting counters\n"); 588 sbiret = SBI_ERR_FAILURE; 589 goto out; 590 } 591 } 592 /* Start the counters that have been configured and requested by the guest */ 593 for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { 594 pmc_index = array_index_nospec(i + ctr_base, 595 RISCV_KVM_MAX_COUNTERS); 596 if (!test_bit(pmc_index, kvpmu->pmc_in_use)) 597 continue; 598 /* The guest started the counter again. Reset the overflow status */ 599 clear_bit(pmc_index, kvpmu->pmc_overflown); 600 pmc = &kvpmu->pmc[pmc_index]; 601 if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE) { 602 pmc->counter_val = ival; 603 } else if (snap_flag_set) { 604 /* The counter index in the snapshot are relative to the counter base */ 605 pmc->counter_val = kvpmu->sdata->ctr_values[i]; 606 } 607 608 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { 609 fevent_code = get_event_code(pmc->event_idx); 610 if (fevent_code >= SBI_PMU_FW_MAX) { 611 sbiret = SBI_ERR_INVALID_PARAM; 612 goto out; 613 } 614 615 /* Check if the counter was already started for some reason */ 616 if (kvpmu->fw_event[fevent_code].started) { 617 sbiret = SBI_ERR_ALREADY_STARTED; 618 continue; 619 } 620 621 kvpmu->fw_event[fevent_code].started = true; 622 kvpmu->fw_event[fevent_code].value = pmc->counter_val; 623 } else if (pmc->perf_event) { 624 if (unlikely(pmc->started)) { 625 sbiret = SBI_ERR_ALREADY_STARTED; 626 continue; 627 } 628 perf_event_period(pmc->perf_event, kvm_pmu_get_sample_period(pmc)); 629 perf_event_enable(pmc->perf_event); 630 pmc->started = true; 631 } else { 632 sbiret = SBI_ERR_INVALID_PARAM; 633 } 634 } 635 636 out: 637 retdata->err_val = sbiret; 638 639 return 0; 640 } 641 642 int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, 643 unsigned long ctr_mask, unsigned long flags, 644 struct kvm_vcpu_sbi_return *retdata) 645 { 646 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 647 int i, pmc_index, sbiret = 0; 648 u64 enabled, running; 649 struct kvm_pmc *pmc; 650 int fevent_code; 651 bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; 652 bool shmem_needs_update = false; 653 654 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) { 655 sbiret = SBI_ERR_INVALID_PARAM; 656 goto out; 657 } 658 659 if (snap_flag_set && kvpmu->snapshot_addr == INVALID_GPA) { 660 sbiret = SBI_ERR_NO_SHMEM; 661 goto out; 662 } 663 664 /* Stop the counters that have been configured and requested by the guest */ 665 for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { 666 pmc_index = array_index_nospec(i + ctr_base, 667 RISCV_KVM_MAX_COUNTERS); 668 if (!test_bit(pmc_index, kvpmu->pmc_in_use)) 669 continue; 670 pmc = &kvpmu->pmc[pmc_index]; 671 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { 672 fevent_code = get_event_code(pmc->event_idx); 673 if (fevent_code >= SBI_PMU_FW_MAX) { 674 sbiret = SBI_ERR_INVALID_PARAM; 675 goto out; 676 } 677 678 if (!kvpmu->fw_event[fevent_code].started) 679 sbiret = SBI_ERR_ALREADY_STOPPED; 680 681 kvpmu->fw_event[fevent_code].started = false; 682 } else if (pmc->perf_event) { 683 if (pmc->started) { 684 /* Stop counting the counter */ 685 perf_event_disable(pmc->perf_event); 686 pmc->started = false; 687 } else { 688 sbiret = SBI_ERR_ALREADY_STOPPED; 689 } 690 691 if (flags & SBI_PMU_STOP_FLAG_RESET) 692 /* Release the counter if this is a reset request */ 693 kvm_pmu_release_perf_event(pmc); 694 } else { 695 sbiret = SBI_ERR_INVALID_PARAM; 696 } 697 698 if (snap_flag_set && !sbiret) { 699 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) 700 pmc->counter_val = kvpmu->fw_event[fevent_code].value; 701 else if (pmc->perf_event) 702 pmc->counter_val += perf_event_read_value(pmc->perf_event, 703 &enabled, &running); 704 /* 705 * The counter and overflow indices in the snapshot region are w.r.to 706 * cbase. Modify the set bit in the counter mask instead of the pmc_index 707 * which indicates the absolute counter index. 708 */ 709 if (test_bit(pmc_index, kvpmu->pmc_overflown)) 710 kvpmu->sdata->ctr_overflow_mask |= BIT(i); 711 kvpmu->sdata->ctr_values[i] = pmc->counter_val; 712 shmem_needs_update = true; 713 } 714 715 if (flags & SBI_PMU_STOP_FLAG_RESET) { 716 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID; 717 clear_bit(pmc_index, kvpmu->pmc_in_use); 718 clear_bit(pmc_index, kvpmu->pmc_overflown); 719 if (snap_flag_set) { 720 /* 721 * Only clear the given counter as the caller is responsible to 722 * validate both the overflow mask and configured counters. 723 */ 724 kvpmu->sdata->ctr_overflow_mask &= ~BIT(i); 725 shmem_needs_update = true; 726 } 727 } 728 } 729 730 if (shmem_needs_update) 731 kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, 732 sizeof(struct riscv_pmu_snapshot_data)); 733 734 out: 735 retdata->err_val = sbiret; 736 737 return 0; 738 } 739 740 int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_base, 741 unsigned long ctr_mask, unsigned long flags, 742 unsigned long eidx, u64 evtdata, 743 struct kvm_vcpu_sbi_return *retdata) 744 { 745 int ctr_idx, sbiret = 0; 746 long ret; 747 bool is_fevent; 748 unsigned long event_code; 749 u32 etype = kvm_pmu_get_perf_event_type(eidx); 750 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 751 struct kvm_pmc *pmc = NULL; 752 struct perf_event_attr attr = { 753 .type = etype, 754 .size = sizeof(struct perf_event_attr), 755 .pinned = true, 756 .disabled = true, 757 /* 758 * It should never reach here if the platform doesn't support the sscofpmf 759 * extension as mode filtering won't work without it. 760 */ 761 .exclude_host = true, 762 .exclude_hv = true, 763 .exclude_user = !!(flags & SBI_PMU_CFG_FLAG_SET_UINH), 764 .exclude_kernel = !!(flags & SBI_PMU_CFG_FLAG_SET_SINH), 765 .config1 = RISCV_PMU_CONFIG1_GUEST_EVENTS, 766 }; 767 768 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) { 769 sbiret = SBI_ERR_INVALID_PARAM; 770 goto out; 771 } 772 773 event_code = get_event_code(eidx); 774 is_fevent = kvm_pmu_is_fw_event(eidx); 775 if (is_fevent && event_code >= SBI_PMU_FW_MAX) { 776 sbiret = SBI_ERR_NOT_SUPPORTED; 777 goto out; 778 } 779 780 /* 781 * SKIP_MATCH flag indicates the caller is aware of the assigned counter 782 * for this event. Just do a sanity check if it already marked used. 783 */ 784 if (flags & SBI_PMU_CFG_FLAG_SKIP_MATCH) { 785 if (!test_bit(ctr_base + __ffs(ctr_mask), kvpmu->pmc_in_use)) { 786 sbiret = SBI_ERR_FAILURE; 787 goto out; 788 } 789 ctr_idx = ctr_base + __ffs(ctr_mask); 790 } else { 791 ctr_idx = pmu_get_pmc_index(kvpmu, eidx, ctr_base, ctr_mask); 792 if (ctr_idx < 0) { 793 sbiret = SBI_ERR_NOT_SUPPORTED; 794 goto out; 795 } 796 } 797 798 ctr_idx = array_index_nospec(ctr_idx, RISCV_KVM_MAX_COUNTERS); 799 pmc = &kvpmu->pmc[ctr_idx]; 800 pmc->idx = ctr_idx; 801 802 if (is_fevent) { 803 if (flags & SBI_PMU_CFG_FLAG_AUTO_START) 804 kvpmu->fw_event[event_code].started = true; 805 } else { 806 ret = kvm_pmu_create_perf_event(pmc, &attr, flags, eidx, evtdata); 807 if (ret) { 808 sbiret = SBI_ERR_NOT_SUPPORTED; 809 goto out; 810 } 811 } 812 813 set_bit(ctr_idx, kvpmu->pmc_in_use); 814 pmc->event_idx = eidx; 815 retdata->out_val = ctr_idx; 816 out: 817 retdata->err_val = sbiret; 818 819 return 0; 820 } 821 822 int kvm_riscv_vcpu_pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx, 823 struct kvm_vcpu_sbi_return *retdata) 824 { 825 int ret; 826 827 ret = pmu_fw_ctr_read_hi(vcpu, cidx, &retdata->out_val); 828 if (ret == -EINVAL) 829 retdata->err_val = SBI_ERR_INVALID_PARAM; 830 831 return 0; 832 } 833 834 int kvm_riscv_vcpu_pmu_fw_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, 835 struct kvm_vcpu_sbi_return *retdata) 836 { 837 int ret; 838 839 ret = pmu_ctr_read(vcpu, cidx, &retdata->out_val); 840 if (ret == -EINVAL) 841 retdata->err_val = SBI_ERR_INVALID_PARAM; 842 843 return 0; 844 } 845 846 void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu) 847 { 848 int i = 0, ret, num_hw_ctrs = 0, hpm_width = 0; 849 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 850 struct kvm_pmc *pmc; 851 852 /* 853 * PMU functionality should be only available to guests if privilege mode 854 * filtering is available in the host. Otherwise, guest will always count 855 * events while the execution is in hypervisor mode. 856 */ 857 if (kvm_riscv_isa_check_host(SSCOFPMF)) 858 return; 859 860 ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs); 861 if (ret < 0 || !hpm_width || !num_hw_ctrs) 862 return; 863 864 /* 865 * Increase the number of hardware counters to offset the time counter. 866 */ 867 kvpmu->num_hw_ctrs = num_hw_ctrs + 1; 868 kvpmu->num_fw_ctrs = SBI_PMU_FW_MAX; 869 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event)); 870 kvpmu->snapshot_addr = INVALID_GPA; 871 872 if (kvpmu->num_hw_ctrs > RISCV_KVM_MAX_HW_CTRS) { 873 pr_warn_once("Limiting the hardware counters to 32 as specified by the ISA"); 874 kvpmu->num_hw_ctrs = RISCV_KVM_MAX_HW_CTRS; 875 } 876 877 /* 878 * There is no correlation between the logical hardware counter and virtual counters. 879 * However, we need to encode a hpmcounter CSR in the counter info field so that 880 * KVM can trap n emulate the read. This works well in the migration use case as 881 * KVM doesn't care if the actual hpmcounter is available in the hardware or not. 882 */ 883 for (i = 0; i < kvm_pmu_num_counters(kvpmu); i++) { 884 /* TIME CSR shouldn't be read from perf interface */ 885 if (i == 1) 886 continue; 887 pmc = &kvpmu->pmc[i]; 888 pmc->idx = i; 889 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID; 890 pmc->vcpu = vcpu; 891 if (i < kvpmu->num_hw_ctrs) { 892 pmc->cinfo.type = SBI_PMU_CTR_TYPE_HW; 893 if (i < 3) 894 /* CY, IR counters */ 895 pmc->cinfo.width = 63; 896 else 897 pmc->cinfo.width = hpm_width; 898 /* 899 * The CSR number doesn't have any relation with the logical 900 * hardware counters. The CSR numbers are encoded sequentially 901 * to avoid maintaining a map between the virtual counter 902 * and CSR number. 903 */ 904 pmc->cinfo.csr = CSR_CYCLE + i; 905 } else { 906 pmc->cinfo.type = SBI_PMU_CTR_TYPE_FW; 907 pmc->cinfo.width = 63; 908 } 909 } 910 911 kvpmu->init_done = true; 912 } 913 914 void kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu *vcpu) 915 { 916 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); 917 struct kvm_pmc *pmc; 918 int i; 919 920 if (!kvpmu) 921 return; 922 923 for_each_set_bit(i, kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS) { 924 pmc = &kvpmu->pmc[i]; 925 pmc->counter_val = 0; 926 kvm_pmu_release_perf_event(pmc); 927 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID; 928 } 929 bitmap_zero(kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS); 930 bitmap_zero(kvpmu->pmc_overflown, RISCV_KVM_MAX_COUNTERS); 931 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event)); 932 kvm_pmu_clear_snapshot_area(vcpu); 933 } 934 935 void kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu *vcpu) 936 { 937 kvm_riscv_vcpu_pmu_deinit(vcpu); 938 } 939