1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * RISC-V performance counter support. 4 * 5 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 6 * 7 * This code is based on ARM perf event code which is in turn based on 8 * sparc64 and x86 code. 9 */ 10 11 #define pr_fmt(fmt) "riscv-pmu-sbi: " fmt 12 13 #include <linux/perf/riscv_pmu.h> 14 #include <linux/platform_device.h> 15 #include <linux/irq.h> 16 #include <linux/irqdomain.h> 17 #include <linux/of_irq.h> 18 #include <linux/of.h> 19 #include <linux/cpu_pm.h> 20 #include <linux/sched/clock.h> 21 #include <linux/soc/andes/irq.h> 22 #include <linux/workqueue.h> 23 24 #include <asm/errata_list.h> 25 #include <asm/sbi.h> 26 #include <asm/cpufeature.h> 27 #include <asm/vendor_extensions.h> 28 #include <asm/vendor_extensions/andes.h> 29 30 #define ALT_SBI_PMU_OVERFLOW(__ovl) \ 31 asm volatile(ALTERNATIVE_2( \ 32 "csrr %0, " __stringify(CSR_SCOUNTOVF), \ 33 "csrr %0, " __stringify(THEAD_C9XX_CSR_SCOUNTEROF), \ 34 THEAD_VENDOR_ID, ERRATA_THEAD_PMU, \ 35 CONFIG_ERRATA_THEAD_PMU, \ 36 "csrr %0, " __stringify(ANDES_CSR_SCOUNTEROF), \ 37 ANDES_VENDOR_ID, \ 38 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \ 39 CONFIG_ANDES_CUSTOM_PMU) \ 40 : "=r" (__ovl) : \ 41 : "memory") 42 43 #define ALT_SBI_PMU_OVF_CLEAR_PENDING(__irq_mask) \ 44 asm volatile(ALTERNATIVE( \ 45 "csrc " __stringify(CSR_IP) ", %0\n\t", \ 46 "csrc " __stringify(ANDES_CSR_SLIP) ", %0\n\t", \ 47 ANDES_VENDOR_ID, \ 48 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \ 49 CONFIG_ANDES_CUSTOM_PMU) \ 50 : : "r"(__irq_mask) \ 51 : "memory") 52 53 #define SYSCTL_NO_USER_ACCESS 0 54 #define SYSCTL_USER_ACCESS 1 55 #define SYSCTL_LEGACY 2 56 57 #define PERF_EVENT_FLAG_NO_USER_ACCESS BIT(SYSCTL_NO_USER_ACCESS) 58 #define PERF_EVENT_FLAG_USER_ACCESS BIT(SYSCTL_USER_ACCESS) 59 #define PERF_EVENT_FLAG_LEGACY BIT(SYSCTL_LEGACY) 60 61 PMU_FORMAT_ATTR(event, "config:0-55"); 62 PMU_FORMAT_ATTR(firmware, "config:62-63"); 63 64 static bool sbi_v2_available; 65 static bool sbi_v3_available; 66 static DEFINE_STATIC_KEY_FALSE(sbi_pmu_snapshot_available); 67 #define sbi_pmu_snapshot_available() \ 68 static_branch_unlikely(&sbi_pmu_snapshot_available) 69 70 static struct attribute *riscv_arch_formats_attr[] = { 71 &format_attr_event.attr, 72 &format_attr_firmware.attr, 73 NULL, 74 }; 75 76 static struct attribute_group riscv_pmu_format_group = { 77 .name = "format", 78 .attrs = riscv_arch_formats_attr, 79 }; 80 81 static const struct attribute_group *riscv_pmu_attr_groups[] = { 82 &riscv_pmu_format_group, 83 NULL, 84 }; 85 86 /* Allow user mode access by default */ 87 static int sysctl_perf_user_access __read_mostly = SYSCTL_USER_ACCESS; 88 89 /* 90 * RISC-V doesn't have heterogeneous harts yet. This need to be part of 91 * per_cpu in case of harts with different pmu counters 92 */ 93 static union sbi_pmu_ctr_info *pmu_ctr_list; 94 static bool riscv_pmu_use_irq; 95 static unsigned int riscv_pmu_irq_num; 96 static unsigned int riscv_pmu_irq_mask; 97 static unsigned int riscv_pmu_irq; 98 99 /* Cache the available counters in a bitmask */ 100 static DECLARE_BITMAP(cmask, RISCV_MAX_COUNTERS); 101 102 static int pmu_event_find_cache(u64 config); 103 struct sbi_pmu_event_data { 104 union { 105 union { 106 struct hw_gen_event { 107 uint32_t event_code:16; 108 uint32_t event_type:4; 109 uint32_t reserved:12; 110 } hw_gen_event; 111 struct hw_cache_event { 112 uint32_t result_id:1; 113 uint32_t op_id:2; 114 uint32_t cache_id:13; 115 uint32_t event_type:4; 116 uint32_t reserved:12; 117 } hw_cache_event; 118 }; 119 uint32_t event_idx; 120 }; 121 }; 122 123 static struct sbi_pmu_event_data pmu_hw_event_map[] = { 124 [PERF_COUNT_HW_CPU_CYCLES] = {.hw_gen_event = { 125 SBI_PMU_HW_CPU_CYCLES, 126 SBI_PMU_EVENT_TYPE_HW, 0}}, 127 [PERF_COUNT_HW_INSTRUCTIONS] = {.hw_gen_event = { 128 SBI_PMU_HW_INSTRUCTIONS, 129 SBI_PMU_EVENT_TYPE_HW, 0}}, 130 [PERF_COUNT_HW_CACHE_REFERENCES] = {.hw_gen_event = { 131 SBI_PMU_HW_CACHE_REFERENCES, 132 SBI_PMU_EVENT_TYPE_HW, 0}}, 133 [PERF_COUNT_HW_CACHE_MISSES] = {.hw_gen_event = { 134 SBI_PMU_HW_CACHE_MISSES, 135 SBI_PMU_EVENT_TYPE_HW, 0}}, 136 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {.hw_gen_event = { 137 SBI_PMU_HW_BRANCH_INSTRUCTIONS, 138 SBI_PMU_EVENT_TYPE_HW, 0}}, 139 [PERF_COUNT_HW_BRANCH_MISSES] = {.hw_gen_event = { 140 SBI_PMU_HW_BRANCH_MISSES, 141 SBI_PMU_EVENT_TYPE_HW, 0}}, 142 [PERF_COUNT_HW_BUS_CYCLES] = {.hw_gen_event = { 143 SBI_PMU_HW_BUS_CYCLES, 144 SBI_PMU_EVENT_TYPE_HW, 0}}, 145 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = {.hw_gen_event = { 146 SBI_PMU_HW_STALLED_CYCLES_FRONTEND, 147 SBI_PMU_EVENT_TYPE_HW, 0}}, 148 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = {.hw_gen_event = { 149 SBI_PMU_HW_STALLED_CYCLES_BACKEND, 150 SBI_PMU_EVENT_TYPE_HW, 0}}, 151 [PERF_COUNT_HW_REF_CPU_CYCLES] = {.hw_gen_event = { 152 SBI_PMU_HW_REF_CPU_CYCLES, 153 SBI_PMU_EVENT_TYPE_HW, 0}}, 154 }; 155 156 #define C(x) PERF_COUNT_HW_CACHE_##x 157 static struct sbi_pmu_event_data pmu_cache_event_map[PERF_COUNT_HW_CACHE_MAX] 158 [PERF_COUNT_HW_CACHE_OP_MAX] 159 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 160 [C(L1D)] = { 161 [C(OP_READ)] = { 162 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 163 C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 164 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 165 C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 166 }, 167 [C(OP_WRITE)] = { 168 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 169 C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 170 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 171 C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 172 }, 173 [C(OP_PREFETCH)] = { 174 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 175 C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 176 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 177 C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 178 }, 179 }, 180 [C(L1I)] = { 181 [C(OP_READ)] = { 182 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 183 C(OP_READ), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 184 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), C(OP_READ), 185 C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 186 }, 187 [C(OP_WRITE)] = { 188 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 189 C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 190 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 191 C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 192 }, 193 [C(OP_PREFETCH)] = { 194 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 195 C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 196 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 197 C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 198 }, 199 }, 200 [C(LL)] = { 201 [C(OP_READ)] = { 202 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 203 C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 204 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 205 C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 206 }, 207 [C(OP_WRITE)] = { 208 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 209 C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 210 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 211 C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 212 }, 213 [C(OP_PREFETCH)] = { 214 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 215 C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 216 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 217 C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 218 }, 219 }, 220 [C(DTLB)] = { 221 [C(OP_READ)] = { 222 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 223 C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 224 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 225 C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 226 }, 227 [C(OP_WRITE)] = { 228 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 229 C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 230 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 231 C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 232 }, 233 [C(OP_PREFETCH)] = { 234 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 235 C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 236 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 237 C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 238 }, 239 }, 240 [C(ITLB)] = { 241 [C(OP_READ)] = { 242 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 243 C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 244 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 245 C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 246 }, 247 [C(OP_WRITE)] = { 248 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 249 C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 250 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 251 C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 252 }, 253 [C(OP_PREFETCH)] = { 254 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 255 C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 256 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 257 C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 258 }, 259 }, 260 [C(BPU)] = { 261 [C(OP_READ)] = { 262 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 263 C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 264 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 265 C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 266 }, 267 [C(OP_WRITE)] = { 268 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 269 C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 270 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 271 C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 272 }, 273 [C(OP_PREFETCH)] = { 274 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 275 C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 276 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 277 C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 278 }, 279 }, 280 [C(NODE)] = { 281 [C(OP_READ)] = { 282 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 283 C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 284 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 285 C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 286 }, 287 [C(OP_WRITE)] = { 288 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 289 C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 290 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 291 C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 292 }, 293 [C(OP_PREFETCH)] = { 294 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS), 295 C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 296 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), 297 C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}}, 298 }, 299 }, 300 }; 301 302 static int pmu_sbi_check_event_info(void) 303 { 304 int num_events = ARRAY_SIZE(pmu_hw_event_map) + PERF_COUNT_HW_CACHE_MAX * 305 PERF_COUNT_HW_CACHE_OP_MAX * PERF_COUNT_HW_CACHE_RESULT_MAX; 306 struct riscv_pmu_event_info *event_info_shmem; 307 phys_addr_t base_addr; 308 int i, j, k, result = 0, count = 0; 309 struct sbiret ret; 310 311 event_info_shmem = kzalloc_objs(*event_info_shmem, num_events); 312 if (!event_info_shmem) 313 return -ENOMEM; 314 315 for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) 316 event_info_shmem[count++].event_idx = pmu_hw_event_map[i].event_idx; 317 318 for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) { 319 for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) { 320 for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++) 321 event_info_shmem[count++].event_idx = 322 pmu_cache_event_map[i][j][k].event_idx; 323 } 324 } 325 326 base_addr = __pa(event_info_shmem); 327 if (IS_ENABLED(CONFIG_32BIT)) 328 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, lower_32_bits(base_addr), 329 upper_32_bits(base_addr), count, 0, 0, 0); 330 else 331 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, base_addr, 0, 332 count, 0, 0, 0); 333 if (ret.error) { 334 result = -EOPNOTSUPP; 335 goto free_mem; 336 } 337 338 for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) { 339 if (!(event_info_shmem[i].output & RISCV_PMU_EVENT_INFO_OUTPUT_MASK)) 340 pmu_hw_event_map[i].event_idx = -ENOENT; 341 } 342 343 count = ARRAY_SIZE(pmu_hw_event_map); 344 345 for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) { 346 for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) { 347 for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++) { 348 if (!(event_info_shmem[count].output & 349 RISCV_PMU_EVENT_INFO_OUTPUT_MASK)) 350 pmu_cache_event_map[i][j][k].event_idx = -ENOENT; 351 count++; 352 } 353 } 354 } 355 356 free_mem: 357 kfree(event_info_shmem); 358 359 return result; 360 } 361 362 static struct sbiret pmu_sbi_ctr_cfg_match(unsigned long cbase, 363 unsigned long ctr_mask, 364 unsigned long cflags, 365 unsigned long event_idx, 366 u64 config) 367 { 368 #if defined(CONFIG_32BIT) 369 return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, 370 ctr_mask, cflags, event_idx, config, config >> 32); 371 #else 372 return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, 373 ctr_mask, cflags, event_idx, config, 0); 374 #endif 375 } 376 377 static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata) 378 { 379 struct sbiret ret = { .error = SBI_ERR_NOT_SUPPORTED }; 380 int i; 381 382 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { 383 if (!cmask[i]) 384 continue; 385 ret = pmu_sbi_ctr_cfg_match(i * BITS_PER_LONG, cmask[i], 0, 386 edata->event_idx, 0); 387 if (!ret.error) { 388 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, 389 ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); 390 return; 391 } 392 } 393 if (ret.error == SBI_ERR_NOT_SUPPORTED) { 394 /* This event cannot be monitored by any counter */ 395 edata->event_idx = -ENOENT; 396 } 397 } 398 399 static void pmu_sbi_check_std_events(struct work_struct *work) 400 { 401 int ret; 402 403 if (sbi_v3_available) { 404 ret = pmu_sbi_check_event_info(); 405 if (ret) 406 pr_err("pmu_sbi_check_event_info failed with error %d\n", ret); 407 return; 408 } 409 410 for (int i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) 411 pmu_sbi_check_event(&pmu_hw_event_map[i]); 412 413 for (int i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) 414 for (int j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) 415 for (int k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++) 416 pmu_sbi_check_event(&pmu_cache_event_map[i][j][k]); 417 } 418 419 static DECLARE_WORK(check_std_events_work, pmu_sbi_check_std_events); 420 421 static int pmu_sbi_ctr_get_width(int idx) 422 { 423 return pmu_ctr_list[idx].width; 424 } 425 426 static bool pmu_sbi_ctr_is_fw(int cidx) 427 { 428 union sbi_pmu_ctr_info *info; 429 430 info = &pmu_ctr_list[cidx]; 431 if (!info) 432 return false; 433 434 return info->type == SBI_PMU_CTR_TYPE_FW; 435 } 436 437 int riscv_pmu_get_event_info(u32 type, u64 config, u64 *econfig) 438 { 439 int ret = -ENOENT; 440 441 switch (type) { 442 case PERF_TYPE_HARDWARE: 443 if (config >= PERF_COUNT_HW_MAX) 444 return -EINVAL; 445 ret = pmu_hw_event_map[config].event_idx; 446 break; 447 case PERF_TYPE_HW_CACHE: 448 ret = pmu_event_find_cache(config); 449 break; 450 case PERF_TYPE_RAW: 451 /* 452 * As per SBI v0.3 specification, 453 * -- the upper 16 bits must be unused for a hardware raw event. 454 * As per SBI v2.0 specification, 455 * -- the upper 8 bits must be unused for a hardware raw event. 456 * Bits 63:62 are used to distinguish between raw events 457 * 00 - Hardware raw event 458 * 10 - SBI firmware events 459 * 11 - Risc-V platform specific firmware event 460 */ 461 switch (config >> 62) { 462 case 0: 463 if (sbi_v3_available) { 464 /* Return error any bits [56-63] is set as it is not allowed by the spec */ 465 if (!(config & ~RISCV_PMU_RAW_EVENT_V2_MASK)) { 466 if (econfig) 467 *econfig = config & RISCV_PMU_RAW_EVENT_V2_MASK; 468 ret = RISCV_PMU_RAW_EVENT_V2_IDX; 469 } 470 /* Return error any bits [48-63] is set as it is not allowed by the spec */ 471 } else if (!(config & ~RISCV_PMU_RAW_EVENT_MASK)) { 472 if (econfig) 473 *econfig = config & RISCV_PMU_RAW_EVENT_MASK; 474 ret = RISCV_PMU_RAW_EVENT_IDX; 475 } 476 break; 477 case 2: 478 ret = (config & 0xFFFF) | (SBI_PMU_EVENT_TYPE_FW << 16); 479 break; 480 case 3: 481 /* 482 * For Risc-V platform specific firmware events 483 * Event code - 0xFFFF 484 * Event data - raw event encoding 485 */ 486 ret = SBI_PMU_EVENT_TYPE_FW << 16 | RISCV_PLAT_FW_EVENT; 487 if (econfig) 488 *econfig = config & RISCV_PMU_PLAT_FW_EVENT_MASK; 489 break; 490 default: 491 break; 492 } 493 break; 494 default: 495 break; 496 } 497 498 return ret; 499 } 500 EXPORT_SYMBOL_GPL(riscv_pmu_get_event_info); 501 502 /* 503 * Returns the counter width of a programmable counter and number of hardware 504 * counters. As we don't support heterogeneous CPUs yet, it is okay to just 505 * return the counter width of the first programmable counter. 506 */ 507 int riscv_pmu_get_hpm_info(u32 *hw_ctr_width, u32 *num_hw_ctr) 508 { 509 int i; 510 union sbi_pmu_ctr_info *info; 511 u32 hpm_width = 0, hpm_count = 0; 512 513 if (bitmap_empty(cmask, RISCV_MAX_COUNTERS)) 514 return -EINVAL; 515 516 for_each_set_bit(i, cmask, RISCV_MAX_COUNTERS) { 517 info = &pmu_ctr_list[i]; 518 if (!info) 519 continue; 520 if (!hpm_width && info->csr != CSR_CYCLE && info->csr != CSR_INSTRET) 521 hpm_width = info->width; 522 if (info->type == SBI_PMU_CTR_TYPE_HW) 523 hpm_count++; 524 } 525 526 *hw_ctr_width = hpm_width; 527 *num_hw_ctr = hpm_count; 528 529 return 0; 530 } 531 EXPORT_SYMBOL_GPL(riscv_pmu_get_hpm_info); 532 533 static uint8_t pmu_sbi_csr_index(struct perf_event *event) 534 { 535 return pmu_ctr_list[event->hw.idx].csr - CSR_CYCLE; 536 } 537 538 static unsigned long pmu_sbi_get_filter_flags(struct perf_event *event) 539 { 540 unsigned long cflags = 0; 541 bool guest_events = false; 542 543 if (event->attr.config1 & RISCV_PMU_CONFIG1_GUEST_EVENTS) 544 guest_events = true; 545 if (event->attr.exclude_kernel) 546 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VSINH : SBI_PMU_CFG_FLAG_SET_SINH; 547 if (event->attr.exclude_user) 548 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VUINH : SBI_PMU_CFG_FLAG_SET_UINH; 549 if (guest_events && event->attr.exclude_hv) 550 cflags |= SBI_PMU_CFG_FLAG_SET_SINH; 551 if (event->attr.exclude_host) 552 cflags |= SBI_PMU_CFG_FLAG_SET_UINH | SBI_PMU_CFG_FLAG_SET_SINH; 553 if (event->attr.exclude_guest) 554 cflags |= SBI_PMU_CFG_FLAG_SET_VSINH | SBI_PMU_CFG_FLAG_SET_VUINH; 555 556 return cflags; 557 } 558 559 static int pmu_sbi_ctr_get_idx(struct perf_event *event) 560 { 561 struct hw_perf_event *hwc = &event->hw; 562 struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); 563 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); 564 struct sbiret ret; 565 int idx, i; 566 u64 cbase = 0, cmask = 0; 567 unsigned long cflags = 0; 568 569 cflags = pmu_sbi_get_filter_flags(event); 570 571 /* 572 * In legacy mode, we have to force the fixed counters for those events 573 * but not in the user access mode as we want to use the other counters 574 * that support sampling/filtering. 575 */ 576 if ((hwc->flags & PERF_EVENT_FLAG_LEGACY) && (event->attr.type == PERF_TYPE_HARDWARE)) { 577 if (event->attr.config == PERF_COUNT_HW_CPU_CYCLES) { 578 cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH; 579 cmask = 1; 580 } else if (event->attr.config == PERF_COUNT_HW_INSTRUCTIONS) { 581 cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH; 582 cmask = BIT(CSR_INSTRET - CSR_CYCLE); 583 } 584 } 585 586 /* retrieve the available counter index */ 587 if (cmask) { 588 ret = pmu_sbi_ctr_cfg_match(cbase, cmask, cflags, hwc->event_base, 589 hwc->config); 590 } else { 591 ret.error = SBI_ERR_NOT_SUPPORTED; 592 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { 593 if (!rvpmu->cmask[i]) 594 continue; 595 cbase = i * BITS_PER_LONG; 596 ret = pmu_sbi_ctr_cfg_match(cbase, rvpmu->cmask[i], cflags, 597 hwc->event_base, hwc->config); 598 if (!ret.error) 599 break; 600 } 601 } 602 if (ret.error) { 603 pr_debug("Not able to find a counter for event %lx config %llx\n", 604 hwc->event_base, hwc->config); 605 return sbi_err_map_linux_errno(ret.error); 606 } 607 608 idx = ret.value; 609 if (!test_bit(idx, rvpmu->cmask) || !pmu_ctr_list[idx].value) 610 return -ENOENT; 611 612 /* Additional sanity check for the counter id */ 613 if (pmu_sbi_ctr_is_fw(idx)) { 614 if (!test_and_set_bit(idx, cpuc->used_fw_ctrs)) 615 return idx; 616 } else { 617 if (!test_and_set_bit(idx, cpuc->used_hw_ctrs)) 618 return idx; 619 } 620 621 return -ENOENT; 622 } 623 624 static void pmu_sbi_ctr_clear_idx(struct perf_event *event) 625 { 626 627 struct hw_perf_event *hwc = &event->hw; 628 struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); 629 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); 630 int idx = hwc->idx; 631 632 if (pmu_sbi_ctr_is_fw(idx)) 633 clear_bit(idx, cpuc->used_fw_ctrs); 634 else 635 clear_bit(idx, cpuc->used_hw_ctrs); 636 } 637 638 static int pmu_event_find_cache(u64 config) 639 { 640 unsigned int cache_type, cache_op, cache_result, ret; 641 642 cache_type = (config >> 0) & 0xff; 643 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 644 return -EINVAL; 645 646 cache_op = (config >> 8) & 0xff; 647 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 648 return -EINVAL; 649 650 cache_result = (config >> 16) & 0xff; 651 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 652 return -EINVAL; 653 654 ret = pmu_cache_event_map[cache_type][cache_op][cache_result].event_idx; 655 656 return ret; 657 } 658 659 static bool pmu_sbi_is_fw_event(struct perf_event *event) 660 { 661 u32 type = event->attr.type; 662 u64 config = event->attr.config; 663 664 if ((type == PERF_TYPE_RAW) && ((config >> 63) == 1)) 665 return true; 666 else 667 return false; 668 } 669 670 static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig) 671 { 672 u32 type = event->attr.type; 673 u64 config = event->attr.config; 674 675 /* 676 * Ensure we are finished checking standard hardware events for 677 * validity before allowing userspace to configure any events. 678 */ 679 flush_work(&check_std_events_work); 680 681 return riscv_pmu_get_event_info(type, config, econfig); 682 } 683 684 static void pmu_sbi_snapshot_free(struct riscv_pmu *pmu) 685 { 686 int cpu; 687 688 for_each_possible_cpu(cpu) { 689 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu); 690 691 if (!cpu_hw_evt->snapshot_addr) 692 continue; 693 694 free_page((unsigned long)cpu_hw_evt->snapshot_addr); 695 cpu_hw_evt->snapshot_addr = NULL; 696 cpu_hw_evt->snapshot_addr_phys = 0; 697 } 698 } 699 700 static int pmu_sbi_snapshot_alloc(struct riscv_pmu *pmu) 701 { 702 int cpu; 703 struct page *snapshot_page; 704 705 for_each_possible_cpu(cpu) { 706 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu); 707 708 snapshot_page = alloc_page(GFP_ATOMIC | __GFP_ZERO); 709 if (!snapshot_page) { 710 pmu_sbi_snapshot_free(pmu); 711 return -ENOMEM; 712 } 713 cpu_hw_evt->snapshot_addr = page_to_virt(snapshot_page); 714 cpu_hw_evt->snapshot_addr_phys = page_to_phys(snapshot_page); 715 } 716 717 return 0; 718 } 719 720 static int pmu_sbi_snapshot_disable(void) 721 { 722 struct sbiret ret; 723 724 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, SBI_SHMEM_DISABLE, 725 SBI_SHMEM_DISABLE, 0, 0, 0, 0); 726 if (ret.error) { 727 pr_warn("failed to disable snapshot shared memory\n"); 728 return sbi_err_map_linux_errno(ret.error); 729 } 730 731 return 0; 732 } 733 734 static int pmu_sbi_snapshot_setup(struct riscv_pmu *pmu, int cpu) 735 { 736 struct cpu_hw_events *cpu_hw_evt; 737 struct sbiret ret = {0}; 738 739 cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu); 740 if (!cpu_hw_evt->snapshot_addr_phys) 741 return -EINVAL; 742 743 if (cpu_hw_evt->snapshot_set_done) 744 return 0; 745 746 if (IS_ENABLED(CONFIG_32BIT)) 747 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, 748 cpu_hw_evt->snapshot_addr_phys, 749 (u64)(cpu_hw_evt->snapshot_addr_phys) >> 32, 0, 0, 0, 0); 750 else 751 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, 752 cpu_hw_evt->snapshot_addr_phys, 0, 0, 0, 0, 0); 753 754 /* Free up the snapshot area memory and fall back to SBI PMU calls without snapshot */ 755 if (ret.error) { 756 if (ret.error != SBI_ERR_NOT_SUPPORTED) 757 pr_warn("pmu snapshot setup failed with error %ld\n", ret.error); 758 return sbi_err_map_linux_errno(ret.error); 759 } 760 761 memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS); 762 cpu_hw_evt->snapshot_set_done = true; 763 764 return 0; 765 } 766 767 static u64 pmu_sbi_ctr_read(struct perf_event *event) 768 { 769 struct hw_perf_event *hwc = &event->hw; 770 int idx = hwc->idx; 771 struct sbiret ret; 772 u64 val = 0; 773 struct riscv_pmu *pmu = to_riscv_pmu(event->pmu); 774 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); 775 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; 776 union sbi_pmu_ctr_info info = pmu_ctr_list[idx]; 777 778 /* Read the value from the shared memory directly only if counter is stopped */ 779 if (sbi_pmu_snapshot_available() && (hwc->state & PERF_HES_STOPPED)) { 780 val = sdata->ctr_values[idx]; 781 return val; 782 } 783 784 if (pmu_sbi_is_fw_event(event)) { 785 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ, 786 hwc->idx, 0, 0, 0, 0, 0); 787 if (ret.error) 788 return 0; 789 790 val = ret.value; 791 if (IS_ENABLED(CONFIG_32BIT) && sbi_v2_available && info.width >= 32) { 792 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ_HI, 793 hwc->idx, 0, 0, 0, 0, 0); 794 if (!ret.error) 795 val |= ((u64)ret.value << 32); 796 else 797 WARN_ONCE(1, "Unable to read upper 32 bits of firmware counter error: %ld\n", 798 ret.error); 799 } 800 } else { 801 val = riscv_pmu_ctr_read_csr(info.csr); 802 if (IS_ENABLED(CONFIG_32BIT)) 803 val |= ((u64)riscv_pmu_ctr_read_csr(info.csr + 0x80)) << 32; 804 } 805 806 return val; 807 } 808 809 static void pmu_sbi_set_scounteren(void *arg) 810 { 811 struct perf_event *event = (struct perf_event *)arg; 812 813 if (event->hw.idx != -1) 814 csr_write(CSR_SCOUNTEREN, 815 csr_read(CSR_SCOUNTEREN) | BIT(pmu_sbi_csr_index(event))); 816 } 817 818 static void pmu_sbi_reset_scounteren(void *arg) 819 { 820 struct perf_event *event = (struct perf_event *)arg; 821 822 if (event->hw.idx != -1) 823 csr_write(CSR_SCOUNTEREN, 824 csr_read(CSR_SCOUNTEREN) & ~BIT(pmu_sbi_csr_index(event))); 825 } 826 827 static void pmu_sbi_ctr_start(struct perf_event *event, u64 ival) 828 { 829 struct sbiret ret; 830 struct hw_perf_event *hwc = &event->hw; 831 unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE; 832 833 /* There is no benefit setting SNAPSHOT FLAG for a single counter */ 834 #if defined(CONFIG_32BIT) 835 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx, 836 1, flag, ival, ival >> 32, 0); 837 #else 838 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx, 839 1, flag, ival, 0, 0); 840 #endif 841 if (ret.error && (ret.error != SBI_ERR_ALREADY_STARTED)) 842 pr_err("Starting counter idx %d failed with error %d\n", 843 hwc->idx, sbi_err_map_linux_errno(ret.error)); 844 845 if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) && 846 (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT)) 847 pmu_sbi_set_scounteren((void *)event); 848 } 849 850 static void pmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag) 851 { 852 struct sbiret ret; 853 struct hw_perf_event *hwc = &event->hw; 854 struct riscv_pmu *pmu = to_riscv_pmu(event->pmu); 855 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); 856 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; 857 858 if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) && 859 (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT)) 860 pmu_sbi_reset_scounteren((void *)event); 861 862 if (sbi_pmu_snapshot_available()) 863 flag |= SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; 864 865 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, hwc->idx, 1, flag, 0, 0, 0); 866 if (!ret.error && sbi_pmu_snapshot_available()) { 867 /* 868 * The counter snapshot is based on the index base specified by hwc->idx. 869 * The actual counter value is updated in shared memory at index 0 when counter 870 * mask is 0x01. To ensure accurate counter values, it's necessary to transfer 871 * the counter value to shared memory. However, if hwc->idx is zero, the counter 872 * value is already correctly updated in shared memory, requiring no further 873 * adjustment. 874 */ 875 if (hwc->idx > 0) { 876 sdata->ctr_values[hwc->idx] = sdata->ctr_values[0]; 877 sdata->ctr_values[0] = 0; 878 } 879 } else if (ret.error && (ret.error != SBI_ERR_ALREADY_STOPPED) && 880 flag != SBI_PMU_STOP_FLAG_RESET) { 881 pr_err("Stopping counter idx %d failed with error %d\n", 882 hwc->idx, sbi_err_map_linux_errno(ret.error)); 883 } 884 } 885 886 static int pmu_sbi_find_num_ctrs(void) 887 { 888 struct sbiret ret; 889 890 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS, 0, 0, 0, 0, 0, 0); 891 if (!ret.error) 892 return ret.value; 893 else 894 return sbi_err_map_linux_errno(ret.error); 895 } 896 897 static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) 898 { 899 struct sbiret ret; 900 int i, num_hw_ctr = 0, num_fw_ctr = 0; 901 union sbi_pmu_ctr_info cinfo; 902 903 pmu_ctr_list = kzalloc_objs(*pmu_ctr_list, nctr); 904 if (!pmu_ctr_list) 905 return -ENOMEM; 906 907 for (i = 0; i < nctr; i++) { 908 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0); 909 if (ret.error) 910 /* The logical counter ids are not expected to be contiguous */ 911 continue; 912 913 set_bit(i, mask); 914 915 cinfo.value = ret.value; 916 if (cinfo.type == SBI_PMU_CTR_TYPE_FW) 917 num_fw_ctr++; 918 else 919 num_hw_ctr++; 920 pmu_ctr_list[i].value = cinfo.value; 921 } 922 923 pr_info("%d firmware and %d hardware counters\n", num_fw_ctr, num_hw_ctr); 924 925 return 0; 926 } 927 928 static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) 929 { 930 int i; 931 932 /* 933 * No need to check the error because we are disabling all the counters 934 * which may include counters that are not enabled yet. 935 */ 936 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { 937 if (!pmu->cmask[i]) 938 continue; 939 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, 940 i * BITS_PER_LONG, pmu->cmask[i], 941 SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); 942 } 943 } 944 945 static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) 946 { 947 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); 948 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; 949 unsigned long flag = 0; 950 int i, idx; 951 struct sbiret ret; 952 u64 temp_ctr_overflow_mask = 0; 953 954 if (sbi_pmu_snapshot_available()) 955 flag = SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; 956 957 /* Reset the shadow copy to avoid save/restore any value from previous overflow */ 958 memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS); 959 960 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { 961 /* No need to check the error here as we can't do anything about the error */ 962 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, i * BITS_PER_LONG, 963 cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0); 964 if (!ret.error && sbi_pmu_snapshot_available()) { 965 /* Save the counter values to avoid clobbering */ 966 for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) 967 cpu_hw_evt->snapshot_cval_shcopy[i * BITS_PER_LONG + idx] = 968 sdata->ctr_values[idx]; 969 /* Save the overflow mask to avoid clobbering */ 970 temp_ctr_overflow_mask |= sdata->ctr_overflow_mask << (i * BITS_PER_LONG); 971 } 972 } 973 974 /* Restore the counter values to the shared memory for used hw counters */ 975 if (sbi_pmu_snapshot_available()) { 976 for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) 977 sdata->ctr_values[idx] = cpu_hw_evt->snapshot_cval_shcopy[idx]; 978 if (temp_ctr_overflow_mask) 979 sdata->ctr_overflow_mask = temp_ctr_overflow_mask; 980 } 981 } 982 983 /* 984 * This function starts all the used counters in two step approach. 985 * Any counter that did not overflow can be start in a single step 986 * while the overflowed counters need to be started with updated initialization 987 * value. 988 */ 989 static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, 990 u64 ctr_ovf_mask) 991 { 992 int idx = 0, i; 993 struct perf_event *event; 994 unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE; 995 unsigned long ctr_start_mask = 0; 996 uint64_t max_period; 997 struct hw_perf_event *hwc; 998 u64 init_val = 0; 999 1000 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { 1001 ctr_start_mask = cpu_hw_evt->used_hw_ctrs[i] & ~ctr_ovf_mask; 1002 /* Start all the counters that did not overflow in a single shot */ 1003 if (ctr_start_mask) { 1004 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, i * BITS_PER_LONG, 1005 ctr_start_mask, 0, 0, 0, 0); 1006 } 1007 } 1008 1009 /* Reinitialize and start all the counter that overflowed */ 1010 while (ctr_ovf_mask) { 1011 if (ctr_ovf_mask & 0x01) { 1012 event = cpu_hw_evt->events[idx]; 1013 hwc = &event->hw; 1014 max_period = riscv_pmu_ctr_get_width_mask(event); 1015 init_val = local64_read(&hwc->prev_count) & max_period; 1016 #if defined(CONFIG_32BIT) 1017 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, 1018 flag, init_val, init_val >> 32, 0); 1019 #else 1020 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, 1021 flag, init_val, 0, 0); 1022 #endif 1023 perf_event_update_userpage(event); 1024 } 1025 ctr_ovf_mask = ctr_ovf_mask >> 1; 1026 idx++; 1027 } 1028 } 1029 1030 static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_evt, 1031 u64 ctr_ovf_mask) 1032 { 1033 int i, idx = 0; 1034 struct perf_event *event; 1035 unsigned long flag = SBI_PMU_START_FLAG_INIT_SNAPSHOT; 1036 u64 max_period, init_val = 0; 1037 struct hw_perf_event *hwc; 1038 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; 1039 1040 for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { 1041 if (ctr_ovf_mask & BIT_ULL(idx)) { 1042 event = cpu_hw_evt->events[idx]; 1043 hwc = &event->hw; 1044 max_period = riscv_pmu_ctr_get_width_mask(event); 1045 init_val = local64_read(&hwc->prev_count) & max_period; 1046 cpu_hw_evt->snapshot_cval_shcopy[idx] = init_val; 1047 } 1048 /* 1049 * We do not need to update the non-overflow counters the previous 1050 * value should have been there already. 1051 */ 1052 } 1053 1054 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { 1055 /* Restore the counter values to relative indices for used hw counters */ 1056 for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) 1057 sdata->ctr_values[idx] = 1058 cpu_hw_evt->snapshot_cval_shcopy[idx + i * BITS_PER_LONG]; 1059 /* Start all the counters in a single shot */ 1060 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx * BITS_PER_LONG, 1061 cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0); 1062 } 1063 } 1064 1065 static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu, 1066 u64 ctr_ovf_mask) 1067 { 1068 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); 1069 1070 if (sbi_pmu_snapshot_available()) 1071 pmu_sbi_start_ovf_ctrs_snapshot(cpu_hw_evt, ctr_ovf_mask); 1072 else 1073 pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask); 1074 } 1075 1076 static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) 1077 { 1078 struct perf_sample_data data; 1079 struct pt_regs *regs; 1080 struct hw_perf_event *hw_evt; 1081 union sbi_pmu_ctr_info *info; 1082 int lidx, hidx, fidx; 1083 struct riscv_pmu *pmu; 1084 struct perf_event *event; 1085 u64 overflow; 1086 u64 overflowed_ctrs = 0; 1087 struct cpu_hw_events *cpu_hw_evt = dev; 1088 u64 start_clock = sched_clock(); 1089 struct riscv_pmu_snapshot_data *sdata; 1090 1091 if (WARN_ON_ONCE(!cpu_hw_evt)) 1092 return IRQ_NONE; 1093 1094 sdata = cpu_hw_evt->snapshot_addr; 1095 1096 /* Firmware counter don't support overflow yet */ 1097 fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS); 1098 if (fidx == RISCV_MAX_COUNTERS) { 1099 csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); 1100 return IRQ_NONE; 1101 } 1102 1103 event = cpu_hw_evt->events[fidx]; 1104 if (!event) { 1105 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); 1106 return IRQ_NONE; 1107 } 1108 1109 pmu = to_riscv_pmu(event->pmu); 1110 pmu_sbi_stop_hw_ctrs(pmu); 1111 1112 /* Overflow status register should only be read after counter are stopped */ 1113 if (sbi_pmu_snapshot_available()) 1114 overflow = sdata->ctr_overflow_mask; 1115 else 1116 ALT_SBI_PMU_OVERFLOW(overflow); 1117 1118 /* 1119 * Overflow interrupt pending bit should only be cleared after stopping 1120 * all the counters to avoid any race condition. 1121 */ 1122 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); 1123 1124 /* No overflow bit is set */ 1125 if (!overflow) 1126 return IRQ_NONE; 1127 1128 regs = get_irq_regs(); 1129 1130 for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { 1131 struct perf_event *event = cpu_hw_evt->events[lidx]; 1132 1133 /* Skip if invalid event or user did not request a sampling */ 1134 if (!event || !is_sampling_event(event)) 1135 continue; 1136 1137 info = &pmu_ctr_list[lidx]; 1138 /* Do a sanity check */ 1139 if (!info || info->type != SBI_PMU_CTR_TYPE_HW) 1140 continue; 1141 1142 if (sbi_pmu_snapshot_available()) 1143 /* SBI implementation already updated the logical indicies */ 1144 hidx = lidx; 1145 else 1146 /* compute hardware counter index */ 1147 hidx = info->csr - CSR_CYCLE; 1148 1149 /* check if the corresponding bit is set in scountovf or overflow mask in shmem */ 1150 if (!(overflow & BIT_ULL(hidx))) 1151 continue; 1152 1153 /* 1154 * Keep a track of overflowed counters so that they can be started 1155 * with updated initial value. 1156 */ 1157 overflowed_ctrs |= BIT_ULL(lidx); 1158 hw_evt = &event->hw; 1159 /* Update the event states here so that we know the state while reading */ 1160 hw_evt->state |= PERF_HES_STOPPED; 1161 riscv_pmu_event_update(event); 1162 hw_evt->state |= PERF_HES_UPTODATE; 1163 perf_sample_data_init(&data, 0, hw_evt->last_period); 1164 if (riscv_pmu_event_set_period(event)) { 1165 /* 1166 * Unlike other ISAs, RISC-V don't have to disable interrupts 1167 * to avoid throttling here. As per the specification, the 1168 * interrupt remains disabled until the OF bit is set. 1169 * Interrupts are enabled again only during the start. 1170 * TODO: We will need to stop the guest counters once 1171 * virtualization support is added. 1172 */ 1173 perf_event_overflow(event, &data, regs); 1174 } 1175 /* Reset the state as we are going to start the counter after the loop */ 1176 hw_evt->state = 0; 1177 } 1178 1179 pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs); 1180 perf_sample_event_took(sched_clock() - start_clock); 1181 1182 return IRQ_HANDLED; 1183 } 1184 1185 static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) 1186 { 1187 struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); 1188 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); 1189 1190 /* 1191 * We keep enabling userspace access to CYCLE, TIME and INSTRET via the 1192 * legacy option but that will be removed in the future. 1193 */ 1194 if (sysctl_perf_user_access == SYSCTL_LEGACY) 1195 csr_write(CSR_SCOUNTEREN, 0x7); 1196 else 1197 csr_write(CSR_SCOUNTEREN, 0x2); 1198 1199 /* Stop all the counters so that they can be enabled from perf */ 1200 pmu_sbi_stop_all(pmu); 1201 1202 if (riscv_pmu_use_irq) { 1203 cpu_hw_evt->irq = riscv_pmu_irq; 1204 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); 1205 enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE); 1206 } 1207 1208 if (sbi_pmu_snapshot_available()) 1209 return pmu_sbi_snapshot_setup(pmu, cpu); 1210 1211 return 0; 1212 } 1213 1214 static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node) 1215 { 1216 if (riscv_pmu_use_irq) { 1217 disable_percpu_irq(riscv_pmu_irq); 1218 } 1219 1220 /* Disable all counters access for user mode now */ 1221 csr_write(CSR_SCOUNTEREN, 0x0); 1222 1223 if (sbi_pmu_snapshot_available()) 1224 return pmu_sbi_snapshot_disable(); 1225 1226 return 0; 1227 } 1228 1229 static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev) 1230 { 1231 int ret; 1232 struct cpu_hw_events __percpu *hw_events = pmu->hw_events; 1233 struct irq_domain *domain = NULL; 1234 1235 if (riscv_isa_extension_available(NULL, SSCOFPMF)) { 1236 riscv_pmu_irq_num = RV_IRQ_PMU; 1237 riscv_pmu_use_irq = true; 1238 } else if (IS_ENABLED(CONFIG_ERRATA_THEAD_PMU) && 1239 riscv_cached_mvendorid(0) == THEAD_VENDOR_ID && 1240 riscv_cached_marchid(0) == 0 && 1241 riscv_cached_mimpid(0) == 0) { 1242 riscv_pmu_irq_num = THEAD_C9XX_RV_IRQ_PMU; 1243 riscv_pmu_use_irq = true; 1244 } else if (riscv_has_vendor_extension_unlikely(ANDES_VENDOR_ID, 1245 RISCV_ISA_VENDOR_EXT_XANDESPMU) && 1246 IS_ENABLED(CONFIG_ANDES_CUSTOM_PMU)) { 1247 riscv_pmu_irq_num = ANDES_SLI_CAUSE_BASE + ANDES_RV_IRQ_PMOVI; 1248 riscv_pmu_use_irq = true; 1249 } 1250 1251 riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG); 1252 1253 if (!riscv_pmu_use_irq) 1254 return -EOPNOTSUPP; 1255 1256 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), 1257 DOMAIN_BUS_ANY); 1258 if (!domain) { 1259 pr_err("Failed to find INTC IRQ root domain\n"); 1260 ret = -ENODEV; 1261 goto err; 1262 } 1263 1264 riscv_pmu_irq = irq_create_mapping(domain, riscv_pmu_irq_num); 1265 if (!riscv_pmu_irq) { 1266 pr_err("Failed to map PMU interrupt for node\n"); 1267 ret = -ENODEV; 1268 goto err; 1269 } 1270 1271 ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events); 1272 if (ret) { 1273 pr_err("registering percpu irq failed [%d]\n", ret); 1274 irq_dispose_mapping(riscv_pmu_irq); 1275 riscv_pmu_irq = 0; 1276 goto err; 1277 } 1278 1279 return 0; 1280 err: 1281 riscv_pmu_use_irq = false; 1282 return ret; 1283 } 1284 1285 #ifdef CONFIG_CPU_PM 1286 static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, 1287 void *v) 1288 { 1289 struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, riscv_pm_nb); 1290 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); 1291 bool enabled = !bitmap_empty(cpuc->used_hw_ctrs, RISCV_MAX_COUNTERS); 1292 struct perf_event *event; 1293 int idx; 1294 1295 if (!enabled) 1296 return NOTIFY_OK; 1297 1298 for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) { 1299 event = cpuc->events[idx]; 1300 if (!event) 1301 continue; 1302 1303 switch (cmd) { 1304 case CPU_PM_ENTER: 1305 /* 1306 * Stop and update the counter 1307 */ 1308 riscv_pmu_stop(event, PERF_EF_UPDATE); 1309 break; 1310 case CPU_PM_EXIT: 1311 case CPU_PM_ENTER_FAILED: 1312 /* 1313 * Restore and enable the counter. 1314 */ 1315 riscv_pmu_start(event, PERF_EF_RELOAD); 1316 break; 1317 default: 1318 break; 1319 } 1320 } 1321 1322 return NOTIFY_OK; 1323 } 1324 1325 static int riscv_pm_pmu_register(struct riscv_pmu *pmu) 1326 { 1327 pmu->riscv_pm_nb.notifier_call = riscv_pm_pmu_notify; 1328 return cpu_pm_register_notifier(&pmu->riscv_pm_nb); 1329 } 1330 1331 static void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) 1332 { 1333 cpu_pm_unregister_notifier(&pmu->riscv_pm_nb); 1334 } 1335 #else 1336 static inline int riscv_pm_pmu_register(struct riscv_pmu *pmu) { return 0; } 1337 static inline void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) { } 1338 #endif 1339 1340 static void riscv_pmu_destroy(struct riscv_pmu *pmu) 1341 { 1342 if (sbi_v2_available) { 1343 if (sbi_pmu_snapshot_available()) { 1344 pmu_sbi_snapshot_disable(); 1345 pmu_sbi_snapshot_free(pmu); 1346 } 1347 } 1348 riscv_pm_pmu_unregister(pmu); 1349 if (!hlist_unhashed(&pmu->node)) 1350 cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node); 1351 } 1352 1353 static void pmu_sbi_event_init(struct perf_event *event) 1354 { 1355 /* 1356 * The permissions are set at event_init so that we do not depend 1357 * on the sysctl value that can change. 1358 */ 1359 if (sysctl_perf_user_access == SYSCTL_NO_USER_ACCESS) 1360 event->hw.flags |= PERF_EVENT_FLAG_NO_USER_ACCESS; 1361 else if (sysctl_perf_user_access == SYSCTL_USER_ACCESS) 1362 event->hw.flags |= PERF_EVENT_FLAG_USER_ACCESS; 1363 else 1364 event->hw.flags |= PERF_EVENT_FLAG_LEGACY; 1365 } 1366 1367 static void pmu_sbi_event_mapped(struct perf_event *event, struct mm_struct *mm) 1368 { 1369 if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS) 1370 return; 1371 1372 if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) { 1373 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES && 1374 event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) { 1375 return; 1376 } 1377 } 1378 1379 /* 1380 * The user mmapped the event to directly access it: this is where 1381 * we determine based on sysctl_perf_user_access if we grant userspace 1382 * the direct access to this event. That means that within the same 1383 * task, some events may be directly accessible and some other may not, 1384 * if the user changes the value of sysctl_perf_user_accesss in the 1385 * meantime. 1386 */ 1387 1388 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; 1389 1390 /* 1391 * We must enable userspace access *before* advertising in the user page 1392 * that it is possible to do so to avoid any race. 1393 * And we must notify all cpus here because threads that currently run 1394 * on other cpus will try to directly access the counter too without 1395 * calling pmu_sbi_ctr_start. 1396 */ 1397 if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS) 1398 on_each_cpu_mask(mm_cpumask(mm), 1399 pmu_sbi_set_scounteren, (void *)event, 1); 1400 } 1401 1402 static void pmu_sbi_event_unmapped(struct perf_event *event, struct mm_struct *mm) 1403 { 1404 if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS) 1405 return; 1406 1407 if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) { 1408 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES && 1409 event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) { 1410 return; 1411 } 1412 } 1413 1414 /* 1415 * Here we can directly remove user access since the user does not have 1416 * access to the user page anymore so we avoid the racy window where the 1417 * user could have read cap_user_rdpmc to true right before we disable 1418 * it. 1419 */ 1420 event->hw.flags &= ~PERF_EVENT_FLAG_USER_READ_CNT; 1421 1422 if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS) 1423 on_each_cpu_mask(mm_cpumask(mm), 1424 pmu_sbi_reset_scounteren, (void *)event, 1); 1425 } 1426 1427 static void riscv_pmu_update_counter_access(void *info) 1428 { 1429 if (sysctl_perf_user_access == SYSCTL_LEGACY) 1430 csr_write(CSR_SCOUNTEREN, 0x7); 1431 else 1432 csr_write(CSR_SCOUNTEREN, 0x2); 1433 } 1434 1435 static int riscv_pmu_proc_user_access_handler(const struct ctl_table *table, 1436 int write, void *buffer, 1437 size_t *lenp, loff_t *ppos) 1438 { 1439 int prev = sysctl_perf_user_access; 1440 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 1441 1442 /* 1443 * Test against the previous value since we clear SCOUNTEREN when 1444 * sysctl_perf_user_access is set to SYSCTL_USER_ACCESS, but we should 1445 * not do that if that was already the case. 1446 */ 1447 if (ret || !write || prev == sysctl_perf_user_access) 1448 return ret; 1449 1450 on_each_cpu(riscv_pmu_update_counter_access, NULL, 1); 1451 1452 return 0; 1453 } 1454 1455 static const struct ctl_table sbi_pmu_sysctl_table[] = { 1456 { 1457 .procname = "perf_user_access", 1458 .data = &sysctl_perf_user_access, 1459 .maxlen = sizeof(unsigned int), 1460 .mode = 0644, 1461 .proc_handler = riscv_pmu_proc_user_access_handler, 1462 .extra1 = SYSCTL_ZERO, 1463 .extra2 = SYSCTL_TWO, 1464 }, 1465 }; 1466 1467 static int pmu_sbi_device_probe(struct platform_device *pdev) 1468 { 1469 struct riscv_pmu *pmu = NULL; 1470 int ret = -ENODEV; 1471 int num_counters; 1472 bool irq_requested = false; 1473 1474 pr_info("SBI PMU extension is available\n"); 1475 pmu = riscv_pmu_alloc(); 1476 if (!pmu) 1477 return -ENOMEM; 1478 1479 num_counters = pmu_sbi_find_num_ctrs(); 1480 if (num_counters < 0) { 1481 pr_err("SBI PMU extension doesn't provide any counters\n"); 1482 goto out_free; 1483 } 1484 1485 /* It is possible to get from SBI more than max number of counters */ 1486 if (num_counters > RISCV_MAX_COUNTERS) { 1487 num_counters = RISCV_MAX_COUNTERS; 1488 pr_info("SBI returned more than maximum number of counters. Limiting the number of counters to %d\n", num_counters); 1489 } 1490 1491 /* cache all the information about counters now */ 1492 if (pmu_sbi_get_ctrinfo(num_counters, cmask)) 1493 goto out_free; 1494 1495 ret = pmu_sbi_setup_irqs(pmu, pdev); 1496 if (ret < 0) { 1497 pr_info("Perf sampling/filtering is not supported as sscof extension is not available\n"); 1498 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1499 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; 1500 } 1501 irq_requested = (ret == 0); 1502 1503 pmu->pmu.attr_groups = riscv_pmu_attr_groups; 1504 pmu->pmu.parent = &pdev->dev; 1505 bitmap_copy(pmu->cmask, cmask, RISCV_MAX_COUNTERS); 1506 pmu->ctr_start = pmu_sbi_ctr_start; 1507 pmu->ctr_stop = pmu_sbi_ctr_stop; 1508 pmu->event_map = pmu_sbi_event_map; 1509 pmu->ctr_get_idx = pmu_sbi_ctr_get_idx; 1510 pmu->ctr_get_width = pmu_sbi_ctr_get_width; 1511 pmu->ctr_clear_idx = pmu_sbi_ctr_clear_idx; 1512 pmu->ctr_read = pmu_sbi_ctr_read; 1513 pmu->event_init = pmu_sbi_event_init; 1514 pmu->event_mapped = pmu_sbi_event_mapped; 1515 pmu->event_unmapped = pmu_sbi_event_unmapped; 1516 pmu->csr_index = pmu_sbi_csr_index; 1517 1518 ret = riscv_pm_pmu_register(pmu); 1519 if (ret) 1520 goto out_destroy; 1521 1522 ret = perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW); 1523 if (ret) 1524 goto out_destroy; 1525 1526 /* SBI PMU Snapsphot is only available in SBI v2.0 */ 1527 if (sbi_v2_available) { 1528 int cpu; 1529 1530 ret = pmu_sbi_snapshot_alloc(pmu); 1531 if (ret) 1532 goto out_unregister; 1533 1534 cpu = get_cpu(); 1535 ret = pmu_sbi_snapshot_setup(pmu, cpu); 1536 put_cpu(); 1537 1538 if (ret) { 1539 /* Snapshot is an optional feature. Continue if not available */ 1540 pmu_sbi_snapshot_free(pmu); 1541 } else { 1542 pr_info("SBI PMU snapshot detected\n"); 1543 /* 1544 * We enable it once here for the boot cpu. If snapshot shmem setup 1545 * fails during cpu hotplug process, it will fail to start the cpu 1546 * as we can not handle hetergenous PMUs with different snapshot 1547 * capability. 1548 */ 1549 static_branch_enable(&sbi_pmu_snapshot_available); 1550 } 1551 } 1552 1553 register_sysctl("kernel", sbi_pmu_sysctl_table); 1554 1555 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node); 1556 if (ret) 1557 goto out_unregister; 1558 1559 /* Asynchronously check which standard events are available */ 1560 schedule_work(&check_std_events_work); 1561 1562 return 0; 1563 1564 out_unregister: 1565 perf_pmu_unregister(&pmu->pmu); 1566 1567 out_destroy: 1568 riscv_pmu_destroy(pmu); 1569 if (irq_requested) { 1570 free_percpu_irq(riscv_pmu_irq, pmu->hw_events); 1571 irq_dispose_mapping(riscv_pmu_irq); 1572 riscv_pmu_irq = 0; 1573 } 1574 1575 out_free: 1576 free_percpu(pmu->hw_events); 1577 kfree(pmu_ctr_list); 1578 pmu_ctr_list = NULL; 1579 kfree(pmu); 1580 return ret; 1581 } 1582 1583 static struct platform_driver pmu_sbi_driver = { 1584 .probe = pmu_sbi_device_probe, 1585 .driver = { 1586 .name = RISCV_PMU_SBI_PDEV_NAME, 1587 }, 1588 }; 1589 1590 static int __init pmu_sbi_devinit(void) 1591 { 1592 int ret; 1593 struct platform_device *pdev; 1594 1595 if (sbi_spec_version < sbi_mk_version(0, 3) || 1596 !sbi_probe_extension(SBI_EXT_PMU)) { 1597 return 0; 1598 } 1599 1600 if (sbi_spec_version >= sbi_mk_version(2, 0)) 1601 sbi_v2_available = true; 1602 1603 if (sbi_spec_version >= sbi_mk_version(3, 0)) 1604 sbi_v3_available = true; 1605 1606 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_STARTING, 1607 "perf/riscv/pmu:starting", 1608 pmu_sbi_starting_cpu, pmu_sbi_dying_cpu); 1609 if (ret) { 1610 pr_err("CPU hotplug notifier could not be registered: %d\n", 1611 ret); 1612 return ret; 1613 } 1614 1615 ret = platform_driver_register(&pmu_sbi_driver); 1616 if (ret) 1617 return ret; 1618 1619 pdev = platform_device_register_simple(RISCV_PMU_SBI_PDEV_NAME, -1, NULL, 0); 1620 if (IS_ERR(pdev)) { 1621 platform_driver_unregister(&pmu_sbi_driver); 1622 return PTR_ERR(pdev); 1623 } 1624 1625 /* Notify legacy implementation that SBI pmu is available*/ 1626 riscv_pmu_legacy_skip_init(); 1627 1628 return ret; 1629 } 1630 device_initcall(pmu_sbi_devinit) 1631