1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * RISC-V performance counter support.
4 *
5 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
6 *
7 * This code is based on ARM perf event code which is in turn based on
8 * sparc64 and x86 code.
9 */
10
11 #define pr_fmt(fmt) "riscv-pmu-sbi: " fmt
12
13 #include <linux/perf/riscv_pmu.h>
14 #include <linux/platform_device.h>
15 #include <linux/irq.h>
16 #include <linux/irqdomain.h>
17 #include <linux/of_irq.h>
18 #include <linux/of.h>
19 #include <linux/cpu_pm.h>
20 #include <linux/sched/clock.h>
21 #include <linux/soc/andes/irq.h>
22 #include <linux/workqueue.h>
23
24 #include <asm/errata_list.h>
25 #include <asm/sbi.h>
26 #include <asm/cpufeature.h>
27 #include <asm/vendor_extensions.h>
28 #include <asm/vendor_extensions/andes.h>
29
30 #define ALT_SBI_PMU_OVERFLOW(__ovl) \
31 asm volatile(ALTERNATIVE_2( \
32 "csrr %0, " __stringify(CSR_SCOUNTOVF), \
33 "csrr %0, " __stringify(THEAD_C9XX_CSR_SCOUNTEROF), \
34 THEAD_VENDOR_ID, ERRATA_THEAD_PMU, \
35 CONFIG_ERRATA_THEAD_PMU, \
36 "csrr %0, " __stringify(ANDES_CSR_SCOUNTEROF), \
37 ANDES_VENDOR_ID, \
38 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \
39 CONFIG_ANDES_CUSTOM_PMU) \
40 : "=r" (__ovl) : \
41 : "memory")
42
43 #define ALT_SBI_PMU_OVF_CLEAR_PENDING(__irq_mask) \
44 asm volatile(ALTERNATIVE( \
45 "csrc " __stringify(CSR_IP) ", %0\n\t", \
46 "csrc " __stringify(ANDES_CSR_SLIP) ", %0\n\t", \
47 ANDES_VENDOR_ID, \
48 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \
49 CONFIG_ANDES_CUSTOM_PMU) \
50 : : "r"(__irq_mask) \
51 : "memory")
52
53 #define SYSCTL_NO_USER_ACCESS 0
54 #define SYSCTL_USER_ACCESS 1
55 #define SYSCTL_LEGACY 2
56
57 #define PERF_EVENT_FLAG_NO_USER_ACCESS BIT(SYSCTL_NO_USER_ACCESS)
58 #define PERF_EVENT_FLAG_USER_ACCESS BIT(SYSCTL_USER_ACCESS)
59 #define PERF_EVENT_FLAG_LEGACY BIT(SYSCTL_LEGACY)
60
61 PMU_FORMAT_ATTR(event, "config:0-55");
62 PMU_FORMAT_ATTR(firmware, "config:62-63");
63
64 static bool sbi_v2_available;
65 static bool sbi_v3_available;
66 static DEFINE_STATIC_KEY_FALSE(sbi_pmu_snapshot_available);
67 #define sbi_pmu_snapshot_available() \
68 static_branch_unlikely(&sbi_pmu_snapshot_available)
69
70 static struct attribute *riscv_arch_formats_attr[] = {
71 &format_attr_event.attr,
72 &format_attr_firmware.attr,
73 NULL,
74 };
75
76 static struct attribute_group riscv_pmu_format_group = {
77 .name = "format",
78 .attrs = riscv_arch_formats_attr,
79 };
80
81 static const struct attribute_group *riscv_pmu_attr_groups[] = {
82 &riscv_pmu_format_group,
83 NULL,
84 };
85
86 /* Allow user mode access by default */
87 static int sysctl_perf_user_access __read_mostly = SYSCTL_USER_ACCESS;
88
89 /*
90 * RISC-V doesn't have heterogeneous harts yet. This need to be part of
91 * per_cpu in case of harts with different pmu counters
92 */
93 static union sbi_pmu_ctr_info *pmu_ctr_list;
94 static bool riscv_pmu_use_irq;
95 static unsigned int riscv_pmu_irq_num;
96 static unsigned int riscv_pmu_irq_mask;
97 static unsigned int riscv_pmu_irq;
98
99 /* Cache the available counters in a bitmask */
100 static unsigned long cmask;
101
102 static int pmu_event_find_cache(u64 config);
103 struct sbi_pmu_event_data {
104 union {
105 union {
106 struct hw_gen_event {
107 uint32_t event_code:16;
108 uint32_t event_type:4;
109 uint32_t reserved:12;
110 } hw_gen_event;
111 struct hw_cache_event {
112 uint32_t result_id:1;
113 uint32_t op_id:2;
114 uint32_t cache_id:13;
115 uint32_t event_type:4;
116 uint32_t reserved:12;
117 } hw_cache_event;
118 };
119 uint32_t event_idx;
120 };
121 };
122
123 static struct sbi_pmu_event_data pmu_hw_event_map[] = {
124 [PERF_COUNT_HW_CPU_CYCLES] = {.hw_gen_event = {
125 SBI_PMU_HW_CPU_CYCLES,
126 SBI_PMU_EVENT_TYPE_HW, 0}},
127 [PERF_COUNT_HW_INSTRUCTIONS] = {.hw_gen_event = {
128 SBI_PMU_HW_INSTRUCTIONS,
129 SBI_PMU_EVENT_TYPE_HW, 0}},
130 [PERF_COUNT_HW_CACHE_REFERENCES] = {.hw_gen_event = {
131 SBI_PMU_HW_CACHE_REFERENCES,
132 SBI_PMU_EVENT_TYPE_HW, 0}},
133 [PERF_COUNT_HW_CACHE_MISSES] = {.hw_gen_event = {
134 SBI_PMU_HW_CACHE_MISSES,
135 SBI_PMU_EVENT_TYPE_HW, 0}},
136 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {.hw_gen_event = {
137 SBI_PMU_HW_BRANCH_INSTRUCTIONS,
138 SBI_PMU_EVENT_TYPE_HW, 0}},
139 [PERF_COUNT_HW_BRANCH_MISSES] = {.hw_gen_event = {
140 SBI_PMU_HW_BRANCH_MISSES,
141 SBI_PMU_EVENT_TYPE_HW, 0}},
142 [PERF_COUNT_HW_BUS_CYCLES] = {.hw_gen_event = {
143 SBI_PMU_HW_BUS_CYCLES,
144 SBI_PMU_EVENT_TYPE_HW, 0}},
145 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = {.hw_gen_event = {
146 SBI_PMU_HW_STALLED_CYCLES_FRONTEND,
147 SBI_PMU_EVENT_TYPE_HW, 0}},
148 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = {.hw_gen_event = {
149 SBI_PMU_HW_STALLED_CYCLES_BACKEND,
150 SBI_PMU_EVENT_TYPE_HW, 0}},
151 [PERF_COUNT_HW_REF_CPU_CYCLES] = {.hw_gen_event = {
152 SBI_PMU_HW_REF_CPU_CYCLES,
153 SBI_PMU_EVENT_TYPE_HW, 0}},
154 };
155
156 #define C(x) PERF_COUNT_HW_CACHE_##x
157 static struct sbi_pmu_event_data pmu_cache_event_map[PERF_COUNT_HW_CACHE_MAX]
158 [PERF_COUNT_HW_CACHE_OP_MAX]
159 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
160 [C(L1D)] = {
161 [C(OP_READ)] = {
162 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
163 C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
164 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
165 C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
166 },
167 [C(OP_WRITE)] = {
168 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
169 C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
170 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
171 C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
172 },
173 [C(OP_PREFETCH)] = {
174 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
175 C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
176 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
177 C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
178 },
179 },
180 [C(L1I)] = {
181 [C(OP_READ)] = {
182 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
183 C(OP_READ), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
184 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), C(OP_READ),
185 C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
186 },
187 [C(OP_WRITE)] = {
188 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
189 C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
190 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
191 C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
192 },
193 [C(OP_PREFETCH)] = {
194 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
195 C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
196 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
197 C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
198 },
199 },
200 [C(LL)] = {
201 [C(OP_READ)] = {
202 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
203 C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
204 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
205 C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
206 },
207 [C(OP_WRITE)] = {
208 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
209 C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
210 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
211 C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
212 },
213 [C(OP_PREFETCH)] = {
214 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
215 C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
216 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
217 C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
218 },
219 },
220 [C(DTLB)] = {
221 [C(OP_READ)] = {
222 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
223 C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
224 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
225 C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
226 },
227 [C(OP_WRITE)] = {
228 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
229 C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
230 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
231 C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
232 },
233 [C(OP_PREFETCH)] = {
234 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
235 C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
236 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
237 C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
238 },
239 },
240 [C(ITLB)] = {
241 [C(OP_READ)] = {
242 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
243 C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
244 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
245 C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
246 },
247 [C(OP_WRITE)] = {
248 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
249 C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
250 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
251 C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
252 },
253 [C(OP_PREFETCH)] = {
254 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
255 C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
256 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
257 C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
258 },
259 },
260 [C(BPU)] = {
261 [C(OP_READ)] = {
262 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
263 C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
264 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
265 C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
266 },
267 [C(OP_WRITE)] = {
268 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
269 C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
270 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
271 C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
272 },
273 [C(OP_PREFETCH)] = {
274 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
275 C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
276 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
277 C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
278 },
279 },
280 [C(NODE)] = {
281 [C(OP_READ)] = {
282 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
283 C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
284 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
285 C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
286 },
287 [C(OP_WRITE)] = {
288 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
289 C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
290 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
291 C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
292 },
293 [C(OP_PREFETCH)] = {
294 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
295 C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
296 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
297 C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
298 },
299 },
300 };
301
pmu_sbi_check_event_info(void)302 static int pmu_sbi_check_event_info(void)
303 {
304 int num_events = ARRAY_SIZE(pmu_hw_event_map) + PERF_COUNT_HW_CACHE_MAX *
305 PERF_COUNT_HW_CACHE_OP_MAX * PERF_COUNT_HW_CACHE_RESULT_MAX;
306 struct riscv_pmu_event_info *event_info_shmem;
307 phys_addr_t base_addr;
308 int i, j, k, result = 0, count = 0;
309 struct sbiret ret;
310
311 event_info_shmem = kzalloc_objs(*event_info_shmem, num_events);
312 if (!event_info_shmem)
313 return -ENOMEM;
314
315 for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++)
316 event_info_shmem[count++].event_idx = pmu_hw_event_map[i].event_idx;
317
318 for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) {
319 for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) {
320 for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++)
321 event_info_shmem[count++].event_idx =
322 pmu_cache_event_map[i][j][k].event_idx;
323 }
324 }
325
326 base_addr = __pa(event_info_shmem);
327 if (IS_ENABLED(CONFIG_32BIT))
328 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, lower_32_bits(base_addr),
329 upper_32_bits(base_addr), count, 0, 0, 0);
330 else
331 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, base_addr, 0,
332 count, 0, 0, 0);
333 if (ret.error) {
334 result = -EOPNOTSUPP;
335 goto free_mem;
336 }
337
338 for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) {
339 if (!(event_info_shmem[i].output & RISCV_PMU_EVENT_INFO_OUTPUT_MASK))
340 pmu_hw_event_map[i].event_idx = -ENOENT;
341 }
342
343 count = ARRAY_SIZE(pmu_hw_event_map);
344
345 for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) {
346 for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) {
347 for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++) {
348 if (!(event_info_shmem[count].output &
349 RISCV_PMU_EVENT_INFO_OUTPUT_MASK))
350 pmu_cache_event_map[i][j][k].event_idx = -ENOENT;
351 count++;
352 }
353 }
354 }
355
356 free_mem:
357 kfree(event_info_shmem);
358
359 return result;
360 }
361
pmu_sbi_check_event(struct sbi_pmu_event_data * edata)362 static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata)
363 {
364 struct sbiret ret;
365
366 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH,
367 0, cmask, 0, edata->event_idx, 0, 0);
368 if (!ret.error) {
369 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
370 ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0);
371 } else if (ret.error == SBI_ERR_NOT_SUPPORTED) {
372 /* This event cannot be monitored by any counter */
373 edata->event_idx = -ENOENT;
374 }
375 }
376
pmu_sbi_check_std_events(struct work_struct * work)377 static void pmu_sbi_check_std_events(struct work_struct *work)
378 {
379 int ret;
380
381 if (sbi_v3_available) {
382 ret = pmu_sbi_check_event_info();
383 if (ret)
384 pr_err("pmu_sbi_check_event_info failed with error %d\n", ret);
385 return;
386 }
387
388 for (int i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++)
389 pmu_sbi_check_event(&pmu_hw_event_map[i]);
390
391 for (int i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++)
392 for (int j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++)
393 for (int k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++)
394 pmu_sbi_check_event(&pmu_cache_event_map[i][j][k]);
395 }
396
397 static DECLARE_WORK(check_std_events_work, pmu_sbi_check_std_events);
398
pmu_sbi_ctr_get_width(int idx)399 static int pmu_sbi_ctr_get_width(int idx)
400 {
401 return pmu_ctr_list[idx].width;
402 }
403
pmu_sbi_ctr_is_fw(int cidx)404 static bool pmu_sbi_ctr_is_fw(int cidx)
405 {
406 union sbi_pmu_ctr_info *info;
407
408 info = &pmu_ctr_list[cidx];
409 if (!info)
410 return false;
411
412 return info->type == SBI_PMU_CTR_TYPE_FW;
413 }
414
riscv_pmu_get_event_info(u32 type,u64 config,u64 * econfig)415 int riscv_pmu_get_event_info(u32 type, u64 config, u64 *econfig)
416 {
417 int ret = -ENOENT;
418
419 switch (type) {
420 case PERF_TYPE_HARDWARE:
421 if (config >= PERF_COUNT_HW_MAX)
422 return -EINVAL;
423 ret = pmu_hw_event_map[config].event_idx;
424 break;
425 case PERF_TYPE_HW_CACHE:
426 ret = pmu_event_find_cache(config);
427 break;
428 case PERF_TYPE_RAW:
429 /*
430 * As per SBI v0.3 specification,
431 * -- the upper 16 bits must be unused for a hardware raw event.
432 * As per SBI v2.0 specification,
433 * -- the upper 8 bits must be unused for a hardware raw event.
434 * Bits 63:62 are used to distinguish between raw events
435 * 00 - Hardware raw event
436 * 10 - SBI firmware events
437 * 11 - Risc-V platform specific firmware event
438 */
439 switch (config >> 62) {
440 case 0:
441 if (sbi_v3_available) {
442 /* Return error any bits [56-63] is set as it is not allowed by the spec */
443 if (!(config & ~RISCV_PMU_RAW_EVENT_V2_MASK)) {
444 if (econfig)
445 *econfig = config & RISCV_PMU_RAW_EVENT_V2_MASK;
446 ret = RISCV_PMU_RAW_EVENT_V2_IDX;
447 }
448 /* Return error any bits [48-63] is set as it is not allowed by the spec */
449 } else if (!(config & ~RISCV_PMU_RAW_EVENT_MASK)) {
450 if (econfig)
451 *econfig = config & RISCV_PMU_RAW_EVENT_MASK;
452 ret = RISCV_PMU_RAW_EVENT_IDX;
453 }
454 break;
455 case 2:
456 ret = (config & 0xFFFF) | (SBI_PMU_EVENT_TYPE_FW << 16);
457 break;
458 case 3:
459 /*
460 * For Risc-V platform specific firmware events
461 * Event code - 0xFFFF
462 * Event data - raw event encoding
463 */
464 ret = SBI_PMU_EVENT_TYPE_FW << 16 | RISCV_PLAT_FW_EVENT;
465 if (econfig)
466 *econfig = config & RISCV_PMU_PLAT_FW_EVENT_MASK;
467 break;
468 default:
469 break;
470 }
471 break;
472 default:
473 break;
474 }
475
476 return ret;
477 }
478 EXPORT_SYMBOL_GPL(riscv_pmu_get_event_info);
479
480 /*
481 * Returns the counter width of a programmable counter and number of hardware
482 * counters. As we don't support heterogeneous CPUs yet, it is okay to just
483 * return the counter width of the first programmable counter.
484 */
riscv_pmu_get_hpm_info(u32 * hw_ctr_width,u32 * num_hw_ctr)485 int riscv_pmu_get_hpm_info(u32 *hw_ctr_width, u32 *num_hw_ctr)
486 {
487 int i;
488 union sbi_pmu_ctr_info *info;
489 u32 hpm_width = 0, hpm_count = 0;
490
491 if (!cmask)
492 return -EINVAL;
493
494 for_each_set_bit(i, &cmask, RISCV_MAX_COUNTERS) {
495 info = &pmu_ctr_list[i];
496 if (!info)
497 continue;
498 if (!hpm_width && info->csr != CSR_CYCLE && info->csr != CSR_INSTRET)
499 hpm_width = info->width;
500 if (info->type == SBI_PMU_CTR_TYPE_HW)
501 hpm_count++;
502 }
503
504 *hw_ctr_width = hpm_width;
505 *num_hw_ctr = hpm_count;
506
507 return 0;
508 }
509 EXPORT_SYMBOL_GPL(riscv_pmu_get_hpm_info);
510
pmu_sbi_csr_index(struct perf_event * event)511 static uint8_t pmu_sbi_csr_index(struct perf_event *event)
512 {
513 return pmu_ctr_list[event->hw.idx].csr - CSR_CYCLE;
514 }
515
pmu_sbi_get_filter_flags(struct perf_event * event)516 static unsigned long pmu_sbi_get_filter_flags(struct perf_event *event)
517 {
518 unsigned long cflags = 0;
519 bool guest_events = false;
520
521 if (event->attr.config1 & RISCV_PMU_CONFIG1_GUEST_EVENTS)
522 guest_events = true;
523 if (event->attr.exclude_kernel)
524 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VSINH : SBI_PMU_CFG_FLAG_SET_SINH;
525 if (event->attr.exclude_user)
526 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VUINH : SBI_PMU_CFG_FLAG_SET_UINH;
527 if (guest_events && event->attr.exclude_hv)
528 cflags |= SBI_PMU_CFG_FLAG_SET_SINH;
529 if (event->attr.exclude_host)
530 cflags |= SBI_PMU_CFG_FLAG_SET_UINH | SBI_PMU_CFG_FLAG_SET_SINH;
531 if (event->attr.exclude_guest)
532 cflags |= SBI_PMU_CFG_FLAG_SET_VSINH | SBI_PMU_CFG_FLAG_SET_VUINH;
533
534 return cflags;
535 }
536
pmu_sbi_ctr_get_idx(struct perf_event * event)537 static int pmu_sbi_ctr_get_idx(struct perf_event *event)
538 {
539 struct hw_perf_event *hwc = &event->hw;
540 struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
541 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
542 struct sbiret ret;
543 int idx;
544 uint64_t cbase = 0, cmask = rvpmu->cmask;
545 unsigned long cflags = 0;
546
547 cflags = pmu_sbi_get_filter_flags(event);
548
549 /*
550 * In legacy mode, we have to force the fixed counters for those events
551 * but not in the user access mode as we want to use the other counters
552 * that support sampling/filtering.
553 */
554 if ((hwc->flags & PERF_EVENT_FLAG_LEGACY) && (event->attr.type == PERF_TYPE_HARDWARE)) {
555 if (event->attr.config == PERF_COUNT_HW_CPU_CYCLES) {
556 cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH;
557 cmask = 1;
558 } else if (event->attr.config == PERF_COUNT_HW_INSTRUCTIONS) {
559 cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH;
560 cmask = BIT(CSR_INSTRET - CSR_CYCLE);
561 }
562 }
563
564 /* retrieve the available counter index */
565 #if defined(CONFIG_32BIT)
566 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase,
567 cmask, cflags, hwc->event_base, hwc->config,
568 hwc->config >> 32);
569 #else
570 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase,
571 cmask, cflags, hwc->event_base, hwc->config, 0);
572 #endif
573 if (ret.error) {
574 pr_debug("Not able to find a counter for event %lx config %llx\n",
575 hwc->event_base, hwc->config);
576 return sbi_err_map_linux_errno(ret.error);
577 }
578
579 idx = ret.value;
580 if (!test_bit(idx, &rvpmu->cmask) || !pmu_ctr_list[idx].value)
581 return -ENOENT;
582
583 /* Additional sanity check for the counter id */
584 if (pmu_sbi_ctr_is_fw(idx)) {
585 if (!test_and_set_bit(idx, cpuc->used_fw_ctrs))
586 return idx;
587 } else {
588 if (!test_and_set_bit(idx, cpuc->used_hw_ctrs))
589 return idx;
590 }
591
592 return -ENOENT;
593 }
594
pmu_sbi_ctr_clear_idx(struct perf_event * event)595 static void pmu_sbi_ctr_clear_idx(struct perf_event *event)
596 {
597
598 struct hw_perf_event *hwc = &event->hw;
599 struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
600 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
601 int idx = hwc->idx;
602
603 if (pmu_sbi_ctr_is_fw(idx))
604 clear_bit(idx, cpuc->used_fw_ctrs);
605 else
606 clear_bit(idx, cpuc->used_hw_ctrs);
607 }
608
pmu_event_find_cache(u64 config)609 static int pmu_event_find_cache(u64 config)
610 {
611 unsigned int cache_type, cache_op, cache_result, ret;
612
613 cache_type = (config >> 0) & 0xff;
614 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
615 return -EINVAL;
616
617 cache_op = (config >> 8) & 0xff;
618 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
619 return -EINVAL;
620
621 cache_result = (config >> 16) & 0xff;
622 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
623 return -EINVAL;
624
625 ret = pmu_cache_event_map[cache_type][cache_op][cache_result].event_idx;
626
627 return ret;
628 }
629
pmu_sbi_is_fw_event(struct perf_event * event)630 static bool pmu_sbi_is_fw_event(struct perf_event *event)
631 {
632 u32 type = event->attr.type;
633 u64 config = event->attr.config;
634
635 if ((type == PERF_TYPE_RAW) && ((config >> 63) == 1))
636 return true;
637 else
638 return false;
639 }
640
pmu_sbi_event_map(struct perf_event * event,u64 * econfig)641 static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig)
642 {
643 u32 type = event->attr.type;
644 u64 config = event->attr.config;
645
646 /*
647 * Ensure we are finished checking standard hardware events for
648 * validity before allowing userspace to configure any events.
649 */
650 flush_work(&check_std_events_work);
651
652 return riscv_pmu_get_event_info(type, config, econfig);
653 }
654
pmu_sbi_snapshot_free(struct riscv_pmu * pmu)655 static void pmu_sbi_snapshot_free(struct riscv_pmu *pmu)
656 {
657 int cpu;
658
659 for_each_possible_cpu(cpu) {
660 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
661
662 if (!cpu_hw_evt->snapshot_addr)
663 continue;
664
665 free_page((unsigned long)cpu_hw_evt->snapshot_addr);
666 cpu_hw_evt->snapshot_addr = NULL;
667 cpu_hw_evt->snapshot_addr_phys = 0;
668 }
669 }
670
pmu_sbi_snapshot_alloc(struct riscv_pmu * pmu)671 static int pmu_sbi_snapshot_alloc(struct riscv_pmu *pmu)
672 {
673 int cpu;
674 struct page *snapshot_page;
675
676 for_each_possible_cpu(cpu) {
677 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
678
679 snapshot_page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
680 if (!snapshot_page) {
681 pmu_sbi_snapshot_free(pmu);
682 return -ENOMEM;
683 }
684 cpu_hw_evt->snapshot_addr = page_to_virt(snapshot_page);
685 cpu_hw_evt->snapshot_addr_phys = page_to_phys(snapshot_page);
686 }
687
688 return 0;
689 }
690
pmu_sbi_snapshot_disable(void)691 static int pmu_sbi_snapshot_disable(void)
692 {
693 struct sbiret ret;
694
695 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, SBI_SHMEM_DISABLE,
696 SBI_SHMEM_DISABLE, 0, 0, 0, 0);
697 if (ret.error) {
698 pr_warn("failed to disable snapshot shared memory\n");
699 return sbi_err_map_linux_errno(ret.error);
700 }
701
702 return 0;
703 }
704
pmu_sbi_snapshot_setup(struct riscv_pmu * pmu,int cpu)705 static int pmu_sbi_snapshot_setup(struct riscv_pmu *pmu, int cpu)
706 {
707 struct cpu_hw_events *cpu_hw_evt;
708 struct sbiret ret = {0};
709
710 cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
711 if (!cpu_hw_evt->snapshot_addr_phys)
712 return -EINVAL;
713
714 if (cpu_hw_evt->snapshot_set_done)
715 return 0;
716
717 if (IS_ENABLED(CONFIG_32BIT))
718 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM,
719 cpu_hw_evt->snapshot_addr_phys,
720 (u64)(cpu_hw_evt->snapshot_addr_phys) >> 32, 0, 0, 0, 0);
721 else
722 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM,
723 cpu_hw_evt->snapshot_addr_phys, 0, 0, 0, 0, 0);
724
725 /* Free up the snapshot area memory and fall back to SBI PMU calls without snapshot */
726 if (ret.error) {
727 if (ret.error != SBI_ERR_NOT_SUPPORTED)
728 pr_warn("pmu snapshot setup failed with error %ld\n", ret.error);
729 return sbi_err_map_linux_errno(ret.error);
730 }
731
732 memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS);
733 cpu_hw_evt->snapshot_set_done = true;
734
735 return 0;
736 }
737
pmu_sbi_ctr_read(struct perf_event * event)738 static u64 pmu_sbi_ctr_read(struct perf_event *event)
739 {
740 struct hw_perf_event *hwc = &event->hw;
741 int idx = hwc->idx;
742 struct sbiret ret;
743 u64 val = 0;
744 struct riscv_pmu *pmu = to_riscv_pmu(event->pmu);
745 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
746 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
747 union sbi_pmu_ctr_info info = pmu_ctr_list[idx];
748
749 /* Read the value from the shared memory directly only if counter is stopped */
750 if (sbi_pmu_snapshot_available() && (hwc->state & PERF_HES_STOPPED)) {
751 val = sdata->ctr_values[idx];
752 return val;
753 }
754
755 if (pmu_sbi_is_fw_event(event)) {
756 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ,
757 hwc->idx, 0, 0, 0, 0, 0);
758 if (ret.error)
759 return 0;
760
761 val = ret.value;
762 if (IS_ENABLED(CONFIG_32BIT) && sbi_v2_available && info.width >= 32) {
763 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ_HI,
764 hwc->idx, 0, 0, 0, 0, 0);
765 if (!ret.error)
766 val |= ((u64)ret.value << 32);
767 else
768 WARN_ONCE(1, "Unable to read upper 32 bits of firmware counter error: %ld\n",
769 ret.error);
770 }
771 } else {
772 val = riscv_pmu_ctr_read_csr(info.csr);
773 if (IS_ENABLED(CONFIG_32BIT))
774 val |= ((u64)riscv_pmu_ctr_read_csr(info.csr + 0x80)) << 32;
775 }
776
777 return val;
778 }
779
pmu_sbi_set_scounteren(void * arg)780 static void pmu_sbi_set_scounteren(void *arg)
781 {
782 struct perf_event *event = (struct perf_event *)arg;
783
784 if (event->hw.idx != -1)
785 csr_write(CSR_SCOUNTEREN,
786 csr_read(CSR_SCOUNTEREN) | BIT(pmu_sbi_csr_index(event)));
787 }
788
pmu_sbi_reset_scounteren(void * arg)789 static void pmu_sbi_reset_scounteren(void *arg)
790 {
791 struct perf_event *event = (struct perf_event *)arg;
792
793 if (event->hw.idx != -1)
794 csr_write(CSR_SCOUNTEREN,
795 csr_read(CSR_SCOUNTEREN) & ~BIT(pmu_sbi_csr_index(event)));
796 }
797
pmu_sbi_ctr_start(struct perf_event * event,u64 ival)798 static void pmu_sbi_ctr_start(struct perf_event *event, u64 ival)
799 {
800 struct sbiret ret;
801 struct hw_perf_event *hwc = &event->hw;
802 unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
803
804 /* There is no benefit setting SNAPSHOT FLAG for a single counter */
805 #if defined(CONFIG_32BIT)
806 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
807 1, flag, ival, ival >> 32, 0);
808 #else
809 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
810 1, flag, ival, 0, 0);
811 #endif
812 if (ret.error && (ret.error != SBI_ERR_ALREADY_STARTED))
813 pr_err("Starting counter idx %d failed with error %d\n",
814 hwc->idx, sbi_err_map_linux_errno(ret.error));
815
816 if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) &&
817 (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT))
818 pmu_sbi_set_scounteren((void *)event);
819 }
820
pmu_sbi_ctr_stop(struct perf_event * event,unsigned long flag)821 static void pmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag)
822 {
823 struct sbiret ret;
824 struct hw_perf_event *hwc = &event->hw;
825 struct riscv_pmu *pmu = to_riscv_pmu(event->pmu);
826 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
827 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
828
829 if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) &&
830 (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT))
831 pmu_sbi_reset_scounteren((void *)event);
832
833 if (sbi_pmu_snapshot_available())
834 flag |= SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
835
836 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, hwc->idx, 1, flag, 0, 0, 0);
837 if (!ret.error && sbi_pmu_snapshot_available()) {
838 /*
839 * The counter snapshot is based on the index base specified by hwc->idx.
840 * The actual counter value is updated in shared memory at index 0 when counter
841 * mask is 0x01. To ensure accurate counter values, it's necessary to transfer
842 * the counter value to shared memory. However, if hwc->idx is zero, the counter
843 * value is already correctly updated in shared memory, requiring no further
844 * adjustment.
845 */
846 if (hwc->idx > 0) {
847 sdata->ctr_values[hwc->idx] = sdata->ctr_values[0];
848 sdata->ctr_values[0] = 0;
849 }
850 } else if (ret.error && (ret.error != SBI_ERR_ALREADY_STOPPED) &&
851 flag != SBI_PMU_STOP_FLAG_RESET) {
852 pr_err("Stopping counter idx %d failed with error %d\n",
853 hwc->idx, sbi_err_map_linux_errno(ret.error));
854 }
855 }
856
pmu_sbi_find_num_ctrs(void)857 static int pmu_sbi_find_num_ctrs(void)
858 {
859 struct sbiret ret;
860
861 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS, 0, 0, 0, 0, 0, 0);
862 if (!ret.error)
863 return ret.value;
864 else
865 return sbi_err_map_linux_errno(ret.error);
866 }
867
pmu_sbi_get_ctrinfo(int nctr,unsigned long * mask)868 static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask)
869 {
870 struct sbiret ret;
871 int i, num_hw_ctr = 0, num_fw_ctr = 0;
872 union sbi_pmu_ctr_info cinfo;
873
874 pmu_ctr_list = kzalloc_objs(*pmu_ctr_list, nctr);
875 if (!pmu_ctr_list)
876 return -ENOMEM;
877
878 for (i = 0; i < nctr; i++) {
879 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0);
880 if (ret.error)
881 /* The logical counter ids are not expected to be contiguous */
882 continue;
883
884 *mask |= BIT(i);
885
886 cinfo.value = ret.value;
887 if (cinfo.type == SBI_PMU_CTR_TYPE_FW)
888 num_fw_ctr++;
889 else
890 num_hw_ctr++;
891 pmu_ctr_list[i].value = cinfo.value;
892 }
893
894 pr_info("%d firmware and %d hardware counters\n", num_fw_ctr, num_hw_ctr);
895
896 return 0;
897 }
898
pmu_sbi_stop_all(struct riscv_pmu * pmu)899 static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu)
900 {
901 /*
902 * No need to check the error because we are disabling all the counters
903 * which may include counters that are not enabled yet.
904 */
905 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
906 0, pmu->cmask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0);
907 }
908
pmu_sbi_stop_hw_ctrs(struct riscv_pmu * pmu)909 static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu)
910 {
911 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
912 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
913 unsigned long flag = 0;
914 int i, idx;
915 struct sbiret ret;
916 u64 temp_ctr_overflow_mask = 0;
917
918 if (sbi_pmu_snapshot_available())
919 flag = SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
920
921 /* Reset the shadow copy to avoid save/restore any value from previous overflow */
922 memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS);
923
924 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
925 /* No need to check the error here as we can't do anything about the error */
926 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, i * BITS_PER_LONG,
927 cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0);
928 if (!ret.error && sbi_pmu_snapshot_available()) {
929 /* Save the counter values to avoid clobbering */
930 for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG)
931 cpu_hw_evt->snapshot_cval_shcopy[i * BITS_PER_LONG + idx] =
932 sdata->ctr_values[idx];
933 /* Save the overflow mask to avoid clobbering */
934 temp_ctr_overflow_mask |= sdata->ctr_overflow_mask << (i * BITS_PER_LONG);
935 }
936 }
937
938 /* Restore the counter values to the shared memory for used hw counters */
939 if (sbi_pmu_snapshot_available()) {
940 for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS)
941 sdata->ctr_values[idx] = cpu_hw_evt->snapshot_cval_shcopy[idx];
942 if (temp_ctr_overflow_mask)
943 sdata->ctr_overflow_mask = temp_ctr_overflow_mask;
944 }
945 }
946
947 /*
948 * This function starts all the used counters in two step approach.
949 * Any counter that did not overflow can be start in a single step
950 * while the overflowed counters need to be started with updated initialization
951 * value.
952 */
pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events * cpu_hw_evt,u64 ctr_ovf_mask)953 static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt,
954 u64 ctr_ovf_mask)
955 {
956 int idx = 0, i;
957 struct perf_event *event;
958 unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
959 unsigned long ctr_start_mask = 0;
960 uint64_t max_period;
961 struct hw_perf_event *hwc;
962 u64 init_val = 0;
963
964 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
965 ctr_start_mask = cpu_hw_evt->used_hw_ctrs[i] & ~ctr_ovf_mask;
966 /* Start all the counters that did not overflow in a single shot */
967 if (ctr_start_mask) {
968 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, i * BITS_PER_LONG,
969 ctr_start_mask, 0, 0, 0, 0);
970 }
971 }
972
973 /* Reinitialize and start all the counter that overflowed */
974 while (ctr_ovf_mask) {
975 if (ctr_ovf_mask & 0x01) {
976 event = cpu_hw_evt->events[idx];
977 hwc = &event->hw;
978 max_period = riscv_pmu_ctr_get_width_mask(event);
979 init_val = local64_read(&hwc->prev_count) & max_period;
980 #if defined(CONFIG_32BIT)
981 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
982 flag, init_val, init_val >> 32, 0);
983 #else
984 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
985 flag, init_val, 0, 0);
986 #endif
987 perf_event_update_userpage(event);
988 }
989 ctr_ovf_mask = ctr_ovf_mask >> 1;
990 idx++;
991 }
992 }
993
pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events * cpu_hw_evt,u64 ctr_ovf_mask)994 static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_evt,
995 u64 ctr_ovf_mask)
996 {
997 int i, idx = 0;
998 struct perf_event *event;
999 unsigned long flag = SBI_PMU_START_FLAG_INIT_SNAPSHOT;
1000 u64 max_period, init_val = 0;
1001 struct hw_perf_event *hwc;
1002 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
1003
1004 for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
1005 if (ctr_ovf_mask & BIT(idx)) {
1006 event = cpu_hw_evt->events[idx];
1007 hwc = &event->hw;
1008 max_period = riscv_pmu_ctr_get_width_mask(event);
1009 init_val = local64_read(&hwc->prev_count) & max_period;
1010 cpu_hw_evt->snapshot_cval_shcopy[idx] = init_val;
1011 }
1012 /*
1013 * We do not need to update the non-overflow counters the previous
1014 * value should have been there already.
1015 */
1016 }
1017
1018 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
1019 /* Restore the counter values to relative indices for used hw counters */
1020 for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG)
1021 sdata->ctr_values[idx] =
1022 cpu_hw_evt->snapshot_cval_shcopy[idx + i * BITS_PER_LONG];
1023 /* Start all the counters in a single shot */
1024 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx * BITS_PER_LONG,
1025 cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0);
1026 }
1027 }
1028
pmu_sbi_start_overflow_mask(struct riscv_pmu * pmu,u64 ctr_ovf_mask)1029 static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu,
1030 u64 ctr_ovf_mask)
1031 {
1032 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
1033
1034 if (sbi_pmu_snapshot_available())
1035 pmu_sbi_start_ovf_ctrs_snapshot(cpu_hw_evt, ctr_ovf_mask);
1036 else
1037 pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask);
1038 }
1039
pmu_sbi_ovf_handler(int irq,void * dev)1040 static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev)
1041 {
1042 struct perf_sample_data data;
1043 struct pt_regs *regs;
1044 struct hw_perf_event *hw_evt;
1045 union sbi_pmu_ctr_info *info;
1046 int lidx, hidx, fidx;
1047 struct riscv_pmu *pmu;
1048 struct perf_event *event;
1049 u64 overflow;
1050 u64 overflowed_ctrs = 0;
1051 struct cpu_hw_events *cpu_hw_evt = dev;
1052 u64 start_clock = sched_clock();
1053 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
1054
1055 if (WARN_ON_ONCE(!cpu_hw_evt))
1056 return IRQ_NONE;
1057
1058 /* Firmware counter don't support overflow yet */
1059 fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS);
1060 if (fidx == RISCV_MAX_COUNTERS) {
1061 csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num));
1062 return IRQ_NONE;
1063 }
1064
1065 event = cpu_hw_evt->events[fidx];
1066 if (!event) {
1067 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
1068 return IRQ_NONE;
1069 }
1070
1071 pmu = to_riscv_pmu(event->pmu);
1072 pmu_sbi_stop_hw_ctrs(pmu);
1073
1074 /* Overflow status register should only be read after counter are stopped */
1075 if (sbi_pmu_snapshot_available())
1076 overflow = sdata->ctr_overflow_mask;
1077 else
1078 ALT_SBI_PMU_OVERFLOW(overflow);
1079
1080 /*
1081 * Overflow interrupt pending bit should only be cleared after stopping
1082 * all the counters to avoid any race condition.
1083 */
1084 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
1085
1086 /* No overflow bit is set */
1087 if (!overflow)
1088 return IRQ_NONE;
1089
1090 regs = get_irq_regs();
1091
1092 for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
1093 struct perf_event *event = cpu_hw_evt->events[lidx];
1094
1095 /* Skip if invalid event or user did not request a sampling */
1096 if (!event || !is_sampling_event(event))
1097 continue;
1098
1099 info = &pmu_ctr_list[lidx];
1100 /* Do a sanity check */
1101 if (!info || info->type != SBI_PMU_CTR_TYPE_HW)
1102 continue;
1103
1104 if (sbi_pmu_snapshot_available())
1105 /* SBI implementation already updated the logical indicies */
1106 hidx = lidx;
1107 else
1108 /* compute hardware counter index */
1109 hidx = info->csr - CSR_CYCLE;
1110
1111 /* check if the corresponding bit is set in scountovf or overflow mask in shmem */
1112 if (!(overflow & BIT(hidx)))
1113 continue;
1114
1115 /*
1116 * Keep a track of overflowed counters so that they can be started
1117 * with updated initial value.
1118 */
1119 overflowed_ctrs |= BIT(lidx);
1120 hw_evt = &event->hw;
1121 /* Update the event states here so that we know the state while reading */
1122 hw_evt->state |= PERF_HES_STOPPED;
1123 riscv_pmu_event_update(event);
1124 hw_evt->state |= PERF_HES_UPTODATE;
1125 perf_sample_data_init(&data, 0, hw_evt->last_period);
1126 if (riscv_pmu_event_set_period(event)) {
1127 /*
1128 * Unlike other ISAs, RISC-V don't have to disable interrupts
1129 * to avoid throttling here. As per the specification, the
1130 * interrupt remains disabled until the OF bit is set.
1131 * Interrupts are enabled again only during the start.
1132 * TODO: We will need to stop the guest counters once
1133 * virtualization support is added.
1134 */
1135 perf_event_overflow(event, &data, regs);
1136 }
1137 /* Reset the state as we are going to start the counter after the loop */
1138 hw_evt->state = 0;
1139 }
1140
1141 pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs);
1142 perf_sample_event_took(sched_clock() - start_clock);
1143
1144 return IRQ_HANDLED;
1145 }
1146
pmu_sbi_starting_cpu(unsigned int cpu,struct hlist_node * node)1147 static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node)
1148 {
1149 struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node);
1150 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
1151
1152 /*
1153 * We keep enabling userspace access to CYCLE, TIME and INSTRET via the
1154 * legacy option but that will be removed in the future.
1155 */
1156 if (sysctl_perf_user_access == SYSCTL_LEGACY)
1157 csr_write(CSR_SCOUNTEREN, 0x7);
1158 else
1159 csr_write(CSR_SCOUNTEREN, 0x2);
1160
1161 /* Stop all the counters so that they can be enabled from perf */
1162 pmu_sbi_stop_all(pmu);
1163
1164 if (riscv_pmu_use_irq) {
1165 cpu_hw_evt->irq = riscv_pmu_irq;
1166 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
1167 enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE);
1168 }
1169
1170 if (sbi_pmu_snapshot_available())
1171 return pmu_sbi_snapshot_setup(pmu, cpu);
1172
1173 return 0;
1174 }
1175
pmu_sbi_dying_cpu(unsigned int cpu,struct hlist_node * node)1176 static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node)
1177 {
1178 if (riscv_pmu_use_irq) {
1179 disable_percpu_irq(riscv_pmu_irq);
1180 }
1181
1182 /* Disable all counters access for user mode now */
1183 csr_write(CSR_SCOUNTEREN, 0x0);
1184
1185 if (sbi_pmu_snapshot_available())
1186 return pmu_sbi_snapshot_disable();
1187
1188 return 0;
1189 }
1190
pmu_sbi_setup_irqs(struct riscv_pmu * pmu,struct platform_device * pdev)1191 static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev)
1192 {
1193 int ret;
1194 struct cpu_hw_events __percpu *hw_events = pmu->hw_events;
1195 struct irq_domain *domain = NULL;
1196
1197 if (riscv_isa_extension_available(NULL, SSCOFPMF)) {
1198 riscv_pmu_irq_num = RV_IRQ_PMU;
1199 riscv_pmu_use_irq = true;
1200 } else if (IS_ENABLED(CONFIG_ERRATA_THEAD_PMU) &&
1201 riscv_cached_mvendorid(0) == THEAD_VENDOR_ID &&
1202 riscv_cached_marchid(0) == 0 &&
1203 riscv_cached_mimpid(0) == 0) {
1204 riscv_pmu_irq_num = THEAD_C9XX_RV_IRQ_PMU;
1205 riscv_pmu_use_irq = true;
1206 } else if (riscv_has_vendor_extension_unlikely(ANDES_VENDOR_ID,
1207 RISCV_ISA_VENDOR_EXT_XANDESPMU) &&
1208 IS_ENABLED(CONFIG_ANDES_CUSTOM_PMU)) {
1209 riscv_pmu_irq_num = ANDES_SLI_CAUSE_BASE + ANDES_RV_IRQ_PMOVI;
1210 riscv_pmu_use_irq = true;
1211 }
1212
1213 riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG);
1214
1215 if (!riscv_pmu_use_irq)
1216 return -EOPNOTSUPP;
1217
1218 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
1219 DOMAIN_BUS_ANY);
1220 if (!domain) {
1221 pr_err("Failed to find INTC IRQ root domain\n");
1222 ret = -ENODEV;
1223 goto err;
1224 }
1225
1226 riscv_pmu_irq = irq_create_mapping(domain, riscv_pmu_irq_num);
1227 if (!riscv_pmu_irq) {
1228 pr_err("Failed to map PMU interrupt for node\n");
1229 ret = -ENODEV;
1230 goto err;
1231 }
1232
1233 ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events);
1234 if (ret) {
1235 pr_err("registering percpu irq failed [%d]\n", ret);
1236 irq_dispose_mapping(riscv_pmu_irq);
1237 riscv_pmu_irq = 0;
1238 goto err;
1239 }
1240
1241 return 0;
1242 err:
1243 riscv_pmu_use_irq = false;
1244 return ret;
1245 }
1246
1247 #ifdef CONFIG_CPU_PM
riscv_pm_pmu_notify(struct notifier_block * b,unsigned long cmd,void * v)1248 static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd,
1249 void *v)
1250 {
1251 struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, riscv_pm_nb);
1252 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
1253 bool enabled = !bitmap_empty(cpuc->used_hw_ctrs, RISCV_MAX_COUNTERS);
1254 struct perf_event *event;
1255 int idx;
1256
1257 if (!enabled)
1258 return NOTIFY_OK;
1259
1260 for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) {
1261 event = cpuc->events[idx];
1262 if (!event)
1263 continue;
1264
1265 switch (cmd) {
1266 case CPU_PM_ENTER:
1267 /*
1268 * Stop and update the counter
1269 */
1270 riscv_pmu_stop(event, PERF_EF_UPDATE);
1271 break;
1272 case CPU_PM_EXIT:
1273 case CPU_PM_ENTER_FAILED:
1274 /*
1275 * Restore and enable the counter.
1276 */
1277 riscv_pmu_start(event, PERF_EF_RELOAD);
1278 break;
1279 default:
1280 break;
1281 }
1282 }
1283
1284 return NOTIFY_OK;
1285 }
1286
riscv_pm_pmu_register(struct riscv_pmu * pmu)1287 static int riscv_pm_pmu_register(struct riscv_pmu *pmu)
1288 {
1289 pmu->riscv_pm_nb.notifier_call = riscv_pm_pmu_notify;
1290 return cpu_pm_register_notifier(&pmu->riscv_pm_nb);
1291 }
1292
riscv_pm_pmu_unregister(struct riscv_pmu * pmu)1293 static void riscv_pm_pmu_unregister(struct riscv_pmu *pmu)
1294 {
1295 cpu_pm_unregister_notifier(&pmu->riscv_pm_nb);
1296 }
1297 #else
riscv_pm_pmu_register(struct riscv_pmu * pmu)1298 static inline int riscv_pm_pmu_register(struct riscv_pmu *pmu) { return 0; }
riscv_pm_pmu_unregister(struct riscv_pmu * pmu)1299 static inline void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) { }
1300 #endif
1301
riscv_pmu_destroy(struct riscv_pmu * pmu)1302 static void riscv_pmu_destroy(struct riscv_pmu *pmu)
1303 {
1304 if (sbi_v2_available) {
1305 if (sbi_pmu_snapshot_available()) {
1306 pmu_sbi_snapshot_disable();
1307 pmu_sbi_snapshot_free(pmu);
1308 }
1309 }
1310 riscv_pm_pmu_unregister(pmu);
1311 if (!hlist_unhashed(&pmu->node))
1312 cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
1313 }
1314
pmu_sbi_event_init(struct perf_event * event)1315 static void pmu_sbi_event_init(struct perf_event *event)
1316 {
1317 /*
1318 * The permissions are set at event_init so that we do not depend
1319 * on the sysctl value that can change.
1320 */
1321 if (sysctl_perf_user_access == SYSCTL_NO_USER_ACCESS)
1322 event->hw.flags |= PERF_EVENT_FLAG_NO_USER_ACCESS;
1323 else if (sysctl_perf_user_access == SYSCTL_USER_ACCESS)
1324 event->hw.flags |= PERF_EVENT_FLAG_USER_ACCESS;
1325 else
1326 event->hw.flags |= PERF_EVENT_FLAG_LEGACY;
1327 }
1328
pmu_sbi_event_mapped(struct perf_event * event,struct mm_struct * mm)1329 static void pmu_sbi_event_mapped(struct perf_event *event, struct mm_struct *mm)
1330 {
1331 if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS)
1332 return;
1333
1334 if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) {
1335 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES &&
1336 event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) {
1337 return;
1338 }
1339 }
1340
1341 /*
1342 * The user mmapped the event to directly access it: this is where
1343 * we determine based on sysctl_perf_user_access if we grant userspace
1344 * the direct access to this event. That means that within the same
1345 * task, some events may be directly accessible and some other may not,
1346 * if the user changes the value of sysctl_perf_user_accesss in the
1347 * meantime.
1348 */
1349
1350 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;
1351
1352 /*
1353 * We must enable userspace access *before* advertising in the user page
1354 * that it is possible to do so to avoid any race.
1355 * And we must notify all cpus here because threads that currently run
1356 * on other cpus will try to directly access the counter too without
1357 * calling pmu_sbi_ctr_start.
1358 */
1359 if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS)
1360 on_each_cpu_mask(mm_cpumask(mm),
1361 pmu_sbi_set_scounteren, (void *)event, 1);
1362 }
1363
pmu_sbi_event_unmapped(struct perf_event * event,struct mm_struct * mm)1364 static void pmu_sbi_event_unmapped(struct perf_event *event, struct mm_struct *mm)
1365 {
1366 if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS)
1367 return;
1368
1369 if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) {
1370 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES &&
1371 event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) {
1372 return;
1373 }
1374 }
1375
1376 /*
1377 * Here we can directly remove user access since the user does not have
1378 * access to the user page anymore so we avoid the racy window where the
1379 * user could have read cap_user_rdpmc to true right before we disable
1380 * it.
1381 */
1382 event->hw.flags &= ~PERF_EVENT_FLAG_USER_READ_CNT;
1383
1384 if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS)
1385 on_each_cpu_mask(mm_cpumask(mm),
1386 pmu_sbi_reset_scounteren, (void *)event, 1);
1387 }
1388
riscv_pmu_update_counter_access(void * info)1389 static void riscv_pmu_update_counter_access(void *info)
1390 {
1391 if (sysctl_perf_user_access == SYSCTL_LEGACY)
1392 csr_write(CSR_SCOUNTEREN, 0x7);
1393 else
1394 csr_write(CSR_SCOUNTEREN, 0x2);
1395 }
1396
riscv_pmu_proc_user_access_handler(const struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)1397 static int riscv_pmu_proc_user_access_handler(const struct ctl_table *table,
1398 int write, void *buffer,
1399 size_t *lenp, loff_t *ppos)
1400 {
1401 int prev = sysctl_perf_user_access;
1402 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
1403
1404 /*
1405 * Test against the previous value since we clear SCOUNTEREN when
1406 * sysctl_perf_user_access is set to SYSCTL_USER_ACCESS, but we should
1407 * not do that if that was already the case.
1408 */
1409 if (ret || !write || prev == sysctl_perf_user_access)
1410 return ret;
1411
1412 on_each_cpu(riscv_pmu_update_counter_access, NULL, 1);
1413
1414 return 0;
1415 }
1416
1417 static const struct ctl_table sbi_pmu_sysctl_table[] = {
1418 {
1419 .procname = "perf_user_access",
1420 .data = &sysctl_perf_user_access,
1421 .maxlen = sizeof(unsigned int),
1422 .mode = 0644,
1423 .proc_handler = riscv_pmu_proc_user_access_handler,
1424 .extra1 = SYSCTL_ZERO,
1425 .extra2 = SYSCTL_TWO,
1426 },
1427 };
1428
pmu_sbi_device_probe(struct platform_device * pdev)1429 static int pmu_sbi_device_probe(struct platform_device *pdev)
1430 {
1431 struct riscv_pmu *pmu = NULL;
1432 int ret = -ENODEV;
1433 int num_counters;
1434 bool irq_requested = false;
1435
1436 pr_info("SBI PMU extension is available\n");
1437 pmu = riscv_pmu_alloc();
1438 if (!pmu)
1439 return -ENOMEM;
1440
1441 num_counters = pmu_sbi_find_num_ctrs();
1442 if (num_counters < 0) {
1443 pr_err("SBI PMU extension doesn't provide any counters\n");
1444 goto out_free;
1445 }
1446
1447 /* It is possible to get from SBI more than max number of counters */
1448 if (num_counters > RISCV_MAX_COUNTERS) {
1449 num_counters = RISCV_MAX_COUNTERS;
1450 pr_info("SBI returned more than maximum number of counters. Limiting the number of counters to %d\n", num_counters);
1451 }
1452
1453 /* cache all the information about counters now */
1454 if (pmu_sbi_get_ctrinfo(num_counters, &cmask))
1455 goto out_free;
1456
1457 ret = pmu_sbi_setup_irqs(pmu, pdev);
1458 if (ret < 0) {
1459 pr_info("Perf sampling/filtering is not supported as sscof extension is not available\n");
1460 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1461 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE;
1462 }
1463 irq_requested = (ret == 0);
1464
1465 pmu->pmu.attr_groups = riscv_pmu_attr_groups;
1466 pmu->pmu.parent = &pdev->dev;
1467 pmu->cmask = cmask;
1468 pmu->ctr_start = pmu_sbi_ctr_start;
1469 pmu->ctr_stop = pmu_sbi_ctr_stop;
1470 pmu->event_map = pmu_sbi_event_map;
1471 pmu->ctr_get_idx = pmu_sbi_ctr_get_idx;
1472 pmu->ctr_get_width = pmu_sbi_ctr_get_width;
1473 pmu->ctr_clear_idx = pmu_sbi_ctr_clear_idx;
1474 pmu->ctr_read = pmu_sbi_ctr_read;
1475 pmu->event_init = pmu_sbi_event_init;
1476 pmu->event_mapped = pmu_sbi_event_mapped;
1477 pmu->event_unmapped = pmu_sbi_event_unmapped;
1478 pmu->csr_index = pmu_sbi_csr_index;
1479
1480 ret = riscv_pm_pmu_register(pmu);
1481 if (ret)
1482 goto out_destroy;
1483
1484 ret = perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW);
1485 if (ret)
1486 goto out_destroy;
1487
1488 /* SBI PMU Snapsphot is only available in SBI v2.0 */
1489 if (sbi_v2_available) {
1490 int cpu;
1491
1492 ret = pmu_sbi_snapshot_alloc(pmu);
1493 if (ret)
1494 goto out_unregister;
1495
1496 cpu = get_cpu();
1497 ret = pmu_sbi_snapshot_setup(pmu, cpu);
1498 put_cpu();
1499
1500 if (ret) {
1501 /* Snapshot is an optional feature. Continue if not available */
1502 pmu_sbi_snapshot_free(pmu);
1503 } else {
1504 pr_info("SBI PMU snapshot detected\n");
1505 /*
1506 * We enable it once here for the boot cpu. If snapshot shmem setup
1507 * fails during cpu hotplug process, it will fail to start the cpu
1508 * as we can not handle hetergenous PMUs with different snapshot
1509 * capability.
1510 */
1511 static_branch_enable(&sbi_pmu_snapshot_available);
1512 }
1513 }
1514
1515 register_sysctl("kernel", sbi_pmu_sysctl_table);
1516
1517 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
1518 if (ret)
1519 goto out_unregister;
1520
1521 /* Asynchronously check which standard events are available */
1522 schedule_work(&check_std_events_work);
1523
1524 return 0;
1525
1526 out_unregister:
1527 perf_pmu_unregister(&pmu->pmu);
1528
1529 out_destroy:
1530 riscv_pmu_destroy(pmu);
1531 if (irq_requested) {
1532 free_percpu_irq(riscv_pmu_irq, pmu->hw_events);
1533 irq_dispose_mapping(riscv_pmu_irq);
1534 riscv_pmu_irq = 0;
1535 }
1536
1537 out_free:
1538 free_percpu(pmu->hw_events);
1539 kfree(pmu_ctr_list);
1540 pmu_ctr_list = NULL;
1541 kfree(pmu);
1542 return ret;
1543 }
1544
1545 static struct platform_driver pmu_sbi_driver = {
1546 .probe = pmu_sbi_device_probe,
1547 .driver = {
1548 .name = RISCV_PMU_SBI_PDEV_NAME,
1549 },
1550 };
1551
pmu_sbi_devinit(void)1552 static int __init pmu_sbi_devinit(void)
1553 {
1554 int ret;
1555 struct platform_device *pdev;
1556
1557 if (sbi_spec_version < sbi_mk_version(0, 3) ||
1558 !sbi_probe_extension(SBI_EXT_PMU)) {
1559 return 0;
1560 }
1561
1562 if (sbi_spec_version >= sbi_mk_version(2, 0))
1563 sbi_v2_available = true;
1564
1565 if (sbi_spec_version >= sbi_mk_version(3, 0))
1566 sbi_v3_available = true;
1567
1568 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_STARTING,
1569 "perf/riscv/pmu:starting",
1570 pmu_sbi_starting_cpu, pmu_sbi_dying_cpu);
1571 if (ret) {
1572 pr_err("CPU hotplug notifier could not be registered: %d\n",
1573 ret);
1574 return ret;
1575 }
1576
1577 ret = platform_driver_register(&pmu_sbi_driver);
1578 if (ret)
1579 return ret;
1580
1581 pdev = platform_device_register_simple(RISCV_PMU_SBI_PDEV_NAME, -1, NULL, 0);
1582 if (IS_ERR(pdev)) {
1583 platform_driver_unregister(&pmu_sbi_driver);
1584 return PTR_ERR(pdev);
1585 }
1586
1587 /* Notify legacy implementation that SBI pmu is available*/
1588 riscv_pmu_legacy_skip_init();
1589
1590 return ret;
1591 }
1592 device_initcall(pmu_sbi_devinit)
1593