1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * RISC-V performance counter support.
4 *
5 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
6 *
7 * This code is based on ARM perf event code which is in turn based on
8 * sparc64 and x86 code.
9 */
10
11 #define pr_fmt(fmt) "riscv-pmu-sbi: " fmt
12
13 #include <linux/perf/riscv_pmu.h>
14 #include <linux/platform_device.h>
15 #include <linux/irq.h>
16 #include <linux/irqdomain.h>
17 #include <linux/of_irq.h>
18 #include <linux/of.h>
19 #include <linux/cpu_pm.h>
20 #include <linux/sched/clock.h>
21 #include <linux/soc/andes/irq.h>
22 #include <linux/workqueue.h>
23
24 #include <asm/errata_list.h>
25 #include <asm/sbi.h>
26 #include <asm/cpufeature.h>
27 #include <asm/vendor_extensions.h>
28 #include <asm/vendor_extensions/andes.h>
29
30 #define ALT_SBI_PMU_OVERFLOW(__ovl) \
31 asm volatile(ALTERNATIVE_2( \
32 "csrr %0, " __stringify(CSR_SCOUNTOVF), \
33 "csrr %0, " __stringify(THEAD_C9XX_CSR_SCOUNTEROF), \
34 THEAD_VENDOR_ID, ERRATA_THEAD_PMU, \
35 CONFIG_ERRATA_THEAD_PMU, \
36 "csrr %0, " __stringify(ANDES_CSR_SCOUNTEROF), \
37 ANDES_VENDOR_ID, \
38 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \
39 CONFIG_ANDES_CUSTOM_PMU) \
40 : "=r" (__ovl) : \
41 : "memory")
42
43 #define ALT_SBI_PMU_OVF_CLEAR_PENDING(__irq_mask) \
44 asm volatile(ALTERNATIVE( \
45 "csrc " __stringify(CSR_IP) ", %0\n\t", \
46 "csrc " __stringify(ANDES_CSR_SLIP) ", %0\n\t", \
47 ANDES_VENDOR_ID, \
48 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \
49 CONFIG_ANDES_CUSTOM_PMU) \
50 : : "r"(__irq_mask) \
51 : "memory")
52
53 #define SYSCTL_NO_USER_ACCESS 0
54 #define SYSCTL_USER_ACCESS 1
55 #define SYSCTL_LEGACY 2
56
57 #define PERF_EVENT_FLAG_NO_USER_ACCESS BIT(SYSCTL_NO_USER_ACCESS)
58 #define PERF_EVENT_FLAG_USER_ACCESS BIT(SYSCTL_USER_ACCESS)
59 #define PERF_EVENT_FLAG_LEGACY BIT(SYSCTL_LEGACY)
60
61 PMU_FORMAT_ATTR(event, "config:0-55");
62 PMU_FORMAT_ATTR(firmware, "config:62-63");
63
64 static bool sbi_v2_available;
65 static bool sbi_v3_available;
66 static DEFINE_STATIC_KEY_FALSE(sbi_pmu_snapshot_available);
67 #define sbi_pmu_snapshot_available() \
68 static_branch_unlikely(&sbi_pmu_snapshot_available)
69
70 static struct attribute *riscv_arch_formats_attr[] = {
71 &format_attr_event.attr,
72 &format_attr_firmware.attr,
73 NULL,
74 };
75
76 static struct attribute_group riscv_pmu_format_group = {
77 .name = "format",
78 .attrs = riscv_arch_formats_attr,
79 };
80
81 static const struct attribute_group *riscv_pmu_attr_groups[] = {
82 &riscv_pmu_format_group,
83 NULL,
84 };
85
86 /* Allow user mode access by default */
87 static int sysctl_perf_user_access __read_mostly = SYSCTL_USER_ACCESS;
88
89 /*
90 * RISC-V doesn't have heterogeneous harts yet. This need to be part of
91 * per_cpu in case of harts with different pmu counters
92 */
93 static union sbi_pmu_ctr_info *pmu_ctr_list;
94 static bool riscv_pmu_use_irq;
95 static unsigned int riscv_pmu_irq_num;
96 static unsigned int riscv_pmu_irq_mask;
97 static unsigned int riscv_pmu_irq;
98
99 /* Cache the available counters in a bitmask */
100 static DECLARE_BITMAP(cmask, RISCV_MAX_COUNTERS);
101
102 static int pmu_event_find_cache(u64 config);
103 struct sbi_pmu_event_data {
104 union {
105 union {
106 struct hw_gen_event {
107 uint32_t event_code:16;
108 uint32_t event_type:4;
109 uint32_t reserved:12;
110 } hw_gen_event;
111 struct hw_cache_event {
112 uint32_t result_id:1;
113 uint32_t op_id:2;
114 uint32_t cache_id:13;
115 uint32_t event_type:4;
116 uint32_t reserved:12;
117 } hw_cache_event;
118 };
119 uint32_t event_idx;
120 };
121 };
122
123 static struct sbi_pmu_event_data pmu_hw_event_map[] = {
124 [PERF_COUNT_HW_CPU_CYCLES] = {.hw_gen_event = {
125 SBI_PMU_HW_CPU_CYCLES,
126 SBI_PMU_EVENT_TYPE_HW, 0}},
127 [PERF_COUNT_HW_INSTRUCTIONS] = {.hw_gen_event = {
128 SBI_PMU_HW_INSTRUCTIONS,
129 SBI_PMU_EVENT_TYPE_HW, 0}},
130 [PERF_COUNT_HW_CACHE_REFERENCES] = {.hw_gen_event = {
131 SBI_PMU_HW_CACHE_REFERENCES,
132 SBI_PMU_EVENT_TYPE_HW, 0}},
133 [PERF_COUNT_HW_CACHE_MISSES] = {.hw_gen_event = {
134 SBI_PMU_HW_CACHE_MISSES,
135 SBI_PMU_EVENT_TYPE_HW, 0}},
136 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {.hw_gen_event = {
137 SBI_PMU_HW_BRANCH_INSTRUCTIONS,
138 SBI_PMU_EVENT_TYPE_HW, 0}},
139 [PERF_COUNT_HW_BRANCH_MISSES] = {.hw_gen_event = {
140 SBI_PMU_HW_BRANCH_MISSES,
141 SBI_PMU_EVENT_TYPE_HW, 0}},
142 [PERF_COUNT_HW_BUS_CYCLES] = {.hw_gen_event = {
143 SBI_PMU_HW_BUS_CYCLES,
144 SBI_PMU_EVENT_TYPE_HW, 0}},
145 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = {.hw_gen_event = {
146 SBI_PMU_HW_STALLED_CYCLES_FRONTEND,
147 SBI_PMU_EVENT_TYPE_HW, 0}},
148 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = {.hw_gen_event = {
149 SBI_PMU_HW_STALLED_CYCLES_BACKEND,
150 SBI_PMU_EVENT_TYPE_HW, 0}},
151 [PERF_COUNT_HW_REF_CPU_CYCLES] = {.hw_gen_event = {
152 SBI_PMU_HW_REF_CPU_CYCLES,
153 SBI_PMU_EVENT_TYPE_HW, 0}},
154 };
155
156 #define C(x) PERF_COUNT_HW_CACHE_##x
157 static struct sbi_pmu_event_data pmu_cache_event_map[PERF_COUNT_HW_CACHE_MAX]
158 [PERF_COUNT_HW_CACHE_OP_MAX]
159 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
160 [C(L1D)] = {
161 [C(OP_READ)] = {
162 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
163 C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
164 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
165 C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
166 },
167 [C(OP_WRITE)] = {
168 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
169 C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
170 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
171 C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
172 },
173 [C(OP_PREFETCH)] = {
174 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
175 C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
176 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
177 C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
178 },
179 },
180 [C(L1I)] = {
181 [C(OP_READ)] = {
182 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
183 C(OP_READ), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
184 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), C(OP_READ),
185 C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
186 },
187 [C(OP_WRITE)] = {
188 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
189 C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
190 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
191 C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
192 },
193 [C(OP_PREFETCH)] = {
194 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
195 C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
196 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
197 C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
198 },
199 },
200 [C(LL)] = {
201 [C(OP_READ)] = {
202 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
203 C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
204 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
205 C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
206 },
207 [C(OP_WRITE)] = {
208 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
209 C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
210 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
211 C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
212 },
213 [C(OP_PREFETCH)] = {
214 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
215 C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
216 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
217 C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
218 },
219 },
220 [C(DTLB)] = {
221 [C(OP_READ)] = {
222 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
223 C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
224 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
225 C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
226 },
227 [C(OP_WRITE)] = {
228 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
229 C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
230 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
231 C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
232 },
233 [C(OP_PREFETCH)] = {
234 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
235 C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
236 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
237 C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
238 },
239 },
240 [C(ITLB)] = {
241 [C(OP_READ)] = {
242 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
243 C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
244 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
245 C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
246 },
247 [C(OP_WRITE)] = {
248 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
249 C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
250 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
251 C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
252 },
253 [C(OP_PREFETCH)] = {
254 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
255 C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
256 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
257 C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
258 },
259 },
260 [C(BPU)] = {
261 [C(OP_READ)] = {
262 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
263 C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
264 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
265 C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
266 },
267 [C(OP_WRITE)] = {
268 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
269 C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
270 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
271 C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
272 },
273 [C(OP_PREFETCH)] = {
274 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
275 C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
276 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
277 C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
278 },
279 },
280 [C(NODE)] = {
281 [C(OP_READ)] = {
282 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
283 C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
284 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
285 C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
286 },
287 [C(OP_WRITE)] = {
288 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
289 C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
290 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
291 C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
292 },
293 [C(OP_PREFETCH)] = {
294 [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
295 C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
296 [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
297 C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
298 },
299 },
300 };
301
pmu_sbi_check_event_info(void)302 static int pmu_sbi_check_event_info(void)
303 {
304 int num_events = ARRAY_SIZE(pmu_hw_event_map) + PERF_COUNT_HW_CACHE_MAX *
305 PERF_COUNT_HW_CACHE_OP_MAX * PERF_COUNT_HW_CACHE_RESULT_MAX;
306 struct riscv_pmu_event_info *event_info_shmem;
307 phys_addr_t base_addr;
308 int i, j, k, result = 0, count = 0;
309 struct sbiret ret;
310
311 event_info_shmem = kzalloc_objs(*event_info_shmem, num_events);
312 if (!event_info_shmem)
313 return -ENOMEM;
314
315 for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++)
316 event_info_shmem[count++].event_idx = pmu_hw_event_map[i].event_idx;
317
318 for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) {
319 for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) {
320 for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++)
321 event_info_shmem[count++].event_idx =
322 pmu_cache_event_map[i][j][k].event_idx;
323 }
324 }
325
326 base_addr = __pa(event_info_shmem);
327 if (IS_ENABLED(CONFIG_32BIT))
328 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, lower_32_bits(base_addr),
329 upper_32_bits(base_addr), count, 0, 0, 0);
330 else
331 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_EVENT_GET_INFO, base_addr, 0,
332 count, 0, 0, 0);
333 if (ret.error) {
334 result = -EOPNOTSUPP;
335 goto free_mem;
336 }
337
338 for (i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++) {
339 if (!(event_info_shmem[i].output & RISCV_PMU_EVENT_INFO_OUTPUT_MASK))
340 pmu_hw_event_map[i].event_idx = -ENOENT;
341 }
342
343 count = ARRAY_SIZE(pmu_hw_event_map);
344
345 for (i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++) {
346 for (j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++) {
347 for (k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++) {
348 if (!(event_info_shmem[count].output &
349 RISCV_PMU_EVENT_INFO_OUTPUT_MASK))
350 pmu_cache_event_map[i][j][k].event_idx = -ENOENT;
351 count++;
352 }
353 }
354 }
355
356 free_mem:
357 kfree(event_info_shmem);
358
359 return result;
360 }
361
pmu_sbi_ctr_cfg_match(unsigned long cbase,unsigned long ctr_mask,unsigned long cflags,unsigned long event_idx,u64 config)362 static struct sbiret pmu_sbi_ctr_cfg_match(unsigned long cbase,
363 unsigned long ctr_mask,
364 unsigned long cflags,
365 unsigned long event_idx,
366 u64 config)
367 {
368 #if defined(CONFIG_32BIT)
369 return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase,
370 ctr_mask, cflags, event_idx, config, config >> 32);
371 #else
372 return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase,
373 ctr_mask, cflags, event_idx, config, 0);
374 #endif
375 }
376
pmu_sbi_check_event(struct sbi_pmu_event_data * edata)377 static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata)
378 {
379 struct sbiret ret = { .error = SBI_ERR_NOT_SUPPORTED };
380 int i;
381
382 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
383 if (!cmask[i])
384 continue;
385 ret = pmu_sbi_ctr_cfg_match(i * BITS_PER_LONG, cmask[i], 0,
386 edata->event_idx, 0);
387 if (!ret.error) {
388 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
389 ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0);
390 return;
391 }
392 }
393 if (ret.error == SBI_ERR_NOT_SUPPORTED) {
394 /* This event cannot be monitored by any counter */
395 edata->event_idx = -ENOENT;
396 }
397 }
398
pmu_sbi_check_std_events(struct work_struct * work)399 static void pmu_sbi_check_std_events(struct work_struct *work)
400 {
401 int ret;
402
403 if (sbi_v3_available) {
404 ret = pmu_sbi_check_event_info();
405 if (ret)
406 pr_err("pmu_sbi_check_event_info failed with error %d\n", ret);
407 return;
408 }
409
410 for (int i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++)
411 pmu_sbi_check_event(&pmu_hw_event_map[i]);
412
413 for (int i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++)
414 for (int j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++)
415 for (int k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++)
416 pmu_sbi_check_event(&pmu_cache_event_map[i][j][k]);
417 }
418
419 static DECLARE_WORK(check_std_events_work, pmu_sbi_check_std_events);
420
pmu_sbi_ctr_get_width(int idx)421 static int pmu_sbi_ctr_get_width(int idx)
422 {
423 return pmu_ctr_list[idx].width;
424 }
425
pmu_sbi_ctr_is_fw(int cidx)426 static bool pmu_sbi_ctr_is_fw(int cidx)
427 {
428 union sbi_pmu_ctr_info *info;
429
430 info = &pmu_ctr_list[cidx];
431 if (!info)
432 return false;
433
434 return info->type == SBI_PMU_CTR_TYPE_FW;
435 }
436
riscv_pmu_get_event_info(u32 type,u64 config,u64 * econfig)437 int riscv_pmu_get_event_info(u32 type, u64 config, u64 *econfig)
438 {
439 int ret = -ENOENT;
440
441 switch (type) {
442 case PERF_TYPE_HARDWARE:
443 if (config >= PERF_COUNT_HW_MAX)
444 return -EINVAL;
445 ret = pmu_hw_event_map[config].event_idx;
446 break;
447 case PERF_TYPE_HW_CACHE:
448 ret = pmu_event_find_cache(config);
449 break;
450 case PERF_TYPE_RAW:
451 /*
452 * As per SBI v0.3 specification,
453 * -- the upper 16 bits must be unused for a hardware raw event.
454 * As per SBI v2.0 specification,
455 * -- the upper 8 bits must be unused for a hardware raw event.
456 * Bits 63:62 are used to distinguish between raw events
457 * 00 - Hardware raw event
458 * 10 - SBI firmware events
459 * 11 - Risc-V platform specific firmware event
460 */
461 switch (config >> 62) {
462 case 0:
463 if (sbi_v3_available) {
464 /* Return error any bits [56-63] is set as it is not allowed by the spec */
465 if (!(config & ~RISCV_PMU_RAW_EVENT_V2_MASK)) {
466 if (econfig)
467 *econfig = config & RISCV_PMU_RAW_EVENT_V2_MASK;
468 ret = RISCV_PMU_RAW_EVENT_V2_IDX;
469 }
470 /* Return error any bits [48-63] is set as it is not allowed by the spec */
471 } else if (!(config & ~RISCV_PMU_RAW_EVENT_MASK)) {
472 if (econfig)
473 *econfig = config & RISCV_PMU_RAW_EVENT_MASK;
474 ret = RISCV_PMU_RAW_EVENT_IDX;
475 }
476 break;
477 case 2:
478 ret = (config & 0xFFFF) | (SBI_PMU_EVENT_TYPE_FW << 16);
479 break;
480 case 3:
481 /*
482 * For Risc-V platform specific firmware events
483 * Event code - 0xFFFF
484 * Event data - raw event encoding
485 */
486 ret = SBI_PMU_EVENT_TYPE_FW << 16 | RISCV_PLAT_FW_EVENT;
487 if (econfig)
488 *econfig = config & RISCV_PMU_PLAT_FW_EVENT_MASK;
489 break;
490 default:
491 break;
492 }
493 break;
494 default:
495 break;
496 }
497
498 return ret;
499 }
500 EXPORT_SYMBOL_GPL(riscv_pmu_get_event_info);
501
502 /*
503 * Returns the counter width of a programmable counter and number of hardware
504 * counters. As we don't support heterogeneous CPUs yet, it is okay to just
505 * return the counter width of the first programmable counter.
506 */
riscv_pmu_get_hpm_info(u32 * hw_ctr_width,u32 * num_hw_ctr)507 int riscv_pmu_get_hpm_info(u32 *hw_ctr_width, u32 *num_hw_ctr)
508 {
509 int i;
510 union sbi_pmu_ctr_info *info;
511 u32 hpm_width = 0, hpm_count = 0;
512
513 if (bitmap_empty(cmask, RISCV_MAX_COUNTERS))
514 return -EINVAL;
515
516 for_each_set_bit(i, cmask, RISCV_MAX_COUNTERS) {
517 info = &pmu_ctr_list[i];
518 if (!info)
519 continue;
520 if (!hpm_width && info->csr != CSR_CYCLE && info->csr != CSR_INSTRET)
521 hpm_width = info->width;
522 if (info->type == SBI_PMU_CTR_TYPE_HW)
523 hpm_count++;
524 }
525
526 *hw_ctr_width = hpm_width;
527 *num_hw_ctr = hpm_count;
528
529 return 0;
530 }
531 EXPORT_SYMBOL_GPL(riscv_pmu_get_hpm_info);
532
pmu_sbi_csr_index(struct perf_event * event)533 static uint8_t pmu_sbi_csr_index(struct perf_event *event)
534 {
535 return pmu_ctr_list[event->hw.idx].csr - CSR_CYCLE;
536 }
537
pmu_sbi_get_filter_flags(struct perf_event * event)538 static unsigned long pmu_sbi_get_filter_flags(struct perf_event *event)
539 {
540 unsigned long cflags = 0;
541 bool guest_events = false;
542
543 if (event->attr.config1 & RISCV_PMU_CONFIG1_GUEST_EVENTS)
544 guest_events = true;
545 if (event->attr.exclude_kernel)
546 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VSINH : SBI_PMU_CFG_FLAG_SET_SINH;
547 if (event->attr.exclude_user)
548 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VUINH : SBI_PMU_CFG_FLAG_SET_UINH;
549 if (guest_events && event->attr.exclude_hv)
550 cflags |= SBI_PMU_CFG_FLAG_SET_SINH;
551 if (event->attr.exclude_host)
552 cflags |= SBI_PMU_CFG_FLAG_SET_UINH | SBI_PMU_CFG_FLAG_SET_SINH;
553 if (event->attr.exclude_guest)
554 cflags |= SBI_PMU_CFG_FLAG_SET_VSINH | SBI_PMU_CFG_FLAG_SET_VUINH;
555
556 return cflags;
557 }
558
pmu_sbi_ctr_get_idx(struct perf_event * event)559 static int pmu_sbi_ctr_get_idx(struct perf_event *event)
560 {
561 struct hw_perf_event *hwc = &event->hw;
562 struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
563 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
564 struct sbiret ret;
565 int idx, i;
566 u64 cbase = 0, cmask = 0;
567 unsigned long cflags = 0;
568
569 cflags = pmu_sbi_get_filter_flags(event);
570
571 /*
572 * In legacy mode, we have to force the fixed counters for those events
573 * but not in the user access mode as we want to use the other counters
574 * that support sampling/filtering.
575 */
576 if ((hwc->flags & PERF_EVENT_FLAG_LEGACY) && (event->attr.type == PERF_TYPE_HARDWARE)) {
577 if (event->attr.config == PERF_COUNT_HW_CPU_CYCLES) {
578 cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH;
579 cmask = 1;
580 } else if (event->attr.config == PERF_COUNT_HW_INSTRUCTIONS) {
581 cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH;
582 cmask = BIT(CSR_INSTRET - CSR_CYCLE);
583 }
584 }
585
586 /* retrieve the available counter index */
587 if (cmask) {
588 ret = pmu_sbi_ctr_cfg_match(cbase, cmask, cflags, hwc->event_base,
589 hwc->config);
590 } else {
591 ret.error = SBI_ERR_NOT_SUPPORTED;
592 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
593 if (!rvpmu->cmask[i])
594 continue;
595 cbase = i * BITS_PER_LONG;
596 ret = pmu_sbi_ctr_cfg_match(cbase, rvpmu->cmask[i], cflags,
597 hwc->event_base, hwc->config);
598 if (!ret.error)
599 break;
600 }
601 }
602 if (ret.error) {
603 pr_debug("Not able to find a counter for event %lx config %llx\n",
604 hwc->event_base, hwc->config);
605 return sbi_err_map_linux_errno(ret.error);
606 }
607
608 idx = ret.value;
609 if (!test_bit(idx, rvpmu->cmask) || !pmu_ctr_list[idx].value)
610 return -ENOENT;
611
612 /* Additional sanity check for the counter id */
613 if (pmu_sbi_ctr_is_fw(idx)) {
614 if (!test_and_set_bit(idx, cpuc->used_fw_ctrs))
615 return idx;
616 } else {
617 if (!test_and_set_bit(idx, cpuc->used_hw_ctrs))
618 return idx;
619 }
620
621 return -ENOENT;
622 }
623
pmu_sbi_ctr_clear_idx(struct perf_event * event)624 static void pmu_sbi_ctr_clear_idx(struct perf_event *event)
625 {
626
627 struct hw_perf_event *hwc = &event->hw;
628 struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
629 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
630 int idx = hwc->idx;
631
632 if (pmu_sbi_ctr_is_fw(idx))
633 clear_bit(idx, cpuc->used_fw_ctrs);
634 else
635 clear_bit(idx, cpuc->used_hw_ctrs);
636 }
637
pmu_event_find_cache(u64 config)638 static int pmu_event_find_cache(u64 config)
639 {
640 unsigned int cache_type, cache_op, cache_result, ret;
641
642 cache_type = (config >> 0) & 0xff;
643 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
644 return -EINVAL;
645
646 cache_op = (config >> 8) & 0xff;
647 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
648 return -EINVAL;
649
650 cache_result = (config >> 16) & 0xff;
651 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
652 return -EINVAL;
653
654 ret = pmu_cache_event_map[cache_type][cache_op][cache_result].event_idx;
655
656 return ret;
657 }
658
pmu_sbi_is_fw_event(struct perf_event * event)659 static bool pmu_sbi_is_fw_event(struct perf_event *event)
660 {
661 u32 type = event->attr.type;
662 u64 config = event->attr.config;
663
664 if ((type == PERF_TYPE_RAW) && ((config >> 63) == 1))
665 return true;
666 else
667 return false;
668 }
669
pmu_sbi_event_map(struct perf_event * event,u64 * econfig)670 static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig)
671 {
672 u32 type = event->attr.type;
673 u64 config = event->attr.config;
674
675 /*
676 * Ensure we are finished checking standard hardware events for
677 * validity before allowing userspace to configure any events.
678 */
679 flush_work(&check_std_events_work);
680
681 return riscv_pmu_get_event_info(type, config, econfig);
682 }
683
pmu_sbi_snapshot_free(struct riscv_pmu * pmu)684 static void pmu_sbi_snapshot_free(struct riscv_pmu *pmu)
685 {
686 int cpu;
687
688 for_each_possible_cpu(cpu) {
689 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
690
691 if (!cpu_hw_evt->snapshot_addr)
692 continue;
693
694 free_page((unsigned long)cpu_hw_evt->snapshot_addr);
695 cpu_hw_evt->snapshot_addr = NULL;
696 cpu_hw_evt->snapshot_addr_phys = 0;
697 }
698 }
699
pmu_sbi_snapshot_alloc(struct riscv_pmu * pmu)700 static int pmu_sbi_snapshot_alloc(struct riscv_pmu *pmu)
701 {
702 int cpu;
703 struct page *snapshot_page;
704
705 for_each_possible_cpu(cpu) {
706 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
707
708 snapshot_page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
709 if (!snapshot_page) {
710 pmu_sbi_snapshot_free(pmu);
711 return -ENOMEM;
712 }
713 cpu_hw_evt->snapshot_addr = page_to_virt(snapshot_page);
714 cpu_hw_evt->snapshot_addr_phys = page_to_phys(snapshot_page);
715 }
716
717 return 0;
718 }
719
pmu_sbi_snapshot_disable(void)720 static int pmu_sbi_snapshot_disable(void)
721 {
722 struct sbiret ret;
723
724 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, SBI_SHMEM_DISABLE,
725 SBI_SHMEM_DISABLE, 0, 0, 0, 0);
726 if (ret.error) {
727 pr_warn("failed to disable snapshot shared memory\n");
728 return sbi_err_map_linux_errno(ret.error);
729 }
730
731 return 0;
732 }
733
pmu_sbi_snapshot_setup(struct riscv_pmu * pmu,int cpu)734 static int pmu_sbi_snapshot_setup(struct riscv_pmu *pmu, int cpu)
735 {
736 struct cpu_hw_events *cpu_hw_evt;
737 struct sbiret ret = {0};
738
739 cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
740 if (!cpu_hw_evt->snapshot_addr_phys)
741 return -EINVAL;
742
743 if (cpu_hw_evt->snapshot_set_done)
744 return 0;
745
746 if (IS_ENABLED(CONFIG_32BIT))
747 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM,
748 cpu_hw_evt->snapshot_addr_phys,
749 (u64)(cpu_hw_evt->snapshot_addr_phys) >> 32, 0, 0, 0, 0);
750 else
751 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM,
752 cpu_hw_evt->snapshot_addr_phys, 0, 0, 0, 0, 0);
753
754 /* Free up the snapshot area memory and fall back to SBI PMU calls without snapshot */
755 if (ret.error) {
756 if (ret.error != SBI_ERR_NOT_SUPPORTED)
757 pr_warn("pmu snapshot setup failed with error %ld\n", ret.error);
758 return sbi_err_map_linux_errno(ret.error);
759 }
760
761 memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS);
762 cpu_hw_evt->snapshot_set_done = true;
763
764 return 0;
765 }
766
pmu_sbi_ctr_read(struct perf_event * event)767 static u64 pmu_sbi_ctr_read(struct perf_event *event)
768 {
769 struct hw_perf_event *hwc = &event->hw;
770 int idx = hwc->idx;
771 struct sbiret ret;
772 u64 val = 0;
773 struct riscv_pmu *pmu = to_riscv_pmu(event->pmu);
774 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
775 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
776 union sbi_pmu_ctr_info info = pmu_ctr_list[idx];
777
778 /* Read the value from the shared memory directly only if counter is stopped */
779 if (sbi_pmu_snapshot_available() && (hwc->state & PERF_HES_STOPPED)) {
780 val = sdata->ctr_values[idx];
781 return val;
782 }
783
784 if (pmu_sbi_is_fw_event(event)) {
785 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ,
786 hwc->idx, 0, 0, 0, 0, 0);
787 if (ret.error)
788 return 0;
789
790 val = ret.value;
791 if (IS_ENABLED(CONFIG_32BIT) && sbi_v2_available && info.width >= 32) {
792 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ_HI,
793 hwc->idx, 0, 0, 0, 0, 0);
794 if (!ret.error)
795 val |= ((u64)ret.value << 32);
796 else
797 WARN_ONCE(1, "Unable to read upper 32 bits of firmware counter error: %ld\n",
798 ret.error);
799 }
800 } else {
801 val = riscv_pmu_ctr_read_csr(info.csr);
802 if (IS_ENABLED(CONFIG_32BIT))
803 val |= ((u64)riscv_pmu_ctr_read_csr(info.csr + 0x80)) << 32;
804 }
805
806 return val;
807 }
808
pmu_sbi_set_scounteren(void * arg)809 static void pmu_sbi_set_scounteren(void *arg)
810 {
811 struct perf_event *event = (struct perf_event *)arg;
812
813 if (event->hw.idx != -1)
814 csr_write(CSR_SCOUNTEREN,
815 csr_read(CSR_SCOUNTEREN) | BIT(pmu_sbi_csr_index(event)));
816 }
817
pmu_sbi_reset_scounteren(void * arg)818 static void pmu_sbi_reset_scounteren(void *arg)
819 {
820 struct perf_event *event = (struct perf_event *)arg;
821
822 if (event->hw.idx != -1)
823 csr_write(CSR_SCOUNTEREN,
824 csr_read(CSR_SCOUNTEREN) & ~BIT(pmu_sbi_csr_index(event)));
825 }
826
pmu_sbi_ctr_start(struct perf_event * event,u64 ival)827 static void pmu_sbi_ctr_start(struct perf_event *event, u64 ival)
828 {
829 struct sbiret ret;
830 struct hw_perf_event *hwc = &event->hw;
831 unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
832
833 /* There is no benefit setting SNAPSHOT FLAG for a single counter */
834 #if defined(CONFIG_32BIT)
835 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
836 1, flag, ival, ival >> 32, 0);
837 #else
838 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
839 1, flag, ival, 0, 0);
840 #endif
841 if (ret.error && (ret.error != SBI_ERR_ALREADY_STARTED))
842 pr_err("Starting counter idx %d failed with error %d\n",
843 hwc->idx, sbi_err_map_linux_errno(ret.error));
844
845 if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) &&
846 (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT))
847 pmu_sbi_set_scounteren((void *)event);
848 }
849
pmu_sbi_ctr_stop(struct perf_event * event,unsigned long flag)850 static void pmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag)
851 {
852 struct sbiret ret;
853 struct hw_perf_event *hwc = &event->hw;
854 struct riscv_pmu *pmu = to_riscv_pmu(event->pmu);
855 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
856 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
857
858 if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) &&
859 (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT))
860 pmu_sbi_reset_scounteren((void *)event);
861
862 if (sbi_pmu_snapshot_available())
863 flag |= SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
864
865 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, hwc->idx, 1, flag, 0, 0, 0);
866 if (!ret.error && sbi_pmu_snapshot_available()) {
867 /*
868 * The counter snapshot is based on the index base specified by hwc->idx.
869 * The actual counter value is updated in shared memory at index 0 when counter
870 * mask is 0x01. To ensure accurate counter values, it's necessary to transfer
871 * the counter value to shared memory. However, if hwc->idx is zero, the counter
872 * value is already correctly updated in shared memory, requiring no further
873 * adjustment.
874 */
875 if (hwc->idx > 0) {
876 sdata->ctr_values[hwc->idx] = sdata->ctr_values[0];
877 sdata->ctr_values[0] = 0;
878 }
879 } else if (ret.error && (ret.error != SBI_ERR_ALREADY_STOPPED) &&
880 flag != SBI_PMU_STOP_FLAG_RESET) {
881 pr_err("Stopping counter idx %d failed with error %d\n",
882 hwc->idx, sbi_err_map_linux_errno(ret.error));
883 }
884 }
885
pmu_sbi_find_num_ctrs(void)886 static int pmu_sbi_find_num_ctrs(void)
887 {
888 struct sbiret ret;
889
890 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS, 0, 0, 0, 0, 0, 0);
891 if (!ret.error)
892 return ret.value;
893 else
894 return sbi_err_map_linux_errno(ret.error);
895 }
896
pmu_sbi_get_ctrinfo(int nctr,unsigned long * mask)897 static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask)
898 {
899 struct sbiret ret;
900 int i, num_hw_ctr = 0, num_fw_ctr = 0;
901 union sbi_pmu_ctr_info cinfo;
902
903 pmu_ctr_list = kzalloc_objs(*pmu_ctr_list, nctr);
904 if (!pmu_ctr_list)
905 return -ENOMEM;
906
907 for (i = 0; i < nctr; i++) {
908 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0);
909 if (ret.error)
910 /* The logical counter ids are not expected to be contiguous */
911 continue;
912
913 set_bit(i, mask);
914
915 cinfo.value = ret.value;
916 if (cinfo.type == SBI_PMU_CTR_TYPE_FW)
917 num_fw_ctr++;
918 else
919 num_hw_ctr++;
920 pmu_ctr_list[i].value = cinfo.value;
921 }
922
923 pr_info("%d firmware and %d hardware counters\n", num_fw_ctr, num_hw_ctr);
924
925 return 0;
926 }
927
pmu_sbi_stop_all(struct riscv_pmu * pmu)928 static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu)
929 {
930 int i;
931
932 /*
933 * No need to check the error because we are disabling all the counters
934 * which may include counters that are not enabled yet.
935 */
936 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
937 if (!pmu->cmask[i])
938 continue;
939 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
940 i * BITS_PER_LONG, pmu->cmask[i],
941 SBI_PMU_STOP_FLAG_RESET, 0, 0, 0);
942 }
943 }
944
pmu_sbi_stop_hw_ctrs(struct riscv_pmu * pmu)945 static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu)
946 {
947 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
948 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
949 unsigned long flag = 0;
950 int i, idx;
951 struct sbiret ret;
952 u64 temp_ctr_overflow_mask = 0;
953
954 if (sbi_pmu_snapshot_available())
955 flag = SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
956
957 /* Reset the shadow copy to avoid save/restore any value from previous overflow */
958 memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS);
959
960 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
961 /* No need to check the error here as we can't do anything about the error */
962 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, i * BITS_PER_LONG,
963 cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0);
964 if (!ret.error && sbi_pmu_snapshot_available()) {
965 /* Save the counter values to avoid clobbering */
966 for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG)
967 cpu_hw_evt->snapshot_cval_shcopy[i * BITS_PER_LONG + idx] =
968 sdata->ctr_values[idx];
969 /* Save the overflow mask to avoid clobbering */
970 temp_ctr_overflow_mask |= sdata->ctr_overflow_mask << (i * BITS_PER_LONG);
971 }
972 }
973
974 /* Restore the counter values to the shared memory for used hw counters */
975 if (sbi_pmu_snapshot_available()) {
976 for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS)
977 sdata->ctr_values[idx] = cpu_hw_evt->snapshot_cval_shcopy[idx];
978 if (temp_ctr_overflow_mask)
979 sdata->ctr_overflow_mask = temp_ctr_overflow_mask;
980 }
981 }
982
983 /*
984 * This function starts all the used counters in two step approach.
985 * Any counter that did not overflow can be start in a single step
986 * while the overflowed counters need to be started with updated initialization
987 * value.
988 */
pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events * cpu_hw_evt,u64 ctr_ovf_mask)989 static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt,
990 u64 ctr_ovf_mask)
991 {
992 int idx = 0, i;
993 struct perf_event *event;
994 unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
995 unsigned long ctr_start_mask = 0;
996 uint64_t max_period;
997 struct hw_perf_event *hwc;
998 u64 init_val = 0;
999
1000 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
1001 ctr_start_mask = cpu_hw_evt->used_hw_ctrs[i] & ~ctr_ovf_mask;
1002 /* Start all the counters that did not overflow in a single shot */
1003 if (ctr_start_mask) {
1004 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, i * BITS_PER_LONG,
1005 ctr_start_mask, 0, 0, 0, 0);
1006 }
1007 }
1008
1009 /* Reinitialize and start all the counter that overflowed */
1010 while (ctr_ovf_mask) {
1011 if (ctr_ovf_mask & 0x01) {
1012 event = cpu_hw_evt->events[idx];
1013 hwc = &event->hw;
1014 max_period = riscv_pmu_ctr_get_width_mask(event);
1015 init_val = local64_read(&hwc->prev_count) & max_period;
1016 #if defined(CONFIG_32BIT)
1017 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
1018 flag, init_val, init_val >> 32, 0);
1019 #else
1020 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
1021 flag, init_val, 0, 0);
1022 #endif
1023 perf_event_update_userpage(event);
1024 }
1025 ctr_ovf_mask = ctr_ovf_mask >> 1;
1026 idx++;
1027 }
1028 }
1029
pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events * cpu_hw_evt,u64 ctr_ovf_mask)1030 static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_evt,
1031 u64 ctr_ovf_mask)
1032 {
1033 int i, idx = 0;
1034 struct perf_event *event;
1035 unsigned long flag = SBI_PMU_START_FLAG_INIT_SNAPSHOT;
1036 u64 max_period, init_val = 0;
1037 struct hw_perf_event *hwc;
1038 struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
1039
1040 for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
1041 if (ctr_ovf_mask & BIT_ULL(idx)) {
1042 event = cpu_hw_evt->events[idx];
1043 hwc = &event->hw;
1044 max_period = riscv_pmu_ctr_get_width_mask(event);
1045 init_val = local64_read(&hwc->prev_count) & max_period;
1046 cpu_hw_evt->snapshot_cval_shcopy[idx] = init_val;
1047 }
1048 /*
1049 * We do not need to update the non-overflow counters the previous
1050 * value should have been there already.
1051 */
1052 }
1053
1054 for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
1055 /* Restore the counter values to relative indices for used hw counters */
1056 for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG)
1057 sdata->ctr_values[idx] =
1058 cpu_hw_evt->snapshot_cval_shcopy[idx + i * BITS_PER_LONG];
1059 /* Start all the counters in a single shot */
1060 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx * BITS_PER_LONG,
1061 cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0);
1062 }
1063 }
1064
pmu_sbi_start_overflow_mask(struct riscv_pmu * pmu,u64 ctr_ovf_mask)1065 static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu,
1066 u64 ctr_ovf_mask)
1067 {
1068 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
1069
1070 if (sbi_pmu_snapshot_available())
1071 pmu_sbi_start_ovf_ctrs_snapshot(cpu_hw_evt, ctr_ovf_mask);
1072 else
1073 pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask);
1074 }
1075
pmu_sbi_ovf_handler(int irq,void * dev)1076 static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev)
1077 {
1078 struct perf_sample_data data;
1079 struct pt_regs *regs;
1080 struct hw_perf_event *hw_evt;
1081 union sbi_pmu_ctr_info *info;
1082 int lidx, hidx, fidx;
1083 struct riscv_pmu *pmu;
1084 struct perf_event *event;
1085 u64 overflow;
1086 u64 overflowed_ctrs = 0;
1087 struct cpu_hw_events *cpu_hw_evt = dev;
1088 u64 start_clock = sched_clock();
1089 struct riscv_pmu_snapshot_data *sdata;
1090
1091 if (WARN_ON_ONCE(!cpu_hw_evt))
1092 return IRQ_NONE;
1093
1094 sdata = cpu_hw_evt->snapshot_addr;
1095
1096 /* Firmware counter don't support overflow yet */
1097 fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS);
1098 if (fidx == RISCV_MAX_COUNTERS) {
1099 csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num));
1100 return IRQ_NONE;
1101 }
1102
1103 event = cpu_hw_evt->events[fidx];
1104 if (!event) {
1105 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
1106 return IRQ_NONE;
1107 }
1108
1109 pmu = to_riscv_pmu(event->pmu);
1110 pmu_sbi_stop_hw_ctrs(pmu);
1111
1112 /* Overflow status register should only be read after counter are stopped */
1113 if (sbi_pmu_snapshot_available())
1114 overflow = sdata->ctr_overflow_mask;
1115 else
1116 ALT_SBI_PMU_OVERFLOW(overflow);
1117
1118 /*
1119 * Overflow interrupt pending bit should only be cleared after stopping
1120 * all the counters to avoid any race condition.
1121 */
1122 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
1123
1124 /* No overflow bit is set */
1125 if (!overflow)
1126 return IRQ_NONE;
1127
1128 regs = get_irq_regs();
1129
1130 for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
1131 struct perf_event *event = cpu_hw_evt->events[lidx];
1132
1133 /* Skip if invalid event or user did not request a sampling */
1134 if (!event || !is_sampling_event(event))
1135 continue;
1136
1137 info = &pmu_ctr_list[lidx];
1138 /* Do a sanity check */
1139 if (!info || info->type != SBI_PMU_CTR_TYPE_HW)
1140 continue;
1141
1142 if (sbi_pmu_snapshot_available())
1143 /* SBI implementation already updated the logical indicies */
1144 hidx = lidx;
1145 else
1146 /* compute hardware counter index */
1147 hidx = info->csr - CSR_CYCLE;
1148
1149 /* check if the corresponding bit is set in scountovf or overflow mask in shmem */
1150 if (!(overflow & BIT_ULL(hidx)))
1151 continue;
1152
1153 /*
1154 * Keep a track of overflowed counters so that they can be started
1155 * with updated initial value.
1156 */
1157 overflowed_ctrs |= BIT_ULL(lidx);
1158 hw_evt = &event->hw;
1159 /* Update the event states here so that we know the state while reading */
1160 hw_evt->state |= PERF_HES_STOPPED;
1161 riscv_pmu_event_update(event);
1162 hw_evt->state |= PERF_HES_UPTODATE;
1163 perf_sample_data_init(&data, 0, hw_evt->last_period);
1164 if (riscv_pmu_event_set_period(event)) {
1165 /*
1166 * Unlike other ISAs, RISC-V don't have to disable interrupts
1167 * to avoid throttling here. As per the specification, the
1168 * interrupt remains disabled until the OF bit is set.
1169 * Interrupts are enabled again only during the start.
1170 * TODO: We will need to stop the guest counters once
1171 * virtualization support is added.
1172 */
1173 perf_event_overflow(event, &data, regs);
1174 }
1175 /* Reset the state as we are going to start the counter after the loop */
1176 hw_evt->state = 0;
1177 }
1178
1179 pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs);
1180 perf_sample_event_took(sched_clock() - start_clock);
1181
1182 return IRQ_HANDLED;
1183 }
1184
pmu_sbi_starting_cpu(unsigned int cpu,struct hlist_node * node)1185 static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node)
1186 {
1187 struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node);
1188 struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
1189
1190 /*
1191 * We keep enabling userspace access to CYCLE, TIME and INSTRET via the
1192 * legacy option but that will be removed in the future.
1193 */
1194 if (sysctl_perf_user_access == SYSCTL_LEGACY)
1195 csr_write(CSR_SCOUNTEREN, 0x7);
1196 else
1197 csr_write(CSR_SCOUNTEREN, 0x2);
1198
1199 /* Stop all the counters so that they can be enabled from perf */
1200 pmu_sbi_stop_all(pmu);
1201
1202 if (riscv_pmu_use_irq) {
1203 cpu_hw_evt->irq = riscv_pmu_irq;
1204 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
1205 enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE);
1206 }
1207
1208 if (sbi_pmu_snapshot_available())
1209 return pmu_sbi_snapshot_setup(pmu, cpu);
1210
1211 return 0;
1212 }
1213
pmu_sbi_dying_cpu(unsigned int cpu,struct hlist_node * node)1214 static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node)
1215 {
1216 if (riscv_pmu_use_irq) {
1217 disable_percpu_irq(riscv_pmu_irq);
1218 }
1219
1220 /* Disable all counters access for user mode now */
1221 csr_write(CSR_SCOUNTEREN, 0x0);
1222
1223 if (sbi_pmu_snapshot_available())
1224 return pmu_sbi_snapshot_disable();
1225
1226 return 0;
1227 }
1228
pmu_sbi_setup_irqs(struct riscv_pmu * pmu,struct platform_device * pdev)1229 static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev)
1230 {
1231 int ret;
1232 struct cpu_hw_events __percpu *hw_events = pmu->hw_events;
1233 struct irq_domain *domain = NULL;
1234
1235 if (riscv_isa_extension_available(NULL, SSCOFPMF)) {
1236 riscv_pmu_irq_num = RV_IRQ_PMU;
1237 riscv_pmu_use_irq = true;
1238 } else if (IS_ENABLED(CONFIG_ERRATA_THEAD_PMU) &&
1239 riscv_cached_mvendorid(0) == THEAD_VENDOR_ID &&
1240 riscv_cached_marchid(0) == 0 &&
1241 riscv_cached_mimpid(0) == 0) {
1242 riscv_pmu_irq_num = THEAD_C9XX_RV_IRQ_PMU;
1243 riscv_pmu_use_irq = true;
1244 } else if (riscv_has_vendor_extension_unlikely(ANDES_VENDOR_ID,
1245 RISCV_ISA_VENDOR_EXT_XANDESPMU) &&
1246 IS_ENABLED(CONFIG_ANDES_CUSTOM_PMU)) {
1247 riscv_pmu_irq_num = ANDES_SLI_CAUSE_BASE + ANDES_RV_IRQ_PMOVI;
1248 riscv_pmu_use_irq = true;
1249 }
1250
1251 riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG);
1252
1253 if (!riscv_pmu_use_irq)
1254 return -EOPNOTSUPP;
1255
1256 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
1257 DOMAIN_BUS_ANY);
1258 if (!domain) {
1259 pr_err("Failed to find INTC IRQ root domain\n");
1260 ret = -ENODEV;
1261 goto err;
1262 }
1263
1264 riscv_pmu_irq = irq_create_mapping(domain, riscv_pmu_irq_num);
1265 if (!riscv_pmu_irq) {
1266 pr_err("Failed to map PMU interrupt for node\n");
1267 ret = -ENODEV;
1268 goto err;
1269 }
1270
1271 ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events);
1272 if (ret) {
1273 pr_err("registering percpu irq failed [%d]\n", ret);
1274 irq_dispose_mapping(riscv_pmu_irq);
1275 riscv_pmu_irq = 0;
1276 goto err;
1277 }
1278
1279 return 0;
1280 err:
1281 riscv_pmu_use_irq = false;
1282 return ret;
1283 }
1284
1285 #ifdef CONFIG_CPU_PM
riscv_pm_pmu_notify(struct notifier_block * b,unsigned long cmd,void * v)1286 static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd,
1287 void *v)
1288 {
1289 struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, riscv_pm_nb);
1290 struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
1291 bool enabled = !bitmap_empty(cpuc->used_hw_ctrs, RISCV_MAX_COUNTERS);
1292 struct perf_event *event;
1293 int idx;
1294
1295 if (!enabled)
1296 return NOTIFY_OK;
1297
1298 for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) {
1299 event = cpuc->events[idx];
1300 if (!event)
1301 continue;
1302
1303 switch (cmd) {
1304 case CPU_PM_ENTER:
1305 /*
1306 * Stop and update the counter
1307 */
1308 riscv_pmu_stop(event, PERF_EF_UPDATE);
1309 break;
1310 case CPU_PM_EXIT:
1311 case CPU_PM_ENTER_FAILED:
1312 /*
1313 * Restore and enable the counter.
1314 */
1315 riscv_pmu_start(event, PERF_EF_RELOAD);
1316 break;
1317 default:
1318 break;
1319 }
1320 }
1321
1322 return NOTIFY_OK;
1323 }
1324
riscv_pm_pmu_register(struct riscv_pmu * pmu)1325 static int riscv_pm_pmu_register(struct riscv_pmu *pmu)
1326 {
1327 pmu->riscv_pm_nb.notifier_call = riscv_pm_pmu_notify;
1328 return cpu_pm_register_notifier(&pmu->riscv_pm_nb);
1329 }
1330
riscv_pm_pmu_unregister(struct riscv_pmu * pmu)1331 static void riscv_pm_pmu_unregister(struct riscv_pmu *pmu)
1332 {
1333 cpu_pm_unregister_notifier(&pmu->riscv_pm_nb);
1334 }
1335 #else
riscv_pm_pmu_register(struct riscv_pmu * pmu)1336 static inline int riscv_pm_pmu_register(struct riscv_pmu *pmu) { return 0; }
riscv_pm_pmu_unregister(struct riscv_pmu * pmu)1337 static inline void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) { }
1338 #endif
1339
riscv_pmu_destroy(struct riscv_pmu * pmu)1340 static void riscv_pmu_destroy(struct riscv_pmu *pmu)
1341 {
1342 if (sbi_v2_available) {
1343 if (sbi_pmu_snapshot_available()) {
1344 pmu_sbi_snapshot_disable();
1345 pmu_sbi_snapshot_free(pmu);
1346 }
1347 }
1348 riscv_pm_pmu_unregister(pmu);
1349 if (!hlist_unhashed(&pmu->node))
1350 cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
1351 }
1352
pmu_sbi_event_init(struct perf_event * event)1353 static void pmu_sbi_event_init(struct perf_event *event)
1354 {
1355 /*
1356 * The permissions are set at event_init so that we do not depend
1357 * on the sysctl value that can change.
1358 */
1359 if (sysctl_perf_user_access == SYSCTL_NO_USER_ACCESS)
1360 event->hw.flags |= PERF_EVENT_FLAG_NO_USER_ACCESS;
1361 else if (sysctl_perf_user_access == SYSCTL_USER_ACCESS)
1362 event->hw.flags |= PERF_EVENT_FLAG_USER_ACCESS;
1363 else
1364 event->hw.flags |= PERF_EVENT_FLAG_LEGACY;
1365 }
1366
pmu_sbi_event_mapped(struct perf_event * event,struct mm_struct * mm)1367 static void pmu_sbi_event_mapped(struct perf_event *event, struct mm_struct *mm)
1368 {
1369 if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS)
1370 return;
1371
1372 if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) {
1373 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES &&
1374 event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) {
1375 return;
1376 }
1377 }
1378
1379 /*
1380 * The user mmapped the event to directly access it: this is where
1381 * we determine based on sysctl_perf_user_access if we grant userspace
1382 * the direct access to this event. That means that within the same
1383 * task, some events may be directly accessible and some other may not,
1384 * if the user changes the value of sysctl_perf_user_accesss in the
1385 * meantime.
1386 */
1387
1388 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;
1389
1390 /*
1391 * We must enable userspace access *before* advertising in the user page
1392 * that it is possible to do so to avoid any race.
1393 * And we must notify all cpus here because threads that currently run
1394 * on other cpus will try to directly access the counter too without
1395 * calling pmu_sbi_ctr_start.
1396 */
1397 if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS)
1398 on_each_cpu_mask(mm_cpumask(mm),
1399 pmu_sbi_set_scounteren, (void *)event, 1);
1400 }
1401
pmu_sbi_event_unmapped(struct perf_event * event,struct mm_struct * mm)1402 static void pmu_sbi_event_unmapped(struct perf_event *event, struct mm_struct *mm)
1403 {
1404 if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS)
1405 return;
1406
1407 if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) {
1408 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES &&
1409 event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) {
1410 return;
1411 }
1412 }
1413
1414 /*
1415 * Here we can directly remove user access since the user does not have
1416 * access to the user page anymore so we avoid the racy window where the
1417 * user could have read cap_user_rdpmc to true right before we disable
1418 * it.
1419 */
1420 event->hw.flags &= ~PERF_EVENT_FLAG_USER_READ_CNT;
1421
1422 if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS)
1423 on_each_cpu_mask(mm_cpumask(mm),
1424 pmu_sbi_reset_scounteren, (void *)event, 1);
1425 }
1426
riscv_pmu_update_counter_access(void * info)1427 static void riscv_pmu_update_counter_access(void *info)
1428 {
1429 if (sysctl_perf_user_access == SYSCTL_LEGACY)
1430 csr_write(CSR_SCOUNTEREN, 0x7);
1431 else
1432 csr_write(CSR_SCOUNTEREN, 0x2);
1433 }
1434
riscv_pmu_proc_user_access_handler(const struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)1435 static int riscv_pmu_proc_user_access_handler(const struct ctl_table *table,
1436 int write, void *buffer,
1437 size_t *lenp, loff_t *ppos)
1438 {
1439 int prev = sysctl_perf_user_access;
1440 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
1441
1442 /*
1443 * Test against the previous value since we clear SCOUNTEREN when
1444 * sysctl_perf_user_access is set to SYSCTL_USER_ACCESS, but we should
1445 * not do that if that was already the case.
1446 */
1447 if (ret || !write || prev == sysctl_perf_user_access)
1448 return ret;
1449
1450 on_each_cpu(riscv_pmu_update_counter_access, NULL, 1);
1451
1452 return 0;
1453 }
1454
1455 static const struct ctl_table sbi_pmu_sysctl_table[] = {
1456 {
1457 .procname = "perf_user_access",
1458 .data = &sysctl_perf_user_access,
1459 .maxlen = sizeof(unsigned int),
1460 .mode = 0644,
1461 .proc_handler = riscv_pmu_proc_user_access_handler,
1462 .extra1 = SYSCTL_ZERO,
1463 .extra2 = SYSCTL_TWO,
1464 },
1465 };
1466
pmu_sbi_device_probe(struct platform_device * pdev)1467 static int pmu_sbi_device_probe(struct platform_device *pdev)
1468 {
1469 struct riscv_pmu *pmu = NULL;
1470 int ret = -ENODEV;
1471 int num_counters;
1472 bool irq_requested = false;
1473
1474 pr_info("SBI PMU extension is available\n");
1475 pmu = riscv_pmu_alloc();
1476 if (!pmu)
1477 return -ENOMEM;
1478
1479 num_counters = pmu_sbi_find_num_ctrs();
1480 if (num_counters < 0) {
1481 pr_err("SBI PMU extension doesn't provide any counters\n");
1482 goto out_free;
1483 }
1484
1485 /* It is possible to get from SBI more than max number of counters */
1486 if (num_counters > RISCV_MAX_COUNTERS) {
1487 num_counters = RISCV_MAX_COUNTERS;
1488 pr_info("SBI returned more than maximum number of counters. Limiting the number of counters to %d\n", num_counters);
1489 }
1490
1491 /* cache all the information about counters now */
1492 if (pmu_sbi_get_ctrinfo(num_counters, cmask))
1493 goto out_free;
1494
1495 ret = pmu_sbi_setup_irqs(pmu, pdev);
1496 if (ret < 0) {
1497 pr_info("Perf sampling/filtering is not supported as sscof extension is not available\n");
1498 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1499 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE;
1500 }
1501 irq_requested = (ret == 0);
1502
1503 pmu->pmu.attr_groups = riscv_pmu_attr_groups;
1504 pmu->pmu.parent = &pdev->dev;
1505 bitmap_copy(pmu->cmask, cmask, RISCV_MAX_COUNTERS);
1506 pmu->ctr_start = pmu_sbi_ctr_start;
1507 pmu->ctr_stop = pmu_sbi_ctr_stop;
1508 pmu->event_map = pmu_sbi_event_map;
1509 pmu->ctr_get_idx = pmu_sbi_ctr_get_idx;
1510 pmu->ctr_get_width = pmu_sbi_ctr_get_width;
1511 pmu->ctr_clear_idx = pmu_sbi_ctr_clear_idx;
1512 pmu->ctr_read = pmu_sbi_ctr_read;
1513 pmu->event_init = pmu_sbi_event_init;
1514 pmu->event_mapped = pmu_sbi_event_mapped;
1515 pmu->event_unmapped = pmu_sbi_event_unmapped;
1516 pmu->csr_index = pmu_sbi_csr_index;
1517
1518 ret = riscv_pm_pmu_register(pmu);
1519 if (ret)
1520 goto out_destroy;
1521
1522 ret = perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW);
1523 if (ret)
1524 goto out_destroy;
1525
1526 /* SBI PMU Snapsphot is only available in SBI v2.0 */
1527 if (sbi_v2_available) {
1528 int cpu;
1529
1530 ret = pmu_sbi_snapshot_alloc(pmu);
1531 if (ret)
1532 goto out_unregister;
1533
1534 cpu = get_cpu();
1535 ret = pmu_sbi_snapshot_setup(pmu, cpu);
1536 put_cpu();
1537
1538 if (ret) {
1539 /* Snapshot is an optional feature. Continue if not available */
1540 pmu_sbi_snapshot_free(pmu);
1541 } else {
1542 pr_info("SBI PMU snapshot detected\n");
1543 /*
1544 * We enable it once here for the boot cpu. If snapshot shmem setup
1545 * fails during cpu hotplug process, it will fail to start the cpu
1546 * as we can not handle hetergenous PMUs with different snapshot
1547 * capability.
1548 */
1549 static_branch_enable(&sbi_pmu_snapshot_available);
1550 }
1551 }
1552
1553 register_sysctl("kernel", sbi_pmu_sysctl_table);
1554
1555 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
1556 if (ret)
1557 goto out_unregister;
1558
1559 /* Asynchronously check which standard events are available */
1560 schedule_work(&check_std_events_work);
1561
1562 return 0;
1563
1564 out_unregister:
1565 perf_pmu_unregister(&pmu->pmu);
1566
1567 out_destroy:
1568 riscv_pmu_destroy(pmu);
1569 if (irq_requested) {
1570 free_percpu_irq(riscv_pmu_irq, pmu->hw_events);
1571 irq_dispose_mapping(riscv_pmu_irq);
1572 riscv_pmu_irq = 0;
1573 }
1574
1575 out_free:
1576 free_percpu(pmu->hw_events);
1577 kfree(pmu_ctr_list);
1578 pmu_ctr_list = NULL;
1579 kfree(pmu);
1580 return ret;
1581 }
1582
1583 static struct platform_driver pmu_sbi_driver = {
1584 .probe = pmu_sbi_device_probe,
1585 .driver = {
1586 .name = RISCV_PMU_SBI_PDEV_NAME,
1587 },
1588 };
1589
pmu_sbi_devinit(void)1590 static int __init pmu_sbi_devinit(void)
1591 {
1592 int ret;
1593 struct platform_device *pdev;
1594
1595 if (sbi_spec_version < sbi_mk_version(0, 3) ||
1596 !sbi_probe_extension(SBI_EXT_PMU)) {
1597 return 0;
1598 }
1599
1600 if (sbi_spec_version >= sbi_mk_version(2, 0))
1601 sbi_v2_available = true;
1602
1603 if (sbi_spec_version >= sbi_mk_version(3, 0))
1604 sbi_v3_available = true;
1605
1606 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_STARTING,
1607 "perf/riscv/pmu:starting",
1608 pmu_sbi_starting_cpu, pmu_sbi_dying_cpu);
1609 if (ret) {
1610 pr_err("CPU hotplug notifier could not be registered: %d\n",
1611 ret);
1612 return ret;
1613 }
1614
1615 ret = platform_driver_register(&pmu_sbi_driver);
1616 if (ret)
1617 return ret;
1618
1619 pdev = platform_device_register_simple(RISCV_PMU_SBI_PDEV_NAME, -1, NULL, 0);
1620 if (IS_ERR(pdev)) {
1621 platform_driver_unregister(&pmu_sbi_driver);
1622 return PTR_ERR(pdev);
1623 }
1624
1625 /* Notify legacy implementation that SBI pmu is available*/
1626 riscv_pmu_legacy_skip_init();
1627
1628 return ret;
1629 }
1630 device_initcall(pmu_sbi_devinit)
1631