1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * HiSilicon SoC L3C uncore Hardware event counters support
4 *
5 * Copyright (C) 2017 HiSilicon Limited
6 * Author: Anurup M <anurup.m@huawei.com>
7 * Shaokun Zhang <zhangshaokun@hisilicon.com>
8 *
9 * This code is based on the uncore PMUs like arm-cci and arm-ccn.
10 */
11 #include <linux/acpi.h>
12 #include <linux/bug.h>
13 #include <linux/cpuhotplug.h>
14 #include <linux/interrupt.h>
15 #include <linux/irq.h>
16 #include <linux/list.h>
17 #include <linux/smp.h>
18
19 #include "hisi_uncore_pmu.h"
20
21 /* L3C register definition */
22 #define L3C_PERF_CTRL 0x0408
23 #define L3C_INT_MASK 0x0800
24 #define L3C_INT_STATUS 0x0808
25 #define L3C_INT_CLEAR 0x080c
26 #define L3C_CORE_CTRL 0x1b04
27 #define L3C_TRACETAG_CTRL 0x1b20
28 #define L3C_DATSRC_TYPE 0x1b48
29 #define L3C_DATSRC_CTRL 0x1bf0
30 #define L3C_EVENT_CTRL 0x1c00
31 #define L3C_VERSION 0x1cf0
32 #define L3C_EVENT_TYPE0 0x1d00
33 /*
34 * If the HW version only supports a 48-bit counter, then
35 * bits [63:48] are reserved, which are Read-As-Zero and
36 * Writes-Ignored.
37 */
38 #define L3C_CNTR0_LOWER 0x1e00
39
40 /* L3C has 8-counters */
41 #define L3C_NR_COUNTERS 0x8
42 #define L3C_MAX_EXT 2
43
44 #define L3C_PERF_CTRL_EN 0x10000
45 #define L3C_TRACETAG_EN BIT(31)
46 #define L3C_TRACETAG_REQ_SHIFT 7
47 #define L3C_TRACETAG_MARK_EN BIT(0)
48 #define L3C_TRACETAG_REQ_EN (L3C_TRACETAG_MARK_EN | BIT(2))
49 #define L3C_TRACETAG_CORE_EN (L3C_TRACETAG_MARK_EN | BIT(3))
50 #define L3C_CORE_EN BIT(20)
51 #define L3C_COER_NONE 0x0
52 #define L3C_DATSRC_MASK 0xFF
53 #define L3C_DATSRC_SKT_EN BIT(23)
54 #define L3C_DATSRC_NONE 0x0
55 #define L3C_EVTYPE_NONE 0xff
56 #define L3C_V1_NR_EVENTS 0x59
57 #define L3C_V2_NR_EVENTS 0xFF
58
59 HISI_PMU_EVENT_ATTR_EXTRACTOR(ext, config, 17, 16);
60 /*
61 * Remain the config1:0-7 for backward compatibility if some existing users
62 * hardcode the config1:0-7 directly without parsing the sysfs attribute.
63 */
64 HISI_PMU_EVENT_ATTR_EXTRACTOR(tt_core_deprecated, config1, 7, 0);
65 HISI_PMU_EVENT_ATTR_EXTRACTOR(tt_req, config1, 10, 8);
66 HISI_PMU_EVENT_ATTR_EXTRACTOR(datasrc_cfg, config1, 15, 11);
67 HISI_PMU_EVENT_ATTR_EXTRACTOR(datasrc_skt, config1, 16, 16);
68 HISI_PMU_EVENT_ATTR_EXTRACTOR(tt_core, config2, 15, 0);
69
70 struct hisi_l3c_pmu {
71 struct hisi_pmu l3c_pmu;
72
73 /* MMIO and IRQ resources for extension events */
74 void __iomem *ext_base[L3C_MAX_EXT];
75 int ext_irq[L3C_MAX_EXT];
76 int ext_num;
77 };
78
79 #define to_hisi_l3c_pmu(_l3c_pmu) \
80 container_of(_l3c_pmu, struct hisi_l3c_pmu, l3c_pmu)
81
82 /*
83 * The hardware counter idx used in counter enable/disable,
84 * interrupt enable/disable and status check, etc.
85 */
86 #define L3C_HW_IDX(_cntr_idx) ((_cntr_idx) % L3C_NR_COUNTERS)
87
88 /* Range of ext counters in used mask. */
89 #define L3C_CNTR_EXT_L(_ext) (((_ext) + 1) * L3C_NR_COUNTERS)
90 #define L3C_CNTR_EXT_H(_ext) (((_ext) + 2) * L3C_NR_COUNTERS)
91
92 struct hisi_l3c_pmu_ext {
93 bool support_ext;
94 };
95
96 static enum cpuhp_state hisi_l3c_pmu_cpuhp_state;
97
support_ext(struct hisi_l3c_pmu * pmu)98 static bool support_ext(struct hisi_l3c_pmu *pmu)
99 {
100 struct hisi_l3c_pmu_ext *l3c_pmu_ext = pmu->l3c_pmu.dev_info->private;
101
102 return l3c_pmu_ext->support_ext;
103 }
104
105 /*
106 * tt_core was extended to cover all the CPUs sharing the L3 and was moved from
107 * config1:0-7 to config2:0-*. Try it first and fallback to tt_core_deprecated
108 * if user's still using the deprecated one.
109 */
hisi_l3c_pmu_get_tt_core(struct perf_event * event)110 static u32 hisi_l3c_pmu_get_tt_core(struct perf_event *event)
111 {
112 u32 core = hisi_get_tt_core(event);
113
114 if (core)
115 return core;
116
117 return hisi_get_tt_core_deprecated(event);
118 }
119
hisi_l3c_pmu_get_event_idx(struct perf_event * event)120 static int hisi_l3c_pmu_get_event_idx(struct perf_event *event)
121 {
122 struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
123 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
124 unsigned long *used_mask = l3c_pmu->pmu_events.used_mask;
125 int ext = hisi_get_ext(event);
126 int idx;
127
128 /*
129 * For an L3C PMU that supports extension events, we can monitor
130 * maximum 2 * num_counters to 3 * num_counters events, depending on
131 * the number of ext regions supported by hardware. Thus use bit
132 * [0, num_counters - 1] for normal events and bit
133 * [ext * num_counters, (ext + 1) * num_counters - 1] for extension
134 * events. The idx allocation will keep unchanged for normal events and
135 * we can also use the idx to distinguish whether it's an extension
136 * event or not.
137 *
138 * Since normal events and extension events locates on the different
139 * address space, save the base address to the event->hw.event_base.
140 */
141 if (ext && !support_ext(hisi_l3c_pmu))
142 return -EOPNOTSUPP;
143
144 if (ext)
145 event->hw.event_base = (unsigned long)hisi_l3c_pmu->ext_base[ext - 1];
146 else
147 event->hw.event_base = (unsigned long)l3c_pmu->base;
148
149 ext -= 1;
150 idx = find_next_zero_bit(used_mask, L3C_CNTR_EXT_H(ext), L3C_CNTR_EXT_L(ext));
151
152 if (idx >= L3C_CNTR_EXT_H(ext))
153 return -EAGAIN;
154
155 set_bit(idx, used_mask);
156
157 return idx;
158 }
159
hisi_l3c_pmu_event_readl(struct hw_perf_event * hwc,u32 reg)160 static u32 hisi_l3c_pmu_event_readl(struct hw_perf_event *hwc, u32 reg)
161 {
162 return readl((void __iomem *)hwc->event_base + reg);
163 }
164
hisi_l3c_pmu_event_writel(struct hw_perf_event * hwc,u32 reg,u32 val)165 static void hisi_l3c_pmu_event_writel(struct hw_perf_event *hwc, u32 reg, u32 val)
166 {
167 writel(val, (void __iomem *)hwc->event_base + reg);
168 }
169
hisi_l3c_pmu_event_readq(struct hw_perf_event * hwc,u32 reg)170 static u64 hisi_l3c_pmu_event_readq(struct hw_perf_event *hwc, u32 reg)
171 {
172 return readq((void __iomem *)hwc->event_base + reg);
173 }
174
hisi_l3c_pmu_event_writeq(struct hw_perf_event * hwc,u32 reg,u64 val)175 static void hisi_l3c_pmu_event_writeq(struct hw_perf_event *hwc, u32 reg, u64 val)
176 {
177 writeq(val, (void __iomem *)hwc->event_base + reg);
178 }
179
hisi_l3c_pmu_config_req_tracetag(struct perf_event * event)180 static void hisi_l3c_pmu_config_req_tracetag(struct perf_event *event)
181 {
182 struct hw_perf_event *hwc = &event->hw;
183 u32 tt_req = hisi_get_tt_req(event);
184
185 if (tt_req) {
186 u32 val;
187
188 /* Set request-type for tracetag */
189 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
190 val |= tt_req << L3C_TRACETAG_REQ_SHIFT;
191 val |= L3C_TRACETAG_REQ_EN;
192 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
193
194 /* Enable request-tracetag statistics */
195 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
196 val |= L3C_TRACETAG_EN;
197 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
198 }
199 }
200
hisi_l3c_pmu_clear_req_tracetag(struct perf_event * event)201 static void hisi_l3c_pmu_clear_req_tracetag(struct perf_event *event)
202 {
203 struct hw_perf_event *hwc = &event->hw;
204 u32 tt_req = hisi_get_tt_req(event);
205
206 if (tt_req) {
207 u32 val;
208
209 /* Clear request-type */
210 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
211 val &= ~(tt_req << L3C_TRACETAG_REQ_SHIFT);
212 val &= ~L3C_TRACETAG_REQ_EN;
213 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
214
215 /* Disable request-tracetag statistics */
216 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
217 val &= ~L3C_TRACETAG_EN;
218 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
219 }
220 }
221
hisi_l3c_pmu_write_ds(struct perf_event * event,u32 ds_cfg)222 static void hisi_l3c_pmu_write_ds(struct perf_event *event, u32 ds_cfg)
223 {
224 struct hw_perf_event *hwc = &event->hw;
225 u32 reg, reg_idx, shift, val;
226 int idx = L3C_HW_IDX(hwc->idx);
227
228 /*
229 * Select the appropriate datasource register(L3C_DATSRC_TYPE0/1).
230 * There are 2 datasource ctrl register for the 8 hardware counters.
231 * Datasrc is 8-bits and for the former 4 hardware counters,
232 * L3C_DATSRC_TYPE0 is chosen. For the latter 4 hardware counters,
233 * L3C_DATSRC_TYPE1 is chosen.
234 */
235 reg = L3C_DATSRC_TYPE + (idx / 4) * 4;
236 reg_idx = idx % 4;
237 shift = 8 * reg_idx;
238
239 val = hisi_l3c_pmu_event_readl(hwc, reg);
240 val &= ~(L3C_DATSRC_MASK << shift);
241 val |= ds_cfg << shift;
242 hisi_l3c_pmu_event_writel(hwc, reg, val);
243 }
244
hisi_l3c_pmu_config_ds(struct perf_event * event)245 static void hisi_l3c_pmu_config_ds(struct perf_event *event)
246 {
247 struct hw_perf_event *hwc = &event->hw;
248 u32 ds_cfg = hisi_get_datasrc_cfg(event);
249 u32 ds_skt = hisi_get_datasrc_skt(event);
250
251 if (ds_cfg)
252 hisi_l3c_pmu_write_ds(event, ds_cfg);
253
254 if (ds_skt) {
255 u32 val;
256
257 val = hisi_l3c_pmu_event_readl(hwc, L3C_DATSRC_CTRL);
258 val |= L3C_DATSRC_SKT_EN;
259 hisi_l3c_pmu_event_writel(hwc, L3C_DATSRC_CTRL, val);
260 }
261 }
262
hisi_l3c_pmu_clear_ds(struct perf_event * event)263 static void hisi_l3c_pmu_clear_ds(struct perf_event *event)
264 {
265 struct hw_perf_event *hwc = &event->hw;
266 u32 ds_cfg = hisi_get_datasrc_cfg(event);
267 u32 ds_skt = hisi_get_datasrc_skt(event);
268
269 if (ds_cfg)
270 hisi_l3c_pmu_write_ds(event, L3C_DATSRC_NONE);
271
272 if (ds_skt) {
273 u32 val;
274
275 val = hisi_l3c_pmu_event_readl(hwc, L3C_DATSRC_CTRL);
276 val &= ~L3C_DATSRC_SKT_EN;
277 hisi_l3c_pmu_event_writel(hwc, L3C_DATSRC_CTRL, val);
278 }
279 }
280
hisi_l3c_pmu_config_core_tracetag(struct perf_event * event)281 static void hisi_l3c_pmu_config_core_tracetag(struct perf_event *event)
282 {
283 struct hw_perf_event *hwc = &event->hw;
284 u32 core = hisi_l3c_pmu_get_tt_core(event);
285
286 if (core) {
287 u32 val;
288
289 /* Config and enable core information */
290 hisi_l3c_pmu_event_writel(hwc, L3C_CORE_CTRL, core);
291 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
292 val |= L3C_CORE_EN;
293 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
294
295 /* Enable core-tracetag statistics */
296 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
297 val |= L3C_TRACETAG_CORE_EN;
298 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
299 }
300 }
301
hisi_l3c_pmu_clear_core_tracetag(struct perf_event * event)302 static void hisi_l3c_pmu_clear_core_tracetag(struct perf_event *event)
303 {
304 struct hw_perf_event *hwc = &event->hw;
305 u32 core = hisi_l3c_pmu_get_tt_core(event);
306
307 if (core) {
308 u32 val;
309
310 /* Clear core information */
311 hisi_l3c_pmu_event_writel(hwc, L3C_CORE_CTRL, L3C_COER_NONE);
312 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
313 val &= ~L3C_CORE_EN;
314 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
315
316 /* Disable core-tracetag statistics */
317 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
318 val &= ~L3C_TRACETAG_CORE_EN;
319 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
320 }
321 }
322
hisi_l3c_pmu_have_filter(struct perf_event * event)323 static bool hisi_l3c_pmu_have_filter(struct perf_event *event)
324 {
325 return hisi_get_tt_req(event) || hisi_l3c_pmu_get_tt_core(event) ||
326 hisi_get_datasrc_cfg(event) || hisi_get_datasrc_skt(event);
327 }
328
hisi_l3c_pmu_enable_filter(struct perf_event * event)329 static void hisi_l3c_pmu_enable_filter(struct perf_event *event)
330 {
331 if (hisi_l3c_pmu_have_filter(event)) {
332 hisi_l3c_pmu_config_req_tracetag(event);
333 hisi_l3c_pmu_config_core_tracetag(event);
334 hisi_l3c_pmu_config_ds(event);
335 }
336 }
337
hisi_l3c_pmu_disable_filter(struct perf_event * event)338 static void hisi_l3c_pmu_disable_filter(struct perf_event *event)
339 {
340 if (hisi_l3c_pmu_have_filter(event)) {
341 hisi_l3c_pmu_clear_ds(event);
342 hisi_l3c_pmu_clear_core_tracetag(event);
343 hisi_l3c_pmu_clear_req_tracetag(event);
344 }
345 }
346
hisi_l3c_pmu_check_filter(struct perf_event * event)347 static int hisi_l3c_pmu_check_filter(struct perf_event *event)
348 {
349 struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
350 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
351 int ext = hisi_get_ext(event);
352
353 if (ext < 0 || ext > hisi_l3c_pmu->ext_num)
354 return -EINVAL;
355
356 if (hisi_get_tt_core(event) && hisi_get_tt_core_deprecated(event))
357 return -EINVAL;
358
359 return 0;
360 }
361
362 /*
363 * Select the counter register offset using the counter index
364 */
hisi_l3c_pmu_get_counter_offset(int cntr_idx)365 static u32 hisi_l3c_pmu_get_counter_offset(int cntr_idx)
366 {
367 return L3C_CNTR0_LOWER + L3C_HW_IDX(cntr_idx) * 8;
368 }
369
hisi_l3c_pmu_read_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)370 static u64 hisi_l3c_pmu_read_counter(struct hisi_pmu *l3c_pmu,
371 struct hw_perf_event *hwc)
372 {
373 return hisi_l3c_pmu_event_readq(hwc, hisi_l3c_pmu_get_counter_offset(hwc->idx));
374 }
375
hisi_l3c_pmu_write_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc,u64 val)376 static void hisi_l3c_pmu_write_counter(struct hisi_pmu *l3c_pmu,
377 struct hw_perf_event *hwc, u64 val)
378 {
379 hisi_l3c_pmu_event_writeq(hwc, hisi_l3c_pmu_get_counter_offset(hwc->idx), val);
380 }
381
hisi_l3c_pmu_write_evtype(struct hisi_pmu * l3c_pmu,int idx,u32 type)382 static void hisi_l3c_pmu_write_evtype(struct hisi_pmu *l3c_pmu, int idx,
383 u32 type)
384 {
385 struct hw_perf_event *hwc = &l3c_pmu->pmu_events.hw_events[idx]->hw;
386 u32 reg, reg_idx, shift, val;
387
388 idx = L3C_HW_IDX(idx);
389
390 /*
391 * Select the appropriate event select register(L3C_EVENT_TYPE0/1).
392 * There are 2 event select registers for the 8 hardware counters.
393 * Event code is 8-bits and for the former 4 hardware counters,
394 * L3C_EVENT_TYPE0 is chosen. For the latter 4 hardware counters,
395 * L3C_EVENT_TYPE1 is chosen.
396 */
397 reg = L3C_EVENT_TYPE0 + (idx / 4) * 4;
398 reg_idx = idx % 4;
399 shift = 8 * reg_idx;
400
401 /* Write event code to L3C_EVENT_TYPEx Register */
402 val = hisi_l3c_pmu_event_readl(hwc, reg);
403 val &= ~(L3C_EVTYPE_NONE << shift);
404 val |= type << shift;
405 hisi_l3c_pmu_event_writel(hwc, reg, val);
406 }
407
hisi_l3c_pmu_start_counters(struct hisi_pmu * l3c_pmu)408 static void hisi_l3c_pmu_start_counters(struct hisi_pmu *l3c_pmu)
409 {
410 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
411 unsigned long *used_mask = l3c_pmu->pmu_events.used_mask;
412 unsigned long used_cntr = find_first_bit(used_mask, l3c_pmu->num_counters);
413 u32 val;
414 int i;
415
416 /*
417 * Check if any counter belongs to the normal range (instead of ext
418 * range). If so, enable it.
419 */
420 if (used_cntr < L3C_NR_COUNTERS) {
421 val = readl(l3c_pmu->base + L3C_PERF_CTRL);
422 val |= L3C_PERF_CTRL_EN;
423 writel(val, l3c_pmu->base + L3C_PERF_CTRL);
424 }
425
426 /* If not, do enable it on ext ranges. */
427 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
428 /* Find used counter in this ext range, skip the range if not. */
429 used_cntr = find_next_bit(used_mask, L3C_CNTR_EXT_H(i), L3C_CNTR_EXT_L(i));
430 if (used_cntr >= L3C_CNTR_EXT_H(i))
431 continue;
432
433 val = readl(hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
434 val |= L3C_PERF_CTRL_EN;
435 writel(val, hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
436 }
437 }
438
hisi_l3c_pmu_stop_counters(struct hisi_pmu * l3c_pmu)439 static void hisi_l3c_pmu_stop_counters(struct hisi_pmu *l3c_pmu)
440 {
441 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
442 unsigned long *used_mask = l3c_pmu->pmu_events.used_mask;
443 unsigned long used_cntr = find_first_bit(used_mask, l3c_pmu->num_counters);
444 u32 val;
445 int i;
446
447 /*
448 * Check if any counter belongs to the normal range (instead of ext
449 * range). If so, stop it.
450 */
451 if (used_cntr < L3C_NR_COUNTERS) {
452 val = readl(l3c_pmu->base + L3C_PERF_CTRL);
453 val &= ~L3C_PERF_CTRL_EN;
454 writel(val, l3c_pmu->base + L3C_PERF_CTRL);
455 }
456
457 /* If not, do stop it on ext ranges. */
458 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
459 /* Find used counter in this ext range, skip the range if not. */
460 used_cntr = find_next_bit(used_mask, L3C_CNTR_EXT_H(i), L3C_CNTR_EXT_L(i));
461 if (used_cntr >= L3C_CNTR_EXT_H(i))
462 continue;
463
464 val = readl(hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
465 val &= ~L3C_PERF_CTRL_EN;
466 writel(val, hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
467 }
468 }
469
hisi_l3c_pmu_enable_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)470 static void hisi_l3c_pmu_enable_counter(struct hisi_pmu *l3c_pmu,
471 struct hw_perf_event *hwc)
472 {
473 u32 val;
474
475 /* Enable counter index in L3C_EVENT_CTRL register */
476 val = hisi_l3c_pmu_event_readl(hwc, L3C_EVENT_CTRL);
477 val |= 1 << L3C_HW_IDX(hwc->idx);
478 hisi_l3c_pmu_event_writel(hwc, L3C_EVENT_CTRL, val);
479 }
480
hisi_l3c_pmu_disable_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)481 static void hisi_l3c_pmu_disable_counter(struct hisi_pmu *l3c_pmu,
482 struct hw_perf_event *hwc)
483 {
484 u32 val;
485
486 /* Clear counter index in L3C_EVENT_CTRL register */
487 val = hisi_l3c_pmu_event_readl(hwc, L3C_EVENT_CTRL);
488 val &= ~(1 << L3C_HW_IDX(hwc->idx));
489 hisi_l3c_pmu_event_writel(hwc, L3C_EVENT_CTRL, val);
490 }
491
hisi_l3c_pmu_enable_counter_int(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)492 static void hisi_l3c_pmu_enable_counter_int(struct hisi_pmu *l3c_pmu,
493 struct hw_perf_event *hwc)
494 {
495 u32 val;
496
497 val = hisi_l3c_pmu_event_readl(hwc, L3C_INT_MASK);
498 /* Write 0 to enable interrupt */
499 val &= ~(1 << L3C_HW_IDX(hwc->idx));
500 hisi_l3c_pmu_event_writel(hwc, L3C_INT_MASK, val);
501 }
502
hisi_l3c_pmu_disable_counter_int(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)503 static void hisi_l3c_pmu_disable_counter_int(struct hisi_pmu *l3c_pmu,
504 struct hw_perf_event *hwc)
505 {
506 u32 val;
507
508 val = hisi_l3c_pmu_event_readl(hwc, L3C_INT_MASK);
509 /* Write 1 to mask interrupt */
510 val |= 1 << L3C_HW_IDX(hwc->idx);
511 hisi_l3c_pmu_event_writel(hwc, L3C_INT_MASK, val);
512 }
513
hisi_l3c_pmu_get_int_status(struct hisi_pmu * l3c_pmu)514 static u32 hisi_l3c_pmu_get_int_status(struct hisi_pmu *l3c_pmu)
515 {
516 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
517 u32 ext_int, status, status_ext = 0;
518 int i;
519
520 status = readl(l3c_pmu->base + L3C_INT_STATUS);
521
522 if (!support_ext(hisi_l3c_pmu))
523 return status;
524
525 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
526 ext_int = readl(hisi_l3c_pmu->ext_base[i] + L3C_INT_STATUS);
527 status_ext |= ext_int << (L3C_NR_COUNTERS * i);
528 }
529
530 return status | (status_ext << L3C_NR_COUNTERS);
531 }
532
hisi_l3c_pmu_clear_int_status(struct hisi_pmu * l3c_pmu,int idx)533 static void hisi_l3c_pmu_clear_int_status(struct hisi_pmu *l3c_pmu, int idx)
534 {
535 struct hw_perf_event *hwc = &l3c_pmu->pmu_events.hw_events[idx]->hw;
536
537 hisi_l3c_pmu_event_writel(hwc, L3C_INT_CLEAR, 1 << L3C_HW_IDX(idx));
538 }
539
hisi_l3c_pmu_init_data(struct platform_device * pdev,struct hisi_pmu * l3c_pmu)540 static int hisi_l3c_pmu_init_data(struct platform_device *pdev,
541 struct hisi_pmu *l3c_pmu)
542 {
543 hisi_uncore_pmu_init_topology(l3c_pmu, &pdev->dev);
544
545 /*
546 * Use the SCCL_ID and CCL_ID to identify the L3C PMU, while
547 * SCCL_ID is in MPIDR[aff2] and CCL_ID is in MPIDR[aff1].
548 */
549 if (l3c_pmu->topo.sccl_id < 0) {
550 dev_err(&pdev->dev, "Can not read l3c sccl-id!\n");
551 return -EINVAL;
552 }
553
554 if (l3c_pmu->topo.ccl_id < 0) {
555 dev_err(&pdev->dev, "Can not read l3c ccl-id!\n");
556 return -EINVAL;
557 }
558
559 l3c_pmu->dev_info = device_get_match_data(&pdev->dev);
560 if (!l3c_pmu->dev_info)
561 return -ENODEV;
562
563 l3c_pmu->base = devm_platform_ioremap_resource(pdev, 0);
564 if (IS_ERR(l3c_pmu->base)) {
565 dev_err(&pdev->dev, "ioremap failed for l3c_pmu resource\n");
566 return PTR_ERR(l3c_pmu->base);
567 }
568
569 l3c_pmu->identifier = readl(l3c_pmu->base + L3C_VERSION);
570
571 return 0;
572 }
573
hisi_l3c_pmu_init_ext(struct hisi_pmu * l3c_pmu,struct platform_device * pdev)574 static int hisi_l3c_pmu_init_ext(struct hisi_pmu *l3c_pmu, struct platform_device *pdev)
575 {
576 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
577 int ret, irq, ext_num, i;
578 char *irqname;
579
580 /* HiSilicon L3C PMU supporting ext should have more than 1 irq resources. */
581 ext_num = platform_irq_count(pdev);
582 if (ext_num < L3C_MAX_EXT)
583 return -ENODEV;
584
585 /*
586 * The number of ext supported equals the number of irq - 1, since one
587 * of the irqs belongs to the normal part of PMU.
588 */
589 hisi_l3c_pmu->ext_num = ext_num - 1;
590
591 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
592 hisi_l3c_pmu->ext_base[i] = devm_platform_ioremap_resource(pdev, i + 1);
593 if (IS_ERR(hisi_l3c_pmu->ext_base[i]))
594 return PTR_ERR(hisi_l3c_pmu->ext_base[i]);
595
596 irq = platform_get_irq(pdev, i + 1);
597 if (irq < 0)
598 return irq;
599
600 irqname = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s ext%d",
601 dev_name(&pdev->dev), i + 1);
602 if (!irqname)
603 return -ENOMEM;
604
605 ret = devm_request_irq(&pdev->dev, irq, hisi_uncore_pmu_isr,
606 IRQF_NOBALANCING | IRQF_NO_THREAD,
607 irqname, l3c_pmu);
608 if (ret < 0)
609 return ret;
610
611 hisi_l3c_pmu->ext_irq[i] = irq;
612 }
613
614 return 0;
615 }
616
617 static struct attribute *hisi_l3c_pmu_v1_format_attr[] = {
618 HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
619 NULL,
620 };
621
622 static const struct attribute_group hisi_l3c_pmu_v1_format_group = {
623 .name = "format",
624 .attrs = hisi_l3c_pmu_v1_format_attr,
625 };
626
627 static struct attribute *hisi_l3c_pmu_v2_format_attr[] = {
628 HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
629 HISI_PMU_FORMAT_ATTR(tt_core_deprecated, "config1:0-7"),
630 HISI_PMU_FORMAT_ATTR(tt_req, "config1:8-10"),
631 HISI_PMU_FORMAT_ATTR(datasrc_cfg, "config1:11-15"),
632 HISI_PMU_FORMAT_ATTR(datasrc_skt, "config1:16"),
633 HISI_PMU_FORMAT_ATTR(tt_core, "config2:0-15"),
634 NULL
635 };
636
637 static const struct attribute_group hisi_l3c_pmu_v2_format_group = {
638 .name = "format",
639 .attrs = hisi_l3c_pmu_v2_format_attr,
640 };
641
642 static struct attribute *hisi_l3c_pmu_v3_format_attr[] = {
643 HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
644 HISI_PMU_FORMAT_ATTR(ext, "config:16-17"),
645 HISI_PMU_FORMAT_ATTR(tt_core_deprecated, "config1:0-7"),
646 HISI_PMU_FORMAT_ATTR(tt_req, "config1:8-10"),
647 HISI_PMU_FORMAT_ATTR(tt_core, "config2:0-15"),
648 NULL
649 };
650
651 static const struct attribute_group hisi_l3c_pmu_v3_format_group = {
652 .name = "format",
653 .attrs = hisi_l3c_pmu_v3_format_attr,
654 };
655
656 static struct attribute *hisi_l3c_pmu_v1_events_attr[] = {
657 HISI_PMU_EVENT_ATTR(rd_cpipe, 0x00),
658 HISI_PMU_EVENT_ATTR(wr_cpipe, 0x01),
659 HISI_PMU_EVENT_ATTR(rd_hit_cpipe, 0x02),
660 HISI_PMU_EVENT_ATTR(wr_hit_cpipe, 0x03),
661 HISI_PMU_EVENT_ATTR(victim_num, 0x04),
662 HISI_PMU_EVENT_ATTR(rd_spipe, 0x20),
663 HISI_PMU_EVENT_ATTR(wr_spipe, 0x21),
664 HISI_PMU_EVENT_ATTR(rd_hit_spipe, 0x22),
665 HISI_PMU_EVENT_ATTR(wr_hit_spipe, 0x23),
666 HISI_PMU_EVENT_ATTR(back_invalid, 0x29),
667 HISI_PMU_EVENT_ATTR(retry_cpu, 0x40),
668 HISI_PMU_EVENT_ATTR(retry_ring, 0x41),
669 HISI_PMU_EVENT_ATTR(prefetch_drop, 0x42),
670 NULL,
671 };
672
673 static const struct attribute_group hisi_l3c_pmu_v1_events_group = {
674 .name = "events",
675 .attrs = hisi_l3c_pmu_v1_events_attr,
676 };
677
678 static struct attribute *hisi_l3c_pmu_v2_events_attr[] = {
679 HISI_PMU_EVENT_ATTR(l3c_hit, 0x48),
680 HISI_PMU_EVENT_ATTR(cycles, 0x7f),
681 HISI_PMU_EVENT_ATTR(l3c_ref, 0xb8),
682 HISI_PMU_EVENT_ATTR(dat_access, 0xb9),
683 NULL
684 };
685
686 static const struct attribute_group hisi_l3c_pmu_v2_events_group = {
687 .name = "events",
688 .attrs = hisi_l3c_pmu_v2_events_attr,
689 };
690
691 static struct attribute *hisi_l3c_pmu_v3_events_attr[] = {
692 HISI_PMU_EVENT_ATTR(rd_spipe, 0x18),
693 HISI_PMU_EVENT_ATTR(rd_hit_spipe, 0x19),
694 HISI_PMU_EVENT_ATTR(wr_spipe, 0x1a),
695 HISI_PMU_EVENT_ATTR(wr_hit_spipe, 0x1b),
696 HISI_PMU_EVENT_ATTR(io_rd_spipe, 0x1c),
697 HISI_PMU_EVENT_ATTR(io_rd_hit_spipe, 0x1d),
698 HISI_PMU_EVENT_ATTR(io_wr_spipe, 0x1e),
699 HISI_PMU_EVENT_ATTR(io_wr_hit_spipe, 0x1f),
700 HISI_PMU_EVENT_ATTR(cycles, 0x7f),
701 HISI_PMU_EVENT_ATTR(l3c_ref, 0xbc),
702 HISI_PMU_EVENT_ATTR(l3c2ring, 0xbd),
703 NULL
704 };
705
706 static const struct attribute_group hisi_l3c_pmu_v3_events_group = {
707 .name = "events",
708 .attrs = hisi_l3c_pmu_v3_events_attr,
709 };
710
711 static const struct attribute_group *hisi_l3c_pmu_v1_attr_groups[] = {
712 &hisi_l3c_pmu_v1_format_group,
713 &hisi_l3c_pmu_v1_events_group,
714 &hisi_pmu_cpumask_attr_group,
715 &hisi_pmu_identifier_group,
716 NULL,
717 };
718
719 static const struct attribute_group *hisi_l3c_pmu_v2_attr_groups[] = {
720 &hisi_l3c_pmu_v2_format_group,
721 &hisi_l3c_pmu_v2_events_group,
722 &hisi_pmu_cpumask_attr_group,
723 &hisi_pmu_identifier_group,
724 NULL
725 };
726
727 static const struct attribute_group *hisi_l3c_pmu_v3_attr_groups[] = {
728 &hisi_l3c_pmu_v3_format_group,
729 &hisi_l3c_pmu_v3_events_group,
730 &hisi_pmu_cpumask_attr_group,
731 &hisi_pmu_identifier_group,
732 NULL
733 };
734
735 static struct hisi_l3c_pmu_ext hisi_l3c_pmu_support_ext = {
736 .support_ext = true,
737 };
738
739 static struct hisi_l3c_pmu_ext hisi_l3c_pmu_not_support_ext = {
740 .support_ext = false,
741 };
742
743 static const struct hisi_pmu_dev_info hisi_l3c_pmu_v1 = {
744 .attr_groups = hisi_l3c_pmu_v1_attr_groups,
745 .counter_bits = 48,
746 .check_event = L3C_V1_NR_EVENTS,
747 .private = &hisi_l3c_pmu_not_support_ext,
748 };
749
750 static const struct hisi_pmu_dev_info hisi_l3c_pmu_v2 = {
751 .attr_groups = hisi_l3c_pmu_v2_attr_groups,
752 .counter_bits = 64,
753 .check_event = L3C_V2_NR_EVENTS,
754 .private = &hisi_l3c_pmu_not_support_ext,
755 };
756
757 static const struct hisi_pmu_dev_info hisi_l3c_pmu_v3 = {
758 .attr_groups = hisi_l3c_pmu_v3_attr_groups,
759 .counter_bits = 64,
760 .check_event = L3C_V2_NR_EVENTS,
761 .private = &hisi_l3c_pmu_support_ext,
762 };
763
764 static const struct hisi_uncore_ops hisi_uncore_l3c_ops = {
765 .write_evtype = hisi_l3c_pmu_write_evtype,
766 .get_event_idx = hisi_l3c_pmu_get_event_idx,
767 .start_counters = hisi_l3c_pmu_start_counters,
768 .stop_counters = hisi_l3c_pmu_stop_counters,
769 .enable_counter = hisi_l3c_pmu_enable_counter,
770 .disable_counter = hisi_l3c_pmu_disable_counter,
771 .enable_counter_int = hisi_l3c_pmu_enable_counter_int,
772 .disable_counter_int = hisi_l3c_pmu_disable_counter_int,
773 .write_counter = hisi_l3c_pmu_write_counter,
774 .read_counter = hisi_l3c_pmu_read_counter,
775 .get_int_status = hisi_l3c_pmu_get_int_status,
776 .clear_int_status = hisi_l3c_pmu_clear_int_status,
777 .enable_filter = hisi_l3c_pmu_enable_filter,
778 .disable_filter = hisi_l3c_pmu_disable_filter,
779 .check_filter = hisi_l3c_pmu_check_filter,
780 };
781
hisi_l3c_pmu_dev_probe(struct platform_device * pdev,struct hisi_pmu * l3c_pmu)782 static int hisi_l3c_pmu_dev_probe(struct platform_device *pdev,
783 struct hisi_pmu *l3c_pmu)
784 {
785 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
786 struct hisi_l3c_pmu_ext *l3c_pmu_dev_ext;
787 int ret;
788
789 ret = hisi_l3c_pmu_init_data(pdev, l3c_pmu);
790 if (ret)
791 return ret;
792
793 ret = hisi_uncore_pmu_init_irq(l3c_pmu, pdev);
794 if (ret)
795 return ret;
796
797 l3c_pmu->pmu_events.attr_groups = l3c_pmu->dev_info->attr_groups;
798 l3c_pmu->counter_bits = l3c_pmu->dev_info->counter_bits;
799 l3c_pmu->check_event = l3c_pmu->dev_info->check_event;
800 l3c_pmu->num_counters = L3C_NR_COUNTERS;
801 l3c_pmu->ops = &hisi_uncore_l3c_ops;
802 l3c_pmu->dev = &pdev->dev;
803 l3c_pmu->on_cpu = -1;
804
805 l3c_pmu_dev_ext = l3c_pmu->dev_info->private;
806 if (l3c_pmu_dev_ext->support_ext) {
807 ret = hisi_l3c_pmu_init_ext(l3c_pmu, pdev);
808 if (ret)
809 return ret;
810 /*
811 * The extension events have their own counters with the
812 * same number of the normal events counters. So we can
813 * have at maximum num_counters * ext events monitored.
814 */
815 l3c_pmu->num_counters += hisi_l3c_pmu->ext_num * L3C_NR_COUNTERS;
816 }
817
818 return 0;
819 }
820
hisi_l3c_pmu_probe(struct platform_device * pdev)821 static int hisi_l3c_pmu_probe(struct platform_device *pdev)
822 {
823 struct hisi_l3c_pmu *hisi_l3c_pmu;
824 struct hisi_pmu *l3c_pmu;
825 char *name;
826 int ret;
827
828 hisi_l3c_pmu = devm_kzalloc(&pdev->dev, sizeof(*hisi_l3c_pmu), GFP_KERNEL);
829 if (!hisi_l3c_pmu)
830 return -ENOMEM;
831
832 l3c_pmu = &hisi_l3c_pmu->l3c_pmu;
833 platform_set_drvdata(pdev, l3c_pmu);
834
835 ret = hisi_l3c_pmu_dev_probe(pdev, l3c_pmu);
836 if (ret)
837 return ret;
838
839 if (l3c_pmu->topo.sub_id >= 0)
840 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%d_l3c%d_%d",
841 l3c_pmu->topo.sccl_id, l3c_pmu->topo.ccl_id,
842 l3c_pmu->topo.sub_id);
843 else
844 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%d_l3c%d",
845 l3c_pmu->topo.sccl_id, l3c_pmu->topo.ccl_id);
846 if (!name)
847 return -ENOMEM;
848
849 ret = cpuhp_state_add_instance(hisi_l3c_pmu_cpuhp_state, &l3c_pmu->node);
850 if (ret) {
851 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
852 return ret;
853 }
854
855 hisi_pmu_init(l3c_pmu, THIS_MODULE);
856
857 ret = perf_pmu_register(&l3c_pmu->pmu, name, -1);
858 if (ret) {
859 dev_err(l3c_pmu->dev, "L3C PMU register failed!\n");
860 cpuhp_state_remove_instance_nocalls(
861 hisi_l3c_pmu_cpuhp_state, &l3c_pmu->node);
862 }
863
864 return ret;
865 }
866
hisi_l3c_pmu_remove(struct platform_device * pdev)867 static void hisi_l3c_pmu_remove(struct platform_device *pdev)
868 {
869 struct hisi_pmu *l3c_pmu = platform_get_drvdata(pdev);
870
871 perf_pmu_unregister(&l3c_pmu->pmu);
872 cpuhp_state_remove_instance_nocalls(hisi_l3c_pmu_cpuhp_state,
873 &l3c_pmu->node);
874 }
875
876 static const struct acpi_device_id hisi_l3c_pmu_acpi_match[] = {
877 { "HISI0213", (kernel_ulong_t)&hisi_l3c_pmu_v1 },
878 { "HISI0214", (kernel_ulong_t)&hisi_l3c_pmu_v2 },
879 { "HISI0215", (kernel_ulong_t)&hisi_l3c_pmu_v3 },
880 {}
881 };
882 MODULE_DEVICE_TABLE(acpi, hisi_l3c_pmu_acpi_match);
883
884 static struct platform_driver hisi_l3c_pmu_driver = {
885 .driver = {
886 .name = "hisi_l3c_pmu",
887 .acpi_match_table = ACPI_PTR(hisi_l3c_pmu_acpi_match),
888 .suppress_bind_attrs = true,
889 },
890 .probe = hisi_l3c_pmu_probe,
891 .remove = hisi_l3c_pmu_remove,
892 };
893
hisi_l3c_pmu_online_cpu(unsigned int cpu,struct hlist_node * node)894 static int hisi_l3c_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
895 {
896 struct hisi_pmu *l3c_pmu = hlist_entry_safe(node, struct hisi_pmu, node);
897 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
898 int ret, i;
899
900 ret = hisi_uncore_pmu_online_cpu(cpu, node);
901 if (ret)
902 return ret;
903
904 /* Avoid L3C pmu not supporting ext from ext irq migrating. */
905 if (!support_ext(hisi_l3c_pmu))
906 return 0;
907
908 for (i = 0; i < hisi_l3c_pmu->ext_num; i++)
909 WARN_ON(irq_set_affinity(hisi_l3c_pmu->ext_irq[i],
910 cpumask_of(l3c_pmu->on_cpu)));
911
912 return 0;
913 }
914
hisi_l3c_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)915 static int hisi_l3c_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
916 {
917 struct hisi_pmu *l3c_pmu = hlist_entry_safe(node, struct hisi_pmu, node);
918 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
919 int ret, i;
920
921 ret = hisi_uncore_pmu_offline_cpu(cpu, node);
922 if (ret)
923 return ret;
924
925 /* If failed to find any available CPU, skip irq migration. */
926 if (l3c_pmu->on_cpu < 0)
927 return 0;
928
929 /* Avoid L3C pmu not supporting ext from ext irq migrating. */
930 if (!support_ext(hisi_l3c_pmu))
931 return 0;
932
933 for (i = 0; i < hisi_l3c_pmu->ext_num; i++)
934 WARN_ON(irq_set_affinity(hisi_l3c_pmu->ext_irq[i],
935 cpumask_of(l3c_pmu->on_cpu)));
936
937 return 0;
938 }
939
hisi_l3c_pmu_module_init(void)940 static int __init hisi_l3c_pmu_module_init(void)
941 {
942 int ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
943 "AP_PERF_ARM_HISI_L3_ONLINE",
944 hisi_l3c_pmu_online_cpu,
945 hisi_l3c_pmu_offline_cpu);
946 if (ret < 0) {
947 pr_err("L3C PMU: Error setup hotplug, ret = %d\n", ret);
948 return ret;
949 }
950
951 hisi_l3c_pmu_cpuhp_state = ret;
952
953 ret = platform_driver_register(&hisi_l3c_pmu_driver);
954 if (ret)
955 cpuhp_remove_multi_state(hisi_l3c_pmu_cpuhp_state);
956
957 return ret;
958 }
959 module_init(hisi_l3c_pmu_module_init);
960
hisi_l3c_pmu_module_exit(void)961 static void __exit hisi_l3c_pmu_module_exit(void)
962 {
963 platform_driver_unregister(&hisi_l3c_pmu_driver);
964 cpuhp_remove_multi_state(hisi_l3c_pmu_cpuhp_state);
965 }
966 module_exit(hisi_l3c_pmu_module_exit);
967
968 MODULE_IMPORT_NS("HISI_PMU");
969 MODULE_DESCRIPTION("HiSilicon SoC L3C uncore PMU driver");
970 MODULE_LICENSE("GPL v2");
971 MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");
972 MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
973