1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * HiSilicon SoC L3C uncore Hardware event counters support
4 *
5 * Copyright (C) 2017 HiSilicon Limited
6 * Author: Anurup M <anurup.m@huawei.com>
7 * Shaokun Zhang <zhangshaokun@hisilicon.com>
8 *
9 * This code is based on the uncore PMUs like arm-cci and arm-ccn.
10 */
11 #include <linux/acpi.h>
12 #include <linux/bug.h>
13 #include <linux/cpuhotplug.h>
14 #include <linux/interrupt.h>
15 #include <linux/irq.h>
16 #include <linux/list.h>
17 #include <linux/smp.h>
18
19 #include "hisi_uncore_pmu.h"
20
21 /* L3C register definition */
22 #define L3C_PERF_CTRL 0x0408
23 #define L3C_INT_MASK 0x0800
24 #define L3C_INT_STATUS 0x0808
25 #define L3C_INT_CLEAR 0x080c
26 #define L3C_CORE_CTRL 0x1b04
27 #define L3C_TRACETAG_CTRL 0x1b20
28 #define L3C_DATSRC_TYPE 0x1b48
29 #define L3C_DATSRC_CTRL 0x1bf0
30 #define L3C_EVENT_CTRL 0x1c00
31 #define L3C_VERSION 0x1cf0
32 #define L3C_EVENT_TYPE0 0x1d00
33 /*
34 * If the HW version only supports a 48-bit counter, then
35 * bits [63:48] are reserved, which are Read-As-Zero and
36 * Writes-Ignored.
37 */
38 #define L3C_CNTR0_LOWER 0x1e00
39
40 /* L3C has 8-counters */
41 #define L3C_NR_COUNTERS 0x8
42 #define L3C_MAX_EXT 2
43
44 #define L3C_PERF_CTRL_EN 0x10000
45 #define L3C_TRACETAG_EN BIT(31)
46 #define L3C_TRACETAG_REQ_SHIFT 7
47 #define L3C_TRACETAG_MARK_EN BIT(0)
48 #define L3C_TRACETAG_REQ_EN (L3C_TRACETAG_MARK_EN | BIT(2))
49 #define L3C_TRACETAG_CORE_EN (L3C_TRACETAG_MARK_EN | BIT(3))
50 #define L3C_CORE_EN BIT(20)
51 #define L3C_COER_NONE 0x0
52 #define L3C_DATSRC_MASK 0xFF
53 #define L3C_DATSRC_SKT_EN BIT(23)
54 #define L3C_DATSRC_NONE 0x0
55 #define L3C_EVTYPE_NONE 0xff
56 #define L3C_V1_NR_EVENTS 0x59
57 #define L3C_V2_NR_EVENTS 0xFF
58
59 HISI_PMU_EVENT_ATTR_EXTRACTOR(ext, config, 17, 16);
60 HISI_PMU_EVENT_ATTR_EXTRACTOR(tt_req, config1, 10, 8);
61 HISI_PMU_EVENT_ATTR_EXTRACTOR(datasrc_cfg, config1, 15, 11);
62 HISI_PMU_EVENT_ATTR_EXTRACTOR(datasrc_skt, config1, 16, 16);
63 HISI_PMU_EVENT_ATTR_EXTRACTOR(tt_core, config2, 15, 0);
64
65 struct hisi_l3c_pmu {
66 struct hisi_pmu l3c_pmu;
67
68 /* MMIO and IRQ resources for extension events */
69 void __iomem *ext_base[L3C_MAX_EXT];
70 int ext_irq[L3C_MAX_EXT];
71 int ext_num;
72 };
73
74 #define to_hisi_l3c_pmu(_l3c_pmu) \
75 container_of(_l3c_pmu, struct hisi_l3c_pmu, l3c_pmu)
76
77 /*
78 * The hardware counter idx used in counter enable/disable,
79 * interrupt enable/disable and status check, etc.
80 */
81 #define L3C_HW_IDX(_cntr_idx) ((_cntr_idx) % L3C_NR_COUNTERS)
82
83 /* Range of ext counters in used mask. */
84 #define L3C_CNTR_EXT_L(_ext) (((_ext) + 1) * L3C_NR_COUNTERS)
85 #define L3C_CNTR_EXT_H(_ext) (((_ext) + 2) * L3C_NR_COUNTERS)
86
87 struct hisi_l3c_pmu_ext {
88 bool support_ext;
89 };
90
support_ext(struct hisi_l3c_pmu * pmu)91 static bool support_ext(struct hisi_l3c_pmu *pmu)
92 {
93 struct hisi_l3c_pmu_ext *l3c_pmu_ext = pmu->l3c_pmu.dev_info->private;
94
95 return l3c_pmu_ext->support_ext;
96 }
97
hisi_l3c_pmu_get_event_idx(struct perf_event * event)98 static int hisi_l3c_pmu_get_event_idx(struct perf_event *event)
99 {
100 struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
101 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
102 unsigned long *used_mask = l3c_pmu->pmu_events.used_mask;
103 int ext = hisi_get_ext(event);
104 int idx;
105
106 /*
107 * For an L3C PMU that supports extension events, we can monitor
108 * maximum 2 * num_counters to 3 * num_counters events, depending on
109 * the number of ext regions supported by hardware. Thus use bit
110 * [0, num_counters - 1] for normal events and bit
111 * [ext * num_counters, (ext + 1) * num_counters - 1] for extension
112 * events. The idx allocation will keep unchanged for normal events and
113 * we can also use the idx to distinguish whether it's an extension
114 * event or not.
115 *
116 * Since normal events and extension events locates on the different
117 * address space, save the base address to the event->hw.event_base.
118 */
119 if (ext && !support_ext(hisi_l3c_pmu))
120 return -EOPNOTSUPP;
121
122 if (ext)
123 event->hw.event_base = (unsigned long)hisi_l3c_pmu->ext_base[ext - 1];
124 else
125 event->hw.event_base = (unsigned long)l3c_pmu->base;
126
127 ext -= 1;
128 idx = find_next_zero_bit(used_mask, L3C_CNTR_EXT_H(ext), L3C_CNTR_EXT_L(ext));
129
130 if (idx >= L3C_CNTR_EXT_H(ext))
131 return -EAGAIN;
132
133 set_bit(idx, used_mask);
134
135 return idx;
136 }
137
hisi_l3c_pmu_event_readl(struct hw_perf_event * hwc,u32 reg)138 static u32 hisi_l3c_pmu_event_readl(struct hw_perf_event *hwc, u32 reg)
139 {
140 return readl((void __iomem *)hwc->event_base + reg);
141 }
142
hisi_l3c_pmu_event_writel(struct hw_perf_event * hwc,u32 reg,u32 val)143 static void hisi_l3c_pmu_event_writel(struct hw_perf_event *hwc, u32 reg, u32 val)
144 {
145 writel(val, (void __iomem *)hwc->event_base + reg);
146 }
147
hisi_l3c_pmu_event_readq(struct hw_perf_event * hwc,u32 reg)148 static u64 hisi_l3c_pmu_event_readq(struct hw_perf_event *hwc, u32 reg)
149 {
150 return readq((void __iomem *)hwc->event_base + reg);
151 }
152
hisi_l3c_pmu_event_writeq(struct hw_perf_event * hwc,u32 reg,u64 val)153 static void hisi_l3c_pmu_event_writeq(struct hw_perf_event *hwc, u32 reg, u64 val)
154 {
155 writeq(val, (void __iomem *)hwc->event_base + reg);
156 }
157
hisi_l3c_pmu_config_req_tracetag(struct perf_event * event)158 static void hisi_l3c_pmu_config_req_tracetag(struct perf_event *event)
159 {
160 struct hw_perf_event *hwc = &event->hw;
161 u32 tt_req = hisi_get_tt_req(event);
162
163 if (tt_req) {
164 u32 val;
165
166 /* Set request-type for tracetag */
167 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
168 val |= tt_req << L3C_TRACETAG_REQ_SHIFT;
169 val |= L3C_TRACETAG_REQ_EN;
170 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
171
172 /* Enable request-tracetag statistics */
173 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
174 val |= L3C_TRACETAG_EN;
175 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
176 }
177 }
178
hisi_l3c_pmu_clear_req_tracetag(struct perf_event * event)179 static void hisi_l3c_pmu_clear_req_tracetag(struct perf_event *event)
180 {
181 struct hw_perf_event *hwc = &event->hw;
182 u32 tt_req = hisi_get_tt_req(event);
183
184 if (tt_req) {
185 u32 val;
186
187 /* Clear request-type */
188 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
189 val &= ~(tt_req << L3C_TRACETAG_REQ_SHIFT);
190 val &= ~L3C_TRACETAG_REQ_EN;
191 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
192
193 /* Disable request-tracetag statistics */
194 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
195 val &= ~L3C_TRACETAG_EN;
196 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
197 }
198 }
199
hisi_l3c_pmu_write_ds(struct perf_event * event,u32 ds_cfg)200 static void hisi_l3c_pmu_write_ds(struct perf_event *event, u32 ds_cfg)
201 {
202 struct hw_perf_event *hwc = &event->hw;
203 u32 reg, reg_idx, shift, val;
204 int idx = L3C_HW_IDX(hwc->idx);
205
206 /*
207 * Select the appropriate datasource register(L3C_DATSRC_TYPE0/1).
208 * There are 2 datasource ctrl register for the 8 hardware counters.
209 * Datasrc is 8-bits and for the former 4 hardware counters,
210 * L3C_DATSRC_TYPE0 is chosen. For the latter 4 hardware counters,
211 * L3C_DATSRC_TYPE1 is chosen.
212 */
213 reg = L3C_DATSRC_TYPE + (idx / 4) * 4;
214 reg_idx = idx % 4;
215 shift = 8 * reg_idx;
216
217 val = hisi_l3c_pmu_event_readl(hwc, reg);
218 val &= ~(L3C_DATSRC_MASK << shift);
219 val |= ds_cfg << shift;
220 hisi_l3c_pmu_event_writel(hwc, reg, val);
221 }
222
hisi_l3c_pmu_config_ds(struct perf_event * event)223 static void hisi_l3c_pmu_config_ds(struct perf_event *event)
224 {
225 struct hw_perf_event *hwc = &event->hw;
226 u32 ds_cfg = hisi_get_datasrc_cfg(event);
227 u32 ds_skt = hisi_get_datasrc_skt(event);
228
229 if (ds_cfg)
230 hisi_l3c_pmu_write_ds(event, ds_cfg);
231
232 if (ds_skt) {
233 u32 val;
234
235 val = hisi_l3c_pmu_event_readl(hwc, L3C_DATSRC_CTRL);
236 val |= L3C_DATSRC_SKT_EN;
237 hisi_l3c_pmu_event_writel(hwc, L3C_DATSRC_CTRL, val);
238 }
239 }
240
hisi_l3c_pmu_clear_ds(struct perf_event * event)241 static void hisi_l3c_pmu_clear_ds(struct perf_event *event)
242 {
243 struct hw_perf_event *hwc = &event->hw;
244 u32 ds_cfg = hisi_get_datasrc_cfg(event);
245 u32 ds_skt = hisi_get_datasrc_skt(event);
246
247 if (ds_cfg)
248 hisi_l3c_pmu_write_ds(event, L3C_DATSRC_NONE);
249
250 if (ds_skt) {
251 u32 val;
252
253 val = hisi_l3c_pmu_event_readl(hwc, L3C_DATSRC_CTRL);
254 val &= ~L3C_DATSRC_SKT_EN;
255 hisi_l3c_pmu_event_writel(hwc, L3C_DATSRC_CTRL, val);
256 }
257 }
258
hisi_l3c_pmu_config_core_tracetag(struct perf_event * event)259 static void hisi_l3c_pmu_config_core_tracetag(struct perf_event *event)
260 {
261 struct hw_perf_event *hwc = &event->hw;
262 u32 core = hisi_get_tt_core(event);
263
264 if (core) {
265 u32 val;
266
267 /* Config and enable core information */
268 hisi_l3c_pmu_event_writel(hwc, L3C_CORE_CTRL, core);
269 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
270 val |= L3C_CORE_EN;
271 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
272
273 /* Enable core-tracetag statistics */
274 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
275 val |= L3C_TRACETAG_CORE_EN;
276 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
277 }
278 }
279
hisi_l3c_pmu_clear_core_tracetag(struct perf_event * event)280 static void hisi_l3c_pmu_clear_core_tracetag(struct perf_event *event)
281 {
282 struct hw_perf_event *hwc = &event->hw;
283 u32 core = hisi_get_tt_core(event);
284
285 if (core) {
286 u32 val;
287
288 /* Clear core information */
289 hisi_l3c_pmu_event_writel(hwc, L3C_CORE_CTRL, L3C_COER_NONE);
290 val = hisi_l3c_pmu_event_readl(hwc, L3C_PERF_CTRL);
291 val &= ~L3C_CORE_EN;
292 hisi_l3c_pmu_event_writel(hwc, L3C_PERF_CTRL, val);
293
294 /* Disable core-tracetag statistics */
295 val = hisi_l3c_pmu_event_readl(hwc, L3C_TRACETAG_CTRL);
296 val &= ~L3C_TRACETAG_CORE_EN;
297 hisi_l3c_pmu_event_writel(hwc, L3C_TRACETAG_CTRL, val);
298 }
299 }
300
hisi_l3c_pmu_have_filter(struct perf_event * event)301 static bool hisi_l3c_pmu_have_filter(struct perf_event *event)
302 {
303 return hisi_get_tt_req(event) || hisi_get_tt_core(event) ||
304 hisi_get_datasrc_cfg(event) || hisi_get_datasrc_skt(event);
305 }
306
hisi_l3c_pmu_enable_filter(struct perf_event * event)307 static void hisi_l3c_pmu_enable_filter(struct perf_event *event)
308 {
309 if (hisi_l3c_pmu_have_filter(event)) {
310 hisi_l3c_pmu_config_req_tracetag(event);
311 hisi_l3c_pmu_config_core_tracetag(event);
312 hisi_l3c_pmu_config_ds(event);
313 }
314 }
315
hisi_l3c_pmu_disable_filter(struct perf_event * event)316 static void hisi_l3c_pmu_disable_filter(struct perf_event *event)
317 {
318 if (hisi_l3c_pmu_have_filter(event)) {
319 hisi_l3c_pmu_clear_ds(event);
320 hisi_l3c_pmu_clear_core_tracetag(event);
321 hisi_l3c_pmu_clear_req_tracetag(event);
322 }
323 }
324
hisi_l3c_pmu_check_filter(struct perf_event * event)325 static int hisi_l3c_pmu_check_filter(struct perf_event *event)
326 {
327 struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
328 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
329 int ext = hisi_get_ext(event);
330
331 if (ext < 0 || ext > hisi_l3c_pmu->ext_num)
332 return -EINVAL;
333
334 return 0;
335 }
336
337 /*
338 * Select the counter register offset using the counter index
339 */
hisi_l3c_pmu_get_counter_offset(int cntr_idx)340 static u32 hisi_l3c_pmu_get_counter_offset(int cntr_idx)
341 {
342 return L3C_CNTR0_LOWER + L3C_HW_IDX(cntr_idx) * 8;
343 }
344
hisi_l3c_pmu_read_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)345 static u64 hisi_l3c_pmu_read_counter(struct hisi_pmu *l3c_pmu,
346 struct hw_perf_event *hwc)
347 {
348 return hisi_l3c_pmu_event_readq(hwc, hisi_l3c_pmu_get_counter_offset(hwc->idx));
349 }
350
hisi_l3c_pmu_write_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc,u64 val)351 static void hisi_l3c_pmu_write_counter(struct hisi_pmu *l3c_pmu,
352 struct hw_perf_event *hwc, u64 val)
353 {
354 hisi_l3c_pmu_event_writeq(hwc, hisi_l3c_pmu_get_counter_offset(hwc->idx), val);
355 }
356
hisi_l3c_pmu_write_evtype(struct hisi_pmu * l3c_pmu,int idx,u32 type)357 static void hisi_l3c_pmu_write_evtype(struct hisi_pmu *l3c_pmu, int idx,
358 u32 type)
359 {
360 struct hw_perf_event *hwc = &l3c_pmu->pmu_events.hw_events[idx]->hw;
361 u32 reg, reg_idx, shift, val;
362
363 idx = L3C_HW_IDX(idx);
364
365 /*
366 * Select the appropriate event select register(L3C_EVENT_TYPE0/1).
367 * There are 2 event select registers for the 8 hardware counters.
368 * Event code is 8-bits and for the former 4 hardware counters,
369 * L3C_EVENT_TYPE0 is chosen. For the latter 4 hardware counters,
370 * L3C_EVENT_TYPE1 is chosen.
371 */
372 reg = L3C_EVENT_TYPE0 + (idx / 4) * 4;
373 reg_idx = idx % 4;
374 shift = 8 * reg_idx;
375
376 /* Write event code to L3C_EVENT_TYPEx Register */
377 val = hisi_l3c_pmu_event_readl(hwc, reg);
378 val &= ~(L3C_EVTYPE_NONE << shift);
379 val |= type << shift;
380 hisi_l3c_pmu_event_writel(hwc, reg, val);
381 }
382
hisi_l3c_pmu_start_counters(struct hisi_pmu * l3c_pmu)383 static void hisi_l3c_pmu_start_counters(struct hisi_pmu *l3c_pmu)
384 {
385 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
386 unsigned long *used_mask = l3c_pmu->pmu_events.used_mask;
387 unsigned long used_cntr = find_first_bit(used_mask, l3c_pmu->num_counters);
388 u32 val;
389 int i;
390
391 /*
392 * Check if any counter belongs to the normal range (instead of ext
393 * range). If so, enable it.
394 */
395 if (used_cntr < L3C_NR_COUNTERS) {
396 val = readl(l3c_pmu->base + L3C_PERF_CTRL);
397 val |= L3C_PERF_CTRL_EN;
398 writel(val, l3c_pmu->base + L3C_PERF_CTRL);
399 }
400
401 /* If not, do enable it on ext ranges. */
402 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
403 /* Find used counter in this ext range, skip the range if not. */
404 used_cntr = find_next_bit(used_mask, L3C_CNTR_EXT_H(i), L3C_CNTR_EXT_L(i));
405 if (used_cntr >= L3C_CNTR_EXT_H(i))
406 continue;
407
408 val = readl(hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
409 val |= L3C_PERF_CTRL_EN;
410 writel(val, hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
411 }
412 }
413
hisi_l3c_pmu_stop_counters(struct hisi_pmu * l3c_pmu)414 static void hisi_l3c_pmu_stop_counters(struct hisi_pmu *l3c_pmu)
415 {
416 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
417 unsigned long *used_mask = l3c_pmu->pmu_events.used_mask;
418 unsigned long used_cntr = find_first_bit(used_mask, l3c_pmu->num_counters);
419 u32 val;
420 int i;
421
422 /*
423 * Check if any counter belongs to the normal range (instead of ext
424 * range). If so, stop it.
425 */
426 if (used_cntr < L3C_NR_COUNTERS) {
427 val = readl(l3c_pmu->base + L3C_PERF_CTRL);
428 val &= ~L3C_PERF_CTRL_EN;
429 writel(val, l3c_pmu->base + L3C_PERF_CTRL);
430 }
431
432 /* If not, do stop it on ext ranges. */
433 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
434 /* Find used counter in this ext range, skip the range if not. */
435 used_cntr = find_next_bit(used_mask, L3C_CNTR_EXT_H(i), L3C_CNTR_EXT_L(i));
436 if (used_cntr >= L3C_CNTR_EXT_H(i))
437 continue;
438
439 val = readl(hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
440 val &= ~L3C_PERF_CTRL_EN;
441 writel(val, hisi_l3c_pmu->ext_base[i] + L3C_PERF_CTRL);
442 }
443 }
444
hisi_l3c_pmu_enable_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)445 static void hisi_l3c_pmu_enable_counter(struct hisi_pmu *l3c_pmu,
446 struct hw_perf_event *hwc)
447 {
448 u32 val;
449
450 /* Enable counter index in L3C_EVENT_CTRL register */
451 val = hisi_l3c_pmu_event_readl(hwc, L3C_EVENT_CTRL);
452 val |= 1 << L3C_HW_IDX(hwc->idx);
453 hisi_l3c_pmu_event_writel(hwc, L3C_EVENT_CTRL, val);
454 }
455
hisi_l3c_pmu_disable_counter(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)456 static void hisi_l3c_pmu_disable_counter(struct hisi_pmu *l3c_pmu,
457 struct hw_perf_event *hwc)
458 {
459 u32 val;
460
461 /* Clear counter index in L3C_EVENT_CTRL register */
462 val = hisi_l3c_pmu_event_readl(hwc, L3C_EVENT_CTRL);
463 val &= ~(1 << L3C_HW_IDX(hwc->idx));
464 hisi_l3c_pmu_event_writel(hwc, L3C_EVENT_CTRL, val);
465 }
466
hisi_l3c_pmu_enable_counter_int(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)467 static void hisi_l3c_pmu_enable_counter_int(struct hisi_pmu *l3c_pmu,
468 struct hw_perf_event *hwc)
469 {
470 u32 val;
471
472 val = hisi_l3c_pmu_event_readl(hwc, L3C_INT_MASK);
473 /* Write 0 to enable interrupt */
474 val &= ~(1 << L3C_HW_IDX(hwc->idx));
475 hisi_l3c_pmu_event_writel(hwc, L3C_INT_MASK, val);
476 }
477
hisi_l3c_pmu_disable_counter_int(struct hisi_pmu * l3c_pmu,struct hw_perf_event * hwc)478 static void hisi_l3c_pmu_disable_counter_int(struct hisi_pmu *l3c_pmu,
479 struct hw_perf_event *hwc)
480 {
481 u32 val;
482
483 val = hisi_l3c_pmu_event_readl(hwc, L3C_INT_MASK);
484 /* Write 1 to mask interrupt */
485 val |= 1 << L3C_HW_IDX(hwc->idx);
486 hisi_l3c_pmu_event_writel(hwc, L3C_INT_MASK, val);
487 }
488
hisi_l3c_pmu_get_int_status(struct hisi_pmu * l3c_pmu)489 static u32 hisi_l3c_pmu_get_int_status(struct hisi_pmu *l3c_pmu)
490 {
491 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
492 u32 ext_int, status, status_ext = 0;
493 int i;
494
495 status = readl(l3c_pmu->base + L3C_INT_STATUS);
496
497 if (!support_ext(hisi_l3c_pmu))
498 return status;
499
500 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
501 ext_int = readl(hisi_l3c_pmu->ext_base[i] + L3C_INT_STATUS);
502 status_ext |= ext_int << (L3C_NR_COUNTERS * i);
503 }
504
505 return status | (status_ext << L3C_NR_COUNTERS);
506 }
507
hisi_l3c_pmu_clear_int_status(struct hisi_pmu * l3c_pmu,int idx)508 static void hisi_l3c_pmu_clear_int_status(struct hisi_pmu *l3c_pmu, int idx)
509 {
510 struct hw_perf_event *hwc = &l3c_pmu->pmu_events.hw_events[idx]->hw;
511
512 hisi_l3c_pmu_event_writel(hwc, L3C_INT_CLEAR, 1 << L3C_HW_IDX(idx));
513 }
514
hisi_l3c_pmu_init_data(struct platform_device * pdev,struct hisi_pmu * l3c_pmu)515 static int hisi_l3c_pmu_init_data(struct platform_device *pdev,
516 struct hisi_pmu *l3c_pmu)
517 {
518 hisi_uncore_pmu_init_topology(l3c_pmu, &pdev->dev);
519
520 /*
521 * Use the SCCL_ID and CCL_ID to identify the L3C PMU, while
522 * SCCL_ID is in MPIDR[aff2] and CCL_ID is in MPIDR[aff1].
523 */
524 if (l3c_pmu->topo.sccl_id < 0) {
525 dev_err(&pdev->dev, "Can not read l3c sccl-id!\n");
526 return -EINVAL;
527 }
528
529 if (l3c_pmu->topo.ccl_id < 0) {
530 dev_err(&pdev->dev, "Can not read l3c ccl-id!\n");
531 return -EINVAL;
532 }
533
534 l3c_pmu->dev_info = device_get_match_data(&pdev->dev);
535 if (!l3c_pmu->dev_info)
536 return -ENODEV;
537
538 l3c_pmu->base = devm_platform_ioremap_resource(pdev, 0);
539 if (IS_ERR(l3c_pmu->base)) {
540 dev_err(&pdev->dev, "ioremap failed for l3c_pmu resource\n");
541 return PTR_ERR(l3c_pmu->base);
542 }
543
544 l3c_pmu->identifier = readl(l3c_pmu->base + L3C_VERSION);
545
546 return 0;
547 }
548
hisi_l3c_pmu_init_ext(struct hisi_pmu * l3c_pmu,struct platform_device * pdev)549 static int hisi_l3c_pmu_init_ext(struct hisi_pmu *l3c_pmu, struct platform_device *pdev)
550 {
551 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
552 int ret, irq, ext_num, i;
553 char *irqname;
554
555 /* HiSilicon L3C PMU supporting ext should have more than 1 irq resources. */
556 ext_num = platform_irq_count(pdev);
557 if (ext_num < L3C_MAX_EXT)
558 return -ENODEV;
559
560 /*
561 * The number of ext supported equals the number of irq - 1, since one
562 * of the irqs belongs to the normal part of PMU.
563 */
564 hisi_l3c_pmu->ext_num = ext_num - 1;
565
566 for (i = 0; i < hisi_l3c_pmu->ext_num; i++) {
567 hisi_l3c_pmu->ext_base[i] = devm_platform_ioremap_resource(pdev, i + 1);
568 if (IS_ERR(hisi_l3c_pmu->ext_base[i]))
569 return PTR_ERR(hisi_l3c_pmu->ext_base[i]);
570
571 irq = platform_get_irq(pdev, i + 1);
572 if (irq < 0)
573 return irq;
574
575 irqname = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s ext%d",
576 dev_name(&pdev->dev), i + 1);
577 if (!irqname)
578 return -ENOMEM;
579
580 ret = devm_request_irq(&pdev->dev, irq, hisi_uncore_pmu_isr,
581 IRQF_NOBALANCING | IRQF_NO_THREAD,
582 irqname, l3c_pmu);
583 if (ret < 0)
584 return dev_err_probe(&pdev->dev, ret,
585 "Fail to request EXT IRQ: %d.\n", irq);
586
587 hisi_l3c_pmu->ext_irq[i] = irq;
588 }
589
590 return 0;
591 }
592
593 static struct attribute *hisi_l3c_pmu_v1_format_attr[] = {
594 HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
595 NULL,
596 };
597
598 static const struct attribute_group hisi_l3c_pmu_v1_format_group = {
599 .name = "format",
600 .attrs = hisi_l3c_pmu_v1_format_attr,
601 };
602
603 static struct attribute *hisi_l3c_pmu_v2_format_attr[] = {
604 HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
605 HISI_PMU_FORMAT_ATTR(tt_core, "config2:0-15"),
606 HISI_PMU_FORMAT_ATTR(tt_req, "config1:8-10"),
607 HISI_PMU_FORMAT_ATTR(datasrc_cfg, "config1:11-15"),
608 HISI_PMU_FORMAT_ATTR(datasrc_skt, "config1:16"),
609 NULL
610 };
611
612 static const struct attribute_group hisi_l3c_pmu_v2_format_group = {
613 .name = "format",
614 .attrs = hisi_l3c_pmu_v2_format_attr,
615 };
616
617 static struct attribute *hisi_l3c_pmu_v3_format_attr[] = {
618 HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
619 HISI_PMU_FORMAT_ATTR(ext, "config:16-17"),
620 HISI_PMU_FORMAT_ATTR(tt_req, "config1:8-10"),
621 HISI_PMU_FORMAT_ATTR(tt_core, "config2:0-15"),
622 NULL
623 };
624
625 static const struct attribute_group hisi_l3c_pmu_v3_format_group = {
626 .name = "format",
627 .attrs = hisi_l3c_pmu_v3_format_attr,
628 };
629
630 static struct attribute *hisi_l3c_pmu_v1_events_attr[] = {
631 HISI_PMU_EVENT_ATTR(rd_cpipe, 0x00),
632 HISI_PMU_EVENT_ATTR(wr_cpipe, 0x01),
633 HISI_PMU_EVENT_ATTR(rd_hit_cpipe, 0x02),
634 HISI_PMU_EVENT_ATTR(wr_hit_cpipe, 0x03),
635 HISI_PMU_EVENT_ATTR(victim_num, 0x04),
636 HISI_PMU_EVENT_ATTR(rd_spipe, 0x20),
637 HISI_PMU_EVENT_ATTR(wr_spipe, 0x21),
638 HISI_PMU_EVENT_ATTR(rd_hit_spipe, 0x22),
639 HISI_PMU_EVENT_ATTR(wr_hit_spipe, 0x23),
640 HISI_PMU_EVENT_ATTR(back_invalid, 0x29),
641 HISI_PMU_EVENT_ATTR(retry_cpu, 0x40),
642 HISI_PMU_EVENT_ATTR(retry_ring, 0x41),
643 HISI_PMU_EVENT_ATTR(prefetch_drop, 0x42),
644 NULL,
645 };
646
647 static const struct attribute_group hisi_l3c_pmu_v1_events_group = {
648 .name = "events",
649 .attrs = hisi_l3c_pmu_v1_events_attr,
650 };
651
652 static struct attribute *hisi_l3c_pmu_v2_events_attr[] = {
653 HISI_PMU_EVENT_ATTR(l3c_hit, 0x48),
654 HISI_PMU_EVENT_ATTR(cycles, 0x7f),
655 HISI_PMU_EVENT_ATTR(l3c_ref, 0xb8),
656 HISI_PMU_EVENT_ATTR(dat_access, 0xb9),
657 NULL
658 };
659
660 static const struct attribute_group hisi_l3c_pmu_v2_events_group = {
661 .name = "events",
662 .attrs = hisi_l3c_pmu_v2_events_attr,
663 };
664
665 static struct attribute *hisi_l3c_pmu_v3_events_attr[] = {
666 HISI_PMU_EVENT_ATTR(rd_spipe, 0x18),
667 HISI_PMU_EVENT_ATTR(rd_hit_spipe, 0x19),
668 HISI_PMU_EVENT_ATTR(wr_spipe, 0x1a),
669 HISI_PMU_EVENT_ATTR(wr_hit_spipe, 0x1b),
670 HISI_PMU_EVENT_ATTR(io_rd_spipe, 0x1c),
671 HISI_PMU_EVENT_ATTR(io_rd_hit_spipe, 0x1d),
672 HISI_PMU_EVENT_ATTR(io_wr_spipe, 0x1e),
673 HISI_PMU_EVENT_ATTR(io_wr_hit_spipe, 0x1f),
674 HISI_PMU_EVENT_ATTR(cycles, 0x7f),
675 HISI_PMU_EVENT_ATTR(l3c_ref, 0xbc),
676 HISI_PMU_EVENT_ATTR(l3c2ring, 0xbd),
677 NULL
678 };
679
680 static const struct attribute_group hisi_l3c_pmu_v3_events_group = {
681 .name = "events",
682 .attrs = hisi_l3c_pmu_v3_events_attr,
683 };
684
685 static const struct attribute_group *hisi_l3c_pmu_v1_attr_groups[] = {
686 &hisi_l3c_pmu_v1_format_group,
687 &hisi_l3c_pmu_v1_events_group,
688 &hisi_pmu_cpumask_attr_group,
689 &hisi_pmu_identifier_group,
690 NULL,
691 };
692
693 static const struct attribute_group *hisi_l3c_pmu_v2_attr_groups[] = {
694 &hisi_l3c_pmu_v2_format_group,
695 &hisi_l3c_pmu_v2_events_group,
696 &hisi_pmu_cpumask_attr_group,
697 &hisi_pmu_identifier_group,
698 NULL
699 };
700
701 static const struct attribute_group *hisi_l3c_pmu_v3_attr_groups[] = {
702 &hisi_l3c_pmu_v3_format_group,
703 &hisi_l3c_pmu_v3_events_group,
704 &hisi_pmu_cpumask_attr_group,
705 &hisi_pmu_identifier_group,
706 NULL
707 };
708
709 static struct hisi_l3c_pmu_ext hisi_l3c_pmu_support_ext = {
710 .support_ext = true,
711 };
712
713 static struct hisi_l3c_pmu_ext hisi_l3c_pmu_not_support_ext = {
714 .support_ext = false,
715 };
716
717 static const struct hisi_pmu_dev_info hisi_l3c_pmu_v1 = {
718 .attr_groups = hisi_l3c_pmu_v1_attr_groups,
719 .counter_bits = 48,
720 .check_event = L3C_V1_NR_EVENTS,
721 .private = &hisi_l3c_pmu_not_support_ext,
722 };
723
724 static const struct hisi_pmu_dev_info hisi_l3c_pmu_v2 = {
725 .attr_groups = hisi_l3c_pmu_v2_attr_groups,
726 .counter_bits = 64,
727 .check_event = L3C_V2_NR_EVENTS,
728 .private = &hisi_l3c_pmu_not_support_ext,
729 };
730
731 static const struct hisi_pmu_dev_info hisi_l3c_pmu_v3 = {
732 .attr_groups = hisi_l3c_pmu_v3_attr_groups,
733 .counter_bits = 64,
734 .check_event = L3C_V2_NR_EVENTS,
735 .private = &hisi_l3c_pmu_support_ext,
736 };
737
738 static const struct hisi_uncore_ops hisi_uncore_l3c_ops = {
739 .write_evtype = hisi_l3c_pmu_write_evtype,
740 .get_event_idx = hisi_l3c_pmu_get_event_idx,
741 .start_counters = hisi_l3c_pmu_start_counters,
742 .stop_counters = hisi_l3c_pmu_stop_counters,
743 .enable_counter = hisi_l3c_pmu_enable_counter,
744 .disable_counter = hisi_l3c_pmu_disable_counter,
745 .enable_counter_int = hisi_l3c_pmu_enable_counter_int,
746 .disable_counter_int = hisi_l3c_pmu_disable_counter_int,
747 .write_counter = hisi_l3c_pmu_write_counter,
748 .read_counter = hisi_l3c_pmu_read_counter,
749 .get_int_status = hisi_l3c_pmu_get_int_status,
750 .clear_int_status = hisi_l3c_pmu_clear_int_status,
751 .enable_filter = hisi_l3c_pmu_enable_filter,
752 .disable_filter = hisi_l3c_pmu_disable_filter,
753 .check_filter = hisi_l3c_pmu_check_filter,
754 };
755
hisi_l3c_pmu_dev_probe(struct platform_device * pdev,struct hisi_pmu * l3c_pmu)756 static int hisi_l3c_pmu_dev_probe(struct platform_device *pdev,
757 struct hisi_pmu *l3c_pmu)
758 {
759 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
760 struct hisi_l3c_pmu_ext *l3c_pmu_dev_ext;
761 int ret;
762
763 ret = hisi_l3c_pmu_init_data(pdev, l3c_pmu);
764 if (ret)
765 return ret;
766
767 ret = hisi_uncore_pmu_init_irq(l3c_pmu, pdev);
768 if (ret)
769 return ret;
770
771 l3c_pmu->pmu_events.attr_groups = l3c_pmu->dev_info->attr_groups;
772 l3c_pmu->counter_bits = l3c_pmu->dev_info->counter_bits;
773 l3c_pmu->check_event = l3c_pmu->dev_info->check_event;
774 l3c_pmu->num_counters = L3C_NR_COUNTERS;
775 l3c_pmu->ops = &hisi_uncore_l3c_ops;
776 l3c_pmu->dev = &pdev->dev;
777 l3c_pmu->on_cpu = -1;
778
779 l3c_pmu_dev_ext = l3c_pmu->dev_info->private;
780 if (l3c_pmu_dev_ext->support_ext) {
781 ret = hisi_l3c_pmu_init_ext(l3c_pmu, pdev);
782 if (ret)
783 return ret;
784 /*
785 * The extension events have their own counters with the
786 * same number of the normal events counters. So we can
787 * have at maximum num_counters * ext events monitored.
788 */
789 l3c_pmu->num_counters += hisi_l3c_pmu->ext_num * L3C_NR_COUNTERS;
790 }
791
792 return 0;
793 }
794
hisi_l3c_pmu_probe(struct platform_device * pdev)795 static int hisi_l3c_pmu_probe(struct platform_device *pdev)
796 {
797 struct hisi_l3c_pmu *hisi_l3c_pmu;
798 struct hisi_pmu *l3c_pmu;
799 char *name;
800 int ret;
801
802 hisi_l3c_pmu = devm_kzalloc(&pdev->dev, sizeof(*hisi_l3c_pmu), GFP_KERNEL);
803 if (!hisi_l3c_pmu)
804 return -ENOMEM;
805
806 l3c_pmu = &hisi_l3c_pmu->l3c_pmu;
807 platform_set_drvdata(pdev, l3c_pmu);
808
809 ret = hisi_l3c_pmu_dev_probe(pdev, l3c_pmu);
810 if (ret)
811 return ret;
812
813 if (l3c_pmu->topo.sub_id >= 0)
814 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%d_l3c%d_%d",
815 l3c_pmu->topo.sccl_id, l3c_pmu->topo.ccl_id,
816 l3c_pmu->topo.sub_id);
817 else
818 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%d_l3c%d",
819 l3c_pmu->topo.sccl_id, l3c_pmu->topo.ccl_id);
820 if (!name)
821 return -ENOMEM;
822
823 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
824 &l3c_pmu->node);
825 if (ret) {
826 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
827 return ret;
828 }
829
830 hisi_pmu_init(l3c_pmu, THIS_MODULE);
831
832 ret = perf_pmu_register(&l3c_pmu->pmu, name, -1);
833 if (ret) {
834 dev_err(l3c_pmu->dev, "L3C PMU register failed!\n");
835 cpuhp_state_remove_instance_nocalls(
836 CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, &l3c_pmu->node);
837 }
838
839 return ret;
840 }
841
hisi_l3c_pmu_remove(struct platform_device * pdev)842 static void hisi_l3c_pmu_remove(struct platform_device *pdev)
843 {
844 struct hisi_pmu *l3c_pmu = platform_get_drvdata(pdev);
845
846 perf_pmu_unregister(&l3c_pmu->pmu);
847 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
848 &l3c_pmu->node);
849 }
850
851 static const struct acpi_device_id hisi_l3c_pmu_acpi_match[] = {
852 { "HISI0213", (kernel_ulong_t)&hisi_l3c_pmu_v1 },
853 { "HISI0214", (kernel_ulong_t)&hisi_l3c_pmu_v2 },
854 { "HISI0215", (kernel_ulong_t)&hisi_l3c_pmu_v3 },
855 {}
856 };
857 MODULE_DEVICE_TABLE(acpi, hisi_l3c_pmu_acpi_match);
858
859 static struct platform_driver hisi_l3c_pmu_driver = {
860 .driver = {
861 .name = "hisi_l3c_pmu",
862 .acpi_match_table = ACPI_PTR(hisi_l3c_pmu_acpi_match),
863 .suppress_bind_attrs = true,
864 },
865 .probe = hisi_l3c_pmu_probe,
866 .remove = hisi_l3c_pmu_remove,
867 };
868
hisi_l3c_pmu_online_cpu(unsigned int cpu,struct hlist_node * node)869 static int hisi_l3c_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
870 {
871 struct hisi_pmu *l3c_pmu = hlist_entry_safe(node, struct hisi_pmu, node);
872 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
873 int ret, i;
874
875 ret = hisi_uncore_pmu_online_cpu(cpu, node);
876 if (ret)
877 return ret;
878
879 /* Avoid L3C pmu not supporting ext from ext irq migrating. */
880 if (!support_ext(hisi_l3c_pmu))
881 return 0;
882
883 for (i = 0; i < hisi_l3c_pmu->ext_num; i++)
884 WARN_ON(irq_set_affinity(hisi_l3c_pmu->ext_irq[i],
885 cpumask_of(l3c_pmu->on_cpu)));
886
887 return 0;
888 }
889
hisi_l3c_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)890 static int hisi_l3c_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
891 {
892 struct hisi_pmu *l3c_pmu = hlist_entry_safe(node, struct hisi_pmu, node);
893 struct hisi_l3c_pmu *hisi_l3c_pmu = to_hisi_l3c_pmu(l3c_pmu);
894 int ret, i;
895
896 ret = hisi_uncore_pmu_offline_cpu(cpu, node);
897 if (ret)
898 return ret;
899
900 /* If failed to find any available CPU, skip irq migration. */
901 if (l3c_pmu->on_cpu < 0)
902 return 0;
903
904 /* Avoid L3C pmu not supporting ext from ext irq migrating. */
905 if (!support_ext(hisi_l3c_pmu))
906 return 0;
907
908 for (i = 0; i < hisi_l3c_pmu->ext_num; i++)
909 WARN_ON(irq_set_affinity(hisi_l3c_pmu->ext_irq[i],
910 cpumask_of(l3c_pmu->on_cpu)));
911
912 return 0;
913 }
914
hisi_l3c_pmu_module_init(void)915 static int __init hisi_l3c_pmu_module_init(void)
916 {
917 int ret;
918
919 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
920 "AP_PERF_ARM_HISI_L3_ONLINE",
921 hisi_l3c_pmu_online_cpu,
922 hisi_l3c_pmu_offline_cpu);
923 if (ret) {
924 pr_err("L3C PMU: Error setup hotplug, ret = %d\n", ret);
925 return ret;
926 }
927
928 ret = platform_driver_register(&hisi_l3c_pmu_driver);
929 if (ret)
930 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE);
931
932 return ret;
933 }
934 module_init(hisi_l3c_pmu_module_init);
935
hisi_l3c_pmu_module_exit(void)936 static void __exit hisi_l3c_pmu_module_exit(void)
937 {
938 platform_driver_unregister(&hisi_l3c_pmu_driver);
939 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE);
940 }
941 module_exit(hisi_l3c_pmu_module_exit);
942
943 MODULE_IMPORT_NS("HISI_PMU");
944 MODULE_DESCRIPTION("HiSilicon SoC L3C uncore PMU driver");
945 MODULE_LICENSE("GPL v2");
946 MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");
947 MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
948