1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Alibaba DDR Sub-System Driveway PMU driver
4 *
5 * Copyright (C) 2022 Alibaba Inc
6 */
7
8 #define ALI_DRW_PMUNAME "ali_drw"
9 #define ALI_DRW_DRVNAME ALI_DRW_PMUNAME "_pmu"
10 #define pr_fmt(fmt) ALI_DRW_DRVNAME ": " fmt
11
12 #include <linux/acpi.h>
13 #include <linux/bitfield.h>
14 #include <linux/bitmap.h>
15 #include <linux/bitops.h>
16 #include <linux/cpuhotplug.h>
17 #include <linux/cpumask.h>
18 #include <linux/device.h>
19 #include <linux/errno.h>
20 #include <linux/interrupt.h>
21 #include <linux/irq.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/mutex.h>
26 #include <linux/perf_event.h>
27 #include <linux/platform_device.h>
28 #include <linux/printk.h>
29 #include <linux/rculist.h>
30 #include <linux/refcount.h>
31
32
33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS 16
34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE 19
35
36 #define ALI_DRW_PMU_PA_SHIFT 12
37 #define ALI_DRW_PMU_CNT_INIT 0x00000000
38 #define ALI_DRW_CNT_MAX_PERIOD 0xffffffff
39 #define ALI_DRW_PMU_CYCLE_EVT_ID 0x80
40
41 #define ALI_DRW_PMU_CNT_CTRL 0xC00
42 #define ALI_DRW_PMU_CNT_RST BIT(2)
43 #define ALI_DRW_PMU_CNT_STOP BIT(1)
44 #define ALI_DRW_PMU_CNT_START BIT(0)
45
46 #define ALI_DRW_PMU_CNT_STATE 0xC04
47 #define ALI_DRW_PMU_TEST_CTRL 0xC08
48 #define ALI_DRW_PMU_CNT_PRELOAD 0xC0C
49
50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK GENMASK(23, 0)
51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK GENMASK(31, 0)
52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH 0xC10
53 #define ALI_DRW_PMU_CYCLE_CNT_LOW 0xC14
54
55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */
56 #define ALI_DRW_PMU_EVENT_SEL0 0xC68
57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/
58 #define ALI_DRW_PMU_EVENT_SELn(n) \
59 (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)
60 #define ALI_DRW_PMCOM_CNT_EN BIT(7)
61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK GENMASK(5, 0)
62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \
63 (8 * (n % 4))
64
65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */
66 #define ALI_DRW_PMU_COMMON_COUNTER0 0xC78
67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \
68 (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))
69
70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL 0xCB8
71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL 0xCBC
72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS 0xCC0
73 #define ALI_DRW_PMU_OV_INTR_CLR 0xCC4
74 #define ALI_DRW_PMU_OV_INTR_STATUS 0xCC8
75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK GENMASK(23, 8)
76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK GENMASK(7, 0)
77 #define ALI_DRW_PMU_OV_INTR_MASK GENMASK_ULL(63, 0)
78
79 static int ali_drw_cpuhp_state_num;
80
81 static LIST_HEAD(ali_drw_pmu_irqs);
82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);
83
84 struct ali_drw_pmu_irq {
85 struct hlist_node node;
86 struct list_head irqs_node;
87 struct list_head pmus_node;
88 int irq_num;
89 int cpu;
90 refcount_t refcount;
91 };
92
93 struct ali_drw_pmu {
94 void __iomem *cfg_base;
95 struct device *dev;
96
97 struct list_head pmus_node;
98 struct ali_drw_pmu_irq *irq;
99 int irq_num;
100 int cpu;
101 DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);
102 struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
103 int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
104
105 struct pmu pmu;
106 };
107
108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))
109
110 #define DRW_CONFIG_EVENTID GENMASK(7, 0)
111 #define GET_DRW_EVENTID(event) FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)
112
ali_drw_pmu_format_show(struct device * dev,struct device_attribute * attr,char * buf)113 static ssize_t ali_drw_pmu_format_show(struct device *dev,
114 struct device_attribute *attr, char *buf)
115 {
116 struct dev_ext_attribute *eattr;
117
118 eattr = container_of(attr, struct dev_ext_attribute, attr);
119
120 return sprintf(buf, "%s\n", (char *)eattr->var);
121 }
122
123 /*
124 * PMU event attributes
125 */
ali_drw_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)126 static ssize_t ali_drw_pmu_event_show(struct device *dev,
127 struct device_attribute *attr, char *page)
128 {
129 struct dev_ext_attribute *eattr;
130
131 eattr = container_of(attr, struct dev_ext_attribute, attr);
132
133 return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
134 }
135
136 #define ALI_DRW_PMU_ATTR(_name, _func, _config) \
137 (&((struct dev_ext_attribute[]) { \
138 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \
139 })[0].attr.attr)
140
141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \
142 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)
143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config) \
144 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)
145
146 static struct attribute *ali_drw_pmu_events_attrs[] = {
147 ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr, 0x0),
148 ALI_DRW_PMU_EVENT_ATTR(hif_wr, 0x1),
149 ALI_DRW_PMU_EVENT_ATTR(hif_rd, 0x2),
150 ALI_DRW_PMU_EVENT_ATTR(hif_rmw, 0x3),
151 ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd, 0x4),
152 ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles, 0x7),
153 ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles, 0x8),
154 ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical, 0x9),
155 ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical, 0xA),
156 ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical, 0xB),
157 ALI_DRW_PMU_EVENT_ATTR(op_is_activate, 0xC),
158 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr, 0xD),
159 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate, 0xE),
160 ALI_DRW_PMU_EVENT_ATTR(op_is_rd, 0xF),
161 ALI_DRW_PMU_EVENT_ATTR(op_is_wr, 0x10),
162 ALI_DRW_PMU_EVENT_ATTR(op_is_mwr, 0x11),
163 ALI_DRW_PMU_EVENT_ATTR(op_is_precharge, 0x12),
164 ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr, 0x13),
165 ALI_DRW_PMU_EVENT_ATTR(precharge_for_other, 0x14),
166 ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions, 0x15),
167 ALI_DRW_PMU_EVENT_ATTR(write_combine, 0x16),
168 ALI_DRW_PMU_EVENT_ATTR(war_hazard, 0x17),
169 ALI_DRW_PMU_EVENT_ATTR(raw_hazard, 0x18),
170 ALI_DRW_PMU_EVENT_ATTR(waw_hazard, 0x19),
171 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0, 0x1A),
172 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1, 0x1B),
173 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2, 0x1C),
174 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3, 0x1D),
175 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0, 0x1E),
176 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1, 0x1F),
177 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2, 0x20),
178 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3, 0x21),
179 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0, 0x26),
180 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1, 0x27),
181 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2, 0x28),
182 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3, 0x29),
183 ALI_DRW_PMU_EVENT_ATTR(op_is_refresh, 0x2A),
184 ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref, 0x2B),
185 ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode, 0x2D),
186 ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl, 0x2E),
187 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),
188 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),
189 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc, 0x34),
190 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr, 0x35),
191 ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr, 0x36),
192 ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart, 0x37),
193 ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch, 0x38),
194 ALI_DRW_PMU_EVENT_ATTR(chi_txreq, 0x39),
195 ALI_DRW_PMU_EVENT_ATTR(chi_txdat, 0x3A),
196 ALI_DRW_PMU_EVENT_ATTR(chi_rxdat, 0x3B),
197 ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp, 0x3C),
198 ALI_DRW_PMU_EVENT_ATTR(tsz_vio, 0x3D),
199 ALI_DRW_PMU_EVENT_ATTR(cycle, 0x80),
200 NULL,
201 };
202
203 static struct attribute_group ali_drw_pmu_events_attr_group = {
204 .name = "events",
205 .attrs = ali_drw_pmu_events_attrs,
206 };
207
208 static struct attribute *ali_drw_pmu_format_attr[] = {
209 ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),
210 NULL,
211 };
212
213 static const struct attribute_group ali_drw_pmu_format_group = {
214 .name = "format",
215 .attrs = ali_drw_pmu_format_attr,
216 };
217
ali_drw_pmu_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
219 struct device_attribute *attr,
220 char *buf)
221 {
222 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
223
224 return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
225 }
226
227 static struct device_attribute ali_drw_pmu_cpumask_attr =
228 __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);
229
230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = {
231 &ali_drw_pmu_cpumask_attr.attr,
232 NULL,
233 };
234
235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {
236 .attrs = ali_drw_pmu_cpumask_attrs,
237 };
238
ali_drw_pmu_identifier_attr_visible(struct kobject * kobj,struct attribute * attr,int n)239 static umode_t ali_drw_pmu_identifier_attr_visible(struct kobject *kobj,
240 struct attribute *attr, int n)
241 {
242 return attr->mode;
243 }
244
245 static DEVICE_STRING_ATTR_RO(ali_drw_pmu_identifier, 0444, "ali_drw_pmu");
246
247 static struct attribute *ali_drw_pmu_identifier_attrs[] = {
248 &dev_attr_ali_drw_pmu_identifier.attr.attr,
249 NULL
250 };
251
252 static const struct attribute_group ali_drw_pmu_identifier_attr_group = {
253 .attrs = ali_drw_pmu_identifier_attrs,
254 .is_visible = ali_drw_pmu_identifier_attr_visible
255 };
256
257 static const struct attribute_group *ali_drw_pmu_attr_groups[] = {
258 &ali_drw_pmu_events_attr_group,
259 &ali_drw_pmu_cpumask_attr_group,
260 &ali_drw_pmu_format_group,
261 &ali_drw_pmu_identifier_attr_group,
262 NULL,
263 };
264
265 /* find a counter for event, then in add func, hw.idx will equal to counter */
ali_drw_get_counter_idx(struct perf_event * event)266 static int ali_drw_get_counter_idx(struct perf_event *event)
267 {
268 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
269 int idx;
270
271 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {
272 if (!test_and_set_bit(idx, drw_pmu->used_mask))
273 return idx;
274 }
275
276 /* The counters are all in use. */
277 return -EBUSY;
278 }
279
ali_drw_pmu_read_counter(struct perf_event * event)280 static u64 ali_drw_pmu_read_counter(struct perf_event *event)
281 {
282 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
283 u64 cycle_high, cycle_low;
284
285 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
286 cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);
287 cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;
288 cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);
289 cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;
290 return (cycle_high << 32 | cycle_low);
291 }
292
293 return readl(drw_pmu->cfg_base +
294 ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));
295 }
296
ali_drw_pmu_event_update(struct perf_event * event)297 static void ali_drw_pmu_event_update(struct perf_event *event)
298 {
299 struct hw_perf_event *hwc = &event->hw;
300 u64 delta, prev, now;
301
302 do {
303 prev = local64_read(&hwc->prev_count);
304 now = ali_drw_pmu_read_counter(event);
305 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
306
307 /* handle overflow. */
308 delta = now - prev;
309 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)
310 delta &= ALI_DRW_PMU_OV_INTR_MASK;
311 else
312 delta &= ALI_DRW_CNT_MAX_PERIOD;
313 local64_add(delta, &event->count);
314 }
315
ali_drw_pmu_event_set_period(struct perf_event * event)316 static void ali_drw_pmu_event_set_period(struct perf_event *event)
317 {
318 u64 pre_val;
319 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
320
321 /* set a preload counter for test purpose */
322 writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,
323 drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
324
325 /* set conunter initial value */
326 pre_val = ALI_DRW_PMU_CNT_INIT;
327 writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
328 local64_set(&event->hw.prev_count, pre_val);
329
330 /* set sel mode to zero to start test */
331 writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
332 }
333
ali_drw_pmu_enable_counter(struct perf_event * event)334 static void ali_drw_pmu_enable_counter(struct perf_event *event)
335 {
336 u32 val, subval, reg, shift;
337 int counter = event->hw.idx;
338 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
339
340 reg = ALI_DRW_PMU_EVENT_SELn(counter);
341 val = readl(drw_pmu->cfg_base + reg);
342 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |
343 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);
344
345 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
346 val &= ~(GENMASK(7, 0) << shift);
347 val |= subval << shift;
348
349 writel(val, drw_pmu->cfg_base + reg);
350 }
351
ali_drw_pmu_disable_counter(struct perf_event * event)352 static void ali_drw_pmu_disable_counter(struct perf_event *event)
353 {
354 u32 val, reg, subval, shift;
355 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
356 int counter = event->hw.idx;
357
358 reg = ALI_DRW_PMU_EVENT_SELn(counter);
359 val = readl(drw_pmu->cfg_base + reg);
360 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |
361 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);
362
363 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
364 val &= ~(GENMASK(7, 0) << shift);
365 val |= subval << shift;
366
367 writel(val, drw_pmu->cfg_base + reg);
368 }
369
ali_drw_pmu_isr(int irq_num,void * data)370 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)
371 {
372 struct ali_drw_pmu_irq *irq = data;
373 struct ali_drw_pmu *drw_pmu;
374 irqreturn_t ret = IRQ_NONE;
375
376 rcu_read_lock();
377 list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {
378 unsigned long status, clr_status;
379 struct perf_event *event;
380 unsigned int idx;
381
382 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
383 event = drw_pmu->events[idx];
384 if (!event)
385 continue;
386 ali_drw_pmu_disable_counter(event);
387 }
388
389 /* common counter intr status */
390 status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);
391 status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
392 if (status) {
393 for_each_set_bit(idx, &status,
394 ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
395 event = drw_pmu->events[idx];
396 if (WARN_ON_ONCE(!event))
397 continue;
398 ali_drw_pmu_event_update(event);
399 ali_drw_pmu_event_set_period(event);
400 }
401
402 /* clear common counter intr status */
403 clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
404 writel(clr_status,
405 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
406 }
407
408 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
409 event = drw_pmu->events[idx];
410 if (!event)
411 continue;
412 if (!(event->hw.state & PERF_HES_STOPPED))
413 ali_drw_pmu_enable_counter(event);
414 }
415 if (status)
416 ret = IRQ_HANDLED;
417 }
418 rcu_read_unlock();
419 return ret;
420 }
421
__ali_drw_pmu_init_irq(struct platform_device * pdev,int irq_num)422 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device
423 *pdev, int irq_num)
424 {
425 int ret;
426 struct ali_drw_pmu_irq *irq;
427
428 list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {
429 if (irq->irq_num == irq_num
430 && refcount_inc_not_zero(&irq->refcount))
431 return irq;
432 }
433
434 irq = kzalloc(sizeof(*irq), GFP_KERNEL);
435 if (!irq)
436 return ERR_PTR(-ENOMEM);
437
438 INIT_LIST_HEAD(&irq->pmus_node);
439
440 /* Pick one CPU to be the preferred one to use */
441 irq->cpu = smp_processor_id();
442 refcount_set(&irq->refcount, 1);
443
444 /*
445 * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same
446 * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers
447 * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not
448 * share with other component.
449 */
450 ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,
451 IRQF_SHARED, dev_name(&pdev->dev), irq);
452 if (ret < 0) {
453 dev_err(&pdev->dev,
454 "Fail to request IRQ:%d ret:%d\n", irq_num, ret);
455 goto out_free;
456 }
457
458 ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
459 if (ret)
460 goto out_free;
461
462 ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,
463 &irq->node);
464 if (ret)
465 goto out_free;
466
467 irq->irq_num = irq_num;
468 list_add(&irq->irqs_node, &ali_drw_pmu_irqs);
469
470 return irq;
471
472 out_free:
473 kfree(irq);
474 return ERR_PTR(ret);
475 }
476
ali_drw_pmu_init_irq(struct ali_drw_pmu * drw_pmu,struct platform_device * pdev)477 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,
478 struct platform_device *pdev)
479 {
480 int irq_num;
481 struct ali_drw_pmu_irq *irq;
482
483 /* Read and init IRQ */
484 irq_num = platform_get_irq(pdev, 0);
485 if (irq_num < 0)
486 return irq_num;
487
488 mutex_lock(&ali_drw_pmu_irqs_lock);
489 irq = __ali_drw_pmu_init_irq(pdev, irq_num);
490 mutex_unlock(&ali_drw_pmu_irqs_lock);
491
492 if (IS_ERR(irq))
493 return PTR_ERR(irq);
494
495 drw_pmu->irq = irq;
496
497 mutex_lock(&ali_drw_pmu_irqs_lock);
498 list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);
499 mutex_unlock(&ali_drw_pmu_irqs_lock);
500
501 return 0;
502 }
503
ali_drw_pmu_uninit_irq(struct ali_drw_pmu * drw_pmu)504 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)
505 {
506 struct ali_drw_pmu_irq *irq = drw_pmu->irq;
507
508 mutex_lock(&ali_drw_pmu_irqs_lock);
509 list_del_rcu(&drw_pmu->pmus_node);
510
511 if (!refcount_dec_and_test(&irq->refcount)) {
512 mutex_unlock(&ali_drw_pmu_irqs_lock);
513 return;
514 }
515
516 list_del(&irq->irqs_node);
517 mutex_unlock(&ali_drw_pmu_irqs_lock);
518
519 WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
520 cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,
521 &irq->node);
522 kfree(irq);
523 }
524
ali_drw_pmu_event_init(struct perf_event * event)525 static int ali_drw_pmu_event_init(struct perf_event *event)
526 {
527 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
528 struct hw_perf_event *hwc = &event->hw;
529 struct perf_event *sibling;
530 struct device *dev = drw_pmu->pmu.dev;
531
532 if (event->attr.type != event->pmu->type)
533 return -ENOENT;
534
535 if (is_sampling_event(event)) {
536 dev_err(dev, "Sampling not supported!\n");
537 return -EOPNOTSUPP;
538 }
539
540 if (event->attach_state & PERF_ATTACH_TASK) {
541 dev_err(dev, "Per-task counter cannot allocate!\n");
542 return -EOPNOTSUPP;
543 }
544
545 event->cpu = drw_pmu->cpu;
546 if (event->cpu < 0) {
547 dev_err(dev, "Per-task mode not supported!\n");
548 return -EOPNOTSUPP;
549 }
550
551 if (event->group_leader != event &&
552 !is_software_event(event->group_leader)) {
553 dev_err(dev, "driveway only allow one event!\n");
554 return -EINVAL;
555 }
556
557 for_each_sibling_event(sibling, event->group_leader) {
558 if (sibling != event && !is_software_event(sibling)) {
559 dev_err(dev, "driveway event not allowed!\n");
560 return -EINVAL;
561 }
562 }
563
564 /* reset all the pmu counters */
565 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
566
567 hwc->idx = -1;
568
569 return 0;
570 }
571
ali_drw_pmu_start(struct perf_event * event,int flags)572 static void ali_drw_pmu_start(struct perf_event *event, int flags)
573 {
574 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
575
576 event->hw.state = 0;
577
578 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
579 writel(ALI_DRW_PMU_CNT_START,
580 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
581 return;
582 }
583
584 ali_drw_pmu_event_set_period(event);
585 if (flags & PERF_EF_RELOAD) {
586 unsigned long prev_raw_count =
587 local64_read(&event->hw.prev_count);
588 writel(prev_raw_count,
589 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
590 }
591
592 ali_drw_pmu_enable_counter(event);
593
594 writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
595 }
596
ali_drw_pmu_stop(struct perf_event * event,int flags)597 static void ali_drw_pmu_stop(struct perf_event *event, int flags)
598 {
599 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
600
601 if (event->hw.state & PERF_HES_STOPPED)
602 return;
603
604 if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)
605 ali_drw_pmu_disable_counter(event);
606
607 writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
608
609 ali_drw_pmu_event_update(event);
610 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
611 }
612
ali_drw_pmu_add(struct perf_event * event,int flags)613 static int ali_drw_pmu_add(struct perf_event *event, int flags)
614 {
615 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
616 struct hw_perf_event *hwc = &event->hw;
617 int idx = -1;
618 int evtid;
619
620 evtid = GET_DRW_EVENTID(event);
621
622 if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {
623 idx = ali_drw_get_counter_idx(event);
624 if (idx < 0)
625 return idx;
626 drw_pmu->events[idx] = event;
627 drw_pmu->evtids[idx] = evtid;
628 }
629 hwc->idx = idx;
630
631 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
632
633 if (flags & PERF_EF_START)
634 ali_drw_pmu_start(event, PERF_EF_RELOAD);
635
636 /* Propagate our changes to the userspace mapping. */
637 perf_event_update_userpage(event);
638
639 return 0;
640 }
641
ali_drw_pmu_del(struct perf_event * event,int flags)642 static void ali_drw_pmu_del(struct perf_event *event, int flags)
643 {
644 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
645 struct hw_perf_event *hwc = &event->hw;
646 int idx = hwc->idx;
647
648 ali_drw_pmu_stop(event, PERF_EF_UPDATE);
649
650 if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
651 drw_pmu->events[idx] = NULL;
652 drw_pmu->evtids[idx] = 0;
653 clear_bit(idx, drw_pmu->used_mask);
654 }
655
656 perf_event_update_userpage(event);
657 }
658
ali_drw_pmu_read(struct perf_event * event)659 static void ali_drw_pmu_read(struct perf_event *event)
660 {
661 ali_drw_pmu_event_update(event);
662 }
663
ali_drw_pmu_probe(struct platform_device * pdev)664 static int ali_drw_pmu_probe(struct platform_device *pdev)
665 {
666 struct ali_drw_pmu *drw_pmu;
667 struct resource *res;
668 char *name;
669 int ret;
670
671 drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);
672 if (!drw_pmu)
673 return -ENOMEM;
674
675 drw_pmu->dev = &pdev->dev;
676 platform_set_drvdata(pdev, drw_pmu);
677
678 drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
679 if (IS_ERR(drw_pmu->cfg_base))
680 return PTR_ERR(drw_pmu->cfg_base);
681
682 name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
683 (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
684 if (!name)
685 return -ENOMEM;
686
687 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
688
689 /* enable the generation of interrupt by all common counters */
690 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
691 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);
692
693 /* clearing interrupt status */
694 writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
695
696 drw_pmu->cpu = smp_processor_id();
697
698 ret = ali_drw_pmu_init_irq(drw_pmu, pdev);
699 if (ret)
700 return ret;
701
702 drw_pmu->pmu = (struct pmu) {
703 .module = THIS_MODULE,
704 .parent = &pdev->dev,
705 .task_ctx_nr = perf_invalid_context,
706 .event_init = ali_drw_pmu_event_init,
707 .add = ali_drw_pmu_add,
708 .del = ali_drw_pmu_del,
709 .start = ali_drw_pmu_start,
710 .stop = ali_drw_pmu_stop,
711 .read = ali_drw_pmu_read,
712 .attr_groups = ali_drw_pmu_attr_groups,
713 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
714 };
715
716 ret = perf_pmu_register(&drw_pmu->pmu, name, -1);
717 if (ret) {
718 dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");
719 ali_drw_pmu_uninit_irq(drw_pmu);
720 }
721
722 return ret;
723 }
724
ali_drw_pmu_remove(struct platform_device * pdev)725 static void ali_drw_pmu_remove(struct platform_device *pdev)
726 {
727 struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);
728
729 /* disable the generation of interrupt by all common counters */
730 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
731 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);
732
733 ali_drw_pmu_uninit_irq(drw_pmu);
734 perf_pmu_unregister(&drw_pmu->pmu);
735 }
736
ali_drw_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)737 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
738 {
739 struct ali_drw_pmu_irq *irq;
740 struct ali_drw_pmu *drw_pmu;
741 unsigned int target;
742
743 irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);
744 if (cpu != irq->cpu)
745 return 0;
746
747 target = cpumask_any_and_but(cpumask_of_node(cpu_to_node(cpu)),
748 cpu_online_mask, cpu);
749 if (target >= nr_cpu_ids)
750 target = cpumask_any_but(cpu_online_mask, cpu);
751
752 if (target >= nr_cpu_ids)
753 return 0;
754
755 /* We're only reading, but this isn't the place to be involving RCU */
756 mutex_lock(&ali_drw_pmu_irqs_lock);
757 list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)
758 perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);
759 mutex_unlock(&ali_drw_pmu_irqs_lock);
760
761 WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
762 irq->cpu = target;
763
764 return 0;
765 }
766
767 /*
768 * Due to historical reasons, the HID used in the production environment is
769 * ARMHD700, so we leave ARMHD700 as Compatible ID.
770 */
771 static const struct acpi_device_id ali_drw_acpi_match[] = {
772 {"BABA5000", 0},
773 {"ARMHD700", 0},
774 {}
775 };
776
777 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);
778
779 static struct platform_driver ali_drw_pmu_driver = {
780 .driver = {
781 .name = "ali_drw_pmu",
782 .acpi_match_table = ali_drw_acpi_match,
783 },
784 .probe = ali_drw_pmu_probe,
785 .remove_new = ali_drw_pmu_remove,
786 };
787
ali_drw_pmu_init(void)788 static int __init ali_drw_pmu_init(void)
789 {
790 int ret;
791
792 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
793 "ali_drw_pmu:online",
794 NULL, ali_drw_pmu_offline_cpu);
795
796 if (ret < 0) {
797 pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",
798 ret);
799 return ret;
800 }
801 ali_drw_cpuhp_state_num = ret;
802
803 ret = platform_driver_register(&ali_drw_pmu_driver);
804 if (ret)
805 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
806
807 return ret;
808 }
809
ali_drw_pmu_exit(void)810 static void __exit ali_drw_pmu_exit(void)
811 {
812 platform_driver_unregister(&ali_drw_pmu_driver);
813 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
814 }
815
816 module_init(ali_drw_pmu_init);
817 module_exit(ali_drw_pmu_exit);
818
819 MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>");
820 MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>");
821 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
822 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");
823 MODULE_LICENSE("GPL v2");
824