1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2022-2024 Arm Limited
3 // NI-700 Network-on-Chip PMU driver
4
5 #include <linux/acpi.h>
6 #include <linux/bitfield.h>
7 #include <linux/interrupt.h>
8 #include <linux/io.h>
9 #include <linux/io-64-nonatomic-lo-hi.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/perf_event.h>
14 #include <linux/platform_device.h>
15 #include <linux/slab.h>
16
17 /* Common registers */
18 #define NI_NODE_TYPE 0x000
19 #define NI_NODE_TYPE_NODE_ID GENMASK(31, 16)
20 #define NI_NODE_TYPE_NODE_TYPE GENMASK(15, 0)
21
22 #define NI_CHILD_NODE_INFO 0x004
23 #define NI_CHILD_PTR(n) (0x008 + (n) * 4)
24
25 #define NI700_PMUSELA 0x00c
26
27 /* Config node */
28 #define NI_PERIPHERAL_ID0 0xfe0
29 #define NI_PIDR0_PART_7_0 GENMASK(7, 0)
30 #define NI_PERIPHERAL_ID1 0xfe4
31 #define NI_PIDR1_PART_11_8 GENMASK(3, 0)
32 #define NI_PERIPHERAL_ID2 0xfe8
33 #define NI_PIDR2_VERSION GENMASK(7, 4)
34
35 /* PMU node */
36 #define NI_PMEVCNTR(n) (0x008 + (n) * 8)
37 #define NI_PMCCNTR_L 0x0f8
38 #define NI_PMCCNTR_U 0x0fc
39 #define NI_PMEVTYPER(n) (0x400 + (n) * 4)
40 #define NI_PMEVTYPER_NODE_TYPE GENMASK(12, 9)
41 #define NI_PMEVTYPER_NODE_ID GENMASK(8, 0)
42 #define NI_PMCNTENSET 0xc00
43 #define NI_PMCNTENCLR 0xc20
44 #define NI_PMINTENSET 0xc40
45 #define NI_PMINTENCLR 0xc60
46 #define NI_PMOVSCLR 0xc80
47 #define NI_PMOVSSET 0xcc0
48 #define NI_PMCFGR 0xe00
49 #define NI_PMCR 0xe04
50 #define NI_PMCR_RESET_CCNT BIT(2)
51 #define NI_PMCR_RESET_EVCNT BIT(1)
52 #define NI_PMCR_ENABLE BIT(0)
53
54 #define NI_NUM_COUNTERS 8
55 #define NI_CCNT_IDX 31
56
57 /* Event attributes */
58 #define NI_CONFIG_TYPE GENMASK_ULL(15, 0)
59 #define NI_CONFIG_NODEID GENMASK_ULL(31, 16)
60 #define NI_CONFIG_EVENTID GENMASK_ULL(47, 32)
61
62 #define NI_EVENT_TYPE(event) FIELD_GET(NI_CONFIG_TYPE, (event)->attr.config)
63 #define NI_EVENT_NODEID(event) FIELD_GET(NI_CONFIG_NODEID, (event)->attr.config)
64 #define NI_EVENT_EVENTID(event) FIELD_GET(NI_CONFIG_EVENTID, (event)->attr.config)
65
66 enum ni_part {
67 PART_NI_700 = 0x43b,
68 PART_NI_710AE = 0x43d,
69 };
70
71 enum ni_node_type {
72 NI_GLOBAL,
73 NI_VOLTAGE,
74 NI_POWER,
75 NI_CLOCK,
76 NI_ASNI,
77 NI_AMNI,
78 NI_PMU,
79 NI_HSNI,
80 NI_HMNI,
81 NI_PMNI,
82 };
83
84 struct arm_ni_node {
85 void __iomem *base;
86 enum ni_node_type type;
87 u16 id;
88 u32 num_components;
89 };
90
91 struct arm_ni_unit {
92 void __iomem *pmusela;
93 enum ni_node_type type;
94 u16 id;
95 bool ns;
96 union {
97 __le64 pmusel;
98 u8 event[8];
99 };
100 };
101
102 struct arm_ni_cd {
103 void __iomem *pmu_base;
104 u16 id;
105 s8 irq_friend;
106 int num_units;
107 int irq;
108 struct pmu pmu;
109 struct arm_ni_unit *units;
110 struct perf_event *evcnt[NI_NUM_COUNTERS];
111 struct perf_event *ccnt;
112 };
113
114 struct arm_ni {
115 struct device *dev;
116 void __iomem *base;
117 enum ni_part part;
118 int id;
119 int cpu;
120 int num_cds;
121 struct hlist_node cpuhp_node;
122 struct arm_ni_cd cds[] __counted_by(num_cds);
123 };
124
125 #define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id])
126 #define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu)
127
128 #define ni_for_each_cd(n, c) \
129 for (struct arm_ni_cd *c = n->cds; c < n->cds + n->num_cds; c++) if (c->pmu_base)
130
131 #define cd_for_each_unit(cd, u) \
132 for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++)
133
134 static int arm_ni_hp_state;
135
136 struct arm_ni_event_attr {
137 struct device_attribute attr;
138 enum ni_node_type type;
139 };
140
141 #define NI_EVENT_ATTR(_name, _type) \
142 (&((struct arm_ni_event_attr[]) {{ \
143 .attr = __ATTR(_name, 0444, arm_ni_event_show, NULL), \
144 .type = _type, \
145 }})[0].attr.attr)
146
arm_ni_event_show(struct device * dev,struct device_attribute * attr,char * buf)147 static ssize_t arm_ni_event_show(struct device *dev,
148 struct device_attribute *attr, char *buf)
149 {
150 struct arm_ni_event_attr *eattr = container_of(attr, typeof(*eattr), attr);
151
152 if (eattr->type == NI_PMU)
153 return sysfs_emit(buf, "type=0x%x\n", eattr->type);
154
155 return sysfs_emit(buf, "type=0x%x,eventid=?,nodeid=?\n", eattr->type);
156 }
157
arm_ni_event_attr_is_visible(struct kobject * kobj,struct attribute * attr,int unused)158 static umode_t arm_ni_event_attr_is_visible(struct kobject *kobj,
159 struct attribute *attr, int unused)
160 {
161 struct device *dev = kobj_to_dev(kobj);
162 struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));
163 struct arm_ni_event_attr *eattr;
164
165 eattr = container_of(attr, typeof(*eattr), attr.attr);
166
167 cd_for_each_unit(cd, unit) {
168 if (unit->type == eattr->type && unit->ns)
169 return attr->mode;
170 }
171
172 return 0;
173 }
174
175 static struct attribute *arm_ni_event_attrs[] = {
176 NI_EVENT_ATTR(asni, NI_ASNI),
177 NI_EVENT_ATTR(amni, NI_AMNI),
178 NI_EVENT_ATTR(cycles, NI_PMU),
179 NI_EVENT_ATTR(hsni, NI_HSNI),
180 NI_EVENT_ATTR(hmni, NI_HMNI),
181 NI_EVENT_ATTR(pmni, NI_PMNI),
182 NULL
183 };
184
185 static const struct attribute_group arm_ni_event_attrs_group = {
186 .name = "events",
187 .attrs = arm_ni_event_attrs,
188 .is_visible = arm_ni_event_attr_is_visible,
189 };
190
191 struct arm_ni_format_attr {
192 struct device_attribute attr;
193 u64 field;
194 };
195
196 #define NI_FORMAT_ATTR(_name, _fld) \
197 (&((struct arm_ni_format_attr[]) {{ \
198 .attr = __ATTR(_name, 0444, arm_ni_format_show, NULL), \
199 .field = _fld, \
200 }})[0].attr.attr)
201
arm_ni_format_show(struct device * dev,struct device_attribute * attr,char * buf)202 static ssize_t arm_ni_format_show(struct device *dev,
203 struct device_attribute *attr, char *buf)
204 {
205 struct arm_ni_format_attr *fmt = container_of(attr, typeof(*fmt), attr);
206
207 return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field);
208 }
209
210 static struct attribute *arm_ni_format_attrs[] = {
211 NI_FORMAT_ATTR(type, NI_CONFIG_TYPE),
212 NI_FORMAT_ATTR(nodeid, NI_CONFIG_NODEID),
213 NI_FORMAT_ATTR(eventid, NI_CONFIG_EVENTID),
214 NULL
215 };
216
217 static const struct attribute_group arm_ni_format_attrs_group = {
218 .name = "format",
219 .attrs = arm_ni_format_attrs,
220 };
221
arm_ni_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)222 static ssize_t arm_ni_cpumask_show(struct device *dev,
223 struct device_attribute *attr, char *buf)
224 {
225 struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
226
227 return cpumap_print_to_pagebuf(true, buf, cpumask_of(ni->cpu));
228 }
229
230 static struct device_attribute arm_ni_cpumask_attr =
231 __ATTR(cpumask, 0444, arm_ni_cpumask_show, NULL);
232
arm_ni_identifier_show(struct device * dev,struct device_attribute * attr,char * buf)233 static ssize_t arm_ni_identifier_show(struct device *dev,
234 struct device_attribute *attr, char *buf)
235 {
236 struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
237 u32 reg = readl_relaxed(ni->base + NI_PERIPHERAL_ID2);
238 int version = FIELD_GET(NI_PIDR2_VERSION, reg);
239
240 return sysfs_emit(buf, "%03x%02x\n", ni->part, version);
241 }
242
243 static struct device_attribute arm_ni_identifier_attr =
244 __ATTR(identifier, 0444, arm_ni_identifier_show, NULL);
245
246 static struct attribute *arm_ni_other_attrs[] = {
247 &arm_ni_cpumask_attr.attr,
248 &arm_ni_identifier_attr.attr,
249 NULL
250 };
251
252 static const struct attribute_group arm_ni_other_attr_group = {
253 .attrs = arm_ni_other_attrs,
254 };
255
256 static const struct attribute_group *arm_ni_attr_groups[] = {
257 &arm_ni_event_attrs_group,
258 &arm_ni_format_attrs_group,
259 &arm_ni_other_attr_group,
260 NULL
261 };
262
arm_ni_pmu_enable(struct pmu * pmu)263 static void arm_ni_pmu_enable(struct pmu *pmu)
264 {
265 writel_relaxed(NI_PMCR_ENABLE, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
266 }
267
arm_ni_pmu_disable(struct pmu * pmu)268 static void arm_ni_pmu_disable(struct pmu *pmu)
269 {
270 writel_relaxed(0, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
271 }
272
273 struct arm_ni_val {
274 unsigned int evcnt;
275 unsigned int ccnt;
276 };
277
arm_ni_val_count_event(struct perf_event * evt,struct arm_ni_val * val)278 static bool arm_ni_val_count_event(struct perf_event *evt, struct arm_ni_val *val)
279 {
280 if (is_software_event(evt))
281 return true;
282
283 if (NI_EVENT_TYPE(evt) == NI_PMU) {
284 val->ccnt++;
285 return val->ccnt <= 1;
286 }
287
288 val->evcnt++;
289 return val->evcnt <= NI_NUM_COUNTERS;
290 }
291
arm_ni_validate_group(struct perf_event * event)292 static int arm_ni_validate_group(struct perf_event *event)
293 {
294 struct perf_event *sibling, *leader = event->group_leader;
295 struct arm_ni_val val = { 0 };
296
297 if (leader == event)
298 return 0;
299
300 arm_ni_val_count_event(event, &val);
301 if (!arm_ni_val_count_event(leader, &val))
302 return -EINVAL;
303
304 for_each_sibling_event(sibling, leader) {
305 if (!arm_ni_val_count_event(sibling, &val))
306 return -EINVAL;
307 }
308 return 0;
309 }
310
arm_ni_event_init(struct perf_event * event)311 static int arm_ni_event_init(struct perf_event *event)
312 {
313 struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
314
315 if (event->attr.type != event->pmu->type)
316 return -ENOENT;
317
318 if (is_sampling_event(event))
319 return -EINVAL;
320
321 event->cpu = cd_to_ni(cd)->cpu;
322 if (NI_EVENT_TYPE(event) == NI_PMU)
323 return arm_ni_validate_group(event);
324
325 cd_for_each_unit(cd, unit) {
326 if (unit->type == NI_EVENT_TYPE(event) &&
327 unit->id == NI_EVENT_NODEID(event) && unit->ns) {
328 event->hw.config_base = (unsigned long)unit;
329 return arm_ni_validate_group(event);
330 }
331 }
332 return -EINVAL;
333 }
334
arm_ni_read_ccnt(struct arm_ni_cd * cd)335 static u64 arm_ni_read_ccnt(struct arm_ni_cd *cd)
336 {
337 u64 l, u_old, u_new;
338 int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */
339
340 u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);
341 do {
342 u_old = u_new;
343 l = readl_relaxed(cd->pmu_base + NI_PMCCNTR_L);
344 u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);
345 } while (u_new != u_old && --retries);
346 WARN_ON(!retries);
347
348 return (u_new << 32) | l;
349 }
350
arm_ni_event_read(struct perf_event * event)351 static void arm_ni_event_read(struct perf_event *event)
352 {
353 struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
354 struct hw_perf_event *hw = &event->hw;
355 u64 count, prev;
356 bool ccnt = hw->idx == NI_CCNT_IDX;
357
358 do {
359 prev = local64_read(&hw->prev_count);
360 if (ccnt)
361 count = arm_ni_read_ccnt(cd);
362 else
363 count = readl_relaxed(cd->pmu_base + NI_PMEVCNTR(hw->idx));
364 } while (local64_cmpxchg(&hw->prev_count, prev, count) != prev);
365
366 count -= prev;
367 if (!ccnt)
368 count = (u32)count;
369 local64_add(count, &event->count);
370 }
371
arm_ni_event_start(struct perf_event * event,int flags)372 static void arm_ni_event_start(struct perf_event *event, int flags)
373 {
374 struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
375
376 writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET);
377 }
378
arm_ni_event_stop(struct perf_event * event,int flags)379 static void arm_ni_event_stop(struct perf_event *event, int flags)
380 {
381 struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
382
383 writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR);
384 if (flags & PERF_EF_UPDATE)
385 arm_ni_event_read(event);
386 }
387
arm_ni_init_ccnt(struct arm_ni_cd * cd)388 static void arm_ni_init_ccnt(struct arm_ni_cd *cd)
389 {
390 local64_set(&cd->ccnt->hw.prev_count, S64_MIN);
391 lo_hi_writeq_relaxed(S64_MIN, cd->pmu_base + NI_PMCCNTR_L);
392 }
393
arm_ni_init_evcnt(struct arm_ni_cd * cd,int idx)394 static void arm_ni_init_evcnt(struct arm_ni_cd *cd, int idx)
395 {
396 local64_set(&cd->evcnt[idx]->hw.prev_count, S32_MIN);
397 writel_relaxed(S32_MIN, cd->pmu_base + NI_PMEVCNTR(idx));
398 }
399
arm_ni_event_add(struct perf_event * event,int flags)400 static int arm_ni_event_add(struct perf_event *event, int flags)
401 {
402 struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
403 struct hw_perf_event *hw = &event->hw;
404 struct arm_ni_unit *unit;
405 enum ni_node_type type = NI_EVENT_TYPE(event);
406 u32 reg;
407
408 if (type == NI_PMU) {
409 if (cd->ccnt)
410 return -ENOSPC;
411 hw->idx = NI_CCNT_IDX;
412 cd->ccnt = event;
413 arm_ni_init_ccnt(cd);
414 } else {
415 hw->idx = 0;
416 while (cd->evcnt[hw->idx]) {
417 if (++hw->idx == NI_NUM_COUNTERS)
418 return -ENOSPC;
419 }
420 cd->evcnt[hw->idx] = event;
421 unit = (void *)hw->config_base;
422 unit->event[hw->idx] = NI_EVENT_EVENTID(event);
423 arm_ni_init_evcnt(cd, hw->idx);
424 lo_hi_writeq_relaxed(le64_to_cpu(unit->pmusel), unit->pmusela);
425
426 reg = FIELD_PREP(NI_PMEVTYPER_NODE_TYPE, type) |
427 FIELD_PREP(NI_PMEVTYPER_NODE_ID, NI_EVENT_NODEID(event));
428 writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx));
429 }
430 if (flags & PERF_EF_START)
431 arm_ni_event_start(event, 0);
432 return 0;
433 }
434
arm_ni_event_del(struct perf_event * event,int flags)435 static void arm_ni_event_del(struct perf_event *event, int flags)
436 {
437 struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
438 struct hw_perf_event *hw = &event->hw;
439
440 arm_ni_event_stop(event, PERF_EF_UPDATE);
441
442 if (hw->idx == NI_CCNT_IDX)
443 cd->ccnt = NULL;
444 else
445 cd->evcnt[hw->idx] = NULL;
446 }
447
arm_ni_handle_irq(int irq,void * dev_id)448 static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id)
449 {
450 struct arm_ni_cd *cd = dev_id;
451 irqreturn_t ret = IRQ_NONE;
452
453 for (;;) {
454 u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);
455
456 if (reg & (1U << NI_CCNT_IDX)) {
457 ret = IRQ_HANDLED;
458 if (!(WARN_ON(!cd->ccnt))) {
459 arm_ni_event_read(cd->ccnt);
460 arm_ni_init_ccnt(cd);
461 }
462 }
463 for (int i = 0; i < NI_NUM_COUNTERS; i++) {
464 if (!(reg & (1U << i)))
465 continue;
466 ret = IRQ_HANDLED;
467 if (!(WARN_ON(!cd->evcnt[i]))) {
468 arm_ni_event_read(cd->evcnt[i]);
469 arm_ni_init_evcnt(cd, i);
470 }
471 }
472 writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);
473 if (!cd->irq_friend)
474 return ret;
475 cd += cd->irq_friend;
476 }
477 }
478
arm_ni_init_cd(struct arm_ni * ni,struct arm_ni_node * node,u64 res_start)479 static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start)
480 {
481 struct arm_ni_cd *cd = ni->cds + node->id;
482 const char *name;
483
484 cd->id = node->id;
485 cd->num_units = node->num_components;
486 cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL);
487 if (!cd->units)
488 return -ENOMEM;
489
490 for (int i = 0; i < cd->num_units; i++) {
491 u32 reg = readl_relaxed(node->base + NI_CHILD_PTR(i));
492 void __iomem *unit_base = ni->base + reg;
493 struct arm_ni_unit *unit = cd->units + i;
494
495 reg = readl_relaxed(unit_base + NI_NODE_TYPE);
496 unit->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
497 unit->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
498
499 switch (unit->type) {
500 case NI_PMU:
501 reg = readl_relaxed(unit_base + NI_PMCFGR);
502 if (!reg) {
503 dev_info(ni->dev, "No access to PMU %d\n", cd->id);
504 devm_kfree(ni->dev, cd->units);
505 return 0;
506 }
507 unit->ns = true;
508 cd->pmu_base = unit_base;
509 break;
510 case NI_ASNI:
511 case NI_AMNI:
512 case NI_HSNI:
513 case NI_HMNI:
514 case NI_PMNI:
515 unit->pmusela = unit_base + NI700_PMUSELA;
516 writel_relaxed(1, unit->pmusela);
517 if (readl_relaxed(unit->pmusela) != 1)
518 dev_info(ni->dev, "No access to node 0x%04x%04x\n", unit->id, unit->type);
519 else
520 unit->ns = true;
521 break;
522 default:
523 /*
524 * e.g. FMU - thankfully bits 3:2 of FMU_ERR_FR0 are RES0 so
525 * can't alias any of the leaf node types we're looking for.
526 */
527 dev_dbg(ni->dev, "Mystery node 0x%04x%04x\n", unit->id, unit->type);
528 break;
529 }
530 }
531
532 res_start += cd->pmu_base - ni->base;
533 if (!devm_request_mem_region(ni->dev, res_start, SZ_4K, dev_name(ni->dev))) {
534 dev_err(ni->dev, "Failed to request PMU region 0x%llx\n", res_start);
535 return -EBUSY;
536 }
537
538 writel_relaxed(NI_PMCR_RESET_CCNT | NI_PMCR_RESET_EVCNT,
539 cd->pmu_base + NI_PMCR);
540 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR);
541 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR);
542
543 cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id);
544 if (cd->irq < 0)
545 return cd->irq;
546
547 cd->pmu = (struct pmu) {
548 .module = THIS_MODULE,
549 .parent = ni->dev,
550 .attr_groups = arm_ni_attr_groups,
551 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
552 .task_ctx_nr = perf_invalid_context,
553 .pmu_enable = arm_ni_pmu_enable,
554 .pmu_disable = arm_ni_pmu_disable,
555 .event_init = arm_ni_event_init,
556 .add = arm_ni_event_add,
557 .del = arm_ni_event_del,
558 .start = arm_ni_event_start,
559 .stop = arm_ni_event_stop,
560 .read = arm_ni_event_read,
561 };
562
563 name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id);
564 if (!name)
565 return -ENOMEM;
566
567 return perf_pmu_register(&cd->pmu, name, -1);
568 }
569
arm_ni_remove(struct platform_device * pdev)570 static void arm_ni_remove(struct platform_device *pdev)
571 {
572 struct arm_ni *ni = platform_get_drvdata(pdev);
573
574 ni_for_each_cd(ni, cd) {
575 writel_relaxed(0, cd->pmu_base + NI_PMCR);
576 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR);
577 perf_pmu_unregister(&cd->pmu);
578 }
579 cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
580 }
581
arm_ni_probe_domain(void __iomem * base,struct arm_ni_node * node)582 static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node)
583 {
584 u32 reg = readl_relaxed(base + NI_NODE_TYPE);
585
586 node->base = base;
587 node->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
588 node->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
589 node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO);
590 }
591
arm_ni_init_irqs(struct arm_ni * ni)592 static int arm_ni_init_irqs(struct arm_ni *ni)
593 {
594 int err;
595
596 ni_for_each_cd(ni, cd) {
597 for (struct arm_ni_cd *prev = cd; prev-- > ni->cds; ) {
598 if (prev->irq == cd->irq) {
599 prev->irq_friend = cd - prev;
600 goto set_inten;
601 }
602 }
603 err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,
604 IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN,
605 dev_name(ni->dev), cd);
606 if (err)
607 return err;
608
609 irq_set_affinity(cd->irq, cpumask_of(ni->cpu));
610 set_inten:
611 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);
612 }
613
614 ni_for_each_cd(ni, cd)
615 if (!cd->irq_friend)
616 enable_irq(cd->irq);
617 return 0;
618 }
619
arm_ni_probe(struct platform_device * pdev)620 static int arm_ni_probe(struct platform_device *pdev)
621 {
622 struct arm_ni_node cfg, vd, pd, cd;
623 struct arm_ni *ni;
624 struct resource *res;
625 void __iomem *base;
626 static atomic_t id;
627 int ret, num_cds;
628 u32 reg, part;
629
630 /*
631 * We want to map the whole configuration space for ease of discovery,
632 * but the PMU pages are the only ones for which we can honestly claim
633 * exclusive ownership, so we'll request them explicitly once found.
634 */
635 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
636 base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
637 if (!base)
638 return -ENOMEM;
639
640 arm_ni_probe_domain(base, &cfg);
641 if (cfg.type != NI_GLOBAL)
642 return -ENODEV;
643
644 reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID0);
645 part = FIELD_GET(NI_PIDR0_PART_7_0, reg);
646 reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID1);
647 part |= FIELD_GET(NI_PIDR1_PART_11_8, reg) << 8;
648
649 switch (part) {
650 case PART_NI_700:
651 case PART_NI_710AE:
652 break;
653 default:
654 dev_WARN(&pdev->dev, "Unknown part number: 0x%03x, this may go badly\n", part);
655 break;
656 }
657
658 num_cds = 0;
659 for (int v = 0; v < cfg.num_components; v++) {
660 reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
661 arm_ni_probe_domain(base + reg, &vd);
662 for (int p = 0; p < vd.num_components; p++) {
663 reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
664 arm_ni_probe_domain(base + reg, &pd);
665 num_cds += pd.num_components;
666 }
667 }
668
669 ni = devm_kzalloc(&pdev->dev, struct_size(ni, cds, num_cds), GFP_KERNEL);
670 if (!ni)
671 return -ENOMEM;
672
673 ni->dev = &pdev->dev;
674 ni->base = base;
675 ni->num_cds = num_cds;
676 ni->part = part;
677 ni->id = atomic_fetch_inc(&id);
678 ni->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));
679 platform_set_drvdata(pdev, ni);
680
681 ret = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
682 if (ret)
683 return ret;
684
685 for (int v = 0; v < cfg.num_components; v++) {
686 reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
687 arm_ni_probe_domain(base + reg, &vd);
688 for (int p = 0; p < vd.num_components; p++) {
689 reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
690 arm_ni_probe_domain(base + reg, &pd);
691 for (int c = 0; c < pd.num_components; c++) {
692 reg = readl_relaxed(pd.base + NI_CHILD_PTR(c));
693 arm_ni_probe_domain(base + reg, &cd);
694 ret = arm_ni_init_cd(ni, &cd, res->start);
695 if (ret) {
696 ni->cds[cd.id].pmu_base = NULL;
697 arm_ni_remove(pdev);
698 return ret;
699 }
700 }
701 }
702 }
703
704 ret = arm_ni_init_irqs(ni);
705 if (ret)
706 arm_ni_remove(pdev);
707
708 return ret;
709 }
710
711 #ifdef CONFIG_OF
712 static const struct of_device_id arm_ni_of_match[] = {
713 { .compatible = "arm,ni-700" },
714 {}
715 };
716 MODULE_DEVICE_TABLE(of, arm_ni_of_match);
717 #endif
718
719 #ifdef CONFIG_ACPI
720 static const struct acpi_device_id arm_ni_acpi_match[] = {
721 { "ARMHCB70" },
722 {}
723 };
724 MODULE_DEVICE_TABLE(acpi, arm_ni_acpi_match);
725 #endif
726
727 static struct platform_driver arm_ni_driver = {
728 .driver = {
729 .name = "arm-ni",
730 .of_match_table = of_match_ptr(arm_ni_of_match),
731 .acpi_match_table = ACPI_PTR(arm_ni_acpi_match),
732 .suppress_bind_attrs = true,
733 },
734 .probe = arm_ni_probe,
735 .remove = arm_ni_remove,
736 };
737
arm_ni_pmu_migrate(struct arm_ni * ni,unsigned int cpu)738 static void arm_ni_pmu_migrate(struct arm_ni *ni, unsigned int cpu)
739 {
740 ni_for_each_cd(ni, cd) {
741 perf_pmu_migrate_context(&cd->pmu, ni->cpu, cpu);
742 irq_set_affinity(cd->irq, cpumask_of(cpu));
743 }
744 ni->cpu = cpu;
745 }
746
arm_ni_pmu_online_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)747 static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
748 {
749 struct arm_ni *ni;
750 int node;
751
752 ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
753 node = dev_to_node(ni->dev);
754 if (cpu_to_node(ni->cpu) != node && cpu_to_node(cpu) == node)
755 arm_ni_pmu_migrate(ni, cpu);
756 return 0;
757 }
758
arm_ni_pmu_offline_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)759 static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
760 {
761 struct arm_ni *ni;
762 unsigned int target;
763 int node;
764
765 ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
766 if (cpu != ni->cpu)
767 return 0;
768
769 node = dev_to_node(ni->dev);
770 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
771 if (target >= nr_cpu_ids)
772 target = cpumask_any_but(cpu_online_mask, cpu);
773
774 if (target < nr_cpu_ids)
775 arm_ni_pmu_migrate(ni, target);
776 return 0;
777 }
778
arm_ni_init(void)779 static int __init arm_ni_init(void)
780 {
781 int ret;
782
783 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
784 "perf/arm/ni:online",
785 arm_ni_pmu_online_cpu,
786 arm_ni_pmu_offline_cpu);
787 if (ret < 0)
788 return ret;
789
790 arm_ni_hp_state = ret;
791
792 ret = platform_driver_register(&arm_ni_driver);
793 if (ret)
794 cpuhp_remove_multi_state(arm_ni_hp_state);
795 return ret;
796 }
797
arm_ni_exit(void)798 static void __exit arm_ni_exit(void)
799 {
800 platform_driver_unregister(&arm_ni_driver);
801 cpuhp_remove_multi_state(arm_ni_hp_state);
802 }
803
804 module_init(arm_ni_init);
805 module_exit(arm_ni_exit);
806
807 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
808 MODULE_DESCRIPTION("Arm NI-700 PMU driver");
809 MODULE_LICENSE("GPL v2");
810