1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2022-2024 Arm Limited 3 // NI-700 Network-on-Chip PMU driver 4 5 #include <linux/acpi.h> 6 #include <linux/bitfield.h> 7 #include <linux/interrupt.h> 8 #include <linux/io.h> 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/perf_event.h> 14 #include <linux/platform_device.h> 15 #include <linux/slab.h> 16 17 /* Common registers */ 18 #define NI_NODE_TYPE 0x000 19 #define NI_NODE_TYPE_NODE_ID GENMASK(31, 16) 20 #define NI_NODE_TYPE_NODE_TYPE GENMASK(15, 0) 21 22 #define NI_CHILD_NODE_INFO 0x004 23 #define NI_CHILD_PTR(n) (0x008 + (n) * 4) 24 #define NI_NUM_SUB_FEATURES 0x100 25 #define NI_SUB_FEATURE_TYPE(n) (0x108 + (n) * 8) 26 #define NI_SUB_FEATURE_PTR(n) (0x10c + (n) * 8) 27 28 #define NI_SUB_FEATURE_TYPE_FCU 0x2 29 30 #define NI700_PMUSELA 0x00c 31 32 /* Config node */ 33 #define NI_PERIPHERAL_ID0 0xfe0 34 #define NI_PIDR0_PART_7_0 GENMASK(7, 0) 35 #define NI_PERIPHERAL_ID1 0xfe4 36 #define NI_PIDR1_PART_11_8 GENMASK(3, 0) 37 #define NI_PERIPHERAL_ID2 0xfe8 38 #define NI_PIDR2_VERSION GENMASK(7, 4) 39 40 /* PMU node */ 41 #define NI700_PMEVCNTR(n) (0x008 + (n) * 8) 42 #define NI700_PMCCNTR_L 0x0f8 43 #define NI_PMEVCNTR(n) (0x200 + (n) * 8) 44 #define NI_PMCCNTR_L 0x2f8 45 #define NI_PMEVTYPER(n) (0x400 + (n) * 4) 46 #define NI_PMEVTYPER_NODE_TYPE GENMASK(12, 9) 47 #define NI_PMEVTYPER_NODE_ID GENMASK(8, 0) 48 #define NI_PMCNTENSET 0xc00 49 #define NI_PMCNTENCLR 0xc20 50 #define NI_PMINTENSET 0xc40 51 #define NI_PMINTENCLR 0xc60 52 #define NI_PMOVSCLR 0xc80 53 #define NI_PMOVSSET 0xcc0 54 #define NI_PMCFGR 0xe00 55 #define NI_PMCR 0xe04 56 #define NI_PMCR_RESET_CCNT BIT(2) 57 #define NI_PMCR_RESET_EVCNT BIT(1) 58 #define NI_PMCR_ENABLE BIT(0) 59 60 #define NI_NUM_COUNTERS 8 61 #define NI_CCNT_IDX 31 62 63 /* Event attributes */ 64 #define NI_CONFIG_TYPE GENMASK_ULL(15, 0) 65 #define NI_CONFIG_NODEID GENMASK_ULL(31, 16) 66 #define NI_CONFIG_EVENTID GENMASK_ULL(47, 32) 67 68 #define NI_EVENT_TYPE(event) FIELD_GET(NI_CONFIG_TYPE, (event)->attr.config) 69 #define NI_EVENT_NODEID(event) FIELD_GET(NI_CONFIG_NODEID, (event)->attr.config) 70 #define NI_EVENT_EVENTID(event) FIELD_GET(NI_CONFIG_EVENTID, (event)->attr.config) 71 72 enum ni_part { 73 PART_NI_700 = 0x43b, 74 PART_NI_710AE = 0x43d, 75 PART_NOC_S3 = 0x43f, 76 PART_SI_L1 = 0x455, 77 }; 78 79 enum ni_node_type { 80 NI_GLOBAL, 81 NI_VOLTAGE, 82 NI_POWER, 83 NI_CLOCK, 84 NI_ASNI, 85 NI_AMNI, 86 NI_PMU, 87 NI_HSNI, 88 NI_HMNI, 89 NI_PMNI, 90 NI_TSNI, 91 NI_TMNI, 92 NI_CMNI = 0x0e, 93 NI_MCN = 0x63, 94 }; 95 96 struct arm_ni_node { 97 void __iomem *base; 98 enum ni_node_type type; 99 u16 id; 100 u32 num_components; 101 }; 102 103 struct arm_ni_unit { 104 void __iomem *pmusela; 105 enum ni_node_type type; 106 u16 id; 107 bool ns; 108 union { 109 __le64 pmusel; 110 u8 event[8]; 111 }; 112 }; 113 114 struct arm_ni_cd { 115 void __iomem *pmu_base; 116 u16 id; 117 s8 irq_friend; 118 int num_units; 119 int irq; 120 struct pmu pmu; 121 struct arm_ni_unit *units; 122 struct perf_event *evcnt[NI_NUM_COUNTERS]; 123 struct perf_event *ccnt; 124 }; 125 126 struct arm_ni { 127 struct device *dev; 128 void __iomem *base; 129 enum ni_part part; 130 int id; 131 int cpu; 132 int num_cds; 133 struct hlist_node cpuhp_node; 134 struct arm_ni_cd cds[] __counted_by(num_cds); 135 }; 136 137 #define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id]) 138 #define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu) 139 140 #define ni_for_each_cd(n, c) \ 141 for (struct arm_ni_cd *c = n->cds; c < n->cds + n->num_cds; c++) if (c->pmu_base) 142 143 #define cd_for_each_unit(cd, u) \ 144 for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++) 145 146 static int arm_ni_hp_state; 147 148 struct arm_ni_event_attr { 149 struct device_attribute attr; 150 enum ni_node_type type; 151 }; 152 153 #define NI_EVENT_ATTR(_name, _type) \ 154 (&((struct arm_ni_event_attr[]) {{ \ 155 .attr = __ATTR(_name, 0444, arm_ni_event_show, NULL), \ 156 .type = _type, \ 157 }})[0].attr.attr) 158 159 static ssize_t arm_ni_event_show(struct device *dev, 160 struct device_attribute *attr, char *buf) 161 { 162 struct arm_ni_event_attr *eattr = container_of(attr, typeof(*eattr), attr); 163 164 if (eattr->type == NI_PMU) 165 return sysfs_emit(buf, "type=0x%x\n", eattr->type); 166 167 return sysfs_emit(buf, "type=0x%x,eventid=?,nodeid=?\n", eattr->type); 168 } 169 170 static umode_t arm_ni_event_attr_is_visible(struct kobject *kobj, 171 struct attribute *attr, int unused) 172 { 173 struct device *dev = kobj_to_dev(kobj); 174 struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev)); 175 struct arm_ni_event_attr *eattr; 176 177 eattr = container_of(attr, typeof(*eattr), attr.attr); 178 179 cd_for_each_unit(cd, unit) { 180 if (unit->type == eattr->type && unit->ns) 181 return attr->mode; 182 } 183 184 return 0; 185 } 186 187 static struct attribute *arm_ni_event_attrs[] = { 188 NI_EVENT_ATTR(asni, NI_ASNI), 189 NI_EVENT_ATTR(amni, NI_AMNI), 190 NI_EVENT_ATTR(cycles, NI_PMU), 191 NI_EVENT_ATTR(hsni, NI_HSNI), 192 NI_EVENT_ATTR(hmni, NI_HMNI), 193 NI_EVENT_ATTR(pmni, NI_PMNI), 194 NI_EVENT_ATTR(tsni, NI_TSNI), 195 NI_EVENT_ATTR(tmni, NI_TMNI), 196 NI_EVENT_ATTR(cmni, NI_CMNI), 197 NULL 198 }; 199 200 static const struct attribute_group arm_ni_event_attrs_group = { 201 .name = "events", 202 .attrs = arm_ni_event_attrs, 203 .is_visible = arm_ni_event_attr_is_visible, 204 }; 205 206 struct arm_ni_format_attr { 207 struct device_attribute attr; 208 u64 field; 209 }; 210 211 #define NI_FORMAT_ATTR(_name, _fld) \ 212 (&((struct arm_ni_format_attr[]) {{ \ 213 .attr = __ATTR(_name, 0444, arm_ni_format_show, NULL), \ 214 .field = _fld, \ 215 }})[0].attr.attr) 216 217 static ssize_t arm_ni_format_show(struct device *dev, 218 struct device_attribute *attr, char *buf) 219 { 220 struct arm_ni_format_attr *fmt = container_of(attr, typeof(*fmt), attr); 221 222 return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field); 223 } 224 225 static struct attribute *arm_ni_format_attrs[] = { 226 NI_FORMAT_ATTR(type, NI_CONFIG_TYPE), 227 NI_FORMAT_ATTR(nodeid, NI_CONFIG_NODEID), 228 NI_FORMAT_ATTR(eventid, NI_CONFIG_EVENTID), 229 NULL 230 }; 231 232 static const struct attribute_group arm_ni_format_attrs_group = { 233 .name = "format", 234 .attrs = arm_ni_format_attrs, 235 }; 236 237 static ssize_t arm_ni_cpumask_show(struct device *dev, 238 struct device_attribute *attr, char *buf) 239 { 240 struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev))); 241 242 return cpumap_print_to_pagebuf(true, buf, cpumask_of(ni->cpu)); 243 } 244 245 static struct device_attribute arm_ni_cpumask_attr = 246 __ATTR(cpumask, 0444, arm_ni_cpumask_show, NULL); 247 248 static ssize_t arm_ni_identifier_show(struct device *dev, 249 struct device_attribute *attr, char *buf) 250 { 251 struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev))); 252 u32 reg = readl_relaxed(ni->base + NI_PERIPHERAL_ID2); 253 int version = FIELD_GET(NI_PIDR2_VERSION, reg); 254 255 return sysfs_emit(buf, "%03x%02x\n", ni->part, version); 256 } 257 258 static struct device_attribute arm_ni_identifier_attr = 259 __ATTR(identifier, 0444, arm_ni_identifier_show, NULL); 260 261 static struct attribute *arm_ni_other_attrs[] = { 262 &arm_ni_cpumask_attr.attr, 263 &arm_ni_identifier_attr.attr, 264 NULL 265 }; 266 267 static const struct attribute_group arm_ni_other_attr_group = { 268 .attrs = arm_ni_other_attrs, 269 }; 270 271 static const struct attribute_group *arm_ni_attr_groups[] = { 272 &arm_ni_event_attrs_group, 273 &arm_ni_format_attrs_group, 274 &arm_ni_other_attr_group, 275 NULL 276 }; 277 278 static void arm_ni_pmu_enable(struct pmu *pmu) 279 { 280 writel_relaxed(NI_PMCR_ENABLE, pmu_to_cd(pmu)->pmu_base + NI_PMCR); 281 } 282 283 static void arm_ni_pmu_disable(struct pmu *pmu) 284 { 285 writel_relaxed(0, pmu_to_cd(pmu)->pmu_base + NI_PMCR); 286 } 287 288 struct arm_ni_val { 289 unsigned int evcnt; 290 unsigned int ccnt; 291 }; 292 293 static bool arm_ni_val_count_event(struct perf_event *evt, struct arm_ni_val *val) 294 { 295 if (is_software_event(evt)) 296 return true; 297 298 if (NI_EVENT_TYPE(evt) == NI_PMU) { 299 val->ccnt++; 300 return val->ccnt <= 1; 301 } 302 303 val->evcnt++; 304 return val->evcnt <= NI_NUM_COUNTERS; 305 } 306 307 static int arm_ni_validate_group(struct perf_event *event) 308 { 309 struct perf_event *sibling, *leader = event->group_leader; 310 struct arm_ni_val val = { 0 }; 311 312 if (leader == event) 313 return 0; 314 315 arm_ni_val_count_event(event, &val); 316 if (!arm_ni_val_count_event(leader, &val)) 317 return -EINVAL; 318 319 for_each_sibling_event(sibling, leader) { 320 if (!arm_ni_val_count_event(sibling, &val)) 321 return -EINVAL; 322 } 323 return 0; 324 } 325 326 static bool arm_ni_is_7xx(const struct arm_ni *ni) 327 { 328 return ni->part == PART_NI_700 || ni->part == PART_NI_710AE; 329 } 330 331 static int arm_ni_event_init(struct perf_event *event) 332 { 333 struct arm_ni_cd *cd = pmu_to_cd(event->pmu); 334 struct arm_ni *ni; 335 336 if (event->attr.type != event->pmu->type) 337 return -ENOENT; 338 339 if (is_sampling_event(event)) 340 return -EINVAL; 341 342 ni = cd_to_ni(cd); 343 event->cpu = ni->cpu; 344 event->hw.flags = arm_ni_is_7xx(ni); 345 346 if (NI_EVENT_TYPE(event) == NI_PMU) 347 return arm_ni_validate_group(event); 348 349 cd_for_each_unit(cd, unit) { 350 if (unit->type == NI_EVENT_TYPE(event) && 351 unit->id == NI_EVENT_NODEID(event) && unit->ns) { 352 event->hw.config_base = (unsigned long)unit; 353 return arm_ni_validate_group(event); 354 } 355 } 356 return -EINVAL; 357 } 358 359 static u64 arm_ni_read_ccnt(void __iomem *pmccntr) 360 { 361 u64 l, u_old, u_new; 362 int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */ 363 364 u_new = readl_relaxed(pmccntr + 4); 365 do { 366 u_old = u_new; 367 l = readl_relaxed(pmccntr); 368 u_new = readl_relaxed(pmccntr + 4); 369 } while (u_new != u_old && --retries); 370 WARN_ON(!retries); 371 372 return (u_new << 32) | l; 373 } 374 375 static void arm_ni_event_read(struct perf_event *event) 376 { 377 struct hw_perf_event *hw = &event->hw; 378 u64 count, prev; 379 bool ccnt = hw->idx == NI_CCNT_IDX; 380 381 do { 382 prev = local64_read(&hw->prev_count); 383 if (ccnt) 384 count = arm_ni_read_ccnt((void __iomem *)event->hw.event_base); 385 else 386 count = readl_relaxed((void __iomem *)event->hw.event_base); 387 } while (local64_cmpxchg(&hw->prev_count, prev, count) != prev); 388 389 count -= prev; 390 if (!ccnt) 391 count = (u32)count; 392 local64_add(count, &event->count); 393 } 394 395 static void arm_ni_event_start(struct perf_event *event, int flags) 396 { 397 struct arm_ni_cd *cd = pmu_to_cd(event->pmu); 398 399 writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET); 400 } 401 402 static void arm_ni_event_stop(struct perf_event *event, int flags) 403 { 404 struct arm_ni_cd *cd = pmu_to_cd(event->pmu); 405 406 writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR); 407 if (flags & PERF_EF_UPDATE) 408 arm_ni_event_read(event); 409 } 410 411 static void arm_ni_init_ccnt(struct hw_perf_event *hw) 412 { 413 local64_set(&hw->prev_count, S64_MIN); 414 lo_hi_writeq_relaxed(S64_MIN, (void __iomem *)hw->event_base); 415 } 416 417 static void arm_ni_init_evcnt(struct hw_perf_event *hw) 418 { 419 local64_set(&hw->prev_count, S32_MIN); 420 writel_relaxed(S32_MIN, (void __iomem *)hw->event_base); 421 } 422 423 static int arm_ni_event_add(struct perf_event *event, int flags) 424 { 425 struct arm_ni_cd *cd = pmu_to_cd(event->pmu); 426 struct hw_perf_event *hw = &event->hw; 427 struct arm_ni_unit *unit; 428 enum ni_node_type type = NI_EVENT_TYPE(event); 429 u32 reg; 430 431 if (type == NI_PMU) { 432 if (cd->ccnt) 433 return -ENOSPC; 434 hw->idx = NI_CCNT_IDX; 435 hw->event_base = (unsigned long)cd->pmu_base + 436 (hw->flags ? NI700_PMCCNTR_L : NI_PMCCNTR_L); 437 cd->ccnt = event; 438 arm_ni_init_ccnt(hw); 439 } else { 440 hw->idx = 0; 441 while (cd->evcnt[hw->idx]) { 442 if (++hw->idx == NI_NUM_COUNTERS) 443 return -ENOSPC; 444 } 445 cd->evcnt[hw->idx] = event; 446 unit = (void *)hw->config_base; 447 unit->event[hw->idx] = NI_EVENT_EVENTID(event); 448 hw->event_base = (unsigned long)cd->pmu_base + 449 (hw->flags ? NI700_PMEVCNTR(hw->idx) : NI_PMEVCNTR(hw->idx)); 450 arm_ni_init_evcnt(hw); 451 lo_hi_writeq_relaxed(le64_to_cpu(unit->pmusel), unit->pmusela); 452 453 reg = FIELD_PREP(NI_PMEVTYPER_NODE_TYPE, type) | 454 FIELD_PREP(NI_PMEVTYPER_NODE_ID, NI_EVENT_NODEID(event)); 455 writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx)); 456 } 457 if (flags & PERF_EF_START) 458 arm_ni_event_start(event, 0); 459 return 0; 460 } 461 462 static void arm_ni_event_del(struct perf_event *event, int flags) 463 { 464 struct arm_ni_cd *cd = pmu_to_cd(event->pmu); 465 struct hw_perf_event *hw = &event->hw; 466 467 arm_ni_event_stop(event, PERF_EF_UPDATE); 468 469 if (hw->idx == NI_CCNT_IDX) 470 cd->ccnt = NULL; 471 else 472 cd->evcnt[hw->idx] = NULL; 473 } 474 475 static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id) 476 { 477 struct arm_ni_cd *cd = dev_id; 478 irqreturn_t ret = IRQ_NONE; 479 480 for (;;) { 481 u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR); 482 483 if (reg & (1U << NI_CCNT_IDX)) { 484 ret = IRQ_HANDLED; 485 if (!(WARN_ON(!cd->ccnt))) { 486 arm_ni_event_read(cd->ccnt); 487 arm_ni_init_ccnt(&cd->ccnt->hw); 488 } 489 } 490 for (int i = 0; i < NI_NUM_COUNTERS; i++) { 491 if (!(reg & (1U << i))) 492 continue; 493 ret = IRQ_HANDLED; 494 if (!(WARN_ON(!cd->evcnt[i]))) { 495 arm_ni_event_read(cd->evcnt[i]); 496 arm_ni_init_evcnt(&cd->evcnt[i]->hw); 497 } 498 } 499 writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR); 500 if (!cd->irq_friend) 501 return ret; 502 cd += cd->irq_friend; 503 } 504 } 505 506 static void __iomem *arm_ni_get_pmusel(struct arm_ni *ni, void __iomem *unit_base) 507 { 508 u32 type, ptr, num; 509 510 if (arm_ni_is_7xx(ni)) 511 return unit_base + NI700_PMUSELA; 512 513 num = readl_relaxed(unit_base + NI_NUM_SUB_FEATURES); 514 for (int i = 0; i < num; i++) { 515 type = readl_relaxed(unit_base + NI_SUB_FEATURE_TYPE(i)); 516 if (type != NI_SUB_FEATURE_TYPE_FCU) 517 continue; 518 ptr = readl_relaxed(unit_base + NI_SUB_FEATURE_PTR(i)); 519 return ni->base + ptr; 520 } 521 /* Should be impossible */ 522 return NULL; 523 } 524 525 static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start) 526 { 527 struct arm_ni_cd *cd = ni->cds + node->id; 528 const char *name; 529 530 cd->id = node->id; 531 cd->num_units = node->num_components; 532 cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL); 533 if (!cd->units) 534 return -ENOMEM; 535 536 for (int i = 0; i < cd->num_units; i++) { 537 u32 reg = readl_relaxed(node->base + NI_CHILD_PTR(i)); 538 void __iomem *unit_base = ni->base + reg; 539 struct arm_ni_unit *unit = cd->units + i; 540 541 reg = readl_relaxed(unit_base + NI_NODE_TYPE); 542 unit->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg); 543 unit->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg); 544 545 switch (unit->type) { 546 case NI_PMU: 547 reg = readl_relaxed(unit_base + NI_PMCFGR); 548 if (!reg) { 549 dev_info(ni->dev, "No access to PMU %d\n", cd->id); 550 devm_kfree(ni->dev, cd->units); 551 return 0; 552 } 553 unit->ns = true; 554 cd->pmu_base = unit_base; 555 break; 556 case NI_ASNI: 557 case NI_AMNI: 558 case NI_HSNI: 559 case NI_HMNI: 560 case NI_PMNI: 561 case NI_TSNI: 562 case NI_TMNI: 563 case NI_CMNI: 564 unit->pmusela = arm_ni_get_pmusel(ni, unit_base); 565 writel_relaxed(1, unit->pmusela); 566 if (readl_relaxed(unit->pmusela) != 1) 567 dev_info(ni->dev, "No access to node 0x%04x%04x\n", unit->id, unit->type); 568 else 569 unit->ns = true; 570 break; 571 case NI_MCN: 572 break; 573 default: 574 /* 575 * e.g. FMU - thankfully bits 3:2 of FMU_ERR_FR0 are RES0 so 576 * can't alias any of the leaf node types we're looking for. 577 */ 578 dev_dbg(ni->dev, "Mystery node 0x%04x%04x\n", unit->id, unit->type); 579 break; 580 } 581 } 582 583 res_start += cd->pmu_base - ni->base; 584 if (!devm_request_mem_region(ni->dev, res_start, SZ_4K, dev_name(ni->dev))) { 585 dev_err(ni->dev, "Failed to request PMU region 0x%llx\n", res_start); 586 return -EBUSY; 587 } 588 589 writel_relaxed(NI_PMCR_RESET_CCNT | NI_PMCR_RESET_EVCNT, 590 cd->pmu_base + NI_PMCR); 591 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR); 592 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR); 593 594 cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id); 595 if (cd->irq < 0) 596 return cd->irq; 597 598 cd->pmu = (struct pmu) { 599 .module = THIS_MODULE, 600 .parent = ni->dev, 601 .attr_groups = arm_ni_attr_groups, 602 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 603 .task_ctx_nr = perf_invalid_context, 604 .pmu_enable = arm_ni_pmu_enable, 605 .pmu_disable = arm_ni_pmu_disable, 606 .event_init = arm_ni_event_init, 607 .add = arm_ni_event_add, 608 .del = arm_ni_event_del, 609 .start = arm_ni_event_start, 610 .stop = arm_ni_event_stop, 611 .read = arm_ni_event_read, 612 }; 613 614 name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id); 615 if (!name) 616 return -ENOMEM; 617 618 return perf_pmu_register(&cd->pmu, name, -1); 619 } 620 621 static void arm_ni_remove(struct platform_device *pdev) 622 { 623 struct arm_ni *ni = platform_get_drvdata(pdev); 624 625 ni_for_each_cd(ni, cd) { 626 writel_relaxed(0, cd->pmu_base + NI_PMCR); 627 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR); 628 perf_pmu_unregister(&cd->pmu); 629 } 630 cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node); 631 } 632 633 static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node) 634 { 635 u32 reg = readl_relaxed(base + NI_NODE_TYPE); 636 637 node->base = base; 638 node->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg); 639 node->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg); 640 node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO); 641 } 642 643 static int arm_ni_init_irqs(struct arm_ni *ni) 644 { 645 int err; 646 647 ni_for_each_cd(ni, cd) { 648 for (struct arm_ni_cd *prev = cd; prev-- > ni->cds; ) { 649 if (prev->irq == cd->irq) { 650 prev->irq_friend = cd - prev; 651 goto set_inten; 652 } 653 } 654 err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq, 655 IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN, 656 dev_name(ni->dev), cd); 657 if (err) 658 return err; 659 660 irq_set_affinity(cd->irq, cpumask_of(ni->cpu)); 661 set_inten: 662 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET); 663 } 664 665 ni_for_each_cd(ni, cd) 666 if (!cd->irq_friend) 667 enable_irq(cd->irq); 668 return 0; 669 } 670 671 static int arm_ni_probe(struct platform_device *pdev) 672 { 673 struct arm_ni_node cfg, vd, pd, cd; 674 struct arm_ni *ni; 675 struct resource *res; 676 void __iomem *base; 677 static atomic_t id; 678 int ret, num_cds; 679 u32 reg, part; 680 681 /* 682 * We want to map the whole configuration space for ease of discovery, 683 * but the PMU pages are the only ones for which we can honestly claim 684 * exclusive ownership, so we'll request them explicitly once found. 685 */ 686 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 687 base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); 688 if (!base) 689 return -ENOMEM; 690 691 arm_ni_probe_domain(base, &cfg); 692 if (cfg.type != NI_GLOBAL) 693 return -ENODEV; 694 695 reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID0); 696 part = FIELD_GET(NI_PIDR0_PART_7_0, reg); 697 reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID1); 698 part |= FIELD_GET(NI_PIDR1_PART_11_8, reg) << 8; 699 700 switch (part) { 701 case PART_NI_700: 702 case PART_NI_710AE: 703 case PART_NOC_S3: 704 case PART_SI_L1: 705 break; 706 default: 707 dev_WARN(&pdev->dev, "Unknown part number: 0x%03x, this may go badly\n", part); 708 break; 709 } 710 711 num_cds = 0; 712 for (int v = 0; v < cfg.num_components; v++) { 713 reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v)); 714 arm_ni_probe_domain(base + reg, &vd); 715 for (int p = 0; p < vd.num_components; p++) { 716 reg = readl_relaxed(vd.base + NI_CHILD_PTR(p)); 717 arm_ni_probe_domain(base + reg, &pd); 718 num_cds += pd.num_components; 719 } 720 } 721 722 ni = devm_kzalloc(&pdev->dev, struct_size(ni, cds, num_cds), GFP_KERNEL); 723 if (!ni) 724 return -ENOMEM; 725 726 ni->dev = &pdev->dev; 727 ni->base = base; 728 ni->num_cds = num_cds; 729 ni->part = part; 730 ni->id = atomic_fetch_inc(&id); 731 ni->cpu = cpumask_local_spread(0, dev_to_node(ni->dev)); 732 platform_set_drvdata(pdev, ni); 733 734 ret = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node); 735 if (ret) 736 return ret; 737 738 for (int v = 0; v < cfg.num_components; v++) { 739 reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v)); 740 arm_ni_probe_domain(base + reg, &vd); 741 for (int p = 0; p < vd.num_components; p++) { 742 reg = readl_relaxed(vd.base + NI_CHILD_PTR(p)); 743 arm_ni_probe_domain(base + reg, &pd); 744 for (int c = 0; c < pd.num_components; c++) { 745 reg = readl_relaxed(pd.base + NI_CHILD_PTR(c)); 746 arm_ni_probe_domain(base + reg, &cd); 747 ret = arm_ni_init_cd(ni, &cd, res->start); 748 if (ret) { 749 ni->cds[cd.id].pmu_base = NULL; 750 arm_ni_remove(pdev); 751 return ret; 752 } 753 } 754 } 755 } 756 757 ret = arm_ni_init_irqs(ni); 758 if (ret) 759 arm_ni_remove(pdev); 760 761 return ret; 762 } 763 764 #ifdef CONFIG_OF 765 static const struct of_device_id arm_ni_of_match[] = { 766 { .compatible = "arm,ni-700" }, 767 {} 768 }; 769 MODULE_DEVICE_TABLE(of, arm_ni_of_match); 770 #endif 771 772 #ifdef CONFIG_ACPI 773 static const struct acpi_device_id arm_ni_acpi_match[] = { 774 { "ARMHCB70" }, 775 {} 776 }; 777 MODULE_DEVICE_TABLE(acpi, arm_ni_acpi_match); 778 #endif 779 780 static struct platform_driver arm_ni_driver = { 781 .driver = { 782 .name = "arm-ni", 783 .of_match_table = of_match_ptr(arm_ni_of_match), 784 .acpi_match_table = ACPI_PTR(arm_ni_acpi_match), 785 .suppress_bind_attrs = true, 786 }, 787 .probe = arm_ni_probe, 788 .remove = arm_ni_remove, 789 }; 790 791 static void arm_ni_pmu_migrate(struct arm_ni *ni, unsigned int cpu) 792 { 793 ni_for_each_cd(ni, cd) { 794 perf_pmu_migrate_context(&cd->pmu, ni->cpu, cpu); 795 irq_set_affinity(cd->irq, cpumask_of(cpu)); 796 } 797 ni->cpu = cpu; 798 } 799 800 static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 801 { 802 struct arm_ni *ni; 803 int node; 804 805 ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node); 806 node = dev_to_node(ni->dev); 807 if (cpu_to_node(ni->cpu) != node && cpu_to_node(cpu) == node) 808 arm_ni_pmu_migrate(ni, cpu); 809 return 0; 810 } 811 812 static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 813 { 814 struct arm_ni *ni; 815 unsigned int target; 816 int node; 817 818 ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node); 819 if (cpu != ni->cpu) 820 return 0; 821 822 node = dev_to_node(ni->dev); 823 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); 824 if (target >= nr_cpu_ids) 825 target = cpumask_any_but(cpu_online_mask, cpu); 826 827 if (target < nr_cpu_ids) 828 arm_ni_pmu_migrate(ni, target); 829 return 0; 830 } 831 832 static int __init arm_ni_init(void) 833 { 834 int ret; 835 836 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 837 "perf/arm/ni:online", 838 arm_ni_pmu_online_cpu, 839 arm_ni_pmu_offline_cpu); 840 if (ret < 0) 841 return ret; 842 843 arm_ni_hp_state = ret; 844 845 ret = platform_driver_register(&arm_ni_driver); 846 if (ret) 847 cpuhp_remove_multi_state(arm_ni_hp_state); 848 return ret; 849 } 850 851 static void __exit arm_ni_exit(void) 852 { 853 platform_driver_unregister(&arm_ni_driver); 854 cpuhp_remove_multi_state(arm_ni_hp_state); 855 } 856 857 module_init(arm_ni_init); 858 module_exit(arm_ni_exit); 859 860 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); 861 MODULE_DESCRIPTION("Arm NI-700 PMU driver"); 862 MODULE_LICENSE("GPL v2"); 863