1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * StarFive's StarLink PMU driver 4 * 5 * Copyright (C) 2023 StarFive Technology Co., Ltd. 6 * 7 * Author: Ji Sheng Teoh <jisheng.teoh@starfivetech.com> 8 * 9 */ 10 11 #define STARLINK_PMU_PDEV_NAME "starfive_starlink_pmu" 12 #define pr_fmt(fmt) STARLINK_PMU_PDEV_NAME ": " fmt 13 14 #include <linux/bitmap.h> 15 #include <linux/cpu_pm.h> 16 #include <linux/io.h> 17 #include <linux/irq.h> 18 #include <linux/kernel.h> 19 #include <linux/module.h> 20 #include <linux/perf_event.h> 21 #include <linux/platform_device.h> 22 #include <linux/sysfs.h> 23 24 #define STARLINK_PMU_MAX_COUNTERS 64 25 #define STARLINK_PMU_NUM_COUNTERS 16 26 #define STARLINK_PMU_IDX_CYCLE_COUNTER 63 27 28 #define STARLINK_PMU_EVENT_SELECT 0x060 29 #define STARLINK_PMU_EVENT_COUNTER 0x160 30 #define STARLINK_PMU_COUNTER_MASK GENMASK_ULL(63, 0) 31 #define STARLINK_PMU_CYCLE_COUNTER 0x058 32 33 #define STARLINK_PMU_CONTROL 0x040 34 #define STARLINK_PMU_GLOBAL_ENABLE BIT_ULL(0) 35 36 #define STARLINK_PMU_INTERRUPT_ENABLE 0x050 37 #define STARLINK_PMU_COUNTER_OVERFLOW_STATUS 0x048 38 #define STARLINK_PMU_CYCLE_OVERFLOW_MASK BIT_ULL(63) 39 40 #define STARLINK_CYCLES 0x058 41 #define CACHE_READ_REQUEST 0x04000701 42 #define CACHE_WRITE_REQUEST 0x03000001 43 #define CACHE_RELEASE_REQUEST 0x0003e001 44 #define CACHE_READ_HIT 0x00901202 45 #define CACHE_READ_MISS 0x04008002 46 #define CACHE_WRITE_HIT 0x006c0002 47 #define CACHE_WRITE_MISS 0x03000002 48 #define CACHE_WRITEBACK 0x00000403 49 50 #define to_starlink_pmu(p) (container_of(p, struct starlink_pmu, pmu)) 51 52 #define STARLINK_FORMAT_ATTR(_name, _config) \ 53 (&((struct dev_ext_attribute[]) { \ 54 { .attr = __ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \ 55 .var = (void *)_config, } \ 56 })[0].attr.attr) 57 58 #define STARLINK_EVENT_ATTR(_name, _id) \ 59 PMU_EVENT_ATTR_ID(_name, starlink_pmu_sysfs_event_show, _id) 60 61 static int starlink_pmu_cpuhp_state; 62 63 struct starlink_hw_events { 64 struct perf_event *events[STARLINK_PMU_MAX_COUNTERS]; 65 DECLARE_BITMAP(used_mask, STARLINK_PMU_MAX_COUNTERS); 66 }; 67 68 struct starlink_pmu { 69 struct pmu pmu; 70 struct starlink_hw_events __percpu *hw_events; 71 struct hlist_node node; 72 struct notifier_block starlink_pmu_pm_nb; 73 void __iomem *pmu_base; 74 cpumask_t cpumask; 75 int irq; 76 }; 77 78 static ssize_t 79 starlink_pmu_sysfs_format_show(struct device *dev, 80 struct device_attribute *attr, 81 char *buf) 82 { 83 struct dev_ext_attribute *eattr = container_of(attr, 84 struct dev_ext_attribute, attr); 85 86 return sysfs_emit(buf, "%s\n", (char *)eattr->var); 87 } 88 89 static struct attribute *starlink_pmu_format_attrs[] = { 90 STARLINK_FORMAT_ATTR(event, "config:0-31"), 91 NULL 92 }; 93 94 static const struct attribute_group starlink_pmu_format_attr_group = { 95 .name = "format", 96 .attrs = starlink_pmu_format_attrs, 97 }; 98 99 static ssize_t 100 starlink_pmu_sysfs_event_show(struct device *dev, 101 struct device_attribute *attr, 102 char *buf) 103 { 104 struct perf_pmu_events_attr *eattr = container_of(attr, 105 struct perf_pmu_events_attr, attr); 106 107 return sysfs_emit(buf, "event=0x%02llx\n", eattr->id); 108 } 109 110 static struct attribute *starlink_pmu_event_attrs[] = { 111 STARLINK_EVENT_ATTR(cycles, STARLINK_CYCLES), 112 STARLINK_EVENT_ATTR(read_request, CACHE_READ_REQUEST), 113 STARLINK_EVENT_ATTR(write_request, CACHE_WRITE_REQUEST), 114 STARLINK_EVENT_ATTR(release_request, CACHE_RELEASE_REQUEST), 115 STARLINK_EVENT_ATTR(read_hit, CACHE_READ_HIT), 116 STARLINK_EVENT_ATTR(read_miss, CACHE_READ_MISS), 117 STARLINK_EVENT_ATTR(write_hit, CACHE_WRITE_HIT), 118 STARLINK_EVENT_ATTR(write_miss, CACHE_WRITE_MISS), 119 STARLINK_EVENT_ATTR(writeback, CACHE_WRITEBACK), 120 NULL 121 }; 122 123 static const struct attribute_group starlink_pmu_events_attr_group = { 124 .name = "events", 125 .attrs = starlink_pmu_event_attrs, 126 }; 127 128 static ssize_t 129 cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) 130 { 131 struct starlink_pmu *starlink_pmu = to_starlink_pmu(dev_get_drvdata(dev)); 132 133 return cpumap_print_to_pagebuf(true, buf, &starlink_pmu->cpumask); 134 } 135 136 static DEVICE_ATTR_RO(cpumask); 137 138 static struct attribute *starlink_pmu_cpumask_attrs[] = { 139 &dev_attr_cpumask.attr, 140 NULL 141 }; 142 143 static const struct attribute_group starlink_pmu_cpumask_attr_group = { 144 .attrs = starlink_pmu_cpumask_attrs, 145 }; 146 147 static const struct attribute_group *starlink_pmu_attr_groups[] = { 148 &starlink_pmu_format_attr_group, 149 &starlink_pmu_events_attr_group, 150 &starlink_pmu_cpumask_attr_group, 151 NULL 152 }; 153 154 static void starlink_pmu_set_event_period(struct perf_event *event) 155 { 156 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); 157 struct hw_perf_event *hwc = &event->hw; 158 int idx = event->hw.idx; 159 160 /* 161 * Program counter to half of it's max count to handle 162 * cases of extreme interrupt latency. 163 */ 164 u64 val = STARLINK_PMU_COUNTER_MASK >> 1; 165 166 local64_set(&hwc->prev_count, val); 167 if (hwc->config == STARLINK_CYCLES) 168 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_CYCLE_COUNTER); 169 else 170 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_EVENT_COUNTER + 171 idx * sizeof(u64)); 172 } 173 174 static void starlink_pmu_counter_start(struct perf_event *event, 175 struct starlink_pmu *starlink_pmu) 176 { 177 struct hw_perf_event *hwc = &event->hw; 178 int idx = event->hw.idx; 179 u64 val; 180 181 /* 182 * Enable counter overflow interrupt[63:0], 183 * which is mapped as follow: 184 * 185 * event counter 0 - Bit [0] 186 * event counter 1 - Bit [1] 187 * ... 188 * cycle counter - Bit [63] 189 */ 190 val = readq(starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); 191 192 if (hwc->config == STARLINK_CYCLES) { 193 /* 194 * Cycle count has its dedicated register, and it starts 195 * counting as soon as STARLINK_PMU_GLOBAL_ENABLE is set. 196 */ 197 val |= STARLINK_PMU_CYCLE_OVERFLOW_MASK; 198 } else { 199 writeq(event->hw.config, starlink_pmu->pmu_base + 200 STARLINK_PMU_EVENT_SELECT + idx * sizeof(u64)); 201 202 val |= BIT_ULL(idx); 203 } 204 205 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); 206 207 writeq(STARLINK_PMU_GLOBAL_ENABLE, starlink_pmu->pmu_base + 208 STARLINK_PMU_CONTROL); 209 } 210 211 static void starlink_pmu_counter_stop(struct perf_event *event, 212 struct starlink_pmu *starlink_pmu) 213 { 214 struct hw_perf_event *hwc = &event->hw; 215 int idx = event->hw.idx; 216 u64 val; 217 218 val = readq(starlink_pmu->pmu_base + STARLINK_PMU_CONTROL); 219 val &= ~STARLINK_PMU_GLOBAL_ENABLE; 220 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_CONTROL); 221 222 val = readq(starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); 223 if (hwc->config == STARLINK_CYCLES) 224 val &= ~STARLINK_PMU_CYCLE_OVERFLOW_MASK; 225 else 226 val &= ~BIT_ULL(idx); 227 228 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); 229 } 230 231 static void starlink_pmu_update(struct perf_event *event) 232 { 233 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); 234 struct hw_perf_event *hwc = &event->hw; 235 int idx = hwc->idx; 236 u64 prev_raw_count, new_raw_count; 237 u64 oldval; 238 u64 delta; 239 240 do { 241 prev_raw_count = local64_read(&hwc->prev_count); 242 if (hwc->config == STARLINK_CYCLES) 243 new_raw_count = readq(starlink_pmu->pmu_base + 244 STARLINK_PMU_CYCLE_COUNTER); 245 else 246 new_raw_count = readq(starlink_pmu->pmu_base + 247 STARLINK_PMU_EVENT_COUNTER + 248 idx * sizeof(u64)); 249 oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, 250 new_raw_count); 251 } while (oldval != prev_raw_count); 252 253 delta = (new_raw_count - prev_raw_count) & STARLINK_PMU_COUNTER_MASK; 254 local64_add(delta, &event->count); 255 } 256 257 static void starlink_pmu_start(struct perf_event *event, int flags) 258 { 259 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); 260 struct hw_perf_event *hwc = &event->hw; 261 262 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 263 return; 264 265 if (flags & PERF_EF_RELOAD) 266 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 267 268 hwc->state = 0; 269 270 starlink_pmu_set_event_period(event); 271 starlink_pmu_counter_start(event, starlink_pmu); 272 273 perf_event_update_userpage(event); 274 } 275 276 static void starlink_pmu_stop(struct perf_event *event, int flags) 277 { 278 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); 279 struct hw_perf_event *hwc = &event->hw; 280 281 if (hwc->state & PERF_HES_STOPPED) 282 return; 283 284 starlink_pmu_counter_stop(event, starlink_pmu); 285 starlink_pmu_update(event); 286 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 287 } 288 289 static int starlink_pmu_add(struct perf_event *event, int flags) 290 { 291 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); 292 struct starlink_hw_events *hw_events = 293 this_cpu_ptr(starlink_pmu->hw_events); 294 struct hw_perf_event *hwc = &event->hw; 295 unsigned long *used_mask = hw_events->used_mask; 296 u32 n_events = STARLINK_PMU_NUM_COUNTERS; 297 int idx; 298 299 /* 300 * Cycle counter has dedicated register to hold counter value. 301 * Event other than cycle count has to be enabled through 302 * event select register, and assigned with independent counter 303 * as they appear. 304 */ 305 306 if (hwc->config == STARLINK_CYCLES) { 307 idx = STARLINK_PMU_IDX_CYCLE_COUNTER; 308 } else { 309 idx = find_first_zero_bit(used_mask, n_events); 310 /* All counter are in use */ 311 if (idx < 0) 312 return idx; 313 314 set_bit(idx, used_mask); 315 } 316 317 hwc->idx = idx; 318 hw_events->events[idx] = event; 319 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 320 321 if (flags & PERF_EF_START) 322 starlink_pmu_start(event, PERF_EF_RELOAD); 323 324 perf_event_update_userpage(event); 325 326 return 0; 327 } 328 329 static void starlink_pmu_del(struct perf_event *event, int flags) 330 { 331 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); 332 struct starlink_hw_events *hw_events = 333 this_cpu_ptr(starlink_pmu->hw_events); 334 struct hw_perf_event *hwc = &event->hw; 335 336 starlink_pmu_stop(event, PERF_EF_UPDATE); 337 hw_events->events[hwc->idx] = NULL; 338 clear_bit(hwc->idx, hw_events->used_mask); 339 340 perf_event_update_userpage(event); 341 } 342 343 static bool starlink_pmu_validate_event_group(struct perf_event *event) 344 { 345 struct perf_event *leader = event->group_leader; 346 struct perf_event *sibling; 347 int counter = 1; 348 349 /* 350 * Ensure hardware events in the group are on the same PMU, 351 * software events are acceptable. 352 */ 353 if (event->group_leader->pmu != event->pmu && 354 !is_software_event(event->group_leader)) 355 return false; 356 357 for_each_sibling_event(sibling, leader) { 358 if (sibling->pmu != event->pmu && !is_software_event(sibling)) 359 return false; 360 361 counter++; 362 } 363 364 return counter <= STARLINK_PMU_NUM_COUNTERS; 365 } 366 367 static int starlink_pmu_event_init(struct perf_event *event) 368 { 369 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); 370 struct hw_perf_event *hwc = &event->hw; 371 372 /* 373 * Sampling is not supported, as counters are shared 374 * by all CPU. 375 */ 376 if (hwc->sample_period) 377 return -EOPNOTSUPP; 378 379 /* 380 * Per-task and attach to a task are not supported, 381 * as uncore events are not specific to any CPU. 382 */ 383 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) 384 return -EOPNOTSUPP; 385 386 if (!starlink_pmu_validate_event_group(event)) 387 return -EINVAL; 388 389 hwc->idx = -1; 390 hwc->config = event->attr.config; 391 event->cpu = cpumask_first(&starlink_pmu->cpumask); 392 393 return 0; 394 } 395 396 static irqreturn_t starlink_pmu_handle_irq(int irq_num, void *data) 397 { 398 struct starlink_pmu *starlink_pmu = data; 399 struct starlink_hw_events *hw_events = 400 this_cpu_ptr(starlink_pmu->hw_events); 401 bool handled = false; 402 int idx; 403 u64 overflow_status; 404 405 for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) { 406 struct perf_event *event = hw_events->events[idx]; 407 408 if (!event) 409 continue; 410 411 overflow_status = readq(starlink_pmu->pmu_base + 412 STARLINK_PMU_COUNTER_OVERFLOW_STATUS); 413 if (!(overflow_status & BIT_ULL(idx))) 414 continue; 415 416 writeq(BIT_ULL(idx), starlink_pmu->pmu_base + 417 STARLINK_PMU_COUNTER_OVERFLOW_STATUS); 418 419 starlink_pmu_update(event); 420 starlink_pmu_set_event_period(event); 421 handled = true; 422 } 423 return IRQ_RETVAL(handled); 424 } 425 426 static int starlink_setup_irqs(struct starlink_pmu *starlink_pmu, 427 struct platform_device *pdev) 428 { 429 int ret, irq; 430 431 irq = platform_get_irq(pdev, 0); 432 if (irq < 0) 433 return -EINVAL; 434 435 ret = devm_request_irq(&pdev->dev, irq, starlink_pmu_handle_irq, 436 0, STARLINK_PMU_PDEV_NAME, starlink_pmu); 437 if (ret) 438 return dev_err_probe(&pdev->dev, ret, "Failed to request IRQ\n"); 439 440 starlink_pmu->irq = irq; 441 442 return 0; 443 } 444 445 static int starlink_pmu_pm_notify(struct notifier_block *b, 446 unsigned long cmd, void *v) 447 { 448 struct starlink_pmu *starlink_pmu = container_of(b, struct starlink_pmu, 449 starlink_pmu_pm_nb); 450 struct starlink_hw_events *hw_events = 451 this_cpu_ptr(starlink_pmu->hw_events); 452 bool enabled = !bitmap_empty(hw_events->used_mask, STARLINK_PMU_MAX_COUNTERS); 453 struct perf_event *event; 454 int idx; 455 456 if (!enabled) 457 return NOTIFY_OK; 458 459 for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) { 460 event = hw_events->events[idx]; 461 if (!event) 462 continue; 463 464 switch (cmd) { 465 case CPU_PM_ENTER: 466 /* Stop and update the counter */ 467 starlink_pmu_stop(event, PERF_EF_UPDATE); 468 break; 469 case CPU_PM_EXIT: 470 case CPU_PM_ENTER_FAILED: 471 /* Restore and enable the counter */ 472 starlink_pmu_start(event, PERF_EF_RELOAD); 473 break; 474 default: 475 break; 476 } 477 } 478 479 return NOTIFY_OK; 480 } 481 482 static int starlink_pmu_pm_register(struct starlink_pmu *starlink_pmu) 483 { 484 if (!IS_ENABLED(CONFIG_CPU_PM)) 485 return 0; 486 487 starlink_pmu->starlink_pmu_pm_nb.notifier_call = starlink_pmu_pm_notify; 488 return cpu_pm_register_notifier(&starlink_pmu->starlink_pmu_pm_nb); 489 } 490 491 static void starlink_pmu_pm_unregister(struct starlink_pmu *starlink_pmu) 492 { 493 if (!IS_ENABLED(CONFIG_CPU_PM)) 494 return; 495 496 cpu_pm_unregister_notifier(&starlink_pmu->starlink_pmu_pm_nb); 497 } 498 499 static void starlink_pmu_destroy(struct starlink_pmu *starlink_pmu) 500 { 501 starlink_pmu_pm_unregister(starlink_pmu); 502 cpuhp_state_remove_instance(starlink_pmu_cpuhp_state, 503 &starlink_pmu->node); 504 } 505 506 static int starlink_pmu_probe(struct platform_device *pdev) 507 { 508 struct starlink_pmu *starlink_pmu; 509 struct starlink_hw_events *hw_events; 510 struct resource *res; 511 int cpuid, i, ret; 512 513 starlink_pmu = devm_kzalloc(&pdev->dev, sizeof(*starlink_pmu), GFP_KERNEL); 514 if (!starlink_pmu) 515 return -ENOMEM; 516 517 starlink_pmu->pmu_base = 518 devm_platform_get_and_ioremap_resource(pdev, 0, &res); 519 if (IS_ERR(starlink_pmu->pmu_base)) 520 return PTR_ERR(starlink_pmu->pmu_base); 521 522 starlink_pmu->hw_events = alloc_percpu_gfp(struct starlink_hw_events, 523 GFP_KERNEL); 524 if (!starlink_pmu->hw_events) { 525 dev_err(&pdev->dev, "Failed to allocate per-cpu PMU data\n"); 526 return -ENOMEM; 527 } 528 529 for_each_possible_cpu(cpuid) { 530 hw_events = per_cpu_ptr(starlink_pmu->hw_events, cpuid); 531 for (i = 0; i < STARLINK_PMU_MAX_COUNTERS; i++) 532 hw_events->events[i] = NULL; 533 } 534 535 ret = starlink_setup_irqs(starlink_pmu, pdev); 536 if (ret) 537 return ret; 538 539 ret = cpuhp_state_add_instance(starlink_pmu_cpuhp_state, 540 &starlink_pmu->node); 541 if (ret) { 542 dev_err(&pdev->dev, "Failed to register hotplug\n"); 543 return ret; 544 } 545 546 ret = starlink_pmu_pm_register(starlink_pmu); 547 if (ret) { 548 cpuhp_state_remove_instance(starlink_pmu_cpuhp_state, 549 &starlink_pmu->node); 550 return ret; 551 } 552 553 starlink_pmu->pmu = (struct pmu) { 554 .task_ctx_nr = perf_invalid_context, 555 .event_init = starlink_pmu_event_init, 556 .add = starlink_pmu_add, 557 .del = starlink_pmu_del, 558 .start = starlink_pmu_start, 559 .stop = starlink_pmu_stop, 560 .read = starlink_pmu_update, 561 .attr_groups = starlink_pmu_attr_groups, 562 }; 563 564 ret = perf_pmu_register(&starlink_pmu->pmu, STARLINK_PMU_PDEV_NAME, -1); 565 if (ret) 566 starlink_pmu_destroy(starlink_pmu); 567 568 return ret; 569 } 570 571 static const struct of_device_id starlink_pmu_of_match[] = { 572 { .compatible = "starfive,jh8100-starlink-pmu" }, 573 {} 574 }; 575 MODULE_DEVICE_TABLE(of, starlink_pmu_of_match); 576 577 static struct platform_driver starlink_pmu_driver = { 578 .driver = { 579 .name = STARLINK_PMU_PDEV_NAME, 580 .of_match_table = starlink_pmu_of_match, 581 .suppress_bind_attrs = true, 582 }, 583 .probe = starlink_pmu_probe, 584 }; 585 586 static int 587 starlink_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 588 { 589 struct starlink_pmu *starlink_pmu = hlist_entry_safe(node, 590 struct starlink_pmu, 591 node); 592 593 if (cpumask_empty(&starlink_pmu->cpumask)) 594 cpumask_set_cpu(cpu, &starlink_pmu->cpumask); 595 596 WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(cpu))); 597 598 return 0; 599 } 600 601 static int 602 starlink_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 603 { 604 struct starlink_pmu *starlink_pmu = hlist_entry_safe(node, 605 struct starlink_pmu, 606 node); 607 unsigned int target; 608 609 if (!cpumask_test_and_clear_cpu(cpu, &starlink_pmu->cpumask)) 610 return 0; 611 612 target = cpumask_any_but(cpu_online_mask, cpu); 613 if (target >= nr_cpu_ids) 614 return 0; 615 616 perf_pmu_migrate_context(&starlink_pmu->pmu, cpu, target); 617 618 cpumask_set_cpu(target, &starlink_pmu->cpumask); 619 WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(target))); 620 621 return 0; 622 } 623 624 static int __init starlink_pmu_init(void) 625 { 626 int ret; 627 628 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 629 "soc/starfive/starlink_pmu:online", 630 starlink_pmu_online_cpu, 631 starlink_pmu_offline_cpu); 632 if (ret < 0) 633 return ret; 634 635 starlink_pmu_cpuhp_state = ret; 636 637 return platform_driver_register(&starlink_pmu_driver); 638 } 639 640 device_initcall(starlink_pmu_init); 641