1 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved. 2 * 3 * This program is free software; you can redistribute it and/or modify 4 * it under the terms of the GNU General Public License version 2 and 5 * only version 2 as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 */ 12 #include <linux/acpi.h> 13 #include <linux/bitops.h> 14 #include <linux/bug.h> 15 #include <linux/cpuhotplug.h> 16 #include <linux/cpumask.h> 17 #include <linux/device.h> 18 #include <linux/errno.h> 19 #include <linux/interrupt.h> 20 #include <linux/irq.h> 21 #include <linux/kernel.h> 22 #include <linux/list.h> 23 #include <linux/percpu.h> 24 #include <linux/perf_event.h> 25 #include <linux/platform_device.h> 26 #include <linux/smp.h> 27 #include <linux/spinlock.h> 28 #include <linux/sysfs.h> 29 #include <linux/types.h> 30 31 #include <asm/barrier.h> 32 #include <asm/local64.h> 33 #include <asm/sysreg.h> 34 35 #define MAX_L2_CTRS 9 36 37 #define L2PMCR_NUM_EV_SHIFT 11 38 #define L2PMCR_NUM_EV_MASK 0x1F 39 40 #define L2PMCR 0x400 41 #define L2PMCNTENCLR 0x403 42 #define L2PMCNTENSET 0x404 43 #define L2PMINTENCLR 0x405 44 #define L2PMINTENSET 0x406 45 #define L2PMOVSCLR 0x407 46 #define L2PMOVSSET 0x408 47 #define L2PMCCNTCR 0x409 48 #define L2PMCCNTR 0x40A 49 #define L2PMCCNTSR 0x40C 50 #define L2PMRESR 0x410 51 #define IA_L2PMXEVCNTCR_BASE 0x420 52 #define IA_L2PMXEVCNTR_BASE 0x421 53 #define IA_L2PMXEVFILTER_BASE 0x423 54 #define IA_L2PMXEVTYPER_BASE 0x424 55 56 #define IA_L2_REG_OFFSET 0x10 57 58 #define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000 59 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004 60 #define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003 61 62 #define L2EVTYPER_REG_SHIFT 3 63 64 #define L2PMRESR_GROUP_BITS 8 65 #define L2PMRESR_GROUP_MASK GENMASK(7, 0) 66 67 #define L2CYCLE_CTR_BIT 31 68 #define L2CYCLE_CTR_RAW_CODE 0xFE 69 70 #define L2PMCR_RESET_ALL 0x6 71 #define L2PMCR_COUNTERS_ENABLE 0x1 72 #define L2PMCR_COUNTERS_DISABLE 0x0 73 74 #define L2PMRESR_EN BIT_ULL(63) 75 76 #define L2_EVT_MASK 0x00000FFF 77 #define L2_EVT_CODE_MASK 0x00000FF0 78 #define L2_EVT_GRP_MASK 0x0000000F 79 #define L2_EVT_CODE_SHIFT 4 80 #define L2_EVT_GRP_SHIFT 0 81 82 #define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT) 83 #define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT) 84 85 #define L2_EVT_GROUP_MAX 7 86 87 #define L2_COUNTER_RELOAD BIT_ULL(31) 88 #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63) 89 90 #define L2CPUSRSELR_EL1 sys_reg(3, 3, 15, 0, 6) 91 #define L2CPUSRDR_EL1 sys_reg(3, 3, 15, 0, 7) 92 93 #define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE) 94 95 static DEFINE_RAW_SPINLOCK(l2_access_lock); 96 97 /** 98 * set_l2_indirect_reg: write value to an L2 register 99 * @reg: Address of L2 register. 100 * @value: Value to be written to register. 101 * 102 * Use architecturally required barriers for ordering between system register 103 * accesses 104 */ 105 static void set_l2_indirect_reg(u64 reg, u64 val) 106 { 107 unsigned long flags; 108 109 raw_spin_lock_irqsave(&l2_access_lock, flags); 110 write_sysreg_s(reg, L2CPUSRSELR_EL1); 111 isb(); 112 write_sysreg_s(val, L2CPUSRDR_EL1); 113 isb(); 114 raw_spin_unlock_irqrestore(&l2_access_lock, flags); 115 } 116 117 /** 118 * get_l2_indirect_reg: read an L2 register value 119 * @reg: Address of L2 register. 120 * 121 * Use architecturally required barriers for ordering between system register 122 * accesses 123 */ 124 static u64 get_l2_indirect_reg(u64 reg) 125 { 126 u64 val; 127 unsigned long flags; 128 129 raw_spin_lock_irqsave(&l2_access_lock, flags); 130 write_sysreg_s(reg, L2CPUSRSELR_EL1); 131 isb(); 132 val = read_sysreg_s(L2CPUSRDR_EL1); 133 raw_spin_unlock_irqrestore(&l2_access_lock, flags); 134 135 return val; 136 } 137 138 struct cluster_pmu; 139 140 /* 141 * Aggregate PMU. Implements the core pmu functions and manages 142 * the hardware PMUs. 143 */ 144 struct l2cache_pmu { 145 struct hlist_node node; 146 u32 num_pmus; 147 struct pmu pmu; 148 int num_counters; 149 cpumask_t cpumask; 150 struct platform_device *pdev; 151 struct cluster_pmu * __percpu *pmu_cluster; 152 struct list_head clusters; 153 }; 154 155 /* 156 * The cache is made up of one or more clusters, each cluster has its own PMU. 157 * Each cluster is associated with one or more CPUs. 158 * This structure represents one of the hardware PMUs. 159 * 160 * Events can be envisioned as a 2-dimensional array. Each column represents 161 * a group of events. There are 8 groups. Only one entry from each 162 * group can be in use at a time. 163 * 164 * Events are specified as 0xCCG, where CC is 2 hex digits specifying 165 * the code (array row) and G specifies the group (column). 166 * 167 * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE 168 * which is outside the above scheme. 169 */ 170 struct cluster_pmu { 171 struct list_head next; 172 struct perf_event *events[MAX_L2_CTRS]; 173 struct l2cache_pmu *l2cache_pmu; 174 DECLARE_BITMAP(used_counters, MAX_L2_CTRS); 175 DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1); 176 int irq; 177 int cluster_id; 178 /* The CPU that is used for collecting events on this cluster */ 179 int on_cpu; 180 /* All the CPUs associated with this cluster */ 181 cpumask_t cluster_cpus; 182 spinlock_t pmu_lock; 183 }; 184 185 #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu)) 186 187 static u32 l2_cycle_ctr_idx; 188 static u32 l2_counter_present_mask; 189 190 static inline u32 idx_to_reg_bit(u32 idx) 191 { 192 if (idx == l2_cycle_ctr_idx) 193 return BIT(L2CYCLE_CTR_BIT); 194 195 return BIT(idx); 196 } 197 198 static inline struct cluster_pmu *get_cluster_pmu( 199 struct l2cache_pmu *l2cache_pmu, int cpu) 200 { 201 return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu); 202 } 203 204 static void cluster_pmu_reset(void) 205 { 206 /* Reset all counters */ 207 set_l2_indirect_reg(L2PMCR, L2PMCR_RESET_ALL); 208 set_l2_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask); 209 set_l2_indirect_reg(L2PMINTENCLR, l2_counter_present_mask); 210 set_l2_indirect_reg(L2PMOVSCLR, l2_counter_present_mask); 211 } 212 213 static inline void cluster_pmu_enable(void) 214 { 215 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE); 216 } 217 218 static inline void cluster_pmu_disable(void) 219 { 220 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE); 221 } 222 223 static inline void cluster_pmu_counter_set_value(u32 idx, u64 value) 224 { 225 if (idx == l2_cycle_ctr_idx) 226 set_l2_indirect_reg(L2PMCCNTR, value); 227 else 228 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value); 229 } 230 231 static inline u64 cluster_pmu_counter_get_value(u32 idx) 232 { 233 u64 value; 234 235 if (idx == l2_cycle_ctr_idx) 236 value = get_l2_indirect_reg(L2PMCCNTR); 237 else 238 value = get_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx)); 239 240 return value; 241 } 242 243 static inline void cluster_pmu_counter_enable(u32 idx) 244 { 245 set_l2_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx)); 246 } 247 248 static inline void cluster_pmu_counter_disable(u32 idx) 249 { 250 set_l2_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx)); 251 } 252 253 static inline void cluster_pmu_counter_enable_interrupt(u32 idx) 254 { 255 set_l2_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx)); 256 } 257 258 static inline void cluster_pmu_counter_disable_interrupt(u32 idx) 259 { 260 set_l2_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx)); 261 } 262 263 static inline void cluster_pmu_set_evccntcr(u32 val) 264 { 265 set_l2_indirect_reg(L2PMCCNTCR, val); 266 } 267 268 static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val) 269 { 270 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val); 271 } 272 273 static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val) 274 { 275 set_l2_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val); 276 } 277 278 static void cluster_pmu_set_resr(struct cluster_pmu *cluster, 279 u32 event_group, u32 event_cc) 280 { 281 u64 field; 282 u64 resr_val; 283 u32 shift; 284 unsigned long flags; 285 286 shift = L2PMRESR_GROUP_BITS * event_group; 287 field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift); 288 289 spin_lock_irqsave(&cluster->pmu_lock, flags); 290 291 resr_val = get_l2_indirect_reg(L2PMRESR); 292 resr_val &= ~(L2PMRESR_GROUP_MASK << shift); 293 resr_val |= field; 294 resr_val |= L2PMRESR_EN; 295 set_l2_indirect_reg(L2PMRESR, resr_val); 296 297 spin_unlock_irqrestore(&cluster->pmu_lock, flags); 298 } 299 300 /* 301 * Hardware allows filtering of events based on the originating 302 * CPU. Turn this off by setting filter bits to allow events from 303 * all CPUS, subunits and ID independent events in this cluster. 304 */ 305 static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr) 306 { 307 u32 val = L2PMXEVFILTER_SUFILTER_ALL | 308 L2PMXEVFILTER_ORGFILTER_IDINDEP | 309 L2PMXEVFILTER_ORGFILTER_ALL; 310 311 set_l2_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val); 312 } 313 314 static inline u32 cluster_pmu_getreset_ovsr(void) 315 { 316 u32 result = get_l2_indirect_reg(L2PMOVSSET); 317 318 set_l2_indirect_reg(L2PMOVSCLR, result); 319 return result; 320 } 321 322 static inline bool cluster_pmu_has_overflowed(u32 ovsr) 323 { 324 return !!(ovsr & l2_counter_present_mask); 325 } 326 327 static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx) 328 { 329 return !!(ovsr & idx_to_reg_bit(idx)); 330 } 331 332 static void l2_cache_event_update(struct perf_event *event) 333 { 334 struct hw_perf_event *hwc = &event->hw; 335 u64 delta, prev, now; 336 u32 idx = hwc->idx; 337 338 do { 339 prev = local64_read(&hwc->prev_count); 340 now = cluster_pmu_counter_get_value(idx); 341 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 342 343 /* 344 * The cycle counter is 64-bit, but all other counters are 345 * 32-bit, and we must handle 32-bit overflow explicitly. 346 */ 347 delta = now - prev; 348 if (idx != l2_cycle_ctr_idx) 349 delta &= 0xffffffff; 350 351 local64_add(delta, &event->count); 352 } 353 354 static void l2_cache_cluster_set_period(struct cluster_pmu *cluster, 355 struct hw_perf_event *hwc) 356 { 357 u32 idx = hwc->idx; 358 u64 new; 359 360 /* 361 * We limit the max period to half the max counter value so 362 * that even in the case of extreme interrupt latency the 363 * counter will (hopefully) not wrap past its initial value. 364 */ 365 if (idx == l2_cycle_ctr_idx) 366 new = L2_CYCLE_COUNTER_RELOAD; 367 else 368 new = L2_COUNTER_RELOAD; 369 370 local64_set(&hwc->prev_count, new); 371 cluster_pmu_counter_set_value(idx, new); 372 } 373 374 static int l2_cache_get_event_idx(struct cluster_pmu *cluster, 375 struct perf_event *event) 376 { 377 struct hw_perf_event *hwc = &event->hw; 378 int idx; 379 int num_ctrs = cluster->l2cache_pmu->num_counters - 1; 380 unsigned int group; 381 382 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { 383 if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters)) 384 return -EAGAIN; 385 386 return l2_cycle_ctr_idx; 387 } 388 389 idx = find_first_zero_bit(cluster->used_counters, num_ctrs); 390 if (idx == num_ctrs) 391 /* The counters are all in use. */ 392 return -EAGAIN; 393 394 /* 395 * Check for column exclusion: event column already in use by another 396 * event. This is for events which are not in the same group. 397 * Conflicting events in the same group are detected in event_init. 398 */ 399 group = L2_EVT_GROUP(hwc->config_base); 400 if (test_bit(group, cluster->used_groups)) 401 return -EAGAIN; 402 403 set_bit(idx, cluster->used_counters); 404 set_bit(group, cluster->used_groups); 405 406 return idx; 407 } 408 409 static void l2_cache_clear_event_idx(struct cluster_pmu *cluster, 410 struct perf_event *event) 411 { 412 struct hw_perf_event *hwc = &event->hw; 413 int idx = hwc->idx; 414 415 clear_bit(idx, cluster->used_counters); 416 if (hwc->config_base != L2CYCLE_CTR_RAW_CODE) 417 clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups); 418 } 419 420 static irqreturn_t l2_cache_handle_irq(int irq_num, void *data) 421 { 422 struct cluster_pmu *cluster = data; 423 int num_counters = cluster->l2cache_pmu->num_counters; 424 u32 ovsr; 425 int idx; 426 427 ovsr = cluster_pmu_getreset_ovsr(); 428 if (!cluster_pmu_has_overflowed(ovsr)) 429 return IRQ_NONE; 430 431 for_each_set_bit(idx, cluster->used_counters, num_counters) { 432 struct perf_event *event = cluster->events[idx]; 433 struct hw_perf_event *hwc; 434 435 if (WARN_ON_ONCE(!event)) 436 continue; 437 438 if (!cluster_pmu_counter_has_overflowed(ovsr, idx)) 439 continue; 440 441 l2_cache_event_update(event); 442 hwc = &event->hw; 443 444 l2_cache_cluster_set_period(cluster, hwc); 445 } 446 447 return IRQ_HANDLED; 448 } 449 450 /* 451 * Implementation of abstract pmu functionality required by 452 * the core perf events code. 453 */ 454 455 static void l2_cache_pmu_enable(struct pmu *pmu) 456 { 457 /* 458 * Although there is only one PMU (per socket) controlling multiple 459 * physical PMUs (per cluster), because we do not support per-task mode 460 * each event is associated with a CPU. Each event has pmu_enable 461 * called on its CPU, so here it is only necessary to enable the 462 * counters for the current CPU. 463 */ 464 465 cluster_pmu_enable(); 466 } 467 468 static void l2_cache_pmu_disable(struct pmu *pmu) 469 { 470 cluster_pmu_disable(); 471 } 472 473 static int l2_cache_event_init(struct perf_event *event) 474 { 475 struct hw_perf_event *hwc = &event->hw; 476 struct cluster_pmu *cluster; 477 struct perf_event *sibling; 478 struct l2cache_pmu *l2cache_pmu; 479 480 if (event->attr.type != event->pmu->type) 481 return -ENOENT; 482 483 l2cache_pmu = to_l2cache_pmu(event->pmu); 484 485 if (hwc->sample_period) { 486 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 487 "Sampling not supported\n"); 488 return -EOPNOTSUPP; 489 } 490 491 if (event->cpu < 0) { 492 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 493 "Per-task mode not supported\n"); 494 return -EOPNOTSUPP; 495 } 496 497 /* We cannot filter accurately so we just don't allow it. */ 498 if (event->attr.exclude_user || event->attr.exclude_kernel || 499 event->attr.exclude_hv || event->attr.exclude_idle) { 500 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 501 "Can't exclude execution levels\n"); 502 return -EOPNOTSUPP; 503 } 504 505 if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) || 506 ((event->attr.config & ~L2_EVT_MASK) != 0)) && 507 (event->attr.config != L2CYCLE_CTR_RAW_CODE)) { 508 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 509 "Invalid config %llx\n", 510 event->attr.config); 511 return -EINVAL; 512 } 513 514 /* Don't allow groups with mixed PMUs, except for s/w events */ 515 if (event->group_leader->pmu != event->pmu && 516 !is_software_event(event->group_leader)) { 517 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 518 "Can't create mixed PMU group\n"); 519 return -EINVAL; 520 } 521 522 list_for_each_entry(sibling, &event->group_leader->sibling_list, 523 group_entry) 524 if (sibling->pmu != event->pmu && 525 !is_software_event(sibling)) { 526 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 527 "Can't create mixed PMU group\n"); 528 return -EINVAL; 529 } 530 531 cluster = get_cluster_pmu(l2cache_pmu, event->cpu); 532 if (!cluster) { 533 /* CPU has not been initialised */ 534 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 535 "CPU%d not associated with L2 cluster\n", event->cpu); 536 return -EINVAL; 537 } 538 539 /* Ensure all events in a group are on the same cpu */ 540 if ((event->group_leader != event) && 541 (cluster->on_cpu != event->group_leader->cpu)) { 542 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 543 "Can't create group on CPUs %d and %d", 544 event->cpu, event->group_leader->cpu); 545 return -EINVAL; 546 } 547 548 if ((event != event->group_leader) && 549 !is_software_event(event->group_leader) && 550 (L2_EVT_GROUP(event->group_leader->attr.config) == 551 L2_EVT_GROUP(event->attr.config))) { 552 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 553 "Column exclusion: conflicting events %llx %llx\n", 554 event->group_leader->attr.config, 555 event->attr.config); 556 return -EINVAL; 557 } 558 559 list_for_each_entry(sibling, &event->group_leader->sibling_list, 560 group_entry) { 561 if ((sibling != event) && 562 !is_software_event(sibling) && 563 (L2_EVT_GROUP(sibling->attr.config) == 564 L2_EVT_GROUP(event->attr.config))) { 565 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 566 "Column exclusion: conflicting events %llx %llx\n", 567 sibling->attr.config, 568 event->attr.config); 569 return -EINVAL; 570 } 571 } 572 573 hwc->idx = -1; 574 hwc->config_base = event->attr.config; 575 576 /* 577 * Ensure all events are on the same cpu so all events are in the 578 * same cpu context, to avoid races on pmu_enable etc. 579 */ 580 event->cpu = cluster->on_cpu; 581 582 return 0; 583 } 584 585 static void l2_cache_event_start(struct perf_event *event, int flags) 586 { 587 struct cluster_pmu *cluster; 588 struct hw_perf_event *hwc = &event->hw; 589 int idx = hwc->idx; 590 u32 config; 591 u32 event_cc, event_group; 592 593 hwc->state = 0; 594 595 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 596 597 l2_cache_cluster_set_period(cluster, hwc); 598 599 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { 600 cluster_pmu_set_evccntcr(0); 601 } else { 602 config = hwc->config_base; 603 event_cc = L2_EVT_CODE(config); 604 event_group = L2_EVT_GROUP(config); 605 606 cluster_pmu_set_evcntcr(idx, 0); 607 cluster_pmu_set_evtyper(idx, event_group); 608 cluster_pmu_set_resr(cluster, event_group, event_cc); 609 cluster_pmu_set_evfilter_sys_mode(idx); 610 } 611 612 cluster_pmu_counter_enable_interrupt(idx); 613 cluster_pmu_counter_enable(idx); 614 } 615 616 static void l2_cache_event_stop(struct perf_event *event, int flags) 617 { 618 struct hw_perf_event *hwc = &event->hw; 619 int idx = hwc->idx; 620 621 if (hwc->state & PERF_HES_STOPPED) 622 return; 623 624 cluster_pmu_counter_disable_interrupt(idx); 625 cluster_pmu_counter_disable(idx); 626 627 if (flags & PERF_EF_UPDATE) 628 l2_cache_event_update(event); 629 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 630 } 631 632 static int l2_cache_event_add(struct perf_event *event, int flags) 633 { 634 struct hw_perf_event *hwc = &event->hw; 635 int idx; 636 int err = 0; 637 struct cluster_pmu *cluster; 638 639 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 640 641 idx = l2_cache_get_event_idx(cluster, event); 642 if (idx < 0) 643 return idx; 644 645 hwc->idx = idx; 646 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 647 cluster->events[idx] = event; 648 local64_set(&hwc->prev_count, 0); 649 650 if (flags & PERF_EF_START) 651 l2_cache_event_start(event, flags); 652 653 /* Propagate changes to the userspace mapping. */ 654 perf_event_update_userpage(event); 655 656 return err; 657 } 658 659 static void l2_cache_event_del(struct perf_event *event, int flags) 660 { 661 struct hw_perf_event *hwc = &event->hw; 662 struct cluster_pmu *cluster; 663 int idx = hwc->idx; 664 665 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 666 667 l2_cache_event_stop(event, flags | PERF_EF_UPDATE); 668 cluster->events[idx] = NULL; 669 l2_cache_clear_event_idx(cluster, event); 670 671 perf_event_update_userpage(event); 672 } 673 674 static void l2_cache_event_read(struct perf_event *event) 675 { 676 l2_cache_event_update(event); 677 } 678 679 static ssize_t l2_cache_pmu_cpumask_show(struct device *dev, 680 struct device_attribute *attr, 681 char *buf) 682 { 683 struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev)); 684 685 return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask); 686 } 687 688 static struct device_attribute l2_cache_pmu_cpumask_attr = 689 __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL); 690 691 static struct attribute *l2_cache_pmu_cpumask_attrs[] = { 692 &l2_cache_pmu_cpumask_attr.attr, 693 NULL, 694 }; 695 696 static struct attribute_group l2_cache_pmu_cpumask_group = { 697 .attrs = l2_cache_pmu_cpumask_attrs, 698 }; 699 700 /* CCG format for perf RAW codes. */ 701 PMU_FORMAT_ATTR(l2_code, "config:4-11"); 702 PMU_FORMAT_ATTR(l2_group, "config:0-3"); 703 static struct attribute *l2_cache_pmu_formats[] = { 704 &format_attr_l2_code.attr, 705 &format_attr_l2_group.attr, 706 NULL, 707 }; 708 709 static struct attribute_group l2_cache_pmu_format_group = { 710 .name = "format", 711 .attrs = l2_cache_pmu_formats, 712 }; 713 714 static const struct attribute_group *l2_cache_pmu_attr_grps[] = { 715 &l2_cache_pmu_format_group, 716 &l2_cache_pmu_cpumask_group, 717 NULL, 718 }; 719 720 /* 721 * Generic device handlers 722 */ 723 724 static const struct acpi_device_id l2_cache_pmu_acpi_match[] = { 725 { "QCOM8130", }, 726 { } 727 }; 728 729 static int get_num_counters(void) 730 { 731 int val; 732 733 val = get_l2_indirect_reg(L2PMCR); 734 735 /* 736 * Read number of counters from L2PMCR and add 1 737 * for the cycle counter. 738 */ 739 return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1; 740 } 741 742 static struct cluster_pmu *l2_cache_associate_cpu_with_cluster( 743 struct l2cache_pmu *l2cache_pmu, int cpu) 744 { 745 u64 mpidr; 746 int cpu_cluster_id; 747 struct cluster_pmu *cluster = NULL; 748 749 /* 750 * This assumes that the cluster_id is in MPIDR[aff1] for 751 * single-threaded cores, and MPIDR[aff2] for multi-threaded 752 * cores. This logic will have to be updated if this changes. 753 */ 754 mpidr = read_cpuid_mpidr(); 755 if (mpidr & MPIDR_MT_BITMASK) 756 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2); 757 else 758 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); 759 760 list_for_each_entry(cluster, &l2cache_pmu->clusters, next) { 761 if (cluster->cluster_id != cpu_cluster_id) 762 continue; 763 764 dev_info(&l2cache_pmu->pdev->dev, 765 "CPU%d associated with cluster %d\n", cpu, 766 cluster->cluster_id); 767 cpumask_set_cpu(cpu, &cluster->cluster_cpus); 768 *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster; 769 break; 770 } 771 772 return cluster; 773 } 774 775 static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 776 { 777 struct cluster_pmu *cluster; 778 struct l2cache_pmu *l2cache_pmu; 779 780 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); 781 cluster = get_cluster_pmu(l2cache_pmu, cpu); 782 if (!cluster) { 783 /* First time this CPU has come online */ 784 cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu); 785 if (!cluster) { 786 /* Only if broken firmware doesn't list every cluster */ 787 WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu); 788 return 0; 789 } 790 } 791 792 /* If another CPU is managing this cluster, we're done */ 793 if (cluster->on_cpu != -1) 794 return 0; 795 796 /* 797 * All CPUs on this cluster were down, use this one. 798 * Reset to put it into sane state. 799 */ 800 cluster->on_cpu = cpu; 801 cpumask_set_cpu(cpu, &l2cache_pmu->cpumask); 802 cluster_pmu_reset(); 803 804 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu))); 805 enable_irq(cluster->irq); 806 807 return 0; 808 } 809 810 static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 811 { 812 struct cluster_pmu *cluster; 813 struct l2cache_pmu *l2cache_pmu; 814 cpumask_t cluster_online_cpus; 815 unsigned int target; 816 817 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); 818 cluster = get_cluster_pmu(l2cache_pmu, cpu); 819 if (!cluster) 820 return 0; 821 822 /* If this CPU is not managing the cluster, we're done */ 823 if (cluster->on_cpu != cpu) 824 return 0; 825 826 /* Give up ownership of cluster */ 827 cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask); 828 cluster->on_cpu = -1; 829 830 /* Any other CPU for this cluster which is still online */ 831 cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus, 832 cpu_online_mask); 833 target = cpumask_any_but(&cluster_online_cpus, cpu); 834 if (target >= nr_cpu_ids) { 835 disable_irq(cluster->irq); 836 return 0; 837 } 838 839 perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target); 840 cluster->on_cpu = target; 841 cpumask_set_cpu(target, &l2cache_pmu->cpumask); 842 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target))); 843 844 return 0; 845 } 846 847 static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) 848 { 849 struct platform_device *pdev = to_platform_device(dev->parent); 850 struct platform_device *sdev = to_platform_device(dev); 851 struct l2cache_pmu *l2cache_pmu = data; 852 struct cluster_pmu *cluster; 853 struct acpi_device *device; 854 unsigned long fw_cluster_id; 855 int err; 856 int irq; 857 858 if (acpi_bus_get_device(ACPI_HANDLE(dev), &device)) 859 return -ENODEV; 860 861 if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) { 862 dev_err(&pdev->dev, "unable to read ACPI uid\n"); 863 return -ENODEV; 864 } 865 866 cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL); 867 if (!cluster) 868 return -ENOMEM; 869 870 INIT_LIST_HEAD(&cluster->next); 871 list_add(&cluster->next, &l2cache_pmu->clusters); 872 cluster->cluster_id = fw_cluster_id; 873 874 irq = platform_get_irq(sdev, 0); 875 if (irq < 0) { 876 dev_err(&pdev->dev, 877 "Failed to get valid irq for cluster %ld\n", 878 fw_cluster_id); 879 return irq; 880 } 881 irq_set_status_flags(irq, IRQ_NOAUTOEN); 882 cluster->irq = irq; 883 884 cluster->l2cache_pmu = l2cache_pmu; 885 cluster->on_cpu = -1; 886 887 err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq, 888 IRQF_NOBALANCING | IRQF_NO_THREAD, 889 "l2-cache-pmu", cluster); 890 if (err) { 891 dev_err(&pdev->dev, 892 "Unable to request IRQ%d for L2 PMU counters\n", irq); 893 return err; 894 } 895 896 dev_info(&pdev->dev, 897 "Registered L2 cache PMU cluster %ld\n", fw_cluster_id); 898 899 spin_lock_init(&cluster->pmu_lock); 900 901 l2cache_pmu->num_pmus++; 902 903 return 0; 904 } 905 906 static int l2_cache_pmu_probe(struct platform_device *pdev) 907 { 908 int err; 909 struct l2cache_pmu *l2cache_pmu; 910 911 l2cache_pmu = 912 devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL); 913 if (!l2cache_pmu) 914 return -ENOMEM; 915 916 INIT_LIST_HEAD(&l2cache_pmu->clusters); 917 918 platform_set_drvdata(pdev, l2cache_pmu); 919 l2cache_pmu->pmu = (struct pmu) { 920 /* suffix is instance id for future use with multiple sockets */ 921 .name = "l2cache_0", 922 .task_ctx_nr = perf_invalid_context, 923 .pmu_enable = l2_cache_pmu_enable, 924 .pmu_disable = l2_cache_pmu_disable, 925 .event_init = l2_cache_event_init, 926 .add = l2_cache_event_add, 927 .del = l2_cache_event_del, 928 .start = l2_cache_event_start, 929 .stop = l2_cache_event_stop, 930 .read = l2_cache_event_read, 931 .attr_groups = l2_cache_pmu_attr_grps, 932 }; 933 934 l2cache_pmu->num_counters = get_num_counters(); 935 l2cache_pmu->pdev = pdev; 936 l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev, 937 struct cluster_pmu *); 938 if (!l2cache_pmu->pmu_cluster) 939 return -ENOMEM; 940 941 l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1; 942 l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) | 943 BIT(L2CYCLE_CTR_BIT); 944 945 cpumask_clear(&l2cache_pmu->cpumask); 946 947 /* Read cluster info and initialize each cluster */ 948 err = device_for_each_child(&pdev->dev, l2cache_pmu, 949 l2_cache_pmu_probe_cluster); 950 if (err) 951 return err; 952 953 if (l2cache_pmu->num_pmus == 0) { 954 dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n"); 955 return -ENODEV; 956 } 957 958 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 959 &l2cache_pmu->node); 960 if (err) { 961 dev_err(&pdev->dev, "Error %d registering hotplug", err); 962 return err; 963 } 964 965 err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1); 966 if (err) { 967 dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err); 968 goto out_unregister; 969 } 970 971 dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n", 972 l2cache_pmu->num_pmus); 973 974 return err; 975 976 out_unregister: 977 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 978 &l2cache_pmu->node); 979 return err; 980 } 981 982 static int l2_cache_pmu_remove(struct platform_device *pdev) 983 { 984 struct l2cache_pmu *l2cache_pmu = 985 to_l2cache_pmu(platform_get_drvdata(pdev)); 986 987 perf_pmu_unregister(&l2cache_pmu->pmu); 988 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 989 &l2cache_pmu->node); 990 return 0; 991 } 992 993 static struct platform_driver l2_cache_pmu_driver = { 994 .driver = { 995 .name = "qcom-l2cache-pmu", 996 .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match), 997 }, 998 .probe = l2_cache_pmu_probe, 999 .remove = l2_cache_pmu_remove, 1000 }; 1001 1002 static int __init register_l2_cache_pmu_driver(void) 1003 { 1004 int err; 1005 1006 err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 1007 "AP_PERF_ARM_QCOM_L2_ONLINE", 1008 l2cache_pmu_online_cpu, 1009 l2cache_pmu_offline_cpu); 1010 if (err) 1011 return err; 1012 1013 return platform_driver_register(&l2_cache_pmu_driver); 1014 } 1015 device_initcall(register_l2_cache_pmu_driver); 1016