1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Performance event support - Freescale Embedded Performance Monitor 4 * 5 * Copyright 2008-2009 Paul Mackerras, IBM Corporation. 6 * Copyright 2010 Freescale Semiconductor, Inc. 7 */ 8 #include <linux/kernel.h> 9 #include <linux/sched.h> 10 #include <linux/perf_event.h> 11 #include <linux/percpu.h> 12 #include <linux/hardirq.h> 13 #include <asm/reg_fsl_emb.h> 14 #include <asm/pmc.h> 15 #include <asm/machdep.h> 16 #include <asm/firmware.h> 17 #include <asm/ptrace.h> 18 19 struct cpu_hw_events { 20 int n_events; 21 int disabled; 22 u8 pmcs_enabled; 23 struct perf_event *event[MAX_HWEVENTS]; 24 }; 25 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); 26 27 static struct fsl_emb_pmu *ppmu; 28 29 /* Number of perf_events counting hardware events */ 30 static atomic_t num_events; 31 /* Used to avoid races in calling reserve/release_pmc_hardware */ 32 static DEFINE_MUTEX(pmc_reserve_mutex); 33 34 static void perf_event_interrupt(struct pt_regs *regs); 35 36 /* 37 * Read one performance monitor counter (PMC). 38 */ 39 static unsigned long read_pmc(int idx) 40 { 41 unsigned long val; 42 43 switch (idx) { 44 case 0: 45 val = mfpmr(PMRN_PMC0); 46 break; 47 case 1: 48 val = mfpmr(PMRN_PMC1); 49 break; 50 case 2: 51 val = mfpmr(PMRN_PMC2); 52 break; 53 case 3: 54 val = mfpmr(PMRN_PMC3); 55 break; 56 case 4: 57 val = mfpmr(PMRN_PMC4); 58 break; 59 case 5: 60 val = mfpmr(PMRN_PMC5); 61 break; 62 default: 63 printk(KERN_ERR "oops trying to read PMC%d\n", idx); 64 val = 0; 65 } 66 return val; 67 } 68 69 /* 70 * Write one PMC. 71 */ 72 static void write_pmc(int idx, unsigned long val) 73 { 74 switch (idx) { 75 case 0: 76 mtpmr(PMRN_PMC0, val); 77 break; 78 case 1: 79 mtpmr(PMRN_PMC1, val); 80 break; 81 case 2: 82 mtpmr(PMRN_PMC2, val); 83 break; 84 case 3: 85 mtpmr(PMRN_PMC3, val); 86 break; 87 case 4: 88 mtpmr(PMRN_PMC4, val); 89 break; 90 case 5: 91 mtpmr(PMRN_PMC5, val); 92 break; 93 default: 94 printk(KERN_ERR "oops trying to write PMC%d\n", idx); 95 } 96 97 isync(); 98 } 99 100 /* 101 * Write one local control A register 102 */ 103 static void write_pmlca(int idx, unsigned long val) 104 { 105 switch (idx) { 106 case 0: 107 mtpmr(PMRN_PMLCA0, val); 108 break; 109 case 1: 110 mtpmr(PMRN_PMLCA1, val); 111 break; 112 case 2: 113 mtpmr(PMRN_PMLCA2, val); 114 break; 115 case 3: 116 mtpmr(PMRN_PMLCA3, val); 117 break; 118 case 4: 119 mtpmr(PMRN_PMLCA4, val); 120 break; 121 case 5: 122 mtpmr(PMRN_PMLCA5, val); 123 break; 124 default: 125 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx); 126 } 127 128 isync(); 129 } 130 131 /* 132 * Write one local control B register 133 */ 134 static void write_pmlcb(int idx, unsigned long val) 135 { 136 switch (idx) { 137 case 0: 138 mtpmr(PMRN_PMLCB0, val); 139 break; 140 case 1: 141 mtpmr(PMRN_PMLCB1, val); 142 break; 143 case 2: 144 mtpmr(PMRN_PMLCB2, val); 145 break; 146 case 3: 147 mtpmr(PMRN_PMLCB3, val); 148 break; 149 case 4: 150 mtpmr(PMRN_PMLCB4, val); 151 break; 152 case 5: 153 mtpmr(PMRN_PMLCB5, val); 154 break; 155 default: 156 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx); 157 } 158 159 isync(); 160 } 161 162 static void fsl_emb_pmu_read(struct perf_event *event) 163 { 164 s64 val, delta, prev; 165 166 if (event->hw.state & PERF_HES_STOPPED) 167 return; 168 169 /* 170 * Performance monitor interrupts come even when interrupts 171 * are soft-disabled, as long as interrupts are hard-enabled. 172 * Therefore we treat them like NMIs. 173 */ 174 do { 175 prev = local64_read(&event->hw.prev_count); 176 barrier(); 177 val = read_pmc(event->hw.idx); 178 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev); 179 180 /* The counters are only 32 bits wide */ 181 delta = (val - prev) & 0xfffffffful; 182 local64_add(delta, &event->count); 183 local64_sub(delta, &event->hw.period_left); 184 } 185 186 /* 187 * Disable all events to prevent PMU interrupts and to allow 188 * events to be added or removed. 189 */ 190 static void fsl_emb_pmu_disable(struct pmu *pmu) 191 { 192 struct cpu_hw_events *cpuhw; 193 unsigned long flags; 194 195 local_irq_save(flags); 196 cpuhw = this_cpu_ptr(&cpu_hw_events); 197 198 if (!cpuhw->disabled) { 199 cpuhw->disabled = 1; 200 201 /* 202 * Check if we ever enabled the PMU on this cpu. 203 */ 204 if (!cpuhw->pmcs_enabled) { 205 ppc_enable_pmcs(); 206 cpuhw->pmcs_enabled = 1; 207 } 208 209 if (atomic_read(&num_events)) { 210 /* 211 * Set the 'freeze all counters' bit, and disable 212 * interrupts. The barrier is to make sure the 213 * mtpmr has been executed and the PMU has frozen 214 * the events before we return. 215 */ 216 217 mtpmr(PMRN_PMGC0, PMGC0_FAC); 218 isync(); 219 } 220 } 221 local_irq_restore(flags); 222 } 223 224 /* 225 * Re-enable all events if disable == 0. 226 * If we were previously disabled and events were added, then 227 * put the new config on the PMU. 228 */ 229 static void fsl_emb_pmu_enable(struct pmu *pmu) 230 { 231 struct cpu_hw_events *cpuhw; 232 unsigned long flags; 233 234 local_irq_save(flags); 235 cpuhw = this_cpu_ptr(&cpu_hw_events); 236 if (!cpuhw->disabled) 237 goto out; 238 239 cpuhw->disabled = 0; 240 ppc_set_pmu_inuse(cpuhw->n_events != 0); 241 242 if (cpuhw->n_events > 0) { 243 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); 244 isync(); 245 } 246 247 out: 248 local_irq_restore(flags); 249 } 250 251 static int collect_events(struct perf_event *group, int max_count, 252 struct perf_event *ctrs[]) 253 { 254 int n = 0; 255 struct perf_event *event; 256 257 if (!is_software_event(group)) { 258 if (n >= max_count) 259 return -1; 260 ctrs[n] = group; 261 n++; 262 } 263 for_each_sibling_event(event, group) { 264 if (!is_software_event(event) && 265 event->state != PERF_EVENT_STATE_OFF) { 266 if (n >= max_count) 267 return -1; 268 ctrs[n] = event; 269 n++; 270 } 271 } 272 return n; 273 } 274 275 /* context locked on entry */ 276 static int fsl_emb_pmu_add(struct perf_event *event, int flags) 277 { 278 struct cpu_hw_events *cpuhw; 279 int ret = -EAGAIN; 280 int num_counters = ppmu->n_counter; 281 u64 val; 282 int i; 283 284 perf_pmu_disable(event->pmu); 285 cpuhw = &get_cpu_var(cpu_hw_events); 286 287 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) 288 num_counters = ppmu->n_restricted; 289 290 /* 291 * Allocate counters from top-down, so that restricted-capable 292 * counters are kept free as long as possible. 293 */ 294 for (i = num_counters - 1; i >= 0; i--) { 295 if (cpuhw->event[i]) 296 continue; 297 298 break; 299 } 300 301 if (i < 0) 302 goto out; 303 304 event->hw.idx = i; 305 cpuhw->event[i] = event; 306 ++cpuhw->n_events; 307 308 val = 0; 309 if (event->hw.sample_period) { 310 s64 left = local64_read(&event->hw.period_left); 311 if (left < 0x80000000L) 312 val = 0x80000000L - left; 313 } 314 local64_set(&event->hw.prev_count, val); 315 316 if (unlikely(!(flags & PERF_EF_START))) { 317 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 318 val = 0; 319 } else { 320 event->hw.state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE); 321 } 322 323 write_pmc(i, val); 324 perf_event_update_userpage(event); 325 326 write_pmlcb(i, event->hw.config >> 32); 327 write_pmlca(i, event->hw.config_base); 328 329 ret = 0; 330 out: 331 put_cpu_var(cpu_hw_events); 332 perf_pmu_enable(event->pmu); 333 return ret; 334 } 335 336 /* context locked on entry */ 337 static void fsl_emb_pmu_del(struct perf_event *event, int flags) 338 { 339 struct cpu_hw_events *cpuhw; 340 int i = event->hw.idx; 341 342 perf_pmu_disable(event->pmu); 343 if (i < 0) 344 goto out; 345 346 fsl_emb_pmu_read(event); 347 348 cpuhw = &get_cpu_var(cpu_hw_events); 349 350 WARN_ON(event != cpuhw->event[event->hw.idx]); 351 352 write_pmlca(i, 0); 353 write_pmlcb(i, 0); 354 write_pmc(i, 0); 355 356 cpuhw->event[i] = NULL; 357 event->hw.idx = -1; 358 359 /* 360 * TODO: if at least one restricted event exists, and we 361 * just freed up a non-restricted-capable counter, and 362 * there is a restricted-capable counter occupied by 363 * a non-restricted event, migrate that event to the 364 * vacated counter. 365 */ 366 367 cpuhw->n_events--; 368 369 out: 370 perf_pmu_enable(event->pmu); 371 put_cpu_var(cpu_hw_events); 372 } 373 374 static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags) 375 { 376 unsigned long flags; 377 unsigned long val; 378 s64 left; 379 380 if (event->hw.idx < 0 || !event->hw.sample_period) 381 return; 382 383 if (!(event->hw.state & PERF_HES_STOPPED)) 384 return; 385 386 if (ef_flags & PERF_EF_RELOAD) 387 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 388 389 local_irq_save(flags); 390 perf_pmu_disable(event->pmu); 391 392 event->hw.state = 0; 393 left = local64_read(&event->hw.period_left); 394 val = 0; 395 if (left < 0x80000000L) 396 val = 0x80000000L - left; 397 write_pmc(event->hw.idx, val); 398 399 perf_event_update_userpage(event); 400 perf_pmu_enable(event->pmu); 401 local_irq_restore(flags); 402 } 403 404 static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags) 405 { 406 unsigned long flags; 407 408 if (event->hw.idx < 0 || !event->hw.sample_period) 409 return; 410 411 if (event->hw.state & PERF_HES_STOPPED) 412 return; 413 414 local_irq_save(flags); 415 perf_pmu_disable(event->pmu); 416 417 fsl_emb_pmu_read(event); 418 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 419 write_pmc(event->hw.idx, 0); 420 421 perf_event_update_userpage(event); 422 perf_pmu_enable(event->pmu); 423 local_irq_restore(flags); 424 } 425 426 /* 427 * Release the PMU if this is the last perf_event. 428 */ 429 static void hw_perf_event_destroy(struct perf_event *event) 430 { 431 if (!atomic_add_unless(&num_events, -1, 1)) { 432 mutex_lock(&pmc_reserve_mutex); 433 if (atomic_dec_return(&num_events) == 0) 434 release_pmc_hardware(); 435 mutex_unlock(&pmc_reserve_mutex); 436 } 437 } 438 439 /* 440 * Translate a generic cache event_id config to a raw event_id code. 441 */ 442 static int hw_perf_cache_event(u64 config, u64 *eventp) 443 { 444 unsigned long type, op, result; 445 int ev; 446 447 if (!ppmu->cache_events) 448 return -EINVAL; 449 450 /* unpack config */ 451 type = config & 0xff; 452 op = (config >> 8) & 0xff; 453 result = (config >> 16) & 0xff; 454 455 if (type >= PERF_COUNT_HW_CACHE_MAX || 456 op >= PERF_COUNT_HW_CACHE_OP_MAX || 457 result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 458 return -EINVAL; 459 460 ev = (*ppmu->cache_events)[type][op][result]; 461 if (ev == 0) 462 return -EOPNOTSUPP; 463 if (ev == -1) 464 return -EINVAL; 465 *eventp = ev; 466 return 0; 467 } 468 469 static int fsl_emb_pmu_event_init(struct perf_event *event) 470 { 471 u64 ev; 472 struct perf_event *events[MAX_HWEVENTS]; 473 int n; 474 int err; 475 int num_restricted; 476 int i; 477 478 if (ppmu->n_counter > MAX_HWEVENTS) { 479 WARN(1, "No. of perf counters (%d) is higher than max array size(%d)\n", 480 ppmu->n_counter, MAX_HWEVENTS); 481 ppmu->n_counter = MAX_HWEVENTS; 482 } 483 484 switch (event->attr.type) { 485 case PERF_TYPE_HARDWARE: 486 ev = event->attr.config; 487 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0) 488 return -EOPNOTSUPP; 489 ev = ppmu->generic_events[ev]; 490 break; 491 492 case PERF_TYPE_HW_CACHE: 493 err = hw_perf_cache_event(event->attr.config, &ev); 494 if (err) 495 return err; 496 break; 497 498 case PERF_TYPE_RAW: 499 ev = event->attr.config; 500 break; 501 502 default: 503 return -ENOENT; 504 } 505 506 event->hw.config = ppmu->xlate_event(ev); 507 if (!(event->hw.config & FSL_EMB_EVENT_VALID)) 508 return -EINVAL; 509 510 /* 511 * If this is in a group, check if it can go on with all the 512 * other hardware events in the group. We assume the event 513 * hasn't been linked into its leader's sibling list at this point. 514 */ 515 n = 0; 516 if (event->group_leader != event) { 517 n = collect_events(event->group_leader, 518 ppmu->n_counter - 1, events); 519 if (n < 0) 520 return -EINVAL; 521 } 522 523 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) { 524 num_restricted = 0; 525 for (i = 0; i < n; i++) { 526 if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED) 527 num_restricted++; 528 } 529 530 if (num_restricted >= ppmu->n_restricted) 531 return -EINVAL; 532 } 533 534 event->hw.idx = -1; 535 536 event->hw.config_base = PMLCA_CE | PMLCA_FCM1 | 537 (u32)((ev << 16) & PMLCA_EVENT_MASK); 538 539 if (event->attr.exclude_user) 540 event->hw.config_base |= PMLCA_FCU; 541 if (event->attr.exclude_kernel) 542 event->hw.config_base |= PMLCA_FCS; 543 if (event->attr.exclude_idle) 544 return -ENOTSUPP; 545 546 event->hw.last_period = event->hw.sample_period; 547 local64_set(&event->hw.period_left, event->hw.last_period); 548 549 /* 550 * See if we need to reserve the PMU. 551 * If no events are currently in use, then we have to take a 552 * mutex to ensure that we don't race with another task doing 553 * reserve_pmc_hardware or release_pmc_hardware. 554 */ 555 err = 0; 556 if (!atomic_inc_not_zero(&num_events)) { 557 mutex_lock(&pmc_reserve_mutex); 558 if (atomic_read(&num_events) == 0 && 559 reserve_pmc_hardware(perf_event_interrupt)) 560 err = -EBUSY; 561 else 562 atomic_inc(&num_events); 563 mutex_unlock(&pmc_reserve_mutex); 564 565 mtpmr(PMRN_PMGC0, PMGC0_FAC); 566 isync(); 567 } 568 event->destroy = hw_perf_event_destroy; 569 570 return err; 571 } 572 573 static struct pmu fsl_emb_pmu = { 574 .pmu_enable = fsl_emb_pmu_enable, 575 .pmu_disable = fsl_emb_pmu_disable, 576 .event_init = fsl_emb_pmu_event_init, 577 .add = fsl_emb_pmu_add, 578 .del = fsl_emb_pmu_del, 579 .start = fsl_emb_pmu_start, 580 .stop = fsl_emb_pmu_stop, 581 .read = fsl_emb_pmu_read, 582 }; 583 584 /* 585 * A counter has overflowed; update its count and record 586 * things if requested. Note that interrupts are hard-disabled 587 * here so there is no possibility of being interrupted. 588 */ 589 static void record_and_restart(struct perf_event *event, unsigned long val, 590 struct pt_regs *regs) 591 { 592 u64 period = event->hw.sample_period; 593 s64 prev, delta, left; 594 int record = 0; 595 596 if (event->hw.state & PERF_HES_STOPPED) { 597 write_pmc(event->hw.idx, 0); 598 return; 599 } 600 601 /* we don't have to worry about interrupts here */ 602 prev = local64_read(&event->hw.prev_count); 603 delta = (val - prev) & 0xfffffffful; 604 local64_add(delta, &event->count); 605 606 /* 607 * See if the total period for this event has expired, 608 * and update for the next period. 609 */ 610 val = 0; 611 left = local64_read(&event->hw.period_left) - delta; 612 if (period) { 613 if (left <= 0) { 614 left += period; 615 if (left <= 0) 616 left = period; 617 record = 1; 618 event->hw.last_period = event->hw.sample_period; 619 } 620 if (left < 0x80000000LL) 621 val = 0x80000000LL - left; 622 } 623 624 write_pmc(event->hw.idx, val); 625 local64_set(&event->hw.prev_count, val); 626 local64_set(&event->hw.period_left, left); 627 perf_event_update_userpage(event); 628 629 /* 630 * Finally record data if requested. 631 */ 632 if (record) { 633 struct perf_sample_data data; 634 635 perf_sample_data_init(&data, 0, event->hw.last_period); 636 637 if (perf_event_overflow(event, &data, regs)) 638 fsl_emb_pmu_stop(event, 0); 639 } 640 } 641 642 static void perf_event_interrupt(struct pt_regs *regs) 643 { 644 int i; 645 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 646 struct perf_event *event; 647 unsigned long val; 648 649 for (i = 0; i < ppmu->n_counter; ++i) { 650 event = cpuhw->event[i]; 651 652 val = read_pmc(i); 653 if ((int)val < 0) { 654 if (event) { 655 /* event has overflowed */ 656 record_and_restart(event, val, regs); 657 } else { 658 /* 659 * Disabled counter is negative, 660 * reset it just in case. 661 */ 662 write_pmc(i, 0); 663 } 664 } 665 } 666 667 /* PMM will keep counters frozen until we return from the interrupt. */ 668 mtmsr(mfmsr() | MSR_PMM); 669 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); 670 isync(); 671 } 672 673 static int fsl_emb_pmu_prepare_cpu(unsigned int cpu) 674 { 675 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); 676 677 memset(cpuhw, 0, sizeof(*cpuhw)); 678 679 return 0; 680 } 681 682 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu) 683 { 684 if (ppmu) 685 return -EBUSY; /* something's already registered */ 686 687 ppmu = pmu; 688 pr_info("%s performance monitor hardware support registered\n", 689 pmu->name); 690 691 perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW); 692 cpuhp_setup_state(CPUHP_PERF_POWER, "perf/powerpc:prepare", 693 fsl_emb_pmu_prepare_cpu, NULL); 694 695 return 0; 696 } 697