1 /* 2 * Performance event support - Freescale Embedded Performance Monitor 3 * 4 * Copyright 2008-2009 Paul Mackerras, IBM Corporation. 5 * Copyright 2010 Freescale Semiconductor, Inc. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/perf_event.h> 15 #include <linux/percpu.h> 16 #include <linux/hardirq.h> 17 #include <asm/reg_fsl_emb.h> 18 #include <asm/pmc.h> 19 #include <asm/machdep.h> 20 #include <asm/firmware.h> 21 #include <asm/ptrace.h> 22 23 struct cpu_hw_events { 24 int n_events; 25 int disabled; 26 u8 pmcs_enabled; 27 struct perf_event *event[MAX_HWEVENTS]; 28 }; 29 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); 30 31 static struct fsl_emb_pmu *ppmu; 32 33 /* Number of perf_events counting hardware events */ 34 static atomic_t num_events; 35 /* Used to avoid races in calling reserve/release_pmc_hardware */ 36 static DEFINE_MUTEX(pmc_reserve_mutex); 37 38 /* 39 * If interrupts were soft-disabled when a PMU interrupt occurs, treat 40 * it as an NMI. 41 */ 42 static inline int perf_intr_is_nmi(struct pt_regs *regs) 43 { 44 #ifdef __powerpc64__ 45 return !regs->softe; 46 #else 47 return 0; 48 #endif 49 } 50 51 static void perf_event_interrupt(struct pt_regs *regs); 52 53 /* 54 * Read one performance monitor counter (PMC). 55 */ 56 static unsigned long read_pmc(int idx) 57 { 58 unsigned long val; 59 60 switch (idx) { 61 case 0: 62 val = mfpmr(PMRN_PMC0); 63 break; 64 case 1: 65 val = mfpmr(PMRN_PMC1); 66 break; 67 case 2: 68 val = mfpmr(PMRN_PMC2); 69 break; 70 case 3: 71 val = mfpmr(PMRN_PMC3); 72 break; 73 default: 74 printk(KERN_ERR "oops trying to read PMC%d\n", idx); 75 val = 0; 76 } 77 return val; 78 } 79 80 /* 81 * Write one PMC. 82 */ 83 static void write_pmc(int idx, unsigned long val) 84 { 85 switch (idx) { 86 case 0: 87 mtpmr(PMRN_PMC0, val); 88 break; 89 case 1: 90 mtpmr(PMRN_PMC1, val); 91 break; 92 case 2: 93 mtpmr(PMRN_PMC2, val); 94 break; 95 case 3: 96 mtpmr(PMRN_PMC3, val); 97 break; 98 default: 99 printk(KERN_ERR "oops trying to write PMC%d\n", idx); 100 } 101 102 isync(); 103 } 104 105 /* 106 * Write one local control A register 107 */ 108 static void write_pmlca(int idx, unsigned long val) 109 { 110 switch (idx) { 111 case 0: 112 mtpmr(PMRN_PMLCA0, val); 113 break; 114 case 1: 115 mtpmr(PMRN_PMLCA1, val); 116 break; 117 case 2: 118 mtpmr(PMRN_PMLCA2, val); 119 break; 120 case 3: 121 mtpmr(PMRN_PMLCA3, val); 122 break; 123 default: 124 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx); 125 } 126 127 isync(); 128 } 129 130 /* 131 * Write one local control B register 132 */ 133 static void write_pmlcb(int idx, unsigned long val) 134 { 135 switch (idx) { 136 case 0: 137 mtpmr(PMRN_PMLCB0, val); 138 break; 139 case 1: 140 mtpmr(PMRN_PMLCB1, val); 141 break; 142 case 2: 143 mtpmr(PMRN_PMLCB2, val); 144 break; 145 case 3: 146 mtpmr(PMRN_PMLCB3, val); 147 break; 148 default: 149 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx); 150 } 151 152 isync(); 153 } 154 155 static void fsl_emb_pmu_read(struct perf_event *event) 156 { 157 s64 val, delta, prev; 158 159 if (event->hw.state & PERF_HES_STOPPED) 160 return; 161 162 /* 163 * Performance monitor interrupts come even when interrupts 164 * are soft-disabled, as long as interrupts are hard-enabled. 165 * Therefore we treat them like NMIs. 166 */ 167 do { 168 prev = local64_read(&event->hw.prev_count); 169 barrier(); 170 val = read_pmc(event->hw.idx); 171 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev); 172 173 /* The counters are only 32 bits wide */ 174 delta = (val - prev) & 0xfffffffful; 175 local64_add(delta, &event->count); 176 local64_sub(delta, &event->hw.period_left); 177 } 178 179 /* 180 * Disable all events to prevent PMU interrupts and to allow 181 * events to be added or removed. 182 */ 183 static void fsl_emb_pmu_disable(struct pmu *pmu) 184 { 185 struct cpu_hw_events *cpuhw; 186 unsigned long flags; 187 188 local_irq_save(flags); 189 cpuhw = &__get_cpu_var(cpu_hw_events); 190 191 if (!cpuhw->disabled) { 192 cpuhw->disabled = 1; 193 194 /* 195 * Check if we ever enabled the PMU on this cpu. 196 */ 197 if (!cpuhw->pmcs_enabled) { 198 ppc_enable_pmcs(); 199 cpuhw->pmcs_enabled = 1; 200 } 201 202 if (atomic_read(&num_events)) { 203 /* 204 * Set the 'freeze all counters' bit, and disable 205 * interrupts. The barrier is to make sure the 206 * mtpmr has been executed and the PMU has frozen 207 * the events before we return. 208 */ 209 210 mtpmr(PMRN_PMGC0, PMGC0_FAC); 211 isync(); 212 } 213 } 214 local_irq_restore(flags); 215 } 216 217 /* 218 * Re-enable all events if disable == 0. 219 * If we were previously disabled and events were added, then 220 * put the new config on the PMU. 221 */ 222 static void fsl_emb_pmu_enable(struct pmu *pmu) 223 { 224 struct cpu_hw_events *cpuhw; 225 unsigned long flags; 226 227 local_irq_save(flags); 228 cpuhw = &__get_cpu_var(cpu_hw_events); 229 if (!cpuhw->disabled) 230 goto out; 231 232 cpuhw->disabled = 0; 233 ppc_set_pmu_inuse(cpuhw->n_events != 0); 234 235 if (cpuhw->n_events > 0) { 236 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); 237 isync(); 238 } 239 240 out: 241 local_irq_restore(flags); 242 } 243 244 static int collect_events(struct perf_event *group, int max_count, 245 struct perf_event *ctrs[]) 246 { 247 int n = 0; 248 struct perf_event *event; 249 250 if (!is_software_event(group)) { 251 if (n >= max_count) 252 return -1; 253 ctrs[n] = group; 254 n++; 255 } 256 list_for_each_entry(event, &group->sibling_list, group_entry) { 257 if (!is_software_event(event) && 258 event->state != PERF_EVENT_STATE_OFF) { 259 if (n >= max_count) 260 return -1; 261 ctrs[n] = event; 262 n++; 263 } 264 } 265 return n; 266 } 267 268 /* context locked on entry */ 269 static int fsl_emb_pmu_add(struct perf_event *event, int flags) 270 { 271 struct cpu_hw_events *cpuhw; 272 int ret = -EAGAIN; 273 int num_counters = ppmu->n_counter; 274 u64 val; 275 int i; 276 277 perf_pmu_disable(event->pmu); 278 cpuhw = &get_cpu_var(cpu_hw_events); 279 280 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) 281 num_counters = ppmu->n_restricted; 282 283 /* 284 * Allocate counters from top-down, so that restricted-capable 285 * counters are kept free as long as possible. 286 */ 287 for (i = num_counters - 1; i >= 0; i--) { 288 if (cpuhw->event[i]) 289 continue; 290 291 break; 292 } 293 294 if (i < 0) 295 goto out; 296 297 event->hw.idx = i; 298 cpuhw->event[i] = event; 299 ++cpuhw->n_events; 300 301 val = 0; 302 if (event->hw.sample_period) { 303 s64 left = local64_read(&event->hw.period_left); 304 if (left < 0x80000000L) 305 val = 0x80000000L - left; 306 } 307 local64_set(&event->hw.prev_count, val); 308 309 if (!(flags & PERF_EF_START)) { 310 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 311 val = 0; 312 } 313 314 write_pmc(i, val); 315 perf_event_update_userpage(event); 316 317 write_pmlcb(i, event->hw.config >> 32); 318 write_pmlca(i, event->hw.config_base); 319 320 ret = 0; 321 out: 322 put_cpu_var(cpu_hw_events); 323 perf_pmu_enable(event->pmu); 324 return ret; 325 } 326 327 /* context locked on entry */ 328 static void fsl_emb_pmu_del(struct perf_event *event, int flags) 329 { 330 struct cpu_hw_events *cpuhw; 331 int i = event->hw.idx; 332 333 perf_pmu_disable(event->pmu); 334 if (i < 0) 335 goto out; 336 337 fsl_emb_pmu_read(event); 338 339 cpuhw = &get_cpu_var(cpu_hw_events); 340 341 WARN_ON(event != cpuhw->event[event->hw.idx]); 342 343 write_pmlca(i, 0); 344 write_pmlcb(i, 0); 345 write_pmc(i, 0); 346 347 cpuhw->event[i] = NULL; 348 event->hw.idx = -1; 349 350 /* 351 * TODO: if at least one restricted event exists, and we 352 * just freed up a non-restricted-capable counter, and 353 * there is a restricted-capable counter occupied by 354 * a non-restricted event, migrate that event to the 355 * vacated counter. 356 */ 357 358 cpuhw->n_events--; 359 360 out: 361 perf_pmu_enable(event->pmu); 362 put_cpu_var(cpu_hw_events); 363 } 364 365 static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags) 366 { 367 unsigned long flags; 368 s64 left; 369 370 if (event->hw.idx < 0 || !event->hw.sample_period) 371 return; 372 373 if (!(event->hw.state & PERF_HES_STOPPED)) 374 return; 375 376 if (ef_flags & PERF_EF_RELOAD) 377 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 378 379 local_irq_save(flags); 380 perf_pmu_disable(event->pmu); 381 382 event->hw.state = 0; 383 left = local64_read(&event->hw.period_left); 384 write_pmc(event->hw.idx, left); 385 386 perf_event_update_userpage(event); 387 perf_pmu_enable(event->pmu); 388 local_irq_restore(flags); 389 } 390 391 static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags) 392 { 393 unsigned long flags; 394 395 if (event->hw.idx < 0 || !event->hw.sample_period) 396 return; 397 398 if (event->hw.state & PERF_HES_STOPPED) 399 return; 400 401 local_irq_save(flags); 402 perf_pmu_disable(event->pmu); 403 404 fsl_emb_pmu_read(event); 405 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 406 write_pmc(event->hw.idx, 0); 407 408 perf_event_update_userpage(event); 409 perf_pmu_enable(event->pmu); 410 local_irq_restore(flags); 411 } 412 413 /* 414 * Release the PMU if this is the last perf_event. 415 */ 416 static void hw_perf_event_destroy(struct perf_event *event) 417 { 418 if (!atomic_add_unless(&num_events, -1, 1)) { 419 mutex_lock(&pmc_reserve_mutex); 420 if (atomic_dec_return(&num_events) == 0) 421 release_pmc_hardware(); 422 mutex_unlock(&pmc_reserve_mutex); 423 } 424 } 425 426 /* 427 * Translate a generic cache event_id config to a raw event_id code. 428 */ 429 static int hw_perf_cache_event(u64 config, u64 *eventp) 430 { 431 unsigned long type, op, result; 432 int ev; 433 434 if (!ppmu->cache_events) 435 return -EINVAL; 436 437 /* unpack config */ 438 type = config & 0xff; 439 op = (config >> 8) & 0xff; 440 result = (config >> 16) & 0xff; 441 442 if (type >= PERF_COUNT_HW_CACHE_MAX || 443 op >= PERF_COUNT_HW_CACHE_OP_MAX || 444 result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 445 return -EINVAL; 446 447 ev = (*ppmu->cache_events)[type][op][result]; 448 if (ev == 0) 449 return -EOPNOTSUPP; 450 if (ev == -1) 451 return -EINVAL; 452 *eventp = ev; 453 return 0; 454 } 455 456 static int fsl_emb_pmu_event_init(struct perf_event *event) 457 { 458 u64 ev; 459 struct perf_event *events[MAX_HWEVENTS]; 460 int n; 461 int err; 462 int num_restricted; 463 int i; 464 465 switch (event->attr.type) { 466 case PERF_TYPE_HARDWARE: 467 ev = event->attr.config; 468 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0) 469 return -EOPNOTSUPP; 470 ev = ppmu->generic_events[ev]; 471 break; 472 473 case PERF_TYPE_HW_CACHE: 474 err = hw_perf_cache_event(event->attr.config, &ev); 475 if (err) 476 return err; 477 break; 478 479 case PERF_TYPE_RAW: 480 ev = event->attr.config; 481 break; 482 483 default: 484 return -ENOENT; 485 } 486 487 event->hw.config = ppmu->xlate_event(ev); 488 if (!(event->hw.config & FSL_EMB_EVENT_VALID)) 489 return -EINVAL; 490 491 /* 492 * If this is in a group, check if it can go on with all the 493 * other hardware events in the group. We assume the event 494 * hasn't been linked into its leader's sibling list at this point. 495 */ 496 n = 0; 497 if (event->group_leader != event) { 498 n = collect_events(event->group_leader, 499 ppmu->n_counter - 1, events); 500 if (n < 0) 501 return -EINVAL; 502 } 503 504 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) { 505 num_restricted = 0; 506 for (i = 0; i < n; i++) { 507 if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED) 508 num_restricted++; 509 } 510 511 if (num_restricted >= ppmu->n_restricted) 512 return -EINVAL; 513 } 514 515 event->hw.idx = -1; 516 517 event->hw.config_base = PMLCA_CE | PMLCA_FCM1 | 518 (u32)((ev << 16) & PMLCA_EVENT_MASK); 519 520 if (event->attr.exclude_user) 521 event->hw.config_base |= PMLCA_FCU; 522 if (event->attr.exclude_kernel) 523 event->hw.config_base |= PMLCA_FCS; 524 if (event->attr.exclude_idle) 525 return -ENOTSUPP; 526 527 event->hw.last_period = event->hw.sample_period; 528 local64_set(&event->hw.period_left, event->hw.last_period); 529 530 /* 531 * See if we need to reserve the PMU. 532 * If no events are currently in use, then we have to take a 533 * mutex to ensure that we don't race with another task doing 534 * reserve_pmc_hardware or release_pmc_hardware. 535 */ 536 err = 0; 537 if (!atomic_inc_not_zero(&num_events)) { 538 mutex_lock(&pmc_reserve_mutex); 539 if (atomic_read(&num_events) == 0 && 540 reserve_pmc_hardware(perf_event_interrupt)) 541 err = -EBUSY; 542 else 543 atomic_inc(&num_events); 544 mutex_unlock(&pmc_reserve_mutex); 545 546 mtpmr(PMRN_PMGC0, PMGC0_FAC); 547 isync(); 548 } 549 event->destroy = hw_perf_event_destroy; 550 551 return err; 552 } 553 554 static struct pmu fsl_emb_pmu = { 555 .pmu_enable = fsl_emb_pmu_enable, 556 .pmu_disable = fsl_emb_pmu_disable, 557 .event_init = fsl_emb_pmu_event_init, 558 .add = fsl_emb_pmu_add, 559 .del = fsl_emb_pmu_del, 560 .start = fsl_emb_pmu_start, 561 .stop = fsl_emb_pmu_stop, 562 .read = fsl_emb_pmu_read, 563 }; 564 565 /* 566 * A counter has overflowed; update its count and record 567 * things if requested. Note that interrupts are hard-disabled 568 * here so there is no possibility of being interrupted. 569 */ 570 static void record_and_restart(struct perf_event *event, unsigned long val, 571 struct pt_regs *regs) 572 { 573 u64 period = event->hw.sample_period; 574 s64 prev, delta, left; 575 int record = 0; 576 577 if (event->hw.state & PERF_HES_STOPPED) { 578 write_pmc(event->hw.idx, 0); 579 return; 580 } 581 582 /* we don't have to worry about interrupts here */ 583 prev = local64_read(&event->hw.prev_count); 584 delta = (val - prev) & 0xfffffffful; 585 local64_add(delta, &event->count); 586 587 /* 588 * See if the total period for this event has expired, 589 * and update for the next period. 590 */ 591 val = 0; 592 left = local64_read(&event->hw.period_left) - delta; 593 if (period) { 594 if (left <= 0) { 595 left += period; 596 if (left <= 0) 597 left = period; 598 record = 1; 599 event->hw.last_period = event->hw.sample_period; 600 } 601 if (left < 0x80000000LL) 602 val = 0x80000000LL - left; 603 } 604 605 write_pmc(event->hw.idx, val); 606 local64_set(&event->hw.prev_count, val); 607 local64_set(&event->hw.period_left, left); 608 perf_event_update_userpage(event); 609 610 /* 611 * Finally record data if requested. 612 */ 613 if (record) { 614 struct perf_sample_data data; 615 616 perf_sample_data_init(&data, 0, event->hw.last_period); 617 618 if (perf_event_overflow(event, &data, regs)) 619 fsl_emb_pmu_stop(event, 0); 620 } 621 } 622 623 static void perf_event_interrupt(struct pt_regs *regs) 624 { 625 int i; 626 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events); 627 struct perf_event *event; 628 unsigned long val; 629 int found = 0; 630 int nmi; 631 632 nmi = perf_intr_is_nmi(regs); 633 if (nmi) 634 nmi_enter(); 635 else 636 irq_enter(); 637 638 for (i = 0; i < ppmu->n_counter; ++i) { 639 event = cpuhw->event[i]; 640 641 val = read_pmc(i); 642 if ((int)val < 0) { 643 if (event) { 644 /* event has overflowed */ 645 found = 1; 646 record_and_restart(event, val, regs); 647 } else { 648 /* 649 * Disabled counter is negative, 650 * reset it just in case. 651 */ 652 write_pmc(i, 0); 653 } 654 } 655 } 656 657 /* PMM will keep counters frozen until we return from the interrupt. */ 658 mtmsr(mfmsr() | MSR_PMM); 659 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); 660 isync(); 661 662 if (nmi) 663 nmi_exit(); 664 else 665 irq_exit(); 666 } 667 668 void hw_perf_event_setup(int cpu) 669 { 670 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); 671 672 memset(cpuhw, 0, sizeof(*cpuhw)); 673 } 674 675 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu) 676 { 677 if (ppmu) 678 return -EBUSY; /* something's already registered */ 679 680 ppmu = pmu; 681 pr_info("%s performance monitor hardware support registered\n", 682 pmu->name); 683 684 perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW); 685 686 return 0; 687 } 688