1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011,2013 Justin Hibbits 5 * Copyright (c) 2005, Joseph Koshy 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/pmc.h> 36 #include <sys/pmckern.h> 37 #include <sys/sysent.h> 38 #include <sys/syslog.h> 39 #include <sys/systm.h> 40 41 #include <machine/pmc_mdep.h> 42 #include <machine/spr.h> 43 #include <machine/pte.h> 44 #include <machine/sr.h> 45 #include <machine/cpu.h> 46 #include <machine/stack.h> 47 48 #include "hwpmc_powerpc.h" 49 50 #ifdef __powerpc64__ 51 #define OFFSET 4 /* Account for the TOC reload slot */ 52 #else 53 #define OFFSET 0 54 #endif 55 56 struct powerpc_cpu **powerpc_pcpu; 57 struct pmc_ppc_event *ppc_event_codes; 58 size_t ppc_event_codes_size; 59 int ppc_event_first; 60 int ppc_event_last; 61 int ppc_max_pmcs; 62 63 void (*powerpc_set_pmc)(int cpu, int ri, int config); 64 pmc_value_t (*powerpc_pmcn_read)(unsigned int pmc); 65 void (*powerpc_pmcn_write)(unsigned int pmc, uint32_t val); 66 void (*powerpc_resume_pmc)(bool ie); 67 68 69 int 70 pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, 71 struct trapframe *tf) 72 { 73 uintptr_t *osp, *sp; 74 uintptr_t pc; 75 int frames = 0; 76 77 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 78 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 79 osp = (uintptr_t *)PAGE_SIZE; 80 81 for (; frames < maxsamples; frames++) { 82 if (sp <= osp) 83 break; 84 #ifdef __powerpc64__ 85 pc = sp[2]; 86 #else 87 pc = sp[1]; 88 #endif 89 if ((pc & 3) || (pc < 0x100)) 90 break; 91 92 /* 93 * trapexit() and asttrapexit() are sentinels 94 * for kernel stack tracing. 95 * */ 96 if (pc + OFFSET == (uintptr_t) &trapexit || 97 pc + OFFSET == (uintptr_t) &asttrapexit) 98 break; 99 100 cc[frames] = pc; 101 osp = sp; 102 sp = (uintptr_t *)*sp; 103 } 104 return (frames); 105 } 106 107 static int 108 powerpc_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 109 { 110 111 return (0); 112 } 113 114 static int 115 powerpc_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 116 { 117 118 return (0); 119 } 120 121 int 122 powerpc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 123 { 124 int error; 125 struct pmc_hw *phw; 126 char powerpc_name[PMC_NAME_MAX]; 127 128 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 129 ("[powerpc,%d], illegal CPU %d", __LINE__, cpu)); 130 131 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 132 snprintf(powerpc_name, sizeof(powerpc_name), "POWERPC-%d", ri); 133 if ((error = copystr(powerpc_name, pi->pm_name, PMC_NAME_MAX, 134 NULL)) != 0) 135 return error; 136 pi->pm_class = powerpc_pcpu[cpu]->pc_class; 137 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 138 pi->pm_enabled = TRUE; 139 *ppmc = phw->phw_pmc; 140 } else { 141 pi->pm_enabled = FALSE; 142 *ppmc = NULL; 143 } 144 145 return (0); 146 } 147 148 int 149 powerpc_get_config(int cpu, int ri, struct pmc **ppm) 150 { 151 152 *ppm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; 153 154 return (0); 155 } 156 157 int 158 powerpc_pcpu_init(struct pmc_mdep *md, int cpu) 159 { 160 struct pmc_cpu *pc; 161 struct powerpc_cpu *pac; 162 struct pmc_hw *phw; 163 int first_ri, i; 164 165 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 166 ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu)); 167 PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu); 168 169 powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu), M_PMC, 170 M_WAITOK|M_ZERO); 171 pac->pc_ppcpmcs = malloc(sizeof(struct pmc_hw) * ppc_max_pmcs, 172 M_PMC, M_WAITOK|M_ZERO); 173 pac->pc_class = 174 md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_class; 175 176 pc = pmc_pcpu[cpu]; 177 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri; 178 KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__)); 179 180 for (i = 0, phw = pac->pc_ppcpmcs; i < ppc_max_pmcs; i++, phw++) { 181 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 182 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); 183 phw->phw_pmc = NULL; 184 pc->pc_hwpmcs[i + first_ri] = phw; 185 } 186 187 return (0); 188 } 189 190 int 191 powerpc_pcpu_fini(struct pmc_mdep *md, int cpu) 192 { 193 PMCDBG1(MDP,INI,1,"powerpc-fini cpu=%d", cpu); 194 195 free(powerpc_pcpu[cpu]->pc_ppcpmcs, M_PMC); 196 free(powerpc_pcpu[cpu], M_PMC); 197 198 return (0); 199 } 200 201 int 202 powerpc_allocate_pmc(int cpu, int ri, struct pmc *pm, 203 const struct pmc_op_pmcallocate *a) 204 { 205 enum pmc_event pe; 206 uint32_t caps, config = 0, counter = 0; 207 int i; 208 209 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 210 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 211 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 212 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 213 214 caps = a->pm_caps; 215 216 pe = a->pm_ev; 217 218 if (pe < ppc_event_first || pe > ppc_event_last) 219 return (EINVAL); 220 221 for (i = 0; i < ppc_event_codes_size; i++) { 222 if (ppc_event_codes[i].pe_event == pe) { 223 config = ppc_event_codes[i].pe_code; 224 counter = ppc_event_codes[i].pe_flags; 225 break; 226 } 227 } 228 if (i == ppc_event_codes_size) 229 return (EINVAL); 230 231 if ((counter & (1 << ri)) == 0) 232 return (EINVAL); 233 234 if (caps & PMC_CAP_SYSTEM) 235 config |= POWERPC_PMC_KERNEL_ENABLE; 236 if (caps & PMC_CAP_USER) 237 config |= POWERPC_PMC_USER_ENABLE; 238 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 239 config |= POWERPC_PMC_ENABLE; 240 241 pm->pm_md.pm_powerpc.pm_powerpc_evsel = config; 242 243 PMCDBG3(MDP,ALL,1,"powerpc-allocate cpu=%d ri=%d -> config=0x%x", 244 cpu, ri, config); 245 return (0); 246 } 247 248 int 249 powerpc_release_pmc(int cpu, int ri, struct pmc *pmc) 250 { 251 struct pmc_hw *phw; 252 253 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 254 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 255 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 256 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 257 258 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 259 KASSERT(phw->phw_pmc == NULL, 260 ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 261 262 return (0); 263 } 264 265 int 266 powerpc_start_pmc(int cpu, int ri) 267 { 268 struct pmc *pm; 269 270 PMCDBG2(MDP,STA,1,"powerpc-start cpu=%d ri=%d", cpu, ri); 271 pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; 272 powerpc_set_pmc(cpu, ri, pm->pm_md.pm_powerpc.pm_powerpc_evsel); 273 274 return (0); 275 } 276 277 int 278 powerpc_stop_pmc(int cpu, int ri) 279 { 280 PMCDBG2(MDP,STO,1, "powerpc-stop cpu=%d ri=%d", cpu, ri); 281 powerpc_set_pmc(cpu, ri, PMCN_NONE); 282 return (0); 283 } 284 285 int 286 powerpc_config_pmc(int cpu, int ri, struct pmc *pm) 287 { 288 struct pmc_hw *phw; 289 290 PMCDBG3(MDP,CFG,1, "powerpc-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 291 292 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 293 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 294 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 295 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 296 297 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 298 299 KASSERT(pm == NULL || phw->phw_pmc == NULL, 300 ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 301 __LINE__, pm, phw->phw_pmc)); 302 303 phw->phw_pmc = pm; 304 305 return (0); 306 } 307 308 pmc_value_t 309 powerpc_pmcn_read_default(unsigned int pmc) 310 { 311 pmc_value_t val; 312 313 if (pmc > ppc_max_pmcs) 314 panic("Invalid PMC number: %d\n", pmc); 315 316 switch (pmc) { 317 case 0: 318 val = mfspr(SPR_PMC1); 319 break; 320 case 1: 321 val = mfspr(SPR_PMC2); 322 break; 323 case 2: 324 val = mfspr(SPR_PMC3); 325 break; 326 case 3: 327 val = mfspr(SPR_PMC4); 328 break; 329 case 4: 330 val = mfspr(SPR_PMC5); 331 break; 332 case 5: 333 val = mfspr(SPR_PMC6); 334 break; 335 case 6: 336 val = mfspr(SPR_PMC7); 337 break; 338 case 7: 339 val = mfspr(SPR_PMC8); 340 break; 341 } 342 343 return (val); 344 } 345 346 void 347 powerpc_pmcn_write_default(unsigned int pmc, uint32_t val) 348 { 349 if (pmc > ppc_max_pmcs) 350 panic("Invalid PMC number: %d\n", pmc); 351 352 switch (pmc) { 353 case 0: 354 mtspr(SPR_PMC1, val); 355 break; 356 case 1: 357 mtspr(SPR_PMC2, val); 358 break; 359 case 2: 360 mtspr(SPR_PMC3, val); 361 break; 362 case 3: 363 mtspr(SPR_PMC4, val); 364 break; 365 case 4: 366 mtspr(SPR_PMC5, val); 367 break; 368 case 5: 369 mtspr(SPR_PMC6, val); 370 break; 371 case 6: 372 mtspr(SPR_PMC7, val); 373 break; 374 case 7: 375 mtspr(SPR_PMC8, val); 376 break; 377 } 378 } 379 380 int 381 powerpc_read_pmc(int cpu, int ri, pmc_value_t *v) 382 { 383 struct pmc *pm; 384 pmc_value_t p, r, tmp; 385 386 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 387 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 388 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 389 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 390 391 pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; 392 KASSERT(pm, 393 ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, 394 ri)); 395 396 /* 397 * After an interrupt occurs because of a PMC overflow, the PMC value 398 * is not always MAX_PMC_VALUE + 1, but may be a little above it. 399 * This may mess up calculations and frustrate machine independent 400 * layer expectations, such as that no value read should be greater 401 * than reload count in sampling mode. 402 * To avoid these issues, use MAX_PMC_VALUE as an upper limit. 403 */ 404 p = MIN(powerpc_pmcn_read(ri), POWERPC_MAX_PMC_VALUE); 405 r = pm->pm_sc.pm_reloadcount; 406 407 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 408 /* 409 * Special case 1: r is too big 410 * This usually happens when a PMC write fails, the PMC is 411 * stopped and then it is read. 412 * 413 * Special case 2: PMC was reseted or has a value 414 * that should not be possible with current r. 415 * 416 * In the above cases, just return 0 instead of an arbitrary 417 * value. 418 */ 419 if (r > POWERPC_MAX_PMC_VALUE || p + r <= POWERPC_MAX_PMC_VALUE) 420 tmp = 0; 421 else 422 tmp = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(p); 423 } else 424 tmp = p + (POWERPC_MAX_PMC_VALUE + 1) * PPC_OVERFLOWCNT(pm); 425 426 PMCDBG5(MDP,REA,1,"ppc-read cpu=%d ri=%d -> %jx (%jx,%jx)", 427 cpu, ri, (uintmax_t)tmp, (uintmax_t)PPC_OVERFLOWCNT(pm), 428 (uintmax_t)p); 429 *v = tmp; 430 return (0); 431 } 432 433 int 434 powerpc_write_pmc(int cpu, int ri, pmc_value_t v) 435 { 436 struct pmc *pm; 437 pmc_value_t vlo; 438 439 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 440 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 441 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 442 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 443 444 pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; 445 446 if (PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { 447 PPC_OVERFLOWCNT(pm) = v / (POWERPC_MAX_PMC_VALUE + 1); 448 vlo = v % (POWERPC_MAX_PMC_VALUE + 1); 449 } else if (v > POWERPC_MAX_PMC_VALUE) { 450 PMCDBG3(MDP,WRI,2, 451 "powerpc-write cpu=%d ri=%d: PMC value is too big: %jx", 452 cpu, ri, (uintmax_t)v); 453 return (EINVAL); 454 } else 455 vlo = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 456 457 PMCDBG5(MDP,WRI,1,"powerpc-write cpu=%d ri=%d -> %jx (%jx,%jx)", 458 cpu, ri, (uintmax_t)v, (uintmax_t)PPC_OVERFLOWCNT(pm), 459 (uintmax_t)vlo); 460 461 powerpc_pmcn_write(ri, vlo); 462 return (0); 463 } 464 465 int 466 powerpc_pmc_intr(struct trapframe *tf) 467 { 468 struct pmc *pm; 469 struct powerpc_cpu *pc; 470 int cpu, error, i, retval; 471 472 cpu = curcpu; 473 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 474 ("[powerpc,%d] out of range CPU %d", __LINE__, cpu)); 475 476 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, 477 TRAPF_USERMODE(tf)); 478 479 retval = 0; 480 pc = powerpc_pcpu[cpu]; 481 482 /* 483 * Look for a running, sampling PMC which has overflowed 484 * and which has a valid 'struct pmc' association. 485 */ 486 for (i = 0; i < ppc_max_pmcs; i++) { 487 if (!POWERPC_PMC_HAS_OVERFLOWED(i)) 488 continue; 489 retval = 1; /* Found an interrupting PMC. */ 490 491 /* 492 * Always clear the PMC, to make it stop interrupting. 493 * If pm is available and in sampling mode, use reload 494 * count, to make PMC read after stop correct. 495 * Otherwise, just reset the PMC. 496 */ 497 if ((pm = pc->pc_ppcpmcs[i].phw_pmc) != NULL && 498 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 499 if (pm->pm_state != PMC_STATE_RUNNING) { 500 powerpc_write_pmc(cpu, i, 501 pm->pm_sc.pm_reloadcount); 502 continue; 503 } 504 } else { 505 if (pm != NULL) { /* !PMC_IS_SAMPLING_MODE */ 506 PPC_OVERFLOWCNT(pm) = (PPC_OVERFLOWCNT(pm) + 507 1) % PPC_OVERFLOWCNT_MAX; 508 PMCDBG3(MDP,INT,2, 509 "cpu=%d ri=%d: overflowcnt=%d", 510 cpu, i, PPC_OVERFLOWCNT(pm)); 511 } 512 513 powerpc_pmcn_write(i, 0); 514 continue; 515 } 516 517 error = pmc_process_interrupt(PMC_HR, pm, tf); 518 if (error != 0) { 519 PMCDBG3(MDP,INT,3, 520 "cpu=%d ri=%d: error %d processing interrupt", 521 cpu, i, error); 522 powerpc_stop_pmc(cpu, i); 523 } 524 525 /* Reload sampling count */ 526 powerpc_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount); 527 } 528 529 if (retval) 530 counter_u64_add(pmc_stats.pm_intr_processed, 1); 531 else 532 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 533 534 /* 535 * Re-enable PERF exceptions if we were able to find the interrupt 536 * source and handle it. Otherwise, it's better to disable PERF 537 * interrupts, to avoid the risk of processing the same interrupt 538 * forever. 539 */ 540 powerpc_resume_pmc(retval != 0); 541 if (retval == 0) 542 log(LOG_WARNING, 543 "pmc_intr: couldn't find interrupting PMC on cpu %d - " 544 "disabling PERF interrupts\n", cpu); 545 546 return (retval); 547 } 548 549 struct pmc_mdep * 550 pmc_md_initialize() 551 { 552 struct pmc_mdep *pmc_mdep; 553 int error; 554 uint16_t vers; 555 556 /* 557 * Allocate space for pointers to PMC HW descriptors and for 558 * the MDEP structure used by MI code. 559 */ 560 powerpc_pcpu = malloc(sizeof(struct powerpc_cpu *) * pmc_cpu_max(), M_PMC, 561 M_WAITOK|M_ZERO); 562 563 /* Just one class */ 564 pmc_mdep = pmc_mdep_alloc(1); 565 566 vers = mfpvr() >> 16; 567 568 pmc_mdep->pmd_switch_in = powerpc_switch_in; 569 pmc_mdep->pmd_switch_out = powerpc_switch_out; 570 571 switch (vers) { 572 case MPC7447A: 573 case MPC7448: 574 case MPC7450: 575 case MPC7455: 576 case MPC7457: 577 error = pmc_mpc7xxx_initialize(pmc_mdep); 578 break; 579 case IBM970: 580 case IBM970FX: 581 case IBM970MP: 582 error = pmc_ppc970_initialize(pmc_mdep); 583 break; 584 case IBMPOWER8E: 585 case IBMPOWER8NVL: 586 case IBMPOWER8: 587 case IBMPOWER9: 588 error = pmc_power8_initialize(pmc_mdep); 589 break; 590 case FSL_E500v1: 591 case FSL_E500v2: 592 case FSL_E500mc: 593 case FSL_E5500: 594 error = pmc_e500_initialize(pmc_mdep); 595 break; 596 default: 597 error = -1; 598 break; 599 } 600 601 if (error != 0) { 602 pmc_mdep_free(pmc_mdep); 603 pmc_mdep = NULL; 604 } 605 606 return (pmc_mdep); 607 } 608 609 void 610 pmc_md_finalize(struct pmc_mdep *md) 611 { 612 613 free(powerpc_pcpu, M_PMC); 614 powerpc_pcpu = NULL; 615 } 616 617 int 618 pmc_save_user_callchain(uintptr_t *cc, int maxsamples, 619 struct trapframe *tf) 620 { 621 uintptr_t *osp, *sp; 622 int frames = 0; 623 624 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 625 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 626 osp = NULL; 627 628 for (; frames < maxsamples; frames++) { 629 if (sp <= osp) 630 break; 631 osp = sp; 632 #ifdef __powerpc64__ 633 /* Check if 32-bit mode. */ 634 if (!(tf->srr1 & PSL_SF)) { 635 cc[frames] = fuword32((uint32_t *)sp + 1); 636 sp = (uintptr_t *)(uintptr_t)fuword32(sp); 637 } else { 638 cc[frames] = fuword(sp + 2); 639 sp = (uintptr_t *)fuword(sp); 640 } 641 #else 642 cc[frames] = fuword32((uint32_t *)sp + 1); 643 sp = (uintptr_t *)fuword32(sp); 644 #endif 645 } 646 647 return (frames); 648 } 649