1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011,2013 Justin Hibbits 5 * Copyright (c) 2005, Joseph Koshy 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31 #include <sys/param.h> 32 #include <sys/pmc.h> 33 #include <sys/pmckern.h> 34 #include <sys/sysent.h> 35 #include <sys/syslog.h> 36 #include <sys/systm.h> 37 38 #include <machine/pmc_mdep.h> 39 #include <machine/spr.h> 40 #include <machine/pte.h> 41 #include <machine/sr.h> 42 #include <machine/cpu.h> 43 #include <machine/stack.h> 44 45 #include "hwpmc_powerpc.h" 46 47 #ifdef __powerpc64__ 48 #define OFFSET 4 /* Account for the TOC reload slot */ 49 #else 50 #define OFFSET 0 51 #endif 52 53 struct powerpc_cpu **powerpc_pcpu; 54 struct pmc_ppc_event *ppc_event_codes; 55 size_t ppc_event_codes_size; 56 int ppc_event_first; 57 int ppc_event_last; 58 int ppc_max_pmcs; 59 enum pmc_class ppc_class; 60 61 void (*powerpc_set_pmc)(int cpu, int ri, int config); 62 pmc_value_t (*powerpc_pmcn_read)(unsigned int pmc); 63 void (*powerpc_pmcn_write)(unsigned int pmc, uint32_t val); 64 void (*powerpc_resume_pmc)(bool ie); 65 66 67 int 68 pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, 69 struct trapframe *tf) 70 { 71 uintptr_t *osp, *sp; 72 uintptr_t pc; 73 int frames = 0; 74 75 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 76 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 77 osp = (uintptr_t *)PAGE_SIZE; 78 79 for (; frames < maxsamples; frames++) { 80 if (sp <= osp) 81 break; 82 #ifdef __powerpc64__ 83 pc = sp[2]; 84 #else 85 pc = sp[1]; 86 #endif 87 if ((pc & 3) || (pc < 0x100)) 88 break; 89 90 /* 91 * trapexit() and asttrapexit() are sentinels 92 * for kernel stack tracing. 93 * */ 94 if (pc + OFFSET == (uintptr_t) &trapexit || 95 pc + OFFSET == (uintptr_t) &asttrapexit) 96 break; 97 98 cc[frames] = pc; 99 osp = sp; 100 sp = (uintptr_t *)*sp; 101 } 102 return (frames); 103 } 104 105 int 106 powerpc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 107 { 108 struct pmc_hw *phw; 109 110 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 111 ("[powerpc,%d], illegal CPU %d", __LINE__, cpu)); 112 113 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 114 115 snprintf(pi->pm_name, sizeof(pi->pm_name), "POWERPC-%d", ri); 116 pi->pm_class = powerpc_pcpu[cpu]->pc_class; 117 118 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 119 pi->pm_enabled = TRUE; 120 *ppmc = phw->phw_pmc; 121 } else { 122 pi->pm_enabled = FALSE; 123 *ppmc = NULL; 124 } 125 126 return (0); 127 } 128 129 int 130 powerpc_get_config(int cpu, int ri, struct pmc **ppm) 131 { 132 133 *ppm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; 134 135 return (0); 136 } 137 138 int 139 powerpc_pcpu_init(struct pmc_mdep *md, int cpu) 140 { 141 struct pmc_cpu *pc; 142 struct powerpc_cpu *pac; 143 struct pmc_hw *phw; 144 int first_ri, i; 145 146 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 147 ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu)); 148 PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu); 149 150 powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu) + 151 ppc_max_pmcs * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); 152 pac->pc_class = 153 md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_class; 154 155 pc = pmc_pcpu[cpu]; 156 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri; 157 KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__)); 158 159 for (i = 0, phw = pac->pc_ppcpmcs; i < ppc_max_pmcs; i++, phw++) { 160 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 161 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); 162 phw->phw_pmc = NULL; 163 pc->pc_hwpmcs[i + first_ri] = phw; 164 } 165 166 return (0); 167 } 168 169 int 170 powerpc_pcpu_fini(struct pmc_mdep *md, int cpu) 171 { 172 PMCDBG1(MDP,INI,1,"powerpc-fini cpu=%d", cpu); 173 174 free(powerpc_pcpu[cpu], M_PMC); 175 powerpc_pcpu[cpu] = NULL; 176 177 return (0); 178 } 179 180 int 181 powerpc_allocate_pmc(int cpu, int ri, struct pmc *pm, 182 const struct pmc_op_pmcallocate *a) 183 { 184 enum pmc_event pe; 185 uint32_t caps, config = 0, counter = 0; 186 int i; 187 188 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 189 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 190 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 191 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 192 193 if (a->pm_class != ppc_class) 194 return (EINVAL); 195 196 caps = a->pm_caps; 197 198 pe = a->pm_ev; 199 200 if (pe < ppc_event_first || pe > ppc_event_last) 201 return (EINVAL); 202 203 for (i = 0; i < ppc_event_codes_size; i++) { 204 if (ppc_event_codes[i].pe_event == pe) { 205 config = ppc_event_codes[i].pe_code; 206 counter = ppc_event_codes[i].pe_flags; 207 break; 208 } 209 } 210 if (i == ppc_event_codes_size) 211 return (EINVAL); 212 213 if ((counter & (1 << ri)) == 0) 214 return (EINVAL); 215 216 if (caps & PMC_CAP_SYSTEM) 217 config |= POWERPC_PMC_KERNEL_ENABLE; 218 if (caps & PMC_CAP_USER) 219 config |= POWERPC_PMC_USER_ENABLE; 220 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 221 config |= POWERPC_PMC_ENABLE; 222 223 pm->pm_md.pm_powerpc.pm_powerpc_evsel = config; 224 225 PMCDBG3(MDP,ALL,1,"powerpc-allocate cpu=%d ri=%d -> config=0x%x", 226 cpu, ri, config); 227 return (0); 228 } 229 230 int 231 powerpc_release_pmc(int cpu, int ri, struct pmc *pmc) 232 { 233 struct pmc_hw *phw __diagused; 234 235 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 236 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 237 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 238 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 239 240 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 241 KASSERT(phw->phw_pmc == NULL, 242 ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 243 244 return (0); 245 } 246 247 int 248 powerpc_start_pmc(int cpu, int ri, struct pmc *pm) 249 { 250 251 PMCDBG2(MDP,STA,1,"powerpc-start cpu=%d ri=%d", cpu, ri); 252 powerpc_set_pmc(cpu, ri, pm->pm_md.pm_powerpc.pm_powerpc_evsel); 253 254 return (0); 255 } 256 257 int 258 powerpc_stop_pmc(int cpu, int ri, struct pmc *pm __unused) 259 { 260 PMCDBG2(MDP,STO,1, "powerpc-stop cpu=%d ri=%d", cpu, ri); 261 powerpc_set_pmc(cpu, ri, PMCN_NONE); 262 return (0); 263 } 264 265 int 266 powerpc_config_pmc(int cpu, int ri, struct pmc *pm) 267 { 268 struct pmc_hw *phw; 269 270 PMCDBG3(MDP,CFG,1, "powerpc-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 271 272 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 273 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 274 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 275 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 276 277 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 278 279 KASSERT(pm == NULL || phw->phw_pmc == NULL, 280 ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 281 __LINE__, pm, phw->phw_pmc)); 282 283 phw->phw_pmc = pm; 284 285 return (0); 286 } 287 288 pmc_value_t 289 powerpc_pmcn_read_default(unsigned int pmc) 290 { 291 pmc_value_t val; 292 293 if (pmc > ppc_max_pmcs) 294 panic("Invalid PMC number: %d\n", pmc); 295 296 switch (pmc) { 297 case 0: 298 val = mfspr(SPR_PMC1); 299 break; 300 case 1: 301 val = mfspr(SPR_PMC2); 302 break; 303 case 2: 304 val = mfspr(SPR_PMC3); 305 break; 306 case 3: 307 val = mfspr(SPR_PMC4); 308 break; 309 case 4: 310 val = mfspr(SPR_PMC5); 311 break; 312 case 5: 313 val = mfspr(SPR_PMC6); 314 break; 315 case 6: 316 val = mfspr(SPR_PMC7); 317 break; 318 case 7: 319 val = mfspr(SPR_PMC8); 320 break; 321 } 322 323 return (val); 324 } 325 326 void 327 powerpc_pmcn_write_default(unsigned int pmc, uint32_t val) 328 { 329 if (pmc > ppc_max_pmcs) 330 panic("Invalid PMC number: %d\n", pmc); 331 332 switch (pmc) { 333 case 0: 334 mtspr(SPR_PMC1, val); 335 break; 336 case 1: 337 mtspr(SPR_PMC2, val); 338 break; 339 case 2: 340 mtspr(SPR_PMC3, val); 341 break; 342 case 3: 343 mtspr(SPR_PMC4, val); 344 break; 345 case 4: 346 mtspr(SPR_PMC5, val); 347 break; 348 case 5: 349 mtspr(SPR_PMC6, val); 350 break; 351 case 6: 352 mtspr(SPR_PMC7, val); 353 break; 354 case 7: 355 mtspr(SPR_PMC8, val); 356 break; 357 } 358 } 359 360 int 361 powerpc_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 362 { 363 pmc_value_t p, r, tmp; 364 365 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 366 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 367 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 368 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 369 370 /* 371 * After an interrupt occurs because of a PMC overflow, the PMC value 372 * is not always MAX_PMC_VALUE + 1, but may be a little above it. 373 * This may mess up calculations and frustrate machine independent 374 * layer expectations, such as that no value read should be greater 375 * than reload count in sampling mode. 376 * To avoid these issues, use MAX_PMC_VALUE as an upper limit. 377 */ 378 p = MIN(powerpc_pmcn_read(ri), POWERPC_MAX_PMC_VALUE); 379 r = pm->pm_sc.pm_reloadcount; 380 381 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 382 /* 383 * Special case 1: r is too big 384 * This usually happens when a PMC write fails, the PMC is 385 * stopped and then it is read. 386 * 387 * Special case 2: PMC was reseted or has a value 388 * that should not be possible with current r. 389 * 390 * In the above cases, just return 0 instead of an arbitrary 391 * value. 392 */ 393 if (r > POWERPC_MAX_PMC_VALUE || p + r <= POWERPC_MAX_PMC_VALUE) 394 tmp = 0; 395 else 396 tmp = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(p); 397 } else 398 tmp = p + (POWERPC_MAX_PMC_VALUE + 1) * PPC_OVERFLOWCNT(pm); 399 400 PMCDBG5(MDP,REA,1,"ppc-read cpu=%d ri=%d -> %jx (%jx,%jx)", 401 cpu, ri, (uintmax_t)tmp, (uintmax_t)PPC_OVERFLOWCNT(pm), 402 (uintmax_t)p); 403 *v = tmp; 404 return (0); 405 } 406 407 int 408 powerpc_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 409 { 410 pmc_value_t vlo; 411 412 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 413 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 414 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 415 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 416 417 if (PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { 418 PPC_OVERFLOWCNT(pm) = v / (POWERPC_MAX_PMC_VALUE + 1); 419 vlo = v % (POWERPC_MAX_PMC_VALUE + 1); 420 } else if (v > POWERPC_MAX_PMC_VALUE) { 421 PMCDBG3(MDP,WRI,2, 422 "powerpc-write cpu=%d ri=%d: PMC value is too big: %jx", 423 cpu, ri, (uintmax_t)v); 424 return (EINVAL); 425 } else 426 vlo = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 427 428 PMCDBG5(MDP,WRI,1,"powerpc-write cpu=%d ri=%d -> %jx (%jx,%jx)", 429 cpu, ri, (uintmax_t)v, (uintmax_t)PPC_OVERFLOWCNT(pm), 430 (uintmax_t)vlo); 431 432 powerpc_pmcn_write(ri, vlo); 433 return (0); 434 } 435 436 int 437 powerpc_pmc_intr(struct trapframe *tf) 438 { 439 struct pmc *pm; 440 struct powerpc_cpu *pc; 441 int cpu, error, i, retval; 442 443 cpu = curcpu; 444 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 445 ("[powerpc,%d] out of range CPU %d", __LINE__, cpu)); 446 447 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, 448 TRAPF_USERMODE(tf)); 449 450 retval = 0; 451 pc = powerpc_pcpu[cpu]; 452 453 /* 454 * Look for a running, sampling PMC which has overflowed 455 * and which has a valid 'struct pmc' association. 456 */ 457 for (i = 0; i < ppc_max_pmcs; i++) { 458 if (!POWERPC_PMC_HAS_OVERFLOWED(i)) 459 continue; 460 retval = 1; /* Found an interrupting PMC. */ 461 462 /* 463 * Always clear the PMC, to make it stop interrupting. 464 * If pm is available and in sampling mode, use reload 465 * count, to make PMC read after stop correct. 466 * Otherwise, just reset the PMC. 467 */ 468 if ((pm = pc->pc_ppcpmcs[i].phw_pmc) != NULL && 469 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 470 if (pm->pm_state != PMC_STATE_RUNNING) { 471 powerpc_write_pmc(cpu, i, pm, 472 pm->pm_sc.pm_reloadcount); 473 continue; 474 } 475 } else { 476 if (pm != NULL) { /* !PMC_IS_SAMPLING_MODE */ 477 PPC_OVERFLOWCNT(pm) = (PPC_OVERFLOWCNT(pm) + 478 1) % PPC_OVERFLOWCNT_MAX; 479 PMCDBG3(MDP,INT,2, 480 "cpu=%d ri=%d: overflowcnt=%d", 481 cpu, i, PPC_OVERFLOWCNT(pm)); 482 } 483 484 powerpc_pmcn_write(i, 0); 485 continue; 486 } 487 488 error = pmc_process_interrupt(PMC_HR, pm, tf); 489 if (error != 0) { 490 PMCDBG3(MDP,INT,3, 491 "cpu=%d ri=%d: error %d processing interrupt", 492 cpu, i, error); 493 powerpc_stop_pmc(cpu, i, pm); 494 } 495 496 /* Reload sampling count */ 497 powerpc_write_pmc(cpu, i, pm, pm->pm_sc.pm_reloadcount); 498 } 499 500 if (retval) 501 counter_u64_add(pmc_stats.pm_intr_processed, 1); 502 else 503 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 504 505 /* 506 * Re-enable PERF exceptions if we were able to find the interrupt 507 * source and handle it. Otherwise, it's better to disable PERF 508 * interrupts, to avoid the risk of processing the same interrupt 509 * forever. 510 */ 511 powerpc_resume_pmc(retval != 0); 512 if (retval == 0) 513 log(LOG_WARNING, 514 "pmc_intr: couldn't find interrupting PMC on cpu %d - " 515 "disabling PERF interrupts\n", cpu); 516 517 return (retval); 518 } 519 520 struct pmc_mdep * 521 pmc_md_initialize(void) 522 { 523 struct pmc_mdep *pmc_mdep; 524 int error; 525 uint16_t vers; 526 527 /* 528 * Allocate space for pointers to PMC HW descriptors and for 529 * the MDEP structure used by MI code. 530 */ 531 powerpc_pcpu = malloc(sizeof(struct powerpc_cpu *) * pmc_cpu_max(), M_PMC, 532 M_WAITOK|M_ZERO); 533 534 /* Just one class */ 535 pmc_mdep = pmc_mdep_alloc(1); 536 537 vers = mfpvr() >> 16; 538 539 switch (vers) { 540 case MPC7447A: 541 case MPC7448: 542 case MPC7450: 543 case MPC7455: 544 case MPC7457: 545 error = pmc_mpc7xxx_initialize(pmc_mdep); 546 break; 547 case IBM970: 548 case IBM970FX: 549 case IBM970MP: 550 error = pmc_ppc970_initialize(pmc_mdep); 551 break; 552 case IBMPOWER8E: 553 case IBMPOWER8NVL: 554 case IBMPOWER8: 555 case IBMPOWER9: 556 error = pmc_power8_initialize(pmc_mdep); 557 break; 558 case FSL_E500v1: 559 case FSL_E500v2: 560 case FSL_E500mc: 561 case FSL_E5500: 562 error = pmc_e500_initialize(pmc_mdep); 563 break; 564 default: 565 error = -1; 566 break; 567 } 568 569 if (error != 0) { 570 pmc_mdep_free(pmc_mdep); 571 pmc_mdep = NULL; 572 } 573 574 /* Set the value for kern.hwpmc.cpuid */ 575 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "%08x", mfpvr()); 576 577 return (pmc_mdep); 578 } 579 580 void 581 pmc_md_finalize(struct pmc_mdep *md) 582 { 583 PMCDBG0(MDP, INI, 1, "powerpc-finalize"); 584 585 for (int i = 0; i < pmc_cpu_max(); i++) 586 KASSERT(powerpc_pcpu[i] == NULL, 587 ("[powerpc,%d] non-null pcpu cpu %d", __LINE__, i)); 588 589 free(powerpc_pcpu, M_PMC); 590 powerpc_pcpu = NULL; 591 } 592 593 int 594 pmc_save_user_callchain(uintptr_t *cc, int maxsamples, 595 struct trapframe *tf) 596 { 597 uintptr_t *osp, *sp; 598 int frames = 0; 599 600 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 601 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 602 osp = NULL; 603 604 for (; frames < maxsamples; frames++) { 605 if (sp <= osp) 606 break; 607 osp = sp; 608 #ifdef __powerpc64__ 609 /* Check if 32-bit mode. */ 610 if (!(tf->srr1 & PSL_SF)) { 611 cc[frames] = fuword32((uint32_t *)sp + 1); 612 sp = (uintptr_t *)(uintptr_t)fuword32(sp); 613 } else { 614 cc[frames] = fuword(sp + 2); 615 sp = (uintptr_t *)fuword(sp); 616 } 617 #else 618 cc[frames] = fuword32((uint32_t *)sp + 1); 619 sp = (uintptr_t *)fuword32(sp); 620 #endif 621 } 622 623 return (frames); 624 } 625