1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011,2013 Justin Hibbits 5 * Copyright (c) 2005, Joseph Koshy 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31 #include <sys/cdefs.h> 32 #include <sys/param.h> 33 #include <sys/pmc.h> 34 #include <sys/pmckern.h> 35 #include <sys/sysent.h> 36 #include <sys/syslog.h> 37 #include <sys/systm.h> 38 39 #include <machine/pmc_mdep.h> 40 #include <machine/spr.h> 41 #include <machine/pte.h> 42 #include <machine/sr.h> 43 #include <machine/cpu.h> 44 #include <machine/stack.h> 45 46 #include "hwpmc_powerpc.h" 47 48 #ifdef __powerpc64__ 49 #define OFFSET 4 /* Account for the TOC reload slot */ 50 #else 51 #define OFFSET 0 52 #endif 53 54 struct powerpc_cpu **powerpc_pcpu; 55 struct pmc_ppc_event *ppc_event_codes; 56 size_t ppc_event_codes_size; 57 int ppc_event_first; 58 int ppc_event_last; 59 int ppc_max_pmcs; 60 enum pmc_class ppc_class; 61 62 void (*powerpc_set_pmc)(int cpu, int ri, int config); 63 pmc_value_t (*powerpc_pmcn_read)(unsigned int pmc); 64 void (*powerpc_pmcn_write)(unsigned int pmc, uint32_t val); 65 void (*powerpc_resume_pmc)(bool ie); 66 67 68 int 69 pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, 70 struct trapframe *tf) 71 { 72 uintptr_t *osp, *sp; 73 uintptr_t pc; 74 int frames = 0; 75 76 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 77 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 78 osp = (uintptr_t *)PAGE_SIZE; 79 80 for (; frames < maxsamples; frames++) { 81 if (sp <= osp) 82 break; 83 #ifdef __powerpc64__ 84 pc = sp[2]; 85 #else 86 pc = sp[1]; 87 #endif 88 if ((pc & 3) || (pc < 0x100)) 89 break; 90 91 /* 92 * trapexit() and asttrapexit() are sentinels 93 * for kernel stack tracing. 94 * */ 95 if (pc + OFFSET == (uintptr_t) &trapexit || 96 pc + OFFSET == (uintptr_t) &asttrapexit) 97 break; 98 99 cc[frames] = pc; 100 osp = sp; 101 sp = (uintptr_t *)*sp; 102 } 103 return (frames); 104 } 105 106 int 107 powerpc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 108 { 109 struct pmc_hw *phw; 110 111 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 112 ("[powerpc,%d], illegal CPU %d", __LINE__, cpu)); 113 114 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 115 116 snprintf(pi->pm_name, sizeof(pi->pm_name), "POWERPC-%d", ri); 117 pi->pm_class = powerpc_pcpu[cpu]->pc_class; 118 119 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 120 pi->pm_enabled = TRUE; 121 *ppmc = phw->phw_pmc; 122 } else { 123 pi->pm_enabled = FALSE; 124 *ppmc = NULL; 125 } 126 127 return (0); 128 } 129 130 int 131 powerpc_get_config(int cpu, int ri, struct pmc **ppm) 132 { 133 134 *ppm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; 135 136 return (0); 137 } 138 139 int 140 powerpc_pcpu_init(struct pmc_mdep *md, int cpu) 141 { 142 struct pmc_cpu *pc; 143 struct powerpc_cpu *pac; 144 struct pmc_hw *phw; 145 int first_ri, i; 146 147 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 148 ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu)); 149 PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu); 150 151 powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu) + 152 ppc_max_pmcs * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); 153 pac->pc_class = 154 md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_class; 155 156 pc = pmc_pcpu[cpu]; 157 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri; 158 KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__)); 159 160 for (i = 0, phw = pac->pc_ppcpmcs; i < ppc_max_pmcs; i++, phw++) { 161 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 162 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); 163 phw->phw_pmc = NULL; 164 pc->pc_hwpmcs[i + first_ri] = phw; 165 } 166 167 return (0); 168 } 169 170 int 171 powerpc_pcpu_fini(struct pmc_mdep *md, int cpu) 172 { 173 PMCDBG1(MDP,INI,1,"powerpc-fini cpu=%d", cpu); 174 175 free(powerpc_pcpu[cpu], M_PMC); 176 powerpc_pcpu[cpu] = NULL; 177 178 return (0); 179 } 180 181 int 182 powerpc_allocate_pmc(int cpu, int ri, struct pmc *pm, 183 const struct pmc_op_pmcallocate *a) 184 { 185 enum pmc_event pe; 186 uint32_t caps, config = 0, counter = 0; 187 int i; 188 189 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 190 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 191 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 192 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 193 194 if (a->pm_class != ppc_class) 195 return (EINVAL); 196 197 caps = a->pm_caps; 198 199 pe = a->pm_ev; 200 201 if (pe < ppc_event_first || pe > ppc_event_last) 202 return (EINVAL); 203 204 for (i = 0; i < ppc_event_codes_size; i++) { 205 if (ppc_event_codes[i].pe_event == pe) { 206 config = ppc_event_codes[i].pe_code; 207 counter = ppc_event_codes[i].pe_flags; 208 break; 209 } 210 } 211 if (i == ppc_event_codes_size) 212 return (EINVAL); 213 214 if ((counter & (1 << ri)) == 0) 215 return (EINVAL); 216 217 if (caps & PMC_CAP_SYSTEM) 218 config |= POWERPC_PMC_KERNEL_ENABLE; 219 if (caps & PMC_CAP_USER) 220 config |= POWERPC_PMC_USER_ENABLE; 221 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 222 config |= POWERPC_PMC_ENABLE; 223 224 pm->pm_md.pm_powerpc.pm_powerpc_evsel = config; 225 226 PMCDBG3(MDP,ALL,1,"powerpc-allocate cpu=%d ri=%d -> config=0x%x", 227 cpu, ri, config); 228 return (0); 229 } 230 231 int 232 powerpc_release_pmc(int cpu, int ri, struct pmc *pmc) 233 { 234 struct pmc_hw *phw __diagused; 235 236 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 237 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 238 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 239 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 240 241 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 242 KASSERT(phw->phw_pmc == NULL, 243 ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 244 245 return (0); 246 } 247 248 int 249 powerpc_start_pmc(int cpu, int ri, struct pmc *pm) 250 { 251 252 PMCDBG2(MDP,STA,1,"powerpc-start cpu=%d ri=%d", cpu, ri); 253 powerpc_set_pmc(cpu, ri, pm->pm_md.pm_powerpc.pm_powerpc_evsel); 254 255 return (0); 256 } 257 258 int 259 powerpc_stop_pmc(int cpu, int ri, struct pmc *pm __unused) 260 { 261 PMCDBG2(MDP,STO,1, "powerpc-stop cpu=%d ri=%d", cpu, ri); 262 powerpc_set_pmc(cpu, ri, PMCN_NONE); 263 return (0); 264 } 265 266 int 267 powerpc_config_pmc(int cpu, int ri, struct pmc *pm) 268 { 269 struct pmc_hw *phw; 270 271 PMCDBG3(MDP,CFG,1, "powerpc-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 272 273 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 274 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 275 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 276 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 277 278 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 279 280 KASSERT(pm == NULL || phw->phw_pmc == NULL, 281 ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 282 __LINE__, pm, phw->phw_pmc)); 283 284 phw->phw_pmc = pm; 285 286 return (0); 287 } 288 289 pmc_value_t 290 powerpc_pmcn_read_default(unsigned int pmc) 291 { 292 pmc_value_t val; 293 294 if (pmc > ppc_max_pmcs) 295 panic("Invalid PMC number: %d\n", pmc); 296 297 switch (pmc) { 298 case 0: 299 val = mfspr(SPR_PMC1); 300 break; 301 case 1: 302 val = mfspr(SPR_PMC2); 303 break; 304 case 2: 305 val = mfspr(SPR_PMC3); 306 break; 307 case 3: 308 val = mfspr(SPR_PMC4); 309 break; 310 case 4: 311 val = mfspr(SPR_PMC5); 312 break; 313 case 5: 314 val = mfspr(SPR_PMC6); 315 break; 316 case 6: 317 val = mfspr(SPR_PMC7); 318 break; 319 case 7: 320 val = mfspr(SPR_PMC8); 321 break; 322 } 323 324 return (val); 325 } 326 327 void 328 powerpc_pmcn_write_default(unsigned int pmc, uint32_t val) 329 { 330 if (pmc > ppc_max_pmcs) 331 panic("Invalid PMC number: %d\n", pmc); 332 333 switch (pmc) { 334 case 0: 335 mtspr(SPR_PMC1, val); 336 break; 337 case 1: 338 mtspr(SPR_PMC2, val); 339 break; 340 case 2: 341 mtspr(SPR_PMC3, val); 342 break; 343 case 3: 344 mtspr(SPR_PMC4, val); 345 break; 346 case 4: 347 mtspr(SPR_PMC5, val); 348 break; 349 case 5: 350 mtspr(SPR_PMC6, val); 351 break; 352 case 6: 353 mtspr(SPR_PMC7, val); 354 break; 355 case 7: 356 mtspr(SPR_PMC8, val); 357 break; 358 } 359 } 360 361 int 362 powerpc_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 363 { 364 pmc_value_t p, r, tmp; 365 366 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 367 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 368 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 369 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 370 371 /* 372 * After an interrupt occurs because of a PMC overflow, the PMC value 373 * is not always MAX_PMC_VALUE + 1, but may be a little above it. 374 * This may mess up calculations and frustrate machine independent 375 * layer expectations, such as that no value read should be greater 376 * than reload count in sampling mode. 377 * To avoid these issues, use MAX_PMC_VALUE as an upper limit. 378 */ 379 p = MIN(powerpc_pmcn_read(ri), POWERPC_MAX_PMC_VALUE); 380 r = pm->pm_sc.pm_reloadcount; 381 382 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 383 /* 384 * Special case 1: r is too big 385 * This usually happens when a PMC write fails, the PMC is 386 * stopped and then it is read. 387 * 388 * Special case 2: PMC was reseted or has a value 389 * that should not be possible with current r. 390 * 391 * In the above cases, just return 0 instead of an arbitrary 392 * value. 393 */ 394 if (r > POWERPC_MAX_PMC_VALUE || p + r <= POWERPC_MAX_PMC_VALUE) 395 tmp = 0; 396 else 397 tmp = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(p); 398 } else 399 tmp = p + (POWERPC_MAX_PMC_VALUE + 1) * PPC_OVERFLOWCNT(pm); 400 401 PMCDBG5(MDP,REA,1,"ppc-read cpu=%d ri=%d -> %jx (%jx,%jx)", 402 cpu, ri, (uintmax_t)tmp, (uintmax_t)PPC_OVERFLOWCNT(pm), 403 (uintmax_t)p); 404 *v = tmp; 405 return (0); 406 } 407 408 int 409 powerpc_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 410 { 411 pmc_value_t vlo; 412 413 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 414 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 415 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 416 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 417 418 if (PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { 419 PPC_OVERFLOWCNT(pm) = v / (POWERPC_MAX_PMC_VALUE + 1); 420 vlo = v % (POWERPC_MAX_PMC_VALUE + 1); 421 } else if (v > POWERPC_MAX_PMC_VALUE) { 422 PMCDBG3(MDP,WRI,2, 423 "powerpc-write cpu=%d ri=%d: PMC value is too big: %jx", 424 cpu, ri, (uintmax_t)v); 425 return (EINVAL); 426 } else 427 vlo = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 428 429 PMCDBG5(MDP,WRI,1,"powerpc-write cpu=%d ri=%d -> %jx (%jx,%jx)", 430 cpu, ri, (uintmax_t)v, (uintmax_t)PPC_OVERFLOWCNT(pm), 431 (uintmax_t)vlo); 432 433 powerpc_pmcn_write(ri, vlo); 434 return (0); 435 } 436 437 int 438 powerpc_pmc_intr(struct trapframe *tf) 439 { 440 struct pmc *pm; 441 struct powerpc_cpu *pc; 442 int cpu, error, i, retval; 443 444 cpu = curcpu; 445 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 446 ("[powerpc,%d] out of range CPU %d", __LINE__, cpu)); 447 448 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, 449 TRAPF_USERMODE(tf)); 450 451 retval = 0; 452 pc = powerpc_pcpu[cpu]; 453 454 /* 455 * Look for a running, sampling PMC which has overflowed 456 * and which has a valid 'struct pmc' association. 457 */ 458 for (i = 0; i < ppc_max_pmcs; i++) { 459 if (!POWERPC_PMC_HAS_OVERFLOWED(i)) 460 continue; 461 retval = 1; /* Found an interrupting PMC. */ 462 463 /* 464 * Always clear the PMC, to make it stop interrupting. 465 * If pm is available and in sampling mode, use reload 466 * count, to make PMC read after stop correct. 467 * Otherwise, just reset the PMC. 468 */ 469 if ((pm = pc->pc_ppcpmcs[i].phw_pmc) != NULL && 470 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 471 if (pm->pm_state != PMC_STATE_RUNNING) { 472 powerpc_write_pmc(cpu, i, pm, 473 pm->pm_sc.pm_reloadcount); 474 continue; 475 } 476 } else { 477 if (pm != NULL) { /* !PMC_IS_SAMPLING_MODE */ 478 PPC_OVERFLOWCNT(pm) = (PPC_OVERFLOWCNT(pm) + 479 1) % PPC_OVERFLOWCNT_MAX; 480 PMCDBG3(MDP,INT,2, 481 "cpu=%d ri=%d: overflowcnt=%d", 482 cpu, i, PPC_OVERFLOWCNT(pm)); 483 } 484 485 powerpc_pmcn_write(i, 0); 486 continue; 487 } 488 489 error = pmc_process_interrupt(PMC_HR, pm, tf); 490 if (error != 0) { 491 PMCDBG3(MDP,INT,3, 492 "cpu=%d ri=%d: error %d processing interrupt", 493 cpu, i, error); 494 powerpc_stop_pmc(cpu, i, pm); 495 } 496 497 /* Reload sampling count */ 498 powerpc_write_pmc(cpu, i, pm, pm->pm_sc.pm_reloadcount); 499 } 500 501 if (retval) 502 counter_u64_add(pmc_stats.pm_intr_processed, 1); 503 else 504 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 505 506 /* 507 * Re-enable PERF exceptions if we were able to find the interrupt 508 * source and handle it. Otherwise, it's better to disable PERF 509 * interrupts, to avoid the risk of processing the same interrupt 510 * forever. 511 */ 512 powerpc_resume_pmc(retval != 0); 513 if (retval == 0) 514 log(LOG_WARNING, 515 "pmc_intr: couldn't find interrupting PMC on cpu %d - " 516 "disabling PERF interrupts\n", cpu); 517 518 return (retval); 519 } 520 521 struct pmc_mdep * 522 pmc_md_initialize(void) 523 { 524 struct pmc_mdep *pmc_mdep; 525 int error; 526 uint16_t vers; 527 528 /* 529 * Allocate space for pointers to PMC HW descriptors and for 530 * the MDEP structure used by MI code. 531 */ 532 powerpc_pcpu = malloc(sizeof(struct powerpc_cpu *) * pmc_cpu_max(), M_PMC, 533 M_WAITOK|M_ZERO); 534 535 /* Just one class */ 536 pmc_mdep = pmc_mdep_alloc(1); 537 538 vers = mfpvr() >> 16; 539 540 switch (vers) { 541 case MPC7447A: 542 case MPC7448: 543 case MPC7450: 544 case MPC7455: 545 case MPC7457: 546 error = pmc_mpc7xxx_initialize(pmc_mdep); 547 break; 548 case IBM970: 549 case IBM970FX: 550 case IBM970MP: 551 error = pmc_ppc970_initialize(pmc_mdep); 552 break; 553 case IBMPOWER8E: 554 case IBMPOWER8NVL: 555 case IBMPOWER8: 556 case IBMPOWER9: 557 error = pmc_power8_initialize(pmc_mdep); 558 break; 559 case FSL_E500v1: 560 case FSL_E500v2: 561 case FSL_E500mc: 562 case FSL_E5500: 563 error = pmc_e500_initialize(pmc_mdep); 564 break; 565 default: 566 error = -1; 567 break; 568 } 569 570 if (error != 0) { 571 pmc_mdep_free(pmc_mdep); 572 pmc_mdep = NULL; 573 } 574 575 /* Set the value for kern.hwpmc.cpuid */ 576 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "%08x", mfpvr()); 577 578 return (pmc_mdep); 579 } 580 581 void 582 pmc_md_finalize(struct pmc_mdep *md) 583 { 584 PMCDBG0(MDP, INI, 1, "powerpc-finalize"); 585 586 for (int i = 0; i < pmc_cpu_max(); i++) 587 KASSERT(powerpc_pcpu[i] == NULL, 588 ("[powerpc,%d] non-null pcpu cpu %d", __LINE__, i)); 589 590 free(powerpc_pcpu, M_PMC); 591 powerpc_pcpu = NULL; 592 } 593 594 int 595 pmc_save_user_callchain(uintptr_t *cc, int maxsamples, 596 struct trapframe *tf) 597 { 598 uintptr_t *osp, *sp; 599 int frames = 0; 600 601 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 602 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 603 osp = NULL; 604 605 for (; frames < maxsamples; frames++) { 606 if (sp <= osp) 607 break; 608 osp = sp; 609 #ifdef __powerpc64__ 610 /* Check if 32-bit mode. */ 611 if (!(tf->srr1 & PSL_SF)) { 612 cc[frames] = fuword32((uint32_t *)sp + 1); 613 sp = (uintptr_t *)(uintptr_t)fuword32(sp); 614 } else { 615 cc[frames] = fuword(sp + 2); 616 sp = (uintptr_t *)fuword(sp); 617 } 618 #else 619 cc[frames] = fuword32((uint32_t *)sp + 1); 620 sp = (uintptr_t *)fuword32(sp); 621 #endif 622 } 623 624 return (frames); 625 } 626