1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011,2013 Justin Hibbits 5 * Copyright (c) 2005, Joseph Koshy 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/pmc.h> 36 #include <sys/pmckern.h> 37 #include <sys/sysent.h> 38 #include <sys/syslog.h> 39 #include <sys/systm.h> 40 41 #include <machine/pmc_mdep.h> 42 #include <machine/spr.h> 43 #include <machine/pte.h> 44 #include <machine/sr.h> 45 #include <machine/cpu.h> 46 #include <machine/stack.h> 47 48 #include "hwpmc_powerpc.h" 49 50 #ifdef __powerpc64__ 51 #define OFFSET 4 /* Account for the TOC reload slot */ 52 #else 53 #define OFFSET 0 54 #endif 55 56 struct powerpc_cpu **powerpc_pcpu; 57 struct pmc_ppc_event *ppc_event_codes; 58 size_t ppc_event_codes_size; 59 int ppc_event_first; 60 int ppc_event_last; 61 int ppc_max_pmcs; 62 enum pmc_class ppc_class; 63 64 void (*powerpc_set_pmc)(int cpu, int ri, int config); 65 pmc_value_t (*powerpc_pmcn_read)(unsigned int pmc); 66 void (*powerpc_pmcn_write)(unsigned int pmc, uint32_t val); 67 void (*powerpc_resume_pmc)(bool ie); 68 69 70 int 71 pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, 72 struct trapframe *tf) 73 { 74 uintptr_t *osp, *sp; 75 uintptr_t pc; 76 int frames = 0; 77 78 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 79 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 80 osp = (uintptr_t *)PAGE_SIZE; 81 82 for (; frames < maxsamples; frames++) { 83 if (sp <= osp) 84 break; 85 #ifdef __powerpc64__ 86 pc = sp[2]; 87 #else 88 pc = sp[1]; 89 #endif 90 if ((pc & 3) || (pc < 0x100)) 91 break; 92 93 /* 94 * trapexit() and asttrapexit() are sentinels 95 * for kernel stack tracing. 96 * */ 97 if (pc + OFFSET == (uintptr_t) &trapexit || 98 pc + OFFSET == (uintptr_t) &asttrapexit) 99 break; 100 101 cc[frames] = pc; 102 osp = sp; 103 sp = (uintptr_t *)*sp; 104 } 105 return (frames); 106 } 107 108 int 109 powerpc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 110 { 111 struct pmc_hw *phw; 112 113 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 114 ("[powerpc,%d], illegal CPU %d", __LINE__, cpu)); 115 116 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 117 118 snprintf(pi->pm_name, sizeof(pi->pm_name), "POWERPC-%d", ri); 119 pi->pm_class = powerpc_pcpu[cpu]->pc_class; 120 121 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 122 pi->pm_enabled = TRUE; 123 *ppmc = phw->phw_pmc; 124 } else { 125 pi->pm_enabled = FALSE; 126 *ppmc = NULL; 127 } 128 129 return (0); 130 } 131 132 int 133 powerpc_get_config(int cpu, int ri, struct pmc **ppm) 134 { 135 136 *ppm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; 137 138 return (0); 139 } 140 141 int 142 powerpc_pcpu_init(struct pmc_mdep *md, int cpu) 143 { 144 struct pmc_cpu *pc; 145 struct powerpc_cpu *pac; 146 struct pmc_hw *phw; 147 int first_ri, i; 148 149 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 150 ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu)); 151 PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu); 152 153 powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu) + 154 ppc_max_pmcs * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); 155 pac->pc_class = 156 md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_class; 157 158 pc = pmc_pcpu[cpu]; 159 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri; 160 KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__)); 161 162 for (i = 0, phw = pac->pc_ppcpmcs; i < ppc_max_pmcs; i++, phw++) { 163 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 164 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); 165 phw->phw_pmc = NULL; 166 pc->pc_hwpmcs[i + first_ri] = phw; 167 } 168 169 return (0); 170 } 171 172 int 173 powerpc_pcpu_fini(struct pmc_mdep *md, int cpu) 174 { 175 PMCDBG1(MDP,INI,1,"powerpc-fini cpu=%d", cpu); 176 177 free(powerpc_pcpu[cpu], M_PMC); 178 powerpc_pcpu[cpu] = NULL; 179 180 return (0); 181 } 182 183 int 184 powerpc_allocate_pmc(int cpu, int ri, struct pmc *pm, 185 const struct pmc_op_pmcallocate *a) 186 { 187 enum pmc_event pe; 188 uint32_t caps, config = 0, counter = 0; 189 int i; 190 191 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 192 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 193 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 194 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 195 196 if (a->pm_class != ppc_class) 197 return (EINVAL); 198 199 caps = a->pm_caps; 200 201 pe = a->pm_ev; 202 203 if (pe < ppc_event_first || pe > ppc_event_last) 204 return (EINVAL); 205 206 for (i = 0; i < ppc_event_codes_size; i++) { 207 if (ppc_event_codes[i].pe_event == pe) { 208 config = ppc_event_codes[i].pe_code; 209 counter = ppc_event_codes[i].pe_flags; 210 break; 211 } 212 } 213 if (i == ppc_event_codes_size) 214 return (EINVAL); 215 216 if ((counter & (1 << ri)) == 0) 217 return (EINVAL); 218 219 if (caps & PMC_CAP_SYSTEM) 220 config |= POWERPC_PMC_KERNEL_ENABLE; 221 if (caps & PMC_CAP_USER) 222 config |= POWERPC_PMC_USER_ENABLE; 223 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 224 config |= POWERPC_PMC_ENABLE; 225 226 pm->pm_md.pm_powerpc.pm_powerpc_evsel = config; 227 228 PMCDBG3(MDP,ALL,1,"powerpc-allocate cpu=%d ri=%d -> config=0x%x", 229 cpu, ri, config); 230 return (0); 231 } 232 233 int 234 powerpc_release_pmc(int cpu, int ri, struct pmc *pmc) 235 { 236 struct pmc_hw *phw __diagused; 237 238 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 239 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 240 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 241 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 242 243 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 244 KASSERT(phw->phw_pmc == NULL, 245 ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 246 247 return (0); 248 } 249 250 int 251 powerpc_start_pmc(int cpu, int ri, struct pmc *pm) 252 { 253 254 PMCDBG2(MDP,STA,1,"powerpc-start cpu=%d ri=%d", cpu, ri); 255 powerpc_set_pmc(cpu, ri, pm->pm_md.pm_powerpc.pm_powerpc_evsel); 256 257 return (0); 258 } 259 260 int 261 powerpc_stop_pmc(int cpu, int ri, struct pmc *pm __unused) 262 { 263 PMCDBG2(MDP,STO,1, "powerpc-stop cpu=%d ri=%d", cpu, ri); 264 powerpc_set_pmc(cpu, ri, PMCN_NONE); 265 return (0); 266 } 267 268 int 269 powerpc_config_pmc(int cpu, int ri, struct pmc *pm) 270 { 271 struct pmc_hw *phw; 272 273 PMCDBG3(MDP,CFG,1, "powerpc-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 274 275 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 276 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 277 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 278 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 279 280 phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; 281 282 KASSERT(pm == NULL || phw->phw_pmc == NULL, 283 ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 284 __LINE__, pm, phw->phw_pmc)); 285 286 phw->phw_pmc = pm; 287 288 return (0); 289 } 290 291 pmc_value_t 292 powerpc_pmcn_read_default(unsigned int pmc) 293 { 294 pmc_value_t val; 295 296 if (pmc > ppc_max_pmcs) 297 panic("Invalid PMC number: %d\n", pmc); 298 299 switch (pmc) { 300 case 0: 301 val = mfspr(SPR_PMC1); 302 break; 303 case 1: 304 val = mfspr(SPR_PMC2); 305 break; 306 case 2: 307 val = mfspr(SPR_PMC3); 308 break; 309 case 3: 310 val = mfspr(SPR_PMC4); 311 break; 312 case 4: 313 val = mfspr(SPR_PMC5); 314 break; 315 case 5: 316 val = mfspr(SPR_PMC6); 317 break; 318 case 6: 319 val = mfspr(SPR_PMC7); 320 break; 321 case 7: 322 val = mfspr(SPR_PMC8); 323 break; 324 } 325 326 return (val); 327 } 328 329 void 330 powerpc_pmcn_write_default(unsigned int pmc, uint32_t val) 331 { 332 if (pmc > ppc_max_pmcs) 333 panic("Invalid PMC number: %d\n", pmc); 334 335 switch (pmc) { 336 case 0: 337 mtspr(SPR_PMC1, val); 338 break; 339 case 1: 340 mtspr(SPR_PMC2, val); 341 break; 342 case 2: 343 mtspr(SPR_PMC3, val); 344 break; 345 case 3: 346 mtspr(SPR_PMC4, val); 347 break; 348 case 4: 349 mtspr(SPR_PMC5, val); 350 break; 351 case 5: 352 mtspr(SPR_PMC6, val); 353 break; 354 case 6: 355 mtspr(SPR_PMC7, val); 356 break; 357 case 7: 358 mtspr(SPR_PMC8, val); 359 break; 360 } 361 } 362 363 int 364 powerpc_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 365 { 366 pmc_value_t p, r, tmp; 367 368 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 369 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 370 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 371 ("[powerpc,%d] illegal row index %d", __LINE__, ri)); 372 373 /* 374 * After an interrupt occurs because of a PMC overflow, the PMC value 375 * is not always MAX_PMC_VALUE + 1, but may be a little above it. 376 * This may mess up calculations and frustrate machine independent 377 * layer expectations, such as that no value read should be greater 378 * than reload count in sampling mode. 379 * To avoid these issues, use MAX_PMC_VALUE as an upper limit. 380 */ 381 p = MIN(powerpc_pmcn_read(ri), POWERPC_MAX_PMC_VALUE); 382 r = pm->pm_sc.pm_reloadcount; 383 384 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 385 /* 386 * Special case 1: r is too big 387 * This usually happens when a PMC write fails, the PMC is 388 * stopped and then it is read. 389 * 390 * Special case 2: PMC was reseted or has a value 391 * that should not be possible with current r. 392 * 393 * In the above cases, just return 0 instead of an arbitrary 394 * value. 395 */ 396 if (r > POWERPC_MAX_PMC_VALUE || p + r <= POWERPC_MAX_PMC_VALUE) 397 tmp = 0; 398 else 399 tmp = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(p); 400 } else 401 tmp = p + (POWERPC_MAX_PMC_VALUE + 1) * PPC_OVERFLOWCNT(pm); 402 403 PMCDBG5(MDP,REA,1,"ppc-read cpu=%d ri=%d -> %jx (%jx,%jx)", 404 cpu, ri, (uintmax_t)tmp, (uintmax_t)PPC_OVERFLOWCNT(pm), 405 (uintmax_t)p); 406 *v = tmp; 407 return (0); 408 } 409 410 int 411 powerpc_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 412 { 413 pmc_value_t vlo; 414 415 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 416 ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); 417 KASSERT(ri >= 0 && ri < ppc_max_pmcs, 418 ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); 419 420 if (PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { 421 PPC_OVERFLOWCNT(pm) = v / (POWERPC_MAX_PMC_VALUE + 1); 422 vlo = v % (POWERPC_MAX_PMC_VALUE + 1); 423 } else if (v > POWERPC_MAX_PMC_VALUE) { 424 PMCDBG3(MDP,WRI,2, 425 "powerpc-write cpu=%d ri=%d: PMC value is too big: %jx", 426 cpu, ri, (uintmax_t)v); 427 return (EINVAL); 428 } else 429 vlo = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 430 431 PMCDBG5(MDP,WRI,1,"powerpc-write cpu=%d ri=%d -> %jx (%jx,%jx)", 432 cpu, ri, (uintmax_t)v, (uintmax_t)PPC_OVERFLOWCNT(pm), 433 (uintmax_t)vlo); 434 435 powerpc_pmcn_write(ri, vlo); 436 return (0); 437 } 438 439 int 440 powerpc_pmc_intr(struct trapframe *tf) 441 { 442 struct pmc *pm; 443 struct powerpc_cpu *pc; 444 int cpu, error, i, retval; 445 446 cpu = curcpu; 447 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 448 ("[powerpc,%d] out of range CPU %d", __LINE__, cpu)); 449 450 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, 451 TRAPF_USERMODE(tf)); 452 453 retval = 0; 454 pc = powerpc_pcpu[cpu]; 455 456 /* 457 * Look for a running, sampling PMC which has overflowed 458 * and which has a valid 'struct pmc' association. 459 */ 460 for (i = 0; i < ppc_max_pmcs; i++) { 461 if (!POWERPC_PMC_HAS_OVERFLOWED(i)) 462 continue; 463 retval = 1; /* Found an interrupting PMC. */ 464 465 /* 466 * Always clear the PMC, to make it stop interrupting. 467 * If pm is available and in sampling mode, use reload 468 * count, to make PMC read after stop correct. 469 * Otherwise, just reset the PMC. 470 */ 471 if ((pm = pc->pc_ppcpmcs[i].phw_pmc) != NULL && 472 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 473 if (pm->pm_state != PMC_STATE_RUNNING) { 474 powerpc_write_pmc(cpu, i, pm, 475 pm->pm_sc.pm_reloadcount); 476 continue; 477 } 478 } else { 479 if (pm != NULL) { /* !PMC_IS_SAMPLING_MODE */ 480 PPC_OVERFLOWCNT(pm) = (PPC_OVERFLOWCNT(pm) + 481 1) % PPC_OVERFLOWCNT_MAX; 482 PMCDBG3(MDP,INT,2, 483 "cpu=%d ri=%d: overflowcnt=%d", 484 cpu, i, PPC_OVERFLOWCNT(pm)); 485 } 486 487 powerpc_pmcn_write(i, 0); 488 continue; 489 } 490 491 error = pmc_process_interrupt(PMC_HR, pm, tf); 492 if (error != 0) { 493 PMCDBG3(MDP,INT,3, 494 "cpu=%d ri=%d: error %d processing interrupt", 495 cpu, i, error); 496 powerpc_stop_pmc(cpu, i, pm); 497 } 498 499 /* Reload sampling count */ 500 powerpc_write_pmc(cpu, i, pm, pm->pm_sc.pm_reloadcount); 501 } 502 503 if (retval) 504 counter_u64_add(pmc_stats.pm_intr_processed, 1); 505 else 506 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 507 508 /* 509 * Re-enable PERF exceptions if we were able to find the interrupt 510 * source and handle it. Otherwise, it's better to disable PERF 511 * interrupts, to avoid the risk of processing the same interrupt 512 * forever. 513 */ 514 powerpc_resume_pmc(retval != 0); 515 if (retval == 0) 516 log(LOG_WARNING, 517 "pmc_intr: couldn't find interrupting PMC on cpu %d - " 518 "disabling PERF interrupts\n", cpu); 519 520 return (retval); 521 } 522 523 struct pmc_mdep * 524 pmc_md_initialize(void) 525 { 526 struct pmc_mdep *pmc_mdep; 527 int error; 528 uint16_t vers; 529 530 /* 531 * Allocate space for pointers to PMC HW descriptors and for 532 * the MDEP structure used by MI code. 533 */ 534 powerpc_pcpu = malloc(sizeof(struct powerpc_cpu *) * pmc_cpu_max(), M_PMC, 535 M_WAITOK|M_ZERO); 536 537 /* Just one class */ 538 pmc_mdep = pmc_mdep_alloc(1); 539 540 vers = mfpvr() >> 16; 541 542 switch (vers) { 543 case MPC7447A: 544 case MPC7448: 545 case MPC7450: 546 case MPC7455: 547 case MPC7457: 548 error = pmc_mpc7xxx_initialize(pmc_mdep); 549 break; 550 case IBM970: 551 case IBM970FX: 552 case IBM970MP: 553 error = pmc_ppc970_initialize(pmc_mdep); 554 break; 555 case IBMPOWER8E: 556 case IBMPOWER8NVL: 557 case IBMPOWER8: 558 case IBMPOWER9: 559 error = pmc_power8_initialize(pmc_mdep); 560 break; 561 case FSL_E500v1: 562 case FSL_E500v2: 563 case FSL_E500mc: 564 case FSL_E5500: 565 error = pmc_e500_initialize(pmc_mdep); 566 break; 567 default: 568 error = -1; 569 break; 570 } 571 572 if (error != 0) { 573 pmc_mdep_free(pmc_mdep); 574 pmc_mdep = NULL; 575 } 576 577 /* Set the value for kern.hwpmc.cpuid */ 578 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "%08x", mfpvr()); 579 580 return (pmc_mdep); 581 } 582 583 void 584 pmc_md_finalize(struct pmc_mdep *md) 585 { 586 587 free(powerpc_pcpu, M_PMC); 588 powerpc_pcpu = NULL; 589 } 590 591 int 592 pmc_save_user_callchain(uintptr_t *cc, int maxsamples, 593 struct trapframe *tf) 594 { 595 uintptr_t *osp, *sp; 596 int frames = 0; 597 598 cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); 599 sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); 600 osp = NULL; 601 602 for (; frames < maxsamples; frames++) { 603 if (sp <= osp) 604 break; 605 osp = sp; 606 #ifdef __powerpc64__ 607 /* Check if 32-bit mode. */ 608 if (!(tf->srr1 & PSL_SF)) { 609 cc[frames] = fuword32((uint32_t *)sp + 1); 610 sp = (uintptr_t *)(uintptr_t)fuword32(sp); 611 } else { 612 cc[frames] = fuword(sp + 2); 613 sp = (uintptr_t *)fuword(sp); 614 } 615 #else 616 cc[frames] = fuword32((uint32_t *)sp + 1); 617 sp = (uintptr_t *)fuword32(sp); 618 #endif 619 } 620 621 return (frames); 622 } 623