1 /*- 2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by the University of Cambridge Computer 6 * Laboratory with support from ARM Ltd. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/pmc.h> 33 #include <sys/pmckern.h> 34 35 #include <machine/pmc_mdep.h> 36 #include <machine/cpu.h> 37 #include <machine/machdep.h> 38 39 #include "opt_acpi.h" 40 41 static int arm64_npmcs; 42 static bool arm64_64bit_events __read_mostly = false; 43 44 struct arm64_event_code_map { 45 enum pmc_event pe_ev; 46 uint8_t pe_code; 47 }; 48 49 /* 50 * Per-processor information. 51 */ 52 struct arm64_cpu { 53 struct pmc_hw *pc_arm64pmcs; 54 }; 55 56 static struct arm64_cpu **arm64_pcpu; 57 58 /* 59 * Interrupt Enable Set Register 60 */ 61 static __inline void 62 arm64_interrupt_enable(uint32_t pmc) 63 { 64 uint32_t reg; 65 66 reg = (1 << pmc); 67 WRITE_SPECIALREG(pmintenset_el1, reg); 68 69 isb(); 70 } 71 72 /* 73 * Interrupt Clear Set Register 74 */ 75 static __inline void 76 arm64_interrupt_disable(uint32_t pmc) 77 { 78 uint32_t reg; 79 80 reg = (1 << pmc); 81 WRITE_SPECIALREG(pmintenclr_el1, reg); 82 83 isb(); 84 } 85 86 /* 87 * Counter Set Enable Register 88 */ 89 static __inline void 90 arm64_counter_enable(unsigned int pmc) 91 { 92 uint32_t reg; 93 94 reg = (1 << pmc); 95 WRITE_SPECIALREG(pmcntenset_el0, reg); 96 97 isb(); 98 } 99 100 /* 101 * Counter Clear Enable Register 102 */ 103 static __inline void 104 arm64_counter_disable(unsigned int pmc) 105 { 106 uint32_t reg; 107 108 reg = (1 << pmc); 109 WRITE_SPECIALREG(pmcntenclr_el0, reg); 110 111 isb(); 112 } 113 114 /* 115 * Performance Monitors Control Register 116 */ 117 static uint64_t 118 arm64_pmcr_read(void) 119 { 120 uint32_t reg; 121 122 reg = READ_SPECIALREG(pmcr_el0); 123 124 return (reg); 125 } 126 127 static void 128 arm64_pmcr_write(uint64_t reg) 129 { 130 131 WRITE_SPECIALREG(pmcr_el0, reg); 132 133 isb(); 134 } 135 136 /* 137 * Performance Count Register N 138 */ 139 static uint64_t 140 arm64_pmcn_read(unsigned int pmc) 141 { 142 143 KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); 144 145 WRITE_SPECIALREG(pmselr_el0, pmc); 146 147 isb(); 148 149 return (READ_SPECIALREG(pmxevcntr_el0)); 150 } 151 152 static void 153 arm64_pmcn_write(unsigned int pmc, uint64_t reg) 154 { 155 156 KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); 157 158 WRITE_SPECIALREG(pmselr_el0, pmc); 159 WRITE_SPECIALREG(pmxevcntr_el0, reg); 160 161 isb(); 162 } 163 164 static int 165 arm64_allocate_pmc(int cpu, int ri, struct pmc *pm, 166 const struct pmc_op_pmcallocate *a) 167 { 168 uint64_t config; 169 enum pmc_event pe; 170 171 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 172 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); 173 KASSERT(ri >= 0 && ri < arm64_npmcs, 174 ("[arm64,%d] illegal row index %d", __LINE__, ri)); 175 176 if (a->pm_class != PMC_CLASS_ARMV8) { 177 return (EINVAL); 178 } 179 pe = a->pm_ev; 180 181 if ((a->pm_flags & PMC_F_EV_PMU) != 0) { 182 config = a->pm_md.pm_md_config; 183 } else { 184 config = (uint32_t)pe - PMC_EV_ARMV8_FIRST; 185 if (config > (PMC_EV_ARMV8_LAST - PMC_EV_ARMV8_FIRST)) 186 return (EINVAL); 187 } 188 189 switch (a->pm_caps & (PMC_CAP_SYSTEM | PMC_CAP_USER)) { 190 case PMC_CAP_SYSTEM: 191 /* Exclude EL0 */ 192 config |= PMEVTYPER_U; 193 if (in_vhe()) { 194 /* If in VHE we need to include EL2 and exclude EL1 */ 195 config |= PMEVTYPER_NSH | PMEVTYPER_P; 196 } 197 break; 198 case PMC_CAP_USER: 199 /* Exclude EL1 */ 200 config |= PMEVTYPER_P; 201 /* Exclude EL2 */ 202 config &= ~PMEVTYPER_NSH; 203 break; 204 default: 205 /* 206 * Trace both USER and SYSTEM if none are specified 207 * (default setting) or if both flags are specified 208 * (user explicitly requested both qualifiers). 209 */ 210 if (in_vhe()) { 211 /* If in VHE we need to include EL2 */ 212 config |= PMEVTYPER_NSH; 213 } 214 break; 215 } 216 217 pm->pm_md.pm_arm64.pm_arm64_evsel = config; 218 PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%lx", ri, 219 config); 220 221 return (0); 222 } 223 224 225 static int 226 arm64_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 227 { 228 pmc_value_t tmp; 229 register_t s; 230 int reg; 231 232 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 233 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); 234 KASSERT(ri >= 0 && ri < arm64_npmcs, 235 ("[arm64,%d] illegal row index %d", __LINE__, ri)); 236 237 /* 238 * Ensure we don't get interrupted while updating the overflow count. 239 */ 240 s = intr_disable(); 241 tmp = arm64_pmcn_read(ri); 242 reg = (1 << ri); 243 if ((READ_SPECIALREG(pmovsclr_el0) & reg) != 0) { 244 /* Clear Overflow Flag */ 245 WRITE_SPECIALREG(pmovsclr_el0, reg); 246 pm->pm_pcpu_state[cpu].pps_overflowcnt++; 247 248 /* Reread counter in case we raced. */ 249 tmp = arm64_pmcn_read(ri); 250 } 251 /* 252 * If the counter is 32-bit increment the upper bits of the counter. 253 * It it is 64-bit then there is nothing we can do as tmp is already 254 * 64-bit. 255 */ 256 if (!arm64_64bit_events) { 257 tmp &= 0xffffffffu; 258 tmp += (uint64_t)pm->pm_pcpu_state[cpu].pps_overflowcnt << 32; 259 } 260 intr_restore(s); 261 262 PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp); 263 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 264 /* 265 * Clamp value to 0 if the counter just overflowed, 266 * otherwise the returned reload count would wrap to a 267 * huge value. 268 */ 269 if ((tmp & (1ull << 63)) == 0) 270 tmp = 0; 271 else 272 tmp = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 273 } 274 *v = tmp; 275 276 return (0); 277 } 278 279 static int 280 arm64_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 281 { 282 283 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 284 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); 285 KASSERT(ri >= 0 && ri < arm64_npmcs, 286 ("[arm64,%d] illegal row-index %d", __LINE__, ri)); 287 288 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 289 v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 290 291 PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v); 292 293 if (!arm64_64bit_events) { 294 pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32; 295 v &= 0xffffffffu; 296 } 297 arm64_pmcn_write(ri, v); 298 299 return (0); 300 } 301 302 static int 303 arm64_config_pmc(int cpu, int ri, struct pmc *pm) 304 { 305 struct pmc_hw *phw; 306 307 PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 308 309 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 310 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); 311 KASSERT(ri >= 0 && ri < arm64_npmcs, 312 ("[arm64,%d] illegal row-index %d", __LINE__, ri)); 313 314 phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; 315 316 KASSERT(pm == NULL || phw->phw_pmc == NULL, 317 ("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 318 __LINE__, pm, phw->phw_pmc)); 319 320 phw->phw_pmc = pm; 321 322 return (0); 323 } 324 325 static int 326 arm64_start_pmc(int cpu, int ri, struct pmc *pm) 327 { 328 uint64_t config; 329 330 config = pm->pm_md.pm_arm64.pm_arm64_evsel; 331 332 /* 333 * Configure the event selection. 334 */ 335 WRITE_SPECIALREG(pmselr_el0, ri); 336 WRITE_SPECIALREG(pmxevtyper_el0, config); 337 338 isb(); 339 340 /* 341 * Enable the PMC. 342 */ 343 arm64_interrupt_enable(ri); 344 arm64_counter_enable(ri); 345 346 return (0); 347 } 348 349 static int 350 arm64_stop_pmc(int cpu, int ri, struct pmc *pm __unused) 351 { 352 /* 353 * Disable the PMCs. 354 */ 355 arm64_counter_disable(ri); 356 arm64_interrupt_disable(ri); 357 358 return (0); 359 } 360 361 static int 362 arm64_release_pmc(int cpu, int ri, struct pmc *pmc) 363 { 364 struct pmc_hw *phw __diagused; 365 366 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 367 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); 368 KASSERT(ri >= 0 && ri < arm64_npmcs, 369 ("[arm64,%d] illegal row-index %d", __LINE__, ri)); 370 371 phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; 372 KASSERT(phw->phw_pmc == NULL, 373 ("[arm64,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 374 375 return (0); 376 } 377 378 static int 379 arm64_intr(struct trapframe *tf) 380 { 381 int retval, ri; 382 struct pmc *pm; 383 int error; 384 int reg, cpu; 385 386 cpu = curcpu; 387 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 388 ("[arm64,%d] CPU %d out of range", __LINE__, cpu)); 389 390 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *)tf, 391 TRAPF_USERMODE(tf)); 392 393 retval = 0; 394 395 for (ri = 0; ri < arm64_npmcs; ri++) { 396 pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; 397 if (pm == NULL) 398 continue; 399 /* Check if counter is overflowed */ 400 reg = (1 << ri); 401 if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0) 402 continue; 403 /* Clear Overflow Flag */ 404 WRITE_SPECIALREG(pmovsclr_el0, reg); 405 406 isb(); 407 408 retval = 1; /* Found an interrupting PMC. */ 409 410 pm->pm_pcpu_state[cpu].pps_overflowcnt += 1; 411 412 if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 413 continue; 414 415 if (pm->pm_state != PMC_STATE_RUNNING) 416 continue; 417 418 error = pmc_process_interrupt(PMC_HR, pm, tf); 419 if (error) 420 arm64_stop_pmc(cpu, ri, pm); 421 422 /* Reload sampling count */ 423 arm64_write_pmc(cpu, ri, pm, pm->pm_sc.pm_reloadcount); 424 } 425 426 return (retval); 427 } 428 429 static int 430 arm64_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 431 { 432 struct pmc_hw *phw; 433 434 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 435 ("[arm64,%d], illegal CPU %d", __LINE__, cpu)); 436 KASSERT(ri >= 0 && ri < arm64_npmcs, 437 ("[arm64,%d] row-index %d out of range", __LINE__, ri)); 438 439 phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; 440 441 snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV8-%d", ri); 442 pi->pm_class = PMC_CLASS_ARMV8; 443 444 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 445 pi->pm_enabled = TRUE; 446 *ppmc = phw->phw_pmc; 447 } else { 448 pi->pm_enabled = FALSE; 449 *ppmc = NULL; 450 } 451 452 return (0); 453 } 454 455 static int 456 arm64_get_config(int cpu, int ri, struct pmc **ppm) 457 { 458 459 *ppm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; 460 461 return (0); 462 } 463 464 static int 465 arm64_pcpu_init(struct pmc_mdep *md, int cpu) 466 { 467 struct arm64_cpu *pac; 468 struct pmc_hw *phw; 469 struct pmc_cpu *pc; 470 uint64_t pmcr; 471 int first_ri; 472 int i; 473 474 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 475 ("[arm64,%d] wrong cpu number %d", __LINE__, cpu)); 476 PMCDBG0(MDP, INI, 1, "arm64-pcpu-init"); 477 478 arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC, 479 M_WAITOK | M_ZERO); 480 481 pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs, 482 M_PMC, M_WAITOK | M_ZERO); 483 pc = pmc_pcpu[cpu]; 484 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri; 485 KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__)); 486 487 for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) { 488 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 489 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); 490 phw->phw_pmc = NULL; 491 pc->pc_hwpmcs[i + first_ri] = phw; 492 } 493 494 /* 495 * Disable all counters and overflow interrupts. Upon reset they are in 496 * an undefined state. 497 * 498 * Don't issue an isb here, just wait for the one in arm64_pmcr_write() 499 * to make the writes visible. 500 */ 501 WRITE_SPECIALREG(pmcntenclr_el0, 0xffffffff); 502 WRITE_SPECIALREG(pmintenclr_el1, 0xffffffff); 503 504 /* Enable unit with a useful default state */ 505 pmcr = PMCR_LC | PMCR_C | PMCR_P | PMCR_E; 506 if (arm64_64bit_events) 507 pmcr |= PMCR_LP; 508 arm64_pmcr_write(pmcr); 509 510 return (0); 511 } 512 513 static int 514 arm64_pcpu_fini(struct pmc_mdep *md, int cpu) 515 { 516 uint64_t pmcr; 517 518 PMCDBG0(MDP, INI, 1, "arm64-pcpu-fini"); 519 520 pmcr = arm64_pmcr_read(); 521 pmcr &= ~PMCR_E; 522 arm64_pmcr_write(pmcr); 523 524 free(arm64_pcpu[cpu]->pc_arm64pmcs, M_PMC); 525 free(arm64_pcpu[cpu], M_PMC); 526 arm64_pcpu[cpu] = NULL; 527 528 return (0); 529 } 530 531 struct pmc_mdep * 532 pmc_arm64_initialize(void) 533 { 534 struct pmc_mdep *pmc_mdep; 535 struct pmc_classdep *pcd; 536 int classes, idcode, impcode; 537 uint64_t dfr; 538 uint64_t pmcr; 539 uint64_t midr; 540 541 pmcr = arm64_pmcr_read(); 542 arm64_npmcs = (pmcr & PMCR_N_MASK) >> PMCR_N_SHIFT; 543 impcode = (pmcr & PMCR_IMP_MASK) >> PMCR_IMP_SHIFT; 544 idcode = (pmcr & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT; 545 546 PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs); 547 548 /* 549 * Write the CPU model to kern.hwpmc.cpuid. 550 * 551 * We zero the variant and revision fields. 552 * 553 * TODO: how to handle differences between cores due to big.LITTLE? 554 * For now, just use MIDR from CPU 0. 555 */ 556 midr = (uint64_t)(pcpu_find(0)->pc_midr); 557 midr &= ~(CPU_VAR_MASK | CPU_REV_MASK); 558 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "0x%016lx", midr); 559 560 /* Check if we have 64-bit counters */ 561 if (get_kernel_reg(ID_AA64DFR0_EL1, &dfr)) { 562 if (ID_AA64DFR0_PMUVer_VAL(dfr) >= ID_AA64DFR0_PMUVer_3_5) 563 arm64_64bit_events = true; 564 } 565 566 /* 567 * Allocate space for pointers to PMC HW descriptors and for 568 * the MDEP structure used by MI code. 569 */ 570 arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(), 571 M_PMC, M_WAITOK | M_ZERO); 572 573 /* One AArch64 CPU class */ 574 classes = 1; 575 576 #ifdef DEV_ACPI 577 /* Query presence of optional classes and set max class. */ 578 if (pmc_cmn600_nclasses() > 0) 579 classes = MAX(classes, PMC_MDEP_CLASS_INDEX_CMN600); 580 if (pmc_dmc620_nclasses() > 0) 581 classes = MAX(classes, PMC_MDEP_CLASS_INDEX_DMC620_C); 582 #endif 583 584 pmc_mdep = pmc_mdep_alloc(classes); 585 586 switch(impcode) { 587 case PMCR_IMP_ARM: 588 switch (idcode) { 589 case PMCR_IDCODE_CORTEX_A76: 590 case PMCR_IDCODE_NEOVERSE_N1: 591 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A76; 592 break; 593 case PMCR_IDCODE_CORTEX_A57: 594 case PMCR_IDCODE_CORTEX_A72: 595 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57; 596 break; 597 default: 598 case PMCR_IDCODE_CORTEX_A53: 599 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; 600 break; 601 } 602 break; 603 default: 604 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; 605 break; 606 } 607 608 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8]; 609 pcd->pcd_caps = ARMV8_PMC_CAPS; 610 pcd->pcd_class = PMC_CLASS_ARMV8; 611 pcd->pcd_num = arm64_npmcs; 612 pcd->pcd_ri = pmc_mdep->pmd_npmc; 613 pcd->pcd_width = 64; 614 615 pcd->pcd_allocate_pmc = arm64_allocate_pmc; 616 pcd->pcd_config_pmc = arm64_config_pmc; 617 pcd->pcd_pcpu_fini = arm64_pcpu_fini; 618 pcd->pcd_pcpu_init = arm64_pcpu_init; 619 pcd->pcd_describe = arm64_describe; 620 pcd->pcd_get_config = arm64_get_config; 621 pcd->pcd_read_pmc = arm64_read_pmc; 622 pcd->pcd_release_pmc = arm64_release_pmc; 623 pcd->pcd_start_pmc = arm64_start_pmc; 624 pcd->pcd_stop_pmc = arm64_stop_pmc; 625 pcd->pcd_write_pmc = arm64_write_pmc; 626 627 pmc_mdep->pmd_intr = arm64_intr; 628 pmc_mdep->pmd_npmc += arm64_npmcs; 629 630 #ifdef DEV_ACPI 631 if (pmc_cmn600_nclasses() > 0) 632 pmc_cmn600_initialize(pmc_mdep); 633 if (pmc_dmc620_nclasses() > 0) { 634 pmc_dmc620_initialize_cd2(pmc_mdep); 635 pmc_dmc620_initialize_c(pmc_mdep); 636 } 637 #endif 638 639 return (pmc_mdep); 640 } 641 642 void 643 pmc_arm64_finalize(struct pmc_mdep *md) 644 { 645 PMCDBG0(MDP, INI, 1, "arm64-finalize"); 646 647 for (int i = 0; i < pmc_cpu_max(); i++) 648 KASSERT(arm64_pcpu[i] == NULL, 649 ("[arm64,%d] non-null pcpu cpu %d", __LINE__, i)); 650 651 free(arm64_pcpu, M_PMC); 652 } 653