1 /*- 2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/pmc.h> 37 #include <sys/pmckern.h> 38 39 #include <machine/pmc_mdep.h> 40 #include <machine/cpu.h> 41 42 #define CPU_ID_CORTEX_VER_MASK 0xff 43 #define CPU_ID_CORTEX_VER_SHIFT 4 44 45 static int armv7_npmcs; 46 47 struct armv7_event_code_map { 48 enum pmc_event pe_ev; 49 uint8_t pe_code; 50 }; 51 52 const struct armv7_event_code_map armv7_event_codes[] = { 53 { PMC_EV_ARMV7_PMNC_SW_INCR, 0x00 }, 54 { PMC_EV_ARMV7_L1_ICACHE_REFILL, 0x01 }, 55 { PMC_EV_ARMV7_ITLB_REFILL, 0x02 }, 56 { PMC_EV_ARMV7_L1_DCACHE_REFILL, 0x03 }, 57 { PMC_EV_ARMV7_L1_DCACHE_ACCESS, 0x04 }, 58 { PMC_EV_ARMV7_DTLB_REFILL, 0x05 }, 59 { PMC_EV_ARMV7_MEM_READ, 0x06 }, 60 { PMC_EV_ARMV7_MEM_WRITE, 0x07 }, 61 { PMC_EV_ARMV7_INSTR_EXECUTED, 0x08 }, 62 { PMC_EV_ARMV7_EXC_TAKEN, 0x09 }, 63 { PMC_EV_ARMV7_EXC_EXECUTED, 0x0A }, 64 { PMC_EV_ARMV7_CID_WRITE, 0x0B }, 65 { PMC_EV_ARMV7_PC_WRITE, 0x0C }, 66 { PMC_EV_ARMV7_PC_IMM_BRANCH, 0x0D }, 67 { PMC_EV_ARMV7_PC_PROC_RETURN, 0x0E }, 68 { PMC_EV_ARMV7_MEM_UNALIGNED_ACCESS, 0x0F }, 69 { PMC_EV_ARMV7_PC_BRANCH_MIS_PRED, 0x10 }, 70 { PMC_EV_ARMV7_CLOCK_CYCLES, 0x11 }, 71 { PMC_EV_ARMV7_PC_BRANCH_PRED, 0x12 }, 72 { PMC_EV_ARMV7_MEM_ACCESS, 0x13 }, 73 { PMC_EV_ARMV7_L1_ICACHE_ACCESS, 0x14 }, 74 { PMC_EV_ARMV7_L1_DCACHE_WB, 0x15 }, 75 { PMC_EV_ARMV7_L2_CACHE_ACCESS, 0x16 }, 76 { PMC_EV_ARMV7_L2_CACHE_REFILL, 0x17 }, 77 { PMC_EV_ARMV7_L2_CACHE_WB, 0x18 }, 78 { PMC_EV_ARMV7_BUS_ACCESS, 0x19 }, 79 { PMC_EV_ARMV7_MEM_ERROR, 0x1A }, 80 { PMC_EV_ARMV7_INSTR_SPEC, 0x1B }, 81 { PMC_EV_ARMV7_TTBR_WRITE, 0x1C }, 82 { PMC_EV_ARMV7_BUS_CYCLES, 0x1D }, 83 { PMC_EV_ARMV7_CPU_CYCLES, 0xFF }, 84 }; 85 86 const int armv7_event_codes_size = 87 sizeof(armv7_event_codes) / sizeof(armv7_event_codes[0]); 88 89 /* 90 * Per-processor information. 91 */ 92 struct armv7_cpu { 93 struct pmc_hw *pc_armv7pmcs; 94 int cortex_ver; 95 }; 96 97 static struct armv7_cpu **armv7_pcpu; 98 99 /* 100 * Interrupt Enable Set Register 101 */ 102 static __inline void 103 armv7_interrupt_enable(uint32_t pmc) 104 { 105 uint32_t reg; 106 107 reg = (1 << pmc); 108 cp15_pminten_set(reg); 109 } 110 111 /* 112 * Interrupt Clear Set Register 113 */ 114 static __inline void 115 armv7_interrupt_disable(uint32_t pmc) 116 { 117 uint32_t reg; 118 119 reg = (1 << pmc); 120 cp15_pminten_clr(reg); 121 } 122 123 /* 124 * Counter Set Enable Register 125 */ 126 static __inline void 127 armv7_counter_enable(unsigned int pmc) 128 { 129 uint32_t reg; 130 131 reg = (1 << pmc); 132 cp15_pmcnten_set(reg); 133 } 134 135 /* 136 * Counter Clear Enable Register 137 */ 138 static __inline void 139 armv7_counter_disable(unsigned int pmc) 140 { 141 uint32_t reg; 142 143 reg = (1 << pmc); 144 cp15_pmcnten_clr(reg); 145 } 146 147 /* 148 * Performance Count Register N 149 */ 150 static uint32_t 151 armv7_pmcn_read(unsigned int pmc) 152 { 153 154 KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); 155 156 cp15_pmselr_set(pmc); 157 return (cp15_pmxevcntr_get()); 158 } 159 160 static uint32_t 161 armv7_pmcn_write(unsigned int pmc, uint32_t reg) 162 { 163 164 KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); 165 166 cp15_pmselr_set(pmc); 167 cp15_pmxevcntr_set(reg); 168 169 return (reg); 170 } 171 172 static int 173 armv7_allocate_pmc(int cpu, int ri, struct pmc *pm, 174 const struct pmc_op_pmcallocate *a) 175 { 176 uint32_t caps, config; 177 struct armv7_cpu *pac; 178 enum pmc_event pe; 179 int i; 180 181 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 182 ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); 183 KASSERT(ri >= 0 && ri < armv7_npmcs, 184 ("[armv7,%d] illegal row index %d", __LINE__, ri)); 185 186 pac = armv7_pcpu[cpu]; 187 188 caps = a->pm_caps; 189 if (a->pm_class != PMC_CLASS_ARMV7) 190 return (EINVAL); 191 pe = a->pm_ev; 192 193 for (i = 0; i < armv7_event_codes_size; i++) { 194 if (armv7_event_codes[i].pe_ev == pe) { 195 config = armv7_event_codes[i].pe_code; 196 break; 197 } 198 } 199 if (i == armv7_event_codes_size) 200 return EINVAL; 201 202 pm->pm_md.pm_armv7.pm_armv7_evsel = config; 203 204 PMCDBG(MDP,ALL,2,"armv7-allocate ri=%d -> config=0x%x", ri, config); 205 206 return 0; 207 } 208 209 210 static int 211 armv7_read_pmc(int cpu, int ri, pmc_value_t *v) 212 { 213 pmc_value_t tmp; 214 struct pmc *pm; 215 216 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 217 ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); 218 KASSERT(ri >= 0 && ri < armv7_npmcs, 219 ("[armv7,%d] illegal row index %d", __LINE__, ri)); 220 221 pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; 222 223 if (pm->pm_md.pm_armv7.pm_armv7_evsel == 0xFF) 224 tmp = cp15_pmccntr_get(); 225 else 226 tmp = armv7_pmcn_read(ri); 227 228 PMCDBG(MDP,REA,2,"armv7-read id=%d -> %jd", ri, tmp); 229 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 230 *v = ARMV7_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 231 else 232 *v = tmp; 233 234 return 0; 235 } 236 237 static int 238 armv7_write_pmc(int cpu, int ri, pmc_value_t v) 239 { 240 struct pmc *pm; 241 242 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 243 ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); 244 KASSERT(ri >= 0 && ri < armv7_npmcs, 245 ("[armv7,%d] illegal row-index %d", __LINE__, ri)); 246 247 pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; 248 249 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 250 v = ARMV7_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 251 252 PMCDBG(MDP,WRI,1,"armv7-write cpu=%d ri=%d v=%jx", cpu, ri, v); 253 254 if (pm->pm_md.pm_armv7.pm_armv7_evsel == 0xFF) 255 cp15_pmccntr_set(v); 256 else 257 armv7_pmcn_write(ri, v); 258 259 return 0; 260 } 261 262 static int 263 armv7_config_pmc(int cpu, int ri, struct pmc *pm) 264 { 265 struct pmc_hw *phw; 266 267 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 268 269 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 270 ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); 271 KASSERT(ri >= 0 && ri < armv7_npmcs, 272 ("[armv7,%d] illegal row-index %d", __LINE__, ri)); 273 274 phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; 275 276 KASSERT(pm == NULL || phw->phw_pmc == NULL, 277 ("[armv7,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 278 __LINE__, pm, phw->phw_pmc)); 279 280 phw->phw_pmc = pm; 281 282 return 0; 283 } 284 285 static int 286 armv7_start_pmc(int cpu, int ri) 287 { 288 struct pmc_hw *phw; 289 uint32_t config; 290 struct pmc *pm; 291 292 phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; 293 pm = phw->phw_pmc; 294 config = pm->pm_md.pm_armv7.pm_armv7_evsel; 295 296 /* 297 * Configure the event selection. 298 */ 299 cp15_pmselr_set(ri); 300 cp15_pmxevtyper_set(config); 301 302 /* 303 * Enable the PMC. 304 */ 305 armv7_interrupt_enable(ri); 306 armv7_counter_enable(ri); 307 308 return 0; 309 } 310 311 static int 312 armv7_stop_pmc(int cpu, int ri) 313 { 314 struct pmc_hw *phw; 315 struct pmc *pm; 316 317 phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; 318 pm = phw->phw_pmc; 319 320 /* 321 * Disable the PMCs. 322 */ 323 armv7_counter_disable(ri); 324 armv7_interrupt_disable(ri); 325 326 return 0; 327 } 328 329 static int 330 armv7_release_pmc(int cpu, int ri, struct pmc *pmc) 331 { 332 struct pmc_hw *phw; 333 334 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 335 ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); 336 KASSERT(ri >= 0 && ri < armv7_npmcs, 337 ("[armv7,%d] illegal row-index %d", __LINE__, ri)); 338 339 phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; 340 KASSERT(phw->phw_pmc == NULL, 341 ("[armv7,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 342 343 return 0; 344 } 345 346 static int 347 armv7_intr(int cpu, struct trapframe *tf) 348 { 349 struct armv7_cpu *pc; 350 int retval, ri; 351 struct pmc *pm; 352 int error; 353 int reg; 354 355 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 356 ("[armv7,%d] CPU %d out of range", __LINE__, cpu)); 357 358 retval = 0; 359 pc = armv7_pcpu[cpu]; 360 361 for (ri = 0; ri < armv7_npmcs; ri++) { 362 pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; 363 if (pm == NULL) 364 continue; 365 if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 366 continue; 367 368 /* Check if counter has overflowed */ 369 if (pm->pm_md.pm_armv7.pm_armv7_evsel == 0xFF) 370 reg = (1 << 31); 371 else 372 reg = (1 << ri); 373 374 if ((cp15_pmovsr_get() & reg) == 0) { 375 continue; 376 } 377 378 /* Clear Overflow Flag */ 379 cp15_pmovsr_set(reg); 380 381 retval = 1; /* Found an interrupting PMC. */ 382 if (pm->pm_state != PMC_STATE_RUNNING) 383 continue; 384 385 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, 386 TRAPF_USERMODE(tf)); 387 if (error) 388 armv7_stop_pmc(cpu, ri); 389 390 /* Reload sampling count */ 391 armv7_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); 392 } 393 394 return (retval); 395 } 396 397 static int 398 armv7_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 399 { 400 char armv7_name[PMC_NAME_MAX]; 401 struct pmc_hw *phw; 402 int error; 403 404 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 405 ("[armv7,%d], illegal CPU %d", __LINE__, cpu)); 406 KASSERT(ri >= 0 && ri < armv7_npmcs, 407 ("[armv7,%d] row-index %d out of range", __LINE__, ri)); 408 409 phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; 410 snprintf(armv7_name, sizeof(armv7_name), "ARMV7-%d", ri); 411 if ((error = copystr(armv7_name, pi->pm_name, PMC_NAME_MAX, 412 NULL)) != 0) 413 return error; 414 pi->pm_class = PMC_CLASS_ARMV7; 415 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 416 pi->pm_enabled = TRUE; 417 *ppmc = phw->phw_pmc; 418 } else { 419 pi->pm_enabled = FALSE; 420 *ppmc = NULL; 421 } 422 423 return (0); 424 } 425 426 static int 427 armv7_get_config(int cpu, int ri, struct pmc **ppm) 428 { 429 430 *ppm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; 431 432 return 0; 433 } 434 435 /* 436 * XXX don't know what we should do here. 437 */ 438 static int 439 armv7_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 440 { 441 442 return 0; 443 } 444 445 static int 446 armv7_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 447 { 448 449 return 0; 450 } 451 452 static int 453 armv7_pcpu_init(struct pmc_mdep *md, int cpu) 454 { 455 struct armv7_cpu *pac; 456 struct pmc_hw *phw; 457 struct pmc_cpu *pc; 458 uint32_t pmnc; 459 int first_ri; 460 int cpuid; 461 int i; 462 463 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 464 ("[armv7,%d] wrong cpu number %d", __LINE__, cpu)); 465 PMCDBG(MDP,INI,1,"armv7-init cpu=%d", cpu); 466 467 armv7_pcpu[cpu] = pac = malloc(sizeof(struct armv7_cpu), M_PMC, 468 M_WAITOK|M_ZERO); 469 470 cpuid = cpu_ident(); 471 pac->cortex_ver = (cpuid >> CPU_ID_CORTEX_VER_SHIFT) & \ 472 CPU_ID_CORTEX_VER_MASK; 473 474 pac->pc_armv7pmcs = malloc(sizeof(struct pmc_hw) * armv7_npmcs, 475 M_PMC, M_WAITOK|M_ZERO); 476 pc = pmc_pcpu[cpu]; 477 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7].pcd_ri; 478 KASSERT(pc != NULL, ("[armv7,%d] NULL per-cpu pointer", __LINE__)); 479 480 for (i = 0, phw = pac->pc_armv7pmcs; i < armv7_npmcs; i++, phw++) { 481 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 482 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); 483 phw->phw_pmc = NULL; 484 pc->pc_hwpmcs[i + first_ri] = phw; 485 } 486 487 /* Enable unit */ 488 pmnc = cp15_pmcr_get(); 489 pmnc |= ARMV7_PMNC_ENABLE; 490 cp15_pmcr_set(pmnc); 491 492 return 0; 493 } 494 495 static int 496 armv7_pcpu_fini(struct pmc_mdep *md, int cpu) 497 { 498 uint32_t pmnc; 499 500 pmnc = cp15_pmcr_get(); 501 pmnc &= ~ARMV7_PMNC_ENABLE; 502 cp15_pmcr_set(pmnc); 503 504 return 0; 505 } 506 507 struct pmc_mdep * 508 pmc_armv7_initialize() 509 { 510 struct pmc_mdep *pmc_mdep; 511 struct pmc_classdep *pcd; 512 int reg; 513 514 reg = cp15_pmcr_get(); 515 516 armv7_npmcs = (reg >> ARMV7_PMNC_N_SHIFT) & \ 517 ARMV7_PMNC_N_MASK; 518 519 PMCDBG(MDP,INI,1,"armv7-init npmcs=%d", armv7_npmcs); 520 521 /* 522 * Allocate space for pointers to PMC HW descriptors and for 523 * the MDEP structure used by MI code. 524 */ 525 armv7_pcpu = malloc(sizeof(struct armv7_cpu *) * pmc_cpu_max(), 526 M_PMC, M_WAITOK | M_ZERO); 527 528 /* Just one class */ 529 pmc_mdep = pmc_mdep_alloc(1); 530 pmc_mdep->pmd_cputype = PMC_CPU_ARMV7; 531 532 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7]; 533 pcd->pcd_caps = ARMV7_PMC_CAPS; 534 pcd->pcd_class = PMC_CLASS_ARMV7; 535 pcd->pcd_num = armv7_npmcs; 536 pcd->pcd_ri = pmc_mdep->pmd_npmc; 537 pcd->pcd_width = 32; 538 539 pcd->pcd_allocate_pmc = armv7_allocate_pmc; 540 pcd->pcd_config_pmc = armv7_config_pmc; 541 pcd->pcd_pcpu_fini = armv7_pcpu_fini; 542 pcd->pcd_pcpu_init = armv7_pcpu_init; 543 pcd->pcd_describe = armv7_describe; 544 pcd->pcd_get_config = armv7_get_config; 545 pcd->pcd_read_pmc = armv7_read_pmc; 546 pcd->pcd_release_pmc = armv7_release_pmc; 547 pcd->pcd_start_pmc = armv7_start_pmc; 548 pcd->pcd_stop_pmc = armv7_stop_pmc; 549 pcd->pcd_write_pmc = armv7_write_pmc; 550 551 pmc_mdep->pmd_intr = armv7_intr; 552 pmc_mdep->pmd_switch_in = armv7_switch_in; 553 pmc_mdep->pmd_switch_out = armv7_switch_out; 554 555 pmc_mdep->pmd_npmc += armv7_npmcs; 556 557 return (pmc_mdep); 558 } 559 560 void 561 pmc_armv7_finalize(struct pmc_mdep *md) 562 { 563 564 } 565