1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2010 Fabien Thomas 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Intel Uncore PMCs. 31 */ 32 33 #include <sys/cdefs.h> 34 #include <sys/param.h> 35 #include <sys/bus.h> 36 #include <sys/pmc.h> 37 #include <sys/pmckern.h> 38 #include <sys/systm.h> 39 40 #include <machine/intr_machdep.h> 41 #include <x86/apicvar.h> 42 #include <machine/cpu.h> 43 #include <machine/cpufunc.h> 44 #include <machine/specialreg.h> 45 46 #define UCF_PMC_CAPS \ 47 (PMC_CAP_READ | PMC_CAP_WRITE) 48 49 #define UCP_PMC_CAPS \ 50 (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ 51 PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) 52 53 #define SELECTSEL(x) \ 54 (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ 55 UCP_CB0_EVSEL0 : UCP_EVSEL0) 56 57 #define SELECTOFF(x) \ 58 (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ 59 UCF_OFFSET_SB : UCF_OFFSET) 60 61 static enum pmc_cputype uncore_cputype; 62 63 struct uncore_cpu { 64 volatile uint32_t pc_ucfctrl; /* Fixed function control. */ 65 volatile uint64_t pc_globalctrl; /* Global control register. */ 66 struct pmc_hw pc_uncorepmcs[]; 67 }; 68 69 static struct uncore_cpu **uncore_pcpu; 70 71 static uint64_t uncore_pmcmask; 72 73 static int uncore_ucf_ri; /* relative index of fixed counters */ 74 static int uncore_ucf_width; 75 static int uncore_ucf_npmc; 76 77 static int uncore_ucp_width; 78 static int uncore_ucp_npmc; 79 80 static int 81 uncore_pcpu_noop(struct pmc_mdep *md, int cpu) 82 { 83 (void) md; 84 (void) cpu; 85 return (0); 86 } 87 88 static int 89 uncore_pcpu_init(struct pmc_mdep *md, int cpu) 90 { 91 struct pmc_cpu *pc; 92 struct uncore_cpu *cc; 93 struct pmc_hw *phw; 94 int uncore_ri, n, npmc; 95 96 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 97 ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); 98 99 PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); 100 101 uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; 102 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; 103 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; 104 105 cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), 106 M_PMC, M_WAITOK | M_ZERO); 107 108 uncore_pcpu[cpu] = cc; 109 pc = pmc_pcpu[cpu]; 110 111 KASSERT(pc != NULL && cc != NULL, 112 ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); 113 114 for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { 115 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 116 PMC_PHW_CPU_TO_STATE(cpu) | 117 PMC_PHW_INDEX_TO_STATE(n + uncore_ri); 118 phw->phw_pmc = NULL; 119 pc->pc_hwpmcs[n + uncore_ri] = phw; 120 } 121 122 return (0); 123 } 124 125 static int 126 uncore_pcpu_fini(struct pmc_mdep *md, int cpu) 127 { 128 int uncore_ri, n, npmc; 129 struct pmc_cpu *pc; 130 struct uncore_cpu *cc; 131 132 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 133 ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); 134 135 PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); 136 137 if ((cc = uncore_pcpu[cpu]) == NULL) 138 return (0); 139 140 uncore_pcpu[cpu] = NULL; 141 142 pc = pmc_pcpu[cpu]; 143 144 KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, 145 cpu)); 146 147 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; 148 uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; 149 150 for (n = 0; n < npmc; n++) 151 wrmsr(SELECTSEL(uncore_cputype) + n, 0); 152 153 wrmsr(UCF_CTRL, 0); 154 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; 155 156 for (n = 0; n < npmc; n++) 157 pc->pc_hwpmcs[n + uncore_ri] = NULL; 158 159 free(cc, M_PMC); 160 161 return (0); 162 } 163 164 /* 165 * Fixed function counters. 166 */ 167 168 static pmc_value_t 169 ucf_perfctr_value_to_reload_count(pmc_value_t v) 170 { 171 172 /* If the PMC has overflowed, return a reload count of zero. */ 173 if ((v & (1ULL << (uncore_ucf_width - 1))) == 0) 174 return (0); 175 v &= (1ULL << uncore_ucf_width) - 1; 176 return (1ULL << uncore_ucf_width) - v; 177 } 178 179 static pmc_value_t 180 ucf_reload_count_to_perfctr_value(pmc_value_t rlc) 181 { 182 return (1ULL << uncore_ucf_width) - rlc; 183 } 184 185 static int 186 ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, 187 const struct pmc_op_pmcallocate *a) 188 { 189 uint32_t flags; 190 191 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 192 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 193 194 PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); 195 196 if (ri < 0 || ri > uncore_ucf_npmc) 197 return (EINVAL); 198 199 if (a->pm_class != PMC_CLASS_UCF) 200 return (EINVAL); 201 202 if ((a->pm_flags & PMC_F_EV_PMU) == 0) 203 return (EINVAL); 204 205 flags = UCF_EN; 206 207 pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); 208 209 PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", 210 (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); 211 212 return (0); 213 } 214 215 static int 216 ucf_config_pmc(int cpu, int ri, struct pmc *pm) 217 { 218 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 219 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 220 221 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 222 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 223 224 PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 225 226 KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, 227 cpu)); 228 229 uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; 230 231 return (0); 232 } 233 234 static int 235 ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 236 { 237 struct pmc_hw *phw; 238 239 phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; 240 241 snprintf(pi->pm_name, sizeof(pi->pm_name), "UCF-%d", ri); 242 pi->pm_class = PMC_CLASS_UCF; 243 244 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 245 pi->pm_enabled = TRUE; 246 *ppmc = phw->phw_pmc; 247 } else { 248 pi->pm_enabled = FALSE; 249 *ppmc = NULL; 250 } 251 252 return (0); 253 } 254 255 static int 256 ucf_get_config(int cpu, int ri, struct pmc **ppm) 257 { 258 *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 259 260 return (0); 261 } 262 263 static int 264 ucf_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 265 { 266 pmc_value_t tmp; 267 268 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 269 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 270 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 271 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 272 273 tmp = rdmsr(UCF_CTR0 + ri); 274 275 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 276 *v = ucf_perfctr_value_to_reload_count(tmp); 277 else 278 *v = tmp; 279 280 PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); 281 282 return (0); 283 } 284 285 static int 286 ucf_release_pmc(int cpu, int ri, struct pmc *pmc) 287 { 288 PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); 289 290 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 291 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 292 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 293 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 294 295 KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, 296 ("[uncore,%d] PHW pmc non-NULL", __LINE__)); 297 298 return (0); 299 } 300 301 static int 302 ucf_start_pmc(int cpu, int ri, struct pmc *pm) 303 { 304 struct uncore_cpu *ucfc; 305 306 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 307 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 308 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 309 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 310 311 PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); 312 313 ucfc = uncore_pcpu[cpu]; 314 ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; 315 316 wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); 317 318 ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); 319 wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); 320 321 PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", 322 ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), 323 ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); 324 325 return (0); 326 } 327 328 static int 329 ucf_stop_pmc(int cpu, int ri, struct pmc *pm __unused) 330 { 331 uint32_t fc; 332 struct uncore_cpu *ucfc; 333 334 PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); 335 336 ucfc = uncore_pcpu[cpu]; 337 338 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 339 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 340 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 341 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 342 343 fc = (UCF_MASK << (ri * 4)); 344 345 ucfc->pc_ucfctrl &= ~fc; 346 347 PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); 348 wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); 349 350 /* Don't need to write UC_GLOBAL_CTRL, one disable is enough. */ 351 352 PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", 353 ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), 354 ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); 355 356 return (0); 357 } 358 359 static int 360 ucf_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 361 { 362 struct uncore_cpu *cc; 363 364 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 365 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 366 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 367 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 368 369 cc = uncore_pcpu[cpu]; 370 371 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 372 v = ucf_reload_count_to_perfctr_value(v); 373 374 wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ 375 wrmsr(UCF_CTR0 + ri, v); 376 wrmsr(UCF_CTRL, cc->pc_ucfctrl); 377 378 PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", 379 cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); 380 381 return (0); 382 } 383 384 385 static void 386 ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) 387 { 388 struct pmc_classdep *pcd; 389 390 KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); 391 392 PMCDBG0(MDP,INI,1, "ucf-initialize"); 393 394 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; 395 396 pcd->pcd_caps = UCF_PMC_CAPS; 397 pcd->pcd_class = PMC_CLASS_UCF; 398 pcd->pcd_num = npmc; 399 pcd->pcd_ri = md->pmd_npmc; 400 pcd->pcd_width = pmcwidth; 401 402 pcd->pcd_allocate_pmc = ucf_allocate_pmc; 403 pcd->pcd_config_pmc = ucf_config_pmc; 404 pcd->pcd_describe = ucf_describe; 405 pcd->pcd_get_config = ucf_get_config; 406 pcd->pcd_get_msr = NULL; 407 pcd->pcd_pcpu_fini = uncore_pcpu_noop; 408 pcd->pcd_pcpu_init = uncore_pcpu_noop; 409 pcd->pcd_read_pmc = ucf_read_pmc; 410 pcd->pcd_release_pmc = ucf_release_pmc; 411 pcd->pcd_start_pmc = ucf_start_pmc; 412 pcd->pcd_stop_pmc = ucf_stop_pmc; 413 pcd->pcd_write_pmc = ucf_write_pmc; 414 415 md->pmd_npmc += npmc; 416 } 417 418 /* 419 * Intel programmable PMCs. 420 */ 421 422 /* 423 * Event descriptor tables. 424 * 425 * For each event id, we track: 426 * 427 * 1. The CPUs that the event is valid for. 428 * 429 * 2. If the event uses a fixed UMASK, the value of the umask field. 430 * If the event doesn't use a fixed UMASK, a mask of legal bits 431 * to check against. 432 */ 433 434 struct ucp_event_descr { 435 enum pmc_event ucp_ev; 436 unsigned char ucp_evcode; 437 unsigned char ucp_umask; 438 unsigned char ucp_flags; 439 }; 440 441 #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ 442 #define UCP_F_WM (1 << 1) /* CPU: Westmere */ 443 #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ 444 #define UCP_F_HW (1 << 3) /* CPU: Haswell */ 445 #define UCP_F_FM (1 << 4) /* Fixed mask */ 446 447 #define UCP_F_ALLCPUS \ 448 (UCP_F_I7 | UCP_F_WM) 449 450 #define UCP_F_CMASK 0xFF000000 451 452 static pmc_value_t 453 ucp_perfctr_value_to_reload_count(pmc_value_t v) 454 { 455 v &= (1ULL << uncore_ucp_width) - 1; 456 return (1ULL << uncore_ucp_width) - v; 457 } 458 459 static pmc_value_t 460 ucp_reload_count_to_perfctr_value(pmc_value_t rlc) 461 { 462 return (1ULL << uncore_ucp_width) - rlc; 463 } 464 465 /* 466 * Counter specific event information for Sandybridge and Haswell 467 */ 468 static int 469 ucp_event_sb_hw_ok_on_counter(uint8_t ev, int ri) 470 { 471 uint32_t mask; 472 473 switch (ev) { 474 /* 475 * Events valid only on counter 0. 476 */ 477 case 0x80: 478 case 0x83: 479 mask = (1 << 0); 480 break; 481 482 default: 483 mask = ~0; /* Any row index is ok. */ 484 } 485 486 return (mask & (1 << ri)); 487 } 488 489 static int 490 ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, 491 const struct pmc_op_pmcallocate *a) 492 { 493 uint8_t ev; 494 const struct pmc_md_ucp_op_pmcallocate *ucp; 495 496 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 497 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 498 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 499 ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); 500 501 if (a->pm_class != PMC_CLASS_UCP) 502 return (EINVAL); 503 504 if ((a->pm_flags & PMC_F_EV_PMU) == 0) 505 return (EINVAL); 506 507 ucp = &a->pm_md.pm_ucp; 508 ev = UCP_EVSEL(ucp->pm_ucp_config); 509 switch (uncore_cputype) { 510 case PMC_CPU_INTEL_HASWELL: 511 case PMC_CPU_INTEL_SANDYBRIDGE: 512 if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) 513 return (EINVAL); 514 break; 515 default: 516 break; 517 } 518 519 pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN; 520 521 return (0); 522 } 523 524 static int 525 ucp_config_pmc(int cpu, int ri, struct pmc *pm) 526 { 527 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 528 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 529 530 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 531 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 532 533 PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 534 535 KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, 536 cpu)); 537 538 uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; 539 540 return (0); 541 } 542 543 static int 544 ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 545 { 546 struct pmc_hw *phw; 547 548 phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; 549 550 snprintf(pi->pm_name, sizeof(pi->pm_name), "UCP-%d", ri); 551 pi->pm_class = PMC_CLASS_UCP; 552 553 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 554 pi->pm_enabled = TRUE; 555 *ppmc = phw->phw_pmc; 556 } else { 557 pi->pm_enabled = FALSE; 558 *ppmc = NULL; 559 } 560 561 return (0); 562 } 563 564 static int 565 ucp_get_config(int cpu, int ri, struct pmc **ppm) 566 { 567 *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; 568 569 return (0); 570 } 571 572 static int 573 ucp_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 574 { 575 pmc_value_t tmp; 576 577 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 578 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 579 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 580 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 581 582 tmp = rdmsr(UCP_PMC0 + ri); 583 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 584 *v = ucp_perfctr_value_to_reload_count(tmp); 585 else 586 *v = tmp; 587 588 PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, 589 ri, *v); 590 591 return (0); 592 } 593 594 static int 595 ucp_release_pmc(int cpu, int ri, struct pmc *pm) 596 { 597 (void) pm; 598 599 PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, 600 pm); 601 602 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 603 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 604 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 605 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 606 607 KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc 608 == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); 609 610 return (0); 611 } 612 613 static int 614 ucp_start_pmc(int cpu, int ri, struct pmc *pm) 615 { 616 uint64_t evsel; 617 struct uncore_cpu *cc; 618 619 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 620 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 621 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 622 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 623 624 cc = uncore_pcpu[cpu]; 625 626 PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); 627 628 evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; 629 630 PMCDBG4(MDP,STA,2, 631 "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", 632 cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); 633 634 wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); 635 636 cc->pc_globalctrl |= (1ULL << ri); 637 wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); 638 639 return (0); 640 } 641 642 static int 643 ucp_stop_pmc(int cpu, int ri, struct pmc *pm __unused) 644 { 645 646 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 647 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 648 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 649 ("[uncore,%d] illegal row index %d", __LINE__, ri)); 650 651 PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); 652 653 /* stop hw. */ 654 wrmsr(SELECTSEL(uncore_cputype) + ri, 0); 655 656 /* Don't need to write UC_GLOBAL_CTRL, one disable is enough. */ 657 658 return (0); 659 } 660 661 static int 662 ucp_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 663 { 664 665 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 666 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 667 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 668 ("[uncore,%d] illegal row index %d", __LINE__, ri)); 669 670 PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, 671 UCP_PMC0 + ri, v); 672 673 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 674 v = ucp_reload_count_to_perfctr_value(v); 675 676 /* 677 * Write the new value to the counter. The counter will be in 678 * a stopped state when the pcd_write() entry point is called. 679 */ 680 681 wrmsr(UCP_PMC0 + ri, v); 682 683 return (0); 684 } 685 686 687 static void 688 ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) 689 { 690 struct pmc_classdep *pcd; 691 692 KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); 693 694 PMCDBG0(MDP,INI,1, "ucp-initialize"); 695 696 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; 697 698 pcd->pcd_caps = UCP_PMC_CAPS; 699 pcd->pcd_class = PMC_CLASS_UCP; 700 pcd->pcd_num = npmc; 701 pcd->pcd_ri = md->pmd_npmc; 702 pcd->pcd_width = pmcwidth; 703 704 pcd->pcd_allocate_pmc = ucp_allocate_pmc; 705 pcd->pcd_config_pmc = ucp_config_pmc; 706 pcd->pcd_describe = ucp_describe; 707 pcd->pcd_get_config = ucp_get_config; 708 pcd->pcd_get_msr = NULL; 709 pcd->pcd_pcpu_fini = uncore_pcpu_fini; 710 pcd->pcd_pcpu_init = uncore_pcpu_init; 711 pcd->pcd_read_pmc = ucp_read_pmc; 712 pcd->pcd_release_pmc = ucp_release_pmc; 713 pcd->pcd_start_pmc = ucp_start_pmc; 714 pcd->pcd_stop_pmc = ucp_stop_pmc; 715 pcd->pcd_write_pmc = ucp_write_pmc; 716 717 md->pmd_npmc += npmc; 718 } 719 720 int 721 pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) 722 { 723 uncore_cputype = md->pmd_cputype; 724 uncore_pmcmask = 0; 725 726 /* 727 * Initialize programmable counters. 728 */ 729 730 uncore_ucp_npmc = 8; 731 uncore_ucp_width = 48; 732 733 uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); 734 735 ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); 736 737 /* 738 * Initialize fixed function counters, if present. 739 */ 740 uncore_ucf_ri = uncore_ucp_npmc; 741 uncore_ucf_npmc = 1; 742 uncore_ucf_width = 48; 743 744 ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); 745 uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); 746 747 PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, 748 uncore_ucf_ri); 749 750 uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, 751 M_ZERO | M_WAITOK); 752 753 return (0); 754 } 755 756 void 757 pmc_uncore_finalize(struct pmc_mdep *md) 758 { 759 PMCDBG0(MDP,INI,1, "uncore-finalize"); 760 761 for (int i = 0; i < pmc_cpu_max(); i++) 762 KASSERT(uncore_pcpu[i] == NULL, 763 ("[uncore,%d] non-null pcpu cpu %d", __LINE__, i)); 764 765 free(uncore_pcpu, M_PMC); 766 uncore_pcpu = NULL; 767 } 768