1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010 Fabien Thomas 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Intel Uncore PMCs. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/bus.h> 38 #include <sys/pmc.h> 39 #include <sys/pmckern.h> 40 #include <sys/systm.h> 41 42 #include <machine/intr_machdep.h> 43 #if (__FreeBSD_version >= 1100000) 44 #include <x86/apicvar.h> 45 #else 46 #include <machine/apicvar.h> 47 #endif 48 #include <machine/cpu.h> 49 #include <machine/cpufunc.h> 50 #include <machine/specialreg.h> 51 52 #define UCF_PMC_CAPS \ 53 (PMC_CAP_READ | PMC_CAP_WRITE) 54 55 #define UCP_PMC_CAPS \ 56 (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ 57 PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) 58 59 #define SELECTSEL(x) \ 60 (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ 61 UCP_CB0_EVSEL0 : UCP_EVSEL0) 62 63 #define SELECTOFF(x) \ 64 (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ 65 UCF_OFFSET_SB : UCF_OFFSET) 66 67 static enum pmc_cputype uncore_cputype; 68 69 struct uncore_cpu { 70 volatile uint32_t pc_resync; 71 volatile uint32_t pc_ucfctrl; /* Fixed function control. */ 72 volatile uint64_t pc_globalctrl; /* Global control register. */ 73 struct pmc_hw pc_uncorepmcs[]; 74 }; 75 76 static struct uncore_cpu **uncore_pcpu; 77 78 static uint64_t uncore_pmcmask; 79 80 static int uncore_ucf_ri; /* relative index of fixed counters */ 81 static int uncore_ucf_width; 82 static int uncore_ucf_npmc; 83 84 static int uncore_ucp_width; 85 static int uncore_ucp_npmc; 86 87 static int 88 uncore_pcpu_noop(struct pmc_mdep *md, int cpu) 89 { 90 (void) md; 91 (void) cpu; 92 return (0); 93 } 94 95 static int 96 uncore_pcpu_init(struct pmc_mdep *md, int cpu) 97 { 98 struct pmc_cpu *pc; 99 struct uncore_cpu *cc; 100 struct pmc_hw *phw; 101 int uncore_ri, n, npmc; 102 103 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 104 ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); 105 106 PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); 107 108 uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; 109 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; 110 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; 111 112 cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), 113 M_PMC, M_WAITOK | M_ZERO); 114 115 uncore_pcpu[cpu] = cc; 116 pc = pmc_pcpu[cpu]; 117 118 KASSERT(pc != NULL && cc != NULL, 119 ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); 120 121 for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { 122 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 123 PMC_PHW_CPU_TO_STATE(cpu) | 124 PMC_PHW_INDEX_TO_STATE(n + uncore_ri); 125 phw->phw_pmc = NULL; 126 pc->pc_hwpmcs[n + uncore_ri] = phw; 127 } 128 129 return (0); 130 } 131 132 static int 133 uncore_pcpu_fini(struct pmc_mdep *md, int cpu) 134 { 135 int uncore_ri, n, npmc; 136 struct pmc_cpu *pc; 137 struct uncore_cpu *cc; 138 139 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 140 ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); 141 142 PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); 143 144 if ((cc = uncore_pcpu[cpu]) == NULL) 145 return (0); 146 147 uncore_pcpu[cpu] = NULL; 148 149 pc = pmc_pcpu[cpu]; 150 151 KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, 152 cpu)); 153 154 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; 155 uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; 156 157 for (n = 0; n < npmc; n++) 158 wrmsr(SELECTSEL(uncore_cputype) + n, 0); 159 160 wrmsr(UCF_CTRL, 0); 161 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; 162 163 for (n = 0; n < npmc; n++) 164 pc->pc_hwpmcs[n + uncore_ri] = NULL; 165 166 free(cc, M_PMC); 167 168 return (0); 169 } 170 171 /* 172 * Fixed function counters. 173 */ 174 175 static pmc_value_t 176 ucf_perfctr_value_to_reload_count(pmc_value_t v) 177 { 178 v &= (1ULL << uncore_ucf_width) - 1; 179 return (1ULL << uncore_ucf_width) - v; 180 } 181 182 static pmc_value_t 183 ucf_reload_count_to_perfctr_value(pmc_value_t rlc) 184 { 185 return (1ULL << uncore_ucf_width) - rlc; 186 } 187 188 static int 189 ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, 190 const struct pmc_op_pmcallocate *a) 191 { 192 uint32_t flags; 193 194 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 195 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 196 197 PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); 198 199 if (ri < 0 || ri > uncore_ucf_npmc) 200 return (EINVAL); 201 202 if (a->pm_class != PMC_CLASS_UCF) 203 return (EINVAL); 204 205 flags = UCF_EN; 206 207 pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); 208 209 PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", 210 (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); 211 212 return (0); 213 } 214 215 static int 216 ucf_config_pmc(int cpu, int ri, struct pmc *pm) 217 { 218 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 219 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 220 221 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 222 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 223 224 PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 225 226 KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, 227 cpu)); 228 229 uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; 230 231 return (0); 232 } 233 234 static int 235 ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 236 { 237 int error; 238 struct pmc_hw *phw; 239 char ucf_name[PMC_NAME_MAX]; 240 241 phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; 242 243 (void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri); 244 if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX, 245 NULL)) != 0) 246 return (error); 247 248 pi->pm_class = PMC_CLASS_UCF; 249 250 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 251 pi->pm_enabled = TRUE; 252 *ppmc = phw->phw_pmc; 253 } else { 254 pi->pm_enabled = FALSE; 255 *ppmc = NULL; 256 } 257 258 return (0); 259 } 260 261 static int 262 ucf_get_config(int cpu, int ri, struct pmc **ppm) 263 { 264 *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 265 266 return (0); 267 } 268 269 static int 270 ucf_read_pmc(int cpu, int ri, pmc_value_t *v) 271 { 272 struct pmc *pm; 273 pmc_value_t tmp; 274 275 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 276 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 277 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 278 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 279 280 pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 281 282 KASSERT(pm, 283 ("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, 284 ri, ri + uncore_ucf_ri)); 285 286 tmp = rdmsr(UCF_CTR0 + ri); 287 288 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 289 *v = ucf_perfctr_value_to_reload_count(tmp); 290 else 291 *v = tmp; 292 293 PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); 294 295 return (0); 296 } 297 298 static int 299 ucf_release_pmc(int cpu, int ri, struct pmc *pmc) 300 { 301 PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); 302 303 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 304 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 305 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 306 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 307 308 KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, 309 ("[uncore,%d] PHW pmc non-NULL", __LINE__)); 310 311 return (0); 312 } 313 314 static int 315 ucf_start_pmc(int cpu, int ri) 316 { 317 struct pmc *pm; 318 struct uncore_cpu *ucfc; 319 320 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 321 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 322 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 323 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 324 325 PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); 326 327 ucfc = uncore_pcpu[cpu]; 328 pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 329 330 ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; 331 332 wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); 333 334 do { 335 ucfc->pc_resync = 0; 336 ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); 337 wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); 338 } while (ucfc->pc_resync != 0); 339 340 PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", 341 ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), 342 ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); 343 344 return (0); 345 } 346 347 static int 348 ucf_stop_pmc(int cpu, int ri) 349 { 350 uint32_t fc; 351 struct uncore_cpu *ucfc; 352 353 PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); 354 355 ucfc = uncore_pcpu[cpu]; 356 357 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 358 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 359 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 360 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 361 362 fc = (UCF_MASK << (ri * 4)); 363 364 ucfc->pc_ucfctrl &= ~fc; 365 366 PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); 367 wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); 368 369 do { 370 ucfc->pc_resync = 0; 371 ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype))); 372 wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); 373 } while (ucfc->pc_resync != 0); 374 375 PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", 376 ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), 377 ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); 378 379 return (0); 380 } 381 382 static int 383 ucf_write_pmc(int cpu, int ri, pmc_value_t v) 384 { 385 struct uncore_cpu *cc; 386 struct pmc *pm; 387 388 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 389 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 390 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 391 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 392 393 cc = uncore_pcpu[cpu]; 394 pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 395 396 KASSERT(pm, 397 ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 398 399 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 400 v = ucf_reload_count_to_perfctr_value(v); 401 402 wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ 403 wrmsr(UCF_CTR0 + ri, v); 404 wrmsr(UCF_CTRL, cc->pc_ucfctrl); 405 406 PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", 407 cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); 408 409 return (0); 410 } 411 412 413 static void 414 ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) 415 { 416 struct pmc_classdep *pcd; 417 418 KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); 419 420 PMCDBG0(MDP,INI,1, "ucf-initialize"); 421 422 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; 423 424 pcd->pcd_caps = UCF_PMC_CAPS; 425 pcd->pcd_class = PMC_CLASS_UCF; 426 pcd->pcd_num = npmc; 427 pcd->pcd_ri = md->pmd_npmc; 428 pcd->pcd_width = pmcwidth; 429 430 pcd->pcd_allocate_pmc = ucf_allocate_pmc; 431 pcd->pcd_config_pmc = ucf_config_pmc; 432 pcd->pcd_describe = ucf_describe; 433 pcd->pcd_get_config = ucf_get_config; 434 pcd->pcd_get_msr = NULL; 435 pcd->pcd_pcpu_fini = uncore_pcpu_noop; 436 pcd->pcd_pcpu_init = uncore_pcpu_noop; 437 pcd->pcd_read_pmc = ucf_read_pmc; 438 pcd->pcd_release_pmc = ucf_release_pmc; 439 pcd->pcd_start_pmc = ucf_start_pmc; 440 pcd->pcd_stop_pmc = ucf_stop_pmc; 441 pcd->pcd_write_pmc = ucf_write_pmc; 442 443 md->pmd_npmc += npmc; 444 } 445 446 /* 447 * Intel programmable PMCs. 448 */ 449 450 /* 451 * Event descriptor tables. 452 * 453 * For each event id, we track: 454 * 455 * 1. The CPUs that the event is valid for. 456 * 457 * 2. If the event uses a fixed UMASK, the value of the umask field. 458 * If the event doesn't use a fixed UMASK, a mask of legal bits 459 * to check against. 460 */ 461 462 struct ucp_event_descr { 463 enum pmc_event ucp_ev; 464 unsigned char ucp_evcode; 465 unsigned char ucp_umask; 466 unsigned char ucp_flags; 467 }; 468 469 #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ 470 #define UCP_F_WM (1 << 1) /* CPU: Westmere */ 471 #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ 472 #define UCP_F_HW (1 << 3) /* CPU: Haswell */ 473 #define UCP_F_FM (1 << 4) /* Fixed mask */ 474 475 #define UCP_F_ALLCPUS \ 476 (UCP_F_I7 | UCP_F_WM) 477 478 #define UCP_F_CMASK 0xFF000000 479 480 static pmc_value_t 481 ucp_perfctr_value_to_reload_count(pmc_value_t v) 482 { 483 v &= (1ULL << uncore_ucp_width) - 1; 484 return (1ULL << uncore_ucp_width) - v; 485 } 486 487 static pmc_value_t 488 ucp_reload_count_to_perfctr_value(pmc_value_t rlc) 489 { 490 return (1ULL << uncore_ucp_width) - rlc; 491 } 492 493 /* 494 * Counter specific event information for Sandybridge and Haswell 495 */ 496 static int 497 ucp_event_sb_hw_ok_on_counter(uint8_t ev, int ri) 498 { 499 uint32_t mask; 500 501 switch (ev) { 502 /* 503 * Events valid only on counter 0. 504 */ 505 case 0x80: 506 case 0x83: 507 mask = (1 << 0); 508 break; 509 510 default: 511 mask = ~0; /* Any row index is ok. */ 512 } 513 514 return (mask & (1 << ri)); 515 } 516 517 static int 518 ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, 519 const struct pmc_op_pmcallocate *a) 520 { 521 uint8_t ev; 522 const struct pmc_md_ucp_op_pmcallocate *ucp; 523 524 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 525 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 526 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 527 ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); 528 529 if (a->pm_class != PMC_CLASS_UCP) 530 return (EINVAL); 531 532 ucp = &a->pm_md.pm_ucp; 533 ev = UCP_EVSEL(ucp->pm_ucp_config); 534 switch (uncore_cputype) { 535 case PMC_CPU_INTEL_HASWELL: 536 case PMC_CPU_INTEL_SANDYBRIDGE: 537 if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) 538 return (EINVAL); 539 break; 540 default: 541 break; 542 } 543 544 pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN; 545 546 return (0); 547 } 548 549 static int 550 ucp_config_pmc(int cpu, int ri, struct pmc *pm) 551 { 552 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 553 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 554 555 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 556 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 557 558 PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 559 560 KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, 561 cpu)); 562 563 uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; 564 565 return (0); 566 } 567 568 static int 569 ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 570 { 571 int error; 572 struct pmc_hw *phw; 573 char ucp_name[PMC_NAME_MAX]; 574 575 phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; 576 577 (void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri); 578 if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX, 579 NULL)) != 0) 580 return (error); 581 582 pi->pm_class = PMC_CLASS_UCP; 583 584 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 585 pi->pm_enabled = TRUE; 586 *ppmc = phw->phw_pmc; 587 } else { 588 pi->pm_enabled = FALSE; 589 *ppmc = NULL; 590 } 591 592 return (0); 593 } 594 595 static int 596 ucp_get_config(int cpu, int ri, struct pmc **ppm) 597 { 598 *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; 599 600 return (0); 601 } 602 603 static int 604 ucp_read_pmc(int cpu, int ri, pmc_value_t *v) 605 { 606 struct pmc *pm; 607 pmc_value_t tmp; 608 609 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 610 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 611 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 612 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 613 614 pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; 615 616 KASSERT(pm, 617 ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, 618 ri)); 619 620 tmp = rdmsr(UCP_PMC0 + ri); 621 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 622 *v = ucp_perfctr_value_to_reload_count(tmp); 623 else 624 *v = tmp; 625 626 PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, 627 ri, *v); 628 629 return (0); 630 } 631 632 static int 633 ucp_release_pmc(int cpu, int ri, struct pmc *pm) 634 { 635 (void) pm; 636 637 PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, 638 pm); 639 640 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 641 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 642 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 643 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 644 645 KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc 646 == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); 647 648 return (0); 649 } 650 651 static int 652 ucp_start_pmc(int cpu, int ri) 653 { 654 struct pmc *pm; 655 uint32_t evsel; 656 struct uncore_cpu *cc; 657 658 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 659 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 660 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 661 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 662 663 cc = uncore_pcpu[cpu]; 664 pm = cc->pc_uncorepmcs[ri].phw_pmc; 665 666 KASSERT(pm, 667 ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", 668 __LINE__, cpu, ri)); 669 670 PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); 671 672 evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; 673 674 PMCDBG4(MDP,STA,2, 675 "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", 676 cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); 677 678 /* Event specific configuration. */ 679 switch (pm->pm_event) { 680 case PMC_EV_UCP_EVENT_0CH_04H_E: 681 case PMC_EV_UCP_EVENT_0CH_08H_E: 682 wrmsr(MSR_GQ_SNOOP_MESF,0x2); 683 break; 684 case PMC_EV_UCP_EVENT_0CH_04H_F: 685 case PMC_EV_UCP_EVENT_0CH_08H_F: 686 wrmsr(MSR_GQ_SNOOP_MESF,0x8); 687 break; 688 case PMC_EV_UCP_EVENT_0CH_04H_M: 689 case PMC_EV_UCP_EVENT_0CH_08H_M: 690 wrmsr(MSR_GQ_SNOOP_MESF,0x1); 691 break; 692 case PMC_EV_UCP_EVENT_0CH_04H_S: 693 case PMC_EV_UCP_EVENT_0CH_08H_S: 694 wrmsr(MSR_GQ_SNOOP_MESF,0x4); 695 break; 696 default: 697 break; 698 } 699 wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); 700 701 do { 702 cc->pc_resync = 0; 703 cc->pc_globalctrl |= (1ULL << ri); 704 wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); 705 } while (cc->pc_resync != 0); 706 707 return (0); 708 } 709 710 static int 711 ucp_stop_pmc(int cpu, int ri) 712 { 713 struct pmc *pm __diagused; 714 struct uncore_cpu *cc; 715 716 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 717 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 718 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 719 ("[uncore,%d] illegal row index %d", __LINE__, ri)); 720 721 cc = uncore_pcpu[cpu]; 722 pm = cc->pc_uncorepmcs[ri].phw_pmc; 723 724 KASSERT(pm, 725 ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, 726 cpu, ri)); 727 728 PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); 729 730 /* stop hw. */ 731 wrmsr(SELECTSEL(uncore_cputype) + ri, 0); 732 733 do { 734 cc->pc_resync = 0; 735 cc->pc_globalctrl &= ~(1ULL << ri); 736 wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); 737 } while (cc->pc_resync != 0); 738 739 return (0); 740 } 741 742 static int 743 ucp_write_pmc(int cpu, int ri, pmc_value_t v) 744 { 745 struct pmc *pm; 746 struct uncore_cpu *cc; 747 748 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 749 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 750 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 751 ("[uncore,%d] illegal row index %d", __LINE__, ri)); 752 753 cc = uncore_pcpu[cpu]; 754 pm = cc->pc_uncorepmcs[ri].phw_pmc; 755 756 KASSERT(pm, 757 ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, 758 cpu, ri)); 759 760 PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, 761 UCP_PMC0 + ri, v); 762 763 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 764 v = ucp_reload_count_to_perfctr_value(v); 765 766 /* 767 * Write the new value to the counter. The counter will be in 768 * a stopped state when the pcd_write() entry point is called. 769 */ 770 771 wrmsr(UCP_PMC0 + ri, v); 772 773 return (0); 774 } 775 776 777 static void 778 ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) 779 { 780 struct pmc_classdep *pcd; 781 782 KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); 783 784 PMCDBG0(MDP,INI,1, "ucp-initialize"); 785 786 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; 787 788 pcd->pcd_caps = UCP_PMC_CAPS; 789 pcd->pcd_class = PMC_CLASS_UCP; 790 pcd->pcd_num = npmc; 791 pcd->pcd_ri = md->pmd_npmc; 792 pcd->pcd_width = pmcwidth; 793 794 pcd->pcd_allocate_pmc = ucp_allocate_pmc; 795 pcd->pcd_config_pmc = ucp_config_pmc; 796 pcd->pcd_describe = ucp_describe; 797 pcd->pcd_get_config = ucp_get_config; 798 pcd->pcd_get_msr = NULL; 799 pcd->pcd_pcpu_fini = uncore_pcpu_fini; 800 pcd->pcd_pcpu_init = uncore_pcpu_init; 801 pcd->pcd_read_pmc = ucp_read_pmc; 802 pcd->pcd_release_pmc = ucp_release_pmc; 803 pcd->pcd_start_pmc = ucp_start_pmc; 804 pcd->pcd_stop_pmc = ucp_stop_pmc; 805 pcd->pcd_write_pmc = ucp_write_pmc; 806 807 md->pmd_npmc += npmc; 808 } 809 810 int 811 pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) 812 { 813 uncore_cputype = md->pmd_cputype; 814 uncore_pmcmask = 0; 815 816 /* 817 * Initialize programmable counters. 818 */ 819 820 uncore_ucp_npmc = 8; 821 uncore_ucp_width = 48; 822 823 uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); 824 825 ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); 826 827 /* 828 * Initialize fixed function counters, if present. 829 */ 830 uncore_ucf_ri = uncore_ucp_npmc; 831 uncore_ucf_npmc = 1; 832 uncore_ucf_width = 48; 833 834 ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); 835 uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); 836 837 PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, 838 uncore_ucf_ri); 839 840 uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, 841 M_ZERO | M_WAITOK); 842 843 return (0); 844 } 845 846 void 847 pmc_uncore_finalize(struct pmc_mdep *md) 848 { 849 PMCDBG0(MDP,INI,1, "uncore-finalize"); 850 851 free(uncore_pcpu, M_PMC); 852 uncore_pcpu = NULL; 853 } 854