1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2026, Netflix, Inc. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* Support for the AMD IBS */ 29 30 #include <sys/param.h> 31 #include <sys/lock.h> 32 #include <sys/malloc.h> 33 #include <sys/mutex.h> 34 #include <sys/pcpu.h> 35 #include <sys/pmc.h> 36 #include <sys/pmckern.h> 37 #include <sys/pmclog.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 #include <sys/systm.h> 41 42 #define EXTERR_CATEGORY EXTERR_CAT_HWPMC_IBS 43 #include <sys/exterrvar.h> 44 45 #include <machine/cpu.h> 46 #include <machine/cpufunc.h> 47 #include <machine/md_var.h> 48 #include <machine/specialreg.h> 49 50 #define IBS_STOP_ITER 50 /* Stopping iterations */ 51 52 /* AMD IBS PMCs */ 53 struct ibs_descr { 54 struct pmc_descr pm_descr; /* "base class" */ 55 }; 56 57 /* 58 * Globals 59 */ 60 static uint64_t ibs_features; 61 static uint64_t ibs_fetch_allowed_mask; 62 static uint64_t ibs_op_allowed_mask; 63 static uint64_t ibs_fetch_ctl2_allowed_mask; 64 static uint64_t ibs_op_ctl2_allowed_mask; 65 static bool ibs_fetch_ctl2_supported; 66 static bool ibs_op_ctl2_supported; 67 68 static uint64_t ibs_fetch_extra_mask; 69 static uint64_t ibs_fetch_ctl2_extra_mask; 70 static uint64_t ibs_op_extra_mask; 71 static uint64_t ibs_op_ctl2_extra_mask; 72 73 SYSCTL_DECL(_kern_hwpmc); 74 75 SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_fetch_extra_mask, CTLFLAG_RDTUN, 76 &ibs_fetch_extra_mask, 0, 77 "Extra allowed bits in the IBS fetch control MSR (override; default 0)"); 78 79 SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_fetch_ctl2_extra_mask, CTLFLAG_RDTUN, 80 &ibs_fetch_ctl2_extra_mask, 0, 81 "Extra allowed bits in the IBS fetch control 2 MSR (override; default 0)"); 82 83 SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_op_extra_mask, CTLFLAG_RDTUN, 84 &ibs_op_extra_mask, 0, 85 "Extra allowed bits in the IBS op control MSR (override; default 0)"); 86 87 SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_op_ctl2_extra_mask, CTLFLAG_RDTUN, 88 &ibs_op_ctl2_extra_mask, 0, 89 "Extra allowed bits in the IBS op control 2 MSR (override; default 0)"); 90 91 /* 92 * Per-processor information 93 */ 94 #define IBS_CPU_RUNNING 1 95 #define IBS_CPU_STOPPING 2 96 #define IBS_CPU_STOPPED 3 97 98 struct ibs_cpu { 99 int pc_status; 100 int pc_nmi_credit; /* latched NMIs already serviced */ 101 struct pmc_hw pc_ibspmcs[IBS_NPMCS]; 102 }; 103 static struct ibs_cpu **ibs_pcpu; 104 105 static void 106 ibs_init_policy(void) 107 { 108 109 ibs_fetch_allowed_mask = IBS_FETCH_ALLOWED_MASK_BASE; 110 ibs_fetch_ctl2_allowed_mask = 0; 111 112 ibs_op_allowed_mask = IBS_OP_CTL_MAXCNTBASEMASK; 113 ibs_op_ctl2_allowed_mask = 0; 114 115 if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0) 116 ibs_fetch_allowed_mask |= IBS_FETCH_CTL_L3MISSONLY; 117 118 if ((ibs_features & CPUID_IBSID_OPCNT) != 0) 119 ibs_op_allowed_mask |= IBS_OP_CTL_COUNTERCONTROL; 120 121 if ((ibs_features & CPUID_IBSID_OPCNTEXT) != 0) 122 ibs_op_allowed_mask |= IBS_OP_CTL_MAXCNTEXTMASK; 123 124 if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0) 125 ibs_op_allowed_mask |= IBS_OP_CTL_L3MISSONLY; 126 127 if ((ibs_features & CPUID_IBSID_FETCHLATFILTERING) != 0) 128 ibs_fetch_ctl2_allowed_mask |= IBS_FETCH_CTL2_LATFILTERMASK; 129 130 if ((ibs_features & CPUID_IBSID_STRMSTANDRMTSOCKET) != 0) 131 ibs_op_ctl2_allowed_mask |= IBS_OP_CTL2_STRMSTFILTER; 132 133 if ((ibs_features & CPUID_IBSID_IBSDIS) != 0) { 134 ibs_fetch_ctl2_supported = true; 135 ibs_op_ctl2_supported = true; 136 } 137 138 /* 139 * ctl2 MSRs only exist on Zen 6; writing them on older silicon 140 * would #GP. 141 */ 142 if (!ibs_fetch_ctl2_supported) 143 ibs_fetch_ctl2_supported = (ibs_fetch_ctl2_allowed_mask != 0); 144 if (!ibs_op_ctl2_supported) 145 ibs_op_ctl2_supported = (ibs_op_ctl2_allowed_mask != 0); 146 } 147 148 static int 149 ibs_validate_fetch_config(uint64_t config) 150 { 151 uint64_t allowed_mask; 152 153 allowed_mask = ibs_fetch_allowed_mask | ibs_fetch_extra_mask; 154 if ((config & ~allowed_mask) != 0) { 155 return (EXTERROR(EINVAL, 156 "IBS fetch ctl config 0x%jx has bits outside allowed mask" 157 " 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); 158 } 159 160 return (0); 161 } 162 163 static int 164 ibs_validate_op_config(uint64_t config) 165 { 166 uint64_t allowed_mask; 167 168 allowed_mask = ibs_op_allowed_mask; 169 170 if ((config & IBS_OP_CTL_LATFLTEN) != 0) { 171 if ((ibs_features & CPUID_IBSID_IBSLOADLATENCYFILT) == 0) 172 return (EXTERROR(EINVAL, 173 "IBS op load-latency filter is not supported")); 174 /* 175 * Zen 6 decouples L3MISSONLY from load-latency filtering 176 * (AMD pub 69205); enforce the pairing only on older parts. 177 */ 178 if ((ibs_features & CPUID_IBSID_IBSDIS) == 0 && 179 (config & IBS_OP_CTL_L3MISSONLY) == 0) 180 return (EXTERROR(EINVAL, 181 "IBS op load-latency filter requires l3miss")); 182 183 allowed_mask |= IBS_OP_CTL_LDLATMASK | IBS_OP_CTL_L3MISSONLY; 184 } 185 186 allowed_mask |= ibs_op_extra_mask; 187 188 if ((config & ~allowed_mask) != 0) { 189 return (EXTERROR(EINVAL, 190 "IBS op ctl config 0x%jx has bits outside allowed mask" 191 " 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); 192 } 193 194 return (0); 195 } 196 197 static int 198 ibs_validate_fetch_ctl2_config(uint64_t config) 199 { 200 uint64_t allowed_mask; 201 202 if (config == 0) 203 return (0); 204 205 if (!ibs_fetch_ctl2_supported) 206 return (EXTERROR(EINVAL, 207 "IBS fetch ctl2 features are not supported on this CPU")); 208 209 allowed_mask = ibs_fetch_ctl2_allowed_mask | ibs_fetch_ctl2_extra_mask; 210 211 if ((config & ~allowed_mask) != 0) { 212 return (EXTERROR(EINVAL, 213 "IBS fetch ctl2 config 0x%jx has bits outside allowed" 214 " mask 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); 215 } 216 217 return (0); 218 } 219 220 static int 221 ibs_validate_op_ctl2_config(uint64_t config) 222 { 223 uint64_t allowed_mask; 224 225 if (config == 0) 226 return (0); 227 228 if (!ibs_op_ctl2_supported) 229 return (EXTERROR(EINVAL, 230 "IBS op ctl2 features are not supported on this CPU")); 231 232 allowed_mask = ibs_op_ctl2_allowed_mask | ibs_op_ctl2_extra_mask; 233 234 if ((config & ~allowed_mask) != 0) { 235 return (EXTERROR(EINVAL, 236 "IBS op ctl2 config 0x%jx has bits outside allowed mask" 237 " 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); 238 } 239 240 return (0); 241 } 242 243 static int 244 ibs_validate_pmc_config(int ri, uint64_t config, uint64_t config2) 245 { 246 int error; 247 248 switch (ri) { 249 case IBS_PMC_FETCH: 250 error = ibs_validate_fetch_config(config); 251 if (error != 0) 252 return (error); 253 return (ibs_validate_fetch_ctl2_config(config2)); 254 case IBS_PMC_OP: 255 error = ibs_validate_op_config(config); 256 if (error != 0) 257 return (error); 258 return (ibs_validate_op_ctl2_config(config2)); 259 default: 260 return (EXTERROR(EINVAL, "Unsupported IBS PMC type %ju", 261 (uintmax_t)ri)); 262 } 263 } 264 265 /* 266 * Read a PMC value from the MSR. 267 */ 268 static int 269 ibs_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 270 { 271 272 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 273 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); 274 KASSERT(ri >= 0 && ri < IBS_NPMCS, 275 ("[ibs,%d] illegal row-index %d", __LINE__, ri)); 276 KASSERT(ibs_pcpu[cpu], 277 ("[ibs,%d] null per-cpu, cpu %d", __LINE__, cpu)); 278 279 /* read the IBS count */ 280 switch (ri) { 281 case IBS_PMC_FETCH: 282 *v = IBS_FETCH_CTL_TO_COUNT(rdmsr(IBS_FETCH_CTL)); 283 break; 284 case IBS_PMC_OP: 285 *v = IBS_OP_CTL_TO_COUNT(rdmsr(IBS_OP_CTL)); 286 break; 287 } 288 289 PMCDBG2(MDP, REA, 2, "ibs-read id=%d -> %jd", ri, *v); 290 291 return (0); 292 } 293 294 /* 295 * Write a PMC MSR. 296 */ 297 static int 298 ibs_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 299 { 300 pmc_value_t m; 301 302 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 303 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); 304 KASSERT(ri >= 0 && ri < IBS_NPMCS, 305 ("[ibs,%d] illegal row-index %d", __LINE__, ri)); 306 307 /* write the IBS count */ 308 switch (ri) { 309 case IBS_PMC_FETCH: 310 m = rdmsr(IBS_FETCH_CTL) & ~IBS_FETCH_CTL_CURCNTMASK; 311 /* Setting a count greater than interval is undefined. */ 312 if (IBS_FETCH_CTL_TO_INTERVAL(m) > v) 313 m |= IBS_FETCH_COUNT_TO_CTL(v); 314 wrmsr(IBS_FETCH_CTL, m); 315 break; 316 case IBS_PMC_OP: 317 m = rdmsr(IBS_OP_CTL) & ~IBS_OP_CTL_CURCNTMASK; 318 /* Setting a count greater than interval is undefined */ 319 if (IBS_OP_CTL_TO_INTERVAL(m) > v) 320 m |= IBS_OP_COUNT_TO_CTL(v); 321 wrmsr(IBS_OP_CTL, m); 322 break; 323 } 324 325 PMCDBG3(MDP, WRI, 1, "ibs-write cpu=%d ri=%d v=%jx", cpu, ri, v); 326 327 return (0); 328 } 329 330 /* 331 * Configure hardware PMC according to the configuration recorded in 'pm'. 332 */ 333 static int 334 ibs_config_pmc(int cpu, int ri, struct pmc *pm) 335 { 336 struct pmc_hw *phw; 337 338 PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 339 340 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 341 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); 342 KASSERT(ri >= 0 && ri < IBS_NPMCS, 343 ("[ibs,%d] illegal row-index %d", __LINE__, ri)); 344 345 phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; 346 347 KASSERT(pm == NULL || phw->phw_pmc == NULL, 348 ("[ibs,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 349 __LINE__, pm, phw->phw_pmc)); 350 351 phw->phw_pmc = pm; 352 353 return (0); 354 } 355 356 /* 357 * Retrieve a configured PMC pointer from hardware state. 358 */ 359 static int 360 ibs_get_config(int cpu, int ri, struct pmc **ppm) 361 { 362 363 *ppm = ibs_pcpu[cpu]->pc_ibspmcs[ri].phw_pmc; 364 365 return (0); 366 } 367 368 /* 369 * Check if a given PMC allocation is feasible. 370 */ 371 static int 372 ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, 373 const struct pmc_op_pmcallocate *a) 374 { 375 uint64_t caps, config, config2; 376 int error; 377 378 KASSERT(ri >= 0 && ri < IBS_NPMCS, 379 ("[ibs,%d] illegal row index %d", __LINE__, ri)); 380 381 /* Row discriminators; the allocation loop probes every row. */ 382 if (a->pm_class != PMC_CLASS_IBS) 383 return (EINVAL); 384 if (a->pm_md.pm_ibs.ibs_type != ri) 385 return (EINVAL); 386 387 caps = pm->pm_caps; 388 389 PMCDBG2(MDP, ALL, 1, "ibs-allocate ri=%d caps=0x%x", ri, caps); 390 391 if ((ibs_features & CPUID_IBSID_ADDRBIT63FILTERING) != 0) { 392 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 393 return (EXTERROR(EINVAL, 394 "IBS requires at least USER or SYSTEM capability")); 395 } else { 396 if ((caps & PMC_CAP_SYSTEM) == 0) 397 return (EXTERROR(EINVAL, 398 "IBS requires SYSTEM capability")); 399 if ((caps & PMC_CAP_USER) != 0) 400 return (EXTERROR(EINVAL, 401 "IBS USER filtering requires Zen 6 addr63 support")); 402 } 403 404 if (!PMC_IS_SAMPLING_MODE(a->pm_mode)) 405 return (EXTERROR(EINVAL, "IBS only supports sampling mode")); 406 407 config = a->pm_md.pm_ibs.ibs_ctl; 408 config2 = a->pm_md.pm_ibs.ibs_ctl2; 409 error = ibs_validate_pmc_config(ri, config, config2); 410 if (error != 0) 411 return (error); 412 pm->pm_md.pm_ibs.ibs_ctl = config; 413 pm->pm_md.pm_ibs.ibs_ctl2 = config2; 414 415 if ((ibs_features & CPUID_IBSID_ADDRBIT63FILTERING) != 0) { 416 if ((caps & PMC_CAP_USER) != 0 && 417 (caps & PMC_CAP_SYSTEM) == 0) { 418 if (ri == IBS_PMC_FETCH) 419 pm->pm_md.pm_ibs.ibs_ctl2 |= 420 IBS_FETCH_CTL2_EXCLADDR63EQ1; 421 else 422 pm->pm_md.pm_ibs.ibs_ctl2 |= 423 IBS_OP_CTL2_EXCLRIP63EQ1; 424 } else if ((caps & PMC_CAP_SYSTEM) != 0 && 425 (caps & PMC_CAP_USER) == 0) { 426 if (ri == IBS_PMC_FETCH) 427 pm->pm_md.pm_ibs.ibs_ctl2 |= 428 IBS_FETCH_CTL2_EXCLADDR63EQ0; 429 else 430 pm->pm_md.pm_ibs.ibs_ctl2 |= 431 IBS_OP_CTL2_EXCLRIP63EQ0; 432 } 433 } 434 435 PMCDBG3(MDP, ALL, 2, 436 "ibs-allocate ri=%d -> config=0x%jx config2=0x%jx", ri, 437 config, config2); 438 439 return (0); 440 } 441 442 /* 443 * Release machine dependent state associated with a PMC. This is a 444 * no-op on this architecture. 445 */ 446 static int 447 ibs_release_pmc(int cpu, int ri, struct pmc *pmc __unused) 448 { 449 struct pmc_hw *phw __diagused; 450 451 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 452 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); 453 KASSERT(ri >= 0 && ri < IBS_NPMCS, 454 ("[ibs,%d] illegal row-index %d", __LINE__, ri)); 455 456 PMCDBG1(MDP, ALL, 1, "ibs-release ri=%d", ri); 457 458 phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; 459 460 KASSERT(phw->phw_pmc == NULL, 461 ("[ibs,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 462 463 return (0); 464 } 465 466 /* 467 * Start a PMC. 468 */ 469 static int 470 ibs_start_pmc(int cpu __diagused, int ri, struct pmc *pm) 471 { 472 uint64_t config; 473 474 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 475 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); 476 KASSERT(ri >= 0 && ri < IBS_NPMCS, 477 ("[ibs,%d] illegal row-index %d", __LINE__, ri)); 478 479 PMCDBG2(MDP, STA, 1, "ibs-start cpu=%d ri=%d", cpu, ri); 480 481 /* 482 * This is used to handle spurious NMIs. All that matters is that it 483 * is not in the stopping state. 484 */ 485 atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_RUNNING); 486 487 /* 488 * Turn on the ENABLE bit. Zeroing out the control register eliminates 489 * stale valid bits from spurious NMIs and it resets the counter. This 490 * is safe here because the counter is not yet enabled; the NMI re-arm 491 * path must not do the same (Family 10h erratum #420). 492 */ 493 switch (ri) { 494 case IBS_PMC_FETCH: 495 wrmsr(IBS_FETCH_CTL, 0); 496 if (ibs_fetch_ctl2_supported) 497 wrmsr(IBS_FETCH_CTL2, 498 pm->pm_md.pm_ibs.ibs_ctl2 & ~IBS_FETCH_CTL2_DISABLE); 499 config = pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE; 500 wrmsr(IBS_FETCH_CTL, config); 501 break; 502 case IBS_PMC_OP: 503 wrmsr(IBS_OP_CTL, 0); 504 if (ibs_op_ctl2_supported) 505 wrmsr(IBS_OP_CTL2, 506 pm->pm_md.pm_ibs.ibs_ctl2 & ~IBS_OP_CTL2_DISABLE); 507 config = pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE; 508 wrmsr(IBS_OP_CTL, config); 509 break; 510 } 511 512 return (0); 513 } 514 515 /* 516 * Stop a PMC. 517 */ 518 static int 519 ibs_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) 520 { 521 int i; 522 uint64_t config, config2; 523 bool use_alt_disable; 524 525 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 526 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); 527 KASSERT(ri >= 0 && ri < IBS_NPMCS, 528 ("[ibs,%d] illegal row-index %d", __LINE__, ri)); 529 530 PMCDBG1(MDP, STO, 1, "ibs-stop ri=%d", ri); 531 532 /* 533 * Turn off the ENABLE bit, but unfortunately there are a few quirks 534 * that generate excess NMIs. Workaround #420 in the Revision Guide 535 * for AMD Family 10h Processors 41322 Rev. 3.92 March 2012. requires 536 * that we clear the max count before clearing enable. 537 * 538 * Even after clearing the counter spurious NMIs are still possible so 539 * we use a per-CPU atomic variable to notify the interrupt handler we 540 * are stopping and discard spurious NMIs. We then retry clearing the 541 * control register for 50us. This gives us enough time and ensures 542 * that the valid bit is not accidently stuck after a spurious NMI. 543 * 544 * On Zen 6 with the alternate disable bit (CPUID IbsDis), assert the 545 * ctl2 DISABLE bit first. This avoids an RMW hazard in ctl1 that the 546 * processor may update concurrently while sampling. 547 */ 548 config = pm->pm_md.pm_ibs.ibs_ctl; 549 config2 = pm->pm_md.pm_ibs.ibs_ctl2; 550 use_alt_disable = (ibs_features & CPUID_IBSID_IBSDIS) != 0; 551 552 atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPING); 553 554 /* 555 * On Zen 6, ctl2 DISABLE is the authoritative stop switch; skip 556 * the legacy ctl1 RMW and clear it directly 557 */ 558 switch (ri) { 559 case IBS_PMC_FETCH: 560 if (use_alt_disable) { 561 wrmsr(IBS_FETCH_CTL2, 562 config2 | IBS_FETCH_CTL2_DISABLE); 563 wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_ENABLE); 564 } else { 565 wrmsr(IBS_FETCH_CTL, 566 config & ~IBS_FETCH_CTL_MAXCNTMASK); 567 DELAY(1); 568 config &= ~IBS_FETCH_CTL_ENABLE; 569 wrmsr(IBS_FETCH_CTL, config); 570 } 571 break; 572 case IBS_PMC_OP: 573 if (use_alt_disable) { 574 wrmsr(IBS_OP_CTL2, 575 config2 | IBS_OP_CTL2_DISABLE); 576 wrmsr(IBS_OP_CTL, config & ~IBS_OP_CTL_ENABLE); 577 } else { 578 wrmsr(IBS_OP_CTL, 579 config & ~IBS_OP_CTL_MAXCNTMASK); 580 DELAY(1); 581 config &= ~IBS_OP_CTL_ENABLE; 582 wrmsr(IBS_OP_CTL, config); 583 } 584 break; 585 } 586 587 for (i = 0; i < IBS_STOP_ITER; i++) { 588 DELAY(1); 589 590 switch (ri) { 591 case IBS_PMC_FETCH: 592 wrmsr(IBS_FETCH_CTL, 0); 593 if (ibs_fetch_ctl2_supported) 594 wrmsr(IBS_FETCH_CTL2, 0); 595 break; 596 case IBS_PMC_OP: 597 wrmsr(IBS_OP_CTL, 0); 598 if (ibs_op_ctl2_supported) 599 wrmsr(IBS_OP_CTL2, 0); 600 break; 601 } 602 } 603 604 atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPED); 605 606 return (0); 607 } 608 609 static void 610 pmc_ibs_process_fetch(struct pmc *pm, struct trapframe *tf, uint64_t config) 611 { 612 struct pmc_multipart mpd; 613 614 if (pm == NULL) 615 return; 616 617 if (pm->pm_state != PMC_STATE_RUNNING) 618 return; 619 620 memset(&mpd, 0, sizeof(mpd)); 621 622 mpd.pl_type = PMC_CC_MULTIPART_IBS_FETCH; 623 mpd.pl_length = PMC_MPIDX_FETCH_MAX; 624 mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config; 625 if ((ibs_features & CPUID_IBSID_IBSFETCHCTLEXTD) != 0) { 626 mpd.pl_mpdata[PMC_MPIDX_FETCH_EXTCTL] = rdmsr(IBS_FETCH_EXTCTL); 627 } 628 mpd.pl_mpdata[PMC_MPIDX_FETCH_LINADDR] = rdmsr(IBS_FETCH_LINADDR); 629 if ((config & IBS_FETCH_CTL_PHYSADDRVALID) != 0) { 630 mpd.pl_mpdata[PMC_MPIDX_FETCH_PHYSADDR] = 631 rdmsr(IBS_FETCH_PHYSADDR); 632 } 633 if (ibs_fetch_ctl2_supported) { 634 mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL2] = rdmsr(IBS_FETCH_CTL2); 635 } 636 637 pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd); 638 639 wrmsr(IBS_FETCH_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE); 640 } 641 642 static void 643 pmc_ibs_process_op(struct pmc *pm, struct trapframe *tf, uint64_t config) 644 { 645 struct pmc_multipart mpd; 646 647 if (pm == NULL) 648 return; 649 650 if (pm->pm_state != PMC_STATE_RUNNING) 651 return; 652 653 memset(&mpd, 0, sizeof(mpd)); 654 655 mpd.pl_type = PMC_CC_MULTIPART_IBS_OP; 656 mpd.pl_length = PMC_MPIDX_OP_MAX; 657 mpd.pl_mpdata[PMC_MPIDX_OP_CTL] = config; 658 mpd.pl_mpdata[PMC_MPIDX_OP_RIP] = rdmsr(IBS_OP_RIP); 659 mpd.pl_mpdata[PMC_MPIDX_OP_DATA] = rdmsr(IBS_OP_DATA); 660 mpd.pl_mpdata[PMC_MPIDX_OP_DATA2] = rdmsr(IBS_OP_DATA2); 661 mpd.pl_mpdata[PMC_MPIDX_OP_DATA3] = rdmsr(IBS_OP_DATA3); 662 mpd.pl_mpdata[PMC_MPIDX_OP_DC_LINADDR] = rdmsr(IBS_OP_DC_LINADDR); 663 mpd.pl_mpdata[PMC_MPIDX_OP_DC_PHYSADDR] = rdmsr(IBS_OP_DC_PHYSADDR); 664 if ((ibs_features & CPUID_IBSID_BRNTRGT) != 0) { 665 mpd.pl_mpdata[PMC_MPIDX_OP_TGT_RIP] = rdmsr(IBS_OP_TGT_RIP); 666 } 667 if ((ibs_features & CPUID_IBSID_IBSOPDATA4) != 0) { 668 mpd.pl_mpdata[PMC_MPIDX_OP_DATA4] = rdmsr(IBS_OP_DATA4); 669 } 670 if (ibs_op_ctl2_supported) { 671 mpd.pl_mpdata[PMC_MPIDX_OP_CTL2] = rdmsr(IBS_OP_CTL2); 672 } 673 674 pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd); 675 676 wrmsr(IBS_OP_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE); 677 } 678 679 /* 680 * Interrupt handler. This function needs to return '1' if the 681 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 682 * to sleep or do anything a 'fast' interrupt handler is not allowed 683 * to do. 684 */ 685 int 686 pmc_ibs_intr(struct trapframe *tf) 687 { 688 struct ibs_cpu *pac; 689 struct pmc *pm; 690 int retval, cpu; 691 uint64_t config; 692 693 if (ibs_pcpu == NULL) 694 return (0); 695 696 cpu = curcpu; 697 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 698 ("[ibs,%d] out of range CPU %d", __LINE__, cpu)); 699 700 PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); 701 702 retval = 0; 703 704 pac = ibs_pcpu[cpu]; 705 706 /* 707 * Fetch and op share an NMI. Both valid bits may be set on entry, 708 * so service each valid source or the skipped unit can stay frozen 709 * with no NMI pending to re-arm it. 710 */ 711 config = rdmsr(IBS_FETCH_CTL); 712 if ((config & IBS_FETCH_CTL_VALID) != 0) { 713 pm = pac->pc_ibspmcs[IBS_PMC_FETCH].phw_pmc; 714 715 retval++; 716 717 pmc_ibs_process_fetch(pm, tf, config); 718 } 719 720 config = rdmsr(IBS_OP_CTL); 721 if ((config & IBS_OP_CTL_VALID) != 0) { 722 pm = pac->pc_ibspmcs[IBS_PMC_OP].phw_pmc; 723 724 retval++; 725 726 pmc_ibs_process_op(pm, tf, config); 727 } 728 729 /* 730 * When both units were serviced, the second unit's NMI may still be 731 * latched and will arrive with no valid bit set. Claim that one 732 * NMI so it is not reported as unknown. 733 */ 734 if (retval == 0 && pac->pc_nmi_credit != 0) 735 retval = 1; 736 pac->pc_nmi_credit = (retval == 2); 737 if (retval > 1) 738 retval = 1; 739 740 if (retval == 0) { 741 // Lets check for a stray NMI when stopping 742 if (atomic_load_int(&pac->pc_status) == IBS_CPU_STOPPING) { 743 return (1); 744 } 745 } 746 747 748 if (retval) 749 counter_u64_add(pmc_stats.pm_intr_processed, 1); 750 else 751 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 752 753 PMCDBG1(MDP, INT, 2, "retval=%d", retval); 754 755 return (retval); 756 } 757 758 /* 759 * Describe a PMC. 760 */ 761 static int 762 ibs_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 763 { 764 struct pmc_hw *phw; 765 766 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 767 ("[ibs,%d] illegal CPU %d", __LINE__, cpu)); 768 KASSERT(ri >= 0 && ri < IBS_NPMCS, 769 ("[ibs,%d] row-index %d out of range", __LINE__, ri)); 770 771 phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; 772 773 if (ri == IBS_PMC_FETCH) { 774 strlcpy(pi->pm_name, "IBS-FETCH", sizeof(pi->pm_name)); 775 pi->pm_class = PMC_CLASS_IBS; 776 pi->pm_enabled = true; 777 *ppmc = phw->phw_pmc; 778 } else { 779 strlcpy(pi->pm_name, "IBS-OP", sizeof(pi->pm_name)); 780 pi->pm_class = PMC_CLASS_IBS; 781 pi->pm_enabled = true; 782 *ppmc = phw->phw_pmc; 783 } 784 785 return (0); 786 } 787 788 /* 789 * Processor-dependent initialization. 790 */ 791 static int 792 ibs_pcpu_init(struct pmc_mdep *md, int cpu) 793 { 794 struct ibs_cpu *pac; 795 struct pmc_cpu *pc; 796 struct pmc_hw *phw; 797 int first_ri, n; 798 799 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 800 ("[ibs,%d] insane cpu number %d", __LINE__, cpu)); 801 802 PMCDBG1(MDP, INI, 1, "ibs-init cpu=%d", cpu); 803 804 ibs_pcpu[cpu] = pac = malloc(sizeof(struct ibs_cpu), M_PMC, 805 M_WAITOK | M_ZERO); 806 807 /* 808 * Set the content of the hardware descriptors to a known 809 * state and initialize pointers in the MI per-cpu descriptor. 810 */ 811 pc = pmc_pcpu[cpu]; 812 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri; 813 814 KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu pointer", __LINE__)); 815 816 for (n = 0, phw = pac->pc_ibspmcs; n < IBS_NPMCS; n++, phw++) { 817 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 818 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 819 phw->phw_pmc = NULL; 820 pc->pc_hwpmcs[n + first_ri] = phw; 821 } 822 823 return (0); 824 } 825 826 /* 827 * Processor-dependent cleanup prior to the KLD being unloaded. 828 */ 829 static int 830 ibs_pcpu_fini(struct pmc_mdep *md, int cpu) 831 { 832 struct ibs_cpu *pac; 833 struct pmc_cpu *pc; 834 int first_ri, i; 835 836 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 837 ("[ibs,%d] insane cpu number (%d)", __LINE__, cpu)); 838 839 PMCDBG1(MDP, INI, 1, "ibs-cleanup cpu=%d", cpu); 840 841 /* 842 * Turn off IBS. 843 */ 844 wrmsr(IBS_FETCH_CTL, 0); 845 wrmsr(IBS_OP_CTL, 0); 846 if (ibs_fetch_ctl2_supported) 847 wrmsr(IBS_FETCH_CTL2, 0); 848 if (ibs_op_ctl2_supported) 849 wrmsr(IBS_OP_CTL2, 0); 850 851 /* 852 * Free up allocated space. 853 */ 854 if ((pac = ibs_pcpu[cpu]) == NULL) 855 return (0); 856 857 ibs_pcpu[cpu] = NULL; 858 859 pc = pmc_pcpu[cpu]; 860 KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu state", __LINE__)); 861 862 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri; 863 864 /* 865 * Reset pointers in the MI 'per-cpu' state. 866 */ 867 for (i = 0; i < IBS_NPMCS; i++) 868 pc->pc_hwpmcs[i + first_ri] = NULL; 869 870 free(pac, M_PMC); 871 872 return (0); 873 } 874 875 /* 876 * Initialize ourselves. 877 */ 878 int 879 pmc_ibs_initialize(struct pmc_mdep *pmc_mdep, int ncpus) 880 { 881 u_int regs[4]; 882 struct pmc_classdep *pcd; 883 884 /* 885 * Allocate space for pointers to PMC HW descriptors and for 886 * the MDEP structure used by MI code. 887 */ 888 ibs_pcpu = malloc(sizeof(struct ibs_cpu *) * pmc_cpu_max(), M_PMC, 889 M_WAITOK | M_ZERO); 890 891 /* Initialize AMD IBS handling. */ 892 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS]; 893 894 pcd->pcd_caps = IBS_PMC_CAPS; 895 pcd->pcd_class = PMC_CLASS_IBS; 896 pcd->pcd_num = IBS_NPMCS; 897 pcd->pcd_ri = pmc_mdep->pmd_npmc; 898 pcd->pcd_width = 0; 899 900 pcd->pcd_allocate_pmc = ibs_allocate_pmc; 901 pcd->pcd_config_pmc = ibs_config_pmc; 902 pcd->pcd_describe = ibs_describe; 903 pcd->pcd_get_config = ibs_get_config; 904 pcd->pcd_pcpu_fini = ibs_pcpu_fini; 905 pcd->pcd_pcpu_init = ibs_pcpu_init; 906 pcd->pcd_release_pmc = ibs_release_pmc; 907 pcd->pcd_start_pmc = ibs_start_pmc; 908 pcd->pcd_stop_pmc = ibs_stop_pmc; 909 pcd->pcd_read_pmc = ibs_read_pmc; 910 pcd->pcd_write_pmc = ibs_write_pmc; 911 912 pmc_mdep->pmd_npmc += IBS_NPMCS; 913 914 if (cpu_exthigh >= CPUID_IBSID) { 915 do_cpuid(CPUID_IBSID, regs); 916 ibs_features = regs[0]; 917 if ((ibs_features & CPUID_IBSID_IBSFFV) == 0) 918 ibs_features = 0; 919 } else { 920 ibs_features = 0; 921 } 922 923 ibs_init_policy(); 924 925 PMCDBG0(MDP, INI, 0, "ibs-initialize"); 926 927 return (0); 928 } 929 930 /* 931 * Finalization code for AMD CPUs. 932 */ 933 void 934 pmc_ibs_finalize(struct pmc_mdep *md) 935 { 936 PMCDBG0(MDP, INI, 1, "ibs-finalize"); 937 938 /* Safe even if the IBS class was skipped at initialize time. */ 939 if (ibs_pcpu == NULL) 940 return; 941 942 for (int i = 0; i < pmc_cpu_max(); i++) 943 KASSERT(ibs_pcpu[i] == NULL, 944 ("[ibs,%d] non-null pcpu cpu %d", __LINE__, i)); 945 946 free(ibs_pcpu, M_PMC); 947 ibs_pcpu = NULL; 948 } 949