1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* Support for the AMD K8 and later processors */ 34 35 #include <sys/param.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/mutex.h> 39 #include <sys/pcpu.h> 40 #include <sys/pmc.h> 41 #include <sys/pmckern.h> 42 #include <sys/smp.h> 43 #include <sys/sysctl.h> 44 #include <sys/systm.h> 45 46 #define EXTERR_CATEGORY EXTERR_CAT_HWPMC_AMD 47 #include <sys/exterrvar.h> 48 49 #include <machine/cpu.h> 50 #include <machine/cpufunc.h> 51 #include <machine/md_var.h> 52 #include <machine/specialreg.h> 53 54 #define OVERFLOW_WAIT_COUNT 50 55 56 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter); 57 58 /* AMD K8 PMCs */ 59 struct amd_descr { 60 struct pmc_descr pm_descr; /* "base class" */ 61 uint32_t pm_evsel; /* address of EVSEL register */ 62 uint32_t pm_perfctr; /* address of PERFCTR register */ 63 enum sub_class pm_subclass; /* register subclass */ 64 }; 65 66 static int amd_npmcs; 67 static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs, amd_umc_npmcs; 68 static struct amd_descr amd_pmcdesc[AMD_NPMCS_MAX]; 69 struct amd_event_code_map { 70 enum pmc_event pe_ev; /* enum value */ 71 uint16_t pe_code; /* encoded event mask */ 72 uint8_t pe_mask; /* bits allowed in unit mask */ 73 }; 74 75 const struct amd_event_code_map amd_event_codes[] = { 76 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 77 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 78 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 79 80 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 81 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 82 0x21, 0x00 }, 83 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 84 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 85 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 86 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 87 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 88 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 89 90 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 91 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 92 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 93 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 94 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 95 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 96 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 97 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 98 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 99 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 100 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 101 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 102 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 103 104 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 105 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 106 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 107 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 108 109 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 110 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 111 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 112 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 113 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 114 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 115 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 116 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 117 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 118 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 119 120 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 121 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 122 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 123 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 124 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 125 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 126 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 127 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 128 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 129 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 130 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 131 0xCA, 0x00 }, 132 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 133 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 134 0xCC, 0x07 }, 135 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 136 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 137 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 138 139 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 140 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 141 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 142 0xD2, 0x00 }, 143 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 144 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 145 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 146 0xD5, 0x00 }, 147 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 148 0xD6, 0x00 }, 149 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 150 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 151 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 152 0xD9, 0x00 }, 153 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 154 0xDA, 0x00 }, 155 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 156 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 157 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 158 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 159 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 160 161 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 162 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 163 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 164 0xE2, 0x00 }, 165 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 166 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 167 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 168 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 169 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 170 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 171 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 172 173 }; 174 175 const int amd_event_codes_size = nitems(amd_event_codes); 176 177 /* 178 * Per-processor information 179 */ 180 struct amd_cpu { 181 struct pmc_hw pc_amdpmcs[AMD_NPMCS_MAX]; 182 }; 183 static struct amd_cpu **amd_pcpu; 184 185 /* Populated by amd_init_policy(); PRECISERETIRE is OR-ed in per-allocation. */ 186 static uint64_t amd_core_allowed_mask; 187 static uint64_t amd_l3_allowed_mask; 188 static uint64_t amd_df_allowed_mask; 189 static uint64_t amd_umc_allowed_mask; 190 191 static uint64_t amd_core_extra_mask; 192 static uint64_t amd_l3_extra_mask; 193 static uint64_t amd_df_extra_mask; 194 static uint64_t amd_umc_extra_mask; 195 196 SYSCTL_DECL(_kern_hwpmc); 197 198 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_core_extra_mask, CTLFLAG_RDTUN, 199 &amd_core_extra_mask, 0, 200 "Extra allowed bits in AMD core PMU PERFEVTSEL (override; default 0)"); 201 202 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_l3_extra_mask, CTLFLAG_RDTUN, 203 &amd_l3_extra_mask, 0, 204 "Extra allowed bits in AMD L3 PMU control (override; default 0)"); 205 206 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_df_extra_mask, CTLFLAG_RDTUN, 207 &amd_df_extra_mask, 0, 208 "Extra allowed bits in AMD DF PMU control (override; default 0)"); 209 210 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_umc_extra_mask, CTLFLAG_RDTUN, 211 &amd_umc_extra_mask, 0, 212 "Extra allowed bits in AMD UMC PMU control (override; default 0)"); 213 214 static void 215 amd_init_policy(void) 216 { 217 int family; 218 219 family = CPUID_TO_FAMILY(cpu_id); 220 221 amd_core_allowed_mask = AMD_VALID_BITS; 222 223 amd_l3_allowed_mask = (family <= 0x17) ? 224 AMD_PMC_L3_FAMILY17_MASK : AMD_PMC_L3_FAMILY19_MASK; 225 226 amd_df_allowed_mask = (family <= 0x19) ? 227 AMD_PMC_DF_FAMILY17_MASK : AMD_PMC_DF_FAMILY1A_MASK; 228 229 amd_umc_allowed_mask = AMD_PMC_UMC_MASK; 230 } 231 232 static uint64_t 233 amd_config_mask(enum sub_class subclass, uint64_t caps) 234 { 235 236 switch (subclass) { 237 case PMC_AMD_SUB_CLASS_CORE: 238 return (amd_core_allowed_mask | amd_core_extra_mask | 239 (((caps & PMC_CAP_PRECISE) != 0) ? 240 AMD_PMC_PRECISERETIRE : 0)); 241 case PMC_AMD_SUB_CLASS_L3_CACHE: 242 return (amd_l3_allowed_mask | amd_l3_extra_mask); 243 case PMC_AMD_SUB_CLASS_DATA_FABRIC: 244 return (amd_df_allowed_mask | amd_df_extra_mask); 245 case PMC_AMD_SUB_CLASS_UMC: 246 return (amd_umc_allowed_mask | amd_umc_extra_mask); 247 default: 248 return (0); 249 } 250 } 251 252 /* 253 * Read a PMC value from the MSR. 254 */ 255 static int 256 amd_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 257 { 258 const struct amd_descr *pd; 259 pmc_value_t tmp; 260 enum pmc_mode mode; 261 262 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 263 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 264 KASSERT(ri >= 0 && ri < amd_npmcs, 265 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 266 KASSERT(amd_pcpu[cpu], 267 ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); 268 269 pd = &amd_pmcdesc[ri]; 270 mode = PMC_TO_MODE(pm); 271 272 PMCDBG2(MDP, REA, 1, "amd-read id=%d class=%d", ri, 273 pd->pm_descr.pd_class); 274 275 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 276 PMCDBG2(MDP, REA, 2, "amd-read (pre-munge) id=%d -> %jd", ri, tmp); 277 if (PMC_IS_SAMPLING_MODE(mode)) { 278 /* 279 * Clamp value to 0 if the counter just overflowed, 280 * otherwise the returned reload count would wrap to a 281 * huge value. 282 */ 283 if ((tmp & (1ULL << 47)) == 0) 284 tmp = 0; 285 else { 286 /* Sign extend 48 bit value to 64 bits. */ 287 tmp = (pmc_value_t) ((int64_t)(tmp << 16) >> 16); 288 tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 289 } 290 } 291 *v = tmp; 292 293 PMCDBG2(MDP, REA, 2, "amd-read (post-munge) id=%d -> %jd", ri, *v); 294 295 return (0); 296 } 297 298 /* 299 * Write a PMC MSR. 300 */ 301 static int 302 amd_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 303 { 304 const struct amd_descr *pd; 305 enum pmc_mode mode; 306 307 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 308 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 309 KASSERT(ri >= 0 && ri < amd_npmcs, 310 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 311 312 pd = &amd_pmcdesc[ri]; 313 mode = PMC_TO_MODE(pm); 314 315 /* use 2's complement of the count for sampling mode PMCs */ 316 if (PMC_IS_SAMPLING_MODE(mode)) 317 v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 318 319 PMCDBG3(MDP, WRI, 1, "amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 320 321 /* write the PMC value */ 322 wrmsr(pd->pm_perfctr, v); 323 return (0); 324 } 325 326 /* 327 * Configure hardware PMC according to the configuration recorded in 'pm'. 328 */ 329 static int 330 amd_config_pmc(int cpu, int ri, struct pmc *pm) 331 { 332 struct pmc_hw *phw; 333 334 PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 335 336 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 337 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 338 KASSERT(ri >= 0 && ri < amd_npmcs, 339 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 340 341 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 342 343 KASSERT(pm == NULL || phw->phw_pmc == NULL, 344 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 345 __LINE__, pm, phw->phw_pmc)); 346 347 phw->phw_pmc = pm; 348 return (0); 349 } 350 351 /* 352 * Retrieve a configured PMC pointer from hardware state. 353 */ 354 static int 355 amd_get_config(int cpu, int ri, struct pmc **ppm) 356 { 357 *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; 358 return (0); 359 } 360 361 /* 362 * Machine-dependent actions taken during the context switch in of a 363 * thread. 364 */ 365 static int 366 amd_switch_in(struct pmc_cpu *pc __pmcdbg_used, struct pmc_process *pp) 367 { 368 PMCDBG3(MDP, SWI, 1, "pc=%p pp=%p enable-msr=%d", pc, pp, 369 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 370 371 /* enable the RDPMC instruction if needed */ 372 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 373 load_cr4(rcr4() | CR4_PCE); 374 375 return (0); 376 } 377 378 /* 379 * Machine-dependent actions taken during the context switch out of a 380 * thread. 381 */ 382 static int 383 amd_switch_out(struct pmc_cpu *pc __pmcdbg_used, 384 struct pmc_process *pp __pmcdbg_used) 385 { 386 PMCDBG3(MDP, SWO, 1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 387 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 388 389 /* always turn off the RDPMC instruction */ 390 load_cr4(rcr4() & ~CR4_PCE); 391 392 return (0); 393 } 394 395 /* 396 * Check if a given PMC allocation is feasible. 397 */ 398 static int 399 amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, 400 const struct pmc_op_pmcallocate *a) 401 { 402 const struct pmc_descr *pd; 403 uint64_t allowed_unitmask, caps, config, unitmask; 404 enum pmc_event pe; 405 int i; 406 407 KASSERT(ri >= 0 && ri < amd_npmcs, 408 ("[amd,%d] illegal row index %d", __LINE__, ri)); 409 410 pd = &amd_pmcdesc[ri].pm_descr; 411 412 /* check class match */ 413 if (pd->pd_class != a->pm_class) 414 return (EINVAL); 415 416 caps = pm->pm_caps; 417 418 PMCDBG2(MDP, ALL, 1,"amd-allocate ri=%d caps=0x%x", ri, caps); 419 420 /* Validate sub-class. */ 421 if (amd_pmcdesc[ri].pm_subclass != a->pm_md.pm_amd.pm_amd_sub_class) 422 return (EINVAL); 423 424 if (((caps & PMC_CAP_PRECISE) != 0) && 425 ((pd->pd_caps & PMC_CAP_PRECISE) == 0)) 426 return (EINVAL); 427 428 /* PMC_F_EV_PMU: config comes from pmu-events tables. */ 429 if ((a->pm_flags & PMC_F_EV_PMU) != 0) { 430 config = a->pm_md.pm_amd.pm_amd_config; 431 if ((config & ~amd_config_mask(amd_pmcdesc[ri].pm_subclass, 432 caps)) != 0) 433 return (EXTERROR(EINVAL, 434 "AMD PMU config has unsupported bits %#jx", 435 (uintmax_t)(config & ~amd_config_mask( 436 amd_pmcdesc[ri].pm_subclass, caps)))); 437 pm->pm_md.pm_amd.pm_amd_evsel = config; 438 PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx", 439 ri, (uintmax_t)config); 440 return (0); 441 } 442 443 /* 444 * Everything below this is for supporting older processors. 445 */ 446 pe = a->pm_ev; 447 448 /* map ev to the correct event mask code */ 449 config = allowed_unitmask = 0; 450 for (i = 0; i < amd_event_codes_size; i++) { 451 if (amd_event_codes[i].pe_ev == pe) { 452 config = 453 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 454 allowed_unitmask = 455 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 456 break; 457 } 458 } 459 if (i == amd_event_codes_size) 460 return (EXTERROR(EINVAL, 461 "AMD legacy event %ju is not supported", 462 (uintmax_t)pe)); 463 464 unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 465 if ((unitmask & ~allowed_unitmask) != 0) /* disallow reserved bits */ 466 return (EXTERROR(EINVAL, 467 "AMD unitmask %#jx exceeds allowed mask %#jx", 468 (uintmax_t)unitmask, (uintmax_t)allowed_unitmask)); 469 470 if (unitmask && (caps & PMC_CAP_QUALIFIER) != 0) 471 config |= unitmask; 472 473 if ((caps & PMC_CAP_THRESHOLD) != 0) 474 config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 475 476 /* Set at least one of the 'usr' or 'os' caps. */ 477 if ((caps & PMC_CAP_USER) != 0) 478 config |= AMD_PMC_USR; 479 if ((caps & PMC_CAP_SYSTEM) != 0) 480 config |= AMD_PMC_OS; 481 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 482 config |= (AMD_PMC_USR|AMD_PMC_OS); 483 484 if ((caps & PMC_CAP_EDGE) != 0) 485 config |= AMD_PMC_EDGE; 486 if ((caps & PMC_CAP_INVERT) != 0) 487 config |= AMD_PMC_INVERT; 488 if ((caps & PMC_CAP_INTERRUPT) != 0) 489 config |= AMD_PMC_INT; 490 491 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 492 493 PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%x", ri, config); 494 495 return (0); 496 } 497 498 /* 499 * Release machine dependent state associated with a PMC. This is a 500 * no-op on this architecture. 501 */ 502 static int 503 amd_release_pmc(int cpu, int ri, struct pmc *pmc __unused) 504 { 505 struct pmc_hw *phw __diagused; 506 507 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 508 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 509 KASSERT(ri >= 0 && ri < amd_npmcs, 510 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 511 512 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 513 514 KASSERT(phw->phw_pmc == NULL, 515 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 516 517 return (0); 518 } 519 520 /* 521 * Start a PMC. 522 */ 523 static int 524 amd_start_pmc(int cpu __diagused, int ri, struct pmc *pm) 525 { 526 const struct amd_descr *pd; 527 uint64_t config; 528 529 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 530 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 531 KASSERT(ri >= 0 && ri < amd_npmcs, 532 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 533 534 pd = &amd_pmcdesc[ri]; 535 536 PMCDBG2(MDP, STA, 1, "amd-start cpu=%d ri=%d", cpu, ri); 537 538 /* 539 * Asserts triggered by DF/UMC counters because all DF/UMC MSRs are 540 * shared. While userspace now honors the per-node flags, we should 541 * enforce this in the kernel. 542 */ 543 /* turn on the PMC ENABLE bit */ 544 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC) { 545 KASSERT(AMD_PMC_UMC_IS_STOPPED(pd->pm_evsel), 546 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", 547 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 548 549 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_UMC_ENABLE; 550 } else { 551 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 552 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", 553 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 554 555 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 556 } 557 558 PMCDBG1(MDP, STA, 2, "amd-start config=0x%x", config); 559 560 wrmsr(pd->pm_evsel, config); 561 return (0); 562 } 563 564 /* 565 * Stop a PMC. 566 */ 567 static int 568 amd_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) 569 { 570 const struct amd_descr *pd; 571 uint64_t config; 572 int i; 573 574 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 575 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 576 KASSERT(ri >= 0 && ri < amd_npmcs, 577 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 578 579 pd = &amd_pmcdesc[ri]; 580 581 PMCDBG1(MDP, STO, 1, "amd-stop ri=%d", ri); 582 583 /* turn off the PMC ENABLE bit */ 584 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC) { 585 KASSERT(!AMD_PMC_UMC_IS_STOPPED(pd->pm_evsel), 586 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 587 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 588 589 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE; 590 } else { 591 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 592 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 593 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 594 595 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 596 } 597 wrmsr(pd->pm_evsel, config); 598 599 /* 600 * Due to NMI latency on newer AMD processors 601 * NMI interrupts are ignored, which leads to 602 * panic or messages based on kernel configuration 603 */ 604 605 /* Wait for the count to be reset */ 606 for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { 607 if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1))) 608 break; 609 610 DELAY(1); 611 } 612 613 return (0); 614 } 615 616 /* 617 * Interrupt handler. This function needs to return '1' if the 618 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 619 * to sleep or do anything a 'fast' interrupt handler is not allowed 620 * to do. 621 */ 622 static int 623 amd_intr(struct trapframe *tf) 624 { 625 struct amd_cpu *pac; 626 struct pmc *pm; 627 pmc_value_t v; 628 uint64_t config, evsel, perfctr; 629 uint32_t active = 0, count = 0; 630 int i, error, retval, cpu; 631 632 cpu = curcpu; 633 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 634 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 635 636 PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); 637 638 retval = 0; 639 640 pac = amd_pcpu[cpu]; 641 642 retval = pmc_ibs_intr(tf); 643 if (retval) 644 goto done; 645 646 /* 647 * look for all PMCs that have interrupted: 648 * - look for a running, sampling PMC which has overflowed 649 * and which has a valid 'struct pmc' association 650 * 651 * If found, we call a helper to process the interrupt. 652 * 653 * PMCs interrupting at the same time are collapsed into 654 * a single interrupt. Check all the valid pmcs for 655 * overflow. 656 */ 657 for (i = 0; i < amd_npmcs; i++) { 658 if (amd_pmcdesc[i].pm_subclass != PMC_AMD_SUB_CLASS_CORE) 659 break; 660 661 if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || 662 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 663 continue; 664 } 665 666 /* Consider pmc with valid handle as active */ 667 active++; 668 669 if (!AMD_PMC_HAS_OVERFLOWED(i)) 670 continue; 671 672 retval = 1; /* Found an interrupting PMC. */ 673 674 if (pm->pm_state != PMC_STATE_RUNNING) 675 continue; 676 677 /* Stop the PMC, reload count. */ 678 evsel = amd_pmcdesc[i].pm_evsel; 679 perfctr = amd_pmcdesc[i].pm_perfctr; 680 v = pm->pm_sc.pm_reloadcount; 681 config = rdmsr(evsel); 682 683 684 if (amd_pmcdesc[i].pm_subclass == PMC_AMD_SUB_CLASS_UMC) { 685 KASSERT((config & ~AMD_PMC_UMC_ENABLE) == 686 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE), 687 ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, 688 (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); 689 690 wrmsr(evsel, config & ~AMD_PMC_UMC_ENABLE); 691 } else { 692 KASSERT((config & ~AMD_PMC_ENABLE) == 693 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 694 ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, 695 (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); 696 697 wrmsr(evsel, config & ~AMD_PMC_ENABLE); 698 } 699 wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 700 701 /* Restart the counter if logging succeeded. */ 702 error = pmc_process_interrupt(PMC_HR, pm, tf); 703 if (error == 0) 704 wrmsr(evsel, config); 705 } 706 707 /* 708 * Due to NMI latency, there can be a scenario in which 709 * multiple pmcs gets serviced in an earlier NMI and we 710 * do not find an overflow in the subsequent NMI. 711 * 712 * For such cases we keep a per-cpu count of active NMIs 713 * and compare it with min(active pmcs, 2) to determine 714 * if this NMI was for a pmc overflow which was serviced 715 * in an earlier request or should be ignored. 716 */ 717 if (retval) { 718 DPCPU_SET(nmi_counter, min(2, active)); 719 } else { 720 if ((count = DPCPU_GET(nmi_counter))) { 721 retval = 1; 722 DPCPU_SET(nmi_counter, --count); 723 } 724 } 725 726 done: 727 if (retval) 728 counter_u64_add(pmc_stats.pm_intr_processed, 1); 729 else 730 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 731 732 PMCDBG1(MDP, INT, 2, "retval=%d", retval); 733 return (retval); 734 } 735 736 /* 737 * Describe a PMC. 738 */ 739 static int 740 amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 741 { 742 const struct amd_descr *pd; 743 struct pmc_hw *phw; 744 745 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 746 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 747 KASSERT(ri >= 0 && ri < amd_npmcs, 748 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 749 750 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 751 pd = &amd_pmcdesc[ri]; 752 753 strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); 754 pi->pm_class = pd->pm_descr.pd_class; 755 756 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) != 0) { 757 pi->pm_enabled = true; 758 *ppmc = phw->phw_pmc; 759 } else { 760 pi->pm_enabled = false; 761 *ppmc = NULL; 762 } 763 764 return (0); 765 } 766 767 /* 768 * Return the MSR address of the given PMC. 769 */ 770 static int 771 amd_get_msr(int ri, uint32_t *msr) 772 { 773 int df_idx; 774 775 KASSERT(ri >= 0 && ri < amd_npmcs, 776 ("[amd,%d] ri %d out of range", __LINE__, ri)); 777 778 /* 779 * Map counter row index to RDPMC ECX value. 780 * 781 * AMD BKDG 24594 rev 3.37, page 440, 782 * "RDPMC Read Performance-Monitoring Counter": 783 * ECX 0-5: Core counters 0-5 784 * ECX 6-9: DF/Northbridge counters 0-3 785 * ECX 10-15: L3 Cache counters 0-5 786 * ECX 16-27: DF/Northbridge counters 4-15 787 * 788 * AMD PPR 57930-A0 section 2.1.9, 789 * "Register Sharing" for DF counter details. 790 */ 791 if (ri < amd_core_npmcs) { 792 /* ECX 0-5: Core counters */ 793 *msr = ri; 794 } else if (ri < amd_core_npmcs + amd_l3_npmcs) { 795 /* ECX 10-15: L3 Cache counters */ 796 *msr = 10 + (ri - amd_core_npmcs); 797 } else { 798 /* ECX 6-9: DF counters 0-3 799 * ECX 16-27: DF counters 4-15 */ 800 df_idx = ri - amd_core_npmcs - amd_l3_npmcs; 801 if (df_idx < 4) 802 *msr = 6 + df_idx; 803 else if (df_idx < 16) 804 *msr = 16 + (df_idx - 4); 805 else 806 return (EINVAL); 807 } 808 return (0); 809 } 810 811 /* 812 * Return the capabilities of the given PMC. 813 */ 814 static int 815 amd_get_caps(int ri, uint32_t *caps) 816 { 817 KASSERT(ri >= 0 && ri < amd_npmcs, 818 ("[amd,%d] ri %d out of range", __LINE__, ri)); 819 820 *caps = amd_pmcdesc[ri].pm_descr.pd_caps; 821 822 return (0); 823 } 824 825 /* 826 * Processor-dependent initialization. 827 */ 828 static int 829 amd_pcpu_init(struct pmc_mdep *md, int cpu) 830 { 831 struct amd_cpu *pac; 832 struct pmc_cpu *pc; 833 struct pmc_hw *phw; 834 int first_ri, n; 835 836 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 837 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 838 839 PMCDBG1(MDP, INI, 1, "amd-init cpu=%d", cpu); 840 841 amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, 842 M_WAITOK | M_ZERO); 843 844 /* 845 * Set the content of the hardware descriptors to a known 846 * state and initialize pointers in the MI per-cpu descriptor. 847 */ 848 pc = pmc_pcpu[cpu]; 849 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8].pcd_ri; 850 851 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); 852 853 for (n = 0, phw = pac->pc_amdpmcs; n < amd_npmcs; n++, phw++) { 854 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 855 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 856 phw->phw_pmc = NULL; 857 pc->pc_hwpmcs[n + first_ri] = phw; 858 } 859 860 return (0); 861 } 862 863 /* 864 * Processor-dependent cleanup prior to the KLD being unloaded. 865 */ 866 static int 867 amd_pcpu_fini(struct pmc_mdep *md, int cpu) 868 { 869 struct amd_cpu *pac; 870 struct pmc_cpu *pc; 871 int first_ri, i; 872 873 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 874 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 875 876 PMCDBG1(MDP, INI, 1, "amd-cleanup cpu=%d", cpu); 877 878 /* 879 * Next, free up allocated space. 880 */ 881 if ((pac = amd_pcpu[cpu]) == NULL) 882 return (0); 883 884 amd_pcpu[cpu] = NULL; 885 886 #ifdef HWPMC_DEBUG 887 for (i = 0; i < AMD_NPMCS_K8; i++) { 888 KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, 889 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 890 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i), 891 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 892 } 893 #endif 894 895 pc = pmc_pcpu[cpu]; 896 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__)); 897 898 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8].pcd_ri; 899 900 /* 901 * Reset pointers in the MI 'per-cpu' state. 902 */ 903 for (i = 0; i < amd_npmcs; i++) 904 pc->pc_hwpmcs[i + first_ri] = NULL; 905 906 free(pac, M_PMC); 907 return (0); 908 } 909 910 /* 911 * Check that the PMC hardware is safe to use. First, we check that the PMCs 912 * are not in use by firmware or another module. Second, if none of the PMC 913 * feature flags are set, we check that the event selector is working, because 914 * virtual machines have no way to communicate the absence of PMCs. 915 */ 916 static int 917 amd_hwcheck(void) 918 { 919 uint64_t reg; 920 int error, i; 921 922 /* 923 * Some PC vendors enable the core counters in firmware to track 924 * performance. The best guess is that this is being used to control 925 * power management from within the SMM mode. We shouldn't just take 926 * over the PMCs in this case. The user should try disabling any 927 * performance monitoring or power management functions in the BIOS to 928 * safely make use of the counters. 929 */ 930 for (i = 0; i < amd_core_npmcs; i++) { 931 error = rdmsr_safe(amd_pmcdesc[i].pm_evsel, ®); 932 if (error != 0) { 933 printf("hwpmc: AMD evsel %d rdmsr failed!\n", i); 934 return (-1); 935 } 936 937 if ((reg & AMD_PMC_ENABLE) != 0) { 938 printf("hwpmc: PMCs maybe in use by firmware!\n"); 939 printf("hwpmc: Disable the PMC use in the BIOS before loading\n"); 940 return (-1); 941 } 942 } 943 944 /* 945 * Unfortunately, there is no way to communicate that the original four 946 * core counters are disabled through CPUIDs alone. We attempt to 947 * write and read back the MSR to validate that it is working. 948 * 949 * Referenced the BIOS and Kernel Developer Guide for AMD Athlon 64 and 950 * AMD Opteron Processors 26094 Rev. 3.24 January, 2005 to ensure these 951 * fields are valid. 952 */ 953 if ((amd_feature2 & AMDID2_PCXC) == 0) { 954 error = wrmsr_safe(AMD_PMC_EVSEL_0, AMD_PMC_OS | AMD_PMC_USR); 955 if (error != 0) { 956 printf("hwpmc: AMD evsel 0 wrmsr failed!\n"); 957 return (-1); 958 } 959 960 error = rdmsr_safe(AMD_PMC_EVSEL_0, ®); 961 if (error != 0) { 962 printf("hwpmc: AMD evsel 0 rdmsr failed!\n"); 963 return (-1); 964 } 965 966 if (reg == 0) { 967 printf("hwpmc: AMD evsel returned invalid value! " 968 "You may be in a VM without PMC support.\n"); 969 return (-1); 970 } 971 972 wrmsr(AMD_PMC_EVSEL_0, 0); 973 } 974 975 return (0); 976 } 977 978 /* 979 * Initialize ourselves. 980 */ 981 struct pmc_mdep * 982 pmc_amd_initialize(void) 983 { 984 u_int regs[4]; 985 struct amd_descr *d; 986 struct pmc_classdep *pcd; 987 struct pmc_mdep *pmc_mdep; 988 enum pmc_cputype cputype; 989 int ncpus, nclasses, i; 990 int family, model, stepping; 991 int error; 992 int pmcs_per_umc; 993 994 /* 995 * The presence of hardware performance counters on the AMD 996 * Athlon, Duron or later processors, is _not_ indicated by 997 * any of the processor feature flags set by the 'CPUID' 998 * instruction, so we only check the 'instruction family' 999 * field returned by CPUID for instruction family >= 6. 1000 */ 1001 1002 family = CPUID_TO_FAMILY(cpu_id); 1003 model = CPUID_TO_MODEL(cpu_id); 1004 stepping = CPUID_TO_STEPPING(cpu_id); 1005 1006 if (family == 0x18) 1007 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X", 1008 family, model, stepping); 1009 else 1010 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X", 1011 family, model, stepping); 1012 1013 switch (cpu_id & 0xF00) { 1014 case 0xF00: /* Athlon64/Opteron processor */ 1015 cputype = PMC_CPU_AMD_K8; 1016 break; 1017 default: 1018 printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, family, 1019 model); 1020 return (NULL); 1021 } 1022 1023 /* 1024 * From PPR for AMD Family 1Ah, a new cpuid leaf specifies the maximum 1025 * number of PMCs of each type. If we do not have that leaf, we use 1026 * the prior default values that are only valid if we have the feature 1027 * bit enabled in CPU. 1028 */ 1029 if ((amd_feature2 & AMDID2_PCXC) != 0) { 1030 amd_core_npmcs = AMD_PMC_CORE_DEFAULT; 1031 } else { 1032 amd_core_npmcs = AMD_NPMCS_K8; 1033 } 1034 amd_l3_npmcs = AMD_PMC_L3_DEFAULT; 1035 amd_df_npmcs = AMD_PMC_DF_DEFAULT; 1036 amd_umc_npmcs = 0; 1037 pmcs_per_umc = 0; 1038 1039 if (cpu_exthigh >= CPUID_EXTPERFMON) { 1040 do_cpuid(CPUID_EXTPERFMON, regs); 1041 if (regs[1] != 0) { 1042 amd_core_npmcs = EXTPERFMON_CORE_PMCS(regs[1]); 1043 amd_df_npmcs = EXTPERFMON_DF_PMCS(regs[1]); 1044 amd_umc_npmcs = EXTPERFMON_UMC_PMCS(regs[1]); 1045 pmcs_per_umc = amd_umc_npmcs / popcntq(regs[2]); 1046 } 1047 } 1048 1049 /* Enable the newer core counters */ 1050 for (i = 0; i < amd_core_npmcs; i++) { 1051 d = &amd_pmcdesc[i]; 1052 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1053 "K8-%d", i); 1054 d->pm_descr.pd_class = PMC_CLASS_K8; 1055 d->pm_descr.pd_caps = AMD_PMC_CAPS; 1056 /* 1057 * Zen 5 can precisely count retire events. 1058 * 1059 * Refer to PPR Vol 1 for AMD Family 1Ah Model 02h C1 57238 1060 * Rev. 0.24 September 29, 2024. 1061 */ 1062 if ((family >= 0x1a) && (i == 2)) 1063 d->pm_descr.pd_caps |= PMC_CAP_PRECISE; 1064 d->pm_descr.pd_width = 48; 1065 if ((amd_feature2 & AMDID2_PCXC) != 0) { 1066 d->pm_evsel = AMD_PMC_CORE_BASE + 2 * i; 1067 d->pm_perfctr = AMD_PMC_CORE_BASE + 2 * i + 1; 1068 } else { 1069 d->pm_evsel = AMD_PMC_EVSEL_0 + i; 1070 d->pm_perfctr = AMD_PMC_PERFCTR_0 + i; 1071 } 1072 d->pm_subclass = PMC_AMD_SUB_CLASS_CORE; 1073 } 1074 amd_npmcs = amd_core_npmcs; 1075 1076 if ((amd_feature2 & AMDID2_PTSCEL2I) != 0) { 1077 /* Enable the LLC/L3 counters */ 1078 for (i = 0; i < amd_l3_npmcs; i++) { 1079 d = &amd_pmcdesc[amd_npmcs + i]; 1080 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1081 "K8-L3-%d", i); 1082 d->pm_descr.pd_class = PMC_CLASS_K8; 1083 d->pm_descr.pd_caps = AMD_PMC_L3_CAPS; 1084 d->pm_descr.pd_width = 48; 1085 d->pm_evsel = AMD_PMC_L3_BASE + 2 * i; 1086 d->pm_perfctr = AMD_PMC_L3_BASE + 2 * i + 1; 1087 d->pm_subclass = PMC_AMD_SUB_CLASS_L3_CACHE; 1088 } 1089 amd_npmcs += amd_l3_npmcs; 1090 } 1091 1092 if ((amd_feature2 & AMDID2_PNXC) != 0) { 1093 /* Enable the data fabric counters */ 1094 for (i = 0; i < amd_df_npmcs; i++) { 1095 d = &amd_pmcdesc[amd_npmcs + i]; 1096 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1097 "K8-DF-%d", i); 1098 d->pm_descr.pd_class = PMC_CLASS_K8; 1099 d->pm_descr.pd_caps = AMD_PMC_DF_CAPS; 1100 d->pm_descr.pd_width = 48; 1101 d->pm_evsel = AMD_PMC_DF_BASE + 2 * i; 1102 d->pm_perfctr = AMD_PMC_DF_BASE + 2 * i + 1; 1103 d->pm_subclass = PMC_AMD_SUB_CLASS_DATA_FABRIC; 1104 } 1105 amd_npmcs += amd_df_npmcs; 1106 } 1107 1108 for (i = 0; i < amd_umc_npmcs; i++) { 1109 d = &amd_pmcdesc[amd_npmcs + i]; 1110 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1111 "K8-UMC%d-%d", i / pmcs_per_umc, i); 1112 d->pm_descr.pd_class = PMC_CLASS_K8; 1113 d->pm_descr.pd_caps = AMD_PMC_UMC_CAPS; 1114 d->pm_descr.pd_width = 48; 1115 d->pm_evsel = AMD_PMC_UMC_BASE + 2 * i; 1116 d->pm_perfctr = AMD_PMC_UMC_BASE + 2 * i + 1; 1117 d->pm_subclass = PMC_AMD_SUB_CLASS_UMC; 1118 } 1119 amd_npmcs += amd_umc_npmcs; 1120 1121 /* 1122 * Sanity check that the hardware is safe to use. Do not read or write 1123 * any of the PMC MSRs until after this check passes. 1124 */ 1125 if (amd_hwcheck() < 0) { 1126 return (NULL); 1127 } 1128 1129 /* 1130 * Allocate space for pointers to PMC HW descriptors and for 1131 * the MDEP structure used by MI code. 1132 */ 1133 amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC, 1134 M_WAITOK | M_ZERO); 1135 1136 /* 1137 * These processors have two or three classes of PMCs: the TSC, 1138 * programmable PMCs, and AMD IBS. One extra class slot is reserved 1139 * for the optional RAPL energy counters. 1140 */ 1141 if ((amd_feature2 & AMDID2_IBS) != 0) { 1142 nclasses = 4; 1143 } else { 1144 nclasses = 3; 1145 } 1146 1147 /* 1148 * Detect support for MPERF and APERF MSRs. tsc_perf_stat is set by the 1149 * kernel's generic TSC initialization (start_TSC(), called at boot via 1150 * cpu_startup() -> startrtclock()), not by hwpmc's TSC PMC class. It is 1151 * set only after confirming both MSRs actually increment (some emulators 1152 * expose the CPUID bit without real MSR support). 1153 */ 1154 if ((cpu_power_ecx & CPUID_PERF_STAT) && (tsc_perf_stat == 1)) { 1155 nclasses++; 1156 } 1157 1158 pmc_mdep = pmc_mdep_alloc(nclasses); 1159 1160 ncpus = pmc_cpu_max(); 1161 1162 /* Initialize the TSC. */ 1163 error = pmc_tsc_initialize(pmc_mdep, ncpus); 1164 if (error != 0) 1165 goto error; 1166 1167 /* Initialize AMD K8 PMC handling. */ 1168 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8]; 1169 1170 pcd->pcd_caps = AMD_PMC_CAPS; 1171 pcd->pcd_class = PMC_CLASS_K8; 1172 pcd->pcd_num = amd_npmcs; 1173 pcd->pcd_ri = pmc_mdep->pmd_npmc; 1174 pcd->pcd_width = 48; 1175 1176 pcd->pcd_allocate_pmc = amd_allocate_pmc; 1177 pcd->pcd_config_pmc = amd_config_pmc; 1178 pcd->pcd_describe = amd_describe; 1179 pcd->pcd_get_config = amd_get_config; 1180 pcd->pcd_get_msr = amd_get_msr; 1181 pcd->pcd_pcpu_fini = amd_pcpu_fini; 1182 pcd->pcd_pcpu_init = amd_pcpu_init; 1183 pcd->pcd_read_pmc = amd_read_pmc; 1184 pcd->pcd_release_pmc = amd_release_pmc; 1185 pcd->pcd_start_pmc = amd_start_pmc; 1186 pcd->pcd_stop_pmc = amd_stop_pmc; 1187 pcd->pcd_write_pmc = amd_write_pmc; 1188 pcd->pcd_get_caps = amd_get_caps; 1189 1190 pmc_mdep->pmd_cputype = cputype; 1191 pmc_mdep->pmd_intr = amd_intr; 1192 pmc_mdep->pmd_switch_in = amd_switch_in; 1193 pmc_mdep->pmd_switch_out = amd_switch_out; 1194 1195 pmc_mdep->pmd_npmc += amd_npmcs; 1196 1197 amd_init_policy(); 1198 1199 PMCDBG0(MDP, INI, 0, "amd-initialize"); 1200 1201 if ((amd_feature2 & AMDID2_IBS) != 0) { 1202 error = pmc_ibs_initialize(pmc_mdep, ncpus); 1203 if (error != 0) 1204 goto error; 1205 } 1206 1207 /* Initialize PERF class. */ 1208 pmc_perf_initialize(pmc_mdep, ncpus, nclasses - 1); 1209 1210 /* RAPL takes the reserved last slot; drop it if the probe fails. */ 1211 error = pmc_rapl_initialize(pmc_mdep, ncpus, pmc_mdep->pmd_nclass - 1); 1212 if (error != 0) 1213 pmc_mdep->pmd_nclass--; 1214 1215 return (pmc_mdep); 1216 1217 error: 1218 free(pmc_mdep, M_PMC); 1219 return (NULL); 1220 } 1221 1222 /* 1223 * Finalization code for AMD CPUs. 1224 */ 1225 void 1226 pmc_amd_finalize(struct pmc_mdep *md) 1227 { 1228 PMCDBG0(MDP, INI, 1, "amd-finalize"); 1229 1230 /* Safe even if the RAPL class was skipped at initialize time. */ 1231 pmc_rapl_finalize(md); 1232 1233 pmc_tsc_finalize(md); 1234 1235 pmc_perf_finalize(md); 1236 1237 for (int i = 0; i < pmc_cpu_max(); i++) 1238 KASSERT(amd_pcpu[i] == NULL, 1239 ("[amd,%d] non-null pcpu cpu %d", __LINE__, i)); 1240 1241 free(amd_pcpu, M_PMC); 1242 amd_pcpu = NULL; 1243 } 1244