1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* Support for the AMD K8 and later processors */ 34 35 #include <sys/param.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/mutex.h> 39 #include <sys/pcpu.h> 40 #include <sys/pmc.h> 41 #include <sys/pmckern.h> 42 #include <sys/smp.h> 43 #include <sys/sysctl.h> 44 #include <sys/systm.h> 45 46 #include <machine/cpu.h> 47 #include <machine/cpufunc.h> 48 #include <machine/md_var.h> 49 #include <machine/specialreg.h> 50 51 #define OVERFLOW_WAIT_COUNT 50 52 53 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter); 54 55 /* AMD K8 PMCs */ 56 struct amd_descr { 57 struct pmc_descr pm_descr; /* "base class" */ 58 uint32_t pm_evsel; /* address of EVSEL register */ 59 uint32_t pm_perfctr; /* address of PERFCTR register */ 60 enum sub_class pm_subclass; /* register subclass */ 61 }; 62 63 static int amd_npmcs; 64 static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs; 65 static struct amd_descr amd_pmcdesc[AMD_NPMCS_MAX]; 66 struct amd_event_code_map { 67 enum pmc_event pe_ev; /* enum value */ 68 uint16_t pe_code; /* encoded event mask */ 69 uint8_t pe_mask; /* bits allowed in unit mask */ 70 }; 71 72 const struct amd_event_code_map amd_event_codes[] = { 73 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 74 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 75 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 76 77 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 78 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 79 0x21, 0x00 }, 80 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 81 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 82 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 83 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 84 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 85 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 86 87 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 88 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 89 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 90 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 91 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 92 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 93 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 94 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 95 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 96 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 97 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 98 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 99 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 100 101 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 102 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 103 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 104 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 105 106 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 107 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 108 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 109 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 110 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 111 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 112 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 113 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 114 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 115 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 116 117 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 118 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 119 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 120 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 121 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 122 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 123 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 124 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 125 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 126 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 127 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 128 0xCA, 0x00 }, 129 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 130 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 131 0xCC, 0x07 }, 132 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 133 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 134 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 135 136 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 137 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 138 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 139 0xD2, 0x00 }, 140 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 141 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 142 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 143 0xD5, 0x00 }, 144 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 145 0xD6, 0x00 }, 146 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 147 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 148 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 149 0xD9, 0x00 }, 150 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 151 0xDA, 0x00 }, 152 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 153 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 154 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 155 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 156 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 157 158 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 159 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 160 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 161 0xE2, 0x00 }, 162 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 163 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 164 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 165 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 166 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 167 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 168 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 169 170 }; 171 172 const int amd_event_codes_size = nitems(amd_event_codes); 173 174 /* 175 * Per-processor information 176 */ 177 struct amd_cpu { 178 struct pmc_hw pc_amdpmcs[AMD_NPMCS_MAX]; 179 }; 180 static struct amd_cpu **amd_pcpu; 181 182 /* Populated by amd_init_policy(); PRECISERETIRE is OR-ed in per-allocation. */ 183 static uint64_t amd_core_allowed_mask; 184 static uint64_t amd_l3_allowed_mask; 185 static uint64_t amd_df_allowed_mask; 186 187 static uint64_t amd_core_extra_mask; 188 static uint64_t amd_l3_extra_mask; 189 static uint64_t amd_df_extra_mask; 190 191 SYSCTL_DECL(_kern_hwpmc); 192 193 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_core_extra_mask, CTLFLAG_RDTUN, 194 &amd_core_extra_mask, 0, 195 "Extra allowed bits in AMD core PMU PERFEVTSEL (override; default 0)"); 196 197 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_l3_extra_mask, CTLFLAG_RDTUN, 198 &amd_l3_extra_mask, 0, 199 "Extra allowed bits in AMD L3 PMU control (override; default 0)"); 200 201 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_df_extra_mask, CTLFLAG_RDTUN, 202 &amd_df_extra_mask, 0, 203 "Extra allowed bits in AMD DF PMU control (override; default 0)"); 204 205 static void 206 amd_init_policy(void) 207 { 208 int family; 209 210 family = CPUID_TO_FAMILY(cpu_id); 211 212 amd_core_allowed_mask = AMD_VALID_BITS; 213 214 amd_l3_allowed_mask = (family <= 0x17) ? 215 AMD_PMC_L3_FAMILY17_MASK : AMD_PMC_L3_FAMILY19_MASK; 216 217 amd_df_allowed_mask = (family <= 0x19) ? 218 AMD_PMC_DF_FAMILY17_MASK : AMD_PMC_DF_FAMILY1A_MASK; 219 } 220 221 static uint64_t 222 amd_config_mask(enum sub_class subclass, uint64_t caps) 223 { 224 225 switch (subclass) { 226 case PMC_AMD_SUB_CLASS_CORE: 227 return (amd_core_allowed_mask | amd_core_extra_mask | 228 (((caps & PMC_CAP_PRECISE) != 0) ? 229 AMD_PMC_PRECISERETIRE : 0)); 230 case PMC_AMD_SUB_CLASS_L3_CACHE: 231 return (amd_l3_allowed_mask | amd_l3_extra_mask); 232 case PMC_AMD_SUB_CLASS_DATA_FABRIC: 233 return (amd_df_allowed_mask | amd_df_extra_mask); 234 default: 235 return (0); 236 } 237 } 238 239 /* 240 * Read a PMC value from the MSR. 241 */ 242 static int 243 amd_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 244 { 245 const struct amd_descr *pd; 246 pmc_value_t tmp; 247 enum pmc_mode mode; 248 249 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 250 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 251 KASSERT(ri >= 0 && ri < amd_npmcs, 252 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 253 KASSERT(amd_pcpu[cpu], 254 ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); 255 256 pd = &amd_pmcdesc[ri]; 257 mode = PMC_TO_MODE(pm); 258 259 PMCDBG2(MDP, REA, 1, "amd-read id=%d class=%d", ri, 260 pd->pm_descr.pd_class); 261 262 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 263 PMCDBG2(MDP, REA, 2, "amd-read (pre-munge) id=%d -> %jd", ri, tmp); 264 if (PMC_IS_SAMPLING_MODE(mode)) { 265 /* 266 * Clamp value to 0 if the counter just overflowed, 267 * otherwise the returned reload count would wrap to a 268 * huge value. 269 */ 270 if ((tmp & (1ULL << 47)) == 0) 271 tmp = 0; 272 else { 273 /* Sign extend 48 bit value to 64 bits. */ 274 tmp = (pmc_value_t) ((int64_t)(tmp << 16) >> 16); 275 tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 276 } 277 } 278 *v = tmp; 279 280 PMCDBG2(MDP, REA, 2, "amd-read (post-munge) id=%d -> %jd", ri, *v); 281 282 return (0); 283 } 284 285 /* 286 * Write a PMC MSR. 287 */ 288 static int 289 amd_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 290 { 291 const struct amd_descr *pd; 292 enum pmc_mode mode; 293 294 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 295 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 296 KASSERT(ri >= 0 && ri < amd_npmcs, 297 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 298 299 pd = &amd_pmcdesc[ri]; 300 mode = PMC_TO_MODE(pm); 301 302 /* use 2's complement of the count for sampling mode PMCs */ 303 if (PMC_IS_SAMPLING_MODE(mode)) 304 v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 305 306 PMCDBG3(MDP, WRI, 1, "amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 307 308 /* write the PMC value */ 309 wrmsr(pd->pm_perfctr, v); 310 return (0); 311 } 312 313 /* 314 * Configure hardware PMC according to the configuration recorded in 'pm'. 315 */ 316 static int 317 amd_config_pmc(int cpu, int ri, struct pmc *pm) 318 { 319 struct pmc_hw *phw; 320 321 PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 322 323 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 324 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 325 KASSERT(ri >= 0 && ri < amd_npmcs, 326 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 327 328 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 329 330 KASSERT(pm == NULL || phw->phw_pmc == NULL, 331 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 332 __LINE__, pm, phw->phw_pmc)); 333 334 phw->phw_pmc = pm; 335 return (0); 336 } 337 338 /* 339 * Retrieve a configured PMC pointer from hardware state. 340 */ 341 static int 342 amd_get_config(int cpu, int ri, struct pmc **ppm) 343 { 344 *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; 345 return (0); 346 } 347 348 /* 349 * Machine-dependent actions taken during the context switch in of a 350 * thread. 351 */ 352 static int 353 amd_switch_in(struct pmc_cpu *pc __pmcdbg_used, struct pmc_process *pp) 354 { 355 PMCDBG3(MDP, SWI, 1, "pc=%p pp=%p enable-msr=%d", pc, pp, 356 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 357 358 /* enable the RDPMC instruction if needed */ 359 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 360 load_cr4(rcr4() | CR4_PCE); 361 362 return (0); 363 } 364 365 /* 366 * Machine-dependent actions taken during the context switch out of a 367 * thread. 368 */ 369 static int 370 amd_switch_out(struct pmc_cpu *pc __pmcdbg_used, 371 struct pmc_process *pp __pmcdbg_used) 372 { 373 PMCDBG3(MDP, SWO, 1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 374 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 375 376 /* always turn off the RDPMC instruction */ 377 load_cr4(rcr4() & ~CR4_PCE); 378 379 return (0); 380 } 381 382 /* 383 * Check if a given PMC allocation is feasible. 384 */ 385 static int 386 amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, 387 const struct pmc_op_pmcallocate *a) 388 { 389 const struct pmc_descr *pd; 390 uint64_t allowed_unitmask, caps, config, unitmask; 391 enum pmc_event pe; 392 int i; 393 394 KASSERT(ri >= 0 && ri < amd_npmcs, 395 ("[amd,%d] illegal row index %d", __LINE__, ri)); 396 397 pd = &amd_pmcdesc[ri].pm_descr; 398 399 /* check class match */ 400 if (pd->pd_class != a->pm_class) 401 return (EINVAL); 402 403 if ((a->pm_flags & PMC_F_EV_PMU) == 0) 404 return (EINVAL); 405 406 caps = pm->pm_caps; 407 408 if (((caps & PMC_CAP_PRECISE) != 0) && 409 ((pd->pd_caps & PMC_CAP_PRECISE) == 0)) 410 return (EINVAL); 411 412 PMCDBG2(MDP, ALL, 1,"amd-allocate ri=%d caps=0x%x", ri, caps); 413 414 /* Validate sub-class. */ 415 if (amd_pmcdesc[ri].pm_subclass != a->pm_md.pm_amd.pm_amd_sub_class) 416 return (EINVAL); 417 418 if (strlen(pmc_cpuid) != 0) { 419 config = a->pm_md.pm_amd.pm_amd_config; 420 if ((config & ~amd_config_mask(amd_pmcdesc[ri].pm_subclass, 421 caps)) != 0) 422 return (EINVAL); 423 pm->pm_md.pm_amd.pm_amd_evsel = config; 424 PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx", 425 ri, (uintmax_t)config); 426 return (0); 427 } 428 429 /* 430 * Everything below this is for supporting older processors. 431 */ 432 pe = a->pm_ev; 433 434 /* map ev to the correct event mask code */ 435 config = allowed_unitmask = 0; 436 for (i = 0; i < amd_event_codes_size; i++) { 437 if (amd_event_codes[i].pe_ev == pe) { 438 config = 439 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 440 allowed_unitmask = 441 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 442 break; 443 } 444 } 445 if (i == amd_event_codes_size) 446 return (EINVAL); 447 448 unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 449 if ((unitmask & ~allowed_unitmask) != 0) /* disallow reserved bits */ 450 return (EINVAL); 451 452 if (unitmask && (caps & PMC_CAP_QUALIFIER) != 0) 453 config |= unitmask; 454 455 if ((caps & PMC_CAP_THRESHOLD) != 0) 456 config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 457 458 /* Set at least one of the 'usr' or 'os' caps. */ 459 if ((caps & PMC_CAP_USER) != 0) 460 config |= AMD_PMC_USR; 461 if ((caps & PMC_CAP_SYSTEM) != 0) 462 config |= AMD_PMC_OS; 463 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 464 config |= (AMD_PMC_USR|AMD_PMC_OS); 465 466 if ((caps & PMC_CAP_EDGE) != 0) 467 config |= AMD_PMC_EDGE; 468 if ((caps & PMC_CAP_INVERT) != 0) 469 config |= AMD_PMC_INVERT; 470 if ((caps & PMC_CAP_INTERRUPT) != 0) 471 config |= AMD_PMC_INT; 472 473 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 474 475 PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%x", ri, config); 476 477 return (0); 478 } 479 480 /* 481 * Release machine dependent state associated with a PMC. This is a 482 * no-op on this architecture. 483 */ 484 static int 485 amd_release_pmc(int cpu, int ri, struct pmc *pmc __unused) 486 { 487 struct pmc_hw *phw __diagused; 488 489 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 490 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 491 KASSERT(ri >= 0 && ri < amd_npmcs, 492 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 493 494 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 495 496 KASSERT(phw->phw_pmc == NULL, 497 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 498 499 return (0); 500 } 501 502 /* 503 * Start a PMC. 504 */ 505 static int 506 amd_start_pmc(int cpu __diagused, int ri, struct pmc *pm) 507 { 508 const struct amd_descr *pd; 509 uint64_t config; 510 511 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 512 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 513 KASSERT(ri >= 0 && ri < amd_npmcs, 514 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 515 516 pd = &amd_pmcdesc[ri]; 517 518 PMCDBG2(MDP, STA, 1, "amd-start cpu=%d ri=%d", cpu, ri); 519 520 /* 521 * Triggered by DF counters because all DF MSRs are shared. We need to 522 * change the code to honor the per-package flag in the JSON event 523 * definitions. 524 */ 525 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 526 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 527 ri, cpu, pd->pm_descr.pd_name)); 528 529 /* turn on the PMC ENABLE bit */ 530 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 531 532 PMCDBG1(MDP, STA, 2, "amd-start config=0x%x", config); 533 534 wrmsr(pd->pm_evsel, config); 535 return (0); 536 } 537 538 /* 539 * Stop a PMC. 540 */ 541 static int 542 amd_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) 543 { 544 const struct amd_descr *pd; 545 uint64_t config; 546 int i; 547 548 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 549 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 550 KASSERT(ri >= 0 && ri < amd_npmcs, 551 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 552 553 pd = &amd_pmcdesc[ri]; 554 555 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 556 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 557 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 558 559 PMCDBG1(MDP, STO, 1, "amd-stop ri=%d", ri); 560 561 /* turn off the PMC ENABLE bit */ 562 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 563 wrmsr(pd->pm_evsel, config); 564 565 /* 566 * Due to NMI latency on newer AMD processors 567 * NMI interrupts are ignored, which leads to 568 * panic or messages based on kernel configuration 569 */ 570 571 /* Wait for the count to be reset */ 572 for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { 573 if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1))) 574 break; 575 576 DELAY(1); 577 } 578 579 return (0); 580 } 581 582 /* 583 * Interrupt handler. This function needs to return '1' if the 584 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 585 * to sleep or do anything a 'fast' interrupt handler is not allowed 586 * to do. 587 */ 588 static int 589 amd_intr(struct trapframe *tf) 590 { 591 struct amd_cpu *pac; 592 struct pmc *pm; 593 pmc_value_t v; 594 uint64_t config, evsel, perfctr; 595 uint32_t active = 0, count = 0; 596 int i, error, retval, cpu; 597 598 cpu = curcpu; 599 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 600 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 601 602 PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); 603 604 retval = 0; 605 606 pac = amd_pcpu[cpu]; 607 608 retval = pmc_ibs_intr(tf); 609 if (retval) 610 goto done; 611 612 /* 613 * look for all PMCs that have interrupted: 614 * - look for a running, sampling PMC which has overflowed 615 * and which has a valid 'struct pmc' association 616 * 617 * If found, we call a helper to process the interrupt. 618 * 619 * PMCs interrupting at the same time are collapsed into 620 * a single interrupt. Check all the valid pmcs for 621 * overflow. 622 */ 623 for (i = 0; i < amd_npmcs; i++) { 624 if (amd_pmcdesc[i].pm_subclass != PMC_AMD_SUB_CLASS_CORE) 625 break; 626 627 if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || 628 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 629 continue; 630 } 631 632 /* Consider pmc with valid handle as active */ 633 active++; 634 635 if (!AMD_PMC_HAS_OVERFLOWED(i)) 636 continue; 637 638 retval = 1; /* Found an interrupting PMC. */ 639 640 if (pm->pm_state != PMC_STATE_RUNNING) 641 continue; 642 643 /* Stop the PMC, reload count. */ 644 evsel = amd_pmcdesc[i].pm_evsel; 645 perfctr = amd_pmcdesc[i].pm_perfctr; 646 v = pm->pm_sc.pm_reloadcount; 647 config = rdmsr(evsel); 648 649 KASSERT((config & ~AMD_PMC_ENABLE) == 650 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 651 ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, 652 (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); 653 654 wrmsr(evsel, config & ~AMD_PMC_ENABLE); 655 wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 656 657 /* Restart the counter if logging succeeded. */ 658 error = pmc_process_interrupt(PMC_HR, pm, tf); 659 if (error == 0) 660 wrmsr(evsel, config); 661 } 662 663 /* 664 * Due to NMI latency, there can be a scenario in which 665 * multiple pmcs gets serviced in an earlier NMI and we 666 * do not find an overflow in the subsequent NMI. 667 * 668 * For such cases we keep a per-cpu count of active NMIs 669 * and compare it with min(active pmcs, 2) to determine 670 * if this NMI was for a pmc overflow which was serviced 671 * in an earlier request or should be ignored. 672 */ 673 if (retval) { 674 DPCPU_SET(nmi_counter, min(2, active)); 675 } else { 676 if ((count = DPCPU_GET(nmi_counter))) { 677 retval = 1; 678 DPCPU_SET(nmi_counter, --count); 679 } 680 } 681 682 done: 683 if (retval) 684 counter_u64_add(pmc_stats.pm_intr_processed, 1); 685 else 686 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 687 688 PMCDBG1(MDP, INT, 2, "retval=%d", retval); 689 return (retval); 690 } 691 692 /* 693 * Describe a PMC. 694 */ 695 static int 696 amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 697 { 698 const struct amd_descr *pd; 699 struct pmc_hw *phw; 700 701 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 702 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 703 KASSERT(ri >= 0 && ri < amd_npmcs, 704 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 705 706 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 707 pd = &amd_pmcdesc[ri]; 708 709 strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); 710 pi->pm_class = pd->pm_descr.pd_class; 711 712 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) != 0) { 713 pi->pm_enabled = true; 714 *ppmc = phw->phw_pmc; 715 } else { 716 pi->pm_enabled = false; 717 *ppmc = NULL; 718 } 719 720 return (0); 721 } 722 723 /* 724 * Return the MSR address of the given PMC. 725 */ 726 static int 727 amd_get_msr(int ri, uint32_t *msr) 728 { 729 int df_idx; 730 731 KASSERT(ri >= 0 && ri < amd_npmcs, 732 ("[amd,%d] ri %d out of range", __LINE__, ri)); 733 734 /* 735 * Map counter row index to RDPMC ECX value. 736 * 737 * AMD BKDG 24594 rev 3.37, page 440, 738 * "RDPMC Read Performance-Monitoring Counter": 739 * ECX 0-5: Core counters 0-5 740 * ECX 6-9: DF/Northbridge counters 0-3 741 * ECX 10-15: L3 Cache counters 0-5 742 * ECX 16-27: DF/Northbridge counters 4-15 743 * 744 * AMD PPR 57930-A0 section 2.1.9, 745 * "Register Sharing" for DF counter details. 746 */ 747 if (ri < amd_core_npmcs) { 748 /* ECX 0-5: Core counters */ 749 *msr = ri; 750 } else if (ri < amd_core_npmcs + amd_l3_npmcs) { 751 /* ECX 10-15: L3 Cache counters */ 752 *msr = 10 + (ri - amd_core_npmcs); 753 } else { 754 /* ECX 6-9: DF counters 0-3 755 * ECX 16-27: DF counters 4-15 */ 756 df_idx = ri - amd_core_npmcs - amd_l3_npmcs; 757 if (df_idx < 4) 758 *msr = 6 + df_idx; 759 else if (df_idx < 16) 760 *msr = 16 + (df_idx - 4); 761 else 762 return (EINVAL); 763 } 764 return (0); 765 } 766 767 /* 768 * Return the capabilities of the given PMC. 769 */ 770 static int 771 amd_get_caps(int ri, uint32_t *caps) 772 { 773 KASSERT(ri >= 0 && ri < amd_npmcs, 774 ("[amd,%d] ri %d out of range", __LINE__, ri)); 775 776 *caps = amd_pmcdesc[ri].pm_descr.pd_caps; 777 778 return (0); 779 } 780 781 /* 782 * Processor-dependent initialization. 783 */ 784 static int 785 amd_pcpu_init(struct pmc_mdep *md, int cpu) 786 { 787 struct amd_cpu *pac; 788 struct pmc_cpu *pc; 789 struct pmc_hw *phw; 790 int first_ri, n; 791 792 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 793 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 794 795 PMCDBG1(MDP, INI, 1, "amd-init cpu=%d", cpu); 796 797 amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, 798 M_WAITOK | M_ZERO); 799 800 /* 801 * Set the content of the hardware descriptors to a known 802 * state and initialize pointers in the MI per-cpu descriptor. 803 */ 804 pc = pmc_pcpu[cpu]; 805 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8].pcd_ri; 806 807 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); 808 809 for (n = 0, phw = pac->pc_amdpmcs; n < amd_npmcs; n++, phw++) { 810 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 811 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 812 phw->phw_pmc = NULL; 813 pc->pc_hwpmcs[n + first_ri] = phw; 814 } 815 816 return (0); 817 } 818 819 /* 820 * Processor-dependent cleanup prior to the KLD being unloaded. 821 */ 822 static int 823 amd_pcpu_fini(struct pmc_mdep *md, int cpu) 824 { 825 struct amd_cpu *pac; 826 struct pmc_cpu *pc; 827 int first_ri, i; 828 829 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 830 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 831 832 PMCDBG1(MDP, INI, 1, "amd-cleanup cpu=%d", cpu); 833 834 /* 835 * Next, free up allocated space. 836 */ 837 if ((pac = amd_pcpu[cpu]) == NULL) 838 return (0); 839 840 amd_pcpu[cpu] = NULL; 841 842 #ifdef HWPMC_DEBUG 843 for (i = 0; i < AMD_NPMCS_K8; i++) { 844 KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, 845 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 846 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i), 847 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 848 } 849 #endif 850 851 pc = pmc_pcpu[cpu]; 852 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__)); 853 854 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8].pcd_ri; 855 856 /* 857 * Reset pointers in the MI 'per-cpu' state. 858 */ 859 for (i = 0; i < amd_npmcs; i++) 860 pc->pc_hwpmcs[i + first_ri] = NULL; 861 862 free(pac, M_PMC); 863 return (0); 864 } 865 866 /* 867 * Initialize ourselves. 868 */ 869 struct pmc_mdep * 870 pmc_amd_initialize(void) 871 { 872 struct amd_descr *d; 873 struct pmc_classdep *pcd; 874 struct pmc_mdep *pmc_mdep; 875 uint64_t reg; 876 enum pmc_cputype cputype; 877 int ncpus, nclasses, i; 878 int family, model, stepping; 879 int error; 880 881 /* 882 * The presence of hardware performance counters on the AMD 883 * Athlon, Duron or later processors, is _not_ indicated by 884 * any of the processor feature flags set by the 'CPUID' 885 * instruction, so we only check the 'instruction family' 886 * field returned by CPUID for instruction family >= 6. 887 */ 888 889 family = CPUID_TO_FAMILY(cpu_id); 890 model = CPUID_TO_MODEL(cpu_id); 891 stepping = CPUID_TO_STEPPING(cpu_id); 892 893 if (family == 0x18) 894 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X", 895 family, model, stepping); 896 else 897 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X", 898 family, model, stepping); 899 900 switch (cpu_id & 0xF00) { 901 case 0xF00: /* Athlon64/Opteron processor */ 902 cputype = PMC_CPU_AMD_K8; 903 break; 904 default: 905 printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, family, 906 model); 907 return (NULL); 908 } 909 910 /* 911 * Unforunately, there is no way to communicate that the original four 912 * core counters are disabled through CPUIDs alone. We attempt to 913 * write and read back the MSR to validate that it is working. 914 * 915 * Referenced the BIOS and Kernel Developer Guide for AMD Athlon 64 and 916 * AMD Opteron Processors 26094 Rev. 3.24 January, 2005 to ensure these 917 * fields are valid. 918 */ 919 if ((amd_feature2 & AMDID2_PCXC) == 0) { 920 error = wrmsr_safe(AMD_PMC_EVSEL_0, AMD_PMC_OS | AMD_PMC_USR); 921 if (error != 0) { 922 printf("hwpmc: AMD evsel 0 wrmsr failed!\n"); 923 return (NULL); 924 } 925 926 error = rdmsr_safe(AMD_PMC_EVSEL_0, ®); 927 if (error != 0) { 928 printf("hwpmc: AMD evsel 0 rdmsr failed!\n"); 929 return (NULL); 930 } 931 932 if (reg == 0) { 933 printf("hwpmc: AMD evsel returned invalid value! " 934 "You may be in a VM without PMC support.\n"); 935 return (NULL); 936 } 937 938 wrmsr(AMD_PMC_EVSEL_0, 0); 939 } 940 941 /* 942 * From PPR for AMD Family 1Ah, a new cpuid leaf specifies the maximum 943 * number of PMCs of each type. If we do not have that leaf, we use 944 * the prior default values that are only valid if we have the feature 945 * bit enabled in CPU. 946 */ 947 if ((amd_feature2 & AMDID2_PCXC) != 0) { 948 amd_core_npmcs = AMD_PMC_CORE_DEFAULT; 949 } else { 950 amd_core_npmcs = AMD_NPMCS_K8; 951 } 952 amd_l3_npmcs = AMD_PMC_L3_DEFAULT; 953 amd_df_npmcs = AMD_PMC_DF_DEFAULT; 954 955 if (cpu_exthigh >= CPUID_EXTPERFMON) { 956 u_int regs[4]; 957 do_cpuid(CPUID_EXTPERFMON, regs); 958 if (regs[1] != 0) { 959 amd_core_npmcs = EXTPERFMON_CORE_PMCS(regs[1]); 960 amd_df_npmcs = EXTPERFMON_DF_PMCS(regs[1]); 961 } 962 } 963 964 /* Enable the newer core counters */ 965 for (i = 0; i < amd_core_npmcs; i++) { 966 d = &amd_pmcdesc[i]; 967 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 968 "K8-%d", i); 969 d->pm_descr.pd_class = PMC_CLASS_K8; 970 d->pm_descr.pd_caps = AMD_PMC_CAPS; 971 /* 972 * Zen 5 can precisely count retire events. 973 * 974 * Refer to PPR Vol 1 for AMD Family 1Ah Model 02h C1 57238 975 * Rev. 0.24 September 29, 2024. 976 */ 977 if ((family >= 0x1a) && (i == 2)) 978 d->pm_descr.pd_caps |= PMC_CAP_PRECISE; 979 d->pm_descr.pd_width = 48; 980 if ((amd_feature2 & AMDID2_PCXC) != 0) { 981 d->pm_evsel = AMD_PMC_CORE_BASE + 2 * i; 982 d->pm_perfctr = AMD_PMC_CORE_BASE + 2 * i + 1; 983 } else { 984 d->pm_evsel = AMD_PMC_EVSEL_0 + i; 985 d->pm_perfctr = AMD_PMC_PERFCTR_0 + i; 986 } 987 d->pm_subclass = PMC_AMD_SUB_CLASS_CORE; 988 } 989 amd_npmcs = amd_core_npmcs; 990 991 if ((amd_feature2 & AMDID2_PTSCEL2I) != 0) { 992 /* Enable the LLC/L3 counters */ 993 for (i = 0; i < amd_l3_npmcs; i++) { 994 d = &amd_pmcdesc[amd_npmcs + i]; 995 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 996 "K8-L3-%d", i); 997 d->pm_descr.pd_class = PMC_CLASS_K8; 998 d->pm_descr.pd_caps = AMD_PMC_L3_CAPS; 999 d->pm_descr.pd_width = 48; 1000 d->pm_evsel = AMD_PMC_L3_BASE + 2 * i; 1001 d->pm_perfctr = AMD_PMC_L3_BASE + 2 * i + 1; 1002 d->pm_subclass = PMC_AMD_SUB_CLASS_L3_CACHE; 1003 } 1004 amd_npmcs += amd_l3_npmcs; 1005 } 1006 1007 if ((amd_feature2 & AMDID2_PNXC) != 0) { 1008 /* Enable the data fabric counters */ 1009 for (i = 0; i < amd_df_npmcs; i++) { 1010 d = &amd_pmcdesc[amd_npmcs + i]; 1011 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1012 "K8-DF-%d", i); 1013 d->pm_descr.pd_class = PMC_CLASS_K8; 1014 d->pm_descr.pd_caps = AMD_PMC_DF_CAPS; 1015 d->pm_descr.pd_width = 48; 1016 d->pm_evsel = AMD_PMC_DF_BASE + 2 * i; 1017 d->pm_perfctr = AMD_PMC_DF_BASE + 2 * i + 1; 1018 d->pm_subclass = PMC_AMD_SUB_CLASS_DATA_FABRIC; 1019 } 1020 amd_npmcs += amd_df_npmcs; 1021 } 1022 1023 /* 1024 * Allocate space for pointers to PMC HW descriptors and for 1025 * the MDEP structure used by MI code. 1026 */ 1027 amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC, 1028 M_WAITOK | M_ZERO); 1029 1030 /* 1031 * These processors have two or three classes of PMCs: the TSC, 1032 * programmable PMCs, and AMD IBS. 1033 */ 1034 if ((amd_feature2 & AMDID2_IBS) != 0) { 1035 nclasses = 3; 1036 } else { 1037 nclasses = 2; 1038 } 1039 1040 pmc_mdep = pmc_mdep_alloc(nclasses); 1041 1042 ncpus = pmc_cpu_max(); 1043 1044 /* Initialize the TSC. */ 1045 error = pmc_tsc_initialize(pmc_mdep, ncpus); 1046 if (error != 0) 1047 goto error; 1048 1049 /* Initialize AMD K8 PMC handling. */ 1050 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8]; 1051 1052 pcd->pcd_caps = AMD_PMC_CAPS; 1053 pcd->pcd_class = PMC_CLASS_K8; 1054 pcd->pcd_num = amd_npmcs; 1055 pcd->pcd_ri = pmc_mdep->pmd_npmc; 1056 pcd->pcd_width = 48; 1057 1058 pcd->pcd_allocate_pmc = amd_allocate_pmc; 1059 pcd->pcd_config_pmc = amd_config_pmc; 1060 pcd->pcd_describe = amd_describe; 1061 pcd->pcd_get_config = amd_get_config; 1062 pcd->pcd_get_msr = amd_get_msr; 1063 pcd->pcd_pcpu_fini = amd_pcpu_fini; 1064 pcd->pcd_pcpu_init = amd_pcpu_init; 1065 pcd->pcd_read_pmc = amd_read_pmc; 1066 pcd->pcd_release_pmc = amd_release_pmc; 1067 pcd->pcd_start_pmc = amd_start_pmc; 1068 pcd->pcd_stop_pmc = amd_stop_pmc; 1069 pcd->pcd_write_pmc = amd_write_pmc; 1070 pcd->pcd_get_caps = amd_get_caps; 1071 1072 pmc_mdep->pmd_cputype = cputype; 1073 pmc_mdep->pmd_intr = amd_intr; 1074 pmc_mdep->pmd_switch_in = amd_switch_in; 1075 pmc_mdep->pmd_switch_out = amd_switch_out; 1076 1077 pmc_mdep->pmd_npmc += amd_npmcs; 1078 1079 amd_init_policy(); 1080 1081 PMCDBG0(MDP, INI, 0, "amd-initialize"); 1082 1083 if (nclasses >= 3) { 1084 error = pmc_ibs_initialize(pmc_mdep, ncpus); 1085 if (error != 0) 1086 goto error; 1087 } 1088 1089 return (pmc_mdep); 1090 1091 error: 1092 free(pmc_mdep, M_PMC); 1093 return (NULL); 1094 } 1095 1096 /* 1097 * Finalization code for AMD CPUs. 1098 */ 1099 void 1100 pmc_amd_finalize(struct pmc_mdep *md) 1101 { 1102 PMCDBG0(MDP, INI, 1, "amd-finalize"); 1103 1104 pmc_tsc_finalize(md); 1105 1106 for (int i = 0; i < pmc_cpu_max(); i++) 1107 KASSERT(amd_pcpu[i] == NULL, 1108 ("[amd,%d] non-null pcpu cpu %d", __LINE__, i)); 1109 1110 free(amd_pcpu, M_PMC); 1111 amd_pcpu = NULL; 1112 } 1113