1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* Support for the AMD K8 and later processors */ 34 35 #include <sys/param.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/mutex.h> 39 #include <sys/pcpu.h> 40 #include <sys/pmc.h> 41 #include <sys/pmckern.h> 42 #include <sys/smp.h> 43 #include <sys/sysctl.h> 44 #include <sys/systm.h> 45 46 #define EXTERR_CATEGORY EXTERR_CAT_HWPMC_AMD 47 #include <sys/exterrvar.h> 48 49 #include <machine/atomic.h> 50 #include <machine/cpu.h> 51 #include <machine/cpufunc.h> 52 #include <machine/md_var.h> 53 #include <machine/specialreg.h> 54 55 #define OVERFLOW_WAIT_COUNT 50 56 57 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter); 58 59 /* AMD K8 PMCs */ 60 struct amd_descr { 61 struct pmc_descr pm_descr; /* "base class" */ 62 uint32_t pm_evsel; /* address of EVSEL register */ 63 uint32_t pm_perfctr; /* address of PERFCTR register */ 64 enum sub_class pm_subclass; /* register subclass */ 65 }; 66 67 static struct amd_descr *amd_pmcdesc; 68 69 static int amd_npmcs; 70 static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs, amd_umc_npmcs; 71 72 struct amd_event_code_map { 73 enum pmc_event pe_ev; /* enum value */ 74 uint16_t pe_code; /* encoded event mask */ 75 uint8_t pe_mask; /* bits allowed in unit mask */ 76 }; 77 78 const struct amd_event_code_map amd_event_codes[] = { 79 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 80 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 81 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 82 83 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 84 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 85 0x21, 0x00 }, 86 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 87 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 88 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 89 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 90 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 91 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 92 93 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 94 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 95 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 96 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 97 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 98 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 99 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 100 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 101 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 102 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 103 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 104 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 105 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 106 107 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 108 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 109 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 110 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 111 112 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 113 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 114 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 115 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 116 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 117 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 118 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 119 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 120 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 121 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 122 123 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 124 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 125 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 126 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 127 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 128 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 129 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 130 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 131 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 132 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 133 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 134 0xCA, 0x00 }, 135 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 136 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 137 0xCC, 0x07 }, 138 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 139 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 140 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 141 142 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 143 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 144 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 145 0xD2, 0x00 }, 146 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 147 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 148 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 149 0xD5, 0x00 }, 150 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 151 0xD6, 0x00 }, 152 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 153 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 154 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 155 0xD9, 0x00 }, 156 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 157 0xDA, 0x00 }, 158 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 159 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 160 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 161 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 162 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 163 164 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 165 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 166 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 167 0xE2, 0x00 }, 168 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 169 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 170 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 171 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 172 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 173 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 174 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 175 176 }; 177 178 const int amd_event_codes_size = nitems(amd_event_codes); 179 180 /* 181 * This struct holds per-processor information. 182 * 183 * pc_global_mask marks the core counters that must run on this CPU now. 184 * pc_virtual_mask marks per-process counters that wait to start at the 185 * next context switch. The switch clears this mask each time, so it 186 * never holds a system-wide counter. System counters and per-process 187 * counters always use different row indices. They never share one 188 * GLOBAL_CTL bit. 189 */ 190 struct amd_cpu { 191 struct pmc_hw *pc_amdpmcs; 192 volatile u_int pc_global_mask; 193 volatile u_int pc_virtual_mask; 194 volatile u_int pc_gate_depth; 195 }; 196 static struct amd_cpu **amd_pcpu; 197 198 /* Populated by amd_init_policy(); PRECISERETIRE is OR-ed in per-allocation. */ 199 static uint64_t amd_core_allowed_mask; 200 static uint64_t amd_l3_allowed_mask; 201 static uint64_t amd_df_allowed_mask; 202 static uint64_t amd_umc_allowed_mask; 203 204 static uint64_t amd_core_extra_mask; 205 static uint64_t amd_l3_extra_mask; 206 static uint64_t amd_df_extra_mask; 207 static uint64_t amd_umc_extra_mask; 208 209 static bool amd_perfmon_v2; 210 /* Mask with one bit for each core counter */ 211 static uint64_t amd_global_cntr_mask; 212 213 SYSCTL_DECL(_kern_hwpmc); 214 215 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_core_extra_mask, CTLFLAG_RDTUN, 216 &amd_core_extra_mask, 0, 217 "Extra allowed bits in AMD core PMU PERFEVTSEL (override; default 0)"); 218 219 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_l3_extra_mask, CTLFLAG_RDTUN, 220 &amd_l3_extra_mask, 0, 221 "Extra allowed bits in AMD L3 PMU control (override; default 0)"); 222 223 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_df_extra_mask, CTLFLAG_RDTUN, 224 &amd_df_extra_mask, 0, 225 "Extra allowed bits in AMD DF PMU control (override; default 0)"); 226 227 SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_umc_extra_mask, CTLFLAG_RDTUN, 228 &amd_umc_extra_mask, 0, 229 "Extra allowed bits in AMD UMC PMU control (override; default 0)"); 230 231 SYSCTL_BOOL(_kern_hwpmc, OID_AUTO, amd_perfmon_v2, CTLFLAG_RD, 232 &amd_perfmon_v2, 0, 233 "AMD PerfMonV2 global-control path selected (read-only)"); 234 235 static void 236 amd_init_policy(void) 237 { 238 int family; 239 240 family = CPUID_TO_FAMILY(cpu_id); 241 242 amd_core_allowed_mask = AMD_VALID_BITS; 243 244 amd_l3_allowed_mask = (family <= 0x17) ? 245 AMD_PMC_L3_FAMILY17_MASK : AMD_PMC_L3_FAMILY19_MASK; 246 247 amd_df_allowed_mask = (family <= 0x19) ? 248 AMD_PMC_DF_FAMILY17_MASK : AMD_PMC_DF_FAMILY1A_MASK; 249 250 amd_umc_allowed_mask = AMD_PMC_UMC_MASK; 251 } 252 253 static uint64_t 254 amd_config_mask(enum sub_class subclass, uint64_t caps) 255 { 256 257 switch (subclass) { 258 case PMC_AMD_SUB_CLASS_CORE: 259 return (amd_core_allowed_mask | amd_core_extra_mask | 260 (((caps & PMC_CAP_PRECISE) != 0) ? 261 AMD_PMC_PRECISERETIRE : 0)); 262 case PMC_AMD_SUB_CLASS_L3_CACHE: 263 return (amd_l3_allowed_mask | amd_l3_extra_mask); 264 case PMC_AMD_SUB_CLASS_DATA_FABRIC: 265 return (amd_df_allowed_mask | amd_df_extra_mask); 266 case PMC_AMD_SUB_CLASS_UMC: 267 return (amd_umc_allowed_mask | amd_umc_extra_mask); 268 default: 269 return (0); 270 } 271 } 272 273 static __inline u_int 274 amd_v2_counter_mask(int ri) 275 { 276 KASSERT(ri >= 0 && ri < amd_core_npmcs, 277 ("[amd,%d] illegal core row-index %d", __LINE__, ri)); 278 return (1U << ri); 279 } 280 281 static __inline void 282 amd_v2_assert_mask(u_int mask) 283 { 284 KASSERT((mask & ~(u_int)amd_global_cntr_mask) == 0, 285 ("[amd,%d] invalid GLOBAL_CTL mask %#x", __LINE__, mask)); 286 } 287 288 static __inline void 289 amd_v2_publish_mask(int cpu) 290 { 291 struct amd_cpu *pac; 292 u_int mask; 293 294 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 295 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 296 pac = amd_pcpu[cpu]; 297 KASSERT(pac != NULL, 298 ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); 299 mask = atomic_load_acq_int(&pac->pc_global_mask); 300 amd_v2_assert_mask(mask); 301 if (atomic_load_acq_int(&pac->pc_gate_depth) == 0) 302 wrmsr(AMD_PMC_GLOBAL_CTL, mask); 303 } 304 305 static void 306 amd_v2_stage_virtual(int cpu, int ri) 307 { 308 struct amd_cpu *pac; 309 u_int mask; 310 311 pac = amd_pcpu[cpu]; 312 mask = amd_v2_counter_mask(ri); 313 atomic_set_int(&pac->pc_virtual_mask, mask); 314 } 315 316 static int 317 amd_start_pmc_all_v2(int cpu) 318 { 319 struct amd_cpu *pac; 320 u_int mask; 321 322 pac = amd_pcpu[cpu]; 323 mask = atomic_load_acq_int(&pac->pc_virtual_mask); 324 if (mask == 0) 325 return (0); 326 amd_v2_assert_mask(mask); 327 atomic_set_int(&pac->pc_global_mask, mask); 328 amd_v2_publish_mask(cpu); 329 return (0); 330 } 331 332 /* Clear only the staged virtual bits. System-wide counters keep running. */ 333 static int 334 amd_stop_pmc_all_v2(int cpu) 335 { 336 struct amd_cpu *pac; 337 u_int mask; 338 339 pac = amd_pcpu[cpu]; 340 mask = atomic_readandclear_int(&pac->pc_virtual_mask); 341 if (mask == 0) 342 return (0); 343 amd_v2_assert_mask(mask); 344 atomic_clear_int(&pac->pc_global_mask, mask); 345 amd_v2_publish_mask(cpu); 346 return (0); 347 } 348 349 static void 350 amd_v2_forget_core(int cpu, int ri, struct pmc *pm) 351 { 352 struct amd_cpu *pac; 353 u_int mask; 354 355 pac = amd_pcpu[cpu]; 356 mask = amd_v2_counter_mask(ri); 357 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) 358 atomic_clear_int(&pac->pc_virtual_mask, mask); 359 if ((atomic_load_acq_int(&pac->pc_global_mask) & mask) != 0) { 360 atomic_clear_int(&pac->pc_global_mask, mask); 361 amd_v2_publish_mask(cpu); 362 } 363 } 364 365 static __inline void 366 amd_v2_disable_all(void) 367 { 368 wrmsr(AMD_PMC_GLOBAL_CTL, 0); 369 } 370 371 static void 372 amd_v2_freeze_core(int cpu) 373 { 374 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 375 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 376 atomic_add_int(&amd_pcpu[cpu]->pc_gate_depth, 1); 377 amd_v2_disable_all(); 378 } 379 380 static void 381 amd_v2_thaw_core(int cpu) 382 { 383 u_int old; 384 385 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 386 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 387 old = atomic_fetchadd_int(&amd_pcpu[cpu]->pc_gate_depth, -1); 388 KASSERT(old > 0, 389 ("AMD PMC gate-depth underflow on CPU %d", cpu)); 390 if (old == 1) { 391 u_int mask; 392 393 mask = atomic_load_acq_int(&amd_pcpu[cpu]->pc_global_mask); 394 amd_v2_assert_mask(mask); 395 wrmsr(AMD_PMC_GLOBAL_CTL, mask); 396 } 397 } 398 399 /* 400 * Read a PMC value from the MSR. 401 */ 402 static int 403 amd_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 404 { 405 const struct amd_descr *pd; 406 pmc_value_t tmp; 407 enum pmc_mode mode; 408 409 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 410 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 411 KASSERT(ri >= 0 && ri < amd_npmcs, 412 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 413 KASSERT(amd_pcpu[cpu], 414 ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); 415 416 pd = &amd_pmcdesc[ri]; 417 mode = PMC_TO_MODE(pm); 418 419 PMCDBG2(MDP, REA, 1, "amd-read id=%d class=%d", ri, 420 pd->pm_descr.pd_class); 421 422 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 423 PMCDBG2(MDP, REA, 2, "amd-read (pre-munge) id=%d -> %jd", ri, tmp); 424 if (PMC_IS_SAMPLING_MODE(mode)) { 425 /* 426 * Clamp value to 0 if the counter just overflowed, 427 * otherwise the returned reload count would wrap to a 428 * huge value. 429 */ 430 if ((tmp & (1ULL << 47)) == 0) 431 tmp = 0; 432 else { 433 /* Sign extend 48 bit value to 64 bits. */ 434 tmp = (pmc_value_t) ((int64_t)(tmp << 16) >> 16); 435 tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 436 } 437 } 438 *v = tmp; 439 440 PMCDBG2(MDP, REA, 2, "amd-read (post-munge) id=%d -> %jd", ri, *v); 441 442 return (0); 443 } 444 445 /* 446 * Write a PMC MSR. 447 */ 448 static int 449 amd_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 450 { 451 const struct amd_descr *pd; 452 enum pmc_mode mode; 453 454 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 455 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 456 KASSERT(ri >= 0 && ri < amd_npmcs, 457 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 458 459 pd = &amd_pmcdesc[ri]; 460 mode = PMC_TO_MODE(pm); 461 462 /* use 2's complement of the count for sampling mode PMCs */ 463 if (PMC_IS_SAMPLING_MODE(mode)) 464 v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 465 466 PMCDBG3(MDP, WRI, 1, "amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 467 468 /* write the PMC value */ 469 wrmsr(pd->pm_perfctr, v); 470 return (0); 471 } 472 473 /* 474 * Configure hardware PMC according to the configuration recorded in 'pm'. 475 */ 476 static int 477 amd_config_pmc(int cpu, int ri, struct pmc *pm) 478 { 479 struct pmc_hw *phw; 480 481 PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 482 483 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 484 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 485 KASSERT(ri >= 0 && ri < amd_npmcs, 486 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 487 488 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 489 490 KASSERT(pm == NULL || phw->phw_pmc == NULL, 491 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 492 __LINE__, pm, phw->phw_pmc)); 493 494 if (amd_perfmon_v2 && pm == NULL && phw->phw_pmc != NULL && 495 amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE) 496 amd_v2_forget_core(cpu, ri, phw->phw_pmc); 497 498 phw->phw_pmc = pm; 499 return (0); 500 } 501 502 /* 503 * Retrieve a configured PMC pointer from hardware state. 504 */ 505 static int 506 amd_get_config(int cpu, int ri, struct pmc **ppm) 507 { 508 *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; 509 return (0); 510 } 511 512 /* 513 * Machine-dependent actions taken during the context switch in of a 514 * thread. 515 */ 516 static int 517 amd_switch_in(struct pmc_cpu *pc __pmcdbg_used, struct pmc_process *pp) 518 { 519 PMCDBG3(MDP, SWI, 1, "pc=%p pp=%p enable-msr=%d", pc, pp, 520 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 521 522 /* enable the RDPMC instruction if needed */ 523 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 524 load_cr4(rcr4() | CR4_PCE); 525 526 return (0); 527 } 528 529 /* 530 * Machine-dependent actions taken during the context switch out of a 531 * thread. 532 */ 533 static int 534 amd_switch_out(struct pmc_cpu *pc __pmcdbg_used, 535 struct pmc_process *pp __pmcdbg_used) 536 { 537 PMCDBG3(MDP, SWO, 1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 538 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 539 540 /* always turn off the RDPMC instruction */ 541 load_cr4(rcr4() & ~CR4_PCE); 542 543 return (0); 544 } 545 546 /* 547 * Check if an event can use row index 'ri'. 548 */ 549 static int 550 amd_can_assign_pmc(int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) 551 { 552 const struct pmc_descr *pd; 553 uint64_t allowed_unitmask, caps, config, unitmask; 554 enum pmc_event pe; 555 int i; 556 557 KASSERT(ri >= 0 && ri < amd_npmcs, 558 ("[amd,%d] illegal row index %d", __LINE__, ri)); 559 560 pd = &amd_pmcdesc[ri].pm_descr; 561 562 if (pd->pd_class != a->pm_class) 563 return (EINVAL); 564 565 caps = pm->pm_caps; 566 567 /* Validate sub-class. */ 568 if (amd_pmcdesc[ri].pm_subclass != a->pm_md.pm_amd.pm_amd_sub_class) 569 return (EINVAL); 570 571 if (((caps & PMC_CAP_PRECISE) != 0) && 572 ((pd->pd_caps & PMC_CAP_PRECISE) == 0)) 573 return (EINVAL); 574 575 /* PMC_F_EV_PMU: config comes from pmu-events tables. */ 576 if ((a->pm_flags & PMC_F_EV_PMU) != 0) { 577 config = a->pm_md.pm_amd.pm_amd_config; 578 if ((config & ~amd_config_mask(amd_pmcdesc[ri].pm_subclass, 579 caps)) != 0) 580 return (EXTERROR(EINVAL, 581 "AMD PMU config has unsupported bits %#jx", 582 (uintmax_t)(config & ~amd_config_mask( 583 amd_pmcdesc[ri].pm_subclass, caps)))); 584 return (0); 585 } 586 587 pe = a->pm_ev; 588 config = allowed_unitmask = 0; 589 for (i = 0; i < amd_event_codes_size; i++) { 590 if (amd_event_codes[i].pe_ev == pe) { 591 config = 592 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 593 allowed_unitmask = 594 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 595 break; 596 } 597 } 598 if (i == amd_event_codes_size) 599 return (EXTERROR(EINVAL, 600 "AMD legacy event %ju is not supported", 601 (uintmax_t)pe)); 602 603 unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 604 if ((unitmask & ~allowed_unitmask) != 0) /* disallow reserved bits */ 605 return (EXTERROR(EINVAL, 606 "AMD unitmask %#jx exceeds allowed mask %#jx", 607 (uintmax_t)unitmask, (uintmax_t)allowed_unitmask)); 608 609 return (0); 610 } 611 612 /* 613 * Check if a given PMC allocation is feasible. 614 */ 615 static int 616 amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, 617 const struct pmc_op_pmcallocate *a) 618 { 619 uint64_t allowed_unitmask, caps, config, unitmask; 620 enum pmc_event pe; 621 int error, i; 622 623 error = amd_can_assign_pmc(ri, pm, a); 624 if (error != 0) 625 return (error); 626 627 caps = pm->pm_caps; 628 PMCDBG2(MDP, ALL, 1,"amd-allocate ri=%d caps=0x%x", ri, caps); 629 630 /* PMC_F_EV_PMU: config comes from pmu-events tables. */ 631 if ((a->pm_flags & PMC_F_EV_PMU) != 0) { 632 config = a->pm_md.pm_amd.pm_amd_config; 633 pm->pm_md.pm_amd.pm_amd_evsel = config; 634 PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx", 635 ri, (uintmax_t)config); 636 return (0); 637 } 638 639 pe = a->pm_ev; 640 config = allowed_unitmask = 0; 641 for (i = 0; i < amd_event_codes_size; i++) { 642 if (amd_event_codes[i].pe_ev == pe) { 643 config = 644 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 645 allowed_unitmask = 646 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 647 break; 648 } 649 } 650 651 unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 652 if (unitmask && (caps & PMC_CAP_QUALIFIER) != 0) 653 config |= unitmask; 654 655 if ((caps & PMC_CAP_THRESHOLD) != 0) 656 config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 657 658 if ((caps & PMC_CAP_USER) != 0) 659 config |= AMD_PMC_USR; 660 if ((caps & PMC_CAP_SYSTEM) != 0) 661 config |= AMD_PMC_OS; 662 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) 663 config |= (AMD_PMC_USR|AMD_PMC_OS); 664 665 if ((caps & PMC_CAP_EDGE) != 0) 666 config |= AMD_PMC_EDGE; 667 if ((caps & PMC_CAP_INVERT) != 0) 668 config |= AMD_PMC_INVERT; 669 if ((caps & PMC_CAP_INTERRUPT) != 0) 670 config |= AMD_PMC_INT; 671 672 pm->pm_md.pm_amd.pm_amd_evsel = config; 673 674 PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%x", ri, config); 675 676 return (0); 677 } 678 679 /* 680 * Release machine dependent state associated with a PMC. This is a 681 * no-op on this architecture. 682 */ 683 static int 684 amd_release_pmc(int cpu, int ri, struct pmc *pmc __unused) 685 { 686 struct pmc_hw *phw __diagused; 687 688 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 689 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 690 KASSERT(ri >= 0 && ri < amd_npmcs, 691 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 692 693 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 694 695 KASSERT(phw->phw_pmc == NULL, 696 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 697 698 return (0); 699 } 700 701 /* 702 * Start a PMC. 703 */ 704 static int 705 amd_start_pmc(int cpu __diagused, int ri, struct pmc *pm) 706 { 707 const struct amd_descr *pd; 708 uint64_t config; 709 710 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 711 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 712 KASSERT(ri >= 0 && ri < amd_npmcs, 713 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 714 715 pd = &amd_pmcdesc[ri]; 716 717 PMCDBG2(MDP, STA, 1, "amd-start cpu=%d ri=%d", cpu, ri); 718 719 /* 720 * Asserts triggered by DF/UMC counters because all DF/UMC MSRs are 721 * shared. While userspace now honors the per-node flags, we should 722 * enforce this in the kernel. 723 */ 724 /* turn on the PMC ENABLE bit */ 725 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC) { 726 KASSERT(AMD_PMC_UMC_IS_STOPPED(pd->pm_evsel), 727 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", 728 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 729 730 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_UMC_ENABLE; 731 } else { 732 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 733 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", 734 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 735 736 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 737 } 738 739 PMCDBG1(MDP, STA, 2, "amd-start config=0x%x", config); 740 741 wrmsr(pd->pm_evsel, config); 742 return (0); 743 } 744 745 /* Start one PMC. pcd_start_all commits the virtual GLOBAL_CTL bits. */ 746 static int 747 amd_start_pmc_v2(int cpu __diagused, int ri, struct pmc *pm) 748 { 749 const struct amd_descr *pd; 750 enum pmc_mode mode; 751 uint64_t config; 752 753 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 754 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 755 KASSERT(ri >= 0 && ri < amd_npmcs, 756 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 757 758 pd = &amd_pmcdesc[ri]; 759 mode = PMC_TO_MODE(pm); 760 761 PMCDBG2(MDP, STA, 1, "amd-start-v2 cpu=%d ri=%d", cpu, ri); 762 763 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && 764 PMC_IS_VIRTUAL_MODE(mode)) 765 amd_v2_stage_virtual(cpu, ri); 766 767 /* 768 * Drop a stale overflow status bit before the row goes live. 769 * amd_stop_pmc_v2() waits only OVERFLOW_WAIT_COUNT microseconds 770 * for the old occupant's in-flight NMI. Group rotation reuses 771 * rows often, so a surviving GLOBAL_STATUS bit would reach the 772 * late handler after the row restarts. The handler would then 773 * blame this PMC: a false sample, plus a mid-window reload for 774 * the wrong event. With the bit clear, the late handler finds 775 * no work and treats the NMI as spurious. Legacy AMD has no 776 * status plane to check, so it only relies on the bounded wait. 777 */ 778 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && 779 PMC_IS_SAMPLING_MODE(mode) && 780 (rdmsr(AMD_PMC_GLOBAL_STATUS) & (1ULL << ri)) != 0) { 781 wrmsr(AMD_PMC_GLOBAL_STATUS_CLR, 1ULL << ri); 782 783 /* 784 * That NMI may still be in flight. No status bit is left 785 * to claim it, so amd_intr_v2() would return 0 and the NMI 786 * would go unhandled (machdep.panic_on_nmi). Give the stray 787 * absorber one credit on this CPU. 788 */ 789 if (DPCPU_GET(nmi_counter) == 0) 790 DPCPU_SET(nmi_counter, 1); 791 } 792 793 /* Enable EVSEL while the virtual slot's global bit is off. */ 794 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 795 wrmsr(pd->pm_evsel, config); 796 797 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && 798 PMC_IS_SYSTEM_MODE(mode)) { 799 atomic_set_int(&amd_pcpu[cpu]->pc_global_mask, 800 amd_v2_counter_mask(ri)); 801 amd_v2_publish_mask(cpu); 802 } 803 804 return (0); 805 } 806 807 /* 808 * Stop a PMC. 809 */ 810 static int 811 amd_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) 812 { 813 const struct amd_descr *pd; 814 uint64_t config; 815 int i; 816 817 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 818 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 819 KASSERT(ri >= 0 && ri < amd_npmcs, 820 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 821 822 pd = &amd_pmcdesc[ri]; 823 824 PMCDBG1(MDP, STO, 1, "amd-stop ri=%d", ri); 825 826 /* turn off the PMC ENABLE bit */ 827 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC) { 828 KASSERT(!AMD_PMC_UMC_IS_STOPPED(pd->pm_evsel), 829 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 830 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 831 832 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE; 833 } else { 834 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 835 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 836 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 837 838 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 839 } 840 wrmsr(pd->pm_evsel, config); 841 842 /* 843 * Due to NMI latency on newer AMD processors 844 * NMI interrupts are ignored, which leads to 845 * panic or messages based on kernel configuration 846 */ 847 848 /* Wait for the count to be reset */ 849 for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { 850 if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1))) 851 break; 852 853 DELAY(1); 854 } 855 856 return (0); 857 } 858 859 /* Stop one PMC. pcd_stop_all already zeroed the virtual GLOBAL_CTL bits. */ 860 static int 861 amd_stop_pmc_v2(int cpu __diagused, int ri, struct pmc *pm) 862 { 863 const struct amd_descr *pd; 864 enum pmc_mode mode; 865 int i; 866 867 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 868 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 869 KASSERT(ri >= 0 && ri < amd_npmcs, 870 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 871 872 pd = &amd_pmcdesc[ri]; 873 mode = PMC_TO_MODE(pm); 874 875 PMCDBG1(MDP, STO, 1, "amd-stop-v2 ri=%d", ri); 876 877 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && 878 PMC_IS_SYSTEM_MODE(mode)) { 879 atomic_clear_int(&amd_pcpu[cpu]->pc_global_mask, 880 amd_v2_counter_mask(ri)); 881 amd_v2_publish_mask(cpu); 882 } 883 884 /* Disable EVSEL after the counter's global bit is off. */ 885 wrmsr(pd->pm_evsel, 886 pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE); 887 888 /* Wait out an in-flight overflow NMI. The handler clears the status bit. */ 889 if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && 890 PMC_IS_SAMPLING_MODE(mode)) { 891 for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { 892 if ((rdmsr(AMD_PMC_GLOBAL_STATUS) & (1ULL << ri)) == 0) 893 break; 894 895 DELAY(1); 896 } 897 } 898 899 return (0); 900 } 901 902 /* 903 * Interrupt handler. This function needs to return '1' if the 904 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 905 * to sleep or do anything a 'fast' interrupt handler is not allowed 906 * to do. 907 */ 908 static int 909 amd_intr(struct trapframe *tf) 910 { 911 struct amd_cpu *pac; 912 struct pmc *pm; 913 pmc_value_t v; 914 uint64_t config, evsel, perfctr; 915 uint32_t active = 0, count = 0; 916 int i, error, retval, cpu; 917 918 cpu = curcpu; 919 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 920 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 921 922 PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); 923 924 retval = 0; 925 926 pac = amd_pcpu[cpu]; 927 928 retval = pmc_ibs_intr(tf); 929 if (retval) 930 goto done; 931 932 /* 933 * look for all PMCs that have interrupted: 934 * - look for a running, sampling PMC which has overflowed 935 * and which has a valid 'struct pmc' association 936 * 937 * If found, we call a helper to process the interrupt. 938 * 939 * PMCs interrupting at the same time are collapsed into 940 * a single interrupt. Check all the valid pmcs for 941 * overflow. 942 */ 943 for (i = 0; i < amd_npmcs; i++) { 944 if (amd_pmcdesc[i].pm_subclass != PMC_AMD_SUB_CLASS_CORE) 945 break; 946 947 if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || 948 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 949 continue; 950 } 951 952 /* Consider pmc with valid handle as active */ 953 active++; 954 955 if (!AMD_PMC_HAS_OVERFLOWED(i)) 956 continue; 957 958 retval = 1; /* Found an interrupting PMC. */ 959 960 if (pm->pm_state != PMC_STATE_RUNNING) 961 continue; 962 963 /* Stop the PMC, reload count. */ 964 evsel = amd_pmcdesc[i].pm_evsel; 965 perfctr = amd_pmcdesc[i].pm_perfctr; 966 v = pm->pm_sc.pm_reloadcount; 967 config = rdmsr(evsel); 968 969 970 if (amd_pmcdesc[i].pm_subclass == PMC_AMD_SUB_CLASS_UMC) { 971 KASSERT((config & ~AMD_PMC_UMC_ENABLE) == 972 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE), 973 ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, 974 (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); 975 976 wrmsr(evsel, config & ~AMD_PMC_UMC_ENABLE); 977 } else { 978 KASSERT((config & ~AMD_PMC_ENABLE) == 979 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 980 ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, 981 (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); 982 983 wrmsr(evsel, config & ~AMD_PMC_ENABLE); 984 } 985 wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 986 987 /* Restart the counter if logging succeeded. */ 988 error = pmc_process_interrupt(PMC_HR, pm, tf); 989 if (error == 0) 990 wrmsr(evsel, config); 991 } 992 993 /* 994 * Due to NMI latency, there can be a scenario in which 995 * multiple pmcs gets serviced in an earlier NMI and we 996 * do not find an overflow in the subsequent NMI. 997 * 998 * For such cases we keep a per-cpu count of active NMIs 999 * and compare it with min(active pmcs, 2) to determine 1000 * if this NMI was for a pmc overflow which was serviced 1001 * in an earlier request or should be ignored. 1002 */ 1003 if (retval) { 1004 DPCPU_SET(nmi_counter, min(2, active)); 1005 } else { 1006 if ((count = DPCPU_GET(nmi_counter))) { 1007 retval = 1; 1008 DPCPU_SET(nmi_counter, --count); 1009 } 1010 } 1011 1012 done: 1013 if (retval) 1014 counter_u64_add(pmc_stats.pm_intr_processed, 1); 1015 else 1016 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 1017 1018 PMCDBG1(MDP, INT, 2, "retval=%d", retval); 1019 return (retval); 1020 } 1021 1022 /* 1023 * This is the v2 interrupt handler: freeze counters; read GLOBAL_STATUS once; 1024 * reload; thaw. 1025 */ 1026 static int 1027 amd_intr_v2(struct trapframe *tf) 1028 { 1029 struct amd_cpu *pac; 1030 struct pmc *pm; 1031 pmc_value_t v; 1032 uint64_t status, pending; 1033 uint32_t active = 0, count = 0; 1034 int i, error, retval, cpu; 1035 1036 cpu = curcpu; 1037 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 1038 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 1039 1040 PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); 1041 1042 retval = 0; 1043 pac = amd_pcpu[cpu]; 1044 1045 retval = pmc_ibs_intr(tf); 1046 if (retval) 1047 goto done; 1048 1049 amd_v2_freeze_core(cpu); 1050 1051 /* Read the overflow bitmap once. */ 1052 status = rdmsr(AMD_PMC_GLOBAL_STATUS); 1053 status &= amd_global_cntr_mask; 1054 1055 /* 1056 * Count all active sampling PMCs, not just the ones that 1057 * overflowed. Count the in-flight NMI too, its counter has 1058 * not overflowed yet. 1059 */ 1060 for (i = 0; i < amd_core_npmcs; i++) { 1061 pm = pac->pc_amdpmcs[i].phw_pmc; 1062 if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 1063 active++; 1064 } 1065 1066 /* ffsll() returns a 1-based bit index, or 0 when no bits are set. */ 1067 pending = status; 1068 while ((i = ffsll(pending) - 1) != -1) { 1069 pending &= ~(1ULL << i); 1070 1071 if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || 1072 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 1073 continue; 1074 } 1075 1076 retval = 1; 1077 1078 if (pm->pm_state != PMC_STATE_RUNNING) 1079 continue; 1080 1081 /* Reload the counter. */ 1082 v = pm->pm_sc.pm_reloadcount; 1083 wrmsr(amd_pmcdesc[i].pm_perfctr, 1084 AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 1085 1086 /* 1087 * On a log failure, leave the PMC disabled. MI code restarts 1088 * it via pcd_start_pmc. 1089 */ 1090 error = pmc_process_interrupt(PMC_HR, pm, tf); 1091 if (error != 0) 1092 wrmsr(amd_pmcdesc[i].pm_evsel, 1093 pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE); 1094 } 1095 1096 /* Acknowledge the overflow bits through GLOBAL_STATUS_CLR. */ 1097 wrmsr(AMD_PMC_GLOBAL_STATUS_CLR, status); 1098 1099 /* Thaw the core counters. */ 1100 amd_v2_thaw_core(cpu); 1101 1102 /* 1103 * An earlier NMI may have serviced this overflow. Absorb the stray 1104 * NMI. 1105 */ 1106 if (retval) { 1107 DPCPU_SET(nmi_counter, min(2, active)); 1108 } else { 1109 if ((count = DPCPU_GET(nmi_counter))) { 1110 retval = 1; 1111 DPCPU_SET(nmi_counter, --count); 1112 } 1113 } 1114 1115 done: 1116 if (retval) 1117 counter_u64_add(pmc_stats.pm_intr_processed, 1); 1118 else 1119 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 1120 1121 PMCDBG1(MDP, INT, 2, "retval=%d", retval); 1122 return (retval); 1123 } 1124 1125 /* 1126 * Describe a PMC. 1127 */ 1128 static int 1129 amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 1130 { 1131 const struct amd_descr *pd; 1132 struct pmc_hw *phw; 1133 1134 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 1135 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 1136 KASSERT(ri >= 0 && ri < amd_npmcs, 1137 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 1138 1139 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; 1140 pd = &amd_pmcdesc[ri]; 1141 1142 strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); 1143 pi->pm_class = pd->pm_descr.pd_class; 1144 1145 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) != 0) { 1146 pi->pm_enabled = true; 1147 *ppmc = phw->phw_pmc; 1148 } else { 1149 pi->pm_enabled = false; 1150 *ppmc = NULL; 1151 } 1152 1153 return (0); 1154 } 1155 1156 /* 1157 * Return the MSR address of the given PMC. 1158 */ 1159 static int 1160 amd_get_msr(int ri, uint32_t *msr) 1161 { 1162 int df_idx; 1163 1164 KASSERT(ri >= 0 && ri < amd_npmcs, 1165 ("[amd,%d] ri %d out of range", __LINE__, ri)); 1166 1167 /* 1168 * Map counter row index to RDPMC ECX value. 1169 * 1170 * AMD BKDG 24594 rev 3.37, page 440, 1171 * "RDPMC Read Performance-Monitoring Counter": 1172 * ECX 0-5: Core counters 0-5 1173 * ECX 6-9: DF/Northbridge counters 0-3 1174 * ECX 10-15: L3 Cache counters 0-5 1175 * ECX 16-27: DF/Northbridge counters 4-15 1176 * 1177 * AMD PPR 57930-A0 section 2.1.9, 1178 * "Register Sharing" for DF counter details. 1179 */ 1180 if (ri < amd_core_npmcs) { 1181 /* ECX 0-5: Core counters */ 1182 *msr = ri; 1183 } else if (ri < amd_core_npmcs + amd_l3_npmcs) { 1184 /* ECX 10-15: L3 Cache counters */ 1185 *msr = 10 + (ri - amd_core_npmcs); 1186 } else { 1187 /* ECX 6-9: DF counters 0-3 1188 * ECX 16-27: DF counters 4-15 */ 1189 df_idx = ri - amd_core_npmcs - amd_l3_npmcs; 1190 if (df_idx < 4) 1191 *msr = 6 + df_idx; 1192 else if (df_idx < 16) 1193 *msr = 16 + (df_idx - 4); 1194 else 1195 return (EINVAL); 1196 } 1197 return (0); 1198 } 1199 1200 /* 1201 * Return the capabilities of the given PMC. 1202 */ 1203 static int 1204 amd_get_caps(int ri, uint32_t *caps) 1205 { 1206 KASSERT(ri >= 0 && ri < amd_npmcs, 1207 ("[amd,%d] ri %d out of range", __LINE__, ri)); 1208 1209 *caps = amd_pmcdesc[ri].pm_descr.pd_caps; 1210 1211 return (0); 1212 } 1213 1214 /* 1215 * Processor-dependent initialization. 1216 */ 1217 static int 1218 amd_pcpu_init(struct pmc_mdep *md, int cpu) 1219 { 1220 struct amd_cpu *pac; 1221 struct pmc_cpu *pc; 1222 struct pmc_hw *phw; 1223 int first_ri, n; 1224 1225 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 1226 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 1227 1228 PMCDBG1(MDP, INI, 1, "amd-init cpu=%d", cpu); 1229 1230 amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, 1231 M_WAITOK | M_ZERO); 1232 pac->pc_amdpmcs = mallocarray(amd_npmcs, sizeof(*pac->pc_amdpmcs), 1233 M_PMC, M_WAITOK | M_ZERO); 1234 1235 if (amd_perfmon_v2) { 1236 KASSERT(atomic_load_acq_int(&pac->pc_gate_depth) == 0, 1237 ("[amd,%d] nonzero initial gate depth on CPU %d", 1238 __LINE__, cpu)); 1239 KASSERT(atomic_load_acq_int(&pac->pc_global_mask) == 0, 1240 ("[amd,%d] nonzero initial desired mask on CPU %d", 1241 __LINE__, cpu)); 1242 KASSERT(atomic_load_acq_int(&pac->pc_virtual_mask) == 0, 1243 ("[amd,%d] nonzero initial virtual mask on CPU %d", 1244 __LINE__, cpu)); 1245 amd_v2_disable_all(); 1246 } 1247 1248 /* 1249 * Set the content of the hardware descriptors to a known 1250 * state and initialize pointers in the MI per-cpu descriptor. 1251 */ 1252 pc = pmc_pcpu[cpu]; 1253 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8].pcd_ri; 1254 1255 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); 1256 1257 for (n = 0, phw = pac->pc_amdpmcs; n < amd_npmcs; n++, phw++) { 1258 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 1259 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 1260 phw->phw_pmc = NULL; 1261 pc->pc_hwpmcs[n + first_ri] = phw; 1262 } 1263 1264 return (0); 1265 } 1266 1267 /* 1268 * Processor-dependent cleanup prior to the KLD being unloaded. 1269 */ 1270 static int 1271 amd_pcpu_fini(struct pmc_mdep *md, int cpu) 1272 { 1273 struct amd_cpu *pac; 1274 struct pmc_cpu *pc; 1275 int first_ri, i; 1276 1277 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 1278 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 1279 1280 PMCDBG1(MDP, INI, 1, "amd-cleanup cpu=%d", cpu); 1281 1282 /* 1283 * Next, free up allocated space. 1284 */ 1285 if ((pac = amd_pcpu[cpu]) == NULL) 1286 return (0); 1287 1288 if (amd_perfmon_v2) { 1289 KASSERT(atomic_load_acq_int(&pac->pc_gate_depth) == 0, 1290 ("[amd,%d] nonzero gate depth on CPU %d", __LINE__, cpu)); 1291 KASSERT(atomic_load_acq_int(&pac->pc_global_mask) == 0, 1292 ("[amd,%d] nonzero desired mask on CPU %d", __LINE__, cpu)); 1293 KASSERT(atomic_load_acq_int(&pac->pc_virtual_mask) == 0, 1294 ("[amd,%d] nonzero virtual mask on CPU %d", __LINE__, cpu)); 1295 amd_v2_disable_all(); 1296 } 1297 amd_pcpu[cpu] = NULL; 1298 1299 #ifdef HWPMC_DEBUG 1300 for (i = 0; i < AMD_NPMCS_K8; i++) { 1301 KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, 1302 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 1303 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i), 1304 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 1305 } 1306 #endif 1307 1308 pc = pmc_pcpu[cpu]; 1309 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__)); 1310 1311 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8].pcd_ri; 1312 1313 /* 1314 * Reset pointers in the MI 'per-cpu' state. 1315 */ 1316 for (i = 0; i < amd_npmcs; i++) 1317 pc->pc_hwpmcs[i + first_ri] = NULL; 1318 1319 free(pac->pc_amdpmcs, M_PMC); 1320 free(pac, M_PMC); 1321 return (0); 1322 } 1323 1324 struct amd_v2_hwcheck_state { 1325 volatile u_int avh_read_error; 1326 volatile u_int avh_enabled; 1327 }; 1328 1329 static void 1330 amd_v2_hwcheck_cpu(void *arg) 1331 { 1332 struct amd_v2_hwcheck_state *state; 1333 uint64_t reg; 1334 int error, i; 1335 1336 state = arg; 1337 for (i = 0; i < amd_core_npmcs; i++) { 1338 error = rdmsr_safe(amd_pmcdesc[i].pm_evsel, ®); 1339 if (error != 0) { 1340 atomic_set_int(&state->avh_read_error, 1); 1341 continue; 1342 } 1343 if ((reg & AMD_PMC_ENABLE) != 0) 1344 atomic_set_int(&state->avh_enabled, 1); 1345 } 1346 } 1347 1348 /* 1349 * Check that the PMC hardware is safe to use. First, we check that the PMCs 1350 * are not in use by firmware or another module. Second, if none of the PMC 1351 * feature flags are set, we check that the event selector is working, because 1352 * virtual machines have no way to communicate the absence of PMCs. 1353 */ 1354 static int 1355 amd_hwcheck(void) 1356 { 1357 struct amd_v2_hwcheck_state state; 1358 uint64_t reg; 1359 int error, i; 1360 1361 if (amd_perfmon_v2) { 1362 state.avh_read_error = 0; 1363 state.avh_enabled = 0; 1364 smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier, 1365 amd_v2_hwcheck_cpu, smp_no_rendezvous_barrier, &state); 1366 if (state.avh_read_error != 0) { 1367 printf("hwpmc: AMD PerfMonV2 EVSEL read failed on one " 1368 "or more CPUs!\n"); 1369 return (-1); 1370 } 1371 if (state.avh_enabled != 0) { 1372 printf("hwpmc: PMCs maybe in use by firmware!\n"); 1373 printf("hwpmc: Disable the PMC use in the BIOS before " 1374 "loading\n"); 1375 return (-1); 1376 } 1377 return (0); 1378 } 1379 1380 /* 1381 * Some PC vendors enable the core counters in firmware to track 1382 * performance. The best guess is that this is being used to control 1383 * power management from within the SMM mode. We shouldn't just take 1384 * over the PMCs in this case. The user should try disabling any 1385 * performance monitoring or power management functions in the BIOS to 1386 * safely make use of the counters. 1387 */ 1388 for (i = 0; i < amd_core_npmcs; i++) { 1389 error = rdmsr_safe(amd_pmcdesc[i].pm_evsel, ®); 1390 if (error != 0) { 1391 printf("hwpmc: AMD evsel %d rdmsr failed!\n", i); 1392 return (-1); 1393 } 1394 1395 if ((reg & AMD_PMC_ENABLE) != 0) { 1396 printf("hwpmc: PMCs maybe in use by firmware!\n"); 1397 printf("hwpmc: Disable the PMC use in the BIOS before loading\n"); 1398 return (-1); 1399 } 1400 } 1401 1402 /* 1403 * Unfortunately, there is no way to communicate that the original four 1404 * core counters are disabled through CPUIDs alone. We attempt to 1405 * write and read back the MSR to validate that it is working. 1406 * 1407 * Referenced the BIOS and Kernel Developer Guide for AMD Athlon 64 and 1408 * AMD Opteron Processors 26094 Rev. 3.24 January, 2005 to ensure these 1409 * fields are valid. 1410 */ 1411 if ((amd_feature2 & AMDID2_PCXC) == 0) { 1412 error = wrmsr_safe(AMD_PMC_EVSEL_0, AMD_PMC_OS | AMD_PMC_USR); 1413 if (error != 0) { 1414 printf("hwpmc: AMD evsel 0 wrmsr failed!\n"); 1415 return (-1); 1416 } 1417 1418 error = rdmsr_safe(AMD_PMC_EVSEL_0, ®); 1419 if (error != 0) { 1420 printf("hwpmc: AMD evsel 0 rdmsr failed!\n"); 1421 return (-1); 1422 } 1423 1424 if (reg == 0) { 1425 printf("hwpmc: AMD evsel returned invalid value! " 1426 "You may be in a VM without PMC support.\n"); 1427 return (-1); 1428 } 1429 1430 wrmsr(AMD_PMC_EVSEL_0, 0); 1431 } 1432 1433 return (0); 1434 } 1435 1436 /* 1437 * Initialize ourselves. 1438 */ 1439 struct pmc_mdep * 1440 pmc_amd_initialize(void) 1441 { 1442 u_int regs[4]; 1443 struct amd_descr *d; 1444 struct pmc_classdep *pcd; 1445 struct pmc_mdep *pmc_mdep; 1446 enum pmc_cputype cputype; 1447 int ncpus, nclasses, i; 1448 int family, model, stepping; 1449 int npmcs_total; 1450 int error; 1451 int pmcs_per_umc; 1452 1453 /* 1454 * The presence of hardware performance counters on the AMD 1455 * Athlon, Duron or later processors, is _not_ indicated by 1456 * any of the processor feature flags set by the 'CPUID' 1457 * instruction, so we only check the 'instruction family' 1458 * field returned by CPUID for instruction family >= 6. 1459 */ 1460 1461 family = CPUID_TO_FAMILY(cpu_id); 1462 model = CPUID_TO_MODEL(cpu_id); 1463 stepping = CPUID_TO_STEPPING(cpu_id); 1464 1465 if (family == 0x18) 1466 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X", 1467 family, model, stepping); 1468 else 1469 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X", 1470 family, model, stepping); 1471 1472 switch (cpu_id & 0xF00) { 1473 case 0xF00: /* Athlon64/Opteron processor */ 1474 cputype = PMC_CPU_AMD_K8; 1475 break; 1476 default: 1477 printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, family, 1478 model); 1479 return (NULL); 1480 } 1481 1482 /* 1483 * From PPR for AMD Family 1Ah, a new cpuid leaf specifies the maximum 1484 * number of PMCs of each type. If we do not have that leaf, we use 1485 * the prior default values that are only valid if we have the feature 1486 * bit enabled in CPU. 1487 */ 1488 if ((amd_feature2 & AMDID2_PCXC) != 0) { 1489 amd_core_npmcs = AMD_PMC_CORE_DEFAULT; 1490 } else { 1491 amd_core_npmcs = AMD_NPMCS_K8; 1492 } 1493 amd_l3_npmcs = AMD_PMC_L3_DEFAULT; 1494 amd_df_npmcs = AMD_PMC_DF_DEFAULT; 1495 amd_umc_npmcs = 0; 1496 pmcs_per_umc = 0; 1497 1498 if (cpu_exthigh >= CPUID_EXTPERFMON) { 1499 do_cpuid(CPUID_EXTPERFMON, regs); 1500 if (regs[1] != 0) { 1501 amd_core_npmcs = EXTPERFMON_CORE_PMCS(regs[1]); 1502 amd_df_npmcs = EXTPERFMON_DF_PMCS(regs[1]); 1503 amd_umc_npmcs = EXTPERFMON_UMC_PMCS(regs[1]); 1504 pmcs_per_umc = amd_umc_npmcs / popcntq(regs[2]); 1505 } 1506 1507 /* EAX bit 0 holds the PerfMonV2 flag. */ 1508 if (EXTPERFMON_PERFMONV2(regs[0]) && family >= 0x19) 1509 amd_perfmon_v2 = true; 1510 } 1511 1512 /* 1513 * Normalize per-class counts against feature bits so that allocation, 1514 * registration, and amd_get_msr() row offsets all use the same values. 1515 * UMC counters have no CPUID feature flag; amd_umc_npmcs is 0 when 1516 * the CPUID leaf is absent. 1517 */ 1518 if ((amd_feature2 & AMDID2_PTSCEL2I) == 0) 1519 amd_l3_npmcs = 0; 1520 if ((amd_feature2 & AMDID2_PNXC) == 0) 1521 amd_df_npmcs = 0; 1522 npmcs_total = amd_core_npmcs + amd_l3_npmcs + amd_df_npmcs + 1523 amd_umc_npmcs; 1524 KASSERT(npmcs_total <= AMD_NPMCS_MAX, 1525 ("%s: npmcs_total %d exceeds AMD_NPMCS_MAX %d", 1526 __func__, npmcs_total, AMD_NPMCS_MAX)); 1527 amd_pmcdesc = mallocarray(npmcs_total, sizeof(*amd_pmcdesc), 1528 M_PMC, M_WAITOK | M_ZERO); 1529 1530 /* Enable the newer core counters */ 1531 for (i = 0; i < amd_core_npmcs; i++) { 1532 d = &amd_pmcdesc[i]; 1533 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1534 "K8-%d", i); 1535 d->pm_descr.pd_class = PMC_CLASS_K8; 1536 d->pm_descr.pd_caps = AMD_PMC_CAPS; 1537 /* 1538 * Zen 5 can precisely count retire events. 1539 * 1540 * Refer to PPR Vol 1 for AMD Family 1Ah Model 02h C1 57238 1541 * Rev. 0.24 September 29, 2024. 1542 */ 1543 if ((family >= 0x1a) && (i == 2)) 1544 d->pm_descr.pd_caps |= PMC_CAP_PRECISE; 1545 d->pm_descr.pd_width = 48; 1546 if ((amd_feature2 & AMDID2_PCXC) != 0) { 1547 d->pm_evsel = AMD_PMC_CORE_BASE + 2 * i; 1548 d->pm_perfctr = AMD_PMC_CORE_BASE + 2 * i + 1; 1549 } else { 1550 d->pm_evsel = AMD_PMC_EVSEL_0 + i; 1551 d->pm_perfctr = AMD_PMC_PERFCTR_0 + i; 1552 } 1553 d->pm_subclass = PMC_AMD_SUB_CLASS_CORE; 1554 } 1555 amd_npmcs = amd_core_npmcs; 1556 1557 if (amd_perfmon_v2) 1558 amd_global_cntr_mask = (1ULL << amd_core_npmcs) - 1; 1559 1560 if ((amd_feature2 & AMDID2_PTSCEL2I) != 0) { 1561 /* Enable the LLC/L3 counters */ 1562 for (i = 0; i < amd_l3_npmcs; i++) { 1563 d = &amd_pmcdesc[amd_npmcs + i]; 1564 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1565 "K8-L3-%d", i); 1566 d->pm_descr.pd_class = PMC_CLASS_K8; 1567 d->pm_descr.pd_caps = AMD_PMC_L3_CAPS; 1568 d->pm_descr.pd_width = 48; 1569 d->pm_evsel = AMD_PMC_L3_BASE + 2 * i; 1570 d->pm_perfctr = AMD_PMC_L3_BASE + 2 * i + 1; 1571 d->pm_subclass = PMC_AMD_SUB_CLASS_L3_CACHE; 1572 } 1573 amd_npmcs += amd_l3_npmcs; 1574 } 1575 1576 if ((amd_feature2 & AMDID2_PNXC) != 0) { 1577 /* Enable the data fabric counters */ 1578 for (i = 0; i < amd_df_npmcs; i++) { 1579 d = &amd_pmcdesc[amd_npmcs + i]; 1580 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1581 "K8-DF-%d", i); 1582 d->pm_descr.pd_class = PMC_CLASS_K8; 1583 d->pm_descr.pd_caps = AMD_PMC_DF_CAPS; 1584 d->pm_descr.pd_width = 48; 1585 d->pm_evsel = AMD_PMC_DF_BASE + 2 * i; 1586 d->pm_perfctr = AMD_PMC_DF_BASE + 2 * i + 1; 1587 d->pm_subclass = PMC_AMD_SUB_CLASS_DATA_FABRIC; 1588 } 1589 amd_npmcs += amd_df_npmcs; 1590 } 1591 1592 KASSERT(amd_npmcs == npmcs_total - amd_umc_npmcs, 1593 ("%s: UMC cursor wrong: got %d expected %d", 1594 __func__, amd_npmcs, npmcs_total - amd_umc_npmcs)); 1595 for (i = 0; i < amd_umc_npmcs; i++) { 1596 d = &amd_pmcdesc[amd_npmcs + i]; 1597 snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, 1598 "K8-UMC%d-%d", i / pmcs_per_umc, i); 1599 d->pm_descr.pd_class = PMC_CLASS_K8; 1600 d->pm_descr.pd_caps = AMD_PMC_UMC_CAPS; 1601 d->pm_descr.pd_width = 48; 1602 d->pm_evsel = AMD_PMC_UMC_BASE + 2 * i; 1603 d->pm_perfctr = AMD_PMC_UMC_BASE + 2 * i + 1; 1604 d->pm_subclass = PMC_AMD_SUB_CLASS_UMC; 1605 } 1606 amd_npmcs += amd_umc_npmcs; 1607 1608 KASSERT(amd_npmcs == npmcs_total, 1609 ("%s: descriptor cursor %d != npmcs_total %d", 1610 __func__, amd_npmcs, npmcs_total)); 1611 1612 /* 1613 * Sanity check that the hardware is safe to use. Do not read or write 1614 * any of the PMC MSRs until after this check passes. 1615 */ 1616 if (amd_hwcheck() < 0) { 1617 free(amd_pmcdesc, M_PMC); 1618 amd_pmcdesc = NULL; 1619 return (NULL); 1620 } 1621 1622 /* 1623 * Allocate space for pointers to PMC HW descriptors and for 1624 * the MDEP structure used by MI code. 1625 */ 1626 amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC, 1627 M_WAITOK | M_ZERO); 1628 1629 /* 1630 * These processors have two or three classes of PMCs: the TSC, 1631 * programmable PMCs, and AMD IBS. One extra class slot is reserved 1632 * for the optional RAPL energy counters. 1633 */ 1634 if ((amd_feature2 & AMDID2_IBS) != 0) { 1635 nclasses = 4; 1636 } else { 1637 nclasses = 3; 1638 } 1639 1640 /* 1641 * Detect support for MPERF and APERF MSRs. tsc_perf_stat is set by the 1642 * kernel's generic TSC initialization (start_TSC(), called at boot via 1643 * cpu_startup() -> startrtclock()), not by hwpmc's TSC PMC class. It is 1644 * set only after confirming both MSRs actually increment (some emulators 1645 * expose the CPUID bit without real MSR support). 1646 * 1647 * Separately, detect support for the IRPERF MSR (AMD-only). If the 1648 * feature is present, HWCR_IRPERFEN should already be set in MSR_HWCR 1649 * by initializecpu() at boot, which also accounts for erratum #1054 1650 * (IRPERF is left disabled on affected Zen 1 parts) and verifies IRPERF 1651 * is actually incrementing on a dead/emulated MSR. This code only reads 1652 * back that already-decided state. 1653 * 1654 * Either condition above causes the PERF class to be registered, so at 1655 * least one of MPERF/APERF or IRPERF is expected to be usable when this 1656 * branch is taken. 1657 */ 1658 if (((cpu_power_ecx & CPUID_PERF_STAT) && (tsc_perf_stat == 1)) || 1659 ((amd_extended_feature_extensions & AMDFEID_IRPERF) && 1660 (rdmsr(MSR_HWCR) & HWCR_IRPERFEN))) { 1661 nclasses++; 1662 } 1663 1664 pmc_mdep = pmc_mdep_alloc(nclasses); 1665 1666 ncpus = pmc_cpu_max(); 1667 1668 /* Initialize the TSC. */ 1669 error = pmc_tsc_initialize(pmc_mdep, ncpus); 1670 if (error != 0) 1671 goto error; 1672 1673 /* Initialize AMD K8 PMC handling. */ 1674 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8]; 1675 1676 pcd->pcd_caps = AMD_PMC_CAPS; 1677 pcd->pcd_class = PMC_CLASS_K8; 1678 pcd->pcd_num = amd_npmcs; 1679 pcd->pcd_ri = pmc_mdep->pmd_npmc; 1680 pcd->pcd_width = 48; 1681 1682 pcd->pcd_allocate_pmc = amd_allocate_pmc; 1683 pcd->pcd_config_pmc = amd_config_pmc; 1684 pcd->pcd_describe = amd_describe; 1685 pcd->pcd_get_config = amd_get_config; 1686 pcd->pcd_get_msr = amd_get_msr; 1687 pcd->pcd_pcpu_fini = amd_pcpu_fini; 1688 pcd->pcd_pcpu_init = amd_pcpu_init; 1689 pcd->pcd_read_pmc = amd_read_pmc; 1690 pcd->pcd_release_pmc = amd_release_pmc; 1691 pcd->pcd_start_pmc = amd_start_pmc; 1692 pcd->pcd_stop_pmc = amd_stop_pmc; 1693 pcd->pcd_write_pmc = amd_write_pmc; 1694 pcd->pcd_get_caps = amd_get_caps; 1695 1696 pmc_mdep->pmd_cputype = cputype; 1697 pmc_mdep->pmd_intr = amd_intr; 1698 pmc_mdep->pmd_switch_in = amd_switch_in; 1699 pmc_mdep->pmd_switch_out = amd_switch_out; 1700 1701 /* For v2, override core control. L3 and DF keep the classic path. */ 1702 if (amd_perfmon_v2) { 1703 pcd->pcd_start_pmc = amd_start_pmc_v2; 1704 pcd->pcd_stop_pmc = amd_stop_pmc_v2; 1705 pcd->pcd_start_all = amd_start_pmc_all_v2; 1706 pcd->pcd_stop_all = amd_stop_pmc_all_v2; 1707 pmc_mdep->pmd_intr = amd_intr_v2; 1708 } 1709 1710 pmc_mdep->pmd_npmc += amd_npmcs; 1711 1712 amd_init_policy(); 1713 1714 PMCDBG0(MDP, INI, 0, "amd-initialize"); 1715 1716 if ((amd_feature2 & AMDID2_IBS) != 0) { 1717 error = pmc_ibs_initialize(pmc_mdep, ncpus); 1718 if (error != 0) 1719 goto error; 1720 } 1721 1722 /* Initialize PERF class. */ 1723 pmc_perf_initialize(pmc_mdep, ncpus, nclasses - 1); 1724 1725 /* RAPL takes the reserved last slot; drop it if the probe fails. */ 1726 error = pmc_rapl_initialize(pmc_mdep, ncpus, pmc_mdep->pmd_nclass - 1); 1727 if (error != 0) 1728 pmc_mdep->pmd_nclass--; 1729 1730 return (pmc_mdep); 1731 1732 error: 1733 free(pmc_mdep, M_PMC); 1734 free(amd_pcpu, M_PMC); 1735 amd_pcpu = NULL; 1736 free(amd_pmcdesc, M_PMC); 1737 amd_pmcdesc = NULL; 1738 amd_npmcs = 0; 1739 return (NULL); 1740 } 1741 1742 /* 1743 * Finalization code for AMD CPUs. 1744 */ 1745 void 1746 pmc_amd_finalize(struct pmc_mdep *md) 1747 { 1748 PMCDBG0(MDP, INI, 1, "amd-finalize"); 1749 1750 /* Safe even if the RAPL class was skipped at initialize time. */ 1751 pmc_rapl_finalize(md); 1752 1753 pmc_tsc_finalize(md); 1754 1755 pmc_perf_finalize(md); 1756 1757 pmc_ibs_finalize(md); 1758 1759 for (int i = 0; i < pmc_cpu_max(); i++) 1760 KASSERT(amd_pcpu[i] == NULL, 1761 ("[amd,%d] non-null pcpu cpu %d", __LINE__, i)); 1762 1763 free(amd_pcpu, M_PMC); 1764 amd_pcpu = NULL; 1765 1766 free(amd_pmcdesc, M_PMC); 1767 amd_pmcdesc = NULL; 1768 } 1769