1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2026, Netflix, Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 /* Support for the AMD IBS */
29
30 #include <sys/param.h>
31 #include <sys/lock.h>
32 #include <sys/malloc.h>
33 #include <sys/mutex.h>
34 #include <sys/pcpu.h>
35 #include <sys/pmc.h>
36 #include <sys/pmckern.h>
37 #include <sys/pmclog.h>
38 #include <sys/smp.h>
39 #include <sys/sysctl.h>
40 #include <sys/systm.h>
41
42 #define EXTERR_CATEGORY EXTERR_CAT_HWPMC_IBS
43 #include <sys/exterrvar.h>
44
45 #include <machine/cpu.h>
46 #include <machine/cpufunc.h>
47 #include <machine/md_var.h>
48 #include <machine/specialreg.h>
49
50 #define IBS_STOP_ITER 50 /* Stopping iterations */
51
52 /* AMD IBS PMCs */
53 struct ibs_descr {
54 struct pmc_descr pm_descr; /* "base class" */
55 };
56
57 /*
58 * Globals
59 */
60 static uint64_t ibs_features;
61 static uint64_t ibs_fetch_allowed_mask;
62 static uint64_t ibs_op_allowed_mask;
63
64 static uint64_t ibs_fetch_extra_mask;
65 static uint64_t ibs_op_extra_mask;
66
67 SYSCTL_DECL(_kern_hwpmc);
68
69 SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_fetch_extra_mask, CTLFLAG_RDTUN,
70 &ibs_fetch_extra_mask, 0,
71 "Extra allowed bits in the IBS fetch control MSR (override; default 0)");
72
73 SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_op_extra_mask, CTLFLAG_RDTUN,
74 &ibs_op_extra_mask, 0,
75 "Extra allowed bits in the IBS op control MSR (override; default 0)");
76
77 /*
78 * Per-processor information
79 */
80 #define IBS_CPU_RUNNING 1
81 #define IBS_CPU_STOPPING 2
82 #define IBS_CPU_STOPPED 3
83
84 struct ibs_cpu {
85 int pc_status;
86 struct pmc_hw pc_ibspmcs[IBS_NPMCS];
87 };
88 static struct ibs_cpu **ibs_pcpu;
89
90 static void
ibs_init_policy(void)91 ibs_init_policy(void)
92 {
93
94 ibs_fetch_allowed_mask = IBS_FETCH_ALLOWED_MASK_BASE;
95
96 ibs_op_allowed_mask = IBS_OP_CTL_MAXCNTBASEMASK;
97
98 if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0)
99 ibs_fetch_allowed_mask |= IBS_FETCH_CTL_L3MISSONLY;
100
101 if ((ibs_features & CPUID_IBSID_OPCNT) != 0)
102 ibs_op_allowed_mask |= IBS_OP_CTL_COUNTERCONTROL;
103
104 if ((ibs_features & CPUID_IBSID_OPCNTEXT) != 0)
105 ibs_op_allowed_mask |= IBS_OP_CTL_MAXCNTEXTMASK;
106
107 if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0)
108 ibs_op_allowed_mask |= IBS_OP_CTL_L3MISSONLY;
109 }
110
111 static int
ibs_validate_fetch_config(uint64_t config)112 ibs_validate_fetch_config(uint64_t config)
113 {
114
115 if ((config & ~(ibs_fetch_allowed_mask | ibs_fetch_extra_mask)) != 0)
116 return (EINVAL);
117
118 return (0);
119 }
120
121 static int
ibs_validate_op_config(uint64_t config)122 ibs_validate_op_config(uint64_t config)
123 {
124 uint64_t allowed_mask;
125
126 allowed_mask = ibs_op_allowed_mask;
127
128 if ((config & IBS_OP_CTL_LATFLTEN) != 0) {
129 if ((ibs_features & CPUID_IBSID_IBSLOADLATENCYFILT) == 0)
130 return (EINVAL);
131 if ((config & IBS_OP_CTL_L3MISSONLY) == 0)
132 return (EINVAL);
133
134 allowed_mask |= IBS_OP_CTL_LDLATMASK | IBS_OP_CTL_L3MISSONLY;
135 }
136
137 allowed_mask |= ibs_op_extra_mask;
138
139 if ((config & ~allowed_mask) != 0)
140 return (EINVAL);
141
142 return (0);
143 }
144
145 static int
ibs_validate_pmc_config(int ri,uint64_t config)146 ibs_validate_pmc_config(int ri, uint64_t config)
147 {
148
149 switch (ri) {
150 case IBS_PMC_FETCH:
151 return (ibs_validate_fetch_config(config));
152 case IBS_PMC_OP:
153 return (ibs_validate_op_config(config));
154 default:
155 return (EINVAL);
156 }
157 }
158
159 /*
160 * Read a PMC value from the MSR.
161 */
162 static int
ibs_read_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t * v)163 ibs_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
164 {
165
166 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
167 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
168 KASSERT(ri >= 0 && ri < IBS_NPMCS,
169 ("[ibs,%d] illegal row-index %d", __LINE__, ri));
170 KASSERT(ibs_pcpu[cpu],
171 ("[ibs,%d] null per-cpu, cpu %d", __LINE__, cpu));
172
173 /* read the IBS count */
174 switch (ri) {
175 case IBS_PMC_FETCH:
176 *v = IBS_FETCH_CTL_TO_COUNT(rdmsr(IBS_FETCH_CTL));
177 break;
178 case IBS_PMC_OP:
179 *v = IBS_OP_CTL_TO_COUNT(rdmsr(IBS_OP_CTL));
180 break;
181 }
182
183 PMCDBG2(MDP, REA, 2, "ibs-read id=%d -> %jd", ri, *v);
184
185 return (0);
186 }
187
188 /*
189 * Write a PMC MSR.
190 */
191 static int
ibs_write_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t v)192 ibs_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
193 {
194 pmc_value_t m;
195
196 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
197 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
198 KASSERT(ri >= 0 && ri < IBS_NPMCS,
199 ("[ibs,%d] illegal row-index %d", __LINE__, ri));
200
201 /* write the IBS count */
202 switch (ri) {
203 case IBS_PMC_FETCH:
204 m = rdmsr(IBS_FETCH_CTL) & ~IBS_FETCH_CTL_CURCNTMASK;
205 /* Setting a count greater than interval is undefined. */
206 if (IBS_FETCH_CTL_TO_INTERVAL(m) > v)
207 m |= IBS_FETCH_COUNT_TO_CTL(v);
208 wrmsr(IBS_FETCH_CTL, m);
209 break;
210 case IBS_PMC_OP:
211 m = rdmsr(IBS_OP_CTL) & ~IBS_OP_CTL_CURCNTMASK;
212 /* Setting a count greater than interval is undefined */
213 if (IBS_OP_CTL_TO_INTERVAL(m) > v)
214 m |= IBS_OP_COUNT_TO_CTL(v);
215 wrmsr(IBS_OP_CTL, m);
216 break;
217 }
218
219 PMCDBG3(MDP, WRI, 1, "ibs-write cpu=%d ri=%d v=%jx", cpu, ri, v);
220
221 return (0);
222 }
223
224 /*
225 * Configure hardware PMC according to the configuration recorded in 'pm'.
226 */
227 static int
ibs_config_pmc(int cpu,int ri,struct pmc * pm)228 ibs_config_pmc(int cpu, int ri, struct pmc *pm)
229 {
230 struct pmc_hw *phw;
231
232 PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
233
234 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
235 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
236 KASSERT(ri >= 0 && ri < IBS_NPMCS,
237 ("[ibs,%d] illegal row-index %d", __LINE__, ri));
238
239 phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri];
240
241 KASSERT(pm == NULL || phw->phw_pmc == NULL,
242 ("[ibs,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
243 __LINE__, pm, phw->phw_pmc));
244
245 phw->phw_pmc = pm;
246
247 return (0);
248 }
249
250 /*
251 * Retrieve a configured PMC pointer from hardware state.
252 */
253 static int
ibs_get_config(int cpu,int ri,struct pmc ** ppm)254 ibs_get_config(int cpu, int ri, struct pmc **ppm)
255 {
256
257 *ppm = ibs_pcpu[cpu]->pc_ibspmcs[ri].phw_pmc;
258
259 return (0);
260 }
261
262 /*
263 * Check if a given PMC allocation is feasible.
264 */
265 static int
ibs_allocate_pmc(int cpu __unused,int ri,struct pmc * pm,const struct pmc_op_pmcallocate * a)266 ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
267 const struct pmc_op_pmcallocate *a)
268 {
269 uint64_t caps, config;
270 int error;
271
272 KASSERT(ri >= 0 && ri < IBS_NPMCS,
273 ("[ibs,%d] illegal row index %d", __LINE__, ri));
274
275 /* check class match */
276 if (a->pm_class != PMC_CLASS_IBS)
277 return (EXTERROR(EINVAL, "PMC class is not IBS"));
278 if (a->pm_md.pm_ibs.ibs_type != ri)
279 return (EXTERROR(EINVAL,
280 "IBS type %ju does not match PMC index %ju",
281 (uint64_t)a->pm_md.pm_ibs.ibs_type, (uint64_t)ri));
282
283 caps = pm->pm_caps;
284
285 PMCDBG2(MDP, ALL, 1, "ibs-allocate ri=%d caps=0x%x", ri, caps);
286
287 if ((caps & PMC_CAP_SYSTEM) == 0)
288 return (EXTERROR(EINVAL, "IBS requires SYSTEM capability"));
289
290 if (!PMC_IS_SAMPLING_MODE(a->pm_mode))
291 return (EINVAL);
292
293 config = a->pm_md.pm_ibs.ibs_ctl;
294 error = ibs_validate_pmc_config(ri, config);
295 if (error != 0)
296 return (error);
297 pm->pm_md.pm_ibs.ibs_ctl = config;
298
299 PMCDBG2(MDP, ALL, 2, "ibs-allocate ri=%d -> config=0x%jx", ri,
300 config);
301
302 return (0);
303 }
304
305 /*
306 * Release machine dependent state associated with a PMC. This is a
307 * no-op on this architecture.
308 */
309 static int
ibs_release_pmc(int cpu,int ri,struct pmc * pmc __unused)310 ibs_release_pmc(int cpu, int ri, struct pmc *pmc __unused)
311 {
312 struct pmc_hw *phw __diagused;
313
314 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
315 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
316 KASSERT(ri >= 0 && ri < IBS_NPMCS,
317 ("[ibs,%d] illegal row-index %d", __LINE__, ri));
318
319 PMCDBG1(MDP, ALL, 1, "ibs-release ri=%d", ri);
320
321 phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri];
322
323 KASSERT(phw->phw_pmc == NULL,
324 ("[ibs,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
325
326 return (0);
327 }
328
329 /*
330 * Start a PMC.
331 */
332 static int
ibs_start_pmc(int cpu __diagused,int ri,struct pmc * pm)333 ibs_start_pmc(int cpu __diagused, int ri, struct pmc *pm)
334 {
335 uint64_t config;
336
337 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
338 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
339 KASSERT(ri >= 0 && ri < IBS_NPMCS,
340 ("[ibs,%d] illegal row-index %d", __LINE__, ri));
341
342 PMCDBG2(MDP, STA, 1, "ibs-start cpu=%d ri=%d", cpu, ri);
343
344 /*
345 * This is used to handle spurious NMIs. All that matters is that it
346 * is not in the stopping state.
347 */
348 atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_RUNNING);
349
350 /*
351 * Turn on the ENABLE bit. Zeroing out the control register eliminates
352 * stale valid bits from spurious NMIs and it resets the counter.
353 */
354 switch (ri) {
355 case IBS_PMC_FETCH:
356 wrmsr(IBS_FETCH_CTL, 0);
357 config = pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE;
358 wrmsr(IBS_FETCH_CTL, config);
359 break;
360 case IBS_PMC_OP:
361 wrmsr(IBS_OP_CTL, 0);
362 config = pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE;
363 wrmsr(IBS_OP_CTL, config);
364 break;
365 }
366
367 return (0);
368 }
369
370 /*
371 * Stop a PMC.
372 */
373 static int
ibs_stop_pmc(int cpu __diagused,int ri,struct pmc * pm)374 ibs_stop_pmc(int cpu __diagused, int ri, struct pmc *pm)
375 {
376 int i;
377 uint64_t config;
378
379 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
380 ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
381 KASSERT(ri >= 0 && ri < IBS_NPMCS,
382 ("[ibs,%d] illegal row-index %d", __LINE__, ri));
383
384 PMCDBG1(MDP, STO, 1, "ibs-stop ri=%d", ri);
385
386 /*
387 * Turn off the ENABLE bit, but unfortunately there are a few quirks
388 * that generate excess NMIs. Workaround #420 in the Revision Guide
389 * for AMD Family 10h Processors 41322 Rev. 3.92 March 2012. requires
390 * that we clear the max count before clearing enable.
391 *
392 * Even after clearing the counter spurious NMIs are still possible so
393 * we use a per-CPU atomic variable to notify the interrupt handler we
394 * are stopping and discard spurious NMIs. We then retry clearing the
395 * control register for 50us. This gives us enough time and ensures
396 * that the valid bit is not accidently stuck after a spurious NMI.
397 */
398 config = pm->pm_md.pm_ibs.ibs_ctl;
399
400 atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPING);
401
402 switch (ri) {
403 case IBS_PMC_FETCH:
404 wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_MAXCNTMASK);
405 DELAY(1);
406 config &= ~IBS_FETCH_CTL_ENABLE;
407 wrmsr(IBS_FETCH_CTL, config);
408 break;
409 case IBS_PMC_OP:
410 wrmsr(IBS_OP_CTL, config & ~IBS_OP_CTL_MAXCNTMASK);
411 DELAY(1);
412 config &= ~IBS_OP_CTL_ENABLE;
413 wrmsr(IBS_OP_CTL, config);
414 break;
415 }
416
417 for (i = 0; i < IBS_STOP_ITER; i++) {
418 DELAY(1);
419
420 switch (ri) {
421 case IBS_PMC_FETCH:
422 wrmsr(IBS_FETCH_CTL, 0);
423 break;
424 case IBS_PMC_OP:
425 wrmsr(IBS_OP_CTL, 0);
426 break;
427 }
428 }
429
430 atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPED);
431
432 return (0);
433 }
434
435 static void
pmc_ibs_process_fetch(struct pmc * pm,struct trapframe * tf,uint64_t config)436 pmc_ibs_process_fetch(struct pmc *pm, struct trapframe *tf, uint64_t config)
437 {
438 struct pmc_multipart mpd;
439
440 if (pm == NULL)
441 return;
442
443 if (pm->pm_state != PMC_STATE_RUNNING)
444 return;
445
446 memset(&mpd, 0, sizeof(mpd));
447
448 mpd.pl_type = PMC_CC_MULTIPART_IBS_FETCH;
449 mpd.pl_length = PMC_MPIDX_FETCH_MAX;
450 mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config;
451 if ((ibs_features & CPUID_IBSID_IBSFETCHCTLEXTD) != 0) {
452 mpd.pl_mpdata[PMC_MPIDX_FETCH_EXTCTL] = rdmsr(IBS_FETCH_EXTCTL);
453 }
454 mpd.pl_mpdata[PMC_MPIDX_FETCH_LINADDR] = rdmsr(IBS_FETCH_LINADDR);
455 if ((config & IBS_FETCH_CTL_PHYSADDRVALID) != 0) {
456 mpd.pl_mpdata[PMC_MPIDX_FETCH_PHYSADDR] =
457 rdmsr(IBS_FETCH_PHYSADDR);
458 }
459
460 pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd);
461
462 wrmsr(IBS_FETCH_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE);
463 }
464
465 static void
pmc_ibs_process_op(struct pmc * pm,struct trapframe * tf,uint64_t config)466 pmc_ibs_process_op(struct pmc *pm, struct trapframe *tf, uint64_t config)
467 {
468 struct pmc_multipart mpd;
469
470 if (pm == NULL)
471 return;
472
473 if (pm->pm_state != PMC_STATE_RUNNING)
474 return;
475
476 memset(&mpd, 0, sizeof(mpd));
477
478 mpd.pl_type = PMC_CC_MULTIPART_IBS_OP;
479 mpd.pl_length = PMC_MPIDX_OP_MAX;
480 mpd.pl_mpdata[PMC_MPIDX_OP_CTL] = config;
481 mpd.pl_mpdata[PMC_MPIDX_OP_RIP] = rdmsr(IBS_OP_RIP);
482 mpd.pl_mpdata[PMC_MPIDX_OP_DATA] = rdmsr(IBS_OP_DATA);
483 mpd.pl_mpdata[PMC_MPIDX_OP_DATA2] = rdmsr(IBS_OP_DATA2);
484 mpd.pl_mpdata[PMC_MPIDX_OP_DATA3] = rdmsr(IBS_OP_DATA3);
485 mpd.pl_mpdata[PMC_MPIDX_OP_DC_LINADDR] = rdmsr(IBS_OP_DC_LINADDR);
486 mpd.pl_mpdata[PMC_MPIDX_OP_DC_PHYSADDR] = rdmsr(IBS_OP_DC_PHYSADDR);
487 if ((ibs_features & CPUID_IBSID_BRNTRGT) != 0) {
488 mpd.pl_mpdata[PMC_MPIDX_OP_TGT_RIP] = rdmsr(IBS_OP_TGT_RIP);
489 }
490 if ((ibs_features & CPUID_IBSID_IBSOPDATA4) != 0) {
491 mpd.pl_mpdata[PMC_MPIDX_OP_DATA4] = rdmsr(IBS_OP_DATA4);
492 }
493
494 pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd);
495
496 wrmsr(IBS_OP_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE);
497 }
498
499 /*
500 * Interrupt handler. This function needs to return '1' if the
501 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed
502 * to sleep or do anything a 'fast' interrupt handler is not allowed
503 * to do.
504 */
505 int
pmc_ibs_intr(struct trapframe * tf)506 pmc_ibs_intr(struct trapframe *tf)
507 {
508 struct ibs_cpu *pac;
509 struct pmc *pm;
510 int retval, cpu;
511 uint64_t config;
512
513 if (ibs_pcpu == NULL)
514 return (0);
515
516 cpu = curcpu;
517 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
518 ("[ibs,%d] out of range CPU %d", __LINE__, cpu));
519
520 PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf));
521
522 retval = 0;
523
524 pac = ibs_pcpu[cpu];
525
526 config = rdmsr(IBS_FETCH_CTL);
527 if ((config & IBS_FETCH_CTL_VALID) != 0) {
528 pm = pac->pc_ibspmcs[IBS_PMC_FETCH].phw_pmc;
529
530 retval = 1;
531
532 pmc_ibs_process_fetch(pm, tf, config);
533 }
534
535 config = rdmsr(IBS_OP_CTL);
536 if ((retval == 0) && ((config & IBS_OP_CTL_VALID) != 0)) {
537 pm = pac->pc_ibspmcs[IBS_PMC_OP].phw_pmc;
538
539 retval = 1;
540
541 pmc_ibs_process_op(pm, tf, config);
542 }
543
544 if (retval == 0) {
545 // Lets check for a stray NMI when stopping
546 if (atomic_load_int(&pac->pc_status) == IBS_CPU_STOPPING) {
547 return (1);
548 }
549 }
550
551
552 if (retval)
553 counter_u64_add(pmc_stats.pm_intr_processed, 1);
554 else
555 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
556
557 PMCDBG1(MDP, INT, 2, "retval=%d", retval);
558
559 return (retval);
560 }
561
562 /*
563 * Describe a PMC.
564 */
565 static int
ibs_describe(int cpu,int ri,struct pmc_info * pi,struct pmc ** ppmc)566 ibs_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
567 {
568 struct pmc_hw *phw;
569
570 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
571 ("[ibs,%d] illegal CPU %d", __LINE__, cpu));
572 KASSERT(ri >= 0 && ri < IBS_NPMCS,
573 ("[ibs,%d] row-index %d out of range", __LINE__, ri));
574
575 phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri];
576
577 if (ri == IBS_PMC_FETCH) {
578 strlcpy(pi->pm_name, "IBS-FETCH", sizeof(pi->pm_name));
579 pi->pm_class = PMC_CLASS_IBS;
580 pi->pm_enabled = true;
581 *ppmc = phw->phw_pmc;
582 } else {
583 strlcpy(pi->pm_name, "IBS-OP", sizeof(pi->pm_name));
584 pi->pm_class = PMC_CLASS_IBS;
585 pi->pm_enabled = true;
586 *ppmc = phw->phw_pmc;
587 }
588
589 return (0);
590 }
591
592 /*
593 * Processor-dependent initialization.
594 */
595 static int
ibs_pcpu_init(struct pmc_mdep * md,int cpu)596 ibs_pcpu_init(struct pmc_mdep *md, int cpu)
597 {
598 struct ibs_cpu *pac;
599 struct pmc_cpu *pc;
600 struct pmc_hw *phw;
601 int first_ri, n;
602
603 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
604 ("[ibs,%d] insane cpu number %d", __LINE__, cpu));
605
606 PMCDBG1(MDP, INI, 1, "ibs-init cpu=%d", cpu);
607
608 ibs_pcpu[cpu] = pac = malloc(sizeof(struct ibs_cpu), M_PMC,
609 M_WAITOK | M_ZERO);
610
611 /*
612 * Set the content of the hardware descriptors to a known
613 * state and initialize pointers in the MI per-cpu descriptor.
614 */
615 pc = pmc_pcpu[cpu];
616 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri;
617
618 KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu pointer", __LINE__));
619
620 for (n = 0, phw = pac->pc_ibspmcs; n < IBS_NPMCS; n++, phw++) {
621 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
622 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
623 phw->phw_pmc = NULL;
624 pc->pc_hwpmcs[n + first_ri] = phw;
625 }
626
627 return (0);
628 }
629
630 /*
631 * Processor-dependent cleanup prior to the KLD being unloaded.
632 */
633 static int
ibs_pcpu_fini(struct pmc_mdep * md,int cpu)634 ibs_pcpu_fini(struct pmc_mdep *md, int cpu)
635 {
636 struct ibs_cpu *pac;
637 struct pmc_cpu *pc;
638 int first_ri, i;
639
640 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
641 ("[ibs,%d] insane cpu number (%d)", __LINE__, cpu));
642
643 PMCDBG1(MDP, INI, 1, "ibs-cleanup cpu=%d", cpu);
644
645 /*
646 * Turn off IBS.
647 */
648 wrmsr(IBS_FETCH_CTL, 0);
649 wrmsr(IBS_OP_CTL, 0);
650
651 /*
652 * Free up allocated space.
653 */
654 if ((pac = ibs_pcpu[cpu]) == NULL)
655 return (0);
656
657 ibs_pcpu[cpu] = NULL;
658
659 pc = pmc_pcpu[cpu];
660 KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu state", __LINE__));
661
662 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri;
663
664 /*
665 * Reset pointers in the MI 'per-cpu' state.
666 */
667 for (i = 0; i < IBS_NPMCS; i++)
668 pc->pc_hwpmcs[i + first_ri] = NULL;
669
670 free(pac, M_PMC);
671
672 return (0);
673 }
674
675 /*
676 * Initialize ourselves.
677 */
678 int
pmc_ibs_initialize(struct pmc_mdep * pmc_mdep,int ncpus)679 pmc_ibs_initialize(struct pmc_mdep *pmc_mdep, int ncpus)
680 {
681 u_int regs[4];
682 struct pmc_classdep *pcd;
683
684 /*
685 * Allocate space for pointers to PMC HW descriptors and for
686 * the MDEP structure used by MI code.
687 */
688 ibs_pcpu = malloc(sizeof(struct ibs_cpu *) * pmc_cpu_max(), M_PMC,
689 M_WAITOK | M_ZERO);
690
691 /* Initialize AMD IBS handling. */
692 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS];
693
694 pcd->pcd_caps = IBS_PMC_CAPS;
695 pcd->pcd_class = PMC_CLASS_IBS;
696 pcd->pcd_num = IBS_NPMCS;
697 pcd->pcd_ri = pmc_mdep->pmd_npmc;
698 pcd->pcd_width = 0;
699
700 pcd->pcd_allocate_pmc = ibs_allocate_pmc;
701 pcd->pcd_config_pmc = ibs_config_pmc;
702 pcd->pcd_describe = ibs_describe;
703 pcd->pcd_get_config = ibs_get_config;
704 pcd->pcd_pcpu_fini = ibs_pcpu_fini;
705 pcd->pcd_pcpu_init = ibs_pcpu_init;
706 pcd->pcd_release_pmc = ibs_release_pmc;
707 pcd->pcd_start_pmc = ibs_start_pmc;
708 pcd->pcd_stop_pmc = ibs_stop_pmc;
709 pcd->pcd_read_pmc = ibs_read_pmc;
710 pcd->pcd_write_pmc = ibs_write_pmc;
711
712 pmc_mdep->pmd_npmc += IBS_NPMCS;
713
714 if (cpu_exthigh >= CPUID_IBSID) {
715 do_cpuid(CPUID_IBSID, regs);
716 ibs_features = regs[0];
717 if ((ibs_features & CPUID_IBSID_IBSFFV) == 0)
718 ibs_features = 0;
719 } else {
720 ibs_features = 0;
721 }
722
723 ibs_init_policy();
724
725 PMCDBG0(MDP, INI, 0, "ibs-initialize");
726
727 return (0);
728 }
729
730 /*
731 * Finalization code for AMD CPUs.
732 */
733 void
pmc_ibs_finalize(struct pmc_mdep * md)734 pmc_ibs_finalize(struct pmc_mdep *md)
735 {
736 PMCDBG0(MDP, INI, 1, "ibs-finalize");
737
738 for (int i = 0; i < pmc_cpu_max(); i++)
739 KASSERT(ibs_pcpu[i] == NULL,
740 ("[ibs,%d] non-null pcpu cpu %d", __LINE__, i));
741
742 free(ibs_pcpu, M_PMC);
743 ibs_pcpu = NULL;
744 }
745