xref: /freebsd/sys/dev/hwpmc/hwpmc_core.c (revision 04e832672159cae412e8984e0b0cabfa6e7428b7)

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2008 Joseph Koshy
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * Intel Core PMCs.
 */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <sys/systm.h>

#include <machine/intr_machdep.h>
#include <x86/apicvar.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>

#define	CORE_CPUID_REQUEST		0xA
#define	CORE_CPUID_REQUEST_SIZE		0x4
#define	CORE_CPUID_EAX			0x0
#define	CORE_CPUID_EBX			0x1
#define	CORE_CPUID_ECX			0x2
#define	CORE_CPUID_EDX			0x3

#define	IAF_PMC_CAPS			\
	(PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
	 PMC_CAP_USER | PMC_CAP_SYSTEM)
#define	IAF_RI_TO_MSR(RI)		((RI) + (1 << 30))

#define	IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
    PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE |	 \
    PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)

#define	EV_IS_NOTARCH		0
#define	EV_IS_ARCH_SUPP		1
#define	EV_IS_ARCH_NOTSUPP	-1

/*
 * "Architectural" events defined by Intel.  The values of these
 * symbols correspond to positions in the bitmask returned by
 * the CPUID.0AH instruction.
 */
enum core_arch_events {
	CORE_AE_BRANCH_INSTRUCTION_RETIRED	= 5,
	CORE_AE_BRANCH_MISSES_RETIRED		= 6,
	CORE_AE_INSTRUCTION_RETIRED		= 1,
	CORE_AE_LLC_MISSES			= 4,
	CORE_AE_LLC_REFERENCE			= 3,
	CORE_AE_UNHALTED_REFERENCE_CYCLES	= 2,
	CORE_AE_UNHALTED_CORE_CYCLES		= 0
};

static enum pmc_cputype	core_cputype;
static int core_version;

struct core_cpu {
	volatile uint32_t	pc_iafctrl;	/* Fixed function control. */
	volatile uint64_t	pc_globalctrl;	/* Global control register. */
	struct pmc_hw		pc_corepmcs[];
};

static struct core_cpu **core_pcpu;

static uint32_t core_architectural_events;
static uint64_t core_pmcmask;

static int core_iaf_ri;		/* relative index of fixed counters */
static int core_iaf_width;
static int core_iaf_npmc;

static int core_iap_width;
static int core_iap_npmc;
static int core_iap_wroffset;

static u_int pmc_alloc_refs;
static bool pmc_tsx_force_abort_set;

static int
core_pcpu_noop(struct pmc_mdep *md, int cpu)
{
	(void) md;
	(void) cpu;
	return (0);
}

static int
core_pcpu_init(struct pmc_mdep *md, int cpu)
{
	struct pmc_cpu *pc;
	struct core_cpu *cc;
	struct pmc_hw *phw;
	int core_ri, n, npmc;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[iaf,%d] insane cpu number %d", __LINE__, cpu));

	PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);

	core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
	npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;

	if (core_version >= 2)
		npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;

	cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw),
	    M_PMC, M_WAITOK | M_ZERO);

	core_pcpu[cpu] = cc;
	pc = pmc_pcpu[cpu];

	KASSERT(pc != NULL && cc != NULL,
	    ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));

	for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
		phw->phw_state 	  = PMC_PHW_FLAG_IS_ENABLED |
		    PMC_PHW_CPU_TO_STATE(cpu) |
		    PMC_PHW_INDEX_TO_STATE(n + core_ri);
		phw->phw_pmc	  = NULL;
		pc->pc_hwpmcs[n + core_ri]  = phw;
	}

	if (core_version >= 2 && vm_guest == VM_GUEST_NO) {
		/* Enable Freezing PMCs on PMI. */
		wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) | 0x1000);
	}

	return (0);
}

static int
core_pcpu_fini(struct pmc_mdep *md, int cpu)
{
	int core_ri, n, npmc;
	struct pmc_cpu *pc;
	struct core_cpu *cc;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] insane cpu number (%d)", __LINE__, cpu));

	PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);

	if ((cc = core_pcpu[cpu]) == NULL)
		return (0);

	core_pcpu[cpu] = NULL;

	pc = pmc_pcpu[cpu];

	KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
		cpu));

	npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
	core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;

	for (n = 0; n < npmc; n++)
		wrmsr(IAP_EVSEL0 + n, 0);

	if (core_version >= 2) {
		wrmsr(IAF_CTRL, 0);
		npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
	}

	for (n = 0; n < npmc; n++)
		pc->pc_hwpmcs[n + core_ri] = NULL;

	free(cc, M_PMC);

	return (0);
}

/*
 * Fixed function counters.
 */

static pmc_value_t
iaf_perfctr_value_to_reload_count(pmc_value_t v)
{

	/* If the PMC has overflowed, return a reload count of zero. */
	if ((v & (1ULL << (core_iaf_width - 1))) == 0)
		return (0);
	v &= (1ULL << core_iaf_width) - 1;
	return (1ULL << core_iaf_width) - v;
}

static pmc_value_t
iaf_reload_count_to_perfctr_value(pmc_value_t rlc)
{
	return (1ULL << core_iaf_width) - rlc;
}

static int
iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
    const struct pmc_op_pmcallocate *a)
{
	uint8_t ev, umask;
	uint32_t caps;
	uint64_t config, flags;
	const struct pmc_md_iap_op_pmcallocate *iap;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU %d", __LINE__, cpu));

	PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);

	if (ri < 0 || ri > core_iaf_npmc)
		return (EINVAL);

	if (a->pm_class != PMC_CLASS_IAF)
		return (EINVAL);

	if ((a->pm_flags & PMC_F_EV_PMU) == 0)
		return (EINVAL);

	iap = &a->pm_md.pm_iap;
	config = iap->pm_iap_config;
	ev = IAP_EVSEL_GET(config);
	umask = IAP_UMASK_GET(config);

	if (ev == 0x0) {
		if (umask != ri + 1)
			return (EINVAL);
	} else {
		switch (ri) {
		case 0:	/* INST_RETIRED.ANY */
			if (ev != 0xC0 || umask != 0x00)
				return (EINVAL);
			break;
		case 1:	/* CPU_CLK_UNHALTED.THREAD */
			if (ev != 0x3C || umask != 0x00)
				return (EINVAL);
			break;
		case 2:	/* CPU_CLK_UNHALTED.REF */
			if (ev != 0x3C || umask != 0x01)
				return (EINVAL);
			break;
		case 3:	/* TOPDOWN.SLOTS */
			if (ev != 0xA4 || umask != 0x01)
				return (EINVAL);
			break;
		default:
			return (EINVAL);
		}
	}

	pmc_alloc_refs++;
	if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 &&
	    !pmc_tsx_force_abort_set) {
		pmc_tsx_force_abort_set = true;
		x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL |
		    MSR_OP_WRITE, 1, NULL);
	}

	flags = 0;
	if (config & IAP_OS)
		flags |= IAF_OS;
	if (config & IAP_USR)
		flags |= IAF_USR;
	if (config & IAP_ANY)
		flags |= IAF_ANY;
	if (config & IAP_INT)
		flags |= IAF_PMI;

	caps = a->pm_caps;
	if (caps & PMC_CAP_INTERRUPT)
		flags |= IAF_PMI;
	if (caps & PMC_CAP_SYSTEM)
		flags |= IAF_OS;
	if (caps & PMC_CAP_USER)
		flags |= IAF_USR;
	if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
		flags |= (IAF_OS | IAF_USR);

	pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));

	PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
	    (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);

	return (0);
}

static int
iaf_config_pmc(int cpu, int ri, struct pmc *pm)
{
	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU %d", __LINE__, cpu));

	KASSERT(ri >= 0 && ri < core_iaf_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);

	KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
	    cpu));

	core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;

	return (0);
}

static int
iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
	struct pmc_hw *phw;

	phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];

	snprintf(pi->pm_name, sizeof(pi->pm_name), "IAF-%d", ri);
	pi->pm_class = PMC_CLASS_IAF;

	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
		pi->pm_enabled = TRUE;
		*ppmc          = phw->phw_pmc;
	} else {
		pi->pm_enabled = FALSE;
		*ppmc          = NULL;
	}

	return (0);
}

static int
iaf_get_config(int cpu, int ri, struct pmc **ppm)
{
	*ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;

	return (0);
}

static int
iaf_get_msr(int ri, uint32_t *msr)
{
	KASSERT(ri >= 0 && ri < core_iaf_npmc,
	    ("[iaf,%d] ri %d out of range", __LINE__, ri));

	*msr = IAF_RI_TO_MSR(ri);

	return (0);
}

static int
iaf_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
{
	pmc_value_t tmp;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iaf_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	tmp = rdpmc(IAF_RI_TO_MSR(ri));

	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
		*v = iaf_perfctr_value_to_reload_count(tmp);
	else
		*v = tmp & ((1ULL << core_iaf_width) - 1);

	PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
	    IAF_RI_TO_MSR(ri), *v);

	return (0);
}

static int
iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
{
	PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iaf_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
	    ("[core,%d] PHW pmc non-NULL", __LINE__));

	MPASS(pmc_alloc_refs > 0);
	if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) {
		pmc_tsx_force_abort_set = false;
		x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL |
		    MSR_OP_WRITE, 0, NULL);
	}

	return (0);
}

static int
iaf_start_pmc(int cpu, int ri, struct pmc *pm)
{
	struct core_cpu *cc;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iaf_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);

	cc = core_pcpu[cpu];
	cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
	wrmsr(IAF_CTRL, cc->pc_iafctrl);

	cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
	wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);

	PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
	    cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
	    cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));

	return (0);
}

static int
iaf_stop_pmc(int cpu, int ri, struct pmc *pm)
{
	struct core_cpu *cc;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iaf_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri);

	cc = core_pcpu[cpu];

	cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4));
	wrmsr(IAF_CTRL, cc->pc_iafctrl);

	/* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */

	PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
	    cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
	    cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));

	return (0);
}

static int
iaf_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
{
	struct core_cpu *cc;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iaf_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	cc = core_pcpu[cpu];

	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
		v = iaf_reload_count_to_perfctr_value(v);

	/* Turn off the fixed counter */
	wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4)));

	wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));

	/* Turn on fixed counters */
	wrmsr(IAF_CTRL, cc->pc_iafctrl);

	PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
	    "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
	    (uintmax_t) rdmsr(IAF_CTRL),
	    (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));

	return (0);
}


static void
iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
{
	struct pmc_classdep *pcd;

	KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__));

	PMCDBG0(MDP,INI,1, "iaf-initialize");

	pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];

	pcd->pcd_caps	= IAF_PMC_CAPS;
	pcd->pcd_class	= PMC_CLASS_IAF;
	pcd->pcd_num	= npmc;
	pcd->pcd_ri	= md->pmd_npmc;
	pcd->pcd_width	= pmcwidth;

	pcd->pcd_allocate_pmc	= iaf_allocate_pmc;
	pcd->pcd_config_pmc	= iaf_config_pmc;
	pcd->pcd_describe	= iaf_describe;
	pcd->pcd_get_config	= iaf_get_config;
	pcd->pcd_get_msr	= iaf_get_msr;
	pcd->pcd_pcpu_fini	= core_pcpu_noop;
	pcd->pcd_pcpu_init	= core_pcpu_noop;
	pcd->pcd_read_pmc	= iaf_read_pmc;
	pcd->pcd_release_pmc	= iaf_release_pmc;
	pcd->pcd_start_pmc	= iaf_start_pmc;
	pcd->pcd_stop_pmc	= iaf_stop_pmc;
	pcd->pcd_write_pmc	= iaf_write_pmc;

	md->pmd_npmc	       += npmc;
}

/*
 * Intel programmable PMCs.
 */

/* Sub fields of UMASK that this event supports. */
#define	IAP_M_CORE		(1 << 0) /* Core specificity */
#define	IAP_M_AGENT		(1 << 1) /* Agent specificity */
#define	IAP_M_PREFETCH		(1 << 2) /* Prefetch */
#define	IAP_M_MESI		(1 << 3) /* MESI */
#define	IAP_M_SNOOPRESPONSE	(1 << 4) /* Snoop response */
#define	IAP_M_SNOOPTYPE		(1 << 5) /* Snoop type */
#define	IAP_M_TRANSITION	(1 << 6) /* Transition */

#define	IAP_F_CORE		(0x3 << 14) /* Core specificity */
#define	IAP_F_AGENT		(0x1 << 13) /* Agent specificity */
#define	IAP_F_PREFETCH		(0x3 << 12) /* Prefetch */
#define	IAP_F_MESI		(0xF <<  8) /* MESI */
#define	IAP_F_SNOOPRESPONSE	(0xB <<  8) /* Snoop response */
#define	IAP_F_SNOOPTYPE		(0x3 <<  8) /* Snoop type */
#define	IAP_F_TRANSITION	(0x1 << 12) /* Transition */

#define	IAP_PREFETCH_RESERVED	(0x2 << 12)
#define	IAP_CORE_THIS		(0x1 << 14)
#define	IAP_CORE_ALL		(0x3 << 14)
#define	IAP_F_CMASK		0xFF000000

static pmc_value_t
iap_perfctr_value_to_reload_count(pmc_value_t v)
{

	/* If the PMC has overflowed, return a reload count of zero. */
	if ((v & (1ULL << (core_iap_width - 1))) == 0)
		return (0);
	v &= (1ULL << core_iap_width) - 1;
	return (1ULL << core_iap_width) - v;
}

static pmc_value_t
iap_reload_count_to_perfctr_value(pmc_value_t rlc)
{
	return (1ULL << core_iap_width) - rlc;
}

static int
iap_pmc_has_overflowed(int ri)
{
	uint64_t v;

	/*
	 * We treat a Core (i.e., Intel architecture v1) PMC as has
	 * having overflowed if its MSB is zero.
	 */
	v = rdpmc(ri);
	return ((v & (1ULL << (core_iap_width - 1))) == 0);
}

static int
iap_event_corei7_ok_on_counter(uint8_t evsel, int ri)
{
	uint32_t mask;

	switch (evsel) {
	/* Events valid only on counter 0, 1. */
	case 0x40:
	case 0x41:
	case 0x42:
	case 0x43:
	case 0x4C:
	case 0x4E:
	case 0x51:
	case 0x52:
	case 0x53:
	case 0x63:
		mask = 0x3;
		break;
	/* Any row index is ok. */
	default:
		mask = ~0;
	}

	return (mask & (1 << ri));
}

static int
iap_event_westmere_ok_on_counter(uint8_t evsel, int ri)
{
	uint32_t mask;

	switch (evsel) {
	/* Events valid only on counter 0. */
	case 0x60:
	case 0xB3:
		mask = 0x1;
		break;

	/* Events valid only on counter 0, 1. */
	case 0x4C:
	case 0x4E:
	case 0x51:
	case 0x52:
	case 0x63:
		mask = 0x3;
		break;
	/* Any row index is ok. */
	default:
		mask = ~0;
	}

	return (mask & (1 << ri));
}

static int
iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri)
{
	uint32_t mask;

	switch (evsel) {
	/* Events valid only on counter 0. */
	case 0xB7:
		mask = 0x1;
		break;
	/* Events valid only on counter 1. */
	case 0xC0:
		mask = 0x2;
		break;
	/* Events valid only on counter 2. */
	case 0x48:
	case 0xA2:
	case 0xA3:
		mask = 0x4;
		break;
	/* Events valid only on counter 3. */
	case 0xBB:
	case 0xCD:
		mask = 0x8;
		break;
	/* Any row index is ok. */
	default:
		mask = ~0;
	}

	return (mask & (1 << ri));
}

static int
iap_event_core_ok_on_counter(uint8_t evsel, int ri)
{
	uint32_t mask;

	switch (evsel) {
		/*
		 * Events valid only on counter 0.
		 */
	case 0x10:
	case 0x14:
	case 0x18:
	case 0xB3:
	case 0xC1:
	case 0xCB:
		mask = (1 << 0);
		break;

		/*
		 * Events valid only on counter 1.
		 */
	case 0x11:
	case 0x12:
	case 0x13:
		mask = (1 << 1);
		break;

	default:
		mask = ~0;	/* Any row index is ok. */
	}

	return (mask & (1 << ri));
}

static int
iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
    const struct pmc_op_pmcallocate *a)
{
	uint8_t ev;
	const struct pmc_md_iap_op_pmcallocate *iap;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[core,%d] illegal row-index value %d", __LINE__, ri));

	if (a->pm_class != PMC_CLASS_IAP)
		return (EINVAL);

	if ((a->pm_flags & PMC_F_EV_PMU) == 0)
		return (EINVAL);

	iap = &a->pm_md.pm_iap;
	ev = IAP_EVSEL_GET(iap->pm_iap_config);

	switch (core_cputype) {
	case PMC_CPU_INTEL_CORE:
	case PMC_CPU_INTEL_CORE2:
	case PMC_CPU_INTEL_CORE2EXTREME:
		if (iap_event_core_ok_on_counter(ev, ri) == 0)
			return (EINVAL);
	case PMC_CPU_INTEL_COREI7:
	case PMC_CPU_INTEL_NEHALEM_EX:
		if (iap_event_corei7_ok_on_counter(ev, ri) == 0)
			return (EINVAL);
		break;
	case PMC_CPU_INTEL_WESTMERE:
	case PMC_CPU_INTEL_WESTMERE_EX:
		if (iap_event_westmere_ok_on_counter(ev, ri) == 0)
			return (EINVAL);
		break;
	case PMC_CPU_INTEL_SANDYBRIDGE:
	case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
	case PMC_CPU_INTEL_IVYBRIDGE:
	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
	case PMC_CPU_INTEL_HASWELL:
	case PMC_CPU_INTEL_HASWELL_XEON:
	case PMC_CPU_INTEL_BROADWELL:
	case PMC_CPU_INTEL_BROADWELL_XEON:
		if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
			return (EINVAL);
		break;
	case PMC_CPU_INTEL_ATOM:
	case PMC_CPU_INTEL_ATOM_SILVERMONT:
	case PMC_CPU_INTEL_ATOM_GOLDMONT:
	case PMC_CPU_INTEL_ATOM_GOLDMONT_P:
	case PMC_CPU_INTEL_ATOM_TREMONT:
	case PMC_CPU_INTEL_SKYLAKE:
	case PMC_CPU_INTEL_SKYLAKE_XEON:
	case PMC_CPU_INTEL_ICELAKE:
	case PMC_CPU_INTEL_ICELAKE_XEON:
	case PMC_CPU_INTEL_ALDERLAKE:
	default:
		break;
	}

	pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config;
	return (0);
}

static int
iap_config_pmc(int cpu, int ri, struct pmc *pm)
{
	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU %d", __LINE__, cpu));

	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);

	KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
	    cpu));

	core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;

	return (0);
}

static int
iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
	struct pmc_hw *phw;

	phw = &core_pcpu[cpu]->pc_corepmcs[ri];

	snprintf(pi->pm_name, sizeof(pi->pm_name), "IAP-%d", ri);
	pi->pm_class = PMC_CLASS_IAP;

	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
		pi->pm_enabled = TRUE;
		*ppmc          = phw->phw_pmc;
	} else {
		pi->pm_enabled = FALSE;
		*ppmc          = NULL;
	}

	return (0);
}

static int
iap_get_config(int cpu, int ri, struct pmc **ppm)
{
	*ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;

	return (0);
}

static int
iap_get_msr(int ri, uint32_t *msr)
{
	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[iap,%d] ri %d out of range", __LINE__, ri));

	*msr = ri;

	return (0);
}

static int
iap_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
{
	pmc_value_t tmp;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	tmp = rdpmc(ri);
	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
		*v = iap_perfctr_value_to_reload_count(tmp);
	else
		*v = tmp & ((1ULL << core_iap_width) - 1);

	PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
	    IAP_PMC0 + ri, *v);

	return (0);
}

static int
iap_release_pmc(int cpu, int ri, struct pmc *pm)
{
	(void) pm;

	PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
	    pm);

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
	    == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));

	return (0);
}

static int
iap_start_pmc(int cpu, int ri, struct pmc *pm)
{
	uint64_t evsel;
	struct core_cpu *cc;

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[core,%d] illegal row-index %d", __LINE__, ri));

	cc = core_pcpu[cpu];

	PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);

	evsel = pm->pm_md.pm_iap.pm_iap_evsel;

	PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
	    cpu, ri, IAP_EVSEL0 + ri, evsel);

	/* Event specific configuration. */

	switch (IAP_EVSEL_GET(evsel)) {
	case 0xB7:
		wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
		break;
	case 0xBB:
		wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
		break;
	default:
		break;
	}

	wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);

	if (core_version >= 2) {
		cc->pc_globalctrl |= (1ULL << ri);
		wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
	}

	return (0);
}

static int
iap_stop_pmc(int cpu, int ri, struct pmc *pm __unused)
{

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[core,%d] illegal row index %d", __LINE__, ri));

	PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);

	wrmsr(IAP_EVSEL0 + ri, 0);

	/* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */

	return (0);
}

static int
iap_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
{

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
	KASSERT(ri >= 0 && ri < core_iap_npmc,
	    ("[core,%d] illegal row index %d", __LINE__, ri));

	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
		v = iap_reload_count_to_perfctr_value(v);

	v &= (1ULL << core_iap_width) - 1;

	PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
	    IAP_PMC0 + ri, v);

	/*
	 * Write the new value to the counter (or it's alias).  The
	 * counter will be in a stopped state when the pcd_write()
	 * entry point is called.
	 */
	wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
	return (0);
}


static void
iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth,
    int flags)
{
	struct pmc_classdep *pcd;

	KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__));

	PMCDBG0(MDP,INI,1, "iap-initialize");

	/* Remember the set of architectural events supported. */
	core_architectural_events = ~flags;

	pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];

	pcd->pcd_caps	= IAP_PMC_CAPS;
	pcd->pcd_class	= PMC_CLASS_IAP;
	pcd->pcd_num	= npmc;
	pcd->pcd_ri	= md->pmd_npmc;
	pcd->pcd_width	= pmcwidth;

	pcd->pcd_allocate_pmc	= iap_allocate_pmc;
	pcd->pcd_config_pmc	= iap_config_pmc;
	pcd->pcd_describe	= iap_describe;
	pcd->pcd_get_config	= iap_get_config;
	pcd->pcd_get_msr	= iap_get_msr;
	pcd->pcd_pcpu_fini	= core_pcpu_fini;
	pcd->pcd_pcpu_init	= core_pcpu_init;
	pcd->pcd_read_pmc	= iap_read_pmc;
	pcd->pcd_release_pmc	= iap_release_pmc;
	pcd->pcd_start_pmc	= iap_start_pmc;
	pcd->pcd_stop_pmc	= iap_stop_pmc;
	pcd->pcd_write_pmc	= iap_write_pmc;

	md->pmd_npmc	       += npmc;
}

static int
core_intr(struct trapframe *tf)
{
	pmc_value_t v;
	struct pmc *pm;
	struct core_cpu *cc;
	int error, found_interrupt, ri;

	PMCDBG3(MDP,INT, 1, "cpu=%d tf=%p um=%d", curcpu, (void *) tf,
	    TRAPF_USERMODE(tf));

	found_interrupt = 0;
	cc = core_pcpu[curcpu];

	for (ri = 0; ri < core_iap_npmc; ri++) {

		if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
			continue;

		if (!iap_pmc_has_overflowed(ri))
			continue;

		found_interrupt = 1;

		if (pm->pm_state != PMC_STATE_RUNNING)
			continue;

		error = pmc_process_interrupt(PMC_HR, pm, tf);

		v = pm->pm_sc.pm_reloadcount;
		v = iap_reload_count_to_perfctr_value(v);

		/*
		 * Stop the counter, reload it but only restart it if
		 * the PMC is not stalled.
		 */
		wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel);
		wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);

		if (__predict_false(error))
			continue;

		wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN);
	}

	if (found_interrupt)
		counter_u64_add(pmc_stats.pm_intr_processed, 1);
	else
		counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	if (found_interrupt)
		lapic_reenable_pcint();

	return (found_interrupt);
}

static int
core2_intr(struct trapframe *tf)
{
	int error, found_interrupt = 0, n, cpu;
	uint64_t flag, intrstatus, intrdisable = 0;
	struct pmc *pm;
	struct core_cpu *cc;
	pmc_value_t v;

	cpu = curcpu;
	PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
	    TRAPF_USERMODE(tf));

	/*
	 * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which
	 * PMCs have a pending PMI interrupt.  We take a 'snapshot' of
	 * the current set of interrupting PMCs and process these
	 * after stopping them.
	 */
	intrstatus = rdmsr(IA_GLOBAL_STATUS);
	PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
	    (uintmax_t) intrstatus);

	/*
	 * Stop PMCs unless hardware already done it.
	 */
	if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0)
		wrmsr(IA_GLOBAL_CTRL, 0);

	cc = core_pcpu[cpu];
	KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));

	/*
	 * Look for interrupts from fixed function PMCs.
	 */
	for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;
	     n++, flag <<= 1) {

		if ((intrstatus & flag) == 0)
			continue;

		found_interrupt = 1;

		pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
		if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
			continue;

		error = pmc_process_interrupt(PMC_HR, pm, tf);
		if (__predict_false(error))
			intrdisable |= flag;

		v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);

		/* Reload sampling count. */
		wrmsr(IAF_CTR0 + n, v);

		PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu,
		    error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n)));
	}

	/*
	 * Process interrupts from the programmable counters.
	 */
	for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) {
		if ((intrstatus & flag) == 0)
			continue;

		found_interrupt = 1;

		pm = cc->pc_corepmcs[n].phw_pmc;
		if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
			continue;

		error = pmc_process_interrupt(PMC_HR, pm, tf);
		if (__predict_false(error))
			intrdisable |= flag;

		v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);

		PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
		    (uintmax_t) v);

		/* Reload sampling count. */
		wrmsr(core_iap_wroffset + IAP_PMC0 + n, v);
	}

	if (found_interrupt)
		counter_u64_add(pmc_stats.pm_intr_processed, 1);
	else
		counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	if (found_interrupt)
		lapic_reenable_pcint();

	/*
	 * Reenable all non-stalled PMCs.
	 */
	if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) {
		wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus);
		cc->pc_globalctrl &= ~intrdisable;
		wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
	} else {
		if (__predict_false(intrdisable)) {
			cc->pc_globalctrl &= ~intrdisable;
			wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
		}
		wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus);
	}

	PMCDBG4(MDP, INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx",
	    cpu, (uintmax_t) rdmsr(IAF_CTRL),
	    (uintmax_t) rdmsr(IA_GLOBAL_CTRL),
	    (uintmax_t) rdmsr(IA_GLOBAL_STATUS));

	return (found_interrupt);
}

int
pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override)
{
	int cpuid[CORE_CPUID_REQUEST_SIZE];
	int flags, nflags;

	do_cpuid(CORE_CPUID_REQUEST, cpuid);

	core_cputype = md->pmd_cputype;
	core_version = (version_override > 0) ? version_override :
	    cpuid[CORE_CPUID_EAX] & 0xFF;

	PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d version=%d",
	    core_cputype, maxcpu, core_version);

	if (core_version < 1 || core_version > 5 ||
	    (core_cputype != PMC_CPU_INTEL_CORE && core_version == 1)) {
		/* Unknown PMC architecture. */
		printf("hwpmc_core: unknown PMC architecture: %d\n",
		    core_version);
		return (EPROGMISMATCH);
	}

	core_iap_wroffset = 0;
	if (cpu_feature2 & CPUID2_PDCM) {
		if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) {
			PMCDBG0(MDP, INI, 1,
			    "core-init full-width write supported");
			core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0;
		} else
			PMCDBG0(MDP, INI, 1,
			    "core-init full-width write NOT supported");
	} else
		PMCDBG0(MDP, INI, 1, "core-init pdcm not supported");

	core_pmcmask = 0;

	/*
	 * Initialize programmable counters.
	 */
	core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF;
	core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF;

	core_pmcmask |= ((1ULL << core_iap_npmc) - 1);

	nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF;
	flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);

	iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags);

	/*
	 * Initialize fixed function counters, if present.
	 */
	if (core_version >= 2) {
		core_iaf_ri = core_iap_npmc;
		core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F;
		core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF;

		iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width);
		core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
	}

	PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
	    core_iaf_ri);

	core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC,
	    M_ZERO | M_WAITOK);

	/*
	 * Choose the appropriate interrupt handler.
	 */
	if (core_version >= 2)
		md->pmd_intr = core2_intr;
	else
		md->pmd_intr = core_intr;

	return (0);
}

void
pmc_core_finalize(struct pmc_mdep *md)
{
	PMCDBG0(MDP,INI,1, "core-finalize");

	for (int i = 0; i < pmc_cpu_max(); i++)
		KASSERT(core_pcpu[i] == NULL,
		    ("[core,%d] non-null pcpu cpu %d", __LINE__, i));

	free(core_pcpu, M_PMC);
	core_pcpu = NULL;
}