57 #define	IAF_RI_TO_MSR(RI)		((RI) + (1 << 30))  argument
222 iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,  in iaf_allocate_pmc()  argument
233 	PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);  in iaf_allocate_pmc()
235 	if (ri < 0 || ri > core_iaf_npmc)  in iaf_allocate_pmc()
250 		if (umask != ri + 1)  in iaf_allocate_pmc()
253 		switch (ri) {  in iaf_allocate_pmc()
303 	pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));  in iaf_allocate_pmc()
312 iaf_config_pmc(int cpu, int ri, struct pmc *pm)  in iaf_config_pmc()  argument
317 	KASSERT(ri >= 0 && ri < core_iaf_npmc,  in iaf_config_pmc()
318 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iaf_config_pmc()
320 	PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);  in iaf_config_pmc()
325 	core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;  in iaf_config_pmc()
331 iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)  in iaf_describe()  argument
335 	phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];  in iaf_describe()
337 	snprintf(pi->pm_name, sizeof(pi->pm_name), "IAF-%d", ri);  in iaf_describe()
352 iaf_get_config(int cpu, int ri, struct pmc **ppm)  in iaf_get_config()  argument
354 	*ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;  in iaf_get_config()
360 iaf_get_msr(int ri, uint32_t *msr)  in iaf_get_msr()  argument
362 	KASSERT(ri >= 0 && ri < core_iaf_npmc,  in iaf_get_msr()
363 	    ("[iaf,%d] ri %d out of range", __LINE__, ri));  in iaf_get_msr()
365 	*msr = IAF_RI_TO_MSR(ri);  in iaf_get_msr()
371 iaf_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)  in iaf_read_pmc()  argument
377 	KASSERT(ri >= 0 && ri < core_iaf_npmc,  in iaf_read_pmc()
378 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iaf_read_pmc()
380 	tmp = rdpmc(IAF_RI_TO_MSR(ri));  in iaf_read_pmc()
387 	PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,  in iaf_read_pmc()
388 	    IAF_RI_TO_MSR(ri), *v);  in iaf_read_pmc()
394 iaf_release_pmc(int cpu, int ri, struct pmc *pmc)  in iaf_release_pmc()  argument
396 	PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);  in iaf_release_pmc()
400 	KASSERT(ri >= 0 && ri < core_iaf_npmc,  in iaf_release_pmc()
401 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iaf_release_pmc()
403 	KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,  in iaf_release_pmc()
417 iaf_start_pmc(int cpu, int ri, struct pmc *pm)  in iaf_start_pmc()  argument
423 	KASSERT(ri >= 0 && ri < core_iaf_npmc,  in iaf_start_pmc()
424 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iaf_start_pmc()
426 	PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);  in iaf_start_pmc()
432 	cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));  in iaf_start_pmc()
443 iaf_stop_pmc(int cpu, int ri, struct pmc *pm)  in iaf_stop_pmc()  argument
449 	KASSERT(ri >= 0 && ri < core_iaf_npmc,  in iaf_stop_pmc()
450 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iaf_stop_pmc()
452 	PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri);  in iaf_stop_pmc()
456 	cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4));  in iaf_stop_pmc()
469 iaf_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)  in iaf_write_pmc()  argument
475 	KASSERT(ri >= 0 && ri < core_iaf_npmc,  in iaf_write_pmc()
476 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iaf_write_pmc()
484 	wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4)));  in iaf_write_pmc()
486 	wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));  in iaf_write_pmc()
491 	PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "  in iaf_write_pmc()
492 	    "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,  in iaf_write_pmc()
494 	    (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));  in iaf_write_pmc()
577 iap_pmc_has_overflowed(int ri)  in iap_pmc_has_overflowed()  argument
585 	v = rdpmc(ri);  in iap_pmc_has_overflowed()
590 iap_event_corei7_ok_on_counter(uint8_t evsel, int ri)  in iap_event_corei7_ok_on_counter()  argument
613 	return (mask & (1 << ri));  in iap_event_corei7_ok_on_counter()
617 iap_event_westmere_ok_on_counter(uint8_t evsel, int ri)  in iap_event_westmere_ok_on_counter()  argument
641 	return (mask & (1 << ri));  in iap_event_westmere_ok_on_counter()
645 iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri)  in iap_event_sb_sbx_ib_ibx_ok_on_counter()  argument
674 	return (mask & (1 << ri));  in iap_event_sb_sbx_ib_ibx_ok_on_counter()
678 iap_event_core_ok_on_counter(uint8_t evsel, int ri)  in iap_event_core_ok_on_counter()  argument
708 	return (mask & (1 << ri));  in iap_event_core_ok_on_counter()
712 iap_allocate_pmc(int cpu, int ri, struct pmc *pm,  in iap_allocate_pmc()  argument
720 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_allocate_pmc()
721 	    ("[core,%d] illegal row-index value %d", __LINE__, ri));  in iap_allocate_pmc()
736 		if (iap_event_core_ok_on_counter(ev, ri) == 0)  in iap_allocate_pmc()
740 		if (iap_event_corei7_ok_on_counter(ev, ri) == 0)  in iap_allocate_pmc()
745 		if (iap_event_westmere_ok_on_counter(ev, ri) == 0)  in iap_allocate_pmc()
756 		if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)  in iap_allocate_pmc()
778 iap_config_pmc(int cpu, int ri, struct pmc *pm)  in iap_config_pmc()  argument
783 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_config_pmc()
784 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iap_config_pmc()
786 	PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);  in iap_config_pmc()
791 	core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;  in iap_config_pmc()
797 iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)  in iap_describe()  argument
801 	phw = &core_pcpu[cpu]->pc_corepmcs[ri];  in iap_describe()
803 	snprintf(pi->pm_name, sizeof(pi->pm_name), "IAP-%d", ri);  in iap_describe()
818 iap_get_config(int cpu, int ri, struct pmc **ppm)  in iap_get_config()  argument
820 	*ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;  in iap_get_config()
826 iap_get_msr(int ri, uint32_t *msr)  in iap_get_msr()  argument
828 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_get_msr()
829 	    ("[iap,%d] ri %d out of range", __LINE__, ri));  in iap_get_msr()
831 	*msr = ri;  in iap_get_msr()
837 iap_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)  in iap_read_pmc()  argument
843 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_read_pmc()
844 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iap_read_pmc()
846 	tmp = rdpmc(ri);  in iap_read_pmc()
852 	PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,  in iap_read_pmc()
853 	    IAP_PMC0 + ri, *v);  in iap_read_pmc()
859 iap_release_pmc(int cpu, int ri, struct pmc *pm)  in iap_release_pmc()  argument
863 	PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,  in iap_release_pmc()
868 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_release_pmc()
869 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iap_release_pmc()
871 	KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc  in iap_release_pmc()
878 iap_start_pmc(int cpu, int ri, struct pmc *pm)  in iap_start_pmc()  argument
885 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_start_pmc()
886 	    ("[core,%d] illegal row-index %d", __LINE__, ri));  in iap_start_pmc()
890 	PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);  in iap_start_pmc()
894 	PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",  in iap_start_pmc()
895 	    cpu, ri, IAP_EVSEL0 + ri, evsel);  in iap_start_pmc()
910 	wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);  in iap_start_pmc()
913 		cc->pc_globalctrl |= (1ULL << ri);  in iap_start_pmc()
921 iap_stop_pmc(int cpu, int ri, struct pmc *pm __unused)  in iap_stop_pmc()  argument
926 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_stop_pmc()
927 	    ("[core,%d] illegal row index %d", __LINE__, ri));  in iap_stop_pmc()
929 	PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);  in iap_stop_pmc()
931 	wrmsr(IAP_EVSEL0 + ri, 0);  in iap_stop_pmc()
939 iap_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)  in iap_write_pmc()  argument
944 	KASSERT(ri >= 0 && ri < core_iap_npmc,  in iap_write_pmc()
945 	    ("[core,%d] illegal row index %d", __LINE__, ri));  in iap_write_pmc()
952 	PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,  in iap_write_pmc()
953 	    IAP_PMC0 + ri, v);  in iap_write_pmc()
960 	wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);  in iap_write_pmc()
1008 	int error, found_interrupt, ri;  in core_intr()  local
1016 	for (ri = 0; ri < core_iap_npmc; ri++) {  in core_intr()
1018 		if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||  in core_intr()
1022 		if (!iap_pmc_has_overflowed(ri))  in core_intr()
1039 		wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel);  in core_intr()
1040 		wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);  in core_intr()
1045 		wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN);  in core_intr()