210 static bool	pmc_can_allocate_row(int ri, enum pmc_mode mode);
211 static bool	pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
609 pmc_ri_to_classdep(struct pmc_mdep *md __unused, int ri, int *adjri)  in pmc_ri_to_classdep()  argument
613 	KASSERT(ri >= 0 && ri < md->pmd_npmc,  in pmc_ri_to_classdep()
614 	    ("[pmc,%d] illegal row-index %d", __LINE__, ri));  in pmc_ri_to_classdep()
616 	pcd = pmc_rowindex_to_classdep[ri];  in pmc_ri_to_classdep()
618 	    ("[pmc,%d] ri %d null pcd", __LINE__, ri));  in pmc_ri_to_classdep()
620 	*adjri = ri - pcd->pcd_ri;  in pmc_ri_to_classdep()
902 	int ri;  in pmc_link_target_process()  local
914 	ri = PMC_TO_ROWINDEX(pm);  in pmc_link_target_process()
917 	    pm, ri, pp);  in pmc_link_target_process()
931 	atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,  in pmc_link_target_process()
940 	pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?  in pmc_link_target_process()
949 			KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,  in pmc_link_target_process()
951 			    "ri=%d", __LINE__, pp->pp_proc->p_pid, ri));  in pmc_link_target_process()
964 	int ri;  in pmc_unlink_target_process()  local
978 	ri = PMC_TO_ROWINDEX(pm);  in pmc_unlink_target_process()
981 	    pm, ri, pp);  in pmc_unlink_target_process()
983 	KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,  in pmc_unlink_target_process()
985 		ri, pm, pp->pp_pmcs[ri].pp_pmc));  in pmc_unlink_target_process()
987 	pp->pp_pmcs[ri].pp_pmc = NULL;  in pmc_unlink_target_process()
988 	pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t)0;  in pmc_unlink_target_process()
994 			pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t)0;  in pmc_unlink_target_process()
1089 	int ri, error;  in pmc_attach_one_process()  local
1108 	ri = PMC_TO_ROWINDEX(pm);  in pmc_attach_one_process()
1120 	if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */  in pmc_attach_one_process()
1125 	if (pp->pp_pmcs[ri].pp_pmc != NULL) {  in pmc_attach_one_process()
1223 	int ri;  in pmc_detach_one_process()  local
1231 	ri = PMC_TO_ROWINDEX(pm);  in pmc_detach_one_process()
1234 	    pm, ri, p, p->p_pid, p->p_comm, flags);  in pmc_detach_one_process()
1239 	if (pp->pp_pmcs[ri].pp_pmc != pm)  in pmc_detach_one_process()
1335 	u_int ri;  in pmc_process_exec()  local
1387 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exec()
1388 		if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)  in pmc_process_exec()
1413 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exec()
1414 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {  in pmc_process_exec()
1452 	u_int adjri, ri;  in pmc_process_csw_in()  local
1473 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_csw_in()
1474 		if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)  in pmc_process_csw_in()
1480 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,  in pmc_process_csw_in()
1482 		    __LINE__, PMC_TO_ROWINDEX(pm), ri));  in pmc_process_csw_in()
1499 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_process_csw_in()
1502 		phw = pc->pc_hwpmcs[ri];  in pmc_process_csw_in()
1539 				if (pt->pt_pmcs[ri].pt_pmcval > 0)  in pmc_process_csw_in()
1540 					newvalue = pt->pt_pmcs[ri].pt_pmcval;  in pmc_process_csw_in()
1551 				newvalue = pp->pp_pmcs[ri].pp_pmcval;  in pmc_process_csw_in()
1552 				pp->pp_pmcs[ri].pp_pmcval =  in pmc_process_csw_in()
1560 			    cpu, ri, newvalue, pm->pm_sc.pm_reloadcount));  in pmc_process_csw_in()
1566 			newvalue = PMC_PCPU_SAVED(cpu, ri) =  in pmc_process_csw_in()
1571 		PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);  in pmc_process_csw_in()
1611 	u_int adjri, ri;  in pmc_process_csw_out()  local
1651 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_csw_out()
1652 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_process_csw_out()
1663 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,  in pmc_process_csw_out()
1665 			__LINE__, PMC_TO_ROWINDEX(pm), ri));  in pmc_process_csw_out()
1689 		    pp->pp_pmcs[ri].pp_pmc != NULL) {  in pmc_process_csw_out()
1690 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,  in pmc_process_csw_out()
1692 				pm, ri, pp->pp_pmcs[ri].pp_pmc));  in pmc_process_csw_out()
1701 				    cpu, ri, newvalue);  in pmc_process_csw_out()
1723 					pt->pt_pmcs[ri].pt_pmcval = newvalue;  in pmc_process_csw_out()
1737 					pp->pp_pmcs[ri].pp_pmcval += newvalue;  in pmc_process_csw_out()
1738 					if (pp->pp_pmcs[ri].pp_pmcval >  in pmc_process_csw_out()
1740 						pp->pp_pmcs[ri].pp_pmcval -=  in pmc_process_csw_out()
1746 				tmp = newvalue - PMC_PCPU_SAVED(cpu, ri);  in pmc_process_csw_out()
1749 				    cpu, ri, tmp);  in pmc_process_csw_out()
1760 				     "incr=%jx", __LINE__, cpu, ri,  in pmc_process_csw_out()
1761 				     newvalue, PMC_PCPU_SAVED(cpu, ri), tmp));  in pmc_process_csw_out()
1765 				pp->pp_pmcs[ri].pp_pmcval += tmp;  in pmc_process_csw_out()
1835 	int ri;  in pmc_process_mmap()  local
1858 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_mmap()
1859 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&  in pmc_process_mmap()
1882 	int ri;  in pmc_process_munmap()  local
1897 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_munmap()
1898 		pm = pp->pp_pmcs[ri].pp_pmc;  in pmc_process_munmap()
2720 	u_int adjri, ri, cpu;  in pmc_release_pmc_descriptor()  local
2725 	ri   = PMC_TO_ROWINDEX(pm);  in pmc_release_pmc_descriptor()
2726 	pcd  = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_release_pmc_descriptor()
2729 	PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,  in pmc_release_pmc_descriptor()
2750 			phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];  in pmc_release_pmc_descriptor()
2754 				__LINE__, ri, phw->phw_pmc, pm));  in pmc_release_pmc_descriptor()
2755 			PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);  in pmc_release_pmc_descriptor()
2762 		PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);  in pmc_release_pmc_descriptor()
2838 		PMC_UNMARK_ROW_STANDALONE(ri);  in pmc_release_pmc_descriptor()
2840 		PMC_UNMARK_ROW_THREAD(ri);  in pmc_release_pmc_descriptor()
2890 pmc_getrowdisp(int ri)  in pmc_getrowdisp()  argument
2892 	return (pmc_pmcdisp[ri]);  in pmc_getrowdisp()
2906 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)  in pmc_can_allocate_rowindex()  argument
2914 	    "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);  in pmc_can_allocate_rowindex()
2925 			if (PMC_TO_ROWINDEX(pm) == ri) {  in pmc_can_allocate_rowindex()
2941 		if (pp->pp_pmcs[ri].pp_pmc != NULL)  in pmc_can_allocate_rowindex()
2945 	    p, p->p_pid, p->p_comm, ri);  in pmc_can_allocate_rowindex()
2954 pmc_can_allocate_row(int ri, enum pmc_mode mode)  in pmc_can_allocate_row()  argument
2960 	PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);  in pmc_can_allocate_row()
2977 	if (!PMC_ROW_DISP_IS_FREE(ri) &&  in pmc_can_allocate_row()
2978 	    !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&  in pmc_can_allocate_row()
2979 	    !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))  in pmc_can_allocate_row()
2985 	PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);  in pmc_can_allocate_row()
3061 	int adjri, error, cpu, ri;  in pmc_start()  local
3067 	ri   = PMC_TO_ROWINDEX(pm);  in pmc_start()
3068 	pcd  = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_start()
3073 	PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);  in pmc_start()
3182 	int adjri, cpu, error, ri;  in pmc_stop()  local
3217 	ri = PMC_TO_ROWINDEX(pm);  in pmc_stop()
3218 	pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_stop()
3667 	u_int cpu, ri, adjri;  in pmc_do_op_pmcrw()  local
3704 		ri = PMC_TO_ROWINDEX(pm);  in pmc_do_op_pmcrw()
3705 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_do_op_pmcrw()
3726 		ri  = PMC_TO_ROWINDEX(pm);  in pmc_do_op_pmcrw()
3727 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_do_op_pmcrw()
3755 		    ri, prw->pm_value, *valp);  in pmc_do_op_pmcrw()
3757 		PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, *valp);  in pmc_do_op_pmcrw()
4124 		int cpu, ri;  in pmc_syscall_handler()  local
4163 		ri = pma.pm_pmc; /* pmc id == row index */  in pmc_syscall_handler()
4164 		if (ri < 0 || ri >= (int) md->pmd_npmc) {  in pmc_syscall_handler()
4174 		if (PMC_ROW_DISP_IS_THREAD(ri) &&  in pmc_syscall_handler()
4189 		phw = pc->pc_hwpmcs[ri];  in pmc_syscall_handler()
4199 				PMC_MARK_ROW_STANDALONE(ri);  in pmc_syscall_handler()
4203 				PMC_UNMARK_ROW_STANDALONE(ri);  in pmc_syscall_handler()
4273 		int adjri, ri;  in pmc_syscall_handler()  local
4319 		ri = PMC_TO_ROWINDEX(pm);  in pmc_syscall_handler()
4320 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_syscall_handler()
4975 	int ri, adjri, cpu;  in pmc_process_exit()  local
5031 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exit()
5037 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_process_exit()
5041 		PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);  in pmc_process_exit()
5046 		PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p state=%d", ri,  in pmc_process_exit()
5047 		    pp->pp_pmcs[ri].pp_pmc, pm, pm->pm_state);  in pmc_process_exit()
5049 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,  in pmc_process_exit()
5051 		    PMC_TO_ROWINDEX(pm), ri));  in pmc_process_exit()
5052 		KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,  in pmc_process_exit()
5053 		    ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri,  in pmc_process_exit()
5054 		    pp->pp_pmcs[ri].pp_pmc));  in pmc_process_exit()
5056 		    ("[pmc,%d] bad runcount ri %d rc %ju", __LINE__, ri,  in pmc_process_exit()
5073 					tmp = newvalue - PMC_PCPU_SAVED(cpu, ri);  in pmc_process_exit()
5077 					pp->pp_pmcs[ri].pp_pmcval += tmp;  in pmc_process_exit()
5084 		    ("[pmc,%d] runcount is %d", __LINE__, ri));  in pmc_process_exit()
5103 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exit()
5104 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {  in pmc_process_exit()
5141 	unsigned int ri;  in pmc_process_fork()  local
5179 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_fork()
5180 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&  in pmc_process_fork()
5209 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_fork()
5210 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&  in pmc_process_fork()
5418 	int c, cpu, error, n, ri;  in pmc_initialize()  local
5494 	for (ri = c = 0; c < md->pmd_nclass; c++) {  in pmc_initialize()
5496 		pcd->pcd_ri = ri;  in pmc_initialize()
5497 		ri += pcd->pcd_num;  in pmc_initialize()
5510 	for (ri = c = 0; c < md->pmd_nclass; c++) {  in pmc_initialize()
5512 		for (n = 0; n < pcd->pcd_num; n++, ri++)  in pmc_initialize()
5513 			pmc_rowindex_to_classdep[ri] = pcd;  in pmc_initialize()
5516 	KASSERT(ri == md->pmd_npmc,  in pmc_initialize()
5518 	    ri, md->pmd_npmc));  in pmc_initialize()