Lines Matching refs:pm
202 static int pmc_add_sample(ring_type_t ring, struct pmc *pm,
206 static int pmc_attach_process(struct proc *p, struct pmc *pm);
209 static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
213 static int pmc_can_attach(struct pmc *pm, struct proc *p);
217 static int pmc_detach_process(struct proc *p, struct pmc *pm);
218 static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
221 static void pmc_destroy_pmc_descriptor(struct pmc *pm);
224 static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
232 static void pmc_link_target_process(struct pmc *pm,
235 static void pmc_log_kernel_mappings(struct pmc *pm);
239 static void pmc_process_allproc(struct pmc *pm);
257 static int pmc_start(struct pmc *pm);
258 static int pmc_stop(struct pmc *pm);
844 struct pmc *pm, *tmp; in pmc_remove_owner() local
854 LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) { in pmc_remove_owner()
855 PMCDBG1(OWN,ORM,2, "pmc=%p", pm); in pmc_remove_owner()
856 KASSERT(pm->pm_owner == po, in pmc_remove_owner()
857 ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po)); in pmc_remove_owner()
859 pmc_release_pmc_descriptor(pm); /* will unlink from the list */ in pmc_remove_owner()
860 pmc_destroy_pmc_descriptor(pm); in pmc_remove_owner()
898 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp) in pmc_link_target_process() argument
905 KASSERT(pm != NULL && pp != NULL, in pmc_link_target_process()
906 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); in pmc_link_target_process()
907 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), in pmc_link_target_process()
909 __LINE__, pm, pp->pp_proc->p_pid)); in pmc_link_target_process()
914 ri = PMC_TO_ROWINDEX(pm); in pmc_link_target_process()
917 pm, ri, pp); in pmc_link_target_process()
920 LIST_FOREACH(pt, &pm->pm_targets, pt_next) { in pmc_link_target_process()
923 __LINE__, pp, pm)); in pmc_link_target_process()
929 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); in pmc_link_target_process()
932 (uintptr_t)pm); in pmc_link_target_process()
934 if (pm->pm_owner->po_owner == pp->pp_proc) in pmc_link_target_process()
935 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER; in pmc_link_target_process()
940 pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ? in pmc_link_target_process()
941 pm->pm_sc.pm_reloadcount : 0; in pmc_link_target_process()
946 if (PMC_TO_MODE(pm) == PMC_MODE_TS) { in pmc_link_target_process()
962 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) in pmc_unlink_target_process() argument
971 KASSERT(pm != NULL && pp != NULL, in pmc_unlink_target_process()
972 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); in pmc_unlink_target_process()
978 ri = PMC_TO_ROWINDEX(pm); in pmc_unlink_target_process()
981 pm, ri, pp); in pmc_unlink_target_process()
983 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm, in pmc_unlink_target_process()
985 ri, pm, pp->pp_pmcs[ri].pp_pmc)); in pmc_unlink_target_process()
991 if (PMC_TO_MODE(pm) == PMC_MODE_TS) { in pmc_unlink_target_process()
999 if (pm->pm_owner->po_owner == pp->pp_proc) { in pmc_unlink_target_process()
1001 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER; in pmc_unlink_target_process()
1007 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next) in pmc_unlink_target_process()
1012 "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); in pmc_unlink_target_process()
1018 if (LIST_EMPTY(&pm->pm_targets)) { in pmc_unlink_target_process()
1019 p = pm->pm_owner->po_owner; in pmc_unlink_target_process()
1033 pmc_can_attach(struct pmc *pm, struct proc *t) in pmc_can_attach() argument
1043 if ((o = pm->pm_owner->po_owner) == t) in pmc_can_attach()
1087 pmc_attach_one_process(struct proc *p, struct pmc *pm) in pmc_attach_one_process() argument
1095 PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, in pmc_attach_one_process()
1096 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); in pmc_attach_one_process()
1108 ri = PMC_TO_ROWINDEX(pm); in pmc_attach_one_process()
1120 if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */ in pmc_attach_one_process()
1130 pmc_link_target_process(pm, pp); in pmc_attach_one_process()
1132 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) && in pmc_attach_one_process()
1133 (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0) in pmc_attach_one_process()
1134 pm->pm_flags |= PMC_F_NEEDS_LOGFILE; in pmc_attach_one_process()
1136 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */ in pmc_attach_one_process()
1139 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) { in pmc_attach_one_process()
1145 pmclog_process_pmcattach(pm, p->p_pid, fullpath); in pmc_attach_one_process()
1148 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) in pmc_attach_one_process()
1149 pmc_log_process_mappings(pm->pm_owner, p); in pmc_attach_one_process()
1164 pmc_attach_process(struct proc *p, struct pmc *pm) in pmc_attach_process() argument
1171 PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, in pmc_attach_process()
1172 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); in pmc_attach_process()
1178 if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0) in pmc_attach_process()
1181 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) in pmc_attach_process()
1182 return (pmc_attach_one_process(p, pm)); in pmc_attach_process()
1192 if ((error = pmc_attach_one_process(p, pm)) != 0) in pmc_attach_process()
1208 (void)pmc_detach_process(top, pm); in pmc_attach_process()
1221 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags) in pmc_detach_one_process() argument
1228 KASSERT(pm != NULL, in pmc_detach_one_process()
1231 ri = PMC_TO_ROWINDEX(pm); in pmc_detach_one_process()
1234 pm, ri, p, p->p_pid, p->p_comm, flags); in pmc_detach_one_process()
1239 if (pp->pp_pmcs[ri].pp_pmc != pm) in pmc_detach_one_process()
1242 pmc_unlink_target_process(pm, pp); in pmc_detach_one_process()
1245 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) in pmc_detach_one_process()
1246 pmclog_process_pmcdetach(pm, p->p_pid); in pmc_detach_one_process()
1276 pmc_detach_process(struct proc *p, struct pmc *pm) in pmc_detach_process() argument
1282 PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, in pmc_detach_process()
1283 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); in pmc_detach_process()
1285 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) in pmc_detach_process()
1286 return (pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE)); in pmc_detach_process()
1297 (void)pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); in pmc_detach_process()
1315 if (LIST_EMPTY(&pm->pm_targets)) in pmc_detach_process()
1316 pm->pm_flags &= ~PMC_F_ATTACH_DONE; in pmc_detach_process()
1330 struct pmc *pm; in pmc_process_exec() local
1388 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) in pmc_process_exec()
1391 po = pm->pm_owner; in pmc_process_exec()
1394 pmclog_process_procexec(po, pm->pm_id, p->p_pid, in pmc_process_exec()
1414 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { in pmc_process_exec()
1415 if (pmc_can_attach(pm, td->td_proc) != 0) { in pmc_process_exec()
1416 pmc_detach_one_process(td->td_proc, pm, in pmc_process_exec()
1443 struct pmc *pm; in pmc_process_csw_in() local
1474 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) in pmc_process_csw_in()
1477 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), in pmc_process_csw_in()
1479 __LINE__, PMC_TO_MODE(pm))); in pmc_process_csw_in()
1480 KASSERT(PMC_TO_ROWINDEX(pm) == ri, in pmc_process_csw_in()
1482 __LINE__, PMC_TO_ROWINDEX(pm), ri)); in pmc_process_csw_in()
1488 if (pm->pm_state != PMC_STATE_RUNNING) in pmc_process_csw_in()
1491 KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0, in pmc_process_csw_in()
1492 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_csw_in()
1493 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_csw_in()
1496 counter_u64_add(pm->pm_runcount, 1); in pmc_process_csw_in()
1500 (void)pcd->pcd_config_pmc(cpu, adjri, pm); in pmc_process_csw_in()
1507 KASSERT(phw->phw_pmc == pm, in pmc_process_csw_in()
1509 phw->phw_pmc, pm)); in pmc_process_csw_in()
1518 if (PMC_TO_MODE(pm) == PMC_MODE_TS) { in pmc_process_csw_in()
1527 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1542 newvalue = pm->pm_sc.pm_reloadcount; in pmc_process_csw_in()
1553 pm->pm_sc.pm_reloadcount; in pmc_process_csw_in()
1555 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1557 pm->pm_sc.pm_reloadcount, in pmc_process_csw_in()
1560 cpu, ri, newvalue, pm->pm_sc.pm_reloadcount)); in pmc_process_csw_in()
1562 KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, in pmc_process_csw_in()
1564 PMC_TO_MODE(pm))); in pmc_process_csw_in()
1565 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1567 pm->pm_gv.pm_savedvalue; in pmc_process_csw_in()
1568 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1573 (void)pcd->pcd_write_pmc(cpu, adjri, pm, newvalue); in pmc_process_csw_in()
1576 if (PMC_TO_MODE(pm) == PMC_MODE_TS) in pmc_process_csw_in()
1577 pm->pm_pcpu_state[cpu].pps_stalled = 0; in pmc_process_csw_in()
1580 pm->pm_pcpu_state[cpu].pps_cpustate = 1; in pmc_process_csw_in()
1583 (void)pcd->pcd_start_pmc(cpu, adjri, pm); in pmc_process_csw_in()
1601 struct pmc *pm; in pmc_process_csw_out() local
1653 pm = NULL; in pmc_process_csw_out()
1654 (void)(*pcd->pcd_get_config)(cpu, adjri, &pm); in pmc_process_csw_out()
1656 if (pm == NULL) /* nothing at this row index */ in pmc_process_csw_out()
1659 mode = PMC_TO_MODE(pm); in pmc_process_csw_out()
1663 KASSERT(PMC_TO_ROWINDEX(pm) == ri, in pmc_process_csw_out()
1665 __LINE__, PMC_TO_ROWINDEX(pm), ri)); in pmc_process_csw_out()
1673 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_process_csw_out()
1674 if (pm->pm_pcpu_state[cpu].pps_stalled == 0) in pmc_process_csw_out()
1675 (void)pcd->pcd_stop_pmc(cpu, adjri, pm); in pmc_process_csw_out()
1677 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_csw_out()
1678 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_csw_out()
1679 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_csw_out()
1682 counter_u64_add(pm->pm_runcount, -1); in pmc_process_csw_out()
1688 if (pm->pm_state != PMC_STATE_DELETED && pp != NULL && in pmc_process_csw_out()
1690 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, in pmc_process_csw_out()
1692 pm, ri, pp->pp_pmcs[ri].pp_pmc)); in pmc_process_csw_out()
1697 (void)pcd->pcd_read_pmc(cpu, adjri, pm, &newvalue); in pmc_process_csw_out()
1711 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1739 pm->pm_sc.pm_reloadcount) { in pmc_process_csw_out()
1741 pm->pm_sc.pm_reloadcount; in pmc_process_csw_out()
1744 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1763 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1764 pm->pm_gv.pm_savedvalue += tmp; in pmc_process_csw_out()
1766 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1768 if (pm->pm_flags & PMC_F_LOG_PROCCSW) in pmc_process_csw_out()
1769 pmclog_process_proccsw(pm, pp, tmp, td); in pmc_process_csw_out()
1830 const struct pmc *pm; in pmc_process_mmap() local
1859 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && in pmc_process_mmap()
1860 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { in pmc_process_mmap()
1861 pmclog_process_map_in(pm->pm_owner, in pmc_process_mmap()
1878 const struct pmc *pm; in pmc_process_munmap() local
1898 pm = pp->pp_pmcs[ri].pp_pmc; in pmc_process_munmap()
1899 if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { in pmc_process_munmap()
1900 pmclog_process_map_out(pm->pm_owner, pid, in pmc_process_munmap()
1910 pmc_log_kernel_mappings(struct pmc *pm) in pmc_log_kernel_mappings() argument
1916 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), in pmc_log_kernel_mappings()
1918 __LINE__, (void *) pm)); in pmc_log_kernel_mappings()
1920 po = pm->pm_owner; in pmc_log_kernel_mappings()
1924 if (PMC_TO_MODE(pm) == PMC_MODE_SS) in pmc_log_kernel_mappings()
1925 pmc_process_allproc(pm); in pmc_log_kernel_mappings()
2654 pmc_destroy_pmc_descriptor(struct pmc *pm) in pmc_destroy_pmc_descriptor() argument
2657 KASSERT(pm->pm_state == PMC_STATE_DELETED || in pmc_destroy_pmc_descriptor()
2658 pm->pm_state == PMC_STATE_FREE, in pmc_destroy_pmc_descriptor()
2660 KASSERT(LIST_EMPTY(&pm->pm_targets), in pmc_destroy_pmc_descriptor()
2662 KASSERT(pm->pm_owner == NULL, in pmc_destroy_pmc_descriptor()
2664 KASSERT(counter_u64_fetch(pm->pm_runcount) == 0, in pmc_destroy_pmc_descriptor()
2666 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_destroy_pmc_descriptor()
2668 counter_u64_free(pm->pm_runcount); in pmc_destroy_pmc_descriptor()
2669 free(pm->pm_pcpu_state, M_PMC); in pmc_destroy_pmc_descriptor()
2670 free(pm, M_PMC); in pmc_destroy_pmc_descriptor()
2674 pmc_wait_for_pmc_idle(struct pmc *pm) in pmc_wait_for_pmc_idle() argument
2685 pmclog_flush(pm->pm_owner, 1); in pmc_wait_for_pmc_idle()
2686 while (counter_u64_fetch(pm->pm_runcount) > 0) { in pmc_wait_for_pmc_idle()
2687 pmclog_flush(pm->pm_owner, 1); in pmc_wait_for_pmc_idle()
2692 "pmc to be free", __LINE__, PMC_TO_ROWINDEX(pm), in pmc_wait_for_pmc_idle()
2693 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_wait_for_pmc_idle()
2711 pmc_release_pmc_descriptor(struct pmc *pm) in pmc_release_pmc_descriptor() argument
2723 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); in pmc_release_pmc_descriptor()
2725 ri = PMC_TO_ROWINDEX(pm); in pmc_release_pmc_descriptor()
2727 mode = PMC_TO_MODE(pm); in pmc_release_pmc_descriptor()
2729 PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, in pmc_release_pmc_descriptor()
2742 cpu = PMC_TO_CPU(pm); in pmc_release_pmc_descriptor()
2746 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_release_pmc_descriptor()
2747 if (pm->pm_state == PMC_STATE_RUNNING && in pmc_release_pmc_descriptor()
2748 pm->pm_pcpu_state[cpu].pps_stalled == 0) { in pmc_release_pmc_descriptor()
2752 KASSERT(phw->phw_pmc == pm, in pmc_release_pmc_descriptor()
2754 __LINE__, ri, phw->phw_pmc, pm)); in pmc_release_pmc_descriptor()
2758 (void)pcd->pcd_stop_pmc(cpu, adjri, pm); in pmc_release_pmc_descriptor()
2769 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) { in pmc_release_pmc_descriptor()
2770 po = pm->pm_owner; in pmc_release_pmc_descriptor()
2778 pm->pm_state = PMC_STATE_DELETED; in pmc_release_pmc_descriptor()
2786 pmc_wait_for_pmc_idle(pm); in pmc_release_pmc_descriptor()
2798 pm->pm_state = PMC_STATE_DELETED; in pmc_release_pmc_descriptor()
2801 pmc_wait_for_pmc_idle(pm); in pmc_release_pmc_descriptor()
2810 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) { in pmc_release_pmc_descriptor()
2812 pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */ in pmc_release_pmc_descriptor()
2832 (void)pcd->pcd_release_pmc(cpu, adjri, pm); in pmc_release_pmc_descriptor()
2837 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) in pmc_release_pmc_descriptor()
2843 if (pm->pm_owner != NULL) { in pmc_release_pmc_descriptor()
2844 LIST_REMOVE(pm, pm_next); in pmc_release_pmc_descriptor()
2845 pm->pm_owner = NULL; in pmc_release_pmc_descriptor()
2908 struct pmc *pm; in pmc_can_allocate_rowindex() local
2924 LIST_FOREACH(pm, &po->po_pmcs, pm_next) { in pmc_can_allocate_rowindex()
2925 if (PMC_TO_ROWINDEX(pm) == ri) { in pmc_can_allocate_rowindex()
2926 mode = PMC_TO_MODE(pm); in pmc_can_allocate_rowindex()
2930 PMC_TO_CPU(pm) == cpu) in pmc_can_allocate_rowindex()
2995 struct pmc *pm; in pmc_find_pmc_descriptor_in_process() local
3001 LIST_FOREACH(pm, &po->po_pmcs, pm_next) { in pmc_find_pmc_descriptor_in_process()
3002 if (pm->pm_id == pmcid) in pmc_find_pmc_descriptor_in_process()
3003 return (pm); in pmc_find_pmc_descriptor_in_process()
3012 struct pmc *pm, *opm; in pmc_find_pmc() local
3041 if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL) in pmc_find_pmc()
3044 PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm); in pmc_find_pmc()
3046 *pmc = pm; in pmc_find_pmc()
3054 pmc_start(struct pmc *pm) in pmc_start() argument
3063 KASSERT(pm != NULL, in pmc_start()
3066 mode = PMC_TO_MODE(pm); in pmc_start()
3067 ri = PMC_TO_ROWINDEX(pm); in pmc_start()
3071 po = pm->pm_owner; in pmc_start()
3073 PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); in pmc_start()
3075 po = pm->pm_owner; in pmc_start()
3081 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) != 0 && in pmc_start()
3089 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) in pmc_start()
3090 pmc_log_kernel_mappings(pm); in pmc_start()
3097 if (LIST_EMPTY(&pm->pm_targets)) { in pmc_start()
3098 error = (pm->pm_flags & PMC_F_ATTACH_DONE) != 0 ? in pmc_start()
3099 ESRCH : pmc_attach_process(po->po_owner, pm); in pmc_start()
3107 pm->pm_state = PMC_STATE_RUNNING; in pmc_start()
3108 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) != 0) in pmc_start()
3144 cpu = PMC_TO_CPU(pm); in pmc_start()
3153 pm->pm_state = PMC_STATE_RUNNING; in pmc_start()
3156 v = PMC_IS_SAMPLING_MODE(mode) ? pm->pm_sc.pm_reloadcount : in pmc_start()
3157 pm->pm_sc.pm_initial; in pmc_start()
3158 if ((error = pcd->pcd_write_pmc(cpu, adjri, pm, v)) == 0) { in pmc_start()
3161 pm->pm_pcpu_state[cpu].pps_stalled = 0; in pmc_start()
3164 pm->pm_pcpu_state[cpu].pps_cpustate = 1; in pmc_start()
3165 error = pcd->pcd_start_pmc(cpu, adjri, pm); in pmc_start()
3177 pmc_stop(struct pmc *pm) in pmc_stop() argument
3184 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); in pmc_stop()
3186 PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, PMC_TO_MODE(pm), in pmc_stop()
3187 PMC_TO_ROWINDEX(pm)); in pmc_stop()
3189 pm->pm_state = PMC_STATE_STOPPED; in pmc_stop()
3198 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) in pmc_stop()
3209 cpu = PMC_TO_CPU(pm); in pmc_stop()
3217 ri = PMC_TO_ROWINDEX(pm); in pmc_stop()
3220 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_stop()
3222 if ((error = pcd->pcd_stop_pmc(cpu, adjri, pm)) == 0) { in pmc_stop()
3223 error = pcd->pcd_read_pmc(cpu, adjri, pm, in pmc_stop()
3224 &pm->pm_sc.pm_initial); in pmc_stop()
3231 po = pm->pm_owner; in pmc_stop()
3232 if (PMC_TO_MODE(pm) == PMC_MODE_SS) { in pmc_stop()
3533 struct pmc *pm; in pmc_do_op_pmcattach() local
3545 error = pmc_find_pmc(a.pm_pmc, &pm); in pmc_do_op_pmcattach()
3549 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) in pmc_do_op_pmcattach()
3553 if (pm->pm_state == PMC_STATE_RUNNING) { in pmc_do_op_pmcattach()
3555 } else if (pm->pm_state != PMC_STATE_ALLOCATED && in pmc_do_op_pmcattach()
3556 pm->pm_state != PMC_STATE_STOPPED) { in pmc_do_op_pmcattach()
3580 error = pmc_attach_process(p, pm); in pmc_do_op_pmcattach()
3591 struct pmc *pm; in pmc_do_op_pmcdetach() local
3600 error = pmc_find_pmc(a.pm_pmc, &pm); in pmc_do_op_pmcdetach()
3619 error = pmc_detach_process(p, pm); in pmc_do_op_pmcdetach()
3631 struct pmc *pm; in pmc_do_op_pmcrelease() local
3646 error = pmc_find_pmc(pmcid, &pm); in pmc_do_op_pmcrelease()
3650 po = pm->pm_owner; in pmc_do_op_pmcrelease()
3651 pmc_release_pmc_descriptor(pm); in pmc_do_op_pmcrelease()
3653 pmc_destroy_pmc_descriptor(pm); in pmc_do_op_pmcrelease()
3666 struct pmc *pm; in pmc_do_op_pmcrw() local
3677 error = pmc_find_pmc(prw->pm_pmcid, &pm); in pmc_do_op_pmcrw()
3682 if (pm->pm_state != PMC_STATE_ALLOCATED && in pmc_do_op_pmcrw()
3683 pm->pm_state != PMC_STATE_STOPPED && in pmc_do_op_pmcrw()
3684 pm->pm_state != PMC_STATE_RUNNING) in pmc_do_op_pmcrw()
3688 if (pm->pm_state == PMC_STATE_RUNNING && in pmc_do_op_pmcrw()
3692 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) { in pmc_do_op_pmcrw()
3704 ri = PMC_TO_ROWINDEX(pm); in pmc_do_op_pmcrw()
3707 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_do_op_pmcrw()
3711 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && in pmc_do_op_pmcrw()
3712 (pm->pm_state == PMC_STATE_RUNNING)) { in pmc_do_op_pmcrw()
3713 error = (*pcd->pcd_read_pmc)(cpu, adjri, pm, in pmc_do_op_pmcrw()
3716 *valp = pm->pm_gv.pm_savedvalue; in pmc_do_op_pmcrw()
3721 pm->pm_gv.pm_savedvalue = prw->pm_value; in pmc_do_op_pmcrw()
3723 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_do_op_pmcrw()
3725 cpu = PMC_TO_CPU(pm); in pmc_do_op_pmcrw()
3726 ri = PMC_TO_ROWINDEX(pm); in pmc_do_op_pmcrw()
3739 error = (*pcd->pcd_read_pmc)(cpu, adjri, pm, valp); in pmc_do_op_pmcrw()
3743 error = (*pcd->pcd_write_pmc)(cpu, adjri, pm, in pmc_do_op_pmcrw()
3813 struct pmc *pm; in pmc_syscall_handler() local
3852 LIST_FOREACH(pm, &po->po_pmcs, pm_next) in pmc_syscall_handler()
3853 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && in pmc_syscall_handler()
3854 pm->pm_state == PMC_STATE_RUNNING) in pmc_syscall_handler()
3855 pmc_stop(pm); in pmc_syscall_handler()
4031 struct pmc *pm; in pmc_syscall_handler() local
4075 if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0) in pmc_syscall_handler()
4087 if (pm == NULL) /* no PMC associated */ in pmc_syscall_handler()
4090 po = pm->pm_owner; in pmc_syscall_handler()
4097 p->pm_mode = PMC_TO_MODE(pm); in pmc_syscall_handler()
4098 p->pm_event = pm->pm_event; in pmc_syscall_handler()
4099 p->pm_flags = pm->pm_flags; in pmc_syscall_handler()
4101 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) in pmc_syscall_handler()
4103 pm->pm_sc.pm_reloadcount; in pmc_syscall_handler()
4274 struct pmc *pm; in pmc_syscall_handler() local
4284 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) in pmc_syscall_handler()
4298 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) || in pmc_syscall_handler()
4299 (pm->pm_flags & PMC_F_DESCENDANTS)) { in pmc_syscall_handler()
4312 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || in pmc_syscall_handler()
4314 pt->pt_process->pp_proc != pm->pm_owner->po_owner) { in pmc_syscall_handler()
4319 ri = PMC_TO_ROWINDEX(pm); in pmc_syscall_handler()
4396 struct pmc *pm; in pmc_syscall_handler() local
4404 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0) in pmc_syscall_handler()
4407 if (pm->pm_state == PMC_STATE_RUNNING) { in pmc_syscall_handler()
4412 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { in pmc_syscall_handler()
4422 pm->pm_sc.pm_reloadcount = MAX(MAX(1, pmc_mincount), in pmc_syscall_handler()
4425 pm->pm_sc.pm_initial = sc.pm_count; in pmc_syscall_handler()
4437 struct pmc *pm; in pmc_syscall_handler() local
4447 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) in pmc_syscall_handler()
4450 KASSERT(pmcid == pm->pm_id, in pmc_syscall_handler()
4452 pm->pm_id, pmcid)); in pmc_syscall_handler()
4454 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ in pmc_syscall_handler()
4456 else if (pm->pm_state != PMC_STATE_STOPPED && in pmc_syscall_handler()
4457 pm->pm_state != PMC_STATE_ALLOCATED) { in pmc_syscall_handler()
4462 error = pmc_start(pm); in pmc_syscall_handler()
4474 struct pmc *pm; in pmc_syscall_handler() local
4489 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) in pmc_syscall_handler()
4492 KASSERT(pmcid == pm->pm_id, in pmc_syscall_handler()
4494 pm->pm_id, pmcid)); in pmc_syscall_handler()
4496 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ in pmc_syscall_handler()
4498 else if (pm->pm_state != PMC_STATE_RUNNING) { in pmc_syscall_handler()
4503 error = pmc_stop(pm); in pmc_syscall_handler()
4601 pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf) in pmc_add_sample() argument
4620 pm->pm_pcpu_state[cpu].pps_stalled = 1; in pmc_add_sample()
4623 cpu, pm, tf, inuserspace, in pmc_add_sample()
4632 PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm, tf, in pmc_add_sample()
4637 ps->ps_pmc = pm; in pmc_add_sample()
4646 callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ? in pmc_add_sample()
4672 KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0, in pmc_add_sample()
4673 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_add_sample()
4674 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_add_sample()
4676 counter_u64_add(pm->pm_runcount, 1); /* hold onto PMC */ in pmc_add_sample()
4694 pmc_process_interrupt(int ring, struct pmc *pm, struct trapframe *tf) in pmc_process_interrupt() argument
4699 if ((pm->pm_flags & PMC_F_USERCALLCHAIN) && in pmc_process_interrupt()
4702 return (pmc_add_sample(PMC_UR, pm, tf)); in pmc_process_interrupt()
4704 return (pmc_add_sample(ring, pm, tf)); in pmc_process_interrupt()
4715 struct pmc *pm; in pmc_capture_user_callchain() local
4758 pm = ps->ps_pmc; in pmc_capture_user_callchain()
4759 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN, in pmc_capture_user_callchain()
4762 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_capture_user_callchain()
4764 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_capture_user_callchain()
4797 counter_u64_add(pm->pm_runcount, -1); in pmc_capture_user_callchain()
4826 struct pmc *pm; in pmc_process_samples() local
4850 pm = ps->ps_pmc; in pmc_process_samples()
4851 if (pm->pm_state != PMC_STATE_RUNNING) in pmc_process_samples()
4854 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_samples()
4855 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_samples()
4856 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_samples()
4857 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), in pmc_process_samples()
4859 pm, PMC_TO_MODE(pm))); in pmc_process_samples()
4861 po = pm->pm_owner; in pmc_process_samples()
4885 pm, ps->ps_nsamples, ps->ps_flags, in pmc_process_samples()
4900 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { in pmc_process_samples()
4906 pmclog_process_callchain(pm, ps); in pmc_process_samples()
4910 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_samples()
4911 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_samples()
4912 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_samples()
4914 counter_u64_add(pm->pm_runcount, -1); in pmc_process_samples()
4934 (void)(*pcd->pcd_get_config)(cpu, adjri, &pm); in pmc_process_samples()
4936 if (pm == NULL || /* !cfg'ed */ in pmc_process_samples()
4937 pm->pm_state != PMC_STATE_RUNNING || /* !active */ in pmc_process_samples()
4938 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ in pmc_process_samples()
4939 !pm->pm_pcpu_state[cpu].pps_cpustate || /* !desired */ in pmc_process_samples()
4940 !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */ in pmc_process_samples()
4943 pm->pm_pcpu_state[cpu].pps_stalled = 0; in pmc_process_samples()
4944 (void)(*pcd->pcd_start_pmc)(cpu, adjri, pm); in pmc_process_samples()
4970 struct pmc *pm; in pmc_process_exit() local
5036 pm = NULL; in pmc_process_exit()
5039 (void)(*pcd->pcd_get_config)(cpu, adjri, &pm); in pmc_process_exit()
5041 PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm); in pmc_process_exit()
5043 if (pm == NULL || !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) in pmc_process_exit()
5047 pp->pp_pmcs[ri].pp_pmc, pm, pm->pm_state); in pmc_process_exit()
5049 KASSERT(PMC_TO_ROWINDEX(pm) == ri, in pmc_process_exit()
5051 PMC_TO_ROWINDEX(pm), ri)); in pmc_process_exit()
5052 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, in pmc_process_exit()
5053 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, in pmc_process_exit()
5055 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_exit()
5057 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_exit()
5065 if (pm->pm_pcpu_state[cpu].pps_cpustate) { in pmc_process_exit()
5066 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_process_exit()
5067 if (!pm->pm_pcpu_state[cpu].pps_stalled) { in pmc_process_exit()
5068 (void)pcd->pcd_stop_pmc(cpu, adjri, pm); in pmc_process_exit()
5070 if (PMC_TO_MODE(pm) == PMC_MODE_TC) { in pmc_process_exit()
5071 pcd->pcd_read_pmc(cpu, adjri, pm, in pmc_process_exit()
5075 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_exit()
5076 pm->pm_gv.pm_savedvalue += tmp; in pmc_process_exit()
5078 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_exit()
5083 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_exit()
5086 counter_u64_add(pm->pm_runcount, -1); in pmc_process_exit()
5104 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { in pmc_process_exit()
5105 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) != 0 && in pmc_process_exit()
5106 PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { in pmc_process_exit()
5107 pmclog_process_procexit(pm, pp); in pmc_process_exit()
5109 pmc_unlink_target_process(pm, pp); in pmc_process_exit()
5138 struct pmc *pm; in pmc_process_fork() local
5180 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL && in pmc_process_fork()
5181 (pm->pm_flags & PMC_F_DESCENDANTS) != 0) { in pmc_process_fork()
5210 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL && in pmc_process_fork()
5211 (pm->pm_flags & PMC_F_DESCENDANTS) != 0) { in pmc_process_fork()
5212 pmc_link_target_process(pm, ppnew); in pmc_process_fork()
5213 po = pm->pm_owner; in pmc_process_fork()
5266 pmc_process_allproc(struct pmc *pm) in pmc_process_allproc() argument
5272 po = pm->pm_owner; in pmc_process_allproc()