Lines Matching full:pmc
47 #include <sys/pmc.h>
102 pmc_value_t *pmc_pcpu_saved; /* saved PMC values: CSW handling */
107 static int *pmc_pmcdisp; /* PMC row dispositions */
118 KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
122 ("[pmc,%d] row disposition error", __LINE__)); \
127 KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
132 KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
139 KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
161 * Hash table of PMC owner descriptors. This table is protected by
162 * the shared PMC "sx" lock.
168 * List of PMC owners with system-wide sampling PMCs.
202 static int pmc_add_sample(ring_type_t ring, struct pmc *pm,
206 static int pmc_attach_process(struct proc *p, struct pmc *pm);
207 static struct pmc *pmc_allocate_pmc_descriptor(void);
209 static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
213 static int pmc_can_attach(struct pmc *pm, struct proc *p);
217 static int pmc_detach_process(struct proc *p, struct pmc *pm);
218 static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
221 static void pmc_destroy_pmc_descriptor(struct pmc *pm);
224 static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
225 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
226 pmc_id_t pmc);
232 static void pmc_link_target_process(struct pmc *pm,
235 static void pmc_log_kernel_mappings(struct pmc *pm);
239 static void pmc_process_allproc(struct pmc *pm);
254 static void pmc_release_pmc_descriptor(struct pmc *pmc);
257 static int pmc_start(struct pmc *pm);
258 static int pmc_stop(struct pmc *pm);
263 static void pmc_unlink_target_process(struct pmc *pmc,
436 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
438 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
440 MODULE_VERSION(pmc, PMC_VERSION);
488 DBG_SET_FLAG_MAJ("pmc", PMC); in pmc_debugflags_parse()
606 * index for the PMC class index.
614 ("[pmc,%d] illegal row-index %d", __LINE__, ri)); in pmc_ri_to_classdep()
618 ("[pmc,%d] ri %d null pcd", __LINE__, ri)); in pmc_ri_to_classdep()
622 ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri)); in pmc_ri_to_classdep()
635 * - PMC descriptors (and attached lists)
681 * time, handlers look at the PMC pointer (phw->phw_pmc) configured
682 * when the PMC was started. If this pointer is NULL, the interrupt
685 * PMC hardware is started. Conversely, this pointer is unset atomically
686 * only after the PMC hardware is stopped.
690 * locks. We also ensure that a PMC's software state is destroyed only
691 * after the PMC is taken off hardware (on all CPUs).
698 * while the target process is running on another. A PMC could also
703 * of 'struct pmc *' pointers, one for each hardware PMC.
705 * - At context switch IN time, each "target" PMC in RUNNING state
706 * gets started on hardware and a pointer to each PMC is copied into
707 * the per-cpu phw array. The 'runcount' for the PMC is
712 * only if the PMC is in a non-deleted state (the PMCs state could
716 * OUT, the PMC could have been released on another CPU. Therefore
718 * OFF PMCs and will update a PMC's saved value only if reachable
721 * - OP PMCRELEASE could be called on a PMC at any time (the PMC could
725 * We prevent further scheduling of the PMC by marking it as in
726 * state 'DELETED'. If the runcount of the PMC is non-zero then
727 * this PMC is currently running on a CPU somewhere. The thread
731 * The contents of a PMC descriptor (struct pmc) are protected using
735 * - Type "pmc-sx", used by the global SX lock.
736 * - Type "pmc-sleep", for sleep mutexes used by logger threads.
737 * - Type "pmc-per-proc", for protecting PMC owner descriptors.
738 * - Type "pmc-leaf", used for all other spin mutexes.
780 ("[pmc,%d] bad cpu number %d", __LINE__, cpu)); in pmc_select_cpu()
783 KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive " in pmc_select_cpu()
793 ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__, in pmc_select_cpu()
844 struct pmc *pm, *tmp; in pmc_remove_owner()
853 /* release all owned PMC descriptors */ in pmc_remove_owner()
855 PMCDBG1(OWN,ORM,2, "pmc=%p", pm); in pmc_remove_owner()
857 ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po)); in pmc_remove_owner()
864 ("[pmc,%d] SS count not zero", __LINE__)); in pmc_remove_owner()
866 ("[pmc,%d] PMC list not empty", __LINE__)); in pmc_remove_owner()
895 * Add an association between a target process and a PMC.
898 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp) in pmc_link_target_process()
906 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); in pmc_link_target_process()
908 ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d", in pmc_link_target_process()
911 ("[pmc,%d] Illegal reference count %d for process record %p", in pmc_link_target_process()
916 PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p", in pmc_link_target_process()
922 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", in pmc_link_target_process()
950 ("[pmc,%d] pt_pmcval not cleared for pid=%d at " in pmc_link_target_process()
959 * Removes the association between a target process and a PMC.
962 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) in pmc_unlink_target_process()
972 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); in pmc_unlink_target_process()
975 ("[pmc,%d] Illegal ref count %d on process record %p", in pmc_unlink_target_process()
980 PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p", in pmc_unlink_target_process()
984 ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__, in pmc_unlink_target_process()
1006 /* Remove the target process from the PMC structure */ in pmc_unlink_target_process()
1011 KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found " in pmc_unlink_target_process()
1012 "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); in pmc_unlink_target_process()
1017 /* if the PMC now lacks targets, send the owner a SIGIO */ in pmc_unlink_target_process()
1029 * Check if PMC 'pm' may be attached to target process 't'.
1033 pmc_can_attach(struct pmc *pm, struct proc *t) in pmc_can_attach()
1035 struct proc *o; /* pmc owner */ in pmc_can_attach()
1040 * A PMC's owner can always attach that PMC to itself. in pmc_can_attach()
1057 * The effective uid of the PMC owner should match at least one in pmc_can_attach()
1084 * Attach a process to a PMC.
1087 pmc_attach_one_process(struct proc *p, struct pmc *pm) in pmc_attach_one_process()
1106 * process descriptor and PMC. in pmc_attach_one_process()
1164 pmc_attach_process(struct proc *p, struct pmc *pm) in pmc_attach_process()
1175 * If this PMC successfully allowed a GETMSR operation in pmc_attach_process()
1186 * this PMC. in pmc_attach_process()
1216 * Detach a process from a PMC. If there are no other PMCs tracking
1221 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags) in pmc_detach_one_process()
1229 ("[pmc,%d] null pm pointer", __LINE__)); in pmc_detach_one_process()
1254 ("[pmc,%d] Illegal refcnt %d for process struct %p", in pmc_detach_one_process()
1257 if (pp->pp_refcnt != 0) /* still a target of some PMC */ in pmc_detach_one_process()
1273 * Detach a process and optionally its descendants from a PMC.
1276 pmc_detach_process(struct proc *p, struct pmc *pm) in pmc_detach_process()
1289 * Traverse all children, detaching them from this PMC. We in pmc_detach_process()
1290 * ignore errors since we could be detaching a PMC from a in pmc_detach_process()
1330 struct pmc *pm; in pmc_process_exec()
1373 * If the process being exec'ed is not the target of any PMC, we are in pmc_process_exec()
1410 * than before, allow it to be the target of a PMC only if in pmc_process_exec()
1411 * the PMC's owner has sufficient privilege. in pmc_process_exec()
1423 ("[pmc,%d] Illegal ref count %u on pp %p", __LINE__, in pmc_process_exec()
1443 struct pmc *pm; in pmc_process_csw_in()
1460 ("[pmc,%d] not my thread state", __LINE__)); in pmc_process_csw_in()
1470 ("[pmc,%d] weird CPU id %d", __LINE__, cpu)); in pmc_process_csw_in()
1478 ("[pmc,%d] Target PMC in non-virtual mode (%d)", in pmc_process_csw_in()
1481 ("[pmc,%d] Row index mismatch pmc %d != ri %d", in pmc_process_csw_in()
1492 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_csw_in()
1495 /* increment PMC runcount */ in pmc_process_csw_in()
1505 ("[pmc,%d] null hw pointer", __LINE__)); in pmc_process_csw_in()
1508 ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__, in pmc_process_csw_in()
1512 * Write out saved value and start the PMC. in pmc_process_csw_in()
1515 * counting mode PMCs use a per-pmc value that is in pmc_process_csw_in()
1524 ("[pmc,%d] No thread found for td=%p", __LINE__, in pmc_process_csw_in()
1558 ("[pmc,%d] pmcval outside of expected range cpu=%d " in pmc_process_csw_in()
1563 ("[pmc,%d] illegal mode=%d", __LINE__, in pmc_process_csw_in()
1575 /* If a sampling mode PMC, reset stalled state. */ in pmc_process_csw_in()
1582 /* Start the PMC. */ in pmc_process_csw_in()
1601 struct pmc *pm; in pmc_process_csw_out()
1637 ("[pmc,%d weird CPU id %d", __LINE__, cpu)); in pmc_process_csw_out()
1642 * When a PMC gets unlinked from a target PMC, it will in pmc_process_csw_out()
1648 * the hardware to determine if a PMC is scheduled on in pmc_process_csw_out()
1661 continue; /* not a process virtual PMC */ in pmc_process_csw_out()
1664 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", in pmc_process_csw_out()
1670 * an interrupt re-enables the PMC after this code has in pmc_process_csw_out()
1678 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_csw_out()
1681 /* reduce this PMC's runcount */ in pmc_process_csw_out()
1685 * If this PMC is associated with this process, in pmc_process_csw_out()
1691 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, in pmc_process_csw_out()
1694 ("[pmc,%d] pp refcnt = %d", __LINE__, in pmc_process_csw_out()
1708 ("[pmc,%d] No thread found for td=%p", in pmc_process_csw_out()
1758 ("[pmc,%d] negative increment cpu=%d " in pmc_process_csw_out()
1792 struct pmc_process *pmc; in pmc_process_thread_add() local
1794 pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE); in pmc_process_thread_add()
1795 if (pmc != NULL) in pmc_process_thread_add()
1796 pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE); in pmc_process_thread_add()
1805 struct pmc_process *pmc; in pmc_process_thread_delete() local
1807 pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE); in pmc_process_thread_delete()
1808 if (pmc != NULL) in pmc_process_thread_delete()
1809 pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc, in pmc_process_thread_delete()
1830 const struct pmc *pm; in pmc_process_mmap()
1856 * Inform sampling PMC owners tracking this process. in pmc_process_mmap()
1878 const struct pmc *pm; in pmc_process_munmap()
1910 pmc_log_kernel_mappings(struct pmc *pm) in pmc_log_kernel_mappings()
1917 ("[pmc,%d] non-sampling PMC (%p) desires mapping information", in pmc_log_kernel_mappings()
2103 panic("[pmc,%d] Cannot find init", __LINE__); in pmc_log_all_process_mappings()
2221 KASSERT(td == curthread, ("[pmc,%d] td != curthread", in pmc_hook_handler()
2228 ("[pmc,%d] invalid td_pinned value", __LINE__)); in pmc_hook_handler()
2238 KASSERT(td == curthread, ("[pmc,%d] td != curthread", in pmc_hook_handler()
2246 ("[pmc,%d] invalid td_pinned value", __LINE__)); in pmc_hook_handler()
2255 * Call soft PMC sampling intr. in pmc_hook_handler()
2270 KASSERT(td == curthread, ("[pmc,%d] td != curthread", in pmc_hook_handler()
2279 KASSERT(td == curthread, ("[pmc,%d] td != curthread", in pmc_hook_handler()
2285 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); in pmc_hook_handler()
2312 mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN); in pmc_allocate_owner_descriptor()
2314 PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p", in pmc_allocate_owner_descriptor()
2433 KASSERT(td != NULL, ("[pmc,%d] called to add NULL td", __LINE__)); in pmc_find_thread_descriptor()
2486 KASSERT(!PROC_LOCKED(p), ("[pmc,%d] proc unexpectedly locked", in pmc_add_thread_descriptors_from_proc()
2585 ("[pmc,%d] Removing process descriptor %p with count %d", in pmc_remove_process_descriptor()
2628 "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po); in pmc_find_owner_descriptor()
2634 * Allocate a pmc descriptor and initialize its fields.
2636 static struct pmc *
2639 struct pmc *pmc; in pmc_allocate_pmc_descriptor() local
2641 pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK | M_ZERO); in pmc_allocate_pmc_descriptor()
2642 pmc->pm_runcount = counter_u64_alloc(M_WAITOK); in pmc_allocate_pmc_descriptor()
2643 pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state) * mp_ncpus, in pmc_allocate_pmc_descriptor()
2645 PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc); in pmc_allocate_pmc_descriptor()
2647 return (pmc); in pmc_allocate_pmc_descriptor()
2651 * Destroy a pmc descriptor.
2654 pmc_destroy_pmc_descriptor(struct pmc *pm) in pmc_destroy_pmc_descriptor()
2659 ("[pmc,%d] destroying non-deleted PMC", __LINE__)); in pmc_destroy_pmc_descriptor()
2661 ("[pmc,%d] destroying pmc with targets", __LINE__)); in pmc_destroy_pmc_descriptor()
2663 ("[pmc,%d] destroying pmc attached to an owner", __LINE__)); in pmc_destroy_pmc_descriptor()
2665 ("[pmc,%d] pmc has non-zero run count %ju", __LINE__, in pmc_destroy_pmc_descriptor()
2674 pmc_wait_for_pmc_idle(struct pmc *pm) in pmc_wait_for_pmc_idle()
2682 * Loop (with a forced context switch) till the PMC's runcount in pmc_wait_for_pmc_idle()
2691 ("[pmc,%d] (ri%d, rc%ju) waiting too long for " in pmc_wait_for_pmc_idle()
2692 "pmc to be free", __LINE__, PMC_TO_ROWINDEX(pm), in pmc_wait_for_pmc_idle()
2702 * - detaches the PMC from hardware
2704 * - removes the PMC from its owner's list
2705 * - destroys the PMC private mutex
2707 * Once this function completes, the given pmc pointer can be freed by
2711 pmc_release_pmc_descriptor(struct pmc *pm) in pmc_release_pmc_descriptor()
2723 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); in pmc_release_pmc_descriptor()
2729 PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, in pmc_release_pmc_descriptor()
2733 * First, we take the PMC off hardware. in pmc_release_pmc_descriptor()
2738 * A system mode PMC runs on a specific CPU. Switch in pmc_release_pmc_descriptor()
2753 ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)", in pmc_release_pmc_descriptor()
2755 PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri); in pmc_release_pmc_descriptor()
2762 PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri); in pmc_release_pmc_descriptor()
2783 * We could have references to this PMC structure in the in pmc_release_pmc_descriptor()
2790 * A virtual PMC could be running on multiple CPUs at a given in pmc_release_pmc_descriptor()
2793 * By marking its state as DELETED, we ensure that this PMC is in pmc_release_pmc_descriptor()
2796 * Then we wait till all CPUs are done with this PMC. in pmc_release_pmc_descriptor()
2804 * At this point the PMC is off all CPUs and cannot be freshly in pmc_release_pmc_descriptor()
2806 * from this PMC. If a process-record's refcount falls to zero, in pmc_release_pmc_descriptor()
2814 PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt); in pmc_release_pmc_descriptor()
2850 * Register an owner and a pmc.
2853 pmc_register_owner(struct proc *p, struct pmc *pmc) in pmc_register_owner() argument
2864 KASSERT(pmc->pm_owner == NULL, in pmc_register_owner()
2865 ("[pmc,%d] attempting to own an initialized PMC", __LINE__)); in pmc_register_owner()
2866 pmc->pm_owner = po; in pmc_register_owner()
2868 LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next); in pmc_register_owner()
2875 pmclog_process_pmcallocate(pmc); in pmc_register_owner()
2877 PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p", in pmc_register_owner()
2878 po, pmc); in pmc_register_owner()
2896 * Check if a PMC at row index 'ri' can be allocated to the current
2900 * - the current process is already being profiled by a PMC at index 'ri',
2902 * - the current process has already allocated a PMC at index 'ri'
2908 struct pmc *pm; in pmc_can_allocate_rowindex()
2913 PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d " in pmc_can_allocate_rowindex()
2917 * We shouldn't have already allocated a process-mode PMC at in pmc_can_allocate_rowindex()
2920 * We shouldn't have allocated a system-wide PMC on the same in pmc_can_allocate_rowindex()
2937 * We also shouldn't be the target of any PMC at this index in pmc_can_allocate_rowindex()
2944 PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok", in pmc_can_allocate_rowindex()
2950 * Check if a given PMC at row index 'ri' can be currently used in
2960 PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode); in pmc_can_allocate_row()
2968 * check disposition for PMC row 'ri': in pmc_can_allocate_row()
2985 PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode); in pmc_can_allocate_row()
2990 * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
2992 static struct pmc *
2995 struct pmc *pm; in pmc_find_pmc_descriptor_in_process()
2998 ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, in pmc_find_pmc_descriptor_in_process()
3010 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc) in pmc_find_pmc() argument
3012 struct pmc *pm, *opm; in pmc_find_pmc()
3016 PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid); in pmc_find_pmc()
3044 PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm); in pmc_find_pmc()
3046 *pmc = pm; in pmc_find_pmc()
3051 * Start a PMC.
3054 pmc_start(struct pmc *pm) in pmc_start()
3064 ("[pmc,%d] null pm", __LINE__)); in pmc_start()
3073 PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); in pmc_start()
3086 * If this is a sampling mode PMC, log mapping information for in pmc_start()
3094 * If a PMCATTACH has never been done on this PMC, in pmc_start()
3103 * If the PMC is attached to its owner, then force a context in pmc_start()
3116 * A system-wide PMC. in pmc_start()
3119 * sampling PMC. in pmc_start()
3135 PMCDBG1(PMC,OPS,1, "po=%p in global list", po); in pmc_start()
3141 * PMC, and start the hardware. in pmc_start()
3151 * so write out the initial value and start the PMC. in pmc_start()
3159 /* If a sampling mode PMC, reset stalled state. */ in pmc_start()
3174 * Stop a PMC.
3177 pmc_stop(struct pmc *pm) in pmc_stop()
3184 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); in pmc_stop()
3186 PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, PMC_TO_MODE(pm), in pmc_stop()
3192 * If the PMC is a virtual mode one, changing the state to non-RUNNING in pmc_stop()
3193 * is enough to ensure that the PMC never gets scheduled. in pmc_stop()
3195 * If this PMC is current running on a CPU, then it will handled in pmc_stop()
3202 * A system-mode PMC. Move to the CPU associated with this PMC, and in pmc_stop()
3211 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); in pmc_stop()
3230 /* Remove this owner from the global list of SS PMC owners. */ in pmc_stop()
3238 PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po); in pmc_stop()
3290 struct pmc *pmc; in pmc_do_op_pmcallocate() local
3333 * Refuse an allocation for a system-wide PMC if this process has been in pmc_do_op_pmcallocate()
3381 /* The requested PMC capabilities should be feasible. */ in pmc_do_op_pmcallocate()
3385 PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", pa->pm_ev, in pmc_do_op_pmcallocate()
3388 pmc = pmc_allocate_pmc_descriptor(); in pmc_do_op_pmcallocate()
3389 pmc->pm_id = PMC_ID_MAKE_ID(cpu, pa->pm_mode, class, PMC_ID_INVALID); in pmc_do_op_pmcallocate()
3390 pmc->pm_event = pa->pm_ev; in pmc_do_op_pmcallocate()
3391 pmc->pm_state = PMC_STATE_FREE; in pmc_do_op_pmcallocate()
3392 pmc->pm_caps = caps; in pmc_do_op_pmcallocate()
3393 pmc->pm_flags = flags; in pmc_do_op_pmcallocate()
3406 pmc->pm_sc.pm_reloadcount = MAX(MAX(1, pmc_mincount), in pmc_do_op_pmcallocate()
3409 pmc->pm_sc.pm_initial = pa->pm_count; in pmc_do_op_pmcallocate()
3432 if (pcd->pcd_allocate_pmc(cpu, adjri, pmc, pa) == 0) { in pmc_do_op_pmcallocate()
3446 if (pcd->pcd_allocate_pmc(td->td_oncpu, adjri, pmc, in pmc_do_op_pmcallocate()
3460 pmc_destroy_pmc_descriptor(pmc); in pmc_do_op_pmcallocate()
3465 pmc->pm_id = PMC_ID_MAKE_ID(cpu, mode, class, n); in pmc_do_op_pmcallocate()
3467 PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x", in pmc_do_op_pmcallocate()
3468 pmc->pm_event, class, mode, n, pmc->pm_id); in pmc_do_op_pmcallocate()
3471 if ((pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW)) != 0) in pmc_do_op_pmcallocate()
3472 pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; in pmc_do_op_pmcallocate()
3476 pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; in pmc_do_op_pmcallocate()
3479 * Configure global pmc's immediately. in pmc_do_op_pmcallocate()
3481 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) { in pmc_do_op_pmcallocate()
3489 (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) { in pmc_do_op_pmcallocate()
3490 (void)pcd->pcd_release_pmc(cpu, adjri, pmc); in pmc_do_op_pmcallocate()
3491 pmc_destroy_pmc_descriptor(pmc); in pmc_do_op_pmcallocate()
3499 pmc->pm_state = PMC_STATE_ALLOCATED; in pmc_do_op_pmcallocate()
3500 pmc->pm_class = class; in pmc_do_op_pmcallocate()
3511 * Register this PMC with the current thread as its owner. in pmc_do_op_pmcallocate()
3513 error = pmc_register_owner(p, pmc); in pmc_do_op_pmcallocate()
3515 pmc_release_pmc_descriptor(pmc); in pmc_do_op_pmcallocate()
3516 pmc_destroy_pmc_descriptor(pmc); in pmc_do_op_pmcallocate()
3523 pa->pm_pmcid = pmc->pm_id; in pmc_do_op_pmcallocate()
3533 struct pmc *pm; in pmc_do_op_pmcattach()
3573 * We are allowed to attach a PMC to a process if we can debug it. in pmc_do_op_pmcattach()
3591 struct pmc *pm; in pmc_do_op_pmcdetach()
3631 struct pmc *pm; in pmc_do_op_pmcrelease()
3635 * Find PMC pointer for the named PMC. in pmc_do_op_pmcrelease()
3638 * PMC, remove all its target threads, and remove the in pmc_do_op_pmcrelease()
3639 * PMC from its owner's list. in pmc_do_op_pmcrelease()
3641 * Remove the owner record if this is the last PMC in pmc_do_op_pmcrelease()
3666 struct pmc *pm; in pmc_do_op_pmcrw()
3670 PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw->pm_pmcid, prw->pm_flags); in pmc_do_op_pmcrw()
3676 /* Locate PMC descriptor. */ in pmc_do_op_pmcrw()
3681 /* Can't read a PMC that hasn't been started. */ in pmc_do_op_pmcrw()
3694 * If this PMC is attached to its owner (i.e., the process in pmc_do_op_pmcrw()
3697 * are only allowed when the PMC is stopped, so only update the in pmc_do_op_pmcrw()
3700 * If the PMC is not running, or is not attached to its owner, in pmc_do_op_pmcrw()
3754 PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx", in pmc_do_op_pmcrw()
3757 PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, *valp); in pmc_do_op_pmcrw()
3775 /* PMC isn't set up yet */ in pmc_syscall_handler()
3813 struct pmc *pm; in pmc_syscall_handler()
4031 struct pmc *pm; in pmc_syscall_handler()
4073 ("[pmc,%d] null pcd ri=%d", __LINE__, n)); in pmc_syscall_handler()
4087 if (pm == NULL) /* no PMC associated */ in pmc_syscall_handler()
4093 ("[pmc,%d] pmc_owner had a null proc pointer", in pmc_syscall_handler()
4108 /* now copy out the PMC info collected */ in pmc_syscall_handler()
4118 * Set the administrative state of a PMC. I.e. whether in pmc_syscall_handler()
4119 * the PMC is to be used or not. in pmc_syscall_handler()
4134 ("[pmc,%d] td != curthread", __LINE__)); in pmc_syscall_handler()
4163 ri = pma.pm_pmc; /* pmc id == row index */ in pmc_syscall_handler()
4170 * We can't disable a PMC with a row-index allocated in pmc_syscall_handler()
4181 * otherwise, this PMC on this CPU is either free or in pmc_syscall_handler()
4215 * Allocate a PMC. in pmc_syscall_handler()
4234 * Attach a PMC to a process. in pmc_syscall_handler()
4249 * Detach an attached PMC from a process. in pmc_syscall_handler()
4274 struct pmc *pm; in pmc_syscall_handler()
4288 * The allocated PMC has to be a process virtual PMC, in pmc_syscall_handler()
4307 * a process that has allocated and attached a PMC to in pmc_syscall_handler()
4308 * itself. Conversely the PMC is only allowed to have in pmc_syscall_handler()
4322 /* PMC class has no 'GETMSR' support */ in pmc_syscall_handler()
4346 * Release an allocated PMC. in pmc_syscall_handler()
4361 * Read and/or write a PMC. in pmc_syscall_handler()
4390 * Set the sampling rate for a sampling mode PMC and the in pmc_syscall_handler()
4391 * initial count for a counting mode PMC. in pmc_syscall_handler()
4396 struct pmc *pm; in pmc_syscall_handler()
4431 * Start a PMC. in pmc_syscall_handler()
4437 struct pmc *pm; in pmc_syscall_handler()
4451 ("[pmc,%d] pmcid %x != id %x", __LINE__, in pmc_syscall_handler()
4468 * Stop a PMC. in pmc_syscall_handler()
4474 struct pmc *pm; in pmc_syscall_handler()
4485 * Mark the PMC as inactive and invoke the MD stop in pmc_syscall_handler()
4493 ("[pmc,%d] pmc id %x != pmcid %x", __LINE__, in pmc_syscall_handler()
4601 pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf) in pmc_add_sample()
4673 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_add_sample()
4676 counter_u64_add(pm->pm_runcount, 1); /* hold onto PMC */ in pmc_add_sample()
4694 pmc_process_interrupt(int ring, struct pmc *pm, struct trapframe *tf) in pmc_process_interrupt()
4715 struct pmc *pm; in pmc_capture_user_callchain()
4730 ("[pmc,%d] Retrieving callchain for thread that doesn't want it", in pmc_capture_user_callchain()
4755 ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__, in pmc_capture_user_callchain()
4760 ("[pmc,%d] Retrieving callchain for PMC that doesn't " in pmc_capture_user_callchain()
4763 ("[pmc,%d] runcount %ju", __LINE__, in pmc_capture_user_callchain()
4826 struct pmc *pm; in pmc_process_samples()
4836 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, in pmc_process_samples()
4855 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_samples()
4858 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__, in pmc_process_samples()
4890 * If this is a process-mode PMC that is attached to in pmc_process_samples()
4895 * Otherwise, this is either a sampling-mode PMC that in pmc_process_samples()
4897 * or a system-wide sampling PMC. Dispatch a log in pmc_process_samples()
4898 * entry to the PMC's owner process. in pmc_process_samples()
4911 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_samples()
4926 * If the NMI handler sets the pm_stalled field of a PMC after in pmc_process_samples()
4927 * the check below, we'll end up processing the stalled PMC at in pmc_process_samples()
4933 ("[pmc,%d] null pcd ri=%d", __LINE__, n)); in pmc_process_samples()
4970 struct pmc *pm; in pmc_process_exit()
4983 * Log a sysexit event to all SS PMC owners. in pmc_process_exit()
5022 PMCDBG2(PRC,EXT,2, "process-exit proc=%p pmc-process=%p", p, pp); in pmc_process_exit()
5033 * Pick up the pmc pointer from hardware state similar to the in pmc_process_exit()
5050 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", __LINE__, in pmc_process_exit()
5053 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, in pmc_process_exit()
5056 ("[pmc,%d] bad runcount ri %d rc %ju", __LINE__, ri, in pmc_process_exit()
5062 * interrupt re-enables the PMC after this code has already in pmc_process_exit()
5084 ("[pmc,%d] runcount is %d", __LINE__, ri)); in pmc_process_exit()
5099 * send a signal to all PMC owner's whose PMCs are orphaned. in pmc_process_exit()
5101 * Log PMC value at exit time if requested. in pmc_process_exit()
5138 struct pmc *pm; in pmc_process_fork()
5165 PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1, in pmc_process_fork()
5266 pmc_process_allproc(struct pmc *pm) in pmc_process_allproc()
5444 PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION); in pmc_initialize()
5501 ("[pmc,%d] no classes or pmcs", __LINE__)); in pmc_initialize()
5517 ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__, in pmc_initialize()
5561 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); in pmc_initialize()
5612 mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf", in pmc_initialize()
5619 pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size, in pmc_initialize()
5627 mtx_init(&pmc_threadfreelist_mtx, "pmc-threadfreelist", "pmc-leaf", in pmc_initialize()
5761 ("[pmc,%d] Global SS owner list not empty", __LINE__)); in pmc_cleanup()
5763 ("[pmc,%d] Global SS count not empty", __LINE__)); in pmc_cleanup()
5765 /* do processor and pmc-class dependent cleanup */ in pmc_cleanup()
5772 PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p", in pmc_cleanup()
5801 ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__, in pmc_cleanup()
5804 ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__, in pmc_cleanup()
5807 ("[pmc,%d] Null userret cpu sample buffer cpu=%d", __LINE__, in pmc_cleanup()