202 static int	pmc_add_sample(ring_type_t ring, struct pmc *pm,
206 static int	pmc_attach_process(struct proc *p, struct pmc *pm);
207 static struct pmc *pmc_allocate_pmc_descriptor(void);
209 static int	pmc_attach_one_process(struct proc *p, struct pmc *pm);
213 static int	pmc_can_attach(struct pmc *pm, struct proc *p);
217 static int	pmc_detach_process(struct proc *p, struct pmc *pm);
218 static int	pmc_detach_one_process(struct proc *p, struct pmc *pm,
221 static void	pmc_destroy_pmc_descriptor(struct pmc *pm);
224 static int	pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
225 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
226     pmc_id_t pmc);
232 static void	pmc_link_target_process(struct pmc *pm,
235 static void	pmc_log_kernel_mappings(struct pmc *pm);
239 static void	pmc_process_allproc(struct pmc *pm);
254 static void	pmc_release_pmc_descriptor(struct pmc *pmc);
257 static int	pmc_start(struct pmc *pm);
258 static int	pmc_stop(struct pmc *pm);
263 static void	pmc_unlink_target_process(struct pmc *pmc,
436 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
438 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
440 MODULE_VERSION(pmc, PMC_VERSION);
844 	struct pmc *pm, *tmp;  in pmc_remove_owner()
898 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)  in pmc_link_target_process()
962 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)  in pmc_unlink_target_process()
1033 pmc_can_attach(struct pmc *pm, struct proc *t)  in pmc_can_attach()
1087 pmc_attach_one_process(struct proc *p, struct pmc *pm)  in pmc_attach_one_process()
1164 pmc_attach_process(struct proc *p, struct pmc *pm)  in pmc_attach_process()
1221 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)  in pmc_detach_one_process()
1276 pmc_detach_process(struct proc *p, struct pmc *pm)  in pmc_detach_process()
1330 	struct pmc *pm;  in pmc_process_exec()
1443 	struct pmc *pm;  in pmc_process_csw_in()
1601 	struct pmc *pm;  in pmc_process_csw_out()
1792 	struct pmc_process *pmc;  in pmc_process_thread_add()  local
1794 	pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);  in pmc_process_thread_add()
1795 	if (pmc != NULL)  in pmc_process_thread_add()
1796 		pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE);  in pmc_process_thread_add()
1805 	struct pmc_process *pmc;  in pmc_process_thread_delete()  local
1807 	pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);  in pmc_process_thread_delete()
1808 	if (pmc != NULL)  in pmc_process_thread_delete()
1809 		pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,  in pmc_process_thread_delete()
1830 	const struct pmc *pm;  in pmc_process_mmap()
1878 	const struct pmc *pm;  in pmc_process_munmap()
1910 pmc_log_kernel_mappings(struct pmc *pm)  in pmc_log_kernel_mappings()
2636 static struct pmc *
2639 	struct pmc *pmc;  in pmc_allocate_pmc_descriptor()  local
2641 	pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK | M_ZERO);  in pmc_allocate_pmc_descriptor()
2642 	pmc->pm_runcount = counter_u64_alloc(M_WAITOK);  in pmc_allocate_pmc_descriptor()
2643 	pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state) * mp_ncpus,  in pmc_allocate_pmc_descriptor()
2645 	PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);  in pmc_allocate_pmc_descriptor()
2647 	return (pmc);  in pmc_allocate_pmc_descriptor()
2654 pmc_destroy_pmc_descriptor(struct pmc *pm)  in pmc_destroy_pmc_descriptor()
2674 pmc_wait_for_pmc_idle(struct pmc *pm)  in pmc_wait_for_pmc_idle()
2711 pmc_release_pmc_descriptor(struct pmc *pm)  in pmc_release_pmc_descriptor()
2853 pmc_register_owner(struct proc *p, struct pmc *pmc)  in pmc_register_owner()  argument
2864 	KASSERT(pmc->pm_owner == NULL,  in pmc_register_owner()
2866 	pmc->pm_owner = po;  in pmc_register_owner()
2868 	LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);  in pmc_register_owner()
2875 		pmclog_process_pmcallocate(pmc);  in pmc_register_owner()
2878 	    po, pmc);  in pmc_register_owner()
2908 	struct pmc *pm;  in pmc_can_allocate_rowindex()
2992 static struct pmc *
2995 	struct pmc *pm;  in pmc_find_pmc_descriptor_in_process()
3010 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)  in pmc_find_pmc()  argument
3012 	struct pmc *pm, *opm;  in pmc_find_pmc()
3046 	*pmc = pm;  in pmc_find_pmc()
3054 pmc_start(struct pmc *pm)  in pmc_start()
3177 pmc_stop(struct pmc *pm)  in pmc_stop()
3290 	struct pmc *pmc;  in pmc_do_op_pmcallocate()  local
3388 	pmc = pmc_allocate_pmc_descriptor();  in pmc_do_op_pmcallocate()
3389 	pmc->pm_id    = PMC_ID_MAKE_ID(cpu, pa->pm_mode, class, PMC_ID_INVALID);  in pmc_do_op_pmcallocate()
3390 	pmc->pm_event = pa->pm_ev;  in pmc_do_op_pmcallocate()
3391 	pmc->pm_state = PMC_STATE_FREE;  in pmc_do_op_pmcallocate()
3392 	pmc->pm_caps  = caps;  in pmc_do_op_pmcallocate()
3393 	pmc->pm_flags = flags;  in pmc_do_op_pmcallocate()
3406 		pmc->pm_sc.pm_reloadcount = MAX(MAX(1, pmc_mincount),  in pmc_do_op_pmcallocate()
3409 		pmc->pm_sc.pm_initial = pa->pm_count;  in pmc_do_op_pmcallocate()
3432 			if (pcd->pcd_allocate_pmc(cpu, adjri, pmc, pa) == 0) {  in pmc_do_op_pmcallocate()
3446 			if (pcd->pcd_allocate_pmc(td->td_oncpu, adjri, pmc,  in pmc_do_op_pmcallocate()
3460 		pmc_destroy_pmc_descriptor(pmc);  in pmc_do_op_pmcallocate()
3465 	pmc->pm_id = PMC_ID_MAKE_ID(cpu, mode, class, n);  in pmc_do_op_pmcallocate()
3468 	    pmc->pm_event, class, mode, n, pmc->pm_id);  in pmc_do_op_pmcallocate()
3471 	if ((pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW)) != 0)  in pmc_do_op_pmcallocate()
3472 		pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;  in pmc_do_op_pmcallocate()
3476 		pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;  in pmc_do_op_pmcallocate()
3481 	if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {  in pmc_do_op_pmcallocate()
3489 		    (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {  in pmc_do_op_pmcallocate()
3490 			(void)pcd->pcd_release_pmc(cpu, adjri, pmc);  in pmc_do_op_pmcallocate()
3491 			pmc_destroy_pmc_descriptor(pmc);  in pmc_do_op_pmcallocate()
3499 	pmc->pm_state = PMC_STATE_ALLOCATED;  in pmc_do_op_pmcallocate()
3500 	pmc->pm_class = class;  in pmc_do_op_pmcallocate()
3513 	error = pmc_register_owner(p, pmc);  in pmc_do_op_pmcallocate()
3515 		pmc_release_pmc_descriptor(pmc);  in pmc_do_op_pmcallocate()
3516 		pmc_destroy_pmc_descriptor(pmc);  in pmc_do_op_pmcallocate()
3523 	pa->pm_pmcid = pmc->pm_id;  in pmc_do_op_pmcallocate()
3533 	struct pmc *pm;  in pmc_do_op_pmcattach()
3591 	struct pmc *pm;  in pmc_do_op_pmcdetach()
3631 	struct pmc *pm;  in pmc_do_op_pmcrelease()
3666 	struct pmc *pm;  in pmc_do_op_pmcrw()
3813 		struct pmc *pm;  in pmc_syscall_handler()
4031 		struct pmc *pm;  in pmc_syscall_handler()
4274 		struct pmc *pm;  in pmc_syscall_handler()
4396 		struct pmc *pm;  in pmc_syscall_handler()
4437 		struct pmc *pm;  in pmc_syscall_handler()
4474 		struct pmc *pm;  in pmc_syscall_handler()
4601 pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf)  in pmc_add_sample()
4694 pmc_process_interrupt(int ring, struct pmc *pm, struct trapframe *tf)  in pmc_process_interrupt()
4715 	struct pmc *pm;  in pmc_capture_user_callchain()
4826 	struct pmc *pm;  in pmc_process_samples()
4970 	struct pmc *pm;  in pmc_process_exit()
5138 	struct pmc *pm;  in pmc_process_fork()
5266 pmc_process_allproc(struct pmc *pm)  in pmc_process_allproc()