xref: /freebsd/sys/dev/hwpmc/hwpmc_mod.c (revision fc9a2b80c4c77096e163cb4a812c41e7e591f2fa)
1ebccf1e3SJoseph Koshy /*-
2ebccf1e3SJoseph Koshy  * Copyright (c) 2003-2005 Joseph Koshy
3ebccf1e3SJoseph Koshy  * All rights reserved.
4ebccf1e3SJoseph Koshy  *
5ebccf1e3SJoseph Koshy  * Redistribution and use in source and binary forms, with or without
6ebccf1e3SJoseph Koshy  * modification, are permitted provided that the following conditions
7ebccf1e3SJoseph Koshy  * are met:
8ebccf1e3SJoseph Koshy  * 1. Redistributions of source code must retain the above copyright
9ebccf1e3SJoseph Koshy  *    notice, this list of conditions and the following disclaimer.
10ebccf1e3SJoseph Koshy  * 2. Redistributions in binary form must reproduce the above copyright
11ebccf1e3SJoseph Koshy  *    notice, this list of conditions and the following disclaimer in the
12ebccf1e3SJoseph Koshy  *    documentation and/or other materials provided with the distribution.
13ebccf1e3SJoseph Koshy  *
14ebccf1e3SJoseph Koshy  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15ebccf1e3SJoseph Koshy  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16ebccf1e3SJoseph Koshy  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17ebccf1e3SJoseph Koshy  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18ebccf1e3SJoseph Koshy  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19ebccf1e3SJoseph Koshy  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20ebccf1e3SJoseph Koshy  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21ebccf1e3SJoseph Koshy  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22ebccf1e3SJoseph Koshy  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23ebccf1e3SJoseph Koshy  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24ebccf1e3SJoseph Koshy  * SUCH DAMAGE.
25ebccf1e3SJoseph Koshy  *
26ebccf1e3SJoseph Koshy  */
27ebccf1e3SJoseph Koshy 
28ebccf1e3SJoseph Koshy #include <sys/cdefs.h>
29ebccf1e3SJoseph Koshy __FBSDID("$FreeBSD$");
30ebccf1e3SJoseph Koshy 
31ebccf1e3SJoseph Koshy #include <sys/param.h>
32ebccf1e3SJoseph Koshy #include <sys/eventhandler.h>
33ebccf1e3SJoseph Koshy #include <sys/jail.h>
34ebccf1e3SJoseph Koshy #include <sys/kernel.h>
35f263522aSJoseph Koshy #include <sys/kthread.h>
36ebccf1e3SJoseph Koshy #include <sys/limits.h>
37ebccf1e3SJoseph Koshy #include <sys/lock.h>
38ebccf1e3SJoseph Koshy #include <sys/malloc.h>
39ebccf1e3SJoseph Koshy #include <sys/module.h>
40ebccf1e3SJoseph Koshy #include <sys/mutex.h>
41ebccf1e3SJoseph Koshy #include <sys/pmc.h>
42ebccf1e3SJoseph Koshy #include <sys/pmckern.h>
43f263522aSJoseph Koshy #include <sys/pmclog.h>
44ebccf1e3SJoseph Koshy #include <sys/proc.h>
45ebccf1e3SJoseph Koshy #include <sys/queue.h>
46f263522aSJoseph Koshy #include <sys/resourcevar.h>
47ebccf1e3SJoseph Koshy #include <sys/sched.h>
48ebccf1e3SJoseph Koshy #include <sys/signalvar.h>
49ebccf1e3SJoseph Koshy #include <sys/smp.h>
50ebccf1e3SJoseph Koshy #include <sys/sx.h>
51ebccf1e3SJoseph Koshy #include <sys/sysctl.h>
52ebccf1e3SJoseph Koshy #include <sys/sysent.h>
53ebccf1e3SJoseph Koshy #include <sys/systm.h>
54f263522aSJoseph Koshy #include <sys/vnode.h>
55ebccf1e3SJoseph Koshy 
56f263522aSJoseph Koshy #include <machine/atomic.h>
57ebccf1e3SJoseph Koshy #include <machine/md_var.h>
58ebccf1e3SJoseph Koshy 
59ebccf1e3SJoseph Koshy /*
60ebccf1e3SJoseph Koshy  * Types
61ebccf1e3SJoseph Koshy  */
62ebccf1e3SJoseph Koshy 
63ebccf1e3SJoseph Koshy enum pmc_flags {
64ebccf1e3SJoseph Koshy 	PMC_FLAG_NONE	  = 0x00, /* do nothing */
65ebccf1e3SJoseph Koshy 	PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
66ebccf1e3SJoseph Koshy 	PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
67ebccf1e3SJoseph Koshy };
68ebccf1e3SJoseph Koshy 
69ebccf1e3SJoseph Koshy /*
70ebccf1e3SJoseph Koshy  * The offset in sysent where the syscall is allocated.
71ebccf1e3SJoseph Koshy  */
72ebccf1e3SJoseph Koshy 
73ebccf1e3SJoseph Koshy static int pmc_syscall_num = NO_SYSCALL;
74ebccf1e3SJoseph Koshy struct pmc_cpu		**pmc_pcpu;	 /* per-cpu state */
75ebccf1e3SJoseph Koshy pmc_value_t		*pmc_pcpu_saved; /* saved PMC values: CSW handling */
76ebccf1e3SJoseph Koshy 
77ebccf1e3SJoseph Koshy #define	PMC_PCPU_SAVED(C,R)	pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
78ebccf1e3SJoseph Koshy 
79ebccf1e3SJoseph Koshy struct mtx_pool		*pmc_mtxpool;
80ebccf1e3SJoseph Koshy static int		*pmc_pmcdisp;	 /* PMC row dispositions */
81ebccf1e3SJoseph Koshy 
82ebccf1e3SJoseph Koshy #define	PMC_ROW_DISP_IS_FREE(R)		(pmc_pmcdisp[(R)] == 0)
83ebccf1e3SJoseph Koshy #define	PMC_ROW_DISP_IS_THREAD(R)	(pmc_pmcdisp[(R)] > 0)
84ebccf1e3SJoseph Koshy #define	PMC_ROW_DISP_IS_STANDALONE(R)	(pmc_pmcdisp[(R)] < 0)
85ebccf1e3SJoseph Koshy 
86ebccf1e3SJoseph Koshy #define	PMC_MARK_ROW_FREE(R) do {					  \
87ebccf1e3SJoseph Koshy 	pmc_pmcdisp[(R)] = 0;						  \
88ebccf1e3SJoseph Koshy } while (0)
89ebccf1e3SJoseph Koshy 
90ebccf1e3SJoseph Koshy #define	PMC_MARK_ROW_STANDALONE(R) do {					  \
91ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
92ebccf1e3SJoseph Koshy 		    __LINE__));						  \
93ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], -1);				  \
94ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] >= (-mp_ncpus), ("[pmc,%d] row "	  \
95ebccf1e3SJoseph Koshy 		"disposition error", __LINE__));			  \
96ebccf1e3SJoseph Koshy } while (0)
97ebccf1e3SJoseph Koshy 
98ebccf1e3SJoseph Koshy #define	PMC_UNMARK_ROW_STANDALONE(R) do { 				  \
99ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], 1);				  \
100ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
101ebccf1e3SJoseph Koshy 		    __LINE__));						  \
102ebccf1e3SJoseph Koshy } while (0)
103ebccf1e3SJoseph Koshy 
104ebccf1e3SJoseph Koshy #define	PMC_MARK_ROW_THREAD(R) do {					  \
105ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
106ebccf1e3SJoseph Koshy 		    __LINE__));						  \
107ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], 1);				  \
108ebccf1e3SJoseph Koshy } while (0)
109ebccf1e3SJoseph Koshy 
110ebccf1e3SJoseph Koshy #define	PMC_UNMARK_ROW_THREAD(R) do {					  \
111ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], -1);				  \
112ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
113ebccf1e3SJoseph Koshy 		    __LINE__));						  \
114ebccf1e3SJoseph Koshy } while (0)
115ebccf1e3SJoseph Koshy 
116ebccf1e3SJoseph Koshy 
117ebccf1e3SJoseph Koshy /* various event handlers */
118ebccf1e3SJoseph Koshy static eventhandler_tag	pmc_exit_tag, pmc_fork_tag;
119ebccf1e3SJoseph Koshy 
120ebccf1e3SJoseph Koshy /* Module statistics */
121ebccf1e3SJoseph Koshy struct pmc_op_getdriverstats pmc_stats;
122ebccf1e3SJoseph Koshy 
123ebccf1e3SJoseph Koshy /* Machine/processor dependent operations */
124ebccf1e3SJoseph Koshy struct pmc_mdep  *md;
125ebccf1e3SJoseph Koshy 
126ebccf1e3SJoseph Koshy /*
127ebccf1e3SJoseph Koshy  * Hash tables mapping owner processes and target threads to PMCs.
128ebccf1e3SJoseph Koshy  */
129ebccf1e3SJoseph Koshy 
130ebccf1e3SJoseph Koshy struct mtx pmc_processhash_mtx;		/* spin mutex */
131ebccf1e3SJoseph Koshy static u_long pmc_processhashmask;
132ebccf1e3SJoseph Koshy static LIST_HEAD(pmc_processhash, pmc_process)	*pmc_processhash;
133ebccf1e3SJoseph Koshy 
134ebccf1e3SJoseph Koshy /*
135ebccf1e3SJoseph Koshy  * Hash table of PMC owner descriptors.  This table is protected by
136ebccf1e3SJoseph Koshy  * the shared PMC "sx" lock.
137ebccf1e3SJoseph Koshy  */
138ebccf1e3SJoseph Koshy 
139ebccf1e3SJoseph Koshy static u_long pmc_ownerhashmask;
140ebccf1e3SJoseph Koshy static LIST_HEAD(pmc_ownerhash, pmc_owner)	*pmc_ownerhash;
141ebccf1e3SJoseph Koshy 
142ebccf1e3SJoseph Koshy /*
143f263522aSJoseph Koshy  * List of PMC owners with system-wide sampling PMCs.
144f263522aSJoseph Koshy  */
145f263522aSJoseph Koshy 
146f263522aSJoseph Koshy static LIST_HEAD(, pmc_owner)			pmc_ss_owners;
147f263522aSJoseph Koshy 
148f263522aSJoseph Koshy 
149f263522aSJoseph Koshy /*
150ebccf1e3SJoseph Koshy  * Prototypes
151ebccf1e3SJoseph Koshy  */
152ebccf1e3SJoseph Koshy 
153342ed5d9SRuslan Ermilov #ifdef	DEBUG
154ebccf1e3SJoseph Koshy static int	pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
155ebccf1e3SJoseph Koshy static int	pmc_debugflags_parse(char *newstr, char *fence);
156ebccf1e3SJoseph Koshy #endif
157ebccf1e3SJoseph Koshy 
158ebccf1e3SJoseph Koshy static int	load(struct module *module, int cmd, void *arg);
159f263522aSJoseph Koshy static int	pmc_attach_process(struct proc *p, struct pmc *pm);
160ebccf1e3SJoseph Koshy static struct pmc *pmc_allocate_pmc_descriptor(void);
161f263522aSJoseph Koshy static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
162f263522aSJoseph Koshy static int	pmc_attach_one_process(struct proc *p, struct pmc *pm);
163c5153e19SJoseph Koshy static int	pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
164c5153e19SJoseph Koshy     int cpu);
165f263522aSJoseph Koshy static int	pmc_can_attach(struct pmc *pm, struct proc *p);
166f263522aSJoseph Koshy static void	pmc_cleanup(void);
167f263522aSJoseph Koshy static int	pmc_detach_process(struct proc *p, struct pmc *pm);
168f263522aSJoseph Koshy static int	pmc_detach_one_process(struct proc *p, struct pmc *pm,
169f263522aSJoseph Koshy     int flags);
170f263522aSJoseph Koshy static void	pmc_destroy_owner_descriptor(struct pmc_owner *po);
171ebccf1e3SJoseph Koshy static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
172ebccf1e3SJoseph Koshy static int	pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
173f263522aSJoseph Koshy static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
174f263522aSJoseph Koshy     pmc_id_t pmc);
175f263522aSJoseph Koshy static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
176f263522aSJoseph Koshy     uint32_t mode);
177c5153e19SJoseph Koshy static void	pmc_force_context_switch(void);
178ebccf1e3SJoseph Koshy static void	pmc_link_target_process(struct pmc *pm,
179ebccf1e3SJoseph Koshy     struct pmc_process *pp);
180f263522aSJoseph Koshy static void	pmc_maybe_remove_owner(struct pmc_owner *po);
181f263522aSJoseph Koshy static void	pmc_process_csw_in(struct thread *td);
182f263522aSJoseph Koshy static void	pmc_process_csw_out(struct thread *td);
183ebccf1e3SJoseph Koshy static void	pmc_process_exit(void *arg, struct proc *p);
184ebccf1e3SJoseph Koshy static void	pmc_process_fork(void *arg, struct proc *p1,
185ebccf1e3SJoseph Koshy     struct proc *p2, int n);
186f263522aSJoseph Koshy static void	pmc_process_samples(int cpu);
187f263522aSJoseph Koshy static void	pmc_release_pmc_descriptor(struct pmc *pmc);
188f263522aSJoseph Koshy static void	pmc_remove_owner(struct pmc_owner *po);
189f263522aSJoseph Koshy static void	pmc_remove_process_descriptor(struct pmc_process *pp);
190f263522aSJoseph Koshy static void	pmc_restore_cpu_binding(struct pmc_binding *pb);
191f263522aSJoseph Koshy static void	pmc_save_cpu_binding(struct pmc_binding *pb);
192f263522aSJoseph Koshy static void	pmc_select_cpu(int cpu);
193ebccf1e3SJoseph Koshy static int	pmc_start(struct pmc *pm);
194ebccf1e3SJoseph Koshy static int	pmc_stop(struct pmc *pm);
195f263522aSJoseph Koshy static int	pmc_syscall_handler(struct thread *td, void *syscall_args);
196f263522aSJoseph Koshy static void	pmc_unlink_target_process(struct pmc *pmc,
197f263522aSJoseph Koshy     struct pmc_process *pp);
198ebccf1e3SJoseph Koshy 
199ebccf1e3SJoseph Koshy /*
200ebccf1e3SJoseph Koshy  * Kernel tunables and sysctl(8) interface.
201ebccf1e3SJoseph Koshy  */
202ebccf1e3SJoseph Koshy 
203ebccf1e3SJoseph Koshy SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
204ebccf1e3SJoseph Koshy 
205342ed5d9SRuslan Ermilov #ifdef	DEBUG
206f263522aSJoseph Koshy struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
207ebccf1e3SJoseph Koshy char	pmc_debugstr[PMC_DEBUG_STRSIZE];
208ebccf1e3SJoseph Koshy TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
209ebccf1e3SJoseph Koshy     sizeof(pmc_debugstr));
210ebccf1e3SJoseph Koshy SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
211ebccf1e3SJoseph Koshy     CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN,
212ebccf1e3SJoseph Koshy     0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
213ebccf1e3SJoseph Koshy #endif
214ebccf1e3SJoseph Koshy 
215ebccf1e3SJoseph Koshy /*
216f263522aSJoseph Koshy  * kern.hwpmc.hashrows -- determines the number of rows in the
217ebccf1e3SJoseph Koshy  * of the hash table used to look up threads
218ebccf1e3SJoseph Koshy  */
219ebccf1e3SJoseph Koshy 
220ebccf1e3SJoseph Koshy static int pmc_hashsize = PMC_HASH_SIZE;
221ebccf1e3SJoseph Koshy TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize);
222ebccf1e3SJoseph Koshy SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD,
223ebccf1e3SJoseph Koshy     &pmc_hashsize, 0, "rows in hash tables");
224ebccf1e3SJoseph Koshy 
225ebccf1e3SJoseph Koshy /*
226f263522aSJoseph Koshy  * kern.hwpmc.nsamples --- number of PC samples per CPU
227ebccf1e3SJoseph Koshy  */
228ebccf1e3SJoseph Koshy 
229f263522aSJoseph Koshy static int pmc_nsamples = PMC_NSAMPLES;
230f263522aSJoseph Koshy TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples);
231f263522aSJoseph Koshy SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD,
232f263522aSJoseph Koshy     &pmc_nsamples, 0, "number of PC samples per CPU");
233ebccf1e3SJoseph Koshy 
234ebccf1e3SJoseph Koshy /*
235f263522aSJoseph Koshy  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
236ebccf1e3SJoseph Koshy  */
237ebccf1e3SJoseph Koshy 
238ebccf1e3SJoseph Koshy static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
239ebccf1e3SJoseph Koshy TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size);
240ebccf1e3SJoseph Koshy SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD,
241ebccf1e3SJoseph Koshy     &pmc_mtxpool_size, 0, "size of spin mutex pool");
242ebccf1e3SJoseph Koshy 
243ebccf1e3SJoseph Koshy 
244ebccf1e3SJoseph Koshy /*
245ebccf1e3SJoseph Koshy  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
246ebccf1e3SJoseph Koshy  * allocate system-wide PMCs.
247ebccf1e3SJoseph Koshy  *
248ebccf1e3SJoseph Koshy  * Allowing unprivileged processes to allocate system PMCs is convenient
249ebccf1e3SJoseph Koshy  * if system-wide measurements need to be taken concurrently with other
250ebccf1e3SJoseph Koshy  * per-process measurements.  This feature is turned off by default.
251ebccf1e3SJoseph Koshy  */
252ebccf1e3SJoseph Koshy 
253ebccf1e3SJoseph Koshy SYSCTL_DECL(_security_bsd);
254ebccf1e3SJoseph Koshy 
255ebccf1e3SJoseph Koshy static int pmc_unprivileged_syspmcs = 0;
256ebccf1e3SJoseph Koshy TUNABLE_INT("security.bsd.unprivileged_syspmcs", &pmc_unprivileged_syspmcs);
257ebccf1e3SJoseph Koshy SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RW,
258ebccf1e3SJoseph Koshy     &pmc_unprivileged_syspmcs, 0,
259ebccf1e3SJoseph Koshy     "allow unprivileged process to allocate system PMCs");
260ebccf1e3SJoseph Koshy 
261f263522aSJoseph Koshy /*
262f263522aSJoseph Koshy  * Hash function.  Discard the lower 2 bits of the pointer since
263f263522aSJoseph Koshy  * these are always zero for our uses.  The hash multiplier is
264f263522aSJoseph Koshy  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
265f263522aSJoseph Koshy  */
266ebccf1e3SJoseph Koshy 
267ebccf1e3SJoseph Koshy #if	LONG_BIT == 64
268ebccf1e3SJoseph Koshy #define	_PMC_HM		11400714819323198486u
269ebccf1e3SJoseph Koshy #elif	LONG_BIT == 32
270ebccf1e3SJoseph Koshy #define	_PMC_HM		2654435769u
271ebccf1e3SJoseph Koshy #else
272ebccf1e3SJoseph Koshy #error 	Must know the size of 'long' to compile
273ebccf1e3SJoseph Koshy #endif
274ebccf1e3SJoseph Koshy 
275ebccf1e3SJoseph Koshy #define	PMC_HASH_PTR(P,M)	((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
276ebccf1e3SJoseph Koshy 
277ebccf1e3SJoseph Koshy /*
278ebccf1e3SJoseph Koshy  * Syscall structures
279ebccf1e3SJoseph Koshy  */
280ebccf1e3SJoseph Koshy 
281ebccf1e3SJoseph Koshy /* The `sysent' for the new syscall */
282ebccf1e3SJoseph Koshy static struct sysent pmc_sysent = {
283ebccf1e3SJoseph Koshy 	2,			/* sy_narg */
284ebccf1e3SJoseph Koshy 	pmc_syscall_handler	/* sy_call */
285ebccf1e3SJoseph Koshy };
286ebccf1e3SJoseph Koshy 
287ebccf1e3SJoseph Koshy static struct syscall_module_data pmc_syscall_mod = {
288ebccf1e3SJoseph Koshy 	load,
289ebccf1e3SJoseph Koshy 	NULL,
290ebccf1e3SJoseph Koshy 	&pmc_syscall_num,
291ebccf1e3SJoseph Koshy 	&pmc_sysent,
292ebccf1e3SJoseph Koshy 	{ 0, NULL }
293ebccf1e3SJoseph Koshy };
294ebccf1e3SJoseph Koshy 
295ebccf1e3SJoseph Koshy static moduledata_t pmc_mod = {
296ebccf1e3SJoseph Koshy 	PMC_MODULE_NAME,
297ebccf1e3SJoseph Koshy 	syscall_module_handler,
298ebccf1e3SJoseph Koshy 	&pmc_syscall_mod
299ebccf1e3SJoseph Koshy };
300ebccf1e3SJoseph Koshy 
301ebccf1e3SJoseph Koshy DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
302ebccf1e3SJoseph Koshy MODULE_VERSION(pmc, PMC_VERSION);
303ebccf1e3SJoseph Koshy 
304342ed5d9SRuslan Ermilov #ifdef	DEBUG
305f263522aSJoseph Koshy enum pmc_dbgparse_state {
306f263522aSJoseph Koshy 	PMCDS_WS,		/* in whitespace */
307f263522aSJoseph Koshy 	PMCDS_MAJOR,		/* seen a major keyword */
308f263522aSJoseph Koshy 	PMCDS_MINOR
309f263522aSJoseph Koshy };
310f263522aSJoseph Koshy 
311ebccf1e3SJoseph Koshy static int
312ebccf1e3SJoseph Koshy pmc_debugflags_parse(char *newstr, char *fence)
313ebccf1e3SJoseph Koshy {
314e1691ef7SJoseph Koshy 	char c, *p, *q;
315f263522aSJoseph Koshy 	struct pmc_debugflags *tmpflags;
316f263522aSJoseph Koshy 	int error, found, *newbits, tmp;
317f263522aSJoseph Koshy 	size_t kwlen;
318ebccf1e3SJoseph Koshy 
319f263522aSJoseph Koshy 	MALLOC(tmpflags, struct pmc_debugflags *, sizeof(*tmpflags),
320f263522aSJoseph Koshy 	    M_PMC, M_WAITOK|M_ZERO);
321ebccf1e3SJoseph Koshy 
322ebccf1e3SJoseph Koshy 	p = newstr;
323f263522aSJoseph Koshy 	error = 0;
324ebccf1e3SJoseph Koshy 
325f263522aSJoseph Koshy 	for (; p < fence && (c = *p); p++) {
326ebccf1e3SJoseph Koshy 
327f263522aSJoseph Koshy 		/* skip white space */
328f263522aSJoseph Koshy 		if (c == ' ' || c == '\t')
329f263522aSJoseph Koshy 			continue;
330f263522aSJoseph Koshy 
331f263522aSJoseph Koshy 		/* look for a keyword followed by "=" */
332f263522aSJoseph Koshy 		for (q = p; p < fence && (c = *p) && c != '='; p++)
333f263522aSJoseph Koshy 			;
334f263522aSJoseph Koshy 		if (c != '=') {
335f263522aSJoseph Koshy 			error = EINVAL;
336f263522aSJoseph Koshy 			goto done;
337ebccf1e3SJoseph Koshy 		}
338ebccf1e3SJoseph Koshy 
339f263522aSJoseph Koshy 		kwlen = p - q;
340f263522aSJoseph Koshy 		newbits = NULL;
341ebccf1e3SJoseph Koshy 
342f263522aSJoseph Koshy 		/* lookup flag group name */
343f263522aSJoseph Koshy #define	DBG_SET_FLAG_MAJ(S,F)						\
344f263522aSJoseph Koshy 		if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)	\
345f263522aSJoseph Koshy 			newbits = &tmpflags->pdb_ ## F;
346ebccf1e3SJoseph Koshy 
347f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("cpu",		CPU);
348f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("csw",		CSW);
349f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("logging",	LOG);
350f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("module",	MOD);
351f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("md", 		MDP);
352f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("owner",	OWN);
353f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("pmc",		PMC);
354f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("process",	PRC);
355f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("sampling", 	SAM);
356ebccf1e3SJoseph Koshy 
357f263522aSJoseph Koshy 		if (newbits == NULL) {
358f263522aSJoseph Koshy 			error = EINVAL;
359f263522aSJoseph Koshy 			goto done;
360ebccf1e3SJoseph Koshy 		}
361ebccf1e3SJoseph Koshy 
362f263522aSJoseph Koshy 		p++;		/* skip the '=' */
363ebccf1e3SJoseph Koshy 
364f263522aSJoseph Koshy 		/* Now parse the individual flags */
365f263522aSJoseph Koshy 		tmp = 0;
366f263522aSJoseph Koshy 	newflag:
367f263522aSJoseph Koshy 		for (q = p; p < fence && (c = *p); p++)
368f263522aSJoseph Koshy 			if (c == ' ' || c == '\t' || c == ',')
369f263522aSJoseph Koshy 				break;
370f263522aSJoseph Koshy 
371f263522aSJoseph Koshy 		/* p == fence or c == ws or c == "," or c == 0 */
372f263522aSJoseph Koshy 
373f263522aSJoseph Koshy 		if ((kwlen = p - q) == 0) {
374f263522aSJoseph Koshy 			*newbits = tmp;
375f263522aSJoseph Koshy 			continue;
376f263522aSJoseph Koshy 		}
377f263522aSJoseph Koshy 
378f263522aSJoseph Koshy 		found = 0;
379f263522aSJoseph Koshy #define	DBG_SET_FLAG_MIN(S,F)						\
380f263522aSJoseph Koshy 		if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)	\
381f263522aSJoseph Koshy 			tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
382f263522aSJoseph Koshy 
383f263522aSJoseph Koshy 		/* a '*' denotes all possible flags in the group */
384f263522aSJoseph Koshy 		if (kwlen == 1 && *q == '*')
385f263522aSJoseph Koshy 			tmp = found = ~0;
386f263522aSJoseph Koshy 		/* look for individual flag names */
387f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("allocaterow", ALR);
388f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("allocate",	ALL);
389f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("attach",	ATT);
390f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("bind",	BND);
391f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("config",	CFG);
392f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("exec",	EXC);
393f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("exit",	EXT);
394f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("find",	FND);
395f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("flush",	FLS);
396f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("fork",	FRK);
397f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("getbuf",	GTB);
398f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("hook",	PMH);
399f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("init",	INI);
400f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("intr",	INT);
401f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("linktarget",	TLK);
402f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("mayberemove", OMR);
403f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("ops",		OPS);
404f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("read",	REA);
405f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("register",	REG);
406f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("release",	REL);
407f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("remove",	ORM);
408f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("sample",	SAM);
409f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("scheduleio",	SIO);
410f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("select",	SEL);
411f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("signal",	SIG);
412f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("swi",		SWI);
413f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("swo",		SWO);
414f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("start",	STA);
415f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("stop",	STO);
416f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("syscall",	PMS);
417f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("unlinktarget", TUL);
418f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("write",	WRI);
419f263522aSJoseph Koshy 		if (found == 0) {
420f263522aSJoseph Koshy 			/* unrecognized flag name */
421f263522aSJoseph Koshy 			error = EINVAL;
422f263522aSJoseph Koshy 			goto done;
423f263522aSJoseph Koshy 		}
424f263522aSJoseph Koshy 
425f263522aSJoseph Koshy 		if (c == 0 || c == ' ' || c == '\t') {	/* end of flag group */
426f263522aSJoseph Koshy 			*newbits = tmp;
427f263522aSJoseph Koshy 			continue;
428f263522aSJoseph Koshy 		}
429f263522aSJoseph Koshy 
430f263522aSJoseph Koshy 		p++;
431f263522aSJoseph Koshy 		goto newflag;
432f263522aSJoseph Koshy 	}
433f263522aSJoseph Koshy 
434f263522aSJoseph Koshy 	/* save the new flag set */
435f263522aSJoseph Koshy 	bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
436f263522aSJoseph Koshy 
437f263522aSJoseph Koshy  done:
438f263522aSJoseph Koshy 	FREE(tmpflags, M_PMC);
439f263522aSJoseph Koshy 	return error;
440ebccf1e3SJoseph Koshy }
441ebccf1e3SJoseph Koshy 
442ebccf1e3SJoseph Koshy static int
443ebccf1e3SJoseph Koshy pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
444ebccf1e3SJoseph Koshy {
445ebccf1e3SJoseph Koshy 	char *fence, *newstr;
446ebccf1e3SJoseph Koshy 	int error;
447ebccf1e3SJoseph Koshy 	unsigned int n;
448ebccf1e3SJoseph Koshy 
449ebccf1e3SJoseph Koshy 	(void) arg1; (void) arg2; /* unused parameters */
450ebccf1e3SJoseph Koshy 
451ebccf1e3SJoseph Koshy 	n = sizeof(pmc_debugstr);
452ebccf1e3SJoseph Koshy 	MALLOC(newstr, char *, n, M_PMC, M_ZERO|M_WAITOK);
453f263522aSJoseph Koshy 	(void) strlcpy(newstr, pmc_debugstr, n);
454ebccf1e3SJoseph Koshy 
455ebccf1e3SJoseph Koshy 	error = sysctl_handle_string(oidp, newstr, n, req);
456ebccf1e3SJoseph Koshy 
457ebccf1e3SJoseph Koshy 	/* if there is a new string, parse and copy it */
458ebccf1e3SJoseph Koshy 	if (error == 0 && req->newptr != NULL) {
459f263522aSJoseph Koshy 		fence = newstr + (n < req->newlen ? n : req->newlen + 1);
460ebccf1e3SJoseph Koshy 		if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
461ebccf1e3SJoseph Koshy 			(void) strlcpy(pmc_debugstr, newstr,
462ebccf1e3SJoseph Koshy 			    sizeof(pmc_debugstr));
463ebccf1e3SJoseph Koshy 	}
464ebccf1e3SJoseph Koshy 
465ebccf1e3SJoseph Koshy 	FREE(newstr, M_PMC);
466ebccf1e3SJoseph Koshy 
467ebccf1e3SJoseph Koshy 	return error;
468ebccf1e3SJoseph Koshy }
469ebccf1e3SJoseph Koshy #endif
470ebccf1e3SJoseph Koshy 
471ebccf1e3SJoseph Koshy /*
472ebccf1e3SJoseph Koshy  * Concurrency Control
473ebccf1e3SJoseph Koshy  *
474ebccf1e3SJoseph Koshy  * The driver manages the following data structures:
475ebccf1e3SJoseph Koshy  *
476ebccf1e3SJoseph Koshy  *   - target process descriptors, one per target process
477ebccf1e3SJoseph Koshy  *   - owner process descriptors (and attached lists), one per owner process
478ebccf1e3SJoseph Koshy  *   - lookup hash tables for owner and target processes
479ebccf1e3SJoseph Koshy  *   - PMC descriptors (and attached lists)
480ebccf1e3SJoseph Koshy  *   - per-cpu hardware state
481ebccf1e3SJoseph Koshy  *   - the 'hook' variable through which the kernel calls into
482ebccf1e3SJoseph Koshy  *     this module
483ebccf1e3SJoseph Koshy  *   - the machine hardware state (managed by the MD layer)
484ebccf1e3SJoseph Koshy  *
485ebccf1e3SJoseph Koshy  * These data structures are accessed from:
486ebccf1e3SJoseph Koshy  *
487ebccf1e3SJoseph Koshy  * - thread context-switch code
488ebccf1e3SJoseph Koshy  * - interrupt handlers (possibly on multiple cpus)
489ebccf1e3SJoseph Koshy  * - kernel threads on multiple cpus running on behalf of user
490ebccf1e3SJoseph Koshy  *   processes doing system calls
491ebccf1e3SJoseph Koshy  * - this driver's private kernel threads
492ebccf1e3SJoseph Koshy  *
493ebccf1e3SJoseph Koshy  * = Locks and Locking strategy =
494ebccf1e3SJoseph Koshy  *
495ebccf1e3SJoseph Koshy  * The driver uses four locking strategies for its operation:
496ebccf1e3SJoseph Koshy  *
497ebccf1e3SJoseph Koshy  * - There is a 'global' SX lock "pmc_sx" that is used to protect
498ebccf1e3SJoseph Koshy  *   the its 'meta-data'.
499ebccf1e3SJoseph Koshy  *
500ebccf1e3SJoseph Koshy  *   Calls into the module (via syscall() or by the kernel) start with
501ebccf1e3SJoseph Koshy  *   this lock being held in exclusive mode.  Depending on the requested
502ebccf1e3SJoseph Koshy  *   operation, the lock may be downgraded to 'shared' mode to allow
503ebccf1e3SJoseph Koshy  *   more concurrent readers into the module.
504ebccf1e3SJoseph Koshy  *
505ebccf1e3SJoseph Koshy  *   This SX lock is held in exclusive mode for any operations that
506ebccf1e3SJoseph Koshy  *   modify the linkages between the driver's internal data structures.
507ebccf1e3SJoseph Koshy  *
508ebccf1e3SJoseph Koshy  *   The 'pmc_hook' function pointer is also protected by this lock.
509ebccf1e3SJoseph Koshy  *   It is only examined with the sx lock held in exclusive mode.  The
510ebccf1e3SJoseph Koshy  *   kernel module is allowed to be unloaded only with the sx lock
511ebccf1e3SJoseph Koshy  *   held in exclusive mode.  In normal syscall handling, after
512ebccf1e3SJoseph Koshy  *   acquiring the pmc_sx lock we first check that 'pmc_hook' is
513ebccf1e3SJoseph Koshy  *   non-null before proceeding.  This prevents races between the
514ebccf1e3SJoseph Koshy  *   thread unloading the module and other threads seeking to use the
515ebccf1e3SJoseph Koshy  *   module.
516ebccf1e3SJoseph Koshy  *
517ebccf1e3SJoseph Koshy  * - Lookups of target process structures and owner process structures
518ebccf1e3SJoseph Koshy  *   cannot use the global "pmc_sx" SX lock because these lookups need
519ebccf1e3SJoseph Koshy  *   to happen during context switches and in other critical sections
520ebccf1e3SJoseph Koshy  *   where sleeping is not allowed.  We protect these lookup tables
521ebccf1e3SJoseph Koshy  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
522ebccf1e3SJoseph Koshy  *   "pmc_ownerhash_mtx".  These are 'leaf' mutexes, in that no other
523ebccf1e3SJoseph Koshy  *   lock is acquired with these locks held.
524ebccf1e3SJoseph Koshy  *
525ebccf1e3SJoseph Koshy  * - Interrupt handlers work in a lock free manner.  At interrupt
526ebccf1e3SJoseph Koshy  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
527ebccf1e3SJoseph Koshy  *   when the PMC was started.  If this pointer is NULL, the interrupt
528ebccf1e3SJoseph Koshy  *   is ignored after updating driver statistics.  We ensure that this
529ebccf1e3SJoseph Koshy  *   pointer is set (using an atomic operation if necessary) before the
530ebccf1e3SJoseph Koshy  *   PMC hardware is started.  Conversely, this pointer is unset atomically
531ebccf1e3SJoseph Koshy  *   only after the PMC hardware is stopped.
532ebccf1e3SJoseph Koshy  *
533ebccf1e3SJoseph Koshy  *   We ensure that everything needed for the operation of an
534ebccf1e3SJoseph Koshy  *   interrupt handler is available without it needing to acquire any
535ebccf1e3SJoseph Koshy  *   locks.  We also ensure that a PMC's software state is destroyed only
536ebccf1e3SJoseph Koshy  *   after the PMC is taken off hardware (on all CPUs).
537ebccf1e3SJoseph Koshy  *
538ebccf1e3SJoseph Koshy  * - Context-switch handling with process-private PMCs needs more
539ebccf1e3SJoseph Koshy  *   care.
540ebccf1e3SJoseph Koshy  *
541ebccf1e3SJoseph Koshy  *   A given process may be the target of multiple PMCs.  For example,
542ebccf1e3SJoseph Koshy  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
543ebccf1e3SJoseph Koshy  *   while the target process is running on another.  A PMC could also
544ebccf1e3SJoseph Koshy  *   be getting released because its owner is exiting.  We tackle
545ebccf1e3SJoseph Koshy  *   these situations in the following manner:
546ebccf1e3SJoseph Koshy  *
547ebccf1e3SJoseph Koshy  *   - each target process structure 'pmc_process' has an array
548ebccf1e3SJoseph Koshy  *     of 'struct pmc *' pointers, one for each hardware PMC.
549ebccf1e3SJoseph Koshy  *
550ebccf1e3SJoseph Koshy  *   - At context switch IN time, each "target" PMC in RUNNING state
551ebccf1e3SJoseph Koshy  *     gets started on hardware and a pointer to each PMC is copied into
552ebccf1e3SJoseph Koshy  *     the per-cpu phw array.  The 'runcount' for the PMC is
553ebccf1e3SJoseph Koshy  *     incremented.
554ebccf1e3SJoseph Koshy  *
555ebccf1e3SJoseph Koshy  *   - At context switch OUT time, all process-virtual PMCs are stopped
556ebccf1e3SJoseph Koshy  *     on hardware.  The saved value is added to the PMCs value field
557ebccf1e3SJoseph Koshy  *     only if the PMC is in a non-deleted state (the PMCs state could
558ebccf1e3SJoseph Koshy  *     have changed during the current time slice).
559ebccf1e3SJoseph Koshy  *
560ebccf1e3SJoseph Koshy  *     Note that since in-between a switch IN on a processor and a switch
561ebccf1e3SJoseph Koshy  *     OUT, the PMC could have been released on another CPU.  Therefore
562ebccf1e3SJoseph Koshy  *     context switch OUT always looks at the hardware state to turn
563ebccf1e3SJoseph Koshy  *     OFF PMCs and will update a PMC's saved value only if reachable
564ebccf1e3SJoseph Koshy  *     from the target process record.
565ebccf1e3SJoseph Koshy  *
566ebccf1e3SJoseph Koshy  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
567ebccf1e3SJoseph Koshy  *     be attached to many processes at the time of the call and could
568ebccf1e3SJoseph Koshy  *     be active on multiple CPUs).
569ebccf1e3SJoseph Koshy  *
570ebccf1e3SJoseph Koshy  *     We prevent further scheduling of the PMC by marking it as in
571ebccf1e3SJoseph Koshy  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
572ebccf1e3SJoseph Koshy  *     this PMC is currently running on a CPU somewhere.  The thread
573ebccf1e3SJoseph Koshy  *     doing the PMCRELEASE operation waits by repeatedly doing an
574ebccf1e3SJoseph Koshy  *     tsleep() till the runcount comes to zero.
575ebccf1e3SJoseph Koshy  *
576ebccf1e3SJoseph Koshy  */
577ebccf1e3SJoseph Koshy 
578ebccf1e3SJoseph Koshy /*
579ebccf1e3SJoseph Koshy  * save the cpu binding of the current kthread
580ebccf1e3SJoseph Koshy  */
581ebccf1e3SJoseph Koshy 
582ebccf1e3SJoseph Koshy static void
583ebccf1e3SJoseph Koshy pmc_save_cpu_binding(struct pmc_binding *pb)
584ebccf1e3SJoseph Koshy {
585ebccf1e3SJoseph Koshy 	PMCDBG(CPU,BND,2, "%s", "save-cpu");
586ebccf1e3SJoseph Koshy 	mtx_lock_spin(&sched_lock);
587ebccf1e3SJoseph Koshy 	pb->pb_bound = sched_is_bound(curthread);
588ebccf1e3SJoseph Koshy 	pb->pb_cpu   = curthread->td_oncpu;
589ebccf1e3SJoseph Koshy 	mtx_unlock_spin(&sched_lock);
590ebccf1e3SJoseph Koshy 	PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
591ebccf1e3SJoseph Koshy }
592ebccf1e3SJoseph Koshy 
593ebccf1e3SJoseph Koshy /*
594ebccf1e3SJoseph Koshy  * restore the cpu binding of the current thread
595ebccf1e3SJoseph Koshy  */
596ebccf1e3SJoseph Koshy 
597ebccf1e3SJoseph Koshy static void
598ebccf1e3SJoseph Koshy pmc_restore_cpu_binding(struct pmc_binding *pb)
599ebccf1e3SJoseph Koshy {
600ebccf1e3SJoseph Koshy 	PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
601ebccf1e3SJoseph Koshy 	    curthread->td_oncpu, pb->pb_cpu);
602ebccf1e3SJoseph Koshy 	mtx_lock_spin(&sched_lock);
603ebccf1e3SJoseph Koshy 	if (pb->pb_bound)
604ebccf1e3SJoseph Koshy 		sched_bind(curthread, pb->pb_cpu);
605ebccf1e3SJoseph Koshy 	else
606ebccf1e3SJoseph Koshy 		sched_unbind(curthread);
607ebccf1e3SJoseph Koshy 	mtx_unlock_spin(&sched_lock);
608ebccf1e3SJoseph Koshy 	PMCDBG(CPU,BND,2, "%s", "restore-cpu done");
609ebccf1e3SJoseph Koshy }
610ebccf1e3SJoseph Koshy 
611ebccf1e3SJoseph Koshy /*
612ebccf1e3SJoseph Koshy  * move execution over the specified cpu and bind it there.
613ebccf1e3SJoseph Koshy  */
614ebccf1e3SJoseph Koshy 
615ebccf1e3SJoseph Koshy static void
616ebccf1e3SJoseph Koshy pmc_select_cpu(int cpu)
617ebccf1e3SJoseph Koshy {
618ebccf1e3SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < mp_ncpus,
619ebccf1e3SJoseph Koshy 	    ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
620ebccf1e3SJoseph Koshy 
621ebccf1e3SJoseph Koshy 	/* never move to a disabled CPU */
622ebccf1e3SJoseph Koshy 	KASSERT(pmc_cpu_is_disabled(cpu) == 0, ("[pmc,%d] selecting "
623ebccf1e3SJoseph Koshy 	    "disabled CPU %d", __LINE__, cpu));
624ebccf1e3SJoseph Koshy 
625ebccf1e3SJoseph Koshy 	PMCDBG(CPU,SEL,2, "select-cpu cpu=%d", cpu);
626ebccf1e3SJoseph Koshy 	mtx_lock_spin(&sched_lock);
627ebccf1e3SJoseph Koshy 	sched_bind(curthread, cpu);
628ebccf1e3SJoseph Koshy 	mtx_unlock_spin(&sched_lock);
629ebccf1e3SJoseph Koshy 
630ebccf1e3SJoseph Koshy 	KASSERT(curthread->td_oncpu == cpu,
631ebccf1e3SJoseph Koshy 	    ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
632ebccf1e3SJoseph Koshy 		cpu, curthread->td_oncpu));
633ebccf1e3SJoseph Koshy 
634ebccf1e3SJoseph Koshy 	PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
635ebccf1e3SJoseph Koshy }
636ebccf1e3SJoseph Koshy 
637ebccf1e3SJoseph Koshy /*
638c5153e19SJoseph Koshy  * Force a context switch.
639c5153e19SJoseph Koshy  *
640c5153e19SJoseph Koshy  * We do this by tsleep'ing for 1 tick -- invoking mi_switch() is not
641c5153e19SJoseph Koshy  * guaranteed to force a context switch.
642c5153e19SJoseph Koshy  */
643c5153e19SJoseph Koshy 
644c5153e19SJoseph Koshy static void
645c5153e19SJoseph Koshy pmc_force_context_switch(void)
646c5153e19SJoseph Koshy {
647c5153e19SJoseph Koshy 	u_char	curpri;
648c5153e19SJoseph Koshy 
649c5153e19SJoseph Koshy 	mtx_lock_spin(&sched_lock);
650c5153e19SJoseph Koshy 	curpri = curthread->td_priority;
651c5153e19SJoseph Koshy 	mtx_unlock_spin(&sched_lock);
652c5153e19SJoseph Koshy 
653c5153e19SJoseph Koshy 	(void) tsleep((void *) pmc_force_context_switch, curpri,
654c5153e19SJoseph Koshy 	    "pmcctx", 1);
655c5153e19SJoseph Koshy 
656c5153e19SJoseph Koshy }
657c5153e19SJoseph Koshy 
658c5153e19SJoseph Koshy /*
659f263522aSJoseph Koshy  * Get the file name for an executable.  This is a simple wrapper
660f263522aSJoseph Koshy  * around vn_fullpath(9).
661ebccf1e3SJoseph Koshy  */
662ebccf1e3SJoseph Koshy 
663f263522aSJoseph Koshy static void
66415139246SJoseph Koshy pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
665ebccf1e3SJoseph Koshy {
666ebccf1e3SJoseph Koshy 	struct thread *td;
667ebccf1e3SJoseph Koshy 
668f263522aSJoseph Koshy 	td = curthread;
669f263522aSJoseph Koshy 	*fullpath = "unknown";
670f263522aSJoseph Koshy 	*freepath = NULL;
671d9dcd4f9SJoseph Koshy 	vn_lock(v, LK_CANRECURSE | LK_EXCLUSIVE | LK_RETRY, td);
67215139246SJoseph Koshy 	vn_fullpath(td, v, fullpath, freepath);
67315139246SJoseph Koshy 	VOP_UNLOCK(v, 0, td);
674ebccf1e3SJoseph Koshy }
675ebccf1e3SJoseph Koshy 
676ebccf1e3SJoseph Koshy /*
677ebccf1e3SJoseph Koshy  * remove an process owning PMCs
678ebccf1e3SJoseph Koshy  */
679ebccf1e3SJoseph Koshy 
680ebccf1e3SJoseph Koshy void
681ebccf1e3SJoseph Koshy pmc_remove_owner(struct pmc_owner *po)
682ebccf1e3SJoseph Koshy {
683f263522aSJoseph Koshy 	struct pmc *pm, *tmp;
684ebccf1e3SJoseph Koshy 
685ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
686ebccf1e3SJoseph Koshy 
687ebccf1e3SJoseph Koshy 	PMCDBG(OWN,ORM,1, "remove-owner po=%p", po);
688ebccf1e3SJoseph Koshy 
689ebccf1e3SJoseph Koshy 	/* Remove descriptor from the owner hash table */
690ebccf1e3SJoseph Koshy 	LIST_REMOVE(po, po_next);
691ebccf1e3SJoseph Koshy 
692f263522aSJoseph Koshy 	/* release all owned PMC descriptors */
693f263522aSJoseph Koshy 	LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
694f263522aSJoseph Koshy 		PMCDBG(OWN,ORM,2, "pmc=%p", pm);
695f263522aSJoseph Koshy 		KASSERT(pm->pm_owner == po,
696f263522aSJoseph Koshy 		    ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
697ebccf1e3SJoseph Koshy 
698f263522aSJoseph Koshy 		pmc_release_pmc_descriptor(pm);	/* will unlink from the list */
699ebccf1e3SJoseph Koshy 	}
700ebccf1e3SJoseph Koshy 
701f263522aSJoseph Koshy 	KASSERT(po->po_sscount == 0,
702f263522aSJoseph Koshy 	    ("[pmc,%d] SS count not zero", __LINE__));
703ebccf1e3SJoseph Koshy 	KASSERT(LIST_EMPTY(&po->po_pmcs),
704ebccf1e3SJoseph Koshy 	    ("[pmc,%d] PMC list not empty", __LINE__));
705ebccf1e3SJoseph Koshy 
706f263522aSJoseph Koshy 	/* de-configure the log file if present */
707c5153e19SJoseph Koshy 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
708f263522aSJoseph Koshy 		pmclog_deconfigure_log(po);
709ebccf1e3SJoseph Koshy }
710ebccf1e3SJoseph Koshy 
711ebccf1e3SJoseph Koshy /*
712ebccf1e3SJoseph Koshy  * remove an owner process record if all conditions are met.
713ebccf1e3SJoseph Koshy  */
714ebccf1e3SJoseph Koshy 
715ebccf1e3SJoseph Koshy static void
716ebccf1e3SJoseph Koshy pmc_maybe_remove_owner(struct pmc_owner *po)
717ebccf1e3SJoseph Koshy {
718ebccf1e3SJoseph Koshy 
719ebccf1e3SJoseph Koshy 	PMCDBG(OWN,OMR,1, "maybe-remove-owner po=%p", po);
720ebccf1e3SJoseph Koshy 
721ebccf1e3SJoseph Koshy 	/*
722ebccf1e3SJoseph Koshy 	 * Remove owner record if
723ebccf1e3SJoseph Koshy 	 * - this process does not own any PMCs
724ebccf1e3SJoseph Koshy 	 * - this process has not allocated a system-wide sampling buffer
725ebccf1e3SJoseph Koshy 	 */
726ebccf1e3SJoseph Koshy 
727ebccf1e3SJoseph Koshy 	if (LIST_EMPTY(&po->po_pmcs) &&
728c5153e19SJoseph Koshy 	    ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
729ebccf1e3SJoseph Koshy 		pmc_remove_owner(po);
730f263522aSJoseph Koshy 		pmc_destroy_owner_descriptor(po);
731ebccf1e3SJoseph Koshy 	}
732ebccf1e3SJoseph Koshy }
733ebccf1e3SJoseph Koshy 
734ebccf1e3SJoseph Koshy /*
735ebccf1e3SJoseph Koshy  * Add an association between a target process and a PMC.
736ebccf1e3SJoseph Koshy  */
737ebccf1e3SJoseph Koshy 
738ebccf1e3SJoseph Koshy static void
739ebccf1e3SJoseph Koshy pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
740ebccf1e3SJoseph Koshy {
741ebccf1e3SJoseph Koshy 	int ri;
742ebccf1e3SJoseph Koshy 	struct pmc_target *pt;
743ebccf1e3SJoseph Koshy 
744ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
745ebccf1e3SJoseph Koshy 
746ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL && pp != NULL,
747ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
748f263522aSJoseph Koshy 	KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
749f263522aSJoseph Koshy 	    ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
750f263522aSJoseph Koshy 		__LINE__, pm, pp->pp_proc->p_pid));
751ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1),
752ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Illegal reference count %d for process record %p",
753ebccf1e3SJoseph Koshy 		__LINE__, pp->pp_refcnt, (void *) pp));
754ebccf1e3SJoseph Koshy 
755c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
756ebccf1e3SJoseph Koshy 
757ebccf1e3SJoseph Koshy 	PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
758ebccf1e3SJoseph Koshy 	    pm, ri, pp);
759ebccf1e3SJoseph Koshy 
760342ed5d9SRuslan Ermilov #ifdef	DEBUG
761ebccf1e3SJoseph Koshy 	LIST_FOREACH(pt, &pm->pm_targets, pt_next)
762ebccf1e3SJoseph Koshy 	    if (pt->pt_process == pp)
763ebccf1e3SJoseph Koshy 		    KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
764ebccf1e3SJoseph Koshy 				__LINE__, pp, pm));
765ebccf1e3SJoseph Koshy #endif
766ebccf1e3SJoseph Koshy 
767ebccf1e3SJoseph Koshy 	MALLOC(pt, struct pmc_target *, sizeof(struct pmc_target),
768ebccf1e3SJoseph Koshy 	    M_PMC, M_ZERO|M_WAITOK);
769ebccf1e3SJoseph Koshy 
770ebccf1e3SJoseph Koshy 	pt->pt_process = pp;
771ebccf1e3SJoseph Koshy 
772ebccf1e3SJoseph Koshy 	LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
773ebccf1e3SJoseph Koshy 
774122eceefSJohn Baldwin 	atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
775122eceefSJohn Baldwin 	    (uintptr_t)pm);
776ebccf1e3SJoseph Koshy 
7776b8c8cd8SJoseph Koshy 	if (pm->pm_owner->po_owner == pp->pp_proc)
778c5153e19SJoseph Koshy 		pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
7796b8c8cd8SJoseph Koshy 
780f263522aSJoseph Koshy 	/*
781f263522aSJoseph Koshy 	 * Initialize the per-process values at this row index.
782f263522aSJoseph Koshy 	 */
783f263522aSJoseph Koshy 	pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
784f263522aSJoseph Koshy 	    pm->pm_sc.pm_reloadcount : 0;
785f263522aSJoseph Koshy 
786ebccf1e3SJoseph Koshy 	pp->pp_refcnt++;
787ebccf1e3SJoseph Koshy 
788ebccf1e3SJoseph Koshy }
789ebccf1e3SJoseph Koshy 
790ebccf1e3SJoseph Koshy /*
791ebccf1e3SJoseph Koshy  * Removes the association between a target process and a PMC.
792ebccf1e3SJoseph Koshy  */
793ebccf1e3SJoseph Koshy 
794ebccf1e3SJoseph Koshy static void
795ebccf1e3SJoseph Koshy pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
796ebccf1e3SJoseph Koshy {
797ebccf1e3SJoseph Koshy 	int ri;
798f263522aSJoseph Koshy 	struct proc *p;
799ebccf1e3SJoseph Koshy 	struct pmc_target *ptgt;
800ebccf1e3SJoseph Koshy 
801ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
802ebccf1e3SJoseph Koshy 
803ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL && pp != NULL,
804ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
805ebccf1e3SJoseph Koshy 
806ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc,
807ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Illegal ref count %d on process record %p",
808ebccf1e3SJoseph Koshy 		__LINE__, pp->pp_refcnt, (void *) pp));
809ebccf1e3SJoseph Koshy 
810c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
811ebccf1e3SJoseph Koshy 
812ebccf1e3SJoseph Koshy 	PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
813ebccf1e3SJoseph Koshy 	    pm, ri, pp);
814ebccf1e3SJoseph Koshy 
815ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
816ebccf1e3SJoseph Koshy 	    ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
817ebccf1e3SJoseph Koshy 		ri, pm, pp->pp_pmcs[ri].pp_pmc));
818ebccf1e3SJoseph Koshy 
819ebccf1e3SJoseph Koshy 	pp->pp_pmcs[ri].pp_pmc = NULL;
820ebccf1e3SJoseph Koshy 	pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
821ebccf1e3SJoseph Koshy 
822c5153e19SJoseph Koshy 	/* Remove owner-specific flags */
823c5153e19SJoseph Koshy 	if (pm->pm_owner->po_owner == pp->pp_proc) {
824c5153e19SJoseph Koshy 		pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
825c5153e19SJoseph Koshy 		pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
826c5153e19SJoseph Koshy 	}
8276b8c8cd8SJoseph Koshy 
828ebccf1e3SJoseph Koshy 	pp->pp_refcnt--;
829ebccf1e3SJoseph Koshy 
830ebccf1e3SJoseph Koshy 	/* Remove the target process from the PMC structure */
831ebccf1e3SJoseph Koshy 	LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
832ebccf1e3SJoseph Koshy 		if (ptgt->pt_process == pp)
833ebccf1e3SJoseph Koshy 			break;
834ebccf1e3SJoseph Koshy 
835ebccf1e3SJoseph Koshy 	KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
836ebccf1e3SJoseph Koshy 		    "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
837ebccf1e3SJoseph Koshy 
838ebccf1e3SJoseph Koshy 	LIST_REMOVE(ptgt, pt_next);
839ebccf1e3SJoseph Koshy 	FREE(ptgt, M_PMC);
840f263522aSJoseph Koshy 
841f263522aSJoseph Koshy 	/* if the PMC now lacks targets, send the owner a SIGIO */
842f263522aSJoseph Koshy 	if (LIST_EMPTY(&pm->pm_targets)) {
843f263522aSJoseph Koshy 		p = pm->pm_owner->po_owner;
844f263522aSJoseph Koshy 		PROC_LOCK(p);
845f263522aSJoseph Koshy 		psignal(p, SIGIO);
846f263522aSJoseph Koshy 		PROC_UNLOCK(p);
847f263522aSJoseph Koshy 
848f263522aSJoseph Koshy 		PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p,
849f263522aSJoseph Koshy 		    SIGIO);
850ebccf1e3SJoseph Koshy 	}
851ebccf1e3SJoseph Koshy }
852ebccf1e3SJoseph Koshy 
853ebccf1e3SJoseph Koshy /*
854ebccf1e3SJoseph Koshy  * Check if PMC 'pm' may be attached to target process 't'.
855ebccf1e3SJoseph Koshy  */
856ebccf1e3SJoseph Koshy 
857ebccf1e3SJoseph Koshy static int
858ebccf1e3SJoseph Koshy pmc_can_attach(struct pmc *pm, struct proc *t)
859ebccf1e3SJoseph Koshy {
860ebccf1e3SJoseph Koshy 	struct proc *o;		/* pmc owner */
861ebccf1e3SJoseph Koshy 	struct ucred *oc, *tc;	/* owner, target credentials */
862ebccf1e3SJoseph Koshy 	int decline_attach, i;
863ebccf1e3SJoseph Koshy 
864ebccf1e3SJoseph Koshy 	/*
865ebccf1e3SJoseph Koshy 	 * A PMC's owner can always attach that PMC to itself.
866ebccf1e3SJoseph Koshy 	 */
867ebccf1e3SJoseph Koshy 
868ebccf1e3SJoseph Koshy 	if ((o = pm->pm_owner->po_owner) == t)
869ebccf1e3SJoseph Koshy 		return 0;
870ebccf1e3SJoseph Koshy 
871ebccf1e3SJoseph Koshy 	PROC_LOCK(o);
872ebccf1e3SJoseph Koshy 	oc = o->p_ucred;
873ebccf1e3SJoseph Koshy 	crhold(oc);
874ebccf1e3SJoseph Koshy 	PROC_UNLOCK(o);
875ebccf1e3SJoseph Koshy 
876ebccf1e3SJoseph Koshy 	PROC_LOCK(t);
877ebccf1e3SJoseph Koshy 	tc = t->p_ucred;
878ebccf1e3SJoseph Koshy 	crhold(tc);
879ebccf1e3SJoseph Koshy 	PROC_UNLOCK(t);
880ebccf1e3SJoseph Koshy 
881ebccf1e3SJoseph Koshy 	/*
882ebccf1e3SJoseph Koshy 	 * The effective uid of the PMC owner should match at least one
883ebccf1e3SJoseph Koshy 	 * of the {effective,real,saved} uids of the target process.
884ebccf1e3SJoseph Koshy 	 */
885ebccf1e3SJoseph Koshy 
886ebccf1e3SJoseph Koshy 	decline_attach = oc->cr_uid != tc->cr_uid &&
887ebccf1e3SJoseph Koshy 	    oc->cr_uid != tc->cr_svuid &&
888ebccf1e3SJoseph Koshy 	    oc->cr_uid != tc->cr_ruid;
889ebccf1e3SJoseph Koshy 
890ebccf1e3SJoseph Koshy 	/*
891ebccf1e3SJoseph Koshy 	 * Every one of the target's group ids, must be in the owner's
892ebccf1e3SJoseph Koshy 	 * group list.
893ebccf1e3SJoseph Koshy 	 */
894ebccf1e3SJoseph Koshy 	for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
895ebccf1e3SJoseph Koshy 		decline_attach = !groupmember(tc->cr_groups[i], oc);
896ebccf1e3SJoseph Koshy 
897ebccf1e3SJoseph Koshy 	/* check the read and saved gids too */
898ebccf1e3SJoseph Koshy 	if (decline_attach == 0)
899ebccf1e3SJoseph Koshy 		decline_attach = !groupmember(tc->cr_rgid, oc) ||
900ebccf1e3SJoseph Koshy 		    !groupmember(tc->cr_svgid, oc);
901ebccf1e3SJoseph Koshy 
902ebccf1e3SJoseph Koshy 	crfree(tc);
903ebccf1e3SJoseph Koshy 	crfree(oc);
904ebccf1e3SJoseph Koshy 
905ebccf1e3SJoseph Koshy 	return !decline_attach;
906ebccf1e3SJoseph Koshy }
907ebccf1e3SJoseph Koshy 
908ebccf1e3SJoseph Koshy /*
909ebccf1e3SJoseph Koshy  * Attach a process to a PMC.
910ebccf1e3SJoseph Koshy  */
911ebccf1e3SJoseph Koshy 
912ebccf1e3SJoseph Koshy static int
913ebccf1e3SJoseph Koshy pmc_attach_one_process(struct proc *p, struct pmc *pm)
914ebccf1e3SJoseph Koshy {
915ebccf1e3SJoseph Koshy 	int ri;
916f263522aSJoseph Koshy 	char *fullpath, *freepath;
917ebccf1e3SJoseph Koshy 	struct pmc_process	*pp;
918ebccf1e3SJoseph Koshy 
919ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
920ebccf1e3SJoseph Koshy 
921ebccf1e3SJoseph Koshy 	PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
922c5153e19SJoseph Koshy 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
923ebccf1e3SJoseph Koshy 
924ebccf1e3SJoseph Koshy 	/*
925ebccf1e3SJoseph Koshy 	 * Locate the process descriptor corresponding to process 'p',
926ebccf1e3SJoseph Koshy 	 * allocating space as needed.
927ebccf1e3SJoseph Koshy 	 *
928ebccf1e3SJoseph Koshy 	 * Verify that rowindex 'pm_rowindex' is free in the process
929ebccf1e3SJoseph Koshy 	 * descriptor.
930ebccf1e3SJoseph Koshy 	 *
931ebccf1e3SJoseph Koshy 	 * If not, allocate space for a descriptor and link the
932ebccf1e3SJoseph Koshy 	 * process descriptor and PMC.
933ebccf1e3SJoseph Koshy 	 */
934c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
935ebccf1e3SJoseph Koshy 
936ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
937ebccf1e3SJoseph Koshy 		return ENOMEM;
938ebccf1e3SJoseph Koshy 
939ebccf1e3SJoseph Koshy 	if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
940ebccf1e3SJoseph Koshy 		return EEXIST;
941ebccf1e3SJoseph Koshy 
942ebccf1e3SJoseph Koshy 	if (pp->pp_pmcs[ri].pp_pmc != NULL)
943ebccf1e3SJoseph Koshy 		return EBUSY;
944ebccf1e3SJoseph Koshy 
945ebccf1e3SJoseph Koshy 	pmc_link_target_process(pm, pp);
946ebccf1e3SJoseph Koshy 
947f263522aSJoseph Koshy 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
948f263522aSJoseph Koshy 	    (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
949f263522aSJoseph Koshy 		pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
950f263522aSJoseph Koshy 
951f263522aSJoseph Koshy 	pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
952f263522aSJoseph Koshy 
953f263522aSJoseph Koshy 	/* issue an attach event to a configured log file */
954f263522aSJoseph Koshy 	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
95515139246SJoseph Koshy 		pmc_getfilename(p->p_textvp, &fullpath, &freepath);
956f263522aSJoseph Koshy 		pmclog_process_pmcattach(pm, p->p_pid, fullpath);
957f263522aSJoseph Koshy 		if (freepath)
958f263522aSJoseph Koshy 			FREE(freepath, M_TEMP);
959f263522aSJoseph Koshy 	}
960ebccf1e3SJoseph Koshy 	/* mark process as using HWPMCs */
961ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
962ebccf1e3SJoseph Koshy 	p->p_flag |= P_HWPMC;
963ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
964ebccf1e3SJoseph Koshy 
965ebccf1e3SJoseph Koshy 	return 0;
966ebccf1e3SJoseph Koshy }
967ebccf1e3SJoseph Koshy 
968ebccf1e3SJoseph Koshy /*
969ebccf1e3SJoseph Koshy  * Attach a process and optionally its children
970ebccf1e3SJoseph Koshy  */
971ebccf1e3SJoseph Koshy 
972ebccf1e3SJoseph Koshy static int
973ebccf1e3SJoseph Koshy pmc_attach_process(struct proc *p, struct pmc *pm)
974ebccf1e3SJoseph Koshy {
975ebccf1e3SJoseph Koshy 	int error;
976ebccf1e3SJoseph Koshy 	struct proc *top;
977ebccf1e3SJoseph Koshy 
978ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
979ebccf1e3SJoseph Koshy 
980ebccf1e3SJoseph Koshy 	PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
981c5153e19SJoseph Koshy 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
982c5153e19SJoseph Koshy 
983c5153e19SJoseph Koshy 
984c5153e19SJoseph Koshy 	/*
985c5153e19SJoseph Koshy 	 * If this PMC successfully allowed a GETMSR operation
986c5153e19SJoseph Koshy 	 * in the past, disallow further ATTACHes.
987c5153e19SJoseph Koshy 	 */
988c5153e19SJoseph Koshy 
989c5153e19SJoseph Koshy 	if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
990c5153e19SJoseph Koshy 		return EPERM;
991ebccf1e3SJoseph Koshy 
992ebccf1e3SJoseph Koshy 	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
993ebccf1e3SJoseph Koshy 		return pmc_attach_one_process(p, pm);
994ebccf1e3SJoseph Koshy 
995ebccf1e3SJoseph Koshy 	/*
996ebccf1e3SJoseph Koshy 	 * Traverse all child processes, attaching them to
997ebccf1e3SJoseph Koshy 	 * this PMC.
998ebccf1e3SJoseph Koshy 	 */
999ebccf1e3SJoseph Koshy 
1000ebccf1e3SJoseph Koshy 	sx_slock(&proctree_lock);
1001ebccf1e3SJoseph Koshy 
1002ebccf1e3SJoseph Koshy 	top = p;
1003ebccf1e3SJoseph Koshy 
1004ebccf1e3SJoseph Koshy 	for (;;) {
1005ebccf1e3SJoseph Koshy 		if ((error = pmc_attach_one_process(p, pm)) != 0)
1006ebccf1e3SJoseph Koshy 			break;
1007ebccf1e3SJoseph Koshy 		if (!LIST_EMPTY(&p->p_children))
1008ebccf1e3SJoseph Koshy 			p = LIST_FIRST(&p->p_children);
1009ebccf1e3SJoseph Koshy 		else for (;;) {
1010ebccf1e3SJoseph Koshy 			if (p == top)
1011ebccf1e3SJoseph Koshy 				goto done;
1012ebccf1e3SJoseph Koshy 			if (LIST_NEXT(p, p_sibling)) {
1013ebccf1e3SJoseph Koshy 				p = LIST_NEXT(p, p_sibling);
1014ebccf1e3SJoseph Koshy 				break;
1015ebccf1e3SJoseph Koshy 			}
1016ebccf1e3SJoseph Koshy 			p = p->p_pptr;
1017ebccf1e3SJoseph Koshy 		}
1018ebccf1e3SJoseph Koshy 	}
1019ebccf1e3SJoseph Koshy 
1020ebccf1e3SJoseph Koshy 	if (error)
1021ebccf1e3SJoseph Koshy 		(void) pmc_detach_process(top, pm);
1022ebccf1e3SJoseph Koshy 
1023ebccf1e3SJoseph Koshy  done:
1024ebccf1e3SJoseph Koshy 	sx_sunlock(&proctree_lock);
1025ebccf1e3SJoseph Koshy 	return error;
1026ebccf1e3SJoseph Koshy }
1027ebccf1e3SJoseph Koshy 
1028ebccf1e3SJoseph Koshy /*
1029ebccf1e3SJoseph Koshy  * Detach a process from a PMC.  If there are no other PMCs tracking
1030ebccf1e3SJoseph Koshy  * this process, remove the process structure from its hash table.  If
1031ebccf1e3SJoseph Koshy  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1032ebccf1e3SJoseph Koshy  */
1033ebccf1e3SJoseph Koshy 
1034ebccf1e3SJoseph Koshy static int
1035ebccf1e3SJoseph Koshy pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1036ebccf1e3SJoseph Koshy {
1037ebccf1e3SJoseph Koshy 	int ri;
1038ebccf1e3SJoseph Koshy 	struct pmc_process *pp;
1039ebccf1e3SJoseph Koshy 
1040ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1041ebccf1e3SJoseph Koshy 
1042ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL,
1043ebccf1e3SJoseph Koshy 	    ("[pmc,%d] null pm pointer", __LINE__));
1044ebccf1e3SJoseph Koshy 
1045c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
1046ebccf1e3SJoseph Koshy 
1047c5153e19SJoseph Koshy 	PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1048c5153e19SJoseph Koshy 	    pm, ri, p, p->p_pid, p->p_comm, flags);
1049ebccf1e3SJoseph Koshy 
1050ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1051ebccf1e3SJoseph Koshy 		return ESRCH;
1052ebccf1e3SJoseph Koshy 
1053ebccf1e3SJoseph Koshy 	if (pp->pp_pmcs[ri].pp_pmc != pm)
1054ebccf1e3SJoseph Koshy 		return EINVAL;
1055ebccf1e3SJoseph Koshy 
1056ebccf1e3SJoseph Koshy 	pmc_unlink_target_process(pm, pp);
1057ebccf1e3SJoseph Koshy 
1058f263522aSJoseph Koshy 	/* Issue a detach entry if a log file is configured */
1059f263522aSJoseph Koshy 	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1060f263522aSJoseph Koshy 		pmclog_process_pmcdetach(pm, p->p_pid);
1061f263522aSJoseph Koshy 
1062ebccf1e3SJoseph Koshy 	/*
1063ebccf1e3SJoseph Koshy 	 * If there are no PMCs targetting this process, we remove its
1064ebccf1e3SJoseph Koshy 	 * descriptor from the target hash table and unset the P_HWPMC
1065ebccf1e3SJoseph Koshy 	 * flag in the struct proc.
1066ebccf1e3SJoseph Koshy 	 */
1067ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc,
1068ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Illegal refcnt %d for process struct %p",
1069ebccf1e3SJoseph Koshy 		__LINE__, pp->pp_refcnt, pp));
1070ebccf1e3SJoseph Koshy 
1071ebccf1e3SJoseph Koshy 	if (pp->pp_refcnt != 0)	/* still a target of some PMC */
1072ebccf1e3SJoseph Koshy 		return 0;
1073ebccf1e3SJoseph Koshy 
1074ebccf1e3SJoseph Koshy 	pmc_remove_process_descriptor(pp);
1075ebccf1e3SJoseph Koshy 
1076ebccf1e3SJoseph Koshy 	if (flags & PMC_FLAG_REMOVE)
1077ebccf1e3SJoseph Koshy 		FREE(pp, M_PMC);
1078ebccf1e3SJoseph Koshy 
1079ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
1080ebccf1e3SJoseph Koshy 	p->p_flag &= ~P_HWPMC;
1081ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
1082ebccf1e3SJoseph Koshy 
1083ebccf1e3SJoseph Koshy 	return 0;
1084ebccf1e3SJoseph Koshy }
1085ebccf1e3SJoseph Koshy 
1086ebccf1e3SJoseph Koshy /*
1087ebccf1e3SJoseph Koshy  * Detach a process and optionally its descendants from a PMC.
1088ebccf1e3SJoseph Koshy  */
1089ebccf1e3SJoseph Koshy 
1090ebccf1e3SJoseph Koshy static int
1091ebccf1e3SJoseph Koshy pmc_detach_process(struct proc *p, struct pmc *pm)
1092ebccf1e3SJoseph Koshy {
1093ebccf1e3SJoseph Koshy 	struct proc *top;
1094ebccf1e3SJoseph Koshy 
1095ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1096ebccf1e3SJoseph Koshy 
1097ebccf1e3SJoseph Koshy 	PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1098c5153e19SJoseph Koshy 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1099ebccf1e3SJoseph Koshy 
1100ebccf1e3SJoseph Koshy 	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1101ebccf1e3SJoseph Koshy 		return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1102ebccf1e3SJoseph Koshy 
1103ebccf1e3SJoseph Koshy 	/*
1104ebccf1e3SJoseph Koshy 	 * Traverse all children, detaching them from this PMC.  We
1105ebccf1e3SJoseph Koshy 	 * ignore errors since we could be detaching a PMC from a
1106ebccf1e3SJoseph Koshy 	 * partially attached proc tree.
1107ebccf1e3SJoseph Koshy 	 */
1108ebccf1e3SJoseph Koshy 
1109ebccf1e3SJoseph Koshy 	sx_slock(&proctree_lock);
1110ebccf1e3SJoseph Koshy 
1111ebccf1e3SJoseph Koshy 	top = p;
1112ebccf1e3SJoseph Koshy 
1113ebccf1e3SJoseph Koshy 	for (;;) {
1114ebccf1e3SJoseph Koshy 		(void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1115ebccf1e3SJoseph Koshy 
1116ebccf1e3SJoseph Koshy 		if (!LIST_EMPTY(&p->p_children))
1117ebccf1e3SJoseph Koshy 			p = LIST_FIRST(&p->p_children);
1118ebccf1e3SJoseph Koshy 		else for (;;) {
1119ebccf1e3SJoseph Koshy 			if (p == top)
1120ebccf1e3SJoseph Koshy 				goto done;
1121ebccf1e3SJoseph Koshy 			if (LIST_NEXT(p, p_sibling)) {
1122ebccf1e3SJoseph Koshy 				p = LIST_NEXT(p, p_sibling);
1123ebccf1e3SJoseph Koshy 				break;
1124ebccf1e3SJoseph Koshy 			}
1125ebccf1e3SJoseph Koshy 			p = p->p_pptr;
1126ebccf1e3SJoseph Koshy 		}
1127ebccf1e3SJoseph Koshy 	}
1128ebccf1e3SJoseph Koshy 
1129ebccf1e3SJoseph Koshy  done:
1130ebccf1e3SJoseph Koshy 	sx_sunlock(&proctree_lock);
1131f263522aSJoseph Koshy 
1132f263522aSJoseph Koshy 	if (LIST_EMPTY(&pm->pm_targets))
1133f263522aSJoseph Koshy 		pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1134f263522aSJoseph Koshy 
1135ebccf1e3SJoseph Koshy 	return 0;
1136ebccf1e3SJoseph Koshy }
1137ebccf1e3SJoseph Koshy 
1138ebccf1e3SJoseph Koshy 
1139ebccf1e3SJoseph Koshy /*
1140ebccf1e3SJoseph Koshy  * Thread context switch IN
1141ebccf1e3SJoseph Koshy  */
1142ebccf1e3SJoseph Koshy 
1143f263522aSJoseph Koshy static void
1144f263522aSJoseph Koshy pmc_process_csw_in(struct thread *td)
1145ebccf1e3SJoseph Koshy {
1146ebccf1e3SJoseph Koshy 	int cpu;
1147ebccf1e3SJoseph Koshy 	unsigned int ri;
1148ebccf1e3SJoseph Koshy 	struct pmc *pm;
1149ebccf1e3SJoseph Koshy 	struct proc *p;
1150ebccf1e3SJoseph Koshy 	struct pmc_cpu *pc;
1151ebccf1e3SJoseph Koshy 	struct pmc_hw *phw;
1152ebccf1e3SJoseph Koshy 	struct pmc_process *pp;
1153ebccf1e3SJoseph Koshy 	pmc_value_t newvalue;
1154ebccf1e3SJoseph Koshy 
1155ebccf1e3SJoseph Koshy 	p = td->td_proc;
1156ebccf1e3SJoseph Koshy 
1157ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1158f263522aSJoseph Koshy 		return;
1159ebccf1e3SJoseph Koshy 
1160ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_proc == td->td_proc,
1161ebccf1e3SJoseph Koshy 	    ("[pmc,%d] not my thread state", __LINE__));
1162ebccf1e3SJoseph Koshy 
1163f263522aSJoseph Koshy 	critical_enter(); /* no preemption from this point */
1164ebccf1e3SJoseph Koshy 
1165ebccf1e3SJoseph Koshy 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1166ebccf1e3SJoseph Koshy 
1167f263522aSJoseph Koshy 	PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1168ebccf1e3SJoseph Koshy 	    p->p_pid, p->p_comm, pp);
1169ebccf1e3SJoseph Koshy 
1170ebccf1e3SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < mp_ncpus,
1171ebccf1e3SJoseph Koshy 	    ("[pmc,%d] wierd CPU id %d", __LINE__, cpu));
1172ebccf1e3SJoseph Koshy 
1173ebccf1e3SJoseph Koshy 	pc = pmc_pcpu[cpu];
1174ebccf1e3SJoseph Koshy 
1175ebccf1e3SJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++) {
1176ebccf1e3SJoseph Koshy 
1177ebccf1e3SJoseph Koshy 		if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1178ebccf1e3SJoseph Koshy 			continue;
1179ebccf1e3SJoseph Koshy 
1180c5153e19SJoseph Koshy 		KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1181ebccf1e3SJoseph Koshy 		    ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1182c5153e19SJoseph Koshy 			__LINE__, PMC_TO_MODE(pm)));
1183ebccf1e3SJoseph Koshy 
1184c5153e19SJoseph Koshy 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1185ebccf1e3SJoseph Koshy 		    ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1186c5153e19SJoseph Koshy 			__LINE__, PMC_TO_ROWINDEX(pm), ri));
1187ebccf1e3SJoseph Koshy 
1188ebccf1e3SJoseph Koshy 		/*
1189ebccf1e3SJoseph Koshy 		 * Only PMCs that are marked as 'RUNNING' need
1190ebccf1e3SJoseph Koshy 		 * be placed on hardware.
1191ebccf1e3SJoseph Koshy 		 */
1192ebccf1e3SJoseph Koshy 
1193ebccf1e3SJoseph Koshy 		if (pm->pm_state != PMC_STATE_RUNNING)
1194ebccf1e3SJoseph Koshy 			continue;
1195ebccf1e3SJoseph Koshy 
1196ebccf1e3SJoseph Koshy 		/* increment PMC runcount */
1197ebccf1e3SJoseph Koshy 		atomic_add_rel_32(&pm->pm_runcount, 1);
1198ebccf1e3SJoseph Koshy 
1199ebccf1e3SJoseph Koshy 		/* configure the HWPMC we are going to use. */
1200ebccf1e3SJoseph Koshy 		md->pmd_config_pmc(cpu, ri, pm);
1201ebccf1e3SJoseph Koshy 
1202ebccf1e3SJoseph Koshy 		phw = pc->pc_hwpmcs[ri];
1203ebccf1e3SJoseph Koshy 
1204ebccf1e3SJoseph Koshy 		KASSERT(phw != NULL,
1205ebccf1e3SJoseph Koshy 		    ("[pmc,%d] null hw pointer", __LINE__));
1206ebccf1e3SJoseph Koshy 
1207ebccf1e3SJoseph Koshy 		KASSERT(phw->phw_pmc == pm,
1208ebccf1e3SJoseph Koshy 		    ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1209ebccf1e3SJoseph Koshy 			phw->phw_pmc, pm));
1210ebccf1e3SJoseph Koshy 
1211f263522aSJoseph Koshy 		/*
1212f263522aSJoseph Koshy 		 * Write out saved value and start the PMC.
1213f263522aSJoseph Koshy 		 *
1214f263522aSJoseph Koshy 		 * Sampling PMCs use a per-process value, while
1215f263522aSJoseph Koshy 		 * counting mode PMCs use a per-pmc value that is
1216f263522aSJoseph Koshy 		 * inherited across descendants.
1217f263522aSJoseph Koshy 		 */
1218f263522aSJoseph Koshy 		if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1219f263522aSJoseph Koshy 			mtx_pool_lock_spin(pmc_mtxpool, pm);
1220f263522aSJoseph Koshy 			newvalue = PMC_PCPU_SAVED(cpu,ri) =
1221f263522aSJoseph Koshy 			    pp->pp_pmcs[ri].pp_pmcval;
1222f263522aSJoseph Koshy 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
1223f263522aSJoseph Koshy 		} else {
1224f263522aSJoseph Koshy 			KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1225f263522aSJoseph Koshy 			    ("[pmc,%d] illegal mode=%d", __LINE__,
1226f263522aSJoseph Koshy 			    PMC_TO_MODE(pm)));
1227ebccf1e3SJoseph Koshy 			mtx_pool_lock_spin(pmc_mtxpool, pm);
1228ebccf1e3SJoseph Koshy 			newvalue = PMC_PCPU_SAVED(cpu, ri) =
1229ebccf1e3SJoseph Koshy 			    pm->pm_gv.pm_savedvalue;
1230ebccf1e3SJoseph Koshy 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
1231f263522aSJoseph Koshy 		}
1232f263522aSJoseph Koshy 
1233f263522aSJoseph Koshy 		PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1234ebccf1e3SJoseph Koshy 
1235ebccf1e3SJoseph Koshy 		md->pmd_write_pmc(cpu, ri, newvalue);
1236ebccf1e3SJoseph Koshy 		md->pmd_start_pmc(cpu, ri);
1237ebccf1e3SJoseph Koshy 	}
1238ebccf1e3SJoseph Koshy 
1239ebccf1e3SJoseph Koshy 	/*
1240ebccf1e3SJoseph Koshy 	 * perform any other architecture/cpu dependent thread
1241ebccf1e3SJoseph Koshy 	 * switch-in actions.
1242ebccf1e3SJoseph Koshy 	 */
1243ebccf1e3SJoseph Koshy 
12446b8c8cd8SJoseph Koshy 	(void) (*md->pmd_switch_in)(pc, pp);
1245ebccf1e3SJoseph Koshy 
1246ebccf1e3SJoseph Koshy 	critical_exit();
1247ebccf1e3SJoseph Koshy 
1248ebccf1e3SJoseph Koshy }
1249ebccf1e3SJoseph Koshy 
1250ebccf1e3SJoseph Koshy /*
1251ebccf1e3SJoseph Koshy  * Thread context switch OUT.
1252ebccf1e3SJoseph Koshy  */
1253ebccf1e3SJoseph Koshy 
1254f263522aSJoseph Koshy static void
1255f263522aSJoseph Koshy pmc_process_csw_out(struct thread *td)
1256ebccf1e3SJoseph Koshy {
1257ebccf1e3SJoseph Koshy 	int cpu;
1258f263522aSJoseph Koshy 	enum pmc_mode mode;
1259ebccf1e3SJoseph Koshy 	unsigned int ri;
1260ebccf1e3SJoseph Koshy 	struct pmc *pm;
1261ebccf1e3SJoseph Koshy 	struct proc *p;
1262ebccf1e3SJoseph Koshy 	struct pmc_cpu *pc;
1263ebccf1e3SJoseph Koshy 	struct pmc_process *pp;
1264f263522aSJoseph Koshy 	int64_t tmp;
1265f263522aSJoseph Koshy 	pmc_value_t newvalue;
1266ebccf1e3SJoseph Koshy 
1267ebccf1e3SJoseph Koshy 	/*
1268ebccf1e3SJoseph Koshy 	 * Locate our process descriptor; this may be NULL if
1269ebccf1e3SJoseph Koshy 	 * this process is exiting and we have already removed
1270ebccf1e3SJoseph Koshy 	 * the process from the target process table.
1271ebccf1e3SJoseph Koshy 	 *
1272ebccf1e3SJoseph Koshy 	 * Note that due to kernel preemption, multiple
1273ebccf1e3SJoseph Koshy 	 * context switches may happen while the process is
1274ebccf1e3SJoseph Koshy 	 * exiting.
1275ebccf1e3SJoseph Koshy 	 *
1276ebccf1e3SJoseph Koshy 	 * Note also that if the target process cannot be
1277ebccf1e3SJoseph Koshy 	 * found we still need to deconfigure any PMCs that
1278ebccf1e3SJoseph Koshy 	 * are currently running on hardware.
1279ebccf1e3SJoseph Koshy 	 */
1280ebccf1e3SJoseph Koshy 
1281ebccf1e3SJoseph Koshy 	p = td->td_proc;
1282ebccf1e3SJoseph Koshy 	pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1283ebccf1e3SJoseph Koshy 
1284ebccf1e3SJoseph Koshy 	/*
1285ebccf1e3SJoseph Koshy 	 * save PMCs
1286ebccf1e3SJoseph Koshy 	 */
1287ebccf1e3SJoseph Koshy 
1288ebccf1e3SJoseph Koshy 	critical_enter();
1289ebccf1e3SJoseph Koshy 
1290ebccf1e3SJoseph Koshy 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1291ebccf1e3SJoseph Koshy 
1292f263522aSJoseph Koshy 	PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1293ebccf1e3SJoseph Koshy 	    p->p_pid, p->p_comm, pp);
1294ebccf1e3SJoseph Koshy 
1295ebccf1e3SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < mp_ncpus,
1296ebccf1e3SJoseph Koshy 	    ("[pmc,%d wierd CPU id %d", __LINE__, cpu));
1297ebccf1e3SJoseph Koshy 
1298ebccf1e3SJoseph Koshy 	pc = pmc_pcpu[cpu];
1299ebccf1e3SJoseph Koshy 
1300ebccf1e3SJoseph Koshy 	/*
1301ebccf1e3SJoseph Koshy 	 * When a PMC gets unlinked from a target PMC, it will
1302ebccf1e3SJoseph Koshy 	 * be removed from the target's pp_pmc[] array.
1303ebccf1e3SJoseph Koshy 	 *
1304ebccf1e3SJoseph Koshy 	 * However, on a MP system, the target could have been
1305ebccf1e3SJoseph Koshy 	 * executing on another CPU at the time of the unlink.
1306ebccf1e3SJoseph Koshy 	 * So, at context switch OUT time, we need to look at
1307ebccf1e3SJoseph Koshy 	 * the hardware to determine if a PMC is scheduled on
1308ebccf1e3SJoseph Koshy 	 * it.
1309ebccf1e3SJoseph Koshy 	 */
1310ebccf1e3SJoseph Koshy 
1311ebccf1e3SJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++) {
1312ebccf1e3SJoseph Koshy 
1313c5153e19SJoseph Koshy 		pm = NULL;
1314c5153e19SJoseph Koshy 		(void) (*md->pmd_get_config)(cpu, ri, &pm);
1315ebccf1e3SJoseph Koshy 
1316ebccf1e3SJoseph Koshy 		if (pm == NULL)	/* nothing at this row index */
1317ebccf1e3SJoseph Koshy 			continue;
1318ebccf1e3SJoseph Koshy 
1319f263522aSJoseph Koshy 		mode = PMC_TO_MODE(pm);
1320f263522aSJoseph Koshy 		if (!PMC_IS_VIRTUAL_MODE(mode))
1321ebccf1e3SJoseph Koshy 			continue; /* not a process virtual PMC */
1322ebccf1e3SJoseph Koshy 
1323c5153e19SJoseph Koshy 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1324ebccf1e3SJoseph Koshy 		    ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1325c5153e19SJoseph Koshy 			__LINE__, PMC_TO_ROWINDEX(pm), ri));
1326ebccf1e3SJoseph Koshy 
1327f263522aSJoseph Koshy 		/* Stop hardware if not already stopped */
1328fbf1556dSJoseph Koshy 		if (pm->pm_stalled == 0)
1329ebccf1e3SJoseph Koshy 			md->pmd_stop_pmc(cpu, ri);
1330ebccf1e3SJoseph Koshy 
1331ebccf1e3SJoseph Koshy 		/* reduce this PMC's runcount */
1332ebccf1e3SJoseph Koshy 		atomic_subtract_rel_32(&pm->pm_runcount, 1);
1333ebccf1e3SJoseph Koshy 
1334ebccf1e3SJoseph Koshy 		/*
1335ebccf1e3SJoseph Koshy 		 * If this PMC is associated with this process,
1336ebccf1e3SJoseph Koshy 		 * save the reading.
1337ebccf1e3SJoseph Koshy 		 */
1338ebccf1e3SJoseph Koshy 
1339ebccf1e3SJoseph Koshy 		if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {
1340ebccf1e3SJoseph Koshy 
1341ebccf1e3SJoseph Koshy 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1342f263522aSJoseph Koshy 			    ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1343f263522aSJoseph Koshy 				pm, ri, pp->pp_pmcs[ri].pp_pmc));
1344ebccf1e3SJoseph Koshy 
1345ebccf1e3SJoseph Koshy 			KASSERT(pp->pp_refcnt > 0,
1346ebccf1e3SJoseph Koshy 			    ("[pmc,%d] pp refcnt = %d", __LINE__,
1347ebccf1e3SJoseph Koshy 				pp->pp_refcnt));
1348ebccf1e3SJoseph Koshy 
1349ebccf1e3SJoseph Koshy 			md->pmd_read_pmc(cpu, ri, &newvalue);
1350ebccf1e3SJoseph Koshy 
1351ebccf1e3SJoseph Koshy 			tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1352ebccf1e3SJoseph Koshy 
1353f263522aSJoseph Koshy 			PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
1354f263522aSJoseph Koshy 			    tmp);
1355f263522aSJoseph Koshy 
1356f263522aSJoseph Koshy 			if (mode == PMC_MODE_TS) {
1357f263522aSJoseph Koshy 
1358f263522aSJoseph Koshy 				/*
1359f263522aSJoseph Koshy 				 * For sampling process-virtual PMCs,
1360f263522aSJoseph Koshy 				 * we expect the count to be
1361f263522aSJoseph Koshy 				 * decreasing as the 'value'
1362f263522aSJoseph Koshy 				 * programmed into the PMC is the
1363f263522aSJoseph Koshy 				 * number of events to be seen till
1364f263522aSJoseph Koshy 				 * the next sampling interrupt.
1365f263522aSJoseph Koshy 				 */
1366f263522aSJoseph Koshy 				if (tmp < 0)
1367f263522aSJoseph Koshy 					tmp += pm->pm_sc.pm_reloadcount;
1368f263522aSJoseph Koshy 				mtx_pool_lock_spin(pmc_mtxpool, pm);
1369f263522aSJoseph Koshy 				pp->pp_pmcs[ri].pp_pmcval -= tmp;
1370f263522aSJoseph Koshy 				if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0)
1371f263522aSJoseph Koshy 					pp->pp_pmcs[ri].pp_pmcval +=
1372f263522aSJoseph Koshy 					    pm->pm_sc.pm_reloadcount;
1373f263522aSJoseph Koshy 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
1374f263522aSJoseph Koshy 
1375f263522aSJoseph Koshy 			} else {
1376f263522aSJoseph Koshy 
1377f263522aSJoseph Koshy 				/*
1378f263522aSJoseph Koshy 				 * For counting process-virtual PMCs,
1379f263522aSJoseph Koshy 				 * we expect the count to be
1380f263522aSJoseph Koshy 				 * increasing monotonically, modulo a 64
1381f263522aSJoseph Koshy 				 * bit wraparound.
1382f263522aSJoseph Koshy 				 */
1383ebccf1e3SJoseph Koshy 				KASSERT((int64_t) tmp >= 0,
1384ebccf1e3SJoseph Koshy 				    ("[pmc,%d] negative increment cpu=%d "
1385ebccf1e3SJoseph Koshy 				     "ri=%d newvalue=%jx saved=%jx "
1386ebccf1e3SJoseph Koshy 				     "incr=%jx", __LINE__, cpu, ri,
1387f263522aSJoseph Koshy 				     newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1388ebccf1e3SJoseph Koshy 
1389ebccf1e3SJoseph Koshy 				mtx_pool_lock_spin(pmc_mtxpool, pm);
1390ebccf1e3SJoseph Koshy 				pm->pm_gv.pm_savedvalue += tmp;
1391ebccf1e3SJoseph Koshy 				pp->pp_pmcs[ri].pp_pmcval += tmp;
1392ebccf1e3SJoseph Koshy 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
1393ebccf1e3SJoseph Koshy 
1394f263522aSJoseph Koshy 				if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1395f263522aSJoseph Koshy 					pmclog_process_proccsw(pm, pp, tmp);
1396f263522aSJoseph Koshy 			}
1397ebccf1e3SJoseph Koshy 		}
1398ebccf1e3SJoseph Koshy 
1399ebccf1e3SJoseph Koshy 		/* mark hardware as free */
1400ebccf1e3SJoseph Koshy 		md->pmd_config_pmc(cpu, ri, NULL);
1401ebccf1e3SJoseph Koshy 	}
1402ebccf1e3SJoseph Koshy 
1403ebccf1e3SJoseph Koshy 	/*
1404ebccf1e3SJoseph Koshy 	 * perform any other architecture/cpu dependent thread
1405ebccf1e3SJoseph Koshy 	 * switch out functions.
1406ebccf1e3SJoseph Koshy 	 */
1407ebccf1e3SJoseph Koshy 
14086b8c8cd8SJoseph Koshy 	(void) (*md->pmd_switch_out)(pc, pp);
1409ebccf1e3SJoseph Koshy 
1410ebccf1e3SJoseph Koshy 	critical_exit();
1411f263522aSJoseph Koshy }
1412f263522aSJoseph Koshy 
1413f263522aSJoseph Koshy /*
1414f263522aSJoseph Koshy  * The 'hook' invoked from the kernel proper
1415f263522aSJoseph Koshy  */
1416f263522aSJoseph Koshy 
1417f263522aSJoseph Koshy 
1418342ed5d9SRuslan Ermilov #ifdef	DEBUG
1419f263522aSJoseph Koshy const char *pmc_hooknames[] = {
1420f263522aSJoseph Koshy 	"",
1421f263522aSJoseph Koshy 	"EXIT",
1422f263522aSJoseph Koshy 	"EXEC",
1423f263522aSJoseph Koshy 	"FORK",
1424f263522aSJoseph Koshy 	"CSW-IN",
1425f263522aSJoseph Koshy 	"CSW-OUT",
1426f263522aSJoseph Koshy 	"SAMPLE"
1427f263522aSJoseph Koshy };
1428f263522aSJoseph Koshy #endif
1429f263522aSJoseph Koshy 
1430f263522aSJoseph Koshy static int
1431f263522aSJoseph Koshy pmc_hook_handler(struct thread *td, int function, void *arg)
1432f263522aSJoseph Koshy {
1433f263522aSJoseph Koshy 
1434f263522aSJoseph Koshy 	PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
1435f263522aSJoseph Koshy 	    pmc_hooknames[function], arg);
1436f263522aSJoseph Koshy 
1437f263522aSJoseph Koshy 	switch (function)
1438f263522aSJoseph Koshy 	{
1439f263522aSJoseph Koshy 
1440f263522aSJoseph Koshy 	/*
1441f263522aSJoseph Koshy 	 * Process exec()
1442f263522aSJoseph Koshy 	 */
1443f263522aSJoseph Koshy 
1444f263522aSJoseph Koshy 	case PMC_FN_PROCESS_EXEC:
1445f263522aSJoseph Koshy 	{
1446f263522aSJoseph Koshy 		char *fullpath, *freepath;
1447f263522aSJoseph Koshy 		unsigned int ri;
1448f263522aSJoseph Koshy 		int is_using_hwpmcs;
1449f263522aSJoseph Koshy 		struct pmc *pm;
1450f263522aSJoseph Koshy 		struct proc *p;
1451f263522aSJoseph Koshy 		struct pmc_owner *po;
1452f263522aSJoseph Koshy 		struct pmc_process *pp;
145315139246SJoseph Koshy 		struct pmckern_procexec *pk;
1454f263522aSJoseph Koshy 
1455f263522aSJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
1456f263522aSJoseph Koshy 
1457f263522aSJoseph Koshy 		p = td->td_proc;
145815139246SJoseph Koshy 		pmc_getfilename(p->p_textvp, &fullpath, &freepath);
145915139246SJoseph Koshy 
146015139246SJoseph Koshy 		pk = (struct pmckern_procexec *) arg;
1461f263522aSJoseph Koshy 
1462f263522aSJoseph Koshy 		/* Inform owners of SS mode PMCs of the exec event. */
1463f263522aSJoseph Koshy 		LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1464f263522aSJoseph Koshy 		    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
146515139246SJoseph Koshy 			    pmclog_process_procexec(po, PMC_ID_INVALID,
146615139246SJoseph Koshy 				p->p_pid, pk->pm_entryaddr, fullpath);
1467f263522aSJoseph Koshy 
1468f263522aSJoseph Koshy 		PROC_LOCK(p);
1469f263522aSJoseph Koshy 		is_using_hwpmcs = p->p_flag & P_HWPMC;
1470f263522aSJoseph Koshy 		PROC_UNLOCK(p);
1471f263522aSJoseph Koshy 
1472f263522aSJoseph Koshy 		if (!is_using_hwpmcs) {
1473f263522aSJoseph Koshy 			if (freepath)
1474f263522aSJoseph Koshy 				FREE(freepath, M_TEMP);
1475f263522aSJoseph Koshy 			break;
1476f263522aSJoseph Koshy 		}
1477f263522aSJoseph Koshy 
1478f263522aSJoseph Koshy 		/*
1479f263522aSJoseph Koshy 		 * PMCs are not inherited across an exec():  remove any
1480f263522aSJoseph Koshy 		 * PMCs that this process is the owner of.
1481f263522aSJoseph Koshy 		 */
1482f263522aSJoseph Koshy 
1483f263522aSJoseph Koshy 		if ((po = pmc_find_owner_descriptor(p)) != NULL) {
1484f263522aSJoseph Koshy 			pmc_remove_owner(po);
1485f263522aSJoseph Koshy 			pmc_destroy_owner_descriptor(po);
1486f263522aSJoseph Koshy 		}
1487f263522aSJoseph Koshy 
1488f263522aSJoseph Koshy 		/*
14892b01a084SJoseph Koshy 		 * If the process being exec'ed is not the target of any
14902b01a084SJoseph Koshy 		 * PMC, we are done.
1491f263522aSJoseph Koshy 		 */
14922b01a084SJoseph Koshy 		if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
14932b01a084SJoseph Koshy 			if (freepath)
14942b01a084SJoseph Koshy 				FREE(freepath, M_TEMP);
1495f263522aSJoseph Koshy 			break;
14962b01a084SJoseph Koshy 		}
1497f263522aSJoseph Koshy 
1498f263522aSJoseph Koshy 		/*
1499f263522aSJoseph Koshy 		 * Log the exec event to all monitoring owners.  Skip
1500f263522aSJoseph Koshy 		 * owners who have already recieved the event because
15012b01a084SJoseph Koshy 		 * they had system sampling PMCs active.
1502f263522aSJoseph Koshy 		 */
1503f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++)
1504f263522aSJoseph Koshy 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
1505f263522aSJoseph Koshy 				po = pm->pm_owner;
1506f263522aSJoseph Koshy 				if (po->po_sscount == 0 &&
1507f263522aSJoseph Koshy 				    po->po_flags & PMC_PO_OWNS_LOGFILE)
150815139246SJoseph Koshy 					pmclog_process_procexec(po, pm->pm_id,
150915139246SJoseph Koshy 					    p->p_pid, pk->pm_entryaddr,
1510f263522aSJoseph Koshy 					    fullpath);
1511f263522aSJoseph Koshy 			}
1512f263522aSJoseph Koshy 
1513f263522aSJoseph Koshy 		if (freepath)
1514f263522aSJoseph Koshy 			FREE(freepath, M_TEMP);
1515f263522aSJoseph Koshy 
1516f263522aSJoseph Koshy 
1517f263522aSJoseph Koshy 		PMCDBG(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
151815139246SJoseph Koshy 		    p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
1519f263522aSJoseph Koshy 
152015139246SJoseph Koshy 		if (pk->pm_credentialschanged == 0) /* no change */
1521f263522aSJoseph Koshy 			break;
1522f263522aSJoseph Koshy 
1523f263522aSJoseph Koshy 		/*
1524f263522aSJoseph Koshy 		 * If the newly exec()'ed process has a different credential
1525f263522aSJoseph Koshy 		 * than before, allow it to be the target of a PMC only if
1526f263522aSJoseph Koshy 		 * the PMC's owner has sufficient priviledge.
1527f263522aSJoseph Koshy 		 */
1528f263522aSJoseph Koshy 
1529f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++)
1530f263522aSJoseph Koshy 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
1531f263522aSJoseph Koshy 				if (pmc_can_attach(pm, td->td_proc) != 0)
1532f263522aSJoseph Koshy 					pmc_detach_one_process(td->td_proc,
1533f263522aSJoseph Koshy 					    pm, PMC_FLAG_NONE);
1534f263522aSJoseph Koshy 
1535f263522aSJoseph Koshy 		KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc,
1536f263522aSJoseph Koshy 		    ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
1537f263522aSJoseph Koshy 			pp->pp_refcnt, pp));
1538f263522aSJoseph Koshy 
1539f263522aSJoseph Koshy 		/*
1540f263522aSJoseph Koshy 		 * If this process is no longer the target of any
1541f263522aSJoseph Koshy 		 * PMCs, we can remove the process entry and free
1542f263522aSJoseph Koshy 		 * up space.
1543f263522aSJoseph Koshy 		 */
1544f263522aSJoseph Koshy 
1545f263522aSJoseph Koshy 		if (pp->pp_refcnt == 0) {
1546f263522aSJoseph Koshy 			pmc_remove_process_descriptor(pp);
1547f263522aSJoseph Koshy 			FREE(pp, M_PMC);
1548f263522aSJoseph Koshy 			break;
1549f263522aSJoseph Koshy 		}
1550ebccf1e3SJoseph Koshy 
1551ebccf1e3SJoseph Koshy 	}
1552ebccf1e3SJoseph Koshy 	break;
1553ebccf1e3SJoseph Koshy 
1554f263522aSJoseph Koshy 	case PMC_FN_CSW_IN:
1555f263522aSJoseph Koshy 		pmc_process_csw_in(td);
1556f263522aSJoseph Koshy 		break;
1557f263522aSJoseph Koshy 
1558f263522aSJoseph Koshy 	case PMC_FN_CSW_OUT:
1559f263522aSJoseph Koshy 		pmc_process_csw_out(td);
1560f263522aSJoseph Koshy 		break;
1561f263522aSJoseph Koshy 
1562f263522aSJoseph Koshy 	/*
1563f263522aSJoseph Koshy 	 * Process accumulated PC samples.
1564f263522aSJoseph Koshy 	 *
1565f263522aSJoseph Koshy 	 * This function is expected to be called by hardclock() for
1566f263522aSJoseph Koshy 	 * each CPU that has accumulated PC samples.
1567f263522aSJoseph Koshy 	 *
1568f263522aSJoseph Koshy 	 * This function is to be executed on the CPU whose samples
1569f263522aSJoseph Koshy 	 * are being processed.
1570f263522aSJoseph Koshy 	 */
1571f263522aSJoseph Koshy 	case PMC_FN_DO_SAMPLES:
1572f263522aSJoseph Koshy 
1573f263522aSJoseph Koshy 		/*
1574f263522aSJoseph Koshy 		 * Clear the cpu specific bit in the CPU mask before
1575f263522aSJoseph Koshy 		 * do the rest of the processing.  If the NMI handler
1576f263522aSJoseph Koshy 		 * gets invoked after the "atomic_clear_int()" call
1577f263522aSJoseph Koshy 		 * below but before "pmc_process_samples()" gets
1578f263522aSJoseph Koshy 		 * around to processing the interrupt, then we will
1579f263522aSJoseph Koshy 		 * come back here at the next hardclock() tick (and
1580f263522aSJoseph Koshy 		 * may find nothing to do if "pmc_process_samples()"
1581f263522aSJoseph Koshy 		 * had already processed the interrupt).  We don't
1582f263522aSJoseph Koshy 		 * lose the interrupt sample.
1583f263522aSJoseph Koshy 		 */
1584f263522aSJoseph Koshy 		atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid)));
1585f263522aSJoseph Koshy 		pmc_process_samples(PCPU_GET(cpuid));
1586f263522aSJoseph Koshy 		break;
1587f263522aSJoseph Koshy 
1588ebccf1e3SJoseph Koshy 	default:
1589342ed5d9SRuslan Ermilov #ifdef	DEBUG
1590ebccf1e3SJoseph Koshy 		KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
1591ebccf1e3SJoseph Koshy #endif
1592ebccf1e3SJoseph Koshy 		break;
1593ebccf1e3SJoseph Koshy 
1594ebccf1e3SJoseph Koshy 	}
1595ebccf1e3SJoseph Koshy 
1596ebccf1e3SJoseph Koshy 	return 0;
1597ebccf1e3SJoseph Koshy }
1598ebccf1e3SJoseph Koshy 
1599ebccf1e3SJoseph Koshy /*
1600ebccf1e3SJoseph Koshy  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
1601ebccf1e3SJoseph Koshy  */
1602ebccf1e3SJoseph Koshy 
1603ebccf1e3SJoseph Koshy static struct pmc_owner *
1604ebccf1e3SJoseph Koshy pmc_allocate_owner_descriptor(struct proc *p)
1605ebccf1e3SJoseph Koshy {
1606ebccf1e3SJoseph Koshy 	uint32_t hindex;
1607ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
1608ebccf1e3SJoseph Koshy 	struct pmc_ownerhash *poh;
1609ebccf1e3SJoseph Koshy 
1610ebccf1e3SJoseph Koshy 	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1611ebccf1e3SJoseph Koshy 	poh = &pmc_ownerhash[hindex];
1612ebccf1e3SJoseph Koshy 
1613ebccf1e3SJoseph Koshy 	/* allocate space for N pointers and one descriptor struct */
1614ebccf1e3SJoseph Koshy 	MALLOC(po, struct pmc_owner *, sizeof(struct pmc_owner),
1615f263522aSJoseph Koshy 	    M_PMC, M_ZERO|M_WAITOK);
1616ebccf1e3SJoseph Koshy 
1617f263522aSJoseph Koshy 	po->po_sscount = po->po_error = po->po_flags = 0;
1618f263522aSJoseph Koshy 	po->po_file  = NULL;
1619ebccf1e3SJoseph Koshy 	po->po_owner = p;
1620f263522aSJoseph Koshy 	po->po_kthread = NULL;
1621ebccf1e3SJoseph Koshy 	LIST_INIT(&po->po_pmcs);
1622ebccf1e3SJoseph Koshy 	LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
1623ebccf1e3SJoseph Koshy 
1624f263522aSJoseph Koshy 	TAILQ_INIT(&po->po_logbuffers);
1625f263522aSJoseph Koshy 	mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc", MTX_SPIN);
1626f263522aSJoseph Koshy 
1627ebccf1e3SJoseph Koshy 	PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
1628ebccf1e3SJoseph Koshy 	    p, p->p_pid, p->p_comm, po);
1629ebccf1e3SJoseph Koshy 
1630ebccf1e3SJoseph Koshy 	return po;
1631ebccf1e3SJoseph Koshy }
1632ebccf1e3SJoseph Koshy 
1633f263522aSJoseph Koshy static void
1634f263522aSJoseph Koshy pmc_destroy_owner_descriptor(struct pmc_owner *po)
1635f263522aSJoseph Koshy {
1636f263522aSJoseph Koshy 
1637f263522aSJoseph Koshy 	PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
1638f263522aSJoseph Koshy 	    po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
1639f263522aSJoseph Koshy 
1640f263522aSJoseph Koshy 	mtx_destroy(&po->po_mtx);
1641f263522aSJoseph Koshy 	FREE(po, M_PMC);
1642f263522aSJoseph Koshy }
1643f263522aSJoseph Koshy 
1644ebccf1e3SJoseph Koshy /*
1645ebccf1e3SJoseph Koshy  * find the descriptor corresponding to process 'p', adding or removing it
1646ebccf1e3SJoseph Koshy  * as specified by 'mode'.
1647ebccf1e3SJoseph Koshy  */
1648ebccf1e3SJoseph Koshy 
1649ebccf1e3SJoseph Koshy static struct pmc_process *
1650ebccf1e3SJoseph Koshy pmc_find_process_descriptor(struct proc *p, uint32_t mode)
1651ebccf1e3SJoseph Koshy {
1652ebccf1e3SJoseph Koshy 	uint32_t hindex;
1653ebccf1e3SJoseph Koshy 	struct pmc_process *pp, *ppnew;
1654ebccf1e3SJoseph Koshy 	struct pmc_processhash *pph;
1655ebccf1e3SJoseph Koshy 
1656ebccf1e3SJoseph Koshy 	hindex = PMC_HASH_PTR(p, pmc_processhashmask);
1657ebccf1e3SJoseph Koshy 	pph = &pmc_processhash[hindex];
1658ebccf1e3SJoseph Koshy 
1659ebccf1e3SJoseph Koshy 	ppnew = NULL;
1660ebccf1e3SJoseph Koshy 
1661ebccf1e3SJoseph Koshy 	/*
1662ebccf1e3SJoseph Koshy 	 * Pre-allocate memory in the FIND_ALLOCATE case since we
1663ebccf1e3SJoseph Koshy 	 * cannot call malloc(9) once we hold a spin lock.
1664ebccf1e3SJoseph Koshy 	 */
1665ebccf1e3SJoseph Koshy 
1666ebccf1e3SJoseph Koshy 	if (mode & PMC_FLAG_ALLOCATE) {
1667ebccf1e3SJoseph Koshy 		/* allocate additional space for 'n' pmc pointers */
1668ebccf1e3SJoseph Koshy 		MALLOC(ppnew, struct pmc_process *,
1669ebccf1e3SJoseph Koshy 		    sizeof(struct pmc_process) + md->pmd_npmc *
1670ebccf1e3SJoseph Koshy 		    sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK);
1671ebccf1e3SJoseph Koshy 	}
1672ebccf1e3SJoseph Koshy 
1673ebccf1e3SJoseph Koshy 	mtx_lock_spin(&pmc_processhash_mtx);
1674ebccf1e3SJoseph Koshy 	LIST_FOREACH(pp, pph, pp_next)
1675ebccf1e3SJoseph Koshy 	    if (pp->pp_proc == p)
1676ebccf1e3SJoseph Koshy 		    break;
1677ebccf1e3SJoseph Koshy 
1678ebccf1e3SJoseph Koshy 	if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
1679ebccf1e3SJoseph Koshy 		LIST_REMOVE(pp, pp_next);
1680ebccf1e3SJoseph Koshy 
1681ebccf1e3SJoseph Koshy 	if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
1682ebccf1e3SJoseph Koshy 	    ppnew != NULL) {
1683ebccf1e3SJoseph Koshy 		ppnew->pp_proc = p;
1684ebccf1e3SJoseph Koshy 		LIST_INSERT_HEAD(pph, ppnew, pp_next);
1685ebccf1e3SJoseph Koshy 		pp = ppnew;
1686ebccf1e3SJoseph Koshy 		ppnew = NULL;
1687ebccf1e3SJoseph Koshy 	}
1688ebccf1e3SJoseph Koshy 	mtx_unlock_spin(&pmc_processhash_mtx);
1689ebccf1e3SJoseph Koshy 
1690ebccf1e3SJoseph Koshy 	if (pp != NULL && ppnew != NULL)
1691ebccf1e3SJoseph Koshy 		FREE(ppnew, M_PMC);
1692ebccf1e3SJoseph Koshy 
1693ebccf1e3SJoseph Koshy 	return pp;
1694ebccf1e3SJoseph Koshy }
1695ebccf1e3SJoseph Koshy 
1696ebccf1e3SJoseph Koshy /*
1697ebccf1e3SJoseph Koshy  * remove a process descriptor from the process hash table.
1698ebccf1e3SJoseph Koshy  */
1699ebccf1e3SJoseph Koshy 
1700ebccf1e3SJoseph Koshy static void
1701ebccf1e3SJoseph Koshy pmc_remove_process_descriptor(struct pmc_process *pp)
1702ebccf1e3SJoseph Koshy {
1703ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_refcnt == 0,
1704ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Removing process descriptor %p with count %d",
1705ebccf1e3SJoseph Koshy 		__LINE__, pp, pp->pp_refcnt));
1706ebccf1e3SJoseph Koshy 
1707ebccf1e3SJoseph Koshy 	mtx_lock_spin(&pmc_processhash_mtx);
1708ebccf1e3SJoseph Koshy 	LIST_REMOVE(pp, pp_next);
1709ebccf1e3SJoseph Koshy 	mtx_unlock_spin(&pmc_processhash_mtx);
1710ebccf1e3SJoseph Koshy }
1711ebccf1e3SJoseph Koshy 
1712ebccf1e3SJoseph Koshy 
1713ebccf1e3SJoseph Koshy /*
1714ebccf1e3SJoseph Koshy  * find an owner descriptor corresponding to proc 'p'
1715ebccf1e3SJoseph Koshy  */
1716ebccf1e3SJoseph Koshy 
1717ebccf1e3SJoseph Koshy static struct pmc_owner *
1718ebccf1e3SJoseph Koshy pmc_find_owner_descriptor(struct proc *p)
1719ebccf1e3SJoseph Koshy {
1720ebccf1e3SJoseph Koshy 	uint32_t hindex;
1721ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
1722ebccf1e3SJoseph Koshy 	struct pmc_ownerhash *poh;
1723ebccf1e3SJoseph Koshy 
1724ebccf1e3SJoseph Koshy 	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1725ebccf1e3SJoseph Koshy 	poh = &pmc_ownerhash[hindex];
1726ebccf1e3SJoseph Koshy 
1727ebccf1e3SJoseph Koshy 	po = NULL;
1728ebccf1e3SJoseph Koshy 	LIST_FOREACH(po, poh, po_next)
1729ebccf1e3SJoseph Koshy 	    if (po->po_owner == p)
1730ebccf1e3SJoseph Koshy 		    break;
1731ebccf1e3SJoseph Koshy 
1732ebccf1e3SJoseph Koshy 	PMCDBG(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
1733ebccf1e3SJoseph Koshy 	    "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
1734ebccf1e3SJoseph Koshy 
1735ebccf1e3SJoseph Koshy 	return po;
1736ebccf1e3SJoseph Koshy }
1737ebccf1e3SJoseph Koshy 
1738ebccf1e3SJoseph Koshy /*
1739ebccf1e3SJoseph Koshy  * pmc_allocate_pmc_descriptor
1740ebccf1e3SJoseph Koshy  *
1741ebccf1e3SJoseph Koshy  * Allocate a pmc descriptor and initialize its
1742ebccf1e3SJoseph Koshy  * fields.
1743ebccf1e3SJoseph Koshy  */
1744ebccf1e3SJoseph Koshy 
1745ebccf1e3SJoseph Koshy static struct pmc *
1746ebccf1e3SJoseph Koshy pmc_allocate_pmc_descriptor(void)
1747ebccf1e3SJoseph Koshy {
1748ebccf1e3SJoseph Koshy 	struct pmc *pmc;
1749ebccf1e3SJoseph Koshy 
1750ebccf1e3SJoseph Koshy 	MALLOC(pmc, struct pmc *, sizeof(struct pmc), M_PMC, M_ZERO|M_WAITOK);
1751ebccf1e3SJoseph Koshy 
1752ebccf1e3SJoseph Koshy 	if (pmc != NULL) {
1753ebccf1e3SJoseph Koshy 		pmc->pm_owner = NULL;
1754ebccf1e3SJoseph Koshy 		LIST_INIT(&pmc->pm_targets);
1755ebccf1e3SJoseph Koshy 	}
1756ebccf1e3SJoseph Koshy 
1757ebccf1e3SJoseph Koshy 	PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
1758ebccf1e3SJoseph Koshy 
1759ebccf1e3SJoseph Koshy 	return pmc;
1760ebccf1e3SJoseph Koshy }
1761ebccf1e3SJoseph Koshy 
1762ebccf1e3SJoseph Koshy /*
1763ebccf1e3SJoseph Koshy  * Destroy a pmc descriptor.
1764ebccf1e3SJoseph Koshy  */
1765ebccf1e3SJoseph Koshy 
1766ebccf1e3SJoseph Koshy static void
1767ebccf1e3SJoseph Koshy pmc_destroy_pmc_descriptor(struct pmc *pm)
1768ebccf1e3SJoseph Koshy {
1769ebccf1e3SJoseph Koshy 	(void) pm;
1770ebccf1e3SJoseph Koshy 
1771342ed5d9SRuslan Ermilov #ifdef	DEBUG
1772ebccf1e3SJoseph Koshy 	KASSERT(pm->pm_state == PMC_STATE_DELETED ||
1773ebccf1e3SJoseph Koshy 	    pm->pm_state == PMC_STATE_FREE,
1774ebccf1e3SJoseph Koshy 	    ("[pmc,%d] destroying non-deleted PMC", __LINE__));
1775ebccf1e3SJoseph Koshy 	KASSERT(LIST_EMPTY(&pm->pm_targets),
1776ebccf1e3SJoseph Koshy 	    ("[pmc,%d] destroying pmc with targets", __LINE__));
1777ebccf1e3SJoseph Koshy 	KASSERT(pm->pm_owner == NULL,
1778ebccf1e3SJoseph Koshy 	    ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
1779ebccf1e3SJoseph Koshy 	KASSERT(pm->pm_runcount == 0,
1780ebccf1e3SJoseph Koshy 	    ("[pmc,%d] pmc has non-zero run count %d", __LINE__,
1781ebccf1e3SJoseph Koshy 		pm->pm_runcount));
1782ebccf1e3SJoseph Koshy #endif
1783ebccf1e3SJoseph Koshy }
1784ebccf1e3SJoseph Koshy 
1785f263522aSJoseph Koshy static void
1786f263522aSJoseph Koshy pmc_wait_for_pmc_idle(struct pmc *pm)
1787f263522aSJoseph Koshy {
1788342ed5d9SRuslan Ermilov #ifdef	DEBUG
1789f263522aSJoseph Koshy 	volatile int maxloop;
1790f263522aSJoseph Koshy 
1791f263522aSJoseph Koshy 	maxloop = 100 * mp_ncpus;
1792f263522aSJoseph Koshy #endif
1793f263522aSJoseph Koshy 
1794f263522aSJoseph Koshy 	/*
1795f263522aSJoseph Koshy 	 * Loop (with a forced context switch) till the PMC's runcount
1796f263522aSJoseph Koshy 	 * comes down to zero.
1797f263522aSJoseph Koshy 	 */
1798f263522aSJoseph Koshy 	while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
1799342ed5d9SRuslan Ermilov #ifdef	DEBUG
1800f263522aSJoseph Koshy 		maxloop--;
1801f263522aSJoseph Koshy 		KASSERT(maxloop > 0,
1802f263522aSJoseph Koshy 		    ("[pmc,%d] (ri%d, rc%d) waiting too long for "
1803f263522aSJoseph Koshy 			"pmc to be free", __LINE__,
1804f263522aSJoseph Koshy 			PMC_TO_ROWINDEX(pm), pm->pm_runcount));
1805f263522aSJoseph Koshy #endif
1806f263522aSJoseph Koshy 		pmc_force_context_switch();
1807f263522aSJoseph Koshy 	}
1808f263522aSJoseph Koshy }
1809f263522aSJoseph Koshy 
1810ebccf1e3SJoseph Koshy /*
1811ebccf1e3SJoseph Koshy  * This function does the following things:
1812ebccf1e3SJoseph Koshy  *
1813ebccf1e3SJoseph Koshy  *  - detaches the PMC from hardware
1814ebccf1e3SJoseph Koshy  *  - unlinks all target threads that were attached to it
1815ebccf1e3SJoseph Koshy  *  - removes the PMC from its owner's list
1816ebccf1e3SJoseph Koshy  *  - destroy's the PMC private mutex
1817ebccf1e3SJoseph Koshy  *
1818ebccf1e3SJoseph Koshy  * Once this function completes, the given pmc pointer can be safely
1819ebccf1e3SJoseph Koshy  * FREE'd by the caller.
1820ebccf1e3SJoseph Koshy  */
1821ebccf1e3SJoseph Koshy 
1822ebccf1e3SJoseph Koshy static void
1823ebccf1e3SJoseph Koshy pmc_release_pmc_descriptor(struct pmc *pm)
1824ebccf1e3SJoseph Koshy {
1825ebccf1e3SJoseph Koshy 	u_int ri, cpu;
1826c5153e19SJoseph Koshy 	enum pmc_mode mode;
1827ebccf1e3SJoseph Koshy 	struct pmc_hw *phw;
1828f263522aSJoseph Koshy 	struct pmc_owner *po;
1829ebccf1e3SJoseph Koshy 	struct pmc_process *pp;
1830ebccf1e3SJoseph Koshy 	struct pmc_target *ptgt, *tmp;
1831ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
1832ebccf1e3SJoseph Koshy 
1833ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1834ebccf1e3SJoseph Koshy 
1835ebccf1e3SJoseph Koshy 	KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
1836ebccf1e3SJoseph Koshy 
1837c5153e19SJoseph Koshy 	ri   = PMC_TO_ROWINDEX(pm);
1838c5153e19SJoseph Koshy 	mode = PMC_TO_MODE(pm);
1839ebccf1e3SJoseph Koshy 
1840ebccf1e3SJoseph Koshy 	PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
1841c5153e19SJoseph Koshy 	    mode);
1842ebccf1e3SJoseph Koshy 
1843ebccf1e3SJoseph Koshy 	/*
1844ebccf1e3SJoseph Koshy 	 * First, we take the PMC off hardware.
1845ebccf1e3SJoseph Koshy 	 */
1846869f4a8bSWarner Losh 	cpu = 0;
1847c5153e19SJoseph Koshy 	if (PMC_IS_SYSTEM_MODE(mode)) {
1848ebccf1e3SJoseph Koshy 
1849ebccf1e3SJoseph Koshy 		/*
1850ebccf1e3SJoseph Koshy 		 * A system mode PMC runs on a specific CPU.  Switch
1851ebccf1e3SJoseph Koshy 		 * to this CPU and turn hardware off.
1852ebccf1e3SJoseph Koshy 		 */
1853ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
1854ebccf1e3SJoseph Koshy 
1855c5153e19SJoseph Koshy 		cpu = PMC_TO_CPU(pm);
1856ebccf1e3SJoseph Koshy 
1857ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
1858ebccf1e3SJoseph Koshy 
1859f263522aSJoseph Koshy 		/* switch off non-stalled CPUs */
1860f263522aSJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING &&
1861fbf1556dSJoseph Koshy 		    pm->pm_stalled == 0) {
1862f263522aSJoseph Koshy 
1863ebccf1e3SJoseph Koshy 			phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1864ebccf1e3SJoseph Koshy 
1865ebccf1e3SJoseph Koshy 			KASSERT(phw->phw_pmc == pm,
1866ebccf1e3SJoseph Koshy 			    ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
1867ebccf1e3SJoseph Koshy 				__LINE__, ri, phw->phw_pmc, pm));
1868ebccf1e3SJoseph Koshy 			PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
1869ebccf1e3SJoseph Koshy 
1870ebccf1e3SJoseph Koshy 			critical_enter();
1871ebccf1e3SJoseph Koshy 			md->pmd_stop_pmc(cpu, ri);
1872ebccf1e3SJoseph Koshy 			critical_exit();
1873ebccf1e3SJoseph Koshy 		}
1874ebccf1e3SJoseph Koshy 
1875ebccf1e3SJoseph Koshy 		PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
1876ebccf1e3SJoseph Koshy 
1877ebccf1e3SJoseph Koshy 		critical_enter();
1878ebccf1e3SJoseph Koshy 		md->pmd_config_pmc(cpu, ri, NULL);
1879ebccf1e3SJoseph Koshy 		critical_exit();
1880ebccf1e3SJoseph Koshy 
1881f263522aSJoseph Koshy 		/* adjust the global and process count of SS mode PMCs */
1882f263522aSJoseph Koshy 		if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
1883f263522aSJoseph Koshy 			po = pm->pm_owner;
1884f263522aSJoseph Koshy 			po->po_sscount--;
1885f263522aSJoseph Koshy 			if (po->po_sscount == 0) {
1886f263522aSJoseph Koshy 				atomic_subtract_rel_int(&pmc_ss_count, 1);
1887f263522aSJoseph Koshy 				LIST_REMOVE(po, po_ssnext);
1888f263522aSJoseph Koshy 			}
1889f263522aSJoseph Koshy 		}
1890f263522aSJoseph Koshy 
1891ebccf1e3SJoseph Koshy 		pm->pm_state = PMC_STATE_DELETED;
1892ebccf1e3SJoseph Koshy 
1893ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
1894ebccf1e3SJoseph Koshy 
1895f263522aSJoseph Koshy 		/*
1896f263522aSJoseph Koshy 		 * We could have references to this PMC structure in
1897f263522aSJoseph Koshy 		 * the per-cpu sample queues.  Wait for the queue to
1898f263522aSJoseph Koshy 		 * drain.
1899f263522aSJoseph Koshy 		 */
1900f263522aSJoseph Koshy 		pmc_wait_for_pmc_idle(pm);
1901f263522aSJoseph Koshy 
1902c5153e19SJoseph Koshy 	} else if (PMC_IS_VIRTUAL_MODE(mode)) {
1903ebccf1e3SJoseph Koshy 
1904ebccf1e3SJoseph Koshy 		/*
1905ebccf1e3SJoseph Koshy 		 * A virtual PMC could be running on multiple CPUs at
1906ebccf1e3SJoseph Koshy 		 * a given instant.
1907ebccf1e3SJoseph Koshy 		 *
1908ebccf1e3SJoseph Koshy 		 * By marking its state as DELETED, we ensure that
1909ebccf1e3SJoseph Koshy 		 * this PMC is never further scheduled on hardware.
1910ebccf1e3SJoseph Koshy 		 *
1911ebccf1e3SJoseph Koshy 		 * Then we wait till all CPUs are done with this PMC.
1912ebccf1e3SJoseph Koshy 		 */
1913ebccf1e3SJoseph Koshy 		pm->pm_state = PMC_STATE_DELETED;
1914ebccf1e3SJoseph Koshy 
1915ebccf1e3SJoseph Koshy 
1916f263522aSJoseph Koshy 		/* Wait for the PMCs runcount to come to zero. */
1917f263522aSJoseph Koshy 		pmc_wait_for_pmc_idle(pm);
1918ebccf1e3SJoseph Koshy 
1919ebccf1e3SJoseph Koshy 		/*
1920ebccf1e3SJoseph Koshy 		 * At this point the PMC is off all CPUs and cannot be
1921ebccf1e3SJoseph Koshy 		 * freshly scheduled onto a CPU.  It is now safe to
1922ebccf1e3SJoseph Koshy 		 * unlink all targets from this PMC.  If a
1923ebccf1e3SJoseph Koshy 		 * process-record's refcount falls to zero, we remove
1924ebccf1e3SJoseph Koshy 		 * it from the hash table.  The module-wide SX lock
1925ebccf1e3SJoseph Koshy 		 * protects us from races.
1926ebccf1e3SJoseph Koshy 		 */
1927ebccf1e3SJoseph Koshy 		LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
1928ebccf1e3SJoseph Koshy 			pp = ptgt->pt_process;
1929ebccf1e3SJoseph Koshy 			pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
1930ebccf1e3SJoseph Koshy 
1931ebccf1e3SJoseph Koshy 			PMCDBG(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
1932ebccf1e3SJoseph Koshy 
1933ebccf1e3SJoseph Koshy 			/*
1934ebccf1e3SJoseph Koshy 			 * If the target process record shows that no
1935ebccf1e3SJoseph Koshy 			 * PMCs are attached to it, reclaim its space.
1936ebccf1e3SJoseph Koshy 			 */
1937ebccf1e3SJoseph Koshy 
1938ebccf1e3SJoseph Koshy 			if (pp->pp_refcnt == 0) {
1939ebccf1e3SJoseph Koshy 				pmc_remove_process_descriptor(pp);
1940ebccf1e3SJoseph Koshy 				FREE(pp, M_PMC);
1941ebccf1e3SJoseph Koshy 			}
1942ebccf1e3SJoseph Koshy 		}
1943ebccf1e3SJoseph Koshy 
1944ebccf1e3SJoseph Koshy 		cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
1945ebccf1e3SJoseph Koshy 
1946ebccf1e3SJoseph Koshy 	}
1947ebccf1e3SJoseph Koshy 
1948ebccf1e3SJoseph Koshy 	/*
1949ebccf1e3SJoseph Koshy 	 * Release any MD resources
1950ebccf1e3SJoseph Koshy 	 */
1951ebccf1e3SJoseph Koshy 
1952ebccf1e3SJoseph Koshy 	(void) md->pmd_release_pmc(cpu, ri, pm);
1953ebccf1e3SJoseph Koshy 
1954ebccf1e3SJoseph Koshy 	/*
1955ebccf1e3SJoseph Koshy 	 * Update row disposition
1956ebccf1e3SJoseph Koshy 	 */
1957ebccf1e3SJoseph Koshy 
1958c5153e19SJoseph Koshy 	if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
1959ebccf1e3SJoseph Koshy 		PMC_UNMARK_ROW_STANDALONE(ri);
1960ebccf1e3SJoseph Koshy 	else
1961ebccf1e3SJoseph Koshy 		PMC_UNMARK_ROW_THREAD(ri);
1962ebccf1e3SJoseph Koshy 
1963ebccf1e3SJoseph Koshy 	/* unlink from the owner's list */
1964f263522aSJoseph Koshy 	if (pm->pm_owner) {
1965f263522aSJoseph Koshy 		LIST_REMOVE(pm, pm_next);
1966f263522aSJoseph Koshy 		pm->pm_owner = NULL;
1967f263522aSJoseph Koshy 	}
1968ebccf1e3SJoseph Koshy 
1969ebccf1e3SJoseph Koshy 	pmc_destroy_pmc_descriptor(pm);
1970ebccf1e3SJoseph Koshy }
1971ebccf1e3SJoseph Koshy 
1972ebccf1e3SJoseph Koshy /*
1973ebccf1e3SJoseph Koshy  * Register an owner and a pmc.
1974ebccf1e3SJoseph Koshy  */
1975ebccf1e3SJoseph Koshy 
1976ebccf1e3SJoseph Koshy static int
1977ebccf1e3SJoseph Koshy pmc_register_owner(struct proc *p, struct pmc *pmc)
1978ebccf1e3SJoseph Koshy {
1979ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
1980ebccf1e3SJoseph Koshy 
1981ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1982ebccf1e3SJoseph Koshy 
1983c5153e19SJoseph Koshy 	if ((po = pmc_find_owner_descriptor(p)) == NULL)
1984f263522aSJoseph Koshy 		if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
1985ebccf1e3SJoseph Koshy 			return ENOMEM;
1986ebccf1e3SJoseph Koshy 
1987ebccf1e3SJoseph Koshy 	KASSERT(pmc->pm_owner == NULL,
1988ebccf1e3SJoseph Koshy 	    ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
1989ebccf1e3SJoseph Koshy 	pmc->pm_owner  = po;
1990ebccf1e3SJoseph Koshy 
1991f263522aSJoseph Koshy 	LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
1992ebccf1e3SJoseph Koshy 
1993ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
1994ebccf1e3SJoseph Koshy 	p->p_flag |= P_HWPMC;
1995ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
1996ebccf1e3SJoseph Koshy 
1997f263522aSJoseph Koshy 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1998f263522aSJoseph Koshy 		pmclog_process_pmcallocate(pmc);
1999f263522aSJoseph Koshy 
2000f263522aSJoseph Koshy 	PMCDBG(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2001f263522aSJoseph Koshy 	    po, pmc);
2002ebccf1e3SJoseph Koshy 
2003ebccf1e3SJoseph Koshy 	return 0;
2004ebccf1e3SJoseph Koshy }
2005ebccf1e3SJoseph Koshy 
2006ebccf1e3SJoseph Koshy /*
2007ebccf1e3SJoseph Koshy  * Return the current row disposition:
2008ebccf1e3SJoseph Koshy  * == 0 => FREE
2009ebccf1e3SJoseph Koshy  *  > 0 => PROCESS MODE
2010ebccf1e3SJoseph Koshy  *  < 0 => SYSTEM MODE
2011ebccf1e3SJoseph Koshy  */
2012ebccf1e3SJoseph Koshy 
2013ebccf1e3SJoseph Koshy int
2014ebccf1e3SJoseph Koshy pmc_getrowdisp(int ri)
2015ebccf1e3SJoseph Koshy {
2016ebccf1e3SJoseph Koshy 	return pmc_pmcdisp[ri];
2017ebccf1e3SJoseph Koshy }
2018ebccf1e3SJoseph Koshy 
2019ebccf1e3SJoseph Koshy /*
2020ebccf1e3SJoseph Koshy  * Check if a PMC at row index 'ri' can be allocated to the current
2021ebccf1e3SJoseph Koshy  * process.
2022ebccf1e3SJoseph Koshy  *
2023ebccf1e3SJoseph Koshy  * Allocation can fail if:
2024ebccf1e3SJoseph Koshy  *   - the current process is already being profiled by a PMC at index 'ri',
2025ebccf1e3SJoseph Koshy  *     attached to it via OP_PMCATTACH.
2026ebccf1e3SJoseph Koshy  *   - the current process has already allocated a PMC at index 'ri'
2027ebccf1e3SJoseph Koshy  *     via OP_ALLOCATE.
2028ebccf1e3SJoseph Koshy  */
2029ebccf1e3SJoseph Koshy 
2030ebccf1e3SJoseph Koshy static int
2031c5153e19SJoseph Koshy pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2032ebccf1e3SJoseph Koshy {
2033c5153e19SJoseph Koshy 	enum pmc_mode mode;
2034c5153e19SJoseph Koshy 	struct pmc *pm;
2035ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
2036ebccf1e3SJoseph Koshy 	struct pmc_process *pp;
2037ebccf1e3SJoseph Koshy 
2038c5153e19SJoseph Koshy 	PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2039c5153e19SJoseph Koshy 	    "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2040ebccf1e3SJoseph Koshy 
2041c5153e19SJoseph Koshy 	/*
2042c5153e19SJoseph Koshy 	 * We shouldn't have already allocated a process-mode PMC at
2043c5153e19SJoseph Koshy 	 * row index 'ri'.
2044c5153e19SJoseph Koshy 	 *
2045c5153e19SJoseph Koshy 	 * We shouldn't have allocated a system-wide PMC on the same
2046c5153e19SJoseph Koshy 	 * CPU and same RI.
2047c5153e19SJoseph Koshy 	 */
2048ebccf1e3SJoseph Koshy 	if ((po = pmc_find_owner_descriptor(p)) != NULL)
2049f263522aSJoseph Koshy 		LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2050c5153e19SJoseph Koshy 		    if (PMC_TO_ROWINDEX(pm) == ri) {
2051c5153e19SJoseph Koshy 			    mode = PMC_TO_MODE(pm);
2052c5153e19SJoseph Koshy 			    if (PMC_IS_VIRTUAL_MODE(mode))
2053ebccf1e3SJoseph Koshy 				    return EEXIST;
2054c5153e19SJoseph Koshy 			    if (PMC_IS_SYSTEM_MODE(mode) &&
2055c5153e19SJoseph Koshy 				(int) PMC_TO_CPU(pm) == cpu)
2056c5153e19SJoseph Koshy 				    return EEXIST;
2057c5153e19SJoseph Koshy 		    }
2058c5153e19SJoseph Koshy 	        }
2059ebccf1e3SJoseph Koshy 
2060c5153e19SJoseph Koshy 	/*
2061c5153e19SJoseph Koshy 	 * We also shouldn't be the target of any PMC at this index
2062c5153e19SJoseph Koshy 	 * since otherwise a PMC_ATTACH to ourselves will fail.
2063c5153e19SJoseph Koshy 	 */
2064ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2065ebccf1e3SJoseph Koshy 		if (pp->pp_pmcs[ri].pp_pmc)
2066ebccf1e3SJoseph Koshy 			return EEXIST;
2067ebccf1e3SJoseph Koshy 
2068ebccf1e3SJoseph Koshy 	PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2069ebccf1e3SJoseph Koshy 	    p, p->p_pid, p->p_comm, ri);
2070ebccf1e3SJoseph Koshy 
2071ebccf1e3SJoseph Koshy 	return 0;
2072ebccf1e3SJoseph Koshy }
2073ebccf1e3SJoseph Koshy 
2074ebccf1e3SJoseph Koshy /*
2075ebccf1e3SJoseph Koshy  * Check if a given PMC at row index 'ri' can be currently used in
2076ebccf1e3SJoseph Koshy  * mode 'mode'.
2077ebccf1e3SJoseph Koshy  */
2078ebccf1e3SJoseph Koshy 
2079ebccf1e3SJoseph Koshy static int
2080ebccf1e3SJoseph Koshy pmc_can_allocate_row(int ri, enum pmc_mode mode)
2081ebccf1e3SJoseph Koshy {
2082ebccf1e3SJoseph Koshy 	enum pmc_disp	disp;
2083ebccf1e3SJoseph Koshy 
2084ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
2085ebccf1e3SJoseph Koshy 
2086ebccf1e3SJoseph Koshy 	PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
2087ebccf1e3SJoseph Koshy 
2088ebccf1e3SJoseph Koshy 	if (PMC_IS_SYSTEM_MODE(mode))
2089ebccf1e3SJoseph Koshy 		disp = PMC_DISP_STANDALONE;
2090ebccf1e3SJoseph Koshy 	else
2091ebccf1e3SJoseph Koshy 		disp = PMC_DISP_THREAD;
2092ebccf1e3SJoseph Koshy 
2093ebccf1e3SJoseph Koshy 	/*
2094ebccf1e3SJoseph Koshy 	 * check disposition for PMC row 'ri':
2095ebccf1e3SJoseph Koshy 	 *
2096ebccf1e3SJoseph Koshy 	 * Expected disposition		Row-disposition		Result
2097ebccf1e3SJoseph Koshy 	 *
2098ebccf1e3SJoseph Koshy 	 * STANDALONE			STANDALONE or FREE	proceed
2099ebccf1e3SJoseph Koshy 	 * STANDALONE			THREAD			fail
2100ebccf1e3SJoseph Koshy 	 * THREAD			THREAD or FREE		proceed
2101ebccf1e3SJoseph Koshy 	 * THREAD			STANDALONE		fail
2102ebccf1e3SJoseph Koshy 	 */
2103ebccf1e3SJoseph Koshy 
2104ebccf1e3SJoseph Koshy 	if (!PMC_ROW_DISP_IS_FREE(ri) &&
2105ebccf1e3SJoseph Koshy 	    !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
2106ebccf1e3SJoseph Koshy 	    !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
2107ebccf1e3SJoseph Koshy 		return EBUSY;
2108ebccf1e3SJoseph Koshy 
2109ebccf1e3SJoseph Koshy 	/*
2110ebccf1e3SJoseph Koshy 	 * All OK
2111ebccf1e3SJoseph Koshy 	 */
2112ebccf1e3SJoseph Koshy 
2113ebccf1e3SJoseph Koshy 	PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
2114ebccf1e3SJoseph Koshy 
2115ebccf1e3SJoseph Koshy 	return 0;
2116ebccf1e3SJoseph Koshy 
2117ebccf1e3SJoseph Koshy }
2118ebccf1e3SJoseph Koshy 
2119ebccf1e3SJoseph Koshy /*
2120c5153e19SJoseph Koshy  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
2121ebccf1e3SJoseph Koshy  */
2122ebccf1e3SJoseph Koshy 
2123ebccf1e3SJoseph Koshy static struct pmc *
2124ebccf1e3SJoseph Koshy pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
2125ebccf1e3SJoseph Koshy {
2126f263522aSJoseph Koshy 	struct pmc *pm;
2127ebccf1e3SJoseph Koshy 
2128c5153e19SJoseph Koshy 	KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
2129c5153e19SJoseph Koshy 	    ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
2130c5153e19SJoseph Koshy 		PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
2131ebccf1e3SJoseph Koshy 
2132f263522aSJoseph Koshy 	LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2133f263522aSJoseph Koshy 	    if (pm->pm_id == pmcid)
2134f263522aSJoseph Koshy 		    return pm;
2135ebccf1e3SJoseph Koshy 
2136ebccf1e3SJoseph Koshy 	return NULL;
2137ebccf1e3SJoseph Koshy }
2138ebccf1e3SJoseph Koshy 
2139ebccf1e3SJoseph Koshy static int
2140ebccf1e3SJoseph Koshy pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
2141ebccf1e3SJoseph Koshy {
2142ebccf1e3SJoseph Koshy 
2143ebccf1e3SJoseph Koshy 	struct pmc *pm;
2144ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
2145ebccf1e3SJoseph Koshy 
2146ebccf1e3SJoseph Koshy 	PMCDBG(PMC,FND,1, "find-pmc id=%d", pmcid);
2147ebccf1e3SJoseph Koshy 
2148ebccf1e3SJoseph Koshy 	if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL)
2149ebccf1e3SJoseph Koshy 		return ESRCH;
2150ebccf1e3SJoseph Koshy 
2151ebccf1e3SJoseph Koshy 	if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
2152ebccf1e3SJoseph Koshy 		return EINVAL;
2153ebccf1e3SJoseph Koshy 
2154ebccf1e3SJoseph Koshy 	PMCDBG(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
2155ebccf1e3SJoseph Koshy 
2156ebccf1e3SJoseph Koshy 	*pmc = pm;
2157ebccf1e3SJoseph Koshy 	return 0;
2158ebccf1e3SJoseph Koshy }
2159ebccf1e3SJoseph Koshy 
2160ebccf1e3SJoseph Koshy /*
2161ebccf1e3SJoseph Koshy  * Start a PMC.
2162ebccf1e3SJoseph Koshy  */
2163ebccf1e3SJoseph Koshy 
2164ebccf1e3SJoseph Koshy static int
2165ebccf1e3SJoseph Koshy pmc_start(struct pmc *pm)
2166ebccf1e3SJoseph Koshy {
2167ebccf1e3SJoseph Koshy 	int error, cpu, ri;
2168c5153e19SJoseph Koshy 	enum pmc_mode mode;
2169f263522aSJoseph Koshy 	struct pmc_owner *po;
2170ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
2171ebccf1e3SJoseph Koshy 
2172ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL,
2173ebccf1e3SJoseph Koshy 	    ("[pmc,%d] null pm", __LINE__));
2174ebccf1e3SJoseph Koshy 
2175c5153e19SJoseph Koshy 	mode = PMC_TO_MODE(pm);
2176c5153e19SJoseph Koshy 	ri   = PMC_TO_ROWINDEX(pm);
2177c5153e19SJoseph Koshy 	error = 0;
2178c5153e19SJoseph Koshy 
2179c5153e19SJoseph Koshy 	PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2180ebccf1e3SJoseph Koshy 
2181f263522aSJoseph Koshy 	po = pm->pm_owner;
2182ebccf1e3SJoseph Koshy 
2183c5153e19SJoseph Koshy 	if (PMC_IS_VIRTUAL_MODE(mode)) {
2184ebccf1e3SJoseph Koshy 
2185ebccf1e3SJoseph Koshy 		/*
2186f263522aSJoseph Koshy 		 * If a PMCATTACH has never been done on this PMC,
2187f263522aSJoseph Koshy 		 * attach it to its owner process.
2188ebccf1e3SJoseph Koshy 		 */
2189ebccf1e3SJoseph Koshy 
2190ebccf1e3SJoseph Koshy 		if (LIST_EMPTY(&pm->pm_targets))
2191f263522aSJoseph Koshy 			error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
2192f263522aSJoseph Koshy 			    pmc_attach_process(po->po_owner, pm);
2193f263522aSJoseph Koshy 
2194f263522aSJoseph Koshy 		/*
2195f263522aSJoseph Koshy 		 * Disallow PMCSTART if a logfile is required but has not
2196f263522aSJoseph Koshy 		 * been configured yet.
2197f263522aSJoseph Koshy 		 */
2198f263522aSJoseph Koshy 
2199f263522aSJoseph Koshy 		if (error == 0 && (pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2200f263522aSJoseph Koshy 		    (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2201f263522aSJoseph Koshy 			error = EDOOFUS;
2202ebccf1e3SJoseph Koshy 
2203c5153e19SJoseph Koshy 		/*
2204c5153e19SJoseph Koshy 		 * If the PMC is attached to its owner, then force a context
2205c5153e19SJoseph Koshy 		 * switch to ensure that the MD state gets set correctly.
2206c5153e19SJoseph Koshy 		 */
2207ebccf1e3SJoseph Koshy 
2208f263522aSJoseph Koshy 		if (error == 0) {
2209f263522aSJoseph Koshy 			pm->pm_state = PMC_STATE_RUNNING;
2210f263522aSJoseph Koshy 			if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2211f263522aSJoseph Koshy 				pmc_force_context_switch();
2212f263522aSJoseph Koshy 		}
2213ebccf1e3SJoseph Koshy 
2214c5153e19SJoseph Koshy 		return error;
2215f263522aSJoseph Koshy 	}
2216ebccf1e3SJoseph Koshy 
2217f263522aSJoseph Koshy 
2218f263522aSJoseph Koshy 	/*
2219f263522aSJoseph Koshy 	 * A system-wide PMC.
2220f263522aSJoseph Koshy 	 */
2221f263522aSJoseph Koshy 
2222f263522aSJoseph Koshy 	if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2223f263522aSJoseph Koshy 	    (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2224f263522aSJoseph Koshy 		return EDOOFUS;	/* programming error */
2225f263522aSJoseph Koshy 
2226f263522aSJoseph Koshy 	/*
2227f263522aSJoseph Koshy 	 * Add the owner to the global list if this is a system-wide
2228f263522aSJoseph Koshy 	 * sampling PMC.
2229f263522aSJoseph Koshy 	 */
2230f263522aSJoseph Koshy 
2231f263522aSJoseph Koshy 	if (mode == PMC_MODE_SS) {
2232f263522aSJoseph Koshy 		if (po->po_sscount == 0) {
2233f263522aSJoseph Koshy 			LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
2234f263522aSJoseph Koshy 			atomic_add_rel_int(&pmc_ss_count, 1);
2235f263522aSJoseph Koshy 			PMCDBG(PMC,OPS,1, "po=%p in global list", po);
2236f263522aSJoseph Koshy 		}
2237f263522aSJoseph Koshy 		po->po_sscount++;
2238ebccf1e3SJoseph Koshy 	}
2239ebccf1e3SJoseph Koshy 
2240ebccf1e3SJoseph Koshy 	/*
2241f263522aSJoseph Koshy 	 * Move to the CPU associated with this
2242ebccf1e3SJoseph Koshy 	 * PMC, and start the hardware.
2243ebccf1e3SJoseph Koshy 	 */
2244ebccf1e3SJoseph Koshy 
2245ebccf1e3SJoseph Koshy 	pmc_save_cpu_binding(&pb);
2246ebccf1e3SJoseph Koshy 
2247c5153e19SJoseph Koshy 	cpu = PMC_TO_CPU(pm);
2248ebccf1e3SJoseph Koshy 
2249ebccf1e3SJoseph Koshy 	if (pmc_cpu_is_disabled(cpu))
2250ebccf1e3SJoseph Koshy 		return ENXIO;
2251ebccf1e3SJoseph Koshy 
2252ebccf1e3SJoseph Koshy 	pmc_select_cpu(cpu);
2253ebccf1e3SJoseph Koshy 
2254ebccf1e3SJoseph Koshy 	/*
2255ebccf1e3SJoseph Koshy 	 * global PMCs are configured at allocation time
2256ebccf1e3SJoseph Koshy 	 * so write out the initial value and start the PMC.
2257ebccf1e3SJoseph Koshy 	 */
2258ebccf1e3SJoseph Koshy 
2259f263522aSJoseph Koshy 	pm->pm_state = PMC_STATE_RUNNING;
2260f263522aSJoseph Koshy 
2261c5153e19SJoseph Koshy 	critical_enter();
2262ebccf1e3SJoseph Koshy 	if ((error = md->pmd_write_pmc(cpu, ri,
2263c5153e19SJoseph Koshy 		 PMC_IS_SAMPLING_MODE(mode) ?
2264ebccf1e3SJoseph Koshy 		 pm->pm_sc.pm_reloadcount :
2265ebccf1e3SJoseph Koshy 		 pm->pm_sc.pm_initial)) == 0)
2266ebccf1e3SJoseph Koshy 		error = md->pmd_start_pmc(cpu, ri);
2267c5153e19SJoseph Koshy 	critical_exit();
2268ebccf1e3SJoseph Koshy 
2269ebccf1e3SJoseph Koshy 	pmc_restore_cpu_binding(&pb);
2270ebccf1e3SJoseph Koshy 
2271ebccf1e3SJoseph Koshy 	return error;
2272ebccf1e3SJoseph Koshy }
2273ebccf1e3SJoseph Koshy 
2274ebccf1e3SJoseph Koshy /*
2275ebccf1e3SJoseph Koshy  * Stop a PMC.
2276ebccf1e3SJoseph Koshy  */
2277ebccf1e3SJoseph Koshy 
2278ebccf1e3SJoseph Koshy static int
2279ebccf1e3SJoseph Koshy pmc_stop(struct pmc *pm)
2280ebccf1e3SJoseph Koshy {
2281c5153e19SJoseph Koshy 	int cpu, error, ri;
2282f263522aSJoseph Koshy 	struct pmc_owner *po;
2283ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
2284ebccf1e3SJoseph Koshy 
2285ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2286ebccf1e3SJoseph Koshy 
2287c5153e19SJoseph Koshy 	PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2288c5153e19SJoseph Koshy 	    PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2289ebccf1e3SJoseph Koshy 
2290ebccf1e3SJoseph Koshy 	pm->pm_state = PMC_STATE_STOPPED;
2291ebccf1e3SJoseph Koshy 
2292ebccf1e3SJoseph Koshy 	/*
2293ebccf1e3SJoseph Koshy 	 * If the PMC is a virtual mode one, changing the state to
2294ebccf1e3SJoseph Koshy 	 * non-RUNNING is enough to ensure that the PMC never gets
2295ebccf1e3SJoseph Koshy 	 * scheduled.
2296ebccf1e3SJoseph Koshy 	 *
2297ebccf1e3SJoseph Koshy 	 * If this PMC is current running on a CPU, then it will
2298ebccf1e3SJoseph Koshy 	 * handled correctly at the time its target process is context
2299ebccf1e3SJoseph Koshy 	 * switched out.
2300ebccf1e3SJoseph Koshy 	 */
2301ebccf1e3SJoseph Koshy 
2302c5153e19SJoseph Koshy 	if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
2303ebccf1e3SJoseph Koshy 		return 0;
2304ebccf1e3SJoseph Koshy 
2305ebccf1e3SJoseph Koshy 	/*
2306ebccf1e3SJoseph Koshy 	 * A system-mode PMC.  Move to the CPU associated with
2307ebccf1e3SJoseph Koshy 	 * this PMC, and stop the hardware.  We update the
2308ebccf1e3SJoseph Koshy 	 * 'initial count' so that a subsequent PMCSTART will
2309ebccf1e3SJoseph Koshy 	 * resume counting from the current hardware count.
2310ebccf1e3SJoseph Koshy 	 */
2311ebccf1e3SJoseph Koshy 
2312ebccf1e3SJoseph Koshy 	pmc_save_cpu_binding(&pb);
2313ebccf1e3SJoseph Koshy 
2314c5153e19SJoseph Koshy 	cpu = PMC_TO_CPU(pm);
2315c5153e19SJoseph Koshy 
2316c5153e19SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < mp_ncpus,
2317c5153e19SJoseph Koshy 	    ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2318ebccf1e3SJoseph Koshy 
2319ebccf1e3SJoseph Koshy 	if (pmc_cpu_is_disabled(cpu))
2320ebccf1e3SJoseph Koshy 		return ENXIO;
2321ebccf1e3SJoseph Koshy 
2322ebccf1e3SJoseph Koshy 	pmc_select_cpu(cpu);
2323ebccf1e3SJoseph Koshy 
2324c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
2325c5153e19SJoseph Koshy 
2326c5153e19SJoseph Koshy 	critical_enter();
2327c5153e19SJoseph Koshy 	if ((error = md->pmd_stop_pmc(cpu, ri)) == 0)
2328c5153e19SJoseph Koshy 		error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial);
2329c5153e19SJoseph Koshy 	critical_exit();
2330ebccf1e3SJoseph Koshy 
2331ebccf1e3SJoseph Koshy 	pmc_restore_cpu_binding(&pb);
2332ebccf1e3SJoseph Koshy 
2333f263522aSJoseph Koshy 	po = pm->pm_owner;
2334f263522aSJoseph Koshy 
2335f263522aSJoseph Koshy 	/* remove this owner from the global list of SS PMC owners */
2336f263522aSJoseph Koshy 	if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
2337f263522aSJoseph Koshy 		po->po_sscount--;
2338f263522aSJoseph Koshy 		if (po->po_sscount == 0) {
2339f263522aSJoseph Koshy 			atomic_subtract_rel_int(&pmc_ss_count, 1);
2340f263522aSJoseph Koshy 			LIST_REMOVE(po, po_ssnext);
2341f263522aSJoseph Koshy 			PMCDBG(PMC,OPS,2,"po=%p removed from global list", po);
2342f263522aSJoseph Koshy 		}
2343f263522aSJoseph Koshy 	}
2344f263522aSJoseph Koshy 
2345ebccf1e3SJoseph Koshy 	return error;
2346ebccf1e3SJoseph Koshy }
2347ebccf1e3SJoseph Koshy 
2348ebccf1e3SJoseph Koshy 
2349342ed5d9SRuslan Ermilov #ifdef	DEBUG
2350ebccf1e3SJoseph Koshy static const char *pmc_op_to_name[] = {
2351ebccf1e3SJoseph Koshy #undef	__PMC_OP
2352ebccf1e3SJoseph Koshy #define	__PMC_OP(N, D)	#N ,
2353ebccf1e3SJoseph Koshy 	__PMC_OPS()
2354ebccf1e3SJoseph Koshy 	NULL
2355ebccf1e3SJoseph Koshy };
2356ebccf1e3SJoseph Koshy #endif
2357ebccf1e3SJoseph Koshy 
2358ebccf1e3SJoseph Koshy /*
2359ebccf1e3SJoseph Koshy  * The syscall interface
2360ebccf1e3SJoseph Koshy  */
2361ebccf1e3SJoseph Koshy 
2362ebccf1e3SJoseph Koshy #define	PMC_GET_SX_XLOCK(...) do {		\
2363ebccf1e3SJoseph Koshy 	sx_xlock(&pmc_sx);			\
2364ebccf1e3SJoseph Koshy 	if (pmc_hook == NULL) {			\
2365ebccf1e3SJoseph Koshy 		sx_xunlock(&pmc_sx);		\
2366ebccf1e3SJoseph Koshy 		return __VA_ARGS__;		\
2367ebccf1e3SJoseph Koshy 	}					\
2368ebccf1e3SJoseph Koshy } while (0)
2369ebccf1e3SJoseph Koshy 
2370ebccf1e3SJoseph Koshy #define	PMC_DOWNGRADE_SX() do {			\
2371ebccf1e3SJoseph Koshy 	sx_downgrade(&pmc_sx);			\
2372ebccf1e3SJoseph Koshy 	is_sx_downgraded = 1;			\
2373ebccf1e3SJoseph Koshy } while (0)
2374ebccf1e3SJoseph Koshy 
2375ebccf1e3SJoseph Koshy static int
2376ebccf1e3SJoseph Koshy pmc_syscall_handler(struct thread *td, void *syscall_args)
2377ebccf1e3SJoseph Koshy {
2378ebccf1e3SJoseph Koshy 	int error, is_sx_downgraded, op;
2379ebccf1e3SJoseph Koshy 	struct pmc_syscall_args *c;
2380ebccf1e3SJoseph Koshy 	void *arg;
2381ebccf1e3SJoseph Koshy 
2382ebccf1e3SJoseph Koshy 	PMC_GET_SX_XLOCK(ENOSYS);
2383ebccf1e3SJoseph Koshy 
2384f263522aSJoseph Koshy 	DROP_GIANT();
2385f263522aSJoseph Koshy 
2386ebccf1e3SJoseph Koshy 	is_sx_downgraded = 0;
2387ebccf1e3SJoseph Koshy 
2388ebccf1e3SJoseph Koshy 	c = (struct pmc_syscall_args *) syscall_args;
2389ebccf1e3SJoseph Koshy 
2390ebccf1e3SJoseph Koshy 	op = c->pmop_code;
2391ebccf1e3SJoseph Koshy 	arg = c->pmop_data;
2392ebccf1e3SJoseph Koshy 
2393ebccf1e3SJoseph Koshy 	PMCDBG(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
2394ebccf1e3SJoseph Koshy 	    pmc_op_to_name[op], arg);
2395ebccf1e3SJoseph Koshy 
2396ebccf1e3SJoseph Koshy 	error = 0;
2397ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_stats.pm_syscalls, 1);
2398ebccf1e3SJoseph Koshy 
2399ebccf1e3SJoseph Koshy 	switch(op)
2400ebccf1e3SJoseph Koshy 	{
2401ebccf1e3SJoseph Koshy 
2402ebccf1e3SJoseph Koshy 
2403ebccf1e3SJoseph Koshy 	/*
2404ebccf1e3SJoseph Koshy 	 * Configure a log file.
2405ebccf1e3SJoseph Koshy 	 *
2406ebccf1e3SJoseph Koshy 	 * XXX This OP will be reworked.
2407ebccf1e3SJoseph Koshy 	 */
2408ebccf1e3SJoseph Koshy 
2409ebccf1e3SJoseph Koshy 	case PMC_OP_CONFIGURELOG:
2410ebccf1e3SJoseph Koshy 	{
2411fc9a2b80SJoseph Koshy 		struct pmc *pm;
2412ebccf1e3SJoseph Koshy 		struct pmc_owner *po;
2413ebccf1e3SJoseph Koshy 		struct pmc_op_configurelog cl;
2414ebccf1e3SJoseph Koshy 		struct proc *p;
2415ebccf1e3SJoseph Koshy 
2416ebccf1e3SJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
2417ebccf1e3SJoseph Koshy 
2418ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
2419ebccf1e3SJoseph Koshy 			break;
2420ebccf1e3SJoseph Koshy 
2421ebccf1e3SJoseph Koshy 		/* mark this process as owning a log file */
2422ebccf1e3SJoseph Koshy 		p = td->td_proc;
2423ebccf1e3SJoseph Koshy 		if ((po = pmc_find_owner_descriptor(p)) == NULL)
2424f263522aSJoseph Koshy 			if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
2425f263522aSJoseph Koshy 				error = ENOMEM;
2426ebccf1e3SJoseph Koshy 				break;
2427f263522aSJoseph Koshy 			}
2428ebccf1e3SJoseph Koshy 
2429f263522aSJoseph Koshy 		/*
2430f263522aSJoseph Koshy 		 * If a valid fd was passed in, try to configure that,
2431f263522aSJoseph Koshy 		 * otherwise if 'fd' was less than zero and there was
2432f263522aSJoseph Koshy 		 * a log file configured, flush its buffers and
2433f263522aSJoseph Koshy 		 * de-configure it.
2434f263522aSJoseph Koshy 		 */
2435f263522aSJoseph Koshy 		if (cl.pm_logfd >= 0)
2436f263522aSJoseph Koshy 			error = pmclog_configure_log(po, cl.pm_logfd);
2437f263522aSJoseph Koshy 		else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
2438f263522aSJoseph Koshy 			pmclog_process_closelog(po);
2439f263522aSJoseph Koshy 			error = pmclog_flush(po);
2440fc9a2b80SJoseph Koshy 			if (error == 0) {
2441fc9a2b80SJoseph Koshy 				LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2442fc9a2b80SJoseph Koshy 				    if (pm->pm_flags & PMC_F_NEEDS_LOGFILE)
2443fc9a2b80SJoseph Koshy 					    pmc_stop(pm);
2444f263522aSJoseph Koshy 				error = pmclog_deconfigure_log(po);
2445fc9a2b80SJoseph Koshy 			}
2446f263522aSJoseph Koshy 		} else
2447f263522aSJoseph Koshy 			error = EINVAL;
2448ebccf1e3SJoseph Koshy 	}
2449ebccf1e3SJoseph Koshy 	break;
2450ebccf1e3SJoseph Koshy 
2451ebccf1e3SJoseph Koshy 
2452ebccf1e3SJoseph Koshy 	/*
2453f263522aSJoseph Koshy 	 * Flush a log file.
2454f263522aSJoseph Koshy 	 */
2455f263522aSJoseph Koshy 
2456f263522aSJoseph Koshy 	case PMC_OP_FLUSHLOG:
2457f263522aSJoseph Koshy 	{
2458f263522aSJoseph Koshy 		struct pmc_owner *po;
2459f263522aSJoseph Koshy 
2460f263522aSJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
2461f263522aSJoseph Koshy 
2462f263522aSJoseph Koshy 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
2463f263522aSJoseph Koshy 			error = EINVAL;
2464f263522aSJoseph Koshy 			break;
2465f263522aSJoseph Koshy 		}
2466f263522aSJoseph Koshy 
2467f263522aSJoseph Koshy 		error = pmclog_flush(po);
2468f263522aSJoseph Koshy 	}
2469f263522aSJoseph Koshy 	break;
2470f263522aSJoseph Koshy 
2471f263522aSJoseph Koshy 	/*
2472ebccf1e3SJoseph Koshy 	 * Retrieve hardware configuration.
2473ebccf1e3SJoseph Koshy 	 */
2474ebccf1e3SJoseph Koshy 
2475ebccf1e3SJoseph Koshy 	case PMC_OP_GETCPUINFO:	/* CPU information */
2476ebccf1e3SJoseph Koshy 	{
2477ebccf1e3SJoseph Koshy 		struct pmc_op_getcpuinfo gci;
2478ebccf1e3SJoseph Koshy 
2479ebccf1e3SJoseph Koshy 		gci.pm_cputype = md->pmd_cputype;
2480c5153e19SJoseph Koshy 		gci.pm_ncpu    = mp_ncpus;
2481ebccf1e3SJoseph Koshy 		gci.pm_npmc    = md->pmd_npmc;
2482ebccf1e3SJoseph Koshy 		gci.pm_nclass  = md->pmd_nclass;
2483ebccf1e3SJoseph Koshy 		bcopy(md->pmd_classes, &gci.pm_classes,
2484ebccf1e3SJoseph Koshy 		    sizeof(gci.pm_classes));
2485ebccf1e3SJoseph Koshy 		error = copyout(&gci, arg, sizeof(gci));
2486ebccf1e3SJoseph Koshy 	}
2487ebccf1e3SJoseph Koshy 	break;
2488ebccf1e3SJoseph Koshy 
2489ebccf1e3SJoseph Koshy 
2490ebccf1e3SJoseph Koshy 	/*
2491ebccf1e3SJoseph Koshy 	 * Get module statistics
2492ebccf1e3SJoseph Koshy 	 */
2493ebccf1e3SJoseph Koshy 
2494ebccf1e3SJoseph Koshy 	case PMC_OP_GETDRIVERSTATS:
2495ebccf1e3SJoseph Koshy 	{
2496ebccf1e3SJoseph Koshy 		struct pmc_op_getdriverstats gms;
2497ebccf1e3SJoseph Koshy 
2498ebccf1e3SJoseph Koshy 		bcopy(&pmc_stats, &gms, sizeof(gms));
2499ebccf1e3SJoseph Koshy 		error = copyout(&gms, arg, sizeof(gms));
2500ebccf1e3SJoseph Koshy 	}
2501ebccf1e3SJoseph Koshy 	break;
2502ebccf1e3SJoseph Koshy 
2503ebccf1e3SJoseph Koshy 
2504ebccf1e3SJoseph Koshy 	/*
2505ebccf1e3SJoseph Koshy 	 * Retrieve module version number
2506ebccf1e3SJoseph Koshy 	 */
2507ebccf1e3SJoseph Koshy 
2508ebccf1e3SJoseph Koshy 	case PMC_OP_GETMODULEVERSION:
2509ebccf1e3SJoseph Koshy 	{
2510f263522aSJoseph Koshy 		uint32_t cv, modv;
2511f263522aSJoseph Koshy 
2512f263522aSJoseph Koshy 		/* retrieve the client's idea of the ABI version */
2513f263522aSJoseph Koshy 		if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
2514f263522aSJoseph Koshy 			break;
2515f263522aSJoseph Koshy 		/* don't service clients newer than our driver */
2516f263522aSJoseph Koshy 		modv = PMC_VERSION;
2517f263522aSJoseph Koshy 		if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
2518f263522aSJoseph Koshy 			error = EPROGMISMATCH;
2519f263522aSJoseph Koshy 			break;
2520f263522aSJoseph Koshy 		}
2521f263522aSJoseph Koshy 		error = copyout(&modv, arg, sizeof(int));
2522ebccf1e3SJoseph Koshy 	}
2523ebccf1e3SJoseph Koshy 	break;
2524ebccf1e3SJoseph Koshy 
2525ebccf1e3SJoseph Koshy 
2526ebccf1e3SJoseph Koshy 	/*
2527ebccf1e3SJoseph Koshy 	 * Retrieve the state of all the PMCs on a given
2528ebccf1e3SJoseph Koshy 	 * CPU.
2529ebccf1e3SJoseph Koshy 	 */
2530ebccf1e3SJoseph Koshy 
2531ebccf1e3SJoseph Koshy 	case PMC_OP_GETPMCINFO:
2532ebccf1e3SJoseph Koshy 	{
2533ebccf1e3SJoseph Koshy 		uint32_t cpu, n, npmc;
2534ebccf1e3SJoseph Koshy 		size_t pmcinfo_size;
2535ebccf1e3SJoseph Koshy 		struct pmc *pm;
2536ebccf1e3SJoseph Koshy 		struct pmc_info *p, *pmcinfo;
2537ebccf1e3SJoseph Koshy 		struct pmc_op_getpmcinfo *gpi;
2538ebccf1e3SJoseph Koshy 		struct pmc_owner *po;
2539ebccf1e3SJoseph Koshy 		struct pmc_binding pb;
2540ebccf1e3SJoseph Koshy 
2541ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
2542ebccf1e3SJoseph Koshy 
2543ebccf1e3SJoseph Koshy 		gpi = (struct pmc_op_getpmcinfo *) arg;
2544ebccf1e3SJoseph Koshy 
2545ebccf1e3SJoseph Koshy 		if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
2546ebccf1e3SJoseph Koshy 			break;
2547ebccf1e3SJoseph Koshy 
2548ebccf1e3SJoseph Koshy 		if (cpu >= (unsigned int) mp_ncpus) {
2549ebccf1e3SJoseph Koshy 			error = EINVAL;
2550ebccf1e3SJoseph Koshy 			break;
2551ebccf1e3SJoseph Koshy 		}
2552ebccf1e3SJoseph Koshy 
2553ebccf1e3SJoseph Koshy 		if (pmc_cpu_is_disabled(cpu)) {
2554ebccf1e3SJoseph Koshy 			error = ENXIO;
2555ebccf1e3SJoseph Koshy 			break;
2556ebccf1e3SJoseph Koshy 		}
2557ebccf1e3SJoseph Koshy 
2558ebccf1e3SJoseph Koshy 		/* switch to CPU 'cpu' */
2559ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
2560ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
2561ebccf1e3SJoseph Koshy 
2562ebccf1e3SJoseph Koshy 		npmc = md->pmd_npmc;
2563ebccf1e3SJoseph Koshy 
2564ebccf1e3SJoseph Koshy 		pmcinfo_size = npmc * sizeof(struct pmc_info);
2565ebccf1e3SJoseph Koshy 		MALLOC(pmcinfo, struct pmc_info *, pmcinfo_size, M_PMC,
2566ebccf1e3SJoseph Koshy 		    M_WAITOK);
2567ebccf1e3SJoseph Koshy 
2568ebccf1e3SJoseph Koshy 		p = pmcinfo;
2569ebccf1e3SJoseph Koshy 
2570ebccf1e3SJoseph Koshy 		for (n = 0; n < md->pmd_npmc; n++, p++) {
2571ebccf1e3SJoseph Koshy 
2572ebccf1e3SJoseph Koshy 			if ((error = md->pmd_describe(cpu, n, p, &pm)) != 0)
2573ebccf1e3SJoseph Koshy 				break;
2574ebccf1e3SJoseph Koshy 
2575ebccf1e3SJoseph Koshy 			if (PMC_ROW_DISP_IS_STANDALONE(n))
2576ebccf1e3SJoseph Koshy 				p->pm_rowdisp = PMC_DISP_STANDALONE;
2577ebccf1e3SJoseph Koshy 			else if (PMC_ROW_DISP_IS_THREAD(n))
2578ebccf1e3SJoseph Koshy 				p->pm_rowdisp = PMC_DISP_THREAD;
2579ebccf1e3SJoseph Koshy 			else
2580ebccf1e3SJoseph Koshy 				p->pm_rowdisp = PMC_DISP_FREE;
2581ebccf1e3SJoseph Koshy 
2582ebccf1e3SJoseph Koshy 			p->pm_ownerpid = -1;
2583ebccf1e3SJoseph Koshy 
2584ebccf1e3SJoseph Koshy 			if (pm == NULL)	/* no PMC associated */
2585ebccf1e3SJoseph Koshy 				continue;
2586ebccf1e3SJoseph Koshy 
2587ebccf1e3SJoseph Koshy 			po = pm->pm_owner;
2588ebccf1e3SJoseph Koshy 
2589ebccf1e3SJoseph Koshy 			KASSERT(po->po_owner != NULL,
2590ebccf1e3SJoseph Koshy 			    ("[pmc,%d] pmc_owner had a null proc pointer",
2591ebccf1e3SJoseph Koshy 				__LINE__));
2592ebccf1e3SJoseph Koshy 
2593ebccf1e3SJoseph Koshy 			p->pm_ownerpid = po->po_owner->p_pid;
2594c5153e19SJoseph Koshy 			p->pm_mode     = PMC_TO_MODE(pm);
2595ebccf1e3SJoseph Koshy 			p->pm_event    = pm->pm_event;
2596ebccf1e3SJoseph Koshy 			p->pm_flags    = pm->pm_flags;
2597ebccf1e3SJoseph Koshy 
2598c5153e19SJoseph Koshy 			if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2599ebccf1e3SJoseph Koshy 				p->pm_reloadcount =
2600ebccf1e3SJoseph Koshy 				    pm->pm_sc.pm_reloadcount;
2601ebccf1e3SJoseph Koshy 		}
2602ebccf1e3SJoseph Koshy 
2603ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
2604ebccf1e3SJoseph Koshy 
2605ebccf1e3SJoseph Koshy 		/* now copy out the PMC info collected */
2606ebccf1e3SJoseph Koshy 		if (error == 0)
2607ebccf1e3SJoseph Koshy 			error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
2608ebccf1e3SJoseph Koshy 
2609ebccf1e3SJoseph Koshy 		FREE(pmcinfo, M_PMC);
2610ebccf1e3SJoseph Koshy 	}
2611ebccf1e3SJoseph Koshy 	break;
2612ebccf1e3SJoseph Koshy 
2613ebccf1e3SJoseph Koshy 
2614ebccf1e3SJoseph Koshy 	/*
2615ebccf1e3SJoseph Koshy 	 * Set the administrative state of a PMC.  I.e. whether
2616ebccf1e3SJoseph Koshy 	 * the PMC is to be used or not.
2617ebccf1e3SJoseph Koshy 	 */
2618ebccf1e3SJoseph Koshy 
2619ebccf1e3SJoseph Koshy 	case PMC_OP_PMCADMIN:
2620ebccf1e3SJoseph Koshy 	{
2621ebccf1e3SJoseph Koshy 		int cpu, ri;
2622ebccf1e3SJoseph Koshy 		enum pmc_state request;
2623ebccf1e3SJoseph Koshy 		struct pmc_cpu *pc;
2624ebccf1e3SJoseph Koshy 		struct pmc_hw *phw;
2625ebccf1e3SJoseph Koshy 		struct pmc_op_pmcadmin pma;
2626ebccf1e3SJoseph Koshy 		struct pmc_binding pb;
2627ebccf1e3SJoseph Koshy 
2628ebccf1e3SJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
2629ebccf1e3SJoseph Koshy 
2630ebccf1e3SJoseph Koshy 		KASSERT(td == curthread,
2631ebccf1e3SJoseph Koshy 		    ("[pmc,%d] td != curthread", __LINE__));
2632ebccf1e3SJoseph Koshy 
2633ebccf1e3SJoseph Koshy 		if (suser(td) || jailed(td->td_ucred)) {
2634ebccf1e3SJoseph Koshy 			error =  EPERM;
2635ebccf1e3SJoseph Koshy 			break;
2636ebccf1e3SJoseph Koshy 		}
2637ebccf1e3SJoseph Koshy 
2638ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
2639ebccf1e3SJoseph Koshy 			break;
2640ebccf1e3SJoseph Koshy 
2641ebccf1e3SJoseph Koshy 		cpu = pma.pm_cpu;
2642ebccf1e3SJoseph Koshy 
2643ebccf1e3SJoseph Koshy 		if (cpu < 0 || cpu >= mp_ncpus) {
2644ebccf1e3SJoseph Koshy 			error = EINVAL;
2645ebccf1e3SJoseph Koshy 			break;
2646ebccf1e3SJoseph Koshy 		}
2647ebccf1e3SJoseph Koshy 
2648ebccf1e3SJoseph Koshy 		if (pmc_cpu_is_disabled(cpu)) {
2649ebccf1e3SJoseph Koshy 			error = ENXIO;
2650ebccf1e3SJoseph Koshy 			break;
2651ebccf1e3SJoseph Koshy 		}
2652ebccf1e3SJoseph Koshy 
2653ebccf1e3SJoseph Koshy 		request = pma.pm_state;
2654ebccf1e3SJoseph Koshy 
2655ebccf1e3SJoseph Koshy 		if (request != PMC_STATE_DISABLED &&
2656ebccf1e3SJoseph Koshy 		    request != PMC_STATE_FREE) {
2657ebccf1e3SJoseph Koshy 			error = EINVAL;
2658ebccf1e3SJoseph Koshy 			break;
2659ebccf1e3SJoseph Koshy 		}
2660ebccf1e3SJoseph Koshy 
2661ebccf1e3SJoseph Koshy 		ri = pma.pm_pmc; /* pmc id == row index */
2662ebccf1e3SJoseph Koshy 		if (ri < 0 || ri >= (int) md->pmd_npmc) {
2663ebccf1e3SJoseph Koshy 			error = EINVAL;
2664ebccf1e3SJoseph Koshy 			break;
2665ebccf1e3SJoseph Koshy 		}
2666ebccf1e3SJoseph Koshy 
2667ebccf1e3SJoseph Koshy 		/*
2668ebccf1e3SJoseph Koshy 		 * We can't disable a PMC with a row-index allocated
2669ebccf1e3SJoseph Koshy 		 * for process virtual PMCs.
2670ebccf1e3SJoseph Koshy 		 */
2671ebccf1e3SJoseph Koshy 
2672ebccf1e3SJoseph Koshy 		if (PMC_ROW_DISP_IS_THREAD(ri) &&
2673ebccf1e3SJoseph Koshy 		    request == PMC_STATE_DISABLED) {
2674ebccf1e3SJoseph Koshy 			error = EBUSY;
2675ebccf1e3SJoseph Koshy 			break;
2676ebccf1e3SJoseph Koshy 		}
2677ebccf1e3SJoseph Koshy 
2678ebccf1e3SJoseph Koshy 		/*
2679ebccf1e3SJoseph Koshy 		 * otherwise, this PMC on this CPU is either free or
2680ebccf1e3SJoseph Koshy 		 * in system-wide mode.
2681ebccf1e3SJoseph Koshy 		 */
2682ebccf1e3SJoseph Koshy 
2683ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
2684ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
2685ebccf1e3SJoseph Koshy 
2686ebccf1e3SJoseph Koshy 		pc  = pmc_pcpu[cpu];
2687ebccf1e3SJoseph Koshy 		phw = pc->pc_hwpmcs[ri];
2688ebccf1e3SJoseph Koshy 
2689ebccf1e3SJoseph Koshy 		/*
2690ebccf1e3SJoseph Koshy 		 * XXX do we need some kind of 'forced' disable?
2691ebccf1e3SJoseph Koshy 		 */
2692ebccf1e3SJoseph Koshy 
2693ebccf1e3SJoseph Koshy 		if (phw->phw_pmc == NULL) {
2694ebccf1e3SJoseph Koshy 			if (request == PMC_STATE_DISABLED &&
2695ebccf1e3SJoseph Koshy 			    (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
2696ebccf1e3SJoseph Koshy 				phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
2697ebccf1e3SJoseph Koshy 				PMC_MARK_ROW_STANDALONE(ri);
2698ebccf1e3SJoseph Koshy 			} else if (request == PMC_STATE_FREE &&
2699ebccf1e3SJoseph Koshy 			    (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
2700ebccf1e3SJoseph Koshy 				phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
2701ebccf1e3SJoseph Koshy 				PMC_UNMARK_ROW_STANDALONE(ri);
2702ebccf1e3SJoseph Koshy 			}
2703ebccf1e3SJoseph Koshy 			/* other cases are a no-op */
2704ebccf1e3SJoseph Koshy 		} else
2705ebccf1e3SJoseph Koshy 			error = EBUSY;
2706ebccf1e3SJoseph Koshy 
2707ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
2708ebccf1e3SJoseph Koshy 	}
2709ebccf1e3SJoseph Koshy 	break;
2710ebccf1e3SJoseph Koshy 
2711ebccf1e3SJoseph Koshy 
2712ebccf1e3SJoseph Koshy 	/*
2713ebccf1e3SJoseph Koshy 	 * Allocate a PMC.
2714ebccf1e3SJoseph Koshy 	 */
2715ebccf1e3SJoseph Koshy 
2716ebccf1e3SJoseph Koshy 	case PMC_OP_PMCALLOCATE:
2717ebccf1e3SJoseph Koshy 	{
2718ebccf1e3SJoseph Koshy 		uint32_t caps;
2719ebccf1e3SJoseph Koshy 		u_int cpu;
2720ebccf1e3SJoseph Koshy 		int n;
2721ebccf1e3SJoseph Koshy 		enum pmc_mode mode;
2722ebccf1e3SJoseph Koshy 		struct pmc *pmc;
2723c5153e19SJoseph Koshy 		struct pmc_hw *phw;
2724ebccf1e3SJoseph Koshy 		struct pmc_op_pmcallocate pa;
2725ebccf1e3SJoseph Koshy 		struct pmc_binding pb;
2726ebccf1e3SJoseph Koshy 
2727ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
2728ebccf1e3SJoseph Koshy 			break;
2729ebccf1e3SJoseph Koshy 
2730ebccf1e3SJoseph Koshy 		caps = pa.pm_caps;
2731ebccf1e3SJoseph Koshy 		mode = pa.pm_mode;
2732ebccf1e3SJoseph Koshy 		cpu  = pa.pm_cpu;
2733ebccf1e3SJoseph Koshy 
2734ebccf1e3SJoseph Koshy 		if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
2735ebccf1e3SJoseph Koshy 		     mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
2736ebccf1e3SJoseph Koshy 		    (cpu != (u_int) PMC_CPU_ANY && cpu >= (u_int) mp_ncpus)) {
2737ebccf1e3SJoseph Koshy 			error = EINVAL;
2738ebccf1e3SJoseph Koshy 			break;
2739ebccf1e3SJoseph Koshy 		}
2740ebccf1e3SJoseph Koshy 
2741ebccf1e3SJoseph Koshy 		/*
2742ebccf1e3SJoseph Koshy 		 * Virtual PMCs should only ask for a default CPU.
2743ebccf1e3SJoseph Koshy 		 * System mode PMCs need to specify a non-default CPU.
2744ebccf1e3SJoseph Koshy 		 */
2745ebccf1e3SJoseph Koshy 
2746ebccf1e3SJoseph Koshy 		if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
2747ebccf1e3SJoseph Koshy 		    (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
2748ebccf1e3SJoseph Koshy 			error = EINVAL;
2749ebccf1e3SJoseph Koshy 			break;
2750ebccf1e3SJoseph Koshy 		}
2751ebccf1e3SJoseph Koshy 
2752ebccf1e3SJoseph Koshy 		/*
2753ebccf1e3SJoseph Koshy 		 * Check that a disabled CPU is not being asked for.
2754ebccf1e3SJoseph Koshy 		 */
2755ebccf1e3SJoseph Koshy 
2756ebccf1e3SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode) && pmc_cpu_is_disabled(cpu)) {
2757ebccf1e3SJoseph Koshy 			error = ENXIO;
2758ebccf1e3SJoseph Koshy 			break;
2759ebccf1e3SJoseph Koshy 		}
2760ebccf1e3SJoseph Koshy 
2761ebccf1e3SJoseph Koshy 		/*
2762ebccf1e3SJoseph Koshy 		 * Refuse an allocation for a system-wide PMC if this
2763ebccf1e3SJoseph Koshy 		 * process has been jailed, or if this process lacks
2764ebccf1e3SJoseph Koshy 		 * super-user credentials and the sysctl tunable
2765ebccf1e3SJoseph Koshy 		 * 'security.bsd.unprivileged_syspmcs' is zero.
2766ebccf1e3SJoseph Koshy 		 */
2767ebccf1e3SJoseph Koshy 
2768ebccf1e3SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode)) {
2769ebccf1e3SJoseph Koshy 			if (jailed(curthread->td_ucred))
2770ebccf1e3SJoseph Koshy 				error = EPERM;
2771ebccf1e3SJoseph Koshy 			else if (suser(curthread) &&
2772ebccf1e3SJoseph Koshy 			    (pmc_unprivileged_syspmcs == 0))
2773ebccf1e3SJoseph Koshy 				error = EPERM;
2774ebccf1e3SJoseph Koshy 		}
2775ebccf1e3SJoseph Koshy 
2776ebccf1e3SJoseph Koshy 		if (error)
2777ebccf1e3SJoseph Koshy 			break;
2778ebccf1e3SJoseph Koshy 
2779ebccf1e3SJoseph Koshy 		/*
2780ebccf1e3SJoseph Koshy 		 * Look for valid values for 'pm_flags'
2781ebccf1e3SJoseph Koshy 		 */
2782ebccf1e3SJoseph Koshy 
2783f263522aSJoseph Koshy 		if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
2784f263522aSJoseph Koshy 		    PMC_F_LOG_PROCEXIT)) != 0) {
2785f263522aSJoseph Koshy 			error = EINVAL;
2786f263522aSJoseph Koshy 			break;
2787f263522aSJoseph Koshy 		}
2788f263522aSJoseph Koshy 
2789f263522aSJoseph Koshy 		/* process logging options are not allowed for system PMCs */
2790f263522aSJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
2791f263522aSJoseph Koshy 		    (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
2792ebccf1e3SJoseph Koshy 			error = EINVAL;
2793ebccf1e3SJoseph Koshy 			break;
2794ebccf1e3SJoseph Koshy 		}
2795ebccf1e3SJoseph Koshy 
2796ebccf1e3SJoseph Koshy 		/*
2797ebccf1e3SJoseph Koshy 		 * All sampling mode PMCs need to be able to interrupt the
2798ebccf1e3SJoseph Koshy 		 * CPU.
2799ebccf1e3SJoseph Koshy 		 */
2800f263522aSJoseph Koshy 		if (PMC_IS_SAMPLING_MODE(mode))
2801ebccf1e3SJoseph Koshy 			caps |= PMC_CAP_INTERRUPT;
2802ebccf1e3SJoseph Koshy 
2803744d6797SJoseph Koshy 		/* A valid class specifier should have been passed in. */
2804744d6797SJoseph Koshy 		for (n = 0; n < md->pmd_nclass; n++)
2805744d6797SJoseph Koshy 			if (md->pmd_classes[n].pm_class == pa.pm_class)
2806744d6797SJoseph Koshy 				break;
2807744d6797SJoseph Koshy 		if (n == md->pmd_nclass) {
2808744d6797SJoseph Koshy 			error = EINVAL;
2809744d6797SJoseph Koshy 			break;
2810744d6797SJoseph Koshy 		}
2811744d6797SJoseph Koshy 
2812744d6797SJoseph Koshy 		/* The requested PMC capabilities should be feasible. */
2813744d6797SJoseph Koshy 		if ((md->pmd_classes[n].pm_caps & caps) != caps) {
2814744d6797SJoseph Koshy 			error = EOPNOTSUPP;
2815744d6797SJoseph Koshy 			break;
2816744d6797SJoseph Koshy 		}
2817744d6797SJoseph Koshy 
2818ebccf1e3SJoseph Koshy 		PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
2819ebccf1e3SJoseph Koshy 		    pa.pm_ev, caps, mode, cpu);
2820ebccf1e3SJoseph Koshy 
2821ebccf1e3SJoseph Koshy 		pmc = pmc_allocate_pmc_descriptor();
2822c5153e19SJoseph Koshy 		pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
2823c5153e19SJoseph Koshy 		    PMC_ID_INVALID);
2824ebccf1e3SJoseph Koshy 		pmc->pm_event = pa.pm_ev;
2825ebccf1e3SJoseph Koshy 		pmc->pm_state = PMC_STATE_FREE;
2826ebccf1e3SJoseph Koshy 		pmc->pm_caps  = caps;
2827ebccf1e3SJoseph Koshy 		pmc->pm_flags = pa.pm_flags;
2828ebccf1e3SJoseph Koshy 
2829ebccf1e3SJoseph Koshy 		/* switch thread to CPU 'cpu' */
2830ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
2831ebccf1e3SJoseph Koshy 
2832ebccf1e3SJoseph Koshy #define	PMC_IS_SHAREABLE_PMC(cpu, n)				\
2833ebccf1e3SJoseph Koshy 	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &		\
2834ebccf1e3SJoseph Koshy 	 PMC_PHW_FLAG_IS_SHAREABLE)
2835ebccf1e3SJoseph Koshy #define	PMC_IS_UNALLOCATED(cpu, n)				\
2836ebccf1e3SJoseph Koshy 	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
2837ebccf1e3SJoseph Koshy 
2838ebccf1e3SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode)) {
2839ebccf1e3SJoseph Koshy 			pmc_select_cpu(cpu);
2840ebccf1e3SJoseph Koshy 			for (n = 0; n < (int) md->pmd_npmc; n++)
2841ebccf1e3SJoseph Koshy 				if (pmc_can_allocate_row(n, mode) == 0 &&
2842ebccf1e3SJoseph Koshy 				    pmc_can_allocate_rowindex(
2843c5153e19SJoseph Koshy 					    curthread->td_proc, n, cpu) == 0 &&
2844ebccf1e3SJoseph Koshy 				    (PMC_IS_UNALLOCATED(cpu, n) ||
2845ebccf1e3SJoseph Koshy 				     PMC_IS_SHAREABLE_PMC(cpu, n)) &&
2846ebccf1e3SJoseph Koshy 				    md->pmd_allocate_pmc(cpu, n, pmc,
2847ebccf1e3SJoseph Koshy 					&pa) == 0)
2848ebccf1e3SJoseph Koshy 					break;
2849ebccf1e3SJoseph Koshy 		} else {
2850ebccf1e3SJoseph Koshy 			/* Process virtual mode */
2851ebccf1e3SJoseph Koshy 			for (n = 0; n < (int) md->pmd_npmc; n++) {
2852ebccf1e3SJoseph Koshy 				if (pmc_can_allocate_row(n, mode) == 0 &&
2853ebccf1e3SJoseph Koshy 				    pmc_can_allocate_rowindex(
2854c5153e19SJoseph Koshy 					    curthread->td_proc, n,
2855c5153e19SJoseph Koshy 					    PMC_CPU_ANY) == 0 &&
2856ebccf1e3SJoseph Koshy 				    md->pmd_allocate_pmc(curthread->td_oncpu,
2857ebccf1e3SJoseph Koshy 					n, pmc, &pa) == 0)
2858ebccf1e3SJoseph Koshy 					break;
2859ebccf1e3SJoseph Koshy 			}
2860ebccf1e3SJoseph Koshy 		}
2861ebccf1e3SJoseph Koshy 
2862ebccf1e3SJoseph Koshy #undef	PMC_IS_UNALLOCATED
2863ebccf1e3SJoseph Koshy #undef	PMC_IS_SHAREABLE_PMC
2864ebccf1e3SJoseph Koshy 
2865ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
2866ebccf1e3SJoseph Koshy 
2867ebccf1e3SJoseph Koshy 		if (n == (int) md->pmd_npmc) {
2868ebccf1e3SJoseph Koshy 			pmc_destroy_pmc_descriptor(pmc);
2869ebccf1e3SJoseph Koshy 			FREE(pmc, M_PMC);
2870ebccf1e3SJoseph Koshy 			pmc = NULL;
2871ebccf1e3SJoseph Koshy 			error = EINVAL;
2872ebccf1e3SJoseph Koshy 			break;
2873ebccf1e3SJoseph Koshy 		}
2874ebccf1e3SJoseph Koshy 
2875c5153e19SJoseph Koshy 		/* Fill in the correct value in the ID field */
2876c5153e19SJoseph Koshy 		pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
2877c5153e19SJoseph Koshy 
2878c5153e19SJoseph Koshy 		PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
2879c5153e19SJoseph Koshy 		    pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
2880ebccf1e3SJoseph Koshy 
2881f263522aSJoseph Koshy 		/* Process mode PMCs with logging enabled need log files */
2882f263522aSJoseph Koshy 		if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
2883f263522aSJoseph Koshy 			pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
2884f263522aSJoseph Koshy 
2885f263522aSJoseph Koshy 		/* All system mode sampling PMCs require a log file */
2886f263522aSJoseph Koshy 		if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
2887f263522aSJoseph Koshy 			pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
2888f263522aSJoseph Koshy 
2889ebccf1e3SJoseph Koshy 		/*
2890ebccf1e3SJoseph Koshy 		 * Configure global pmc's immediately
2891ebccf1e3SJoseph Koshy 		 */
2892ebccf1e3SJoseph Koshy 
2893c5153e19SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
2894c5153e19SJoseph Koshy 
2895c5153e19SJoseph Koshy 			pmc_save_cpu_binding(&pb);
2896c5153e19SJoseph Koshy 			pmc_select_cpu(cpu);
2897c5153e19SJoseph Koshy 
2898c5153e19SJoseph Koshy 			phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
2899c5153e19SJoseph Koshy 
2900c5153e19SJoseph Koshy 			if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
2901c5153e19SJoseph Koshy 			    (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
2902ebccf1e3SJoseph Koshy 				(void) md->pmd_release_pmc(cpu, n, pmc);
2903ebccf1e3SJoseph Koshy 				pmc_destroy_pmc_descriptor(pmc);
2904ebccf1e3SJoseph Koshy 				FREE(pmc, M_PMC);
2905ebccf1e3SJoseph Koshy 				pmc = NULL;
2906c5153e19SJoseph Koshy 				pmc_restore_cpu_binding(&pb);
2907c5153e19SJoseph Koshy 				error = EPERM;
2908ebccf1e3SJoseph Koshy 				break;
2909ebccf1e3SJoseph Koshy 			}
2910ebccf1e3SJoseph Koshy 
2911c5153e19SJoseph Koshy 			pmc_restore_cpu_binding(&pb);
2912c5153e19SJoseph Koshy 		}
2913ebccf1e3SJoseph Koshy 
2914ebccf1e3SJoseph Koshy 		pmc->pm_state    = PMC_STATE_ALLOCATED;
2915ebccf1e3SJoseph Koshy 
2916ebccf1e3SJoseph Koshy 		/*
2917ebccf1e3SJoseph Koshy 		 * mark row disposition
2918ebccf1e3SJoseph Koshy 		 */
2919ebccf1e3SJoseph Koshy 
2920ebccf1e3SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode))
2921ebccf1e3SJoseph Koshy 			PMC_MARK_ROW_STANDALONE(n);
2922ebccf1e3SJoseph Koshy 		else
2923ebccf1e3SJoseph Koshy 			PMC_MARK_ROW_THREAD(n);
2924ebccf1e3SJoseph Koshy 
2925ebccf1e3SJoseph Koshy 		/*
2926ebccf1e3SJoseph Koshy 		 * Register this PMC with the current thread as its owner.
2927ebccf1e3SJoseph Koshy 		 */
2928ebccf1e3SJoseph Koshy 
2929ebccf1e3SJoseph Koshy 		if ((error =
2930ebccf1e3SJoseph Koshy 		    pmc_register_owner(curthread->td_proc, pmc)) != 0) {
2931ebccf1e3SJoseph Koshy 			pmc_release_pmc_descriptor(pmc);
2932ebccf1e3SJoseph Koshy 			FREE(pmc, M_PMC);
2933ebccf1e3SJoseph Koshy 			pmc = NULL;
2934ebccf1e3SJoseph Koshy 			break;
2935ebccf1e3SJoseph Koshy 		}
2936ebccf1e3SJoseph Koshy 
2937ebccf1e3SJoseph Koshy 		/*
2938ebccf1e3SJoseph Koshy 		 * Return the allocated index.
2939ebccf1e3SJoseph Koshy 		 */
2940ebccf1e3SJoseph Koshy 
2941c5153e19SJoseph Koshy 		pa.pm_pmcid = pmc->pm_id;
2942ebccf1e3SJoseph Koshy 
2943ebccf1e3SJoseph Koshy 		error = copyout(&pa, arg, sizeof(pa));
2944ebccf1e3SJoseph Koshy 	}
2945ebccf1e3SJoseph Koshy 	break;
2946ebccf1e3SJoseph Koshy 
2947ebccf1e3SJoseph Koshy 
2948ebccf1e3SJoseph Koshy 	/*
2949ebccf1e3SJoseph Koshy 	 * Attach a PMC to a process.
2950ebccf1e3SJoseph Koshy 	 */
2951ebccf1e3SJoseph Koshy 
2952ebccf1e3SJoseph Koshy 	case PMC_OP_PMCATTACH:
2953ebccf1e3SJoseph Koshy 	{
2954ebccf1e3SJoseph Koshy 		struct pmc *pm;
2955ebccf1e3SJoseph Koshy 		struct proc *p;
2956ebccf1e3SJoseph Koshy 		struct pmc_op_pmcattach a;
2957ebccf1e3SJoseph Koshy 
2958ebccf1e3SJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
2959ebccf1e3SJoseph Koshy 
2960ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &a, sizeof(a))) != 0)
2961ebccf1e3SJoseph Koshy 			break;
2962ebccf1e3SJoseph Koshy 
2963ebccf1e3SJoseph Koshy 		if (a.pm_pid < 0) {
2964ebccf1e3SJoseph Koshy 			error = EINVAL;
2965ebccf1e3SJoseph Koshy 			break;
2966ebccf1e3SJoseph Koshy 		} else if (a.pm_pid == 0)
2967ebccf1e3SJoseph Koshy 			a.pm_pid = td->td_proc->p_pid;
2968ebccf1e3SJoseph Koshy 
2969ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
2970ebccf1e3SJoseph Koshy 			break;
2971ebccf1e3SJoseph Koshy 
2972c5153e19SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
2973ebccf1e3SJoseph Koshy 			error = EINVAL;
2974ebccf1e3SJoseph Koshy 			break;
2975ebccf1e3SJoseph Koshy 		}
2976ebccf1e3SJoseph Koshy 
2977ebccf1e3SJoseph Koshy 		/* PMCs may be (re)attached only when allocated or stopped */
2978ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING) {
2979ebccf1e3SJoseph Koshy 			error = EBUSY;
2980ebccf1e3SJoseph Koshy 			break;
2981ebccf1e3SJoseph Koshy 		} else if (pm->pm_state != PMC_STATE_ALLOCATED &&
2982ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_STOPPED) {
2983ebccf1e3SJoseph Koshy 			error = EINVAL;
2984ebccf1e3SJoseph Koshy 			break;
2985ebccf1e3SJoseph Koshy 		}
2986ebccf1e3SJoseph Koshy 
2987ebccf1e3SJoseph Koshy 		/* lookup pid */
2988ebccf1e3SJoseph Koshy 		if ((p = pfind(a.pm_pid)) == NULL) {
2989ebccf1e3SJoseph Koshy 			error = ESRCH;
2990ebccf1e3SJoseph Koshy 			break;
2991ebccf1e3SJoseph Koshy 		}
2992ebccf1e3SJoseph Koshy 
2993ebccf1e3SJoseph Koshy 		/*
2994ebccf1e3SJoseph Koshy 		 * Ignore processes that are working on exiting.
2995ebccf1e3SJoseph Koshy 		 */
2996ebccf1e3SJoseph Koshy 		if (p->p_flag & P_WEXIT) {
2997ebccf1e3SJoseph Koshy 			error = ESRCH;
2998ebccf1e3SJoseph Koshy 			PROC_UNLOCK(p);	/* pfind() returns a locked process */
2999ebccf1e3SJoseph Koshy 			break;
3000ebccf1e3SJoseph Koshy 		}
3001ebccf1e3SJoseph Koshy 
3002ebccf1e3SJoseph Koshy 		/*
3003ebccf1e3SJoseph Koshy 		 * we are allowed to attach a PMC to a process if
3004ebccf1e3SJoseph Koshy 		 * we can debug it.
3005ebccf1e3SJoseph Koshy 		 */
3006ebccf1e3SJoseph Koshy 		error = p_candebug(curthread, p);
3007ebccf1e3SJoseph Koshy 
3008ebccf1e3SJoseph Koshy 		PROC_UNLOCK(p);
3009ebccf1e3SJoseph Koshy 
3010ebccf1e3SJoseph Koshy 		if (error == 0)
3011ebccf1e3SJoseph Koshy 			error = pmc_attach_process(p, pm);
3012ebccf1e3SJoseph Koshy 	}
3013ebccf1e3SJoseph Koshy 	break;
3014ebccf1e3SJoseph Koshy 
3015ebccf1e3SJoseph Koshy 
3016ebccf1e3SJoseph Koshy 	/*
3017ebccf1e3SJoseph Koshy 	 * Detach an attached PMC from a process.
3018ebccf1e3SJoseph Koshy 	 */
3019ebccf1e3SJoseph Koshy 
3020ebccf1e3SJoseph Koshy 	case PMC_OP_PMCDETACH:
3021ebccf1e3SJoseph Koshy 	{
3022ebccf1e3SJoseph Koshy 		struct pmc *pm;
3023ebccf1e3SJoseph Koshy 		struct proc *p;
3024ebccf1e3SJoseph Koshy 		struct pmc_op_pmcattach a;
3025ebccf1e3SJoseph Koshy 
3026ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &a, sizeof(a))) != 0)
3027ebccf1e3SJoseph Koshy 			break;
3028ebccf1e3SJoseph Koshy 
3029ebccf1e3SJoseph Koshy 		if (a.pm_pid < 0) {
3030ebccf1e3SJoseph Koshy 			error = EINVAL;
3031ebccf1e3SJoseph Koshy 			break;
3032ebccf1e3SJoseph Koshy 		} else if (a.pm_pid == 0)
3033ebccf1e3SJoseph Koshy 			a.pm_pid = td->td_proc->p_pid;
3034ebccf1e3SJoseph Koshy 
3035ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3036ebccf1e3SJoseph Koshy 			break;
3037ebccf1e3SJoseph Koshy 
3038ebccf1e3SJoseph Koshy 		if ((p = pfind(a.pm_pid)) == NULL) {
3039ebccf1e3SJoseph Koshy 			error = ESRCH;
3040ebccf1e3SJoseph Koshy 			break;
3041ebccf1e3SJoseph Koshy 		}
3042ebccf1e3SJoseph Koshy 
3043ebccf1e3SJoseph Koshy 		/*
3044ebccf1e3SJoseph Koshy 		 * Treat processes that are in the process of exiting
3045ebccf1e3SJoseph Koshy 		 * as if they were not present.
3046ebccf1e3SJoseph Koshy 		 */
3047ebccf1e3SJoseph Koshy 
3048ebccf1e3SJoseph Koshy 		if (p->p_flag & P_WEXIT)
3049ebccf1e3SJoseph Koshy 			error = ESRCH;
3050ebccf1e3SJoseph Koshy 
3051ebccf1e3SJoseph Koshy 		PROC_UNLOCK(p);	/* pfind() returns a locked process */
3052ebccf1e3SJoseph Koshy 
3053ebccf1e3SJoseph Koshy 		if (error == 0)
3054ebccf1e3SJoseph Koshy 			error = pmc_detach_process(p, pm);
3055ebccf1e3SJoseph Koshy 	}
3056ebccf1e3SJoseph Koshy 	break;
3057ebccf1e3SJoseph Koshy 
3058ebccf1e3SJoseph Koshy 
3059ebccf1e3SJoseph Koshy 	/*
3060f263522aSJoseph Koshy 	 * Retrieve the MSR number associated with the counter
3061f263522aSJoseph Koshy 	 * 'pmc_id'.  This allows processes to directly use RDPMC
3062f263522aSJoseph Koshy 	 * instructions to read their PMCs, without the overhead of a
3063f263522aSJoseph Koshy 	 * system call.
3064f263522aSJoseph Koshy 	 */
3065f263522aSJoseph Koshy 
3066f263522aSJoseph Koshy 	case PMC_OP_PMCGETMSR:
3067f263522aSJoseph Koshy 	{
3068f263522aSJoseph Koshy 		int ri;
3069f263522aSJoseph Koshy 		struct pmc	*pm;
3070f263522aSJoseph Koshy 		struct pmc_target *pt;
3071f263522aSJoseph Koshy 		struct pmc_op_getmsr gm;
3072f263522aSJoseph Koshy 
3073f263522aSJoseph Koshy 		PMC_DOWNGRADE_SX();
3074f263522aSJoseph Koshy 
3075f263522aSJoseph Koshy 		/* CPU has no 'GETMSR' support */
3076f263522aSJoseph Koshy 		if (md->pmd_get_msr == NULL) {
3077f263522aSJoseph Koshy 			error = ENOSYS;
3078f263522aSJoseph Koshy 			break;
3079f263522aSJoseph Koshy 		}
3080f263522aSJoseph Koshy 
3081f263522aSJoseph Koshy 		if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3082f263522aSJoseph Koshy 			break;
3083f263522aSJoseph Koshy 
3084f263522aSJoseph Koshy 		if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3085f263522aSJoseph Koshy 			break;
3086f263522aSJoseph Koshy 
3087f263522aSJoseph Koshy 		/*
3088f263522aSJoseph Koshy 		 * The allocated PMC has to be a process virtual PMC,
3089f263522aSJoseph Koshy 		 * i.e., of type MODE_T[CS].  Global PMCs can only be
3090f263522aSJoseph Koshy 		 * read using the PMCREAD operation since they may be
3091f263522aSJoseph Koshy 		 * allocated on a different CPU than the one we could
3092f263522aSJoseph Koshy 		 * be running on at the time of the RDPMC instruction.
3093f263522aSJoseph Koshy 		 *
3094f263522aSJoseph Koshy 		 * The GETMSR operation is not allowed for PMCs that
3095f263522aSJoseph Koshy 		 * are inherited across processes.
3096f263522aSJoseph Koshy 		 */
3097f263522aSJoseph Koshy 
3098f263522aSJoseph Koshy 		if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
3099f263522aSJoseph Koshy 		    (pm->pm_flags & PMC_F_DESCENDANTS)) {
3100f263522aSJoseph Koshy 			error = EINVAL;
3101f263522aSJoseph Koshy 			break;
3102f263522aSJoseph Koshy 		}
3103f263522aSJoseph Koshy 
3104f263522aSJoseph Koshy 		/*
3105f263522aSJoseph Koshy 		 * It only makes sense to use a RDPMC (or its
3106f263522aSJoseph Koshy 		 * equivalent instruction on non-x86 architectures) on
3107f263522aSJoseph Koshy 		 * a process that has allocated and attached a PMC to
3108f263522aSJoseph Koshy 		 * itself.  Conversely the PMC is only allowed to have
3109f263522aSJoseph Koshy 		 * one process attached to it -- its owner.
3110f263522aSJoseph Koshy 		 */
3111f263522aSJoseph Koshy 
3112f263522aSJoseph Koshy 		if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3113f263522aSJoseph Koshy 		    LIST_NEXT(pt, pt_next) != NULL ||
3114f263522aSJoseph Koshy 		    pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3115f263522aSJoseph Koshy 			error = EINVAL;
3116f263522aSJoseph Koshy 			break;
3117f263522aSJoseph Koshy 		}
3118f263522aSJoseph Koshy 
3119f263522aSJoseph Koshy 		ri = PMC_TO_ROWINDEX(pm);
3120f263522aSJoseph Koshy 
3121f263522aSJoseph Koshy 		if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0)
3122f263522aSJoseph Koshy 			break;
3123f263522aSJoseph Koshy 
3124f263522aSJoseph Koshy 		if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3125f263522aSJoseph Koshy 			break;
3126f263522aSJoseph Koshy 
3127f263522aSJoseph Koshy 		/*
3128f263522aSJoseph Koshy 		 * Mark our process as using MSRs.  Update machine
3129f263522aSJoseph Koshy 		 * state using a forced context switch.
3130f263522aSJoseph Koshy 		 */
3131f263522aSJoseph Koshy 
3132f263522aSJoseph Koshy 		pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
3133f263522aSJoseph Koshy 		pmc_force_context_switch();
3134f263522aSJoseph Koshy 
3135f263522aSJoseph Koshy 	}
3136f263522aSJoseph Koshy 	break;
3137f263522aSJoseph Koshy 
3138f263522aSJoseph Koshy 	/*
3139ebccf1e3SJoseph Koshy 	 * Release an allocated PMC
3140ebccf1e3SJoseph Koshy 	 */
3141ebccf1e3SJoseph Koshy 
3142ebccf1e3SJoseph Koshy 	case PMC_OP_PMCRELEASE:
3143ebccf1e3SJoseph Koshy 	{
3144ebccf1e3SJoseph Koshy 		pmc_id_t pmcid;
3145ebccf1e3SJoseph Koshy 		struct pmc *pm;
3146ebccf1e3SJoseph Koshy 		struct pmc_owner *po;
3147ebccf1e3SJoseph Koshy 		struct pmc_op_simple sp;
3148ebccf1e3SJoseph Koshy 
3149ebccf1e3SJoseph Koshy 		/*
3150ebccf1e3SJoseph Koshy 		 * Find PMC pointer for the named PMC.
3151ebccf1e3SJoseph Koshy 		 *
3152ebccf1e3SJoseph Koshy 		 * Use pmc_release_pmc_descriptor() to switch off the
3153ebccf1e3SJoseph Koshy 		 * PMC, remove all its target threads, and remove the
3154ebccf1e3SJoseph Koshy 		 * PMC from its owner's list.
3155ebccf1e3SJoseph Koshy 		 *
3156ebccf1e3SJoseph Koshy 		 * Remove the owner record if this is the last PMC
3157ebccf1e3SJoseph Koshy 		 * owned.
3158ebccf1e3SJoseph Koshy 		 *
3159ebccf1e3SJoseph Koshy 		 * Free up space.
3160ebccf1e3SJoseph Koshy 		 */
3161ebccf1e3SJoseph Koshy 
3162ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3163ebccf1e3SJoseph Koshy 			break;
3164ebccf1e3SJoseph Koshy 
3165ebccf1e3SJoseph Koshy 		pmcid = sp.pm_pmcid;
3166ebccf1e3SJoseph Koshy 
3167ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3168ebccf1e3SJoseph Koshy 			break;
3169ebccf1e3SJoseph Koshy 
3170ebccf1e3SJoseph Koshy 		po = pm->pm_owner;
3171ebccf1e3SJoseph Koshy 		pmc_release_pmc_descriptor(pm);
3172ebccf1e3SJoseph Koshy 		pmc_maybe_remove_owner(po);
3173ebccf1e3SJoseph Koshy 
3174ebccf1e3SJoseph Koshy 		FREE(pm, M_PMC);
3175ebccf1e3SJoseph Koshy 	}
3176ebccf1e3SJoseph Koshy 	break;
3177ebccf1e3SJoseph Koshy 
3178ebccf1e3SJoseph Koshy 
3179ebccf1e3SJoseph Koshy 	/*
3180ebccf1e3SJoseph Koshy 	 * Read and/or write a PMC.
3181ebccf1e3SJoseph Koshy 	 */
3182ebccf1e3SJoseph Koshy 
3183ebccf1e3SJoseph Koshy 	case PMC_OP_PMCRW:
3184ebccf1e3SJoseph Koshy 	{
3185ebccf1e3SJoseph Koshy 		uint32_t cpu, ri;
3186ebccf1e3SJoseph Koshy 		struct pmc *pm;
3187ebccf1e3SJoseph Koshy 		struct pmc_op_pmcrw *pprw;
3188ebccf1e3SJoseph Koshy 		struct pmc_op_pmcrw prw;
3189ebccf1e3SJoseph Koshy 		struct pmc_binding pb;
3190ebccf1e3SJoseph Koshy 		pmc_value_t oldvalue;
3191ebccf1e3SJoseph Koshy 
3192ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
3193ebccf1e3SJoseph Koshy 
3194ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
3195ebccf1e3SJoseph Koshy 			break;
3196ebccf1e3SJoseph Koshy 
3197869f4a8bSWarner Losh 		ri = 0;
3198ebccf1e3SJoseph Koshy 		PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
3199ebccf1e3SJoseph Koshy 		    prw.pm_flags);
3200ebccf1e3SJoseph Koshy 
3201ebccf1e3SJoseph Koshy 		/* must have at least one flag set */
3202ebccf1e3SJoseph Koshy 		if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
3203ebccf1e3SJoseph Koshy 			error = EINVAL;
3204ebccf1e3SJoseph Koshy 			break;
3205ebccf1e3SJoseph Koshy 		}
3206ebccf1e3SJoseph Koshy 
3207ebccf1e3SJoseph Koshy 		/* locate pmc descriptor */
3208ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
3209ebccf1e3SJoseph Koshy 			break;
3210ebccf1e3SJoseph Koshy 
3211ebccf1e3SJoseph Koshy 		/* Can't read a PMC that hasn't been started. */
3212ebccf1e3SJoseph Koshy 		if (pm->pm_state != PMC_STATE_ALLOCATED &&
3213ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_STOPPED &&
3214ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_RUNNING) {
3215ebccf1e3SJoseph Koshy 			error = EINVAL;
3216ebccf1e3SJoseph Koshy 			break;
3217ebccf1e3SJoseph Koshy 		}
3218ebccf1e3SJoseph Koshy 
3219ebccf1e3SJoseph Koshy 		/* writing a new value is allowed only for 'STOPPED' pmcs */
3220ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING &&
3221ebccf1e3SJoseph Koshy 		    (prw.pm_flags & PMC_F_NEWVALUE)) {
3222ebccf1e3SJoseph Koshy 			error = EBUSY;
3223ebccf1e3SJoseph Koshy 			break;
3224ebccf1e3SJoseph Koshy 		}
3225ebccf1e3SJoseph Koshy 
3226c5153e19SJoseph Koshy 		if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
3227ebccf1e3SJoseph Koshy 
3228c5153e19SJoseph Koshy 			/*
3229c5153e19SJoseph Koshy 			 * If this PMC is attached to its owner (i.e.,
3230c5153e19SJoseph Koshy 			 * the process requesting this operation) and
3231c5153e19SJoseph Koshy 			 * is running, then attempt to get an
3232c5153e19SJoseph Koshy 			 * upto-date reading from hardware for a READ.
3233c5153e19SJoseph Koshy 			 * Writes are only allowed when the PMC is
3234c5153e19SJoseph Koshy 			 * stopped, so only update the saved value
3235c5153e19SJoseph Koshy 			 * field.
3236c5153e19SJoseph Koshy 			 *
3237c5153e19SJoseph Koshy 			 * If the PMC is not running, or is not
3238c5153e19SJoseph Koshy 			 * attached to its owner, read/write to the
3239c5153e19SJoseph Koshy 			 * savedvalue field.
3240c5153e19SJoseph Koshy 			 */
3241c5153e19SJoseph Koshy 
3242c5153e19SJoseph Koshy 			ri = PMC_TO_ROWINDEX(pm);
3243c5153e19SJoseph Koshy 
3244ebccf1e3SJoseph Koshy 			mtx_pool_lock_spin(pmc_mtxpool, pm);
3245c5153e19SJoseph Koshy 			cpu = curthread->td_oncpu;
3246c5153e19SJoseph Koshy 
3247c5153e19SJoseph Koshy 			if (prw.pm_flags & PMC_F_OLDVALUE) {
3248c5153e19SJoseph Koshy 				if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3249c5153e19SJoseph Koshy 				    (pm->pm_state == PMC_STATE_RUNNING))
3250c5153e19SJoseph Koshy 					error = (*md->pmd_read_pmc)(cpu, ri,
3251c5153e19SJoseph Koshy 					    &oldvalue);
3252c5153e19SJoseph Koshy 				else
3253ebccf1e3SJoseph Koshy 					oldvalue = pm->pm_gv.pm_savedvalue;
3254c5153e19SJoseph Koshy 			}
3255ebccf1e3SJoseph Koshy 			if (prw.pm_flags & PMC_F_NEWVALUE)
3256ebccf1e3SJoseph Koshy 				pm->pm_gv.pm_savedvalue = prw.pm_value;
3257c5153e19SJoseph Koshy 
3258ebccf1e3SJoseph Koshy 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
3259ebccf1e3SJoseph Koshy 
3260ebccf1e3SJoseph Koshy 		} else { /* System mode PMCs */
3261c5153e19SJoseph Koshy 			cpu = PMC_TO_CPU(pm);
3262c5153e19SJoseph Koshy 			ri  = PMC_TO_ROWINDEX(pm);
3263ebccf1e3SJoseph Koshy 
3264ebccf1e3SJoseph Koshy 			if (pmc_cpu_is_disabled(cpu)) {
3265ebccf1e3SJoseph Koshy 				error = ENXIO;
3266ebccf1e3SJoseph Koshy 				break;
3267ebccf1e3SJoseph Koshy 			}
3268ebccf1e3SJoseph Koshy 
3269ebccf1e3SJoseph Koshy 			/* move this thread to CPU 'cpu' */
3270ebccf1e3SJoseph Koshy 			pmc_save_cpu_binding(&pb);
3271ebccf1e3SJoseph Koshy 			pmc_select_cpu(cpu);
3272ebccf1e3SJoseph Koshy 
3273c5153e19SJoseph Koshy 			critical_enter();
3274ebccf1e3SJoseph Koshy 			/* save old value */
3275ebccf1e3SJoseph Koshy 			if (prw.pm_flags & PMC_F_OLDVALUE)
3276ebccf1e3SJoseph Koshy 				if ((error = (*md->pmd_read_pmc)(cpu, ri,
3277ebccf1e3SJoseph Koshy 					 &oldvalue)))
3278ebccf1e3SJoseph Koshy 					goto error;
3279ebccf1e3SJoseph Koshy 			/* write out new value */
3280ebccf1e3SJoseph Koshy 			if (prw.pm_flags & PMC_F_NEWVALUE)
3281ebccf1e3SJoseph Koshy 				error = (*md->pmd_write_pmc)(cpu, ri,
3282ebccf1e3SJoseph Koshy 				    prw.pm_value);
3283ebccf1e3SJoseph Koshy 		error:
3284c5153e19SJoseph Koshy 			critical_exit();
3285ebccf1e3SJoseph Koshy 			pmc_restore_cpu_binding(&pb);
3286ebccf1e3SJoseph Koshy 			if (error)
3287ebccf1e3SJoseph Koshy 				break;
3288ebccf1e3SJoseph Koshy 		}
3289ebccf1e3SJoseph Koshy 
3290ebccf1e3SJoseph Koshy 		pprw = (struct pmc_op_pmcrw *) arg;
3291ebccf1e3SJoseph Koshy 
3292342ed5d9SRuslan Ermilov #ifdef	DEBUG
3293ebccf1e3SJoseph Koshy 		if (prw.pm_flags & PMC_F_NEWVALUE)
3294ebccf1e3SJoseph Koshy 			PMCDBG(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
3295ebccf1e3SJoseph Koshy 			    ri, prw.pm_value, oldvalue);
3296ebccf1e3SJoseph Koshy 		else
3297ebccf1e3SJoseph Koshy 			PMCDBG(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
3298ebccf1e3SJoseph Koshy #endif
3299ebccf1e3SJoseph Koshy 
3300ebccf1e3SJoseph Koshy 		/* return old value if requested */
3301ebccf1e3SJoseph Koshy 		if (prw.pm_flags & PMC_F_OLDVALUE)
3302ebccf1e3SJoseph Koshy 			if ((error = copyout(&oldvalue, &pprw->pm_value,
3303ebccf1e3SJoseph Koshy 				 sizeof(prw.pm_value))))
3304ebccf1e3SJoseph Koshy 				break;
3305ebccf1e3SJoseph Koshy 
3306ebccf1e3SJoseph Koshy 	}
3307ebccf1e3SJoseph Koshy 	break;
3308ebccf1e3SJoseph Koshy 
3309ebccf1e3SJoseph Koshy 
3310ebccf1e3SJoseph Koshy 	/*
3311ebccf1e3SJoseph Koshy 	 * Set the sampling rate for a sampling mode PMC and the
3312ebccf1e3SJoseph Koshy 	 * initial count for a counting mode PMC.
3313ebccf1e3SJoseph Koshy 	 */
3314ebccf1e3SJoseph Koshy 
3315ebccf1e3SJoseph Koshy 	case PMC_OP_PMCSETCOUNT:
3316ebccf1e3SJoseph Koshy 	{
3317ebccf1e3SJoseph Koshy 		struct pmc *pm;
3318ebccf1e3SJoseph Koshy 		struct pmc_op_pmcsetcount sc;
3319ebccf1e3SJoseph Koshy 
3320ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
3321ebccf1e3SJoseph Koshy 
3322ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
3323ebccf1e3SJoseph Koshy 			break;
3324ebccf1e3SJoseph Koshy 
3325ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
3326ebccf1e3SJoseph Koshy 			break;
3327ebccf1e3SJoseph Koshy 
3328ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING) {
3329ebccf1e3SJoseph Koshy 			error = EBUSY;
3330ebccf1e3SJoseph Koshy 			break;
3331ebccf1e3SJoseph Koshy 		}
3332ebccf1e3SJoseph Koshy 
3333c5153e19SJoseph Koshy 		if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3334ebccf1e3SJoseph Koshy 			pm->pm_sc.pm_reloadcount = sc.pm_count;
3335ebccf1e3SJoseph Koshy 		else
3336ebccf1e3SJoseph Koshy 			pm->pm_sc.pm_initial = sc.pm_count;
3337ebccf1e3SJoseph Koshy 	}
3338ebccf1e3SJoseph Koshy 	break;
3339ebccf1e3SJoseph Koshy 
3340ebccf1e3SJoseph Koshy 
3341ebccf1e3SJoseph Koshy 	/*
3342ebccf1e3SJoseph Koshy 	 * Start a PMC.
3343ebccf1e3SJoseph Koshy 	 */
3344ebccf1e3SJoseph Koshy 
3345ebccf1e3SJoseph Koshy 	case PMC_OP_PMCSTART:
3346ebccf1e3SJoseph Koshy 	{
3347ebccf1e3SJoseph Koshy 		pmc_id_t pmcid;
3348ebccf1e3SJoseph Koshy 		struct pmc *pm;
3349ebccf1e3SJoseph Koshy 		struct pmc_op_simple sp;
3350ebccf1e3SJoseph Koshy 
3351ebccf1e3SJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
3352ebccf1e3SJoseph Koshy 
3353ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3354ebccf1e3SJoseph Koshy 			break;
3355ebccf1e3SJoseph Koshy 
3356ebccf1e3SJoseph Koshy 		pmcid = sp.pm_pmcid;
3357ebccf1e3SJoseph Koshy 
3358ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3359ebccf1e3SJoseph Koshy 			break;
3360ebccf1e3SJoseph Koshy 
3361c5153e19SJoseph Koshy 		KASSERT(pmcid == pm->pm_id,
3362c5153e19SJoseph Koshy 		    ("[pmc,%d] pmcid %x != id %x", __LINE__,
3363c5153e19SJoseph Koshy 			pm->pm_id, pmcid));
3364ebccf1e3SJoseph Koshy 
3365ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
3366ebccf1e3SJoseph Koshy 			break;
3367ebccf1e3SJoseph Koshy 		else if (pm->pm_state != PMC_STATE_STOPPED &&
3368ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_ALLOCATED) {
3369ebccf1e3SJoseph Koshy 			error = EINVAL;
3370ebccf1e3SJoseph Koshy 			break;
3371ebccf1e3SJoseph Koshy 		}
3372ebccf1e3SJoseph Koshy 
3373ebccf1e3SJoseph Koshy 		error = pmc_start(pm);
3374ebccf1e3SJoseph Koshy 	}
3375ebccf1e3SJoseph Koshy 	break;
3376ebccf1e3SJoseph Koshy 
3377ebccf1e3SJoseph Koshy 
3378ebccf1e3SJoseph Koshy 	/*
3379ebccf1e3SJoseph Koshy 	 * Stop a PMC.
3380ebccf1e3SJoseph Koshy 	 */
3381ebccf1e3SJoseph Koshy 
3382ebccf1e3SJoseph Koshy 	case PMC_OP_PMCSTOP:
3383ebccf1e3SJoseph Koshy 	{
3384ebccf1e3SJoseph Koshy 		pmc_id_t pmcid;
3385ebccf1e3SJoseph Koshy 		struct pmc *pm;
3386ebccf1e3SJoseph Koshy 		struct pmc_op_simple sp;
3387ebccf1e3SJoseph Koshy 
3388ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
3389ebccf1e3SJoseph Koshy 
3390ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3391ebccf1e3SJoseph Koshy 			break;
3392ebccf1e3SJoseph Koshy 
3393ebccf1e3SJoseph Koshy 		pmcid = sp.pm_pmcid;
3394ebccf1e3SJoseph Koshy 
3395ebccf1e3SJoseph Koshy 		/*
3396ebccf1e3SJoseph Koshy 		 * Mark the PMC as inactive and invoke the MD stop
3397ebccf1e3SJoseph Koshy 		 * routines if needed.
3398ebccf1e3SJoseph Koshy 		 */
3399ebccf1e3SJoseph Koshy 
3400ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3401ebccf1e3SJoseph Koshy 			break;
3402ebccf1e3SJoseph Koshy 
3403c5153e19SJoseph Koshy 		KASSERT(pmcid == pm->pm_id,
3404c5153e19SJoseph Koshy 		    ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
3405c5153e19SJoseph Koshy 			pm->pm_id, pmcid));
3406ebccf1e3SJoseph Koshy 
3407ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
3408ebccf1e3SJoseph Koshy 			break;
3409ebccf1e3SJoseph Koshy 		else if (pm->pm_state != PMC_STATE_RUNNING) {
3410ebccf1e3SJoseph Koshy 			error = EINVAL;
3411ebccf1e3SJoseph Koshy 			break;
3412ebccf1e3SJoseph Koshy 		}
3413ebccf1e3SJoseph Koshy 
3414ebccf1e3SJoseph Koshy 		error = pmc_stop(pm);
3415ebccf1e3SJoseph Koshy 	}
3416ebccf1e3SJoseph Koshy 	break;
3417ebccf1e3SJoseph Koshy 
3418ebccf1e3SJoseph Koshy 
3419ebccf1e3SJoseph Koshy 	/*
3420fbf1556dSJoseph Koshy 	 * Write a user supplied value to the log file.
3421ebccf1e3SJoseph Koshy 	 */
3422ebccf1e3SJoseph Koshy 
3423ebccf1e3SJoseph Koshy 	case PMC_OP_WRITELOG:
3424ebccf1e3SJoseph Koshy 	{
3425f263522aSJoseph Koshy 		struct pmc_op_writelog wl;
3426f263522aSJoseph Koshy 		struct pmc_owner *po;
3427ebccf1e3SJoseph Koshy 
3428ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
3429ebccf1e3SJoseph Koshy 
3430f263522aSJoseph Koshy 		if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
3431ebccf1e3SJoseph Koshy 			break;
3432ebccf1e3SJoseph Koshy 
3433f263522aSJoseph Koshy 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3434ebccf1e3SJoseph Koshy 			error = EINVAL;
3435ebccf1e3SJoseph Koshy 			break;
3436ebccf1e3SJoseph Koshy 		}
3437ebccf1e3SJoseph Koshy 
3438f263522aSJoseph Koshy 		if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
3439c5153e19SJoseph Koshy 			error = EINVAL;
3440c5153e19SJoseph Koshy 			break;
3441c5153e19SJoseph Koshy 		}
3442c5153e19SJoseph Koshy 
3443f263522aSJoseph Koshy 		error = pmclog_process_userlog(po, &wl);
3444ebccf1e3SJoseph Koshy 	}
3445ebccf1e3SJoseph Koshy 	break;
3446f263522aSJoseph Koshy 
3447ebccf1e3SJoseph Koshy 
3448ebccf1e3SJoseph Koshy 	default:
3449ebccf1e3SJoseph Koshy 		error = EINVAL;
3450ebccf1e3SJoseph Koshy 		break;
3451ebccf1e3SJoseph Koshy 	}
3452ebccf1e3SJoseph Koshy 
3453ebccf1e3SJoseph Koshy 	if (is_sx_downgraded)
3454ebccf1e3SJoseph Koshy 		sx_sunlock(&pmc_sx);
3455ebccf1e3SJoseph Koshy 	else
3456ebccf1e3SJoseph Koshy 		sx_xunlock(&pmc_sx);
3457ebccf1e3SJoseph Koshy 
3458ebccf1e3SJoseph Koshy 	if (error)
3459ebccf1e3SJoseph Koshy 		atomic_add_int(&pmc_stats.pm_syscall_errors, 1);
3460ebccf1e3SJoseph Koshy 
3461f263522aSJoseph Koshy 	PICKUP_GIANT();
3462f263522aSJoseph Koshy 
3463ebccf1e3SJoseph Koshy 	return error;
3464ebccf1e3SJoseph Koshy }
3465ebccf1e3SJoseph Koshy 
3466ebccf1e3SJoseph Koshy /*
3467ebccf1e3SJoseph Koshy  * Helper functions
3468ebccf1e3SJoseph Koshy  */
3469ebccf1e3SJoseph Koshy 
3470f263522aSJoseph Koshy 
3471ebccf1e3SJoseph Koshy /*
3472f263522aSJoseph Koshy  * Interrupt processing.
3473f263522aSJoseph Koshy  *
3474f263522aSJoseph Koshy  * Find a free slot in the per-cpu array of PC samples and write the
3475f263522aSJoseph Koshy  * current (PMC,PID,PC) triple to it.  If an event was successfully
3476f263522aSJoseph Koshy  * added, a bit is set in mask 'pmc_cpumask' denoting that the
3477f263522aSJoseph Koshy  * DO_SAMPLES hook needs to be invoked from the clock handler.
3478f263522aSJoseph Koshy  *
3479f263522aSJoseph Koshy  * This function is meant to be called from an NMI handler.  It cannot
3480f263522aSJoseph Koshy  * use any of the locking primitives supplied by the OS.
3481ebccf1e3SJoseph Koshy  */
3482ebccf1e3SJoseph Koshy 
3483f263522aSJoseph Koshy int
348415139246SJoseph Koshy pmc_process_interrupt(int cpu, struct pmc *pm, uintfptr_t pc, int usermode)
3485ebccf1e3SJoseph Koshy {
3486f263522aSJoseph Koshy 	int error, ri;
3487f263522aSJoseph Koshy 	struct thread *td;
3488f263522aSJoseph Koshy 	struct pmc_sample *ps;
3489f263522aSJoseph Koshy 	struct pmc_samplebuffer *psb;
3490ebccf1e3SJoseph Koshy 
3491f263522aSJoseph Koshy 	error = 0;
3492f263522aSJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
3493ebccf1e3SJoseph Koshy 
3494f263522aSJoseph Koshy 	psb = pmc_pcpu[cpu]->pc_sb;
3495ebccf1e3SJoseph Koshy 
3496f263522aSJoseph Koshy 	ps = psb->ps_write;
3497f263522aSJoseph Koshy 	if (ps->ps_pc) {	/* in use, reader hasn't caught up */
3498fbf1556dSJoseph Koshy 		pm->pm_stalled = 1;
3499f263522aSJoseph Koshy 		atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
3500f263522aSJoseph Koshy 		PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d",
350115139246SJoseph Koshy 		    cpu, pm, (uint64_t) pc, usermode,
3502f263522aSJoseph Koshy 		    (int) (psb->ps_write - psb->ps_samples),
3503f263522aSJoseph Koshy 		    (int) (psb->ps_read - psb->ps_samples));
3504f263522aSJoseph Koshy 		error = ENOMEM;
3505f263522aSJoseph Koshy 		goto done;
3506ebccf1e3SJoseph Koshy 	}
3507ebccf1e3SJoseph Koshy 
3508f263522aSJoseph Koshy 	/* fill in entry */
3509f263522aSJoseph Koshy 	PMCDBG(SAM,INT,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, pm,
351015139246SJoseph Koshy 	    (uint64_t) pc, usermode,
3511f263522aSJoseph Koshy 	    (int) (psb->ps_write - psb->ps_samples),
3512f263522aSJoseph Koshy 	    (int) (psb->ps_read - psb->ps_samples));
3513f263522aSJoseph Koshy 
3514f263522aSJoseph Koshy 	atomic_add_rel_32(&pm->pm_runcount, 1);		/* hold onto PMC */
3515f263522aSJoseph Koshy 	ps->ps_pmc = pm;
3516f263522aSJoseph Koshy 	if ((td = curthread) && td->td_proc)
3517f263522aSJoseph Koshy 		ps->ps_pid = td->td_proc->p_pid;
3518f263522aSJoseph Koshy 	else
3519f263522aSJoseph Koshy 		ps->ps_pid = -1;
3520f263522aSJoseph Koshy 	ps->ps_usermode = usermode;
3521f263522aSJoseph Koshy 	ps->ps_pc = pc;		/* mark entry as in use */
3522f263522aSJoseph Koshy 
3523f263522aSJoseph Koshy 	/* increment write pointer, modulo ring buffer size */
3524f263522aSJoseph Koshy 	ps++;
3525f263522aSJoseph Koshy 	if (ps == psb->ps_fence)
3526f263522aSJoseph Koshy 		psb->ps_write = psb->ps_samples;
3527f263522aSJoseph Koshy 	else
3528f263522aSJoseph Koshy 		psb->ps_write = ps;
3529f263522aSJoseph Koshy 
3530f263522aSJoseph Koshy  done:
3531f263522aSJoseph Koshy 	/* mark CPU as needing processing */
3532f263522aSJoseph Koshy 	atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3533f263522aSJoseph Koshy 
3534f263522aSJoseph Koshy 	return error;
3535f263522aSJoseph Koshy }
3536f263522aSJoseph Koshy 
3537f263522aSJoseph Koshy 
3538ebccf1e3SJoseph Koshy /*
3539f263522aSJoseph Koshy  * Process saved PC samples.
3540ebccf1e3SJoseph Koshy  */
3541ebccf1e3SJoseph Koshy 
3542ebccf1e3SJoseph Koshy static void
3543f263522aSJoseph Koshy pmc_process_samples(int cpu)
3544ebccf1e3SJoseph Koshy {
3545f263522aSJoseph Koshy 	int n, ri;
3546f263522aSJoseph Koshy 	struct pmc *pm;
3547f263522aSJoseph Koshy 	struct thread *td;
3548f263522aSJoseph Koshy 	struct pmc_owner *po;
3549f263522aSJoseph Koshy 	struct pmc_sample *ps;
3550f263522aSJoseph Koshy 	struct pmc_samplebuffer *psb;
3551ebccf1e3SJoseph Koshy 
3552f263522aSJoseph Koshy 	KASSERT(PCPU_GET(cpuid) == cpu,
3553f263522aSJoseph Koshy 	    ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
3554f263522aSJoseph Koshy 		PCPU_GET(cpuid), cpu));
3555ebccf1e3SJoseph Koshy 
3556f263522aSJoseph Koshy 	psb = pmc_pcpu[cpu]->pc_sb;
3557f263522aSJoseph Koshy 
3558f263522aSJoseph Koshy 	for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
3559f263522aSJoseph Koshy 
3560f263522aSJoseph Koshy 		ps = psb->ps_read;
3561f263522aSJoseph Koshy 		if (ps->ps_pc == (uintfptr_t) 0)	/* no data */
3562f263522aSJoseph Koshy 			break;
3563f263522aSJoseph Koshy 
3564f263522aSJoseph Koshy 		pm = ps->ps_pmc;
3565f263522aSJoseph Koshy 		po = pm->pm_owner;
3566f263522aSJoseph Koshy 
3567f263522aSJoseph Koshy 		KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
3568f263522aSJoseph Koshy 		    ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
3569f263522aSJoseph Koshy 			pm, PMC_TO_MODE(pm)));
3570f263522aSJoseph Koshy 
3571f263522aSJoseph Koshy 		/* Ignore PMCs that have been switched off */
3572f263522aSJoseph Koshy 		if (pm->pm_state != PMC_STATE_RUNNING)
3573f263522aSJoseph Koshy 			goto entrydone;
3574f263522aSJoseph Koshy 
3575f263522aSJoseph Koshy 		PMCDBG(SAM,OPS,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu,
357615139246SJoseph Koshy 		    pm, (uint64_t) ps->ps_pc, ps->ps_usermode,
3577f263522aSJoseph Koshy 		    (int) (psb->ps_write - psb->ps_samples),
3578f263522aSJoseph Koshy 		    (int) (psb->ps_read - psb->ps_samples));
3579f263522aSJoseph Koshy 
3580f263522aSJoseph Koshy 		/*
3581f263522aSJoseph Koshy 		 * If this is a process-mode PMC that is attached to
3582f263522aSJoseph Koshy 		 * its owner, and if the PC is in user mode, update
3583f263522aSJoseph Koshy 		 * profiling statistics like timer-based profiling
3584f263522aSJoseph Koshy 		 * would have done.
3585f263522aSJoseph Koshy 		 */
3586f263522aSJoseph Koshy 		if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
3587f263522aSJoseph Koshy 			if (ps->ps_usermode) {
3588f263522aSJoseph Koshy 				td = FIRST_THREAD_IN_PROC(po->po_owner);
3589f263522aSJoseph Koshy 				addupc_intr(td, ps->ps_pc, 1);
3590f263522aSJoseph Koshy 			}
3591f263522aSJoseph Koshy 			goto entrydone;
3592f263522aSJoseph Koshy 		}
3593f263522aSJoseph Koshy 
3594f263522aSJoseph Koshy 		/*
3595f263522aSJoseph Koshy 		 * Otherwise, this is either a sampling mode PMC that
3596f263522aSJoseph Koshy 		 * is attached to a different process than its owner,
3597f263522aSJoseph Koshy 		 * or a system-wide sampling PMC.  Dispatch a log
3598f263522aSJoseph Koshy 		 * entry to the PMC's owner process.
3599f263522aSJoseph Koshy 		 */
3600f263522aSJoseph Koshy 
3601f263522aSJoseph Koshy 		pmclog_process_pcsample(pm, ps);
3602f263522aSJoseph Koshy 
3603f263522aSJoseph Koshy 	entrydone:
3604f263522aSJoseph Koshy 		ps->ps_pc = (uintfptr_t) 0;	/* mark entry as free */
3605f263522aSJoseph Koshy 		atomic_subtract_rel_32(&pm->pm_runcount, 1);
3606f263522aSJoseph Koshy 
3607f263522aSJoseph Koshy 		/* increment read pointer, modulo sample size */
3608f263522aSJoseph Koshy 		if (++ps == psb->ps_fence)
3609f263522aSJoseph Koshy 			psb->ps_read = psb->ps_samples;
3610f263522aSJoseph Koshy 		else
3611f263522aSJoseph Koshy 			psb->ps_read = ps;
3612f263522aSJoseph Koshy 	}
3613f263522aSJoseph Koshy 
3614f263522aSJoseph Koshy 	atomic_add_int(&pmc_stats.pm_log_sweeps, 1);
3615f263522aSJoseph Koshy 
3616f263522aSJoseph Koshy 	/* Do not re-enable stalled PMCs if we failed to process any samples */
3617f263522aSJoseph Koshy 	if (n == 0)
3618ebccf1e3SJoseph Koshy 		return;
3619f263522aSJoseph Koshy 
3620f263522aSJoseph Koshy 	/*
3621f263522aSJoseph Koshy 	 * Restart any stalled sampling PMCs on this CPU.
3622f263522aSJoseph Koshy 	 *
3623fbf1556dSJoseph Koshy 	 * If the NMI handler sets the pm_stalled field of a PMC after
3624fbf1556dSJoseph Koshy 	 * the check below, we'll end up processing the stalled PMC at
3625fbf1556dSJoseph Koshy 	 * the next hardclock tick.
3626f263522aSJoseph Koshy 	 */
3627f263522aSJoseph Koshy 	for (n = 0; n < md->pmd_npmc; n++) {
3628f263522aSJoseph Koshy 		(void) (*md->pmd_get_config)(cpu,n,&pm);
3629f263522aSJoseph Koshy 		if (pm == NULL ||			 /* !cfg'ed */
3630f263522aSJoseph Koshy 		    pm->pm_state != PMC_STATE_RUNNING || /* !active */
3631f263522aSJoseph Koshy 		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
3632fbf1556dSJoseph Koshy 		    pm->pm_stalled == 0) /* !stalled */
3633f263522aSJoseph Koshy 			continue;
3634f263522aSJoseph Koshy 
3635fbf1556dSJoseph Koshy 		pm->pm_stalled = 0;
3636f263522aSJoseph Koshy 		ri = PMC_TO_ROWINDEX(pm);
3637f263522aSJoseph Koshy 		(*md->pmd_start_pmc)(cpu, ri);
3638f263522aSJoseph Koshy 	}
3639ebccf1e3SJoseph Koshy }
3640ebccf1e3SJoseph Koshy 
3641ebccf1e3SJoseph Koshy /*
3642ebccf1e3SJoseph Koshy  * Event handlers.
3643ebccf1e3SJoseph Koshy  */
3644ebccf1e3SJoseph Koshy 
3645ebccf1e3SJoseph Koshy /*
3646ebccf1e3SJoseph Koshy  * Handle a process exit.
3647ebccf1e3SJoseph Koshy  *
3648f263522aSJoseph Koshy  * Remove this process from all hash tables.  If this process
3649f263522aSJoseph Koshy  * owned any PMCs, turn off those PMCs and deallocate them,
3650f263522aSJoseph Koshy  * removing any associations with target processes.
3651f263522aSJoseph Koshy  *
3652f263522aSJoseph Koshy  * This function will be called by the last 'thread' of a
3653f263522aSJoseph Koshy  * process.
3654f263522aSJoseph Koshy  *
3655ebccf1e3SJoseph Koshy  * XXX This eventhandler gets called early in the exit process.
3656ebccf1e3SJoseph Koshy  * Consider using a 'hook' invocation from thread_exit() or equivalent
3657ebccf1e3SJoseph Koshy  * spot.  Another negative is that kse_exit doesn't seem to call
3658ebccf1e3SJoseph Koshy  * exit1() [??].
3659f263522aSJoseph Koshy  *
3660ebccf1e3SJoseph Koshy  */
3661ebccf1e3SJoseph Koshy 
3662ebccf1e3SJoseph Koshy static void
3663ebccf1e3SJoseph Koshy pmc_process_exit(void *arg __unused, struct proc *p)
3664ebccf1e3SJoseph Koshy {
3665ebccf1e3SJoseph Koshy 	int is_using_hwpmcs;
3666f263522aSJoseph Koshy 	int cpu;
3667f263522aSJoseph Koshy 	unsigned int ri;
3668f263522aSJoseph Koshy 	struct pmc *pm;
3669f263522aSJoseph Koshy 	struct pmc_process *pp;
3670f263522aSJoseph Koshy 	struct pmc_owner *po;
3671f263522aSJoseph Koshy 	pmc_value_t newvalue, tmp;
3672ebccf1e3SJoseph Koshy 
3673ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
3674ebccf1e3SJoseph Koshy 	is_using_hwpmcs = p->p_flag & P_HWPMC;
3675ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
3676ebccf1e3SJoseph Koshy 
3677f263522aSJoseph Koshy 	/*
3678f263522aSJoseph Koshy 	 * Log a sysexit event to all SS PMC owners.
3679f263522aSJoseph Koshy 	 */
3680f263522aSJoseph Koshy 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
3681f263522aSJoseph Koshy 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
3682f263522aSJoseph Koshy 		    pmclog_process_sysexit(po, p->p_pid);
3683f263522aSJoseph Koshy 
3684f263522aSJoseph Koshy 	if (!is_using_hwpmcs)
3685f263522aSJoseph Koshy 		return;
3686f263522aSJoseph Koshy 
3687f263522aSJoseph Koshy 	PMC_GET_SX_XLOCK();
3688ebccf1e3SJoseph Koshy 	PMCDBG(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
3689ebccf1e3SJoseph Koshy 	    p->p_comm);
3690ebccf1e3SJoseph Koshy 
3691f263522aSJoseph Koshy 	/*
3692f263522aSJoseph Koshy 	 * Since this code is invoked by the last thread in an exiting
3693f263522aSJoseph Koshy 	 * process, we would have context switched IN at some prior
3694f263522aSJoseph Koshy 	 * point.  However, with PREEMPTION, kernel mode context
3695f263522aSJoseph Koshy 	 * switches may happen any time, so we want to disable a
3696f263522aSJoseph Koshy 	 * context switch OUT till we get any PMCs targetting this
3697f263522aSJoseph Koshy 	 * process off the hardware.
3698f263522aSJoseph Koshy 	 *
3699f263522aSJoseph Koshy 	 * We also need to atomically remove this process'
3700f263522aSJoseph Koshy 	 * entry from our target process hash table, using
3701f263522aSJoseph Koshy 	 * PMC_FLAG_REMOVE.
3702f263522aSJoseph Koshy 	 */
3703f263522aSJoseph Koshy 	PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
3704f263522aSJoseph Koshy 	    p->p_comm);
3705f263522aSJoseph Koshy 
3706f263522aSJoseph Koshy 	critical_enter(); /* no preemption */
3707f263522aSJoseph Koshy 
3708f263522aSJoseph Koshy 	cpu = curthread->td_oncpu;
3709f263522aSJoseph Koshy 
3710f263522aSJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p,
3711f263522aSJoseph Koshy 		 PMC_FLAG_REMOVE)) != NULL) {
3712f263522aSJoseph Koshy 
3713f263522aSJoseph Koshy 		PMCDBG(PRC,EXT,2,
3714f263522aSJoseph Koshy 		    "process-exit proc=%p pmc-process=%p", p, pp);
3715f263522aSJoseph Koshy 
3716f263522aSJoseph Koshy 		/*
3717f263522aSJoseph Koshy 		 * The exiting process could the target of
3718f263522aSJoseph Koshy 		 * some PMCs which will be running on
3719f263522aSJoseph Koshy 		 * currently executing CPU.
3720f263522aSJoseph Koshy 		 *
3721f263522aSJoseph Koshy 		 * We need to turn these PMCs off like we
3722f263522aSJoseph Koshy 		 * would do at context switch OUT time.
3723f263522aSJoseph Koshy 		 */
3724f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++) {
3725f263522aSJoseph Koshy 
3726f263522aSJoseph Koshy 			/*
3727f263522aSJoseph Koshy 			 * Pick up the pmc pointer from hardware
3728f263522aSJoseph Koshy 			 * state similar to the CSW_OUT code.
3729f263522aSJoseph Koshy 			 */
3730f263522aSJoseph Koshy 			pm = NULL;
3731f263522aSJoseph Koshy 			(void) (*md->pmd_get_config)(cpu, ri, &pm);
3732f263522aSJoseph Koshy 
3733f263522aSJoseph Koshy 			PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
3734f263522aSJoseph Koshy 
3735f263522aSJoseph Koshy 			if (pm == NULL ||
3736f263522aSJoseph Koshy 			    !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
3737f263522aSJoseph Koshy 				continue;
3738f263522aSJoseph Koshy 
3739f263522aSJoseph Koshy 			PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
3740f263522aSJoseph Koshy 			    "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
3741f263522aSJoseph Koshy 			    pm, pm->pm_state);
3742f263522aSJoseph Koshy 
3743f263522aSJoseph Koshy 			KASSERT(PMC_TO_ROWINDEX(pm) == ri,
3744f263522aSJoseph Koshy 			    ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
3745f263522aSJoseph Koshy 				__LINE__, PMC_TO_ROWINDEX(pm), ri));
3746f263522aSJoseph Koshy 
3747f263522aSJoseph Koshy 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
3748f263522aSJoseph Koshy 			    ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
3749f263522aSJoseph Koshy 				__LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
3750f263522aSJoseph Koshy 
3751f263522aSJoseph Koshy 			(void) md->pmd_stop_pmc(cpu, ri);
3752f263522aSJoseph Koshy 
3753f263522aSJoseph Koshy 			KASSERT(pm->pm_runcount > 0,
3754f263522aSJoseph Koshy 			    ("[pmc,%d] bad runcount ri %d rc %d",
3755f263522aSJoseph Koshy 				__LINE__, ri, pm->pm_runcount));
3756f263522aSJoseph Koshy 
3757fbf1556dSJoseph Koshy 			/* Stop hardware only if it is actually running */
3758f263522aSJoseph Koshy 			if (pm->pm_state == PMC_STATE_RUNNING &&
3759fbf1556dSJoseph Koshy 			    pm->pm_stalled == 0) {
3760f263522aSJoseph Koshy 				md->pmd_read_pmc(cpu, ri, &newvalue);
3761f263522aSJoseph Koshy 				tmp = newvalue -
3762f263522aSJoseph Koshy 				    PMC_PCPU_SAVED(cpu,ri);
3763f263522aSJoseph Koshy 
3764f263522aSJoseph Koshy 				mtx_pool_lock_spin(pmc_mtxpool, pm);
3765f263522aSJoseph Koshy 				pm->pm_gv.pm_savedvalue += tmp;
3766f263522aSJoseph Koshy 				pp->pp_pmcs[ri].pp_pmcval += tmp;
3767f263522aSJoseph Koshy 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
3768ebccf1e3SJoseph Koshy 			}
3769f263522aSJoseph Koshy 
3770f263522aSJoseph Koshy 			atomic_subtract_rel_32(&pm->pm_runcount,1);
3771f263522aSJoseph Koshy 
3772f263522aSJoseph Koshy 			KASSERT((int) pm->pm_runcount >= 0,
3773f263522aSJoseph Koshy 			    ("[pmc,%d] runcount is %d", __LINE__, ri));
3774f263522aSJoseph Koshy 
3775f263522aSJoseph Koshy 			(void) md->pmd_config_pmc(cpu, ri, NULL);
3776f263522aSJoseph Koshy 		}
3777f263522aSJoseph Koshy 
3778f263522aSJoseph Koshy 		/*
3779f263522aSJoseph Koshy 		 * Inform the MD layer of this pseudo "context switch
3780f263522aSJoseph Koshy 		 * out"
3781f263522aSJoseph Koshy 		 */
3782f263522aSJoseph Koshy 		(void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
3783f263522aSJoseph Koshy 
3784f263522aSJoseph Koshy 		critical_exit(); /* ok to be pre-empted now */
3785f263522aSJoseph Koshy 
3786f263522aSJoseph Koshy 		/*
3787f263522aSJoseph Koshy 		 * Unlink this process from the PMCs that are
3788f263522aSJoseph Koshy 		 * targetting it.  This will send a signal to
3789f263522aSJoseph Koshy 		 * all PMC owner's whose PMCs are orphaned.
3790f263522aSJoseph Koshy 		 *
3791f263522aSJoseph Koshy 		 * Log PMC value at exit time if requested.
3792f263522aSJoseph Koshy 		 */
3793f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++)
3794f263522aSJoseph Koshy 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
3795fbf1556dSJoseph Koshy 				if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
3796fbf1556dSJoseph Koshy 				    PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
3797f263522aSJoseph Koshy 					pmclog_process_procexit(pm, pp);
3798f263522aSJoseph Koshy 				pmc_unlink_target_process(pm, pp);
3799f263522aSJoseph Koshy 			}
3800f263522aSJoseph Koshy 		FREE(pp, M_PMC);
3801f263522aSJoseph Koshy 
3802f263522aSJoseph Koshy 	} else
3803f263522aSJoseph Koshy 		critical_exit(); /* pp == NULL */
3804f263522aSJoseph Koshy 
3805f263522aSJoseph Koshy 
3806f263522aSJoseph Koshy 	/*
3807f263522aSJoseph Koshy 	 * If the process owned PMCs, free them up and free up
3808f263522aSJoseph Koshy 	 * memory.
3809f263522aSJoseph Koshy 	 */
3810f263522aSJoseph Koshy 	if ((po = pmc_find_owner_descriptor(p)) != NULL) {
3811f263522aSJoseph Koshy 		pmc_remove_owner(po);
3812f263522aSJoseph Koshy 		pmc_destroy_owner_descriptor(po);
3813f263522aSJoseph Koshy 	}
3814f263522aSJoseph Koshy 
3815f263522aSJoseph Koshy 	sx_xunlock(&pmc_sx);
3816ebccf1e3SJoseph Koshy }
3817ebccf1e3SJoseph Koshy 
3818ebccf1e3SJoseph Koshy /*
3819ebccf1e3SJoseph Koshy  * Handle a process fork.
3820ebccf1e3SJoseph Koshy  *
3821ebccf1e3SJoseph Koshy  * If the parent process 'p1' is under HWPMC monitoring, then copy
3822ebccf1e3SJoseph Koshy  * over any attached PMCs that have 'do_descendants' semantics.
3823ebccf1e3SJoseph Koshy  */
3824ebccf1e3SJoseph Koshy 
3825ebccf1e3SJoseph Koshy static void
3826f263522aSJoseph Koshy pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
3827ebccf1e3SJoseph Koshy     int flags)
3828ebccf1e3SJoseph Koshy {
3829ebccf1e3SJoseph Koshy 	int is_using_hwpmcs;
3830f263522aSJoseph Koshy 	unsigned int ri;
3831f263522aSJoseph Koshy 	uint32_t do_descendants;
3832f263522aSJoseph Koshy 	struct pmc *pm;
3833f263522aSJoseph Koshy 	struct pmc_owner *po;
3834f263522aSJoseph Koshy 	struct pmc_process *ppnew, *ppold;
3835ebccf1e3SJoseph Koshy 
3836ebccf1e3SJoseph Koshy 	(void) flags;		/* unused parameter */
3837ebccf1e3SJoseph Koshy 
3838ebccf1e3SJoseph Koshy 	PROC_LOCK(p1);
3839ebccf1e3SJoseph Koshy 	is_using_hwpmcs = p1->p_flag & P_HWPMC;
3840ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p1);
3841ebccf1e3SJoseph Koshy 
3842f263522aSJoseph Koshy 	/*
3843f263522aSJoseph Koshy 	 * If there are system-wide sampling PMCs active, we need to
3844f263522aSJoseph Koshy 	 * log all fork events to their owner's logs.
3845f263522aSJoseph Koshy 	 */
3846f263522aSJoseph Koshy 
3847f263522aSJoseph Koshy 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
3848f263522aSJoseph Koshy 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
3849f263522aSJoseph Koshy 		    pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
3850f263522aSJoseph Koshy 
3851f263522aSJoseph Koshy 	if (!is_using_hwpmcs)
3852f263522aSJoseph Koshy 		return;
3853f263522aSJoseph Koshy 
3854ebccf1e3SJoseph Koshy 	PMC_GET_SX_XLOCK();
3855f263522aSJoseph Koshy 	PMCDBG(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
3856f263522aSJoseph Koshy 	    p1->p_pid, p1->p_comm, newproc);
3857f263522aSJoseph Koshy 
3858f263522aSJoseph Koshy 	/*
3859f263522aSJoseph Koshy 	 * If the parent process (curthread->td_proc) is a
3860f263522aSJoseph Koshy 	 * target of any PMCs, look for PMCs that are to be
3861f263522aSJoseph Koshy 	 * inherited, and link these into the new process
3862f263522aSJoseph Koshy 	 * descriptor.
3863f263522aSJoseph Koshy 	 */
3864f263522aSJoseph Koshy 	if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
3865f263522aSJoseph Koshy 		 PMC_FLAG_NONE)) == NULL)
3866f263522aSJoseph Koshy 		goto done;		/* nothing to do */
3867f263522aSJoseph Koshy 
3868f263522aSJoseph Koshy 	do_descendants = 0;
3869f263522aSJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++)
3870f263522aSJoseph Koshy 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
3871f263522aSJoseph Koshy 			do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
3872f263522aSJoseph Koshy 	if (do_descendants == 0) /* nothing to do */
3873f263522aSJoseph Koshy 		goto done;
3874f263522aSJoseph Koshy 
3875f263522aSJoseph Koshy 	/* allocate a descriptor for the new process  */
3876f263522aSJoseph Koshy 	if ((ppnew = pmc_find_process_descriptor(newproc,
3877f263522aSJoseph Koshy 		 PMC_FLAG_ALLOCATE)) == NULL)
3878f263522aSJoseph Koshy 		goto done;
3879f263522aSJoseph Koshy 
3880f263522aSJoseph Koshy 	/*
3881f263522aSJoseph Koshy 	 * Run through all PMCs that were targeting the old process
3882f263522aSJoseph Koshy 	 * and which specified F_DESCENDANTS and attach them to the
3883f263522aSJoseph Koshy 	 * new process.
3884f263522aSJoseph Koshy 	 *
3885f263522aSJoseph Koshy 	 * Log the fork event to all owners of PMCs attached to this
3886f263522aSJoseph Koshy 	 * process, if not already logged.
3887f263522aSJoseph Koshy 	 */
3888f263522aSJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++)
3889f263522aSJoseph Koshy 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
3890f263522aSJoseph Koshy 		    (pm->pm_flags & PMC_F_DESCENDANTS)) {
3891f263522aSJoseph Koshy 			pmc_link_target_process(pm, ppnew);
3892f263522aSJoseph Koshy 			po = pm->pm_owner;
3893f263522aSJoseph Koshy 			if (po->po_sscount == 0 &&
3894f263522aSJoseph Koshy 			    po->po_flags & PMC_PO_OWNS_LOGFILE)
3895f263522aSJoseph Koshy 				pmclog_process_procfork(po, p1->p_pid,
3896f263522aSJoseph Koshy 				    newproc->p_pid);
3897ebccf1e3SJoseph Koshy 		}
3898f263522aSJoseph Koshy 
3899f263522aSJoseph Koshy 	/*
3900f263522aSJoseph Koshy 	 * Now mark the new process as being tracked by this driver.
3901f263522aSJoseph Koshy 	 */
3902f263522aSJoseph Koshy 	PROC_LOCK(newproc);
3903f263522aSJoseph Koshy 	newproc->p_flag |= P_HWPMC;
3904f263522aSJoseph Koshy 	PROC_UNLOCK(newproc);
3905f263522aSJoseph Koshy 
3906f263522aSJoseph Koshy  done:
3907f263522aSJoseph Koshy 	sx_xunlock(&pmc_sx);
3908ebccf1e3SJoseph Koshy }
3909ebccf1e3SJoseph Koshy 
3910ebccf1e3SJoseph Koshy 
3911ebccf1e3SJoseph Koshy /*
3912ebccf1e3SJoseph Koshy  * initialization
3913ebccf1e3SJoseph Koshy  */
3914ebccf1e3SJoseph Koshy 
3915ebccf1e3SJoseph Koshy static const char *pmc_name_of_pmcclass[] = {
3916ebccf1e3SJoseph Koshy #undef	__PMC_CLASS
3917ebccf1e3SJoseph Koshy #define	__PMC_CLASS(N) #N ,
3918ebccf1e3SJoseph Koshy 	__PMC_CLASSES()
3919ebccf1e3SJoseph Koshy };
3920ebccf1e3SJoseph Koshy 
3921ebccf1e3SJoseph Koshy static int
3922ebccf1e3SJoseph Koshy pmc_initialize(void)
3923ebccf1e3SJoseph Koshy {
3924f263522aSJoseph Koshy 	int cpu, error, n;
3925ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
3926f263522aSJoseph Koshy 	struct pmc_samplebuffer *sb;
3927ebccf1e3SJoseph Koshy 
3928ebccf1e3SJoseph Koshy 	md = NULL;
3929ebccf1e3SJoseph Koshy 	error = 0;
3930ebccf1e3SJoseph Koshy 
3931342ed5d9SRuslan Ermilov #ifdef	DEBUG
3932ebccf1e3SJoseph Koshy 	/* parse debug flags first */
3933ebccf1e3SJoseph Koshy 	if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
3934ebccf1e3SJoseph Koshy 		pmc_debugstr, sizeof(pmc_debugstr)))
3935ebccf1e3SJoseph Koshy 		pmc_debugflags_parse(pmc_debugstr,
3936ebccf1e3SJoseph Koshy 		    pmc_debugstr+strlen(pmc_debugstr));
3937ebccf1e3SJoseph Koshy #endif
3938ebccf1e3SJoseph Koshy 
3939ebccf1e3SJoseph Koshy 	PMCDBG(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
3940ebccf1e3SJoseph Koshy 
3941fadcc6e2SJoseph Koshy 	/* check kernel version */
3942fadcc6e2SJoseph Koshy 	if (pmc_kernel_version != PMC_VERSION) {
3943fadcc6e2SJoseph Koshy 		if (pmc_kernel_version == 0)
3944fadcc6e2SJoseph Koshy 			printf("hwpmc: this kernel has not been compiled with "
3945fadcc6e2SJoseph Koshy 			    "'options HWPMC_HOOKS'.\n");
3946fadcc6e2SJoseph Koshy 		else
3947fadcc6e2SJoseph Koshy 			printf("hwpmc: kernel version (0x%x) does not match "
3948fadcc6e2SJoseph Koshy 			    "module version (0x%x).\n", pmc_kernel_version,
3949fadcc6e2SJoseph Koshy 			    PMC_VERSION);
3950fadcc6e2SJoseph Koshy 		return EPROGMISMATCH;
3951fadcc6e2SJoseph Koshy 	}
3952fadcc6e2SJoseph Koshy 
3953ebccf1e3SJoseph Koshy 	/*
3954ebccf1e3SJoseph Koshy 	 * check sysctl parameters
3955ebccf1e3SJoseph Koshy 	 */
3956ebccf1e3SJoseph Koshy 
3957ebccf1e3SJoseph Koshy 	if (pmc_hashsize <= 0) {
3958f263522aSJoseph Koshy 		(void) printf("hwpmc: tunable hashsize=%d must be greater "
3959f263522aSJoseph Koshy 		    "than zero.\n", pmc_hashsize);
3960ebccf1e3SJoseph Koshy 		pmc_hashsize = PMC_HASH_SIZE;
3961ebccf1e3SJoseph Koshy 	}
3962ebccf1e3SJoseph Koshy 
3963f263522aSJoseph Koshy 	if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
3964ba8ae2deSJoseph Koshy 		(void) printf("hwpmc: tunable nsamples=%d out of range.\n",
3965ba8ae2deSJoseph Koshy 		    pmc_nsamples);
3966f263522aSJoseph Koshy 		pmc_nsamples = PMC_NSAMPLES;
3967f263522aSJoseph Koshy 	}
3968f263522aSJoseph Koshy 
3969f263522aSJoseph Koshy 	md = pmc_md_initialize();
3970ebccf1e3SJoseph Koshy 
3971ebccf1e3SJoseph Koshy 	if (md == NULL || md->pmd_init == NULL)
3972ebccf1e3SJoseph Koshy 		return ENOSYS;
3973ebccf1e3SJoseph Koshy 
3974ebccf1e3SJoseph Koshy 	/* allocate space for the per-cpu array */
3975ebccf1e3SJoseph Koshy 	MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *),
3976ebccf1e3SJoseph Koshy 	    M_PMC, M_WAITOK|M_ZERO);
3977ebccf1e3SJoseph Koshy 
3978ebccf1e3SJoseph Koshy 	/* per-cpu 'saved values' for managing process-mode PMCs */
3979ebccf1e3SJoseph Koshy 	MALLOC(pmc_pcpu_saved, pmc_value_t *,
3980ebccf1e3SJoseph Koshy 	    sizeof(pmc_value_t) * mp_ncpus * md->pmd_npmc, M_PMC, M_WAITOK);
3981ebccf1e3SJoseph Koshy 
3982ebccf1e3SJoseph Koshy 	/* perform cpu dependent initialization */
3983ebccf1e3SJoseph Koshy 	pmc_save_cpu_binding(&pb);
3984ebccf1e3SJoseph Koshy 	for (cpu = 0; cpu < mp_ncpus; cpu++) {
3985ebccf1e3SJoseph Koshy 		if (pmc_cpu_is_disabled(cpu))
3986ebccf1e3SJoseph Koshy 			continue;
3987ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
3988ebccf1e3SJoseph Koshy 		if ((error = md->pmd_init(cpu)) != 0)
3989ebccf1e3SJoseph Koshy 			break;
3990ebccf1e3SJoseph Koshy 	}
3991ebccf1e3SJoseph Koshy 	pmc_restore_cpu_binding(&pb);
3992ebccf1e3SJoseph Koshy 
3993ebccf1e3SJoseph Koshy 	if (error != 0)
3994ebccf1e3SJoseph Koshy 		return error;
3995ebccf1e3SJoseph Koshy 
3996f263522aSJoseph Koshy 	/* allocate space for the sample array */
3997f263522aSJoseph Koshy 	for (cpu = 0; cpu < mp_ncpus; cpu++) {
3998f263522aSJoseph Koshy 		if (pmc_cpu_is_disabled(cpu))
3999f263522aSJoseph Koshy 			continue;
4000f263522aSJoseph Koshy 		MALLOC(sb, struct pmc_samplebuffer *,
4001f263522aSJoseph Koshy 		    sizeof(struct pmc_samplebuffer) +
4002f263522aSJoseph Koshy 		    pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
4003f263522aSJoseph Koshy 		    M_WAITOK|M_ZERO);
4004f263522aSJoseph Koshy 
4005f263522aSJoseph Koshy 		sb->ps_read = sb->ps_write = sb->ps_samples;
4006ba8ae2deSJoseph Koshy 		sb->ps_fence = sb->ps_samples + pmc_nsamples;
4007f263522aSJoseph Koshy 		KASSERT(pmc_pcpu[cpu] != NULL,
4008f263522aSJoseph Koshy 		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
4009f263522aSJoseph Koshy 
4010f263522aSJoseph Koshy 		pmc_pcpu[cpu]->pc_sb = sb;
4011f263522aSJoseph Koshy 	}
4012f263522aSJoseph Koshy 
4013ebccf1e3SJoseph Koshy 	/* allocate space for the row disposition array */
4014ebccf1e3SJoseph Koshy 	pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
4015ebccf1e3SJoseph Koshy 	    M_PMC, M_WAITOK|M_ZERO);
4016ebccf1e3SJoseph Koshy 
4017ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp != NULL,
4018ebccf1e3SJoseph Koshy 	    ("[pmc,%d] pmcdisp allocation returned NULL", __LINE__));
4019ebccf1e3SJoseph Koshy 
4020ebccf1e3SJoseph Koshy 	/* mark all PMCs as available */
4021ebccf1e3SJoseph Koshy 	for (n = 0; n < (int) md->pmd_npmc; n++)
4022ebccf1e3SJoseph Koshy 		PMC_MARK_ROW_FREE(n);
4023ebccf1e3SJoseph Koshy 
4024ebccf1e3SJoseph Koshy 	/* allocate thread hash tables */
4025ebccf1e3SJoseph Koshy 	pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
4026ebccf1e3SJoseph Koshy 	    &pmc_ownerhashmask);
4027ebccf1e3SJoseph Koshy 
4028ebccf1e3SJoseph Koshy 	pmc_processhash = hashinit(pmc_hashsize, M_PMC,
4029ebccf1e3SJoseph Koshy 	    &pmc_processhashmask);
4030ebccf1e3SJoseph Koshy 	mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc", MTX_SPIN);
4031ebccf1e3SJoseph Koshy 
4032f263522aSJoseph Koshy 	LIST_INIT(&pmc_ss_owners);
4033f263522aSJoseph Koshy 	pmc_ss_count = 0;
4034f263522aSJoseph Koshy 
4035ebccf1e3SJoseph Koshy 	/* allocate a pool of spin mutexes */
4036ebccf1e3SJoseph Koshy 	pmc_mtxpool = mtx_pool_create("pmc", pmc_mtxpool_size, MTX_SPIN);
4037ebccf1e3SJoseph Koshy 
4038ebccf1e3SJoseph Koshy 	PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
4039ebccf1e3SJoseph Koshy 	    "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
4040ebccf1e3SJoseph Koshy 	    pmc_processhash, pmc_processhashmask);
4041ebccf1e3SJoseph Koshy 
4042ebccf1e3SJoseph Koshy 	/* register process {exit,fork,exec} handlers */
4043ebccf1e3SJoseph Koshy 	pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
4044ebccf1e3SJoseph Koshy 	    pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
4045ebccf1e3SJoseph Koshy 	pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
4046ebccf1e3SJoseph Koshy 	    pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
4047ebccf1e3SJoseph Koshy 
4048f263522aSJoseph Koshy 	/* initialize logging */
4049f263522aSJoseph Koshy 	pmclog_initialize();
4050f263522aSJoseph Koshy 
4051ebccf1e3SJoseph Koshy 	/* set hook functions */
4052ebccf1e3SJoseph Koshy 	pmc_intr = md->pmd_intr;
4053ebccf1e3SJoseph Koshy 	pmc_hook = pmc_hook_handler;
4054ebccf1e3SJoseph Koshy 
4055ebccf1e3SJoseph Koshy 	if (error == 0) {
4056ebccf1e3SJoseph Koshy 		printf(PMC_MODULE_NAME ":");
4057a8eb16c5SJoseph Koshy 		for (n = 0; n < (int) md->pmd_nclass; n++) {
4058a8eb16c5SJoseph Koshy 			printf(" %s/%d/0x%b",
4059c5153e19SJoseph Koshy 			    pmc_name_of_pmcclass[md->pmd_classes[n].pm_class],
4060a8eb16c5SJoseph Koshy 			    md->pmd_nclasspmcs[n],
4061a8eb16c5SJoseph Koshy 			    md->pmd_classes[n].pm_caps,
4062a8eb16c5SJoseph Koshy 			    "\20"
4063a8eb16c5SJoseph Koshy 			    "\1INT\2USR\3SYS\4EDG\5THR"
4064a8eb16c5SJoseph Koshy 			    "\6REA\7WRI\10INV\11QUA\12PRC"
4065a8eb16c5SJoseph Koshy 			    "\13TAG\14CSC");
4066a8eb16c5SJoseph Koshy 		}
4067ebccf1e3SJoseph Koshy 		printf("\n");
4068ebccf1e3SJoseph Koshy 	}
4069ebccf1e3SJoseph Koshy 
4070ebccf1e3SJoseph Koshy 	return error;
4071ebccf1e3SJoseph Koshy }
4072ebccf1e3SJoseph Koshy 
4073ebccf1e3SJoseph Koshy /* prepare to be unloaded */
4074ebccf1e3SJoseph Koshy static void
4075ebccf1e3SJoseph Koshy pmc_cleanup(void)
4076ebccf1e3SJoseph Koshy {
4077ebccf1e3SJoseph Koshy 	int cpu;
4078ebccf1e3SJoseph Koshy 	struct pmc_ownerhash *ph;
4079ebccf1e3SJoseph Koshy 	struct pmc_owner *po, *tmp;
4080ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
4081342ed5d9SRuslan Ermilov #ifdef	DEBUG
4082ebccf1e3SJoseph Koshy 	struct pmc_processhash *prh;
4083ebccf1e3SJoseph Koshy #endif
4084ebccf1e3SJoseph Koshy 
4085ebccf1e3SJoseph Koshy 	PMCDBG(MOD,INI,0, "%s", "cleanup");
4086ebccf1e3SJoseph Koshy 
4087f263522aSJoseph Koshy 	/* switch off sampling */
4088f263522aSJoseph Koshy 	atomic_store_rel_int(&pmc_cpumask, 0);
4089f263522aSJoseph Koshy 	pmc_intr = NULL;
4090ebccf1e3SJoseph Koshy 
4091ebccf1e3SJoseph Koshy 	sx_xlock(&pmc_sx);
4092ebccf1e3SJoseph Koshy 	if (pmc_hook == NULL) {	/* being unloaded already */
4093ebccf1e3SJoseph Koshy 		sx_xunlock(&pmc_sx);
4094ebccf1e3SJoseph Koshy 		return;
4095ebccf1e3SJoseph Koshy 	}
4096ebccf1e3SJoseph Koshy 
4097ebccf1e3SJoseph Koshy 	pmc_hook = NULL; /* prevent new threads from entering module */
4098ebccf1e3SJoseph Koshy 
4099ebccf1e3SJoseph Koshy 	/* deregister event handlers */
4100ebccf1e3SJoseph Koshy 	EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
4101ebccf1e3SJoseph Koshy 	EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
4102ebccf1e3SJoseph Koshy 
4103ebccf1e3SJoseph Koshy 	/* send SIGBUS to all owner threads, free up allocations */
4104ebccf1e3SJoseph Koshy 	if (pmc_ownerhash)
4105ebccf1e3SJoseph Koshy 		for (ph = pmc_ownerhash;
4106ebccf1e3SJoseph Koshy 		     ph <= &pmc_ownerhash[pmc_ownerhashmask];
4107ebccf1e3SJoseph Koshy 		     ph++) {
4108ebccf1e3SJoseph Koshy 			LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
4109ebccf1e3SJoseph Koshy 				pmc_remove_owner(po);
4110ebccf1e3SJoseph Koshy 
4111ebccf1e3SJoseph Koshy 				/* send SIGBUS to owner processes */
4112ebccf1e3SJoseph Koshy 				PMCDBG(MOD,INI,2, "cleanup signal proc=%p "
4113ebccf1e3SJoseph Koshy 				    "(%d, %s)", po->po_owner,
4114ebccf1e3SJoseph Koshy 				    po->po_owner->p_pid,
4115ebccf1e3SJoseph Koshy 				    po->po_owner->p_comm);
4116ebccf1e3SJoseph Koshy 
4117ebccf1e3SJoseph Koshy 				PROC_LOCK(po->po_owner);
4118ebccf1e3SJoseph Koshy 				psignal(po->po_owner, SIGBUS);
4119ebccf1e3SJoseph Koshy 				PROC_UNLOCK(po->po_owner);
4120f263522aSJoseph Koshy 
4121f263522aSJoseph Koshy 				pmc_destroy_owner_descriptor(po);
4122ebccf1e3SJoseph Koshy 			}
4123ebccf1e3SJoseph Koshy 		}
4124ebccf1e3SJoseph Koshy 
4125ebccf1e3SJoseph Koshy 	/* reclaim allocated data structures */
4126ebccf1e3SJoseph Koshy 	if (pmc_mtxpool)
4127ebccf1e3SJoseph Koshy 		mtx_pool_destroy(&pmc_mtxpool);
4128ebccf1e3SJoseph Koshy 
4129ebccf1e3SJoseph Koshy 	mtx_destroy(&pmc_processhash_mtx);
4130ebccf1e3SJoseph Koshy 	if (pmc_processhash) {
4131342ed5d9SRuslan Ermilov #ifdef	DEBUG
4132ebccf1e3SJoseph Koshy 		struct pmc_process *pp;
4133ebccf1e3SJoseph Koshy 
4134ebccf1e3SJoseph Koshy 		PMCDBG(MOD,INI,3, "%s", "destroy process hash");
4135ebccf1e3SJoseph Koshy 		for (prh = pmc_processhash;
4136ebccf1e3SJoseph Koshy 		     prh <= &pmc_processhash[pmc_processhashmask];
4137ebccf1e3SJoseph Koshy 		     prh++)
4138ebccf1e3SJoseph Koshy 			LIST_FOREACH(pp, prh, pp_next)
4139ebccf1e3SJoseph Koshy 			    PMCDBG(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
4140ebccf1e3SJoseph Koshy #endif
4141ebccf1e3SJoseph Koshy 
4142ebccf1e3SJoseph Koshy 		hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
4143ebccf1e3SJoseph Koshy 		pmc_processhash = NULL;
4144ebccf1e3SJoseph Koshy 	}
4145ebccf1e3SJoseph Koshy 
4146ebccf1e3SJoseph Koshy 	if (pmc_ownerhash) {
4147ebccf1e3SJoseph Koshy 		PMCDBG(MOD,INI,3, "%s", "destroy owner hash");
4148ebccf1e3SJoseph Koshy 		hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
4149ebccf1e3SJoseph Koshy 		pmc_ownerhash = NULL;
4150ebccf1e3SJoseph Koshy 	}
4151ebccf1e3SJoseph Koshy 
4152f263522aSJoseph Koshy 	KASSERT(LIST_EMPTY(&pmc_ss_owners),
4153f263522aSJoseph Koshy 	    ("[pmc,%d] Global SS owner list not empty", __LINE__));
4154f263522aSJoseph Koshy 	KASSERT(pmc_ss_count == 0,
4155f263522aSJoseph Koshy 	    ("[pmc,%d] Global SS count not empty", __LINE__));
4156f263522aSJoseph Koshy 
4157ba8ae2deSJoseph Koshy 	/* free the per-cpu sample buffers */
4158ba8ae2deSJoseph Koshy 	for (cpu = 0; cpu < mp_ncpus; cpu++) {
4159ba8ae2deSJoseph Koshy 		if (pmc_cpu_is_disabled(cpu))
4160ba8ae2deSJoseph Koshy 			continue;
4161ba8ae2deSJoseph Koshy 		KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
4162ba8ae2deSJoseph Koshy 		    ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
4163ba8ae2deSJoseph Koshy 			cpu));
4164ba8ae2deSJoseph Koshy 		FREE(pmc_pcpu[cpu]->pc_sb, M_PMC);
4165ba8ae2deSJoseph Koshy 		pmc_pcpu[cpu]->pc_sb = NULL;
4166ba8ae2deSJoseph Koshy 	}
4167ba8ae2deSJoseph Koshy 
4168ebccf1e3SJoseph Koshy  	/* do processor dependent cleanup */
4169ebccf1e3SJoseph Koshy 	PMCDBG(MOD,INI,3, "%s", "md cleanup");
4170ebccf1e3SJoseph Koshy 	if (md) {
4171ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
4172ebccf1e3SJoseph Koshy 		for (cpu = 0; cpu < mp_ncpus; cpu++) {
4173ebccf1e3SJoseph Koshy 			PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
4174ebccf1e3SJoseph Koshy 			    cpu, pmc_pcpu[cpu]);
4175ebccf1e3SJoseph Koshy 			if (pmc_cpu_is_disabled(cpu))
4176ebccf1e3SJoseph Koshy 				continue;
4177ebccf1e3SJoseph Koshy 			pmc_select_cpu(cpu);
4178ebccf1e3SJoseph Koshy 			if (pmc_pcpu[cpu])
4179ebccf1e3SJoseph Koshy 				(void) md->pmd_cleanup(cpu);
4180ebccf1e3SJoseph Koshy 		}
4181ebccf1e3SJoseph Koshy 		FREE(md, M_PMC);
4182ebccf1e3SJoseph Koshy 		md = NULL;
4183ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
4184ebccf1e3SJoseph Koshy 	}
4185ebccf1e3SJoseph Koshy 
4186ebccf1e3SJoseph Koshy 	/* deallocate per-cpu structures */
4187ebccf1e3SJoseph Koshy 	FREE(pmc_pcpu, M_PMC);
4188ebccf1e3SJoseph Koshy 	pmc_pcpu = NULL;
4189ebccf1e3SJoseph Koshy 
4190ebccf1e3SJoseph Koshy 	FREE(pmc_pcpu_saved, M_PMC);
4191ebccf1e3SJoseph Koshy 	pmc_pcpu_saved = NULL;
4192ebccf1e3SJoseph Koshy 
4193ebccf1e3SJoseph Koshy 	if (pmc_pmcdisp) {
4194ebccf1e3SJoseph Koshy 		FREE(pmc_pmcdisp, M_PMC);
4195ebccf1e3SJoseph Koshy 		pmc_pmcdisp = NULL;
4196ebccf1e3SJoseph Koshy 	}
4197ebccf1e3SJoseph Koshy 
4198f263522aSJoseph Koshy 	pmclog_shutdown();
4199f263522aSJoseph Koshy 
4200ebccf1e3SJoseph Koshy 	sx_xunlock(&pmc_sx); 	/* we are done */
4201ebccf1e3SJoseph Koshy }
4202ebccf1e3SJoseph Koshy 
4203ebccf1e3SJoseph Koshy /*
4204ebccf1e3SJoseph Koshy  * The function called at load/unload.
4205ebccf1e3SJoseph Koshy  */
4206ebccf1e3SJoseph Koshy 
4207ebccf1e3SJoseph Koshy static int
4208ebccf1e3SJoseph Koshy load (struct module *module __unused, int cmd, void *arg __unused)
4209ebccf1e3SJoseph Koshy {
4210ebccf1e3SJoseph Koshy 	int error;
4211ebccf1e3SJoseph Koshy 
4212ebccf1e3SJoseph Koshy 	error = 0;
4213ebccf1e3SJoseph Koshy 
4214ebccf1e3SJoseph Koshy 	switch (cmd) {
4215ebccf1e3SJoseph Koshy 	case MOD_LOAD :
4216ebccf1e3SJoseph Koshy 		/* initialize the subsystem */
4217ebccf1e3SJoseph Koshy 		error = pmc_initialize();
4218ebccf1e3SJoseph Koshy 		if (error != 0)
4219ebccf1e3SJoseph Koshy 			break;
4220ebccf1e3SJoseph Koshy 		PMCDBG(MOD,INI,1, "syscall=%d ncpus=%d",
4221ebccf1e3SJoseph Koshy 		    pmc_syscall_num, mp_ncpus);
4222ebccf1e3SJoseph Koshy 		break;
4223ebccf1e3SJoseph Koshy 
4224ebccf1e3SJoseph Koshy 
4225ebccf1e3SJoseph Koshy 	case MOD_UNLOAD :
4226ebccf1e3SJoseph Koshy 	case MOD_SHUTDOWN:
4227ebccf1e3SJoseph Koshy 		pmc_cleanup();
4228ebccf1e3SJoseph Koshy 		PMCDBG(MOD,INI,1, "%s", "unloaded");
4229ebccf1e3SJoseph Koshy 		break;
4230ebccf1e3SJoseph Koshy 
4231ebccf1e3SJoseph Koshy 	default :
4232ebccf1e3SJoseph Koshy 		error = EINVAL;	/* XXX should panic(9) */
4233ebccf1e3SJoseph Koshy 		break;
4234ebccf1e3SJoseph Koshy 	}
4235ebccf1e3SJoseph Koshy 
4236ebccf1e3SJoseph Koshy 	return error;
4237ebccf1e3SJoseph Koshy }
4238ebccf1e3SJoseph Koshy 
4239ebccf1e3SJoseph Koshy /* memory pool */
4240ebccf1e3SJoseph Koshy MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module");
4241