xref: /freebsd/sys/dev/hwpmc/hwpmc_mod.c (revision a1b5a8955e828f049b425b7b2141999823059923)
1 /*-
2  * Copyright (c) 2003-2008 Joseph Koshy
3  * Copyright (c) 2007 The FreeBSD Foundation
4  * All rights reserved.
5  *
6  * Portions of this software were developed by A. Joseph Koshy under
7  * sponsorship from the FreeBSD Foundation and Google, Inc.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  */
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include <sys/param.h>
36 #include <sys/eventhandler.h>
37 #include <sys/jail.h>
38 #include <sys/kernel.h>
39 #include <sys/kthread.h>
40 #include <sys/limits.h>
41 #include <sys/lock.h>
42 #include <sys/malloc.h>
43 #include <sys/module.h>
44 #include <sys/mutex.h>
45 #include <sys/pmc.h>
46 #include <sys/pmckern.h>
47 #include <sys/pmclog.h>
48 #include <sys/priv.h>
49 #include <sys/proc.h>
50 #include <sys/queue.h>
51 #include <sys/resourcevar.h>
52 #include <sys/sched.h>
53 #include <sys/signalvar.h>
54 #include <sys/smp.h>
55 #include <sys/sx.h>
56 #include <sys/sysctl.h>
57 #include <sys/sysent.h>
58 #include <sys/systm.h>
59 #include <sys/vnode.h>
60 
61 #include <sys/linker.h>		/* needs to be after <sys/malloc.h> */
62 
63 #include <machine/atomic.h>
64 #include <machine/md_var.h>
65 
66 /*
67  * Types
68  */
69 
70 enum pmc_flags {
71 	PMC_FLAG_NONE	  = 0x00, /* do nothing */
72 	PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
73 	PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
74 };
75 
76 /*
77  * The offset in sysent where the syscall is allocated.
78  */
79 
80 static int pmc_syscall_num = NO_SYSCALL;
81 struct pmc_cpu		**pmc_pcpu;	 /* per-cpu state */
82 pmc_value_t		*pmc_pcpu_saved; /* saved PMC values: CSW handling */
83 
84 #define	PMC_PCPU_SAVED(C,R)	pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
85 
86 struct mtx_pool		*pmc_mtxpool;
87 static int		*pmc_pmcdisp;	 /* PMC row dispositions */
88 
89 #define	PMC_ROW_DISP_IS_FREE(R)		(pmc_pmcdisp[(R)] == 0)
90 #define	PMC_ROW_DISP_IS_THREAD(R)	(pmc_pmcdisp[(R)] > 0)
91 #define	PMC_ROW_DISP_IS_STANDALONE(R)	(pmc_pmcdisp[(R)] < 0)
92 
93 #define	PMC_MARK_ROW_FREE(R) do {					  \
94 	pmc_pmcdisp[(R)] = 0;						  \
95 } while (0)
96 
97 #define	PMC_MARK_ROW_STANDALONE(R) do {					  \
98 	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
99 		    __LINE__));						  \
100 	atomic_add_int(&pmc_pmcdisp[(R)], -1);				  \
101 	KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()),		  \
102 		("[pmc,%d] row disposition error", __LINE__));		  \
103 } while (0)
104 
105 #define	PMC_UNMARK_ROW_STANDALONE(R) do { 				  \
106 	atomic_add_int(&pmc_pmcdisp[(R)], 1);				  \
107 	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
108 		    __LINE__));						  \
109 } while (0)
110 
111 #define	PMC_MARK_ROW_THREAD(R) do {					  \
112 	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
113 		    __LINE__));						  \
114 	atomic_add_int(&pmc_pmcdisp[(R)], 1);				  \
115 } while (0)
116 
117 #define	PMC_UNMARK_ROW_THREAD(R) do {					  \
118 	atomic_add_int(&pmc_pmcdisp[(R)], -1);				  \
119 	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
120 		    __LINE__));						  \
121 } while (0)
122 
123 
124 /* various event handlers */
125 static eventhandler_tag	pmc_exit_tag, pmc_fork_tag;
126 
127 /* Module statistics */
128 struct pmc_op_getdriverstats pmc_stats;
129 
130 /* Machine/processor dependent operations */
131 struct pmc_mdep  *md;
132 
133 /*
134  * Hash tables mapping owner processes and target threads to PMCs.
135  */
136 
137 struct mtx pmc_processhash_mtx;		/* spin mutex */
138 static u_long pmc_processhashmask;
139 static LIST_HEAD(pmc_processhash, pmc_process)	*pmc_processhash;
140 
141 /*
142  * Hash table of PMC owner descriptors.  This table is protected by
143  * the shared PMC "sx" lock.
144  */
145 
146 static u_long pmc_ownerhashmask;
147 static LIST_HEAD(pmc_ownerhash, pmc_owner)	*pmc_ownerhash;
148 
149 /*
150  * List of PMC owners with system-wide sampling PMCs.
151  */
152 
153 static LIST_HEAD(, pmc_owner)			pmc_ss_owners;
154 
155 
156 /*
157  * Prototypes
158  */
159 
160 #ifdef	DEBUG
161 static int	pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
162 static int	pmc_debugflags_parse(char *newstr, char *fence);
163 #endif
164 
165 static int	load(struct module *module, int cmd, void *arg);
166 static int	pmc_attach_process(struct proc *p, struct pmc *pm);
167 static struct pmc *pmc_allocate_pmc_descriptor(void);
168 static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
169 static int	pmc_attach_one_process(struct proc *p, struct pmc *pm);
170 static int	pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
171     int cpu);
172 static int	pmc_can_attach(struct pmc *pm, struct proc *p);
173 static void	pmc_capture_user_callchain(int cpu, struct trapframe *tf);
174 static void	pmc_cleanup(void);
175 static int	pmc_detach_process(struct proc *p, struct pmc *pm);
176 static int	pmc_detach_one_process(struct proc *p, struct pmc *pm,
177     int flags);
178 static void	pmc_destroy_owner_descriptor(struct pmc_owner *po);
179 static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
180 static int	pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
181 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
182     pmc_id_t pmc);
183 static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
184     uint32_t mode);
185 static void	pmc_force_context_switch(void);
186 static void	pmc_link_target_process(struct pmc *pm,
187     struct pmc_process *pp);
188 static void	pmc_log_all_process_mappings(struct pmc_owner *po);
189 static void	pmc_log_kernel_mappings(struct pmc *pm);
190 static void	pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
191 static void	pmc_maybe_remove_owner(struct pmc_owner *po);
192 static void	pmc_process_csw_in(struct thread *td);
193 static void	pmc_process_csw_out(struct thread *td);
194 static void	pmc_process_exit(void *arg, struct proc *p);
195 static void	pmc_process_fork(void *arg, struct proc *p1,
196     struct proc *p2, int n);
197 static void	pmc_process_samples(int cpu);
198 static void	pmc_release_pmc_descriptor(struct pmc *pmc);
199 static void	pmc_remove_owner(struct pmc_owner *po);
200 static void	pmc_remove_process_descriptor(struct pmc_process *pp);
201 static void	pmc_restore_cpu_binding(struct pmc_binding *pb);
202 static void	pmc_save_cpu_binding(struct pmc_binding *pb);
203 static void	pmc_select_cpu(int cpu);
204 static int	pmc_start(struct pmc *pm);
205 static int	pmc_stop(struct pmc *pm);
206 static int	pmc_syscall_handler(struct thread *td, void *syscall_args);
207 static void	pmc_unlink_target_process(struct pmc *pmc,
208     struct pmc_process *pp);
209 
210 /*
211  * Kernel tunables and sysctl(8) interface.
212  */
213 
214 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
215 
216 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
217 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "callchaindepth", &pmc_callchaindepth);
218 SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_TUN|CTLFLAG_RD,
219     &pmc_callchaindepth, 0, "depth of call chain records");
220 
221 #ifdef	DEBUG
222 struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
223 char	pmc_debugstr[PMC_DEBUG_STRSIZE];
224 TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
225     sizeof(pmc_debugstr));
226 SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
227     CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN,
228     0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
229 #endif
230 
231 /*
232  * kern.hwpmc.hashrows -- determines the number of rows in the
233  * of the hash table used to look up threads
234  */
235 
236 static int pmc_hashsize = PMC_HASH_SIZE;
237 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize);
238 SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD,
239     &pmc_hashsize, 0, "rows in hash tables");
240 
241 /*
242  * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
243  */
244 
245 static int pmc_nsamples = PMC_NSAMPLES;
246 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples);
247 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD,
248     &pmc_nsamples, 0, "number of PC samples per CPU");
249 
250 
251 /*
252  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
253  */
254 
255 static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
256 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size);
257 SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD,
258     &pmc_mtxpool_size, 0, "size of spin mutex pool");
259 
260 
261 /*
262  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
263  * allocate system-wide PMCs.
264  *
265  * Allowing unprivileged processes to allocate system PMCs is convenient
266  * if system-wide measurements need to be taken concurrently with other
267  * per-process measurements.  This feature is turned off by default.
268  */
269 
270 static int pmc_unprivileged_syspmcs = 0;
271 TUNABLE_INT("security.bsd.unprivileged_syspmcs", &pmc_unprivileged_syspmcs);
272 SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RW,
273     &pmc_unprivileged_syspmcs, 0,
274     "allow unprivileged process to allocate system PMCs");
275 
276 /*
277  * Hash function.  Discard the lower 2 bits of the pointer since
278  * these are always zero for our uses.  The hash multiplier is
279  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
280  */
281 
282 #if	LONG_BIT == 64
283 #define	_PMC_HM		11400714819323198486u
284 #elif	LONG_BIT == 32
285 #define	_PMC_HM		2654435769u
286 #else
287 #error 	Must know the size of 'long' to compile
288 #endif
289 
290 #define	PMC_HASH_PTR(P,M)	((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
291 
292 /*
293  * Syscall structures
294  */
295 
296 /* The `sysent' for the new syscall */
297 static struct sysent pmc_sysent = {
298 	2,			/* sy_narg */
299 	pmc_syscall_handler	/* sy_call */
300 };
301 
302 static struct syscall_module_data pmc_syscall_mod = {
303 	load,
304 	NULL,
305 	&pmc_syscall_num,
306 	&pmc_sysent,
307 	{ 0, NULL }
308 };
309 
310 static moduledata_t pmc_mod = {
311 	PMC_MODULE_NAME,
312 	syscall_module_handler,
313 	&pmc_syscall_mod
314 };
315 
316 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
317 MODULE_VERSION(pmc, PMC_VERSION);
318 
319 #ifdef	DEBUG
320 enum pmc_dbgparse_state {
321 	PMCDS_WS,		/* in whitespace */
322 	PMCDS_MAJOR,		/* seen a major keyword */
323 	PMCDS_MINOR
324 };
325 
326 static int
327 pmc_debugflags_parse(char *newstr, char *fence)
328 {
329 	char c, *p, *q;
330 	struct pmc_debugflags *tmpflags;
331 	int error, found, *newbits, tmp;
332 	size_t kwlen;
333 
334 	tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
335 
336 	p = newstr;
337 	error = 0;
338 
339 	for (; p < fence && (c = *p); p++) {
340 
341 		/* skip white space */
342 		if (c == ' ' || c == '\t')
343 			continue;
344 
345 		/* look for a keyword followed by "=" */
346 		for (q = p; p < fence && (c = *p) && c != '='; p++)
347 			;
348 		if (c != '=') {
349 			error = EINVAL;
350 			goto done;
351 		}
352 
353 		kwlen = p - q;
354 		newbits = NULL;
355 
356 		/* lookup flag group name */
357 #define	DBG_SET_FLAG_MAJ(S,F)						\
358 		if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)	\
359 			newbits = &tmpflags->pdb_ ## F;
360 
361 		DBG_SET_FLAG_MAJ("cpu",		CPU);
362 		DBG_SET_FLAG_MAJ("csw",		CSW);
363 		DBG_SET_FLAG_MAJ("logging",	LOG);
364 		DBG_SET_FLAG_MAJ("module",	MOD);
365 		DBG_SET_FLAG_MAJ("md", 		MDP);
366 		DBG_SET_FLAG_MAJ("owner",	OWN);
367 		DBG_SET_FLAG_MAJ("pmc",		PMC);
368 		DBG_SET_FLAG_MAJ("process",	PRC);
369 		DBG_SET_FLAG_MAJ("sampling", 	SAM);
370 
371 		if (newbits == NULL) {
372 			error = EINVAL;
373 			goto done;
374 		}
375 
376 		p++;		/* skip the '=' */
377 
378 		/* Now parse the individual flags */
379 		tmp = 0;
380 	newflag:
381 		for (q = p; p < fence && (c = *p); p++)
382 			if (c == ' ' || c == '\t' || c == ',')
383 				break;
384 
385 		/* p == fence or c == ws or c == "," or c == 0 */
386 
387 		if ((kwlen = p - q) == 0) {
388 			*newbits = tmp;
389 			continue;
390 		}
391 
392 		found = 0;
393 #define	DBG_SET_FLAG_MIN(S,F)						\
394 		if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)	\
395 			tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
396 
397 		/* a '*' denotes all possible flags in the group */
398 		if (kwlen == 1 && *q == '*')
399 			tmp = found = ~0;
400 		/* look for individual flag names */
401 		DBG_SET_FLAG_MIN("allocaterow", ALR);
402 		DBG_SET_FLAG_MIN("allocate",	ALL);
403 		DBG_SET_FLAG_MIN("attach",	ATT);
404 		DBG_SET_FLAG_MIN("bind",	BND);
405 		DBG_SET_FLAG_MIN("config",	CFG);
406 		DBG_SET_FLAG_MIN("exec",	EXC);
407 		DBG_SET_FLAG_MIN("exit",	EXT);
408 		DBG_SET_FLAG_MIN("find",	FND);
409 		DBG_SET_FLAG_MIN("flush",	FLS);
410 		DBG_SET_FLAG_MIN("fork",	FRK);
411 		DBG_SET_FLAG_MIN("getbuf",	GTB);
412 		DBG_SET_FLAG_MIN("hook",	PMH);
413 		DBG_SET_FLAG_MIN("init",	INI);
414 		DBG_SET_FLAG_MIN("intr",	INT);
415 		DBG_SET_FLAG_MIN("linktarget",	TLK);
416 		DBG_SET_FLAG_MIN("mayberemove", OMR);
417 		DBG_SET_FLAG_MIN("ops",		OPS);
418 		DBG_SET_FLAG_MIN("read",	REA);
419 		DBG_SET_FLAG_MIN("register",	REG);
420 		DBG_SET_FLAG_MIN("release",	REL);
421 		DBG_SET_FLAG_MIN("remove",	ORM);
422 		DBG_SET_FLAG_MIN("sample",	SAM);
423 		DBG_SET_FLAG_MIN("scheduleio",	SIO);
424 		DBG_SET_FLAG_MIN("select",	SEL);
425 		DBG_SET_FLAG_MIN("signal",	SIG);
426 		DBG_SET_FLAG_MIN("swi",		SWI);
427 		DBG_SET_FLAG_MIN("swo",		SWO);
428 		DBG_SET_FLAG_MIN("start",	STA);
429 		DBG_SET_FLAG_MIN("stop",	STO);
430 		DBG_SET_FLAG_MIN("syscall",	PMS);
431 		DBG_SET_FLAG_MIN("unlinktarget", TUL);
432 		DBG_SET_FLAG_MIN("write",	WRI);
433 		if (found == 0) {
434 			/* unrecognized flag name */
435 			error = EINVAL;
436 			goto done;
437 		}
438 
439 		if (c == 0 || c == ' ' || c == '\t') {	/* end of flag group */
440 			*newbits = tmp;
441 			continue;
442 		}
443 
444 		p++;
445 		goto newflag;
446 	}
447 
448 	/* save the new flag set */
449 	bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
450 
451  done:
452 	free(tmpflags, M_PMC);
453 	return error;
454 }
455 
456 static int
457 pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
458 {
459 	char *fence, *newstr;
460 	int error;
461 	unsigned int n;
462 
463 	(void) arg1; (void) arg2; /* unused parameters */
464 
465 	n = sizeof(pmc_debugstr);
466 	newstr = malloc(n, M_PMC, M_ZERO|M_WAITOK);
467 	(void) strlcpy(newstr, pmc_debugstr, n);
468 
469 	error = sysctl_handle_string(oidp, newstr, n, req);
470 
471 	/* if there is a new string, parse and copy it */
472 	if (error == 0 && req->newptr != NULL) {
473 		fence = newstr + (n < req->newlen ? n : req->newlen + 1);
474 		if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
475 			(void) strlcpy(pmc_debugstr, newstr,
476 			    sizeof(pmc_debugstr));
477 	}
478 
479 	free(newstr, M_PMC);
480 
481 	return error;
482 }
483 #endif
484 
485 /*
486  * Concurrency Control
487  *
488  * The driver manages the following data structures:
489  *
490  *   - target process descriptors, one per target process
491  *   - owner process descriptors (and attached lists), one per owner process
492  *   - lookup hash tables for owner and target processes
493  *   - PMC descriptors (and attached lists)
494  *   - per-cpu hardware state
495  *   - the 'hook' variable through which the kernel calls into
496  *     this module
497  *   - the machine hardware state (managed by the MD layer)
498  *
499  * These data structures are accessed from:
500  *
501  * - thread context-switch code
502  * - interrupt handlers (possibly on multiple cpus)
503  * - kernel threads on multiple cpus running on behalf of user
504  *   processes doing system calls
505  * - this driver's private kernel threads
506  *
507  * = Locks and Locking strategy =
508  *
509  * The driver uses four locking strategies for its operation:
510  *
511  * - The global SX lock "pmc_sx" is used to protect internal
512  *   data structures.
513  *
514  *   Calls into the module by syscall() start with this lock being
515  *   held in exclusive mode.  Depending on the requested operation,
516  *   the lock may be downgraded to 'shared' mode to allow more
517  *   concurrent readers into the module.  Calls into the module from
518  *   other parts of the kernel acquire the lock in shared mode.
519  *
520  *   This SX lock is held in exclusive mode for any operations that
521  *   modify the linkages between the driver's internal data structures.
522  *
523  *   The 'pmc_hook' function pointer is also protected by this lock.
524  *   It is only examined with the sx lock held in exclusive mode.  The
525  *   kernel module is allowed to be unloaded only with the sx lock held
526  *   in exclusive mode.  In normal syscall handling, after acquiring the
527  *   pmc_sx lock we first check that 'pmc_hook' is non-null before
528  *   proceeding.  This prevents races between the thread unloading the module
529  *   and other threads seeking to use the module.
530  *
531  * - Lookups of target process structures and owner process structures
532  *   cannot use the global "pmc_sx" SX lock because these lookups need
533  *   to happen during context switches and in other critical sections
534  *   where sleeping is not allowed.  We protect these lookup tables
535  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
536  *   "pmc_ownerhash_mtx".
537  *
538  * - Interrupt handlers work in a lock free manner.  At interrupt
539  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
540  *   when the PMC was started.  If this pointer is NULL, the interrupt
541  *   is ignored after updating driver statistics.  We ensure that this
542  *   pointer is set (using an atomic operation if necessary) before the
543  *   PMC hardware is started.  Conversely, this pointer is unset atomically
544  *   only after the PMC hardware is stopped.
545  *
546  *   We ensure that everything needed for the operation of an
547  *   interrupt handler is available without it needing to acquire any
548  *   locks.  We also ensure that a PMC's software state is destroyed only
549  *   after the PMC is taken off hardware (on all CPUs).
550  *
551  * - Context-switch handling with process-private PMCs needs more
552  *   care.
553  *
554  *   A given process may be the target of multiple PMCs.  For example,
555  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
556  *   while the target process is running on another.  A PMC could also
557  *   be getting released because its owner is exiting.  We tackle
558  *   these situations in the following manner:
559  *
560  *   - each target process structure 'pmc_process' has an array
561  *     of 'struct pmc *' pointers, one for each hardware PMC.
562  *
563  *   - At context switch IN time, each "target" PMC in RUNNING state
564  *     gets started on hardware and a pointer to each PMC is copied into
565  *     the per-cpu phw array.  The 'runcount' for the PMC is
566  *     incremented.
567  *
568  *   - At context switch OUT time, all process-virtual PMCs are stopped
569  *     on hardware.  The saved value is added to the PMCs value field
570  *     only if the PMC is in a non-deleted state (the PMCs state could
571  *     have changed during the current time slice).
572  *
573  *     Note that since in-between a switch IN on a processor and a switch
574  *     OUT, the PMC could have been released on another CPU.  Therefore
575  *     context switch OUT always looks at the hardware state to turn
576  *     OFF PMCs and will update a PMC's saved value only if reachable
577  *     from the target process record.
578  *
579  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
580  *     be attached to many processes at the time of the call and could
581  *     be active on multiple CPUs).
582  *
583  *     We prevent further scheduling of the PMC by marking it as in
584  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
585  *     this PMC is currently running on a CPU somewhere.  The thread
586  *     doing the PMCRELEASE operation waits by repeatedly doing a
587  *     pause() till the runcount comes to zero.
588  *
589  * The contents of a PMC descriptor (struct pmc) are protected using
590  * a spin-mutex.  In order to save space, we use a mutex pool.
591  *
592  * In terms of lock types used by witness(4), we use:
593  * - Type "pmc-sx", used by the global SX lock.
594  * - Type "pmc-sleep", for sleep mutexes used by logger threads.
595  * - Type "pmc-per-proc", for protecting PMC owner descriptors.
596  * - Type "pmc-leaf", used for all other spin mutexes.
597  */
598 
599 /*
600  * save the cpu binding of the current kthread
601  */
602 
603 static void
604 pmc_save_cpu_binding(struct pmc_binding *pb)
605 {
606 	PMCDBG(CPU,BND,2, "%s", "save-cpu");
607 	thread_lock(curthread);
608 	pb->pb_bound = sched_is_bound(curthread);
609 	pb->pb_cpu   = curthread->td_oncpu;
610 	thread_unlock(curthread);
611 	PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
612 }
613 
614 /*
615  * restore the cpu binding of the current thread
616  */
617 
618 static void
619 pmc_restore_cpu_binding(struct pmc_binding *pb)
620 {
621 	PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
622 	    curthread->td_oncpu, pb->pb_cpu);
623 	thread_lock(curthread);
624 	if (pb->pb_bound)
625 		sched_bind(curthread, pb->pb_cpu);
626 	else
627 		sched_unbind(curthread);
628 	thread_unlock(curthread);
629 	PMCDBG(CPU,BND,2, "%s", "restore-cpu done");
630 }
631 
632 /*
633  * move execution over the specified cpu and bind it there.
634  */
635 
636 static void
637 pmc_select_cpu(int cpu)
638 {
639 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
640 	    ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
641 
642 	/* Never move to an inactive CPU. */
643 	KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
644 	    "CPU %d", __LINE__, cpu));
645 
646 	PMCDBG(CPU,SEL,2, "select-cpu cpu=%d", cpu);
647 	thread_lock(curthread);
648 	sched_bind(curthread, cpu);
649 	thread_unlock(curthread);
650 
651 	KASSERT(curthread->td_oncpu == cpu,
652 	    ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
653 		cpu, curthread->td_oncpu));
654 
655 	PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
656 }
657 
658 /*
659  * Force a context switch.
660  *
661  * We do this by pause'ing for 1 tick -- invoking mi_switch() is not
662  * guaranteed to force a context switch.
663  */
664 
665 static void
666 pmc_force_context_switch(void)
667 {
668 
669 	pause("pmcctx", 1);
670 }
671 
672 /*
673  * Get the file name for an executable.  This is a simple wrapper
674  * around vn_fullpath(9).
675  */
676 
677 static void
678 pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
679 {
680 
681 	*fullpath = "unknown";
682 	*freepath = NULL;
683 	vn_fullpath(curthread, v, fullpath, freepath);
684 }
685 
686 /*
687  * remove an process owning PMCs
688  */
689 
690 void
691 pmc_remove_owner(struct pmc_owner *po)
692 {
693 	struct pmc *pm, *tmp;
694 
695 	sx_assert(&pmc_sx, SX_XLOCKED);
696 
697 	PMCDBG(OWN,ORM,1, "remove-owner po=%p", po);
698 
699 	/* Remove descriptor from the owner hash table */
700 	LIST_REMOVE(po, po_next);
701 
702 	/* release all owned PMC descriptors */
703 	LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
704 		PMCDBG(OWN,ORM,2, "pmc=%p", pm);
705 		KASSERT(pm->pm_owner == po,
706 		    ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
707 
708 		pmc_release_pmc_descriptor(pm);	/* will unlink from the list */
709 	}
710 
711 	KASSERT(po->po_sscount == 0,
712 	    ("[pmc,%d] SS count not zero", __LINE__));
713 	KASSERT(LIST_EMPTY(&po->po_pmcs),
714 	    ("[pmc,%d] PMC list not empty", __LINE__));
715 
716 	/* de-configure the log file if present */
717 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
718 		pmclog_deconfigure_log(po);
719 }
720 
721 /*
722  * remove an owner process record if all conditions are met.
723  */
724 
725 static void
726 pmc_maybe_remove_owner(struct pmc_owner *po)
727 {
728 
729 	PMCDBG(OWN,OMR,1, "maybe-remove-owner po=%p", po);
730 
731 	/*
732 	 * Remove owner record if
733 	 * - this process does not own any PMCs
734 	 * - this process has not allocated a system-wide sampling buffer
735 	 */
736 
737 	if (LIST_EMPTY(&po->po_pmcs) &&
738 	    ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
739 		pmc_remove_owner(po);
740 		pmc_destroy_owner_descriptor(po);
741 	}
742 }
743 
744 /*
745  * Add an association between a target process and a PMC.
746  */
747 
748 static void
749 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
750 {
751 	int ri;
752 	struct pmc_target *pt;
753 
754 	sx_assert(&pmc_sx, SX_XLOCKED);
755 
756 	KASSERT(pm != NULL && pp != NULL,
757 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
758 	KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
759 	    ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
760 		__LINE__, pm, pp->pp_proc->p_pid));
761 	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1),
762 	    ("[pmc,%d] Illegal reference count %d for process record %p",
763 		__LINE__, pp->pp_refcnt, (void *) pp));
764 
765 	ri = PMC_TO_ROWINDEX(pm);
766 
767 	PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
768 	    pm, ri, pp);
769 
770 #ifdef	DEBUG
771 	LIST_FOREACH(pt, &pm->pm_targets, pt_next)
772 	    if (pt->pt_process == pp)
773 		    KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
774 				__LINE__, pp, pm));
775 #endif
776 
777 	pt = malloc(sizeof(struct pmc_target), M_PMC, M_ZERO|M_WAITOK);
778 
779 	pt->pt_process = pp;
780 
781 	LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
782 
783 	atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
784 	    (uintptr_t)pm);
785 
786 	if (pm->pm_owner->po_owner == pp->pp_proc)
787 		pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
788 
789 	/*
790 	 * Initialize the per-process values at this row index.
791 	 */
792 	pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
793 	    pm->pm_sc.pm_reloadcount : 0;
794 
795 	pp->pp_refcnt++;
796 
797 }
798 
799 /*
800  * Removes the association between a target process and a PMC.
801  */
802 
803 static void
804 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
805 {
806 	int ri;
807 	struct proc *p;
808 	struct pmc_target *ptgt;
809 
810 	sx_assert(&pmc_sx, SX_XLOCKED);
811 
812 	KASSERT(pm != NULL && pp != NULL,
813 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
814 
815 	KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc,
816 	    ("[pmc,%d] Illegal ref count %d on process record %p",
817 		__LINE__, pp->pp_refcnt, (void *) pp));
818 
819 	ri = PMC_TO_ROWINDEX(pm);
820 
821 	PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
822 	    pm, ri, pp);
823 
824 	KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
825 	    ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
826 		ri, pm, pp->pp_pmcs[ri].pp_pmc));
827 
828 	pp->pp_pmcs[ri].pp_pmc = NULL;
829 	pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
830 
831 	/* Remove owner-specific flags */
832 	if (pm->pm_owner->po_owner == pp->pp_proc) {
833 		pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
834 		pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
835 	}
836 
837 	pp->pp_refcnt--;
838 
839 	/* Remove the target process from the PMC structure */
840 	LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
841 		if (ptgt->pt_process == pp)
842 			break;
843 
844 	KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
845 		    "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
846 
847 	LIST_REMOVE(ptgt, pt_next);
848 	free(ptgt, M_PMC);
849 
850 	/* if the PMC now lacks targets, send the owner a SIGIO */
851 	if (LIST_EMPTY(&pm->pm_targets)) {
852 		p = pm->pm_owner->po_owner;
853 		PROC_LOCK(p);
854 		psignal(p, SIGIO);
855 		PROC_UNLOCK(p);
856 
857 		PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p,
858 		    SIGIO);
859 	}
860 }
861 
862 /*
863  * Check if PMC 'pm' may be attached to target process 't'.
864  */
865 
866 static int
867 pmc_can_attach(struct pmc *pm, struct proc *t)
868 {
869 	struct proc *o;		/* pmc owner */
870 	struct ucred *oc, *tc;	/* owner, target credentials */
871 	int decline_attach, i;
872 
873 	/*
874 	 * A PMC's owner can always attach that PMC to itself.
875 	 */
876 
877 	if ((o = pm->pm_owner->po_owner) == t)
878 		return 0;
879 
880 	PROC_LOCK(o);
881 	oc = o->p_ucred;
882 	crhold(oc);
883 	PROC_UNLOCK(o);
884 
885 	PROC_LOCK(t);
886 	tc = t->p_ucred;
887 	crhold(tc);
888 	PROC_UNLOCK(t);
889 
890 	/*
891 	 * The effective uid of the PMC owner should match at least one
892 	 * of the {effective,real,saved} uids of the target process.
893 	 */
894 
895 	decline_attach = oc->cr_uid != tc->cr_uid &&
896 	    oc->cr_uid != tc->cr_svuid &&
897 	    oc->cr_uid != tc->cr_ruid;
898 
899 	/*
900 	 * Every one of the target's group ids, must be in the owner's
901 	 * group list.
902 	 */
903 	for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
904 		decline_attach = !groupmember(tc->cr_groups[i], oc);
905 
906 	/* check the read and saved gids too */
907 	if (decline_attach == 0)
908 		decline_attach = !groupmember(tc->cr_rgid, oc) ||
909 		    !groupmember(tc->cr_svgid, oc);
910 
911 	crfree(tc);
912 	crfree(oc);
913 
914 	return !decline_attach;
915 }
916 
917 /*
918  * Attach a process to a PMC.
919  */
920 
921 static int
922 pmc_attach_one_process(struct proc *p, struct pmc *pm)
923 {
924 	int ri;
925 	char *fullpath, *freepath;
926 	struct pmc_process	*pp;
927 
928 	sx_assert(&pmc_sx, SX_XLOCKED);
929 
930 	PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
931 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
932 
933 	/*
934 	 * Locate the process descriptor corresponding to process 'p',
935 	 * allocating space as needed.
936 	 *
937 	 * Verify that rowindex 'pm_rowindex' is free in the process
938 	 * descriptor.
939 	 *
940 	 * If not, allocate space for a descriptor and link the
941 	 * process descriptor and PMC.
942 	 */
943 	ri = PMC_TO_ROWINDEX(pm);
944 
945 	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
946 		return ENOMEM;
947 
948 	if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
949 		return EEXIST;
950 
951 	if (pp->pp_pmcs[ri].pp_pmc != NULL)
952 		return EBUSY;
953 
954 	pmc_link_target_process(pm, pp);
955 
956 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
957 	    (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
958 		pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
959 
960 	pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
961 
962 	/* issue an attach event to a configured log file */
963 	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
964 		pmc_getfilename(p->p_textvp, &fullpath, &freepath);
965 		if (p->p_flag & P_KTHREAD) {
966 			fullpath = kernelname;
967 			freepath = NULL;
968 		} else
969 			pmclog_process_pmcattach(pm, p->p_pid, fullpath);
970 		if (freepath)
971 			free(freepath, M_TEMP);
972 		if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
973 			pmc_log_process_mappings(pm->pm_owner, p);
974 	}
975 	/* mark process as using HWPMCs */
976 	PROC_LOCK(p);
977 	p->p_flag |= P_HWPMC;
978 	PROC_UNLOCK(p);
979 
980 	return 0;
981 }
982 
983 /*
984  * Attach a process and optionally its children
985  */
986 
987 static int
988 pmc_attach_process(struct proc *p, struct pmc *pm)
989 {
990 	int error;
991 	struct proc *top;
992 
993 	sx_assert(&pmc_sx, SX_XLOCKED);
994 
995 	PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
996 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
997 
998 
999 	/*
1000 	 * If this PMC successfully allowed a GETMSR operation
1001 	 * in the past, disallow further ATTACHes.
1002 	 */
1003 
1004 	if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
1005 		return EPERM;
1006 
1007 	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1008 		return pmc_attach_one_process(p, pm);
1009 
1010 	/*
1011 	 * Traverse all child processes, attaching them to
1012 	 * this PMC.
1013 	 */
1014 
1015 	sx_slock(&proctree_lock);
1016 
1017 	top = p;
1018 
1019 	for (;;) {
1020 		if ((error = pmc_attach_one_process(p, pm)) != 0)
1021 			break;
1022 		if (!LIST_EMPTY(&p->p_children))
1023 			p = LIST_FIRST(&p->p_children);
1024 		else for (;;) {
1025 			if (p == top)
1026 				goto done;
1027 			if (LIST_NEXT(p, p_sibling)) {
1028 				p = LIST_NEXT(p, p_sibling);
1029 				break;
1030 			}
1031 			p = p->p_pptr;
1032 		}
1033 	}
1034 
1035 	if (error)
1036 		(void) pmc_detach_process(top, pm);
1037 
1038  done:
1039 	sx_sunlock(&proctree_lock);
1040 	return error;
1041 }
1042 
1043 /*
1044  * Detach a process from a PMC.  If there are no other PMCs tracking
1045  * this process, remove the process structure from its hash table.  If
1046  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1047  */
1048 
1049 static int
1050 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1051 {
1052 	int ri;
1053 	struct pmc_process *pp;
1054 
1055 	sx_assert(&pmc_sx, SX_XLOCKED);
1056 
1057 	KASSERT(pm != NULL,
1058 	    ("[pmc,%d] null pm pointer", __LINE__));
1059 
1060 	ri = PMC_TO_ROWINDEX(pm);
1061 
1062 	PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1063 	    pm, ri, p, p->p_pid, p->p_comm, flags);
1064 
1065 	if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1066 		return ESRCH;
1067 
1068 	if (pp->pp_pmcs[ri].pp_pmc != pm)
1069 		return EINVAL;
1070 
1071 	pmc_unlink_target_process(pm, pp);
1072 
1073 	/* Issue a detach entry if a log file is configured */
1074 	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1075 		pmclog_process_pmcdetach(pm, p->p_pid);
1076 
1077 	/*
1078 	 * If there are no PMCs targetting this process, we remove its
1079 	 * descriptor from the target hash table and unset the P_HWPMC
1080 	 * flag in the struct proc.
1081 	 */
1082 	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc,
1083 	    ("[pmc,%d] Illegal refcnt %d for process struct %p",
1084 		__LINE__, pp->pp_refcnt, pp));
1085 
1086 	if (pp->pp_refcnt != 0)	/* still a target of some PMC */
1087 		return 0;
1088 
1089 	pmc_remove_process_descriptor(pp);
1090 
1091 	if (flags & PMC_FLAG_REMOVE)
1092 		free(pp, M_PMC);
1093 
1094 	PROC_LOCK(p);
1095 	p->p_flag &= ~P_HWPMC;
1096 	PROC_UNLOCK(p);
1097 
1098 	return 0;
1099 }
1100 
1101 /*
1102  * Detach a process and optionally its descendants from a PMC.
1103  */
1104 
1105 static int
1106 pmc_detach_process(struct proc *p, struct pmc *pm)
1107 {
1108 	struct proc *top;
1109 
1110 	sx_assert(&pmc_sx, SX_XLOCKED);
1111 
1112 	PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1113 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1114 
1115 	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1116 		return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1117 
1118 	/*
1119 	 * Traverse all children, detaching them from this PMC.  We
1120 	 * ignore errors since we could be detaching a PMC from a
1121 	 * partially attached proc tree.
1122 	 */
1123 
1124 	sx_slock(&proctree_lock);
1125 
1126 	top = p;
1127 
1128 	for (;;) {
1129 		(void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1130 
1131 		if (!LIST_EMPTY(&p->p_children))
1132 			p = LIST_FIRST(&p->p_children);
1133 		else for (;;) {
1134 			if (p == top)
1135 				goto done;
1136 			if (LIST_NEXT(p, p_sibling)) {
1137 				p = LIST_NEXT(p, p_sibling);
1138 				break;
1139 			}
1140 			p = p->p_pptr;
1141 		}
1142 	}
1143 
1144  done:
1145 	sx_sunlock(&proctree_lock);
1146 
1147 	if (LIST_EMPTY(&pm->pm_targets))
1148 		pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1149 
1150 	return 0;
1151 }
1152 
1153 
1154 /*
1155  * Thread context switch IN
1156  */
1157 
1158 static void
1159 pmc_process_csw_in(struct thread *td)
1160 {
1161 	int cpu;
1162 	unsigned int ri;
1163 	struct pmc *pm;
1164 	struct proc *p;
1165 	struct pmc_cpu *pc;
1166 	struct pmc_hw *phw;
1167 	struct pmc_process *pp;
1168 	pmc_value_t newvalue;
1169 
1170 	p = td->td_proc;
1171 
1172 	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1173 		return;
1174 
1175 	KASSERT(pp->pp_proc == td->td_proc,
1176 	    ("[pmc,%d] not my thread state", __LINE__));
1177 
1178 	critical_enter(); /* no preemption from this point */
1179 
1180 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1181 
1182 	PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1183 	    p->p_pid, p->p_comm, pp);
1184 
1185 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1186 	    ("[pmc,%d] wierd CPU id %d", __LINE__, cpu));
1187 
1188 	pc = pmc_pcpu[cpu];
1189 
1190 	for (ri = 0; ri < md->pmd_npmc; ri++) {
1191 
1192 		if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1193 			continue;
1194 
1195 		KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1196 		    ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1197 			__LINE__, PMC_TO_MODE(pm)));
1198 
1199 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1200 		    ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1201 			__LINE__, PMC_TO_ROWINDEX(pm), ri));
1202 
1203 		/*
1204 		 * Only PMCs that are marked as 'RUNNING' need
1205 		 * be placed on hardware.
1206 		 */
1207 
1208 		if (pm->pm_state != PMC_STATE_RUNNING)
1209 			continue;
1210 
1211 		/* increment PMC runcount */
1212 		atomic_add_rel_32(&pm->pm_runcount, 1);
1213 
1214 		/* configure the HWPMC we are going to use. */
1215 		md->pmd_config_pmc(cpu, ri, pm);
1216 
1217 		phw = pc->pc_hwpmcs[ri];
1218 
1219 		KASSERT(phw != NULL,
1220 		    ("[pmc,%d] null hw pointer", __LINE__));
1221 
1222 		KASSERT(phw->phw_pmc == pm,
1223 		    ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1224 			phw->phw_pmc, pm));
1225 
1226 		/*
1227 		 * Write out saved value and start the PMC.
1228 		 *
1229 		 * Sampling PMCs use a per-process value, while
1230 		 * counting mode PMCs use a per-pmc value that is
1231 		 * inherited across descendants.
1232 		 */
1233 		if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1234 			mtx_pool_lock_spin(pmc_mtxpool, pm);
1235 			newvalue = PMC_PCPU_SAVED(cpu,ri) =
1236 			    pp->pp_pmcs[ri].pp_pmcval;
1237 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
1238 		} else {
1239 			KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1240 			    ("[pmc,%d] illegal mode=%d", __LINE__,
1241 			    PMC_TO_MODE(pm)));
1242 			mtx_pool_lock_spin(pmc_mtxpool, pm);
1243 			newvalue = PMC_PCPU_SAVED(cpu, ri) =
1244 			    pm->pm_gv.pm_savedvalue;
1245 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
1246 		}
1247 
1248 		PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1249 
1250 		md->pmd_write_pmc(cpu, ri, newvalue);
1251 		md->pmd_start_pmc(cpu, ri);
1252 	}
1253 
1254 	/*
1255 	 * perform any other architecture/cpu dependent thread
1256 	 * switch-in actions.
1257 	 */
1258 
1259 	(void) (*md->pmd_switch_in)(pc, pp);
1260 
1261 	critical_exit();
1262 
1263 }
1264 
1265 /*
1266  * Thread context switch OUT.
1267  */
1268 
1269 static void
1270 pmc_process_csw_out(struct thread *td)
1271 {
1272 	int cpu;
1273 	enum pmc_mode mode;
1274 	unsigned int ri;
1275 	struct pmc *pm;
1276 	struct proc *p;
1277 	struct pmc_cpu *pc;
1278 	struct pmc_process *pp;
1279 	int64_t tmp;
1280 	pmc_value_t newvalue;
1281 
1282 	/*
1283 	 * Locate our process descriptor; this may be NULL if
1284 	 * this process is exiting and we have already removed
1285 	 * the process from the target process table.
1286 	 *
1287 	 * Note that due to kernel preemption, multiple
1288 	 * context switches may happen while the process is
1289 	 * exiting.
1290 	 *
1291 	 * Note also that if the target process cannot be
1292 	 * found we still need to deconfigure any PMCs that
1293 	 * are currently running on hardware.
1294 	 */
1295 
1296 	p = td->td_proc;
1297 	pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1298 
1299 	/*
1300 	 * save PMCs
1301 	 */
1302 
1303 	critical_enter();
1304 
1305 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1306 
1307 	PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1308 	    p->p_pid, p->p_comm, pp);
1309 
1310 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1311 	    ("[pmc,%d wierd CPU id %d", __LINE__, cpu));
1312 
1313 	pc = pmc_pcpu[cpu];
1314 
1315 	/*
1316 	 * When a PMC gets unlinked from a target PMC, it will
1317 	 * be removed from the target's pp_pmc[] array.
1318 	 *
1319 	 * However, on a MP system, the target could have been
1320 	 * executing on another CPU at the time of the unlink.
1321 	 * So, at context switch OUT time, we need to look at
1322 	 * the hardware to determine if a PMC is scheduled on
1323 	 * it.
1324 	 */
1325 
1326 	for (ri = 0; ri < md->pmd_npmc; ri++) {
1327 
1328 		pm = NULL;
1329 		(void) (*md->pmd_get_config)(cpu, ri, &pm);
1330 
1331 		if (pm == NULL)	/* nothing at this row index */
1332 			continue;
1333 
1334 		mode = PMC_TO_MODE(pm);
1335 		if (!PMC_IS_VIRTUAL_MODE(mode))
1336 			continue; /* not a process virtual PMC */
1337 
1338 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1339 		    ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1340 			__LINE__, PMC_TO_ROWINDEX(pm), ri));
1341 
1342 		/* Stop hardware if not already stopped */
1343 		if (pm->pm_stalled == 0)
1344 			md->pmd_stop_pmc(cpu, ri);
1345 
1346 		/* reduce this PMC's runcount */
1347 		atomic_subtract_rel_32(&pm->pm_runcount, 1);
1348 
1349 		/*
1350 		 * If this PMC is associated with this process,
1351 		 * save the reading.
1352 		 */
1353 
1354 		if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {
1355 
1356 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1357 			    ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1358 				pm, ri, pp->pp_pmcs[ri].pp_pmc));
1359 
1360 			KASSERT(pp->pp_refcnt > 0,
1361 			    ("[pmc,%d] pp refcnt = %d", __LINE__,
1362 				pp->pp_refcnt));
1363 
1364 			md->pmd_read_pmc(cpu, ri, &newvalue);
1365 
1366 			tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1367 
1368 			PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
1369 			    tmp);
1370 
1371 			if (mode == PMC_MODE_TS) {
1372 
1373 				/*
1374 				 * For sampling process-virtual PMCs,
1375 				 * we expect the count to be
1376 				 * decreasing as the 'value'
1377 				 * programmed into the PMC is the
1378 				 * number of events to be seen till
1379 				 * the next sampling interrupt.
1380 				 */
1381 				if (tmp < 0)
1382 					tmp += pm->pm_sc.pm_reloadcount;
1383 				mtx_pool_lock_spin(pmc_mtxpool, pm);
1384 				pp->pp_pmcs[ri].pp_pmcval -= tmp;
1385 				if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0)
1386 					pp->pp_pmcs[ri].pp_pmcval +=
1387 					    pm->pm_sc.pm_reloadcount;
1388 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
1389 
1390 			} else {
1391 
1392 				/*
1393 				 * For counting process-virtual PMCs,
1394 				 * we expect the count to be
1395 				 * increasing monotonically, modulo a 64
1396 				 * bit wraparound.
1397 				 */
1398 				KASSERT((int64_t) tmp >= 0,
1399 				    ("[pmc,%d] negative increment cpu=%d "
1400 				     "ri=%d newvalue=%jx saved=%jx "
1401 				     "incr=%jx", __LINE__, cpu, ri,
1402 				     newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1403 
1404 				mtx_pool_lock_spin(pmc_mtxpool, pm);
1405 				pm->pm_gv.pm_savedvalue += tmp;
1406 				pp->pp_pmcs[ri].pp_pmcval += tmp;
1407 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
1408 
1409 				if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1410 					pmclog_process_proccsw(pm, pp, tmp);
1411 			}
1412 		}
1413 
1414 		/* mark hardware as free */
1415 		md->pmd_config_pmc(cpu, ri, NULL);
1416 	}
1417 
1418 	/*
1419 	 * perform any other architecture/cpu dependent thread
1420 	 * switch out functions.
1421 	 */
1422 
1423 	(void) (*md->pmd_switch_out)(pc, pp);
1424 
1425 	critical_exit();
1426 }
1427 
1428 /*
1429  * Log a KLD operation.
1430  */
1431 
1432 static void
1433 pmc_process_kld_load(struct pmckern_map_in *pkm)
1434 {
1435 	struct pmc_owner *po;
1436 
1437 	sx_assert(&pmc_sx, SX_LOCKED);
1438 
1439 	/*
1440 	 * Notify owners of system sampling PMCs about KLD operations.
1441 	 */
1442 
1443 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1444 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1445 	    	pmclog_process_map_in(po, (pid_t) -1, pkm->pm_address,
1446 		    (char *) pkm->pm_file);
1447 
1448 	/*
1449 	 * TODO: Notify owners of (all) process-sampling PMCs too.
1450 	 */
1451 
1452 	return;
1453 }
1454 
1455 static void
1456 pmc_process_kld_unload(struct pmckern_map_out *pkm)
1457 {
1458 	struct pmc_owner *po;
1459 
1460 	sx_assert(&pmc_sx, SX_LOCKED);
1461 
1462 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1463 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1464 		pmclog_process_map_out(po, (pid_t) -1,
1465 		    pkm->pm_address, pkm->pm_address + pkm->pm_size);
1466 
1467 	/*
1468 	 * TODO: Notify owners of process-sampling PMCs.
1469 	 */
1470 }
1471 
1472 /*
1473  * A mapping change for a process.
1474  */
1475 
1476 static void
1477 pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
1478 {
1479 	int ri;
1480 	pid_t pid;
1481 	char *fullpath, *freepath;
1482 	const struct pmc *pm;
1483 	struct pmc_owner *po;
1484 	const struct pmc_process *pp;
1485 
1486 	freepath = fullpath = NULL;
1487 	pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
1488 
1489 	pid = td->td_proc->p_pid;
1490 
1491 	/* Inform owners of all system-wide sampling PMCs. */
1492 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1493 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1494 		pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
1495 
1496 	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1497 		goto done;
1498 
1499 	/*
1500 	 * Inform sampling PMC owners tracking this process.
1501 	 */
1502 	for (ri = 0; ri < md->pmd_npmc; ri++)
1503 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1504 		    PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1505 			pmclog_process_map_in(pm->pm_owner,
1506 			    pid, pkm->pm_address, fullpath);
1507 
1508   done:
1509 	if (freepath)
1510 		free(freepath, M_TEMP);
1511 }
1512 
1513 
1514 /*
1515  * Log an munmap request.
1516  */
1517 
1518 static void
1519 pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
1520 {
1521 	int ri;
1522 	pid_t pid;
1523 	struct pmc_owner *po;
1524 	const struct pmc *pm;
1525 	const struct pmc_process *pp;
1526 
1527 	pid = td->td_proc->p_pid;
1528 
1529 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1530 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1531 		pmclog_process_map_out(po, pid, pkm->pm_address,
1532 		    pkm->pm_address + pkm->pm_size);
1533 
1534 	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1535 		return;
1536 
1537 	for (ri = 0; ri < md->pmd_npmc; ri++)
1538 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1539 		    PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1540 			pmclog_process_map_out(pm->pm_owner, pid,
1541 			    pkm->pm_address, pkm->pm_address + pkm->pm_size);
1542 }
1543 
1544 /*
1545  * Log mapping information about the kernel.
1546  */
1547 
1548 static void
1549 pmc_log_kernel_mappings(struct pmc *pm)
1550 {
1551 	struct pmc_owner *po;
1552 	struct pmckern_map_in *km, *kmbase;
1553 
1554 	sx_assert(&pmc_sx, SX_LOCKED);
1555 	KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1556 	    ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1557 		__LINE__, (void *) pm));
1558 
1559 	po = pm->pm_owner;
1560 
1561 	if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1562 		return;
1563 
1564 	/*
1565 	 * Log the current set of kernel modules.
1566 	 */
1567 	kmbase = linker_hwpmc_list_objects();
1568 	for (km = kmbase; km->pm_file != NULL; km++) {
1569 		PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file,
1570 		    (void *) km->pm_address);
1571 		pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1572 		    km->pm_file);
1573 	}
1574 	free(kmbase, M_LINKER);
1575 
1576 	po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1577 }
1578 
1579 /*
1580  * Log the mappings for a single process.
1581  */
1582 
1583 static void
1584 pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1585 {
1586 }
1587 
1588 /*
1589  * Log mappings for all processes in the system.
1590  */
1591 
1592 static void
1593 pmc_log_all_process_mappings(struct pmc_owner *po)
1594 {
1595 	struct proc *p, *top;
1596 
1597 	sx_assert(&pmc_sx, SX_XLOCKED);
1598 
1599 	if ((p = pfind(1)) == NULL)
1600 		panic("[pmc,%d] Cannot find init", __LINE__);
1601 
1602 	PROC_UNLOCK(p);
1603 
1604 	sx_slock(&proctree_lock);
1605 
1606 	top = p;
1607 
1608 	for (;;) {
1609 		pmc_log_process_mappings(po, p);
1610 		if (!LIST_EMPTY(&p->p_children))
1611 			p = LIST_FIRST(&p->p_children);
1612 		else for (;;) {
1613 			if (p == top)
1614 				goto done;
1615 			if (LIST_NEXT(p, p_sibling)) {
1616 				p = LIST_NEXT(p, p_sibling);
1617 				break;
1618 			}
1619 			p = p->p_pptr;
1620 		}
1621 	}
1622  done:
1623 	sx_sunlock(&proctree_lock);
1624 }
1625 
1626 /*
1627  * The 'hook' invoked from the kernel proper
1628  */
1629 
1630 
1631 #ifdef	DEBUG
1632 const char *pmc_hooknames[] = {
1633 	/* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
1634 	"",
1635 	"EXEC",
1636 	"CSW-IN",
1637 	"CSW-OUT",
1638 	"SAMPLE",
1639 	"KLDLOAD",
1640 	"KLDUNLOAD",
1641 	"MMAP",
1642 	"MUNMAP",
1643 	"CALLCHAIN"
1644 };
1645 #endif
1646 
1647 static int
1648 pmc_hook_handler(struct thread *td, int function, void *arg)
1649 {
1650 
1651 	PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
1652 	    pmc_hooknames[function], arg);
1653 
1654 	switch (function)
1655 	{
1656 
1657 	/*
1658 	 * Process exec()
1659 	 */
1660 
1661 	case PMC_FN_PROCESS_EXEC:
1662 	{
1663 		char *fullpath, *freepath;
1664 		unsigned int ri;
1665 		int is_using_hwpmcs;
1666 		struct pmc *pm;
1667 		struct proc *p;
1668 		struct pmc_owner *po;
1669 		struct pmc_process *pp;
1670 		struct pmckern_procexec *pk;
1671 
1672 		sx_assert(&pmc_sx, SX_XLOCKED);
1673 
1674 		p = td->td_proc;
1675 		pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1676 
1677 		pk = (struct pmckern_procexec *) arg;
1678 
1679 		/* Inform owners of SS mode PMCs of the exec event. */
1680 		LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1681 		    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1682 			    pmclog_process_procexec(po, PMC_ID_INVALID,
1683 				p->p_pid, pk->pm_entryaddr, fullpath);
1684 
1685 		PROC_LOCK(p);
1686 		is_using_hwpmcs = p->p_flag & P_HWPMC;
1687 		PROC_UNLOCK(p);
1688 
1689 		if (!is_using_hwpmcs) {
1690 			if (freepath)
1691 				free(freepath, M_TEMP);
1692 			break;
1693 		}
1694 
1695 		/*
1696 		 * PMCs are not inherited across an exec():  remove any
1697 		 * PMCs that this process is the owner of.
1698 		 */
1699 
1700 		if ((po = pmc_find_owner_descriptor(p)) != NULL) {
1701 			pmc_remove_owner(po);
1702 			pmc_destroy_owner_descriptor(po);
1703 		}
1704 
1705 		/*
1706 		 * If the process being exec'ed is not the target of any
1707 		 * PMC, we are done.
1708 		 */
1709 		if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
1710 			if (freepath)
1711 				free(freepath, M_TEMP);
1712 			break;
1713 		}
1714 
1715 		/*
1716 		 * Log the exec event to all monitoring owners.  Skip
1717 		 * owners who have already recieved the event because
1718 		 * they had system sampling PMCs active.
1719 		 */
1720 		for (ri = 0; ri < md->pmd_npmc; ri++)
1721 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
1722 				po = pm->pm_owner;
1723 				if (po->po_sscount == 0 &&
1724 				    po->po_flags & PMC_PO_OWNS_LOGFILE)
1725 					pmclog_process_procexec(po, pm->pm_id,
1726 					    p->p_pid, pk->pm_entryaddr,
1727 					    fullpath);
1728 			}
1729 
1730 		if (freepath)
1731 			free(freepath, M_TEMP);
1732 
1733 
1734 		PMCDBG(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
1735 		    p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
1736 
1737 		if (pk->pm_credentialschanged == 0) /* no change */
1738 			break;
1739 
1740 		/*
1741 		 * If the newly exec()'ed process has a different credential
1742 		 * than before, allow it to be the target of a PMC only if
1743 		 * the PMC's owner has sufficient priviledge.
1744 		 */
1745 
1746 		for (ri = 0; ri < md->pmd_npmc; ri++)
1747 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
1748 				if (pmc_can_attach(pm, td->td_proc) != 0)
1749 					pmc_detach_one_process(td->td_proc,
1750 					    pm, PMC_FLAG_NONE);
1751 
1752 		KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc,
1753 		    ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
1754 			pp->pp_refcnt, pp));
1755 
1756 		/*
1757 		 * If this process is no longer the target of any
1758 		 * PMCs, we can remove the process entry and free
1759 		 * up space.
1760 		 */
1761 
1762 		if (pp->pp_refcnt == 0) {
1763 			pmc_remove_process_descriptor(pp);
1764 			free(pp, M_PMC);
1765 			break;
1766 		}
1767 
1768 	}
1769 	break;
1770 
1771 	case PMC_FN_CSW_IN:
1772 		pmc_process_csw_in(td);
1773 		break;
1774 
1775 	case PMC_FN_CSW_OUT:
1776 		pmc_process_csw_out(td);
1777 		break;
1778 
1779 	/*
1780 	 * Process accumulated PC samples.
1781 	 *
1782 	 * This function is expected to be called by hardclock() for
1783 	 * each CPU that has accumulated PC samples.
1784 	 *
1785 	 * This function is to be executed on the CPU whose samples
1786 	 * are being processed.
1787 	 */
1788 	case PMC_FN_DO_SAMPLES:
1789 
1790 		/*
1791 		 * Clear the cpu specific bit in the CPU mask before
1792 		 * do the rest of the processing.  If the NMI handler
1793 		 * gets invoked after the "atomic_clear_int()" call
1794 		 * below but before "pmc_process_samples()" gets
1795 		 * around to processing the interrupt, then we will
1796 		 * come back here at the next hardclock() tick (and
1797 		 * may find nothing to do if "pmc_process_samples()"
1798 		 * had already processed the interrupt).  We don't
1799 		 * lose the interrupt sample.
1800 		 */
1801 		atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid)));
1802 		pmc_process_samples(PCPU_GET(cpuid));
1803 		break;
1804 
1805 
1806 	case PMC_FN_KLD_LOAD:
1807 		sx_assert(&pmc_sx, SX_LOCKED);
1808 		pmc_process_kld_load((struct pmckern_map_in *) arg);
1809 		break;
1810 
1811 	case PMC_FN_KLD_UNLOAD:
1812 		sx_assert(&pmc_sx, SX_LOCKED);
1813 		pmc_process_kld_unload((struct pmckern_map_out *) arg);
1814 		break;
1815 
1816 	case PMC_FN_MMAP:
1817 		sx_assert(&pmc_sx, SX_LOCKED);
1818 		pmc_process_mmap(td, (struct pmckern_map_in *) arg);
1819 		break;
1820 
1821 	case PMC_FN_MUNMAP:
1822 		sx_assert(&pmc_sx, SX_LOCKED);
1823 		pmc_process_munmap(td, (struct pmckern_map_out *) arg);
1824 		break;
1825 
1826 	case PMC_FN_USER_CALLCHAIN:
1827 		/*
1828 		 * Record a call chain.
1829 		 */
1830 		pmc_capture_user_callchain(PCPU_GET(cpuid),
1831 		    (struct trapframe *) arg);
1832 		break;
1833 
1834 	default:
1835 #ifdef	DEBUG
1836 		KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
1837 #endif
1838 		break;
1839 
1840 	}
1841 
1842 	return 0;
1843 }
1844 
1845 /*
1846  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
1847  */
1848 
1849 static struct pmc_owner *
1850 pmc_allocate_owner_descriptor(struct proc *p)
1851 {
1852 	uint32_t hindex;
1853 	struct pmc_owner *po;
1854 	struct pmc_ownerhash *poh;
1855 
1856 	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1857 	poh = &pmc_ownerhash[hindex];
1858 
1859 	/* allocate space for N pointers and one descriptor struct */
1860 	po = malloc(sizeof(struct pmc_owner), M_PMC, M_ZERO|M_WAITOK);
1861 
1862 	po->po_sscount = po->po_error = po->po_flags = 0;
1863 	po->po_file  = NULL;
1864 	po->po_owner = p;
1865 	po->po_kthread = NULL;
1866 	LIST_INIT(&po->po_pmcs);
1867 	LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
1868 
1869 	TAILQ_INIT(&po->po_logbuffers);
1870 	mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
1871 
1872 	PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
1873 	    p, p->p_pid, p->p_comm, po);
1874 
1875 	return po;
1876 }
1877 
1878 static void
1879 pmc_destroy_owner_descriptor(struct pmc_owner *po)
1880 {
1881 
1882 	PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
1883 	    po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
1884 
1885 	mtx_destroy(&po->po_mtx);
1886 	free(po, M_PMC);
1887 }
1888 
1889 /*
1890  * find the descriptor corresponding to process 'p', adding or removing it
1891  * as specified by 'mode'.
1892  */
1893 
1894 static struct pmc_process *
1895 pmc_find_process_descriptor(struct proc *p, uint32_t mode)
1896 {
1897 	uint32_t hindex;
1898 	struct pmc_process *pp, *ppnew;
1899 	struct pmc_processhash *pph;
1900 
1901 	hindex = PMC_HASH_PTR(p, pmc_processhashmask);
1902 	pph = &pmc_processhash[hindex];
1903 
1904 	ppnew = NULL;
1905 
1906 	/*
1907 	 * Pre-allocate memory in the FIND_ALLOCATE case since we
1908 	 * cannot call malloc(9) once we hold a spin lock.
1909 	 */
1910 
1911 	if (mode & PMC_FLAG_ALLOCATE) {
1912 		/* allocate additional space for 'n' pmc pointers */
1913 		ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
1914 		    sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK);
1915 	}
1916 
1917 	mtx_lock_spin(&pmc_processhash_mtx);
1918 	LIST_FOREACH(pp, pph, pp_next)
1919 	    if (pp->pp_proc == p)
1920 		    break;
1921 
1922 	if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
1923 		LIST_REMOVE(pp, pp_next);
1924 
1925 	if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
1926 	    ppnew != NULL) {
1927 		ppnew->pp_proc = p;
1928 		LIST_INSERT_HEAD(pph, ppnew, pp_next);
1929 		pp = ppnew;
1930 		ppnew = NULL;
1931 	}
1932 	mtx_unlock_spin(&pmc_processhash_mtx);
1933 
1934 	if (pp != NULL && ppnew != NULL)
1935 		free(ppnew, M_PMC);
1936 
1937 	return pp;
1938 }
1939 
1940 /*
1941  * remove a process descriptor from the process hash table.
1942  */
1943 
1944 static void
1945 pmc_remove_process_descriptor(struct pmc_process *pp)
1946 {
1947 	KASSERT(pp->pp_refcnt == 0,
1948 	    ("[pmc,%d] Removing process descriptor %p with count %d",
1949 		__LINE__, pp, pp->pp_refcnt));
1950 
1951 	mtx_lock_spin(&pmc_processhash_mtx);
1952 	LIST_REMOVE(pp, pp_next);
1953 	mtx_unlock_spin(&pmc_processhash_mtx);
1954 }
1955 
1956 
1957 /*
1958  * find an owner descriptor corresponding to proc 'p'
1959  */
1960 
1961 static struct pmc_owner *
1962 pmc_find_owner_descriptor(struct proc *p)
1963 {
1964 	uint32_t hindex;
1965 	struct pmc_owner *po;
1966 	struct pmc_ownerhash *poh;
1967 
1968 	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1969 	poh = &pmc_ownerhash[hindex];
1970 
1971 	po = NULL;
1972 	LIST_FOREACH(po, poh, po_next)
1973 	    if (po->po_owner == p)
1974 		    break;
1975 
1976 	PMCDBG(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
1977 	    "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
1978 
1979 	return po;
1980 }
1981 
1982 /*
1983  * pmc_allocate_pmc_descriptor
1984  *
1985  * Allocate a pmc descriptor and initialize its
1986  * fields.
1987  */
1988 
1989 static struct pmc *
1990 pmc_allocate_pmc_descriptor(void)
1991 {
1992 	struct pmc *pmc;
1993 
1994 	pmc = malloc(sizeof(struct pmc), M_PMC, M_ZERO|M_WAITOK);
1995 
1996 	if (pmc != NULL) {
1997 		pmc->pm_owner = NULL;
1998 		LIST_INIT(&pmc->pm_targets);
1999 	}
2000 
2001 	PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2002 
2003 	return pmc;
2004 }
2005 
2006 /*
2007  * Destroy a pmc descriptor.
2008  */
2009 
2010 static void
2011 pmc_destroy_pmc_descriptor(struct pmc *pm)
2012 {
2013 	(void) pm;
2014 
2015 #ifdef	DEBUG
2016 	KASSERT(pm->pm_state == PMC_STATE_DELETED ||
2017 	    pm->pm_state == PMC_STATE_FREE,
2018 	    ("[pmc,%d] destroying non-deleted PMC", __LINE__));
2019 	KASSERT(LIST_EMPTY(&pm->pm_targets),
2020 	    ("[pmc,%d] destroying pmc with targets", __LINE__));
2021 	KASSERT(pm->pm_owner == NULL,
2022 	    ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
2023 	KASSERT(pm->pm_runcount == 0,
2024 	    ("[pmc,%d] pmc has non-zero run count %d", __LINE__,
2025 		pm->pm_runcount));
2026 #endif
2027 }
2028 
2029 static void
2030 pmc_wait_for_pmc_idle(struct pmc *pm)
2031 {
2032 #ifdef	DEBUG
2033 	volatile int maxloop;
2034 
2035 	maxloop = 100 * pmc_cpu_max();
2036 #endif
2037 
2038 	/*
2039 	 * Loop (with a forced context switch) till the PMC's runcount
2040 	 * comes down to zero.
2041 	 */
2042 	while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
2043 #ifdef	DEBUG
2044 		maxloop--;
2045 		KASSERT(maxloop > 0,
2046 		    ("[pmc,%d] (ri%d, rc%d) waiting too long for "
2047 			"pmc to be free", __LINE__,
2048 			PMC_TO_ROWINDEX(pm), pm->pm_runcount));
2049 #endif
2050 		pmc_force_context_switch();
2051 	}
2052 }
2053 
2054 /*
2055  * This function does the following things:
2056  *
2057  *  - detaches the PMC from hardware
2058  *  - unlinks all target threads that were attached to it
2059  *  - removes the PMC from its owner's list
2060  *  - destroy's the PMC private mutex
2061  *
2062  * Once this function completes, the given pmc pointer can be safely
2063  * FREE'd by the caller.
2064  */
2065 
2066 static void
2067 pmc_release_pmc_descriptor(struct pmc *pm)
2068 {
2069 	u_int ri, cpu;
2070 	enum pmc_mode mode;
2071 	struct pmc_hw *phw;
2072 	struct pmc_owner *po;
2073 	struct pmc_process *pp;
2074 	struct pmc_target *ptgt, *tmp;
2075 	struct pmc_binding pb;
2076 
2077 	sx_assert(&pmc_sx, SX_XLOCKED);
2078 
2079 	KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2080 
2081 	ri   = PMC_TO_ROWINDEX(pm);
2082 	mode = PMC_TO_MODE(pm);
2083 
2084 	PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2085 	    mode);
2086 
2087 	/*
2088 	 * First, we take the PMC off hardware.
2089 	 */
2090 	cpu = 0;
2091 	if (PMC_IS_SYSTEM_MODE(mode)) {
2092 
2093 		/*
2094 		 * A system mode PMC runs on a specific CPU.  Switch
2095 		 * to this CPU and turn hardware off.
2096 		 */
2097 		pmc_save_cpu_binding(&pb);
2098 
2099 		cpu = PMC_TO_CPU(pm);
2100 
2101 		pmc_select_cpu(cpu);
2102 
2103 		/* switch off non-stalled CPUs */
2104 		if (pm->pm_state == PMC_STATE_RUNNING &&
2105 		    pm->pm_stalled == 0) {
2106 
2107 			phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2108 
2109 			KASSERT(phw->phw_pmc == pm,
2110 			    ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2111 				__LINE__, ri, phw->phw_pmc, pm));
2112 			PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2113 
2114 			critical_enter();
2115 			md->pmd_stop_pmc(cpu, ri);
2116 			critical_exit();
2117 		}
2118 
2119 		PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2120 
2121 		critical_enter();
2122 		md->pmd_config_pmc(cpu, ri, NULL);
2123 		critical_exit();
2124 
2125 		/* adjust the global and process count of SS mode PMCs */
2126 		if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2127 			po = pm->pm_owner;
2128 			po->po_sscount--;
2129 			if (po->po_sscount == 0) {
2130 				atomic_subtract_rel_int(&pmc_ss_count, 1);
2131 				LIST_REMOVE(po, po_ssnext);
2132 			}
2133 		}
2134 
2135 		pm->pm_state = PMC_STATE_DELETED;
2136 
2137 		pmc_restore_cpu_binding(&pb);
2138 
2139 		/*
2140 		 * We could have references to this PMC structure in
2141 		 * the per-cpu sample queues.  Wait for the queue to
2142 		 * drain.
2143 		 */
2144 		pmc_wait_for_pmc_idle(pm);
2145 
2146 	} else if (PMC_IS_VIRTUAL_MODE(mode)) {
2147 
2148 		/*
2149 		 * A virtual PMC could be running on multiple CPUs at
2150 		 * a given instant.
2151 		 *
2152 		 * By marking its state as DELETED, we ensure that
2153 		 * this PMC is never further scheduled on hardware.
2154 		 *
2155 		 * Then we wait till all CPUs are done with this PMC.
2156 		 */
2157 		pm->pm_state = PMC_STATE_DELETED;
2158 
2159 
2160 		/* Wait for the PMCs runcount to come to zero. */
2161 		pmc_wait_for_pmc_idle(pm);
2162 
2163 		/*
2164 		 * At this point the PMC is off all CPUs and cannot be
2165 		 * freshly scheduled onto a CPU.  It is now safe to
2166 		 * unlink all targets from this PMC.  If a
2167 		 * process-record's refcount falls to zero, we remove
2168 		 * it from the hash table.  The module-wide SX lock
2169 		 * protects us from races.
2170 		 */
2171 		LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
2172 			pp = ptgt->pt_process;
2173 			pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
2174 
2175 			PMCDBG(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
2176 
2177 			/*
2178 			 * If the target process record shows that no
2179 			 * PMCs are attached to it, reclaim its space.
2180 			 */
2181 
2182 			if (pp->pp_refcnt == 0) {
2183 				pmc_remove_process_descriptor(pp);
2184 				free(pp, M_PMC);
2185 			}
2186 		}
2187 
2188 		cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2189 
2190 	}
2191 
2192 	/*
2193 	 * Release any MD resources
2194 	 */
2195 
2196 	(void) md->pmd_release_pmc(cpu, ri, pm);
2197 
2198 	/*
2199 	 * Update row disposition
2200 	 */
2201 
2202 	if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2203 		PMC_UNMARK_ROW_STANDALONE(ri);
2204 	else
2205 		PMC_UNMARK_ROW_THREAD(ri);
2206 
2207 	/* unlink from the owner's list */
2208 	if (pm->pm_owner) {
2209 		LIST_REMOVE(pm, pm_next);
2210 		pm->pm_owner = NULL;
2211 	}
2212 
2213 	pmc_destroy_pmc_descriptor(pm);
2214 }
2215 
2216 /*
2217  * Register an owner and a pmc.
2218  */
2219 
2220 static int
2221 pmc_register_owner(struct proc *p, struct pmc *pmc)
2222 {
2223 	struct pmc_owner *po;
2224 
2225 	sx_assert(&pmc_sx, SX_XLOCKED);
2226 
2227 	if ((po = pmc_find_owner_descriptor(p)) == NULL)
2228 		if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
2229 			return ENOMEM;
2230 
2231 	KASSERT(pmc->pm_owner == NULL,
2232 	    ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2233 	pmc->pm_owner  = po;
2234 
2235 	LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
2236 
2237 	PROC_LOCK(p);
2238 	p->p_flag |= P_HWPMC;
2239 	PROC_UNLOCK(p);
2240 
2241 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2242 		pmclog_process_pmcallocate(pmc);
2243 
2244 	PMCDBG(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2245 	    po, pmc);
2246 
2247 	return 0;
2248 }
2249 
2250 /*
2251  * Return the current row disposition:
2252  * == 0 => FREE
2253  *  > 0 => PROCESS MODE
2254  *  < 0 => SYSTEM MODE
2255  */
2256 
2257 int
2258 pmc_getrowdisp(int ri)
2259 {
2260 	return pmc_pmcdisp[ri];
2261 }
2262 
2263 /*
2264  * Check if a PMC at row index 'ri' can be allocated to the current
2265  * process.
2266  *
2267  * Allocation can fail if:
2268  *   - the current process is already being profiled by a PMC at index 'ri',
2269  *     attached to it via OP_PMCATTACH.
2270  *   - the current process has already allocated a PMC at index 'ri'
2271  *     via OP_ALLOCATE.
2272  */
2273 
2274 static int
2275 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2276 {
2277 	enum pmc_mode mode;
2278 	struct pmc *pm;
2279 	struct pmc_owner *po;
2280 	struct pmc_process *pp;
2281 
2282 	PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2283 	    "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2284 
2285 	/*
2286 	 * We shouldn't have already allocated a process-mode PMC at
2287 	 * row index 'ri'.
2288 	 *
2289 	 * We shouldn't have allocated a system-wide PMC on the same
2290 	 * CPU and same RI.
2291 	 */
2292 	if ((po = pmc_find_owner_descriptor(p)) != NULL)
2293 		LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2294 		    if (PMC_TO_ROWINDEX(pm) == ri) {
2295 			    mode = PMC_TO_MODE(pm);
2296 			    if (PMC_IS_VIRTUAL_MODE(mode))
2297 				    return EEXIST;
2298 			    if (PMC_IS_SYSTEM_MODE(mode) &&
2299 				(int) PMC_TO_CPU(pm) == cpu)
2300 				    return EEXIST;
2301 		    }
2302 	        }
2303 
2304 	/*
2305 	 * We also shouldn't be the target of any PMC at this index
2306 	 * since otherwise a PMC_ATTACH to ourselves will fail.
2307 	 */
2308 	if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2309 		if (pp->pp_pmcs[ri].pp_pmc)
2310 			return EEXIST;
2311 
2312 	PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2313 	    p, p->p_pid, p->p_comm, ri);
2314 
2315 	return 0;
2316 }
2317 
2318 /*
2319  * Check if a given PMC at row index 'ri' can be currently used in
2320  * mode 'mode'.
2321  */
2322 
2323 static int
2324 pmc_can_allocate_row(int ri, enum pmc_mode mode)
2325 {
2326 	enum pmc_disp	disp;
2327 
2328 	sx_assert(&pmc_sx, SX_XLOCKED);
2329 
2330 	PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
2331 
2332 	if (PMC_IS_SYSTEM_MODE(mode))
2333 		disp = PMC_DISP_STANDALONE;
2334 	else
2335 		disp = PMC_DISP_THREAD;
2336 
2337 	/*
2338 	 * check disposition for PMC row 'ri':
2339 	 *
2340 	 * Expected disposition		Row-disposition		Result
2341 	 *
2342 	 * STANDALONE			STANDALONE or FREE	proceed
2343 	 * STANDALONE			THREAD			fail
2344 	 * THREAD			THREAD or FREE		proceed
2345 	 * THREAD			STANDALONE		fail
2346 	 */
2347 
2348 	if (!PMC_ROW_DISP_IS_FREE(ri) &&
2349 	    !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
2350 	    !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
2351 		return EBUSY;
2352 
2353 	/*
2354 	 * All OK
2355 	 */
2356 
2357 	PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
2358 
2359 	return 0;
2360 
2361 }
2362 
2363 /*
2364  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
2365  */
2366 
2367 static struct pmc *
2368 pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
2369 {
2370 	struct pmc *pm;
2371 
2372 	KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
2373 	    ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
2374 		PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
2375 
2376 	LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2377 	    if (pm->pm_id == pmcid)
2378 		    return pm;
2379 
2380 	return NULL;
2381 }
2382 
2383 static int
2384 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
2385 {
2386 
2387 	struct pmc *pm;
2388 	struct pmc_owner *po;
2389 
2390 	PMCDBG(PMC,FND,1, "find-pmc id=%d", pmcid);
2391 
2392 	if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL)
2393 		return ESRCH;
2394 
2395 	if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
2396 		return EINVAL;
2397 
2398 	PMCDBG(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
2399 
2400 	*pmc = pm;
2401 	return 0;
2402 }
2403 
2404 /*
2405  * Start a PMC.
2406  */
2407 
2408 static int
2409 pmc_start(struct pmc *pm)
2410 {
2411 	int error, cpu, ri;
2412 	enum pmc_mode mode;
2413 	struct pmc_owner *po;
2414 	struct pmc_binding pb;
2415 
2416 	KASSERT(pm != NULL,
2417 	    ("[pmc,%d] null pm", __LINE__));
2418 
2419 	mode = PMC_TO_MODE(pm);
2420 	ri   = PMC_TO_ROWINDEX(pm);
2421 	error = 0;
2422 
2423 	PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2424 
2425 	po = pm->pm_owner;
2426 
2427 	/*
2428 	 * Disallow PMCSTART if a logfile is required but has not been
2429 	 * configured yet.
2430 	 */
2431 	if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2432 	    (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2433 		return EDOOFUS;	/* programming error */
2434 
2435 	/*
2436 	 * If this is a sampling mode PMC, log mapping information for
2437 	 * the kernel modules that are currently loaded.
2438 	 */
2439 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2440 	    pmc_log_kernel_mappings(pm);
2441 
2442 	if (PMC_IS_VIRTUAL_MODE(mode)) {
2443 
2444 		/*
2445 		 * If a PMCATTACH has never been done on this PMC,
2446 		 * attach it to its owner process.
2447 		 */
2448 
2449 		if (LIST_EMPTY(&pm->pm_targets))
2450 			error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
2451 			    pmc_attach_process(po->po_owner, pm);
2452 
2453 		/*
2454 		 * If the PMC is attached to its owner, then force a context
2455 		 * switch to ensure that the MD state gets set correctly.
2456 		 */
2457 
2458 		if (error == 0) {
2459 			pm->pm_state = PMC_STATE_RUNNING;
2460 			if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2461 				pmc_force_context_switch();
2462 		}
2463 
2464 		return error;
2465 	}
2466 
2467 
2468 	/*
2469 	 * A system-wide PMC.
2470 	 *
2471 	 * Add the owner to the global list if this is a system-wide
2472 	 * sampling PMC.
2473 	 */
2474 
2475 	if (mode == PMC_MODE_SS) {
2476 		if (po->po_sscount == 0) {
2477 			LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
2478 			atomic_add_rel_int(&pmc_ss_count, 1);
2479 			PMCDBG(PMC,OPS,1, "po=%p in global list", po);
2480 		}
2481 		po->po_sscount++;
2482 	}
2483 
2484 	/* Log mapping information for all processes in the system. */
2485 	pmc_log_all_process_mappings(po);
2486 
2487 	/*
2488 	 * Move to the CPU associated with this
2489 	 * PMC, and start the hardware.
2490 	 */
2491 
2492 	pmc_save_cpu_binding(&pb);
2493 
2494 	cpu = PMC_TO_CPU(pm);
2495 
2496 	if (!pmc_cpu_is_active(cpu))
2497 		return ENXIO;
2498 
2499 	pmc_select_cpu(cpu);
2500 
2501 	/*
2502 	 * global PMCs are configured at allocation time
2503 	 * so write out the initial value and start the PMC.
2504 	 */
2505 
2506 	pm->pm_state = PMC_STATE_RUNNING;
2507 
2508 	critical_enter();
2509 	if ((error = md->pmd_write_pmc(cpu, ri,
2510 		 PMC_IS_SAMPLING_MODE(mode) ?
2511 		 pm->pm_sc.pm_reloadcount :
2512 		 pm->pm_sc.pm_initial)) == 0)
2513 		error = md->pmd_start_pmc(cpu, ri);
2514 	critical_exit();
2515 
2516 	pmc_restore_cpu_binding(&pb);
2517 
2518 	return error;
2519 }
2520 
2521 /*
2522  * Stop a PMC.
2523  */
2524 
2525 static int
2526 pmc_stop(struct pmc *pm)
2527 {
2528 	int cpu, error, ri;
2529 	struct pmc_owner *po;
2530 	struct pmc_binding pb;
2531 
2532 	KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2533 
2534 	PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2535 	    PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2536 
2537 	pm->pm_state = PMC_STATE_STOPPED;
2538 
2539 	/*
2540 	 * If the PMC is a virtual mode one, changing the state to
2541 	 * non-RUNNING is enough to ensure that the PMC never gets
2542 	 * scheduled.
2543 	 *
2544 	 * If this PMC is current running on a CPU, then it will
2545 	 * handled correctly at the time its target process is context
2546 	 * switched out.
2547 	 */
2548 
2549 	if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
2550 		return 0;
2551 
2552 	/*
2553 	 * A system-mode PMC.  Move to the CPU associated with
2554 	 * this PMC, and stop the hardware.  We update the
2555 	 * 'initial count' so that a subsequent PMCSTART will
2556 	 * resume counting from the current hardware count.
2557 	 */
2558 
2559 	pmc_save_cpu_binding(&pb);
2560 
2561 	cpu = PMC_TO_CPU(pm);
2562 
2563 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2564 	    ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2565 
2566 	if (!pmc_cpu_is_active(cpu))
2567 		return ENXIO;
2568 
2569 	pmc_select_cpu(cpu);
2570 
2571 	ri = PMC_TO_ROWINDEX(pm);
2572 
2573 	critical_enter();
2574 	if ((error = md->pmd_stop_pmc(cpu, ri)) == 0)
2575 		error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial);
2576 	critical_exit();
2577 
2578 	pmc_restore_cpu_binding(&pb);
2579 
2580 	po = pm->pm_owner;
2581 
2582 	/* remove this owner from the global list of SS PMC owners */
2583 	if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
2584 		po->po_sscount--;
2585 		if (po->po_sscount == 0) {
2586 			atomic_subtract_rel_int(&pmc_ss_count, 1);
2587 			LIST_REMOVE(po, po_ssnext);
2588 			PMCDBG(PMC,OPS,2,"po=%p removed from global list", po);
2589 		}
2590 	}
2591 
2592 	return error;
2593 }
2594 
2595 
2596 #ifdef	DEBUG
2597 static const char *pmc_op_to_name[] = {
2598 #undef	__PMC_OP
2599 #define	__PMC_OP(N, D)	#N ,
2600 	__PMC_OPS()
2601 	NULL
2602 };
2603 #endif
2604 
2605 /*
2606  * The syscall interface
2607  */
2608 
2609 #define	PMC_GET_SX_XLOCK(...) do {		\
2610 	sx_xlock(&pmc_sx);			\
2611 	if (pmc_hook == NULL) {			\
2612 		sx_xunlock(&pmc_sx);		\
2613 		return __VA_ARGS__;		\
2614 	}					\
2615 } while (0)
2616 
2617 #define	PMC_DOWNGRADE_SX() do {			\
2618 	sx_downgrade(&pmc_sx);			\
2619 	is_sx_downgraded = 1;			\
2620 } while (0)
2621 
2622 static int
2623 pmc_syscall_handler(struct thread *td, void *syscall_args)
2624 {
2625 	int error, is_sx_downgraded, op;
2626 	struct pmc_syscall_args *c;
2627 	void *arg;
2628 
2629 	PMC_GET_SX_XLOCK(ENOSYS);
2630 
2631 	DROP_GIANT();
2632 
2633 	is_sx_downgraded = 0;
2634 
2635 	c = (struct pmc_syscall_args *) syscall_args;
2636 
2637 	op = c->pmop_code;
2638 	arg = c->pmop_data;
2639 
2640 	PMCDBG(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
2641 	    pmc_op_to_name[op], arg);
2642 
2643 	error = 0;
2644 	atomic_add_int(&pmc_stats.pm_syscalls, 1);
2645 
2646 	switch(op)
2647 	{
2648 
2649 
2650 	/*
2651 	 * Configure a log file.
2652 	 *
2653 	 * XXX This OP will be reworked.
2654 	 */
2655 
2656 	case PMC_OP_CONFIGURELOG:
2657 	{
2658 		struct proc *p;
2659 		struct pmc *pm;
2660 		struct pmc_owner *po;
2661 		struct pmc_op_configurelog cl;
2662 
2663 		sx_assert(&pmc_sx, SX_XLOCKED);
2664 
2665 		if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
2666 			break;
2667 
2668 		/* mark this process as owning a log file */
2669 		p = td->td_proc;
2670 		if ((po = pmc_find_owner_descriptor(p)) == NULL)
2671 			if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
2672 				error = ENOMEM;
2673 				break;
2674 			}
2675 
2676 		/*
2677 		 * If a valid fd was passed in, try to configure that,
2678 		 * otherwise if 'fd' was less than zero and there was
2679 		 * a log file configured, flush its buffers and
2680 		 * de-configure it.
2681 		 */
2682 		if (cl.pm_logfd >= 0)
2683 			error = pmclog_configure_log(po, cl.pm_logfd);
2684 		else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
2685 			pmclog_process_closelog(po);
2686 			error = pmclog_flush(po);
2687 			if (error == 0) {
2688 				LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2689 				    if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
2690 					pm->pm_state == PMC_STATE_RUNNING)
2691 					    pmc_stop(pm);
2692 				error = pmclog_deconfigure_log(po);
2693 			}
2694 		} else
2695 			error = EINVAL;
2696 
2697 		if (error)
2698 			break;
2699 	}
2700 	break;
2701 
2702 
2703 	/*
2704 	 * Flush a log file.
2705 	 */
2706 
2707 	case PMC_OP_FLUSHLOG:
2708 	{
2709 		struct pmc_owner *po;
2710 
2711 		sx_assert(&pmc_sx, SX_XLOCKED);
2712 
2713 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
2714 			error = EINVAL;
2715 			break;
2716 		}
2717 
2718 		error = pmclog_flush(po);
2719 	}
2720 	break;
2721 
2722 	/*
2723 	 * Retrieve hardware configuration.
2724 	 */
2725 
2726 	case PMC_OP_GETCPUINFO:	/* CPU information */
2727 	{
2728 		struct pmc_op_getcpuinfo gci;
2729 
2730 		gci.pm_cputype = md->pmd_cputype;
2731 		gci.pm_ncpu    = pmc_cpu_max();
2732 		gci.pm_npmc    = md->pmd_npmc;
2733 		gci.pm_nclass  = md->pmd_nclass;
2734 		bcopy(md->pmd_classes, &gci.pm_classes,
2735 		    sizeof(gci.pm_classes));
2736 		error = copyout(&gci, arg, sizeof(gci));
2737 	}
2738 	break;
2739 
2740 
2741 	/*
2742 	 * Get module statistics
2743 	 */
2744 
2745 	case PMC_OP_GETDRIVERSTATS:
2746 	{
2747 		struct pmc_op_getdriverstats gms;
2748 
2749 		bcopy(&pmc_stats, &gms, sizeof(gms));
2750 		error = copyout(&gms, arg, sizeof(gms));
2751 	}
2752 	break;
2753 
2754 
2755 	/*
2756 	 * Retrieve module version number
2757 	 */
2758 
2759 	case PMC_OP_GETMODULEVERSION:
2760 	{
2761 		uint32_t cv, modv;
2762 
2763 		/* retrieve the client's idea of the ABI version */
2764 		if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
2765 			break;
2766 		/* don't service clients newer than our driver */
2767 		modv = PMC_VERSION;
2768 		if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
2769 			error = EPROGMISMATCH;
2770 			break;
2771 		}
2772 		error = copyout(&modv, arg, sizeof(int));
2773 	}
2774 	break;
2775 
2776 
2777 	/*
2778 	 * Retrieve the state of all the PMCs on a given
2779 	 * CPU.
2780 	 */
2781 
2782 	case PMC_OP_GETPMCINFO:
2783 	{
2784 		uint32_t cpu, n, npmc;
2785 		size_t pmcinfo_size;
2786 		struct pmc *pm;
2787 		struct pmc_info *p, *pmcinfo;
2788 		struct pmc_op_getpmcinfo *gpi;
2789 		struct pmc_owner *po;
2790 		struct pmc_binding pb;
2791 
2792 		PMC_DOWNGRADE_SX();
2793 
2794 		gpi = (struct pmc_op_getpmcinfo *) arg;
2795 
2796 		if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
2797 			break;
2798 
2799 		if (cpu >= pmc_cpu_max()) {
2800 			error = EINVAL;
2801 			break;
2802 		}
2803 
2804 		if (!pmc_cpu_is_active(cpu)) {
2805 			error = ENXIO;
2806 			break;
2807 		}
2808 
2809 		/* switch to CPU 'cpu' */
2810 		pmc_save_cpu_binding(&pb);
2811 		pmc_select_cpu(cpu);
2812 
2813 		npmc = md->pmd_npmc;
2814 
2815 		pmcinfo_size = npmc * sizeof(struct pmc_info);
2816 		pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
2817 
2818 		p = pmcinfo;
2819 
2820 		for (n = 0; n < md->pmd_npmc; n++, p++) {
2821 
2822 			if ((error = md->pmd_describe(cpu, n, p, &pm)) != 0)
2823 				break;
2824 
2825 			if (PMC_ROW_DISP_IS_STANDALONE(n))
2826 				p->pm_rowdisp = PMC_DISP_STANDALONE;
2827 			else if (PMC_ROW_DISP_IS_THREAD(n))
2828 				p->pm_rowdisp = PMC_DISP_THREAD;
2829 			else
2830 				p->pm_rowdisp = PMC_DISP_FREE;
2831 
2832 			p->pm_ownerpid = -1;
2833 
2834 			if (pm == NULL)	/* no PMC associated */
2835 				continue;
2836 
2837 			po = pm->pm_owner;
2838 
2839 			KASSERT(po->po_owner != NULL,
2840 			    ("[pmc,%d] pmc_owner had a null proc pointer",
2841 				__LINE__));
2842 
2843 			p->pm_ownerpid = po->po_owner->p_pid;
2844 			p->pm_mode     = PMC_TO_MODE(pm);
2845 			p->pm_event    = pm->pm_event;
2846 			p->pm_flags    = pm->pm_flags;
2847 
2848 			if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2849 				p->pm_reloadcount =
2850 				    pm->pm_sc.pm_reloadcount;
2851 		}
2852 
2853 		pmc_restore_cpu_binding(&pb);
2854 
2855 		/* now copy out the PMC info collected */
2856 		if (error == 0)
2857 			error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
2858 
2859 		free(pmcinfo, M_PMC);
2860 	}
2861 	break;
2862 
2863 
2864 	/*
2865 	 * Set the administrative state of a PMC.  I.e. whether
2866 	 * the PMC is to be used or not.
2867 	 */
2868 
2869 	case PMC_OP_PMCADMIN:
2870 	{
2871 		int cpu, ri;
2872 		enum pmc_state request;
2873 		struct pmc_cpu *pc;
2874 		struct pmc_hw *phw;
2875 		struct pmc_op_pmcadmin pma;
2876 		struct pmc_binding pb;
2877 
2878 		sx_assert(&pmc_sx, SX_XLOCKED);
2879 
2880 		KASSERT(td == curthread,
2881 		    ("[pmc,%d] td != curthread", __LINE__));
2882 
2883 		error = priv_check(td, PRIV_PMC_MANAGE);
2884 		if (error)
2885 			break;
2886 
2887 		if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
2888 			break;
2889 
2890 		cpu = pma.pm_cpu;
2891 
2892 		if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
2893 			error = EINVAL;
2894 			break;
2895 		}
2896 
2897 		if (!pmc_cpu_is_active(cpu)) {
2898 			error = ENXIO;
2899 			break;
2900 		}
2901 
2902 		request = pma.pm_state;
2903 
2904 		if (request != PMC_STATE_DISABLED &&
2905 		    request != PMC_STATE_FREE) {
2906 			error = EINVAL;
2907 			break;
2908 		}
2909 
2910 		ri = pma.pm_pmc; /* pmc id == row index */
2911 		if (ri < 0 || ri >= (int) md->pmd_npmc) {
2912 			error = EINVAL;
2913 			break;
2914 		}
2915 
2916 		/*
2917 		 * We can't disable a PMC with a row-index allocated
2918 		 * for process virtual PMCs.
2919 		 */
2920 
2921 		if (PMC_ROW_DISP_IS_THREAD(ri) &&
2922 		    request == PMC_STATE_DISABLED) {
2923 			error = EBUSY;
2924 			break;
2925 		}
2926 
2927 		/*
2928 		 * otherwise, this PMC on this CPU is either free or
2929 		 * in system-wide mode.
2930 		 */
2931 
2932 		pmc_save_cpu_binding(&pb);
2933 		pmc_select_cpu(cpu);
2934 
2935 		pc  = pmc_pcpu[cpu];
2936 		phw = pc->pc_hwpmcs[ri];
2937 
2938 		/*
2939 		 * XXX do we need some kind of 'forced' disable?
2940 		 */
2941 
2942 		if (phw->phw_pmc == NULL) {
2943 			if (request == PMC_STATE_DISABLED &&
2944 			    (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
2945 				phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
2946 				PMC_MARK_ROW_STANDALONE(ri);
2947 			} else if (request == PMC_STATE_FREE &&
2948 			    (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
2949 				phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
2950 				PMC_UNMARK_ROW_STANDALONE(ri);
2951 			}
2952 			/* other cases are a no-op */
2953 		} else
2954 			error = EBUSY;
2955 
2956 		pmc_restore_cpu_binding(&pb);
2957 	}
2958 	break;
2959 
2960 
2961 	/*
2962 	 * Allocate a PMC.
2963 	 */
2964 
2965 	case PMC_OP_PMCALLOCATE:
2966 	{
2967 		uint32_t caps;
2968 		u_int cpu;
2969 		int n;
2970 		enum pmc_mode mode;
2971 		struct pmc *pmc;
2972 		struct pmc_hw *phw;
2973 		struct pmc_op_pmcallocate pa;
2974 		struct pmc_binding pb;
2975 
2976 		if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
2977 			break;
2978 
2979 		caps = pa.pm_caps;
2980 		mode = pa.pm_mode;
2981 		cpu  = pa.pm_cpu;
2982 
2983 		if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
2984 		     mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
2985 		    (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
2986 			error = EINVAL;
2987 			break;
2988 		}
2989 
2990 		/*
2991 		 * Virtual PMCs should only ask for a default CPU.
2992 		 * System mode PMCs need to specify a non-default CPU.
2993 		 */
2994 
2995 		if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
2996 		    (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
2997 			error = EINVAL;
2998 			break;
2999 		}
3000 
3001 		/*
3002 		 * Check that an inactive CPU is not being asked for.
3003 		 */
3004 
3005 		if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
3006 			error = ENXIO;
3007 			break;
3008 		}
3009 
3010 		/*
3011 		 * Refuse an allocation for a system-wide PMC if this
3012 		 * process has been jailed, or if this process lacks
3013 		 * super-user credentials and the sysctl tunable
3014 		 * 'security.bsd.unprivileged_syspmcs' is zero.
3015 		 */
3016 
3017 		if (PMC_IS_SYSTEM_MODE(mode)) {
3018 			if (jailed(curthread->td_ucred)) {
3019 				error = EPERM;
3020 				break;
3021 			}
3022 			if (!pmc_unprivileged_syspmcs) {
3023 				error = priv_check(curthread,
3024 				    PRIV_PMC_SYSTEM);
3025 				if (error)
3026 					break;
3027 			}
3028 		}
3029 
3030 		if (error)
3031 			break;
3032 
3033 		/*
3034 		 * Look for valid values for 'pm_flags'
3035 		 */
3036 
3037 		if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
3038 		    PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) {
3039 			error = EINVAL;
3040 			break;
3041 		}
3042 
3043 		/* process logging options are not allowed for system PMCs */
3044 		if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3045 		    (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3046 			error = EINVAL;
3047 			break;
3048 		}
3049 
3050 		/*
3051 		 * All sampling mode PMCs need to be able to interrupt the
3052 		 * CPU.
3053 		 */
3054 		if (PMC_IS_SAMPLING_MODE(mode))
3055 			caps |= PMC_CAP_INTERRUPT;
3056 
3057 		/* A valid class specifier should have been passed in. */
3058 		for (n = 0; n < md->pmd_nclass; n++)
3059 			if (md->pmd_classes[n].pm_class == pa.pm_class)
3060 				break;
3061 		if (n == md->pmd_nclass) {
3062 			error = EINVAL;
3063 			break;
3064 		}
3065 
3066 		/* The requested PMC capabilities should be feasible. */
3067 		if ((md->pmd_classes[n].pm_caps & caps) != caps) {
3068 			error = EOPNOTSUPP;
3069 			break;
3070 		}
3071 
3072 		PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3073 		    pa.pm_ev, caps, mode, cpu);
3074 
3075 		pmc = pmc_allocate_pmc_descriptor();
3076 		pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
3077 		    PMC_ID_INVALID);
3078 		pmc->pm_event = pa.pm_ev;
3079 		pmc->pm_state = PMC_STATE_FREE;
3080 		pmc->pm_caps  = caps;
3081 		pmc->pm_flags = pa.pm_flags;
3082 
3083 		/* switch thread to CPU 'cpu' */
3084 		pmc_save_cpu_binding(&pb);
3085 
3086 #define	PMC_IS_SHAREABLE_PMC(cpu, n)				\
3087 	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &		\
3088 	 PMC_PHW_FLAG_IS_SHAREABLE)
3089 #define	PMC_IS_UNALLOCATED(cpu, n)				\
3090 	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3091 
3092 		if (PMC_IS_SYSTEM_MODE(mode)) {
3093 			pmc_select_cpu(cpu);
3094 			for (n = 0; n < (int) md->pmd_npmc; n++)
3095 				if (pmc_can_allocate_row(n, mode) == 0 &&
3096 				    pmc_can_allocate_rowindex(
3097 					    curthread->td_proc, n, cpu) == 0 &&
3098 				    (PMC_IS_UNALLOCATED(cpu, n) ||
3099 				     PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3100 				    md->pmd_allocate_pmc(cpu, n, pmc,
3101 					&pa) == 0)
3102 					break;
3103 		} else {
3104 			/* Process virtual mode */
3105 			for (n = 0; n < (int) md->pmd_npmc; n++) {
3106 				if (pmc_can_allocate_row(n, mode) == 0 &&
3107 				    pmc_can_allocate_rowindex(
3108 					    curthread->td_proc, n,
3109 					    PMC_CPU_ANY) == 0 &&
3110 				    md->pmd_allocate_pmc(curthread->td_oncpu,
3111 					n, pmc, &pa) == 0)
3112 					break;
3113 			}
3114 		}
3115 
3116 #undef	PMC_IS_UNALLOCATED
3117 #undef	PMC_IS_SHAREABLE_PMC
3118 
3119 		pmc_restore_cpu_binding(&pb);
3120 
3121 		if (n == (int) md->pmd_npmc) {
3122 			pmc_destroy_pmc_descriptor(pmc);
3123 			free(pmc, M_PMC);
3124 			pmc = NULL;
3125 			error = EINVAL;
3126 			break;
3127 		}
3128 
3129 		/* Fill in the correct value in the ID field */
3130 		pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
3131 
3132 		PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
3133 		    pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
3134 
3135 		/* Process mode PMCs with logging enabled need log files */
3136 		if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
3137 			pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3138 
3139 		/* All system mode sampling PMCs require a log file */
3140 		if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
3141 			pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3142 
3143 		/*
3144 		 * Configure global pmc's immediately
3145 		 */
3146 
3147 		if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
3148 
3149 			pmc_save_cpu_binding(&pb);
3150 			pmc_select_cpu(cpu);
3151 
3152 			phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
3153 
3154 			if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
3155 			    (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
3156 				(void) md->pmd_release_pmc(cpu, n, pmc);
3157 				pmc_destroy_pmc_descriptor(pmc);
3158 				free(pmc, M_PMC);
3159 				pmc = NULL;
3160 				pmc_restore_cpu_binding(&pb);
3161 				error = EPERM;
3162 				break;
3163 			}
3164 
3165 			pmc_restore_cpu_binding(&pb);
3166 		}
3167 
3168 		pmc->pm_state    = PMC_STATE_ALLOCATED;
3169 
3170 		/*
3171 		 * mark row disposition
3172 		 */
3173 
3174 		if (PMC_IS_SYSTEM_MODE(mode))
3175 			PMC_MARK_ROW_STANDALONE(n);
3176 		else
3177 			PMC_MARK_ROW_THREAD(n);
3178 
3179 		/*
3180 		 * Register this PMC with the current thread as its owner.
3181 		 */
3182 
3183 		if ((error =
3184 		    pmc_register_owner(curthread->td_proc, pmc)) != 0) {
3185 			pmc_release_pmc_descriptor(pmc);
3186 			free(pmc, M_PMC);
3187 			pmc = NULL;
3188 			break;
3189 		}
3190 
3191 		/*
3192 		 * Return the allocated index.
3193 		 */
3194 
3195 		pa.pm_pmcid = pmc->pm_id;
3196 
3197 		error = copyout(&pa, arg, sizeof(pa));
3198 	}
3199 	break;
3200 
3201 
3202 	/*
3203 	 * Attach a PMC to a process.
3204 	 */
3205 
3206 	case PMC_OP_PMCATTACH:
3207 	{
3208 		struct pmc *pm;
3209 		struct proc *p;
3210 		struct pmc_op_pmcattach a;
3211 
3212 		sx_assert(&pmc_sx, SX_XLOCKED);
3213 
3214 		if ((error = copyin(arg, &a, sizeof(a))) != 0)
3215 			break;
3216 
3217 		if (a.pm_pid < 0) {
3218 			error = EINVAL;
3219 			break;
3220 		} else if (a.pm_pid == 0)
3221 			a.pm_pid = td->td_proc->p_pid;
3222 
3223 		if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3224 			break;
3225 
3226 		if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
3227 			error = EINVAL;
3228 			break;
3229 		}
3230 
3231 		/* PMCs may be (re)attached only when allocated or stopped */
3232 		if (pm->pm_state == PMC_STATE_RUNNING) {
3233 			error = EBUSY;
3234 			break;
3235 		} else if (pm->pm_state != PMC_STATE_ALLOCATED &&
3236 		    pm->pm_state != PMC_STATE_STOPPED) {
3237 			error = EINVAL;
3238 			break;
3239 		}
3240 
3241 		/* lookup pid */
3242 		if ((p = pfind(a.pm_pid)) == NULL) {
3243 			error = ESRCH;
3244 			break;
3245 		}
3246 
3247 		/*
3248 		 * Ignore processes that are working on exiting.
3249 		 */
3250 		if (p->p_flag & P_WEXIT) {
3251 			error = ESRCH;
3252 			PROC_UNLOCK(p);	/* pfind() returns a locked process */
3253 			break;
3254 		}
3255 
3256 		/*
3257 		 * we are allowed to attach a PMC to a process if
3258 		 * we can debug it.
3259 		 */
3260 		error = p_candebug(curthread, p);
3261 
3262 		PROC_UNLOCK(p);
3263 
3264 		if (error == 0)
3265 			error = pmc_attach_process(p, pm);
3266 	}
3267 	break;
3268 
3269 
3270 	/*
3271 	 * Detach an attached PMC from a process.
3272 	 */
3273 
3274 	case PMC_OP_PMCDETACH:
3275 	{
3276 		struct pmc *pm;
3277 		struct proc *p;
3278 		struct pmc_op_pmcattach a;
3279 
3280 		if ((error = copyin(arg, &a, sizeof(a))) != 0)
3281 			break;
3282 
3283 		if (a.pm_pid < 0) {
3284 			error = EINVAL;
3285 			break;
3286 		} else if (a.pm_pid == 0)
3287 			a.pm_pid = td->td_proc->p_pid;
3288 
3289 		if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3290 			break;
3291 
3292 		if ((p = pfind(a.pm_pid)) == NULL) {
3293 			error = ESRCH;
3294 			break;
3295 		}
3296 
3297 		/*
3298 		 * Treat processes that are in the process of exiting
3299 		 * as if they were not present.
3300 		 */
3301 
3302 		if (p->p_flag & P_WEXIT)
3303 			error = ESRCH;
3304 
3305 		PROC_UNLOCK(p);	/* pfind() returns a locked process */
3306 
3307 		if (error == 0)
3308 			error = pmc_detach_process(p, pm);
3309 	}
3310 	break;
3311 
3312 
3313 	/*
3314 	 * Retrieve the MSR number associated with the counter
3315 	 * 'pmc_id'.  This allows processes to directly use RDPMC
3316 	 * instructions to read their PMCs, without the overhead of a
3317 	 * system call.
3318 	 */
3319 
3320 	case PMC_OP_PMCGETMSR:
3321 	{
3322 		int ri;
3323 		struct pmc	*pm;
3324 		struct pmc_target *pt;
3325 		struct pmc_op_getmsr gm;
3326 
3327 		PMC_DOWNGRADE_SX();
3328 
3329 		/* CPU has no 'GETMSR' support */
3330 		if (md->pmd_get_msr == NULL) {
3331 			error = ENOSYS;
3332 			break;
3333 		}
3334 
3335 		if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3336 			break;
3337 
3338 		if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3339 			break;
3340 
3341 		/*
3342 		 * The allocated PMC has to be a process virtual PMC,
3343 		 * i.e., of type MODE_T[CS].  Global PMCs can only be
3344 		 * read using the PMCREAD operation since they may be
3345 		 * allocated on a different CPU than the one we could
3346 		 * be running on at the time of the RDPMC instruction.
3347 		 *
3348 		 * The GETMSR operation is not allowed for PMCs that
3349 		 * are inherited across processes.
3350 		 */
3351 
3352 		if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
3353 		    (pm->pm_flags & PMC_F_DESCENDANTS)) {
3354 			error = EINVAL;
3355 			break;
3356 		}
3357 
3358 		/*
3359 		 * It only makes sense to use a RDPMC (or its
3360 		 * equivalent instruction on non-x86 architectures) on
3361 		 * a process that has allocated and attached a PMC to
3362 		 * itself.  Conversely the PMC is only allowed to have
3363 		 * one process attached to it -- its owner.
3364 		 */
3365 
3366 		if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3367 		    LIST_NEXT(pt, pt_next) != NULL ||
3368 		    pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3369 			error = EINVAL;
3370 			break;
3371 		}
3372 
3373 		ri = PMC_TO_ROWINDEX(pm);
3374 
3375 		if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0)
3376 			break;
3377 
3378 		if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3379 			break;
3380 
3381 		/*
3382 		 * Mark our process as using MSRs.  Update machine
3383 		 * state using a forced context switch.
3384 		 */
3385 
3386 		pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
3387 		pmc_force_context_switch();
3388 
3389 	}
3390 	break;
3391 
3392 	/*
3393 	 * Release an allocated PMC
3394 	 */
3395 
3396 	case PMC_OP_PMCRELEASE:
3397 	{
3398 		pmc_id_t pmcid;
3399 		struct pmc *pm;
3400 		struct pmc_owner *po;
3401 		struct pmc_op_simple sp;
3402 
3403 		/*
3404 		 * Find PMC pointer for the named PMC.
3405 		 *
3406 		 * Use pmc_release_pmc_descriptor() to switch off the
3407 		 * PMC, remove all its target threads, and remove the
3408 		 * PMC from its owner's list.
3409 		 *
3410 		 * Remove the owner record if this is the last PMC
3411 		 * owned.
3412 		 *
3413 		 * Free up space.
3414 		 */
3415 
3416 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3417 			break;
3418 
3419 		pmcid = sp.pm_pmcid;
3420 
3421 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3422 			break;
3423 
3424 		po = pm->pm_owner;
3425 		pmc_release_pmc_descriptor(pm);
3426 		pmc_maybe_remove_owner(po);
3427 
3428 		free(pm, M_PMC);
3429 	}
3430 	break;
3431 
3432 
3433 	/*
3434 	 * Read and/or write a PMC.
3435 	 */
3436 
3437 	case PMC_OP_PMCRW:
3438 	{
3439 		uint32_t cpu, ri;
3440 		struct pmc *pm;
3441 		struct pmc_op_pmcrw *pprw;
3442 		struct pmc_op_pmcrw prw;
3443 		struct pmc_binding pb;
3444 		pmc_value_t oldvalue;
3445 
3446 		PMC_DOWNGRADE_SX();
3447 
3448 		if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
3449 			break;
3450 
3451 		ri = 0;
3452 		PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
3453 		    prw.pm_flags);
3454 
3455 		/* must have at least one flag set */
3456 		if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
3457 			error = EINVAL;
3458 			break;
3459 		}
3460 
3461 		/* locate pmc descriptor */
3462 		if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
3463 			break;
3464 
3465 		/* Can't read a PMC that hasn't been started. */
3466 		if (pm->pm_state != PMC_STATE_ALLOCATED &&
3467 		    pm->pm_state != PMC_STATE_STOPPED &&
3468 		    pm->pm_state != PMC_STATE_RUNNING) {
3469 			error = EINVAL;
3470 			break;
3471 		}
3472 
3473 		/* writing a new value is allowed only for 'STOPPED' pmcs */
3474 		if (pm->pm_state == PMC_STATE_RUNNING &&
3475 		    (prw.pm_flags & PMC_F_NEWVALUE)) {
3476 			error = EBUSY;
3477 			break;
3478 		}
3479 
3480 		if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
3481 
3482 			/*
3483 			 * If this PMC is attached to its owner (i.e.,
3484 			 * the process requesting this operation) and
3485 			 * is running, then attempt to get an
3486 			 * upto-date reading from hardware for a READ.
3487 			 * Writes are only allowed when the PMC is
3488 			 * stopped, so only update the saved value
3489 			 * field.
3490 			 *
3491 			 * If the PMC is not running, or is not
3492 			 * attached to its owner, read/write to the
3493 			 * savedvalue field.
3494 			 */
3495 
3496 			ri = PMC_TO_ROWINDEX(pm);
3497 
3498 			mtx_pool_lock_spin(pmc_mtxpool, pm);
3499 			cpu = curthread->td_oncpu;
3500 
3501 			if (prw.pm_flags & PMC_F_OLDVALUE) {
3502 				if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3503 				    (pm->pm_state == PMC_STATE_RUNNING))
3504 					error = (*md->pmd_read_pmc)(cpu, ri,
3505 					    &oldvalue);
3506 				else
3507 					oldvalue = pm->pm_gv.pm_savedvalue;
3508 			}
3509 			if (prw.pm_flags & PMC_F_NEWVALUE)
3510 				pm->pm_gv.pm_savedvalue = prw.pm_value;
3511 
3512 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
3513 
3514 		} else { /* System mode PMCs */
3515 			cpu = PMC_TO_CPU(pm);
3516 			ri  = PMC_TO_ROWINDEX(pm);
3517 
3518 			if (!pmc_cpu_is_active(cpu)) {
3519 				error = ENXIO;
3520 				break;
3521 			}
3522 
3523 			/* move this thread to CPU 'cpu' */
3524 			pmc_save_cpu_binding(&pb);
3525 			pmc_select_cpu(cpu);
3526 
3527 			critical_enter();
3528 			/* save old value */
3529 			if (prw.pm_flags & PMC_F_OLDVALUE)
3530 				if ((error = (*md->pmd_read_pmc)(cpu, ri,
3531 					 &oldvalue)))
3532 					goto error;
3533 			/* write out new value */
3534 			if (prw.pm_flags & PMC_F_NEWVALUE)
3535 				error = (*md->pmd_write_pmc)(cpu, ri,
3536 				    prw.pm_value);
3537 		error:
3538 			critical_exit();
3539 			pmc_restore_cpu_binding(&pb);
3540 			if (error)
3541 				break;
3542 		}
3543 
3544 		pprw = (struct pmc_op_pmcrw *) arg;
3545 
3546 #ifdef	DEBUG
3547 		if (prw.pm_flags & PMC_F_NEWVALUE)
3548 			PMCDBG(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
3549 			    ri, prw.pm_value, oldvalue);
3550 		else if (prw.pm_flags & PMC_F_OLDVALUE)
3551 			PMCDBG(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
3552 #endif
3553 
3554 		/* return old value if requested */
3555 		if (prw.pm_flags & PMC_F_OLDVALUE)
3556 			if ((error = copyout(&oldvalue, &pprw->pm_value,
3557 				 sizeof(prw.pm_value))))
3558 				break;
3559 
3560 	}
3561 	break;
3562 
3563 
3564 	/*
3565 	 * Set the sampling rate for a sampling mode PMC and the
3566 	 * initial count for a counting mode PMC.
3567 	 */
3568 
3569 	case PMC_OP_PMCSETCOUNT:
3570 	{
3571 		struct pmc *pm;
3572 		struct pmc_op_pmcsetcount sc;
3573 
3574 		PMC_DOWNGRADE_SX();
3575 
3576 		if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
3577 			break;
3578 
3579 		if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
3580 			break;
3581 
3582 		if (pm->pm_state == PMC_STATE_RUNNING) {
3583 			error = EBUSY;
3584 			break;
3585 		}
3586 
3587 		if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3588 			pm->pm_sc.pm_reloadcount = sc.pm_count;
3589 		else
3590 			pm->pm_sc.pm_initial = sc.pm_count;
3591 	}
3592 	break;
3593 
3594 
3595 	/*
3596 	 * Start a PMC.
3597 	 */
3598 
3599 	case PMC_OP_PMCSTART:
3600 	{
3601 		pmc_id_t pmcid;
3602 		struct pmc *pm;
3603 		struct pmc_op_simple sp;
3604 
3605 		sx_assert(&pmc_sx, SX_XLOCKED);
3606 
3607 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3608 			break;
3609 
3610 		pmcid = sp.pm_pmcid;
3611 
3612 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3613 			break;
3614 
3615 		KASSERT(pmcid == pm->pm_id,
3616 		    ("[pmc,%d] pmcid %x != id %x", __LINE__,
3617 			pm->pm_id, pmcid));
3618 
3619 		if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
3620 			break;
3621 		else if (pm->pm_state != PMC_STATE_STOPPED &&
3622 		    pm->pm_state != PMC_STATE_ALLOCATED) {
3623 			error = EINVAL;
3624 			break;
3625 		}
3626 
3627 		error = pmc_start(pm);
3628 	}
3629 	break;
3630 
3631 
3632 	/*
3633 	 * Stop a PMC.
3634 	 */
3635 
3636 	case PMC_OP_PMCSTOP:
3637 	{
3638 		pmc_id_t pmcid;
3639 		struct pmc *pm;
3640 		struct pmc_op_simple sp;
3641 
3642 		PMC_DOWNGRADE_SX();
3643 
3644 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3645 			break;
3646 
3647 		pmcid = sp.pm_pmcid;
3648 
3649 		/*
3650 		 * Mark the PMC as inactive and invoke the MD stop
3651 		 * routines if needed.
3652 		 */
3653 
3654 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3655 			break;
3656 
3657 		KASSERT(pmcid == pm->pm_id,
3658 		    ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
3659 			pm->pm_id, pmcid));
3660 
3661 		if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
3662 			break;
3663 		else if (pm->pm_state != PMC_STATE_RUNNING) {
3664 			error = EINVAL;
3665 			break;
3666 		}
3667 
3668 		error = pmc_stop(pm);
3669 	}
3670 	break;
3671 
3672 
3673 	/*
3674 	 * Write a user supplied value to the log file.
3675 	 */
3676 
3677 	case PMC_OP_WRITELOG:
3678 	{
3679 		struct pmc_op_writelog wl;
3680 		struct pmc_owner *po;
3681 
3682 		PMC_DOWNGRADE_SX();
3683 
3684 		if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
3685 			break;
3686 
3687 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3688 			error = EINVAL;
3689 			break;
3690 		}
3691 
3692 		if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
3693 			error = EINVAL;
3694 			break;
3695 		}
3696 
3697 		error = pmclog_process_userlog(po, &wl);
3698 	}
3699 	break;
3700 
3701 
3702 	default:
3703 		error = EINVAL;
3704 		break;
3705 	}
3706 
3707 	if (is_sx_downgraded)
3708 		sx_sunlock(&pmc_sx);
3709 	else
3710 		sx_xunlock(&pmc_sx);
3711 
3712 	if (error)
3713 		atomic_add_int(&pmc_stats.pm_syscall_errors, 1);
3714 
3715 	PICKUP_GIANT();
3716 
3717 	return error;
3718 }
3719 
3720 /*
3721  * Helper functions
3722  */
3723 
3724 
3725 /*
3726  * Mark the thread as needing callchain capture and post an AST.  The
3727  * actual callchain capture will be done in a context where it is safe
3728  * to take page faults.
3729  */
3730 
3731 static void
3732 pmc_post_callchain_ast(void)
3733 {
3734 	struct thread *td;
3735 
3736 	td = curthread;
3737 
3738 	/*
3739 	 * Mark this thread as needing processing in ast().
3740 	 * td->td_pflags will be safe to touch as the process was in
3741 	 * user space when it was interrupted.
3742 	 */
3743 	td->td_pflags |= TDP_CALLCHAIN;
3744 
3745 	/*
3746 	 * Again, since we've entered this function directly from
3747 	 * userland, `td' is guaranteed to be not locked by this CPU,
3748 	 * so its safe to try acquire the thread lock even though we
3749 	 * are executing in an NMI context.  We need to acquire this
3750 	 * lock before touching `td_flags' because other CPUs may be
3751 	 * in the process of touching this field.
3752 	 */
3753 	thread_lock(td);
3754 	td->td_flags |= TDF_ASTPENDING;
3755 	thread_unlock(td);
3756 
3757 	return;
3758 }
3759 
3760 /*
3761  * Interrupt processing.
3762  *
3763  * Find a free slot in the per-cpu array of samples and capture the
3764  * current callchain there.  If a sample was successfully added, a bit
3765  * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
3766  * needs to be invoked from the clock handler.
3767  *
3768  * This function is meant to be called from an NMI handler.  It cannot
3769  * use any of the locking primitives supplied by the OS.
3770  */
3771 
3772 int
3773 pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
3774     int inuserspace)
3775 {
3776 	int error, callchaindepth;
3777 	struct thread *td;
3778 	struct pmc_sample *ps;
3779 	struct pmc_samplebuffer *psb;
3780 
3781 	error = 0;
3782 
3783 	/*
3784 	 * Allocate space for a sample buffer.
3785 	 */
3786 	psb = pmc_pcpu[cpu]->pc_sb;
3787 
3788 	ps = psb->ps_write;
3789 	if (ps->ps_nsamples) {	/* in use, reader hasn't caught up */
3790 		pm->pm_stalled = 1;
3791 		atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
3792 		PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
3793 		    cpu, pm, (void *) tf, inuserspace,
3794 		    (int) (psb->ps_write - psb->ps_samples),
3795 		    (int) (psb->ps_read - psb->ps_samples));
3796 		error = ENOMEM;
3797 		goto done;
3798 	}
3799 
3800 
3801 	/* Fill in entry. */
3802 	PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
3803 	    (void *) tf, inuserspace,
3804 	    (int) (psb->ps_write - psb->ps_samples),
3805 	    (int) (psb->ps_read - psb->ps_samples));
3806 
3807 	atomic_add_rel_32(&pm->pm_runcount, 1);	/* hold onto PMC */
3808 	ps->ps_pmc = pm;
3809 	if ((td = curthread) && td->td_proc)
3810 		ps->ps_pid = td->td_proc->p_pid;
3811 	else
3812 		ps->ps_pid = -1;
3813 	ps->ps_cpu = cpu;
3814 	ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
3815 
3816 	callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
3817 	    pmc_callchaindepth : 1;
3818 
3819 	if (callchaindepth == 1)
3820 		ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
3821 	else {
3822 		/*
3823 		 * Kernel stack traversals can be done immediately,
3824 		 * while we defer to an AST for user space traversals.
3825 		 */
3826 		if (!inuserspace)
3827 			callchaindepth =
3828 			    pmc_save_kernel_callchain(ps->ps_pc,
3829 				callchaindepth, tf);
3830 		else {
3831 			pmc_post_callchain_ast();
3832 			callchaindepth = PMC_SAMPLE_INUSE;
3833 		}
3834 	}
3835 
3836 	ps->ps_nsamples = callchaindepth;	/* mark entry as in use */
3837 
3838 	/* increment write pointer, modulo ring buffer size */
3839 	ps++;
3840 	if (ps == psb->ps_fence)
3841 		psb->ps_write = psb->ps_samples;
3842 	else
3843 		psb->ps_write = ps;
3844 
3845  done:
3846 	/* mark CPU as needing processing */
3847 	atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3848 
3849 	return (error);
3850 }
3851 
3852 /*
3853  * Capture a user call chain.  This function will be called from ast()
3854  * before control returns to userland and before the process gets
3855  * rescheduled.
3856  */
3857 
3858 static void
3859 pmc_capture_user_callchain(int cpu, struct trapframe *tf)
3860 {
3861 	int i;
3862 	struct pmc *pm;
3863 	struct pmc_sample *ps;
3864 	struct pmc_samplebuffer *psb;
3865 
3866 	psb = pmc_pcpu[cpu]->pc_sb;
3867 
3868 	/*
3869 	 * Iterate through all deferred callchain requests.
3870 	 */
3871 
3872 	for (i = 0; i < pmc_nsamples; i++) {
3873 
3874 		ps = &psb->ps_samples[i];
3875 		if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
3876 			continue;
3877 
3878 		pm = ps->ps_pmc;
3879 
3880 		KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
3881 		    ("[pmc,%d] Retrieving callchain for PMC that doesn't "
3882 			"want it", __LINE__));
3883 
3884 		/*
3885 		 * Retrieve the callchain and mark the sample buffer
3886 		 * as 'processable' by the timer tick sweep code.
3887 		 */
3888 		ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc,
3889 		    pmc_callchaindepth, tf);
3890 	}
3891 
3892 	return;
3893 }
3894 
3895 
3896 /*
3897  * Process saved PC samples.
3898  */
3899 
3900 static void
3901 pmc_process_samples(int cpu)
3902 {
3903 	int n, ri;
3904 	struct pmc *pm;
3905 	struct thread *td;
3906 	struct pmc_owner *po;
3907 	struct pmc_sample *ps;
3908 	struct pmc_samplebuffer *psb;
3909 
3910 	KASSERT(PCPU_GET(cpuid) == cpu,
3911 	    ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
3912 		PCPU_GET(cpuid), cpu));
3913 
3914 	psb = pmc_pcpu[cpu]->pc_sb;
3915 
3916 	for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
3917 
3918 		ps = psb->ps_read;
3919 		if (ps->ps_nsamples == PMC_SAMPLE_FREE)
3920 			break;
3921 		if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
3922 			/* Need a rescan at a later time. */
3923 			atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3924 			break;
3925 		}
3926 
3927 		pm = ps->ps_pmc;
3928 		po = pm->pm_owner;
3929 
3930 		KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
3931 		    ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
3932 			pm, PMC_TO_MODE(pm)));
3933 
3934 		/* Ignore PMCs that have been switched off */
3935 		if (pm->pm_state != PMC_STATE_RUNNING)
3936 			goto entrydone;
3937 
3938 		PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
3939 		    pm, ps->ps_nsamples, ps->ps_flags,
3940 		    (int) (psb->ps_write - psb->ps_samples),
3941 		    (int) (psb->ps_read - psb->ps_samples));
3942 
3943 		/*
3944 		 * If this is a process-mode PMC that is attached to
3945 		 * its owner, and if the PC is in user mode, update
3946 		 * profiling statistics like timer-based profiling
3947 		 * would have done.
3948 		 */
3949 		if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
3950 			if (ps->ps_flags & PMC_CC_F_USERSPACE) {
3951 				td = FIRST_THREAD_IN_PROC(po->po_owner);
3952 				addupc_intr(td, ps->ps_pc[0], 1);
3953 			}
3954 			goto entrydone;
3955 		}
3956 
3957 		/*
3958 		 * Otherwise, this is either a sampling mode PMC that
3959 		 * is attached to a different process than its owner,
3960 		 * or a system-wide sampling PMC.  Dispatch a log
3961 		 * entry to the PMC's owner process.
3962 		 */
3963 
3964 		pmclog_process_callchain(pm, ps);
3965 
3966 	entrydone:
3967 		ps->ps_nsamples = 0;	/* mark entry as free */
3968 		atomic_subtract_rel_32(&pm->pm_runcount, 1);
3969 
3970 		/* increment read pointer, modulo sample size */
3971 		if (++ps == psb->ps_fence)
3972 			psb->ps_read = psb->ps_samples;
3973 		else
3974 			psb->ps_read = ps;
3975 	}
3976 
3977 	atomic_add_int(&pmc_stats.pm_log_sweeps, 1);
3978 
3979 	/* Do not re-enable stalled PMCs if we failed to process any samples */
3980 	if (n == 0)
3981 		return;
3982 
3983 	/*
3984 	 * Restart any stalled sampling PMCs on this CPU.
3985 	 *
3986 	 * If the NMI handler sets the pm_stalled field of a PMC after
3987 	 * the check below, we'll end up processing the stalled PMC at
3988 	 * the next hardclock tick.
3989 	 */
3990 	for (n = 0; n < md->pmd_npmc; n++) {
3991 		(void) (*md->pmd_get_config)(cpu,n,&pm);
3992 		if (pm == NULL ||			 /* !cfg'ed */
3993 		    pm->pm_state != PMC_STATE_RUNNING || /* !active */
3994 		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
3995 		    pm->pm_stalled == 0) /* !stalled */
3996 			continue;
3997 
3998 		pm->pm_stalled = 0;
3999 		ri = PMC_TO_ROWINDEX(pm);
4000 		(*md->pmd_start_pmc)(cpu, ri);
4001 	}
4002 }
4003 
4004 /*
4005  * Event handlers.
4006  */
4007 
4008 /*
4009  * Handle a process exit.
4010  *
4011  * Remove this process from all hash tables.  If this process
4012  * owned any PMCs, turn off those PMCs and deallocate them,
4013  * removing any associations with target processes.
4014  *
4015  * This function will be called by the last 'thread' of a
4016  * process.
4017  *
4018  * XXX This eventhandler gets called early in the exit process.
4019  * Consider using a 'hook' invocation from thread_exit() or equivalent
4020  * spot.  Another negative is that kse_exit doesn't seem to call
4021  * exit1() [??].
4022  *
4023  */
4024 
4025 static void
4026 pmc_process_exit(void *arg __unused, struct proc *p)
4027 {
4028 	int is_using_hwpmcs;
4029 	int cpu;
4030 	unsigned int ri;
4031 	struct pmc *pm;
4032 	struct pmc_process *pp;
4033 	struct pmc_owner *po;
4034 	pmc_value_t newvalue, tmp;
4035 
4036 	PROC_LOCK(p);
4037 	is_using_hwpmcs = p->p_flag & P_HWPMC;
4038 	PROC_UNLOCK(p);
4039 
4040 	/*
4041 	 * Log a sysexit event to all SS PMC owners.
4042 	 */
4043 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
4044 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
4045 		    pmclog_process_sysexit(po, p->p_pid);
4046 
4047 	if (!is_using_hwpmcs)
4048 		return;
4049 
4050 	PMC_GET_SX_XLOCK();
4051 	PMCDBG(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
4052 	    p->p_comm);
4053 
4054 	/*
4055 	 * Since this code is invoked by the last thread in an exiting
4056 	 * process, we would have context switched IN at some prior
4057 	 * point.  However, with PREEMPTION, kernel mode context
4058 	 * switches may happen any time, so we want to disable a
4059 	 * context switch OUT till we get any PMCs targetting this
4060 	 * process off the hardware.
4061 	 *
4062 	 * We also need to atomically remove this process'
4063 	 * entry from our target process hash table, using
4064 	 * PMC_FLAG_REMOVE.
4065 	 */
4066 	PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
4067 	    p->p_comm);
4068 
4069 	critical_enter(); /* no preemption */
4070 
4071 	cpu = curthread->td_oncpu;
4072 
4073 	if ((pp = pmc_find_process_descriptor(p,
4074 		 PMC_FLAG_REMOVE)) != NULL) {
4075 
4076 		PMCDBG(PRC,EXT,2,
4077 		    "process-exit proc=%p pmc-process=%p", p, pp);
4078 
4079 		/*
4080 		 * The exiting process could the target of
4081 		 * some PMCs which will be running on
4082 		 * currently executing CPU.
4083 		 *
4084 		 * We need to turn these PMCs off like we
4085 		 * would do at context switch OUT time.
4086 		 */
4087 		for (ri = 0; ri < md->pmd_npmc; ri++) {
4088 
4089 			/*
4090 			 * Pick up the pmc pointer from hardware
4091 			 * state similar to the CSW_OUT code.
4092 			 */
4093 			pm = NULL;
4094 			(void) (*md->pmd_get_config)(cpu, ri, &pm);
4095 
4096 			PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
4097 
4098 			if (pm == NULL ||
4099 			    !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
4100 				continue;
4101 
4102 			PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
4103 			    "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
4104 			    pm, pm->pm_state);
4105 
4106 			KASSERT(PMC_TO_ROWINDEX(pm) == ri,
4107 			    ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
4108 				__LINE__, PMC_TO_ROWINDEX(pm), ri));
4109 
4110 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
4111 			    ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
4112 				__LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
4113 
4114 			(void) md->pmd_stop_pmc(cpu, ri);
4115 
4116 			KASSERT(pm->pm_runcount > 0,
4117 			    ("[pmc,%d] bad runcount ri %d rc %d",
4118 				__LINE__, ri, pm->pm_runcount));
4119 
4120 			/* Stop hardware only if it is actually running */
4121 			if (pm->pm_state == PMC_STATE_RUNNING &&
4122 			    pm->pm_stalled == 0) {
4123 				md->pmd_read_pmc(cpu, ri, &newvalue);
4124 				tmp = newvalue -
4125 				    PMC_PCPU_SAVED(cpu,ri);
4126 
4127 				mtx_pool_lock_spin(pmc_mtxpool, pm);
4128 				pm->pm_gv.pm_savedvalue += tmp;
4129 				pp->pp_pmcs[ri].pp_pmcval += tmp;
4130 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
4131 			}
4132 
4133 			atomic_subtract_rel_32(&pm->pm_runcount,1);
4134 
4135 			KASSERT((int) pm->pm_runcount >= 0,
4136 			    ("[pmc,%d] runcount is %d", __LINE__, ri));
4137 
4138 			(void) md->pmd_config_pmc(cpu, ri, NULL);
4139 		}
4140 
4141 		/*
4142 		 * Inform the MD layer of this pseudo "context switch
4143 		 * out"
4144 		 */
4145 		(void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
4146 
4147 		critical_exit(); /* ok to be pre-empted now */
4148 
4149 		/*
4150 		 * Unlink this process from the PMCs that are
4151 		 * targetting it.  This will send a signal to
4152 		 * all PMC owner's whose PMCs are orphaned.
4153 		 *
4154 		 * Log PMC value at exit time if requested.
4155 		 */
4156 		for (ri = 0; ri < md->pmd_npmc; ri++)
4157 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
4158 				if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
4159 				    PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
4160 					pmclog_process_procexit(pm, pp);
4161 				pmc_unlink_target_process(pm, pp);
4162 			}
4163 		free(pp, M_PMC);
4164 
4165 	} else
4166 		critical_exit(); /* pp == NULL */
4167 
4168 
4169 	/*
4170 	 * If the process owned PMCs, free them up and free up
4171 	 * memory.
4172 	 */
4173 	if ((po = pmc_find_owner_descriptor(p)) != NULL) {
4174 		pmc_remove_owner(po);
4175 		pmc_destroy_owner_descriptor(po);
4176 	}
4177 
4178 	sx_xunlock(&pmc_sx);
4179 }
4180 
4181 /*
4182  * Handle a process fork.
4183  *
4184  * If the parent process 'p1' is under HWPMC monitoring, then copy
4185  * over any attached PMCs that have 'do_descendants' semantics.
4186  */
4187 
4188 static void
4189 pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
4190     int flags)
4191 {
4192 	int is_using_hwpmcs;
4193 	unsigned int ri;
4194 	uint32_t do_descendants;
4195 	struct pmc *pm;
4196 	struct pmc_owner *po;
4197 	struct pmc_process *ppnew, *ppold;
4198 
4199 	(void) flags;		/* unused parameter */
4200 
4201 	PROC_LOCK(p1);
4202 	is_using_hwpmcs = p1->p_flag & P_HWPMC;
4203 	PROC_UNLOCK(p1);
4204 
4205 	/*
4206 	 * If there are system-wide sampling PMCs active, we need to
4207 	 * log all fork events to their owner's logs.
4208 	 */
4209 
4210 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
4211 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
4212 		    pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
4213 
4214 	if (!is_using_hwpmcs)
4215 		return;
4216 
4217 	PMC_GET_SX_XLOCK();
4218 	PMCDBG(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
4219 	    p1->p_pid, p1->p_comm, newproc);
4220 
4221 	/*
4222 	 * If the parent process (curthread->td_proc) is a
4223 	 * target of any PMCs, look for PMCs that are to be
4224 	 * inherited, and link these into the new process
4225 	 * descriptor.
4226 	 */
4227 	if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
4228 		 PMC_FLAG_NONE)) == NULL)
4229 		goto done;		/* nothing to do */
4230 
4231 	do_descendants = 0;
4232 	for (ri = 0; ri < md->pmd_npmc; ri++)
4233 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
4234 			do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
4235 	if (do_descendants == 0) /* nothing to do */
4236 		goto done;
4237 
4238 	/* allocate a descriptor for the new process  */
4239 	if ((ppnew = pmc_find_process_descriptor(newproc,
4240 		 PMC_FLAG_ALLOCATE)) == NULL)
4241 		goto done;
4242 
4243 	/*
4244 	 * Run through all PMCs that were targeting the old process
4245 	 * and which specified F_DESCENDANTS and attach them to the
4246 	 * new process.
4247 	 *
4248 	 * Log the fork event to all owners of PMCs attached to this
4249 	 * process, if not already logged.
4250 	 */
4251 	for (ri = 0; ri < md->pmd_npmc; ri++)
4252 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
4253 		    (pm->pm_flags & PMC_F_DESCENDANTS)) {
4254 			pmc_link_target_process(pm, ppnew);
4255 			po = pm->pm_owner;
4256 			if (po->po_sscount == 0 &&
4257 			    po->po_flags & PMC_PO_OWNS_LOGFILE)
4258 				pmclog_process_procfork(po, p1->p_pid,
4259 				    newproc->p_pid);
4260 		}
4261 
4262 	/*
4263 	 * Now mark the new process as being tracked by this driver.
4264 	 */
4265 	PROC_LOCK(newproc);
4266 	newproc->p_flag |= P_HWPMC;
4267 	PROC_UNLOCK(newproc);
4268 
4269  done:
4270 	sx_xunlock(&pmc_sx);
4271 }
4272 
4273 
4274 /*
4275  * initialization
4276  */
4277 
4278 static const char *pmc_name_of_pmcclass[] = {
4279 #undef	__PMC_CLASS
4280 #define	__PMC_CLASS(N) #N ,
4281 	__PMC_CLASSES()
4282 };
4283 
4284 static int
4285 pmc_initialize(void)
4286 {
4287 	int cpu, error, n;
4288 	unsigned int maxcpu;
4289 	struct pmc_binding pb;
4290 	struct pmc_sample *ps;
4291 	struct pmc_samplebuffer *sb;
4292 
4293 	md = NULL;
4294 	error = 0;
4295 
4296 #ifdef	DEBUG
4297 	/* parse debug flags first */
4298 	if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
4299 		pmc_debugstr, sizeof(pmc_debugstr)))
4300 		pmc_debugflags_parse(pmc_debugstr,
4301 		    pmc_debugstr+strlen(pmc_debugstr));
4302 #endif
4303 
4304 	PMCDBG(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
4305 
4306 	/* check kernel version */
4307 	if (pmc_kernel_version != PMC_VERSION) {
4308 		if (pmc_kernel_version == 0)
4309 			printf("hwpmc: this kernel has not been compiled with "
4310 			    "'options HWPMC_HOOKS'.\n");
4311 		else
4312 			printf("hwpmc: kernel version (0x%x) does not match "
4313 			    "module version (0x%x).\n", pmc_kernel_version,
4314 			    PMC_VERSION);
4315 		return EPROGMISMATCH;
4316 	}
4317 
4318 	/*
4319 	 * check sysctl parameters
4320 	 */
4321 
4322 	if (pmc_hashsize <= 0) {
4323 		(void) printf("hwpmc: tunable \"hashsize\"=%d must be "
4324 		    "greater than zero.\n", pmc_hashsize);
4325 		pmc_hashsize = PMC_HASH_SIZE;
4326 	}
4327 
4328 	if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
4329 		(void) printf("hwpmc: tunable \"nsamples\"=%d out of "
4330 		    "range.\n", pmc_nsamples);
4331 		pmc_nsamples = PMC_NSAMPLES;
4332 	}
4333 
4334 	if (pmc_callchaindepth <= 0 ||
4335 	    pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
4336 		(void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
4337 		    "range.\n", pmc_callchaindepth);
4338 		pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
4339 	}
4340 
4341 	md = pmc_md_initialize();
4342 
4343 	if (md == NULL || md->pmd_init == NULL)
4344 		return ENOSYS;
4345 
4346 	maxcpu = pmc_cpu_max();
4347 
4348 	/* allocate space for the per-cpu array */
4349 	pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *),
4350 	    M_PMC, M_WAITOK|M_ZERO);
4351 
4352 	/* per-cpu 'saved values' for managing process-mode PMCs */
4353 	pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
4354 	    M_PMC, M_WAITOK);
4355 
4356 	/* Perform CPU-dependent initialization. */
4357 	pmc_save_cpu_binding(&pb);
4358 	for (cpu = 0; cpu < maxcpu; cpu++) {
4359 		if (!pmc_cpu_is_active(cpu))
4360 			continue;
4361 		pmc_select_cpu(cpu);
4362 		if ((error = md->pmd_init(cpu)) != 0)
4363 			break;
4364 	}
4365 	pmc_restore_cpu_binding(&pb);
4366 
4367 	if (error != 0)
4368 		return error;
4369 
4370 	/* allocate space for the sample array */
4371 	for (cpu = 0; cpu < maxcpu; cpu++) {
4372 		if (!pmc_cpu_is_active(cpu))
4373 			continue;
4374 		sb = malloc(sizeof(struct pmc_samplebuffer) +
4375 		    pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
4376 		    M_WAITOK|M_ZERO);
4377 
4378 		sb->ps_read = sb->ps_write = sb->ps_samples;
4379 		sb->ps_fence = sb->ps_samples + pmc_nsamples;
4380 		KASSERT(pmc_pcpu[cpu] != NULL,
4381 		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
4382 
4383 		sb->ps_callchains = malloc(pmc_callchaindepth *
4384 		    pmc_nsamples * sizeof(uintptr_t),
4385 		    M_PMC, M_WAITOK|M_ZERO);
4386 
4387 		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
4388 			ps->ps_pc = sb->ps_callchains +
4389 			    (n * pmc_callchaindepth);
4390 
4391 		pmc_pcpu[cpu]->pc_sb = sb;
4392 	}
4393 
4394 	/* allocate space for the row disposition array */
4395 	pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
4396 	    M_PMC, M_WAITOK|M_ZERO);
4397 
4398 	KASSERT(pmc_pmcdisp != NULL,
4399 	    ("[pmc,%d] pmcdisp allocation returned NULL", __LINE__));
4400 
4401 	/* mark all PMCs as available */
4402 	for (n = 0; n < (int) md->pmd_npmc; n++)
4403 		PMC_MARK_ROW_FREE(n);
4404 
4405 	/* allocate thread hash tables */
4406 	pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
4407 	    &pmc_ownerhashmask);
4408 
4409 	pmc_processhash = hashinit(pmc_hashsize, M_PMC,
4410 	    &pmc_processhashmask);
4411 	mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
4412 	    MTX_SPIN);
4413 
4414 	LIST_INIT(&pmc_ss_owners);
4415 	pmc_ss_count = 0;
4416 
4417 	/* allocate a pool of spin mutexes */
4418 	pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
4419 	    MTX_SPIN);
4420 
4421 	PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
4422 	    "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
4423 	    pmc_processhash, pmc_processhashmask);
4424 
4425 	/* register process {exit,fork,exec} handlers */
4426 	pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
4427 	    pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
4428 	pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
4429 	    pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
4430 
4431 	/* initialize logging */
4432 	pmclog_initialize();
4433 
4434 	/* set hook functions */
4435 	pmc_intr = md->pmd_intr;
4436 	pmc_hook = pmc_hook_handler;
4437 
4438 	if (error == 0) {
4439 		printf(PMC_MODULE_NAME ":");
4440 		for (n = 0; n < (int) md->pmd_nclass; n++) {
4441 			printf(" %s/%d/0x%b",
4442 			    pmc_name_of_pmcclass[md->pmd_classes[n].pm_class],
4443 			    md->pmd_nclasspmcs[n],
4444 			    md->pmd_classes[n].pm_caps,
4445 			    "\20"
4446 			    "\1INT\2USR\3SYS\4EDG\5THR"
4447 			    "\6REA\7WRI\10INV\11QUA\12PRC"
4448 			    "\13TAG\14CSC");
4449 		}
4450 		printf("\n");
4451 	}
4452 
4453 	return error;
4454 }
4455 
4456 /* prepare to be unloaded */
4457 static void
4458 pmc_cleanup(void)
4459 {
4460 	int cpu;
4461 	unsigned int maxcpu;
4462 	struct pmc_ownerhash *ph;
4463 	struct pmc_owner *po, *tmp;
4464 	struct pmc_binding pb;
4465 #ifdef	DEBUG
4466 	struct pmc_processhash *prh;
4467 #endif
4468 
4469 	PMCDBG(MOD,INI,0, "%s", "cleanup");
4470 
4471 	/* switch off sampling */
4472 	atomic_store_rel_int(&pmc_cpumask, 0);
4473 	pmc_intr = NULL;
4474 
4475 	sx_xlock(&pmc_sx);
4476 	if (pmc_hook == NULL) {	/* being unloaded already */
4477 		sx_xunlock(&pmc_sx);
4478 		return;
4479 	}
4480 
4481 	pmc_hook = NULL; /* prevent new threads from entering module */
4482 
4483 	/* deregister event handlers */
4484 	EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
4485 	EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
4486 
4487 	/* send SIGBUS to all owner threads, free up allocations */
4488 	if (pmc_ownerhash)
4489 		for (ph = pmc_ownerhash;
4490 		     ph <= &pmc_ownerhash[pmc_ownerhashmask];
4491 		     ph++) {
4492 			LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
4493 				pmc_remove_owner(po);
4494 
4495 				/* send SIGBUS to owner processes */
4496 				PMCDBG(MOD,INI,2, "cleanup signal proc=%p "
4497 				    "(%d, %s)", po->po_owner,
4498 				    po->po_owner->p_pid,
4499 				    po->po_owner->p_comm);
4500 
4501 				PROC_LOCK(po->po_owner);
4502 				psignal(po->po_owner, SIGBUS);
4503 				PROC_UNLOCK(po->po_owner);
4504 
4505 				pmc_destroy_owner_descriptor(po);
4506 			}
4507 		}
4508 
4509 	/* reclaim allocated data structures */
4510 	if (pmc_mtxpool)
4511 		mtx_pool_destroy(&pmc_mtxpool);
4512 
4513 	mtx_destroy(&pmc_processhash_mtx);
4514 	if (pmc_processhash) {
4515 #ifdef	DEBUG
4516 		struct pmc_process *pp;
4517 
4518 		PMCDBG(MOD,INI,3, "%s", "destroy process hash");
4519 		for (prh = pmc_processhash;
4520 		     prh <= &pmc_processhash[pmc_processhashmask];
4521 		     prh++)
4522 			LIST_FOREACH(pp, prh, pp_next)
4523 			    PMCDBG(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
4524 #endif
4525 
4526 		hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
4527 		pmc_processhash = NULL;
4528 	}
4529 
4530 	if (pmc_ownerhash) {
4531 		PMCDBG(MOD,INI,3, "%s", "destroy owner hash");
4532 		hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
4533 		pmc_ownerhash = NULL;
4534 	}
4535 
4536 	KASSERT(LIST_EMPTY(&pmc_ss_owners),
4537 	    ("[pmc,%d] Global SS owner list not empty", __LINE__));
4538 	KASSERT(pmc_ss_count == 0,
4539 	    ("[pmc,%d] Global SS count not empty", __LINE__));
4540 
4541 	/* Free the per-cpu sample buffers. */
4542 	maxcpu = pmc_cpu_max();
4543 	for (cpu = 0; cpu < maxcpu; cpu++) {
4544 		if (!pmc_cpu_is_active(cpu))
4545 			continue;
4546 		KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
4547 		    ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
4548 			cpu));
4549 		free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC);
4550 		free(pmc_pcpu[cpu]->pc_sb, M_PMC);
4551 		pmc_pcpu[cpu]->pc_sb = NULL;
4552 	}
4553 
4554  	/* do processor dependent cleanup */
4555 	PMCDBG(MOD,INI,3, "%s", "md cleanup");
4556 	if (md) {
4557 		pmc_save_cpu_binding(&pb);
4558 		for (cpu = 0; cpu < maxcpu; cpu++) {
4559 			PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
4560 			    cpu, pmc_pcpu[cpu]);
4561 			if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
4562 				continue;
4563 			pmc_select_cpu(cpu);
4564 			if (md->pmd_cleanup)
4565 				md->pmd_cleanup(cpu);
4566 		}
4567 		free(md, M_PMC);
4568 		md = NULL;
4569 		pmc_restore_cpu_binding(&pb);
4570 	}
4571 
4572 	/* deallocate per-cpu structures */
4573 	free(pmc_pcpu, M_PMC);
4574 	pmc_pcpu = NULL;
4575 
4576 	free(pmc_pcpu_saved, M_PMC);
4577 	pmc_pcpu_saved = NULL;
4578 
4579 	if (pmc_pmcdisp) {
4580 		free(pmc_pmcdisp, M_PMC);
4581 		pmc_pmcdisp = NULL;
4582 	}
4583 
4584 	pmclog_shutdown();
4585 
4586 	sx_xunlock(&pmc_sx); 	/* we are done */
4587 }
4588 
4589 /*
4590  * The function called at load/unload.
4591  */
4592 
4593 static int
4594 load (struct module *module __unused, int cmd, void *arg __unused)
4595 {
4596 	int error;
4597 
4598 	error = 0;
4599 
4600 	switch (cmd) {
4601 	case MOD_LOAD :
4602 		/* initialize the subsystem */
4603 		error = pmc_initialize();
4604 		if (error != 0)
4605 			break;
4606 		PMCDBG(MOD,INI,1, "syscall=%d maxcpu=%d",
4607 		    pmc_syscall_num, pmc_cpu_max());
4608 		break;
4609 
4610 
4611 	case MOD_UNLOAD :
4612 	case MOD_SHUTDOWN:
4613 		pmc_cleanup();
4614 		PMCDBG(MOD,INI,1, "%s", "unloaded");
4615 		break;
4616 
4617 	default :
4618 		error = EINVAL;	/* XXX should panic(9) */
4619 		break;
4620 	}
4621 
4622 	return error;
4623 }
4624 
4625 /* memory pool */
4626 MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module");
4627