xref: /freebsd/sys/dev/hwpmc/hwpmc_mod.c (revision 0f00315cb37468757755d73b2ca49dd2e386f1b1)
1ebccf1e3SJoseph Koshy /*-
2718cf2ccSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3718cf2ccSPedro F. Giffuni  *
4122ccdc1SJoseph Koshy  * Copyright (c) 2003-2008 Joseph Koshy
5d07f36b0SJoseph Koshy  * Copyright (c) 2007 The FreeBSD Foundation
6e6b475e0SMatt Macy  * Copyright (c) 2018 Matthew Macy
7ebccf1e3SJoseph Koshy  * All rights reserved.
8ebccf1e3SJoseph Koshy  *
9d07f36b0SJoseph Koshy  * Portions of this software were developed by A. Joseph Koshy under
10d07f36b0SJoseph Koshy  * sponsorship from the FreeBSD Foundation and Google, Inc.
11d07f36b0SJoseph Koshy  *
12ebccf1e3SJoseph Koshy  * Redistribution and use in source and binary forms, with or without
13ebccf1e3SJoseph Koshy  * modification, are permitted provided that the following conditions
14ebccf1e3SJoseph Koshy  * are met:
15ebccf1e3SJoseph Koshy  * 1. Redistributions of source code must retain the above copyright
16ebccf1e3SJoseph Koshy  *    notice, this list of conditions and the following disclaimer.
17ebccf1e3SJoseph Koshy  * 2. Redistributions in binary form must reproduce the above copyright
18ebccf1e3SJoseph Koshy  *    notice, this list of conditions and the following disclaimer in the
19ebccf1e3SJoseph Koshy  *    documentation and/or other materials provided with the distribution.
20ebccf1e3SJoseph Koshy  *
21ebccf1e3SJoseph Koshy  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
22ebccf1e3SJoseph Koshy  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23ebccf1e3SJoseph Koshy  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24ebccf1e3SJoseph Koshy  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
25ebccf1e3SJoseph Koshy  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26ebccf1e3SJoseph Koshy  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27ebccf1e3SJoseph Koshy  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28ebccf1e3SJoseph Koshy  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29ebccf1e3SJoseph Koshy  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30ebccf1e3SJoseph Koshy  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31ebccf1e3SJoseph Koshy  * SUCH DAMAGE.
32ebccf1e3SJoseph Koshy  *
33ebccf1e3SJoseph Koshy  */
34ebccf1e3SJoseph Koshy 
35ebccf1e3SJoseph Koshy #include <sys/cdefs.h>
36ebccf1e3SJoseph Koshy __FBSDID("$FreeBSD$");
37ebccf1e3SJoseph Koshy 
38ebccf1e3SJoseph Koshy #include <sys/param.h>
39ebccf1e3SJoseph Koshy #include <sys/eventhandler.h>
40ebccf1e3SJoseph Koshy #include <sys/jail.h>
41ebccf1e3SJoseph Koshy #include <sys/kernel.h>
42f263522aSJoseph Koshy #include <sys/kthread.h>
43ebccf1e3SJoseph Koshy #include <sys/limits.h>
44ebccf1e3SJoseph Koshy #include <sys/lock.h>
45ebccf1e3SJoseph Koshy #include <sys/malloc.h>
46ebccf1e3SJoseph Koshy #include <sys/module.h>
47e9b5dc16SJoseph Koshy #include <sys/mount.h>
48ebccf1e3SJoseph Koshy #include <sys/mutex.h>
49ebccf1e3SJoseph Koshy #include <sys/pmc.h>
50ebccf1e3SJoseph Koshy #include <sys/pmckern.h>
51f263522aSJoseph Koshy #include <sys/pmclog.h>
52acd3428bSRobert Watson #include <sys/priv.h>
53ebccf1e3SJoseph Koshy #include <sys/proc.h>
54ebccf1e3SJoseph Koshy #include <sys/queue.h>
55f263522aSJoseph Koshy #include <sys/resourcevar.h>
5689f6b863SAttilio Rao #include <sys/rwlock.h>
57ebccf1e3SJoseph Koshy #include <sys/sched.h>
58ebccf1e3SJoseph Koshy #include <sys/signalvar.h>
59ebccf1e3SJoseph Koshy #include <sys/smp.h>
60ebccf1e3SJoseph Koshy #include <sys/sx.h>
61ebccf1e3SJoseph Koshy #include <sys/sysctl.h>
62ebccf1e3SJoseph Koshy #include <sys/sysent.h>
63ebccf1e3SJoseph Koshy #include <sys/systm.h>
64f263522aSJoseph Koshy #include <sys/vnode.h>
65ebccf1e3SJoseph Koshy 
6649874f6eSJoseph Koshy #include <sys/linker.h>		/* needs to be after <sys/malloc.h> */
6749874f6eSJoseph Koshy 
68f263522aSJoseph Koshy #include <machine/atomic.h>
69ebccf1e3SJoseph Koshy #include <machine/md_var.h>
70ebccf1e3SJoseph Koshy 
71bf792d68SJoseph Koshy #include <vm/vm.h>
72bf792d68SJoseph Koshy #include <vm/vm_extern.h>
73bf792d68SJoseph Koshy #include <vm/pmap.h>
74bf792d68SJoseph Koshy #include <vm/vm_map.h>
75bf792d68SJoseph Koshy #include <vm/vm_object.h>
76bf792d68SJoseph Koshy 
77f5f9340bSFabien Thomas #include "hwpmc_soft.h"
78f5f9340bSFabien Thomas 
79f1401123SMatt Macy #ifdef NUMA
80f1401123SMatt Macy #define NDOMAINS vm_ndomains
81f1401123SMatt Macy #else
82f1401123SMatt Macy #define NDOMAINS 1
83f1401123SMatt Macy #define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
84f1401123SMatt Macy #define free_domain(addr, type) free(addr, type)
85f1401123SMatt Macy #endif
86f1401123SMatt Macy 
87ebccf1e3SJoseph Koshy /*
88ebccf1e3SJoseph Koshy  * Types
89ebccf1e3SJoseph Koshy  */
90ebccf1e3SJoseph Koshy 
91ebccf1e3SJoseph Koshy enum pmc_flags {
92ebccf1e3SJoseph Koshy 	PMC_FLAG_NONE	  = 0x00, /* do nothing */
93ebccf1e3SJoseph Koshy 	PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
94ebccf1e3SJoseph Koshy 	PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
95ebccf1e3SJoseph Koshy };
96ebccf1e3SJoseph Koshy 
97ebccf1e3SJoseph Koshy /*
98ebccf1e3SJoseph Koshy  * The offset in sysent where the syscall is allocated.
99ebccf1e3SJoseph Koshy  */
100ebccf1e3SJoseph Koshy 
101ebccf1e3SJoseph Koshy static int pmc_syscall_num = NO_SYSCALL;
102ebccf1e3SJoseph Koshy struct pmc_cpu		**pmc_pcpu;	 /* per-cpu state */
103ebccf1e3SJoseph Koshy pmc_value_t		*pmc_pcpu_saved; /* saved PMC values: CSW handling */
104ebccf1e3SJoseph Koshy 
105ebccf1e3SJoseph Koshy #define	PMC_PCPU_SAVED(C,R)	pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
106ebccf1e3SJoseph Koshy 
107ebccf1e3SJoseph Koshy struct mtx_pool		*pmc_mtxpool;
108ebccf1e3SJoseph Koshy static int		*pmc_pmcdisp;	 /* PMC row dispositions */
109ebccf1e3SJoseph Koshy 
110ebccf1e3SJoseph Koshy #define	PMC_ROW_DISP_IS_FREE(R)		(pmc_pmcdisp[(R)] == 0)
111ebccf1e3SJoseph Koshy #define	PMC_ROW_DISP_IS_THREAD(R)	(pmc_pmcdisp[(R)] > 0)
112ebccf1e3SJoseph Koshy #define	PMC_ROW_DISP_IS_STANDALONE(R)	(pmc_pmcdisp[(R)] < 0)
113ebccf1e3SJoseph Koshy 
114ebccf1e3SJoseph Koshy #define	PMC_MARK_ROW_FREE(R) do {					  \
115ebccf1e3SJoseph Koshy 	pmc_pmcdisp[(R)] = 0;						  \
116ebccf1e3SJoseph Koshy } while (0)
117ebccf1e3SJoseph Koshy 
118ebccf1e3SJoseph Koshy #define	PMC_MARK_ROW_STANDALONE(R) do {					  \
119ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
120ebccf1e3SJoseph Koshy 		    __LINE__));						  \
121ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], -1);				  \
122122ccdc1SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()),		  \
123122ccdc1SJoseph Koshy 		("[pmc,%d] row disposition error", __LINE__));		  \
124ebccf1e3SJoseph Koshy } while (0)
125ebccf1e3SJoseph Koshy 
126ebccf1e3SJoseph Koshy #define	PMC_UNMARK_ROW_STANDALONE(R) do { 				  \
127ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], 1);				  \
128ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
129ebccf1e3SJoseph Koshy 		    __LINE__));						  \
130ebccf1e3SJoseph Koshy } while (0)
131ebccf1e3SJoseph Koshy 
132ebccf1e3SJoseph Koshy #define	PMC_MARK_ROW_THREAD(R) do {					  \
133ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
134ebccf1e3SJoseph Koshy 		    __LINE__));						  \
135ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], 1);				  \
136ebccf1e3SJoseph Koshy } while (0)
137ebccf1e3SJoseph Koshy 
138ebccf1e3SJoseph Koshy #define	PMC_UNMARK_ROW_THREAD(R) do {					  \
139ebccf1e3SJoseph Koshy 	atomic_add_int(&pmc_pmcdisp[(R)], -1);				  \
140ebccf1e3SJoseph Koshy 	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
141ebccf1e3SJoseph Koshy 		    __LINE__));						  \
142ebccf1e3SJoseph Koshy } while (0)
143ebccf1e3SJoseph Koshy 
144ebccf1e3SJoseph Koshy 
145ebccf1e3SJoseph Koshy /* various event handlers */
14629f4e216SMark Johnston static eventhandler_tag	pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
14729f4e216SMark Johnston     pmc_kld_unload_tag;
148ebccf1e3SJoseph Koshy 
149ebccf1e3SJoseph Koshy /* Module statistics */
150e6b475e0SMatt Macy struct pmc_driverstats pmc_stats;
151e6b475e0SMatt Macy 
152ebccf1e3SJoseph Koshy 
153ebccf1e3SJoseph Koshy /* Machine/processor dependent operations */
1540cfab8ddSJoseph Koshy static struct pmc_mdep  *md;
155ebccf1e3SJoseph Koshy 
156ebccf1e3SJoseph Koshy /*
157ebccf1e3SJoseph Koshy  * Hash tables mapping owner processes and target threads to PMCs.
158ebccf1e3SJoseph Koshy  */
159ebccf1e3SJoseph Koshy 
160ebccf1e3SJoseph Koshy struct mtx pmc_processhash_mtx;		/* spin mutex */
161ebccf1e3SJoseph Koshy static u_long pmc_processhashmask;
162ebccf1e3SJoseph Koshy static LIST_HEAD(pmc_processhash, pmc_process)	*pmc_processhash;
163ebccf1e3SJoseph Koshy 
164ebccf1e3SJoseph Koshy /*
165ebccf1e3SJoseph Koshy  * Hash table of PMC owner descriptors.  This table is protected by
166ebccf1e3SJoseph Koshy  * the shared PMC "sx" lock.
167ebccf1e3SJoseph Koshy  */
168ebccf1e3SJoseph Koshy 
169ebccf1e3SJoseph Koshy static u_long pmc_ownerhashmask;
170ebccf1e3SJoseph Koshy static LIST_HEAD(pmc_ownerhash, pmc_owner)	*pmc_ownerhash;
171ebccf1e3SJoseph Koshy 
172ebccf1e3SJoseph Koshy /*
173f263522aSJoseph Koshy  * List of PMC owners with system-wide sampling PMCs.
174f263522aSJoseph Koshy  */
175f263522aSJoseph Koshy 
176f263522aSJoseph Koshy static LIST_HEAD(, pmc_owner)			pmc_ss_owners;
177f263522aSJoseph Koshy 
178f263522aSJoseph Koshy 
179f263522aSJoseph Koshy /*
180e829eb6dSJoseph Koshy  * A map of row indices to classdep structures.
181e829eb6dSJoseph Koshy  */
182e829eb6dSJoseph Koshy static struct pmc_classdep **pmc_rowindex_to_classdep;
183e829eb6dSJoseph Koshy 
184e829eb6dSJoseph Koshy /*
185ebccf1e3SJoseph Koshy  * Prototypes
186ebccf1e3SJoseph Koshy  */
187ebccf1e3SJoseph Koshy 
188680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
189ebccf1e3SJoseph Koshy static int	pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
190ebccf1e3SJoseph Koshy static int	pmc_debugflags_parse(char *newstr, char *fence);
191ebccf1e3SJoseph Koshy #endif
192ebccf1e3SJoseph Koshy 
193ebccf1e3SJoseph Koshy static int	load(struct module *module, int cmd, void *arg);
194f263522aSJoseph Koshy static int	pmc_attach_process(struct proc *p, struct pmc *pm);
195ebccf1e3SJoseph Koshy static struct pmc *pmc_allocate_pmc_descriptor(void);
196f263522aSJoseph Koshy static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
197f263522aSJoseph Koshy static int	pmc_attach_one_process(struct proc *p, struct pmc *pm);
198c5153e19SJoseph Koshy static int	pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
199c5153e19SJoseph Koshy     int cpu);
200f263522aSJoseph Koshy static int	pmc_can_attach(struct pmc *pm, struct proc *p);
201f5f9340bSFabien Thomas static void	pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
202f263522aSJoseph Koshy static void	pmc_cleanup(void);
203f263522aSJoseph Koshy static int	pmc_detach_process(struct proc *p, struct pmc *pm);
204f263522aSJoseph Koshy static int	pmc_detach_one_process(struct proc *p, struct pmc *pm,
205f263522aSJoseph Koshy     int flags);
206f263522aSJoseph Koshy static void	pmc_destroy_owner_descriptor(struct pmc_owner *po);
20706796b67SMark Johnston static void	pmc_destroy_pmc_descriptor(struct pmc *pm);
208ebccf1e3SJoseph Koshy static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
209ebccf1e3SJoseph Koshy static int	pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
210f263522aSJoseph Koshy static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
211f263522aSJoseph Koshy     pmc_id_t pmc);
212f263522aSJoseph Koshy static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
213f263522aSJoseph Koshy     uint32_t mode);
214c5153e19SJoseph Koshy static void	pmc_force_context_switch(void);
215ebccf1e3SJoseph Koshy static void	pmc_link_target_process(struct pmc *pm,
216ebccf1e3SJoseph Koshy     struct pmc_process *pp);
217d07f36b0SJoseph Koshy static void	pmc_log_all_process_mappings(struct pmc_owner *po);
218d07f36b0SJoseph Koshy static void	pmc_log_kernel_mappings(struct pmc *pm);
219d07f36b0SJoseph Koshy static void	pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
220f263522aSJoseph Koshy static void	pmc_maybe_remove_owner(struct pmc_owner *po);
221f263522aSJoseph Koshy static void	pmc_process_csw_in(struct thread *td);
222f263522aSJoseph Koshy static void	pmc_process_csw_out(struct thread *td);
223ebccf1e3SJoseph Koshy static void	pmc_process_exit(void *arg, struct proc *p);
224ebccf1e3SJoseph Koshy static void	pmc_process_fork(void *arg, struct proc *p1,
225ebccf1e3SJoseph Koshy     struct proc *p2, int n);
226f5f9340bSFabien Thomas static void	pmc_process_samples(int cpu, int soft);
227f263522aSJoseph Koshy static void	pmc_release_pmc_descriptor(struct pmc *pmc);
228f263522aSJoseph Koshy static void	pmc_remove_owner(struct pmc_owner *po);
229f263522aSJoseph Koshy static void	pmc_remove_process_descriptor(struct pmc_process *pp);
230f263522aSJoseph Koshy static void	pmc_restore_cpu_binding(struct pmc_binding *pb);
231f263522aSJoseph Koshy static void	pmc_save_cpu_binding(struct pmc_binding *pb);
232f263522aSJoseph Koshy static void	pmc_select_cpu(int cpu);
233ebccf1e3SJoseph Koshy static int	pmc_start(struct pmc *pm);
234ebccf1e3SJoseph Koshy static int	pmc_stop(struct pmc *pm);
235f263522aSJoseph Koshy static int	pmc_syscall_handler(struct thread *td, void *syscall_args);
236f263522aSJoseph Koshy static void	pmc_unlink_target_process(struct pmc *pmc,
237f263522aSJoseph Koshy     struct pmc_process *pp);
238f5f9340bSFabien Thomas static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
239f5f9340bSFabien Thomas static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
240f5f9340bSFabien Thomas static struct pmc_mdep *pmc_generic_cpu_initialize(void);
241f5f9340bSFabien Thomas static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
242ebccf1e3SJoseph Koshy 
243ebccf1e3SJoseph Koshy /*
244ebccf1e3SJoseph Koshy  * Kernel tunables and sysctl(8) interface.
245ebccf1e3SJoseph Koshy  */
246ebccf1e3SJoseph Koshy 
247f5f9340bSFabien Thomas SYSCTL_DECL(_kern_hwpmc);
248e6b475e0SMatt Macy SYSCTL_NODE(_kern_hwpmc, OID_AUTO, stats, CTLFLAG_RW, 0, "HWPMC stats");
249e6b475e0SMatt Macy 
250e6b475e0SMatt Macy 
251e6b475e0SMatt Macy /* Stats. */
252e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_ignored, CTLFLAG_RW,
253e6b475e0SMatt Macy 				   &pmc_stats.pm_intr_ignored, "# of interrupts ignored");
254e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_processed, CTLFLAG_RW,
255e6b475e0SMatt Macy 				   &pmc_stats.pm_intr_processed, "# of interrupts processed");
256e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_bufferfull, CTLFLAG_RW,
257e6b475e0SMatt Macy 				   &pmc_stats.pm_intr_bufferfull, "# of interrupts where buffer was full");
258e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscalls, CTLFLAG_RW,
259e6b475e0SMatt Macy 				   &pmc_stats.pm_syscalls, "# of syscalls");
260e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscall_errors, CTLFLAG_RW,
261e6b475e0SMatt Macy 				   &pmc_stats.pm_syscall_errors, "# of syscall_errors");
262e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests, CTLFLAG_RW,
263e6b475e0SMatt Macy 				   &pmc_stats.pm_buffer_requests, "# of buffer requests");
264e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests_failed, CTLFLAG_RW,
265e6b475e0SMatt Macy 				   &pmc_stats.pm_buffer_requests_failed, "# of buffer requests which failed");
266e6b475e0SMatt Macy SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, log_sweeps, CTLFLAG_RW,
267e6b475e0SMatt Macy 				   &pmc_stats.pm_log_sweeps, "# of ?");
268ebccf1e3SJoseph Koshy 
269d07f36b0SJoseph Koshy static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
270af3b2549SHans Petter Selasky SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN,
271d07f36b0SJoseph Koshy     &pmc_callchaindepth, 0, "depth of call chain records");
272d07f36b0SJoseph Koshy 
273e6b475e0SMatt Macy char pmc_cpuid[64];
274e6b475e0SMatt Macy SYSCTL_STRING(_kern_hwpmc, OID_AUTO, cpuid, CTLFLAG_RD,
275e6b475e0SMatt Macy 	pmc_cpuid, 0, "cpu version string");
276680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
277f263522aSJoseph Koshy struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
278ebccf1e3SJoseph Koshy char	pmc_debugstr[PMC_DEBUG_STRSIZE];
279ebccf1e3SJoseph Koshy TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
280ebccf1e3SJoseph Koshy     sizeof(pmc_debugstr));
281ebccf1e3SJoseph Koshy SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
282af3b2549SHans Petter Selasky     CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
283ebccf1e3SJoseph Koshy     0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
284ebccf1e3SJoseph Koshy #endif
285ebccf1e3SJoseph Koshy 
286e6b475e0SMatt Macy 
287ebccf1e3SJoseph Koshy /*
288f263522aSJoseph Koshy  * kern.hwpmc.hashrows -- determines the number of rows in the
289ebccf1e3SJoseph Koshy  * of the hash table used to look up threads
290ebccf1e3SJoseph Koshy  */
291ebccf1e3SJoseph Koshy 
292ebccf1e3SJoseph Koshy static int pmc_hashsize = PMC_HASH_SIZE;
293af3b2549SHans Petter Selasky SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN,
294ebccf1e3SJoseph Koshy     &pmc_hashsize, 0, "rows in hash tables");
295ebccf1e3SJoseph Koshy 
296ebccf1e3SJoseph Koshy /*
297d07f36b0SJoseph Koshy  * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
298ebccf1e3SJoseph Koshy  */
299ebccf1e3SJoseph Koshy 
300f263522aSJoseph Koshy static int pmc_nsamples = PMC_NSAMPLES;
301af3b2549SHans Petter Selasky SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN,
302f263522aSJoseph Koshy     &pmc_nsamples, 0, "number of PC samples per CPU");
303ebccf1e3SJoseph Koshy 
304d07f36b0SJoseph Koshy 
305ebccf1e3SJoseph Koshy /*
306f263522aSJoseph Koshy  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
307ebccf1e3SJoseph Koshy  */
308ebccf1e3SJoseph Koshy 
309ebccf1e3SJoseph Koshy static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
310af3b2549SHans Petter Selasky SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN,
311ebccf1e3SJoseph Koshy     &pmc_mtxpool_size, 0, "size of spin mutex pool");
312ebccf1e3SJoseph Koshy 
313ebccf1e3SJoseph Koshy 
314ebccf1e3SJoseph Koshy /*
315ebccf1e3SJoseph Koshy  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
316ebccf1e3SJoseph Koshy  * allocate system-wide PMCs.
317ebccf1e3SJoseph Koshy  *
318ebccf1e3SJoseph Koshy  * Allowing unprivileged processes to allocate system PMCs is convenient
319ebccf1e3SJoseph Koshy  * if system-wide measurements need to be taken concurrently with other
320ebccf1e3SJoseph Koshy  * per-process measurements.  This feature is turned off by default.
321ebccf1e3SJoseph Koshy  */
322ebccf1e3SJoseph Koshy 
323ebccf1e3SJoseph Koshy static int pmc_unprivileged_syspmcs = 0;
324af3b2549SHans Petter Selasky SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN,
325ebccf1e3SJoseph Koshy     &pmc_unprivileged_syspmcs, 0,
326ebccf1e3SJoseph Koshy     "allow unprivileged process to allocate system PMCs");
327ebccf1e3SJoseph Koshy 
328f263522aSJoseph Koshy /*
329f263522aSJoseph Koshy  * Hash function.  Discard the lower 2 bits of the pointer since
330f263522aSJoseph Koshy  * these are always zero for our uses.  The hash multiplier is
331f263522aSJoseph Koshy  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
332f263522aSJoseph Koshy  */
333ebccf1e3SJoseph Koshy 
334ebccf1e3SJoseph Koshy #if	LONG_BIT == 64
335ebccf1e3SJoseph Koshy #define	_PMC_HM		11400714819323198486u
336ebccf1e3SJoseph Koshy #elif	LONG_BIT == 32
337ebccf1e3SJoseph Koshy #define	_PMC_HM		2654435769u
338ebccf1e3SJoseph Koshy #else
339ebccf1e3SJoseph Koshy #error 	Must know the size of 'long' to compile
340ebccf1e3SJoseph Koshy #endif
341ebccf1e3SJoseph Koshy 
342ebccf1e3SJoseph Koshy #define	PMC_HASH_PTR(P,M)	((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
343ebccf1e3SJoseph Koshy 
344ebccf1e3SJoseph Koshy /*
345ebccf1e3SJoseph Koshy  * Syscall structures
346ebccf1e3SJoseph Koshy  */
347ebccf1e3SJoseph Koshy 
348ebccf1e3SJoseph Koshy /* The `sysent' for the new syscall */
349ebccf1e3SJoseph Koshy static struct sysent pmc_sysent = {
3501cfbc451SKonstantin Belousov 	.sy_narg =	2,
3511cfbc451SKonstantin Belousov 	.sy_call =	pmc_syscall_handler,
352ebccf1e3SJoseph Koshy };
353ebccf1e3SJoseph Koshy 
354ebccf1e3SJoseph Koshy static struct syscall_module_data pmc_syscall_mod = {
3551cfbc451SKonstantin Belousov 	.chainevh =	load,
3561cfbc451SKonstantin Belousov 	.chainarg =	NULL,
3571cfbc451SKonstantin Belousov 	.offset =	&pmc_syscall_num,
3581cfbc451SKonstantin Belousov 	.new_sysent =	&pmc_sysent,
3591cfbc451SKonstantin Belousov 	.old_sysent =	{ .sy_narg = 0, .sy_call = NULL },
3601cfbc451SKonstantin Belousov 	.flags =	SY_THR_STATIC_KLD,
361ebccf1e3SJoseph Koshy };
362ebccf1e3SJoseph Koshy 
363ebccf1e3SJoseph Koshy static moduledata_t pmc_mod = {
3641cfbc451SKonstantin Belousov 	.name =		PMC_MODULE_NAME,
3651cfbc451SKonstantin Belousov 	.evhand =	syscall_module_handler,
3661cfbc451SKonstantin Belousov 	.priv =		&pmc_syscall_mod,
367ebccf1e3SJoseph Koshy };
368ebccf1e3SJoseph Koshy 
369fdce57a0SJohn Baldwin #ifdef EARLY_AP_STARTUP
370fdce57a0SJohn Baldwin DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
371fdce57a0SJohn Baldwin #else
372ebccf1e3SJoseph Koshy DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
373fdce57a0SJohn Baldwin #endif
374ebccf1e3SJoseph Koshy MODULE_VERSION(pmc, PMC_VERSION);
375ebccf1e3SJoseph Koshy 
376680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
377f263522aSJoseph Koshy enum pmc_dbgparse_state {
378f263522aSJoseph Koshy 	PMCDS_WS,		/* in whitespace */
379f263522aSJoseph Koshy 	PMCDS_MAJOR,		/* seen a major keyword */
380f263522aSJoseph Koshy 	PMCDS_MINOR
381f263522aSJoseph Koshy };
382f263522aSJoseph Koshy 
383ebccf1e3SJoseph Koshy static int
384ebccf1e3SJoseph Koshy pmc_debugflags_parse(char *newstr, char *fence)
385ebccf1e3SJoseph Koshy {
386e1691ef7SJoseph Koshy 	char c, *p, *q;
387f263522aSJoseph Koshy 	struct pmc_debugflags *tmpflags;
388f263522aSJoseph Koshy 	int error, found, *newbits, tmp;
389f263522aSJoseph Koshy 	size_t kwlen;
390ebccf1e3SJoseph Koshy 
391e11e3f18SDag-Erling Smørgrav 	tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
392ebccf1e3SJoseph Koshy 
393ebccf1e3SJoseph Koshy 	p = newstr;
394f263522aSJoseph Koshy 	error = 0;
395ebccf1e3SJoseph Koshy 
396f263522aSJoseph Koshy 	for (; p < fence && (c = *p); p++) {
397ebccf1e3SJoseph Koshy 
398f263522aSJoseph Koshy 		/* skip white space */
399f263522aSJoseph Koshy 		if (c == ' ' || c == '\t')
400f263522aSJoseph Koshy 			continue;
401f263522aSJoseph Koshy 
402f263522aSJoseph Koshy 		/* look for a keyword followed by "=" */
403f263522aSJoseph Koshy 		for (q = p; p < fence && (c = *p) && c != '='; p++)
404f263522aSJoseph Koshy 			;
405f263522aSJoseph Koshy 		if (c != '=') {
406f263522aSJoseph Koshy 			error = EINVAL;
407f263522aSJoseph Koshy 			goto done;
408ebccf1e3SJoseph Koshy 		}
409ebccf1e3SJoseph Koshy 
410f263522aSJoseph Koshy 		kwlen = p - q;
411f263522aSJoseph Koshy 		newbits = NULL;
412ebccf1e3SJoseph Koshy 
413f263522aSJoseph Koshy 		/* lookup flag group name */
414f263522aSJoseph Koshy #define	DBG_SET_FLAG_MAJ(S,F)						\
415f263522aSJoseph Koshy 		if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)	\
416f263522aSJoseph Koshy 			newbits = &tmpflags->pdb_ ## F;
417ebccf1e3SJoseph Koshy 
418f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("cpu",		CPU);
419f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("csw",		CSW);
420f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("logging",	LOG);
421f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("module",	MOD);
422f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("md", 		MDP);
423f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("owner",	OWN);
424f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("pmc",		PMC);
425f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("process",	PRC);
426f263522aSJoseph Koshy 		DBG_SET_FLAG_MAJ("sampling", 	SAM);
427ebccf1e3SJoseph Koshy 
428f263522aSJoseph Koshy 		if (newbits == NULL) {
429f263522aSJoseph Koshy 			error = EINVAL;
430f263522aSJoseph Koshy 			goto done;
431ebccf1e3SJoseph Koshy 		}
432ebccf1e3SJoseph Koshy 
433f263522aSJoseph Koshy 		p++;		/* skip the '=' */
434ebccf1e3SJoseph Koshy 
435f263522aSJoseph Koshy 		/* Now parse the individual flags */
436f263522aSJoseph Koshy 		tmp = 0;
437f263522aSJoseph Koshy 	newflag:
438f263522aSJoseph Koshy 		for (q = p; p < fence && (c = *p); p++)
439f263522aSJoseph Koshy 			if (c == ' ' || c == '\t' || c == ',')
440f263522aSJoseph Koshy 				break;
441f263522aSJoseph Koshy 
442f263522aSJoseph Koshy 		/* p == fence or c == ws or c == "," or c == 0 */
443f263522aSJoseph Koshy 
444f263522aSJoseph Koshy 		if ((kwlen = p - q) == 0) {
445f263522aSJoseph Koshy 			*newbits = tmp;
446f263522aSJoseph Koshy 			continue;
447f263522aSJoseph Koshy 		}
448f263522aSJoseph Koshy 
449f263522aSJoseph Koshy 		found = 0;
450f263522aSJoseph Koshy #define	DBG_SET_FLAG_MIN(S,F)						\
451f263522aSJoseph Koshy 		if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)	\
452f263522aSJoseph Koshy 			tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
453f263522aSJoseph Koshy 
454f263522aSJoseph Koshy 		/* a '*' denotes all possible flags in the group */
455f263522aSJoseph Koshy 		if (kwlen == 1 && *q == '*')
456f263522aSJoseph Koshy 			tmp = found = ~0;
457f263522aSJoseph Koshy 		/* look for individual flag names */
458f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("allocaterow", ALR);
459f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("allocate",	ALL);
460f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("attach",	ATT);
461f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("bind",	BND);
462f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("config",	CFG);
463f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("exec",	EXC);
464f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("exit",	EXT);
465f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("find",	FND);
466f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("flush",	FLS);
467f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("fork",	FRK);
468f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("getbuf",	GTB);
469f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("hook",	PMH);
470f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("init",	INI);
471f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("intr",	INT);
472f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("linktarget",	TLK);
473f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("mayberemove", OMR);
474f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("ops",		OPS);
475f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("read",	REA);
476f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("register",	REG);
477f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("release",	REL);
478f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("remove",	ORM);
479f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("sample",	SAM);
480f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("scheduleio",	SIO);
481f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("select",	SEL);
482f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("signal",	SIG);
483f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("swi",		SWI);
484f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("swo",		SWO);
485f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("start",	STA);
486f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("stop",	STO);
487f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("syscall",	PMS);
488f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("unlinktarget", TUL);
489f263522aSJoseph Koshy 		DBG_SET_FLAG_MIN("write",	WRI);
490f263522aSJoseph Koshy 		if (found == 0) {
491f263522aSJoseph Koshy 			/* unrecognized flag name */
492f263522aSJoseph Koshy 			error = EINVAL;
493f263522aSJoseph Koshy 			goto done;
494f263522aSJoseph Koshy 		}
495f263522aSJoseph Koshy 
496f263522aSJoseph Koshy 		if (c == 0 || c == ' ' || c == '\t') {	/* end of flag group */
497f263522aSJoseph Koshy 			*newbits = tmp;
498f263522aSJoseph Koshy 			continue;
499f263522aSJoseph Koshy 		}
500f263522aSJoseph Koshy 
501f263522aSJoseph Koshy 		p++;
502f263522aSJoseph Koshy 		goto newflag;
503f263522aSJoseph Koshy 	}
504f263522aSJoseph Koshy 
505f263522aSJoseph Koshy 	/* save the new flag set */
506f263522aSJoseph Koshy 	bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
507f263522aSJoseph Koshy 
508f263522aSJoseph Koshy  done:
5091ede983cSDag-Erling Smørgrav 	free(tmpflags, M_PMC);
510f263522aSJoseph Koshy 	return error;
511ebccf1e3SJoseph Koshy }
512ebccf1e3SJoseph Koshy 
513ebccf1e3SJoseph Koshy static int
514ebccf1e3SJoseph Koshy pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
515ebccf1e3SJoseph Koshy {
516ebccf1e3SJoseph Koshy 	char *fence, *newstr;
517ebccf1e3SJoseph Koshy 	int error;
518ebccf1e3SJoseph Koshy 	unsigned int n;
519ebccf1e3SJoseph Koshy 
520ebccf1e3SJoseph Koshy 	(void) arg1; (void) arg2; /* unused parameters */
521ebccf1e3SJoseph Koshy 
522ebccf1e3SJoseph Koshy 	n = sizeof(pmc_debugstr);
523e829eb6dSJoseph Koshy 	newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
524f263522aSJoseph Koshy 	(void) strlcpy(newstr, pmc_debugstr, n);
525ebccf1e3SJoseph Koshy 
526ebccf1e3SJoseph Koshy 	error = sysctl_handle_string(oidp, newstr, n, req);
527ebccf1e3SJoseph Koshy 
528ebccf1e3SJoseph Koshy 	/* if there is a new string, parse and copy it */
529ebccf1e3SJoseph Koshy 	if (error == 0 && req->newptr != NULL) {
530f263522aSJoseph Koshy 		fence = newstr + (n < req->newlen ? n : req->newlen + 1);
531ebccf1e3SJoseph Koshy 		if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
532ebccf1e3SJoseph Koshy 			(void) strlcpy(pmc_debugstr, newstr,
533ebccf1e3SJoseph Koshy 			    sizeof(pmc_debugstr));
534ebccf1e3SJoseph Koshy 	}
535ebccf1e3SJoseph Koshy 
5361ede983cSDag-Erling Smørgrav 	free(newstr, M_PMC);
537ebccf1e3SJoseph Koshy 
538ebccf1e3SJoseph Koshy 	return error;
539ebccf1e3SJoseph Koshy }
540ebccf1e3SJoseph Koshy #endif
541ebccf1e3SJoseph Koshy 
542ebccf1e3SJoseph Koshy /*
543e829eb6dSJoseph Koshy  * Map a row index to a classdep structure and return the adjusted row
544e829eb6dSJoseph Koshy  * index for the PMC class index.
545e829eb6dSJoseph Koshy  */
546e829eb6dSJoseph Koshy static struct pmc_classdep *
547e829eb6dSJoseph Koshy pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
548e829eb6dSJoseph Koshy {
549e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
550e829eb6dSJoseph Koshy 
551e829eb6dSJoseph Koshy 	(void) md;
552e829eb6dSJoseph Koshy 
553e829eb6dSJoseph Koshy 	KASSERT(ri >= 0 && ri < md->pmd_npmc,
554e829eb6dSJoseph Koshy 	    ("[pmc,%d] illegal row-index %d", __LINE__, ri));
555e829eb6dSJoseph Koshy 
556e829eb6dSJoseph Koshy 	pcd = pmc_rowindex_to_classdep[ri];
557e829eb6dSJoseph Koshy 
558e829eb6dSJoseph Koshy 	KASSERT(pcd != NULL,
559e5087dd8SRui Paulo 	    ("[pmc,%d] ri %d null pcd", __LINE__, ri));
560e829eb6dSJoseph Koshy 
561e829eb6dSJoseph Koshy 	*adjri = ri - pcd->pcd_ri;
562e829eb6dSJoseph Koshy 
563e829eb6dSJoseph Koshy 	KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
564e829eb6dSJoseph Koshy 	    ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
565e829eb6dSJoseph Koshy 
566e829eb6dSJoseph Koshy 	return (pcd);
567e829eb6dSJoseph Koshy }
568e829eb6dSJoseph Koshy 
569e829eb6dSJoseph Koshy /*
570ebccf1e3SJoseph Koshy  * Concurrency Control
571ebccf1e3SJoseph Koshy  *
572ebccf1e3SJoseph Koshy  * The driver manages the following data structures:
573ebccf1e3SJoseph Koshy  *
574ebccf1e3SJoseph Koshy  *   - target process descriptors, one per target process
575ebccf1e3SJoseph Koshy  *   - owner process descriptors (and attached lists), one per owner process
576ebccf1e3SJoseph Koshy  *   - lookup hash tables for owner and target processes
577ebccf1e3SJoseph Koshy  *   - PMC descriptors (and attached lists)
578ebccf1e3SJoseph Koshy  *   - per-cpu hardware state
579ebccf1e3SJoseph Koshy  *   - the 'hook' variable through which the kernel calls into
580ebccf1e3SJoseph Koshy  *     this module
581ebccf1e3SJoseph Koshy  *   - the machine hardware state (managed by the MD layer)
582ebccf1e3SJoseph Koshy  *
583ebccf1e3SJoseph Koshy  * These data structures are accessed from:
584ebccf1e3SJoseph Koshy  *
585ebccf1e3SJoseph Koshy  * - thread context-switch code
586ebccf1e3SJoseph Koshy  * - interrupt handlers (possibly on multiple cpus)
587ebccf1e3SJoseph Koshy  * - kernel threads on multiple cpus running on behalf of user
588ebccf1e3SJoseph Koshy  *   processes doing system calls
589ebccf1e3SJoseph Koshy  * - this driver's private kernel threads
590ebccf1e3SJoseph Koshy  *
591ebccf1e3SJoseph Koshy  * = Locks and Locking strategy =
592ebccf1e3SJoseph Koshy  *
593ebccf1e3SJoseph Koshy  * The driver uses four locking strategies for its operation:
594ebccf1e3SJoseph Koshy  *
595382d30cdSJoseph Koshy  * - The global SX lock "pmc_sx" is used to protect internal
596382d30cdSJoseph Koshy  *   data structures.
597ebccf1e3SJoseph Koshy  *
598382d30cdSJoseph Koshy  *   Calls into the module by syscall() start with this lock being
599382d30cdSJoseph Koshy  *   held in exclusive mode.  Depending on the requested operation,
600382d30cdSJoseph Koshy  *   the lock may be downgraded to 'shared' mode to allow more
601382d30cdSJoseph Koshy  *   concurrent readers into the module.  Calls into the module from
602382d30cdSJoseph Koshy  *   other parts of the kernel acquire the lock in shared mode.
603ebccf1e3SJoseph Koshy  *
604ebccf1e3SJoseph Koshy  *   This SX lock is held in exclusive mode for any operations that
605ebccf1e3SJoseph Koshy  *   modify the linkages between the driver's internal data structures.
606ebccf1e3SJoseph Koshy  *
607ebccf1e3SJoseph Koshy  *   The 'pmc_hook' function pointer is also protected by this lock.
608ebccf1e3SJoseph Koshy  *   It is only examined with the sx lock held in exclusive mode.  The
609382d30cdSJoseph Koshy  *   kernel module is allowed to be unloaded only with the sx lock held
610382d30cdSJoseph Koshy  *   in exclusive mode.  In normal syscall handling, after acquiring the
611382d30cdSJoseph Koshy  *   pmc_sx lock we first check that 'pmc_hook' is non-null before
612382d30cdSJoseph Koshy  *   proceeding.  This prevents races between the thread unloading the module
613382d30cdSJoseph Koshy  *   and other threads seeking to use the module.
614ebccf1e3SJoseph Koshy  *
615ebccf1e3SJoseph Koshy  * - Lookups of target process structures and owner process structures
616ebccf1e3SJoseph Koshy  *   cannot use the global "pmc_sx" SX lock because these lookups need
617ebccf1e3SJoseph Koshy  *   to happen during context switches and in other critical sections
618ebccf1e3SJoseph Koshy  *   where sleeping is not allowed.  We protect these lookup tables
619ebccf1e3SJoseph Koshy  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
620382d30cdSJoseph Koshy  *   "pmc_ownerhash_mtx".
621ebccf1e3SJoseph Koshy  *
622ebccf1e3SJoseph Koshy  * - Interrupt handlers work in a lock free manner.  At interrupt
623ebccf1e3SJoseph Koshy  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
624ebccf1e3SJoseph Koshy  *   when the PMC was started.  If this pointer is NULL, the interrupt
625ebccf1e3SJoseph Koshy  *   is ignored after updating driver statistics.  We ensure that this
626ebccf1e3SJoseph Koshy  *   pointer is set (using an atomic operation if necessary) before the
627ebccf1e3SJoseph Koshy  *   PMC hardware is started.  Conversely, this pointer is unset atomically
628ebccf1e3SJoseph Koshy  *   only after the PMC hardware is stopped.
629ebccf1e3SJoseph Koshy  *
630ebccf1e3SJoseph Koshy  *   We ensure that everything needed for the operation of an
631ebccf1e3SJoseph Koshy  *   interrupt handler is available without it needing to acquire any
632ebccf1e3SJoseph Koshy  *   locks.  We also ensure that a PMC's software state is destroyed only
633ebccf1e3SJoseph Koshy  *   after the PMC is taken off hardware (on all CPUs).
634ebccf1e3SJoseph Koshy  *
635ebccf1e3SJoseph Koshy  * - Context-switch handling with process-private PMCs needs more
636ebccf1e3SJoseph Koshy  *   care.
637ebccf1e3SJoseph Koshy  *
638ebccf1e3SJoseph Koshy  *   A given process may be the target of multiple PMCs.  For example,
639ebccf1e3SJoseph Koshy  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
640ebccf1e3SJoseph Koshy  *   while the target process is running on another.  A PMC could also
641ebccf1e3SJoseph Koshy  *   be getting released because its owner is exiting.  We tackle
642ebccf1e3SJoseph Koshy  *   these situations in the following manner:
643ebccf1e3SJoseph Koshy  *
644ebccf1e3SJoseph Koshy  *   - each target process structure 'pmc_process' has an array
645ebccf1e3SJoseph Koshy  *     of 'struct pmc *' pointers, one for each hardware PMC.
646ebccf1e3SJoseph Koshy  *
647ebccf1e3SJoseph Koshy  *   - At context switch IN time, each "target" PMC in RUNNING state
648ebccf1e3SJoseph Koshy  *     gets started on hardware and a pointer to each PMC is copied into
649ebccf1e3SJoseph Koshy  *     the per-cpu phw array.  The 'runcount' for the PMC is
650ebccf1e3SJoseph Koshy  *     incremented.
651ebccf1e3SJoseph Koshy  *
652ebccf1e3SJoseph Koshy  *   - At context switch OUT time, all process-virtual PMCs are stopped
653ebccf1e3SJoseph Koshy  *     on hardware.  The saved value is added to the PMCs value field
654ebccf1e3SJoseph Koshy  *     only if the PMC is in a non-deleted state (the PMCs state could
655ebccf1e3SJoseph Koshy  *     have changed during the current time slice).
656ebccf1e3SJoseph Koshy  *
657ebccf1e3SJoseph Koshy  *     Note that since in-between a switch IN on a processor and a switch
658ebccf1e3SJoseph Koshy  *     OUT, the PMC could have been released on another CPU.  Therefore
659ebccf1e3SJoseph Koshy  *     context switch OUT always looks at the hardware state to turn
660ebccf1e3SJoseph Koshy  *     OFF PMCs and will update a PMC's saved value only if reachable
661ebccf1e3SJoseph Koshy  *     from the target process record.
662ebccf1e3SJoseph Koshy  *
663ebccf1e3SJoseph Koshy  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
664ebccf1e3SJoseph Koshy  *     be attached to many processes at the time of the call and could
665ebccf1e3SJoseph Koshy  *     be active on multiple CPUs).
666ebccf1e3SJoseph Koshy  *
667ebccf1e3SJoseph Koshy  *     We prevent further scheduling of the PMC by marking it as in
668ebccf1e3SJoseph Koshy  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
669ebccf1e3SJoseph Koshy  *     this PMC is currently running on a CPU somewhere.  The thread
6704d70511aSJohn Baldwin  *     doing the PMCRELEASE operation waits by repeatedly doing a
6714d70511aSJohn Baldwin  *     pause() till the runcount comes to zero.
672ebccf1e3SJoseph Koshy  *
673382d30cdSJoseph Koshy  * The contents of a PMC descriptor (struct pmc) are protected using
674382d30cdSJoseph Koshy  * a spin-mutex.  In order to save space, we use a mutex pool.
675382d30cdSJoseph Koshy  *
676382d30cdSJoseph Koshy  * In terms of lock types used by witness(4), we use:
677382d30cdSJoseph Koshy  * - Type "pmc-sx", used by the global SX lock.
678382d30cdSJoseph Koshy  * - Type "pmc-sleep", for sleep mutexes used by logger threads.
679382d30cdSJoseph Koshy  * - Type "pmc-per-proc", for protecting PMC owner descriptors.
680382d30cdSJoseph Koshy  * - Type "pmc-leaf", used for all other spin mutexes.
681ebccf1e3SJoseph Koshy  */
682ebccf1e3SJoseph Koshy 
683ebccf1e3SJoseph Koshy /*
684ebccf1e3SJoseph Koshy  * save the cpu binding of the current kthread
685ebccf1e3SJoseph Koshy  */
686ebccf1e3SJoseph Koshy 
687ebccf1e3SJoseph Koshy static void
688ebccf1e3SJoseph Koshy pmc_save_cpu_binding(struct pmc_binding *pb)
689ebccf1e3SJoseph Koshy {
6904a3690dfSJohn Baldwin 	PMCDBG0(CPU,BND,2, "save-cpu");
691982d11f8SJeff Roberson 	thread_lock(curthread);
692ebccf1e3SJoseph Koshy 	pb->pb_bound = sched_is_bound(curthread);
693ebccf1e3SJoseph Koshy 	pb->pb_cpu   = curthread->td_oncpu;
694982d11f8SJeff Roberson 	thread_unlock(curthread);
6954a3690dfSJohn Baldwin 	PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
696ebccf1e3SJoseph Koshy }
697ebccf1e3SJoseph Koshy 
698ebccf1e3SJoseph Koshy /*
699ebccf1e3SJoseph Koshy  * restore the cpu binding of the current thread
700ebccf1e3SJoseph Koshy  */
701ebccf1e3SJoseph Koshy 
702ebccf1e3SJoseph Koshy static void
703ebccf1e3SJoseph Koshy pmc_restore_cpu_binding(struct pmc_binding *pb)
704ebccf1e3SJoseph Koshy {
7054a3690dfSJohn Baldwin 	PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
706ebccf1e3SJoseph Koshy 	    curthread->td_oncpu, pb->pb_cpu);
707982d11f8SJeff Roberson 	thread_lock(curthread);
708ebccf1e3SJoseph Koshy 	if (pb->pb_bound)
709ebccf1e3SJoseph Koshy 		sched_bind(curthread, pb->pb_cpu);
710ebccf1e3SJoseph Koshy 	else
711ebccf1e3SJoseph Koshy 		sched_unbind(curthread);
712982d11f8SJeff Roberson 	thread_unlock(curthread);
7134a3690dfSJohn Baldwin 	PMCDBG0(CPU,BND,2, "restore-cpu done");
714ebccf1e3SJoseph Koshy }
715ebccf1e3SJoseph Koshy 
716ebccf1e3SJoseph Koshy /*
717ebccf1e3SJoseph Koshy  * move execution over the specified cpu and bind it there.
718ebccf1e3SJoseph Koshy  */
719ebccf1e3SJoseph Koshy 
720ebccf1e3SJoseph Koshy static void
721ebccf1e3SJoseph Koshy pmc_select_cpu(int cpu)
722ebccf1e3SJoseph Koshy {
723122ccdc1SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
724ebccf1e3SJoseph Koshy 	    ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
725ebccf1e3SJoseph Koshy 
726122ccdc1SJoseph Koshy 	/* Never move to an inactive CPU. */
727122ccdc1SJoseph Koshy 	KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
728122ccdc1SJoseph Koshy 	    "CPU %d", __LINE__, cpu));
729ebccf1e3SJoseph Koshy 
7304a3690dfSJohn Baldwin 	PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu);
731982d11f8SJeff Roberson 	thread_lock(curthread);
732ebccf1e3SJoseph Koshy 	sched_bind(curthread, cpu);
733982d11f8SJeff Roberson 	thread_unlock(curthread);
734ebccf1e3SJoseph Koshy 
735ebccf1e3SJoseph Koshy 	KASSERT(curthread->td_oncpu == cpu,
736ebccf1e3SJoseph Koshy 	    ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
737ebccf1e3SJoseph Koshy 		cpu, curthread->td_oncpu));
738ebccf1e3SJoseph Koshy 
7394a3690dfSJohn Baldwin 	PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
740ebccf1e3SJoseph Koshy }
741ebccf1e3SJoseph Koshy 
742ebccf1e3SJoseph Koshy /*
743c5153e19SJoseph Koshy  * Force a context switch.
744c5153e19SJoseph Koshy  *
7454d70511aSJohn Baldwin  * We do this by pause'ing for 1 tick -- invoking mi_switch() is not
746c5153e19SJoseph Koshy  * guaranteed to force a context switch.
747c5153e19SJoseph Koshy  */
748c5153e19SJoseph Koshy 
749c5153e19SJoseph Koshy static void
750c5153e19SJoseph Koshy pmc_force_context_switch(void)
751c5153e19SJoseph Koshy {
752c5153e19SJoseph Koshy 
7534d70511aSJohn Baldwin 	pause("pmcctx", 1);
754c5153e19SJoseph Koshy }
755c5153e19SJoseph Koshy 
756c5153e19SJoseph Koshy /*
757f263522aSJoseph Koshy  * Get the file name for an executable.  This is a simple wrapper
758f263522aSJoseph Koshy  * around vn_fullpath(9).
759ebccf1e3SJoseph Koshy  */
760ebccf1e3SJoseph Koshy 
761f263522aSJoseph Koshy static void
76215139246SJoseph Koshy pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
763ebccf1e3SJoseph Koshy {
764ebccf1e3SJoseph Koshy 
765f263522aSJoseph Koshy 	*fullpath = "unknown";
766f263522aSJoseph Koshy 	*freepath = NULL;
76722db15c0SAttilio Rao 	vn_fullpath(curthread, v, fullpath, freepath);
768ebccf1e3SJoseph Koshy }
769ebccf1e3SJoseph Koshy 
770ebccf1e3SJoseph Koshy /*
771ebccf1e3SJoseph Koshy  * remove an process owning PMCs
772ebccf1e3SJoseph Koshy  */
773ebccf1e3SJoseph Koshy 
774ebccf1e3SJoseph Koshy void
775ebccf1e3SJoseph Koshy pmc_remove_owner(struct pmc_owner *po)
776ebccf1e3SJoseph Koshy {
777f263522aSJoseph Koshy 	struct pmc *pm, *tmp;
778ebccf1e3SJoseph Koshy 
779ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
780ebccf1e3SJoseph Koshy 
7814a3690dfSJohn Baldwin 	PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po);
782ebccf1e3SJoseph Koshy 
783ebccf1e3SJoseph Koshy 	/* Remove descriptor from the owner hash table */
784ebccf1e3SJoseph Koshy 	LIST_REMOVE(po, po_next);
785ebccf1e3SJoseph Koshy 
786f263522aSJoseph Koshy 	/* release all owned PMC descriptors */
787f263522aSJoseph Koshy 	LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
7884a3690dfSJohn Baldwin 		PMCDBG1(OWN,ORM,2, "pmc=%p", pm);
789f263522aSJoseph Koshy 		KASSERT(pm->pm_owner == po,
790f263522aSJoseph Koshy 		    ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
791ebccf1e3SJoseph Koshy 
792f263522aSJoseph Koshy 		pmc_release_pmc_descriptor(pm);	/* will unlink from the list */
79306796b67SMark Johnston 		pmc_destroy_pmc_descriptor(pm);
794ebccf1e3SJoseph Koshy 	}
795ebccf1e3SJoseph Koshy 
796f263522aSJoseph Koshy 	KASSERT(po->po_sscount == 0,
797f263522aSJoseph Koshy 	    ("[pmc,%d] SS count not zero", __LINE__));
798ebccf1e3SJoseph Koshy 	KASSERT(LIST_EMPTY(&po->po_pmcs),
799ebccf1e3SJoseph Koshy 	    ("[pmc,%d] PMC list not empty", __LINE__));
800ebccf1e3SJoseph Koshy 
801f263522aSJoseph Koshy 	/* de-configure the log file if present */
802c5153e19SJoseph Koshy 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
803f263522aSJoseph Koshy 		pmclog_deconfigure_log(po);
804ebccf1e3SJoseph Koshy }
805ebccf1e3SJoseph Koshy 
806ebccf1e3SJoseph Koshy /*
807ebccf1e3SJoseph Koshy  * remove an owner process record if all conditions are met.
808ebccf1e3SJoseph Koshy  */
809ebccf1e3SJoseph Koshy 
810ebccf1e3SJoseph Koshy static void
811ebccf1e3SJoseph Koshy pmc_maybe_remove_owner(struct pmc_owner *po)
812ebccf1e3SJoseph Koshy {
813ebccf1e3SJoseph Koshy 
8144a3690dfSJohn Baldwin 	PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po);
815ebccf1e3SJoseph Koshy 
816ebccf1e3SJoseph Koshy 	/*
817ebccf1e3SJoseph Koshy 	 * Remove owner record if
818ebccf1e3SJoseph Koshy 	 * - this process does not own any PMCs
819ebccf1e3SJoseph Koshy 	 * - this process has not allocated a system-wide sampling buffer
820ebccf1e3SJoseph Koshy 	 */
821ebccf1e3SJoseph Koshy 
822ebccf1e3SJoseph Koshy 	if (LIST_EMPTY(&po->po_pmcs) &&
823c5153e19SJoseph Koshy 	    ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
824ebccf1e3SJoseph Koshy 		pmc_remove_owner(po);
825f263522aSJoseph Koshy 		pmc_destroy_owner_descriptor(po);
826ebccf1e3SJoseph Koshy 	}
827ebccf1e3SJoseph Koshy }
828ebccf1e3SJoseph Koshy 
829ebccf1e3SJoseph Koshy /*
830ebccf1e3SJoseph Koshy  * Add an association between a target process and a PMC.
831ebccf1e3SJoseph Koshy  */
832ebccf1e3SJoseph Koshy 
833ebccf1e3SJoseph Koshy static void
834ebccf1e3SJoseph Koshy pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
835ebccf1e3SJoseph Koshy {
836ebccf1e3SJoseph Koshy 	int ri;
837ebccf1e3SJoseph Koshy 	struct pmc_target *pt;
838ebccf1e3SJoseph Koshy 
839ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
840ebccf1e3SJoseph Koshy 
841ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL && pp != NULL,
842ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
843f263522aSJoseph Koshy 	KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
844f263522aSJoseph Koshy 	    ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
845f263522aSJoseph Koshy 		__LINE__, pm, pp->pp_proc->p_pid));
846fa630f35SFabien Thomas 	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
847ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Illegal reference count %d for process record %p",
848ebccf1e3SJoseph Koshy 		__LINE__, pp->pp_refcnt, (void *) pp));
849ebccf1e3SJoseph Koshy 
850c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
851ebccf1e3SJoseph Koshy 
8524a3690dfSJohn Baldwin 	PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
853ebccf1e3SJoseph Koshy 	    pm, ri, pp);
854ebccf1e3SJoseph Koshy 
855680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
856ebccf1e3SJoseph Koshy 	LIST_FOREACH(pt, &pm->pm_targets, pt_next)
857ebccf1e3SJoseph Koshy 	    if (pt->pt_process == pp)
858ebccf1e3SJoseph Koshy 		    KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
859ebccf1e3SJoseph Koshy 				__LINE__, pp, pm));
860ebccf1e3SJoseph Koshy #endif
861ebccf1e3SJoseph Koshy 
862e829eb6dSJoseph Koshy 	pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
863ebccf1e3SJoseph Koshy 	pt->pt_process = pp;
864ebccf1e3SJoseph Koshy 
865ebccf1e3SJoseph Koshy 	LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
866ebccf1e3SJoseph Koshy 
867122eceefSJohn Baldwin 	atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
868122eceefSJohn Baldwin 	    (uintptr_t)pm);
869ebccf1e3SJoseph Koshy 
8706b8c8cd8SJoseph Koshy 	if (pm->pm_owner->po_owner == pp->pp_proc)
871c5153e19SJoseph Koshy 		pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
8726b8c8cd8SJoseph Koshy 
873f263522aSJoseph Koshy 	/*
874f263522aSJoseph Koshy 	 * Initialize the per-process values at this row index.
875f263522aSJoseph Koshy 	 */
876f263522aSJoseph Koshy 	pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
877f263522aSJoseph Koshy 	    pm->pm_sc.pm_reloadcount : 0;
878f263522aSJoseph Koshy 
879ebccf1e3SJoseph Koshy 	pp->pp_refcnt++;
880ebccf1e3SJoseph Koshy 
881ebccf1e3SJoseph Koshy }
882ebccf1e3SJoseph Koshy 
883ebccf1e3SJoseph Koshy /*
884ebccf1e3SJoseph Koshy  * Removes the association between a target process and a PMC.
885ebccf1e3SJoseph Koshy  */
886ebccf1e3SJoseph Koshy 
887ebccf1e3SJoseph Koshy static void
888ebccf1e3SJoseph Koshy pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
889ebccf1e3SJoseph Koshy {
890ebccf1e3SJoseph Koshy 	int ri;
891f263522aSJoseph Koshy 	struct proc *p;
892ebccf1e3SJoseph Koshy 	struct pmc_target *ptgt;
893ebccf1e3SJoseph Koshy 
894ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
895ebccf1e3SJoseph Koshy 
896ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL && pp != NULL,
897ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
898ebccf1e3SJoseph Koshy 
899fa630f35SFabien Thomas 	KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
900ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Illegal ref count %d on process record %p",
901ebccf1e3SJoseph Koshy 		__LINE__, pp->pp_refcnt, (void *) pp));
902ebccf1e3SJoseph Koshy 
903c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
904ebccf1e3SJoseph Koshy 
9054a3690dfSJohn Baldwin 	PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
906ebccf1e3SJoseph Koshy 	    pm, ri, pp);
907ebccf1e3SJoseph Koshy 
908ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
909ebccf1e3SJoseph Koshy 	    ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
910ebccf1e3SJoseph Koshy 		ri, pm, pp->pp_pmcs[ri].pp_pmc));
911ebccf1e3SJoseph Koshy 
912ebccf1e3SJoseph Koshy 	pp->pp_pmcs[ri].pp_pmc = NULL;
913ebccf1e3SJoseph Koshy 	pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
914ebccf1e3SJoseph Koshy 
915c5153e19SJoseph Koshy 	/* Remove owner-specific flags */
916c5153e19SJoseph Koshy 	if (pm->pm_owner->po_owner == pp->pp_proc) {
917c5153e19SJoseph Koshy 		pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
918c5153e19SJoseph Koshy 		pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
919c5153e19SJoseph Koshy 	}
9206b8c8cd8SJoseph Koshy 
921ebccf1e3SJoseph Koshy 	pp->pp_refcnt--;
922ebccf1e3SJoseph Koshy 
923ebccf1e3SJoseph Koshy 	/* Remove the target process from the PMC structure */
924ebccf1e3SJoseph Koshy 	LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
925ebccf1e3SJoseph Koshy 		if (ptgt->pt_process == pp)
926ebccf1e3SJoseph Koshy 			break;
927ebccf1e3SJoseph Koshy 
928ebccf1e3SJoseph Koshy 	KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
929ebccf1e3SJoseph Koshy 		    "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
930ebccf1e3SJoseph Koshy 
931ebccf1e3SJoseph Koshy 	LIST_REMOVE(ptgt, pt_next);
9321ede983cSDag-Erling Smørgrav 	free(ptgt, M_PMC);
933f263522aSJoseph Koshy 
934f263522aSJoseph Koshy 	/* if the PMC now lacks targets, send the owner a SIGIO */
935f263522aSJoseph Koshy 	if (LIST_EMPTY(&pm->pm_targets)) {
936f263522aSJoseph Koshy 		p = pm->pm_owner->po_owner;
937f263522aSJoseph Koshy 		PROC_LOCK(p);
9388451d0ddSKip Macy 		kern_psignal(p, SIGIO);
939f263522aSJoseph Koshy 		PROC_UNLOCK(p);
940f263522aSJoseph Koshy 
9414a3690dfSJohn Baldwin 		PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p,
942f263522aSJoseph Koshy 		    SIGIO);
943ebccf1e3SJoseph Koshy 	}
944ebccf1e3SJoseph Koshy }
945ebccf1e3SJoseph Koshy 
946ebccf1e3SJoseph Koshy /*
947ebccf1e3SJoseph Koshy  * Check if PMC 'pm' may be attached to target process 't'.
948ebccf1e3SJoseph Koshy  */
949ebccf1e3SJoseph Koshy 
950ebccf1e3SJoseph Koshy static int
951ebccf1e3SJoseph Koshy pmc_can_attach(struct pmc *pm, struct proc *t)
952ebccf1e3SJoseph Koshy {
953ebccf1e3SJoseph Koshy 	struct proc *o;		/* pmc owner */
954ebccf1e3SJoseph Koshy 	struct ucred *oc, *tc;	/* owner, target credentials */
955ebccf1e3SJoseph Koshy 	int decline_attach, i;
956ebccf1e3SJoseph Koshy 
957ebccf1e3SJoseph Koshy 	/*
958ebccf1e3SJoseph Koshy 	 * A PMC's owner can always attach that PMC to itself.
959ebccf1e3SJoseph Koshy 	 */
960ebccf1e3SJoseph Koshy 
961ebccf1e3SJoseph Koshy 	if ((o = pm->pm_owner->po_owner) == t)
962ebccf1e3SJoseph Koshy 		return 0;
963ebccf1e3SJoseph Koshy 
964ebccf1e3SJoseph Koshy 	PROC_LOCK(o);
965ebccf1e3SJoseph Koshy 	oc = o->p_ucred;
966ebccf1e3SJoseph Koshy 	crhold(oc);
967ebccf1e3SJoseph Koshy 	PROC_UNLOCK(o);
968ebccf1e3SJoseph Koshy 
969ebccf1e3SJoseph Koshy 	PROC_LOCK(t);
970ebccf1e3SJoseph Koshy 	tc = t->p_ucred;
971ebccf1e3SJoseph Koshy 	crhold(tc);
972ebccf1e3SJoseph Koshy 	PROC_UNLOCK(t);
973ebccf1e3SJoseph Koshy 
974ebccf1e3SJoseph Koshy 	/*
975ebccf1e3SJoseph Koshy 	 * The effective uid of the PMC owner should match at least one
976ebccf1e3SJoseph Koshy 	 * of the {effective,real,saved} uids of the target process.
977ebccf1e3SJoseph Koshy 	 */
978ebccf1e3SJoseph Koshy 
979ebccf1e3SJoseph Koshy 	decline_attach = oc->cr_uid != tc->cr_uid &&
980ebccf1e3SJoseph Koshy 	    oc->cr_uid != tc->cr_svuid &&
981ebccf1e3SJoseph Koshy 	    oc->cr_uid != tc->cr_ruid;
982ebccf1e3SJoseph Koshy 
983ebccf1e3SJoseph Koshy 	/*
984ebccf1e3SJoseph Koshy 	 * Every one of the target's group ids, must be in the owner's
985ebccf1e3SJoseph Koshy 	 * group list.
986ebccf1e3SJoseph Koshy 	 */
987ebccf1e3SJoseph Koshy 	for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
988ebccf1e3SJoseph Koshy 		decline_attach = !groupmember(tc->cr_groups[i], oc);
989ebccf1e3SJoseph Koshy 
990ebccf1e3SJoseph Koshy 	/* check the read and saved gids too */
991ebccf1e3SJoseph Koshy 	if (decline_attach == 0)
992ebccf1e3SJoseph Koshy 		decline_attach = !groupmember(tc->cr_rgid, oc) ||
993ebccf1e3SJoseph Koshy 		    !groupmember(tc->cr_svgid, oc);
994ebccf1e3SJoseph Koshy 
995ebccf1e3SJoseph Koshy 	crfree(tc);
996ebccf1e3SJoseph Koshy 	crfree(oc);
997ebccf1e3SJoseph Koshy 
998ebccf1e3SJoseph Koshy 	return !decline_attach;
999ebccf1e3SJoseph Koshy }
1000ebccf1e3SJoseph Koshy 
1001ebccf1e3SJoseph Koshy /*
1002ebccf1e3SJoseph Koshy  * Attach a process to a PMC.
1003ebccf1e3SJoseph Koshy  */
1004ebccf1e3SJoseph Koshy 
1005ebccf1e3SJoseph Koshy static int
1006ebccf1e3SJoseph Koshy pmc_attach_one_process(struct proc *p, struct pmc *pm)
1007ebccf1e3SJoseph Koshy {
1008ebccf1e3SJoseph Koshy 	int ri;
1009f263522aSJoseph Koshy 	char *fullpath, *freepath;
1010ebccf1e3SJoseph Koshy 	struct pmc_process	*pp;
1011ebccf1e3SJoseph Koshy 
1012ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1013ebccf1e3SJoseph Koshy 
10144a3690dfSJohn Baldwin 	PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
1015c5153e19SJoseph Koshy 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1016ebccf1e3SJoseph Koshy 
1017ebccf1e3SJoseph Koshy 	/*
1018ebccf1e3SJoseph Koshy 	 * Locate the process descriptor corresponding to process 'p',
1019ebccf1e3SJoseph Koshy 	 * allocating space as needed.
1020ebccf1e3SJoseph Koshy 	 *
1021ebccf1e3SJoseph Koshy 	 * Verify that rowindex 'pm_rowindex' is free in the process
1022ebccf1e3SJoseph Koshy 	 * descriptor.
1023ebccf1e3SJoseph Koshy 	 *
1024ebccf1e3SJoseph Koshy 	 * If not, allocate space for a descriptor and link the
1025ebccf1e3SJoseph Koshy 	 * process descriptor and PMC.
1026ebccf1e3SJoseph Koshy 	 */
1027c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
1028ebccf1e3SJoseph Koshy 
1029ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
1030ebccf1e3SJoseph Koshy 		return ENOMEM;
1031ebccf1e3SJoseph Koshy 
1032ebccf1e3SJoseph Koshy 	if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
1033ebccf1e3SJoseph Koshy 		return EEXIST;
1034ebccf1e3SJoseph Koshy 
1035ebccf1e3SJoseph Koshy 	if (pp->pp_pmcs[ri].pp_pmc != NULL)
1036ebccf1e3SJoseph Koshy 		return EBUSY;
1037ebccf1e3SJoseph Koshy 
1038ebccf1e3SJoseph Koshy 	pmc_link_target_process(pm, pp);
1039ebccf1e3SJoseph Koshy 
1040f263522aSJoseph Koshy 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
1041f263522aSJoseph Koshy 	    (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
1042f263522aSJoseph Koshy 		pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
1043f263522aSJoseph Koshy 
1044f263522aSJoseph Koshy 	pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
1045f263522aSJoseph Koshy 
1046f263522aSJoseph Koshy 	/* issue an attach event to a configured log file */
1047f263522aSJoseph Koshy 	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
1048db57c70aSKonstantin Belousov 		if (p->p_flag & P_KPROC) {
1049d67023a1SJeff Roberson 			fullpath = kernelname;
1050d67023a1SJeff Roberson 			freepath = NULL;
10510fb2c5d6SKonstantin Belousov 		} else {
10520fb2c5d6SKonstantin Belousov 			pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1053f263522aSJoseph Koshy 			pmclog_process_pmcattach(pm, p->p_pid, fullpath);
10540fb2c5d6SKonstantin Belousov 		}
10551ede983cSDag-Erling Smørgrav 		free(freepath, M_TEMP);
1056d07f36b0SJoseph Koshy 		if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1057d07f36b0SJoseph Koshy 			pmc_log_process_mappings(pm->pm_owner, p);
1058f263522aSJoseph Koshy 	}
1059ebccf1e3SJoseph Koshy 	/* mark process as using HWPMCs */
1060ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
1061ebccf1e3SJoseph Koshy 	p->p_flag |= P_HWPMC;
1062ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
1063ebccf1e3SJoseph Koshy 
1064ebccf1e3SJoseph Koshy 	return 0;
1065ebccf1e3SJoseph Koshy }
1066ebccf1e3SJoseph Koshy 
1067ebccf1e3SJoseph Koshy /*
1068ebccf1e3SJoseph Koshy  * Attach a process and optionally its children
1069ebccf1e3SJoseph Koshy  */
1070ebccf1e3SJoseph Koshy 
1071ebccf1e3SJoseph Koshy static int
1072ebccf1e3SJoseph Koshy pmc_attach_process(struct proc *p, struct pmc *pm)
1073ebccf1e3SJoseph Koshy {
1074ebccf1e3SJoseph Koshy 	int error;
1075ebccf1e3SJoseph Koshy 	struct proc *top;
1076ebccf1e3SJoseph Koshy 
1077ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1078ebccf1e3SJoseph Koshy 
10794a3690dfSJohn Baldwin 	PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
1080c5153e19SJoseph Koshy 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1081c5153e19SJoseph Koshy 
1082c5153e19SJoseph Koshy 
1083c5153e19SJoseph Koshy 	/*
1084c5153e19SJoseph Koshy 	 * If this PMC successfully allowed a GETMSR operation
1085c5153e19SJoseph Koshy 	 * in the past, disallow further ATTACHes.
1086c5153e19SJoseph Koshy 	 */
1087c5153e19SJoseph Koshy 
1088c5153e19SJoseph Koshy 	if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
1089c5153e19SJoseph Koshy 		return EPERM;
1090ebccf1e3SJoseph Koshy 
1091ebccf1e3SJoseph Koshy 	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1092ebccf1e3SJoseph Koshy 		return pmc_attach_one_process(p, pm);
1093ebccf1e3SJoseph Koshy 
1094ebccf1e3SJoseph Koshy 	/*
1095ebccf1e3SJoseph Koshy 	 * Traverse all child processes, attaching them to
1096ebccf1e3SJoseph Koshy 	 * this PMC.
1097ebccf1e3SJoseph Koshy 	 */
1098ebccf1e3SJoseph Koshy 
1099ebccf1e3SJoseph Koshy 	sx_slock(&proctree_lock);
1100ebccf1e3SJoseph Koshy 
1101ebccf1e3SJoseph Koshy 	top = p;
1102ebccf1e3SJoseph Koshy 
1103ebccf1e3SJoseph Koshy 	for (;;) {
1104ebccf1e3SJoseph Koshy 		if ((error = pmc_attach_one_process(p, pm)) != 0)
1105ebccf1e3SJoseph Koshy 			break;
1106ebccf1e3SJoseph Koshy 		if (!LIST_EMPTY(&p->p_children))
1107ebccf1e3SJoseph Koshy 			p = LIST_FIRST(&p->p_children);
1108ebccf1e3SJoseph Koshy 		else for (;;) {
1109ebccf1e3SJoseph Koshy 			if (p == top)
1110ebccf1e3SJoseph Koshy 				goto done;
1111ebccf1e3SJoseph Koshy 			if (LIST_NEXT(p, p_sibling)) {
1112ebccf1e3SJoseph Koshy 				p = LIST_NEXT(p, p_sibling);
1113ebccf1e3SJoseph Koshy 				break;
1114ebccf1e3SJoseph Koshy 			}
1115ebccf1e3SJoseph Koshy 			p = p->p_pptr;
1116ebccf1e3SJoseph Koshy 		}
1117ebccf1e3SJoseph Koshy 	}
1118ebccf1e3SJoseph Koshy 
1119ebccf1e3SJoseph Koshy 	if (error)
1120ebccf1e3SJoseph Koshy 		(void) pmc_detach_process(top, pm);
1121ebccf1e3SJoseph Koshy 
1122ebccf1e3SJoseph Koshy  done:
1123ebccf1e3SJoseph Koshy 	sx_sunlock(&proctree_lock);
1124ebccf1e3SJoseph Koshy 	return error;
1125ebccf1e3SJoseph Koshy }
1126ebccf1e3SJoseph Koshy 
1127ebccf1e3SJoseph Koshy /*
1128ebccf1e3SJoseph Koshy  * Detach a process from a PMC.  If there are no other PMCs tracking
1129ebccf1e3SJoseph Koshy  * this process, remove the process structure from its hash table.  If
1130ebccf1e3SJoseph Koshy  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1131ebccf1e3SJoseph Koshy  */
1132ebccf1e3SJoseph Koshy 
1133ebccf1e3SJoseph Koshy static int
1134ebccf1e3SJoseph Koshy pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1135ebccf1e3SJoseph Koshy {
1136ebccf1e3SJoseph Koshy 	int ri;
1137ebccf1e3SJoseph Koshy 	struct pmc_process *pp;
1138ebccf1e3SJoseph Koshy 
1139ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1140ebccf1e3SJoseph Koshy 
1141ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL,
1142ebccf1e3SJoseph Koshy 	    ("[pmc,%d] null pm pointer", __LINE__));
1143ebccf1e3SJoseph Koshy 
1144c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
1145ebccf1e3SJoseph Koshy 
11464a3690dfSJohn Baldwin 	PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1147c5153e19SJoseph Koshy 	    pm, ri, p, p->p_pid, p->p_comm, flags);
1148ebccf1e3SJoseph Koshy 
1149ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1150ebccf1e3SJoseph Koshy 		return ESRCH;
1151ebccf1e3SJoseph Koshy 
1152ebccf1e3SJoseph Koshy 	if (pp->pp_pmcs[ri].pp_pmc != pm)
1153ebccf1e3SJoseph Koshy 		return EINVAL;
1154ebccf1e3SJoseph Koshy 
1155ebccf1e3SJoseph Koshy 	pmc_unlink_target_process(pm, pp);
1156ebccf1e3SJoseph Koshy 
1157f263522aSJoseph Koshy 	/* Issue a detach entry if a log file is configured */
1158f263522aSJoseph Koshy 	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1159f263522aSJoseph Koshy 		pmclog_process_pmcdetach(pm, p->p_pid);
1160f263522aSJoseph Koshy 
1161ebccf1e3SJoseph Koshy 	/*
1162b790c193SPedro F. Giffuni 	 * If there are no PMCs targeting this process, we remove its
1163ebccf1e3SJoseph Koshy 	 * descriptor from the target hash table and unset the P_HWPMC
1164ebccf1e3SJoseph Koshy 	 * flag in the struct proc.
1165ebccf1e3SJoseph Koshy 	 */
1166fa630f35SFabien Thomas 	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1167ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Illegal refcnt %d for process struct %p",
1168ebccf1e3SJoseph Koshy 		__LINE__, pp->pp_refcnt, pp));
1169ebccf1e3SJoseph Koshy 
1170ebccf1e3SJoseph Koshy 	if (pp->pp_refcnt != 0)	/* still a target of some PMC */
1171ebccf1e3SJoseph Koshy 		return 0;
1172ebccf1e3SJoseph Koshy 
1173ebccf1e3SJoseph Koshy 	pmc_remove_process_descriptor(pp);
1174ebccf1e3SJoseph Koshy 
1175ebccf1e3SJoseph Koshy 	if (flags & PMC_FLAG_REMOVE)
11761ede983cSDag-Erling Smørgrav 		free(pp, M_PMC);
1177ebccf1e3SJoseph Koshy 
1178ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
1179ebccf1e3SJoseph Koshy 	p->p_flag &= ~P_HWPMC;
1180ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
1181ebccf1e3SJoseph Koshy 
1182ebccf1e3SJoseph Koshy 	return 0;
1183ebccf1e3SJoseph Koshy }
1184ebccf1e3SJoseph Koshy 
1185ebccf1e3SJoseph Koshy /*
1186ebccf1e3SJoseph Koshy  * Detach a process and optionally its descendants from a PMC.
1187ebccf1e3SJoseph Koshy  */
1188ebccf1e3SJoseph Koshy 
1189ebccf1e3SJoseph Koshy static int
1190ebccf1e3SJoseph Koshy pmc_detach_process(struct proc *p, struct pmc *pm)
1191ebccf1e3SJoseph Koshy {
1192ebccf1e3SJoseph Koshy 	struct proc *top;
1193ebccf1e3SJoseph Koshy 
1194ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1195ebccf1e3SJoseph Koshy 
11964a3690dfSJohn Baldwin 	PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1197c5153e19SJoseph Koshy 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1198ebccf1e3SJoseph Koshy 
1199ebccf1e3SJoseph Koshy 	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1200ebccf1e3SJoseph Koshy 		return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1201ebccf1e3SJoseph Koshy 
1202ebccf1e3SJoseph Koshy 	/*
1203ebccf1e3SJoseph Koshy 	 * Traverse all children, detaching them from this PMC.  We
1204ebccf1e3SJoseph Koshy 	 * ignore errors since we could be detaching a PMC from a
1205ebccf1e3SJoseph Koshy 	 * partially attached proc tree.
1206ebccf1e3SJoseph Koshy 	 */
1207ebccf1e3SJoseph Koshy 
1208ebccf1e3SJoseph Koshy 	sx_slock(&proctree_lock);
1209ebccf1e3SJoseph Koshy 
1210ebccf1e3SJoseph Koshy 	top = p;
1211ebccf1e3SJoseph Koshy 
1212ebccf1e3SJoseph Koshy 	for (;;) {
1213ebccf1e3SJoseph Koshy 		(void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1214ebccf1e3SJoseph Koshy 
1215ebccf1e3SJoseph Koshy 		if (!LIST_EMPTY(&p->p_children))
1216ebccf1e3SJoseph Koshy 			p = LIST_FIRST(&p->p_children);
1217ebccf1e3SJoseph Koshy 		else for (;;) {
1218ebccf1e3SJoseph Koshy 			if (p == top)
1219ebccf1e3SJoseph Koshy 				goto done;
1220ebccf1e3SJoseph Koshy 			if (LIST_NEXT(p, p_sibling)) {
1221ebccf1e3SJoseph Koshy 				p = LIST_NEXT(p, p_sibling);
1222ebccf1e3SJoseph Koshy 				break;
1223ebccf1e3SJoseph Koshy 			}
1224ebccf1e3SJoseph Koshy 			p = p->p_pptr;
1225ebccf1e3SJoseph Koshy 		}
1226ebccf1e3SJoseph Koshy 	}
1227ebccf1e3SJoseph Koshy 
1228ebccf1e3SJoseph Koshy  done:
1229ebccf1e3SJoseph Koshy 	sx_sunlock(&proctree_lock);
1230f263522aSJoseph Koshy 
1231f263522aSJoseph Koshy 	if (LIST_EMPTY(&pm->pm_targets))
1232f263522aSJoseph Koshy 		pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1233f263522aSJoseph Koshy 
1234ebccf1e3SJoseph Koshy 	return 0;
1235ebccf1e3SJoseph Koshy }
1236ebccf1e3SJoseph Koshy 
1237ebccf1e3SJoseph Koshy 
1238ebccf1e3SJoseph Koshy /*
1239ebccf1e3SJoseph Koshy  * Thread context switch IN
1240ebccf1e3SJoseph Koshy  */
1241ebccf1e3SJoseph Koshy 
1242f263522aSJoseph Koshy static void
1243f263522aSJoseph Koshy pmc_process_csw_in(struct thread *td)
1244ebccf1e3SJoseph Koshy {
1245ebccf1e3SJoseph Koshy 	int cpu;
1246e829eb6dSJoseph Koshy 	unsigned int adjri, ri;
1247ebccf1e3SJoseph Koshy 	struct pmc *pm;
1248ebccf1e3SJoseph Koshy 	struct proc *p;
1249ebccf1e3SJoseph Koshy 	struct pmc_cpu *pc;
1250ebccf1e3SJoseph Koshy 	struct pmc_hw *phw;
1251ebccf1e3SJoseph Koshy 	pmc_value_t newvalue;
1252e829eb6dSJoseph Koshy 	struct pmc_process *pp;
1253e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
1254ebccf1e3SJoseph Koshy 
1255ebccf1e3SJoseph Koshy 	p = td->td_proc;
1256ebccf1e3SJoseph Koshy 
1257ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1258f263522aSJoseph Koshy 		return;
1259ebccf1e3SJoseph Koshy 
1260ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_proc == td->td_proc,
1261ebccf1e3SJoseph Koshy 	    ("[pmc,%d] not my thread state", __LINE__));
1262ebccf1e3SJoseph Koshy 
1263f263522aSJoseph Koshy 	critical_enter(); /* no preemption from this point */
1264ebccf1e3SJoseph Koshy 
1265ebccf1e3SJoseph Koshy 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1266ebccf1e3SJoseph Koshy 
12674a3690dfSJohn Baldwin 	PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1268ebccf1e3SJoseph Koshy 	    p->p_pid, p->p_comm, pp);
1269ebccf1e3SJoseph Koshy 
1270122ccdc1SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1271b790c193SPedro F. Giffuni 	    ("[pmc,%d] weird CPU id %d", __LINE__, cpu));
1272ebccf1e3SJoseph Koshy 
1273ebccf1e3SJoseph Koshy 	pc = pmc_pcpu[cpu];
1274ebccf1e3SJoseph Koshy 
1275ebccf1e3SJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++) {
1276ebccf1e3SJoseph Koshy 
1277ebccf1e3SJoseph Koshy 		if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1278ebccf1e3SJoseph Koshy 			continue;
1279ebccf1e3SJoseph Koshy 
1280c5153e19SJoseph Koshy 		KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1281ebccf1e3SJoseph Koshy 		    ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1282c5153e19SJoseph Koshy 			__LINE__, PMC_TO_MODE(pm)));
1283ebccf1e3SJoseph Koshy 
1284c5153e19SJoseph Koshy 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1285ebccf1e3SJoseph Koshy 		    ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1286c5153e19SJoseph Koshy 			__LINE__, PMC_TO_ROWINDEX(pm), ri));
1287ebccf1e3SJoseph Koshy 
1288ebccf1e3SJoseph Koshy 		/*
1289ebccf1e3SJoseph Koshy 		 * Only PMCs that are marked as 'RUNNING' need
1290ebccf1e3SJoseph Koshy 		 * be placed on hardware.
1291ebccf1e3SJoseph Koshy 		 */
1292ebccf1e3SJoseph Koshy 
1293ebccf1e3SJoseph Koshy 		if (pm->pm_state != PMC_STATE_RUNNING)
1294ebccf1e3SJoseph Koshy 			continue;
1295ebccf1e3SJoseph Koshy 
1296ebccf1e3SJoseph Koshy 		/* increment PMC runcount */
1297e6b475e0SMatt Macy 		counter_u64_add(pm->pm_runcount, 1);
1298ebccf1e3SJoseph Koshy 
1299ebccf1e3SJoseph Koshy 		/* configure the HWPMC we are going to use. */
1300e829eb6dSJoseph Koshy 		pcd = pmc_ri_to_classdep(md, ri, &adjri);
1301e829eb6dSJoseph Koshy 		pcd->pcd_config_pmc(cpu, adjri, pm);
1302ebccf1e3SJoseph Koshy 
1303ebccf1e3SJoseph Koshy 		phw = pc->pc_hwpmcs[ri];
1304ebccf1e3SJoseph Koshy 
1305ebccf1e3SJoseph Koshy 		KASSERT(phw != NULL,
1306ebccf1e3SJoseph Koshy 		    ("[pmc,%d] null hw pointer", __LINE__));
1307ebccf1e3SJoseph Koshy 
1308ebccf1e3SJoseph Koshy 		KASSERT(phw->phw_pmc == pm,
1309ebccf1e3SJoseph Koshy 		    ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1310ebccf1e3SJoseph Koshy 			phw->phw_pmc, pm));
1311ebccf1e3SJoseph Koshy 
1312f263522aSJoseph Koshy 		/*
1313f263522aSJoseph Koshy 		 * Write out saved value and start the PMC.
1314f263522aSJoseph Koshy 		 *
1315f263522aSJoseph Koshy 		 * Sampling PMCs use a per-process value, while
1316f263522aSJoseph Koshy 		 * counting mode PMCs use a per-pmc value that is
1317f263522aSJoseph Koshy 		 * inherited across descendants.
1318f263522aSJoseph Koshy 		 */
1319f263522aSJoseph Koshy 		if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1320f263522aSJoseph Koshy 			mtx_pool_lock_spin(pmc_mtxpool, pm);
13215eaa6f01SJonathan T. Looney 
13225eaa6f01SJonathan T. Looney 			/*
13235eaa6f01SJonathan T. Looney 			 * Use the saved value calculated after the most recent
13245eaa6f01SJonathan T. Looney 			 * thread switch out to start this counter.  Reset
13255eaa6f01SJonathan T. Looney 			 * the saved count in case another thread from this
13265eaa6f01SJonathan T. Looney 			 * process switches in before any threads switch out.
13275eaa6f01SJonathan T. Looney 			 */
1328f263522aSJoseph Koshy 			newvalue = PMC_PCPU_SAVED(cpu,ri) =
1329f263522aSJoseph Koshy 			    pp->pp_pmcs[ri].pp_pmcval;
13305eaa6f01SJonathan T. Looney 			pp->pp_pmcs[ri].pp_pmcval = pm->pm_sc.pm_reloadcount;
1331f263522aSJoseph Koshy 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
1332f263522aSJoseph Koshy 		} else {
1333f263522aSJoseph Koshy 			KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1334f263522aSJoseph Koshy 			    ("[pmc,%d] illegal mode=%d", __LINE__,
1335f263522aSJoseph Koshy 			    PMC_TO_MODE(pm)));
1336ebccf1e3SJoseph Koshy 			mtx_pool_lock_spin(pmc_mtxpool, pm);
1337ebccf1e3SJoseph Koshy 			newvalue = PMC_PCPU_SAVED(cpu, ri) =
1338ebccf1e3SJoseph Koshy 			    pm->pm_gv.pm_savedvalue;
1339ebccf1e3SJoseph Koshy 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
1340f263522aSJoseph Koshy 		}
1341f263522aSJoseph Koshy 
13424a3690dfSJohn Baldwin 		PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1343ebccf1e3SJoseph Koshy 
1344e829eb6dSJoseph Koshy 		pcd->pcd_write_pmc(cpu, adjri, newvalue);
1345a3924968SJonathan T. Looney 
1346a3924968SJonathan T. Looney 		/* If a sampling mode PMC, reset stalled state. */
1347a3924968SJonathan T. Looney 		if (PMC_TO_MODE(pm) == PMC_MODE_TS)
1348e6b475e0SMatt Macy 			pm->pm_pcpu_state[cpu].pps_stalled = 0;
1349a3924968SJonathan T. Looney 
1350a3924968SJonathan T. Looney 		/* Indicate that we desire this to run. */
1351e6b475e0SMatt Macy 		pm->pm_pcpu_state[cpu].pps_cpustate = 1;
1352a3924968SJonathan T. Looney 
1353a3924968SJonathan T. Looney 		/* Start the PMC. */
1354e829eb6dSJoseph Koshy 		pcd->pcd_start_pmc(cpu, adjri);
1355ebccf1e3SJoseph Koshy 	}
1356ebccf1e3SJoseph Koshy 
1357ebccf1e3SJoseph Koshy 	/*
1358ebccf1e3SJoseph Koshy 	 * perform any other architecture/cpu dependent thread
1359ebccf1e3SJoseph Koshy 	 * switch-in actions.
1360ebccf1e3SJoseph Koshy 	 */
1361ebccf1e3SJoseph Koshy 
13626b8c8cd8SJoseph Koshy 	(void) (*md->pmd_switch_in)(pc, pp);
1363ebccf1e3SJoseph Koshy 
1364ebccf1e3SJoseph Koshy 	critical_exit();
1365ebccf1e3SJoseph Koshy 
1366ebccf1e3SJoseph Koshy }
1367ebccf1e3SJoseph Koshy 
1368ebccf1e3SJoseph Koshy /*
1369ebccf1e3SJoseph Koshy  * Thread context switch OUT.
1370ebccf1e3SJoseph Koshy  */
1371ebccf1e3SJoseph Koshy 
1372f263522aSJoseph Koshy static void
1373f263522aSJoseph Koshy pmc_process_csw_out(struct thread *td)
1374ebccf1e3SJoseph Koshy {
1375ebccf1e3SJoseph Koshy 	int cpu;
1376e829eb6dSJoseph Koshy 	int64_t tmp;
1377ebccf1e3SJoseph Koshy 	struct pmc *pm;
1378ebccf1e3SJoseph Koshy 	struct proc *p;
1379e829eb6dSJoseph Koshy 	enum pmc_mode mode;
1380ebccf1e3SJoseph Koshy 	struct pmc_cpu *pc;
1381f263522aSJoseph Koshy 	pmc_value_t newvalue;
1382e829eb6dSJoseph Koshy 	unsigned int adjri, ri;
1383e829eb6dSJoseph Koshy 	struct pmc_process *pp;
1384e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
1385e829eb6dSJoseph Koshy 
1386ebccf1e3SJoseph Koshy 
1387ebccf1e3SJoseph Koshy 	/*
1388ebccf1e3SJoseph Koshy 	 * Locate our process descriptor; this may be NULL if
1389ebccf1e3SJoseph Koshy 	 * this process is exiting and we have already removed
1390ebccf1e3SJoseph Koshy 	 * the process from the target process table.
1391ebccf1e3SJoseph Koshy 	 *
1392ebccf1e3SJoseph Koshy 	 * Note that due to kernel preemption, multiple
1393ebccf1e3SJoseph Koshy 	 * context switches may happen while the process is
1394ebccf1e3SJoseph Koshy 	 * exiting.
1395ebccf1e3SJoseph Koshy 	 *
1396ebccf1e3SJoseph Koshy 	 * Note also that if the target process cannot be
1397ebccf1e3SJoseph Koshy 	 * found we still need to deconfigure any PMCs that
1398ebccf1e3SJoseph Koshy 	 * are currently running on hardware.
1399ebccf1e3SJoseph Koshy 	 */
1400ebccf1e3SJoseph Koshy 
1401ebccf1e3SJoseph Koshy 	p = td->td_proc;
1402ebccf1e3SJoseph Koshy 	pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1403ebccf1e3SJoseph Koshy 
1404ebccf1e3SJoseph Koshy 	/*
1405ebccf1e3SJoseph Koshy 	 * save PMCs
1406ebccf1e3SJoseph Koshy 	 */
1407ebccf1e3SJoseph Koshy 
1408ebccf1e3SJoseph Koshy 	critical_enter();
1409ebccf1e3SJoseph Koshy 
1410ebccf1e3SJoseph Koshy 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1411ebccf1e3SJoseph Koshy 
14124a3690dfSJohn Baldwin 	PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1413ebccf1e3SJoseph Koshy 	    p->p_pid, p->p_comm, pp);
1414ebccf1e3SJoseph Koshy 
1415122ccdc1SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1416b790c193SPedro F. Giffuni 	    ("[pmc,%d weird CPU id %d", __LINE__, cpu));
1417ebccf1e3SJoseph Koshy 
1418ebccf1e3SJoseph Koshy 	pc = pmc_pcpu[cpu];
1419ebccf1e3SJoseph Koshy 
1420ebccf1e3SJoseph Koshy 	/*
1421ebccf1e3SJoseph Koshy 	 * When a PMC gets unlinked from a target PMC, it will
1422ebccf1e3SJoseph Koshy 	 * be removed from the target's pp_pmc[] array.
1423ebccf1e3SJoseph Koshy 	 *
1424ebccf1e3SJoseph Koshy 	 * However, on a MP system, the target could have been
1425ebccf1e3SJoseph Koshy 	 * executing on another CPU at the time of the unlink.
1426ebccf1e3SJoseph Koshy 	 * So, at context switch OUT time, we need to look at
1427ebccf1e3SJoseph Koshy 	 * the hardware to determine if a PMC is scheduled on
1428ebccf1e3SJoseph Koshy 	 * it.
1429ebccf1e3SJoseph Koshy 	 */
1430ebccf1e3SJoseph Koshy 
1431ebccf1e3SJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++) {
1432ebccf1e3SJoseph Koshy 
1433e829eb6dSJoseph Koshy 		pcd = pmc_ri_to_classdep(md, ri, &adjri);
1434c5153e19SJoseph Koshy 		pm  = NULL;
1435e829eb6dSJoseph Koshy 		(void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
1436ebccf1e3SJoseph Koshy 
1437ebccf1e3SJoseph Koshy 		if (pm == NULL)	/* nothing at this row index */
1438ebccf1e3SJoseph Koshy 			continue;
1439ebccf1e3SJoseph Koshy 
1440f263522aSJoseph Koshy 		mode = PMC_TO_MODE(pm);
1441f263522aSJoseph Koshy 		if (!PMC_IS_VIRTUAL_MODE(mode))
1442ebccf1e3SJoseph Koshy 			continue; /* not a process virtual PMC */
1443ebccf1e3SJoseph Koshy 
1444c5153e19SJoseph Koshy 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1445ebccf1e3SJoseph Koshy 		    ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1446c5153e19SJoseph Koshy 			__LINE__, PMC_TO_ROWINDEX(pm), ri));
1447ebccf1e3SJoseph Koshy 
1448a3924968SJonathan T. Looney 		/*
1449a3924968SJonathan T. Looney 		 * Change desired state, and then stop if not stalled.
1450a3924968SJonathan T. Looney 		 * This two-step dance should avoid race conditions where
1451a3924968SJonathan T. Looney 		 * an interrupt re-enables the PMC after this code has
1452a3924968SJonathan T. Looney 		 * already checked the pm_stalled flag.
1453a3924968SJonathan T. Looney 		 */
1454e6b475e0SMatt Macy 		pm->pm_pcpu_state[cpu].pps_cpustate = 0;
1455e6b475e0SMatt Macy 		if (pm->pm_pcpu_state[cpu].pps_stalled == 0)
1456e829eb6dSJoseph Koshy 			pcd->pcd_stop_pmc(cpu, adjri);
1457ebccf1e3SJoseph Koshy 
1458ebccf1e3SJoseph Koshy 		/* reduce this PMC's runcount */
1459e6b475e0SMatt Macy 		counter_u64_add(pm->pm_runcount, -1);
1460ebccf1e3SJoseph Koshy 
1461ebccf1e3SJoseph Koshy 		/*
1462ebccf1e3SJoseph Koshy 		 * If this PMC is associated with this process,
1463ebccf1e3SJoseph Koshy 		 * save the reading.
1464ebccf1e3SJoseph Koshy 		 */
1465ebccf1e3SJoseph Koshy 
1466593077d6SAndriy Gapon 		if (pm->pm_state != PMC_STATE_DELETED && pp != NULL &&
1467593077d6SAndriy Gapon 		    pp->pp_pmcs[ri].pp_pmc != NULL) {
1468ebccf1e3SJoseph Koshy 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1469f263522aSJoseph Koshy 			    ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1470f263522aSJoseph Koshy 				pm, ri, pp->pp_pmcs[ri].pp_pmc));
1471ebccf1e3SJoseph Koshy 
1472ebccf1e3SJoseph Koshy 			KASSERT(pp->pp_refcnt > 0,
1473ebccf1e3SJoseph Koshy 			    ("[pmc,%d] pp refcnt = %d", __LINE__,
1474ebccf1e3SJoseph Koshy 				pp->pp_refcnt));
1475ebccf1e3SJoseph Koshy 
1476e829eb6dSJoseph Koshy 			pcd->pcd_read_pmc(cpu, adjri, &newvalue);
1477ebccf1e3SJoseph Koshy 
1478f263522aSJoseph Koshy 			if (mode == PMC_MODE_TS) {
14795eaa6f01SJonathan T. Looney 				PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (samp)",
14805eaa6f01SJonathan T. Looney 				    cpu, ri, PMC_PCPU_SAVED(cpu,ri) - newvalue);
1481f263522aSJoseph Koshy 
1482f263522aSJoseph Koshy 				/*
1483f263522aSJoseph Koshy 				 * For sampling process-virtual PMCs,
14845eaa6f01SJonathan T. Looney 				 * newvalue is the number of events to be seen
14855eaa6f01SJonathan T. Looney 				 * until the next sampling interrupt.
14865eaa6f01SJonathan T. Looney 				 * We can just add the events left from this
14875eaa6f01SJonathan T. Looney 				 * invocation to the counter, then adjust
14885eaa6f01SJonathan T. Looney 				 * in case we overflow our range.
14895eaa6f01SJonathan T. Looney 				 *
14905eaa6f01SJonathan T. Looney 				 * (Recall that we reload the counter every
14915eaa6f01SJonathan T. Looney 				 * time we use it.)
1492f263522aSJoseph Koshy 				 */
1493f263522aSJoseph Koshy 				mtx_pool_lock_spin(pmc_mtxpool, pm);
14945eaa6f01SJonathan T. Looney 
14955eaa6f01SJonathan T. Looney 				pp->pp_pmcs[ri].pp_pmcval += newvalue;
14965eaa6f01SJonathan T. Looney 				if (pp->pp_pmcs[ri].pp_pmcval >
14975eaa6f01SJonathan T. Looney 				    pm->pm_sc.pm_reloadcount)
14985eaa6f01SJonathan T. Looney 					pp->pp_pmcs[ri].pp_pmcval -=
1499f263522aSJoseph Koshy 					    pm->pm_sc.pm_reloadcount;
15005eaa6f01SJonathan T. Looney 				KASSERT(pp->pp_pmcs[ri].pp_pmcval > 0 &&
15015eaa6f01SJonathan T. Looney 				    pp->pp_pmcs[ri].pp_pmcval <=
15025eaa6f01SJonathan T. Looney 				    pm->pm_sc.pm_reloadcount,
15035eaa6f01SJonathan T. Looney 				    ("[pmc,%d] pp_pmcval outside of expected "
15045eaa6f01SJonathan T. Looney 				    "range cpu=%d ri=%d pp_pmcval=%jx "
15055eaa6f01SJonathan T. Looney 				    "pm_reloadcount=%jx", __LINE__, cpu, ri,
15065eaa6f01SJonathan T. Looney 				    pp->pp_pmcs[ri].pp_pmcval,
15075eaa6f01SJonathan T. Looney 				    pm->pm_sc.pm_reloadcount));
1508f263522aSJoseph Koshy 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
1509f263522aSJoseph Koshy 
1510f263522aSJoseph Koshy 			} else {
15115eaa6f01SJonathan T. Looney 				tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
15125eaa6f01SJonathan T. Looney 
15135eaa6f01SJonathan T. Looney 				PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (count)",
15145eaa6f01SJonathan T. Looney 				    cpu, ri, tmp);
1515f263522aSJoseph Koshy 
1516f263522aSJoseph Koshy 				/*
1517f263522aSJoseph Koshy 				 * For counting process-virtual PMCs,
1518f263522aSJoseph Koshy 				 * we expect the count to be
1519f263522aSJoseph Koshy 				 * increasing monotonically, modulo a 64
1520f263522aSJoseph Koshy 				 * bit wraparound.
1521f263522aSJoseph Koshy 				 */
15220c8cc7b0SKonstantin Belousov 				KASSERT(tmp >= 0,
1523ebccf1e3SJoseph Koshy 				    ("[pmc,%d] negative increment cpu=%d "
1524ebccf1e3SJoseph Koshy 				     "ri=%d newvalue=%jx saved=%jx "
1525ebccf1e3SJoseph Koshy 				     "incr=%jx", __LINE__, cpu, ri,
1526f263522aSJoseph Koshy 				     newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1527ebccf1e3SJoseph Koshy 
1528ebccf1e3SJoseph Koshy 				mtx_pool_lock_spin(pmc_mtxpool, pm);
1529ebccf1e3SJoseph Koshy 				pm->pm_gv.pm_savedvalue += tmp;
1530ebccf1e3SJoseph Koshy 				pp->pp_pmcs[ri].pp_pmcval += tmp;
1531ebccf1e3SJoseph Koshy 				mtx_pool_unlock_spin(pmc_mtxpool, pm);
1532ebccf1e3SJoseph Koshy 
1533f263522aSJoseph Koshy 				if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1534f263522aSJoseph Koshy 					pmclog_process_proccsw(pm, pp, tmp);
1535f263522aSJoseph Koshy 			}
1536ebccf1e3SJoseph Koshy 		}
1537ebccf1e3SJoseph Koshy 
1538ebccf1e3SJoseph Koshy 		/* mark hardware as free */
1539e829eb6dSJoseph Koshy 		pcd->pcd_config_pmc(cpu, adjri, NULL);
1540ebccf1e3SJoseph Koshy 	}
1541ebccf1e3SJoseph Koshy 
1542ebccf1e3SJoseph Koshy 	/*
1543ebccf1e3SJoseph Koshy 	 * perform any other architecture/cpu dependent thread
1544ebccf1e3SJoseph Koshy 	 * switch out functions.
1545ebccf1e3SJoseph Koshy 	 */
1546ebccf1e3SJoseph Koshy 
15476b8c8cd8SJoseph Koshy 	(void) (*md->pmd_switch_out)(pc, pp);
1548ebccf1e3SJoseph Koshy 
1549ebccf1e3SJoseph Koshy 	critical_exit();
1550f263522aSJoseph Koshy }
1551f263522aSJoseph Koshy 
1552f263522aSJoseph Koshy /*
155349874f6eSJoseph Koshy  * A mapping change for a process.
155449874f6eSJoseph Koshy  */
155549874f6eSJoseph Koshy 
155649874f6eSJoseph Koshy static void
155749874f6eSJoseph Koshy pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
155849874f6eSJoseph Koshy {
155949874f6eSJoseph Koshy 	int ri;
156049874f6eSJoseph Koshy 	pid_t pid;
156149874f6eSJoseph Koshy 	char *fullpath, *freepath;
156249874f6eSJoseph Koshy 	const struct pmc *pm;
156349874f6eSJoseph Koshy 	struct pmc_owner *po;
156449874f6eSJoseph Koshy 	const struct pmc_process *pp;
156549874f6eSJoseph Koshy 
156649874f6eSJoseph Koshy 	freepath = fullpath = NULL;
1567*0f00315cSMatt Macy 	epoch_exit(global_epoch);
156849874f6eSJoseph Koshy 	pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
156949874f6eSJoseph Koshy 
157049874f6eSJoseph Koshy 	pid = td->td_proc->p_pid;
157149874f6eSJoseph Koshy 
1572*0f00315cSMatt Macy 	epoch_enter(global_epoch);
157349874f6eSJoseph Koshy 	/* Inform owners of all system-wide sampling PMCs. */
1574*0f00315cSMatt Macy 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
157549874f6eSJoseph Koshy 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
157649874f6eSJoseph Koshy 			pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
157749874f6eSJoseph Koshy 
157849874f6eSJoseph Koshy 	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
157949874f6eSJoseph Koshy 		goto done;
158049874f6eSJoseph Koshy 
158149874f6eSJoseph Koshy 	/*
158249874f6eSJoseph Koshy 	 * Inform sampling PMC owners tracking this process.
158349874f6eSJoseph Koshy 	 */
158449874f6eSJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++)
158549874f6eSJoseph Koshy 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
158649874f6eSJoseph Koshy 		    PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
158749874f6eSJoseph Koshy 			pmclog_process_map_in(pm->pm_owner,
158849874f6eSJoseph Koshy 			    pid, pkm->pm_address, fullpath);
158949874f6eSJoseph Koshy 
159049874f6eSJoseph Koshy   done:
159149874f6eSJoseph Koshy 	if (freepath)
15921ede983cSDag-Erling Smørgrav 		free(freepath, M_TEMP);
159349874f6eSJoseph Koshy }
159449874f6eSJoseph Koshy 
159549874f6eSJoseph Koshy 
159649874f6eSJoseph Koshy /*
159749874f6eSJoseph Koshy  * Log an munmap request.
159849874f6eSJoseph Koshy  */
159949874f6eSJoseph Koshy 
160049874f6eSJoseph Koshy static void
160149874f6eSJoseph Koshy pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
160249874f6eSJoseph Koshy {
160349874f6eSJoseph Koshy 	int ri;
160449874f6eSJoseph Koshy 	pid_t pid;
160549874f6eSJoseph Koshy 	struct pmc_owner *po;
160649874f6eSJoseph Koshy 	const struct pmc *pm;
160749874f6eSJoseph Koshy 	const struct pmc_process *pp;
160849874f6eSJoseph Koshy 
160949874f6eSJoseph Koshy 	pid = td->td_proc->p_pid;
161049874f6eSJoseph Koshy 
1611*0f00315cSMatt Macy 	epoch_enter(global_epoch);
1612*0f00315cSMatt Macy 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
161349874f6eSJoseph Koshy 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
161449874f6eSJoseph Koshy 		pmclog_process_map_out(po, pid, pkm->pm_address,
161549874f6eSJoseph Koshy 		    pkm->pm_address + pkm->pm_size);
1616*0f00315cSMatt Macy 	epoch_exit(global_epoch);
161749874f6eSJoseph Koshy 
161849874f6eSJoseph Koshy 	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
161949874f6eSJoseph Koshy 		return;
162049874f6eSJoseph Koshy 
162149874f6eSJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++)
162249874f6eSJoseph Koshy 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
162349874f6eSJoseph Koshy 		    PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
16247c9f60e3SJoseph Koshy 			pmclog_process_map_out(pm->pm_owner, pid,
16257c9f60e3SJoseph Koshy 			    pkm->pm_address, pkm->pm_address + pkm->pm_size);
162649874f6eSJoseph Koshy }
162749874f6eSJoseph Koshy 
162849874f6eSJoseph Koshy /*
1629d07f36b0SJoseph Koshy  * Log mapping information about the kernel.
1630d07f36b0SJoseph Koshy  */
1631d07f36b0SJoseph Koshy 
1632d07f36b0SJoseph Koshy static void
1633d07f36b0SJoseph Koshy pmc_log_kernel_mappings(struct pmc *pm)
1634d07f36b0SJoseph Koshy {
1635d07f36b0SJoseph Koshy 	struct pmc_owner *po;
1636d07f36b0SJoseph Koshy 	struct pmckern_map_in *km, *kmbase;
1637d07f36b0SJoseph Koshy 
1638*0f00315cSMatt Macy 	MPASS(in_epoch() || sx_xlocked(&pmc_sx));
1639d07f36b0SJoseph Koshy 	KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1640d07f36b0SJoseph Koshy 	    ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1641d07f36b0SJoseph Koshy 		__LINE__, (void *) pm));
1642d07f36b0SJoseph Koshy 
1643d07f36b0SJoseph Koshy 	po = pm->pm_owner;
1644d07f36b0SJoseph Koshy 
1645d07f36b0SJoseph Koshy 	if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1646d07f36b0SJoseph Koshy 		return;
1647d07f36b0SJoseph Koshy 
1648d07f36b0SJoseph Koshy 	/*
1649d07f36b0SJoseph Koshy 	 * Log the current set of kernel modules.
1650d07f36b0SJoseph Koshy 	 */
1651d07f36b0SJoseph Koshy 	kmbase = linker_hwpmc_list_objects();
1652d07f36b0SJoseph Koshy 	for (km = kmbase; km->pm_file != NULL; km++) {
16534a3690dfSJohn Baldwin 		PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file,
1654d07f36b0SJoseph Koshy 		    (void *) km->pm_address);
1655d07f36b0SJoseph Koshy 		pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1656d07f36b0SJoseph Koshy 		    km->pm_file);
1657d07f36b0SJoseph Koshy 	}
16581ede983cSDag-Erling Smørgrav 	free(kmbase, M_LINKER);
1659d07f36b0SJoseph Koshy 
1660d07f36b0SJoseph Koshy 	po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1661d07f36b0SJoseph Koshy }
1662d07f36b0SJoseph Koshy 
1663d07f36b0SJoseph Koshy /*
1664d07f36b0SJoseph Koshy  * Log the mappings for a single process.
1665d07f36b0SJoseph Koshy  */
1666d07f36b0SJoseph Koshy 
1667d07f36b0SJoseph Koshy static void
1668d07f36b0SJoseph Koshy pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1669d07f36b0SJoseph Koshy {
1670bf792d68SJoseph Koshy 	vm_map_t map;
1671bf792d68SJoseph Koshy 	struct vnode *vp;
1672bf792d68SJoseph Koshy 	struct vmspace *vm;
1673bf792d68SJoseph Koshy 	vm_map_entry_t entry;
1674bf792d68SJoseph Koshy 	vm_offset_t last_end;
1675bf792d68SJoseph Koshy 	u_int last_timestamp;
1676bf792d68SJoseph Koshy 	struct vnode *last_vp;
1677bf792d68SJoseph Koshy 	vm_offset_t start_addr;
1678bf792d68SJoseph Koshy 	vm_object_t obj, lobj, tobj;
1679bf792d68SJoseph Koshy 	char *fullpath, *freepath;
1680bf792d68SJoseph Koshy 
1681bf792d68SJoseph Koshy 	last_vp = NULL;
1682bf792d68SJoseph Koshy 	last_end = (vm_offset_t) 0;
1683bf792d68SJoseph Koshy 	fullpath = freepath = NULL;
1684bf792d68SJoseph Koshy 
1685bf792d68SJoseph Koshy 	if ((vm = vmspace_acquire_ref(p)) == NULL)
1686bf792d68SJoseph Koshy 		return;
1687bf792d68SJoseph Koshy 
1688bf792d68SJoseph Koshy 	map = &vm->vm_map;
1689bf792d68SJoseph Koshy 	vm_map_lock_read(map);
1690bf792d68SJoseph Koshy 
1691bf792d68SJoseph Koshy 	for (entry = map->header.next; entry != &map->header; entry = entry->next) {
1692bf792d68SJoseph Koshy 
1693bf792d68SJoseph Koshy 		if (entry == NULL) {
16944a3690dfSJohn Baldwin 			PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
1695bf792d68SJoseph Koshy 			    "NULL! pid=%d vm_map=%p\n", p->p_pid, map);
1696bf792d68SJoseph Koshy 			break;
1697bf792d68SJoseph Koshy 		}
1698bf792d68SJoseph Koshy 
1699bf792d68SJoseph Koshy 		/*
1700bf792d68SJoseph Koshy 		 * We only care about executable map entries.
1701bf792d68SJoseph Koshy 		 */
1702bf792d68SJoseph Koshy 		if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) ||
1703bf792d68SJoseph Koshy 		    !(entry->protection & VM_PROT_EXECUTE) ||
1704bf792d68SJoseph Koshy 		    (entry->object.vm_object == NULL)) {
1705bf792d68SJoseph Koshy 			continue;
1706bf792d68SJoseph Koshy 		}
1707bf792d68SJoseph Koshy 
1708bf792d68SJoseph Koshy 		obj = entry->object.vm_object;
170966c392dfSAlan Cox 		VM_OBJECT_RLOCK(obj);
1710bf792d68SJoseph Koshy 
1711bf792d68SJoseph Koshy 		/*
1712bf792d68SJoseph Koshy 		 * Walk the backing_object list to find the base
1713bf792d68SJoseph Koshy 		 * (non-shadowed) vm_object.
1714bf792d68SJoseph Koshy 		 */
1715bf792d68SJoseph Koshy 		for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
1716bf792d68SJoseph Koshy 			if (tobj != obj)
171766c392dfSAlan Cox 				VM_OBJECT_RLOCK(tobj);
1718bf792d68SJoseph Koshy 			if (lobj != obj)
171966c392dfSAlan Cox 				VM_OBJECT_RUNLOCK(lobj);
1720bf792d68SJoseph Koshy 			lobj = tobj;
1721bf792d68SJoseph Koshy 		}
1722bf792d68SJoseph Koshy 
1723bf792d68SJoseph Koshy 		/*
1724bf792d68SJoseph Koshy 		 * At this point lobj is the base vm_object and it is locked.
1725bf792d68SJoseph Koshy 		 */
1726bf792d68SJoseph Koshy 		if (lobj == NULL) {
17274a3690dfSJohn Baldwin 			PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
1728bf792d68SJoseph Koshy 			    "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
172966c392dfSAlan Cox 			VM_OBJECT_RUNLOCK(obj);
1730bf792d68SJoseph Koshy 			continue;
1731bf792d68SJoseph Koshy 		}
1732bf792d68SJoseph Koshy 
173363e4c6cdSEric van Gyzen 		vp = vm_object_vnode(lobj);
173463e4c6cdSEric van Gyzen 		if (vp == NULL) {
1735bf792d68SJoseph Koshy 			if (lobj != obj)
173666c392dfSAlan Cox 				VM_OBJECT_RUNLOCK(lobj);
173766c392dfSAlan Cox 			VM_OBJECT_RUNLOCK(obj);
1738bf792d68SJoseph Koshy 			continue;
1739bf792d68SJoseph Koshy 		}
1740bf792d68SJoseph Koshy 
1741bf792d68SJoseph Koshy 		/*
1742bf792d68SJoseph Koshy 		 * Skip contiguous regions that point to the same
1743bf792d68SJoseph Koshy 		 * vnode, so we don't emit redundant MAP-IN
1744bf792d68SJoseph Koshy 		 * directives.
1745bf792d68SJoseph Koshy 		 */
174663e4c6cdSEric van Gyzen 		if (entry->start == last_end && vp == last_vp) {
1747bf792d68SJoseph Koshy 			last_end = entry->end;
1748bf792d68SJoseph Koshy 			if (lobj != obj)
174966c392dfSAlan Cox 				VM_OBJECT_RUNLOCK(lobj);
175066c392dfSAlan Cox 			VM_OBJECT_RUNLOCK(obj);
1751bf792d68SJoseph Koshy 			continue;
1752bf792d68SJoseph Koshy 		}
1753bf792d68SJoseph Koshy 
1754bf792d68SJoseph Koshy 		/*
1755bf792d68SJoseph Koshy 		 * We don't want to keep the proc's vm_map or this
1756bf792d68SJoseph Koshy 		 * vm_object locked while we walk the pathname, since
1757bf792d68SJoseph Koshy 		 * vn_fullpath() can sleep.  However, if we drop the
1758bf792d68SJoseph Koshy 		 * lock, it's possible for concurrent activity to
1759bf792d68SJoseph Koshy 		 * modify the vm_map list.  To protect against this,
1760bf792d68SJoseph Koshy 		 * we save the vm_map timestamp before we release the
1761bf792d68SJoseph Koshy 		 * lock, and check it after we reacquire the lock
1762bf792d68SJoseph Koshy 		 * below.
1763bf792d68SJoseph Koshy 		 */
1764bf792d68SJoseph Koshy 		start_addr = entry->start;
1765bf792d68SJoseph Koshy 		last_end = entry->end;
1766bf792d68SJoseph Koshy 		last_timestamp = map->timestamp;
1767bf792d68SJoseph Koshy 		vm_map_unlock_read(map);
1768bf792d68SJoseph Koshy 
1769bf792d68SJoseph Koshy 		vref(vp);
1770bf792d68SJoseph Koshy 		if (lobj != obj)
177166c392dfSAlan Cox 			VM_OBJECT_RUNLOCK(lobj);
1772bf792d68SJoseph Koshy 
177366c392dfSAlan Cox 		VM_OBJECT_RUNLOCK(obj);
1774bf792d68SJoseph Koshy 
1775bf792d68SJoseph Koshy 		freepath = NULL;
1776bf792d68SJoseph Koshy 		pmc_getfilename(vp, &fullpath, &freepath);
1777bf792d68SJoseph Koshy 		last_vp = vp;
1778e9b5dc16SJoseph Koshy 
1779bf792d68SJoseph Koshy 		vrele(vp);
1780e9b5dc16SJoseph Koshy 
1781bf792d68SJoseph Koshy 		vp = NULL;
1782bf792d68SJoseph Koshy 		pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
1783bf792d68SJoseph Koshy 		if (freepath)
1784bf792d68SJoseph Koshy 			free(freepath, M_TEMP);
1785bf792d68SJoseph Koshy 
1786bf792d68SJoseph Koshy 		vm_map_lock_read(map);
1787bf792d68SJoseph Koshy 
1788bf792d68SJoseph Koshy 		/*
1789bf792d68SJoseph Koshy 		 * If our saved timestamp doesn't match, this means
1790bf792d68SJoseph Koshy 		 * that the vm_map was modified out from under us and
1791bf792d68SJoseph Koshy 		 * we can't trust our current "entry" pointer.  Do a
1792bf792d68SJoseph Koshy 		 * new lookup for this entry.  If there is no entry
1793bf792d68SJoseph Koshy 		 * for this address range, vm_map_lookup_entry() will
1794bf792d68SJoseph Koshy 		 * return the previous one, so we always want to go to
1795bf792d68SJoseph Koshy 		 * entry->next on the next loop iteration.
1796bf792d68SJoseph Koshy 		 *
1797bf792d68SJoseph Koshy 		 * There is an edge condition here that can occur if
1798bf792d68SJoseph Koshy 		 * there is no entry at or before this address.  In
1799bf792d68SJoseph Koshy 		 * this situation, vm_map_lookup_entry returns
1800bf792d68SJoseph Koshy 		 * &map->header, which would cause our loop to abort
1801bf792d68SJoseph Koshy 		 * without processing the rest of the map.  However,
1802bf792d68SJoseph Koshy 		 * in practice this will never happen for process
1803bf792d68SJoseph Koshy 		 * vm_map.  This is because the executable's text
1804bf792d68SJoseph Koshy 		 * segment is the first mapping in the proc's address
1805bf792d68SJoseph Koshy 		 * space, and this mapping is never removed until the
1806bf792d68SJoseph Koshy 		 * process exits, so there will always be a non-header
1807bf792d68SJoseph Koshy 		 * entry at or before the requested address for
1808bf792d68SJoseph Koshy 		 * vm_map_lookup_entry to return.
1809bf792d68SJoseph Koshy 		 */
1810bf792d68SJoseph Koshy 		if (map->timestamp != last_timestamp)
1811bf792d68SJoseph Koshy 			vm_map_lookup_entry(map, last_end - 1, &entry);
1812bf792d68SJoseph Koshy 	}
1813bf792d68SJoseph Koshy 
1814bf792d68SJoseph Koshy 	vm_map_unlock_read(map);
1815bf792d68SJoseph Koshy 	vmspace_free(vm);
1816bf792d68SJoseph Koshy 	return;
1817d07f36b0SJoseph Koshy }
1818d07f36b0SJoseph Koshy 
1819d07f36b0SJoseph Koshy /*
1820d07f36b0SJoseph Koshy  * Log mappings for all processes in the system.
1821d07f36b0SJoseph Koshy  */
1822d07f36b0SJoseph Koshy 
1823d07f36b0SJoseph Koshy static void
1824d07f36b0SJoseph Koshy pmc_log_all_process_mappings(struct pmc_owner *po)
1825d07f36b0SJoseph Koshy {
1826d07f36b0SJoseph Koshy 	struct proc *p, *top;
1827d07f36b0SJoseph Koshy 
1828d07f36b0SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
1829d07f36b0SJoseph Koshy 
1830d07f36b0SJoseph Koshy 	if ((p = pfind(1)) == NULL)
1831d07f36b0SJoseph Koshy 		panic("[pmc,%d] Cannot find init", __LINE__);
1832d07f36b0SJoseph Koshy 
1833d07f36b0SJoseph Koshy 	PROC_UNLOCK(p);
1834d07f36b0SJoseph Koshy 
1835d07f36b0SJoseph Koshy 	sx_slock(&proctree_lock);
1836d07f36b0SJoseph Koshy 
1837d07f36b0SJoseph Koshy 	top = p;
1838d07f36b0SJoseph Koshy 
1839d07f36b0SJoseph Koshy 	for (;;) {
1840d07f36b0SJoseph Koshy 		pmc_log_process_mappings(po, p);
1841d07f36b0SJoseph Koshy 		if (!LIST_EMPTY(&p->p_children))
1842d07f36b0SJoseph Koshy 			p = LIST_FIRST(&p->p_children);
1843d07f36b0SJoseph Koshy 		else for (;;) {
1844d07f36b0SJoseph Koshy 			if (p == top)
1845d07f36b0SJoseph Koshy 				goto done;
1846d07f36b0SJoseph Koshy 			if (LIST_NEXT(p, p_sibling)) {
1847d07f36b0SJoseph Koshy 				p = LIST_NEXT(p, p_sibling);
1848d07f36b0SJoseph Koshy 				break;
1849d07f36b0SJoseph Koshy 			}
1850d07f36b0SJoseph Koshy 			p = p->p_pptr;
1851d07f36b0SJoseph Koshy 		}
1852d07f36b0SJoseph Koshy 	}
1853d07f36b0SJoseph Koshy  done:
1854d07f36b0SJoseph Koshy 	sx_sunlock(&proctree_lock);
1855d07f36b0SJoseph Koshy }
1856d07f36b0SJoseph Koshy 
1857d07f36b0SJoseph Koshy /*
1858f263522aSJoseph Koshy  * The 'hook' invoked from the kernel proper
1859f263522aSJoseph Koshy  */
1860f263522aSJoseph Koshy 
1861f263522aSJoseph Koshy 
1862680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
1863f263522aSJoseph Koshy const char *pmc_hooknames[] = {
186449874f6eSJoseph Koshy 	/* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
1865f263522aSJoseph Koshy 	"",
1866f263522aSJoseph Koshy 	"EXEC",
1867f263522aSJoseph Koshy 	"CSW-IN",
1868f263522aSJoseph Koshy 	"CSW-OUT",
186949874f6eSJoseph Koshy 	"SAMPLE",
187029f4e216SMark Johnston 	"UNUSED1",
187129f4e216SMark Johnston 	"UNUSED2",
187249874f6eSJoseph Koshy 	"MMAP",
1873d07f36b0SJoseph Koshy 	"MUNMAP",
1874f5f9340bSFabien Thomas 	"CALLCHAIN-NMI",
1875f5f9340bSFabien Thomas 	"CALLCHAIN-SOFT",
1876f5f9340bSFabien Thomas 	"SOFTSAMPLING"
1877f263522aSJoseph Koshy };
1878f263522aSJoseph Koshy #endif
1879f263522aSJoseph Koshy 
1880f263522aSJoseph Koshy static int
1881f263522aSJoseph Koshy pmc_hook_handler(struct thread *td, int function, void *arg)
1882f263522aSJoseph Koshy {
1883f263522aSJoseph Koshy 
18844a3690dfSJohn Baldwin 	PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
1885f263522aSJoseph Koshy 	    pmc_hooknames[function], arg);
1886f263522aSJoseph Koshy 
1887f263522aSJoseph Koshy 	switch (function)
1888f263522aSJoseph Koshy 	{
1889f263522aSJoseph Koshy 
1890f263522aSJoseph Koshy 	/*
1891f263522aSJoseph Koshy 	 * Process exec()
1892f263522aSJoseph Koshy 	 */
1893f263522aSJoseph Koshy 
1894f263522aSJoseph Koshy 	case PMC_FN_PROCESS_EXEC:
1895f263522aSJoseph Koshy 	{
1896f263522aSJoseph Koshy 		char *fullpath, *freepath;
1897f263522aSJoseph Koshy 		unsigned int ri;
1898f263522aSJoseph Koshy 		int is_using_hwpmcs;
1899f263522aSJoseph Koshy 		struct pmc *pm;
1900f263522aSJoseph Koshy 		struct proc *p;
1901f263522aSJoseph Koshy 		struct pmc_owner *po;
1902f263522aSJoseph Koshy 		struct pmc_process *pp;
190315139246SJoseph Koshy 		struct pmckern_procexec *pk;
1904f263522aSJoseph Koshy 
1905f263522aSJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
1906f263522aSJoseph Koshy 
1907f263522aSJoseph Koshy 		p = td->td_proc;
190815139246SJoseph Koshy 		pmc_getfilename(p->p_textvp, &fullpath, &freepath);
190915139246SJoseph Koshy 
191015139246SJoseph Koshy 		pk = (struct pmckern_procexec *) arg;
1911f263522aSJoseph Koshy 
1912*0f00315cSMatt Macy 		epoch_enter(global_epoch);
1913f263522aSJoseph Koshy 		/* Inform owners of SS mode PMCs of the exec event. */
1914*0f00315cSMatt Macy 		CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1915f263522aSJoseph Koshy 		    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
191615139246SJoseph Koshy 			    pmclog_process_procexec(po, PMC_ID_INVALID,
191715139246SJoseph Koshy 				p->p_pid, pk->pm_entryaddr, fullpath);
1918*0f00315cSMatt Macy 		epoch_exit(global_epoch);
1919f263522aSJoseph Koshy 
1920f263522aSJoseph Koshy 		PROC_LOCK(p);
1921f263522aSJoseph Koshy 		is_using_hwpmcs = p->p_flag & P_HWPMC;
1922f263522aSJoseph Koshy 		PROC_UNLOCK(p);
1923f263522aSJoseph Koshy 
1924f263522aSJoseph Koshy 		if (!is_using_hwpmcs) {
1925f263522aSJoseph Koshy 			if (freepath)
19261ede983cSDag-Erling Smørgrav 				free(freepath, M_TEMP);
1927f263522aSJoseph Koshy 			break;
1928f263522aSJoseph Koshy 		}
1929f263522aSJoseph Koshy 
1930f263522aSJoseph Koshy 		/*
1931f263522aSJoseph Koshy 		 * PMCs are not inherited across an exec():  remove any
1932f263522aSJoseph Koshy 		 * PMCs that this process is the owner of.
1933f263522aSJoseph Koshy 		 */
1934f263522aSJoseph Koshy 
1935f263522aSJoseph Koshy 		if ((po = pmc_find_owner_descriptor(p)) != NULL) {
1936f263522aSJoseph Koshy 			pmc_remove_owner(po);
1937f263522aSJoseph Koshy 			pmc_destroy_owner_descriptor(po);
1938f263522aSJoseph Koshy 		}
1939f263522aSJoseph Koshy 
1940f263522aSJoseph Koshy 		/*
19412b01a084SJoseph Koshy 		 * If the process being exec'ed is not the target of any
19422b01a084SJoseph Koshy 		 * PMC, we are done.
1943f263522aSJoseph Koshy 		 */
19442b01a084SJoseph Koshy 		if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
19452b01a084SJoseph Koshy 			if (freepath)
19461ede983cSDag-Erling Smørgrav 				free(freepath, M_TEMP);
1947f263522aSJoseph Koshy 			break;
19482b01a084SJoseph Koshy 		}
1949f263522aSJoseph Koshy 
1950f263522aSJoseph Koshy 		/*
1951f263522aSJoseph Koshy 		 * Log the exec event to all monitoring owners.  Skip
1952b790c193SPedro F. Giffuni 		 * owners who have already received the event because
19532b01a084SJoseph Koshy 		 * they had system sampling PMCs active.
1954f263522aSJoseph Koshy 		 */
1955f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++)
1956f263522aSJoseph Koshy 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
1957f263522aSJoseph Koshy 				po = pm->pm_owner;
1958f263522aSJoseph Koshy 				if (po->po_sscount == 0 &&
1959f263522aSJoseph Koshy 				    po->po_flags & PMC_PO_OWNS_LOGFILE)
196015139246SJoseph Koshy 					pmclog_process_procexec(po, pm->pm_id,
196115139246SJoseph Koshy 					    p->p_pid, pk->pm_entryaddr,
1962f263522aSJoseph Koshy 					    fullpath);
1963f263522aSJoseph Koshy 			}
1964f263522aSJoseph Koshy 
1965f263522aSJoseph Koshy 		if (freepath)
19661ede983cSDag-Erling Smørgrav 			free(freepath, M_TEMP);
1967f263522aSJoseph Koshy 
1968f263522aSJoseph Koshy 
19694a3690dfSJohn Baldwin 		PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
197015139246SJoseph Koshy 		    p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
1971f263522aSJoseph Koshy 
197215139246SJoseph Koshy 		if (pk->pm_credentialschanged == 0) /* no change */
1973f263522aSJoseph Koshy 			break;
1974f263522aSJoseph Koshy 
1975f263522aSJoseph Koshy 		/*
1976f263522aSJoseph Koshy 		 * If the newly exec()'ed process has a different credential
1977f263522aSJoseph Koshy 		 * than before, allow it to be the target of a PMC only if
1978b790c193SPedro F. Giffuni 		 * the PMC's owner has sufficient privilege.
1979f263522aSJoseph Koshy 		 */
1980f263522aSJoseph Koshy 
1981f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++)
1982f263522aSJoseph Koshy 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
1983f263522aSJoseph Koshy 				if (pmc_can_attach(pm, td->td_proc) != 0)
1984f263522aSJoseph Koshy 					pmc_detach_one_process(td->td_proc,
1985f263522aSJoseph Koshy 					    pm, PMC_FLAG_NONE);
1986f263522aSJoseph Koshy 
1987fa630f35SFabien Thomas 		KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1988f263522aSJoseph Koshy 		    ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
1989f263522aSJoseph Koshy 			pp->pp_refcnt, pp));
1990f263522aSJoseph Koshy 
1991f263522aSJoseph Koshy 		/*
1992f263522aSJoseph Koshy 		 * If this process is no longer the target of any
1993f263522aSJoseph Koshy 		 * PMCs, we can remove the process entry and free
1994f263522aSJoseph Koshy 		 * up space.
1995f263522aSJoseph Koshy 		 */
1996f263522aSJoseph Koshy 
1997f263522aSJoseph Koshy 		if (pp->pp_refcnt == 0) {
1998f263522aSJoseph Koshy 			pmc_remove_process_descriptor(pp);
19991ede983cSDag-Erling Smørgrav 			free(pp, M_PMC);
2000f263522aSJoseph Koshy 			break;
2001f263522aSJoseph Koshy 		}
2002ebccf1e3SJoseph Koshy 
2003ebccf1e3SJoseph Koshy 	}
2004ebccf1e3SJoseph Koshy 	break;
2005ebccf1e3SJoseph Koshy 
2006f263522aSJoseph Koshy 	case PMC_FN_CSW_IN:
2007f263522aSJoseph Koshy 		pmc_process_csw_in(td);
2008f263522aSJoseph Koshy 		break;
2009f263522aSJoseph Koshy 
2010f263522aSJoseph Koshy 	case PMC_FN_CSW_OUT:
2011f263522aSJoseph Koshy 		pmc_process_csw_out(td);
2012f263522aSJoseph Koshy 		break;
2013f263522aSJoseph Koshy 
2014f263522aSJoseph Koshy 	/*
2015f263522aSJoseph Koshy 	 * Process accumulated PC samples.
2016f263522aSJoseph Koshy 	 *
2017f263522aSJoseph Koshy 	 * This function is expected to be called by hardclock() for
2018f263522aSJoseph Koshy 	 * each CPU that has accumulated PC samples.
2019f263522aSJoseph Koshy 	 *
2020f263522aSJoseph Koshy 	 * This function is to be executed on the CPU whose samples
2021f263522aSJoseph Koshy 	 * are being processed.
2022f263522aSJoseph Koshy 	 */
2023f263522aSJoseph Koshy 	case PMC_FN_DO_SAMPLES:
2024f263522aSJoseph Koshy 
2025f263522aSJoseph Koshy 		/*
2026f263522aSJoseph Koshy 		 * Clear the cpu specific bit in the CPU mask before
2027f263522aSJoseph Koshy 		 * do the rest of the processing.  If the NMI handler
2028f263522aSJoseph Koshy 		 * gets invoked after the "atomic_clear_int()" call
2029f263522aSJoseph Koshy 		 * below but before "pmc_process_samples()" gets
2030f263522aSJoseph Koshy 		 * around to processing the interrupt, then we will
2031f263522aSJoseph Koshy 		 * come back here at the next hardclock() tick (and
2032f263522aSJoseph Koshy 		 * may find nothing to do if "pmc_process_samples()"
2033f263522aSJoseph Koshy 		 * had already processed the interrupt).  We don't
2034f263522aSJoseph Koshy 		 * lose the interrupt sample.
2035f263522aSJoseph Koshy 		 */
2036e6b475e0SMatt Macy 		DPCPU_SET(pmc_sampled, 0);
2037f5f9340bSFabien Thomas 		pmc_process_samples(PCPU_GET(cpuid), PMC_HR);
2038f5f9340bSFabien Thomas 		pmc_process_samples(PCPU_GET(cpuid), PMC_SR);
2039f263522aSJoseph Koshy 		break;
2040f263522aSJoseph Koshy 
204149874f6eSJoseph Koshy 	case PMC_FN_MMAP:
2042*0f00315cSMatt Macy 		MPASS(in_epoch() || sx_xlocked(&pmc_sx));
204349874f6eSJoseph Koshy 		pmc_process_mmap(td, (struct pmckern_map_in *) arg);
204449874f6eSJoseph Koshy 		break;
204549874f6eSJoseph Koshy 
204649874f6eSJoseph Koshy 	case PMC_FN_MUNMAP:
2047*0f00315cSMatt Macy 		MPASS(in_epoch() || sx_xlocked(&pmc_sx));
204849874f6eSJoseph Koshy 		pmc_process_munmap(td, (struct pmckern_map_out *) arg);
204949874f6eSJoseph Koshy 		break;
205049874f6eSJoseph Koshy 
2051d07f36b0SJoseph Koshy 	case PMC_FN_USER_CALLCHAIN:
2052d07f36b0SJoseph Koshy 		/*
2053d07f36b0SJoseph Koshy 		 * Record a call chain.
2054d07f36b0SJoseph Koshy 		 */
20556fe00c78SJoseph Koshy 		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
20566fe00c78SJoseph Koshy 		    __LINE__));
2057f5f9340bSFabien Thomas 
2058f5f9340bSFabien Thomas 		pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
2059d07f36b0SJoseph Koshy 		    (struct trapframe *) arg);
20606fe00c78SJoseph Koshy 		td->td_pflags &= ~TDP_CALLCHAIN;
2061d07f36b0SJoseph Koshy 		break;
2062d07f36b0SJoseph Koshy 
2063f5f9340bSFabien Thomas 	case PMC_FN_USER_CALLCHAIN_SOFT:
2064f5f9340bSFabien Thomas 		/*
2065f5f9340bSFabien Thomas 		 * Record a call chain.
2066f5f9340bSFabien Thomas 		 */
2067f5f9340bSFabien Thomas 		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2068f5f9340bSFabien Thomas 		    __LINE__));
2069f5f9340bSFabien Thomas 		pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR,
2070f5f9340bSFabien Thomas 		    (struct trapframe *) arg);
2071f5f9340bSFabien Thomas 		td->td_pflags &= ~TDP_CALLCHAIN;
2072f5f9340bSFabien Thomas 		break;
2073f5f9340bSFabien Thomas 
2074f5f9340bSFabien Thomas 	case PMC_FN_SOFT_SAMPLING:
2075f5f9340bSFabien Thomas 		/*
2076f5f9340bSFabien Thomas 		 * Call soft PMC sampling intr.
2077f5f9340bSFabien Thomas 		 */
2078f5f9340bSFabien Thomas 		pmc_soft_intr((struct pmckern_soft *) arg);
2079f5f9340bSFabien Thomas 		break;
2080f5f9340bSFabien Thomas 
2081ebccf1e3SJoseph Koshy 	default:
2082680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
2083ebccf1e3SJoseph Koshy 		KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
2084ebccf1e3SJoseph Koshy #endif
2085ebccf1e3SJoseph Koshy 		break;
2086ebccf1e3SJoseph Koshy 
2087ebccf1e3SJoseph Koshy 	}
2088ebccf1e3SJoseph Koshy 
2089ebccf1e3SJoseph Koshy 	return 0;
2090ebccf1e3SJoseph Koshy }
2091ebccf1e3SJoseph Koshy 
2092ebccf1e3SJoseph Koshy /*
2093ebccf1e3SJoseph Koshy  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
2094ebccf1e3SJoseph Koshy  */
2095ebccf1e3SJoseph Koshy 
2096ebccf1e3SJoseph Koshy static struct pmc_owner *
2097ebccf1e3SJoseph Koshy pmc_allocate_owner_descriptor(struct proc *p)
2098ebccf1e3SJoseph Koshy {
2099ebccf1e3SJoseph Koshy 	uint32_t hindex;
2100ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
2101ebccf1e3SJoseph Koshy 	struct pmc_ownerhash *poh;
2102ebccf1e3SJoseph Koshy 
2103ebccf1e3SJoseph Koshy 	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2104ebccf1e3SJoseph Koshy 	poh = &pmc_ownerhash[hindex];
2105ebccf1e3SJoseph Koshy 
2106ebccf1e3SJoseph Koshy 	/* allocate space for N pointers and one descriptor struct */
2107e829eb6dSJoseph Koshy 	po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
2108ebccf1e3SJoseph Koshy 	po->po_owner = p;
2109ebccf1e3SJoseph Koshy 	LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
2110ebccf1e3SJoseph Koshy 
2111f263522aSJoseph Koshy 	TAILQ_INIT(&po->po_logbuffers);
2112382d30cdSJoseph Koshy 	mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
2113f263522aSJoseph Koshy 
21144a3690dfSJohn Baldwin 	PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
2115ebccf1e3SJoseph Koshy 	    p, p->p_pid, p->p_comm, po);
2116ebccf1e3SJoseph Koshy 
2117ebccf1e3SJoseph Koshy 	return po;
2118ebccf1e3SJoseph Koshy }
2119ebccf1e3SJoseph Koshy 
2120f263522aSJoseph Koshy static void
2121f263522aSJoseph Koshy pmc_destroy_owner_descriptor(struct pmc_owner *po)
2122f263522aSJoseph Koshy {
2123f263522aSJoseph Koshy 
21244a3690dfSJohn Baldwin 	PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
2125f263522aSJoseph Koshy 	    po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
2126f263522aSJoseph Koshy 
2127f263522aSJoseph Koshy 	mtx_destroy(&po->po_mtx);
21281ede983cSDag-Erling Smørgrav 	free(po, M_PMC);
2129f263522aSJoseph Koshy }
2130f263522aSJoseph Koshy 
2131ebccf1e3SJoseph Koshy /*
2132ebccf1e3SJoseph Koshy  * find the descriptor corresponding to process 'p', adding or removing it
2133ebccf1e3SJoseph Koshy  * as specified by 'mode'.
2134ebccf1e3SJoseph Koshy  */
2135ebccf1e3SJoseph Koshy 
2136ebccf1e3SJoseph Koshy static struct pmc_process *
2137ebccf1e3SJoseph Koshy pmc_find_process_descriptor(struct proc *p, uint32_t mode)
2138ebccf1e3SJoseph Koshy {
2139ebccf1e3SJoseph Koshy 	uint32_t hindex;
2140ebccf1e3SJoseph Koshy 	struct pmc_process *pp, *ppnew;
2141ebccf1e3SJoseph Koshy 	struct pmc_processhash *pph;
2142ebccf1e3SJoseph Koshy 
2143ebccf1e3SJoseph Koshy 	hindex = PMC_HASH_PTR(p, pmc_processhashmask);
2144ebccf1e3SJoseph Koshy 	pph = &pmc_processhash[hindex];
2145ebccf1e3SJoseph Koshy 
2146ebccf1e3SJoseph Koshy 	ppnew = NULL;
2147ebccf1e3SJoseph Koshy 
2148ebccf1e3SJoseph Koshy 	/*
2149ebccf1e3SJoseph Koshy 	 * Pre-allocate memory in the FIND_ALLOCATE case since we
2150ebccf1e3SJoseph Koshy 	 * cannot call malloc(9) once we hold a spin lock.
2151ebccf1e3SJoseph Koshy 	 */
2152e829eb6dSJoseph Koshy 	if (mode & PMC_FLAG_ALLOCATE)
21531ede983cSDag-Erling Smørgrav 		ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
2154e829eb6dSJoseph Koshy 		    sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
2155ebccf1e3SJoseph Koshy 
2156ebccf1e3SJoseph Koshy 	mtx_lock_spin(&pmc_processhash_mtx);
2157ebccf1e3SJoseph Koshy 	LIST_FOREACH(pp, pph, pp_next)
2158ebccf1e3SJoseph Koshy 	    if (pp->pp_proc == p)
2159ebccf1e3SJoseph Koshy 		    break;
2160ebccf1e3SJoseph Koshy 
2161ebccf1e3SJoseph Koshy 	if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
2162ebccf1e3SJoseph Koshy 		LIST_REMOVE(pp, pp_next);
2163ebccf1e3SJoseph Koshy 
2164ebccf1e3SJoseph Koshy 	if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
2165ebccf1e3SJoseph Koshy 	    ppnew != NULL) {
2166ebccf1e3SJoseph Koshy 		ppnew->pp_proc = p;
2167ebccf1e3SJoseph Koshy 		LIST_INSERT_HEAD(pph, ppnew, pp_next);
2168ebccf1e3SJoseph Koshy 		pp = ppnew;
2169ebccf1e3SJoseph Koshy 		ppnew = NULL;
2170ebccf1e3SJoseph Koshy 	}
2171ebccf1e3SJoseph Koshy 	mtx_unlock_spin(&pmc_processhash_mtx);
2172ebccf1e3SJoseph Koshy 
2173ebccf1e3SJoseph Koshy 	if (pp != NULL && ppnew != NULL)
21741ede983cSDag-Erling Smørgrav 		free(ppnew, M_PMC);
2175ebccf1e3SJoseph Koshy 
2176ebccf1e3SJoseph Koshy 	return pp;
2177ebccf1e3SJoseph Koshy }
2178ebccf1e3SJoseph Koshy 
2179ebccf1e3SJoseph Koshy /*
2180ebccf1e3SJoseph Koshy  * remove a process descriptor from the process hash table.
2181ebccf1e3SJoseph Koshy  */
2182ebccf1e3SJoseph Koshy 
2183ebccf1e3SJoseph Koshy static void
2184ebccf1e3SJoseph Koshy pmc_remove_process_descriptor(struct pmc_process *pp)
2185ebccf1e3SJoseph Koshy {
2186ebccf1e3SJoseph Koshy 	KASSERT(pp->pp_refcnt == 0,
2187ebccf1e3SJoseph Koshy 	    ("[pmc,%d] Removing process descriptor %p with count %d",
2188ebccf1e3SJoseph Koshy 		__LINE__, pp, pp->pp_refcnt));
2189ebccf1e3SJoseph Koshy 
2190ebccf1e3SJoseph Koshy 	mtx_lock_spin(&pmc_processhash_mtx);
2191ebccf1e3SJoseph Koshy 	LIST_REMOVE(pp, pp_next);
2192ebccf1e3SJoseph Koshy 	mtx_unlock_spin(&pmc_processhash_mtx);
2193ebccf1e3SJoseph Koshy }
2194ebccf1e3SJoseph Koshy 
2195ebccf1e3SJoseph Koshy 
2196ebccf1e3SJoseph Koshy /*
2197ebccf1e3SJoseph Koshy  * find an owner descriptor corresponding to proc 'p'
2198ebccf1e3SJoseph Koshy  */
2199ebccf1e3SJoseph Koshy 
2200ebccf1e3SJoseph Koshy static struct pmc_owner *
2201ebccf1e3SJoseph Koshy pmc_find_owner_descriptor(struct proc *p)
2202ebccf1e3SJoseph Koshy {
2203ebccf1e3SJoseph Koshy 	uint32_t hindex;
2204ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
2205ebccf1e3SJoseph Koshy 	struct pmc_ownerhash *poh;
2206ebccf1e3SJoseph Koshy 
2207ebccf1e3SJoseph Koshy 	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2208ebccf1e3SJoseph Koshy 	poh = &pmc_ownerhash[hindex];
2209ebccf1e3SJoseph Koshy 
2210ebccf1e3SJoseph Koshy 	po = NULL;
2211ebccf1e3SJoseph Koshy 	LIST_FOREACH(po, poh, po_next)
2212ebccf1e3SJoseph Koshy 	    if (po->po_owner == p)
2213ebccf1e3SJoseph Koshy 		    break;
2214ebccf1e3SJoseph Koshy 
22154a3690dfSJohn Baldwin 	PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
2216ebccf1e3SJoseph Koshy 	    "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
2217ebccf1e3SJoseph Koshy 
2218ebccf1e3SJoseph Koshy 	return po;
2219ebccf1e3SJoseph Koshy }
2220ebccf1e3SJoseph Koshy 
2221ebccf1e3SJoseph Koshy /*
2222ebccf1e3SJoseph Koshy  * pmc_allocate_pmc_descriptor
2223ebccf1e3SJoseph Koshy  *
2224ebccf1e3SJoseph Koshy  * Allocate a pmc descriptor and initialize its
2225ebccf1e3SJoseph Koshy  * fields.
2226ebccf1e3SJoseph Koshy  */
2227ebccf1e3SJoseph Koshy 
2228ebccf1e3SJoseph Koshy static struct pmc *
2229ebccf1e3SJoseph Koshy pmc_allocate_pmc_descriptor(void)
2230ebccf1e3SJoseph Koshy {
2231ebccf1e3SJoseph Koshy 	struct pmc *pmc;
2232ebccf1e3SJoseph Koshy 
2233e829eb6dSJoseph Koshy 	pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
2234e6b475e0SMatt Macy 	pmc->pm_runcount = counter_u64_alloc(M_WAITOK);
2235e6b475e0SMatt Macy 	pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state)*mp_ncpus, M_PMC, M_WAITOK|M_ZERO);
22364a3690dfSJohn Baldwin 	PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2237ebccf1e3SJoseph Koshy 
2238ebccf1e3SJoseph Koshy 	return pmc;
2239ebccf1e3SJoseph Koshy }
2240ebccf1e3SJoseph Koshy 
2241ebccf1e3SJoseph Koshy /*
2242ebccf1e3SJoseph Koshy  * Destroy a pmc descriptor.
2243ebccf1e3SJoseph Koshy  */
2244ebccf1e3SJoseph Koshy 
2245ebccf1e3SJoseph Koshy static void
2246ebccf1e3SJoseph Koshy pmc_destroy_pmc_descriptor(struct pmc *pm)
2247ebccf1e3SJoseph Koshy {
2248ebccf1e3SJoseph Koshy 
2249ebccf1e3SJoseph Koshy 	KASSERT(pm->pm_state == PMC_STATE_DELETED ||
2250ebccf1e3SJoseph Koshy 	    pm->pm_state == PMC_STATE_FREE,
2251ebccf1e3SJoseph Koshy 	    ("[pmc,%d] destroying non-deleted PMC", __LINE__));
2252ebccf1e3SJoseph Koshy 	KASSERT(LIST_EMPTY(&pm->pm_targets),
2253ebccf1e3SJoseph Koshy 	    ("[pmc,%d] destroying pmc with targets", __LINE__));
2254ebccf1e3SJoseph Koshy 	KASSERT(pm->pm_owner == NULL,
2255ebccf1e3SJoseph Koshy 	    ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
2256e6b475e0SMatt Macy 	KASSERT(counter_u64_fetch(pm->pm_runcount) == 0,
2257e6b475e0SMatt Macy 	    ("[pmc,%d] pmc has non-zero run count %ld", __LINE__,
2258e6b475e0SMatt Macy 		 (unsigned long)counter_u64_fetch(pm->pm_runcount)));
225906796b67SMark Johnston 
2260e6b475e0SMatt Macy 	counter_u64_free(pm->pm_runcount);
2261e6b475e0SMatt Macy 	free(pm->pm_pcpu_state, M_PMC);
226206796b67SMark Johnston 	free(pm, M_PMC);
2263ebccf1e3SJoseph Koshy }
2264ebccf1e3SJoseph Koshy 
2265f263522aSJoseph Koshy static void
2266f263522aSJoseph Koshy pmc_wait_for_pmc_idle(struct pmc *pm)
2267f263522aSJoseph Koshy {
2268680f1afdSJohn Baldwin #ifdef HWPMC_DEBUG
2269f263522aSJoseph Koshy 	volatile int maxloop;
2270f263522aSJoseph Koshy 
2271122ccdc1SJoseph Koshy 	maxloop = 100 * pmc_cpu_max();
2272f263522aSJoseph Koshy #endif
2273f263522aSJoseph Koshy 	/*
2274f263522aSJoseph Koshy 	 * Loop (with a forced context switch) till the PMC's runcount
2275f263522aSJoseph Koshy 	 * comes down to zero.
2276f263522aSJoseph Koshy 	 */
2277e6b475e0SMatt Macy 	while (counter_u64_fetch(pm->pm_runcount) > 0) {
2278680f1afdSJohn Baldwin #ifdef HWPMC_DEBUG
2279f263522aSJoseph Koshy 		maxloop--;
2280f263522aSJoseph Koshy 		KASSERT(maxloop > 0,
2281e6b475e0SMatt Macy 		    ("[pmc,%d] (ri%d, rc%ld) waiting too long for "
2282f263522aSJoseph Koshy 			"pmc to be free", __LINE__,
2283e6b475e0SMatt Macy 			 PMC_TO_ROWINDEX(pm), (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2284f263522aSJoseph Koshy #endif
2285f263522aSJoseph Koshy 		pmc_force_context_switch();
2286f263522aSJoseph Koshy 	}
2287f263522aSJoseph Koshy }
2288f263522aSJoseph Koshy 
2289ebccf1e3SJoseph Koshy /*
2290ebccf1e3SJoseph Koshy  * This function does the following things:
2291ebccf1e3SJoseph Koshy  *
2292ebccf1e3SJoseph Koshy  *  - detaches the PMC from hardware
2293ebccf1e3SJoseph Koshy  *  - unlinks all target threads that were attached to it
2294ebccf1e3SJoseph Koshy  *  - removes the PMC from its owner's list
229506796b67SMark Johnston  *  - destroys the PMC private mutex
2296ebccf1e3SJoseph Koshy  *
229706796b67SMark Johnston  * Once this function completes, the given pmc pointer can be freed by
229806796b67SMark Johnston  * calling pmc_destroy_pmc_descriptor().
2299ebccf1e3SJoseph Koshy  */
2300ebccf1e3SJoseph Koshy 
2301ebccf1e3SJoseph Koshy static void
2302ebccf1e3SJoseph Koshy pmc_release_pmc_descriptor(struct pmc *pm)
2303ebccf1e3SJoseph Koshy {
2304c5153e19SJoseph Koshy 	enum pmc_mode mode;
2305ebccf1e3SJoseph Koshy 	struct pmc_hw *phw;
2306e829eb6dSJoseph Koshy 	u_int adjri, ri, cpu;
2307f263522aSJoseph Koshy 	struct pmc_owner *po;
2308ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
2309e829eb6dSJoseph Koshy 	struct pmc_process *pp;
2310e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
2311e829eb6dSJoseph Koshy 	struct pmc_target *ptgt, *tmp;
2312ebccf1e3SJoseph Koshy 
2313ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
2314ebccf1e3SJoseph Koshy 
2315ebccf1e3SJoseph Koshy 	KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2316ebccf1e3SJoseph Koshy 
2317c5153e19SJoseph Koshy 	ri   = PMC_TO_ROWINDEX(pm);
2318e829eb6dSJoseph Koshy 	pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2319c5153e19SJoseph Koshy 	mode = PMC_TO_MODE(pm);
2320ebccf1e3SJoseph Koshy 
23214a3690dfSJohn Baldwin 	PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2322c5153e19SJoseph Koshy 	    mode);
2323ebccf1e3SJoseph Koshy 
2324ebccf1e3SJoseph Koshy 	/*
2325ebccf1e3SJoseph Koshy 	 * First, we take the PMC off hardware.
2326ebccf1e3SJoseph Koshy 	 */
2327869f4a8bSWarner Losh 	cpu = 0;
2328c5153e19SJoseph Koshy 	if (PMC_IS_SYSTEM_MODE(mode)) {
2329ebccf1e3SJoseph Koshy 
2330ebccf1e3SJoseph Koshy 		/*
2331ebccf1e3SJoseph Koshy 		 * A system mode PMC runs on a specific CPU.  Switch
2332ebccf1e3SJoseph Koshy 		 * to this CPU and turn hardware off.
2333ebccf1e3SJoseph Koshy 		 */
2334ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
2335ebccf1e3SJoseph Koshy 
2336c5153e19SJoseph Koshy 		cpu = PMC_TO_CPU(pm);
2337ebccf1e3SJoseph Koshy 
2338ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
2339ebccf1e3SJoseph Koshy 
2340f263522aSJoseph Koshy 		/* switch off non-stalled CPUs */
2341e6b475e0SMatt Macy 		pm->pm_pcpu_state[cpu].pps_cpustate = 0;
2342f263522aSJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING &&
2343e6b475e0SMatt Macy 			pm->pm_pcpu_state[cpu].pps_stalled == 0) {
2344f263522aSJoseph Koshy 
2345ebccf1e3SJoseph Koshy 			phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2346ebccf1e3SJoseph Koshy 
2347ebccf1e3SJoseph Koshy 			KASSERT(phw->phw_pmc == pm,
2348ebccf1e3SJoseph Koshy 			    ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2349ebccf1e3SJoseph Koshy 				__LINE__, ri, phw->phw_pmc, pm));
23504a3690dfSJohn Baldwin 			PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2351ebccf1e3SJoseph Koshy 
2352ebccf1e3SJoseph Koshy 			critical_enter();
2353e829eb6dSJoseph Koshy 			pcd->pcd_stop_pmc(cpu, adjri);
2354ebccf1e3SJoseph Koshy 			critical_exit();
2355ebccf1e3SJoseph Koshy 		}
2356ebccf1e3SJoseph Koshy 
23574a3690dfSJohn Baldwin 		PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2358ebccf1e3SJoseph Koshy 
2359ebccf1e3SJoseph Koshy 		critical_enter();
2360e829eb6dSJoseph Koshy 		pcd->pcd_config_pmc(cpu, adjri, NULL);
2361ebccf1e3SJoseph Koshy 		critical_exit();
2362ebccf1e3SJoseph Koshy 
2363f263522aSJoseph Koshy 		/* adjust the global and process count of SS mode PMCs */
2364f263522aSJoseph Koshy 		if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2365f263522aSJoseph Koshy 			po = pm->pm_owner;
2366f263522aSJoseph Koshy 			po->po_sscount--;
2367f263522aSJoseph Koshy 			if (po->po_sscount == 0) {
2368f263522aSJoseph Koshy 				atomic_subtract_rel_int(&pmc_ss_count, 1);
2369*0f00315cSMatt Macy 				CK_LIST_REMOVE(po, po_ssnext);
2370*0f00315cSMatt Macy 				epoch_wait(global_epoch);
2371f263522aSJoseph Koshy 			}
2372f263522aSJoseph Koshy 		}
2373f263522aSJoseph Koshy 
2374ebccf1e3SJoseph Koshy 		pm->pm_state = PMC_STATE_DELETED;
2375ebccf1e3SJoseph Koshy 
2376ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
2377ebccf1e3SJoseph Koshy 
2378f263522aSJoseph Koshy 		/*
2379f263522aSJoseph Koshy 		 * We could have references to this PMC structure in
2380f263522aSJoseph Koshy 		 * the per-cpu sample queues.  Wait for the queue to
2381f263522aSJoseph Koshy 		 * drain.
2382f263522aSJoseph Koshy 		 */
2383f263522aSJoseph Koshy 		pmc_wait_for_pmc_idle(pm);
2384f263522aSJoseph Koshy 
2385c5153e19SJoseph Koshy 	} else if (PMC_IS_VIRTUAL_MODE(mode)) {
2386ebccf1e3SJoseph Koshy 
2387ebccf1e3SJoseph Koshy 		/*
2388ebccf1e3SJoseph Koshy 		 * A virtual PMC could be running on multiple CPUs at
2389ebccf1e3SJoseph Koshy 		 * a given instant.
2390ebccf1e3SJoseph Koshy 		 *
2391ebccf1e3SJoseph Koshy 		 * By marking its state as DELETED, we ensure that
2392ebccf1e3SJoseph Koshy 		 * this PMC is never further scheduled on hardware.
2393ebccf1e3SJoseph Koshy 		 *
2394ebccf1e3SJoseph Koshy 		 * Then we wait till all CPUs are done with this PMC.
2395ebccf1e3SJoseph Koshy 		 */
2396ebccf1e3SJoseph Koshy 		pm->pm_state = PMC_STATE_DELETED;
2397ebccf1e3SJoseph Koshy 
2398ebccf1e3SJoseph Koshy 
2399f263522aSJoseph Koshy 		/* Wait for the PMCs runcount to come to zero. */
2400f263522aSJoseph Koshy 		pmc_wait_for_pmc_idle(pm);
2401ebccf1e3SJoseph Koshy 
2402ebccf1e3SJoseph Koshy 		/*
2403ebccf1e3SJoseph Koshy 		 * At this point the PMC is off all CPUs and cannot be
2404ebccf1e3SJoseph Koshy 		 * freshly scheduled onto a CPU.  It is now safe to
2405ebccf1e3SJoseph Koshy 		 * unlink all targets from this PMC.  If a
2406ebccf1e3SJoseph Koshy 		 * process-record's refcount falls to zero, we remove
2407ebccf1e3SJoseph Koshy 		 * it from the hash table.  The module-wide SX lock
2408ebccf1e3SJoseph Koshy 		 * protects us from races.
2409ebccf1e3SJoseph Koshy 		 */
2410ebccf1e3SJoseph Koshy 		LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
2411ebccf1e3SJoseph Koshy 			pp = ptgt->pt_process;
2412ebccf1e3SJoseph Koshy 			pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
2413ebccf1e3SJoseph Koshy 
24144a3690dfSJohn Baldwin 			PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
2415ebccf1e3SJoseph Koshy 
2416ebccf1e3SJoseph Koshy 			/*
2417ebccf1e3SJoseph Koshy 			 * If the target process record shows that no
2418ebccf1e3SJoseph Koshy 			 * PMCs are attached to it, reclaim its space.
2419ebccf1e3SJoseph Koshy 			 */
2420ebccf1e3SJoseph Koshy 
2421ebccf1e3SJoseph Koshy 			if (pp->pp_refcnt == 0) {
2422ebccf1e3SJoseph Koshy 				pmc_remove_process_descriptor(pp);
24231ede983cSDag-Erling Smørgrav 				free(pp, M_PMC);
2424ebccf1e3SJoseph Koshy 			}
2425ebccf1e3SJoseph Koshy 		}
2426ebccf1e3SJoseph Koshy 
2427ebccf1e3SJoseph Koshy 		cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2428ebccf1e3SJoseph Koshy 
2429ebccf1e3SJoseph Koshy 	}
2430ebccf1e3SJoseph Koshy 
2431ebccf1e3SJoseph Koshy 	/*
2432ebccf1e3SJoseph Koshy 	 * Release any MD resources
2433ebccf1e3SJoseph Koshy 	 */
2434e829eb6dSJoseph Koshy 	(void) pcd->pcd_release_pmc(cpu, adjri, pm);
2435ebccf1e3SJoseph Koshy 
2436ebccf1e3SJoseph Koshy 	/*
2437ebccf1e3SJoseph Koshy 	 * Update row disposition
2438ebccf1e3SJoseph Koshy 	 */
2439ebccf1e3SJoseph Koshy 
2440c5153e19SJoseph Koshy 	if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2441ebccf1e3SJoseph Koshy 		PMC_UNMARK_ROW_STANDALONE(ri);
2442ebccf1e3SJoseph Koshy 	else
2443ebccf1e3SJoseph Koshy 		PMC_UNMARK_ROW_THREAD(ri);
2444ebccf1e3SJoseph Koshy 
2445ebccf1e3SJoseph Koshy 	/* unlink from the owner's list */
2446f263522aSJoseph Koshy 	if (pm->pm_owner) {
2447f263522aSJoseph Koshy 		LIST_REMOVE(pm, pm_next);
2448f263522aSJoseph Koshy 		pm->pm_owner = NULL;
2449f263522aSJoseph Koshy 	}
2450ebccf1e3SJoseph Koshy }
2451ebccf1e3SJoseph Koshy 
2452ebccf1e3SJoseph Koshy /*
2453ebccf1e3SJoseph Koshy  * Register an owner and a pmc.
2454ebccf1e3SJoseph Koshy  */
2455ebccf1e3SJoseph Koshy 
2456ebccf1e3SJoseph Koshy static int
2457ebccf1e3SJoseph Koshy pmc_register_owner(struct proc *p, struct pmc *pmc)
2458ebccf1e3SJoseph Koshy {
2459ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
2460ebccf1e3SJoseph Koshy 
2461ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
2462ebccf1e3SJoseph Koshy 
2463c5153e19SJoseph Koshy 	if ((po = pmc_find_owner_descriptor(p)) == NULL)
2464f263522aSJoseph Koshy 		if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
2465ebccf1e3SJoseph Koshy 			return ENOMEM;
2466ebccf1e3SJoseph Koshy 
2467ebccf1e3SJoseph Koshy 	KASSERT(pmc->pm_owner == NULL,
2468ebccf1e3SJoseph Koshy 	    ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2469ebccf1e3SJoseph Koshy 	pmc->pm_owner  = po;
2470ebccf1e3SJoseph Koshy 
2471f263522aSJoseph Koshy 	LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
2472ebccf1e3SJoseph Koshy 
2473ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
2474ebccf1e3SJoseph Koshy 	p->p_flag |= P_HWPMC;
2475ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
2476ebccf1e3SJoseph Koshy 
2477f263522aSJoseph Koshy 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2478f263522aSJoseph Koshy 		pmclog_process_pmcallocate(pmc);
2479f263522aSJoseph Koshy 
24804a3690dfSJohn Baldwin 	PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2481f263522aSJoseph Koshy 	    po, pmc);
2482ebccf1e3SJoseph Koshy 
2483ebccf1e3SJoseph Koshy 	return 0;
2484ebccf1e3SJoseph Koshy }
2485ebccf1e3SJoseph Koshy 
2486ebccf1e3SJoseph Koshy /*
2487ebccf1e3SJoseph Koshy  * Return the current row disposition:
2488ebccf1e3SJoseph Koshy  * == 0 => FREE
2489ebccf1e3SJoseph Koshy  *  > 0 => PROCESS MODE
2490ebccf1e3SJoseph Koshy  *  < 0 => SYSTEM MODE
2491ebccf1e3SJoseph Koshy  */
2492ebccf1e3SJoseph Koshy 
2493ebccf1e3SJoseph Koshy int
2494ebccf1e3SJoseph Koshy pmc_getrowdisp(int ri)
2495ebccf1e3SJoseph Koshy {
2496ebccf1e3SJoseph Koshy 	return pmc_pmcdisp[ri];
2497ebccf1e3SJoseph Koshy }
2498ebccf1e3SJoseph Koshy 
2499ebccf1e3SJoseph Koshy /*
2500ebccf1e3SJoseph Koshy  * Check if a PMC at row index 'ri' can be allocated to the current
2501ebccf1e3SJoseph Koshy  * process.
2502ebccf1e3SJoseph Koshy  *
2503ebccf1e3SJoseph Koshy  * Allocation can fail if:
2504ebccf1e3SJoseph Koshy  *   - the current process is already being profiled by a PMC at index 'ri',
2505ebccf1e3SJoseph Koshy  *     attached to it via OP_PMCATTACH.
2506ebccf1e3SJoseph Koshy  *   - the current process has already allocated a PMC at index 'ri'
2507ebccf1e3SJoseph Koshy  *     via OP_ALLOCATE.
2508ebccf1e3SJoseph Koshy  */
2509ebccf1e3SJoseph Koshy 
2510ebccf1e3SJoseph Koshy static int
2511c5153e19SJoseph Koshy pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2512ebccf1e3SJoseph Koshy {
2513c5153e19SJoseph Koshy 	enum pmc_mode mode;
2514c5153e19SJoseph Koshy 	struct pmc *pm;
2515ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
2516ebccf1e3SJoseph Koshy 	struct pmc_process *pp;
2517ebccf1e3SJoseph Koshy 
25184a3690dfSJohn Baldwin 	PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2519c5153e19SJoseph Koshy 	    "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2520ebccf1e3SJoseph Koshy 
2521c5153e19SJoseph Koshy 	/*
2522c5153e19SJoseph Koshy 	 * We shouldn't have already allocated a process-mode PMC at
2523c5153e19SJoseph Koshy 	 * row index 'ri'.
2524c5153e19SJoseph Koshy 	 *
2525c5153e19SJoseph Koshy 	 * We shouldn't have allocated a system-wide PMC on the same
2526c5153e19SJoseph Koshy 	 * CPU and same RI.
2527c5153e19SJoseph Koshy 	 */
2528ebccf1e3SJoseph Koshy 	if ((po = pmc_find_owner_descriptor(p)) != NULL)
2529f263522aSJoseph Koshy 		LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2530c5153e19SJoseph Koshy 		    if (PMC_TO_ROWINDEX(pm) == ri) {
2531c5153e19SJoseph Koshy 			    mode = PMC_TO_MODE(pm);
2532c5153e19SJoseph Koshy 			    if (PMC_IS_VIRTUAL_MODE(mode))
2533ebccf1e3SJoseph Koshy 				    return EEXIST;
2534c5153e19SJoseph Koshy 			    if (PMC_IS_SYSTEM_MODE(mode) &&
2535c5153e19SJoseph Koshy 				(int) PMC_TO_CPU(pm) == cpu)
2536c5153e19SJoseph Koshy 				    return EEXIST;
2537c5153e19SJoseph Koshy 		    }
2538c5153e19SJoseph Koshy 	        }
2539ebccf1e3SJoseph Koshy 
2540c5153e19SJoseph Koshy 	/*
2541c5153e19SJoseph Koshy 	 * We also shouldn't be the target of any PMC at this index
2542c5153e19SJoseph Koshy 	 * since otherwise a PMC_ATTACH to ourselves will fail.
2543c5153e19SJoseph Koshy 	 */
2544ebccf1e3SJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2545ebccf1e3SJoseph Koshy 		if (pp->pp_pmcs[ri].pp_pmc)
2546ebccf1e3SJoseph Koshy 			return EEXIST;
2547ebccf1e3SJoseph Koshy 
25484a3690dfSJohn Baldwin 	PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2549ebccf1e3SJoseph Koshy 	    p, p->p_pid, p->p_comm, ri);
2550ebccf1e3SJoseph Koshy 
2551ebccf1e3SJoseph Koshy 	return 0;
2552ebccf1e3SJoseph Koshy }
2553ebccf1e3SJoseph Koshy 
2554ebccf1e3SJoseph Koshy /*
2555ebccf1e3SJoseph Koshy  * Check if a given PMC at row index 'ri' can be currently used in
2556ebccf1e3SJoseph Koshy  * mode 'mode'.
2557ebccf1e3SJoseph Koshy  */
2558ebccf1e3SJoseph Koshy 
2559ebccf1e3SJoseph Koshy static int
2560ebccf1e3SJoseph Koshy pmc_can_allocate_row(int ri, enum pmc_mode mode)
2561ebccf1e3SJoseph Koshy {
2562ebccf1e3SJoseph Koshy 	enum pmc_disp	disp;
2563ebccf1e3SJoseph Koshy 
2564ebccf1e3SJoseph Koshy 	sx_assert(&pmc_sx, SX_XLOCKED);
2565ebccf1e3SJoseph Koshy 
25664a3690dfSJohn Baldwin 	PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
2567ebccf1e3SJoseph Koshy 
2568ebccf1e3SJoseph Koshy 	if (PMC_IS_SYSTEM_MODE(mode))
2569ebccf1e3SJoseph Koshy 		disp = PMC_DISP_STANDALONE;
2570ebccf1e3SJoseph Koshy 	else
2571ebccf1e3SJoseph Koshy 		disp = PMC_DISP_THREAD;
2572ebccf1e3SJoseph Koshy 
2573ebccf1e3SJoseph Koshy 	/*
2574ebccf1e3SJoseph Koshy 	 * check disposition for PMC row 'ri':
2575ebccf1e3SJoseph Koshy 	 *
2576ebccf1e3SJoseph Koshy 	 * Expected disposition		Row-disposition		Result
2577ebccf1e3SJoseph Koshy 	 *
2578ebccf1e3SJoseph Koshy 	 * STANDALONE			STANDALONE or FREE	proceed
2579ebccf1e3SJoseph Koshy 	 * STANDALONE			THREAD			fail
2580ebccf1e3SJoseph Koshy 	 * THREAD			THREAD or FREE		proceed
2581ebccf1e3SJoseph Koshy 	 * THREAD			STANDALONE		fail
2582ebccf1e3SJoseph Koshy 	 */
2583ebccf1e3SJoseph Koshy 
2584ebccf1e3SJoseph Koshy 	if (!PMC_ROW_DISP_IS_FREE(ri) &&
2585ebccf1e3SJoseph Koshy 	    !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
2586ebccf1e3SJoseph Koshy 	    !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
2587ebccf1e3SJoseph Koshy 		return EBUSY;
2588ebccf1e3SJoseph Koshy 
2589ebccf1e3SJoseph Koshy 	/*
2590ebccf1e3SJoseph Koshy 	 * All OK
2591ebccf1e3SJoseph Koshy 	 */
2592ebccf1e3SJoseph Koshy 
25934a3690dfSJohn Baldwin 	PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
2594ebccf1e3SJoseph Koshy 
2595ebccf1e3SJoseph Koshy 	return 0;
2596ebccf1e3SJoseph Koshy 
2597ebccf1e3SJoseph Koshy }
2598ebccf1e3SJoseph Koshy 
2599ebccf1e3SJoseph Koshy /*
2600c5153e19SJoseph Koshy  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
2601ebccf1e3SJoseph Koshy  */
2602ebccf1e3SJoseph Koshy 
2603ebccf1e3SJoseph Koshy static struct pmc *
2604ebccf1e3SJoseph Koshy pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
2605ebccf1e3SJoseph Koshy {
2606f263522aSJoseph Koshy 	struct pmc *pm;
2607ebccf1e3SJoseph Koshy 
2608c5153e19SJoseph Koshy 	KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
2609c5153e19SJoseph Koshy 	    ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
2610c5153e19SJoseph Koshy 		PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
2611ebccf1e3SJoseph Koshy 
2612f263522aSJoseph Koshy 	LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2613f263522aSJoseph Koshy 	    if (pm->pm_id == pmcid)
2614f263522aSJoseph Koshy 		    return pm;
2615ebccf1e3SJoseph Koshy 
2616ebccf1e3SJoseph Koshy 	return NULL;
2617ebccf1e3SJoseph Koshy }
2618ebccf1e3SJoseph Koshy 
2619ebccf1e3SJoseph Koshy static int
2620ebccf1e3SJoseph Koshy pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
2621ebccf1e3SJoseph Koshy {
2622ebccf1e3SJoseph Koshy 
2623e6f47577SBjoern A. Zeeb 	struct pmc *pm, *opm;
2624ebccf1e3SJoseph Koshy 	struct pmc_owner *po;
2625e6f47577SBjoern A. Zeeb 	struct pmc_process *pp;
2626ebccf1e3SJoseph Koshy 
26274a3690dfSJohn Baldwin 	PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid);
2628d2cd6388SKonstantin Belousov 	if (PMC_ID_TO_ROWINDEX(pmcid) >= md->pmd_npmc)
2629d2cd6388SKonstantin Belousov 		return (EINVAL);
2630ebccf1e3SJoseph Koshy 
2631e6f47577SBjoern A. Zeeb 	if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) {
2632e6f47577SBjoern A. Zeeb 		/*
2633e6f47577SBjoern A. Zeeb 		 * In case of PMC_F_DESCENDANTS child processes we will not find
2634e6f47577SBjoern A. Zeeb 		 * the current process in the owners hash list.  Find the owner
2635e6f47577SBjoern A. Zeeb 		 * process first and from there lookup the po.
2636e6f47577SBjoern A. Zeeb 		 */
2637e6f47577SBjoern A. Zeeb 		if ((pp = pmc_find_process_descriptor(curthread->td_proc,
2638e6f47577SBjoern A. Zeeb 		    PMC_FLAG_NONE)) == NULL) {
2639ebccf1e3SJoseph Koshy 			return ESRCH;
2640e6f47577SBjoern A. Zeeb 		} else {
2641e6f47577SBjoern A. Zeeb 			opm = pp->pp_pmcs[PMC_ID_TO_ROWINDEX(pmcid)].pp_pmc;
2642e6f47577SBjoern A. Zeeb 			if (opm == NULL)
2643e6f47577SBjoern A. Zeeb 				return ESRCH;
2644e6f47577SBjoern A. Zeeb 			if ((opm->pm_flags & (PMC_F_ATTACHED_TO_OWNER|
2645e6f47577SBjoern A. Zeeb 			    PMC_F_DESCENDANTS)) != (PMC_F_ATTACHED_TO_OWNER|
2646e6f47577SBjoern A. Zeeb 			    PMC_F_DESCENDANTS))
2647e6f47577SBjoern A. Zeeb 				return ESRCH;
2648e6f47577SBjoern A. Zeeb 			po = opm->pm_owner;
2649e6f47577SBjoern A. Zeeb 		}
2650e6f47577SBjoern A. Zeeb 	}
2651ebccf1e3SJoseph Koshy 
2652ebccf1e3SJoseph Koshy 	if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
2653ebccf1e3SJoseph Koshy 		return EINVAL;
2654ebccf1e3SJoseph Koshy 
26554a3690dfSJohn Baldwin 	PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
2656ebccf1e3SJoseph Koshy 
2657ebccf1e3SJoseph Koshy 	*pmc = pm;
2658ebccf1e3SJoseph Koshy 	return 0;
2659ebccf1e3SJoseph Koshy }
2660ebccf1e3SJoseph Koshy 
2661ebccf1e3SJoseph Koshy /*
2662ebccf1e3SJoseph Koshy  * Start a PMC.
2663ebccf1e3SJoseph Koshy  */
2664ebccf1e3SJoseph Koshy 
2665ebccf1e3SJoseph Koshy static int
2666ebccf1e3SJoseph Koshy pmc_start(struct pmc *pm)
2667ebccf1e3SJoseph Koshy {
2668c5153e19SJoseph Koshy 	enum pmc_mode mode;
2669f263522aSJoseph Koshy 	struct pmc_owner *po;
2670ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
2671e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
2672e829eb6dSJoseph Koshy 	int adjri, error, cpu, ri;
2673ebccf1e3SJoseph Koshy 
2674ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL,
2675ebccf1e3SJoseph Koshy 	    ("[pmc,%d] null pm", __LINE__));
2676ebccf1e3SJoseph Koshy 
2677c5153e19SJoseph Koshy 	mode = PMC_TO_MODE(pm);
2678c5153e19SJoseph Koshy 	ri   = PMC_TO_ROWINDEX(pm);
2679e829eb6dSJoseph Koshy 	pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2680e829eb6dSJoseph Koshy 
2681c5153e19SJoseph Koshy 	error = 0;
2682c5153e19SJoseph Koshy 
26834a3690dfSJohn Baldwin 	PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2684ebccf1e3SJoseph Koshy 
2685f263522aSJoseph Koshy 	po = pm->pm_owner;
2686ebccf1e3SJoseph Koshy 
2687d07f36b0SJoseph Koshy 	/*
2688d07f36b0SJoseph Koshy 	 * Disallow PMCSTART if a logfile is required but has not been
2689d07f36b0SJoseph Koshy 	 * configured yet.
2690d07f36b0SJoseph Koshy 	 */
2691d07f36b0SJoseph Koshy 	if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2692d07f36b0SJoseph Koshy 	    (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2693e829eb6dSJoseph Koshy 		return (EDOOFUS);	/* programming error */
2694d07f36b0SJoseph Koshy 
2695d07f36b0SJoseph Koshy 	/*
2696d07f36b0SJoseph Koshy 	 * If this is a sampling mode PMC, log mapping information for
2697d07f36b0SJoseph Koshy 	 * the kernel modules that are currently loaded.
2698d07f36b0SJoseph Koshy 	 */
2699d07f36b0SJoseph Koshy 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2700d07f36b0SJoseph Koshy 	    pmc_log_kernel_mappings(pm);
2701d07f36b0SJoseph Koshy 
2702c5153e19SJoseph Koshy 	if (PMC_IS_VIRTUAL_MODE(mode)) {
2703ebccf1e3SJoseph Koshy 
2704ebccf1e3SJoseph Koshy 		/*
2705f263522aSJoseph Koshy 		 * If a PMCATTACH has never been done on this PMC,
2706f263522aSJoseph Koshy 		 * attach it to its owner process.
2707ebccf1e3SJoseph Koshy 		 */
2708ebccf1e3SJoseph Koshy 
2709ebccf1e3SJoseph Koshy 		if (LIST_EMPTY(&pm->pm_targets))
2710f263522aSJoseph Koshy 			error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
2711f263522aSJoseph Koshy 			    pmc_attach_process(po->po_owner, pm);
2712f263522aSJoseph Koshy 
2713f263522aSJoseph Koshy 		/*
2714c5153e19SJoseph Koshy 		 * If the PMC is attached to its owner, then force a context
2715c5153e19SJoseph Koshy 		 * switch to ensure that the MD state gets set correctly.
2716c5153e19SJoseph Koshy 		 */
2717ebccf1e3SJoseph Koshy 
2718f263522aSJoseph Koshy 		if (error == 0) {
2719f263522aSJoseph Koshy 			pm->pm_state = PMC_STATE_RUNNING;
2720f263522aSJoseph Koshy 			if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2721f263522aSJoseph Koshy 				pmc_force_context_switch();
2722f263522aSJoseph Koshy 		}
2723ebccf1e3SJoseph Koshy 
2724e829eb6dSJoseph Koshy 		return (error);
2725f263522aSJoseph Koshy 	}
2726ebccf1e3SJoseph Koshy 
2727f263522aSJoseph Koshy 
2728f263522aSJoseph Koshy 	/*
2729f263522aSJoseph Koshy 	 * A system-wide PMC.
2730d07f36b0SJoseph Koshy 	 *
2731f263522aSJoseph Koshy 	 * Add the owner to the global list if this is a system-wide
2732f263522aSJoseph Koshy 	 * sampling PMC.
2733f263522aSJoseph Koshy 	 */
2734f263522aSJoseph Koshy 
2735f263522aSJoseph Koshy 	if (mode == PMC_MODE_SS) {
2736f263522aSJoseph Koshy 		if (po->po_sscount == 0) {
2737*0f00315cSMatt Macy 			CK_LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
2738f263522aSJoseph Koshy 			atomic_add_rel_int(&pmc_ss_count, 1);
27394a3690dfSJohn Baldwin 			PMCDBG1(PMC,OPS,1, "po=%p in global list", po);
2740f263522aSJoseph Koshy 		}
2741f263522aSJoseph Koshy 		po->po_sscount++;
2742ebccf1e3SJoseph Koshy 
2743bf792d68SJoseph Koshy 		/*
2744bf792d68SJoseph Koshy 		 * Log mapping information for all existing processes in the
2745bf792d68SJoseph Koshy 		 * system.  Subsequent mappings are logged as they happen;
2746bf792d68SJoseph Koshy 		 * see pmc_process_mmap().
2747bf792d68SJoseph Koshy 		 */
2748bf792d68SJoseph Koshy 		if (po->po_logprocmaps == 0) {
2749d07f36b0SJoseph Koshy 			pmc_log_all_process_mappings(po);
2750bf792d68SJoseph Koshy 			po->po_logprocmaps = 1;
2751bf792d68SJoseph Koshy 		}
275204001891SRyan Stone 	}
275349874f6eSJoseph Koshy 
2754ebccf1e3SJoseph Koshy 	/*
2755f263522aSJoseph Koshy 	 * Move to the CPU associated with this
2756ebccf1e3SJoseph Koshy 	 * PMC, and start the hardware.
2757ebccf1e3SJoseph Koshy 	 */
2758ebccf1e3SJoseph Koshy 
2759ebccf1e3SJoseph Koshy 	pmc_save_cpu_binding(&pb);
2760ebccf1e3SJoseph Koshy 
2761c5153e19SJoseph Koshy 	cpu = PMC_TO_CPU(pm);
2762ebccf1e3SJoseph Koshy 
2763122ccdc1SJoseph Koshy 	if (!pmc_cpu_is_active(cpu))
2764e829eb6dSJoseph Koshy 		return (ENXIO);
2765ebccf1e3SJoseph Koshy 
2766ebccf1e3SJoseph Koshy 	pmc_select_cpu(cpu);
2767ebccf1e3SJoseph Koshy 
2768ebccf1e3SJoseph Koshy 	/*
2769ebccf1e3SJoseph Koshy 	 * global PMCs are configured at allocation time
2770ebccf1e3SJoseph Koshy 	 * so write out the initial value and start the PMC.
2771ebccf1e3SJoseph Koshy 	 */
2772ebccf1e3SJoseph Koshy 
2773f263522aSJoseph Koshy 	pm->pm_state = PMC_STATE_RUNNING;
2774f263522aSJoseph Koshy 
2775c5153e19SJoseph Koshy 	critical_enter();
2776e829eb6dSJoseph Koshy 	if ((error = pcd->pcd_write_pmc(cpu, adjri,
2777c5153e19SJoseph Koshy 		 PMC_IS_SAMPLING_MODE(mode) ?
2778ebccf1e3SJoseph Koshy 		 pm->pm_sc.pm_reloadcount :
2779a3924968SJonathan T. Looney 		 pm->pm_sc.pm_initial)) == 0) {
2780a3924968SJonathan T. Looney 		/* If a sampling mode PMC, reset stalled state. */
2781a3924968SJonathan T. Looney 		if (PMC_IS_SAMPLING_MODE(mode))
2782e6b475e0SMatt Macy 			pm->pm_pcpu_state[cpu].pps_stalled = 0;
2783a3924968SJonathan T. Looney 
2784a3924968SJonathan T. Looney 		/* Indicate that we desire this to run. Start it. */
2785e6b475e0SMatt Macy 		pm->pm_pcpu_state[cpu].pps_cpustate = 1;
2786e829eb6dSJoseph Koshy 		error = pcd->pcd_start_pmc(cpu, adjri);
2787a3924968SJonathan T. Looney 	}
2788c5153e19SJoseph Koshy 	critical_exit();
2789ebccf1e3SJoseph Koshy 
2790ebccf1e3SJoseph Koshy 	pmc_restore_cpu_binding(&pb);
2791ebccf1e3SJoseph Koshy 
2792e829eb6dSJoseph Koshy 	return (error);
2793ebccf1e3SJoseph Koshy }
2794ebccf1e3SJoseph Koshy 
2795ebccf1e3SJoseph Koshy /*
2796ebccf1e3SJoseph Koshy  * Stop a PMC.
2797ebccf1e3SJoseph Koshy  */
2798ebccf1e3SJoseph Koshy 
2799ebccf1e3SJoseph Koshy static int
2800ebccf1e3SJoseph Koshy pmc_stop(struct pmc *pm)
2801ebccf1e3SJoseph Koshy {
2802f263522aSJoseph Koshy 	struct pmc_owner *po;
2803ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
2804e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
2805e829eb6dSJoseph Koshy 	int adjri, cpu, error, ri;
2806ebccf1e3SJoseph Koshy 
2807ebccf1e3SJoseph Koshy 	KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2808ebccf1e3SJoseph Koshy 
28094a3690dfSJohn Baldwin 	PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2810c5153e19SJoseph Koshy 	    PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2811ebccf1e3SJoseph Koshy 
2812ebccf1e3SJoseph Koshy 	pm->pm_state = PMC_STATE_STOPPED;
2813ebccf1e3SJoseph Koshy 
2814ebccf1e3SJoseph Koshy 	/*
2815ebccf1e3SJoseph Koshy 	 * If the PMC is a virtual mode one, changing the state to
2816ebccf1e3SJoseph Koshy 	 * non-RUNNING is enough to ensure that the PMC never gets
2817ebccf1e3SJoseph Koshy 	 * scheduled.
2818ebccf1e3SJoseph Koshy 	 *
2819ebccf1e3SJoseph Koshy 	 * If this PMC is current running on a CPU, then it will
2820ebccf1e3SJoseph Koshy 	 * handled correctly at the time its target process is context
2821ebccf1e3SJoseph Koshy 	 * switched out.
2822ebccf1e3SJoseph Koshy 	 */
2823ebccf1e3SJoseph Koshy 
2824c5153e19SJoseph Koshy 	if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
2825ebccf1e3SJoseph Koshy 		return 0;
2826ebccf1e3SJoseph Koshy 
2827ebccf1e3SJoseph Koshy 	/*
2828ebccf1e3SJoseph Koshy 	 * A system-mode PMC.  Move to the CPU associated with
2829ebccf1e3SJoseph Koshy 	 * this PMC, and stop the hardware.  We update the
2830ebccf1e3SJoseph Koshy 	 * 'initial count' so that a subsequent PMCSTART will
2831ebccf1e3SJoseph Koshy 	 * resume counting from the current hardware count.
2832ebccf1e3SJoseph Koshy 	 */
2833ebccf1e3SJoseph Koshy 
2834ebccf1e3SJoseph Koshy 	pmc_save_cpu_binding(&pb);
2835ebccf1e3SJoseph Koshy 
2836c5153e19SJoseph Koshy 	cpu = PMC_TO_CPU(pm);
2837c5153e19SJoseph Koshy 
2838122ccdc1SJoseph Koshy 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2839c5153e19SJoseph Koshy 	    ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2840ebccf1e3SJoseph Koshy 
2841122ccdc1SJoseph Koshy 	if (!pmc_cpu_is_active(cpu))
2842ebccf1e3SJoseph Koshy 		return ENXIO;
2843ebccf1e3SJoseph Koshy 
2844ebccf1e3SJoseph Koshy 	pmc_select_cpu(cpu);
2845ebccf1e3SJoseph Koshy 
2846c5153e19SJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
2847e829eb6dSJoseph Koshy 	pcd = pmc_ri_to_classdep(md, ri, &adjri);
2848c5153e19SJoseph Koshy 
2849e6b475e0SMatt Macy 	pm->pm_pcpu_state[cpu].pps_cpustate = 0;
2850c5153e19SJoseph Koshy 	critical_enter();
2851e829eb6dSJoseph Koshy 	if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
2852e829eb6dSJoseph Koshy 		error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
2853c5153e19SJoseph Koshy 	critical_exit();
2854ebccf1e3SJoseph Koshy 
2855ebccf1e3SJoseph Koshy 	pmc_restore_cpu_binding(&pb);
2856ebccf1e3SJoseph Koshy 
2857f263522aSJoseph Koshy 	po = pm->pm_owner;
2858f263522aSJoseph Koshy 
2859f263522aSJoseph Koshy 	/* remove this owner from the global list of SS PMC owners */
2860f263522aSJoseph Koshy 	if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
2861f263522aSJoseph Koshy 		po->po_sscount--;
2862f263522aSJoseph Koshy 		if (po->po_sscount == 0) {
2863f263522aSJoseph Koshy 			atomic_subtract_rel_int(&pmc_ss_count, 1);
2864*0f00315cSMatt Macy 			CK_LIST_REMOVE(po, po_ssnext);
2865*0f00315cSMatt Macy 			epoch_wait(global_epoch);
28664a3690dfSJohn Baldwin 			PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po);
2867f263522aSJoseph Koshy 		}
2868f263522aSJoseph Koshy 	}
2869f263522aSJoseph Koshy 
2870e829eb6dSJoseph Koshy 	return (error);
2871ebccf1e3SJoseph Koshy }
2872ebccf1e3SJoseph Koshy 
2873ebccf1e3SJoseph Koshy 
2874680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
2875ebccf1e3SJoseph Koshy static const char *pmc_op_to_name[] = {
2876ebccf1e3SJoseph Koshy #undef	__PMC_OP
2877ebccf1e3SJoseph Koshy #define	__PMC_OP(N, D)	#N ,
2878ebccf1e3SJoseph Koshy 	__PMC_OPS()
2879ebccf1e3SJoseph Koshy 	NULL
2880ebccf1e3SJoseph Koshy };
2881ebccf1e3SJoseph Koshy #endif
2882ebccf1e3SJoseph Koshy 
2883ebccf1e3SJoseph Koshy /*
2884ebccf1e3SJoseph Koshy  * The syscall interface
2885ebccf1e3SJoseph Koshy  */
2886ebccf1e3SJoseph Koshy 
2887ebccf1e3SJoseph Koshy #define	PMC_GET_SX_XLOCK(...) do {		\
2888ebccf1e3SJoseph Koshy 	sx_xlock(&pmc_sx);			\
2889ebccf1e3SJoseph Koshy 	if (pmc_hook == NULL) {			\
2890ebccf1e3SJoseph Koshy 		sx_xunlock(&pmc_sx);		\
2891ebccf1e3SJoseph Koshy 		return __VA_ARGS__;		\
2892ebccf1e3SJoseph Koshy 	}					\
2893ebccf1e3SJoseph Koshy } while (0)
2894ebccf1e3SJoseph Koshy 
2895ebccf1e3SJoseph Koshy #define	PMC_DOWNGRADE_SX() do {			\
2896ebccf1e3SJoseph Koshy 	sx_downgrade(&pmc_sx);			\
2897ebccf1e3SJoseph Koshy 	is_sx_downgraded = 1;			\
2898ebccf1e3SJoseph Koshy } while (0)
2899ebccf1e3SJoseph Koshy 
2900ebccf1e3SJoseph Koshy static int
2901ebccf1e3SJoseph Koshy pmc_syscall_handler(struct thread *td, void *syscall_args)
2902ebccf1e3SJoseph Koshy {
290320b555e1SKonstantin Belousov 	int error, is_sx_downgraded, op;
2904ebccf1e3SJoseph Koshy 	struct pmc_syscall_args *c;
290520b555e1SKonstantin Belousov 	void *pmclog_proc_handle;
2906ebccf1e3SJoseph Koshy 	void *arg;
2907ebccf1e3SJoseph Koshy 
2908ebccf1e3SJoseph Koshy 	c = (struct pmc_syscall_args *)syscall_args;
2909ebccf1e3SJoseph Koshy 	op = c->pmop_code;
2910ebccf1e3SJoseph Koshy 	arg = c->pmop_data;
2911*0f00315cSMatt Macy 	/* PMC isn't set up yet */
2912*0f00315cSMatt Macy 	if (pmc_hook == NULL)
2913*0f00315cSMatt Macy 		return (EINVAL);
291420b555e1SKonstantin Belousov 	if (op == PMC_OP_CONFIGURELOG) {
291520b555e1SKonstantin Belousov 		/*
291620b555e1SKonstantin Belousov 		 * We cannot create the logging process inside
291720b555e1SKonstantin Belousov 		 * pmclog_configure_log() because there is a LOR
291820b555e1SKonstantin Belousov 		 * between pmc_sx and process structure locks.
291920b555e1SKonstantin Belousov 		 * Instead, pre-create the process and ignite the loop
292020b555e1SKonstantin Belousov 		 * if everything is fine, otherwise direct the process
292120b555e1SKonstantin Belousov 		 * to exit.
292220b555e1SKonstantin Belousov 		 */
292320b555e1SKonstantin Belousov 		error = pmclog_proc_create(td, &pmclog_proc_handle);
292420b555e1SKonstantin Belousov 		if (error != 0)
292520b555e1SKonstantin Belousov 			goto done_syscall;
292620b555e1SKonstantin Belousov 	}
292720b555e1SKonstantin Belousov 
292820b555e1SKonstantin Belousov 	PMC_GET_SX_XLOCK(ENOSYS);
292920b555e1SKonstantin Belousov 	is_sx_downgraded = 0;
29304a3690dfSJohn Baldwin 	PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
2931ebccf1e3SJoseph Koshy 	    pmc_op_to_name[op], arg);
2932ebccf1e3SJoseph Koshy 
2933ebccf1e3SJoseph Koshy 	error = 0;
2934e6b475e0SMatt Macy 	counter_u64_add(pmc_stats.pm_syscalls, 1);
2935ebccf1e3SJoseph Koshy 
2936cf9ef806SKonstantin Belousov 	switch (op) {
2937ebccf1e3SJoseph Koshy 
2938ebccf1e3SJoseph Koshy 
2939ebccf1e3SJoseph Koshy 	/*
2940ebccf1e3SJoseph Koshy 	 * Configure a log file.
2941ebccf1e3SJoseph Koshy 	 *
2942ebccf1e3SJoseph Koshy 	 * XXX This OP will be reworked.
2943ebccf1e3SJoseph Koshy 	 */
2944ebccf1e3SJoseph Koshy 
2945ebccf1e3SJoseph Koshy 	case PMC_OP_CONFIGURELOG:
2946ebccf1e3SJoseph Koshy 	{
294749874f6eSJoseph Koshy 		struct proc *p;
2948fc9a2b80SJoseph Koshy 		struct pmc *pm;
2949ebccf1e3SJoseph Koshy 		struct pmc_owner *po;
2950ebccf1e3SJoseph Koshy 		struct pmc_op_configurelog cl;
2951ebccf1e3SJoseph Koshy 
295220b555e1SKonstantin Belousov 		if ((error = copyin(arg, &cl, sizeof(cl))) != 0) {
295320b555e1SKonstantin Belousov 			pmclog_proc_ignite(pmclog_proc_handle, NULL);
2954ebccf1e3SJoseph Koshy 			break;
295520b555e1SKonstantin Belousov 		}
2956ebccf1e3SJoseph Koshy 
2957ebccf1e3SJoseph Koshy 		/* mark this process as owning a log file */
2958ebccf1e3SJoseph Koshy 		p = td->td_proc;
2959ebccf1e3SJoseph Koshy 		if ((po = pmc_find_owner_descriptor(p)) == NULL)
2960f263522aSJoseph Koshy 			if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
296120b555e1SKonstantin Belousov 				pmclog_proc_ignite(pmclog_proc_handle, NULL);
2962f263522aSJoseph Koshy 				error = ENOMEM;
2963ebccf1e3SJoseph Koshy 				break;
2964f263522aSJoseph Koshy 			}
2965ebccf1e3SJoseph Koshy 
2966f263522aSJoseph Koshy 		/*
2967f263522aSJoseph Koshy 		 * If a valid fd was passed in, try to configure that,
2968f263522aSJoseph Koshy 		 * otherwise if 'fd' was less than zero and there was
2969f263522aSJoseph Koshy 		 * a log file configured, flush its buffers and
2970f263522aSJoseph Koshy 		 * de-configure it.
2971f263522aSJoseph Koshy 		 */
2972ca2d94beSAttilio Rao 		if (cl.pm_logfd >= 0) {
29730cfab8ddSJoseph Koshy 			error = pmclog_configure_log(md, po, cl.pm_logfd);
297420b555e1SKonstantin Belousov 			pmclog_proc_ignite(pmclog_proc_handle, error == 0 ?
297520b555e1SKonstantin Belousov 			    po : NULL);
2976ca2d94beSAttilio Rao 		} else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
297720b555e1SKonstantin Belousov 			pmclog_proc_ignite(pmclog_proc_handle, NULL);
2978dceed24aSFabien Thomas 			error = pmclog_close(po);
2979fc9a2b80SJoseph Koshy 			if (error == 0) {
2980fc9a2b80SJoseph Koshy 				LIST_FOREACH(pm, &po->po_pmcs, pm_next)
298166f3bc78SJoseph Koshy 				    if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
298266f3bc78SJoseph Koshy 					pm->pm_state == PMC_STATE_RUNNING)
2983fc9a2b80SJoseph Koshy 					    pmc_stop(pm);
2984f263522aSJoseph Koshy 				error = pmclog_deconfigure_log(po);
2985fc9a2b80SJoseph Koshy 			}
298620b555e1SKonstantin Belousov 		} else {
298720b555e1SKonstantin Belousov 			pmclog_proc_ignite(pmclog_proc_handle, NULL);
2988f263522aSJoseph Koshy 			error = EINVAL;
298920b555e1SKonstantin Belousov 		}
2990ebccf1e3SJoseph Koshy 	}
2991ebccf1e3SJoseph Koshy 	break;
2992ebccf1e3SJoseph Koshy 
2993ebccf1e3SJoseph Koshy 	/*
2994f263522aSJoseph Koshy 	 * Flush a log file.
2995f263522aSJoseph Koshy 	 */
2996f263522aSJoseph Koshy 
2997f263522aSJoseph Koshy 	case PMC_OP_FLUSHLOG:
2998f263522aSJoseph Koshy 	{
2999f263522aSJoseph Koshy 		struct pmc_owner *po;
3000f263522aSJoseph Koshy 
3001f263522aSJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
3002f263522aSJoseph Koshy 
3003f263522aSJoseph Koshy 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3004f263522aSJoseph Koshy 			error = EINVAL;
3005f263522aSJoseph Koshy 			break;
3006f263522aSJoseph Koshy 		}
3007f263522aSJoseph Koshy 
3008f263522aSJoseph Koshy 		error = pmclog_flush(po);
3009f263522aSJoseph Koshy 	}
3010f263522aSJoseph Koshy 	break;
3011f263522aSJoseph Koshy 
3012f263522aSJoseph Koshy 	/*
3013dceed24aSFabien Thomas 	 * Close a log file.
3014dceed24aSFabien Thomas 	 */
3015dceed24aSFabien Thomas 
3016dceed24aSFabien Thomas 	case PMC_OP_CLOSELOG:
3017dceed24aSFabien Thomas 	{
3018dceed24aSFabien Thomas 		struct pmc_owner *po;
3019dceed24aSFabien Thomas 
3020dceed24aSFabien Thomas 		sx_assert(&pmc_sx, SX_XLOCKED);
3021dceed24aSFabien Thomas 
3022dceed24aSFabien Thomas 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3023dceed24aSFabien Thomas 			error = EINVAL;
3024dceed24aSFabien Thomas 			break;
3025dceed24aSFabien Thomas 		}
3026dceed24aSFabien Thomas 
3027dceed24aSFabien Thomas 		error = pmclog_close(po);
3028dceed24aSFabien Thomas 	}
3029dceed24aSFabien Thomas 	break;
3030dceed24aSFabien Thomas 
3031dceed24aSFabien Thomas 	/*
3032ebccf1e3SJoseph Koshy 	 * Retrieve hardware configuration.
3033ebccf1e3SJoseph Koshy 	 */
3034ebccf1e3SJoseph Koshy 
3035ebccf1e3SJoseph Koshy 	case PMC_OP_GETCPUINFO:	/* CPU information */
3036ebccf1e3SJoseph Koshy 	{
3037ebccf1e3SJoseph Koshy 		struct pmc_op_getcpuinfo gci;
3038e829eb6dSJoseph Koshy 		struct pmc_classinfo *pci;
3039e829eb6dSJoseph Koshy 		struct pmc_classdep *pcd;
3040e829eb6dSJoseph Koshy 		int cl;
3041ebccf1e3SJoseph Koshy 
3042ebccf1e3SJoseph Koshy 		gci.pm_cputype = md->pmd_cputype;
3043122ccdc1SJoseph Koshy 		gci.pm_ncpu    = pmc_cpu_max();
3044ebccf1e3SJoseph Koshy 		gci.pm_npmc    = md->pmd_npmc;
3045ebccf1e3SJoseph Koshy 		gci.pm_nclass  = md->pmd_nclass;
3046e829eb6dSJoseph Koshy 		pci = gci.pm_classes;
3047e829eb6dSJoseph Koshy 		pcd = md->pmd_classdep;
3048e829eb6dSJoseph Koshy 		for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
3049e829eb6dSJoseph Koshy 			pci->pm_caps  = pcd->pcd_caps;
3050e829eb6dSJoseph Koshy 			pci->pm_class = pcd->pcd_class;
3051e829eb6dSJoseph Koshy 			pci->pm_width = pcd->pcd_width;
3052e829eb6dSJoseph Koshy 			pci->pm_num   = pcd->pcd_num;
3053e829eb6dSJoseph Koshy 		}
3054ebccf1e3SJoseph Koshy 		error = copyout(&gci, arg, sizeof(gci));
3055ebccf1e3SJoseph Koshy 	}
3056ebccf1e3SJoseph Koshy 	break;
3057ebccf1e3SJoseph Koshy 
3058f5f9340bSFabien Thomas 	/*
3059f5f9340bSFabien Thomas 	 * Retrieve soft events list.
3060f5f9340bSFabien Thomas 	 */
3061f5f9340bSFabien Thomas 	case PMC_OP_GETDYNEVENTINFO:
3062f5f9340bSFabien Thomas 	{
3063f5f9340bSFabien Thomas 		enum pmc_class			cl;
3064f5f9340bSFabien Thomas 		enum pmc_event			ev;
3065f5f9340bSFabien Thomas 		struct pmc_op_getdyneventinfo	*gei;
3066f5f9340bSFabien Thomas 		struct pmc_dyn_event_descr	dev;
3067f5f9340bSFabien Thomas 		struct pmc_soft			*ps;
3068f5f9340bSFabien Thomas 		uint32_t			nevent;
3069f5f9340bSFabien Thomas 
3070f5f9340bSFabien Thomas 		sx_assert(&pmc_sx, SX_LOCKED);
3071f5f9340bSFabien Thomas 
3072f5f9340bSFabien Thomas 		gei = (struct pmc_op_getdyneventinfo *) arg;
3073f5f9340bSFabien Thomas 
3074f5f9340bSFabien Thomas 		if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
3075f5f9340bSFabien Thomas 			break;
3076f5f9340bSFabien Thomas 
3077f5f9340bSFabien Thomas 		/* Only SOFT class is dynamic. */
3078f5f9340bSFabien Thomas 		if (cl != PMC_CLASS_SOFT) {
3079f5f9340bSFabien Thomas 			error = EINVAL;
3080f5f9340bSFabien Thomas 			break;
3081f5f9340bSFabien Thomas 		}
3082f5f9340bSFabien Thomas 
3083f5f9340bSFabien Thomas 		nevent = 0;
3084ca57f64fSSean Bruno 		for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
3085f5f9340bSFabien Thomas 			ps = pmc_soft_ev_acquire(ev);
3086f5f9340bSFabien Thomas 			if (ps == NULL)
3087f5f9340bSFabien Thomas 				continue;
3088f5f9340bSFabien Thomas 			bcopy(&ps->ps_ev, &dev, sizeof(dev));
3089f5f9340bSFabien Thomas 			pmc_soft_ev_release(ps);
3090f5f9340bSFabien Thomas 
3091f5f9340bSFabien Thomas 			error = copyout(&dev,
3092f5f9340bSFabien Thomas 			    &gei->pm_events[nevent],
3093f5f9340bSFabien Thomas 			    sizeof(struct pmc_dyn_event_descr));
3094f5f9340bSFabien Thomas 			if (error != 0)
3095f5f9340bSFabien Thomas 				break;
3096f5f9340bSFabien Thomas 			nevent++;
3097f5f9340bSFabien Thomas 		}
3098f5f9340bSFabien Thomas 		if (error != 0)
3099f5f9340bSFabien Thomas 			break;
3100f5f9340bSFabien Thomas 
3101f5f9340bSFabien Thomas 		error = copyout(&nevent, &gei->pm_nevent,
3102f5f9340bSFabien Thomas 		    sizeof(nevent));
3103f5f9340bSFabien Thomas 	}
3104f5f9340bSFabien Thomas 	break;
3105ebccf1e3SJoseph Koshy 
3106ebccf1e3SJoseph Koshy 	/*
3107ebccf1e3SJoseph Koshy 	 * Get module statistics
3108ebccf1e3SJoseph Koshy 	 */
3109ebccf1e3SJoseph Koshy 
3110ebccf1e3SJoseph Koshy 	case PMC_OP_GETDRIVERSTATS:
3111ebccf1e3SJoseph Koshy 	{
3112ebccf1e3SJoseph Koshy 		struct pmc_op_getdriverstats gms;
3113e6b475e0SMatt Macy #define CFETCH(a, b, field) a.field = counter_u64_fetch(b.field)
3114e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_intr_ignored);
3115e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_intr_processed);
3116e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_intr_bufferfull);
3117e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_syscalls);
3118e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_syscall_errors);
3119e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_buffer_requests);
3120e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_buffer_requests_failed);
3121e6b475e0SMatt Macy 		CFETCH(gms, pmc_stats, pm_log_sweeps);
3122e6b475e0SMatt Macy #undef CFETCH
3123ebccf1e3SJoseph Koshy 		error = copyout(&gms, arg, sizeof(gms));
3124ebccf1e3SJoseph Koshy 	}
3125ebccf1e3SJoseph Koshy 	break;
3126ebccf1e3SJoseph Koshy 
3127ebccf1e3SJoseph Koshy 
3128ebccf1e3SJoseph Koshy 	/*
3129ebccf1e3SJoseph Koshy 	 * Retrieve module version number
3130ebccf1e3SJoseph Koshy 	 */
3131ebccf1e3SJoseph Koshy 
3132ebccf1e3SJoseph Koshy 	case PMC_OP_GETMODULEVERSION:
3133ebccf1e3SJoseph Koshy 	{
3134f263522aSJoseph Koshy 		uint32_t cv, modv;
3135f263522aSJoseph Koshy 
3136f263522aSJoseph Koshy 		/* retrieve the client's idea of the ABI version */
3137f263522aSJoseph Koshy 		if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
3138f263522aSJoseph Koshy 			break;
3139f263522aSJoseph Koshy 		/* don't service clients newer than our driver */
3140f263522aSJoseph Koshy 		modv = PMC_VERSION;
3141f263522aSJoseph Koshy 		if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
3142f263522aSJoseph Koshy 			error = EPROGMISMATCH;
3143f263522aSJoseph Koshy 			break;
3144f263522aSJoseph Koshy 		}
3145f263522aSJoseph Koshy 		error = copyout(&modv, arg, sizeof(int));
3146ebccf1e3SJoseph Koshy 	}
3147ebccf1e3SJoseph Koshy 	break;
3148ebccf1e3SJoseph Koshy 
3149ebccf1e3SJoseph Koshy 
3150ebccf1e3SJoseph Koshy 	/*
3151ebccf1e3SJoseph Koshy 	 * Retrieve the state of all the PMCs on a given
3152ebccf1e3SJoseph Koshy 	 * CPU.
3153ebccf1e3SJoseph Koshy 	 */
3154ebccf1e3SJoseph Koshy 
3155ebccf1e3SJoseph Koshy 	case PMC_OP_GETPMCINFO:
3156ebccf1e3SJoseph Koshy 	{
3157e829eb6dSJoseph Koshy 		int ari;
3158ebccf1e3SJoseph Koshy 		struct pmc *pm;
3159e829eb6dSJoseph Koshy 		size_t pmcinfo_size;
3160e829eb6dSJoseph Koshy 		uint32_t cpu, n, npmc;
3161ebccf1e3SJoseph Koshy 		struct pmc_owner *po;
3162ebccf1e3SJoseph Koshy 		struct pmc_binding pb;
3163e829eb6dSJoseph Koshy 		struct pmc_classdep *pcd;
3164e829eb6dSJoseph Koshy 		struct pmc_info *p, *pmcinfo;
3165e829eb6dSJoseph Koshy 		struct pmc_op_getpmcinfo *gpi;
3166ebccf1e3SJoseph Koshy 
3167ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
3168ebccf1e3SJoseph Koshy 
3169ebccf1e3SJoseph Koshy 		gpi = (struct pmc_op_getpmcinfo *) arg;
3170ebccf1e3SJoseph Koshy 
3171ebccf1e3SJoseph Koshy 		if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
3172ebccf1e3SJoseph Koshy 			break;
3173ebccf1e3SJoseph Koshy 
3174122ccdc1SJoseph Koshy 		if (cpu >= pmc_cpu_max()) {
3175ebccf1e3SJoseph Koshy 			error = EINVAL;
3176ebccf1e3SJoseph Koshy 			break;
3177ebccf1e3SJoseph Koshy 		}
3178ebccf1e3SJoseph Koshy 
3179122ccdc1SJoseph Koshy 		if (!pmc_cpu_is_active(cpu)) {
3180ebccf1e3SJoseph Koshy 			error = ENXIO;
3181ebccf1e3SJoseph Koshy 			break;
3182ebccf1e3SJoseph Koshy 		}
3183ebccf1e3SJoseph Koshy 
3184ebccf1e3SJoseph Koshy 		/* switch to CPU 'cpu' */
3185ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
3186ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
3187ebccf1e3SJoseph Koshy 
3188ebccf1e3SJoseph Koshy 		npmc = md->pmd_npmc;
3189ebccf1e3SJoseph Koshy 
3190ebccf1e3SJoseph Koshy 		pmcinfo_size = npmc * sizeof(struct pmc_info);
3191e11e3f18SDag-Erling Smørgrav 		pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
3192ebccf1e3SJoseph Koshy 
3193ebccf1e3SJoseph Koshy 		p = pmcinfo;
3194ebccf1e3SJoseph Koshy 
3195ebccf1e3SJoseph Koshy 		for (n = 0; n < md->pmd_npmc; n++, p++) {
3196ebccf1e3SJoseph Koshy 
3197e829eb6dSJoseph Koshy 			pcd = pmc_ri_to_classdep(md, n, &ari);
3198e829eb6dSJoseph Koshy 
3199e829eb6dSJoseph Koshy 			KASSERT(pcd != NULL,
3200e829eb6dSJoseph Koshy 			    ("[pmc,%d] null pcd ri=%d", __LINE__, n));
3201e829eb6dSJoseph Koshy 
3202e829eb6dSJoseph Koshy 			if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
3203ebccf1e3SJoseph Koshy 				break;
3204ebccf1e3SJoseph Koshy 
3205ebccf1e3SJoseph Koshy 			if (PMC_ROW_DISP_IS_STANDALONE(n))
3206ebccf1e3SJoseph Koshy 				p->pm_rowdisp = PMC_DISP_STANDALONE;
3207ebccf1e3SJoseph Koshy 			else if (PMC_ROW_DISP_IS_THREAD(n))
3208ebccf1e3SJoseph Koshy 				p->pm_rowdisp = PMC_DISP_THREAD;
3209ebccf1e3SJoseph Koshy 			else
3210ebccf1e3SJoseph Koshy 				p->pm_rowdisp = PMC_DISP_FREE;
3211ebccf1e3SJoseph Koshy 
3212ebccf1e3SJoseph Koshy 			p->pm_ownerpid = -1;
3213ebccf1e3SJoseph Koshy 
3214ebccf1e3SJoseph Koshy 			if (pm == NULL)	/* no PMC associated */
3215ebccf1e3SJoseph Koshy 				continue;
3216ebccf1e3SJoseph Koshy 
3217ebccf1e3SJoseph Koshy 			po = pm->pm_owner;
3218ebccf1e3SJoseph Koshy 
3219ebccf1e3SJoseph Koshy 			KASSERT(po->po_owner != NULL,
3220ebccf1e3SJoseph Koshy 			    ("[pmc,%d] pmc_owner had a null proc pointer",
3221ebccf1e3SJoseph Koshy 				__LINE__));
3222ebccf1e3SJoseph Koshy 
3223ebccf1e3SJoseph Koshy 			p->pm_ownerpid = po->po_owner->p_pid;
3224c5153e19SJoseph Koshy 			p->pm_mode     = PMC_TO_MODE(pm);
3225ebccf1e3SJoseph Koshy 			p->pm_event    = pm->pm_event;
3226ebccf1e3SJoseph Koshy 			p->pm_flags    = pm->pm_flags;
3227ebccf1e3SJoseph Koshy 
3228c5153e19SJoseph Koshy 			if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3229ebccf1e3SJoseph Koshy 				p->pm_reloadcount =
3230ebccf1e3SJoseph Koshy 				    pm->pm_sc.pm_reloadcount;
3231ebccf1e3SJoseph Koshy 		}
3232ebccf1e3SJoseph Koshy 
3233ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
3234ebccf1e3SJoseph Koshy 
3235ebccf1e3SJoseph Koshy 		/* now copy out the PMC info collected */
3236ebccf1e3SJoseph Koshy 		if (error == 0)
3237ebccf1e3SJoseph Koshy 			error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
3238ebccf1e3SJoseph Koshy 
32391ede983cSDag-Erling Smørgrav 		free(pmcinfo, M_PMC);
3240ebccf1e3SJoseph Koshy 	}
3241ebccf1e3SJoseph Koshy 	break;
3242ebccf1e3SJoseph Koshy 
3243ebccf1e3SJoseph Koshy 
3244ebccf1e3SJoseph Koshy 	/*
3245ebccf1e3SJoseph Koshy 	 * Set the administrative state of a PMC.  I.e. whether
3246ebccf1e3SJoseph Koshy 	 * the PMC is to be used or not.
3247ebccf1e3SJoseph Koshy 	 */
3248ebccf1e3SJoseph Koshy 
3249ebccf1e3SJoseph Koshy 	case PMC_OP_PMCADMIN:
3250ebccf1e3SJoseph Koshy 	{
3251ebccf1e3SJoseph Koshy 		int cpu, ri;
3252ebccf1e3SJoseph Koshy 		enum pmc_state request;
3253ebccf1e3SJoseph Koshy 		struct pmc_cpu *pc;
3254ebccf1e3SJoseph Koshy 		struct pmc_hw *phw;
3255ebccf1e3SJoseph Koshy 		struct pmc_op_pmcadmin pma;
3256ebccf1e3SJoseph Koshy 		struct pmc_binding pb;
3257ebccf1e3SJoseph Koshy 
3258ebccf1e3SJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
3259ebccf1e3SJoseph Koshy 
3260ebccf1e3SJoseph Koshy 		KASSERT(td == curthread,
3261ebccf1e3SJoseph Koshy 		    ("[pmc,%d] td != curthread", __LINE__));
3262ebccf1e3SJoseph Koshy 
3263acd3428bSRobert Watson 		error = priv_check(td, PRIV_PMC_MANAGE);
3264acd3428bSRobert Watson 		if (error)
3265ebccf1e3SJoseph Koshy 			break;
3266ebccf1e3SJoseph Koshy 
3267ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
3268ebccf1e3SJoseph Koshy 			break;
3269ebccf1e3SJoseph Koshy 
3270ebccf1e3SJoseph Koshy 		cpu = pma.pm_cpu;
3271ebccf1e3SJoseph Koshy 
3272122ccdc1SJoseph Koshy 		if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
3273ebccf1e3SJoseph Koshy 			error = EINVAL;
3274ebccf1e3SJoseph Koshy 			break;
3275ebccf1e3SJoseph Koshy 		}
3276ebccf1e3SJoseph Koshy 
3277122ccdc1SJoseph Koshy 		if (!pmc_cpu_is_active(cpu)) {
3278ebccf1e3SJoseph Koshy 			error = ENXIO;
3279ebccf1e3SJoseph Koshy 			break;
3280ebccf1e3SJoseph Koshy 		}
3281ebccf1e3SJoseph Koshy 
3282ebccf1e3SJoseph Koshy 		request = pma.pm_state;
3283ebccf1e3SJoseph Koshy 
3284ebccf1e3SJoseph Koshy 		if (request != PMC_STATE_DISABLED &&
3285ebccf1e3SJoseph Koshy 		    request != PMC_STATE_FREE) {
3286ebccf1e3SJoseph Koshy 			error = EINVAL;
3287ebccf1e3SJoseph Koshy 			break;
3288ebccf1e3SJoseph Koshy 		}
3289ebccf1e3SJoseph Koshy 
3290ebccf1e3SJoseph Koshy 		ri = pma.pm_pmc; /* pmc id == row index */
3291ebccf1e3SJoseph Koshy 		if (ri < 0 || ri >= (int) md->pmd_npmc) {
3292ebccf1e3SJoseph Koshy 			error = EINVAL;
3293ebccf1e3SJoseph Koshy 			break;
3294ebccf1e3SJoseph Koshy 		}
3295ebccf1e3SJoseph Koshy 
3296ebccf1e3SJoseph Koshy 		/*
3297ebccf1e3SJoseph Koshy 		 * We can't disable a PMC with a row-index allocated
3298ebccf1e3SJoseph Koshy 		 * for process virtual PMCs.
3299ebccf1e3SJoseph Koshy 		 */
3300ebccf1e3SJoseph Koshy 
3301ebccf1e3SJoseph Koshy 		if (PMC_ROW_DISP_IS_THREAD(ri) &&
3302ebccf1e3SJoseph Koshy 		    request == PMC_STATE_DISABLED) {
3303ebccf1e3SJoseph Koshy 			error = EBUSY;
3304ebccf1e3SJoseph Koshy 			break;
3305ebccf1e3SJoseph Koshy 		}
3306ebccf1e3SJoseph Koshy 
3307ebccf1e3SJoseph Koshy 		/*
3308ebccf1e3SJoseph Koshy 		 * otherwise, this PMC on this CPU is either free or
3309ebccf1e3SJoseph Koshy 		 * in system-wide mode.
3310ebccf1e3SJoseph Koshy 		 */
3311ebccf1e3SJoseph Koshy 
3312ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
3313ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
3314ebccf1e3SJoseph Koshy 
3315ebccf1e3SJoseph Koshy 		pc  = pmc_pcpu[cpu];
3316ebccf1e3SJoseph Koshy 		phw = pc->pc_hwpmcs[ri];
3317ebccf1e3SJoseph Koshy 
3318ebccf1e3SJoseph Koshy 		/*
3319ebccf1e3SJoseph Koshy 		 * XXX do we need some kind of 'forced' disable?
3320ebccf1e3SJoseph Koshy 		 */
3321ebccf1e3SJoseph Koshy 
3322ebccf1e3SJoseph Koshy 		if (phw->phw_pmc == NULL) {
3323ebccf1e3SJoseph Koshy 			if (request == PMC_STATE_DISABLED &&
3324ebccf1e3SJoseph Koshy 			    (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
3325ebccf1e3SJoseph Koshy 				phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
3326ebccf1e3SJoseph Koshy 				PMC_MARK_ROW_STANDALONE(ri);
3327ebccf1e3SJoseph Koshy 			} else if (request == PMC_STATE_FREE &&
3328ebccf1e3SJoseph Koshy 			    (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
3329ebccf1e3SJoseph Koshy 				phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
3330ebccf1e3SJoseph Koshy 				PMC_UNMARK_ROW_STANDALONE(ri);
3331ebccf1e3SJoseph Koshy 			}
3332ebccf1e3SJoseph Koshy 			/* other cases are a no-op */
3333ebccf1e3SJoseph Koshy 		} else
3334ebccf1e3SJoseph Koshy 			error = EBUSY;
3335ebccf1e3SJoseph Koshy 
3336ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
3337ebccf1e3SJoseph Koshy 	}
3338ebccf1e3SJoseph Koshy 	break;
3339ebccf1e3SJoseph Koshy 
3340ebccf1e3SJoseph Koshy 
3341ebccf1e3SJoseph Koshy 	/*
3342ebccf1e3SJoseph Koshy 	 * Allocate a PMC.
3343ebccf1e3SJoseph Koshy 	 */
3344ebccf1e3SJoseph Koshy 
3345ebccf1e3SJoseph Koshy 	case PMC_OP_PMCALLOCATE:
3346ebccf1e3SJoseph Koshy 	{
3347e829eb6dSJoseph Koshy 		int adjri, n;
3348ebccf1e3SJoseph Koshy 		u_int cpu;
3349e829eb6dSJoseph Koshy 		uint32_t caps;
3350ebccf1e3SJoseph Koshy 		struct pmc *pmc;
3351e829eb6dSJoseph Koshy 		enum pmc_mode mode;
3352c5153e19SJoseph Koshy 		struct pmc_hw *phw;
3353ebccf1e3SJoseph Koshy 		struct pmc_binding pb;
3354e829eb6dSJoseph Koshy 		struct pmc_classdep *pcd;
3355e829eb6dSJoseph Koshy 		struct pmc_op_pmcallocate pa;
3356ebccf1e3SJoseph Koshy 
3357ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
3358ebccf1e3SJoseph Koshy 			break;
3359ebccf1e3SJoseph Koshy 
3360ebccf1e3SJoseph Koshy 		caps = pa.pm_caps;
3361ebccf1e3SJoseph Koshy 		mode = pa.pm_mode;
3362ebccf1e3SJoseph Koshy 		cpu  = pa.pm_cpu;
3363ebccf1e3SJoseph Koshy 
3364ebccf1e3SJoseph Koshy 		if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
3365ebccf1e3SJoseph Koshy 		     mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
3366122ccdc1SJoseph Koshy 		    (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
3367ebccf1e3SJoseph Koshy 			error = EINVAL;
3368ebccf1e3SJoseph Koshy 			break;
3369ebccf1e3SJoseph Koshy 		}
3370ebccf1e3SJoseph Koshy 
3371ebccf1e3SJoseph Koshy 		/*
3372ebccf1e3SJoseph Koshy 		 * Virtual PMCs should only ask for a default CPU.
3373ebccf1e3SJoseph Koshy 		 * System mode PMCs need to specify a non-default CPU.
3374ebccf1e3SJoseph Koshy 		 */
3375ebccf1e3SJoseph Koshy 
3376ebccf1e3SJoseph Koshy 		if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
3377ebccf1e3SJoseph Koshy 		    (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
3378ebccf1e3SJoseph Koshy 			error = EINVAL;
3379ebccf1e3SJoseph Koshy 			break;
3380ebccf1e3SJoseph Koshy 		}
3381ebccf1e3SJoseph Koshy 
3382ebccf1e3SJoseph Koshy 		/*
3383122ccdc1SJoseph Koshy 		 * Check that an inactive CPU is not being asked for.
3384ebccf1e3SJoseph Koshy 		 */
3385ebccf1e3SJoseph Koshy 
3386122ccdc1SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
3387ebccf1e3SJoseph Koshy 			error = ENXIO;
3388ebccf1e3SJoseph Koshy 			break;
3389ebccf1e3SJoseph Koshy 		}
3390ebccf1e3SJoseph Koshy 
3391ebccf1e3SJoseph Koshy 		/*
3392ebccf1e3SJoseph Koshy 		 * Refuse an allocation for a system-wide PMC if this
3393ebccf1e3SJoseph Koshy 		 * process has been jailed, or if this process lacks
3394ebccf1e3SJoseph Koshy 		 * super-user credentials and the sysctl tunable
3395ebccf1e3SJoseph Koshy 		 * 'security.bsd.unprivileged_syspmcs' is zero.
3396ebccf1e3SJoseph Koshy 		 */
3397ebccf1e3SJoseph Koshy 
3398ebccf1e3SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode)) {
3399acd3428bSRobert Watson 			if (jailed(curthread->td_ucred)) {
3400ebccf1e3SJoseph Koshy 				error = EPERM;
3401acd3428bSRobert Watson 				break;
3402acd3428bSRobert Watson 			}
3403acd3428bSRobert Watson 			if (!pmc_unprivileged_syspmcs) {
3404acd3428bSRobert Watson 				error = priv_check(curthread,
3405acd3428bSRobert Watson 				    PRIV_PMC_SYSTEM);
3406acd3428bSRobert Watson 				if (error)
3407acd3428bSRobert Watson 					break;
3408acd3428bSRobert Watson 			}
3409ebccf1e3SJoseph Koshy 		}
3410ebccf1e3SJoseph Koshy 
3411ebccf1e3SJoseph Koshy 		/*
3412ebccf1e3SJoseph Koshy 		 * Look for valid values for 'pm_flags'
3413ebccf1e3SJoseph Koshy 		 */
3414ebccf1e3SJoseph Koshy 
3415f263522aSJoseph Koshy 		if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
3416d07f36b0SJoseph Koshy 		    PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) {
3417f263522aSJoseph Koshy 			error = EINVAL;
3418f263522aSJoseph Koshy 			break;
3419f263522aSJoseph Koshy 		}
3420f263522aSJoseph Koshy 
3421f263522aSJoseph Koshy 		/* process logging options are not allowed for system PMCs */
3422f263522aSJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3423f263522aSJoseph Koshy 		    (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3424ebccf1e3SJoseph Koshy 			error = EINVAL;
3425ebccf1e3SJoseph Koshy 			break;
3426ebccf1e3SJoseph Koshy 		}
3427ebccf1e3SJoseph Koshy 
3428ebccf1e3SJoseph Koshy 		/*
3429ebccf1e3SJoseph Koshy 		 * All sampling mode PMCs need to be able to interrupt the
3430ebccf1e3SJoseph Koshy 		 * CPU.
3431ebccf1e3SJoseph Koshy 		 */
3432f263522aSJoseph Koshy 		if (PMC_IS_SAMPLING_MODE(mode))
3433ebccf1e3SJoseph Koshy 			caps |= PMC_CAP_INTERRUPT;
3434ebccf1e3SJoseph Koshy 
3435744d6797SJoseph Koshy 		/* A valid class specifier should have been passed in. */
3436744d6797SJoseph Koshy 		for (n = 0; n < md->pmd_nclass; n++)
3437e829eb6dSJoseph Koshy 			if (md->pmd_classdep[n].pcd_class == pa.pm_class)
3438744d6797SJoseph Koshy 				break;
3439744d6797SJoseph Koshy 		if (n == md->pmd_nclass) {
3440744d6797SJoseph Koshy 			error = EINVAL;
3441744d6797SJoseph Koshy 			break;
3442744d6797SJoseph Koshy 		}
3443744d6797SJoseph Koshy 
3444744d6797SJoseph Koshy 		/* The requested PMC capabilities should be feasible. */
3445e829eb6dSJoseph Koshy 		if ((md->pmd_classdep[n].pcd_caps & caps) != caps) {
3446744d6797SJoseph Koshy 			error = EOPNOTSUPP;
3447744d6797SJoseph Koshy 			break;
3448744d6797SJoseph Koshy 		}
3449744d6797SJoseph Koshy 
34504a3690dfSJohn Baldwin 		PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3451ebccf1e3SJoseph Koshy 		    pa.pm_ev, caps, mode, cpu);
3452ebccf1e3SJoseph Koshy 
3453ebccf1e3SJoseph Koshy 		pmc = pmc_allocate_pmc_descriptor();
3454c5153e19SJoseph Koshy 		pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
3455c5153e19SJoseph Koshy 		    PMC_ID_INVALID);
3456ebccf1e3SJoseph Koshy 		pmc->pm_event = pa.pm_ev;
3457ebccf1e3SJoseph Koshy 		pmc->pm_state = PMC_STATE_FREE;
3458ebccf1e3SJoseph Koshy 		pmc->pm_caps  = caps;
3459ebccf1e3SJoseph Koshy 		pmc->pm_flags = pa.pm_flags;
3460ebccf1e3SJoseph Koshy 
3461ebccf1e3SJoseph Koshy 		/* switch thread to CPU 'cpu' */
3462ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
3463ebccf1e3SJoseph Koshy 
3464ebccf1e3SJoseph Koshy #define	PMC_IS_SHAREABLE_PMC(cpu, n)				\
3465ebccf1e3SJoseph Koshy 	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &		\
3466ebccf1e3SJoseph Koshy 	 PMC_PHW_FLAG_IS_SHAREABLE)
3467ebccf1e3SJoseph Koshy #define	PMC_IS_UNALLOCATED(cpu, n)				\
3468ebccf1e3SJoseph Koshy 	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3469ebccf1e3SJoseph Koshy 
3470ebccf1e3SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode)) {
3471ebccf1e3SJoseph Koshy 			pmc_select_cpu(cpu);
3472e829eb6dSJoseph Koshy 			for (n = 0; n < (int) md->pmd_npmc; n++) {
3473e829eb6dSJoseph Koshy 				pcd = pmc_ri_to_classdep(md, n, &adjri);
3474ebccf1e3SJoseph Koshy 				if (pmc_can_allocate_row(n, mode) == 0 &&
3475ebccf1e3SJoseph Koshy 				    pmc_can_allocate_rowindex(
3476c5153e19SJoseph Koshy 					    curthread->td_proc, n, cpu) == 0 &&
3477ebccf1e3SJoseph Koshy 				    (PMC_IS_UNALLOCATED(cpu, n) ||
3478ebccf1e3SJoseph Koshy 				     PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3479e829eb6dSJoseph Koshy 				    pcd->pcd_allocate_pmc(cpu, adjri, pmc,
3480ebccf1e3SJoseph Koshy 					&pa) == 0)
3481ebccf1e3SJoseph Koshy 					break;
3482e829eb6dSJoseph Koshy 			}
3483ebccf1e3SJoseph Koshy 		} else {
3484ebccf1e3SJoseph Koshy 			/* Process virtual mode */
3485ebccf1e3SJoseph Koshy 			for (n = 0; n < (int) md->pmd_npmc; n++) {
3486e829eb6dSJoseph Koshy 				pcd = pmc_ri_to_classdep(md, n, &adjri);
3487ebccf1e3SJoseph Koshy 				if (pmc_can_allocate_row(n, mode) == 0 &&
3488ebccf1e3SJoseph Koshy 				    pmc_can_allocate_rowindex(
3489c5153e19SJoseph Koshy 					    curthread->td_proc, n,
3490c5153e19SJoseph Koshy 					    PMC_CPU_ANY) == 0 &&
3491e829eb6dSJoseph Koshy 				    pcd->pcd_allocate_pmc(curthread->td_oncpu,
3492e829eb6dSJoseph Koshy 					adjri, pmc, &pa) == 0)
3493ebccf1e3SJoseph Koshy 					break;
3494ebccf1e3SJoseph Koshy 			}
3495ebccf1e3SJoseph Koshy 		}
3496ebccf1e3SJoseph Koshy 
3497ebccf1e3SJoseph Koshy #undef	PMC_IS_UNALLOCATED
3498ebccf1e3SJoseph Koshy #undef	PMC_IS_SHAREABLE_PMC
3499ebccf1e3SJoseph Koshy 
3500ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
3501ebccf1e3SJoseph Koshy 
3502ebccf1e3SJoseph Koshy 		if (n == (int) md->pmd_npmc) {
3503ebccf1e3SJoseph Koshy 			pmc_destroy_pmc_descriptor(pmc);
3504ebccf1e3SJoseph Koshy 			pmc = NULL;
3505ebccf1e3SJoseph Koshy 			error = EINVAL;
3506ebccf1e3SJoseph Koshy 			break;
3507ebccf1e3SJoseph Koshy 		}
3508ebccf1e3SJoseph Koshy 
3509c5153e19SJoseph Koshy 		/* Fill in the correct value in the ID field */
3510c5153e19SJoseph Koshy 		pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
3511c5153e19SJoseph Koshy 
35124a3690dfSJohn Baldwin 		PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
3513c5153e19SJoseph Koshy 		    pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
3514ebccf1e3SJoseph Koshy 
3515f263522aSJoseph Koshy 		/* Process mode PMCs with logging enabled need log files */
3516f263522aSJoseph Koshy 		if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
3517f263522aSJoseph Koshy 			pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3518f263522aSJoseph Koshy 
3519f263522aSJoseph Koshy 		/* All system mode sampling PMCs require a log file */
3520f263522aSJoseph Koshy 		if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
3521f263522aSJoseph Koshy 			pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3522f263522aSJoseph Koshy 
3523ebccf1e3SJoseph Koshy 		/*
3524ebccf1e3SJoseph Koshy 		 * Configure global pmc's immediately
3525ebccf1e3SJoseph Koshy 		 */
3526ebccf1e3SJoseph Koshy 
3527c5153e19SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
3528c5153e19SJoseph Koshy 
3529c5153e19SJoseph Koshy 			pmc_save_cpu_binding(&pb);
3530c5153e19SJoseph Koshy 			pmc_select_cpu(cpu);
3531c5153e19SJoseph Koshy 
3532c5153e19SJoseph Koshy 			phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
3533e829eb6dSJoseph Koshy 			pcd = pmc_ri_to_classdep(md, n, &adjri);
3534c5153e19SJoseph Koshy 
3535c5153e19SJoseph Koshy 			if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
3536e829eb6dSJoseph Koshy 			    (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
3537e829eb6dSJoseph Koshy 				(void) pcd->pcd_release_pmc(cpu, adjri, pmc);
3538ebccf1e3SJoseph Koshy 				pmc_destroy_pmc_descriptor(pmc);
3539ebccf1e3SJoseph Koshy 				pmc = NULL;
3540c5153e19SJoseph Koshy 				pmc_restore_cpu_binding(&pb);
3541c5153e19SJoseph Koshy 				error = EPERM;
3542ebccf1e3SJoseph Koshy 				break;
3543ebccf1e3SJoseph Koshy 			}
3544ebccf1e3SJoseph Koshy 
3545c5153e19SJoseph Koshy 			pmc_restore_cpu_binding(&pb);
3546c5153e19SJoseph Koshy 		}
3547ebccf1e3SJoseph Koshy 
3548ebccf1e3SJoseph Koshy 		pmc->pm_state    = PMC_STATE_ALLOCATED;
3549ebccf1e3SJoseph Koshy 
3550ebccf1e3SJoseph Koshy 		/*
3551ebccf1e3SJoseph Koshy 		 * mark row disposition
3552ebccf1e3SJoseph Koshy 		 */
3553ebccf1e3SJoseph Koshy 
3554ebccf1e3SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(mode))
3555ebccf1e3SJoseph Koshy 			PMC_MARK_ROW_STANDALONE(n);
3556ebccf1e3SJoseph Koshy 		else
3557ebccf1e3SJoseph Koshy 			PMC_MARK_ROW_THREAD(n);
3558ebccf1e3SJoseph Koshy 
3559ebccf1e3SJoseph Koshy 		/*
3560ebccf1e3SJoseph Koshy 		 * Register this PMC with the current thread as its owner.
3561ebccf1e3SJoseph Koshy 		 */
3562ebccf1e3SJoseph Koshy 
3563ebccf1e3SJoseph Koshy 		if ((error =
3564ebccf1e3SJoseph Koshy 		    pmc_register_owner(curthread->td_proc, pmc)) != 0) {
3565ebccf1e3SJoseph Koshy 			pmc_release_pmc_descriptor(pmc);
356606796b67SMark Johnston 			pmc_destroy_pmc_descriptor(pmc);
3567ebccf1e3SJoseph Koshy 			pmc = NULL;
3568ebccf1e3SJoseph Koshy 			break;
3569ebccf1e3SJoseph Koshy 		}
3570ebccf1e3SJoseph Koshy 
3571ebccf1e3SJoseph Koshy 		/*
3572ebccf1e3SJoseph Koshy 		 * Return the allocated index.
3573ebccf1e3SJoseph Koshy 		 */
3574ebccf1e3SJoseph Koshy 
3575c5153e19SJoseph Koshy 		pa.pm_pmcid = pmc->pm_id;
3576ebccf1e3SJoseph Koshy 
3577ebccf1e3SJoseph Koshy 		error = copyout(&pa, arg, sizeof(pa));
3578ebccf1e3SJoseph Koshy 	}
3579ebccf1e3SJoseph Koshy 	break;
3580ebccf1e3SJoseph Koshy 
3581ebccf1e3SJoseph Koshy 
3582ebccf1e3SJoseph Koshy 	/*
3583ebccf1e3SJoseph Koshy 	 * Attach a PMC to a process.
3584ebccf1e3SJoseph Koshy 	 */
3585ebccf1e3SJoseph Koshy 
3586ebccf1e3SJoseph Koshy 	case PMC_OP_PMCATTACH:
3587ebccf1e3SJoseph Koshy 	{
3588ebccf1e3SJoseph Koshy 		struct pmc *pm;
3589ebccf1e3SJoseph Koshy 		struct proc *p;
3590ebccf1e3SJoseph Koshy 		struct pmc_op_pmcattach a;
3591ebccf1e3SJoseph Koshy 
3592ebccf1e3SJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
3593ebccf1e3SJoseph Koshy 
3594ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &a, sizeof(a))) != 0)
3595ebccf1e3SJoseph Koshy 			break;
3596ebccf1e3SJoseph Koshy 
3597ebccf1e3SJoseph Koshy 		if (a.pm_pid < 0) {
3598ebccf1e3SJoseph Koshy 			error = EINVAL;
3599ebccf1e3SJoseph Koshy 			break;
3600ebccf1e3SJoseph Koshy 		} else if (a.pm_pid == 0)
3601ebccf1e3SJoseph Koshy 			a.pm_pid = td->td_proc->p_pid;
3602ebccf1e3SJoseph Koshy 
3603ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3604ebccf1e3SJoseph Koshy 			break;
3605ebccf1e3SJoseph Koshy 
3606c5153e19SJoseph Koshy 		if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
3607ebccf1e3SJoseph Koshy 			error = EINVAL;
3608ebccf1e3SJoseph Koshy 			break;
3609ebccf1e3SJoseph Koshy 		}
3610ebccf1e3SJoseph Koshy 
3611ebccf1e3SJoseph Koshy 		/* PMCs may be (re)attached only when allocated or stopped */
3612ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING) {
3613ebccf1e3SJoseph Koshy 			error = EBUSY;
3614ebccf1e3SJoseph Koshy 			break;
3615ebccf1e3SJoseph Koshy 		} else if (pm->pm_state != PMC_STATE_ALLOCATED &&
3616ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_STOPPED) {
3617ebccf1e3SJoseph Koshy 			error = EINVAL;
3618ebccf1e3SJoseph Koshy 			break;
3619ebccf1e3SJoseph Koshy 		}
3620ebccf1e3SJoseph Koshy 
3621ebccf1e3SJoseph Koshy 		/* lookup pid */
3622ebccf1e3SJoseph Koshy 		if ((p = pfind(a.pm_pid)) == NULL) {
3623ebccf1e3SJoseph Koshy 			error = ESRCH;
3624ebccf1e3SJoseph Koshy 			break;
3625ebccf1e3SJoseph Koshy 		}
3626ebccf1e3SJoseph Koshy 
3627ebccf1e3SJoseph Koshy 		/*
3628ebccf1e3SJoseph Koshy 		 * Ignore processes that are working on exiting.
3629ebccf1e3SJoseph Koshy 		 */
3630ebccf1e3SJoseph Koshy 		if (p->p_flag & P_WEXIT) {
3631ebccf1e3SJoseph Koshy 			error = ESRCH;
3632ebccf1e3SJoseph Koshy 			PROC_UNLOCK(p);	/* pfind() returns a locked process */
3633ebccf1e3SJoseph Koshy 			break;
3634ebccf1e3SJoseph Koshy 		}
3635ebccf1e3SJoseph Koshy 
3636ebccf1e3SJoseph Koshy 		/*
3637ebccf1e3SJoseph Koshy 		 * we are allowed to attach a PMC to a process if
3638ebccf1e3SJoseph Koshy 		 * we can debug it.
3639ebccf1e3SJoseph Koshy 		 */
3640ebccf1e3SJoseph Koshy 		error = p_candebug(curthread, p);
3641ebccf1e3SJoseph Koshy 
3642ebccf1e3SJoseph Koshy 		PROC_UNLOCK(p);
3643ebccf1e3SJoseph Koshy 
3644ebccf1e3SJoseph Koshy 		if (error == 0)
3645ebccf1e3SJoseph Koshy 			error = pmc_attach_process(p, pm);
3646ebccf1e3SJoseph Koshy 	}
3647ebccf1e3SJoseph Koshy 	break;
3648ebccf1e3SJoseph Koshy 
3649ebccf1e3SJoseph Koshy 
3650ebccf1e3SJoseph Koshy 	/*
3651ebccf1e3SJoseph Koshy 	 * Detach an attached PMC from a process.
3652ebccf1e3SJoseph Koshy 	 */
3653ebccf1e3SJoseph Koshy 
3654ebccf1e3SJoseph Koshy 	case PMC_OP_PMCDETACH:
3655ebccf1e3SJoseph Koshy 	{
3656ebccf1e3SJoseph Koshy 		struct pmc *pm;
3657ebccf1e3SJoseph Koshy 		struct proc *p;
3658ebccf1e3SJoseph Koshy 		struct pmc_op_pmcattach a;
3659ebccf1e3SJoseph Koshy 
3660ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &a, sizeof(a))) != 0)
3661ebccf1e3SJoseph Koshy 			break;
3662ebccf1e3SJoseph Koshy 
3663ebccf1e3SJoseph Koshy 		if (a.pm_pid < 0) {
3664ebccf1e3SJoseph Koshy 			error = EINVAL;
3665ebccf1e3SJoseph Koshy 			break;
3666ebccf1e3SJoseph Koshy 		} else if (a.pm_pid == 0)
3667ebccf1e3SJoseph Koshy 			a.pm_pid = td->td_proc->p_pid;
3668ebccf1e3SJoseph Koshy 
3669ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3670ebccf1e3SJoseph Koshy 			break;
3671ebccf1e3SJoseph Koshy 
3672ebccf1e3SJoseph Koshy 		if ((p = pfind(a.pm_pid)) == NULL) {
3673ebccf1e3SJoseph Koshy 			error = ESRCH;
3674ebccf1e3SJoseph Koshy 			break;
3675ebccf1e3SJoseph Koshy 		}
3676ebccf1e3SJoseph Koshy 
3677ebccf1e3SJoseph Koshy 		/*
3678ebccf1e3SJoseph Koshy 		 * Treat processes that are in the process of exiting
3679ebccf1e3SJoseph Koshy 		 * as if they were not present.
3680ebccf1e3SJoseph Koshy 		 */
3681ebccf1e3SJoseph Koshy 
3682ebccf1e3SJoseph Koshy 		if (p->p_flag & P_WEXIT)
3683ebccf1e3SJoseph Koshy 			error = ESRCH;
3684ebccf1e3SJoseph Koshy 
3685ebccf1e3SJoseph Koshy 		PROC_UNLOCK(p);	/* pfind() returns a locked process */
3686ebccf1e3SJoseph Koshy 
3687ebccf1e3SJoseph Koshy 		if (error == 0)
3688ebccf1e3SJoseph Koshy 			error = pmc_detach_process(p, pm);
3689ebccf1e3SJoseph Koshy 	}
3690ebccf1e3SJoseph Koshy 	break;
3691ebccf1e3SJoseph Koshy 
3692ebccf1e3SJoseph Koshy 
3693ebccf1e3SJoseph Koshy 	/*
3694f263522aSJoseph Koshy 	 * Retrieve the MSR number associated with the counter
3695f263522aSJoseph Koshy 	 * 'pmc_id'.  This allows processes to directly use RDPMC
3696f263522aSJoseph Koshy 	 * instructions to read their PMCs, without the overhead of a
3697f263522aSJoseph Koshy 	 * system call.
3698f263522aSJoseph Koshy 	 */
3699f263522aSJoseph Koshy 
3700f263522aSJoseph Koshy 	case PMC_OP_PMCGETMSR:
3701f263522aSJoseph Koshy 	{
3702e829eb6dSJoseph Koshy 		int adjri, ri;
3703f263522aSJoseph Koshy 		struct pmc *pm;
3704f263522aSJoseph Koshy 		struct pmc_target *pt;
3705f263522aSJoseph Koshy 		struct pmc_op_getmsr gm;
3706e829eb6dSJoseph Koshy 		struct pmc_classdep *pcd;
3707f263522aSJoseph Koshy 
3708f263522aSJoseph Koshy 		PMC_DOWNGRADE_SX();
3709f263522aSJoseph Koshy 
3710f263522aSJoseph Koshy 		if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3711f263522aSJoseph Koshy 			break;
3712f263522aSJoseph Koshy 
3713f263522aSJoseph Koshy 		if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3714f263522aSJoseph Koshy 			break;
3715f263522aSJoseph Koshy 
3716f263522aSJoseph Koshy 		/*
3717f263522aSJoseph Koshy 		 * The allocated PMC has to be a process virtual PMC,
3718f263522aSJoseph Koshy 		 * i.e., of type MODE_T[CS].  Global PMCs can only be
3719f263522aSJoseph Koshy 		 * read using the PMCREAD operation since they may be
3720f263522aSJoseph Koshy 		 * allocated on a different CPU than the one we could
3721f263522aSJoseph Koshy 		 * be running on at the time of the RDPMC instruction.
3722f263522aSJoseph Koshy 		 *
3723f263522aSJoseph Koshy 		 * The GETMSR operation is not allowed for PMCs that
3724f263522aSJoseph Koshy 		 * are inherited across processes.
3725f263522aSJoseph Koshy 		 */
3726f263522aSJoseph Koshy 
3727f263522aSJoseph Koshy 		if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
3728f263522aSJoseph Koshy 		    (pm->pm_flags & PMC_F_DESCENDANTS)) {
3729f263522aSJoseph Koshy 			error = EINVAL;
3730f263522aSJoseph Koshy 			break;
3731f263522aSJoseph Koshy 		}
3732f263522aSJoseph Koshy 
3733f263522aSJoseph Koshy 		/*
3734f263522aSJoseph Koshy 		 * It only makes sense to use a RDPMC (or its
3735f263522aSJoseph Koshy 		 * equivalent instruction on non-x86 architectures) on
3736f263522aSJoseph Koshy 		 * a process that has allocated and attached a PMC to
3737f263522aSJoseph Koshy 		 * itself.  Conversely the PMC is only allowed to have
3738f263522aSJoseph Koshy 		 * one process attached to it -- its owner.
3739f263522aSJoseph Koshy 		 */
3740f263522aSJoseph Koshy 
3741f263522aSJoseph Koshy 		if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3742f263522aSJoseph Koshy 		    LIST_NEXT(pt, pt_next) != NULL ||
3743f263522aSJoseph Koshy 		    pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3744f263522aSJoseph Koshy 			error = EINVAL;
3745f263522aSJoseph Koshy 			break;
3746f263522aSJoseph Koshy 		}
3747f263522aSJoseph Koshy 
3748f263522aSJoseph Koshy 		ri = PMC_TO_ROWINDEX(pm);
3749e829eb6dSJoseph Koshy 		pcd = pmc_ri_to_classdep(md, ri, &adjri);
3750f263522aSJoseph Koshy 
3751e829eb6dSJoseph Koshy 		/* PMC class has no 'GETMSR' support */
3752e829eb6dSJoseph Koshy 		if (pcd->pcd_get_msr == NULL) {
3753e829eb6dSJoseph Koshy 			error = ENOSYS;
3754e829eb6dSJoseph Koshy 			break;
3755e829eb6dSJoseph Koshy 		}
3756e829eb6dSJoseph Koshy 
3757e829eb6dSJoseph Koshy 		if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
3758f263522aSJoseph Koshy 			break;
3759f263522aSJoseph Koshy 
3760f263522aSJoseph Koshy 		if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3761f263522aSJoseph Koshy 			break;
3762f263522aSJoseph Koshy 
3763f263522aSJoseph Koshy 		/*
3764f263522aSJoseph Koshy 		 * Mark our process as using MSRs.  Update machine
3765f263522aSJoseph Koshy 		 * state using a forced context switch.
3766f263522aSJoseph Koshy 		 */
3767f263522aSJoseph Koshy 
3768f263522aSJoseph Koshy 		pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
3769f263522aSJoseph Koshy 		pmc_force_context_switch();
3770f263522aSJoseph Koshy 
3771f263522aSJoseph Koshy 	}
3772f263522aSJoseph Koshy 	break;
3773f263522aSJoseph Koshy 
3774f263522aSJoseph Koshy 	/*
3775ebccf1e3SJoseph Koshy 	 * Release an allocated PMC
3776ebccf1e3SJoseph Koshy 	 */
3777ebccf1e3SJoseph Koshy 
3778ebccf1e3SJoseph Koshy 	case PMC_OP_PMCRELEASE:
3779ebccf1e3SJoseph Koshy 	{
3780ebccf1e3SJoseph Koshy 		pmc_id_t pmcid;
3781ebccf1e3SJoseph Koshy 		struct pmc *pm;
3782ebccf1e3SJoseph Koshy 		struct pmc_owner *po;
3783ebccf1e3SJoseph Koshy 		struct pmc_op_simple sp;
3784ebccf1e3SJoseph Koshy 
3785ebccf1e3SJoseph Koshy 		/*
3786ebccf1e3SJoseph Koshy 		 * Find PMC pointer for the named PMC.
3787ebccf1e3SJoseph Koshy 		 *
3788ebccf1e3SJoseph Koshy 		 * Use pmc_release_pmc_descriptor() to switch off the
3789ebccf1e3SJoseph Koshy 		 * PMC, remove all its target threads, and remove the
3790ebccf1e3SJoseph Koshy 		 * PMC from its owner's list.
3791ebccf1e3SJoseph Koshy 		 *
3792ebccf1e3SJoseph Koshy 		 * Remove the owner record if this is the last PMC
3793ebccf1e3SJoseph Koshy 		 * owned.
3794ebccf1e3SJoseph Koshy 		 *
3795ebccf1e3SJoseph Koshy 		 * Free up space.
3796ebccf1e3SJoseph Koshy 		 */
3797ebccf1e3SJoseph Koshy 
3798ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3799ebccf1e3SJoseph Koshy 			break;
3800ebccf1e3SJoseph Koshy 
3801ebccf1e3SJoseph Koshy 		pmcid = sp.pm_pmcid;
3802ebccf1e3SJoseph Koshy 
3803ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3804ebccf1e3SJoseph Koshy 			break;
3805ebccf1e3SJoseph Koshy 
3806ebccf1e3SJoseph Koshy 		po = pm->pm_owner;
3807ebccf1e3SJoseph Koshy 		pmc_release_pmc_descriptor(pm);
3808ebccf1e3SJoseph Koshy 		pmc_maybe_remove_owner(po);
380906796b67SMark Johnston 		pmc_destroy_pmc_descriptor(pm);
3810ebccf1e3SJoseph Koshy 	}
3811ebccf1e3SJoseph Koshy 	break;
3812ebccf1e3SJoseph Koshy 
3813ebccf1e3SJoseph Koshy 
3814ebccf1e3SJoseph Koshy 	/*
3815ebccf1e3SJoseph Koshy 	 * Read and/or write a PMC.
3816ebccf1e3SJoseph Koshy 	 */
3817ebccf1e3SJoseph Koshy 
3818ebccf1e3SJoseph Koshy 	case PMC_OP_PMCRW:
3819ebccf1e3SJoseph Koshy 	{
3820e829eb6dSJoseph Koshy 		int adjri;
3821ebccf1e3SJoseph Koshy 		struct pmc *pm;
3822e829eb6dSJoseph Koshy 		uint32_t cpu, ri;
3823ebccf1e3SJoseph Koshy 		pmc_value_t oldvalue;
3824e829eb6dSJoseph Koshy 		struct pmc_binding pb;
3825e829eb6dSJoseph Koshy 		struct pmc_op_pmcrw prw;
3826e829eb6dSJoseph Koshy 		struct pmc_classdep *pcd;
3827e829eb6dSJoseph Koshy 		struct pmc_op_pmcrw *pprw;
3828ebccf1e3SJoseph Koshy 
3829ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
3830ebccf1e3SJoseph Koshy 
3831ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
3832ebccf1e3SJoseph Koshy 			break;
3833ebccf1e3SJoseph Koshy 
3834869f4a8bSWarner Losh 		ri = 0;
38354a3690dfSJohn Baldwin 		PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
3836ebccf1e3SJoseph Koshy 		    prw.pm_flags);
3837ebccf1e3SJoseph Koshy 
3838ebccf1e3SJoseph Koshy 		/* must have at least one flag set */
3839ebccf1e3SJoseph Koshy 		if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
3840ebccf1e3SJoseph Koshy 			error = EINVAL;
3841ebccf1e3SJoseph Koshy 			break;
3842ebccf1e3SJoseph Koshy 		}
3843ebccf1e3SJoseph Koshy 
3844ebccf1e3SJoseph Koshy 		/* locate pmc descriptor */
3845ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
3846ebccf1e3SJoseph Koshy 			break;
3847ebccf1e3SJoseph Koshy 
3848ebccf1e3SJoseph Koshy 		/* Can't read a PMC that hasn't been started. */
3849ebccf1e3SJoseph Koshy 		if (pm->pm_state != PMC_STATE_ALLOCATED &&
3850ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_STOPPED &&
3851ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_RUNNING) {
3852ebccf1e3SJoseph Koshy 			error = EINVAL;
3853ebccf1e3SJoseph Koshy 			break;
3854ebccf1e3SJoseph Koshy 		}
3855ebccf1e3SJoseph Koshy 
3856ebccf1e3SJoseph Koshy 		/* writing a new value is allowed only for 'STOPPED' pmcs */
3857ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING &&
3858ebccf1e3SJoseph Koshy 		    (prw.pm_flags & PMC_F_NEWVALUE)) {
3859ebccf1e3SJoseph Koshy 			error = EBUSY;
3860ebccf1e3SJoseph Koshy 			break;
3861ebccf1e3SJoseph Koshy 		}
3862ebccf1e3SJoseph Koshy 
3863c5153e19SJoseph Koshy 		if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
3864ebccf1e3SJoseph Koshy 
3865c5153e19SJoseph Koshy 			/*
3866c5153e19SJoseph Koshy 			 * If this PMC is attached to its owner (i.e.,
3867c5153e19SJoseph Koshy 			 * the process requesting this operation) and
3868c5153e19SJoseph Koshy 			 * is running, then attempt to get an
3869c5153e19SJoseph Koshy 			 * upto-date reading from hardware for a READ.
3870c5153e19SJoseph Koshy 			 * Writes are only allowed when the PMC is
3871c5153e19SJoseph Koshy 			 * stopped, so only update the saved value
3872c5153e19SJoseph Koshy 			 * field.
3873c5153e19SJoseph Koshy 			 *
3874c5153e19SJoseph Koshy 			 * If the PMC is not running, or is not
3875c5153e19SJoseph Koshy 			 * attached to its owner, read/write to the
3876c5153e19SJoseph Koshy 			 * savedvalue field.
3877c5153e19SJoseph Koshy 			 */
3878c5153e19SJoseph Koshy 
3879c5153e19SJoseph Koshy 			ri = PMC_TO_ROWINDEX(pm);
3880e829eb6dSJoseph Koshy 			pcd = pmc_ri_to_classdep(md, ri, &adjri);
3881c5153e19SJoseph Koshy 
3882ebccf1e3SJoseph Koshy 			mtx_pool_lock_spin(pmc_mtxpool, pm);
3883c5153e19SJoseph Koshy 			cpu = curthread->td_oncpu;
3884c5153e19SJoseph Koshy 
3885c5153e19SJoseph Koshy 			if (prw.pm_flags & PMC_F_OLDVALUE) {
3886c5153e19SJoseph Koshy 				if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3887c5153e19SJoseph Koshy 				    (pm->pm_state == PMC_STATE_RUNNING))
3888e829eb6dSJoseph Koshy 					error = (*pcd->pcd_read_pmc)(cpu, adjri,
3889c5153e19SJoseph Koshy 					    &oldvalue);
3890c5153e19SJoseph Koshy 				else
3891ebccf1e3SJoseph Koshy 					oldvalue = pm->pm_gv.pm_savedvalue;
3892c5153e19SJoseph Koshy 			}
3893ebccf1e3SJoseph Koshy 			if (prw.pm_flags & PMC_F_NEWVALUE)
3894ebccf1e3SJoseph Koshy 				pm->pm_gv.pm_savedvalue = prw.pm_value;
3895c5153e19SJoseph Koshy 
3896ebccf1e3SJoseph Koshy 			mtx_pool_unlock_spin(pmc_mtxpool, pm);
3897ebccf1e3SJoseph Koshy 
3898ebccf1e3SJoseph Koshy 		} else { /* System mode PMCs */
3899c5153e19SJoseph Koshy 			cpu = PMC_TO_CPU(pm);
3900c5153e19SJoseph Koshy 			ri  = PMC_TO_ROWINDEX(pm);
3901e829eb6dSJoseph Koshy 			pcd = pmc_ri_to_classdep(md, ri, &adjri);
3902ebccf1e3SJoseph Koshy 
3903122ccdc1SJoseph Koshy 			if (!pmc_cpu_is_active(cpu)) {
3904ebccf1e3SJoseph Koshy 				error = ENXIO;
3905ebccf1e3SJoseph Koshy 				break;
3906ebccf1e3SJoseph Koshy 			}
3907ebccf1e3SJoseph Koshy 
3908ebccf1e3SJoseph Koshy 			/* move this thread to CPU 'cpu' */
3909ebccf1e3SJoseph Koshy 			pmc_save_cpu_binding(&pb);
3910ebccf1e3SJoseph Koshy 			pmc_select_cpu(cpu);
3911ebccf1e3SJoseph Koshy 
3912c5153e19SJoseph Koshy 			critical_enter();
3913ebccf1e3SJoseph Koshy 			/* save old value */
3914ebccf1e3SJoseph Koshy 			if (prw.pm_flags & PMC_F_OLDVALUE)
3915e829eb6dSJoseph Koshy 				if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
3916ebccf1e3SJoseph Koshy 					 &oldvalue)))
3917ebccf1e3SJoseph Koshy 					goto error;
3918ebccf1e3SJoseph Koshy 			/* write out new value */
3919ebccf1e3SJoseph Koshy 			if (prw.pm_flags & PMC_F_NEWVALUE)
3920e829eb6dSJoseph Koshy 				error = (*pcd->pcd_write_pmc)(cpu, adjri,
3921ebccf1e3SJoseph Koshy 				    prw.pm_value);
3922ebccf1e3SJoseph Koshy 		error:
3923c5153e19SJoseph Koshy 			critical_exit();
3924ebccf1e3SJoseph Koshy 			pmc_restore_cpu_binding(&pb);
3925ebccf1e3SJoseph Koshy 			if (error)
3926ebccf1e3SJoseph Koshy 				break;
3927ebccf1e3SJoseph Koshy 		}
3928ebccf1e3SJoseph Koshy 
3929ebccf1e3SJoseph Koshy 		pprw = (struct pmc_op_pmcrw *) arg;
3930ebccf1e3SJoseph Koshy 
3931680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
3932ebccf1e3SJoseph Koshy 		if (prw.pm_flags & PMC_F_NEWVALUE)
39334a3690dfSJohn Baldwin 			PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
3934ebccf1e3SJoseph Koshy 			    ri, prw.pm_value, oldvalue);
39359d0a2685SJoseph Koshy 		else if (prw.pm_flags & PMC_F_OLDVALUE)
39364a3690dfSJohn Baldwin 			PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
3937ebccf1e3SJoseph Koshy #endif
3938ebccf1e3SJoseph Koshy 
3939ebccf1e3SJoseph Koshy 		/* return old value if requested */
3940ebccf1e3SJoseph Koshy 		if (prw.pm_flags & PMC_F_OLDVALUE)
3941ebccf1e3SJoseph Koshy 			if ((error = copyout(&oldvalue, &pprw->pm_value,
3942ebccf1e3SJoseph Koshy 				 sizeof(prw.pm_value))))
3943ebccf1e3SJoseph Koshy 				break;
3944ebccf1e3SJoseph Koshy 
3945ebccf1e3SJoseph Koshy 	}
3946ebccf1e3SJoseph Koshy 	break;
3947ebccf1e3SJoseph Koshy 
3948ebccf1e3SJoseph Koshy 
3949ebccf1e3SJoseph Koshy 	/*
3950ebccf1e3SJoseph Koshy 	 * Set the sampling rate for a sampling mode PMC and the
3951ebccf1e3SJoseph Koshy 	 * initial count for a counting mode PMC.
3952ebccf1e3SJoseph Koshy 	 */
3953ebccf1e3SJoseph Koshy 
3954ebccf1e3SJoseph Koshy 	case PMC_OP_PMCSETCOUNT:
3955ebccf1e3SJoseph Koshy 	{
3956ebccf1e3SJoseph Koshy 		struct pmc *pm;
3957ebccf1e3SJoseph Koshy 		struct pmc_op_pmcsetcount sc;
3958ebccf1e3SJoseph Koshy 
3959ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
3960ebccf1e3SJoseph Koshy 
3961ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
3962ebccf1e3SJoseph Koshy 			break;
3963ebccf1e3SJoseph Koshy 
3964ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
3965ebccf1e3SJoseph Koshy 			break;
3966ebccf1e3SJoseph Koshy 
3967ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING) {
3968ebccf1e3SJoseph Koshy 			error = EBUSY;
3969ebccf1e3SJoseph Koshy 			break;
3970ebccf1e3SJoseph Koshy 		}
3971ebccf1e3SJoseph Koshy 
3972c5153e19SJoseph Koshy 		if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3973ebccf1e3SJoseph Koshy 			pm->pm_sc.pm_reloadcount = sc.pm_count;
3974ebccf1e3SJoseph Koshy 		else
3975ebccf1e3SJoseph Koshy 			pm->pm_sc.pm_initial = sc.pm_count;
3976ebccf1e3SJoseph Koshy 	}
3977ebccf1e3SJoseph Koshy 	break;
3978ebccf1e3SJoseph Koshy 
3979ebccf1e3SJoseph Koshy 
3980ebccf1e3SJoseph Koshy 	/*
3981ebccf1e3SJoseph Koshy 	 * Start a PMC.
3982ebccf1e3SJoseph Koshy 	 */
3983ebccf1e3SJoseph Koshy 
3984ebccf1e3SJoseph Koshy 	case PMC_OP_PMCSTART:
3985ebccf1e3SJoseph Koshy 	{
3986ebccf1e3SJoseph Koshy 		pmc_id_t pmcid;
3987ebccf1e3SJoseph Koshy 		struct pmc *pm;
3988ebccf1e3SJoseph Koshy 		struct pmc_op_simple sp;
3989ebccf1e3SJoseph Koshy 
3990ebccf1e3SJoseph Koshy 		sx_assert(&pmc_sx, SX_XLOCKED);
3991ebccf1e3SJoseph Koshy 
3992ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3993ebccf1e3SJoseph Koshy 			break;
3994ebccf1e3SJoseph Koshy 
3995ebccf1e3SJoseph Koshy 		pmcid = sp.pm_pmcid;
3996ebccf1e3SJoseph Koshy 
3997ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3998ebccf1e3SJoseph Koshy 			break;
3999ebccf1e3SJoseph Koshy 
4000c5153e19SJoseph Koshy 		KASSERT(pmcid == pm->pm_id,
4001c5153e19SJoseph Koshy 		    ("[pmc,%d] pmcid %x != id %x", __LINE__,
4002c5153e19SJoseph Koshy 			pm->pm_id, pmcid));
4003ebccf1e3SJoseph Koshy 
4004ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
4005ebccf1e3SJoseph Koshy 			break;
4006ebccf1e3SJoseph Koshy 		else if (pm->pm_state != PMC_STATE_STOPPED &&
4007ebccf1e3SJoseph Koshy 		    pm->pm_state != PMC_STATE_ALLOCATED) {
4008ebccf1e3SJoseph Koshy 			error = EINVAL;
4009ebccf1e3SJoseph Koshy 			break;
4010ebccf1e3SJoseph Koshy 		}
4011ebccf1e3SJoseph Koshy 
4012ebccf1e3SJoseph Koshy 		error = pmc_start(pm);
4013ebccf1e3SJoseph Koshy 	}
4014ebccf1e3SJoseph Koshy 	break;
4015ebccf1e3SJoseph Koshy 
4016ebccf1e3SJoseph Koshy 
4017ebccf1e3SJoseph Koshy 	/*
4018ebccf1e3SJoseph Koshy 	 * Stop a PMC.
4019ebccf1e3SJoseph Koshy 	 */
4020ebccf1e3SJoseph Koshy 
4021ebccf1e3SJoseph Koshy 	case PMC_OP_PMCSTOP:
4022ebccf1e3SJoseph Koshy 	{
4023ebccf1e3SJoseph Koshy 		pmc_id_t pmcid;
4024ebccf1e3SJoseph Koshy 		struct pmc *pm;
4025ebccf1e3SJoseph Koshy 		struct pmc_op_simple sp;
4026ebccf1e3SJoseph Koshy 
4027ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
4028ebccf1e3SJoseph Koshy 
4029ebccf1e3SJoseph Koshy 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4030ebccf1e3SJoseph Koshy 			break;
4031ebccf1e3SJoseph Koshy 
4032ebccf1e3SJoseph Koshy 		pmcid = sp.pm_pmcid;
4033ebccf1e3SJoseph Koshy 
4034ebccf1e3SJoseph Koshy 		/*
4035ebccf1e3SJoseph Koshy 		 * Mark the PMC as inactive and invoke the MD stop
4036ebccf1e3SJoseph Koshy 		 * routines if needed.
4037ebccf1e3SJoseph Koshy 		 */
4038ebccf1e3SJoseph Koshy 
4039ebccf1e3SJoseph Koshy 		if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4040ebccf1e3SJoseph Koshy 			break;
4041ebccf1e3SJoseph Koshy 
4042c5153e19SJoseph Koshy 		KASSERT(pmcid == pm->pm_id,
4043c5153e19SJoseph Koshy 		    ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
4044c5153e19SJoseph Koshy 			pm->pm_id, pmcid));
4045ebccf1e3SJoseph Koshy 
4046ebccf1e3SJoseph Koshy 		if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
4047ebccf1e3SJoseph Koshy 			break;
4048ebccf1e3SJoseph Koshy 		else if (pm->pm_state != PMC_STATE_RUNNING) {
4049ebccf1e3SJoseph Koshy 			error = EINVAL;
4050ebccf1e3SJoseph Koshy 			break;
4051ebccf1e3SJoseph Koshy 		}
4052ebccf1e3SJoseph Koshy 
4053ebccf1e3SJoseph Koshy 		error = pmc_stop(pm);
4054ebccf1e3SJoseph Koshy 	}
4055ebccf1e3SJoseph Koshy 	break;
4056ebccf1e3SJoseph Koshy 
4057ebccf1e3SJoseph Koshy 
4058ebccf1e3SJoseph Koshy 	/*
4059fbf1556dSJoseph Koshy 	 * Write a user supplied value to the log file.
4060ebccf1e3SJoseph Koshy 	 */
4061ebccf1e3SJoseph Koshy 
4062ebccf1e3SJoseph Koshy 	case PMC_OP_WRITELOG:
4063ebccf1e3SJoseph Koshy 	{
4064f263522aSJoseph Koshy 		struct pmc_op_writelog wl;
4065f263522aSJoseph Koshy 		struct pmc_owner *po;
4066ebccf1e3SJoseph Koshy 
4067ebccf1e3SJoseph Koshy 		PMC_DOWNGRADE_SX();
4068ebccf1e3SJoseph Koshy 
4069f263522aSJoseph Koshy 		if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
4070ebccf1e3SJoseph Koshy 			break;
4071ebccf1e3SJoseph Koshy 
4072f263522aSJoseph Koshy 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
4073ebccf1e3SJoseph Koshy 			error = EINVAL;
4074ebccf1e3SJoseph Koshy 			break;
4075ebccf1e3SJoseph Koshy 		}
4076ebccf1e3SJoseph Koshy 
4077f263522aSJoseph Koshy 		if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
4078c5153e19SJoseph Koshy 			error = EINVAL;
4079c5153e19SJoseph Koshy 			break;
4080c5153e19SJoseph Koshy 		}
4081c5153e19SJoseph Koshy 
4082f263522aSJoseph Koshy 		error = pmclog_process_userlog(po, &wl);
4083ebccf1e3SJoseph Koshy 	}
4084ebccf1e3SJoseph Koshy 	break;
4085f263522aSJoseph Koshy 
4086ebccf1e3SJoseph Koshy 
4087ebccf1e3SJoseph Koshy 	default:
4088ebccf1e3SJoseph Koshy 		error = EINVAL;
4089ebccf1e3SJoseph Koshy 		break;
4090ebccf1e3SJoseph Koshy 	}
4091ebccf1e3SJoseph Koshy 
4092ebccf1e3SJoseph Koshy 	if (is_sx_downgraded)
4093ebccf1e3SJoseph Koshy 		sx_sunlock(&pmc_sx);
4094ebccf1e3SJoseph Koshy 	else
4095ebccf1e3SJoseph Koshy 		sx_xunlock(&pmc_sx);
409620b555e1SKonstantin Belousov done_syscall:
4097ebccf1e3SJoseph Koshy 	if (error)
4098e6b475e0SMatt Macy 		counter_u64_add(pmc_stats.pm_syscall_errors, 1);
4099ebccf1e3SJoseph Koshy 
4100cf9ef806SKonstantin Belousov 	return (error);
4101ebccf1e3SJoseph Koshy }
4102ebccf1e3SJoseph Koshy 
4103ebccf1e3SJoseph Koshy /*
4104ebccf1e3SJoseph Koshy  * Helper functions
4105ebccf1e3SJoseph Koshy  */
4106ebccf1e3SJoseph Koshy 
4107f263522aSJoseph Koshy 
4108ebccf1e3SJoseph Koshy /*
4109d07f36b0SJoseph Koshy  * Mark the thread as needing callchain capture and post an AST.  The
4110d07f36b0SJoseph Koshy  * actual callchain capture will be done in a context where it is safe
4111d07f36b0SJoseph Koshy  * to take page faults.
4112d07f36b0SJoseph Koshy  */
4113d07f36b0SJoseph Koshy 
4114d07f36b0SJoseph Koshy static void
41156fe00c78SJoseph Koshy pmc_post_callchain_callback(void)
4116d07f36b0SJoseph Koshy {
4117d07f36b0SJoseph Koshy 	struct thread *td;
4118d07f36b0SJoseph Koshy 
4119d07f36b0SJoseph Koshy 	td = curthread;
4120d07f36b0SJoseph Koshy 
41210e031404SFabien Thomas 	/*
41220e031404SFabien Thomas 	 * If there is multiple PMCs for the same interrupt ignore new post
41230e031404SFabien Thomas 	 */
41240e031404SFabien Thomas 	if (td->td_pflags & TDP_CALLCHAIN)
41250e031404SFabien Thomas 		return;
41266fe00c78SJoseph Koshy 
4127d07f36b0SJoseph Koshy 	/*
41286fe00c78SJoseph Koshy 	 * Mark this thread as needing callchain capture.
41296fe00c78SJoseph Koshy 	 * `td->td_pflags' will be safe to touch because this thread
41306fe00c78SJoseph Koshy 	 * was in user space when it was interrupted.
4131d07f36b0SJoseph Koshy 	 */
4132d07f36b0SJoseph Koshy 	td->td_pflags |= TDP_CALLCHAIN;
4133d07f36b0SJoseph Koshy 
4134d07f36b0SJoseph Koshy 	/*
41356fe00c78SJoseph Koshy 	 * Don't let this thread migrate between CPUs until callchain
41366fe00c78SJoseph Koshy 	 * capture completes.
4137d07f36b0SJoseph Koshy 	 */
41386fe00c78SJoseph Koshy 	sched_pin();
4139d07f36b0SJoseph Koshy 
4140d07f36b0SJoseph Koshy 	return;
4141d07f36b0SJoseph Koshy }
4142d07f36b0SJoseph Koshy 
4143d07f36b0SJoseph Koshy /*
4144f263522aSJoseph Koshy  * Interrupt processing.
4145f263522aSJoseph Koshy  *
4146d07f36b0SJoseph Koshy  * Find a free slot in the per-cpu array of samples and capture the
4147d07f36b0SJoseph Koshy  * current callchain there.  If a sample was successfully added, a bit
4148d07f36b0SJoseph Koshy  * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
4149d07f36b0SJoseph Koshy  * needs to be invoked from the clock handler.
4150f263522aSJoseph Koshy  *
4151f263522aSJoseph Koshy  * This function is meant to be called from an NMI handler.  It cannot
4152f263522aSJoseph Koshy  * use any of the locking primitives supplied by the OS.
4153ebccf1e3SJoseph Koshy  */
4154ebccf1e3SJoseph Koshy 
4155f263522aSJoseph Koshy int
4156f5f9340bSFabien Thomas pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf,
4157d07f36b0SJoseph Koshy     int inuserspace)
4158ebccf1e3SJoseph Koshy {
4159d07f36b0SJoseph Koshy 	int error, callchaindepth;
4160f263522aSJoseph Koshy 	struct thread *td;
4161f263522aSJoseph Koshy 	struct pmc_sample *ps;
4162f263522aSJoseph Koshy 	struct pmc_samplebuffer *psb;
4163ebccf1e3SJoseph Koshy 
4164f263522aSJoseph Koshy 	error = 0;
4165ebccf1e3SJoseph Koshy 
4166d07f36b0SJoseph Koshy 	/*
4167d07f36b0SJoseph Koshy 	 * Allocate space for a sample buffer.
4168d07f36b0SJoseph Koshy 	 */
4169f5f9340bSFabien Thomas 	psb = pmc_pcpu[cpu]->pc_sb[ring];
4170ebccf1e3SJoseph Koshy 
4171f263522aSJoseph Koshy 	ps = psb->ps_write;
4172d07f36b0SJoseph Koshy 	if (ps->ps_nsamples) {	/* in use, reader hasn't caught up */
4173e6b475e0SMatt Macy 		pm->pm_pcpu_state[cpu].pps_stalled = 1;
4174e6b475e0SMatt Macy 		counter_u64_add(pmc_stats.pm_intr_bufferfull, 1);
41754a3690dfSJohn Baldwin 		PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
4176d07f36b0SJoseph Koshy 		    cpu, pm, (void *) tf, inuserspace,
4177f263522aSJoseph Koshy 		    (int) (psb->ps_write - psb->ps_samples),
4178f263522aSJoseph Koshy 		    (int) (psb->ps_read - psb->ps_samples));
4179c66ea2eeSJonathan T. Looney 		callchaindepth = 1;
4180f263522aSJoseph Koshy 		error = ENOMEM;
4181f263522aSJoseph Koshy 		goto done;
4182ebccf1e3SJoseph Koshy 	}
4183ebccf1e3SJoseph Koshy 
4184d07f36b0SJoseph Koshy 
4185d07f36b0SJoseph Koshy 	/* Fill in entry. */
41864a3690dfSJohn Baldwin 	PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
4187d07f36b0SJoseph Koshy 	    (void *) tf, inuserspace,
4188f263522aSJoseph Koshy 	    (int) (psb->ps_write - psb->ps_samples),
4189f263522aSJoseph Koshy 	    (int) (psb->ps_read - psb->ps_samples));
4190f263522aSJoseph Koshy 
4191e6b475e0SMatt Macy 	KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
4192e6b475e0SMatt Macy 	    ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4193e6b475e0SMatt Macy 		 (unsigned long)counter_u64_fetch(pm->pm_runcount)));
41946fe00c78SJoseph Koshy 
4195e6b475e0SMatt Macy 	counter_u64_add(pm->pm_runcount, 1);	/* hold onto PMC */
4196f5f9340bSFabien Thomas 
4197f263522aSJoseph Koshy 	ps->ps_pmc = pm;
4198f263522aSJoseph Koshy 	if ((td = curthread) && td->td_proc)
4199f263522aSJoseph Koshy 		ps->ps_pid = td->td_proc->p_pid;
4200f263522aSJoseph Koshy 	else
4201f263522aSJoseph Koshy 		ps->ps_pid = -1;
4202d07f36b0SJoseph Koshy 	ps->ps_cpu = cpu;
42036fe00c78SJoseph Koshy 	ps->ps_td = td;
4204d07f36b0SJoseph Koshy 	ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
4205d07f36b0SJoseph Koshy 
4206d07f36b0SJoseph Koshy 	callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
4207d07f36b0SJoseph Koshy 	    pmc_callchaindepth : 1;
4208d07f36b0SJoseph Koshy 
4209d07f36b0SJoseph Koshy 	if (callchaindepth == 1)
4210d07f36b0SJoseph Koshy 		ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
4211d07f36b0SJoseph Koshy 	else {
4212d07f36b0SJoseph Koshy 		/*
4213d07f36b0SJoseph Koshy 		 * Kernel stack traversals can be done immediately,
4214d07f36b0SJoseph Koshy 		 * while we defer to an AST for user space traversals.
4215d07f36b0SJoseph Koshy 		 */
4216f5f9340bSFabien Thomas 		if (!inuserspace) {
4217d07f36b0SJoseph Koshy 			callchaindepth =
4218d07f36b0SJoseph Koshy 			    pmc_save_kernel_callchain(ps->ps_pc,
4219d07f36b0SJoseph Koshy 				callchaindepth, tf);
4220f5f9340bSFabien Thomas 		} else {
42216fe00c78SJoseph Koshy 			pmc_post_callchain_callback();
4222d07f36b0SJoseph Koshy 			callchaindepth = PMC_SAMPLE_INUSE;
4223d07f36b0SJoseph Koshy 		}
4224d07f36b0SJoseph Koshy 	}
4225d07f36b0SJoseph Koshy 
4226d07f36b0SJoseph Koshy 	ps->ps_nsamples = callchaindepth;	/* mark entry as in use */
4227f263522aSJoseph Koshy 
4228f263522aSJoseph Koshy 	/* increment write pointer, modulo ring buffer size */
4229f263522aSJoseph Koshy 	ps++;
4230f263522aSJoseph Koshy 	if (ps == psb->ps_fence)
4231f263522aSJoseph Koshy 		psb->ps_write = psb->ps_samples;
4232f263522aSJoseph Koshy 	else
4233f263522aSJoseph Koshy 		psb->ps_write = ps;
4234f263522aSJoseph Koshy 
4235f263522aSJoseph Koshy  done:
4236f263522aSJoseph Koshy 	/* mark CPU as needing processing */
4237c66ea2eeSJonathan T. Looney 	if (callchaindepth != PMC_SAMPLE_INUSE)
4238e6b475e0SMatt Macy 		DPCPU_SET(pmc_sampled, 1);
4239f263522aSJoseph Koshy 
4240d07f36b0SJoseph Koshy 	return (error);
4241d07f36b0SJoseph Koshy }
4242d07f36b0SJoseph Koshy 
4243d07f36b0SJoseph Koshy /*
4244d07f36b0SJoseph Koshy  * Capture a user call chain.  This function will be called from ast()
4245d07f36b0SJoseph Koshy  * before control returns to userland and before the process gets
4246d07f36b0SJoseph Koshy  * rescheduled.
4247d07f36b0SJoseph Koshy  */
4248d07f36b0SJoseph Koshy 
4249d07f36b0SJoseph Koshy static void
4250f5f9340bSFabien Thomas pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
4251d07f36b0SJoseph Koshy {
4252d07f36b0SJoseph Koshy 	struct pmc *pm;
42536fe00c78SJoseph Koshy 	struct thread *td;
4254c66ea2eeSJonathan T. Looney 	struct pmc_sample *ps, *ps_end;
4255d07f36b0SJoseph Koshy 	struct pmc_samplebuffer *psb;
42566fe00c78SJoseph Koshy #ifdef	INVARIANTS
42576fe00c78SJoseph Koshy 	int ncallchains;
42587bc7e3cdSAndrew Turner 	int nfree;
42596fe00c78SJoseph Koshy #endif
42606fe00c78SJoseph Koshy 
4261f5f9340bSFabien Thomas 	psb = pmc_pcpu[cpu]->pc_sb[ring];
42626fe00c78SJoseph Koshy 	td = curthread;
42636fe00c78SJoseph Koshy 
42646fe00c78SJoseph Koshy 	KASSERT(td->td_pflags & TDP_CALLCHAIN,
42656fe00c78SJoseph Koshy 	    ("[pmc,%d] Retrieving callchain for thread that doesn't want it",
42666fe00c78SJoseph Koshy 		__LINE__));
42676fe00c78SJoseph Koshy 
42686fe00c78SJoseph Koshy #ifdef	INVARIANTS
42696fe00c78SJoseph Koshy 	ncallchains = 0;
42707bc7e3cdSAndrew Turner 	nfree = 0;
42716fe00c78SJoseph Koshy #endif
4272d07f36b0SJoseph Koshy 
4273d07f36b0SJoseph Koshy 	/*
4274d07f36b0SJoseph Koshy 	 * Iterate through all deferred callchain requests.
4275c66ea2eeSJonathan T. Looney 	 * Walk from the current read pointer to the current
4276c66ea2eeSJonathan T. Looney 	 * write pointer.
4277d07f36b0SJoseph Koshy 	 */
4278d07f36b0SJoseph Koshy 
4279c66ea2eeSJonathan T. Looney 	ps = psb->ps_read;
4280c66ea2eeSJonathan T. Looney 	ps_end = psb->ps_write;
4281c66ea2eeSJonathan T. Looney 	do {
42827bc7e3cdSAndrew Turner #ifdef	INVARIANTS
42836cb40391SZbigniew Bodek 		if ((ps->ps_pmc == NULL) ||
42846cb40391SZbigniew Bodek 		    (ps->ps_pmc->pm_state != PMC_STATE_RUNNING))
42857bc7e3cdSAndrew Turner 			nfree++;
42867bc7e3cdSAndrew Turner #endif
4287d07f36b0SJoseph Koshy 		if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
4288c66ea2eeSJonathan T. Looney 			goto next;
42896fe00c78SJoseph Koshy 		if (ps->ps_td != td)
4290c66ea2eeSJonathan T. Looney 			goto next;
42916fe00c78SJoseph Koshy 
42926fe00c78SJoseph Koshy 		KASSERT(ps->ps_cpu == cpu,
42936fe00c78SJoseph Koshy 		    ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
42946fe00c78SJoseph Koshy 			ps->ps_cpu, PCPU_GET(cpuid)));
4295d07f36b0SJoseph Koshy 
4296d07f36b0SJoseph Koshy 		pm = ps->ps_pmc;
4297d07f36b0SJoseph Koshy 
4298d07f36b0SJoseph Koshy 		KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
4299d07f36b0SJoseph Koshy 		    ("[pmc,%d] Retrieving callchain for PMC that doesn't "
4300d07f36b0SJoseph Koshy 			"want it", __LINE__));
4301d07f36b0SJoseph Koshy 
4302e6b475e0SMatt Macy 		KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4303e6b475e0SMatt Macy 		    ("[pmc,%d] runcount %ld", __LINE__, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
43046fe00c78SJoseph Koshy 
4305d07f36b0SJoseph Koshy 		/*
4306d07f36b0SJoseph Koshy 		 * Retrieve the callchain and mark the sample buffer
4307d07f36b0SJoseph Koshy 		 * as 'processable' by the timer tick sweep code.
4308d07f36b0SJoseph Koshy 		 */
4309d07f36b0SJoseph Koshy 		ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc,
4310d07f36b0SJoseph Koshy 		    pmc_callchaindepth, tf);
43116fe00c78SJoseph Koshy 
43126fe00c78SJoseph Koshy #ifdef	INVARIANTS
43136fe00c78SJoseph Koshy 		ncallchains++;
43146fe00c78SJoseph Koshy #endif
4315c66ea2eeSJonathan T. Looney 
4316c66ea2eeSJonathan T. Looney next:
4317c66ea2eeSJonathan T. Looney 		/* increment the pointer, modulo sample ring size */
4318c66ea2eeSJonathan T. Looney 		if (++ps == psb->ps_fence)
4319c66ea2eeSJonathan T. Looney 			ps = psb->ps_samples;
4320c66ea2eeSJonathan T. Looney 	} while (ps != ps_end);
4321d07f36b0SJoseph Koshy 
43226cb40391SZbigniew Bodek #ifdef	INVARIANTS
43237bc7e3cdSAndrew Turner 	KASSERT(ncallchains > 0 || nfree > 0,
43246fe00c78SJoseph Koshy 	    ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
43256fe00c78SJoseph Koshy 		cpu));
43266cb40391SZbigniew Bodek #endif
43276fe00c78SJoseph Koshy 
43285584e917SAttilio Rao 	KASSERT(td->td_pinned == 1,
4329f5f9340bSFabien Thomas 	    ("[pmc,%d] invalid td_pinned value", __LINE__));
4330f5f9340bSFabien Thomas 	sched_unpin();	/* Can migrate safely now. */
4331f5f9340bSFabien Thomas 
4332c66ea2eeSJonathan T. Looney 	/* mark CPU as needing processing */
4333e6b475e0SMatt Macy 	DPCPU_SET(pmc_sampled, 1);
4334f263522aSJoseph Koshy }
4335f263522aSJoseph Koshy 
4336ebccf1e3SJoseph Koshy /*
4337f263522aSJoseph Koshy  * Process saved PC samples.
4338ebccf1e3SJoseph Koshy  */
4339ebccf1e3SJoseph Koshy 
4340ebccf1e3SJoseph Koshy static void
4341f5f9340bSFabien Thomas pmc_process_samples(int cpu, int ring)
4342ebccf1e3SJoseph Koshy {
4343f263522aSJoseph Koshy 	struct pmc *pm;
43440cfab8ddSJoseph Koshy 	int adjri, n;
4345f263522aSJoseph Koshy 	struct thread *td;
4346f263522aSJoseph Koshy 	struct pmc_owner *po;
4347f263522aSJoseph Koshy 	struct pmc_sample *ps;
4348e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
4349f263522aSJoseph Koshy 	struct pmc_samplebuffer *psb;
4350ebccf1e3SJoseph Koshy 
4351f263522aSJoseph Koshy 	KASSERT(PCPU_GET(cpuid) == cpu,
4352f263522aSJoseph Koshy 	    ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
4353f263522aSJoseph Koshy 		PCPU_GET(cpuid), cpu));
4354ebccf1e3SJoseph Koshy 
4355f5f9340bSFabien Thomas 	psb = pmc_pcpu[cpu]->pc_sb[ring];
4356f263522aSJoseph Koshy 
4357f263522aSJoseph Koshy 	for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
4358f263522aSJoseph Koshy 
4359f263522aSJoseph Koshy 		ps = psb->ps_read;
4360d07f36b0SJoseph Koshy 		if (ps->ps_nsamples == PMC_SAMPLE_FREE)
4361f263522aSJoseph Koshy 			break;
4362f263522aSJoseph Koshy 
4363f263522aSJoseph Koshy 		pm = ps->ps_pmc;
43646fe00c78SJoseph Koshy 
4365e6b475e0SMatt Macy 		KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4366e6b475e0SMatt Macy 		    ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4367e6b475e0SMatt Macy 			 (unsigned long)counter_u64_fetch(pm->pm_runcount)));
43686fe00c78SJoseph Koshy 
4369f263522aSJoseph Koshy 		po = pm->pm_owner;
4370f263522aSJoseph Koshy 
4371f263522aSJoseph Koshy 		KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
4372f263522aSJoseph Koshy 		    ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
4373f263522aSJoseph Koshy 			pm, PMC_TO_MODE(pm)));
4374f263522aSJoseph Koshy 
4375f263522aSJoseph Koshy 		/* Ignore PMCs that have been switched off */
4376f263522aSJoseph Koshy 		if (pm->pm_state != PMC_STATE_RUNNING)
4377f263522aSJoseph Koshy 			goto entrydone;
4378f263522aSJoseph Koshy 
4379f5f9340bSFabien Thomas 		/* If there is a pending AST wait for completion */
4380f5f9340bSFabien Thomas 		if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
4381f5f9340bSFabien Thomas 			/* Need a rescan at a later time. */
4382e6b475e0SMatt Macy 			DPCPU_SET(pmc_sampled, 1);
4383f5f9340bSFabien Thomas 			break;
4384f5f9340bSFabien Thomas 		}
4385f5f9340bSFabien Thomas 
43864a3690dfSJohn Baldwin 		PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
4387d07f36b0SJoseph Koshy 		    pm, ps->ps_nsamples, ps->ps_flags,
4388f263522aSJoseph Koshy 		    (int) (psb->ps_write - psb->ps_samples),
4389f263522aSJoseph Koshy 		    (int) (psb->ps_read - psb->ps_samples));
4390f263522aSJoseph Koshy 
4391f263522aSJoseph Koshy 		/*
4392f263522aSJoseph Koshy 		 * If this is a process-mode PMC that is attached to
4393f263522aSJoseph Koshy 		 * its owner, and if the PC is in user mode, update
4394f263522aSJoseph Koshy 		 * profiling statistics like timer-based profiling
4395f263522aSJoseph Koshy 		 * would have done.
4396f263522aSJoseph Koshy 		 */
4397f263522aSJoseph Koshy 		if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
4398d07f36b0SJoseph Koshy 			if (ps->ps_flags & PMC_CC_F_USERSPACE) {
4399f263522aSJoseph Koshy 				td = FIRST_THREAD_IN_PROC(po->po_owner);
4400d07f36b0SJoseph Koshy 				addupc_intr(td, ps->ps_pc[0], 1);
4401f263522aSJoseph Koshy 			}
4402f263522aSJoseph Koshy 			goto entrydone;
4403f263522aSJoseph Koshy 		}
4404f263522aSJoseph Koshy 
4405f263522aSJoseph Koshy 		/*
4406f263522aSJoseph Koshy 		 * Otherwise, this is either a sampling mode PMC that
4407f263522aSJoseph Koshy 		 * is attached to a different process than its owner,
4408f263522aSJoseph Koshy 		 * or a system-wide sampling PMC.  Dispatch a log
4409f263522aSJoseph Koshy 		 * entry to the PMC's owner process.
4410f263522aSJoseph Koshy 		 */
4411d07f36b0SJoseph Koshy 		pmclog_process_callchain(pm, ps);
4412f263522aSJoseph Koshy 
4413f263522aSJoseph Koshy 	entrydone:
4414d07f36b0SJoseph Koshy 		ps->ps_nsamples = 0; /* mark entry as free */
4415e6b475e0SMatt Macy 		counter_u64_add(pm->pm_runcount, -1);
4416f263522aSJoseph Koshy 
4417f263522aSJoseph Koshy 		/* increment read pointer, modulo sample size */
4418f263522aSJoseph Koshy 		if (++ps == psb->ps_fence)
4419f263522aSJoseph Koshy 			psb->ps_read = psb->ps_samples;
4420f263522aSJoseph Koshy 		else
4421f263522aSJoseph Koshy 			psb->ps_read = ps;
4422f263522aSJoseph Koshy 	}
4423f263522aSJoseph Koshy 
4424e6b475e0SMatt Macy 	counter_u64_add(pmc_stats.pm_log_sweeps, 1);
4425f263522aSJoseph Koshy 
4426f263522aSJoseph Koshy 	/* Do not re-enable stalled PMCs if we failed to process any samples */
4427f263522aSJoseph Koshy 	if (n == 0)
4428ebccf1e3SJoseph Koshy 		return;
4429f263522aSJoseph Koshy 
4430f263522aSJoseph Koshy 	/*
4431f263522aSJoseph Koshy 	 * Restart any stalled sampling PMCs on this CPU.
4432f263522aSJoseph Koshy 	 *
4433fbf1556dSJoseph Koshy 	 * If the NMI handler sets the pm_stalled field of a PMC after
4434fbf1556dSJoseph Koshy 	 * the check below, we'll end up processing the stalled PMC at
4435fbf1556dSJoseph Koshy 	 * the next hardclock tick.
4436f263522aSJoseph Koshy 	 */
4437f263522aSJoseph Koshy 	for (n = 0; n < md->pmd_npmc; n++) {
4438e829eb6dSJoseph Koshy 		pcd = pmc_ri_to_classdep(md, n, &adjri);
4439e829eb6dSJoseph Koshy 		KASSERT(pcd != NULL,
4440e829eb6dSJoseph Koshy 		    ("[pmc,%d] null pcd ri=%d", __LINE__, n));
4441e829eb6dSJoseph Koshy 		(void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
4442e829eb6dSJoseph Koshy 
4443f263522aSJoseph Koshy 		if (pm == NULL ||			 /* !cfg'ed */
4444f263522aSJoseph Koshy 		    pm->pm_state != PMC_STATE_RUNNING || /* !active */
4445f263522aSJoseph Koshy 		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
4446e6b475e0SMatt Macy 			!pm->pm_pcpu_state[cpu].pps_cpustate  || /* !desired */
4447e6b475e0SMatt Macy 		    !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */
4448f263522aSJoseph Koshy 			continue;
4449f263522aSJoseph Koshy 
4450e6b475e0SMatt Macy 		pm->pm_pcpu_state[cpu].pps_stalled = 0;
4451e829eb6dSJoseph Koshy 		(*pcd->pcd_start_pmc)(cpu, adjri);
4452f263522aSJoseph Koshy 	}
4453ebccf1e3SJoseph Koshy }
4454ebccf1e3SJoseph Koshy 
4455ebccf1e3SJoseph Koshy /*
4456ebccf1e3SJoseph Koshy  * Event handlers.
4457ebccf1e3SJoseph Koshy  */
4458ebccf1e3SJoseph Koshy 
4459ebccf1e3SJoseph Koshy /*
4460ebccf1e3SJoseph Koshy  * Handle a process exit.
4461ebccf1e3SJoseph Koshy  *
4462f263522aSJoseph Koshy  * Remove this process from all hash tables.  If this process
4463f263522aSJoseph Koshy  * owned any PMCs, turn off those PMCs and deallocate them,
4464f263522aSJoseph Koshy  * removing any associations with target processes.
4465f263522aSJoseph Koshy  *
4466f263522aSJoseph Koshy  * This function will be called by the last 'thread' of a
4467f263522aSJoseph Koshy  * process.
4468f263522aSJoseph Koshy  *
4469ebccf1e3SJoseph Koshy  * XXX This eventhandler gets called early in the exit process.
4470ebccf1e3SJoseph Koshy  * Consider using a 'hook' invocation from thread_exit() or equivalent
4471ebccf1e3SJoseph Koshy  * spot.  Another negative is that kse_exit doesn't seem to call
4472ebccf1e3SJoseph Koshy  * exit1() [??].
4473f263522aSJoseph Koshy  *
4474ebccf1e3SJoseph Koshy  */
4475ebccf1e3SJoseph Koshy 
4476ebccf1e3SJoseph Koshy static void
4477ebccf1e3SJoseph Koshy pmc_process_exit(void *arg __unused, struct proc *p)
4478ebccf1e3SJoseph Koshy {
4479f263522aSJoseph Koshy 	struct pmc *pm;
4480e829eb6dSJoseph Koshy 	int adjri, cpu;
4481e829eb6dSJoseph Koshy 	unsigned int ri;
4482e829eb6dSJoseph Koshy 	int is_using_hwpmcs;
4483f263522aSJoseph Koshy 	struct pmc_owner *po;
4484e829eb6dSJoseph Koshy 	struct pmc_process *pp;
4485e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
4486f263522aSJoseph Koshy 	pmc_value_t newvalue, tmp;
4487ebccf1e3SJoseph Koshy 
4488ebccf1e3SJoseph Koshy 	PROC_LOCK(p);
4489ebccf1e3SJoseph Koshy 	is_using_hwpmcs = p->p_flag & P_HWPMC;
4490ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p);
4491ebccf1e3SJoseph Koshy 
4492f263522aSJoseph Koshy 	/*
4493f263522aSJoseph Koshy 	 * Log a sysexit event to all SS PMC owners.
4494f263522aSJoseph Koshy 	 */
4495*0f00315cSMatt Macy 	epoch_enter(global_epoch);
4496*0f00315cSMatt Macy 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
4497f263522aSJoseph Koshy 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
4498f263522aSJoseph Koshy 		    pmclog_process_sysexit(po, p->p_pid);
4499*0f00315cSMatt Macy 	epoch_exit(global_epoch);
4500f263522aSJoseph Koshy 
4501f263522aSJoseph Koshy 	if (!is_using_hwpmcs)
4502f263522aSJoseph Koshy 		return;
4503f263522aSJoseph Koshy 
4504f263522aSJoseph Koshy 	PMC_GET_SX_XLOCK();
45054a3690dfSJohn Baldwin 	PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
4506ebccf1e3SJoseph Koshy 	    p->p_comm);
4507ebccf1e3SJoseph Koshy 
4508f263522aSJoseph Koshy 	/*
4509f263522aSJoseph Koshy 	 * Since this code is invoked by the last thread in an exiting
4510f263522aSJoseph Koshy 	 * process, we would have context switched IN at some prior
4511f263522aSJoseph Koshy 	 * point.  However, with PREEMPTION, kernel mode context
4512f263522aSJoseph Koshy 	 * switches may happen any time, so we want to disable a
4513b790c193SPedro F. Giffuni 	 * context switch OUT till we get any PMCs targeting this
4514f263522aSJoseph Koshy 	 * process off the hardware.
4515f263522aSJoseph Koshy 	 *
4516f263522aSJoseph Koshy 	 * We also need to atomically remove this process'
4517f263522aSJoseph Koshy 	 * entry from our target process hash table, using
4518f263522aSJoseph Koshy 	 * PMC_FLAG_REMOVE.
4519f263522aSJoseph Koshy 	 */
45204a3690dfSJohn Baldwin 	PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
4521f263522aSJoseph Koshy 	    p->p_comm);
4522f263522aSJoseph Koshy 
4523f263522aSJoseph Koshy 	critical_enter(); /* no preemption */
4524f263522aSJoseph Koshy 
4525f263522aSJoseph Koshy 	cpu = curthread->td_oncpu;
4526f263522aSJoseph Koshy 
4527f263522aSJoseph Koshy 	if ((pp = pmc_find_process_descriptor(p,
4528f263522aSJoseph Koshy 		 PMC_FLAG_REMOVE)) != NULL) {
4529f263522aSJoseph Koshy 
45304a3690dfSJohn Baldwin 		PMCDBG2(PRC,EXT,2,
4531f263522aSJoseph Koshy 		    "process-exit proc=%p pmc-process=%p", p, pp);
4532f263522aSJoseph Koshy 
4533f263522aSJoseph Koshy 		/*
4534f263522aSJoseph Koshy 		 * The exiting process could the target of
4535f263522aSJoseph Koshy 		 * some PMCs which will be running on
4536f263522aSJoseph Koshy 		 * currently executing CPU.
4537f263522aSJoseph Koshy 		 *
4538f263522aSJoseph Koshy 		 * We need to turn these PMCs off like we
4539f263522aSJoseph Koshy 		 * would do at context switch OUT time.
4540f263522aSJoseph Koshy 		 */
4541f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++) {
4542f263522aSJoseph Koshy 
4543f263522aSJoseph Koshy 			/*
4544f263522aSJoseph Koshy 			 * Pick up the pmc pointer from hardware
4545f263522aSJoseph Koshy 			 * state similar to the CSW_OUT code.
4546f263522aSJoseph Koshy 			 */
4547f263522aSJoseph Koshy 			pm = NULL;
4548e829eb6dSJoseph Koshy 
4549e829eb6dSJoseph Koshy 			pcd = pmc_ri_to_classdep(md, ri, &adjri);
4550e829eb6dSJoseph Koshy 
4551e829eb6dSJoseph Koshy 			(void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
4552f263522aSJoseph Koshy 
45534a3690dfSJohn Baldwin 			PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
4554f263522aSJoseph Koshy 
4555f263522aSJoseph Koshy 			if (pm == NULL ||
4556f263522aSJoseph Koshy 			    !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
4557f263522aSJoseph Koshy 				continue;
4558f263522aSJoseph Koshy 
45594a3690dfSJohn Baldwin 			PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
4560f263522aSJoseph Koshy 			    "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
4561f263522aSJoseph Koshy 			    pm, pm->pm_state);
4562f263522aSJoseph Koshy 
4563f263522aSJoseph Koshy 			KASSERT(PMC_TO_ROWINDEX(pm) == ri,
4564f263522aSJoseph Koshy 			    ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
4565f263522aSJoseph Koshy 				__LINE__, PMC_TO_ROWINDEX(pm), ri));
4566f263522aSJoseph Koshy 
4567f263522aSJoseph Koshy 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
4568f263522aSJoseph Koshy 			    ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
4569f263522aSJoseph Koshy 				__LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
4570f263522aSJoseph Koshy 
4571e6b475e0SMatt Macy 			KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4572e6b475e0SMatt Macy 			    ("[pmc,%d] bad runcount ri %d rc %ld",
4573e6b475e0SMatt Macy 				 __LINE__, ri, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4574f263522aSJoseph Koshy 
4575a3924968SJonathan T. Looney 			/*
4576a3924968SJonathan T. Looney 			 * Change desired state, and then stop if not
4577a3924968SJonathan T. Looney 			 * stalled. This two-step dance should avoid
4578a3924968SJonathan T. Looney 			 * race conditions where an interrupt re-enables
4579a3924968SJonathan T. Looney 			 * the PMC after this code has already checked
4580a3924968SJonathan T. Looney 			 * the pm_stalled flag.
4581a3924968SJonathan T. Looney 			 */
4582e6b475e0SMatt Macy 			if (pm->pm_pcpu_state[cpu].pps_cpustate) {
4583e6b475e0SMatt Macy 				pm->pm_pcpu_state[cpu].pps_cpustate = 0;
4584e6b475e0SMatt Macy 				if (!pm->pm_pcpu_state[cpu].pps_stalled) {
4585a3924968SJonathan T. Looney 					(void) pcd->pcd_stop_pmc(cpu, adjri);
4586a3924968SJonathan T. Looney 					pcd->pcd_read_pmc(cpu, adjri,
4587a3924968SJonathan T. Looney 					    &newvalue);
4588f263522aSJoseph Koshy 					tmp = newvalue -
4589f263522aSJoseph Koshy 					    PMC_PCPU_SAVED(cpu,ri);
4590f263522aSJoseph Koshy 
4591f263522aSJoseph Koshy 					mtx_pool_lock_spin(pmc_mtxpool, pm);
4592f263522aSJoseph Koshy 					pm->pm_gv.pm_savedvalue += tmp;
4593f263522aSJoseph Koshy 					pp->pp_pmcs[ri].pp_pmcval += tmp;
4594f263522aSJoseph Koshy 					mtx_pool_unlock_spin(pmc_mtxpool, pm);
4595ebccf1e3SJoseph Koshy 				}
4596a3924968SJonathan T. Looney 			}
4597f263522aSJoseph Koshy 
4598e6b475e0SMatt Macy 			counter_u64_add(pm->pm_runcount, -1);
4599f263522aSJoseph Koshy 
4600e6b475e0SMatt Macy 			KASSERT((int) counter_u64_fetch(pm->pm_runcount) >= 0,
4601f263522aSJoseph Koshy 			    ("[pmc,%d] runcount is %d", __LINE__, ri));
4602f263522aSJoseph Koshy 
4603e829eb6dSJoseph Koshy 			(void) pcd->pcd_config_pmc(cpu, adjri, NULL);
4604f263522aSJoseph Koshy 		}
4605f263522aSJoseph Koshy 
4606f263522aSJoseph Koshy 		/*
4607f263522aSJoseph Koshy 		 * Inform the MD layer of this pseudo "context switch
4608f263522aSJoseph Koshy 		 * out"
4609f263522aSJoseph Koshy 		 */
4610f263522aSJoseph Koshy 		(void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
4611f263522aSJoseph Koshy 
4612f263522aSJoseph Koshy 		critical_exit(); /* ok to be pre-empted now */
4613f263522aSJoseph Koshy 
4614f263522aSJoseph Koshy 		/*
4615f263522aSJoseph Koshy 		 * Unlink this process from the PMCs that are
4616b790c193SPedro F. Giffuni 		 * targeting it.  This will send a signal to
4617f263522aSJoseph Koshy 		 * all PMC owner's whose PMCs are orphaned.
4618f263522aSJoseph Koshy 		 *
4619f263522aSJoseph Koshy 		 * Log PMC value at exit time if requested.
4620f263522aSJoseph Koshy 		 */
4621f263522aSJoseph Koshy 		for (ri = 0; ri < md->pmd_npmc; ri++)
4622f263522aSJoseph Koshy 			if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
4623fbf1556dSJoseph Koshy 				if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
4624fbf1556dSJoseph Koshy 				    PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
4625f263522aSJoseph Koshy 					pmclog_process_procexit(pm, pp);
4626f263522aSJoseph Koshy 				pmc_unlink_target_process(pm, pp);
4627f263522aSJoseph Koshy 			}
46281ede983cSDag-Erling Smørgrav 		free(pp, M_PMC);
4629f263522aSJoseph Koshy 
4630f263522aSJoseph Koshy 	} else
4631f263522aSJoseph Koshy 		critical_exit(); /* pp == NULL */
4632f263522aSJoseph Koshy 
4633f263522aSJoseph Koshy 
4634f263522aSJoseph Koshy 	/*
4635f263522aSJoseph Koshy 	 * If the process owned PMCs, free them up and free up
4636f263522aSJoseph Koshy 	 * memory.
4637f263522aSJoseph Koshy 	 */
4638f263522aSJoseph Koshy 	if ((po = pmc_find_owner_descriptor(p)) != NULL) {
4639f263522aSJoseph Koshy 		pmc_remove_owner(po);
4640f263522aSJoseph Koshy 		pmc_destroy_owner_descriptor(po);
4641f263522aSJoseph Koshy 	}
4642f263522aSJoseph Koshy 
4643f263522aSJoseph Koshy 	sx_xunlock(&pmc_sx);
4644ebccf1e3SJoseph Koshy }
4645ebccf1e3SJoseph Koshy 
4646ebccf1e3SJoseph Koshy /*
4647ebccf1e3SJoseph Koshy  * Handle a process fork.
4648ebccf1e3SJoseph Koshy  *
4649ebccf1e3SJoseph Koshy  * If the parent process 'p1' is under HWPMC monitoring, then copy
4650ebccf1e3SJoseph Koshy  * over any attached PMCs that have 'do_descendants' semantics.
4651ebccf1e3SJoseph Koshy  */
4652ebccf1e3SJoseph Koshy 
4653ebccf1e3SJoseph Koshy static void
4654f263522aSJoseph Koshy pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
4655ebccf1e3SJoseph Koshy     int flags)
4656ebccf1e3SJoseph Koshy {
4657ebccf1e3SJoseph Koshy 	int is_using_hwpmcs;
4658f263522aSJoseph Koshy 	unsigned int ri;
4659f263522aSJoseph Koshy 	uint32_t do_descendants;
4660f263522aSJoseph Koshy 	struct pmc *pm;
4661f263522aSJoseph Koshy 	struct pmc_owner *po;
4662f263522aSJoseph Koshy 	struct pmc_process *ppnew, *ppold;
4663ebccf1e3SJoseph Koshy 
4664ebccf1e3SJoseph Koshy 	(void) flags;		/* unused parameter */
4665ebccf1e3SJoseph Koshy 
4666ebccf1e3SJoseph Koshy 	PROC_LOCK(p1);
4667ebccf1e3SJoseph Koshy 	is_using_hwpmcs = p1->p_flag & P_HWPMC;
4668ebccf1e3SJoseph Koshy 	PROC_UNLOCK(p1);
4669ebccf1e3SJoseph Koshy 
4670f263522aSJoseph Koshy 	/*
4671f263522aSJoseph Koshy 	 * If there are system-wide sampling PMCs active, we need to
4672f263522aSJoseph Koshy 	 * log all fork events to their owner's logs.
4673f263522aSJoseph Koshy 	 */
4674*0f00315cSMatt Macy 	epoch_enter(global_epoch);
4675*0f00315cSMatt Macy 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
4676f263522aSJoseph Koshy 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
4677f263522aSJoseph Koshy 		    pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
4678*0f00315cSMatt Macy 	epoch_exit(global_epoch);
4679f263522aSJoseph Koshy 
4680f263522aSJoseph Koshy 	if (!is_using_hwpmcs)
4681f263522aSJoseph Koshy 		return;
4682f263522aSJoseph Koshy 
4683ebccf1e3SJoseph Koshy 	PMC_GET_SX_XLOCK();
46844a3690dfSJohn Baldwin 	PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
4685f263522aSJoseph Koshy 	    p1->p_pid, p1->p_comm, newproc);
4686f263522aSJoseph Koshy 
4687f263522aSJoseph Koshy 	/*
4688f263522aSJoseph Koshy 	 * If the parent process (curthread->td_proc) is a
4689f263522aSJoseph Koshy 	 * target of any PMCs, look for PMCs that are to be
4690f263522aSJoseph Koshy 	 * inherited, and link these into the new process
4691f263522aSJoseph Koshy 	 * descriptor.
4692f263522aSJoseph Koshy 	 */
4693f263522aSJoseph Koshy 	if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
4694f263522aSJoseph Koshy 		 PMC_FLAG_NONE)) == NULL)
4695f263522aSJoseph Koshy 		goto done;		/* nothing to do */
4696f263522aSJoseph Koshy 
4697f263522aSJoseph Koshy 	do_descendants = 0;
4698f263522aSJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++)
4699f263522aSJoseph Koshy 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
4700f263522aSJoseph Koshy 			do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
4701f263522aSJoseph Koshy 	if (do_descendants == 0) /* nothing to do */
4702f263522aSJoseph Koshy 		goto done;
4703f263522aSJoseph Koshy 
4704f263522aSJoseph Koshy 	/* allocate a descriptor for the new process  */
4705f263522aSJoseph Koshy 	if ((ppnew = pmc_find_process_descriptor(newproc,
4706f263522aSJoseph Koshy 		 PMC_FLAG_ALLOCATE)) == NULL)
4707f263522aSJoseph Koshy 		goto done;
4708f263522aSJoseph Koshy 
4709f263522aSJoseph Koshy 	/*
4710f263522aSJoseph Koshy 	 * Run through all PMCs that were targeting the old process
4711f263522aSJoseph Koshy 	 * and which specified F_DESCENDANTS and attach them to the
4712f263522aSJoseph Koshy 	 * new process.
4713f263522aSJoseph Koshy 	 *
4714f263522aSJoseph Koshy 	 * Log the fork event to all owners of PMCs attached to this
4715f263522aSJoseph Koshy 	 * process, if not already logged.
4716f263522aSJoseph Koshy 	 */
4717f263522aSJoseph Koshy 	for (ri = 0; ri < md->pmd_npmc; ri++)
4718f263522aSJoseph Koshy 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
4719f263522aSJoseph Koshy 		    (pm->pm_flags & PMC_F_DESCENDANTS)) {
4720f263522aSJoseph Koshy 			pmc_link_target_process(pm, ppnew);
4721f263522aSJoseph Koshy 			po = pm->pm_owner;
4722f263522aSJoseph Koshy 			if (po->po_sscount == 0 &&
4723f263522aSJoseph Koshy 			    po->po_flags & PMC_PO_OWNS_LOGFILE)
4724f263522aSJoseph Koshy 				pmclog_process_procfork(po, p1->p_pid,
4725f263522aSJoseph Koshy 				    newproc->p_pid);
4726ebccf1e3SJoseph Koshy 		}
4727f263522aSJoseph Koshy 
4728f263522aSJoseph Koshy 	/*
4729f263522aSJoseph Koshy 	 * Now mark the new process as being tracked by this driver.
4730f263522aSJoseph Koshy 	 */
4731f263522aSJoseph Koshy 	PROC_LOCK(newproc);
4732f263522aSJoseph Koshy 	newproc->p_flag |= P_HWPMC;
4733f263522aSJoseph Koshy 	PROC_UNLOCK(newproc);
4734f263522aSJoseph Koshy 
4735f263522aSJoseph Koshy  done:
4736f263522aSJoseph Koshy 	sx_xunlock(&pmc_sx);
4737ebccf1e3SJoseph Koshy }
4738ebccf1e3SJoseph Koshy 
473929f4e216SMark Johnston static void
474029f4e216SMark Johnston pmc_kld_load(void *arg __unused, linker_file_t lf)
474129f4e216SMark Johnston {
474229f4e216SMark Johnston 	struct pmc_owner *po;
474329f4e216SMark Johnston 
474429f4e216SMark Johnston 	/*
474529f4e216SMark Johnston 	 * Notify owners of system sampling PMCs about KLD operations.
474629f4e216SMark Johnston 	 */
4747*0f00315cSMatt Macy 	epoch_enter(global_epoch);
4748*0f00315cSMatt Macy 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
474929f4e216SMark Johnston 		if (po->po_flags & PMC_PO_OWNS_LOGFILE)
475029f4e216SMark Johnston 			pmclog_process_map_in(po, (pid_t) -1,
475129f4e216SMark Johnston 			    (uintfptr_t) lf->address, lf->filename);
4752*0f00315cSMatt Macy 	epoch_exit(global_epoch);
475329f4e216SMark Johnston 
475429f4e216SMark Johnston 	/*
475529f4e216SMark Johnston 	 * TODO: Notify owners of (all) process-sampling PMCs too.
475629f4e216SMark Johnston 	 */
475729f4e216SMark Johnston }
475829f4e216SMark Johnston 
475929f4e216SMark Johnston static void
476029f4e216SMark Johnston pmc_kld_unload(void *arg __unused, const char *filename __unused,
476129f4e216SMark Johnston     caddr_t address, size_t size)
476229f4e216SMark Johnston {
476329f4e216SMark Johnston 	struct pmc_owner *po;
476429f4e216SMark Johnston 
4765*0f00315cSMatt Macy 	epoch_enter(global_epoch);
4766*0f00315cSMatt Macy 	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
476729f4e216SMark Johnston 		if (po->po_flags & PMC_PO_OWNS_LOGFILE)
476829f4e216SMark Johnston 			pmclog_process_map_out(po, (pid_t) -1,
476929f4e216SMark Johnston 			    (uintfptr_t) address, (uintfptr_t) address + size);
4770*0f00315cSMatt Macy 	epoch_exit(global_epoch);
477129f4e216SMark Johnston 
477229f4e216SMark Johnston 	/*
477329f4e216SMark Johnston 	 * TODO: Notify owners of process-sampling PMCs.
477429f4e216SMark Johnston 	 */
477529f4e216SMark Johnston }
4776ebccf1e3SJoseph Koshy 
4777ebccf1e3SJoseph Koshy /*
4778ebccf1e3SJoseph Koshy  * initialization
4779ebccf1e3SJoseph Koshy  */
47800ceb54c2SJohn Baldwin static const char *
47810ceb54c2SJohn Baldwin pmc_name_of_pmcclass(enum pmc_class class)
47820ceb54c2SJohn Baldwin {
4783ebccf1e3SJoseph Koshy 
47840ceb54c2SJohn Baldwin 	switch (class) {
4785ebccf1e3SJoseph Koshy #undef	__PMC_CLASS
47860ceb54c2SJohn Baldwin #define	__PMC_CLASS(S,V,D)						\
47870ceb54c2SJohn Baldwin 	case PMC_CLASS_##S:						\
47880ceb54c2SJohn Baldwin 		return #S;
47890ceb54c2SJohn Baldwin 	__PMC_CLASSES();
47900ceb54c2SJohn Baldwin 	default:
47910ceb54c2SJohn Baldwin 		return ("<unknown>");
47920ceb54c2SJohn Baldwin 	}
47930ceb54c2SJohn Baldwin }
4794ebccf1e3SJoseph Koshy 
4795f5f9340bSFabien Thomas /*
4796f5f9340bSFabien Thomas  * Base class initializer: allocate structure and set default classes.
4797f5f9340bSFabien Thomas  */
4798f5f9340bSFabien Thomas struct pmc_mdep *
4799f5f9340bSFabien Thomas pmc_mdep_alloc(int nclasses)
4800f5f9340bSFabien Thomas {
4801f5f9340bSFabien Thomas 	struct pmc_mdep *md;
4802f5f9340bSFabien Thomas 	int	n;
4803f5f9340bSFabien Thomas 
4804f5f9340bSFabien Thomas 	/* SOFT + md classes */
4805f5f9340bSFabien Thomas 	n = 1 + nclasses;
4806f5f9340bSFabien Thomas 	md = malloc(sizeof(struct pmc_mdep) + n *
4807f5f9340bSFabien Thomas 	    sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
4808f5f9340bSFabien Thomas 	md->pmd_nclass = n;
4809f5f9340bSFabien Thomas 
4810f5f9340bSFabien Thomas 	/* Add base class. */
4811f5f9340bSFabien Thomas 	pmc_soft_initialize(md);
4812f5f9340bSFabien Thomas 	return md;
4813f5f9340bSFabien Thomas }
4814f5f9340bSFabien Thomas 
4815f5f9340bSFabien Thomas void
4816f5f9340bSFabien Thomas pmc_mdep_free(struct pmc_mdep *md)
4817f5f9340bSFabien Thomas {
4818f5f9340bSFabien Thomas 	pmc_soft_finalize(md);
4819f5f9340bSFabien Thomas 	free(md, M_PMC);
4820f5f9340bSFabien Thomas }
4821f5f9340bSFabien Thomas 
4822f5f9340bSFabien Thomas static int
4823f5f9340bSFabien Thomas generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
4824f5f9340bSFabien Thomas {
4825f5f9340bSFabien Thomas 	(void) pc; (void) pp;
4826f5f9340bSFabien Thomas 
4827f5f9340bSFabien Thomas 	return (0);
4828f5f9340bSFabien Thomas }
4829f5f9340bSFabien Thomas 
4830f5f9340bSFabien Thomas static int
4831f5f9340bSFabien Thomas generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
4832f5f9340bSFabien Thomas {
4833f5f9340bSFabien Thomas 	(void) pc; (void) pp;
4834f5f9340bSFabien Thomas 
4835f5f9340bSFabien Thomas 	return (0);
4836f5f9340bSFabien Thomas }
4837f5f9340bSFabien Thomas 
4838f5f9340bSFabien Thomas static struct pmc_mdep *
4839f5f9340bSFabien Thomas pmc_generic_cpu_initialize(void)
4840f5f9340bSFabien Thomas {
4841f5f9340bSFabien Thomas 	struct pmc_mdep *md;
4842f5f9340bSFabien Thomas 
4843f5f9340bSFabien Thomas 	md = pmc_mdep_alloc(0);
4844f5f9340bSFabien Thomas 
4845f5f9340bSFabien Thomas 	md->pmd_cputype    = PMC_CPU_GENERIC;
4846f5f9340bSFabien Thomas 
4847f5f9340bSFabien Thomas 	md->pmd_pcpu_init  = NULL;
4848f5f9340bSFabien Thomas 	md->pmd_pcpu_fini  = NULL;
4849f5f9340bSFabien Thomas 	md->pmd_switch_in  = generic_switch_in;
4850f5f9340bSFabien Thomas 	md->pmd_switch_out = generic_switch_out;
4851f5f9340bSFabien Thomas 
4852f5f9340bSFabien Thomas 	return (md);
4853f5f9340bSFabien Thomas }
4854f5f9340bSFabien Thomas 
4855f5f9340bSFabien Thomas static void
4856f5f9340bSFabien Thomas pmc_generic_cpu_finalize(struct pmc_mdep *md)
4857f5f9340bSFabien Thomas {
4858f5f9340bSFabien Thomas 	(void) md;
4859f5f9340bSFabien Thomas }
4860f5f9340bSFabien Thomas 
4861f5f9340bSFabien Thomas 
4862ebccf1e3SJoseph Koshy static int
4863ebccf1e3SJoseph Koshy pmc_initialize(void)
4864ebccf1e3SJoseph Koshy {
4865e829eb6dSJoseph Koshy 	int c, cpu, error, n, ri;
4866e6b475e0SMatt Macy 	unsigned int maxcpu, domain;
4867e6b475e0SMatt Macy 	struct pcpu *pc;
4868ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
4869d07f36b0SJoseph Koshy 	struct pmc_sample *ps;
4870e829eb6dSJoseph Koshy 	struct pmc_classdep *pcd;
4871f263522aSJoseph Koshy 	struct pmc_samplebuffer *sb;
4872ebccf1e3SJoseph Koshy 
4873ebccf1e3SJoseph Koshy 	md = NULL;
4874ebccf1e3SJoseph Koshy 	error = 0;
4875ebccf1e3SJoseph Koshy 
4876e6b475e0SMatt Macy 	pmc_stats.pm_intr_ignored = counter_u64_alloc(M_WAITOK);
4877e6b475e0SMatt Macy 	pmc_stats.pm_intr_processed = counter_u64_alloc(M_WAITOK);
4878e6b475e0SMatt Macy 	pmc_stats.pm_intr_bufferfull = counter_u64_alloc(M_WAITOK);
4879e6b475e0SMatt Macy 	pmc_stats.pm_syscalls = counter_u64_alloc(M_WAITOK);
4880e6b475e0SMatt Macy 	pmc_stats.pm_syscall_errors = counter_u64_alloc(M_WAITOK);
4881e6b475e0SMatt Macy 	pmc_stats.pm_buffer_requests = counter_u64_alloc(M_WAITOK);
4882e6b475e0SMatt Macy 	pmc_stats.pm_buffer_requests_failed = counter_u64_alloc(M_WAITOK);
4883e6b475e0SMatt Macy 	pmc_stats.pm_log_sweeps = counter_u64_alloc(M_WAITOK);
4884e6b475e0SMatt Macy 
4885680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
4886ebccf1e3SJoseph Koshy 	/* parse debug flags first */
4887ebccf1e3SJoseph Koshy 	if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
4888ebccf1e3SJoseph Koshy 		pmc_debugstr, sizeof(pmc_debugstr)))
4889ebccf1e3SJoseph Koshy 		pmc_debugflags_parse(pmc_debugstr,
4890ebccf1e3SJoseph Koshy 		    pmc_debugstr+strlen(pmc_debugstr));
4891ebccf1e3SJoseph Koshy #endif
4892ebccf1e3SJoseph Koshy 
48934a3690dfSJohn Baldwin 	PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
4894ebccf1e3SJoseph Koshy 
4895fadcc6e2SJoseph Koshy 	/* check kernel version */
4896fadcc6e2SJoseph Koshy 	if (pmc_kernel_version != PMC_VERSION) {
4897fadcc6e2SJoseph Koshy 		if (pmc_kernel_version == 0)
4898fadcc6e2SJoseph Koshy 			printf("hwpmc: this kernel has not been compiled with "
4899fadcc6e2SJoseph Koshy 			    "'options HWPMC_HOOKS'.\n");
4900fadcc6e2SJoseph Koshy 		else
4901fadcc6e2SJoseph Koshy 			printf("hwpmc: kernel version (0x%x) does not match "
4902fadcc6e2SJoseph Koshy 			    "module version (0x%x).\n", pmc_kernel_version,
4903fadcc6e2SJoseph Koshy 			    PMC_VERSION);
4904fadcc6e2SJoseph Koshy 		return EPROGMISMATCH;
4905fadcc6e2SJoseph Koshy 	}
4906fadcc6e2SJoseph Koshy 
4907ebccf1e3SJoseph Koshy 	/*
4908ebccf1e3SJoseph Koshy 	 * check sysctl parameters
4909ebccf1e3SJoseph Koshy 	 */
4910ebccf1e3SJoseph Koshy 
4911ebccf1e3SJoseph Koshy 	if (pmc_hashsize <= 0) {
4912d07f36b0SJoseph Koshy 		(void) printf("hwpmc: tunable \"hashsize\"=%d must be "
4913d07f36b0SJoseph Koshy 		    "greater than zero.\n", pmc_hashsize);
4914ebccf1e3SJoseph Koshy 		pmc_hashsize = PMC_HASH_SIZE;
4915ebccf1e3SJoseph Koshy 	}
4916ebccf1e3SJoseph Koshy 
4917f263522aSJoseph Koshy 	if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
4918d07f36b0SJoseph Koshy 		(void) printf("hwpmc: tunable \"nsamples\"=%d out of "
4919d07f36b0SJoseph Koshy 		    "range.\n", pmc_nsamples);
4920f263522aSJoseph Koshy 		pmc_nsamples = PMC_NSAMPLES;
4921f263522aSJoseph Koshy 	}
4922f263522aSJoseph Koshy 
4923d07f36b0SJoseph Koshy 	if (pmc_callchaindepth <= 0 ||
4924d07f36b0SJoseph Koshy 	    pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
4925d07f36b0SJoseph Koshy 		(void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
4926201b654eSEd Maste 		    "range - using %d.\n", pmc_callchaindepth,
4927201b654eSEd Maste 		    PMC_CALLCHAIN_DEPTH_MAX);
4928201b654eSEd Maste 		pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX;
4929d07f36b0SJoseph Koshy 	}
4930d07f36b0SJoseph Koshy 
4931f263522aSJoseph Koshy 	md = pmc_md_initialize();
4932f5f9340bSFabien Thomas 	if (md == NULL) {
4933f5f9340bSFabien Thomas 		/* Default to generic CPU. */
4934f5f9340bSFabien Thomas 		md = pmc_generic_cpu_initialize();
4935e829eb6dSJoseph Koshy 		if (md == NULL)
4936e829eb6dSJoseph Koshy 			return (ENOSYS);
4937f5f9340bSFabien Thomas         }
4938e829eb6dSJoseph Koshy 
4939e829eb6dSJoseph Koshy 	KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
4940e829eb6dSJoseph Koshy 	    ("[pmc,%d] no classes or pmcs", __LINE__));
4941e829eb6dSJoseph Koshy 
4942e829eb6dSJoseph Koshy 	/* Compute the map from row-indices to classdep pointers. */
4943e829eb6dSJoseph Koshy 	pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
4944e829eb6dSJoseph Koshy 	    md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
4945e829eb6dSJoseph Koshy 
4946e829eb6dSJoseph Koshy 	for (n = 0; n < md->pmd_npmc; n++)
4947e829eb6dSJoseph Koshy 		pmc_rowindex_to_classdep[n] = NULL;
4948e829eb6dSJoseph Koshy 	for (ri = c = 0; c < md->pmd_nclass; c++) {
4949e829eb6dSJoseph Koshy 		pcd = &md->pmd_classdep[c];
4950e829eb6dSJoseph Koshy 		for (n = 0; n < pcd->pcd_num; n++, ri++)
4951e829eb6dSJoseph Koshy 			pmc_rowindex_to_classdep[ri] = pcd;
4952e829eb6dSJoseph Koshy 	}
4953e829eb6dSJoseph Koshy 
4954e829eb6dSJoseph Koshy 	KASSERT(ri == md->pmd_npmc,
4955e829eb6dSJoseph Koshy 	    ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
4956e829eb6dSJoseph Koshy 	    ri, md->pmd_npmc));
4957ebccf1e3SJoseph Koshy 
4958122ccdc1SJoseph Koshy 	maxcpu = pmc_cpu_max();
4959122ccdc1SJoseph Koshy 
4960ebccf1e3SJoseph Koshy 	/* allocate space for the per-cpu array */
4961e829eb6dSJoseph Koshy 	pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
4962e829eb6dSJoseph Koshy 	    M_WAITOK|M_ZERO);
4963ebccf1e3SJoseph Koshy 
4964ebccf1e3SJoseph Koshy 	/* per-cpu 'saved values' for managing process-mode PMCs */
4965e11e3f18SDag-Erling Smørgrav 	pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
4966e11e3f18SDag-Erling Smørgrav 	    M_PMC, M_WAITOK);
4967ebccf1e3SJoseph Koshy 
4968122ccdc1SJoseph Koshy 	/* Perform CPU-dependent initialization. */
4969ebccf1e3SJoseph Koshy 	pmc_save_cpu_binding(&pb);
4970e829eb6dSJoseph Koshy 	error = 0;
4971e829eb6dSJoseph Koshy 	for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
4972122ccdc1SJoseph Koshy 		if (!pmc_cpu_is_active(cpu))
4973ebccf1e3SJoseph Koshy 			continue;
4974ebccf1e3SJoseph Koshy 		pmc_select_cpu(cpu);
4975e829eb6dSJoseph Koshy 		pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
4976e829eb6dSJoseph Koshy 		    md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
4977e829eb6dSJoseph Koshy 		    M_WAITOK|M_ZERO);
4978e829eb6dSJoseph Koshy 		if (md->pmd_pcpu_init)
49790cfab8ddSJoseph Koshy 			error = md->pmd_pcpu_init(md, cpu);
4980e829eb6dSJoseph Koshy 		for (n = 0; error == 0 && n < md->pmd_nclass; n++)
4981e829eb6dSJoseph Koshy 			error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
4982ebccf1e3SJoseph Koshy 	}
4983ebccf1e3SJoseph Koshy 	pmc_restore_cpu_binding(&pb);
4984ebccf1e3SJoseph Koshy 
4985e829eb6dSJoseph Koshy 	if (error)
4986e829eb6dSJoseph Koshy 		return (error);
4987ebccf1e3SJoseph Koshy 
4988f263522aSJoseph Koshy 	/* allocate space for the sample array */
4989122ccdc1SJoseph Koshy 	for (cpu = 0; cpu < maxcpu; cpu++) {
4990122ccdc1SJoseph Koshy 		if (!pmc_cpu_is_active(cpu))
4991f263522aSJoseph Koshy 			continue;
4992e6b475e0SMatt Macy 		pc = pcpu_find(cpu);
4993e6b475e0SMatt Macy 		domain = pc->pc_domain;
4994e6b475e0SMatt Macy 		sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
4995e6b475e0SMatt Macy 			pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
4996f263522aSJoseph Koshy 		    M_WAITOK|M_ZERO);
4997f263522aSJoseph Koshy 		sb->ps_read = sb->ps_write = sb->ps_samples;
4998ba8ae2deSJoseph Koshy 		sb->ps_fence = sb->ps_samples + pmc_nsamples;
4999e829eb6dSJoseph Koshy 
5000f263522aSJoseph Koshy 		KASSERT(pmc_pcpu[cpu] != NULL,
5001f263522aSJoseph Koshy 		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5002f263522aSJoseph Koshy 
5003e6b475e0SMatt Macy 		sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5004e6b475e0SMatt Macy 			sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5005d07f36b0SJoseph Koshy 
5006d07f36b0SJoseph Koshy 		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5007d07f36b0SJoseph Koshy 			ps->ps_pc = sb->ps_callchains +
5008d07f36b0SJoseph Koshy 			    (n * pmc_callchaindepth);
5009d07f36b0SJoseph Koshy 
5010f5f9340bSFabien Thomas 		pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
5011f5f9340bSFabien Thomas 
5012e6b475e0SMatt Macy 		sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5013e6b475e0SMatt Macy 			pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5014f5f9340bSFabien Thomas 		    M_WAITOK|M_ZERO);
5015f5f9340bSFabien Thomas 		sb->ps_read = sb->ps_write = sb->ps_samples;
5016f5f9340bSFabien Thomas 		sb->ps_fence = sb->ps_samples + pmc_nsamples;
5017f5f9340bSFabien Thomas 
5018f5f9340bSFabien Thomas 		KASSERT(pmc_pcpu[cpu] != NULL,
5019f5f9340bSFabien Thomas 		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5020f5f9340bSFabien Thomas 
5021e6b475e0SMatt Macy 		sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5022e6b475e0SMatt Macy 			sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5023f5f9340bSFabien Thomas 
5024f5f9340bSFabien Thomas 		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5025f5f9340bSFabien Thomas 			ps->ps_pc = sb->ps_callchains +
5026f5f9340bSFabien Thomas 			    (n * pmc_callchaindepth);
5027f5f9340bSFabien Thomas 
5028f5f9340bSFabien Thomas 		pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
5029f263522aSJoseph Koshy 	}
5030f263522aSJoseph Koshy 
5031ebccf1e3SJoseph Koshy 	/* allocate space for the row disposition array */
5032ebccf1e3SJoseph Koshy 	pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
5033ebccf1e3SJoseph Koshy 	    M_PMC, M_WAITOK|M_ZERO);
5034ebccf1e3SJoseph Koshy 
5035ebccf1e3SJoseph Koshy 	/* mark all PMCs as available */
5036ebccf1e3SJoseph Koshy 	for (n = 0; n < (int) md->pmd_npmc; n++)
5037ebccf1e3SJoseph Koshy 		PMC_MARK_ROW_FREE(n);
5038ebccf1e3SJoseph Koshy 
5039ebccf1e3SJoseph Koshy 	/* allocate thread hash tables */
5040ebccf1e3SJoseph Koshy 	pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
5041ebccf1e3SJoseph Koshy 	    &pmc_ownerhashmask);
5042ebccf1e3SJoseph Koshy 
5043ebccf1e3SJoseph Koshy 	pmc_processhash = hashinit(pmc_hashsize, M_PMC,
5044ebccf1e3SJoseph Koshy 	    &pmc_processhashmask);
5045382d30cdSJoseph Koshy 	mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
5046382d30cdSJoseph Koshy 	    MTX_SPIN);
5047ebccf1e3SJoseph Koshy 
5048f263522aSJoseph Koshy 	LIST_INIT(&pmc_ss_owners);
5049f263522aSJoseph Koshy 	pmc_ss_count = 0;
5050f263522aSJoseph Koshy 
5051ebccf1e3SJoseph Koshy 	/* allocate a pool of spin mutexes */
5052382d30cdSJoseph Koshy 	pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
5053382d30cdSJoseph Koshy 	    MTX_SPIN);
5054ebccf1e3SJoseph Koshy 
50554a3690dfSJohn Baldwin 	PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
5056ebccf1e3SJoseph Koshy 	    "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
5057ebccf1e3SJoseph Koshy 	    pmc_processhash, pmc_processhashmask);
5058ebccf1e3SJoseph Koshy 
5059ebccf1e3SJoseph Koshy 	/* register process {exit,fork,exec} handlers */
5060ebccf1e3SJoseph Koshy 	pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
5061ebccf1e3SJoseph Koshy 	    pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
5062ebccf1e3SJoseph Koshy 	pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
5063ebccf1e3SJoseph Koshy 	    pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
5064ebccf1e3SJoseph Koshy 
506529f4e216SMark Johnston 	/* register kld event handlers */
506629f4e216SMark Johnston 	pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
506729f4e216SMark Johnston 	    NULL, EVENTHANDLER_PRI_ANY);
506829f4e216SMark Johnston 	pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload,
506929f4e216SMark Johnston 	    NULL, EVENTHANDLER_PRI_ANY);
507029f4e216SMark Johnston 
5071f263522aSJoseph Koshy 	/* initialize logging */
5072f263522aSJoseph Koshy 	pmclog_initialize();
5073f263522aSJoseph Koshy 
5074ebccf1e3SJoseph Koshy 	/* set hook functions */
5075ebccf1e3SJoseph Koshy 	pmc_intr = md->pmd_intr;
5076*0f00315cSMatt Macy 	wmb();
5077ebccf1e3SJoseph Koshy 	pmc_hook = pmc_hook_handler;
5078ebccf1e3SJoseph Koshy 
5079ebccf1e3SJoseph Koshy 	if (error == 0) {
5080ebccf1e3SJoseph Koshy 		printf(PMC_MODULE_NAME ":");
5081a8eb16c5SJoseph Koshy 		for (n = 0; n < (int) md->pmd_nclass; n++) {
5082e829eb6dSJoseph Koshy 			pcd = &md->pmd_classdep[n];
50837dada264SJoseph Koshy 			printf(" %s/%d/%d/0x%b",
50840ceb54c2SJohn Baldwin 			    pmc_name_of_pmcclass(pcd->pcd_class),
5085e829eb6dSJoseph Koshy 			    pcd->pcd_num,
50867dada264SJoseph Koshy 			    pcd->pcd_width,
5087e829eb6dSJoseph Koshy 			    pcd->pcd_caps,
5088a8eb16c5SJoseph Koshy 			    "\20"
5089a8eb16c5SJoseph Koshy 			    "\1INT\2USR\3SYS\4EDG\5THR"
5090a8eb16c5SJoseph Koshy 			    "\6REA\7WRI\10INV\11QUA\12PRC"
5091a8eb16c5SJoseph Koshy 			    "\13TAG\14CSC");
5092a8eb16c5SJoseph Koshy 		}
5093ebccf1e3SJoseph Koshy 		printf("\n");
5094ebccf1e3SJoseph Koshy 	}
5095ebccf1e3SJoseph Koshy 
5096e829eb6dSJoseph Koshy 	return (error);
5097ebccf1e3SJoseph Koshy }
5098ebccf1e3SJoseph Koshy 
5099ebccf1e3SJoseph Koshy /* prepare to be unloaded */
5100ebccf1e3SJoseph Koshy static void
5101ebccf1e3SJoseph Koshy pmc_cleanup(void)
5102ebccf1e3SJoseph Koshy {
5103e829eb6dSJoseph Koshy 	int c, cpu;
5104122ccdc1SJoseph Koshy 	unsigned int maxcpu;
5105ebccf1e3SJoseph Koshy 	struct pmc_ownerhash *ph;
5106ebccf1e3SJoseph Koshy 	struct pmc_owner *po, *tmp;
5107ebccf1e3SJoseph Koshy 	struct pmc_binding pb;
5108680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
5109ebccf1e3SJoseph Koshy 	struct pmc_processhash *prh;
5110ebccf1e3SJoseph Koshy #endif
5111ebccf1e3SJoseph Koshy 
51124a3690dfSJohn Baldwin 	PMCDBG0(MOD,INI,0, "cleanup");
5113ebccf1e3SJoseph Koshy 
5114f263522aSJoseph Koshy 	/* switch off sampling */
5115e6b475e0SMatt Macy 	CPU_FOREACH(cpu)
5116e6b475e0SMatt Macy 		DPCPU_ID_SET(cpu, pmc_sampled, 0);
5117f263522aSJoseph Koshy 	pmc_intr = NULL;
5118ebccf1e3SJoseph Koshy 
5119ebccf1e3SJoseph Koshy 	sx_xlock(&pmc_sx);
5120ebccf1e3SJoseph Koshy 	if (pmc_hook == NULL) {	/* being unloaded already */
5121ebccf1e3SJoseph Koshy 		sx_xunlock(&pmc_sx);
5122ebccf1e3SJoseph Koshy 		return;
5123ebccf1e3SJoseph Koshy 	}
5124ebccf1e3SJoseph Koshy 
5125ebccf1e3SJoseph Koshy 	pmc_hook = NULL; /* prevent new threads from entering module */
5126ebccf1e3SJoseph Koshy 
5127ebccf1e3SJoseph Koshy 	/* deregister event handlers */
5128ebccf1e3SJoseph Koshy 	EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
5129ebccf1e3SJoseph Koshy 	EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
513029f4e216SMark Johnston 	EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
513129f4e216SMark Johnston 	EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);
5132ebccf1e3SJoseph Koshy 
5133ebccf1e3SJoseph Koshy 	/* send SIGBUS to all owner threads, free up allocations */
5134ebccf1e3SJoseph Koshy 	if (pmc_ownerhash)
5135ebccf1e3SJoseph Koshy 		for (ph = pmc_ownerhash;
5136ebccf1e3SJoseph Koshy 		     ph <= &pmc_ownerhash[pmc_ownerhashmask];
5137ebccf1e3SJoseph Koshy 		     ph++) {
5138ebccf1e3SJoseph Koshy 			LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
5139ebccf1e3SJoseph Koshy 				pmc_remove_owner(po);
5140ebccf1e3SJoseph Koshy 
5141ebccf1e3SJoseph Koshy 				/* send SIGBUS to owner processes */
51424a3690dfSJohn Baldwin 				PMCDBG3(MOD,INI,2, "cleanup signal proc=%p "
5143ebccf1e3SJoseph Koshy 				    "(%d, %s)", po->po_owner,
5144ebccf1e3SJoseph Koshy 				    po->po_owner->p_pid,
5145ebccf1e3SJoseph Koshy 				    po->po_owner->p_comm);
5146ebccf1e3SJoseph Koshy 
5147ebccf1e3SJoseph Koshy 				PROC_LOCK(po->po_owner);
51488451d0ddSKip Macy 				kern_psignal(po->po_owner, SIGBUS);
5149ebccf1e3SJoseph Koshy 				PROC_UNLOCK(po->po_owner);
5150f263522aSJoseph Koshy 
5151f263522aSJoseph Koshy 				pmc_destroy_owner_descriptor(po);
5152ebccf1e3SJoseph Koshy 			}
5153ebccf1e3SJoseph Koshy 		}
5154ebccf1e3SJoseph Koshy 
5155ebccf1e3SJoseph Koshy 	/* reclaim allocated data structures */
5156ebccf1e3SJoseph Koshy 	if (pmc_mtxpool)
5157ebccf1e3SJoseph Koshy 		mtx_pool_destroy(&pmc_mtxpool);
5158ebccf1e3SJoseph Koshy 
5159ebccf1e3SJoseph Koshy 	mtx_destroy(&pmc_processhash_mtx);
5160ebccf1e3SJoseph Koshy 	if (pmc_processhash) {
5161680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
5162ebccf1e3SJoseph Koshy 		struct pmc_process *pp;
5163ebccf1e3SJoseph Koshy 
51644a3690dfSJohn Baldwin 		PMCDBG0(MOD,INI,3, "destroy process hash");
5165ebccf1e3SJoseph Koshy 		for (prh = pmc_processhash;
5166ebccf1e3SJoseph Koshy 		     prh <= &pmc_processhash[pmc_processhashmask];
5167ebccf1e3SJoseph Koshy 		     prh++)
5168ebccf1e3SJoseph Koshy 			LIST_FOREACH(pp, prh, pp_next)
51694a3690dfSJohn Baldwin 			    PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
5170ebccf1e3SJoseph Koshy #endif
5171ebccf1e3SJoseph Koshy 
5172ebccf1e3SJoseph Koshy 		hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
5173ebccf1e3SJoseph Koshy 		pmc_processhash = NULL;
5174ebccf1e3SJoseph Koshy 	}
5175ebccf1e3SJoseph Koshy 
5176ebccf1e3SJoseph Koshy 	if (pmc_ownerhash) {
51774a3690dfSJohn Baldwin 		PMCDBG0(MOD,INI,3, "destroy owner hash");
5178ebccf1e3SJoseph Koshy 		hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
5179ebccf1e3SJoseph Koshy 		pmc_ownerhash = NULL;
5180ebccf1e3SJoseph Koshy 	}
5181ebccf1e3SJoseph Koshy 
5182f263522aSJoseph Koshy 	KASSERT(LIST_EMPTY(&pmc_ss_owners),
5183f263522aSJoseph Koshy 	    ("[pmc,%d] Global SS owner list not empty", __LINE__));
5184f263522aSJoseph Koshy 	KASSERT(pmc_ss_count == 0,
5185f263522aSJoseph Koshy 	    ("[pmc,%d] Global SS count not empty", __LINE__));
5186f263522aSJoseph Koshy 
5187e829eb6dSJoseph Koshy  	/* do processor and pmc-class dependent cleanup */
5188122ccdc1SJoseph Koshy 	maxcpu = pmc_cpu_max();
5189ba8ae2deSJoseph Koshy 
51904a3690dfSJohn Baldwin 	PMCDBG0(MOD,INI,3, "md cleanup");
5191ebccf1e3SJoseph Koshy 	if (md) {
5192ebccf1e3SJoseph Koshy 		pmc_save_cpu_binding(&pb);
5193122ccdc1SJoseph Koshy 		for (cpu = 0; cpu < maxcpu; cpu++) {
51944a3690dfSJohn Baldwin 			PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
5195ebccf1e3SJoseph Koshy 			    cpu, pmc_pcpu[cpu]);
5196122ccdc1SJoseph Koshy 			if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
5197ebccf1e3SJoseph Koshy 				continue;
5198ebccf1e3SJoseph Koshy 			pmc_select_cpu(cpu);
5199e829eb6dSJoseph Koshy 			for (c = 0; c < md->pmd_nclass; c++)
5200e829eb6dSJoseph Koshy 				md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
5201e829eb6dSJoseph Koshy 			if (md->pmd_pcpu_fini)
52020cfab8ddSJoseph Koshy 				md->pmd_pcpu_fini(md, cpu);
5203ebccf1e3SJoseph Koshy 		}
5204bc4f0adeSJoseph Koshy 
5205f5f9340bSFabien Thomas 		if (md->pmd_cputype == PMC_CPU_GENERIC)
5206f5f9340bSFabien Thomas 			pmc_generic_cpu_finalize(md);
5207f5f9340bSFabien Thomas 		else
5208bc4f0adeSJoseph Koshy 			pmc_md_finalize(md);
5209bc4f0adeSJoseph Koshy 
5210f5f9340bSFabien Thomas 		pmc_mdep_free(md);
5211ebccf1e3SJoseph Koshy 		md = NULL;
5212ebccf1e3SJoseph Koshy 		pmc_restore_cpu_binding(&pb);
5213ebccf1e3SJoseph Koshy 	}
5214ebccf1e3SJoseph Koshy 
5215e829eb6dSJoseph Koshy 	/* Free per-cpu descriptors. */
5216e829eb6dSJoseph Koshy 	for (cpu = 0; cpu < maxcpu; cpu++) {
5217e829eb6dSJoseph Koshy 		if (!pmc_cpu_is_active(cpu))
5218e829eb6dSJoseph Koshy 			continue;
5219f5f9340bSFabien Thomas 		KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
5220f5f9340bSFabien Thomas 		    ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
5221e829eb6dSJoseph Koshy 			cpu));
5222f5f9340bSFabien Thomas 		KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
5223f5f9340bSFabien Thomas 		    ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
5224f5f9340bSFabien Thomas 			cpu));
5225e6b475e0SMatt Macy 		free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
5226e6b475e0SMatt Macy 		free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
5227e6b475e0SMatt Macy 		free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
5228e6b475e0SMatt Macy 		free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
5229e6b475e0SMatt Macy 		free_domain(pmc_pcpu[cpu], M_PMC);
5230e829eb6dSJoseph Koshy 	}
5231e829eb6dSJoseph Koshy 
52321ede983cSDag-Erling Smørgrav 	free(pmc_pcpu, M_PMC);
5233ebccf1e3SJoseph Koshy 	pmc_pcpu = NULL;
5234ebccf1e3SJoseph Koshy 
52351ede983cSDag-Erling Smørgrav 	free(pmc_pcpu_saved, M_PMC);
5236ebccf1e3SJoseph Koshy 	pmc_pcpu_saved = NULL;
5237ebccf1e3SJoseph Koshy 
5238ebccf1e3SJoseph Koshy 	if (pmc_pmcdisp) {
52391ede983cSDag-Erling Smørgrav 		free(pmc_pmcdisp, M_PMC);
5240ebccf1e3SJoseph Koshy 		pmc_pmcdisp = NULL;
5241ebccf1e3SJoseph Koshy 	}
5242ebccf1e3SJoseph Koshy 
5243e829eb6dSJoseph Koshy 	if (pmc_rowindex_to_classdep) {
5244e829eb6dSJoseph Koshy 		free(pmc_rowindex_to_classdep, M_PMC);
5245e829eb6dSJoseph Koshy 		pmc_rowindex_to_classdep = NULL;
5246e829eb6dSJoseph Koshy 	}
5247e829eb6dSJoseph Koshy 
5248f263522aSJoseph Koshy 	pmclog_shutdown();
5249e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_intr_ignored);
5250e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_intr_processed);
5251e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_intr_bufferfull);
5252e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_syscalls);
5253e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_syscall_errors);
5254e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_buffer_requests);
5255e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_buffer_requests_failed);
5256e6b475e0SMatt Macy 	counter_u64_free(pmc_stats.pm_log_sweeps);
5257ebccf1e3SJoseph Koshy 	sx_xunlock(&pmc_sx); 	/* we are done */
5258ebccf1e3SJoseph Koshy }
5259ebccf1e3SJoseph Koshy 
5260ebccf1e3SJoseph Koshy /*
5261ebccf1e3SJoseph Koshy  * The function called at load/unload.
5262ebccf1e3SJoseph Koshy  */
5263ebccf1e3SJoseph Koshy 
5264ebccf1e3SJoseph Koshy static int
5265ebccf1e3SJoseph Koshy load (struct module *module __unused, int cmd, void *arg __unused)
5266ebccf1e3SJoseph Koshy {
5267ebccf1e3SJoseph Koshy 	int error;
5268ebccf1e3SJoseph Koshy 
5269ebccf1e3SJoseph Koshy 	error = 0;
5270ebccf1e3SJoseph Koshy 
5271ebccf1e3SJoseph Koshy 	switch (cmd) {
5272ebccf1e3SJoseph Koshy 	case MOD_LOAD :
5273ebccf1e3SJoseph Koshy 		/* initialize the subsystem */
5274ebccf1e3SJoseph Koshy 		error = pmc_initialize();
5275ebccf1e3SJoseph Koshy 		if (error != 0)
5276ebccf1e3SJoseph Koshy 			break;
52774a3690dfSJohn Baldwin 		PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d",
5278122ccdc1SJoseph Koshy 		    pmc_syscall_num, pmc_cpu_max());
5279ebccf1e3SJoseph Koshy 		break;
5280ebccf1e3SJoseph Koshy 
5281ebccf1e3SJoseph Koshy 
5282ebccf1e3SJoseph Koshy 	case MOD_UNLOAD :
5283ebccf1e3SJoseph Koshy 	case MOD_SHUTDOWN:
5284ebccf1e3SJoseph Koshy 		pmc_cleanup();
52854a3690dfSJohn Baldwin 		PMCDBG0(MOD,INI,1, "unloaded");
5286ebccf1e3SJoseph Koshy 		break;
5287ebccf1e3SJoseph Koshy 
5288ebccf1e3SJoseph Koshy 	default :
5289ebccf1e3SJoseph Koshy 		error = EINVAL;	/* XXX should panic(9) */
5290ebccf1e3SJoseph Koshy 		break;
5291ebccf1e3SJoseph Koshy 	}
5292ebccf1e3SJoseph Koshy 
5293ebccf1e3SJoseph Koshy 	return error;
5294ebccf1e3SJoseph Koshy }
5295