xref: /freebsd/sys/dev/hwpmc/hwpmc_logging.c (revision 9978bd996b69dc6b320efce25e9d42dbdb25ac8c)
1f263522aSJoseph Koshy /*-
2718cf2ccSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3718cf2ccSPedro F. Giffuni  *
4d07f36b0SJoseph Koshy  * Copyright (c) 2005-2007 Joseph Koshy
5d07f36b0SJoseph Koshy  * Copyright (c) 2007 The FreeBSD Foundation
6e6b475e0SMatt Macy  * Copyright (c) 2018 Matthew Macy
7f263522aSJoseph Koshy  * All rights reserved.
8f263522aSJoseph Koshy  *
9d07f36b0SJoseph Koshy  * Portions of this software were developed by A. Joseph Koshy under
10d07f36b0SJoseph Koshy  * sponsorship from the FreeBSD Foundation and Google, Inc.
11d07f36b0SJoseph Koshy  *
12f263522aSJoseph Koshy  * Redistribution and use in source and binary forms, with or without
13f263522aSJoseph Koshy  * modification, are permitted provided that the following conditions
14f263522aSJoseph Koshy  * are met:
15f263522aSJoseph Koshy  * 1. Redistributions of source code must retain the above copyright
16f263522aSJoseph Koshy  *    notice, this list of conditions and the following disclaimer.
17f263522aSJoseph Koshy  * 2. Redistributions in binary form must reproduce the above copyright
18f263522aSJoseph Koshy  *    notice, this list of conditions and the following disclaimer in the
19f263522aSJoseph Koshy  *    documentation and/or other materials provided with the distribution.
20f263522aSJoseph Koshy  *
21f263522aSJoseph Koshy  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
22f263522aSJoseph Koshy  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23f263522aSJoseph Koshy  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24f263522aSJoseph Koshy  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
25f263522aSJoseph Koshy  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26f263522aSJoseph Koshy  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27f263522aSJoseph Koshy  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28f263522aSJoseph Koshy  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29f263522aSJoseph Koshy  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30f263522aSJoseph Koshy  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31f263522aSJoseph Koshy  * SUCH DAMAGE.
32f263522aSJoseph Koshy  *
33f263522aSJoseph Koshy  */
34f263522aSJoseph Koshy 
35f263522aSJoseph Koshy /*
36f263522aSJoseph Koshy  * Logging code for hwpmc(4)
37f263522aSJoseph Koshy  */
38f263522aSJoseph Koshy 
39f263522aSJoseph Koshy #include <sys/cdefs.h>
40f263522aSJoseph Koshy __FBSDID("$FreeBSD$");
41f263522aSJoseph Koshy 
42f263522aSJoseph Koshy #include <sys/param.h>
434a144410SRobert Watson #include <sys/capsicum.h>
44*9978bd99SMark Johnston #include <sys/domainset.h>
45f263522aSJoseph Koshy #include <sys/file.h>
46f263522aSJoseph Koshy #include <sys/kernel.h>
47f263522aSJoseph Koshy #include <sys/kthread.h>
48f263522aSJoseph Koshy #include <sys/lock.h>
49f263522aSJoseph Koshy #include <sys/module.h>
50f263522aSJoseph Koshy #include <sys/mutex.h>
51f263522aSJoseph Koshy #include <sys/pmc.h>
52ca2d94beSAttilio Rao #include <sys/pmckern.h>
53f263522aSJoseph Koshy #include <sys/pmclog.h>
54f263522aSJoseph Koshy #include <sys/proc.h>
55e6b475e0SMatt Macy #include <sys/sched.h>
56f263522aSJoseph Koshy #include <sys/signalvar.h>
57e6b475e0SMatt Macy #include <sys/smp.h>
58ea4d25f9SKonstantin Belousov #include <sys/syscallsubr.h>
59f263522aSJoseph Koshy #include <sys/sysctl.h>
60f263522aSJoseph Koshy #include <sys/systm.h>
61f263522aSJoseph Koshy #include <sys/uio.h>
62f263522aSJoseph Koshy #include <sys/unistd.h>
63f263522aSJoseph Koshy #include <sys/vnode.h>
64f263522aSJoseph Koshy 
65f992dd4bSMatt Macy #if defined(__i386__) || defined(__amd64__)
66f992dd4bSMatt Macy #include <machine/clock.h>
67f992dd4bSMatt Macy #endif
68f992dd4bSMatt Macy 
69102ccac2SMatt Macy #define curdomain PCPU_GET(domain)
70f1401123SMatt Macy 
71f263522aSJoseph Koshy /*
72f263522aSJoseph Koshy  * Sysctl tunables
73f263522aSJoseph Koshy  */
74f263522aSJoseph Koshy 
75f263522aSJoseph Koshy SYSCTL_DECL(_kern_hwpmc);
76f263522aSJoseph Koshy 
77f263522aSJoseph Koshy /*
78f263522aSJoseph Koshy  * kern.hwpmc.logbuffersize -- size of the per-cpu owner buffers.
79f263522aSJoseph Koshy  */
80f263522aSJoseph Koshy 
81f263522aSJoseph Koshy static int pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
82b01c40f1SRandall Stewart #if (__FreeBSD_version < 1100000)
83b01c40f1SRandall Stewart TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "logbuffersize", &pmclog_buffer_size);
84b01c40f1SRandall Stewart #endif
85af3b2549SHans Petter Selasky SYSCTL_INT(_kern_hwpmc, OID_AUTO, logbuffersize, CTLFLAG_RDTUN,
86f263522aSJoseph Koshy     &pmclog_buffer_size, 0, "size of log buffers in kilobytes");
87f263522aSJoseph Koshy 
88f263522aSJoseph Koshy /*
89f263522aSJoseph Koshy  * kern.hwpmc.nbuffer -- number of global log buffers
90f263522aSJoseph Koshy  */
91f263522aSJoseph Koshy 
92e6b475e0SMatt Macy static int pmc_nlogbuffers_pcpu = PMC_NLOGBUFFERS_PCPU;
93b01c40f1SRandall Stewart #if (__FreeBSD_version < 1100000)
94e6b475e0SMatt Macy TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nbuffers", &pmc_nlogbuffers_pcpu);
95b01c40f1SRandall Stewart #endif
96e6b475e0SMatt Macy SYSCTL_INT(_kern_hwpmc, OID_AUTO, nbuffers_pcpu, CTLFLAG_RDTUN,
97e6b475e0SMatt Macy     &pmc_nlogbuffers_pcpu, 0, "number of log buffers per cpu");
98f263522aSJoseph Koshy 
99f263522aSJoseph Koshy /*
100f263522aSJoseph Koshy  * Global log buffer list and associated spin lock.
101f263522aSJoseph Koshy  */
102f263522aSJoseph Koshy 
103f263522aSJoseph Koshy static struct mtx pmc_kthread_mtx;	/* sleep lock */
104f263522aSJoseph Koshy 
105e6b475e0SMatt Macy #define	PMCLOG_INIT_BUFFER_DESCRIPTOR(D, buf, domain) do {						\
106e6b475e0SMatt Macy 		(D)->plb_fence = ((char *) (buf)) +	1024*pmclog_buffer_size;			\
107e6b475e0SMatt Macy 		(D)->plb_base  = (D)->plb_ptr = ((char *) (buf));				\
108e6b475e0SMatt Macy 		(D)->plb_domain = domain; \
109f263522aSJoseph Koshy 	} while (0)
110f263522aSJoseph Koshy 
111e6b475e0SMatt Macy #define	PMCLOG_RESET_BUFFER_DESCRIPTOR(D) do {			\
112e6b475e0SMatt Macy 		(D)->plb_ptr  = (D)->plb_base; \
113e6b475e0SMatt Macy 	} while (0)
114f263522aSJoseph Koshy 
115f263522aSJoseph Koshy /*
116f263522aSJoseph Koshy  * Log file record constructors.
117f263522aSJoseph Koshy  */
118fbf1556dSJoseph Koshy #define	_PMCLOG_TO_HEADER(T,L)						\
119fbf1556dSJoseph Koshy 	((PMCLOG_HEADER_MAGIC << 24) |					\
120fbf1556dSJoseph Koshy 	 (PMCLOG_TYPE_ ## T << 16)   |					\
121fbf1556dSJoseph Koshy 	 ((L) & 0xFFFF))
122fbf1556dSJoseph Koshy 
123f263522aSJoseph Koshy /* reserve LEN bytes of space and initialize the entry header */
124f992dd4bSMatt Macy #define	_PMCLOG_RESERVE_SAFE(PO,TYPE,LEN,ACTION, TSC) do {	\
125f263522aSJoseph Koshy 		uint32_t *_le;						\
126f263522aSJoseph Koshy 		int _len = roundup((LEN), sizeof(uint32_t));	\
127f992dd4bSMatt Macy 		struct pmclog_header *ph;							\
128f263522aSJoseph Koshy 		if ((_le = pmclog_reserve((PO), _len)) == NULL) {	\
129f263522aSJoseph Koshy 			ACTION;											\
130f263522aSJoseph Koshy 		}													\
131f992dd4bSMatt Macy 		ph = (struct pmclog_header *)_le;					\
132f992dd4bSMatt Macy 		ph->pl_header =_PMCLOG_TO_HEADER(TYPE,_len);	\
133f992dd4bSMatt Macy 		ph->pl_tsc = (TSC);									\
134f992dd4bSMatt Macy 		_le += sizeof(*ph)/4	/* skip over timestamp */
135f263522aSJoseph Koshy 
136e6b475e0SMatt Macy /* reserve LEN bytes of space and initialize the entry header */
137e6b475e0SMatt Macy #define	_PMCLOG_RESERVE(PO,TYPE,LEN,ACTION) do {			\
138e6b475e0SMatt Macy 		uint32_t *_le;						\
139e6b475e0SMatt Macy 		int _len = roundup((LEN), sizeof(uint32_t));	\
140f992dd4bSMatt Macy 		uint64_t tsc;										\
141f992dd4bSMatt Macy 		struct pmclog_header *ph;							\
142f992dd4bSMatt Macy 		tsc = pmc_rdtsc();									\
143e6b475e0SMatt Macy 		spinlock_enter();									\
144e6b475e0SMatt Macy 		if ((_le = pmclog_reserve((PO), _len)) == NULL) {	\
145e6b475e0SMatt Macy 			spinlock_exit();								\
146e6b475e0SMatt Macy 			ACTION;											\
147e6b475e0SMatt Macy 		}												\
148f992dd4bSMatt Macy 		ph = (struct pmclog_header *)_le;					\
149f992dd4bSMatt Macy 		ph->pl_header =_PMCLOG_TO_HEADER(TYPE,_len);	\
150f992dd4bSMatt Macy 		ph->pl_tsc = tsc;									\
151f992dd4bSMatt Macy 		_le += sizeof(*ph)/4	/* skip over timestamp */
152e6b475e0SMatt Macy 
153e6b475e0SMatt Macy 
154f992dd4bSMatt Macy 
155f992dd4bSMatt Macy #define	PMCLOG_RESERVE_SAFE(P,T,L,TSC)		_PMCLOG_RESERVE_SAFE(P,T,L,return,TSC)
156f263522aSJoseph Koshy #define	PMCLOG_RESERVE(P,T,L)		_PMCLOG_RESERVE(P,T,L,return)
157f263522aSJoseph Koshy #define	PMCLOG_RESERVE_WITH_ERROR(P,T,L) _PMCLOG_RESERVE(P,T,L,		\
158f263522aSJoseph Koshy 	error=ENOMEM;goto error)
159f263522aSJoseph Koshy 
160f263522aSJoseph Koshy #define	PMCLOG_EMIT32(V)	do { *_le++ = (V); } while (0)
161f263522aSJoseph Koshy #define	PMCLOG_EMIT64(V)	do { 					\
162f263522aSJoseph Koshy 		*_le++ = (uint32_t) ((V) & 0xFFFFFFFF);			\
163f263522aSJoseph Koshy 		*_le++ = (uint32_t) (((V) >> 32) & 0xFFFFFFFF);		\
164f263522aSJoseph Koshy 	} while (0)
165f263522aSJoseph Koshy 
166f263522aSJoseph Koshy 
167f263522aSJoseph Koshy /* Emit a string.  Caution: does NOT update _le, so needs to be last */
168f263522aSJoseph Koshy #define	PMCLOG_EMITSTRING(S,L)	do { bcopy((S), _le, (L)); } while (0)
169f5f9340bSFabien Thomas #define	PMCLOG_EMITNULLSTRING(L) do { bzero(_le, (L)); } while (0)
170f263522aSJoseph Koshy 
171e6b475e0SMatt Macy #define	PMCLOG_DESPATCH_SAFE(PO)						\
172e6b475e0SMatt Macy 	    pmclog_release((PO));						\
173e6b475e0SMatt Macy 	} while (0)
174e6b475e0SMatt Macy 
17539446ce5SMatt Macy #define	PMCLOG_DESPATCH_SCHED_LOCK(PO)						\
17639446ce5SMatt Macy 	     pmclog_release_flags((PO), 0);							\
17739446ce5SMatt Macy 	} while (0)
17839446ce5SMatt Macy 
179f263522aSJoseph Koshy #define	PMCLOG_DESPATCH(PO)							\
180f263522aSJoseph Koshy 	    pmclog_release((PO));						\
181e6b475e0SMatt Macy 		spinlock_exit();							\
182e6b475e0SMatt Macy 	} while (0)
183e6b475e0SMatt Macy 
184e6b475e0SMatt Macy #define	PMCLOG_DESPATCH_SYNC(PO)						\
18539446ce5SMatt Macy 	    pmclog_schedule_io((PO), 1);						\
186e6b475e0SMatt Macy 		spinlock_exit();								\
187f263522aSJoseph Koshy 		} while (0)
188f263522aSJoseph Koshy 
189f263522aSJoseph Koshy 
190f992dd4bSMatt Macy #define TSDELTA 4
191f263522aSJoseph Koshy /*
192f263522aSJoseph Koshy  * Assertions about the log file format.
193f263522aSJoseph Koshy  */
194f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_callchain) == 7*4 + TSDELTA +
195d07f36b0SJoseph Koshy     PMC_CALLCHAIN_DEPTH_MAX*sizeof(uintfptr_t));
196f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_closelog) == 3*4 + TSDELTA);
197f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_dropnotify) == 3*4 + TSDELTA);
198f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_map_in) == PATH_MAX + TSDELTA +
199f992dd4bSMatt Macy     5*4 + sizeof(uintfptr_t));
20049874f6eSJoseph Koshy CTASSERT(offsetof(struct pmclog_map_in,pl_pathname) ==
201f992dd4bSMatt Macy     5*4 + TSDELTA + sizeof(uintfptr_t));
202f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_map_out) == 5*4 + 2*sizeof(uintfptr_t) + TSDELTA);
203f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_pmcallocate) == 9*4 + TSDELTA);
204f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_pmcattach) == 5*4 + PATH_MAX + TSDELTA);
205f992dd4bSMatt Macy CTASSERT(offsetof(struct pmclog_pmcattach,pl_pathname) == 5*4 + TSDELTA);
206f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_pmcdetach) == 5*4 + TSDELTA);
207f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_proccsw) == 7*4 + 8 + TSDELTA);
208f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_procexec) == 5*4 + PATH_MAX +
209f992dd4bSMatt Macy     sizeof(uintfptr_t) + TSDELTA);
210f992dd4bSMatt Macy CTASSERT(offsetof(struct pmclog_procexec,pl_pathname) == 5*4 + TSDELTA +
21115139246SJoseph Koshy     sizeof(uintfptr_t));
212f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_procexit) == 5*4 + 8 + TSDELTA);
213f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_procfork) == 5*4 + TSDELTA);
214f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_sysexit) == 6*4);
215f992dd4bSMatt Macy CTASSERT(sizeof(struct pmclog_userdata) == 6*4);
216f263522aSJoseph Koshy 
217f263522aSJoseph Koshy /*
218f263522aSJoseph Koshy  * Log buffer structure
219f263522aSJoseph Koshy  */
220f263522aSJoseph Koshy 
221f263522aSJoseph Koshy struct pmclog_buffer {
222f263522aSJoseph Koshy 	TAILQ_ENTRY(pmclog_buffer) plb_next;
223f263522aSJoseph Koshy 	char 		*plb_base;
224f263522aSJoseph Koshy 	char		*plb_ptr;
225f263522aSJoseph Koshy 	char 		*plb_fence;
226e6b475e0SMatt Macy 	uint16_t	 plb_domain;
227e6b475e0SMatt Macy } __aligned(CACHE_LINE_SIZE);
228e6b475e0SMatt Macy 
229f263522aSJoseph Koshy /*
230f263522aSJoseph Koshy  * Prototypes
231f263522aSJoseph Koshy  */
232f263522aSJoseph Koshy 
233f263522aSJoseph Koshy static int pmclog_get_buffer(struct pmc_owner *po);
234f263522aSJoseph Koshy static void pmclog_loop(void *arg);
235f263522aSJoseph Koshy static void pmclog_release(struct pmc_owner *po);
236f263522aSJoseph Koshy static uint32_t *pmclog_reserve(struct pmc_owner *po, int length);
23739446ce5SMatt Macy static void pmclog_schedule_io(struct pmc_owner *po, int wakeup);
238d9f1b8dbSMatt Macy static void pmclog_schedule_all(struct pmc_owner *po);
239f263522aSJoseph Koshy static void pmclog_stop_kthread(struct pmc_owner *po);
240f263522aSJoseph Koshy 
241f263522aSJoseph Koshy /*
242f263522aSJoseph Koshy  * Helper functions
243f263522aSJoseph Koshy  */
244f263522aSJoseph Koshy 
245e6b475e0SMatt Macy static inline void
246e6b475e0SMatt Macy pmc_plb_rele_unlocked(struct pmclog_buffer *plb)
247e6b475e0SMatt Macy {
248e6b475e0SMatt Macy 	TAILQ_INSERT_HEAD(&pmc_dom_hdrs[plb->plb_domain]->pdbh_head, plb, plb_next);
249e6b475e0SMatt Macy }
250e6b475e0SMatt Macy 
251e6b475e0SMatt Macy static inline void
252e6b475e0SMatt Macy pmc_plb_rele(struct pmclog_buffer *plb)
253e6b475e0SMatt Macy {
254e6b475e0SMatt Macy 	mtx_lock_spin(&pmc_dom_hdrs[plb->plb_domain]->pdbh_mtx);
255e6b475e0SMatt Macy 	pmc_plb_rele_unlocked(plb);
256e6b475e0SMatt Macy 	mtx_unlock_spin(&pmc_dom_hdrs[plb->plb_domain]->pdbh_mtx);
257e6b475e0SMatt Macy }
258e6b475e0SMatt Macy 
259f263522aSJoseph Koshy /*
260f263522aSJoseph Koshy  * Get a log buffer
261f263522aSJoseph Koshy  */
262f263522aSJoseph Koshy static int
263f263522aSJoseph Koshy pmclog_get_buffer(struct pmc_owner *po)
264f263522aSJoseph Koshy {
265f263522aSJoseph Koshy 	struct pmclog_buffer *plb;
266e6b475e0SMatt Macy 	int domain;
267f263522aSJoseph Koshy 
268e6b475e0SMatt Macy 	KASSERT(po->po_curbuf[curcpu] == NULL,
2691ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p current buffer still valid", __LINE__, po));
270f263522aSJoseph Koshy 
271102ccac2SMatt Macy 	domain = curdomain;
2720f00315cSMatt Macy 	MPASS(pmc_dom_hdrs[domain]);
273e6b475e0SMatt Macy 	mtx_lock_spin(&pmc_dom_hdrs[domain]->pdbh_mtx);
274e6b475e0SMatt Macy 	if ((plb = TAILQ_FIRST(&pmc_dom_hdrs[domain]->pdbh_head)) != NULL)
275e6b475e0SMatt Macy 		TAILQ_REMOVE(&pmc_dom_hdrs[domain]->pdbh_head, plb, plb_next);
276e6b475e0SMatt Macy 	mtx_unlock_spin(&pmc_dom_hdrs[domain]->pdbh_mtx);
277f263522aSJoseph Koshy 
2784a3690dfSJohn Baldwin 	PMCDBG2(LOG,GTB,1, "po=%p plb=%p", po, plb);
279f263522aSJoseph Koshy 
280680f1afdSJohn Baldwin #ifdef	HWPMC_DEBUG
281f263522aSJoseph Koshy 	if (plb)
282f263522aSJoseph Koshy 		KASSERT(plb->plb_ptr == plb->plb_base &&
283f263522aSJoseph Koshy 		    plb->plb_base < plb->plb_fence,
2841ad08c6aSJoseph Koshy 		    ("[pmclog,%d] po=%p buffer invariants: ptr=%p "
285f263522aSJoseph Koshy 		    "base=%p fence=%p", __LINE__, po, plb->plb_ptr,
286f263522aSJoseph Koshy 		    plb->plb_base, plb->plb_fence));
287f263522aSJoseph Koshy #endif
288f263522aSJoseph Koshy 
289e6b475e0SMatt Macy 	po->po_curbuf[curcpu] = plb;
290f263522aSJoseph Koshy 
291f263522aSJoseph Koshy 	/* update stats */
292e6b475e0SMatt Macy 	counter_u64_add(pmc_stats.pm_buffer_requests, 1);
293f263522aSJoseph Koshy 	if (plb == NULL)
294e6b475e0SMatt Macy 		counter_u64_add(pmc_stats.pm_buffer_requests_failed, 1);
295f263522aSJoseph Koshy 
2961ad08c6aSJoseph Koshy 	return (plb ? 0 : ENOMEM);
297f263522aSJoseph Koshy }
298f263522aSJoseph Koshy 
29920b555e1SKonstantin Belousov struct pmclog_proc_init_args {
30020b555e1SKonstantin Belousov 	struct proc *kthr;
30120b555e1SKonstantin Belousov 	struct pmc_owner *po;
30220b555e1SKonstantin Belousov 	bool exit;
30320b555e1SKonstantin Belousov 	bool acted;
30420b555e1SKonstantin Belousov };
30520b555e1SKonstantin Belousov 
30620b555e1SKonstantin Belousov int
30720b555e1SKonstantin Belousov pmclog_proc_create(struct thread *td, void **handlep)
30820b555e1SKonstantin Belousov {
30920b555e1SKonstantin Belousov 	struct pmclog_proc_init_args *ia;
31020b555e1SKonstantin Belousov 	int error;
31120b555e1SKonstantin Belousov 
31220b555e1SKonstantin Belousov 	ia = malloc(sizeof(*ia), M_TEMP, M_WAITOK | M_ZERO);
31320b555e1SKonstantin Belousov 	error = kproc_create(pmclog_loop, ia, &ia->kthr,
31420b555e1SKonstantin Belousov 	    RFHIGHPID, 0, "hwpmc: proc(%d)", td->td_proc->p_pid);
31520b555e1SKonstantin Belousov 	if (error == 0)
31620b555e1SKonstantin Belousov 		*handlep = ia;
31720b555e1SKonstantin Belousov 	return (error);
31820b555e1SKonstantin Belousov }
31920b555e1SKonstantin Belousov 
32020b555e1SKonstantin Belousov void
32120b555e1SKonstantin Belousov pmclog_proc_ignite(void *handle, struct pmc_owner *po)
32220b555e1SKonstantin Belousov {
32320b555e1SKonstantin Belousov 	struct pmclog_proc_init_args *ia;
32420b555e1SKonstantin Belousov 
32520b555e1SKonstantin Belousov 	ia = handle;
32620b555e1SKonstantin Belousov 	mtx_lock(&pmc_kthread_mtx);
32720b555e1SKonstantin Belousov 	MPASS(!ia->acted);
32820b555e1SKonstantin Belousov 	MPASS(ia->po == NULL);
32920b555e1SKonstantin Belousov 	MPASS(!ia->exit);
33020b555e1SKonstantin Belousov 	MPASS(ia->kthr != NULL);
33120b555e1SKonstantin Belousov 	if (po == NULL) {
33220b555e1SKonstantin Belousov 		ia->exit = true;
33320b555e1SKonstantin Belousov 	} else {
33420b555e1SKonstantin Belousov 		ia->po = po;
33520b555e1SKonstantin Belousov 		KASSERT(po->po_kthread == NULL,
33620b555e1SKonstantin Belousov 		    ("[pmclog,%d] po=%p kthread (%p) already present",
33720b555e1SKonstantin Belousov 		    __LINE__, po, po->po_kthread));
33820b555e1SKonstantin Belousov 		po->po_kthread = ia->kthr;
33920b555e1SKonstantin Belousov 	}
34020b555e1SKonstantin Belousov 	wakeup(ia);
34120b555e1SKonstantin Belousov 	while (!ia->acted)
34220b555e1SKonstantin Belousov 		msleep(ia, &pmc_kthread_mtx, PWAIT, "pmclogw", 0);
34320b555e1SKonstantin Belousov 	mtx_unlock(&pmc_kthread_mtx);
34420b555e1SKonstantin Belousov 	free(ia, M_TEMP);
34520b555e1SKonstantin Belousov }
34620b555e1SKonstantin Belousov 
347f263522aSJoseph Koshy /*
348f263522aSJoseph Koshy  * Log handler loop.
349f263522aSJoseph Koshy  *
350f263522aSJoseph Koshy  * This function is executed by each pmc owner's helper thread.
351f263522aSJoseph Koshy  */
352f263522aSJoseph Koshy static void
353f263522aSJoseph Koshy pmclog_loop(void *arg)
354f263522aSJoseph Koshy {
35520b555e1SKonstantin Belousov 	struct pmclog_proc_init_args *ia;
356f263522aSJoseph Koshy 	struct pmc_owner *po;
357f263522aSJoseph Koshy 	struct pmclog_buffer *lb;
3585eaf27d8SFabien Thomas 	struct proc *p;
359f263522aSJoseph Koshy 	struct ucred *ownercred;
360f263522aSJoseph Koshy 	struct ucred *mycred;
361f263522aSJoseph Koshy 	struct thread *td;
362ea4d25f9SKonstantin Belousov 	sigset_t unb;
363f263522aSJoseph Koshy 	struct uio auio;
364f263522aSJoseph Koshy 	struct iovec aiov;
365f263522aSJoseph Koshy 	size_t nbytes;
36620b555e1SKonstantin Belousov 	int error;
367f263522aSJoseph Koshy 
368f263522aSJoseph Koshy 	td = curthread;
369ea4d25f9SKonstantin Belousov 
370ea4d25f9SKonstantin Belousov 	SIGEMPTYSET(unb);
371ea4d25f9SKonstantin Belousov 	SIGADDSET(unb, SIGHUP);
372ea4d25f9SKonstantin Belousov 	(void)kern_sigprocmask(td, SIG_UNBLOCK, &unb, NULL, 0);
373ea4d25f9SKonstantin Belousov 
37420b555e1SKonstantin Belousov 	ia = arg;
37520b555e1SKonstantin Belousov 	MPASS(ia->kthr == curproc);
37620b555e1SKonstantin Belousov 	MPASS(!ia->acted);
37720b555e1SKonstantin Belousov 	mtx_lock(&pmc_kthread_mtx);
37820b555e1SKonstantin Belousov 	while (ia->po == NULL && !ia->exit)
37920b555e1SKonstantin Belousov 		msleep(ia, &pmc_kthread_mtx, PWAIT, "pmclogi", 0);
38020b555e1SKonstantin Belousov 	if (ia->exit) {
38120b555e1SKonstantin Belousov 		ia->acted = true;
38220b555e1SKonstantin Belousov 		wakeup(ia);
38320b555e1SKonstantin Belousov 		mtx_unlock(&pmc_kthread_mtx);
38420b555e1SKonstantin Belousov 		kproc_exit(0);
38520b555e1SKonstantin Belousov 	}
38620b555e1SKonstantin Belousov 	MPASS(ia->po != NULL);
38720b555e1SKonstantin Belousov 	po = ia->po;
38820b555e1SKonstantin Belousov 	ia->acted = true;
38920b555e1SKonstantin Belousov 	wakeup(ia);
39020b555e1SKonstantin Belousov 	mtx_unlock(&pmc_kthread_mtx);
39120b555e1SKonstantin Belousov 	ia = NULL;
39220b555e1SKonstantin Belousov 
39320b555e1SKonstantin Belousov 	p = po->po_owner;
394f263522aSJoseph Koshy 	mycred = td->td_ucred;
395f263522aSJoseph Koshy 
3965eaf27d8SFabien Thomas 	PROC_LOCK(p);
3975eaf27d8SFabien Thomas 	ownercred = crhold(p->p_ucred);
3985eaf27d8SFabien Thomas 	PROC_UNLOCK(p);
399f263522aSJoseph Koshy 
4004a3690dfSJohn Baldwin 	PMCDBG2(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread);
401f263522aSJoseph Koshy 	KASSERT(po->po_kthread == curthread->td_proc,
4021ad08c6aSJoseph Koshy 	    ("[pmclog,%d] proc mismatch po=%p po/kt=%p curproc=%p", __LINE__,
403f263522aSJoseph Koshy 		po, po->po_kthread, curthread->td_proc));
404f263522aSJoseph Koshy 
405f263522aSJoseph Koshy 	lb = NULL;
406f263522aSJoseph Koshy 
407f263522aSJoseph Koshy 
408f263522aSJoseph Koshy 	/*
409f263522aSJoseph Koshy 	 * Loop waiting for I/O requests to be added to the owner
410f263522aSJoseph Koshy 	 * struct's queue.  The loop is exited when the log file
411f263522aSJoseph Koshy 	 * is deconfigured.
412f263522aSJoseph Koshy 	 */
413f263522aSJoseph Koshy 
414f263522aSJoseph Koshy 	mtx_lock(&pmc_kthread_mtx);
415f263522aSJoseph Koshy 
416f263522aSJoseph Koshy 	for (;;) {
417f263522aSJoseph Koshy 
418f263522aSJoseph Koshy 		/* check if we've been asked to exit */
419f263522aSJoseph Koshy 		if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
420f263522aSJoseph Koshy 			break;
421f263522aSJoseph Koshy 
422f263522aSJoseph Koshy 		if (lb == NULL) { /* look for a fresh buffer to write */
423f263522aSJoseph Koshy 			mtx_lock_spin(&po->po_mtx);
424f263522aSJoseph Koshy 			if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
425f263522aSJoseph Koshy 				mtx_unlock_spin(&po->po_mtx);
426f263522aSJoseph Koshy 
4276f33c108SFabien Thomas 				/* No more buffers and shutdown required. */
428ea4d25f9SKonstantin Belousov 				if (po->po_flags & PMC_PO_SHUTDOWN)
4296f33c108SFabien Thomas 					break;
430dceed24aSFabien Thomas 
431cb239408SJoseph Koshy 				(void) msleep(po, &pmc_kthread_mtx, PWAIT,
43239446ce5SMatt Macy 				    "pmcloop", 250);
433f263522aSJoseph Koshy 				continue;
434f263522aSJoseph Koshy 			}
435f263522aSJoseph Koshy 
436f263522aSJoseph Koshy 			TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
437f263522aSJoseph Koshy 			mtx_unlock_spin(&po->po_mtx);
438f263522aSJoseph Koshy 		}
439f263522aSJoseph Koshy 
440f263522aSJoseph Koshy 		mtx_unlock(&pmc_kthread_mtx);
441f263522aSJoseph Koshy 
442f263522aSJoseph Koshy 		/* process the request */
4434a3690dfSJohn Baldwin 		PMCDBG3(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
444f263522aSJoseph Koshy 		    lb->plb_base, lb->plb_ptr);
445f263522aSJoseph Koshy 		/* change our thread's credentials before issuing the I/O */
446f263522aSJoseph Koshy 
447f263522aSJoseph Koshy 		aiov.iov_base = lb->plb_base;
448f263522aSJoseph Koshy 		aiov.iov_len  = nbytes = lb->plb_ptr - lb->plb_base;
449f263522aSJoseph Koshy 
450f263522aSJoseph Koshy 		auio.uio_iov    = &aiov;
451f263522aSJoseph Koshy 		auio.uio_iovcnt = 1;
452f263522aSJoseph Koshy 		auio.uio_offset = -1;
453f263522aSJoseph Koshy 		auio.uio_resid  = nbytes;
454f263522aSJoseph Koshy 		auio.uio_rw     = UIO_WRITE;
455f263522aSJoseph Koshy 		auio.uio_segflg = UIO_SYSSPACE;
456f263522aSJoseph Koshy 		auio.uio_td     = td;
457f263522aSJoseph Koshy 
458f263522aSJoseph Koshy 		/* switch thread credentials -- see kern_ktrace.c */
459f263522aSJoseph Koshy 		td->td_ucred = ownercred;
460f263522aSJoseph Koshy 		error = fo_write(po->po_file, &auio, ownercred, 0, td);
461f263522aSJoseph Koshy 		td->td_ucred = mycred;
462f263522aSJoseph Koshy 
463f263522aSJoseph Koshy 		if (error) {
464f263522aSJoseph Koshy 			/* XXX some errors are recoverable */
465f263522aSJoseph Koshy 			/* send a SIGIO to the owner and exit */
4665eaf27d8SFabien Thomas 			PROC_LOCK(p);
4678451d0ddSKip Macy 			kern_psignal(p, SIGIO);
4685eaf27d8SFabien Thomas 			PROC_UNLOCK(p);
4695eaf27d8SFabien Thomas 
4705eaf27d8SFabien Thomas 			mtx_lock(&pmc_kthread_mtx);
471f263522aSJoseph Koshy 
472f263522aSJoseph Koshy 			po->po_error = error; /* save for flush log */
473f263522aSJoseph Koshy 
4744a3690dfSJohn Baldwin 			PMCDBG2(LOG,WRI,2, "po=%p error=%d", po, error);
475f263522aSJoseph Koshy 
476f263522aSJoseph Koshy 			break;
477f263522aSJoseph Koshy 		}
478f263522aSJoseph Koshy 
4795eaf27d8SFabien Thomas 		mtx_lock(&pmc_kthread_mtx);
4805eaf27d8SFabien Thomas 
481f263522aSJoseph Koshy 		/* put the used buffer back into the global pool */
482e6b475e0SMatt Macy 		PMCLOG_RESET_BUFFER_DESCRIPTOR(lb);
483f263522aSJoseph Koshy 
484e6b475e0SMatt Macy 		pmc_plb_rele(lb);
485f263522aSJoseph Koshy 		lb = NULL;
486f263522aSJoseph Koshy 	}
487f263522aSJoseph Koshy 
4886f33c108SFabien Thomas 	wakeup_one(po->po_kthread);
489f263522aSJoseph Koshy 	po->po_kthread = NULL;
490f263522aSJoseph Koshy 
491f263522aSJoseph Koshy 	mtx_unlock(&pmc_kthread_mtx);
492f263522aSJoseph Koshy 
493f263522aSJoseph Koshy 	/* return the current I/O buffer to the global pool */
494f263522aSJoseph Koshy 	if (lb) {
495e6b475e0SMatt Macy 		PMCLOG_RESET_BUFFER_DESCRIPTOR(lb);
496f263522aSJoseph Koshy 
497e6b475e0SMatt Macy 		pmc_plb_rele(lb);
498f263522aSJoseph Koshy 	}
499f263522aSJoseph Koshy 
500f263522aSJoseph Koshy 	/*
501f263522aSJoseph Koshy 	 * Exit this thread, signalling the waiter
502f263522aSJoseph Koshy 	 */
503f263522aSJoseph Koshy 
504f263522aSJoseph Koshy 	crfree(ownercred);
505f263522aSJoseph Koshy 
5063745c395SJulian Elischer 	kproc_exit(0);
507f263522aSJoseph Koshy }
508f263522aSJoseph Koshy 
509f263522aSJoseph Koshy /*
510f263522aSJoseph Koshy  * Release and log entry and schedule an I/O if needed.
511f263522aSJoseph Koshy  */
512f263522aSJoseph Koshy 
513f263522aSJoseph Koshy static void
51439446ce5SMatt Macy pmclog_release_flags(struct pmc_owner *po, int wakeup)
515f263522aSJoseph Koshy {
516e6b475e0SMatt Macy 	struct pmclog_buffer *plb;
517e6b475e0SMatt Macy 
518e6b475e0SMatt Macy 	plb = po->po_curbuf[curcpu];
519e6b475e0SMatt Macy 	KASSERT(plb->plb_ptr >= plb->plb_base,
5201ad08c6aSJoseph Koshy 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
521e6b475e0SMatt Macy 		po, plb->plb_ptr, plb->plb_base));
522e6b475e0SMatt Macy 	KASSERT(plb->plb_ptr <= plb->plb_fence,
5231ad08c6aSJoseph Koshy 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
524e6b475e0SMatt Macy 		po, plb->plb_ptr, plb->plb_fence));
525f263522aSJoseph Koshy 
526f263522aSJoseph Koshy 	/* schedule an I/O if we've filled a buffer */
527e6b475e0SMatt Macy 	if (plb->plb_ptr >= plb->plb_fence)
52839446ce5SMatt Macy 		pmclog_schedule_io(po, wakeup);
529f263522aSJoseph Koshy 
5304a3690dfSJohn Baldwin 	PMCDBG1(LOG,REL,1, "po=%p", po);
531f263522aSJoseph Koshy }
532f263522aSJoseph Koshy 
53339446ce5SMatt Macy static void
53439446ce5SMatt Macy pmclog_release(struct pmc_owner *po)
53539446ce5SMatt Macy {
53639446ce5SMatt Macy 
53739446ce5SMatt Macy 	pmclog_release_flags(po, 1);
53839446ce5SMatt Macy }
53939446ce5SMatt Macy 
540f263522aSJoseph Koshy 
541f263522aSJoseph Koshy /*
542f263522aSJoseph Koshy  * Attempt to reserve 'length' bytes of space in an owner's log
543f263522aSJoseph Koshy  * buffer.  The function returns a pointer to 'length' bytes of space
544f263522aSJoseph Koshy  * if there was enough space or returns NULL if no space was
545f263522aSJoseph Koshy  * available.  Non-null returns do so with the po mutex locked.  The
546f263522aSJoseph Koshy  * caller must invoke pmclog_release() on the pmc owner structure
547f263522aSJoseph Koshy  * when done.
548f263522aSJoseph Koshy  */
549f263522aSJoseph Koshy 
550f263522aSJoseph Koshy static uint32_t *
551f263522aSJoseph Koshy pmclog_reserve(struct pmc_owner *po, int length)
552f263522aSJoseph Koshy {
553fbf1556dSJoseph Koshy 	uintptr_t newptr, oldptr;
554e6b475e0SMatt Macy 	struct pmclog_buffer *plb, **pplb;
555f263522aSJoseph Koshy 
5564a3690dfSJohn Baldwin 	PMCDBG2(LOG,ALL,1, "po=%p len=%d", po, length);
557f263522aSJoseph Koshy 
558f263522aSJoseph Koshy 	KASSERT(length % sizeof(uint32_t) == 0,
559f263522aSJoseph Koshy 	    ("[pmclog,%d] length not a multiple of word size", __LINE__));
560f263522aSJoseph Koshy 
561b44906e5SFabien Thomas 	/* No more data when shutdown in progress. */
562e6b475e0SMatt Macy 	if (po->po_flags & PMC_PO_SHUTDOWN)
563b44906e5SFabien Thomas 		return (NULL);
564b44906e5SFabien Thomas 
565e6b475e0SMatt Macy 	pplb = &po->po_curbuf[curcpu];
566e6b475e0SMatt Macy 	if (*pplb == NULL && pmclog_get_buffer(po) != 0)
567e6b475e0SMatt Macy 		goto fail;
568f263522aSJoseph Koshy 
569e6b475e0SMatt Macy 	KASSERT(*pplb != NULL,
5701ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p no current buffer", __LINE__, po));
571f263522aSJoseph Koshy 
572e6b475e0SMatt Macy 	plb = *pplb;
573e6b475e0SMatt Macy 	KASSERT(plb->plb_ptr >= plb->plb_base &&
574e6b475e0SMatt Macy 	    plb->plb_ptr <= plb->plb_fence,
5751ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
576e6b475e0SMatt Macy 		__LINE__, po, plb->plb_ptr, plb->plb_base,
577e6b475e0SMatt Macy 		plb->plb_fence));
578f263522aSJoseph Koshy 
579e6b475e0SMatt Macy 	oldptr = (uintptr_t) plb->plb_ptr;
580f263522aSJoseph Koshy 	newptr = oldptr + length;
581f263522aSJoseph Koshy 
582fbf1556dSJoseph Koshy 	KASSERT(oldptr != (uintptr_t) NULL,
5831ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p Null log buffer pointer", __LINE__, po));
584f263522aSJoseph Koshy 
585f263522aSJoseph Koshy 	/*
586f263522aSJoseph Koshy 	 * If we have space in the current buffer, return a pointer to
587f263522aSJoseph Koshy 	 * available space with the PO structure locked.
588f263522aSJoseph Koshy 	 */
589e6b475e0SMatt Macy 	if (newptr <= (uintptr_t) plb->plb_fence) {
590e6b475e0SMatt Macy 		plb->plb_ptr = (char *) newptr;
591f263522aSJoseph Koshy 		goto done;
592f263522aSJoseph Koshy 	}
593f263522aSJoseph Koshy 
594fbf1556dSJoseph Koshy 	/*
595fbf1556dSJoseph Koshy 	 * Otherwise, schedule the current buffer for output and get a
596fbf1556dSJoseph Koshy 	 * fresh buffer.
597fbf1556dSJoseph Koshy 	 */
59839446ce5SMatt Macy 	pmclog_schedule_io(po, 0);
599f263522aSJoseph Koshy 
600e6b475e0SMatt Macy 	if (pmclog_get_buffer(po) != 0)
601e6b475e0SMatt Macy 		goto fail;
602f263522aSJoseph Koshy 
603e6b475e0SMatt Macy 	plb = *pplb;
604e6b475e0SMatt Macy 	KASSERT(plb != NULL,
6051ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p no current buffer", __LINE__, po));
606f263522aSJoseph Koshy 
607e6b475e0SMatt Macy 	KASSERT(plb->plb_ptr != NULL,
6081ad08c6aSJoseph Koshy 	    ("[pmclog,%d] null return from pmc_get_log_buffer", __LINE__));
609f263522aSJoseph Koshy 
610e6b475e0SMatt Macy 	KASSERT(plb->plb_ptr == plb->plb_base &&
611e6b475e0SMatt Macy 	    plb->plb_ptr <= plb->plb_fence,
6121ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
613e6b475e0SMatt Macy 		__LINE__, po, plb->plb_ptr, plb->plb_base,
614e6b475e0SMatt Macy 		plb->plb_fence));
615f263522aSJoseph Koshy 
616e6b475e0SMatt Macy 	oldptr = (uintptr_t) plb->plb_ptr;
617f263522aSJoseph Koshy 
618f263522aSJoseph Koshy  done:
6191ad08c6aSJoseph Koshy 	return ((uint32_t *) oldptr);
620e6b475e0SMatt Macy  fail:
621e6b475e0SMatt Macy 	return (NULL);
622f263522aSJoseph Koshy }
623f263522aSJoseph Koshy 
624f263522aSJoseph Koshy /*
625f263522aSJoseph Koshy  * Schedule an I/O.
626f263522aSJoseph Koshy  *
627f263522aSJoseph Koshy  * Transfer the current buffer to the helper kthread.
628f263522aSJoseph Koshy  */
629f263522aSJoseph Koshy 
630f263522aSJoseph Koshy static void
63139446ce5SMatt Macy pmclog_schedule_io(struct pmc_owner *po, int wakeup)
632f263522aSJoseph Koshy {
633e6b475e0SMatt Macy 	struct pmclog_buffer *plb;
634f263522aSJoseph Koshy 
635e6b475e0SMatt Macy 	plb = po->po_curbuf[curcpu];
636e6b475e0SMatt Macy 	po->po_curbuf[curcpu] = NULL;
637e6b475e0SMatt Macy 	KASSERT(plb != NULL,
638e6b475e0SMatt Macy 	    ("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po));
639e6b475e0SMatt Macy 	KASSERT(plb->plb_ptr >= plb->plb_base,
6401ad08c6aSJoseph Koshy 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
641e6b475e0SMatt Macy 		po, plb->plb_ptr, plb->plb_base));
642e6b475e0SMatt Macy 	KASSERT(plb->plb_ptr <= plb->plb_fence,
6431ad08c6aSJoseph Koshy 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
644e6b475e0SMatt Macy 		po, plb->plb_ptr, plb->plb_fence));
645f263522aSJoseph Koshy 
6464a3690dfSJohn Baldwin 	PMCDBG1(LOG,SIO, 1, "po=%p", po);
647f263522aSJoseph Koshy 
648f263522aSJoseph Koshy 	/*
649f263522aSJoseph Koshy 	 * Add the current buffer to the tail of the buffer list and
650f263522aSJoseph Koshy 	 * wakeup the helper.
651f263522aSJoseph Koshy 	 */
652e6b475e0SMatt Macy 	mtx_lock_spin(&po->po_mtx);
653e6b475e0SMatt Macy 	TAILQ_INSERT_TAIL(&po->po_logbuffers, plb, plb_next);
654e6b475e0SMatt Macy 	mtx_unlock_spin(&po->po_mtx);
65539446ce5SMatt Macy 	if (wakeup)
656f263522aSJoseph Koshy 		wakeup_one(po);
657f263522aSJoseph Koshy }
658f263522aSJoseph Koshy 
659f263522aSJoseph Koshy /*
660f263522aSJoseph Koshy  * Stop the helper kthread.
661f263522aSJoseph Koshy  */
662f263522aSJoseph Koshy 
663f263522aSJoseph Koshy static void
664f263522aSJoseph Koshy pmclog_stop_kthread(struct pmc_owner *po)
665f263522aSJoseph Koshy {
6665eaf27d8SFabien Thomas 
6675eaf27d8SFabien Thomas 	mtx_lock(&pmc_kthread_mtx);
668f263522aSJoseph Koshy 	po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
669ea4d25f9SKonstantin Belousov 	if (po->po_kthread != NULL) {
670ea4d25f9SKonstantin Belousov 		PROC_LOCK(po->po_kthread);
671ea4d25f9SKonstantin Belousov 		kern_psignal(po->po_kthread, SIGHUP);
672ea4d25f9SKonstantin Belousov 		PROC_UNLOCK(po->po_kthread);
673ea4d25f9SKonstantin Belousov 	}
674f263522aSJoseph Koshy 	wakeup_one(po);
675ea4d25f9SKonstantin Belousov 	while (po->po_kthread)
676fbf1556dSJoseph Koshy 		msleep(po->po_kthread, &pmc_kthread_mtx, PPAUSE, "pmckstp", 0);
6775eaf27d8SFabien Thomas 	mtx_unlock(&pmc_kthread_mtx);
678f263522aSJoseph Koshy }
679f263522aSJoseph Koshy 
680f263522aSJoseph Koshy /*
681f263522aSJoseph Koshy  * Public functions
682f263522aSJoseph Koshy  */
683f263522aSJoseph Koshy 
684f263522aSJoseph Koshy /*
685f263522aSJoseph Koshy  * Configure a log file for pmc owner 'po'.
686f263522aSJoseph Koshy  *
687f263522aSJoseph Koshy  * Parameter 'logfd' is a file handle referencing an open file in the
688f263522aSJoseph Koshy  * owner process.  This file needs to have been opened for writing.
689f263522aSJoseph Koshy  */
690f263522aSJoseph Koshy 
691f263522aSJoseph Koshy int
6920cfab8ddSJoseph Koshy pmclog_configure_log(struct pmc_mdep *md, struct pmc_owner *po, int logfd)
693f263522aSJoseph Koshy {
694f263522aSJoseph Koshy 	struct proc *p;
695f992dd4bSMatt Macy 	struct timespec ts;
696f992dd4bSMatt Macy 	uint64_t tsc;
69720b555e1SKonstantin Belousov 	int error;
69820b555e1SKonstantin Belousov 
69920b555e1SKonstantin Belousov 	sx_assert(&pmc_sx, SA_XLOCKED);
7004a3690dfSJohn Baldwin 	PMCDBG2(LOG,CFG,1, "config po=%p logfd=%d", po, logfd);
701f263522aSJoseph Koshy 
702f263522aSJoseph Koshy 	p = po->po_owner;
703f263522aSJoseph Koshy 
704f263522aSJoseph Koshy 	/* return EBUSY if a log file was already present */
705f263522aSJoseph Koshy 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
7061ad08c6aSJoseph Koshy 		return (EBUSY);
707f263522aSJoseph Koshy 
708f263522aSJoseph Koshy 	KASSERT(po->po_file == NULL,
7091ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p file (%p) already present", __LINE__, po,
710f263522aSJoseph Koshy 		po->po_file));
711f263522aSJoseph Koshy 
712f263522aSJoseph Koshy 	/* get a reference to the file state */
713cbd92ce6SMatt Macy 	error = fget_write(curthread, logfd, &cap_write_rights, &po->po_file);
714f263522aSJoseph Koshy 	if (error)
715f263522aSJoseph Koshy 		goto error;
716de8d8ca4SRandall Stewart 
717f263522aSJoseph Koshy 	/* mark process as owning a log file */
718f263522aSJoseph Koshy 	po->po_flags |= PMC_PO_OWNS_LOGFILE;
719f263522aSJoseph Koshy 
720f263522aSJoseph Koshy 	/* mark process as using HWPMCs */
721f263522aSJoseph Koshy 	PROC_LOCK(p);
722f263522aSJoseph Koshy 	p->p_flag |= P_HWPMC;
723f263522aSJoseph Koshy 	PROC_UNLOCK(p);
724f992dd4bSMatt Macy 	nanotime(&ts);
725f992dd4bSMatt Macy 	tsc = pmc_rdtsc();
726f263522aSJoseph Koshy 	/* create a log initialization entry */
727f263522aSJoseph Koshy 	PMCLOG_RESERVE_WITH_ERROR(po, INITIALIZE,
728f263522aSJoseph Koshy 	    sizeof(struct pmclog_initialize));
729f263522aSJoseph Koshy 	PMCLOG_EMIT32(PMC_VERSION);
730f263522aSJoseph Koshy 	PMCLOG_EMIT32(md->pmd_cputype);
731f992dd4bSMatt Macy #if defined(__i386__) || defined(__amd64__)
732f992dd4bSMatt Macy 	PMCLOG_EMIT64(tsc_freq);
733f992dd4bSMatt Macy #else
734f992dd4bSMatt Macy 	/* other architectures will need to fill this in */
7359616acdeSMatt Macy 	PMCLOG_EMIT32(0);
7369616acdeSMatt Macy 	PMCLOG_EMIT32(0);
737f992dd4bSMatt Macy #endif
738f992dd4bSMatt Macy 	memcpy(_le, &ts, sizeof(ts));
739f992dd4bSMatt Macy 	_le += sizeof(ts)/4;
74007d80fd8SMatt Macy 	PMCLOG_EMITSTRING(pmc_cpuid, PMC_CPUID_LEN);
741e6b475e0SMatt Macy 	PMCLOG_DESPATCH_SYNC(po);
742f263522aSJoseph Koshy 
7431ad08c6aSJoseph Koshy 	return (0);
744f263522aSJoseph Koshy 
745f263522aSJoseph Koshy  error:
7461ad08c6aSJoseph Koshy 	KASSERT(po->po_kthread == NULL, ("[pmclog,%d] po=%p kthread not "
7471ad08c6aSJoseph Koshy 	    "stopped", __LINE__, po));
748f263522aSJoseph Koshy 
749f263522aSJoseph Koshy 	if (po->po_file)
750f263522aSJoseph Koshy 		(void) fdrop(po->po_file, curthread);
751f263522aSJoseph Koshy 	po->po_file  = NULL;	/* clear file and error state */
752f263522aSJoseph Koshy 	po->po_error = 0;
753f4dd123eSKonstantin Belousov 	po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
754f263522aSJoseph Koshy 
7551ad08c6aSJoseph Koshy 	return (error);
756f263522aSJoseph Koshy }
757f263522aSJoseph Koshy 
758f263522aSJoseph Koshy 
759f263522aSJoseph Koshy /*
760f263522aSJoseph Koshy  * De-configure a log file.  This will throw away any buffers queued
761f263522aSJoseph Koshy  * for this owner process.
762f263522aSJoseph Koshy  */
763f263522aSJoseph Koshy 
764f263522aSJoseph Koshy int
765f263522aSJoseph Koshy pmclog_deconfigure_log(struct pmc_owner *po)
766f263522aSJoseph Koshy {
767f263522aSJoseph Koshy 	int error;
768f263522aSJoseph Koshy 	struct pmclog_buffer *lb;
769f263522aSJoseph Koshy 
7704a3690dfSJohn Baldwin 	PMCDBG1(LOG,CFG,1, "de-config po=%p", po);
771f263522aSJoseph Koshy 
772f263522aSJoseph Koshy 	if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
7731ad08c6aSJoseph Koshy 		return (EINVAL);
774f263522aSJoseph Koshy 
775fc9a2b80SJoseph Koshy 	KASSERT(po->po_sscount == 0,
7761ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p still owning SS PMCs", __LINE__, po));
777f263522aSJoseph Koshy 	KASSERT(po->po_file != NULL,
7781ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p no log file", __LINE__, po));
779f263522aSJoseph Koshy 
780f263522aSJoseph Koshy 	/* stop the kthread, this will reset the 'OWNS_LOGFILE' flag */
781f263522aSJoseph Koshy 	pmclog_stop_kthread(po);
782f263522aSJoseph Koshy 
783f263522aSJoseph Koshy 	KASSERT(po->po_kthread == NULL,
7841ad08c6aSJoseph Koshy 	    ("[pmclog,%d] po=%p kthread not stopped", __LINE__, po));
785f263522aSJoseph Koshy 
786f263522aSJoseph Koshy 	/* return all queued log buffers to the global pool */
787f263522aSJoseph Koshy 	while ((lb = TAILQ_FIRST(&po->po_logbuffers)) != NULL) {
788f263522aSJoseph Koshy 		TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
789e6b475e0SMatt Macy 		PMCLOG_RESET_BUFFER_DESCRIPTOR(lb);
790e6b475e0SMatt Macy 		pmc_plb_rele(lb);
791f263522aSJoseph Koshy 	}
792e6b475e0SMatt Macy 	for (int i = 0; i < mp_ncpus; i++) {
793e6b475e0SMatt Macy 		thread_lock(curthread);
794e6b475e0SMatt Macy 		sched_bind(curthread, i);
795e6b475e0SMatt Macy 		thread_unlock(curthread);
796f263522aSJoseph Koshy 		/* return the 'current' buffer to the global pool */
797e6b475e0SMatt Macy 		if ((lb = po->po_curbuf[curcpu]) != NULL) {
798e6b475e0SMatt Macy 			PMCLOG_RESET_BUFFER_DESCRIPTOR(lb);
799e6b475e0SMatt Macy 			pmc_plb_rele(lb);
800f263522aSJoseph Koshy 		}
801e6b475e0SMatt Macy 	}
802e6b475e0SMatt Macy 	thread_lock(curthread);
803e6b475e0SMatt Macy 	sched_unbind(curthread);
804e6b475e0SMatt Macy 	thread_unlock(curthread);
805f263522aSJoseph Koshy 
806f263522aSJoseph Koshy 	/* drop a reference to the fd */
8071121a374SKonstantin Belousov 	if (po->po_file != NULL) {
808f263522aSJoseph Koshy 		error = fdrop(po->po_file, curthread);
809f263522aSJoseph Koshy 		po->po_file = NULL;
8101121a374SKonstantin Belousov 	} else
8111121a374SKonstantin Belousov 		error = 0;
812f263522aSJoseph Koshy 	po->po_error = 0;
813f263522aSJoseph Koshy 
8141ad08c6aSJoseph Koshy 	return (error);
815f263522aSJoseph Koshy }
816f263522aSJoseph Koshy 
817f263522aSJoseph Koshy /*
818f263522aSJoseph Koshy  * Flush a process' log buffer.
819f263522aSJoseph Koshy  */
820f263522aSJoseph Koshy 
821f263522aSJoseph Koshy int
822ebfaf69cSMatt Macy pmclog_flush(struct pmc_owner *po, int force)
823f263522aSJoseph Koshy {
824b44906e5SFabien Thomas 	int error;
825f263522aSJoseph Koshy 
8264a3690dfSJohn Baldwin 	PMCDBG1(LOG,FLS,1, "po=%p", po);
827f263522aSJoseph Koshy 
828f263522aSJoseph Koshy 	/*
829f263522aSJoseph Koshy 	 * If there is a pending error recorded by the logger thread,
830f263522aSJoseph Koshy 	 * return that.
831f263522aSJoseph Koshy 	 */
832f263522aSJoseph Koshy 	if (po->po_error)
8331ad08c6aSJoseph Koshy 		return (po->po_error);
834f263522aSJoseph Koshy 
835f263522aSJoseph Koshy 	error = 0;
836f263522aSJoseph Koshy 
837f263522aSJoseph Koshy 	/*
838f263522aSJoseph Koshy 	 * Check that we do have an active log file.
839f263522aSJoseph Koshy 	 */
840f263522aSJoseph Koshy 	mtx_lock(&pmc_kthread_mtx);
841f263522aSJoseph Koshy 	if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
842f263522aSJoseph Koshy 		error = EINVAL;
843f263522aSJoseph Koshy 		goto error;
844f263522aSJoseph Koshy 	}
845f263522aSJoseph Koshy 
846d9f1b8dbSMatt Macy 	pmclog_schedule_all(po);
847dceed24aSFabien Thomas  error:
848dceed24aSFabien Thomas 	mtx_unlock(&pmc_kthread_mtx);
849dceed24aSFabien Thomas 
850dceed24aSFabien Thomas 	return (error);
851dceed24aSFabien Thomas }
852dceed24aSFabien Thomas 
853e6b475e0SMatt Macy static void
854d9f1b8dbSMatt Macy pmclog_schedule_one_cond(struct pmc_owner *po)
855e6b475e0SMatt Macy {
856e6b475e0SMatt Macy 	struct pmclog_buffer *plb;
8575de96e33SMatt Macy 	int cpu;
858e6b475e0SMatt Macy 
859e6b475e0SMatt Macy 	spinlock_enter();
8605de96e33SMatt Macy 	cpu = curcpu;
861e6b475e0SMatt Macy 	/* tell hardclock not to run again */
8625de96e33SMatt Macy 	if (PMC_CPU_HAS_SAMPLES(cpu))
863d626a614SMatt Macy 		PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
864d9f1b8dbSMatt Macy 
8655de96e33SMatt Macy 	plb = po->po_curbuf[cpu];
866e6b475e0SMatt Macy 	if (plb && plb->plb_ptr != plb->plb_base)
86739446ce5SMatt Macy 		pmclog_schedule_io(po, 1);
868e6b475e0SMatt Macy 	spinlock_exit();
869e6b475e0SMatt Macy }
870e6b475e0SMatt Macy 
871e6b475e0SMatt Macy static void
872d9f1b8dbSMatt Macy pmclog_schedule_all(struct pmc_owner *po)
873e6b475e0SMatt Macy {
874e6b475e0SMatt Macy 	/*
875e6b475e0SMatt Macy 	 * Schedule the current buffer if any and not empty.
876e6b475e0SMatt Macy 	 */
877e6b475e0SMatt Macy 	for (int i = 0; i < mp_ncpus; i++) {
878e6b475e0SMatt Macy 		thread_lock(curthread);
879e6b475e0SMatt Macy 		sched_bind(curthread, i);
880e6b475e0SMatt Macy 		thread_unlock(curthread);
881d9f1b8dbSMatt Macy 		pmclog_schedule_one_cond(po);
882e6b475e0SMatt Macy 	}
883e6b475e0SMatt Macy 	thread_lock(curthread);
884e6b475e0SMatt Macy 	sched_unbind(curthread);
885e6b475e0SMatt Macy 	thread_unlock(curthread);
886e6b475e0SMatt Macy }
887e6b475e0SMatt Macy 
888dceed24aSFabien Thomas int
889dceed24aSFabien Thomas pmclog_close(struct pmc_owner *po)
890dceed24aSFabien Thomas {
891dceed24aSFabien Thomas 
8924a3690dfSJohn Baldwin 	PMCDBG1(LOG,CLO,1, "po=%p", po);
893dceed24aSFabien Thomas 
8949f4f1d4dSFabien Thomas 	pmclog_process_closelog(po);
8959f4f1d4dSFabien Thomas 
896dceed24aSFabien Thomas 	mtx_lock(&pmc_kthread_mtx);
897f2daab2cSMatt Macy 	/*
898f2daab2cSMatt Macy 	 * Initiate shutdown: no new data queued,
899f2daab2cSMatt Macy 	 * thread will close file on last block.
900f2daab2cSMatt Macy 	 */
901f2daab2cSMatt Macy 	po->po_flags |= PMC_PO_SHUTDOWN;
902f2daab2cSMatt Macy 	/* give time for all to see */
903f2daab2cSMatt Macy 	DELAY(50);
904dceed24aSFabien Thomas 
905dceed24aSFabien Thomas 	/*
906dceed24aSFabien Thomas 	 * Schedule the current buffer.
907f263522aSJoseph Koshy 	 */
908d9f1b8dbSMatt Macy 	pmclog_schedule_all(po);
909dceed24aSFabien Thomas 	wakeup_one(po);
910f263522aSJoseph Koshy 
911f263522aSJoseph Koshy 	mtx_unlock(&pmc_kthread_mtx);
912f263522aSJoseph Koshy 
913dceed24aSFabien Thomas 	return (0);
914f263522aSJoseph Koshy }
915f263522aSJoseph Koshy 
916d07f36b0SJoseph Koshy void
917d07f36b0SJoseph Koshy pmclog_process_callchain(struct pmc *pm, struct pmc_sample *ps)
918d07f36b0SJoseph Koshy {
919d07f36b0SJoseph Koshy 	int n, recordlen;
920d07f36b0SJoseph Koshy 	uint32_t flags;
921d07f36b0SJoseph Koshy 	struct pmc_owner *po;
922d07f36b0SJoseph Koshy 
9234a3690dfSJohn Baldwin 	PMCDBG3(LOG,SAM,1,"pm=%p pid=%d n=%d", pm, ps->ps_pid,
924d07f36b0SJoseph Koshy 	    ps->ps_nsamples);
925d07f36b0SJoseph Koshy 
926d07f36b0SJoseph Koshy 	recordlen = offsetof(struct pmclog_callchain, pl_pc) +
927d07f36b0SJoseph Koshy 	    ps->ps_nsamples * sizeof(uintfptr_t);
928d07f36b0SJoseph Koshy 	po = pm->pm_owner;
929d07f36b0SJoseph Koshy 	flags = PMC_CALLCHAIN_TO_CPUFLAGS(ps->ps_cpu,ps->ps_flags);
930f992dd4bSMatt Macy 	PMCLOG_RESERVE_SAFE(po, CALLCHAIN, recordlen, ps->ps_tsc);
931d07f36b0SJoseph Koshy 	PMCLOG_EMIT32(ps->ps_pid);
932a85289cfSMatt Macy 	PMCLOG_EMIT32(ps->ps_tid);
933d07f36b0SJoseph Koshy 	PMCLOG_EMIT32(pm->pm_id);
934d07f36b0SJoseph Koshy 	PMCLOG_EMIT32(flags);
935d07f36b0SJoseph Koshy 	for (n = 0; n < ps->ps_nsamples; n++)
936d07f36b0SJoseph Koshy 		PMCLOG_EMITADDR(ps->ps_pc[n]);
937e6b475e0SMatt Macy 	PMCLOG_DESPATCH_SAFE(po);
938d07f36b0SJoseph Koshy }
939f263522aSJoseph Koshy 
940f263522aSJoseph Koshy void
941f263522aSJoseph Koshy pmclog_process_closelog(struct pmc_owner *po)
942f263522aSJoseph Koshy {
943f263522aSJoseph Koshy 	PMCLOG_RESERVE(po,CLOSELOG,sizeof(struct pmclog_closelog));
944e6b475e0SMatt Macy 	PMCLOG_DESPATCH_SYNC(po);
945f263522aSJoseph Koshy }
946f263522aSJoseph Koshy 
947f263522aSJoseph Koshy void
948f263522aSJoseph Koshy pmclog_process_dropnotify(struct pmc_owner *po)
949f263522aSJoseph Koshy {
950f263522aSJoseph Koshy 	PMCLOG_RESERVE(po,DROPNOTIFY,sizeof(struct pmclog_dropnotify));
951f263522aSJoseph Koshy 	PMCLOG_DESPATCH(po);
952f263522aSJoseph Koshy }
953f263522aSJoseph Koshy 
954f263522aSJoseph Koshy void
95549874f6eSJoseph Koshy pmclog_process_map_in(struct pmc_owner *po, pid_t pid, uintfptr_t start,
95649874f6eSJoseph Koshy     const char *path)
957f263522aSJoseph Koshy {
958f263522aSJoseph Koshy 	int pathlen, recordlen;
959f263522aSJoseph Koshy 
96049874f6eSJoseph Koshy 	KASSERT(path != NULL, ("[pmclog,%d] map-in, null path", __LINE__));
96149874f6eSJoseph Koshy 
962f263522aSJoseph Koshy 	pathlen = strlen(path) + 1;	/* #bytes for path name */
96349874f6eSJoseph Koshy 	recordlen = offsetof(struct pmclog_map_in, pl_pathname) +
964f263522aSJoseph Koshy 	    pathlen;
965f263522aSJoseph Koshy 
96649874f6eSJoseph Koshy 	PMCLOG_RESERVE(po, MAP_IN, recordlen);
967f263522aSJoseph Koshy 	PMCLOG_EMIT32(pid);
968f992dd4bSMatt Macy 	PMCLOG_EMIT32(0);
96949874f6eSJoseph Koshy 	PMCLOG_EMITADDR(start);
970f263522aSJoseph Koshy 	PMCLOG_EMITSTRING(path,pathlen);
9712ce69a4dSMatt Macy 	PMCLOG_DESPATCH_SYNC(po);
972f263522aSJoseph Koshy }
973f263522aSJoseph Koshy 
97449874f6eSJoseph Koshy void
97549874f6eSJoseph Koshy pmclog_process_map_out(struct pmc_owner *po, pid_t pid, uintfptr_t start,
97649874f6eSJoseph Koshy     uintfptr_t end)
97749874f6eSJoseph Koshy {
97849874f6eSJoseph Koshy 	KASSERT(start <= end, ("[pmclog,%d] start > end", __LINE__));
97949874f6eSJoseph Koshy 
98049874f6eSJoseph Koshy 	PMCLOG_RESERVE(po, MAP_OUT, sizeof(struct pmclog_map_out));
98149874f6eSJoseph Koshy 	PMCLOG_EMIT32(pid);
982f992dd4bSMatt Macy 	PMCLOG_EMIT32(0);
98349874f6eSJoseph Koshy 	PMCLOG_EMITADDR(start);
98449874f6eSJoseph Koshy 	PMCLOG_EMITADDR(end);
98549874f6eSJoseph Koshy 	PMCLOG_DESPATCH(po);
98649874f6eSJoseph Koshy }
987f263522aSJoseph Koshy 
988f263522aSJoseph Koshy void
989f263522aSJoseph Koshy pmclog_process_pmcallocate(struct pmc *pm)
990f263522aSJoseph Koshy {
991f263522aSJoseph Koshy 	struct pmc_owner *po;
992f5f9340bSFabien Thomas 	struct pmc_soft *ps;
993f263522aSJoseph Koshy 
994f263522aSJoseph Koshy 	po = pm->pm_owner;
995f263522aSJoseph Koshy 
9964a3690dfSJohn Baldwin 	PMCDBG1(LOG,ALL,1, "pm=%p", pm);
997f263522aSJoseph Koshy 
998f5f9340bSFabien Thomas 	if (PMC_TO_CLASS(pm) == PMC_CLASS_SOFT) {
999f5f9340bSFabien Thomas 		PMCLOG_RESERVE(po, PMCALLOCATEDYN,
1000f5f9340bSFabien Thomas 		    sizeof(struct pmclog_pmcallocatedyn));
1001f5f9340bSFabien Thomas 		PMCLOG_EMIT32(pm->pm_id);
1002f5f9340bSFabien Thomas 		PMCLOG_EMIT32(pm->pm_event);
1003f5f9340bSFabien Thomas 		PMCLOG_EMIT32(pm->pm_flags);
1004f992dd4bSMatt Macy 		PMCLOG_EMIT32(0);
1005b2ca2e50SMatt Macy 		PMCLOG_EMIT64(pm->pm_sc.pm_reloadcount);
1006f5f9340bSFabien Thomas 		ps = pmc_soft_ev_acquire(pm->pm_event);
1007f5f9340bSFabien Thomas 		if (ps != NULL)
1008f5f9340bSFabien Thomas 			PMCLOG_EMITSTRING(ps->ps_ev.pm_ev_name,PMC_NAME_MAX);
1009f5f9340bSFabien Thomas 		else
1010f5f9340bSFabien Thomas 			PMCLOG_EMITNULLSTRING(PMC_NAME_MAX);
1011f5f9340bSFabien Thomas 		pmc_soft_ev_release(ps);
1012e6b475e0SMatt Macy 		PMCLOG_DESPATCH_SYNC(po);
1013f5f9340bSFabien Thomas 	} else {
1014f5f9340bSFabien Thomas 		PMCLOG_RESERVE(po, PMCALLOCATE,
1015f5f9340bSFabien Thomas 		    sizeof(struct pmclog_pmcallocate));
1016f263522aSJoseph Koshy 		PMCLOG_EMIT32(pm->pm_id);
1017f263522aSJoseph Koshy 		PMCLOG_EMIT32(pm->pm_event);
1018f263522aSJoseph Koshy 		PMCLOG_EMIT32(pm->pm_flags);
1019f992dd4bSMatt Macy 		PMCLOG_EMIT32(0);
1020b2ca2e50SMatt Macy 		PMCLOG_EMIT64(pm->pm_sc.pm_reloadcount);
1021e6b475e0SMatt Macy 		PMCLOG_DESPATCH_SYNC(po);
1022f263522aSJoseph Koshy 	}
1023f5f9340bSFabien Thomas }
1024f263522aSJoseph Koshy 
1025f263522aSJoseph Koshy void
1026f263522aSJoseph Koshy pmclog_process_pmcattach(struct pmc *pm, pid_t pid, char *path)
1027f263522aSJoseph Koshy {
1028f263522aSJoseph Koshy 	int pathlen, recordlen;
1029f263522aSJoseph Koshy 	struct pmc_owner *po;
1030f263522aSJoseph Koshy 
10314a3690dfSJohn Baldwin 	PMCDBG2(LOG,ATT,1,"pm=%p pid=%d", pm, pid);
1032f263522aSJoseph Koshy 
1033f263522aSJoseph Koshy 	po = pm->pm_owner;
1034f263522aSJoseph Koshy 
1035f263522aSJoseph Koshy 	pathlen = strlen(path) + 1;	/* #bytes for the string */
1036f263522aSJoseph Koshy 	recordlen = offsetof(struct pmclog_pmcattach, pl_pathname) + pathlen;
1037f263522aSJoseph Koshy 
1038f263522aSJoseph Koshy 	PMCLOG_RESERVE(po, PMCATTACH, recordlen);
1039f263522aSJoseph Koshy 	PMCLOG_EMIT32(pm->pm_id);
1040f263522aSJoseph Koshy 	PMCLOG_EMIT32(pid);
1041f263522aSJoseph Koshy 	PMCLOG_EMITSTRING(path, pathlen);
1042e6b475e0SMatt Macy 	PMCLOG_DESPATCH_SYNC(po);
1043f263522aSJoseph Koshy }
1044f263522aSJoseph Koshy 
1045f263522aSJoseph Koshy void
1046f263522aSJoseph Koshy pmclog_process_pmcdetach(struct pmc *pm, pid_t pid)
1047f263522aSJoseph Koshy {
1048f263522aSJoseph Koshy 	struct pmc_owner *po;
1049f263522aSJoseph Koshy 
10504a3690dfSJohn Baldwin 	PMCDBG2(LOG,ATT,1,"!pm=%p pid=%d", pm, pid);
1051f263522aSJoseph Koshy 
1052f263522aSJoseph Koshy 	po = pm->pm_owner;
1053f263522aSJoseph Koshy 
1054f263522aSJoseph Koshy 	PMCLOG_RESERVE(po, PMCDETACH, sizeof(struct pmclog_pmcdetach));
1055f263522aSJoseph Koshy 	PMCLOG_EMIT32(pm->pm_id);
1056f263522aSJoseph Koshy 	PMCLOG_EMIT32(pid);
1057e6b475e0SMatt Macy 	PMCLOG_DESPATCH_SYNC(po);
1058f263522aSJoseph Koshy }
1059f263522aSJoseph Koshy 
1060ebfaf69cSMatt Macy void
1061ebfaf69cSMatt Macy pmclog_process_proccreate(struct pmc_owner *po, struct proc *p, int sync)
1062ebfaf69cSMatt Macy {
1063ebfaf69cSMatt Macy 	if (sync) {
1064ebfaf69cSMatt Macy 		PMCLOG_RESERVE(po, PROC_CREATE, sizeof(struct pmclog_proccreate));
1065ebfaf69cSMatt Macy 		PMCLOG_EMIT32(p->p_pid);
1066b2ca2e50SMatt Macy 		PMCLOG_EMIT32(p->p_flag);
1067ebfaf69cSMatt Macy 		PMCLOG_EMITSTRING(p->p_comm, MAXCOMLEN+1);
1068ebfaf69cSMatt Macy 		PMCLOG_DESPATCH_SYNC(po);
1069ebfaf69cSMatt Macy 	} else {
1070ebfaf69cSMatt Macy 		PMCLOG_RESERVE(po, PROC_CREATE, sizeof(struct pmclog_proccreate));
1071ebfaf69cSMatt Macy 		PMCLOG_EMIT32(p->p_pid);
1072b2ca2e50SMatt Macy 		PMCLOG_EMIT32(p->p_flag);
1073ebfaf69cSMatt Macy 		PMCLOG_EMITSTRING(p->p_comm, MAXCOMLEN+1);
1074ebfaf69cSMatt Macy 		PMCLOG_DESPATCH(po);
1075ebfaf69cSMatt Macy 	}
1076ebfaf69cSMatt Macy }
1077ebfaf69cSMatt Macy 
1078f263522aSJoseph Koshy /*
1079f263522aSJoseph Koshy  * Log a context switch event to the log file.
1080f263522aSJoseph Koshy  */
1081f263522aSJoseph Koshy 
1082f263522aSJoseph Koshy void
1083a85289cfSMatt Macy pmclog_process_proccsw(struct pmc *pm, struct pmc_process *pp, pmc_value_t v, struct thread *td)
1084f263522aSJoseph Koshy {
1085f263522aSJoseph Koshy 	struct pmc_owner *po;
1086f263522aSJoseph Koshy 
1087f263522aSJoseph Koshy 	KASSERT(pm->pm_flags & PMC_F_LOG_PROCCSW,
1088f263522aSJoseph Koshy 	    ("[pmclog,%d] log-process-csw called gratuitously", __LINE__));
1089f263522aSJoseph Koshy 
10904a3690dfSJohn Baldwin 	PMCDBG3(LOG,SWO,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
1091f263522aSJoseph Koshy 	    v);
1092f263522aSJoseph Koshy 
1093f263522aSJoseph Koshy 	po = pm->pm_owner;
1094f263522aSJoseph Koshy 
1095f992dd4bSMatt Macy 	PMCLOG_RESERVE_SAFE(po, PROCCSW, sizeof(struct pmclog_proccsw), pmc_rdtsc());
1096f263522aSJoseph Koshy 	PMCLOG_EMIT64(v);
1097f992dd4bSMatt Macy 	PMCLOG_EMIT32(pm->pm_id);
1098f263522aSJoseph Koshy 	PMCLOG_EMIT32(pp->pp_proc->p_pid);
1099a85289cfSMatt Macy 	PMCLOG_EMIT32(td->td_tid);
1100f992dd4bSMatt Macy 	PMCLOG_EMIT32(0);
110139446ce5SMatt Macy 	PMCLOG_DESPATCH_SCHED_LOCK(po);
1102f263522aSJoseph Koshy }
1103f263522aSJoseph Koshy 
1104f263522aSJoseph Koshy void
110515139246SJoseph Koshy pmclog_process_procexec(struct pmc_owner *po, pmc_id_t pmid, pid_t pid,
110615139246SJoseph Koshy     uintfptr_t startaddr, char *path)
1107f263522aSJoseph Koshy {
1108f263522aSJoseph Koshy 	int pathlen, recordlen;
1109f263522aSJoseph Koshy 
11104a3690dfSJohn Baldwin 	PMCDBG3(LOG,EXC,1,"po=%p pid=%d path=\"%s\"", po, pid, path);
1111f263522aSJoseph Koshy 
1112f263522aSJoseph Koshy 	pathlen   = strlen(path) + 1;	/* #bytes for the path */
1113f263522aSJoseph Koshy 	recordlen = offsetof(struct pmclog_procexec, pl_pathname) + pathlen;
1114f263522aSJoseph Koshy 	PMCLOG_RESERVE(po, PROCEXEC, recordlen);
1115f263522aSJoseph Koshy 	PMCLOG_EMIT32(pid);
11165506ceb8SMatt Macy 	PMCLOG_EMIT32(pmid);
1117959826caSMatt Macy 	PMCLOG_EMITADDR(startaddr);
1118f263522aSJoseph Koshy 	PMCLOG_EMITSTRING(path,pathlen);
1119ebfaf69cSMatt Macy 	PMCLOG_DESPATCH_SYNC(po);
1120f263522aSJoseph Koshy }
1121f263522aSJoseph Koshy 
1122f263522aSJoseph Koshy /*
1123f263522aSJoseph Koshy  * Log a process exit event (and accumulated pmc value) to the log file.
1124f263522aSJoseph Koshy  */
1125f263522aSJoseph Koshy 
1126f263522aSJoseph Koshy void
1127f263522aSJoseph Koshy pmclog_process_procexit(struct pmc *pm, struct pmc_process *pp)
1128f263522aSJoseph Koshy {
1129f263522aSJoseph Koshy 	int ri;
1130f263522aSJoseph Koshy 	struct pmc_owner *po;
1131f263522aSJoseph Koshy 
1132f263522aSJoseph Koshy 	ri = PMC_TO_ROWINDEX(pm);
11334a3690dfSJohn Baldwin 	PMCDBG3(LOG,EXT,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
1134f263522aSJoseph Koshy 	    pp->pp_pmcs[ri].pp_pmcval);
1135f263522aSJoseph Koshy 
1136f263522aSJoseph Koshy 	po = pm->pm_owner;
1137f263522aSJoseph Koshy 
1138f263522aSJoseph Koshy 	PMCLOG_RESERVE(po, PROCEXIT, sizeof(struct pmclog_procexit));
1139f263522aSJoseph Koshy 	PMCLOG_EMIT32(pm->pm_id);
11405506ceb8SMatt Macy 	PMCLOG_EMIT32(pp->pp_proc->p_pid);
1141959826caSMatt Macy 	PMCLOG_EMIT64(pp->pp_pmcs[ri].pp_pmcval);
1142f263522aSJoseph Koshy 	PMCLOG_DESPATCH(po);
1143f263522aSJoseph Koshy }
1144f263522aSJoseph Koshy 
1145f263522aSJoseph Koshy /*
1146f263522aSJoseph Koshy  * Log a fork event.
1147f263522aSJoseph Koshy  */
1148f263522aSJoseph Koshy 
1149f263522aSJoseph Koshy void
1150f263522aSJoseph Koshy pmclog_process_procfork(struct pmc_owner *po, pid_t oldpid, pid_t newpid)
1151f263522aSJoseph Koshy {
1152f263522aSJoseph Koshy 	PMCLOG_RESERVE(po, PROCFORK, sizeof(struct pmclog_procfork));
1153f263522aSJoseph Koshy 	PMCLOG_EMIT32(oldpid);
1154f263522aSJoseph Koshy 	PMCLOG_EMIT32(newpid);
1155f263522aSJoseph Koshy 	PMCLOG_DESPATCH(po);
1156f263522aSJoseph Koshy }
1157f263522aSJoseph Koshy 
1158f263522aSJoseph Koshy /*
1159f263522aSJoseph Koshy  * Log a process exit event of the form suitable for system-wide PMCs.
1160f263522aSJoseph Koshy  */
1161f263522aSJoseph Koshy 
1162f263522aSJoseph Koshy void
1163f263522aSJoseph Koshy pmclog_process_sysexit(struct pmc_owner *po, pid_t pid)
1164f263522aSJoseph Koshy {
1165f263522aSJoseph Koshy 	PMCLOG_RESERVE(po, SYSEXIT, sizeof(struct pmclog_sysexit));
1166f263522aSJoseph Koshy 	PMCLOG_EMIT32(pid);
1167f263522aSJoseph Koshy 	PMCLOG_DESPATCH(po);
1168f263522aSJoseph Koshy }
1169f263522aSJoseph Koshy 
1170ebfaf69cSMatt Macy void
1171ebfaf69cSMatt Macy pmclog_process_threadcreate(struct pmc_owner *po, struct thread *td, int sync)
1172ebfaf69cSMatt Macy {
1173ebfaf69cSMatt Macy 	struct proc *p;
1174ebfaf69cSMatt Macy 
1175ebfaf69cSMatt Macy 	p = td->td_proc;
1176ebfaf69cSMatt Macy 	if (sync) {
1177ebfaf69cSMatt Macy 		PMCLOG_RESERVE(po, THR_CREATE, sizeof(struct pmclog_threadcreate));
1178ebfaf69cSMatt Macy 		PMCLOG_EMIT32(td->td_tid);
1179ebfaf69cSMatt Macy 		PMCLOG_EMIT32(p->p_pid);
1180b2ca2e50SMatt Macy 		PMCLOG_EMIT32(p->p_flag);
1181f992dd4bSMatt Macy 		PMCLOG_EMIT32(0);
1182ebfaf69cSMatt Macy 		PMCLOG_EMITSTRING(td->td_name, MAXCOMLEN+1);
1183ebfaf69cSMatt Macy 		PMCLOG_DESPATCH_SYNC(po);
1184ebfaf69cSMatt Macy 	} else {
1185ebfaf69cSMatt Macy 		PMCLOG_RESERVE(po, THR_CREATE, sizeof(struct pmclog_threadcreate));
1186ebfaf69cSMatt Macy 		PMCLOG_EMIT32(td->td_tid);
1187ebfaf69cSMatt Macy 		PMCLOG_EMIT32(p->p_pid);
1188b2ca2e50SMatt Macy 		PMCLOG_EMIT32(p->p_flag);
1189f992dd4bSMatt Macy 		PMCLOG_EMIT32(0);
1190ebfaf69cSMatt Macy 		PMCLOG_EMITSTRING(td->td_name, MAXCOMLEN+1);
1191ebfaf69cSMatt Macy 		PMCLOG_DESPATCH(po);
1192ebfaf69cSMatt Macy 	}
1193ebfaf69cSMatt Macy }
1194ebfaf69cSMatt Macy 
1195ebfaf69cSMatt Macy void
1196ebfaf69cSMatt Macy pmclog_process_threadexit(struct pmc_owner *po, struct thread *td)
1197ebfaf69cSMatt Macy {
1198ebfaf69cSMatt Macy 
1199ebfaf69cSMatt Macy 	PMCLOG_RESERVE(po, THR_EXIT, sizeof(struct pmclog_threadexit));
1200ebfaf69cSMatt Macy 	PMCLOG_EMIT32(td->td_tid);
1201ebfaf69cSMatt Macy 	PMCLOG_DESPATCH(po);
1202ebfaf69cSMatt Macy }
1203ebfaf69cSMatt Macy 
1204f263522aSJoseph Koshy /*
1205f263522aSJoseph Koshy  * Write a user log entry.
1206f263522aSJoseph Koshy  */
1207f263522aSJoseph Koshy 
1208f263522aSJoseph Koshy int
1209f263522aSJoseph Koshy pmclog_process_userlog(struct pmc_owner *po, struct pmc_op_writelog *wl)
1210f263522aSJoseph Koshy {
1211f263522aSJoseph Koshy 	int error;
1212f263522aSJoseph Koshy 
12134a3690dfSJohn Baldwin 	PMCDBG2(LOG,WRI,1, "writelog po=%p ud=0x%x", po, wl->pm_userdata);
1214f263522aSJoseph Koshy 
1215f263522aSJoseph Koshy 	error = 0;
1216f263522aSJoseph Koshy 
1217f263522aSJoseph Koshy 	PMCLOG_RESERVE_WITH_ERROR(po, USERDATA,
1218f263522aSJoseph Koshy 	    sizeof(struct pmclog_userdata));
1219f263522aSJoseph Koshy 	PMCLOG_EMIT32(wl->pm_userdata);
1220f263522aSJoseph Koshy 	PMCLOG_DESPATCH(po);
1221f263522aSJoseph Koshy 
1222f263522aSJoseph Koshy  error:
12231ad08c6aSJoseph Koshy 	return (error);
1224f263522aSJoseph Koshy }
1225f263522aSJoseph Koshy 
1226f263522aSJoseph Koshy /*
1227f263522aSJoseph Koshy  * Initialization.
1228f263522aSJoseph Koshy  *
1229f263522aSJoseph Koshy  * Create a pool of log buffers and initialize mutexes.
1230f263522aSJoseph Koshy  */
1231f263522aSJoseph Koshy 
1232f263522aSJoseph Koshy void
1233f263522aSJoseph Koshy pmclog_initialize()
1234f263522aSJoseph Koshy {
1235f263522aSJoseph Koshy 	struct pmclog_buffer *plb;
1236*9978bd99SMark Johnston 	int domain, ncpus, total;
1237f263522aSJoseph Koshy 
1238e6b475e0SMatt Macy 	if (pmclog_buffer_size <= 0 || pmclog_buffer_size > 16*1024) {
12391ad08c6aSJoseph Koshy 		(void) printf("hwpmc: tunable logbuffersize=%d must be "
1240e6b475e0SMatt Macy 					  "greater than zero and less than or equal to 16MB.\n",
1241e6b475e0SMatt Macy 					  pmclog_buffer_size);
1242f263522aSJoseph Koshy 		pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
1243f263522aSJoseph Koshy 	}
1244f263522aSJoseph Koshy 
1245e6b475e0SMatt Macy 	if (pmc_nlogbuffers_pcpu <= 0) {
1246f263522aSJoseph Koshy 		(void) printf("hwpmc: tunable nlogbuffers=%d must be greater "
1247e6b475e0SMatt Macy 					  "than zero.\n", pmc_nlogbuffers_pcpu);
1248e6b475e0SMatt Macy 		pmc_nlogbuffers_pcpu = PMC_NLOGBUFFERS_PCPU;
1249f263522aSJoseph Koshy 	}
1250e6b475e0SMatt Macy 	if (pmc_nlogbuffers_pcpu*pmclog_buffer_size > 32*1024) {
1251e6b475e0SMatt Macy 		(void) printf("hwpmc: memory allocated pcpu must be less than 32MB (is %dK).\n",
1252e6b475e0SMatt Macy 					  pmc_nlogbuffers_pcpu*pmclog_buffer_size);
1253e6b475e0SMatt Macy 		pmc_nlogbuffers_pcpu = PMC_NLOGBUFFERS_PCPU;
1254e6b475e0SMatt Macy 		pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
1255f263522aSJoseph Koshy 	}
125672ac73faSMatt Macy 	for (domain = 0; domain < vm_ndomains; domain++) {
1257*9978bd99SMark Johnston 		ncpus = pmc_dom_hdrs[domain]->pdbh_ncpus;
1258*9978bd99SMark Johnston 		total = ncpus * pmc_nlogbuffers_pcpu;
1259e6b475e0SMatt Macy 
1260*9978bd99SMark Johnston 		plb = malloc_domainset(sizeof(struct pmclog_buffer) * total,
1261*9978bd99SMark Johnston 		    M_PMC, DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
1262e6b475e0SMatt Macy 		pmc_dom_hdrs[domain]->pdbh_plbs = plb;
1263*9978bd99SMark Johnston 		for (; total > 0; total--, plb++) {
1264e6b475e0SMatt Macy 			void *buf;
1265e6b475e0SMatt Macy 
1266*9978bd99SMark Johnston 			buf = malloc_domainset(1024 * pmclog_buffer_size, M_PMC,
1267*9978bd99SMark Johnston 			    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
1268e6b475e0SMatt Macy 			PMCLOG_INIT_BUFFER_DESCRIPTOR(plb, buf, domain);
1269e6b475e0SMatt Macy 			pmc_plb_rele_unlocked(plb);
1270e6b475e0SMatt Macy 		}
1271e6b475e0SMatt Macy 	}
1272382d30cdSJoseph Koshy 	mtx_init(&pmc_kthread_mtx, "pmc-kthread", "pmc-sleep", MTX_DEF);
1273f263522aSJoseph Koshy }
1274f263522aSJoseph Koshy 
1275f263522aSJoseph Koshy /*
1276f263522aSJoseph Koshy  * Shutdown logging.
1277f263522aSJoseph Koshy  *
1278f263522aSJoseph Koshy  * Destroy mutexes and release memory back the to free pool.
1279f263522aSJoseph Koshy  */
1280f263522aSJoseph Koshy 
1281f263522aSJoseph Koshy void
1282f263522aSJoseph Koshy pmclog_shutdown()
1283f263522aSJoseph Koshy {
1284f263522aSJoseph Koshy 	struct pmclog_buffer *plb;
1285e6b475e0SMatt Macy 	int domain;
1286f263522aSJoseph Koshy 
1287f263522aSJoseph Koshy 	mtx_destroy(&pmc_kthread_mtx);
1288f263522aSJoseph Koshy 
128972ac73faSMatt Macy 	for (domain = 0; domain < vm_ndomains; domain++) {
1290e6b475e0SMatt Macy 		while ((plb = TAILQ_FIRST(&pmc_dom_hdrs[domain]->pdbh_head)) != NULL) {
1291e6b475e0SMatt Macy 			TAILQ_REMOVE(&pmc_dom_hdrs[domain]->pdbh_head, plb, plb_next);
1292e6b475e0SMatt Macy 			free(plb->plb_base, M_PMC);
1293e6b475e0SMatt Macy 		}
1294e6b475e0SMatt Macy 		free(pmc_dom_hdrs[domain]->pdbh_plbs, M_PMC);
1295f263522aSJoseph Koshy 	}
1296f263522aSJoseph Koshy }
1297