xref: /freebsd/sys/dev/hwpmc/hwpmc_logging.c (revision 56e53cb8ef000c3ef72337a4095987a932cdedef)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2005-2007 Joseph Koshy
5  * Copyright (c) 2007 The FreeBSD Foundation
6  * All rights reserved.
7  *
8  * Portions of this software were developed by A. Joseph Koshy under
9  * sponsorship from the FreeBSD Foundation and Google, Inc.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  *
32  */
33 
34 /*
35  * Logging code for hwpmc(4)
36  */
37 
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 #include <sys/param.h>
42 #include <sys/capsicum.h>
43 #include <sys/file.h>
44 #include <sys/kernel.h>
45 #include <sys/kthread.h>
46 #include <sys/lock.h>
47 #include <sys/module.h>
48 #include <sys/mutex.h>
49 #include <sys/pmc.h>
50 #include <sys/pmckern.h>
51 #include <sys/pmclog.h>
52 #include <sys/proc.h>
53 #include <sys/signalvar.h>
54 #include <sys/syscallsubr.h>
55 #include <sys/sysctl.h>
56 #include <sys/systm.h>
57 #include <sys/uio.h>
58 #include <sys/unistd.h>
59 #include <sys/vnode.h>
60 
61 /*
62  * Sysctl tunables
63  */
64 
65 SYSCTL_DECL(_kern_hwpmc);
66 
67 /*
68  * kern.hwpmc.logbuffersize -- size of the per-cpu owner buffers.
69  */
70 
71 static int pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
72 #if (__FreeBSD_version < 1100000)
73 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "logbuffersize", &pmclog_buffer_size);
74 #endif
75 SYSCTL_INT(_kern_hwpmc, OID_AUTO, logbuffersize, CTLFLAG_RDTUN,
76     &pmclog_buffer_size, 0, "size of log buffers in kilobytes");
77 
78 /*
79  * kern.hwpmc.nbuffer -- number of global log buffers
80  */
81 
82 static int pmc_nlogbuffers = PMC_NLOGBUFFERS;
83 #if (__FreeBSD_version < 1100000)
84 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nbuffers", &pmc_nlogbuffers);
85 #endif
86 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nbuffers, CTLFLAG_RDTUN,
87     &pmc_nlogbuffers, 0, "number of global log buffers");
88 
89 /*
90  * Global log buffer list and associated spin lock.
91  */
92 
93 TAILQ_HEAD(, pmclog_buffer) pmc_bufferlist =
94 	TAILQ_HEAD_INITIALIZER(pmc_bufferlist);
95 static struct mtx pmc_bufferlist_mtx;	/* spin lock */
96 static struct mtx pmc_kthread_mtx;	/* sleep lock */
97 
98 #define	PMCLOG_INIT_BUFFER_DESCRIPTOR(D) do {				\
99 		const int __roundup = roundup(sizeof(*D),		\
100 			sizeof(uint32_t));				\
101 		(D)->plb_fence = ((char *) (D)) +			\
102 			 1024*pmclog_buffer_size;			\
103 		(D)->plb_base  = (D)->plb_ptr = ((char *) (D)) +	\
104 			__roundup;					\
105 	} while (0)
106 
107 
108 /*
109  * Log file record constructors.
110  */
111 #define	_PMCLOG_TO_HEADER(T,L)						\
112 	((PMCLOG_HEADER_MAGIC << 24) |					\
113 	 (PMCLOG_TYPE_ ## T << 16)   |					\
114 	 ((L) & 0xFFFF))
115 
116 /* reserve LEN bytes of space and initialize the entry header */
117 #define	_PMCLOG_RESERVE(PO,TYPE,LEN,ACTION) do {			\
118 		uint32_t *_le;						\
119 		int _len = roundup((LEN), sizeof(uint32_t));		\
120 		if ((_le = pmclog_reserve((PO), _len)) == NULL) {	\
121 			ACTION;						\
122 		}							\
123 		*_le = _PMCLOG_TO_HEADER(TYPE,_len);			\
124 		_le += 3	/* skip over timestamp */
125 
126 #define	PMCLOG_RESERVE(P,T,L)		_PMCLOG_RESERVE(P,T,L,return)
127 #define	PMCLOG_RESERVE_WITH_ERROR(P,T,L) _PMCLOG_RESERVE(P,T,L,		\
128 	error=ENOMEM;goto error)
129 
130 #define	PMCLOG_EMIT32(V)	do { *_le++ = (V); } while (0)
131 #define	PMCLOG_EMIT64(V)	do { 					\
132 		*_le++ = (uint32_t) ((V) & 0xFFFFFFFF);			\
133 		*_le++ = (uint32_t) (((V) >> 32) & 0xFFFFFFFF);		\
134 	} while (0)
135 
136 
137 /* Emit a string.  Caution: does NOT update _le, so needs to be last */
138 #define	PMCLOG_EMITSTRING(S,L)	do { bcopy((S), _le, (L)); } while (0)
139 #define	PMCLOG_EMITNULLSTRING(L) do { bzero(_le, (L)); } while (0)
140 
141 #define	PMCLOG_DESPATCH(PO)						\
142 		pmclog_release((PO));					\
143 	} while (0)
144 
145 
146 /*
147  * Assertions about the log file format.
148  */
149 
150 CTASSERT(sizeof(struct pmclog_callchain) == 6*4 +
151     PMC_CALLCHAIN_DEPTH_MAX*sizeof(uintfptr_t));
152 CTASSERT(sizeof(struct pmclog_closelog) == 3*4);
153 CTASSERT(sizeof(struct pmclog_dropnotify) == 3*4);
154 CTASSERT(sizeof(struct pmclog_map_in) == PATH_MAX +
155     4*4 + sizeof(uintfptr_t));
156 CTASSERT(offsetof(struct pmclog_map_in,pl_pathname) ==
157     4*4 + sizeof(uintfptr_t));
158 CTASSERT(sizeof(struct pmclog_map_out) == 4*4 + 2*sizeof(uintfptr_t));
159 CTASSERT(sizeof(struct pmclog_pcsample) == 6*4 + sizeof(uintfptr_t));
160 CTASSERT(sizeof(struct pmclog_pmcallocate) == 6*4);
161 CTASSERT(sizeof(struct pmclog_pmcattach) == 5*4 + PATH_MAX);
162 CTASSERT(offsetof(struct pmclog_pmcattach,pl_pathname) == 5*4);
163 CTASSERT(sizeof(struct pmclog_pmcdetach) == 5*4);
164 CTASSERT(sizeof(struct pmclog_proccsw) == 5*4 + 8);
165 CTASSERT(sizeof(struct pmclog_procexec) == 5*4 + PATH_MAX +
166     sizeof(uintfptr_t));
167 CTASSERT(offsetof(struct pmclog_procexec,pl_pathname) == 5*4 +
168     sizeof(uintfptr_t));
169 CTASSERT(sizeof(struct pmclog_procexit) == 5*4 + 8);
170 CTASSERT(sizeof(struct pmclog_procfork) == 5*4);
171 CTASSERT(sizeof(struct pmclog_sysexit) == 4*4);
172 CTASSERT(sizeof(struct pmclog_userdata) == 4*4);
173 
174 /*
175  * Log buffer structure
176  */
177 
178 struct pmclog_buffer {
179 	TAILQ_ENTRY(pmclog_buffer) plb_next;
180 	char 		*plb_base;
181 	char		*plb_ptr;
182 	char 		*plb_fence;
183 };
184 
185 /*
186  * Prototypes
187  */
188 
189 static int pmclog_get_buffer(struct pmc_owner *po);
190 static void pmclog_loop(void *arg);
191 static void pmclog_release(struct pmc_owner *po);
192 static uint32_t *pmclog_reserve(struct pmc_owner *po, int length);
193 static void pmclog_schedule_io(struct pmc_owner *po);
194 static void pmclog_stop_kthread(struct pmc_owner *po);
195 
196 /*
197  * Helper functions
198  */
199 
200 /*
201  * Get a log buffer
202  */
203 
204 static int
205 pmclog_get_buffer(struct pmc_owner *po)
206 {
207 	struct pmclog_buffer *plb;
208 
209 	mtx_assert(&po->po_mtx, MA_OWNED);
210 
211 	KASSERT(po->po_curbuf == NULL,
212 	    ("[pmclog,%d] po=%p current buffer still valid", __LINE__, po));
213 
214 	mtx_lock_spin(&pmc_bufferlist_mtx);
215 	if ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL)
216 		TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next);
217 	mtx_unlock_spin(&pmc_bufferlist_mtx);
218 
219 	PMCDBG2(LOG,GTB,1, "po=%p plb=%p", po, plb);
220 
221 #ifdef	HWPMC_DEBUG
222 	if (plb)
223 		KASSERT(plb->plb_ptr == plb->plb_base &&
224 		    plb->plb_base < plb->plb_fence,
225 		    ("[pmclog,%d] po=%p buffer invariants: ptr=%p "
226 		    "base=%p fence=%p", __LINE__, po, plb->plb_ptr,
227 		    plb->plb_base, plb->plb_fence));
228 #endif
229 
230 	po->po_curbuf = plb;
231 
232 	/* update stats */
233 	atomic_add_int(&pmc_stats.pm_buffer_requests, 1);
234 	if (plb == NULL)
235 		atomic_add_int(&pmc_stats.pm_buffer_requests_failed, 1);
236 
237 	return (plb ? 0 : ENOMEM);
238 }
239 
240 struct pmclog_proc_init_args {
241 	struct proc *kthr;
242 	struct pmc_owner *po;
243 	bool exit;
244 	bool acted;
245 };
246 
247 int
248 pmclog_proc_create(struct thread *td, void **handlep)
249 {
250 	struct pmclog_proc_init_args *ia;
251 	int error;
252 
253 	ia = malloc(sizeof(*ia), M_TEMP, M_WAITOK | M_ZERO);
254 	error = kproc_create(pmclog_loop, ia, &ia->kthr,
255 	    RFHIGHPID, 0, "hwpmc: proc(%d)", td->td_proc->p_pid);
256 	if (error == 0)
257 		*handlep = ia;
258 	return (error);
259 }
260 
261 void
262 pmclog_proc_ignite(void *handle, struct pmc_owner *po)
263 {
264 	struct pmclog_proc_init_args *ia;
265 
266 	ia = handle;
267 	mtx_lock(&pmc_kthread_mtx);
268 	MPASS(!ia->acted);
269 	MPASS(ia->po == NULL);
270 	MPASS(!ia->exit);
271 	MPASS(ia->kthr != NULL);
272 	if (po == NULL) {
273 		ia->exit = true;
274 	} else {
275 		ia->po = po;
276 		KASSERT(po->po_kthread == NULL,
277 		    ("[pmclog,%d] po=%p kthread (%p) already present",
278 		    __LINE__, po, po->po_kthread));
279 		po->po_kthread = ia->kthr;
280 	}
281 	wakeup(ia);
282 	while (!ia->acted)
283 		msleep(ia, &pmc_kthread_mtx, PWAIT, "pmclogw", 0);
284 	mtx_unlock(&pmc_kthread_mtx);
285 	free(ia, M_TEMP);
286 }
287 
288 /*
289  * Log handler loop.
290  *
291  * This function is executed by each pmc owner's helper thread.
292  */
293 
294 static void
295 pmclog_loop(void *arg)
296 {
297 	struct pmclog_proc_init_args *ia;
298 	struct pmc_owner *po;
299 	struct pmclog_buffer *lb;
300 	struct proc *p;
301 	struct ucred *ownercred;
302 	struct ucred *mycred;
303 	struct thread *td;
304 	sigset_t unb;
305 	struct uio auio;
306 	struct iovec aiov;
307 	size_t nbytes;
308 	int error;
309 
310 	td = curthread;
311 
312 	SIGEMPTYSET(unb);
313 	SIGADDSET(unb, SIGHUP);
314 	(void)kern_sigprocmask(td, SIG_UNBLOCK, &unb, NULL, 0);
315 
316 	ia = arg;
317 	MPASS(ia->kthr == curproc);
318 	MPASS(!ia->acted);
319 	mtx_lock(&pmc_kthread_mtx);
320 	while (ia->po == NULL && !ia->exit)
321 		msleep(ia, &pmc_kthread_mtx, PWAIT, "pmclogi", 0);
322 	if (ia->exit) {
323 		ia->acted = true;
324 		wakeup(ia);
325 		mtx_unlock(&pmc_kthread_mtx);
326 		kproc_exit(0);
327 	}
328 	MPASS(ia->po != NULL);
329 	po = ia->po;
330 	ia->acted = true;
331 	wakeup(ia);
332 	mtx_unlock(&pmc_kthread_mtx);
333 	ia = NULL;
334 
335 	p = po->po_owner;
336 	mycred = td->td_ucred;
337 
338 	PROC_LOCK(p);
339 	ownercred = crhold(p->p_ucred);
340 	PROC_UNLOCK(p);
341 
342 	PMCDBG2(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread);
343 	KASSERT(po->po_kthread == curthread->td_proc,
344 	    ("[pmclog,%d] proc mismatch po=%p po/kt=%p curproc=%p", __LINE__,
345 		po, po->po_kthread, curthread->td_proc));
346 
347 	lb = NULL;
348 
349 
350 	/*
351 	 * Loop waiting for I/O requests to be added to the owner
352 	 * struct's queue.  The loop is exited when the log file
353 	 * is deconfigured.
354 	 */
355 
356 	mtx_lock(&pmc_kthread_mtx);
357 
358 	for (;;) {
359 
360 		/* check if we've been asked to exit */
361 		if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
362 			break;
363 
364 		if (lb == NULL) { /* look for a fresh buffer to write */
365 			mtx_lock_spin(&po->po_mtx);
366 			if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
367 				mtx_unlock_spin(&po->po_mtx);
368 
369 				/* No more buffers and shutdown required. */
370 				if (po->po_flags & PMC_PO_SHUTDOWN)
371 					break;
372 
373 				(void) msleep(po, &pmc_kthread_mtx, PWAIT,
374 				    "pmcloop", 0);
375 				continue;
376 			}
377 
378 			TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
379 			mtx_unlock_spin(&po->po_mtx);
380 		}
381 
382 		mtx_unlock(&pmc_kthread_mtx);
383 
384 		/* process the request */
385 		PMCDBG3(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
386 		    lb->plb_base, lb->plb_ptr);
387 		/* change our thread's credentials before issuing the I/O */
388 
389 		aiov.iov_base = lb->plb_base;
390 		aiov.iov_len  = nbytes = lb->plb_ptr - lb->plb_base;
391 
392 		auio.uio_iov    = &aiov;
393 		auio.uio_iovcnt = 1;
394 		auio.uio_offset = -1;
395 		auio.uio_resid  = nbytes;
396 		auio.uio_rw     = UIO_WRITE;
397 		auio.uio_segflg = UIO_SYSSPACE;
398 		auio.uio_td     = td;
399 
400 		/* switch thread credentials -- see kern_ktrace.c */
401 		td->td_ucred = ownercred;
402 		error = fo_write(po->po_file, &auio, ownercred, 0, td);
403 		td->td_ucred = mycred;
404 
405 		if (error) {
406 			/* XXX some errors are recoverable */
407 			/* send a SIGIO to the owner and exit */
408 			PROC_LOCK(p);
409 			kern_psignal(p, SIGIO);
410 			PROC_UNLOCK(p);
411 
412 			mtx_lock(&pmc_kthread_mtx);
413 
414 			po->po_error = error; /* save for flush log */
415 
416 			PMCDBG2(LOG,WRI,2, "po=%p error=%d", po, error);
417 
418 			break;
419 		}
420 
421 		mtx_lock(&pmc_kthread_mtx);
422 
423 		/* put the used buffer back into the global pool */
424 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
425 
426 		mtx_lock_spin(&pmc_bufferlist_mtx);
427 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
428 		mtx_unlock_spin(&pmc_bufferlist_mtx);
429 
430 		lb = NULL;
431 	}
432 
433 	wakeup_one(po->po_kthread);
434 	po->po_kthread = NULL;
435 
436 	mtx_unlock(&pmc_kthread_mtx);
437 
438 	/* return the current I/O buffer to the global pool */
439 	if (lb) {
440 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
441 
442 		mtx_lock_spin(&pmc_bufferlist_mtx);
443 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
444 		mtx_unlock_spin(&pmc_bufferlist_mtx);
445 	}
446 
447 	/*
448 	 * Exit this thread, signalling the waiter
449 	 */
450 
451 	crfree(ownercred);
452 
453 	kproc_exit(0);
454 }
455 
456 /*
457  * Release and log entry and schedule an I/O if needed.
458  */
459 
460 static void
461 pmclog_release(struct pmc_owner *po)
462 {
463 	KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
464 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
465 		po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
466 	KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
467 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
468 		po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
469 
470 	/* schedule an I/O if we've filled a buffer */
471 	if (po->po_curbuf->plb_ptr >= po->po_curbuf->plb_fence)
472 		pmclog_schedule_io(po);
473 
474 	mtx_unlock_spin(&po->po_mtx);
475 
476 	PMCDBG1(LOG,REL,1, "po=%p", po);
477 }
478 
479 
480 /*
481  * Attempt to reserve 'length' bytes of space in an owner's log
482  * buffer.  The function returns a pointer to 'length' bytes of space
483  * if there was enough space or returns NULL if no space was
484  * available.  Non-null returns do so with the po mutex locked.  The
485  * caller must invoke pmclog_release() on the pmc owner structure
486  * when done.
487  */
488 
489 static uint32_t *
490 pmclog_reserve(struct pmc_owner *po, int length)
491 {
492 	uintptr_t newptr, oldptr;
493 	uint32_t *lh;
494 	struct timespec ts;
495 
496 	PMCDBG2(LOG,ALL,1, "po=%p len=%d", po, length);
497 
498 	KASSERT(length % sizeof(uint32_t) == 0,
499 	    ("[pmclog,%d] length not a multiple of word size", __LINE__));
500 
501 	mtx_lock_spin(&po->po_mtx);
502 
503 	/* No more data when shutdown in progress. */
504 	if (po->po_flags & PMC_PO_SHUTDOWN) {
505 		mtx_unlock_spin(&po->po_mtx);
506 		return (NULL);
507 	}
508 
509 	if (po->po_curbuf == NULL)
510 		if (pmclog_get_buffer(po) != 0) {
511 			mtx_unlock_spin(&po->po_mtx);
512 			return (NULL);
513 		}
514 
515 	KASSERT(po->po_curbuf != NULL,
516 	    ("[pmclog,%d] po=%p no current buffer", __LINE__, po));
517 
518 	KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base &&
519 	    po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
520 	    ("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
521 		__LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
522 		po->po_curbuf->plb_fence));
523 
524 	oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
525 	newptr = oldptr + length;
526 
527 	KASSERT(oldptr != (uintptr_t) NULL,
528 	    ("[pmclog,%d] po=%p Null log buffer pointer", __LINE__, po));
529 
530 	/*
531 	 * If we have space in the current buffer, return a pointer to
532 	 * available space with the PO structure locked.
533 	 */
534 	if (newptr <= (uintptr_t) po->po_curbuf->plb_fence) {
535 		po->po_curbuf->plb_ptr = (char *) newptr;
536 		goto done;
537 	}
538 
539 	/*
540 	 * Otherwise, schedule the current buffer for output and get a
541 	 * fresh buffer.
542 	 */
543 	pmclog_schedule_io(po);
544 
545 	if (pmclog_get_buffer(po) != 0) {
546 		mtx_unlock_spin(&po->po_mtx);
547 		return (NULL);
548 	}
549 
550 	KASSERT(po->po_curbuf != NULL,
551 	    ("[pmclog,%d] po=%p no current buffer", __LINE__, po));
552 
553 	KASSERT(po->po_curbuf->plb_ptr != NULL,
554 	    ("[pmclog,%d] null return from pmc_get_log_buffer", __LINE__));
555 
556 	KASSERT(po->po_curbuf->plb_ptr == po->po_curbuf->plb_base &&
557 	    po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
558 	    ("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
559 		__LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
560 		po->po_curbuf->plb_fence));
561 
562 	oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
563 
564  done:
565 	lh = (uint32_t *) oldptr;
566 	lh++;				/* skip header */
567 	getnanotime(&ts);		/* fill in the timestamp */
568 	*lh++ = ts.tv_sec & 0xFFFFFFFF;
569 	*lh++ = ts.tv_nsec & 0xFFFFFFF;
570 	return ((uint32_t *) oldptr);
571 }
572 
573 /*
574  * Schedule an I/O.
575  *
576  * Transfer the current buffer to the helper kthread.
577  */
578 
579 static void
580 pmclog_schedule_io(struct pmc_owner *po)
581 {
582 	KASSERT(po->po_curbuf != NULL,
583 	    ("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po));
584 
585 	KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
586 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
587 		po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
588 	KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
589 	    ("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
590 		po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
591 
592 	PMCDBG1(LOG,SIO, 1, "po=%p", po);
593 
594 	mtx_assert(&po->po_mtx, MA_OWNED);
595 
596 	/*
597 	 * Add the current buffer to the tail of the buffer list and
598 	 * wakeup the helper.
599 	 */
600 	TAILQ_INSERT_TAIL(&po->po_logbuffers, po->po_curbuf, plb_next);
601 	po->po_curbuf = NULL;
602 	wakeup_one(po);
603 }
604 
605 /*
606  * Stop the helper kthread.
607  */
608 
609 static void
610 pmclog_stop_kthread(struct pmc_owner *po)
611 {
612 
613 	mtx_lock(&pmc_kthread_mtx);
614 	po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
615 	if (po->po_kthread != NULL) {
616 		PROC_LOCK(po->po_kthread);
617 		kern_psignal(po->po_kthread, SIGHUP);
618 		PROC_UNLOCK(po->po_kthread);
619 	}
620 	wakeup_one(po);
621 	while (po->po_kthread)
622 		msleep(po->po_kthread, &pmc_kthread_mtx, PPAUSE, "pmckstp", 0);
623 	mtx_unlock(&pmc_kthread_mtx);
624 }
625 
626 /*
627  * Public functions
628  */
629 
630 /*
631  * Configure a log file for pmc owner 'po'.
632  *
633  * Parameter 'logfd' is a file handle referencing an open file in the
634  * owner process.  This file needs to have been opened for writing.
635  */
636 
637 int
638 pmclog_configure_log(struct pmc_mdep *md, struct pmc_owner *po, int logfd)
639 {
640 	struct proc *p;
641 	cap_rights_t rights;
642 	int error;
643 
644 	sx_assert(&pmc_sx, SA_XLOCKED);
645 	PMCDBG2(LOG,CFG,1, "config po=%p logfd=%d", po, logfd);
646 
647 	p = po->po_owner;
648 
649 	/* return EBUSY if a log file was already present */
650 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
651 		return (EBUSY);
652 
653 	KASSERT(po->po_file == NULL,
654 	    ("[pmclog,%d] po=%p file (%p) already present", __LINE__, po,
655 		po->po_file));
656 
657 	/* get a reference to the file state */
658 	error = fget_write(curthread, logfd,
659 	    cap_rights_init(&rights, CAP_WRITE), &po->po_file);
660 	if (error)
661 		goto error;
662 
663 	/* mark process as owning a log file */
664 	po->po_flags |= PMC_PO_OWNS_LOGFILE;
665 
666 	/* mark process as using HWPMCs */
667 	PROC_LOCK(p);
668 	p->p_flag |= P_HWPMC;
669 	PROC_UNLOCK(p);
670 
671 	/* create a log initialization entry */
672 	PMCLOG_RESERVE_WITH_ERROR(po, INITIALIZE,
673 	    sizeof(struct pmclog_initialize));
674 	PMCLOG_EMIT32(PMC_VERSION);
675 	PMCLOG_EMIT32(md->pmd_cputype);
676 	PMCLOG_DESPATCH(po);
677 
678 	return (0);
679 
680  error:
681 	KASSERT(po->po_kthread == NULL, ("[pmclog,%d] po=%p kthread not "
682 	    "stopped", __LINE__, po));
683 
684 	if (po->po_file)
685 		(void) fdrop(po->po_file, curthread);
686 	po->po_file  = NULL;	/* clear file and error state */
687 	po->po_error = 0;
688 	po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
689 
690 	return (error);
691 }
692 
693 
694 /*
695  * De-configure a log file.  This will throw away any buffers queued
696  * for this owner process.
697  */
698 
699 int
700 pmclog_deconfigure_log(struct pmc_owner *po)
701 {
702 	int error;
703 	struct pmclog_buffer *lb;
704 
705 	PMCDBG1(LOG,CFG,1, "de-config po=%p", po);
706 
707 	if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
708 		return (EINVAL);
709 
710 	KASSERT(po->po_sscount == 0,
711 	    ("[pmclog,%d] po=%p still owning SS PMCs", __LINE__, po));
712 	KASSERT(po->po_file != NULL,
713 	    ("[pmclog,%d] po=%p no log file", __LINE__, po));
714 
715 	/* stop the kthread, this will reset the 'OWNS_LOGFILE' flag */
716 	pmclog_stop_kthread(po);
717 
718 	KASSERT(po->po_kthread == NULL,
719 	    ("[pmclog,%d] po=%p kthread not stopped", __LINE__, po));
720 
721 	/* return all queued log buffers to the global pool */
722 	while ((lb = TAILQ_FIRST(&po->po_logbuffers)) != NULL) {
723 		TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
724 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
725 		mtx_lock_spin(&pmc_bufferlist_mtx);
726 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
727 		mtx_unlock_spin(&pmc_bufferlist_mtx);
728 	}
729 
730 	/* return the 'current' buffer to the global pool */
731 	if ((lb = po->po_curbuf) != NULL) {
732 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
733 		mtx_lock_spin(&pmc_bufferlist_mtx);
734 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
735 		mtx_unlock_spin(&pmc_bufferlist_mtx);
736 	}
737 
738 	/* drop a reference to the fd */
739 	if (po->po_file != NULL) {
740 		error = fdrop(po->po_file, curthread);
741 		po->po_file = NULL;
742 	} else
743 		error = 0;
744 	po->po_error = 0;
745 
746 	return (error);
747 }
748 
749 /*
750  * Flush a process' log buffer.
751  */
752 
753 int
754 pmclog_flush(struct pmc_owner *po)
755 {
756 	int error;
757 	struct pmclog_buffer *lb;
758 
759 	PMCDBG1(LOG,FLS,1, "po=%p", po);
760 
761 	/*
762 	 * If there is a pending error recorded by the logger thread,
763 	 * return that.
764 	 */
765 	if (po->po_error)
766 		return (po->po_error);
767 
768 	error = 0;
769 
770 	/*
771 	 * Check that we do have an active log file.
772 	 */
773 	mtx_lock(&pmc_kthread_mtx);
774 	if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
775 		error = EINVAL;
776 		goto error;
777 	}
778 
779 	/*
780 	 * Schedule the current buffer if any and not empty.
781 	 */
782 	mtx_lock_spin(&po->po_mtx);
783 	lb = po->po_curbuf;
784 	if (lb && lb->plb_ptr != lb->plb_base) {
785 		pmclog_schedule_io(po);
786 	} else
787 		error = ENOBUFS;
788 	mtx_unlock_spin(&po->po_mtx);
789 
790  error:
791 	mtx_unlock(&pmc_kthread_mtx);
792 
793 	return (error);
794 }
795 
796 int
797 pmclog_close(struct pmc_owner *po)
798 {
799 
800 	PMCDBG1(LOG,CLO,1, "po=%p", po);
801 
802 	pmclog_process_closelog(po);
803 
804 	mtx_lock(&pmc_kthread_mtx);
805 
806 	/*
807 	 * Schedule the current buffer.
808 	 */
809 	mtx_lock_spin(&po->po_mtx);
810 	if (po->po_curbuf)
811 		pmclog_schedule_io(po);
812 	else
813 		wakeup_one(po);
814 	mtx_unlock_spin(&po->po_mtx);
815 
816 	/*
817 	 * Initiate shutdown: no new data queued,
818 	 * thread will close file on last block.
819 	 */
820 	po->po_flags |= PMC_PO_SHUTDOWN;
821 
822 	mtx_unlock(&pmc_kthread_mtx);
823 
824 	return (0);
825 }
826 
827 void
828 pmclog_process_callchain(struct pmc *pm, struct pmc_sample *ps)
829 {
830 	int n, recordlen;
831 	uint32_t flags;
832 	struct pmc_owner *po;
833 
834 	PMCDBG3(LOG,SAM,1,"pm=%p pid=%d n=%d", pm, ps->ps_pid,
835 	    ps->ps_nsamples);
836 
837 	recordlen = offsetof(struct pmclog_callchain, pl_pc) +
838 	    ps->ps_nsamples * sizeof(uintfptr_t);
839 	po = pm->pm_owner;
840 	flags = PMC_CALLCHAIN_TO_CPUFLAGS(ps->ps_cpu,ps->ps_flags);
841 	PMCLOG_RESERVE(po, CALLCHAIN, recordlen);
842 	PMCLOG_EMIT32(ps->ps_pid);
843 	PMCLOG_EMIT32(pm->pm_id);
844 	PMCLOG_EMIT32(flags);
845 	for (n = 0; n < ps->ps_nsamples; n++)
846 		PMCLOG_EMITADDR(ps->ps_pc[n]);
847 	PMCLOG_DESPATCH(po);
848 }
849 
850 void
851 pmclog_process_closelog(struct pmc_owner *po)
852 {
853 	PMCLOG_RESERVE(po,CLOSELOG,sizeof(struct pmclog_closelog));
854 	PMCLOG_DESPATCH(po);
855 }
856 
857 void
858 pmclog_process_dropnotify(struct pmc_owner *po)
859 {
860 	PMCLOG_RESERVE(po,DROPNOTIFY,sizeof(struct pmclog_dropnotify));
861 	PMCLOG_DESPATCH(po);
862 }
863 
864 void
865 pmclog_process_map_in(struct pmc_owner *po, pid_t pid, uintfptr_t start,
866     const char *path)
867 {
868 	int pathlen, recordlen;
869 
870 	KASSERT(path != NULL, ("[pmclog,%d] map-in, null path", __LINE__));
871 
872 	pathlen = strlen(path) + 1;	/* #bytes for path name */
873 	recordlen = offsetof(struct pmclog_map_in, pl_pathname) +
874 	    pathlen;
875 
876 	PMCLOG_RESERVE(po, MAP_IN, recordlen);
877 	PMCLOG_EMIT32(pid);
878 	PMCLOG_EMITADDR(start);
879 	PMCLOG_EMITSTRING(path,pathlen);
880 	PMCLOG_DESPATCH(po);
881 }
882 
883 void
884 pmclog_process_map_out(struct pmc_owner *po, pid_t pid, uintfptr_t start,
885     uintfptr_t end)
886 {
887 	KASSERT(start <= end, ("[pmclog,%d] start > end", __LINE__));
888 
889 	PMCLOG_RESERVE(po, MAP_OUT, sizeof(struct pmclog_map_out));
890 	PMCLOG_EMIT32(pid);
891 	PMCLOG_EMITADDR(start);
892 	PMCLOG_EMITADDR(end);
893 	PMCLOG_DESPATCH(po);
894 }
895 
896 void
897 pmclog_process_pmcallocate(struct pmc *pm)
898 {
899 	struct pmc_owner *po;
900 	struct pmc_soft *ps;
901 
902 	po = pm->pm_owner;
903 
904 	PMCDBG1(LOG,ALL,1, "pm=%p", pm);
905 
906 	if (PMC_TO_CLASS(pm) == PMC_CLASS_SOFT) {
907 		PMCLOG_RESERVE(po, PMCALLOCATEDYN,
908 		    sizeof(struct pmclog_pmcallocatedyn));
909 		PMCLOG_EMIT32(pm->pm_id);
910 		PMCLOG_EMIT32(pm->pm_event);
911 		PMCLOG_EMIT32(pm->pm_flags);
912 		ps = pmc_soft_ev_acquire(pm->pm_event);
913 		if (ps != NULL)
914 			PMCLOG_EMITSTRING(ps->ps_ev.pm_ev_name,PMC_NAME_MAX);
915 		else
916 			PMCLOG_EMITNULLSTRING(PMC_NAME_MAX);
917 		pmc_soft_ev_release(ps);
918 		PMCLOG_DESPATCH(po);
919 	} else {
920 		PMCLOG_RESERVE(po, PMCALLOCATE,
921 		    sizeof(struct pmclog_pmcallocate));
922 		PMCLOG_EMIT32(pm->pm_id);
923 		PMCLOG_EMIT32(pm->pm_event);
924 		PMCLOG_EMIT32(pm->pm_flags);
925 		PMCLOG_DESPATCH(po);
926 	}
927 }
928 
929 void
930 pmclog_process_pmcattach(struct pmc *pm, pid_t pid, char *path)
931 {
932 	int pathlen, recordlen;
933 	struct pmc_owner *po;
934 
935 	PMCDBG2(LOG,ATT,1,"pm=%p pid=%d", pm, pid);
936 
937 	po = pm->pm_owner;
938 
939 	pathlen = strlen(path) + 1;	/* #bytes for the string */
940 	recordlen = offsetof(struct pmclog_pmcattach, pl_pathname) + pathlen;
941 
942 	PMCLOG_RESERVE(po, PMCATTACH, recordlen);
943 	PMCLOG_EMIT32(pm->pm_id);
944 	PMCLOG_EMIT32(pid);
945 	PMCLOG_EMITSTRING(path, pathlen);
946 	PMCLOG_DESPATCH(po);
947 }
948 
949 void
950 pmclog_process_pmcdetach(struct pmc *pm, pid_t pid)
951 {
952 	struct pmc_owner *po;
953 
954 	PMCDBG2(LOG,ATT,1,"!pm=%p pid=%d", pm, pid);
955 
956 	po = pm->pm_owner;
957 
958 	PMCLOG_RESERVE(po, PMCDETACH, sizeof(struct pmclog_pmcdetach));
959 	PMCLOG_EMIT32(pm->pm_id);
960 	PMCLOG_EMIT32(pid);
961 	PMCLOG_DESPATCH(po);
962 }
963 
964 /*
965  * Log a context switch event to the log file.
966  */
967 
968 void
969 pmclog_process_proccsw(struct pmc *pm, struct pmc_process *pp, pmc_value_t v)
970 {
971 	struct pmc_owner *po;
972 
973 	KASSERT(pm->pm_flags & PMC_F_LOG_PROCCSW,
974 	    ("[pmclog,%d] log-process-csw called gratuitously", __LINE__));
975 
976 	PMCDBG3(LOG,SWO,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
977 	    v);
978 
979 	po = pm->pm_owner;
980 
981 	PMCLOG_RESERVE(po, PROCCSW, sizeof(struct pmclog_proccsw));
982 	PMCLOG_EMIT32(pm->pm_id);
983 	PMCLOG_EMIT64(v);
984 	PMCLOG_EMIT32(pp->pp_proc->p_pid);
985 	PMCLOG_DESPATCH(po);
986 }
987 
988 void
989 pmclog_process_procexec(struct pmc_owner *po, pmc_id_t pmid, pid_t pid,
990     uintfptr_t startaddr, char *path)
991 {
992 	int pathlen, recordlen;
993 
994 	PMCDBG3(LOG,EXC,1,"po=%p pid=%d path=\"%s\"", po, pid, path);
995 
996 	pathlen   = strlen(path) + 1;	/* #bytes for the path */
997 	recordlen = offsetof(struct pmclog_procexec, pl_pathname) + pathlen;
998 
999 	PMCLOG_RESERVE(po, PROCEXEC, recordlen);
1000 	PMCLOG_EMIT32(pid);
1001 	PMCLOG_EMITADDR(startaddr);
1002 	PMCLOG_EMIT32(pmid);
1003 	PMCLOG_EMITSTRING(path,pathlen);
1004 	PMCLOG_DESPATCH(po);
1005 }
1006 
1007 /*
1008  * Log a process exit event (and accumulated pmc value) to the log file.
1009  */
1010 
1011 void
1012 pmclog_process_procexit(struct pmc *pm, struct pmc_process *pp)
1013 {
1014 	int ri;
1015 	struct pmc_owner *po;
1016 
1017 	ri = PMC_TO_ROWINDEX(pm);
1018 	PMCDBG3(LOG,EXT,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
1019 	    pp->pp_pmcs[ri].pp_pmcval);
1020 
1021 	po = pm->pm_owner;
1022 
1023 	PMCLOG_RESERVE(po, PROCEXIT, sizeof(struct pmclog_procexit));
1024 	PMCLOG_EMIT32(pm->pm_id);
1025 	PMCLOG_EMIT64(pp->pp_pmcs[ri].pp_pmcval);
1026 	PMCLOG_EMIT32(pp->pp_proc->p_pid);
1027 	PMCLOG_DESPATCH(po);
1028 }
1029 
1030 /*
1031  * Log a fork event.
1032  */
1033 
1034 void
1035 pmclog_process_procfork(struct pmc_owner *po, pid_t oldpid, pid_t newpid)
1036 {
1037 	PMCLOG_RESERVE(po, PROCFORK, sizeof(struct pmclog_procfork));
1038 	PMCLOG_EMIT32(oldpid);
1039 	PMCLOG_EMIT32(newpid);
1040 	PMCLOG_DESPATCH(po);
1041 }
1042 
1043 /*
1044  * Log a process exit event of the form suitable for system-wide PMCs.
1045  */
1046 
1047 void
1048 pmclog_process_sysexit(struct pmc_owner *po, pid_t pid)
1049 {
1050 	PMCLOG_RESERVE(po, SYSEXIT, sizeof(struct pmclog_sysexit));
1051 	PMCLOG_EMIT32(pid);
1052 	PMCLOG_DESPATCH(po);
1053 }
1054 
1055 /*
1056  * Write a user log entry.
1057  */
1058 
1059 int
1060 pmclog_process_userlog(struct pmc_owner *po, struct pmc_op_writelog *wl)
1061 {
1062 	int error;
1063 
1064 	PMCDBG2(LOG,WRI,1, "writelog po=%p ud=0x%x", po, wl->pm_userdata);
1065 
1066 	error = 0;
1067 
1068 	PMCLOG_RESERVE_WITH_ERROR(po, USERDATA,
1069 	    sizeof(struct pmclog_userdata));
1070 	PMCLOG_EMIT32(wl->pm_userdata);
1071 	PMCLOG_DESPATCH(po);
1072 
1073  error:
1074 	return (error);
1075 }
1076 
1077 /*
1078  * Initialization.
1079  *
1080  * Create a pool of log buffers and initialize mutexes.
1081  */
1082 
1083 void
1084 pmclog_initialize()
1085 {
1086 	int n;
1087 	struct pmclog_buffer *plb;
1088 
1089 	if (pmclog_buffer_size <= 0) {
1090 		(void) printf("hwpmc: tunable logbuffersize=%d must be "
1091 		    "greater than zero.\n", pmclog_buffer_size);
1092 		pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
1093 	}
1094 
1095 	if (pmc_nlogbuffers <= 0) {
1096 		(void) printf("hwpmc: tunable nlogbuffers=%d must be greater "
1097 		    "than zero.\n", pmc_nlogbuffers);
1098 		pmc_nlogbuffers = PMC_NLOGBUFFERS;
1099 	}
1100 
1101 	/* create global pool of log buffers */
1102 	for (n = 0; n < pmc_nlogbuffers; n++) {
1103 		plb = malloc(1024 * pmclog_buffer_size, M_PMC,
1104 		    M_WAITOK|M_ZERO);
1105 		PMCLOG_INIT_BUFFER_DESCRIPTOR(plb);
1106 		TAILQ_INSERT_HEAD(&pmc_bufferlist, plb, plb_next);
1107 	}
1108 	mtx_init(&pmc_bufferlist_mtx, "pmc-buffer-list", "pmc-leaf",
1109 	    MTX_SPIN);
1110 	mtx_init(&pmc_kthread_mtx, "pmc-kthread", "pmc-sleep", MTX_DEF);
1111 }
1112 
1113 /*
1114  * Shutdown logging.
1115  *
1116  * Destroy mutexes and release memory back the to free pool.
1117  */
1118 
1119 void
1120 pmclog_shutdown()
1121 {
1122 	struct pmclog_buffer *plb;
1123 
1124 	mtx_destroy(&pmc_kthread_mtx);
1125 	mtx_destroy(&pmc_bufferlist_mtx);
1126 
1127 	while ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL) {
1128 		TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next);
1129 		free(plb, M_PMC);
1130 	}
1131 }
1132