xref: /freebsd/usr.sbin/pmcstat/pmcstat_log.c (revision 069ac18495ad8fde2748bc94b0f80a50250bb01d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2005-2007, Joseph Koshy
5  * Copyright (c) 2007 The FreeBSD Foundation
6  * All rights reserved.
7  *
8  * Portions of this software were developed by A. Joseph Koshy under
9  * sponsorship from the FreeBSD Foundation and Google, Inc.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  */
32 
33 /*
34  * Transform a hwpmc(4) log into human readable form, and into
35  * gprof(1) compatible profiles.
36  */
37 
38 #include <sys/param.h>
39 #include <sys/endian.h>
40 #include <sys/cpuset.h>
41 #include <sys/gmon.h>
42 #include <sys/imgact_aout.h>
43 #include <sys/imgact_elf.h>
44 #include <sys/mman.h>
45 #include <sys/pmc.h>
46 #include <sys/queue.h>
47 #include <sys/socket.h>
48 #include <sys/stat.h>
49 #include <sys/wait.h>
50 
51 #include <netinet/in.h>
52 
53 #include <assert.h>
54 #include <curses.h>
55 #include <err.h>
56 #include <errno.h>
57 #include <fcntl.h>
58 #include <gelf.h>
59 #include <libgen.h>
60 #include <limits.h>
61 #include <netdb.h>
62 #include <pmc.h>
63 #include <pmclog.h>
64 #include <sysexits.h>
65 #include <stdint.h>
66 #include <stdio.h>
67 #include <stdlib.h>
68 #include <string.h>
69 #include <unistd.h>
70 
71 #include "pmcstat.h"
72 #include "pmcstat_log.h"
73 #include "pmcstat_top.h"
74 
75 /*
76  * PUBLIC INTERFACES
77  *
78  * pmcstat_initialize_logging()	initialize this module, called first
79  * pmcstat_shutdown_logging()		orderly shutdown, called last
80  * pmcstat_open_log()			open an eventlog for processing
81  * pmcstat_process_log()		print/convert an event log
82  * pmcstat_display_log()		top mode display for the log
83  * pmcstat_close_log()			finish processing an event log
84  *
85  * IMPLEMENTATION NOTES
86  *
87  * We correlate each 'callchain' or 'sample' entry seen in the event
88  * log back to an executable object in the system. Executable objects
89  * include:
90  * 	- program executables,
91  *	- shared libraries loaded by the runtime loader,
92  *	- dlopen()'ed objects loaded by the program,
93  *	- the runtime loader itself,
94  *	- the kernel and kernel modules.
95  *
96  * Each process that we know about is treated as a set of regions that
97  * map to executable objects.  Processes are described by
98  * 'pmcstat_process' structures.  Executable objects are tracked by
99  * 'pmcstat_image' structures.  The kernel and kernel modules are
100  * common to all processes (they reside at the same virtual addresses
101  * for all processes).  Individual processes can have their text
102  * segments and shared libraries loaded at process-specific locations.
103  *
104  * A given executable object can be in use by multiple processes
105  * (e.g., libc.so) and loaded at a different address in each.
106  * pmcstat_pcmap structures track per-image mappings.
107  *
108  * The sample log could have samples from multiple PMCs; we
109  * generate one 'gmon.out' profile per PMC.
110  *
111  * IMPLEMENTATION OF GMON OUTPUT
112  *
113  * Each executable object gets one 'gmon.out' profile, per PMC in
114  * use.  Creation of 'gmon.out' profiles is done lazily.  The
115  * 'gmon.out' profiles generated for a given sampling PMC are
116  * aggregates of all the samples for that particular executable
117  * object.
118  *
119  * IMPLEMENTATION OF SYSTEM-WIDE CALLGRAPH OUTPUT
120  *
121  * Each active pmcid has its own callgraph structure, described by a
122  * 'struct pmcstat_callgraph'.  Given a process id and a list of pc
123  * values, we map each pc value to a tuple (image, symbol), where
124  * 'image' denotes an executable object and 'symbol' is the closest
125  * symbol that precedes the pc value.  Each pc value in the list is
126  * also given a 'rank' that reflects its depth in the call stack.
127  */
128 
129 struct pmcstat_pmcs pmcstat_pmcs = LIST_HEAD_INITIALIZER(pmcstat_pmcs);
130 
131 /*
132  * All image descriptors are kept in a hash table.
133  */
134 struct pmcstat_image_hash_list pmcstat_image_hash[PMCSTAT_NHASH];
135 
136 /*
137  * All process descriptors are kept in a hash table.
138  */
139 struct pmcstat_process_hash_list pmcstat_process_hash[PMCSTAT_NHASH];
140 
141 struct pmcstat_stats pmcstat_stats; /* statistics */
142 static int ps_samples_period; /* samples count between top refresh. */
143 
144 struct pmcstat_process *pmcstat_kernproc; /* kernel 'process' */
145 
146 #include "pmcpl_gprof.h"
147 #include "pmcpl_callgraph.h"
148 #include "pmcpl_annotate.h"
149 #include "pmcpl_annotate_cg.h"
150 #include "pmcpl_calltree.h"
151 
152 static struct pmc_plugins plugins[] = {
153 	{
154 		.pl_name		= "none",
155 	},
156 	{
157 		.pl_name		= "callgraph",
158 		.pl_init		= pmcpl_cg_init,
159 		.pl_shutdown		= pmcpl_cg_shutdown,
160 		.pl_process		= pmcpl_cg_process,
161 		.pl_topkeypress		= pmcpl_cg_topkeypress,
162 		.pl_topdisplay		= pmcpl_cg_topdisplay
163 	},
164 	{
165 		.pl_name		= "gprof",
166 		.pl_shutdown		= pmcpl_gmon_shutdown,
167 		.pl_process		= pmcpl_gmon_process,
168 		.pl_initimage		= pmcpl_gmon_initimage,
169 		.pl_shutdownimage	= pmcpl_gmon_shutdownimage,
170 		.pl_newpmc		= pmcpl_gmon_newpmc
171 	},
172 	{
173 		.pl_name		= "annotate",
174 		.pl_process		= pmcpl_annotate_process
175 	},
176 	{
177 		.pl_name		= "calltree",
178 		.pl_configure		= pmcpl_ct_configure,
179 		.pl_init		= pmcpl_ct_init,
180 		.pl_shutdown		= pmcpl_ct_shutdown,
181 		.pl_process		= pmcpl_ct_process,
182 		.pl_topkeypress		= pmcpl_ct_topkeypress,
183 		.pl_topdisplay		= pmcpl_ct_topdisplay
184 	},
185 	{
186 		.pl_name		= "annotate_cg",
187 		.pl_process		= pmcpl_annotate_cg_process
188 	},
189 
190 	{
191 		.pl_name		= NULL
192 	}
193 };
194 
195 static int pmcstat_mergepmc;
196 
197 int pmcstat_pmcinfilter = 0; /* PMC filter for top mode. */
198 float pmcstat_threshold = 0.5; /* Cost filter for top mode. */
199 
200 /*
201  * Prototypes
202  */
203 
204 static void pmcstat_stats_reset(int _reset_global);
205 
206 /*
207  * PMC count.
208  */
209 int pmcstat_npmcs;
210 
211 /*
212  * PMC Top mode pause state.
213  */
214 static int pmcstat_pause;
215 
216 static void
217 pmcstat_stats_reset(int reset_global)
218 {
219 	struct pmcstat_pmcrecord *pr;
220 
221 	/* Flush PMCs stats. */
222 	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next) {
223 		pr->pr_samples = 0;
224 		pr->pr_dubious_frames = 0;
225 	}
226 	ps_samples_period = 0;
227 
228 	/* Flush global stats. */
229 	if (reset_global)
230 		bzero(&pmcstat_stats, sizeof(struct pmcstat_stats));
231 }
232 
233 /*
234  * Resolve file name and line number for the given address.
235  */
236 int
237 pmcstat_image_addr2line(struct pmcstat_image *image, uintfptr_t addr,
238     char *sourcefile, size_t sourcefile_len, unsigned *sourceline,
239     char *funcname, size_t funcname_len)
240 {
241 	static int addr2line_warn = 0;
242 
243 	char *sep, cmdline[PATH_MAX], imagepath[PATH_MAX];
244 	unsigned l;
245 	int fd;
246 
247 	if (image->pi_addr2line == NULL) {
248 		/* Try default debug file location. */
249 		snprintf(imagepath, sizeof(imagepath),
250 		    "/usr/lib/debug/%s%s.debug",
251 		    args.pa_fsroot,
252 		    pmcstat_string_unintern(image->pi_fullpath));
253 		fd = open(imagepath, O_RDONLY);
254 		if (fd < 0) {
255 			/* Old kernel symbol path. */
256 			snprintf(imagepath, sizeof(imagepath), "%s%s.symbols",
257 			    args.pa_fsroot,
258 			    pmcstat_string_unintern(image->pi_fullpath));
259 			fd = open(imagepath, O_RDONLY);
260 			if (fd < 0) {
261 				snprintf(imagepath, sizeof(imagepath), "%s%s",
262 				    args.pa_fsroot,
263 				    pmcstat_string_unintern(
264 				        image->pi_fullpath));
265 			}
266 		}
267 		if (fd >= 0)
268 			close(fd);
269 		/*
270 		 * New addr2line support recursive inline function with -i
271 		 * but the format does not add a marker when no more entries
272 		 * are available.
273 		 */
274 		snprintf(cmdline, sizeof(cmdline), "addr2line -Cfe \"%s\"",
275 		    imagepath);
276 		image->pi_addr2line = popen(cmdline, "r+");
277 		if (image->pi_addr2line == NULL) {
278 			if (!addr2line_warn) {
279 				addr2line_warn = 1;
280 				warnx(
281 "WARNING: addr2line is needed for source code information."
282 				    );
283 			}
284 			return (0);
285 		}
286 	}
287 
288 	if (feof(image->pi_addr2line) || ferror(image->pi_addr2line)) {
289 		warnx("WARNING: addr2line pipe error");
290 		pclose(image->pi_addr2line);
291 		image->pi_addr2line = NULL;
292 		return (0);
293 	}
294 
295 	fprintf(image->pi_addr2line, "%p\n", (void *)addr);
296 
297 	if (fgets(funcname, funcname_len, image->pi_addr2line) == NULL) {
298 		warnx("WARNING: addr2line function name read error");
299 		return (0);
300 	}
301 	sep = strchr(funcname, '\n');
302 	if (sep != NULL)
303 		*sep = '\0';
304 
305 	if (fgets(sourcefile, sourcefile_len, image->pi_addr2line) == NULL) {
306 		warnx("WARNING: addr2line source file read error");
307 		return (0);
308 	}
309 	sep = strchr(sourcefile, ':');
310 	if (sep == NULL) {
311 		warnx("WARNING: addr2line source line separator missing");
312 		return (0);
313 	}
314 	*sep = '\0';
315 	l = atoi(sep+1);
316 	if (l == 0)
317 		return (0);
318 	*sourceline = l;
319 	return (1);
320 }
321 
322 /*
323  * Given a pmcid in use, find its human-readable name.
324  */
325 
326 const char *
327 pmcstat_pmcid_to_name(pmc_id_t pmcid)
328 {
329 	struct pmcstat_pmcrecord *pr;
330 
331 	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
332 	    if (pr->pr_pmcid == pmcid)
333 		    return (pmcstat_string_unintern(pr->pr_pmcname));
334 
335 	return NULL;
336 }
337 
338 /*
339  * Convert PMC index to name.
340  */
341 
342 const char *
343 pmcstat_pmcindex_to_name(int pmcin)
344 {
345 	struct pmcstat_pmcrecord *pr;
346 
347 	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
348 		if (pr->pr_pmcin == pmcin)
349 			return pmcstat_string_unintern(pr->pr_pmcname);
350 
351 	return NULL;
352 }
353 
354 /*
355  * Return PMC record with given index.
356  */
357 
358 struct pmcstat_pmcrecord *
359 pmcstat_pmcindex_to_pmcr(int pmcin)
360 {
361 	struct pmcstat_pmcrecord *pr;
362 
363 	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
364 		if (pr->pr_pmcin == pmcin)
365 			return pr;
366 
367 	return NULL;
368 }
369 
370 /*
371  * Print log entries as text.
372  */
373 
374 static int
375 pmcstat_print_log(void)
376 {
377 	struct pmclog_ev ev;
378 	uint32_t npc;
379 
380 	while (pmclog_read(args.pa_logparser, &ev) == 0) {
381 		assert(ev.pl_state == PMCLOG_OK);
382 		switch (ev.pl_type) {
383 		case PMCLOG_TYPE_CALLCHAIN:
384 			PMCSTAT_PRINT_ENTRY("callchain",
385 			    "%d 0x%x %d %d %c", ev.pl_u.pl_cc.pl_pid,
386 			    ev.pl_u.pl_cc.pl_pmcid,
387 			    PMC_CALLCHAIN_CPUFLAGS_TO_CPU(ev.pl_u.pl_cc. \
388 				pl_cpuflags), ev.pl_u.pl_cc.pl_npc,
389 			    PMC_CALLCHAIN_CPUFLAGS_TO_USERMODE(ev.pl_u.pl_cc.\
390 			        pl_cpuflags) ? 'u' : 's');
391 			for (npc = 0; npc < ev.pl_u.pl_cc.pl_npc; npc++)
392 				PMCSTAT_PRINT_ENTRY("...", "%p",
393 				    (void *) ev.pl_u.pl_cc.pl_pc[npc]);
394 			break;
395 		case PMCLOG_TYPE_CLOSELOG:
396 			PMCSTAT_PRINT_ENTRY("closelog",);
397 			break;
398 		case PMCLOG_TYPE_DROPNOTIFY:
399 			PMCSTAT_PRINT_ENTRY("drop",);
400 			break;
401 		case PMCLOG_TYPE_INITIALIZE:
402 			PMCSTAT_PRINT_ENTRY("initlog","0x%x \"%s\"",
403 			    ev.pl_u.pl_i.pl_version,
404 			    pmc_name_of_cputype(ev.pl_u.pl_i.pl_arch));
405 			if ((ev.pl_u.pl_i.pl_version & 0xFF000000) !=
406 			    PMC_VERSION_MAJOR << 24)
407 				warnx(
408 "WARNING: Log version 0x%x != expected version 0x%x.",
409 				    ev.pl_u.pl_i.pl_version, PMC_VERSION);
410 			break;
411 		case PMCLOG_TYPE_MAP_IN:
412 			PMCSTAT_PRINT_ENTRY("map-in","%d %p \"%s\"",
413 			    ev.pl_u.pl_mi.pl_pid,
414 			    (void *) ev.pl_u.pl_mi.pl_start,
415 			    ev.pl_u.pl_mi.pl_pathname);
416 			break;
417 		case PMCLOG_TYPE_MAP_OUT:
418 			PMCSTAT_PRINT_ENTRY("map-out","%d %p %p",
419 			    ev.pl_u.pl_mo.pl_pid,
420 			    (void *) ev.pl_u.pl_mo.pl_start,
421 			    (void *) ev.pl_u.pl_mo.pl_end);
422 			break;
423 		case PMCLOG_TYPE_PMCALLOCATE:
424 			PMCSTAT_PRINT_ENTRY("allocate","0x%x \"%s\" 0x%x",
425 			    ev.pl_u.pl_a.pl_pmcid,
426 			    ev.pl_u.pl_a.pl_evname,
427 			    ev.pl_u.pl_a.pl_flags);
428 			break;
429 		case PMCLOG_TYPE_PMCALLOCATEDYN:
430 			PMCSTAT_PRINT_ENTRY("allocatedyn","0x%x \"%s\" 0x%x",
431 			    ev.pl_u.pl_ad.pl_pmcid,
432 			    ev.pl_u.pl_ad.pl_evname,
433 			    ev.pl_u.pl_ad.pl_flags);
434 			break;
435 		case PMCLOG_TYPE_PMCATTACH:
436 			PMCSTAT_PRINT_ENTRY("attach","0x%x %d \"%s\"",
437 			    ev.pl_u.pl_t.pl_pmcid,
438 			    ev.pl_u.pl_t.pl_pid,
439 			    ev.pl_u.pl_t.pl_pathname);
440 			break;
441 		case PMCLOG_TYPE_PMCDETACH:
442 			PMCSTAT_PRINT_ENTRY("detach","0x%x %d",
443 			    ev.pl_u.pl_d.pl_pmcid,
444 			    ev.pl_u.pl_d.pl_pid);
445 			break;
446 		case PMCLOG_TYPE_PROCCSW:
447 			PMCSTAT_PRINT_ENTRY("cswval","0x%x %d %jd",
448 			    ev.pl_u.pl_c.pl_pmcid,
449 			    ev.pl_u.pl_c.pl_pid,
450 			    ev.pl_u.pl_c.pl_value);
451 			break;
452 		case PMCLOG_TYPE_PROC_CREATE:
453 			PMCSTAT_PRINT_ENTRY("create","%d %x \"%s\"",
454 			    ev.pl_u.pl_pc.pl_pid,
455 			    ev.pl_u.pl_pc.pl_flags,
456 			    ev.pl_u.pl_pc.pl_pcomm);
457 			break;
458 		case PMCLOG_TYPE_PROCEXEC:
459 			PMCSTAT_PRINT_ENTRY("exec","0x%x %d %p %p \"%s\"",
460 			    ev.pl_u.pl_x.pl_pmcid,
461 			    ev.pl_u.pl_x.pl_pid,
462 			    (void *)ev.pl_u.pl_x.pl_baseaddr,
463 			    (void *)ev.pl_u.pl_x.pl_dynaddr,
464 			    ev.pl_u.pl_x.pl_pathname);
465 			break;
466 		case PMCLOG_TYPE_PROCEXIT:
467 			PMCSTAT_PRINT_ENTRY("exitval","0x%x %d %jd",
468 			    ev.pl_u.pl_e.pl_pmcid,
469 			    ev.pl_u.pl_e.pl_pid,
470 			    ev.pl_u.pl_e.pl_value);
471 			break;
472 		case PMCLOG_TYPE_PROCFORK:
473 			PMCSTAT_PRINT_ENTRY("fork","%d %d",
474 			    ev.pl_u.pl_f.pl_oldpid,
475 			    ev.pl_u.pl_f.pl_newpid);
476 			break;
477 		case PMCLOG_TYPE_USERDATA:
478 			PMCSTAT_PRINT_ENTRY("userdata","0x%x",
479 			    ev.pl_u.pl_u.pl_userdata);
480 			break;
481 		case PMCLOG_TYPE_SYSEXIT:
482 			PMCSTAT_PRINT_ENTRY("exit","%d",
483 			    ev.pl_u.pl_se.pl_pid);
484 			break;
485 		case PMCLOG_TYPE_THR_CREATE:
486 			PMCSTAT_PRINT_ENTRY("thr-create","%d %d %x \"%s\"",
487 			    ev.pl_u.pl_tc.pl_tid,
488 			    ev.pl_u.pl_tc.pl_pid,
489 			    ev.pl_u.pl_tc.pl_flags,
490 			    ev.pl_u.pl_tc.pl_tdname);
491 			break;
492 		case PMCLOG_TYPE_THR_EXIT:
493 			PMCSTAT_PRINT_ENTRY("thr-exit","%d",
494 			    ev.pl_u.pl_tc.pl_tid);
495 			break;
496 		default:
497 			fprintf(args.pa_printfile, "unknown event (type %d).\n",
498 			    ev.pl_type);
499 		}
500 	}
501 
502 	if (ev.pl_state == PMCLOG_EOF)
503 		return (PMCSTAT_FINISHED);
504 	else if (ev.pl_state == PMCLOG_REQUIRE_DATA)
505 		return (PMCSTAT_RUNNING);
506 
507 	errx(EX_DATAERR,
508 	    "ERROR: event parsing failed (record %jd, offset 0x%jx).",
509 	    (uintmax_t) ev.pl_count + 1, ev.pl_offset);
510 	/*NOTREACHED*/
511 }
512 
513 /*
514  * Public Interfaces.
515  */
516 
517 /*
518  * Process a log file in offline analysis mode.
519  */
520 
521 int
522 pmcstat_process_log(void)
523 {
524 
525 	/*
526 	 * If analysis has not been asked for, just print the log to
527 	 * the current output file.
528 	 */
529 	if (args.pa_flags & FLAG_DO_PRINT)
530 		return (pmcstat_print_log());
531 	else
532 		return (pmcstat_analyze_log(&args, plugins, &pmcstat_stats, pmcstat_kernproc,
533 		    pmcstat_mergepmc, &pmcstat_npmcs, &ps_samples_period));
534 }
535 
536 /*
537  * Refresh top display.
538  */
539 
540 static void
541 pmcstat_refresh_top(void)
542 {
543 	int v_attrs;
544 	float v;
545 	char pmcname[40];
546 	struct pmcstat_pmcrecord *pmcpr;
547 
548 	/* If in pause mode do not refresh display. */
549 	if (pmcstat_pause)
550 		return;
551 
552 	/* Wait until PMC pop in the log. */
553 	pmcpr = pmcstat_pmcindex_to_pmcr(pmcstat_pmcinfilter);
554 	if (pmcpr == NULL)
555 		return;
556 
557 	/* Format PMC name. */
558 	if (pmcstat_mergepmc)
559 		snprintf(pmcname, sizeof(pmcname), "[%s]",
560 		    pmcstat_string_unintern(pmcpr->pr_pmcname));
561 	else
562 		snprintf(pmcname, sizeof(pmcname), "%s.%d",
563 		    pmcstat_string_unintern(pmcpr->pr_pmcname),
564 		    pmcstat_pmcinfilter);
565 
566 	/* Format samples count. */
567 	if (ps_samples_period > 0)
568 		v = (pmcpr->pr_samples * 100.0) / ps_samples_period;
569 	else
570 		v = 0.;
571 	v_attrs = PMCSTAT_ATTRPERCENT(v);
572 
573 	PMCSTAT_PRINTBEGIN();
574 	PMCSTAT_PRINTW("PMC: %s Samples: %u ",
575 	    pmcname,
576 	    pmcpr->pr_samples);
577 	PMCSTAT_ATTRON(v_attrs);
578 	PMCSTAT_PRINTW("(%.1f%%) ", v);
579 	PMCSTAT_ATTROFF(v_attrs);
580 	PMCSTAT_PRINTW(", %u unresolved\n\n",
581 	    pmcpr->pr_dubious_frames);
582 	if (plugins[args.pa_plugin].pl_topdisplay != NULL)
583 		plugins[args.pa_plugin].pl_topdisplay();
584 	PMCSTAT_PRINTEND();
585 }
586 
587 /*
588  * Find the next pmc index to display.
589  */
590 
591 static void
592 pmcstat_changefilter(void)
593 {
594 	int pmcin;
595 	struct pmcstat_pmcrecord *pmcr;
596 
597 	/*
598 	 * Find the next merge target.
599 	 */
600 	if (pmcstat_mergepmc) {
601 		pmcin = pmcstat_pmcinfilter;
602 
603 		do {
604 			pmcr = pmcstat_pmcindex_to_pmcr(pmcstat_pmcinfilter);
605 			if (pmcr == NULL || pmcr == pmcr->pr_merge)
606 				break;
607 
608 			pmcstat_pmcinfilter++;
609 			if (pmcstat_pmcinfilter >= pmcstat_npmcs)
610 				pmcstat_pmcinfilter = 0;
611 
612 		} while (pmcstat_pmcinfilter != pmcin);
613 	}
614 }
615 
616 /*
617  * Top mode keypress.
618  */
619 
620 int
621 pmcstat_keypress_log(void)
622 {
623 	int c, ret = 0;
624 	WINDOW *w;
625 
626 	w = newwin(1, 0, 1, 0);
627 	c = wgetch(w);
628 	wprintw(w, "Key: %c => ", c);
629 	switch (c) {
630 	case 'A':
631 		if (args.pa_flags & FLAG_SKIP_TOP_FN_RES)
632 			args.pa_flags &= ~FLAG_SKIP_TOP_FN_RES;
633 		else
634 			args.pa_flags |= FLAG_SKIP_TOP_FN_RES;
635 		break;
636 	case 'c':
637 		wprintw(w, "enter mode 'd' or 'a' => ");
638 		c = wgetch(w);
639 		if (c == 'd') {
640 			args.pa_topmode = PMCSTAT_TOP_DELTA;
641 			wprintw(w, "switching to delta mode");
642 		} else {
643 			args.pa_topmode = PMCSTAT_TOP_ACCUM;
644 			wprintw(w, "switching to accumulation mode");
645 		}
646 		break;
647 	case 'I':
648 		if (args.pa_flags & FLAG_SHOW_OFFSET)
649 			args.pa_flags &= ~FLAG_SHOW_OFFSET;
650 		else
651 			args.pa_flags |= FLAG_SHOW_OFFSET;
652 		break;
653 	case 'm':
654 		pmcstat_mergepmc = !pmcstat_mergepmc;
655 		/*
656 		 * Changing merge state require data reset.
657 		 */
658 		if (plugins[args.pa_plugin].pl_shutdown != NULL)
659 			plugins[args.pa_plugin].pl_shutdown(NULL);
660 		pmcstat_stats_reset(0);
661 		if (plugins[args.pa_plugin].pl_init != NULL)
662 			plugins[args.pa_plugin].pl_init();
663 
664 		/* Update filter to be on a merge target. */
665 		pmcstat_changefilter();
666 		wprintw(w, "merge PMC %s", pmcstat_mergepmc ? "on" : "off");
667 		break;
668 	case 'n':
669 		/* Close current plugin. */
670 		if (plugins[args.pa_plugin].pl_shutdown != NULL)
671 			plugins[args.pa_plugin].pl_shutdown(NULL);
672 
673 		/* Find next top display available. */
674 		do {
675 			args.pa_plugin++;
676 			if (plugins[args.pa_plugin].pl_name == NULL)
677 				args.pa_plugin = 0;
678 		} while (plugins[args.pa_plugin].pl_topdisplay == NULL);
679 
680 		/* Open new plugin. */
681 		pmcstat_stats_reset(0);
682 		if (plugins[args.pa_plugin].pl_init != NULL)
683 			plugins[args.pa_plugin].pl_init();
684 		wprintw(w, "switching to plugin %s",
685 		    plugins[args.pa_plugin].pl_name);
686 		break;
687 	case 'p':
688 		pmcstat_pmcinfilter++;
689 		if (pmcstat_pmcinfilter >= pmcstat_npmcs)
690 			pmcstat_pmcinfilter = 0;
691 		pmcstat_changefilter();
692 		wprintw(w, "switching to PMC %s.%d",
693 		    pmcstat_pmcindex_to_name(pmcstat_pmcinfilter),
694 		    pmcstat_pmcinfilter);
695 		break;
696 	case ' ':
697 		pmcstat_pause = !pmcstat_pause;
698 		if (pmcstat_pause)
699 			wprintw(w, "pause => press space again to continue");
700 		break;
701 	case 'q':
702 		wprintw(w, "exiting...");
703 		ret = 1;
704 		break;
705 	default:
706 		if (plugins[args.pa_plugin].pl_topkeypress != NULL)
707 			if (plugins[args.pa_plugin].pl_topkeypress(c, (void *)w))
708 				ret = 1;
709 	}
710 
711 	wrefresh(w);
712 	delwin(w);
713 	return ret;
714 }
715 
716 
717 /*
718  * Top mode display.
719  */
720 
721 void
722 pmcstat_display_log(void)
723 {
724 
725 	pmcstat_refresh_top();
726 
727 	/* Reset everything if delta mode. */
728 	if (args.pa_topmode == PMCSTAT_TOP_DELTA) {
729 		if (plugins[args.pa_plugin].pl_shutdown != NULL)
730 			plugins[args.pa_plugin].pl_shutdown(NULL);
731 		pmcstat_stats_reset(0);
732 		if (plugins[args.pa_plugin].pl_init != NULL)
733 			plugins[args.pa_plugin].pl_init();
734 	}
735 }
736 
737 /*
738  * Configure a plugins.
739  */
740 
741 void
742 pmcstat_pluginconfigure_log(char *opt)
743 {
744 
745 	if (strncmp(opt, "threshold=", 10) == 0) {
746 		pmcstat_threshold = atof(opt+10);
747 	} else {
748 		if (plugins[args.pa_plugin].pl_configure != NULL) {
749 			if (!plugins[args.pa_plugin].pl_configure(opt))
750 				err(EX_USAGE,
751 				    "ERROR: unknown option <%s>.", opt);
752 		}
753 	}
754 }
755 
756 void
757 pmcstat_log_shutdown_logging(void)
758 {
759 
760 	pmcstat_shutdown_logging(&args, plugins, &pmcstat_stats);
761 }
762 
763 void
764 pmcstat_log_initialize_logging(void)
765 {
766 
767 	pmcstat_initialize_logging(&pmcstat_kernproc,
768 	    &args, plugins, &pmcstat_npmcs, &pmcstat_mergepmc);
769 }
770