xref: /freebsd/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c (revision f5f7c05209ca2c3748fd8b27c5e80ffad49120eb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Copyright (c) 2011, Joyent, Inc. All rights reserved.
28  */
29 
30 #include <stdlib.h>
31 #include <strings.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <limits.h>
35 #include <assert.h>
36 #include <ctype.h>
37 #if defined(sun)
38 #include <alloca.h>
39 #endif
40 #include <dt_impl.h>
41 #if !defined(sun)
42 #include <libproc_compat.h>
43 #endif
44 
45 #define	DT_MASK_LO 0x00000000FFFFFFFFULL
46 
47 /*
48  * We declare this here because (1) we need it and (2) we want to avoid a
49  * dependency on libm in libdtrace.
50  */
51 static long double
52 dt_fabsl(long double x)
53 {
54 	if (x < 0)
55 		return (-x);
56 
57 	return (x);
58 }
59 
60 /*
61  * 128-bit arithmetic functions needed to support the stddev() aggregating
62  * action.
63  */
64 static int
65 dt_gt_128(uint64_t *a, uint64_t *b)
66 {
67 	return (a[1] > b[1] || (a[1] == b[1] && a[0] > b[0]));
68 }
69 
70 static int
71 dt_ge_128(uint64_t *a, uint64_t *b)
72 {
73 	return (a[1] > b[1] || (a[1] == b[1] && a[0] >= b[0]));
74 }
75 
76 static int
77 dt_le_128(uint64_t *a, uint64_t *b)
78 {
79 	return (a[1] < b[1] || (a[1] == b[1] && a[0] <= b[0]));
80 }
81 
82 /*
83  * Shift the 128-bit value in a by b. If b is positive, shift left.
84  * If b is negative, shift right.
85  */
86 static void
87 dt_shift_128(uint64_t *a, int b)
88 {
89 	uint64_t mask;
90 
91 	if (b == 0)
92 		return;
93 
94 	if (b < 0) {
95 		b = -b;
96 		if (b >= 64) {
97 			a[0] = a[1] >> (b - 64);
98 			a[1] = 0;
99 		} else {
100 			a[0] >>= b;
101 			mask = 1LL << (64 - b);
102 			mask -= 1;
103 			a[0] |= ((a[1] & mask) << (64 - b));
104 			a[1] >>= b;
105 		}
106 	} else {
107 		if (b >= 64) {
108 			a[1] = a[0] << (b - 64);
109 			a[0] = 0;
110 		} else {
111 			a[1] <<= b;
112 			mask = a[0] >> (64 - b);
113 			a[1] |= mask;
114 			a[0] <<= b;
115 		}
116 	}
117 }
118 
119 static int
120 dt_nbits_128(uint64_t *a)
121 {
122 	int nbits = 0;
123 	uint64_t tmp[2];
124 	uint64_t zero[2] = { 0, 0 };
125 
126 	tmp[0] = a[0];
127 	tmp[1] = a[1];
128 
129 	dt_shift_128(tmp, -1);
130 	while (dt_gt_128(tmp, zero)) {
131 		dt_shift_128(tmp, -1);
132 		nbits++;
133 	}
134 
135 	return (nbits);
136 }
137 
138 static void
139 dt_subtract_128(uint64_t *minuend, uint64_t *subtrahend, uint64_t *difference)
140 {
141 	uint64_t result[2];
142 
143 	result[0] = minuend[0] - subtrahend[0];
144 	result[1] = minuend[1] - subtrahend[1] -
145 	    (minuend[0] < subtrahend[0] ? 1 : 0);
146 
147 	difference[0] = result[0];
148 	difference[1] = result[1];
149 }
150 
151 static void
152 dt_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
153 {
154 	uint64_t result[2];
155 
156 	result[0] = addend1[0] + addend2[0];
157 	result[1] = addend1[1] + addend2[1] +
158 	    (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
159 
160 	sum[0] = result[0];
161 	sum[1] = result[1];
162 }
163 
164 /*
165  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
166  * use native multiplication on those, and then re-combine into the
167  * resulting 128-bit value.
168  *
169  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
170  *     hi1 * hi2 << 64 +
171  *     hi1 * lo2 << 32 +
172  *     hi2 * lo1 << 32 +
173  *     lo1 * lo2
174  */
175 static void
176 dt_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
177 {
178 	uint64_t hi1, hi2, lo1, lo2;
179 	uint64_t tmp[2];
180 
181 	hi1 = factor1 >> 32;
182 	hi2 = factor2 >> 32;
183 
184 	lo1 = factor1 & DT_MASK_LO;
185 	lo2 = factor2 & DT_MASK_LO;
186 
187 	product[0] = lo1 * lo2;
188 	product[1] = hi1 * hi2;
189 
190 	tmp[0] = hi1 * lo2;
191 	tmp[1] = 0;
192 	dt_shift_128(tmp, 32);
193 	dt_add_128(product, tmp, product);
194 
195 	tmp[0] = hi2 * lo1;
196 	tmp[1] = 0;
197 	dt_shift_128(tmp, 32);
198 	dt_add_128(product, tmp, product);
199 }
200 
201 /*
202  * This is long-hand division.
203  *
204  * We initialize subtrahend by shifting divisor left as far as possible. We
205  * loop, comparing subtrahend to dividend:  if subtrahend is smaller, we
206  * subtract and set the appropriate bit in the result.  We then shift
207  * subtrahend right by one bit for the next comparison.
208  */
209 static void
210 dt_divide_128(uint64_t *dividend, uint64_t divisor, uint64_t *quotient)
211 {
212 	uint64_t result[2] = { 0, 0 };
213 	uint64_t remainder[2];
214 	uint64_t subtrahend[2];
215 	uint64_t divisor_128[2];
216 	uint64_t mask[2] = { 1, 0 };
217 	int log = 0;
218 
219 	assert(divisor != 0);
220 
221 	divisor_128[0] = divisor;
222 	divisor_128[1] = 0;
223 
224 	remainder[0] = dividend[0];
225 	remainder[1] = dividend[1];
226 
227 	subtrahend[0] = divisor;
228 	subtrahend[1] = 0;
229 
230 	while (divisor > 0) {
231 		log++;
232 		divisor >>= 1;
233 	}
234 
235 	dt_shift_128(subtrahend, 128 - log);
236 	dt_shift_128(mask, 128 - log);
237 
238 	while (dt_ge_128(remainder, divisor_128)) {
239 		if (dt_ge_128(remainder, subtrahend)) {
240 			dt_subtract_128(remainder, subtrahend, remainder);
241 			result[0] |= mask[0];
242 			result[1] |= mask[1];
243 		}
244 
245 		dt_shift_128(subtrahend, -1);
246 		dt_shift_128(mask, -1);
247 	}
248 
249 	quotient[0] = result[0];
250 	quotient[1] = result[1];
251 }
252 
253 /*
254  * This is the long-hand method of calculating a square root.
255  * The algorithm is as follows:
256  *
257  * 1. Group the digits by 2 from the right.
258  * 2. Over the leftmost group, find the largest single-digit number
259  *    whose square is less than that group.
260  * 3. Subtract the result of the previous step (2 or 4, depending) and
261  *    bring down the next two-digit group.
262  * 4. For the result R we have so far, find the largest single-digit number
263  *    x such that 2 * R * 10 * x + x^2 is less than the result from step 3.
264  *    (Note that this is doubling R and performing a decimal left-shift by 1
265  *    and searching for the appropriate decimal to fill the one's place.)
266  *    The value x is the next digit in the square root.
267  * Repeat steps 3 and 4 until the desired precision is reached.  (We're
268  * dealing with integers, so the above is sufficient.)
269  *
270  * In decimal, the square root of 582,734 would be calculated as so:
271  *
272  *     __7__6__3
273  *    | 58 27 34
274  *     -49       (7^2 == 49 => 7 is the first digit in the square root)
275  *      --
276  *       9 27    (Subtract and bring down the next group.)
277  * 146   8 76    (2 * 7 * 10 * 6 + 6^2 == 876 => 6 is the next digit in
278  *      -----     the square root)
279  *         51 34 (Subtract and bring down the next group.)
280  * 1523    45 69 (2 * 76 * 10 * 3 + 3^2 == 4569 => 3 is the next digit in
281  *         -----  the square root)
282  *          5 65 (remainder)
283  *
284  * The above algorithm applies similarly in binary, but note that the
285  * only possible non-zero value for x in step 4 is 1, so step 4 becomes a
286  * simple decision: is 2 * R * 2 * 1 + 1^2 (aka R << 2 + 1) less than the
287  * preceding difference?
288  *
289  * In binary, the square root of 11011011 would be calculated as so:
290  *
291  *     __1__1__1__0
292  *    | 11 01 10 11
293  *      01          (0 << 2 + 1 == 1 < 11 => this bit is 1)
294  *      --
295  *      10 01 10 11
296  * 101   1 01       (1 << 2 + 1 == 101 < 1001 => next bit is 1)
297  *      -----
298  *       1 00 10 11
299  * 1101    11 01    (11 << 2 + 1 == 1101 < 10010 => next bit is 1)
300  *       -------
301  *          1 01 11
302  * 11101    1 11 01 (111 << 2 + 1 == 11101 > 10111 => last bit is 0)
303  *
304  */
305 static uint64_t
306 dt_sqrt_128(uint64_t *square)
307 {
308 	uint64_t result[2] = { 0, 0 };
309 	uint64_t diff[2] = { 0, 0 };
310 	uint64_t one[2] = { 1, 0 };
311 	uint64_t next_pair[2];
312 	uint64_t next_try[2];
313 	uint64_t bit_pairs, pair_shift;
314 	int i;
315 
316 	bit_pairs = dt_nbits_128(square) / 2;
317 	pair_shift = bit_pairs * 2;
318 
319 	for (i = 0; i <= bit_pairs; i++) {
320 		/*
321 		 * Bring down the next pair of bits.
322 		 */
323 		next_pair[0] = square[0];
324 		next_pair[1] = square[1];
325 		dt_shift_128(next_pair, -pair_shift);
326 		next_pair[0] &= 0x3;
327 		next_pair[1] = 0;
328 
329 		dt_shift_128(diff, 2);
330 		dt_add_128(diff, next_pair, diff);
331 
332 		/*
333 		 * next_try = R << 2 + 1
334 		 */
335 		next_try[0] = result[0];
336 		next_try[1] = result[1];
337 		dt_shift_128(next_try, 2);
338 		dt_add_128(next_try, one, next_try);
339 
340 		if (dt_le_128(next_try, diff)) {
341 			dt_subtract_128(diff, next_try, diff);
342 			dt_shift_128(result, 1);
343 			dt_add_128(result, one, result);
344 		} else {
345 			dt_shift_128(result, 1);
346 		}
347 
348 		pair_shift -= 2;
349 	}
350 
351 	assert(result[1] == 0);
352 
353 	return (result[0]);
354 }
355 
356 uint64_t
357 dt_stddev(uint64_t *data, uint64_t normal)
358 {
359 	uint64_t avg_of_squares[2];
360 	uint64_t square_of_avg[2];
361 	int64_t norm_avg;
362 	uint64_t diff[2];
363 
364 	/*
365 	 * The standard approximation for standard deviation is
366 	 * sqrt(average(x**2) - average(x)**2), i.e. the square root
367 	 * of the average of the squares minus the square of the average.
368 	 */
369 	dt_divide_128(data + 2, normal, avg_of_squares);
370 	dt_divide_128(avg_of_squares, data[0], avg_of_squares);
371 
372 	norm_avg = (int64_t)data[1] / (int64_t)normal / (int64_t)data[0];
373 
374 	if (norm_avg < 0)
375 		norm_avg = -norm_avg;
376 
377 	dt_multiply_128((uint64_t)norm_avg, (uint64_t)norm_avg, square_of_avg);
378 
379 	dt_subtract_128(avg_of_squares, square_of_avg, diff);
380 
381 	return (dt_sqrt_128(diff));
382 }
383 
384 static int
385 dt_flowindent(dtrace_hdl_t *dtp, dtrace_probedata_t *data, dtrace_epid_t last,
386     dtrace_bufdesc_t *buf, size_t offs)
387 {
388 	dtrace_probedesc_t *pd = data->dtpda_pdesc, *npd;
389 	dtrace_eprobedesc_t *epd = data->dtpda_edesc, *nepd;
390 	char *p = pd->dtpd_provider, *n = pd->dtpd_name, *sub;
391 	dtrace_flowkind_t flow = DTRACEFLOW_NONE;
392 	const char *str = NULL;
393 	static const char *e_str[2] = { " -> ", " => " };
394 	static const char *r_str[2] = { " <- ", " <= " };
395 	static const char *ent = "entry", *ret = "return";
396 	static int entlen = 0, retlen = 0;
397 	dtrace_epid_t next, id = epd->dtepd_epid;
398 	int rval;
399 
400 	if (entlen == 0) {
401 		assert(retlen == 0);
402 		entlen = strlen(ent);
403 		retlen = strlen(ret);
404 	}
405 
406 	/*
407 	 * If the name of the probe is "entry" or ends with "-entry", we
408 	 * treat it as an entry; if it is "return" or ends with "-return",
409 	 * we treat it as a return.  (This allows application-provided probes
410 	 * like "method-entry" or "function-entry" to participate in flow
411 	 * indentation -- without accidentally misinterpreting popular probe
412 	 * names like "carpentry", "gentry" or "Coventry".)
413 	 */
414 	if ((sub = strstr(n, ent)) != NULL && sub[entlen] == '\0' &&
415 	    (sub == n || sub[-1] == '-')) {
416 		flow = DTRACEFLOW_ENTRY;
417 		str = e_str[strcmp(p, "syscall") == 0];
418 	} else if ((sub = strstr(n, ret)) != NULL && sub[retlen] == '\0' &&
419 	    (sub == n || sub[-1] == '-')) {
420 		flow = DTRACEFLOW_RETURN;
421 		str = r_str[strcmp(p, "syscall") == 0];
422 	}
423 
424 	/*
425 	 * If we're going to indent this, we need to check the ID of our last
426 	 * call.  If we're looking at the same probe ID but a different EPID,
427 	 * we _don't_ want to indent.  (Yes, there are some minor holes in
428 	 * this scheme -- it's a heuristic.)
429 	 */
430 	if (flow == DTRACEFLOW_ENTRY) {
431 		if ((last != DTRACE_EPIDNONE && id != last &&
432 		    pd->dtpd_id == dtp->dt_pdesc[last]->dtpd_id))
433 			flow = DTRACEFLOW_NONE;
434 	}
435 
436 	/*
437 	 * If we're going to unindent this, it's more difficult to see if
438 	 * we don't actually want to unindent it -- we need to look at the
439 	 * _next_ EPID.
440 	 */
441 	if (flow == DTRACEFLOW_RETURN) {
442 		offs += epd->dtepd_size;
443 
444 		do {
445 			if (offs >= buf->dtbd_size) {
446 				/*
447 				 * We're at the end -- maybe.  If the oldest
448 				 * record is non-zero, we need to wrap.
449 				 */
450 				if (buf->dtbd_oldest != 0) {
451 					offs = 0;
452 				} else {
453 					goto out;
454 				}
455 			}
456 
457 			next = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
458 
459 			if (next == DTRACE_EPIDNONE)
460 				offs += sizeof (id);
461 		} while (next == DTRACE_EPIDNONE);
462 
463 		if ((rval = dt_epid_lookup(dtp, next, &nepd, &npd)) != 0)
464 			return (rval);
465 
466 		if (next != id && npd->dtpd_id == pd->dtpd_id)
467 			flow = DTRACEFLOW_NONE;
468 	}
469 
470 out:
471 	if (flow == DTRACEFLOW_ENTRY || flow == DTRACEFLOW_RETURN) {
472 		data->dtpda_prefix = str;
473 	} else {
474 		data->dtpda_prefix = "| ";
475 	}
476 
477 	if (flow == DTRACEFLOW_RETURN && data->dtpda_indent > 0)
478 		data->dtpda_indent -= 2;
479 
480 	data->dtpda_flow = flow;
481 
482 	return (0);
483 }
484 
485 static int
486 dt_nullprobe()
487 {
488 	return (DTRACE_CONSUME_THIS);
489 }
490 
491 static int
492 dt_nullrec()
493 {
494 	return (DTRACE_CONSUME_NEXT);
495 }
496 
497 int
498 dt_print_quantline(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
499     uint64_t normal, long double total, char positives, char negatives)
500 {
501 	long double f;
502 	uint_t depth, len = 40;
503 
504 	const char *ats = "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@";
505 	const char *spaces = "                                        ";
506 
507 	assert(strlen(ats) == len && strlen(spaces) == len);
508 	assert(!(total == 0 && (positives || negatives)));
509 	assert(!(val < 0 && !negatives));
510 	assert(!(val > 0 && !positives));
511 	assert(!(val != 0 && total == 0));
512 
513 	if (!negatives) {
514 		if (positives) {
515 			f = (dt_fabsl((long double)val) * len) / total;
516 			depth = (uint_t)(f + 0.5);
517 		} else {
518 			depth = 0;
519 		}
520 
521 		return (dt_printf(dtp, fp, "|%s%s %-9lld\n", ats + len - depth,
522 		    spaces + depth, (long long)val / normal));
523 	}
524 
525 	if (!positives) {
526 		f = (dt_fabsl((long double)val) * len) / total;
527 		depth = (uint_t)(f + 0.5);
528 
529 		return (dt_printf(dtp, fp, "%s%s| %-9lld\n", spaces + depth,
530 		    ats + len - depth, (long long)val / normal));
531 	}
532 
533 	/*
534 	 * If we're here, we have both positive and negative bucket values.
535 	 * To express this graphically, we're going to generate both positive
536 	 * and negative bars separated by a centerline.  These bars are half
537 	 * the size of normal quantize()/lquantize() bars, so we divide the
538 	 * length in half before calculating the bar length.
539 	 */
540 	len /= 2;
541 	ats = &ats[len];
542 	spaces = &spaces[len];
543 
544 	f = (dt_fabsl((long double)val) * len) / total;
545 	depth = (uint_t)(f + 0.5);
546 
547 	if (val <= 0) {
548 		return (dt_printf(dtp, fp, "%s%s|%*s %-9lld\n", spaces + depth,
549 		    ats + len - depth, len, "", (long long)val / normal));
550 	} else {
551 		return (dt_printf(dtp, fp, "%20s|%s%s %-9lld\n", "",
552 		    ats + len - depth, spaces + depth,
553 		    (long long)val / normal));
554 	}
555 }
556 
557 int
558 dt_print_quantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
559     size_t size, uint64_t normal)
560 {
561 	const int64_t *data = addr;
562 	int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;
563 	long double total = 0;
564 	char positives = 0, negatives = 0;
565 
566 	if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
567 		return (dt_set_errno(dtp, EDT_DMISMATCH));
568 
569 	while (first_bin < DTRACE_QUANTIZE_NBUCKETS - 1 && data[first_bin] == 0)
570 		first_bin++;
571 
572 	if (first_bin == DTRACE_QUANTIZE_NBUCKETS - 1) {
573 		/*
574 		 * There isn't any data.  This is possible if (and only if)
575 		 * negative increment values have been used.  In this case,
576 		 * we'll print the buckets around 0.
577 		 */
578 		first_bin = DTRACE_QUANTIZE_ZEROBUCKET - 1;
579 		last_bin = DTRACE_QUANTIZE_ZEROBUCKET + 1;
580 	} else {
581 		if (first_bin > 0)
582 			first_bin--;
583 
584 		while (last_bin > 0 && data[last_bin] == 0)
585 			last_bin--;
586 
587 		if (last_bin < DTRACE_QUANTIZE_NBUCKETS - 1)
588 			last_bin++;
589 	}
590 
591 	for (i = first_bin; i <= last_bin; i++) {
592 		positives |= (data[i] > 0);
593 		negatives |= (data[i] < 0);
594 		total += dt_fabsl((long double)data[i]);
595 	}
596 
597 	if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
598 	    "------------- Distribution -------------", "count") < 0)
599 		return (-1);
600 
601 	for (i = first_bin; i <= last_bin; i++) {
602 		if (dt_printf(dtp, fp, "%16lld ",
603 		    (long long)DTRACE_QUANTIZE_BUCKETVAL(i)) < 0)
604 			return (-1);
605 
606 		if (dt_print_quantline(dtp, fp, data[i], normal, total,
607 		    positives, negatives) < 0)
608 			return (-1);
609 	}
610 
611 	return (0);
612 }
613 
614 int
615 dt_print_lquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
616     size_t size, uint64_t normal)
617 {
618 	const int64_t *data = addr;
619 	int i, first_bin, last_bin, base;
620 	uint64_t arg;
621 	long double total = 0;
622 	uint16_t step, levels;
623 	char positives = 0, negatives = 0;
624 
625 	if (size < sizeof (uint64_t))
626 		return (dt_set_errno(dtp, EDT_DMISMATCH));
627 
628 	arg = *data++;
629 	size -= sizeof (uint64_t);
630 
631 	base = DTRACE_LQUANTIZE_BASE(arg);
632 	step = DTRACE_LQUANTIZE_STEP(arg);
633 	levels = DTRACE_LQUANTIZE_LEVELS(arg);
634 
635 	first_bin = 0;
636 	last_bin = levels + 1;
637 
638 	if (size != sizeof (uint64_t) * (levels + 2))
639 		return (dt_set_errno(dtp, EDT_DMISMATCH));
640 
641 	while (first_bin <= levels + 1 && data[first_bin] == 0)
642 		first_bin++;
643 
644 	if (first_bin > levels + 1) {
645 		first_bin = 0;
646 		last_bin = 2;
647 	} else {
648 		if (first_bin > 0)
649 			first_bin--;
650 
651 		while (last_bin > 0 && data[last_bin] == 0)
652 			last_bin--;
653 
654 		if (last_bin < levels + 1)
655 			last_bin++;
656 	}
657 
658 	for (i = first_bin; i <= last_bin; i++) {
659 		positives |= (data[i] > 0);
660 		negatives |= (data[i] < 0);
661 		total += dt_fabsl((long double)data[i]);
662 	}
663 
664 	if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
665 	    "------------- Distribution -------------", "count") < 0)
666 		return (-1);
667 
668 	for (i = first_bin; i <= last_bin; i++) {
669 		char c[32];
670 		int err;
671 
672 		if (i == 0) {
673 			(void) snprintf(c, sizeof (c), "< %d",
674 			    base / (uint32_t)normal);
675 			err = dt_printf(dtp, fp, "%16s ", c);
676 		} else if (i == levels + 1) {
677 			(void) snprintf(c, sizeof (c), ">= %d",
678 			    base + (levels * step));
679 			err = dt_printf(dtp, fp, "%16s ", c);
680 		} else {
681 			err = dt_printf(dtp, fp, "%16d ",
682 			    base + (i - 1) * step);
683 		}
684 
685 		if (err < 0 || dt_print_quantline(dtp, fp, data[i], normal,
686 		    total, positives, negatives) < 0)
687 			return (-1);
688 	}
689 
690 	return (0);
691 }
692 
693 int
694 dt_print_llquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
695     size_t size, uint64_t normal)
696 {
697 	int i, first_bin, last_bin, bin = 1, order, levels;
698 	uint16_t factor, low, high, nsteps;
699 	const int64_t *data = addr;
700 	int64_t value = 1, next, step;
701 	char positives = 0, negatives = 0;
702 	long double total = 0;
703 	uint64_t arg;
704 	char c[32];
705 
706 	if (size < sizeof (uint64_t))
707 		return (dt_set_errno(dtp, EDT_DMISMATCH));
708 
709 	arg = *data++;
710 	size -= sizeof (uint64_t);
711 
712 	factor = DTRACE_LLQUANTIZE_FACTOR(arg);
713 	low = DTRACE_LLQUANTIZE_LOW(arg);
714 	high = DTRACE_LLQUANTIZE_HIGH(arg);
715 	nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
716 
717 	/*
718 	 * We don't expect to be handed invalid llquantize() parameters here,
719 	 * but sanity check them (to a degree) nonetheless.
720 	 */
721 	if (size > INT32_MAX || factor < 2 || low >= high ||
722 	    nsteps == 0 || factor > nsteps)
723 		return (dt_set_errno(dtp, EDT_DMISMATCH));
724 
725 	levels = (int)size / sizeof (uint64_t);
726 
727 	first_bin = 0;
728 	last_bin = levels - 1;
729 
730 	while (first_bin < levels && data[first_bin] == 0)
731 		first_bin++;
732 
733 	if (first_bin == levels) {
734 		first_bin = 0;
735 		last_bin = 1;
736 	} else {
737 		if (first_bin > 0)
738 			first_bin--;
739 
740 		while (last_bin > 0 && data[last_bin] == 0)
741 			last_bin--;
742 
743 		if (last_bin < levels - 1)
744 			last_bin++;
745 	}
746 
747 	for (i = first_bin; i <= last_bin; i++) {
748 		positives |= (data[i] > 0);
749 		negatives |= (data[i] < 0);
750 		total += dt_fabsl((long double)data[i]);
751 	}
752 
753 	if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
754 	    "------------- Distribution -------------", "count") < 0)
755 		return (-1);
756 
757 	for (order = 0; order < low; order++)
758 		value *= factor;
759 
760 	next = value * factor;
761 	step = next > nsteps ? next / nsteps : 1;
762 
763 	if (first_bin == 0) {
764 		(void) snprintf(c, sizeof (c), "< %lld", (long long)value);
765 
766 		if (dt_printf(dtp, fp, "%16s ", c) < 0)
767 			return (-1);
768 
769 		if (dt_print_quantline(dtp, fp, data[0], normal,
770 		    total, positives, negatives) < 0)
771 			return (-1);
772 	}
773 
774 	while (order <= high) {
775 		if (bin >= first_bin && bin <= last_bin) {
776 			if (dt_printf(dtp, fp, "%16lld ", (long long)value) < 0)
777 				return (-1);
778 
779 			if (dt_print_quantline(dtp, fp, data[bin],
780 			    normal, total, positives, negatives) < 0)
781 				return (-1);
782 		}
783 
784 		assert(value < next);
785 		bin++;
786 
787 		if ((value += step) != next)
788 			continue;
789 
790 		next = value * factor;
791 		step = next > nsteps ? next / nsteps : 1;
792 		order++;
793 	}
794 
795 	if (last_bin < bin)
796 		return (0);
797 
798 	assert(last_bin == bin);
799 	(void) snprintf(c, sizeof (c), ">= %lld", (long long)value);
800 
801 	if (dt_printf(dtp, fp, "%16s ", c) < 0)
802 		return (-1);
803 
804 	return (dt_print_quantline(dtp, fp, data[bin], normal,
805 	    total, positives, negatives));
806 }
807 
808 /*ARGSUSED*/
809 static int
810 dt_print_average(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
811     size_t size, uint64_t normal)
812 {
813 	/* LINTED - alignment */
814 	int64_t *data = (int64_t *)addr;
815 
816 	return (dt_printf(dtp, fp, " %16lld", data[0] ?
817 	    (long long)(data[1] / (int64_t)normal / data[0]) : 0));
818 }
819 
820 /*ARGSUSED*/
821 static int
822 dt_print_stddev(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
823     size_t size, uint64_t normal)
824 {
825 	/* LINTED - alignment */
826 	uint64_t *data = (uint64_t *)addr;
827 
828 	return (dt_printf(dtp, fp, " %16llu", data[0] ?
829 	    (unsigned long long) dt_stddev(data, normal) : 0));
830 }
831 
832 /*ARGSUSED*/
833 int
834 dt_print_bytes(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
835     size_t nbytes, int width, int quiet, int raw)
836 {
837 	/*
838 	 * If the byte stream is a series of printable characters, followed by
839 	 * a terminating byte, we print it out as a string.  Otherwise, we
840 	 * assume that it's something else and just print the bytes.
841 	 */
842 	int i, j, margin = 5;
843 	char *c = (char *)addr;
844 
845 	if (nbytes == 0)
846 		return (0);
847 
848 	if (raw || dtp->dt_options[DTRACEOPT_RAWBYTES] != DTRACEOPT_UNSET)
849 		goto raw;
850 
851 	for (i = 0; i < nbytes; i++) {
852 		/*
853 		 * We define a "printable character" to be one for which
854 		 * isprint(3C) returns non-zero, isspace(3C) returns non-zero,
855 		 * or a character which is either backspace or the bell.
856 		 * Backspace and the bell are regrettably special because
857 		 * they fail the first two tests -- and yet they are entirely
858 		 * printable.  These are the only two control characters that
859 		 * have meaning for the terminal and for which isprint(3C) and
860 		 * isspace(3C) return 0.
861 		 */
862 		if (isprint(c[i]) || isspace(c[i]) ||
863 		    c[i] == '\b' || c[i] == '\a')
864 			continue;
865 
866 		if (c[i] == '\0' && i > 0) {
867 			/*
868 			 * This looks like it might be a string.  Before we
869 			 * assume that it is indeed a string, check the
870 			 * remainder of the byte range; if it contains
871 			 * additional non-nul characters, we'll assume that
872 			 * it's a binary stream that just happens to look like
873 			 * a string, and we'll print out the individual bytes.
874 			 */
875 			for (j = i + 1; j < nbytes; j++) {
876 				if (c[j] != '\0')
877 					break;
878 			}
879 
880 			if (j != nbytes)
881 				break;
882 
883 			if (quiet)
884 				return (dt_printf(dtp, fp, "%s", c));
885 			else
886 				return (dt_printf(dtp, fp, "  %-*s", width, c));
887 		}
888 
889 		break;
890 	}
891 
892 	if (i == nbytes) {
893 		/*
894 		 * The byte range is all printable characters, but there is
895 		 * no trailing nul byte.  We'll assume that it's a string and
896 		 * print it as such.
897 		 */
898 		char *s = alloca(nbytes + 1);
899 		bcopy(c, s, nbytes);
900 		s[nbytes] = '\0';
901 		return (dt_printf(dtp, fp, "  %-*s", width, s));
902 	}
903 
904 raw:
905 	if (dt_printf(dtp, fp, "\n%*s      ", margin, "") < 0)
906 		return (-1);
907 
908 	for (i = 0; i < 16; i++)
909 		if (dt_printf(dtp, fp, "  %c", "0123456789abcdef"[i]) < 0)
910 			return (-1);
911 
912 	if (dt_printf(dtp, fp, "  0123456789abcdef\n") < 0)
913 		return (-1);
914 
915 
916 	for (i = 0; i < nbytes; i += 16) {
917 		if (dt_printf(dtp, fp, "%*s%5x:", margin, "", i) < 0)
918 			return (-1);
919 
920 		for (j = i; j < i + 16 && j < nbytes; j++) {
921 			if (dt_printf(dtp, fp, " %02x", (uchar_t)c[j]) < 0)
922 				return (-1);
923 		}
924 
925 		while (j++ % 16) {
926 			if (dt_printf(dtp, fp, "   ") < 0)
927 				return (-1);
928 		}
929 
930 		if (dt_printf(dtp, fp, "  ") < 0)
931 			return (-1);
932 
933 		for (j = i; j < i + 16 && j < nbytes; j++) {
934 			if (dt_printf(dtp, fp, "%c",
935 			    c[j] < ' ' || c[j] > '~' ? '.' : c[j]) < 0)
936 				return (-1);
937 		}
938 
939 		if (dt_printf(dtp, fp, "\n") < 0)
940 			return (-1);
941 	}
942 
943 	return (0);
944 }
945 
946 int
947 dt_print_stack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
948     caddr_t addr, int depth, int size)
949 {
950 	dtrace_syminfo_t dts;
951 	GElf_Sym sym;
952 	int i, indent;
953 	char c[PATH_MAX * 2];
954 	uint64_t pc;
955 
956 	if (dt_printf(dtp, fp, "\n") < 0)
957 		return (-1);
958 
959 	if (format == NULL)
960 		format = "%s";
961 
962 	if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
963 		indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
964 	else
965 		indent = _dtrace_stkindent;
966 
967 	for (i = 0; i < depth; i++) {
968 		switch (size) {
969 		case sizeof (uint32_t):
970 			/* LINTED - alignment */
971 			pc = *((uint32_t *)addr);
972 			break;
973 
974 		case sizeof (uint64_t):
975 			/* LINTED - alignment */
976 			pc = *((uint64_t *)addr);
977 			break;
978 
979 		default:
980 			return (dt_set_errno(dtp, EDT_BADSTACKPC));
981 		}
982 
983 		if (pc == 0)
984 			break;
985 
986 		addr += size;
987 
988 		if (dt_printf(dtp, fp, "%*s", indent, "") < 0)
989 			return (-1);
990 
991 		if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
992 			if (pc > sym.st_value) {
993 				(void) snprintf(c, sizeof (c), "%s`%s+0x%llx",
994 				    dts.dts_object, dts.dts_name,
995 				    (u_longlong_t)(pc - sym.st_value));
996 			} else {
997 				(void) snprintf(c, sizeof (c), "%s`%s",
998 				    dts.dts_object, dts.dts_name);
999 			}
1000 		} else {
1001 			/*
1002 			 * We'll repeat the lookup, but this time we'll specify
1003 			 * a NULL GElf_Sym -- indicating that we're only
1004 			 * interested in the containing module.
1005 			 */
1006 			if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1007 				(void) snprintf(c, sizeof (c), "%s`0x%llx",
1008 				    dts.dts_object, (u_longlong_t)pc);
1009 			} else {
1010 				(void) snprintf(c, sizeof (c), "0x%llx",
1011 				    (u_longlong_t)pc);
1012 			}
1013 		}
1014 
1015 		if (dt_printf(dtp, fp, format, c) < 0)
1016 			return (-1);
1017 
1018 		if (dt_printf(dtp, fp, "\n") < 0)
1019 			return (-1);
1020 	}
1021 
1022 	return (0);
1023 }
1024 
1025 int
1026 dt_print_ustack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
1027     caddr_t addr, uint64_t arg)
1028 {
1029 	/* LINTED - alignment */
1030 	uint64_t *pc = (uint64_t *)addr;
1031 	uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
1032 	uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
1033 	const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
1034 	const char *str = strsize ? strbase : NULL;
1035 	int err = 0;
1036 
1037 	char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
1038 	struct ps_prochandle *P;
1039 	GElf_Sym sym;
1040 	int i, indent;
1041 	pid_t pid;
1042 
1043 	if (depth == 0)
1044 		return (0);
1045 
1046 	pid = (pid_t)*pc++;
1047 
1048 	if (dt_printf(dtp, fp, "\n") < 0)
1049 		return (-1);
1050 
1051 	if (format == NULL)
1052 		format = "%s";
1053 
1054 	if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
1055 		indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
1056 	else
1057 		indent = _dtrace_stkindent;
1058 
1059 	/*
1060 	 * Ultimately, we need to add an entry point in the library vector for
1061 	 * determining <symbol, offset> from <pid, address>.  For now, if
1062 	 * this is a vector open, we just print the raw address or string.
1063 	 */
1064 	if (dtp->dt_vector == NULL)
1065 		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1066 	else
1067 		P = NULL;
1068 
1069 	if (P != NULL)
1070 		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1071 
1072 	for (i = 0; i < depth && pc[i] != 0; i++) {
1073 		const prmap_t *map;
1074 
1075 		if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1076 			break;
1077 
1078 		if (P != NULL && Plookup_by_addr(P, pc[i],
1079 		    name, sizeof (name), &sym) == 0) {
1080 			(void) Pobjname(P, pc[i], objname, sizeof (objname));
1081 
1082 			if (pc[i] > sym.st_value) {
1083 				(void) snprintf(c, sizeof (c),
1084 				    "%s`%s+0x%llx", dt_basename(objname), name,
1085 				    (u_longlong_t)(pc[i] - sym.st_value));
1086 			} else {
1087 				(void) snprintf(c, sizeof (c),
1088 				    "%s`%s", dt_basename(objname), name);
1089 			}
1090 		} else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
1091 		    (P != NULL && ((map = Paddr_to_map(P, pc[i])) == NULL ||
1092 		    (map->pr_mflags & MA_WRITE)))) {
1093 			/*
1094 			 * If the current string pointer in the string table
1095 			 * does not point to an empty string _and_ the program
1096 			 * counter falls in a writable region, we'll use the
1097 			 * string from the string table instead of the raw
1098 			 * address.  This last condition is necessary because
1099 			 * some (broken) ustack helpers will return a string
1100 			 * even for a program counter that they can't
1101 			 * identify.  If we have a string for a program
1102 			 * counter that falls in a segment that isn't
1103 			 * writable, we assume that we have fallen into this
1104 			 * case and we refuse to use the string.
1105 			 */
1106 			(void) snprintf(c, sizeof (c), "%s", str);
1107 		} else {
1108 			if (P != NULL && Pobjname(P, pc[i], objname,
1109 			    sizeof (objname)) != 0) {
1110 				(void) snprintf(c, sizeof (c), "%s`0x%llx",
1111 				    dt_basename(objname), (u_longlong_t)pc[i]);
1112 			} else {
1113 				(void) snprintf(c, sizeof (c), "0x%llx",
1114 				    (u_longlong_t)pc[i]);
1115 			}
1116 		}
1117 
1118 		if ((err = dt_printf(dtp, fp, format, c)) < 0)
1119 			break;
1120 
1121 		if ((err = dt_printf(dtp, fp, "\n")) < 0)
1122 			break;
1123 
1124 		if (str != NULL && str[0] == '@') {
1125 			/*
1126 			 * If the first character of the string is an "at" sign,
1127 			 * then the string is inferred to be an annotation --
1128 			 * and it is printed out beneath the frame and offset
1129 			 * with brackets.
1130 			 */
1131 			if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1132 				break;
1133 
1134 			(void) snprintf(c, sizeof (c), "  [ %s ]", &str[1]);
1135 
1136 			if ((err = dt_printf(dtp, fp, format, c)) < 0)
1137 				break;
1138 
1139 			if ((err = dt_printf(dtp, fp, "\n")) < 0)
1140 				break;
1141 		}
1142 
1143 		if (str != NULL) {
1144 			str += strlen(str) + 1;
1145 			if (str - strbase >= strsize)
1146 				str = NULL;
1147 		}
1148 	}
1149 
1150 	if (P != NULL) {
1151 		dt_proc_unlock(dtp, P);
1152 		dt_proc_release(dtp, P);
1153 	}
1154 
1155 	return (err);
1156 }
1157 
1158 static int
1159 dt_print_usym(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr, dtrace_actkind_t act)
1160 {
1161 	/* LINTED - alignment */
1162 	uint64_t pid = ((uint64_t *)addr)[0];
1163 	/* LINTED - alignment */
1164 	uint64_t pc = ((uint64_t *)addr)[1];
1165 	const char *format = "  %-50s";
1166 	char *s;
1167 	int n, len = 256;
1168 
1169 	if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
1170 		struct ps_prochandle *P;
1171 
1172 		if ((P = dt_proc_grab(dtp, pid,
1173 		    PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
1174 			GElf_Sym sym;
1175 
1176 			dt_proc_lock(dtp, P);
1177 
1178 			if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
1179 				pc = sym.st_value;
1180 
1181 			dt_proc_unlock(dtp, P);
1182 			dt_proc_release(dtp, P);
1183 		}
1184 	}
1185 
1186 	do {
1187 		n = len;
1188 		s = alloca(n);
1189 	} while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) > n);
1190 
1191 	return (dt_printf(dtp, fp, format, s));
1192 }
1193 
1194 int
1195 dt_print_umod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1196 {
1197 	/* LINTED - alignment */
1198 	uint64_t pid = ((uint64_t *)addr)[0];
1199 	/* LINTED - alignment */
1200 	uint64_t pc = ((uint64_t *)addr)[1];
1201 	int err = 0;
1202 
1203 	char objname[PATH_MAX], c[PATH_MAX * 2];
1204 	struct ps_prochandle *P;
1205 
1206 	if (format == NULL)
1207 		format = "  %-50s";
1208 
1209 	/*
1210 	 * See the comment in dt_print_ustack() for the rationale for
1211 	 * printing raw addresses in the vectored case.
1212 	 */
1213 	if (dtp->dt_vector == NULL)
1214 		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1215 	else
1216 		P = NULL;
1217 
1218 	if (P != NULL)
1219 		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1220 
1221 	if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != 0) {
1222 		(void) snprintf(c, sizeof (c), "%s", dt_basename(objname));
1223 	} else {
1224 		(void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1225 	}
1226 
1227 	err = dt_printf(dtp, fp, format, c);
1228 
1229 	if (P != NULL) {
1230 		dt_proc_unlock(dtp, P);
1231 		dt_proc_release(dtp, P);
1232 	}
1233 
1234 	return (err);
1235 }
1236 
1237 int
1238 dt_print_memory(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1239 {
1240 	int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1241 	size_t nbytes = *((uintptr_t *) addr);
1242 
1243 	return (dt_print_bytes(dtp, fp, addr + sizeof(uintptr_t),
1244 	    nbytes, 50, quiet, 1));
1245 }
1246 
1247 typedef struct dt_type_cbdata {
1248 	dtrace_hdl_t		*dtp;
1249 	dtrace_typeinfo_t	dtt;
1250 	caddr_t			addr;
1251 	caddr_t			addrend;
1252 	const char		*name;
1253 	int			f_type;
1254 	int			indent;
1255 	int			type_width;
1256 	int			name_width;
1257 	FILE			*fp;
1258 } dt_type_cbdata_t;
1259 
1260 static int	dt_print_type_data(dt_type_cbdata_t *, ctf_id_t);
1261 
1262 static int
1263 dt_print_type_member(const char *name, ctf_id_t type, ulong_t off, void *arg)
1264 {
1265 	dt_type_cbdata_t cbdata;
1266 	dt_type_cbdata_t *cbdatap = arg;
1267 	ssize_t ssz;
1268 
1269 	if ((ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type)) <= 0)
1270 		return (0);
1271 
1272 	off /= 8;
1273 
1274 	cbdata = *cbdatap;
1275 	cbdata.name = name;
1276 	cbdata.addr += off;
1277 	cbdata.addrend = cbdata.addr + ssz;
1278 
1279 	return (dt_print_type_data(&cbdata, type));
1280 }
1281 
1282 static int
1283 dt_print_type_width(const char *name, ctf_id_t type, ulong_t off, void *arg)
1284 {
1285 	char buf[DT_TYPE_NAMELEN];
1286 	char *p;
1287 	dt_type_cbdata_t *cbdatap = arg;
1288 	size_t sz = strlen(name);
1289 
1290 	ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1291 
1292 	if ((p = strchr(buf, '[')) != NULL)
1293 		p[-1] = '\0';
1294 	else
1295 		p = "";
1296 
1297 	sz += strlen(p);
1298 
1299 	if (sz > cbdatap->name_width)
1300 		cbdatap->name_width = sz;
1301 
1302 	sz = strlen(buf);
1303 
1304 	if (sz > cbdatap->type_width)
1305 		cbdatap->type_width = sz;
1306 
1307 	return (0);
1308 }
1309 
1310 static int
1311 dt_print_type_data(dt_type_cbdata_t *cbdatap, ctf_id_t type)
1312 {
1313 	caddr_t addr = cbdatap->addr;
1314 	caddr_t addrend = cbdatap->addrend;
1315 	char buf[DT_TYPE_NAMELEN];
1316 	char *p;
1317 	int cnt = 0;
1318 	uint_t kind = ctf_type_kind(cbdatap->dtt.dtt_ctfp, type);
1319 	ssize_t ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type);
1320 
1321 	ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1322 
1323 	if ((p = strchr(buf, '[')) != NULL)
1324 		p[-1] = '\0';
1325 	else
1326 		p = "";
1327 
1328 	if (cbdatap->f_type) {
1329 		int type_width = roundup(cbdatap->type_width + 1, 4);
1330 		int name_width = roundup(cbdatap->name_width + 1, 4);
1331 
1332 		name_width -= strlen(cbdatap->name);
1333 
1334 		dt_printf(cbdatap->dtp, cbdatap->fp, "%*s%-*s%s%-*s	= ",cbdatap->indent * 4,"",type_width,buf,cbdatap->name,name_width,p);
1335 	}
1336 
1337 	while (addr < addrend) {
1338 		dt_type_cbdata_t cbdata;
1339 		ctf_arinfo_t arinfo;
1340 		ctf_encoding_t cte;
1341 		uintptr_t *up;
1342 		void *vp = addr;
1343 		cbdata = *cbdatap;
1344 		cbdata.name = "";
1345 		cbdata.addr = addr;
1346 		cbdata.addrend = addr + ssz;
1347 		cbdata.f_type = 0;
1348 		cbdata.indent++;
1349 		cbdata.type_width = 0;
1350 		cbdata.name_width = 0;
1351 
1352 		if (cnt > 0)
1353 			dt_printf(cbdatap->dtp, cbdatap->fp, "%*s", cbdatap->indent * 4,"");
1354 
1355 		switch (kind) {
1356 		case CTF_K_INTEGER:
1357 			if (ctf_type_encoding(cbdatap->dtt.dtt_ctfp, type, &cte) != 0)
1358 				return (-1);
1359 			if ((cte.cte_format & CTF_INT_SIGNED) != 0)
1360 				switch (cte.cte_bits) {
1361 				case 8:
1362 					if (isprint(*((char *) vp)))
1363 						dt_printf(cbdatap->dtp, cbdatap->fp, "'%c', ", *((char *) vp));
1364 					dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((char *) vp), *((char *) vp));
1365 					break;
1366 				case 16:
1367 					dt_printf(cbdatap->dtp, cbdatap->fp, "%hd (0x%hx);\n", *((short *) vp), *((u_short *) vp));
1368 					break;
1369 				case 32:
1370 					dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((int *) vp), *((u_int *) vp));
1371 					break;
1372 				case 64:
1373 					dt_printf(cbdatap->dtp, cbdatap->fp, "%jd (0x%jx);\n", *((long long *) vp), *((unsigned long long *) vp));
1374 					break;
1375 				default:
1376 					dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1377 					break;
1378 				}
1379 			else
1380 				switch (cte.cte_bits) {
1381 				case 8:
1382 					dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((uint8_t *) vp) & 0xff, *((uint8_t *) vp) & 0xff);
1383 					break;
1384 				case 16:
1385 					dt_printf(cbdatap->dtp, cbdatap->fp, "%hu (0x%hx);\n", *((u_short *) vp), *((u_short *) vp));
1386 					break;
1387 				case 32:
1388 					dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((u_int *) vp), *((u_int *) vp));
1389 					break;
1390 				case 64:
1391 					dt_printf(cbdatap->dtp, cbdatap->fp, "%ju (0x%jx);\n", *((unsigned long long *) vp), *((unsigned long long *) vp));
1392 					break;
1393 				default:
1394 					dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1395 					break;
1396 				}
1397 			break;
1398 		case CTF_K_FLOAT:
1399 			dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FLOAT: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1400 			break;
1401 		case CTF_K_POINTER:
1402 			dt_printf(cbdatap->dtp, cbdatap->fp, "%p;\n", *((void **) addr));
1403 			break;
1404 		case CTF_K_ARRAY:
1405 			if (ctf_array_info(cbdatap->dtt.dtt_ctfp, type, &arinfo) != 0)
1406 				return (-1);
1407 			dt_printf(cbdatap->dtp, cbdatap->fp, "{\n%*s",cbdata.indent * 4,"");
1408 			dt_print_type_data(&cbdata, arinfo.ctr_contents);
1409 			dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1410 			break;
1411 		case CTF_K_FUNCTION:
1412 			dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FUNCTION:\n");
1413 			break;
1414 		case CTF_K_STRUCT:
1415 			cbdata.f_type = 1;
1416 			if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1417 			    dt_print_type_width, &cbdata) != 0)
1418 				return (-1);
1419 			dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1420 			if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1421 			    dt_print_type_member, &cbdata) != 0)
1422 				return (-1);
1423 			dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1424 			break;
1425 		case CTF_K_UNION:
1426 			cbdata.f_type = 1;
1427 			if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1428 			    dt_print_type_width, &cbdata) != 0)
1429 				return (-1);
1430 			dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1431 			if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1432 			    dt_print_type_member, &cbdata) != 0)
1433 				return (-1);
1434 			dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1435 			break;
1436 		case CTF_K_ENUM:
1437 			dt_printf(cbdatap->dtp, cbdatap->fp, "%s;\n", ctf_enum_name(cbdatap->dtt.dtt_ctfp, type, *((int *) vp)));
1438 			break;
1439 		case CTF_K_TYPEDEF:
1440 			dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1441 			break;
1442 		case CTF_K_VOLATILE:
1443 			if (cbdatap->f_type)
1444 				dt_printf(cbdatap->dtp, cbdatap->fp, "volatile ");
1445 			dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1446 			break;
1447 		case CTF_K_CONST:
1448 			if (cbdatap->f_type)
1449 				dt_printf(cbdatap->dtp, cbdatap->fp, "const ");
1450 			dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1451 			break;
1452 		case CTF_K_RESTRICT:
1453 			if (cbdatap->f_type)
1454 				dt_printf(cbdatap->dtp, cbdatap->fp, "restrict ");
1455 			dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1456 			break;
1457 		default:
1458 			break;
1459 		}
1460 
1461 		addr += ssz;
1462 		cnt++;
1463 	}
1464 
1465 	return (0);
1466 }
1467 
1468 static int
1469 dt_print_type(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1470 {
1471 	caddr_t addrend;
1472 	char *p;
1473 	dtrace_typeinfo_t dtt;
1474 	dt_type_cbdata_t cbdata;
1475 	int num = 0;
1476 	int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1477 	ssize_t ssz;
1478 
1479 	if (!quiet)
1480 		dt_printf(dtp, fp, "\n");
1481 
1482 	/* Get the total number of bytes of data buffered. */
1483 	size_t nbytes = *((uintptr_t *) addr);
1484 	addr += sizeof(uintptr_t);
1485 
1486 	/*
1487 	 * Get the size of the type so that we can check that it matches
1488 	 * the CTF data we look up and so that we can figure out how many
1489 	 * type elements are buffered.
1490 	 */
1491 	size_t typs = *((uintptr_t *) addr);
1492 	addr += sizeof(uintptr_t);
1493 
1494 	/*
1495 	 * Point to the type string in the buffer. Get it's string
1496 	 * length and round it up to become the offset to the start
1497 	 * of the buffered type data which we would like to be aligned
1498 	 * for easy access.
1499 	 */
1500 	char *strp = (char *) addr;
1501 	int offset = roundup(strlen(strp) + 1, sizeof(uintptr_t));
1502 
1503 	/*
1504 	 * The type string might have a format such as 'int [20]'.
1505 	 * Check if there is an array dimension present.
1506 	 */
1507 	if ((p = strchr(strp, '[')) != NULL) {
1508 		/* Strip off the array dimension. */
1509 		*p++ = '\0';
1510 
1511 		for (; *p != '\0' && *p != ']'; p++)
1512 			num = num * 10 + *p - '0';
1513 	} else
1514 		/* No array dimension, so default. */
1515 		num = 1;
1516 
1517 	/* Lookup the CTF type from the type string. */
1518 	if (dtrace_lookup_by_type(dtp,  DTRACE_OBJ_EVERY, strp, &dtt) < 0)
1519 		return (-1);
1520 
1521 	/* Offset the buffer address to the start of the data... */
1522 	addr += offset;
1523 
1524 	ssz = ctf_type_size(dtt.dtt_ctfp, dtt.dtt_type);
1525 
1526 	if (typs != ssz) {
1527 		printf("Expected type size from buffer (%lu) to match type size looked up now (%ld)\n", (u_long) typs, (long) ssz);
1528 		return (-1);
1529 	}
1530 
1531 	cbdata.dtp = dtp;
1532 	cbdata.dtt = dtt;
1533 	cbdata.name = "";
1534 	cbdata.addr = addr;
1535 	cbdata.addrend = addr + nbytes;
1536 	cbdata.indent = 1;
1537 	cbdata.f_type = 1;
1538 	cbdata.type_width = 0;
1539 	cbdata.name_width = 0;
1540 	cbdata.fp = fp;
1541 
1542 	return (dt_print_type_data(&cbdata, dtt.dtt_type));
1543 }
1544 
1545 static int
1546 dt_print_sym(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1547 {
1548 	/* LINTED - alignment */
1549 	uint64_t pc = *((uint64_t *)addr);
1550 	dtrace_syminfo_t dts;
1551 	GElf_Sym sym;
1552 	char c[PATH_MAX * 2];
1553 
1554 	if (format == NULL)
1555 		format = "  %-50s";
1556 
1557 	if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1558 		(void) snprintf(c, sizeof (c), "%s`%s",
1559 		    dts.dts_object, dts.dts_name);
1560 	} else {
1561 		/*
1562 		 * We'll repeat the lookup, but this time we'll specify a
1563 		 * NULL GElf_Sym -- indicating that we're only interested in
1564 		 * the containing module.
1565 		 */
1566 		if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1567 			(void) snprintf(c, sizeof (c), "%s`0x%llx",
1568 			    dts.dts_object, (u_longlong_t)pc);
1569 		} else {
1570 			(void) snprintf(c, sizeof (c), "0x%llx",
1571 			    (u_longlong_t)pc);
1572 		}
1573 	}
1574 
1575 	if (dt_printf(dtp, fp, format, c) < 0)
1576 		return (-1);
1577 
1578 	return (0);
1579 }
1580 
1581 int
1582 dt_print_mod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1583 {
1584 	/* LINTED - alignment */
1585 	uint64_t pc = *((uint64_t *)addr);
1586 	dtrace_syminfo_t dts;
1587 	char c[PATH_MAX * 2];
1588 
1589 	if (format == NULL)
1590 		format = "  %-50s";
1591 
1592 	if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1593 		(void) snprintf(c, sizeof (c), "%s", dts.dts_object);
1594 	} else {
1595 		(void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1596 	}
1597 
1598 	if (dt_printf(dtp, fp, format, c) < 0)
1599 		return (-1);
1600 
1601 	return (0);
1602 }
1603 
1604 typedef struct dt_normal {
1605 	dtrace_aggvarid_t dtnd_id;
1606 	uint64_t dtnd_normal;
1607 } dt_normal_t;
1608 
1609 static int
1610 dt_normalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1611 {
1612 	dt_normal_t *normal = arg;
1613 	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1614 	dtrace_aggvarid_t id = normal->dtnd_id;
1615 
1616 	if (agg->dtagd_nrecs == 0)
1617 		return (DTRACE_AGGWALK_NEXT);
1618 
1619 	if (agg->dtagd_varid != id)
1620 		return (DTRACE_AGGWALK_NEXT);
1621 
1622 	((dtrace_aggdata_t *)aggdata)->dtada_normal = normal->dtnd_normal;
1623 	return (DTRACE_AGGWALK_NORMALIZE);
1624 }
1625 
1626 static int
1627 dt_normalize(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1628 {
1629 	dt_normal_t normal;
1630 	caddr_t addr;
1631 
1632 	/*
1633 	 * We (should) have two records:  the aggregation ID followed by the
1634 	 * normalization value.
1635 	 */
1636 	addr = base + rec->dtrd_offset;
1637 
1638 	if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1639 		return (dt_set_errno(dtp, EDT_BADNORMAL));
1640 
1641 	/* LINTED - alignment */
1642 	normal.dtnd_id = *((dtrace_aggvarid_t *)addr);
1643 	rec++;
1644 
1645 	if (rec->dtrd_action != DTRACEACT_LIBACT)
1646 		return (dt_set_errno(dtp, EDT_BADNORMAL));
1647 
1648 	if (rec->dtrd_arg != DT_ACT_NORMALIZE)
1649 		return (dt_set_errno(dtp, EDT_BADNORMAL));
1650 
1651 	addr = base + rec->dtrd_offset;
1652 
1653 	switch (rec->dtrd_size) {
1654 	case sizeof (uint64_t):
1655 		/* LINTED - alignment */
1656 		normal.dtnd_normal = *((uint64_t *)addr);
1657 		break;
1658 	case sizeof (uint32_t):
1659 		/* LINTED - alignment */
1660 		normal.dtnd_normal = *((uint32_t *)addr);
1661 		break;
1662 	case sizeof (uint16_t):
1663 		/* LINTED - alignment */
1664 		normal.dtnd_normal = *((uint16_t *)addr);
1665 		break;
1666 	case sizeof (uint8_t):
1667 		normal.dtnd_normal = *((uint8_t *)addr);
1668 		break;
1669 	default:
1670 		return (dt_set_errno(dtp, EDT_BADNORMAL));
1671 	}
1672 
1673 	(void) dtrace_aggregate_walk(dtp, dt_normalize_agg, &normal);
1674 
1675 	return (0);
1676 }
1677 
1678 static int
1679 dt_denormalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1680 {
1681 	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1682 	dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1683 
1684 	if (agg->dtagd_nrecs == 0)
1685 		return (DTRACE_AGGWALK_NEXT);
1686 
1687 	if (agg->dtagd_varid != id)
1688 		return (DTRACE_AGGWALK_NEXT);
1689 
1690 	return (DTRACE_AGGWALK_DENORMALIZE);
1691 }
1692 
1693 static int
1694 dt_clear_agg(const dtrace_aggdata_t *aggdata, void *arg)
1695 {
1696 	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1697 	dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1698 
1699 	if (agg->dtagd_nrecs == 0)
1700 		return (DTRACE_AGGWALK_NEXT);
1701 
1702 	if (agg->dtagd_varid != id)
1703 		return (DTRACE_AGGWALK_NEXT);
1704 
1705 	return (DTRACE_AGGWALK_CLEAR);
1706 }
1707 
1708 typedef struct dt_trunc {
1709 	dtrace_aggvarid_t dttd_id;
1710 	uint64_t dttd_remaining;
1711 } dt_trunc_t;
1712 
1713 static int
1714 dt_trunc_agg(const dtrace_aggdata_t *aggdata, void *arg)
1715 {
1716 	dt_trunc_t *trunc = arg;
1717 	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1718 	dtrace_aggvarid_t id = trunc->dttd_id;
1719 
1720 	if (agg->dtagd_nrecs == 0)
1721 		return (DTRACE_AGGWALK_NEXT);
1722 
1723 	if (agg->dtagd_varid != id)
1724 		return (DTRACE_AGGWALK_NEXT);
1725 
1726 	if (trunc->dttd_remaining == 0)
1727 		return (DTRACE_AGGWALK_REMOVE);
1728 
1729 	trunc->dttd_remaining--;
1730 	return (DTRACE_AGGWALK_NEXT);
1731 }
1732 
1733 static int
1734 dt_trunc(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1735 {
1736 	dt_trunc_t trunc;
1737 	caddr_t addr;
1738 	int64_t remaining;
1739 	int (*func)(dtrace_hdl_t *, dtrace_aggregate_f *, void *);
1740 
1741 	/*
1742 	 * We (should) have two records:  the aggregation ID followed by the
1743 	 * number of aggregation entries after which the aggregation is to be
1744 	 * truncated.
1745 	 */
1746 	addr = base + rec->dtrd_offset;
1747 
1748 	if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1749 		return (dt_set_errno(dtp, EDT_BADTRUNC));
1750 
1751 	/* LINTED - alignment */
1752 	trunc.dttd_id = *((dtrace_aggvarid_t *)addr);
1753 	rec++;
1754 
1755 	if (rec->dtrd_action != DTRACEACT_LIBACT)
1756 		return (dt_set_errno(dtp, EDT_BADTRUNC));
1757 
1758 	if (rec->dtrd_arg != DT_ACT_TRUNC)
1759 		return (dt_set_errno(dtp, EDT_BADTRUNC));
1760 
1761 	addr = base + rec->dtrd_offset;
1762 
1763 	switch (rec->dtrd_size) {
1764 	case sizeof (uint64_t):
1765 		/* LINTED - alignment */
1766 		remaining = *((int64_t *)addr);
1767 		break;
1768 	case sizeof (uint32_t):
1769 		/* LINTED - alignment */
1770 		remaining = *((int32_t *)addr);
1771 		break;
1772 	case sizeof (uint16_t):
1773 		/* LINTED - alignment */
1774 		remaining = *((int16_t *)addr);
1775 		break;
1776 	case sizeof (uint8_t):
1777 		remaining = *((int8_t *)addr);
1778 		break;
1779 	default:
1780 		return (dt_set_errno(dtp, EDT_BADNORMAL));
1781 	}
1782 
1783 	if (remaining < 0) {
1784 		func = dtrace_aggregate_walk_valsorted;
1785 		remaining = -remaining;
1786 	} else {
1787 		func = dtrace_aggregate_walk_valrevsorted;
1788 	}
1789 
1790 	assert(remaining >= 0);
1791 	trunc.dttd_remaining = remaining;
1792 
1793 	(void) func(dtp, dt_trunc_agg, &trunc);
1794 
1795 	return (0);
1796 }
1797 
1798 static int
1799 dt_print_datum(dtrace_hdl_t *dtp, FILE *fp, dtrace_recdesc_t *rec,
1800     caddr_t addr, size_t size, uint64_t normal)
1801 {
1802 	int err;
1803 	dtrace_actkind_t act = rec->dtrd_action;
1804 
1805 	switch (act) {
1806 	case DTRACEACT_STACK:
1807 		return (dt_print_stack(dtp, fp, NULL, addr,
1808 		    rec->dtrd_arg, rec->dtrd_size / rec->dtrd_arg));
1809 
1810 	case DTRACEACT_USTACK:
1811 	case DTRACEACT_JSTACK:
1812 		return (dt_print_ustack(dtp, fp, NULL, addr, rec->dtrd_arg));
1813 
1814 	case DTRACEACT_USYM:
1815 	case DTRACEACT_UADDR:
1816 		return (dt_print_usym(dtp, fp, addr, act));
1817 
1818 	case DTRACEACT_UMOD:
1819 		return (dt_print_umod(dtp, fp, NULL, addr));
1820 
1821 	case DTRACEACT_SYM:
1822 		return (dt_print_sym(dtp, fp, NULL, addr));
1823 
1824 	case DTRACEACT_MOD:
1825 		return (dt_print_mod(dtp, fp, NULL, addr));
1826 
1827 	case DTRACEAGG_QUANTIZE:
1828 		return (dt_print_quantize(dtp, fp, addr, size, normal));
1829 
1830 	case DTRACEAGG_LQUANTIZE:
1831 		return (dt_print_lquantize(dtp, fp, addr, size, normal));
1832 
1833 	case DTRACEAGG_LLQUANTIZE:
1834 		return (dt_print_llquantize(dtp, fp, addr, size, normal));
1835 
1836 	case DTRACEAGG_AVG:
1837 		return (dt_print_average(dtp, fp, addr, size, normal));
1838 
1839 	case DTRACEAGG_STDDEV:
1840 		return (dt_print_stddev(dtp, fp, addr, size, normal));
1841 
1842 	default:
1843 		break;
1844 	}
1845 
1846 	switch (size) {
1847 	case sizeof (uint64_t):
1848 		err = dt_printf(dtp, fp, " %16lld",
1849 		    /* LINTED - alignment */
1850 		    (long long)*((uint64_t *)addr) / normal);
1851 		break;
1852 	case sizeof (uint32_t):
1853 		/* LINTED - alignment */
1854 		err = dt_printf(dtp, fp, " %8d", *((uint32_t *)addr) /
1855 		    (uint32_t)normal);
1856 		break;
1857 	case sizeof (uint16_t):
1858 		/* LINTED - alignment */
1859 		err = dt_printf(dtp, fp, " %5d", *((uint16_t *)addr) /
1860 		    (uint32_t)normal);
1861 		break;
1862 	case sizeof (uint8_t):
1863 		err = dt_printf(dtp, fp, " %3d", *((uint8_t *)addr) /
1864 		    (uint32_t)normal);
1865 		break;
1866 	default:
1867 		err = dt_print_bytes(dtp, fp, addr, size, 50, 0, 0);
1868 		break;
1869 	}
1870 
1871 	return (err);
1872 }
1873 
1874 int
1875 dt_print_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
1876 {
1877 	int i, aggact = 0;
1878 	dt_print_aggdata_t *pd = arg;
1879 	const dtrace_aggdata_t *aggdata = aggsdata[0];
1880 	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1881 	FILE *fp = pd->dtpa_fp;
1882 	dtrace_hdl_t *dtp = pd->dtpa_dtp;
1883 	dtrace_recdesc_t *rec;
1884 	dtrace_actkind_t act;
1885 	caddr_t addr;
1886 	size_t size;
1887 
1888 	/*
1889 	 * Iterate over each record description in the key, printing the traced
1890 	 * data, skipping the first datum (the tuple member created by the
1891 	 * compiler).
1892 	 */
1893 	for (i = 1; i < agg->dtagd_nrecs; i++) {
1894 		rec = &agg->dtagd_rec[i];
1895 		act = rec->dtrd_action;
1896 		addr = aggdata->dtada_data + rec->dtrd_offset;
1897 		size = rec->dtrd_size;
1898 
1899 		if (DTRACEACT_ISAGG(act)) {
1900 			aggact = i;
1901 			break;
1902 		}
1903 
1904 		if (dt_print_datum(dtp, fp, rec, addr, size, 1) < 0)
1905 			return (-1);
1906 
1907 		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1908 		    DTRACE_BUFDATA_AGGKEY) < 0)
1909 			return (-1);
1910 	}
1911 
1912 	assert(aggact != 0);
1913 
1914 	for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
1915 		uint64_t normal;
1916 
1917 		aggdata = aggsdata[i];
1918 		agg = aggdata->dtada_desc;
1919 		rec = &agg->dtagd_rec[aggact];
1920 		act = rec->dtrd_action;
1921 		addr = aggdata->dtada_data + rec->dtrd_offset;
1922 		size = rec->dtrd_size;
1923 
1924 		assert(DTRACEACT_ISAGG(act));
1925 		normal = aggdata->dtada_normal;
1926 
1927 		if (dt_print_datum(dtp, fp, rec, addr, size, normal) < 0)
1928 			return (-1);
1929 
1930 		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1931 		    DTRACE_BUFDATA_AGGVAL) < 0)
1932 			return (-1);
1933 
1934 		if (!pd->dtpa_allunprint)
1935 			agg->dtagd_flags |= DTRACE_AGD_PRINTED;
1936 	}
1937 
1938 	if (dt_printf(dtp, fp, "\n") < 0)
1939 		return (-1);
1940 
1941 	if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
1942 	    DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0)
1943 		return (-1);
1944 
1945 	return (0);
1946 }
1947 
1948 int
1949 dt_print_agg(const dtrace_aggdata_t *aggdata, void *arg)
1950 {
1951 	dt_print_aggdata_t *pd = arg;
1952 	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1953 	dtrace_aggvarid_t aggvarid = pd->dtpa_id;
1954 
1955 	if (pd->dtpa_allunprint) {
1956 		if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
1957 			return (0);
1958 	} else {
1959 		/*
1960 		 * If we're not printing all unprinted aggregations, then the
1961 		 * aggregation variable ID denotes a specific aggregation
1962 		 * variable that we should print -- skip any other aggregations
1963 		 * that we encounter.
1964 		 */
1965 		if (agg->dtagd_nrecs == 0)
1966 			return (0);
1967 
1968 		if (aggvarid != agg->dtagd_varid)
1969 			return (0);
1970 	}
1971 
1972 	return (dt_print_aggs(&aggdata, 1, arg));
1973 }
1974 
1975 int
1976 dt_setopt(dtrace_hdl_t *dtp, const dtrace_probedata_t *data,
1977     const char *option, const char *value)
1978 {
1979 	int len, rval;
1980 	char *msg;
1981 	const char *errstr;
1982 	dtrace_setoptdata_t optdata;
1983 
1984 	bzero(&optdata, sizeof (optdata));
1985 	(void) dtrace_getopt(dtp, option, &optdata.dtsda_oldval);
1986 
1987 	if (dtrace_setopt(dtp, option, value) == 0) {
1988 		(void) dtrace_getopt(dtp, option, &optdata.dtsda_newval);
1989 		optdata.dtsda_probe = data;
1990 		optdata.dtsda_option = option;
1991 		optdata.dtsda_handle = dtp;
1992 
1993 		if ((rval = dt_handle_setopt(dtp, &optdata)) != 0)
1994 			return (rval);
1995 
1996 		return (0);
1997 	}
1998 
1999 	errstr = dtrace_errmsg(dtp, dtrace_errno(dtp));
2000 	len = strlen(option) + strlen(value) + strlen(errstr) + 80;
2001 	msg = alloca(len);
2002 
2003 	(void) snprintf(msg, len, "couldn't set option \"%s\" to \"%s\": %s\n",
2004 	    option, value, errstr);
2005 
2006 	if ((rval = dt_handle_liberr(dtp, data, msg)) == 0)
2007 		return (0);
2008 
2009 	return (rval);
2010 }
2011 
2012 static int
2013 dt_consume_cpu(dtrace_hdl_t *dtp, FILE *fp, int cpu, dtrace_bufdesc_t *buf,
2014     dtrace_consume_probe_f *efunc, dtrace_consume_rec_f *rfunc, void *arg)
2015 {
2016 	dtrace_epid_t id;
2017 	size_t offs, start = buf->dtbd_oldest, end = buf->dtbd_size;
2018 	int flow = (dtp->dt_options[DTRACEOPT_FLOWINDENT] != DTRACEOPT_UNSET);
2019 	int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
2020 	int rval, i, n;
2021 	dtrace_epid_t last = DTRACE_EPIDNONE;
2022 	dtrace_probedata_t data;
2023 	uint64_t drops;
2024 	caddr_t addr;
2025 
2026 	bzero(&data, sizeof (data));
2027 	data.dtpda_handle = dtp;
2028 	data.dtpda_cpu = cpu;
2029 
2030 again:
2031 	for (offs = start; offs < end; ) {
2032 		dtrace_eprobedesc_t *epd;
2033 
2034 		/*
2035 		 * We're guaranteed to have an ID.
2036 		 */
2037 		id = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
2038 
2039 		if (id == DTRACE_EPIDNONE) {
2040 			/*
2041 			 * This is filler to assure proper alignment of the
2042 			 * next record; we simply ignore it.
2043 			 */
2044 			offs += sizeof (id);
2045 			continue;
2046 		}
2047 
2048 		if ((rval = dt_epid_lookup(dtp, id, &data.dtpda_edesc,
2049 		    &data.dtpda_pdesc)) != 0)
2050 			return (rval);
2051 
2052 		epd = data.dtpda_edesc;
2053 		data.dtpda_data = buf->dtbd_data + offs;
2054 
2055 		if (data.dtpda_edesc->dtepd_uarg != DT_ECB_DEFAULT) {
2056 			rval = dt_handle(dtp, &data);
2057 
2058 			if (rval == DTRACE_CONSUME_NEXT)
2059 				goto nextepid;
2060 
2061 			if (rval == DTRACE_CONSUME_ERROR)
2062 				return (-1);
2063 		}
2064 
2065 		if (flow)
2066 			(void) dt_flowindent(dtp, &data, last, buf, offs);
2067 
2068 		rval = (*efunc)(&data, arg);
2069 
2070 		if (flow) {
2071 			if (data.dtpda_flow == DTRACEFLOW_ENTRY)
2072 				data.dtpda_indent += 2;
2073 		}
2074 
2075 		if (rval == DTRACE_CONSUME_NEXT)
2076 			goto nextepid;
2077 
2078 		if (rval == DTRACE_CONSUME_ABORT)
2079 			return (dt_set_errno(dtp, EDT_DIRABORT));
2080 
2081 		if (rval != DTRACE_CONSUME_THIS)
2082 			return (dt_set_errno(dtp, EDT_BADRVAL));
2083 
2084 		for (i = 0; i < epd->dtepd_nrecs; i++) {
2085 			dtrace_recdesc_t *rec = &epd->dtepd_rec[i];
2086 			dtrace_actkind_t act = rec->dtrd_action;
2087 
2088 			data.dtpda_data = buf->dtbd_data + offs +
2089 			    rec->dtrd_offset;
2090 			addr = data.dtpda_data;
2091 
2092 			if (act == DTRACEACT_LIBACT) {
2093 				uint64_t arg = rec->dtrd_arg;
2094 				dtrace_aggvarid_t id;
2095 
2096 				switch (arg) {
2097 				case DT_ACT_CLEAR:
2098 					/* LINTED - alignment */
2099 					id = *((dtrace_aggvarid_t *)addr);
2100 					(void) dtrace_aggregate_walk(dtp,
2101 					    dt_clear_agg, &id);
2102 					continue;
2103 
2104 				case DT_ACT_DENORMALIZE:
2105 					/* LINTED - alignment */
2106 					id = *((dtrace_aggvarid_t *)addr);
2107 					(void) dtrace_aggregate_walk(dtp,
2108 					    dt_denormalize_agg, &id);
2109 					continue;
2110 
2111 				case DT_ACT_FTRUNCATE:
2112 					if (fp == NULL)
2113 						continue;
2114 
2115 					(void) fflush(fp);
2116 					(void) ftruncate(fileno(fp), 0);
2117 					(void) fseeko(fp, 0, SEEK_SET);
2118 					continue;
2119 
2120 				case DT_ACT_NORMALIZE:
2121 					if (i == epd->dtepd_nrecs - 1)
2122 						return (dt_set_errno(dtp,
2123 						    EDT_BADNORMAL));
2124 
2125 					if (dt_normalize(dtp,
2126 					    buf->dtbd_data + offs, rec) != 0)
2127 						return (-1);
2128 
2129 					i++;
2130 					continue;
2131 
2132 				case DT_ACT_SETOPT: {
2133 					uint64_t *opts = dtp->dt_options;
2134 					dtrace_recdesc_t *valrec;
2135 					uint32_t valsize;
2136 					caddr_t val;
2137 					int rv;
2138 
2139 					if (i == epd->dtepd_nrecs - 1) {
2140 						return (dt_set_errno(dtp,
2141 						    EDT_BADSETOPT));
2142 					}
2143 
2144 					valrec = &epd->dtepd_rec[++i];
2145 					valsize = valrec->dtrd_size;
2146 
2147 					if (valrec->dtrd_action != act ||
2148 					    valrec->dtrd_arg != arg) {
2149 						return (dt_set_errno(dtp,
2150 						    EDT_BADSETOPT));
2151 					}
2152 
2153 					if (valsize > sizeof (uint64_t)) {
2154 						val = buf->dtbd_data + offs +
2155 						    valrec->dtrd_offset;
2156 					} else {
2157 						val = "1";
2158 					}
2159 
2160 					rv = dt_setopt(dtp, &data, addr, val);
2161 
2162 					if (rv != 0)
2163 						return (-1);
2164 
2165 					flow = (opts[DTRACEOPT_FLOWINDENT] !=
2166 					    DTRACEOPT_UNSET);
2167 					quiet = (opts[DTRACEOPT_QUIET] !=
2168 					    DTRACEOPT_UNSET);
2169 
2170 					continue;
2171 				}
2172 
2173 				case DT_ACT_TRUNC:
2174 					if (i == epd->dtepd_nrecs - 1)
2175 						return (dt_set_errno(dtp,
2176 						    EDT_BADTRUNC));
2177 
2178 					if (dt_trunc(dtp,
2179 					    buf->dtbd_data + offs, rec) != 0)
2180 						return (-1);
2181 
2182 					i++;
2183 					continue;
2184 
2185 				default:
2186 					continue;
2187 				}
2188 			}
2189 
2190 			rval = (*rfunc)(&data, rec, arg);
2191 
2192 			if (rval == DTRACE_CONSUME_NEXT)
2193 				continue;
2194 
2195 			if (rval == DTRACE_CONSUME_ABORT)
2196 				return (dt_set_errno(dtp, EDT_DIRABORT));
2197 
2198 			if (rval != DTRACE_CONSUME_THIS)
2199 				return (dt_set_errno(dtp, EDT_BADRVAL));
2200 
2201 			if (act == DTRACEACT_STACK) {
2202 				int depth = rec->dtrd_arg;
2203 
2204 				if (dt_print_stack(dtp, fp, NULL, addr, depth,
2205 				    rec->dtrd_size / depth) < 0)
2206 					return (-1);
2207 				goto nextrec;
2208 			}
2209 
2210 			if (act == DTRACEACT_USTACK ||
2211 			    act == DTRACEACT_JSTACK) {
2212 				if (dt_print_ustack(dtp, fp, NULL,
2213 				    addr, rec->dtrd_arg) < 0)
2214 					return (-1);
2215 				goto nextrec;
2216 			}
2217 
2218 			if (act == DTRACEACT_SYM) {
2219 				if (dt_print_sym(dtp, fp, NULL, addr) < 0)
2220 					return (-1);
2221 				goto nextrec;
2222 			}
2223 
2224 			if (act == DTRACEACT_MOD) {
2225 				if (dt_print_mod(dtp, fp, NULL, addr) < 0)
2226 					return (-1);
2227 				goto nextrec;
2228 			}
2229 
2230 			if (act == DTRACEACT_USYM || act == DTRACEACT_UADDR) {
2231 				if (dt_print_usym(dtp, fp, addr, act) < 0)
2232 					return (-1);
2233 				goto nextrec;
2234 			}
2235 
2236 			if (act == DTRACEACT_UMOD) {
2237 				if (dt_print_umod(dtp, fp, NULL, addr) < 0)
2238 					return (-1);
2239 				goto nextrec;
2240 			}
2241 
2242 			if (act == DTRACEACT_PRINTM) {
2243 				if (dt_print_memory(dtp, fp, addr) < 0)
2244 					return (-1);
2245 				goto nextrec;
2246 			}
2247 
2248 			if (act == DTRACEACT_PRINTT) {
2249 				if (dt_print_type(dtp, fp, addr) < 0)
2250 					return (-1);
2251 				goto nextrec;
2252 			}
2253 
2254 			if (DTRACEACT_ISPRINTFLIKE(act)) {
2255 				void *fmtdata;
2256 				int (*func)(dtrace_hdl_t *, FILE *, void *,
2257 				    const dtrace_probedata_t *,
2258 				    const dtrace_recdesc_t *, uint_t,
2259 				    const void *buf, size_t);
2260 
2261 				if ((fmtdata = dt_format_lookup(dtp,
2262 				    rec->dtrd_format)) == NULL)
2263 					goto nofmt;
2264 
2265 				switch (act) {
2266 				case DTRACEACT_PRINTF:
2267 					func = dtrace_fprintf;
2268 					break;
2269 				case DTRACEACT_PRINTA:
2270 					func = dtrace_fprinta;
2271 					break;
2272 				case DTRACEACT_SYSTEM:
2273 					func = dtrace_system;
2274 					break;
2275 				case DTRACEACT_FREOPEN:
2276 					func = dtrace_freopen;
2277 					break;
2278 				}
2279 
2280 				n = (*func)(dtp, fp, fmtdata, &data,
2281 				    rec, epd->dtepd_nrecs - i,
2282 				    (uchar_t *)buf->dtbd_data + offs,
2283 				    buf->dtbd_size - offs);
2284 
2285 				if (n < 0)
2286 					return (-1); /* errno is set for us */
2287 
2288 				if (n > 0)
2289 					i += n - 1;
2290 				goto nextrec;
2291 			}
2292 
2293 nofmt:
2294 			if (act == DTRACEACT_PRINTA) {
2295 				dt_print_aggdata_t pd;
2296 				dtrace_aggvarid_t *aggvars;
2297 				int j, naggvars = 0;
2298 				size_t size = ((epd->dtepd_nrecs - i) *
2299 				    sizeof (dtrace_aggvarid_t));
2300 
2301 				if ((aggvars = dt_alloc(dtp, size)) == NULL)
2302 					return (-1);
2303 
2304 				/*
2305 				 * This might be a printa() with multiple
2306 				 * aggregation variables.  We need to scan
2307 				 * forward through the records until we find
2308 				 * a record from a different statement.
2309 				 */
2310 				for (j = i; j < epd->dtepd_nrecs; j++) {
2311 					dtrace_recdesc_t *nrec;
2312 					caddr_t naddr;
2313 
2314 					nrec = &epd->dtepd_rec[j];
2315 
2316 					if (nrec->dtrd_uarg != rec->dtrd_uarg)
2317 						break;
2318 
2319 					if (nrec->dtrd_action != act) {
2320 						return (dt_set_errno(dtp,
2321 						    EDT_BADAGG));
2322 					}
2323 
2324 					naddr = buf->dtbd_data + offs +
2325 					    nrec->dtrd_offset;
2326 
2327 					aggvars[naggvars++] =
2328 					    /* LINTED - alignment */
2329 					    *((dtrace_aggvarid_t *)naddr);
2330 				}
2331 
2332 				i = j - 1;
2333 				bzero(&pd, sizeof (pd));
2334 				pd.dtpa_dtp = dtp;
2335 				pd.dtpa_fp = fp;
2336 
2337 				assert(naggvars >= 1);
2338 
2339 				if (naggvars == 1) {
2340 					pd.dtpa_id = aggvars[0];
2341 					dt_free(dtp, aggvars);
2342 
2343 					if (dt_printf(dtp, fp, "\n") < 0 ||
2344 					    dtrace_aggregate_walk_sorted(dtp,
2345 					    dt_print_agg, &pd) < 0)
2346 						return (-1);
2347 					goto nextrec;
2348 				}
2349 
2350 				if (dt_printf(dtp, fp, "\n") < 0 ||
2351 				    dtrace_aggregate_walk_joined(dtp, aggvars,
2352 				    naggvars, dt_print_aggs, &pd) < 0) {
2353 					dt_free(dtp, aggvars);
2354 					return (-1);
2355 				}
2356 
2357 				dt_free(dtp, aggvars);
2358 				goto nextrec;
2359 			}
2360 
2361 			switch (rec->dtrd_size) {
2362 			case sizeof (uint64_t):
2363 				n = dt_printf(dtp, fp,
2364 				    quiet ? "%lld" : " %16lld",
2365 				    /* LINTED - alignment */
2366 				    *((unsigned long long *)addr));
2367 				break;
2368 			case sizeof (uint32_t):
2369 				n = dt_printf(dtp, fp, quiet ? "%d" : " %8d",
2370 				    /* LINTED - alignment */
2371 				    *((uint32_t *)addr));
2372 				break;
2373 			case sizeof (uint16_t):
2374 				n = dt_printf(dtp, fp, quiet ? "%d" : " %5d",
2375 				    /* LINTED - alignment */
2376 				    *((uint16_t *)addr));
2377 				break;
2378 			case sizeof (uint8_t):
2379 				n = dt_printf(dtp, fp, quiet ? "%d" : " %3d",
2380 				    *((uint8_t *)addr));
2381 				break;
2382 			default:
2383 				n = dt_print_bytes(dtp, fp, addr,
2384 				    rec->dtrd_size, 33, quiet, 0);
2385 				break;
2386 			}
2387 
2388 			if (n < 0)
2389 				return (-1); /* errno is set for us */
2390 
2391 nextrec:
2392 			if (dt_buffered_flush(dtp, &data, rec, NULL, 0) < 0)
2393 				return (-1); /* errno is set for us */
2394 		}
2395 
2396 		/*
2397 		 * Call the record callback with a NULL record to indicate
2398 		 * that we're done processing this EPID.
2399 		 */
2400 		rval = (*rfunc)(&data, NULL, arg);
2401 nextepid:
2402 		offs += epd->dtepd_size;
2403 		last = id;
2404 	}
2405 
2406 	if (buf->dtbd_oldest != 0 && start == buf->dtbd_oldest) {
2407 		end = buf->dtbd_oldest;
2408 		start = 0;
2409 		goto again;
2410 	}
2411 
2412 	if ((drops = buf->dtbd_drops) == 0)
2413 		return (0);
2414 
2415 	/*
2416 	 * Explicitly zero the drops to prevent us from processing them again.
2417 	 */
2418 	buf->dtbd_drops = 0;
2419 
2420 	return (dt_handle_cpudrop(dtp, cpu, DTRACEDROP_PRINCIPAL, drops));
2421 }
2422 
2423 typedef struct dt_begin {
2424 	dtrace_consume_probe_f *dtbgn_probefunc;
2425 	dtrace_consume_rec_f *dtbgn_recfunc;
2426 	void *dtbgn_arg;
2427 	dtrace_handle_err_f *dtbgn_errhdlr;
2428 	void *dtbgn_errarg;
2429 	int dtbgn_beginonly;
2430 } dt_begin_t;
2431 
2432 static int
2433 dt_consume_begin_probe(const dtrace_probedata_t *data, void *arg)
2434 {
2435 	dt_begin_t *begin = (dt_begin_t *)arg;
2436 	dtrace_probedesc_t *pd = data->dtpda_pdesc;
2437 
2438 	int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2439 	int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2440 
2441 	if (begin->dtbgn_beginonly) {
2442 		if (!(r1 && r2))
2443 			return (DTRACE_CONSUME_NEXT);
2444 	} else {
2445 		if (r1 && r2)
2446 			return (DTRACE_CONSUME_NEXT);
2447 	}
2448 
2449 	/*
2450 	 * We have a record that we're interested in.  Now call the underlying
2451 	 * probe function...
2452 	 */
2453 	return (begin->dtbgn_probefunc(data, begin->dtbgn_arg));
2454 }
2455 
2456 static int
2457 dt_consume_begin_record(const dtrace_probedata_t *data,
2458     const dtrace_recdesc_t *rec, void *arg)
2459 {
2460 	dt_begin_t *begin = (dt_begin_t *)arg;
2461 
2462 	return (begin->dtbgn_recfunc(data, rec, begin->dtbgn_arg));
2463 }
2464 
2465 static int
2466 dt_consume_begin_error(const dtrace_errdata_t *data, void *arg)
2467 {
2468 	dt_begin_t *begin = (dt_begin_t *)arg;
2469 	dtrace_probedesc_t *pd = data->dteda_pdesc;
2470 
2471 	int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2472 	int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2473 
2474 	if (begin->dtbgn_beginonly) {
2475 		if (!(r1 && r2))
2476 			return (DTRACE_HANDLE_OK);
2477 	} else {
2478 		if (r1 && r2)
2479 			return (DTRACE_HANDLE_OK);
2480 	}
2481 
2482 	return (begin->dtbgn_errhdlr(data, begin->dtbgn_errarg));
2483 }
2484 
2485 static int
2486 dt_consume_begin(dtrace_hdl_t *dtp, FILE *fp, dtrace_bufdesc_t *buf,
2487     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2488 {
2489 	/*
2490 	 * There's this idea that the BEGIN probe should be processed before
2491 	 * everything else, and that the END probe should be processed after
2492 	 * anything else.  In the common case, this is pretty easy to deal
2493 	 * with.  However, a situation may arise where the BEGIN enabling and
2494 	 * END enabling are on the same CPU, and some enabling in the middle
2495 	 * occurred on a different CPU.  To deal with this (blech!) we need to
2496 	 * consume the BEGIN buffer up until the end of the BEGIN probe, and
2497 	 * then set it aside.  We will then process every other CPU, and then
2498 	 * we'll return to the BEGIN CPU and process the rest of the data
2499 	 * (which will inevitably include the END probe, if any).  Making this
2500 	 * even more complicated (!) is the library's ERROR enabling.  Because
2501 	 * this enabling is processed before we even get into the consume call
2502 	 * back, any ERROR firing would result in the library's ERROR enabling
2503 	 * being processed twice -- once in our first pass (for BEGIN probes),
2504 	 * and again in our second pass (for everything but BEGIN probes).  To
2505 	 * deal with this, we interpose on the ERROR handler to assure that we
2506 	 * only process ERROR enablings induced by BEGIN enablings in the
2507 	 * first pass, and that we only process ERROR enablings _not_ induced
2508 	 * by BEGIN enablings in the second pass.
2509 	 */
2510 	dt_begin_t begin;
2511 	processorid_t cpu = dtp->dt_beganon;
2512 	dtrace_bufdesc_t nbuf;
2513 #if !defined(sun)
2514 	dtrace_bufdesc_t *pbuf;
2515 #endif
2516 	int rval, i;
2517 	static int max_ncpus;
2518 	dtrace_optval_t size;
2519 
2520 	dtp->dt_beganon = -1;
2521 
2522 #if defined(sun)
2523 	if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2524 #else
2525 	if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2526 #endif
2527 		/*
2528 		 * We really don't expect this to fail, but it is at least
2529 		 * technically possible for this to fail with ENOENT.  In this
2530 		 * case, we just drive on...
2531 		 */
2532 		if (errno == ENOENT)
2533 			return (0);
2534 
2535 		return (dt_set_errno(dtp, errno));
2536 	}
2537 
2538 	if (!dtp->dt_stopped || buf->dtbd_cpu != dtp->dt_endedon) {
2539 		/*
2540 		 * This is the simple case.  We're either not stopped, or if
2541 		 * we are, we actually processed any END probes on another
2542 		 * CPU.  We can simply consume this buffer and return.
2543 		 */
2544 		return (dt_consume_cpu(dtp, fp, cpu, buf, pf, rf, arg));
2545 	}
2546 
2547 	begin.dtbgn_probefunc = pf;
2548 	begin.dtbgn_recfunc = rf;
2549 	begin.dtbgn_arg = arg;
2550 	begin.dtbgn_beginonly = 1;
2551 
2552 	/*
2553 	 * We need to interpose on the ERROR handler to be sure that we
2554 	 * only process ERRORs induced by BEGIN.
2555 	 */
2556 	begin.dtbgn_errhdlr = dtp->dt_errhdlr;
2557 	begin.dtbgn_errarg = dtp->dt_errarg;
2558 	dtp->dt_errhdlr = dt_consume_begin_error;
2559 	dtp->dt_errarg = &begin;
2560 
2561 	rval = dt_consume_cpu(dtp, fp, cpu, buf, dt_consume_begin_probe,
2562 	    dt_consume_begin_record, &begin);
2563 
2564 	dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2565 	dtp->dt_errarg = begin.dtbgn_errarg;
2566 
2567 	if (rval != 0)
2568 		return (rval);
2569 
2570 	/*
2571 	 * Now allocate a new buffer.  We'll use this to deal with every other
2572 	 * CPU.
2573 	 */
2574 	bzero(&nbuf, sizeof (dtrace_bufdesc_t));
2575 	(void) dtrace_getopt(dtp, "bufsize", &size);
2576 	if ((nbuf.dtbd_data = malloc(size)) == NULL)
2577 		return (dt_set_errno(dtp, EDT_NOMEM));
2578 
2579 	if (max_ncpus == 0)
2580 		max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2581 
2582 	for (i = 0; i < max_ncpus; i++) {
2583 		nbuf.dtbd_cpu = i;
2584 
2585 		if (i == cpu)
2586 			continue;
2587 
2588 #if defined(sun)
2589 		if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &nbuf) == -1) {
2590 #else
2591 		pbuf = &nbuf;
2592 		if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &pbuf) == -1) {
2593 #endif
2594 			/*
2595 			 * If we failed with ENOENT, it may be because the
2596 			 * CPU was unconfigured -- this is okay.  Any other
2597 			 * error, however, is unexpected.
2598 			 */
2599 			if (errno == ENOENT)
2600 				continue;
2601 
2602 			free(nbuf.dtbd_data);
2603 
2604 			return (dt_set_errno(dtp, errno));
2605 		}
2606 
2607 		if ((rval = dt_consume_cpu(dtp, fp,
2608 		    i, &nbuf, pf, rf, arg)) != 0) {
2609 			free(nbuf.dtbd_data);
2610 			return (rval);
2611 		}
2612 	}
2613 
2614 	free(nbuf.dtbd_data);
2615 
2616 	/*
2617 	 * Okay -- we're done with the other buffers.  Now we want to
2618 	 * reconsume the first buffer -- but this time we're looking for
2619 	 * everything _but_ BEGIN.  And of course, in order to only consume
2620 	 * those ERRORs _not_ associated with BEGIN, we need to reinstall our
2621 	 * ERROR interposition function...
2622 	 */
2623 	begin.dtbgn_beginonly = 0;
2624 
2625 	assert(begin.dtbgn_errhdlr == dtp->dt_errhdlr);
2626 	assert(begin.dtbgn_errarg == dtp->dt_errarg);
2627 	dtp->dt_errhdlr = dt_consume_begin_error;
2628 	dtp->dt_errarg = &begin;
2629 
2630 	rval = dt_consume_cpu(dtp, fp, cpu, buf, dt_consume_begin_probe,
2631 	    dt_consume_begin_record, &begin);
2632 
2633 	dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2634 	dtp->dt_errarg = begin.dtbgn_errarg;
2635 
2636 	return (rval);
2637 }
2638 
2639 int
2640 dtrace_consume(dtrace_hdl_t *dtp, FILE *fp,
2641     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2642 {
2643 	dtrace_bufdesc_t *buf = &dtp->dt_buf;
2644 	dtrace_optval_t size;
2645 	static int max_ncpus;
2646 	int i, rval;
2647 	dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_SWITCHRATE];
2648 	hrtime_t now = gethrtime();
2649 
2650 	if (dtp->dt_lastswitch != 0) {
2651 		if (now - dtp->dt_lastswitch < interval)
2652 			return (0);
2653 
2654 		dtp->dt_lastswitch += interval;
2655 	} else {
2656 		dtp->dt_lastswitch = now;
2657 	}
2658 
2659 	if (!dtp->dt_active)
2660 		return (dt_set_errno(dtp, EINVAL));
2661 
2662 	if (max_ncpus == 0)
2663 		max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2664 
2665 	if (pf == NULL)
2666 		pf = (dtrace_consume_probe_f *)dt_nullprobe;
2667 
2668 	if (rf == NULL)
2669 		rf = (dtrace_consume_rec_f *)dt_nullrec;
2670 
2671 	if (buf->dtbd_data == NULL) {
2672 		(void) dtrace_getopt(dtp, "bufsize", &size);
2673 		if ((buf->dtbd_data = malloc(size)) == NULL)
2674 			return (dt_set_errno(dtp, EDT_NOMEM));
2675 
2676 		buf->dtbd_size = size;
2677 	}
2678 
2679 	/*
2680 	 * If we have just begun, we want to first process the CPU that
2681 	 * executed the BEGIN probe (if any).
2682 	 */
2683 	if (dtp->dt_active && dtp->dt_beganon != -1) {
2684 		buf->dtbd_cpu = dtp->dt_beganon;
2685 		if ((rval = dt_consume_begin(dtp, fp, buf, pf, rf, arg)) != 0)
2686 			return (rval);
2687 	}
2688 
2689 	for (i = 0; i < max_ncpus; i++) {
2690 		buf->dtbd_cpu = i;
2691 
2692 		/*
2693 		 * If we have stopped, we want to process the CPU on which the
2694 		 * END probe was processed only _after_ we have processed
2695 		 * everything else.
2696 		 */
2697 		if (dtp->dt_stopped && (i == dtp->dt_endedon))
2698 			continue;
2699 
2700 #if defined(sun)
2701 		if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2702 #else
2703 		if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2704 #endif
2705 			/*
2706 			 * If we failed with ENOENT, it may be because the
2707 			 * CPU was unconfigured -- this is okay.  Any other
2708 			 * error, however, is unexpected.
2709 			 */
2710 			if (errno == ENOENT)
2711 				continue;
2712 
2713 			return (dt_set_errno(dtp, errno));
2714 		}
2715 
2716 		if ((rval = dt_consume_cpu(dtp, fp, i, buf, pf, rf, arg)) != 0)
2717 			return (rval);
2718 	}
2719 
2720 	if (!dtp->dt_stopped)
2721 		return (0);
2722 
2723 	buf->dtbd_cpu = dtp->dt_endedon;
2724 
2725 #if defined(sun)
2726 	if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2727 #else
2728 	if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2729 #endif
2730 		/*
2731 		 * This _really_ shouldn't fail, but it is strictly speaking
2732 		 * possible for this to return ENOENT if the CPU that called
2733 		 * the END enabling somehow managed to become unconfigured.
2734 		 * It's unclear how the user can possibly expect anything
2735 		 * rational to happen in this case -- the state has been thrown
2736 		 * out along with the unconfigured CPU -- so we'll just drive
2737 		 * on...
2738 		 */
2739 		if (errno == ENOENT)
2740 			return (0);
2741 
2742 		return (dt_set_errno(dtp, errno));
2743 	}
2744 
2745 	return (dt_consume_cpu(dtp, fp, dtp->dt_endedon, buf, pf, rf, arg));
2746 }
2747