xref: /freebsd/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c (revision 3877025f52ee205fe99ad4ff68229933d57e4bcb)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2023, Domagoj Stolfa. All rights reserved.
 * Copyright (c) 2017, Joyent, Inc. All rights reserved.
 * Copyright (c) 2012 by Delphix. All rights reserved.
 */

#include <stdlib.h>
#include <strings.h>
#include <errno.h>
#include <unistd.h>
#include <limits.h>
#include <assert.h>
#include <ctype.h>
#ifdef illumos
#include <alloca.h>
#endif
#include <dt_impl.h>
#include <dt_pq.h>
#include <dt_oformat.h>
#ifndef illumos
#include <libproc_compat.h>
#endif

#define	DT_MASK_LO 0x00000000FFFFFFFFULL

#define	dt_format_sym(dtp, addr) dt_print_sym((dtp), NULL, NULL, addr)

typedef struct dt_prepare_args {
	int first_bin;
	int last_bin;
	union {
		struct lquantize_args {
#define lquantize_step		u.lquantize.step
#define lquantize_levels	u.lquantize.levels
#define lquantize_base		u.lquantize.base
			int base;
			uint16_t step;
			uint16_t levels;
		} lquantize;
		struct llquantize_args {
#define	llquantize_next		u.llquantize.next
#define	llquantize_step		u.llquantize.step
#define	llquantize_value	u.llquantize.value
#define	llquantize_levels	u.llquantize.levels
#define	llquantize_order	u.llquantize.order
#define	llquantize_factor	u.llquantize.factor
#define	llquantize_low		u.llquantize.low
#define	llquantize_high		u.llquantize.high
#define	llquantize_nsteps	u.llquantize.nsteps
			int64_t next;
			int64_t step;
			int64_t value;
			int levels;
			int order;
			uint16_t factor;
			uint16_t low;
			uint16_t high;
			uint16_t nsteps;
		} llquantize;
	} u;
} dt_prepare_args_t;

/*
 * We declare this here because (1) we need it and (2) we want to avoid a
 * dependency on libm in libdtrace.
 */
static long double
dt_fabsl(long double x)
{
	if (x < 0)
		return (-x);

	return (x);
}

static int
dt_ndigits(long long val)
{
	int rval = 1;
	long long cmp = 10;

	if (val < 0) {
		val = val == INT64_MIN ? INT64_MAX : -val;
		rval++;
	}

	while (val > cmp && cmp > 0) {
		rval++;
		cmp *= 10;
	}

	return (rval < 4 ? 4 : rval);
}

/*
 * 128-bit arithmetic functions needed to support the stddev() aggregating
 * action.
 */
static int
dt_gt_128(uint64_t *a, uint64_t *b)
{
	return (a[1] > b[1] || (a[1] == b[1] && a[0] > b[0]));
}

static int
dt_ge_128(uint64_t *a, uint64_t *b)
{
	return (a[1] > b[1] || (a[1] == b[1] && a[0] >= b[0]));
}

static int
dt_le_128(uint64_t *a, uint64_t *b)
{
	return (a[1] < b[1] || (a[1] == b[1] && a[0] <= b[0]));
}

/*
 * Shift the 128-bit value in a by b. If b is positive, shift left.
 * If b is negative, shift right.
 */
static void
dt_shift_128(uint64_t *a, int b)
{
	uint64_t mask;

	if (b == 0)
		return;

	if (b < 0) {
		b = -b;
		if (b >= 64) {
			a[0] = a[1] >> (b - 64);
			a[1] = 0;
		} else {
			a[0] >>= b;
			mask = 1LL << (64 - b);
			mask -= 1;
			a[0] |= ((a[1] & mask) << (64 - b));
			a[1] >>= b;
		}
	} else {
		if (b >= 64) {
			a[1] = a[0] << (b - 64);
			a[0] = 0;
		} else {
			a[1] <<= b;
			mask = a[0] >> (64 - b);
			a[1] |= mask;
			a[0] <<= b;
		}
	}
}

static int
dt_nbits_128(uint64_t *a)
{
	int nbits = 0;
	uint64_t tmp[2];
	uint64_t zero[2] = { 0, 0 };

	tmp[0] = a[0];
	tmp[1] = a[1];

	dt_shift_128(tmp, -1);
	while (dt_gt_128(tmp, zero)) {
		dt_shift_128(tmp, -1);
		nbits++;
	}

	return (nbits);
}

static void
dt_subtract_128(uint64_t *minuend, uint64_t *subtrahend, uint64_t *difference)
{
	uint64_t result[2];

	result[0] = minuend[0] - subtrahend[0];
	result[1] = minuend[1] - subtrahend[1] -
	    (minuend[0] < subtrahend[0] ? 1 : 0);

	difference[0] = result[0];
	difference[1] = result[1];
}

static void
dt_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
{
	uint64_t result[2];

	result[0] = addend1[0] + addend2[0];
	result[1] = addend1[1] + addend2[1] +
	    (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);

	sum[0] = result[0];
	sum[1] = result[1];
}

/*
 * The basic idea is to break the 2 64-bit values into 4 32-bit values,
 * use native multiplication on those, and then re-combine into the
 * resulting 128-bit value.
 *
 * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
 *     hi1 * hi2 << 64 +
 *     hi1 * lo2 << 32 +
 *     hi2 * lo1 << 32 +
 *     lo1 * lo2
 */
static void
dt_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
{
	uint64_t hi1, hi2, lo1, lo2;
	uint64_t tmp[2];

	hi1 = factor1 >> 32;
	hi2 = factor2 >> 32;

	lo1 = factor1 & DT_MASK_LO;
	lo2 = factor2 & DT_MASK_LO;

	product[0] = lo1 * lo2;
	product[1] = hi1 * hi2;

	tmp[0] = hi1 * lo2;
	tmp[1] = 0;
	dt_shift_128(tmp, 32);
	dt_add_128(product, tmp, product);

	tmp[0] = hi2 * lo1;
	tmp[1] = 0;
	dt_shift_128(tmp, 32);
	dt_add_128(product, tmp, product);
}

/*
 * This is long-hand division.
 *
 * We initialize subtrahend by shifting divisor left as far as possible. We
 * loop, comparing subtrahend to dividend:  if subtrahend is smaller, we
 * subtract and set the appropriate bit in the result.  We then shift
 * subtrahend right by one bit for the next comparison.
 */
static void
dt_divide_128(uint64_t *dividend, uint64_t divisor, uint64_t *quotient)
{
	uint64_t result[2] = { 0, 0 };
	uint64_t remainder[2];
	uint64_t subtrahend[2];
	uint64_t divisor_128[2];
	uint64_t mask[2] = { 1, 0 };
	int log = 0;

	assert(divisor != 0);

	divisor_128[0] = divisor;
	divisor_128[1] = 0;

	remainder[0] = dividend[0];
	remainder[1] = dividend[1];

	subtrahend[0] = divisor;
	subtrahend[1] = 0;

	while (divisor > 0) {
		log++;
		divisor >>= 1;
	}

	dt_shift_128(subtrahend, 128 - log);
	dt_shift_128(mask, 128 - log);

	while (dt_ge_128(remainder, divisor_128)) {
		if (dt_ge_128(remainder, subtrahend)) {
			dt_subtract_128(remainder, subtrahend, remainder);
			result[0] |= mask[0];
			result[1] |= mask[1];
		}

		dt_shift_128(subtrahend, -1);
		dt_shift_128(mask, -1);
	}

	quotient[0] = result[0];
	quotient[1] = result[1];
}

/*
 * This is the long-hand method of calculating a square root.
 * The algorithm is as follows:
 *
 * 1. Group the digits by 2 from the right.
 * 2. Over the leftmost group, find the largest single-digit number
 *    whose square is less than that group.
 * 3. Subtract the result of the previous step (2 or 4, depending) and
 *    bring down the next two-digit group.
 * 4. For the result R we have so far, find the largest single-digit number
 *    x such that 2 * R * 10 * x + x^2 is less than the result from step 3.
 *    (Note that this is doubling R and performing a decimal left-shift by 1
 *    and searching for the appropriate decimal to fill the one's place.)
 *    The value x is the next digit in the square root.
 * Repeat steps 3 and 4 until the desired precision is reached.  (We're
 * dealing with integers, so the above is sufficient.)
 *
 * In decimal, the square root of 582,734 would be calculated as so:
 *
 *     __7__6__3
 *    | 58 27 34
 *     -49       (7^2 == 49 => 7 is the first digit in the square root)
 *      --
 *       9 27    (Subtract and bring down the next group.)
 * 146   8 76    (2 * 7 * 10 * 6 + 6^2 == 876 => 6 is the next digit in
 *      -----     the square root)
 *         51 34 (Subtract and bring down the next group.)
 * 1523    45 69 (2 * 76 * 10 * 3 + 3^2 == 4569 => 3 is the next digit in
 *         -----  the square root)
 *          5 65 (remainder)
 *
 * The above algorithm applies similarly in binary, but note that the
 * only possible non-zero value for x in step 4 is 1, so step 4 becomes a
 * simple decision: is 2 * R * 2 * 1 + 1^2 (aka R << 2 + 1) less than the
 * preceding difference?
 *
 * In binary, the square root of 11011011 would be calculated as so:
 *
 *     __1__1__1__0
 *    | 11 01 10 11
 *      01          (0 << 2 + 1 == 1 < 11 => this bit is 1)
 *      --
 *      10 01 10 11
 * 101   1 01       (1 << 2 + 1 == 101 < 1001 => next bit is 1)
 *      -----
 *       1 00 10 11
 * 1101    11 01    (11 << 2 + 1 == 1101 < 10010 => next bit is 1)
 *       -------
 *          1 01 11
 * 11101    1 11 01 (111 << 2 + 1 == 11101 > 10111 => last bit is 0)
 *
 */
static uint64_t
dt_sqrt_128(uint64_t *square)
{
	uint64_t result[2] = { 0, 0 };
	uint64_t diff[2] = { 0, 0 };
	uint64_t one[2] = { 1, 0 };
	uint64_t next_pair[2];
	uint64_t next_try[2];
	uint64_t bit_pairs, pair_shift;
	int i;

	bit_pairs = dt_nbits_128(square) / 2;
	pair_shift = bit_pairs * 2;

	for (i = 0; i <= bit_pairs; i++) {
		/*
		 * Bring down the next pair of bits.
		 */
		next_pair[0] = square[0];
		next_pair[1] = square[1];
		dt_shift_128(next_pair, -pair_shift);
		next_pair[0] &= 0x3;
		next_pair[1] = 0;

		dt_shift_128(diff, 2);
		dt_add_128(diff, next_pair, diff);

		/*
		 * next_try = R << 2 + 1
		 */
		next_try[0] = result[0];
		next_try[1] = result[1];
		dt_shift_128(next_try, 2);
		dt_add_128(next_try, one, next_try);

		if (dt_le_128(next_try, diff)) {
			dt_subtract_128(diff, next_try, diff);
			dt_shift_128(result, 1);
			dt_add_128(result, one, result);
		} else {
			dt_shift_128(result, 1);
		}

		pair_shift -= 2;
	}

	assert(result[1] == 0);

	return (result[0]);
}

uint64_t
dt_stddev(uint64_t *data, uint64_t normal)
{
	uint64_t avg_of_squares[2];
	uint64_t square_of_avg[2];
	int64_t norm_avg;
	uint64_t diff[2];

	if (data[0] == 0)
		return (0);

	/*
	 * The standard approximation for standard deviation is
	 * sqrt(average(x**2) - average(x)**2), i.e. the square root
	 * of the average of the squares minus the square of the average.
	 * When normalizing, we should divide the sum of x**2 by normal**2.
	 */
	dt_divide_128(data + 2, normal, avg_of_squares);
	dt_divide_128(avg_of_squares, normal, avg_of_squares);
	dt_divide_128(avg_of_squares, data[0], avg_of_squares);

	norm_avg = (int64_t)data[1] / (int64_t)normal / (int64_t)data[0];

	if (norm_avg < 0)
		norm_avg = -norm_avg;

	dt_multiply_128((uint64_t)norm_avg, (uint64_t)norm_avg, square_of_avg);

	dt_subtract_128(avg_of_squares, square_of_avg, diff);

	return (dt_sqrt_128(diff));
}

static int
dt_flowindent(dtrace_hdl_t *dtp, dtrace_probedata_t *data, dtrace_epid_t last,
    dtrace_bufdesc_t *buf, size_t offs)
{
	dtrace_probedesc_t *pd = data->dtpda_pdesc, *npd;
	dtrace_eprobedesc_t *epd = data->dtpda_edesc, *nepd;
	char *p = pd->dtpd_provider, *n = pd->dtpd_name, *sub;
	dtrace_flowkind_t flow = DTRACEFLOW_NONE;
	const char *str = NULL;
	static const char *e_str[2] = { " -> ", " => " };
	static const char *r_str[2] = { " <- ", " <= " };
	static const char *ent = "entry", *ret = "return";
	static int entlen = 0, retlen = 0;
	dtrace_epid_t next, id = epd->dtepd_epid;
	int rval;

	if (entlen == 0) {
		assert(retlen == 0);
		entlen = strlen(ent);
		retlen = strlen(ret);
	}

	/*
	 * If the name of the probe is "entry" or ends with "-entry", we
	 * treat it as an entry; if it is "return" or ends with "-return",
	 * we treat it as a return.  (This allows application-provided probes
	 * like "method-entry" or "function-entry" to participate in flow
	 * indentation -- without accidentally misinterpreting popular probe
	 * names like "carpentry", "gentry" or "Coventry".)
	 */
	if ((sub = strstr(n, ent)) != NULL && sub[entlen] == '\0' &&
	    (sub == n || sub[-1] == '-')) {
		flow = DTRACEFLOW_ENTRY;
		str = e_str[strcmp(p, "syscall") == 0];
	} else if ((sub = strstr(n, ret)) != NULL && sub[retlen] == '\0' &&
	    (sub == n || sub[-1] == '-')) {
		flow = DTRACEFLOW_RETURN;
		str = r_str[strcmp(p, "syscall") == 0];
	}

	/*
	 * If we're going to indent this, we need to check the ID of our last
	 * call.  If we're looking at the same probe ID but a different EPID,
	 * we _don't_ want to indent.  (Yes, there are some minor holes in
	 * this scheme -- it's a heuristic.)
	 */
	if (flow == DTRACEFLOW_ENTRY) {
		if ((last != DTRACE_EPIDNONE && id != last &&
		    pd->dtpd_id == dtp->dt_pdesc[last]->dtpd_id))
			flow = DTRACEFLOW_NONE;
	}

	/*
	 * If we're going to unindent this, it's more difficult to see if
	 * we don't actually want to unindent it -- we need to look at the
	 * _next_ EPID.
	 */
	if (flow == DTRACEFLOW_RETURN) {
		offs += epd->dtepd_size;

		do {
			if (offs >= buf->dtbd_size)
				goto out;

			next = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);

			if (next == DTRACE_EPIDNONE)
				offs += sizeof (id);
		} while (next == DTRACE_EPIDNONE);

		if ((rval = dt_epid_lookup(dtp, next, &nepd, &npd)) != 0)
			return (rval);

		if (next != id && npd->dtpd_id == pd->dtpd_id)
			flow = DTRACEFLOW_NONE;
	}

out:
	if (flow == DTRACEFLOW_ENTRY || flow == DTRACEFLOW_RETURN) {
		data->dtpda_prefix = str;
	} else {
		data->dtpda_prefix = "| ";
	}

	if (flow == DTRACEFLOW_RETURN && data->dtpda_indent > 0)
		data->dtpda_indent -= 2;

	data->dtpda_flow = flow;

	return (0);
}

static int
dt_nullprobe()
{
	return (DTRACE_CONSUME_THIS);
}

static int
dt_nullrec()
{
	return (DTRACE_CONSUME_NEXT);
}

static void
dt_quantize_total(dtrace_hdl_t *dtp, int64_t datum, long double *total)
{
	long double val = dt_fabsl((long double)datum);

	if (dtp->dt_options[DTRACEOPT_AGGZOOM] == DTRACEOPT_UNSET) {
		*total += val;
		return;
	}

	/*
	 * If we're zooming in on an aggregation, we want the height of the
	 * highest value to be approximately 95% of total bar height -- so we
	 * adjust up by the reciprocal of DTRACE_AGGZOOM_MAX when comparing to
	 * our highest value.
	 */
	val *= 1 / DTRACE_AGGZOOM_MAX;

	if (*total < val)
		*total = val;
}

static int
dt_print_quanthdr(dtrace_hdl_t *dtp, FILE *fp, int width)
{
	return (dt_printf(dtp, fp, "\n%*s %41s %-9s\n",
	    width ? width : 16, width ? "key" : "value",
	    "------------- Distribution -------------", "count"));
}

static int
dt_print_quanthdr_packed(dtrace_hdl_t *dtp, FILE *fp, int width,
    const dtrace_aggdata_t *aggdata, dtrace_actkind_t action)
{
	int min = aggdata->dtada_minbin, max = aggdata->dtada_maxbin;
	int minwidth, maxwidth, i;

	assert(action == DTRACEAGG_QUANTIZE || action == DTRACEAGG_LQUANTIZE);

	if (action == DTRACEAGG_QUANTIZE) {
		if (min != 0 && min != DTRACE_QUANTIZE_ZEROBUCKET)
			min--;

		if (max < DTRACE_QUANTIZE_NBUCKETS - 1)
			max++;

		minwidth = dt_ndigits(DTRACE_QUANTIZE_BUCKETVAL(min));
		maxwidth = dt_ndigits(DTRACE_QUANTIZE_BUCKETVAL(max));
	} else {
		maxwidth = 8;
		minwidth = maxwidth - 1;
		max++;
	}

	if (dt_printf(dtp, fp, "\n%*s %*s .",
	    width, width > 0 ? "key" : "", minwidth, "min") < 0)
		return (-1);

	for (i = min; i <= max; i++) {
		if (dt_printf(dtp, fp, "-") < 0)
			return (-1);
	}

	return (dt_printf(dtp, fp, ". %*s | count\n", -maxwidth, "max"));
}

/*
 * We use a subset of the Unicode Block Elements (U+2588 through U+258F,
 * inclusive) to represent aggregations via UTF-8 -- which are expressed via
 * 3-byte UTF-8 sequences.
 */
#define	DTRACE_AGGUTF8_FULL	0x2588
#define	DTRACE_AGGUTF8_BASE	0x258f
#define	DTRACE_AGGUTF8_LEVELS	8

#define	DTRACE_AGGUTF8_BYTE0(val)	(0xe0 | ((val) >> 12))
#define	DTRACE_AGGUTF8_BYTE1(val)	(0x80 | (((val) >> 6) & 0x3f))
#define	DTRACE_AGGUTF8_BYTE2(val)	(0x80 | ((val) & 0x3f))

static int
dt_print_quantline_utf8(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
    uint64_t normal, long double total)
{
	uint_t len = 40, i, whole, partial;
	long double f = (dt_fabsl((long double)val) * len) / total;
	const char *spaces = "                                        ";

	whole = (uint_t)f;
	partial = (uint_t)((f - (long double)(uint_t)f) *
	    (long double)DTRACE_AGGUTF8_LEVELS);

	if (dt_printf(dtp, fp, "|") < 0)
		return (-1);

	for (i = 0; i < whole; i++) {
		if (dt_printf(dtp, fp, "%c%c%c",
		    DTRACE_AGGUTF8_BYTE0(DTRACE_AGGUTF8_FULL),
		    DTRACE_AGGUTF8_BYTE1(DTRACE_AGGUTF8_FULL),
		    DTRACE_AGGUTF8_BYTE2(DTRACE_AGGUTF8_FULL)) < 0)
			return (-1);
	}

	if (partial != 0) {
		partial = DTRACE_AGGUTF8_BASE - (partial - 1);

		if (dt_printf(dtp, fp, "%c%c%c",
		    DTRACE_AGGUTF8_BYTE0(partial),
		    DTRACE_AGGUTF8_BYTE1(partial),
		    DTRACE_AGGUTF8_BYTE2(partial)) < 0)
			return (-1);

		i++;
	}

	return (dt_printf(dtp, fp, "%s %-9lld\n", spaces + i,
	    (long long)val / normal));
}

static int
dt_print_quantline(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
    uint64_t normal, long double total, char positives, char negatives)
{
	long double f;
	uint_t depth, len = 40;

	const char *ats = "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@";
	const char *spaces = "                                        ";

	assert(strlen(ats) == len && strlen(spaces) == len);
	assert(!(total == 0 && (positives || negatives)));
	assert(!(val < 0 && !negatives));
	assert(!(val > 0 && !positives));
	assert(!(val != 0 && total == 0));

	if (!negatives) {
		if (positives) {
			if (dtp->dt_encoding == DT_ENCODING_UTF8) {
				return (dt_print_quantline_utf8(dtp, fp, val,
				    normal, total));
			}

			f = (dt_fabsl((long double)val) * len) / total;
			depth = (uint_t)(f + 0.5);
		} else {
			depth = 0;
		}

		return (dt_printf(dtp, fp, "|%s%s %-9lld\n", ats + len - depth,
		    spaces + depth, (long long)val / normal));
	}

	if (!positives) {
		f = (dt_fabsl((long double)val) * len) / total;
		depth = (uint_t)(f + 0.5);

		return (dt_printf(dtp, fp, "%s%s| %-9lld\n", spaces + depth,
		    ats + len - depth, (long long)val / normal));
	}

	/*
	 * If we're here, we have both positive and negative bucket values.
	 * To express this graphically, we're going to generate both positive
	 * and negative bars separated by a centerline.  These bars are half
	 * the size of normal quantize()/lquantize() bars, so we divide the
	 * length in half before calculating the bar length.
	 */
	len /= 2;
	ats = &ats[len];
	spaces = &spaces[len];

	f = (dt_fabsl((long double)val) * len) / total;
	depth = (uint_t)(f + 0.5);

	if (val <= 0) {
		return (dt_printf(dtp, fp, "%s%s|%*s %-9lld\n", spaces + depth,
		    ats + len - depth, len, "", (long long)val / normal));
	} else {
		return (dt_printf(dtp, fp, "%20s|%s%s %-9lld\n", "",
		    ats + len - depth, spaces + depth,
		    (long long)val / normal));
	}
}

/*
 * As with UTF-8 printing of aggregations, we use a subset of the Unicode
 * Block Elements (U+2581 through U+2588, inclusive) to represent our packed
 * aggregation.
 */
#define	DTRACE_AGGPACK_BASE	0x2581
#define	DTRACE_AGGPACK_LEVELS	8

static int
dt_print_packed(dtrace_hdl_t *dtp, FILE *fp,
    long double datum, long double total)
{
	static boolean_t utf8_checked = B_FALSE;
	static boolean_t utf8;
	char *ascii = "__xxxxXX";
	char *neg = "vvvvVV";
	unsigned int len;
	long double val;

	if (!utf8_checked) {
		char *term;

		/*
		 * We want to determine if we can reasonably emit UTF-8 for our
		 * packed aggregation.  To do this, we will check for terminals
		 * that are known to be primitive to emit UTF-8 on these.
		 */
		utf8_checked = B_TRUE;

		if (dtp->dt_encoding == DT_ENCODING_ASCII) {
			utf8 = B_FALSE;
		} else if (dtp->dt_encoding == DT_ENCODING_UTF8) {
			utf8 = B_TRUE;
		} else if ((term = getenv("TERM")) != NULL &&
		    (strcmp(term, "sun") == 0 ||
		    strcmp(term, "sun-color") == 0 ||
		    strcmp(term, "dumb") == 0)) {
			utf8 = B_FALSE;
		} else {
			utf8 = B_TRUE;
		}
	}

	if (datum == 0)
		return (dt_printf(dtp, fp, " "));

	if (datum < 0) {
		len = strlen(neg);
		val = dt_fabsl(datum * (len - 1)) / total;
		return (dt_printf(dtp, fp, "%c", neg[(uint_t)(val + 0.5)]));
	}

	if (utf8) {
		int block = DTRACE_AGGPACK_BASE + (unsigned int)(((datum *
		    (DTRACE_AGGPACK_LEVELS - 1)) / total) + 0.5);

		return (dt_printf(dtp, fp, "%c%c%c",
		    DTRACE_AGGUTF8_BYTE0(block),
		    DTRACE_AGGUTF8_BYTE1(block),
		    DTRACE_AGGUTF8_BYTE2(block)));
	}

	len = strlen(ascii);
	val = (datum * (len - 1)) / total;
	return (dt_printf(dtp, fp, "%c", ascii[(uint_t)(val + 0.5)]));
}

static const int64_t *
dt_format_quantize_prepare(dtrace_hdl_t *dtp, const void *addr, size_t size,
    dt_prepare_args_t *args)
{
	const int64_t *data = addr;
	int first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;

	if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t)) {
		(void) dt_set_errno(dtp, EDT_DMISMATCH);
		return (NULL);
	}

	while (first_bin < DTRACE_QUANTIZE_NBUCKETS - 1 && data[first_bin] == 0)
		first_bin++;

	if (first_bin == DTRACE_QUANTIZE_NBUCKETS - 1) {
		/*
		 * There isn't any data.  This is possible if the aggregation
		 * has been clear()'d or if negative increment values have been
		 * used.  Regardless, we'll print the buckets around 0.
		 */
		first_bin = DTRACE_QUANTIZE_ZEROBUCKET - 1;
		last_bin = DTRACE_QUANTIZE_ZEROBUCKET + 1;
	} else {
		if (first_bin > 0)
			first_bin--;

		while (last_bin > 0 && data[last_bin] == 0)
			last_bin--;

		if (last_bin < DTRACE_QUANTIZE_NBUCKETS - 1)
			last_bin++;
	}

	args->first_bin = first_bin;
	args->last_bin = last_bin;
	return (data);
}

int
dt_format_quantize(dtrace_hdl_t *dtp, const void *addr, size_t size,
    uint64_t normal)
{
	const int64_t *data;
	dt_prepare_args_t args = { 0 };
	int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;

	data = dt_format_quantize_prepare(dtp, addr, size, &args);
	/* dt_errno is set for us */
	if (data == NULL)
		return (-1);

	first_bin = args.first_bin;
	last_bin = args.last_bin;

	xo_open_list("buckets");
	for (i = first_bin; i <= last_bin; i++) {
		long long value = (long long)DTRACE_QUANTIZE_BUCKETVAL(i);
		xo_open_instance("buckets");
		xo_emit("{:value/%lld} {:count/%lld}", value,
		    (long long)data[i] / normal);
		xo_close_instance("buckets");
	}
	xo_close_list("buckets");

	return (0);
}

int
dt_print_quantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
    size_t size, uint64_t normal)
{
	const int64_t *data;
	dt_prepare_args_t args = { 0 };
	int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;
	long double total = 0;
	char positives = 0, negatives = 0;

	data = dt_format_quantize_prepare(dtp, addr, size, &args);
	/* dt_errno is set for us */
	if (data == NULL)
		return (-1);

	first_bin = args.first_bin;
	last_bin = args.last_bin;

	for (i = first_bin; i <= last_bin; i++) {
		positives |= (data[i] > 0);
		negatives |= (data[i] < 0);
		dt_quantize_total(dtp, data[i], &total);
	}

	if (dt_print_quanthdr(dtp, fp, 0) < 0)
		return (-1);

	for (i = first_bin; i <= last_bin; i++) {
		if (dt_printf(dtp, fp, "%16lld ",
		    (long long)DTRACE_QUANTIZE_BUCKETVAL(i)) < 0)
			return (-1);

		if (dt_print_quantline(dtp, fp, data[i], normal, total,
		    positives, negatives) < 0)
			return (-1);
	}

	return (0);
}

int
dt_print_quantize_packed(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
    size_t size, const dtrace_aggdata_t *aggdata)
{
	const int64_t *data = addr;
	long double total = 0, count = 0;
	int min = aggdata->dtada_minbin, max = aggdata->dtada_maxbin, i;
	int64_t minval, maxval;

	if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
		return (dt_set_errno(dtp, EDT_DMISMATCH));

	if (min != 0 && min != DTRACE_QUANTIZE_ZEROBUCKET)
		min--;

	if (max < DTRACE_QUANTIZE_NBUCKETS - 1)
		max++;

	minval = DTRACE_QUANTIZE_BUCKETVAL(min);
	maxval = DTRACE_QUANTIZE_BUCKETVAL(max);

	if (dt_printf(dtp, fp, " %*lld :", dt_ndigits(minval),
	    (long long)minval) < 0)
		return (-1);

	for (i = min; i <= max; i++) {
		dt_quantize_total(dtp, data[i], &total);
		count += data[i];
	}

	for (i = min; i <= max; i++) {
		if (dt_print_packed(dtp, fp, data[i], total) < 0)
			return (-1);
	}

	if (dt_printf(dtp, fp, ": %*lld | %lld\n",
	    -dt_ndigits(maxval), (long long)maxval, (long long)count) < 0)
		return (-1);

	return (0);
}

static const int64_t *
dt_format_lquantize_prepare(dtrace_hdl_t *dtp, const void *addr, size_t size,
    dt_prepare_args_t *args)
{
	const int64_t *data = addr;
	int first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1, base;
	uint64_t arg;
	uint16_t step, levels;

	if (size < sizeof (uint64_t)) {
		(void) dt_set_errno(dtp, EDT_DMISMATCH);
		return (NULL);
	}

	arg = *data++;
	size -= sizeof (uint64_t);

	base = DTRACE_LQUANTIZE_BASE(arg);
	step = DTRACE_LQUANTIZE_STEP(arg);
	levels = DTRACE_LQUANTIZE_LEVELS(arg);

	first_bin = 0;
	last_bin = levels + 1;

	if (size != sizeof (uint64_t) * (levels + 2)) {
		(void) dt_set_errno(dtp, EDT_DMISMATCH);
		return (NULL);
	}

	while (first_bin <= levels + 1 && data[first_bin] == 0)
		first_bin++;

	if (first_bin > levels + 1) {
		first_bin = 0;
		last_bin = 2;
	} else {
		if (first_bin > 0)
			first_bin--;

		while (last_bin > 0 && data[last_bin] == 0)
			last_bin--;

		if (last_bin < levels + 1)
			last_bin++;
	}

	args->first_bin = first_bin;
	args->last_bin = last_bin;
	args->lquantize_base = base;
	args->lquantize_step = step;
	args->lquantize_levels = levels;
	return (data);
}

int
dt_format_lquantize(dtrace_hdl_t *dtp, const void *addr, size_t size,
    uint64_t normal)
{
	const int64_t *data;
	dt_prepare_args_t args = { 0 };
	int i, first_bin, last_bin, base;
	uint16_t step, levels;

	data = dt_format_lquantize_prepare(dtp, addr, size, &args);
	/* dt_errno is set for us */
	if (data == NULL)
		return (-1);

	first_bin = args.first_bin;
	last_bin = args.last_bin;
	step = args.lquantize_step;
	levels = args.lquantize_levels;
	base = args.lquantize_base;

	xo_open_list("buckets");
	for (i = first_bin; i <= last_bin; i++) {
		char c[32];
		int err;

		xo_open_instance("buckets");
		if (i == 0) {
			xo_emit("{:value/%d} {:operator/%s}", base, "<");
		} else if (i == levels + 1) {
			xo_emit("{:value/%d} {:operator/%s}",
			    base + (levels * step), ">=");
		} else {
			xo_emit("{:value/%d}", base + (i - 1) * step);
		}

		xo_emit("{:count/%lld}", (long long)data[i] / normal);
		xo_close_instance("buckets");
	}
	xo_close_list("buckets");

	return (0);
}

int
dt_print_lquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
    size_t size, uint64_t normal)
{
	const int64_t *data;
	dt_prepare_args_t args = { 0 };
	int i, first_bin, last_bin, base;
	uint64_t arg;
	long double total = 0;
	uint16_t step, levels;
	char positives = 0, negatives = 0;

	data = dt_format_lquantize_prepare(dtp, addr, size, &args);
	/* dt_errno is set for us */
	if (data == NULL)
		return (-1);

	first_bin = args.first_bin;
	last_bin = args.last_bin;
	step = args.lquantize_step;
	levels = args.lquantize_levels;
	base = args.lquantize_base;

	for (i = first_bin; i <= last_bin; i++) {
		positives |= (data[i] > 0);
		negatives |= (data[i] < 0);
		dt_quantize_total(dtp, data[i], &total);
	}

	if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
	    "------------- Distribution -------------", "count") < 0)
		return (-1);

	for (i = first_bin; i <= last_bin; i++) {
		char c[32];
		int err;

		if (i == 0) {
			(void) snprintf(c, sizeof (c), "< %d", base);
			err = dt_printf(dtp, fp, "%16s ", c);
		} else if (i == levels + 1) {
			(void) snprintf(c, sizeof (c), ">= %d",
			    base + (levels * step));
			err = dt_printf(dtp, fp, "%16s ", c);
		} else {
			err = dt_printf(dtp, fp, "%16d ",
			    base + (i - 1) * step);
		}

		if (err < 0 || dt_print_quantline(dtp, fp, data[i], normal,
		    total, positives, negatives) < 0)
			return (-1);
	}

	return (0);
}

/*ARGSUSED*/
int
dt_print_lquantize_packed(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
    size_t size, const dtrace_aggdata_t *aggdata)
{
	const int64_t *data = addr;
	long double total = 0, count = 0;
	int min, max, base, err;
	uint64_t arg;
	uint16_t step, levels;
	char c[32];
	unsigned int i;

	if (size < sizeof (uint64_t))
		return (dt_set_errno(dtp, EDT_DMISMATCH));

	arg = *data++;
	size -= sizeof (uint64_t);

	base = DTRACE_LQUANTIZE_BASE(arg);
	step = DTRACE_LQUANTIZE_STEP(arg);
	levels = DTRACE_LQUANTIZE_LEVELS(arg);

	if (size != sizeof (uint64_t) * (levels + 2))
		return (dt_set_errno(dtp, EDT_DMISMATCH));

	min = 0;
	max = levels + 1;

	if (min == 0) {
		(void) snprintf(c, sizeof (c), "< %d", base);
		err = dt_printf(dtp, fp, "%8s :", c);
	} else {
		err = dt_printf(dtp, fp, "%8d :", base + (min - 1) * step);
	}

	if (err < 0)
		return (-1);

	for (i = min; i <= max; i++) {
		dt_quantize_total(dtp, data[i], &total);
		count += data[i];
	}

	for (i = min; i <= max; i++) {
		if (dt_print_packed(dtp, fp, data[i], total) < 0)
			return (-1);
	}

	(void) snprintf(c, sizeof (c), ">= %d", base + (levels * step));
	return (dt_printf(dtp, fp, ": %-8s | %lld\n", c, (long long)count));
}

static const int64_t *
dt_format_llquantize_prepare(dtrace_hdl_t *dtp, const void *addr, size_t size,
    dt_prepare_args_t *args)
{
	int i, first_bin, last_bin, bin = 1, order, levels;
	uint16_t factor, low, high, nsteps;
	const int64_t *data = addr;
	int64_t value = 1, next, step;
	uint64_t arg;

	if (size < sizeof(uint64_t)) {
		(void) dt_set_errno(dtp, EDT_DMISMATCH);
		return (NULL);
	}

	arg = *data++;
	size -= sizeof (uint64_t);

	factor = DTRACE_LLQUANTIZE_FACTOR(arg);
	low = DTRACE_LLQUANTIZE_LOW(arg);
	high = DTRACE_LLQUANTIZE_HIGH(arg);
	nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);

	/*
	 * We don't expect to be handed invalid llquantize() parameters here,
	 * but sanity check them (to a degree) nonetheless.
	 */
	if (size > INT32_MAX || factor < 2 || low >= high ||
	    nsteps == 0 || factor > nsteps) {
		(void) dt_set_errno(dtp, EDT_DMISMATCH);
		return (NULL);
	}

	levels = (int)size / sizeof (uint64_t);

	first_bin = 0;
	last_bin = levels - 1;

	while (first_bin < levels && data[first_bin] == 0)
		first_bin++;

	if (first_bin == levels) {
		first_bin = 0;
		last_bin = 1;
	} else {
		if (first_bin > 0)
			first_bin--;

		while (last_bin > 0 && data[last_bin] == 0)
			last_bin--;

		if (last_bin < levels - 1)
			last_bin++;
	}

	for (order = 0; order < low; order++)
		value *= factor;

	next = value * factor;
	step = next > nsteps ? next / nsteps : 1;

	args->first_bin = first_bin;
	args->last_bin = last_bin;
	args->llquantize_factor = factor;
	args->llquantize_low = low;
	args->llquantize_high = high;
	args->llquantize_nsteps = nsteps;
	args->llquantize_levels = levels;
	args->llquantize_order = order;
	args->llquantize_next = next;
	args->llquantize_step = step;
	args->llquantize_value = value;

	return (data);
}

int
dt_format_llquantize(dtrace_hdl_t *dtp, const void *addr, size_t size,
    uint64_t normal)
{
	int first_bin, last_bin, bin = 1, order, levels;
	uint16_t factor, low, high, nsteps;
	const int64_t *data;
	dt_prepare_args_t args = { 0 };
	int64_t value = 1, next, step;
	uint64_t arg;
	char c[32];

	data = dt_format_llquantize_prepare(dtp, addr, size, &args);
	/* dt_errno is set for us */
	if (data == NULL)
		return (-1);

	first_bin = args.first_bin;
	last_bin = args.last_bin;
	factor = args.llquantize_factor;
	low = args.llquantize_low;
	high = args.llquantize_high;
	nsteps = args.llquantize_nsteps;
	levels = args.llquantize_levels;
	order = args.llquantize_order;
	next = args.llquantize_next;
	step = args.llquantize_step;
	value = args.llquantize_value;

	xo_open_list("buckets");
	if (first_bin == 0) {
		/*
		 * We have to represent < value somehow in JSON, so we bundle an
		 * optional "operator" in llquantize buckets.
		 */
		xo_open_instance("buckets");
		xo_emit("{:value/%lld} {:count/%lld} {:operator/%s}",
		    (long long)value, (long long)data[0] / normal, "<");
		xo_close_instance("buckets");
	}

	while (order <= high) {
		if (bin >= first_bin && bin <= last_bin) {
			xo_open_instance("buckets");
			xo_emit("{:value/%lld} {:count/%lld}", (long long)value,
			    (long long)data[bin] / normal);
			xo_close_instance("buckets");
		}

		assert(value < next);
		bin++;

		if ((value += step) != next)
			continue;

		next = value * factor;
		step = next > nsteps ? next / nsteps : 1;
		order++;
	}

	if (last_bin < bin) {
		xo_close_list("buckets");
		return (0);
	}

	assert(last_bin == bin);
	xo_open_instance("buckets");
	xo_emit("{:value/%lld} {:count/%lld} {:operator/%s}", (long long)value,
	    (long long)data[bin] / normal, ">=");
	xo_close_instance("buckets");

	xo_close_list("buckets");
	return (0);
}

int
dt_print_llquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
    size_t size, uint64_t normal)
{
	int i, first_bin, last_bin, bin = 1, order, levels;
	uint16_t factor, low, high, nsteps;
	const int64_t *data;
	dt_prepare_args_t args = { 0 };
	int64_t value = 1, next, step;
	char positives = 0, negatives = 0;
	long double total = 0;
	uint64_t arg;
	char c[32];

	data = dt_format_llquantize_prepare(dtp, addr, size, &args);
	/* dt_errno is set for us */
	if (data == NULL)
		return (-1);

	first_bin = args.first_bin;
	last_bin = args.last_bin;
	factor = args.llquantize_factor;
	low = args.llquantize_low;
	high = args.llquantize_high;
	nsteps = args.llquantize_nsteps;
	levels = args.llquantize_levels;
	order = args.llquantize_order;
	next = args.llquantize_next;
	step = args.llquantize_step;
	value = args.llquantize_value;

	for (i = first_bin; i <= last_bin; i++) {
		positives |= (data[i] > 0);
		negatives |= (data[i] < 0);
		dt_quantize_total(dtp, data[i], &total);
	}

	if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
	    "------------- Distribution -------------", "count") < 0)
		return (-1);

	if (first_bin == 0) {
		(void) snprintf(c, sizeof (c), "< %lld", (long long)value);

		if (dt_printf(dtp, fp, "%16s ", c) < 0)
			return (-1);

		if (dt_print_quantline(dtp, fp, data[0], normal,
		    total, positives, negatives) < 0)
			return (-1);
	}

	while (order <= high) {
		if (bin >= first_bin && bin <= last_bin) {
			if (dt_printf(dtp, fp, "%16lld ", (long long)value) < 0)
				return (-1);

			if (dt_print_quantline(dtp, fp, data[bin],
			    normal, total, positives, negatives) < 0)
				return (-1);
		}

		assert(value < next);
		bin++;

		if ((value += step) != next)
			continue;

		next = value * factor;
		step = next > nsteps ? next / nsteps : 1;
		order++;
	}

	if (last_bin < bin)
		return (0);

	assert(last_bin == bin);
	(void) snprintf(c, sizeof (c), ">= %lld", (long long)value);

	if (dt_printf(dtp, fp, "%16s ", c) < 0)
		return (-1);

	return (dt_print_quantline(dtp, fp, data[bin], normal,
	    total, positives, negatives));
}

static int
dt_format_average(dtrace_hdl_t *dtp, caddr_t addr, size_t size, uint64_t normal)
{
	int64_t *data = (int64_t *)addr;

	xo_emit("{:average/%lld}",
	    data[0] ? (long long)(data[1] / (int64_t)normal / data[0]) : 0);
	return (0);
}

/*ARGSUSED*/
static int
dt_print_average(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
    size_t size, uint64_t normal)
{
	/* LINTED - alignment */
	int64_t *data = (int64_t *)addr;

	return (dt_printf(dtp, fp, " %16lld", data[0] ?
	    (long long)(data[1] / (int64_t)normal / data[0]) : 0));
}

static int
dt_format_stddev(dtrace_hdl_t *dtp, caddr_t addr, size_t size, uint64_t normal)
{
	uint64_t *data = (uint64_t *)addr;

	xo_emit("{:stddev/%llu}",
	    data[0] ? (unsigned long long)dt_stddev(data, normal) : 0);
	return (0);
}

/*ARGSUSED*/
static int
dt_print_stddev(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
    size_t size, uint64_t normal)
{
	/* LINTED - alignment */
	uint64_t *data = (uint64_t *)addr;

	return (dt_printf(dtp, fp, " %16llu", data[0] ?
	    (unsigned long long) dt_stddev(data, normal) : 0));
}

/*ARGSUSED*/
static int
dt_print_bytes(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
    size_t nbytes, int width, int quiet, int forceraw)
{
	/*
	 * If the byte stream is a series of printable characters, followed by
	 * a terminating byte, we print it out as a string.  Otherwise, we
	 * assume that it's something else and just print the bytes.
	 */
	int i, j, margin = 5;
	char *c = (char *)addr;

	if (nbytes == 0)
		return (0);

	if (forceraw)
		goto raw;

	if (dtp->dt_options[DTRACEOPT_RAWBYTES] != DTRACEOPT_UNSET)
		goto raw;

	for (i = 0; i < nbytes; i++) {
		/*
		 * We define a "printable character" to be one for which
		 * isprint(3C) returns non-zero, isspace(3C) returns non-zero,
		 * or a character which is either backspace or the bell.
		 * Backspace and the bell are regrettably special because
		 * they fail the first two tests -- and yet they are entirely
		 * printable.  These are the only two control characters that
		 * have meaning for the terminal and for which isprint(3C) and
		 * isspace(3C) return 0.
		 */
		if (isprint(c[i]) || isspace(c[i]) ||
		    c[i] == '\b' || c[i] == '\a')
			continue;

		if (c[i] == '\0' && i > 0) {
			/*
			 * This looks like it might be a string.  Before we
			 * assume that it is indeed a string, check the
			 * remainder of the byte range; if it contains
			 * additional non-nul characters, we'll assume that
			 * it's a binary stream that just happens to look like
			 * a string, and we'll print out the individual bytes.
			 */
			for (j = i + 1; j < nbytes; j++) {
				if (c[j] != '\0')
					break;
			}

			if (j != nbytes)
				break;

			if (quiet) {
				return (dt_printf(dtp, fp, "%s", c));
			} else {
				return (dt_printf(dtp, fp, " %s%*s",
				    width < 0 ? " " : "", width, c));
			}
		}

		break;
	}

	if (i == nbytes) {
		/*
		 * The byte range is all printable characters, but there is
		 * no trailing nul byte.  We'll assume that it's a string and
		 * print it as such.
		 */
		char *s = alloca(nbytes + 1);
		bcopy(c, s, nbytes);
		s[nbytes] = '\0';
		return (dt_printf(dtp, fp, "  %-*s", width, s));
	}

raw:
	if (dt_printf(dtp, fp, "\n%*s      ", margin, "") < 0)
		return (-1);

	for (i = 0; i < 16; i++)
		if (dt_printf(dtp, fp, "  %c", "0123456789abcdef"[i]) < 0)
			return (-1);

	if (dt_printf(dtp, fp, "  0123456789abcdef\n") < 0)
		return (-1);


	for (i = 0; i < nbytes; i += 16) {
		if (dt_printf(dtp, fp, "%*s%5x:", margin, "", i) < 0)
			return (-1);

		for (j = i; j < i + 16 && j < nbytes; j++) {
			if (dt_printf(dtp, fp, " %02x", (uchar_t)c[j]) < 0)
				return (-1);
		}

		while (j++ % 16) {
			if (dt_printf(dtp, fp, "   ") < 0)
				return (-1);
		}

		if (dt_printf(dtp, fp, "  ") < 0)
			return (-1);

		for (j = i; j < i + 16 && j < nbytes; j++) {
			if (dt_printf(dtp, fp, "%c",
			    c[j] < ' ' || c[j] > '~' ? '.' : c[j]) < 0)
				return (-1);
		}

		if (dt_printf(dtp, fp, "\n") < 0)
			return (-1);
	}

	return (0);
}

int
dt_format_stack(dtrace_hdl_t *dtp, caddr_t addr, int depth, int size)
{
	dtrace_syminfo_t dts;
	GElf_Sym sym;
	int i;
	uint64_t pc;

	xo_open_list("stack-frames");
	for (i = 0; i < depth; i++) {
		switch (size) {
		case sizeof (uint32_t):
			pc = *((uint32_t *)addr);
			break;

		case sizeof (uint64_t):
			pc = *((uint64_t *)addr);
			break;

		default:
			return (dt_set_errno(dtp, EDT_BADSTACKPC));
		}

		if (pc == 0)
			break;

		addr += size;

		xo_open_instance("stack-frames");
		if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
			if (pc > sym.st_value) {
				xo_emit("{:symbol/%s`%s+0x%llx} {:module/%s} "
					"{:name/%s} {:offset/0x%llx}",
				    dts.dts_object, dts.dts_name,
				    (u_longlong_t)(pc - sym.st_value),
				    dts.dts_object, dts.dts_name,
				    (u_longlong_t)(pc - sym.st_value));
			} else {
				xo_emit("{:symbol/%s`%s} {:module/%s} "
					"{:name/%s}",
				    dts.dts_object, dts.dts_name,
				    dts.dts_object, dts.dts_name);
			}
		} else {
			/*
			 * We'll repeat the lookup, but this time we'll specify
			 * a NULL GElf_Sym -- indicating that we're only
			 * interested in the containing module.
			 */
			if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
				xo_emit("{:symbol/%s`0x%llx} {:module/%s} "
					"{:offset/0x%llx}",
				    dts.dts_object, (u_longlong_t)pc,
				    dts.dts_object, (u_longlong_t)pc);
			} else {
				xo_emit("{:symbol/0x%llx} {:offset/0x%llx}",
				    (u_longlong_t)pc, (u_longlong_t)pc);
			}
		}
		xo_close_instance("stack-frames");
	}
	xo_close_list("stack-frames");

	return (0);
}

int
dt_format_ustack(dtrace_hdl_t *dtp, caddr_t addr, uint64_t arg)
{
	uint64_t *pc = (uint64_t *)addr;
	uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
	uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
	const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
	const char *str = strsize ? strbase : NULL;
	int err = 0;

	char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
	struct ps_prochandle *P;
	GElf_Sym sym;
	int i, indent;
	pid_t pid;

	if (depth == 0)
		return (0);

	pid = (pid_t)*pc++;

	/*
	 * Ultimately, we need to add an entry point in the library vector for
	 * determining <symbol, offset> from <pid, address>.  For now, if
	 * this is a vector open, we just print the raw address or string.
	 */
	if (dtp->dt_vector == NULL)
		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
	else
		P = NULL;

	if (P != NULL)
		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */

	xo_open_list("ustack-frames");
	for (i = 0; i < depth && pc[i] != 0; i++) {
		const prmap_t *map;

		xo_open_instance("ustack-frames");
		if (P != NULL && Plookup_by_addr(P, pc[i],
		    name, sizeof (name), &sym) == 0) {
			(void) Pobjname(P, pc[i], objname, sizeof (objname));

			if (pc[i] > sym.st_value) {
				xo_emit("{:symbol/%s`%s+0x%llx} {:module/%s} "
					"{:name/%s} {:offset/0x%llx}",
				    dt_basename(objname), name,
				    (u_longlong_t)(pc[i] - sym.st_value),
				    dt_basename(objname), name,
				    (u_longlong_t)(pc[i] - sym.st_value));
			} else {
				xo_emit("{:symbol/%s`%s} {:module/%s} "
					"{:name/%s}",
				    dt_basename(objname), name,
				    dt_basename(objname), name);
			}
		} else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
		    (P != NULL && ((map = Paddr_to_map(P, pc[i])) == NULL ||
		    (map->pr_mflags & MA_WRITE)))) {
			/*
			 * If the current string pointer in the string table
			 * does not point to an empty string _and_ the program
			 * counter falls in a writable region, we'll use the
			 * string from the string table instead of the raw
			 * address.  This last condition is necessary because
			 * some (broken) ustack helpers will return a string
			 * even for a program counter that they can't
			 * identify.  If we have a string for a program
			 * counter that falls in a segment that isn't
			 * writable, we assume that we have fallen into this
			 * case and we refuse to use the string.
			 */
			xo_emit("{:symbol/%s}", str);
		} else {
			if (P != NULL && Pobjname(P, pc[i], objname,
			    sizeof (objname)) != 0) {
				xo_emit("{:symbol/%s`0x%llx} {:module/%s} "
					"{:offset/0x%llx}",
				    dt_basename(objname), (u_longlong_t)pc[i],
				    dt_basename(objname), (u_longlong_t)pc[i]);
			} else {
				xo_emit("{:symbol/0x%llx} {:offset/0x%llx}",
				    (u_longlong_t)pc[i], (u_longlong_t)pc[i]);
			}
		}

		if (str != NULL && str[0] == '@') {
			/*
			 * If the first character of the string is an "at" sign,
			 * then the string is inferred to be an annotation --
			 * and it is printed out beneath the frame and offset
			 * with brackets.
			 */
			xo_emit("{:annotation/%s}", &str[1]);
		}

		if (str != NULL) {
			str += strlen(str) + 1;
			if (str - strbase >= strsize)
				str = NULL;
		}
		xo_close_instance("ustack-frames");
	}
	xo_close_list("ustack-frames");

	if (P != NULL) {
		dt_proc_unlock(dtp, P);
		dt_proc_release(dtp, P);
	}

	return (err);
}

int
dt_print_stack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
    caddr_t addr, int depth, int size)
{
	dtrace_syminfo_t dts;
	GElf_Sym sym;
	int i, indent;
	char c[PATH_MAX * 2];
	uint64_t pc;

	if (dt_printf(dtp, fp, "\n") < 0)
		return (-1);

	if (format == NULL)
		format = "%s";

	if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
		indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
	else
		indent = _dtrace_stkindent;

	for (i = 0; i < depth; i++) {
		switch (size) {
		case sizeof (uint32_t):
			/* LINTED - alignment */
			pc = *((uint32_t *)addr);
			break;

		case sizeof (uint64_t):
			/* LINTED - alignment */
			pc = *((uint64_t *)addr);
			break;

		default:
			return (dt_set_errno(dtp, EDT_BADSTACKPC));
		}

		if (pc == 0)
			break;

		addr += size;

		if (dt_printf(dtp, fp, "%*s", indent, "") < 0)
			return (-1);

		if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
			if (pc > sym.st_value) {
				(void) snprintf(c, sizeof (c), "%s`%s+0x%llx",
				    dts.dts_object, dts.dts_name,
				    (u_longlong_t)(pc - sym.st_value));
			} else {
				(void) snprintf(c, sizeof (c), "%s`%s",
				    dts.dts_object, dts.dts_name);
			}
		} else {
			/*
			 * We'll repeat the lookup, but this time we'll specify
			 * a NULL GElf_Sym -- indicating that we're only
			 * interested in the containing module.
			 */
			if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
				(void) snprintf(c, sizeof (c), "%s`0x%llx",
				    dts.dts_object, (u_longlong_t)pc);
			} else {
				(void) snprintf(c, sizeof (c), "0x%llx",
				    (u_longlong_t)pc);
			}
		}

		if (dt_printf(dtp, fp, format, c) < 0)
			return (-1);

		if (dt_printf(dtp, fp, "\n") < 0)
			return (-1);
	}

	return (0);
}

int
dt_print_ustack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
    caddr_t addr, uint64_t arg)
{
	/* LINTED - alignment */
	uint64_t *pc = (uint64_t *)addr;
	uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
	uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
	const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
	const char *str = strsize ? strbase : NULL;
	int err = 0;

	char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
	struct ps_prochandle *P;
	GElf_Sym sym;
	int i, indent;
	pid_t pid;

	if (depth == 0)
		return (0);

	pid = (pid_t)*pc++;

	if (dt_printf(dtp, fp, "\n") < 0)
		return (-1);

	if (format == NULL)
		format = "%s";

	if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
		indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
	else
		indent = _dtrace_stkindent;

	/*
	 * Ultimately, we need to add an entry point in the library vector for
	 * determining <symbol, offset> from <pid, address>.  For now, if
	 * this is a vector open, we just print the raw address or string.
	 */
	if (dtp->dt_vector == NULL)
		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
	else
		P = NULL;

	if (P != NULL)
		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */

	for (i = 0; i < depth && pc[i] != 0; i++) {
		const prmap_t *map;

		if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
			break;

		if (P != NULL && Plookup_by_addr(P, pc[i],
		    name, sizeof (name), &sym) == 0) {
			(void) Pobjname(P, pc[i], objname, sizeof (objname));

			if (pc[i] > sym.st_value) {
				(void) snprintf(c, sizeof (c),
				    "%s`%s+0x%llx", dt_basename(objname), name,
				    (u_longlong_t)(pc[i] - sym.st_value));
			} else {
				(void) snprintf(c, sizeof (c),
				    "%s`%s", dt_basename(objname), name);
			}
		} else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
		    (P != NULL && ((map = Paddr_to_map(P, pc[i])) == NULL ||
		    (map->pr_mflags & MA_WRITE)))) {
			/*
			 * If the current string pointer in the string table
			 * does not point to an empty string _and_ the program
			 * counter falls in a writable region, we'll use the
			 * string from the string table instead of the raw
			 * address.  This last condition is necessary because
			 * some (broken) ustack helpers will return a string
			 * even for a program counter that they can't
			 * identify.  If we have a string for a program
			 * counter that falls in a segment that isn't
			 * writable, we assume that we have fallen into this
			 * case and we refuse to use the string.
			 */
			(void) snprintf(c, sizeof (c), "%s", str);
		} else {
			if (P != NULL && Pobjname(P, pc[i], objname,
			    sizeof (objname)) != 0) {
				(void) snprintf(c, sizeof (c), "%s`0x%llx",
				    dt_basename(objname), (u_longlong_t)pc[i]);
			} else {
				(void) snprintf(c, sizeof (c), "0x%llx",
				    (u_longlong_t)pc[i]);
			}
		}

		if ((err = dt_printf(dtp, fp, format, c)) < 0)
			break;

		if ((err = dt_printf(dtp, fp, "\n")) < 0)
			break;

		if (str != NULL && str[0] == '@') {
			/*
			 * If the first character of the string is an "at" sign,
			 * then the string is inferred to be an annotation --
			 * and it is printed out beneath the frame and offset
			 * with brackets.
			 */
			if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
				break;

			(void) snprintf(c, sizeof (c), "  [ %s ]", &str[1]);

			if ((err = dt_printf(dtp, fp, format, c)) < 0)
				break;

			if ((err = dt_printf(dtp, fp, "\n")) < 0)
				break;
		}

		if (str != NULL) {
			str += strlen(str) + 1;
			if (str - strbase >= strsize)
				str = NULL;
		}
	}

	if (P != NULL) {
		dt_proc_unlock(dtp, P);
		dt_proc_release(dtp, P);
	}

	return (err);
}

static int
dt_format_usym(dtrace_hdl_t *dtp, caddr_t addr, dtrace_actkind_t act)
{
	uint64_t pid = ((uint64_t *)addr)[0];
	uint64_t pc = ((uint64_t *)addr)[1];
	char *s;
	int n, len = 256;

	if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
		struct ps_prochandle *P;

		if ((P = dt_proc_grab(dtp, pid,
		    PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
			GElf_Sym sym;

			dt_proc_lock(dtp, P);

			if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
				pc = sym.st_value;

			dt_proc_unlock(dtp, P);
			dt_proc_release(dtp, P);
		}
	}

	do {
		n = len;
		s = alloca(n);
	} while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) > n);

	xo_emit("{:usym/%s}", s);
	return (0);
}


static int
dt_print_usym(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr, dtrace_actkind_t act)
{
	/* LINTED - alignment */
	uint64_t pid = ((uint64_t *)addr)[0];
	/* LINTED - alignment */
	uint64_t pc = ((uint64_t *)addr)[1];
	const char *format = "  %-50s";
	char *s;
	int n, len = 256;

	if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
		struct ps_prochandle *P;

		if ((P = dt_proc_grab(dtp, pid,
		    PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
			GElf_Sym sym;

			dt_proc_lock(dtp, P);

			if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
				pc = sym.st_value;

			dt_proc_unlock(dtp, P);
			dt_proc_release(dtp, P);
		}
	}

	do {
		n = len;
		s = alloca(n);
	} while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) > n);

	return (dt_printf(dtp, fp, format, s));
}

int
dt_format_umod(dtrace_hdl_t *dtp, caddr_t addr)
{
	uint64_t pid = ((uint64_t *)addr)[0];
	uint64_t pc = ((uint64_t *)addr)[1];
	int err = 0;

	char objname[PATH_MAX];
	struct ps_prochandle *P;

	/*
	 * See the comment in dt_print_ustack() for the rationale for
	 * printing raw addresses in the vectored case.
	 */
	if (dtp->dt_vector == NULL)
		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
	else
		P = NULL;

	if (P != NULL)
		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */

	if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != 0) {
		xo_emit("{:umod/%s}", dt_basename(objname));
	} else {
		xo_emit("{:umod/0x%llx}", (u_longlong_t)pc);
	}

	if (P != NULL) {
		dt_proc_unlock(dtp, P);
		dt_proc_release(dtp, P);
	}

	return (0);
}

int
dt_print_umod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
{
	/* LINTED - alignment */
	uint64_t pid = ((uint64_t *)addr)[0];
	/* LINTED - alignment */
	uint64_t pc = ((uint64_t *)addr)[1];
	int err = 0;

	char objname[PATH_MAX], c[PATH_MAX * 2];
	struct ps_prochandle *P;

	if (format == NULL)
		format = "  %-50s";

	/*
	 * See the comment in dt_print_ustack() for the rationale for
	 * printing raw addresses in the vectored case.
	 */
	if (dtp->dt_vector == NULL)
		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
	else
		P = NULL;

	if (P != NULL)
		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */

	if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != 0) {
		(void) snprintf(c, sizeof (c), "%s", dt_basename(objname));
	} else {
		(void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
	}

	err = dt_printf(dtp, fp, format, c);

	if (P != NULL) {
		dt_proc_unlock(dtp, P);
		dt_proc_release(dtp, P);
	}

	return (err);
}

static int
dt_print_sym(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
{
	/* LINTED - alignment */
	uint64_t pc = *((uint64_t *)addr);
	dtrace_syminfo_t dts;
	GElf_Sym sym;
	char c[PATH_MAX * 2];

	if (format == NULL)
		format = "  %-50s";

	if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
		if (dtp->dt_oformat)
			xo_emit("{:sym/%s`%s} {:object/%s} {:name/%s}",
			    dts.dts_object, dts.dts_name, dts.dts_object,
			    dts.dts_name);
		else
			(void) snprintf(c, sizeof (c), "%s`%s",
			    dts.dts_object, dts.dts_name);
	} else {
		/*
		 * We'll repeat the lookup, but this time we'll specify a
		 * NULL GElf_Sym -- indicating that we're only interested in
		 * the containing module.
		 */
		if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
			if (dtp->dt_oformat)
				xo_emit("{:sym/%s`0x%llx} {:object/%s} "
					"{:offset/0x%llx}",
				    dts.dts_object, (u_longlong_t)pc,
				    dts.dts_object, (u_longlong_t)pc);
			else
				(void) snprintf(c, sizeof (c), "%s`0x%llx",
				    dts.dts_object, (u_longlong_t)pc);
		} else {
			if (dtp->dt_oformat)
				xo_emit("{:sym/0x%llx} {:offset/0x%llx}",
				    (u_longlong_t)pc, (u_longlong_t)pc);
			else
				(void) snprintf(c, sizeof (c), "0x%llx",
				    (u_longlong_t)pc);
		}
	}

	if (dtp->dt_oformat != 0 && dt_printf(dtp, fp, format, c) < 0)
		return (-1);

	return (0);
}

int
dt_format_mod(dtrace_hdl_t *dtp, caddr_t addr)
{
	/* LINTED - alignment */
	uint64_t pc = *((uint64_t *)addr);
	dtrace_syminfo_t dts;

	if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
		xo_emit("{:mod/%s}", dts.dts_object);
	} else {
		xo_emit("{:mod/0x%llx}", (u_longlong_t)pc);
	}

	return (0);
}

int
dt_print_mod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
{
	/* LINTED - alignment */
	uint64_t pc = *((uint64_t *)addr);
	dtrace_syminfo_t dts;
	char c[PATH_MAX * 2];

	if (format == NULL)
		format = "  %-50s";

	if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
		(void) snprintf(c, sizeof (c), "%s", dts.dts_object);
	} else {
		(void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
	}

	if (dt_printf(dtp, fp, format, c) < 0)
		return (-1);

	return (0);
}

static char *
dt_format_bytes_get(dtrace_hdl_t *dtp, caddr_t addr, size_t nbytes)
{
	char *s = dt_alloc(dtp, nbytes * 2 + 2 + 1); /* 2 bytes per byte + 0x + '\0' */
	char t[6];
	char *c = (char *)addr;
	size_t i, j;

	if (s == NULL)
		return (NULL);

	/*
	 * XXX: Some duplication with dt_print_bytes().
	 */
	for (i = 0; i < nbytes; i++) {
		if (isprint(c[i]) || isspace(c[i]) || c[i] == '\b' || c[i] == '\a')
			continue;

		if (c[i] == '\0' && i > 0) {
			for (j = i + 1; j < nbytes; j++) {
				if (c[j] != '\0')
					break;
			}

			if (j != nbytes)
				break;

			memcpy(s, c, nbytes);
			return (s);
		}

		break;
	}

	if (i == nbytes) {
		memcpy(s, c, nbytes);
		s[nbytes] = '\0';
		return (s);
	}

	s[0] = '0';
	s[1] = 'x';
	for (i = 0; i < nbytes; i++) {
		snprintf(t, sizeof(t), "%02x", (uchar_t)c[i]);
		memcpy(s + (i * 2) + 2, t, 2);
	}

	s[nbytes * 2 + 2] = 0;
	return (s);
}

static int
dt_format_memory(dtrace_hdl_t *dtp, caddr_t addr)
{
	size_t nbytes = *((size_t *) addr);
	char *s;

	s = dt_format_bytes_get(dtp, addr + sizeof(size_t), nbytes);
	if (s == NULL)
		return (-1);

	xo_emit("{:printm/%s}", s);
	dt_free(dtp, s);

	return (0);
}

static int
dt_print_memory(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
{
	int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
	size_t nbytes = *((size_t *) addr);

	return (dt_print_bytes(dtp, fp, addr + sizeof(size_t),
	    nbytes, 50, quiet, 1));
}

typedef struct dt_normal {
	dtrace_aggvarid_t dtnd_id;
	uint64_t dtnd_normal;
} dt_normal_t;

static int
dt_normalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
{
	dt_normal_t *normal = arg;
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	dtrace_aggvarid_t id = normal->dtnd_id;

	if (agg->dtagd_nrecs == 0)
		return (DTRACE_AGGWALK_NEXT);

	if (agg->dtagd_varid != id)
		return (DTRACE_AGGWALK_NEXT);

	((dtrace_aggdata_t *)aggdata)->dtada_normal = normal->dtnd_normal;
	return (DTRACE_AGGWALK_NORMALIZE);
}

static int
dt_normalize(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
{
	dt_normal_t normal;
	caddr_t addr;

	/*
	 * We (should) have two records:  the aggregation ID followed by the
	 * normalization value.
	 */
	addr = base + rec->dtrd_offset;

	if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
		return (dt_set_errno(dtp, EDT_BADNORMAL));

	/* LINTED - alignment */
	normal.dtnd_id = *((dtrace_aggvarid_t *)addr);
	rec++;

	if (rec->dtrd_action != DTRACEACT_LIBACT)
		return (dt_set_errno(dtp, EDT_BADNORMAL));

	if (rec->dtrd_arg != DT_ACT_NORMALIZE)
		return (dt_set_errno(dtp, EDT_BADNORMAL));

	addr = base + rec->dtrd_offset;

	switch (rec->dtrd_size) {
	case sizeof (uint64_t):
		/* LINTED - alignment */
		normal.dtnd_normal = *((uint64_t *)addr);
		break;
	case sizeof (uint32_t):
		/* LINTED - alignment */
		normal.dtnd_normal = *((uint32_t *)addr);
		break;
	case sizeof (uint16_t):
		/* LINTED - alignment */
		normal.dtnd_normal = *((uint16_t *)addr);
		break;
	case sizeof (uint8_t):
		normal.dtnd_normal = *((uint8_t *)addr);
		break;
	default:
		return (dt_set_errno(dtp, EDT_BADNORMAL));
	}

	(void) dtrace_aggregate_walk(dtp, dt_normalize_agg, &normal);

	return (0);
}

static int
dt_denormalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
{
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);

	if (agg->dtagd_nrecs == 0)
		return (DTRACE_AGGWALK_NEXT);

	if (agg->dtagd_varid != id)
		return (DTRACE_AGGWALK_NEXT);

	return (DTRACE_AGGWALK_DENORMALIZE);
}

static int
dt_clear_agg(const dtrace_aggdata_t *aggdata, void *arg)
{
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);

	if (agg->dtagd_nrecs == 0)
		return (DTRACE_AGGWALK_NEXT);

	if (agg->dtagd_varid != id)
		return (DTRACE_AGGWALK_NEXT);

	return (DTRACE_AGGWALK_CLEAR);
}

typedef struct dt_trunc {
	dtrace_aggvarid_t dttd_id;
	uint64_t dttd_remaining;
} dt_trunc_t;

static int
dt_trunc_agg(const dtrace_aggdata_t *aggdata, void *arg)
{
	dt_trunc_t *trunc = arg;
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	dtrace_aggvarid_t id = trunc->dttd_id;

	if (agg->dtagd_nrecs == 0)
		return (DTRACE_AGGWALK_NEXT);

	if (agg->dtagd_varid != id)
		return (DTRACE_AGGWALK_NEXT);

	if (trunc->dttd_remaining == 0)
		return (DTRACE_AGGWALK_REMOVE);

	trunc->dttd_remaining--;
	return (DTRACE_AGGWALK_NEXT);
}

static int
dt_trunc(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
{
	dt_trunc_t trunc;
	caddr_t addr;
	int64_t remaining;
	int (*func)(dtrace_hdl_t *, dtrace_aggregate_f *, void *);

	/*
	 * We (should) have two records:  the aggregation ID followed by the
	 * number of aggregation entries after which the aggregation is to be
	 * truncated.
	 */
	addr = base + rec->dtrd_offset;

	if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
		return (dt_set_errno(dtp, EDT_BADTRUNC));

	/* LINTED - alignment */
	trunc.dttd_id = *((dtrace_aggvarid_t *)addr);
	rec++;

	if (rec->dtrd_action != DTRACEACT_LIBACT)
		return (dt_set_errno(dtp, EDT_BADTRUNC));

	if (rec->dtrd_arg != DT_ACT_TRUNC)
		return (dt_set_errno(dtp, EDT_BADTRUNC));

	addr = base + rec->dtrd_offset;

	switch (rec->dtrd_size) {
	case sizeof (uint64_t):
		/* LINTED - alignment */
		remaining = *((int64_t *)addr);
		break;
	case sizeof (uint32_t):
		/* LINTED - alignment */
		remaining = *((int32_t *)addr);
		break;
	case sizeof (uint16_t):
		/* LINTED - alignment */
		remaining = *((int16_t *)addr);
		break;
	case sizeof (uint8_t):
		remaining = *((int8_t *)addr);
		break;
	default:
		return (dt_set_errno(dtp, EDT_BADNORMAL));
	}

	if (remaining < 0) {
		func = dtrace_aggregate_walk_valsorted;
		remaining = -remaining;
	} else {
		func = dtrace_aggregate_walk_valrevsorted;
	}

	assert(remaining >= 0);
	trunc.dttd_remaining = remaining;

	(void) func(dtp, dt_trunc_agg, &trunc);

	return (0);
}

static int
dt_format_datum(dtrace_hdl_t *dtp, dtrace_recdesc_t *rec, caddr_t addr,
    size_t size, const dtrace_aggdata_t *aggdata, uint64_t normal,
    dt_print_aggdata_t *pd)
{
	dtrace_actkind_t act = rec->dtrd_action;
	boolean_t packed = pd->dtpa_agghist || pd->dtpa_aggpack;
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	char fmt[512];
	char *s;

	if (packed && pd->dtpa_agghisthdr != agg->dtagd_varid)
		pd->dtpa_agghisthdr = agg->dtagd_varid;

	switch (act) {
	case DTRACEACT_STACK:
		return (dt_format_stack(dtp, addr, rec->dtrd_arg,
		    rec->dtrd_size / rec->dtrd_arg));

	case DTRACEACT_USTACK:
	case DTRACEACT_JSTACK:
		return (dt_format_ustack(dtp, addr, rec->dtrd_arg));

	case DTRACEACT_USYM:
	case DTRACEACT_UADDR:
		return (dt_format_usym(dtp, addr, act));

	case DTRACEACT_UMOD:
		return (dt_format_umod(dtp, addr));

	case DTRACEACT_SYM:
		return (dt_format_sym(dtp, addr));
	case DTRACEACT_MOD:
		return (dt_format_mod(dtp, addr));

	case DTRACEAGG_QUANTIZE:
		return (dt_format_quantize(dtp, addr, size, normal));

	case DTRACEAGG_LQUANTIZE:
		return (dt_format_lquantize(dtp, addr, size, normal));

	case DTRACEAGG_LLQUANTIZE:
		return (dt_format_llquantize(dtp, addr, size, normal));

	case DTRACEAGG_AVG:
		return (dt_format_average(dtp, addr, size, normal));

	case DTRACEAGG_STDDEV:
		return (dt_format_stddev(dtp, addr, size, normal));

	default:
		break;
	}

	switch (size) {
	case sizeof (uint64_t):
		snprintf(fmt, sizeof(fmt), "{:%s/%%lld}", pd->dtpa_keyname);
		xo_emit(fmt, (long long)*((uint64_t *)addr) / normal);
		break;
	case sizeof (uint32_t):
		snprintf(fmt, sizeof(fmt), "{:%s/%%d}", pd->dtpa_keyname);
		xo_emit(fmt, *((uint32_t *)addr) / (uint32_t)normal);
		break;
	case sizeof (uint16_t):
		snprintf(fmt, sizeof(fmt), "{:%s/%%d}", pd->dtpa_keyname);
		xo_emit(fmt, *((uint16_t *)addr) / (uint32_t)normal);
		break;
	case sizeof (uint8_t):
		snprintf(fmt, sizeof(fmt), "{:%s/%%d}", pd->dtpa_keyname);
		xo_emit(fmt, *((uint8_t *)addr) / (uint32_t)normal);
		break;
	default:
		s = dt_format_bytes_get(dtp, addr, size);
		if (s == NULL)
			return (-1);

		xo_emit("{:value/%s}", s);
		dt_free(dtp, s);
		break;
	}

	return (0);
}

static int
dt_print_datum(dtrace_hdl_t *dtp, FILE *fp, dtrace_recdesc_t *rec,
    caddr_t addr, size_t size, const dtrace_aggdata_t *aggdata,
    uint64_t normal, dt_print_aggdata_t *pd)
{
	int err, width;
	dtrace_actkind_t act = rec->dtrd_action;
	boolean_t packed = pd->dtpa_agghist || pd->dtpa_aggpack;
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;

	static struct {
		size_t size;
		int width;
		int packedwidth;
	} *fmt, fmttab[] = {
		{ sizeof (uint8_t),	3,	3 },
		{ sizeof (uint16_t),	5,	5 },
		{ sizeof (uint32_t),	8,	8 },
		{ sizeof (uint64_t),	16,	16 },
		{ 0,			-50,	16 }
	};

	if (packed && pd->dtpa_agghisthdr != agg->dtagd_varid) {
		dtrace_recdesc_t *r;

		width = 0;

		/*
		 * To print our quantization header for either an agghist or
		 * aggpack aggregation, we need to iterate through all of our
		 * of our records to determine their width.
		 */
		for (r = rec; !DTRACEACT_ISAGG(r->dtrd_action); r++) {
			for (fmt = fmttab; fmt->size &&
			    fmt->size != r->dtrd_size; fmt++)
				continue;

			width += fmt->packedwidth + 1;
		}

		if (pd->dtpa_agghist) {
			if (dt_print_quanthdr(dtp, fp, width) < 0)
				return (-1);
		} else {
			if (dt_print_quanthdr_packed(dtp, fp,
			    width, aggdata, r->dtrd_action) < 0)
				return (-1);
		}

		pd->dtpa_agghisthdr = agg->dtagd_varid;
	}

	if (pd->dtpa_agghist && DTRACEACT_ISAGG(act)) {
		char positives = aggdata->dtada_flags & DTRACE_A_HASPOSITIVES;
		char negatives = aggdata->dtada_flags & DTRACE_A_HASNEGATIVES;
		int64_t val;

		assert(act == DTRACEAGG_SUM || act == DTRACEAGG_COUNT);
		val = (long long)*((uint64_t *)addr);

		if (dt_printf(dtp, fp, " ") < 0)
			return (-1);

		return (dt_print_quantline(dtp, fp, val, normal,
		    aggdata->dtada_total, positives, negatives));
	}

	if (pd->dtpa_aggpack && DTRACEACT_ISAGG(act)) {
		switch (act) {
		case DTRACEAGG_QUANTIZE:
			return (dt_print_quantize_packed(dtp,
			    fp, addr, size, aggdata));
		case DTRACEAGG_LQUANTIZE:
			return (dt_print_lquantize_packed(dtp,
			    fp, addr, size, aggdata));
		default:
			break;
		}
	}

	switch (act) {
	case DTRACEACT_STACK:
		return (dt_print_stack(dtp, fp, NULL, addr,
		    rec->dtrd_arg, rec->dtrd_size / rec->dtrd_arg));

	case DTRACEACT_USTACK:
	case DTRACEACT_JSTACK:
		return (dt_print_ustack(dtp, fp, NULL, addr, rec->dtrd_arg));

	case DTRACEACT_USYM:
	case DTRACEACT_UADDR:
		return (dt_print_usym(dtp, fp, addr, act));

	case DTRACEACT_UMOD:
		return (dt_print_umod(dtp, fp, NULL, addr));

	case DTRACEACT_SYM:
		return (dt_print_sym(dtp, fp, NULL, addr));

	case DTRACEACT_MOD:
		return (dt_print_mod(dtp, fp, NULL, addr));

	case DTRACEAGG_QUANTIZE:
		return (dt_print_quantize(dtp, fp, addr, size, normal));

	case DTRACEAGG_LQUANTIZE:
		return (dt_print_lquantize(dtp, fp, addr, size, normal));

	case DTRACEAGG_LLQUANTIZE:
		return (dt_print_llquantize(dtp, fp, addr, size, normal));

	case DTRACEAGG_AVG:
		return (dt_print_average(dtp, fp, addr, size, normal));

	case DTRACEAGG_STDDEV:
		return (dt_print_stddev(dtp, fp, addr, size, normal));

	default:
		break;
	}

	for (fmt = fmttab; fmt->size && fmt->size != size; fmt++)
		continue;

	width = packed ? fmt->packedwidth : fmt->width;

	switch (size) {
	case sizeof (uint64_t):
		err = dt_printf(dtp, fp, " %*lld", width,
		    /* LINTED - alignment */
		    (long long)*((uint64_t *)addr) / normal);
		break;
	case sizeof (uint32_t):
		/* LINTED - alignment */
		err = dt_printf(dtp, fp, " %*d", width, *((uint32_t *)addr) /
		    (uint32_t)normal);
		break;
	case sizeof (uint16_t):
		/* LINTED - alignment */
		err = dt_printf(dtp, fp, " %*d", width, *((uint16_t *)addr) /
		    (uint32_t)normal);
		break;
	case sizeof (uint8_t):
		err = dt_printf(dtp, fp, " %*d", width, *((uint8_t *)addr) /
		    (uint32_t)normal);
		break;
	default:
		err = dt_print_bytes(dtp, fp, addr, size, width, 0, 0);
		break;
	}

	return (err);
}

int
dt_format_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
{
	int i, aggact = 0;
	dt_print_aggdata_t *pd = arg;
	const dtrace_aggdata_t *aggdata = aggsdata[0];
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	dtrace_hdl_t *dtp = pd->dtpa_dtp;
	dtrace_recdesc_t *rec;
	dtrace_actkind_t act;
	caddr_t addr;
	size_t size;

	if (pd->dtpa_aggname == NULL)
		pd->dtpa_aggname = agg->dtagd_name;

	xo_open_instance("aggregation-data");
	strcpy(pd->dtpa_keyname, "value");
	xo_open_list("keys");

	/*
	 * Iterate over each record description in the key, printing the traced
	 * data, skipping the first datum (the tuple member created by the
	 * compiler).
	 */
	for (i = 1; i < agg->dtagd_nrecs; i++) {
		rec = &agg->dtagd_rec[i];
		act = rec->dtrd_action;
		addr = aggdata->dtada_data + rec->dtrd_offset;
		size = rec->dtrd_size;

		if (DTRACEACT_ISAGG(act)) {
			aggact = i;
			break;
		}

		xo_open_instance("keys");
		if (dt_format_datum(dtp, rec, addr,
		    size, aggdata, 1, pd) < 0) {
			xo_close_instance("keys");
			xo_close_instance("aggregation-data");
			return (-1);
		}
		xo_close_instance("keys");

		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
		    DTRACE_BUFDATA_AGGKEY) < 0) {
			xo_close_instance("aggregation-data");
			return (-1);
		}
	}
	xo_close_list("keys");

	assert(aggact != 0);

	for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
		uint64_t normal;

		aggdata = aggsdata[i];
		agg = aggdata->dtada_desc;
		rec = &agg->dtagd_rec[aggact];
		act = rec->dtrd_action;
		addr = aggdata->dtada_data + rec->dtrd_offset;
		size = rec->dtrd_size;

		assert(DTRACEACT_ISAGG(act));

		switch (act) {
		case DTRACEAGG_MIN:
			strcpy(pd->dtpa_keyname, "min");
			break;
		case DTRACEAGG_MAX:
			strcpy(pd->dtpa_keyname, "max");
			break;
		case DTRACEAGG_COUNT:
			strcpy(pd->dtpa_keyname, "count");
			break;
		case DTRACEAGG_SUM:
			strcpy(pd->dtpa_keyname, "sum");
			break;
		default:
			strcpy(pd->dtpa_keyname, "UNKNOWN");
			break;
		}

		normal = aggdata->dtada_normal;

		if (dt_format_datum(dtp, rec, addr, size,
		    aggdata, normal, pd) < 0) {
			xo_close_instance("aggregation-data");
			return (-1);
		}

		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
		    DTRACE_BUFDATA_AGGVAL) < 0) {
			xo_close_instance("aggregation-data");
			return (-1);
		}

		if (!pd->dtpa_allunprint)
			agg->dtagd_flags |= DTRACE_AGD_PRINTED;
	}

	if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
	    DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0) {
		xo_close_instance("aggregation-data");
		return (-1);
	}

	xo_close_instance("aggregation-data");
	return (0);
}

int
dt_print_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
{
	int i, aggact = 0;
	dt_print_aggdata_t *pd = arg;
	const dtrace_aggdata_t *aggdata = aggsdata[0];
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	FILE *fp = pd->dtpa_fp;
	dtrace_hdl_t *dtp = pd->dtpa_dtp;
	dtrace_recdesc_t *rec;
	dtrace_actkind_t act;
	caddr_t addr;
	size_t size;

	pd->dtpa_agghist = (aggdata->dtada_flags & DTRACE_A_TOTAL);
	pd->dtpa_aggpack = (aggdata->dtada_flags & DTRACE_A_MINMAXBIN);

	/*
	 * Iterate over each record description in the key, printing the traced
	 * data, skipping the first datum (the tuple member created by the
	 * compiler).
	 */
	for (i = 1; i < agg->dtagd_nrecs; i++) {
		rec = &agg->dtagd_rec[i];
		act = rec->dtrd_action;
		addr = aggdata->dtada_data + rec->dtrd_offset;
		size = rec->dtrd_size;

		if (DTRACEACT_ISAGG(act)) {
			aggact = i;
			break;
		}

		if (dt_print_datum(dtp, fp, rec, addr,
		    size, aggdata, 1, pd) < 0)
			return (-1);

		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
		    DTRACE_BUFDATA_AGGKEY) < 0)
			return (-1);
	}

	assert(aggact != 0);

	for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
		uint64_t normal;

		aggdata = aggsdata[i];
		agg = aggdata->dtada_desc;
		rec = &agg->dtagd_rec[aggact];
		act = rec->dtrd_action;
		addr = aggdata->dtada_data + rec->dtrd_offset;
		size = rec->dtrd_size;

		assert(DTRACEACT_ISAGG(act));
		normal = aggdata->dtada_normal;

		if (dt_print_datum(dtp, fp, rec, addr,
		    size, aggdata, normal, pd) < 0)
			return (-1);

		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
		    DTRACE_BUFDATA_AGGVAL) < 0)
			return (-1);

		if (!pd->dtpa_allunprint)
			agg->dtagd_flags |= DTRACE_AGD_PRINTED;
	}

	if (!pd->dtpa_agghist && !pd->dtpa_aggpack) {
		if (dt_printf(dtp, fp, "\n") < 0)
			return (-1);
	}

	if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
	    DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0)
		return (-1);

	return (0);
}

int
dt_format_agg(const dtrace_aggdata_t *aggdata, void *arg)
{
	dt_print_aggdata_t *pd = arg;
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	dtrace_aggvarid_t aggvarid = pd->dtpa_id;

	if (pd->dtpa_allunprint) {
		if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
			return (0);
	} else {
		/*
		 * If we're not printing all unprinted aggregations, then the
		 * aggregation variable ID denotes a specific aggregation
		 * variable that we should print -- skip any other aggregations
		 * that we encounter.
		 */
		if (agg->dtagd_nrecs == 0)
			return (0);

		if (aggvarid != agg->dtagd_varid)
			return (0);
	}

	return (dt_format_aggs(&aggdata, 1, arg));
}

int
dt_print_agg(const dtrace_aggdata_t *aggdata, void *arg)
{
	dt_print_aggdata_t *pd = arg;
	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
	dtrace_aggvarid_t aggvarid = pd->dtpa_id;

	if (pd->dtpa_allunprint) {
		if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
			return (0);
	} else {
		/*
		 * If we're not printing all unprinted aggregations, then the
		 * aggregation variable ID denotes a specific aggregation
		 * variable that we should print -- skip any other aggregations
		 * that we encounter.
		 */
		if (agg->dtagd_nrecs == 0)
			return (0);

		if (aggvarid != agg->dtagd_varid)
			return (0);
	}

	return (dt_print_aggs(&aggdata, 1, arg));
}

int
dt_setopt(dtrace_hdl_t *dtp, const dtrace_probedata_t *data,
    const char *option, const char *value)
{
	int len, rval;
	char *msg;
	const char *errstr;
	dtrace_setoptdata_t optdata;

	bzero(&optdata, sizeof (optdata));
	(void) dtrace_getopt(dtp, option, &optdata.dtsda_oldval);

	if (dtrace_setopt(dtp, option, value) == 0) {
		(void) dtrace_getopt(dtp, option, &optdata.dtsda_newval);
		optdata.dtsda_probe = data;
		optdata.dtsda_option = option;
		optdata.dtsda_handle = dtp;

		if ((rval = dt_handle_setopt(dtp, &optdata)) != 0)
			return (rval);

		return (0);
	}

	errstr = dtrace_errmsg(dtp, dtrace_errno(dtp));
	len = strlen(option) + strlen(value) + strlen(errstr) + 80;
	msg = alloca(len);

	(void) snprintf(msg, len, "couldn't set option \"%s\" to \"%s\": %s\n",
	    option, value, errstr);

	if ((rval = dt_handle_liberr(dtp, data, msg)) == 0)
		return (0);

	return (rval);
}

/*
 * Helper functions to help maintain style(9) in dt_consume_cpu().
 */
static int
dt_oformat_agg_sorted(dtrace_hdl_t *dtp, dtrace_aggregate_f *func,
    dt_print_aggdata_t *pd)
{
	int r;

	r = dtrace_aggregate_walk_sorted(dtp, dt_format_agg, pd);
	if (r < 0) {
		xo_close_list("aggregation-data");
		xo_emit("{:aggregation-name/%s}", pd->dtpa_aggname);
		xo_close_instance("output");
	}

	return (r);
}

static void
dt_oformat_agg_name(dt_print_aggdata_t *pd)
{

	xo_close_list("aggregation-data");
	xo_emit("{:aggregation-name/%s}", pd->dtpa_aggname);
}

static int
dt_consume_cpu(dtrace_hdl_t *dtp, FILE *fp, int cpu,
    dtrace_bufdesc_t *buf, boolean_t just_one,
    dtrace_consume_probe_f *efunc, dtrace_consume_rec_f *rfunc, void *arg)
{
	dtrace_epid_t id;
	size_t offs;
	int flow = (dtp->dt_options[DTRACEOPT_FLOWINDENT] != DTRACEOPT_UNSET);
	int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
	int rval, i, n;
	uint64_t tracememsize = 0;
	dtrace_probedata_t data;
	uint64_t drops;
	size_t skip_format;

	bzero(&data, sizeof (data));
	data.dtpda_handle = dtp;
	data.dtpda_cpu = cpu;
	data.dtpda_flow = dtp->dt_flow;
	data.dtpda_indent = dtp->dt_indent;
	data.dtpda_prefix = dtp->dt_prefix;

	for (offs = buf->dtbd_oldest; offs < buf->dtbd_size; ) {
		dtrace_eprobedesc_t *epd;

		/*
		 * We're guaranteed to have an ID.
		 */
		id = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);

		if (id == DTRACE_EPIDNONE) {
			/*
			 * This is filler to assure proper alignment of the
			 * next record; we simply ignore it.
			 */
			offs += sizeof (id);
			continue;
		}

		if ((rval = dt_epid_lookup(dtp, id, &data.dtpda_edesc,
		    &data.dtpda_pdesc)) != 0)
			return (rval);

		epd = data.dtpda_edesc;
		data.dtpda_data = buf->dtbd_data + offs;
		data.dtpda_timestamp = DTRACE_RECORD_LOAD_TIMESTAMP(
		    (struct dtrace_rechdr *)data.dtpda_data);

		if (data.dtpda_edesc->dtepd_uarg != DT_ECB_DEFAULT) {
			rval = dt_handle(dtp, &data);

			if (rval == DTRACE_CONSUME_NEXT)
				goto nextepid;

			if (rval == DTRACE_CONSUME_ERROR)
				return (-1);
		}

		if (flow)
			(void) dt_flowindent(dtp, &data, dtp->dt_last_epid,
			    buf, offs);

		if (dtp->dt_oformat)
			xo_open_instance("probes");
		rval = (*efunc)(&data, arg);

		if (flow) {
			if (data.dtpda_flow == DTRACEFLOW_ENTRY)
				data.dtpda_indent += 2;
		}

		if (rval == DTRACE_CONSUME_NEXT)
			goto nextepid;

		if (rval == DTRACE_CONSUME_ABORT)
			return (dt_set_errno(dtp, EDT_DIRABORT));

		if (rval != DTRACE_CONSUME_THIS)
			return (dt_set_errno(dtp, EDT_BADRVAL));

		skip_format = 0;
		if (dtp->dt_oformat)
			xo_open_list("output");
		for (i = 0; i < epd->dtepd_nrecs; i++) {
			caddr_t addr;
			dtrace_recdesc_t *rec = &epd->dtepd_rec[i];
			dtrace_actkind_t act = rec->dtrd_action;

			if (skip_format > 0)
				skip_format--;

			data.dtpda_data = buf->dtbd_data + offs +
			    rec->dtrd_offset;
			addr = data.dtpda_data;

			if (act == DTRACEACT_LIBACT) {
				uint64_t arg = rec->dtrd_arg;
				dtrace_aggvarid_t id;

				switch (arg) {
				case DT_ACT_CLEAR:
					/* LINTED - alignment */
					id = *((dtrace_aggvarid_t *)addr);
					(void) dtrace_aggregate_walk(dtp,
					    dt_clear_agg, &id);
					continue;

				case DT_ACT_DENORMALIZE:
					/* LINTED - alignment */
					id = *((dtrace_aggvarid_t *)addr);
					(void) dtrace_aggregate_walk(dtp,
					    dt_denormalize_agg, &id);
					continue;

				case DT_ACT_FTRUNCATE:
					if (fp == NULL)
						continue;

					(void) fflush(fp);
					(void) ftruncate(fileno(fp), 0);
					(void) fseeko(fp, 0, SEEK_SET);
					continue;

				case DT_ACT_NORMALIZE:
					if (i == epd->dtepd_nrecs - 1)
						return (dt_set_errno(dtp,
						    EDT_BADNORMAL));

					if (dt_normalize(dtp,
					    buf->dtbd_data + offs, rec) != 0)
						return (-1);

					i++;
					continue;

				case DT_ACT_SETOPT: {
					uint64_t *opts = dtp->dt_options;
					dtrace_recdesc_t *valrec;
					uint32_t valsize;
					caddr_t val;
					int rv;

					if (i == epd->dtepd_nrecs - 1) {
						return (dt_set_errno(dtp,
						    EDT_BADSETOPT));
					}

					valrec = &epd->dtepd_rec[++i];
					valsize = valrec->dtrd_size;

					if (valrec->dtrd_action != act ||
					    valrec->dtrd_arg != arg) {
						return (dt_set_errno(dtp,
						    EDT_BADSETOPT));
					}

					if (valsize > sizeof (uint64_t)) {
						val = buf->dtbd_data + offs +
						    valrec->dtrd_offset;
					} else {
						val = "1";
					}

					rv = dt_setopt(dtp, &data, addr, val);

					if (rv != 0)
						return (-1);

					flow = (opts[DTRACEOPT_FLOWINDENT] !=
					    DTRACEOPT_UNSET);
					quiet = (opts[DTRACEOPT_QUIET] !=
					    DTRACEOPT_UNSET);

					continue;
				}

				case DT_ACT_TRUNC:
					if (i == epd->dtepd_nrecs - 1)
						return (dt_set_errno(dtp,
						    EDT_BADTRUNC));

					if (dt_trunc(dtp,
					    buf->dtbd_data + offs, rec) != 0)
						return (-1);

					i++;
					continue;

				default:
					continue;
				}
			}

			if (act == DTRACEACT_TRACEMEM_DYNSIZE &&
			    rec->dtrd_size == sizeof (uint64_t)) {
			    	/* LINTED - alignment */
				tracememsize = *((unsigned long long *)addr);
				continue;
			}

			rval = (*rfunc)(&data, rec, arg);

			if (rval == DTRACE_CONSUME_NEXT)
				continue;

			if (rval == DTRACE_CONSUME_ABORT)
				return (dt_set_errno(dtp, EDT_DIRABORT));

			if (rval != DTRACE_CONSUME_THIS)
				return (dt_set_errno(dtp, EDT_BADRVAL));

			if (dtp->dt_oformat && rec->dtrd_size > 0)
				xo_open_instance("output");
			if (act == DTRACEACT_STACK) {
				int depth = rec->dtrd_arg;

				if (dtp->dt_oformat) {
					if (dt_format_stack(dtp, addr, depth,
					    rec->dtrd_size / depth) < 0) {
						xo_close_instance("output");
						return (-1);
					}
				} else {
					if (dt_print_stack(dtp,
					    fp, NULL, addr, depth,
					    rec->dtrd_size / depth) < 0)
					return (-1);
				}
				goto nextrec;
			}

			if (act == DTRACEACT_USTACK ||
			    act == DTRACEACT_JSTACK) {
				if (dtp->dt_oformat) {
					if (dt_format_ustack(dtp, addr,
					    rec->dtrd_arg) < 0) {
						xo_close_instance("output");
						return (-1);
					}
				} else {
					if (dt_print_ustack(dtp, fp, NULL,
					    addr, rec->dtrd_arg) < 0)
						return (-1);
				}
				goto nextrec;
			}

			if (act == DTRACEACT_SYM) {
				if (dtp->dt_oformat) {
					if (dt_format_sym(dtp, addr) < 0) {
						xo_close_instance("output");
						return (-1);
					}
				} else {
					if (dt_print_sym(dtp, fp, NULL, addr) < 0)
						return (-1);
				}
				goto nextrec;
			}

			if (act == DTRACEACT_MOD) {
				if (dtp->dt_oformat) {
					if (dt_format_mod(dtp, addr) < 0) {
						xo_close_instance("output");
						return (-1);
					}
				} else {
					if (dt_print_mod(dtp, fp, NULL, addr) < 0)
						return (-1);
				}
				goto nextrec;
			}

			if (act == DTRACEACT_USYM || act == DTRACEACT_UADDR) {
				if (dtp->dt_oformat) {
					if (dt_format_usym(dtp, addr, act) < 0) {
						xo_close_instance("output");
						return (-1);
					}
				} else {
					if (dt_print_usym(dtp, fp, addr, act) < 0)
						return (-1);
				}
				goto nextrec;
			}

			if (act == DTRACEACT_UMOD) {
				if (dtp->dt_oformat) {
					if (dt_format_umod(dtp, addr) < 0) {
						xo_close_instance("output");
						return (-1);
					}
				} else {
					if (dt_print_umod(dtp, fp, NULL, addr) < 0)
						return (-1);
				}
				goto nextrec;
			}

			if (act == DTRACEACT_PRINTM) {
				if (dtp->dt_oformat) {
					if (dt_format_memory(dtp, addr) < 0) {
						xo_close_instance("output");
						return (-1);
					}
				} else {
					if (dt_print_memory(dtp, fp, addr) < 0)
						return (-1);
				}
				goto nextrec;
			}

			if (dtp->dt_oformat == DTRACE_OFORMAT_TEXT &&
			    DTRACEACT_ISPRINTFLIKE(act)) {
				void *fmtdata;
				int (*func)(dtrace_hdl_t *, FILE *, void *,
				    const dtrace_probedata_t *,
				    const dtrace_recdesc_t *, uint_t,
				    const void *buf, size_t);

				if ((fmtdata = dt_format_lookup(dtp,
				    rec->dtrd_format)) == NULL)
					goto nofmt;

				switch (act) {
				case DTRACEACT_PRINTF:
					func = dtrace_fprintf;
					break;
				case DTRACEACT_PRINTA:
					func = dtrace_fprinta;
					break;
				case DTRACEACT_SYSTEM:
					func = dtrace_system;
					break;
				case DTRACEACT_FREOPEN:
					func = dtrace_freopen;
					break;
				}

				n = (*func)(dtp, fp, fmtdata, &data,
				    rec, epd->dtepd_nrecs - i,
				    (uchar_t *)buf->dtbd_data + offs,
				    buf->dtbd_size - offs);

				if (n < 0)
					return (-1); /* errno is set for us */

				if (n > 0)
					i += n - 1;
				goto nextrec;
			}

			/*
			 * We don't care about a formatted printa, system or
			 * freopen for oformat.
			 */
			if (dtp->dt_oformat && act == DTRACEACT_PRINTF &&
			    skip_format == 0) {
				void *fmtdata;
				if ((fmtdata = dt_format_lookup(dtp,
				    rec->dtrd_format)) == NULL)
					goto nofmt;

				n = dtrace_sprintf(dtp, fp, fmtdata, rec,
				    epd->dtepd_nrecs - i,
				    (uchar_t *)buf->dtbd_data + offs,
				    buf->dtbd_size - offs);

				if (n < 0) {
					xo_close_instance("output");
					return (-1); /* errno is set for us */
				}

				xo_emit("{:message/%s}", dtp->dt_sprintf_buf);
				skip_format += n;

				/*
				 * We want the "message" object to be its own
				 * thing, but we still want to process the
				 * current DIFEXPR in case there is a value
				 * attached to it. If there is, we need to
				 * re-open a new output instance, as otherwise
				 * the message ends up bundled with the first
				 * value.
				 *
				 * XXX: There is an edge case where a
				 * printf("hello"); will produce a DIFO that
				 * returns 0 attached to it and we have no good
				 * way to determine if this 0 value is because
				 * there's no real data attached to the printf
				 * as an argument, or it's because the argument
				 * actually returns 0.
				 */
				if (skip_format == 0)
					goto nextrec;

				xo_close_instance("output");
				xo_open_instance("output");
			}

			/*
			 * If this is a DIF expression, and the record has a
			 * format set, this indicates we have a CTF type name
			 * associated with the data and we should try to print
			 * it out by type.
			 */
			if (act == DTRACEACT_DIFEXPR) {
				const char *strdata = dt_strdata_lookup(dtp,
				    rec->dtrd_format);
				if (strdata != NULL) {
					if (dtp->dt_oformat)
						n = dtrace_format_print(dtp, fp,
						    strdata, addr,
						    rec->dtrd_size);
					else
						n = dtrace_print(dtp, fp,
						    strdata, addr,
						    rec->dtrd_size);

					/*
					 * dtrace_print() will return -1 on
					 * error, or return the number of bytes
					 * consumed.  It will return 0 if the
					 * type couldn't be determined, and we
					 * should fall through to the normal
					 * trace method.
					 */
					if (n < 0) {
						if (dtp->dt_oformat)
							xo_close_instance(
							    "output");
						return (-1);
					}

					if (n > 0)
						goto nextrec;
				}
			}

nofmt:
			if (act == DTRACEACT_PRINTA) {
				dt_print_aggdata_t pd;
				dtrace_aggvarid_t *aggvars;
				int j, naggvars = 0;
				size_t size = ((epd->dtepd_nrecs - i) *
				    sizeof (dtrace_aggvarid_t));

				if ((aggvars = dt_alloc(dtp, size)) == NULL) {
					if (dtp->dt_oformat)
						xo_close_instance("output");
					return (-1);
				}

				/*
				 * This might be a printa() with multiple
				 * aggregation variables.  We need to scan
				 * forward through the records until we find
				 * a record from a different statement.
				 */
				for (j = i; j < epd->dtepd_nrecs; j++) {
					dtrace_recdesc_t *nrec;
					caddr_t naddr;

					nrec = &epd->dtepd_rec[j];

					if (nrec->dtrd_uarg != rec->dtrd_uarg)
						break;

					if (nrec->dtrd_action != act) {
						if (dtp->dt_oformat)
							xo_close_instance(
							    "output");
						return (dt_set_errno(dtp,
						    EDT_BADAGG));
					}

					naddr = buf->dtbd_data + offs +
					    nrec->dtrd_offset;

					aggvars[naggvars++] =
					    /* LINTED - alignment */
					    *((dtrace_aggvarid_t *)naddr);
				}

				i = j - 1;
				bzero(&pd, sizeof (pd));
				pd.dtpa_dtp = dtp;
				pd.dtpa_fp = fp;

				assert(naggvars >= 1);

				if (dtp->dt_oformat)
					xo_open_list("aggregation-data");
				if (naggvars == 1) {
					pd.dtpa_id = aggvars[0];
					dt_free(dtp, aggvars);

					if (dtp->dt_oformat) {
						n = dt_oformat_agg_sorted(dtp,
						    dt_format_agg, &pd);
						if (n < 0)
							return (-1);
					} else {
						if (dt_printf(dtp, fp, "\n") < 0 ||
						    dtrace_aggregate_walk_sorted(dtp,
						    dt_print_agg, &pd) < 0)
							return (-1);
					}

					if (dtp->dt_oformat)
						dt_oformat_agg_name(&pd);
					goto nextrec;
				}

				if (dtp->dt_oformat) {
					if (dtrace_aggregate_walk_joined(dtp,
					    aggvars, naggvars,
					    dt_format_aggs, &pd) < 0) {
						dt_oformat_agg_name(&pd);
						xo_close_instance("output");
						dt_free(dtp, aggvars);
						return (-1);
					}
				} else {
					if (dt_printf(dtp, fp, "\n") < 0 ||
					    dtrace_aggregate_walk_joined(dtp,
					    aggvars, naggvars,
					    dt_print_aggs, &pd) < 0) {
						dt_free(dtp, aggvars);
						return (-1);
					}
				}

				if (dtp->dt_oformat)
					dt_oformat_agg_name(&pd);
				dt_free(dtp, aggvars);
				goto nextrec;
			}

			if (act == DTRACEACT_TRACEMEM) {
				if (tracememsize == 0 ||
				    tracememsize > rec->dtrd_size) {
					tracememsize = rec->dtrd_size;
				}

				if (dtp->dt_oformat) {
					char *s;

					s = dt_format_bytes_get(dtp, addr,
					    tracememsize);
					n = xo_emit("{:tracemem/%s}", s);
					dt_free(dtp, s);
				} else {
					n = dt_print_bytes(dtp, fp, addr,
					    tracememsize, -33, quiet, 1);
				}

				tracememsize = 0;

				if (n < 0)
					return (-1);

				goto nextrec;
			}

			switch (rec->dtrd_size) {
			case sizeof (uint64_t):
				if (dtp->dt_oformat) {
					xo_emit("{:value/%lld}",
					    *((unsigned long long *)addr));
					n = 0;
				} else
					n = dt_printf(dtp, fp,
					    quiet ? "%lld" : " %16lld",
					    /* LINTED - alignment */
					    *((unsigned long long *)addr));
				break;
			case sizeof (uint32_t):
				if (dtp->dt_oformat) {
					xo_emit("{:value/%d}",
					    *((uint32_t *)addr));
					n = 0;
				} else
					n = dt_printf(dtp, fp,
					    quiet ? "%d" : " %8d",
					    /* LINTED - alignment */
					    *((uint32_t *)addr));
				break;
			case sizeof (uint16_t):
				if (dtp->dt_oformat) {
					xo_emit("{:value/%d}",
					    *((uint16_t *)addr));
					n = 0;
				} else
					n = dt_printf(dtp, fp,
					    quiet ? "%d" : " %5d",
					    /* LINTED - alignment */
					    *((uint16_t *)addr));
				break;
			case sizeof (uint8_t):
				if (dtp->dt_oformat) {
					xo_emit("{:value/%d}",
					    *((uint8_t *)addr));
					n = 0;
				} else
					n = dt_printf(dtp, fp,
					    quiet ? "%d" : " %3d",
					    *((uint8_t *)addr));
				break;
			default:
				if (dtp->dt_oformat && rec->dtrd_size > 0) {
					char *s;

					s = dt_format_bytes_get(dtp, addr,
					    rec->dtrd_size);
					xo_emit("{:value/%s}", s);
					dt_free(dtp, s);
					n = 0;
				} else {
					n = dt_print_bytes(dtp, fp, addr,
					    rec->dtrd_size, -33, quiet, 0);
				}
				break;
			}

			if (dtp->dt_oformat && rec->dtrd_size > 0)
				xo_close_instance("output");

			if (n < 0)
				return (-1); /* errno is set for us */

nextrec:
			if (dt_buffered_flush(dtp, &data, rec, NULL, 0) < 0)
				return (-1); /* errno is set for us */
		}

		/*
		 * Call the record callback with a NULL record to indicate
		 * that we're done processing this EPID.
		 */
		rval = (*rfunc)(&data, NULL, arg);
nextepid:
		offs += epd->dtepd_size;
		dtp->dt_last_epid = id;

		if (dtp->dt_oformat) {
			xo_close_list("output");
			xo_close_instance("probes");
			xo_flush();
		}
		if (just_one) {
			buf->dtbd_oldest = offs;
			break;
		}
	}

	dtp->dt_flow = data.dtpda_flow;
	dtp->dt_indent = data.dtpda_indent;
	dtp->dt_prefix = data.dtpda_prefix;

	if ((drops = buf->dtbd_drops) == 0)
		return (0);

	/*
	 * Explicitly zero the drops to prevent us from processing them again.
	 */
	buf->dtbd_drops = 0;

	if (dtp->dt_oformat) {
		xo_open_instance("probes");
		dt_oformat_drop(dtp, cpu);
	}
	rval = dt_handle_cpudrop(dtp, cpu, DTRACEDROP_PRINCIPAL, drops);
	if (dtp->dt_oformat)
		xo_close_instance("probes");

	return (rval);
}

/*
 * Reduce memory usage by shrinking the buffer if it's no more than half full.
 * Note, we need to preserve the alignment of the data at dtbd_oldest, which is
 * only 4-byte aligned.
 */
static void
dt_realloc_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf, int cursize)
{
	uint64_t used = buf->dtbd_size - buf->dtbd_oldest;
	if (used < cursize / 2) {
		int misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
		char *newdata = dt_alloc(dtp, used + misalign);
		if (newdata == NULL)
			return;
		bzero(newdata, misalign);
		bcopy(buf->dtbd_data + buf->dtbd_oldest,
		    newdata + misalign, used);
		dt_free(dtp, buf->dtbd_data);
		buf->dtbd_oldest = misalign;
		buf->dtbd_size = used + misalign;
		buf->dtbd_data = newdata;
	}
}

/*
 * If the ring buffer has wrapped, the data is not in order.  Rearrange it
 * so that it is.  Note, we need to preserve the alignment of the data at
 * dtbd_oldest, which is only 4-byte aligned.
 */
static int
dt_unring_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
{
	int misalign;
	char *newdata, *ndp;

	if (buf->dtbd_oldest == 0)
		return (0);

	misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
	newdata = ndp = dt_alloc(dtp, buf->dtbd_size + misalign);

	if (newdata == NULL)
		return (-1);

	assert(0 == (buf->dtbd_size & (sizeof (uint64_t) - 1)));

	bzero(ndp, misalign);
	ndp += misalign;

	bcopy(buf->dtbd_data + buf->dtbd_oldest, ndp,
	    buf->dtbd_size - buf->dtbd_oldest);
	ndp += buf->dtbd_size - buf->dtbd_oldest;

	bcopy(buf->dtbd_data, ndp, buf->dtbd_oldest);

	dt_free(dtp, buf->dtbd_data);
	buf->dtbd_oldest = misalign;
	buf->dtbd_data = newdata;
	buf->dtbd_size += misalign;

	return (0);
}

static void
dt_put_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
{
	dt_free(dtp, buf->dtbd_data);
	dt_free(dtp, buf);
}

/*
 * Returns 0 on success, in which case *cbp will be filled in if we retrieved
 * data, or NULL if there is no data for this CPU.
 * Returns -1 on failure and sets dt_errno.
 */
static int
dt_get_buf(dtrace_hdl_t *dtp, int cpu, dtrace_bufdesc_t **bufp)
{
	dtrace_optval_t size;
	dtrace_bufdesc_t *buf = dt_zalloc(dtp, sizeof (*buf));
	int error, rval;

	if (buf == NULL)
		return (-1);

	(void) dtrace_getopt(dtp, "bufsize", &size);
	buf->dtbd_data = dt_alloc(dtp, size);
	if (buf->dtbd_data == NULL) {
		dt_free(dtp, buf);
		return (-1);
	}
	buf->dtbd_size = size;
	buf->dtbd_cpu = cpu;

#ifdef illumos
	if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
#else
	if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
#endif
		/*
		 * If we failed with ENOENT, it may be because the
		 * CPU was unconfigured -- this is okay.  Any other
		 * error, however, is unexpected.
		 */
		if (errno == ENOENT) {
			*bufp = NULL;
			rval = 0;
		} else
			rval = dt_set_errno(dtp, errno);

		dt_put_buf(dtp, buf);
		return (rval);
	}

	error = dt_unring_buf(dtp, buf);
	if (error != 0) {
		dt_put_buf(dtp, buf);
		return (error);
	}
	dt_realloc_buf(dtp, buf, size);

	*bufp = buf;
	return (0);
}

typedef struct dt_begin {
	dtrace_consume_probe_f *dtbgn_probefunc;
	dtrace_consume_rec_f *dtbgn_recfunc;
	void *dtbgn_arg;
	dtrace_handle_err_f *dtbgn_errhdlr;
	void *dtbgn_errarg;
	int dtbgn_beginonly;
} dt_begin_t;

static int
dt_consume_begin_probe(const dtrace_probedata_t *data, void *arg)
{
	dt_begin_t *begin = arg;
	dtrace_probedesc_t *pd = data->dtpda_pdesc;

	int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
	int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);

	if (begin->dtbgn_beginonly) {
		if (!(r1 && r2))
			return (DTRACE_CONSUME_NEXT);
	} else {
		if (r1 && r2)
			return (DTRACE_CONSUME_NEXT);
	}

	/*
	 * We have a record that we're interested in.  Now call the underlying
	 * probe function...
	 */
	return (begin->dtbgn_probefunc(data, begin->dtbgn_arg));
}

static int
dt_consume_begin_record(const dtrace_probedata_t *data,
    const dtrace_recdesc_t *rec, void *arg)
{
	dt_begin_t *begin = arg;

	return (begin->dtbgn_recfunc(data, rec, begin->dtbgn_arg));
}

static int
dt_consume_begin_error(const dtrace_errdata_t *data, void *arg)
{
	dt_begin_t *begin = (dt_begin_t *)arg;
	dtrace_probedesc_t *pd = data->dteda_pdesc;

	int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
	int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);

	if (begin->dtbgn_beginonly) {
		if (!(r1 && r2))
			return (DTRACE_HANDLE_OK);
	} else {
		if (r1 && r2)
			return (DTRACE_HANDLE_OK);
	}

	return (begin->dtbgn_errhdlr(data, begin->dtbgn_errarg));
}

static int
dt_consume_begin(dtrace_hdl_t *dtp, FILE *fp,
    dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
{
	/*
	 * There's this idea that the BEGIN probe should be processed before
	 * everything else, and that the END probe should be processed after
	 * anything else.  In the common case, this is pretty easy to deal
	 * with.  However, a situation may arise where the BEGIN enabling and
	 * END enabling are on the same CPU, and some enabling in the middle
	 * occurred on a different CPU.  To deal with this (blech!) we need to
	 * consume the BEGIN buffer up until the end of the BEGIN probe, and
	 * then set it aside.  We will then process every other CPU, and then
	 * we'll return to the BEGIN CPU and process the rest of the data
	 * (which will inevitably include the END probe, if any).  Making this
	 * even more complicated (!) is the library's ERROR enabling.  Because
	 * this enabling is processed before we even get into the consume call
	 * back, any ERROR firing would result in the library's ERROR enabling
	 * being processed twice -- once in our first pass (for BEGIN probes),
	 * and again in our second pass (for everything but BEGIN probes).  To
	 * deal with this, we interpose on the ERROR handler to assure that we
	 * only process ERROR enablings induced by BEGIN enablings in the
	 * first pass, and that we only process ERROR enablings _not_ induced
	 * by BEGIN enablings in the second pass.
	 */

	dt_begin_t begin;
	processorid_t cpu = dtp->dt_beganon;
	int rval, i;
	static int max_ncpus;
	dtrace_bufdesc_t *buf;

	dtp->dt_beganon = -1;

	if (dt_get_buf(dtp, cpu, &buf) != 0)
		return (-1);
	if (buf == NULL)
		return (0);

	if (!dtp->dt_stopped || buf->dtbd_cpu != dtp->dt_endedon) {
		/*
		 * This is the simple case.  We're either not stopped, or if
		 * we are, we actually processed any END probes on another
		 * CPU.  We can simply consume this buffer and return.
		 */
		rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
		    pf, rf, arg);
		dt_put_buf(dtp, buf);
		return (rval);
	}

	begin.dtbgn_probefunc = pf;
	begin.dtbgn_recfunc = rf;
	begin.dtbgn_arg = arg;
	begin.dtbgn_beginonly = 1;

	/*
	 * We need to interpose on the ERROR handler to be sure that we
	 * only process ERRORs induced by BEGIN.
	 */
	begin.dtbgn_errhdlr = dtp->dt_errhdlr;
	begin.dtbgn_errarg = dtp->dt_errarg;
	dtp->dt_errhdlr = dt_consume_begin_error;
	dtp->dt_errarg = &begin;

	rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
	    dt_consume_begin_probe, dt_consume_begin_record, &begin);

	dtp->dt_errhdlr = begin.dtbgn_errhdlr;
	dtp->dt_errarg = begin.dtbgn_errarg;

	if (rval != 0) {
		dt_put_buf(dtp, buf);
		return (rval);
	}

	if (max_ncpus == 0 && (max_ncpus = dt_cpu_maxid(dtp) + 1) <= 0)
		return (-1);

	for (i = 0; i < max_ncpus; i++) {
		dtrace_bufdesc_t *nbuf;
		if (i == cpu)
			continue;

		if (dt_get_buf(dtp, i, &nbuf) != 0) {
			dt_put_buf(dtp, buf);
			return (-1);
		}
		if (nbuf == NULL)
			continue;

		rval = dt_consume_cpu(dtp, fp, i, nbuf, B_FALSE,
		    pf, rf, arg);
		dt_put_buf(dtp, nbuf);
		if (rval != 0) {
			dt_put_buf(dtp, buf);
			return (rval);
		}
	}

	/*
	 * Okay -- we're done with the other buffers.  Now we want to
	 * reconsume the first buffer -- but this time we're looking for
	 * everything _but_ BEGIN.  And of course, in order to only consume
	 * those ERRORs _not_ associated with BEGIN, we need to reinstall our
	 * ERROR interposition function...
	 */
	begin.dtbgn_beginonly = 0;

	assert(begin.dtbgn_errhdlr == dtp->dt_errhdlr);
	assert(begin.dtbgn_errarg == dtp->dt_errarg);
	dtp->dt_errhdlr = dt_consume_begin_error;
	dtp->dt_errarg = &begin;

	rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
	    dt_consume_begin_probe, dt_consume_begin_record, &begin);

	dtp->dt_errhdlr = begin.dtbgn_errhdlr;
	dtp->dt_errarg = begin.dtbgn_errarg;

	return (rval);
}

/* ARGSUSED */
static uint64_t
dt_buf_oldest(void *elem, void *arg)
{
	dtrace_bufdesc_t *buf = elem;
	size_t offs = buf->dtbd_oldest;

	while (offs < buf->dtbd_size) {
		dtrace_rechdr_t *dtrh =
		    /* LINTED - alignment */
		    (dtrace_rechdr_t *)(buf->dtbd_data + offs);
		if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
			offs += sizeof (dtrace_epid_t);
		} else {
			return (DTRACE_RECORD_LOAD_TIMESTAMP(dtrh));
		}
	}

	/* There are no records left; use the time the buffer was retrieved. */
	return (buf->dtbd_timestamp);
}

int
dtrace_consume(dtrace_hdl_t *dtp, FILE *fp,
    dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
{
	dtrace_optval_t size;
	static int max_ncpus;
	int i, rval;
	dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_SWITCHRATE];
	hrtime_t now = gethrtime();

	if (dtp->dt_lastswitch != 0) {
		if (now - dtp->dt_lastswitch < interval)
			return (0);

		dtp->dt_lastswitch += interval;
	} else {
		dtp->dt_lastswitch = now;
	}

	if (!dtp->dt_active)
		return (dt_set_errno(dtp, EINVAL));

	if (max_ncpus == 0 && (max_ncpus = dt_cpu_maxid(dtp) + 1) <= 0)
		return (-1);

	if (pf == NULL)
		pf = (dtrace_consume_probe_f *)dt_nullprobe;

	if (rf == NULL)
		rf = (dtrace_consume_rec_f *)dt_nullrec;

	if (dtp->dt_options[DTRACEOPT_TEMPORAL] == DTRACEOPT_UNSET) {
		/*
		 * The output will not be in the order it was traced.  Rather,
		 * we will consume all of the data from each CPU's buffer in
		 * turn.  We apply special handling for the records from BEGIN
		 * and END probes so that they are consumed first and last,
		 * respectively.
		 *
		 * If we have just begun, we want to first process the CPU that
		 * executed the BEGIN probe (if any).
		 */
		if (dtp->dt_active && dtp->dt_beganon != -1 &&
		    (rval = dt_consume_begin(dtp, fp, pf, rf, arg)) != 0)
			return (rval);

		for (i = 0; i < max_ncpus; i++) {
			dtrace_bufdesc_t *buf;

			/*
			 * If we have stopped, we want to process the CPU on
			 * which the END probe was processed only _after_ we
			 * have processed everything else.
			 */
			if (dtp->dt_stopped && (i == dtp->dt_endedon))
				continue;

			if (dt_get_buf(dtp, i, &buf) != 0)
				return (-1);
			if (buf == NULL)
				continue;

			dtp->dt_flow = 0;
			dtp->dt_indent = 0;
			dtp->dt_prefix = NULL;
			rval = dt_consume_cpu(dtp, fp, i,
			    buf, B_FALSE, pf, rf, arg);
			dt_put_buf(dtp, buf);
			if (rval != 0)
				return (rval);
		}
		if (dtp->dt_stopped) {
			dtrace_bufdesc_t *buf;

			if (dt_get_buf(dtp, dtp->dt_endedon, &buf) != 0)
				return (-1);
			if (buf == NULL)
				return (0);

			rval = dt_consume_cpu(dtp, fp, dtp->dt_endedon,
			    buf, B_FALSE, pf, rf, arg);
			dt_put_buf(dtp, buf);
			return (rval);
		}
	} else {
		/*
		 * The output will be in the order it was traced (or for
		 * speculations, when it was committed).  We retrieve a buffer
		 * from each CPU and put it into a priority queue, which sorts
		 * based on the first entry in the buffer.  This is sufficient
		 * because entries within a buffer are already sorted.
		 *
		 * We then consume records one at a time, always consuming the
		 * oldest record, as determined by the priority queue.  When
		 * we reach the end of the time covered by these buffers,
		 * we need to stop and retrieve more records on the next pass.
		 * The kernel tells us the time covered by each buffer, in
		 * dtbd_timestamp.  The first buffer's timestamp tells us the
		 * time covered by all buffers, as subsequently retrieved
		 * buffers will cover to a more recent time.
		 */

		uint64_t *drops = alloca(max_ncpus * sizeof (uint64_t));
		uint64_t first_timestamp = 0;
		uint_t cookie = 0;
		dtrace_bufdesc_t *buf;

		bzero(drops, max_ncpus * sizeof (uint64_t));

		if (dtp->dt_bufq == NULL) {
			dtp->dt_bufq = dt_pq_init(dtp, max_ncpus * 2,
			    dt_buf_oldest, NULL);
			if (dtp->dt_bufq == NULL) /* ENOMEM */
				return (-1);
		}

		/* Retrieve data from each CPU. */
		(void) dtrace_getopt(dtp, "bufsize", &size);
		for (i = 0; i < max_ncpus; i++) {
			dtrace_bufdesc_t *buf;

			if (dt_get_buf(dtp, i, &buf) != 0)
				return (-1);
			if (buf != NULL) {
				if (first_timestamp == 0)
					first_timestamp = buf->dtbd_timestamp;
				assert(buf->dtbd_timestamp >= first_timestamp);

				dt_pq_insert(dtp->dt_bufq, buf);
				drops[i] = buf->dtbd_drops;
				buf->dtbd_drops = 0;
			}
		}

		/* Consume records. */
		for (;;) {
			dtrace_bufdesc_t *buf = dt_pq_pop(dtp->dt_bufq);
			uint64_t timestamp;

			if (buf == NULL)
				break;

			timestamp = dt_buf_oldest(buf, dtp);
			if (timestamp == buf->dtbd_timestamp) {
				/*
				 * We've reached the end of the time covered
				 * by this buffer.  If this is the oldest
				 * buffer, we must do another pass
				 * to retrieve more data.
				 */
				dt_put_buf(dtp, buf);
				if (timestamp == first_timestamp &&
				    !dtp->dt_stopped)
					break;
				continue;
			}
			assert(timestamp >= dtp->dt_last_timestamp);
			dtp->dt_last_timestamp = timestamp;

			if ((rval = dt_consume_cpu(dtp, fp,
			    buf->dtbd_cpu, buf, B_TRUE, pf, rf, arg)) != 0)
				return (rval);
			dt_pq_insert(dtp->dt_bufq, buf);
		}

		/* Consume drops. */
		for (i = 0; i < max_ncpus; i++) {
			if (drops[i] != 0) {
				int error;

				if (dtp->dt_oformat) {
					xo_open_instance("probes");
					dt_oformat_drop(dtp, i);
				}
				error = dt_handle_cpudrop(dtp, i,
				    DTRACEDROP_PRINCIPAL, drops[i]);
				if (dtp->dt_oformat)
					xo_close_instance("probes");
				if (error != 0)
					return (error);
			}
		}

		/*
		 * Reduce memory usage by re-allocating smaller buffers
		 * for the "remnants".
		 */
		while (buf = dt_pq_walk(dtp->dt_bufq, &cookie))
			dt_realloc_buf(dtp, buf, buf->dtbd_size);
	}

	return (0);
}

void
dtrace_oformat_probe(dtrace_hdl_t *dtp __unused, const dtrace_probedata_t *data,
    processorid_t cpu, dtrace_probedesc_t *pd)
{

	xo_emit("{:timestamp/%llu} {:cpu/%d} {:id/%d} {:provider/%s} "
		"{:module/%s} {:function/%s} {:name/%s}",
	    (unsigned long long)data->dtpda_timestamp, cpu, pd->dtpd_id,
	    pd->dtpd_provider, pd->dtpd_mod, pd->dtpd_func, pd->dtpd_name);
}

void
dt_oformat_drop(dtrace_hdl_t *dtp, processorid_t cpu)
{
	xo_emit("{:cpu/%d} {:id/%d} {:provider/%s} "
		"{:module/%s} {:function/%s} {:name/%s}",
	    cpu, -1, "dtrace", "INTERNAL", "INTERNAL", "DROP");
}