xref: /freebsd/sys/kern/subr_stats.c (revision 5b56413d04e608379c9a306373554a8e4d321bc0)
1 /*-
2  * Copyright (c) 2014-2018 Netflix, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 /*
28  * Author: Lawrence Stewart <lstewart@netflix.com>
29  */
30 
31 #include <sys/param.h>
32 #include <sys/arb.h>
33 #include <sys/ctype.h>
34 #include <sys/errno.h>
35 #include <sys/hash.h>
36 #include <sys/limits.h>
37 #include <sys/malloc.h>
38 #include <sys/qmath.h>
39 #include <sys/sbuf.h>
40 #if defined(DIAGNOSTIC)
41 #include <sys/tree.h>
42 #endif
43 #include <sys/stats.h> /* Must come after qmath.h and arb.h */
44 #include <sys/stddef.h>
45 #include <sys/stdint.h>
46 #include <sys/time.h>
47 
48 #ifdef _KERNEL
49 #include <sys/kernel.h>
50 #include <sys/lock.h>
51 #include <sys/rwlock.h>
52 #include <sys/sysctl.h>
53 #include <sys/systm.h>
54 #else /* ! _KERNEL */
55 #include <pthread.h>
56 #include <stdbool.h>
57 #include <stdio.h>
58 #include <stdlib.h>
59 #include <string.h>
60 #endif /* _KERNEL */
61 
62 struct voistatdata_voistate {
63 	/* Previous VOI value for diff calculation. */
64 	struct voistatdata_numeric prev;
65 };
66 
67 #define	VS_VSDVALID	0x0001	/* Stat's voistatdata updated at least once. */
68 struct voistat {
69 	int8_t		stype;		/* Type of stat e.g. VS_STYPE_SUM. */
70 	enum vsd_dtype	dtype : 8;	/* Data type of this stat's data. */
71 	uint16_t	data_off;	/* Blob offset for this stat's data. */
72 	uint16_t	dsz;		/* Size of stat's data. */
73 #define	VS_EBITS 8
74 	uint16_t	errs : VS_EBITS;/* Non-wrapping error count. */
75 	uint16_t	flags : 16 - VS_EBITS;
76 };
77 /* The voistat error count is capped to avoid wrapping. */
78 #define	VS_INCERRS(vs) do {						\
79 	if ((vs)->errs < (1U << VS_EBITS) - 1)				\
80 		(vs)->errs++;						\
81 } while (0)
82 
83 /*
84  * Ideas for flags:
85  *   - Global or entity specific (global would imply use of counter(9)?)
86  *   - Whether to reset stats on read or not
87  *   - Signal an overflow?
88  *   - Compressed voistat array
89  */
90 #define	VOI_REQSTATE	0x0001	/* VOI requires VS_STYPE_VOISTATE. */
91 struct voi {
92 	int16_t		id;		/* VOI id. */
93 	enum vsd_dtype	dtype : 8;	/* Data type of the VOI itself. */
94 	int8_t		voistatmaxid;	/* Largest allocated voistat index. */
95 	uint16_t	stats_off;	/* Blob offset for this VOIs stats. */
96 	uint16_t	flags;
97 };
98 
99 /*
100  * Memory for the entire blob is allocated as a slab and then offsets are
101  * maintained to carve up the slab into sections holding different data types.
102  *
103  * Ideas for flags:
104  * - Compressed voi array (trade off memory usage vs search time)
105  * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib)
106  */
107 struct statsblobv1 {
108 	uint8_t		abi;
109 	uint8_t		endian;
110 	uint16_t	flags;
111 	uint16_t	maxsz;
112 	uint16_t	cursz;
113 	/* Fields from here down are opaque to consumers. */
114 	uint32_t	tplhash;	/* Base template hash ID. */
115 	uint16_t	stats_off;	/* voistat array blob offset. */
116 	uint16_t	statsdata_off;	/* voistatdata array blob offset. */
117 	sbintime_t	created;	/* Blob creation time. */
118 	sbintime_t	lastrst;	/* Time of last reset. */
119 	struct voi	vois[];		/* Array indexed by [voi_id]. */
120 } __aligned(sizeof(void *));
121 _Static_assert(offsetof(struct statsblobv1, cursz) +
122     SIZEOF_MEMBER(struct statsblobv1, cursz) ==
123     offsetof(struct statsblob, opaque),
124     "statsblobv1 ABI mismatch");
125 
126 struct statsblobv1_tpl {
127 	struct metablob		*mb;
128 	struct statsblobv1	*sb;
129 };
130 
131 /* Context passed to iterator callbacks. */
132 struct sb_iter_ctx {
133 	void		*usrctx;	/* Caller supplied context. */
134 	uint32_t	flags;		/* Flags for current iteration. */
135 	int16_t		vslot;		/* struct voi slot index. */
136 	int8_t		vsslot;		/* struct voistat slot index. */
137 };
138 
139 struct sb_tostrcb_ctx {
140 	struct sbuf		*buf;
141 	struct statsblob_tpl	*tpl;
142 	enum sb_str_fmt	fmt;
143 	uint32_t		flags;
144 };
145 
146 struct sb_visitcb_ctx {
147 	stats_blob_visitcb_t	cb;
148 	void			*usrctx;
149 };
150 
151 /* Stats blob iterator callback. */
152 typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v,
153     struct voistat *vs, struct sb_iter_ctx *ctx);
154 
155 #ifdef _KERNEL
156 static struct rwlock tpllistlock;
157 RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock");
158 #define	TPL_LIST_RLOCK() rw_rlock(&tpllistlock)
159 #define	TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock)
160 #define	TPL_LIST_WLOCK() rw_wlock(&tpllistlock)
161 #define	TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock)
162 #define	TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED)
163 #define	TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED)
164 #define	TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED)
165 MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory");
166 #define	stats_free(ptr) free((ptr), M_STATS)
167 #else /* ! _KERNEL */
168 static void stats_constructor(void);
169 static void stats_destructor(void);
170 static pthread_rwlock_t tpllistlock;
171 #define	TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock)
172 #define	TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock)
173 #define	TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK()
174 #define	TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock)
175 #define	TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK()
176 #define	TPL_LIST_LOCK_ASSERT() do { } while (0)
177 #define	TPL_LIST_RLOCK_ASSERT() do { } while (0)
178 #define	TPL_LIST_WLOCK_ASSERT() do { } while (0)
179 #ifdef NDEBUG
180 #define	KASSERT(cond, msg) do {} while (0)
181 #define	stats_abort() do {} while (0)
182 #else /* ! NDEBUG */
183 #define	KASSERT(cond, msg) do { \
184 	if (!(cond)) { \
185 		panic msg; \
186 	} \
187 } while (0)
188 #define	stats_abort() abort()
189 #endif /* NDEBUG */
190 #define	stats_free(ptr) free(ptr)
191 #define	panic(fmt, ...) do { \
192 	fprintf(stderr, (fmt), ##__VA_ARGS__); \
193 	stats_abort(); \
194 } while (0)
195 #endif /* _KERNEL */
196 
197 #define	SB_V1_MAXSZ 65535
198 
199 /* Obtain a blob offset pointer. */
200 #define	BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off)))
201 
202 /*
203  * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a
204  * power of 2 size, we can shift instead of divide. The shift amount must be
205  * updated if sizeof(struct voi) ever changes, which the assert should catch.
206  */
207 #define	NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \
208     sizeof(struct statsblobv1)) >> 3))
209 _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch");
210 
211 /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */
212 const char *vs_stype2name[VS_NUM_STYPES] = {
213 	[VS_STYPE_VOISTATE] = "VOISTATE",
214 	[VS_STYPE_SUM] = "SUM",
215 	[VS_STYPE_MAX] = "MAX",
216 	[VS_STYPE_MIN] = "MIN",
217 	[VS_STYPE_HIST] = "HIST",
218 	[VS_STYPE_TDGST] = "TDGST",
219 };
220 
221 const char *vs_stype2desc[VS_NUM_STYPES] = {
222 	[VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)",
223 	[VS_STYPE_SUM] = "Simple arithmetic accumulator",
224 	[VS_STYPE_MAX] = "Maximum observed VOI value",
225 	[VS_STYPE_MIN] = "Minimum observed VOI value",
226 	[VS_STYPE_HIST] = "Histogram of observed VOI values",
227 	[VS_STYPE_TDGST] = "t-digest of observed VOI values",
228 };
229 
230 const char *vsd_dtype2name[VSD_NUM_DTYPES] = {
231 	[VSD_DTYPE_VOISTATE] = "VOISTATE",
232 	[VSD_DTYPE_INT_S32] = "INT_S32",
233 	[VSD_DTYPE_INT_U32] = "INT_U32",
234 	[VSD_DTYPE_INT_S64] = "INT_S64",
235 	[VSD_DTYPE_INT_U64] = "INT_U64",
236 	[VSD_DTYPE_INT_SLONG] = "INT_SLONG",
237 	[VSD_DTYPE_INT_ULONG] = "INT_ULONG",
238 	[VSD_DTYPE_Q_S32] = "Q_S32",
239 	[VSD_DTYPE_Q_U32] = "Q_U32",
240 	[VSD_DTYPE_Q_S64] = "Q_S64",
241 	[VSD_DTYPE_Q_U64] = "Q_U64",
242 	[VSD_DTYPE_CRHIST32] = "CRHIST32",
243 	[VSD_DTYPE_DRHIST32] = "DRHIST32",
244 	[VSD_DTYPE_DVHIST32] = "DVHIST32",
245 	[VSD_DTYPE_CRHIST64] = "CRHIST64",
246 	[VSD_DTYPE_DRHIST64] = "DRHIST64",
247 	[VSD_DTYPE_DVHIST64] = "DVHIST64",
248 	[VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32",
249 	[VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64",
250 };
251 
252 const size_t vsd_dtype2size[VSD_NUM_DTYPES] = {
253 	[VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate),
254 	[VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32),
255 	[VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32),
256 	[VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64),
257 	[VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64),
258 	[VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong),
259 	[VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong),
260 	[VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32),
261 	[VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32),
262 	[VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64),
263 	[VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64),
264 	[VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32),
265 	[VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32),
266 	[VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32),
267 	[VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64),
268 	[VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64),
269 	[VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64),
270 	[VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32),
271 	[VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64),
272 };
273 
274 static const bool vsd_compoundtype[VSD_NUM_DTYPES] = {
275 	[VSD_DTYPE_VOISTATE] = true,
276 	[VSD_DTYPE_INT_S32] = false,
277 	[VSD_DTYPE_INT_U32] = false,
278 	[VSD_DTYPE_INT_S64] = false,
279 	[VSD_DTYPE_INT_U64] = false,
280 	[VSD_DTYPE_INT_SLONG] = false,
281 	[VSD_DTYPE_INT_ULONG] = false,
282 	[VSD_DTYPE_Q_S32] = false,
283 	[VSD_DTYPE_Q_U32] = false,
284 	[VSD_DTYPE_Q_S64] = false,
285 	[VSD_DTYPE_Q_U64] = false,
286 	[VSD_DTYPE_CRHIST32] = true,
287 	[VSD_DTYPE_DRHIST32] = true,
288 	[VSD_DTYPE_DVHIST32] = true,
289 	[VSD_DTYPE_CRHIST64] = true,
290 	[VSD_DTYPE_DRHIST64] = true,
291 	[VSD_DTYPE_DVHIST64] = true,
292 	[VSD_DTYPE_TDGSTCLUST32] = true,
293 	[VSD_DTYPE_TDGSTCLUST64] = true,
294 };
295 
296 const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = {
297 	[LIM_MIN] = {
298 		[VSD_DTYPE_VOISTATE] = {},
299 		[VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}},
300 		[VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}},
301 		[VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}},
302 		[VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}},
303 		[VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}},
304 		[VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}},
305 		[VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}},
306 		[VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}},
307 		[VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}},
308 		[VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}},
309 	},
310 	[LIM_MAX] = {
311 		[VSD_DTYPE_VOISTATE] = {},
312 		[VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}},
313 		[VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}},
314 		[VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}},
315 		[VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}},
316 		[VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}},
317 		[VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}},
318 		[VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}},
319 		[VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}},
320 		[VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}},
321 		[VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}},
322 	}
323 };
324 
325 /* tpllistlock protects tpllist and ntpl */
326 static uint32_t ntpl;
327 static struct statsblob_tpl **tpllist;
328 
329 static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz,
330     int flags);
331 //static void stats_v1_blob_finalise(struct statsblobv1 *sb);
332 static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
333     uint32_t flags);
334 static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
335     int newvoistatbytes, int newvoistatdatabytes);
336 static void stats_v1_blob_iter(struct statsblobv1 *sb,
337     stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags);
338 static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype,
339     struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt);
340 
341 static inline int
342 ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2)
343 {
344 
345 	KASSERT(Q_PRECEQ(c1->mu, c2->mu),
346 	    ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
347 	    Q_RELPREC(c1->mu, c2->mu)));
348 
349        return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
350 }
351 ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp);
352 
353 static inline int
354 ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2)
355 {
356 
357 	KASSERT(Q_PRECEQ(c1->mu, c2->mu),
358 	    ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
359 	    Q_RELPREC(c1->mu, c2->mu)));
360 
361        return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
362 }
363 ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp);
364 
365 #ifdef DIAGNOSTIC
366 RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp);
367 RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp);
368 #endif
369 
370 static inline sbintime_t
371 stats_sbinuptime(void)
372 {
373 	sbintime_t sbt;
374 #ifdef _KERNEL
375 
376 	sbt = sbinuptime();
377 #else /* ! _KERNEL */
378 	struct timespec tp;
379 
380 	clock_gettime(CLOCK_MONOTONIC_FAST, &tp);
381 	sbt = tstosbt(tp);
382 #endif /* _KERNEL */
383 
384 	return (sbt);
385 }
386 
387 static inline void *
388 stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags)
389 {
390 
391 #ifdef _KERNEL
392 	/* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */
393 	if (!(flags & (M_WAITOK | M_NOWAIT)))
394 		flags |= M_NOWAIT;
395 	ptr = realloc(ptr, newsz, M_STATS, flags);
396 #else /* ! _KERNEL */
397 	ptr = realloc(ptr, newsz);
398 	if ((flags & M_ZERO) && ptr != NULL) {
399 		if (oldsz == 0)
400 			memset(ptr, '\0', newsz);
401 		else if (newsz > oldsz)
402 			memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz);
403 	}
404 #endif /* _KERNEL */
405 
406 	return (ptr);
407 }
408 
409 static inline char *
410 stats_strdup(const char *s,
411 #ifdef _KERNEL
412     int flags)
413 {
414 	char *copy;
415 	size_t len;
416 
417 	if (!(flags & (M_WAITOK | M_NOWAIT)))
418 		flags |= M_NOWAIT;
419 
420 	len = strlen(s) + 1;
421 	if ((copy = malloc(len, M_STATS, flags)) != NULL)
422 		bcopy(s, copy, len);
423 
424 	return (copy);
425 #else
426     int flags __unused)
427 {
428 	return (strdup(s));
429 #endif
430 }
431 
432 static inline void
433 stats_tpl_update_hash(struct statsblob_tpl *tpl)
434 {
435 
436 	TPL_LIST_WLOCK_ASSERT();
437 	tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0);
438 	for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) {
439 		if (tpl->mb->voi_meta[voi_id].name != NULL)
440 			tpl->mb->tplhash = hash32_str(
441 			    tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash);
442 	}
443 	tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz,
444 	    tpl->mb->tplhash);
445 }
446 
447 static inline uint64_t
448 stats_pow_u64(uint64_t base, uint64_t exp)
449 {
450 	uint64_t result = 1;
451 
452 	while (exp) {
453 		if (exp & 1)
454 			result *= base;
455 		exp >>= 1;
456 		base *= base;
457 	}
458 
459 	return (result);
460 }
461 
462 static inline int
463 stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt,
464     struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub)
465 {
466 	uint64_t step = 0;
467 	int error = 0;
468 
469 	switch (info->scheme) {
470 	case BKT_LIN:
471 		step = info->lin.stepinc;
472 		break;
473 	case BKT_EXP:
474 		step = stats_pow_u64(info->exp.stepbase,
475 		    info->exp.stepexp + curbkt);
476 		break;
477 	case BKT_LINEXP:
478 		{
479 		uint64_t curstepexp = 1;
480 
481 		switch (info->voi_dtype) {
482 		case VSD_DTYPE_INT_S32:
483 			while ((int32_t)stats_pow_u64(info->linexp.stepbase,
484 			    curstepexp) <= bkt_lb->int32.s32)
485 				curstepexp++;
486 			break;
487 		case VSD_DTYPE_INT_U32:
488 			while ((uint32_t)stats_pow_u64(info->linexp.stepbase,
489 			    curstepexp) <= bkt_lb->int32.u32)
490 				curstepexp++;
491 			break;
492 		case VSD_DTYPE_INT_S64:
493 			while ((int64_t)stats_pow_u64(info->linexp.stepbase,
494 			    curstepexp) <= bkt_lb->int64.s64)
495 				curstepexp++;
496 			break;
497 		case VSD_DTYPE_INT_U64:
498 			while ((uint64_t)stats_pow_u64(info->linexp.stepbase,
499 			    curstepexp) <= bkt_lb->int64.u64)
500 				curstepexp++;
501 			break;
502 		case VSD_DTYPE_INT_SLONG:
503 			while ((long)stats_pow_u64(info->linexp.stepbase,
504 			    curstepexp) <= bkt_lb->intlong.slong)
505 				curstepexp++;
506 			break;
507 		case VSD_DTYPE_INT_ULONG:
508 			while ((unsigned long)stats_pow_u64(info->linexp.stepbase,
509 			    curstepexp) <= bkt_lb->intlong.ulong)
510 				curstepexp++;
511 			break;
512 		case VSD_DTYPE_Q_S32:
513 			while ((s32q_t)stats_pow_u64(info->linexp.stepbase,
514 			    curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32))
515 			break;
516 		case VSD_DTYPE_Q_U32:
517 			while ((u32q_t)stats_pow_u64(info->linexp.stepbase,
518 			    curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32))
519 			break;
520 		case VSD_DTYPE_Q_S64:
521 			while ((s64q_t)stats_pow_u64(info->linexp.stepbase,
522 			    curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64))
523 				curstepexp++;
524 			break;
525 		case VSD_DTYPE_Q_U64:
526 			while ((u64q_t)stats_pow_u64(info->linexp.stepbase,
527 			    curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64))
528 				curstepexp++;
529 			break;
530 		default:
531 			break;
532 		}
533 
534 		step = stats_pow_u64(info->linexp.stepbase, curstepexp) /
535 		    info->linexp.linstepdiv;
536 		if (step == 0)
537 			step = 1;
538 		break;
539 		}
540 	default:
541 		break;
542 	}
543 
544 	if (info->scheme == BKT_USR) {
545 		*bkt_lb = info->usr.bkts[curbkt].lb;
546 		*bkt_ub = info->usr.bkts[curbkt].ub;
547 	} else if (step != 0) {
548 		switch (info->voi_dtype) {
549 		case VSD_DTYPE_INT_S32:
550 			bkt_ub->int32.s32 += (int32_t)step;
551 			break;
552 		case VSD_DTYPE_INT_U32:
553 			bkt_ub->int32.u32 += (uint32_t)step;
554 			break;
555 		case VSD_DTYPE_INT_S64:
556 			bkt_ub->int64.s64 += (int64_t)step;
557 			break;
558 		case VSD_DTYPE_INT_U64:
559 			bkt_ub->int64.u64 += (uint64_t)step;
560 			break;
561 		case VSD_DTYPE_INT_SLONG:
562 			bkt_ub->intlong.slong += (long)step;
563 			break;
564 		case VSD_DTYPE_INT_ULONG:
565 			bkt_ub->intlong.ulong += (unsigned long)step;
566 			break;
567 		case VSD_DTYPE_Q_S32:
568 			error = Q_QADDI(&bkt_ub->q32.sq32, step);
569 			break;
570 		case VSD_DTYPE_Q_U32:
571 			error = Q_QADDI(&bkt_ub->q32.uq32, step);
572 			break;
573 		case VSD_DTYPE_Q_S64:
574 			error = Q_QADDI(&bkt_ub->q64.sq64, step);
575 			break;
576 		case VSD_DTYPE_Q_U64:
577 			error = Q_QADDI(&bkt_ub->q64.uq64, step);
578 			break;
579 		default:
580 			break;
581 		}
582 	} else { /* info->scheme != BKT_USR && step == 0 */
583 		return (EINVAL);
584 	}
585 
586 	return (error);
587 }
588 
589 static uint32_t
590 stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info)
591 {
592 	struct voistatdata_numeric bkt_lb, bkt_ub;
593 	uint32_t nbkts;
594 	int done;
595 
596 	if (info->scheme == BKT_USR) {
597 		/* XXXLAS: Setting info->{lb,ub} from macro is tricky. */
598 		info->lb = info->usr.bkts[0].lb;
599 		info->ub = info->usr.bkts[info->usr.nbkts - 1].lb;
600 	}
601 
602 	nbkts = 0;
603 	done = 0;
604 	bkt_ub = info->lb;
605 
606 	do {
607 		bkt_lb = bkt_ub;
608 		if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub))
609 			return (0);
610 
611 		if (info->scheme == BKT_USR)
612 			done = (nbkts == info->usr.nbkts);
613 		else {
614 			switch (info->voi_dtype) {
615 			case VSD_DTYPE_INT_S32:
616 				done = (bkt_ub.int32.s32 > info->ub.int32.s32);
617 				break;
618 			case VSD_DTYPE_INT_U32:
619 				done = (bkt_ub.int32.u32 > info->ub.int32.u32);
620 				break;
621 			case VSD_DTYPE_INT_S64:
622 				done = (bkt_ub.int64.s64 > info->ub.int64.s64);
623 				break;
624 			case VSD_DTYPE_INT_U64:
625 				done = (bkt_ub.int64.u64 > info->ub.int64.u64);
626 				break;
627 			case VSD_DTYPE_INT_SLONG:
628 				done = (bkt_ub.intlong.slong >
629 				    info->ub.intlong.slong);
630 				break;
631 			case VSD_DTYPE_INT_ULONG:
632 				done = (bkt_ub.intlong.ulong >
633 				    info->ub.intlong.ulong);
634 				break;
635 			case VSD_DTYPE_Q_S32:
636 				done = Q_QGTQ(bkt_ub.q32.sq32,
637 				    info->ub.q32.sq32);
638 				break;
639 			case VSD_DTYPE_Q_U32:
640 				done = Q_QGTQ(bkt_ub.q32.uq32,
641 				    info->ub.q32.uq32);
642 				break;
643 			case VSD_DTYPE_Q_S64:
644 				done = Q_QGTQ(bkt_ub.q64.sq64,
645 				    info->ub.q64.sq64);
646 				break;
647 			case VSD_DTYPE_Q_U64:
648 				done = Q_QGTQ(bkt_ub.q64.uq64,
649 				    info->ub.q64.uq64);
650 				break;
651 			default:
652 				return (0);
653 			}
654 		}
655 	} while (!done);
656 
657 	if (info->flags & VSD_HIST_LBOUND_INF)
658 		nbkts++;
659 	if (info->flags & VSD_HIST_UBOUND_INF)
660 		nbkts++;
661 
662 	return (nbkts);
663 }
664 
665 int
666 stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
667     struct vss_hist_hlpr_info *info)
668 {
669 	struct voistatdata_hist *hist;
670 	struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub,
671 	    *ubinfbktlb, *ubinfbktub;
672 	uint32_t bkt, nbkts, nloop;
673 
674 	if (vss == NULL || info == NULL || (info->flags &
675 	(VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype ==
676 	VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64)))
677 		return (EINVAL);
678 
679 	info->voi_dtype = voi_dtype;
680 
681 	if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0)
682 		return (EINVAL);
683 
684 	switch (info->hist_dtype) {
685 	case VSD_DTYPE_CRHIST32:
686 		vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts);
687 		break;
688 	case VSD_DTYPE_DRHIST32:
689 		vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts);
690 		break;
691 	case VSD_DTYPE_DVHIST32:
692 		vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts);
693 		break;
694 	case VSD_DTYPE_CRHIST64:
695 		vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts);
696 		break;
697 	case VSD_DTYPE_DRHIST64:
698 		vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts);
699 		break;
700 	case VSD_DTYPE_DVHIST64:
701 		vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts);
702 		break;
703 	default:
704 		return (EINVAL);
705 	}
706 
707 	vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
708 	if (vss->iv == NULL)
709 		return (ENOMEM);
710 
711 	hist = (struct voistatdata_hist *)vss->iv;
712 	bkt_ub = info->lb;
713 
714 	for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0;
715 	    bkt < nbkts;
716 	    bkt++, nloop++) {
717 		bkt_lb = bkt_ub;
718 		if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub))
719 			return (EINVAL);
720 
721 		switch (info->hist_dtype) {
722 		case VSD_DTYPE_CRHIST32:
723 			VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb;
724 			break;
725 		case VSD_DTYPE_DRHIST32:
726 			VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb;
727 			VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub;
728 			break;
729 		case VSD_DTYPE_DVHIST32:
730 			VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb;
731 			break;
732 		case VSD_DTYPE_CRHIST64:
733 			VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb;
734 			break;
735 		case VSD_DTYPE_DRHIST64:
736 			VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb;
737 			VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub;
738 			break;
739 		case VSD_DTYPE_DVHIST64:
740 			VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb;
741 			break;
742 		default:
743 			return (EINVAL);
744 		}
745 	}
746 
747 	lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL;
748 
749 	switch (info->hist_dtype) {
750 	case VSD_DTYPE_CRHIST32:
751 		lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb;
752 		ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb;
753 		break;
754 	case VSD_DTYPE_DRHIST32:
755 		lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb;
756 		lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub;
757 		ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb;
758 		ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub;
759 		break;
760 	case VSD_DTYPE_CRHIST64:
761 		lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb;
762 		ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb;
763 		break;
764 	case VSD_DTYPE_DRHIST64:
765 		lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb;
766 		lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub;
767 		ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb;
768 		ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub;
769 		break;
770 	case VSD_DTYPE_DVHIST32:
771 	case VSD_DTYPE_DVHIST64:
772 		break;
773 	default:
774 		return (EINVAL);
775 	}
776 
777 	if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) {
778 		*lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype];
779 		/*
780 		 * Assignment from numeric_limit array for Q types assigns max
781 		 * possible integral/fractional value for underlying data type,
782 		 * but we must set control bits for this specific histogram per
783 		 * the user's choice of fractional bits, which we extract from
784 		 * info->lb.
785 		 */
786 		if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
787 		    info->voi_dtype == VSD_DTYPE_Q_U32) {
788 			/* Signedness doesn't matter for setting control bits. */
789 			Q_SCVAL(lbinfbktlb->q32.sq32,
790 			    Q_GCVAL(info->lb.q32.sq32));
791 		} else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
792 		    info->voi_dtype == VSD_DTYPE_Q_U64) {
793 			/* Signedness doesn't matter for setting control bits. */
794 			Q_SCVAL(lbinfbktlb->q64.sq64,
795 			    Q_GCVAL(info->lb.q64.sq64));
796 		}
797 		if (lbinfbktub)
798 			*lbinfbktub = info->lb;
799 	}
800 	if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) {
801 		*ubinfbktlb = bkt_lb;
802 		if (ubinfbktub) {
803 			*ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype];
804 			if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
805 			    info->voi_dtype == VSD_DTYPE_Q_U32) {
806 				Q_SCVAL(ubinfbktub->q32.sq32,
807 				    Q_GCVAL(info->lb.q32.sq32));
808 			} else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
809 			    info->voi_dtype == VSD_DTYPE_Q_U64) {
810 				Q_SCVAL(ubinfbktub->q64.sq64,
811 				    Q_GCVAL(info->lb.q64.sq64));
812 			}
813 		}
814 	}
815 
816 	return (0);
817 }
818 
819 int
820 stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
821     struct vss_tdgst_hlpr_info *info)
822 {
823 	struct voistatdata_tdgst *tdgst;
824 	struct ctdth32 *ctd32tree;
825 	struct ctdth64 *ctd64tree;
826 	struct voistatdata_tdgstctd32 *ctd32;
827 	struct voistatdata_tdgstctd64 *ctd64;
828 
829 	info->voi_dtype = voi_dtype;
830 
831 	switch (info->tdgst_dtype) {
832 	case VSD_DTYPE_TDGSTCLUST32:
833 		vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds);
834 		break;
835 	case VSD_DTYPE_TDGSTCLUST64:
836 		vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds);
837 		break;
838 	default:
839 		return (EINVAL);
840 	}
841 
842 	vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
843 	if (vss->iv == NULL)
844 		return (ENOMEM);
845 
846 	tdgst = (struct voistatdata_tdgst *)vss->iv;
847 
848 	switch (info->tdgst_dtype) {
849 	case VSD_DTYPE_TDGSTCLUST32:
850 		ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
851 		ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) {
852 			Q_INI(&ctd32->mu, 0, 0, info->prec);
853 		}
854 		break;
855 	case VSD_DTYPE_TDGSTCLUST64:
856 		ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
857 		ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) {
858 			Q_INI(&ctd64->mu, 0, 0, info->prec);
859 		}
860 		break;
861 	default:
862 		return (EINVAL);
863 	}
864 
865 	return (0);
866 }
867 
868 int
869 stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
870     struct vss_numeric_hlpr_info *info)
871 {
872 	struct voistatdata_numeric iv;
873 
874 	switch (vss->stype) {
875 	case VS_STYPE_SUM:
876 		iv = stats_ctor_vsd_numeric(0);
877 		break;
878 	case VS_STYPE_MIN:
879 		iv = numeric_limits[LIM_MAX][voi_dtype];
880 		break;
881 	case VS_STYPE_MAX:
882 		iv = numeric_limits[LIM_MIN][voi_dtype];
883 		break;
884 	default:
885 		return (EINVAL);
886 	}
887 
888 	vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0);
889 	if (vss->iv == NULL)
890 		return (ENOMEM);
891 
892 	vss->vs_dtype = voi_dtype;
893 	vss->vsdsz = vsd_dtype2size[voi_dtype];
894 	switch (voi_dtype) {
895 	case VSD_DTYPE_INT_S32:
896 		*((int32_t *)vss->iv) = iv.int32.s32;
897 		break;
898 	case VSD_DTYPE_INT_U32:
899 		*((uint32_t *)vss->iv) = iv.int32.u32;
900 		break;
901 	case VSD_DTYPE_INT_S64:
902 		*((int64_t *)vss->iv) = iv.int64.s64;
903 		break;
904 	case VSD_DTYPE_INT_U64:
905 		*((uint64_t *)vss->iv) = iv.int64.u64;
906 		break;
907 	case VSD_DTYPE_INT_SLONG:
908 		*((long *)vss->iv) = iv.intlong.slong;
909 		break;
910 	case VSD_DTYPE_INT_ULONG:
911 		*((unsigned long *)vss->iv) = iv.intlong.ulong;
912 		break;
913 	case VSD_DTYPE_Q_S32:
914 		*((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32,
915 		    Q_CTRLINI(info->prec));
916 		break;
917 	case VSD_DTYPE_Q_U32:
918 		*((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32,
919 		    Q_CTRLINI(info->prec));
920 		break;
921 	case VSD_DTYPE_Q_S64:
922 		*((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64,
923 		    Q_CTRLINI(info->prec));
924 		break;
925 	case VSD_DTYPE_Q_U64:
926 		*((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64,
927 		    Q_CTRLINI(info->prec));
928 		break;
929 	default:
930 		break;
931 	}
932 
933 	return (0);
934 }
935 
936 int
937 stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss,
938     struct voistatspec *vss)
939 {
940 	int i, ret;
941 
942 	for (i = nvss - 1; i >= 0; i--) {
943 		if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i],
944 		    vss[i].hlprinfo)) != 0)
945 			return (ret);
946 	}
947 
948 	return (0);
949 }
950 
951 void
952 stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss)
953 {
954 	int i;
955 
956 	for (i = nvss - 1; i >= 0; i--) {
957 		if (vss[i].hlpr) {
958 			stats_free((void *)vss[i].iv);
959 			vss[i].iv = NULL;
960 		}
961 	}
962 }
963 
964 int
965 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl)
966 {
967 	int error;
968 
969 	error = 0;
970 
971 	TPL_LIST_WLOCK();
972 	if (tpl_id < 0 || tpl_id >= (int)ntpl) {
973 		error = ENOENT;
974 	} else {
975 		*tpl = tpllist[tpl_id];
976 		/* XXXLAS: Acquire refcount on tpl. */
977 	}
978 	TPL_LIST_WUNLOCK();
979 
980 	return (error);
981 }
982 
983 int
984 stats_tpl_fetch_allocid(const char *name, uint32_t hash)
985 {
986 	int i, tpl_id;
987 
988 	tpl_id = -ESRCH;
989 
990 	TPL_LIST_RLOCK();
991 	for (i = ntpl - 1; i >= 0; i--) {
992 		if (name != NULL) {
993 			if (strlen(name) == strlen(tpllist[i]->mb->tplname) &&
994 			    strncmp(name, tpllist[i]->mb->tplname,
995 			    TPL_MAX_NAME_LEN) == 0 && (!hash || hash ==
996 			    tpllist[i]->mb->tplhash)) {
997 				tpl_id = i;
998 				break;
999 			}
1000 		} else if (hash == tpllist[i]->mb->tplhash) {
1001 			tpl_id = i;
1002 			break;
1003 		}
1004 	}
1005 	TPL_LIST_RUNLOCK();
1006 
1007 	return (tpl_id);
1008 }
1009 
1010 int
1011 stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len)
1012 {
1013 	int error;
1014 
1015 	error = 0;
1016 
1017 	TPL_LIST_RLOCK();
1018 	if (tpl_id < ntpl) {
1019 		if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname))
1020 			strlcpy(buf, tpllist[tpl_id]->mb->tplname, len);
1021 		else
1022 			error = EOVERFLOW;
1023 	} else
1024 		error = ENOENT;
1025 	TPL_LIST_RUNLOCK();
1026 
1027 	return (error);
1028 }
1029 
1030 int
1031 stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates,
1032     void *seed_bytes, size_t seed_len)
1033 {
1034 	uint32_t cum_pct, rnd_pct;
1035 	int i;
1036 
1037 	cum_pct = 0;
1038 
1039 	/*
1040 	 * Choose a pseudorandom or seeded number in range [0,100] and use
1041 	 * it to make a sampling decision and template selection where required.
1042 	 * If no seed is supplied, a PRNG is used to generate a pseudorandom
1043 	 * number so that every selection is independent. If a seed is supplied,
1044 	 * the caller desires random selection across different seeds, but
1045 	 * deterministic selection given the same seed. This is achieved by
1046 	 * hashing the seed and using the hash as the random number source.
1047 	 *
1048 	 * XXXLAS: Characterise hash function output distribution.
1049 	 */
1050 	if (seed_bytes == NULL)
1051 		rnd_pct = random() / (INT32_MAX / 100);
1052 	else
1053 		rnd_pct = hash32_buf(seed_bytes, seed_len, 0) /
1054 		    (UINT32_MAX / 100U);
1055 
1056 	/*
1057 	 * We map the randomly selected percentage on to the interval [0,100]
1058 	 * consisting of the cumulatively summed template sampling percentages.
1059 	 * The difference between the cumulative sum of all template sampling
1060 	 * percentages and 100 is treated as a NULL assignment i.e. no stats
1061 	 * template will be assigned, and -1 returned instead.
1062 	 */
1063 	for (i = 0; i < nrates; i++) {
1064 		cum_pct += rates[i].tpl_sample_pct;
1065 
1066 		KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__,
1067 		    cum_pct));
1068 		if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0)
1069 			continue;
1070 
1071 		return (rates[i].tpl_slot_id);
1072 	}
1073 
1074 	return (-1);
1075 }
1076 
1077 int
1078 stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz,
1079     struct statsblobv1 *src, uint32_t flags)
1080 {
1081 	int error, tmperror;
1082 
1083 	error = tmperror = 0;
1084 
1085 	if (src == NULL || dst == NULL ||
1086 	    src->cursz < sizeof(struct statsblob) ||
1087 	    ((flags & SB_CLONE_ALLOCDST) &&
1088 	    (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) {
1089 		error = EINVAL;
1090 	} else if (flags & SB_CLONE_ALLOCDST) {
1091 		*dst = stats_realloc(NULL, 0, src->cursz, 0);
1092 		if (*dst)
1093 			(*dst)->maxsz = dstmaxsz = src->cursz;
1094 		else
1095 			error = ENOMEM;
1096 	} else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) {
1097 		error = EINVAL;
1098 	}
1099 
1100 	if (!error) {
1101 		size_t postcurszlen;
1102 
1103 		/*
1104 		 * Clone src into dst except for the maxsz field. If dst is too
1105 		 * small to hold all of src, only copy src's header and return
1106 		 * EOVERFLOW.
1107 		 */
1108 #ifdef _KERNEL
1109 		if (flags & SB_CLONE_USRDSTNOFAULT)
1110 			error = copyout_nofault(src, *dst,
1111 			    offsetof(struct statsblob, maxsz));
1112 		else if (flags & SB_CLONE_USRDST)
1113 			error = copyout(src, *dst,
1114 			    offsetof(struct statsblob, maxsz));
1115 		else
1116 #endif
1117 			memcpy(*dst, src, offsetof(struct statsblob, maxsz));
1118 #ifdef _KERNEL
1119 		if (error != 0)
1120 			goto out;
1121 #endif
1122 
1123 
1124 		if (dstmaxsz >= src->cursz) {
1125 			postcurszlen = src->cursz -
1126 			    offsetof(struct statsblob, cursz);
1127 		} else {
1128 			error = EOVERFLOW;
1129 			postcurszlen = sizeof(struct statsblob) -
1130 			    offsetof(struct statsblob, cursz);
1131 		}
1132 #ifdef _KERNEL
1133 		if (flags & SB_CLONE_USRDSTNOFAULT)
1134 			tmperror = copyout_nofault(&(src->cursz), &((*dst)->cursz),
1135 			    postcurszlen);
1136 		else if (flags & SB_CLONE_USRDST)
1137 			tmperror = copyout(&(src->cursz), &((*dst)->cursz),
1138 			    postcurszlen);
1139 		else
1140 #endif
1141 			memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen);
1142 
1143 		error = error ? error : tmperror;
1144 	}
1145 #ifdef _KERNEL
1146 out:
1147 #endif
1148 
1149 	return (error);
1150 }
1151 
1152 int
1153 stats_v1_tpl_alloc(const char *name, uint32_t flags __unused)
1154 {
1155 	struct statsblobv1_tpl *tpl, **newtpllist;
1156 	struct statsblobv1 *tpl_sb;
1157 	struct metablob *tpl_mb;
1158 	int tpl_id;
1159 
1160 	if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN)
1161 		return (-EINVAL);
1162 
1163 	if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0)
1164 		return (-EEXIST);
1165 
1166 	tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO);
1167 	tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO);
1168 	tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO);
1169 
1170 	if (tpl_mb != NULL && name != NULL)
1171 		tpl_mb->tplname = stats_strdup(name, 0);
1172 
1173 	if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL ||
1174 	    tpl_mb->tplname == NULL) {
1175 		stats_free(tpl);
1176 		stats_free(tpl_sb);
1177 		if (tpl_mb != NULL) {
1178 			stats_free(tpl_mb->tplname);
1179 			stats_free(tpl_mb);
1180 		}
1181 		return (-ENOMEM);
1182 	}
1183 
1184 	tpl->mb = tpl_mb;
1185 	tpl->sb = tpl_sb;
1186 
1187 	tpl_sb->abi = STATS_ABI_V1;
1188 	tpl_sb->endian =
1189 #if BYTE_ORDER == LITTLE_ENDIAN
1190 	    SB_LE;
1191 #elif BYTE_ORDER == BIG_ENDIAN
1192 	    SB_BE;
1193 #else
1194 	    SB_UE;
1195 #endif
1196 	tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1);
1197 	tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1);
1198 
1199 	TPL_LIST_WLOCK();
1200 	newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *),
1201 	    (ntpl + 1) * sizeof(void *), 0);
1202 	if (newtpllist != NULL) {
1203 		tpl_id = ntpl++;
1204 		tpllist = (struct statsblob_tpl **)newtpllist;
1205 		tpllist[tpl_id] = (struct statsblob_tpl *)tpl;
1206 		stats_tpl_update_hash(tpllist[tpl_id]);
1207 	} else {
1208 		stats_free(tpl);
1209 		stats_free(tpl_sb);
1210 		if (tpl_mb != NULL) {
1211 			stats_free(tpl_mb->tplname);
1212 			stats_free(tpl_mb);
1213 		}
1214 		tpl_id = -ENOMEM;
1215 	}
1216 	TPL_LIST_WUNLOCK();
1217 
1218 	return (tpl_id);
1219 }
1220 
1221 int
1222 stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name,
1223     enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss,
1224     uint32_t flags)
1225 {
1226 	struct voi *voi;
1227 	struct voistat *tmpstat;
1228 	struct statsblobv1 *tpl_sb;
1229 	struct metablob *tpl_mb;
1230 	int error, i, newstatdataidx, newvoibytes, newvoistatbytes,
1231 	    newvoistatdatabytes, newvoistatmaxid;
1232 	uint32_t nbytes;
1233 
1234 	if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES ||
1235 	    nvss == 0 || vss == NULL)
1236 		return (EINVAL);
1237 
1238 	error = nbytes = newvoibytes = newvoistatbytes =
1239 	    newvoistatdatabytes = 0;
1240 	newvoistatmaxid = -1;
1241 
1242 	/* Calculate the number of bytes required for the new voistats. */
1243 	for (i = nvss - 1; i >= 0; i--) {
1244 		if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES ||
1245 		    vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES ||
1246 		    vss[i].iv == NULL || vss[i].vsdsz == 0)
1247 			return (EINVAL);
1248 		if ((int)vss[i].stype > newvoistatmaxid)
1249 			newvoistatmaxid = vss[i].stype;
1250 		newvoistatdatabytes += vss[i].vsdsz;
1251 	}
1252 
1253 	if (flags & SB_VOI_RELUPDATE) {
1254 		/* XXXLAS: VOI state bytes may need to vary based on stat types. */
1255 		newvoistatdatabytes += sizeof(struct voistatdata_voistate);
1256 	}
1257 	nbytes += newvoistatdatabytes;
1258 
1259 	TPL_LIST_WLOCK();
1260 	if (tpl_id < ntpl) {
1261 		tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb;
1262 		tpl_mb = tpllist[tpl_id]->mb;
1263 
1264 		if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) {
1265 			/* Adding a new VOI and associated stats. */
1266 			if (voi_id >= NVOIS(tpl_sb)) {
1267 				/* We need to grow the tpl_sb->vois array. */
1268 				newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) *
1269 				    sizeof(struct voi);
1270 				nbytes += newvoibytes;
1271 			}
1272 			newvoistatbytes =
1273 			    (newvoistatmaxid + 1) * sizeof(struct voistat);
1274 		} else {
1275 			/* Adding stats to an existing VOI. */
1276 			if (newvoistatmaxid >
1277 			    tpl_sb->vois[voi_id].voistatmaxid) {
1278 				newvoistatbytes = (newvoistatmaxid -
1279 				    tpl_sb->vois[voi_id].voistatmaxid) *
1280 				    sizeof(struct voistat);
1281 			}
1282 			/* XXXLAS: KPI does not yet support expanding VOIs. */
1283 			error = EOPNOTSUPP;
1284 		}
1285 		nbytes += newvoistatbytes;
1286 
1287 		if (!error && newvoibytes > 0) {
1288 			struct voi_meta *voi_meta = tpl_mb->voi_meta;
1289 
1290 			voi_meta = stats_realloc(voi_meta, voi_meta == NULL ?
1291 			    0 : NVOIS(tpl_sb) * sizeof(struct voi_meta),
1292 			    (1 + voi_id) * sizeof(struct voi_meta),
1293 			    M_ZERO);
1294 
1295 			if (voi_meta == NULL)
1296 				error = ENOMEM;
1297 			else
1298 				tpl_mb->voi_meta = voi_meta;
1299 		}
1300 
1301 		if (!error) {
1302 			/* NB: Resizing can change where tpl_sb points. */
1303 			error = stats_v1_blob_expand(&tpl_sb, newvoibytes,
1304 			    newvoistatbytes, newvoistatdatabytes);
1305 		}
1306 
1307 		if (!error) {
1308 			tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name,
1309 			    0);
1310 			if (tpl_mb->voi_meta[voi_id].name == NULL)
1311 				error = ENOMEM;
1312 		}
1313 
1314 		if (!error) {
1315 			/* Update the template list with the resized pointer. */
1316 			tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb;
1317 
1318 			/* Update the template. */
1319 			voi = &tpl_sb->vois[voi_id];
1320 
1321 			if (voi->id < 0) {
1322 				/* VOI is new and needs to be initialised. */
1323 				voi->id = voi_id;
1324 				voi->dtype = voi_dtype;
1325 				voi->stats_off = tpl_sb->stats_off;
1326 				if (flags & SB_VOI_RELUPDATE)
1327 					voi->flags |= VOI_REQSTATE;
1328 			} else {
1329 				/*
1330 				 * XXXLAS: When this else block is written, the
1331 				 * "KPI does not yet support expanding VOIs"
1332 				 * error earlier in this function can be
1333 				 * removed. What is required here is to shuffle
1334 				 * the voistat array such that the new stats for
1335 				 * the voi are contiguous, which will displace
1336 				 * stats for other vois that reside after the
1337 				 * voi being updated. The other vois then need
1338 				 * to have their stats_off adjusted post
1339 				 * shuffle.
1340 				 */
1341 			}
1342 
1343 			voi->voistatmaxid = newvoistatmaxid;
1344 			newstatdataidx = 0;
1345 
1346 			if (voi->flags & VOI_REQSTATE) {
1347 				/* Initialise the voistate stat in slot 0. */
1348 				tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off);
1349 				tmpstat->stype = VS_STYPE_VOISTATE;
1350 				tmpstat->flags = 0;
1351 				tmpstat->dtype = VSD_DTYPE_VOISTATE;
1352 				newstatdataidx = tmpstat->dsz =
1353 				    sizeof(struct voistatdata_numeric);
1354 				tmpstat->data_off = tpl_sb->statsdata_off;
1355 			}
1356 
1357 			for (i = 0; (uint32_t)i < nvss; i++) {
1358 				tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off +
1359 				    (vss[i].stype * sizeof(struct voistat)));
1360 				KASSERT(tmpstat->stype < 0, ("voistat %p "
1361 				    "already initialised", tmpstat));
1362 				tmpstat->stype = vss[i].stype;
1363 				tmpstat->flags = vss[i].flags;
1364 				tmpstat->dtype = vss[i].vs_dtype;
1365 				tmpstat->dsz = vss[i].vsdsz;
1366 				tmpstat->data_off = tpl_sb->statsdata_off +
1367 				    newstatdataidx;
1368 				memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off),
1369 				    vss[i].iv, vss[i].vsdsz);
1370 				newstatdataidx += vss[i].vsdsz;
1371 			}
1372 
1373 			/* Update the template version hash. */
1374 			stats_tpl_update_hash(tpllist[tpl_id]);
1375 			/* XXXLAS: Confirm tpl name/hash pair remains unique. */
1376 		}
1377 	} else
1378 		error = EINVAL;
1379 	TPL_LIST_WUNLOCK();
1380 
1381 	return (error);
1382 }
1383 
1384 struct statsblobv1 *
1385 stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused)
1386 {
1387 	struct statsblobv1 *sb;
1388 	int error;
1389 
1390 	sb = NULL;
1391 
1392 	TPL_LIST_RLOCK();
1393 	if (tpl_id < ntpl) {
1394 		sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0);
1395 		if (sb != NULL) {
1396 			sb->maxsz = tpllist[tpl_id]->sb->maxsz;
1397 			error = stats_v1_blob_init_locked(sb, tpl_id, 0);
1398 		} else
1399 			error = ENOMEM;
1400 
1401 		if (error) {
1402 			stats_free(sb);
1403 			sb = NULL;
1404 		}
1405 	}
1406 	TPL_LIST_RUNLOCK();
1407 
1408 	return (sb);
1409 }
1410 
1411 void
1412 stats_v1_blob_destroy(struct statsblobv1 *sb)
1413 {
1414 
1415 	stats_free(sb);
1416 }
1417 
1418 int
1419 stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id,
1420     enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd,
1421     size_t *retvsdsz)
1422 {
1423 	struct voi *v;
1424 	struct voistat *vs;
1425 
1426 	if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 ||
1427 	    voi_id >= NVOIS(sb))
1428 		return (EINVAL);
1429 
1430 	v = &sb->vois[voi_id];
1431 	if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid)
1432 		return (EINVAL);
1433 
1434 	vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat)));
1435 	*retvsd = BLOB_OFFSET(sb, vs->data_off);
1436 	if (retdtype != NULL)
1437 		*retdtype = vs->dtype;
1438 	if (retvsdsz != NULL)
1439 		*retvsdsz = vs->dsz;
1440 
1441 	return (0);
1442 }
1443 
1444 int
1445 stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags)
1446 {
1447 	int error;
1448 
1449 	error = 0;
1450 
1451 	TPL_LIST_RLOCK();
1452 	if (sb == NULL || tpl_id >= ntpl) {
1453 		error = EINVAL;
1454 	} else {
1455 		error = stats_v1_blob_init_locked(sb, tpl_id, flags);
1456 	}
1457 	TPL_LIST_RUNLOCK();
1458 
1459 	return (error);
1460 }
1461 
1462 static inline int
1463 stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
1464     uint32_t flags __unused)
1465 {
1466 	int error;
1467 
1468 	TPL_LIST_RLOCK_ASSERT();
1469 	error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW;
1470 	KASSERT(!error,
1471 	    ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz));
1472 
1473 	if (!error) {
1474 		memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz);
1475 		sb->created = sb->lastrst = stats_sbinuptime();
1476 		sb->tplhash = tpllist[tpl_id]->mb->tplhash;
1477 	}
1478 
1479 	return (error);
1480 }
1481 
1482 static int
1483 stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
1484     int newvoistatbytes, int newvoistatdatabytes)
1485 {
1486 	struct statsblobv1 *sb;
1487 	struct voi *tmpvoi;
1488 	struct voistat *tmpvoistat, *voistat_array;
1489 	int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats;
1490 
1491 	KASSERT(newvoibytes % sizeof(struct voi) == 0,
1492 	    ("Bad newvoibytes %d", newvoibytes));
1493 	KASSERT(newvoistatbytes % sizeof(struct voistat) == 0,
1494 	    ("Bad newvoistatbytes %d", newvoistatbytes));
1495 
1496 	error = ((newvoibytes % sizeof(struct voi) == 0) &&
1497 	    (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL;
1498 	sb = *sbpp;
1499 	nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes;
1500 
1501 	/*
1502 	 * XXXLAS: Required until we gain support for flags which alter the
1503 	 * units of size/offset fields in key structs.
1504 	 */
1505 	if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ))
1506 		error = EFBIG;
1507 
1508 	if (!error && (sb->cursz + nbytes > sb->maxsz)) {
1509 		/* Need to expand our blob. */
1510 		sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO);
1511 		if (sb != NULL) {
1512 			sb->maxsz = sb->cursz + nbytes;
1513 			*sbpp = sb;
1514 		} else
1515 		    error = ENOMEM;
1516 	}
1517 
1518 	if (!error) {
1519 		/*
1520 		 * Shuffle memory within the expanded blob working from the end
1521 		 * backwards, leaving gaps for the new voistat and voistatdata
1522 		 * structs at the beginning of their respective blob regions,
1523 		 * and for the new voi structs at the end of their blob region.
1524 		 */
1525 		memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes),
1526 		    BLOB_OFFSET(sb, sb->statsdata_off),
1527 		    sb->cursz - sb->statsdata_off);
1528 		memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes +
1529 		    newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off),
1530 		    sb->statsdata_off - sb->stats_off);
1531 
1532 		/* First index of new voi/voistat structs to be initialised. */
1533 		idxnewvois = NVOIS(sb);
1534 		idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1;
1535 
1536 		/* Update housekeeping variables and offsets. */
1537 		sb->cursz += nbytes;
1538 		sb->stats_off += newvoibytes;
1539 		sb->statsdata_off += newvoibytes + newvoistatbytes;
1540 
1541 		/* XXXLAS: Zeroing not strictly needed but aids debugging. */
1542 		memset(&sb->vois[idxnewvois], '\0', newvoibytes);
1543 		memset(BLOB_OFFSET(sb, sb->stats_off), '\0',
1544 		    newvoistatbytes);
1545 		memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0',
1546 		    newvoistatdatabytes);
1547 
1548 		/* Initialise new voi array members and update offsets. */
1549 		for (i = 0; i < NVOIS(sb); i++) {
1550 			tmpvoi = &sb->vois[i];
1551 			if (i >= idxnewvois) {
1552 				tmpvoi->id = tmpvoi->voistatmaxid = -1;
1553 			} else if (tmpvoi->id > -1) {
1554 				tmpvoi->stats_off += newvoibytes +
1555 				    newvoistatbytes;
1556 			}
1557 		}
1558 
1559 		/* Initialise new voistat array members and update offsets. */
1560 		nvoistats = (sb->statsdata_off - sb->stats_off) /
1561 		    sizeof(struct voistat);
1562 		voistat_array = BLOB_OFFSET(sb, sb->stats_off);
1563 		for (i = 0; i < nvoistats; i++) {
1564 			tmpvoistat = &voistat_array[i];
1565 			if (i <= idxnewvoistats) {
1566 				tmpvoistat->stype = -1;
1567 			} else if (tmpvoistat->stype > -1) {
1568 				tmpvoistat->data_off += nbytes;
1569 			}
1570 		}
1571 	}
1572 
1573 	return (error);
1574 }
1575 
1576 static void
1577 stats_v1_blob_finalise(struct statsblobv1 *sb __unused)
1578 {
1579 
1580 	/* XXXLAS: Fill this in. */
1581 }
1582 
1583 static void
1584 stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb,
1585     void *usrctx, uint32_t flags)
1586 {
1587 	struct voi *v;
1588 	struct voistat *vs;
1589 	struct sb_iter_ctx ctx;
1590 	int i, j, firstvoi;
1591 
1592 	ctx.usrctx = usrctx;
1593 	ctx.flags = SB_IT_FIRST_CB;
1594 	firstvoi = 1;
1595 
1596 	for (i = 0; i < NVOIS(sb); i++) {
1597 		v = &sb->vois[i];
1598 		ctx.vslot = i;
1599 		ctx.vsslot = -1;
1600 		ctx.flags |= SB_IT_FIRST_VOISTAT;
1601 
1602 		if (firstvoi)
1603 			ctx.flags |= SB_IT_FIRST_VOI;
1604 		else if (i == (NVOIS(sb) - 1))
1605 			ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB;
1606 
1607 		if (v->id < 0 && (flags & SB_IT_NULLVOI)) {
1608 			if (icb(sb, v, NULL, &ctx))
1609 				return;
1610 			firstvoi = 0;
1611 			ctx.flags &= ~SB_IT_FIRST_CB;
1612 		}
1613 
1614 		/* If NULL voi, v->voistatmaxid == -1 */
1615 		for (j = 0; j <= v->voistatmaxid; j++) {
1616 			vs = &((struct voistat *)BLOB_OFFSET(sb,
1617 			    v->stats_off))[j];
1618 			if (vs->stype < 0 &&
1619 			    !(flags & SB_IT_NULLVOISTAT))
1620 				continue;
1621 
1622 			if (j == v->voistatmaxid) {
1623 				ctx.flags |= SB_IT_LAST_VOISTAT;
1624 				if (i == (NVOIS(sb) - 1))
1625 					ctx.flags |=
1626 					    SB_IT_LAST_CB;
1627 			} else
1628 				ctx.flags &= ~SB_IT_LAST_CB;
1629 
1630 			ctx.vsslot = j;
1631 			if (icb(sb, v, vs, &ctx))
1632 				return;
1633 
1634 			ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT |
1635 			    SB_IT_LAST_VOISTAT);
1636 		}
1637 		ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI);
1638 	}
1639 }
1640 
1641 static inline void
1642 stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused,
1643     const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype,
1644     size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1645 {
1646 	const struct ctdth32 *ctd32tree;
1647 	const struct ctdth64 *ctd64tree;
1648 	const struct voistatdata_tdgstctd32 *ctd32;
1649 	const struct voistatdata_tdgstctd64 *ctd64;
1650 	const char *fmtstr;
1651 	uint64_t smplcnt, compcnt;
1652 	int is32bit, qmaxstrlen;
1653 	uint16_t maxctds, curctds;
1654 
1655 	switch (tdgst_dtype) {
1656 	case VSD_DTYPE_TDGSTCLUST32:
1657 		smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt;
1658 		compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt;
1659 		maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1660 		curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1661 		ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree;
1662 		ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) :
1663 		    ARB_CMIN(ctdth32, ctd32tree));
1664 		qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10);
1665 		is32bit = 1;
1666 		ctd64tree = NULL;
1667 		ctd64 = NULL;
1668 		break;
1669 	case VSD_DTYPE_TDGSTCLUST64:
1670 		smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt;
1671 		compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt;
1672 		maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1673 		curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1674 		ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree;
1675 		ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) :
1676 		    ARB_CMIN(ctdth64, ctd64tree));
1677 		qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10);
1678 		is32bit = 0;
1679 		ctd32tree = NULL;
1680 		ctd32 = NULL;
1681 		break;
1682 	default:
1683 		return;
1684 	}
1685 
1686 	switch (fmt) {
1687 	case SB_STRFMT_FREEFORM:
1688 		fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu";
1689 		break;
1690 	case SB_STRFMT_JSON:
1691 	default:
1692 		fmtstr =
1693 		    "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu,"
1694 		    "\"nctds\":%hu,\"ctds\":[";
1695 		break;
1696 	}
1697 	sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt,
1698 	    maxctds, curctds);
1699 
1700 	while ((is32bit ? NULL != ctd32 : NULL != ctd64)) {
1701 		char qstr[qmaxstrlen];
1702 
1703 		switch (fmt) {
1704 		case SB_STRFMT_FREEFORM:
1705 			fmtstr = "\n\t\t\t\t";
1706 			break;
1707 		case SB_STRFMT_JSON:
1708 		default:
1709 			fmtstr = "{";
1710 			break;
1711 		}
1712 		sbuf_cat(buf, fmtstr);
1713 
1714 		if (objdump) {
1715 			switch (fmt) {
1716 			case SB_STRFMT_FREEFORM:
1717 				fmtstr = "ctd[%hu].";
1718 				break;
1719 			case SB_STRFMT_JSON:
1720 			default:
1721 				fmtstr = "\"ctd\":%hu,";
1722 				break;
1723 			}
1724 			sbuf_printf(buf, fmtstr, is32bit ?
1725 			    ARB_SELFIDX(ctd32tree, ctd32) :
1726 			    ARB_SELFIDX(ctd64tree, ctd64));
1727 		}
1728 
1729 		switch (fmt) {
1730 		case SB_STRFMT_FREEFORM:
1731 			fmtstr = "{mu=";
1732 			break;
1733 		case SB_STRFMT_JSON:
1734 		default:
1735 			fmtstr = "\"mu\":";
1736 			break;
1737 		}
1738 		sbuf_cat(buf, fmtstr);
1739 		Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr,
1740 		    sizeof(qstr));
1741 		sbuf_cat(buf, qstr);
1742 
1743 		switch (fmt) {
1744 		case SB_STRFMT_FREEFORM:
1745 			fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1746 			break;
1747 		case SB_STRFMT_JSON:
1748 		default:
1749 			fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1750 			break;
1751 		}
1752 		sbuf_printf(buf, fmtstr,
1753 		    is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt);
1754 
1755 		if (is32bit)
1756 			ctd32 = (objdump ? ARB_CNODE(ctd32tree,
1757 			    ARB_SELFIDX(ctd32tree, ctd32) + 1) :
1758 			    ARB_CNEXT(ctdth32, ctd32tree, ctd32));
1759 		else
1760 			ctd64 = (objdump ? ARB_CNODE(ctd64tree,
1761 			    ARB_SELFIDX(ctd64tree, ctd64) + 1) :
1762 			    ARB_CNEXT(ctdth64, ctd64tree, ctd64));
1763 
1764 		if (fmt == SB_STRFMT_JSON &&
1765 		    (is32bit ? NULL != ctd32 : NULL != ctd64))
1766 			sbuf_putc(buf, ',');
1767 	}
1768 	if (fmt == SB_STRFMT_JSON)
1769 		sbuf_cat(buf, "]");
1770 }
1771 
1772 static inline void
1773 stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype,
1774     const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype,
1775     size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1776 {
1777 	const struct voistatdata_numeric *bkt_lb, *bkt_ub;
1778 	const char *fmtstr;
1779 	int is32bit;
1780 	uint16_t i, nbkts;
1781 
1782 	switch (hist_dtype) {
1783 	case VSD_DTYPE_CRHIST32:
1784 		nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz);
1785 		is32bit = 1;
1786 		break;
1787 	case VSD_DTYPE_DRHIST32:
1788 		nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz);
1789 		is32bit = 1;
1790 		break;
1791 	case VSD_DTYPE_DVHIST32:
1792 		nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz);
1793 		is32bit = 1;
1794 		break;
1795 	case VSD_DTYPE_CRHIST64:
1796 		nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz);
1797 		is32bit = 0;
1798 		break;
1799 	case VSD_DTYPE_DRHIST64:
1800 		nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz);
1801 		is32bit = 0;
1802 		break;
1803 	case VSD_DTYPE_DVHIST64:
1804 		nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz);
1805 		is32bit = 0;
1806 		break;
1807 	default:
1808 		return;
1809 	}
1810 
1811 	switch (fmt) {
1812 	case SB_STRFMT_FREEFORM:
1813 		fmtstr = "nbkts=%hu, ";
1814 		break;
1815 	case SB_STRFMT_JSON:
1816 	default:
1817 		fmtstr = "\"nbkts\":%hu,";
1818 		break;
1819 	}
1820 	sbuf_printf(buf, fmtstr, nbkts);
1821 
1822 	switch (fmt) {
1823 		case SB_STRFMT_FREEFORM:
1824 			fmtstr = (is32bit ? "oob=%u" : "oob=%ju");
1825 			break;
1826 		case SB_STRFMT_JSON:
1827 		default:
1828 			fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" :
1829 			    "\"oob\":%ju,\"bkts\":[");
1830 			break;
1831 	}
1832 	sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist,
1833 	    hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist,
1834 	    hist_dtype, oob));
1835 
1836 	for (i = 0; i < nbkts; i++) {
1837 		switch (hist_dtype) {
1838 		case VSD_DTYPE_CRHIST32:
1839 		case VSD_DTYPE_CRHIST64:
1840 			bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype,
1841 			    bkts[i].lb);
1842 			if (i < nbkts - 1)
1843 				bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist,
1844 				    hist_dtype, bkts[i + 1].lb);
1845 			else
1846 				bkt_ub = &numeric_limits[LIM_MAX][voi_dtype];
1847 			break;
1848 		case VSD_DTYPE_DRHIST32:
1849 		case VSD_DTYPE_DRHIST64:
1850 			bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1851 			    bkts[i].lb);
1852 			bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1853 			    bkts[i].ub);
1854 			break;
1855 		case VSD_DTYPE_DVHIST32:
1856 		case VSD_DTYPE_DVHIST64:
1857 			bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist,
1858 			    hist_dtype, bkts[i].val);
1859 			break;
1860 		default:
1861 			break;
1862 		}
1863 
1864 		switch (fmt) {
1865 		case SB_STRFMT_FREEFORM:
1866 			fmtstr = "\n\t\t\t\t";
1867 			break;
1868 		case SB_STRFMT_JSON:
1869 		default:
1870 			fmtstr = "{";
1871 			break;
1872 		}
1873 		sbuf_cat(buf, fmtstr);
1874 
1875 		if (objdump) {
1876 			switch (fmt) {
1877 			case SB_STRFMT_FREEFORM:
1878 				fmtstr = "bkt[%hu].";
1879 				break;
1880 			case SB_STRFMT_JSON:
1881 			default:
1882 				fmtstr = "\"bkt\":%hu,";
1883 				break;
1884 			}
1885 			sbuf_printf(buf, fmtstr, i);
1886 		}
1887 
1888 		switch (fmt) {
1889 		case SB_STRFMT_FREEFORM:
1890 			fmtstr = "{lb=";
1891 			break;
1892 		case SB_STRFMT_JSON:
1893 		default:
1894 			fmtstr = "\"lb\":";
1895 			break;
1896 		}
1897 		sbuf_cat(buf, fmtstr);
1898 		stats_voistatdata_tostr((const struct voistatdata *)bkt_lb,
1899 		    voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1900 		    fmt, buf, objdump);
1901 
1902 		switch (fmt) {
1903 		case SB_STRFMT_FREEFORM:
1904 			fmtstr = ",ub=";
1905 			break;
1906 		case SB_STRFMT_JSON:
1907 		default:
1908 			fmtstr = ",\"ub\":";
1909 			break;
1910 		}
1911 		sbuf_cat(buf, fmtstr);
1912 		stats_voistatdata_tostr((const struct voistatdata *)bkt_ub,
1913 		    voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1914 		    fmt, buf, objdump);
1915 
1916 		switch (fmt) {
1917 		case SB_STRFMT_FREEFORM:
1918 			fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1919 			break;
1920 		case SB_STRFMT_JSON:
1921 		default:
1922 			fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1923 			break;
1924 		}
1925 		sbuf_printf(buf, fmtstr, is32bit ?
1926 		    VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) :
1927 		    (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype,
1928 		    bkts[i].cnt));
1929 
1930 		if (fmt == SB_STRFMT_JSON && i < nbkts - 1)
1931 			sbuf_putc(buf, ',');
1932 	}
1933 	if (fmt == SB_STRFMT_JSON)
1934 		sbuf_cat(buf, "]");
1935 }
1936 
1937 int
1938 stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype,
1939     enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt,
1940     struct sbuf *buf, int objdump)
1941 {
1942 	const char *fmtstr;
1943 
1944 	if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES ||
1945 	    vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS)
1946 		return (EINVAL);
1947 
1948 	switch (vsd_dtype) {
1949 	case VSD_DTYPE_VOISTATE:
1950 		switch (fmt) {
1951 		case SB_STRFMT_FREEFORM:
1952 			fmtstr = "prev=";
1953 			break;
1954 		case SB_STRFMT_JSON:
1955 		default:
1956 			fmtstr = "\"prev\":";
1957 			break;
1958 		}
1959 		sbuf_cat(buf, fmtstr);
1960 		/*
1961 		 * Render prev by passing it as *vsd and voi_dtype as vsd_dtype.
1962 		 */
1963 		stats_voistatdata_tostr(
1964 		    (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev,
1965 		    voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump);
1966 		break;
1967 	case VSD_DTYPE_INT_S32:
1968 		sbuf_printf(buf, "%d", vsd->int32.s32);
1969 		break;
1970 	case VSD_DTYPE_INT_U32:
1971 		sbuf_printf(buf, "%u", vsd->int32.u32);
1972 		break;
1973 	case VSD_DTYPE_INT_S64:
1974 		sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64);
1975 		break;
1976 	case VSD_DTYPE_INT_U64:
1977 		sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64);
1978 		break;
1979 	case VSD_DTYPE_INT_SLONG:
1980 		sbuf_printf(buf, "%ld", vsd->intlong.slong);
1981 		break;
1982 	case VSD_DTYPE_INT_ULONG:
1983 		sbuf_printf(buf, "%lu", vsd->intlong.ulong);
1984 		break;
1985 	case VSD_DTYPE_Q_S32:
1986 		{
1987 		char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)];
1988 		Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr));
1989 		sbuf_cat(buf, qstr);
1990 		}
1991 		break;
1992 	case VSD_DTYPE_Q_U32:
1993 		{
1994 		char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)];
1995 		Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr));
1996 		sbuf_cat(buf, qstr);
1997 		}
1998 		break;
1999 	case VSD_DTYPE_Q_S64:
2000 		{
2001 		char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)];
2002 		Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr));
2003 		sbuf_cat(buf, qstr);
2004 		}
2005 		break;
2006 	case VSD_DTYPE_Q_U64:
2007 		{
2008 		char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)];
2009 		Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr));
2010 		sbuf_cat(buf, qstr);
2011 		}
2012 		break;
2013 	case VSD_DTYPE_CRHIST32:
2014 	case VSD_DTYPE_DRHIST32:
2015 	case VSD_DTYPE_DVHIST32:
2016 	case VSD_DTYPE_CRHIST64:
2017 	case VSD_DTYPE_DRHIST64:
2018 	case VSD_DTYPE_DVHIST64:
2019 		stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd),
2020 		    vsd_dtype, vsd_sz, fmt, buf, objdump);
2021 		break;
2022 	case VSD_DTYPE_TDGSTCLUST32:
2023 	case VSD_DTYPE_TDGSTCLUST64:
2024 		stats_voistatdata_tdgst_tostr(voi_dtype,
2025 		    CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf,
2026 		    objdump);
2027 		break;
2028 	default:
2029 		break;
2030 	}
2031 
2032 	return (sbuf_error(buf));
2033 }
2034 
2035 static void
2036 stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v,
2037     struct voistat *vs, struct sb_iter_ctx *ctx)
2038 {
2039 	struct sb_tostrcb_ctx *sctx;
2040 	struct metablob *tpl_mb;
2041 	struct sbuf *buf;
2042 	void *vsd;
2043 	uint8_t dump;
2044 
2045 	sctx = ctx->usrctx;
2046 	buf = sctx->buf;
2047 	tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2048 	dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2049 
2050 	if (ctx->flags & SB_IT_FIRST_CB) {
2051 		sbuf_printf(buf, "struct statsblobv1@%p", sb);
2052 		if (dump) {
2053 			sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, "
2054 			    "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, "
2055 			    "stats_off=%hu, statsdata_off=%hu",
2056 			    sb->abi, sb->endian, sb->maxsz, sb->cursz,
2057 			    sb->created, sb->lastrst, sb->flags, sb->stats_off,
2058 			    sb->statsdata_off);
2059 		}
2060 		sbuf_printf(buf, ", tplhash=%u", sb->tplhash);
2061 	}
2062 
2063 	if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2064 		sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id);
2065 		if (v->id < 0)
2066 			return;
2067 		sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" :
2068 		    tpl_mb->voi_meta[v->id].name);
2069 		if (dump)
2070 		    sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, "
2071 		    "voistatmaxid=%hhd, stats_off=%hu", v->flags,
2072 		    vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off);
2073 	}
2074 
2075 	if (!dump && vs->stype <= 0)
2076 		return;
2077 
2078 	sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot);
2079 	if (vs->stype < 0) {
2080 		sbuf_printf(buf, "%hhd", vs->stype);
2081 		return;
2082 	} else
2083 		sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype],
2084 		    vs->errs);
2085 	vsd = BLOB_OFFSET(sb, vs->data_off);
2086 	if (dump)
2087 		sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, "
2088 		    "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype],
2089 		    vs->dsz, vs->data_off);
2090 
2091 	sbuf_cat(buf, "\n\t\t\tvoistatdata: ");
2092 	stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2093 	    sctx->fmt, buf, dump);
2094 }
2095 
2096 static void
2097 stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2098     struct sb_iter_ctx *ctx)
2099 {
2100 	struct sb_tostrcb_ctx *sctx;
2101 	struct metablob *tpl_mb;
2102 	struct sbuf *buf;
2103 	const char *fmtstr;
2104 	void *vsd;
2105 	uint8_t dump;
2106 
2107 	sctx = ctx->usrctx;
2108 	buf = sctx->buf;
2109 	tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2110 	dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2111 
2112 	if (ctx->flags & SB_IT_FIRST_CB) {
2113 		sbuf_putc(buf, '{');
2114 		if (dump) {
2115 			sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu,"
2116 			    "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd,"
2117 			    "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu,"
2118 			    "\"statsdata_off\":%hu,", sb->abi,
2119 			    sb->endian, sb->maxsz, sb->cursz, sb->created,
2120 			    sb->lastrst, sb->flags, sb->stats_off,
2121 			    sb->statsdata_off);
2122 		}
2123 
2124 		if (tpl_mb == NULL)
2125 			fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{";
2126 		else
2127 			fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{";
2128 
2129 		sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null",
2130 		    sb->tplhash);
2131 	}
2132 
2133 	if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2134 		if (dump) {
2135 			sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot,
2136 			    v->id);
2137 			if (v->id < 0) {
2138 				sbuf_cat(buf, "},");
2139 				return;
2140 			}
2141 
2142 			if (tpl_mb == NULL)
2143 				fmtstr = ",\"name\":%s,\"flags\":%hu,"
2144 				    "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2145 				    "\"stats_off\":%hu,";
2146 			else
2147 				fmtstr = ",\"name\":\"%s\",\"flags\":%hu,"
2148 				    "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2149 				    "\"stats_off\":%hu,";
2150 
2151 			sbuf_printf(buf, fmtstr, tpl_mb ?
2152 			    tpl_mb->voi_meta[v->id].name : "null", v->flags,
2153 			    vsd_dtype2name[v->dtype], v->voistatmaxid,
2154 			    v->stats_off);
2155 		} else {
2156 			if (tpl_mb == NULL) {
2157 				sbuf_printf(buf, "\"[%hd]\":{", v->id);
2158 			} else {
2159 				sbuf_printf(buf, "\"%s\":{",
2160 				    tpl_mb->voi_meta[v->id].name);
2161 			}
2162 		}
2163 		sbuf_cat(buf, "\"stats\":{");
2164 	}
2165 
2166 	vsd = BLOB_OFFSET(sb, vs->data_off);
2167 	if (dump) {
2168 		sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot);
2169 		if (vs->stype < 0) {
2170 			sbuf_cat(buf, "{\"stype\":-1},");
2171 			return;
2172 		}
2173 		sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu,"
2174 		    "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{",
2175 		    vs_stype2name[vs->stype], vs->errs, vs->flags,
2176 		    vsd_dtype2name[vs->dtype], vs->data_off);
2177 	} else if (vs->stype > 0) {
2178 		if (tpl_mb == NULL)
2179 			sbuf_printf(buf, "\"[%hhd]\":", vs->stype);
2180 		else
2181 			sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]);
2182 	} else
2183 		return;
2184 
2185 	if ((vs->flags & VS_VSDVALID) || dump) {
2186 		if (!dump)
2187 			sbuf_printf(buf, "{\"errs\":%hu,", vs->errs);
2188 		/* Simple non-compound VSD types need a key. */
2189 		if (!vsd_compoundtype[vs->dtype])
2190 			sbuf_cat(buf, "\"val\":");
2191 		stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2192 		    sctx->fmt, buf, dump);
2193 		sbuf_cat(buf, dump ? "}}" : "}");
2194 	} else
2195 		sbuf_cat(buf, dump ? "null}" : "null");
2196 
2197 	if (ctx->flags & SB_IT_LAST_VOISTAT)
2198 		sbuf_cat(buf, "}}");
2199 
2200 	if (ctx->flags & SB_IT_LAST_CB)
2201 		sbuf_cat(buf, "}}");
2202 	else
2203 		sbuf_putc(buf, ',');
2204 }
2205 
2206 static int
2207 stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2208     struct sb_iter_ctx *ctx)
2209 {
2210 	struct sb_tostrcb_ctx *sctx;
2211 
2212 	sctx = ctx->usrctx;
2213 
2214 	switch (sctx->fmt) {
2215 	case SB_STRFMT_FREEFORM:
2216 		stats_v1_itercb_tostr_freeform(sb, v, vs, ctx);
2217 		break;
2218 	case SB_STRFMT_JSON:
2219 		stats_v1_itercb_tostr_json(sb, v, vs, ctx);
2220 		break;
2221 	default:
2222 		break;
2223 	}
2224 
2225 	return (sbuf_error(sctx->buf));
2226 }
2227 
2228 int
2229 stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf,
2230     enum sb_str_fmt fmt, uint32_t flags)
2231 {
2232 	struct sb_tostrcb_ctx sctx;
2233 	uint32_t iflags;
2234 
2235 	if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL ||
2236 	    fmt >= SB_STRFMT_NUM_FMTS)
2237 		return (EINVAL);
2238 
2239 	sctx.buf = buf;
2240 	sctx.fmt = fmt;
2241 	sctx.flags = flags;
2242 
2243 	if (flags & SB_TOSTR_META) {
2244 		if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash),
2245 		    &sctx.tpl))
2246 			return (EINVAL);
2247 	} else
2248 		sctx.tpl = NULL;
2249 
2250 	iflags = 0;
2251 	if (flags & SB_TOSTR_OBJDUMP)
2252 		iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT);
2253 	stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags);
2254 
2255 	return (sbuf_error(buf));
2256 }
2257 
2258 static int
2259 stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v,
2260     struct voistat *vs, struct sb_iter_ctx *ctx)
2261 {
2262 	struct sb_visitcb_ctx *vctx;
2263 	struct sb_visit sbv;
2264 
2265 	vctx = ctx->usrctx;
2266 
2267 	sbv.tplhash = sb->tplhash;
2268 	sbv.voi_id = v->id;
2269 	sbv.voi_dtype = v->dtype;
2270 	sbv.vs_stype = vs->stype;
2271 	sbv.vs_dtype = vs->dtype;
2272 	sbv.vs_dsz = vs->dsz;
2273 	sbv.vs_data = BLOB_OFFSET(sb, vs->data_off);
2274 	sbv.vs_errs = vs->errs;
2275 	sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB |
2276 	    SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
2277 	    SB_IT_LAST_VOISTAT);
2278 
2279 	return (vctx->cb(&sbv, vctx->usrctx));
2280 }
2281 
2282 int
2283 stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func,
2284     void *usrctx)
2285 {
2286 	struct sb_visitcb_ctx vctx;
2287 
2288 	if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL)
2289 		return (EINVAL);
2290 
2291 	vctx.cb = func;
2292 	vctx.usrctx = usrctx;
2293 
2294 	stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0);
2295 
2296 	return (0);
2297 }
2298 
2299 static int
2300 stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused,
2301     struct voistat *vs, struct sb_iter_ctx *ctx __unused)
2302 {
2303 	void *vsd;
2304 
2305 	if (vs->stype == VS_STYPE_VOISTATE)
2306 		return (0);
2307 
2308 	vsd = BLOB_OFFSET(sb, vs->data_off);
2309 
2310 	/* Perform the stat type's default reset action. */
2311 	switch (vs->stype) {
2312 	case VS_STYPE_SUM:
2313 		switch (vs->dtype) {
2314 		case VSD_DTYPE_Q_S32:
2315 			Q_SIFVAL(VSD(q32, vsd)->sq32, 0);
2316 			break;
2317 		case VSD_DTYPE_Q_U32:
2318 			Q_SIFVAL(VSD(q32, vsd)->uq32, 0);
2319 			break;
2320 		case VSD_DTYPE_Q_S64:
2321 			Q_SIFVAL(VSD(q64, vsd)->sq64, 0);
2322 			break;
2323 		case VSD_DTYPE_Q_U64:
2324 			Q_SIFVAL(VSD(q64, vsd)->uq64, 0);
2325 			break;
2326 		default:
2327 			bzero(vsd, vs->dsz);
2328 			break;
2329 		}
2330 		break;
2331 	case VS_STYPE_MAX:
2332 		switch (vs->dtype) {
2333 		case VSD_DTYPE_Q_S32:
2334 			Q_SIFVAL(VSD(q32, vsd)->sq32,
2335 			    Q_IFMINVAL(VSD(q32, vsd)->sq32));
2336 			break;
2337 		case VSD_DTYPE_Q_U32:
2338 			Q_SIFVAL(VSD(q32, vsd)->uq32,
2339 			    Q_IFMINVAL(VSD(q32, vsd)->uq32));
2340 			break;
2341 		case VSD_DTYPE_Q_S64:
2342 			Q_SIFVAL(VSD(q64, vsd)->sq64,
2343 			    Q_IFMINVAL(VSD(q64, vsd)->sq64));
2344 			break;
2345 		case VSD_DTYPE_Q_U64:
2346 			Q_SIFVAL(VSD(q64, vsd)->uq64,
2347 			    Q_IFMINVAL(VSD(q64, vsd)->uq64));
2348 			break;
2349 		default:
2350 			memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype],
2351 			    vs->dsz);
2352 			break;
2353 		}
2354 		break;
2355 	case VS_STYPE_MIN:
2356 		switch (vs->dtype) {
2357 		case VSD_DTYPE_Q_S32:
2358 			Q_SIFVAL(VSD(q32, vsd)->sq32,
2359 			    Q_IFMAXVAL(VSD(q32, vsd)->sq32));
2360 			break;
2361 		case VSD_DTYPE_Q_U32:
2362 			Q_SIFVAL(VSD(q32, vsd)->uq32,
2363 			    Q_IFMAXVAL(VSD(q32, vsd)->uq32));
2364 			break;
2365 		case VSD_DTYPE_Q_S64:
2366 			Q_SIFVAL(VSD(q64, vsd)->sq64,
2367 			    Q_IFMAXVAL(VSD(q64, vsd)->sq64));
2368 			break;
2369 		case VSD_DTYPE_Q_U64:
2370 			Q_SIFVAL(VSD(q64, vsd)->uq64,
2371 			    Q_IFMAXVAL(VSD(q64, vsd)->uq64));
2372 			break;
2373 		default:
2374 			memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype],
2375 			    vs->dsz);
2376 			break;
2377 		}
2378 		break;
2379 	case VS_STYPE_HIST:
2380 		{
2381 		/* Reset bucket counts. */
2382 		struct voistatdata_hist *hist;
2383 		int i, is32bit;
2384 		uint16_t nbkts;
2385 
2386 		hist = VSD(hist, vsd);
2387 		switch (vs->dtype) {
2388 		case VSD_DTYPE_CRHIST32:
2389 			nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2390 			is32bit = 1;
2391 			break;
2392 		case VSD_DTYPE_DRHIST32:
2393 			nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2394 			is32bit = 1;
2395 			break;
2396 		case VSD_DTYPE_DVHIST32:
2397 			nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2398 			is32bit = 1;
2399 			break;
2400 		case VSD_DTYPE_CRHIST64:
2401 			nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2402 			is32bit = 0;
2403 			break;
2404 		case VSD_DTYPE_DRHIST64:
2405 			nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2406 			is32bit = 0;
2407 			break;
2408 		case VSD_DTYPE_DVHIST64:
2409 			nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2410 			is32bit = 0;
2411 			break;
2412 		default:
2413 			return (0);
2414 		}
2415 
2416 		bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob),
2417 		    is32bit ? sizeof(uint32_t) : sizeof(uint64_t));
2418 		for (i = nbkts - 1; i >= 0; i--) {
2419 			bzero(VSD_HIST_FIELDPTR(hist, vs->dtype,
2420 			    bkts[i].cnt), is32bit ? sizeof(uint32_t) :
2421 			    sizeof(uint64_t));
2422 		}
2423 		break;
2424 		}
2425 	case VS_STYPE_TDGST:
2426 		{
2427 		/* Reset sample count centroids array/tree. */
2428 		struct voistatdata_tdgst *tdgst;
2429 		struct ctdth32 *ctd32tree;
2430 		struct ctdth64 *ctd64tree;
2431 		struct voistatdata_tdgstctd32 *ctd32;
2432 		struct voistatdata_tdgstctd64 *ctd64;
2433 
2434 		tdgst = VSD(tdgst, vsd);
2435 		switch (vs->dtype) {
2436 		case VSD_DTYPE_TDGSTCLUST32:
2437 			VSD(tdgstclust32, tdgst)->smplcnt = 0;
2438 			VSD(tdgstclust32, tdgst)->compcnt = 0;
2439 			ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2440 			ARB_INIT(ctd32, ctdlnk, ctd32tree,
2441 			    ARB_MAXNODES(ctd32tree)) {
2442 				ctd32->cnt = 0;
2443 				Q_SIFVAL(ctd32->mu, 0);
2444 			}
2445 #ifdef DIAGNOSTIC
2446 			RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2447 #endif
2448 		break;
2449 		case VSD_DTYPE_TDGSTCLUST64:
2450 			VSD(tdgstclust64, tdgst)->smplcnt = 0;
2451 			VSD(tdgstclust64, tdgst)->compcnt = 0;
2452 			ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2453 			ARB_INIT(ctd64, ctdlnk, ctd64tree,
2454 			    ARB_MAXNODES(ctd64tree)) {
2455 				ctd64->cnt = 0;
2456 				Q_SIFVAL(ctd64->mu, 0);
2457 			}
2458 #ifdef DIAGNOSTIC
2459 			RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2460 #endif
2461 		break;
2462 		default:
2463 			return (0);
2464 		}
2465 		break;
2466 		}
2467 	default:
2468 		KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
2469 		break;
2470 	}
2471 
2472 	vs->errs = 0;
2473 	vs->flags &= ~VS_VSDVALID;
2474 
2475 	return (0);
2476 }
2477 
2478 int
2479 stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz,
2480     struct statsblobv1 *src, uint32_t flags)
2481 {
2482 	int error;
2483 
2484 	if (src != NULL && src->abi == STATS_ABI_V1) {
2485 		error = stats_v1_blob_clone(dst, dstmaxsz, src, flags);
2486 		if (!error) {
2487 			if (flags & SB_CLONE_RSTSRC) {
2488 				stats_v1_blob_iter(src,
2489 				    stats_v1_icb_reset_voistat, NULL, 0);
2490 				src->lastrst = stats_sbinuptime();
2491 			}
2492 			stats_v1_blob_finalise(*dst);
2493 		}
2494 	} else
2495 		error = EINVAL;
2496 
2497 	return (error);
2498 }
2499 
2500 static inline int
2501 stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused,
2502     struct voistatdata *voival, struct voistat *vs, void *vsd)
2503 {
2504 	int error;
2505 
2506 	KASSERT(vs->dtype < VSD_NUM_DTYPES,
2507 	    ("Unknown VSD dtype %d", vs->dtype));
2508 
2509 	error = 0;
2510 
2511 	switch (vs->dtype) {
2512 	case VSD_DTYPE_INT_S32:
2513 		if (VSD(int32, vsd)->s32 < voival->int32.s32) {
2514 			VSD(int32, vsd)->s32 = voival->int32.s32;
2515 			vs->flags |= VS_VSDVALID;
2516 		}
2517 		break;
2518 	case VSD_DTYPE_INT_U32:
2519 		if (VSD(int32, vsd)->u32 < voival->int32.u32) {
2520 			VSD(int32, vsd)->u32 = voival->int32.u32;
2521 			vs->flags |= VS_VSDVALID;
2522 		}
2523 		break;
2524 	case VSD_DTYPE_INT_S64:
2525 		if (VSD(int64, vsd)->s64 < voival->int64.s64) {
2526 			VSD(int64, vsd)->s64 = voival->int64.s64;
2527 			vs->flags |= VS_VSDVALID;
2528 		}
2529 		break;
2530 	case VSD_DTYPE_INT_U64:
2531 		if (VSD(int64, vsd)->u64 < voival->int64.u64) {
2532 			VSD(int64, vsd)->u64 = voival->int64.u64;
2533 			vs->flags |= VS_VSDVALID;
2534 		}
2535 		break;
2536 	case VSD_DTYPE_INT_SLONG:
2537 		if (VSD(intlong, vsd)->slong < voival->intlong.slong) {
2538 			VSD(intlong, vsd)->slong = voival->intlong.slong;
2539 			vs->flags |= VS_VSDVALID;
2540 		}
2541 		break;
2542 	case VSD_DTYPE_INT_ULONG:
2543 		if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) {
2544 			VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2545 			vs->flags |= VS_VSDVALID;
2546 		}
2547 		break;
2548 	case VSD_DTYPE_Q_S32:
2549 		if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2550 		    (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2551 		    voival->q32.sq32)))) {
2552 			vs->flags |= VS_VSDVALID;
2553 		}
2554 		break;
2555 	case VSD_DTYPE_Q_U32:
2556 		if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2557 		    (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2558 		    voival->q32.uq32)))) {
2559 			vs->flags |= VS_VSDVALID;
2560 		}
2561 		break;
2562 	case VSD_DTYPE_Q_S64:
2563 		if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2564 		    (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2565 		    voival->q64.sq64)))) {
2566 			vs->flags |= VS_VSDVALID;
2567 		}
2568 		break;
2569 	case VSD_DTYPE_Q_U64:
2570 		if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2571 		    (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2572 		    voival->q64.uq64)))) {
2573 			vs->flags |= VS_VSDVALID;
2574 		}
2575 		break;
2576 	default:
2577 		error = EINVAL;
2578 		break;
2579 	}
2580 
2581 	return (error);
2582 }
2583 
2584 static inline int
2585 stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused,
2586     struct voistatdata *voival, struct voistat *vs, void *vsd)
2587 {
2588 	int error;
2589 
2590 	KASSERT(vs->dtype < VSD_NUM_DTYPES,
2591 	    ("Unknown VSD dtype %d", vs->dtype));
2592 
2593 	error = 0;
2594 
2595 	switch (vs->dtype) {
2596 	case VSD_DTYPE_INT_S32:
2597 		if (VSD(int32, vsd)->s32 > voival->int32.s32) {
2598 			VSD(int32, vsd)->s32 = voival->int32.s32;
2599 			vs->flags |= VS_VSDVALID;
2600 		}
2601 		break;
2602 	case VSD_DTYPE_INT_U32:
2603 		if (VSD(int32, vsd)->u32 > voival->int32.u32) {
2604 			VSD(int32, vsd)->u32 = voival->int32.u32;
2605 			vs->flags |= VS_VSDVALID;
2606 		}
2607 		break;
2608 	case VSD_DTYPE_INT_S64:
2609 		if (VSD(int64, vsd)->s64 > voival->int64.s64) {
2610 			VSD(int64, vsd)->s64 = voival->int64.s64;
2611 			vs->flags |= VS_VSDVALID;
2612 		}
2613 		break;
2614 	case VSD_DTYPE_INT_U64:
2615 		if (VSD(int64, vsd)->u64 > voival->int64.u64) {
2616 			VSD(int64, vsd)->u64 = voival->int64.u64;
2617 			vs->flags |= VS_VSDVALID;
2618 		}
2619 		break;
2620 	case VSD_DTYPE_INT_SLONG:
2621 		if (VSD(intlong, vsd)->slong > voival->intlong.slong) {
2622 			VSD(intlong, vsd)->slong = voival->intlong.slong;
2623 			vs->flags |= VS_VSDVALID;
2624 		}
2625 		break;
2626 	case VSD_DTYPE_INT_ULONG:
2627 		if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) {
2628 			VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2629 			vs->flags |= VS_VSDVALID;
2630 		}
2631 		break;
2632 	case VSD_DTYPE_Q_S32:
2633 		if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2634 		    (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2635 		    voival->q32.sq32)))) {
2636 			vs->flags |= VS_VSDVALID;
2637 		}
2638 		break;
2639 	case VSD_DTYPE_Q_U32:
2640 		if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2641 		    (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2642 		    voival->q32.uq32)))) {
2643 			vs->flags |= VS_VSDVALID;
2644 		}
2645 		break;
2646 	case VSD_DTYPE_Q_S64:
2647 		if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2648 		    (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2649 		    voival->q64.sq64)))) {
2650 			vs->flags |= VS_VSDVALID;
2651 		}
2652 		break;
2653 	case VSD_DTYPE_Q_U64:
2654 		if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2655 		    (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2656 		    voival->q64.uq64)))) {
2657 			vs->flags |= VS_VSDVALID;
2658 		}
2659 		break;
2660 	default:
2661 		error = EINVAL;
2662 		break;
2663 	}
2664 
2665 	return (error);
2666 }
2667 
2668 static inline int
2669 stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused,
2670     struct voistatdata *voival, struct voistat *vs, void *vsd)
2671 {
2672 	int error;
2673 
2674 	KASSERT(vs->dtype < VSD_NUM_DTYPES,
2675 	    ("Unknown VSD dtype %d", vs->dtype));
2676 
2677 	error = 0;
2678 
2679 	switch (vs->dtype) {
2680 	case VSD_DTYPE_INT_S32:
2681 		VSD(int32, vsd)->s32 += voival->int32.s32;
2682 		break;
2683 	case VSD_DTYPE_INT_U32:
2684 		VSD(int32, vsd)->u32 += voival->int32.u32;
2685 		break;
2686 	case VSD_DTYPE_INT_S64:
2687 		VSD(int64, vsd)->s64 += voival->int64.s64;
2688 		break;
2689 	case VSD_DTYPE_INT_U64:
2690 		VSD(int64, vsd)->u64 += voival->int64.u64;
2691 		break;
2692 	case VSD_DTYPE_INT_SLONG:
2693 		VSD(intlong, vsd)->slong += voival->intlong.slong;
2694 		break;
2695 	case VSD_DTYPE_INT_ULONG:
2696 		VSD(intlong, vsd)->ulong += voival->intlong.ulong;
2697 		break;
2698 	case VSD_DTYPE_Q_S32:
2699 		error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32);
2700 		break;
2701 	case VSD_DTYPE_Q_U32:
2702 		error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32);
2703 		break;
2704 	case VSD_DTYPE_Q_S64:
2705 		error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64);
2706 		break;
2707 	case VSD_DTYPE_Q_U64:
2708 		error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64);
2709 		break;
2710 	default:
2711 		error = EINVAL;
2712 		break;
2713 	}
2714 
2715 	if (!error)
2716 		vs->flags |= VS_VSDVALID;
2717 
2718 	return (error);
2719 }
2720 
2721 static inline int
2722 stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival,
2723     struct voistat *vs, struct voistatdata_hist *hist)
2724 {
2725 	struct voistatdata_numeric *bkt_lb, *bkt_ub;
2726 	uint64_t *oob64, *cnt64;
2727 	uint32_t *oob32, *cnt32;
2728 	int error, i, found, is32bit, has_ub, eq_only;
2729 
2730 	error = 0;
2731 
2732 	switch (vs->dtype) {
2733 	case VSD_DTYPE_CRHIST32:
2734 		i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2735 		is32bit = 1;
2736 		has_ub = eq_only = 0;
2737 		oob32 = &VSD(crhist32, hist)->oob;
2738 		break;
2739 	case VSD_DTYPE_DRHIST32:
2740 		i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2741 		is32bit = has_ub = 1;
2742 		eq_only = 0;
2743 		oob32 = &VSD(drhist32, hist)->oob;
2744 		break;
2745 	case VSD_DTYPE_DVHIST32:
2746 		i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2747 		is32bit = eq_only = 1;
2748 		has_ub = 0;
2749 		oob32 = &VSD(dvhist32, hist)->oob;
2750 		break;
2751 	case VSD_DTYPE_CRHIST64:
2752 		i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2753 		is32bit = has_ub = eq_only = 0;
2754 		oob64 = &VSD(crhist64, hist)->oob;
2755 		break;
2756 	case VSD_DTYPE_DRHIST64:
2757 		i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2758 		is32bit = eq_only = 0;
2759 		has_ub = 1;
2760 		oob64 = &VSD(drhist64, hist)->oob;
2761 		break;
2762 	case VSD_DTYPE_DVHIST64:
2763 		i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2764 		is32bit = has_ub = 0;
2765 		eq_only = 1;
2766 		oob64 = &VSD(dvhist64, hist)->oob;
2767 		break;
2768 	default:
2769 		return (EINVAL);
2770 	}
2771 	i--; /* Adjust for 0-based array index. */
2772 
2773 	/* XXXLAS: Should probably use a better bucket search algorithm. ARB? */
2774 	for (found = 0; i >= 0 && !found; i--) {
2775 		switch (vs->dtype) {
2776 		case VSD_DTYPE_CRHIST32:
2777 			bkt_lb = &VSD(crhist32, hist)->bkts[i].lb;
2778 			cnt32 = &VSD(crhist32, hist)->bkts[i].cnt;
2779 			break;
2780 		case VSD_DTYPE_DRHIST32:
2781 			bkt_lb = &VSD(drhist32, hist)->bkts[i].lb;
2782 			bkt_ub = &VSD(drhist32, hist)->bkts[i].ub;
2783 			cnt32 = &VSD(drhist32, hist)->bkts[i].cnt;
2784 			break;
2785 		case VSD_DTYPE_DVHIST32:
2786 			bkt_lb = &VSD(dvhist32, hist)->bkts[i].val;
2787 			cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt;
2788 			break;
2789 		case VSD_DTYPE_CRHIST64:
2790 			bkt_lb = &VSD(crhist64, hist)->bkts[i].lb;
2791 			cnt64 = &VSD(crhist64, hist)->bkts[i].cnt;
2792 			break;
2793 		case VSD_DTYPE_DRHIST64:
2794 			bkt_lb = &VSD(drhist64, hist)->bkts[i].lb;
2795 			bkt_ub = &VSD(drhist64, hist)->bkts[i].ub;
2796 			cnt64 = &VSD(drhist64, hist)->bkts[i].cnt;
2797 			break;
2798 		case VSD_DTYPE_DVHIST64:
2799 			bkt_lb = &VSD(dvhist64, hist)->bkts[i].val;
2800 			cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt;
2801 			break;
2802 		default:
2803 			return (EINVAL);
2804 		}
2805 
2806 		switch (voi_dtype) {
2807 		case VSD_DTYPE_INT_S32:
2808 			if (voival->int32.s32 >= bkt_lb->int32.s32) {
2809 				if ((eq_only && voival->int32.s32 ==
2810 				    bkt_lb->int32.s32) ||
2811 				    (!eq_only && (!has_ub ||
2812 				    voival->int32.s32 < bkt_ub->int32.s32)))
2813 					found = 1;
2814 			}
2815 			break;
2816 		case VSD_DTYPE_INT_U32:
2817 			if (voival->int32.u32 >= bkt_lb->int32.u32) {
2818 				if ((eq_only && voival->int32.u32 ==
2819 				    bkt_lb->int32.u32) ||
2820 				    (!eq_only && (!has_ub ||
2821 				    voival->int32.u32 < bkt_ub->int32.u32)))
2822 					found = 1;
2823 			}
2824 			break;
2825 		case VSD_DTYPE_INT_S64:
2826 			if (voival->int64.s64 >= bkt_lb->int64.s64)
2827 				if ((eq_only && voival->int64.s64 ==
2828 				    bkt_lb->int64.s64) ||
2829 				    (!eq_only && (!has_ub ||
2830 				    voival->int64.s64 < bkt_ub->int64.s64)))
2831 					found = 1;
2832 			break;
2833 		case VSD_DTYPE_INT_U64:
2834 			if (voival->int64.u64 >= bkt_lb->int64.u64)
2835 				if ((eq_only && voival->int64.u64 ==
2836 				    bkt_lb->int64.u64) ||
2837 				    (!eq_only && (!has_ub ||
2838 				    voival->int64.u64 < bkt_ub->int64.u64)))
2839 					found = 1;
2840 			break;
2841 		case VSD_DTYPE_INT_SLONG:
2842 			if (voival->intlong.slong >= bkt_lb->intlong.slong)
2843 				if ((eq_only && voival->intlong.slong ==
2844 				    bkt_lb->intlong.slong) ||
2845 				    (!eq_only && (!has_ub ||
2846 				    voival->intlong.slong <
2847 				    bkt_ub->intlong.slong)))
2848 					found = 1;
2849 			break;
2850 		case VSD_DTYPE_INT_ULONG:
2851 			if (voival->intlong.ulong >= bkt_lb->intlong.ulong)
2852 				if ((eq_only && voival->intlong.ulong ==
2853 				    bkt_lb->intlong.ulong) ||
2854 				    (!eq_only && (!has_ub ||
2855 				    voival->intlong.ulong <
2856 				    bkt_ub->intlong.ulong)))
2857 					found = 1;
2858 			break;
2859 		case VSD_DTYPE_Q_S32:
2860 			if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32))
2861 				if ((eq_only && Q_QEQ(voival->q32.sq32,
2862 				    bkt_lb->q32.sq32)) ||
2863 				    (!eq_only && (!has_ub ||
2864 				    Q_QLTQ(voival->q32.sq32,
2865 				    bkt_ub->q32.sq32))))
2866 					found = 1;
2867 			break;
2868 		case VSD_DTYPE_Q_U32:
2869 			if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32))
2870 				if ((eq_only && Q_QEQ(voival->q32.uq32,
2871 				    bkt_lb->q32.uq32)) ||
2872 				    (!eq_only && (!has_ub ||
2873 				    Q_QLTQ(voival->q32.uq32,
2874 				    bkt_ub->q32.uq32))))
2875 					found = 1;
2876 			break;
2877 		case VSD_DTYPE_Q_S64:
2878 			if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64))
2879 				if ((eq_only && Q_QEQ(voival->q64.sq64,
2880 				    bkt_lb->q64.sq64)) ||
2881 				    (!eq_only && (!has_ub ||
2882 				    Q_QLTQ(voival->q64.sq64,
2883 				    bkt_ub->q64.sq64))))
2884 					found = 1;
2885 			break;
2886 		case VSD_DTYPE_Q_U64:
2887 			if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64))
2888 				if ((eq_only && Q_QEQ(voival->q64.uq64,
2889 				    bkt_lb->q64.uq64)) ||
2890 				    (!eq_only && (!has_ub ||
2891 				    Q_QLTQ(voival->q64.uq64,
2892 				    bkt_ub->q64.uq64))))
2893 					found = 1;
2894 			break;
2895 		default:
2896 			break;
2897 		}
2898 	}
2899 
2900 	if (found) {
2901 		if (is32bit)
2902 			*cnt32 += 1;
2903 		else
2904 			*cnt64 += 1;
2905 	} else {
2906 		if (is32bit)
2907 			*oob32 += 1;
2908 		else
2909 			*oob64 += 1;
2910 	}
2911 
2912 	vs->flags |= VS_VSDVALID;
2913 	return (error);
2914 }
2915 
2916 static inline int
2917 stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype,
2918     struct voistatdata_tdgst *tdgst, int attempt)
2919 {
2920 	struct ctdth32 *ctd32tree;
2921 	struct ctdth64 *ctd64tree;
2922 	struct voistatdata_tdgstctd32 *ctd32;
2923 	struct voistatdata_tdgstctd64 *ctd64;
2924 	uint64_t ebits, idxmask;
2925 	uint32_t bitsperidx, nebits;
2926 	int error, idx, is32bit, maxctds, remctds, tmperr;
2927 
2928 	error = 0;
2929 
2930 	switch (vs_dtype) {
2931 	case VSD_DTYPE_TDGSTCLUST32:
2932 		ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2933 		if (!ARB_FULL(ctd32tree))
2934 			return (0);
2935 		VSD(tdgstclust32, tdgst)->compcnt++;
2936 		maxctds = remctds = ARB_MAXNODES(ctd32tree);
2937 		ARB_RESET_TREE(ctd32tree, ctdth32, maxctds);
2938 		VSD(tdgstclust32, tdgst)->smplcnt = 0;
2939 		is32bit = 1;
2940 		ctd64tree = NULL;
2941 		ctd64 = NULL;
2942 #ifdef DIAGNOSTIC
2943 		RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2944 #endif
2945 		break;
2946 	case VSD_DTYPE_TDGSTCLUST64:
2947 		ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2948 		if (!ARB_FULL(ctd64tree))
2949 			return (0);
2950 		VSD(tdgstclust64, tdgst)->compcnt++;
2951 		maxctds = remctds = ARB_MAXNODES(ctd64tree);
2952 		ARB_RESET_TREE(ctd64tree, ctdth64, maxctds);
2953 		VSD(tdgstclust64, tdgst)->smplcnt = 0;
2954 		is32bit = 0;
2955 		ctd32tree = NULL;
2956 		ctd32 = NULL;
2957 #ifdef DIAGNOSTIC
2958 		RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2959 #endif
2960 		break;
2961 	default:
2962 		return (EINVAL);
2963 	}
2964 
2965 	/*
2966 	 * Rebuild the t-digest ARB by pseudorandomly selecting centroids and
2967 	 * re-inserting the mu/cnt of each as a value and corresponding weight.
2968 	 */
2969 
2970 	/*
2971 	 * XXXCEM: random(9) is currently rand(3), not random(3).  rand(3)
2972 	 * RAND_MAX happens to be approximately 31 bits (range [0,
2973 	 * 0x7ffffffd]), so the math kinda works out.  When/if this portion of
2974 	 * the code is compiled in userspace, it gets the random(3) behavior,
2975 	 * which has expected range [0, 0x7fffffff].
2976 	 */
2977 #define	bitsperrand 31
2978 	ebits = 0;
2979 	nebits = 0;
2980 	bitsperidx = fls(maxctds);
2981 	KASSERT(bitsperidx <= sizeof(ebits) << 3,
2982 	    ("%s: bitsperidx=%d, ebits=%d",
2983 	    __func__, bitsperidx, (int)(sizeof(ebits) << 3)));
2984 	idxmask = (UINT64_C(1) << bitsperidx) - 1;
2985 
2986 	/* Initialise the free list with randomised centroid indices. */
2987 	for (; remctds > 0; remctds--) {
2988 		while (nebits < bitsperidx) {
2989 			ebits |= ((uint64_t)random()) << nebits;
2990 			nebits += bitsperrand;
2991 			if (nebits > (sizeof(ebits) << 3))
2992 				nebits = sizeof(ebits) << 3;
2993 		}
2994 		idx = ebits & idxmask;
2995 		nebits -= bitsperidx;
2996 		ebits >>= bitsperidx;
2997 
2998 		/*
2999 		 * Select the next centroid to put on the ARB free list. We
3000 		 * start with the centroid at our randomly selected array index,
3001 		 * and work our way forwards until finding one (the latter
3002 		 * aspect reduces re-insertion randomness, but is good enough).
3003 		 */
3004 		do {
3005 			if (idx >= maxctds)
3006 				idx %= maxctds;
3007 
3008 			if (is32bit)
3009 				ctd32 = ARB_NODE(ctd32tree, idx);
3010 			else
3011 				ctd64 = ARB_NODE(ctd64tree, idx);
3012 		} while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) :
3013 		    ARB_ISFREE(ctd64, ctdlnk)) && ++idx);
3014 
3015 		/* Put the centroid on the ARB free list. */
3016 		if (is32bit)
3017 			ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk);
3018 		else
3019 			ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk);
3020 	}
3021 
3022 	/*
3023 	 * The free list now contains the randomised indices of every centroid.
3024 	 * Walk the free list from start to end, re-inserting each centroid's
3025 	 * mu/cnt. The tdgst_add() call may or may not consume the free centroid
3026 	 * we re-insert values from during each loop iteration, so we must latch
3027 	 * the index of the next free list centroid before the re-insertion
3028 	 * call. The previous loop above should have left the centroid pointer
3029 	 * pointing to the element at the head of the free list.
3030 	 */
3031 	KASSERT((is32bit ?
3032 	    ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) :
3033 	    ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)),
3034 	    ("%s: t-digest ARB@%p free list bug", __func__,
3035 	    (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3036 	remctds = maxctds;
3037 	while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) {
3038 		tmperr = 0;
3039 		if (is32bit) {
3040 			s64q_t x;
3041 
3042 			idx = ARB_NEXTFREEIDX(ctd32, ctdlnk);
3043 			/* Cloning a s32q_t into a s64q_t should never fail. */
3044 			tmperr = Q_QCLONEQ(&x, ctd32->mu);
3045 			tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add(
3046 			    vs_dtype, tdgst, x, ctd32->cnt, attempt);
3047 			ctd32 = ARB_NODE(ctd32tree, idx);
3048 			KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk),
3049 			    ("%s: t-digest ARB@%p free list bug", __func__,
3050 			    ctd32tree));
3051 		} else {
3052 			idx = ARB_NEXTFREEIDX(ctd64, ctdlnk);
3053 			tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst,
3054 			    ctd64->mu, ctd64->cnt, attempt);
3055 			ctd64 = ARB_NODE(ctd64tree, idx);
3056 			KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk),
3057 			    ("%s: t-digest ARB@%p free list bug", __func__,
3058 			    ctd64tree));
3059 		}
3060 		/*
3061 		 * This process should not produce errors, bugs notwithstanding.
3062 		 * Just in case, latch any errors and attempt all re-insertions.
3063 		 */
3064 		error = tmperr ? tmperr : error;
3065 		remctds--;
3066 	}
3067 
3068 	KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__,
3069 	    (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3070 
3071 	return (error);
3072 }
3073 
3074 static inline int
3075 stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
3076     s64q_t x, uint64_t weight, int attempt)
3077 {
3078 #ifdef DIAGNOSTIC
3079 	char qstr[Q_MAXSTRLEN(x, 10)];
3080 #endif
3081 	struct ctdth32 *ctd32tree;
3082 	struct ctdth64 *ctd64tree;
3083 	void *closest, *cur, *lb, *ub;
3084 	struct voistatdata_tdgstctd32 *ctd32;
3085 	struct voistatdata_tdgstctd64 *ctd64;
3086 	uint64_t cnt, smplcnt, sum, tmpsum;
3087 	s64q_t k, minz, q, z;
3088 	int error, is32bit, n;
3089 
3090 	error = 0;
3091 	minz = Q_INI(&z, 0, 0, Q_NFBITS(x));
3092 
3093 	switch (vs_dtype) {
3094 	case VSD_DTYPE_TDGSTCLUST32:
3095 		if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt)
3096 			error = EOVERFLOW;
3097 		smplcnt = VSD(tdgstclust32, tdgst)->smplcnt;
3098 		ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
3099 		is32bit = 1;
3100 		ctd64tree = NULL;
3101 		ctd64 = NULL;
3102 		break;
3103 	case VSD_DTYPE_TDGSTCLUST64:
3104 		if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt)
3105 			error = EOVERFLOW;
3106 		smplcnt = VSD(tdgstclust64, tdgst)->smplcnt;
3107 		ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
3108 		is32bit = 0;
3109 		ctd32tree = NULL;
3110 		ctd32 = NULL;
3111 		break;
3112 	default:
3113 		error = EINVAL;
3114 		break;
3115 	}
3116 
3117 	if (error)
3118 		return (error);
3119 
3120 	/*
3121 	 * Inspired by Ted Dunning's AVLTreeDigest.java
3122 	 */
3123 	do {
3124 #if defined(DIAGNOSTIC)
3125 		KASSERT(attempt < 5,
3126 		    ("%s: Too many attempts", __func__));
3127 #endif
3128 		if (attempt >= 5)
3129 			return (EAGAIN);
3130 
3131 		Q_SIFVAL(minz, Q_IFMAXVAL(minz));
3132 		closest = ub = NULL;
3133 		sum = tmpsum = 0;
3134 
3135 		if (is32bit)
3136 			lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree));
3137 		else
3138 			lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree));
3139 
3140 		if (lb == NULL) /* Empty tree. */
3141 			lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) :
3142 			    (void *)ARB_ROOT(ctd64tree));
3143 
3144 		/*
3145 		 * Find the set of centroids with minimum distance to x and
3146 		 * compute the sum of counts for all centroids with mean less
3147 		 * than the first centroid in the set.
3148 		 */
3149 		for (; cur != NULL;
3150 		    cur = (is32bit ?
3151 		    (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3152 		    (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3153 			if (is32bit) {
3154 				cnt = ctd32->cnt;
3155 				KASSERT(Q_PRECEQ(ctd32->mu, x),
3156 				    ("%s: Q_RELPREC(mu,x)=%d", __func__,
3157 				    Q_RELPREC(ctd32->mu, x)));
3158 				/* Ok to assign as both have same precision. */
3159 				z = ctd32->mu;
3160 			} else {
3161 				cnt = ctd64->cnt;
3162 				KASSERT(Q_PRECEQ(ctd64->mu, x),
3163 				    ("%s: Q_RELPREC(mu,x)=%d", __func__,
3164 				    Q_RELPREC(ctd64->mu, x)));
3165 				/* Ok to assign as both have same precision. */
3166 				z = ctd64->mu;
3167 			}
3168 
3169 			error = Q_QSUBQ(&z, x);
3170 #if defined(DIAGNOSTIC)
3171 			KASSERT(!error, ("%s: unexpected error %d", __func__,
3172 			    error));
3173 #endif
3174 			if (error)
3175 				return (error);
3176 
3177 			z = Q_QABS(z);
3178 			if (Q_QLTQ(z, minz)) {
3179 				minz = z;
3180 				lb = cur;
3181 				sum = tmpsum;
3182 				tmpsum += cnt;
3183 			} else if (Q_QGTQ(z, minz)) {
3184 				ub = cur;
3185 				break;
3186 			}
3187 		}
3188 
3189 		cur = (is32bit ?
3190 		    (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) :
3191 		    (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb));
3192 
3193 		for (n = 0; cur != ub; cur = (is32bit ?
3194 		    (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3195 		    (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3196 			if (is32bit)
3197 				cnt = ctd32->cnt;
3198 			else
3199 				cnt = ctd64->cnt;
3200 
3201 			q = Q_CTRLINI(16);
3202 			if (smplcnt == 1)
3203 				error = Q_QFRACI(&q, 1, 2);
3204 			else
3205 				/* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */
3206 				error = Q_QFRACI(&q, (sum << 1) + cnt - 1,
3207 				    (smplcnt - 1) << 1);
3208 			k = q;
3209 			/* k = q x 4 x samplcnt x attempt */
3210 			error |= Q_QMULI(&k, 4 * smplcnt * attempt);
3211 			/* k = k x (1 - q) */
3212 			error |= Q_QSUBI(&q, 1);
3213 			q = Q_QABS(q);
3214 			error |= Q_QMULQ(&k, q);
3215 #if defined(DIAGNOSTIC)
3216 #if !defined(_KERNEL)
3217 			double q_dbl, k_dbl, q2d, k2d;
3218 			q2d = Q_Q2D(q);
3219 			k2d = Q_Q2D(k);
3220 			q_dbl = smplcnt == 1 ? 0.5 :
3221 			    (sum + ((cnt - 1)  / 2.0)) / (double)(smplcnt - 1);
3222 			k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt;
3223 			/*
3224 			 * If the difference between q and q_dbl is greater than
3225 			 * the fractional precision of q, something is off.
3226 			 * NB: q is holding the value of 1 - q
3227 			 */
3228 			q_dbl = 1.0 - q_dbl;
3229 			KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) <
3230 			    (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))),
3231 			    ("Q-type q bad precision"));
3232 			KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) <
3233 			    1.0 + (0.01 * smplcnt),
3234 			    ("Q-type k bad precision"));
3235 #endif /* !_KERNEL */
3236 			KASSERT(!error, ("%s: unexpected error %d", __func__,
3237 			    error));
3238 #endif /* DIAGNOSTIC */
3239 			if (error)
3240 				return (error);
3241 			if ((is32bit && ((ctd32->cnt + weight) <=
3242 			    (uint64_t)Q_GIVAL(k))) ||
3243 			    (!is32bit && ((ctd64->cnt + weight) <=
3244 			    (uint64_t)Q_GIVAL(k)))) {
3245 				n++;
3246 				/* random() produces 31 bits. */
3247 				if (random() < (INT32_MAX / n))
3248 					closest = cur;
3249 			}
3250 			sum += cnt;
3251 		}
3252 	} while (closest == NULL &&
3253 	    (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) &&
3254 	    (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst,
3255 	    attempt++)) == 0);
3256 
3257 	if (error)
3258 		return (error);
3259 
3260 	if (closest != NULL) {
3261 		/* Merge with an existing centroid. */
3262 		if (is32bit) {
3263 			ctd32 = (struct voistatdata_tdgstctd32 *)closest;
3264 			error = Q_QSUBQ(&x, ctd32->mu);
3265 			/*
3266 			 * The following calculation "x / (cnt + weight)"
3267 			 * computes the amount by which to adjust the centroid's
3268 			 * mu value in order to merge in the VOI sample.
3269 			 *
3270 			 * It can underflow (Q_QDIVI() returns ERANGE) when the
3271 			 * user centroids' fractional precision (which is
3272 			 * inherited by 'x') is too low to represent the result.
3273 			 *
3274 			 * A sophisticated approach to dealing with this issue
3275 			 * would minimise accumulation of error by tracking
3276 			 * underflow per centroid and making an adjustment when
3277 			 * a LSB's worth of underflow has accumulated.
3278 			 *
3279 			 * A simpler approach is to let the result underflow
3280 			 * i.e. merge the VOI sample into the centroid without
3281 			 * adjusting the centroid's mu, and rely on the user to
3282 			 * specify their t-digest with sufficient centroid
3283 			 * fractional precision such that the accumulation of
3284 			 * error from multiple underflows is of no material
3285 			 * consequence to the centroid's final value of mu.
3286 			 *
3287 			 * For the moment, the latter approach is employed by
3288 			 * simply ignoring ERANGE here.
3289 			 *
3290 			 * XXXLAS: Per-centroid underflow tracking is likely too
3291 			 * onerous, but it probably makes sense to accumulate a
3292 			 * single underflow error variable across all centroids
3293 			 * and report it as part of the digest to provide
3294 			 * additional visibility into the digest's fidelity.
3295 			 */
3296 			error = error ? error :
3297 			    Q_QDIVI(&x, ctd32->cnt + weight);
3298 			if ((error && error != ERANGE)
3299 			    || (error = Q_QADDQ(&ctd32->mu, x))) {
3300 #ifdef DIAGNOSTIC
3301 				KASSERT(!error, ("%s: unexpected error %d",
3302 				    __func__, error));
3303 #endif
3304 				return (error);
3305 			}
3306 			ctd32->cnt += weight;
3307 			error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) ==
3308 			    NULL ? 0 : EALREADY;
3309 #ifdef DIAGNOSTIC
3310 			RB_REINSERT(rbctdth32,
3311 			    &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3312 #endif
3313 		} else {
3314 			ctd64 = (struct voistatdata_tdgstctd64 *)closest;
3315 			error = Q_QSUBQ(&x, ctd64->mu);
3316 			error = error ? error :
3317 			    Q_QDIVI(&x, ctd64->cnt + weight);
3318 			/* Refer to is32bit ERANGE discussion above. */
3319 			if ((error && error != ERANGE)
3320 			    || (error = Q_QADDQ(&ctd64->mu, x))) {
3321 				KASSERT(!error, ("%s: unexpected error %d",
3322 				    __func__, error));
3323 				return (error);
3324 			}
3325 			ctd64->cnt += weight;
3326 			error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) ==
3327 			    NULL ? 0 : EALREADY;
3328 #ifdef DIAGNOSTIC
3329 			RB_REINSERT(rbctdth64,
3330 			    &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64);
3331 #endif
3332 		}
3333 	} else {
3334 		/*
3335 		 * Add a new centroid. If digest compression is working
3336 		 * correctly, there should always be at least one free.
3337 		 */
3338 		if (is32bit) {
3339 			ctd32 = ARB_GETFREE(ctd32tree, ctdlnk);
3340 #ifdef DIAGNOSTIC
3341 			KASSERT(ctd32 != NULL,
3342 			    ("%s: t-digest@%p has no free centroids",
3343 			    __func__, tdgst));
3344 #endif
3345 			if (ctd32 == NULL)
3346 				return (EAGAIN);
3347 			if ((error = Q_QCPYVALQ(&ctd32->mu, x)))
3348 				return (error);
3349 			ctd32->cnt = weight;
3350 			error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ?
3351 			    0 : EALREADY;
3352 #ifdef DIAGNOSTIC
3353 			RB_INSERT(rbctdth32,
3354 			    &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3355 #endif
3356 		} else {
3357 			ctd64 = ARB_GETFREE(ctd64tree, ctdlnk);
3358 #ifdef DIAGNOSTIC
3359 			KASSERT(ctd64 != NULL,
3360 			    ("%s: t-digest@%p has no free centroids",
3361 			    __func__, tdgst));
3362 #endif
3363 			if (ctd64 == NULL) /* Should not happen. */
3364 				return (EAGAIN);
3365 			/* Direct assignment ok as both have same type/prec. */
3366 			ctd64->mu = x;
3367 			ctd64->cnt = weight;
3368 			error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ?
3369 			    0 : EALREADY;
3370 #ifdef DIAGNOSTIC
3371 			RB_INSERT(rbctdth64, &VSD(tdgstclust64,
3372 			    tdgst)->rbctdtree, ctd64);
3373 #endif
3374 		}
3375 	}
3376 
3377 	if (is32bit)
3378 		VSD(tdgstclust32, tdgst)->smplcnt += weight;
3379 	else {
3380 		VSD(tdgstclust64, tdgst)->smplcnt += weight;
3381 
3382 #ifdef DIAGNOSTIC
3383 		struct rbctdth64 *rbctdtree =
3384 		    &VSD(tdgstclust64, tdgst)->rbctdtree;
3385 		struct voistatdata_tdgstctd64 *rbctd64;
3386 		int i = 0;
3387 		ARB_FOREACH(ctd64, ctdth64, ctd64tree) {
3388 			rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) :
3389 			    RB_NEXT(rbctdth64, rbctdtree, rbctd64));
3390 
3391 			if (i >= ARB_CURNODES(ctd64tree)
3392 			    || ctd64 != rbctd64
3393 			    || ARB_MIN(ctdth64, ctd64tree) !=
3394 			       RB_MIN(rbctdth64, rbctdtree)
3395 			    || ARB_MAX(ctdth64, ctd64tree) !=
3396 			       RB_MAX(rbctdth64, rbctdtree)
3397 			    || ARB_LEFTIDX(ctd64, ctdlnk) !=
3398 			       ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk))
3399 			    || ARB_RIGHTIDX(ctd64, ctdlnk) !=
3400 			       ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk))
3401 			    || ARB_PARENTIDX(ctd64, ctdlnk) !=
3402 			       ARB_SELFIDX(ctd64tree,
3403 			       RB_PARENT(rbctd64, rblnk))) {
3404 				Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr));
3405 				printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3406 				    "mu=%s\n",
3407 				    (int)ARB_SELFIDX(ctd64tree, ctd64),
3408 				    ARB_PARENTIDX(ctd64, ctdlnk),
3409 				    ARB_LEFTIDX(ctd64, ctdlnk),
3410 				    ARB_RIGHTIDX(ctd64, ctdlnk),
3411 				    ARB_COLOR(ctd64, ctdlnk),
3412 				    qstr);
3413 
3414 				Q_TOSTR(rbctd64->mu, -1, 10, qstr,
3415 				    sizeof(qstr));
3416 				struct voistatdata_tdgstctd64 *parent;
3417 				parent = RB_PARENT(rbctd64, rblnk);
3418 				int rb_color =
3419 					parent == NULL ? 0 :
3420 					RB_LEFT(parent, rblnk) == rbctd64 ?
3421 					(_RB_BITSUP(parent, rblnk) & _RB_L) != 0 :
3422  					(_RB_BITSUP(parent, rblnk) & _RB_R) != 0;
3423 				printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3424 				    "mu=%s\n",
3425 				    (int)ARB_SELFIDX(ctd64tree, rbctd64),
3426 				    (int)ARB_SELFIDX(ctd64tree,
3427 				      RB_PARENT(rbctd64, rblnk)),
3428 				    (int)ARB_SELFIDX(ctd64tree,
3429 				      RB_LEFT(rbctd64, rblnk)),
3430 				    (int)ARB_SELFIDX(ctd64tree,
3431 				      RB_RIGHT(rbctd64, rblnk)),
3432 				    rb_color,
3433 				    qstr);
3434 
3435 				panic("RB@%p and ARB@%p trees differ\n",
3436 				    rbctdtree, ctd64tree);
3437 			}
3438 			i++;
3439 		}
3440 #endif /* DIAGNOSTIC */
3441 	}
3442 
3443 	return (error);
3444 }
3445 
3446 static inline int
3447 stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival,
3448     struct voistat *vs, struct voistatdata_tdgst *tdgst)
3449 {
3450 	s64q_t x;
3451 	int error;
3452 
3453 	error = 0;
3454 
3455 	switch (vs->dtype) {
3456 	case VSD_DTYPE_TDGSTCLUST32:
3457 		/* Use same precision as the user's centroids. */
3458 		Q_INI(&x, 0, 0, Q_NFBITS(
3459 		    ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu));
3460 		break;
3461 	case VSD_DTYPE_TDGSTCLUST64:
3462 		/* Use same precision as the user's centroids. */
3463 		Q_INI(&x, 0, 0, Q_NFBITS(
3464 		    ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu));
3465 		break;
3466 	default:
3467 		KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 ||
3468 		    vs->dtype == VSD_DTYPE_TDGSTCLUST64,
3469 		    ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>",
3470 		    __func__, vs->dtype));
3471 		return (EINVAL);
3472 	}
3473 
3474 	/*
3475 	 * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid
3476 	 * returning EOVERFLOW if the voival would have fit in a u64q_t.
3477 	 */
3478 	switch (voi_dtype) {
3479 	case VSD_DTYPE_INT_S32:
3480 		error = Q_QCPYVALI(&x, voival->int32.s32);
3481 		break;
3482 	case VSD_DTYPE_INT_U32:
3483 		error = Q_QCPYVALI(&x, voival->int32.u32);
3484 		break;
3485 	case VSD_DTYPE_INT_S64:
3486 		error = Q_QCPYVALI(&x, voival->int64.s64);
3487 		break;
3488 	case VSD_DTYPE_INT_U64:
3489 		error = Q_QCPYVALI(&x, voival->int64.u64);
3490 		break;
3491 	case VSD_DTYPE_INT_SLONG:
3492 		error = Q_QCPYVALI(&x, voival->intlong.slong);
3493 		break;
3494 	case VSD_DTYPE_INT_ULONG:
3495 		error = Q_QCPYVALI(&x, voival->intlong.ulong);
3496 		break;
3497 	case VSD_DTYPE_Q_S32:
3498 		error = Q_QCPYVALQ(&x, voival->q32.sq32);
3499 		break;
3500 	case VSD_DTYPE_Q_U32:
3501 		error = Q_QCPYVALQ(&x, voival->q32.uq32);
3502 		break;
3503 	case VSD_DTYPE_Q_S64:
3504 		error = Q_QCPYVALQ(&x, voival->q64.sq64);
3505 		break;
3506 	case VSD_DTYPE_Q_U64:
3507 		error = Q_QCPYVALQ(&x, voival->q64.uq64);
3508 		break;
3509 	default:
3510 		error = EINVAL;
3511 		break;
3512 	}
3513 
3514 	if (error ||
3515 	    (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1)))
3516 		return (error);
3517 
3518 	vs->flags |= VS_VSDVALID;
3519 	return (0);
3520 }
3521 
3522 int
3523 stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id,
3524     enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags)
3525 {
3526 	struct voi *v;
3527 	struct voistat *vs;
3528 	void *statevsd, *vsd;
3529 	int error, i, tmperr;
3530 
3531 	error = 0;
3532 
3533 	if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) ||
3534 	    voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL)
3535 		return (EINVAL);
3536 	v = &sb->vois[voi_id];
3537 	if (voi_dtype != v->dtype || v->id < 0 ||
3538 	    ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE)))
3539 		return (EINVAL);
3540 
3541 	vs = BLOB_OFFSET(sb, v->stats_off);
3542 	if (v->flags & VOI_REQSTATE)
3543 		statevsd = BLOB_OFFSET(sb, vs->data_off);
3544 	else
3545 		statevsd = NULL;
3546 
3547 	if (flags & SB_VOI_RELUPDATE) {
3548 		switch (voi_dtype) {
3549 		case VSD_DTYPE_INT_S32:
3550 			voival->int32.s32 +=
3551 			    VSD(voistate, statevsd)->prev.int32.s32;
3552 			break;
3553 		case VSD_DTYPE_INT_U32:
3554 			voival->int32.u32 +=
3555 			    VSD(voistate, statevsd)->prev.int32.u32;
3556 			break;
3557 		case VSD_DTYPE_INT_S64:
3558 			voival->int64.s64 +=
3559 			    VSD(voistate, statevsd)->prev.int64.s64;
3560 			break;
3561 		case VSD_DTYPE_INT_U64:
3562 			voival->int64.u64 +=
3563 			    VSD(voistate, statevsd)->prev.int64.u64;
3564 			break;
3565 		case VSD_DTYPE_INT_SLONG:
3566 			voival->intlong.slong +=
3567 			    VSD(voistate, statevsd)->prev.intlong.slong;
3568 			break;
3569 		case VSD_DTYPE_INT_ULONG:
3570 			voival->intlong.ulong +=
3571 			    VSD(voistate, statevsd)->prev.intlong.ulong;
3572 			break;
3573 		case VSD_DTYPE_Q_S32:
3574 			error = Q_QADDQ(&voival->q32.sq32,
3575 			    VSD(voistate, statevsd)->prev.q32.sq32);
3576 			break;
3577 		case VSD_DTYPE_Q_U32:
3578 			error = Q_QADDQ(&voival->q32.uq32,
3579 			    VSD(voistate, statevsd)->prev.q32.uq32);
3580 			break;
3581 		case VSD_DTYPE_Q_S64:
3582 			error = Q_QADDQ(&voival->q64.sq64,
3583 			    VSD(voistate, statevsd)->prev.q64.sq64);
3584 			break;
3585 		case VSD_DTYPE_Q_U64:
3586 			error = Q_QADDQ(&voival->q64.uq64,
3587 			    VSD(voistate, statevsd)->prev.q64.uq64);
3588 			break;
3589 		default:
3590 			KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3591 			break;
3592 		}
3593 	}
3594 
3595 	if (error)
3596 		return (error);
3597 
3598 	for (i = v->voistatmaxid; i > 0; i--) {
3599 		vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i];
3600 		if (vs->stype < 0)
3601 			continue;
3602 
3603 		vsd = BLOB_OFFSET(sb, vs->data_off);
3604 
3605 		switch (vs->stype) {
3606 		case VS_STYPE_MAX:
3607 			tmperr = stats_v1_voi_update_max(voi_dtype, voival,
3608 			    vs, vsd);
3609 			break;
3610 		case VS_STYPE_MIN:
3611 			tmperr = stats_v1_voi_update_min(voi_dtype, voival,
3612 			    vs, vsd);
3613 			break;
3614 		case VS_STYPE_SUM:
3615 			tmperr = stats_v1_voi_update_sum(voi_dtype, voival,
3616 			    vs, vsd);
3617 			break;
3618 		case VS_STYPE_HIST:
3619 			tmperr = stats_v1_voi_update_hist(voi_dtype, voival,
3620 			    vs, vsd);
3621 			break;
3622 		case VS_STYPE_TDGST:
3623 			tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival,
3624 			    vs, vsd);
3625 			break;
3626 		default:
3627 			KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
3628 			break;
3629 		}
3630 
3631 		if (tmperr) {
3632 			error = tmperr;
3633 			VS_INCERRS(vs);
3634 		}
3635 	}
3636 
3637 	if (statevsd) {
3638 		switch (voi_dtype) {
3639 		case VSD_DTYPE_INT_S32:
3640 			VSD(voistate, statevsd)->prev.int32.s32 =
3641 			    voival->int32.s32;
3642 			break;
3643 		case VSD_DTYPE_INT_U32:
3644 			VSD(voistate, statevsd)->prev.int32.u32 =
3645 			    voival->int32.u32;
3646 			break;
3647 		case VSD_DTYPE_INT_S64:
3648 			VSD(voistate, statevsd)->prev.int64.s64 =
3649 			    voival->int64.s64;
3650 			break;
3651 		case VSD_DTYPE_INT_U64:
3652 			VSD(voistate, statevsd)->prev.int64.u64 =
3653 			    voival->int64.u64;
3654 			break;
3655 		case VSD_DTYPE_INT_SLONG:
3656 			VSD(voistate, statevsd)->prev.intlong.slong =
3657 			    voival->intlong.slong;
3658 			break;
3659 		case VSD_DTYPE_INT_ULONG:
3660 			VSD(voistate, statevsd)->prev.intlong.ulong =
3661 			    voival->intlong.ulong;
3662 			break;
3663 		case VSD_DTYPE_Q_S32:
3664 			error = Q_QCPYVALQ(
3665 			    &VSD(voistate, statevsd)->prev.q32.sq32,
3666 			    voival->q32.sq32);
3667 			break;
3668 		case VSD_DTYPE_Q_U32:
3669 			error = Q_QCPYVALQ(
3670 			    &VSD(voistate, statevsd)->prev.q32.uq32,
3671 			    voival->q32.uq32);
3672 			break;
3673 		case VSD_DTYPE_Q_S64:
3674 			error = Q_QCPYVALQ(
3675 			    &VSD(voistate, statevsd)->prev.q64.sq64,
3676 			    voival->q64.sq64);
3677 			break;
3678 		case VSD_DTYPE_Q_U64:
3679 			error = Q_QCPYVALQ(
3680 			    &VSD(voistate, statevsd)->prev.q64.uq64,
3681 			    voival->q64.uq64);
3682 			break;
3683 		default:
3684 			KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3685 			break;
3686 		}
3687 	}
3688 
3689 	return (error);
3690 }
3691 
3692 #ifdef _KERNEL
3693 
3694 static void
3695 stats_init(void *arg)
3696 {
3697 
3698 }
3699 SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL);
3700 
3701 /*
3702  * Sysctl handler to display the list of available stats templates.
3703  */
3704 static int
3705 stats_tpl_list_available(SYSCTL_HANDLER_ARGS)
3706 {
3707 	struct sbuf *s;
3708 	int err, i;
3709 
3710 	err = 0;
3711 
3712 	/* We can tolerate ntpl being stale, so do not take the lock. */
3713 	s = sbuf_new(NULL, NULL, /* +1 per tpl for , */
3714 	    ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN);
3715 	if (s == NULL)
3716 		return (ENOMEM);
3717 
3718 	TPL_LIST_RLOCK();
3719 	for (i = 0; i < ntpl; i++) {
3720 		err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "",
3721 		    tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash);
3722 		if (err) {
3723 			/* Sbuf overflow condition. */
3724 			err = EOVERFLOW;
3725 			break;
3726 		}
3727 	}
3728 	TPL_LIST_RUNLOCK();
3729 
3730 	if (!err) {
3731 		sbuf_finish(s);
3732 		err = sysctl_handle_string(oidp, sbuf_data(s), 0, req);
3733 	}
3734 
3735 	sbuf_delete(s);
3736 	return (err);
3737 }
3738 
3739 /*
3740  * Called by subsystem-specific sysctls to report and/or parse the list of
3741  * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t
3742  * conformant function pointer must be passed in as arg1, which is used to
3743  * interact with the subsystem's stats template sample rates list. If arg2 > 0,
3744  * a zero-initialised allocation of arg2-sized contextual memory is
3745  * heap-allocated and passed in to all subsystem callbacks made during the
3746  * operation of stats_tpl_sample_rates().
3747  *
3748  * XXXLAS: Assumes templates are never removed, which is currently true but may
3749  * need to be reworked in future if dynamic template management becomes a
3750  * requirement e.g. to support kernel module based templates.
3751  */
3752 int
3753 stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS)
3754 {
3755 	char kvpair_fmt[16], tplspec_fmt[16];
3756 	char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN];
3757 	char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */
3758 	stats_tpl_sr_cb_t subsys_cb;
3759 	void *subsys_ctx;
3760 	char *buf, *new_rates_usr_str, *tpl_name_p;
3761 	struct stats_tpl_sample_rate *rates;
3762 	struct sbuf *s, _s;
3763 	uint32_t cum_pct, pct, tpl_hash;
3764 	int err, i, off, len, newlen, nrates;
3765 
3766 	buf = NULL;
3767 	rates = NULL;
3768 	err = nrates = 0;
3769 	subsys_cb = (stats_tpl_sr_cb_t)arg1;
3770 	KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__));
3771 	if (arg2 > 0)
3772 		subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO);
3773 	else
3774 		subsys_ctx = NULL;
3775 
3776 	/* Grab current count of subsystem rates. */
3777 	err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx);
3778 	if (err)
3779 		goto done;
3780 
3781 	/* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */
3782 	len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5));
3783 
3784 	if (req->oldptr != NULL || req->newptr != NULL)
3785 		buf = malloc(len, M_TEMP, M_WAITOK);
3786 
3787 	if (req->oldptr != NULL) {
3788 		if (nrates == 0) {
3789 			/* No rates, so return an empty string via oldptr. */
3790 			err = SYSCTL_OUT(req, "", 1);
3791 			if (err)
3792 				goto done;
3793 			goto process_new;
3794 		}
3795 
3796 		s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL);
3797 
3798 		/* Grab locked count of, and ptr to, subsystem rates. */
3799 		err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates,
3800 		    subsys_ctx);
3801 		if (err)
3802 			goto done;
3803 		TPL_LIST_RLOCK();
3804 		for (i = 0; i < nrates && !err; i++) {
3805 			err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "",
3806 			    tpllist[rates[i].tpl_slot_id]->mb->tplname,
3807 			    tpllist[rates[i].tpl_slot_id]->mb->tplhash,
3808 			    rates[i].tpl_sample_pct);
3809 		}
3810 		TPL_LIST_RUNLOCK();
3811 		/* Tell subsystem that we're done with its rates list. */
3812 		err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx);
3813 		if (err)
3814 			goto done;
3815 
3816 		err = sbuf_finish(s);
3817 		if (err)
3818 			goto done; /* We lost a race for buf to be too small. */
3819 
3820 		/* Return the rendered string data via oldptr. */
3821 		err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s));
3822 	} else {
3823 		/* Return the upper bound size for buffer sizing requests. */
3824 		err = SYSCTL_OUT(req, NULL, len);
3825 	}
3826 
3827 process_new:
3828 	if (err || req->newptr == NULL)
3829 		goto done;
3830 
3831 	newlen = req->newlen - req->newidx;
3832 	err = SYSCTL_IN(req, buf, newlen);
3833 	if (err)
3834 		goto done;
3835 
3836 	/*
3837 	 * Initialise format strings at run time.
3838 	 *
3839 	 * Write the max template spec string length into the
3840 	 * template_spec=percent key-value pair parsing format string as:
3841 	 *     " %<width>[^=]=%u %n"
3842 	 *
3843 	 * Write the max template name string length into the tplname:tplhash
3844 	 * parsing format string as:
3845 	 *     "%<width>[^:]:%u"
3846 	 *
3847 	 * Subtract 1 for \0 appended by sscanf().
3848 	 */
3849 	sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1);
3850 	sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1);
3851 
3852 	/*
3853 	 * Parse each CSV key-value pair specifying a template and its sample
3854 	 * percentage. Whitespace either side of a key-value pair is ignored.
3855 	 * Templates can be specified by name, hash, or name and hash per the
3856 	 * following formats (chars in [] are optional):
3857 	 *    ["]<tplname>["]=<percent>
3858 	 *    :hash=pct
3859 	 *    ["]<tplname>["]:hash=<percent>
3860 	 */
3861 	cum_pct = nrates = 0;
3862 	rates = NULL;
3863 	buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */
3864 	new_rates_usr_str = buf;
3865 	while (isspace(*new_rates_usr_str))
3866 		new_rates_usr_str++; /* Skip leading whitespace. */
3867 	while (*new_rates_usr_str != '\0') {
3868 		tpl_name_p = tpl_name;
3869 		tpl_name[0] = '\0';
3870 		tpl_hash = 0;
3871 		off = 0;
3872 
3873 		/*
3874 		 * Parse key-value pair which must perform 2 conversions, then
3875 		 * parse the template spec to extract either name, hash, or name
3876 		 * and hash depending on the three possible spec formats. The
3877 		 * tplspec_fmt format specifier parses name or name and hash
3878 		 * template specs, while the ":%u" format specifier parses
3879 		 * hash-only template specs. If parsing is successfull, ensure
3880 		 * the cumulative sampling percentage does not exceed 100.
3881 		 */
3882 		err = EINVAL;
3883 		if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct,
3884 		    &off))
3885 			break;
3886 		if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) &&
3887 		    (1 != sscanf(tpl_spec, ":%u", &tpl_hash)))
3888 			break;
3889 		if ((cum_pct += pct) > 100)
3890 			break;
3891 		err = 0;
3892 
3893 		/* Strip surrounding "" from template name if present. */
3894 		len = strlen(tpl_name);
3895 		if (len > 0) {
3896 			if (tpl_name[len - 1] == '"')
3897 				tpl_name[--len] = '\0';
3898 			if (tpl_name[0] == '"') {
3899 				tpl_name_p++;
3900 				len--;
3901 			}
3902 		}
3903 
3904 		rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */
3905 		    (nrates + 1) * sizeof(*rates), M_WAITOK);
3906 		rates[nrates].tpl_slot_id =
3907 		    stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash);
3908 		if (rates[nrates].tpl_slot_id < 0) {
3909 			err = -rates[nrates].tpl_slot_id;
3910 			break;
3911 		}
3912 		rates[nrates].tpl_sample_pct = pct;
3913 		nrates++;
3914 		new_rates_usr_str += off;
3915 		if (*new_rates_usr_str != ',')
3916 			break; /* End-of-input or malformed. */
3917 		new_rates_usr_str++; /* Move past comma to next pair. */
3918 	}
3919 
3920 	if (!err) {
3921 		if ((new_rates_usr_str - buf) < newlen) {
3922 			/* Entire input has not been consumed. */
3923 			err = EINVAL;
3924 		} else {
3925 			/*
3926 			 * Give subsystem the new rates. They'll return the
3927 			 * appropriate rates pointer for us to garbage collect.
3928 			 */
3929 			err = subsys_cb(TPL_SR_PUT, &rates, &nrates,
3930 			    subsys_ctx);
3931 		}
3932 	}
3933 	stats_free(rates);
3934 
3935 done:
3936 	free(buf, M_TEMP);
3937 	free(subsys_ctx, M_TEMP);
3938 	return (err);
3939 }
3940 
3941 SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
3942     "stats(9) MIB");
3943 
3944 SYSCTL_PROC(_kern_stats, OID_AUTO, templates,
3945     CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
3946     stats_tpl_list_available, "A",
3947     "list the name/hash of all available stats(9) templates");
3948 
3949 #else /* ! _KERNEL */
3950 
3951 static void __attribute__ ((constructor))
3952 stats_constructor(void)
3953 {
3954 
3955 	pthread_rwlock_init(&tpllistlock, NULL);
3956 }
3957 
3958 static void __attribute__ ((destructor))
3959 stats_destructor(void)
3960 {
3961 
3962 	pthread_rwlock_destroy(&tpllistlock);
3963 }
3964 
3965 #endif /* _KERNEL */
3966