11a13f2e6SEdward Tomasz Napierala /*-
21a13f2e6SEdward Tomasz Napierala * Copyright (c) 2014-2018 Netflix, Inc.
31a13f2e6SEdward Tomasz Napierala * All rights reserved.
41a13f2e6SEdward Tomasz Napierala *
51a13f2e6SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without
61a13f2e6SEdward Tomasz Napierala * modification, are permitted provided that the following conditions
71a13f2e6SEdward Tomasz Napierala * are met:
81a13f2e6SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright
91a13f2e6SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer.
101a13f2e6SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright
111a13f2e6SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the
121a13f2e6SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution.
131a13f2e6SEdward Tomasz Napierala *
141a13f2e6SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
151a13f2e6SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
161a13f2e6SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
171a13f2e6SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
181a13f2e6SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
191a13f2e6SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
201a13f2e6SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
211a13f2e6SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
221a13f2e6SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
231a13f2e6SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
241a13f2e6SEdward Tomasz Napierala * SUCH DAMAGE.
251a13f2e6SEdward Tomasz Napierala */
261a13f2e6SEdward Tomasz Napierala
271a13f2e6SEdward Tomasz Napierala /*
281a13f2e6SEdward Tomasz Napierala * Author: Lawrence Stewart <lstewart@netflix.com>
291a13f2e6SEdward Tomasz Napierala */
301a13f2e6SEdward Tomasz Napierala
311a13f2e6SEdward Tomasz Napierala #include <sys/param.h>
321a13f2e6SEdward Tomasz Napierala #include <sys/arb.h>
331a13f2e6SEdward Tomasz Napierala #include <sys/ctype.h>
341a13f2e6SEdward Tomasz Napierala #include <sys/errno.h>
351a13f2e6SEdward Tomasz Napierala #include <sys/hash.h>
361a13f2e6SEdward Tomasz Napierala #include <sys/limits.h>
371a13f2e6SEdward Tomasz Napierala #include <sys/malloc.h>
381a13f2e6SEdward Tomasz Napierala #include <sys/qmath.h>
391a13f2e6SEdward Tomasz Napierala #include <sys/sbuf.h>
401a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC)
411a13f2e6SEdward Tomasz Napierala #include <sys/tree.h>
421a13f2e6SEdward Tomasz Napierala #endif
431a13f2e6SEdward Tomasz Napierala #include <sys/stats.h> /* Must come after qmath.h and arb.h */
441a13f2e6SEdward Tomasz Napierala #include <sys/stddef.h>
451a13f2e6SEdward Tomasz Napierala #include <sys/stdint.h>
461a13f2e6SEdward Tomasz Napierala #include <sys/time.h>
471a13f2e6SEdward Tomasz Napierala
481a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
491a13f2e6SEdward Tomasz Napierala #include <sys/kernel.h>
501a13f2e6SEdward Tomasz Napierala #include <sys/lock.h>
511a13f2e6SEdward Tomasz Napierala #include <sys/rwlock.h>
521a13f2e6SEdward Tomasz Napierala #include <sys/sysctl.h>
531a13f2e6SEdward Tomasz Napierala #include <sys/systm.h>
541a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */
551a13f2e6SEdward Tomasz Napierala #include <pthread.h>
561a13f2e6SEdward Tomasz Napierala #include <stdbool.h>
571a13f2e6SEdward Tomasz Napierala #include <stdio.h>
581a13f2e6SEdward Tomasz Napierala #include <stdlib.h>
591a13f2e6SEdward Tomasz Napierala #include <string.h>
601a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */
611a13f2e6SEdward Tomasz Napierala
621a13f2e6SEdward Tomasz Napierala struct voistatdata_voistate {
631a13f2e6SEdward Tomasz Napierala /* Previous VOI value for diff calculation. */
641a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric prev;
651a13f2e6SEdward Tomasz Napierala };
661a13f2e6SEdward Tomasz Napierala
671a13f2e6SEdward Tomasz Napierala #define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */
681a13f2e6SEdward Tomasz Napierala struct voistat {
691a13f2e6SEdward Tomasz Napierala int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */
701a13f2e6SEdward Tomasz Napierala enum vsd_dtype dtype : 8; /* Data type of this stat's data. */
711a13f2e6SEdward Tomasz Napierala uint16_t data_off; /* Blob offset for this stat's data. */
721a13f2e6SEdward Tomasz Napierala uint16_t dsz; /* Size of stat's data. */
731a13f2e6SEdward Tomasz Napierala #define VS_EBITS 8
741a13f2e6SEdward Tomasz Napierala uint16_t errs : VS_EBITS;/* Non-wrapping error count. */
751a13f2e6SEdward Tomasz Napierala uint16_t flags : 16 - VS_EBITS;
761a13f2e6SEdward Tomasz Napierala };
771a13f2e6SEdward Tomasz Napierala /* The voistat error count is capped to avoid wrapping. */
781a13f2e6SEdward Tomasz Napierala #define VS_INCERRS(vs) do { \
791a13f2e6SEdward Tomasz Napierala if ((vs)->errs < (1U << VS_EBITS) - 1) \
801a13f2e6SEdward Tomasz Napierala (vs)->errs++; \
811a13f2e6SEdward Tomasz Napierala } while (0)
821a13f2e6SEdward Tomasz Napierala
831a13f2e6SEdward Tomasz Napierala /*
841a13f2e6SEdward Tomasz Napierala * Ideas for flags:
851a13f2e6SEdward Tomasz Napierala * - Global or entity specific (global would imply use of counter(9)?)
861a13f2e6SEdward Tomasz Napierala * - Whether to reset stats on read or not
871a13f2e6SEdward Tomasz Napierala * - Signal an overflow?
881a13f2e6SEdward Tomasz Napierala * - Compressed voistat array
891a13f2e6SEdward Tomasz Napierala */
901a13f2e6SEdward Tomasz Napierala #define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */
911a13f2e6SEdward Tomasz Napierala struct voi {
921a13f2e6SEdward Tomasz Napierala int16_t id; /* VOI id. */
931a13f2e6SEdward Tomasz Napierala enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */
941a13f2e6SEdward Tomasz Napierala int8_t voistatmaxid; /* Largest allocated voistat index. */
951a13f2e6SEdward Tomasz Napierala uint16_t stats_off; /* Blob offset for this VOIs stats. */
961a13f2e6SEdward Tomasz Napierala uint16_t flags;
971a13f2e6SEdward Tomasz Napierala };
981a13f2e6SEdward Tomasz Napierala
991a13f2e6SEdward Tomasz Napierala /*
1001a13f2e6SEdward Tomasz Napierala * Memory for the entire blob is allocated as a slab and then offsets are
1011a13f2e6SEdward Tomasz Napierala * maintained to carve up the slab into sections holding different data types.
1021a13f2e6SEdward Tomasz Napierala *
1031a13f2e6SEdward Tomasz Napierala * Ideas for flags:
1041a13f2e6SEdward Tomasz Napierala * - Compressed voi array (trade off memory usage vs search time)
1051a13f2e6SEdward Tomasz Napierala * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib)
1061a13f2e6SEdward Tomasz Napierala */
1071a13f2e6SEdward Tomasz Napierala struct statsblobv1 {
1081a13f2e6SEdward Tomasz Napierala uint8_t abi;
1091a13f2e6SEdward Tomasz Napierala uint8_t endian;
1101a13f2e6SEdward Tomasz Napierala uint16_t flags;
1111a13f2e6SEdward Tomasz Napierala uint16_t maxsz;
1121a13f2e6SEdward Tomasz Napierala uint16_t cursz;
1131a13f2e6SEdward Tomasz Napierala /* Fields from here down are opaque to consumers. */
1141a13f2e6SEdward Tomasz Napierala uint32_t tplhash; /* Base template hash ID. */
1151a13f2e6SEdward Tomasz Napierala uint16_t stats_off; /* voistat array blob offset. */
1161a13f2e6SEdward Tomasz Napierala uint16_t statsdata_off; /* voistatdata array blob offset. */
1171a13f2e6SEdward Tomasz Napierala sbintime_t created; /* Blob creation time. */
1181a13f2e6SEdward Tomasz Napierala sbintime_t lastrst; /* Time of last reset. */
1191a13f2e6SEdward Tomasz Napierala struct voi vois[]; /* Array indexed by [voi_id]. */
1201a13f2e6SEdward Tomasz Napierala } __aligned(sizeof(void *));
1211a13f2e6SEdward Tomasz Napierala _Static_assert(offsetof(struct statsblobv1, cursz) +
12289f34d46SBrooks Davis SIZEOF_MEMBER(struct statsblobv1, cursz) ==
12389f34d46SBrooks Davis offsetof(struct statsblob, opaque),
1241a13f2e6SEdward Tomasz Napierala "statsblobv1 ABI mismatch");
1251a13f2e6SEdward Tomasz Napierala
1261a13f2e6SEdward Tomasz Napierala struct statsblobv1_tpl {
1271a13f2e6SEdward Tomasz Napierala struct metablob *mb;
1281a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb;
1291a13f2e6SEdward Tomasz Napierala };
1301a13f2e6SEdward Tomasz Napierala
1311a13f2e6SEdward Tomasz Napierala /* Context passed to iterator callbacks. */
1321a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx {
1331a13f2e6SEdward Tomasz Napierala void *usrctx; /* Caller supplied context. */
1341a13f2e6SEdward Tomasz Napierala uint32_t flags; /* Flags for current iteration. */
1351a13f2e6SEdward Tomasz Napierala int16_t vslot; /* struct voi slot index. */
1361a13f2e6SEdward Tomasz Napierala int8_t vsslot; /* struct voistat slot index. */
1371a13f2e6SEdward Tomasz Napierala };
1381a13f2e6SEdward Tomasz Napierala
1391a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx {
1401a13f2e6SEdward Tomasz Napierala struct sbuf *buf;
1411a13f2e6SEdward Tomasz Napierala struct statsblob_tpl *tpl;
1421a13f2e6SEdward Tomasz Napierala enum sb_str_fmt fmt;
1431a13f2e6SEdward Tomasz Napierala uint32_t flags;
1441a13f2e6SEdward Tomasz Napierala };
1451a13f2e6SEdward Tomasz Napierala
1461a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx {
1471a13f2e6SEdward Tomasz Napierala stats_blob_visitcb_t cb;
1481a13f2e6SEdward Tomasz Napierala void *usrctx;
1491a13f2e6SEdward Tomasz Napierala };
1501a13f2e6SEdward Tomasz Napierala
1511a13f2e6SEdward Tomasz Napierala /* Stats blob iterator callback. */
1521a13f2e6SEdward Tomasz Napierala typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v,
1531a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx);
1541a13f2e6SEdward Tomasz Napierala
1551a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
1561a13f2e6SEdward Tomasz Napierala static struct rwlock tpllistlock;
1571a13f2e6SEdward Tomasz Napierala RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock");
1581a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock)
1591a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock)
1601a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock)
1611a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock)
1621a13f2e6SEdward Tomasz Napierala #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED)
1631a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED)
1641a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED)
1651a13f2e6SEdward Tomasz Napierala MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory");
1661a13f2e6SEdward Tomasz Napierala #define stats_free(ptr) free((ptr), M_STATS)
1671a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */
1681a13f2e6SEdward Tomasz Napierala static void stats_constructor(void);
1691a13f2e6SEdward Tomasz Napierala static void stats_destructor(void);
1701a13f2e6SEdward Tomasz Napierala static pthread_rwlock_t tpllistlock;
1711a13f2e6SEdward Tomasz Napierala #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock)
1721a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock)
1731a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK()
1741a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock)
1751a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK()
1761a13f2e6SEdward Tomasz Napierala #define TPL_LIST_LOCK_ASSERT() do { } while (0)
1771a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK_ASSERT() do { } while (0)
1781a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK_ASSERT() do { } while (0)
1791a13f2e6SEdward Tomasz Napierala #ifdef NDEBUG
1801a13f2e6SEdward Tomasz Napierala #define KASSERT(cond, msg) do {} while (0)
1811a13f2e6SEdward Tomasz Napierala #define stats_abort() do {} while (0)
1821a13f2e6SEdward Tomasz Napierala #else /* ! NDEBUG */
1831a13f2e6SEdward Tomasz Napierala #define KASSERT(cond, msg) do { \
1841a13f2e6SEdward Tomasz Napierala if (!(cond)) { \
1851a13f2e6SEdward Tomasz Napierala panic msg; \
1861a13f2e6SEdward Tomasz Napierala } \
1871a13f2e6SEdward Tomasz Napierala } while (0)
1881a13f2e6SEdward Tomasz Napierala #define stats_abort() abort()
1891a13f2e6SEdward Tomasz Napierala #endif /* NDEBUG */
1901a13f2e6SEdward Tomasz Napierala #define stats_free(ptr) free(ptr)
1911a13f2e6SEdward Tomasz Napierala #define panic(fmt, ...) do { \
1921a13f2e6SEdward Tomasz Napierala fprintf(stderr, (fmt), ##__VA_ARGS__); \
1931a13f2e6SEdward Tomasz Napierala stats_abort(); \
1941a13f2e6SEdward Tomasz Napierala } while (0)
1951a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */
1961a13f2e6SEdward Tomasz Napierala
1971a13f2e6SEdward Tomasz Napierala #define SB_V1_MAXSZ 65535
1981a13f2e6SEdward Tomasz Napierala
1991a13f2e6SEdward Tomasz Napierala /* Obtain a blob offset pointer. */
2001a13f2e6SEdward Tomasz Napierala #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off)))
2011a13f2e6SEdward Tomasz Napierala
2021a13f2e6SEdward Tomasz Napierala /*
2031a13f2e6SEdward Tomasz Napierala * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a
2041a13f2e6SEdward Tomasz Napierala * power of 2 size, we can shift instead of divide. The shift amount must be
2051a13f2e6SEdward Tomasz Napierala * updated if sizeof(struct voi) ever changes, which the assert should catch.
2061a13f2e6SEdward Tomasz Napierala */
2071a13f2e6SEdward Tomasz Napierala #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \
2081a13f2e6SEdward Tomasz Napierala sizeof(struct statsblobv1)) >> 3))
2091a13f2e6SEdward Tomasz Napierala _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch");
2101a13f2e6SEdward Tomasz Napierala
2111a13f2e6SEdward Tomasz Napierala /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */
2121a13f2e6SEdward Tomasz Napierala const char *vs_stype2name[VS_NUM_STYPES] = {
2131a13f2e6SEdward Tomasz Napierala [VS_STYPE_VOISTATE] = "VOISTATE",
2141a13f2e6SEdward Tomasz Napierala [VS_STYPE_SUM] = "SUM",
2151a13f2e6SEdward Tomasz Napierala [VS_STYPE_MAX] = "MAX",
2161a13f2e6SEdward Tomasz Napierala [VS_STYPE_MIN] = "MIN",
2171a13f2e6SEdward Tomasz Napierala [VS_STYPE_HIST] = "HIST",
2181a13f2e6SEdward Tomasz Napierala [VS_STYPE_TDGST] = "TDGST",
2191a13f2e6SEdward Tomasz Napierala };
2201a13f2e6SEdward Tomasz Napierala
2211a13f2e6SEdward Tomasz Napierala const char *vs_stype2desc[VS_NUM_STYPES] = {
2221a13f2e6SEdward Tomasz Napierala [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)",
2231a13f2e6SEdward Tomasz Napierala [VS_STYPE_SUM] = "Simple arithmetic accumulator",
2241a13f2e6SEdward Tomasz Napierala [VS_STYPE_MAX] = "Maximum observed VOI value",
2251a13f2e6SEdward Tomasz Napierala [VS_STYPE_MIN] = "Minimum observed VOI value",
2261a13f2e6SEdward Tomasz Napierala [VS_STYPE_HIST] = "Histogram of observed VOI values",
2271a13f2e6SEdward Tomasz Napierala [VS_STYPE_TDGST] = "t-digest of observed VOI values",
2281a13f2e6SEdward Tomasz Napierala };
2291a13f2e6SEdward Tomasz Napierala
2301a13f2e6SEdward Tomasz Napierala const char *vsd_dtype2name[VSD_NUM_DTYPES] = {
2311a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = "VOISTATE",
2321a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = "INT_S32",
2331a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = "INT_U32",
2341a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = "INT_S64",
2351a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = "INT_U64",
2361a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = "INT_SLONG",
2371a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = "INT_ULONG",
2381a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = "Q_S32",
2391a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = "Q_U32",
2401a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = "Q_S64",
2411a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = "Q_U64",
2421a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = "CRHIST32",
2431a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = "DRHIST32",
2441a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = "DVHIST32",
2451a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = "CRHIST64",
2461a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = "DRHIST64",
2471a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = "DVHIST64",
2481a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32",
2491a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64",
2501a13f2e6SEdward Tomasz Napierala };
2511a13f2e6SEdward Tomasz Napierala
2521a13f2e6SEdward Tomasz Napierala const size_t vsd_dtype2size[VSD_NUM_DTYPES] = {
2531a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate),
2541a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32),
2551a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32),
2561a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64),
2571a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64),
2581a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong),
2591a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong),
2601a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32),
2611a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32),
2621a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64),
2631a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64),
2641a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32),
2651a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32),
2661a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32),
2671a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64),
2681a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64),
2691a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64),
2701a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32),
2711a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64),
2721a13f2e6SEdward Tomasz Napierala };
2731a13f2e6SEdward Tomasz Napierala
2741a13f2e6SEdward Tomasz Napierala static const bool vsd_compoundtype[VSD_NUM_DTYPES] = {
2751a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = true,
2761a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = false,
2771a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = false,
2781a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = false,
2791a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = false,
2801a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = false,
2811a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = false,
2821a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = false,
2831a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = false,
2841a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = false,
2851a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = false,
2861a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = true,
2871a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = true,
2881a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = true,
2891a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = true,
2901a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = true,
2911a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = true,
2921a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = true,
2931a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = true,
2941a13f2e6SEdward Tomasz Napierala };
2951a13f2e6SEdward Tomasz Napierala
2961a13f2e6SEdward Tomasz Napierala const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = {
2971a13f2e6SEdward Tomasz Napierala [LIM_MIN] = {
298*a5affc0cSAndrew Turner [VSD_DTYPE_VOISTATE] = {},
2991a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}},
3001a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}},
3011a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}},
3021a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}},
3031a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}},
3041a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}},
3051a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}},
3061a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}},
3071a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}},
3081a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}},
3091a13f2e6SEdward Tomasz Napierala },
3101a13f2e6SEdward Tomasz Napierala [LIM_MAX] = {
311*a5affc0cSAndrew Turner [VSD_DTYPE_VOISTATE] = {},
3121a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}},
3131a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}},
3141a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}},
3151a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}},
3161a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}},
3171a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}},
3181a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}},
3191a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}},
3201a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}},
3211a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}},
3221a13f2e6SEdward Tomasz Napierala }
3231a13f2e6SEdward Tomasz Napierala };
3241a13f2e6SEdward Tomasz Napierala
3251a13f2e6SEdward Tomasz Napierala /* tpllistlock protects tpllist and ntpl */
3261a13f2e6SEdward Tomasz Napierala static uint32_t ntpl;
3271a13f2e6SEdward Tomasz Napierala static struct statsblob_tpl **tpllist;
3281a13f2e6SEdward Tomasz Napierala
3291a13f2e6SEdward Tomasz Napierala static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz,
3301a13f2e6SEdward Tomasz Napierala int flags);
3311a13f2e6SEdward Tomasz Napierala //static void stats_v1_blob_finalise(struct statsblobv1 *sb);
3321a13f2e6SEdward Tomasz Napierala static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
3331a13f2e6SEdward Tomasz Napierala uint32_t flags);
3341a13f2e6SEdward Tomasz Napierala static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
3351a13f2e6SEdward Tomasz Napierala int newvoistatbytes, int newvoistatdatabytes);
3361a13f2e6SEdward Tomasz Napierala static void stats_v1_blob_iter(struct statsblobv1 *sb,
3371a13f2e6SEdward Tomasz Napierala stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags);
3381a13f2e6SEdward Tomasz Napierala static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype,
3391a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt);
3401a13f2e6SEdward Tomasz Napierala
3411a13f2e6SEdward Tomasz Napierala static inline int
ctd32cmp(const struct voistatdata_tdgstctd32 * c1,const struct voistatdata_tdgstctd32 * c2)3421a13f2e6SEdward Tomasz Napierala ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2)
3431a13f2e6SEdward Tomasz Napierala {
3441a13f2e6SEdward Tomasz Napierala
3451a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(c1->mu, c2->mu),
3461a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
3471a13f2e6SEdward Tomasz Napierala Q_RELPREC(c1->mu, c2->mu)));
3481a13f2e6SEdward Tomasz Napierala
3491a13f2e6SEdward Tomasz Napierala return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
3501a13f2e6SEdward Tomasz Napierala }
3511a13f2e6SEdward Tomasz Napierala ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp);
3521a13f2e6SEdward Tomasz Napierala
3531a13f2e6SEdward Tomasz Napierala static inline int
ctd64cmp(const struct voistatdata_tdgstctd64 * c1,const struct voistatdata_tdgstctd64 * c2)3541a13f2e6SEdward Tomasz Napierala ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2)
3551a13f2e6SEdward Tomasz Napierala {
3561a13f2e6SEdward Tomasz Napierala
3571a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(c1->mu, c2->mu),
3581a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
3591a13f2e6SEdward Tomasz Napierala Q_RELPREC(c1->mu, c2->mu)));
3601a13f2e6SEdward Tomasz Napierala
3611a13f2e6SEdward Tomasz Napierala return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
3621a13f2e6SEdward Tomasz Napierala }
3631a13f2e6SEdward Tomasz Napierala ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp);
3641a13f2e6SEdward Tomasz Napierala
3651a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
3661a13f2e6SEdward Tomasz Napierala RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp);
3671a13f2e6SEdward Tomasz Napierala RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp);
3681a13f2e6SEdward Tomasz Napierala #endif
3691a13f2e6SEdward Tomasz Napierala
3701a13f2e6SEdward Tomasz Napierala static inline sbintime_t
stats_sbinuptime(void)3711a13f2e6SEdward Tomasz Napierala stats_sbinuptime(void)
3721a13f2e6SEdward Tomasz Napierala {
3731a13f2e6SEdward Tomasz Napierala sbintime_t sbt;
3741a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
3751a13f2e6SEdward Tomasz Napierala
3761a13f2e6SEdward Tomasz Napierala sbt = sbinuptime();
3771a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */
3781a13f2e6SEdward Tomasz Napierala struct timespec tp;
3791a13f2e6SEdward Tomasz Napierala
3801a13f2e6SEdward Tomasz Napierala clock_gettime(CLOCK_MONOTONIC_FAST, &tp);
3811a13f2e6SEdward Tomasz Napierala sbt = tstosbt(tp);
3821a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */
3831a13f2e6SEdward Tomasz Napierala
3841a13f2e6SEdward Tomasz Napierala return (sbt);
3851a13f2e6SEdward Tomasz Napierala }
3861a13f2e6SEdward Tomasz Napierala
3871a13f2e6SEdward Tomasz Napierala static inline void *
stats_realloc(void * ptr,size_t oldsz,size_t newsz,int flags)3881a13f2e6SEdward Tomasz Napierala stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags)
3891a13f2e6SEdward Tomasz Napierala {
3901a13f2e6SEdward Tomasz Napierala
3911a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
3921a13f2e6SEdward Tomasz Napierala /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */
3931a13f2e6SEdward Tomasz Napierala if (!(flags & (M_WAITOK | M_NOWAIT)))
3941a13f2e6SEdward Tomasz Napierala flags |= M_NOWAIT;
3951a13f2e6SEdward Tomasz Napierala ptr = realloc(ptr, newsz, M_STATS, flags);
3961a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */
3971a13f2e6SEdward Tomasz Napierala ptr = realloc(ptr, newsz);
3981a13f2e6SEdward Tomasz Napierala if ((flags & M_ZERO) && ptr != NULL) {
3991a13f2e6SEdward Tomasz Napierala if (oldsz == 0)
4001a13f2e6SEdward Tomasz Napierala memset(ptr, '\0', newsz);
4011a13f2e6SEdward Tomasz Napierala else if (newsz > oldsz)
4021a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz);
4031a13f2e6SEdward Tomasz Napierala }
4041a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */
4051a13f2e6SEdward Tomasz Napierala
4061a13f2e6SEdward Tomasz Napierala return (ptr);
4071a13f2e6SEdward Tomasz Napierala }
4081a13f2e6SEdward Tomasz Napierala
4091a13f2e6SEdward Tomasz Napierala static inline char *
stats_strdup(const char * s,int flags)4101a13f2e6SEdward Tomasz Napierala stats_strdup(const char *s,
4111a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
4121a13f2e6SEdward Tomasz Napierala int flags)
4131a13f2e6SEdward Tomasz Napierala {
4141a13f2e6SEdward Tomasz Napierala char *copy;
4151a13f2e6SEdward Tomasz Napierala size_t len;
4161a13f2e6SEdward Tomasz Napierala
4171a13f2e6SEdward Tomasz Napierala if (!(flags & (M_WAITOK | M_NOWAIT)))
4181a13f2e6SEdward Tomasz Napierala flags |= M_NOWAIT;
4191a13f2e6SEdward Tomasz Napierala
4201a13f2e6SEdward Tomasz Napierala len = strlen(s) + 1;
4211a13f2e6SEdward Tomasz Napierala if ((copy = malloc(len, M_STATS, flags)) != NULL)
4221a13f2e6SEdward Tomasz Napierala bcopy(s, copy, len);
4231a13f2e6SEdward Tomasz Napierala
4241a13f2e6SEdward Tomasz Napierala return (copy);
4251a13f2e6SEdward Tomasz Napierala #else
4261a13f2e6SEdward Tomasz Napierala int flags __unused)
4271a13f2e6SEdward Tomasz Napierala {
4281a13f2e6SEdward Tomasz Napierala return (strdup(s));
4291a13f2e6SEdward Tomasz Napierala #endif
4301a13f2e6SEdward Tomasz Napierala }
4311a13f2e6SEdward Tomasz Napierala
4321a13f2e6SEdward Tomasz Napierala static inline void
4331a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(struct statsblob_tpl *tpl)
4341a13f2e6SEdward Tomasz Napierala {
4351a13f2e6SEdward Tomasz Napierala
4361a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK_ASSERT();
4371a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0);
4381a13f2e6SEdward Tomasz Napierala for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) {
4391a13f2e6SEdward Tomasz Napierala if (tpl->mb->voi_meta[voi_id].name != NULL)
4401a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_str(
4411a13f2e6SEdward Tomasz Napierala tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash);
4421a13f2e6SEdward Tomasz Napierala }
4431a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz,
4441a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash);
4451a13f2e6SEdward Tomasz Napierala }
4461a13f2e6SEdward Tomasz Napierala
4471a13f2e6SEdward Tomasz Napierala static inline uint64_t
4481a13f2e6SEdward Tomasz Napierala stats_pow_u64(uint64_t base, uint64_t exp)
4491a13f2e6SEdward Tomasz Napierala {
4501a13f2e6SEdward Tomasz Napierala uint64_t result = 1;
4511a13f2e6SEdward Tomasz Napierala
4521a13f2e6SEdward Tomasz Napierala while (exp) {
4531a13f2e6SEdward Tomasz Napierala if (exp & 1)
4541a13f2e6SEdward Tomasz Napierala result *= base;
4551a13f2e6SEdward Tomasz Napierala exp >>= 1;
4561a13f2e6SEdward Tomasz Napierala base *= base;
4571a13f2e6SEdward Tomasz Napierala }
4581a13f2e6SEdward Tomasz Napierala
4591a13f2e6SEdward Tomasz Napierala return (result);
4601a13f2e6SEdward Tomasz Napierala }
4611a13f2e6SEdward Tomasz Napierala
4621a13f2e6SEdward Tomasz Napierala static inline int
4631a13f2e6SEdward Tomasz Napierala stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt,
4641a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub)
4651a13f2e6SEdward Tomasz Napierala {
4661a13f2e6SEdward Tomasz Napierala uint64_t step = 0;
4671a13f2e6SEdward Tomasz Napierala int error = 0;
4681a13f2e6SEdward Tomasz Napierala
4691a13f2e6SEdward Tomasz Napierala switch (info->scheme) {
4701a13f2e6SEdward Tomasz Napierala case BKT_LIN:
4711a13f2e6SEdward Tomasz Napierala step = info->lin.stepinc;
4721a13f2e6SEdward Tomasz Napierala break;
4731a13f2e6SEdward Tomasz Napierala case BKT_EXP:
4741a13f2e6SEdward Tomasz Napierala step = stats_pow_u64(info->exp.stepbase,
4751a13f2e6SEdward Tomasz Napierala info->exp.stepexp + curbkt);
4761a13f2e6SEdward Tomasz Napierala break;
4771a13f2e6SEdward Tomasz Napierala case BKT_LINEXP:
4781a13f2e6SEdward Tomasz Napierala {
4791a13f2e6SEdward Tomasz Napierala uint64_t curstepexp = 1;
4801a13f2e6SEdward Tomasz Napierala
4811a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) {
4821a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
4831a13f2e6SEdward Tomasz Napierala while ((int32_t)stats_pow_u64(info->linexp.stepbase,
4841a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int32.s32)
4851a13f2e6SEdward Tomasz Napierala curstepexp++;
4861a13f2e6SEdward Tomasz Napierala break;
4871a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
4881a13f2e6SEdward Tomasz Napierala while ((uint32_t)stats_pow_u64(info->linexp.stepbase,
4891a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int32.u32)
4901a13f2e6SEdward Tomasz Napierala curstepexp++;
4911a13f2e6SEdward Tomasz Napierala break;
4921a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
4931a13f2e6SEdward Tomasz Napierala while ((int64_t)stats_pow_u64(info->linexp.stepbase,
4941a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int64.s64)
4951a13f2e6SEdward Tomasz Napierala curstepexp++;
4961a13f2e6SEdward Tomasz Napierala break;
4971a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
4981a13f2e6SEdward Tomasz Napierala while ((uint64_t)stats_pow_u64(info->linexp.stepbase,
4991a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int64.u64)
5001a13f2e6SEdward Tomasz Napierala curstepexp++;
5011a13f2e6SEdward Tomasz Napierala break;
5021a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
5031a13f2e6SEdward Tomasz Napierala while ((long)stats_pow_u64(info->linexp.stepbase,
5041a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->intlong.slong)
5051a13f2e6SEdward Tomasz Napierala curstepexp++;
5061a13f2e6SEdward Tomasz Napierala break;
5071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
5081a13f2e6SEdward Tomasz Napierala while ((unsigned long)stats_pow_u64(info->linexp.stepbase,
5091a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->intlong.ulong)
5101a13f2e6SEdward Tomasz Napierala curstepexp++;
5111a13f2e6SEdward Tomasz Napierala break;
5121a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
5131a13f2e6SEdward Tomasz Napierala while ((s32q_t)stats_pow_u64(info->linexp.stepbase,
5141a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32))
5151a13f2e6SEdward Tomasz Napierala break;
5161a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
5171a13f2e6SEdward Tomasz Napierala while ((u32q_t)stats_pow_u64(info->linexp.stepbase,
5181a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32))
5191a13f2e6SEdward Tomasz Napierala break;
5201a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
5211a13f2e6SEdward Tomasz Napierala while ((s64q_t)stats_pow_u64(info->linexp.stepbase,
5221a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64))
5231a13f2e6SEdward Tomasz Napierala curstepexp++;
5241a13f2e6SEdward Tomasz Napierala break;
5251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
5261a13f2e6SEdward Tomasz Napierala while ((u64q_t)stats_pow_u64(info->linexp.stepbase,
5271a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64))
5281a13f2e6SEdward Tomasz Napierala curstepexp++;
5291a13f2e6SEdward Tomasz Napierala break;
5301a13f2e6SEdward Tomasz Napierala default:
5311a13f2e6SEdward Tomasz Napierala break;
5321a13f2e6SEdward Tomasz Napierala }
5331a13f2e6SEdward Tomasz Napierala
5341a13f2e6SEdward Tomasz Napierala step = stats_pow_u64(info->linexp.stepbase, curstepexp) /
5351a13f2e6SEdward Tomasz Napierala info->linexp.linstepdiv;
5361a13f2e6SEdward Tomasz Napierala if (step == 0)
5371a13f2e6SEdward Tomasz Napierala step = 1;
5381a13f2e6SEdward Tomasz Napierala break;
5391a13f2e6SEdward Tomasz Napierala }
5401a13f2e6SEdward Tomasz Napierala default:
5411a13f2e6SEdward Tomasz Napierala break;
5421a13f2e6SEdward Tomasz Napierala }
5431a13f2e6SEdward Tomasz Napierala
5441a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) {
5451a13f2e6SEdward Tomasz Napierala *bkt_lb = info->usr.bkts[curbkt].lb;
5461a13f2e6SEdward Tomasz Napierala *bkt_ub = info->usr.bkts[curbkt].ub;
5471a13f2e6SEdward Tomasz Napierala } else if (step != 0) {
5481a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) {
5491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
5501a13f2e6SEdward Tomasz Napierala bkt_ub->int32.s32 += (int32_t)step;
5511a13f2e6SEdward Tomasz Napierala break;
5521a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
5531a13f2e6SEdward Tomasz Napierala bkt_ub->int32.u32 += (uint32_t)step;
5541a13f2e6SEdward Tomasz Napierala break;
5551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
5561a13f2e6SEdward Tomasz Napierala bkt_ub->int64.s64 += (int64_t)step;
5571a13f2e6SEdward Tomasz Napierala break;
5581a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
5591a13f2e6SEdward Tomasz Napierala bkt_ub->int64.u64 += (uint64_t)step;
5601a13f2e6SEdward Tomasz Napierala break;
5611a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
5621a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.slong += (long)step;
5631a13f2e6SEdward Tomasz Napierala break;
5641a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
5651a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.ulong += (unsigned long)step;
5661a13f2e6SEdward Tomasz Napierala break;
5671a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
5681a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q32.sq32, step);
5691a13f2e6SEdward Tomasz Napierala break;
5701a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
5711a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q32.uq32, step);
5721a13f2e6SEdward Tomasz Napierala break;
5731a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
5741a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q64.sq64, step);
5751a13f2e6SEdward Tomasz Napierala break;
5761a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
5771a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q64.uq64, step);
5781a13f2e6SEdward Tomasz Napierala break;
5791a13f2e6SEdward Tomasz Napierala default:
5801a13f2e6SEdward Tomasz Napierala break;
5811a13f2e6SEdward Tomasz Napierala }
5821a13f2e6SEdward Tomasz Napierala } else { /* info->scheme != BKT_USR && step == 0 */
5831a13f2e6SEdward Tomasz Napierala return (EINVAL);
5841a13f2e6SEdward Tomasz Napierala }
5851a13f2e6SEdward Tomasz Napierala
5861a13f2e6SEdward Tomasz Napierala return (error);
5871a13f2e6SEdward Tomasz Napierala }
5881a13f2e6SEdward Tomasz Napierala
5891a13f2e6SEdward Tomasz Napierala static uint32_t
5901a13f2e6SEdward Tomasz Napierala stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info)
5911a13f2e6SEdward Tomasz Napierala {
5921a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric bkt_lb, bkt_ub;
5931a13f2e6SEdward Tomasz Napierala uint32_t nbkts;
5941a13f2e6SEdward Tomasz Napierala int done;
5951a13f2e6SEdward Tomasz Napierala
5961a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) {
5971a13f2e6SEdward Tomasz Napierala /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */
5981a13f2e6SEdward Tomasz Napierala info->lb = info->usr.bkts[0].lb;
5991a13f2e6SEdward Tomasz Napierala info->ub = info->usr.bkts[info->usr.nbkts - 1].lb;
6001a13f2e6SEdward Tomasz Napierala }
6011a13f2e6SEdward Tomasz Napierala
6021a13f2e6SEdward Tomasz Napierala nbkts = 0;
6031a13f2e6SEdward Tomasz Napierala done = 0;
6041a13f2e6SEdward Tomasz Napierala bkt_ub = info->lb;
6051a13f2e6SEdward Tomasz Napierala
6061a13f2e6SEdward Tomasz Napierala do {
6071a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub;
6081a13f2e6SEdward Tomasz Napierala if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub))
6091a13f2e6SEdward Tomasz Napierala return (0);
6101a13f2e6SEdward Tomasz Napierala
6111a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR)
6121a13f2e6SEdward Tomasz Napierala done = (nbkts == info->usr.nbkts);
6131a13f2e6SEdward Tomasz Napierala else {
6141a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) {
6151a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
6161a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int32.s32 > info->ub.int32.s32);
6171a13f2e6SEdward Tomasz Napierala break;
6181a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
6191a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int32.u32 > info->ub.int32.u32);
6201a13f2e6SEdward Tomasz Napierala break;
6211a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
6221a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int64.s64 > info->ub.int64.s64);
6231a13f2e6SEdward Tomasz Napierala break;
6241a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
6251a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int64.u64 > info->ub.int64.u64);
6261a13f2e6SEdward Tomasz Napierala break;
6271a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
6281a13f2e6SEdward Tomasz Napierala done = (bkt_ub.intlong.slong >
6291a13f2e6SEdward Tomasz Napierala info->ub.intlong.slong);
6301a13f2e6SEdward Tomasz Napierala break;
6311a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
6321a13f2e6SEdward Tomasz Napierala done = (bkt_ub.intlong.ulong >
6331a13f2e6SEdward Tomasz Napierala info->ub.intlong.ulong);
6341a13f2e6SEdward Tomasz Napierala break;
6351a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
6361a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q32.sq32,
6371a13f2e6SEdward Tomasz Napierala info->ub.q32.sq32);
6381a13f2e6SEdward Tomasz Napierala break;
6391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
6401a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q32.uq32,
6411a13f2e6SEdward Tomasz Napierala info->ub.q32.uq32);
6421a13f2e6SEdward Tomasz Napierala break;
6431a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
6441a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q64.sq64,
6451a13f2e6SEdward Tomasz Napierala info->ub.q64.sq64);
6461a13f2e6SEdward Tomasz Napierala break;
6471a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
6481a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q64.uq64,
6491a13f2e6SEdward Tomasz Napierala info->ub.q64.uq64);
6501a13f2e6SEdward Tomasz Napierala break;
6511a13f2e6SEdward Tomasz Napierala default:
6521a13f2e6SEdward Tomasz Napierala return (0);
6531a13f2e6SEdward Tomasz Napierala }
6541a13f2e6SEdward Tomasz Napierala }
6551a13f2e6SEdward Tomasz Napierala } while (!done);
6561a13f2e6SEdward Tomasz Napierala
6571a13f2e6SEdward Tomasz Napierala if (info->flags & VSD_HIST_LBOUND_INF)
6581a13f2e6SEdward Tomasz Napierala nbkts++;
6591a13f2e6SEdward Tomasz Napierala if (info->flags & VSD_HIST_UBOUND_INF)
6601a13f2e6SEdward Tomasz Napierala nbkts++;
6611a13f2e6SEdward Tomasz Napierala
6621a13f2e6SEdward Tomasz Napierala return (nbkts);
6631a13f2e6SEdward Tomasz Napierala }
6641a13f2e6SEdward Tomasz Napierala
6651a13f2e6SEdward Tomasz Napierala int
6661a13f2e6SEdward Tomasz Napierala stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
6671a13f2e6SEdward Tomasz Napierala struct vss_hist_hlpr_info *info)
6681a13f2e6SEdward Tomasz Napierala {
6691a13f2e6SEdward Tomasz Napierala struct voistatdata_hist *hist;
6701a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub,
6711a13f2e6SEdward Tomasz Napierala *ubinfbktlb, *ubinfbktub;
6721a13f2e6SEdward Tomasz Napierala uint32_t bkt, nbkts, nloop;
6731a13f2e6SEdward Tomasz Napierala
6741a13f2e6SEdward Tomasz Napierala if (vss == NULL || info == NULL || (info->flags &
6751a13f2e6SEdward Tomasz Napierala (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype ==
6761a13f2e6SEdward Tomasz Napierala VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64)))
6771a13f2e6SEdward Tomasz Napierala return (EINVAL);
6781a13f2e6SEdward Tomasz Napierala
6791a13f2e6SEdward Tomasz Napierala info->voi_dtype = voi_dtype;
6801a13f2e6SEdward Tomasz Napierala
6811a13f2e6SEdward Tomasz Napierala if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0)
6821a13f2e6SEdward Tomasz Napierala return (EINVAL);
6831a13f2e6SEdward Tomasz Napierala
6841a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) {
6851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
6861a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts);
6871a13f2e6SEdward Tomasz Napierala break;
6881a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
6891a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts);
6901a13f2e6SEdward Tomasz Napierala break;
6911a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
6921a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts);
6931a13f2e6SEdward Tomasz Napierala break;
6941a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
6951a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts);
6961a13f2e6SEdward Tomasz Napierala break;
6971a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
6981a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts);
6991a13f2e6SEdward Tomasz Napierala break;
7001a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
7011a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts);
7021a13f2e6SEdward Tomasz Napierala break;
7031a13f2e6SEdward Tomasz Napierala default:
7041a13f2e6SEdward Tomasz Napierala return (EINVAL);
7051a13f2e6SEdward Tomasz Napierala }
7061a13f2e6SEdward Tomasz Napierala
7071a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
7081a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL)
7091a13f2e6SEdward Tomasz Napierala return (ENOMEM);
7101a13f2e6SEdward Tomasz Napierala
7111a13f2e6SEdward Tomasz Napierala hist = (struct voistatdata_hist *)vss->iv;
7121a13f2e6SEdward Tomasz Napierala bkt_ub = info->lb;
7131a13f2e6SEdward Tomasz Napierala
7141a13f2e6SEdward Tomasz Napierala for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0;
7151a13f2e6SEdward Tomasz Napierala bkt < nbkts;
7161a13f2e6SEdward Tomasz Napierala bkt++, nloop++) {
7171a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub;
7181a13f2e6SEdward Tomasz Napierala if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub))
7191a13f2e6SEdward Tomasz Napierala return (EINVAL);
7201a13f2e6SEdward Tomasz Napierala
7211a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) {
7221a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
7231a13f2e6SEdward Tomasz Napierala VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb;
7241a13f2e6SEdward Tomasz Napierala break;
7251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
7261a13f2e6SEdward Tomasz Napierala VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb;
7271a13f2e6SEdward Tomasz Napierala VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub;
7281a13f2e6SEdward Tomasz Napierala break;
7291a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
7301a13f2e6SEdward Tomasz Napierala VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb;
7311a13f2e6SEdward Tomasz Napierala break;
7321a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
7331a13f2e6SEdward Tomasz Napierala VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb;
7341a13f2e6SEdward Tomasz Napierala break;
7351a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
7361a13f2e6SEdward Tomasz Napierala VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb;
7371a13f2e6SEdward Tomasz Napierala VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub;
7381a13f2e6SEdward Tomasz Napierala break;
7391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
7401a13f2e6SEdward Tomasz Napierala VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb;
7411a13f2e6SEdward Tomasz Napierala break;
7421a13f2e6SEdward Tomasz Napierala default:
7431a13f2e6SEdward Tomasz Napierala return (EINVAL);
7441a13f2e6SEdward Tomasz Napierala }
7451a13f2e6SEdward Tomasz Napierala }
7461a13f2e6SEdward Tomasz Napierala
7471a13f2e6SEdward Tomasz Napierala lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL;
7481a13f2e6SEdward Tomasz Napierala
7491a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) {
7501a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
7511a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb;
7521a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb;
7531a13f2e6SEdward Tomasz Napierala break;
7541a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
7551a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb;
7561a13f2e6SEdward Tomasz Napierala lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub;
7571a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb;
7581a13f2e6SEdward Tomasz Napierala ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub;
7591a13f2e6SEdward Tomasz Napierala break;
7601a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
7611a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb;
7621a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb;
7631a13f2e6SEdward Tomasz Napierala break;
7641a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
7651a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb;
7661a13f2e6SEdward Tomasz Napierala lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub;
7671a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb;
7681a13f2e6SEdward Tomasz Napierala ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub;
7691a13f2e6SEdward Tomasz Napierala break;
7701a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
7711a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
7721a13f2e6SEdward Tomasz Napierala break;
7731a13f2e6SEdward Tomasz Napierala default:
7741a13f2e6SEdward Tomasz Napierala return (EINVAL);
7751a13f2e6SEdward Tomasz Napierala }
7761a13f2e6SEdward Tomasz Napierala
7771a13f2e6SEdward Tomasz Napierala if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) {
7781a13f2e6SEdward Tomasz Napierala *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype];
7791a13f2e6SEdward Tomasz Napierala /*
7801a13f2e6SEdward Tomasz Napierala * Assignment from numeric_limit array for Q types assigns max
7811a13f2e6SEdward Tomasz Napierala * possible integral/fractional value for underlying data type,
7821a13f2e6SEdward Tomasz Napierala * but we must set control bits for this specific histogram per
7831a13f2e6SEdward Tomasz Napierala * the user's choice of fractional bits, which we extract from
7841a13f2e6SEdward Tomasz Napierala * info->lb.
7851a13f2e6SEdward Tomasz Napierala */
7861a13f2e6SEdward Tomasz Napierala if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
7871a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U32) {
7881a13f2e6SEdward Tomasz Napierala /* Signedness doesn't matter for setting control bits. */
7891a13f2e6SEdward Tomasz Napierala Q_SCVAL(lbinfbktlb->q32.sq32,
7901a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q32.sq32));
7911a13f2e6SEdward Tomasz Napierala } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
7921a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U64) {
7931a13f2e6SEdward Tomasz Napierala /* Signedness doesn't matter for setting control bits. */
7941a13f2e6SEdward Tomasz Napierala Q_SCVAL(lbinfbktlb->q64.sq64,
7951a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q64.sq64));
7961a13f2e6SEdward Tomasz Napierala }
7971a13f2e6SEdward Tomasz Napierala if (lbinfbktub)
7981a13f2e6SEdward Tomasz Napierala *lbinfbktub = info->lb;
7991a13f2e6SEdward Tomasz Napierala }
8001a13f2e6SEdward Tomasz Napierala if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) {
8011a13f2e6SEdward Tomasz Napierala *ubinfbktlb = bkt_lb;
8021a13f2e6SEdward Tomasz Napierala if (ubinfbktub) {
8031a13f2e6SEdward Tomasz Napierala *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype];
8041a13f2e6SEdward Tomasz Napierala if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
8051a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U32) {
8061a13f2e6SEdward Tomasz Napierala Q_SCVAL(ubinfbktub->q32.sq32,
8071a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q32.sq32));
8081a13f2e6SEdward Tomasz Napierala } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
8091a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U64) {
8101a13f2e6SEdward Tomasz Napierala Q_SCVAL(ubinfbktub->q64.sq64,
8111a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q64.sq64));
8121a13f2e6SEdward Tomasz Napierala }
8131a13f2e6SEdward Tomasz Napierala }
8141a13f2e6SEdward Tomasz Napierala }
8151a13f2e6SEdward Tomasz Napierala
8161a13f2e6SEdward Tomasz Napierala return (0);
8171a13f2e6SEdward Tomasz Napierala }
8181a13f2e6SEdward Tomasz Napierala
8191a13f2e6SEdward Tomasz Napierala int
8201a13f2e6SEdward Tomasz Napierala stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
8211a13f2e6SEdward Tomasz Napierala struct vss_tdgst_hlpr_info *info)
8221a13f2e6SEdward Tomasz Napierala {
8231a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst;
8241a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree;
8251a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree;
8261a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32;
8271a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64;
8281a13f2e6SEdward Tomasz Napierala
8291a13f2e6SEdward Tomasz Napierala info->voi_dtype = voi_dtype;
8301a13f2e6SEdward Tomasz Napierala
8311a13f2e6SEdward Tomasz Napierala switch (info->tdgst_dtype) {
8321a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
8331a13f2e6SEdward Tomasz Napierala vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds);
8341a13f2e6SEdward Tomasz Napierala break;
8351a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
8361a13f2e6SEdward Tomasz Napierala vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds);
8371a13f2e6SEdward Tomasz Napierala break;
8381a13f2e6SEdward Tomasz Napierala default:
8391a13f2e6SEdward Tomasz Napierala return (EINVAL);
8401a13f2e6SEdward Tomasz Napierala }
8411a13f2e6SEdward Tomasz Napierala
8421a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
8431a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL)
8441a13f2e6SEdward Tomasz Napierala return (ENOMEM);
8451a13f2e6SEdward Tomasz Napierala
8461a13f2e6SEdward Tomasz Napierala tdgst = (struct voistatdata_tdgst *)vss->iv;
8471a13f2e6SEdward Tomasz Napierala
8481a13f2e6SEdward Tomasz Napierala switch (info->tdgst_dtype) {
8491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
8501a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
8511a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) {
8521a13f2e6SEdward Tomasz Napierala Q_INI(&ctd32->mu, 0, 0, info->prec);
8531a13f2e6SEdward Tomasz Napierala }
8541a13f2e6SEdward Tomasz Napierala break;
8551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
8561a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
8571a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) {
8581a13f2e6SEdward Tomasz Napierala Q_INI(&ctd64->mu, 0, 0, info->prec);
8591a13f2e6SEdward Tomasz Napierala }
8601a13f2e6SEdward Tomasz Napierala break;
8611a13f2e6SEdward Tomasz Napierala default:
8621a13f2e6SEdward Tomasz Napierala return (EINVAL);
8631a13f2e6SEdward Tomasz Napierala }
8641a13f2e6SEdward Tomasz Napierala
8651a13f2e6SEdward Tomasz Napierala return (0);
8661a13f2e6SEdward Tomasz Napierala }
8671a13f2e6SEdward Tomasz Napierala
8681a13f2e6SEdward Tomasz Napierala int
8691a13f2e6SEdward Tomasz Napierala stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
8701a13f2e6SEdward Tomasz Napierala struct vss_numeric_hlpr_info *info)
8711a13f2e6SEdward Tomasz Napierala {
8721a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric iv;
8731a13f2e6SEdward Tomasz Napierala
8741a13f2e6SEdward Tomasz Napierala switch (vss->stype) {
8751a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM:
8761a13f2e6SEdward Tomasz Napierala iv = stats_ctor_vsd_numeric(0);
8771a13f2e6SEdward Tomasz Napierala break;
8781a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN:
8791a13f2e6SEdward Tomasz Napierala iv = numeric_limits[LIM_MAX][voi_dtype];
8801a13f2e6SEdward Tomasz Napierala break;
8811a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX:
8821a13f2e6SEdward Tomasz Napierala iv = numeric_limits[LIM_MIN][voi_dtype];
8831a13f2e6SEdward Tomasz Napierala break;
8841a13f2e6SEdward Tomasz Napierala default:
8851a13f2e6SEdward Tomasz Napierala return (EINVAL);
8861a13f2e6SEdward Tomasz Napierala }
8871a13f2e6SEdward Tomasz Napierala
8881a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0);
8891a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL)
8901a13f2e6SEdward Tomasz Napierala return (ENOMEM);
8911a13f2e6SEdward Tomasz Napierala
8921a13f2e6SEdward Tomasz Napierala vss->vs_dtype = voi_dtype;
8931a13f2e6SEdward Tomasz Napierala vss->vsdsz = vsd_dtype2size[voi_dtype];
8941a13f2e6SEdward Tomasz Napierala switch (voi_dtype) {
8951a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
8961a13f2e6SEdward Tomasz Napierala *((int32_t *)vss->iv) = iv.int32.s32;
8971a13f2e6SEdward Tomasz Napierala break;
8981a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
8991a13f2e6SEdward Tomasz Napierala *((uint32_t *)vss->iv) = iv.int32.u32;
9001a13f2e6SEdward Tomasz Napierala break;
9011a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
9021a13f2e6SEdward Tomasz Napierala *((int64_t *)vss->iv) = iv.int64.s64;
9031a13f2e6SEdward Tomasz Napierala break;
9041a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
9051a13f2e6SEdward Tomasz Napierala *((uint64_t *)vss->iv) = iv.int64.u64;
9061a13f2e6SEdward Tomasz Napierala break;
9071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
9081a13f2e6SEdward Tomasz Napierala *((long *)vss->iv) = iv.intlong.slong;
9091a13f2e6SEdward Tomasz Napierala break;
9101a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
9111a13f2e6SEdward Tomasz Napierala *((unsigned long *)vss->iv) = iv.intlong.ulong;
9121a13f2e6SEdward Tomasz Napierala break;
9131a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
9141a13f2e6SEdward Tomasz Napierala *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32,
9151a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec));
9161a13f2e6SEdward Tomasz Napierala break;
9171a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
9181a13f2e6SEdward Tomasz Napierala *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32,
9191a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec));
9201a13f2e6SEdward Tomasz Napierala break;
9211a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
9221a13f2e6SEdward Tomasz Napierala *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64,
9231a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec));
9241a13f2e6SEdward Tomasz Napierala break;
9251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
9261a13f2e6SEdward Tomasz Napierala *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64,
9271a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec));
9281a13f2e6SEdward Tomasz Napierala break;
9291a13f2e6SEdward Tomasz Napierala default:
9301a13f2e6SEdward Tomasz Napierala break;
9311a13f2e6SEdward Tomasz Napierala }
9321a13f2e6SEdward Tomasz Napierala
9331a13f2e6SEdward Tomasz Napierala return (0);
9341a13f2e6SEdward Tomasz Napierala }
9351a13f2e6SEdward Tomasz Napierala
9361a13f2e6SEdward Tomasz Napierala int
9371a13f2e6SEdward Tomasz Napierala stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss,
9381a13f2e6SEdward Tomasz Napierala struct voistatspec *vss)
9391a13f2e6SEdward Tomasz Napierala {
9401a13f2e6SEdward Tomasz Napierala int i, ret;
9411a13f2e6SEdward Tomasz Napierala
9421a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) {
9431a13f2e6SEdward Tomasz Napierala if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i],
9441a13f2e6SEdward Tomasz Napierala vss[i].hlprinfo)) != 0)
9451a13f2e6SEdward Tomasz Napierala return (ret);
9461a13f2e6SEdward Tomasz Napierala }
9471a13f2e6SEdward Tomasz Napierala
9481a13f2e6SEdward Tomasz Napierala return (0);
9491a13f2e6SEdward Tomasz Napierala }
9501a13f2e6SEdward Tomasz Napierala
9511a13f2e6SEdward Tomasz Napierala void
9521a13f2e6SEdward Tomasz Napierala stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss)
9531a13f2e6SEdward Tomasz Napierala {
9541a13f2e6SEdward Tomasz Napierala int i;
9551a13f2e6SEdward Tomasz Napierala
9561a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) {
9571a13f2e6SEdward Tomasz Napierala if (vss[i].hlpr) {
9581a13f2e6SEdward Tomasz Napierala stats_free((void *)vss[i].iv);
9591a13f2e6SEdward Tomasz Napierala vss[i].iv = NULL;
9601a13f2e6SEdward Tomasz Napierala }
9611a13f2e6SEdward Tomasz Napierala }
9621a13f2e6SEdward Tomasz Napierala }
9631a13f2e6SEdward Tomasz Napierala
9641a13f2e6SEdward Tomasz Napierala int
9651a13f2e6SEdward Tomasz Napierala stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl)
9661a13f2e6SEdward Tomasz Napierala {
9671a13f2e6SEdward Tomasz Napierala int error;
9681a13f2e6SEdward Tomasz Napierala
9691a13f2e6SEdward Tomasz Napierala error = 0;
9701a13f2e6SEdward Tomasz Napierala
9711a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK();
9721a13f2e6SEdward Tomasz Napierala if (tpl_id < 0 || tpl_id >= (int)ntpl) {
9731a13f2e6SEdward Tomasz Napierala error = ENOENT;
9741a13f2e6SEdward Tomasz Napierala } else {
9751a13f2e6SEdward Tomasz Napierala *tpl = tpllist[tpl_id];
9761a13f2e6SEdward Tomasz Napierala /* XXXLAS: Acquire refcount on tpl. */
9771a13f2e6SEdward Tomasz Napierala }
9781a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK();
9791a13f2e6SEdward Tomasz Napierala
9801a13f2e6SEdward Tomasz Napierala return (error);
9811a13f2e6SEdward Tomasz Napierala }
9821a13f2e6SEdward Tomasz Napierala
9831a13f2e6SEdward Tomasz Napierala int
9841a13f2e6SEdward Tomasz Napierala stats_tpl_fetch_allocid(const char *name, uint32_t hash)
9851a13f2e6SEdward Tomasz Napierala {
9861a13f2e6SEdward Tomasz Napierala int i, tpl_id;
9871a13f2e6SEdward Tomasz Napierala
9881a13f2e6SEdward Tomasz Napierala tpl_id = -ESRCH;
9891a13f2e6SEdward Tomasz Napierala
9901a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK();
9911a13f2e6SEdward Tomasz Napierala for (i = ntpl - 1; i >= 0; i--) {
9921a13f2e6SEdward Tomasz Napierala if (name != NULL) {
9931a13f2e6SEdward Tomasz Napierala if (strlen(name) == strlen(tpllist[i]->mb->tplname) &&
9941a13f2e6SEdward Tomasz Napierala strncmp(name, tpllist[i]->mb->tplname,
9951a13f2e6SEdward Tomasz Napierala TPL_MAX_NAME_LEN) == 0 && (!hash || hash ==
9961a13f2e6SEdward Tomasz Napierala tpllist[i]->mb->tplhash)) {
9971a13f2e6SEdward Tomasz Napierala tpl_id = i;
9981a13f2e6SEdward Tomasz Napierala break;
9991a13f2e6SEdward Tomasz Napierala }
10001a13f2e6SEdward Tomasz Napierala } else if (hash == tpllist[i]->mb->tplhash) {
10011a13f2e6SEdward Tomasz Napierala tpl_id = i;
10021a13f2e6SEdward Tomasz Napierala break;
10031a13f2e6SEdward Tomasz Napierala }
10041a13f2e6SEdward Tomasz Napierala }
10051a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK();
10061a13f2e6SEdward Tomasz Napierala
10071a13f2e6SEdward Tomasz Napierala return (tpl_id);
10081a13f2e6SEdward Tomasz Napierala }
10091a13f2e6SEdward Tomasz Napierala
10101a13f2e6SEdward Tomasz Napierala int
10111a13f2e6SEdward Tomasz Napierala stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len)
10121a13f2e6SEdward Tomasz Napierala {
10131a13f2e6SEdward Tomasz Napierala int error;
10141a13f2e6SEdward Tomasz Napierala
10151a13f2e6SEdward Tomasz Napierala error = 0;
10161a13f2e6SEdward Tomasz Napierala
10171a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK();
10181a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) {
10191a13f2e6SEdward Tomasz Napierala if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname))
10201a13f2e6SEdward Tomasz Napierala strlcpy(buf, tpllist[tpl_id]->mb->tplname, len);
10211a13f2e6SEdward Tomasz Napierala else
10221a13f2e6SEdward Tomasz Napierala error = EOVERFLOW;
10231a13f2e6SEdward Tomasz Napierala } else
10241a13f2e6SEdward Tomasz Napierala error = ENOENT;
10251a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK();
10261a13f2e6SEdward Tomasz Napierala
10271a13f2e6SEdward Tomasz Napierala return (error);
10281a13f2e6SEdward Tomasz Napierala }
10291a13f2e6SEdward Tomasz Napierala
10301a13f2e6SEdward Tomasz Napierala int
10311a13f2e6SEdward Tomasz Napierala stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates,
10321a13f2e6SEdward Tomasz Napierala void *seed_bytes, size_t seed_len)
10331a13f2e6SEdward Tomasz Napierala {
10341a13f2e6SEdward Tomasz Napierala uint32_t cum_pct, rnd_pct;
10351a13f2e6SEdward Tomasz Napierala int i;
10361a13f2e6SEdward Tomasz Napierala
10371a13f2e6SEdward Tomasz Napierala cum_pct = 0;
10381a13f2e6SEdward Tomasz Napierala
10391a13f2e6SEdward Tomasz Napierala /*
10401a13f2e6SEdward Tomasz Napierala * Choose a pseudorandom or seeded number in range [0,100] and use
10411a13f2e6SEdward Tomasz Napierala * it to make a sampling decision and template selection where required.
10421a13f2e6SEdward Tomasz Napierala * If no seed is supplied, a PRNG is used to generate a pseudorandom
10431a13f2e6SEdward Tomasz Napierala * number so that every selection is independent. If a seed is supplied,
10441a13f2e6SEdward Tomasz Napierala * the caller desires random selection across different seeds, but
10451a13f2e6SEdward Tomasz Napierala * deterministic selection given the same seed. This is achieved by
10461a13f2e6SEdward Tomasz Napierala * hashing the seed and using the hash as the random number source.
10471a13f2e6SEdward Tomasz Napierala *
10481a13f2e6SEdward Tomasz Napierala * XXXLAS: Characterise hash function output distribution.
10491a13f2e6SEdward Tomasz Napierala */
10501a13f2e6SEdward Tomasz Napierala if (seed_bytes == NULL)
10511a13f2e6SEdward Tomasz Napierala rnd_pct = random() / (INT32_MAX / 100);
10521a13f2e6SEdward Tomasz Napierala else
10531a13f2e6SEdward Tomasz Napierala rnd_pct = hash32_buf(seed_bytes, seed_len, 0) /
10541a13f2e6SEdward Tomasz Napierala (UINT32_MAX / 100U);
10551a13f2e6SEdward Tomasz Napierala
10561a13f2e6SEdward Tomasz Napierala /*
10571a13f2e6SEdward Tomasz Napierala * We map the randomly selected percentage on to the interval [0,100]
10581a13f2e6SEdward Tomasz Napierala * consisting of the cumulatively summed template sampling percentages.
10591a13f2e6SEdward Tomasz Napierala * The difference between the cumulative sum of all template sampling
10601a13f2e6SEdward Tomasz Napierala * percentages and 100 is treated as a NULL assignment i.e. no stats
10611a13f2e6SEdward Tomasz Napierala * template will be assigned, and -1 returned instead.
10621a13f2e6SEdward Tomasz Napierala */
10631a13f2e6SEdward Tomasz Napierala for (i = 0; i < nrates; i++) {
10641a13f2e6SEdward Tomasz Napierala cum_pct += rates[i].tpl_sample_pct;
10651a13f2e6SEdward Tomasz Napierala
10661a13f2e6SEdward Tomasz Napierala KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__,
10671a13f2e6SEdward Tomasz Napierala cum_pct));
10681a13f2e6SEdward Tomasz Napierala if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0)
10691a13f2e6SEdward Tomasz Napierala continue;
10701a13f2e6SEdward Tomasz Napierala
10711a13f2e6SEdward Tomasz Napierala return (rates[i].tpl_slot_id);
10721a13f2e6SEdward Tomasz Napierala }
10731a13f2e6SEdward Tomasz Napierala
10741a13f2e6SEdward Tomasz Napierala return (-1);
10751a13f2e6SEdward Tomasz Napierala }
10761a13f2e6SEdward Tomasz Napierala
10771a13f2e6SEdward Tomasz Napierala int
10781a13f2e6SEdward Tomasz Napierala stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz,
10791a13f2e6SEdward Tomasz Napierala struct statsblobv1 *src, uint32_t flags)
10801a13f2e6SEdward Tomasz Napierala {
10817eb92c50SLawrence Stewart int error, tmperror;
10821a13f2e6SEdward Tomasz Napierala
10837eb92c50SLawrence Stewart error = tmperror = 0;
10841a13f2e6SEdward Tomasz Napierala
10851a13f2e6SEdward Tomasz Napierala if (src == NULL || dst == NULL ||
10861a13f2e6SEdward Tomasz Napierala src->cursz < sizeof(struct statsblob) ||
10871a13f2e6SEdward Tomasz Napierala ((flags & SB_CLONE_ALLOCDST) &&
10881a13f2e6SEdward Tomasz Napierala (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) {
10891a13f2e6SEdward Tomasz Napierala error = EINVAL;
10901a13f2e6SEdward Tomasz Napierala } else if (flags & SB_CLONE_ALLOCDST) {
10911a13f2e6SEdward Tomasz Napierala *dst = stats_realloc(NULL, 0, src->cursz, 0);
10921a13f2e6SEdward Tomasz Napierala if (*dst)
10931a13f2e6SEdward Tomasz Napierala (*dst)->maxsz = dstmaxsz = src->cursz;
10941a13f2e6SEdward Tomasz Napierala else
10951a13f2e6SEdward Tomasz Napierala error = ENOMEM;
10961a13f2e6SEdward Tomasz Napierala } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) {
10971a13f2e6SEdward Tomasz Napierala error = EINVAL;
10981a13f2e6SEdward Tomasz Napierala }
10991a13f2e6SEdward Tomasz Napierala
11001a13f2e6SEdward Tomasz Napierala if (!error) {
11011a13f2e6SEdward Tomasz Napierala size_t postcurszlen;
11021a13f2e6SEdward Tomasz Napierala
11031a13f2e6SEdward Tomasz Napierala /*
11041a13f2e6SEdward Tomasz Napierala * Clone src into dst except for the maxsz field. If dst is too
11051a13f2e6SEdward Tomasz Napierala * small to hold all of src, only copy src's header and return
11061a13f2e6SEdward Tomasz Napierala * EOVERFLOW.
11071a13f2e6SEdward Tomasz Napierala */
11081a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
11091a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_USRDSTNOFAULT)
1110a0993376SMark Johnston error = copyout_nofault(src, *dst,
11111a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, maxsz));
11121a13f2e6SEdward Tomasz Napierala else if (flags & SB_CLONE_USRDST)
1113a0993376SMark Johnston error = copyout(src, *dst,
1114a0993376SMark Johnston offsetof(struct statsblob, maxsz));
11151a13f2e6SEdward Tomasz Napierala else
11161a13f2e6SEdward Tomasz Napierala #endif
11171a13f2e6SEdward Tomasz Napierala memcpy(*dst, src, offsetof(struct statsblob, maxsz));
1118a0993376SMark Johnston #ifdef _KERNEL
1119a0993376SMark Johnston if (error != 0)
1120a0993376SMark Johnston goto out;
1121a0993376SMark Johnston #endif
1122a0993376SMark Johnston
11231a13f2e6SEdward Tomasz Napierala
11241a13f2e6SEdward Tomasz Napierala if (dstmaxsz >= src->cursz) {
11251a13f2e6SEdward Tomasz Napierala postcurszlen = src->cursz -
11261a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, cursz);
11271a13f2e6SEdward Tomasz Napierala } else {
11281a13f2e6SEdward Tomasz Napierala error = EOVERFLOW;
11291a13f2e6SEdward Tomasz Napierala postcurszlen = sizeof(struct statsblob) -
11301a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, cursz);
11311a13f2e6SEdward Tomasz Napierala }
11321a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
11331a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_USRDSTNOFAULT)
11347eb92c50SLawrence Stewart tmperror = copyout_nofault(&(src->cursz), &((*dst)->cursz),
11351a13f2e6SEdward Tomasz Napierala postcurszlen);
11361a13f2e6SEdward Tomasz Napierala else if (flags & SB_CLONE_USRDST)
11377eb92c50SLawrence Stewart tmperror = copyout(&(src->cursz), &((*dst)->cursz),
1138a0993376SMark Johnston postcurszlen);
11391a13f2e6SEdward Tomasz Napierala else
11401a13f2e6SEdward Tomasz Napierala #endif
11411a13f2e6SEdward Tomasz Napierala memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen);
11427eb92c50SLawrence Stewart
11437eb92c50SLawrence Stewart error = error ? error : tmperror;
11441a13f2e6SEdward Tomasz Napierala }
1145a0993376SMark Johnston #ifdef _KERNEL
1146a0993376SMark Johnston out:
1147a0993376SMark Johnston #endif
11481a13f2e6SEdward Tomasz Napierala
11491a13f2e6SEdward Tomasz Napierala return (error);
11501a13f2e6SEdward Tomasz Napierala }
11511a13f2e6SEdward Tomasz Napierala
11521a13f2e6SEdward Tomasz Napierala int
11531a13f2e6SEdward Tomasz Napierala stats_v1_tpl_alloc(const char *name, uint32_t flags __unused)
11541a13f2e6SEdward Tomasz Napierala {
11551a13f2e6SEdward Tomasz Napierala struct statsblobv1_tpl *tpl, **newtpllist;
11561a13f2e6SEdward Tomasz Napierala struct statsblobv1 *tpl_sb;
11571a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb;
11581a13f2e6SEdward Tomasz Napierala int tpl_id;
11591a13f2e6SEdward Tomasz Napierala
11601a13f2e6SEdward Tomasz Napierala if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN)
11611a13f2e6SEdward Tomasz Napierala return (-EINVAL);
11621a13f2e6SEdward Tomasz Napierala
11631a13f2e6SEdward Tomasz Napierala if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0)
11641a13f2e6SEdward Tomasz Napierala return (-EEXIST);
11651a13f2e6SEdward Tomasz Napierala
11661a13f2e6SEdward Tomasz Napierala tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO);
11671a13f2e6SEdward Tomasz Napierala tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO);
11681a13f2e6SEdward Tomasz Napierala tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO);
11691a13f2e6SEdward Tomasz Napierala
11701a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL && name != NULL)
11711a13f2e6SEdward Tomasz Napierala tpl_mb->tplname = stats_strdup(name, 0);
11721a13f2e6SEdward Tomasz Napierala
11731a13f2e6SEdward Tomasz Napierala if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL ||
11741a13f2e6SEdward Tomasz Napierala tpl_mb->tplname == NULL) {
11751a13f2e6SEdward Tomasz Napierala stats_free(tpl);
11761a13f2e6SEdward Tomasz Napierala stats_free(tpl_sb);
11771a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL) {
11781a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb->tplname);
11791a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb);
11801a13f2e6SEdward Tomasz Napierala }
11811a13f2e6SEdward Tomasz Napierala return (-ENOMEM);
11821a13f2e6SEdward Tomasz Napierala }
11831a13f2e6SEdward Tomasz Napierala
11841a13f2e6SEdward Tomasz Napierala tpl->mb = tpl_mb;
11851a13f2e6SEdward Tomasz Napierala tpl->sb = tpl_sb;
11861a13f2e6SEdward Tomasz Napierala
11871a13f2e6SEdward Tomasz Napierala tpl_sb->abi = STATS_ABI_V1;
11881a13f2e6SEdward Tomasz Napierala tpl_sb->endian =
11891a13f2e6SEdward Tomasz Napierala #if BYTE_ORDER == LITTLE_ENDIAN
11901a13f2e6SEdward Tomasz Napierala SB_LE;
11911a13f2e6SEdward Tomasz Napierala #elif BYTE_ORDER == BIG_ENDIAN
11921a13f2e6SEdward Tomasz Napierala SB_BE;
11931a13f2e6SEdward Tomasz Napierala #else
11941a13f2e6SEdward Tomasz Napierala SB_UE;
11951a13f2e6SEdward Tomasz Napierala #endif
11961a13f2e6SEdward Tomasz Napierala tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1);
11971a13f2e6SEdward Tomasz Napierala tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1);
11981a13f2e6SEdward Tomasz Napierala
11991a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK();
12001a13f2e6SEdward Tomasz Napierala newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *),
12011a13f2e6SEdward Tomasz Napierala (ntpl + 1) * sizeof(void *), 0);
12021a13f2e6SEdward Tomasz Napierala if (newtpllist != NULL) {
12031a13f2e6SEdward Tomasz Napierala tpl_id = ntpl++;
12041a13f2e6SEdward Tomasz Napierala tpllist = (struct statsblob_tpl **)newtpllist;
12051a13f2e6SEdward Tomasz Napierala tpllist[tpl_id] = (struct statsblob_tpl *)tpl;
12061a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(tpllist[tpl_id]);
12071a13f2e6SEdward Tomasz Napierala } else {
12081a13f2e6SEdward Tomasz Napierala stats_free(tpl);
12091a13f2e6SEdward Tomasz Napierala stats_free(tpl_sb);
12101a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL) {
12111a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb->tplname);
12121a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb);
12131a13f2e6SEdward Tomasz Napierala }
12141a13f2e6SEdward Tomasz Napierala tpl_id = -ENOMEM;
12151a13f2e6SEdward Tomasz Napierala }
12161a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK();
12171a13f2e6SEdward Tomasz Napierala
12181a13f2e6SEdward Tomasz Napierala return (tpl_id);
12191a13f2e6SEdward Tomasz Napierala }
12201a13f2e6SEdward Tomasz Napierala
12211a13f2e6SEdward Tomasz Napierala int
12221a13f2e6SEdward Tomasz Napierala stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name,
12231a13f2e6SEdward Tomasz Napierala enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss,
12241a13f2e6SEdward Tomasz Napierala uint32_t flags)
12251a13f2e6SEdward Tomasz Napierala {
12261a13f2e6SEdward Tomasz Napierala struct voi *voi;
12271a13f2e6SEdward Tomasz Napierala struct voistat *tmpstat;
12281a13f2e6SEdward Tomasz Napierala struct statsblobv1 *tpl_sb;
12291a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb;
12301a13f2e6SEdward Tomasz Napierala int error, i, newstatdataidx, newvoibytes, newvoistatbytes,
12311a13f2e6SEdward Tomasz Napierala newvoistatdatabytes, newvoistatmaxid;
12321a13f2e6SEdward Tomasz Napierala uint32_t nbytes;
12331a13f2e6SEdward Tomasz Napierala
12341a13f2e6SEdward Tomasz Napierala if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES ||
12351a13f2e6SEdward Tomasz Napierala nvss == 0 || vss == NULL)
12361a13f2e6SEdward Tomasz Napierala return (EINVAL);
12371a13f2e6SEdward Tomasz Napierala
12381a13f2e6SEdward Tomasz Napierala error = nbytes = newvoibytes = newvoistatbytes =
12391a13f2e6SEdward Tomasz Napierala newvoistatdatabytes = 0;
12401a13f2e6SEdward Tomasz Napierala newvoistatmaxid = -1;
12411a13f2e6SEdward Tomasz Napierala
12421a13f2e6SEdward Tomasz Napierala /* Calculate the number of bytes required for the new voistats. */
12431a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) {
12441a13f2e6SEdward Tomasz Napierala if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES ||
12451a13f2e6SEdward Tomasz Napierala vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES ||
12461a13f2e6SEdward Tomasz Napierala vss[i].iv == NULL || vss[i].vsdsz == 0)
12471a13f2e6SEdward Tomasz Napierala return (EINVAL);
12481a13f2e6SEdward Tomasz Napierala if ((int)vss[i].stype > newvoistatmaxid)
12491a13f2e6SEdward Tomasz Napierala newvoistatmaxid = vss[i].stype;
12501a13f2e6SEdward Tomasz Napierala newvoistatdatabytes += vss[i].vsdsz;
12511a13f2e6SEdward Tomasz Napierala }
12521a13f2e6SEdward Tomasz Napierala
12531a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) {
12541a13f2e6SEdward Tomasz Napierala /* XXXLAS: VOI state bytes may need to vary based on stat types. */
12551a13f2e6SEdward Tomasz Napierala newvoistatdatabytes += sizeof(struct voistatdata_voistate);
12561a13f2e6SEdward Tomasz Napierala }
12571a13f2e6SEdward Tomasz Napierala nbytes += newvoistatdatabytes;
12581a13f2e6SEdward Tomasz Napierala
12591a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK();
12601a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) {
12611a13f2e6SEdward Tomasz Napierala tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb;
12621a13f2e6SEdward Tomasz Napierala tpl_mb = tpllist[tpl_id]->mb;
12631a13f2e6SEdward Tomasz Napierala
12641a13f2e6SEdward Tomasz Napierala if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) {
12651a13f2e6SEdward Tomasz Napierala /* Adding a new VOI and associated stats. */
12661a13f2e6SEdward Tomasz Napierala if (voi_id >= NVOIS(tpl_sb)) {
12671a13f2e6SEdward Tomasz Napierala /* We need to grow the tpl_sb->vois array. */
12681a13f2e6SEdward Tomasz Napierala newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) *
12691a13f2e6SEdward Tomasz Napierala sizeof(struct voi);
12701a13f2e6SEdward Tomasz Napierala nbytes += newvoibytes;
12711a13f2e6SEdward Tomasz Napierala }
12721a13f2e6SEdward Tomasz Napierala newvoistatbytes =
12731a13f2e6SEdward Tomasz Napierala (newvoistatmaxid + 1) * sizeof(struct voistat);
12741a13f2e6SEdward Tomasz Napierala } else {
12751a13f2e6SEdward Tomasz Napierala /* Adding stats to an existing VOI. */
12761a13f2e6SEdward Tomasz Napierala if (newvoistatmaxid >
12771a13f2e6SEdward Tomasz Napierala tpl_sb->vois[voi_id].voistatmaxid) {
12781a13f2e6SEdward Tomasz Napierala newvoistatbytes = (newvoistatmaxid -
12791a13f2e6SEdward Tomasz Napierala tpl_sb->vois[voi_id].voistatmaxid) *
12801a13f2e6SEdward Tomasz Napierala sizeof(struct voistat);
12811a13f2e6SEdward Tomasz Napierala }
12821a13f2e6SEdward Tomasz Napierala /* XXXLAS: KPI does not yet support expanding VOIs. */
12831a13f2e6SEdward Tomasz Napierala error = EOPNOTSUPP;
12841a13f2e6SEdward Tomasz Napierala }
12851a13f2e6SEdward Tomasz Napierala nbytes += newvoistatbytes;
12861a13f2e6SEdward Tomasz Napierala
12871a13f2e6SEdward Tomasz Napierala if (!error && newvoibytes > 0) {
12881a13f2e6SEdward Tomasz Napierala struct voi_meta *voi_meta = tpl_mb->voi_meta;
12891a13f2e6SEdward Tomasz Napierala
12901a13f2e6SEdward Tomasz Napierala voi_meta = stats_realloc(voi_meta, voi_meta == NULL ?
12911a13f2e6SEdward Tomasz Napierala 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta),
12921a13f2e6SEdward Tomasz Napierala (1 + voi_id) * sizeof(struct voi_meta),
12931a13f2e6SEdward Tomasz Napierala M_ZERO);
12941a13f2e6SEdward Tomasz Napierala
12951a13f2e6SEdward Tomasz Napierala if (voi_meta == NULL)
12961a13f2e6SEdward Tomasz Napierala error = ENOMEM;
12971a13f2e6SEdward Tomasz Napierala else
12981a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta = voi_meta;
12991a13f2e6SEdward Tomasz Napierala }
13001a13f2e6SEdward Tomasz Napierala
13011a13f2e6SEdward Tomasz Napierala if (!error) {
13021a13f2e6SEdward Tomasz Napierala /* NB: Resizing can change where tpl_sb points. */
13031a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_expand(&tpl_sb, newvoibytes,
13041a13f2e6SEdward Tomasz Napierala newvoistatbytes, newvoistatdatabytes);
13051a13f2e6SEdward Tomasz Napierala }
13061a13f2e6SEdward Tomasz Napierala
13071a13f2e6SEdward Tomasz Napierala if (!error) {
13081a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name,
13091a13f2e6SEdward Tomasz Napierala 0);
13101a13f2e6SEdward Tomasz Napierala if (tpl_mb->voi_meta[voi_id].name == NULL)
13111a13f2e6SEdward Tomasz Napierala error = ENOMEM;
13121a13f2e6SEdward Tomasz Napierala }
13131a13f2e6SEdward Tomasz Napierala
13141a13f2e6SEdward Tomasz Napierala if (!error) {
13151a13f2e6SEdward Tomasz Napierala /* Update the template list with the resized pointer. */
13161a13f2e6SEdward Tomasz Napierala tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb;
13171a13f2e6SEdward Tomasz Napierala
13181a13f2e6SEdward Tomasz Napierala /* Update the template. */
13191a13f2e6SEdward Tomasz Napierala voi = &tpl_sb->vois[voi_id];
13201a13f2e6SEdward Tomasz Napierala
13211a13f2e6SEdward Tomasz Napierala if (voi->id < 0) {
13221a13f2e6SEdward Tomasz Napierala /* VOI is new and needs to be initialised. */
13231a13f2e6SEdward Tomasz Napierala voi->id = voi_id;
13241a13f2e6SEdward Tomasz Napierala voi->dtype = voi_dtype;
13251a13f2e6SEdward Tomasz Napierala voi->stats_off = tpl_sb->stats_off;
13261a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE)
13271a13f2e6SEdward Tomasz Napierala voi->flags |= VOI_REQSTATE;
13281a13f2e6SEdward Tomasz Napierala } else {
13291a13f2e6SEdward Tomasz Napierala /*
13301a13f2e6SEdward Tomasz Napierala * XXXLAS: When this else block is written, the
13311a13f2e6SEdward Tomasz Napierala * "KPI does not yet support expanding VOIs"
13321a13f2e6SEdward Tomasz Napierala * error earlier in this function can be
13331a13f2e6SEdward Tomasz Napierala * removed. What is required here is to shuffle
13341a13f2e6SEdward Tomasz Napierala * the voistat array such that the new stats for
13351a13f2e6SEdward Tomasz Napierala * the voi are contiguous, which will displace
13361a13f2e6SEdward Tomasz Napierala * stats for other vois that reside after the
13371a13f2e6SEdward Tomasz Napierala * voi being updated. The other vois then need
13381a13f2e6SEdward Tomasz Napierala * to have their stats_off adjusted post
13391a13f2e6SEdward Tomasz Napierala * shuffle.
13401a13f2e6SEdward Tomasz Napierala */
13411a13f2e6SEdward Tomasz Napierala }
13421a13f2e6SEdward Tomasz Napierala
13431a13f2e6SEdward Tomasz Napierala voi->voistatmaxid = newvoistatmaxid;
13441a13f2e6SEdward Tomasz Napierala newstatdataidx = 0;
13451a13f2e6SEdward Tomasz Napierala
13461a13f2e6SEdward Tomasz Napierala if (voi->flags & VOI_REQSTATE) {
13471a13f2e6SEdward Tomasz Napierala /* Initialise the voistate stat in slot 0. */
13481a13f2e6SEdward Tomasz Napierala tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off);
13491a13f2e6SEdward Tomasz Napierala tmpstat->stype = VS_STYPE_VOISTATE;
13501a13f2e6SEdward Tomasz Napierala tmpstat->flags = 0;
13511a13f2e6SEdward Tomasz Napierala tmpstat->dtype = VSD_DTYPE_VOISTATE;
13521a13f2e6SEdward Tomasz Napierala newstatdataidx = tmpstat->dsz =
13531a13f2e6SEdward Tomasz Napierala sizeof(struct voistatdata_numeric);
13541a13f2e6SEdward Tomasz Napierala tmpstat->data_off = tpl_sb->statsdata_off;
13551a13f2e6SEdward Tomasz Napierala }
13561a13f2e6SEdward Tomasz Napierala
13571a13f2e6SEdward Tomasz Napierala for (i = 0; (uint32_t)i < nvss; i++) {
13581a13f2e6SEdward Tomasz Napierala tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off +
13591a13f2e6SEdward Tomasz Napierala (vss[i].stype * sizeof(struct voistat)));
13601a13f2e6SEdward Tomasz Napierala KASSERT(tmpstat->stype < 0, ("voistat %p "
13611a13f2e6SEdward Tomasz Napierala "already initialised", tmpstat));
13621a13f2e6SEdward Tomasz Napierala tmpstat->stype = vss[i].stype;
13631a13f2e6SEdward Tomasz Napierala tmpstat->flags = vss[i].flags;
13641a13f2e6SEdward Tomasz Napierala tmpstat->dtype = vss[i].vs_dtype;
13651a13f2e6SEdward Tomasz Napierala tmpstat->dsz = vss[i].vsdsz;
13661a13f2e6SEdward Tomasz Napierala tmpstat->data_off = tpl_sb->statsdata_off +
13671a13f2e6SEdward Tomasz Napierala newstatdataidx;
13681a13f2e6SEdward Tomasz Napierala memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off),
13691a13f2e6SEdward Tomasz Napierala vss[i].iv, vss[i].vsdsz);
13701a13f2e6SEdward Tomasz Napierala newstatdataidx += vss[i].vsdsz;
13711a13f2e6SEdward Tomasz Napierala }
13721a13f2e6SEdward Tomasz Napierala
13731a13f2e6SEdward Tomasz Napierala /* Update the template version hash. */
13741a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(tpllist[tpl_id]);
13751a13f2e6SEdward Tomasz Napierala /* XXXLAS: Confirm tpl name/hash pair remains unique. */
13761a13f2e6SEdward Tomasz Napierala }
13771a13f2e6SEdward Tomasz Napierala } else
13781a13f2e6SEdward Tomasz Napierala error = EINVAL;
13791a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK();
13801a13f2e6SEdward Tomasz Napierala
13811a13f2e6SEdward Tomasz Napierala return (error);
13821a13f2e6SEdward Tomasz Napierala }
13831a13f2e6SEdward Tomasz Napierala
13841a13f2e6SEdward Tomasz Napierala struct statsblobv1 *
13851a13f2e6SEdward Tomasz Napierala stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused)
13861a13f2e6SEdward Tomasz Napierala {
13871a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb;
13881a13f2e6SEdward Tomasz Napierala int error;
13891a13f2e6SEdward Tomasz Napierala
13901a13f2e6SEdward Tomasz Napierala sb = NULL;
13911a13f2e6SEdward Tomasz Napierala
13921a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK();
13931a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) {
13941a13f2e6SEdward Tomasz Napierala sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0);
13951a13f2e6SEdward Tomasz Napierala if (sb != NULL) {
13961a13f2e6SEdward Tomasz Napierala sb->maxsz = tpllist[tpl_id]->sb->maxsz;
13971a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_init_locked(sb, tpl_id, 0);
13981a13f2e6SEdward Tomasz Napierala } else
13991a13f2e6SEdward Tomasz Napierala error = ENOMEM;
14001a13f2e6SEdward Tomasz Napierala
14011a13f2e6SEdward Tomasz Napierala if (error) {
14021a13f2e6SEdward Tomasz Napierala stats_free(sb);
14031a13f2e6SEdward Tomasz Napierala sb = NULL;
14041a13f2e6SEdward Tomasz Napierala }
14051a13f2e6SEdward Tomasz Napierala }
14061a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK();
14071a13f2e6SEdward Tomasz Napierala
14081a13f2e6SEdward Tomasz Napierala return (sb);
14091a13f2e6SEdward Tomasz Napierala }
14101a13f2e6SEdward Tomasz Napierala
14111a13f2e6SEdward Tomasz Napierala void
14121a13f2e6SEdward Tomasz Napierala stats_v1_blob_destroy(struct statsblobv1 *sb)
14131a13f2e6SEdward Tomasz Napierala {
14141a13f2e6SEdward Tomasz Napierala
14151a13f2e6SEdward Tomasz Napierala stats_free(sb);
14161a13f2e6SEdward Tomasz Napierala }
14171a13f2e6SEdward Tomasz Napierala
14181a13f2e6SEdward Tomasz Napierala int
14191a13f2e6SEdward Tomasz Napierala stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id,
14201a13f2e6SEdward Tomasz Napierala enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd,
14211a13f2e6SEdward Tomasz Napierala size_t *retvsdsz)
14221a13f2e6SEdward Tomasz Napierala {
14231a13f2e6SEdward Tomasz Napierala struct voi *v;
14241a13f2e6SEdward Tomasz Napierala struct voistat *vs;
14251a13f2e6SEdward Tomasz Napierala
14261a13f2e6SEdward Tomasz Napierala if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 ||
14271a13f2e6SEdward Tomasz Napierala voi_id >= NVOIS(sb))
14281a13f2e6SEdward Tomasz Napierala return (EINVAL);
14291a13f2e6SEdward Tomasz Napierala
14301a13f2e6SEdward Tomasz Napierala v = &sb->vois[voi_id];
14311a13f2e6SEdward Tomasz Napierala if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid)
14321a13f2e6SEdward Tomasz Napierala return (EINVAL);
14331a13f2e6SEdward Tomasz Napierala
14341a13f2e6SEdward Tomasz Napierala vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat)));
14351a13f2e6SEdward Tomasz Napierala *retvsd = BLOB_OFFSET(sb, vs->data_off);
14361a13f2e6SEdward Tomasz Napierala if (retdtype != NULL)
14371a13f2e6SEdward Tomasz Napierala *retdtype = vs->dtype;
14381a13f2e6SEdward Tomasz Napierala if (retvsdsz != NULL)
14391a13f2e6SEdward Tomasz Napierala *retvsdsz = vs->dsz;
14401a13f2e6SEdward Tomasz Napierala
14411a13f2e6SEdward Tomasz Napierala return (0);
14421a13f2e6SEdward Tomasz Napierala }
14431a13f2e6SEdward Tomasz Napierala
14441a13f2e6SEdward Tomasz Napierala int
14451a13f2e6SEdward Tomasz Napierala stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags)
14461a13f2e6SEdward Tomasz Napierala {
14471a13f2e6SEdward Tomasz Napierala int error;
14481a13f2e6SEdward Tomasz Napierala
14491a13f2e6SEdward Tomasz Napierala error = 0;
14501a13f2e6SEdward Tomasz Napierala
14511a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK();
14521a13f2e6SEdward Tomasz Napierala if (sb == NULL || tpl_id >= ntpl) {
14531a13f2e6SEdward Tomasz Napierala error = EINVAL;
14541a13f2e6SEdward Tomasz Napierala } else {
14551a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_init_locked(sb, tpl_id, flags);
14561a13f2e6SEdward Tomasz Napierala }
14571a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK();
14581a13f2e6SEdward Tomasz Napierala
14591a13f2e6SEdward Tomasz Napierala return (error);
14601a13f2e6SEdward Tomasz Napierala }
14611a13f2e6SEdward Tomasz Napierala
14621a13f2e6SEdward Tomasz Napierala static inline int
14631a13f2e6SEdward Tomasz Napierala stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
14641a13f2e6SEdward Tomasz Napierala uint32_t flags __unused)
14651a13f2e6SEdward Tomasz Napierala {
14661a13f2e6SEdward Tomasz Napierala int error;
14671a13f2e6SEdward Tomasz Napierala
14681a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK_ASSERT();
14691a13f2e6SEdward Tomasz Napierala error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW;
14701a13f2e6SEdward Tomasz Napierala KASSERT(!error,
14711a13f2e6SEdward Tomasz Napierala ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz));
14721a13f2e6SEdward Tomasz Napierala
14731a13f2e6SEdward Tomasz Napierala if (!error) {
14741a13f2e6SEdward Tomasz Napierala memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz);
14751a13f2e6SEdward Tomasz Napierala sb->created = sb->lastrst = stats_sbinuptime();
14761a13f2e6SEdward Tomasz Napierala sb->tplhash = tpllist[tpl_id]->mb->tplhash;
14771a13f2e6SEdward Tomasz Napierala }
14781a13f2e6SEdward Tomasz Napierala
14791a13f2e6SEdward Tomasz Napierala return (error);
14801a13f2e6SEdward Tomasz Napierala }
14811a13f2e6SEdward Tomasz Napierala
14821a13f2e6SEdward Tomasz Napierala static int
14831a13f2e6SEdward Tomasz Napierala stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
14841a13f2e6SEdward Tomasz Napierala int newvoistatbytes, int newvoistatdatabytes)
14851a13f2e6SEdward Tomasz Napierala {
14861a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb;
14871a13f2e6SEdward Tomasz Napierala struct voi *tmpvoi;
14881a13f2e6SEdward Tomasz Napierala struct voistat *tmpvoistat, *voistat_array;
14891a13f2e6SEdward Tomasz Napierala int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats;
14901a13f2e6SEdward Tomasz Napierala
14911a13f2e6SEdward Tomasz Napierala KASSERT(newvoibytes % sizeof(struct voi) == 0,
14921a13f2e6SEdward Tomasz Napierala ("Bad newvoibytes %d", newvoibytes));
14931a13f2e6SEdward Tomasz Napierala KASSERT(newvoistatbytes % sizeof(struct voistat) == 0,
14941a13f2e6SEdward Tomasz Napierala ("Bad newvoistatbytes %d", newvoistatbytes));
14951a13f2e6SEdward Tomasz Napierala
14961a13f2e6SEdward Tomasz Napierala error = ((newvoibytes % sizeof(struct voi) == 0) &&
14971a13f2e6SEdward Tomasz Napierala (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL;
14981a13f2e6SEdward Tomasz Napierala sb = *sbpp;
14991a13f2e6SEdward Tomasz Napierala nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes;
15001a13f2e6SEdward Tomasz Napierala
15011a13f2e6SEdward Tomasz Napierala /*
15021a13f2e6SEdward Tomasz Napierala * XXXLAS: Required until we gain support for flags which alter the
15031a13f2e6SEdward Tomasz Napierala * units of size/offset fields in key structs.
15041a13f2e6SEdward Tomasz Napierala */
15051a13f2e6SEdward Tomasz Napierala if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ))
15061a13f2e6SEdward Tomasz Napierala error = EFBIG;
15071a13f2e6SEdward Tomasz Napierala
15081a13f2e6SEdward Tomasz Napierala if (!error && (sb->cursz + nbytes > sb->maxsz)) {
15091a13f2e6SEdward Tomasz Napierala /* Need to expand our blob. */
15101a13f2e6SEdward Tomasz Napierala sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO);
15111a13f2e6SEdward Tomasz Napierala if (sb != NULL) {
15121a13f2e6SEdward Tomasz Napierala sb->maxsz = sb->cursz + nbytes;
15131a13f2e6SEdward Tomasz Napierala *sbpp = sb;
15141a13f2e6SEdward Tomasz Napierala } else
15151a13f2e6SEdward Tomasz Napierala error = ENOMEM;
15161a13f2e6SEdward Tomasz Napierala }
15171a13f2e6SEdward Tomasz Napierala
15181a13f2e6SEdward Tomasz Napierala if (!error) {
15191a13f2e6SEdward Tomasz Napierala /*
15201a13f2e6SEdward Tomasz Napierala * Shuffle memory within the expanded blob working from the end
15211a13f2e6SEdward Tomasz Napierala * backwards, leaving gaps for the new voistat and voistatdata
15221a13f2e6SEdward Tomasz Napierala * structs at the beginning of their respective blob regions,
15231a13f2e6SEdward Tomasz Napierala * and for the new voi structs at the end of their blob region.
15241a13f2e6SEdward Tomasz Napierala */
15251a13f2e6SEdward Tomasz Napierala memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes),
15261a13f2e6SEdward Tomasz Napierala BLOB_OFFSET(sb, sb->statsdata_off),
15271a13f2e6SEdward Tomasz Napierala sb->cursz - sb->statsdata_off);
15281a13f2e6SEdward Tomasz Napierala memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes +
15291a13f2e6SEdward Tomasz Napierala newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off),
15301a13f2e6SEdward Tomasz Napierala sb->statsdata_off - sb->stats_off);
15311a13f2e6SEdward Tomasz Napierala
15321a13f2e6SEdward Tomasz Napierala /* First index of new voi/voistat structs to be initialised. */
15331a13f2e6SEdward Tomasz Napierala idxnewvois = NVOIS(sb);
15341a13f2e6SEdward Tomasz Napierala idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1;
15351a13f2e6SEdward Tomasz Napierala
15361a13f2e6SEdward Tomasz Napierala /* Update housekeeping variables and offsets. */
15371a13f2e6SEdward Tomasz Napierala sb->cursz += nbytes;
15381a13f2e6SEdward Tomasz Napierala sb->stats_off += newvoibytes;
15391a13f2e6SEdward Tomasz Napierala sb->statsdata_off += newvoibytes + newvoistatbytes;
15401a13f2e6SEdward Tomasz Napierala
15411a13f2e6SEdward Tomasz Napierala /* XXXLAS: Zeroing not strictly needed but aids debugging. */
15421a13f2e6SEdward Tomasz Napierala memset(&sb->vois[idxnewvois], '\0', newvoibytes);
15431a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(sb, sb->stats_off), '\0',
15441a13f2e6SEdward Tomasz Napierala newvoistatbytes);
15451a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0',
15461a13f2e6SEdward Tomasz Napierala newvoistatdatabytes);
15471a13f2e6SEdward Tomasz Napierala
15481a13f2e6SEdward Tomasz Napierala /* Initialise new voi array members and update offsets. */
15491a13f2e6SEdward Tomasz Napierala for (i = 0; i < NVOIS(sb); i++) {
15501a13f2e6SEdward Tomasz Napierala tmpvoi = &sb->vois[i];
15511a13f2e6SEdward Tomasz Napierala if (i >= idxnewvois) {
15521a13f2e6SEdward Tomasz Napierala tmpvoi->id = tmpvoi->voistatmaxid = -1;
15531a13f2e6SEdward Tomasz Napierala } else if (tmpvoi->id > -1) {
15541a13f2e6SEdward Tomasz Napierala tmpvoi->stats_off += newvoibytes +
15551a13f2e6SEdward Tomasz Napierala newvoistatbytes;
15561a13f2e6SEdward Tomasz Napierala }
15571a13f2e6SEdward Tomasz Napierala }
15581a13f2e6SEdward Tomasz Napierala
15591a13f2e6SEdward Tomasz Napierala /* Initialise new voistat array members and update offsets. */
15601a13f2e6SEdward Tomasz Napierala nvoistats = (sb->statsdata_off - sb->stats_off) /
15611a13f2e6SEdward Tomasz Napierala sizeof(struct voistat);
15621a13f2e6SEdward Tomasz Napierala voistat_array = BLOB_OFFSET(sb, sb->stats_off);
15631a13f2e6SEdward Tomasz Napierala for (i = 0; i < nvoistats; i++) {
15641a13f2e6SEdward Tomasz Napierala tmpvoistat = &voistat_array[i];
15651a13f2e6SEdward Tomasz Napierala if (i <= idxnewvoistats) {
15661a13f2e6SEdward Tomasz Napierala tmpvoistat->stype = -1;
15671a13f2e6SEdward Tomasz Napierala } else if (tmpvoistat->stype > -1) {
15681a13f2e6SEdward Tomasz Napierala tmpvoistat->data_off += nbytes;
15691a13f2e6SEdward Tomasz Napierala }
15701a13f2e6SEdward Tomasz Napierala }
15711a13f2e6SEdward Tomasz Napierala }
15721a13f2e6SEdward Tomasz Napierala
15731a13f2e6SEdward Tomasz Napierala return (error);
15741a13f2e6SEdward Tomasz Napierala }
15751a13f2e6SEdward Tomasz Napierala
15761a13f2e6SEdward Tomasz Napierala static void
15771a13f2e6SEdward Tomasz Napierala stats_v1_blob_finalise(struct statsblobv1 *sb __unused)
15781a13f2e6SEdward Tomasz Napierala {
15791a13f2e6SEdward Tomasz Napierala
15801a13f2e6SEdward Tomasz Napierala /* XXXLAS: Fill this in. */
15811a13f2e6SEdward Tomasz Napierala }
15821a13f2e6SEdward Tomasz Napierala
15831a13f2e6SEdward Tomasz Napierala static void
15841a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb,
15851a13f2e6SEdward Tomasz Napierala void *usrctx, uint32_t flags)
15861a13f2e6SEdward Tomasz Napierala {
15871a13f2e6SEdward Tomasz Napierala struct voi *v;
15881a13f2e6SEdward Tomasz Napierala struct voistat *vs;
15891a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx ctx;
15901a13f2e6SEdward Tomasz Napierala int i, j, firstvoi;
15911a13f2e6SEdward Tomasz Napierala
15921a13f2e6SEdward Tomasz Napierala ctx.usrctx = usrctx;
1593fbb9879cSJohn Baldwin ctx.flags = SB_IT_FIRST_CB;
15941a13f2e6SEdward Tomasz Napierala firstvoi = 1;
15951a13f2e6SEdward Tomasz Napierala
15961a13f2e6SEdward Tomasz Napierala for (i = 0; i < NVOIS(sb); i++) {
15971a13f2e6SEdward Tomasz Napierala v = &sb->vois[i];
15981a13f2e6SEdward Tomasz Napierala ctx.vslot = i;
15991a13f2e6SEdward Tomasz Napierala ctx.vsslot = -1;
16001a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_FIRST_VOISTAT;
16011a13f2e6SEdward Tomasz Napierala
16021a13f2e6SEdward Tomasz Napierala if (firstvoi)
16031a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_FIRST_VOI;
16041a13f2e6SEdward Tomasz Napierala else if (i == (NVOIS(sb) - 1))
16051a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB;
16061a13f2e6SEdward Tomasz Napierala
16071a13f2e6SEdward Tomasz Napierala if (v->id < 0 && (flags & SB_IT_NULLVOI)) {
16081a13f2e6SEdward Tomasz Napierala if (icb(sb, v, NULL, &ctx))
16091a13f2e6SEdward Tomasz Napierala return;
16101a13f2e6SEdward Tomasz Napierala firstvoi = 0;
16111a13f2e6SEdward Tomasz Napierala ctx.flags &= ~SB_IT_FIRST_CB;
16121a13f2e6SEdward Tomasz Napierala }
16131a13f2e6SEdward Tomasz Napierala
16141a13f2e6SEdward Tomasz Napierala /* If NULL voi, v->voistatmaxid == -1 */
16151a13f2e6SEdward Tomasz Napierala for (j = 0; j <= v->voistatmaxid; j++) {
16161a13f2e6SEdward Tomasz Napierala vs = &((struct voistat *)BLOB_OFFSET(sb,
16171a13f2e6SEdward Tomasz Napierala v->stats_off))[j];
16181a13f2e6SEdward Tomasz Napierala if (vs->stype < 0 &&
16191a13f2e6SEdward Tomasz Napierala !(flags & SB_IT_NULLVOISTAT))
16201a13f2e6SEdward Tomasz Napierala continue;
16211a13f2e6SEdward Tomasz Napierala
16221a13f2e6SEdward Tomasz Napierala if (j == v->voistatmaxid) {
16231a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_LAST_VOISTAT;
16241a13f2e6SEdward Tomasz Napierala if (i == (NVOIS(sb) - 1))
16251a13f2e6SEdward Tomasz Napierala ctx.flags |=
16261a13f2e6SEdward Tomasz Napierala SB_IT_LAST_CB;
16271a13f2e6SEdward Tomasz Napierala } else
16281a13f2e6SEdward Tomasz Napierala ctx.flags &= ~SB_IT_LAST_CB;
16291a13f2e6SEdward Tomasz Napierala
16301a13f2e6SEdward Tomasz Napierala ctx.vsslot = j;
16311a13f2e6SEdward Tomasz Napierala if (icb(sb, v, vs, &ctx))
16321a13f2e6SEdward Tomasz Napierala return;
16331a13f2e6SEdward Tomasz Napierala
16341a13f2e6SEdward Tomasz Napierala ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT |
16351a13f2e6SEdward Tomasz Napierala SB_IT_LAST_VOISTAT);
16361a13f2e6SEdward Tomasz Napierala }
16371a13f2e6SEdward Tomasz Napierala ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI);
16381a13f2e6SEdward Tomasz Napierala }
16391a13f2e6SEdward Tomasz Napierala }
16401a13f2e6SEdward Tomasz Napierala
16411a13f2e6SEdward Tomasz Napierala static inline void
16421a13f2e6SEdward Tomasz Napierala stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused,
16431a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype,
16441a13f2e6SEdward Tomasz Napierala size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
16451a13f2e6SEdward Tomasz Napierala {
16461a13f2e6SEdward Tomasz Napierala const struct ctdth32 *ctd32tree;
16471a13f2e6SEdward Tomasz Napierala const struct ctdth64 *ctd64tree;
16481a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgstctd32 *ctd32;
16491a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgstctd64 *ctd64;
16501a13f2e6SEdward Tomasz Napierala const char *fmtstr;
16511a13f2e6SEdward Tomasz Napierala uint64_t smplcnt, compcnt;
16521a13f2e6SEdward Tomasz Napierala int is32bit, qmaxstrlen;
16531a13f2e6SEdward Tomasz Napierala uint16_t maxctds, curctds;
16541a13f2e6SEdward Tomasz Napierala
16551a13f2e6SEdward Tomasz Napierala switch (tdgst_dtype) {
16561a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
16571a13f2e6SEdward Tomasz Napierala smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt;
16581a13f2e6SEdward Tomasz Napierala compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt;
16591a13f2e6SEdward Tomasz Napierala maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
16601a13f2e6SEdward Tomasz Napierala curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
16611a13f2e6SEdward Tomasz Napierala ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree;
16621a13f2e6SEdward Tomasz Napierala ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) :
16631a13f2e6SEdward Tomasz Napierala ARB_CMIN(ctdth32, ctd32tree));
16641a13f2e6SEdward Tomasz Napierala qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10);
16651a13f2e6SEdward Tomasz Napierala is32bit = 1;
16661a13f2e6SEdward Tomasz Napierala ctd64tree = NULL;
16671a13f2e6SEdward Tomasz Napierala ctd64 = NULL;
16681a13f2e6SEdward Tomasz Napierala break;
16691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
16701a13f2e6SEdward Tomasz Napierala smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt;
16711a13f2e6SEdward Tomasz Napierala compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt;
16721a13f2e6SEdward Tomasz Napierala maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
16731a13f2e6SEdward Tomasz Napierala curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
16741a13f2e6SEdward Tomasz Napierala ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree;
16751a13f2e6SEdward Tomasz Napierala ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) :
16761a13f2e6SEdward Tomasz Napierala ARB_CMIN(ctdth64, ctd64tree));
16771a13f2e6SEdward Tomasz Napierala qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10);
16781a13f2e6SEdward Tomasz Napierala is32bit = 0;
16791a13f2e6SEdward Tomasz Napierala ctd32tree = NULL;
16801a13f2e6SEdward Tomasz Napierala ctd32 = NULL;
16811a13f2e6SEdward Tomasz Napierala break;
16821a13f2e6SEdward Tomasz Napierala default:
16831a13f2e6SEdward Tomasz Napierala return;
16841a13f2e6SEdward Tomasz Napierala }
16851a13f2e6SEdward Tomasz Napierala
16861a13f2e6SEdward Tomasz Napierala switch (fmt) {
16871a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
16881a13f2e6SEdward Tomasz Napierala fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu";
16891a13f2e6SEdward Tomasz Napierala break;
16901a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
16911a13f2e6SEdward Tomasz Napierala default:
16921a13f2e6SEdward Tomasz Napierala fmtstr =
16931a13f2e6SEdward Tomasz Napierala "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu,"
16941a13f2e6SEdward Tomasz Napierala "\"nctds\":%hu,\"ctds\":[";
16951a13f2e6SEdward Tomasz Napierala break;
16961a13f2e6SEdward Tomasz Napierala }
16971a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt,
16981a13f2e6SEdward Tomasz Napierala maxctds, curctds);
16991a13f2e6SEdward Tomasz Napierala
17001a13f2e6SEdward Tomasz Napierala while ((is32bit ? NULL != ctd32 : NULL != ctd64)) {
17011a13f2e6SEdward Tomasz Napierala char qstr[qmaxstrlen];
17021a13f2e6SEdward Tomasz Napierala
17031a13f2e6SEdward Tomasz Napierala switch (fmt) {
17041a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
17051a13f2e6SEdward Tomasz Napierala fmtstr = "\n\t\t\t\t";
17061a13f2e6SEdward Tomasz Napierala break;
17071a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
17081a13f2e6SEdward Tomasz Napierala default:
17091a13f2e6SEdward Tomasz Napierala fmtstr = "{";
17101a13f2e6SEdward Tomasz Napierala break;
17111a13f2e6SEdward Tomasz Napierala }
17121a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr);
17131a13f2e6SEdward Tomasz Napierala
17141a13f2e6SEdward Tomasz Napierala if (objdump) {
17151a13f2e6SEdward Tomasz Napierala switch (fmt) {
17161a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
17171a13f2e6SEdward Tomasz Napierala fmtstr = "ctd[%hu].";
17181a13f2e6SEdward Tomasz Napierala break;
17191a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
17201a13f2e6SEdward Tomasz Napierala default:
17211a13f2e6SEdward Tomasz Napierala fmtstr = "\"ctd\":%hu,";
17221a13f2e6SEdward Tomasz Napierala break;
17231a13f2e6SEdward Tomasz Napierala }
17241a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ?
17251a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd32tree, ctd32) :
17261a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, ctd64));
17271a13f2e6SEdward Tomasz Napierala }
17281a13f2e6SEdward Tomasz Napierala
17291a13f2e6SEdward Tomasz Napierala switch (fmt) {
17301a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
17311a13f2e6SEdward Tomasz Napierala fmtstr = "{mu=";
17321a13f2e6SEdward Tomasz Napierala break;
17331a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
17341a13f2e6SEdward Tomasz Napierala default:
17351a13f2e6SEdward Tomasz Napierala fmtstr = "\"mu\":";
17361a13f2e6SEdward Tomasz Napierala break;
17371a13f2e6SEdward Tomasz Napierala }
17381a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr);
17391a13f2e6SEdward Tomasz Napierala Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr,
17401a13f2e6SEdward Tomasz Napierala sizeof(qstr));
17411a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr);
17421a13f2e6SEdward Tomasz Napierala
17431a13f2e6SEdward Tomasz Napierala switch (fmt) {
17441a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
17451a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
17461a13f2e6SEdward Tomasz Napierala break;
17471a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
17481a13f2e6SEdward Tomasz Napierala default:
17491a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
17501a13f2e6SEdward Tomasz Napierala break;
17511a13f2e6SEdward Tomasz Napierala }
17521a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr,
17531a13f2e6SEdward Tomasz Napierala is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt);
17541a13f2e6SEdward Tomasz Napierala
17551a13f2e6SEdward Tomasz Napierala if (is32bit)
17561a13f2e6SEdward Tomasz Napierala ctd32 = (objdump ? ARB_CNODE(ctd32tree,
17571a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd32tree, ctd32) + 1) :
17581a13f2e6SEdward Tomasz Napierala ARB_CNEXT(ctdth32, ctd32tree, ctd32));
17591a13f2e6SEdward Tomasz Napierala else
17601a13f2e6SEdward Tomasz Napierala ctd64 = (objdump ? ARB_CNODE(ctd64tree,
17611a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, ctd64) + 1) :
17621a13f2e6SEdward Tomasz Napierala ARB_CNEXT(ctdth64, ctd64tree, ctd64));
17631a13f2e6SEdward Tomasz Napierala
17641a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON &&
17651a13f2e6SEdward Tomasz Napierala (is32bit ? NULL != ctd32 : NULL != ctd64))
17661a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ',');
17671a13f2e6SEdward Tomasz Napierala }
17681a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON)
17691a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "]");
17701a13f2e6SEdward Tomasz Napierala }
17711a13f2e6SEdward Tomasz Napierala
17721a13f2e6SEdward Tomasz Napierala static inline void
17731a13f2e6SEdward Tomasz Napierala stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype,
17741a13f2e6SEdward Tomasz Napierala const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype,
17751a13f2e6SEdward Tomasz Napierala size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
17761a13f2e6SEdward Tomasz Napierala {
17771a13f2e6SEdward Tomasz Napierala const struct voistatdata_numeric *bkt_lb, *bkt_ub;
17781a13f2e6SEdward Tomasz Napierala const char *fmtstr;
17791a13f2e6SEdward Tomasz Napierala int is32bit;
17801a13f2e6SEdward Tomasz Napierala uint16_t i, nbkts;
17811a13f2e6SEdward Tomasz Napierala
17821a13f2e6SEdward Tomasz Napierala switch (hist_dtype) {
17831a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
17841a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz);
17851a13f2e6SEdward Tomasz Napierala is32bit = 1;
17861a13f2e6SEdward Tomasz Napierala break;
17871a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
17881a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz);
17891a13f2e6SEdward Tomasz Napierala is32bit = 1;
17901a13f2e6SEdward Tomasz Napierala break;
17911a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
17921a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz);
17931a13f2e6SEdward Tomasz Napierala is32bit = 1;
17941a13f2e6SEdward Tomasz Napierala break;
17951a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
17961a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz);
17971a13f2e6SEdward Tomasz Napierala is32bit = 0;
17981a13f2e6SEdward Tomasz Napierala break;
17991a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
18001a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz);
18011a13f2e6SEdward Tomasz Napierala is32bit = 0;
18021a13f2e6SEdward Tomasz Napierala break;
18031a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
18041a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz);
18051a13f2e6SEdward Tomasz Napierala is32bit = 0;
18061a13f2e6SEdward Tomasz Napierala break;
18071a13f2e6SEdward Tomasz Napierala default:
18081a13f2e6SEdward Tomasz Napierala return;
18091a13f2e6SEdward Tomasz Napierala }
18101a13f2e6SEdward Tomasz Napierala
18111a13f2e6SEdward Tomasz Napierala switch (fmt) {
18121a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
18131a13f2e6SEdward Tomasz Napierala fmtstr = "nbkts=%hu, ";
18141a13f2e6SEdward Tomasz Napierala break;
18151a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
18161a13f2e6SEdward Tomasz Napierala default:
18171a13f2e6SEdward Tomasz Napierala fmtstr = "\"nbkts\":%hu,";
18181a13f2e6SEdward Tomasz Napierala break;
18191a13f2e6SEdward Tomasz Napierala }
18201a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, nbkts);
18211a13f2e6SEdward Tomasz Napierala
18221a13f2e6SEdward Tomasz Napierala switch (fmt) {
18231a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
18241a13f2e6SEdward Tomasz Napierala fmtstr = (is32bit ? "oob=%u" : "oob=%ju");
18251a13f2e6SEdward Tomasz Napierala break;
18261a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
18271a13f2e6SEdward Tomasz Napierala default:
18281a13f2e6SEdward Tomasz Napierala fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" :
18291a13f2e6SEdward Tomasz Napierala "\"oob\":%ju,\"bkts\":[");
18301a13f2e6SEdward Tomasz Napierala break;
18311a13f2e6SEdward Tomasz Napierala }
18321a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist,
18331a13f2e6SEdward Tomasz Napierala hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist,
18341a13f2e6SEdward Tomasz Napierala hist_dtype, oob));
18351a13f2e6SEdward Tomasz Napierala
18361a13f2e6SEdward Tomasz Napierala for (i = 0; i < nbkts; i++) {
18371a13f2e6SEdward Tomasz Napierala switch (hist_dtype) {
18381a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
18391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
18401a13f2e6SEdward Tomasz Napierala bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype,
18411a13f2e6SEdward Tomasz Napierala bkts[i].lb);
18421a13f2e6SEdward Tomasz Napierala if (i < nbkts - 1)
18431a13f2e6SEdward Tomasz Napierala bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist,
18441a13f2e6SEdward Tomasz Napierala hist_dtype, bkts[i + 1].lb);
18451a13f2e6SEdward Tomasz Napierala else
18461a13f2e6SEdward Tomasz Napierala bkt_ub = &numeric_limits[LIM_MAX][voi_dtype];
18471a13f2e6SEdward Tomasz Napierala break;
18481a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
18491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
18501a13f2e6SEdward Tomasz Napierala bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
18511a13f2e6SEdward Tomasz Napierala bkts[i].lb);
18521a13f2e6SEdward Tomasz Napierala bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
18531a13f2e6SEdward Tomasz Napierala bkts[i].ub);
18541a13f2e6SEdward Tomasz Napierala break;
18551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
18561a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
18571a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist,
18581a13f2e6SEdward Tomasz Napierala hist_dtype, bkts[i].val);
18591a13f2e6SEdward Tomasz Napierala break;
18601a13f2e6SEdward Tomasz Napierala default:
18611a13f2e6SEdward Tomasz Napierala break;
18621a13f2e6SEdward Tomasz Napierala }
18631a13f2e6SEdward Tomasz Napierala
18641a13f2e6SEdward Tomasz Napierala switch (fmt) {
18651a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
18661a13f2e6SEdward Tomasz Napierala fmtstr = "\n\t\t\t\t";
18671a13f2e6SEdward Tomasz Napierala break;
18681a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
18691a13f2e6SEdward Tomasz Napierala default:
18701a13f2e6SEdward Tomasz Napierala fmtstr = "{";
18711a13f2e6SEdward Tomasz Napierala break;
18721a13f2e6SEdward Tomasz Napierala }
18731a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr);
18741a13f2e6SEdward Tomasz Napierala
18751a13f2e6SEdward Tomasz Napierala if (objdump) {
18761a13f2e6SEdward Tomasz Napierala switch (fmt) {
18771a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
18781a13f2e6SEdward Tomasz Napierala fmtstr = "bkt[%hu].";
18791a13f2e6SEdward Tomasz Napierala break;
18801a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
18811a13f2e6SEdward Tomasz Napierala default:
18821a13f2e6SEdward Tomasz Napierala fmtstr = "\"bkt\":%hu,";
18831a13f2e6SEdward Tomasz Napierala break;
18841a13f2e6SEdward Tomasz Napierala }
18851a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, i);
18861a13f2e6SEdward Tomasz Napierala }
18871a13f2e6SEdward Tomasz Napierala
18881a13f2e6SEdward Tomasz Napierala switch (fmt) {
18891a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
18901a13f2e6SEdward Tomasz Napierala fmtstr = "{lb=";
18911a13f2e6SEdward Tomasz Napierala break;
18921a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
18931a13f2e6SEdward Tomasz Napierala default:
18941a13f2e6SEdward Tomasz Napierala fmtstr = "\"lb\":";
18951a13f2e6SEdward Tomasz Napierala break;
18961a13f2e6SEdward Tomasz Napierala }
18971a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr);
18981a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr((const struct voistatdata *)bkt_lb,
18991a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
19001a13f2e6SEdward Tomasz Napierala fmt, buf, objdump);
19011a13f2e6SEdward Tomasz Napierala
19021a13f2e6SEdward Tomasz Napierala switch (fmt) {
19031a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
19041a13f2e6SEdward Tomasz Napierala fmtstr = ",ub=";
19051a13f2e6SEdward Tomasz Napierala break;
19061a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
19071a13f2e6SEdward Tomasz Napierala default:
19081a13f2e6SEdward Tomasz Napierala fmtstr = ",\"ub\":";
19091a13f2e6SEdward Tomasz Napierala break;
19101a13f2e6SEdward Tomasz Napierala }
19111a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr);
19121a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr((const struct voistatdata *)bkt_ub,
19131a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
19141a13f2e6SEdward Tomasz Napierala fmt, buf, objdump);
19151a13f2e6SEdward Tomasz Napierala
19161a13f2e6SEdward Tomasz Napierala switch (fmt) {
19171a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
19181a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
19191a13f2e6SEdward Tomasz Napierala break;
19201a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
19211a13f2e6SEdward Tomasz Napierala default:
19221a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
19231a13f2e6SEdward Tomasz Napierala break;
19241a13f2e6SEdward Tomasz Napierala }
19251a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ?
19261a13f2e6SEdward Tomasz Napierala VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) :
19271a13f2e6SEdward Tomasz Napierala (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype,
19281a13f2e6SEdward Tomasz Napierala bkts[i].cnt));
19291a13f2e6SEdward Tomasz Napierala
19301a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON && i < nbkts - 1)
19311a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ',');
19321a13f2e6SEdward Tomasz Napierala }
19331a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON)
19341a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "]");
19351a13f2e6SEdward Tomasz Napierala }
19361a13f2e6SEdward Tomasz Napierala
19371a13f2e6SEdward Tomasz Napierala int
19381a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype,
19391a13f2e6SEdward Tomasz Napierala enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt,
19401a13f2e6SEdward Tomasz Napierala struct sbuf *buf, int objdump)
19411a13f2e6SEdward Tomasz Napierala {
19421a13f2e6SEdward Tomasz Napierala const char *fmtstr;
19431a13f2e6SEdward Tomasz Napierala
19441a13f2e6SEdward Tomasz Napierala if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES ||
19451a13f2e6SEdward Tomasz Napierala vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS)
19461a13f2e6SEdward Tomasz Napierala return (EINVAL);
19471a13f2e6SEdward Tomasz Napierala
19481a13f2e6SEdward Tomasz Napierala switch (vsd_dtype) {
19491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_VOISTATE:
19501a13f2e6SEdward Tomasz Napierala switch (fmt) {
19511a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
19521a13f2e6SEdward Tomasz Napierala fmtstr = "prev=";
19531a13f2e6SEdward Tomasz Napierala break;
19541a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
19551a13f2e6SEdward Tomasz Napierala default:
19561a13f2e6SEdward Tomasz Napierala fmtstr = "\"prev\":";
19571a13f2e6SEdward Tomasz Napierala break;
19581a13f2e6SEdward Tomasz Napierala }
19591a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr);
19601a13f2e6SEdward Tomasz Napierala /*
19611a13f2e6SEdward Tomasz Napierala * Render prev by passing it as *vsd and voi_dtype as vsd_dtype.
19621a13f2e6SEdward Tomasz Napierala */
19631a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(
19641a13f2e6SEdward Tomasz Napierala (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev,
19651a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump);
19661a13f2e6SEdward Tomasz Napierala break;
19671a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
19681a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%d", vsd->int32.s32);
19691a13f2e6SEdward Tomasz Napierala break;
19701a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
19711a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%u", vsd->int32.u32);
19721a13f2e6SEdward Tomasz Napierala break;
19731a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
19741a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64);
19751a13f2e6SEdward Tomasz Napierala break;
19761a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
19771a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64);
19781a13f2e6SEdward Tomasz Napierala break;
19791a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
19801a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%ld", vsd->intlong.slong);
19811a13f2e6SEdward Tomasz Napierala break;
19821a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
19831a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%lu", vsd->intlong.ulong);
19841a13f2e6SEdward Tomasz Napierala break;
19851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
19861a13f2e6SEdward Tomasz Napierala {
19871a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)];
19881a13f2e6SEdward Tomasz Napierala Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr));
19891a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr);
19901a13f2e6SEdward Tomasz Napierala }
19911a13f2e6SEdward Tomasz Napierala break;
19921a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
19931a13f2e6SEdward Tomasz Napierala {
19941a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)];
19951a13f2e6SEdward Tomasz Napierala Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr));
19961a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr);
19971a13f2e6SEdward Tomasz Napierala }
19981a13f2e6SEdward Tomasz Napierala break;
19991a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
20001a13f2e6SEdward Tomasz Napierala {
20011a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)];
20021a13f2e6SEdward Tomasz Napierala Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr));
20031a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr);
20041a13f2e6SEdward Tomasz Napierala }
20051a13f2e6SEdward Tomasz Napierala break;
20061a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
20071a13f2e6SEdward Tomasz Napierala {
20081a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)];
20091a13f2e6SEdward Tomasz Napierala Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr));
20101a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr);
20111a13f2e6SEdward Tomasz Napierala }
20121a13f2e6SEdward Tomasz Napierala break;
20131a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
20141a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
20151a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
20161a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
20171a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
20181a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
20191a13f2e6SEdward Tomasz Napierala stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd),
20201a13f2e6SEdward Tomasz Napierala vsd_dtype, vsd_sz, fmt, buf, objdump);
20211a13f2e6SEdward Tomasz Napierala break;
20221a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
20231a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
20241a13f2e6SEdward Tomasz Napierala stats_voistatdata_tdgst_tostr(voi_dtype,
20251a13f2e6SEdward Tomasz Napierala CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf,
20261a13f2e6SEdward Tomasz Napierala objdump);
20271a13f2e6SEdward Tomasz Napierala break;
20281a13f2e6SEdward Tomasz Napierala default:
20291a13f2e6SEdward Tomasz Napierala break;
20301a13f2e6SEdward Tomasz Napierala }
20311a13f2e6SEdward Tomasz Napierala
20321a13f2e6SEdward Tomasz Napierala return (sbuf_error(buf));
20331a13f2e6SEdward Tomasz Napierala }
20341a13f2e6SEdward Tomasz Napierala
20351a13f2e6SEdward Tomasz Napierala static void
20361a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v,
20371a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx)
20381a13f2e6SEdward Tomasz Napierala {
20391a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx;
20401a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb;
20411a13f2e6SEdward Tomasz Napierala struct sbuf *buf;
20421a13f2e6SEdward Tomasz Napierala void *vsd;
20431a13f2e6SEdward Tomasz Napierala uint8_t dump;
20441a13f2e6SEdward Tomasz Napierala
20451a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx;
20461a13f2e6SEdward Tomasz Napierala buf = sctx->buf;
20471a13f2e6SEdward Tomasz Napierala tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
20481a13f2e6SEdward Tomasz Napierala dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
20491a13f2e6SEdward Tomasz Napierala
20501a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_CB) {
20511a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "struct statsblobv1@%p", sb);
20521a13f2e6SEdward Tomasz Napierala if (dump) {
20531a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, "
20541a13f2e6SEdward Tomasz Napierala "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, "
20551a13f2e6SEdward Tomasz Napierala "stats_off=%hu, statsdata_off=%hu",
20561a13f2e6SEdward Tomasz Napierala sb->abi, sb->endian, sb->maxsz, sb->cursz,
20571a13f2e6SEdward Tomasz Napierala sb->created, sb->lastrst, sb->flags, sb->stats_off,
20581a13f2e6SEdward Tomasz Napierala sb->statsdata_off);
20591a13f2e6SEdward Tomasz Napierala }
20601a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", tplhash=%u", sb->tplhash);
20611a13f2e6SEdward Tomasz Napierala }
20621a13f2e6SEdward Tomasz Napierala
20631a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_VOISTAT) {
20641a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id);
20651a13f2e6SEdward Tomasz Napierala if (v->id < 0)
20661a13f2e6SEdward Tomasz Napierala return;
20671a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" :
20681a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name);
20691a13f2e6SEdward Tomasz Napierala if (dump)
20701a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, "
20711a13f2e6SEdward Tomasz Napierala "voistatmaxid=%hhd, stats_off=%hu", v->flags,
20721a13f2e6SEdward Tomasz Napierala vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off);
20731a13f2e6SEdward Tomasz Napierala }
20741a13f2e6SEdward Tomasz Napierala
20751a13f2e6SEdward Tomasz Napierala if (!dump && vs->stype <= 0)
20761a13f2e6SEdward Tomasz Napierala return;
20771a13f2e6SEdward Tomasz Napierala
20781a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot);
20791a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) {
20801a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%hhd", vs->stype);
20811a13f2e6SEdward Tomasz Napierala return;
20821a13f2e6SEdward Tomasz Napierala } else
20831a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype],
20841a13f2e6SEdward Tomasz Napierala vs->errs);
20851a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off);
20861a13f2e6SEdward Tomasz Napierala if (dump)
20871a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, "
20881a13f2e6SEdward Tomasz Napierala "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype],
20891a13f2e6SEdward Tomasz Napierala vs->dsz, vs->data_off);
20901a13f2e6SEdward Tomasz Napierala
20910a713948SAlexander Motin sbuf_cat(buf, "\n\t\t\tvoistatdata: ");
20921a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
20931a13f2e6SEdward Tomasz Napierala sctx->fmt, buf, dump);
20941a13f2e6SEdward Tomasz Napierala }
20951a13f2e6SEdward Tomasz Napierala
20961a13f2e6SEdward Tomasz Napierala static void
20971a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
20981a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx *ctx)
20991a13f2e6SEdward Tomasz Napierala {
21001a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx;
21011a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb;
21021a13f2e6SEdward Tomasz Napierala struct sbuf *buf;
21031a13f2e6SEdward Tomasz Napierala const char *fmtstr;
21041a13f2e6SEdward Tomasz Napierala void *vsd;
21051a13f2e6SEdward Tomasz Napierala uint8_t dump;
21061a13f2e6SEdward Tomasz Napierala
21071a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx;
21081a13f2e6SEdward Tomasz Napierala buf = sctx->buf;
21091a13f2e6SEdward Tomasz Napierala tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
21101a13f2e6SEdward Tomasz Napierala dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
21111a13f2e6SEdward Tomasz Napierala
21121a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_CB) {
21131a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, '{');
21141a13f2e6SEdward Tomasz Napierala if (dump) {
21151a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu,"
21161a13f2e6SEdward Tomasz Napierala "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd,"
21171a13f2e6SEdward Tomasz Napierala "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu,"
21181a13f2e6SEdward Tomasz Napierala "\"statsdata_off\":%hu,", sb->abi,
21191a13f2e6SEdward Tomasz Napierala sb->endian, sb->maxsz, sb->cursz, sb->created,
21201a13f2e6SEdward Tomasz Napierala sb->lastrst, sb->flags, sb->stats_off,
21211a13f2e6SEdward Tomasz Napierala sb->statsdata_off);
21221a13f2e6SEdward Tomasz Napierala }
21231a13f2e6SEdward Tomasz Napierala
21241a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL)
21251a13f2e6SEdward Tomasz Napierala fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{";
21261a13f2e6SEdward Tomasz Napierala else
21271a13f2e6SEdward Tomasz Napierala fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{";
21281a13f2e6SEdward Tomasz Napierala
21291a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null",
21301a13f2e6SEdward Tomasz Napierala sb->tplhash);
21311a13f2e6SEdward Tomasz Napierala }
21321a13f2e6SEdward Tomasz Napierala
21331a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_VOISTAT) {
21341a13f2e6SEdward Tomasz Napierala if (dump) {
21351a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot,
21361a13f2e6SEdward Tomasz Napierala v->id);
21371a13f2e6SEdward Tomasz Napierala if (v->id < 0) {
21380a713948SAlexander Motin sbuf_cat(buf, "},");
21391a13f2e6SEdward Tomasz Napierala return;
21401a13f2e6SEdward Tomasz Napierala }
21411a13f2e6SEdward Tomasz Napierala
21421a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL)
21431a13f2e6SEdward Tomasz Napierala fmtstr = ",\"name\":%s,\"flags\":%hu,"
21441a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
21451a13f2e6SEdward Tomasz Napierala "\"stats_off\":%hu,";
21461a13f2e6SEdward Tomasz Napierala else
21471a13f2e6SEdward Tomasz Napierala fmtstr = ",\"name\":\"%s\",\"flags\":%hu,"
21481a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
21491a13f2e6SEdward Tomasz Napierala "\"stats_off\":%hu,";
21501a13f2e6SEdward Tomasz Napierala
21511a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, tpl_mb ?
21521a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name : "null", v->flags,
21531a13f2e6SEdward Tomasz Napierala vsd_dtype2name[v->dtype], v->voistatmaxid,
21541a13f2e6SEdward Tomasz Napierala v->stats_off);
21551a13f2e6SEdward Tomasz Napierala } else {
21561a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) {
21571a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hd]\":{", v->id);
21581a13f2e6SEdward Tomasz Napierala } else {
21591a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"%s\":{",
21601a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name);
21611a13f2e6SEdward Tomasz Napierala }
21621a13f2e6SEdward Tomasz Napierala }
21631a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "\"stats\":{");
21641a13f2e6SEdward Tomasz Napierala }
21651a13f2e6SEdward Tomasz Napierala
21661a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off);
21671a13f2e6SEdward Tomasz Napierala if (dump) {
21681a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot);
21691a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) {
21700a713948SAlexander Motin sbuf_cat(buf, "{\"stype\":-1},");
21711a13f2e6SEdward Tomasz Napierala return;
21721a13f2e6SEdward Tomasz Napierala }
21731a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu,"
21741a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{",
21751a13f2e6SEdward Tomasz Napierala vs_stype2name[vs->stype], vs->errs, vs->flags,
21761a13f2e6SEdward Tomasz Napierala vsd_dtype2name[vs->dtype], vs->data_off);
21771a13f2e6SEdward Tomasz Napierala } else if (vs->stype > 0) {
21781a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL)
21791a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hhd]\":", vs->stype);
21801a13f2e6SEdward Tomasz Napierala else
21811a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]);
21821a13f2e6SEdward Tomasz Napierala } else
21831a13f2e6SEdward Tomasz Napierala return;
21841a13f2e6SEdward Tomasz Napierala
21851a13f2e6SEdward Tomasz Napierala if ((vs->flags & VS_VSDVALID) || dump) {
21861a13f2e6SEdward Tomasz Napierala if (!dump)
21871a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"errs\":%hu,", vs->errs);
21881a13f2e6SEdward Tomasz Napierala /* Simple non-compound VSD types need a key. */
21891a13f2e6SEdward Tomasz Napierala if (!vsd_compoundtype[vs->dtype])
21901a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "\"val\":");
21911a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
21921a13f2e6SEdward Tomasz Napierala sctx->fmt, buf, dump);
21931a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, dump ? "}}" : "}");
21941a13f2e6SEdward Tomasz Napierala } else
21951a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, dump ? "null}" : "null");
21961a13f2e6SEdward Tomasz Napierala
21971a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_LAST_VOISTAT)
21981a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "}}");
21991a13f2e6SEdward Tomasz Napierala
22001a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_LAST_CB)
22011a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "}}");
22021a13f2e6SEdward Tomasz Napierala else
22031a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ',');
22041a13f2e6SEdward Tomasz Napierala }
22051a13f2e6SEdward Tomasz Napierala
22061a13f2e6SEdward Tomasz Napierala static int
22071a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
22081a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx *ctx)
22091a13f2e6SEdward Tomasz Napierala {
22101a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx;
22111a13f2e6SEdward Tomasz Napierala
22121a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx;
22131a13f2e6SEdward Tomasz Napierala
22141a13f2e6SEdward Tomasz Napierala switch (sctx->fmt) {
22151a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM:
22161a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_freeform(sb, v, vs, ctx);
22171a13f2e6SEdward Tomasz Napierala break;
22181a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON:
22191a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_json(sb, v, vs, ctx);
22201a13f2e6SEdward Tomasz Napierala break;
22211a13f2e6SEdward Tomasz Napierala default:
22221a13f2e6SEdward Tomasz Napierala break;
22231a13f2e6SEdward Tomasz Napierala }
22241a13f2e6SEdward Tomasz Napierala
22251a13f2e6SEdward Tomasz Napierala return (sbuf_error(sctx->buf));
22261a13f2e6SEdward Tomasz Napierala }
22271a13f2e6SEdward Tomasz Napierala
22281a13f2e6SEdward Tomasz Napierala int
22291a13f2e6SEdward Tomasz Napierala stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf,
22301a13f2e6SEdward Tomasz Napierala enum sb_str_fmt fmt, uint32_t flags)
22311a13f2e6SEdward Tomasz Napierala {
22321a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx sctx;
22331a13f2e6SEdward Tomasz Napierala uint32_t iflags;
22341a13f2e6SEdward Tomasz Napierala
22351a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL ||
22361a13f2e6SEdward Tomasz Napierala fmt >= SB_STRFMT_NUM_FMTS)
22371a13f2e6SEdward Tomasz Napierala return (EINVAL);
22381a13f2e6SEdward Tomasz Napierala
22391a13f2e6SEdward Tomasz Napierala sctx.buf = buf;
22401a13f2e6SEdward Tomasz Napierala sctx.fmt = fmt;
22411a13f2e6SEdward Tomasz Napierala sctx.flags = flags;
22421a13f2e6SEdward Tomasz Napierala
22431a13f2e6SEdward Tomasz Napierala if (flags & SB_TOSTR_META) {
22441a13f2e6SEdward Tomasz Napierala if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash),
22451a13f2e6SEdward Tomasz Napierala &sctx.tpl))
22461a13f2e6SEdward Tomasz Napierala return (EINVAL);
22471a13f2e6SEdward Tomasz Napierala } else
22481a13f2e6SEdward Tomasz Napierala sctx.tpl = NULL;
22491a13f2e6SEdward Tomasz Napierala
22501a13f2e6SEdward Tomasz Napierala iflags = 0;
22511a13f2e6SEdward Tomasz Napierala if (flags & SB_TOSTR_OBJDUMP)
22521a13f2e6SEdward Tomasz Napierala iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT);
22531a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags);
22541a13f2e6SEdward Tomasz Napierala
22551a13f2e6SEdward Tomasz Napierala return (sbuf_error(buf));
22561a13f2e6SEdward Tomasz Napierala }
22571a13f2e6SEdward Tomasz Napierala
22581a13f2e6SEdward Tomasz Napierala static int
22591a13f2e6SEdward Tomasz Napierala stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v,
22601a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx)
22611a13f2e6SEdward Tomasz Napierala {
22621a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx *vctx;
22631a13f2e6SEdward Tomasz Napierala struct sb_visit sbv;
22641a13f2e6SEdward Tomasz Napierala
22651a13f2e6SEdward Tomasz Napierala vctx = ctx->usrctx;
22661a13f2e6SEdward Tomasz Napierala
22671a13f2e6SEdward Tomasz Napierala sbv.tplhash = sb->tplhash;
22681a13f2e6SEdward Tomasz Napierala sbv.voi_id = v->id;
22691a13f2e6SEdward Tomasz Napierala sbv.voi_dtype = v->dtype;
22701a13f2e6SEdward Tomasz Napierala sbv.vs_stype = vs->stype;
22711a13f2e6SEdward Tomasz Napierala sbv.vs_dtype = vs->dtype;
22721a13f2e6SEdward Tomasz Napierala sbv.vs_dsz = vs->dsz;
22731a13f2e6SEdward Tomasz Napierala sbv.vs_data = BLOB_OFFSET(sb, vs->data_off);
22741a13f2e6SEdward Tomasz Napierala sbv.vs_errs = vs->errs;
22751a13f2e6SEdward Tomasz Napierala sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB |
22761a13f2e6SEdward Tomasz Napierala SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
22771a13f2e6SEdward Tomasz Napierala SB_IT_LAST_VOISTAT);
22781a13f2e6SEdward Tomasz Napierala
22791a13f2e6SEdward Tomasz Napierala return (vctx->cb(&sbv, vctx->usrctx));
22801a13f2e6SEdward Tomasz Napierala }
22811a13f2e6SEdward Tomasz Napierala
22821a13f2e6SEdward Tomasz Napierala int
22831a13f2e6SEdward Tomasz Napierala stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func,
22841a13f2e6SEdward Tomasz Napierala void *usrctx)
22851a13f2e6SEdward Tomasz Napierala {
22861a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx vctx;
22871a13f2e6SEdward Tomasz Napierala
22881a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL)
22891a13f2e6SEdward Tomasz Napierala return (EINVAL);
22901a13f2e6SEdward Tomasz Napierala
22911a13f2e6SEdward Tomasz Napierala vctx.cb = func;
22921a13f2e6SEdward Tomasz Napierala vctx.usrctx = usrctx;
22931a13f2e6SEdward Tomasz Napierala
22941a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0);
22951a13f2e6SEdward Tomasz Napierala
22961a13f2e6SEdward Tomasz Napierala return (0);
22971a13f2e6SEdward Tomasz Napierala }
22981a13f2e6SEdward Tomasz Napierala
22991a13f2e6SEdward Tomasz Napierala static int
23001a13f2e6SEdward Tomasz Napierala stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused,
23011a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx __unused)
23021a13f2e6SEdward Tomasz Napierala {
23031a13f2e6SEdward Tomasz Napierala void *vsd;
23041a13f2e6SEdward Tomasz Napierala
23051a13f2e6SEdward Tomasz Napierala if (vs->stype == VS_STYPE_VOISTATE)
23061a13f2e6SEdward Tomasz Napierala return (0);
23071a13f2e6SEdward Tomasz Napierala
23081a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off);
23091a13f2e6SEdward Tomasz Napierala
23101a13f2e6SEdward Tomasz Napierala /* Perform the stat type's default reset action. */
23111a13f2e6SEdward Tomasz Napierala switch (vs->stype) {
23121a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM:
23131a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
23141a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
23151a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32, 0);
23161a13f2e6SEdward Tomasz Napierala break;
23171a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
23181a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32, 0);
23191a13f2e6SEdward Tomasz Napierala break;
23201a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
23211a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64, 0);
23221a13f2e6SEdward Tomasz Napierala break;
23231a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
23241a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64, 0);
23251a13f2e6SEdward Tomasz Napierala break;
23261a13f2e6SEdward Tomasz Napierala default:
23271a13f2e6SEdward Tomasz Napierala bzero(vsd, vs->dsz);
23281a13f2e6SEdward Tomasz Napierala break;
23291a13f2e6SEdward Tomasz Napierala }
23301a13f2e6SEdward Tomasz Napierala break;
23311a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX:
23321a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
23331a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
23341a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32,
23351a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q32, vsd)->sq32));
23361a13f2e6SEdward Tomasz Napierala break;
23371a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
23381a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32,
23391a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q32, vsd)->uq32));
23401a13f2e6SEdward Tomasz Napierala break;
23411a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
23421a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64,
23431a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q64, vsd)->sq64));
23441a13f2e6SEdward Tomasz Napierala break;
23451a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
23461a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64,
23471a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q64, vsd)->uq64));
23481a13f2e6SEdward Tomasz Napierala break;
23491a13f2e6SEdward Tomasz Napierala default:
23501a13f2e6SEdward Tomasz Napierala memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype],
23511a13f2e6SEdward Tomasz Napierala vs->dsz);
23521a13f2e6SEdward Tomasz Napierala break;
23531a13f2e6SEdward Tomasz Napierala }
23541a13f2e6SEdward Tomasz Napierala break;
23551a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN:
23561a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
23571a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
23581a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32,
23591a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q32, vsd)->sq32));
23601a13f2e6SEdward Tomasz Napierala break;
23611a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
23621a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32,
23631a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q32, vsd)->uq32));
23641a13f2e6SEdward Tomasz Napierala break;
23651a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
23661a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64,
23671a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q64, vsd)->sq64));
23681a13f2e6SEdward Tomasz Napierala break;
23691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
23701a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64,
23711a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q64, vsd)->uq64));
23721a13f2e6SEdward Tomasz Napierala break;
23731a13f2e6SEdward Tomasz Napierala default:
23741a13f2e6SEdward Tomasz Napierala memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype],
23751a13f2e6SEdward Tomasz Napierala vs->dsz);
23761a13f2e6SEdward Tomasz Napierala break;
23771a13f2e6SEdward Tomasz Napierala }
23781a13f2e6SEdward Tomasz Napierala break;
23791a13f2e6SEdward Tomasz Napierala case VS_STYPE_HIST:
23801a13f2e6SEdward Tomasz Napierala {
23811a13f2e6SEdward Tomasz Napierala /* Reset bucket counts. */
23821a13f2e6SEdward Tomasz Napierala struct voistatdata_hist *hist;
23831a13f2e6SEdward Tomasz Napierala int i, is32bit;
23841a13f2e6SEdward Tomasz Napierala uint16_t nbkts;
23851a13f2e6SEdward Tomasz Napierala
23861a13f2e6SEdward Tomasz Napierala hist = VSD(hist, vsd);
23871a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
23881a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
23891a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
23901a13f2e6SEdward Tomasz Napierala is32bit = 1;
23911a13f2e6SEdward Tomasz Napierala break;
23921a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
23931a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
23941a13f2e6SEdward Tomasz Napierala is32bit = 1;
23951a13f2e6SEdward Tomasz Napierala break;
23961a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
23971a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
23981a13f2e6SEdward Tomasz Napierala is32bit = 1;
23991a13f2e6SEdward Tomasz Napierala break;
24001a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
24011a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
24021a13f2e6SEdward Tomasz Napierala is32bit = 0;
24031a13f2e6SEdward Tomasz Napierala break;
24041a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
24051a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
24061a13f2e6SEdward Tomasz Napierala is32bit = 0;
24071a13f2e6SEdward Tomasz Napierala break;
24081a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
24091a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
24101a13f2e6SEdward Tomasz Napierala is32bit = 0;
24111a13f2e6SEdward Tomasz Napierala break;
24121a13f2e6SEdward Tomasz Napierala default:
24131a13f2e6SEdward Tomasz Napierala return (0);
24141a13f2e6SEdward Tomasz Napierala }
24151a13f2e6SEdward Tomasz Napierala
24161a13f2e6SEdward Tomasz Napierala bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob),
24171a13f2e6SEdward Tomasz Napierala is32bit ? sizeof(uint32_t) : sizeof(uint64_t));
24181a13f2e6SEdward Tomasz Napierala for (i = nbkts - 1; i >= 0; i--) {
24191a13f2e6SEdward Tomasz Napierala bzero(VSD_HIST_FIELDPTR(hist, vs->dtype,
24201a13f2e6SEdward Tomasz Napierala bkts[i].cnt), is32bit ? sizeof(uint32_t) :
24211a13f2e6SEdward Tomasz Napierala sizeof(uint64_t));
24221a13f2e6SEdward Tomasz Napierala }
24231a13f2e6SEdward Tomasz Napierala break;
24241a13f2e6SEdward Tomasz Napierala }
24251a13f2e6SEdward Tomasz Napierala case VS_STYPE_TDGST:
24261a13f2e6SEdward Tomasz Napierala {
24271a13f2e6SEdward Tomasz Napierala /* Reset sample count centroids array/tree. */
24281a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst;
24291a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree;
24301a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree;
24311a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32;
24321a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64;
24331a13f2e6SEdward Tomasz Napierala
24341a13f2e6SEdward Tomasz Napierala tdgst = VSD(tdgst, vsd);
24351a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
24361a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
24371a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt = 0;
24381a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->compcnt = 0;
24391a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
24401a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd32, ctdlnk, ctd32tree,
24411a13f2e6SEdward Tomasz Napierala ARB_MAXNODES(ctd32tree)) {
24421a13f2e6SEdward Tomasz Napierala ctd32->cnt = 0;
24431a13f2e6SEdward Tomasz Napierala Q_SIFVAL(ctd32->mu, 0);
24441a13f2e6SEdward Tomasz Napierala }
24451a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
24461a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
24471a13f2e6SEdward Tomasz Napierala #endif
24481a13f2e6SEdward Tomasz Napierala break;
24491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
24501a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt = 0;
24511a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->compcnt = 0;
24521a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
24531a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd64, ctdlnk, ctd64tree,
24541a13f2e6SEdward Tomasz Napierala ARB_MAXNODES(ctd64tree)) {
24551a13f2e6SEdward Tomasz Napierala ctd64->cnt = 0;
24561a13f2e6SEdward Tomasz Napierala Q_SIFVAL(ctd64->mu, 0);
24571a13f2e6SEdward Tomasz Napierala }
24581a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
24591a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
24601a13f2e6SEdward Tomasz Napierala #endif
24611a13f2e6SEdward Tomasz Napierala break;
24621a13f2e6SEdward Tomasz Napierala default:
24631a13f2e6SEdward Tomasz Napierala return (0);
24641a13f2e6SEdward Tomasz Napierala }
24651a13f2e6SEdward Tomasz Napierala break;
24661a13f2e6SEdward Tomasz Napierala }
24671a13f2e6SEdward Tomasz Napierala default:
24681a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
24691a13f2e6SEdward Tomasz Napierala break;
24701a13f2e6SEdward Tomasz Napierala }
24711a13f2e6SEdward Tomasz Napierala
24721a13f2e6SEdward Tomasz Napierala vs->errs = 0;
24731a13f2e6SEdward Tomasz Napierala vs->flags &= ~VS_VSDVALID;
24741a13f2e6SEdward Tomasz Napierala
24751a13f2e6SEdward Tomasz Napierala return (0);
24761a13f2e6SEdward Tomasz Napierala }
24771a13f2e6SEdward Tomasz Napierala
24781a13f2e6SEdward Tomasz Napierala int
24791a13f2e6SEdward Tomasz Napierala stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz,
24801a13f2e6SEdward Tomasz Napierala struct statsblobv1 *src, uint32_t flags)
24811a13f2e6SEdward Tomasz Napierala {
24821a13f2e6SEdward Tomasz Napierala int error;
24831a13f2e6SEdward Tomasz Napierala
24841a13f2e6SEdward Tomasz Napierala if (src != NULL && src->abi == STATS_ABI_V1) {
24851a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_clone(dst, dstmaxsz, src, flags);
24861a13f2e6SEdward Tomasz Napierala if (!error) {
24871a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_RSTSRC) {
24881a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(src,
24891a13f2e6SEdward Tomasz Napierala stats_v1_icb_reset_voistat, NULL, 0);
24901a13f2e6SEdward Tomasz Napierala src->lastrst = stats_sbinuptime();
24911a13f2e6SEdward Tomasz Napierala }
24921a13f2e6SEdward Tomasz Napierala stats_v1_blob_finalise(*dst);
24931a13f2e6SEdward Tomasz Napierala }
24941a13f2e6SEdward Tomasz Napierala } else
24951a13f2e6SEdward Tomasz Napierala error = EINVAL;
24961a13f2e6SEdward Tomasz Napierala
24971a13f2e6SEdward Tomasz Napierala return (error);
24981a13f2e6SEdward Tomasz Napierala }
24991a13f2e6SEdward Tomasz Napierala
25001a13f2e6SEdward Tomasz Napierala static inline int
25011a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused,
25021a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd)
25031a13f2e6SEdward Tomasz Napierala {
25041a13f2e6SEdward Tomasz Napierala int error;
25051a13f2e6SEdward Tomasz Napierala
25061a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES,
25071a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype));
25081a13f2e6SEdward Tomasz Napierala
25091a13f2e6SEdward Tomasz Napierala error = 0;
25101a13f2e6SEdward Tomasz Napierala
25111a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
25121a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
25131a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->s32 < voival->int32.s32) {
25141a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 = voival->int32.s32;
25151a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25161a13f2e6SEdward Tomasz Napierala }
25171a13f2e6SEdward Tomasz Napierala break;
25181a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
25191a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->u32 < voival->int32.u32) {
25201a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 = voival->int32.u32;
25211a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25221a13f2e6SEdward Tomasz Napierala }
25231a13f2e6SEdward Tomasz Napierala break;
25241a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
25251a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->s64 < voival->int64.s64) {
25261a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 = voival->int64.s64;
25271a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25281a13f2e6SEdward Tomasz Napierala }
25291a13f2e6SEdward Tomasz Napierala break;
25301a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
25311a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->u64 < voival->int64.u64) {
25321a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 = voival->int64.u64;
25331a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25341a13f2e6SEdward Tomasz Napierala }
25351a13f2e6SEdward Tomasz Napierala break;
25361a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
25371a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->slong < voival->intlong.slong) {
25381a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong = voival->intlong.slong;
25391a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25401a13f2e6SEdward Tomasz Napierala }
25411a13f2e6SEdward Tomasz Napierala break;
25421a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
25431a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) {
25441a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong = voival->intlong.ulong;
25451a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25461a13f2e6SEdward Tomasz Napierala }
25471a13f2e6SEdward Tomasz Napierala break;
25481a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
25491a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
25501a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
25511a13f2e6SEdward Tomasz Napierala voival->q32.sq32)))) {
25521a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25531a13f2e6SEdward Tomasz Napierala }
25541a13f2e6SEdward Tomasz Napierala break;
25551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
25561a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
25571a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
25581a13f2e6SEdward Tomasz Napierala voival->q32.uq32)))) {
25591a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25601a13f2e6SEdward Tomasz Napierala }
25611a13f2e6SEdward Tomasz Napierala break;
25621a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
25631a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
25641a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
25651a13f2e6SEdward Tomasz Napierala voival->q64.sq64)))) {
25661a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25671a13f2e6SEdward Tomasz Napierala }
25681a13f2e6SEdward Tomasz Napierala break;
25691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
25701a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
25711a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
25721a13f2e6SEdward Tomasz Napierala voival->q64.uq64)))) {
25731a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
25741a13f2e6SEdward Tomasz Napierala }
25751a13f2e6SEdward Tomasz Napierala break;
25761a13f2e6SEdward Tomasz Napierala default:
25771a13f2e6SEdward Tomasz Napierala error = EINVAL;
25781a13f2e6SEdward Tomasz Napierala break;
25791a13f2e6SEdward Tomasz Napierala }
25801a13f2e6SEdward Tomasz Napierala
25811a13f2e6SEdward Tomasz Napierala return (error);
25821a13f2e6SEdward Tomasz Napierala }
25831a13f2e6SEdward Tomasz Napierala
25841a13f2e6SEdward Tomasz Napierala static inline int
25851a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused,
25861a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd)
25871a13f2e6SEdward Tomasz Napierala {
25881a13f2e6SEdward Tomasz Napierala int error;
25891a13f2e6SEdward Tomasz Napierala
25901a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES,
25911a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype));
25921a13f2e6SEdward Tomasz Napierala
25931a13f2e6SEdward Tomasz Napierala error = 0;
25941a13f2e6SEdward Tomasz Napierala
25951a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
25961a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
25971a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->s32 > voival->int32.s32) {
25981a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 = voival->int32.s32;
25991a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26001a13f2e6SEdward Tomasz Napierala }
26011a13f2e6SEdward Tomasz Napierala break;
26021a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
26031a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->u32 > voival->int32.u32) {
26041a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 = voival->int32.u32;
26051a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26061a13f2e6SEdward Tomasz Napierala }
26071a13f2e6SEdward Tomasz Napierala break;
26081a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
26091a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->s64 > voival->int64.s64) {
26101a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 = voival->int64.s64;
26111a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26121a13f2e6SEdward Tomasz Napierala }
26131a13f2e6SEdward Tomasz Napierala break;
26141a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
26151a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->u64 > voival->int64.u64) {
26161a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 = voival->int64.u64;
26171a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26181a13f2e6SEdward Tomasz Napierala }
26191a13f2e6SEdward Tomasz Napierala break;
26201a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
26211a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->slong > voival->intlong.slong) {
26221a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong = voival->intlong.slong;
26231a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26241a13f2e6SEdward Tomasz Napierala }
26251a13f2e6SEdward Tomasz Napierala break;
26261a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
26271a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) {
26281a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong = voival->intlong.ulong;
26291a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26301a13f2e6SEdward Tomasz Napierala }
26311a13f2e6SEdward Tomasz Napierala break;
26321a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
26331a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
26341a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
26351a13f2e6SEdward Tomasz Napierala voival->q32.sq32)))) {
26361a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26371a13f2e6SEdward Tomasz Napierala }
26381a13f2e6SEdward Tomasz Napierala break;
26391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
26401a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
26411a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
26421a13f2e6SEdward Tomasz Napierala voival->q32.uq32)))) {
26431a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26441a13f2e6SEdward Tomasz Napierala }
26451a13f2e6SEdward Tomasz Napierala break;
26461a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
26471a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
26481a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
26491a13f2e6SEdward Tomasz Napierala voival->q64.sq64)))) {
26501a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26511a13f2e6SEdward Tomasz Napierala }
26521a13f2e6SEdward Tomasz Napierala break;
26531a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
26541a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
26551a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
26561a13f2e6SEdward Tomasz Napierala voival->q64.uq64)))) {
26571a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
26581a13f2e6SEdward Tomasz Napierala }
26591a13f2e6SEdward Tomasz Napierala break;
26601a13f2e6SEdward Tomasz Napierala default:
26611a13f2e6SEdward Tomasz Napierala error = EINVAL;
26621a13f2e6SEdward Tomasz Napierala break;
26631a13f2e6SEdward Tomasz Napierala }
26641a13f2e6SEdward Tomasz Napierala
26651a13f2e6SEdward Tomasz Napierala return (error);
26661a13f2e6SEdward Tomasz Napierala }
26671a13f2e6SEdward Tomasz Napierala
26681a13f2e6SEdward Tomasz Napierala static inline int
26691a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused,
26701a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd)
26711a13f2e6SEdward Tomasz Napierala {
26721a13f2e6SEdward Tomasz Napierala int error;
26731a13f2e6SEdward Tomasz Napierala
26741a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES,
26751a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype));
26761a13f2e6SEdward Tomasz Napierala
26771a13f2e6SEdward Tomasz Napierala error = 0;
26781a13f2e6SEdward Tomasz Napierala
26791a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
26801a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
26811a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 += voival->int32.s32;
26821a13f2e6SEdward Tomasz Napierala break;
26831a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
26841a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 += voival->int32.u32;
26851a13f2e6SEdward Tomasz Napierala break;
26861a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
26871a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 += voival->int64.s64;
26881a13f2e6SEdward Tomasz Napierala break;
26891a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
26901a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 += voival->int64.u64;
26911a13f2e6SEdward Tomasz Napierala break;
26921a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
26931a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong += voival->intlong.slong;
26941a13f2e6SEdward Tomasz Napierala break;
26951a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
26961a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong += voival->intlong.ulong;
26971a13f2e6SEdward Tomasz Napierala break;
26981a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
26991a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32);
27001a13f2e6SEdward Tomasz Napierala break;
27011a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
27021a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32);
27031a13f2e6SEdward Tomasz Napierala break;
27041a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
27051a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64);
27061a13f2e6SEdward Tomasz Napierala break;
27071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
27081a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64);
27091a13f2e6SEdward Tomasz Napierala break;
27101a13f2e6SEdward Tomasz Napierala default:
27111a13f2e6SEdward Tomasz Napierala error = EINVAL;
27121a13f2e6SEdward Tomasz Napierala break;
27131a13f2e6SEdward Tomasz Napierala }
27141a13f2e6SEdward Tomasz Napierala
27151a13f2e6SEdward Tomasz Napierala if (!error)
27161a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
27171a13f2e6SEdward Tomasz Napierala
27181a13f2e6SEdward Tomasz Napierala return (error);
27191a13f2e6SEdward Tomasz Napierala }
27201a13f2e6SEdward Tomasz Napierala
27211a13f2e6SEdward Tomasz Napierala static inline int
27221a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival,
27231a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct voistatdata_hist *hist)
27241a13f2e6SEdward Tomasz Napierala {
27251a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric *bkt_lb, *bkt_ub;
27261a13f2e6SEdward Tomasz Napierala uint64_t *oob64, *cnt64;
27271a13f2e6SEdward Tomasz Napierala uint32_t *oob32, *cnt32;
27281a13f2e6SEdward Tomasz Napierala int error, i, found, is32bit, has_ub, eq_only;
27291a13f2e6SEdward Tomasz Napierala
27301a13f2e6SEdward Tomasz Napierala error = 0;
27311a13f2e6SEdward Tomasz Napierala
27321a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
27331a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
27341a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
27351a13f2e6SEdward Tomasz Napierala is32bit = 1;
27361a13f2e6SEdward Tomasz Napierala has_ub = eq_only = 0;
27371a13f2e6SEdward Tomasz Napierala oob32 = &VSD(crhist32, hist)->oob;
27381a13f2e6SEdward Tomasz Napierala break;
27391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
27401a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
27411a13f2e6SEdward Tomasz Napierala is32bit = has_ub = 1;
27421a13f2e6SEdward Tomasz Napierala eq_only = 0;
27431a13f2e6SEdward Tomasz Napierala oob32 = &VSD(drhist32, hist)->oob;
27441a13f2e6SEdward Tomasz Napierala break;
27451a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
27461a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
27471a13f2e6SEdward Tomasz Napierala is32bit = eq_only = 1;
27481a13f2e6SEdward Tomasz Napierala has_ub = 0;
27491a13f2e6SEdward Tomasz Napierala oob32 = &VSD(dvhist32, hist)->oob;
27501a13f2e6SEdward Tomasz Napierala break;
27511a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
27521a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
27531a13f2e6SEdward Tomasz Napierala is32bit = has_ub = eq_only = 0;
27541a13f2e6SEdward Tomasz Napierala oob64 = &VSD(crhist64, hist)->oob;
27551a13f2e6SEdward Tomasz Napierala break;
27561a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
27571a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
27581a13f2e6SEdward Tomasz Napierala is32bit = eq_only = 0;
27591a13f2e6SEdward Tomasz Napierala has_ub = 1;
27601a13f2e6SEdward Tomasz Napierala oob64 = &VSD(drhist64, hist)->oob;
27611a13f2e6SEdward Tomasz Napierala break;
27621a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
27631a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
27641a13f2e6SEdward Tomasz Napierala is32bit = has_ub = 0;
27651a13f2e6SEdward Tomasz Napierala eq_only = 1;
27661a13f2e6SEdward Tomasz Napierala oob64 = &VSD(dvhist64, hist)->oob;
27671a13f2e6SEdward Tomasz Napierala break;
27681a13f2e6SEdward Tomasz Napierala default:
27691a13f2e6SEdward Tomasz Napierala return (EINVAL);
27701a13f2e6SEdward Tomasz Napierala }
27711a13f2e6SEdward Tomasz Napierala i--; /* Adjust for 0-based array index. */
27721a13f2e6SEdward Tomasz Napierala
27731a13f2e6SEdward Tomasz Napierala /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */
27741a13f2e6SEdward Tomasz Napierala for (found = 0; i >= 0 && !found; i--) {
27751a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
27761a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32:
27771a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(crhist32, hist)->bkts[i].lb;
27781a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(crhist32, hist)->bkts[i].cnt;
27791a13f2e6SEdward Tomasz Napierala break;
27801a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32:
27811a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(drhist32, hist)->bkts[i].lb;
27821a13f2e6SEdward Tomasz Napierala bkt_ub = &VSD(drhist32, hist)->bkts[i].ub;
27831a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(drhist32, hist)->bkts[i].cnt;
27841a13f2e6SEdward Tomasz Napierala break;
27851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32:
27861a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(dvhist32, hist)->bkts[i].val;
27871a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt;
27881a13f2e6SEdward Tomasz Napierala break;
27891a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64:
27901a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(crhist64, hist)->bkts[i].lb;
27911a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(crhist64, hist)->bkts[i].cnt;
27921a13f2e6SEdward Tomasz Napierala break;
27931a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64:
27941a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(drhist64, hist)->bkts[i].lb;
27951a13f2e6SEdward Tomasz Napierala bkt_ub = &VSD(drhist64, hist)->bkts[i].ub;
27961a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(drhist64, hist)->bkts[i].cnt;
27971a13f2e6SEdward Tomasz Napierala break;
27981a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64:
27991a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(dvhist64, hist)->bkts[i].val;
28001a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt;
28011a13f2e6SEdward Tomasz Napierala break;
28021a13f2e6SEdward Tomasz Napierala default:
28031a13f2e6SEdward Tomasz Napierala return (EINVAL);
28041a13f2e6SEdward Tomasz Napierala }
28051a13f2e6SEdward Tomasz Napierala
28061a13f2e6SEdward Tomasz Napierala switch (voi_dtype) {
28071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
28081a13f2e6SEdward Tomasz Napierala if (voival->int32.s32 >= bkt_lb->int32.s32) {
28091a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int32.s32 ==
28101a13f2e6SEdward Tomasz Napierala bkt_lb->int32.s32) ||
28111a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28121a13f2e6SEdward Tomasz Napierala voival->int32.s32 < bkt_ub->int32.s32)))
28131a13f2e6SEdward Tomasz Napierala found = 1;
28141a13f2e6SEdward Tomasz Napierala }
28151a13f2e6SEdward Tomasz Napierala break;
28161a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
28171a13f2e6SEdward Tomasz Napierala if (voival->int32.u32 >= bkt_lb->int32.u32) {
28181a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int32.u32 ==
28191a13f2e6SEdward Tomasz Napierala bkt_lb->int32.u32) ||
28201a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28211a13f2e6SEdward Tomasz Napierala voival->int32.u32 < bkt_ub->int32.u32)))
28221a13f2e6SEdward Tomasz Napierala found = 1;
28231a13f2e6SEdward Tomasz Napierala }
28241a13f2e6SEdward Tomasz Napierala break;
28251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
28261a13f2e6SEdward Tomasz Napierala if (voival->int64.s64 >= bkt_lb->int64.s64)
28271a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int64.s64 ==
28281a13f2e6SEdward Tomasz Napierala bkt_lb->int64.s64) ||
28291a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28301a13f2e6SEdward Tomasz Napierala voival->int64.s64 < bkt_ub->int64.s64)))
28311a13f2e6SEdward Tomasz Napierala found = 1;
28321a13f2e6SEdward Tomasz Napierala break;
28331a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
28341a13f2e6SEdward Tomasz Napierala if (voival->int64.u64 >= bkt_lb->int64.u64)
28351a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int64.u64 ==
28361a13f2e6SEdward Tomasz Napierala bkt_lb->int64.u64) ||
28371a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28381a13f2e6SEdward Tomasz Napierala voival->int64.u64 < bkt_ub->int64.u64)))
28391a13f2e6SEdward Tomasz Napierala found = 1;
28401a13f2e6SEdward Tomasz Napierala break;
28411a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
28421a13f2e6SEdward Tomasz Napierala if (voival->intlong.slong >= bkt_lb->intlong.slong)
28431a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->intlong.slong ==
28441a13f2e6SEdward Tomasz Napierala bkt_lb->intlong.slong) ||
28451a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28461a13f2e6SEdward Tomasz Napierala voival->intlong.slong <
28471a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.slong)))
28481a13f2e6SEdward Tomasz Napierala found = 1;
28491a13f2e6SEdward Tomasz Napierala break;
28501a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
28511a13f2e6SEdward Tomasz Napierala if (voival->intlong.ulong >= bkt_lb->intlong.ulong)
28521a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->intlong.ulong ==
28531a13f2e6SEdward Tomasz Napierala bkt_lb->intlong.ulong) ||
28541a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28551a13f2e6SEdward Tomasz Napierala voival->intlong.ulong <
28561a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.ulong)))
28571a13f2e6SEdward Tomasz Napierala found = 1;
28581a13f2e6SEdward Tomasz Napierala break;
28591a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
28601a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32))
28611a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q32.sq32,
28621a13f2e6SEdward Tomasz Napierala bkt_lb->q32.sq32)) ||
28631a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28641a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q32.sq32,
28651a13f2e6SEdward Tomasz Napierala bkt_ub->q32.sq32))))
28661a13f2e6SEdward Tomasz Napierala found = 1;
28671a13f2e6SEdward Tomasz Napierala break;
28681a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
28691a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32))
28701a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q32.uq32,
28711a13f2e6SEdward Tomasz Napierala bkt_lb->q32.uq32)) ||
28721a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28731a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q32.uq32,
28741a13f2e6SEdward Tomasz Napierala bkt_ub->q32.uq32))))
28751a13f2e6SEdward Tomasz Napierala found = 1;
28761a13f2e6SEdward Tomasz Napierala break;
28771a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
28781a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64))
28791a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q64.sq64,
28801a13f2e6SEdward Tomasz Napierala bkt_lb->q64.sq64)) ||
28811a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28821a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q64.sq64,
28831a13f2e6SEdward Tomasz Napierala bkt_ub->q64.sq64))))
28841a13f2e6SEdward Tomasz Napierala found = 1;
28851a13f2e6SEdward Tomasz Napierala break;
28861a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
28871a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64))
28881a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q64.uq64,
28891a13f2e6SEdward Tomasz Napierala bkt_lb->q64.uq64)) ||
28901a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub ||
28911a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q64.uq64,
28921a13f2e6SEdward Tomasz Napierala bkt_ub->q64.uq64))))
28931a13f2e6SEdward Tomasz Napierala found = 1;
28941a13f2e6SEdward Tomasz Napierala break;
28951a13f2e6SEdward Tomasz Napierala default:
28961a13f2e6SEdward Tomasz Napierala break;
28971a13f2e6SEdward Tomasz Napierala }
28981a13f2e6SEdward Tomasz Napierala }
28991a13f2e6SEdward Tomasz Napierala
29001a13f2e6SEdward Tomasz Napierala if (found) {
29011a13f2e6SEdward Tomasz Napierala if (is32bit)
29021a13f2e6SEdward Tomasz Napierala *cnt32 += 1;
29031a13f2e6SEdward Tomasz Napierala else
29041a13f2e6SEdward Tomasz Napierala *cnt64 += 1;
29051a13f2e6SEdward Tomasz Napierala } else {
29061a13f2e6SEdward Tomasz Napierala if (is32bit)
29071a13f2e6SEdward Tomasz Napierala *oob32 += 1;
29081a13f2e6SEdward Tomasz Napierala else
29091a13f2e6SEdward Tomasz Napierala *oob64 += 1;
29101a13f2e6SEdward Tomasz Napierala }
29111a13f2e6SEdward Tomasz Napierala
29121a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
29131a13f2e6SEdward Tomasz Napierala return (error);
29141a13f2e6SEdward Tomasz Napierala }
29151a13f2e6SEdward Tomasz Napierala
29161a13f2e6SEdward Tomasz Napierala static inline int
29171a13f2e6SEdward Tomasz Napierala stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype,
29181a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst, int attempt)
29191a13f2e6SEdward Tomasz Napierala {
29201a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree;
29211a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree;
29221a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32;
29231a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64;
29241a13f2e6SEdward Tomasz Napierala uint64_t ebits, idxmask;
29251a13f2e6SEdward Tomasz Napierala uint32_t bitsperidx, nebits;
29261a13f2e6SEdward Tomasz Napierala int error, idx, is32bit, maxctds, remctds, tmperr;
29271a13f2e6SEdward Tomasz Napierala
29281a13f2e6SEdward Tomasz Napierala error = 0;
29291a13f2e6SEdward Tomasz Napierala
29301a13f2e6SEdward Tomasz Napierala switch (vs_dtype) {
29311a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
29321a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
29331a13f2e6SEdward Tomasz Napierala if (!ARB_FULL(ctd32tree))
29341a13f2e6SEdward Tomasz Napierala return (0);
29351a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->compcnt++;
29361a13f2e6SEdward Tomasz Napierala maxctds = remctds = ARB_MAXNODES(ctd32tree);
29371a13f2e6SEdward Tomasz Napierala ARB_RESET_TREE(ctd32tree, ctdth32, maxctds);
29381a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt = 0;
29391a13f2e6SEdward Tomasz Napierala is32bit = 1;
29401a13f2e6SEdward Tomasz Napierala ctd64tree = NULL;
29411a13f2e6SEdward Tomasz Napierala ctd64 = NULL;
29421a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
29431a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
29441a13f2e6SEdward Tomasz Napierala #endif
29451a13f2e6SEdward Tomasz Napierala break;
29461a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
29471a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
29481a13f2e6SEdward Tomasz Napierala if (!ARB_FULL(ctd64tree))
29491a13f2e6SEdward Tomasz Napierala return (0);
29501a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->compcnt++;
29511a13f2e6SEdward Tomasz Napierala maxctds = remctds = ARB_MAXNODES(ctd64tree);
29521a13f2e6SEdward Tomasz Napierala ARB_RESET_TREE(ctd64tree, ctdth64, maxctds);
29531a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt = 0;
29541a13f2e6SEdward Tomasz Napierala is32bit = 0;
29551a13f2e6SEdward Tomasz Napierala ctd32tree = NULL;
29561a13f2e6SEdward Tomasz Napierala ctd32 = NULL;
29571a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
29581a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
29591a13f2e6SEdward Tomasz Napierala #endif
29601a13f2e6SEdward Tomasz Napierala break;
29611a13f2e6SEdward Tomasz Napierala default:
29621a13f2e6SEdward Tomasz Napierala return (EINVAL);
29631a13f2e6SEdward Tomasz Napierala }
29641a13f2e6SEdward Tomasz Napierala
29651a13f2e6SEdward Tomasz Napierala /*
29661a13f2e6SEdward Tomasz Napierala * Rebuild the t-digest ARB by pseudorandomly selecting centroids and
29671a13f2e6SEdward Tomasz Napierala * re-inserting the mu/cnt of each as a value and corresponding weight.
29681a13f2e6SEdward Tomasz Napierala */
29691a13f2e6SEdward Tomasz Napierala
2970f3bae413SConrad Meyer /*
2971f3bae413SConrad Meyer * XXXCEM: random(9) is currently rand(3), not random(3). rand(3)
2972f3bae413SConrad Meyer * RAND_MAX happens to be approximately 31 bits (range [0,
2973f3bae413SConrad Meyer * 0x7ffffffd]), so the math kinda works out. When/if this portion of
2974f3bae413SConrad Meyer * the code is compiled in userspace, it gets the random(3) behavior,
2975f3bae413SConrad Meyer * which has expected range [0, 0x7fffffff].
2976f3bae413SConrad Meyer */
2977f3bae413SConrad Meyer #define bitsperrand 31
29781a13f2e6SEdward Tomasz Napierala ebits = 0;
29791a13f2e6SEdward Tomasz Napierala nebits = 0;
29801a13f2e6SEdward Tomasz Napierala bitsperidx = fls(maxctds);
29811a13f2e6SEdward Tomasz Napierala KASSERT(bitsperidx <= sizeof(ebits) << 3,
29821a13f2e6SEdward Tomasz Napierala ("%s: bitsperidx=%d, ebits=%d",
29831a13f2e6SEdward Tomasz Napierala __func__, bitsperidx, (int)(sizeof(ebits) << 3)));
29841a13f2e6SEdward Tomasz Napierala idxmask = (UINT64_C(1) << bitsperidx) - 1;
29851a13f2e6SEdward Tomasz Napierala
29861a13f2e6SEdward Tomasz Napierala /* Initialise the free list with randomised centroid indices. */
29871a13f2e6SEdward Tomasz Napierala for (; remctds > 0; remctds--) {
29881a13f2e6SEdward Tomasz Napierala while (nebits < bitsperidx) {
29891a13f2e6SEdward Tomasz Napierala ebits |= ((uint64_t)random()) << nebits;
29901a13f2e6SEdward Tomasz Napierala nebits += bitsperrand;
29911a13f2e6SEdward Tomasz Napierala if (nebits > (sizeof(ebits) << 3))
29921a13f2e6SEdward Tomasz Napierala nebits = sizeof(ebits) << 3;
29931a13f2e6SEdward Tomasz Napierala }
29941a13f2e6SEdward Tomasz Napierala idx = ebits & idxmask;
29951a13f2e6SEdward Tomasz Napierala nebits -= bitsperidx;
29961a13f2e6SEdward Tomasz Napierala ebits >>= bitsperidx;
29971a13f2e6SEdward Tomasz Napierala
29981a13f2e6SEdward Tomasz Napierala /*
29991a13f2e6SEdward Tomasz Napierala * Select the next centroid to put on the ARB free list. We
30001a13f2e6SEdward Tomasz Napierala * start with the centroid at our randomly selected array index,
30011a13f2e6SEdward Tomasz Napierala * and work our way forwards until finding one (the latter
30021a13f2e6SEdward Tomasz Napierala * aspect reduces re-insertion randomness, but is good enough).
30031a13f2e6SEdward Tomasz Napierala */
30041a13f2e6SEdward Tomasz Napierala do {
30051a13f2e6SEdward Tomasz Napierala if (idx >= maxctds)
30061a13f2e6SEdward Tomasz Napierala idx %= maxctds;
30071a13f2e6SEdward Tomasz Napierala
30081a13f2e6SEdward Tomasz Napierala if (is32bit)
30091a13f2e6SEdward Tomasz Napierala ctd32 = ARB_NODE(ctd32tree, idx);
30101a13f2e6SEdward Tomasz Napierala else
30111a13f2e6SEdward Tomasz Napierala ctd64 = ARB_NODE(ctd64tree, idx);
30121a13f2e6SEdward Tomasz Napierala } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) :
30131a13f2e6SEdward Tomasz Napierala ARB_ISFREE(ctd64, ctdlnk)) && ++idx);
30141a13f2e6SEdward Tomasz Napierala
30151a13f2e6SEdward Tomasz Napierala /* Put the centroid on the ARB free list. */
30161a13f2e6SEdward Tomasz Napierala if (is32bit)
30171a13f2e6SEdward Tomasz Napierala ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk);
30181a13f2e6SEdward Tomasz Napierala else
30191a13f2e6SEdward Tomasz Napierala ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk);
30201a13f2e6SEdward Tomasz Napierala }
30211a13f2e6SEdward Tomasz Napierala
30221a13f2e6SEdward Tomasz Napierala /*
30231a13f2e6SEdward Tomasz Napierala * The free list now contains the randomised indices of every centroid.
30241a13f2e6SEdward Tomasz Napierala * Walk the free list from start to end, re-inserting each centroid's
30251a13f2e6SEdward Tomasz Napierala * mu/cnt. The tdgst_add() call may or may not consume the free centroid
30261a13f2e6SEdward Tomasz Napierala * we re-insert values from during each loop iteration, so we must latch
30271a13f2e6SEdward Tomasz Napierala * the index of the next free list centroid before the re-insertion
30281a13f2e6SEdward Tomasz Napierala * call. The previous loop above should have left the centroid pointer
30291a13f2e6SEdward Tomasz Napierala * pointing to the element at the head of the free list.
30301a13f2e6SEdward Tomasz Napierala */
30311a13f2e6SEdward Tomasz Napierala KASSERT((is32bit ?
30321a13f2e6SEdward Tomasz Napierala ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) :
30331a13f2e6SEdward Tomasz Napierala ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)),
30341a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__,
30351a13f2e6SEdward Tomasz Napierala (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
30361a13f2e6SEdward Tomasz Napierala remctds = maxctds;
30371a13f2e6SEdward Tomasz Napierala while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) {
30381a13f2e6SEdward Tomasz Napierala tmperr = 0;
30391a13f2e6SEdward Tomasz Napierala if (is32bit) {
30401a13f2e6SEdward Tomasz Napierala s64q_t x;
30411a13f2e6SEdward Tomasz Napierala
30421a13f2e6SEdward Tomasz Napierala idx = ARB_NEXTFREEIDX(ctd32, ctdlnk);
30431a13f2e6SEdward Tomasz Napierala /* Cloning a s32q_t into a s64q_t should never fail. */
30441a13f2e6SEdward Tomasz Napierala tmperr = Q_QCLONEQ(&x, ctd32->mu);
30451a13f2e6SEdward Tomasz Napierala tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add(
30461a13f2e6SEdward Tomasz Napierala vs_dtype, tdgst, x, ctd32->cnt, attempt);
30471a13f2e6SEdward Tomasz Napierala ctd32 = ARB_NODE(ctd32tree, idx);
30481a13f2e6SEdward Tomasz Napierala KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk),
30491a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__,
30501a13f2e6SEdward Tomasz Napierala ctd32tree));
30511a13f2e6SEdward Tomasz Napierala } else {
30521a13f2e6SEdward Tomasz Napierala idx = ARB_NEXTFREEIDX(ctd64, ctdlnk);
30531a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst,
30541a13f2e6SEdward Tomasz Napierala ctd64->mu, ctd64->cnt, attempt);
30551a13f2e6SEdward Tomasz Napierala ctd64 = ARB_NODE(ctd64tree, idx);
30561a13f2e6SEdward Tomasz Napierala KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk),
30571a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__,
30581a13f2e6SEdward Tomasz Napierala ctd64tree));
30591a13f2e6SEdward Tomasz Napierala }
30601a13f2e6SEdward Tomasz Napierala /*
30611a13f2e6SEdward Tomasz Napierala * This process should not produce errors, bugs notwithstanding.
30621a13f2e6SEdward Tomasz Napierala * Just in case, latch any errors and attempt all re-insertions.
30631a13f2e6SEdward Tomasz Napierala */
30641a13f2e6SEdward Tomasz Napierala error = tmperr ? tmperr : error;
30651a13f2e6SEdward Tomasz Napierala remctds--;
30661a13f2e6SEdward Tomasz Napierala }
30671a13f2e6SEdward Tomasz Napierala
30681a13f2e6SEdward Tomasz Napierala KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__,
30691a13f2e6SEdward Tomasz Napierala (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
30701a13f2e6SEdward Tomasz Napierala
30711a13f2e6SEdward Tomasz Napierala return (error);
30721a13f2e6SEdward Tomasz Napierala }
30731a13f2e6SEdward Tomasz Napierala
30741a13f2e6SEdward Tomasz Napierala static inline int
30751a13f2e6SEdward Tomasz Napierala stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
30761a13f2e6SEdward Tomasz Napierala s64q_t x, uint64_t weight, int attempt)
30771a13f2e6SEdward Tomasz Napierala {
30781a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
30791a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(x, 10)];
30801a13f2e6SEdward Tomasz Napierala #endif
30811a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree;
30821a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree;
30831a13f2e6SEdward Tomasz Napierala void *closest, *cur, *lb, *ub;
30841a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32;
30851a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64;
30861a13f2e6SEdward Tomasz Napierala uint64_t cnt, smplcnt, sum, tmpsum;
30871a13f2e6SEdward Tomasz Napierala s64q_t k, minz, q, z;
30881a13f2e6SEdward Tomasz Napierala int error, is32bit, n;
30891a13f2e6SEdward Tomasz Napierala
30901a13f2e6SEdward Tomasz Napierala error = 0;
30911a13f2e6SEdward Tomasz Napierala minz = Q_INI(&z, 0, 0, Q_NFBITS(x));
30921a13f2e6SEdward Tomasz Napierala
30931a13f2e6SEdward Tomasz Napierala switch (vs_dtype) {
30941a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
30951a13f2e6SEdward Tomasz Napierala if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt)
30961a13f2e6SEdward Tomasz Napierala error = EOVERFLOW;
30971a13f2e6SEdward Tomasz Napierala smplcnt = VSD(tdgstclust32, tdgst)->smplcnt;
30981a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
30991a13f2e6SEdward Tomasz Napierala is32bit = 1;
31001a13f2e6SEdward Tomasz Napierala ctd64tree = NULL;
31011a13f2e6SEdward Tomasz Napierala ctd64 = NULL;
31021a13f2e6SEdward Tomasz Napierala break;
31031a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
31041a13f2e6SEdward Tomasz Napierala if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt)
31051a13f2e6SEdward Tomasz Napierala error = EOVERFLOW;
31061a13f2e6SEdward Tomasz Napierala smplcnt = VSD(tdgstclust64, tdgst)->smplcnt;
31071a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
31081a13f2e6SEdward Tomasz Napierala is32bit = 0;
31091a13f2e6SEdward Tomasz Napierala ctd32tree = NULL;
31101a13f2e6SEdward Tomasz Napierala ctd32 = NULL;
31111a13f2e6SEdward Tomasz Napierala break;
31121a13f2e6SEdward Tomasz Napierala default:
31131a13f2e6SEdward Tomasz Napierala error = EINVAL;
31141a13f2e6SEdward Tomasz Napierala break;
31151a13f2e6SEdward Tomasz Napierala }
31161a13f2e6SEdward Tomasz Napierala
31171a13f2e6SEdward Tomasz Napierala if (error)
31181a13f2e6SEdward Tomasz Napierala return (error);
31191a13f2e6SEdward Tomasz Napierala
31201a13f2e6SEdward Tomasz Napierala /*
31211a13f2e6SEdward Tomasz Napierala * Inspired by Ted Dunning's AVLTreeDigest.java
31221a13f2e6SEdward Tomasz Napierala */
31231a13f2e6SEdward Tomasz Napierala do {
31241a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC)
31251a13f2e6SEdward Tomasz Napierala KASSERT(attempt < 5,
31261a13f2e6SEdward Tomasz Napierala ("%s: Too many attempts", __func__));
31271a13f2e6SEdward Tomasz Napierala #endif
31281a13f2e6SEdward Tomasz Napierala if (attempt >= 5)
31291a13f2e6SEdward Tomasz Napierala return (EAGAIN);
31301a13f2e6SEdward Tomasz Napierala
31311a13f2e6SEdward Tomasz Napierala Q_SIFVAL(minz, Q_IFMAXVAL(minz));
31321a13f2e6SEdward Tomasz Napierala closest = ub = NULL;
31331a13f2e6SEdward Tomasz Napierala sum = tmpsum = 0;
31341a13f2e6SEdward Tomasz Napierala
31351a13f2e6SEdward Tomasz Napierala if (is32bit)
31361a13f2e6SEdward Tomasz Napierala lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree));
31371a13f2e6SEdward Tomasz Napierala else
31381a13f2e6SEdward Tomasz Napierala lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree));
31391a13f2e6SEdward Tomasz Napierala
31401a13f2e6SEdward Tomasz Napierala if (lb == NULL) /* Empty tree. */
31411a13f2e6SEdward Tomasz Napierala lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) :
31421a13f2e6SEdward Tomasz Napierala (void *)ARB_ROOT(ctd64tree));
31431a13f2e6SEdward Tomasz Napierala
31441a13f2e6SEdward Tomasz Napierala /*
31451a13f2e6SEdward Tomasz Napierala * Find the set of centroids with minimum distance to x and
31461a13f2e6SEdward Tomasz Napierala * compute the sum of counts for all centroids with mean less
31471a13f2e6SEdward Tomasz Napierala * than the first centroid in the set.
31481a13f2e6SEdward Tomasz Napierala */
31491a13f2e6SEdward Tomasz Napierala for (; cur != NULL;
31501a13f2e6SEdward Tomasz Napierala cur = (is32bit ?
31511a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
31521a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
31531a13f2e6SEdward Tomasz Napierala if (is32bit) {
31541a13f2e6SEdward Tomasz Napierala cnt = ctd32->cnt;
31551a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(ctd32->mu, x),
31561a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(mu,x)=%d", __func__,
31571a13f2e6SEdward Tomasz Napierala Q_RELPREC(ctd32->mu, x)));
31581a13f2e6SEdward Tomasz Napierala /* Ok to assign as both have same precision. */
31591a13f2e6SEdward Tomasz Napierala z = ctd32->mu;
31601a13f2e6SEdward Tomasz Napierala } else {
31611a13f2e6SEdward Tomasz Napierala cnt = ctd64->cnt;
31621a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(ctd64->mu, x),
31631a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(mu,x)=%d", __func__,
31641a13f2e6SEdward Tomasz Napierala Q_RELPREC(ctd64->mu, x)));
31651a13f2e6SEdward Tomasz Napierala /* Ok to assign as both have same precision. */
31661a13f2e6SEdward Tomasz Napierala z = ctd64->mu;
31671a13f2e6SEdward Tomasz Napierala }
31681a13f2e6SEdward Tomasz Napierala
31691a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&z, x);
31701a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC)
31711a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", __func__,
31721a13f2e6SEdward Tomasz Napierala error));
31731a13f2e6SEdward Tomasz Napierala #endif
31741a13f2e6SEdward Tomasz Napierala if (error)
31751a13f2e6SEdward Tomasz Napierala return (error);
31761a13f2e6SEdward Tomasz Napierala
31771a13f2e6SEdward Tomasz Napierala z = Q_QABS(z);
31781a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(z, minz)) {
31791a13f2e6SEdward Tomasz Napierala minz = z;
31801a13f2e6SEdward Tomasz Napierala lb = cur;
31811a13f2e6SEdward Tomasz Napierala sum = tmpsum;
31821a13f2e6SEdward Tomasz Napierala tmpsum += cnt;
31831a13f2e6SEdward Tomasz Napierala } else if (Q_QGTQ(z, minz)) {
31841a13f2e6SEdward Tomasz Napierala ub = cur;
31851a13f2e6SEdward Tomasz Napierala break;
31861a13f2e6SEdward Tomasz Napierala }
31871a13f2e6SEdward Tomasz Napierala }
31881a13f2e6SEdward Tomasz Napierala
31891a13f2e6SEdward Tomasz Napierala cur = (is32bit ?
31901a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) :
31911a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb));
31921a13f2e6SEdward Tomasz Napierala
31931a13f2e6SEdward Tomasz Napierala for (n = 0; cur != ub; cur = (is32bit ?
31941a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
31951a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
31961a13f2e6SEdward Tomasz Napierala if (is32bit)
31971a13f2e6SEdward Tomasz Napierala cnt = ctd32->cnt;
31981a13f2e6SEdward Tomasz Napierala else
31991a13f2e6SEdward Tomasz Napierala cnt = ctd64->cnt;
32001a13f2e6SEdward Tomasz Napierala
32011a13f2e6SEdward Tomasz Napierala q = Q_CTRLINI(16);
32021a13f2e6SEdward Tomasz Napierala if (smplcnt == 1)
32031a13f2e6SEdward Tomasz Napierala error = Q_QFRACI(&q, 1, 2);
32041a13f2e6SEdward Tomasz Napierala else
32051a13f2e6SEdward Tomasz Napierala /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */
32061a13f2e6SEdward Tomasz Napierala error = Q_QFRACI(&q, (sum << 1) + cnt - 1,
32071a13f2e6SEdward Tomasz Napierala (smplcnt - 1) << 1);
32081a13f2e6SEdward Tomasz Napierala k = q;
32091a13f2e6SEdward Tomasz Napierala /* k = q x 4 x samplcnt x attempt */
32101a13f2e6SEdward Tomasz Napierala error |= Q_QMULI(&k, 4 * smplcnt * attempt);
32111a13f2e6SEdward Tomasz Napierala /* k = k x (1 - q) */
32121a13f2e6SEdward Tomasz Napierala error |= Q_QSUBI(&q, 1);
32131a13f2e6SEdward Tomasz Napierala q = Q_QABS(q);
32141a13f2e6SEdward Tomasz Napierala error |= Q_QMULQ(&k, q);
32151a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC)
32161a13f2e6SEdward Tomasz Napierala #if !defined(_KERNEL)
32171a13f2e6SEdward Tomasz Napierala double q_dbl, k_dbl, q2d, k2d;
32181a13f2e6SEdward Tomasz Napierala q2d = Q_Q2D(q);
32191a13f2e6SEdward Tomasz Napierala k2d = Q_Q2D(k);
32201a13f2e6SEdward Tomasz Napierala q_dbl = smplcnt == 1 ? 0.5 :
32211a13f2e6SEdward Tomasz Napierala (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1);
32221a13f2e6SEdward Tomasz Napierala k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt;
32231a13f2e6SEdward Tomasz Napierala /*
32241a13f2e6SEdward Tomasz Napierala * If the difference between q and q_dbl is greater than
32251a13f2e6SEdward Tomasz Napierala * the fractional precision of q, something is off.
32261a13f2e6SEdward Tomasz Napierala * NB: q is holding the value of 1 - q
32271a13f2e6SEdward Tomasz Napierala */
32281a13f2e6SEdward Tomasz Napierala q_dbl = 1.0 - q_dbl;
32291a13f2e6SEdward Tomasz Napierala KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) <
32301a13f2e6SEdward Tomasz Napierala (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))),
32311a13f2e6SEdward Tomasz Napierala ("Q-type q bad precision"));
32321a13f2e6SEdward Tomasz Napierala KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) <
32331a13f2e6SEdward Tomasz Napierala 1.0 + (0.01 * smplcnt),
32341a13f2e6SEdward Tomasz Napierala ("Q-type k bad precision"));
32351a13f2e6SEdward Tomasz Napierala #endif /* !_KERNEL */
32361a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", __func__,
32371a13f2e6SEdward Tomasz Napierala error));
32381a13f2e6SEdward Tomasz Napierala #endif /* DIAGNOSTIC */
32391a13f2e6SEdward Tomasz Napierala if (error)
32401a13f2e6SEdward Tomasz Napierala return (error);
32411a13f2e6SEdward Tomasz Napierala if ((is32bit && ((ctd32->cnt + weight) <=
32421a13f2e6SEdward Tomasz Napierala (uint64_t)Q_GIVAL(k))) ||
32431a13f2e6SEdward Tomasz Napierala (!is32bit && ((ctd64->cnt + weight) <=
32441a13f2e6SEdward Tomasz Napierala (uint64_t)Q_GIVAL(k)))) {
32451a13f2e6SEdward Tomasz Napierala n++;
32461a13f2e6SEdward Tomasz Napierala /* random() produces 31 bits. */
32471a13f2e6SEdward Tomasz Napierala if (random() < (INT32_MAX / n))
32481a13f2e6SEdward Tomasz Napierala closest = cur;
32491a13f2e6SEdward Tomasz Napierala }
32501a13f2e6SEdward Tomasz Napierala sum += cnt;
32511a13f2e6SEdward Tomasz Napierala }
32521a13f2e6SEdward Tomasz Napierala } while (closest == NULL &&
32531a13f2e6SEdward Tomasz Napierala (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) &&
32541a13f2e6SEdward Tomasz Napierala (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst,
32551a13f2e6SEdward Tomasz Napierala attempt++)) == 0);
32561a13f2e6SEdward Tomasz Napierala
32571a13f2e6SEdward Tomasz Napierala if (error)
32581a13f2e6SEdward Tomasz Napierala return (error);
32591a13f2e6SEdward Tomasz Napierala
32601a13f2e6SEdward Tomasz Napierala if (closest != NULL) {
32611a13f2e6SEdward Tomasz Napierala /* Merge with an existing centroid. */
32621a13f2e6SEdward Tomasz Napierala if (is32bit) {
32631a13f2e6SEdward Tomasz Napierala ctd32 = (struct voistatdata_tdgstctd32 *)closest;
32641a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&x, ctd32->mu);
32651eb402e4SLawrence Stewart /*
32661eb402e4SLawrence Stewart * The following calculation "x / (cnt + weight)"
32671eb402e4SLawrence Stewart * computes the amount by which to adjust the centroid's
32681eb402e4SLawrence Stewart * mu value in order to merge in the VOI sample.
32691eb402e4SLawrence Stewart *
32701eb402e4SLawrence Stewart * It can underflow (Q_QDIVI() returns ERANGE) when the
32711eb402e4SLawrence Stewart * user centroids' fractional precision (which is
32721eb402e4SLawrence Stewart * inherited by 'x') is too low to represent the result.
32731eb402e4SLawrence Stewart *
32741eb402e4SLawrence Stewart * A sophisticated approach to dealing with this issue
32751eb402e4SLawrence Stewart * would minimise accumulation of error by tracking
32761eb402e4SLawrence Stewart * underflow per centroid and making an adjustment when
32771eb402e4SLawrence Stewart * a LSB's worth of underflow has accumulated.
32781eb402e4SLawrence Stewart *
32791eb402e4SLawrence Stewart * A simpler approach is to let the result underflow
32801eb402e4SLawrence Stewart * i.e. merge the VOI sample into the centroid without
32811eb402e4SLawrence Stewart * adjusting the centroid's mu, and rely on the user to
32821eb402e4SLawrence Stewart * specify their t-digest with sufficient centroid
32831eb402e4SLawrence Stewart * fractional precision such that the accumulation of
32841eb402e4SLawrence Stewart * error from multiple underflows is of no material
32851eb402e4SLawrence Stewart * consequence to the centroid's final value of mu.
32861eb402e4SLawrence Stewart *
32871eb402e4SLawrence Stewart * For the moment, the latter approach is employed by
32881eb402e4SLawrence Stewart * simply ignoring ERANGE here.
32891eb402e4SLawrence Stewart *
32901eb402e4SLawrence Stewart * XXXLAS: Per-centroid underflow tracking is likely too
32911eb402e4SLawrence Stewart * onerous, but it probably makes sense to accumulate a
32921eb402e4SLawrence Stewart * single underflow error variable across all centroids
32931eb402e4SLawrence Stewart * and report it as part of the digest to provide
32941eb402e4SLawrence Stewart * additional visibility into the digest's fidelity.
32951eb402e4SLawrence Stewart */
32961a13f2e6SEdward Tomasz Napierala error = error ? error :
32971a13f2e6SEdward Tomasz Napierala Q_QDIVI(&x, ctd32->cnt + weight);
32981eb402e4SLawrence Stewart if ((error && error != ERANGE)
32991eb402e4SLawrence Stewart || (error = Q_QADDQ(&ctd32->mu, x))) {
33001a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33011a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d",
33021a13f2e6SEdward Tomasz Napierala __func__, error));
33031a13f2e6SEdward Tomasz Napierala #endif
33041a13f2e6SEdward Tomasz Napierala return (error);
33051a13f2e6SEdward Tomasz Napierala }
33061a13f2e6SEdward Tomasz Napierala ctd32->cnt += weight;
33071a13f2e6SEdward Tomasz Napierala error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) ==
33081a13f2e6SEdward Tomasz Napierala NULL ? 0 : EALREADY;
33091a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33101a13f2e6SEdward Tomasz Napierala RB_REINSERT(rbctdth32,
33111a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
33121a13f2e6SEdward Tomasz Napierala #endif
33131a13f2e6SEdward Tomasz Napierala } else {
33141a13f2e6SEdward Tomasz Napierala ctd64 = (struct voistatdata_tdgstctd64 *)closest;
33151a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&x, ctd64->mu);
33161a13f2e6SEdward Tomasz Napierala error = error ? error :
33171a13f2e6SEdward Tomasz Napierala Q_QDIVI(&x, ctd64->cnt + weight);
33181eb402e4SLawrence Stewart /* Refer to is32bit ERANGE discussion above. */
33191eb402e4SLawrence Stewart if ((error && error != ERANGE)
33201eb402e4SLawrence Stewart || (error = Q_QADDQ(&ctd64->mu, x))) {
33211a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d",
33221a13f2e6SEdward Tomasz Napierala __func__, error));
33231a13f2e6SEdward Tomasz Napierala return (error);
33241a13f2e6SEdward Tomasz Napierala }
33251a13f2e6SEdward Tomasz Napierala ctd64->cnt += weight;
33261a13f2e6SEdward Tomasz Napierala error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) ==
33271a13f2e6SEdward Tomasz Napierala NULL ? 0 : EALREADY;
33281a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33291a13f2e6SEdward Tomasz Napierala RB_REINSERT(rbctdth64,
33301a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64);
33311a13f2e6SEdward Tomasz Napierala #endif
33321a13f2e6SEdward Tomasz Napierala }
33331a13f2e6SEdward Tomasz Napierala } else {
33341a13f2e6SEdward Tomasz Napierala /*
33351a13f2e6SEdward Tomasz Napierala * Add a new centroid. If digest compression is working
33361a13f2e6SEdward Tomasz Napierala * correctly, there should always be at least one free.
33371a13f2e6SEdward Tomasz Napierala */
33381a13f2e6SEdward Tomasz Napierala if (is32bit) {
33391a13f2e6SEdward Tomasz Napierala ctd32 = ARB_GETFREE(ctd32tree, ctdlnk);
33401a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33411a13f2e6SEdward Tomasz Napierala KASSERT(ctd32 != NULL,
33421a13f2e6SEdward Tomasz Napierala ("%s: t-digest@%p has no free centroids",
33431a13f2e6SEdward Tomasz Napierala __func__, tdgst));
33441a13f2e6SEdward Tomasz Napierala #endif
33451a13f2e6SEdward Tomasz Napierala if (ctd32 == NULL)
33461a13f2e6SEdward Tomasz Napierala return (EAGAIN);
33471a13f2e6SEdward Tomasz Napierala if ((error = Q_QCPYVALQ(&ctd32->mu, x)))
33481a13f2e6SEdward Tomasz Napierala return (error);
33491a13f2e6SEdward Tomasz Napierala ctd32->cnt = weight;
33501a13f2e6SEdward Tomasz Napierala error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ?
33511a13f2e6SEdward Tomasz Napierala 0 : EALREADY;
33521a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33531a13f2e6SEdward Tomasz Napierala RB_INSERT(rbctdth32,
33541a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
33551a13f2e6SEdward Tomasz Napierala #endif
33561a13f2e6SEdward Tomasz Napierala } else {
33571a13f2e6SEdward Tomasz Napierala ctd64 = ARB_GETFREE(ctd64tree, ctdlnk);
33581a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33591a13f2e6SEdward Tomasz Napierala KASSERT(ctd64 != NULL,
33601a13f2e6SEdward Tomasz Napierala ("%s: t-digest@%p has no free centroids",
33611a13f2e6SEdward Tomasz Napierala __func__, tdgst));
33621a13f2e6SEdward Tomasz Napierala #endif
33631a13f2e6SEdward Tomasz Napierala if (ctd64 == NULL) /* Should not happen. */
33641a13f2e6SEdward Tomasz Napierala return (EAGAIN);
33651a13f2e6SEdward Tomasz Napierala /* Direct assignment ok as both have same type/prec. */
33661a13f2e6SEdward Tomasz Napierala ctd64->mu = x;
33671a13f2e6SEdward Tomasz Napierala ctd64->cnt = weight;
33681a13f2e6SEdward Tomasz Napierala error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ?
33691a13f2e6SEdward Tomasz Napierala 0 : EALREADY;
33701a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33711a13f2e6SEdward Tomasz Napierala RB_INSERT(rbctdth64, &VSD(tdgstclust64,
33721a13f2e6SEdward Tomasz Napierala tdgst)->rbctdtree, ctd64);
33731a13f2e6SEdward Tomasz Napierala #endif
33741a13f2e6SEdward Tomasz Napierala }
33751a13f2e6SEdward Tomasz Napierala }
33761a13f2e6SEdward Tomasz Napierala
33771a13f2e6SEdward Tomasz Napierala if (is32bit)
33781a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt += weight;
33791a13f2e6SEdward Tomasz Napierala else {
33801a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt += weight;
33811a13f2e6SEdward Tomasz Napierala
33821a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC
33831a13f2e6SEdward Tomasz Napierala struct rbctdth64 *rbctdtree =
33841a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust64, tdgst)->rbctdtree;
33851a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *rbctd64;
33861a13f2e6SEdward Tomasz Napierala int i = 0;
33871a13f2e6SEdward Tomasz Napierala ARB_FOREACH(ctd64, ctdth64, ctd64tree) {
33881a13f2e6SEdward Tomasz Napierala rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) :
33891a13f2e6SEdward Tomasz Napierala RB_NEXT(rbctdth64, rbctdtree, rbctd64));
33901a13f2e6SEdward Tomasz Napierala
33911a13f2e6SEdward Tomasz Napierala if (i >= ARB_CURNODES(ctd64tree)
33921a13f2e6SEdward Tomasz Napierala || ctd64 != rbctd64
33931a13f2e6SEdward Tomasz Napierala || ARB_MIN(ctdth64, ctd64tree) !=
33941a13f2e6SEdward Tomasz Napierala RB_MIN(rbctdth64, rbctdtree)
33951a13f2e6SEdward Tomasz Napierala || ARB_MAX(ctdth64, ctd64tree) !=
33961a13f2e6SEdward Tomasz Napierala RB_MAX(rbctdth64, rbctdtree)
33971a13f2e6SEdward Tomasz Napierala || ARB_LEFTIDX(ctd64, ctdlnk) !=
33981a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk))
33991a13f2e6SEdward Tomasz Napierala || ARB_RIGHTIDX(ctd64, ctdlnk) !=
34001a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk))
34011a13f2e6SEdward Tomasz Napierala || ARB_PARENTIDX(ctd64, ctdlnk) !=
34021a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree,
34031a13f2e6SEdward Tomasz Napierala RB_PARENT(rbctd64, rblnk))) {
34041a13f2e6SEdward Tomasz Napierala Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr));
34051a13f2e6SEdward Tomasz Napierala printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
34061a13f2e6SEdward Tomasz Napierala "mu=%s\n",
34071a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, ctd64),
34081a13f2e6SEdward Tomasz Napierala ARB_PARENTIDX(ctd64, ctdlnk),
34091a13f2e6SEdward Tomasz Napierala ARB_LEFTIDX(ctd64, ctdlnk),
34101a13f2e6SEdward Tomasz Napierala ARB_RIGHTIDX(ctd64, ctdlnk),
34111a13f2e6SEdward Tomasz Napierala ARB_COLOR(ctd64, ctdlnk),
34121a13f2e6SEdward Tomasz Napierala qstr);
34131a13f2e6SEdward Tomasz Napierala
34141a13f2e6SEdward Tomasz Napierala Q_TOSTR(rbctd64->mu, -1, 10, qstr,
34151a13f2e6SEdward Tomasz Napierala sizeof(qstr));
34165d913868SDoug Moore struct voistatdata_tdgstctd64 *parent;
34175d913868SDoug Moore parent = RB_PARENT(rbctd64, rblnk);
34185d913868SDoug Moore int rb_color =
34195d913868SDoug Moore parent == NULL ? 0 :
34205d913868SDoug Moore RB_LEFT(parent, rblnk) == rbctd64 ?
3421d0354fa7SDoug Moore (_RB_BITSUP(parent, rblnk) & _RB_L) != 0 :
3422d0354fa7SDoug Moore (_RB_BITSUP(parent, rblnk) & _RB_R) != 0;
34231a13f2e6SEdward Tomasz Napierala printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
34241a13f2e6SEdward Tomasz Napierala "mu=%s\n",
34251a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, rbctd64),
34261a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree,
34271a13f2e6SEdward Tomasz Napierala RB_PARENT(rbctd64, rblnk)),
34281a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree,
34291a13f2e6SEdward Tomasz Napierala RB_LEFT(rbctd64, rblnk)),
34301a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree,
34311a13f2e6SEdward Tomasz Napierala RB_RIGHT(rbctd64, rblnk)),
34325d913868SDoug Moore rb_color,
34331a13f2e6SEdward Tomasz Napierala qstr);
34341a13f2e6SEdward Tomasz Napierala
34351a13f2e6SEdward Tomasz Napierala panic("RB@%p and ARB@%p trees differ\n",
34361a13f2e6SEdward Tomasz Napierala rbctdtree, ctd64tree);
34371a13f2e6SEdward Tomasz Napierala }
34381a13f2e6SEdward Tomasz Napierala i++;
34391a13f2e6SEdward Tomasz Napierala }
34401a13f2e6SEdward Tomasz Napierala #endif /* DIAGNOSTIC */
34411a13f2e6SEdward Tomasz Napierala }
34421a13f2e6SEdward Tomasz Napierala
34431a13f2e6SEdward Tomasz Napierala return (error);
34441a13f2e6SEdward Tomasz Napierala }
34451a13f2e6SEdward Tomasz Napierala
34461a13f2e6SEdward Tomasz Napierala static inline int
34471a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival,
34481a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct voistatdata_tdgst *tdgst)
34491a13f2e6SEdward Tomasz Napierala {
34501a13f2e6SEdward Tomasz Napierala s64q_t x;
34511a13f2e6SEdward Tomasz Napierala int error;
34521a13f2e6SEdward Tomasz Napierala
34531a13f2e6SEdward Tomasz Napierala error = 0;
34541a13f2e6SEdward Tomasz Napierala
34551a13f2e6SEdward Tomasz Napierala switch (vs->dtype) {
34561a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32:
34571a13f2e6SEdward Tomasz Napierala /* Use same precision as the user's centroids. */
34581a13f2e6SEdward Tomasz Napierala Q_INI(&x, 0, 0, Q_NFBITS(
34591a13f2e6SEdward Tomasz Napierala ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu));
34601a13f2e6SEdward Tomasz Napierala break;
34611a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64:
34621a13f2e6SEdward Tomasz Napierala /* Use same precision as the user's centroids. */
34631a13f2e6SEdward Tomasz Napierala Q_INI(&x, 0, 0, Q_NFBITS(
34641a13f2e6SEdward Tomasz Napierala ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu));
34651a13f2e6SEdward Tomasz Napierala break;
34661a13f2e6SEdward Tomasz Napierala default:
34671a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 ||
34681a13f2e6SEdward Tomasz Napierala vs->dtype == VSD_DTYPE_TDGSTCLUST64,
34691a13f2e6SEdward Tomasz Napierala ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>",
34701a13f2e6SEdward Tomasz Napierala __func__, vs->dtype));
34711a13f2e6SEdward Tomasz Napierala return (EINVAL);
34721a13f2e6SEdward Tomasz Napierala }
34731a13f2e6SEdward Tomasz Napierala
34741a13f2e6SEdward Tomasz Napierala /*
34751a13f2e6SEdward Tomasz Napierala * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid
34761a13f2e6SEdward Tomasz Napierala * returning EOVERFLOW if the voival would have fit in a u64q_t.
34771a13f2e6SEdward Tomasz Napierala */
34781a13f2e6SEdward Tomasz Napierala switch (voi_dtype) {
34791a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
34801a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int32.s32);
34811a13f2e6SEdward Tomasz Napierala break;
34821a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
34831a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int32.u32);
34841a13f2e6SEdward Tomasz Napierala break;
34851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
34861a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int64.s64);
34871a13f2e6SEdward Tomasz Napierala break;
34881a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
34891a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int64.u64);
34901a13f2e6SEdward Tomasz Napierala break;
34911a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
34921a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->intlong.slong);
34931a13f2e6SEdward Tomasz Napierala break;
34941a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
34951a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->intlong.ulong);
34961a13f2e6SEdward Tomasz Napierala break;
34971a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
34981a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q32.sq32);
34991a13f2e6SEdward Tomasz Napierala break;
35001a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
35011a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q32.uq32);
35021a13f2e6SEdward Tomasz Napierala break;
35031a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
35041a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q64.sq64);
35051a13f2e6SEdward Tomasz Napierala break;
35061a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
35071a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q64.uq64);
35081a13f2e6SEdward Tomasz Napierala break;
35091a13f2e6SEdward Tomasz Napierala default:
35101a13f2e6SEdward Tomasz Napierala error = EINVAL;
35111a13f2e6SEdward Tomasz Napierala break;
35121a13f2e6SEdward Tomasz Napierala }
35131a13f2e6SEdward Tomasz Napierala
35141a13f2e6SEdward Tomasz Napierala if (error ||
35151a13f2e6SEdward Tomasz Napierala (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1)))
35161a13f2e6SEdward Tomasz Napierala return (error);
35171a13f2e6SEdward Tomasz Napierala
35181a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID;
35191a13f2e6SEdward Tomasz Napierala return (0);
35201a13f2e6SEdward Tomasz Napierala }
35211a13f2e6SEdward Tomasz Napierala
35221a13f2e6SEdward Tomasz Napierala int
35231a13f2e6SEdward Tomasz Napierala stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id,
35241a13f2e6SEdward Tomasz Napierala enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags)
35251a13f2e6SEdward Tomasz Napierala {
35261a13f2e6SEdward Tomasz Napierala struct voi *v;
35271a13f2e6SEdward Tomasz Napierala struct voistat *vs;
35281a13f2e6SEdward Tomasz Napierala void *statevsd, *vsd;
35291a13f2e6SEdward Tomasz Napierala int error, i, tmperr;
35301a13f2e6SEdward Tomasz Napierala
35311a13f2e6SEdward Tomasz Napierala error = 0;
35321a13f2e6SEdward Tomasz Napierala
35331a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) ||
35341a13f2e6SEdward Tomasz Napierala voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL)
35351a13f2e6SEdward Tomasz Napierala return (EINVAL);
35361a13f2e6SEdward Tomasz Napierala v = &sb->vois[voi_id];
35371a13f2e6SEdward Tomasz Napierala if (voi_dtype != v->dtype || v->id < 0 ||
35381a13f2e6SEdward Tomasz Napierala ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE)))
35391a13f2e6SEdward Tomasz Napierala return (EINVAL);
35401a13f2e6SEdward Tomasz Napierala
35411a13f2e6SEdward Tomasz Napierala vs = BLOB_OFFSET(sb, v->stats_off);
35421a13f2e6SEdward Tomasz Napierala if (v->flags & VOI_REQSTATE)
35431a13f2e6SEdward Tomasz Napierala statevsd = BLOB_OFFSET(sb, vs->data_off);
35441a13f2e6SEdward Tomasz Napierala else
35451a13f2e6SEdward Tomasz Napierala statevsd = NULL;
35461a13f2e6SEdward Tomasz Napierala
35471a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) {
35481a13f2e6SEdward Tomasz Napierala switch (voi_dtype) {
35491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
35501a13f2e6SEdward Tomasz Napierala voival->int32.s32 +=
35511a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.s32;
35521a13f2e6SEdward Tomasz Napierala break;
35531a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
35541a13f2e6SEdward Tomasz Napierala voival->int32.u32 +=
35551a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.u32;
35561a13f2e6SEdward Tomasz Napierala break;
35571a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
35581a13f2e6SEdward Tomasz Napierala voival->int64.s64 +=
35591a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.s64;
35601a13f2e6SEdward Tomasz Napierala break;
35611a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
35621a13f2e6SEdward Tomasz Napierala voival->int64.u64 +=
35631a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.u64;
35641a13f2e6SEdward Tomasz Napierala break;
35651a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
35661a13f2e6SEdward Tomasz Napierala voival->intlong.slong +=
35671a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.slong;
35681a13f2e6SEdward Tomasz Napierala break;
35691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
35701a13f2e6SEdward Tomasz Napierala voival->intlong.ulong +=
35711a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.ulong;
35721a13f2e6SEdward Tomasz Napierala break;
35731a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
35741a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q32.sq32,
35751a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q32.sq32);
35761a13f2e6SEdward Tomasz Napierala break;
35771a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
35781a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q32.uq32,
35791a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q32.uq32);
35801a13f2e6SEdward Tomasz Napierala break;
35811a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
35821a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q64.sq64,
35831a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q64.sq64);
35841a13f2e6SEdward Tomasz Napierala break;
35851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
35861a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q64.uq64,
35871a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q64.uq64);
35881a13f2e6SEdward Tomasz Napierala break;
35891a13f2e6SEdward Tomasz Napierala default:
35901a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
35911a13f2e6SEdward Tomasz Napierala break;
35921a13f2e6SEdward Tomasz Napierala }
35931a13f2e6SEdward Tomasz Napierala }
35941a13f2e6SEdward Tomasz Napierala
35951a13f2e6SEdward Tomasz Napierala if (error)
35961a13f2e6SEdward Tomasz Napierala return (error);
35971a13f2e6SEdward Tomasz Napierala
35981a13f2e6SEdward Tomasz Napierala for (i = v->voistatmaxid; i > 0; i--) {
35991a13f2e6SEdward Tomasz Napierala vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i];
36001a13f2e6SEdward Tomasz Napierala if (vs->stype < 0)
36011a13f2e6SEdward Tomasz Napierala continue;
36021a13f2e6SEdward Tomasz Napierala
36031a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off);
36041a13f2e6SEdward Tomasz Napierala
36051a13f2e6SEdward Tomasz Napierala switch (vs->stype) {
36061a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX:
36071a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_max(voi_dtype, voival,
36081a13f2e6SEdward Tomasz Napierala vs, vsd);
36091a13f2e6SEdward Tomasz Napierala break;
36101a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN:
36111a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_min(voi_dtype, voival,
36121a13f2e6SEdward Tomasz Napierala vs, vsd);
36131a13f2e6SEdward Tomasz Napierala break;
36141a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM:
36151a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_sum(voi_dtype, voival,
36161a13f2e6SEdward Tomasz Napierala vs, vsd);
36171a13f2e6SEdward Tomasz Napierala break;
36181a13f2e6SEdward Tomasz Napierala case VS_STYPE_HIST:
36191a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_hist(voi_dtype, voival,
36201a13f2e6SEdward Tomasz Napierala vs, vsd);
36211a13f2e6SEdward Tomasz Napierala break;
36221a13f2e6SEdward Tomasz Napierala case VS_STYPE_TDGST:
36231a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival,
36241a13f2e6SEdward Tomasz Napierala vs, vsd);
36251a13f2e6SEdward Tomasz Napierala break;
36261a13f2e6SEdward Tomasz Napierala default:
36271a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
36281a13f2e6SEdward Tomasz Napierala break;
36291a13f2e6SEdward Tomasz Napierala }
36301a13f2e6SEdward Tomasz Napierala
36311a13f2e6SEdward Tomasz Napierala if (tmperr) {
36321a13f2e6SEdward Tomasz Napierala error = tmperr;
36331a13f2e6SEdward Tomasz Napierala VS_INCERRS(vs);
36341a13f2e6SEdward Tomasz Napierala }
36351a13f2e6SEdward Tomasz Napierala }
36361a13f2e6SEdward Tomasz Napierala
36371a13f2e6SEdward Tomasz Napierala if (statevsd) {
36381a13f2e6SEdward Tomasz Napierala switch (voi_dtype) {
36391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32:
36401a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.s32 =
36411a13f2e6SEdward Tomasz Napierala voival->int32.s32;
36421a13f2e6SEdward Tomasz Napierala break;
36431a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32:
36441a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.u32 =
36451a13f2e6SEdward Tomasz Napierala voival->int32.u32;
36461a13f2e6SEdward Tomasz Napierala break;
36471a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64:
36481a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.s64 =
36491a13f2e6SEdward Tomasz Napierala voival->int64.s64;
36501a13f2e6SEdward Tomasz Napierala break;
36511a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64:
36521a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.u64 =
36531a13f2e6SEdward Tomasz Napierala voival->int64.u64;
36541a13f2e6SEdward Tomasz Napierala break;
36551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG:
36561a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.slong =
36571a13f2e6SEdward Tomasz Napierala voival->intlong.slong;
36581a13f2e6SEdward Tomasz Napierala break;
36591a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG:
36601a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.ulong =
36611a13f2e6SEdward Tomasz Napierala voival->intlong.ulong;
36621a13f2e6SEdward Tomasz Napierala break;
36631a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32:
36641a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(
36651a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q32.sq32,
36661a13f2e6SEdward Tomasz Napierala voival->q32.sq32);
36671a13f2e6SEdward Tomasz Napierala break;
36681a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32:
36691a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(
36701a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q32.uq32,
36711a13f2e6SEdward Tomasz Napierala voival->q32.uq32);
36721a13f2e6SEdward Tomasz Napierala break;
36731a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64:
36741a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(
36751a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q64.sq64,
36761a13f2e6SEdward Tomasz Napierala voival->q64.sq64);
36771a13f2e6SEdward Tomasz Napierala break;
36781a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64:
36791a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(
36801a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q64.uq64,
36811a13f2e6SEdward Tomasz Napierala voival->q64.uq64);
36821a13f2e6SEdward Tomasz Napierala break;
36831a13f2e6SEdward Tomasz Napierala default:
36841a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
36851a13f2e6SEdward Tomasz Napierala break;
36861a13f2e6SEdward Tomasz Napierala }
36871a13f2e6SEdward Tomasz Napierala }
36881a13f2e6SEdward Tomasz Napierala
36891a13f2e6SEdward Tomasz Napierala return (error);
36901a13f2e6SEdward Tomasz Napierala }
36911a13f2e6SEdward Tomasz Napierala
36921a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL
36931a13f2e6SEdward Tomasz Napierala
36941a13f2e6SEdward Tomasz Napierala static void
36951a13f2e6SEdward Tomasz Napierala stats_init(void *arg)
36961a13f2e6SEdward Tomasz Napierala {
36971a13f2e6SEdward Tomasz Napierala
36981a13f2e6SEdward Tomasz Napierala }
36991a13f2e6SEdward Tomasz Napierala SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL);
37001a13f2e6SEdward Tomasz Napierala
37011a13f2e6SEdward Tomasz Napierala /*
37021a13f2e6SEdward Tomasz Napierala * Sysctl handler to display the list of available stats templates.
37031a13f2e6SEdward Tomasz Napierala */
37041a13f2e6SEdward Tomasz Napierala static int
37051a13f2e6SEdward Tomasz Napierala stats_tpl_list_available(SYSCTL_HANDLER_ARGS)
37061a13f2e6SEdward Tomasz Napierala {
37071a13f2e6SEdward Tomasz Napierala struct sbuf *s;
37081a13f2e6SEdward Tomasz Napierala int err, i;
37091a13f2e6SEdward Tomasz Napierala
37101a13f2e6SEdward Tomasz Napierala err = 0;
37111a13f2e6SEdward Tomasz Napierala
37121a13f2e6SEdward Tomasz Napierala /* We can tolerate ntpl being stale, so do not take the lock. */
37131a13f2e6SEdward Tomasz Napierala s = sbuf_new(NULL, NULL, /* +1 per tpl for , */
37141a13f2e6SEdward Tomasz Napierala ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN);
37151a13f2e6SEdward Tomasz Napierala if (s == NULL)
37161a13f2e6SEdward Tomasz Napierala return (ENOMEM);
37171a13f2e6SEdward Tomasz Napierala
37181a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK();
37191a13f2e6SEdward Tomasz Napierala for (i = 0; i < ntpl; i++) {
37201a13f2e6SEdward Tomasz Napierala err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "",
37211a13f2e6SEdward Tomasz Napierala tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash);
37221a13f2e6SEdward Tomasz Napierala if (err) {
37231a13f2e6SEdward Tomasz Napierala /* Sbuf overflow condition. */
37241a13f2e6SEdward Tomasz Napierala err = EOVERFLOW;
37251a13f2e6SEdward Tomasz Napierala break;
37261a13f2e6SEdward Tomasz Napierala }
37271a13f2e6SEdward Tomasz Napierala }
37281a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK();
37291a13f2e6SEdward Tomasz Napierala
37301a13f2e6SEdward Tomasz Napierala if (!err) {
37311a13f2e6SEdward Tomasz Napierala sbuf_finish(s);
37321a13f2e6SEdward Tomasz Napierala err = sysctl_handle_string(oidp, sbuf_data(s), 0, req);
37331a13f2e6SEdward Tomasz Napierala }
37341a13f2e6SEdward Tomasz Napierala
37351a13f2e6SEdward Tomasz Napierala sbuf_delete(s);
37361a13f2e6SEdward Tomasz Napierala return (err);
37371a13f2e6SEdward Tomasz Napierala }
37381a13f2e6SEdward Tomasz Napierala
37391a13f2e6SEdward Tomasz Napierala /*
37401a13f2e6SEdward Tomasz Napierala * Called by subsystem-specific sysctls to report and/or parse the list of
37411a13f2e6SEdward Tomasz Napierala * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t
37421a13f2e6SEdward Tomasz Napierala * conformant function pointer must be passed in as arg1, which is used to
37431a13f2e6SEdward Tomasz Napierala * interact with the subsystem's stats template sample rates list. If arg2 > 0,
37441a13f2e6SEdward Tomasz Napierala * a zero-initialised allocation of arg2-sized contextual memory is
37451a13f2e6SEdward Tomasz Napierala * heap-allocated and passed in to all subsystem callbacks made during the
37461a13f2e6SEdward Tomasz Napierala * operation of stats_tpl_sample_rates().
37471a13f2e6SEdward Tomasz Napierala *
37481a13f2e6SEdward Tomasz Napierala * XXXLAS: Assumes templates are never removed, which is currently true but may
37491a13f2e6SEdward Tomasz Napierala * need to be reworked in future if dynamic template management becomes a
37501a13f2e6SEdward Tomasz Napierala * requirement e.g. to support kernel module based templates.
37511a13f2e6SEdward Tomasz Napierala */
37521a13f2e6SEdward Tomasz Napierala int
37531a13f2e6SEdward Tomasz Napierala stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS)
37541a13f2e6SEdward Tomasz Napierala {
37551a13f2e6SEdward Tomasz Napierala char kvpair_fmt[16], tplspec_fmt[16];
37561a13f2e6SEdward Tomasz Napierala char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN];
37571a13f2e6SEdward Tomasz Napierala char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */
37581a13f2e6SEdward Tomasz Napierala stats_tpl_sr_cb_t subsys_cb;
37591a13f2e6SEdward Tomasz Napierala void *subsys_ctx;
37601a13f2e6SEdward Tomasz Napierala char *buf, *new_rates_usr_str, *tpl_name_p;
37611a13f2e6SEdward Tomasz Napierala struct stats_tpl_sample_rate *rates;
37621a13f2e6SEdward Tomasz Napierala struct sbuf *s, _s;
37631a13f2e6SEdward Tomasz Napierala uint32_t cum_pct, pct, tpl_hash;
37641a13f2e6SEdward Tomasz Napierala int err, i, off, len, newlen, nrates;
37651a13f2e6SEdward Tomasz Napierala
37661a13f2e6SEdward Tomasz Napierala buf = NULL;
37671a13f2e6SEdward Tomasz Napierala rates = NULL;
37681a13f2e6SEdward Tomasz Napierala err = nrates = 0;
37691a13f2e6SEdward Tomasz Napierala subsys_cb = (stats_tpl_sr_cb_t)arg1;
37701a13f2e6SEdward Tomasz Napierala KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__));
37711a13f2e6SEdward Tomasz Napierala if (arg2 > 0)
37721a13f2e6SEdward Tomasz Napierala subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO);
37731a13f2e6SEdward Tomasz Napierala else
37741a13f2e6SEdward Tomasz Napierala subsys_ctx = NULL;
37751a13f2e6SEdward Tomasz Napierala
37761a13f2e6SEdward Tomasz Napierala /* Grab current count of subsystem rates. */
37771a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx);
37781a13f2e6SEdward Tomasz Napierala if (err)
37791a13f2e6SEdward Tomasz Napierala goto done;
37801a13f2e6SEdward Tomasz Napierala
37811a13f2e6SEdward Tomasz Napierala /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */
37821a13f2e6SEdward Tomasz Napierala len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5));
37831a13f2e6SEdward Tomasz Napierala
37841a13f2e6SEdward Tomasz Napierala if (req->oldptr != NULL || req->newptr != NULL)
37851a13f2e6SEdward Tomasz Napierala buf = malloc(len, M_TEMP, M_WAITOK);
37861a13f2e6SEdward Tomasz Napierala
37871a13f2e6SEdward Tomasz Napierala if (req->oldptr != NULL) {
37881a13f2e6SEdward Tomasz Napierala if (nrates == 0) {
37891a13f2e6SEdward Tomasz Napierala /* No rates, so return an empty string via oldptr. */
37901a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, "", 1);
37911a13f2e6SEdward Tomasz Napierala if (err)
37921a13f2e6SEdward Tomasz Napierala goto done;
37931a13f2e6SEdward Tomasz Napierala goto process_new;
37941a13f2e6SEdward Tomasz Napierala }
37951a13f2e6SEdward Tomasz Napierala
37961a13f2e6SEdward Tomasz Napierala s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL);
37971a13f2e6SEdward Tomasz Napierala
37981a13f2e6SEdward Tomasz Napierala /* Grab locked count of, and ptr to, subsystem rates. */
37991a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates,
38001a13f2e6SEdward Tomasz Napierala subsys_ctx);
38011a13f2e6SEdward Tomasz Napierala if (err)
38021a13f2e6SEdward Tomasz Napierala goto done;
38031a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK();
38041a13f2e6SEdward Tomasz Napierala for (i = 0; i < nrates && !err; i++) {
38051a13f2e6SEdward Tomasz Napierala err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "",
38061a13f2e6SEdward Tomasz Napierala tpllist[rates[i].tpl_slot_id]->mb->tplname,
38071a13f2e6SEdward Tomasz Napierala tpllist[rates[i].tpl_slot_id]->mb->tplhash,
38081a13f2e6SEdward Tomasz Napierala rates[i].tpl_sample_pct);
38091a13f2e6SEdward Tomasz Napierala }
38101a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK();
38111a13f2e6SEdward Tomasz Napierala /* Tell subsystem that we're done with its rates list. */
38121a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx);
38131a13f2e6SEdward Tomasz Napierala if (err)
38141a13f2e6SEdward Tomasz Napierala goto done;
38151a13f2e6SEdward Tomasz Napierala
38161a13f2e6SEdward Tomasz Napierala err = sbuf_finish(s);
38171a13f2e6SEdward Tomasz Napierala if (err)
38181a13f2e6SEdward Tomasz Napierala goto done; /* We lost a race for buf to be too small. */
38191a13f2e6SEdward Tomasz Napierala
38201a13f2e6SEdward Tomasz Napierala /* Return the rendered string data via oldptr. */
38211a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s));
38221a13f2e6SEdward Tomasz Napierala } else {
38231a13f2e6SEdward Tomasz Napierala /* Return the upper bound size for buffer sizing requests. */
38241a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, NULL, len);
38251a13f2e6SEdward Tomasz Napierala }
38261a13f2e6SEdward Tomasz Napierala
38271a13f2e6SEdward Tomasz Napierala process_new:
38281a13f2e6SEdward Tomasz Napierala if (err || req->newptr == NULL)
38291a13f2e6SEdward Tomasz Napierala goto done;
38301a13f2e6SEdward Tomasz Napierala
38311a13f2e6SEdward Tomasz Napierala newlen = req->newlen - req->newidx;
38321a13f2e6SEdward Tomasz Napierala err = SYSCTL_IN(req, buf, newlen);
38331a13f2e6SEdward Tomasz Napierala if (err)
38341a13f2e6SEdward Tomasz Napierala goto done;
38351a13f2e6SEdward Tomasz Napierala
38361a13f2e6SEdward Tomasz Napierala /*
38371a13f2e6SEdward Tomasz Napierala * Initialise format strings at run time.
38381a13f2e6SEdward Tomasz Napierala *
38391a13f2e6SEdward Tomasz Napierala * Write the max template spec string length into the
38401a13f2e6SEdward Tomasz Napierala * template_spec=percent key-value pair parsing format string as:
38411a13f2e6SEdward Tomasz Napierala * " %<width>[^=]=%u %n"
38421a13f2e6SEdward Tomasz Napierala *
38431a13f2e6SEdward Tomasz Napierala * Write the max template name string length into the tplname:tplhash
38441a13f2e6SEdward Tomasz Napierala * parsing format string as:
38451a13f2e6SEdward Tomasz Napierala * "%<width>[^:]:%u"
38461a13f2e6SEdward Tomasz Napierala *
38471a13f2e6SEdward Tomasz Napierala * Subtract 1 for \0 appended by sscanf().
38481a13f2e6SEdward Tomasz Napierala */
38491a13f2e6SEdward Tomasz Napierala sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1);
38501a13f2e6SEdward Tomasz Napierala sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1);
38511a13f2e6SEdward Tomasz Napierala
38521a13f2e6SEdward Tomasz Napierala /*
38531a13f2e6SEdward Tomasz Napierala * Parse each CSV key-value pair specifying a template and its sample
38541a13f2e6SEdward Tomasz Napierala * percentage. Whitespace either side of a key-value pair is ignored.
38551a13f2e6SEdward Tomasz Napierala * Templates can be specified by name, hash, or name and hash per the
38561a13f2e6SEdward Tomasz Napierala * following formats (chars in [] are optional):
38571a13f2e6SEdward Tomasz Napierala * ["]<tplname>["]=<percent>
38581a13f2e6SEdward Tomasz Napierala * :hash=pct
38591a13f2e6SEdward Tomasz Napierala * ["]<tplname>["]:hash=<percent>
38601a13f2e6SEdward Tomasz Napierala */
38611a13f2e6SEdward Tomasz Napierala cum_pct = nrates = 0;
38621a13f2e6SEdward Tomasz Napierala rates = NULL;
38631a13f2e6SEdward Tomasz Napierala buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */
38641a13f2e6SEdward Tomasz Napierala new_rates_usr_str = buf;
38651a13f2e6SEdward Tomasz Napierala while (isspace(*new_rates_usr_str))
38661a13f2e6SEdward Tomasz Napierala new_rates_usr_str++; /* Skip leading whitespace. */
38671a13f2e6SEdward Tomasz Napierala while (*new_rates_usr_str != '\0') {
38681a13f2e6SEdward Tomasz Napierala tpl_name_p = tpl_name;
38691a13f2e6SEdward Tomasz Napierala tpl_name[0] = '\0';
38701a13f2e6SEdward Tomasz Napierala tpl_hash = 0;
38711a13f2e6SEdward Tomasz Napierala off = 0;
38721a13f2e6SEdward Tomasz Napierala
38731a13f2e6SEdward Tomasz Napierala /*
38741a13f2e6SEdward Tomasz Napierala * Parse key-value pair which must perform 2 conversions, then
38751a13f2e6SEdward Tomasz Napierala * parse the template spec to extract either name, hash, or name
38761a13f2e6SEdward Tomasz Napierala * and hash depending on the three possible spec formats. The
38771a13f2e6SEdward Tomasz Napierala * tplspec_fmt format specifier parses name or name and hash
38781a13f2e6SEdward Tomasz Napierala * template specs, while the ":%u" format specifier parses
38791a13f2e6SEdward Tomasz Napierala * hash-only template specs. If parsing is successfull, ensure
38801a13f2e6SEdward Tomasz Napierala * the cumulative sampling percentage does not exceed 100.
38811a13f2e6SEdward Tomasz Napierala */
38821a13f2e6SEdward Tomasz Napierala err = EINVAL;
38831a13f2e6SEdward Tomasz Napierala if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct,
38841a13f2e6SEdward Tomasz Napierala &off))
38851a13f2e6SEdward Tomasz Napierala break;
38861a13f2e6SEdward Tomasz Napierala if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) &&
38871a13f2e6SEdward Tomasz Napierala (1 != sscanf(tpl_spec, ":%u", &tpl_hash)))
38881a13f2e6SEdward Tomasz Napierala break;
38891a13f2e6SEdward Tomasz Napierala if ((cum_pct += pct) > 100)
38901a13f2e6SEdward Tomasz Napierala break;
38911a13f2e6SEdward Tomasz Napierala err = 0;
38921a13f2e6SEdward Tomasz Napierala
38931a13f2e6SEdward Tomasz Napierala /* Strip surrounding "" from template name if present. */
38941a13f2e6SEdward Tomasz Napierala len = strlen(tpl_name);
38951a13f2e6SEdward Tomasz Napierala if (len > 0) {
38961a13f2e6SEdward Tomasz Napierala if (tpl_name[len - 1] == '"')
38971a13f2e6SEdward Tomasz Napierala tpl_name[--len] = '\0';
38981a13f2e6SEdward Tomasz Napierala if (tpl_name[0] == '"') {
38991a13f2e6SEdward Tomasz Napierala tpl_name_p++;
39001a13f2e6SEdward Tomasz Napierala len--;
39011a13f2e6SEdward Tomasz Napierala }
39021a13f2e6SEdward Tomasz Napierala }
39031a13f2e6SEdward Tomasz Napierala
39041a13f2e6SEdward Tomasz Napierala rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */
39051a13f2e6SEdward Tomasz Napierala (nrates + 1) * sizeof(*rates), M_WAITOK);
39061a13f2e6SEdward Tomasz Napierala rates[nrates].tpl_slot_id =
39071a13f2e6SEdward Tomasz Napierala stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash);
39081a13f2e6SEdward Tomasz Napierala if (rates[nrates].tpl_slot_id < 0) {
39091a13f2e6SEdward Tomasz Napierala err = -rates[nrates].tpl_slot_id;
39101a13f2e6SEdward Tomasz Napierala break;
39111a13f2e6SEdward Tomasz Napierala }
39121a13f2e6SEdward Tomasz Napierala rates[nrates].tpl_sample_pct = pct;
39131a13f2e6SEdward Tomasz Napierala nrates++;
39141a13f2e6SEdward Tomasz Napierala new_rates_usr_str += off;
39151a13f2e6SEdward Tomasz Napierala if (*new_rates_usr_str != ',')
39161a13f2e6SEdward Tomasz Napierala break; /* End-of-input or malformed. */
39171a13f2e6SEdward Tomasz Napierala new_rates_usr_str++; /* Move past comma to next pair. */
39181a13f2e6SEdward Tomasz Napierala }
39191a13f2e6SEdward Tomasz Napierala
39201a13f2e6SEdward Tomasz Napierala if (!err) {
39211a13f2e6SEdward Tomasz Napierala if ((new_rates_usr_str - buf) < newlen) {
39221a13f2e6SEdward Tomasz Napierala /* Entire input has not been consumed. */
39231a13f2e6SEdward Tomasz Napierala err = EINVAL;
39241a13f2e6SEdward Tomasz Napierala } else {
39251a13f2e6SEdward Tomasz Napierala /*
39261a13f2e6SEdward Tomasz Napierala * Give subsystem the new rates. They'll return the
39271a13f2e6SEdward Tomasz Napierala * appropriate rates pointer for us to garbage collect.
39281a13f2e6SEdward Tomasz Napierala */
39291a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_PUT, &rates, &nrates,
39301a13f2e6SEdward Tomasz Napierala subsys_ctx);
39311a13f2e6SEdward Tomasz Napierala }
39321a13f2e6SEdward Tomasz Napierala }
39331a13f2e6SEdward Tomasz Napierala stats_free(rates);
39341a13f2e6SEdward Tomasz Napierala
39351a13f2e6SEdward Tomasz Napierala done:
39361a13f2e6SEdward Tomasz Napierala free(buf, M_TEMP);
39371a13f2e6SEdward Tomasz Napierala free(subsys_ctx, M_TEMP);
39381a13f2e6SEdward Tomasz Napierala return (err);
39391a13f2e6SEdward Tomasz Napierala }
39401a13f2e6SEdward Tomasz Napierala
39417029da5cSPawel Biernacki SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
39421a13f2e6SEdward Tomasz Napierala "stats(9) MIB");
39431a13f2e6SEdward Tomasz Napierala
39447029da5cSPawel Biernacki SYSCTL_PROC(_kern_stats, OID_AUTO, templates,
39457029da5cSPawel Biernacki CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
39467029da5cSPawel Biernacki stats_tpl_list_available, "A",
39471a13f2e6SEdward Tomasz Napierala "list the name/hash of all available stats(9) templates");
39481a13f2e6SEdward Tomasz Napierala
39491a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */
39501a13f2e6SEdward Tomasz Napierala
39511a13f2e6SEdward Tomasz Napierala static void __attribute__ ((constructor))
39521a13f2e6SEdward Tomasz Napierala stats_constructor(void)
39531a13f2e6SEdward Tomasz Napierala {
39541a13f2e6SEdward Tomasz Napierala
39551a13f2e6SEdward Tomasz Napierala pthread_rwlock_init(&tpllistlock, NULL);
39561a13f2e6SEdward Tomasz Napierala }
39571a13f2e6SEdward Tomasz Napierala
39581a13f2e6SEdward Tomasz Napierala static void __attribute__ ((destructor))
39591a13f2e6SEdward Tomasz Napierala stats_destructor(void)
39601a13f2e6SEdward Tomasz Napierala {
39611a13f2e6SEdward Tomasz Napierala
39621a13f2e6SEdward Tomasz Napierala pthread_rwlock_destroy(&tpllistlock);
39631a13f2e6SEdward Tomasz Napierala }
39641a13f2e6SEdward Tomasz Napierala
39651a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */
3966