11a13f2e6SEdward Tomasz Napierala /*- 21a13f2e6SEdward Tomasz Napierala * Copyright (c) 2014-2018 Netflix, Inc. 31a13f2e6SEdward Tomasz Napierala * All rights reserved. 41a13f2e6SEdward Tomasz Napierala * 51a13f2e6SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 61a13f2e6SEdward Tomasz Napierala * modification, are permitted provided that the following conditions 71a13f2e6SEdward Tomasz Napierala * are met: 81a13f2e6SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 91a13f2e6SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 101a13f2e6SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 111a13f2e6SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 121a13f2e6SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 131a13f2e6SEdward Tomasz Napierala * 141a13f2e6SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 151a13f2e6SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 161a13f2e6SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 171a13f2e6SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 181a13f2e6SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 191a13f2e6SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 201a13f2e6SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 211a13f2e6SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 221a13f2e6SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 231a13f2e6SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 241a13f2e6SEdward Tomasz Napierala * SUCH DAMAGE. 251a13f2e6SEdward Tomasz Napierala * 261a13f2e6SEdward Tomasz Napierala * $FreeBSD$ 271a13f2e6SEdward Tomasz Napierala */ 281a13f2e6SEdward Tomasz Napierala 291a13f2e6SEdward Tomasz Napierala /* 301a13f2e6SEdward Tomasz Napierala * Author: Lawrence Stewart <lstewart@netflix.com> 311a13f2e6SEdward Tomasz Napierala */ 321a13f2e6SEdward Tomasz Napierala 331a13f2e6SEdward Tomasz Napierala #include <sys/cdefs.h> 341a13f2e6SEdward Tomasz Napierala __FBSDID("$FreeBSD$"); 351a13f2e6SEdward Tomasz Napierala 361a13f2e6SEdward Tomasz Napierala #include <sys/param.h> 371a13f2e6SEdward Tomasz Napierala #include <sys/arb.h> 381a13f2e6SEdward Tomasz Napierala #include <sys/ctype.h> 391a13f2e6SEdward Tomasz Napierala #include <sys/errno.h> 401a13f2e6SEdward Tomasz Napierala #include <sys/hash.h> 411a13f2e6SEdward Tomasz Napierala #include <sys/limits.h> 421a13f2e6SEdward Tomasz Napierala #include <sys/malloc.h> 431a13f2e6SEdward Tomasz Napierala #include <sys/qmath.h> 441a13f2e6SEdward Tomasz Napierala #include <sys/sbuf.h> 451a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 461a13f2e6SEdward Tomasz Napierala #include <sys/tree.h> 471a13f2e6SEdward Tomasz Napierala #endif 481a13f2e6SEdward Tomasz Napierala #include <sys/stats.h> /* Must come after qmath.h and arb.h */ 491a13f2e6SEdward Tomasz Napierala #include <sys/stddef.h> 501a13f2e6SEdward Tomasz Napierala #include <sys/stdint.h> 511a13f2e6SEdward Tomasz Napierala #include <sys/time.h> 521a13f2e6SEdward Tomasz Napierala 531a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 541a13f2e6SEdward Tomasz Napierala #include <sys/kernel.h> 551a13f2e6SEdward Tomasz Napierala #include <sys/lock.h> 561a13f2e6SEdward Tomasz Napierala #include <sys/rwlock.h> 571a13f2e6SEdward Tomasz Napierala #include <sys/sysctl.h> 581a13f2e6SEdward Tomasz Napierala #include <sys/systm.h> 591a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 601a13f2e6SEdward Tomasz Napierala #include <pthread.h> 611a13f2e6SEdward Tomasz Napierala #include <stdbool.h> 621a13f2e6SEdward Tomasz Napierala #include <stdio.h> 631a13f2e6SEdward Tomasz Napierala #include <stdlib.h> 641a13f2e6SEdward Tomasz Napierala #include <string.h> 651a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 661a13f2e6SEdward Tomasz Napierala 671a13f2e6SEdward Tomasz Napierala struct voistatdata_voistate { 681a13f2e6SEdward Tomasz Napierala /* Previous VOI value for diff calculation. */ 691a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric prev; 701a13f2e6SEdward Tomasz Napierala }; 711a13f2e6SEdward Tomasz Napierala 721a13f2e6SEdward Tomasz Napierala #define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */ 731a13f2e6SEdward Tomasz Napierala struct voistat { 741a13f2e6SEdward Tomasz Napierala int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */ 751a13f2e6SEdward Tomasz Napierala enum vsd_dtype dtype : 8; /* Data type of this stat's data. */ 761a13f2e6SEdward Tomasz Napierala uint16_t data_off; /* Blob offset for this stat's data. */ 771a13f2e6SEdward Tomasz Napierala uint16_t dsz; /* Size of stat's data. */ 781a13f2e6SEdward Tomasz Napierala #define VS_EBITS 8 791a13f2e6SEdward Tomasz Napierala uint16_t errs : VS_EBITS;/* Non-wrapping error count. */ 801a13f2e6SEdward Tomasz Napierala uint16_t flags : 16 - VS_EBITS; 811a13f2e6SEdward Tomasz Napierala }; 821a13f2e6SEdward Tomasz Napierala /* The voistat error count is capped to avoid wrapping. */ 831a13f2e6SEdward Tomasz Napierala #define VS_INCERRS(vs) do { \ 841a13f2e6SEdward Tomasz Napierala if ((vs)->errs < (1U << VS_EBITS) - 1) \ 851a13f2e6SEdward Tomasz Napierala (vs)->errs++; \ 861a13f2e6SEdward Tomasz Napierala } while (0) 871a13f2e6SEdward Tomasz Napierala 881a13f2e6SEdward Tomasz Napierala /* 891a13f2e6SEdward Tomasz Napierala * Ideas for flags: 901a13f2e6SEdward Tomasz Napierala * - Global or entity specific (global would imply use of counter(9)?) 911a13f2e6SEdward Tomasz Napierala * - Whether to reset stats on read or not 921a13f2e6SEdward Tomasz Napierala * - Signal an overflow? 931a13f2e6SEdward Tomasz Napierala * - Compressed voistat array 941a13f2e6SEdward Tomasz Napierala */ 951a13f2e6SEdward Tomasz Napierala #define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */ 961a13f2e6SEdward Tomasz Napierala struct voi { 971a13f2e6SEdward Tomasz Napierala int16_t id; /* VOI id. */ 981a13f2e6SEdward Tomasz Napierala enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */ 991a13f2e6SEdward Tomasz Napierala int8_t voistatmaxid; /* Largest allocated voistat index. */ 1001a13f2e6SEdward Tomasz Napierala uint16_t stats_off; /* Blob offset for this VOIs stats. */ 1011a13f2e6SEdward Tomasz Napierala uint16_t flags; 1021a13f2e6SEdward Tomasz Napierala }; 1031a13f2e6SEdward Tomasz Napierala 1041a13f2e6SEdward Tomasz Napierala /* 1051a13f2e6SEdward Tomasz Napierala * Memory for the entire blob is allocated as a slab and then offsets are 1061a13f2e6SEdward Tomasz Napierala * maintained to carve up the slab into sections holding different data types. 1071a13f2e6SEdward Tomasz Napierala * 1081a13f2e6SEdward Tomasz Napierala * Ideas for flags: 1091a13f2e6SEdward Tomasz Napierala * - Compressed voi array (trade off memory usage vs search time) 1101a13f2e6SEdward Tomasz Napierala * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib) 1111a13f2e6SEdward Tomasz Napierala */ 1121a13f2e6SEdward Tomasz Napierala struct statsblobv1 { 1131a13f2e6SEdward Tomasz Napierala uint8_t abi; 1141a13f2e6SEdward Tomasz Napierala uint8_t endian; 1151a13f2e6SEdward Tomasz Napierala uint16_t flags; 1161a13f2e6SEdward Tomasz Napierala uint16_t maxsz; 1171a13f2e6SEdward Tomasz Napierala uint16_t cursz; 1181a13f2e6SEdward Tomasz Napierala /* Fields from here down are opaque to consumers. */ 1191a13f2e6SEdward Tomasz Napierala uint32_t tplhash; /* Base template hash ID. */ 1201a13f2e6SEdward Tomasz Napierala uint16_t stats_off; /* voistat array blob offset. */ 1211a13f2e6SEdward Tomasz Napierala uint16_t statsdata_off; /* voistatdata array blob offset. */ 1221a13f2e6SEdward Tomasz Napierala sbintime_t created; /* Blob creation time. */ 1231a13f2e6SEdward Tomasz Napierala sbintime_t lastrst; /* Time of last reset. */ 1241a13f2e6SEdward Tomasz Napierala struct voi vois[]; /* Array indexed by [voi_id]. */ 1251a13f2e6SEdward Tomasz Napierala } __aligned(sizeof(void *)); 1261a13f2e6SEdward Tomasz Napierala _Static_assert(offsetof(struct statsblobv1, cursz) + 12789f34d46SBrooks Davis SIZEOF_MEMBER(struct statsblobv1, cursz) == 12889f34d46SBrooks Davis offsetof(struct statsblob, opaque), 1291a13f2e6SEdward Tomasz Napierala "statsblobv1 ABI mismatch"); 1301a13f2e6SEdward Tomasz Napierala 1311a13f2e6SEdward Tomasz Napierala struct statsblobv1_tpl { 1321a13f2e6SEdward Tomasz Napierala struct metablob *mb; 1331a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb; 1341a13f2e6SEdward Tomasz Napierala }; 1351a13f2e6SEdward Tomasz Napierala 1361a13f2e6SEdward Tomasz Napierala /* Context passed to iterator callbacks. */ 1371a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx { 1381a13f2e6SEdward Tomasz Napierala void *usrctx; /* Caller supplied context. */ 1391a13f2e6SEdward Tomasz Napierala uint32_t flags; /* Flags for current iteration. */ 1401a13f2e6SEdward Tomasz Napierala int16_t vslot; /* struct voi slot index. */ 1411a13f2e6SEdward Tomasz Napierala int8_t vsslot; /* struct voistat slot index. */ 1421a13f2e6SEdward Tomasz Napierala }; 1431a13f2e6SEdward Tomasz Napierala 1441a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx { 1451a13f2e6SEdward Tomasz Napierala struct sbuf *buf; 1461a13f2e6SEdward Tomasz Napierala struct statsblob_tpl *tpl; 1471a13f2e6SEdward Tomasz Napierala enum sb_str_fmt fmt; 1481a13f2e6SEdward Tomasz Napierala uint32_t flags; 1491a13f2e6SEdward Tomasz Napierala }; 1501a13f2e6SEdward Tomasz Napierala 1511a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx { 1521a13f2e6SEdward Tomasz Napierala stats_blob_visitcb_t cb; 1531a13f2e6SEdward Tomasz Napierala void *usrctx; 1541a13f2e6SEdward Tomasz Napierala }; 1551a13f2e6SEdward Tomasz Napierala 1561a13f2e6SEdward Tomasz Napierala /* Stats blob iterator callback. */ 1571a13f2e6SEdward Tomasz Napierala typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v, 1581a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx); 1591a13f2e6SEdward Tomasz Napierala 1601a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 1611a13f2e6SEdward Tomasz Napierala static struct rwlock tpllistlock; 1621a13f2e6SEdward Tomasz Napierala RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock"); 1631a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock) 1641a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock) 1651a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock) 1661a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock) 1671a13f2e6SEdward Tomasz Napierala #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED) 1681a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED) 1691a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED) 1701a13f2e6SEdward Tomasz Napierala MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory"); 1711a13f2e6SEdward Tomasz Napierala #define stats_free(ptr) free((ptr), M_STATS) 1721a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 1731a13f2e6SEdward Tomasz Napierala static void stats_constructor(void); 1741a13f2e6SEdward Tomasz Napierala static void stats_destructor(void); 1751a13f2e6SEdward Tomasz Napierala static pthread_rwlock_t tpllistlock; 1761a13f2e6SEdward Tomasz Napierala #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock) 1771a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock) 1781a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK() 1791a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock) 1801a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK() 1811a13f2e6SEdward Tomasz Napierala #define TPL_LIST_LOCK_ASSERT() do { } while (0) 1821a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK_ASSERT() do { } while (0) 1831a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK_ASSERT() do { } while (0) 1841a13f2e6SEdward Tomasz Napierala #ifdef NDEBUG 1851a13f2e6SEdward Tomasz Napierala #define KASSERT(cond, msg) do {} while (0) 1861a13f2e6SEdward Tomasz Napierala #define stats_abort() do {} while (0) 1871a13f2e6SEdward Tomasz Napierala #else /* ! NDEBUG */ 1881a13f2e6SEdward Tomasz Napierala #define KASSERT(cond, msg) do { \ 1891a13f2e6SEdward Tomasz Napierala if (!(cond)) { \ 1901a13f2e6SEdward Tomasz Napierala panic msg; \ 1911a13f2e6SEdward Tomasz Napierala } \ 1921a13f2e6SEdward Tomasz Napierala } while (0) 1931a13f2e6SEdward Tomasz Napierala #define stats_abort() abort() 1941a13f2e6SEdward Tomasz Napierala #endif /* NDEBUG */ 1951a13f2e6SEdward Tomasz Napierala #define stats_free(ptr) free(ptr) 1961a13f2e6SEdward Tomasz Napierala #define panic(fmt, ...) do { \ 1971a13f2e6SEdward Tomasz Napierala fprintf(stderr, (fmt), ##__VA_ARGS__); \ 1981a13f2e6SEdward Tomasz Napierala stats_abort(); \ 1991a13f2e6SEdward Tomasz Napierala } while (0) 2001a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 2011a13f2e6SEdward Tomasz Napierala 2021a13f2e6SEdward Tomasz Napierala #define SB_V1_MAXSZ 65535 2031a13f2e6SEdward Tomasz Napierala 2041a13f2e6SEdward Tomasz Napierala /* Obtain a blob offset pointer. */ 2051a13f2e6SEdward Tomasz Napierala #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off))) 2061a13f2e6SEdward Tomasz Napierala 2071a13f2e6SEdward Tomasz Napierala /* 2081a13f2e6SEdward Tomasz Napierala * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a 2091a13f2e6SEdward Tomasz Napierala * power of 2 size, we can shift instead of divide. The shift amount must be 2101a13f2e6SEdward Tomasz Napierala * updated if sizeof(struct voi) ever changes, which the assert should catch. 2111a13f2e6SEdward Tomasz Napierala */ 2121a13f2e6SEdward Tomasz Napierala #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \ 2131a13f2e6SEdward Tomasz Napierala sizeof(struct statsblobv1)) >> 3)) 2141a13f2e6SEdward Tomasz Napierala _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch"); 2151a13f2e6SEdward Tomasz Napierala 2161a13f2e6SEdward Tomasz Napierala /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */ 2171a13f2e6SEdward Tomasz Napierala const char *vs_stype2name[VS_NUM_STYPES] = { 2181a13f2e6SEdward Tomasz Napierala [VS_STYPE_VOISTATE] = "VOISTATE", 2191a13f2e6SEdward Tomasz Napierala [VS_STYPE_SUM] = "SUM", 2201a13f2e6SEdward Tomasz Napierala [VS_STYPE_MAX] = "MAX", 2211a13f2e6SEdward Tomasz Napierala [VS_STYPE_MIN] = "MIN", 2221a13f2e6SEdward Tomasz Napierala [VS_STYPE_HIST] = "HIST", 2231a13f2e6SEdward Tomasz Napierala [VS_STYPE_TDGST] = "TDGST", 2241a13f2e6SEdward Tomasz Napierala }; 2251a13f2e6SEdward Tomasz Napierala 2261a13f2e6SEdward Tomasz Napierala const char *vs_stype2desc[VS_NUM_STYPES] = { 2271a13f2e6SEdward Tomasz Napierala [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)", 2281a13f2e6SEdward Tomasz Napierala [VS_STYPE_SUM] = "Simple arithmetic accumulator", 2291a13f2e6SEdward Tomasz Napierala [VS_STYPE_MAX] = "Maximum observed VOI value", 2301a13f2e6SEdward Tomasz Napierala [VS_STYPE_MIN] = "Minimum observed VOI value", 2311a13f2e6SEdward Tomasz Napierala [VS_STYPE_HIST] = "Histogram of observed VOI values", 2321a13f2e6SEdward Tomasz Napierala [VS_STYPE_TDGST] = "t-digest of observed VOI values", 2331a13f2e6SEdward Tomasz Napierala }; 2341a13f2e6SEdward Tomasz Napierala 2351a13f2e6SEdward Tomasz Napierala const char *vsd_dtype2name[VSD_NUM_DTYPES] = { 2361a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = "VOISTATE", 2371a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = "INT_S32", 2381a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = "INT_U32", 2391a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = "INT_S64", 2401a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = "INT_U64", 2411a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = "INT_SLONG", 2421a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = "INT_ULONG", 2431a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = "Q_S32", 2441a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = "Q_U32", 2451a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = "Q_S64", 2461a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = "Q_U64", 2471a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = "CRHIST32", 2481a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = "DRHIST32", 2491a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = "DVHIST32", 2501a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = "CRHIST64", 2511a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = "DRHIST64", 2521a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = "DVHIST64", 2531a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32", 2541a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64", 2551a13f2e6SEdward Tomasz Napierala }; 2561a13f2e6SEdward Tomasz Napierala 2571a13f2e6SEdward Tomasz Napierala const size_t vsd_dtype2size[VSD_NUM_DTYPES] = { 2581a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate), 2591a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32), 2601a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32), 2611a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64), 2621a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64), 2631a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong), 2641a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong), 2651a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32), 2661a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32), 2671a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64), 2681a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64), 2691a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32), 2701a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32), 2711a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32), 2721a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64), 2731a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64), 2741a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64), 2751a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32), 2761a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64), 2771a13f2e6SEdward Tomasz Napierala }; 2781a13f2e6SEdward Tomasz Napierala 2791a13f2e6SEdward Tomasz Napierala static const bool vsd_compoundtype[VSD_NUM_DTYPES] = { 2801a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = true, 2811a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = false, 2821a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = false, 2831a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = false, 2841a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = false, 2851a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = false, 2861a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = false, 2871a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = false, 2881a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = false, 2891a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = false, 2901a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = false, 2911a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = true, 2921a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = true, 2931a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = true, 2941a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = true, 2951a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = true, 2961a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = true, 2971a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = true, 2981a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = true, 2991a13f2e6SEdward Tomasz Napierala }; 3001a13f2e6SEdward Tomasz Napierala 3011a13f2e6SEdward Tomasz Napierala const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = { 3021a13f2e6SEdward Tomasz Napierala [LIM_MIN] = { 3031a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = {0}, 3041a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}}, 3051a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}}, 3061a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}}, 3071a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}}, 3081a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}}, 3091a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}}, 3101a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}}, 3111a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}}, 3121a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}}, 3131a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}}, 3141a13f2e6SEdward Tomasz Napierala }, 3151a13f2e6SEdward Tomasz Napierala [LIM_MAX] = { 3161a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = {0}, 3171a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}}, 3181a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}}, 3191a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}}, 3201a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}}, 3211a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}}, 3221a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}}, 3231a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}}, 3241a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}}, 3251a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}}, 3261a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}}, 3271a13f2e6SEdward Tomasz Napierala } 3281a13f2e6SEdward Tomasz Napierala }; 3291a13f2e6SEdward Tomasz Napierala 3301a13f2e6SEdward Tomasz Napierala /* tpllistlock protects tpllist and ntpl */ 3311a13f2e6SEdward Tomasz Napierala static uint32_t ntpl; 3321a13f2e6SEdward Tomasz Napierala static struct statsblob_tpl **tpllist; 3331a13f2e6SEdward Tomasz Napierala 3341a13f2e6SEdward Tomasz Napierala static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz, 3351a13f2e6SEdward Tomasz Napierala int flags); 3361a13f2e6SEdward Tomasz Napierala //static void stats_v1_blob_finalise(struct statsblobv1 *sb); 3371a13f2e6SEdward Tomasz Napierala static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 3381a13f2e6SEdward Tomasz Napierala uint32_t flags); 3391a13f2e6SEdward Tomasz Napierala static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 3401a13f2e6SEdward Tomasz Napierala int newvoistatbytes, int newvoistatdatabytes); 3411a13f2e6SEdward Tomasz Napierala static void stats_v1_blob_iter(struct statsblobv1 *sb, 3421a13f2e6SEdward Tomasz Napierala stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags); 3431a13f2e6SEdward Tomasz Napierala static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, 3441a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt); 3451a13f2e6SEdward Tomasz Napierala 3461a13f2e6SEdward Tomasz Napierala static inline int 3471a13f2e6SEdward Tomasz Napierala ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2) 3481a13f2e6SEdward Tomasz Napierala { 3491a13f2e6SEdward Tomasz Napierala 3501a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(c1->mu, c2->mu), 3511a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 3521a13f2e6SEdward Tomasz Napierala Q_RELPREC(c1->mu, c2->mu))); 3531a13f2e6SEdward Tomasz Napierala 3541a13f2e6SEdward Tomasz Napierala return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 3551a13f2e6SEdward Tomasz Napierala } 3561a13f2e6SEdward Tomasz Napierala ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp); 3571a13f2e6SEdward Tomasz Napierala 3581a13f2e6SEdward Tomasz Napierala static inline int 3591a13f2e6SEdward Tomasz Napierala ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2) 3601a13f2e6SEdward Tomasz Napierala { 3611a13f2e6SEdward Tomasz Napierala 3621a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(c1->mu, c2->mu), 3631a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 3641a13f2e6SEdward Tomasz Napierala Q_RELPREC(c1->mu, c2->mu))); 3651a13f2e6SEdward Tomasz Napierala 3661a13f2e6SEdward Tomasz Napierala return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 3671a13f2e6SEdward Tomasz Napierala } 3681a13f2e6SEdward Tomasz Napierala ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp); 3691a13f2e6SEdward Tomasz Napierala 3701a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3711a13f2e6SEdward Tomasz Napierala RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp); 3721a13f2e6SEdward Tomasz Napierala RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp); 3731a13f2e6SEdward Tomasz Napierala #endif 3741a13f2e6SEdward Tomasz Napierala 3751a13f2e6SEdward Tomasz Napierala static inline sbintime_t 3761a13f2e6SEdward Tomasz Napierala stats_sbinuptime(void) 3771a13f2e6SEdward Tomasz Napierala { 3781a13f2e6SEdward Tomasz Napierala sbintime_t sbt; 3791a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 3801a13f2e6SEdward Tomasz Napierala 3811a13f2e6SEdward Tomasz Napierala sbt = sbinuptime(); 3821a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 3831a13f2e6SEdward Tomasz Napierala struct timespec tp; 3841a13f2e6SEdward Tomasz Napierala 3851a13f2e6SEdward Tomasz Napierala clock_gettime(CLOCK_MONOTONIC_FAST, &tp); 3861a13f2e6SEdward Tomasz Napierala sbt = tstosbt(tp); 3871a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 3881a13f2e6SEdward Tomasz Napierala 3891a13f2e6SEdward Tomasz Napierala return (sbt); 3901a13f2e6SEdward Tomasz Napierala } 3911a13f2e6SEdward Tomasz Napierala 3921a13f2e6SEdward Tomasz Napierala static inline void * 3931a13f2e6SEdward Tomasz Napierala stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags) 3941a13f2e6SEdward Tomasz Napierala { 3951a13f2e6SEdward Tomasz Napierala 3961a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 3971a13f2e6SEdward Tomasz Napierala /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */ 3981a13f2e6SEdward Tomasz Napierala if (!(flags & (M_WAITOK | M_NOWAIT))) 3991a13f2e6SEdward Tomasz Napierala flags |= M_NOWAIT; 4001a13f2e6SEdward Tomasz Napierala ptr = realloc(ptr, newsz, M_STATS, flags); 4011a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 4021a13f2e6SEdward Tomasz Napierala ptr = realloc(ptr, newsz); 4031a13f2e6SEdward Tomasz Napierala if ((flags & M_ZERO) && ptr != NULL) { 4041a13f2e6SEdward Tomasz Napierala if (oldsz == 0) 4051a13f2e6SEdward Tomasz Napierala memset(ptr, '\0', newsz); 4061a13f2e6SEdward Tomasz Napierala else if (newsz > oldsz) 4071a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz); 4081a13f2e6SEdward Tomasz Napierala } 4091a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 4101a13f2e6SEdward Tomasz Napierala 4111a13f2e6SEdward Tomasz Napierala return (ptr); 4121a13f2e6SEdward Tomasz Napierala } 4131a13f2e6SEdward Tomasz Napierala 4141a13f2e6SEdward Tomasz Napierala static inline char * 4151a13f2e6SEdward Tomasz Napierala stats_strdup(const char *s, 4161a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 4171a13f2e6SEdward Tomasz Napierala int flags) 4181a13f2e6SEdward Tomasz Napierala { 4191a13f2e6SEdward Tomasz Napierala char *copy; 4201a13f2e6SEdward Tomasz Napierala size_t len; 4211a13f2e6SEdward Tomasz Napierala 4221a13f2e6SEdward Tomasz Napierala if (!(flags & (M_WAITOK | M_NOWAIT))) 4231a13f2e6SEdward Tomasz Napierala flags |= M_NOWAIT; 4241a13f2e6SEdward Tomasz Napierala 4251a13f2e6SEdward Tomasz Napierala len = strlen(s) + 1; 4261a13f2e6SEdward Tomasz Napierala if ((copy = malloc(len, M_STATS, flags)) != NULL) 4271a13f2e6SEdward Tomasz Napierala bcopy(s, copy, len); 4281a13f2e6SEdward Tomasz Napierala 4291a13f2e6SEdward Tomasz Napierala return (copy); 4301a13f2e6SEdward Tomasz Napierala #else 4311a13f2e6SEdward Tomasz Napierala int flags __unused) 4321a13f2e6SEdward Tomasz Napierala { 4331a13f2e6SEdward Tomasz Napierala return (strdup(s)); 4341a13f2e6SEdward Tomasz Napierala #endif 4351a13f2e6SEdward Tomasz Napierala } 4361a13f2e6SEdward Tomasz Napierala 4371a13f2e6SEdward Tomasz Napierala static inline void 4381a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(struct statsblob_tpl *tpl) 4391a13f2e6SEdward Tomasz Napierala { 4401a13f2e6SEdward Tomasz Napierala 4411a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK_ASSERT(); 4421a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0); 4431a13f2e6SEdward Tomasz Napierala for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) { 4441a13f2e6SEdward Tomasz Napierala if (tpl->mb->voi_meta[voi_id].name != NULL) 4451a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_str( 4461a13f2e6SEdward Tomasz Napierala tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash); 4471a13f2e6SEdward Tomasz Napierala } 4481a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz, 4491a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash); 4501a13f2e6SEdward Tomasz Napierala } 4511a13f2e6SEdward Tomasz Napierala 4521a13f2e6SEdward Tomasz Napierala static inline uint64_t 4531a13f2e6SEdward Tomasz Napierala stats_pow_u64(uint64_t base, uint64_t exp) 4541a13f2e6SEdward Tomasz Napierala { 4551a13f2e6SEdward Tomasz Napierala uint64_t result = 1; 4561a13f2e6SEdward Tomasz Napierala 4571a13f2e6SEdward Tomasz Napierala while (exp) { 4581a13f2e6SEdward Tomasz Napierala if (exp & 1) 4591a13f2e6SEdward Tomasz Napierala result *= base; 4601a13f2e6SEdward Tomasz Napierala exp >>= 1; 4611a13f2e6SEdward Tomasz Napierala base *= base; 4621a13f2e6SEdward Tomasz Napierala } 4631a13f2e6SEdward Tomasz Napierala 4641a13f2e6SEdward Tomasz Napierala return (result); 4651a13f2e6SEdward Tomasz Napierala } 4661a13f2e6SEdward Tomasz Napierala 4671a13f2e6SEdward Tomasz Napierala static inline int 4681a13f2e6SEdward Tomasz Napierala stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt, 4691a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub) 4701a13f2e6SEdward Tomasz Napierala { 4711a13f2e6SEdward Tomasz Napierala uint64_t step = 0; 4721a13f2e6SEdward Tomasz Napierala int error = 0; 4731a13f2e6SEdward Tomasz Napierala 4741a13f2e6SEdward Tomasz Napierala switch (info->scheme) { 4751a13f2e6SEdward Tomasz Napierala case BKT_LIN: 4761a13f2e6SEdward Tomasz Napierala step = info->lin.stepinc; 4771a13f2e6SEdward Tomasz Napierala break; 4781a13f2e6SEdward Tomasz Napierala case BKT_EXP: 4791a13f2e6SEdward Tomasz Napierala step = stats_pow_u64(info->exp.stepbase, 4801a13f2e6SEdward Tomasz Napierala info->exp.stepexp + curbkt); 4811a13f2e6SEdward Tomasz Napierala break; 4821a13f2e6SEdward Tomasz Napierala case BKT_LINEXP: 4831a13f2e6SEdward Tomasz Napierala { 4841a13f2e6SEdward Tomasz Napierala uint64_t curstepexp = 1; 4851a13f2e6SEdward Tomasz Napierala 4861a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) { 4871a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 4881a13f2e6SEdward Tomasz Napierala while ((int32_t)stats_pow_u64(info->linexp.stepbase, 4891a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int32.s32) 4901a13f2e6SEdward Tomasz Napierala curstepexp++; 4911a13f2e6SEdward Tomasz Napierala break; 4921a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 4931a13f2e6SEdward Tomasz Napierala while ((uint32_t)stats_pow_u64(info->linexp.stepbase, 4941a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int32.u32) 4951a13f2e6SEdward Tomasz Napierala curstepexp++; 4961a13f2e6SEdward Tomasz Napierala break; 4971a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 4981a13f2e6SEdward Tomasz Napierala while ((int64_t)stats_pow_u64(info->linexp.stepbase, 4991a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int64.s64) 5001a13f2e6SEdward Tomasz Napierala curstepexp++; 5011a13f2e6SEdward Tomasz Napierala break; 5021a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 5031a13f2e6SEdward Tomasz Napierala while ((uint64_t)stats_pow_u64(info->linexp.stepbase, 5041a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int64.u64) 5051a13f2e6SEdward Tomasz Napierala curstepexp++; 5061a13f2e6SEdward Tomasz Napierala break; 5071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 5081a13f2e6SEdward Tomasz Napierala while ((long)stats_pow_u64(info->linexp.stepbase, 5091a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->intlong.slong) 5101a13f2e6SEdward Tomasz Napierala curstepexp++; 5111a13f2e6SEdward Tomasz Napierala break; 5121a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 5131a13f2e6SEdward Tomasz Napierala while ((unsigned long)stats_pow_u64(info->linexp.stepbase, 5141a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->intlong.ulong) 5151a13f2e6SEdward Tomasz Napierala curstepexp++; 5161a13f2e6SEdward Tomasz Napierala break; 5171a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 5181a13f2e6SEdward Tomasz Napierala while ((s32q_t)stats_pow_u64(info->linexp.stepbase, 5191a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32)) 5201a13f2e6SEdward Tomasz Napierala break; 5211a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 5221a13f2e6SEdward Tomasz Napierala while ((u32q_t)stats_pow_u64(info->linexp.stepbase, 5231a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32)) 5241a13f2e6SEdward Tomasz Napierala break; 5251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 5261a13f2e6SEdward Tomasz Napierala while ((s64q_t)stats_pow_u64(info->linexp.stepbase, 5271a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64)) 5281a13f2e6SEdward Tomasz Napierala curstepexp++; 5291a13f2e6SEdward Tomasz Napierala break; 5301a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 5311a13f2e6SEdward Tomasz Napierala while ((u64q_t)stats_pow_u64(info->linexp.stepbase, 5321a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64)) 5331a13f2e6SEdward Tomasz Napierala curstepexp++; 5341a13f2e6SEdward Tomasz Napierala break; 5351a13f2e6SEdward Tomasz Napierala default: 5361a13f2e6SEdward Tomasz Napierala break; 5371a13f2e6SEdward Tomasz Napierala } 5381a13f2e6SEdward Tomasz Napierala 5391a13f2e6SEdward Tomasz Napierala step = stats_pow_u64(info->linexp.stepbase, curstepexp) / 5401a13f2e6SEdward Tomasz Napierala info->linexp.linstepdiv; 5411a13f2e6SEdward Tomasz Napierala if (step == 0) 5421a13f2e6SEdward Tomasz Napierala step = 1; 5431a13f2e6SEdward Tomasz Napierala break; 5441a13f2e6SEdward Tomasz Napierala } 5451a13f2e6SEdward Tomasz Napierala default: 5461a13f2e6SEdward Tomasz Napierala break; 5471a13f2e6SEdward Tomasz Napierala } 5481a13f2e6SEdward Tomasz Napierala 5491a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) { 5501a13f2e6SEdward Tomasz Napierala *bkt_lb = info->usr.bkts[curbkt].lb; 5511a13f2e6SEdward Tomasz Napierala *bkt_ub = info->usr.bkts[curbkt].ub; 5521a13f2e6SEdward Tomasz Napierala } else if (step != 0) { 5531a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) { 5541a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 5551a13f2e6SEdward Tomasz Napierala bkt_ub->int32.s32 += (int32_t)step; 5561a13f2e6SEdward Tomasz Napierala break; 5571a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 5581a13f2e6SEdward Tomasz Napierala bkt_ub->int32.u32 += (uint32_t)step; 5591a13f2e6SEdward Tomasz Napierala break; 5601a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 5611a13f2e6SEdward Tomasz Napierala bkt_ub->int64.s64 += (int64_t)step; 5621a13f2e6SEdward Tomasz Napierala break; 5631a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 5641a13f2e6SEdward Tomasz Napierala bkt_ub->int64.u64 += (uint64_t)step; 5651a13f2e6SEdward Tomasz Napierala break; 5661a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 5671a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.slong += (long)step; 5681a13f2e6SEdward Tomasz Napierala break; 5691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 5701a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.ulong += (unsigned long)step; 5711a13f2e6SEdward Tomasz Napierala break; 5721a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 5731a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q32.sq32, step); 5741a13f2e6SEdward Tomasz Napierala break; 5751a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 5761a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q32.uq32, step); 5771a13f2e6SEdward Tomasz Napierala break; 5781a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 5791a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q64.sq64, step); 5801a13f2e6SEdward Tomasz Napierala break; 5811a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 5821a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q64.uq64, step); 5831a13f2e6SEdward Tomasz Napierala break; 5841a13f2e6SEdward Tomasz Napierala default: 5851a13f2e6SEdward Tomasz Napierala break; 5861a13f2e6SEdward Tomasz Napierala } 5871a13f2e6SEdward Tomasz Napierala } else { /* info->scheme != BKT_USR && step == 0 */ 5881a13f2e6SEdward Tomasz Napierala return (EINVAL); 5891a13f2e6SEdward Tomasz Napierala } 5901a13f2e6SEdward Tomasz Napierala 5911a13f2e6SEdward Tomasz Napierala return (error); 5921a13f2e6SEdward Tomasz Napierala } 5931a13f2e6SEdward Tomasz Napierala 5941a13f2e6SEdward Tomasz Napierala static uint32_t 5951a13f2e6SEdward Tomasz Napierala stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info) 5961a13f2e6SEdward Tomasz Napierala { 5971a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric bkt_lb, bkt_ub; 5981a13f2e6SEdward Tomasz Napierala uint32_t nbkts; 5991a13f2e6SEdward Tomasz Napierala int done; 6001a13f2e6SEdward Tomasz Napierala 6011a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) { 6021a13f2e6SEdward Tomasz Napierala /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */ 6031a13f2e6SEdward Tomasz Napierala info->lb = info->usr.bkts[0].lb; 6041a13f2e6SEdward Tomasz Napierala info->ub = info->usr.bkts[info->usr.nbkts - 1].lb; 6051a13f2e6SEdward Tomasz Napierala } 6061a13f2e6SEdward Tomasz Napierala 6071a13f2e6SEdward Tomasz Napierala nbkts = 0; 6081a13f2e6SEdward Tomasz Napierala done = 0; 6091a13f2e6SEdward Tomasz Napierala bkt_ub = info->lb; 6101a13f2e6SEdward Tomasz Napierala 6111a13f2e6SEdward Tomasz Napierala do { 6121a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub; 6131a13f2e6SEdward Tomasz Napierala if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub)) 6141a13f2e6SEdward Tomasz Napierala return (0); 6151a13f2e6SEdward Tomasz Napierala 6161a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) 6171a13f2e6SEdward Tomasz Napierala done = (nbkts == info->usr.nbkts); 6181a13f2e6SEdward Tomasz Napierala else { 6191a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) { 6201a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 6211a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int32.s32 > info->ub.int32.s32); 6221a13f2e6SEdward Tomasz Napierala break; 6231a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 6241a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int32.u32 > info->ub.int32.u32); 6251a13f2e6SEdward Tomasz Napierala break; 6261a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 6271a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int64.s64 > info->ub.int64.s64); 6281a13f2e6SEdward Tomasz Napierala break; 6291a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 6301a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int64.u64 > info->ub.int64.u64); 6311a13f2e6SEdward Tomasz Napierala break; 6321a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 6331a13f2e6SEdward Tomasz Napierala done = (bkt_ub.intlong.slong > 6341a13f2e6SEdward Tomasz Napierala info->ub.intlong.slong); 6351a13f2e6SEdward Tomasz Napierala break; 6361a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 6371a13f2e6SEdward Tomasz Napierala done = (bkt_ub.intlong.ulong > 6381a13f2e6SEdward Tomasz Napierala info->ub.intlong.ulong); 6391a13f2e6SEdward Tomasz Napierala break; 6401a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 6411a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q32.sq32, 6421a13f2e6SEdward Tomasz Napierala info->ub.q32.sq32); 6431a13f2e6SEdward Tomasz Napierala break; 6441a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 6451a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q32.uq32, 6461a13f2e6SEdward Tomasz Napierala info->ub.q32.uq32); 6471a13f2e6SEdward Tomasz Napierala break; 6481a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 6491a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q64.sq64, 6501a13f2e6SEdward Tomasz Napierala info->ub.q64.sq64); 6511a13f2e6SEdward Tomasz Napierala break; 6521a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 6531a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q64.uq64, 6541a13f2e6SEdward Tomasz Napierala info->ub.q64.uq64); 6551a13f2e6SEdward Tomasz Napierala break; 6561a13f2e6SEdward Tomasz Napierala default: 6571a13f2e6SEdward Tomasz Napierala return (0); 6581a13f2e6SEdward Tomasz Napierala } 6591a13f2e6SEdward Tomasz Napierala } 6601a13f2e6SEdward Tomasz Napierala } while (!done); 6611a13f2e6SEdward Tomasz Napierala 6621a13f2e6SEdward Tomasz Napierala if (info->flags & VSD_HIST_LBOUND_INF) 6631a13f2e6SEdward Tomasz Napierala nbkts++; 6641a13f2e6SEdward Tomasz Napierala if (info->flags & VSD_HIST_UBOUND_INF) 6651a13f2e6SEdward Tomasz Napierala nbkts++; 6661a13f2e6SEdward Tomasz Napierala 6671a13f2e6SEdward Tomasz Napierala return (nbkts); 6681a13f2e6SEdward Tomasz Napierala } 6691a13f2e6SEdward Tomasz Napierala 6701a13f2e6SEdward Tomasz Napierala int 6711a13f2e6SEdward Tomasz Napierala stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 6721a13f2e6SEdward Tomasz Napierala struct vss_hist_hlpr_info *info) 6731a13f2e6SEdward Tomasz Napierala { 6741a13f2e6SEdward Tomasz Napierala struct voistatdata_hist *hist; 6751a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub, 6761a13f2e6SEdward Tomasz Napierala *ubinfbktlb, *ubinfbktub; 6771a13f2e6SEdward Tomasz Napierala uint32_t bkt, nbkts, nloop; 6781a13f2e6SEdward Tomasz Napierala 6791a13f2e6SEdward Tomasz Napierala if (vss == NULL || info == NULL || (info->flags & 6801a13f2e6SEdward Tomasz Napierala (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype == 6811a13f2e6SEdward Tomasz Napierala VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64))) 6821a13f2e6SEdward Tomasz Napierala return (EINVAL); 6831a13f2e6SEdward Tomasz Napierala 6841a13f2e6SEdward Tomasz Napierala info->voi_dtype = voi_dtype; 6851a13f2e6SEdward Tomasz Napierala 6861a13f2e6SEdward Tomasz Napierala if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0) 6871a13f2e6SEdward Tomasz Napierala return (EINVAL); 6881a13f2e6SEdward Tomasz Napierala 6891a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) { 6901a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 6911a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts); 6921a13f2e6SEdward Tomasz Napierala break; 6931a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 6941a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts); 6951a13f2e6SEdward Tomasz Napierala break; 6961a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 6971a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts); 6981a13f2e6SEdward Tomasz Napierala break; 6991a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 7001a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts); 7011a13f2e6SEdward Tomasz Napierala break; 7021a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 7031a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts); 7041a13f2e6SEdward Tomasz Napierala break; 7051a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 7061a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts); 7071a13f2e6SEdward Tomasz Napierala break; 7081a13f2e6SEdward Tomasz Napierala default: 7091a13f2e6SEdward Tomasz Napierala return (EINVAL); 7101a13f2e6SEdward Tomasz Napierala } 7111a13f2e6SEdward Tomasz Napierala 7121a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 7131a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL) 7141a13f2e6SEdward Tomasz Napierala return (ENOMEM); 7151a13f2e6SEdward Tomasz Napierala 7161a13f2e6SEdward Tomasz Napierala hist = (struct voistatdata_hist *)vss->iv; 7171a13f2e6SEdward Tomasz Napierala bkt_ub = info->lb; 7181a13f2e6SEdward Tomasz Napierala 7191a13f2e6SEdward Tomasz Napierala for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0; 7201a13f2e6SEdward Tomasz Napierala bkt < nbkts; 7211a13f2e6SEdward Tomasz Napierala bkt++, nloop++) { 7221a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub; 7231a13f2e6SEdward Tomasz Napierala if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub)) 7241a13f2e6SEdward Tomasz Napierala return (EINVAL); 7251a13f2e6SEdward Tomasz Napierala 7261a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) { 7271a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 7281a13f2e6SEdward Tomasz Napierala VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb; 7291a13f2e6SEdward Tomasz Napierala break; 7301a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 7311a13f2e6SEdward Tomasz Napierala VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb; 7321a13f2e6SEdward Tomasz Napierala VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub; 7331a13f2e6SEdward Tomasz Napierala break; 7341a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 7351a13f2e6SEdward Tomasz Napierala VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb; 7361a13f2e6SEdward Tomasz Napierala break; 7371a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 7381a13f2e6SEdward Tomasz Napierala VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb; 7391a13f2e6SEdward Tomasz Napierala break; 7401a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 7411a13f2e6SEdward Tomasz Napierala VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb; 7421a13f2e6SEdward Tomasz Napierala VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub; 7431a13f2e6SEdward Tomasz Napierala break; 7441a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 7451a13f2e6SEdward Tomasz Napierala VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb; 7461a13f2e6SEdward Tomasz Napierala break; 7471a13f2e6SEdward Tomasz Napierala default: 7481a13f2e6SEdward Tomasz Napierala return (EINVAL); 7491a13f2e6SEdward Tomasz Napierala } 7501a13f2e6SEdward Tomasz Napierala } 7511a13f2e6SEdward Tomasz Napierala 7521a13f2e6SEdward Tomasz Napierala lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL; 7531a13f2e6SEdward Tomasz Napierala 7541a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) { 7551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 7561a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb; 7571a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb; 7581a13f2e6SEdward Tomasz Napierala break; 7591a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 7601a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb; 7611a13f2e6SEdward Tomasz Napierala lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub; 7621a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb; 7631a13f2e6SEdward Tomasz Napierala ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub; 7641a13f2e6SEdward Tomasz Napierala break; 7651a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 7661a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb; 7671a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb; 7681a13f2e6SEdward Tomasz Napierala break; 7691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 7701a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb; 7711a13f2e6SEdward Tomasz Napierala lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub; 7721a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb; 7731a13f2e6SEdward Tomasz Napierala ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub; 7741a13f2e6SEdward Tomasz Napierala break; 7751a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 7761a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 7771a13f2e6SEdward Tomasz Napierala break; 7781a13f2e6SEdward Tomasz Napierala default: 7791a13f2e6SEdward Tomasz Napierala return (EINVAL); 7801a13f2e6SEdward Tomasz Napierala } 7811a13f2e6SEdward Tomasz Napierala 7821a13f2e6SEdward Tomasz Napierala if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) { 7831a13f2e6SEdward Tomasz Napierala *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype]; 7841a13f2e6SEdward Tomasz Napierala /* 7851a13f2e6SEdward Tomasz Napierala * Assignment from numeric_limit array for Q types assigns max 7861a13f2e6SEdward Tomasz Napierala * possible integral/fractional value for underlying data type, 7871a13f2e6SEdward Tomasz Napierala * but we must set control bits for this specific histogram per 7881a13f2e6SEdward Tomasz Napierala * the user's choice of fractional bits, which we extract from 7891a13f2e6SEdward Tomasz Napierala * info->lb. 7901a13f2e6SEdward Tomasz Napierala */ 7911a13f2e6SEdward Tomasz Napierala if (info->voi_dtype == VSD_DTYPE_Q_S32 || 7921a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U32) { 7931a13f2e6SEdward Tomasz Napierala /* Signedness doesn't matter for setting control bits. */ 7941a13f2e6SEdward Tomasz Napierala Q_SCVAL(lbinfbktlb->q32.sq32, 7951a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q32.sq32)); 7961a13f2e6SEdward Tomasz Napierala } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 7971a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U64) { 7981a13f2e6SEdward Tomasz Napierala /* Signedness doesn't matter for setting control bits. */ 7991a13f2e6SEdward Tomasz Napierala Q_SCVAL(lbinfbktlb->q64.sq64, 8001a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q64.sq64)); 8011a13f2e6SEdward Tomasz Napierala } 8021a13f2e6SEdward Tomasz Napierala if (lbinfbktub) 8031a13f2e6SEdward Tomasz Napierala *lbinfbktub = info->lb; 8041a13f2e6SEdward Tomasz Napierala } 8051a13f2e6SEdward Tomasz Napierala if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) { 8061a13f2e6SEdward Tomasz Napierala *ubinfbktlb = bkt_lb; 8071a13f2e6SEdward Tomasz Napierala if (ubinfbktub) { 8081a13f2e6SEdward Tomasz Napierala *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype]; 8091a13f2e6SEdward Tomasz Napierala if (info->voi_dtype == VSD_DTYPE_Q_S32 || 8101a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U32) { 8111a13f2e6SEdward Tomasz Napierala Q_SCVAL(ubinfbktub->q32.sq32, 8121a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q32.sq32)); 8131a13f2e6SEdward Tomasz Napierala } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 8141a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U64) { 8151a13f2e6SEdward Tomasz Napierala Q_SCVAL(ubinfbktub->q64.sq64, 8161a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q64.sq64)); 8171a13f2e6SEdward Tomasz Napierala } 8181a13f2e6SEdward Tomasz Napierala } 8191a13f2e6SEdward Tomasz Napierala } 8201a13f2e6SEdward Tomasz Napierala 8211a13f2e6SEdward Tomasz Napierala return (0); 8221a13f2e6SEdward Tomasz Napierala } 8231a13f2e6SEdward Tomasz Napierala 8241a13f2e6SEdward Tomasz Napierala int 8251a13f2e6SEdward Tomasz Napierala stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 8261a13f2e6SEdward Tomasz Napierala struct vss_tdgst_hlpr_info *info) 8271a13f2e6SEdward Tomasz Napierala { 8281a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst; 8291a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 8301a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 8311a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 8321a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 8331a13f2e6SEdward Tomasz Napierala 8341a13f2e6SEdward Tomasz Napierala info->voi_dtype = voi_dtype; 8351a13f2e6SEdward Tomasz Napierala 8361a13f2e6SEdward Tomasz Napierala switch (info->tdgst_dtype) { 8371a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 8381a13f2e6SEdward Tomasz Napierala vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds); 8391a13f2e6SEdward Tomasz Napierala break; 8401a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 8411a13f2e6SEdward Tomasz Napierala vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds); 8421a13f2e6SEdward Tomasz Napierala break; 8431a13f2e6SEdward Tomasz Napierala default: 8441a13f2e6SEdward Tomasz Napierala return (EINVAL); 8451a13f2e6SEdward Tomasz Napierala } 8461a13f2e6SEdward Tomasz Napierala 8471a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 8481a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL) 8491a13f2e6SEdward Tomasz Napierala return (ENOMEM); 8501a13f2e6SEdward Tomasz Napierala 8511a13f2e6SEdward Tomasz Napierala tdgst = (struct voistatdata_tdgst *)vss->iv; 8521a13f2e6SEdward Tomasz Napierala 8531a13f2e6SEdward Tomasz Napierala switch (info->tdgst_dtype) { 8541a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 8551a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 8561a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) { 8571a13f2e6SEdward Tomasz Napierala Q_INI(&ctd32->mu, 0, 0, info->prec); 8581a13f2e6SEdward Tomasz Napierala } 8591a13f2e6SEdward Tomasz Napierala break; 8601a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 8611a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 8621a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) { 8631a13f2e6SEdward Tomasz Napierala Q_INI(&ctd64->mu, 0, 0, info->prec); 8641a13f2e6SEdward Tomasz Napierala } 8651a13f2e6SEdward Tomasz Napierala break; 8661a13f2e6SEdward Tomasz Napierala default: 8671a13f2e6SEdward Tomasz Napierala return (EINVAL); 8681a13f2e6SEdward Tomasz Napierala } 8691a13f2e6SEdward Tomasz Napierala 8701a13f2e6SEdward Tomasz Napierala return (0); 8711a13f2e6SEdward Tomasz Napierala } 8721a13f2e6SEdward Tomasz Napierala 8731a13f2e6SEdward Tomasz Napierala int 8741a13f2e6SEdward Tomasz Napierala stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 8751a13f2e6SEdward Tomasz Napierala struct vss_numeric_hlpr_info *info) 8761a13f2e6SEdward Tomasz Napierala { 8771a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric iv; 8781a13f2e6SEdward Tomasz Napierala 8791a13f2e6SEdward Tomasz Napierala switch (vss->stype) { 8801a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM: 8811a13f2e6SEdward Tomasz Napierala iv = stats_ctor_vsd_numeric(0); 8821a13f2e6SEdward Tomasz Napierala break; 8831a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN: 8841a13f2e6SEdward Tomasz Napierala iv = numeric_limits[LIM_MAX][voi_dtype]; 8851a13f2e6SEdward Tomasz Napierala break; 8861a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX: 8871a13f2e6SEdward Tomasz Napierala iv = numeric_limits[LIM_MIN][voi_dtype]; 8881a13f2e6SEdward Tomasz Napierala break; 8891a13f2e6SEdward Tomasz Napierala default: 8901a13f2e6SEdward Tomasz Napierala return (EINVAL); 8911a13f2e6SEdward Tomasz Napierala } 8921a13f2e6SEdward Tomasz Napierala 8931a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0); 8941a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL) 8951a13f2e6SEdward Tomasz Napierala return (ENOMEM); 8961a13f2e6SEdward Tomasz Napierala 8971a13f2e6SEdward Tomasz Napierala vss->vs_dtype = voi_dtype; 8981a13f2e6SEdward Tomasz Napierala vss->vsdsz = vsd_dtype2size[voi_dtype]; 8991a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 9001a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 9011a13f2e6SEdward Tomasz Napierala *((int32_t *)vss->iv) = iv.int32.s32; 9021a13f2e6SEdward Tomasz Napierala break; 9031a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 9041a13f2e6SEdward Tomasz Napierala *((uint32_t *)vss->iv) = iv.int32.u32; 9051a13f2e6SEdward Tomasz Napierala break; 9061a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 9071a13f2e6SEdward Tomasz Napierala *((int64_t *)vss->iv) = iv.int64.s64; 9081a13f2e6SEdward Tomasz Napierala break; 9091a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 9101a13f2e6SEdward Tomasz Napierala *((uint64_t *)vss->iv) = iv.int64.u64; 9111a13f2e6SEdward Tomasz Napierala break; 9121a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 9131a13f2e6SEdward Tomasz Napierala *((long *)vss->iv) = iv.intlong.slong; 9141a13f2e6SEdward Tomasz Napierala break; 9151a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 9161a13f2e6SEdward Tomasz Napierala *((unsigned long *)vss->iv) = iv.intlong.ulong; 9171a13f2e6SEdward Tomasz Napierala break; 9181a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 9191a13f2e6SEdward Tomasz Napierala *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32, 9201a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 9211a13f2e6SEdward Tomasz Napierala break; 9221a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 9231a13f2e6SEdward Tomasz Napierala *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32, 9241a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 9251a13f2e6SEdward Tomasz Napierala break; 9261a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 9271a13f2e6SEdward Tomasz Napierala *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64, 9281a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 9291a13f2e6SEdward Tomasz Napierala break; 9301a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 9311a13f2e6SEdward Tomasz Napierala *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64, 9321a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 9331a13f2e6SEdward Tomasz Napierala break; 9341a13f2e6SEdward Tomasz Napierala default: 9351a13f2e6SEdward Tomasz Napierala break; 9361a13f2e6SEdward Tomasz Napierala } 9371a13f2e6SEdward Tomasz Napierala 9381a13f2e6SEdward Tomasz Napierala return (0); 9391a13f2e6SEdward Tomasz Napierala } 9401a13f2e6SEdward Tomasz Napierala 9411a13f2e6SEdward Tomasz Napierala int 9421a13f2e6SEdward Tomasz Napierala stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss, 9431a13f2e6SEdward Tomasz Napierala struct voistatspec *vss) 9441a13f2e6SEdward Tomasz Napierala { 9451a13f2e6SEdward Tomasz Napierala int i, ret; 9461a13f2e6SEdward Tomasz Napierala 9471a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) { 9481a13f2e6SEdward Tomasz Napierala if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i], 9491a13f2e6SEdward Tomasz Napierala vss[i].hlprinfo)) != 0) 9501a13f2e6SEdward Tomasz Napierala return (ret); 9511a13f2e6SEdward Tomasz Napierala } 9521a13f2e6SEdward Tomasz Napierala 9531a13f2e6SEdward Tomasz Napierala return (0); 9541a13f2e6SEdward Tomasz Napierala } 9551a13f2e6SEdward Tomasz Napierala 9561a13f2e6SEdward Tomasz Napierala void 9571a13f2e6SEdward Tomasz Napierala stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss) 9581a13f2e6SEdward Tomasz Napierala { 9591a13f2e6SEdward Tomasz Napierala int i; 9601a13f2e6SEdward Tomasz Napierala 9611a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) { 9621a13f2e6SEdward Tomasz Napierala if (vss[i].hlpr) { 9631a13f2e6SEdward Tomasz Napierala stats_free((void *)vss[i].iv); 9641a13f2e6SEdward Tomasz Napierala vss[i].iv = NULL; 9651a13f2e6SEdward Tomasz Napierala } 9661a13f2e6SEdward Tomasz Napierala } 9671a13f2e6SEdward Tomasz Napierala } 9681a13f2e6SEdward Tomasz Napierala 9691a13f2e6SEdward Tomasz Napierala int 9701a13f2e6SEdward Tomasz Napierala stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl) 9711a13f2e6SEdward Tomasz Napierala { 9721a13f2e6SEdward Tomasz Napierala int error; 9731a13f2e6SEdward Tomasz Napierala 9741a13f2e6SEdward Tomasz Napierala error = 0; 9751a13f2e6SEdward Tomasz Napierala 9761a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK(); 9771a13f2e6SEdward Tomasz Napierala if (tpl_id < 0 || tpl_id >= (int)ntpl) { 9781a13f2e6SEdward Tomasz Napierala error = ENOENT; 9791a13f2e6SEdward Tomasz Napierala } else { 9801a13f2e6SEdward Tomasz Napierala *tpl = tpllist[tpl_id]; 9811a13f2e6SEdward Tomasz Napierala /* XXXLAS: Acquire refcount on tpl. */ 9821a13f2e6SEdward Tomasz Napierala } 9831a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK(); 9841a13f2e6SEdward Tomasz Napierala 9851a13f2e6SEdward Tomasz Napierala return (error); 9861a13f2e6SEdward Tomasz Napierala } 9871a13f2e6SEdward Tomasz Napierala 9881a13f2e6SEdward Tomasz Napierala int 9891a13f2e6SEdward Tomasz Napierala stats_tpl_fetch_allocid(const char *name, uint32_t hash) 9901a13f2e6SEdward Tomasz Napierala { 9911a13f2e6SEdward Tomasz Napierala int i, tpl_id; 9921a13f2e6SEdward Tomasz Napierala 9931a13f2e6SEdward Tomasz Napierala tpl_id = -ESRCH; 9941a13f2e6SEdward Tomasz Napierala 9951a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 9961a13f2e6SEdward Tomasz Napierala for (i = ntpl - 1; i >= 0; i--) { 9971a13f2e6SEdward Tomasz Napierala if (name != NULL) { 9981a13f2e6SEdward Tomasz Napierala if (strlen(name) == strlen(tpllist[i]->mb->tplname) && 9991a13f2e6SEdward Tomasz Napierala strncmp(name, tpllist[i]->mb->tplname, 10001a13f2e6SEdward Tomasz Napierala TPL_MAX_NAME_LEN) == 0 && (!hash || hash == 10011a13f2e6SEdward Tomasz Napierala tpllist[i]->mb->tplhash)) { 10021a13f2e6SEdward Tomasz Napierala tpl_id = i; 10031a13f2e6SEdward Tomasz Napierala break; 10041a13f2e6SEdward Tomasz Napierala } 10051a13f2e6SEdward Tomasz Napierala } else if (hash == tpllist[i]->mb->tplhash) { 10061a13f2e6SEdward Tomasz Napierala tpl_id = i; 10071a13f2e6SEdward Tomasz Napierala break; 10081a13f2e6SEdward Tomasz Napierala } 10091a13f2e6SEdward Tomasz Napierala } 10101a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 10111a13f2e6SEdward Tomasz Napierala 10121a13f2e6SEdward Tomasz Napierala return (tpl_id); 10131a13f2e6SEdward Tomasz Napierala } 10141a13f2e6SEdward Tomasz Napierala 10151a13f2e6SEdward Tomasz Napierala int 10161a13f2e6SEdward Tomasz Napierala stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len) 10171a13f2e6SEdward Tomasz Napierala { 10181a13f2e6SEdward Tomasz Napierala int error; 10191a13f2e6SEdward Tomasz Napierala 10201a13f2e6SEdward Tomasz Napierala error = 0; 10211a13f2e6SEdward Tomasz Napierala 10221a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 10231a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) { 10241a13f2e6SEdward Tomasz Napierala if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname)) 10251a13f2e6SEdward Tomasz Napierala strlcpy(buf, tpllist[tpl_id]->mb->tplname, len); 10261a13f2e6SEdward Tomasz Napierala else 10271a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 10281a13f2e6SEdward Tomasz Napierala } else 10291a13f2e6SEdward Tomasz Napierala error = ENOENT; 10301a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 10311a13f2e6SEdward Tomasz Napierala 10321a13f2e6SEdward Tomasz Napierala return (error); 10331a13f2e6SEdward Tomasz Napierala } 10341a13f2e6SEdward Tomasz Napierala 10351a13f2e6SEdward Tomasz Napierala int 10361a13f2e6SEdward Tomasz Napierala stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates, 10371a13f2e6SEdward Tomasz Napierala void *seed_bytes, size_t seed_len) 10381a13f2e6SEdward Tomasz Napierala { 10391a13f2e6SEdward Tomasz Napierala uint32_t cum_pct, rnd_pct; 10401a13f2e6SEdward Tomasz Napierala int i; 10411a13f2e6SEdward Tomasz Napierala 10421a13f2e6SEdward Tomasz Napierala cum_pct = 0; 10431a13f2e6SEdward Tomasz Napierala 10441a13f2e6SEdward Tomasz Napierala /* 10451a13f2e6SEdward Tomasz Napierala * Choose a pseudorandom or seeded number in range [0,100] and use 10461a13f2e6SEdward Tomasz Napierala * it to make a sampling decision and template selection where required. 10471a13f2e6SEdward Tomasz Napierala * If no seed is supplied, a PRNG is used to generate a pseudorandom 10481a13f2e6SEdward Tomasz Napierala * number so that every selection is independent. If a seed is supplied, 10491a13f2e6SEdward Tomasz Napierala * the caller desires random selection across different seeds, but 10501a13f2e6SEdward Tomasz Napierala * deterministic selection given the same seed. This is achieved by 10511a13f2e6SEdward Tomasz Napierala * hashing the seed and using the hash as the random number source. 10521a13f2e6SEdward Tomasz Napierala * 10531a13f2e6SEdward Tomasz Napierala * XXXLAS: Characterise hash function output distribution. 10541a13f2e6SEdward Tomasz Napierala */ 10551a13f2e6SEdward Tomasz Napierala if (seed_bytes == NULL) 10561a13f2e6SEdward Tomasz Napierala rnd_pct = random() / (INT32_MAX / 100); 10571a13f2e6SEdward Tomasz Napierala else 10581a13f2e6SEdward Tomasz Napierala rnd_pct = hash32_buf(seed_bytes, seed_len, 0) / 10591a13f2e6SEdward Tomasz Napierala (UINT32_MAX / 100U); 10601a13f2e6SEdward Tomasz Napierala 10611a13f2e6SEdward Tomasz Napierala /* 10621a13f2e6SEdward Tomasz Napierala * We map the randomly selected percentage on to the interval [0,100] 10631a13f2e6SEdward Tomasz Napierala * consisting of the cumulatively summed template sampling percentages. 10641a13f2e6SEdward Tomasz Napierala * The difference between the cumulative sum of all template sampling 10651a13f2e6SEdward Tomasz Napierala * percentages and 100 is treated as a NULL assignment i.e. no stats 10661a13f2e6SEdward Tomasz Napierala * template will be assigned, and -1 returned instead. 10671a13f2e6SEdward Tomasz Napierala */ 10681a13f2e6SEdward Tomasz Napierala for (i = 0; i < nrates; i++) { 10691a13f2e6SEdward Tomasz Napierala cum_pct += rates[i].tpl_sample_pct; 10701a13f2e6SEdward Tomasz Napierala 10711a13f2e6SEdward Tomasz Napierala KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__, 10721a13f2e6SEdward Tomasz Napierala cum_pct)); 10731a13f2e6SEdward Tomasz Napierala if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0) 10741a13f2e6SEdward Tomasz Napierala continue; 10751a13f2e6SEdward Tomasz Napierala 10761a13f2e6SEdward Tomasz Napierala return (rates[i].tpl_slot_id); 10771a13f2e6SEdward Tomasz Napierala } 10781a13f2e6SEdward Tomasz Napierala 10791a13f2e6SEdward Tomasz Napierala return (-1); 10801a13f2e6SEdward Tomasz Napierala } 10811a13f2e6SEdward Tomasz Napierala 10821a13f2e6SEdward Tomasz Napierala int 10831a13f2e6SEdward Tomasz Napierala stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz, 10841a13f2e6SEdward Tomasz Napierala struct statsblobv1 *src, uint32_t flags) 10851a13f2e6SEdward Tomasz Napierala { 10861a13f2e6SEdward Tomasz Napierala int error; 10871a13f2e6SEdward Tomasz Napierala 10881a13f2e6SEdward Tomasz Napierala error = 0; 10891a13f2e6SEdward Tomasz Napierala 10901a13f2e6SEdward Tomasz Napierala if (src == NULL || dst == NULL || 10911a13f2e6SEdward Tomasz Napierala src->cursz < sizeof(struct statsblob) || 10921a13f2e6SEdward Tomasz Napierala ((flags & SB_CLONE_ALLOCDST) && 10931a13f2e6SEdward Tomasz Napierala (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) { 10941a13f2e6SEdward Tomasz Napierala error = EINVAL; 10951a13f2e6SEdward Tomasz Napierala } else if (flags & SB_CLONE_ALLOCDST) { 10961a13f2e6SEdward Tomasz Napierala *dst = stats_realloc(NULL, 0, src->cursz, 0); 10971a13f2e6SEdward Tomasz Napierala if (*dst) 10981a13f2e6SEdward Tomasz Napierala (*dst)->maxsz = dstmaxsz = src->cursz; 10991a13f2e6SEdward Tomasz Napierala else 11001a13f2e6SEdward Tomasz Napierala error = ENOMEM; 11011a13f2e6SEdward Tomasz Napierala } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) { 11021a13f2e6SEdward Tomasz Napierala error = EINVAL; 11031a13f2e6SEdward Tomasz Napierala } 11041a13f2e6SEdward Tomasz Napierala 11051a13f2e6SEdward Tomasz Napierala if (!error) { 11061a13f2e6SEdward Tomasz Napierala size_t postcurszlen; 11071a13f2e6SEdward Tomasz Napierala 11081a13f2e6SEdward Tomasz Napierala /* 11091a13f2e6SEdward Tomasz Napierala * Clone src into dst except for the maxsz field. If dst is too 11101a13f2e6SEdward Tomasz Napierala * small to hold all of src, only copy src's header and return 11111a13f2e6SEdward Tomasz Napierala * EOVERFLOW. 11121a13f2e6SEdward Tomasz Napierala */ 11131a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 11141a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_USRDSTNOFAULT) 11151a13f2e6SEdward Tomasz Napierala copyout_nofault(src, *dst, 11161a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, maxsz)); 11171a13f2e6SEdward Tomasz Napierala else if (flags & SB_CLONE_USRDST) 11181a13f2e6SEdward Tomasz Napierala copyout(src, *dst, offsetof(struct statsblob, maxsz)); 11191a13f2e6SEdward Tomasz Napierala else 11201a13f2e6SEdward Tomasz Napierala #endif 11211a13f2e6SEdward Tomasz Napierala memcpy(*dst, src, offsetof(struct statsblob, maxsz)); 11221a13f2e6SEdward Tomasz Napierala 11231a13f2e6SEdward Tomasz Napierala if (dstmaxsz >= src->cursz) { 11241a13f2e6SEdward Tomasz Napierala postcurszlen = src->cursz - 11251a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, cursz); 11261a13f2e6SEdward Tomasz Napierala } else { 11271a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 11281a13f2e6SEdward Tomasz Napierala postcurszlen = sizeof(struct statsblob) - 11291a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, cursz); 11301a13f2e6SEdward Tomasz Napierala } 11311a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 11321a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_USRDSTNOFAULT) 11331a13f2e6SEdward Tomasz Napierala copyout_nofault(&(src->cursz), &((*dst)->cursz), 11341a13f2e6SEdward Tomasz Napierala postcurszlen); 11351a13f2e6SEdward Tomasz Napierala else if (flags & SB_CLONE_USRDST) 11361a13f2e6SEdward Tomasz Napierala copyout(&(src->cursz), &((*dst)->cursz), postcurszlen); 11371a13f2e6SEdward Tomasz Napierala else 11381a13f2e6SEdward Tomasz Napierala #endif 11391a13f2e6SEdward Tomasz Napierala memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen); 11401a13f2e6SEdward Tomasz Napierala } 11411a13f2e6SEdward Tomasz Napierala 11421a13f2e6SEdward Tomasz Napierala return (error); 11431a13f2e6SEdward Tomasz Napierala } 11441a13f2e6SEdward Tomasz Napierala 11451a13f2e6SEdward Tomasz Napierala int 11461a13f2e6SEdward Tomasz Napierala stats_v1_tpl_alloc(const char *name, uint32_t flags __unused) 11471a13f2e6SEdward Tomasz Napierala { 11481a13f2e6SEdward Tomasz Napierala struct statsblobv1_tpl *tpl, **newtpllist; 11491a13f2e6SEdward Tomasz Napierala struct statsblobv1 *tpl_sb; 11501a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 11511a13f2e6SEdward Tomasz Napierala int tpl_id; 11521a13f2e6SEdward Tomasz Napierala 11531a13f2e6SEdward Tomasz Napierala if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN) 11541a13f2e6SEdward Tomasz Napierala return (-EINVAL); 11551a13f2e6SEdward Tomasz Napierala 11561a13f2e6SEdward Tomasz Napierala if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0) 11571a13f2e6SEdward Tomasz Napierala return (-EEXIST); 11581a13f2e6SEdward Tomasz Napierala 11591a13f2e6SEdward Tomasz Napierala tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO); 11601a13f2e6SEdward Tomasz Napierala tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO); 11611a13f2e6SEdward Tomasz Napierala tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO); 11621a13f2e6SEdward Tomasz Napierala 11631a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL && name != NULL) 11641a13f2e6SEdward Tomasz Napierala tpl_mb->tplname = stats_strdup(name, 0); 11651a13f2e6SEdward Tomasz Napierala 11661a13f2e6SEdward Tomasz Napierala if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL || 11671a13f2e6SEdward Tomasz Napierala tpl_mb->tplname == NULL) { 11681a13f2e6SEdward Tomasz Napierala stats_free(tpl); 11691a13f2e6SEdward Tomasz Napierala stats_free(tpl_sb); 11701a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL) { 11711a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb->tplname); 11721a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb); 11731a13f2e6SEdward Tomasz Napierala } 11741a13f2e6SEdward Tomasz Napierala return (-ENOMEM); 11751a13f2e6SEdward Tomasz Napierala } 11761a13f2e6SEdward Tomasz Napierala 11771a13f2e6SEdward Tomasz Napierala tpl->mb = tpl_mb; 11781a13f2e6SEdward Tomasz Napierala tpl->sb = tpl_sb; 11791a13f2e6SEdward Tomasz Napierala 11801a13f2e6SEdward Tomasz Napierala tpl_sb->abi = STATS_ABI_V1; 11811a13f2e6SEdward Tomasz Napierala tpl_sb->endian = 11821a13f2e6SEdward Tomasz Napierala #if BYTE_ORDER == LITTLE_ENDIAN 11831a13f2e6SEdward Tomasz Napierala SB_LE; 11841a13f2e6SEdward Tomasz Napierala #elif BYTE_ORDER == BIG_ENDIAN 11851a13f2e6SEdward Tomasz Napierala SB_BE; 11861a13f2e6SEdward Tomasz Napierala #else 11871a13f2e6SEdward Tomasz Napierala SB_UE; 11881a13f2e6SEdward Tomasz Napierala #endif 11891a13f2e6SEdward Tomasz Napierala tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1); 11901a13f2e6SEdward Tomasz Napierala tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1); 11911a13f2e6SEdward Tomasz Napierala 11921a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK(); 11931a13f2e6SEdward Tomasz Napierala newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *), 11941a13f2e6SEdward Tomasz Napierala (ntpl + 1) * sizeof(void *), 0); 11951a13f2e6SEdward Tomasz Napierala if (newtpllist != NULL) { 11961a13f2e6SEdward Tomasz Napierala tpl_id = ntpl++; 11971a13f2e6SEdward Tomasz Napierala tpllist = (struct statsblob_tpl **)newtpllist; 11981a13f2e6SEdward Tomasz Napierala tpllist[tpl_id] = (struct statsblob_tpl *)tpl; 11991a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(tpllist[tpl_id]); 12001a13f2e6SEdward Tomasz Napierala } else { 12011a13f2e6SEdward Tomasz Napierala stats_free(tpl); 12021a13f2e6SEdward Tomasz Napierala stats_free(tpl_sb); 12031a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL) { 12041a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb->tplname); 12051a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb); 12061a13f2e6SEdward Tomasz Napierala } 12071a13f2e6SEdward Tomasz Napierala tpl_id = -ENOMEM; 12081a13f2e6SEdward Tomasz Napierala } 12091a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK(); 12101a13f2e6SEdward Tomasz Napierala 12111a13f2e6SEdward Tomasz Napierala return (tpl_id); 12121a13f2e6SEdward Tomasz Napierala } 12131a13f2e6SEdward Tomasz Napierala 12141a13f2e6SEdward Tomasz Napierala int 12151a13f2e6SEdward Tomasz Napierala stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name, 12161a13f2e6SEdward Tomasz Napierala enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss, 12171a13f2e6SEdward Tomasz Napierala uint32_t flags) 12181a13f2e6SEdward Tomasz Napierala { 12191a13f2e6SEdward Tomasz Napierala struct voi *voi; 12201a13f2e6SEdward Tomasz Napierala struct voistat *tmpstat; 12211a13f2e6SEdward Tomasz Napierala struct statsblobv1 *tpl_sb; 12221a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 12231a13f2e6SEdward Tomasz Napierala int error, i, newstatdataidx, newvoibytes, newvoistatbytes, 12241a13f2e6SEdward Tomasz Napierala newvoistatdatabytes, newvoistatmaxid; 12251a13f2e6SEdward Tomasz Napierala uint32_t nbytes; 12261a13f2e6SEdward Tomasz Napierala 12271a13f2e6SEdward Tomasz Napierala if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || 12281a13f2e6SEdward Tomasz Napierala nvss == 0 || vss == NULL) 12291a13f2e6SEdward Tomasz Napierala return (EINVAL); 12301a13f2e6SEdward Tomasz Napierala 12311a13f2e6SEdward Tomasz Napierala error = nbytes = newvoibytes = newvoistatbytes = 12321a13f2e6SEdward Tomasz Napierala newvoistatdatabytes = 0; 12331a13f2e6SEdward Tomasz Napierala newvoistatmaxid = -1; 12341a13f2e6SEdward Tomasz Napierala 12351a13f2e6SEdward Tomasz Napierala /* Calculate the number of bytes required for the new voistats. */ 12361a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) { 12371a13f2e6SEdward Tomasz Napierala if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES || 12381a13f2e6SEdward Tomasz Napierala vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES || 12391a13f2e6SEdward Tomasz Napierala vss[i].iv == NULL || vss[i].vsdsz == 0) 12401a13f2e6SEdward Tomasz Napierala return (EINVAL); 12411a13f2e6SEdward Tomasz Napierala if ((int)vss[i].stype > newvoistatmaxid) 12421a13f2e6SEdward Tomasz Napierala newvoistatmaxid = vss[i].stype; 12431a13f2e6SEdward Tomasz Napierala newvoistatdatabytes += vss[i].vsdsz; 12441a13f2e6SEdward Tomasz Napierala } 12451a13f2e6SEdward Tomasz Napierala 12461a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) { 12471a13f2e6SEdward Tomasz Napierala /* XXXLAS: VOI state bytes may need to vary based on stat types. */ 12481a13f2e6SEdward Tomasz Napierala newvoistatdatabytes += sizeof(struct voistatdata_voistate); 12491a13f2e6SEdward Tomasz Napierala } 12501a13f2e6SEdward Tomasz Napierala nbytes += newvoistatdatabytes; 12511a13f2e6SEdward Tomasz Napierala 12521a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK(); 12531a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) { 12541a13f2e6SEdward Tomasz Napierala tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb; 12551a13f2e6SEdward Tomasz Napierala tpl_mb = tpllist[tpl_id]->mb; 12561a13f2e6SEdward Tomasz Napierala 12571a13f2e6SEdward Tomasz Napierala if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) { 12581a13f2e6SEdward Tomasz Napierala /* Adding a new VOI and associated stats. */ 12591a13f2e6SEdward Tomasz Napierala if (voi_id >= NVOIS(tpl_sb)) { 12601a13f2e6SEdward Tomasz Napierala /* We need to grow the tpl_sb->vois array. */ 12611a13f2e6SEdward Tomasz Napierala newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) * 12621a13f2e6SEdward Tomasz Napierala sizeof(struct voi); 12631a13f2e6SEdward Tomasz Napierala nbytes += newvoibytes; 12641a13f2e6SEdward Tomasz Napierala } 12651a13f2e6SEdward Tomasz Napierala newvoistatbytes = 12661a13f2e6SEdward Tomasz Napierala (newvoistatmaxid + 1) * sizeof(struct voistat); 12671a13f2e6SEdward Tomasz Napierala } else { 12681a13f2e6SEdward Tomasz Napierala /* Adding stats to an existing VOI. */ 12691a13f2e6SEdward Tomasz Napierala if (newvoistatmaxid > 12701a13f2e6SEdward Tomasz Napierala tpl_sb->vois[voi_id].voistatmaxid) { 12711a13f2e6SEdward Tomasz Napierala newvoistatbytes = (newvoistatmaxid - 12721a13f2e6SEdward Tomasz Napierala tpl_sb->vois[voi_id].voistatmaxid) * 12731a13f2e6SEdward Tomasz Napierala sizeof(struct voistat); 12741a13f2e6SEdward Tomasz Napierala } 12751a13f2e6SEdward Tomasz Napierala /* XXXLAS: KPI does not yet support expanding VOIs. */ 12761a13f2e6SEdward Tomasz Napierala error = EOPNOTSUPP; 12771a13f2e6SEdward Tomasz Napierala } 12781a13f2e6SEdward Tomasz Napierala nbytes += newvoistatbytes; 12791a13f2e6SEdward Tomasz Napierala 12801a13f2e6SEdward Tomasz Napierala if (!error && newvoibytes > 0) { 12811a13f2e6SEdward Tomasz Napierala struct voi_meta *voi_meta = tpl_mb->voi_meta; 12821a13f2e6SEdward Tomasz Napierala 12831a13f2e6SEdward Tomasz Napierala voi_meta = stats_realloc(voi_meta, voi_meta == NULL ? 12841a13f2e6SEdward Tomasz Napierala 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta), 12851a13f2e6SEdward Tomasz Napierala (1 + voi_id) * sizeof(struct voi_meta), 12861a13f2e6SEdward Tomasz Napierala M_ZERO); 12871a13f2e6SEdward Tomasz Napierala 12881a13f2e6SEdward Tomasz Napierala if (voi_meta == NULL) 12891a13f2e6SEdward Tomasz Napierala error = ENOMEM; 12901a13f2e6SEdward Tomasz Napierala else 12911a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta = voi_meta; 12921a13f2e6SEdward Tomasz Napierala } 12931a13f2e6SEdward Tomasz Napierala 12941a13f2e6SEdward Tomasz Napierala if (!error) { 12951a13f2e6SEdward Tomasz Napierala /* NB: Resizing can change where tpl_sb points. */ 12961a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_expand(&tpl_sb, newvoibytes, 12971a13f2e6SEdward Tomasz Napierala newvoistatbytes, newvoistatdatabytes); 12981a13f2e6SEdward Tomasz Napierala } 12991a13f2e6SEdward Tomasz Napierala 13001a13f2e6SEdward Tomasz Napierala if (!error) { 13011a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name, 13021a13f2e6SEdward Tomasz Napierala 0); 13031a13f2e6SEdward Tomasz Napierala if (tpl_mb->voi_meta[voi_id].name == NULL) 13041a13f2e6SEdward Tomasz Napierala error = ENOMEM; 13051a13f2e6SEdward Tomasz Napierala } 13061a13f2e6SEdward Tomasz Napierala 13071a13f2e6SEdward Tomasz Napierala if (!error) { 13081a13f2e6SEdward Tomasz Napierala /* Update the template list with the resized pointer. */ 13091a13f2e6SEdward Tomasz Napierala tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb; 13101a13f2e6SEdward Tomasz Napierala 13111a13f2e6SEdward Tomasz Napierala /* Update the template. */ 13121a13f2e6SEdward Tomasz Napierala voi = &tpl_sb->vois[voi_id]; 13131a13f2e6SEdward Tomasz Napierala 13141a13f2e6SEdward Tomasz Napierala if (voi->id < 0) { 13151a13f2e6SEdward Tomasz Napierala /* VOI is new and needs to be initialised. */ 13161a13f2e6SEdward Tomasz Napierala voi->id = voi_id; 13171a13f2e6SEdward Tomasz Napierala voi->dtype = voi_dtype; 13181a13f2e6SEdward Tomasz Napierala voi->stats_off = tpl_sb->stats_off; 13191a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) 13201a13f2e6SEdward Tomasz Napierala voi->flags |= VOI_REQSTATE; 13211a13f2e6SEdward Tomasz Napierala } else { 13221a13f2e6SEdward Tomasz Napierala /* 13231a13f2e6SEdward Tomasz Napierala * XXXLAS: When this else block is written, the 13241a13f2e6SEdward Tomasz Napierala * "KPI does not yet support expanding VOIs" 13251a13f2e6SEdward Tomasz Napierala * error earlier in this function can be 13261a13f2e6SEdward Tomasz Napierala * removed. What is required here is to shuffle 13271a13f2e6SEdward Tomasz Napierala * the voistat array such that the new stats for 13281a13f2e6SEdward Tomasz Napierala * the voi are contiguous, which will displace 13291a13f2e6SEdward Tomasz Napierala * stats for other vois that reside after the 13301a13f2e6SEdward Tomasz Napierala * voi being updated. The other vois then need 13311a13f2e6SEdward Tomasz Napierala * to have their stats_off adjusted post 13321a13f2e6SEdward Tomasz Napierala * shuffle. 13331a13f2e6SEdward Tomasz Napierala */ 13341a13f2e6SEdward Tomasz Napierala } 13351a13f2e6SEdward Tomasz Napierala 13361a13f2e6SEdward Tomasz Napierala voi->voistatmaxid = newvoistatmaxid; 13371a13f2e6SEdward Tomasz Napierala newstatdataidx = 0; 13381a13f2e6SEdward Tomasz Napierala 13391a13f2e6SEdward Tomasz Napierala if (voi->flags & VOI_REQSTATE) { 13401a13f2e6SEdward Tomasz Napierala /* Initialise the voistate stat in slot 0. */ 13411a13f2e6SEdward Tomasz Napierala tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off); 13421a13f2e6SEdward Tomasz Napierala tmpstat->stype = VS_STYPE_VOISTATE; 13431a13f2e6SEdward Tomasz Napierala tmpstat->flags = 0; 13441a13f2e6SEdward Tomasz Napierala tmpstat->dtype = VSD_DTYPE_VOISTATE; 13451a13f2e6SEdward Tomasz Napierala newstatdataidx = tmpstat->dsz = 13461a13f2e6SEdward Tomasz Napierala sizeof(struct voistatdata_numeric); 13471a13f2e6SEdward Tomasz Napierala tmpstat->data_off = tpl_sb->statsdata_off; 13481a13f2e6SEdward Tomasz Napierala } 13491a13f2e6SEdward Tomasz Napierala 13501a13f2e6SEdward Tomasz Napierala for (i = 0; (uint32_t)i < nvss; i++) { 13511a13f2e6SEdward Tomasz Napierala tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off + 13521a13f2e6SEdward Tomasz Napierala (vss[i].stype * sizeof(struct voistat))); 13531a13f2e6SEdward Tomasz Napierala KASSERT(tmpstat->stype < 0, ("voistat %p " 13541a13f2e6SEdward Tomasz Napierala "already initialised", tmpstat)); 13551a13f2e6SEdward Tomasz Napierala tmpstat->stype = vss[i].stype; 13561a13f2e6SEdward Tomasz Napierala tmpstat->flags = vss[i].flags; 13571a13f2e6SEdward Tomasz Napierala tmpstat->dtype = vss[i].vs_dtype; 13581a13f2e6SEdward Tomasz Napierala tmpstat->dsz = vss[i].vsdsz; 13591a13f2e6SEdward Tomasz Napierala tmpstat->data_off = tpl_sb->statsdata_off + 13601a13f2e6SEdward Tomasz Napierala newstatdataidx; 13611a13f2e6SEdward Tomasz Napierala memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off), 13621a13f2e6SEdward Tomasz Napierala vss[i].iv, vss[i].vsdsz); 13631a13f2e6SEdward Tomasz Napierala newstatdataidx += vss[i].vsdsz; 13641a13f2e6SEdward Tomasz Napierala } 13651a13f2e6SEdward Tomasz Napierala 13661a13f2e6SEdward Tomasz Napierala /* Update the template version hash. */ 13671a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(tpllist[tpl_id]); 13681a13f2e6SEdward Tomasz Napierala /* XXXLAS: Confirm tpl name/hash pair remains unique. */ 13691a13f2e6SEdward Tomasz Napierala } 13701a13f2e6SEdward Tomasz Napierala } else 13711a13f2e6SEdward Tomasz Napierala error = EINVAL; 13721a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK(); 13731a13f2e6SEdward Tomasz Napierala 13741a13f2e6SEdward Tomasz Napierala return (error); 13751a13f2e6SEdward Tomasz Napierala } 13761a13f2e6SEdward Tomasz Napierala 13771a13f2e6SEdward Tomasz Napierala struct statsblobv1 * 13781a13f2e6SEdward Tomasz Napierala stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused) 13791a13f2e6SEdward Tomasz Napierala { 13801a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb; 13811a13f2e6SEdward Tomasz Napierala int error; 13821a13f2e6SEdward Tomasz Napierala 13831a13f2e6SEdward Tomasz Napierala sb = NULL; 13841a13f2e6SEdward Tomasz Napierala 13851a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 13861a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) { 13871a13f2e6SEdward Tomasz Napierala sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0); 13881a13f2e6SEdward Tomasz Napierala if (sb != NULL) { 13891a13f2e6SEdward Tomasz Napierala sb->maxsz = tpllist[tpl_id]->sb->maxsz; 13901a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_init_locked(sb, tpl_id, 0); 13911a13f2e6SEdward Tomasz Napierala } else 13921a13f2e6SEdward Tomasz Napierala error = ENOMEM; 13931a13f2e6SEdward Tomasz Napierala 13941a13f2e6SEdward Tomasz Napierala if (error) { 13951a13f2e6SEdward Tomasz Napierala stats_free(sb); 13961a13f2e6SEdward Tomasz Napierala sb = NULL; 13971a13f2e6SEdward Tomasz Napierala } 13981a13f2e6SEdward Tomasz Napierala } 13991a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 14001a13f2e6SEdward Tomasz Napierala 14011a13f2e6SEdward Tomasz Napierala return (sb); 14021a13f2e6SEdward Tomasz Napierala } 14031a13f2e6SEdward Tomasz Napierala 14041a13f2e6SEdward Tomasz Napierala void 14051a13f2e6SEdward Tomasz Napierala stats_v1_blob_destroy(struct statsblobv1 *sb) 14061a13f2e6SEdward Tomasz Napierala { 14071a13f2e6SEdward Tomasz Napierala 14081a13f2e6SEdward Tomasz Napierala stats_free(sb); 14091a13f2e6SEdward Tomasz Napierala } 14101a13f2e6SEdward Tomasz Napierala 14111a13f2e6SEdward Tomasz Napierala int 14121a13f2e6SEdward Tomasz Napierala stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id, 14131a13f2e6SEdward Tomasz Napierala enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd, 14141a13f2e6SEdward Tomasz Napierala size_t *retvsdsz) 14151a13f2e6SEdward Tomasz Napierala { 14161a13f2e6SEdward Tomasz Napierala struct voi *v; 14171a13f2e6SEdward Tomasz Napierala struct voistat *vs; 14181a13f2e6SEdward Tomasz Napierala 14191a13f2e6SEdward Tomasz Napierala if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 || 14201a13f2e6SEdward Tomasz Napierala voi_id >= NVOIS(sb)) 14211a13f2e6SEdward Tomasz Napierala return (EINVAL); 14221a13f2e6SEdward Tomasz Napierala 14231a13f2e6SEdward Tomasz Napierala v = &sb->vois[voi_id]; 14241a13f2e6SEdward Tomasz Napierala if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid) 14251a13f2e6SEdward Tomasz Napierala return (EINVAL); 14261a13f2e6SEdward Tomasz Napierala 14271a13f2e6SEdward Tomasz Napierala vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat))); 14281a13f2e6SEdward Tomasz Napierala *retvsd = BLOB_OFFSET(sb, vs->data_off); 14291a13f2e6SEdward Tomasz Napierala if (retdtype != NULL) 14301a13f2e6SEdward Tomasz Napierala *retdtype = vs->dtype; 14311a13f2e6SEdward Tomasz Napierala if (retvsdsz != NULL) 14321a13f2e6SEdward Tomasz Napierala *retvsdsz = vs->dsz; 14331a13f2e6SEdward Tomasz Napierala 14341a13f2e6SEdward Tomasz Napierala return (0); 14351a13f2e6SEdward Tomasz Napierala } 14361a13f2e6SEdward Tomasz Napierala 14371a13f2e6SEdward Tomasz Napierala int 14381a13f2e6SEdward Tomasz Napierala stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags) 14391a13f2e6SEdward Tomasz Napierala { 14401a13f2e6SEdward Tomasz Napierala int error; 14411a13f2e6SEdward Tomasz Napierala 14421a13f2e6SEdward Tomasz Napierala error = 0; 14431a13f2e6SEdward Tomasz Napierala 14441a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 14451a13f2e6SEdward Tomasz Napierala if (sb == NULL || tpl_id >= ntpl) { 14461a13f2e6SEdward Tomasz Napierala error = EINVAL; 14471a13f2e6SEdward Tomasz Napierala } else { 14481a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_init_locked(sb, tpl_id, flags); 14491a13f2e6SEdward Tomasz Napierala } 14501a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 14511a13f2e6SEdward Tomasz Napierala 14521a13f2e6SEdward Tomasz Napierala return (error); 14531a13f2e6SEdward Tomasz Napierala } 14541a13f2e6SEdward Tomasz Napierala 14551a13f2e6SEdward Tomasz Napierala static inline int 14561a13f2e6SEdward Tomasz Napierala stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 14571a13f2e6SEdward Tomasz Napierala uint32_t flags __unused) 14581a13f2e6SEdward Tomasz Napierala { 14591a13f2e6SEdward Tomasz Napierala int error; 14601a13f2e6SEdward Tomasz Napierala 14611a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK_ASSERT(); 14621a13f2e6SEdward Tomasz Napierala error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW; 14631a13f2e6SEdward Tomasz Napierala KASSERT(!error, 14641a13f2e6SEdward Tomasz Napierala ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz)); 14651a13f2e6SEdward Tomasz Napierala 14661a13f2e6SEdward Tomasz Napierala if (!error) { 14671a13f2e6SEdward Tomasz Napierala memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz); 14681a13f2e6SEdward Tomasz Napierala sb->created = sb->lastrst = stats_sbinuptime(); 14691a13f2e6SEdward Tomasz Napierala sb->tplhash = tpllist[tpl_id]->mb->tplhash; 14701a13f2e6SEdward Tomasz Napierala } 14711a13f2e6SEdward Tomasz Napierala 14721a13f2e6SEdward Tomasz Napierala return (error); 14731a13f2e6SEdward Tomasz Napierala } 14741a13f2e6SEdward Tomasz Napierala 14751a13f2e6SEdward Tomasz Napierala static int 14761a13f2e6SEdward Tomasz Napierala stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 14771a13f2e6SEdward Tomasz Napierala int newvoistatbytes, int newvoistatdatabytes) 14781a13f2e6SEdward Tomasz Napierala { 14791a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb; 14801a13f2e6SEdward Tomasz Napierala struct voi *tmpvoi; 14811a13f2e6SEdward Tomasz Napierala struct voistat *tmpvoistat, *voistat_array; 14821a13f2e6SEdward Tomasz Napierala int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats; 14831a13f2e6SEdward Tomasz Napierala 14841a13f2e6SEdward Tomasz Napierala KASSERT(newvoibytes % sizeof(struct voi) == 0, 14851a13f2e6SEdward Tomasz Napierala ("Bad newvoibytes %d", newvoibytes)); 14861a13f2e6SEdward Tomasz Napierala KASSERT(newvoistatbytes % sizeof(struct voistat) == 0, 14871a13f2e6SEdward Tomasz Napierala ("Bad newvoistatbytes %d", newvoistatbytes)); 14881a13f2e6SEdward Tomasz Napierala 14891a13f2e6SEdward Tomasz Napierala error = ((newvoibytes % sizeof(struct voi) == 0) && 14901a13f2e6SEdward Tomasz Napierala (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL; 14911a13f2e6SEdward Tomasz Napierala sb = *sbpp; 14921a13f2e6SEdward Tomasz Napierala nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes; 14931a13f2e6SEdward Tomasz Napierala 14941a13f2e6SEdward Tomasz Napierala /* 14951a13f2e6SEdward Tomasz Napierala * XXXLAS: Required until we gain support for flags which alter the 14961a13f2e6SEdward Tomasz Napierala * units of size/offset fields in key structs. 14971a13f2e6SEdward Tomasz Napierala */ 14981a13f2e6SEdward Tomasz Napierala if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ)) 14991a13f2e6SEdward Tomasz Napierala error = EFBIG; 15001a13f2e6SEdward Tomasz Napierala 15011a13f2e6SEdward Tomasz Napierala if (!error && (sb->cursz + nbytes > sb->maxsz)) { 15021a13f2e6SEdward Tomasz Napierala /* Need to expand our blob. */ 15031a13f2e6SEdward Tomasz Napierala sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO); 15041a13f2e6SEdward Tomasz Napierala if (sb != NULL) { 15051a13f2e6SEdward Tomasz Napierala sb->maxsz = sb->cursz + nbytes; 15061a13f2e6SEdward Tomasz Napierala *sbpp = sb; 15071a13f2e6SEdward Tomasz Napierala } else 15081a13f2e6SEdward Tomasz Napierala error = ENOMEM; 15091a13f2e6SEdward Tomasz Napierala } 15101a13f2e6SEdward Tomasz Napierala 15111a13f2e6SEdward Tomasz Napierala if (!error) { 15121a13f2e6SEdward Tomasz Napierala /* 15131a13f2e6SEdward Tomasz Napierala * Shuffle memory within the expanded blob working from the end 15141a13f2e6SEdward Tomasz Napierala * backwards, leaving gaps for the new voistat and voistatdata 15151a13f2e6SEdward Tomasz Napierala * structs at the beginning of their respective blob regions, 15161a13f2e6SEdward Tomasz Napierala * and for the new voi structs at the end of their blob region. 15171a13f2e6SEdward Tomasz Napierala */ 15181a13f2e6SEdward Tomasz Napierala memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes), 15191a13f2e6SEdward Tomasz Napierala BLOB_OFFSET(sb, sb->statsdata_off), 15201a13f2e6SEdward Tomasz Napierala sb->cursz - sb->statsdata_off); 15211a13f2e6SEdward Tomasz Napierala memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes + 15221a13f2e6SEdward Tomasz Napierala newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off), 15231a13f2e6SEdward Tomasz Napierala sb->statsdata_off - sb->stats_off); 15241a13f2e6SEdward Tomasz Napierala 15251a13f2e6SEdward Tomasz Napierala /* First index of new voi/voistat structs to be initialised. */ 15261a13f2e6SEdward Tomasz Napierala idxnewvois = NVOIS(sb); 15271a13f2e6SEdward Tomasz Napierala idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1; 15281a13f2e6SEdward Tomasz Napierala 15291a13f2e6SEdward Tomasz Napierala /* Update housekeeping variables and offsets. */ 15301a13f2e6SEdward Tomasz Napierala sb->cursz += nbytes; 15311a13f2e6SEdward Tomasz Napierala sb->stats_off += newvoibytes; 15321a13f2e6SEdward Tomasz Napierala sb->statsdata_off += newvoibytes + newvoistatbytes; 15331a13f2e6SEdward Tomasz Napierala 15341a13f2e6SEdward Tomasz Napierala /* XXXLAS: Zeroing not strictly needed but aids debugging. */ 15351a13f2e6SEdward Tomasz Napierala memset(&sb->vois[idxnewvois], '\0', newvoibytes); 15361a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(sb, sb->stats_off), '\0', 15371a13f2e6SEdward Tomasz Napierala newvoistatbytes); 15381a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0', 15391a13f2e6SEdward Tomasz Napierala newvoistatdatabytes); 15401a13f2e6SEdward Tomasz Napierala 15411a13f2e6SEdward Tomasz Napierala /* Initialise new voi array members and update offsets. */ 15421a13f2e6SEdward Tomasz Napierala for (i = 0; i < NVOIS(sb); i++) { 15431a13f2e6SEdward Tomasz Napierala tmpvoi = &sb->vois[i]; 15441a13f2e6SEdward Tomasz Napierala if (i >= idxnewvois) { 15451a13f2e6SEdward Tomasz Napierala tmpvoi->id = tmpvoi->voistatmaxid = -1; 15461a13f2e6SEdward Tomasz Napierala } else if (tmpvoi->id > -1) { 15471a13f2e6SEdward Tomasz Napierala tmpvoi->stats_off += newvoibytes + 15481a13f2e6SEdward Tomasz Napierala newvoistatbytes; 15491a13f2e6SEdward Tomasz Napierala } 15501a13f2e6SEdward Tomasz Napierala } 15511a13f2e6SEdward Tomasz Napierala 15521a13f2e6SEdward Tomasz Napierala /* Initialise new voistat array members and update offsets. */ 15531a13f2e6SEdward Tomasz Napierala nvoistats = (sb->statsdata_off - sb->stats_off) / 15541a13f2e6SEdward Tomasz Napierala sizeof(struct voistat); 15551a13f2e6SEdward Tomasz Napierala voistat_array = BLOB_OFFSET(sb, sb->stats_off); 15561a13f2e6SEdward Tomasz Napierala for (i = 0; i < nvoistats; i++) { 15571a13f2e6SEdward Tomasz Napierala tmpvoistat = &voistat_array[i]; 15581a13f2e6SEdward Tomasz Napierala if (i <= idxnewvoistats) { 15591a13f2e6SEdward Tomasz Napierala tmpvoistat->stype = -1; 15601a13f2e6SEdward Tomasz Napierala } else if (tmpvoistat->stype > -1) { 15611a13f2e6SEdward Tomasz Napierala tmpvoistat->data_off += nbytes; 15621a13f2e6SEdward Tomasz Napierala } 15631a13f2e6SEdward Tomasz Napierala } 15641a13f2e6SEdward Tomasz Napierala } 15651a13f2e6SEdward Tomasz Napierala 15661a13f2e6SEdward Tomasz Napierala return (error); 15671a13f2e6SEdward Tomasz Napierala } 15681a13f2e6SEdward Tomasz Napierala 15691a13f2e6SEdward Tomasz Napierala static void 15701a13f2e6SEdward Tomasz Napierala stats_v1_blob_finalise(struct statsblobv1 *sb __unused) 15711a13f2e6SEdward Tomasz Napierala { 15721a13f2e6SEdward Tomasz Napierala 15731a13f2e6SEdward Tomasz Napierala /* XXXLAS: Fill this in. */ 15741a13f2e6SEdward Tomasz Napierala } 15751a13f2e6SEdward Tomasz Napierala 15761a13f2e6SEdward Tomasz Napierala static void 15771a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb, 15781a13f2e6SEdward Tomasz Napierala void *usrctx, uint32_t flags) 15791a13f2e6SEdward Tomasz Napierala { 15801a13f2e6SEdward Tomasz Napierala struct voi *v; 15811a13f2e6SEdward Tomasz Napierala struct voistat *vs; 15821a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx ctx; 15831a13f2e6SEdward Tomasz Napierala int i, j, firstvoi; 15841a13f2e6SEdward Tomasz Napierala 15851a13f2e6SEdward Tomasz Napierala ctx.usrctx = usrctx; 1586fbb9879cSJohn Baldwin ctx.flags = SB_IT_FIRST_CB; 15871a13f2e6SEdward Tomasz Napierala firstvoi = 1; 15881a13f2e6SEdward Tomasz Napierala 15891a13f2e6SEdward Tomasz Napierala for (i = 0; i < NVOIS(sb); i++) { 15901a13f2e6SEdward Tomasz Napierala v = &sb->vois[i]; 15911a13f2e6SEdward Tomasz Napierala ctx.vslot = i; 15921a13f2e6SEdward Tomasz Napierala ctx.vsslot = -1; 15931a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_FIRST_VOISTAT; 15941a13f2e6SEdward Tomasz Napierala 15951a13f2e6SEdward Tomasz Napierala if (firstvoi) 15961a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_FIRST_VOI; 15971a13f2e6SEdward Tomasz Napierala else if (i == (NVOIS(sb) - 1)) 15981a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB; 15991a13f2e6SEdward Tomasz Napierala 16001a13f2e6SEdward Tomasz Napierala if (v->id < 0 && (flags & SB_IT_NULLVOI)) { 16011a13f2e6SEdward Tomasz Napierala if (icb(sb, v, NULL, &ctx)) 16021a13f2e6SEdward Tomasz Napierala return; 16031a13f2e6SEdward Tomasz Napierala firstvoi = 0; 16041a13f2e6SEdward Tomasz Napierala ctx.flags &= ~SB_IT_FIRST_CB; 16051a13f2e6SEdward Tomasz Napierala } 16061a13f2e6SEdward Tomasz Napierala 16071a13f2e6SEdward Tomasz Napierala /* If NULL voi, v->voistatmaxid == -1 */ 16081a13f2e6SEdward Tomasz Napierala for (j = 0; j <= v->voistatmaxid; j++) { 16091a13f2e6SEdward Tomasz Napierala vs = &((struct voistat *)BLOB_OFFSET(sb, 16101a13f2e6SEdward Tomasz Napierala v->stats_off))[j]; 16111a13f2e6SEdward Tomasz Napierala if (vs->stype < 0 && 16121a13f2e6SEdward Tomasz Napierala !(flags & SB_IT_NULLVOISTAT)) 16131a13f2e6SEdward Tomasz Napierala continue; 16141a13f2e6SEdward Tomasz Napierala 16151a13f2e6SEdward Tomasz Napierala if (j == v->voistatmaxid) { 16161a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_LAST_VOISTAT; 16171a13f2e6SEdward Tomasz Napierala if (i == (NVOIS(sb) - 1)) 16181a13f2e6SEdward Tomasz Napierala ctx.flags |= 16191a13f2e6SEdward Tomasz Napierala SB_IT_LAST_CB; 16201a13f2e6SEdward Tomasz Napierala } else 16211a13f2e6SEdward Tomasz Napierala ctx.flags &= ~SB_IT_LAST_CB; 16221a13f2e6SEdward Tomasz Napierala 16231a13f2e6SEdward Tomasz Napierala ctx.vsslot = j; 16241a13f2e6SEdward Tomasz Napierala if (icb(sb, v, vs, &ctx)) 16251a13f2e6SEdward Tomasz Napierala return; 16261a13f2e6SEdward Tomasz Napierala 16271a13f2e6SEdward Tomasz Napierala ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT | 16281a13f2e6SEdward Tomasz Napierala SB_IT_LAST_VOISTAT); 16291a13f2e6SEdward Tomasz Napierala } 16301a13f2e6SEdward Tomasz Napierala ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI); 16311a13f2e6SEdward Tomasz Napierala } 16321a13f2e6SEdward Tomasz Napierala } 16331a13f2e6SEdward Tomasz Napierala 16341a13f2e6SEdward Tomasz Napierala static inline void 16351a13f2e6SEdward Tomasz Napierala stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused, 16361a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype, 16371a13f2e6SEdward Tomasz Napierala size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 16381a13f2e6SEdward Tomasz Napierala { 16391a13f2e6SEdward Tomasz Napierala const struct ctdth32 *ctd32tree; 16401a13f2e6SEdward Tomasz Napierala const struct ctdth64 *ctd64tree; 16411a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgstctd32 *ctd32; 16421a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgstctd64 *ctd64; 16431a13f2e6SEdward Tomasz Napierala const char *fmtstr; 16441a13f2e6SEdward Tomasz Napierala uint64_t smplcnt, compcnt; 16451a13f2e6SEdward Tomasz Napierala int is32bit, qmaxstrlen; 16461a13f2e6SEdward Tomasz Napierala uint16_t maxctds, curctds; 16471a13f2e6SEdward Tomasz Napierala 16481a13f2e6SEdward Tomasz Napierala switch (tdgst_dtype) { 16491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 16501a13f2e6SEdward Tomasz Napierala smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt; 16511a13f2e6SEdward Tomasz Napierala compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt; 16521a13f2e6SEdward Tomasz Napierala maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 16531a13f2e6SEdward Tomasz Napierala curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 16541a13f2e6SEdward Tomasz Napierala ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree; 16551a13f2e6SEdward Tomasz Napierala ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) : 16561a13f2e6SEdward Tomasz Napierala ARB_CMIN(ctdth32, ctd32tree)); 16571a13f2e6SEdward Tomasz Napierala qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10); 16581a13f2e6SEdward Tomasz Napierala is32bit = 1; 16591a13f2e6SEdward Tomasz Napierala ctd64tree = NULL; 16601a13f2e6SEdward Tomasz Napierala ctd64 = NULL; 16611a13f2e6SEdward Tomasz Napierala break; 16621a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 16631a13f2e6SEdward Tomasz Napierala smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt; 16641a13f2e6SEdward Tomasz Napierala compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt; 16651a13f2e6SEdward Tomasz Napierala maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 16661a13f2e6SEdward Tomasz Napierala curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 16671a13f2e6SEdward Tomasz Napierala ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree; 16681a13f2e6SEdward Tomasz Napierala ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) : 16691a13f2e6SEdward Tomasz Napierala ARB_CMIN(ctdth64, ctd64tree)); 16701a13f2e6SEdward Tomasz Napierala qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10); 16711a13f2e6SEdward Tomasz Napierala is32bit = 0; 16721a13f2e6SEdward Tomasz Napierala ctd32tree = NULL; 16731a13f2e6SEdward Tomasz Napierala ctd32 = NULL; 16741a13f2e6SEdward Tomasz Napierala break; 16751a13f2e6SEdward Tomasz Napierala default: 16761a13f2e6SEdward Tomasz Napierala return; 16771a13f2e6SEdward Tomasz Napierala } 16781a13f2e6SEdward Tomasz Napierala 16791a13f2e6SEdward Tomasz Napierala switch (fmt) { 16801a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 16811a13f2e6SEdward Tomasz Napierala fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu"; 16821a13f2e6SEdward Tomasz Napierala break; 16831a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 16841a13f2e6SEdward Tomasz Napierala default: 16851a13f2e6SEdward Tomasz Napierala fmtstr = 16861a13f2e6SEdward Tomasz Napierala "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu," 16871a13f2e6SEdward Tomasz Napierala "\"nctds\":%hu,\"ctds\":["; 16881a13f2e6SEdward Tomasz Napierala break; 16891a13f2e6SEdward Tomasz Napierala } 16901a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt, 16911a13f2e6SEdward Tomasz Napierala maxctds, curctds); 16921a13f2e6SEdward Tomasz Napierala 16931a13f2e6SEdward Tomasz Napierala while ((is32bit ? NULL != ctd32 : NULL != ctd64)) { 16941a13f2e6SEdward Tomasz Napierala char qstr[qmaxstrlen]; 16951a13f2e6SEdward Tomasz Napierala 16961a13f2e6SEdward Tomasz Napierala switch (fmt) { 16971a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 16981a13f2e6SEdward Tomasz Napierala fmtstr = "\n\t\t\t\t"; 16991a13f2e6SEdward Tomasz Napierala break; 17001a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 17011a13f2e6SEdward Tomasz Napierala default: 17021a13f2e6SEdward Tomasz Napierala fmtstr = "{"; 17031a13f2e6SEdward Tomasz Napierala break; 17041a13f2e6SEdward Tomasz Napierala } 17051a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 17061a13f2e6SEdward Tomasz Napierala 17071a13f2e6SEdward Tomasz Napierala if (objdump) { 17081a13f2e6SEdward Tomasz Napierala switch (fmt) { 17091a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 17101a13f2e6SEdward Tomasz Napierala fmtstr = "ctd[%hu]."; 17111a13f2e6SEdward Tomasz Napierala break; 17121a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 17131a13f2e6SEdward Tomasz Napierala default: 17141a13f2e6SEdward Tomasz Napierala fmtstr = "\"ctd\":%hu,"; 17151a13f2e6SEdward Tomasz Napierala break; 17161a13f2e6SEdward Tomasz Napierala } 17171a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ? 17181a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd32tree, ctd32) : 17191a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, ctd64)); 17201a13f2e6SEdward Tomasz Napierala } 17211a13f2e6SEdward Tomasz Napierala 17221a13f2e6SEdward Tomasz Napierala switch (fmt) { 17231a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 17241a13f2e6SEdward Tomasz Napierala fmtstr = "{mu="; 17251a13f2e6SEdward Tomasz Napierala break; 17261a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 17271a13f2e6SEdward Tomasz Napierala default: 17281a13f2e6SEdward Tomasz Napierala fmtstr = "\"mu\":"; 17291a13f2e6SEdward Tomasz Napierala break; 17301a13f2e6SEdward Tomasz Napierala } 17311a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 17321a13f2e6SEdward Tomasz Napierala Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr, 17331a13f2e6SEdward Tomasz Napierala sizeof(qstr)); 17341a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 17351a13f2e6SEdward Tomasz Napierala 17361a13f2e6SEdward Tomasz Napierala switch (fmt) { 17371a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 17381a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 17391a13f2e6SEdward Tomasz Napierala break; 17401a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 17411a13f2e6SEdward Tomasz Napierala default: 17421a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 17431a13f2e6SEdward Tomasz Napierala break; 17441a13f2e6SEdward Tomasz Napierala } 17451a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, 17461a13f2e6SEdward Tomasz Napierala is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt); 17471a13f2e6SEdward Tomasz Napierala 17481a13f2e6SEdward Tomasz Napierala if (is32bit) 17491a13f2e6SEdward Tomasz Napierala ctd32 = (objdump ? ARB_CNODE(ctd32tree, 17501a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd32tree, ctd32) + 1) : 17511a13f2e6SEdward Tomasz Napierala ARB_CNEXT(ctdth32, ctd32tree, ctd32)); 17521a13f2e6SEdward Tomasz Napierala else 17531a13f2e6SEdward Tomasz Napierala ctd64 = (objdump ? ARB_CNODE(ctd64tree, 17541a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, ctd64) + 1) : 17551a13f2e6SEdward Tomasz Napierala ARB_CNEXT(ctdth64, ctd64tree, ctd64)); 17561a13f2e6SEdward Tomasz Napierala 17571a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON && 17581a13f2e6SEdward Tomasz Napierala (is32bit ? NULL != ctd32 : NULL != ctd64)) 17591a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ','); 17601a13f2e6SEdward Tomasz Napierala } 17611a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON) 17621a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "]"); 17631a13f2e6SEdward Tomasz Napierala } 17641a13f2e6SEdward Tomasz Napierala 17651a13f2e6SEdward Tomasz Napierala static inline void 17661a13f2e6SEdward Tomasz Napierala stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype, 17671a13f2e6SEdward Tomasz Napierala const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype, 17681a13f2e6SEdward Tomasz Napierala size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 17691a13f2e6SEdward Tomasz Napierala { 17701a13f2e6SEdward Tomasz Napierala const struct voistatdata_numeric *bkt_lb, *bkt_ub; 17711a13f2e6SEdward Tomasz Napierala const char *fmtstr; 17721a13f2e6SEdward Tomasz Napierala int is32bit; 17731a13f2e6SEdward Tomasz Napierala uint16_t i, nbkts; 17741a13f2e6SEdward Tomasz Napierala 17751a13f2e6SEdward Tomasz Napierala switch (hist_dtype) { 17761a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 17771a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz); 17781a13f2e6SEdward Tomasz Napierala is32bit = 1; 17791a13f2e6SEdward Tomasz Napierala break; 17801a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 17811a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz); 17821a13f2e6SEdward Tomasz Napierala is32bit = 1; 17831a13f2e6SEdward Tomasz Napierala break; 17841a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 17851a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz); 17861a13f2e6SEdward Tomasz Napierala is32bit = 1; 17871a13f2e6SEdward Tomasz Napierala break; 17881a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 17891a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz); 17901a13f2e6SEdward Tomasz Napierala is32bit = 0; 17911a13f2e6SEdward Tomasz Napierala break; 17921a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 17931a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz); 17941a13f2e6SEdward Tomasz Napierala is32bit = 0; 17951a13f2e6SEdward Tomasz Napierala break; 17961a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 17971a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz); 17981a13f2e6SEdward Tomasz Napierala is32bit = 0; 17991a13f2e6SEdward Tomasz Napierala break; 18001a13f2e6SEdward Tomasz Napierala default: 18011a13f2e6SEdward Tomasz Napierala return; 18021a13f2e6SEdward Tomasz Napierala } 18031a13f2e6SEdward Tomasz Napierala 18041a13f2e6SEdward Tomasz Napierala switch (fmt) { 18051a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 18061a13f2e6SEdward Tomasz Napierala fmtstr = "nbkts=%hu, "; 18071a13f2e6SEdward Tomasz Napierala break; 18081a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 18091a13f2e6SEdward Tomasz Napierala default: 18101a13f2e6SEdward Tomasz Napierala fmtstr = "\"nbkts\":%hu,"; 18111a13f2e6SEdward Tomasz Napierala break; 18121a13f2e6SEdward Tomasz Napierala } 18131a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, nbkts); 18141a13f2e6SEdward Tomasz Napierala 18151a13f2e6SEdward Tomasz Napierala switch (fmt) { 18161a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 18171a13f2e6SEdward Tomasz Napierala fmtstr = (is32bit ? "oob=%u" : "oob=%ju"); 18181a13f2e6SEdward Tomasz Napierala break; 18191a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 18201a13f2e6SEdward Tomasz Napierala default: 18211a13f2e6SEdward Tomasz Napierala fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" : 18221a13f2e6SEdward Tomasz Napierala "\"oob\":%ju,\"bkts\":["); 18231a13f2e6SEdward Tomasz Napierala break; 18241a13f2e6SEdward Tomasz Napierala } 18251a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist, 18261a13f2e6SEdward Tomasz Napierala hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, 18271a13f2e6SEdward Tomasz Napierala hist_dtype, oob)); 18281a13f2e6SEdward Tomasz Napierala 18291a13f2e6SEdward Tomasz Napierala for (i = 0; i < nbkts; i++) { 18301a13f2e6SEdward Tomasz Napierala switch (hist_dtype) { 18311a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 18321a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 18331a13f2e6SEdward Tomasz Napierala bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype, 18341a13f2e6SEdward Tomasz Napierala bkts[i].lb); 18351a13f2e6SEdward Tomasz Napierala if (i < nbkts - 1) 18361a13f2e6SEdward Tomasz Napierala bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist, 18371a13f2e6SEdward Tomasz Napierala hist_dtype, bkts[i + 1].lb); 18381a13f2e6SEdward Tomasz Napierala else 18391a13f2e6SEdward Tomasz Napierala bkt_ub = &numeric_limits[LIM_MAX][voi_dtype]; 18401a13f2e6SEdward Tomasz Napierala break; 18411a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 18421a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 18431a13f2e6SEdward Tomasz Napierala bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 18441a13f2e6SEdward Tomasz Napierala bkts[i].lb); 18451a13f2e6SEdward Tomasz Napierala bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 18461a13f2e6SEdward Tomasz Napierala bkts[i].ub); 18471a13f2e6SEdward Tomasz Napierala break; 18481a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 18491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 18501a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist, 18511a13f2e6SEdward Tomasz Napierala hist_dtype, bkts[i].val); 18521a13f2e6SEdward Tomasz Napierala break; 18531a13f2e6SEdward Tomasz Napierala default: 18541a13f2e6SEdward Tomasz Napierala break; 18551a13f2e6SEdward Tomasz Napierala } 18561a13f2e6SEdward Tomasz Napierala 18571a13f2e6SEdward Tomasz Napierala switch (fmt) { 18581a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 18591a13f2e6SEdward Tomasz Napierala fmtstr = "\n\t\t\t\t"; 18601a13f2e6SEdward Tomasz Napierala break; 18611a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 18621a13f2e6SEdward Tomasz Napierala default: 18631a13f2e6SEdward Tomasz Napierala fmtstr = "{"; 18641a13f2e6SEdward Tomasz Napierala break; 18651a13f2e6SEdward Tomasz Napierala } 18661a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 18671a13f2e6SEdward Tomasz Napierala 18681a13f2e6SEdward Tomasz Napierala if (objdump) { 18691a13f2e6SEdward Tomasz Napierala switch (fmt) { 18701a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 18711a13f2e6SEdward Tomasz Napierala fmtstr = "bkt[%hu]."; 18721a13f2e6SEdward Tomasz Napierala break; 18731a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 18741a13f2e6SEdward Tomasz Napierala default: 18751a13f2e6SEdward Tomasz Napierala fmtstr = "\"bkt\":%hu,"; 18761a13f2e6SEdward Tomasz Napierala break; 18771a13f2e6SEdward Tomasz Napierala } 18781a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, i); 18791a13f2e6SEdward Tomasz Napierala } 18801a13f2e6SEdward Tomasz Napierala 18811a13f2e6SEdward Tomasz Napierala switch (fmt) { 18821a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 18831a13f2e6SEdward Tomasz Napierala fmtstr = "{lb="; 18841a13f2e6SEdward Tomasz Napierala break; 18851a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 18861a13f2e6SEdward Tomasz Napierala default: 18871a13f2e6SEdward Tomasz Napierala fmtstr = "\"lb\":"; 18881a13f2e6SEdward Tomasz Napierala break; 18891a13f2e6SEdward Tomasz Napierala } 18901a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 18911a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr((const struct voistatdata *)bkt_lb, 18921a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 18931a13f2e6SEdward Tomasz Napierala fmt, buf, objdump); 18941a13f2e6SEdward Tomasz Napierala 18951a13f2e6SEdward Tomasz Napierala switch (fmt) { 18961a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 18971a13f2e6SEdward Tomasz Napierala fmtstr = ",ub="; 18981a13f2e6SEdward Tomasz Napierala break; 18991a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 19001a13f2e6SEdward Tomasz Napierala default: 19011a13f2e6SEdward Tomasz Napierala fmtstr = ",\"ub\":"; 19021a13f2e6SEdward Tomasz Napierala break; 19031a13f2e6SEdward Tomasz Napierala } 19041a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 19051a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr((const struct voistatdata *)bkt_ub, 19061a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 19071a13f2e6SEdward Tomasz Napierala fmt, buf, objdump); 19081a13f2e6SEdward Tomasz Napierala 19091a13f2e6SEdward Tomasz Napierala switch (fmt) { 19101a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 19111a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 19121a13f2e6SEdward Tomasz Napierala break; 19131a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 19141a13f2e6SEdward Tomasz Napierala default: 19151a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 19161a13f2e6SEdward Tomasz Napierala break; 19171a13f2e6SEdward Tomasz Napierala } 19181a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ? 19191a13f2e6SEdward Tomasz Napierala VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) : 19201a13f2e6SEdward Tomasz Napierala (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, 19211a13f2e6SEdward Tomasz Napierala bkts[i].cnt)); 19221a13f2e6SEdward Tomasz Napierala 19231a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON && i < nbkts - 1) 19241a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ','); 19251a13f2e6SEdward Tomasz Napierala } 19261a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON) 19271a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "]"); 19281a13f2e6SEdward Tomasz Napierala } 19291a13f2e6SEdward Tomasz Napierala 19301a13f2e6SEdward Tomasz Napierala int 19311a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype, 19321a13f2e6SEdward Tomasz Napierala enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt, 19331a13f2e6SEdward Tomasz Napierala struct sbuf *buf, int objdump) 19341a13f2e6SEdward Tomasz Napierala { 19351a13f2e6SEdward Tomasz Napierala const char *fmtstr; 19361a13f2e6SEdward Tomasz Napierala 19371a13f2e6SEdward Tomasz Napierala if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES || 19381a13f2e6SEdward Tomasz Napierala vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS) 19391a13f2e6SEdward Tomasz Napierala return (EINVAL); 19401a13f2e6SEdward Tomasz Napierala 19411a13f2e6SEdward Tomasz Napierala switch (vsd_dtype) { 19421a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_VOISTATE: 19431a13f2e6SEdward Tomasz Napierala switch (fmt) { 19441a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 19451a13f2e6SEdward Tomasz Napierala fmtstr = "prev="; 19461a13f2e6SEdward Tomasz Napierala break; 19471a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 19481a13f2e6SEdward Tomasz Napierala default: 19491a13f2e6SEdward Tomasz Napierala fmtstr = "\"prev\":"; 19501a13f2e6SEdward Tomasz Napierala break; 19511a13f2e6SEdward Tomasz Napierala } 19521a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 19531a13f2e6SEdward Tomasz Napierala /* 19541a13f2e6SEdward Tomasz Napierala * Render prev by passing it as *vsd and voi_dtype as vsd_dtype. 19551a13f2e6SEdward Tomasz Napierala */ 19561a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr( 19571a13f2e6SEdward Tomasz Napierala (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev, 19581a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump); 19591a13f2e6SEdward Tomasz Napierala break; 19601a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 19611a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%d", vsd->int32.s32); 19621a13f2e6SEdward Tomasz Napierala break; 19631a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 19641a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%u", vsd->int32.u32); 19651a13f2e6SEdward Tomasz Napierala break; 19661a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 19671a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64); 19681a13f2e6SEdward Tomasz Napierala break; 19691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 19701a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64); 19711a13f2e6SEdward Tomasz Napierala break; 19721a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 19731a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%ld", vsd->intlong.slong); 19741a13f2e6SEdward Tomasz Napierala break; 19751a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 19761a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%lu", vsd->intlong.ulong); 19771a13f2e6SEdward Tomasz Napierala break; 19781a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 19791a13f2e6SEdward Tomasz Napierala { 19801a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)]; 19811a13f2e6SEdward Tomasz Napierala Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr)); 19821a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 19831a13f2e6SEdward Tomasz Napierala } 19841a13f2e6SEdward Tomasz Napierala break; 19851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 19861a13f2e6SEdward Tomasz Napierala { 19871a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)]; 19881a13f2e6SEdward Tomasz Napierala Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr)); 19891a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 19901a13f2e6SEdward Tomasz Napierala } 19911a13f2e6SEdward Tomasz Napierala break; 19921a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 19931a13f2e6SEdward Tomasz Napierala { 19941a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)]; 19951a13f2e6SEdward Tomasz Napierala Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr)); 19961a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 19971a13f2e6SEdward Tomasz Napierala } 19981a13f2e6SEdward Tomasz Napierala break; 19991a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 20001a13f2e6SEdward Tomasz Napierala { 20011a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)]; 20021a13f2e6SEdward Tomasz Napierala Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr)); 20031a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 20041a13f2e6SEdward Tomasz Napierala } 20051a13f2e6SEdward Tomasz Napierala break; 20061a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 20071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 20081a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 20091a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 20101a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 20111a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 20121a13f2e6SEdward Tomasz Napierala stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd), 20131a13f2e6SEdward Tomasz Napierala vsd_dtype, vsd_sz, fmt, buf, objdump); 20141a13f2e6SEdward Tomasz Napierala break; 20151a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 20161a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 20171a13f2e6SEdward Tomasz Napierala stats_voistatdata_tdgst_tostr(voi_dtype, 20181a13f2e6SEdward Tomasz Napierala CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf, 20191a13f2e6SEdward Tomasz Napierala objdump); 20201a13f2e6SEdward Tomasz Napierala break; 20211a13f2e6SEdward Tomasz Napierala default: 20221a13f2e6SEdward Tomasz Napierala break; 20231a13f2e6SEdward Tomasz Napierala } 20241a13f2e6SEdward Tomasz Napierala 20251a13f2e6SEdward Tomasz Napierala return (sbuf_error(buf)); 20261a13f2e6SEdward Tomasz Napierala } 20271a13f2e6SEdward Tomasz Napierala 20281a13f2e6SEdward Tomasz Napierala static void 20291a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v, 20301a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx) 20311a13f2e6SEdward Tomasz Napierala { 20321a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx; 20331a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 20341a13f2e6SEdward Tomasz Napierala struct sbuf *buf; 20351a13f2e6SEdward Tomasz Napierala void *vsd; 20361a13f2e6SEdward Tomasz Napierala uint8_t dump; 20371a13f2e6SEdward Tomasz Napierala 20381a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx; 20391a13f2e6SEdward Tomasz Napierala buf = sctx->buf; 20401a13f2e6SEdward Tomasz Napierala tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 20411a13f2e6SEdward Tomasz Napierala dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 20421a13f2e6SEdward Tomasz Napierala 20431a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_CB) { 20441a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "struct statsblobv1@%p", sb); 20451a13f2e6SEdward Tomasz Napierala if (dump) { 20461a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, " 20471a13f2e6SEdward Tomasz Napierala "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, " 20481a13f2e6SEdward Tomasz Napierala "stats_off=%hu, statsdata_off=%hu", 20491a13f2e6SEdward Tomasz Napierala sb->abi, sb->endian, sb->maxsz, sb->cursz, 20501a13f2e6SEdward Tomasz Napierala sb->created, sb->lastrst, sb->flags, sb->stats_off, 20511a13f2e6SEdward Tomasz Napierala sb->statsdata_off); 20521a13f2e6SEdward Tomasz Napierala } 20531a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", tplhash=%u", sb->tplhash); 20541a13f2e6SEdward Tomasz Napierala } 20551a13f2e6SEdward Tomasz Napierala 20561a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_VOISTAT) { 20571a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id); 20581a13f2e6SEdward Tomasz Napierala if (v->id < 0) 20591a13f2e6SEdward Tomasz Napierala return; 20601a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" : 20611a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name); 20621a13f2e6SEdward Tomasz Napierala if (dump) 20631a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, " 20641a13f2e6SEdward Tomasz Napierala "voistatmaxid=%hhd, stats_off=%hu", v->flags, 20651a13f2e6SEdward Tomasz Napierala vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off); 20661a13f2e6SEdward Tomasz Napierala } 20671a13f2e6SEdward Tomasz Napierala 20681a13f2e6SEdward Tomasz Napierala if (!dump && vs->stype <= 0) 20691a13f2e6SEdward Tomasz Napierala return; 20701a13f2e6SEdward Tomasz Napierala 20711a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot); 20721a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) { 20731a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%hhd", vs->stype); 20741a13f2e6SEdward Tomasz Napierala return; 20751a13f2e6SEdward Tomasz Napierala } else 20761a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype], 20771a13f2e6SEdward Tomasz Napierala vs->errs); 20781a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 20791a13f2e6SEdward Tomasz Napierala if (dump) 20801a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, " 20811a13f2e6SEdward Tomasz Napierala "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype], 20821a13f2e6SEdward Tomasz Napierala vs->dsz, vs->data_off); 20831a13f2e6SEdward Tomasz Napierala 20841a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\t\t\tvoistatdata: "); 20851a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 20861a13f2e6SEdward Tomasz Napierala sctx->fmt, buf, dump); 20871a13f2e6SEdward Tomasz Napierala } 20881a13f2e6SEdward Tomasz Napierala 20891a13f2e6SEdward Tomasz Napierala static void 20901a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 20911a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx *ctx) 20921a13f2e6SEdward Tomasz Napierala { 20931a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx; 20941a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 20951a13f2e6SEdward Tomasz Napierala struct sbuf *buf; 20961a13f2e6SEdward Tomasz Napierala const char *fmtstr; 20971a13f2e6SEdward Tomasz Napierala void *vsd; 20981a13f2e6SEdward Tomasz Napierala uint8_t dump; 20991a13f2e6SEdward Tomasz Napierala 21001a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx; 21011a13f2e6SEdward Tomasz Napierala buf = sctx->buf; 21021a13f2e6SEdward Tomasz Napierala tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 21031a13f2e6SEdward Tomasz Napierala dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 21041a13f2e6SEdward Tomasz Napierala 21051a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_CB) { 21061a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, '{'); 21071a13f2e6SEdward Tomasz Napierala if (dump) { 21081a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu," 21091a13f2e6SEdward Tomasz Napierala "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd," 21101a13f2e6SEdward Tomasz Napierala "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu," 21111a13f2e6SEdward Tomasz Napierala "\"statsdata_off\":%hu,", sb->abi, 21121a13f2e6SEdward Tomasz Napierala sb->endian, sb->maxsz, sb->cursz, sb->created, 21131a13f2e6SEdward Tomasz Napierala sb->lastrst, sb->flags, sb->stats_off, 21141a13f2e6SEdward Tomasz Napierala sb->statsdata_off); 21151a13f2e6SEdward Tomasz Napierala } 21161a13f2e6SEdward Tomasz Napierala 21171a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) 21181a13f2e6SEdward Tomasz Napierala fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{"; 21191a13f2e6SEdward Tomasz Napierala else 21201a13f2e6SEdward Tomasz Napierala fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{"; 21211a13f2e6SEdward Tomasz Napierala 21221a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null", 21231a13f2e6SEdward Tomasz Napierala sb->tplhash); 21241a13f2e6SEdward Tomasz Napierala } 21251a13f2e6SEdward Tomasz Napierala 21261a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_VOISTAT) { 21271a13f2e6SEdward Tomasz Napierala if (dump) { 21281a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot, 21291a13f2e6SEdward Tomasz Napierala v->id); 21301a13f2e6SEdward Tomasz Napierala if (v->id < 0) { 21311a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "},"); 21321a13f2e6SEdward Tomasz Napierala return; 21331a13f2e6SEdward Tomasz Napierala } 21341a13f2e6SEdward Tomasz Napierala 21351a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) 21361a13f2e6SEdward Tomasz Napierala fmtstr = ",\"name\":%s,\"flags\":%hu," 21371a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 21381a13f2e6SEdward Tomasz Napierala "\"stats_off\":%hu,"; 21391a13f2e6SEdward Tomasz Napierala else 21401a13f2e6SEdward Tomasz Napierala fmtstr = ",\"name\":\"%s\",\"flags\":%hu," 21411a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 21421a13f2e6SEdward Tomasz Napierala "\"stats_off\":%hu,"; 21431a13f2e6SEdward Tomasz Napierala 21441a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, tpl_mb ? 21451a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name : "null", v->flags, 21461a13f2e6SEdward Tomasz Napierala vsd_dtype2name[v->dtype], v->voistatmaxid, 21471a13f2e6SEdward Tomasz Napierala v->stats_off); 21481a13f2e6SEdward Tomasz Napierala } else { 21491a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) { 21501a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hd]\":{", v->id); 21511a13f2e6SEdward Tomasz Napierala } else { 21521a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"%s\":{", 21531a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name); 21541a13f2e6SEdward Tomasz Napierala } 21551a13f2e6SEdward Tomasz Napierala } 21561a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "\"stats\":{"); 21571a13f2e6SEdward Tomasz Napierala } 21581a13f2e6SEdward Tomasz Napierala 21591a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 21601a13f2e6SEdward Tomasz Napierala if (dump) { 21611a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot); 21621a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) { 21631a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"stype\":-1},"); 21641a13f2e6SEdward Tomasz Napierala return; 21651a13f2e6SEdward Tomasz Napierala } 21661a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu," 21671a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{", 21681a13f2e6SEdward Tomasz Napierala vs_stype2name[vs->stype], vs->errs, vs->flags, 21691a13f2e6SEdward Tomasz Napierala vsd_dtype2name[vs->dtype], vs->data_off); 21701a13f2e6SEdward Tomasz Napierala } else if (vs->stype > 0) { 21711a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) 21721a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hhd]\":", vs->stype); 21731a13f2e6SEdward Tomasz Napierala else 21741a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]); 21751a13f2e6SEdward Tomasz Napierala } else 21761a13f2e6SEdward Tomasz Napierala return; 21771a13f2e6SEdward Tomasz Napierala 21781a13f2e6SEdward Tomasz Napierala if ((vs->flags & VS_VSDVALID) || dump) { 21791a13f2e6SEdward Tomasz Napierala if (!dump) 21801a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"errs\":%hu,", vs->errs); 21811a13f2e6SEdward Tomasz Napierala /* Simple non-compound VSD types need a key. */ 21821a13f2e6SEdward Tomasz Napierala if (!vsd_compoundtype[vs->dtype]) 21831a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "\"val\":"); 21841a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 21851a13f2e6SEdward Tomasz Napierala sctx->fmt, buf, dump); 21861a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, dump ? "}}" : "}"); 21871a13f2e6SEdward Tomasz Napierala } else 21881a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, dump ? "null}" : "null"); 21891a13f2e6SEdward Tomasz Napierala 21901a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_LAST_VOISTAT) 21911a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "}}"); 21921a13f2e6SEdward Tomasz Napierala 21931a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_LAST_CB) 21941a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "}}"); 21951a13f2e6SEdward Tomasz Napierala else 21961a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ','); 21971a13f2e6SEdward Tomasz Napierala } 21981a13f2e6SEdward Tomasz Napierala 21991a13f2e6SEdward Tomasz Napierala static int 22001a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 22011a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx *ctx) 22021a13f2e6SEdward Tomasz Napierala { 22031a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx; 22041a13f2e6SEdward Tomasz Napierala 22051a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx; 22061a13f2e6SEdward Tomasz Napierala 22071a13f2e6SEdward Tomasz Napierala switch (sctx->fmt) { 22081a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 22091a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_freeform(sb, v, vs, ctx); 22101a13f2e6SEdward Tomasz Napierala break; 22111a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 22121a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_json(sb, v, vs, ctx); 22131a13f2e6SEdward Tomasz Napierala break; 22141a13f2e6SEdward Tomasz Napierala default: 22151a13f2e6SEdward Tomasz Napierala break; 22161a13f2e6SEdward Tomasz Napierala } 22171a13f2e6SEdward Tomasz Napierala 22181a13f2e6SEdward Tomasz Napierala return (sbuf_error(sctx->buf)); 22191a13f2e6SEdward Tomasz Napierala } 22201a13f2e6SEdward Tomasz Napierala 22211a13f2e6SEdward Tomasz Napierala int 22221a13f2e6SEdward Tomasz Napierala stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf, 22231a13f2e6SEdward Tomasz Napierala enum sb_str_fmt fmt, uint32_t flags) 22241a13f2e6SEdward Tomasz Napierala { 22251a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx sctx; 22261a13f2e6SEdward Tomasz Napierala uint32_t iflags; 22271a13f2e6SEdward Tomasz Napierala 22281a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL || 22291a13f2e6SEdward Tomasz Napierala fmt >= SB_STRFMT_NUM_FMTS) 22301a13f2e6SEdward Tomasz Napierala return (EINVAL); 22311a13f2e6SEdward Tomasz Napierala 22321a13f2e6SEdward Tomasz Napierala sctx.buf = buf; 22331a13f2e6SEdward Tomasz Napierala sctx.fmt = fmt; 22341a13f2e6SEdward Tomasz Napierala sctx.flags = flags; 22351a13f2e6SEdward Tomasz Napierala 22361a13f2e6SEdward Tomasz Napierala if (flags & SB_TOSTR_META) { 22371a13f2e6SEdward Tomasz Napierala if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash), 22381a13f2e6SEdward Tomasz Napierala &sctx.tpl)) 22391a13f2e6SEdward Tomasz Napierala return (EINVAL); 22401a13f2e6SEdward Tomasz Napierala } else 22411a13f2e6SEdward Tomasz Napierala sctx.tpl = NULL; 22421a13f2e6SEdward Tomasz Napierala 22431a13f2e6SEdward Tomasz Napierala iflags = 0; 22441a13f2e6SEdward Tomasz Napierala if (flags & SB_TOSTR_OBJDUMP) 22451a13f2e6SEdward Tomasz Napierala iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT); 22461a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags); 22471a13f2e6SEdward Tomasz Napierala 22481a13f2e6SEdward Tomasz Napierala return (sbuf_error(buf)); 22491a13f2e6SEdward Tomasz Napierala } 22501a13f2e6SEdward Tomasz Napierala 22511a13f2e6SEdward Tomasz Napierala static int 22521a13f2e6SEdward Tomasz Napierala stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v, 22531a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx) 22541a13f2e6SEdward Tomasz Napierala { 22551a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx *vctx; 22561a13f2e6SEdward Tomasz Napierala struct sb_visit sbv; 22571a13f2e6SEdward Tomasz Napierala 22581a13f2e6SEdward Tomasz Napierala vctx = ctx->usrctx; 22591a13f2e6SEdward Tomasz Napierala 22601a13f2e6SEdward Tomasz Napierala sbv.tplhash = sb->tplhash; 22611a13f2e6SEdward Tomasz Napierala sbv.voi_id = v->id; 22621a13f2e6SEdward Tomasz Napierala sbv.voi_dtype = v->dtype; 22631a13f2e6SEdward Tomasz Napierala sbv.vs_stype = vs->stype; 22641a13f2e6SEdward Tomasz Napierala sbv.vs_dtype = vs->dtype; 22651a13f2e6SEdward Tomasz Napierala sbv.vs_dsz = vs->dsz; 22661a13f2e6SEdward Tomasz Napierala sbv.vs_data = BLOB_OFFSET(sb, vs->data_off); 22671a13f2e6SEdward Tomasz Napierala sbv.vs_errs = vs->errs; 22681a13f2e6SEdward Tomasz Napierala sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB | 22691a13f2e6SEdward Tomasz Napierala SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT | 22701a13f2e6SEdward Tomasz Napierala SB_IT_LAST_VOISTAT); 22711a13f2e6SEdward Tomasz Napierala 22721a13f2e6SEdward Tomasz Napierala return (vctx->cb(&sbv, vctx->usrctx)); 22731a13f2e6SEdward Tomasz Napierala } 22741a13f2e6SEdward Tomasz Napierala 22751a13f2e6SEdward Tomasz Napierala int 22761a13f2e6SEdward Tomasz Napierala stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func, 22771a13f2e6SEdward Tomasz Napierala void *usrctx) 22781a13f2e6SEdward Tomasz Napierala { 22791a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx vctx; 22801a13f2e6SEdward Tomasz Napierala 22811a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL) 22821a13f2e6SEdward Tomasz Napierala return (EINVAL); 22831a13f2e6SEdward Tomasz Napierala 22841a13f2e6SEdward Tomasz Napierala vctx.cb = func; 22851a13f2e6SEdward Tomasz Napierala vctx.usrctx = usrctx; 22861a13f2e6SEdward Tomasz Napierala 22871a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0); 22881a13f2e6SEdward Tomasz Napierala 22891a13f2e6SEdward Tomasz Napierala return (0); 22901a13f2e6SEdward Tomasz Napierala } 22911a13f2e6SEdward Tomasz Napierala 22921a13f2e6SEdward Tomasz Napierala static int 22931a13f2e6SEdward Tomasz Napierala stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused, 22941a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx __unused) 22951a13f2e6SEdward Tomasz Napierala { 22961a13f2e6SEdward Tomasz Napierala void *vsd; 22971a13f2e6SEdward Tomasz Napierala 22981a13f2e6SEdward Tomasz Napierala if (vs->stype == VS_STYPE_VOISTATE) 22991a13f2e6SEdward Tomasz Napierala return (0); 23001a13f2e6SEdward Tomasz Napierala 23011a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 23021a13f2e6SEdward Tomasz Napierala 23031a13f2e6SEdward Tomasz Napierala /* Perform the stat type's default reset action. */ 23041a13f2e6SEdward Tomasz Napierala switch (vs->stype) { 23051a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM: 23061a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 23071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 23081a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32, 0); 23091a13f2e6SEdward Tomasz Napierala break; 23101a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 23111a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32, 0); 23121a13f2e6SEdward Tomasz Napierala break; 23131a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 23141a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64, 0); 23151a13f2e6SEdward Tomasz Napierala break; 23161a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 23171a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64, 0); 23181a13f2e6SEdward Tomasz Napierala break; 23191a13f2e6SEdward Tomasz Napierala default: 23201a13f2e6SEdward Tomasz Napierala bzero(vsd, vs->dsz); 23211a13f2e6SEdward Tomasz Napierala break; 23221a13f2e6SEdward Tomasz Napierala } 23231a13f2e6SEdward Tomasz Napierala break; 23241a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX: 23251a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 23261a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 23271a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32, 23281a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q32, vsd)->sq32)); 23291a13f2e6SEdward Tomasz Napierala break; 23301a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 23311a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32, 23321a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q32, vsd)->uq32)); 23331a13f2e6SEdward Tomasz Napierala break; 23341a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 23351a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64, 23361a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q64, vsd)->sq64)); 23371a13f2e6SEdward Tomasz Napierala break; 23381a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 23391a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64, 23401a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q64, vsd)->uq64)); 23411a13f2e6SEdward Tomasz Napierala break; 23421a13f2e6SEdward Tomasz Napierala default: 23431a13f2e6SEdward Tomasz Napierala memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype], 23441a13f2e6SEdward Tomasz Napierala vs->dsz); 23451a13f2e6SEdward Tomasz Napierala break; 23461a13f2e6SEdward Tomasz Napierala } 23471a13f2e6SEdward Tomasz Napierala break; 23481a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN: 23491a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 23501a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 23511a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32, 23521a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q32, vsd)->sq32)); 23531a13f2e6SEdward Tomasz Napierala break; 23541a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 23551a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32, 23561a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q32, vsd)->uq32)); 23571a13f2e6SEdward Tomasz Napierala break; 23581a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 23591a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64, 23601a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q64, vsd)->sq64)); 23611a13f2e6SEdward Tomasz Napierala break; 23621a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 23631a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64, 23641a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q64, vsd)->uq64)); 23651a13f2e6SEdward Tomasz Napierala break; 23661a13f2e6SEdward Tomasz Napierala default: 23671a13f2e6SEdward Tomasz Napierala memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype], 23681a13f2e6SEdward Tomasz Napierala vs->dsz); 23691a13f2e6SEdward Tomasz Napierala break; 23701a13f2e6SEdward Tomasz Napierala } 23711a13f2e6SEdward Tomasz Napierala break; 23721a13f2e6SEdward Tomasz Napierala case VS_STYPE_HIST: 23731a13f2e6SEdward Tomasz Napierala { 23741a13f2e6SEdward Tomasz Napierala /* Reset bucket counts. */ 23751a13f2e6SEdward Tomasz Napierala struct voistatdata_hist *hist; 23761a13f2e6SEdward Tomasz Napierala int i, is32bit; 23771a13f2e6SEdward Tomasz Napierala uint16_t nbkts; 23781a13f2e6SEdward Tomasz Napierala 23791a13f2e6SEdward Tomasz Napierala hist = VSD(hist, vsd); 23801a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 23811a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 23821a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 23831a13f2e6SEdward Tomasz Napierala is32bit = 1; 23841a13f2e6SEdward Tomasz Napierala break; 23851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 23861a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 23871a13f2e6SEdward Tomasz Napierala is32bit = 1; 23881a13f2e6SEdward Tomasz Napierala break; 23891a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 23901a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 23911a13f2e6SEdward Tomasz Napierala is32bit = 1; 23921a13f2e6SEdward Tomasz Napierala break; 23931a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 23941a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 23951a13f2e6SEdward Tomasz Napierala is32bit = 0; 23961a13f2e6SEdward Tomasz Napierala break; 23971a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 23981a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 23991a13f2e6SEdward Tomasz Napierala is32bit = 0; 24001a13f2e6SEdward Tomasz Napierala break; 24011a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 24021a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 24031a13f2e6SEdward Tomasz Napierala is32bit = 0; 24041a13f2e6SEdward Tomasz Napierala break; 24051a13f2e6SEdward Tomasz Napierala default: 24061a13f2e6SEdward Tomasz Napierala return (0); 24071a13f2e6SEdward Tomasz Napierala } 24081a13f2e6SEdward Tomasz Napierala 24091a13f2e6SEdward Tomasz Napierala bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob), 24101a13f2e6SEdward Tomasz Napierala is32bit ? sizeof(uint32_t) : sizeof(uint64_t)); 24111a13f2e6SEdward Tomasz Napierala for (i = nbkts - 1; i >= 0; i--) { 24121a13f2e6SEdward Tomasz Napierala bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, 24131a13f2e6SEdward Tomasz Napierala bkts[i].cnt), is32bit ? sizeof(uint32_t) : 24141a13f2e6SEdward Tomasz Napierala sizeof(uint64_t)); 24151a13f2e6SEdward Tomasz Napierala } 24161a13f2e6SEdward Tomasz Napierala break; 24171a13f2e6SEdward Tomasz Napierala } 24181a13f2e6SEdward Tomasz Napierala case VS_STYPE_TDGST: 24191a13f2e6SEdward Tomasz Napierala { 24201a13f2e6SEdward Tomasz Napierala /* Reset sample count centroids array/tree. */ 24211a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst; 24221a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 24231a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 24241a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 24251a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 24261a13f2e6SEdward Tomasz Napierala 24271a13f2e6SEdward Tomasz Napierala tdgst = VSD(tdgst, vsd); 24281a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 24291a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 24301a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt = 0; 24311a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->compcnt = 0; 24321a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 24331a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd32, ctdlnk, ctd32tree, 24341a13f2e6SEdward Tomasz Napierala ARB_MAXNODES(ctd32tree)) { 24351a13f2e6SEdward Tomasz Napierala ctd32->cnt = 0; 24361a13f2e6SEdward Tomasz Napierala Q_SIFVAL(ctd32->mu, 0); 24371a13f2e6SEdward Tomasz Napierala } 24381a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 24391a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 24401a13f2e6SEdward Tomasz Napierala #endif 24411a13f2e6SEdward Tomasz Napierala break; 24421a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 24431a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt = 0; 24441a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->compcnt = 0; 24451a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 24461a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd64, ctdlnk, ctd64tree, 24471a13f2e6SEdward Tomasz Napierala ARB_MAXNODES(ctd64tree)) { 24481a13f2e6SEdward Tomasz Napierala ctd64->cnt = 0; 24491a13f2e6SEdward Tomasz Napierala Q_SIFVAL(ctd64->mu, 0); 24501a13f2e6SEdward Tomasz Napierala } 24511a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 24521a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 24531a13f2e6SEdward Tomasz Napierala #endif 24541a13f2e6SEdward Tomasz Napierala break; 24551a13f2e6SEdward Tomasz Napierala default: 24561a13f2e6SEdward Tomasz Napierala return (0); 24571a13f2e6SEdward Tomasz Napierala } 24581a13f2e6SEdward Tomasz Napierala break; 24591a13f2e6SEdward Tomasz Napierala } 24601a13f2e6SEdward Tomasz Napierala default: 24611a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 24621a13f2e6SEdward Tomasz Napierala break; 24631a13f2e6SEdward Tomasz Napierala } 24641a13f2e6SEdward Tomasz Napierala 24651a13f2e6SEdward Tomasz Napierala vs->errs = 0; 24661a13f2e6SEdward Tomasz Napierala vs->flags &= ~VS_VSDVALID; 24671a13f2e6SEdward Tomasz Napierala 24681a13f2e6SEdward Tomasz Napierala return (0); 24691a13f2e6SEdward Tomasz Napierala } 24701a13f2e6SEdward Tomasz Napierala 24711a13f2e6SEdward Tomasz Napierala int 24721a13f2e6SEdward Tomasz Napierala stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz, 24731a13f2e6SEdward Tomasz Napierala struct statsblobv1 *src, uint32_t flags) 24741a13f2e6SEdward Tomasz Napierala { 24751a13f2e6SEdward Tomasz Napierala int error; 24761a13f2e6SEdward Tomasz Napierala 24771a13f2e6SEdward Tomasz Napierala if (src != NULL && src->abi == STATS_ABI_V1) { 24781a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_clone(dst, dstmaxsz, src, flags); 24791a13f2e6SEdward Tomasz Napierala if (!error) { 24801a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_RSTSRC) { 24811a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(src, 24821a13f2e6SEdward Tomasz Napierala stats_v1_icb_reset_voistat, NULL, 0); 24831a13f2e6SEdward Tomasz Napierala src->lastrst = stats_sbinuptime(); 24841a13f2e6SEdward Tomasz Napierala } 24851a13f2e6SEdward Tomasz Napierala stats_v1_blob_finalise(*dst); 24861a13f2e6SEdward Tomasz Napierala } 24871a13f2e6SEdward Tomasz Napierala } else 24881a13f2e6SEdward Tomasz Napierala error = EINVAL; 24891a13f2e6SEdward Tomasz Napierala 24901a13f2e6SEdward Tomasz Napierala return (error); 24911a13f2e6SEdward Tomasz Napierala } 24921a13f2e6SEdward Tomasz Napierala 24931a13f2e6SEdward Tomasz Napierala static inline int 24941a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused, 24951a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd) 24961a13f2e6SEdward Tomasz Napierala { 24971a13f2e6SEdward Tomasz Napierala int error; 24981a13f2e6SEdward Tomasz Napierala 24991a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES, 25001a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype)); 25011a13f2e6SEdward Tomasz Napierala 25021a13f2e6SEdward Tomasz Napierala error = 0; 25031a13f2e6SEdward Tomasz Napierala 25041a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 25051a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 25061a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->s32 < voival->int32.s32) { 25071a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 = voival->int32.s32; 25081a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25091a13f2e6SEdward Tomasz Napierala } 25101a13f2e6SEdward Tomasz Napierala break; 25111a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 25121a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->u32 < voival->int32.u32) { 25131a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 = voival->int32.u32; 25141a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25151a13f2e6SEdward Tomasz Napierala } 25161a13f2e6SEdward Tomasz Napierala break; 25171a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 25181a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->s64 < voival->int64.s64) { 25191a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 = voival->int64.s64; 25201a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25211a13f2e6SEdward Tomasz Napierala } 25221a13f2e6SEdward Tomasz Napierala break; 25231a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 25241a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->u64 < voival->int64.u64) { 25251a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 = voival->int64.u64; 25261a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25271a13f2e6SEdward Tomasz Napierala } 25281a13f2e6SEdward Tomasz Napierala break; 25291a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 25301a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->slong < voival->intlong.slong) { 25311a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong = voival->intlong.slong; 25321a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25331a13f2e6SEdward Tomasz Napierala } 25341a13f2e6SEdward Tomasz Napierala break; 25351a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 25361a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) { 25371a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong = voival->intlong.ulong; 25381a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25391a13f2e6SEdward Tomasz Napierala } 25401a13f2e6SEdward Tomasz Napierala break; 25411a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 25421a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 25431a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 25441a13f2e6SEdward Tomasz Napierala voival->q32.sq32)))) { 25451a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25461a13f2e6SEdward Tomasz Napierala } 25471a13f2e6SEdward Tomasz Napierala break; 25481a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 25491a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 25501a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 25511a13f2e6SEdward Tomasz Napierala voival->q32.uq32)))) { 25521a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25531a13f2e6SEdward Tomasz Napierala } 25541a13f2e6SEdward Tomasz Napierala break; 25551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 25561a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 25571a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 25581a13f2e6SEdward Tomasz Napierala voival->q64.sq64)))) { 25591a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25601a13f2e6SEdward Tomasz Napierala } 25611a13f2e6SEdward Tomasz Napierala break; 25621a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 25631a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 25641a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 25651a13f2e6SEdward Tomasz Napierala voival->q64.uq64)))) { 25661a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25671a13f2e6SEdward Tomasz Napierala } 25681a13f2e6SEdward Tomasz Napierala break; 25691a13f2e6SEdward Tomasz Napierala default: 25701a13f2e6SEdward Tomasz Napierala error = EINVAL; 25711a13f2e6SEdward Tomasz Napierala break; 25721a13f2e6SEdward Tomasz Napierala } 25731a13f2e6SEdward Tomasz Napierala 25741a13f2e6SEdward Tomasz Napierala return (error); 25751a13f2e6SEdward Tomasz Napierala } 25761a13f2e6SEdward Tomasz Napierala 25771a13f2e6SEdward Tomasz Napierala static inline int 25781a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused, 25791a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd) 25801a13f2e6SEdward Tomasz Napierala { 25811a13f2e6SEdward Tomasz Napierala int error; 25821a13f2e6SEdward Tomasz Napierala 25831a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES, 25841a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype)); 25851a13f2e6SEdward Tomasz Napierala 25861a13f2e6SEdward Tomasz Napierala error = 0; 25871a13f2e6SEdward Tomasz Napierala 25881a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 25891a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 25901a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->s32 > voival->int32.s32) { 25911a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 = voival->int32.s32; 25921a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25931a13f2e6SEdward Tomasz Napierala } 25941a13f2e6SEdward Tomasz Napierala break; 25951a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 25961a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->u32 > voival->int32.u32) { 25971a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 = voival->int32.u32; 25981a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 25991a13f2e6SEdward Tomasz Napierala } 26001a13f2e6SEdward Tomasz Napierala break; 26011a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 26021a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->s64 > voival->int64.s64) { 26031a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 = voival->int64.s64; 26041a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26051a13f2e6SEdward Tomasz Napierala } 26061a13f2e6SEdward Tomasz Napierala break; 26071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 26081a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->u64 > voival->int64.u64) { 26091a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 = voival->int64.u64; 26101a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26111a13f2e6SEdward Tomasz Napierala } 26121a13f2e6SEdward Tomasz Napierala break; 26131a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 26141a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->slong > voival->intlong.slong) { 26151a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong = voival->intlong.slong; 26161a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26171a13f2e6SEdward Tomasz Napierala } 26181a13f2e6SEdward Tomasz Napierala break; 26191a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 26201a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) { 26211a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong = voival->intlong.ulong; 26221a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26231a13f2e6SEdward Tomasz Napierala } 26241a13f2e6SEdward Tomasz Napierala break; 26251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 26261a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 26271a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 26281a13f2e6SEdward Tomasz Napierala voival->q32.sq32)))) { 26291a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26301a13f2e6SEdward Tomasz Napierala } 26311a13f2e6SEdward Tomasz Napierala break; 26321a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 26331a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 26341a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 26351a13f2e6SEdward Tomasz Napierala voival->q32.uq32)))) { 26361a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26371a13f2e6SEdward Tomasz Napierala } 26381a13f2e6SEdward Tomasz Napierala break; 26391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 26401a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 26411a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 26421a13f2e6SEdward Tomasz Napierala voival->q64.sq64)))) { 26431a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26441a13f2e6SEdward Tomasz Napierala } 26451a13f2e6SEdward Tomasz Napierala break; 26461a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 26471a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 26481a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 26491a13f2e6SEdward Tomasz Napierala voival->q64.uq64)))) { 26501a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 26511a13f2e6SEdward Tomasz Napierala } 26521a13f2e6SEdward Tomasz Napierala break; 26531a13f2e6SEdward Tomasz Napierala default: 26541a13f2e6SEdward Tomasz Napierala error = EINVAL; 26551a13f2e6SEdward Tomasz Napierala break; 26561a13f2e6SEdward Tomasz Napierala } 26571a13f2e6SEdward Tomasz Napierala 26581a13f2e6SEdward Tomasz Napierala return (error); 26591a13f2e6SEdward Tomasz Napierala } 26601a13f2e6SEdward Tomasz Napierala 26611a13f2e6SEdward Tomasz Napierala static inline int 26621a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused, 26631a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd) 26641a13f2e6SEdward Tomasz Napierala { 26651a13f2e6SEdward Tomasz Napierala int error; 26661a13f2e6SEdward Tomasz Napierala 26671a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES, 26681a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype)); 26691a13f2e6SEdward Tomasz Napierala 26701a13f2e6SEdward Tomasz Napierala error = 0; 26711a13f2e6SEdward Tomasz Napierala 26721a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 26731a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 26741a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 += voival->int32.s32; 26751a13f2e6SEdward Tomasz Napierala break; 26761a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 26771a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 += voival->int32.u32; 26781a13f2e6SEdward Tomasz Napierala break; 26791a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 26801a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 += voival->int64.s64; 26811a13f2e6SEdward Tomasz Napierala break; 26821a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 26831a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 += voival->int64.u64; 26841a13f2e6SEdward Tomasz Napierala break; 26851a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 26861a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong += voival->intlong.slong; 26871a13f2e6SEdward Tomasz Napierala break; 26881a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 26891a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong += voival->intlong.ulong; 26901a13f2e6SEdward Tomasz Napierala break; 26911a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 26921a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32); 26931a13f2e6SEdward Tomasz Napierala break; 26941a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 26951a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32); 26961a13f2e6SEdward Tomasz Napierala break; 26971a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 26981a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64); 26991a13f2e6SEdward Tomasz Napierala break; 27001a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 27011a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64); 27021a13f2e6SEdward Tomasz Napierala break; 27031a13f2e6SEdward Tomasz Napierala default: 27041a13f2e6SEdward Tomasz Napierala error = EINVAL; 27051a13f2e6SEdward Tomasz Napierala break; 27061a13f2e6SEdward Tomasz Napierala } 27071a13f2e6SEdward Tomasz Napierala 27081a13f2e6SEdward Tomasz Napierala if (!error) 27091a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 27101a13f2e6SEdward Tomasz Napierala 27111a13f2e6SEdward Tomasz Napierala return (error); 27121a13f2e6SEdward Tomasz Napierala } 27131a13f2e6SEdward Tomasz Napierala 27141a13f2e6SEdward Tomasz Napierala static inline int 27151a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival, 27161a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct voistatdata_hist *hist) 27171a13f2e6SEdward Tomasz Napierala { 27181a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric *bkt_lb, *bkt_ub; 27191a13f2e6SEdward Tomasz Napierala uint64_t *oob64, *cnt64; 27201a13f2e6SEdward Tomasz Napierala uint32_t *oob32, *cnt32; 27211a13f2e6SEdward Tomasz Napierala int error, i, found, is32bit, has_ub, eq_only; 27221a13f2e6SEdward Tomasz Napierala 27231a13f2e6SEdward Tomasz Napierala error = 0; 27241a13f2e6SEdward Tomasz Napierala 27251a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 27261a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 27271a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 27281a13f2e6SEdward Tomasz Napierala is32bit = 1; 27291a13f2e6SEdward Tomasz Napierala has_ub = eq_only = 0; 27301a13f2e6SEdward Tomasz Napierala oob32 = &VSD(crhist32, hist)->oob; 27311a13f2e6SEdward Tomasz Napierala break; 27321a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 27331a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 27341a13f2e6SEdward Tomasz Napierala is32bit = has_ub = 1; 27351a13f2e6SEdward Tomasz Napierala eq_only = 0; 27361a13f2e6SEdward Tomasz Napierala oob32 = &VSD(drhist32, hist)->oob; 27371a13f2e6SEdward Tomasz Napierala break; 27381a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 27391a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 27401a13f2e6SEdward Tomasz Napierala is32bit = eq_only = 1; 27411a13f2e6SEdward Tomasz Napierala has_ub = 0; 27421a13f2e6SEdward Tomasz Napierala oob32 = &VSD(dvhist32, hist)->oob; 27431a13f2e6SEdward Tomasz Napierala break; 27441a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 27451a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 27461a13f2e6SEdward Tomasz Napierala is32bit = has_ub = eq_only = 0; 27471a13f2e6SEdward Tomasz Napierala oob64 = &VSD(crhist64, hist)->oob; 27481a13f2e6SEdward Tomasz Napierala break; 27491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 27501a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 27511a13f2e6SEdward Tomasz Napierala is32bit = eq_only = 0; 27521a13f2e6SEdward Tomasz Napierala has_ub = 1; 27531a13f2e6SEdward Tomasz Napierala oob64 = &VSD(drhist64, hist)->oob; 27541a13f2e6SEdward Tomasz Napierala break; 27551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 27561a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 27571a13f2e6SEdward Tomasz Napierala is32bit = has_ub = 0; 27581a13f2e6SEdward Tomasz Napierala eq_only = 1; 27591a13f2e6SEdward Tomasz Napierala oob64 = &VSD(dvhist64, hist)->oob; 27601a13f2e6SEdward Tomasz Napierala break; 27611a13f2e6SEdward Tomasz Napierala default: 27621a13f2e6SEdward Tomasz Napierala return (EINVAL); 27631a13f2e6SEdward Tomasz Napierala } 27641a13f2e6SEdward Tomasz Napierala i--; /* Adjust for 0-based array index. */ 27651a13f2e6SEdward Tomasz Napierala 27661a13f2e6SEdward Tomasz Napierala /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */ 27671a13f2e6SEdward Tomasz Napierala for (found = 0; i >= 0 && !found; i--) { 27681a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 27691a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 27701a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(crhist32, hist)->bkts[i].lb; 27711a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(crhist32, hist)->bkts[i].cnt; 27721a13f2e6SEdward Tomasz Napierala break; 27731a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 27741a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(drhist32, hist)->bkts[i].lb; 27751a13f2e6SEdward Tomasz Napierala bkt_ub = &VSD(drhist32, hist)->bkts[i].ub; 27761a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(drhist32, hist)->bkts[i].cnt; 27771a13f2e6SEdward Tomasz Napierala break; 27781a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 27791a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(dvhist32, hist)->bkts[i].val; 27801a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt; 27811a13f2e6SEdward Tomasz Napierala break; 27821a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 27831a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(crhist64, hist)->bkts[i].lb; 27841a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(crhist64, hist)->bkts[i].cnt; 27851a13f2e6SEdward Tomasz Napierala break; 27861a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 27871a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(drhist64, hist)->bkts[i].lb; 27881a13f2e6SEdward Tomasz Napierala bkt_ub = &VSD(drhist64, hist)->bkts[i].ub; 27891a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(drhist64, hist)->bkts[i].cnt; 27901a13f2e6SEdward Tomasz Napierala break; 27911a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 27921a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(dvhist64, hist)->bkts[i].val; 27931a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt; 27941a13f2e6SEdward Tomasz Napierala break; 27951a13f2e6SEdward Tomasz Napierala default: 27961a13f2e6SEdward Tomasz Napierala return (EINVAL); 27971a13f2e6SEdward Tomasz Napierala } 27981a13f2e6SEdward Tomasz Napierala 27991a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 28001a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 28011a13f2e6SEdward Tomasz Napierala if (voival->int32.s32 >= bkt_lb->int32.s32) { 28021a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int32.s32 == 28031a13f2e6SEdward Tomasz Napierala bkt_lb->int32.s32) || 28041a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28051a13f2e6SEdward Tomasz Napierala voival->int32.s32 < bkt_ub->int32.s32))) 28061a13f2e6SEdward Tomasz Napierala found = 1; 28071a13f2e6SEdward Tomasz Napierala } 28081a13f2e6SEdward Tomasz Napierala break; 28091a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 28101a13f2e6SEdward Tomasz Napierala if (voival->int32.u32 >= bkt_lb->int32.u32) { 28111a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int32.u32 == 28121a13f2e6SEdward Tomasz Napierala bkt_lb->int32.u32) || 28131a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28141a13f2e6SEdward Tomasz Napierala voival->int32.u32 < bkt_ub->int32.u32))) 28151a13f2e6SEdward Tomasz Napierala found = 1; 28161a13f2e6SEdward Tomasz Napierala } 28171a13f2e6SEdward Tomasz Napierala break; 28181a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 28191a13f2e6SEdward Tomasz Napierala if (voival->int64.s64 >= bkt_lb->int64.s64) 28201a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int64.s64 == 28211a13f2e6SEdward Tomasz Napierala bkt_lb->int64.s64) || 28221a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28231a13f2e6SEdward Tomasz Napierala voival->int64.s64 < bkt_ub->int64.s64))) 28241a13f2e6SEdward Tomasz Napierala found = 1; 28251a13f2e6SEdward Tomasz Napierala break; 28261a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 28271a13f2e6SEdward Tomasz Napierala if (voival->int64.u64 >= bkt_lb->int64.u64) 28281a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int64.u64 == 28291a13f2e6SEdward Tomasz Napierala bkt_lb->int64.u64) || 28301a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28311a13f2e6SEdward Tomasz Napierala voival->int64.u64 < bkt_ub->int64.u64))) 28321a13f2e6SEdward Tomasz Napierala found = 1; 28331a13f2e6SEdward Tomasz Napierala break; 28341a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 28351a13f2e6SEdward Tomasz Napierala if (voival->intlong.slong >= bkt_lb->intlong.slong) 28361a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->intlong.slong == 28371a13f2e6SEdward Tomasz Napierala bkt_lb->intlong.slong) || 28381a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28391a13f2e6SEdward Tomasz Napierala voival->intlong.slong < 28401a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.slong))) 28411a13f2e6SEdward Tomasz Napierala found = 1; 28421a13f2e6SEdward Tomasz Napierala break; 28431a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 28441a13f2e6SEdward Tomasz Napierala if (voival->intlong.ulong >= bkt_lb->intlong.ulong) 28451a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->intlong.ulong == 28461a13f2e6SEdward Tomasz Napierala bkt_lb->intlong.ulong) || 28471a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28481a13f2e6SEdward Tomasz Napierala voival->intlong.ulong < 28491a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.ulong))) 28501a13f2e6SEdward Tomasz Napierala found = 1; 28511a13f2e6SEdward Tomasz Napierala break; 28521a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 28531a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32)) 28541a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q32.sq32, 28551a13f2e6SEdward Tomasz Napierala bkt_lb->q32.sq32)) || 28561a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28571a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q32.sq32, 28581a13f2e6SEdward Tomasz Napierala bkt_ub->q32.sq32)))) 28591a13f2e6SEdward Tomasz Napierala found = 1; 28601a13f2e6SEdward Tomasz Napierala break; 28611a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 28621a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32)) 28631a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q32.uq32, 28641a13f2e6SEdward Tomasz Napierala bkt_lb->q32.uq32)) || 28651a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28661a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q32.uq32, 28671a13f2e6SEdward Tomasz Napierala bkt_ub->q32.uq32)))) 28681a13f2e6SEdward Tomasz Napierala found = 1; 28691a13f2e6SEdward Tomasz Napierala break; 28701a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 28711a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64)) 28721a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q64.sq64, 28731a13f2e6SEdward Tomasz Napierala bkt_lb->q64.sq64)) || 28741a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28751a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q64.sq64, 28761a13f2e6SEdward Tomasz Napierala bkt_ub->q64.sq64)))) 28771a13f2e6SEdward Tomasz Napierala found = 1; 28781a13f2e6SEdward Tomasz Napierala break; 28791a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 28801a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64)) 28811a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q64.uq64, 28821a13f2e6SEdward Tomasz Napierala bkt_lb->q64.uq64)) || 28831a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 28841a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q64.uq64, 28851a13f2e6SEdward Tomasz Napierala bkt_ub->q64.uq64)))) 28861a13f2e6SEdward Tomasz Napierala found = 1; 28871a13f2e6SEdward Tomasz Napierala break; 28881a13f2e6SEdward Tomasz Napierala default: 28891a13f2e6SEdward Tomasz Napierala break; 28901a13f2e6SEdward Tomasz Napierala } 28911a13f2e6SEdward Tomasz Napierala } 28921a13f2e6SEdward Tomasz Napierala 28931a13f2e6SEdward Tomasz Napierala if (found) { 28941a13f2e6SEdward Tomasz Napierala if (is32bit) 28951a13f2e6SEdward Tomasz Napierala *cnt32 += 1; 28961a13f2e6SEdward Tomasz Napierala else 28971a13f2e6SEdward Tomasz Napierala *cnt64 += 1; 28981a13f2e6SEdward Tomasz Napierala } else { 28991a13f2e6SEdward Tomasz Napierala if (is32bit) 29001a13f2e6SEdward Tomasz Napierala *oob32 += 1; 29011a13f2e6SEdward Tomasz Napierala else 29021a13f2e6SEdward Tomasz Napierala *oob64 += 1; 29031a13f2e6SEdward Tomasz Napierala } 29041a13f2e6SEdward Tomasz Napierala 29051a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 29061a13f2e6SEdward Tomasz Napierala return (error); 29071a13f2e6SEdward Tomasz Napierala } 29081a13f2e6SEdward Tomasz Napierala 29091a13f2e6SEdward Tomasz Napierala static inline int 29101a13f2e6SEdward Tomasz Napierala stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype, 29111a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst, int attempt) 29121a13f2e6SEdward Tomasz Napierala { 29131a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 29141a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 29151a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 29161a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 29171a13f2e6SEdward Tomasz Napierala uint64_t ebits, idxmask; 29181a13f2e6SEdward Tomasz Napierala uint32_t bitsperidx, nebits; 29191a13f2e6SEdward Tomasz Napierala int error, idx, is32bit, maxctds, remctds, tmperr; 29201a13f2e6SEdward Tomasz Napierala 29211a13f2e6SEdward Tomasz Napierala error = 0; 29221a13f2e6SEdward Tomasz Napierala 29231a13f2e6SEdward Tomasz Napierala switch (vs_dtype) { 29241a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 29251a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 29261a13f2e6SEdward Tomasz Napierala if (!ARB_FULL(ctd32tree)) 29271a13f2e6SEdward Tomasz Napierala return (0); 29281a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->compcnt++; 29291a13f2e6SEdward Tomasz Napierala maxctds = remctds = ARB_MAXNODES(ctd32tree); 29301a13f2e6SEdward Tomasz Napierala ARB_RESET_TREE(ctd32tree, ctdth32, maxctds); 29311a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt = 0; 29321a13f2e6SEdward Tomasz Napierala is32bit = 1; 29331a13f2e6SEdward Tomasz Napierala ctd64tree = NULL; 29341a13f2e6SEdward Tomasz Napierala ctd64 = NULL; 29351a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 29361a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 29371a13f2e6SEdward Tomasz Napierala #endif 29381a13f2e6SEdward Tomasz Napierala break; 29391a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 29401a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 29411a13f2e6SEdward Tomasz Napierala if (!ARB_FULL(ctd64tree)) 29421a13f2e6SEdward Tomasz Napierala return (0); 29431a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->compcnt++; 29441a13f2e6SEdward Tomasz Napierala maxctds = remctds = ARB_MAXNODES(ctd64tree); 29451a13f2e6SEdward Tomasz Napierala ARB_RESET_TREE(ctd64tree, ctdth64, maxctds); 29461a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt = 0; 29471a13f2e6SEdward Tomasz Napierala is32bit = 0; 29481a13f2e6SEdward Tomasz Napierala ctd32tree = NULL; 29491a13f2e6SEdward Tomasz Napierala ctd32 = NULL; 29501a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 29511a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 29521a13f2e6SEdward Tomasz Napierala #endif 29531a13f2e6SEdward Tomasz Napierala break; 29541a13f2e6SEdward Tomasz Napierala default: 29551a13f2e6SEdward Tomasz Napierala return (EINVAL); 29561a13f2e6SEdward Tomasz Napierala } 29571a13f2e6SEdward Tomasz Napierala 29581a13f2e6SEdward Tomasz Napierala /* 29591a13f2e6SEdward Tomasz Napierala * Rebuild the t-digest ARB by pseudorandomly selecting centroids and 29601a13f2e6SEdward Tomasz Napierala * re-inserting the mu/cnt of each as a value and corresponding weight. 29611a13f2e6SEdward Tomasz Napierala */ 29621a13f2e6SEdward Tomasz Napierala 2963f3bae413SConrad Meyer /* 2964f3bae413SConrad Meyer * XXXCEM: random(9) is currently rand(3), not random(3). rand(3) 2965f3bae413SConrad Meyer * RAND_MAX happens to be approximately 31 bits (range [0, 2966f3bae413SConrad Meyer * 0x7ffffffd]), so the math kinda works out. When/if this portion of 2967f3bae413SConrad Meyer * the code is compiled in userspace, it gets the random(3) behavior, 2968f3bae413SConrad Meyer * which has expected range [0, 0x7fffffff]. 2969f3bae413SConrad Meyer */ 2970f3bae413SConrad Meyer #define bitsperrand 31 29711a13f2e6SEdward Tomasz Napierala ebits = 0; 29721a13f2e6SEdward Tomasz Napierala nebits = 0; 29731a13f2e6SEdward Tomasz Napierala bitsperidx = fls(maxctds); 29741a13f2e6SEdward Tomasz Napierala KASSERT(bitsperidx <= sizeof(ebits) << 3, 29751a13f2e6SEdward Tomasz Napierala ("%s: bitsperidx=%d, ebits=%d", 29761a13f2e6SEdward Tomasz Napierala __func__, bitsperidx, (int)(sizeof(ebits) << 3))); 29771a13f2e6SEdward Tomasz Napierala idxmask = (UINT64_C(1) << bitsperidx) - 1; 29781a13f2e6SEdward Tomasz Napierala 29791a13f2e6SEdward Tomasz Napierala /* Initialise the free list with randomised centroid indices. */ 29801a13f2e6SEdward Tomasz Napierala for (; remctds > 0; remctds--) { 29811a13f2e6SEdward Tomasz Napierala while (nebits < bitsperidx) { 29821a13f2e6SEdward Tomasz Napierala ebits |= ((uint64_t)random()) << nebits; 29831a13f2e6SEdward Tomasz Napierala nebits += bitsperrand; 29841a13f2e6SEdward Tomasz Napierala if (nebits > (sizeof(ebits) << 3)) 29851a13f2e6SEdward Tomasz Napierala nebits = sizeof(ebits) << 3; 29861a13f2e6SEdward Tomasz Napierala } 29871a13f2e6SEdward Tomasz Napierala idx = ebits & idxmask; 29881a13f2e6SEdward Tomasz Napierala nebits -= bitsperidx; 29891a13f2e6SEdward Tomasz Napierala ebits >>= bitsperidx; 29901a13f2e6SEdward Tomasz Napierala 29911a13f2e6SEdward Tomasz Napierala /* 29921a13f2e6SEdward Tomasz Napierala * Select the next centroid to put on the ARB free list. We 29931a13f2e6SEdward Tomasz Napierala * start with the centroid at our randomly selected array index, 29941a13f2e6SEdward Tomasz Napierala * and work our way forwards until finding one (the latter 29951a13f2e6SEdward Tomasz Napierala * aspect reduces re-insertion randomness, but is good enough). 29961a13f2e6SEdward Tomasz Napierala */ 29971a13f2e6SEdward Tomasz Napierala do { 29981a13f2e6SEdward Tomasz Napierala if (idx >= maxctds) 29991a13f2e6SEdward Tomasz Napierala idx %= maxctds; 30001a13f2e6SEdward Tomasz Napierala 30011a13f2e6SEdward Tomasz Napierala if (is32bit) 30021a13f2e6SEdward Tomasz Napierala ctd32 = ARB_NODE(ctd32tree, idx); 30031a13f2e6SEdward Tomasz Napierala else 30041a13f2e6SEdward Tomasz Napierala ctd64 = ARB_NODE(ctd64tree, idx); 30051a13f2e6SEdward Tomasz Napierala } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) : 30061a13f2e6SEdward Tomasz Napierala ARB_ISFREE(ctd64, ctdlnk)) && ++idx); 30071a13f2e6SEdward Tomasz Napierala 30081a13f2e6SEdward Tomasz Napierala /* Put the centroid on the ARB free list. */ 30091a13f2e6SEdward Tomasz Napierala if (is32bit) 30101a13f2e6SEdward Tomasz Napierala ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk); 30111a13f2e6SEdward Tomasz Napierala else 30121a13f2e6SEdward Tomasz Napierala ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk); 30131a13f2e6SEdward Tomasz Napierala } 30141a13f2e6SEdward Tomasz Napierala 30151a13f2e6SEdward Tomasz Napierala /* 30161a13f2e6SEdward Tomasz Napierala * The free list now contains the randomised indices of every centroid. 30171a13f2e6SEdward Tomasz Napierala * Walk the free list from start to end, re-inserting each centroid's 30181a13f2e6SEdward Tomasz Napierala * mu/cnt. The tdgst_add() call may or may not consume the free centroid 30191a13f2e6SEdward Tomasz Napierala * we re-insert values from during each loop iteration, so we must latch 30201a13f2e6SEdward Tomasz Napierala * the index of the next free list centroid before the re-insertion 30211a13f2e6SEdward Tomasz Napierala * call. The previous loop above should have left the centroid pointer 30221a13f2e6SEdward Tomasz Napierala * pointing to the element at the head of the free list. 30231a13f2e6SEdward Tomasz Napierala */ 30241a13f2e6SEdward Tomasz Napierala KASSERT((is32bit ? 30251a13f2e6SEdward Tomasz Napierala ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) : 30261a13f2e6SEdward Tomasz Napierala ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)), 30271a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__, 30281a13f2e6SEdward Tomasz Napierala (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 30291a13f2e6SEdward Tomasz Napierala remctds = maxctds; 30301a13f2e6SEdward Tomasz Napierala while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) { 30311a13f2e6SEdward Tomasz Napierala tmperr = 0; 30321a13f2e6SEdward Tomasz Napierala if (is32bit) { 30331a13f2e6SEdward Tomasz Napierala s64q_t x; 30341a13f2e6SEdward Tomasz Napierala 30351a13f2e6SEdward Tomasz Napierala idx = ARB_NEXTFREEIDX(ctd32, ctdlnk); 30361a13f2e6SEdward Tomasz Napierala /* Cloning a s32q_t into a s64q_t should never fail. */ 30371a13f2e6SEdward Tomasz Napierala tmperr = Q_QCLONEQ(&x, ctd32->mu); 30381a13f2e6SEdward Tomasz Napierala tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add( 30391a13f2e6SEdward Tomasz Napierala vs_dtype, tdgst, x, ctd32->cnt, attempt); 30401a13f2e6SEdward Tomasz Napierala ctd32 = ARB_NODE(ctd32tree, idx); 30411a13f2e6SEdward Tomasz Napierala KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk), 30421a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__, 30431a13f2e6SEdward Tomasz Napierala ctd32tree)); 30441a13f2e6SEdward Tomasz Napierala } else { 30451a13f2e6SEdward Tomasz Napierala idx = ARB_NEXTFREEIDX(ctd64, ctdlnk); 30461a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst, 30471a13f2e6SEdward Tomasz Napierala ctd64->mu, ctd64->cnt, attempt); 30481a13f2e6SEdward Tomasz Napierala ctd64 = ARB_NODE(ctd64tree, idx); 30491a13f2e6SEdward Tomasz Napierala KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk), 30501a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__, 30511a13f2e6SEdward Tomasz Napierala ctd64tree)); 30521a13f2e6SEdward Tomasz Napierala } 30531a13f2e6SEdward Tomasz Napierala /* 30541a13f2e6SEdward Tomasz Napierala * This process should not produce errors, bugs notwithstanding. 30551a13f2e6SEdward Tomasz Napierala * Just in case, latch any errors and attempt all re-insertions. 30561a13f2e6SEdward Tomasz Napierala */ 30571a13f2e6SEdward Tomasz Napierala error = tmperr ? tmperr : error; 30581a13f2e6SEdward Tomasz Napierala remctds--; 30591a13f2e6SEdward Tomasz Napierala } 30601a13f2e6SEdward Tomasz Napierala 30611a13f2e6SEdward Tomasz Napierala KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__, 30621a13f2e6SEdward Tomasz Napierala (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 30631a13f2e6SEdward Tomasz Napierala 30641a13f2e6SEdward Tomasz Napierala return (error); 30651a13f2e6SEdward Tomasz Napierala } 30661a13f2e6SEdward Tomasz Napierala 30671a13f2e6SEdward Tomasz Napierala static inline int 30681a13f2e6SEdward Tomasz Napierala stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst, 30691a13f2e6SEdward Tomasz Napierala s64q_t x, uint64_t weight, int attempt) 30701a13f2e6SEdward Tomasz Napierala { 30711a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 30721a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(x, 10)]; 30731a13f2e6SEdward Tomasz Napierala #endif 30741a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 30751a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 30761a13f2e6SEdward Tomasz Napierala void *closest, *cur, *lb, *ub; 30771a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 30781a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 30791a13f2e6SEdward Tomasz Napierala uint64_t cnt, smplcnt, sum, tmpsum; 30801a13f2e6SEdward Tomasz Napierala s64q_t k, minz, q, z; 30811a13f2e6SEdward Tomasz Napierala int error, is32bit, n; 30821a13f2e6SEdward Tomasz Napierala 30831a13f2e6SEdward Tomasz Napierala error = 0; 30841a13f2e6SEdward Tomasz Napierala minz = Q_INI(&z, 0, 0, Q_NFBITS(x)); 30851a13f2e6SEdward Tomasz Napierala 30861a13f2e6SEdward Tomasz Napierala switch (vs_dtype) { 30871a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 30881a13f2e6SEdward Tomasz Napierala if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt) 30891a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 30901a13f2e6SEdward Tomasz Napierala smplcnt = VSD(tdgstclust32, tdgst)->smplcnt; 30911a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 30921a13f2e6SEdward Tomasz Napierala is32bit = 1; 30931a13f2e6SEdward Tomasz Napierala ctd64tree = NULL; 30941a13f2e6SEdward Tomasz Napierala ctd64 = NULL; 30951a13f2e6SEdward Tomasz Napierala break; 30961a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 30971a13f2e6SEdward Tomasz Napierala if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt) 30981a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 30991a13f2e6SEdward Tomasz Napierala smplcnt = VSD(tdgstclust64, tdgst)->smplcnt; 31001a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 31011a13f2e6SEdward Tomasz Napierala is32bit = 0; 31021a13f2e6SEdward Tomasz Napierala ctd32tree = NULL; 31031a13f2e6SEdward Tomasz Napierala ctd32 = NULL; 31041a13f2e6SEdward Tomasz Napierala break; 31051a13f2e6SEdward Tomasz Napierala default: 31061a13f2e6SEdward Tomasz Napierala error = EINVAL; 31071a13f2e6SEdward Tomasz Napierala break; 31081a13f2e6SEdward Tomasz Napierala } 31091a13f2e6SEdward Tomasz Napierala 31101a13f2e6SEdward Tomasz Napierala if (error) 31111a13f2e6SEdward Tomasz Napierala return (error); 31121a13f2e6SEdward Tomasz Napierala 31131a13f2e6SEdward Tomasz Napierala /* 31141a13f2e6SEdward Tomasz Napierala * Inspired by Ted Dunning's AVLTreeDigest.java 31151a13f2e6SEdward Tomasz Napierala */ 31161a13f2e6SEdward Tomasz Napierala do { 31171a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 31181a13f2e6SEdward Tomasz Napierala KASSERT(attempt < 5, 31191a13f2e6SEdward Tomasz Napierala ("%s: Too many attempts", __func__)); 31201a13f2e6SEdward Tomasz Napierala #endif 31211a13f2e6SEdward Tomasz Napierala if (attempt >= 5) 31221a13f2e6SEdward Tomasz Napierala return (EAGAIN); 31231a13f2e6SEdward Tomasz Napierala 31241a13f2e6SEdward Tomasz Napierala Q_SIFVAL(minz, Q_IFMAXVAL(minz)); 31251a13f2e6SEdward Tomasz Napierala closest = ub = NULL; 31261a13f2e6SEdward Tomasz Napierala sum = tmpsum = 0; 31271a13f2e6SEdward Tomasz Napierala 31281a13f2e6SEdward Tomasz Napierala if (is32bit) 31291a13f2e6SEdward Tomasz Napierala lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree)); 31301a13f2e6SEdward Tomasz Napierala else 31311a13f2e6SEdward Tomasz Napierala lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree)); 31321a13f2e6SEdward Tomasz Napierala 31331a13f2e6SEdward Tomasz Napierala if (lb == NULL) /* Empty tree. */ 31341a13f2e6SEdward Tomasz Napierala lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) : 31351a13f2e6SEdward Tomasz Napierala (void *)ARB_ROOT(ctd64tree)); 31361a13f2e6SEdward Tomasz Napierala 31371a13f2e6SEdward Tomasz Napierala /* 31381a13f2e6SEdward Tomasz Napierala * Find the set of centroids with minimum distance to x and 31391a13f2e6SEdward Tomasz Napierala * compute the sum of counts for all centroids with mean less 31401a13f2e6SEdward Tomasz Napierala * than the first centroid in the set. 31411a13f2e6SEdward Tomasz Napierala */ 31421a13f2e6SEdward Tomasz Napierala for (; cur != NULL; 31431a13f2e6SEdward Tomasz Napierala cur = (is32bit ? 31441a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 31451a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 31461a13f2e6SEdward Tomasz Napierala if (is32bit) { 31471a13f2e6SEdward Tomasz Napierala cnt = ctd32->cnt; 31481a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(ctd32->mu, x), 31491a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(mu,x)=%d", __func__, 31501a13f2e6SEdward Tomasz Napierala Q_RELPREC(ctd32->mu, x))); 31511a13f2e6SEdward Tomasz Napierala /* Ok to assign as both have same precision. */ 31521a13f2e6SEdward Tomasz Napierala z = ctd32->mu; 31531a13f2e6SEdward Tomasz Napierala } else { 31541a13f2e6SEdward Tomasz Napierala cnt = ctd64->cnt; 31551a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(ctd64->mu, x), 31561a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(mu,x)=%d", __func__, 31571a13f2e6SEdward Tomasz Napierala Q_RELPREC(ctd64->mu, x))); 31581a13f2e6SEdward Tomasz Napierala /* Ok to assign as both have same precision. */ 31591a13f2e6SEdward Tomasz Napierala z = ctd64->mu; 31601a13f2e6SEdward Tomasz Napierala } 31611a13f2e6SEdward Tomasz Napierala 31621a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&z, x); 31631a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 31641a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", __func__, 31651a13f2e6SEdward Tomasz Napierala error)); 31661a13f2e6SEdward Tomasz Napierala #endif 31671a13f2e6SEdward Tomasz Napierala if (error) 31681a13f2e6SEdward Tomasz Napierala return (error); 31691a13f2e6SEdward Tomasz Napierala 31701a13f2e6SEdward Tomasz Napierala z = Q_QABS(z); 31711a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(z, minz)) { 31721a13f2e6SEdward Tomasz Napierala minz = z; 31731a13f2e6SEdward Tomasz Napierala lb = cur; 31741a13f2e6SEdward Tomasz Napierala sum = tmpsum; 31751a13f2e6SEdward Tomasz Napierala tmpsum += cnt; 31761a13f2e6SEdward Tomasz Napierala } else if (Q_QGTQ(z, minz)) { 31771a13f2e6SEdward Tomasz Napierala ub = cur; 31781a13f2e6SEdward Tomasz Napierala break; 31791a13f2e6SEdward Tomasz Napierala } 31801a13f2e6SEdward Tomasz Napierala } 31811a13f2e6SEdward Tomasz Napierala 31821a13f2e6SEdward Tomasz Napierala cur = (is32bit ? 31831a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) : 31841a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb)); 31851a13f2e6SEdward Tomasz Napierala 31861a13f2e6SEdward Tomasz Napierala for (n = 0; cur != ub; cur = (is32bit ? 31871a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 31881a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 31891a13f2e6SEdward Tomasz Napierala if (is32bit) 31901a13f2e6SEdward Tomasz Napierala cnt = ctd32->cnt; 31911a13f2e6SEdward Tomasz Napierala else 31921a13f2e6SEdward Tomasz Napierala cnt = ctd64->cnt; 31931a13f2e6SEdward Tomasz Napierala 31941a13f2e6SEdward Tomasz Napierala q = Q_CTRLINI(16); 31951a13f2e6SEdward Tomasz Napierala if (smplcnt == 1) 31961a13f2e6SEdward Tomasz Napierala error = Q_QFRACI(&q, 1, 2); 31971a13f2e6SEdward Tomasz Napierala else 31981a13f2e6SEdward Tomasz Napierala /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */ 31991a13f2e6SEdward Tomasz Napierala error = Q_QFRACI(&q, (sum << 1) + cnt - 1, 32001a13f2e6SEdward Tomasz Napierala (smplcnt - 1) << 1); 32011a13f2e6SEdward Tomasz Napierala k = q; 32021a13f2e6SEdward Tomasz Napierala /* k = q x 4 x samplcnt x attempt */ 32031a13f2e6SEdward Tomasz Napierala error |= Q_QMULI(&k, 4 * smplcnt * attempt); 32041a13f2e6SEdward Tomasz Napierala /* k = k x (1 - q) */ 32051a13f2e6SEdward Tomasz Napierala error |= Q_QSUBI(&q, 1); 32061a13f2e6SEdward Tomasz Napierala q = Q_QABS(q); 32071a13f2e6SEdward Tomasz Napierala error |= Q_QMULQ(&k, q); 32081a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 32091a13f2e6SEdward Tomasz Napierala #if !defined(_KERNEL) 32101a13f2e6SEdward Tomasz Napierala double q_dbl, k_dbl, q2d, k2d; 32111a13f2e6SEdward Tomasz Napierala q2d = Q_Q2D(q); 32121a13f2e6SEdward Tomasz Napierala k2d = Q_Q2D(k); 32131a13f2e6SEdward Tomasz Napierala q_dbl = smplcnt == 1 ? 0.5 : 32141a13f2e6SEdward Tomasz Napierala (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1); 32151a13f2e6SEdward Tomasz Napierala k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt; 32161a13f2e6SEdward Tomasz Napierala /* 32171a13f2e6SEdward Tomasz Napierala * If the difference between q and q_dbl is greater than 32181a13f2e6SEdward Tomasz Napierala * the fractional precision of q, something is off. 32191a13f2e6SEdward Tomasz Napierala * NB: q is holding the value of 1 - q 32201a13f2e6SEdward Tomasz Napierala */ 32211a13f2e6SEdward Tomasz Napierala q_dbl = 1.0 - q_dbl; 32221a13f2e6SEdward Tomasz Napierala KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) < 32231a13f2e6SEdward Tomasz Napierala (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))), 32241a13f2e6SEdward Tomasz Napierala ("Q-type q bad precision")); 32251a13f2e6SEdward Tomasz Napierala KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) < 32261a13f2e6SEdward Tomasz Napierala 1.0 + (0.01 * smplcnt), 32271a13f2e6SEdward Tomasz Napierala ("Q-type k bad precision")); 32281a13f2e6SEdward Tomasz Napierala #endif /* !_KERNEL */ 32291a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", __func__, 32301a13f2e6SEdward Tomasz Napierala error)); 32311a13f2e6SEdward Tomasz Napierala #endif /* DIAGNOSTIC */ 32321a13f2e6SEdward Tomasz Napierala if (error) 32331a13f2e6SEdward Tomasz Napierala return (error); 32341a13f2e6SEdward Tomasz Napierala if ((is32bit && ((ctd32->cnt + weight) <= 32351a13f2e6SEdward Tomasz Napierala (uint64_t)Q_GIVAL(k))) || 32361a13f2e6SEdward Tomasz Napierala (!is32bit && ((ctd64->cnt + weight) <= 32371a13f2e6SEdward Tomasz Napierala (uint64_t)Q_GIVAL(k)))) { 32381a13f2e6SEdward Tomasz Napierala n++; 32391a13f2e6SEdward Tomasz Napierala /* random() produces 31 bits. */ 32401a13f2e6SEdward Tomasz Napierala if (random() < (INT32_MAX / n)) 32411a13f2e6SEdward Tomasz Napierala closest = cur; 32421a13f2e6SEdward Tomasz Napierala } 32431a13f2e6SEdward Tomasz Napierala sum += cnt; 32441a13f2e6SEdward Tomasz Napierala } 32451a13f2e6SEdward Tomasz Napierala } while (closest == NULL && 32461a13f2e6SEdward Tomasz Napierala (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) && 32471a13f2e6SEdward Tomasz Napierala (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst, 32481a13f2e6SEdward Tomasz Napierala attempt++)) == 0); 32491a13f2e6SEdward Tomasz Napierala 32501a13f2e6SEdward Tomasz Napierala if (error) 32511a13f2e6SEdward Tomasz Napierala return (error); 32521a13f2e6SEdward Tomasz Napierala 32531a13f2e6SEdward Tomasz Napierala if (closest != NULL) { 32541a13f2e6SEdward Tomasz Napierala /* Merge with an existing centroid. */ 32551a13f2e6SEdward Tomasz Napierala if (is32bit) { 32561a13f2e6SEdward Tomasz Napierala ctd32 = (struct voistatdata_tdgstctd32 *)closest; 32571a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&x, ctd32->mu); 32581a13f2e6SEdward Tomasz Napierala error = error ? error : 32591a13f2e6SEdward Tomasz Napierala Q_QDIVI(&x, ctd32->cnt + weight); 32601a13f2e6SEdward Tomasz Napierala if (error || (error = Q_QADDQ(&ctd32->mu, x))) { 32611a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 32621a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", 32631a13f2e6SEdward Tomasz Napierala __func__, error)); 32641a13f2e6SEdward Tomasz Napierala #endif 32651a13f2e6SEdward Tomasz Napierala return (error); 32661a13f2e6SEdward Tomasz Napierala } 32671a13f2e6SEdward Tomasz Napierala ctd32->cnt += weight; 32681a13f2e6SEdward Tomasz Napierala error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) == 32691a13f2e6SEdward Tomasz Napierala NULL ? 0 : EALREADY; 32701a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 32711a13f2e6SEdward Tomasz Napierala RB_REINSERT(rbctdth32, 32721a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 32731a13f2e6SEdward Tomasz Napierala #endif 32741a13f2e6SEdward Tomasz Napierala } else { 32751a13f2e6SEdward Tomasz Napierala ctd64 = (struct voistatdata_tdgstctd64 *)closest; 32761a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&x, ctd64->mu); 32771a13f2e6SEdward Tomasz Napierala error = error ? error : 32781a13f2e6SEdward Tomasz Napierala Q_QDIVI(&x, ctd64->cnt + weight); 32791a13f2e6SEdward Tomasz Napierala if (error || (error = Q_QADDQ(&ctd64->mu, x))) { 32801a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", 32811a13f2e6SEdward Tomasz Napierala __func__, error)); 32821a13f2e6SEdward Tomasz Napierala return (error); 32831a13f2e6SEdward Tomasz Napierala } 32841a13f2e6SEdward Tomasz Napierala ctd64->cnt += weight; 32851a13f2e6SEdward Tomasz Napierala error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) == 32861a13f2e6SEdward Tomasz Napierala NULL ? 0 : EALREADY; 32871a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 32881a13f2e6SEdward Tomasz Napierala RB_REINSERT(rbctdth64, 32891a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64); 32901a13f2e6SEdward Tomasz Napierala #endif 32911a13f2e6SEdward Tomasz Napierala } 32921a13f2e6SEdward Tomasz Napierala } else { 32931a13f2e6SEdward Tomasz Napierala /* 32941a13f2e6SEdward Tomasz Napierala * Add a new centroid. If digest compression is working 32951a13f2e6SEdward Tomasz Napierala * correctly, there should always be at least one free. 32961a13f2e6SEdward Tomasz Napierala */ 32971a13f2e6SEdward Tomasz Napierala if (is32bit) { 32981a13f2e6SEdward Tomasz Napierala ctd32 = ARB_GETFREE(ctd32tree, ctdlnk); 32991a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 33001a13f2e6SEdward Tomasz Napierala KASSERT(ctd32 != NULL, 33011a13f2e6SEdward Tomasz Napierala ("%s: t-digest@%p has no free centroids", 33021a13f2e6SEdward Tomasz Napierala __func__, tdgst)); 33031a13f2e6SEdward Tomasz Napierala #endif 33041a13f2e6SEdward Tomasz Napierala if (ctd32 == NULL) 33051a13f2e6SEdward Tomasz Napierala return (EAGAIN); 33061a13f2e6SEdward Tomasz Napierala if ((error = Q_QCPYVALQ(&ctd32->mu, x))) 33071a13f2e6SEdward Tomasz Napierala return (error); 33081a13f2e6SEdward Tomasz Napierala ctd32->cnt = weight; 33091a13f2e6SEdward Tomasz Napierala error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ? 33101a13f2e6SEdward Tomasz Napierala 0 : EALREADY; 33111a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 33121a13f2e6SEdward Tomasz Napierala RB_INSERT(rbctdth32, 33131a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 33141a13f2e6SEdward Tomasz Napierala #endif 33151a13f2e6SEdward Tomasz Napierala } else { 33161a13f2e6SEdward Tomasz Napierala ctd64 = ARB_GETFREE(ctd64tree, ctdlnk); 33171a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 33181a13f2e6SEdward Tomasz Napierala KASSERT(ctd64 != NULL, 33191a13f2e6SEdward Tomasz Napierala ("%s: t-digest@%p has no free centroids", 33201a13f2e6SEdward Tomasz Napierala __func__, tdgst)); 33211a13f2e6SEdward Tomasz Napierala #endif 33221a13f2e6SEdward Tomasz Napierala if (ctd64 == NULL) /* Should not happen. */ 33231a13f2e6SEdward Tomasz Napierala return (EAGAIN); 33241a13f2e6SEdward Tomasz Napierala /* Direct assignment ok as both have same type/prec. */ 33251a13f2e6SEdward Tomasz Napierala ctd64->mu = x; 33261a13f2e6SEdward Tomasz Napierala ctd64->cnt = weight; 33271a13f2e6SEdward Tomasz Napierala error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ? 33281a13f2e6SEdward Tomasz Napierala 0 : EALREADY; 33291a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 33301a13f2e6SEdward Tomasz Napierala RB_INSERT(rbctdth64, &VSD(tdgstclust64, 33311a13f2e6SEdward Tomasz Napierala tdgst)->rbctdtree, ctd64); 33321a13f2e6SEdward Tomasz Napierala #endif 33331a13f2e6SEdward Tomasz Napierala } 33341a13f2e6SEdward Tomasz Napierala } 33351a13f2e6SEdward Tomasz Napierala 33361a13f2e6SEdward Tomasz Napierala if (is32bit) 33371a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt += weight; 33381a13f2e6SEdward Tomasz Napierala else { 33391a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt += weight; 33401a13f2e6SEdward Tomasz Napierala 33411a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 33421a13f2e6SEdward Tomasz Napierala struct rbctdth64 *rbctdtree = 33431a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust64, tdgst)->rbctdtree; 33441a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *rbctd64; 33451a13f2e6SEdward Tomasz Napierala int i = 0; 33461a13f2e6SEdward Tomasz Napierala ARB_FOREACH(ctd64, ctdth64, ctd64tree) { 33471a13f2e6SEdward Tomasz Napierala rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) : 33481a13f2e6SEdward Tomasz Napierala RB_NEXT(rbctdth64, rbctdtree, rbctd64)); 33491a13f2e6SEdward Tomasz Napierala 33501a13f2e6SEdward Tomasz Napierala if (i >= ARB_CURNODES(ctd64tree) 33511a13f2e6SEdward Tomasz Napierala || ctd64 != rbctd64 33521a13f2e6SEdward Tomasz Napierala || ARB_MIN(ctdth64, ctd64tree) != 33531a13f2e6SEdward Tomasz Napierala RB_MIN(rbctdth64, rbctdtree) 33541a13f2e6SEdward Tomasz Napierala || ARB_MAX(ctdth64, ctd64tree) != 33551a13f2e6SEdward Tomasz Napierala RB_MAX(rbctdth64, rbctdtree) 33561a13f2e6SEdward Tomasz Napierala || ARB_LEFTIDX(ctd64, ctdlnk) != 33571a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk)) 33581a13f2e6SEdward Tomasz Napierala || ARB_RIGHTIDX(ctd64, ctdlnk) != 33591a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk)) 33601a13f2e6SEdward Tomasz Napierala || ARB_PARENTIDX(ctd64, ctdlnk) != 33611a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, 33621a13f2e6SEdward Tomasz Napierala RB_PARENT(rbctd64, rblnk))) { 33631a13f2e6SEdward Tomasz Napierala Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr)); 33641a13f2e6SEdward Tomasz Napierala printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 33651a13f2e6SEdward Tomasz Napierala "mu=%s\n", 33661a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, ctd64), 33671a13f2e6SEdward Tomasz Napierala ARB_PARENTIDX(ctd64, ctdlnk), 33681a13f2e6SEdward Tomasz Napierala ARB_LEFTIDX(ctd64, ctdlnk), 33691a13f2e6SEdward Tomasz Napierala ARB_RIGHTIDX(ctd64, ctdlnk), 33701a13f2e6SEdward Tomasz Napierala ARB_COLOR(ctd64, ctdlnk), 33711a13f2e6SEdward Tomasz Napierala qstr); 33721a13f2e6SEdward Tomasz Napierala 33731a13f2e6SEdward Tomasz Napierala Q_TOSTR(rbctd64->mu, -1, 10, qstr, 33741a13f2e6SEdward Tomasz Napierala sizeof(qstr)); 33751a13f2e6SEdward Tomasz Napierala printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 33761a13f2e6SEdward Tomasz Napierala "mu=%s\n", 33771a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, rbctd64), 33781a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, 33791a13f2e6SEdward Tomasz Napierala RB_PARENT(rbctd64, rblnk)), 33801a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, 33811a13f2e6SEdward Tomasz Napierala RB_LEFT(rbctd64, rblnk)), 33821a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, 33831a13f2e6SEdward Tomasz Napierala RB_RIGHT(rbctd64, rblnk)), 33841a13f2e6SEdward Tomasz Napierala RB_COLOR(rbctd64, rblnk), 33851a13f2e6SEdward Tomasz Napierala qstr); 33861a13f2e6SEdward Tomasz Napierala 33871a13f2e6SEdward Tomasz Napierala panic("RB@%p and ARB@%p trees differ\n", 33881a13f2e6SEdward Tomasz Napierala rbctdtree, ctd64tree); 33891a13f2e6SEdward Tomasz Napierala } 33901a13f2e6SEdward Tomasz Napierala i++; 33911a13f2e6SEdward Tomasz Napierala } 33921a13f2e6SEdward Tomasz Napierala #endif /* DIAGNOSTIC */ 33931a13f2e6SEdward Tomasz Napierala } 33941a13f2e6SEdward Tomasz Napierala 33951a13f2e6SEdward Tomasz Napierala return (error); 33961a13f2e6SEdward Tomasz Napierala } 33971a13f2e6SEdward Tomasz Napierala 33981a13f2e6SEdward Tomasz Napierala static inline int 33991a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival, 34001a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct voistatdata_tdgst *tdgst) 34011a13f2e6SEdward Tomasz Napierala { 34021a13f2e6SEdward Tomasz Napierala s64q_t x; 34031a13f2e6SEdward Tomasz Napierala int error; 34041a13f2e6SEdward Tomasz Napierala 34051a13f2e6SEdward Tomasz Napierala error = 0; 34061a13f2e6SEdward Tomasz Napierala 34071a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 34081a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 34091a13f2e6SEdward Tomasz Napierala /* Use same precision as the user's centroids. */ 34101a13f2e6SEdward Tomasz Napierala Q_INI(&x, 0, 0, Q_NFBITS( 34111a13f2e6SEdward Tomasz Napierala ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu)); 34121a13f2e6SEdward Tomasz Napierala break; 34131a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 34141a13f2e6SEdward Tomasz Napierala /* Use same precision as the user's centroids. */ 34151a13f2e6SEdward Tomasz Napierala Q_INI(&x, 0, 0, Q_NFBITS( 34161a13f2e6SEdward Tomasz Napierala ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu)); 34171a13f2e6SEdward Tomasz Napierala break; 34181a13f2e6SEdward Tomasz Napierala default: 34191a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 || 34201a13f2e6SEdward Tomasz Napierala vs->dtype == VSD_DTYPE_TDGSTCLUST64, 34211a13f2e6SEdward Tomasz Napierala ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>", 34221a13f2e6SEdward Tomasz Napierala __func__, vs->dtype)); 34231a13f2e6SEdward Tomasz Napierala return (EINVAL); 34241a13f2e6SEdward Tomasz Napierala } 34251a13f2e6SEdward Tomasz Napierala 34261a13f2e6SEdward Tomasz Napierala /* 34271a13f2e6SEdward Tomasz Napierala * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid 34281a13f2e6SEdward Tomasz Napierala * returning EOVERFLOW if the voival would have fit in a u64q_t. 34291a13f2e6SEdward Tomasz Napierala */ 34301a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 34311a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 34321a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int32.s32); 34331a13f2e6SEdward Tomasz Napierala break; 34341a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 34351a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int32.u32); 34361a13f2e6SEdward Tomasz Napierala break; 34371a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 34381a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int64.s64); 34391a13f2e6SEdward Tomasz Napierala break; 34401a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 34411a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int64.u64); 34421a13f2e6SEdward Tomasz Napierala break; 34431a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 34441a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->intlong.slong); 34451a13f2e6SEdward Tomasz Napierala break; 34461a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 34471a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->intlong.ulong); 34481a13f2e6SEdward Tomasz Napierala break; 34491a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 34501a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q32.sq32); 34511a13f2e6SEdward Tomasz Napierala break; 34521a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 34531a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q32.uq32); 34541a13f2e6SEdward Tomasz Napierala break; 34551a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 34561a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q64.sq64); 34571a13f2e6SEdward Tomasz Napierala break; 34581a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 34591a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q64.uq64); 34601a13f2e6SEdward Tomasz Napierala break; 34611a13f2e6SEdward Tomasz Napierala default: 34621a13f2e6SEdward Tomasz Napierala error = EINVAL; 34631a13f2e6SEdward Tomasz Napierala break; 34641a13f2e6SEdward Tomasz Napierala } 34651a13f2e6SEdward Tomasz Napierala 34661a13f2e6SEdward Tomasz Napierala if (error || 34671a13f2e6SEdward Tomasz Napierala (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1))) 34681a13f2e6SEdward Tomasz Napierala return (error); 34691a13f2e6SEdward Tomasz Napierala 34701a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 34711a13f2e6SEdward Tomasz Napierala return (0); 34721a13f2e6SEdward Tomasz Napierala } 34731a13f2e6SEdward Tomasz Napierala 34741a13f2e6SEdward Tomasz Napierala int 34751a13f2e6SEdward Tomasz Napierala stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id, 34761a13f2e6SEdward Tomasz Napierala enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags) 34771a13f2e6SEdward Tomasz Napierala { 34781a13f2e6SEdward Tomasz Napierala struct voi *v; 34791a13f2e6SEdward Tomasz Napierala struct voistat *vs; 34801a13f2e6SEdward Tomasz Napierala void *statevsd, *vsd; 34811a13f2e6SEdward Tomasz Napierala int error, i, tmperr; 34821a13f2e6SEdward Tomasz Napierala 34831a13f2e6SEdward Tomasz Napierala error = 0; 34841a13f2e6SEdward Tomasz Napierala 34851a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) || 34861a13f2e6SEdward Tomasz Napierala voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL) 34871a13f2e6SEdward Tomasz Napierala return (EINVAL); 34881a13f2e6SEdward Tomasz Napierala v = &sb->vois[voi_id]; 34891a13f2e6SEdward Tomasz Napierala if (voi_dtype != v->dtype || v->id < 0 || 34901a13f2e6SEdward Tomasz Napierala ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE))) 34911a13f2e6SEdward Tomasz Napierala return (EINVAL); 34921a13f2e6SEdward Tomasz Napierala 34931a13f2e6SEdward Tomasz Napierala vs = BLOB_OFFSET(sb, v->stats_off); 34941a13f2e6SEdward Tomasz Napierala if (v->flags & VOI_REQSTATE) 34951a13f2e6SEdward Tomasz Napierala statevsd = BLOB_OFFSET(sb, vs->data_off); 34961a13f2e6SEdward Tomasz Napierala else 34971a13f2e6SEdward Tomasz Napierala statevsd = NULL; 34981a13f2e6SEdward Tomasz Napierala 34991a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) { 35001a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 35011a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 35021a13f2e6SEdward Tomasz Napierala voival->int32.s32 += 35031a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.s32; 35041a13f2e6SEdward Tomasz Napierala break; 35051a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 35061a13f2e6SEdward Tomasz Napierala voival->int32.u32 += 35071a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.u32; 35081a13f2e6SEdward Tomasz Napierala break; 35091a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 35101a13f2e6SEdward Tomasz Napierala voival->int64.s64 += 35111a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.s64; 35121a13f2e6SEdward Tomasz Napierala break; 35131a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 35141a13f2e6SEdward Tomasz Napierala voival->int64.u64 += 35151a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.u64; 35161a13f2e6SEdward Tomasz Napierala break; 35171a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 35181a13f2e6SEdward Tomasz Napierala voival->intlong.slong += 35191a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.slong; 35201a13f2e6SEdward Tomasz Napierala break; 35211a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 35221a13f2e6SEdward Tomasz Napierala voival->intlong.ulong += 35231a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.ulong; 35241a13f2e6SEdward Tomasz Napierala break; 35251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 35261a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q32.sq32, 35271a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q32.sq32); 35281a13f2e6SEdward Tomasz Napierala break; 35291a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 35301a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q32.uq32, 35311a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q32.uq32); 35321a13f2e6SEdward Tomasz Napierala break; 35331a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 35341a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q64.sq64, 35351a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q64.sq64); 35361a13f2e6SEdward Tomasz Napierala break; 35371a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 35381a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q64.uq64, 35391a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q64.uq64); 35401a13f2e6SEdward Tomasz Napierala break; 35411a13f2e6SEdward Tomasz Napierala default: 35421a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 35431a13f2e6SEdward Tomasz Napierala break; 35441a13f2e6SEdward Tomasz Napierala } 35451a13f2e6SEdward Tomasz Napierala } 35461a13f2e6SEdward Tomasz Napierala 35471a13f2e6SEdward Tomasz Napierala if (error) 35481a13f2e6SEdward Tomasz Napierala return (error); 35491a13f2e6SEdward Tomasz Napierala 35501a13f2e6SEdward Tomasz Napierala for (i = v->voistatmaxid; i > 0; i--) { 35511a13f2e6SEdward Tomasz Napierala vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i]; 35521a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) 35531a13f2e6SEdward Tomasz Napierala continue; 35541a13f2e6SEdward Tomasz Napierala 35551a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 35561a13f2e6SEdward Tomasz Napierala 35571a13f2e6SEdward Tomasz Napierala switch (vs->stype) { 35581a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX: 35591a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_max(voi_dtype, voival, 35601a13f2e6SEdward Tomasz Napierala vs, vsd); 35611a13f2e6SEdward Tomasz Napierala break; 35621a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN: 35631a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_min(voi_dtype, voival, 35641a13f2e6SEdward Tomasz Napierala vs, vsd); 35651a13f2e6SEdward Tomasz Napierala break; 35661a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM: 35671a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_sum(voi_dtype, voival, 35681a13f2e6SEdward Tomasz Napierala vs, vsd); 35691a13f2e6SEdward Tomasz Napierala break; 35701a13f2e6SEdward Tomasz Napierala case VS_STYPE_HIST: 35711a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_hist(voi_dtype, voival, 35721a13f2e6SEdward Tomasz Napierala vs, vsd); 35731a13f2e6SEdward Tomasz Napierala break; 35741a13f2e6SEdward Tomasz Napierala case VS_STYPE_TDGST: 35751a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival, 35761a13f2e6SEdward Tomasz Napierala vs, vsd); 35771a13f2e6SEdward Tomasz Napierala break; 35781a13f2e6SEdward Tomasz Napierala default: 35791a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 35801a13f2e6SEdward Tomasz Napierala break; 35811a13f2e6SEdward Tomasz Napierala } 35821a13f2e6SEdward Tomasz Napierala 35831a13f2e6SEdward Tomasz Napierala if (tmperr) { 35841a13f2e6SEdward Tomasz Napierala error = tmperr; 35851a13f2e6SEdward Tomasz Napierala VS_INCERRS(vs); 35861a13f2e6SEdward Tomasz Napierala } 35871a13f2e6SEdward Tomasz Napierala } 35881a13f2e6SEdward Tomasz Napierala 35891a13f2e6SEdward Tomasz Napierala if (statevsd) { 35901a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 35911a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 35921a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.s32 = 35931a13f2e6SEdward Tomasz Napierala voival->int32.s32; 35941a13f2e6SEdward Tomasz Napierala break; 35951a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 35961a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.u32 = 35971a13f2e6SEdward Tomasz Napierala voival->int32.u32; 35981a13f2e6SEdward Tomasz Napierala break; 35991a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 36001a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.s64 = 36011a13f2e6SEdward Tomasz Napierala voival->int64.s64; 36021a13f2e6SEdward Tomasz Napierala break; 36031a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 36041a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.u64 = 36051a13f2e6SEdward Tomasz Napierala voival->int64.u64; 36061a13f2e6SEdward Tomasz Napierala break; 36071a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 36081a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.slong = 36091a13f2e6SEdward Tomasz Napierala voival->intlong.slong; 36101a13f2e6SEdward Tomasz Napierala break; 36111a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 36121a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.ulong = 36131a13f2e6SEdward Tomasz Napierala voival->intlong.ulong; 36141a13f2e6SEdward Tomasz Napierala break; 36151a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 36161a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 36171a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q32.sq32, 36181a13f2e6SEdward Tomasz Napierala voival->q32.sq32); 36191a13f2e6SEdward Tomasz Napierala break; 36201a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 36211a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 36221a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q32.uq32, 36231a13f2e6SEdward Tomasz Napierala voival->q32.uq32); 36241a13f2e6SEdward Tomasz Napierala break; 36251a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 36261a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 36271a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q64.sq64, 36281a13f2e6SEdward Tomasz Napierala voival->q64.sq64); 36291a13f2e6SEdward Tomasz Napierala break; 36301a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 36311a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 36321a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q64.uq64, 36331a13f2e6SEdward Tomasz Napierala voival->q64.uq64); 36341a13f2e6SEdward Tomasz Napierala break; 36351a13f2e6SEdward Tomasz Napierala default: 36361a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 36371a13f2e6SEdward Tomasz Napierala break; 36381a13f2e6SEdward Tomasz Napierala } 36391a13f2e6SEdward Tomasz Napierala } 36401a13f2e6SEdward Tomasz Napierala 36411a13f2e6SEdward Tomasz Napierala return (error); 36421a13f2e6SEdward Tomasz Napierala } 36431a13f2e6SEdward Tomasz Napierala 36441a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 36451a13f2e6SEdward Tomasz Napierala 36461a13f2e6SEdward Tomasz Napierala static void 36471a13f2e6SEdward Tomasz Napierala stats_init(void *arg) 36481a13f2e6SEdward Tomasz Napierala { 36491a13f2e6SEdward Tomasz Napierala 36501a13f2e6SEdward Tomasz Napierala } 36511a13f2e6SEdward Tomasz Napierala SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL); 36521a13f2e6SEdward Tomasz Napierala 36531a13f2e6SEdward Tomasz Napierala /* 36541a13f2e6SEdward Tomasz Napierala * Sysctl handler to display the list of available stats templates. 36551a13f2e6SEdward Tomasz Napierala */ 36561a13f2e6SEdward Tomasz Napierala static int 36571a13f2e6SEdward Tomasz Napierala stats_tpl_list_available(SYSCTL_HANDLER_ARGS) 36581a13f2e6SEdward Tomasz Napierala { 36591a13f2e6SEdward Tomasz Napierala struct sbuf *s; 36601a13f2e6SEdward Tomasz Napierala int err, i; 36611a13f2e6SEdward Tomasz Napierala 36621a13f2e6SEdward Tomasz Napierala err = 0; 36631a13f2e6SEdward Tomasz Napierala 36641a13f2e6SEdward Tomasz Napierala /* We can tolerate ntpl being stale, so do not take the lock. */ 36651a13f2e6SEdward Tomasz Napierala s = sbuf_new(NULL, NULL, /* +1 per tpl for , */ 36661a13f2e6SEdward Tomasz Napierala ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN); 36671a13f2e6SEdward Tomasz Napierala if (s == NULL) 36681a13f2e6SEdward Tomasz Napierala return (ENOMEM); 36691a13f2e6SEdward Tomasz Napierala 36701a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 36711a13f2e6SEdward Tomasz Napierala for (i = 0; i < ntpl; i++) { 36721a13f2e6SEdward Tomasz Napierala err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "", 36731a13f2e6SEdward Tomasz Napierala tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash); 36741a13f2e6SEdward Tomasz Napierala if (err) { 36751a13f2e6SEdward Tomasz Napierala /* Sbuf overflow condition. */ 36761a13f2e6SEdward Tomasz Napierala err = EOVERFLOW; 36771a13f2e6SEdward Tomasz Napierala break; 36781a13f2e6SEdward Tomasz Napierala } 36791a13f2e6SEdward Tomasz Napierala } 36801a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 36811a13f2e6SEdward Tomasz Napierala 36821a13f2e6SEdward Tomasz Napierala if (!err) { 36831a13f2e6SEdward Tomasz Napierala sbuf_finish(s); 36841a13f2e6SEdward Tomasz Napierala err = sysctl_handle_string(oidp, sbuf_data(s), 0, req); 36851a13f2e6SEdward Tomasz Napierala } 36861a13f2e6SEdward Tomasz Napierala 36871a13f2e6SEdward Tomasz Napierala sbuf_delete(s); 36881a13f2e6SEdward Tomasz Napierala return (err); 36891a13f2e6SEdward Tomasz Napierala } 36901a13f2e6SEdward Tomasz Napierala 36911a13f2e6SEdward Tomasz Napierala /* 36921a13f2e6SEdward Tomasz Napierala * Called by subsystem-specific sysctls to report and/or parse the list of 36931a13f2e6SEdward Tomasz Napierala * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t 36941a13f2e6SEdward Tomasz Napierala * conformant function pointer must be passed in as arg1, which is used to 36951a13f2e6SEdward Tomasz Napierala * interact with the subsystem's stats template sample rates list. If arg2 > 0, 36961a13f2e6SEdward Tomasz Napierala * a zero-initialised allocation of arg2-sized contextual memory is 36971a13f2e6SEdward Tomasz Napierala * heap-allocated and passed in to all subsystem callbacks made during the 36981a13f2e6SEdward Tomasz Napierala * operation of stats_tpl_sample_rates(). 36991a13f2e6SEdward Tomasz Napierala * 37001a13f2e6SEdward Tomasz Napierala * XXXLAS: Assumes templates are never removed, which is currently true but may 37011a13f2e6SEdward Tomasz Napierala * need to be reworked in future if dynamic template management becomes a 37021a13f2e6SEdward Tomasz Napierala * requirement e.g. to support kernel module based templates. 37031a13f2e6SEdward Tomasz Napierala */ 37041a13f2e6SEdward Tomasz Napierala int 37051a13f2e6SEdward Tomasz Napierala stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS) 37061a13f2e6SEdward Tomasz Napierala { 37071a13f2e6SEdward Tomasz Napierala char kvpair_fmt[16], tplspec_fmt[16]; 37081a13f2e6SEdward Tomasz Napierala char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN]; 37091a13f2e6SEdward Tomasz Napierala char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */ 37101a13f2e6SEdward Tomasz Napierala stats_tpl_sr_cb_t subsys_cb; 37111a13f2e6SEdward Tomasz Napierala void *subsys_ctx; 37121a13f2e6SEdward Tomasz Napierala char *buf, *new_rates_usr_str, *tpl_name_p; 37131a13f2e6SEdward Tomasz Napierala struct stats_tpl_sample_rate *rates; 37141a13f2e6SEdward Tomasz Napierala struct sbuf *s, _s; 37151a13f2e6SEdward Tomasz Napierala uint32_t cum_pct, pct, tpl_hash; 37161a13f2e6SEdward Tomasz Napierala int err, i, off, len, newlen, nrates; 37171a13f2e6SEdward Tomasz Napierala 37181a13f2e6SEdward Tomasz Napierala buf = NULL; 37191a13f2e6SEdward Tomasz Napierala rates = NULL; 37201a13f2e6SEdward Tomasz Napierala err = nrates = 0; 37211a13f2e6SEdward Tomasz Napierala subsys_cb = (stats_tpl_sr_cb_t)arg1; 37221a13f2e6SEdward Tomasz Napierala KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__)); 37231a13f2e6SEdward Tomasz Napierala if (arg2 > 0) 37241a13f2e6SEdward Tomasz Napierala subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO); 37251a13f2e6SEdward Tomasz Napierala else 37261a13f2e6SEdward Tomasz Napierala subsys_ctx = NULL; 37271a13f2e6SEdward Tomasz Napierala 37281a13f2e6SEdward Tomasz Napierala /* Grab current count of subsystem rates. */ 37291a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx); 37301a13f2e6SEdward Tomasz Napierala if (err) 37311a13f2e6SEdward Tomasz Napierala goto done; 37321a13f2e6SEdward Tomasz Napierala 37331a13f2e6SEdward Tomasz Napierala /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */ 37341a13f2e6SEdward Tomasz Napierala len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5)); 37351a13f2e6SEdward Tomasz Napierala 37361a13f2e6SEdward Tomasz Napierala if (req->oldptr != NULL || req->newptr != NULL) 37371a13f2e6SEdward Tomasz Napierala buf = malloc(len, M_TEMP, M_WAITOK); 37381a13f2e6SEdward Tomasz Napierala 37391a13f2e6SEdward Tomasz Napierala if (req->oldptr != NULL) { 37401a13f2e6SEdward Tomasz Napierala if (nrates == 0) { 37411a13f2e6SEdward Tomasz Napierala /* No rates, so return an empty string via oldptr. */ 37421a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, "", 1); 37431a13f2e6SEdward Tomasz Napierala if (err) 37441a13f2e6SEdward Tomasz Napierala goto done; 37451a13f2e6SEdward Tomasz Napierala goto process_new; 37461a13f2e6SEdward Tomasz Napierala } 37471a13f2e6SEdward Tomasz Napierala 37481a13f2e6SEdward Tomasz Napierala s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL); 37491a13f2e6SEdward Tomasz Napierala 37501a13f2e6SEdward Tomasz Napierala /* Grab locked count of, and ptr to, subsystem rates. */ 37511a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates, 37521a13f2e6SEdward Tomasz Napierala subsys_ctx); 37531a13f2e6SEdward Tomasz Napierala if (err) 37541a13f2e6SEdward Tomasz Napierala goto done; 37551a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 37561a13f2e6SEdward Tomasz Napierala for (i = 0; i < nrates && !err; i++) { 37571a13f2e6SEdward Tomasz Napierala err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "", 37581a13f2e6SEdward Tomasz Napierala tpllist[rates[i].tpl_slot_id]->mb->tplname, 37591a13f2e6SEdward Tomasz Napierala tpllist[rates[i].tpl_slot_id]->mb->tplhash, 37601a13f2e6SEdward Tomasz Napierala rates[i].tpl_sample_pct); 37611a13f2e6SEdward Tomasz Napierala } 37621a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 37631a13f2e6SEdward Tomasz Napierala /* Tell subsystem that we're done with its rates list. */ 37641a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx); 37651a13f2e6SEdward Tomasz Napierala if (err) 37661a13f2e6SEdward Tomasz Napierala goto done; 37671a13f2e6SEdward Tomasz Napierala 37681a13f2e6SEdward Tomasz Napierala err = sbuf_finish(s); 37691a13f2e6SEdward Tomasz Napierala if (err) 37701a13f2e6SEdward Tomasz Napierala goto done; /* We lost a race for buf to be too small. */ 37711a13f2e6SEdward Tomasz Napierala 37721a13f2e6SEdward Tomasz Napierala /* Return the rendered string data via oldptr. */ 37731a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s)); 37741a13f2e6SEdward Tomasz Napierala } else { 37751a13f2e6SEdward Tomasz Napierala /* Return the upper bound size for buffer sizing requests. */ 37761a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, NULL, len); 37771a13f2e6SEdward Tomasz Napierala } 37781a13f2e6SEdward Tomasz Napierala 37791a13f2e6SEdward Tomasz Napierala process_new: 37801a13f2e6SEdward Tomasz Napierala if (err || req->newptr == NULL) 37811a13f2e6SEdward Tomasz Napierala goto done; 37821a13f2e6SEdward Tomasz Napierala 37831a13f2e6SEdward Tomasz Napierala newlen = req->newlen - req->newidx; 37841a13f2e6SEdward Tomasz Napierala err = SYSCTL_IN(req, buf, newlen); 37851a13f2e6SEdward Tomasz Napierala if (err) 37861a13f2e6SEdward Tomasz Napierala goto done; 37871a13f2e6SEdward Tomasz Napierala 37881a13f2e6SEdward Tomasz Napierala /* 37891a13f2e6SEdward Tomasz Napierala * Initialise format strings at run time. 37901a13f2e6SEdward Tomasz Napierala * 37911a13f2e6SEdward Tomasz Napierala * Write the max template spec string length into the 37921a13f2e6SEdward Tomasz Napierala * template_spec=percent key-value pair parsing format string as: 37931a13f2e6SEdward Tomasz Napierala * " %<width>[^=]=%u %n" 37941a13f2e6SEdward Tomasz Napierala * 37951a13f2e6SEdward Tomasz Napierala * Write the max template name string length into the tplname:tplhash 37961a13f2e6SEdward Tomasz Napierala * parsing format string as: 37971a13f2e6SEdward Tomasz Napierala * "%<width>[^:]:%u" 37981a13f2e6SEdward Tomasz Napierala * 37991a13f2e6SEdward Tomasz Napierala * Subtract 1 for \0 appended by sscanf(). 38001a13f2e6SEdward Tomasz Napierala */ 38011a13f2e6SEdward Tomasz Napierala sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1); 38021a13f2e6SEdward Tomasz Napierala sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1); 38031a13f2e6SEdward Tomasz Napierala 38041a13f2e6SEdward Tomasz Napierala /* 38051a13f2e6SEdward Tomasz Napierala * Parse each CSV key-value pair specifying a template and its sample 38061a13f2e6SEdward Tomasz Napierala * percentage. Whitespace either side of a key-value pair is ignored. 38071a13f2e6SEdward Tomasz Napierala * Templates can be specified by name, hash, or name and hash per the 38081a13f2e6SEdward Tomasz Napierala * following formats (chars in [] are optional): 38091a13f2e6SEdward Tomasz Napierala * ["]<tplname>["]=<percent> 38101a13f2e6SEdward Tomasz Napierala * :hash=pct 38111a13f2e6SEdward Tomasz Napierala * ["]<tplname>["]:hash=<percent> 38121a13f2e6SEdward Tomasz Napierala */ 38131a13f2e6SEdward Tomasz Napierala cum_pct = nrates = 0; 38141a13f2e6SEdward Tomasz Napierala rates = NULL; 38151a13f2e6SEdward Tomasz Napierala buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */ 38161a13f2e6SEdward Tomasz Napierala new_rates_usr_str = buf; 38171a13f2e6SEdward Tomasz Napierala while (isspace(*new_rates_usr_str)) 38181a13f2e6SEdward Tomasz Napierala new_rates_usr_str++; /* Skip leading whitespace. */ 38191a13f2e6SEdward Tomasz Napierala while (*new_rates_usr_str != '\0') { 38201a13f2e6SEdward Tomasz Napierala tpl_name_p = tpl_name; 38211a13f2e6SEdward Tomasz Napierala tpl_name[0] = '\0'; 38221a13f2e6SEdward Tomasz Napierala tpl_hash = 0; 38231a13f2e6SEdward Tomasz Napierala off = 0; 38241a13f2e6SEdward Tomasz Napierala 38251a13f2e6SEdward Tomasz Napierala /* 38261a13f2e6SEdward Tomasz Napierala * Parse key-value pair which must perform 2 conversions, then 38271a13f2e6SEdward Tomasz Napierala * parse the template spec to extract either name, hash, or name 38281a13f2e6SEdward Tomasz Napierala * and hash depending on the three possible spec formats. The 38291a13f2e6SEdward Tomasz Napierala * tplspec_fmt format specifier parses name or name and hash 38301a13f2e6SEdward Tomasz Napierala * template specs, while the ":%u" format specifier parses 38311a13f2e6SEdward Tomasz Napierala * hash-only template specs. If parsing is successfull, ensure 38321a13f2e6SEdward Tomasz Napierala * the cumulative sampling percentage does not exceed 100. 38331a13f2e6SEdward Tomasz Napierala */ 38341a13f2e6SEdward Tomasz Napierala err = EINVAL; 38351a13f2e6SEdward Tomasz Napierala if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct, 38361a13f2e6SEdward Tomasz Napierala &off)) 38371a13f2e6SEdward Tomasz Napierala break; 38381a13f2e6SEdward Tomasz Napierala if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) && 38391a13f2e6SEdward Tomasz Napierala (1 != sscanf(tpl_spec, ":%u", &tpl_hash))) 38401a13f2e6SEdward Tomasz Napierala break; 38411a13f2e6SEdward Tomasz Napierala if ((cum_pct += pct) > 100) 38421a13f2e6SEdward Tomasz Napierala break; 38431a13f2e6SEdward Tomasz Napierala err = 0; 38441a13f2e6SEdward Tomasz Napierala 38451a13f2e6SEdward Tomasz Napierala /* Strip surrounding "" from template name if present. */ 38461a13f2e6SEdward Tomasz Napierala len = strlen(tpl_name); 38471a13f2e6SEdward Tomasz Napierala if (len > 0) { 38481a13f2e6SEdward Tomasz Napierala if (tpl_name[len - 1] == '"') 38491a13f2e6SEdward Tomasz Napierala tpl_name[--len] = '\0'; 38501a13f2e6SEdward Tomasz Napierala if (tpl_name[0] == '"') { 38511a13f2e6SEdward Tomasz Napierala tpl_name_p++; 38521a13f2e6SEdward Tomasz Napierala len--; 38531a13f2e6SEdward Tomasz Napierala } 38541a13f2e6SEdward Tomasz Napierala } 38551a13f2e6SEdward Tomasz Napierala 38561a13f2e6SEdward Tomasz Napierala rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */ 38571a13f2e6SEdward Tomasz Napierala (nrates + 1) * sizeof(*rates), M_WAITOK); 38581a13f2e6SEdward Tomasz Napierala rates[nrates].tpl_slot_id = 38591a13f2e6SEdward Tomasz Napierala stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash); 38601a13f2e6SEdward Tomasz Napierala if (rates[nrates].tpl_slot_id < 0) { 38611a13f2e6SEdward Tomasz Napierala err = -rates[nrates].tpl_slot_id; 38621a13f2e6SEdward Tomasz Napierala break; 38631a13f2e6SEdward Tomasz Napierala } 38641a13f2e6SEdward Tomasz Napierala rates[nrates].tpl_sample_pct = pct; 38651a13f2e6SEdward Tomasz Napierala nrates++; 38661a13f2e6SEdward Tomasz Napierala new_rates_usr_str += off; 38671a13f2e6SEdward Tomasz Napierala if (*new_rates_usr_str != ',') 38681a13f2e6SEdward Tomasz Napierala break; /* End-of-input or malformed. */ 38691a13f2e6SEdward Tomasz Napierala new_rates_usr_str++; /* Move past comma to next pair. */ 38701a13f2e6SEdward Tomasz Napierala } 38711a13f2e6SEdward Tomasz Napierala 38721a13f2e6SEdward Tomasz Napierala if (!err) { 38731a13f2e6SEdward Tomasz Napierala if ((new_rates_usr_str - buf) < newlen) { 38741a13f2e6SEdward Tomasz Napierala /* Entire input has not been consumed. */ 38751a13f2e6SEdward Tomasz Napierala err = EINVAL; 38761a13f2e6SEdward Tomasz Napierala } else { 38771a13f2e6SEdward Tomasz Napierala /* 38781a13f2e6SEdward Tomasz Napierala * Give subsystem the new rates. They'll return the 38791a13f2e6SEdward Tomasz Napierala * appropriate rates pointer for us to garbage collect. 38801a13f2e6SEdward Tomasz Napierala */ 38811a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_PUT, &rates, &nrates, 38821a13f2e6SEdward Tomasz Napierala subsys_ctx); 38831a13f2e6SEdward Tomasz Napierala } 38841a13f2e6SEdward Tomasz Napierala } 38851a13f2e6SEdward Tomasz Napierala stats_free(rates); 38861a13f2e6SEdward Tomasz Napierala 38871a13f2e6SEdward Tomasz Napierala done: 38881a13f2e6SEdward Tomasz Napierala free(buf, M_TEMP); 38891a13f2e6SEdward Tomasz Napierala free(subsys_ctx, M_TEMP); 38901a13f2e6SEdward Tomasz Napierala return (err); 38911a13f2e6SEdward Tomasz Napierala } 38921a13f2e6SEdward Tomasz Napierala 3893*7029da5cSPawel Biernacki SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 38941a13f2e6SEdward Tomasz Napierala "stats(9) MIB"); 38951a13f2e6SEdward Tomasz Napierala 3896*7029da5cSPawel Biernacki SYSCTL_PROC(_kern_stats, OID_AUTO, templates, 3897*7029da5cSPawel Biernacki CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 3898*7029da5cSPawel Biernacki stats_tpl_list_available, "A", 38991a13f2e6SEdward Tomasz Napierala "list the name/hash of all available stats(9) templates"); 39001a13f2e6SEdward Tomasz Napierala 39011a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 39021a13f2e6SEdward Tomasz Napierala 39031a13f2e6SEdward Tomasz Napierala static void __attribute__ ((constructor)) 39041a13f2e6SEdward Tomasz Napierala stats_constructor(void) 39051a13f2e6SEdward Tomasz Napierala { 39061a13f2e6SEdward Tomasz Napierala 39071a13f2e6SEdward Tomasz Napierala pthread_rwlock_init(&tpllistlock, NULL); 39081a13f2e6SEdward Tomasz Napierala } 39091a13f2e6SEdward Tomasz Napierala 39101a13f2e6SEdward Tomasz Napierala static void __attribute__ ((destructor)) 39111a13f2e6SEdward Tomasz Napierala stats_destructor(void) 39121a13f2e6SEdward Tomasz Napierala { 39131a13f2e6SEdward Tomasz Napierala 39141a13f2e6SEdward Tomasz Napierala pthread_rwlock_destroy(&tpllistlock); 39151a13f2e6SEdward Tomasz Napierala } 39161a13f2e6SEdward Tomasz Napierala 39171a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 3918