1*1a13f2e6SEdward Tomasz Napierala /*- 2*1a13f2e6SEdward Tomasz Napierala * Copyright (c) 2014-2018 Netflix, Inc. 3*1a13f2e6SEdward Tomasz Napierala * All rights reserved. 4*1a13f2e6SEdward Tomasz Napierala * 5*1a13f2e6SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 6*1a13f2e6SEdward Tomasz Napierala * modification, are permitted provided that the following conditions 7*1a13f2e6SEdward Tomasz Napierala * are met: 8*1a13f2e6SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 9*1a13f2e6SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 10*1a13f2e6SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 11*1a13f2e6SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 12*1a13f2e6SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 13*1a13f2e6SEdward Tomasz Napierala * 14*1a13f2e6SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15*1a13f2e6SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16*1a13f2e6SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17*1a13f2e6SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18*1a13f2e6SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19*1a13f2e6SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20*1a13f2e6SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21*1a13f2e6SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22*1a13f2e6SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23*1a13f2e6SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24*1a13f2e6SEdward Tomasz Napierala * SUCH DAMAGE. 25*1a13f2e6SEdward Tomasz Napierala * 26*1a13f2e6SEdward Tomasz Napierala * $FreeBSD$ 27*1a13f2e6SEdward Tomasz Napierala */ 28*1a13f2e6SEdward Tomasz Napierala 29*1a13f2e6SEdward Tomasz Napierala /* 30*1a13f2e6SEdward Tomasz Napierala * Author: Lawrence Stewart <lstewart@netflix.com> 31*1a13f2e6SEdward Tomasz Napierala */ 32*1a13f2e6SEdward Tomasz Napierala 33*1a13f2e6SEdward Tomasz Napierala #include <sys/cdefs.h> 34*1a13f2e6SEdward Tomasz Napierala __FBSDID("$FreeBSD$"); 35*1a13f2e6SEdward Tomasz Napierala 36*1a13f2e6SEdward Tomasz Napierala #include <sys/param.h> 37*1a13f2e6SEdward Tomasz Napierala #include <sys/arb.h> 38*1a13f2e6SEdward Tomasz Napierala #include <sys/ctype.h> 39*1a13f2e6SEdward Tomasz Napierala #include <sys/errno.h> 40*1a13f2e6SEdward Tomasz Napierala #include <sys/hash.h> 41*1a13f2e6SEdward Tomasz Napierala #include <sys/limits.h> 42*1a13f2e6SEdward Tomasz Napierala #include <sys/malloc.h> 43*1a13f2e6SEdward Tomasz Napierala #include <sys/qmath.h> 44*1a13f2e6SEdward Tomasz Napierala #include <sys/sbuf.h> 45*1a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 46*1a13f2e6SEdward Tomasz Napierala #include <sys/tree.h> 47*1a13f2e6SEdward Tomasz Napierala #endif 48*1a13f2e6SEdward Tomasz Napierala #include <sys/stats.h> /* Must come after qmath.h and arb.h */ 49*1a13f2e6SEdward Tomasz Napierala #include <sys/stddef.h> 50*1a13f2e6SEdward Tomasz Napierala #include <sys/stdint.h> 51*1a13f2e6SEdward Tomasz Napierala #include <sys/time.h> 52*1a13f2e6SEdward Tomasz Napierala 53*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 54*1a13f2e6SEdward Tomasz Napierala #include <sys/kernel.h> 55*1a13f2e6SEdward Tomasz Napierala #include <sys/lock.h> 56*1a13f2e6SEdward Tomasz Napierala #include <sys/rwlock.h> 57*1a13f2e6SEdward Tomasz Napierala #include <sys/sysctl.h> 58*1a13f2e6SEdward Tomasz Napierala #include <sys/systm.h> 59*1a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 60*1a13f2e6SEdward Tomasz Napierala #include <pthread.h> 61*1a13f2e6SEdward Tomasz Napierala #include <stdbool.h> 62*1a13f2e6SEdward Tomasz Napierala #include <stdio.h> 63*1a13f2e6SEdward Tomasz Napierala #include <stdlib.h> 64*1a13f2e6SEdward Tomasz Napierala #include <string.h> 65*1a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 66*1a13f2e6SEdward Tomasz Napierala 67*1a13f2e6SEdward Tomasz Napierala struct voistatdata_voistate { 68*1a13f2e6SEdward Tomasz Napierala /* Previous VOI value for diff calculation. */ 69*1a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric prev; 70*1a13f2e6SEdward Tomasz Napierala }; 71*1a13f2e6SEdward Tomasz Napierala 72*1a13f2e6SEdward Tomasz Napierala #define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */ 73*1a13f2e6SEdward Tomasz Napierala struct voistat { 74*1a13f2e6SEdward Tomasz Napierala int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */ 75*1a13f2e6SEdward Tomasz Napierala enum vsd_dtype dtype : 8; /* Data type of this stat's data. */ 76*1a13f2e6SEdward Tomasz Napierala uint16_t data_off; /* Blob offset for this stat's data. */ 77*1a13f2e6SEdward Tomasz Napierala uint16_t dsz; /* Size of stat's data. */ 78*1a13f2e6SEdward Tomasz Napierala #define VS_EBITS 8 79*1a13f2e6SEdward Tomasz Napierala uint16_t errs : VS_EBITS;/* Non-wrapping error count. */ 80*1a13f2e6SEdward Tomasz Napierala uint16_t flags : 16 - VS_EBITS; 81*1a13f2e6SEdward Tomasz Napierala }; 82*1a13f2e6SEdward Tomasz Napierala /* The voistat error count is capped to avoid wrapping. */ 83*1a13f2e6SEdward Tomasz Napierala #define VS_INCERRS(vs) do { \ 84*1a13f2e6SEdward Tomasz Napierala if ((vs)->errs < (1U << VS_EBITS) - 1) \ 85*1a13f2e6SEdward Tomasz Napierala (vs)->errs++; \ 86*1a13f2e6SEdward Tomasz Napierala } while (0) 87*1a13f2e6SEdward Tomasz Napierala 88*1a13f2e6SEdward Tomasz Napierala /* 89*1a13f2e6SEdward Tomasz Napierala * Ideas for flags: 90*1a13f2e6SEdward Tomasz Napierala * - Global or entity specific (global would imply use of counter(9)?) 91*1a13f2e6SEdward Tomasz Napierala * - Whether to reset stats on read or not 92*1a13f2e6SEdward Tomasz Napierala * - Signal an overflow? 93*1a13f2e6SEdward Tomasz Napierala * - Compressed voistat array 94*1a13f2e6SEdward Tomasz Napierala */ 95*1a13f2e6SEdward Tomasz Napierala #define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */ 96*1a13f2e6SEdward Tomasz Napierala struct voi { 97*1a13f2e6SEdward Tomasz Napierala int16_t id; /* VOI id. */ 98*1a13f2e6SEdward Tomasz Napierala enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */ 99*1a13f2e6SEdward Tomasz Napierala int8_t voistatmaxid; /* Largest allocated voistat index. */ 100*1a13f2e6SEdward Tomasz Napierala uint16_t stats_off; /* Blob offset for this VOIs stats. */ 101*1a13f2e6SEdward Tomasz Napierala uint16_t flags; 102*1a13f2e6SEdward Tomasz Napierala }; 103*1a13f2e6SEdward Tomasz Napierala 104*1a13f2e6SEdward Tomasz Napierala /* 105*1a13f2e6SEdward Tomasz Napierala * Memory for the entire blob is allocated as a slab and then offsets are 106*1a13f2e6SEdward Tomasz Napierala * maintained to carve up the slab into sections holding different data types. 107*1a13f2e6SEdward Tomasz Napierala * 108*1a13f2e6SEdward Tomasz Napierala * Ideas for flags: 109*1a13f2e6SEdward Tomasz Napierala * - Compressed voi array (trade off memory usage vs search time) 110*1a13f2e6SEdward Tomasz Napierala * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib) 111*1a13f2e6SEdward Tomasz Napierala */ 112*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 { 113*1a13f2e6SEdward Tomasz Napierala uint8_t abi; 114*1a13f2e6SEdward Tomasz Napierala uint8_t endian; 115*1a13f2e6SEdward Tomasz Napierala uint16_t flags; 116*1a13f2e6SEdward Tomasz Napierala uint16_t maxsz; 117*1a13f2e6SEdward Tomasz Napierala uint16_t cursz; 118*1a13f2e6SEdward Tomasz Napierala /* Fields from here down are opaque to consumers. */ 119*1a13f2e6SEdward Tomasz Napierala uint32_t tplhash; /* Base template hash ID. */ 120*1a13f2e6SEdward Tomasz Napierala uint16_t stats_off; /* voistat array blob offset. */ 121*1a13f2e6SEdward Tomasz Napierala uint16_t statsdata_off; /* voistatdata array blob offset. */ 122*1a13f2e6SEdward Tomasz Napierala sbintime_t created; /* Blob creation time. */ 123*1a13f2e6SEdward Tomasz Napierala sbintime_t lastrst; /* Time of last reset. */ 124*1a13f2e6SEdward Tomasz Napierala struct voi vois[]; /* Array indexed by [voi_id]. */ 125*1a13f2e6SEdward Tomasz Napierala } __aligned(sizeof(void *)); 126*1a13f2e6SEdward Tomasz Napierala _Static_assert(offsetof(struct statsblobv1, cursz) + 127*1a13f2e6SEdward Tomasz Napierala SIZEOF_MEMBER(struct statsblobv1, cursz) == sizeof(struct statsblob), 128*1a13f2e6SEdward Tomasz Napierala "statsblobv1 ABI mismatch"); 129*1a13f2e6SEdward Tomasz Napierala 130*1a13f2e6SEdward Tomasz Napierala struct statsblobv1_tpl { 131*1a13f2e6SEdward Tomasz Napierala struct metablob *mb; 132*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb; 133*1a13f2e6SEdward Tomasz Napierala }; 134*1a13f2e6SEdward Tomasz Napierala 135*1a13f2e6SEdward Tomasz Napierala /* Context passed to iterator callbacks. */ 136*1a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx { 137*1a13f2e6SEdward Tomasz Napierala void *usrctx; /* Caller supplied context. */ 138*1a13f2e6SEdward Tomasz Napierala uint32_t flags; /* Flags for current iteration. */ 139*1a13f2e6SEdward Tomasz Napierala int16_t vslot; /* struct voi slot index. */ 140*1a13f2e6SEdward Tomasz Napierala int8_t vsslot; /* struct voistat slot index. */ 141*1a13f2e6SEdward Tomasz Napierala }; 142*1a13f2e6SEdward Tomasz Napierala 143*1a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx { 144*1a13f2e6SEdward Tomasz Napierala struct sbuf *buf; 145*1a13f2e6SEdward Tomasz Napierala struct statsblob_tpl *tpl; 146*1a13f2e6SEdward Tomasz Napierala enum sb_str_fmt fmt; 147*1a13f2e6SEdward Tomasz Napierala uint32_t flags; 148*1a13f2e6SEdward Tomasz Napierala }; 149*1a13f2e6SEdward Tomasz Napierala 150*1a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx { 151*1a13f2e6SEdward Tomasz Napierala stats_blob_visitcb_t cb; 152*1a13f2e6SEdward Tomasz Napierala void *usrctx; 153*1a13f2e6SEdward Tomasz Napierala }; 154*1a13f2e6SEdward Tomasz Napierala 155*1a13f2e6SEdward Tomasz Napierala /* Stats blob iterator callback. */ 156*1a13f2e6SEdward Tomasz Napierala typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v, 157*1a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx); 158*1a13f2e6SEdward Tomasz Napierala 159*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 160*1a13f2e6SEdward Tomasz Napierala static struct rwlock tpllistlock; 161*1a13f2e6SEdward Tomasz Napierala RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock"); 162*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock) 163*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock) 164*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock) 165*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock) 166*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED) 167*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED) 168*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED) 169*1a13f2e6SEdward Tomasz Napierala MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory"); 170*1a13f2e6SEdward Tomasz Napierala #define stats_free(ptr) free((ptr), M_STATS) 171*1a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 172*1a13f2e6SEdward Tomasz Napierala static void stats_constructor(void); 173*1a13f2e6SEdward Tomasz Napierala static void stats_destructor(void); 174*1a13f2e6SEdward Tomasz Napierala static pthread_rwlock_t tpllistlock; 175*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock) 176*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock) 177*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK() 178*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock) 179*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK() 180*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_LOCK_ASSERT() do { } while (0) 181*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_RLOCK_ASSERT() do { } while (0) 182*1a13f2e6SEdward Tomasz Napierala #define TPL_LIST_WLOCK_ASSERT() do { } while (0) 183*1a13f2e6SEdward Tomasz Napierala #ifdef NDEBUG 184*1a13f2e6SEdward Tomasz Napierala #define KASSERT(cond, msg) do {} while (0) 185*1a13f2e6SEdward Tomasz Napierala #define stats_abort() do {} while (0) 186*1a13f2e6SEdward Tomasz Napierala #else /* ! NDEBUG */ 187*1a13f2e6SEdward Tomasz Napierala #define KASSERT(cond, msg) do { \ 188*1a13f2e6SEdward Tomasz Napierala if (!(cond)) { \ 189*1a13f2e6SEdward Tomasz Napierala panic msg; \ 190*1a13f2e6SEdward Tomasz Napierala } \ 191*1a13f2e6SEdward Tomasz Napierala } while (0) 192*1a13f2e6SEdward Tomasz Napierala #define stats_abort() abort() 193*1a13f2e6SEdward Tomasz Napierala #endif /* NDEBUG */ 194*1a13f2e6SEdward Tomasz Napierala #define stats_free(ptr) free(ptr) 195*1a13f2e6SEdward Tomasz Napierala #define panic(fmt, ...) do { \ 196*1a13f2e6SEdward Tomasz Napierala fprintf(stderr, (fmt), ##__VA_ARGS__); \ 197*1a13f2e6SEdward Tomasz Napierala stats_abort(); \ 198*1a13f2e6SEdward Tomasz Napierala } while (0) 199*1a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 200*1a13f2e6SEdward Tomasz Napierala 201*1a13f2e6SEdward Tomasz Napierala #define SB_V1_MAXSZ 65535 202*1a13f2e6SEdward Tomasz Napierala 203*1a13f2e6SEdward Tomasz Napierala /* Obtain a blob offset pointer. */ 204*1a13f2e6SEdward Tomasz Napierala #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off))) 205*1a13f2e6SEdward Tomasz Napierala 206*1a13f2e6SEdward Tomasz Napierala /* 207*1a13f2e6SEdward Tomasz Napierala * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a 208*1a13f2e6SEdward Tomasz Napierala * power of 2 size, we can shift instead of divide. The shift amount must be 209*1a13f2e6SEdward Tomasz Napierala * updated if sizeof(struct voi) ever changes, which the assert should catch. 210*1a13f2e6SEdward Tomasz Napierala */ 211*1a13f2e6SEdward Tomasz Napierala #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \ 212*1a13f2e6SEdward Tomasz Napierala sizeof(struct statsblobv1)) >> 3)) 213*1a13f2e6SEdward Tomasz Napierala _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch"); 214*1a13f2e6SEdward Tomasz Napierala 215*1a13f2e6SEdward Tomasz Napierala /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */ 216*1a13f2e6SEdward Tomasz Napierala const char *vs_stype2name[VS_NUM_STYPES] = { 217*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_VOISTATE] = "VOISTATE", 218*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_SUM] = "SUM", 219*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_MAX] = "MAX", 220*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_MIN] = "MIN", 221*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_HIST] = "HIST", 222*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_TDGST] = "TDGST", 223*1a13f2e6SEdward Tomasz Napierala }; 224*1a13f2e6SEdward Tomasz Napierala 225*1a13f2e6SEdward Tomasz Napierala const char *vs_stype2desc[VS_NUM_STYPES] = { 226*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)", 227*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_SUM] = "Simple arithmetic accumulator", 228*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_MAX] = "Maximum observed VOI value", 229*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_MIN] = "Minimum observed VOI value", 230*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_HIST] = "Histogram of observed VOI values", 231*1a13f2e6SEdward Tomasz Napierala [VS_STYPE_TDGST] = "t-digest of observed VOI values", 232*1a13f2e6SEdward Tomasz Napierala }; 233*1a13f2e6SEdward Tomasz Napierala 234*1a13f2e6SEdward Tomasz Napierala const char *vsd_dtype2name[VSD_NUM_DTYPES] = { 235*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = "VOISTATE", 236*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = "INT_S32", 237*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = "INT_U32", 238*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = "INT_S64", 239*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = "INT_U64", 240*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = "INT_SLONG", 241*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = "INT_ULONG", 242*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = "Q_S32", 243*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = "Q_U32", 244*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = "Q_S64", 245*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = "Q_U64", 246*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = "CRHIST32", 247*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = "DRHIST32", 248*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = "DVHIST32", 249*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = "CRHIST64", 250*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = "DRHIST64", 251*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = "DVHIST64", 252*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32", 253*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64", 254*1a13f2e6SEdward Tomasz Napierala }; 255*1a13f2e6SEdward Tomasz Napierala 256*1a13f2e6SEdward Tomasz Napierala const size_t vsd_dtype2size[VSD_NUM_DTYPES] = { 257*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate), 258*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32), 259*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32), 260*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64), 261*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64), 262*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong), 263*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong), 264*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32), 265*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32), 266*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64), 267*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64), 268*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32), 269*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32), 270*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32), 271*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64), 272*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64), 273*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64), 274*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32), 275*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64), 276*1a13f2e6SEdward Tomasz Napierala }; 277*1a13f2e6SEdward Tomasz Napierala 278*1a13f2e6SEdward Tomasz Napierala static const bool vsd_compoundtype[VSD_NUM_DTYPES] = { 279*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = true, 280*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = false, 281*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = false, 282*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = false, 283*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = false, 284*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = false, 285*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = false, 286*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = false, 287*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = false, 288*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = false, 289*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = false, 290*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST32] = true, 291*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST32] = true, 292*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST32] = true, 293*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_CRHIST64] = true, 294*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DRHIST64] = true, 295*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_DVHIST64] = true, 296*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST32] = true, 297*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_TDGSTCLUST64] = true, 298*1a13f2e6SEdward Tomasz Napierala }; 299*1a13f2e6SEdward Tomasz Napierala 300*1a13f2e6SEdward Tomasz Napierala const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = { 301*1a13f2e6SEdward Tomasz Napierala [LIM_MIN] = { 302*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = {0}, 303*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}}, 304*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}}, 305*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}}, 306*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}}, 307*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}}, 308*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}}, 309*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}}, 310*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}}, 311*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}}, 312*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}}, 313*1a13f2e6SEdward Tomasz Napierala }, 314*1a13f2e6SEdward Tomasz Napierala [LIM_MAX] = { 315*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_VOISTATE] = {0}, 316*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}}, 317*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}}, 318*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}}, 319*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}}, 320*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}}, 321*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}}, 322*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}}, 323*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}}, 324*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}}, 325*1a13f2e6SEdward Tomasz Napierala [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}}, 326*1a13f2e6SEdward Tomasz Napierala } 327*1a13f2e6SEdward Tomasz Napierala }; 328*1a13f2e6SEdward Tomasz Napierala 329*1a13f2e6SEdward Tomasz Napierala /* tpllistlock protects tpllist and ntpl */ 330*1a13f2e6SEdward Tomasz Napierala static uint32_t ntpl; 331*1a13f2e6SEdward Tomasz Napierala static struct statsblob_tpl **tpllist; 332*1a13f2e6SEdward Tomasz Napierala 333*1a13f2e6SEdward Tomasz Napierala static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz, 334*1a13f2e6SEdward Tomasz Napierala int flags); 335*1a13f2e6SEdward Tomasz Napierala //static void stats_v1_blob_finalise(struct statsblobv1 *sb); 336*1a13f2e6SEdward Tomasz Napierala static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 337*1a13f2e6SEdward Tomasz Napierala uint32_t flags); 338*1a13f2e6SEdward Tomasz Napierala static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 339*1a13f2e6SEdward Tomasz Napierala int newvoistatbytes, int newvoistatdatabytes); 340*1a13f2e6SEdward Tomasz Napierala static void stats_v1_blob_iter(struct statsblobv1 *sb, 341*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags); 342*1a13f2e6SEdward Tomasz Napierala static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, 343*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt); 344*1a13f2e6SEdward Tomasz Napierala 345*1a13f2e6SEdward Tomasz Napierala static inline int 346*1a13f2e6SEdward Tomasz Napierala ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2) 347*1a13f2e6SEdward Tomasz Napierala { 348*1a13f2e6SEdward Tomasz Napierala 349*1a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(c1->mu, c2->mu), 350*1a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 351*1a13f2e6SEdward Tomasz Napierala Q_RELPREC(c1->mu, c2->mu))); 352*1a13f2e6SEdward Tomasz Napierala 353*1a13f2e6SEdward Tomasz Napierala return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 354*1a13f2e6SEdward Tomasz Napierala } 355*1a13f2e6SEdward Tomasz Napierala ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp); 356*1a13f2e6SEdward Tomasz Napierala 357*1a13f2e6SEdward Tomasz Napierala static inline int 358*1a13f2e6SEdward Tomasz Napierala ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2) 359*1a13f2e6SEdward Tomasz Napierala { 360*1a13f2e6SEdward Tomasz Napierala 361*1a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(c1->mu, c2->mu), 362*1a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 363*1a13f2e6SEdward Tomasz Napierala Q_RELPREC(c1->mu, c2->mu))); 364*1a13f2e6SEdward Tomasz Napierala 365*1a13f2e6SEdward Tomasz Napierala return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 366*1a13f2e6SEdward Tomasz Napierala } 367*1a13f2e6SEdward Tomasz Napierala ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp); 368*1a13f2e6SEdward Tomasz Napierala 369*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 370*1a13f2e6SEdward Tomasz Napierala RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp); 371*1a13f2e6SEdward Tomasz Napierala RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp); 372*1a13f2e6SEdward Tomasz Napierala #endif 373*1a13f2e6SEdward Tomasz Napierala 374*1a13f2e6SEdward Tomasz Napierala static inline sbintime_t 375*1a13f2e6SEdward Tomasz Napierala stats_sbinuptime(void) 376*1a13f2e6SEdward Tomasz Napierala { 377*1a13f2e6SEdward Tomasz Napierala sbintime_t sbt; 378*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 379*1a13f2e6SEdward Tomasz Napierala 380*1a13f2e6SEdward Tomasz Napierala sbt = sbinuptime(); 381*1a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 382*1a13f2e6SEdward Tomasz Napierala struct timespec tp; 383*1a13f2e6SEdward Tomasz Napierala 384*1a13f2e6SEdward Tomasz Napierala clock_gettime(CLOCK_MONOTONIC_FAST, &tp); 385*1a13f2e6SEdward Tomasz Napierala sbt = tstosbt(tp); 386*1a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 387*1a13f2e6SEdward Tomasz Napierala 388*1a13f2e6SEdward Tomasz Napierala return (sbt); 389*1a13f2e6SEdward Tomasz Napierala } 390*1a13f2e6SEdward Tomasz Napierala 391*1a13f2e6SEdward Tomasz Napierala static inline void * 392*1a13f2e6SEdward Tomasz Napierala stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags) 393*1a13f2e6SEdward Tomasz Napierala { 394*1a13f2e6SEdward Tomasz Napierala 395*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 396*1a13f2e6SEdward Tomasz Napierala /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */ 397*1a13f2e6SEdward Tomasz Napierala if (!(flags & (M_WAITOK | M_NOWAIT))) 398*1a13f2e6SEdward Tomasz Napierala flags |= M_NOWAIT; 399*1a13f2e6SEdward Tomasz Napierala ptr = realloc(ptr, newsz, M_STATS, flags); 400*1a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 401*1a13f2e6SEdward Tomasz Napierala ptr = realloc(ptr, newsz); 402*1a13f2e6SEdward Tomasz Napierala if ((flags & M_ZERO) && ptr != NULL) { 403*1a13f2e6SEdward Tomasz Napierala if (oldsz == 0) 404*1a13f2e6SEdward Tomasz Napierala memset(ptr, '\0', newsz); 405*1a13f2e6SEdward Tomasz Napierala else if (newsz > oldsz) 406*1a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz); 407*1a13f2e6SEdward Tomasz Napierala } 408*1a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 409*1a13f2e6SEdward Tomasz Napierala 410*1a13f2e6SEdward Tomasz Napierala return (ptr); 411*1a13f2e6SEdward Tomasz Napierala } 412*1a13f2e6SEdward Tomasz Napierala 413*1a13f2e6SEdward Tomasz Napierala static inline char * 414*1a13f2e6SEdward Tomasz Napierala stats_strdup(const char *s, 415*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 416*1a13f2e6SEdward Tomasz Napierala int flags) 417*1a13f2e6SEdward Tomasz Napierala { 418*1a13f2e6SEdward Tomasz Napierala char *copy; 419*1a13f2e6SEdward Tomasz Napierala size_t len; 420*1a13f2e6SEdward Tomasz Napierala 421*1a13f2e6SEdward Tomasz Napierala if (!(flags & (M_WAITOK | M_NOWAIT))) 422*1a13f2e6SEdward Tomasz Napierala flags |= M_NOWAIT; 423*1a13f2e6SEdward Tomasz Napierala 424*1a13f2e6SEdward Tomasz Napierala len = strlen(s) + 1; 425*1a13f2e6SEdward Tomasz Napierala if ((copy = malloc(len, M_STATS, flags)) != NULL) 426*1a13f2e6SEdward Tomasz Napierala bcopy(s, copy, len); 427*1a13f2e6SEdward Tomasz Napierala 428*1a13f2e6SEdward Tomasz Napierala return (copy); 429*1a13f2e6SEdward Tomasz Napierala #else 430*1a13f2e6SEdward Tomasz Napierala int flags __unused) 431*1a13f2e6SEdward Tomasz Napierala { 432*1a13f2e6SEdward Tomasz Napierala return (strdup(s)); 433*1a13f2e6SEdward Tomasz Napierala #endif 434*1a13f2e6SEdward Tomasz Napierala } 435*1a13f2e6SEdward Tomasz Napierala 436*1a13f2e6SEdward Tomasz Napierala static inline void 437*1a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(struct statsblob_tpl *tpl) 438*1a13f2e6SEdward Tomasz Napierala { 439*1a13f2e6SEdward Tomasz Napierala 440*1a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK_ASSERT(); 441*1a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0); 442*1a13f2e6SEdward Tomasz Napierala for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) { 443*1a13f2e6SEdward Tomasz Napierala if (tpl->mb->voi_meta[voi_id].name != NULL) 444*1a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_str( 445*1a13f2e6SEdward Tomasz Napierala tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash); 446*1a13f2e6SEdward Tomasz Napierala } 447*1a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz, 448*1a13f2e6SEdward Tomasz Napierala tpl->mb->tplhash); 449*1a13f2e6SEdward Tomasz Napierala } 450*1a13f2e6SEdward Tomasz Napierala 451*1a13f2e6SEdward Tomasz Napierala static inline uint64_t 452*1a13f2e6SEdward Tomasz Napierala stats_pow_u64(uint64_t base, uint64_t exp) 453*1a13f2e6SEdward Tomasz Napierala { 454*1a13f2e6SEdward Tomasz Napierala uint64_t result = 1; 455*1a13f2e6SEdward Tomasz Napierala 456*1a13f2e6SEdward Tomasz Napierala while (exp) { 457*1a13f2e6SEdward Tomasz Napierala if (exp & 1) 458*1a13f2e6SEdward Tomasz Napierala result *= base; 459*1a13f2e6SEdward Tomasz Napierala exp >>= 1; 460*1a13f2e6SEdward Tomasz Napierala base *= base; 461*1a13f2e6SEdward Tomasz Napierala } 462*1a13f2e6SEdward Tomasz Napierala 463*1a13f2e6SEdward Tomasz Napierala return (result); 464*1a13f2e6SEdward Tomasz Napierala } 465*1a13f2e6SEdward Tomasz Napierala 466*1a13f2e6SEdward Tomasz Napierala static inline int 467*1a13f2e6SEdward Tomasz Napierala stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt, 468*1a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub) 469*1a13f2e6SEdward Tomasz Napierala { 470*1a13f2e6SEdward Tomasz Napierala uint64_t step = 0; 471*1a13f2e6SEdward Tomasz Napierala int error = 0; 472*1a13f2e6SEdward Tomasz Napierala 473*1a13f2e6SEdward Tomasz Napierala switch (info->scheme) { 474*1a13f2e6SEdward Tomasz Napierala case BKT_LIN: 475*1a13f2e6SEdward Tomasz Napierala step = info->lin.stepinc; 476*1a13f2e6SEdward Tomasz Napierala break; 477*1a13f2e6SEdward Tomasz Napierala case BKT_EXP: 478*1a13f2e6SEdward Tomasz Napierala step = stats_pow_u64(info->exp.stepbase, 479*1a13f2e6SEdward Tomasz Napierala info->exp.stepexp + curbkt); 480*1a13f2e6SEdward Tomasz Napierala break; 481*1a13f2e6SEdward Tomasz Napierala case BKT_LINEXP: 482*1a13f2e6SEdward Tomasz Napierala { 483*1a13f2e6SEdward Tomasz Napierala uint64_t curstepexp = 1; 484*1a13f2e6SEdward Tomasz Napierala 485*1a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) { 486*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 487*1a13f2e6SEdward Tomasz Napierala while ((int32_t)stats_pow_u64(info->linexp.stepbase, 488*1a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int32.s32) 489*1a13f2e6SEdward Tomasz Napierala curstepexp++; 490*1a13f2e6SEdward Tomasz Napierala break; 491*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 492*1a13f2e6SEdward Tomasz Napierala while ((uint32_t)stats_pow_u64(info->linexp.stepbase, 493*1a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int32.u32) 494*1a13f2e6SEdward Tomasz Napierala curstepexp++; 495*1a13f2e6SEdward Tomasz Napierala break; 496*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 497*1a13f2e6SEdward Tomasz Napierala while ((int64_t)stats_pow_u64(info->linexp.stepbase, 498*1a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int64.s64) 499*1a13f2e6SEdward Tomasz Napierala curstepexp++; 500*1a13f2e6SEdward Tomasz Napierala break; 501*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 502*1a13f2e6SEdward Tomasz Napierala while ((uint64_t)stats_pow_u64(info->linexp.stepbase, 503*1a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->int64.u64) 504*1a13f2e6SEdward Tomasz Napierala curstepexp++; 505*1a13f2e6SEdward Tomasz Napierala break; 506*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 507*1a13f2e6SEdward Tomasz Napierala while ((long)stats_pow_u64(info->linexp.stepbase, 508*1a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->intlong.slong) 509*1a13f2e6SEdward Tomasz Napierala curstepexp++; 510*1a13f2e6SEdward Tomasz Napierala break; 511*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 512*1a13f2e6SEdward Tomasz Napierala while ((unsigned long)stats_pow_u64(info->linexp.stepbase, 513*1a13f2e6SEdward Tomasz Napierala curstepexp) <= bkt_lb->intlong.ulong) 514*1a13f2e6SEdward Tomasz Napierala curstepexp++; 515*1a13f2e6SEdward Tomasz Napierala break; 516*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 517*1a13f2e6SEdward Tomasz Napierala while ((s32q_t)stats_pow_u64(info->linexp.stepbase, 518*1a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32)) 519*1a13f2e6SEdward Tomasz Napierala break; 520*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 521*1a13f2e6SEdward Tomasz Napierala while ((u32q_t)stats_pow_u64(info->linexp.stepbase, 522*1a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32)) 523*1a13f2e6SEdward Tomasz Napierala break; 524*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 525*1a13f2e6SEdward Tomasz Napierala while ((s64q_t)stats_pow_u64(info->linexp.stepbase, 526*1a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64)) 527*1a13f2e6SEdward Tomasz Napierala curstepexp++; 528*1a13f2e6SEdward Tomasz Napierala break; 529*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 530*1a13f2e6SEdward Tomasz Napierala while ((u64q_t)stats_pow_u64(info->linexp.stepbase, 531*1a13f2e6SEdward Tomasz Napierala curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64)) 532*1a13f2e6SEdward Tomasz Napierala curstepexp++; 533*1a13f2e6SEdward Tomasz Napierala break; 534*1a13f2e6SEdward Tomasz Napierala default: 535*1a13f2e6SEdward Tomasz Napierala break; 536*1a13f2e6SEdward Tomasz Napierala } 537*1a13f2e6SEdward Tomasz Napierala 538*1a13f2e6SEdward Tomasz Napierala step = stats_pow_u64(info->linexp.stepbase, curstepexp) / 539*1a13f2e6SEdward Tomasz Napierala info->linexp.linstepdiv; 540*1a13f2e6SEdward Tomasz Napierala if (step == 0) 541*1a13f2e6SEdward Tomasz Napierala step = 1; 542*1a13f2e6SEdward Tomasz Napierala break; 543*1a13f2e6SEdward Tomasz Napierala } 544*1a13f2e6SEdward Tomasz Napierala default: 545*1a13f2e6SEdward Tomasz Napierala break; 546*1a13f2e6SEdward Tomasz Napierala } 547*1a13f2e6SEdward Tomasz Napierala 548*1a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) { 549*1a13f2e6SEdward Tomasz Napierala *bkt_lb = info->usr.bkts[curbkt].lb; 550*1a13f2e6SEdward Tomasz Napierala *bkt_ub = info->usr.bkts[curbkt].ub; 551*1a13f2e6SEdward Tomasz Napierala } else if (step != 0) { 552*1a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) { 553*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 554*1a13f2e6SEdward Tomasz Napierala bkt_ub->int32.s32 += (int32_t)step; 555*1a13f2e6SEdward Tomasz Napierala break; 556*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 557*1a13f2e6SEdward Tomasz Napierala bkt_ub->int32.u32 += (uint32_t)step; 558*1a13f2e6SEdward Tomasz Napierala break; 559*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 560*1a13f2e6SEdward Tomasz Napierala bkt_ub->int64.s64 += (int64_t)step; 561*1a13f2e6SEdward Tomasz Napierala break; 562*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 563*1a13f2e6SEdward Tomasz Napierala bkt_ub->int64.u64 += (uint64_t)step; 564*1a13f2e6SEdward Tomasz Napierala break; 565*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 566*1a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.slong += (long)step; 567*1a13f2e6SEdward Tomasz Napierala break; 568*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 569*1a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.ulong += (unsigned long)step; 570*1a13f2e6SEdward Tomasz Napierala break; 571*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 572*1a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q32.sq32, step); 573*1a13f2e6SEdward Tomasz Napierala break; 574*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 575*1a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q32.uq32, step); 576*1a13f2e6SEdward Tomasz Napierala break; 577*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 578*1a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q64.sq64, step); 579*1a13f2e6SEdward Tomasz Napierala break; 580*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 581*1a13f2e6SEdward Tomasz Napierala error = Q_QADDI(&bkt_ub->q64.uq64, step); 582*1a13f2e6SEdward Tomasz Napierala break; 583*1a13f2e6SEdward Tomasz Napierala default: 584*1a13f2e6SEdward Tomasz Napierala break; 585*1a13f2e6SEdward Tomasz Napierala } 586*1a13f2e6SEdward Tomasz Napierala } else { /* info->scheme != BKT_USR && step == 0 */ 587*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 588*1a13f2e6SEdward Tomasz Napierala } 589*1a13f2e6SEdward Tomasz Napierala 590*1a13f2e6SEdward Tomasz Napierala return (error); 591*1a13f2e6SEdward Tomasz Napierala } 592*1a13f2e6SEdward Tomasz Napierala 593*1a13f2e6SEdward Tomasz Napierala static uint32_t 594*1a13f2e6SEdward Tomasz Napierala stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info) 595*1a13f2e6SEdward Tomasz Napierala { 596*1a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric bkt_lb, bkt_ub; 597*1a13f2e6SEdward Tomasz Napierala uint32_t nbkts; 598*1a13f2e6SEdward Tomasz Napierala int done; 599*1a13f2e6SEdward Tomasz Napierala 600*1a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) { 601*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */ 602*1a13f2e6SEdward Tomasz Napierala info->lb = info->usr.bkts[0].lb; 603*1a13f2e6SEdward Tomasz Napierala info->ub = info->usr.bkts[info->usr.nbkts - 1].lb; 604*1a13f2e6SEdward Tomasz Napierala } 605*1a13f2e6SEdward Tomasz Napierala 606*1a13f2e6SEdward Tomasz Napierala nbkts = 0; 607*1a13f2e6SEdward Tomasz Napierala done = 0; 608*1a13f2e6SEdward Tomasz Napierala bkt_ub = info->lb; 609*1a13f2e6SEdward Tomasz Napierala 610*1a13f2e6SEdward Tomasz Napierala do { 611*1a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub; 612*1a13f2e6SEdward Tomasz Napierala if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub)) 613*1a13f2e6SEdward Tomasz Napierala return (0); 614*1a13f2e6SEdward Tomasz Napierala 615*1a13f2e6SEdward Tomasz Napierala if (info->scheme == BKT_USR) 616*1a13f2e6SEdward Tomasz Napierala done = (nbkts == info->usr.nbkts); 617*1a13f2e6SEdward Tomasz Napierala else { 618*1a13f2e6SEdward Tomasz Napierala switch (info->voi_dtype) { 619*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 620*1a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int32.s32 > info->ub.int32.s32); 621*1a13f2e6SEdward Tomasz Napierala break; 622*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 623*1a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int32.u32 > info->ub.int32.u32); 624*1a13f2e6SEdward Tomasz Napierala break; 625*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 626*1a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int64.s64 > info->ub.int64.s64); 627*1a13f2e6SEdward Tomasz Napierala break; 628*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 629*1a13f2e6SEdward Tomasz Napierala done = (bkt_ub.int64.u64 > info->ub.int64.u64); 630*1a13f2e6SEdward Tomasz Napierala break; 631*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 632*1a13f2e6SEdward Tomasz Napierala done = (bkt_ub.intlong.slong > 633*1a13f2e6SEdward Tomasz Napierala info->ub.intlong.slong); 634*1a13f2e6SEdward Tomasz Napierala break; 635*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 636*1a13f2e6SEdward Tomasz Napierala done = (bkt_ub.intlong.ulong > 637*1a13f2e6SEdward Tomasz Napierala info->ub.intlong.ulong); 638*1a13f2e6SEdward Tomasz Napierala break; 639*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 640*1a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q32.sq32, 641*1a13f2e6SEdward Tomasz Napierala info->ub.q32.sq32); 642*1a13f2e6SEdward Tomasz Napierala break; 643*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 644*1a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q32.uq32, 645*1a13f2e6SEdward Tomasz Napierala info->ub.q32.uq32); 646*1a13f2e6SEdward Tomasz Napierala break; 647*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 648*1a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q64.sq64, 649*1a13f2e6SEdward Tomasz Napierala info->ub.q64.sq64); 650*1a13f2e6SEdward Tomasz Napierala break; 651*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 652*1a13f2e6SEdward Tomasz Napierala done = Q_QGTQ(bkt_ub.q64.uq64, 653*1a13f2e6SEdward Tomasz Napierala info->ub.q64.uq64); 654*1a13f2e6SEdward Tomasz Napierala break; 655*1a13f2e6SEdward Tomasz Napierala default: 656*1a13f2e6SEdward Tomasz Napierala return (0); 657*1a13f2e6SEdward Tomasz Napierala } 658*1a13f2e6SEdward Tomasz Napierala } 659*1a13f2e6SEdward Tomasz Napierala } while (!done); 660*1a13f2e6SEdward Tomasz Napierala 661*1a13f2e6SEdward Tomasz Napierala if (info->flags & VSD_HIST_LBOUND_INF) 662*1a13f2e6SEdward Tomasz Napierala nbkts++; 663*1a13f2e6SEdward Tomasz Napierala if (info->flags & VSD_HIST_UBOUND_INF) 664*1a13f2e6SEdward Tomasz Napierala nbkts++; 665*1a13f2e6SEdward Tomasz Napierala 666*1a13f2e6SEdward Tomasz Napierala return (nbkts); 667*1a13f2e6SEdward Tomasz Napierala } 668*1a13f2e6SEdward Tomasz Napierala 669*1a13f2e6SEdward Tomasz Napierala int 670*1a13f2e6SEdward Tomasz Napierala stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 671*1a13f2e6SEdward Tomasz Napierala struct vss_hist_hlpr_info *info) 672*1a13f2e6SEdward Tomasz Napierala { 673*1a13f2e6SEdward Tomasz Napierala struct voistatdata_hist *hist; 674*1a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub, 675*1a13f2e6SEdward Tomasz Napierala *ubinfbktlb, *ubinfbktub; 676*1a13f2e6SEdward Tomasz Napierala uint32_t bkt, nbkts, nloop; 677*1a13f2e6SEdward Tomasz Napierala 678*1a13f2e6SEdward Tomasz Napierala if (vss == NULL || info == NULL || (info->flags & 679*1a13f2e6SEdward Tomasz Napierala (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype == 680*1a13f2e6SEdward Tomasz Napierala VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64))) 681*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 682*1a13f2e6SEdward Tomasz Napierala 683*1a13f2e6SEdward Tomasz Napierala info->voi_dtype = voi_dtype; 684*1a13f2e6SEdward Tomasz Napierala 685*1a13f2e6SEdward Tomasz Napierala if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0) 686*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 687*1a13f2e6SEdward Tomasz Napierala 688*1a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) { 689*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 690*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts); 691*1a13f2e6SEdward Tomasz Napierala break; 692*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 693*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts); 694*1a13f2e6SEdward Tomasz Napierala break; 695*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 696*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts); 697*1a13f2e6SEdward Tomasz Napierala break; 698*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 699*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts); 700*1a13f2e6SEdward Tomasz Napierala break; 701*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 702*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts); 703*1a13f2e6SEdward Tomasz Napierala break; 704*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 705*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts); 706*1a13f2e6SEdward Tomasz Napierala break; 707*1a13f2e6SEdward Tomasz Napierala default: 708*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 709*1a13f2e6SEdward Tomasz Napierala } 710*1a13f2e6SEdward Tomasz Napierala 711*1a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 712*1a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL) 713*1a13f2e6SEdward Tomasz Napierala return (ENOMEM); 714*1a13f2e6SEdward Tomasz Napierala 715*1a13f2e6SEdward Tomasz Napierala hist = (struct voistatdata_hist *)vss->iv; 716*1a13f2e6SEdward Tomasz Napierala bkt_ub = info->lb; 717*1a13f2e6SEdward Tomasz Napierala 718*1a13f2e6SEdward Tomasz Napierala for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0; 719*1a13f2e6SEdward Tomasz Napierala bkt < nbkts; 720*1a13f2e6SEdward Tomasz Napierala bkt++, nloop++) { 721*1a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub; 722*1a13f2e6SEdward Tomasz Napierala if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub)) 723*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 724*1a13f2e6SEdward Tomasz Napierala 725*1a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) { 726*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 727*1a13f2e6SEdward Tomasz Napierala VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb; 728*1a13f2e6SEdward Tomasz Napierala break; 729*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 730*1a13f2e6SEdward Tomasz Napierala VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb; 731*1a13f2e6SEdward Tomasz Napierala VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub; 732*1a13f2e6SEdward Tomasz Napierala break; 733*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 734*1a13f2e6SEdward Tomasz Napierala VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb; 735*1a13f2e6SEdward Tomasz Napierala break; 736*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 737*1a13f2e6SEdward Tomasz Napierala VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb; 738*1a13f2e6SEdward Tomasz Napierala break; 739*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 740*1a13f2e6SEdward Tomasz Napierala VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb; 741*1a13f2e6SEdward Tomasz Napierala VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub; 742*1a13f2e6SEdward Tomasz Napierala break; 743*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 744*1a13f2e6SEdward Tomasz Napierala VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb; 745*1a13f2e6SEdward Tomasz Napierala break; 746*1a13f2e6SEdward Tomasz Napierala default: 747*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 748*1a13f2e6SEdward Tomasz Napierala } 749*1a13f2e6SEdward Tomasz Napierala } 750*1a13f2e6SEdward Tomasz Napierala 751*1a13f2e6SEdward Tomasz Napierala lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL; 752*1a13f2e6SEdward Tomasz Napierala 753*1a13f2e6SEdward Tomasz Napierala switch (info->hist_dtype) { 754*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 755*1a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb; 756*1a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb; 757*1a13f2e6SEdward Tomasz Napierala break; 758*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 759*1a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb; 760*1a13f2e6SEdward Tomasz Napierala lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub; 761*1a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb; 762*1a13f2e6SEdward Tomasz Napierala ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub; 763*1a13f2e6SEdward Tomasz Napierala break; 764*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 765*1a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb; 766*1a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb; 767*1a13f2e6SEdward Tomasz Napierala break; 768*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 769*1a13f2e6SEdward Tomasz Napierala lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb; 770*1a13f2e6SEdward Tomasz Napierala lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub; 771*1a13f2e6SEdward Tomasz Napierala ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb; 772*1a13f2e6SEdward Tomasz Napierala ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub; 773*1a13f2e6SEdward Tomasz Napierala break; 774*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 775*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 776*1a13f2e6SEdward Tomasz Napierala break; 777*1a13f2e6SEdward Tomasz Napierala default: 778*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 779*1a13f2e6SEdward Tomasz Napierala } 780*1a13f2e6SEdward Tomasz Napierala 781*1a13f2e6SEdward Tomasz Napierala if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) { 782*1a13f2e6SEdward Tomasz Napierala *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype]; 783*1a13f2e6SEdward Tomasz Napierala /* 784*1a13f2e6SEdward Tomasz Napierala * Assignment from numeric_limit array for Q types assigns max 785*1a13f2e6SEdward Tomasz Napierala * possible integral/fractional value for underlying data type, 786*1a13f2e6SEdward Tomasz Napierala * but we must set control bits for this specific histogram per 787*1a13f2e6SEdward Tomasz Napierala * the user's choice of fractional bits, which we extract from 788*1a13f2e6SEdward Tomasz Napierala * info->lb. 789*1a13f2e6SEdward Tomasz Napierala */ 790*1a13f2e6SEdward Tomasz Napierala if (info->voi_dtype == VSD_DTYPE_Q_S32 || 791*1a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U32) { 792*1a13f2e6SEdward Tomasz Napierala /* Signedness doesn't matter for setting control bits. */ 793*1a13f2e6SEdward Tomasz Napierala Q_SCVAL(lbinfbktlb->q32.sq32, 794*1a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q32.sq32)); 795*1a13f2e6SEdward Tomasz Napierala } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 796*1a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U64) { 797*1a13f2e6SEdward Tomasz Napierala /* Signedness doesn't matter for setting control bits. */ 798*1a13f2e6SEdward Tomasz Napierala Q_SCVAL(lbinfbktlb->q64.sq64, 799*1a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q64.sq64)); 800*1a13f2e6SEdward Tomasz Napierala } 801*1a13f2e6SEdward Tomasz Napierala if (lbinfbktub) 802*1a13f2e6SEdward Tomasz Napierala *lbinfbktub = info->lb; 803*1a13f2e6SEdward Tomasz Napierala } 804*1a13f2e6SEdward Tomasz Napierala if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) { 805*1a13f2e6SEdward Tomasz Napierala *ubinfbktlb = bkt_lb; 806*1a13f2e6SEdward Tomasz Napierala if (ubinfbktub) { 807*1a13f2e6SEdward Tomasz Napierala *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype]; 808*1a13f2e6SEdward Tomasz Napierala if (info->voi_dtype == VSD_DTYPE_Q_S32 || 809*1a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U32) { 810*1a13f2e6SEdward Tomasz Napierala Q_SCVAL(ubinfbktub->q32.sq32, 811*1a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q32.sq32)); 812*1a13f2e6SEdward Tomasz Napierala } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 813*1a13f2e6SEdward Tomasz Napierala info->voi_dtype == VSD_DTYPE_Q_U64) { 814*1a13f2e6SEdward Tomasz Napierala Q_SCVAL(ubinfbktub->q64.sq64, 815*1a13f2e6SEdward Tomasz Napierala Q_GCVAL(info->lb.q64.sq64)); 816*1a13f2e6SEdward Tomasz Napierala } 817*1a13f2e6SEdward Tomasz Napierala } 818*1a13f2e6SEdward Tomasz Napierala } 819*1a13f2e6SEdward Tomasz Napierala 820*1a13f2e6SEdward Tomasz Napierala return (0); 821*1a13f2e6SEdward Tomasz Napierala } 822*1a13f2e6SEdward Tomasz Napierala 823*1a13f2e6SEdward Tomasz Napierala int 824*1a13f2e6SEdward Tomasz Napierala stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 825*1a13f2e6SEdward Tomasz Napierala struct vss_tdgst_hlpr_info *info) 826*1a13f2e6SEdward Tomasz Napierala { 827*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst; 828*1a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 829*1a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 830*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 831*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 832*1a13f2e6SEdward Tomasz Napierala 833*1a13f2e6SEdward Tomasz Napierala info->voi_dtype = voi_dtype; 834*1a13f2e6SEdward Tomasz Napierala 835*1a13f2e6SEdward Tomasz Napierala switch (info->tdgst_dtype) { 836*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 837*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds); 838*1a13f2e6SEdward Tomasz Napierala break; 839*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 840*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds); 841*1a13f2e6SEdward Tomasz Napierala break; 842*1a13f2e6SEdward Tomasz Napierala default: 843*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 844*1a13f2e6SEdward Tomasz Napierala } 845*1a13f2e6SEdward Tomasz Napierala 846*1a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 847*1a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL) 848*1a13f2e6SEdward Tomasz Napierala return (ENOMEM); 849*1a13f2e6SEdward Tomasz Napierala 850*1a13f2e6SEdward Tomasz Napierala tdgst = (struct voistatdata_tdgst *)vss->iv; 851*1a13f2e6SEdward Tomasz Napierala 852*1a13f2e6SEdward Tomasz Napierala switch (info->tdgst_dtype) { 853*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 854*1a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 855*1a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) { 856*1a13f2e6SEdward Tomasz Napierala Q_INI(&ctd32->mu, 0, 0, info->prec); 857*1a13f2e6SEdward Tomasz Napierala } 858*1a13f2e6SEdward Tomasz Napierala break; 859*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 860*1a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 861*1a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) { 862*1a13f2e6SEdward Tomasz Napierala Q_INI(&ctd64->mu, 0, 0, info->prec); 863*1a13f2e6SEdward Tomasz Napierala } 864*1a13f2e6SEdward Tomasz Napierala break; 865*1a13f2e6SEdward Tomasz Napierala default: 866*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 867*1a13f2e6SEdward Tomasz Napierala } 868*1a13f2e6SEdward Tomasz Napierala 869*1a13f2e6SEdward Tomasz Napierala return (0); 870*1a13f2e6SEdward Tomasz Napierala } 871*1a13f2e6SEdward Tomasz Napierala 872*1a13f2e6SEdward Tomasz Napierala int 873*1a13f2e6SEdward Tomasz Napierala stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 874*1a13f2e6SEdward Tomasz Napierala struct vss_numeric_hlpr_info *info) 875*1a13f2e6SEdward Tomasz Napierala { 876*1a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric iv; 877*1a13f2e6SEdward Tomasz Napierala 878*1a13f2e6SEdward Tomasz Napierala switch (vss->stype) { 879*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM: 880*1a13f2e6SEdward Tomasz Napierala iv = stats_ctor_vsd_numeric(0); 881*1a13f2e6SEdward Tomasz Napierala break; 882*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN: 883*1a13f2e6SEdward Tomasz Napierala iv = numeric_limits[LIM_MAX][voi_dtype]; 884*1a13f2e6SEdward Tomasz Napierala break; 885*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX: 886*1a13f2e6SEdward Tomasz Napierala iv = numeric_limits[LIM_MIN][voi_dtype]; 887*1a13f2e6SEdward Tomasz Napierala break; 888*1a13f2e6SEdward Tomasz Napierala default: 889*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 890*1a13f2e6SEdward Tomasz Napierala } 891*1a13f2e6SEdward Tomasz Napierala 892*1a13f2e6SEdward Tomasz Napierala vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0); 893*1a13f2e6SEdward Tomasz Napierala if (vss->iv == NULL) 894*1a13f2e6SEdward Tomasz Napierala return (ENOMEM); 895*1a13f2e6SEdward Tomasz Napierala 896*1a13f2e6SEdward Tomasz Napierala vss->vs_dtype = voi_dtype; 897*1a13f2e6SEdward Tomasz Napierala vss->vsdsz = vsd_dtype2size[voi_dtype]; 898*1a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 899*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 900*1a13f2e6SEdward Tomasz Napierala *((int32_t *)vss->iv) = iv.int32.s32; 901*1a13f2e6SEdward Tomasz Napierala break; 902*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 903*1a13f2e6SEdward Tomasz Napierala *((uint32_t *)vss->iv) = iv.int32.u32; 904*1a13f2e6SEdward Tomasz Napierala break; 905*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 906*1a13f2e6SEdward Tomasz Napierala *((int64_t *)vss->iv) = iv.int64.s64; 907*1a13f2e6SEdward Tomasz Napierala break; 908*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 909*1a13f2e6SEdward Tomasz Napierala *((uint64_t *)vss->iv) = iv.int64.u64; 910*1a13f2e6SEdward Tomasz Napierala break; 911*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 912*1a13f2e6SEdward Tomasz Napierala *((long *)vss->iv) = iv.intlong.slong; 913*1a13f2e6SEdward Tomasz Napierala break; 914*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 915*1a13f2e6SEdward Tomasz Napierala *((unsigned long *)vss->iv) = iv.intlong.ulong; 916*1a13f2e6SEdward Tomasz Napierala break; 917*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 918*1a13f2e6SEdward Tomasz Napierala *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32, 919*1a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 920*1a13f2e6SEdward Tomasz Napierala break; 921*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 922*1a13f2e6SEdward Tomasz Napierala *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32, 923*1a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 924*1a13f2e6SEdward Tomasz Napierala break; 925*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 926*1a13f2e6SEdward Tomasz Napierala *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64, 927*1a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 928*1a13f2e6SEdward Tomasz Napierala break; 929*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 930*1a13f2e6SEdward Tomasz Napierala *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64, 931*1a13f2e6SEdward Tomasz Napierala Q_CTRLINI(info->prec)); 932*1a13f2e6SEdward Tomasz Napierala break; 933*1a13f2e6SEdward Tomasz Napierala default: 934*1a13f2e6SEdward Tomasz Napierala break; 935*1a13f2e6SEdward Tomasz Napierala } 936*1a13f2e6SEdward Tomasz Napierala 937*1a13f2e6SEdward Tomasz Napierala return (0); 938*1a13f2e6SEdward Tomasz Napierala } 939*1a13f2e6SEdward Tomasz Napierala 940*1a13f2e6SEdward Tomasz Napierala int 941*1a13f2e6SEdward Tomasz Napierala stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss, 942*1a13f2e6SEdward Tomasz Napierala struct voistatspec *vss) 943*1a13f2e6SEdward Tomasz Napierala { 944*1a13f2e6SEdward Tomasz Napierala int i, ret; 945*1a13f2e6SEdward Tomasz Napierala 946*1a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) { 947*1a13f2e6SEdward Tomasz Napierala if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i], 948*1a13f2e6SEdward Tomasz Napierala vss[i].hlprinfo)) != 0) 949*1a13f2e6SEdward Tomasz Napierala return (ret); 950*1a13f2e6SEdward Tomasz Napierala } 951*1a13f2e6SEdward Tomasz Napierala 952*1a13f2e6SEdward Tomasz Napierala return (0); 953*1a13f2e6SEdward Tomasz Napierala } 954*1a13f2e6SEdward Tomasz Napierala 955*1a13f2e6SEdward Tomasz Napierala void 956*1a13f2e6SEdward Tomasz Napierala stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss) 957*1a13f2e6SEdward Tomasz Napierala { 958*1a13f2e6SEdward Tomasz Napierala int i; 959*1a13f2e6SEdward Tomasz Napierala 960*1a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) { 961*1a13f2e6SEdward Tomasz Napierala if (vss[i].hlpr) { 962*1a13f2e6SEdward Tomasz Napierala stats_free((void *)vss[i].iv); 963*1a13f2e6SEdward Tomasz Napierala vss[i].iv = NULL; 964*1a13f2e6SEdward Tomasz Napierala } 965*1a13f2e6SEdward Tomasz Napierala } 966*1a13f2e6SEdward Tomasz Napierala } 967*1a13f2e6SEdward Tomasz Napierala 968*1a13f2e6SEdward Tomasz Napierala int 969*1a13f2e6SEdward Tomasz Napierala stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl) 970*1a13f2e6SEdward Tomasz Napierala { 971*1a13f2e6SEdward Tomasz Napierala int error; 972*1a13f2e6SEdward Tomasz Napierala 973*1a13f2e6SEdward Tomasz Napierala error = 0; 974*1a13f2e6SEdward Tomasz Napierala 975*1a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK(); 976*1a13f2e6SEdward Tomasz Napierala if (tpl_id < 0 || tpl_id >= (int)ntpl) { 977*1a13f2e6SEdward Tomasz Napierala error = ENOENT; 978*1a13f2e6SEdward Tomasz Napierala } else { 979*1a13f2e6SEdward Tomasz Napierala *tpl = tpllist[tpl_id]; 980*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: Acquire refcount on tpl. */ 981*1a13f2e6SEdward Tomasz Napierala } 982*1a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK(); 983*1a13f2e6SEdward Tomasz Napierala 984*1a13f2e6SEdward Tomasz Napierala return (error); 985*1a13f2e6SEdward Tomasz Napierala } 986*1a13f2e6SEdward Tomasz Napierala 987*1a13f2e6SEdward Tomasz Napierala int 988*1a13f2e6SEdward Tomasz Napierala stats_tpl_fetch_allocid(const char *name, uint32_t hash) 989*1a13f2e6SEdward Tomasz Napierala { 990*1a13f2e6SEdward Tomasz Napierala int i, tpl_id; 991*1a13f2e6SEdward Tomasz Napierala 992*1a13f2e6SEdward Tomasz Napierala tpl_id = -ESRCH; 993*1a13f2e6SEdward Tomasz Napierala 994*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 995*1a13f2e6SEdward Tomasz Napierala for (i = ntpl - 1; i >= 0; i--) { 996*1a13f2e6SEdward Tomasz Napierala if (name != NULL) { 997*1a13f2e6SEdward Tomasz Napierala if (strlen(name) == strlen(tpllist[i]->mb->tplname) && 998*1a13f2e6SEdward Tomasz Napierala strncmp(name, tpllist[i]->mb->tplname, 999*1a13f2e6SEdward Tomasz Napierala TPL_MAX_NAME_LEN) == 0 && (!hash || hash == 1000*1a13f2e6SEdward Tomasz Napierala tpllist[i]->mb->tplhash)) { 1001*1a13f2e6SEdward Tomasz Napierala tpl_id = i; 1002*1a13f2e6SEdward Tomasz Napierala break; 1003*1a13f2e6SEdward Tomasz Napierala } 1004*1a13f2e6SEdward Tomasz Napierala } else if (hash == tpllist[i]->mb->tplhash) { 1005*1a13f2e6SEdward Tomasz Napierala tpl_id = i; 1006*1a13f2e6SEdward Tomasz Napierala break; 1007*1a13f2e6SEdward Tomasz Napierala } 1008*1a13f2e6SEdward Tomasz Napierala } 1009*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 1010*1a13f2e6SEdward Tomasz Napierala 1011*1a13f2e6SEdward Tomasz Napierala return (tpl_id); 1012*1a13f2e6SEdward Tomasz Napierala } 1013*1a13f2e6SEdward Tomasz Napierala 1014*1a13f2e6SEdward Tomasz Napierala int 1015*1a13f2e6SEdward Tomasz Napierala stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len) 1016*1a13f2e6SEdward Tomasz Napierala { 1017*1a13f2e6SEdward Tomasz Napierala int error; 1018*1a13f2e6SEdward Tomasz Napierala 1019*1a13f2e6SEdward Tomasz Napierala error = 0; 1020*1a13f2e6SEdward Tomasz Napierala 1021*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 1022*1a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) { 1023*1a13f2e6SEdward Tomasz Napierala if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname)) 1024*1a13f2e6SEdward Tomasz Napierala strlcpy(buf, tpllist[tpl_id]->mb->tplname, len); 1025*1a13f2e6SEdward Tomasz Napierala else 1026*1a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 1027*1a13f2e6SEdward Tomasz Napierala } else 1028*1a13f2e6SEdward Tomasz Napierala error = ENOENT; 1029*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 1030*1a13f2e6SEdward Tomasz Napierala 1031*1a13f2e6SEdward Tomasz Napierala return (error); 1032*1a13f2e6SEdward Tomasz Napierala } 1033*1a13f2e6SEdward Tomasz Napierala 1034*1a13f2e6SEdward Tomasz Napierala int 1035*1a13f2e6SEdward Tomasz Napierala stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates, 1036*1a13f2e6SEdward Tomasz Napierala void *seed_bytes, size_t seed_len) 1037*1a13f2e6SEdward Tomasz Napierala { 1038*1a13f2e6SEdward Tomasz Napierala uint32_t cum_pct, rnd_pct; 1039*1a13f2e6SEdward Tomasz Napierala int i; 1040*1a13f2e6SEdward Tomasz Napierala 1041*1a13f2e6SEdward Tomasz Napierala cum_pct = 0; 1042*1a13f2e6SEdward Tomasz Napierala 1043*1a13f2e6SEdward Tomasz Napierala /* 1044*1a13f2e6SEdward Tomasz Napierala * Choose a pseudorandom or seeded number in range [0,100] and use 1045*1a13f2e6SEdward Tomasz Napierala * it to make a sampling decision and template selection where required. 1046*1a13f2e6SEdward Tomasz Napierala * If no seed is supplied, a PRNG is used to generate a pseudorandom 1047*1a13f2e6SEdward Tomasz Napierala * number so that every selection is independent. If a seed is supplied, 1048*1a13f2e6SEdward Tomasz Napierala * the caller desires random selection across different seeds, but 1049*1a13f2e6SEdward Tomasz Napierala * deterministic selection given the same seed. This is achieved by 1050*1a13f2e6SEdward Tomasz Napierala * hashing the seed and using the hash as the random number source. 1051*1a13f2e6SEdward Tomasz Napierala * 1052*1a13f2e6SEdward Tomasz Napierala * XXXLAS: Characterise hash function output distribution. 1053*1a13f2e6SEdward Tomasz Napierala */ 1054*1a13f2e6SEdward Tomasz Napierala if (seed_bytes == NULL) 1055*1a13f2e6SEdward Tomasz Napierala rnd_pct = random() / (INT32_MAX / 100); 1056*1a13f2e6SEdward Tomasz Napierala else 1057*1a13f2e6SEdward Tomasz Napierala rnd_pct = hash32_buf(seed_bytes, seed_len, 0) / 1058*1a13f2e6SEdward Tomasz Napierala (UINT32_MAX / 100U); 1059*1a13f2e6SEdward Tomasz Napierala 1060*1a13f2e6SEdward Tomasz Napierala /* 1061*1a13f2e6SEdward Tomasz Napierala * We map the randomly selected percentage on to the interval [0,100] 1062*1a13f2e6SEdward Tomasz Napierala * consisting of the cumulatively summed template sampling percentages. 1063*1a13f2e6SEdward Tomasz Napierala * The difference between the cumulative sum of all template sampling 1064*1a13f2e6SEdward Tomasz Napierala * percentages and 100 is treated as a NULL assignment i.e. no stats 1065*1a13f2e6SEdward Tomasz Napierala * template will be assigned, and -1 returned instead. 1066*1a13f2e6SEdward Tomasz Napierala */ 1067*1a13f2e6SEdward Tomasz Napierala for (i = 0; i < nrates; i++) { 1068*1a13f2e6SEdward Tomasz Napierala cum_pct += rates[i].tpl_sample_pct; 1069*1a13f2e6SEdward Tomasz Napierala 1070*1a13f2e6SEdward Tomasz Napierala KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__, 1071*1a13f2e6SEdward Tomasz Napierala cum_pct)); 1072*1a13f2e6SEdward Tomasz Napierala if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0) 1073*1a13f2e6SEdward Tomasz Napierala continue; 1074*1a13f2e6SEdward Tomasz Napierala 1075*1a13f2e6SEdward Tomasz Napierala return (rates[i].tpl_slot_id); 1076*1a13f2e6SEdward Tomasz Napierala } 1077*1a13f2e6SEdward Tomasz Napierala 1078*1a13f2e6SEdward Tomasz Napierala return (-1); 1079*1a13f2e6SEdward Tomasz Napierala } 1080*1a13f2e6SEdward Tomasz Napierala 1081*1a13f2e6SEdward Tomasz Napierala int 1082*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz, 1083*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 *src, uint32_t flags) 1084*1a13f2e6SEdward Tomasz Napierala { 1085*1a13f2e6SEdward Tomasz Napierala int error; 1086*1a13f2e6SEdward Tomasz Napierala 1087*1a13f2e6SEdward Tomasz Napierala error = 0; 1088*1a13f2e6SEdward Tomasz Napierala 1089*1a13f2e6SEdward Tomasz Napierala if (src == NULL || dst == NULL || 1090*1a13f2e6SEdward Tomasz Napierala src->cursz < sizeof(struct statsblob) || 1091*1a13f2e6SEdward Tomasz Napierala ((flags & SB_CLONE_ALLOCDST) && 1092*1a13f2e6SEdward Tomasz Napierala (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) { 1093*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 1094*1a13f2e6SEdward Tomasz Napierala } else if (flags & SB_CLONE_ALLOCDST) { 1095*1a13f2e6SEdward Tomasz Napierala *dst = stats_realloc(NULL, 0, src->cursz, 0); 1096*1a13f2e6SEdward Tomasz Napierala if (*dst) 1097*1a13f2e6SEdward Tomasz Napierala (*dst)->maxsz = dstmaxsz = src->cursz; 1098*1a13f2e6SEdward Tomasz Napierala else 1099*1a13f2e6SEdward Tomasz Napierala error = ENOMEM; 1100*1a13f2e6SEdward Tomasz Napierala } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) { 1101*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 1102*1a13f2e6SEdward Tomasz Napierala } 1103*1a13f2e6SEdward Tomasz Napierala 1104*1a13f2e6SEdward Tomasz Napierala if (!error) { 1105*1a13f2e6SEdward Tomasz Napierala size_t postcurszlen; 1106*1a13f2e6SEdward Tomasz Napierala 1107*1a13f2e6SEdward Tomasz Napierala /* 1108*1a13f2e6SEdward Tomasz Napierala * Clone src into dst except for the maxsz field. If dst is too 1109*1a13f2e6SEdward Tomasz Napierala * small to hold all of src, only copy src's header and return 1110*1a13f2e6SEdward Tomasz Napierala * EOVERFLOW. 1111*1a13f2e6SEdward Tomasz Napierala */ 1112*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 1113*1a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_USRDSTNOFAULT) 1114*1a13f2e6SEdward Tomasz Napierala copyout_nofault(src, *dst, 1115*1a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, maxsz)); 1116*1a13f2e6SEdward Tomasz Napierala else if (flags & SB_CLONE_USRDST) 1117*1a13f2e6SEdward Tomasz Napierala copyout(src, *dst, offsetof(struct statsblob, maxsz)); 1118*1a13f2e6SEdward Tomasz Napierala else 1119*1a13f2e6SEdward Tomasz Napierala #endif 1120*1a13f2e6SEdward Tomasz Napierala memcpy(*dst, src, offsetof(struct statsblob, maxsz)); 1121*1a13f2e6SEdward Tomasz Napierala 1122*1a13f2e6SEdward Tomasz Napierala if (dstmaxsz >= src->cursz) { 1123*1a13f2e6SEdward Tomasz Napierala postcurszlen = src->cursz - 1124*1a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, cursz); 1125*1a13f2e6SEdward Tomasz Napierala } else { 1126*1a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 1127*1a13f2e6SEdward Tomasz Napierala postcurszlen = sizeof(struct statsblob) - 1128*1a13f2e6SEdward Tomasz Napierala offsetof(struct statsblob, cursz); 1129*1a13f2e6SEdward Tomasz Napierala } 1130*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 1131*1a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_USRDSTNOFAULT) 1132*1a13f2e6SEdward Tomasz Napierala copyout_nofault(&(src->cursz), &((*dst)->cursz), 1133*1a13f2e6SEdward Tomasz Napierala postcurszlen); 1134*1a13f2e6SEdward Tomasz Napierala else if (flags & SB_CLONE_USRDST) 1135*1a13f2e6SEdward Tomasz Napierala copyout(&(src->cursz), &((*dst)->cursz), postcurszlen); 1136*1a13f2e6SEdward Tomasz Napierala else 1137*1a13f2e6SEdward Tomasz Napierala #endif 1138*1a13f2e6SEdward Tomasz Napierala memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen); 1139*1a13f2e6SEdward Tomasz Napierala } 1140*1a13f2e6SEdward Tomasz Napierala 1141*1a13f2e6SEdward Tomasz Napierala return (error); 1142*1a13f2e6SEdward Tomasz Napierala } 1143*1a13f2e6SEdward Tomasz Napierala 1144*1a13f2e6SEdward Tomasz Napierala int 1145*1a13f2e6SEdward Tomasz Napierala stats_v1_tpl_alloc(const char *name, uint32_t flags __unused) 1146*1a13f2e6SEdward Tomasz Napierala { 1147*1a13f2e6SEdward Tomasz Napierala struct statsblobv1_tpl *tpl, **newtpllist; 1148*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 *tpl_sb; 1149*1a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 1150*1a13f2e6SEdward Tomasz Napierala int tpl_id; 1151*1a13f2e6SEdward Tomasz Napierala 1152*1a13f2e6SEdward Tomasz Napierala if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN) 1153*1a13f2e6SEdward Tomasz Napierala return (-EINVAL); 1154*1a13f2e6SEdward Tomasz Napierala 1155*1a13f2e6SEdward Tomasz Napierala if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0) 1156*1a13f2e6SEdward Tomasz Napierala return (-EEXIST); 1157*1a13f2e6SEdward Tomasz Napierala 1158*1a13f2e6SEdward Tomasz Napierala tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO); 1159*1a13f2e6SEdward Tomasz Napierala tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO); 1160*1a13f2e6SEdward Tomasz Napierala tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO); 1161*1a13f2e6SEdward Tomasz Napierala 1162*1a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL && name != NULL) 1163*1a13f2e6SEdward Tomasz Napierala tpl_mb->tplname = stats_strdup(name, 0); 1164*1a13f2e6SEdward Tomasz Napierala 1165*1a13f2e6SEdward Tomasz Napierala if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL || 1166*1a13f2e6SEdward Tomasz Napierala tpl_mb->tplname == NULL) { 1167*1a13f2e6SEdward Tomasz Napierala stats_free(tpl); 1168*1a13f2e6SEdward Tomasz Napierala stats_free(tpl_sb); 1169*1a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL) { 1170*1a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb->tplname); 1171*1a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb); 1172*1a13f2e6SEdward Tomasz Napierala } 1173*1a13f2e6SEdward Tomasz Napierala return (-ENOMEM); 1174*1a13f2e6SEdward Tomasz Napierala } 1175*1a13f2e6SEdward Tomasz Napierala 1176*1a13f2e6SEdward Tomasz Napierala tpl->mb = tpl_mb; 1177*1a13f2e6SEdward Tomasz Napierala tpl->sb = tpl_sb; 1178*1a13f2e6SEdward Tomasz Napierala 1179*1a13f2e6SEdward Tomasz Napierala tpl_sb->abi = STATS_ABI_V1; 1180*1a13f2e6SEdward Tomasz Napierala tpl_sb->endian = 1181*1a13f2e6SEdward Tomasz Napierala #if BYTE_ORDER == LITTLE_ENDIAN 1182*1a13f2e6SEdward Tomasz Napierala SB_LE; 1183*1a13f2e6SEdward Tomasz Napierala #elif BYTE_ORDER == BIG_ENDIAN 1184*1a13f2e6SEdward Tomasz Napierala SB_BE; 1185*1a13f2e6SEdward Tomasz Napierala #else 1186*1a13f2e6SEdward Tomasz Napierala SB_UE; 1187*1a13f2e6SEdward Tomasz Napierala #endif 1188*1a13f2e6SEdward Tomasz Napierala tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1); 1189*1a13f2e6SEdward Tomasz Napierala tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1); 1190*1a13f2e6SEdward Tomasz Napierala 1191*1a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK(); 1192*1a13f2e6SEdward Tomasz Napierala newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *), 1193*1a13f2e6SEdward Tomasz Napierala (ntpl + 1) * sizeof(void *), 0); 1194*1a13f2e6SEdward Tomasz Napierala if (newtpllist != NULL) { 1195*1a13f2e6SEdward Tomasz Napierala tpl_id = ntpl++; 1196*1a13f2e6SEdward Tomasz Napierala tpllist = (struct statsblob_tpl **)newtpllist; 1197*1a13f2e6SEdward Tomasz Napierala tpllist[tpl_id] = (struct statsblob_tpl *)tpl; 1198*1a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(tpllist[tpl_id]); 1199*1a13f2e6SEdward Tomasz Napierala } else { 1200*1a13f2e6SEdward Tomasz Napierala stats_free(tpl); 1201*1a13f2e6SEdward Tomasz Napierala stats_free(tpl_sb); 1202*1a13f2e6SEdward Tomasz Napierala if (tpl_mb != NULL) { 1203*1a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb->tplname); 1204*1a13f2e6SEdward Tomasz Napierala stats_free(tpl_mb); 1205*1a13f2e6SEdward Tomasz Napierala } 1206*1a13f2e6SEdward Tomasz Napierala tpl_id = -ENOMEM; 1207*1a13f2e6SEdward Tomasz Napierala } 1208*1a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK(); 1209*1a13f2e6SEdward Tomasz Napierala 1210*1a13f2e6SEdward Tomasz Napierala return (tpl_id); 1211*1a13f2e6SEdward Tomasz Napierala } 1212*1a13f2e6SEdward Tomasz Napierala 1213*1a13f2e6SEdward Tomasz Napierala int 1214*1a13f2e6SEdward Tomasz Napierala stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name, 1215*1a13f2e6SEdward Tomasz Napierala enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss, 1216*1a13f2e6SEdward Tomasz Napierala uint32_t flags) 1217*1a13f2e6SEdward Tomasz Napierala { 1218*1a13f2e6SEdward Tomasz Napierala struct voi *voi; 1219*1a13f2e6SEdward Tomasz Napierala struct voistat *tmpstat; 1220*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 *tpl_sb; 1221*1a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 1222*1a13f2e6SEdward Tomasz Napierala int error, i, newstatdataidx, newvoibytes, newvoistatbytes, 1223*1a13f2e6SEdward Tomasz Napierala newvoistatdatabytes, newvoistatmaxid; 1224*1a13f2e6SEdward Tomasz Napierala uint32_t nbytes; 1225*1a13f2e6SEdward Tomasz Napierala 1226*1a13f2e6SEdward Tomasz Napierala if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || 1227*1a13f2e6SEdward Tomasz Napierala nvss == 0 || vss == NULL) 1228*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 1229*1a13f2e6SEdward Tomasz Napierala 1230*1a13f2e6SEdward Tomasz Napierala error = nbytes = newvoibytes = newvoistatbytes = 1231*1a13f2e6SEdward Tomasz Napierala newvoistatdatabytes = 0; 1232*1a13f2e6SEdward Tomasz Napierala newvoistatmaxid = -1; 1233*1a13f2e6SEdward Tomasz Napierala 1234*1a13f2e6SEdward Tomasz Napierala /* Calculate the number of bytes required for the new voistats. */ 1235*1a13f2e6SEdward Tomasz Napierala for (i = nvss - 1; i >= 0; i--) { 1236*1a13f2e6SEdward Tomasz Napierala if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES || 1237*1a13f2e6SEdward Tomasz Napierala vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES || 1238*1a13f2e6SEdward Tomasz Napierala vss[i].iv == NULL || vss[i].vsdsz == 0) 1239*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 1240*1a13f2e6SEdward Tomasz Napierala if ((int)vss[i].stype > newvoistatmaxid) 1241*1a13f2e6SEdward Tomasz Napierala newvoistatmaxid = vss[i].stype; 1242*1a13f2e6SEdward Tomasz Napierala newvoistatdatabytes += vss[i].vsdsz; 1243*1a13f2e6SEdward Tomasz Napierala } 1244*1a13f2e6SEdward Tomasz Napierala 1245*1a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) { 1246*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: VOI state bytes may need to vary based on stat types. */ 1247*1a13f2e6SEdward Tomasz Napierala newvoistatdatabytes += sizeof(struct voistatdata_voistate); 1248*1a13f2e6SEdward Tomasz Napierala } 1249*1a13f2e6SEdward Tomasz Napierala nbytes += newvoistatdatabytes; 1250*1a13f2e6SEdward Tomasz Napierala 1251*1a13f2e6SEdward Tomasz Napierala TPL_LIST_WLOCK(); 1252*1a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) { 1253*1a13f2e6SEdward Tomasz Napierala tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb; 1254*1a13f2e6SEdward Tomasz Napierala tpl_mb = tpllist[tpl_id]->mb; 1255*1a13f2e6SEdward Tomasz Napierala 1256*1a13f2e6SEdward Tomasz Napierala if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) { 1257*1a13f2e6SEdward Tomasz Napierala /* Adding a new VOI and associated stats. */ 1258*1a13f2e6SEdward Tomasz Napierala if (voi_id >= NVOIS(tpl_sb)) { 1259*1a13f2e6SEdward Tomasz Napierala /* We need to grow the tpl_sb->vois array. */ 1260*1a13f2e6SEdward Tomasz Napierala newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) * 1261*1a13f2e6SEdward Tomasz Napierala sizeof(struct voi); 1262*1a13f2e6SEdward Tomasz Napierala nbytes += newvoibytes; 1263*1a13f2e6SEdward Tomasz Napierala } 1264*1a13f2e6SEdward Tomasz Napierala newvoistatbytes = 1265*1a13f2e6SEdward Tomasz Napierala (newvoistatmaxid + 1) * sizeof(struct voistat); 1266*1a13f2e6SEdward Tomasz Napierala } else { 1267*1a13f2e6SEdward Tomasz Napierala /* Adding stats to an existing VOI. */ 1268*1a13f2e6SEdward Tomasz Napierala if (newvoistatmaxid > 1269*1a13f2e6SEdward Tomasz Napierala tpl_sb->vois[voi_id].voistatmaxid) { 1270*1a13f2e6SEdward Tomasz Napierala newvoistatbytes = (newvoistatmaxid - 1271*1a13f2e6SEdward Tomasz Napierala tpl_sb->vois[voi_id].voistatmaxid) * 1272*1a13f2e6SEdward Tomasz Napierala sizeof(struct voistat); 1273*1a13f2e6SEdward Tomasz Napierala } 1274*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: KPI does not yet support expanding VOIs. */ 1275*1a13f2e6SEdward Tomasz Napierala error = EOPNOTSUPP; 1276*1a13f2e6SEdward Tomasz Napierala } 1277*1a13f2e6SEdward Tomasz Napierala nbytes += newvoistatbytes; 1278*1a13f2e6SEdward Tomasz Napierala 1279*1a13f2e6SEdward Tomasz Napierala if (!error && newvoibytes > 0) { 1280*1a13f2e6SEdward Tomasz Napierala struct voi_meta *voi_meta = tpl_mb->voi_meta; 1281*1a13f2e6SEdward Tomasz Napierala 1282*1a13f2e6SEdward Tomasz Napierala voi_meta = stats_realloc(voi_meta, voi_meta == NULL ? 1283*1a13f2e6SEdward Tomasz Napierala 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta), 1284*1a13f2e6SEdward Tomasz Napierala (1 + voi_id) * sizeof(struct voi_meta), 1285*1a13f2e6SEdward Tomasz Napierala M_ZERO); 1286*1a13f2e6SEdward Tomasz Napierala 1287*1a13f2e6SEdward Tomasz Napierala if (voi_meta == NULL) 1288*1a13f2e6SEdward Tomasz Napierala error = ENOMEM; 1289*1a13f2e6SEdward Tomasz Napierala else 1290*1a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta = voi_meta; 1291*1a13f2e6SEdward Tomasz Napierala } 1292*1a13f2e6SEdward Tomasz Napierala 1293*1a13f2e6SEdward Tomasz Napierala if (!error) { 1294*1a13f2e6SEdward Tomasz Napierala /* NB: Resizing can change where tpl_sb points. */ 1295*1a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_expand(&tpl_sb, newvoibytes, 1296*1a13f2e6SEdward Tomasz Napierala newvoistatbytes, newvoistatdatabytes); 1297*1a13f2e6SEdward Tomasz Napierala } 1298*1a13f2e6SEdward Tomasz Napierala 1299*1a13f2e6SEdward Tomasz Napierala if (!error) { 1300*1a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name, 1301*1a13f2e6SEdward Tomasz Napierala 0); 1302*1a13f2e6SEdward Tomasz Napierala if (tpl_mb->voi_meta[voi_id].name == NULL) 1303*1a13f2e6SEdward Tomasz Napierala error = ENOMEM; 1304*1a13f2e6SEdward Tomasz Napierala } 1305*1a13f2e6SEdward Tomasz Napierala 1306*1a13f2e6SEdward Tomasz Napierala if (!error) { 1307*1a13f2e6SEdward Tomasz Napierala /* Update the template list with the resized pointer. */ 1308*1a13f2e6SEdward Tomasz Napierala tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb; 1309*1a13f2e6SEdward Tomasz Napierala 1310*1a13f2e6SEdward Tomasz Napierala /* Update the template. */ 1311*1a13f2e6SEdward Tomasz Napierala voi = &tpl_sb->vois[voi_id]; 1312*1a13f2e6SEdward Tomasz Napierala 1313*1a13f2e6SEdward Tomasz Napierala if (voi->id < 0) { 1314*1a13f2e6SEdward Tomasz Napierala /* VOI is new and needs to be initialised. */ 1315*1a13f2e6SEdward Tomasz Napierala voi->id = voi_id; 1316*1a13f2e6SEdward Tomasz Napierala voi->dtype = voi_dtype; 1317*1a13f2e6SEdward Tomasz Napierala voi->stats_off = tpl_sb->stats_off; 1318*1a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) 1319*1a13f2e6SEdward Tomasz Napierala voi->flags |= VOI_REQSTATE; 1320*1a13f2e6SEdward Tomasz Napierala } else { 1321*1a13f2e6SEdward Tomasz Napierala /* 1322*1a13f2e6SEdward Tomasz Napierala * XXXLAS: When this else block is written, the 1323*1a13f2e6SEdward Tomasz Napierala * "KPI does not yet support expanding VOIs" 1324*1a13f2e6SEdward Tomasz Napierala * error earlier in this function can be 1325*1a13f2e6SEdward Tomasz Napierala * removed. What is required here is to shuffle 1326*1a13f2e6SEdward Tomasz Napierala * the voistat array such that the new stats for 1327*1a13f2e6SEdward Tomasz Napierala * the voi are contiguous, which will displace 1328*1a13f2e6SEdward Tomasz Napierala * stats for other vois that reside after the 1329*1a13f2e6SEdward Tomasz Napierala * voi being updated. The other vois then need 1330*1a13f2e6SEdward Tomasz Napierala * to have their stats_off adjusted post 1331*1a13f2e6SEdward Tomasz Napierala * shuffle. 1332*1a13f2e6SEdward Tomasz Napierala */ 1333*1a13f2e6SEdward Tomasz Napierala } 1334*1a13f2e6SEdward Tomasz Napierala 1335*1a13f2e6SEdward Tomasz Napierala voi->voistatmaxid = newvoistatmaxid; 1336*1a13f2e6SEdward Tomasz Napierala newstatdataidx = 0; 1337*1a13f2e6SEdward Tomasz Napierala 1338*1a13f2e6SEdward Tomasz Napierala if (voi->flags & VOI_REQSTATE) { 1339*1a13f2e6SEdward Tomasz Napierala /* Initialise the voistate stat in slot 0. */ 1340*1a13f2e6SEdward Tomasz Napierala tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off); 1341*1a13f2e6SEdward Tomasz Napierala tmpstat->stype = VS_STYPE_VOISTATE; 1342*1a13f2e6SEdward Tomasz Napierala tmpstat->flags = 0; 1343*1a13f2e6SEdward Tomasz Napierala tmpstat->dtype = VSD_DTYPE_VOISTATE; 1344*1a13f2e6SEdward Tomasz Napierala newstatdataidx = tmpstat->dsz = 1345*1a13f2e6SEdward Tomasz Napierala sizeof(struct voistatdata_numeric); 1346*1a13f2e6SEdward Tomasz Napierala tmpstat->data_off = tpl_sb->statsdata_off; 1347*1a13f2e6SEdward Tomasz Napierala } 1348*1a13f2e6SEdward Tomasz Napierala 1349*1a13f2e6SEdward Tomasz Napierala for (i = 0; (uint32_t)i < nvss; i++) { 1350*1a13f2e6SEdward Tomasz Napierala tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off + 1351*1a13f2e6SEdward Tomasz Napierala (vss[i].stype * sizeof(struct voistat))); 1352*1a13f2e6SEdward Tomasz Napierala KASSERT(tmpstat->stype < 0, ("voistat %p " 1353*1a13f2e6SEdward Tomasz Napierala "already initialised", tmpstat)); 1354*1a13f2e6SEdward Tomasz Napierala tmpstat->stype = vss[i].stype; 1355*1a13f2e6SEdward Tomasz Napierala tmpstat->flags = vss[i].flags; 1356*1a13f2e6SEdward Tomasz Napierala tmpstat->dtype = vss[i].vs_dtype; 1357*1a13f2e6SEdward Tomasz Napierala tmpstat->dsz = vss[i].vsdsz; 1358*1a13f2e6SEdward Tomasz Napierala tmpstat->data_off = tpl_sb->statsdata_off + 1359*1a13f2e6SEdward Tomasz Napierala newstatdataidx; 1360*1a13f2e6SEdward Tomasz Napierala memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off), 1361*1a13f2e6SEdward Tomasz Napierala vss[i].iv, vss[i].vsdsz); 1362*1a13f2e6SEdward Tomasz Napierala newstatdataidx += vss[i].vsdsz; 1363*1a13f2e6SEdward Tomasz Napierala } 1364*1a13f2e6SEdward Tomasz Napierala 1365*1a13f2e6SEdward Tomasz Napierala /* Update the template version hash. */ 1366*1a13f2e6SEdward Tomasz Napierala stats_tpl_update_hash(tpllist[tpl_id]); 1367*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: Confirm tpl name/hash pair remains unique. */ 1368*1a13f2e6SEdward Tomasz Napierala } 1369*1a13f2e6SEdward Tomasz Napierala } else 1370*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 1371*1a13f2e6SEdward Tomasz Napierala TPL_LIST_WUNLOCK(); 1372*1a13f2e6SEdward Tomasz Napierala 1373*1a13f2e6SEdward Tomasz Napierala return (error); 1374*1a13f2e6SEdward Tomasz Napierala } 1375*1a13f2e6SEdward Tomasz Napierala 1376*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 * 1377*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused) 1378*1a13f2e6SEdward Tomasz Napierala { 1379*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb; 1380*1a13f2e6SEdward Tomasz Napierala int error; 1381*1a13f2e6SEdward Tomasz Napierala 1382*1a13f2e6SEdward Tomasz Napierala sb = NULL; 1383*1a13f2e6SEdward Tomasz Napierala 1384*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 1385*1a13f2e6SEdward Tomasz Napierala if (tpl_id < ntpl) { 1386*1a13f2e6SEdward Tomasz Napierala sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0); 1387*1a13f2e6SEdward Tomasz Napierala if (sb != NULL) { 1388*1a13f2e6SEdward Tomasz Napierala sb->maxsz = tpllist[tpl_id]->sb->maxsz; 1389*1a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_init_locked(sb, tpl_id, 0); 1390*1a13f2e6SEdward Tomasz Napierala } else 1391*1a13f2e6SEdward Tomasz Napierala error = ENOMEM; 1392*1a13f2e6SEdward Tomasz Napierala 1393*1a13f2e6SEdward Tomasz Napierala if (error) { 1394*1a13f2e6SEdward Tomasz Napierala stats_free(sb); 1395*1a13f2e6SEdward Tomasz Napierala sb = NULL; 1396*1a13f2e6SEdward Tomasz Napierala } 1397*1a13f2e6SEdward Tomasz Napierala } 1398*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 1399*1a13f2e6SEdward Tomasz Napierala 1400*1a13f2e6SEdward Tomasz Napierala return (sb); 1401*1a13f2e6SEdward Tomasz Napierala } 1402*1a13f2e6SEdward Tomasz Napierala 1403*1a13f2e6SEdward Tomasz Napierala void 1404*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_destroy(struct statsblobv1 *sb) 1405*1a13f2e6SEdward Tomasz Napierala { 1406*1a13f2e6SEdward Tomasz Napierala 1407*1a13f2e6SEdward Tomasz Napierala stats_free(sb); 1408*1a13f2e6SEdward Tomasz Napierala } 1409*1a13f2e6SEdward Tomasz Napierala 1410*1a13f2e6SEdward Tomasz Napierala int 1411*1a13f2e6SEdward Tomasz Napierala stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id, 1412*1a13f2e6SEdward Tomasz Napierala enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd, 1413*1a13f2e6SEdward Tomasz Napierala size_t *retvsdsz) 1414*1a13f2e6SEdward Tomasz Napierala { 1415*1a13f2e6SEdward Tomasz Napierala struct voi *v; 1416*1a13f2e6SEdward Tomasz Napierala struct voistat *vs; 1417*1a13f2e6SEdward Tomasz Napierala 1418*1a13f2e6SEdward Tomasz Napierala if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 || 1419*1a13f2e6SEdward Tomasz Napierala voi_id >= NVOIS(sb)) 1420*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 1421*1a13f2e6SEdward Tomasz Napierala 1422*1a13f2e6SEdward Tomasz Napierala v = &sb->vois[voi_id]; 1423*1a13f2e6SEdward Tomasz Napierala if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid) 1424*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 1425*1a13f2e6SEdward Tomasz Napierala 1426*1a13f2e6SEdward Tomasz Napierala vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat))); 1427*1a13f2e6SEdward Tomasz Napierala *retvsd = BLOB_OFFSET(sb, vs->data_off); 1428*1a13f2e6SEdward Tomasz Napierala if (retdtype != NULL) 1429*1a13f2e6SEdward Tomasz Napierala *retdtype = vs->dtype; 1430*1a13f2e6SEdward Tomasz Napierala if (retvsdsz != NULL) 1431*1a13f2e6SEdward Tomasz Napierala *retvsdsz = vs->dsz; 1432*1a13f2e6SEdward Tomasz Napierala 1433*1a13f2e6SEdward Tomasz Napierala return (0); 1434*1a13f2e6SEdward Tomasz Napierala } 1435*1a13f2e6SEdward Tomasz Napierala 1436*1a13f2e6SEdward Tomasz Napierala int 1437*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags) 1438*1a13f2e6SEdward Tomasz Napierala { 1439*1a13f2e6SEdward Tomasz Napierala int error; 1440*1a13f2e6SEdward Tomasz Napierala 1441*1a13f2e6SEdward Tomasz Napierala error = 0; 1442*1a13f2e6SEdward Tomasz Napierala 1443*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 1444*1a13f2e6SEdward Tomasz Napierala if (sb == NULL || tpl_id >= ntpl) { 1445*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 1446*1a13f2e6SEdward Tomasz Napierala } else { 1447*1a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_init_locked(sb, tpl_id, flags); 1448*1a13f2e6SEdward Tomasz Napierala } 1449*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 1450*1a13f2e6SEdward Tomasz Napierala 1451*1a13f2e6SEdward Tomasz Napierala return (error); 1452*1a13f2e6SEdward Tomasz Napierala } 1453*1a13f2e6SEdward Tomasz Napierala 1454*1a13f2e6SEdward Tomasz Napierala static inline int 1455*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 1456*1a13f2e6SEdward Tomasz Napierala uint32_t flags __unused) 1457*1a13f2e6SEdward Tomasz Napierala { 1458*1a13f2e6SEdward Tomasz Napierala int error; 1459*1a13f2e6SEdward Tomasz Napierala 1460*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK_ASSERT(); 1461*1a13f2e6SEdward Tomasz Napierala error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW; 1462*1a13f2e6SEdward Tomasz Napierala KASSERT(!error, 1463*1a13f2e6SEdward Tomasz Napierala ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz)); 1464*1a13f2e6SEdward Tomasz Napierala 1465*1a13f2e6SEdward Tomasz Napierala if (!error) { 1466*1a13f2e6SEdward Tomasz Napierala memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz); 1467*1a13f2e6SEdward Tomasz Napierala sb->created = sb->lastrst = stats_sbinuptime(); 1468*1a13f2e6SEdward Tomasz Napierala sb->tplhash = tpllist[tpl_id]->mb->tplhash; 1469*1a13f2e6SEdward Tomasz Napierala } 1470*1a13f2e6SEdward Tomasz Napierala 1471*1a13f2e6SEdward Tomasz Napierala return (error); 1472*1a13f2e6SEdward Tomasz Napierala } 1473*1a13f2e6SEdward Tomasz Napierala 1474*1a13f2e6SEdward Tomasz Napierala static int 1475*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 1476*1a13f2e6SEdward Tomasz Napierala int newvoistatbytes, int newvoistatdatabytes) 1477*1a13f2e6SEdward Tomasz Napierala { 1478*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 *sb; 1479*1a13f2e6SEdward Tomasz Napierala struct voi *tmpvoi; 1480*1a13f2e6SEdward Tomasz Napierala struct voistat *tmpvoistat, *voistat_array; 1481*1a13f2e6SEdward Tomasz Napierala int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats; 1482*1a13f2e6SEdward Tomasz Napierala 1483*1a13f2e6SEdward Tomasz Napierala KASSERT(newvoibytes % sizeof(struct voi) == 0, 1484*1a13f2e6SEdward Tomasz Napierala ("Bad newvoibytes %d", newvoibytes)); 1485*1a13f2e6SEdward Tomasz Napierala KASSERT(newvoistatbytes % sizeof(struct voistat) == 0, 1486*1a13f2e6SEdward Tomasz Napierala ("Bad newvoistatbytes %d", newvoistatbytes)); 1487*1a13f2e6SEdward Tomasz Napierala 1488*1a13f2e6SEdward Tomasz Napierala error = ((newvoibytes % sizeof(struct voi) == 0) && 1489*1a13f2e6SEdward Tomasz Napierala (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL; 1490*1a13f2e6SEdward Tomasz Napierala sb = *sbpp; 1491*1a13f2e6SEdward Tomasz Napierala nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes; 1492*1a13f2e6SEdward Tomasz Napierala 1493*1a13f2e6SEdward Tomasz Napierala /* 1494*1a13f2e6SEdward Tomasz Napierala * XXXLAS: Required until we gain support for flags which alter the 1495*1a13f2e6SEdward Tomasz Napierala * units of size/offset fields in key structs. 1496*1a13f2e6SEdward Tomasz Napierala */ 1497*1a13f2e6SEdward Tomasz Napierala if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ)) 1498*1a13f2e6SEdward Tomasz Napierala error = EFBIG; 1499*1a13f2e6SEdward Tomasz Napierala 1500*1a13f2e6SEdward Tomasz Napierala if (!error && (sb->cursz + nbytes > sb->maxsz)) { 1501*1a13f2e6SEdward Tomasz Napierala /* Need to expand our blob. */ 1502*1a13f2e6SEdward Tomasz Napierala sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO); 1503*1a13f2e6SEdward Tomasz Napierala if (sb != NULL) { 1504*1a13f2e6SEdward Tomasz Napierala sb->maxsz = sb->cursz + nbytes; 1505*1a13f2e6SEdward Tomasz Napierala *sbpp = sb; 1506*1a13f2e6SEdward Tomasz Napierala } else 1507*1a13f2e6SEdward Tomasz Napierala error = ENOMEM; 1508*1a13f2e6SEdward Tomasz Napierala } 1509*1a13f2e6SEdward Tomasz Napierala 1510*1a13f2e6SEdward Tomasz Napierala if (!error) { 1511*1a13f2e6SEdward Tomasz Napierala /* 1512*1a13f2e6SEdward Tomasz Napierala * Shuffle memory within the expanded blob working from the end 1513*1a13f2e6SEdward Tomasz Napierala * backwards, leaving gaps for the new voistat and voistatdata 1514*1a13f2e6SEdward Tomasz Napierala * structs at the beginning of their respective blob regions, 1515*1a13f2e6SEdward Tomasz Napierala * and for the new voi structs at the end of their blob region. 1516*1a13f2e6SEdward Tomasz Napierala */ 1517*1a13f2e6SEdward Tomasz Napierala memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes), 1518*1a13f2e6SEdward Tomasz Napierala BLOB_OFFSET(sb, sb->statsdata_off), 1519*1a13f2e6SEdward Tomasz Napierala sb->cursz - sb->statsdata_off); 1520*1a13f2e6SEdward Tomasz Napierala memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes + 1521*1a13f2e6SEdward Tomasz Napierala newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off), 1522*1a13f2e6SEdward Tomasz Napierala sb->statsdata_off - sb->stats_off); 1523*1a13f2e6SEdward Tomasz Napierala 1524*1a13f2e6SEdward Tomasz Napierala /* First index of new voi/voistat structs to be initialised. */ 1525*1a13f2e6SEdward Tomasz Napierala idxnewvois = NVOIS(sb); 1526*1a13f2e6SEdward Tomasz Napierala idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1; 1527*1a13f2e6SEdward Tomasz Napierala 1528*1a13f2e6SEdward Tomasz Napierala /* Update housekeeping variables and offsets. */ 1529*1a13f2e6SEdward Tomasz Napierala sb->cursz += nbytes; 1530*1a13f2e6SEdward Tomasz Napierala sb->stats_off += newvoibytes; 1531*1a13f2e6SEdward Tomasz Napierala sb->statsdata_off += newvoibytes + newvoistatbytes; 1532*1a13f2e6SEdward Tomasz Napierala 1533*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: Zeroing not strictly needed but aids debugging. */ 1534*1a13f2e6SEdward Tomasz Napierala memset(&sb->vois[idxnewvois], '\0', newvoibytes); 1535*1a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(sb, sb->stats_off), '\0', 1536*1a13f2e6SEdward Tomasz Napierala newvoistatbytes); 1537*1a13f2e6SEdward Tomasz Napierala memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0', 1538*1a13f2e6SEdward Tomasz Napierala newvoistatdatabytes); 1539*1a13f2e6SEdward Tomasz Napierala 1540*1a13f2e6SEdward Tomasz Napierala /* Initialise new voi array members and update offsets. */ 1541*1a13f2e6SEdward Tomasz Napierala for (i = 0; i < NVOIS(sb); i++) { 1542*1a13f2e6SEdward Tomasz Napierala tmpvoi = &sb->vois[i]; 1543*1a13f2e6SEdward Tomasz Napierala if (i >= idxnewvois) { 1544*1a13f2e6SEdward Tomasz Napierala tmpvoi->id = tmpvoi->voistatmaxid = -1; 1545*1a13f2e6SEdward Tomasz Napierala } else if (tmpvoi->id > -1) { 1546*1a13f2e6SEdward Tomasz Napierala tmpvoi->stats_off += newvoibytes + 1547*1a13f2e6SEdward Tomasz Napierala newvoistatbytes; 1548*1a13f2e6SEdward Tomasz Napierala } 1549*1a13f2e6SEdward Tomasz Napierala } 1550*1a13f2e6SEdward Tomasz Napierala 1551*1a13f2e6SEdward Tomasz Napierala /* Initialise new voistat array members and update offsets. */ 1552*1a13f2e6SEdward Tomasz Napierala nvoistats = (sb->statsdata_off - sb->stats_off) / 1553*1a13f2e6SEdward Tomasz Napierala sizeof(struct voistat); 1554*1a13f2e6SEdward Tomasz Napierala voistat_array = BLOB_OFFSET(sb, sb->stats_off); 1555*1a13f2e6SEdward Tomasz Napierala for (i = 0; i < nvoistats; i++) { 1556*1a13f2e6SEdward Tomasz Napierala tmpvoistat = &voistat_array[i]; 1557*1a13f2e6SEdward Tomasz Napierala if (i <= idxnewvoistats) { 1558*1a13f2e6SEdward Tomasz Napierala tmpvoistat->stype = -1; 1559*1a13f2e6SEdward Tomasz Napierala } else if (tmpvoistat->stype > -1) { 1560*1a13f2e6SEdward Tomasz Napierala tmpvoistat->data_off += nbytes; 1561*1a13f2e6SEdward Tomasz Napierala } 1562*1a13f2e6SEdward Tomasz Napierala } 1563*1a13f2e6SEdward Tomasz Napierala } 1564*1a13f2e6SEdward Tomasz Napierala 1565*1a13f2e6SEdward Tomasz Napierala return (error); 1566*1a13f2e6SEdward Tomasz Napierala } 1567*1a13f2e6SEdward Tomasz Napierala 1568*1a13f2e6SEdward Tomasz Napierala static void 1569*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_finalise(struct statsblobv1 *sb __unused) 1570*1a13f2e6SEdward Tomasz Napierala { 1571*1a13f2e6SEdward Tomasz Napierala 1572*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: Fill this in. */ 1573*1a13f2e6SEdward Tomasz Napierala } 1574*1a13f2e6SEdward Tomasz Napierala 1575*1a13f2e6SEdward Tomasz Napierala static void 1576*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb, 1577*1a13f2e6SEdward Tomasz Napierala void *usrctx, uint32_t flags) 1578*1a13f2e6SEdward Tomasz Napierala { 1579*1a13f2e6SEdward Tomasz Napierala struct voi *v; 1580*1a13f2e6SEdward Tomasz Napierala struct voistat *vs; 1581*1a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx ctx; 1582*1a13f2e6SEdward Tomasz Napierala int i, j, firstvoi; 1583*1a13f2e6SEdward Tomasz Napierala 1584*1a13f2e6SEdward Tomasz Napierala ctx.usrctx = usrctx; 1585*1a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_FIRST_CB; 1586*1a13f2e6SEdward Tomasz Napierala ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT | 1587*1a13f2e6SEdward Tomasz Napierala SB_IT_LAST_VOISTAT); 1588*1a13f2e6SEdward Tomasz Napierala firstvoi = 1; 1589*1a13f2e6SEdward Tomasz Napierala 1590*1a13f2e6SEdward Tomasz Napierala for (i = 0; i < NVOIS(sb); i++) { 1591*1a13f2e6SEdward Tomasz Napierala v = &sb->vois[i]; 1592*1a13f2e6SEdward Tomasz Napierala ctx.vslot = i; 1593*1a13f2e6SEdward Tomasz Napierala ctx.vsslot = -1; 1594*1a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_FIRST_VOISTAT; 1595*1a13f2e6SEdward Tomasz Napierala 1596*1a13f2e6SEdward Tomasz Napierala if (firstvoi) 1597*1a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_FIRST_VOI; 1598*1a13f2e6SEdward Tomasz Napierala else if (i == (NVOIS(sb) - 1)) 1599*1a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB; 1600*1a13f2e6SEdward Tomasz Napierala 1601*1a13f2e6SEdward Tomasz Napierala if (v->id < 0 && (flags & SB_IT_NULLVOI)) { 1602*1a13f2e6SEdward Tomasz Napierala if (icb(sb, v, NULL, &ctx)) 1603*1a13f2e6SEdward Tomasz Napierala return; 1604*1a13f2e6SEdward Tomasz Napierala firstvoi = 0; 1605*1a13f2e6SEdward Tomasz Napierala ctx.flags &= ~SB_IT_FIRST_CB; 1606*1a13f2e6SEdward Tomasz Napierala } 1607*1a13f2e6SEdward Tomasz Napierala 1608*1a13f2e6SEdward Tomasz Napierala /* If NULL voi, v->voistatmaxid == -1 */ 1609*1a13f2e6SEdward Tomasz Napierala for (j = 0; j <= v->voistatmaxid; j++) { 1610*1a13f2e6SEdward Tomasz Napierala vs = &((struct voistat *)BLOB_OFFSET(sb, 1611*1a13f2e6SEdward Tomasz Napierala v->stats_off))[j]; 1612*1a13f2e6SEdward Tomasz Napierala if (vs->stype < 0 && 1613*1a13f2e6SEdward Tomasz Napierala !(flags & SB_IT_NULLVOISTAT)) 1614*1a13f2e6SEdward Tomasz Napierala continue; 1615*1a13f2e6SEdward Tomasz Napierala 1616*1a13f2e6SEdward Tomasz Napierala if (j == v->voistatmaxid) { 1617*1a13f2e6SEdward Tomasz Napierala ctx.flags |= SB_IT_LAST_VOISTAT; 1618*1a13f2e6SEdward Tomasz Napierala if (i == (NVOIS(sb) - 1)) 1619*1a13f2e6SEdward Tomasz Napierala ctx.flags |= 1620*1a13f2e6SEdward Tomasz Napierala SB_IT_LAST_CB; 1621*1a13f2e6SEdward Tomasz Napierala } else 1622*1a13f2e6SEdward Tomasz Napierala ctx.flags &= ~SB_IT_LAST_CB; 1623*1a13f2e6SEdward Tomasz Napierala 1624*1a13f2e6SEdward Tomasz Napierala ctx.vsslot = j; 1625*1a13f2e6SEdward Tomasz Napierala if (icb(sb, v, vs, &ctx)) 1626*1a13f2e6SEdward Tomasz Napierala return; 1627*1a13f2e6SEdward Tomasz Napierala 1628*1a13f2e6SEdward Tomasz Napierala ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT | 1629*1a13f2e6SEdward Tomasz Napierala SB_IT_LAST_VOISTAT); 1630*1a13f2e6SEdward Tomasz Napierala } 1631*1a13f2e6SEdward Tomasz Napierala ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI); 1632*1a13f2e6SEdward Tomasz Napierala } 1633*1a13f2e6SEdward Tomasz Napierala } 1634*1a13f2e6SEdward Tomasz Napierala 1635*1a13f2e6SEdward Tomasz Napierala static inline void 1636*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused, 1637*1a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype, 1638*1a13f2e6SEdward Tomasz Napierala size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 1639*1a13f2e6SEdward Tomasz Napierala { 1640*1a13f2e6SEdward Tomasz Napierala const struct ctdth32 *ctd32tree; 1641*1a13f2e6SEdward Tomasz Napierala const struct ctdth64 *ctd64tree; 1642*1a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgstctd32 *ctd32; 1643*1a13f2e6SEdward Tomasz Napierala const struct voistatdata_tdgstctd64 *ctd64; 1644*1a13f2e6SEdward Tomasz Napierala const char *fmtstr; 1645*1a13f2e6SEdward Tomasz Napierala uint64_t smplcnt, compcnt; 1646*1a13f2e6SEdward Tomasz Napierala int is32bit, qmaxstrlen; 1647*1a13f2e6SEdward Tomasz Napierala uint16_t maxctds, curctds; 1648*1a13f2e6SEdward Tomasz Napierala 1649*1a13f2e6SEdward Tomasz Napierala switch (tdgst_dtype) { 1650*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 1651*1a13f2e6SEdward Tomasz Napierala smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt; 1652*1a13f2e6SEdward Tomasz Napierala compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt; 1653*1a13f2e6SEdward Tomasz Napierala maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 1654*1a13f2e6SEdward Tomasz Napierala curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 1655*1a13f2e6SEdward Tomasz Napierala ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree; 1656*1a13f2e6SEdward Tomasz Napierala ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) : 1657*1a13f2e6SEdward Tomasz Napierala ARB_CMIN(ctdth32, ctd32tree)); 1658*1a13f2e6SEdward Tomasz Napierala qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10); 1659*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 1660*1a13f2e6SEdward Tomasz Napierala ctd64tree = NULL; 1661*1a13f2e6SEdward Tomasz Napierala ctd64 = NULL; 1662*1a13f2e6SEdward Tomasz Napierala break; 1663*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 1664*1a13f2e6SEdward Tomasz Napierala smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt; 1665*1a13f2e6SEdward Tomasz Napierala compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt; 1666*1a13f2e6SEdward Tomasz Napierala maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 1667*1a13f2e6SEdward Tomasz Napierala curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 1668*1a13f2e6SEdward Tomasz Napierala ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree; 1669*1a13f2e6SEdward Tomasz Napierala ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) : 1670*1a13f2e6SEdward Tomasz Napierala ARB_CMIN(ctdth64, ctd64tree)); 1671*1a13f2e6SEdward Tomasz Napierala qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10); 1672*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 1673*1a13f2e6SEdward Tomasz Napierala ctd32tree = NULL; 1674*1a13f2e6SEdward Tomasz Napierala ctd32 = NULL; 1675*1a13f2e6SEdward Tomasz Napierala break; 1676*1a13f2e6SEdward Tomasz Napierala default: 1677*1a13f2e6SEdward Tomasz Napierala return; 1678*1a13f2e6SEdward Tomasz Napierala } 1679*1a13f2e6SEdward Tomasz Napierala 1680*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1681*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1682*1a13f2e6SEdward Tomasz Napierala fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu"; 1683*1a13f2e6SEdward Tomasz Napierala break; 1684*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1685*1a13f2e6SEdward Tomasz Napierala default: 1686*1a13f2e6SEdward Tomasz Napierala fmtstr = 1687*1a13f2e6SEdward Tomasz Napierala "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu," 1688*1a13f2e6SEdward Tomasz Napierala "\"nctds\":%hu,\"ctds\":["; 1689*1a13f2e6SEdward Tomasz Napierala break; 1690*1a13f2e6SEdward Tomasz Napierala } 1691*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt, 1692*1a13f2e6SEdward Tomasz Napierala maxctds, curctds); 1693*1a13f2e6SEdward Tomasz Napierala 1694*1a13f2e6SEdward Tomasz Napierala while ((is32bit ? NULL != ctd32 : NULL != ctd64)) { 1695*1a13f2e6SEdward Tomasz Napierala char qstr[qmaxstrlen]; 1696*1a13f2e6SEdward Tomasz Napierala 1697*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1698*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1699*1a13f2e6SEdward Tomasz Napierala fmtstr = "\n\t\t\t\t"; 1700*1a13f2e6SEdward Tomasz Napierala break; 1701*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1702*1a13f2e6SEdward Tomasz Napierala default: 1703*1a13f2e6SEdward Tomasz Napierala fmtstr = "{"; 1704*1a13f2e6SEdward Tomasz Napierala break; 1705*1a13f2e6SEdward Tomasz Napierala } 1706*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 1707*1a13f2e6SEdward Tomasz Napierala 1708*1a13f2e6SEdward Tomasz Napierala if (objdump) { 1709*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1710*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1711*1a13f2e6SEdward Tomasz Napierala fmtstr = "ctd[%hu]."; 1712*1a13f2e6SEdward Tomasz Napierala break; 1713*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1714*1a13f2e6SEdward Tomasz Napierala default: 1715*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"ctd\":%hu,"; 1716*1a13f2e6SEdward Tomasz Napierala break; 1717*1a13f2e6SEdward Tomasz Napierala } 1718*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ? 1719*1a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd32tree, ctd32) : 1720*1a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, ctd64)); 1721*1a13f2e6SEdward Tomasz Napierala } 1722*1a13f2e6SEdward Tomasz Napierala 1723*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1724*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1725*1a13f2e6SEdward Tomasz Napierala fmtstr = "{mu="; 1726*1a13f2e6SEdward Tomasz Napierala break; 1727*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1728*1a13f2e6SEdward Tomasz Napierala default: 1729*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"mu\":"; 1730*1a13f2e6SEdward Tomasz Napierala break; 1731*1a13f2e6SEdward Tomasz Napierala } 1732*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 1733*1a13f2e6SEdward Tomasz Napierala Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr, 1734*1a13f2e6SEdward Tomasz Napierala sizeof(qstr)); 1735*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 1736*1a13f2e6SEdward Tomasz Napierala 1737*1a13f2e6SEdward Tomasz Napierala 1738*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1739*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1740*1a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 1741*1a13f2e6SEdward Tomasz Napierala break; 1742*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1743*1a13f2e6SEdward Tomasz Napierala default: 1744*1a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 1745*1a13f2e6SEdward Tomasz Napierala break; 1746*1a13f2e6SEdward Tomasz Napierala } 1747*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, 1748*1a13f2e6SEdward Tomasz Napierala is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt); 1749*1a13f2e6SEdward Tomasz Napierala 1750*1a13f2e6SEdward Tomasz Napierala if (is32bit) 1751*1a13f2e6SEdward Tomasz Napierala ctd32 = (objdump ? ARB_CNODE(ctd32tree, 1752*1a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd32tree, ctd32) + 1) : 1753*1a13f2e6SEdward Tomasz Napierala ARB_CNEXT(ctdth32, ctd32tree, ctd32)); 1754*1a13f2e6SEdward Tomasz Napierala else 1755*1a13f2e6SEdward Tomasz Napierala ctd64 = (objdump ? ARB_CNODE(ctd64tree, 1756*1a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, ctd64) + 1) : 1757*1a13f2e6SEdward Tomasz Napierala ARB_CNEXT(ctdth64, ctd64tree, ctd64)); 1758*1a13f2e6SEdward Tomasz Napierala 1759*1a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON && 1760*1a13f2e6SEdward Tomasz Napierala (is32bit ? NULL != ctd32 : NULL != ctd64)) 1761*1a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ','); 1762*1a13f2e6SEdward Tomasz Napierala } 1763*1a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON) 1764*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "]"); 1765*1a13f2e6SEdward Tomasz Napierala } 1766*1a13f2e6SEdward Tomasz Napierala 1767*1a13f2e6SEdward Tomasz Napierala static inline void 1768*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype, 1769*1a13f2e6SEdward Tomasz Napierala const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype, 1770*1a13f2e6SEdward Tomasz Napierala size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 1771*1a13f2e6SEdward Tomasz Napierala { 1772*1a13f2e6SEdward Tomasz Napierala const struct voistatdata_numeric *bkt_lb, *bkt_ub; 1773*1a13f2e6SEdward Tomasz Napierala const char *fmtstr; 1774*1a13f2e6SEdward Tomasz Napierala int is32bit; 1775*1a13f2e6SEdward Tomasz Napierala uint16_t i, nbkts; 1776*1a13f2e6SEdward Tomasz Napierala 1777*1a13f2e6SEdward Tomasz Napierala switch (hist_dtype) { 1778*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 1779*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz); 1780*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 1781*1a13f2e6SEdward Tomasz Napierala break; 1782*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 1783*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz); 1784*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 1785*1a13f2e6SEdward Tomasz Napierala break; 1786*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 1787*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz); 1788*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 1789*1a13f2e6SEdward Tomasz Napierala break; 1790*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 1791*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz); 1792*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 1793*1a13f2e6SEdward Tomasz Napierala break; 1794*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 1795*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz); 1796*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 1797*1a13f2e6SEdward Tomasz Napierala break; 1798*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 1799*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz); 1800*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 1801*1a13f2e6SEdward Tomasz Napierala break; 1802*1a13f2e6SEdward Tomasz Napierala default: 1803*1a13f2e6SEdward Tomasz Napierala return; 1804*1a13f2e6SEdward Tomasz Napierala } 1805*1a13f2e6SEdward Tomasz Napierala 1806*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1807*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1808*1a13f2e6SEdward Tomasz Napierala fmtstr = "nbkts=%hu, "; 1809*1a13f2e6SEdward Tomasz Napierala break; 1810*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1811*1a13f2e6SEdward Tomasz Napierala default: 1812*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"nbkts\":%hu,"; 1813*1a13f2e6SEdward Tomasz Napierala break; 1814*1a13f2e6SEdward Tomasz Napierala } 1815*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, nbkts); 1816*1a13f2e6SEdward Tomasz Napierala 1817*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1818*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1819*1a13f2e6SEdward Tomasz Napierala fmtstr = (is32bit ? "oob=%u" : "oob=%ju"); 1820*1a13f2e6SEdward Tomasz Napierala break; 1821*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1822*1a13f2e6SEdward Tomasz Napierala default: 1823*1a13f2e6SEdward Tomasz Napierala fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" : 1824*1a13f2e6SEdward Tomasz Napierala "\"oob\":%ju,\"bkts\":["); 1825*1a13f2e6SEdward Tomasz Napierala break; 1826*1a13f2e6SEdward Tomasz Napierala } 1827*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist, 1828*1a13f2e6SEdward Tomasz Napierala hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, 1829*1a13f2e6SEdward Tomasz Napierala hist_dtype, oob)); 1830*1a13f2e6SEdward Tomasz Napierala 1831*1a13f2e6SEdward Tomasz Napierala for (i = 0; i < nbkts; i++) { 1832*1a13f2e6SEdward Tomasz Napierala switch (hist_dtype) { 1833*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 1834*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 1835*1a13f2e6SEdward Tomasz Napierala bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype, 1836*1a13f2e6SEdward Tomasz Napierala bkts[i].lb); 1837*1a13f2e6SEdward Tomasz Napierala if (i < nbkts - 1) 1838*1a13f2e6SEdward Tomasz Napierala bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist, 1839*1a13f2e6SEdward Tomasz Napierala hist_dtype, bkts[i + 1].lb); 1840*1a13f2e6SEdward Tomasz Napierala else 1841*1a13f2e6SEdward Tomasz Napierala bkt_ub = &numeric_limits[LIM_MAX][voi_dtype]; 1842*1a13f2e6SEdward Tomasz Napierala break; 1843*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 1844*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 1845*1a13f2e6SEdward Tomasz Napierala bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 1846*1a13f2e6SEdward Tomasz Napierala bkts[i].lb); 1847*1a13f2e6SEdward Tomasz Napierala bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 1848*1a13f2e6SEdward Tomasz Napierala bkts[i].ub); 1849*1a13f2e6SEdward Tomasz Napierala break; 1850*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 1851*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 1852*1a13f2e6SEdward Tomasz Napierala bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist, 1853*1a13f2e6SEdward Tomasz Napierala hist_dtype, bkts[i].val); 1854*1a13f2e6SEdward Tomasz Napierala break; 1855*1a13f2e6SEdward Tomasz Napierala default: 1856*1a13f2e6SEdward Tomasz Napierala break; 1857*1a13f2e6SEdward Tomasz Napierala } 1858*1a13f2e6SEdward Tomasz Napierala 1859*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1860*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1861*1a13f2e6SEdward Tomasz Napierala fmtstr = "\n\t\t\t\t"; 1862*1a13f2e6SEdward Tomasz Napierala break; 1863*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1864*1a13f2e6SEdward Tomasz Napierala default: 1865*1a13f2e6SEdward Tomasz Napierala fmtstr = "{"; 1866*1a13f2e6SEdward Tomasz Napierala break; 1867*1a13f2e6SEdward Tomasz Napierala } 1868*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 1869*1a13f2e6SEdward Tomasz Napierala 1870*1a13f2e6SEdward Tomasz Napierala if (objdump) { 1871*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1872*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1873*1a13f2e6SEdward Tomasz Napierala fmtstr = "bkt[%hu]."; 1874*1a13f2e6SEdward Tomasz Napierala break; 1875*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1876*1a13f2e6SEdward Tomasz Napierala default: 1877*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"bkt\":%hu,"; 1878*1a13f2e6SEdward Tomasz Napierala break; 1879*1a13f2e6SEdward Tomasz Napierala } 1880*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, i); 1881*1a13f2e6SEdward Tomasz Napierala } 1882*1a13f2e6SEdward Tomasz Napierala 1883*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1884*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1885*1a13f2e6SEdward Tomasz Napierala fmtstr = "{lb="; 1886*1a13f2e6SEdward Tomasz Napierala break; 1887*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1888*1a13f2e6SEdward Tomasz Napierala default: 1889*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"lb\":"; 1890*1a13f2e6SEdward Tomasz Napierala break; 1891*1a13f2e6SEdward Tomasz Napierala } 1892*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 1893*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr((const struct voistatdata *)bkt_lb, 1894*1a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 1895*1a13f2e6SEdward Tomasz Napierala fmt, buf, objdump); 1896*1a13f2e6SEdward Tomasz Napierala 1897*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1898*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1899*1a13f2e6SEdward Tomasz Napierala fmtstr = ",ub="; 1900*1a13f2e6SEdward Tomasz Napierala break; 1901*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1902*1a13f2e6SEdward Tomasz Napierala default: 1903*1a13f2e6SEdward Tomasz Napierala fmtstr = ",\"ub\":"; 1904*1a13f2e6SEdward Tomasz Napierala break; 1905*1a13f2e6SEdward Tomasz Napierala } 1906*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 1907*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr((const struct voistatdata *)bkt_ub, 1908*1a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 1909*1a13f2e6SEdward Tomasz Napierala fmt, buf, objdump); 1910*1a13f2e6SEdward Tomasz Napierala 1911*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1912*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1913*1a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 1914*1a13f2e6SEdward Tomasz Napierala break; 1915*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1916*1a13f2e6SEdward Tomasz Napierala default: 1917*1a13f2e6SEdward Tomasz Napierala fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 1918*1a13f2e6SEdward Tomasz Napierala break; 1919*1a13f2e6SEdward Tomasz Napierala } 1920*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, is32bit ? 1921*1a13f2e6SEdward Tomasz Napierala VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) : 1922*1a13f2e6SEdward Tomasz Napierala (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, 1923*1a13f2e6SEdward Tomasz Napierala bkts[i].cnt)); 1924*1a13f2e6SEdward Tomasz Napierala 1925*1a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON && i < nbkts - 1) 1926*1a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ','); 1927*1a13f2e6SEdward Tomasz Napierala } 1928*1a13f2e6SEdward Tomasz Napierala if (fmt == SB_STRFMT_JSON) 1929*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "]"); 1930*1a13f2e6SEdward Tomasz Napierala } 1931*1a13f2e6SEdward Tomasz Napierala 1932*1a13f2e6SEdward Tomasz Napierala int 1933*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype, 1934*1a13f2e6SEdward Tomasz Napierala enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt, 1935*1a13f2e6SEdward Tomasz Napierala struct sbuf *buf, int objdump) 1936*1a13f2e6SEdward Tomasz Napierala { 1937*1a13f2e6SEdward Tomasz Napierala const char *fmtstr; 1938*1a13f2e6SEdward Tomasz Napierala 1939*1a13f2e6SEdward Tomasz Napierala if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES || 1940*1a13f2e6SEdward Tomasz Napierala vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS) 1941*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 1942*1a13f2e6SEdward Tomasz Napierala 1943*1a13f2e6SEdward Tomasz Napierala switch (vsd_dtype) { 1944*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_VOISTATE: 1945*1a13f2e6SEdward Tomasz Napierala switch (fmt) { 1946*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 1947*1a13f2e6SEdward Tomasz Napierala fmtstr = "prev="; 1948*1a13f2e6SEdward Tomasz Napierala break; 1949*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 1950*1a13f2e6SEdward Tomasz Napierala default: 1951*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"prev\":"; 1952*1a13f2e6SEdward Tomasz Napierala break; 1953*1a13f2e6SEdward Tomasz Napierala } 1954*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, fmtstr); 1955*1a13f2e6SEdward Tomasz Napierala /* 1956*1a13f2e6SEdward Tomasz Napierala * Render prev by passing it as *vsd and voi_dtype as vsd_dtype. 1957*1a13f2e6SEdward Tomasz Napierala */ 1958*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr( 1959*1a13f2e6SEdward Tomasz Napierala (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev, 1960*1a13f2e6SEdward Tomasz Napierala voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump); 1961*1a13f2e6SEdward Tomasz Napierala break; 1962*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 1963*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%d", vsd->int32.s32); 1964*1a13f2e6SEdward Tomasz Napierala break; 1965*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 1966*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%u", vsd->int32.u32); 1967*1a13f2e6SEdward Tomasz Napierala break; 1968*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 1969*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64); 1970*1a13f2e6SEdward Tomasz Napierala break; 1971*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 1972*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64); 1973*1a13f2e6SEdward Tomasz Napierala break; 1974*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 1975*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%ld", vsd->intlong.slong); 1976*1a13f2e6SEdward Tomasz Napierala break; 1977*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 1978*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%lu", vsd->intlong.ulong); 1979*1a13f2e6SEdward Tomasz Napierala break; 1980*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 1981*1a13f2e6SEdward Tomasz Napierala { 1982*1a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)]; 1983*1a13f2e6SEdward Tomasz Napierala Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr)); 1984*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 1985*1a13f2e6SEdward Tomasz Napierala } 1986*1a13f2e6SEdward Tomasz Napierala break; 1987*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 1988*1a13f2e6SEdward Tomasz Napierala { 1989*1a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)]; 1990*1a13f2e6SEdward Tomasz Napierala Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr)); 1991*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 1992*1a13f2e6SEdward Tomasz Napierala } 1993*1a13f2e6SEdward Tomasz Napierala break; 1994*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 1995*1a13f2e6SEdward Tomasz Napierala { 1996*1a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)]; 1997*1a13f2e6SEdward Tomasz Napierala Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr)); 1998*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 1999*1a13f2e6SEdward Tomasz Napierala } 2000*1a13f2e6SEdward Tomasz Napierala break; 2001*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2002*1a13f2e6SEdward Tomasz Napierala { 2003*1a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)]; 2004*1a13f2e6SEdward Tomasz Napierala Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr)); 2005*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, qstr); 2006*1a13f2e6SEdward Tomasz Napierala } 2007*1a13f2e6SEdward Tomasz Napierala break; 2008*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 2009*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 2010*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 2011*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 2012*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 2013*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 2014*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd), 2015*1a13f2e6SEdward Tomasz Napierala vsd_dtype, vsd_sz, fmt, buf, objdump); 2016*1a13f2e6SEdward Tomasz Napierala break; 2017*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 2018*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 2019*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tdgst_tostr(voi_dtype, 2020*1a13f2e6SEdward Tomasz Napierala CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf, 2021*1a13f2e6SEdward Tomasz Napierala objdump); 2022*1a13f2e6SEdward Tomasz Napierala break; 2023*1a13f2e6SEdward Tomasz Napierala default: 2024*1a13f2e6SEdward Tomasz Napierala break; 2025*1a13f2e6SEdward Tomasz Napierala } 2026*1a13f2e6SEdward Tomasz Napierala 2027*1a13f2e6SEdward Tomasz Napierala return (sbuf_error(buf)); 2028*1a13f2e6SEdward Tomasz Napierala } 2029*1a13f2e6SEdward Tomasz Napierala 2030*1a13f2e6SEdward Tomasz Napierala static void 2031*1a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v, 2032*1a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx) 2033*1a13f2e6SEdward Tomasz Napierala { 2034*1a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx; 2035*1a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 2036*1a13f2e6SEdward Tomasz Napierala struct sbuf *buf; 2037*1a13f2e6SEdward Tomasz Napierala void *vsd; 2038*1a13f2e6SEdward Tomasz Napierala uint8_t dump; 2039*1a13f2e6SEdward Tomasz Napierala 2040*1a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx; 2041*1a13f2e6SEdward Tomasz Napierala buf = sctx->buf; 2042*1a13f2e6SEdward Tomasz Napierala tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 2043*1a13f2e6SEdward Tomasz Napierala dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 2044*1a13f2e6SEdward Tomasz Napierala 2045*1a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_CB) { 2046*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "struct statsblobv1@%p", sb); 2047*1a13f2e6SEdward Tomasz Napierala if (dump) { 2048*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, " 2049*1a13f2e6SEdward Tomasz Napierala "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, " 2050*1a13f2e6SEdward Tomasz Napierala "stats_off=%hu, statsdata_off=%hu", 2051*1a13f2e6SEdward Tomasz Napierala sb->abi, sb->endian, sb->maxsz, sb->cursz, 2052*1a13f2e6SEdward Tomasz Napierala sb->created, sb->lastrst, sb->flags, sb->stats_off, 2053*1a13f2e6SEdward Tomasz Napierala sb->statsdata_off); 2054*1a13f2e6SEdward Tomasz Napierala } 2055*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", tplhash=%u", sb->tplhash); 2056*1a13f2e6SEdward Tomasz Napierala } 2057*1a13f2e6SEdward Tomasz Napierala 2058*1a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_VOISTAT) { 2059*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id); 2060*1a13f2e6SEdward Tomasz Napierala if (v->id < 0) 2061*1a13f2e6SEdward Tomasz Napierala return; 2062*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" : 2063*1a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name); 2064*1a13f2e6SEdward Tomasz Napierala if (dump) 2065*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, " 2066*1a13f2e6SEdward Tomasz Napierala "voistatmaxid=%hhd, stats_off=%hu", v->flags, 2067*1a13f2e6SEdward Tomasz Napierala vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off); 2068*1a13f2e6SEdward Tomasz Napierala } 2069*1a13f2e6SEdward Tomasz Napierala 2070*1a13f2e6SEdward Tomasz Napierala if (!dump && vs->stype <= 0) 2071*1a13f2e6SEdward Tomasz Napierala return; 2072*1a13f2e6SEdward Tomasz Napierala 2073*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot); 2074*1a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) { 2075*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%hhd", vs->stype); 2076*1a13f2e6SEdward Tomasz Napierala return; 2077*1a13f2e6SEdward Tomasz Napierala } else 2078*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype], 2079*1a13f2e6SEdward Tomasz Napierala vs->errs); 2080*1a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 2081*1a13f2e6SEdward Tomasz Napierala if (dump) 2082*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, " 2083*1a13f2e6SEdward Tomasz Napierala "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype], 2084*1a13f2e6SEdward Tomasz Napierala vs->dsz, vs->data_off); 2085*1a13f2e6SEdward Tomasz Napierala 2086*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\n\t\t\tvoistatdata: "); 2087*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 2088*1a13f2e6SEdward Tomasz Napierala sctx->fmt, buf, dump); 2089*1a13f2e6SEdward Tomasz Napierala } 2090*1a13f2e6SEdward Tomasz Napierala 2091*1a13f2e6SEdward Tomasz Napierala static void 2092*1a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 2093*1a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx *ctx) 2094*1a13f2e6SEdward Tomasz Napierala { 2095*1a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx; 2096*1a13f2e6SEdward Tomasz Napierala struct metablob *tpl_mb; 2097*1a13f2e6SEdward Tomasz Napierala struct sbuf *buf; 2098*1a13f2e6SEdward Tomasz Napierala const char *fmtstr; 2099*1a13f2e6SEdward Tomasz Napierala void *vsd; 2100*1a13f2e6SEdward Tomasz Napierala uint8_t dump; 2101*1a13f2e6SEdward Tomasz Napierala 2102*1a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx; 2103*1a13f2e6SEdward Tomasz Napierala buf = sctx->buf; 2104*1a13f2e6SEdward Tomasz Napierala tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 2105*1a13f2e6SEdward Tomasz Napierala dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 2106*1a13f2e6SEdward Tomasz Napierala 2107*1a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_CB) { 2108*1a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, '{'); 2109*1a13f2e6SEdward Tomasz Napierala if (dump) { 2110*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu," 2111*1a13f2e6SEdward Tomasz Napierala "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd," 2112*1a13f2e6SEdward Tomasz Napierala "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu," 2113*1a13f2e6SEdward Tomasz Napierala "\"statsdata_off\":%hu,", sb->abi, 2114*1a13f2e6SEdward Tomasz Napierala sb->endian, sb->maxsz, sb->cursz, sb->created, 2115*1a13f2e6SEdward Tomasz Napierala sb->lastrst, sb->flags, sb->stats_off, 2116*1a13f2e6SEdward Tomasz Napierala sb->statsdata_off); 2117*1a13f2e6SEdward Tomasz Napierala } 2118*1a13f2e6SEdward Tomasz Napierala 2119*1a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) 2120*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{"; 2121*1a13f2e6SEdward Tomasz Napierala else 2122*1a13f2e6SEdward Tomasz Napierala fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{"; 2123*1a13f2e6SEdward Tomasz Napierala 2124*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null", 2125*1a13f2e6SEdward Tomasz Napierala sb->tplhash); 2126*1a13f2e6SEdward Tomasz Napierala } 2127*1a13f2e6SEdward Tomasz Napierala 2128*1a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_FIRST_VOISTAT) { 2129*1a13f2e6SEdward Tomasz Napierala if (dump) { 2130*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot, 2131*1a13f2e6SEdward Tomasz Napierala v->id); 2132*1a13f2e6SEdward Tomasz Napierala if (v->id < 0) { 2133*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "},"); 2134*1a13f2e6SEdward Tomasz Napierala return; 2135*1a13f2e6SEdward Tomasz Napierala } 2136*1a13f2e6SEdward Tomasz Napierala 2137*1a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) 2138*1a13f2e6SEdward Tomasz Napierala fmtstr = ",\"name\":%s,\"flags\":%hu," 2139*1a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 2140*1a13f2e6SEdward Tomasz Napierala "\"stats_off\":%hu,"; 2141*1a13f2e6SEdward Tomasz Napierala else 2142*1a13f2e6SEdward Tomasz Napierala fmtstr = ",\"name\":\"%s\",\"flags\":%hu," 2143*1a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 2144*1a13f2e6SEdward Tomasz Napierala "\"stats_off\":%hu,"; 2145*1a13f2e6SEdward Tomasz Napierala 2146*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, fmtstr, tpl_mb ? 2147*1a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name : "null", v->flags, 2148*1a13f2e6SEdward Tomasz Napierala vsd_dtype2name[v->dtype], v->voistatmaxid, 2149*1a13f2e6SEdward Tomasz Napierala v->stats_off); 2150*1a13f2e6SEdward Tomasz Napierala } else { 2151*1a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) { 2152*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hd]\":{", v->id); 2153*1a13f2e6SEdward Tomasz Napierala } else { 2154*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"%s\":{", 2155*1a13f2e6SEdward Tomasz Napierala tpl_mb->voi_meta[v->id].name); 2156*1a13f2e6SEdward Tomasz Napierala } 2157*1a13f2e6SEdward Tomasz Napierala } 2158*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "\"stats\":{"); 2159*1a13f2e6SEdward Tomasz Napierala } 2160*1a13f2e6SEdward Tomasz Napierala 2161*1a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 2162*1a13f2e6SEdward Tomasz Napierala if (dump) { 2163*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot); 2164*1a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) { 2165*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"stype\":-1},"); 2166*1a13f2e6SEdward Tomasz Napierala return; 2167*1a13f2e6SEdward Tomasz Napierala } 2168*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu," 2169*1a13f2e6SEdward Tomasz Napierala "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{", 2170*1a13f2e6SEdward Tomasz Napierala vs_stype2name[vs->stype], vs->errs, vs->flags, 2171*1a13f2e6SEdward Tomasz Napierala vsd_dtype2name[vs->dtype], vs->data_off); 2172*1a13f2e6SEdward Tomasz Napierala } else if (vs->stype > 0) { 2173*1a13f2e6SEdward Tomasz Napierala if (tpl_mb == NULL) 2174*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"[%hhd]\":", vs->stype); 2175*1a13f2e6SEdward Tomasz Napierala else 2176*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]); 2177*1a13f2e6SEdward Tomasz Napierala } else 2178*1a13f2e6SEdward Tomasz Napierala return; 2179*1a13f2e6SEdward Tomasz Napierala 2180*1a13f2e6SEdward Tomasz Napierala if ((vs->flags & VS_VSDVALID) || dump) { 2181*1a13f2e6SEdward Tomasz Napierala if (!dump) 2182*1a13f2e6SEdward Tomasz Napierala sbuf_printf(buf, "{\"errs\":%hu,", vs->errs); 2183*1a13f2e6SEdward Tomasz Napierala /* Simple non-compound VSD types need a key. */ 2184*1a13f2e6SEdward Tomasz Napierala if (!vsd_compoundtype[vs->dtype]) 2185*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "\"val\":"); 2186*1a13f2e6SEdward Tomasz Napierala stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 2187*1a13f2e6SEdward Tomasz Napierala sctx->fmt, buf, dump); 2188*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, dump ? "}}" : "}"); 2189*1a13f2e6SEdward Tomasz Napierala } else 2190*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, dump ? "null}" : "null"); 2191*1a13f2e6SEdward Tomasz Napierala 2192*1a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_LAST_VOISTAT) 2193*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "}}"); 2194*1a13f2e6SEdward Tomasz Napierala 2195*1a13f2e6SEdward Tomasz Napierala if (ctx->flags & SB_IT_LAST_CB) 2196*1a13f2e6SEdward Tomasz Napierala sbuf_cat(buf, "}}"); 2197*1a13f2e6SEdward Tomasz Napierala else 2198*1a13f2e6SEdward Tomasz Napierala sbuf_putc(buf, ','); 2199*1a13f2e6SEdward Tomasz Napierala } 2200*1a13f2e6SEdward Tomasz Napierala 2201*1a13f2e6SEdward Tomasz Napierala static int 2202*1a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 2203*1a13f2e6SEdward Tomasz Napierala struct sb_iter_ctx *ctx) 2204*1a13f2e6SEdward Tomasz Napierala { 2205*1a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx *sctx; 2206*1a13f2e6SEdward Tomasz Napierala 2207*1a13f2e6SEdward Tomasz Napierala sctx = ctx->usrctx; 2208*1a13f2e6SEdward Tomasz Napierala 2209*1a13f2e6SEdward Tomasz Napierala switch (sctx->fmt) { 2210*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_FREEFORM: 2211*1a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_freeform(sb, v, vs, ctx); 2212*1a13f2e6SEdward Tomasz Napierala break; 2213*1a13f2e6SEdward Tomasz Napierala case SB_STRFMT_JSON: 2214*1a13f2e6SEdward Tomasz Napierala stats_v1_itercb_tostr_json(sb, v, vs, ctx); 2215*1a13f2e6SEdward Tomasz Napierala break; 2216*1a13f2e6SEdward Tomasz Napierala default: 2217*1a13f2e6SEdward Tomasz Napierala break; 2218*1a13f2e6SEdward Tomasz Napierala } 2219*1a13f2e6SEdward Tomasz Napierala 2220*1a13f2e6SEdward Tomasz Napierala return (sbuf_error(sctx->buf)); 2221*1a13f2e6SEdward Tomasz Napierala } 2222*1a13f2e6SEdward Tomasz Napierala 2223*1a13f2e6SEdward Tomasz Napierala int 2224*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf, 2225*1a13f2e6SEdward Tomasz Napierala enum sb_str_fmt fmt, uint32_t flags) 2226*1a13f2e6SEdward Tomasz Napierala { 2227*1a13f2e6SEdward Tomasz Napierala struct sb_tostrcb_ctx sctx; 2228*1a13f2e6SEdward Tomasz Napierala uint32_t iflags; 2229*1a13f2e6SEdward Tomasz Napierala 2230*1a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL || 2231*1a13f2e6SEdward Tomasz Napierala fmt >= SB_STRFMT_NUM_FMTS) 2232*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 2233*1a13f2e6SEdward Tomasz Napierala 2234*1a13f2e6SEdward Tomasz Napierala sctx.buf = buf; 2235*1a13f2e6SEdward Tomasz Napierala sctx.fmt = fmt; 2236*1a13f2e6SEdward Tomasz Napierala sctx.flags = flags; 2237*1a13f2e6SEdward Tomasz Napierala 2238*1a13f2e6SEdward Tomasz Napierala if (flags & SB_TOSTR_META) { 2239*1a13f2e6SEdward Tomasz Napierala if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash), 2240*1a13f2e6SEdward Tomasz Napierala &sctx.tpl)) 2241*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 2242*1a13f2e6SEdward Tomasz Napierala } else 2243*1a13f2e6SEdward Tomasz Napierala sctx.tpl = NULL; 2244*1a13f2e6SEdward Tomasz Napierala 2245*1a13f2e6SEdward Tomasz Napierala iflags = 0; 2246*1a13f2e6SEdward Tomasz Napierala if (flags & SB_TOSTR_OBJDUMP) 2247*1a13f2e6SEdward Tomasz Napierala iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT); 2248*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags); 2249*1a13f2e6SEdward Tomasz Napierala 2250*1a13f2e6SEdward Tomasz Napierala return (sbuf_error(buf)); 2251*1a13f2e6SEdward Tomasz Napierala } 2252*1a13f2e6SEdward Tomasz Napierala 2253*1a13f2e6SEdward Tomasz Napierala static int 2254*1a13f2e6SEdward Tomasz Napierala stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v, 2255*1a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx) 2256*1a13f2e6SEdward Tomasz Napierala { 2257*1a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx *vctx; 2258*1a13f2e6SEdward Tomasz Napierala struct sb_visit sbv; 2259*1a13f2e6SEdward Tomasz Napierala 2260*1a13f2e6SEdward Tomasz Napierala vctx = ctx->usrctx; 2261*1a13f2e6SEdward Tomasz Napierala 2262*1a13f2e6SEdward Tomasz Napierala sbv.tplhash = sb->tplhash; 2263*1a13f2e6SEdward Tomasz Napierala sbv.voi_id = v->id; 2264*1a13f2e6SEdward Tomasz Napierala sbv.voi_dtype = v->dtype; 2265*1a13f2e6SEdward Tomasz Napierala sbv.vs_stype = vs->stype; 2266*1a13f2e6SEdward Tomasz Napierala sbv.vs_dtype = vs->dtype; 2267*1a13f2e6SEdward Tomasz Napierala sbv.vs_dsz = vs->dsz; 2268*1a13f2e6SEdward Tomasz Napierala sbv.vs_data = BLOB_OFFSET(sb, vs->data_off); 2269*1a13f2e6SEdward Tomasz Napierala sbv.vs_errs = vs->errs; 2270*1a13f2e6SEdward Tomasz Napierala sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB | 2271*1a13f2e6SEdward Tomasz Napierala SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT | 2272*1a13f2e6SEdward Tomasz Napierala SB_IT_LAST_VOISTAT); 2273*1a13f2e6SEdward Tomasz Napierala 2274*1a13f2e6SEdward Tomasz Napierala return (vctx->cb(&sbv, vctx->usrctx)); 2275*1a13f2e6SEdward Tomasz Napierala } 2276*1a13f2e6SEdward Tomasz Napierala 2277*1a13f2e6SEdward Tomasz Napierala int 2278*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func, 2279*1a13f2e6SEdward Tomasz Napierala void *usrctx) 2280*1a13f2e6SEdward Tomasz Napierala { 2281*1a13f2e6SEdward Tomasz Napierala struct sb_visitcb_ctx vctx; 2282*1a13f2e6SEdward Tomasz Napierala 2283*1a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL) 2284*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 2285*1a13f2e6SEdward Tomasz Napierala 2286*1a13f2e6SEdward Tomasz Napierala vctx.cb = func; 2287*1a13f2e6SEdward Tomasz Napierala vctx.usrctx = usrctx; 2288*1a13f2e6SEdward Tomasz Napierala 2289*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0); 2290*1a13f2e6SEdward Tomasz Napierala 2291*1a13f2e6SEdward Tomasz Napierala return (0); 2292*1a13f2e6SEdward Tomasz Napierala } 2293*1a13f2e6SEdward Tomasz Napierala 2294*1a13f2e6SEdward Tomasz Napierala static int 2295*1a13f2e6SEdward Tomasz Napierala stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused, 2296*1a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct sb_iter_ctx *ctx __unused) 2297*1a13f2e6SEdward Tomasz Napierala { 2298*1a13f2e6SEdward Tomasz Napierala void *vsd; 2299*1a13f2e6SEdward Tomasz Napierala 2300*1a13f2e6SEdward Tomasz Napierala if (vs->stype == VS_STYPE_VOISTATE) 2301*1a13f2e6SEdward Tomasz Napierala return (0); 2302*1a13f2e6SEdward Tomasz Napierala 2303*1a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 2304*1a13f2e6SEdward Tomasz Napierala 2305*1a13f2e6SEdward Tomasz Napierala /* Perform the stat type's default reset action. */ 2306*1a13f2e6SEdward Tomasz Napierala switch (vs->stype) { 2307*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM: 2308*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2309*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 2310*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32, 0); 2311*1a13f2e6SEdward Tomasz Napierala break; 2312*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 2313*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32, 0); 2314*1a13f2e6SEdward Tomasz Napierala break; 2315*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 2316*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64, 0); 2317*1a13f2e6SEdward Tomasz Napierala break; 2318*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2319*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64, 0); 2320*1a13f2e6SEdward Tomasz Napierala break; 2321*1a13f2e6SEdward Tomasz Napierala default: 2322*1a13f2e6SEdward Tomasz Napierala bzero(vsd, vs->dsz); 2323*1a13f2e6SEdward Tomasz Napierala break; 2324*1a13f2e6SEdward Tomasz Napierala } 2325*1a13f2e6SEdward Tomasz Napierala break; 2326*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX: 2327*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2328*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 2329*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32, 2330*1a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q32, vsd)->sq32)); 2331*1a13f2e6SEdward Tomasz Napierala break; 2332*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 2333*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32, 2334*1a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q32, vsd)->uq32)); 2335*1a13f2e6SEdward Tomasz Napierala break; 2336*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 2337*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64, 2338*1a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q64, vsd)->sq64)); 2339*1a13f2e6SEdward Tomasz Napierala break; 2340*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2341*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64, 2342*1a13f2e6SEdward Tomasz Napierala Q_IFMINVAL(VSD(q64, vsd)->uq64)); 2343*1a13f2e6SEdward Tomasz Napierala break; 2344*1a13f2e6SEdward Tomasz Napierala default: 2345*1a13f2e6SEdward Tomasz Napierala memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype], 2346*1a13f2e6SEdward Tomasz Napierala vs->dsz); 2347*1a13f2e6SEdward Tomasz Napierala break; 2348*1a13f2e6SEdward Tomasz Napierala } 2349*1a13f2e6SEdward Tomasz Napierala break; 2350*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN: 2351*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2352*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 2353*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->sq32, 2354*1a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q32, vsd)->sq32)); 2355*1a13f2e6SEdward Tomasz Napierala break; 2356*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 2357*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q32, vsd)->uq32, 2358*1a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q32, vsd)->uq32)); 2359*1a13f2e6SEdward Tomasz Napierala break; 2360*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 2361*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->sq64, 2362*1a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q64, vsd)->sq64)); 2363*1a13f2e6SEdward Tomasz Napierala break; 2364*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2365*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(VSD(q64, vsd)->uq64, 2366*1a13f2e6SEdward Tomasz Napierala Q_IFMAXVAL(VSD(q64, vsd)->uq64)); 2367*1a13f2e6SEdward Tomasz Napierala break; 2368*1a13f2e6SEdward Tomasz Napierala default: 2369*1a13f2e6SEdward Tomasz Napierala memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype], 2370*1a13f2e6SEdward Tomasz Napierala vs->dsz); 2371*1a13f2e6SEdward Tomasz Napierala break; 2372*1a13f2e6SEdward Tomasz Napierala } 2373*1a13f2e6SEdward Tomasz Napierala break; 2374*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_HIST: 2375*1a13f2e6SEdward Tomasz Napierala { 2376*1a13f2e6SEdward Tomasz Napierala /* Reset bucket counts. */ 2377*1a13f2e6SEdward Tomasz Napierala struct voistatdata_hist *hist; 2378*1a13f2e6SEdward Tomasz Napierala int i, is32bit; 2379*1a13f2e6SEdward Tomasz Napierala uint16_t nbkts; 2380*1a13f2e6SEdward Tomasz Napierala 2381*1a13f2e6SEdward Tomasz Napierala hist = VSD(hist, vsd); 2382*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2383*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 2384*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 2385*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 2386*1a13f2e6SEdward Tomasz Napierala break; 2387*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 2388*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 2389*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 2390*1a13f2e6SEdward Tomasz Napierala break; 2391*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 2392*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 2393*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 2394*1a13f2e6SEdward Tomasz Napierala break; 2395*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 2396*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 2397*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 2398*1a13f2e6SEdward Tomasz Napierala break; 2399*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 2400*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 2401*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 2402*1a13f2e6SEdward Tomasz Napierala break; 2403*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 2404*1a13f2e6SEdward Tomasz Napierala nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 2405*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 2406*1a13f2e6SEdward Tomasz Napierala break; 2407*1a13f2e6SEdward Tomasz Napierala default: 2408*1a13f2e6SEdward Tomasz Napierala return (0); 2409*1a13f2e6SEdward Tomasz Napierala } 2410*1a13f2e6SEdward Tomasz Napierala 2411*1a13f2e6SEdward Tomasz Napierala bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob), 2412*1a13f2e6SEdward Tomasz Napierala is32bit ? sizeof(uint32_t) : sizeof(uint64_t)); 2413*1a13f2e6SEdward Tomasz Napierala for (i = nbkts - 1; i >= 0; i--) { 2414*1a13f2e6SEdward Tomasz Napierala bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, 2415*1a13f2e6SEdward Tomasz Napierala bkts[i].cnt), is32bit ? sizeof(uint32_t) : 2416*1a13f2e6SEdward Tomasz Napierala sizeof(uint64_t)); 2417*1a13f2e6SEdward Tomasz Napierala } 2418*1a13f2e6SEdward Tomasz Napierala break; 2419*1a13f2e6SEdward Tomasz Napierala } 2420*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_TDGST: 2421*1a13f2e6SEdward Tomasz Napierala { 2422*1a13f2e6SEdward Tomasz Napierala /* Reset sample count centroids array/tree. */ 2423*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst; 2424*1a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 2425*1a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 2426*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 2427*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 2428*1a13f2e6SEdward Tomasz Napierala 2429*1a13f2e6SEdward Tomasz Napierala tdgst = VSD(tdgst, vsd); 2430*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2431*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 2432*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt = 0; 2433*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->compcnt = 0; 2434*1a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 2435*1a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd32, ctdlnk, ctd32tree, 2436*1a13f2e6SEdward Tomasz Napierala ARB_MAXNODES(ctd32tree)) { 2437*1a13f2e6SEdward Tomasz Napierala ctd32->cnt = 0; 2438*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(ctd32->mu, 0); 2439*1a13f2e6SEdward Tomasz Napierala } 2440*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 2441*1a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 2442*1a13f2e6SEdward Tomasz Napierala #endif 2443*1a13f2e6SEdward Tomasz Napierala break; 2444*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 2445*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt = 0; 2446*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->compcnt = 0; 2447*1a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 2448*1a13f2e6SEdward Tomasz Napierala ARB_INIT(ctd64, ctdlnk, ctd64tree, 2449*1a13f2e6SEdward Tomasz Napierala ARB_MAXNODES(ctd64tree)) { 2450*1a13f2e6SEdward Tomasz Napierala ctd64->cnt = 0; 2451*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(ctd64->mu, 0); 2452*1a13f2e6SEdward Tomasz Napierala } 2453*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 2454*1a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 2455*1a13f2e6SEdward Tomasz Napierala #endif 2456*1a13f2e6SEdward Tomasz Napierala break; 2457*1a13f2e6SEdward Tomasz Napierala default: 2458*1a13f2e6SEdward Tomasz Napierala return (0); 2459*1a13f2e6SEdward Tomasz Napierala } 2460*1a13f2e6SEdward Tomasz Napierala break; 2461*1a13f2e6SEdward Tomasz Napierala } 2462*1a13f2e6SEdward Tomasz Napierala default: 2463*1a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 2464*1a13f2e6SEdward Tomasz Napierala break; 2465*1a13f2e6SEdward Tomasz Napierala } 2466*1a13f2e6SEdward Tomasz Napierala 2467*1a13f2e6SEdward Tomasz Napierala vs->errs = 0; 2468*1a13f2e6SEdward Tomasz Napierala vs->flags &= ~VS_VSDVALID; 2469*1a13f2e6SEdward Tomasz Napierala 2470*1a13f2e6SEdward Tomasz Napierala return (0); 2471*1a13f2e6SEdward Tomasz Napierala } 2472*1a13f2e6SEdward Tomasz Napierala 2473*1a13f2e6SEdward Tomasz Napierala int 2474*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz, 2475*1a13f2e6SEdward Tomasz Napierala struct statsblobv1 *src, uint32_t flags) 2476*1a13f2e6SEdward Tomasz Napierala { 2477*1a13f2e6SEdward Tomasz Napierala int error; 2478*1a13f2e6SEdward Tomasz Napierala 2479*1a13f2e6SEdward Tomasz Napierala if (src != NULL && src->abi == STATS_ABI_V1) { 2480*1a13f2e6SEdward Tomasz Napierala error = stats_v1_blob_clone(dst, dstmaxsz, src, flags); 2481*1a13f2e6SEdward Tomasz Napierala if (!error) { 2482*1a13f2e6SEdward Tomasz Napierala if (flags & SB_CLONE_RSTSRC) { 2483*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_iter(src, 2484*1a13f2e6SEdward Tomasz Napierala stats_v1_icb_reset_voistat, NULL, 0); 2485*1a13f2e6SEdward Tomasz Napierala src->lastrst = stats_sbinuptime(); 2486*1a13f2e6SEdward Tomasz Napierala } 2487*1a13f2e6SEdward Tomasz Napierala stats_v1_blob_finalise(*dst); 2488*1a13f2e6SEdward Tomasz Napierala } 2489*1a13f2e6SEdward Tomasz Napierala } else 2490*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 2491*1a13f2e6SEdward Tomasz Napierala 2492*1a13f2e6SEdward Tomasz Napierala return (error); 2493*1a13f2e6SEdward Tomasz Napierala } 2494*1a13f2e6SEdward Tomasz Napierala 2495*1a13f2e6SEdward Tomasz Napierala static inline int 2496*1a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused, 2497*1a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd) 2498*1a13f2e6SEdward Tomasz Napierala { 2499*1a13f2e6SEdward Tomasz Napierala int error; 2500*1a13f2e6SEdward Tomasz Napierala 2501*1a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES, 2502*1a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype)); 2503*1a13f2e6SEdward Tomasz Napierala 2504*1a13f2e6SEdward Tomasz Napierala error = 0; 2505*1a13f2e6SEdward Tomasz Napierala 2506*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2507*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 2508*1a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->s32 < voival->int32.s32) { 2509*1a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 = voival->int32.s32; 2510*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2511*1a13f2e6SEdward Tomasz Napierala } 2512*1a13f2e6SEdward Tomasz Napierala break; 2513*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 2514*1a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->u32 < voival->int32.u32) { 2515*1a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 = voival->int32.u32; 2516*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2517*1a13f2e6SEdward Tomasz Napierala } 2518*1a13f2e6SEdward Tomasz Napierala break; 2519*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 2520*1a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->s64 < voival->int64.s64) { 2521*1a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 = voival->int64.s64; 2522*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2523*1a13f2e6SEdward Tomasz Napierala } 2524*1a13f2e6SEdward Tomasz Napierala break; 2525*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 2526*1a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->u64 < voival->int64.u64) { 2527*1a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 = voival->int64.u64; 2528*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2529*1a13f2e6SEdward Tomasz Napierala } 2530*1a13f2e6SEdward Tomasz Napierala break; 2531*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 2532*1a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->slong < voival->intlong.slong) { 2533*1a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong = voival->intlong.slong; 2534*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2535*1a13f2e6SEdward Tomasz Napierala } 2536*1a13f2e6SEdward Tomasz Napierala break; 2537*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 2538*1a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) { 2539*1a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong = voival->intlong.ulong; 2540*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2541*1a13f2e6SEdward Tomasz Napierala } 2542*1a13f2e6SEdward Tomasz Napierala break; 2543*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 2544*1a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 2545*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 2546*1a13f2e6SEdward Tomasz Napierala voival->q32.sq32)))) { 2547*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2548*1a13f2e6SEdward Tomasz Napierala } 2549*1a13f2e6SEdward Tomasz Napierala break; 2550*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 2551*1a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 2552*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 2553*1a13f2e6SEdward Tomasz Napierala voival->q32.uq32)))) { 2554*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2555*1a13f2e6SEdward Tomasz Napierala } 2556*1a13f2e6SEdward Tomasz Napierala break; 2557*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 2558*1a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 2559*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 2560*1a13f2e6SEdward Tomasz Napierala voival->q64.sq64)))) { 2561*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2562*1a13f2e6SEdward Tomasz Napierala } 2563*1a13f2e6SEdward Tomasz Napierala break; 2564*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2565*1a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 2566*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 2567*1a13f2e6SEdward Tomasz Napierala voival->q64.uq64)))) { 2568*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2569*1a13f2e6SEdward Tomasz Napierala } 2570*1a13f2e6SEdward Tomasz Napierala break; 2571*1a13f2e6SEdward Tomasz Napierala default: 2572*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 2573*1a13f2e6SEdward Tomasz Napierala break; 2574*1a13f2e6SEdward Tomasz Napierala } 2575*1a13f2e6SEdward Tomasz Napierala 2576*1a13f2e6SEdward Tomasz Napierala return (error); 2577*1a13f2e6SEdward Tomasz Napierala } 2578*1a13f2e6SEdward Tomasz Napierala 2579*1a13f2e6SEdward Tomasz Napierala static inline int 2580*1a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused, 2581*1a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd) 2582*1a13f2e6SEdward Tomasz Napierala { 2583*1a13f2e6SEdward Tomasz Napierala int error; 2584*1a13f2e6SEdward Tomasz Napierala 2585*1a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES, 2586*1a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype)); 2587*1a13f2e6SEdward Tomasz Napierala 2588*1a13f2e6SEdward Tomasz Napierala error = 0; 2589*1a13f2e6SEdward Tomasz Napierala 2590*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2591*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 2592*1a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->s32 > voival->int32.s32) { 2593*1a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 = voival->int32.s32; 2594*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2595*1a13f2e6SEdward Tomasz Napierala } 2596*1a13f2e6SEdward Tomasz Napierala break; 2597*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 2598*1a13f2e6SEdward Tomasz Napierala if (VSD(int32, vsd)->u32 > voival->int32.u32) { 2599*1a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 = voival->int32.u32; 2600*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2601*1a13f2e6SEdward Tomasz Napierala } 2602*1a13f2e6SEdward Tomasz Napierala break; 2603*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 2604*1a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->s64 > voival->int64.s64) { 2605*1a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 = voival->int64.s64; 2606*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2607*1a13f2e6SEdward Tomasz Napierala } 2608*1a13f2e6SEdward Tomasz Napierala break; 2609*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 2610*1a13f2e6SEdward Tomasz Napierala if (VSD(int64, vsd)->u64 > voival->int64.u64) { 2611*1a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 = voival->int64.u64; 2612*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2613*1a13f2e6SEdward Tomasz Napierala } 2614*1a13f2e6SEdward Tomasz Napierala break; 2615*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 2616*1a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->slong > voival->intlong.slong) { 2617*1a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong = voival->intlong.slong; 2618*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2619*1a13f2e6SEdward Tomasz Napierala } 2620*1a13f2e6SEdward Tomasz Napierala break; 2621*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 2622*1a13f2e6SEdward Tomasz Napierala if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) { 2623*1a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong = voival->intlong.ulong; 2624*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2625*1a13f2e6SEdward Tomasz Napierala } 2626*1a13f2e6SEdward Tomasz Napierala break; 2627*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 2628*1a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 2629*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 2630*1a13f2e6SEdward Tomasz Napierala voival->q32.sq32)))) { 2631*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2632*1a13f2e6SEdward Tomasz Napierala } 2633*1a13f2e6SEdward Tomasz Napierala break; 2634*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 2635*1a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 2636*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 2637*1a13f2e6SEdward Tomasz Napierala voival->q32.uq32)))) { 2638*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2639*1a13f2e6SEdward Tomasz Napierala } 2640*1a13f2e6SEdward Tomasz Napierala break; 2641*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 2642*1a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 2643*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 2644*1a13f2e6SEdward Tomasz Napierala voival->q64.sq64)))) { 2645*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2646*1a13f2e6SEdward Tomasz Napierala } 2647*1a13f2e6SEdward Tomasz Napierala break; 2648*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2649*1a13f2e6SEdward Tomasz Napierala if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 2650*1a13f2e6SEdward Tomasz Napierala (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 2651*1a13f2e6SEdward Tomasz Napierala voival->q64.uq64)))) { 2652*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2653*1a13f2e6SEdward Tomasz Napierala } 2654*1a13f2e6SEdward Tomasz Napierala break; 2655*1a13f2e6SEdward Tomasz Napierala default: 2656*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 2657*1a13f2e6SEdward Tomasz Napierala break; 2658*1a13f2e6SEdward Tomasz Napierala } 2659*1a13f2e6SEdward Tomasz Napierala 2660*1a13f2e6SEdward Tomasz Napierala return (error); 2661*1a13f2e6SEdward Tomasz Napierala } 2662*1a13f2e6SEdward Tomasz Napierala 2663*1a13f2e6SEdward Tomasz Napierala static inline int 2664*1a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused, 2665*1a13f2e6SEdward Tomasz Napierala struct voistatdata *voival, struct voistat *vs, void *vsd) 2666*1a13f2e6SEdward Tomasz Napierala { 2667*1a13f2e6SEdward Tomasz Napierala int error; 2668*1a13f2e6SEdward Tomasz Napierala 2669*1a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype < VSD_NUM_DTYPES, 2670*1a13f2e6SEdward Tomasz Napierala ("Unknown VSD dtype %d", vs->dtype)); 2671*1a13f2e6SEdward Tomasz Napierala 2672*1a13f2e6SEdward Tomasz Napierala error = 0; 2673*1a13f2e6SEdward Tomasz Napierala 2674*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2675*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 2676*1a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->s32 += voival->int32.s32; 2677*1a13f2e6SEdward Tomasz Napierala break; 2678*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 2679*1a13f2e6SEdward Tomasz Napierala VSD(int32, vsd)->u32 += voival->int32.u32; 2680*1a13f2e6SEdward Tomasz Napierala break; 2681*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 2682*1a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->s64 += voival->int64.s64; 2683*1a13f2e6SEdward Tomasz Napierala break; 2684*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 2685*1a13f2e6SEdward Tomasz Napierala VSD(int64, vsd)->u64 += voival->int64.u64; 2686*1a13f2e6SEdward Tomasz Napierala break; 2687*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 2688*1a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->slong += voival->intlong.slong; 2689*1a13f2e6SEdward Tomasz Napierala break; 2690*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 2691*1a13f2e6SEdward Tomasz Napierala VSD(intlong, vsd)->ulong += voival->intlong.ulong; 2692*1a13f2e6SEdward Tomasz Napierala break; 2693*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 2694*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32); 2695*1a13f2e6SEdward Tomasz Napierala break; 2696*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 2697*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32); 2698*1a13f2e6SEdward Tomasz Napierala break; 2699*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 2700*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64); 2701*1a13f2e6SEdward Tomasz Napierala break; 2702*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2703*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64); 2704*1a13f2e6SEdward Tomasz Napierala break; 2705*1a13f2e6SEdward Tomasz Napierala default: 2706*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 2707*1a13f2e6SEdward Tomasz Napierala break; 2708*1a13f2e6SEdward Tomasz Napierala } 2709*1a13f2e6SEdward Tomasz Napierala 2710*1a13f2e6SEdward Tomasz Napierala if (!error) 2711*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2712*1a13f2e6SEdward Tomasz Napierala 2713*1a13f2e6SEdward Tomasz Napierala return (error); 2714*1a13f2e6SEdward Tomasz Napierala } 2715*1a13f2e6SEdward Tomasz Napierala 2716*1a13f2e6SEdward Tomasz Napierala static inline int 2717*1a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival, 2718*1a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct voistatdata_hist *hist) 2719*1a13f2e6SEdward Tomasz Napierala { 2720*1a13f2e6SEdward Tomasz Napierala struct voistatdata_numeric *bkt_lb, *bkt_ub; 2721*1a13f2e6SEdward Tomasz Napierala uint64_t *oob64, *cnt64; 2722*1a13f2e6SEdward Tomasz Napierala uint32_t *oob32, *cnt32; 2723*1a13f2e6SEdward Tomasz Napierala int error, i, found, is32bit, has_ub, eq_only; 2724*1a13f2e6SEdward Tomasz Napierala 2725*1a13f2e6SEdward Tomasz Napierala error = 0; 2726*1a13f2e6SEdward Tomasz Napierala 2727*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2728*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 2729*1a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 2730*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 2731*1a13f2e6SEdward Tomasz Napierala has_ub = eq_only = 0; 2732*1a13f2e6SEdward Tomasz Napierala oob32 = &VSD(crhist32, hist)->oob; 2733*1a13f2e6SEdward Tomasz Napierala break; 2734*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 2735*1a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 2736*1a13f2e6SEdward Tomasz Napierala is32bit = has_ub = 1; 2737*1a13f2e6SEdward Tomasz Napierala eq_only = 0; 2738*1a13f2e6SEdward Tomasz Napierala oob32 = &VSD(drhist32, hist)->oob; 2739*1a13f2e6SEdward Tomasz Napierala break; 2740*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 2741*1a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 2742*1a13f2e6SEdward Tomasz Napierala is32bit = eq_only = 1; 2743*1a13f2e6SEdward Tomasz Napierala has_ub = 0; 2744*1a13f2e6SEdward Tomasz Napierala oob32 = &VSD(dvhist32, hist)->oob; 2745*1a13f2e6SEdward Tomasz Napierala break; 2746*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 2747*1a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 2748*1a13f2e6SEdward Tomasz Napierala is32bit = has_ub = eq_only = 0; 2749*1a13f2e6SEdward Tomasz Napierala oob64 = &VSD(crhist64, hist)->oob; 2750*1a13f2e6SEdward Tomasz Napierala break; 2751*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 2752*1a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 2753*1a13f2e6SEdward Tomasz Napierala is32bit = eq_only = 0; 2754*1a13f2e6SEdward Tomasz Napierala has_ub = 1; 2755*1a13f2e6SEdward Tomasz Napierala oob64 = &VSD(drhist64, hist)->oob; 2756*1a13f2e6SEdward Tomasz Napierala break; 2757*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 2758*1a13f2e6SEdward Tomasz Napierala i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 2759*1a13f2e6SEdward Tomasz Napierala is32bit = has_ub = 0; 2760*1a13f2e6SEdward Tomasz Napierala eq_only = 1; 2761*1a13f2e6SEdward Tomasz Napierala oob64 = &VSD(dvhist64, hist)->oob; 2762*1a13f2e6SEdward Tomasz Napierala break; 2763*1a13f2e6SEdward Tomasz Napierala default: 2764*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 2765*1a13f2e6SEdward Tomasz Napierala } 2766*1a13f2e6SEdward Tomasz Napierala i--; /* Adjust for 0-based array index. */ 2767*1a13f2e6SEdward Tomasz Napierala 2768*1a13f2e6SEdward Tomasz Napierala /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */ 2769*1a13f2e6SEdward Tomasz Napierala for (found = 0; i >= 0 && !found; i--) { 2770*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 2771*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST32: 2772*1a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(crhist32, hist)->bkts[i].lb; 2773*1a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(crhist32, hist)->bkts[i].cnt; 2774*1a13f2e6SEdward Tomasz Napierala break; 2775*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST32: 2776*1a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(drhist32, hist)->bkts[i].lb; 2777*1a13f2e6SEdward Tomasz Napierala bkt_ub = &VSD(drhist32, hist)->bkts[i].ub; 2778*1a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(drhist32, hist)->bkts[i].cnt; 2779*1a13f2e6SEdward Tomasz Napierala break; 2780*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST32: 2781*1a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(dvhist32, hist)->bkts[i].val; 2782*1a13f2e6SEdward Tomasz Napierala cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt; 2783*1a13f2e6SEdward Tomasz Napierala break; 2784*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_CRHIST64: 2785*1a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(crhist64, hist)->bkts[i].lb; 2786*1a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(crhist64, hist)->bkts[i].cnt; 2787*1a13f2e6SEdward Tomasz Napierala break; 2788*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DRHIST64: 2789*1a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(drhist64, hist)->bkts[i].lb; 2790*1a13f2e6SEdward Tomasz Napierala bkt_ub = &VSD(drhist64, hist)->bkts[i].ub; 2791*1a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(drhist64, hist)->bkts[i].cnt; 2792*1a13f2e6SEdward Tomasz Napierala break; 2793*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_DVHIST64: 2794*1a13f2e6SEdward Tomasz Napierala bkt_lb = &VSD(dvhist64, hist)->bkts[i].val; 2795*1a13f2e6SEdward Tomasz Napierala cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt; 2796*1a13f2e6SEdward Tomasz Napierala break; 2797*1a13f2e6SEdward Tomasz Napierala default: 2798*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 2799*1a13f2e6SEdward Tomasz Napierala } 2800*1a13f2e6SEdward Tomasz Napierala 2801*1a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 2802*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 2803*1a13f2e6SEdward Tomasz Napierala if (voival->int32.s32 >= bkt_lb->int32.s32) { 2804*1a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int32.s32 == 2805*1a13f2e6SEdward Tomasz Napierala bkt_lb->int32.s32) || 2806*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2807*1a13f2e6SEdward Tomasz Napierala voival->int32.s32 < bkt_ub->int32.s32))) 2808*1a13f2e6SEdward Tomasz Napierala found = 1; 2809*1a13f2e6SEdward Tomasz Napierala } 2810*1a13f2e6SEdward Tomasz Napierala break; 2811*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 2812*1a13f2e6SEdward Tomasz Napierala if (voival->int32.u32 >= bkt_lb->int32.u32) { 2813*1a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int32.u32 == 2814*1a13f2e6SEdward Tomasz Napierala bkt_lb->int32.u32) || 2815*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2816*1a13f2e6SEdward Tomasz Napierala voival->int32.u32 < bkt_ub->int32.u32))) 2817*1a13f2e6SEdward Tomasz Napierala found = 1; 2818*1a13f2e6SEdward Tomasz Napierala } 2819*1a13f2e6SEdward Tomasz Napierala break; 2820*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 2821*1a13f2e6SEdward Tomasz Napierala if (voival->int64.s64 >= bkt_lb->int64.s64) 2822*1a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int64.s64 == 2823*1a13f2e6SEdward Tomasz Napierala bkt_lb->int64.s64) || 2824*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2825*1a13f2e6SEdward Tomasz Napierala voival->int64.s64 < bkt_ub->int64.s64))) 2826*1a13f2e6SEdward Tomasz Napierala found = 1; 2827*1a13f2e6SEdward Tomasz Napierala break; 2828*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 2829*1a13f2e6SEdward Tomasz Napierala if (voival->int64.u64 >= bkt_lb->int64.u64) 2830*1a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->int64.u64 == 2831*1a13f2e6SEdward Tomasz Napierala bkt_lb->int64.u64) || 2832*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2833*1a13f2e6SEdward Tomasz Napierala voival->int64.u64 < bkt_ub->int64.u64))) 2834*1a13f2e6SEdward Tomasz Napierala found = 1; 2835*1a13f2e6SEdward Tomasz Napierala break; 2836*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 2837*1a13f2e6SEdward Tomasz Napierala if (voival->intlong.slong >= bkt_lb->intlong.slong) 2838*1a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->intlong.slong == 2839*1a13f2e6SEdward Tomasz Napierala bkt_lb->intlong.slong) || 2840*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2841*1a13f2e6SEdward Tomasz Napierala voival->intlong.slong < 2842*1a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.slong))) 2843*1a13f2e6SEdward Tomasz Napierala found = 1; 2844*1a13f2e6SEdward Tomasz Napierala break; 2845*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 2846*1a13f2e6SEdward Tomasz Napierala if (voival->intlong.ulong >= bkt_lb->intlong.ulong) 2847*1a13f2e6SEdward Tomasz Napierala if ((eq_only && voival->intlong.ulong == 2848*1a13f2e6SEdward Tomasz Napierala bkt_lb->intlong.ulong) || 2849*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2850*1a13f2e6SEdward Tomasz Napierala voival->intlong.ulong < 2851*1a13f2e6SEdward Tomasz Napierala bkt_ub->intlong.ulong))) 2852*1a13f2e6SEdward Tomasz Napierala found = 1; 2853*1a13f2e6SEdward Tomasz Napierala break; 2854*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 2855*1a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32)) 2856*1a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q32.sq32, 2857*1a13f2e6SEdward Tomasz Napierala bkt_lb->q32.sq32)) || 2858*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2859*1a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q32.sq32, 2860*1a13f2e6SEdward Tomasz Napierala bkt_ub->q32.sq32)))) 2861*1a13f2e6SEdward Tomasz Napierala found = 1; 2862*1a13f2e6SEdward Tomasz Napierala break; 2863*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 2864*1a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32)) 2865*1a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q32.uq32, 2866*1a13f2e6SEdward Tomasz Napierala bkt_lb->q32.uq32)) || 2867*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2868*1a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q32.uq32, 2869*1a13f2e6SEdward Tomasz Napierala bkt_ub->q32.uq32)))) 2870*1a13f2e6SEdward Tomasz Napierala found = 1; 2871*1a13f2e6SEdward Tomasz Napierala break; 2872*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 2873*1a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64)) 2874*1a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q64.sq64, 2875*1a13f2e6SEdward Tomasz Napierala bkt_lb->q64.sq64)) || 2876*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2877*1a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q64.sq64, 2878*1a13f2e6SEdward Tomasz Napierala bkt_ub->q64.sq64)))) 2879*1a13f2e6SEdward Tomasz Napierala found = 1; 2880*1a13f2e6SEdward Tomasz Napierala break; 2881*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 2882*1a13f2e6SEdward Tomasz Napierala if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64)) 2883*1a13f2e6SEdward Tomasz Napierala if ((eq_only && Q_QEQ(voival->q64.uq64, 2884*1a13f2e6SEdward Tomasz Napierala bkt_lb->q64.uq64)) || 2885*1a13f2e6SEdward Tomasz Napierala (!eq_only && (!has_ub || 2886*1a13f2e6SEdward Tomasz Napierala Q_QLTQ(voival->q64.uq64, 2887*1a13f2e6SEdward Tomasz Napierala bkt_ub->q64.uq64)))) 2888*1a13f2e6SEdward Tomasz Napierala found = 1; 2889*1a13f2e6SEdward Tomasz Napierala break; 2890*1a13f2e6SEdward Tomasz Napierala default: 2891*1a13f2e6SEdward Tomasz Napierala break; 2892*1a13f2e6SEdward Tomasz Napierala } 2893*1a13f2e6SEdward Tomasz Napierala } 2894*1a13f2e6SEdward Tomasz Napierala 2895*1a13f2e6SEdward Tomasz Napierala if (found) { 2896*1a13f2e6SEdward Tomasz Napierala if (is32bit) 2897*1a13f2e6SEdward Tomasz Napierala *cnt32 += 1; 2898*1a13f2e6SEdward Tomasz Napierala else 2899*1a13f2e6SEdward Tomasz Napierala *cnt64 += 1; 2900*1a13f2e6SEdward Tomasz Napierala } else { 2901*1a13f2e6SEdward Tomasz Napierala if (is32bit) 2902*1a13f2e6SEdward Tomasz Napierala *oob32 += 1; 2903*1a13f2e6SEdward Tomasz Napierala else 2904*1a13f2e6SEdward Tomasz Napierala *oob64 += 1; 2905*1a13f2e6SEdward Tomasz Napierala } 2906*1a13f2e6SEdward Tomasz Napierala 2907*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 2908*1a13f2e6SEdward Tomasz Napierala return (error); 2909*1a13f2e6SEdward Tomasz Napierala } 2910*1a13f2e6SEdward Tomasz Napierala 2911*1a13f2e6SEdward Tomasz Napierala static inline int 2912*1a13f2e6SEdward Tomasz Napierala stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype, 2913*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgst *tdgst, int attempt) 2914*1a13f2e6SEdward Tomasz Napierala { 2915*1a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 2916*1a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 2917*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 2918*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 2919*1a13f2e6SEdward Tomasz Napierala uint64_t ebits, idxmask; 2920*1a13f2e6SEdward Tomasz Napierala uint32_t bitsperidx, nebits; 2921*1a13f2e6SEdward Tomasz Napierala int error, idx, is32bit, maxctds, remctds, tmperr; 2922*1a13f2e6SEdward Tomasz Napierala 2923*1a13f2e6SEdward Tomasz Napierala error = 0; 2924*1a13f2e6SEdward Tomasz Napierala 2925*1a13f2e6SEdward Tomasz Napierala switch (vs_dtype) { 2926*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 2927*1a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 2928*1a13f2e6SEdward Tomasz Napierala if (!ARB_FULL(ctd32tree)) 2929*1a13f2e6SEdward Tomasz Napierala return (0); 2930*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->compcnt++; 2931*1a13f2e6SEdward Tomasz Napierala maxctds = remctds = ARB_MAXNODES(ctd32tree); 2932*1a13f2e6SEdward Tomasz Napierala ARB_RESET_TREE(ctd32tree, ctdth32, maxctds); 2933*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt = 0; 2934*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 2935*1a13f2e6SEdward Tomasz Napierala ctd64tree = NULL; 2936*1a13f2e6SEdward Tomasz Napierala ctd64 = NULL; 2937*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 2938*1a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 2939*1a13f2e6SEdward Tomasz Napierala #endif 2940*1a13f2e6SEdward Tomasz Napierala break; 2941*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 2942*1a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 2943*1a13f2e6SEdward Tomasz Napierala if (!ARB_FULL(ctd64tree)) 2944*1a13f2e6SEdward Tomasz Napierala return (0); 2945*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->compcnt++; 2946*1a13f2e6SEdward Tomasz Napierala maxctds = remctds = ARB_MAXNODES(ctd64tree); 2947*1a13f2e6SEdward Tomasz Napierala ARB_RESET_TREE(ctd64tree, ctdth64, maxctds); 2948*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt = 0; 2949*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 2950*1a13f2e6SEdward Tomasz Napierala ctd32tree = NULL; 2951*1a13f2e6SEdward Tomasz Napierala ctd32 = NULL; 2952*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 2953*1a13f2e6SEdward Tomasz Napierala RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 2954*1a13f2e6SEdward Tomasz Napierala #endif 2955*1a13f2e6SEdward Tomasz Napierala break; 2956*1a13f2e6SEdward Tomasz Napierala default: 2957*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 2958*1a13f2e6SEdward Tomasz Napierala } 2959*1a13f2e6SEdward Tomasz Napierala 2960*1a13f2e6SEdward Tomasz Napierala /* 2961*1a13f2e6SEdward Tomasz Napierala * Rebuild the t-digest ARB by pseudorandomly selecting centroids and 2962*1a13f2e6SEdward Tomasz Napierala * re-inserting the mu/cnt of each as a value and corresponding weight. 2963*1a13f2e6SEdward Tomasz Napierala */ 2964*1a13f2e6SEdward Tomasz Napierala 2965*1a13f2e6SEdward Tomasz Napierala #define bitsperrand 31 /* Per random(3). */ 2966*1a13f2e6SEdward Tomasz Napierala ebits = 0; 2967*1a13f2e6SEdward Tomasz Napierala nebits = 0; 2968*1a13f2e6SEdward Tomasz Napierala bitsperidx = fls(maxctds); 2969*1a13f2e6SEdward Tomasz Napierala KASSERT(bitsperidx <= sizeof(ebits) << 3, 2970*1a13f2e6SEdward Tomasz Napierala ("%s: bitsperidx=%d, ebits=%d", 2971*1a13f2e6SEdward Tomasz Napierala __func__, bitsperidx, (int)(sizeof(ebits) << 3))); 2972*1a13f2e6SEdward Tomasz Napierala idxmask = (UINT64_C(1) << bitsperidx) - 1; 2973*1a13f2e6SEdward Tomasz Napierala srandom(stats_sbinuptime()); 2974*1a13f2e6SEdward Tomasz Napierala 2975*1a13f2e6SEdward Tomasz Napierala /* Initialise the free list with randomised centroid indices. */ 2976*1a13f2e6SEdward Tomasz Napierala for (; remctds > 0; remctds--) { 2977*1a13f2e6SEdward Tomasz Napierala while (nebits < bitsperidx) { 2978*1a13f2e6SEdward Tomasz Napierala ebits |= ((uint64_t)random()) << nebits; 2979*1a13f2e6SEdward Tomasz Napierala nebits += bitsperrand; 2980*1a13f2e6SEdward Tomasz Napierala if (nebits > (sizeof(ebits) << 3)) 2981*1a13f2e6SEdward Tomasz Napierala nebits = sizeof(ebits) << 3; 2982*1a13f2e6SEdward Tomasz Napierala } 2983*1a13f2e6SEdward Tomasz Napierala idx = ebits & idxmask; 2984*1a13f2e6SEdward Tomasz Napierala nebits -= bitsperidx; 2985*1a13f2e6SEdward Tomasz Napierala ebits >>= bitsperidx; 2986*1a13f2e6SEdward Tomasz Napierala 2987*1a13f2e6SEdward Tomasz Napierala /* 2988*1a13f2e6SEdward Tomasz Napierala * Select the next centroid to put on the ARB free list. We 2989*1a13f2e6SEdward Tomasz Napierala * start with the centroid at our randomly selected array index, 2990*1a13f2e6SEdward Tomasz Napierala * and work our way forwards until finding one (the latter 2991*1a13f2e6SEdward Tomasz Napierala * aspect reduces re-insertion randomness, but is good enough). 2992*1a13f2e6SEdward Tomasz Napierala */ 2993*1a13f2e6SEdward Tomasz Napierala do { 2994*1a13f2e6SEdward Tomasz Napierala if (idx >= maxctds) 2995*1a13f2e6SEdward Tomasz Napierala idx %= maxctds; 2996*1a13f2e6SEdward Tomasz Napierala 2997*1a13f2e6SEdward Tomasz Napierala if (is32bit) 2998*1a13f2e6SEdward Tomasz Napierala ctd32 = ARB_NODE(ctd32tree, idx); 2999*1a13f2e6SEdward Tomasz Napierala else 3000*1a13f2e6SEdward Tomasz Napierala ctd64 = ARB_NODE(ctd64tree, idx); 3001*1a13f2e6SEdward Tomasz Napierala } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) : 3002*1a13f2e6SEdward Tomasz Napierala ARB_ISFREE(ctd64, ctdlnk)) && ++idx); 3003*1a13f2e6SEdward Tomasz Napierala 3004*1a13f2e6SEdward Tomasz Napierala /* Put the centroid on the ARB free list. */ 3005*1a13f2e6SEdward Tomasz Napierala if (is32bit) 3006*1a13f2e6SEdward Tomasz Napierala ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk); 3007*1a13f2e6SEdward Tomasz Napierala else 3008*1a13f2e6SEdward Tomasz Napierala ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk); 3009*1a13f2e6SEdward Tomasz Napierala } 3010*1a13f2e6SEdward Tomasz Napierala 3011*1a13f2e6SEdward Tomasz Napierala /* 3012*1a13f2e6SEdward Tomasz Napierala * The free list now contains the randomised indices of every centroid. 3013*1a13f2e6SEdward Tomasz Napierala * Walk the free list from start to end, re-inserting each centroid's 3014*1a13f2e6SEdward Tomasz Napierala * mu/cnt. The tdgst_add() call may or may not consume the free centroid 3015*1a13f2e6SEdward Tomasz Napierala * we re-insert values from during each loop iteration, so we must latch 3016*1a13f2e6SEdward Tomasz Napierala * the index of the next free list centroid before the re-insertion 3017*1a13f2e6SEdward Tomasz Napierala * call. The previous loop above should have left the centroid pointer 3018*1a13f2e6SEdward Tomasz Napierala * pointing to the element at the head of the free list. 3019*1a13f2e6SEdward Tomasz Napierala */ 3020*1a13f2e6SEdward Tomasz Napierala KASSERT((is32bit ? 3021*1a13f2e6SEdward Tomasz Napierala ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) : 3022*1a13f2e6SEdward Tomasz Napierala ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)), 3023*1a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__, 3024*1a13f2e6SEdward Tomasz Napierala (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 3025*1a13f2e6SEdward Tomasz Napierala remctds = maxctds; 3026*1a13f2e6SEdward Tomasz Napierala while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) { 3027*1a13f2e6SEdward Tomasz Napierala tmperr = 0; 3028*1a13f2e6SEdward Tomasz Napierala if (is32bit) { 3029*1a13f2e6SEdward Tomasz Napierala s64q_t x; 3030*1a13f2e6SEdward Tomasz Napierala 3031*1a13f2e6SEdward Tomasz Napierala idx = ARB_NEXTFREEIDX(ctd32, ctdlnk); 3032*1a13f2e6SEdward Tomasz Napierala /* Cloning a s32q_t into a s64q_t should never fail. */ 3033*1a13f2e6SEdward Tomasz Napierala tmperr = Q_QCLONEQ(&x, ctd32->mu); 3034*1a13f2e6SEdward Tomasz Napierala tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add( 3035*1a13f2e6SEdward Tomasz Napierala vs_dtype, tdgst, x, ctd32->cnt, attempt); 3036*1a13f2e6SEdward Tomasz Napierala ctd32 = ARB_NODE(ctd32tree, idx); 3037*1a13f2e6SEdward Tomasz Napierala KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk), 3038*1a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__, 3039*1a13f2e6SEdward Tomasz Napierala ctd32tree)); 3040*1a13f2e6SEdward Tomasz Napierala } else { 3041*1a13f2e6SEdward Tomasz Napierala idx = ARB_NEXTFREEIDX(ctd64, ctdlnk); 3042*1a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst, 3043*1a13f2e6SEdward Tomasz Napierala ctd64->mu, ctd64->cnt, attempt); 3044*1a13f2e6SEdward Tomasz Napierala ctd64 = ARB_NODE(ctd64tree, idx); 3045*1a13f2e6SEdward Tomasz Napierala KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk), 3046*1a13f2e6SEdward Tomasz Napierala ("%s: t-digest ARB@%p free list bug", __func__, 3047*1a13f2e6SEdward Tomasz Napierala ctd64tree)); 3048*1a13f2e6SEdward Tomasz Napierala } 3049*1a13f2e6SEdward Tomasz Napierala /* 3050*1a13f2e6SEdward Tomasz Napierala * This process should not produce errors, bugs notwithstanding. 3051*1a13f2e6SEdward Tomasz Napierala * Just in case, latch any errors and attempt all re-insertions. 3052*1a13f2e6SEdward Tomasz Napierala */ 3053*1a13f2e6SEdward Tomasz Napierala error = tmperr ? tmperr : error; 3054*1a13f2e6SEdward Tomasz Napierala remctds--; 3055*1a13f2e6SEdward Tomasz Napierala } 3056*1a13f2e6SEdward Tomasz Napierala 3057*1a13f2e6SEdward Tomasz Napierala KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__, 3058*1a13f2e6SEdward Tomasz Napierala (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 3059*1a13f2e6SEdward Tomasz Napierala 3060*1a13f2e6SEdward Tomasz Napierala return (error); 3061*1a13f2e6SEdward Tomasz Napierala } 3062*1a13f2e6SEdward Tomasz Napierala 3063*1a13f2e6SEdward Tomasz Napierala static inline int 3064*1a13f2e6SEdward Tomasz Napierala stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst, 3065*1a13f2e6SEdward Tomasz Napierala s64q_t x, uint64_t weight, int attempt) 3066*1a13f2e6SEdward Tomasz Napierala { 3067*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3068*1a13f2e6SEdward Tomasz Napierala char qstr[Q_MAXSTRLEN(x, 10)]; 3069*1a13f2e6SEdward Tomasz Napierala #endif 3070*1a13f2e6SEdward Tomasz Napierala struct ctdth32 *ctd32tree; 3071*1a13f2e6SEdward Tomasz Napierala struct ctdth64 *ctd64tree; 3072*1a13f2e6SEdward Tomasz Napierala void *closest, *cur, *lb, *ub; 3073*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd32 *ctd32; 3074*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *ctd64; 3075*1a13f2e6SEdward Tomasz Napierala uint64_t cnt, smplcnt, sum, tmpsum; 3076*1a13f2e6SEdward Tomasz Napierala s64q_t k, minz, q, z; 3077*1a13f2e6SEdward Tomasz Napierala int error, is32bit, n; 3078*1a13f2e6SEdward Tomasz Napierala 3079*1a13f2e6SEdward Tomasz Napierala error = 0; 3080*1a13f2e6SEdward Tomasz Napierala minz = Q_INI(&z, 0, 0, Q_NFBITS(x)); 3081*1a13f2e6SEdward Tomasz Napierala 3082*1a13f2e6SEdward Tomasz Napierala switch (vs_dtype) { 3083*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 3084*1a13f2e6SEdward Tomasz Napierala if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt) 3085*1a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 3086*1a13f2e6SEdward Tomasz Napierala smplcnt = VSD(tdgstclust32, tdgst)->smplcnt; 3087*1a13f2e6SEdward Tomasz Napierala ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 3088*1a13f2e6SEdward Tomasz Napierala is32bit = 1; 3089*1a13f2e6SEdward Tomasz Napierala ctd64tree = NULL; 3090*1a13f2e6SEdward Tomasz Napierala ctd64 = NULL; 3091*1a13f2e6SEdward Tomasz Napierala break; 3092*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 3093*1a13f2e6SEdward Tomasz Napierala if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt) 3094*1a13f2e6SEdward Tomasz Napierala error = EOVERFLOW; 3095*1a13f2e6SEdward Tomasz Napierala smplcnt = VSD(tdgstclust64, tdgst)->smplcnt; 3096*1a13f2e6SEdward Tomasz Napierala ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 3097*1a13f2e6SEdward Tomasz Napierala is32bit = 0; 3098*1a13f2e6SEdward Tomasz Napierala ctd32tree = NULL; 3099*1a13f2e6SEdward Tomasz Napierala ctd32 = NULL; 3100*1a13f2e6SEdward Tomasz Napierala break; 3101*1a13f2e6SEdward Tomasz Napierala default: 3102*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 3103*1a13f2e6SEdward Tomasz Napierala break; 3104*1a13f2e6SEdward Tomasz Napierala } 3105*1a13f2e6SEdward Tomasz Napierala 3106*1a13f2e6SEdward Tomasz Napierala if (error) 3107*1a13f2e6SEdward Tomasz Napierala return (error); 3108*1a13f2e6SEdward Tomasz Napierala 3109*1a13f2e6SEdward Tomasz Napierala /* 3110*1a13f2e6SEdward Tomasz Napierala * Inspired by Ted Dunning's AVLTreeDigest.java 3111*1a13f2e6SEdward Tomasz Napierala */ 3112*1a13f2e6SEdward Tomasz Napierala do { 3113*1a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 3114*1a13f2e6SEdward Tomasz Napierala KASSERT(attempt < 5, 3115*1a13f2e6SEdward Tomasz Napierala ("%s: Too many attempts", __func__)); 3116*1a13f2e6SEdward Tomasz Napierala #endif 3117*1a13f2e6SEdward Tomasz Napierala if (attempt >= 5) 3118*1a13f2e6SEdward Tomasz Napierala return (EAGAIN); 3119*1a13f2e6SEdward Tomasz Napierala 3120*1a13f2e6SEdward Tomasz Napierala Q_SIFVAL(minz, Q_IFMAXVAL(minz)); 3121*1a13f2e6SEdward Tomasz Napierala closest = ub = NULL; 3122*1a13f2e6SEdward Tomasz Napierala sum = tmpsum = 0; 3123*1a13f2e6SEdward Tomasz Napierala 3124*1a13f2e6SEdward Tomasz Napierala if (is32bit) 3125*1a13f2e6SEdward Tomasz Napierala lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree)); 3126*1a13f2e6SEdward Tomasz Napierala else 3127*1a13f2e6SEdward Tomasz Napierala lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree)); 3128*1a13f2e6SEdward Tomasz Napierala 3129*1a13f2e6SEdward Tomasz Napierala if (lb == NULL) /* Empty tree. */ 3130*1a13f2e6SEdward Tomasz Napierala lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) : 3131*1a13f2e6SEdward Tomasz Napierala (void *)ARB_ROOT(ctd64tree)); 3132*1a13f2e6SEdward Tomasz Napierala 3133*1a13f2e6SEdward Tomasz Napierala /* 3134*1a13f2e6SEdward Tomasz Napierala * Find the set of centroids with minimum distance to x and 3135*1a13f2e6SEdward Tomasz Napierala * compute the sum of counts for all centroids with mean less 3136*1a13f2e6SEdward Tomasz Napierala * than the first centroid in the set. 3137*1a13f2e6SEdward Tomasz Napierala */ 3138*1a13f2e6SEdward Tomasz Napierala for (; cur != NULL; 3139*1a13f2e6SEdward Tomasz Napierala cur = (is32bit ? 3140*1a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 3141*1a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 3142*1a13f2e6SEdward Tomasz Napierala if (is32bit) { 3143*1a13f2e6SEdward Tomasz Napierala cnt = ctd32->cnt; 3144*1a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(ctd32->mu, x), 3145*1a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(mu,x)=%d", __func__, 3146*1a13f2e6SEdward Tomasz Napierala Q_RELPREC(ctd32->mu, x))); 3147*1a13f2e6SEdward Tomasz Napierala /* Ok to assign as both have same precision. */ 3148*1a13f2e6SEdward Tomasz Napierala z = ctd32->mu; 3149*1a13f2e6SEdward Tomasz Napierala } else { 3150*1a13f2e6SEdward Tomasz Napierala cnt = ctd64->cnt; 3151*1a13f2e6SEdward Tomasz Napierala KASSERT(Q_PRECEQ(ctd64->mu, x), 3152*1a13f2e6SEdward Tomasz Napierala ("%s: Q_RELPREC(mu,x)=%d", __func__, 3153*1a13f2e6SEdward Tomasz Napierala Q_RELPREC(ctd64->mu, x))); 3154*1a13f2e6SEdward Tomasz Napierala /* Ok to assign as both have same precision. */ 3155*1a13f2e6SEdward Tomasz Napierala z = ctd64->mu; 3156*1a13f2e6SEdward Tomasz Napierala } 3157*1a13f2e6SEdward Tomasz Napierala 3158*1a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&z, x); 3159*1a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 3160*1a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", __func__, 3161*1a13f2e6SEdward Tomasz Napierala error)); 3162*1a13f2e6SEdward Tomasz Napierala #endif 3163*1a13f2e6SEdward Tomasz Napierala if (error) 3164*1a13f2e6SEdward Tomasz Napierala return (error); 3165*1a13f2e6SEdward Tomasz Napierala 3166*1a13f2e6SEdward Tomasz Napierala z = Q_QABS(z); 3167*1a13f2e6SEdward Tomasz Napierala if (Q_QLTQ(z, minz)) { 3168*1a13f2e6SEdward Tomasz Napierala minz = z; 3169*1a13f2e6SEdward Tomasz Napierala lb = cur; 3170*1a13f2e6SEdward Tomasz Napierala sum = tmpsum; 3171*1a13f2e6SEdward Tomasz Napierala tmpsum += cnt; 3172*1a13f2e6SEdward Tomasz Napierala } else if (Q_QGTQ(z, minz)) { 3173*1a13f2e6SEdward Tomasz Napierala ub = cur; 3174*1a13f2e6SEdward Tomasz Napierala break; 3175*1a13f2e6SEdward Tomasz Napierala } 3176*1a13f2e6SEdward Tomasz Napierala } 3177*1a13f2e6SEdward Tomasz Napierala 3178*1a13f2e6SEdward Tomasz Napierala cur = (is32bit ? 3179*1a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) : 3180*1a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb)); 3181*1a13f2e6SEdward Tomasz Napierala 3182*1a13f2e6SEdward Tomasz Napierala for (n = 0; cur != ub; cur = (is32bit ? 3183*1a13f2e6SEdward Tomasz Napierala (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 3184*1a13f2e6SEdward Tomasz Napierala (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 3185*1a13f2e6SEdward Tomasz Napierala if (is32bit) 3186*1a13f2e6SEdward Tomasz Napierala cnt = ctd32->cnt; 3187*1a13f2e6SEdward Tomasz Napierala else 3188*1a13f2e6SEdward Tomasz Napierala cnt = ctd64->cnt; 3189*1a13f2e6SEdward Tomasz Napierala 3190*1a13f2e6SEdward Tomasz Napierala q = Q_CTRLINI(16); 3191*1a13f2e6SEdward Tomasz Napierala if (smplcnt == 1) 3192*1a13f2e6SEdward Tomasz Napierala error = Q_QFRACI(&q, 1, 2); 3193*1a13f2e6SEdward Tomasz Napierala else 3194*1a13f2e6SEdward Tomasz Napierala /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */ 3195*1a13f2e6SEdward Tomasz Napierala error = Q_QFRACI(&q, (sum << 1) + cnt - 1, 3196*1a13f2e6SEdward Tomasz Napierala (smplcnt - 1) << 1); 3197*1a13f2e6SEdward Tomasz Napierala k = q; 3198*1a13f2e6SEdward Tomasz Napierala /* k = q x 4 x samplcnt x attempt */ 3199*1a13f2e6SEdward Tomasz Napierala error |= Q_QMULI(&k, 4 * smplcnt * attempt); 3200*1a13f2e6SEdward Tomasz Napierala /* k = k x (1 - q) */ 3201*1a13f2e6SEdward Tomasz Napierala error |= Q_QSUBI(&q, 1); 3202*1a13f2e6SEdward Tomasz Napierala q = Q_QABS(q); 3203*1a13f2e6SEdward Tomasz Napierala error |= Q_QMULQ(&k, q); 3204*1a13f2e6SEdward Tomasz Napierala #if defined(DIAGNOSTIC) 3205*1a13f2e6SEdward Tomasz Napierala #if !defined(_KERNEL) 3206*1a13f2e6SEdward Tomasz Napierala double q_dbl, k_dbl, q2d, k2d; 3207*1a13f2e6SEdward Tomasz Napierala q2d = Q_Q2D(q); 3208*1a13f2e6SEdward Tomasz Napierala k2d = Q_Q2D(k); 3209*1a13f2e6SEdward Tomasz Napierala q_dbl = smplcnt == 1 ? 0.5 : 3210*1a13f2e6SEdward Tomasz Napierala (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1); 3211*1a13f2e6SEdward Tomasz Napierala k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt; 3212*1a13f2e6SEdward Tomasz Napierala /* 3213*1a13f2e6SEdward Tomasz Napierala * If the difference between q and q_dbl is greater than 3214*1a13f2e6SEdward Tomasz Napierala * the fractional precision of q, something is off. 3215*1a13f2e6SEdward Tomasz Napierala * NB: q is holding the value of 1 - q 3216*1a13f2e6SEdward Tomasz Napierala */ 3217*1a13f2e6SEdward Tomasz Napierala q_dbl = 1.0 - q_dbl; 3218*1a13f2e6SEdward Tomasz Napierala KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) < 3219*1a13f2e6SEdward Tomasz Napierala (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))), 3220*1a13f2e6SEdward Tomasz Napierala ("Q-type q bad precision")); 3221*1a13f2e6SEdward Tomasz Napierala KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) < 3222*1a13f2e6SEdward Tomasz Napierala 1.0 + (0.01 * smplcnt), 3223*1a13f2e6SEdward Tomasz Napierala ("Q-type k bad precision")); 3224*1a13f2e6SEdward Tomasz Napierala #endif /* !_KERNEL */ 3225*1a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", __func__, 3226*1a13f2e6SEdward Tomasz Napierala error)); 3227*1a13f2e6SEdward Tomasz Napierala #endif /* DIAGNOSTIC */ 3228*1a13f2e6SEdward Tomasz Napierala if (error) 3229*1a13f2e6SEdward Tomasz Napierala return (error); 3230*1a13f2e6SEdward Tomasz Napierala if ((is32bit && ((ctd32->cnt + weight) <= 3231*1a13f2e6SEdward Tomasz Napierala (uint64_t)Q_GIVAL(k))) || 3232*1a13f2e6SEdward Tomasz Napierala (!is32bit && ((ctd64->cnt + weight) <= 3233*1a13f2e6SEdward Tomasz Napierala (uint64_t)Q_GIVAL(k)))) { 3234*1a13f2e6SEdward Tomasz Napierala n++; 3235*1a13f2e6SEdward Tomasz Napierala /* random() produces 31 bits. */ 3236*1a13f2e6SEdward Tomasz Napierala if (random() < (INT32_MAX / n)) 3237*1a13f2e6SEdward Tomasz Napierala closest = cur; 3238*1a13f2e6SEdward Tomasz Napierala } 3239*1a13f2e6SEdward Tomasz Napierala sum += cnt; 3240*1a13f2e6SEdward Tomasz Napierala } 3241*1a13f2e6SEdward Tomasz Napierala } while (closest == NULL && 3242*1a13f2e6SEdward Tomasz Napierala (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) && 3243*1a13f2e6SEdward Tomasz Napierala (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst, 3244*1a13f2e6SEdward Tomasz Napierala attempt++)) == 0); 3245*1a13f2e6SEdward Tomasz Napierala 3246*1a13f2e6SEdward Tomasz Napierala if (error) 3247*1a13f2e6SEdward Tomasz Napierala return (error); 3248*1a13f2e6SEdward Tomasz Napierala 3249*1a13f2e6SEdward Tomasz Napierala if (closest != NULL) { 3250*1a13f2e6SEdward Tomasz Napierala /* Merge with an existing centroid. */ 3251*1a13f2e6SEdward Tomasz Napierala if (is32bit) { 3252*1a13f2e6SEdward Tomasz Napierala ctd32 = (struct voistatdata_tdgstctd32 *)closest; 3253*1a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&x, ctd32->mu); 3254*1a13f2e6SEdward Tomasz Napierala error = error ? error : 3255*1a13f2e6SEdward Tomasz Napierala Q_QDIVI(&x, ctd32->cnt + weight); 3256*1a13f2e6SEdward Tomasz Napierala if (error || (error = Q_QADDQ(&ctd32->mu, x))) { 3257*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3258*1a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", 3259*1a13f2e6SEdward Tomasz Napierala __func__, error)); 3260*1a13f2e6SEdward Tomasz Napierala #endif 3261*1a13f2e6SEdward Tomasz Napierala return (error); 3262*1a13f2e6SEdward Tomasz Napierala } 3263*1a13f2e6SEdward Tomasz Napierala ctd32->cnt += weight; 3264*1a13f2e6SEdward Tomasz Napierala error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) == 3265*1a13f2e6SEdward Tomasz Napierala NULL ? 0 : EALREADY; 3266*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3267*1a13f2e6SEdward Tomasz Napierala RB_REINSERT(rbctdth32, 3268*1a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 3269*1a13f2e6SEdward Tomasz Napierala #endif 3270*1a13f2e6SEdward Tomasz Napierala } else { 3271*1a13f2e6SEdward Tomasz Napierala ctd64 = (struct voistatdata_tdgstctd64 *)closest; 3272*1a13f2e6SEdward Tomasz Napierala error = Q_QSUBQ(&x, ctd64->mu); 3273*1a13f2e6SEdward Tomasz Napierala error = error ? error : 3274*1a13f2e6SEdward Tomasz Napierala Q_QDIVI(&x, ctd64->cnt + weight); 3275*1a13f2e6SEdward Tomasz Napierala if (error || (error = Q_QADDQ(&ctd64->mu, x))) { 3276*1a13f2e6SEdward Tomasz Napierala KASSERT(!error, ("%s: unexpected error %d", 3277*1a13f2e6SEdward Tomasz Napierala __func__, error)); 3278*1a13f2e6SEdward Tomasz Napierala return (error); 3279*1a13f2e6SEdward Tomasz Napierala } 3280*1a13f2e6SEdward Tomasz Napierala ctd64->cnt += weight; 3281*1a13f2e6SEdward Tomasz Napierala error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) == 3282*1a13f2e6SEdward Tomasz Napierala NULL ? 0 : EALREADY; 3283*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3284*1a13f2e6SEdward Tomasz Napierala RB_REINSERT(rbctdth64, 3285*1a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64); 3286*1a13f2e6SEdward Tomasz Napierala #endif 3287*1a13f2e6SEdward Tomasz Napierala } 3288*1a13f2e6SEdward Tomasz Napierala } else { 3289*1a13f2e6SEdward Tomasz Napierala /* 3290*1a13f2e6SEdward Tomasz Napierala * Add a new centroid. If digest compression is working 3291*1a13f2e6SEdward Tomasz Napierala * correctly, there should always be at least one free. 3292*1a13f2e6SEdward Tomasz Napierala */ 3293*1a13f2e6SEdward Tomasz Napierala if (is32bit) { 3294*1a13f2e6SEdward Tomasz Napierala ctd32 = ARB_GETFREE(ctd32tree, ctdlnk); 3295*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3296*1a13f2e6SEdward Tomasz Napierala KASSERT(ctd32 != NULL, 3297*1a13f2e6SEdward Tomasz Napierala ("%s: t-digest@%p has no free centroids", 3298*1a13f2e6SEdward Tomasz Napierala __func__, tdgst)); 3299*1a13f2e6SEdward Tomasz Napierala #endif 3300*1a13f2e6SEdward Tomasz Napierala if (ctd32 == NULL) 3301*1a13f2e6SEdward Tomasz Napierala return (EAGAIN); 3302*1a13f2e6SEdward Tomasz Napierala if ((error = Q_QCPYVALQ(&ctd32->mu, x))) 3303*1a13f2e6SEdward Tomasz Napierala return (error); 3304*1a13f2e6SEdward Tomasz Napierala ctd32->cnt = weight; 3305*1a13f2e6SEdward Tomasz Napierala error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ? 3306*1a13f2e6SEdward Tomasz Napierala 0 : EALREADY; 3307*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3308*1a13f2e6SEdward Tomasz Napierala RB_INSERT(rbctdth32, 3309*1a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 3310*1a13f2e6SEdward Tomasz Napierala #endif 3311*1a13f2e6SEdward Tomasz Napierala } else { 3312*1a13f2e6SEdward Tomasz Napierala ctd64 = ARB_GETFREE(ctd64tree, ctdlnk); 3313*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3314*1a13f2e6SEdward Tomasz Napierala KASSERT(ctd64 != NULL, 3315*1a13f2e6SEdward Tomasz Napierala ("%s: t-digest@%p has no free centroids", 3316*1a13f2e6SEdward Tomasz Napierala __func__, tdgst)); 3317*1a13f2e6SEdward Tomasz Napierala #endif 3318*1a13f2e6SEdward Tomasz Napierala if (ctd64 == NULL) /* Should not happen. */ 3319*1a13f2e6SEdward Tomasz Napierala return (EAGAIN); 3320*1a13f2e6SEdward Tomasz Napierala /* Direct assignment ok as both have same type/prec. */ 3321*1a13f2e6SEdward Tomasz Napierala ctd64->mu = x; 3322*1a13f2e6SEdward Tomasz Napierala ctd64->cnt = weight; 3323*1a13f2e6SEdward Tomasz Napierala error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ? 3324*1a13f2e6SEdward Tomasz Napierala 0 : EALREADY; 3325*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3326*1a13f2e6SEdward Tomasz Napierala RB_INSERT(rbctdth64, &VSD(tdgstclust64, 3327*1a13f2e6SEdward Tomasz Napierala tdgst)->rbctdtree, ctd64); 3328*1a13f2e6SEdward Tomasz Napierala #endif 3329*1a13f2e6SEdward Tomasz Napierala } 3330*1a13f2e6SEdward Tomasz Napierala } 3331*1a13f2e6SEdward Tomasz Napierala 3332*1a13f2e6SEdward Tomasz Napierala if (is32bit) 3333*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust32, tdgst)->smplcnt += weight; 3334*1a13f2e6SEdward Tomasz Napierala else { 3335*1a13f2e6SEdward Tomasz Napierala VSD(tdgstclust64, tdgst)->smplcnt += weight; 3336*1a13f2e6SEdward Tomasz Napierala 3337*1a13f2e6SEdward Tomasz Napierala #ifdef DIAGNOSTIC 3338*1a13f2e6SEdward Tomasz Napierala struct rbctdth64 *rbctdtree = 3339*1a13f2e6SEdward Tomasz Napierala &VSD(tdgstclust64, tdgst)->rbctdtree; 3340*1a13f2e6SEdward Tomasz Napierala struct voistatdata_tdgstctd64 *rbctd64; 3341*1a13f2e6SEdward Tomasz Napierala int i = 0; 3342*1a13f2e6SEdward Tomasz Napierala ARB_FOREACH(ctd64, ctdth64, ctd64tree) { 3343*1a13f2e6SEdward Tomasz Napierala rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) : 3344*1a13f2e6SEdward Tomasz Napierala RB_NEXT(rbctdth64, rbctdtree, rbctd64)); 3345*1a13f2e6SEdward Tomasz Napierala 3346*1a13f2e6SEdward Tomasz Napierala if (i >= ARB_CURNODES(ctd64tree) 3347*1a13f2e6SEdward Tomasz Napierala || ctd64 != rbctd64 3348*1a13f2e6SEdward Tomasz Napierala || ARB_MIN(ctdth64, ctd64tree) != 3349*1a13f2e6SEdward Tomasz Napierala RB_MIN(rbctdth64, rbctdtree) 3350*1a13f2e6SEdward Tomasz Napierala || ARB_MAX(ctdth64, ctd64tree) != 3351*1a13f2e6SEdward Tomasz Napierala RB_MAX(rbctdth64, rbctdtree) 3352*1a13f2e6SEdward Tomasz Napierala || ARB_LEFTIDX(ctd64, ctdlnk) != 3353*1a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk)) 3354*1a13f2e6SEdward Tomasz Napierala || ARB_RIGHTIDX(ctd64, ctdlnk) != 3355*1a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk)) 3356*1a13f2e6SEdward Tomasz Napierala || ARB_PARENTIDX(ctd64, ctdlnk) != 3357*1a13f2e6SEdward Tomasz Napierala ARB_SELFIDX(ctd64tree, 3358*1a13f2e6SEdward Tomasz Napierala RB_PARENT(rbctd64, rblnk))) { 3359*1a13f2e6SEdward Tomasz Napierala Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr)); 3360*1a13f2e6SEdward Tomasz Napierala printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 3361*1a13f2e6SEdward Tomasz Napierala "mu=%s\n", 3362*1a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, ctd64), 3363*1a13f2e6SEdward Tomasz Napierala ARB_PARENTIDX(ctd64, ctdlnk), 3364*1a13f2e6SEdward Tomasz Napierala ARB_LEFTIDX(ctd64, ctdlnk), 3365*1a13f2e6SEdward Tomasz Napierala ARB_RIGHTIDX(ctd64, ctdlnk), 3366*1a13f2e6SEdward Tomasz Napierala ARB_COLOR(ctd64, ctdlnk), 3367*1a13f2e6SEdward Tomasz Napierala qstr); 3368*1a13f2e6SEdward Tomasz Napierala 3369*1a13f2e6SEdward Tomasz Napierala Q_TOSTR(rbctd64->mu, -1, 10, qstr, 3370*1a13f2e6SEdward Tomasz Napierala sizeof(qstr)); 3371*1a13f2e6SEdward Tomasz Napierala printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 3372*1a13f2e6SEdward Tomasz Napierala "mu=%s\n", 3373*1a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, rbctd64), 3374*1a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, 3375*1a13f2e6SEdward Tomasz Napierala RB_PARENT(rbctd64, rblnk)), 3376*1a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, 3377*1a13f2e6SEdward Tomasz Napierala RB_LEFT(rbctd64, rblnk)), 3378*1a13f2e6SEdward Tomasz Napierala (int)ARB_SELFIDX(ctd64tree, 3379*1a13f2e6SEdward Tomasz Napierala RB_RIGHT(rbctd64, rblnk)), 3380*1a13f2e6SEdward Tomasz Napierala RB_COLOR(rbctd64, rblnk), 3381*1a13f2e6SEdward Tomasz Napierala qstr); 3382*1a13f2e6SEdward Tomasz Napierala 3383*1a13f2e6SEdward Tomasz Napierala panic("RB@%p and ARB@%p trees differ\n", 3384*1a13f2e6SEdward Tomasz Napierala rbctdtree, ctd64tree); 3385*1a13f2e6SEdward Tomasz Napierala } 3386*1a13f2e6SEdward Tomasz Napierala i++; 3387*1a13f2e6SEdward Tomasz Napierala } 3388*1a13f2e6SEdward Tomasz Napierala #endif /* DIAGNOSTIC */ 3389*1a13f2e6SEdward Tomasz Napierala } 3390*1a13f2e6SEdward Tomasz Napierala 3391*1a13f2e6SEdward Tomasz Napierala return (error); 3392*1a13f2e6SEdward Tomasz Napierala } 3393*1a13f2e6SEdward Tomasz Napierala 3394*1a13f2e6SEdward Tomasz Napierala static inline int 3395*1a13f2e6SEdward Tomasz Napierala stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival, 3396*1a13f2e6SEdward Tomasz Napierala struct voistat *vs, struct voistatdata_tdgst *tdgst) 3397*1a13f2e6SEdward Tomasz Napierala { 3398*1a13f2e6SEdward Tomasz Napierala s64q_t x; 3399*1a13f2e6SEdward Tomasz Napierala int error; 3400*1a13f2e6SEdward Tomasz Napierala 3401*1a13f2e6SEdward Tomasz Napierala error = 0; 3402*1a13f2e6SEdward Tomasz Napierala 3403*1a13f2e6SEdward Tomasz Napierala switch (vs->dtype) { 3404*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST32: 3405*1a13f2e6SEdward Tomasz Napierala /* Use same precision as the user's centroids. */ 3406*1a13f2e6SEdward Tomasz Napierala Q_INI(&x, 0, 0, Q_NFBITS( 3407*1a13f2e6SEdward Tomasz Napierala ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu)); 3408*1a13f2e6SEdward Tomasz Napierala break; 3409*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_TDGSTCLUST64: 3410*1a13f2e6SEdward Tomasz Napierala /* Use same precision as the user's centroids. */ 3411*1a13f2e6SEdward Tomasz Napierala Q_INI(&x, 0, 0, Q_NFBITS( 3412*1a13f2e6SEdward Tomasz Napierala ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu)); 3413*1a13f2e6SEdward Tomasz Napierala break; 3414*1a13f2e6SEdward Tomasz Napierala default: 3415*1a13f2e6SEdward Tomasz Napierala KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 || 3416*1a13f2e6SEdward Tomasz Napierala vs->dtype == VSD_DTYPE_TDGSTCLUST64, 3417*1a13f2e6SEdward Tomasz Napierala ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>", 3418*1a13f2e6SEdward Tomasz Napierala __func__, vs->dtype)); 3419*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 3420*1a13f2e6SEdward Tomasz Napierala } 3421*1a13f2e6SEdward Tomasz Napierala 3422*1a13f2e6SEdward Tomasz Napierala /* 3423*1a13f2e6SEdward Tomasz Napierala * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid 3424*1a13f2e6SEdward Tomasz Napierala * returning EOVERFLOW if the voival would have fit in a u64q_t. 3425*1a13f2e6SEdward Tomasz Napierala */ 3426*1a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 3427*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 3428*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int32.s32); 3429*1a13f2e6SEdward Tomasz Napierala break; 3430*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 3431*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int32.u32); 3432*1a13f2e6SEdward Tomasz Napierala break; 3433*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 3434*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int64.s64); 3435*1a13f2e6SEdward Tomasz Napierala break; 3436*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 3437*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->int64.u64); 3438*1a13f2e6SEdward Tomasz Napierala break; 3439*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 3440*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->intlong.slong); 3441*1a13f2e6SEdward Tomasz Napierala break; 3442*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 3443*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALI(&x, voival->intlong.ulong); 3444*1a13f2e6SEdward Tomasz Napierala break; 3445*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 3446*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q32.sq32); 3447*1a13f2e6SEdward Tomasz Napierala break; 3448*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 3449*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q32.uq32); 3450*1a13f2e6SEdward Tomasz Napierala break; 3451*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 3452*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q64.sq64); 3453*1a13f2e6SEdward Tomasz Napierala break; 3454*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 3455*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ(&x, voival->q64.uq64); 3456*1a13f2e6SEdward Tomasz Napierala break; 3457*1a13f2e6SEdward Tomasz Napierala default: 3458*1a13f2e6SEdward Tomasz Napierala error = EINVAL; 3459*1a13f2e6SEdward Tomasz Napierala break; 3460*1a13f2e6SEdward Tomasz Napierala } 3461*1a13f2e6SEdward Tomasz Napierala 3462*1a13f2e6SEdward Tomasz Napierala if (error || 3463*1a13f2e6SEdward Tomasz Napierala (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1))) 3464*1a13f2e6SEdward Tomasz Napierala return (error); 3465*1a13f2e6SEdward Tomasz Napierala 3466*1a13f2e6SEdward Tomasz Napierala vs->flags |= VS_VSDVALID; 3467*1a13f2e6SEdward Tomasz Napierala return (0); 3468*1a13f2e6SEdward Tomasz Napierala } 3469*1a13f2e6SEdward Tomasz Napierala 3470*1a13f2e6SEdward Tomasz Napierala int 3471*1a13f2e6SEdward Tomasz Napierala stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id, 3472*1a13f2e6SEdward Tomasz Napierala enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags) 3473*1a13f2e6SEdward Tomasz Napierala { 3474*1a13f2e6SEdward Tomasz Napierala struct voi *v; 3475*1a13f2e6SEdward Tomasz Napierala struct voistat *vs; 3476*1a13f2e6SEdward Tomasz Napierala void *statevsd, *vsd; 3477*1a13f2e6SEdward Tomasz Napierala int error, i, tmperr; 3478*1a13f2e6SEdward Tomasz Napierala 3479*1a13f2e6SEdward Tomasz Napierala error = 0; 3480*1a13f2e6SEdward Tomasz Napierala 3481*1a13f2e6SEdward Tomasz Napierala if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) || 3482*1a13f2e6SEdward Tomasz Napierala voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL) 3483*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 3484*1a13f2e6SEdward Tomasz Napierala v = &sb->vois[voi_id]; 3485*1a13f2e6SEdward Tomasz Napierala if (voi_dtype != v->dtype || v->id < 0 || 3486*1a13f2e6SEdward Tomasz Napierala ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE))) 3487*1a13f2e6SEdward Tomasz Napierala return (EINVAL); 3488*1a13f2e6SEdward Tomasz Napierala 3489*1a13f2e6SEdward Tomasz Napierala vs = BLOB_OFFSET(sb, v->stats_off); 3490*1a13f2e6SEdward Tomasz Napierala if (v->flags & VOI_REQSTATE) 3491*1a13f2e6SEdward Tomasz Napierala statevsd = BLOB_OFFSET(sb, vs->data_off); 3492*1a13f2e6SEdward Tomasz Napierala else 3493*1a13f2e6SEdward Tomasz Napierala statevsd = NULL; 3494*1a13f2e6SEdward Tomasz Napierala 3495*1a13f2e6SEdward Tomasz Napierala if (flags & SB_VOI_RELUPDATE) { 3496*1a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 3497*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 3498*1a13f2e6SEdward Tomasz Napierala voival->int32.s32 += 3499*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.s32; 3500*1a13f2e6SEdward Tomasz Napierala break; 3501*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 3502*1a13f2e6SEdward Tomasz Napierala voival->int32.u32 += 3503*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.u32; 3504*1a13f2e6SEdward Tomasz Napierala break; 3505*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 3506*1a13f2e6SEdward Tomasz Napierala voival->int64.s64 += 3507*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.s64; 3508*1a13f2e6SEdward Tomasz Napierala break; 3509*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 3510*1a13f2e6SEdward Tomasz Napierala voival->int64.u64 += 3511*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.u64; 3512*1a13f2e6SEdward Tomasz Napierala break; 3513*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 3514*1a13f2e6SEdward Tomasz Napierala voival->intlong.slong += 3515*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.slong; 3516*1a13f2e6SEdward Tomasz Napierala break; 3517*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 3518*1a13f2e6SEdward Tomasz Napierala voival->intlong.ulong += 3519*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.ulong; 3520*1a13f2e6SEdward Tomasz Napierala break; 3521*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 3522*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q32.sq32, 3523*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q32.sq32); 3524*1a13f2e6SEdward Tomasz Napierala break; 3525*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 3526*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q32.uq32, 3527*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q32.uq32); 3528*1a13f2e6SEdward Tomasz Napierala break; 3529*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 3530*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q64.sq64, 3531*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q64.sq64); 3532*1a13f2e6SEdward Tomasz Napierala break; 3533*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 3534*1a13f2e6SEdward Tomasz Napierala error = Q_QADDQ(&voival->q64.uq64, 3535*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.q64.uq64); 3536*1a13f2e6SEdward Tomasz Napierala break; 3537*1a13f2e6SEdward Tomasz Napierala default: 3538*1a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 3539*1a13f2e6SEdward Tomasz Napierala break; 3540*1a13f2e6SEdward Tomasz Napierala } 3541*1a13f2e6SEdward Tomasz Napierala } 3542*1a13f2e6SEdward Tomasz Napierala 3543*1a13f2e6SEdward Tomasz Napierala if (error) 3544*1a13f2e6SEdward Tomasz Napierala return (error); 3545*1a13f2e6SEdward Tomasz Napierala 3546*1a13f2e6SEdward Tomasz Napierala for (i = v->voistatmaxid; i > 0; i--) { 3547*1a13f2e6SEdward Tomasz Napierala vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i]; 3548*1a13f2e6SEdward Tomasz Napierala if (vs->stype < 0) 3549*1a13f2e6SEdward Tomasz Napierala continue; 3550*1a13f2e6SEdward Tomasz Napierala 3551*1a13f2e6SEdward Tomasz Napierala vsd = BLOB_OFFSET(sb, vs->data_off); 3552*1a13f2e6SEdward Tomasz Napierala 3553*1a13f2e6SEdward Tomasz Napierala switch (vs->stype) { 3554*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_MAX: 3555*1a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_max(voi_dtype, voival, 3556*1a13f2e6SEdward Tomasz Napierala vs, vsd); 3557*1a13f2e6SEdward Tomasz Napierala break; 3558*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_MIN: 3559*1a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_min(voi_dtype, voival, 3560*1a13f2e6SEdward Tomasz Napierala vs, vsd); 3561*1a13f2e6SEdward Tomasz Napierala break; 3562*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_SUM: 3563*1a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_sum(voi_dtype, voival, 3564*1a13f2e6SEdward Tomasz Napierala vs, vsd); 3565*1a13f2e6SEdward Tomasz Napierala break; 3566*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_HIST: 3567*1a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_hist(voi_dtype, voival, 3568*1a13f2e6SEdward Tomasz Napierala vs, vsd); 3569*1a13f2e6SEdward Tomasz Napierala break; 3570*1a13f2e6SEdward Tomasz Napierala case VS_STYPE_TDGST: 3571*1a13f2e6SEdward Tomasz Napierala tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival, 3572*1a13f2e6SEdward Tomasz Napierala vs, vsd); 3573*1a13f2e6SEdward Tomasz Napierala break; 3574*1a13f2e6SEdward Tomasz Napierala default: 3575*1a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 3576*1a13f2e6SEdward Tomasz Napierala break; 3577*1a13f2e6SEdward Tomasz Napierala } 3578*1a13f2e6SEdward Tomasz Napierala 3579*1a13f2e6SEdward Tomasz Napierala if (tmperr) { 3580*1a13f2e6SEdward Tomasz Napierala error = tmperr; 3581*1a13f2e6SEdward Tomasz Napierala VS_INCERRS(vs); 3582*1a13f2e6SEdward Tomasz Napierala } 3583*1a13f2e6SEdward Tomasz Napierala } 3584*1a13f2e6SEdward Tomasz Napierala 3585*1a13f2e6SEdward Tomasz Napierala if (statevsd) { 3586*1a13f2e6SEdward Tomasz Napierala switch (voi_dtype) { 3587*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S32: 3588*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.s32 = 3589*1a13f2e6SEdward Tomasz Napierala voival->int32.s32; 3590*1a13f2e6SEdward Tomasz Napierala break; 3591*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U32: 3592*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int32.u32 = 3593*1a13f2e6SEdward Tomasz Napierala voival->int32.u32; 3594*1a13f2e6SEdward Tomasz Napierala break; 3595*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_S64: 3596*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.s64 = 3597*1a13f2e6SEdward Tomasz Napierala voival->int64.s64; 3598*1a13f2e6SEdward Tomasz Napierala break; 3599*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_U64: 3600*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.int64.u64 = 3601*1a13f2e6SEdward Tomasz Napierala voival->int64.u64; 3602*1a13f2e6SEdward Tomasz Napierala break; 3603*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_SLONG: 3604*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.slong = 3605*1a13f2e6SEdward Tomasz Napierala voival->intlong.slong; 3606*1a13f2e6SEdward Tomasz Napierala break; 3607*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_INT_ULONG: 3608*1a13f2e6SEdward Tomasz Napierala VSD(voistate, statevsd)->prev.intlong.ulong = 3609*1a13f2e6SEdward Tomasz Napierala voival->intlong.ulong; 3610*1a13f2e6SEdward Tomasz Napierala break; 3611*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S32: 3612*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 3613*1a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q32.sq32, 3614*1a13f2e6SEdward Tomasz Napierala voival->q32.sq32); 3615*1a13f2e6SEdward Tomasz Napierala break; 3616*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U32: 3617*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 3618*1a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q32.uq32, 3619*1a13f2e6SEdward Tomasz Napierala voival->q32.uq32); 3620*1a13f2e6SEdward Tomasz Napierala break; 3621*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_S64: 3622*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 3623*1a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q64.sq64, 3624*1a13f2e6SEdward Tomasz Napierala voival->q64.sq64); 3625*1a13f2e6SEdward Tomasz Napierala break; 3626*1a13f2e6SEdward Tomasz Napierala case VSD_DTYPE_Q_U64: 3627*1a13f2e6SEdward Tomasz Napierala error = Q_QCPYVALQ( 3628*1a13f2e6SEdward Tomasz Napierala &VSD(voistate, statevsd)->prev.q64.uq64, 3629*1a13f2e6SEdward Tomasz Napierala voival->q64.uq64); 3630*1a13f2e6SEdward Tomasz Napierala break; 3631*1a13f2e6SEdward Tomasz Napierala default: 3632*1a13f2e6SEdward Tomasz Napierala KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 3633*1a13f2e6SEdward Tomasz Napierala break; 3634*1a13f2e6SEdward Tomasz Napierala } 3635*1a13f2e6SEdward Tomasz Napierala } 3636*1a13f2e6SEdward Tomasz Napierala 3637*1a13f2e6SEdward Tomasz Napierala return (error); 3638*1a13f2e6SEdward Tomasz Napierala } 3639*1a13f2e6SEdward Tomasz Napierala 3640*1a13f2e6SEdward Tomasz Napierala #ifdef _KERNEL 3641*1a13f2e6SEdward Tomasz Napierala 3642*1a13f2e6SEdward Tomasz Napierala static void 3643*1a13f2e6SEdward Tomasz Napierala stats_init(void *arg) 3644*1a13f2e6SEdward Tomasz Napierala { 3645*1a13f2e6SEdward Tomasz Napierala 3646*1a13f2e6SEdward Tomasz Napierala } 3647*1a13f2e6SEdward Tomasz Napierala SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL); 3648*1a13f2e6SEdward Tomasz Napierala 3649*1a13f2e6SEdward Tomasz Napierala /* 3650*1a13f2e6SEdward Tomasz Napierala * Sysctl handler to display the list of available stats templates. 3651*1a13f2e6SEdward Tomasz Napierala */ 3652*1a13f2e6SEdward Tomasz Napierala static int 3653*1a13f2e6SEdward Tomasz Napierala stats_tpl_list_available(SYSCTL_HANDLER_ARGS) 3654*1a13f2e6SEdward Tomasz Napierala { 3655*1a13f2e6SEdward Tomasz Napierala struct sbuf *s; 3656*1a13f2e6SEdward Tomasz Napierala int err, i; 3657*1a13f2e6SEdward Tomasz Napierala 3658*1a13f2e6SEdward Tomasz Napierala err = 0; 3659*1a13f2e6SEdward Tomasz Napierala 3660*1a13f2e6SEdward Tomasz Napierala /* We can tolerate ntpl being stale, so do not take the lock. */ 3661*1a13f2e6SEdward Tomasz Napierala s = sbuf_new(NULL, NULL, /* +1 per tpl for , */ 3662*1a13f2e6SEdward Tomasz Napierala ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN); 3663*1a13f2e6SEdward Tomasz Napierala if (s == NULL) 3664*1a13f2e6SEdward Tomasz Napierala return (ENOMEM); 3665*1a13f2e6SEdward Tomasz Napierala 3666*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 3667*1a13f2e6SEdward Tomasz Napierala for (i = 0; i < ntpl; i++) { 3668*1a13f2e6SEdward Tomasz Napierala err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "", 3669*1a13f2e6SEdward Tomasz Napierala tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash); 3670*1a13f2e6SEdward Tomasz Napierala if (err) { 3671*1a13f2e6SEdward Tomasz Napierala /* Sbuf overflow condition. */ 3672*1a13f2e6SEdward Tomasz Napierala err = EOVERFLOW; 3673*1a13f2e6SEdward Tomasz Napierala break; 3674*1a13f2e6SEdward Tomasz Napierala } 3675*1a13f2e6SEdward Tomasz Napierala } 3676*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 3677*1a13f2e6SEdward Tomasz Napierala 3678*1a13f2e6SEdward Tomasz Napierala if (!err) { 3679*1a13f2e6SEdward Tomasz Napierala sbuf_finish(s); 3680*1a13f2e6SEdward Tomasz Napierala err = sysctl_handle_string(oidp, sbuf_data(s), 0, req); 3681*1a13f2e6SEdward Tomasz Napierala } 3682*1a13f2e6SEdward Tomasz Napierala 3683*1a13f2e6SEdward Tomasz Napierala sbuf_delete(s); 3684*1a13f2e6SEdward Tomasz Napierala return (err); 3685*1a13f2e6SEdward Tomasz Napierala } 3686*1a13f2e6SEdward Tomasz Napierala 3687*1a13f2e6SEdward Tomasz Napierala /* 3688*1a13f2e6SEdward Tomasz Napierala * Called by subsystem-specific sysctls to report and/or parse the list of 3689*1a13f2e6SEdward Tomasz Napierala * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t 3690*1a13f2e6SEdward Tomasz Napierala * conformant function pointer must be passed in as arg1, which is used to 3691*1a13f2e6SEdward Tomasz Napierala * interact with the subsystem's stats template sample rates list. If arg2 > 0, 3692*1a13f2e6SEdward Tomasz Napierala * a zero-initialised allocation of arg2-sized contextual memory is 3693*1a13f2e6SEdward Tomasz Napierala * heap-allocated and passed in to all subsystem callbacks made during the 3694*1a13f2e6SEdward Tomasz Napierala * operation of stats_tpl_sample_rates(). 3695*1a13f2e6SEdward Tomasz Napierala * 3696*1a13f2e6SEdward Tomasz Napierala * XXXLAS: Assumes templates are never removed, which is currently true but may 3697*1a13f2e6SEdward Tomasz Napierala * need to be reworked in future if dynamic template management becomes a 3698*1a13f2e6SEdward Tomasz Napierala * requirement e.g. to support kernel module based templates. 3699*1a13f2e6SEdward Tomasz Napierala */ 3700*1a13f2e6SEdward Tomasz Napierala int 3701*1a13f2e6SEdward Tomasz Napierala stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS) 3702*1a13f2e6SEdward Tomasz Napierala { 3703*1a13f2e6SEdward Tomasz Napierala char kvpair_fmt[16], tplspec_fmt[16]; 3704*1a13f2e6SEdward Tomasz Napierala char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN]; 3705*1a13f2e6SEdward Tomasz Napierala char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */ 3706*1a13f2e6SEdward Tomasz Napierala stats_tpl_sr_cb_t subsys_cb; 3707*1a13f2e6SEdward Tomasz Napierala void *subsys_ctx; 3708*1a13f2e6SEdward Tomasz Napierala char *buf, *new_rates_usr_str, *tpl_name_p; 3709*1a13f2e6SEdward Tomasz Napierala struct stats_tpl_sample_rate *rates; 3710*1a13f2e6SEdward Tomasz Napierala struct sbuf *s, _s; 3711*1a13f2e6SEdward Tomasz Napierala uint32_t cum_pct, pct, tpl_hash; 3712*1a13f2e6SEdward Tomasz Napierala int err, i, off, len, newlen, nrates; 3713*1a13f2e6SEdward Tomasz Napierala 3714*1a13f2e6SEdward Tomasz Napierala buf = NULL; 3715*1a13f2e6SEdward Tomasz Napierala rates = NULL; 3716*1a13f2e6SEdward Tomasz Napierala err = nrates = 0; 3717*1a13f2e6SEdward Tomasz Napierala subsys_cb = (stats_tpl_sr_cb_t)arg1; 3718*1a13f2e6SEdward Tomasz Napierala KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__)); 3719*1a13f2e6SEdward Tomasz Napierala if (arg2 > 0) 3720*1a13f2e6SEdward Tomasz Napierala subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO); 3721*1a13f2e6SEdward Tomasz Napierala else 3722*1a13f2e6SEdward Tomasz Napierala subsys_ctx = NULL; 3723*1a13f2e6SEdward Tomasz Napierala 3724*1a13f2e6SEdward Tomasz Napierala /* Grab current count of subsystem rates. */ 3725*1a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx); 3726*1a13f2e6SEdward Tomasz Napierala if (err) 3727*1a13f2e6SEdward Tomasz Napierala goto done; 3728*1a13f2e6SEdward Tomasz Napierala 3729*1a13f2e6SEdward Tomasz Napierala /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */ 3730*1a13f2e6SEdward Tomasz Napierala len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5)); 3731*1a13f2e6SEdward Tomasz Napierala 3732*1a13f2e6SEdward Tomasz Napierala if (req->oldptr != NULL || req->newptr != NULL) 3733*1a13f2e6SEdward Tomasz Napierala buf = malloc(len, M_TEMP, M_WAITOK); 3734*1a13f2e6SEdward Tomasz Napierala 3735*1a13f2e6SEdward Tomasz Napierala if (req->oldptr != NULL) { 3736*1a13f2e6SEdward Tomasz Napierala if (nrates == 0) { 3737*1a13f2e6SEdward Tomasz Napierala /* No rates, so return an empty string via oldptr. */ 3738*1a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, "", 1); 3739*1a13f2e6SEdward Tomasz Napierala if (err) 3740*1a13f2e6SEdward Tomasz Napierala goto done; 3741*1a13f2e6SEdward Tomasz Napierala goto process_new; 3742*1a13f2e6SEdward Tomasz Napierala } 3743*1a13f2e6SEdward Tomasz Napierala 3744*1a13f2e6SEdward Tomasz Napierala s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL); 3745*1a13f2e6SEdward Tomasz Napierala 3746*1a13f2e6SEdward Tomasz Napierala /* Grab locked count of, and ptr to, subsystem rates. */ 3747*1a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates, 3748*1a13f2e6SEdward Tomasz Napierala subsys_ctx); 3749*1a13f2e6SEdward Tomasz Napierala if (err) 3750*1a13f2e6SEdward Tomasz Napierala goto done; 3751*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RLOCK(); 3752*1a13f2e6SEdward Tomasz Napierala for (i = 0; i < nrates && !err; i++) { 3753*1a13f2e6SEdward Tomasz Napierala err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "", 3754*1a13f2e6SEdward Tomasz Napierala tpllist[rates[i].tpl_slot_id]->mb->tplname, 3755*1a13f2e6SEdward Tomasz Napierala tpllist[rates[i].tpl_slot_id]->mb->tplhash, 3756*1a13f2e6SEdward Tomasz Napierala rates[i].tpl_sample_pct); 3757*1a13f2e6SEdward Tomasz Napierala } 3758*1a13f2e6SEdward Tomasz Napierala TPL_LIST_RUNLOCK(); 3759*1a13f2e6SEdward Tomasz Napierala /* Tell subsystem that we're done with its rates list. */ 3760*1a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx); 3761*1a13f2e6SEdward Tomasz Napierala if (err) 3762*1a13f2e6SEdward Tomasz Napierala goto done; 3763*1a13f2e6SEdward Tomasz Napierala 3764*1a13f2e6SEdward Tomasz Napierala err = sbuf_finish(s); 3765*1a13f2e6SEdward Tomasz Napierala if (err) 3766*1a13f2e6SEdward Tomasz Napierala goto done; /* We lost a race for buf to be too small. */ 3767*1a13f2e6SEdward Tomasz Napierala 3768*1a13f2e6SEdward Tomasz Napierala /* Return the rendered string data via oldptr. */ 3769*1a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s)); 3770*1a13f2e6SEdward Tomasz Napierala } else { 3771*1a13f2e6SEdward Tomasz Napierala /* Return the upper bound size for buffer sizing requests. */ 3772*1a13f2e6SEdward Tomasz Napierala err = SYSCTL_OUT(req, NULL, len); 3773*1a13f2e6SEdward Tomasz Napierala } 3774*1a13f2e6SEdward Tomasz Napierala 3775*1a13f2e6SEdward Tomasz Napierala process_new: 3776*1a13f2e6SEdward Tomasz Napierala if (err || req->newptr == NULL) 3777*1a13f2e6SEdward Tomasz Napierala goto done; 3778*1a13f2e6SEdward Tomasz Napierala 3779*1a13f2e6SEdward Tomasz Napierala newlen = req->newlen - req->newidx; 3780*1a13f2e6SEdward Tomasz Napierala err = SYSCTL_IN(req, buf, newlen); 3781*1a13f2e6SEdward Tomasz Napierala if (err) 3782*1a13f2e6SEdward Tomasz Napierala goto done; 3783*1a13f2e6SEdward Tomasz Napierala 3784*1a13f2e6SEdward Tomasz Napierala /* 3785*1a13f2e6SEdward Tomasz Napierala * Initialise format strings at run time. 3786*1a13f2e6SEdward Tomasz Napierala * 3787*1a13f2e6SEdward Tomasz Napierala * Write the max template spec string length into the 3788*1a13f2e6SEdward Tomasz Napierala * template_spec=percent key-value pair parsing format string as: 3789*1a13f2e6SEdward Tomasz Napierala * " %<width>[^=]=%u %n" 3790*1a13f2e6SEdward Tomasz Napierala * 3791*1a13f2e6SEdward Tomasz Napierala * Write the max template name string length into the tplname:tplhash 3792*1a13f2e6SEdward Tomasz Napierala * parsing format string as: 3793*1a13f2e6SEdward Tomasz Napierala * "%<width>[^:]:%u" 3794*1a13f2e6SEdward Tomasz Napierala * 3795*1a13f2e6SEdward Tomasz Napierala * Subtract 1 for \0 appended by sscanf(). 3796*1a13f2e6SEdward Tomasz Napierala */ 3797*1a13f2e6SEdward Tomasz Napierala sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1); 3798*1a13f2e6SEdward Tomasz Napierala sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1); 3799*1a13f2e6SEdward Tomasz Napierala 3800*1a13f2e6SEdward Tomasz Napierala /* 3801*1a13f2e6SEdward Tomasz Napierala * Parse each CSV key-value pair specifying a template and its sample 3802*1a13f2e6SEdward Tomasz Napierala * percentage. Whitespace either side of a key-value pair is ignored. 3803*1a13f2e6SEdward Tomasz Napierala * Templates can be specified by name, hash, or name and hash per the 3804*1a13f2e6SEdward Tomasz Napierala * following formats (chars in [] are optional): 3805*1a13f2e6SEdward Tomasz Napierala * ["]<tplname>["]=<percent> 3806*1a13f2e6SEdward Tomasz Napierala * :hash=pct 3807*1a13f2e6SEdward Tomasz Napierala * ["]<tplname>["]:hash=<percent> 3808*1a13f2e6SEdward Tomasz Napierala */ 3809*1a13f2e6SEdward Tomasz Napierala cum_pct = nrates = 0; 3810*1a13f2e6SEdward Tomasz Napierala rates = NULL; 3811*1a13f2e6SEdward Tomasz Napierala buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */ 3812*1a13f2e6SEdward Tomasz Napierala new_rates_usr_str = buf; 3813*1a13f2e6SEdward Tomasz Napierala while (isspace(*new_rates_usr_str)) 3814*1a13f2e6SEdward Tomasz Napierala new_rates_usr_str++; /* Skip leading whitespace. */ 3815*1a13f2e6SEdward Tomasz Napierala while (*new_rates_usr_str != '\0') { 3816*1a13f2e6SEdward Tomasz Napierala tpl_name_p = tpl_name; 3817*1a13f2e6SEdward Tomasz Napierala tpl_name[0] = '\0'; 3818*1a13f2e6SEdward Tomasz Napierala tpl_hash = 0; 3819*1a13f2e6SEdward Tomasz Napierala off = 0; 3820*1a13f2e6SEdward Tomasz Napierala 3821*1a13f2e6SEdward Tomasz Napierala /* 3822*1a13f2e6SEdward Tomasz Napierala * Parse key-value pair which must perform 2 conversions, then 3823*1a13f2e6SEdward Tomasz Napierala * parse the template spec to extract either name, hash, or name 3824*1a13f2e6SEdward Tomasz Napierala * and hash depending on the three possible spec formats. The 3825*1a13f2e6SEdward Tomasz Napierala * tplspec_fmt format specifier parses name or name and hash 3826*1a13f2e6SEdward Tomasz Napierala * template specs, while the ":%u" format specifier parses 3827*1a13f2e6SEdward Tomasz Napierala * hash-only template specs. If parsing is successfull, ensure 3828*1a13f2e6SEdward Tomasz Napierala * the cumulative sampling percentage does not exceed 100. 3829*1a13f2e6SEdward Tomasz Napierala */ 3830*1a13f2e6SEdward Tomasz Napierala err = EINVAL; 3831*1a13f2e6SEdward Tomasz Napierala if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct, 3832*1a13f2e6SEdward Tomasz Napierala &off)) 3833*1a13f2e6SEdward Tomasz Napierala break; 3834*1a13f2e6SEdward Tomasz Napierala if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) && 3835*1a13f2e6SEdward Tomasz Napierala (1 != sscanf(tpl_spec, ":%u", &tpl_hash))) 3836*1a13f2e6SEdward Tomasz Napierala break; 3837*1a13f2e6SEdward Tomasz Napierala if ((cum_pct += pct) > 100) 3838*1a13f2e6SEdward Tomasz Napierala break; 3839*1a13f2e6SEdward Tomasz Napierala err = 0; 3840*1a13f2e6SEdward Tomasz Napierala 3841*1a13f2e6SEdward Tomasz Napierala /* Strip surrounding "" from template name if present. */ 3842*1a13f2e6SEdward Tomasz Napierala len = strlen(tpl_name); 3843*1a13f2e6SEdward Tomasz Napierala if (len > 0) { 3844*1a13f2e6SEdward Tomasz Napierala if (tpl_name[len - 1] == '"') 3845*1a13f2e6SEdward Tomasz Napierala tpl_name[--len] = '\0'; 3846*1a13f2e6SEdward Tomasz Napierala if (tpl_name[0] == '"') { 3847*1a13f2e6SEdward Tomasz Napierala tpl_name_p++; 3848*1a13f2e6SEdward Tomasz Napierala len--; 3849*1a13f2e6SEdward Tomasz Napierala } 3850*1a13f2e6SEdward Tomasz Napierala } 3851*1a13f2e6SEdward Tomasz Napierala 3852*1a13f2e6SEdward Tomasz Napierala rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */ 3853*1a13f2e6SEdward Tomasz Napierala (nrates + 1) * sizeof(*rates), M_WAITOK); 3854*1a13f2e6SEdward Tomasz Napierala rates[nrates].tpl_slot_id = 3855*1a13f2e6SEdward Tomasz Napierala stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash); 3856*1a13f2e6SEdward Tomasz Napierala if (rates[nrates].tpl_slot_id < 0) { 3857*1a13f2e6SEdward Tomasz Napierala err = -rates[nrates].tpl_slot_id; 3858*1a13f2e6SEdward Tomasz Napierala break; 3859*1a13f2e6SEdward Tomasz Napierala } 3860*1a13f2e6SEdward Tomasz Napierala rates[nrates].tpl_sample_pct = pct; 3861*1a13f2e6SEdward Tomasz Napierala nrates++; 3862*1a13f2e6SEdward Tomasz Napierala new_rates_usr_str += off; 3863*1a13f2e6SEdward Tomasz Napierala if (*new_rates_usr_str != ',') 3864*1a13f2e6SEdward Tomasz Napierala break; /* End-of-input or malformed. */ 3865*1a13f2e6SEdward Tomasz Napierala new_rates_usr_str++; /* Move past comma to next pair. */ 3866*1a13f2e6SEdward Tomasz Napierala } 3867*1a13f2e6SEdward Tomasz Napierala 3868*1a13f2e6SEdward Tomasz Napierala if (!err) { 3869*1a13f2e6SEdward Tomasz Napierala if ((new_rates_usr_str - buf) < newlen) { 3870*1a13f2e6SEdward Tomasz Napierala /* Entire input has not been consumed. */ 3871*1a13f2e6SEdward Tomasz Napierala err = EINVAL; 3872*1a13f2e6SEdward Tomasz Napierala } else { 3873*1a13f2e6SEdward Tomasz Napierala /* 3874*1a13f2e6SEdward Tomasz Napierala * Give subsystem the new rates. They'll return the 3875*1a13f2e6SEdward Tomasz Napierala * appropriate rates pointer for us to garbage collect. 3876*1a13f2e6SEdward Tomasz Napierala */ 3877*1a13f2e6SEdward Tomasz Napierala err = subsys_cb(TPL_SR_PUT, &rates, &nrates, 3878*1a13f2e6SEdward Tomasz Napierala subsys_ctx); 3879*1a13f2e6SEdward Tomasz Napierala } 3880*1a13f2e6SEdward Tomasz Napierala } 3881*1a13f2e6SEdward Tomasz Napierala stats_free(rates); 3882*1a13f2e6SEdward Tomasz Napierala 3883*1a13f2e6SEdward Tomasz Napierala done: 3884*1a13f2e6SEdward Tomasz Napierala free(buf, M_TEMP); 3885*1a13f2e6SEdward Tomasz Napierala free(subsys_ctx, M_TEMP); 3886*1a13f2e6SEdward Tomasz Napierala return (err); 3887*1a13f2e6SEdward Tomasz Napierala } 3888*1a13f2e6SEdward Tomasz Napierala 3889*1a13f2e6SEdward Tomasz Napierala SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW, NULL, 3890*1a13f2e6SEdward Tomasz Napierala "stats(9) MIB"); 3891*1a13f2e6SEdward Tomasz Napierala 3892*1a13f2e6SEdward Tomasz Napierala SYSCTL_PROC(_kern_stats, OID_AUTO, templates, CTLTYPE_STRING|CTLFLAG_RD, 3893*1a13f2e6SEdward Tomasz Napierala NULL, 0, stats_tpl_list_available, "A", 3894*1a13f2e6SEdward Tomasz Napierala "list the name/hash of all available stats(9) templates"); 3895*1a13f2e6SEdward Tomasz Napierala 3896*1a13f2e6SEdward Tomasz Napierala #else /* ! _KERNEL */ 3897*1a13f2e6SEdward Tomasz Napierala 3898*1a13f2e6SEdward Tomasz Napierala static void __attribute__ ((constructor)) 3899*1a13f2e6SEdward Tomasz Napierala stats_constructor(void) 3900*1a13f2e6SEdward Tomasz Napierala { 3901*1a13f2e6SEdward Tomasz Napierala 3902*1a13f2e6SEdward Tomasz Napierala pthread_rwlock_init(&tpllistlock, NULL); 3903*1a13f2e6SEdward Tomasz Napierala } 3904*1a13f2e6SEdward Tomasz Napierala 3905*1a13f2e6SEdward Tomasz Napierala static void __attribute__ ((destructor)) 3906*1a13f2e6SEdward Tomasz Napierala stats_destructor(void) 3907*1a13f2e6SEdward Tomasz Napierala { 3908*1a13f2e6SEdward Tomasz Napierala 3909*1a13f2e6SEdward Tomasz Napierala pthread_rwlock_destroy(&tpllistlock); 3910*1a13f2e6SEdward Tomasz Napierala } 3911*1a13f2e6SEdward Tomasz Napierala 3912*1a13f2e6SEdward Tomasz Napierala #endif /* _KERNEL */ 3913