1 /*- 2 * Copyright (c) 2014-2018 Netflix, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * Author: Lawrence Stewart <lstewart@netflix.com> 29 */ 30 31 #include <sys/cdefs.h> 32 #include <sys/param.h> 33 #include <sys/arb.h> 34 #include <sys/ctype.h> 35 #include <sys/errno.h> 36 #include <sys/hash.h> 37 #include <sys/limits.h> 38 #include <sys/malloc.h> 39 #include <sys/qmath.h> 40 #include <sys/sbuf.h> 41 #if defined(DIAGNOSTIC) 42 #include <sys/tree.h> 43 #endif 44 #include <sys/stats.h> /* Must come after qmath.h and arb.h */ 45 #include <sys/stddef.h> 46 #include <sys/stdint.h> 47 #include <sys/time.h> 48 49 #ifdef _KERNEL 50 #include <sys/kernel.h> 51 #include <sys/lock.h> 52 #include <sys/rwlock.h> 53 #include <sys/sysctl.h> 54 #include <sys/systm.h> 55 #else /* ! _KERNEL */ 56 #include <pthread.h> 57 #include <stdbool.h> 58 #include <stdio.h> 59 #include <stdlib.h> 60 #include <string.h> 61 #endif /* _KERNEL */ 62 63 struct voistatdata_voistate { 64 /* Previous VOI value for diff calculation. */ 65 struct voistatdata_numeric prev; 66 }; 67 68 #define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */ 69 struct voistat { 70 int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */ 71 enum vsd_dtype dtype : 8; /* Data type of this stat's data. */ 72 uint16_t data_off; /* Blob offset for this stat's data. */ 73 uint16_t dsz; /* Size of stat's data. */ 74 #define VS_EBITS 8 75 uint16_t errs : VS_EBITS;/* Non-wrapping error count. */ 76 uint16_t flags : 16 - VS_EBITS; 77 }; 78 /* The voistat error count is capped to avoid wrapping. */ 79 #define VS_INCERRS(vs) do { \ 80 if ((vs)->errs < (1U << VS_EBITS) - 1) \ 81 (vs)->errs++; \ 82 } while (0) 83 84 /* 85 * Ideas for flags: 86 * - Global or entity specific (global would imply use of counter(9)?) 87 * - Whether to reset stats on read or not 88 * - Signal an overflow? 89 * - Compressed voistat array 90 */ 91 #define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */ 92 struct voi { 93 int16_t id; /* VOI id. */ 94 enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */ 95 int8_t voistatmaxid; /* Largest allocated voistat index. */ 96 uint16_t stats_off; /* Blob offset for this VOIs stats. */ 97 uint16_t flags; 98 }; 99 100 /* 101 * Memory for the entire blob is allocated as a slab and then offsets are 102 * maintained to carve up the slab into sections holding different data types. 103 * 104 * Ideas for flags: 105 * - Compressed voi array (trade off memory usage vs search time) 106 * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib) 107 */ 108 struct statsblobv1 { 109 uint8_t abi; 110 uint8_t endian; 111 uint16_t flags; 112 uint16_t maxsz; 113 uint16_t cursz; 114 /* Fields from here down are opaque to consumers. */ 115 uint32_t tplhash; /* Base template hash ID. */ 116 uint16_t stats_off; /* voistat array blob offset. */ 117 uint16_t statsdata_off; /* voistatdata array blob offset. */ 118 sbintime_t created; /* Blob creation time. */ 119 sbintime_t lastrst; /* Time of last reset. */ 120 struct voi vois[]; /* Array indexed by [voi_id]. */ 121 } __aligned(sizeof(void *)); 122 _Static_assert(offsetof(struct statsblobv1, cursz) + 123 SIZEOF_MEMBER(struct statsblobv1, cursz) == 124 offsetof(struct statsblob, opaque), 125 "statsblobv1 ABI mismatch"); 126 127 struct statsblobv1_tpl { 128 struct metablob *mb; 129 struct statsblobv1 *sb; 130 }; 131 132 /* Context passed to iterator callbacks. */ 133 struct sb_iter_ctx { 134 void *usrctx; /* Caller supplied context. */ 135 uint32_t flags; /* Flags for current iteration. */ 136 int16_t vslot; /* struct voi slot index. */ 137 int8_t vsslot; /* struct voistat slot index. */ 138 }; 139 140 struct sb_tostrcb_ctx { 141 struct sbuf *buf; 142 struct statsblob_tpl *tpl; 143 enum sb_str_fmt fmt; 144 uint32_t flags; 145 }; 146 147 struct sb_visitcb_ctx { 148 stats_blob_visitcb_t cb; 149 void *usrctx; 150 }; 151 152 /* Stats blob iterator callback. */ 153 typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v, 154 struct voistat *vs, struct sb_iter_ctx *ctx); 155 156 #ifdef _KERNEL 157 static struct rwlock tpllistlock; 158 RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock"); 159 #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock) 160 #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock) 161 #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock) 162 #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock) 163 #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED) 164 #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED) 165 #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED) 166 MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory"); 167 #define stats_free(ptr) free((ptr), M_STATS) 168 #else /* ! _KERNEL */ 169 static void stats_constructor(void); 170 static void stats_destructor(void); 171 static pthread_rwlock_t tpllistlock; 172 #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock) 173 #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock) 174 #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK() 175 #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock) 176 #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK() 177 #define TPL_LIST_LOCK_ASSERT() do { } while (0) 178 #define TPL_LIST_RLOCK_ASSERT() do { } while (0) 179 #define TPL_LIST_WLOCK_ASSERT() do { } while (0) 180 #ifdef NDEBUG 181 #define KASSERT(cond, msg) do {} while (0) 182 #define stats_abort() do {} while (0) 183 #else /* ! NDEBUG */ 184 #define KASSERT(cond, msg) do { \ 185 if (!(cond)) { \ 186 panic msg; \ 187 } \ 188 } while (0) 189 #define stats_abort() abort() 190 #endif /* NDEBUG */ 191 #define stats_free(ptr) free(ptr) 192 #define panic(fmt, ...) do { \ 193 fprintf(stderr, (fmt), ##__VA_ARGS__); \ 194 stats_abort(); \ 195 } while (0) 196 #endif /* _KERNEL */ 197 198 #define SB_V1_MAXSZ 65535 199 200 /* Obtain a blob offset pointer. */ 201 #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off))) 202 203 /* 204 * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a 205 * power of 2 size, we can shift instead of divide. The shift amount must be 206 * updated if sizeof(struct voi) ever changes, which the assert should catch. 207 */ 208 #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \ 209 sizeof(struct statsblobv1)) >> 3)) 210 _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch"); 211 212 /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */ 213 const char *vs_stype2name[VS_NUM_STYPES] = { 214 [VS_STYPE_VOISTATE] = "VOISTATE", 215 [VS_STYPE_SUM] = "SUM", 216 [VS_STYPE_MAX] = "MAX", 217 [VS_STYPE_MIN] = "MIN", 218 [VS_STYPE_HIST] = "HIST", 219 [VS_STYPE_TDGST] = "TDGST", 220 }; 221 222 const char *vs_stype2desc[VS_NUM_STYPES] = { 223 [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)", 224 [VS_STYPE_SUM] = "Simple arithmetic accumulator", 225 [VS_STYPE_MAX] = "Maximum observed VOI value", 226 [VS_STYPE_MIN] = "Minimum observed VOI value", 227 [VS_STYPE_HIST] = "Histogram of observed VOI values", 228 [VS_STYPE_TDGST] = "t-digest of observed VOI values", 229 }; 230 231 const char *vsd_dtype2name[VSD_NUM_DTYPES] = { 232 [VSD_DTYPE_VOISTATE] = "VOISTATE", 233 [VSD_DTYPE_INT_S32] = "INT_S32", 234 [VSD_DTYPE_INT_U32] = "INT_U32", 235 [VSD_DTYPE_INT_S64] = "INT_S64", 236 [VSD_DTYPE_INT_U64] = "INT_U64", 237 [VSD_DTYPE_INT_SLONG] = "INT_SLONG", 238 [VSD_DTYPE_INT_ULONG] = "INT_ULONG", 239 [VSD_DTYPE_Q_S32] = "Q_S32", 240 [VSD_DTYPE_Q_U32] = "Q_U32", 241 [VSD_DTYPE_Q_S64] = "Q_S64", 242 [VSD_DTYPE_Q_U64] = "Q_U64", 243 [VSD_DTYPE_CRHIST32] = "CRHIST32", 244 [VSD_DTYPE_DRHIST32] = "DRHIST32", 245 [VSD_DTYPE_DVHIST32] = "DVHIST32", 246 [VSD_DTYPE_CRHIST64] = "CRHIST64", 247 [VSD_DTYPE_DRHIST64] = "DRHIST64", 248 [VSD_DTYPE_DVHIST64] = "DVHIST64", 249 [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32", 250 [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64", 251 }; 252 253 const size_t vsd_dtype2size[VSD_NUM_DTYPES] = { 254 [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate), 255 [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32), 256 [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32), 257 [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64), 258 [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64), 259 [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong), 260 [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong), 261 [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32), 262 [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32), 263 [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64), 264 [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64), 265 [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32), 266 [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32), 267 [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32), 268 [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64), 269 [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64), 270 [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64), 271 [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32), 272 [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64), 273 }; 274 275 static const bool vsd_compoundtype[VSD_NUM_DTYPES] = { 276 [VSD_DTYPE_VOISTATE] = true, 277 [VSD_DTYPE_INT_S32] = false, 278 [VSD_DTYPE_INT_U32] = false, 279 [VSD_DTYPE_INT_S64] = false, 280 [VSD_DTYPE_INT_U64] = false, 281 [VSD_DTYPE_INT_SLONG] = false, 282 [VSD_DTYPE_INT_ULONG] = false, 283 [VSD_DTYPE_Q_S32] = false, 284 [VSD_DTYPE_Q_U32] = false, 285 [VSD_DTYPE_Q_S64] = false, 286 [VSD_DTYPE_Q_U64] = false, 287 [VSD_DTYPE_CRHIST32] = true, 288 [VSD_DTYPE_DRHIST32] = true, 289 [VSD_DTYPE_DVHIST32] = true, 290 [VSD_DTYPE_CRHIST64] = true, 291 [VSD_DTYPE_DRHIST64] = true, 292 [VSD_DTYPE_DVHIST64] = true, 293 [VSD_DTYPE_TDGSTCLUST32] = true, 294 [VSD_DTYPE_TDGSTCLUST64] = true, 295 }; 296 297 const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = { 298 [LIM_MIN] = { 299 [VSD_DTYPE_VOISTATE] = {0}, 300 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}}, 301 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}}, 302 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}}, 303 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}}, 304 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}}, 305 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}}, 306 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}}, 307 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}}, 308 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}}, 309 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}}, 310 }, 311 [LIM_MAX] = { 312 [VSD_DTYPE_VOISTATE] = {0}, 313 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}}, 314 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}}, 315 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}}, 316 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}}, 317 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}}, 318 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}}, 319 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}}, 320 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}}, 321 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}}, 322 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}}, 323 } 324 }; 325 326 /* tpllistlock protects tpllist and ntpl */ 327 static uint32_t ntpl; 328 static struct statsblob_tpl **tpllist; 329 330 static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz, 331 int flags); 332 //static void stats_v1_blob_finalise(struct statsblobv1 *sb); 333 static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 334 uint32_t flags); 335 static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 336 int newvoistatbytes, int newvoistatdatabytes); 337 static void stats_v1_blob_iter(struct statsblobv1 *sb, 338 stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags); 339 static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, 340 struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt); 341 342 static inline int 343 ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2) 344 { 345 346 KASSERT(Q_PRECEQ(c1->mu, c2->mu), 347 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 348 Q_RELPREC(c1->mu, c2->mu))); 349 350 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 351 } 352 ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp); 353 354 static inline int 355 ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2) 356 { 357 358 KASSERT(Q_PRECEQ(c1->mu, c2->mu), 359 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 360 Q_RELPREC(c1->mu, c2->mu))); 361 362 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 363 } 364 ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp); 365 366 #ifdef DIAGNOSTIC 367 RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp); 368 RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp); 369 #endif 370 371 static inline sbintime_t 372 stats_sbinuptime(void) 373 { 374 sbintime_t sbt; 375 #ifdef _KERNEL 376 377 sbt = sbinuptime(); 378 #else /* ! _KERNEL */ 379 struct timespec tp; 380 381 clock_gettime(CLOCK_MONOTONIC_FAST, &tp); 382 sbt = tstosbt(tp); 383 #endif /* _KERNEL */ 384 385 return (sbt); 386 } 387 388 static inline void * 389 stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags) 390 { 391 392 #ifdef _KERNEL 393 /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */ 394 if (!(flags & (M_WAITOK | M_NOWAIT))) 395 flags |= M_NOWAIT; 396 ptr = realloc(ptr, newsz, M_STATS, flags); 397 #else /* ! _KERNEL */ 398 ptr = realloc(ptr, newsz); 399 if ((flags & M_ZERO) && ptr != NULL) { 400 if (oldsz == 0) 401 memset(ptr, '\0', newsz); 402 else if (newsz > oldsz) 403 memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz); 404 } 405 #endif /* _KERNEL */ 406 407 return (ptr); 408 } 409 410 static inline char * 411 stats_strdup(const char *s, 412 #ifdef _KERNEL 413 int flags) 414 { 415 char *copy; 416 size_t len; 417 418 if (!(flags & (M_WAITOK | M_NOWAIT))) 419 flags |= M_NOWAIT; 420 421 len = strlen(s) + 1; 422 if ((copy = malloc(len, M_STATS, flags)) != NULL) 423 bcopy(s, copy, len); 424 425 return (copy); 426 #else 427 int flags __unused) 428 { 429 return (strdup(s)); 430 #endif 431 } 432 433 static inline void 434 stats_tpl_update_hash(struct statsblob_tpl *tpl) 435 { 436 437 TPL_LIST_WLOCK_ASSERT(); 438 tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0); 439 for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) { 440 if (tpl->mb->voi_meta[voi_id].name != NULL) 441 tpl->mb->tplhash = hash32_str( 442 tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash); 443 } 444 tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz, 445 tpl->mb->tplhash); 446 } 447 448 static inline uint64_t 449 stats_pow_u64(uint64_t base, uint64_t exp) 450 { 451 uint64_t result = 1; 452 453 while (exp) { 454 if (exp & 1) 455 result *= base; 456 exp >>= 1; 457 base *= base; 458 } 459 460 return (result); 461 } 462 463 static inline int 464 stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt, 465 struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub) 466 { 467 uint64_t step = 0; 468 int error = 0; 469 470 switch (info->scheme) { 471 case BKT_LIN: 472 step = info->lin.stepinc; 473 break; 474 case BKT_EXP: 475 step = stats_pow_u64(info->exp.stepbase, 476 info->exp.stepexp + curbkt); 477 break; 478 case BKT_LINEXP: 479 { 480 uint64_t curstepexp = 1; 481 482 switch (info->voi_dtype) { 483 case VSD_DTYPE_INT_S32: 484 while ((int32_t)stats_pow_u64(info->linexp.stepbase, 485 curstepexp) <= bkt_lb->int32.s32) 486 curstepexp++; 487 break; 488 case VSD_DTYPE_INT_U32: 489 while ((uint32_t)stats_pow_u64(info->linexp.stepbase, 490 curstepexp) <= bkt_lb->int32.u32) 491 curstepexp++; 492 break; 493 case VSD_DTYPE_INT_S64: 494 while ((int64_t)stats_pow_u64(info->linexp.stepbase, 495 curstepexp) <= bkt_lb->int64.s64) 496 curstepexp++; 497 break; 498 case VSD_DTYPE_INT_U64: 499 while ((uint64_t)stats_pow_u64(info->linexp.stepbase, 500 curstepexp) <= bkt_lb->int64.u64) 501 curstepexp++; 502 break; 503 case VSD_DTYPE_INT_SLONG: 504 while ((long)stats_pow_u64(info->linexp.stepbase, 505 curstepexp) <= bkt_lb->intlong.slong) 506 curstepexp++; 507 break; 508 case VSD_DTYPE_INT_ULONG: 509 while ((unsigned long)stats_pow_u64(info->linexp.stepbase, 510 curstepexp) <= bkt_lb->intlong.ulong) 511 curstepexp++; 512 break; 513 case VSD_DTYPE_Q_S32: 514 while ((s32q_t)stats_pow_u64(info->linexp.stepbase, 515 curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32)) 516 break; 517 case VSD_DTYPE_Q_U32: 518 while ((u32q_t)stats_pow_u64(info->linexp.stepbase, 519 curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32)) 520 break; 521 case VSD_DTYPE_Q_S64: 522 while ((s64q_t)stats_pow_u64(info->linexp.stepbase, 523 curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64)) 524 curstepexp++; 525 break; 526 case VSD_DTYPE_Q_U64: 527 while ((u64q_t)stats_pow_u64(info->linexp.stepbase, 528 curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64)) 529 curstepexp++; 530 break; 531 default: 532 break; 533 } 534 535 step = stats_pow_u64(info->linexp.stepbase, curstepexp) / 536 info->linexp.linstepdiv; 537 if (step == 0) 538 step = 1; 539 break; 540 } 541 default: 542 break; 543 } 544 545 if (info->scheme == BKT_USR) { 546 *bkt_lb = info->usr.bkts[curbkt].lb; 547 *bkt_ub = info->usr.bkts[curbkt].ub; 548 } else if (step != 0) { 549 switch (info->voi_dtype) { 550 case VSD_DTYPE_INT_S32: 551 bkt_ub->int32.s32 += (int32_t)step; 552 break; 553 case VSD_DTYPE_INT_U32: 554 bkt_ub->int32.u32 += (uint32_t)step; 555 break; 556 case VSD_DTYPE_INT_S64: 557 bkt_ub->int64.s64 += (int64_t)step; 558 break; 559 case VSD_DTYPE_INT_U64: 560 bkt_ub->int64.u64 += (uint64_t)step; 561 break; 562 case VSD_DTYPE_INT_SLONG: 563 bkt_ub->intlong.slong += (long)step; 564 break; 565 case VSD_DTYPE_INT_ULONG: 566 bkt_ub->intlong.ulong += (unsigned long)step; 567 break; 568 case VSD_DTYPE_Q_S32: 569 error = Q_QADDI(&bkt_ub->q32.sq32, step); 570 break; 571 case VSD_DTYPE_Q_U32: 572 error = Q_QADDI(&bkt_ub->q32.uq32, step); 573 break; 574 case VSD_DTYPE_Q_S64: 575 error = Q_QADDI(&bkt_ub->q64.sq64, step); 576 break; 577 case VSD_DTYPE_Q_U64: 578 error = Q_QADDI(&bkt_ub->q64.uq64, step); 579 break; 580 default: 581 break; 582 } 583 } else { /* info->scheme != BKT_USR && step == 0 */ 584 return (EINVAL); 585 } 586 587 return (error); 588 } 589 590 static uint32_t 591 stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info) 592 { 593 struct voistatdata_numeric bkt_lb, bkt_ub; 594 uint32_t nbkts; 595 int done; 596 597 if (info->scheme == BKT_USR) { 598 /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */ 599 info->lb = info->usr.bkts[0].lb; 600 info->ub = info->usr.bkts[info->usr.nbkts - 1].lb; 601 } 602 603 nbkts = 0; 604 done = 0; 605 bkt_ub = info->lb; 606 607 do { 608 bkt_lb = bkt_ub; 609 if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub)) 610 return (0); 611 612 if (info->scheme == BKT_USR) 613 done = (nbkts == info->usr.nbkts); 614 else { 615 switch (info->voi_dtype) { 616 case VSD_DTYPE_INT_S32: 617 done = (bkt_ub.int32.s32 > info->ub.int32.s32); 618 break; 619 case VSD_DTYPE_INT_U32: 620 done = (bkt_ub.int32.u32 > info->ub.int32.u32); 621 break; 622 case VSD_DTYPE_INT_S64: 623 done = (bkt_ub.int64.s64 > info->ub.int64.s64); 624 break; 625 case VSD_DTYPE_INT_U64: 626 done = (bkt_ub.int64.u64 > info->ub.int64.u64); 627 break; 628 case VSD_DTYPE_INT_SLONG: 629 done = (bkt_ub.intlong.slong > 630 info->ub.intlong.slong); 631 break; 632 case VSD_DTYPE_INT_ULONG: 633 done = (bkt_ub.intlong.ulong > 634 info->ub.intlong.ulong); 635 break; 636 case VSD_DTYPE_Q_S32: 637 done = Q_QGTQ(bkt_ub.q32.sq32, 638 info->ub.q32.sq32); 639 break; 640 case VSD_DTYPE_Q_U32: 641 done = Q_QGTQ(bkt_ub.q32.uq32, 642 info->ub.q32.uq32); 643 break; 644 case VSD_DTYPE_Q_S64: 645 done = Q_QGTQ(bkt_ub.q64.sq64, 646 info->ub.q64.sq64); 647 break; 648 case VSD_DTYPE_Q_U64: 649 done = Q_QGTQ(bkt_ub.q64.uq64, 650 info->ub.q64.uq64); 651 break; 652 default: 653 return (0); 654 } 655 } 656 } while (!done); 657 658 if (info->flags & VSD_HIST_LBOUND_INF) 659 nbkts++; 660 if (info->flags & VSD_HIST_UBOUND_INF) 661 nbkts++; 662 663 return (nbkts); 664 } 665 666 int 667 stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 668 struct vss_hist_hlpr_info *info) 669 { 670 struct voistatdata_hist *hist; 671 struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub, 672 *ubinfbktlb, *ubinfbktub; 673 uint32_t bkt, nbkts, nloop; 674 675 if (vss == NULL || info == NULL || (info->flags & 676 (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype == 677 VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64))) 678 return (EINVAL); 679 680 info->voi_dtype = voi_dtype; 681 682 if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0) 683 return (EINVAL); 684 685 switch (info->hist_dtype) { 686 case VSD_DTYPE_CRHIST32: 687 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts); 688 break; 689 case VSD_DTYPE_DRHIST32: 690 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts); 691 break; 692 case VSD_DTYPE_DVHIST32: 693 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts); 694 break; 695 case VSD_DTYPE_CRHIST64: 696 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts); 697 break; 698 case VSD_DTYPE_DRHIST64: 699 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts); 700 break; 701 case VSD_DTYPE_DVHIST64: 702 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts); 703 break; 704 default: 705 return (EINVAL); 706 } 707 708 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 709 if (vss->iv == NULL) 710 return (ENOMEM); 711 712 hist = (struct voistatdata_hist *)vss->iv; 713 bkt_ub = info->lb; 714 715 for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0; 716 bkt < nbkts; 717 bkt++, nloop++) { 718 bkt_lb = bkt_ub; 719 if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub)) 720 return (EINVAL); 721 722 switch (info->hist_dtype) { 723 case VSD_DTYPE_CRHIST32: 724 VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb; 725 break; 726 case VSD_DTYPE_DRHIST32: 727 VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb; 728 VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub; 729 break; 730 case VSD_DTYPE_DVHIST32: 731 VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb; 732 break; 733 case VSD_DTYPE_CRHIST64: 734 VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb; 735 break; 736 case VSD_DTYPE_DRHIST64: 737 VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb; 738 VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub; 739 break; 740 case VSD_DTYPE_DVHIST64: 741 VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb; 742 break; 743 default: 744 return (EINVAL); 745 } 746 } 747 748 lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL; 749 750 switch (info->hist_dtype) { 751 case VSD_DTYPE_CRHIST32: 752 lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb; 753 ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb; 754 break; 755 case VSD_DTYPE_DRHIST32: 756 lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb; 757 lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub; 758 ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb; 759 ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub; 760 break; 761 case VSD_DTYPE_CRHIST64: 762 lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb; 763 ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb; 764 break; 765 case VSD_DTYPE_DRHIST64: 766 lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb; 767 lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub; 768 ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb; 769 ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub; 770 break; 771 case VSD_DTYPE_DVHIST32: 772 case VSD_DTYPE_DVHIST64: 773 break; 774 default: 775 return (EINVAL); 776 } 777 778 if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) { 779 *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype]; 780 /* 781 * Assignment from numeric_limit array for Q types assigns max 782 * possible integral/fractional value for underlying data type, 783 * but we must set control bits for this specific histogram per 784 * the user's choice of fractional bits, which we extract from 785 * info->lb. 786 */ 787 if (info->voi_dtype == VSD_DTYPE_Q_S32 || 788 info->voi_dtype == VSD_DTYPE_Q_U32) { 789 /* Signedness doesn't matter for setting control bits. */ 790 Q_SCVAL(lbinfbktlb->q32.sq32, 791 Q_GCVAL(info->lb.q32.sq32)); 792 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 793 info->voi_dtype == VSD_DTYPE_Q_U64) { 794 /* Signedness doesn't matter for setting control bits. */ 795 Q_SCVAL(lbinfbktlb->q64.sq64, 796 Q_GCVAL(info->lb.q64.sq64)); 797 } 798 if (lbinfbktub) 799 *lbinfbktub = info->lb; 800 } 801 if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) { 802 *ubinfbktlb = bkt_lb; 803 if (ubinfbktub) { 804 *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype]; 805 if (info->voi_dtype == VSD_DTYPE_Q_S32 || 806 info->voi_dtype == VSD_DTYPE_Q_U32) { 807 Q_SCVAL(ubinfbktub->q32.sq32, 808 Q_GCVAL(info->lb.q32.sq32)); 809 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 810 info->voi_dtype == VSD_DTYPE_Q_U64) { 811 Q_SCVAL(ubinfbktub->q64.sq64, 812 Q_GCVAL(info->lb.q64.sq64)); 813 } 814 } 815 } 816 817 return (0); 818 } 819 820 int 821 stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 822 struct vss_tdgst_hlpr_info *info) 823 { 824 struct voistatdata_tdgst *tdgst; 825 struct ctdth32 *ctd32tree; 826 struct ctdth64 *ctd64tree; 827 struct voistatdata_tdgstctd32 *ctd32; 828 struct voistatdata_tdgstctd64 *ctd64; 829 830 info->voi_dtype = voi_dtype; 831 832 switch (info->tdgst_dtype) { 833 case VSD_DTYPE_TDGSTCLUST32: 834 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds); 835 break; 836 case VSD_DTYPE_TDGSTCLUST64: 837 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds); 838 break; 839 default: 840 return (EINVAL); 841 } 842 843 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 844 if (vss->iv == NULL) 845 return (ENOMEM); 846 847 tdgst = (struct voistatdata_tdgst *)vss->iv; 848 849 switch (info->tdgst_dtype) { 850 case VSD_DTYPE_TDGSTCLUST32: 851 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 852 ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) { 853 Q_INI(&ctd32->mu, 0, 0, info->prec); 854 } 855 break; 856 case VSD_DTYPE_TDGSTCLUST64: 857 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 858 ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) { 859 Q_INI(&ctd64->mu, 0, 0, info->prec); 860 } 861 break; 862 default: 863 return (EINVAL); 864 } 865 866 return (0); 867 } 868 869 int 870 stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 871 struct vss_numeric_hlpr_info *info) 872 { 873 struct voistatdata_numeric iv; 874 875 switch (vss->stype) { 876 case VS_STYPE_SUM: 877 iv = stats_ctor_vsd_numeric(0); 878 break; 879 case VS_STYPE_MIN: 880 iv = numeric_limits[LIM_MAX][voi_dtype]; 881 break; 882 case VS_STYPE_MAX: 883 iv = numeric_limits[LIM_MIN][voi_dtype]; 884 break; 885 default: 886 return (EINVAL); 887 } 888 889 vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0); 890 if (vss->iv == NULL) 891 return (ENOMEM); 892 893 vss->vs_dtype = voi_dtype; 894 vss->vsdsz = vsd_dtype2size[voi_dtype]; 895 switch (voi_dtype) { 896 case VSD_DTYPE_INT_S32: 897 *((int32_t *)vss->iv) = iv.int32.s32; 898 break; 899 case VSD_DTYPE_INT_U32: 900 *((uint32_t *)vss->iv) = iv.int32.u32; 901 break; 902 case VSD_DTYPE_INT_S64: 903 *((int64_t *)vss->iv) = iv.int64.s64; 904 break; 905 case VSD_DTYPE_INT_U64: 906 *((uint64_t *)vss->iv) = iv.int64.u64; 907 break; 908 case VSD_DTYPE_INT_SLONG: 909 *((long *)vss->iv) = iv.intlong.slong; 910 break; 911 case VSD_DTYPE_INT_ULONG: 912 *((unsigned long *)vss->iv) = iv.intlong.ulong; 913 break; 914 case VSD_DTYPE_Q_S32: 915 *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32, 916 Q_CTRLINI(info->prec)); 917 break; 918 case VSD_DTYPE_Q_U32: 919 *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32, 920 Q_CTRLINI(info->prec)); 921 break; 922 case VSD_DTYPE_Q_S64: 923 *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64, 924 Q_CTRLINI(info->prec)); 925 break; 926 case VSD_DTYPE_Q_U64: 927 *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64, 928 Q_CTRLINI(info->prec)); 929 break; 930 default: 931 break; 932 } 933 934 return (0); 935 } 936 937 int 938 stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss, 939 struct voistatspec *vss) 940 { 941 int i, ret; 942 943 for (i = nvss - 1; i >= 0; i--) { 944 if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i], 945 vss[i].hlprinfo)) != 0) 946 return (ret); 947 } 948 949 return (0); 950 } 951 952 void 953 stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss) 954 { 955 int i; 956 957 for (i = nvss - 1; i >= 0; i--) { 958 if (vss[i].hlpr) { 959 stats_free((void *)vss[i].iv); 960 vss[i].iv = NULL; 961 } 962 } 963 } 964 965 int 966 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl) 967 { 968 int error; 969 970 error = 0; 971 972 TPL_LIST_WLOCK(); 973 if (tpl_id < 0 || tpl_id >= (int)ntpl) { 974 error = ENOENT; 975 } else { 976 *tpl = tpllist[tpl_id]; 977 /* XXXLAS: Acquire refcount on tpl. */ 978 } 979 TPL_LIST_WUNLOCK(); 980 981 return (error); 982 } 983 984 int 985 stats_tpl_fetch_allocid(const char *name, uint32_t hash) 986 { 987 int i, tpl_id; 988 989 tpl_id = -ESRCH; 990 991 TPL_LIST_RLOCK(); 992 for (i = ntpl - 1; i >= 0; i--) { 993 if (name != NULL) { 994 if (strlen(name) == strlen(tpllist[i]->mb->tplname) && 995 strncmp(name, tpllist[i]->mb->tplname, 996 TPL_MAX_NAME_LEN) == 0 && (!hash || hash == 997 tpllist[i]->mb->tplhash)) { 998 tpl_id = i; 999 break; 1000 } 1001 } else if (hash == tpllist[i]->mb->tplhash) { 1002 tpl_id = i; 1003 break; 1004 } 1005 } 1006 TPL_LIST_RUNLOCK(); 1007 1008 return (tpl_id); 1009 } 1010 1011 int 1012 stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len) 1013 { 1014 int error; 1015 1016 error = 0; 1017 1018 TPL_LIST_RLOCK(); 1019 if (tpl_id < ntpl) { 1020 if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname)) 1021 strlcpy(buf, tpllist[tpl_id]->mb->tplname, len); 1022 else 1023 error = EOVERFLOW; 1024 } else 1025 error = ENOENT; 1026 TPL_LIST_RUNLOCK(); 1027 1028 return (error); 1029 } 1030 1031 int 1032 stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates, 1033 void *seed_bytes, size_t seed_len) 1034 { 1035 uint32_t cum_pct, rnd_pct; 1036 int i; 1037 1038 cum_pct = 0; 1039 1040 /* 1041 * Choose a pseudorandom or seeded number in range [0,100] and use 1042 * it to make a sampling decision and template selection where required. 1043 * If no seed is supplied, a PRNG is used to generate a pseudorandom 1044 * number so that every selection is independent. If a seed is supplied, 1045 * the caller desires random selection across different seeds, but 1046 * deterministic selection given the same seed. This is achieved by 1047 * hashing the seed and using the hash as the random number source. 1048 * 1049 * XXXLAS: Characterise hash function output distribution. 1050 */ 1051 if (seed_bytes == NULL) 1052 rnd_pct = random() / (INT32_MAX / 100); 1053 else 1054 rnd_pct = hash32_buf(seed_bytes, seed_len, 0) / 1055 (UINT32_MAX / 100U); 1056 1057 /* 1058 * We map the randomly selected percentage on to the interval [0,100] 1059 * consisting of the cumulatively summed template sampling percentages. 1060 * The difference between the cumulative sum of all template sampling 1061 * percentages and 100 is treated as a NULL assignment i.e. no stats 1062 * template will be assigned, and -1 returned instead. 1063 */ 1064 for (i = 0; i < nrates; i++) { 1065 cum_pct += rates[i].tpl_sample_pct; 1066 1067 KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__, 1068 cum_pct)); 1069 if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0) 1070 continue; 1071 1072 return (rates[i].tpl_slot_id); 1073 } 1074 1075 return (-1); 1076 } 1077 1078 int 1079 stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz, 1080 struct statsblobv1 *src, uint32_t flags) 1081 { 1082 int error; 1083 1084 error = 0; 1085 1086 if (src == NULL || dst == NULL || 1087 src->cursz < sizeof(struct statsblob) || 1088 ((flags & SB_CLONE_ALLOCDST) && 1089 (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) { 1090 error = EINVAL; 1091 } else if (flags & SB_CLONE_ALLOCDST) { 1092 *dst = stats_realloc(NULL, 0, src->cursz, 0); 1093 if (*dst) 1094 (*dst)->maxsz = dstmaxsz = src->cursz; 1095 else 1096 error = ENOMEM; 1097 } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) { 1098 error = EINVAL; 1099 } 1100 1101 if (!error) { 1102 size_t postcurszlen; 1103 1104 /* 1105 * Clone src into dst except for the maxsz field. If dst is too 1106 * small to hold all of src, only copy src's header and return 1107 * EOVERFLOW. 1108 */ 1109 #ifdef _KERNEL 1110 if (flags & SB_CLONE_USRDSTNOFAULT) 1111 copyout_nofault(src, *dst, 1112 offsetof(struct statsblob, maxsz)); 1113 else if (flags & SB_CLONE_USRDST) 1114 copyout(src, *dst, offsetof(struct statsblob, maxsz)); 1115 else 1116 #endif 1117 memcpy(*dst, src, offsetof(struct statsblob, maxsz)); 1118 1119 if (dstmaxsz >= src->cursz) { 1120 postcurszlen = src->cursz - 1121 offsetof(struct statsblob, cursz); 1122 } else { 1123 error = EOVERFLOW; 1124 postcurszlen = sizeof(struct statsblob) - 1125 offsetof(struct statsblob, cursz); 1126 } 1127 #ifdef _KERNEL 1128 if (flags & SB_CLONE_USRDSTNOFAULT) 1129 copyout_nofault(&(src->cursz), &((*dst)->cursz), 1130 postcurszlen); 1131 else if (flags & SB_CLONE_USRDST) 1132 copyout(&(src->cursz), &((*dst)->cursz), postcurszlen); 1133 else 1134 #endif 1135 memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen); 1136 } 1137 1138 return (error); 1139 } 1140 1141 int 1142 stats_v1_tpl_alloc(const char *name, uint32_t flags __unused) 1143 { 1144 struct statsblobv1_tpl *tpl, **newtpllist; 1145 struct statsblobv1 *tpl_sb; 1146 struct metablob *tpl_mb; 1147 int tpl_id; 1148 1149 if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN) 1150 return (-EINVAL); 1151 1152 if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0) 1153 return (-EEXIST); 1154 1155 tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO); 1156 tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO); 1157 tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO); 1158 1159 if (tpl_mb != NULL && name != NULL) 1160 tpl_mb->tplname = stats_strdup(name, 0); 1161 1162 if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL || 1163 tpl_mb->tplname == NULL) { 1164 stats_free(tpl); 1165 stats_free(tpl_sb); 1166 if (tpl_mb != NULL) { 1167 stats_free(tpl_mb->tplname); 1168 stats_free(tpl_mb); 1169 } 1170 return (-ENOMEM); 1171 } 1172 1173 tpl->mb = tpl_mb; 1174 tpl->sb = tpl_sb; 1175 1176 tpl_sb->abi = STATS_ABI_V1; 1177 tpl_sb->endian = 1178 #if BYTE_ORDER == LITTLE_ENDIAN 1179 SB_LE; 1180 #elif BYTE_ORDER == BIG_ENDIAN 1181 SB_BE; 1182 #else 1183 SB_UE; 1184 #endif 1185 tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1); 1186 tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1); 1187 1188 TPL_LIST_WLOCK(); 1189 newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *), 1190 (ntpl + 1) * sizeof(void *), 0); 1191 if (newtpllist != NULL) { 1192 tpl_id = ntpl++; 1193 tpllist = (struct statsblob_tpl **)newtpllist; 1194 tpllist[tpl_id] = (struct statsblob_tpl *)tpl; 1195 stats_tpl_update_hash(tpllist[tpl_id]); 1196 } else { 1197 stats_free(tpl); 1198 stats_free(tpl_sb); 1199 if (tpl_mb != NULL) { 1200 stats_free(tpl_mb->tplname); 1201 stats_free(tpl_mb); 1202 } 1203 tpl_id = -ENOMEM; 1204 } 1205 TPL_LIST_WUNLOCK(); 1206 1207 return (tpl_id); 1208 } 1209 1210 int 1211 stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name, 1212 enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss, 1213 uint32_t flags) 1214 { 1215 struct voi *voi; 1216 struct voistat *tmpstat; 1217 struct statsblobv1 *tpl_sb; 1218 struct metablob *tpl_mb; 1219 int error, i, newstatdataidx, newvoibytes, newvoistatbytes, 1220 newvoistatdatabytes, newvoistatmaxid; 1221 uint32_t nbytes; 1222 1223 if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || 1224 nvss == 0 || vss == NULL) 1225 return (EINVAL); 1226 1227 error = nbytes = newvoibytes = newvoistatbytes = 1228 newvoistatdatabytes = 0; 1229 newvoistatmaxid = -1; 1230 1231 /* Calculate the number of bytes required for the new voistats. */ 1232 for (i = nvss - 1; i >= 0; i--) { 1233 if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES || 1234 vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES || 1235 vss[i].iv == NULL || vss[i].vsdsz == 0) 1236 return (EINVAL); 1237 if ((int)vss[i].stype > newvoistatmaxid) 1238 newvoistatmaxid = vss[i].stype; 1239 newvoistatdatabytes += vss[i].vsdsz; 1240 } 1241 1242 if (flags & SB_VOI_RELUPDATE) { 1243 /* XXXLAS: VOI state bytes may need to vary based on stat types. */ 1244 newvoistatdatabytes += sizeof(struct voistatdata_voistate); 1245 } 1246 nbytes += newvoistatdatabytes; 1247 1248 TPL_LIST_WLOCK(); 1249 if (tpl_id < ntpl) { 1250 tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb; 1251 tpl_mb = tpllist[tpl_id]->mb; 1252 1253 if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) { 1254 /* Adding a new VOI and associated stats. */ 1255 if (voi_id >= NVOIS(tpl_sb)) { 1256 /* We need to grow the tpl_sb->vois array. */ 1257 newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) * 1258 sizeof(struct voi); 1259 nbytes += newvoibytes; 1260 } 1261 newvoistatbytes = 1262 (newvoistatmaxid + 1) * sizeof(struct voistat); 1263 } else { 1264 /* Adding stats to an existing VOI. */ 1265 if (newvoistatmaxid > 1266 tpl_sb->vois[voi_id].voistatmaxid) { 1267 newvoistatbytes = (newvoistatmaxid - 1268 tpl_sb->vois[voi_id].voistatmaxid) * 1269 sizeof(struct voistat); 1270 } 1271 /* XXXLAS: KPI does not yet support expanding VOIs. */ 1272 error = EOPNOTSUPP; 1273 } 1274 nbytes += newvoistatbytes; 1275 1276 if (!error && newvoibytes > 0) { 1277 struct voi_meta *voi_meta = tpl_mb->voi_meta; 1278 1279 voi_meta = stats_realloc(voi_meta, voi_meta == NULL ? 1280 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta), 1281 (1 + voi_id) * sizeof(struct voi_meta), 1282 M_ZERO); 1283 1284 if (voi_meta == NULL) 1285 error = ENOMEM; 1286 else 1287 tpl_mb->voi_meta = voi_meta; 1288 } 1289 1290 if (!error) { 1291 /* NB: Resizing can change where tpl_sb points. */ 1292 error = stats_v1_blob_expand(&tpl_sb, newvoibytes, 1293 newvoistatbytes, newvoistatdatabytes); 1294 } 1295 1296 if (!error) { 1297 tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name, 1298 0); 1299 if (tpl_mb->voi_meta[voi_id].name == NULL) 1300 error = ENOMEM; 1301 } 1302 1303 if (!error) { 1304 /* Update the template list with the resized pointer. */ 1305 tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb; 1306 1307 /* Update the template. */ 1308 voi = &tpl_sb->vois[voi_id]; 1309 1310 if (voi->id < 0) { 1311 /* VOI is new and needs to be initialised. */ 1312 voi->id = voi_id; 1313 voi->dtype = voi_dtype; 1314 voi->stats_off = tpl_sb->stats_off; 1315 if (flags & SB_VOI_RELUPDATE) 1316 voi->flags |= VOI_REQSTATE; 1317 } else { 1318 /* 1319 * XXXLAS: When this else block is written, the 1320 * "KPI does not yet support expanding VOIs" 1321 * error earlier in this function can be 1322 * removed. What is required here is to shuffle 1323 * the voistat array such that the new stats for 1324 * the voi are contiguous, which will displace 1325 * stats for other vois that reside after the 1326 * voi being updated. The other vois then need 1327 * to have their stats_off adjusted post 1328 * shuffle. 1329 */ 1330 } 1331 1332 voi->voistatmaxid = newvoistatmaxid; 1333 newstatdataidx = 0; 1334 1335 if (voi->flags & VOI_REQSTATE) { 1336 /* Initialise the voistate stat in slot 0. */ 1337 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off); 1338 tmpstat->stype = VS_STYPE_VOISTATE; 1339 tmpstat->flags = 0; 1340 tmpstat->dtype = VSD_DTYPE_VOISTATE; 1341 newstatdataidx = tmpstat->dsz = 1342 sizeof(struct voistatdata_numeric); 1343 tmpstat->data_off = tpl_sb->statsdata_off; 1344 } 1345 1346 for (i = 0; (uint32_t)i < nvss; i++) { 1347 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off + 1348 (vss[i].stype * sizeof(struct voistat))); 1349 KASSERT(tmpstat->stype < 0, ("voistat %p " 1350 "already initialised", tmpstat)); 1351 tmpstat->stype = vss[i].stype; 1352 tmpstat->flags = vss[i].flags; 1353 tmpstat->dtype = vss[i].vs_dtype; 1354 tmpstat->dsz = vss[i].vsdsz; 1355 tmpstat->data_off = tpl_sb->statsdata_off + 1356 newstatdataidx; 1357 memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off), 1358 vss[i].iv, vss[i].vsdsz); 1359 newstatdataidx += vss[i].vsdsz; 1360 } 1361 1362 /* Update the template version hash. */ 1363 stats_tpl_update_hash(tpllist[tpl_id]); 1364 /* XXXLAS: Confirm tpl name/hash pair remains unique. */ 1365 } 1366 } else 1367 error = EINVAL; 1368 TPL_LIST_WUNLOCK(); 1369 1370 return (error); 1371 } 1372 1373 struct statsblobv1 * 1374 stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused) 1375 { 1376 struct statsblobv1 *sb; 1377 int error; 1378 1379 sb = NULL; 1380 1381 TPL_LIST_RLOCK(); 1382 if (tpl_id < ntpl) { 1383 sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0); 1384 if (sb != NULL) { 1385 sb->maxsz = tpllist[tpl_id]->sb->maxsz; 1386 error = stats_v1_blob_init_locked(sb, tpl_id, 0); 1387 } else 1388 error = ENOMEM; 1389 1390 if (error) { 1391 stats_free(sb); 1392 sb = NULL; 1393 } 1394 } 1395 TPL_LIST_RUNLOCK(); 1396 1397 return (sb); 1398 } 1399 1400 void 1401 stats_v1_blob_destroy(struct statsblobv1 *sb) 1402 { 1403 1404 stats_free(sb); 1405 } 1406 1407 int 1408 stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id, 1409 enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd, 1410 size_t *retvsdsz) 1411 { 1412 struct voi *v; 1413 struct voistat *vs; 1414 1415 if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 || 1416 voi_id >= NVOIS(sb)) 1417 return (EINVAL); 1418 1419 v = &sb->vois[voi_id]; 1420 if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid) 1421 return (EINVAL); 1422 1423 vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat))); 1424 *retvsd = BLOB_OFFSET(sb, vs->data_off); 1425 if (retdtype != NULL) 1426 *retdtype = vs->dtype; 1427 if (retvsdsz != NULL) 1428 *retvsdsz = vs->dsz; 1429 1430 return (0); 1431 } 1432 1433 int 1434 stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags) 1435 { 1436 int error; 1437 1438 error = 0; 1439 1440 TPL_LIST_RLOCK(); 1441 if (sb == NULL || tpl_id >= ntpl) { 1442 error = EINVAL; 1443 } else { 1444 error = stats_v1_blob_init_locked(sb, tpl_id, flags); 1445 } 1446 TPL_LIST_RUNLOCK(); 1447 1448 return (error); 1449 } 1450 1451 static inline int 1452 stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 1453 uint32_t flags __unused) 1454 { 1455 int error; 1456 1457 TPL_LIST_RLOCK_ASSERT(); 1458 error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW; 1459 KASSERT(!error, 1460 ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz)); 1461 1462 if (!error) { 1463 memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz); 1464 sb->created = sb->lastrst = stats_sbinuptime(); 1465 sb->tplhash = tpllist[tpl_id]->mb->tplhash; 1466 } 1467 1468 return (error); 1469 } 1470 1471 static int 1472 stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 1473 int newvoistatbytes, int newvoistatdatabytes) 1474 { 1475 struct statsblobv1 *sb; 1476 struct voi *tmpvoi; 1477 struct voistat *tmpvoistat, *voistat_array; 1478 int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats; 1479 1480 KASSERT(newvoibytes % sizeof(struct voi) == 0, 1481 ("Bad newvoibytes %d", newvoibytes)); 1482 KASSERT(newvoistatbytes % sizeof(struct voistat) == 0, 1483 ("Bad newvoistatbytes %d", newvoistatbytes)); 1484 1485 error = ((newvoibytes % sizeof(struct voi) == 0) && 1486 (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL; 1487 sb = *sbpp; 1488 nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes; 1489 1490 /* 1491 * XXXLAS: Required until we gain support for flags which alter the 1492 * units of size/offset fields in key structs. 1493 */ 1494 if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ)) 1495 error = EFBIG; 1496 1497 if (!error && (sb->cursz + nbytes > sb->maxsz)) { 1498 /* Need to expand our blob. */ 1499 sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO); 1500 if (sb != NULL) { 1501 sb->maxsz = sb->cursz + nbytes; 1502 *sbpp = sb; 1503 } else 1504 error = ENOMEM; 1505 } 1506 1507 if (!error) { 1508 /* 1509 * Shuffle memory within the expanded blob working from the end 1510 * backwards, leaving gaps for the new voistat and voistatdata 1511 * structs at the beginning of their respective blob regions, 1512 * and for the new voi structs at the end of their blob region. 1513 */ 1514 memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes), 1515 BLOB_OFFSET(sb, sb->statsdata_off), 1516 sb->cursz - sb->statsdata_off); 1517 memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes + 1518 newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off), 1519 sb->statsdata_off - sb->stats_off); 1520 1521 /* First index of new voi/voistat structs to be initialised. */ 1522 idxnewvois = NVOIS(sb); 1523 idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1; 1524 1525 /* Update housekeeping variables and offsets. */ 1526 sb->cursz += nbytes; 1527 sb->stats_off += newvoibytes; 1528 sb->statsdata_off += newvoibytes + newvoistatbytes; 1529 1530 /* XXXLAS: Zeroing not strictly needed but aids debugging. */ 1531 memset(&sb->vois[idxnewvois], '\0', newvoibytes); 1532 memset(BLOB_OFFSET(sb, sb->stats_off), '\0', 1533 newvoistatbytes); 1534 memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0', 1535 newvoistatdatabytes); 1536 1537 /* Initialise new voi array members and update offsets. */ 1538 for (i = 0; i < NVOIS(sb); i++) { 1539 tmpvoi = &sb->vois[i]; 1540 if (i >= idxnewvois) { 1541 tmpvoi->id = tmpvoi->voistatmaxid = -1; 1542 } else if (tmpvoi->id > -1) { 1543 tmpvoi->stats_off += newvoibytes + 1544 newvoistatbytes; 1545 } 1546 } 1547 1548 /* Initialise new voistat array members and update offsets. */ 1549 nvoistats = (sb->statsdata_off - sb->stats_off) / 1550 sizeof(struct voistat); 1551 voistat_array = BLOB_OFFSET(sb, sb->stats_off); 1552 for (i = 0; i < nvoistats; i++) { 1553 tmpvoistat = &voistat_array[i]; 1554 if (i <= idxnewvoistats) { 1555 tmpvoistat->stype = -1; 1556 } else if (tmpvoistat->stype > -1) { 1557 tmpvoistat->data_off += nbytes; 1558 } 1559 } 1560 } 1561 1562 return (error); 1563 } 1564 1565 static void 1566 stats_v1_blob_finalise(struct statsblobv1 *sb __unused) 1567 { 1568 1569 /* XXXLAS: Fill this in. */ 1570 } 1571 1572 static void 1573 stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb, 1574 void *usrctx, uint32_t flags) 1575 { 1576 struct voi *v; 1577 struct voistat *vs; 1578 struct sb_iter_ctx ctx; 1579 int i, j, firstvoi; 1580 1581 ctx.usrctx = usrctx; 1582 ctx.flags = SB_IT_FIRST_CB; 1583 firstvoi = 1; 1584 1585 for (i = 0; i < NVOIS(sb); i++) { 1586 v = &sb->vois[i]; 1587 ctx.vslot = i; 1588 ctx.vsslot = -1; 1589 ctx.flags |= SB_IT_FIRST_VOISTAT; 1590 1591 if (firstvoi) 1592 ctx.flags |= SB_IT_FIRST_VOI; 1593 else if (i == (NVOIS(sb) - 1)) 1594 ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB; 1595 1596 if (v->id < 0 && (flags & SB_IT_NULLVOI)) { 1597 if (icb(sb, v, NULL, &ctx)) 1598 return; 1599 firstvoi = 0; 1600 ctx.flags &= ~SB_IT_FIRST_CB; 1601 } 1602 1603 /* If NULL voi, v->voistatmaxid == -1 */ 1604 for (j = 0; j <= v->voistatmaxid; j++) { 1605 vs = &((struct voistat *)BLOB_OFFSET(sb, 1606 v->stats_off))[j]; 1607 if (vs->stype < 0 && 1608 !(flags & SB_IT_NULLVOISTAT)) 1609 continue; 1610 1611 if (j == v->voistatmaxid) { 1612 ctx.flags |= SB_IT_LAST_VOISTAT; 1613 if (i == (NVOIS(sb) - 1)) 1614 ctx.flags |= 1615 SB_IT_LAST_CB; 1616 } else 1617 ctx.flags &= ~SB_IT_LAST_CB; 1618 1619 ctx.vsslot = j; 1620 if (icb(sb, v, vs, &ctx)) 1621 return; 1622 1623 ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT | 1624 SB_IT_LAST_VOISTAT); 1625 } 1626 ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI); 1627 } 1628 } 1629 1630 static inline void 1631 stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused, 1632 const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype, 1633 size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 1634 { 1635 const struct ctdth32 *ctd32tree; 1636 const struct ctdth64 *ctd64tree; 1637 const struct voistatdata_tdgstctd32 *ctd32; 1638 const struct voistatdata_tdgstctd64 *ctd64; 1639 const char *fmtstr; 1640 uint64_t smplcnt, compcnt; 1641 int is32bit, qmaxstrlen; 1642 uint16_t maxctds, curctds; 1643 1644 switch (tdgst_dtype) { 1645 case VSD_DTYPE_TDGSTCLUST32: 1646 smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt; 1647 compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt; 1648 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 1649 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 1650 ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree; 1651 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) : 1652 ARB_CMIN(ctdth32, ctd32tree)); 1653 qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10); 1654 is32bit = 1; 1655 ctd64tree = NULL; 1656 ctd64 = NULL; 1657 break; 1658 case VSD_DTYPE_TDGSTCLUST64: 1659 smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt; 1660 compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt; 1661 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 1662 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 1663 ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree; 1664 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) : 1665 ARB_CMIN(ctdth64, ctd64tree)); 1666 qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10); 1667 is32bit = 0; 1668 ctd32tree = NULL; 1669 ctd32 = NULL; 1670 break; 1671 default: 1672 return; 1673 } 1674 1675 switch (fmt) { 1676 case SB_STRFMT_FREEFORM: 1677 fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu"; 1678 break; 1679 case SB_STRFMT_JSON: 1680 default: 1681 fmtstr = 1682 "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu," 1683 "\"nctds\":%hu,\"ctds\":["; 1684 break; 1685 } 1686 sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt, 1687 maxctds, curctds); 1688 1689 while ((is32bit ? NULL != ctd32 : NULL != ctd64)) { 1690 char qstr[qmaxstrlen]; 1691 1692 switch (fmt) { 1693 case SB_STRFMT_FREEFORM: 1694 fmtstr = "\n\t\t\t\t"; 1695 break; 1696 case SB_STRFMT_JSON: 1697 default: 1698 fmtstr = "{"; 1699 break; 1700 } 1701 sbuf_cat(buf, fmtstr); 1702 1703 if (objdump) { 1704 switch (fmt) { 1705 case SB_STRFMT_FREEFORM: 1706 fmtstr = "ctd[%hu]."; 1707 break; 1708 case SB_STRFMT_JSON: 1709 default: 1710 fmtstr = "\"ctd\":%hu,"; 1711 break; 1712 } 1713 sbuf_printf(buf, fmtstr, is32bit ? 1714 ARB_SELFIDX(ctd32tree, ctd32) : 1715 ARB_SELFIDX(ctd64tree, ctd64)); 1716 } 1717 1718 switch (fmt) { 1719 case SB_STRFMT_FREEFORM: 1720 fmtstr = "{mu="; 1721 break; 1722 case SB_STRFMT_JSON: 1723 default: 1724 fmtstr = "\"mu\":"; 1725 break; 1726 } 1727 sbuf_cat(buf, fmtstr); 1728 Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr, 1729 sizeof(qstr)); 1730 sbuf_cat(buf, qstr); 1731 1732 switch (fmt) { 1733 case SB_STRFMT_FREEFORM: 1734 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 1735 break; 1736 case SB_STRFMT_JSON: 1737 default: 1738 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 1739 break; 1740 } 1741 sbuf_printf(buf, fmtstr, 1742 is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt); 1743 1744 if (is32bit) 1745 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 1746 ARB_SELFIDX(ctd32tree, ctd32) + 1) : 1747 ARB_CNEXT(ctdth32, ctd32tree, ctd32)); 1748 else 1749 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 1750 ARB_SELFIDX(ctd64tree, ctd64) + 1) : 1751 ARB_CNEXT(ctdth64, ctd64tree, ctd64)); 1752 1753 if (fmt == SB_STRFMT_JSON && 1754 (is32bit ? NULL != ctd32 : NULL != ctd64)) 1755 sbuf_putc(buf, ','); 1756 } 1757 if (fmt == SB_STRFMT_JSON) 1758 sbuf_cat(buf, "]"); 1759 } 1760 1761 static inline void 1762 stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype, 1763 const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype, 1764 size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 1765 { 1766 const struct voistatdata_numeric *bkt_lb, *bkt_ub; 1767 const char *fmtstr; 1768 int is32bit; 1769 uint16_t i, nbkts; 1770 1771 switch (hist_dtype) { 1772 case VSD_DTYPE_CRHIST32: 1773 nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz); 1774 is32bit = 1; 1775 break; 1776 case VSD_DTYPE_DRHIST32: 1777 nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz); 1778 is32bit = 1; 1779 break; 1780 case VSD_DTYPE_DVHIST32: 1781 nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz); 1782 is32bit = 1; 1783 break; 1784 case VSD_DTYPE_CRHIST64: 1785 nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz); 1786 is32bit = 0; 1787 break; 1788 case VSD_DTYPE_DRHIST64: 1789 nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz); 1790 is32bit = 0; 1791 break; 1792 case VSD_DTYPE_DVHIST64: 1793 nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz); 1794 is32bit = 0; 1795 break; 1796 default: 1797 return; 1798 } 1799 1800 switch (fmt) { 1801 case SB_STRFMT_FREEFORM: 1802 fmtstr = "nbkts=%hu, "; 1803 break; 1804 case SB_STRFMT_JSON: 1805 default: 1806 fmtstr = "\"nbkts\":%hu,"; 1807 break; 1808 } 1809 sbuf_printf(buf, fmtstr, nbkts); 1810 1811 switch (fmt) { 1812 case SB_STRFMT_FREEFORM: 1813 fmtstr = (is32bit ? "oob=%u" : "oob=%ju"); 1814 break; 1815 case SB_STRFMT_JSON: 1816 default: 1817 fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" : 1818 "\"oob\":%ju,\"bkts\":["); 1819 break; 1820 } 1821 sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist, 1822 hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, 1823 hist_dtype, oob)); 1824 1825 for (i = 0; i < nbkts; i++) { 1826 switch (hist_dtype) { 1827 case VSD_DTYPE_CRHIST32: 1828 case VSD_DTYPE_CRHIST64: 1829 bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype, 1830 bkts[i].lb); 1831 if (i < nbkts - 1) 1832 bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist, 1833 hist_dtype, bkts[i + 1].lb); 1834 else 1835 bkt_ub = &numeric_limits[LIM_MAX][voi_dtype]; 1836 break; 1837 case VSD_DTYPE_DRHIST32: 1838 case VSD_DTYPE_DRHIST64: 1839 bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 1840 bkts[i].lb); 1841 bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 1842 bkts[i].ub); 1843 break; 1844 case VSD_DTYPE_DVHIST32: 1845 case VSD_DTYPE_DVHIST64: 1846 bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist, 1847 hist_dtype, bkts[i].val); 1848 break; 1849 default: 1850 break; 1851 } 1852 1853 switch (fmt) { 1854 case SB_STRFMT_FREEFORM: 1855 fmtstr = "\n\t\t\t\t"; 1856 break; 1857 case SB_STRFMT_JSON: 1858 default: 1859 fmtstr = "{"; 1860 break; 1861 } 1862 sbuf_cat(buf, fmtstr); 1863 1864 if (objdump) { 1865 switch (fmt) { 1866 case SB_STRFMT_FREEFORM: 1867 fmtstr = "bkt[%hu]."; 1868 break; 1869 case SB_STRFMT_JSON: 1870 default: 1871 fmtstr = "\"bkt\":%hu,"; 1872 break; 1873 } 1874 sbuf_printf(buf, fmtstr, i); 1875 } 1876 1877 switch (fmt) { 1878 case SB_STRFMT_FREEFORM: 1879 fmtstr = "{lb="; 1880 break; 1881 case SB_STRFMT_JSON: 1882 default: 1883 fmtstr = "\"lb\":"; 1884 break; 1885 } 1886 sbuf_cat(buf, fmtstr); 1887 stats_voistatdata_tostr((const struct voistatdata *)bkt_lb, 1888 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 1889 fmt, buf, objdump); 1890 1891 switch (fmt) { 1892 case SB_STRFMT_FREEFORM: 1893 fmtstr = ",ub="; 1894 break; 1895 case SB_STRFMT_JSON: 1896 default: 1897 fmtstr = ",\"ub\":"; 1898 break; 1899 } 1900 sbuf_cat(buf, fmtstr); 1901 stats_voistatdata_tostr((const struct voistatdata *)bkt_ub, 1902 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 1903 fmt, buf, objdump); 1904 1905 switch (fmt) { 1906 case SB_STRFMT_FREEFORM: 1907 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 1908 break; 1909 case SB_STRFMT_JSON: 1910 default: 1911 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 1912 break; 1913 } 1914 sbuf_printf(buf, fmtstr, is32bit ? 1915 VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) : 1916 (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, 1917 bkts[i].cnt)); 1918 1919 if (fmt == SB_STRFMT_JSON && i < nbkts - 1) 1920 sbuf_putc(buf, ','); 1921 } 1922 if (fmt == SB_STRFMT_JSON) 1923 sbuf_cat(buf, "]"); 1924 } 1925 1926 int 1927 stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype, 1928 enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt, 1929 struct sbuf *buf, int objdump) 1930 { 1931 const char *fmtstr; 1932 1933 if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES || 1934 vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS) 1935 return (EINVAL); 1936 1937 switch (vsd_dtype) { 1938 case VSD_DTYPE_VOISTATE: 1939 switch (fmt) { 1940 case SB_STRFMT_FREEFORM: 1941 fmtstr = "prev="; 1942 break; 1943 case SB_STRFMT_JSON: 1944 default: 1945 fmtstr = "\"prev\":"; 1946 break; 1947 } 1948 sbuf_cat(buf, fmtstr); 1949 /* 1950 * Render prev by passing it as *vsd and voi_dtype as vsd_dtype. 1951 */ 1952 stats_voistatdata_tostr( 1953 (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev, 1954 voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump); 1955 break; 1956 case VSD_DTYPE_INT_S32: 1957 sbuf_printf(buf, "%d", vsd->int32.s32); 1958 break; 1959 case VSD_DTYPE_INT_U32: 1960 sbuf_printf(buf, "%u", vsd->int32.u32); 1961 break; 1962 case VSD_DTYPE_INT_S64: 1963 sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64); 1964 break; 1965 case VSD_DTYPE_INT_U64: 1966 sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64); 1967 break; 1968 case VSD_DTYPE_INT_SLONG: 1969 sbuf_printf(buf, "%ld", vsd->intlong.slong); 1970 break; 1971 case VSD_DTYPE_INT_ULONG: 1972 sbuf_printf(buf, "%lu", vsd->intlong.ulong); 1973 break; 1974 case VSD_DTYPE_Q_S32: 1975 { 1976 char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)]; 1977 Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr)); 1978 sbuf_cat(buf, qstr); 1979 } 1980 break; 1981 case VSD_DTYPE_Q_U32: 1982 { 1983 char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)]; 1984 Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr)); 1985 sbuf_cat(buf, qstr); 1986 } 1987 break; 1988 case VSD_DTYPE_Q_S64: 1989 { 1990 char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)]; 1991 Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr)); 1992 sbuf_cat(buf, qstr); 1993 } 1994 break; 1995 case VSD_DTYPE_Q_U64: 1996 { 1997 char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)]; 1998 Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr)); 1999 sbuf_cat(buf, qstr); 2000 } 2001 break; 2002 case VSD_DTYPE_CRHIST32: 2003 case VSD_DTYPE_DRHIST32: 2004 case VSD_DTYPE_DVHIST32: 2005 case VSD_DTYPE_CRHIST64: 2006 case VSD_DTYPE_DRHIST64: 2007 case VSD_DTYPE_DVHIST64: 2008 stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd), 2009 vsd_dtype, vsd_sz, fmt, buf, objdump); 2010 break; 2011 case VSD_DTYPE_TDGSTCLUST32: 2012 case VSD_DTYPE_TDGSTCLUST64: 2013 stats_voistatdata_tdgst_tostr(voi_dtype, 2014 CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf, 2015 objdump); 2016 break; 2017 default: 2018 break; 2019 } 2020 2021 return (sbuf_error(buf)); 2022 } 2023 2024 static void 2025 stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v, 2026 struct voistat *vs, struct sb_iter_ctx *ctx) 2027 { 2028 struct sb_tostrcb_ctx *sctx; 2029 struct metablob *tpl_mb; 2030 struct sbuf *buf; 2031 void *vsd; 2032 uint8_t dump; 2033 2034 sctx = ctx->usrctx; 2035 buf = sctx->buf; 2036 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 2037 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 2038 2039 if (ctx->flags & SB_IT_FIRST_CB) { 2040 sbuf_printf(buf, "struct statsblobv1@%p", sb); 2041 if (dump) { 2042 sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, " 2043 "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, " 2044 "stats_off=%hu, statsdata_off=%hu", 2045 sb->abi, sb->endian, sb->maxsz, sb->cursz, 2046 sb->created, sb->lastrst, sb->flags, sb->stats_off, 2047 sb->statsdata_off); 2048 } 2049 sbuf_printf(buf, ", tplhash=%u", sb->tplhash); 2050 } 2051 2052 if (ctx->flags & SB_IT_FIRST_VOISTAT) { 2053 sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id); 2054 if (v->id < 0) 2055 return; 2056 sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" : 2057 tpl_mb->voi_meta[v->id].name); 2058 if (dump) 2059 sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, " 2060 "voistatmaxid=%hhd, stats_off=%hu", v->flags, 2061 vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off); 2062 } 2063 2064 if (!dump && vs->stype <= 0) 2065 return; 2066 2067 sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot); 2068 if (vs->stype < 0) { 2069 sbuf_printf(buf, "%hhd", vs->stype); 2070 return; 2071 } else 2072 sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype], 2073 vs->errs); 2074 vsd = BLOB_OFFSET(sb, vs->data_off); 2075 if (dump) 2076 sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, " 2077 "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype], 2078 vs->dsz, vs->data_off); 2079 2080 sbuf_printf(buf, "\n\t\t\tvoistatdata: "); 2081 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 2082 sctx->fmt, buf, dump); 2083 } 2084 2085 static void 2086 stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 2087 struct sb_iter_ctx *ctx) 2088 { 2089 struct sb_tostrcb_ctx *sctx; 2090 struct metablob *tpl_mb; 2091 struct sbuf *buf; 2092 const char *fmtstr; 2093 void *vsd; 2094 uint8_t dump; 2095 2096 sctx = ctx->usrctx; 2097 buf = sctx->buf; 2098 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 2099 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 2100 2101 if (ctx->flags & SB_IT_FIRST_CB) { 2102 sbuf_putc(buf, '{'); 2103 if (dump) { 2104 sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu," 2105 "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd," 2106 "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu," 2107 "\"statsdata_off\":%hu,", sb->abi, 2108 sb->endian, sb->maxsz, sb->cursz, sb->created, 2109 sb->lastrst, sb->flags, sb->stats_off, 2110 sb->statsdata_off); 2111 } 2112 2113 if (tpl_mb == NULL) 2114 fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{"; 2115 else 2116 fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{"; 2117 2118 sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null", 2119 sb->tplhash); 2120 } 2121 2122 if (ctx->flags & SB_IT_FIRST_VOISTAT) { 2123 if (dump) { 2124 sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot, 2125 v->id); 2126 if (v->id < 0) { 2127 sbuf_printf(buf, "},"); 2128 return; 2129 } 2130 2131 if (tpl_mb == NULL) 2132 fmtstr = ",\"name\":%s,\"flags\":%hu," 2133 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 2134 "\"stats_off\":%hu,"; 2135 else 2136 fmtstr = ",\"name\":\"%s\",\"flags\":%hu," 2137 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 2138 "\"stats_off\":%hu,"; 2139 2140 sbuf_printf(buf, fmtstr, tpl_mb ? 2141 tpl_mb->voi_meta[v->id].name : "null", v->flags, 2142 vsd_dtype2name[v->dtype], v->voistatmaxid, 2143 v->stats_off); 2144 } else { 2145 if (tpl_mb == NULL) { 2146 sbuf_printf(buf, "\"[%hd]\":{", v->id); 2147 } else { 2148 sbuf_printf(buf, "\"%s\":{", 2149 tpl_mb->voi_meta[v->id].name); 2150 } 2151 } 2152 sbuf_cat(buf, "\"stats\":{"); 2153 } 2154 2155 vsd = BLOB_OFFSET(sb, vs->data_off); 2156 if (dump) { 2157 sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot); 2158 if (vs->stype < 0) { 2159 sbuf_printf(buf, "{\"stype\":-1},"); 2160 return; 2161 } 2162 sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu," 2163 "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{", 2164 vs_stype2name[vs->stype], vs->errs, vs->flags, 2165 vsd_dtype2name[vs->dtype], vs->data_off); 2166 } else if (vs->stype > 0) { 2167 if (tpl_mb == NULL) 2168 sbuf_printf(buf, "\"[%hhd]\":", vs->stype); 2169 else 2170 sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]); 2171 } else 2172 return; 2173 2174 if ((vs->flags & VS_VSDVALID) || dump) { 2175 if (!dump) 2176 sbuf_printf(buf, "{\"errs\":%hu,", vs->errs); 2177 /* Simple non-compound VSD types need a key. */ 2178 if (!vsd_compoundtype[vs->dtype]) 2179 sbuf_cat(buf, "\"val\":"); 2180 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 2181 sctx->fmt, buf, dump); 2182 sbuf_cat(buf, dump ? "}}" : "}"); 2183 } else 2184 sbuf_cat(buf, dump ? "null}" : "null"); 2185 2186 if (ctx->flags & SB_IT_LAST_VOISTAT) 2187 sbuf_cat(buf, "}}"); 2188 2189 if (ctx->flags & SB_IT_LAST_CB) 2190 sbuf_cat(buf, "}}"); 2191 else 2192 sbuf_putc(buf, ','); 2193 } 2194 2195 static int 2196 stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 2197 struct sb_iter_ctx *ctx) 2198 { 2199 struct sb_tostrcb_ctx *sctx; 2200 2201 sctx = ctx->usrctx; 2202 2203 switch (sctx->fmt) { 2204 case SB_STRFMT_FREEFORM: 2205 stats_v1_itercb_tostr_freeform(sb, v, vs, ctx); 2206 break; 2207 case SB_STRFMT_JSON: 2208 stats_v1_itercb_tostr_json(sb, v, vs, ctx); 2209 break; 2210 default: 2211 break; 2212 } 2213 2214 return (sbuf_error(sctx->buf)); 2215 } 2216 2217 int 2218 stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf, 2219 enum sb_str_fmt fmt, uint32_t flags) 2220 { 2221 struct sb_tostrcb_ctx sctx; 2222 uint32_t iflags; 2223 2224 if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL || 2225 fmt >= SB_STRFMT_NUM_FMTS) 2226 return (EINVAL); 2227 2228 sctx.buf = buf; 2229 sctx.fmt = fmt; 2230 sctx.flags = flags; 2231 2232 if (flags & SB_TOSTR_META) { 2233 if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash), 2234 &sctx.tpl)) 2235 return (EINVAL); 2236 } else 2237 sctx.tpl = NULL; 2238 2239 iflags = 0; 2240 if (flags & SB_TOSTR_OBJDUMP) 2241 iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT); 2242 stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags); 2243 2244 return (sbuf_error(buf)); 2245 } 2246 2247 static int 2248 stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v, 2249 struct voistat *vs, struct sb_iter_ctx *ctx) 2250 { 2251 struct sb_visitcb_ctx *vctx; 2252 struct sb_visit sbv; 2253 2254 vctx = ctx->usrctx; 2255 2256 sbv.tplhash = sb->tplhash; 2257 sbv.voi_id = v->id; 2258 sbv.voi_dtype = v->dtype; 2259 sbv.vs_stype = vs->stype; 2260 sbv.vs_dtype = vs->dtype; 2261 sbv.vs_dsz = vs->dsz; 2262 sbv.vs_data = BLOB_OFFSET(sb, vs->data_off); 2263 sbv.vs_errs = vs->errs; 2264 sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB | 2265 SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT | 2266 SB_IT_LAST_VOISTAT); 2267 2268 return (vctx->cb(&sbv, vctx->usrctx)); 2269 } 2270 2271 int 2272 stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func, 2273 void *usrctx) 2274 { 2275 struct sb_visitcb_ctx vctx; 2276 2277 if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL) 2278 return (EINVAL); 2279 2280 vctx.cb = func; 2281 vctx.usrctx = usrctx; 2282 2283 stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0); 2284 2285 return (0); 2286 } 2287 2288 static int 2289 stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused, 2290 struct voistat *vs, struct sb_iter_ctx *ctx __unused) 2291 { 2292 void *vsd; 2293 2294 if (vs->stype == VS_STYPE_VOISTATE) 2295 return (0); 2296 2297 vsd = BLOB_OFFSET(sb, vs->data_off); 2298 2299 /* Perform the stat type's default reset action. */ 2300 switch (vs->stype) { 2301 case VS_STYPE_SUM: 2302 switch (vs->dtype) { 2303 case VSD_DTYPE_Q_S32: 2304 Q_SIFVAL(VSD(q32, vsd)->sq32, 0); 2305 break; 2306 case VSD_DTYPE_Q_U32: 2307 Q_SIFVAL(VSD(q32, vsd)->uq32, 0); 2308 break; 2309 case VSD_DTYPE_Q_S64: 2310 Q_SIFVAL(VSD(q64, vsd)->sq64, 0); 2311 break; 2312 case VSD_DTYPE_Q_U64: 2313 Q_SIFVAL(VSD(q64, vsd)->uq64, 0); 2314 break; 2315 default: 2316 bzero(vsd, vs->dsz); 2317 break; 2318 } 2319 break; 2320 case VS_STYPE_MAX: 2321 switch (vs->dtype) { 2322 case VSD_DTYPE_Q_S32: 2323 Q_SIFVAL(VSD(q32, vsd)->sq32, 2324 Q_IFMINVAL(VSD(q32, vsd)->sq32)); 2325 break; 2326 case VSD_DTYPE_Q_U32: 2327 Q_SIFVAL(VSD(q32, vsd)->uq32, 2328 Q_IFMINVAL(VSD(q32, vsd)->uq32)); 2329 break; 2330 case VSD_DTYPE_Q_S64: 2331 Q_SIFVAL(VSD(q64, vsd)->sq64, 2332 Q_IFMINVAL(VSD(q64, vsd)->sq64)); 2333 break; 2334 case VSD_DTYPE_Q_U64: 2335 Q_SIFVAL(VSD(q64, vsd)->uq64, 2336 Q_IFMINVAL(VSD(q64, vsd)->uq64)); 2337 break; 2338 default: 2339 memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype], 2340 vs->dsz); 2341 break; 2342 } 2343 break; 2344 case VS_STYPE_MIN: 2345 switch (vs->dtype) { 2346 case VSD_DTYPE_Q_S32: 2347 Q_SIFVAL(VSD(q32, vsd)->sq32, 2348 Q_IFMAXVAL(VSD(q32, vsd)->sq32)); 2349 break; 2350 case VSD_DTYPE_Q_U32: 2351 Q_SIFVAL(VSD(q32, vsd)->uq32, 2352 Q_IFMAXVAL(VSD(q32, vsd)->uq32)); 2353 break; 2354 case VSD_DTYPE_Q_S64: 2355 Q_SIFVAL(VSD(q64, vsd)->sq64, 2356 Q_IFMAXVAL(VSD(q64, vsd)->sq64)); 2357 break; 2358 case VSD_DTYPE_Q_U64: 2359 Q_SIFVAL(VSD(q64, vsd)->uq64, 2360 Q_IFMAXVAL(VSD(q64, vsd)->uq64)); 2361 break; 2362 default: 2363 memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype], 2364 vs->dsz); 2365 break; 2366 } 2367 break; 2368 case VS_STYPE_HIST: 2369 { 2370 /* Reset bucket counts. */ 2371 struct voistatdata_hist *hist; 2372 int i, is32bit; 2373 uint16_t nbkts; 2374 2375 hist = VSD(hist, vsd); 2376 switch (vs->dtype) { 2377 case VSD_DTYPE_CRHIST32: 2378 nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 2379 is32bit = 1; 2380 break; 2381 case VSD_DTYPE_DRHIST32: 2382 nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 2383 is32bit = 1; 2384 break; 2385 case VSD_DTYPE_DVHIST32: 2386 nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 2387 is32bit = 1; 2388 break; 2389 case VSD_DTYPE_CRHIST64: 2390 nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 2391 is32bit = 0; 2392 break; 2393 case VSD_DTYPE_DRHIST64: 2394 nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 2395 is32bit = 0; 2396 break; 2397 case VSD_DTYPE_DVHIST64: 2398 nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 2399 is32bit = 0; 2400 break; 2401 default: 2402 return (0); 2403 } 2404 2405 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob), 2406 is32bit ? sizeof(uint32_t) : sizeof(uint64_t)); 2407 for (i = nbkts - 1; i >= 0; i--) { 2408 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, 2409 bkts[i].cnt), is32bit ? sizeof(uint32_t) : 2410 sizeof(uint64_t)); 2411 } 2412 break; 2413 } 2414 case VS_STYPE_TDGST: 2415 { 2416 /* Reset sample count centroids array/tree. */ 2417 struct voistatdata_tdgst *tdgst; 2418 struct ctdth32 *ctd32tree; 2419 struct ctdth64 *ctd64tree; 2420 struct voistatdata_tdgstctd32 *ctd32; 2421 struct voistatdata_tdgstctd64 *ctd64; 2422 2423 tdgst = VSD(tdgst, vsd); 2424 switch (vs->dtype) { 2425 case VSD_DTYPE_TDGSTCLUST32: 2426 VSD(tdgstclust32, tdgst)->smplcnt = 0; 2427 VSD(tdgstclust32, tdgst)->compcnt = 0; 2428 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 2429 ARB_INIT(ctd32, ctdlnk, ctd32tree, 2430 ARB_MAXNODES(ctd32tree)) { 2431 ctd32->cnt = 0; 2432 Q_SIFVAL(ctd32->mu, 0); 2433 } 2434 #ifdef DIAGNOSTIC 2435 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 2436 #endif 2437 break; 2438 case VSD_DTYPE_TDGSTCLUST64: 2439 VSD(tdgstclust64, tdgst)->smplcnt = 0; 2440 VSD(tdgstclust64, tdgst)->compcnt = 0; 2441 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 2442 ARB_INIT(ctd64, ctdlnk, ctd64tree, 2443 ARB_MAXNODES(ctd64tree)) { 2444 ctd64->cnt = 0; 2445 Q_SIFVAL(ctd64->mu, 0); 2446 } 2447 #ifdef DIAGNOSTIC 2448 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 2449 #endif 2450 break; 2451 default: 2452 return (0); 2453 } 2454 break; 2455 } 2456 default: 2457 KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 2458 break; 2459 } 2460 2461 vs->errs = 0; 2462 vs->flags &= ~VS_VSDVALID; 2463 2464 return (0); 2465 } 2466 2467 int 2468 stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz, 2469 struct statsblobv1 *src, uint32_t flags) 2470 { 2471 int error; 2472 2473 if (src != NULL && src->abi == STATS_ABI_V1) { 2474 error = stats_v1_blob_clone(dst, dstmaxsz, src, flags); 2475 if (!error) { 2476 if (flags & SB_CLONE_RSTSRC) { 2477 stats_v1_blob_iter(src, 2478 stats_v1_icb_reset_voistat, NULL, 0); 2479 src->lastrst = stats_sbinuptime(); 2480 } 2481 stats_v1_blob_finalise(*dst); 2482 } 2483 } else 2484 error = EINVAL; 2485 2486 return (error); 2487 } 2488 2489 static inline int 2490 stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused, 2491 struct voistatdata *voival, struct voistat *vs, void *vsd) 2492 { 2493 int error; 2494 2495 KASSERT(vs->dtype < VSD_NUM_DTYPES, 2496 ("Unknown VSD dtype %d", vs->dtype)); 2497 2498 error = 0; 2499 2500 switch (vs->dtype) { 2501 case VSD_DTYPE_INT_S32: 2502 if (VSD(int32, vsd)->s32 < voival->int32.s32) { 2503 VSD(int32, vsd)->s32 = voival->int32.s32; 2504 vs->flags |= VS_VSDVALID; 2505 } 2506 break; 2507 case VSD_DTYPE_INT_U32: 2508 if (VSD(int32, vsd)->u32 < voival->int32.u32) { 2509 VSD(int32, vsd)->u32 = voival->int32.u32; 2510 vs->flags |= VS_VSDVALID; 2511 } 2512 break; 2513 case VSD_DTYPE_INT_S64: 2514 if (VSD(int64, vsd)->s64 < voival->int64.s64) { 2515 VSD(int64, vsd)->s64 = voival->int64.s64; 2516 vs->flags |= VS_VSDVALID; 2517 } 2518 break; 2519 case VSD_DTYPE_INT_U64: 2520 if (VSD(int64, vsd)->u64 < voival->int64.u64) { 2521 VSD(int64, vsd)->u64 = voival->int64.u64; 2522 vs->flags |= VS_VSDVALID; 2523 } 2524 break; 2525 case VSD_DTYPE_INT_SLONG: 2526 if (VSD(intlong, vsd)->slong < voival->intlong.slong) { 2527 VSD(intlong, vsd)->slong = voival->intlong.slong; 2528 vs->flags |= VS_VSDVALID; 2529 } 2530 break; 2531 case VSD_DTYPE_INT_ULONG: 2532 if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) { 2533 VSD(intlong, vsd)->ulong = voival->intlong.ulong; 2534 vs->flags |= VS_VSDVALID; 2535 } 2536 break; 2537 case VSD_DTYPE_Q_S32: 2538 if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 2539 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 2540 voival->q32.sq32)))) { 2541 vs->flags |= VS_VSDVALID; 2542 } 2543 break; 2544 case VSD_DTYPE_Q_U32: 2545 if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 2546 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 2547 voival->q32.uq32)))) { 2548 vs->flags |= VS_VSDVALID; 2549 } 2550 break; 2551 case VSD_DTYPE_Q_S64: 2552 if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 2553 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 2554 voival->q64.sq64)))) { 2555 vs->flags |= VS_VSDVALID; 2556 } 2557 break; 2558 case VSD_DTYPE_Q_U64: 2559 if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 2560 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 2561 voival->q64.uq64)))) { 2562 vs->flags |= VS_VSDVALID; 2563 } 2564 break; 2565 default: 2566 error = EINVAL; 2567 break; 2568 } 2569 2570 return (error); 2571 } 2572 2573 static inline int 2574 stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused, 2575 struct voistatdata *voival, struct voistat *vs, void *vsd) 2576 { 2577 int error; 2578 2579 KASSERT(vs->dtype < VSD_NUM_DTYPES, 2580 ("Unknown VSD dtype %d", vs->dtype)); 2581 2582 error = 0; 2583 2584 switch (vs->dtype) { 2585 case VSD_DTYPE_INT_S32: 2586 if (VSD(int32, vsd)->s32 > voival->int32.s32) { 2587 VSD(int32, vsd)->s32 = voival->int32.s32; 2588 vs->flags |= VS_VSDVALID; 2589 } 2590 break; 2591 case VSD_DTYPE_INT_U32: 2592 if (VSD(int32, vsd)->u32 > voival->int32.u32) { 2593 VSD(int32, vsd)->u32 = voival->int32.u32; 2594 vs->flags |= VS_VSDVALID; 2595 } 2596 break; 2597 case VSD_DTYPE_INT_S64: 2598 if (VSD(int64, vsd)->s64 > voival->int64.s64) { 2599 VSD(int64, vsd)->s64 = voival->int64.s64; 2600 vs->flags |= VS_VSDVALID; 2601 } 2602 break; 2603 case VSD_DTYPE_INT_U64: 2604 if (VSD(int64, vsd)->u64 > voival->int64.u64) { 2605 VSD(int64, vsd)->u64 = voival->int64.u64; 2606 vs->flags |= VS_VSDVALID; 2607 } 2608 break; 2609 case VSD_DTYPE_INT_SLONG: 2610 if (VSD(intlong, vsd)->slong > voival->intlong.slong) { 2611 VSD(intlong, vsd)->slong = voival->intlong.slong; 2612 vs->flags |= VS_VSDVALID; 2613 } 2614 break; 2615 case VSD_DTYPE_INT_ULONG: 2616 if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) { 2617 VSD(intlong, vsd)->ulong = voival->intlong.ulong; 2618 vs->flags |= VS_VSDVALID; 2619 } 2620 break; 2621 case VSD_DTYPE_Q_S32: 2622 if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 2623 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 2624 voival->q32.sq32)))) { 2625 vs->flags |= VS_VSDVALID; 2626 } 2627 break; 2628 case VSD_DTYPE_Q_U32: 2629 if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 2630 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 2631 voival->q32.uq32)))) { 2632 vs->flags |= VS_VSDVALID; 2633 } 2634 break; 2635 case VSD_DTYPE_Q_S64: 2636 if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 2637 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 2638 voival->q64.sq64)))) { 2639 vs->flags |= VS_VSDVALID; 2640 } 2641 break; 2642 case VSD_DTYPE_Q_U64: 2643 if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 2644 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 2645 voival->q64.uq64)))) { 2646 vs->flags |= VS_VSDVALID; 2647 } 2648 break; 2649 default: 2650 error = EINVAL; 2651 break; 2652 } 2653 2654 return (error); 2655 } 2656 2657 static inline int 2658 stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused, 2659 struct voistatdata *voival, struct voistat *vs, void *vsd) 2660 { 2661 int error; 2662 2663 KASSERT(vs->dtype < VSD_NUM_DTYPES, 2664 ("Unknown VSD dtype %d", vs->dtype)); 2665 2666 error = 0; 2667 2668 switch (vs->dtype) { 2669 case VSD_DTYPE_INT_S32: 2670 VSD(int32, vsd)->s32 += voival->int32.s32; 2671 break; 2672 case VSD_DTYPE_INT_U32: 2673 VSD(int32, vsd)->u32 += voival->int32.u32; 2674 break; 2675 case VSD_DTYPE_INT_S64: 2676 VSD(int64, vsd)->s64 += voival->int64.s64; 2677 break; 2678 case VSD_DTYPE_INT_U64: 2679 VSD(int64, vsd)->u64 += voival->int64.u64; 2680 break; 2681 case VSD_DTYPE_INT_SLONG: 2682 VSD(intlong, vsd)->slong += voival->intlong.slong; 2683 break; 2684 case VSD_DTYPE_INT_ULONG: 2685 VSD(intlong, vsd)->ulong += voival->intlong.ulong; 2686 break; 2687 case VSD_DTYPE_Q_S32: 2688 error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32); 2689 break; 2690 case VSD_DTYPE_Q_U32: 2691 error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32); 2692 break; 2693 case VSD_DTYPE_Q_S64: 2694 error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64); 2695 break; 2696 case VSD_DTYPE_Q_U64: 2697 error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64); 2698 break; 2699 default: 2700 error = EINVAL; 2701 break; 2702 } 2703 2704 if (!error) 2705 vs->flags |= VS_VSDVALID; 2706 2707 return (error); 2708 } 2709 2710 static inline int 2711 stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival, 2712 struct voistat *vs, struct voistatdata_hist *hist) 2713 { 2714 struct voistatdata_numeric *bkt_lb, *bkt_ub; 2715 uint64_t *oob64, *cnt64; 2716 uint32_t *oob32, *cnt32; 2717 int error, i, found, is32bit, has_ub, eq_only; 2718 2719 error = 0; 2720 2721 switch (vs->dtype) { 2722 case VSD_DTYPE_CRHIST32: 2723 i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 2724 is32bit = 1; 2725 has_ub = eq_only = 0; 2726 oob32 = &VSD(crhist32, hist)->oob; 2727 break; 2728 case VSD_DTYPE_DRHIST32: 2729 i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 2730 is32bit = has_ub = 1; 2731 eq_only = 0; 2732 oob32 = &VSD(drhist32, hist)->oob; 2733 break; 2734 case VSD_DTYPE_DVHIST32: 2735 i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 2736 is32bit = eq_only = 1; 2737 has_ub = 0; 2738 oob32 = &VSD(dvhist32, hist)->oob; 2739 break; 2740 case VSD_DTYPE_CRHIST64: 2741 i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 2742 is32bit = has_ub = eq_only = 0; 2743 oob64 = &VSD(crhist64, hist)->oob; 2744 break; 2745 case VSD_DTYPE_DRHIST64: 2746 i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 2747 is32bit = eq_only = 0; 2748 has_ub = 1; 2749 oob64 = &VSD(drhist64, hist)->oob; 2750 break; 2751 case VSD_DTYPE_DVHIST64: 2752 i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 2753 is32bit = has_ub = 0; 2754 eq_only = 1; 2755 oob64 = &VSD(dvhist64, hist)->oob; 2756 break; 2757 default: 2758 return (EINVAL); 2759 } 2760 i--; /* Adjust for 0-based array index. */ 2761 2762 /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */ 2763 for (found = 0; i >= 0 && !found; i--) { 2764 switch (vs->dtype) { 2765 case VSD_DTYPE_CRHIST32: 2766 bkt_lb = &VSD(crhist32, hist)->bkts[i].lb; 2767 cnt32 = &VSD(crhist32, hist)->bkts[i].cnt; 2768 break; 2769 case VSD_DTYPE_DRHIST32: 2770 bkt_lb = &VSD(drhist32, hist)->bkts[i].lb; 2771 bkt_ub = &VSD(drhist32, hist)->bkts[i].ub; 2772 cnt32 = &VSD(drhist32, hist)->bkts[i].cnt; 2773 break; 2774 case VSD_DTYPE_DVHIST32: 2775 bkt_lb = &VSD(dvhist32, hist)->bkts[i].val; 2776 cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt; 2777 break; 2778 case VSD_DTYPE_CRHIST64: 2779 bkt_lb = &VSD(crhist64, hist)->bkts[i].lb; 2780 cnt64 = &VSD(crhist64, hist)->bkts[i].cnt; 2781 break; 2782 case VSD_DTYPE_DRHIST64: 2783 bkt_lb = &VSD(drhist64, hist)->bkts[i].lb; 2784 bkt_ub = &VSD(drhist64, hist)->bkts[i].ub; 2785 cnt64 = &VSD(drhist64, hist)->bkts[i].cnt; 2786 break; 2787 case VSD_DTYPE_DVHIST64: 2788 bkt_lb = &VSD(dvhist64, hist)->bkts[i].val; 2789 cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt; 2790 break; 2791 default: 2792 return (EINVAL); 2793 } 2794 2795 switch (voi_dtype) { 2796 case VSD_DTYPE_INT_S32: 2797 if (voival->int32.s32 >= bkt_lb->int32.s32) { 2798 if ((eq_only && voival->int32.s32 == 2799 bkt_lb->int32.s32) || 2800 (!eq_only && (!has_ub || 2801 voival->int32.s32 < bkt_ub->int32.s32))) 2802 found = 1; 2803 } 2804 break; 2805 case VSD_DTYPE_INT_U32: 2806 if (voival->int32.u32 >= bkt_lb->int32.u32) { 2807 if ((eq_only && voival->int32.u32 == 2808 bkt_lb->int32.u32) || 2809 (!eq_only && (!has_ub || 2810 voival->int32.u32 < bkt_ub->int32.u32))) 2811 found = 1; 2812 } 2813 break; 2814 case VSD_DTYPE_INT_S64: 2815 if (voival->int64.s64 >= bkt_lb->int64.s64) 2816 if ((eq_only && voival->int64.s64 == 2817 bkt_lb->int64.s64) || 2818 (!eq_only && (!has_ub || 2819 voival->int64.s64 < bkt_ub->int64.s64))) 2820 found = 1; 2821 break; 2822 case VSD_DTYPE_INT_U64: 2823 if (voival->int64.u64 >= bkt_lb->int64.u64) 2824 if ((eq_only && voival->int64.u64 == 2825 bkt_lb->int64.u64) || 2826 (!eq_only && (!has_ub || 2827 voival->int64.u64 < bkt_ub->int64.u64))) 2828 found = 1; 2829 break; 2830 case VSD_DTYPE_INT_SLONG: 2831 if (voival->intlong.slong >= bkt_lb->intlong.slong) 2832 if ((eq_only && voival->intlong.slong == 2833 bkt_lb->intlong.slong) || 2834 (!eq_only && (!has_ub || 2835 voival->intlong.slong < 2836 bkt_ub->intlong.slong))) 2837 found = 1; 2838 break; 2839 case VSD_DTYPE_INT_ULONG: 2840 if (voival->intlong.ulong >= bkt_lb->intlong.ulong) 2841 if ((eq_only && voival->intlong.ulong == 2842 bkt_lb->intlong.ulong) || 2843 (!eq_only && (!has_ub || 2844 voival->intlong.ulong < 2845 bkt_ub->intlong.ulong))) 2846 found = 1; 2847 break; 2848 case VSD_DTYPE_Q_S32: 2849 if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32)) 2850 if ((eq_only && Q_QEQ(voival->q32.sq32, 2851 bkt_lb->q32.sq32)) || 2852 (!eq_only && (!has_ub || 2853 Q_QLTQ(voival->q32.sq32, 2854 bkt_ub->q32.sq32)))) 2855 found = 1; 2856 break; 2857 case VSD_DTYPE_Q_U32: 2858 if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32)) 2859 if ((eq_only && Q_QEQ(voival->q32.uq32, 2860 bkt_lb->q32.uq32)) || 2861 (!eq_only && (!has_ub || 2862 Q_QLTQ(voival->q32.uq32, 2863 bkt_ub->q32.uq32)))) 2864 found = 1; 2865 break; 2866 case VSD_DTYPE_Q_S64: 2867 if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64)) 2868 if ((eq_only && Q_QEQ(voival->q64.sq64, 2869 bkt_lb->q64.sq64)) || 2870 (!eq_only && (!has_ub || 2871 Q_QLTQ(voival->q64.sq64, 2872 bkt_ub->q64.sq64)))) 2873 found = 1; 2874 break; 2875 case VSD_DTYPE_Q_U64: 2876 if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64)) 2877 if ((eq_only && Q_QEQ(voival->q64.uq64, 2878 bkt_lb->q64.uq64)) || 2879 (!eq_only && (!has_ub || 2880 Q_QLTQ(voival->q64.uq64, 2881 bkt_ub->q64.uq64)))) 2882 found = 1; 2883 break; 2884 default: 2885 break; 2886 } 2887 } 2888 2889 if (found) { 2890 if (is32bit) 2891 *cnt32 += 1; 2892 else 2893 *cnt64 += 1; 2894 } else { 2895 if (is32bit) 2896 *oob32 += 1; 2897 else 2898 *oob64 += 1; 2899 } 2900 2901 vs->flags |= VS_VSDVALID; 2902 return (error); 2903 } 2904 2905 static inline int 2906 stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype, 2907 struct voistatdata_tdgst *tdgst, int attempt) 2908 { 2909 struct ctdth32 *ctd32tree; 2910 struct ctdth64 *ctd64tree; 2911 struct voistatdata_tdgstctd32 *ctd32; 2912 struct voistatdata_tdgstctd64 *ctd64; 2913 uint64_t ebits, idxmask; 2914 uint32_t bitsperidx, nebits; 2915 int error, idx, is32bit, maxctds, remctds, tmperr; 2916 2917 error = 0; 2918 2919 switch (vs_dtype) { 2920 case VSD_DTYPE_TDGSTCLUST32: 2921 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 2922 if (!ARB_FULL(ctd32tree)) 2923 return (0); 2924 VSD(tdgstclust32, tdgst)->compcnt++; 2925 maxctds = remctds = ARB_MAXNODES(ctd32tree); 2926 ARB_RESET_TREE(ctd32tree, ctdth32, maxctds); 2927 VSD(tdgstclust32, tdgst)->smplcnt = 0; 2928 is32bit = 1; 2929 ctd64tree = NULL; 2930 ctd64 = NULL; 2931 #ifdef DIAGNOSTIC 2932 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 2933 #endif 2934 break; 2935 case VSD_DTYPE_TDGSTCLUST64: 2936 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 2937 if (!ARB_FULL(ctd64tree)) 2938 return (0); 2939 VSD(tdgstclust64, tdgst)->compcnt++; 2940 maxctds = remctds = ARB_MAXNODES(ctd64tree); 2941 ARB_RESET_TREE(ctd64tree, ctdth64, maxctds); 2942 VSD(tdgstclust64, tdgst)->smplcnt = 0; 2943 is32bit = 0; 2944 ctd32tree = NULL; 2945 ctd32 = NULL; 2946 #ifdef DIAGNOSTIC 2947 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 2948 #endif 2949 break; 2950 default: 2951 return (EINVAL); 2952 } 2953 2954 /* 2955 * Rebuild the t-digest ARB by pseudorandomly selecting centroids and 2956 * re-inserting the mu/cnt of each as a value and corresponding weight. 2957 */ 2958 2959 /* 2960 * XXXCEM: random(9) is currently rand(3), not random(3). rand(3) 2961 * RAND_MAX happens to be approximately 31 bits (range [0, 2962 * 0x7ffffffd]), so the math kinda works out. When/if this portion of 2963 * the code is compiled in userspace, it gets the random(3) behavior, 2964 * which has expected range [0, 0x7fffffff]. 2965 */ 2966 #define bitsperrand 31 2967 ebits = 0; 2968 nebits = 0; 2969 bitsperidx = fls(maxctds); 2970 KASSERT(bitsperidx <= sizeof(ebits) << 3, 2971 ("%s: bitsperidx=%d, ebits=%d", 2972 __func__, bitsperidx, (int)(sizeof(ebits) << 3))); 2973 idxmask = (UINT64_C(1) << bitsperidx) - 1; 2974 2975 /* Initialise the free list with randomised centroid indices. */ 2976 for (; remctds > 0; remctds--) { 2977 while (nebits < bitsperidx) { 2978 ebits |= ((uint64_t)random()) << nebits; 2979 nebits += bitsperrand; 2980 if (nebits > (sizeof(ebits) << 3)) 2981 nebits = sizeof(ebits) << 3; 2982 } 2983 idx = ebits & idxmask; 2984 nebits -= bitsperidx; 2985 ebits >>= bitsperidx; 2986 2987 /* 2988 * Select the next centroid to put on the ARB free list. We 2989 * start with the centroid at our randomly selected array index, 2990 * and work our way forwards until finding one (the latter 2991 * aspect reduces re-insertion randomness, but is good enough). 2992 */ 2993 do { 2994 if (idx >= maxctds) 2995 idx %= maxctds; 2996 2997 if (is32bit) 2998 ctd32 = ARB_NODE(ctd32tree, idx); 2999 else 3000 ctd64 = ARB_NODE(ctd64tree, idx); 3001 } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) : 3002 ARB_ISFREE(ctd64, ctdlnk)) && ++idx); 3003 3004 /* Put the centroid on the ARB free list. */ 3005 if (is32bit) 3006 ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk); 3007 else 3008 ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk); 3009 } 3010 3011 /* 3012 * The free list now contains the randomised indices of every centroid. 3013 * Walk the free list from start to end, re-inserting each centroid's 3014 * mu/cnt. The tdgst_add() call may or may not consume the free centroid 3015 * we re-insert values from during each loop iteration, so we must latch 3016 * the index of the next free list centroid before the re-insertion 3017 * call. The previous loop above should have left the centroid pointer 3018 * pointing to the element at the head of the free list. 3019 */ 3020 KASSERT((is32bit ? 3021 ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) : 3022 ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)), 3023 ("%s: t-digest ARB@%p free list bug", __func__, 3024 (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 3025 remctds = maxctds; 3026 while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) { 3027 tmperr = 0; 3028 if (is32bit) { 3029 s64q_t x; 3030 3031 idx = ARB_NEXTFREEIDX(ctd32, ctdlnk); 3032 /* Cloning a s32q_t into a s64q_t should never fail. */ 3033 tmperr = Q_QCLONEQ(&x, ctd32->mu); 3034 tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add( 3035 vs_dtype, tdgst, x, ctd32->cnt, attempt); 3036 ctd32 = ARB_NODE(ctd32tree, idx); 3037 KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk), 3038 ("%s: t-digest ARB@%p free list bug", __func__, 3039 ctd32tree)); 3040 } else { 3041 idx = ARB_NEXTFREEIDX(ctd64, ctdlnk); 3042 tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst, 3043 ctd64->mu, ctd64->cnt, attempt); 3044 ctd64 = ARB_NODE(ctd64tree, idx); 3045 KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk), 3046 ("%s: t-digest ARB@%p free list bug", __func__, 3047 ctd64tree)); 3048 } 3049 /* 3050 * This process should not produce errors, bugs notwithstanding. 3051 * Just in case, latch any errors and attempt all re-insertions. 3052 */ 3053 error = tmperr ? tmperr : error; 3054 remctds--; 3055 } 3056 3057 KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__, 3058 (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 3059 3060 return (error); 3061 } 3062 3063 static inline int 3064 stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst, 3065 s64q_t x, uint64_t weight, int attempt) 3066 { 3067 #ifdef DIAGNOSTIC 3068 char qstr[Q_MAXSTRLEN(x, 10)]; 3069 #endif 3070 struct ctdth32 *ctd32tree; 3071 struct ctdth64 *ctd64tree; 3072 void *closest, *cur, *lb, *ub; 3073 struct voistatdata_tdgstctd32 *ctd32; 3074 struct voistatdata_tdgstctd64 *ctd64; 3075 uint64_t cnt, smplcnt, sum, tmpsum; 3076 s64q_t k, minz, q, z; 3077 int error, is32bit, n; 3078 3079 error = 0; 3080 minz = Q_INI(&z, 0, 0, Q_NFBITS(x)); 3081 3082 switch (vs_dtype) { 3083 case VSD_DTYPE_TDGSTCLUST32: 3084 if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt) 3085 error = EOVERFLOW; 3086 smplcnt = VSD(tdgstclust32, tdgst)->smplcnt; 3087 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 3088 is32bit = 1; 3089 ctd64tree = NULL; 3090 ctd64 = NULL; 3091 break; 3092 case VSD_DTYPE_TDGSTCLUST64: 3093 if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt) 3094 error = EOVERFLOW; 3095 smplcnt = VSD(tdgstclust64, tdgst)->smplcnt; 3096 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 3097 is32bit = 0; 3098 ctd32tree = NULL; 3099 ctd32 = NULL; 3100 break; 3101 default: 3102 error = EINVAL; 3103 break; 3104 } 3105 3106 if (error) 3107 return (error); 3108 3109 /* 3110 * Inspired by Ted Dunning's AVLTreeDigest.java 3111 */ 3112 do { 3113 #if defined(DIAGNOSTIC) 3114 KASSERT(attempt < 5, 3115 ("%s: Too many attempts", __func__)); 3116 #endif 3117 if (attempt >= 5) 3118 return (EAGAIN); 3119 3120 Q_SIFVAL(minz, Q_IFMAXVAL(minz)); 3121 closest = ub = NULL; 3122 sum = tmpsum = 0; 3123 3124 if (is32bit) 3125 lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree)); 3126 else 3127 lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree)); 3128 3129 if (lb == NULL) /* Empty tree. */ 3130 lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) : 3131 (void *)ARB_ROOT(ctd64tree)); 3132 3133 /* 3134 * Find the set of centroids with minimum distance to x and 3135 * compute the sum of counts for all centroids with mean less 3136 * than the first centroid in the set. 3137 */ 3138 for (; cur != NULL; 3139 cur = (is32bit ? 3140 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 3141 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 3142 if (is32bit) { 3143 cnt = ctd32->cnt; 3144 KASSERT(Q_PRECEQ(ctd32->mu, x), 3145 ("%s: Q_RELPREC(mu,x)=%d", __func__, 3146 Q_RELPREC(ctd32->mu, x))); 3147 /* Ok to assign as both have same precision. */ 3148 z = ctd32->mu; 3149 } else { 3150 cnt = ctd64->cnt; 3151 KASSERT(Q_PRECEQ(ctd64->mu, x), 3152 ("%s: Q_RELPREC(mu,x)=%d", __func__, 3153 Q_RELPREC(ctd64->mu, x))); 3154 /* Ok to assign as both have same precision. */ 3155 z = ctd64->mu; 3156 } 3157 3158 error = Q_QSUBQ(&z, x); 3159 #if defined(DIAGNOSTIC) 3160 KASSERT(!error, ("%s: unexpected error %d", __func__, 3161 error)); 3162 #endif 3163 if (error) 3164 return (error); 3165 3166 z = Q_QABS(z); 3167 if (Q_QLTQ(z, minz)) { 3168 minz = z; 3169 lb = cur; 3170 sum = tmpsum; 3171 tmpsum += cnt; 3172 } else if (Q_QGTQ(z, minz)) { 3173 ub = cur; 3174 break; 3175 } 3176 } 3177 3178 cur = (is32bit ? 3179 (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) : 3180 (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb)); 3181 3182 for (n = 0; cur != ub; cur = (is32bit ? 3183 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 3184 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 3185 if (is32bit) 3186 cnt = ctd32->cnt; 3187 else 3188 cnt = ctd64->cnt; 3189 3190 q = Q_CTRLINI(16); 3191 if (smplcnt == 1) 3192 error = Q_QFRACI(&q, 1, 2); 3193 else 3194 /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */ 3195 error = Q_QFRACI(&q, (sum << 1) + cnt - 1, 3196 (smplcnt - 1) << 1); 3197 k = q; 3198 /* k = q x 4 x samplcnt x attempt */ 3199 error |= Q_QMULI(&k, 4 * smplcnt * attempt); 3200 /* k = k x (1 - q) */ 3201 error |= Q_QSUBI(&q, 1); 3202 q = Q_QABS(q); 3203 error |= Q_QMULQ(&k, q); 3204 #if defined(DIAGNOSTIC) 3205 #if !defined(_KERNEL) 3206 double q_dbl, k_dbl, q2d, k2d; 3207 q2d = Q_Q2D(q); 3208 k2d = Q_Q2D(k); 3209 q_dbl = smplcnt == 1 ? 0.5 : 3210 (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1); 3211 k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt; 3212 /* 3213 * If the difference between q and q_dbl is greater than 3214 * the fractional precision of q, something is off. 3215 * NB: q is holding the value of 1 - q 3216 */ 3217 q_dbl = 1.0 - q_dbl; 3218 KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) < 3219 (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))), 3220 ("Q-type q bad precision")); 3221 KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) < 3222 1.0 + (0.01 * smplcnt), 3223 ("Q-type k bad precision")); 3224 #endif /* !_KERNEL */ 3225 KASSERT(!error, ("%s: unexpected error %d", __func__, 3226 error)); 3227 #endif /* DIAGNOSTIC */ 3228 if (error) 3229 return (error); 3230 if ((is32bit && ((ctd32->cnt + weight) <= 3231 (uint64_t)Q_GIVAL(k))) || 3232 (!is32bit && ((ctd64->cnt + weight) <= 3233 (uint64_t)Q_GIVAL(k)))) { 3234 n++; 3235 /* random() produces 31 bits. */ 3236 if (random() < (INT32_MAX / n)) 3237 closest = cur; 3238 } 3239 sum += cnt; 3240 } 3241 } while (closest == NULL && 3242 (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) && 3243 (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst, 3244 attempt++)) == 0); 3245 3246 if (error) 3247 return (error); 3248 3249 if (closest != NULL) { 3250 /* Merge with an existing centroid. */ 3251 if (is32bit) { 3252 ctd32 = (struct voistatdata_tdgstctd32 *)closest; 3253 error = Q_QSUBQ(&x, ctd32->mu); 3254 /* 3255 * The following calculation "x / (cnt + weight)" 3256 * computes the amount by which to adjust the centroid's 3257 * mu value in order to merge in the VOI sample. 3258 * 3259 * It can underflow (Q_QDIVI() returns ERANGE) when the 3260 * user centroids' fractional precision (which is 3261 * inherited by 'x') is too low to represent the result. 3262 * 3263 * A sophisticated approach to dealing with this issue 3264 * would minimise accumulation of error by tracking 3265 * underflow per centroid and making an adjustment when 3266 * a LSB's worth of underflow has accumulated. 3267 * 3268 * A simpler approach is to let the result underflow 3269 * i.e. merge the VOI sample into the centroid without 3270 * adjusting the centroid's mu, and rely on the user to 3271 * specify their t-digest with sufficient centroid 3272 * fractional precision such that the accumulation of 3273 * error from multiple underflows is of no material 3274 * consequence to the centroid's final value of mu. 3275 * 3276 * For the moment, the latter approach is employed by 3277 * simply ignoring ERANGE here. 3278 * 3279 * XXXLAS: Per-centroid underflow tracking is likely too 3280 * onerous, but it probably makes sense to accumulate a 3281 * single underflow error variable across all centroids 3282 * and report it as part of the digest to provide 3283 * additional visibility into the digest's fidelity. 3284 */ 3285 error = error ? error : 3286 Q_QDIVI(&x, ctd32->cnt + weight); 3287 if ((error && error != ERANGE) 3288 || (error = Q_QADDQ(&ctd32->mu, x))) { 3289 #ifdef DIAGNOSTIC 3290 KASSERT(!error, ("%s: unexpected error %d", 3291 __func__, error)); 3292 #endif 3293 return (error); 3294 } 3295 ctd32->cnt += weight; 3296 error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) == 3297 NULL ? 0 : EALREADY; 3298 #ifdef DIAGNOSTIC 3299 RB_REINSERT(rbctdth32, 3300 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 3301 #endif 3302 } else { 3303 ctd64 = (struct voistatdata_tdgstctd64 *)closest; 3304 error = Q_QSUBQ(&x, ctd64->mu); 3305 error = error ? error : 3306 Q_QDIVI(&x, ctd64->cnt + weight); 3307 /* Refer to is32bit ERANGE discussion above. */ 3308 if ((error && error != ERANGE) 3309 || (error = Q_QADDQ(&ctd64->mu, x))) { 3310 KASSERT(!error, ("%s: unexpected error %d", 3311 __func__, error)); 3312 return (error); 3313 } 3314 ctd64->cnt += weight; 3315 error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) == 3316 NULL ? 0 : EALREADY; 3317 #ifdef DIAGNOSTIC 3318 RB_REINSERT(rbctdth64, 3319 &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64); 3320 #endif 3321 } 3322 } else { 3323 /* 3324 * Add a new centroid. If digest compression is working 3325 * correctly, there should always be at least one free. 3326 */ 3327 if (is32bit) { 3328 ctd32 = ARB_GETFREE(ctd32tree, ctdlnk); 3329 #ifdef DIAGNOSTIC 3330 KASSERT(ctd32 != NULL, 3331 ("%s: t-digest@%p has no free centroids", 3332 __func__, tdgst)); 3333 #endif 3334 if (ctd32 == NULL) 3335 return (EAGAIN); 3336 if ((error = Q_QCPYVALQ(&ctd32->mu, x))) 3337 return (error); 3338 ctd32->cnt = weight; 3339 error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ? 3340 0 : EALREADY; 3341 #ifdef DIAGNOSTIC 3342 RB_INSERT(rbctdth32, 3343 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 3344 #endif 3345 } else { 3346 ctd64 = ARB_GETFREE(ctd64tree, ctdlnk); 3347 #ifdef DIAGNOSTIC 3348 KASSERT(ctd64 != NULL, 3349 ("%s: t-digest@%p has no free centroids", 3350 __func__, tdgst)); 3351 #endif 3352 if (ctd64 == NULL) /* Should not happen. */ 3353 return (EAGAIN); 3354 /* Direct assignment ok as both have same type/prec. */ 3355 ctd64->mu = x; 3356 ctd64->cnt = weight; 3357 error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ? 3358 0 : EALREADY; 3359 #ifdef DIAGNOSTIC 3360 RB_INSERT(rbctdth64, &VSD(tdgstclust64, 3361 tdgst)->rbctdtree, ctd64); 3362 #endif 3363 } 3364 } 3365 3366 if (is32bit) 3367 VSD(tdgstclust32, tdgst)->smplcnt += weight; 3368 else { 3369 VSD(tdgstclust64, tdgst)->smplcnt += weight; 3370 3371 #ifdef DIAGNOSTIC 3372 struct rbctdth64 *rbctdtree = 3373 &VSD(tdgstclust64, tdgst)->rbctdtree; 3374 struct voistatdata_tdgstctd64 *rbctd64; 3375 int i = 0; 3376 ARB_FOREACH(ctd64, ctdth64, ctd64tree) { 3377 rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) : 3378 RB_NEXT(rbctdth64, rbctdtree, rbctd64)); 3379 3380 if (i >= ARB_CURNODES(ctd64tree) 3381 || ctd64 != rbctd64 3382 || ARB_MIN(ctdth64, ctd64tree) != 3383 RB_MIN(rbctdth64, rbctdtree) 3384 || ARB_MAX(ctdth64, ctd64tree) != 3385 RB_MAX(rbctdth64, rbctdtree) 3386 || ARB_LEFTIDX(ctd64, ctdlnk) != 3387 ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk)) 3388 || ARB_RIGHTIDX(ctd64, ctdlnk) != 3389 ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk)) 3390 || ARB_PARENTIDX(ctd64, ctdlnk) != 3391 ARB_SELFIDX(ctd64tree, 3392 RB_PARENT(rbctd64, rblnk))) { 3393 Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr)); 3394 printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 3395 "mu=%s\n", 3396 (int)ARB_SELFIDX(ctd64tree, ctd64), 3397 ARB_PARENTIDX(ctd64, ctdlnk), 3398 ARB_LEFTIDX(ctd64, ctdlnk), 3399 ARB_RIGHTIDX(ctd64, ctdlnk), 3400 ARB_COLOR(ctd64, ctdlnk), 3401 qstr); 3402 3403 Q_TOSTR(rbctd64->mu, -1, 10, qstr, 3404 sizeof(qstr)); 3405 struct voistatdata_tdgstctd64 *parent; 3406 parent = RB_PARENT(rbctd64, rblnk); 3407 int rb_color = 3408 parent == NULL ? 0 : 3409 RB_LEFT(parent, rblnk) == rbctd64 ? 3410 (_RB_BITSUP(parent, rblnk) & _RB_L) != 0 : 3411 (_RB_BITSUP(parent, rblnk) & _RB_R) != 0; 3412 printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 3413 "mu=%s\n", 3414 (int)ARB_SELFIDX(ctd64tree, rbctd64), 3415 (int)ARB_SELFIDX(ctd64tree, 3416 RB_PARENT(rbctd64, rblnk)), 3417 (int)ARB_SELFIDX(ctd64tree, 3418 RB_LEFT(rbctd64, rblnk)), 3419 (int)ARB_SELFIDX(ctd64tree, 3420 RB_RIGHT(rbctd64, rblnk)), 3421 rb_color, 3422 qstr); 3423 3424 panic("RB@%p and ARB@%p trees differ\n", 3425 rbctdtree, ctd64tree); 3426 } 3427 i++; 3428 } 3429 #endif /* DIAGNOSTIC */ 3430 } 3431 3432 return (error); 3433 } 3434 3435 static inline int 3436 stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival, 3437 struct voistat *vs, struct voistatdata_tdgst *tdgst) 3438 { 3439 s64q_t x; 3440 int error; 3441 3442 error = 0; 3443 3444 switch (vs->dtype) { 3445 case VSD_DTYPE_TDGSTCLUST32: 3446 /* Use same precision as the user's centroids. */ 3447 Q_INI(&x, 0, 0, Q_NFBITS( 3448 ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu)); 3449 break; 3450 case VSD_DTYPE_TDGSTCLUST64: 3451 /* Use same precision as the user's centroids. */ 3452 Q_INI(&x, 0, 0, Q_NFBITS( 3453 ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu)); 3454 break; 3455 default: 3456 KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 || 3457 vs->dtype == VSD_DTYPE_TDGSTCLUST64, 3458 ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>", 3459 __func__, vs->dtype)); 3460 return (EINVAL); 3461 } 3462 3463 /* 3464 * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid 3465 * returning EOVERFLOW if the voival would have fit in a u64q_t. 3466 */ 3467 switch (voi_dtype) { 3468 case VSD_DTYPE_INT_S32: 3469 error = Q_QCPYVALI(&x, voival->int32.s32); 3470 break; 3471 case VSD_DTYPE_INT_U32: 3472 error = Q_QCPYVALI(&x, voival->int32.u32); 3473 break; 3474 case VSD_DTYPE_INT_S64: 3475 error = Q_QCPYVALI(&x, voival->int64.s64); 3476 break; 3477 case VSD_DTYPE_INT_U64: 3478 error = Q_QCPYVALI(&x, voival->int64.u64); 3479 break; 3480 case VSD_DTYPE_INT_SLONG: 3481 error = Q_QCPYVALI(&x, voival->intlong.slong); 3482 break; 3483 case VSD_DTYPE_INT_ULONG: 3484 error = Q_QCPYVALI(&x, voival->intlong.ulong); 3485 break; 3486 case VSD_DTYPE_Q_S32: 3487 error = Q_QCPYVALQ(&x, voival->q32.sq32); 3488 break; 3489 case VSD_DTYPE_Q_U32: 3490 error = Q_QCPYVALQ(&x, voival->q32.uq32); 3491 break; 3492 case VSD_DTYPE_Q_S64: 3493 error = Q_QCPYVALQ(&x, voival->q64.sq64); 3494 break; 3495 case VSD_DTYPE_Q_U64: 3496 error = Q_QCPYVALQ(&x, voival->q64.uq64); 3497 break; 3498 default: 3499 error = EINVAL; 3500 break; 3501 } 3502 3503 if (error || 3504 (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1))) 3505 return (error); 3506 3507 vs->flags |= VS_VSDVALID; 3508 return (0); 3509 } 3510 3511 int 3512 stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id, 3513 enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags) 3514 { 3515 struct voi *v; 3516 struct voistat *vs; 3517 void *statevsd, *vsd; 3518 int error, i, tmperr; 3519 3520 error = 0; 3521 3522 if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) || 3523 voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL) 3524 return (EINVAL); 3525 v = &sb->vois[voi_id]; 3526 if (voi_dtype != v->dtype || v->id < 0 || 3527 ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE))) 3528 return (EINVAL); 3529 3530 vs = BLOB_OFFSET(sb, v->stats_off); 3531 if (v->flags & VOI_REQSTATE) 3532 statevsd = BLOB_OFFSET(sb, vs->data_off); 3533 else 3534 statevsd = NULL; 3535 3536 if (flags & SB_VOI_RELUPDATE) { 3537 switch (voi_dtype) { 3538 case VSD_DTYPE_INT_S32: 3539 voival->int32.s32 += 3540 VSD(voistate, statevsd)->prev.int32.s32; 3541 break; 3542 case VSD_DTYPE_INT_U32: 3543 voival->int32.u32 += 3544 VSD(voistate, statevsd)->prev.int32.u32; 3545 break; 3546 case VSD_DTYPE_INT_S64: 3547 voival->int64.s64 += 3548 VSD(voistate, statevsd)->prev.int64.s64; 3549 break; 3550 case VSD_DTYPE_INT_U64: 3551 voival->int64.u64 += 3552 VSD(voistate, statevsd)->prev.int64.u64; 3553 break; 3554 case VSD_DTYPE_INT_SLONG: 3555 voival->intlong.slong += 3556 VSD(voistate, statevsd)->prev.intlong.slong; 3557 break; 3558 case VSD_DTYPE_INT_ULONG: 3559 voival->intlong.ulong += 3560 VSD(voistate, statevsd)->prev.intlong.ulong; 3561 break; 3562 case VSD_DTYPE_Q_S32: 3563 error = Q_QADDQ(&voival->q32.sq32, 3564 VSD(voistate, statevsd)->prev.q32.sq32); 3565 break; 3566 case VSD_DTYPE_Q_U32: 3567 error = Q_QADDQ(&voival->q32.uq32, 3568 VSD(voistate, statevsd)->prev.q32.uq32); 3569 break; 3570 case VSD_DTYPE_Q_S64: 3571 error = Q_QADDQ(&voival->q64.sq64, 3572 VSD(voistate, statevsd)->prev.q64.sq64); 3573 break; 3574 case VSD_DTYPE_Q_U64: 3575 error = Q_QADDQ(&voival->q64.uq64, 3576 VSD(voistate, statevsd)->prev.q64.uq64); 3577 break; 3578 default: 3579 KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 3580 break; 3581 } 3582 } 3583 3584 if (error) 3585 return (error); 3586 3587 for (i = v->voistatmaxid; i > 0; i--) { 3588 vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i]; 3589 if (vs->stype < 0) 3590 continue; 3591 3592 vsd = BLOB_OFFSET(sb, vs->data_off); 3593 3594 switch (vs->stype) { 3595 case VS_STYPE_MAX: 3596 tmperr = stats_v1_voi_update_max(voi_dtype, voival, 3597 vs, vsd); 3598 break; 3599 case VS_STYPE_MIN: 3600 tmperr = stats_v1_voi_update_min(voi_dtype, voival, 3601 vs, vsd); 3602 break; 3603 case VS_STYPE_SUM: 3604 tmperr = stats_v1_voi_update_sum(voi_dtype, voival, 3605 vs, vsd); 3606 break; 3607 case VS_STYPE_HIST: 3608 tmperr = stats_v1_voi_update_hist(voi_dtype, voival, 3609 vs, vsd); 3610 break; 3611 case VS_STYPE_TDGST: 3612 tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival, 3613 vs, vsd); 3614 break; 3615 default: 3616 KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 3617 break; 3618 } 3619 3620 if (tmperr) { 3621 error = tmperr; 3622 VS_INCERRS(vs); 3623 } 3624 } 3625 3626 if (statevsd) { 3627 switch (voi_dtype) { 3628 case VSD_DTYPE_INT_S32: 3629 VSD(voistate, statevsd)->prev.int32.s32 = 3630 voival->int32.s32; 3631 break; 3632 case VSD_DTYPE_INT_U32: 3633 VSD(voistate, statevsd)->prev.int32.u32 = 3634 voival->int32.u32; 3635 break; 3636 case VSD_DTYPE_INT_S64: 3637 VSD(voistate, statevsd)->prev.int64.s64 = 3638 voival->int64.s64; 3639 break; 3640 case VSD_DTYPE_INT_U64: 3641 VSD(voistate, statevsd)->prev.int64.u64 = 3642 voival->int64.u64; 3643 break; 3644 case VSD_DTYPE_INT_SLONG: 3645 VSD(voistate, statevsd)->prev.intlong.slong = 3646 voival->intlong.slong; 3647 break; 3648 case VSD_DTYPE_INT_ULONG: 3649 VSD(voistate, statevsd)->prev.intlong.ulong = 3650 voival->intlong.ulong; 3651 break; 3652 case VSD_DTYPE_Q_S32: 3653 error = Q_QCPYVALQ( 3654 &VSD(voistate, statevsd)->prev.q32.sq32, 3655 voival->q32.sq32); 3656 break; 3657 case VSD_DTYPE_Q_U32: 3658 error = Q_QCPYVALQ( 3659 &VSD(voistate, statevsd)->prev.q32.uq32, 3660 voival->q32.uq32); 3661 break; 3662 case VSD_DTYPE_Q_S64: 3663 error = Q_QCPYVALQ( 3664 &VSD(voistate, statevsd)->prev.q64.sq64, 3665 voival->q64.sq64); 3666 break; 3667 case VSD_DTYPE_Q_U64: 3668 error = Q_QCPYVALQ( 3669 &VSD(voistate, statevsd)->prev.q64.uq64, 3670 voival->q64.uq64); 3671 break; 3672 default: 3673 KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 3674 break; 3675 } 3676 } 3677 3678 return (error); 3679 } 3680 3681 #ifdef _KERNEL 3682 3683 static void 3684 stats_init(void *arg) 3685 { 3686 3687 } 3688 SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL); 3689 3690 /* 3691 * Sysctl handler to display the list of available stats templates. 3692 */ 3693 static int 3694 stats_tpl_list_available(SYSCTL_HANDLER_ARGS) 3695 { 3696 struct sbuf *s; 3697 int err, i; 3698 3699 err = 0; 3700 3701 /* We can tolerate ntpl being stale, so do not take the lock. */ 3702 s = sbuf_new(NULL, NULL, /* +1 per tpl for , */ 3703 ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN); 3704 if (s == NULL) 3705 return (ENOMEM); 3706 3707 TPL_LIST_RLOCK(); 3708 for (i = 0; i < ntpl; i++) { 3709 err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "", 3710 tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash); 3711 if (err) { 3712 /* Sbuf overflow condition. */ 3713 err = EOVERFLOW; 3714 break; 3715 } 3716 } 3717 TPL_LIST_RUNLOCK(); 3718 3719 if (!err) { 3720 sbuf_finish(s); 3721 err = sysctl_handle_string(oidp, sbuf_data(s), 0, req); 3722 } 3723 3724 sbuf_delete(s); 3725 return (err); 3726 } 3727 3728 /* 3729 * Called by subsystem-specific sysctls to report and/or parse the list of 3730 * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t 3731 * conformant function pointer must be passed in as arg1, which is used to 3732 * interact with the subsystem's stats template sample rates list. If arg2 > 0, 3733 * a zero-initialised allocation of arg2-sized contextual memory is 3734 * heap-allocated and passed in to all subsystem callbacks made during the 3735 * operation of stats_tpl_sample_rates(). 3736 * 3737 * XXXLAS: Assumes templates are never removed, which is currently true but may 3738 * need to be reworked in future if dynamic template management becomes a 3739 * requirement e.g. to support kernel module based templates. 3740 */ 3741 int 3742 stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS) 3743 { 3744 char kvpair_fmt[16], tplspec_fmt[16]; 3745 char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN]; 3746 char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */ 3747 stats_tpl_sr_cb_t subsys_cb; 3748 void *subsys_ctx; 3749 char *buf, *new_rates_usr_str, *tpl_name_p; 3750 struct stats_tpl_sample_rate *rates; 3751 struct sbuf *s, _s; 3752 uint32_t cum_pct, pct, tpl_hash; 3753 int err, i, off, len, newlen, nrates; 3754 3755 buf = NULL; 3756 rates = NULL; 3757 err = nrates = 0; 3758 subsys_cb = (stats_tpl_sr_cb_t)arg1; 3759 KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__)); 3760 if (arg2 > 0) 3761 subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO); 3762 else 3763 subsys_ctx = NULL; 3764 3765 /* Grab current count of subsystem rates. */ 3766 err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx); 3767 if (err) 3768 goto done; 3769 3770 /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */ 3771 len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5)); 3772 3773 if (req->oldptr != NULL || req->newptr != NULL) 3774 buf = malloc(len, M_TEMP, M_WAITOK); 3775 3776 if (req->oldptr != NULL) { 3777 if (nrates == 0) { 3778 /* No rates, so return an empty string via oldptr. */ 3779 err = SYSCTL_OUT(req, "", 1); 3780 if (err) 3781 goto done; 3782 goto process_new; 3783 } 3784 3785 s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL); 3786 3787 /* Grab locked count of, and ptr to, subsystem rates. */ 3788 err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates, 3789 subsys_ctx); 3790 if (err) 3791 goto done; 3792 TPL_LIST_RLOCK(); 3793 for (i = 0; i < nrates && !err; i++) { 3794 err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "", 3795 tpllist[rates[i].tpl_slot_id]->mb->tplname, 3796 tpllist[rates[i].tpl_slot_id]->mb->tplhash, 3797 rates[i].tpl_sample_pct); 3798 } 3799 TPL_LIST_RUNLOCK(); 3800 /* Tell subsystem that we're done with its rates list. */ 3801 err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx); 3802 if (err) 3803 goto done; 3804 3805 err = sbuf_finish(s); 3806 if (err) 3807 goto done; /* We lost a race for buf to be too small. */ 3808 3809 /* Return the rendered string data via oldptr. */ 3810 err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s)); 3811 } else { 3812 /* Return the upper bound size for buffer sizing requests. */ 3813 err = SYSCTL_OUT(req, NULL, len); 3814 } 3815 3816 process_new: 3817 if (err || req->newptr == NULL) 3818 goto done; 3819 3820 newlen = req->newlen - req->newidx; 3821 err = SYSCTL_IN(req, buf, newlen); 3822 if (err) 3823 goto done; 3824 3825 /* 3826 * Initialise format strings at run time. 3827 * 3828 * Write the max template spec string length into the 3829 * template_spec=percent key-value pair parsing format string as: 3830 * " %<width>[^=]=%u %n" 3831 * 3832 * Write the max template name string length into the tplname:tplhash 3833 * parsing format string as: 3834 * "%<width>[^:]:%u" 3835 * 3836 * Subtract 1 for \0 appended by sscanf(). 3837 */ 3838 sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1); 3839 sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1); 3840 3841 /* 3842 * Parse each CSV key-value pair specifying a template and its sample 3843 * percentage. Whitespace either side of a key-value pair is ignored. 3844 * Templates can be specified by name, hash, or name and hash per the 3845 * following formats (chars in [] are optional): 3846 * ["]<tplname>["]=<percent> 3847 * :hash=pct 3848 * ["]<tplname>["]:hash=<percent> 3849 */ 3850 cum_pct = nrates = 0; 3851 rates = NULL; 3852 buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */ 3853 new_rates_usr_str = buf; 3854 while (isspace(*new_rates_usr_str)) 3855 new_rates_usr_str++; /* Skip leading whitespace. */ 3856 while (*new_rates_usr_str != '\0') { 3857 tpl_name_p = tpl_name; 3858 tpl_name[0] = '\0'; 3859 tpl_hash = 0; 3860 off = 0; 3861 3862 /* 3863 * Parse key-value pair which must perform 2 conversions, then 3864 * parse the template spec to extract either name, hash, or name 3865 * and hash depending on the three possible spec formats. The 3866 * tplspec_fmt format specifier parses name or name and hash 3867 * template specs, while the ":%u" format specifier parses 3868 * hash-only template specs. If parsing is successfull, ensure 3869 * the cumulative sampling percentage does not exceed 100. 3870 */ 3871 err = EINVAL; 3872 if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct, 3873 &off)) 3874 break; 3875 if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) && 3876 (1 != sscanf(tpl_spec, ":%u", &tpl_hash))) 3877 break; 3878 if ((cum_pct += pct) > 100) 3879 break; 3880 err = 0; 3881 3882 /* Strip surrounding "" from template name if present. */ 3883 len = strlen(tpl_name); 3884 if (len > 0) { 3885 if (tpl_name[len - 1] == '"') 3886 tpl_name[--len] = '\0'; 3887 if (tpl_name[0] == '"') { 3888 tpl_name_p++; 3889 len--; 3890 } 3891 } 3892 3893 rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */ 3894 (nrates + 1) * sizeof(*rates), M_WAITOK); 3895 rates[nrates].tpl_slot_id = 3896 stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash); 3897 if (rates[nrates].tpl_slot_id < 0) { 3898 err = -rates[nrates].tpl_slot_id; 3899 break; 3900 } 3901 rates[nrates].tpl_sample_pct = pct; 3902 nrates++; 3903 new_rates_usr_str += off; 3904 if (*new_rates_usr_str != ',') 3905 break; /* End-of-input or malformed. */ 3906 new_rates_usr_str++; /* Move past comma to next pair. */ 3907 } 3908 3909 if (!err) { 3910 if ((new_rates_usr_str - buf) < newlen) { 3911 /* Entire input has not been consumed. */ 3912 err = EINVAL; 3913 } else { 3914 /* 3915 * Give subsystem the new rates. They'll return the 3916 * appropriate rates pointer for us to garbage collect. 3917 */ 3918 err = subsys_cb(TPL_SR_PUT, &rates, &nrates, 3919 subsys_ctx); 3920 } 3921 } 3922 stats_free(rates); 3923 3924 done: 3925 free(buf, M_TEMP); 3926 free(subsys_ctx, M_TEMP); 3927 return (err); 3928 } 3929 3930 SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 3931 "stats(9) MIB"); 3932 3933 SYSCTL_PROC(_kern_stats, OID_AUTO, templates, 3934 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 3935 stats_tpl_list_available, "A", 3936 "list the name/hash of all available stats(9) templates"); 3937 3938 #else /* ! _KERNEL */ 3939 3940 static void __attribute__ ((constructor)) 3941 stats_constructor(void) 3942 { 3943 3944 pthread_rwlock_init(&tpllistlock, NULL); 3945 } 3946 3947 static void __attribute__ ((destructor)) 3948 stats_destructor(void) 3949 { 3950 3951 pthread_rwlock_destroy(&tpllistlock); 3952 } 3953 3954 #endif /* _KERNEL */ 3955