1 /*- 2 * Copyright (c) 2014-2018 Netflix, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * Author: Lawrence Stewart <lstewart@netflix.com> 29 */ 30 31 #include <sys/param.h> 32 #include <sys/arb.h> 33 #include <sys/ctype.h> 34 #include <sys/errno.h> 35 #include <sys/hash.h> 36 #include <sys/limits.h> 37 #include <sys/malloc.h> 38 #include <sys/qmath.h> 39 #include <sys/sbuf.h> 40 #if defined(DIAGNOSTIC) 41 #include <sys/tree.h> 42 #endif 43 #include <sys/stats.h> /* Must come after qmath.h and arb.h */ 44 #include <sys/stddef.h> 45 #include <sys/stdint.h> 46 #include <sys/time.h> 47 48 #ifdef _KERNEL 49 #include <sys/kernel.h> 50 #include <sys/lock.h> 51 #include <sys/rwlock.h> 52 #include <sys/sysctl.h> 53 #include <sys/systm.h> 54 #else /* ! _KERNEL */ 55 #include <pthread.h> 56 #include <stdbool.h> 57 #include <stdio.h> 58 #include <stdlib.h> 59 #include <string.h> 60 #endif /* _KERNEL */ 61 62 struct voistatdata_voistate { 63 /* Previous VOI value for diff calculation. */ 64 struct voistatdata_numeric prev; 65 }; 66 67 #define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */ 68 struct voistat { 69 int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */ 70 enum vsd_dtype dtype : 8; /* Data type of this stat's data. */ 71 uint16_t data_off; /* Blob offset for this stat's data. */ 72 uint16_t dsz; /* Size of stat's data. */ 73 #define VS_EBITS 8 74 uint16_t errs : VS_EBITS;/* Non-wrapping error count. */ 75 uint16_t flags : 16 - VS_EBITS; 76 }; 77 /* The voistat error count is capped to avoid wrapping. */ 78 #define VS_INCERRS(vs) do { \ 79 if ((vs)->errs < (1U << VS_EBITS) - 1) \ 80 (vs)->errs++; \ 81 } while (0) 82 83 /* 84 * Ideas for flags: 85 * - Global or entity specific (global would imply use of counter(9)?) 86 * - Whether to reset stats on read or not 87 * - Signal an overflow? 88 * - Compressed voistat array 89 */ 90 #define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */ 91 struct voi { 92 int16_t id; /* VOI id. */ 93 enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */ 94 int8_t voistatmaxid; /* Largest allocated voistat index. */ 95 uint16_t stats_off; /* Blob offset for this VOIs stats. */ 96 uint16_t flags; 97 }; 98 99 /* 100 * Memory for the entire blob is allocated as a slab and then offsets are 101 * maintained to carve up the slab into sections holding different data types. 102 * 103 * Ideas for flags: 104 * - Compressed voi array (trade off memory usage vs search time) 105 * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib) 106 */ 107 struct statsblobv1 { 108 uint8_t abi; 109 uint8_t endian; 110 uint16_t flags; 111 uint16_t maxsz; 112 uint16_t cursz; 113 /* Fields from here down are opaque to consumers. */ 114 uint32_t tplhash; /* Base template hash ID. */ 115 uint16_t stats_off; /* voistat array blob offset. */ 116 uint16_t statsdata_off; /* voistatdata array blob offset. */ 117 sbintime_t created; /* Blob creation time. */ 118 sbintime_t lastrst; /* Time of last reset. */ 119 struct voi vois[]; /* Array indexed by [voi_id]. */ 120 } __aligned(sizeof(void *)); 121 _Static_assert(offsetof(struct statsblobv1, cursz) + 122 SIZEOF_MEMBER(struct statsblobv1, cursz) == 123 offsetof(struct statsblob, opaque), 124 "statsblobv1 ABI mismatch"); 125 126 struct statsblobv1_tpl { 127 struct metablob *mb; 128 struct statsblobv1 *sb; 129 }; 130 131 /* Context passed to iterator callbacks. */ 132 struct sb_iter_ctx { 133 void *usrctx; /* Caller supplied context. */ 134 uint32_t flags; /* Flags for current iteration. */ 135 int16_t vslot; /* struct voi slot index. */ 136 int8_t vsslot; /* struct voistat slot index. */ 137 }; 138 139 struct sb_tostrcb_ctx { 140 struct sbuf *buf; 141 struct statsblob_tpl *tpl; 142 enum sb_str_fmt fmt; 143 uint32_t flags; 144 }; 145 146 struct sb_visitcb_ctx { 147 stats_blob_visitcb_t cb; 148 void *usrctx; 149 }; 150 151 /* Stats blob iterator callback. */ 152 typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v, 153 struct voistat *vs, struct sb_iter_ctx *ctx); 154 155 #ifdef _KERNEL 156 static struct rwlock tpllistlock; 157 RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock"); 158 #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock) 159 #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock) 160 #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock) 161 #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock) 162 #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED) 163 #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED) 164 #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED) 165 MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory"); 166 #define stats_free(ptr) free((ptr), M_STATS) 167 #else /* ! _KERNEL */ 168 static void stats_constructor(void); 169 static void stats_destructor(void); 170 static pthread_rwlock_t tpllistlock; 171 #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock) 172 #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock) 173 #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK() 174 #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock) 175 #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK() 176 #define TPL_LIST_LOCK_ASSERT() do { } while (0) 177 #define TPL_LIST_RLOCK_ASSERT() do { } while (0) 178 #define TPL_LIST_WLOCK_ASSERT() do { } while (0) 179 #ifdef NDEBUG 180 #define KASSERT(cond, msg) do {} while (0) 181 #define stats_abort() do {} while (0) 182 #else /* ! NDEBUG */ 183 #define KASSERT(cond, msg) do { \ 184 if (!(cond)) { \ 185 panic msg; \ 186 } \ 187 } while (0) 188 #define stats_abort() abort() 189 #endif /* NDEBUG */ 190 #define stats_free(ptr) free(ptr) 191 #define panic(fmt, ...) do { \ 192 fprintf(stderr, (fmt), ##__VA_ARGS__); \ 193 stats_abort(); \ 194 } while (0) 195 #endif /* _KERNEL */ 196 197 #define SB_V1_MAXSZ 65535 198 199 /* Obtain a blob offset pointer. */ 200 #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off))) 201 202 /* 203 * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a 204 * power of 2 size, we can shift instead of divide. The shift amount must be 205 * updated if sizeof(struct voi) ever changes, which the assert should catch. 206 */ 207 #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \ 208 sizeof(struct statsblobv1)) >> 3)) 209 _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch"); 210 211 /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */ 212 const char *vs_stype2name[VS_NUM_STYPES] = { 213 [VS_STYPE_VOISTATE] = "VOISTATE", 214 [VS_STYPE_SUM] = "SUM", 215 [VS_STYPE_MAX] = "MAX", 216 [VS_STYPE_MIN] = "MIN", 217 [VS_STYPE_HIST] = "HIST", 218 [VS_STYPE_TDGST] = "TDGST", 219 }; 220 221 const char *vs_stype2desc[VS_NUM_STYPES] = { 222 [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)", 223 [VS_STYPE_SUM] = "Simple arithmetic accumulator", 224 [VS_STYPE_MAX] = "Maximum observed VOI value", 225 [VS_STYPE_MIN] = "Minimum observed VOI value", 226 [VS_STYPE_HIST] = "Histogram of observed VOI values", 227 [VS_STYPE_TDGST] = "t-digest of observed VOI values", 228 }; 229 230 const char *vsd_dtype2name[VSD_NUM_DTYPES] = { 231 [VSD_DTYPE_VOISTATE] = "VOISTATE", 232 [VSD_DTYPE_INT_S32] = "INT_S32", 233 [VSD_DTYPE_INT_U32] = "INT_U32", 234 [VSD_DTYPE_INT_S64] = "INT_S64", 235 [VSD_DTYPE_INT_U64] = "INT_U64", 236 [VSD_DTYPE_INT_SLONG] = "INT_SLONG", 237 [VSD_DTYPE_INT_ULONG] = "INT_ULONG", 238 [VSD_DTYPE_Q_S32] = "Q_S32", 239 [VSD_DTYPE_Q_U32] = "Q_U32", 240 [VSD_DTYPE_Q_S64] = "Q_S64", 241 [VSD_DTYPE_Q_U64] = "Q_U64", 242 [VSD_DTYPE_CRHIST32] = "CRHIST32", 243 [VSD_DTYPE_DRHIST32] = "DRHIST32", 244 [VSD_DTYPE_DVHIST32] = "DVHIST32", 245 [VSD_DTYPE_CRHIST64] = "CRHIST64", 246 [VSD_DTYPE_DRHIST64] = "DRHIST64", 247 [VSD_DTYPE_DVHIST64] = "DVHIST64", 248 [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32", 249 [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64", 250 }; 251 252 const size_t vsd_dtype2size[VSD_NUM_DTYPES] = { 253 [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate), 254 [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32), 255 [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32), 256 [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64), 257 [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64), 258 [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong), 259 [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong), 260 [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32), 261 [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32), 262 [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64), 263 [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64), 264 [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32), 265 [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32), 266 [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32), 267 [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64), 268 [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64), 269 [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64), 270 [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32), 271 [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64), 272 }; 273 274 static const bool vsd_compoundtype[VSD_NUM_DTYPES] = { 275 [VSD_DTYPE_VOISTATE] = true, 276 [VSD_DTYPE_INT_S32] = false, 277 [VSD_DTYPE_INT_U32] = false, 278 [VSD_DTYPE_INT_S64] = false, 279 [VSD_DTYPE_INT_U64] = false, 280 [VSD_DTYPE_INT_SLONG] = false, 281 [VSD_DTYPE_INT_ULONG] = false, 282 [VSD_DTYPE_Q_S32] = false, 283 [VSD_DTYPE_Q_U32] = false, 284 [VSD_DTYPE_Q_S64] = false, 285 [VSD_DTYPE_Q_U64] = false, 286 [VSD_DTYPE_CRHIST32] = true, 287 [VSD_DTYPE_DRHIST32] = true, 288 [VSD_DTYPE_DVHIST32] = true, 289 [VSD_DTYPE_CRHIST64] = true, 290 [VSD_DTYPE_DRHIST64] = true, 291 [VSD_DTYPE_DVHIST64] = true, 292 [VSD_DTYPE_TDGSTCLUST32] = true, 293 [VSD_DTYPE_TDGSTCLUST64] = true, 294 }; 295 296 const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = { 297 [LIM_MIN] = { 298 [VSD_DTYPE_VOISTATE] = {0}, 299 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}}, 300 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}}, 301 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}}, 302 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}}, 303 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}}, 304 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}}, 305 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}}, 306 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}}, 307 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}}, 308 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}}, 309 }, 310 [LIM_MAX] = { 311 [VSD_DTYPE_VOISTATE] = {0}, 312 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}}, 313 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}}, 314 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}}, 315 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}}, 316 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}}, 317 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}}, 318 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}}, 319 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}}, 320 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}}, 321 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}}, 322 } 323 }; 324 325 /* tpllistlock protects tpllist and ntpl */ 326 static uint32_t ntpl; 327 static struct statsblob_tpl **tpllist; 328 329 static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz, 330 int flags); 331 //static void stats_v1_blob_finalise(struct statsblobv1 *sb); 332 static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 333 uint32_t flags); 334 static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 335 int newvoistatbytes, int newvoistatdatabytes); 336 static void stats_v1_blob_iter(struct statsblobv1 *sb, 337 stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags); 338 static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, 339 struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt); 340 341 static inline int 342 ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2) 343 { 344 345 KASSERT(Q_PRECEQ(c1->mu, c2->mu), 346 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 347 Q_RELPREC(c1->mu, c2->mu))); 348 349 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 350 } 351 ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp); 352 353 static inline int 354 ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2) 355 { 356 357 KASSERT(Q_PRECEQ(c1->mu, c2->mu), 358 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, 359 Q_RELPREC(c1->mu, c2->mu))); 360 361 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); 362 } 363 ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp); 364 365 #ifdef DIAGNOSTIC 366 RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp); 367 RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp); 368 #endif 369 370 static inline sbintime_t 371 stats_sbinuptime(void) 372 { 373 sbintime_t sbt; 374 #ifdef _KERNEL 375 376 sbt = sbinuptime(); 377 #else /* ! _KERNEL */ 378 struct timespec tp; 379 380 clock_gettime(CLOCK_MONOTONIC_FAST, &tp); 381 sbt = tstosbt(tp); 382 #endif /* _KERNEL */ 383 384 return (sbt); 385 } 386 387 static inline void * 388 stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags) 389 { 390 391 #ifdef _KERNEL 392 /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */ 393 if (!(flags & (M_WAITOK | M_NOWAIT))) 394 flags |= M_NOWAIT; 395 ptr = realloc(ptr, newsz, M_STATS, flags); 396 #else /* ! _KERNEL */ 397 ptr = realloc(ptr, newsz); 398 if ((flags & M_ZERO) && ptr != NULL) { 399 if (oldsz == 0) 400 memset(ptr, '\0', newsz); 401 else if (newsz > oldsz) 402 memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz); 403 } 404 #endif /* _KERNEL */ 405 406 return (ptr); 407 } 408 409 static inline char * 410 stats_strdup(const char *s, 411 #ifdef _KERNEL 412 int flags) 413 { 414 char *copy; 415 size_t len; 416 417 if (!(flags & (M_WAITOK | M_NOWAIT))) 418 flags |= M_NOWAIT; 419 420 len = strlen(s) + 1; 421 if ((copy = malloc(len, M_STATS, flags)) != NULL) 422 bcopy(s, copy, len); 423 424 return (copy); 425 #else 426 int flags __unused) 427 { 428 return (strdup(s)); 429 #endif 430 } 431 432 static inline void 433 stats_tpl_update_hash(struct statsblob_tpl *tpl) 434 { 435 436 TPL_LIST_WLOCK_ASSERT(); 437 tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0); 438 for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) { 439 if (tpl->mb->voi_meta[voi_id].name != NULL) 440 tpl->mb->tplhash = hash32_str( 441 tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash); 442 } 443 tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz, 444 tpl->mb->tplhash); 445 } 446 447 static inline uint64_t 448 stats_pow_u64(uint64_t base, uint64_t exp) 449 { 450 uint64_t result = 1; 451 452 while (exp) { 453 if (exp & 1) 454 result *= base; 455 exp >>= 1; 456 base *= base; 457 } 458 459 return (result); 460 } 461 462 static inline int 463 stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt, 464 struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub) 465 { 466 uint64_t step = 0; 467 int error = 0; 468 469 switch (info->scheme) { 470 case BKT_LIN: 471 step = info->lin.stepinc; 472 break; 473 case BKT_EXP: 474 step = stats_pow_u64(info->exp.stepbase, 475 info->exp.stepexp + curbkt); 476 break; 477 case BKT_LINEXP: 478 { 479 uint64_t curstepexp = 1; 480 481 switch (info->voi_dtype) { 482 case VSD_DTYPE_INT_S32: 483 while ((int32_t)stats_pow_u64(info->linexp.stepbase, 484 curstepexp) <= bkt_lb->int32.s32) 485 curstepexp++; 486 break; 487 case VSD_DTYPE_INT_U32: 488 while ((uint32_t)stats_pow_u64(info->linexp.stepbase, 489 curstepexp) <= bkt_lb->int32.u32) 490 curstepexp++; 491 break; 492 case VSD_DTYPE_INT_S64: 493 while ((int64_t)stats_pow_u64(info->linexp.stepbase, 494 curstepexp) <= bkt_lb->int64.s64) 495 curstepexp++; 496 break; 497 case VSD_DTYPE_INT_U64: 498 while ((uint64_t)stats_pow_u64(info->linexp.stepbase, 499 curstepexp) <= bkt_lb->int64.u64) 500 curstepexp++; 501 break; 502 case VSD_DTYPE_INT_SLONG: 503 while ((long)stats_pow_u64(info->linexp.stepbase, 504 curstepexp) <= bkt_lb->intlong.slong) 505 curstepexp++; 506 break; 507 case VSD_DTYPE_INT_ULONG: 508 while ((unsigned long)stats_pow_u64(info->linexp.stepbase, 509 curstepexp) <= bkt_lb->intlong.ulong) 510 curstepexp++; 511 break; 512 case VSD_DTYPE_Q_S32: 513 while ((s32q_t)stats_pow_u64(info->linexp.stepbase, 514 curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32)) 515 break; 516 case VSD_DTYPE_Q_U32: 517 while ((u32q_t)stats_pow_u64(info->linexp.stepbase, 518 curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32)) 519 break; 520 case VSD_DTYPE_Q_S64: 521 while ((s64q_t)stats_pow_u64(info->linexp.stepbase, 522 curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64)) 523 curstepexp++; 524 break; 525 case VSD_DTYPE_Q_U64: 526 while ((u64q_t)stats_pow_u64(info->linexp.stepbase, 527 curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64)) 528 curstepexp++; 529 break; 530 default: 531 break; 532 } 533 534 step = stats_pow_u64(info->linexp.stepbase, curstepexp) / 535 info->linexp.linstepdiv; 536 if (step == 0) 537 step = 1; 538 break; 539 } 540 default: 541 break; 542 } 543 544 if (info->scheme == BKT_USR) { 545 *bkt_lb = info->usr.bkts[curbkt].lb; 546 *bkt_ub = info->usr.bkts[curbkt].ub; 547 } else if (step != 0) { 548 switch (info->voi_dtype) { 549 case VSD_DTYPE_INT_S32: 550 bkt_ub->int32.s32 += (int32_t)step; 551 break; 552 case VSD_DTYPE_INT_U32: 553 bkt_ub->int32.u32 += (uint32_t)step; 554 break; 555 case VSD_DTYPE_INT_S64: 556 bkt_ub->int64.s64 += (int64_t)step; 557 break; 558 case VSD_DTYPE_INT_U64: 559 bkt_ub->int64.u64 += (uint64_t)step; 560 break; 561 case VSD_DTYPE_INT_SLONG: 562 bkt_ub->intlong.slong += (long)step; 563 break; 564 case VSD_DTYPE_INT_ULONG: 565 bkt_ub->intlong.ulong += (unsigned long)step; 566 break; 567 case VSD_DTYPE_Q_S32: 568 error = Q_QADDI(&bkt_ub->q32.sq32, step); 569 break; 570 case VSD_DTYPE_Q_U32: 571 error = Q_QADDI(&bkt_ub->q32.uq32, step); 572 break; 573 case VSD_DTYPE_Q_S64: 574 error = Q_QADDI(&bkt_ub->q64.sq64, step); 575 break; 576 case VSD_DTYPE_Q_U64: 577 error = Q_QADDI(&bkt_ub->q64.uq64, step); 578 break; 579 default: 580 break; 581 } 582 } else { /* info->scheme != BKT_USR && step == 0 */ 583 return (EINVAL); 584 } 585 586 return (error); 587 } 588 589 static uint32_t 590 stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info) 591 { 592 struct voistatdata_numeric bkt_lb, bkt_ub; 593 uint32_t nbkts; 594 int done; 595 596 if (info->scheme == BKT_USR) { 597 /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */ 598 info->lb = info->usr.bkts[0].lb; 599 info->ub = info->usr.bkts[info->usr.nbkts - 1].lb; 600 } 601 602 nbkts = 0; 603 done = 0; 604 bkt_ub = info->lb; 605 606 do { 607 bkt_lb = bkt_ub; 608 if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub)) 609 return (0); 610 611 if (info->scheme == BKT_USR) 612 done = (nbkts == info->usr.nbkts); 613 else { 614 switch (info->voi_dtype) { 615 case VSD_DTYPE_INT_S32: 616 done = (bkt_ub.int32.s32 > info->ub.int32.s32); 617 break; 618 case VSD_DTYPE_INT_U32: 619 done = (bkt_ub.int32.u32 > info->ub.int32.u32); 620 break; 621 case VSD_DTYPE_INT_S64: 622 done = (bkt_ub.int64.s64 > info->ub.int64.s64); 623 break; 624 case VSD_DTYPE_INT_U64: 625 done = (bkt_ub.int64.u64 > info->ub.int64.u64); 626 break; 627 case VSD_DTYPE_INT_SLONG: 628 done = (bkt_ub.intlong.slong > 629 info->ub.intlong.slong); 630 break; 631 case VSD_DTYPE_INT_ULONG: 632 done = (bkt_ub.intlong.ulong > 633 info->ub.intlong.ulong); 634 break; 635 case VSD_DTYPE_Q_S32: 636 done = Q_QGTQ(bkt_ub.q32.sq32, 637 info->ub.q32.sq32); 638 break; 639 case VSD_DTYPE_Q_U32: 640 done = Q_QGTQ(bkt_ub.q32.uq32, 641 info->ub.q32.uq32); 642 break; 643 case VSD_DTYPE_Q_S64: 644 done = Q_QGTQ(bkt_ub.q64.sq64, 645 info->ub.q64.sq64); 646 break; 647 case VSD_DTYPE_Q_U64: 648 done = Q_QGTQ(bkt_ub.q64.uq64, 649 info->ub.q64.uq64); 650 break; 651 default: 652 return (0); 653 } 654 } 655 } while (!done); 656 657 if (info->flags & VSD_HIST_LBOUND_INF) 658 nbkts++; 659 if (info->flags & VSD_HIST_UBOUND_INF) 660 nbkts++; 661 662 return (nbkts); 663 } 664 665 int 666 stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 667 struct vss_hist_hlpr_info *info) 668 { 669 struct voistatdata_hist *hist; 670 struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub, 671 *ubinfbktlb, *ubinfbktub; 672 uint32_t bkt, nbkts, nloop; 673 674 if (vss == NULL || info == NULL || (info->flags & 675 (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype == 676 VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64))) 677 return (EINVAL); 678 679 info->voi_dtype = voi_dtype; 680 681 if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0) 682 return (EINVAL); 683 684 switch (info->hist_dtype) { 685 case VSD_DTYPE_CRHIST32: 686 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts); 687 break; 688 case VSD_DTYPE_DRHIST32: 689 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts); 690 break; 691 case VSD_DTYPE_DVHIST32: 692 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts); 693 break; 694 case VSD_DTYPE_CRHIST64: 695 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts); 696 break; 697 case VSD_DTYPE_DRHIST64: 698 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts); 699 break; 700 case VSD_DTYPE_DVHIST64: 701 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts); 702 break; 703 default: 704 return (EINVAL); 705 } 706 707 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 708 if (vss->iv == NULL) 709 return (ENOMEM); 710 711 hist = (struct voistatdata_hist *)vss->iv; 712 bkt_ub = info->lb; 713 714 for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0; 715 bkt < nbkts; 716 bkt++, nloop++) { 717 bkt_lb = bkt_ub; 718 if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub)) 719 return (EINVAL); 720 721 switch (info->hist_dtype) { 722 case VSD_DTYPE_CRHIST32: 723 VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb; 724 break; 725 case VSD_DTYPE_DRHIST32: 726 VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb; 727 VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub; 728 break; 729 case VSD_DTYPE_DVHIST32: 730 VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb; 731 break; 732 case VSD_DTYPE_CRHIST64: 733 VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb; 734 break; 735 case VSD_DTYPE_DRHIST64: 736 VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb; 737 VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub; 738 break; 739 case VSD_DTYPE_DVHIST64: 740 VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb; 741 break; 742 default: 743 return (EINVAL); 744 } 745 } 746 747 lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL; 748 749 switch (info->hist_dtype) { 750 case VSD_DTYPE_CRHIST32: 751 lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb; 752 ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb; 753 break; 754 case VSD_DTYPE_DRHIST32: 755 lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb; 756 lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub; 757 ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb; 758 ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub; 759 break; 760 case VSD_DTYPE_CRHIST64: 761 lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb; 762 ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb; 763 break; 764 case VSD_DTYPE_DRHIST64: 765 lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb; 766 lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub; 767 ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb; 768 ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub; 769 break; 770 case VSD_DTYPE_DVHIST32: 771 case VSD_DTYPE_DVHIST64: 772 break; 773 default: 774 return (EINVAL); 775 } 776 777 if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) { 778 *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype]; 779 /* 780 * Assignment from numeric_limit array for Q types assigns max 781 * possible integral/fractional value for underlying data type, 782 * but we must set control bits for this specific histogram per 783 * the user's choice of fractional bits, which we extract from 784 * info->lb. 785 */ 786 if (info->voi_dtype == VSD_DTYPE_Q_S32 || 787 info->voi_dtype == VSD_DTYPE_Q_U32) { 788 /* Signedness doesn't matter for setting control bits. */ 789 Q_SCVAL(lbinfbktlb->q32.sq32, 790 Q_GCVAL(info->lb.q32.sq32)); 791 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 792 info->voi_dtype == VSD_DTYPE_Q_U64) { 793 /* Signedness doesn't matter for setting control bits. */ 794 Q_SCVAL(lbinfbktlb->q64.sq64, 795 Q_GCVAL(info->lb.q64.sq64)); 796 } 797 if (lbinfbktub) 798 *lbinfbktub = info->lb; 799 } 800 if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) { 801 *ubinfbktlb = bkt_lb; 802 if (ubinfbktub) { 803 *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype]; 804 if (info->voi_dtype == VSD_DTYPE_Q_S32 || 805 info->voi_dtype == VSD_DTYPE_Q_U32) { 806 Q_SCVAL(ubinfbktub->q32.sq32, 807 Q_GCVAL(info->lb.q32.sq32)); 808 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || 809 info->voi_dtype == VSD_DTYPE_Q_U64) { 810 Q_SCVAL(ubinfbktub->q64.sq64, 811 Q_GCVAL(info->lb.q64.sq64)); 812 } 813 } 814 } 815 816 return (0); 817 } 818 819 int 820 stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 821 struct vss_tdgst_hlpr_info *info) 822 { 823 struct voistatdata_tdgst *tdgst; 824 struct ctdth32 *ctd32tree; 825 struct ctdth64 *ctd64tree; 826 struct voistatdata_tdgstctd32 *ctd32; 827 struct voistatdata_tdgstctd64 *ctd64; 828 829 info->voi_dtype = voi_dtype; 830 831 switch (info->tdgst_dtype) { 832 case VSD_DTYPE_TDGSTCLUST32: 833 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds); 834 break; 835 case VSD_DTYPE_TDGSTCLUST64: 836 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds); 837 break; 838 default: 839 return (EINVAL); 840 } 841 842 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); 843 if (vss->iv == NULL) 844 return (ENOMEM); 845 846 tdgst = (struct voistatdata_tdgst *)vss->iv; 847 848 switch (info->tdgst_dtype) { 849 case VSD_DTYPE_TDGSTCLUST32: 850 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 851 ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) { 852 Q_INI(&ctd32->mu, 0, 0, info->prec); 853 } 854 break; 855 case VSD_DTYPE_TDGSTCLUST64: 856 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 857 ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) { 858 Q_INI(&ctd64->mu, 0, 0, info->prec); 859 } 860 break; 861 default: 862 return (EINVAL); 863 } 864 865 return (0); 866 } 867 868 int 869 stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, 870 struct vss_numeric_hlpr_info *info) 871 { 872 struct voistatdata_numeric iv; 873 874 switch (vss->stype) { 875 case VS_STYPE_SUM: 876 iv = stats_ctor_vsd_numeric(0); 877 break; 878 case VS_STYPE_MIN: 879 iv = numeric_limits[LIM_MAX][voi_dtype]; 880 break; 881 case VS_STYPE_MAX: 882 iv = numeric_limits[LIM_MIN][voi_dtype]; 883 break; 884 default: 885 return (EINVAL); 886 } 887 888 vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0); 889 if (vss->iv == NULL) 890 return (ENOMEM); 891 892 vss->vs_dtype = voi_dtype; 893 vss->vsdsz = vsd_dtype2size[voi_dtype]; 894 switch (voi_dtype) { 895 case VSD_DTYPE_INT_S32: 896 *((int32_t *)vss->iv) = iv.int32.s32; 897 break; 898 case VSD_DTYPE_INT_U32: 899 *((uint32_t *)vss->iv) = iv.int32.u32; 900 break; 901 case VSD_DTYPE_INT_S64: 902 *((int64_t *)vss->iv) = iv.int64.s64; 903 break; 904 case VSD_DTYPE_INT_U64: 905 *((uint64_t *)vss->iv) = iv.int64.u64; 906 break; 907 case VSD_DTYPE_INT_SLONG: 908 *((long *)vss->iv) = iv.intlong.slong; 909 break; 910 case VSD_DTYPE_INT_ULONG: 911 *((unsigned long *)vss->iv) = iv.intlong.ulong; 912 break; 913 case VSD_DTYPE_Q_S32: 914 *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32, 915 Q_CTRLINI(info->prec)); 916 break; 917 case VSD_DTYPE_Q_U32: 918 *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32, 919 Q_CTRLINI(info->prec)); 920 break; 921 case VSD_DTYPE_Q_S64: 922 *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64, 923 Q_CTRLINI(info->prec)); 924 break; 925 case VSD_DTYPE_Q_U64: 926 *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64, 927 Q_CTRLINI(info->prec)); 928 break; 929 default: 930 break; 931 } 932 933 return (0); 934 } 935 936 int 937 stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss, 938 struct voistatspec *vss) 939 { 940 int i, ret; 941 942 for (i = nvss - 1; i >= 0; i--) { 943 if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i], 944 vss[i].hlprinfo)) != 0) 945 return (ret); 946 } 947 948 return (0); 949 } 950 951 void 952 stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss) 953 { 954 int i; 955 956 for (i = nvss - 1; i >= 0; i--) { 957 if (vss[i].hlpr) { 958 stats_free((void *)vss[i].iv); 959 vss[i].iv = NULL; 960 } 961 } 962 } 963 964 int 965 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl) 966 { 967 int error; 968 969 error = 0; 970 971 TPL_LIST_WLOCK(); 972 if (tpl_id < 0 || tpl_id >= (int)ntpl) { 973 error = ENOENT; 974 } else { 975 *tpl = tpllist[tpl_id]; 976 /* XXXLAS: Acquire refcount on tpl. */ 977 } 978 TPL_LIST_WUNLOCK(); 979 980 return (error); 981 } 982 983 int 984 stats_tpl_fetch_allocid(const char *name, uint32_t hash) 985 { 986 int i, tpl_id; 987 988 tpl_id = -ESRCH; 989 990 TPL_LIST_RLOCK(); 991 for (i = ntpl - 1; i >= 0; i--) { 992 if (name != NULL) { 993 if (strlen(name) == strlen(tpllist[i]->mb->tplname) && 994 strncmp(name, tpllist[i]->mb->tplname, 995 TPL_MAX_NAME_LEN) == 0 && (!hash || hash == 996 tpllist[i]->mb->tplhash)) { 997 tpl_id = i; 998 break; 999 } 1000 } else if (hash == tpllist[i]->mb->tplhash) { 1001 tpl_id = i; 1002 break; 1003 } 1004 } 1005 TPL_LIST_RUNLOCK(); 1006 1007 return (tpl_id); 1008 } 1009 1010 int 1011 stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len) 1012 { 1013 int error; 1014 1015 error = 0; 1016 1017 TPL_LIST_RLOCK(); 1018 if (tpl_id < ntpl) { 1019 if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname)) 1020 strlcpy(buf, tpllist[tpl_id]->mb->tplname, len); 1021 else 1022 error = EOVERFLOW; 1023 } else 1024 error = ENOENT; 1025 TPL_LIST_RUNLOCK(); 1026 1027 return (error); 1028 } 1029 1030 int 1031 stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates, 1032 void *seed_bytes, size_t seed_len) 1033 { 1034 uint32_t cum_pct, rnd_pct; 1035 int i; 1036 1037 cum_pct = 0; 1038 1039 /* 1040 * Choose a pseudorandom or seeded number in range [0,100] and use 1041 * it to make a sampling decision and template selection where required. 1042 * If no seed is supplied, a PRNG is used to generate a pseudorandom 1043 * number so that every selection is independent. If a seed is supplied, 1044 * the caller desires random selection across different seeds, but 1045 * deterministic selection given the same seed. This is achieved by 1046 * hashing the seed and using the hash as the random number source. 1047 * 1048 * XXXLAS: Characterise hash function output distribution. 1049 */ 1050 if (seed_bytes == NULL) 1051 rnd_pct = random() / (INT32_MAX / 100); 1052 else 1053 rnd_pct = hash32_buf(seed_bytes, seed_len, 0) / 1054 (UINT32_MAX / 100U); 1055 1056 /* 1057 * We map the randomly selected percentage on to the interval [0,100] 1058 * consisting of the cumulatively summed template sampling percentages. 1059 * The difference between the cumulative sum of all template sampling 1060 * percentages and 100 is treated as a NULL assignment i.e. no stats 1061 * template will be assigned, and -1 returned instead. 1062 */ 1063 for (i = 0; i < nrates; i++) { 1064 cum_pct += rates[i].tpl_sample_pct; 1065 1066 KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__, 1067 cum_pct)); 1068 if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0) 1069 continue; 1070 1071 return (rates[i].tpl_slot_id); 1072 } 1073 1074 return (-1); 1075 } 1076 1077 int 1078 stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz, 1079 struct statsblobv1 *src, uint32_t flags) 1080 { 1081 int error, tmperror; 1082 1083 error = tmperror = 0; 1084 1085 if (src == NULL || dst == NULL || 1086 src->cursz < sizeof(struct statsblob) || 1087 ((flags & SB_CLONE_ALLOCDST) && 1088 (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) { 1089 error = EINVAL; 1090 } else if (flags & SB_CLONE_ALLOCDST) { 1091 *dst = stats_realloc(NULL, 0, src->cursz, 0); 1092 if (*dst) 1093 (*dst)->maxsz = dstmaxsz = src->cursz; 1094 else 1095 error = ENOMEM; 1096 } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) { 1097 error = EINVAL; 1098 } 1099 1100 if (!error) { 1101 size_t postcurszlen; 1102 1103 /* 1104 * Clone src into dst except for the maxsz field. If dst is too 1105 * small to hold all of src, only copy src's header and return 1106 * EOVERFLOW. 1107 */ 1108 #ifdef _KERNEL 1109 if (flags & SB_CLONE_USRDSTNOFAULT) 1110 error = copyout_nofault(src, *dst, 1111 offsetof(struct statsblob, maxsz)); 1112 else if (flags & SB_CLONE_USRDST) 1113 error = copyout(src, *dst, 1114 offsetof(struct statsblob, maxsz)); 1115 else 1116 #endif 1117 memcpy(*dst, src, offsetof(struct statsblob, maxsz)); 1118 #ifdef _KERNEL 1119 if (error != 0) 1120 goto out; 1121 #endif 1122 1123 1124 if (dstmaxsz >= src->cursz) { 1125 postcurszlen = src->cursz - 1126 offsetof(struct statsblob, cursz); 1127 } else { 1128 error = EOVERFLOW; 1129 postcurszlen = sizeof(struct statsblob) - 1130 offsetof(struct statsblob, cursz); 1131 } 1132 #ifdef _KERNEL 1133 if (flags & SB_CLONE_USRDSTNOFAULT) 1134 tmperror = copyout_nofault(&(src->cursz), &((*dst)->cursz), 1135 postcurszlen); 1136 else if (flags & SB_CLONE_USRDST) 1137 tmperror = copyout(&(src->cursz), &((*dst)->cursz), 1138 postcurszlen); 1139 else 1140 #endif 1141 memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen); 1142 1143 error = error ? error : tmperror; 1144 } 1145 #ifdef _KERNEL 1146 out: 1147 #endif 1148 1149 return (error); 1150 } 1151 1152 int 1153 stats_v1_tpl_alloc(const char *name, uint32_t flags __unused) 1154 { 1155 struct statsblobv1_tpl *tpl, **newtpllist; 1156 struct statsblobv1 *tpl_sb; 1157 struct metablob *tpl_mb; 1158 int tpl_id; 1159 1160 if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN) 1161 return (-EINVAL); 1162 1163 if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0) 1164 return (-EEXIST); 1165 1166 tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO); 1167 tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO); 1168 tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO); 1169 1170 if (tpl_mb != NULL && name != NULL) 1171 tpl_mb->tplname = stats_strdup(name, 0); 1172 1173 if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL || 1174 tpl_mb->tplname == NULL) { 1175 stats_free(tpl); 1176 stats_free(tpl_sb); 1177 if (tpl_mb != NULL) { 1178 stats_free(tpl_mb->tplname); 1179 stats_free(tpl_mb); 1180 } 1181 return (-ENOMEM); 1182 } 1183 1184 tpl->mb = tpl_mb; 1185 tpl->sb = tpl_sb; 1186 1187 tpl_sb->abi = STATS_ABI_V1; 1188 tpl_sb->endian = 1189 #if BYTE_ORDER == LITTLE_ENDIAN 1190 SB_LE; 1191 #elif BYTE_ORDER == BIG_ENDIAN 1192 SB_BE; 1193 #else 1194 SB_UE; 1195 #endif 1196 tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1); 1197 tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1); 1198 1199 TPL_LIST_WLOCK(); 1200 newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *), 1201 (ntpl + 1) * sizeof(void *), 0); 1202 if (newtpllist != NULL) { 1203 tpl_id = ntpl++; 1204 tpllist = (struct statsblob_tpl **)newtpllist; 1205 tpllist[tpl_id] = (struct statsblob_tpl *)tpl; 1206 stats_tpl_update_hash(tpllist[tpl_id]); 1207 } else { 1208 stats_free(tpl); 1209 stats_free(tpl_sb); 1210 if (tpl_mb != NULL) { 1211 stats_free(tpl_mb->tplname); 1212 stats_free(tpl_mb); 1213 } 1214 tpl_id = -ENOMEM; 1215 } 1216 TPL_LIST_WUNLOCK(); 1217 1218 return (tpl_id); 1219 } 1220 1221 int 1222 stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name, 1223 enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss, 1224 uint32_t flags) 1225 { 1226 struct voi *voi; 1227 struct voistat *tmpstat; 1228 struct statsblobv1 *tpl_sb; 1229 struct metablob *tpl_mb; 1230 int error, i, newstatdataidx, newvoibytes, newvoistatbytes, 1231 newvoistatdatabytes, newvoistatmaxid; 1232 uint32_t nbytes; 1233 1234 if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || 1235 nvss == 0 || vss == NULL) 1236 return (EINVAL); 1237 1238 error = nbytes = newvoibytes = newvoistatbytes = 1239 newvoistatdatabytes = 0; 1240 newvoistatmaxid = -1; 1241 1242 /* Calculate the number of bytes required for the new voistats. */ 1243 for (i = nvss - 1; i >= 0; i--) { 1244 if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES || 1245 vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES || 1246 vss[i].iv == NULL || vss[i].vsdsz == 0) 1247 return (EINVAL); 1248 if ((int)vss[i].stype > newvoistatmaxid) 1249 newvoistatmaxid = vss[i].stype; 1250 newvoistatdatabytes += vss[i].vsdsz; 1251 } 1252 1253 if (flags & SB_VOI_RELUPDATE) { 1254 /* XXXLAS: VOI state bytes may need to vary based on stat types. */ 1255 newvoistatdatabytes += sizeof(struct voistatdata_voistate); 1256 } 1257 nbytes += newvoistatdatabytes; 1258 1259 TPL_LIST_WLOCK(); 1260 if (tpl_id < ntpl) { 1261 tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb; 1262 tpl_mb = tpllist[tpl_id]->mb; 1263 1264 if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) { 1265 /* Adding a new VOI and associated stats. */ 1266 if (voi_id >= NVOIS(tpl_sb)) { 1267 /* We need to grow the tpl_sb->vois array. */ 1268 newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) * 1269 sizeof(struct voi); 1270 nbytes += newvoibytes; 1271 } 1272 newvoistatbytes = 1273 (newvoistatmaxid + 1) * sizeof(struct voistat); 1274 } else { 1275 /* Adding stats to an existing VOI. */ 1276 if (newvoistatmaxid > 1277 tpl_sb->vois[voi_id].voistatmaxid) { 1278 newvoistatbytes = (newvoistatmaxid - 1279 tpl_sb->vois[voi_id].voistatmaxid) * 1280 sizeof(struct voistat); 1281 } 1282 /* XXXLAS: KPI does not yet support expanding VOIs. */ 1283 error = EOPNOTSUPP; 1284 } 1285 nbytes += newvoistatbytes; 1286 1287 if (!error && newvoibytes > 0) { 1288 struct voi_meta *voi_meta = tpl_mb->voi_meta; 1289 1290 voi_meta = stats_realloc(voi_meta, voi_meta == NULL ? 1291 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta), 1292 (1 + voi_id) * sizeof(struct voi_meta), 1293 M_ZERO); 1294 1295 if (voi_meta == NULL) 1296 error = ENOMEM; 1297 else 1298 tpl_mb->voi_meta = voi_meta; 1299 } 1300 1301 if (!error) { 1302 /* NB: Resizing can change where tpl_sb points. */ 1303 error = stats_v1_blob_expand(&tpl_sb, newvoibytes, 1304 newvoistatbytes, newvoistatdatabytes); 1305 } 1306 1307 if (!error) { 1308 tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name, 1309 0); 1310 if (tpl_mb->voi_meta[voi_id].name == NULL) 1311 error = ENOMEM; 1312 } 1313 1314 if (!error) { 1315 /* Update the template list with the resized pointer. */ 1316 tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb; 1317 1318 /* Update the template. */ 1319 voi = &tpl_sb->vois[voi_id]; 1320 1321 if (voi->id < 0) { 1322 /* VOI is new and needs to be initialised. */ 1323 voi->id = voi_id; 1324 voi->dtype = voi_dtype; 1325 voi->stats_off = tpl_sb->stats_off; 1326 if (flags & SB_VOI_RELUPDATE) 1327 voi->flags |= VOI_REQSTATE; 1328 } else { 1329 /* 1330 * XXXLAS: When this else block is written, the 1331 * "KPI does not yet support expanding VOIs" 1332 * error earlier in this function can be 1333 * removed. What is required here is to shuffle 1334 * the voistat array such that the new stats for 1335 * the voi are contiguous, which will displace 1336 * stats for other vois that reside after the 1337 * voi being updated. The other vois then need 1338 * to have their stats_off adjusted post 1339 * shuffle. 1340 */ 1341 } 1342 1343 voi->voistatmaxid = newvoistatmaxid; 1344 newstatdataidx = 0; 1345 1346 if (voi->flags & VOI_REQSTATE) { 1347 /* Initialise the voistate stat in slot 0. */ 1348 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off); 1349 tmpstat->stype = VS_STYPE_VOISTATE; 1350 tmpstat->flags = 0; 1351 tmpstat->dtype = VSD_DTYPE_VOISTATE; 1352 newstatdataidx = tmpstat->dsz = 1353 sizeof(struct voistatdata_numeric); 1354 tmpstat->data_off = tpl_sb->statsdata_off; 1355 } 1356 1357 for (i = 0; (uint32_t)i < nvss; i++) { 1358 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off + 1359 (vss[i].stype * sizeof(struct voistat))); 1360 KASSERT(tmpstat->stype < 0, ("voistat %p " 1361 "already initialised", tmpstat)); 1362 tmpstat->stype = vss[i].stype; 1363 tmpstat->flags = vss[i].flags; 1364 tmpstat->dtype = vss[i].vs_dtype; 1365 tmpstat->dsz = vss[i].vsdsz; 1366 tmpstat->data_off = tpl_sb->statsdata_off + 1367 newstatdataidx; 1368 memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off), 1369 vss[i].iv, vss[i].vsdsz); 1370 newstatdataidx += vss[i].vsdsz; 1371 } 1372 1373 /* Update the template version hash. */ 1374 stats_tpl_update_hash(tpllist[tpl_id]); 1375 /* XXXLAS: Confirm tpl name/hash pair remains unique. */ 1376 } 1377 } else 1378 error = EINVAL; 1379 TPL_LIST_WUNLOCK(); 1380 1381 return (error); 1382 } 1383 1384 struct statsblobv1 * 1385 stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused) 1386 { 1387 struct statsblobv1 *sb; 1388 int error; 1389 1390 sb = NULL; 1391 1392 TPL_LIST_RLOCK(); 1393 if (tpl_id < ntpl) { 1394 sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0); 1395 if (sb != NULL) { 1396 sb->maxsz = tpllist[tpl_id]->sb->maxsz; 1397 error = stats_v1_blob_init_locked(sb, tpl_id, 0); 1398 } else 1399 error = ENOMEM; 1400 1401 if (error) { 1402 stats_free(sb); 1403 sb = NULL; 1404 } 1405 } 1406 TPL_LIST_RUNLOCK(); 1407 1408 return (sb); 1409 } 1410 1411 void 1412 stats_v1_blob_destroy(struct statsblobv1 *sb) 1413 { 1414 1415 stats_free(sb); 1416 } 1417 1418 int 1419 stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id, 1420 enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd, 1421 size_t *retvsdsz) 1422 { 1423 struct voi *v; 1424 struct voistat *vs; 1425 1426 if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 || 1427 voi_id >= NVOIS(sb)) 1428 return (EINVAL); 1429 1430 v = &sb->vois[voi_id]; 1431 if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid) 1432 return (EINVAL); 1433 1434 vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat))); 1435 *retvsd = BLOB_OFFSET(sb, vs->data_off); 1436 if (retdtype != NULL) 1437 *retdtype = vs->dtype; 1438 if (retvsdsz != NULL) 1439 *retvsdsz = vs->dsz; 1440 1441 return (0); 1442 } 1443 1444 int 1445 stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags) 1446 { 1447 int error; 1448 1449 error = 0; 1450 1451 TPL_LIST_RLOCK(); 1452 if (sb == NULL || tpl_id >= ntpl) { 1453 error = EINVAL; 1454 } else { 1455 error = stats_v1_blob_init_locked(sb, tpl_id, flags); 1456 } 1457 TPL_LIST_RUNLOCK(); 1458 1459 return (error); 1460 } 1461 1462 static inline int 1463 stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, 1464 uint32_t flags __unused) 1465 { 1466 int error; 1467 1468 TPL_LIST_RLOCK_ASSERT(); 1469 error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW; 1470 KASSERT(!error, 1471 ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz)); 1472 1473 if (!error) { 1474 memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz); 1475 sb->created = sb->lastrst = stats_sbinuptime(); 1476 sb->tplhash = tpllist[tpl_id]->mb->tplhash; 1477 } 1478 1479 return (error); 1480 } 1481 1482 static int 1483 stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, 1484 int newvoistatbytes, int newvoistatdatabytes) 1485 { 1486 struct statsblobv1 *sb; 1487 struct voi *tmpvoi; 1488 struct voistat *tmpvoistat, *voistat_array; 1489 int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats; 1490 1491 KASSERT(newvoibytes % sizeof(struct voi) == 0, 1492 ("Bad newvoibytes %d", newvoibytes)); 1493 KASSERT(newvoistatbytes % sizeof(struct voistat) == 0, 1494 ("Bad newvoistatbytes %d", newvoistatbytes)); 1495 1496 error = ((newvoibytes % sizeof(struct voi) == 0) && 1497 (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL; 1498 sb = *sbpp; 1499 nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes; 1500 1501 /* 1502 * XXXLAS: Required until we gain support for flags which alter the 1503 * units of size/offset fields in key structs. 1504 */ 1505 if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ)) 1506 error = EFBIG; 1507 1508 if (!error && (sb->cursz + nbytes > sb->maxsz)) { 1509 /* Need to expand our blob. */ 1510 sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO); 1511 if (sb != NULL) { 1512 sb->maxsz = sb->cursz + nbytes; 1513 *sbpp = sb; 1514 } else 1515 error = ENOMEM; 1516 } 1517 1518 if (!error) { 1519 /* 1520 * Shuffle memory within the expanded blob working from the end 1521 * backwards, leaving gaps for the new voistat and voistatdata 1522 * structs at the beginning of their respective blob regions, 1523 * and for the new voi structs at the end of their blob region. 1524 */ 1525 memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes), 1526 BLOB_OFFSET(sb, sb->statsdata_off), 1527 sb->cursz - sb->statsdata_off); 1528 memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes + 1529 newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off), 1530 sb->statsdata_off - sb->stats_off); 1531 1532 /* First index of new voi/voistat structs to be initialised. */ 1533 idxnewvois = NVOIS(sb); 1534 idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1; 1535 1536 /* Update housekeeping variables and offsets. */ 1537 sb->cursz += nbytes; 1538 sb->stats_off += newvoibytes; 1539 sb->statsdata_off += newvoibytes + newvoistatbytes; 1540 1541 /* XXXLAS: Zeroing not strictly needed but aids debugging. */ 1542 memset(&sb->vois[idxnewvois], '\0', newvoibytes); 1543 memset(BLOB_OFFSET(sb, sb->stats_off), '\0', 1544 newvoistatbytes); 1545 memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0', 1546 newvoistatdatabytes); 1547 1548 /* Initialise new voi array members and update offsets. */ 1549 for (i = 0; i < NVOIS(sb); i++) { 1550 tmpvoi = &sb->vois[i]; 1551 if (i >= idxnewvois) { 1552 tmpvoi->id = tmpvoi->voistatmaxid = -1; 1553 } else if (tmpvoi->id > -1) { 1554 tmpvoi->stats_off += newvoibytes + 1555 newvoistatbytes; 1556 } 1557 } 1558 1559 /* Initialise new voistat array members and update offsets. */ 1560 nvoistats = (sb->statsdata_off - sb->stats_off) / 1561 sizeof(struct voistat); 1562 voistat_array = BLOB_OFFSET(sb, sb->stats_off); 1563 for (i = 0; i < nvoistats; i++) { 1564 tmpvoistat = &voistat_array[i]; 1565 if (i <= idxnewvoistats) { 1566 tmpvoistat->stype = -1; 1567 } else if (tmpvoistat->stype > -1) { 1568 tmpvoistat->data_off += nbytes; 1569 } 1570 } 1571 } 1572 1573 return (error); 1574 } 1575 1576 static void 1577 stats_v1_blob_finalise(struct statsblobv1 *sb __unused) 1578 { 1579 1580 /* XXXLAS: Fill this in. */ 1581 } 1582 1583 static void 1584 stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb, 1585 void *usrctx, uint32_t flags) 1586 { 1587 struct voi *v; 1588 struct voistat *vs; 1589 struct sb_iter_ctx ctx; 1590 int i, j, firstvoi; 1591 1592 ctx.usrctx = usrctx; 1593 ctx.flags = SB_IT_FIRST_CB; 1594 firstvoi = 1; 1595 1596 for (i = 0; i < NVOIS(sb); i++) { 1597 v = &sb->vois[i]; 1598 ctx.vslot = i; 1599 ctx.vsslot = -1; 1600 ctx.flags |= SB_IT_FIRST_VOISTAT; 1601 1602 if (firstvoi) 1603 ctx.flags |= SB_IT_FIRST_VOI; 1604 else if (i == (NVOIS(sb) - 1)) 1605 ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB; 1606 1607 if (v->id < 0 && (flags & SB_IT_NULLVOI)) { 1608 if (icb(sb, v, NULL, &ctx)) 1609 return; 1610 firstvoi = 0; 1611 ctx.flags &= ~SB_IT_FIRST_CB; 1612 } 1613 1614 /* If NULL voi, v->voistatmaxid == -1 */ 1615 for (j = 0; j <= v->voistatmaxid; j++) { 1616 vs = &((struct voistat *)BLOB_OFFSET(sb, 1617 v->stats_off))[j]; 1618 if (vs->stype < 0 && 1619 !(flags & SB_IT_NULLVOISTAT)) 1620 continue; 1621 1622 if (j == v->voistatmaxid) { 1623 ctx.flags |= SB_IT_LAST_VOISTAT; 1624 if (i == (NVOIS(sb) - 1)) 1625 ctx.flags |= 1626 SB_IT_LAST_CB; 1627 } else 1628 ctx.flags &= ~SB_IT_LAST_CB; 1629 1630 ctx.vsslot = j; 1631 if (icb(sb, v, vs, &ctx)) 1632 return; 1633 1634 ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT | 1635 SB_IT_LAST_VOISTAT); 1636 } 1637 ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI); 1638 } 1639 } 1640 1641 static inline void 1642 stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused, 1643 const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype, 1644 size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 1645 { 1646 const struct ctdth32 *ctd32tree; 1647 const struct ctdth64 *ctd64tree; 1648 const struct voistatdata_tdgstctd32 *ctd32; 1649 const struct voistatdata_tdgstctd64 *ctd64; 1650 const char *fmtstr; 1651 uint64_t smplcnt, compcnt; 1652 int is32bit, qmaxstrlen; 1653 uint16_t maxctds, curctds; 1654 1655 switch (tdgst_dtype) { 1656 case VSD_DTYPE_TDGSTCLUST32: 1657 smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt; 1658 compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt; 1659 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 1660 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); 1661 ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree; 1662 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) : 1663 ARB_CMIN(ctdth32, ctd32tree)); 1664 qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10); 1665 is32bit = 1; 1666 ctd64tree = NULL; 1667 ctd64 = NULL; 1668 break; 1669 case VSD_DTYPE_TDGSTCLUST64: 1670 smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt; 1671 compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt; 1672 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 1673 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); 1674 ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree; 1675 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) : 1676 ARB_CMIN(ctdth64, ctd64tree)); 1677 qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10); 1678 is32bit = 0; 1679 ctd32tree = NULL; 1680 ctd32 = NULL; 1681 break; 1682 default: 1683 return; 1684 } 1685 1686 switch (fmt) { 1687 case SB_STRFMT_FREEFORM: 1688 fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu"; 1689 break; 1690 case SB_STRFMT_JSON: 1691 default: 1692 fmtstr = 1693 "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu," 1694 "\"nctds\":%hu,\"ctds\":["; 1695 break; 1696 } 1697 sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt, 1698 maxctds, curctds); 1699 1700 while ((is32bit ? NULL != ctd32 : NULL != ctd64)) { 1701 char qstr[qmaxstrlen]; 1702 1703 switch (fmt) { 1704 case SB_STRFMT_FREEFORM: 1705 fmtstr = "\n\t\t\t\t"; 1706 break; 1707 case SB_STRFMT_JSON: 1708 default: 1709 fmtstr = "{"; 1710 break; 1711 } 1712 sbuf_cat(buf, fmtstr); 1713 1714 if (objdump) { 1715 switch (fmt) { 1716 case SB_STRFMT_FREEFORM: 1717 fmtstr = "ctd[%hu]."; 1718 break; 1719 case SB_STRFMT_JSON: 1720 default: 1721 fmtstr = "\"ctd\":%hu,"; 1722 break; 1723 } 1724 sbuf_printf(buf, fmtstr, is32bit ? 1725 ARB_SELFIDX(ctd32tree, ctd32) : 1726 ARB_SELFIDX(ctd64tree, ctd64)); 1727 } 1728 1729 switch (fmt) { 1730 case SB_STRFMT_FREEFORM: 1731 fmtstr = "{mu="; 1732 break; 1733 case SB_STRFMT_JSON: 1734 default: 1735 fmtstr = "\"mu\":"; 1736 break; 1737 } 1738 sbuf_cat(buf, fmtstr); 1739 Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr, 1740 sizeof(qstr)); 1741 sbuf_cat(buf, qstr); 1742 1743 switch (fmt) { 1744 case SB_STRFMT_FREEFORM: 1745 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 1746 break; 1747 case SB_STRFMT_JSON: 1748 default: 1749 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 1750 break; 1751 } 1752 sbuf_printf(buf, fmtstr, 1753 is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt); 1754 1755 if (is32bit) 1756 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 1757 ARB_SELFIDX(ctd32tree, ctd32) + 1) : 1758 ARB_CNEXT(ctdth32, ctd32tree, ctd32)); 1759 else 1760 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 1761 ARB_SELFIDX(ctd64tree, ctd64) + 1) : 1762 ARB_CNEXT(ctdth64, ctd64tree, ctd64)); 1763 1764 if (fmt == SB_STRFMT_JSON && 1765 (is32bit ? NULL != ctd32 : NULL != ctd64)) 1766 sbuf_putc(buf, ','); 1767 } 1768 if (fmt == SB_STRFMT_JSON) 1769 sbuf_cat(buf, "]"); 1770 } 1771 1772 static inline void 1773 stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype, 1774 const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype, 1775 size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) 1776 { 1777 const struct voistatdata_numeric *bkt_lb, *bkt_ub; 1778 const char *fmtstr; 1779 int is32bit; 1780 uint16_t i, nbkts; 1781 1782 switch (hist_dtype) { 1783 case VSD_DTYPE_CRHIST32: 1784 nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz); 1785 is32bit = 1; 1786 break; 1787 case VSD_DTYPE_DRHIST32: 1788 nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz); 1789 is32bit = 1; 1790 break; 1791 case VSD_DTYPE_DVHIST32: 1792 nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz); 1793 is32bit = 1; 1794 break; 1795 case VSD_DTYPE_CRHIST64: 1796 nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz); 1797 is32bit = 0; 1798 break; 1799 case VSD_DTYPE_DRHIST64: 1800 nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz); 1801 is32bit = 0; 1802 break; 1803 case VSD_DTYPE_DVHIST64: 1804 nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz); 1805 is32bit = 0; 1806 break; 1807 default: 1808 return; 1809 } 1810 1811 switch (fmt) { 1812 case SB_STRFMT_FREEFORM: 1813 fmtstr = "nbkts=%hu, "; 1814 break; 1815 case SB_STRFMT_JSON: 1816 default: 1817 fmtstr = "\"nbkts\":%hu,"; 1818 break; 1819 } 1820 sbuf_printf(buf, fmtstr, nbkts); 1821 1822 switch (fmt) { 1823 case SB_STRFMT_FREEFORM: 1824 fmtstr = (is32bit ? "oob=%u" : "oob=%ju"); 1825 break; 1826 case SB_STRFMT_JSON: 1827 default: 1828 fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" : 1829 "\"oob\":%ju,\"bkts\":["); 1830 break; 1831 } 1832 sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist, 1833 hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, 1834 hist_dtype, oob)); 1835 1836 for (i = 0; i < nbkts; i++) { 1837 switch (hist_dtype) { 1838 case VSD_DTYPE_CRHIST32: 1839 case VSD_DTYPE_CRHIST64: 1840 bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype, 1841 bkts[i].lb); 1842 if (i < nbkts - 1) 1843 bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist, 1844 hist_dtype, bkts[i + 1].lb); 1845 else 1846 bkt_ub = &numeric_limits[LIM_MAX][voi_dtype]; 1847 break; 1848 case VSD_DTYPE_DRHIST32: 1849 case VSD_DTYPE_DRHIST64: 1850 bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 1851 bkts[i].lb); 1852 bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, 1853 bkts[i].ub); 1854 break; 1855 case VSD_DTYPE_DVHIST32: 1856 case VSD_DTYPE_DVHIST64: 1857 bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist, 1858 hist_dtype, bkts[i].val); 1859 break; 1860 default: 1861 break; 1862 } 1863 1864 switch (fmt) { 1865 case SB_STRFMT_FREEFORM: 1866 fmtstr = "\n\t\t\t\t"; 1867 break; 1868 case SB_STRFMT_JSON: 1869 default: 1870 fmtstr = "{"; 1871 break; 1872 } 1873 sbuf_cat(buf, fmtstr); 1874 1875 if (objdump) { 1876 switch (fmt) { 1877 case SB_STRFMT_FREEFORM: 1878 fmtstr = "bkt[%hu]."; 1879 break; 1880 case SB_STRFMT_JSON: 1881 default: 1882 fmtstr = "\"bkt\":%hu,"; 1883 break; 1884 } 1885 sbuf_printf(buf, fmtstr, i); 1886 } 1887 1888 switch (fmt) { 1889 case SB_STRFMT_FREEFORM: 1890 fmtstr = "{lb="; 1891 break; 1892 case SB_STRFMT_JSON: 1893 default: 1894 fmtstr = "\"lb\":"; 1895 break; 1896 } 1897 sbuf_cat(buf, fmtstr); 1898 stats_voistatdata_tostr((const struct voistatdata *)bkt_lb, 1899 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 1900 fmt, buf, objdump); 1901 1902 switch (fmt) { 1903 case SB_STRFMT_FREEFORM: 1904 fmtstr = ",ub="; 1905 break; 1906 case SB_STRFMT_JSON: 1907 default: 1908 fmtstr = ",\"ub\":"; 1909 break; 1910 } 1911 sbuf_cat(buf, fmtstr); 1912 stats_voistatdata_tostr((const struct voistatdata *)bkt_ub, 1913 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), 1914 fmt, buf, objdump); 1915 1916 switch (fmt) { 1917 case SB_STRFMT_FREEFORM: 1918 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; 1919 break; 1920 case SB_STRFMT_JSON: 1921 default: 1922 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; 1923 break; 1924 } 1925 sbuf_printf(buf, fmtstr, is32bit ? 1926 VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) : 1927 (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, 1928 bkts[i].cnt)); 1929 1930 if (fmt == SB_STRFMT_JSON && i < nbkts - 1) 1931 sbuf_putc(buf, ','); 1932 } 1933 if (fmt == SB_STRFMT_JSON) 1934 sbuf_cat(buf, "]"); 1935 } 1936 1937 int 1938 stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype, 1939 enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt, 1940 struct sbuf *buf, int objdump) 1941 { 1942 const char *fmtstr; 1943 1944 if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES || 1945 vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS) 1946 return (EINVAL); 1947 1948 switch (vsd_dtype) { 1949 case VSD_DTYPE_VOISTATE: 1950 switch (fmt) { 1951 case SB_STRFMT_FREEFORM: 1952 fmtstr = "prev="; 1953 break; 1954 case SB_STRFMT_JSON: 1955 default: 1956 fmtstr = "\"prev\":"; 1957 break; 1958 } 1959 sbuf_cat(buf, fmtstr); 1960 /* 1961 * Render prev by passing it as *vsd and voi_dtype as vsd_dtype. 1962 */ 1963 stats_voistatdata_tostr( 1964 (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev, 1965 voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump); 1966 break; 1967 case VSD_DTYPE_INT_S32: 1968 sbuf_printf(buf, "%d", vsd->int32.s32); 1969 break; 1970 case VSD_DTYPE_INT_U32: 1971 sbuf_printf(buf, "%u", vsd->int32.u32); 1972 break; 1973 case VSD_DTYPE_INT_S64: 1974 sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64); 1975 break; 1976 case VSD_DTYPE_INT_U64: 1977 sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64); 1978 break; 1979 case VSD_DTYPE_INT_SLONG: 1980 sbuf_printf(buf, "%ld", vsd->intlong.slong); 1981 break; 1982 case VSD_DTYPE_INT_ULONG: 1983 sbuf_printf(buf, "%lu", vsd->intlong.ulong); 1984 break; 1985 case VSD_DTYPE_Q_S32: 1986 { 1987 char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)]; 1988 Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr)); 1989 sbuf_cat(buf, qstr); 1990 } 1991 break; 1992 case VSD_DTYPE_Q_U32: 1993 { 1994 char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)]; 1995 Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr)); 1996 sbuf_cat(buf, qstr); 1997 } 1998 break; 1999 case VSD_DTYPE_Q_S64: 2000 { 2001 char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)]; 2002 Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr)); 2003 sbuf_cat(buf, qstr); 2004 } 2005 break; 2006 case VSD_DTYPE_Q_U64: 2007 { 2008 char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)]; 2009 Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr)); 2010 sbuf_cat(buf, qstr); 2011 } 2012 break; 2013 case VSD_DTYPE_CRHIST32: 2014 case VSD_DTYPE_DRHIST32: 2015 case VSD_DTYPE_DVHIST32: 2016 case VSD_DTYPE_CRHIST64: 2017 case VSD_DTYPE_DRHIST64: 2018 case VSD_DTYPE_DVHIST64: 2019 stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd), 2020 vsd_dtype, vsd_sz, fmt, buf, objdump); 2021 break; 2022 case VSD_DTYPE_TDGSTCLUST32: 2023 case VSD_DTYPE_TDGSTCLUST64: 2024 stats_voistatdata_tdgst_tostr(voi_dtype, 2025 CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf, 2026 objdump); 2027 break; 2028 default: 2029 break; 2030 } 2031 2032 return (sbuf_error(buf)); 2033 } 2034 2035 static void 2036 stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v, 2037 struct voistat *vs, struct sb_iter_ctx *ctx) 2038 { 2039 struct sb_tostrcb_ctx *sctx; 2040 struct metablob *tpl_mb; 2041 struct sbuf *buf; 2042 void *vsd; 2043 uint8_t dump; 2044 2045 sctx = ctx->usrctx; 2046 buf = sctx->buf; 2047 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 2048 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 2049 2050 if (ctx->flags & SB_IT_FIRST_CB) { 2051 sbuf_printf(buf, "struct statsblobv1@%p", sb); 2052 if (dump) { 2053 sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, " 2054 "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, " 2055 "stats_off=%hu, statsdata_off=%hu", 2056 sb->abi, sb->endian, sb->maxsz, sb->cursz, 2057 sb->created, sb->lastrst, sb->flags, sb->stats_off, 2058 sb->statsdata_off); 2059 } 2060 sbuf_printf(buf, ", tplhash=%u", sb->tplhash); 2061 } 2062 2063 if (ctx->flags & SB_IT_FIRST_VOISTAT) { 2064 sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id); 2065 if (v->id < 0) 2066 return; 2067 sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" : 2068 tpl_mb->voi_meta[v->id].name); 2069 if (dump) 2070 sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, " 2071 "voistatmaxid=%hhd, stats_off=%hu", v->flags, 2072 vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off); 2073 } 2074 2075 if (!dump && vs->stype <= 0) 2076 return; 2077 2078 sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot); 2079 if (vs->stype < 0) { 2080 sbuf_printf(buf, "%hhd", vs->stype); 2081 return; 2082 } else 2083 sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype], 2084 vs->errs); 2085 vsd = BLOB_OFFSET(sb, vs->data_off); 2086 if (dump) 2087 sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, " 2088 "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype], 2089 vs->dsz, vs->data_off); 2090 2091 sbuf_cat(buf, "\n\t\t\tvoistatdata: "); 2092 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 2093 sctx->fmt, buf, dump); 2094 } 2095 2096 static void 2097 stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 2098 struct sb_iter_ctx *ctx) 2099 { 2100 struct sb_tostrcb_ctx *sctx; 2101 struct metablob *tpl_mb; 2102 struct sbuf *buf; 2103 const char *fmtstr; 2104 void *vsd; 2105 uint8_t dump; 2106 2107 sctx = ctx->usrctx; 2108 buf = sctx->buf; 2109 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; 2110 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); 2111 2112 if (ctx->flags & SB_IT_FIRST_CB) { 2113 sbuf_putc(buf, '{'); 2114 if (dump) { 2115 sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu," 2116 "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd," 2117 "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu," 2118 "\"statsdata_off\":%hu,", sb->abi, 2119 sb->endian, sb->maxsz, sb->cursz, sb->created, 2120 sb->lastrst, sb->flags, sb->stats_off, 2121 sb->statsdata_off); 2122 } 2123 2124 if (tpl_mb == NULL) 2125 fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{"; 2126 else 2127 fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{"; 2128 2129 sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null", 2130 sb->tplhash); 2131 } 2132 2133 if (ctx->flags & SB_IT_FIRST_VOISTAT) { 2134 if (dump) { 2135 sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot, 2136 v->id); 2137 if (v->id < 0) { 2138 sbuf_cat(buf, "},"); 2139 return; 2140 } 2141 2142 if (tpl_mb == NULL) 2143 fmtstr = ",\"name\":%s,\"flags\":%hu," 2144 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 2145 "\"stats_off\":%hu,"; 2146 else 2147 fmtstr = ",\"name\":\"%s\",\"flags\":%hu," 2148 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," 2149 "\"stats_off\":%hu,"; 2150 2151 sbuf_printf(buf, fmtstr, tpl_mb ? 2152 tpl_mb->voi_meta[v->id].name : "null", v->flags, 2153 vsd_dtype2name[v->dtype], v->voistatmaxid, 2154 v->stats_off); 2155 } else { 2156 if (tpl_mb == NULL) { 2157 sbuf_printf(buf, "\"[%hd]\":{", v->id); 2158 } else { 2159 sbuf_printf(buf, "\"%s\":{", 2160 tpl_mb->voi_meta[v->id].name); 2161 } 2162 } 2163 sbuf_cat(buf, "\"stats\":{"); 2164 } 2165 2166 vsd = BLOB_OFFSET(sb, vs->data_off); 2167 if (dump) { 2168 sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot); 2169 if (vs->stype < 0) { 2170 sbuf_cat(buf, "{\"stype\":-1},"); 2171 return; 2172 } 2173 sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu," 2174 "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{", 2175 vs_stype2name[vs->stype], vs->errs, vs->flags, 2176 vsd_dtype2name[vs->dtype], vs->data_off); 2177 } else if (vs->stype > 0) { 2178 if (tpl_mb == NULL) 2179 sbuf_printf(buf, "\"[%hhd]\":", vs->stype); 2180 else 2181 sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]); 2182 } else 2183 return; 2184 2185 if ((vs->flags & VS_VSDVALID) || dump) { 2186 if (!dump) 2187 sbuf_printf(buf, "{\"errs\":%hu,", vs->errs); 2188 /* Simple non-compound VSD types need a key. */ 2189 if (!vsd_compoundtype[vs->dtype]) 2190 sbuf_cat(buf, "\"val\":"); 2191 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, 2192 sctx->fmt, buf, dump); 2193 sbuf_cat(buf, dump ? "}}" : "}"); 2194 } else 2195 sbuf_cat(buf, dump ? "null}" : "null"); 2196 2197 if (ctx->flags & SB_IT_LAST_VOISTAT) 2198 sbuf_cat(buf, "}}"); 2199 2200 if (ctx->flags & SB_IT_LAST_CB) 2201 sbuf_cat(buf, "}}"); 2202 else 2203 sbuf_putc(buf, ','); 2204 } 2205 2206 static int 2207 stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, 2208 struct sb_iter_ctx *ctx) 2209 { 2210 struct sb_tostrcb_ctx *sctx; 2211 2212 sctx = ctx->usrctx; 2213 2214 switch (sctx->fmt) { 2215 case SB_STRFMT_FREEFORM: 2216 stats_v1_itercb_tostr_freeform(sb, v, vs, ctx); 2217 break; 2218 case SB_STRFMT_JSON: 2219 stats_v1_itercb_tostr_json(sb, v, vs, ctx); 2220 break; 2221 default: 2222 break; 2223 } 2224 2225 return (sbuf_error(sctx->buf)); 2226 } 2227 2228 int 2229 stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf, 2230 enum sb_str_fmt fmt, uint32_t flags) 2231 { 2232 struct sb_tostrcb_ctx sctx; 2233 uint32_t iflags; 2234 2235 if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL || 2236 fmt >= SB_STRFMT_NUM_FMTS) 2237 return (EINVAL); 2238 2239 sctx.buf = buf; 2240 sctx.fmt = fmt; 2241 sctx.flags = flags; 2242 2243 if (flags & SB_TOSTR_META) { 2244 if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash), 2245 &sctx.tpl)) 2246 return (EINVAL); 2247 } else 2248 sctx.tpl = NULL; 2249 2250 iflags = 0; 2251 if (flags & SB_TOSTR_OBJDUMP) 2252 iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT); 2253 stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags); 2254 2255 return (sbuf_error(buf)); 2256 } 2257 2258 static int 2259 stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v, 2260 struct voistat *vs, struct sb_iter_ctx *ctx) 2261 { 2262 struct sb_visitcb_ctx *vctx; 2263 struct sb_visit sbv; 2264 2265 vctx = ctx->usrctx; 2266 2267 sbv.tplhash = sb->tplhash; 2268 sbv.voi_id = v->id; 2269 sbv.voi_dtype = v->dtype; 2270 sbv.vs_stype = vs->stype; 2271 sbv.vs_dtype = vs->dtype; 2272 sbv.vs_dsz = vs->dsz; 2273 sbv.vs_data = BLOB_OFFSET(sb, vs->data_off); 2274 sbv.vs_errs = vs->errs; 2275 sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB | 2276 SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT | 2277 SB_IT_LAST_VOISTAT); 2278 2279 return (vctx->cb(&sbv, vctx->usrctx)); 2280 } 2281 2282 int 2283 stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func, 2284 void *usrctx) 2285 { 2286 struct sb_visitcb_ctx vctx; 2287 2288 if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL) 2289 return (EINVAL); 2290 2291 vctx.cb = func; 2292 vctx.usrctx = usrctx; 2293 2294 stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0); 2295 2296 return (0); 2297 } 2298 2299 static int 2300 stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused, 2301 struct voistat *vs, struct sb_iter_ctx *ctx __unused) 2302 { 2303 void *vsd; 2304 2305 if (vs->stype == VS_STYPE_VOISTATE) 2306 return (0); 2307 2308 vsd = BLOB_OFFSET(sb, vs->data_off); 2309 2310 /* Perform the stat type's default reset action. */ 2311 switch (vs->stype) { 2312 case VS_STYPE_SUM: 2313 switch (vs->dtype) { 2314 case VSD_DTYPE_Q_S32: 2315 Q_SIFVAL(VSD(q32, vsd)->sq32, 0); 2316 break; 2317 case VSD_DTYPE_Q_U32: 2318 Q_SIFVAL(VSD(q32, vsd)->uq32, 0); 2319 break; 2320 case VSD_DTYPE_Q_S64: 2321 Q_SIFVAL(VSD(q64, vsd)->sq64, 0); 2322 break; 2323 case VSD_DTYPE_Q_U64: 2324 Q_SIFVAL(VSD(q64, vsd)->uq64, 0); 2325 break; 2326 default: 2327 bzero(vsd, vs->dsz); 2328 break; 2329 } 2330 break; 2331 case VS_STYPE_MAX: 2332 switch (vs->dtype) { 2333 case VSD_DTYPE_Q_S32: 2334 Q_SIFVAL(VSD(q32, vsd)->sq32, 2335 Q_IFMINVAL(VSD(q32, vsd)->sq32)); 2336 break; 2337 case VSD_DTYPE_Q_U32: 2338 Q_SIFVAL(VSD(q32, vsd)->uq32, 2339 Q_IFMINVAL(VSD(q32, vsd)->uq32)); 2340 break; 2341 case VSD_DTYPE_Q_S64: 2342 Q_SIFVAL(VSD(q64, vsd)->sq64, 2343 Q_IFMINVAL(VSD(q64, vsd)->sq64)); 2344 break; 2345 case VSD_DTYPE_Q_U64: 2346 Q_SIFVAL(VSD(q64, vsd)->uq64, 2347 Q_IFMINVAL(VSD(q64, vsd)->uq64)); 2348 break; 2349 default: 2350 memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype], 2351 vs->dsz); 2352 break; 2353 } 2354 break; 2355 case VS_STYPE_MIN: 2356 switch (vs->dtype) { 2357 case VSD_DTYPE_Q_S32: 2358 Q_SIFVAL(VSD(q32, vsd)->sq32, 2359 Q_IFMAXVAL(VSD(q32, vsd)->sq32)); 2360 break; 2361 case VSD_DTYPE_Q_U32: 2362 Q_SIFVAL(VSD(q32, vsd)->uq32, 2363 Q_IFMAXVAL(VSD(q32, vsd)->uq32)); 2364 break; 2365 case VSD_DTYPE_Q_S64: 2366 Q_SIFVAL(VSD(q64, vsd)->sq64, 2367 Q_IFMAXVAL(VSD(q64, vsd)->sq64)); 2368 break; 2369 case VSD_DTYPE_Q_U64: 2370 Q_SIFVAL(VSD(q64, vsd)->uq64, 2371 Q_IFMAXVAL(VSD(q64, vsd)->uq64)); 2372 break; 2373 default: 2374 memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype], 2375 vs->dsz); 2376 break; 2377 } 2378 break; 2379 case VS_STYPE_HIST: 2380 { 2381 /* Reset bucket counts. */ 2382 struct voistatdata_hist *hist; 2383 int i, is32bit; 2384 uint16_t nbkts; 2385 2386 hist = VSD(hist, vsd); 2387 switch (vs->dtype) { 2388 case VSD_DTYPE_CRHIST32: 2389 nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 2390 is32bit = 1; 2391 break; 2392 case VSD_DTYPE_DRHIST32: 2393 nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 2394 is32bit = 1; 2395 break; 2396 case VSD_DTYPE_DVHIST32: 2397 nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 2398 is32bit = 1; 2399 break; 2400 case VSD_DTYPE_CRHIST64: 2401 nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 2402 is32bit = 0; 2403 break; 2404 case VSD_DTYPE_DRHIST64: 2405 nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 2406 is32bit = 0; 2407 break; 2408 case VSD_DTYPE_DVHIST64: 2409 nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 2410 is32bit = 0; 2411 break; 2412 default: 2413 return (0); 2414 } 2415 2416 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob), 2417 is32bit ? sizeof(uint32_t) : sizeof(uint64_t)); 2418 for (i = nbkts - 1; i >= 0; i--) { 2419 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, 2420 bkts[i].cnt), is32bit ? sizeof(uint32_t) : 2421 sizeof(uint64_t)); 2422 } 2423 break; 2424 } 2425 case VS_STYPE_TDGST: 2426 { 2427 /* Reset sample count centroids array/tree. */ 2428 struct voistatdata_tdgst *tdgst; 2429 struct ctdth32 *ctd32tree; 2430 struct ctdth64 *ctd64tree; 2431 struct voistatdata_tdgstctd32 *ctd32; 2432 struct voistatdata_tdgstctd64 *ctd64; 2433 2434 tdgst = VSD(tdgst, vsd); 2435 switch (vs->dtype) { 2436 case VSD_DTYPE_TDGSTCLUST32: 2437 VSD(tdgstclust32, tdgst)->smplcnt = 0; 2438 VSD(tdgstclust32, tdgst)->compcnt = 0; 2439 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 2440 ARB_INIT(ctd32, ctdlnk, ctd32tree, 2441 ARB_MAXNODES(ctd32tree)) { 2442 ctd32->cnt = 0; 2443 Q_SIFVAL(ctd32->mu, 0); 2444 } 2445 #ifdef DIAGNOSTIC 2446 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 2447 #endif 2448 break; 2449 case VSD_DTYPE_TDGSTCLUST64: 2450 VSD(tdgstclust64, tdgst)->smplcnt = 0; 2451 VSD(tdgstclust64, tdgst)->compcnt = 0; 2452 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 2453 ARB_INIT(ctd64, ctdlnk, ctd64tree, 2454 ARB_MAXNODES(ctd64tree)) { 2455 ctd64->cnt = 0; 2456 Q_SIFVAL(ctd64->mu, 0); 2457 } 2458 #ifdef DIAGNOSTIC 2459 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 2460 #endif 2461 break; 2462 default: 2463 return (0); 2464 } 2465 break; 2466 } 2467 default: 2468 KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 2469 break; 2470 } 2471 2472 vs->errs = 0; 2473 vs->flags &= ~VS_VSDVALID; 2474 2475 return (0); 2476 } 2477 2478 int 2479 stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz, 2480 struct statsblobv1 *src, uint32_t flags) 2481 { 2482 int error; 2483 2484 if (src != NULL && src->abi == STATS_ABI_V1) { 2485 error = stats_v1_blob_clone(dst, dstmaxsz, src, flags); 2486 if (!error) { 2487 if (flags & SB_CLONE_RSTSRC) { 2488 stats_v1_blob_iter(src, 2489 stats_v1_icb_reset_voistat, NULL, 0); 2490 src->lastrst = stats_sbinuptime(); 2491 } 2492 stats_v1_blob_finalise(*dst); 2493 } 2494 } else 2495 error = EINVAL; 2496 2497 return (error); 2498 } 2499 2500 static inline int 2501 stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused, 2502 struct voistatdata *voival, struct voistat *vs, void *vsd) 2503 { 2504 int error; 2505 2506 KASSERT(vs->dtype < VSD_NUM_DTYPES, 2507 ("Unknown VSD dtype %d", vs->dtype)); 2508 2509 error = 0; 2510 2511 switch (vs->dtype) { 2512 case VSD_DTYPE_INT_S32: 2513 if (VSD(int32, vsd)->s32 < voival->int32.s32) { 2514 VSD(int32, vsd)->s32 = voival->int32.s32; 2515 vs->flags |= VS_VSDVALID; 2516 } 2517 break; 2518 case VSD_DTYPE_INT_U32: 2519 if (VSD(int32, vsd)->u32 < voival->int32.u32) { 2520 VSD(int32, vsd)->u32 = voival->int32.u32; 2521 vs->flags |= VS_VSDVALID; 2522 } 2523 break; 2524 case VSD_DTYPE_INT_S64: 2525 if (VSD(int64, vsd)->s64 < voival->int64.s64) { 2526 VSD(int64, vsd)->s64 = voival->int64.s64; 2527 vs->flags |= VS_VSDVALID; 2528 } 2529 break; 2530 case VSD_DTYPE_INT_U64: 2531 if (VSD(int64, vsd)->u64 < voival->int64.u64) { 2532 VSD(int64, vsd)->u64 = voival->int64.u64; 2533 vs->flags |= VS_VSDVALID; 2534 } 2535 break; 2536 case VSD_DTYPE_INT_SLONG: 2537 if (VSD(intlong, vsd)->slong < voival->intlong.slong) { 2538 VSD(intlong, vsd)->slong = voival->intlong.slong; 2539 vs->flags |= VS_VSDVALID; 2540 } 2541 break; 2542 case VSD_DTYPE_INT_ULONG: 2543 if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) { 2544 VSD(intlong, vsd)->ulong = voival->intlong.ulong; 2545 vs->flags |= VS_VSDVALID; 2546 } 2547 break; 2548 case VSD_DTYPE_Q_S32: 2549 if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 2550 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 2551 voival->q32.sq32)))) { 2552 vs->flags |= VS_VSDVALID; 2553 } 2554 break; 2555 case VSD_DTYPE_Q_U32: 2556 if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 2557 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 2558 voival->q32.uq32)))) { 2559 vs->flags |= VS_VSDVALID; 2560 } 2561 break; 2562 case VSD_DTYPE_Q_S64: 2563 if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 2564 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 2565 voival->q64.sq64)))) { 2566 vs->flags |= VS_VSDVALID; 2567 } 2568 break; 2569 case VSD_DTYPE_Q_U64: 2570 if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 2571 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 2572 voival->q64.uq64)))) { 2573 vs->flags |= VS_VSDVALID; 2574 } 2575 break; 2576 default: 2577 error = EINVAL; 2578 break; 2579 } 2580 2581 return (error); 2582 } 2583 2584 static inline int 2585 stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused, 2586 struct voistatdata *voival, struct voistat *vs, void *vsd) 2587 { 2588 int error; 2589 2590 KASSERT(vs->dtype < VSD_NUM_DTYPES, 2591 ("Unknown VSD dtype %d", vs->dtype)); 2592 2593 error = 0; 2594 2595 switch (vs->dtype) { 2596 case VSD_DTYPE_INT_S32: 2597 if (VSD(int32, vsd)->s32 > voival->int32.s32) { 2598 VSD(int32, vsd)->s32 = voival->int32.s32; 2599 vs->flags |= VS_VSDVALID; 2600 } 2601 break; 2602 case VSD_DTYPE_INT_U32: 2603 if (VSD(int32, vsd)->u32 > voival->int32.u32) { 2604 VSD(int32, vsd)->u32 = voival->int32.u32; 2605 vs->flags |= VS_VSDVALID; 2606 } 2607 break; 2608 case VSD_DTYPE_INT_S64: 2609 if (VSD(int64, vsd)->s64 > voival->int64.s64) { 2610 VSD(int64, vsd)->s64 = voival->int64.s64; 2611 vs->flags |= VS_VSDVALID; 2612 } 2613 break; 2614 case VSD_DTYPE_INT_U64: 2615 if (VSD(int64, vsd)->u64 > voival->int64.u64) { 2616 VSD(int64, vsd)->u64 = voival->int64.u64; 2617 vs->flags |= VS_VSDVALID; 2618 } 2619 break; 2620 case VSD_DTYPE_INT_SLONG: 2621 if (VSD(intlong, vsd)->slong > voival->intlong.slong) { 2622 VSD(intlong, vsd)->slong = voival->intlong.slong; 2623 vs->flags |= VS_VSDVALID; 2624 } 2625 break; 2626 case VSD_DTYPE_INT_ULONG: 2627 if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) { 2628 VSD(intlong, vsd)->ulong = voival->intlong.ulong; 2629 vs->flags |= VS_VSDVALID; 2630 } 2631 break; 2632 case VSD_DTYPE_Q_S32: 2633 if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && 2634 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, 2635 voival->q32.sq32)))) { 2636 vs->flags |= VS_VSDVALID; 2637 } 2638 break; 2639 case VSD_DTYPE_Q_U32: 2640 if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && 2641 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, 2642 voival->q32.uq32)))) { 2643 vs->flags |= VS_VSDVALID; 2644 } 2645 break; 2646 case VSD_DTYPE_Q_S64: 2647 if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && 2648 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, 2649 voival->q64.sq64)))) { 2650 vs->flags |= VS_VSDVALID; 2651 } 2652 break; 2653 case VSD_DTYPE_Q_U64: 2654 if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && 2655 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, 2656 voival->q64.uq64)))) { 2657 vs->flags |= VS_VSDVALID; 2658 } 2659 break; 2660 default: 2661 error = EINVAL; 2662 break; 2663 } 2664 2665 return (error); 2666 } 2667 2668 static inline int 2669 stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused, 2670 struct voistatdata *voival, struct voistat *vs, void *vsd) 2671 { 2672 int error; 2673 2674 KASSERT(vs->dtype < VSD_NUM_DTYPES, 2675 ("Unknown VSD dtype %d", vs->dtype)); 2676 2677 error = 0; 2678 2679 switch (vs->dtype) { 2680 case VSD_DTYPE_INT_S32: 2681 VSD(int32, vsd)->s32 += voival->int32.s32; 2682 break; 2683 case VSD_DTYPE_INT_U32: 2684 VSD(int32, vsd)->u32 += voival->int32.u32; 2685 break; 2686 case VSD_DTYPE_INT_S64: 2687 VSD(int64, vsd)->s64 += voival->int64.s64; 2688 break; 2689 case VSD_DTYPE_INT_U64: 2690 VSD(int64, vsd)->u64 += voival->int64.u64; 2691 break; 2692 case VSD_DTYPE_INT_SLONG: 2693 VSD(intlong, vsd)->slong += voival->intlong.slong; 2694 break; 2695 case VSD_DTYPE_INT_ULONG: 2696 VSD(intlong, vsd)->ulong += voival->intlong.ulong; 2697 break; 2698 case VSD_DTYPE_Q_S32: 2699 error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32); 2700 break; 2701 case VSD_DTYPE_Q_U32: 2702 error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32); 2703 break; 2704 case VSD_DTYPE_Q_S64: 2705 error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64); 2706 break; 2707 case VSD_DTYPE_Q_U64: 2708 error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64); 2709 break; 2710 default: 2711 error = EINVAL; 2712 break; 2713 } 2714 2715 if (!error) 2716 vs->flags |= VS_VSDVALID; 2717 2718 return (error); 2719 } 2720 2721 static inline int 2722 stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival, 2723 struct voistat *vs, struct voistatdata_hist *hist) 2724 { 2725 struct voistatdata_numeric *bkt_lb, *bkt_ub; 2726 uint64_t *oob64, *cnt64; 2727 uint32_t *oob32, *cnt32; 2728 int error, i, found, is32bit, has_ub, eq_only; 2729 2730 error = 0; 2731 2732 switch (vs->dtype) { 2733 case VSD_DTYPE_CRHIST32: 2734 i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); 2735 is32bit = 1; 2736 has_ub = eq_only = 0; 2737 oob32 = &VSD(crhist32, hist)->oob; 2738 break; 2739 case VSD_DTYPE_DRHIST32: 2740 i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); 2741 is32bit = has_ub = 1; 2742 eq_only = 0; 2743 oob32 = &VSD(drhist32, hist)->oob; 2744 break; 2745 case VSD_DTYPE_DVHIST32: 2746 i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); 2747 is32bit = eq_only = 1; 2748 has_ub = 0; 2749 oob32 = &VSD(dvhist32, hist)->oob; 2750 break; 2751 case VSD_DTYPE_CRHIST64: 2752 i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); 2753 is32bit = has_ub = eq_only = 0; 2754 oob64 = &VSD(crhist64, hist)->oob; 2755 break; 2756 case VSD_DTYPE_DRHIST64: 2757 i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); 2758 is32bit = eq_only = 0; 2759 has_ub = 1; 2760 oob64 = &VSD(drhist64, hist)->oob; 2761 break; 2762 case VSD_DTYPE_DVHIST64: 2763 i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); 2764 is32bit = has_ub = 0; 2765 eq_only = 1; 2766 oob64 = &VSD(dvhist64, hist)->oob; 2767 break; 2768 default: 2769 return (EINVAL); 2770 } 2771 i--; /* Adjust for 0-based array index. */ 2772 2773 /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */ 2774 for (found = 0; i >= 0 && !found; i--) { 2775 switch (vs->dtype) { 2776 case VSD_DTYPE_CRHIST32: 2777 bkt_lb = &VSD(crhist32, hist)->bkts[i].lb; 2778 cnt32 = &VSD(crhist32, hist)->bkts[i].cnt; 2779 break; 2780 case VSD_DTYPE_DRHIST32: 2781 bkt_lb = &VSD(drhist32, hist)->bkts[i].lb; 2782 bkt_ub = &VSD(drhist32, hist)->bkts[i].ub; 2783 cnt32 = &VSD(drhist32, hist)->bkts[i].cnt; 2784 break; 2785 case VSD_DTYPE_DVHIST32: 2786 bkt_lb = &VSD(dvhist32, hist)->bkts[i].val; 2787 cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt; 2788 break; 2789 case VSD_DTYPE_CRHIST64: 2790 bkt_lb = &VSD(crhist64, hist)->bkts[i].lb; 2791 cnt64 = &VSD(crhist64, hist)->bkts[i].cnt; 2792 break; 2793 case VSD_DTYPE_DRHIST64: 2794 bkt_lb = &VSD(drhist64, hist)->bkts[i].lb; 2795 bkt_ub = &VSD(drhist64, hist)->bkts[i].ub; 2796 cnt64 = &VSD(drhist64, hist)->bkts[i].cnt; 2797 break; 2798 case VSD_DTYPE_DVHIST64: 2799 bkt_lb = &VSD(dvhist64, hist)->bkts[i].val; 2800 cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt; 2801 break; 2802 default: 2803 return (EINVAL); 2804 } 2805 2806 switch (voi_dtype) { 2807 case VSD_DTYPE_INT_S32: 2808 if (voival->int32.s32 >= bkt_lb->int32.s32) { 2809 if ((eq_only && voival->int32.s32 == 2810 bkt_lb->int32.s32) || 2811 (!eq_only && (!has_ub || 2812 voival->int32.s32 < bkt_ub->int32.s32))) 2813 found = 1; 2814 } 2815 break; 2816 case VSD_DTYPE_INT_U32: 2817 if (voival->int32.u32 >= bkt_lb->int32.u32) { 2818 if ((eq_only && voival->int32.u32 == 2819 bkt_lb->int32.u32) || 2820 (!eq_only && (!has_ub || 2821 voival->int32.u32 < bkt_ub->int32.u32))) 2822 found = 1; 2823 } 2824 break; 2825 case VSD_DTYPE_INT_S64: 2826 if (voival->int64.s64 >= bkt_lb->int64.s64) 2827 if ((eq_only && voival->int64.s64 == 2828 bkt_lb->int64.s64) || 2829 (!eq_only && (!has_ub || 2830 voival->int64.s64 < bkt_ub->int64.s64))) 2831 found = 1; 2832 break; 2833 case VSD_DTYPE_INT_U64: 2834 if (voival->int64.u64 >= bkt_lb->int64.u64) 2835 if ((eq_only && voival->int64.u64 == 2836 bkt_lb->int64.u64) || 2837 (!eq_only && (!has_ub || 2838 voival->int64.u64 < bkt_ub->int64.u64))) 2839 found = 1; 2840 break; 2841 case VSD_DTYPE_INT_SLONG: 2842 if (voival->intlong.slong >= bkt_lb->intlong.slong) 2843 if ((eq_only && voival->intlong.slong == 2844 bkt_lb->intlong.slong) || 2845 (!eq_only && (!has_ub || 2846 voival->intlong.slong < 2847 bkt_ub->intlong.slong))) 2848 found = 1; 2849 break; 2850 case VSD_DTYPE_INT_ULONG: 2851 if (voival->intlong.ulong >= bkt_lb->intlong.ulong) 2852 if ((eq_only && voival->intlong.ulong == 2853 bkt_lb->intlong.ulong) || 2854 (!eq_only && (!has_ub || 2855 voival->intlong.ulong < 2856 bkt_ub->intlong.ulong))) 2857 found = 1; 2858 break; 2859 case VSD_DTYPE_Q_S32: 2860 if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32)) 2861 if ((eq_only && Q_QEQ(voival->q32.sq32, 2862 bkt_lb->q32.sq32)) || 2863 (!eq_only && (!has_ub || 2864 Q_QLTQ(voival->q32.sq32, 2865 bkt_ub->q32.sq32)))) 2866 found = 1; 2867 break; 2868 case VSD_DTYPE_Q_U32: 2869 if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32)) 2870 if ((eq_only && Q_QEQ(voival->q32.uq32, 2871 bkt_lb->q32.uq32)) || 2872 (!eq_only && (!has_ub || 2873 Q_QLTQ(voival->q32.uq32, 2874 bkt_ub->q32.uq32)))) 2875 found = 1; 2876 break; 2877 case VSD_DTYPE_Q_S64: 2878 if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64)) 2879 if ((eq_only && Q_QEQ(voival->q64.sq64, 2880 bkt_lb->q64.sq64)) || 2881 (!eq_only && (!has_ub || 2882 Q_QLTQ(voival->q64.sq64, 2883 bkt_ub->q64.sq64)))) 2884 found = 1; 2885 break; 2886 case VSD_DTYPE_Q_U64: 2887 if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64)) 2888 if ((eq_only && Q_QEQ(voival->q64.uq64, 2889 bkt_lb->q64.uq64)) || 2890 (!eq_only && (!has_ub || 2891 Q_QLTQ(voival->q64.uq64, 2892 bkt_ub->q64.uq64)))) 2893 found = 1; 2894 break; 2895 default: 2896 break; 2897 } 2898 } 2899 2900 if (found) { 2901 if (is32bit) 2902 *cnt32 += 1; 2903 else 2904 *cnt64 += 1; 2905 } else { 2906 if (is32bit) 2907 *oob32 += 1; 2908 else 2909 *oob64 += 1; 2910 } 2911 2912 vs->flags |= VS_VSDVALID; 2913 return (error); 2914 } 2915 2916 static inline int 2917 stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype, 2918 struct voistatdata_tdgst *tdgst, int attempt) 2919 { 2920 struct ctdth32 *ctd32tree; 2921 struct ctdth64 *ctd64tree; 2922 struct voistatdata_tdgstctd32 *ctd32; 2923 struct voistatdata_tdgstctd64 *ctd64; 2924 uint64_t ebits, idxmask; 2925 uint32_t bitsperidx, nebits; 2926 int error, idx, is32bit, maxctds, remctds, tmperr; 2927 2928 error = 0; 2929 2930 switch (vs_dtype) { 2931 case VSD_DTYPE_TDGSTCLUST32: 2932 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 2933 if (!ARB_FULL(ctd32tree)) 2934 return (0); 2935 VSD(tdgstclust32, tdgst)->compcnt++; 2936 maxctds = remctds = ARB_MAXNODES(ctd32tree); 2937 ARB_RESET_TREE(ctd32tree, ctdth32, maxctds); 2938 VSD(tdgstclust32, tdgst)->smplcnt = 0; 2939 is32bit = 1; 2940 ctd64tree = NULL; 2941 ctd64 = NULL; 2942 #ifdef DIAGNOSTIC 2943 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); 2944 #endif 2945 break; 2946 case VSD_DTYPE_TDGSTCLUST64: 2947 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 2948 if (!ARB_FULL(ctd64tree)) 2949 return (0); 2950 VSD(tdgstclust64, tdgst)->compcnt++; 2951 maxctds = remctds = ARB_MAXNODES(ctd64tree); 2952 ARB_RESET_TREE(ctd64tree, ctdth64, maxctds); 2953 VSD(tdgstclust64, tdgst)->smplcnt = 0; 2954 is32bit = 0; 2955 ctd32tree = NULL; 2956 ctd32 = NULL; 2957 #ifdef DIAGNOSTIC 2958 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); 2959 #endif 2960 break; 2961 default: 2962 return (EINVAL); 2963 } 2964 2965 /* 2966 * Rebuild the t-digest ARB by pseudorandomly selecting centroids and 2967 * re-inserting the mu/cnt of each as a value and corresponding weight. 2968 */ 2969 2970 /* 2971 * XXXCEM: random(9) is currently rand(3), not random(3). rand(3) 2972 * RAND_MAX happens to be approximately 31 bits (range [0, 2973 * 0x7ffffffd]), so the math kinda works out. When/if this portion of 2974 * the code is compiled in userspace, it gets the random(3) behavior, 2975 * which has expected range [0, 0x7fffffff]. 2976 */ 2977 #define bitsperrand 31 2978 ebits = 0; 2979 nebits = 0; 2980 bitsperidx = fls(maxctds); 2981 KASSERT(bitsperidx <= sizeof(ebits) << 3, 2982 ("%s: bitsperidx=%d, ebits=%d", 2983 __func__, bitsperidx, (int)(sizeof(ebits) << 3))); 2984 idxmask = (UINT64_C(1) << bitsperidx) - 1; 2985 2986 /* Initialise the free list with randomised centroid indices. */ 2987 for (; remctds > 0; remctds--) { 2988 while (nebits < bitsperidx) { 2989 ebits |= ((uint64_t)random()) << nebits; 2990 nebits += bitsperrand; 2991 if (nebits > (sizeof(ebits) << 3)) 2992 nebits = sizeof(ebits) << 3; 2993 } 2994 idx = ebits & idxmask; 2995 nebits -= bitsperidx; 2996 ebits >>= bitsperidx; 2997 2998 /* 2999 * Select the next centroid to put on the ARB free list. We 3000 * start with the centroid at our randomly selected array index, 3001 * and work our way forwards until finding one (the latter 3002 * aspect reduces re-insertion randomness, but is good enough). 3003 */ 3004 do { 3005 if (idx >= maxctds) 3006 idx %= maxctds; 3007 3008 if (is32bit) 3009 ctd32 = ARB_NODE(ctd32tree, idx); 3010 else 3011 ctd64 = ARB_NODE(ctd64tree, idx); 3012 } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) : 3013 ARB_ISFREE(ctd64, ctdlnk)) && ++idx); 3014 3015 /* Put the centroid on the ARB free list. */ 3016 if (is32bit) 3017 ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk); 3018 else 3019 ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk); 3020 } 3021 3022 /* 3023 * The free list now contains the randomised indices of every centroid. 3024 * Walk the free list from start to end, re-inserting each centroid's 3025 * mu/cnt. The tdgst_add() call may or may not consume the free centroid 3026 * we re-insert values from during each loop iteration, so we must latch 3027 * the index of the next free list centroid before the re-insertion 3028 * call. The previous loop above should have left the centroid pointer 3029 * pointing to the element at the head of the free list. 3030 */ 3031 KASSERT((is32bit ? 3032 ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) : 3033 ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)), 3034 ("%s: t-digest ARB@%p free list bug", __func__, 3035 (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 3036 remctds = maxctds; 3037 while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) { 3038 tmperr = 0; 3039 if (is32bit) { 3040 s64q_t x; 3041 3042 idx = ARB_NEXTFREEIDX(ctd32, ctdlnk); 3043 /* Cloning a s32q_t into a s64q_t should never fail. */ 3044 tmperr = Q_QCLONEQ(&x, ctd32->mu); 3045 tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add( 3046 vs_dtype, tdgst, x, ctd32->cnt, attempt); 3047 ctd32 = ARB_NODE(ctd32tree, idx); 3048 KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk), 3049 ("%s: t-digest ARB@%p free list bug", __func__, 3050 ctd32tree)); 3051 } else { 3052 idx = ARB_NEXTFREEIDX(ctd64, ctdlnk); 3053 tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst, 3054 ctd64->mu, ctd64->cnt, attempt); 3055 ctd64 = ARB_NODE(ctd64tree, idx); 3056 KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk), 3057 ("%s: t-digest ARB@%p free list bug", __func__, 3058 ctd64tree)); 3059 } 3060 /* 3061 * This process should not produce errors, bugs notwithstanding. 3062 * Just in case, latch any errors and attempt all re-insertions. 3063 */ 3064 error = tmperr ? tmperr : error; 3065 remctds--; 3066 } 3067 3068 KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__, 3069 (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); 3070 3071 return (error); 3072 } 3073 3074 static inline int 3075 stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst, 3076 s64q_t x, uint64_t weight, int attempt) 3077 { 3078 #ifdef DIAGNOSTIC 3079 char qstr[Q_MAXSTRLEN(x, 10)]; 3080 #endif 3081 struct ctdth32 *ctd32tree; 3082 struct ctdth64 *ctd64tree; 3083 void *closest, *cur, *lb, *ub; 3084 struct voistatdata_tdgstctd32 *ctd32; 3085 struct voistatdata_tdgstctd64 *ctd64; 3086 uint64_t cnt, smplcnt, sum, tmpsum; 3087 s64q_t k, minz, q, z; 3088 int error, is32bit, n; 3089 3090 error = 0; 3091 minz = Q_INI(&z, 0, 0, Q_NFBITS(x)); 3092 3093 switch (vs_dtype) { 3094 case VSD_DTYPE_TDGSTCLUST32: 3095 if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt) 3096 error = EOVERFLOW; 3097 smplcnt = VSD(tdgstclust32, tdgst)->smplcnt; 3098 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; 3099 is32bit = 1; 3100 ctd64tree = NULL; 3101 ctd64 = NULL; 3102 break; 3103 case VSD_DTYPE_TDGSTCLUST64: 3104 if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt) 3105 error = EOVERFLOW; 3106 smplcnt = VSD(tdgstclust64, tdgst)->smplcnt; 3107 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; 3108 is32bit = 0; 3109 ctd32tree = NULL; 3110 ctd32 = NULL; 3111 break; 3112 default: 3113 error = EINVAL; 3114 break; 3115 } 3116 3117 if (error) 3118 return (error); 3119 3120 /* 3121 * Inspired by Ted Dunning's AVLTreeDigest.java 3122 */ 3123 do { 3124 #if defined(DIAGNOSTIC) 3125 KASSERT(attempt < 5, 3126 ("%s: Too many attempts", __func__)); 3127 #endif 3128 if (attempt >= 5) 3129 return (EAGAIN); 3130 3131 Q_SIFVAL(minz, Q_IFMAXVAL(minz)); 3132 closest = ub = NULL; 3133 sum = tmpsum = 0; 3134 3135 if (is32bit) 3136 lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree)); 3137 else 3138 lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree)); 3139 3140 if (lb == NULL) /* Empty tree. */ 3141 lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) : 3142 (void *)ARB_ROOT(ctd64tree)); 3143 3144 /* 3145 * Find the set of centroids with minimum distance to x and 3146 * compute the sum of counts for all centroids with mean less 3147 * than the first centroid in the set. 3148 */ 3149 for (; cur != NULL; 3150 cur = (is32bit ? 3151 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 3152 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 3153 if (is32bit) { 3154 cnt = ctd32->cnt; 3155 KASSERT(Q_PRECEQ(ctd32->mu, x), 3156 ("%s: Q_RELPREC(mu,x)=%d", __func__, 3157 Q_RELPREC(ctd32->mu, x))); 3158 /* Ok to assign as both have same precision. */ 3159 z = ctd32->mu; 3160 } else { 3161 cnt = ctd64->cnt; 3162 KASSERT(Q_PRECEQ(ctd64->mu, x), 3163 ("%s: Q_RELPREC(mu,x)=%d", __func__, 3164 Q_RELPREC(ctd64->mu, x))); 3165 /* Ok to assign as both have same precision. */ 3166 z = ctd64->mu; 3167 } 3168 3169 error = Q_QSUBQ(&z, x); 3170 #if defined(DIAGNOSTIC) 3171 KASSERT(!error, ("%s: unexpected error %d", __func__, 3172 error)); 3173 #endif 3174 if (error) 3175 return (error); 3176 3177 z = Q_QABS(z); 3178 if (Q_QLTQ(z, minz)) { 3179 minz = z; 3180 lb = cur; 3181 sum = tmpsum; 3182 tmpsum += cnt; 3183 } else if (Q_QGTQ(z, minz)) { 3184 ub = cur; 3185 break; 3186 } 3187 } 3188 3189 cur = (is32bit ? 3190 (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) : 3191 (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb)); 3192 3193 for (n = 0; cur != ub; cur = (is32bit ? 3194 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : 3195 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { 3196 if (is32bit) 3197 cnt = ctd32->cnt; 3198 else 3199 cnt = ctd64->cnt; 3200 3201 q = Q_CTRLINI(16); 3202 if (smplcnt == 1) 3203 error = Q_QFRACI(&q, 1, 2); 3204 else 3205 /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */ 3206 error = Q_QFRACI(&q, (sum << 1) + cnt - 1, 3207 (smplcnt - 1) << 1); 3208 k = q; 3209 /* k = q x 4 x samplcnt x attempt */ 3210 error |= Q_QMULI(&k, 4 * smplcnt * attempt); 3211 /* k = k x (1 - q) */ 3212 error |= Q_QSUBI(&q, 1); 3213 q = Q_QABS(q); 3214 error |= Q_QMULQ(&k, q); 3215 #if defined(DIAGNOSTIC) 3216 #if !defined(_KERNEL) 3217 double q_dbl, k_dbl, q2d, k2d; 3218 q2d = Q_Q2D(q); 3219 k2d = Q_Q2D(k); 3220 q_dbl = smplcnt == 1 ? 0.5 : 3221 (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1); 3222 k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt; 3223 /* 3224 * If the difference between q and q_dbl is greater than 3225 * the fractional precision of q, something is off. 3226 * NB: q is holding the value of 1 - q 3227 */ 3228 q_dbl = 1.0 - q_dbl; 3229 KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) < 3230 (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))), 3231 ("Q-type q bad precision")); 3232 KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) < 3233 1.0 + (0.01 * smplcnt), 3234 ("Q-type k bad precision")); 3235 #endif /* !_KERNEL */ 3236 KASSERT(!error, ("%s: unexpected error %d", __func__, 3237 error)); 3238 #endif /* DIAGNOSTIC */ 3239 if (error) 3240 return (error); 3241 if ((is32bit && ((ctd32->cnt + weight) <= 3242 (uint64_t)Q_GIVAL(k))) || 3243 (!is32bit && ((ctd64->cnt + weight) <= 3244 (uint64_t)Q_GIVAL(k)))) { 3245 n++; 3246 /* random() produces 31 bits. */ 3247 if (random() < (INT32_MAX / n)) 3248 closest = cur; 3249 } 3250 sum += cnt; 3251 } 3252 } while (closest == NULL && 3253 (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) && 3254 (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst, 3255 attempt++)) == 0); 3256 3257 if (error) 3258 return (error); 3259 3260 if (closest != NULL) { 3261 /* Merge with an existing centroid. */ 3262 if (is32bit) { 3263 ctd32 = (struct voistatdata_tdgstctd32 *)closest; 3264 error = Q_QSUBQ(&x, ctd32->mu); 3265 /* 3266 * The following calculation "x / (cnt + weight)" 3267 * computes the amount by which to adjust the centroid's 3268 * mu value in order to merge in the VOI sample. 3269 * 3270 * It can underflow (Q_QDIVI() returns ERANGE) when the 3271 * user centroids' fractional precision (which is 3272 * inherited by 'x') is too low to represent the result. 3273 * 3274 * A sophisticated approach to dealing with this issue 3275 * would minimise accumulation of error by tracking 3276 * underflow per centroid and making an adjustment when 3277 * a LSB's worth of underflow has accumulated. 3278 * 3279 * A simpler approach is to let the result underflow 3280 * i.e. merge the VOI sample into the centroid without 3281 * adjusting the centroid's mu, and rely on the user to 3282 * specify their t-digest with sufficient centroid 3283 * fractional precision such that the accumulation of 3284 * error from multiple underflows is of no material 3285 * consequence to the centroid's final value of mu. 3286 * 3287 * For the moment, the latter approach is employed by 3288 * simply ignoring ERANGE here. 3289 * 3290 * XXXLAS: Per-centroid underflow tracking is likely too 3291 * onerous, but it probably makes sense to accumulate a 3292 * single underflow error variable across all centroids 3293 * and report it as part of the digest to provide 3294 * additional visibility into the digest's fidelity. 3295 */ 3296 error = error ? error : 3297 Q_QDIVI(&x, ctd32->cnt + weight); 3298 if ((error && error != ERANGE) 3299 || (error = Q_QADDQ(&ctd32->mu, x))) { 3300 #ifdef DIAGNOSTIC 3301 KASSERT(!error, ("%s: unexpected error %d", 3302 __func__, error)); 3303 #endif 3304 return (error); 3305 } 3306 ctd32->cnt += weight; 3307 error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) == 3308 NULL ? 0 : EALREADY; 3309 #ifdef DIAGNOSTIC 3310 RB_REINSERT(rbctdth32, 3311 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 3312 #endif 3313 } else { 3314 ctd64 = (struct voistatdata_tdgstctd64 *)closest; 3315 error = Q_QSUBQ(&x, ctd64->mu); 3316 error = error ? error : 3317 Q_QDIVI(&x, ctd64->cnt + weight); 3318 /* Refer to is32bit ERANGE discussion above. */ 3319 if ((error && error != ERANGE) 3320 || (error = Q_QADDQ(&ctd64->mu, x))) { 3321 KASSERT(!error, ("%s: unexpected error %d", 3322 __func__, error)); 3323 return (error); 3324 } 3325 ctd64->cnt += weight; 3326 error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) == 3327 NULL ? 0 : EALREADY; 3328 #ifdef DIAGNOSTIC 3329 RB_REINSERT(rbctdth64, 3330 &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64); 3331 #endif 3332 } 3333 } else { 3334 /* 3335 * Add a new centroid. If digest compression is working 3336 * correctly, there should always be at least one free. 3337 */ 3338 if (is32bit) { 3339 ctd32 = ARB_GETFREE(ctd32tree, ctdlnk); 3340 #ifdef DIAGNOSTIC 3341 KASSERT(ctd32 != NULL, 3342 ("%s: t-digest@%p has no free centroids", 3343 __func__, tdgst)); 3344 #endif 3345 if (ctd32 == NULL) 3346 return (EAGAIN); 3347 if ((error = Q_QCPYVALQ(&ctd32->mu, x))) 3348 return (error); 3349 ctd32->cnt = weight; 3350 error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ? 3351 0 : EALREADY; 3352 #ifdef DIAGNOSTIC 3353 RB_INSERT(rbctdth32, 3354 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); 3355 #endif 3356 } else { 3357 ctd64 = ARB_GETFREE(ctd64tree, ctdlnk); 3358 #ifdef DIAGNOSTIC 3359 KASSERT(ctd64 != NULL, 3360 ("%s: t-digest@%p has no free centroids", 3361 __func__, tdgst)); 3362 #endif 3363 if (ctd64 == NULL) /* Should not happen. */ 3364 return (EAGAIN); 3365 /* Direct assignment ok as both have same type/prec. */ 3366 ctd64->mu = x; 3367 ctd64->cnt = weight; 3368 error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ? 3369 0 : EALREADY; 3370 #ifdef DIAGNOSTIC 3371 RB_INSERT(rbctdth64, &VSD(tdgstclust64, 3372 tdgst)->rbctdtree, ctd64); 3373 #endif 3374 } 3375 } 3376 3377 if (is32bit) 3378 VSD(tdgstclust32, tdgst)->smplcnt += weight; 3379 else { 3380 VSD(tdgstclust64, tdgst)->smplcnt += weight; 3381 3382 #ifdef DIAGNOSTIC 3383 struct rbctdth64 *rbctdtree = 3384 &VSD(tdgstclust64, tdgst)->rbctdtree; 3385 struct voistatdata_tdgstctd64 *rbctd64; 3386 int i = 0; 3387 ARB_FOREACH(ctd64, ctdth64, ctd64tree) { 3388 rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) : 3389 RB_NEXT(rbctdth64, rbctdtree, rbctd64)); 3390 3391 if (i >= ARB_CURNODES(ctd64tree) 3392 || ctd64 != rbctd64 3393 || ARB_MIN(ctdth64, ctd64tree) != 3394 RB_MIN(rbctdth64, rbctdtree) 3395 || ARB_MAX(ctdth64, ctd64tree) != 3396 RB_MAX(rbctdth64, rbctdtree) 3397 || ARB_LEFTIDX(ctd64, ctdlnk) != 3398 ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk)) 3399 || ARB_RIGHTIDX(ctd64, ctdlnk) != 3400 ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk)) 3401 || ARB_PARENTIDX(ctd64, ctdlnk) != 3402 ARB_SELFIDX(ctd64tree, 3403 RB_PARENT(rbctd64, rblnk))) { 3404 Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr)); 3405 printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 3406 "mu=%s\n", 3407 (int)ARB_SELFIDX(ctd64tree, ctd64), 3408 ARB_PARENTIDX(ctd64, ctdlnk), 3409 ARB_LEFTIDX(ctd64, ctdlnk), 3410 ARB_RIGHTIDX(ctd64, ctdlnk), 3411 ARB_COLOR(ctd64, ctdlnk), 3412 qstr); 3413 3414 Q_TOSTR(rbctd64->mu, -1, 10, qstr, 3415 sizeof(qstr)); 3416 struct voistatdata_tdgstctd64 *parent; 3417 parent = RB_PARENT(rbctd64, rblnk); 3418 int rb_color = 3419 parent == NULL ? 0 : 3420 RB_LEFT(parent, rblnk) == rbctd64 ? 3421 (_RB_BITSUP(parent, rblnk) & _RB_L) != 0 : 3422 (_RB_BITSUP(parent, rblnk) & _RB_R) != 0; 3423 printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d " 3424 "mu=%s\n", 3425 (int)ARB_SELFIDX(ctd64tree, rbctd64), 3426 (int)ARB_SELFIDX(ctd64tree, 3427 RB_PARENT(rbctd64, rblnk)), 3428 (int)ARB_SELFIDX(ctd64tree, 3429 RB_LEFT(rbctd64, rblnk)), 3430 (int)ARB_SELFIDX(ctd64tree, 3431 RB_RIGHT(rbctd64, rblnk)), 3432 rb_color, 3433 qstr); 3434 3435 panic("RB@%p and ARB@%p trees differ\n", 3436 rbctdtree, ctd64tree); 3437 } 3438 i++; 3439 } 3440 #endif /* DIAGNOSTIC */ 3441 } 3442 3443 return (error); 3444 } 3445 3446 static inline int 3447 stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival, 3448 struct voistat *vs, struct voistatdata_tdgst *tdgst) 3449 { 3450 s64q_t x; 3451 int error; 3452 3453 error = 0; 3454 3455 switch (vs->dtype) { 3456 case VSD_DTYPE_TDGSTCLUST32: 3457 /* Use same precision as the user's centroids. */ 3458 Q_INI(&x, 0, 0, Q_NFBITS( 3459 ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu)); 3460 break; 3461 case VSD_DTYPE_TDGSTCLUST64: 3462 /* Use same precision as the user's centroids. */ 3463 Q_INI(&x, 0, 0, Q_NFBITS( 3464 ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu)); 3465 break; 3466 default: 3467 KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 || 3468 vs->dtype == VSD_DTYPE_TDGSTCLUST64, 3469 ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>", 3470 __func__, vs->dtype)); 3471 return (EINVAL); 3472 } 3473 3474 /* 3475 * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid 3476 * returning EOVERFLOW if the voival would have fit in a u64q_t. 3477 */ 3478 switch (voi_dtype) { 3479 case VSD_DTYPE_INT_S32: 3480 error = Q_QCPYVALI(&x, voival->int32.s32); 3481 break; 3482 case VSD_DTYPE_INT_U32: 3483 error = Q_QCPYVALI(&x, voival->int32.u32); 3484 break; 3485 case VSD_DTYPE_INT_S64: 3486 error = Q_QCPYVALI(&x, voival->int64.s64); 3487 break; 3488 case VSD_DTYPE_INT_U64: 3489 error = Q_QCPYVALI(&x, voival->int64.u64); 3490 break; 3491 case VSD_DTYPE_INT_SLONG: 3492 error = Q_QCPYVALI(&x, voival->intlong.slong); 3493 break; 3494 case VSD_DTYPE_INT_ULONG: 3495 error = Q_QCPYVALI(&x, voival->intlong.ulong); 3496 break; 3497 case VSD_DTYPE_Q_S32: 3498 error = Q_QCPYVALQ(&x, voival->q32.sq32); 3499 break; 3500 case VSD_DTYPE_Q_U32: 3501 error = Q_QCPYVALQ(&x, voival->q32.uq32); 3502 break; 3503 case VSD_DTYPE_Q_S64: 3504 error = Q_QCPYVALQ(&x, voival->q64.sq64); 3505 break; 3506 case VSD_DTYPE_Q_U64: 3507 error = Q_QCPYVALQ(&x, voival->q64.uq64); 3508 break; 3509 default: 3510 error = EINVAL; 3511 break; 3512 } 3513 3514 if (error || 3515 (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1))) 3516 return (error); 3517 3518 vs->flags |= VS_VSDVALID; 3519 return (0); 3520 } 3521 3522 int 3523 stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id, 3524 enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags) 3525 { 3526 struct voi *v; 3527 struct voistat *vs; 3528 void *statevsd, *vsd; 3529 int error, i, tmperr; 3530 3531 error = 0; 3532 3533 if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) || 3534 voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL) 3535 return (EINVAL); 3536 v = &sb->vois[voi_id]; 3537 if (voi_dtype != v->dtype || v->id < 0 || 3538 ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE))) 3539 return (EINVAL); 3540 3541 vs = BLOB_OFFSET(sb, v->stats_off); 3542 if (v->flags & VOI_REQSTATE) 3543 statevsd = BLOB_OFFSET(sb, vs->data_off); 3544 else 3545 statevsd = NULL; 3546 3547 if (flags & SB_VOI_RELUPDATE) { 3548 switch (voi_dtype) { 3549 case VSD_DTYPE_INT_S32: 3550 voival->int32.s32 += 3551 VSD(voistate, statevsd)->prev.int32.s32; 3552 break; 3553 case VSD_DTYPE_INT_U32: 3554 voival->int32.u32 += 3555 VSD(voistate, statevsd)->prev.int32.u32; 3556 break; 3557 case VSD_DTYPE_INT_S64: 3558 voival->int64.s64 += 3559 VSD(voistate, statevsd)->prev.int64.s64; 3560 break; 3561 case VSD_DTYPE_INT_U64: 3562 voival->int64.u64 += 3563 VSD(voistate, statevsd)->prev.int64.u64; 3564 break; 3565 case VSD_DTYPE_INT_SLONG: 3566 voival->intlong.slong += 3567 VSD(voistate, statevsd)->prev.intlong.slong; 3568 break; 3569 case VSD_DTYPE_INT_ULONG: 3570 voival->intlong.ulong += 3571 VSD(voistate, statevsd)->prev.intlong.ulong; 3572 break; 3573 case VSD_DTYPE_Q_S32: 3574 error = Q_QADDQ(&voival->q32.sq32, 3575 VSD(voistate, statevsd)->prev.q32.sq32); 3576 break; 3577 case VSD_DTYPE_Q_U32: 3578 error = Q_QADDQ(&voival->q32.uq32, 3579 VSD(voistate, statevsd)->prev.q32.uq32); 3580 break; 3581 case VSD_DTYPE_Q_S64: 3582 error = Q_QADDQ(&voival->q64.sq64, 3583 VSD(voistate, statevsd)->prev.q64.sq64); 3584 break; 3585 case VSD_DTYPE_Q_U64: 3586 error = Q_QADDQ(&voival->q64.uq64, 3587 VSD(voistate, statevsd)->prev.q64.uq64); 3588 break; 3589 default: 3590 KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 3591 break; 3592 } 3593 } 3594 3595 if (error) 3596 return (error); 3597 3598 for (i = v->voistatmaxid; i > 0; i--) { 3599 vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i]; 3600 if (vs->stype < 0) 3601 continue; 3602 3603 vsd = BLOB_OFFSET(sb, vs->data_off); 3604 3605 switch (vs->stype) { 3606 case VS_STYPE_MAX: 3607 tmperr = stats_v1_voi_update_max(voi_dtype, voival, 3608 vs, vsd); 3609 break; 3610 case VS_STYPE_MIN: 3611 tmperr = stats_v1_voi_update_min(voi_dtype, voival, 3612 vs, vsd); 3613 break; 3614 case VS_STYPE_SUM: 3615 tmperr = stats_v1_voi_update_sum(voi_dtype, voival, 3616 vs, vsd); 3617 break; 3618 case VS_STYPE_HIST: 3619 tmperr = stats_v1_voi_update_hist(voi_dtype, voival, 3620 vs, vsd); 3621 break; 3622 case VS_STYPE_TDGST: 3623 tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival, 3624 vs, vsd); 3625 break; 3626 default: 3627 KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); 3628 break; 3629 } 3630 3631 if (tmperr) { 3632 error = tmperr; 3633 VS_INCERRS(vs); 3634 } 3635 } 3636 3637 if (statevsd) { 3638 switch (voi_dtype) { 3639 case VSD_DTYPE_INT_S32: 3640 VSD(voistate, statevsd)->prev.int32.s32 = 3641 voival->int32.s32; 3642 break; 3643 case VSD_DTYPE_INT_U32: 3644 VSD(voistate, statevsd)->prev.int32.u32 = 3645 voival->int32.u32; 3646 break; 3647 case VSD_DTYPE_INT_S64: 3648 VSD(voistate, statevsd)->prev.int64.s64 = 3649 voival->int64.s64; 3650 break; 3651 case VSD_DTYPE_INT_U64: 3652 VSD(voistate, statevsd)->prev.int64.u64 = 3653 voival->int64.u64; 3654 break; 3655 case VSD_DTYPE_INT_SLONG: 3656 VSD(voistate, statevsd)->prev.intlong.slong = 3657 voival->intlong.slong; 3658 break; 3659 case VSD_DTYPE_INT_ULONG: 3660 VSD(voistate, statevsd)->prev.intlong.ulong = 3661 voival->intlong.ulong; 3662 break; 3663 case VSD_DTYPE_Q_S32: 3664 error = Q_QCPYVALQ( 3665 &VSD(voistate, statevsd)->prev.q32.sq32, 3666 voival->q32.sq32); 3667 break; 3668 case VSD_DTYPE_Q_U32: 3669 error = Q_QCPYVALQ( 3670 &VSD(voistate, statevsd)->prev.q32.uq32, 3671 voival->q32.uq32); 3672 break; 3673 case VSD_DTYPE_Q_S64: 3674 error = Q_QCPYVALQ( 3675 &VSD(voistate, statevsd)->prev.q64.sq64, 3676 voival->q64.sq64); 3677 break; 3678 case VSD_DTYPE_Q_U64: 3679 error = Q_QCPYVALQ( 3680 &VSD(voistate, statevsd)->prev.q64.uq64, 3681 voival->q64.uq64); 3682 break; 3683 default: 3684 KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); 3685 break; 3686 } 3687 } 3688 3689 return (error); 3690 } 3691 3692 #ifdef _KERNEL 3693 3694 static void 3695 stats_init(void *arg) 3696 { 3697 3698 } 3699 SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL); 3700 3701 /* 3702 * Sysctl handler to display the list of available stats templates. 3703 */ 3704 static int 3705 stats_tpl_list_available(SYSCTL_HANDLER_ARGS) 3706 { 3707 struct sbuf *s; 3708 int err, i; 3709 3710 err = 0; 3711 3712 /* We can tolerate ntpl being stale, so do not take the lock. */ 3713 s = sbuf_new(NULL, NULL, /* +1 per tpl for , */ 3714 ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN); 3715 if (s == NULL) 3716 return (ENOMEM); 3717 3718 TPL_LIST_RLOCK(); 3719 for (i = 0; i < ntpl; i++) { 3720 err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "", 3721 tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash); 3722 if (err) { 3723 /* Sbuf overflow condition. */ 3724 err = EOVERFLOW; 3725 break; 3726 } 3727 } 3728 TPL_LIST_RUNLOCK(); 3729 3730 if (!err) { 3731 sbuf_finish(s); 3732 err = sysctl_handle_string(oidp, sbuf_data(s), 0, req); 3733 } 3734 3735 sbuf_delete(s); 3736 return (err); 3737 } 3738 3739 /* 3740 * Called by subsystem-specific sysctls to report and/or parse the list of 3741 * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t 3742 * conformant function pointer must be passed in as arg1, which is used to 3743 * interact with the subsystem's stats template sample rates list. If arg2 > 0, 3744 * a zero-initialised allocation of arg2-sized contextual memory is 3745 * heap-allocated and passed in to all subsystem callbacks made during the 3746 * operation of stats_tpl_sample_rates(). 3747 * 3748 * XXXLAS: Assumes templates are never removed, which is currently true but may 3749 * need to be reworked in future if dynamic template management becomes a 3750 * requirement e.g. to support kernel module based templates. 3751 */ 3752 int 3753 stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS) 3754 { 3755 char kvpair_fmt[16], tplspec_fmt[16]; 3756 char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN]; 3757 char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */ 3758 stats_tpl_sr_cb_t subsys_cb; 3759 void *subsys_ctx; 3760 char *buf, *new_rates_usr_str, *tpl_name_p; 3761 struct stats_tpl_sample_rate *rates; 3762 struct sbuf *s, _s; 3763 uint32_t cum_pct, pct, tpl_hash; 3764 int err, i, off, len, newlen, nrates; 3765 3766 buf = NULL; 3767 rates = NULL; 3768 err = nrates = 0; 3769 subsys_cb = (stats_tpl_sr_cb_t)arg1; 3770 KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__)); 3771 if (arg2 > 0) 3772 subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO); 3773 else 3774 subsys_ctx = NULL; 3775 3776 /* Grab current count of subsystem rates. */ 3777 err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx); 3778 if (err) 3779 goto done; 3780 3781 /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */ 3782 len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5)); 3783 3784 if (req->oldptr != NULL || req->newptr != NULL) 3785 buf = malloc(len, M_TEMP, M_WAITOK); 3786 3787 if (req->oldptr != NULL) { 3788 if (nrates == 0) { 3789 /* No rates, so return an empty string via oldptr. */ 3790 err = SYSCTL_OUT(req, "", 1); 3791 if (err) 3792 goto done; 3793 goto process_new; 3794 } 3795 3796 s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL); 3797 3798 /* Grab locked count of, and ptr to, subsystem rates. */ 3799 err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates, 3800 subsys_ctx); 3801 if (err) 3802 goto done; 3803 TPL_LIST_RLOCK(); 3804 for (i = 0; i < nrates && !err; i++) { 3805 err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "", 3806 tpllist[rates[i].tpl_slot_id]->mb->tplname, 3807 tpllist[rates[i].tpl_slot_id]->mb->tplhash, 3808 rates[i].tpl_sample_pct); 3809 } 3810 TPL_LIST_RUNLOCK(); 3811 /* Tell subsystem that we're done with its rates list. */ 3812 err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx); 3813 if (err) 3814 goto done; 3815 3816 err = sbuf_finish(s); 3817 if (err) 3818 goto done; /* We lost a race for buf to be too small. */ 3819 3820 /* Return the rendered string data via oldptr. */ 3821 err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s)); 3822 } else { 3823 /* Return the upper bound size for buffer sizing requests. */ 3824 err = SYSCTL_OUT(req, NULL, len); 3825 } 3826 3827 process_new: 3828 if (err || req->newptr == NULL) 3829 goto done; 3830 3831 newlen = req->newlen - req->newidx; 3832 err = SYSCTL_IN(req, buf, newlen); 3833 if (err) 3834 goto done; 3835 3836 /* 3837 * Initialise format strings at run time. 3838 * 3839 * Write the max template spec string length into the 3840 * template_spec=percent key-value pair parsing format string as: 3841 * " %<width>[^=]=%u %n" 3842 * 3843 * Write the max template name string length into the tplname:tplhash 3844 * parsing format string as: 3845 * "%<width>[^:]:%u" 3846 * 3847 * Subtract 1 for \0 appended by sscanf(). 3848 */ 3849 sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1); 3850 sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1); 3851 3852 /* 3853 * Parse each CSV key-value pair specifying a template and its sample 3854 * percentage. Whitespace either side of a key-value pair is ignored. 3855 * Templates can be specified by name, hash, or name and hash per the 3856 * following formats (chars in [] are optional): 3857 * ["]<tplname>["]=<percent> 3858 * :hash=pct 3859 * ["]<tplname>["]:hash=<percent> 3860 */ 3861 cum_pct = nrates = 0; 3862 rates = NULL; 3863 buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */ 3864 new_rates_usr_str = buf; 3865 while (isspace(*new_rates_usr_str)) 3866 new_rates_usr_str++; /* Skip leading whitespace. */ 3867 while (*new_rates_usr_str != '\0') { 3868 tpl_name_p = tpl_name; 3869 tpl_name[0] = '\0'; 3870 tpl_hash = 0; 3871 off = 0; 3872 3873 /* 3874 * Parse key-value pair which must perform 2 conversions, then 3875 * parse the template spec to extract either name, hash, or name 3876 * and hash depending on the three possible spec formats. The 3877 * tplspec_fmt format specifier parses name or name and hash 3878 * template specs, while the ":%u" format specifier parses 3879 * hash-only template specs. If parsing is successfull, ensure 3880 * the cumulative sampling percentage does not exceed 100. 3881 */ 3882 err = EINVAL; 3883 if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct, 3884 &off)) 3885 break; 3886 if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) && 3887 (1 != sscanf(tpl_spec, ":%u", &tpl_hash))) 3888 break; 3889 if ((cum_pct += pct) > 100) 3890 break; 3891 err = 0; 3892 3893 /* Strip surrounding "" from template name if present. */ 3894 len = strlen(tpl_name); 3895 if (len > 0) { 3896 if (tpl_name[len - 1] == '"') 3897 tpl_name[--len] = '\0'; 3898 if (tpl_name[0] == '"') { 3899 tpl_name_p++; 3900 len--; 3901 } 3902 } 3903 3904 rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */ 3905 (nrates + 1) * sizeof(*rates), M_WAITOK); 3906 rates[nrates].tpl_slot_id = 3907 stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash); 3908 if (rates[nrates].tpl_slot_id < 0) { 3909 err = -rates[nrates].tpl_slot_id; 3910 break; 3911 } 3912 rates[nrates].tpl_sample_pct = pct; 3913 nrates++; 3914 new_rates_usr_str += off; 3915 if (*new_rates_usr_str != ',') 3916 break; /* End-of-input or malformed. */ 3917 new_rates_usr_str++; /* Move past comma to next pair. */ 3918 } 3919 3920 if (!err) { 3921 if ((new_rates_usr_str - buf) < newlen) { 3922 /* Entire input has not been consumed. */ 3923 err = EINVAL; 3924 } else { 3925 /* 3926 * Give subsystem the new rates. They'll return the 3927 * appropriate rates pointer for us to garbage collect. 3928 */ 3929 err = subsys_cb(TPL_SR_PUT, &rates, &nrates, 3930 subsys_ctx); 3931 } 3932 } 3933 stats_free(rates); 3934 3935 done: 3936 free(buf, M_TEMP); 3937 free(subsys_ctx, M_TEMP); 3938 return (err); 3939 } 3940 3941 SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 3942 "stats(9) MIB"); 3943 3944 SYSCTL_PROC(_kern_stats, OID_AUTO, templates, 3945 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 3946 stats_tpl_list_available, "A", 3947 "list the name/hash of all available stats(9) templates"); 3948 3949 #else /* ! _KERNEL */ 3950 3951 static void __attribute__ ((constructor)) 3952 stats_constructor(void) 3953 { 3954 3955 pthread_rwlock_init(&tpllistlock, NULL); 3956 } 3957 3958 static void __attribute__ ((destructor)) 3959 stats_destructor(void) 3960 { 3961 3962 pthread_rwlock_destroy(&tpllistlock); 3963 } 3964 3965 #endif /* _KERNEL */ 3966