Lines Matching +full:tpl +full:- +full:support

1 /*-
2 * Copyright (c) 2014-2018 Netflix, Inc.
74 uint16_t errs : VS_EBITS;/* Non-wrapping error count. */
75 uint16_t flags : 16 - VS_EBITS;
79 if ((vs)->errs < (1U << VS_EBITS) - 1) \
80 (vs)->errs++; \
85 * - Global or entity specific (global would imply use of counter(9)?)
86 * - Whether to reset stats on read or not
87 * - Signal an overflow?
88 * - Compressed voistat array
104 * - Compressed voi array (trade off memory usage vs search time)
105 * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib)
141 struct statsblob_tpl *tpl; member
207 #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \
227 [VS_STYPE_TDGST] = "t-digest of observed VOI values",
345 KASSERT(Q_PRECEQ(c1->mu, c2->mu), in ctd32cmp()
346 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, in ctd32cmp()
347 Q_RELPREC(c1->mu, c2->mu))); in ctd32cmp()
349 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); in ctd32cmp()
357 KASSERT(Q_PRECEQ(c1->mu, c2->mu), in ctd64cmp()
358 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, in ctd64cmp()
359 Q_RELPREC(c1->mu, c2->mu))); in ctd64cmp()
361 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); in ctd64cmp()
402 memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz); in stats_realloc()
433 stats_tpl_update_hash(struct statsblob_tpl *tpl) argument
437 tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0);
438 for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) {
439 if (tpl->mb->voi_meta[voi_id].name != NULL)
440 tpl->mb->tplhash = hash32_str(
441 tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash);
443 tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz,
444 tpl->mb->tplhash);
469 switch (info->scheme) {
471 step = info->lin.stepinc;
474 step = stats_pow_u64(info->exp.stepbase,
475 info->exp.stepexp + curbkt);
481 switch (info->voi_dtype) {
483 while ((int32_t)stats_pow_u64(info->linexp.stepbase,
484 curstepexp) <= bkt_lb->int32.s32)
488 while ((uint32_t)stats_pow_u64(info->linexp.stepbase,
489 curstepexp) <= bkt_lb->int32.u32)
493 while ((int64_t)stats_pow_u64(info->linexp.stepbase,
494 curstepexp) <= bkt_lb->int64.s64)
498 while ((uint64_t)stats_pow_u64(info->linexp.stepbase,
499 curstepexp) <= bkt_lb->int64.u64)
503 while ((long)stats_pow_u64(info->linexp.stepbase,
504 curstepexp) <= bkt_lb->intlong.slong)
508 while ((unsigned long)stats_pow_u64(info->linexp.stepbase,
509 curstepexp) <= bkt_lb->intlong.ulong)
513 while ((s32q_t)stats_pow_u64(info->linexp.stepbase,
514 curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32))
517 while ((u32q_t)stats_pow_u64(info->linexp.stepbase,
518 curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32))
521 while ((s64q_t)stats_pow_u64(info->linexp.stepbase,
522 curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64))
526 while ((u64q_t)stats_pow_u64(info->linexp.stepbase,
527 curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64))
534 step = stats_pow_u64(info->linexp.stepbase, curstepexp) /
535 info->linexp.linstepdiv;
544 if (info->scheme == BKT_USR) {
545 *bkt_lb = info->usr.bkts[curbkt].lb;
546 *bkt_ub = info->usr.bkts[curbkt].ub;
548 switch (info->voi_dtype) {
550 bkt_ub->int32.s32 += (int32_t)step;
553 bkt_ub->int32.u32 += (uint32_t)step;
556 bkt_ub->int64.s64 += (int64_t)step;
559 bkt_ub->int64.u64 += (uint64_t)step;
562 bkt_ub->intlong.slong += (long)step;
565 bkt_ub->intlong.ulong += (unsigned long)step;
568 error = Q_QADDI(&bkt_ub->q32.sq32, step);
571 error = Q_QADDI(&bkt_ub->q32.uq32, step);
574 error = Q_QADDI(&bkt_ub->q64.sq64, step);
577 error = Q_QADDI(&bkt_ub->q64.uq64, step);
582 } else { /* info->scheme != BKT_USR && step == 0 */
596 if (info->scheme == BKT_USR) {
597 /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */
598 info->lb = info->usr.bkts[0].lb;
599 info->ub = info->usr.bkts[info->usr.nbkts - 1].lb;
604 bkt_ub = info->lb;
611 if (info->scheme == BKT_USR)
612 done = (nbkts == info->usr.nbkts);
614 switch (info->voi_dtype) {
616 done = (bkt_ub.int32.s32 > info->ub.int32.s32);
619 done = (bkt_ub.int32.u32 > info->ub.int32.u32);
622 done = (bkt_ub.int64.s64 > info->ub.int64.s64);
625 done = (bkt_ub.int64.u64 > info->ub.int64.u64);
629 info->ub.intlong.slong);
633 info->ub.intlong.ulong);
637 info->ub.q32.sq32);
641 info->ub.q32.uq32);
645 info->ub.q64.sq64);
649 info->ub.q64.uq64);
657 if (info->flags & VSD_HIST_LBOUND_INF)
659 if (info->flags & VSD_HIST_UBOUND_INF)
674 if (vss == NULL || info == NULL || (info->flags &
675 (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype ==
676 VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64)))
679 info->voi_dtype = voi_dtype;
684 switch (info->hist_dtype) {
686 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts);
689 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts);
692 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts);
695 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts);
698 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts);
701 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts);
707 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
708 if (vss->iv == NULL)
711 hist = (struct voistatdata_hist *)vss->iv;
712 bkt_ub = info->lb;
714 for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0;
721 switch (info->hist_dtype) {
723 VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb;
726 VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb;
727 VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub;
730 VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb;
733 VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb;
736 VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb;
737 VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub;
740 VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb;
749 switch (info->hist_dtype) {
751 lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb;
752 ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb;
755 lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb;
756 lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub;
757 ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb;
758 ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub;
761 lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb;
762 ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb;
765 lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb;
766 lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub;
767 ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb;
768 ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub;
777 if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) {
778 *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype];
784 * info->lb.
786 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
787 info->voi_dtype == VSD_DTYPE_Q_U32) {
789 Q_SCVAL(lbinfbktlb->q32.sq32,
790 Q_GCVAL(info->lb.q32.sq32));
791 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
792 info->voi_dtype == VSD_DTYPE_Q_U64) {
794 Q_SCVAL(lbinfbktlb->q64.sq64,
795 Q_GCVAL(info->lb.q64.sq64));
798 *lbinfbktub = info->lb;
800 if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) {
803 *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype];
804 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
805 info->voi_dtype == VSD_DTYPE_Q_U32) {
806 Q_SCVAL(ubinfbktub->q32.sq32,
807 Q_GCVAL(info->lb.q32.sq32));
808 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
809 info->voi_dtype == VSD_DTYPE_Q_U64) {
810 Q_SCVAL(ubinfbktub->q64.sq64,
811 Q_GCVAL(info->lb.q64.sq64));
829 info->voi_dtype = voi_dtype;
831 switch (info->tdgst_dtype) {
833 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds);
836 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds);
842 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
843 if (vss->iv == NULL)
846 tdgst = (struct voistatdata_tdgst *)vss->iv;
848 switch (info->tdgst_dtype) {
850 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
851 ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) {
852 Q_INI(&ctd32->mu, 0, 0, info->prec);
856 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
857 ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) {
858 Q_INI(&ctd64->mu, 0, 0, info->prec);
874 switch (vss->stype) {
888 vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0);
889 if (vss->iv == NULL)
892 vss->vs_dtype = voi_dtype;
893 vss->vsdsz = vsd_dtype2size[voi_dtype];
896 *((int32_t *)vss->iv) = iv.int32.s32;
899 *((uint32_t *)vss->iv) = iv.int32.u32;
902 *((int64_t *)vss->iv) = iv.int64.s64;
905 *((uint64_t *)vss->iv) = iv.int64.u64;
908 *((long *)vss->iv) = iv.intlong.slong;
911 *((unsigned long *)vss->iv) = iv.intlong.ulong;
914 *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32,
915 Q_CTRLINI(info->prec));
918 *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32,
919 Q_CTRLINI(info->prec));
922 *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64,
923 Q_CTRLINI(info->prec));
926 *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64,
927 Q_CTRLINI(info->prec));
942 for (i = nvss - 1; i >= 0; i--) {
956 for (i = nvss - 1; i >= 0; i--) {
965 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl) argument
975 *tpl = tpllist[tpl_id];
976 /* XXXLAS: Acquire refcount on tpl. */
988 tpl_id = -ESRCH;
991 for (i = ntpl - 1; i >= 0; i--) {
993 if (strlen(name) == strlen(tpllist[i]->mb->tplname) &&
994 strncmp(name, tpllist[i]->mb->tplname,
996 tpllist[i]->mb->tplhash)) {
1000 } else if (hash == tpllist[i]->mb->tplhash) {
1019 if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname))
1020 strlcpy(buf, tpllist[tpl_id]->mb->tplname, len);
1061 * template will be assigned, and -1 returned instead.
1074 return (-1);
1086 src->cursz < sizeof(struct statsblob) ||
1091 *dst = stats_realloc(NULL, 0, src->cursz, 0);
1093 (*dst)->maxsz = dstmaxsz = src->cursz;
1124 if (dstmaxsz >= src->cursz) {
1125 postcurszlen = src->cursz -
1129 postcurszlen = sizeof(struct statsblob) -
1134 tmperror = copyout_nofault(&(src->cursz), &((*dst)->cursz),
1137 tmperror = copyout(&(src->cursz), &((*dst)->cursz),
1141 memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen);
1155 struct statsblobv1_tpl *tpl, **newtpllist; local
1161 return (-EINVAL);
1164 return (-EEXIST);
1166 tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO);
1171 tpl_mb->tplname = stats_strdup(name, 0);
1173 if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL ||
1174 tpl_mb->tplname == NULL) {
1175 stats_free(tpl);
1178 stats_free(tpl_mb->tplname);
1181 return (-ENOMEM);
1184 tpl->mb = tpl_mb;
1185 tpl->sb = tpl_sb;
1187 tpl_sb->abi = STATS_ABI_V1;
1188 tpl_sb->endian =
1196 tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1);
1197 tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1);
1205 tpllist[tpl_id] = (struct statsblob_tpl *)tpl;
1208 stats_free(tpl);
1211 stats_free(tpl_mb->tplname);
1214 tpl_id = -ENOMEM;
1240 newvoistatmaxid = -1;
1243 for (i = nvss - 1; i >= 0; i--) {
1261 tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb;
1262 tpl_mb = tpllist[tpl_id]->mb;
1264 if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) {
1267 /* We need to grow the tpl_sb->vois array. */
1268 newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) *
1277 tpl_sb->vois[voi_id].voistatmaxid) {
1278 newvoistatbytes = (newvoistatmaxid -
1279 tpl_sb->vois[voi_id].voistatmaxid) *
1282 /* XXXLAS: KPI does not yet support expanding VOIs. */
1288 struct voi_meta *voi_meta = tpl_mb->voi_meta;
1298 tpl_mb->voi_meta = voi_meta;
1308 tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name,
1310 if (tpl_mb->voi_meta[voi_id].name == NULL)
1316 tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb;
1319 voi = &tpl_sb->vois[voi_id];
1321 if (voi->id < 0) {
1323 voi->id = voi_id;
1324 voi->dtype = voi_dtype;
1325 voi->stats_off = tpl_sb->stats_off;
1327 voi->flags |= VOI_REQSTATE;
1331 * "KPI does not yet support expanding VOIs"
1343 voi->voistatmaxid = newvoistatmaxid;
1346 if (voi->flags & VOI_REQSTATE) {
1348 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off);
1349 tmpstat->stype = VS_STYPE_VOISTATE;
1350 tmpstat->flags = 0;
1351 tmpstat->dtype = VSD_DTYPE_VOISTATE;
1352 newstatdataidx = tmpstat->dsz =
1354 tmpstat->data_off = tpl_sb->statsdata_off;
1358 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off +
1360 KASSERT(tmpstat->stype < 0, ("voistat %p "
1362 tmpstat->stype = vss[i].stype;
1363 tmpstat->flags = vss[i].flags;
1364 tmpstat->dtype = vss[i].vs_dtype;
1365 tmpstat->dsz = vss[i].vsdsz;
1366 tmpstat->data_off = tpl_sb->statsdata_off +
1368 memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off),
1375 /* XXXLAS: Confirm tpl name/hash pair remains unique. */
1394 sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0);
1396 sb->maxsz = tpllist[tpl_id]->sb->maxsz;
1426 if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 ||
1430 v = &sb->vois[voi_id];
1431 if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid)
1434 vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat)));
1435 *retvsd = BLOB_OFFSET(sb, vs->data_off);
1437 *retdtype = vs->dtype;
1439 *retvsdsz = vs->dsz;
1469 error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW;
1471 ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz));
1474 memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz);
1475 sb->created = sb->lastrst = stats_sbinuptime();
1476 sb->tplhash = tpllist[tpl_id]->mb->tplhash;
1502 * XXXLAS: Required until we gain support for flags which alter the
1505 if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ))
1508 if (!error && (sb->cursz + nbytes > sb->maxsz)) {
1510 sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO);
1512 sb->maxsz = sb->cursz + nbytes;
1525 memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes),
1526 BLOB_OFFSET(sb, sb->statsdata_off),
1527 sb->cursz - sb->statsdata_off);
1528 memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes +
1529 newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off),
1530 sb->statsdata_off - sb->stats_off);
1534 idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1;
1537 sb->cursz += nbytes;
1538 sb->stats_off += newvoibytes;
1539 sb->statsdata_off += newvoibytes + newvoistatbytes;
1542 memset(&sb->vois[idxnewvois], '\0', newvoibytes);
1543 memset(BLOB_OFFSET(sb, sb->stats_off), '\0',
1545 memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0',
1550 tmpvoi = &sb->vois[i];
1552 tmpvoi->id = tmpvoi->voistatmaxid = -1;
1553 } else if (tmpvoi->id > -1) {
1554 tmpvoi->stats_off += newvoibytes +
1560 nvoistats = (sb->statsdata_off - sb->stats_off) /
1562 voistat_array = BLOB_OFFSET(sb, sb->stats_off);
1566 tmpvoistat->stype = -1;
1567 } else if (tmpvoistat->stype > -1) {
1568 tmpvoistat->data_off += nbytes;
1597 v = &sb->vois[i];
1599 ctx.vsslot = -1;
1604 else if (i == (NVOIS(sb) - 1))
1607 if (v->id < 0 && (flags & SB_IT_NULLVOI)) {
1614 /* If NULL voi, v->voistatmaxid == -1 */
1615 for (j = 0; j <= v->voistatmaxid; j++) {
1617 v->stats_off))[j];
1618 if (vs->stype < 0 &&
1622 if (j == v->voistatmaxid) {
1624 if (i == (NVOIS(sb) - 1))
1657 smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt;
1658 compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt;
1659 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1660 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1661 ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree;
1664 qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10);
1670 smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt;
1671 compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt;
1672 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1673 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1674 ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree;
1677 qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10);
1739 Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr,
1753 is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt);
1842 if (i < nbkts - 1)
1930 if (fmt == SB_STRFMT_JSON && i < nbkts - 1)
1964 (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev,
1968 sbuf_printf(buf, "%d", vsd->int32.s32);
1971 sbuf_printf(buf, "%u", vsd->int32.u32);
1974 sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64);
1977 sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64);
1980 sbuf_printf(buf, "%ld", vsd->intlong.slong);
1983 sbuf_printf(buf, "%lu", vsd->intlong.ulong);
1987 char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)];
1988 Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr));
1994 char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)];
1995 Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr));
2001 char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)];
2002 Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr));
2008 char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)];
2009 Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr));
2045 sctx = ctx->usrctx;
2046 buf = sctx->buf;
2047 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2048 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2050 if (ctx->flags & SB_IT_FIRST_CB) {
2056 sb->abi, sb->endian, sb->maxsz, sb->cursz,
2057 sb->created, sb->lastrst, sb->flags, sb->stats_off,
2058 sb->statsdata_off);
2060 sbuf_printf(buf, ", tplhash=%u", sb->tplhash);
2063 if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2064 sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id);
2065 if (v->id < 0)
2068 tpl_mb->voi_meta[v->id].name);
2071 "voistatmaxid=%hhd, stats_off=%hu", v->flags,
2072 vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off);
2075 if (!dump && vs->stype <= 0)
2078 sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot);
2079 if (vs->stype < 0) {
2080 sbuf_printf(buf, "%hhd", vs->stype);
2083 sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype],
2084 vs->errs);
2085 vsd = BLOB_OFFSET(sb, vs->data_off);
2088 "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype],
2089 vs->dsz, vs->data_off);
2092 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2093 sctx->fmt, buf, dump);
2107 sctx = ctx->usrctx;
2108 buf = sctx->buf;
2109 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2110 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2112 if (ctx->flags & SB_IT_FIRST_CB) {
2118 "\"statsdata_off\":%hu,", sb->abi,
2119 sb->endian, sb->maxsz, sb->cursz, sb->created,
2120 sb->lastrst, sb->flags, sb->stats_off,
2121 sb->statsdata_off);
2129 sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null",
2130 sb->tplhash);
2133 if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2135 sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot,
2136 v->id);
2137 if (v->id < 0) {
2152 tpl_mb->voi_meta[v->id].name : "null", v->flags,
2153 vsd_dtype2name[v->dtype], v->voistatmaxid,
2154 v->stats_off);
2157 sbuf_printf(buf, "\"[%hd]\":{", v->id);
2160 tpl_mb->voi_meta[v->id].name);
2166 vsd = BLOB_OFFSET(sb, vs->data_off);
2168 sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot);
2169 if (vs->stype < 0) {
2170 sbuf_cat(buf, "{\"stype\":-1},");
2175 vs_stype2name[vs->stype], vs->errs, vs->flags,
2176 vsd_dtype2name[vs->dtype], vs->data_off);
2177 } else if (vs->stype > 0) {
2179 sbuf_printf(buf, "\"[%hhd]\":", vs->stype);
2181 sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]);
2185 if ((vs->flags & VS_VSDVALID) || dump) {
2187 sbuf_printf(buf, "{\"errs\":%hu,", vs->errs);
2188 /* Simple non-compound VSD types need a key. */
2189 if (!vsd_compoundtype[vs->dtype])
2191 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2192 sctx->fmt, buf, dump);
2197 if (ctx->flags & SB_IT_LAST_VOISTAT)
2200 if (ctx->flags & SB_IT_LAST_CB)
2212 sctx = ctx->usrctx;
2214 switch (sctx->fmt) {
2225 return (sbuf_error(sctx->buf));
2235 if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL ||
2244 if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash),
2245 &sctx.tpl))
2248 sctx.tpl = NULL;
2265 vctx = ctx->usrctx;
2267 sbv.tplhash = sb->tplhash;
2268 sbv.voi_id = v->id;
2269 sbv.voi_dtype = v->dtype;
2270 sbv.vs_stype = vs->stype;
2271 sbv.vs_dtype = vs->dtype;
2272 sbv.vs_dsz = vs->dsz;
2273 sbv.vs_data = BLOB_OFFSET(sb, vs->data_off);
2274 sbv.vs_errs = vs->errs;
2275 sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB |
2279 return (vctx->cb(&sbv, vctx->usrctx));
2288 if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL)
2305 if (vs->stype == VS_STYPE_VOISTATE)
2308 vsd = BLOB_OFFSET(sb, vs->data_off);
2311 switch (vs->stype) {
2313 switch (vs->dtype) {
2315 Q_SIFVAL(VSD(q32, vsd)->sq32, 0);
2318 Q_SIFVAL(VSD(q32, vsd)->uq32, 0);
2321 Q_SIFVAL(VSD(q64, vsd)->sq64, 0);
2324 Q_SIFVAL(VSD(q64, vsd)->uq64, 0);
2327 bzero(vsd, vs->dsz);
2332 switch (vs->dtype) {
2334 Q_SIFVAL(VSD(q32, vsd)->sq32,
2335 Q_IFMINVAL(VSD(q32, vsd)->sq32));
2338 Q_SIFVAL(VSD(q32, vsd)->uq32,
2339 Q_IFMINVAL(VSD(q32, vsd)->uq32));
2342 Q_SIFVAL(VSD(q64, vsd)->sq64,
2343 Q_IFMINVAL(VSD(q64, vsd)->sq64));
2346 Q_SIFVAL(VSD(q64, vsd)->uq64,
2347 Q_IFMINVAL(VSD(q64, vsd)->uq64));
2350 memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype],
2351 vs->dsz);
2356 switch (vs->dtype) {
2358 Q_SIFVAL(VSD(q32, vsd)->sq32,
2359 Q_IFMAXVAL(VSD(q32, vsd)->sq32));
2362 Q_SIFVAL(VSD(q32, vsd)->uq32,
2363 Q_IFMAXVAL(VSD(q32, vsd)->uq32));
2366 Q_SIFVAL(VSD(q64, vsd)->sq64,
2367 Q_IFMAXVAL(VSD(q64, vsd)->sq64));
2370 Q_SIFVAL(VSD(q64, vsd)->uq64,
2371 Q_IFMAXVAL(VSD(q64, vsd)->uq64));
2374 memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype],
2375 vs->dsz);
2387 switch (vs->dtype) {
2389 nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2393 nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2397 nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2401 nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2405 nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2409 nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2416 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob),
2418 for (i = nbkts - 1; i >= 0; i--) {
2419 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype,
2435 switch (vs->dtype) {
2437 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2438 VSD(tdgstclust32, tdgst)->compcnt = 0;
2439 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2442 ctd32->cnt = 0;
2443 Q_SIFVAL(ctd32->mu, 0);
2446 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2450 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2451 VSD(tdgstclust64, tdgst)->compcnt = 0;
2452 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2455 ctd64->cnt = 0;
2456 Q_SIFVAL(ctd64->mu, 0);
2459 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2468 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
2472 vs->errs = 0;
2473 vs->flags &= ~VS_VSDVALID;
2484 if (src != NULL && src->abi == STATS_ABI_V1) {
2490 src->lastrst = stats_sbinuptime();
2506 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2507 ("Unknown VSD dtype %d", vs->dtype));
2511 switch (vs->dtype) {
2513 if (VSD(int32, vsd)->s32 < voival->int32.s32) {
2514 VSD(int32, vsd)->s32 = voival->int32.s32;
2515 vs->flags |= VS_VSDVALID;
2519 if (VSD(int32, vsd)->u32 < voival->int32.u32) {
2520 VSD(int32, vsd)->u32 = voival->int32.u32;
2521 vs->flags |= VS_VSDVALID;
2525 if (VSD(int64, vsd)->s64 < voival->int64.s64) {
2526 VSD(int64, vsd)->s64 = voival->int64.s64;
2527 vs->flags |= VS_VSDVALID;
2531 if (VSD(int64, vsd)->u64 < voival->int64.u64) {
2532 VSD(int64, vsd)->u64 = voival->int64.u64;
2533 vs->flags |= VS_VSDVALID;
2537 if (VSD(intlong, vsd)->slong < voival->intlong.slong) {
2538 VSD(intlong, vsd)->slong = voival->intlong.slong;
2539 vs->flags |= VS_VSDVALID;
2543 if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) {
2544 VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2545 vs->flags |= VS_VSDVALID;
2549 if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2550 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2551 voival->q32.sq32)))) {
2552 vs->flags |= VS_VSDVALID;
2556 if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2557 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2558 voival->q32.uq32)))) {
2559 vs->flags |= VS_VSDVALID;
2563 if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2564 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2565 voival->q64.sq64)))) {
2566 vs->flags |= VS_VSDVALID;
2570 if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2571 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2572 voival->q64.uq64)))) {
2573 vs->flags |= VS_VSDVALID;
2590 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2591 ("Unknown VSD dtype %d", vs->dtype));
2595 switch (vs->dtype) {
2597 if (VSD(int32, vsd)->s32 > voival->int32.s32) {
2598 VSD(int32, vsd)->s32 = voival->int32.s32;
2599 vs->flags |= VS_VSDVALID;
2603 if (VSD(int32, vsd)->u32 > voival->int32.u32) {
2604 VSD(int32, vsd)->u32 = voival->int32.u32;
2605 vs->flags |= VS_VSDVALID;
2609 if (VSD(int64, vsd)->s64 > voival->int64.s64) {
2610 VSD(int64, vsd)->s64 = voival->int64.s64;
2611 vs->flags |= VS_VSDVALID;
2615 if (VSD(int64, vsd)->u64 > voival->int64.u64) {
2616 VSD(int64, vsd)->u64 = voival->int64.u64;
2617 vs->flags |= VS_VSDVALID;
2621 if (VSD(intlong, vsd)->slong > voival->intlong.slong) {
2622 VSD(intlong, vsd)->slong = voival->intlong.slong;
2623 vs->flags |= VS_VSDVALID;
2627 if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) {
2628 VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2629 vs->flags |= VS_VSDVALID;
2633 if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2634 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2635 voival->q32.sq32)))) {
2636 vs->flags |= VS_VSDVALID;
2640 if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2641 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2642 voival->q32.uq32)))) {
2643 vs->flags |= VS_VSDVALID;
2647 if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2648 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2649 voival->q64.sq64)))) {
2650 vs->flags |= VS_VSDVALID;
2654 if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2655 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2656 voival->q64.uq64)))) {
2657 vs->flags |= VS_VSDVALID;
2674 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2675 ("Unknown VSD dtype %d", vs->dtype));
2679 switch (vs->dtype) {
2681 VSD(int32, vsd)->s32 += voival->int32.s32;
2684 VSD(int32, vsd)->u32 += voival->int32.u32;
2687 VSD(int64, vsd)->s64 += voival->int64.s64;
2690 VSD(int64, vsd)->u64 += voival->int64.u64;
2693 VSD(intlong, vsd)->slong += voival->intlong.slong;
2696 VSD(intlong, vsd)->ulong += voival->intlong.ulong;
2699 error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32);
2702 error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32);
2705 error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64);
2708 error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64);
2716 vs->flags |= VS_VSDVALID;
2732 switch (vs->dtype) {
2734 i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2737 oob32 = &VSD(crhist32, hist)->oob;
2740 i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2743 oob32 = &VSD(drhist32, hist)->oob;
2746 i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2749 oob32 = &VSD(dvhist32, hist)->oob;
2752 i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2754 oob64 = &VSD(crhist64, hist)->oob;
2757 i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2760 oob64 = &VSD(drhist64, hist)->oob;
2763 i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2766 oob64 = &VSD(dvhist64, hist)->oob;
2771 i--; /* Adjust for 0-based array index. */
2774 for (found = 0; i >= 0 && !found; i--) {
2775 switch (vs->dtype) {
2777 bkt_lb = &VSD(crhist32, hist)->bkts[i].lb;
2778 cnt32 = &VSD(crhist32, hist)->bkts[i].cnt;
2781 bkt_lb = &VSD(drhist32, hist)->bkts[i].lb;
2782 bkt_ub = &VSD(drhist32, hist)->bkts[i].ub;
2783 cnt32 = &VSD(drhist32, hist)->bkts[i].cnt;
2786 bkt_lb = &VSD(dvhist32, hist)->bkts[i].val;
2787 cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt;
2790 bkt_lb = &VSD(crhist64, hist)->bkts[i].lb;
2791 cnt64 = &VSD(crhist64, hist)->bkts[i].cnt;
2794 bkt_lb = &VSD(drhist64, hist)->bkts[i].lb;
2795 bkt_ub = &VSD(drhist64, hist)->bkts[i].ub;
2796 cnt64 = &VSD(drhist64, hist)->bkts[i].cnt;
2799 bkt_lb = &VSD(dvhist64, hist)->bkts[i].val;
2800 cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt;
2808 if (voival->int32.s32 >= bkt_lb->int32.s32) {
2809 if ((eq_only && voival->int32.s32 ==
2810 bkt_lb->int32.s32) ||
2812 voival->int32.s32 < bkt_ub->int32.s32)))
2817 if (voival->int32.u32 >= bkt_lb->int32.u32) {
2818 if ((eq_only && voival->int32.u32 ==
2819 bkt_lb->int32.u32) ||
2821 voival->int32.u32 < bkt_ub->int32.u32)))
2826 if (voival->int64.s64 >= bkt_lb->int64.s64)
2827 if ((eq_only && voival->int64.s64 ==
2828 bkt_lb->int64.s64) ||
2830 voival->int64.s64 < bkt_ub->int64.s64)))
2834 if (voival->int64.u64 >= bkt_lb->int64.u64)
2835 if ((eq_only && voival->int64.u64 ==
2836 bkt_lb->int64.u64) ||
2838 voival->int64.u64 < bkt_ub->int64.u64)))
2842 if (voival->intlong.slong >= bkt_lb->intlong.slong)
2843 if ((eq_only && voival->intlong.slong ==
2844 bkt_lb->intlong.slong) ||
2846 voival->intlong.slong <
2847 bkt_ub->intlong.slong)))
2851 if (voival->intlong.ulong >= bkt_lb->intlong.ulong)
2852 if ((eq_only && voival->intlong.ulong ==
2853 bkt_lb->intlong.ulong) ||
2855 voival->intlong.ulong <
2856 bkt_ub->intlong.ulong)))
2860 if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32))
2861 if ((eq_only && Q_QEQ(voival->q32.sq32,
2862 bkt_lb->q32.sq32)) ||
2864 Q_QLTQ(voival->q32.sq32,
2865 bkt_ub->q32.sq32))))
2869 if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32))
2870 if ((eq_only && Q_QEQ(voival->q32.uq32,
2871 bkt_lb->q32.uq32)) ||
2873 Q_QLTQ(voival->q32.uq32,
2874 bkt_ub->q32.uq32))))
2878 if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64))
2879 if ((eq_only && Q_QEQ(voival->q64.sq64,
2880 bkt_lb->q64.sq64)) ||
2882 Q_QLTQ(voival->q64.sq64,
2883 bkt_ub->q64.sq64))))
2887 if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64))
2888 if ((eq_only && Q_QEQ(voival->q64.uq64,
2889 bkt_lb->q64.uq64)) ||
2891 Q_QLTQ(voival->q64.uq64,
2892 bkt_ub->q64.uq64))))
2912 vs->flags |= VS_VSDVALID;
2932 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2935 VSD(tdgstclust32, tdgst)->compcnt++;
2938 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2943 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2947 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2950 VSD(tdgstclust64, tdgst)->compcnt++;
2953 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2958 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2966 * Rebuild the t-digest ARB by pseudorandomly selecting centroids and
2967 * re-inserting the mu/cnt of each as a value and corresponding weight.
2984 idxmask = (UINT64_C(1) << bitsperidx) - 1;
2987 for (; remctds > 0; remctds--) {
2995 nebits -= bitsperidx;
3002 * aspect reduces re-insertion randomness, but is good enough).
3024 * Walk the free list from start to end, re-inserting each centroid's
3026 * we re-insert values from during each loop iteration, so we must latch
3027 * the index of the next free list centroid before the re-insertion
3034 ("%s: t-digest ARB@%p free list bug", __func__,
3044 tmperr = Q_QCLONEQ(&x, ctd32->mu);
3046 vs_dtype, tdgst, x, ctd32->cnt, attempt);
3049 ("%s: t-digest ARB@%p free list bug", __func__,
3054 ctd64->mu, ctd64->cnt, attempt);
3057 ("%s: t-digest ARB@%p free list bug", __func__,
3062 * Just in case, latch any errors and attempt all re-insertions.
3065 remctds--;
3068 KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__,
3095 if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt)
3097 smplcnt = VSD(tdgstclust32, tdgst)->smplcnt;
3098 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
3104 if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt)
3106 smplcnt = VSD(tdgstclust64, tdgst)->smplcnt;
3107 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
3154 cnt = ctd32->cnt;
3155 KASSERT(Q_PRECEQ(ctd32->mu, x),
3157 Q_RELPREC(ctd32->mu, x)));
3159 z = ctd32->mu;
3161 cnt = ctd64->cnt;
3162 KASSERT(Q_PRECEQ(ctd64->mu, x),
3164 Q_RELPREC(ctd64->mu, x)));
3166 z = ctd64->mu;
3197 cnt = ctd32->cnt;
3199 cnt = ctd64->cnt;
3205 /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */
3206 error = Q_QFRACI(&q, (sum << 1) + cnt - 1,
3207 (smplcnt - 1) << 1);
3211 /* k = k x (1 - q) */
3221 (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1);
3222 k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt;
3226 * NB: q is holding the value of 1 - q
3228 q_dbl = 1.0 - q_dbl;
3229 KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) <
3231 ("Q-type q bad precision"));
3232 KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) <
3234 ("Q-type k bad precision"));
3241 if ((is32bit && ((ctd32->cnt + weight) <=
3243 (!is32bit && ((ctd64->cnt + weight) <=
3264 error = Q_QSUBQ(&x, ctd32->mu);
3282 * specify their t-digest with sufficient centroid
3290 * XXXLAS: Per-centroid underflow tracking is likely too
3297 Q_QDIVI(&x, ctd32->cnt + weight);
3299 || (error = Q_QADDQ(&ctd32->mu, x))) {
3306 ctd32->cnt += weight;
3311 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3315 error = Q_QSUBQ(&x, ctd64->mu);
3317 Q_QDIVI(&x, ctd64->cnt + weight);
3320 || (error = Q_QADDQ(&ctd64->mu, x))) {
3325 ctd64->cnt += weight;
3330 &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64);
3342 ("%s: t-digest@%p has no free centroids",
3347 if ((error = Q_QCPYVALQ(&ctd32->mu, x)))
3349 ctd32->cnt = weight;
3354 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3360 ("%s: t-digest@%p has no free centroids",
3366 ctd64->mu = x;
3367 ctd64->cnt = weight;
3372 tdgst)->rbctdtree, ctd64);
3378 VSD(tdgstclust32, tdgst)->smplcnt += weight;
3380 VSD(tdgstclust64, tdgst)->smplcnt += weight;
3384 &VSD(tdgstclust64, tdgst)->rbctdtree;
3404 Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr));
3414 Q_TOSTR(rbctd64->mu, -1, 10, qstr,
3455 switch (vs->dtype) {
3459 ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu));
3464 ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu));
3467 KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 ||
3468 vs->dtype == VSD_DTYPE_TDGSTCLUST64,
3469 ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>",
3470 __func__, vs->dtype));
3480 error = Q_QCPYVALI(&x, voival->int32.s32);
3483 error = Q_QCPYVALI(&x, voival->int32.u32);
3486 error = Q_QCPYVALI(&x, voival->int64.s64);
3489 error = Q_QCPYVALI(&x, voival->int64.u64);
3492 error = Q_QCPYVALI(&x, voival->intlong.slong);
3495 error = Q_QCPYVALI(&x, voival->intlong.ulong);
3498 error = Q_QCPYVALQ(&x, voival->q32.sq32);
3501 error = Q_QCPYVALQ(&x, voival->q32.uq32);
3504 error = Q_QCPYVALQ(&x, voival->q64.sq64);
3507 error = Q_QCPYVALQ(&x, voival->q64.uq64);
3515 (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1)))
3518 vs->flags |= VS_VSDVALID;
3533 if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) ||
3536 v = &sb->vois[voi_id];
3537 if (voi_dtype != v->dtype || v->id < 0 ||
3538 ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE)))
3541 vs = BLOB_OFFSET(sb, v->stats_off);
3542 if (v->flags & VOI_REQSTATE)
3543 statevsd = BLOB_OFFSET(sb, vs->data_off);
3550 voival->int32.s32 +=
3551 VSD(voistate, statevsd)->prev.int32.s32;
3554 voival->int32.u32 +=
3555 VSD(voistate, statevsd)->prev.int32.u32;
3558 voival->int64.s64 +=
3559 VSD(voistate, statevsd)->prev.int64.s64;
3562 voival->int64.u64 +=
3563 VSD(voistate, statevsd)->prev.int64.u64;
3566 voival->intlong.slong +=
3567 VSD(voistate, statevsd)->prev.intlong.slong;
3570 voival->intlong.ulong +=
3571 VSD(voistate, statevsd)->prev.intlong.ulong;
3574 error = Q_QADDQ(&voival->q32.sq32,
3575 VSD(voistate, statevsd)->prev.q32.sq32);
3578 error = Q_QADDQ(&voival->q32.uq32,
3579 VSD(voistate, statevsd)->prev.q32.uq32);
3582 error = Q_QADDQ(&voival->q64.sq64,
3583 VSD(voistate, statevsd)->prev.q64.sq64);
3586 error = Q_QADDQ(&voival->q64.uq64,
3587 VSD(voistate, statevsd)->prev.q64.uq64);
3598 for (i = v->voistatmaxid; i > 0; i--) {
3599 vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i];
3600 if (vs->stype < 0)
3603 vsd = BLOB_OFFSET(sb, vs->data_off);
3605 switch (vs->stype) {
3627 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
3640 VSD(voistate, statevsd)->prev.int32.s32 =
3641 voival->int32.s32;
3644 VSD(voistate, statevsd)->prev.int32.u32 =
3645 voival->int32.u32;
3648 VSD(voistate, statevsd)->prev.int64.s64 =
3649 voival->int64.s64;
3652 VSD(voistate, statevsd)->prev.int64.u64 =
3653 voival->int64.u64;
3656 VSD(voistate, statevsd)->prev.intlong.slong =
3657 voival->intlong.slong;
3660 VSD(voistate, statevsd)->prev.intlong.ulong =
3661 voival->intlong.ulong;
3665 &VSD(voistate, statevsd)->prev.q32.sq32,
3666 voival->q32.sq32);
3670 &VSD(voistate, statevsd)->prev.q32.uq32,
3671 voival->q32.uq32);
3675 &VSD(voistate, statevsd)->prev.q64.sq64,
3676 voival->q64.sq64);
3680 &VSD(voistate, statevsd)->prev.q64.uq64,
3681 voival->q64.uq64);
3713 s = sbuf_new(NULL, NULL, /* +1 per tpl for , */
3721 tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash);
3740 * Called by subsystem-specific sysctls to report and/or parse the list of
3744 * a zero-initialised allocation of arg2-sized contextual memory is
3745 * heap-allocated and passed in to all subsystem callbacks made during the
3750 * requirement e.g. to support kernel module based templates.
3782 len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5));
3784 if (req->oldptr != NULL || req->newptr != NULL)
3787 if (req->oldptr != NULL) {
3806 tpllist[rates[i].tpl_slot_id]->mb->tplname,
3807 tpllist[rates[i].tpl_slot_id]->mb->tplhash,
3828 if (err || req->newptr == NULL)
3831 newlen = req->newlen - req->newidx;
3840 * template_spec=percent key-value pair parsing format string as:
3849 sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1);
3850 sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1);
3853 * Parse each CSV key-value pair specifying a template and its sample
3854 * percentage. Whitespace either side of a key-value pair is ignored.
3874 * Parse key-value pair which must perform 2 conversions, then
3879 * hash-only template specs. If parsing is successfull, ensure
3896 if (tpl_name[len - 1] == '"')
3897 tpl_name[--len] = '\0';
3900 len--;
3909 err = -rates[nrates].tpl_slot_id;
3916 break; /* End-of-input or malformed. */
3921 if ((new_rates_usr_str - buf) < newlen) {