Lines Matching +full:current +full:- +full:limiting
1 /*-
4 * SPDX-License-Identifier: BSD-2-Clause
60 * Default I/O scheduler for FreeBSD. This implementation is just a thin-vineer
81 * where N is the number of samples that 86% of the current
85 * alpha_bits = -log_2(alpha)
86 * alpha = 2^-alpha_bits
103 * published read-only since at present they are compile time constants.
130 #define LAT_BUCKETS 20 /* < 20us < 40us ... < 2^(n-1)*20us >= 2^(n-1)*20us */
248 * Information about the current rate limiters, if any
253 int current; /* Current rate limiter */ member
254 int l_value1; /* per-limiter scratch value 1. */
255 int l_value2; /* per-limiter scratch value 2. */
263 int total; /* Total for all time -- wraps */
264 int in; /* number queued all time -- wraps */
265 int out; /* number completed all time -- wraps */
266 int errs; /* Number of I/Os completed with error -- wraps */
288 set_max = 0, /* current = max */
330 int current_read_bias; /* Current read bias state */
358 int lim = ios->limiter; in cam_iosched_limiter_init()
373 int lim = ios->limiter; in cam_iosched_limiter_tick()
388 int lim = ios->limiter; in cam_iosched_limiter_iop()
403 int lim = ios->limiter; in cam_iosched_limiter_caniop()
418 int lim = ios->limiter; in cam_iosched_limiter_iodone()
438 if (ios->current <= 0 || ios->pending < ios->current) in cam_iosched_qd_iop()
448 if (ios->current <= 0 || ios->pending < ios->current) in cam_iosched_qd_caniop()
458 if (ios->current <= 0 || ios->pending != ios->current) in cam_iosched_qd_iodone()
468 ios->l_value1 = ios->current / ios->softc->quanta; in cam_iosched_iops_init()
469 if (ios->l_value1 <= 0) in cam_iosched_iops_init()
470 ios->l_value1 = 1; in cam_iosched_iops_init()
471 ios->l_value2 = 0; in cam_iosched_iops_init()
485 new_ios = (int)((ios->current * (uint64_t)ios->softc->this_frac) >> 16); in cam_iosched_iops_tick()
486 if (new_ios < 1 && ios->l_value2 < ios->current) { in cam_iosched_iops_tick()
488 ios->l_value2++; in cam_iosched_iops_tick()
496 if ((ios->softc->total_ticks % ios->softc->quanta) == 0) { in cam_iosched_iops_tick()
497 ios->l_value1 = new_ios; in cam_iosched_iops_tick()
498 ios->l_value2 = 1; in cam_iosched_iops_tick()
500 ios->l_value1 += new_ios; in cam_iosched_iops_tick()
512 * otherwise wait. If current iops is 0, treat that in cam_iosched_iops_caniop()
515 if (ios->current > 0 && ios->l_value1 <= 0) in cam_iosched_iops_caniop()
527 ios->l_value1--; in cam_iosched_iops_iop()
536 /* ios->current is in kB/s, so scale to bytes */ in cam_iosched_bw_init()
537 ios->l_value1 = ios->current * 1000 / ios->softc->quanta; in cam_iosched_bw_init()
551 * then too bad, that's lost. Also, ios->current in cam_iosched_bw_tick()
558 bw = (int)((ios->current * 1000ull * (uint64_t)ios->softc->this_frac) >> 16); in cam_iosched_bw_tick()
559 if (ios->l_value1 < bw * 4) in cam_iosched_bw_tick()
560 ios->l_value1 += bw; in cam_iosched_bw_tick()
582 * Also note that if the current limit is <= 0, in cam_iosched_bw_caniop()
585 if (ios->current > 0 && ios->l_value1 <= 0) in cam_iosched_bw_caniop()
598 ios->l_value1 -= bp->bio_length; in cam_iosched_bw_iop()
612 callout_reset(&isc->ticker, hz / isc->quanta, cam_iosched_ticker, isc); in cam_iosched_ticker()
615 delta = now - isc->last_time; in cam_iosched_ticker()
616 …isc->this_frac = (uint32_t)delta >> 16; /* Note: discards seconds -- should be 0 harmless if not … in cam_iosched_ticker()
617 isc->last_time = now; in cam_iosched_ticker()
619 cam_iosched_cl_maybe_steer(&isc->cl); in cam_iosched_ticker()
621 cam_iosched_limiter_tick(&isc->read_stats); in cam_iosched_ticker()
622 cam_iosched_limiter_tick(&isc->write_stats); in cam_iosched_ticker()
623 cam_iosched_limiter_tick(&isc->trim_stats); in cam_iosched_ticker()
625 isc->schedfnc(isc->periph); in cam_iosched_ticker()
628 * isc->load is an EMA of the pending I/Os at each tick. The number of in cam_iosched_ticker()
638 * The reason for the rate limiting bit is because those I/Os in cam_iosched_ticker()
648 pending = isc->read_stats.pending + isc->write_stats.pending /* + isc->trim_stats.pending */; in cam_iosched_ticker()
649 pending += !!(isc->read_stats.state_flags & IOP_RATE_LIMITED) * isc->read_stats.queued + in cam_iosched_ticker()
650 !!(isc->write_stats.state_flags & IOP_RATE_LIMITED) * isc->write_stats.queued /* + in cam_iosched_ticker()
651 !!(isc->trim_stats.state_flags & IOP_RATE_LIMITED) * isc->trim_stats.queued */ ; in cam_iosched_ticker()
653 pending /= isc->periph->path->device->ccbq.total_openings; in cam_iosched_ticker()
655 …isc->load = (pending + (isc->load << 13) - isc->load) >> 13; /* see above: 13 -> 16139 / 200/s = ~… in cam_iosched_ticker()
657 isc->total_ticks++; in cam_iosched_ticker()
664 clp->next_steer = sbinuptime(); in cam_iosched_cl_init()
665 clp->softc = isc; in cam_iosched_cl_init()
666 clp->steer_interval = SBT_1S * 5; /* Let's start out steering every 5s */ in cam_iosched_cl_init()
667 clp->lolat = 5 * SBT_1MS; in cam_iosched_cl_init()
668 clp->hilat = 15 * SBT_1MS; in cam_iosched_cl_init()
669 clp->alpha = 20; /* Alpha == gain. 20 = .2 */ in cam_iosched_cl_init()
670 clp->type = set_max; in cam_iosched_cl_init()
680 isc = clp->softc; in cam_iosched_cl_maybe_steer()
681 now = isc->last_time; in cam_iosched_cl_maybe_steer()
682 if (now < clp->next_steer) in cam_iosched_cl_maybe_steer()
685 clp->next_steer = now + clp->steer_interval; in cam_iosched_cl_maybe_steer()
686 switch (clp->type) { in cam_iosched_cl_maybe_steer()
688 if (isc->write_stats.current != isc->write_stats.max) in cam_iosched_cl_maybe_steer()
690 isc->write_stats.current, isc->write_stats.max); in cam_iosched_cl_maybe_steer()
691 isc->read_stats.current = isc->read_stats.max; in cam_iosched_cl_maybe_steer()
692 isc->write_stats.current = isc->write_stats.max; in cam_iosched_cl_maybe_steer()
693 isc->trim_stats.current = isc->trim_stats.max; in cam_iosched_cl_maybe_steer()
696 old = isc->write_stats.current; in cam_iosched_cl_maybe_steer()
697 lat = isc->read_stats.ema; in cam_iosched_cl_maybe_steer()
699 * Simple PLL-like engine. Since we're steering to a range for in cam_iosched_cl_maybe_steer()
709 * boiler control PLL. this may result in over-steering while in cam_iosched_cl_maybe_steer()
724 if (lat < clp->lolat || isc->read_stats.total - clp->last_count < 10) in cam_iosched_cl_maybe_steer()
725 isc->write_stats.current = isc->write_stats.current * in cam_iosched_cl_maybe_steer()
726 (100 + clp->alpha) / 100; /* Scale up */ in cam_iosched_cl_maybe_steer()
727 else if (lat > clp->hilat) in cam_iosched_cl_maybe_steer()
728 isc->write_stats.current = isc->write_stats.current * in cam_iosched_cl_maybe_steer()
729 (100 - clp->alpha) / 100; /* Scale down */ in cam_iosched_cl_maybe_steer()
730 clp->last_count = isc->read_stats.total; in cam_iosched_cl_maybe_steer()
736 if (isc->write_stats.current < isc->write_stats.min) in cam_iosched_cl_maybe_steer()
737 isc->write_stats.current = isc->write_stats.min; in cam_iosched_cl_maybe_steer()
738 if (isc->write_stats.current > isc->write_stats.max) in cam_iosched_cl_maybe_steer()
739 isc->write_stats.current = isc->write_stats.max; in cam_iosched_cl_maybe_steer()
740 if (old != isc->write_stats.current && iosched_debug) in cam_iosched_cl_maybe_steer()
742 old, isc->write_stats.current, in cam_iosched_cl_maybe_steer()
771 return !!(isc->flags & CAM_IOSCHED_FLAG_WORK_FLAGS); in cam_iosched_has_flagged_work()
779 struct bio *rbp = bioq_first(&isc->bio_queue); in cam_iosched_has_io()
780 struct bio *wbp = bioq_first(&isc->write_queue); in cam_iosched_has_io()
782 cam_iosched_limiter_caniop(&isc->write_stats, wbp) == 0; in cam_iosched_has_io()
784 cam_iosched_limiter_caniop(&isc->read_stats, rbp) == 0; in cam_iosched_has_io()
786 …an write %d: pending_writes %d max_writes %d\n", can_write, isc->write_stats.pending, isc->write_s… in cam_iosched_has_io()
787 …n read %d: read_stats.pending %d max_reads %d\n", can_read, isc->read_stats.pending, isc->read_sta… in cam_iosched_has_io()
788 printf("Queued reads %d writes %d\n", isc->read_stats.queued, isc->write_stats.queued); in cam_iosched_has_io()
793 return bioq_first(&isc->bio_queue) != NULL; in cam_iosched_has_io()
801 bp = bioq_first(&isc->trim_queue); in cam_iosched_has_more_trim()
805 * If we're limiting trims, then defer action on trims in cam_iosched_has_more_trim()
808 if (bp == NULL || cam_iosched_limiter_caniop(&isc->trim_stats, bp) != 0) in cam_iosched_has_more_trim()
818 if (isc->trim_goal > 0) { in cam_iosched_has_more_trim()
819 if (isc->queued_trims >= isc->trim_goal) in cam_iosched_has_more_trim()
821 if (isc->queued_trims > 0 && in cam_iosched_has_more_trim()
822 isc->trim_ticks > 0 && in cam_iosched_has_more_trim()
823 ticks - isc->last_trim_tick > isc->trim_ticks) in cam_iosched_has_more_trim()
829 return !(isc->flags & CAM_IOSCHED_FLAG_TRIM_ACTIVE) && bp != NULL; in cam_iosched_has_more_trim()
832 #define cam_iosched_sort_queue(isc) ((isc)->sort_io_queue >= 0 ? \
833 (isc)->sort_io_queue : cam_sort_io_queues)
855 ios->limiter = none; in cam_iosched_iop_stats_init()
856 ios->in = 0; in cam_iosched_iop_stats_init()
857 ios->max = ios->current = 300000; in cam_iosched_iop_stats_init()
858 ios->min = 1; in cam_iosched_iop_stats_init()
859 ios->out = 0; in cam_iosched_iop_stats_init()
860 ios->errs = 0; in cam_iosched_iop_stats_init()
861 ios->pending = 0; in cam_iosched_iop_stats_init()
862 ios->queued = 0; in cam_iosched_iop_stats_init()
863 ios->total = 0; in cam_iosched_iop_stats_init()
864 ios->ema = 0; in cam_iosched_iop_stats_init()
865 ios->emvar = 0; in cam_iosched_iop_stats_init()
866 ios->bad_latency = SBT_1S / 2; /* Default to 500ms */ in cam_iosched_iop_stats_init()
867 ios->softc = isc; in cam_iosched_iop_stats_init()
881 isc = ios->softc; in cam_iosched_limiter_sysctl()
882 value = ios->limiter; in cam_iosched_limiter_sysctl()
890 if (error != 0 || req->newptr == NULL) in cam_iosched_limiter_sysctl()
893 cam_periph_lock(isc->periph); in cam_iosched_limiter_sysctl()
898 ios->limiter = i; in cam_iosched_limiter_sysctl()
901 ios->limiter = value; in cam_iosched_limiter_sysctl()
902 cam_periph_unlock(isc->periph); in cam_iosched_limiter_sysctl()
906 callout_reset(&isc->ticker, hz / isc->quanta, cam_iosched_ticker, isc); in cam_iosched_limiter_sysctl()
907 isc->flags |= CAM_IOSCHED_FLAG_CALLOUT_ACTIVE; in cam_iosched_limiter_sysctl()
909 cam_periph_unlock(isc->periph); in cam_iosched_limiter_sysctl()
913 cam_periph_unlock(isc->periph); in cam_iosched_limiter_sysctl()
927 isc = clp->softc; in cam_iosched_control_type_sysctl()
928 value = clp->type; in cam_iosched_control_type_sysctl()
936 if (error != 0 || req->newptr == NULL) in cam_iosched_control_type_sysctl()
942 cam_periph_lock(isc->periph); in cam_iosched_control_type_sysctl()
943 clp->type = i; in cam_iosched_control_type_sysctl()
944 cam_periph_unlock(isc->periph); in cam_iosched_control_type_sysctl()
963 if (error != 0 || req->newptr == NULL) in cam_iosched_sbintime_sysctl()
982 for (i = 0; i < LAT_BUCKETS - 1; i++) in cam_iosched_sysctl_latencies()
984 sbuf_printf(&sb, "%jd", (intmax_t)latencies[LAT_BUCKETS - 1]); in cam_iosched_sysctl_latencies()
1001 if ((error != 0) || (req->newptr == NULL)) in cam_iosched_quanta_sysctl()
1018 ios->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx, in cam_iosched_iop_stats_sysctl_init()
1019 SYSCTL_CHILDREN(isc->sysctl_tree), OID_AUTO, name, in cam_iosched_iop_stats_sysctl_init()
1021 n = SYSCTL_CHILDREN(ios->sysctl_tree); in cam_iosched_iop_stats_sysctl_init()
1022 ctx = &ios->sysctl_ctx; in cam_iosched_iop_stats_sysctl_init()
1026 &ios->ema, in cam_iosched_iop_stats_sysctl_init()
1030 &ios->emvar, in cam_iosched_iop_stats_sysctl_init()
1035 &ios->pending, 0, in cam_iosched_iop_stats_sysctl_init()
1039 &ios->total, 0, in cam_iosched_iop_stats_sysctl_init()
1043 &ios->queued, 0, in cam_iosched_iop_stats_sysctl_init()
1047 &ios->in, 0, in cam_iosched_iop_stats_sysctl_init()
1051 &ios->out, 0, in cam_iosched_iop_stats_sysctl_init()
1055 &ios->errs, 0, in cam_iosched_iop_stats_sysctl_init()
1059 &ios->too_long, 0, in cam_iosched_iop_stats_sysctl_init()
1064 &ios->bad_latency, 0, cam_iosched_sbintime_sysctl, "A", in cam_iosched_iop_stats_sysctl_init()
1071 "Current limiting type."); in cam_iosched_iop_stats_sysctl_init()
1074 &ios->min, 0, in cam_iosched_iop_stats_sysctl_init()
1078 &ios->max, 0, in cam_iosched_iop_stats_sysctl_init()
1081 OID_AUTO, "current", CTLFLAG_RW, in cam_iosched_iop_stats_sysctl_init()
1082 &ios->current, 0, in cam_iosched_iop_stats_sysctl_init()
1083 "current resource"); in cam_iosched_iop_stats_sysctl_init()
1088 &ios->latencies, 0, in cam_iosched_iop_stats_sysctl_init()
1101 if (ios->sysctl_tree) in cam_iosched_iop_stats_fini()
1102 if (sysctl_ctx_free(&ios->sysctl_ctx) != 0) in cam_iosched_iop_stats_fini()
1113 clp = &isc->cl; in cam_iosched_cl_sysctl_init()
1114 clp->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx, in cam_iosched_cl_sysctl_init()
1115 SYSCTL_CHILDREN(isc->sysctl_tree), OID_AUTO, "control", in cam_iosched_cl_sysctl_init()
1117 n = SYSCTL_CHILDREN(clp->sysctl_tree); in cam_iosched_cl_sysctl_init()
1118 ctx = &clp->sysctl_ctx; in cam_iosched_cl_sysctl_init()
1128 &clp->steer_interval, 0, cam_iosched_sbintime_sysctl, "A", in cam_iosched_cl_sysctl_init()
1133 &clp->lolat, 0, cam_iosched_sbintime_sysctl, "A", in cam_iosched_cl_sysctl_init()
1138 &clp->hilat, 0, cam_iosched_sbintime_sysctl, "A", in cam_iosched_cl_sysctl_init()
1142 &clp->alpha, 0, in cam_iosched_cl_sysctl_init()
1149 if (clp->sysctl_tree) in cam_iosched_cl_sysctl_fini()
1150 if (sysctl_ctx_free(&clp->sysctl_ctx) != 0) in cam_iosched_cl_sysctl_fini()
1168 isc->disk = dp; in cam_iosched_init()
1169 isc->schedfnc = schedfnc; in cam_iosched_init()
1174 isc->sort_io_queue = -1; in cam_iosched_init()
1175 bioq_init(&isc->bio_queue); in cam_iosched_init()
1176 bioq_init(&isc->trim_queue); in cam_iosched_init()
1179 bioq_init(&isc->write_queue); in cam_iosched_init()
1180 isc->read_bias = default_read_bias; in cam_iosched_init()
1181 isc->current_read_bias = 0; in cam_iosched_init()
1182 isc->quanta = min(hz, 200); in cam_iosched_init()
1183 cam_iosched_iop_stats_init(isc, &isc->read_stats); in cam_iosched_init()
1184 cam_iosched_iop_stats_init(isc, &isc->write_stats); in cam_iosched_init()
1185 cam_iosched_iop_stats_init(isc, &isc->trim_stats); in cam_iosched_init()
1186 isc->trim_stats.max = 1; /* Trims are special: one at a time for now */ in cam_iosched_init()
1187 isc->last_time = sbinuptime(); in cam_iosched_init()
1188 callout_init_mtx(&isc->ticker, cam_periph_mtx(periph), 0); in cam_iosched_init()
1189 isc->periph = periph; in cam_iosched_init()
1190 cam_iosched_cl_init(&isc->cl, isc); in cam_iosched_init()
1191 callout_reset(&isc->ticker, hz / isc->quanta, cam_iosched_ticker, isc); in cam_iosched_init()
1192 isc->flags |= CAM_IOSCHED_FLAG_CALLOUT_ACTIVE; in cam_iosched_init()
1210 cam_iosched_iop_stats_fini(&isc->read_stats); in cam_iosched_fini()
1211 cam_iosched_iop_stats_fini(&isc->write_stats); in cam_iosched_fini()
1212 cam_iosched_iop_stats_fini(&isc->trim_stats); in cam_iosched_fini()
1213 cam_iosched_cl_sysctl_fini(&isc->cl); in cam_iosched_fini()
1214 if (isc->sysctl_tree) in cam_iosched_fini()
1215 if (sysctl_ctx_free(&isc->sysctl_ctx) != 0) in cam_iosched_fini()
1217 if (isc->flags & CAM_IOSCHED_FLAG_CALLOUT_ACTIVE) { in cam_iosched_fini()
1218 callout_drain(&isc->ticker); in cam_iosched_fini()
1219 isc->flags &= ~ CAM_IOSCHED_FLAG_CALLOUT_ACTIVE; in cam_iosched_fini()
1238 &isc->sort_io_queue, 0, in cam_iosched_sysctl_init()
1242 &isc->trim_goal, 0, in cam_iosched_sysctl_init()
1246 &isc->trim_goal, 0, in cam_iosched_sysctl_init()
1253 isc->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx, in cam_iosched_sysctl_init()
1256 n = SYSCTL_CHILDREN(isc->sysctl_tree); in cam_iosched_sysctl_init()
1257 ctx = &isc->sysctl_ctx; in cam_iosched_sysctl_init()
1259 cam_iosched_iop_stats_sysctl_init(isc, &isc->read_stats, "read"); in cam_iosched_sysctl_init()
1260 cam_iosched_iop_stats_sysctl_init(isc, &isc->write_stats, "write"); in cam_iosched_sysctl_init()
1261 cam_iosched_iop_stats_sysctl_init(isc, &isc->trim_stats, "trim"); in cam_iosched_sysctl_init()
1266 &isc->read_bias, default_read_bias, in cam_iosched_sysctl_init()
1271 &isc->quanta, 0, cam_iosched_quanta_sysctl, "I", in cam_iosched_sysctl_init()
1276 &isc->total_ticks, 0, in cam_iosched_sysctl_init()
1281 &isc->load, 0, in cam_iosched_sysctl_init()
1286 &isc->max_lat, 0, in cam_iosched_sysctl_init()
1296 isc->latfcn = fnp; in cam_iosched_set_latfcn()
1297 isc->latarg = argp; in cam_iosched_set_latfcn()
1326 isc->trim_goal = goal; in cam_iosched_set_trim_goal()
1333 isc->trim_ticks = trim_ticks; in cam_iosched_set_trim_ticks()
1344 bioq_flush(&isc->bio_queue, stp, err); in cam_iosched_flush()
1345 bioq_flush(&isc->trim_queue, stp, err); in cam_iosched_flush()
1348 bioq_flush(&isc->write_queue, stp, err); in cam_iosched_flush()
1363 * the NAND media. Limiting the queue depth like this will also limit in cam_iosched_get_write()
1367 bp = bioq_first(&isc->write_queue); in cam_iosched_get_write()
1375 * If pending read, prefer that based on current read bias in cam_iosched_get_write()
1378 if (bioq_first(&isc->bio_queue) && isc->current_read_bias) { in cam_iosched_get_write()
1382 isc->current_read_bias, isc->write_stats.queued, in cam_iosched_get_write()
1383 isc->read_stats.queued); in cam_iosched_get_write()
1384 isc->current_read_bias--; in cam_iosched_get_write()
1385 /* We're not limiting writes, per se, just doing reads first */ in cam_iosched_get_write()
1390 * See if our current limiter allows this I/O. in cam_iosched_get_write()
1392 if (cam_iosched_limiter_iop(&isc->write_stats, bp) != 0) { in cam_iosched_get_write()
1395 isc->write_stats.state_flags |= IOP_RATE_LIMITED; in cam_iosched_get_write()
1403 isc->current_read_bias = isc->read_bias; in cam_iosched_get_write()
1404 bioq_remove(&isc->write_queue, bp); in cam_iosched_get_write()
1405 if (bp->bio_cmd == BIO_WRITE) { in cam_iosched_get_write()
1406 isc->write_stats.queued--; in cam_iosched_get_write()
1407 isc->write_stats.total++; in cam_iosched_get_write()
1408 isc->write_stats.pending++; in cam_iosched_get_write()
1411 printf("HWQ : %p %#x\n", bp, bp->bio_cmd); in cam_iosched_get_write()
1412 isc->write_stats.state_flags &= ~IOP_RATE_LIMITED; in cam_iosched_get_write()
1423 bioq_insert_head(&isc->trim_queue, bp); in cam_iosched_put_back_trim()
1424 if (isc->queued_trims == 0) in cam_iosched_put_back_trim()
1425 isc->last_trim_tick = ticks; in cam_iosched_put_back_trim()
1426 isc->queued_trims++; in cam_iosched_put_back_trim()
1428 isc->trim_stats.queued++; in cam_iosched_put_back_trim()
1429 isc->trim_stats.total--; /* since we put it back, don't double count */ in cam_iosched_put_back_trim()
1430 isc->trim_stats.pending--; in cam_iosched_put_back_trim()
1446 bp = bioq_first(&isc->trim_queue); in cam_iosched_next_trim()
1449 bioq_remove(&isc->trim_queue, bp); in cam_iosched_next_trim()
1450 isc->queued_trims--; in cam_iosched_next_trim()
1451 isc->last_trim_tick = ticks; /* Reset the tick timer when we take trims */ in cam_iosched_next_trim()
1453 isc->trim_stats.queued--; in cam_iosched_next_trim()
1454 isc->trim_stats.total++; in cam_iosched_next_trim()
1455 isc->trim_stats.pending++; in cam_iosched_next_trim()
1477 bp = bioq_first(&isc->trim_queue); in cam_iosched_get_trim()
1482 * If pending read, prefer that based on current read bias setting. The in cam_iosched_get_trim()
1487 if (bioq_first(&isc->bio_queue) && isc->current_read_bias) { in cam_iosched_get_trim()
1491 isc->current_read_bias, isc->trim_stats.queued, in cam_iosched_get_trim()
1492 isc->read_stats.queued); in cam_iosched_get_trim()
1493 isc->current_read_bias--; in cam_iosched_get_trim()
1494 /* We're not limiting TRIMS, per se, just doing reads first */ in cam_iosched_get_trim()
1500 isc->current_read_bias = isc->read_bias; in cam_iosched_get_trim()
1504 * See if our current limiter allows this I/O. Because we only call this in cam_iosched_get_trim()
1511 * have the effect of limiting the iops as seen from the upper layers. in cam_iosched_get_trim()
1513 if (cam_iosched_limiter_iop(&isc->trim_stats, bp) != 0) { in cam_iosched_get_trim()
1516 isc->trim_stats.state_flags |= IOP_RATE_LIMITED; in cam_iosched_get_trim()
1519 isc->current_read_bias = isc->read_bias; in cam_iosched_get_trim()
1520 isc->trim_stats.state_flags &= ~IOP_RATE_LIMITED; in cam_iosched_get_trim()
1536 if (bp == NULL || bp->bio_flags & BIO_ORDERED) in bio_next()
1544 return ios->state_flags & IOP_RATE_LIMITED; in cam_iosched_rate_limited()
1585 for (bp = bioq_first(&isc->bio_queue); bp != NULL; in cam_iosched_next_bio()
1593 if (bp->bio_cmd == BIO_READ) { in cam_iosched_next_bio()
1594 if (cam_iosched_rate_limited(&isc->read_stats) || in cam_iosched_next_bio()
1595 cam_iosched_limiter_iop(&isc->read_stats, bp) != 0) { in cam_iosched_next_bio()
1596 isc->read_stats.state_flags |= IOP_RATE_LIMITED; in cam_iosched_next_bio()
1599 isc->read_stats.state_flags &= ~IOP_RATE_LIMITED; in cam_iosched_next_bio()
1606 * in the queue after we transition from 0 to non-zero. in cam_iosched_next_bio()
1608 if (bp->bio_cmd == BIO_WRITE) { in cam_iosched_next_bio()
1609 if (cam_iosched_rate_limited(&isc->write_stats) || in cam_iosched_next_bio()
1610 cam_iosched_limiter_iop(&isc->write_stats, bp) != 0) { in cam_iosched_next_bio()
1611 isc->write_stats.state_flags |= IOP_RATE_LIMITED; in cam_iosched_next_bio()
1614 isc->write_stats.state_flags &= ~IOP_RATE_LIMITED; in cam_iosched_next_bio()
1624 bp = bioq_first(&isc->bio_queue); in cam_iosched_next_bio()
1628 bioq_remove(&isc->bio_queue, bp); in cam_iosched_next_bio()
1631 if (bp->bio_cmd == BIO_READ) { in cam_iosched_next_bio()
1632 isc->read_stats.queued--; in cam_iosched_next_bio()
1633 isc->read_stats.total++; in cam_iosched_next_bio()
1634 isc->read_stats.pending++; in cam_iosched_next_bio()
1635 } else if (bp->bio_cmd == BIO_WRITE) { in cam_iosched_next_bio()
1636 isc->write_stats.queued--; in cam_iosched_next_bio()
1637 isc->write_stats.total++; in cam_iosched_next_bio()
1638 isc->write_stats.pending++; in cam_iosched_next_bio()
1642 printf("HWQ : %p %#x\n", bp, bp->bio_cmd); in cam_iosched_next_bio()
1669 if (bp->bio_cmd == BIO_SPEEDUP) { in cam_iosched_queue_work()
1674 while (bioq_first(&isc->trim_queue) && in cam_iosched_queue_work()
1675 (bp->bio_length == 0 || len < bp->bio_length)) { in cam_iosched_queue_work()
1676 nbp = bioq_takefirst(&isc->trim_queue); in cam_iosched_queue_work()
1677 len += nbp->bio_length; in cam_iosched_queue_work()
1678 nbp->bio_error = 0; in cam_iosched_queue_work()
1681 if (bp->bio_length > 0) { in cam_iosched_queue_work()
1682 if (bp->bio_length > len) in cam_iosched_queue_work()
1683 bp->bio_resid = bp->bio_length - len; in cam_iosched_queue_work()
1685 bp->bio_resid = 0; in cam_iosched_queue_work()
1687 bp->bio_error = 0; in cam_iosched_queue_work()
1694 * set the last tick time to one less than the current ticks minus the in cam_iosched_queue_work()
1697 if (bp->bio_cmd == BIO_FLUSH && isc->trim_ticks > 0) in cam_iosched_queue_work()
1698 isc->last_trim_tick = ticks - isc->trim_ticks - 1; in cam_iosched_queue_work()
1704 if (bp->bio_cmd == BIO_DELETE) { in cam_iosched_queue_work()
1705 bioq_insert_tail(&isc->trim_queue, bp); in cam_iosched_queue_work()
1706 if (isc->queued_trims == 0) in cam_iosched_queue_work()
1707 isc->last_trim_tick = ticks; in cam_iosched_queue_work()
1708 isc->queued_trims++; in cam_iosched_queue_work()
1710 isc->trim_stats.in++; in cam_iosched_queue_work()
1711 isc->trim_stats.queued++; in cam_iosched_queue_work()
1715 else if (do_dynamic_iosched && isc->read_bias != 0 && in cam_iosched_queue_work()
1716 (bp->bio_cmd != BIO_READ)) { in cam_iosched_queue_work()
1718 bioq_disksort(&isc->write_queue, bp); in cam_iosched_queue_work()
1720 bioq_insert_tail(&isc->write_queue, bp); in cam_iosched_queue_work()
1722 printf("Qw : %p %#x\n", bp, bp->bio_cmd); in cam_iosched_queue_work()
1723 if (bp->bio_cmd == BIO_WRITE) { in cam_iosched_queue_work()
1724 isc->write_stats.in++; in cam_iosched_queue_work()
1725 isc->write_stats.queued++; in cam_iosched_queue_work()
1731 bioq_disksort(&isc->bio_queue, bp); in cam_iosched_queue_work()
1733 bioq_insert_tail(&isc->bio_queue, bp); in cam_iosched_queue_work()
1736 printf("Qr : %p %#x\n", bp, bp->bio_cmd); in cam_iosched_queue_work()
1737 if (bp->bio_cmd == BIO_READ) { in cam_iosched_queue_work()
1738 isc->read_stats.in++; in cam_iosched_queue_work()
1739 isc->read_stats.queued++; in cam_iosched_queue_work()
1740 } else if (bp->bio_cmd == BIO_WRITE) { in cam_iosched_queue_work()
1741 isc->write_stats.in++; in cam_iosched_queue_work()
1742 isc->write_stats.queued++; in cam_iosched_queue_work()
1766 isc->flags &= ~CAM_IOSCHED_FLAG_TRIM_ACTIVE; in cam_iosched_trim_done()
1783 printf("done: %p %#x\n", bp, bp->bio_cmd); in cam_iosched_bio_complete()
1784 if (bp->bio_cmd == BIO_WRITE) { in cam_iosched_bio_complete()
1785 retval = cam_iosched_limiter_iodone(&isc->write_stats, bp); in cam_iosched_bio_complete()
1786 if ((bp->bio_flags & BIO_ERROR) != 0) in cam_iosched_bio_complete()
1787 isc->write_stats.errs++; in cam_iosched_bio_complete()
1788 isc->write_stats.out++; in cam_iosched_bio_complete()
1789 isc->write_stats.pending--; in cam_iosched_bio_complete()
1790 } else if (bp->bio_cmd == BIO_READ) { in cam_iosched_bio_complete()
1791 retval = cam_iosched_limiter_iodone(&isc->read_stats, bp); in cam_iosched_bio_complete()
1792 if ((bp->bio_flags & BIO_ERROR) != 0) in cam_iosched_bio_complete()
1793 isc->read_stats.errs++; in cam_iosched_bio_complete()
1794 isc->read_stats.out++; in cam_iosched_bio_complete()
1795 isc->read_stats.pending--; in cam_iosched_bio_complete()
1796 } else if (bp->bio_cmd == BIO_DELETE) { in cam_iosched_bio_complete()
1797 if ((bp->bio_flags & BIO_ERROR) != 0) in cam_iosched_bio_complete()
1798 isc->trim_stats.errs++; in cam_iosched_bio_complete()
1799 isc->trim_stats.out++; in cam_iosched_bio_complete()
1800 isc->trim_stats.pending--; in cam_iosched_bio_complete()
1801 } else if (bp->bio_cmd != BIO_FLUSH) { in cam_iosched_bio_complete()
1803 printf("Completing command with bio_cmd == %#x\n", bp->bio_cmd); in cam_iosched_bio_complete()
1806 if ((bp->bio_flags & BIO_ERROR) == 0 && done_ccb != NULL && in cam_iosched_bio_complete()
1807 (done_ccb->ccb_h.status & CAM_QOS_VALID) != 0) { in cam_iosched_bio_complete()
1810 sim_latency = cam_iosched_sbintime_t(done_ccb->ccb_h.qos.periph_data); in cam_iosched_bio_complete()
1818 if (isc->latfcn && isc->max_lat != 0 && sim_latency > isc->max_lat) in cam_iosched_bio_complete()
1819 isc->latfcn(isc->latarg, sim_latency, bp); in cam_iosched_bio_complete()
1834 isc->flags |= CAM_IOSCHED_FLAG_TRIM_ACTIVE; in cam_iosched_submit_trim()
1844 isc->sort_io_queue = val; in cam_iosched_set_sort_queue()
1850 return isc->flags & flags; in cam_iosched_has_work_flags()
1856 isc->flags |= flags; in cam_iosched_set_work_flags()
1862 isc->flags &= ~flags; in cam_iosched_clr_work_flags()
1869 * http://www.embedded.com/electronics-blogs/programmer-s-toolbox/4219659/Integer-Square-Roots
1884 uint64_t bit = 1ULL << (sizeof(uint64_t) * NBBY - 2); in isqrt64()
1901 val -= res + bit; in isqrt64()
1911 static sbintime_t latencies[LAT_BUCKETS - 1] = {
1939 daddr_t lba = bp->bio_pblkno; in cam_iosched_devctl_outlier()
1940 daddr_t cnt = bp->bio_bcount / iop->softc->disk->d_sectorsize; in cam_iosched_devctl_outlier()
1950 iop->softc->periph->periph_name, in cam_iosched_devctl_outlier()
1951 iop->softc->periph->unit_number, in cam_iosched_devctl_outlier()
1970 if (sim_latency > iop->bad_latency) { in cam_iosched_update()
1972 iop->too_long++; in cam_iosched_update()
1979 for (i = 0; i < LAT_BUCKETS - 1; i++) { in cam_iosched_update()
1981 iop->latencies[i]++; in cam_iosched_update()
1985 if (i == LAT_BUCKETS - 1) in cam_iosched_update()
1986 iop->latencies[i]++; /* Put all > 8192ms values into the last bucket. */ in cam_iosched_update()
1990 * (2 ^ -alpha_bits). For more info see the NIST statistical in cam_iosched_update()
1993 * ema_t = y_t * alpha + ema_t-1 * (1 - alpha) [nist] in cam_iosched_update()
1994 * ema_t = y_t * alpha + ema_t-1 - alpha * ema_t-1 in cam_iosched_update()
1995 * ema_t = alpha * y_t - alpha * ema_t-1 + ema_t-1 in cam_iosched_update()
1997 * sub e == ema_t-1, b == 1/alpha (== 1 << alpha_bits), d == y_t - ema_t-1 in cam_iosched_update()
1998 * = y_t/b - e/b + be/b in cam_iosched_update()
1999 * = (y_t - e + be) / b in cam_iosched_update()
2006 * diff_t = y_t - ema_t-1 in cam_iosched_update()
2007 * emvar_t = (1 - alpha) * (emavar_t-1 + diff_t^2 * alpha) in cam_iosched_update()
2008 * = emavar_t-1 - alpha * emavar_t-1 + delta_t^2 * alpha - (delta_t * alpha)^2 in cam_iosched_update()
2009 * sub b == 1/alpha (== 1 << alpha_bits), e == emavar_t-1, d = delta_t^2 in cam_iosched_update()
2010 * = e - e/b + dd/b + dd/bb in cam_iosched_update()
2011 * = (bbe - be + bdd + dd) / bb in cam_iosched_update()
2012 * = (bbe + b(dd-e) + dd) / bb (which is expanded below bb = 1<<(2*alpha_bits)) in cam_iosched_update()
2024 delta = (y - iop->ema); /* d */ in cam_iosched_update()
2025 iop->ema = ((iop->ema << alpha_bits) + delta) >> alpha_bits; in cam_iosched_update()
2043 iop->emvar = ((iop->emvar << (2 * alpha_bits)) + /* bbe */ in cam_iosched_update()
2044 ((deltasq - iop->emvar) << alpha_bits) + /* b(dd-e) */ in cam_iosched_update()
2047 iop->sd = (sbintime_t)isqrt64((uint64_t)iop->emvar) << 12; in cam_iosched_update()
2054 switch (bp->bio_cmd) { in cam_iosched_io_metric_update()
2056 cam_iosched_update(&isc->read_stats, sim_latency, bp); in cam_iosched_io_metric_update()
2059 cam_iosched_update(&isc->write_stats, sim_latency, bp); in cam_iosched_io_metric_update()
2062 cam_iosched_update(&isc->trim_stats, sim_latency, bp); in cam_iosched_io_metric_update()
2075 TAILQ_FOREACH(bp, &bq->queue, bio_queue) { in biolen()
2093 db_printf("pending_reads: %d\n", isc->read_stats.pending); in DB_SHOW_COMMAND()
2094 db_printf("min_reads: %d\n", isc->read_stats.min); in DB_SHOW_COMMAND()
2095 db_printf("max_reads: %d\n", isc->read_stats.max); in DB_SHOW_COMMAND()
2096 db_printf("reads: %d\n", isc->read_stats.total); in DB_SHOW_COMMAND()
2097 db_printf("in_reads: %d\n", isc->read_stats.in); in DB_SHOW_COMMAND()
2098 db_printf("out_reads: %d\n", isc->read_stats.out); in DB_SHOW_COMMAND()
2099 db_printf("queued_reads: %d\n", isc->read_stats.queued); in DB_SHOW_COMMAND()
2100 db_printf("Read Q len %d\n", biolen(&isc->bio_queue)); in DB_SHOW_COMMAND()
2101 db_printf("pending_writes: %d\n", isc->write_stats.pending); in DB_SHOW_COMMAND()
2102 db_printf("min_writes: %d\n", isc->write_stats.min); in DB_SHOW_COMMAND()
2103 db_printf("max_writes: %d\n", isc->write_stats.max); in DB_SHOW_COMMAND()
2104 db_printf("writes: %d\n", isc->write_stats.total); in DB_SHOW_COMMAND()
2105 db_printf("in_writes: %d\n", isc->write_stats.in); in DB_SHOW_COMMAND()
2106 db_printf("out_writes: %d\n", isc->write_stats.out); in DB_SHOW_COMMAND()
2107 db_printf("queued_writes: %d\n", isc->write_stats.queued); in DB_SHOW_COMMAND()
2108 db_printf("Write Q len %d\n", biolen(&isc->write_queue)); in DB_SHOW_COMMAND()
2109 db_printf("pending_trims: %d\n", isc->trim_stats.pending); in DB_SHOW_COMMAND()
2110 db_printf("min_trims: %d\n", isc->trim_stats.min); in DB_SHOW_COMMAND()
2111 db_printf("max_trims: %d\n", isc->trim_stats.max); in DB_SHOW_COMMAND()
2112 db_printf("trims: %d\n", isc->trim_stats.total); in DB_SHOW_COMMAND()
2113 db_printf("in_trims: %d\n", isc->trim_stats.in); in DB_SHOW_COMMAND()
2114 db_printf("out_trims: %d\n", isc->trim_stats.out); in DB_SHOW_COMMAND()
2115 db_printf("queued_trims: %d\n", isc->trim_stats.queued); in DB_SHOW_COMMAND()
2116 db_printf("Trim Q len %d\n", biolen(&isc->trim_queue)); in DB_SHOW_COMMAND()
2117 db_printf("read_bias: %d\n", isc->read_bias); in DB_SHOW_COMMAND()
2118 db_printf("current_read_bias: %d\n", isc->current_read_bias); in DB_SHOW_COMMAND()
2120 (isc->flags & CAM_IOSCHED_FLAG_TRIM_ACTIVE) ? "yes" : "no"); in DB_SHOW_COMMAND()