356 cam_iosched_limiter_init(struct iop_stats *ios)  in cam_iosched_limiter_init()  argument
358 	int lim = ios->limiter;  in cam_iosched_limiter_init()
365 		return limsw[lim].l_init(ios);  in cam_iosched_limiter_init()
371 cam_iosched_limiter_tick(struct iop_stats *ios)  in cam_iosched_limiter_tick()  argument
373 	int lim = ios->limiter;  in cam_iosched_limiter_tick()
380 		return limsw[lim].l_tick(ios);  in cam_iosched_limiter_tick()
386 cam_iosched_limiter_iop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_limiter_iop()  argument
388 	int lim = ios->limiter;  in cam_iosched_limiter_iop()
395 		return limsw[lim].l_iop(ios, bp);  in cam_iosched_limiter_iop()
401 cam_iosched_limiter_caniop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_limiter_caniop()  argument
403 	int lim = ios->limiter;  in cam_iosched_limiter_caniop()
410 		return limsw[lim].l_caniop(ios, bp);  in cam_iosched_limiter_caniop()
416 cam_iosched_limiter_iodone(struct iop_stats *ios, struct bio *bp)  in cam_iosched_limiter_iodone()  argument
418 	int lim = ios->limiter;  in cam_iosched_limiter_iodone()
425 		return limsw[lim].l_iodone(ios, bp);  in cam_iosched_limiter_iodone()
435 cam_iosched_qd_iop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_qd_iop()  argument
438 	if (ios->current <= 0 || ios->pending < ios->current)  in cam_iosched_qd_iop()
445 cam_iosched_qd_caniop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_qd_caniop()  argument
448 	if (ios->current <= 0 || ios->pending < ios->current)  in cam_iosched_qd_caniop()
455 cam_iosched_qd_iodone(struct iop_stats *ios, struct bio *bp)  in cam_iosched_qd_iodone()  argument
458 	if (ios->current <= 0 || ios->pending != ios->current)  in cam_iosched_qd_iodone()
465 cam_iosched_iops_init(struct iop_stats *ios)  in cam_iosched_iops_init()  argument
468 	ios->l_value1 = ios->current / ios->softc->quanta;  in cam_iosched_iops_init()
469 	if (ios->l_value1 <= 0)  in cam_iosched_iops_init()
470 		ios->l_value1 = 1;  in cam_iosched_iops_init()
471 	ios->l_value2 = 0;  in cam_iosched_iops_init()
477 cam_iosched_iops_tick(struct iop_stats *ios)  in cam_iosched_iops_tick()  argument
483 	 * the IOs for this interval have been spent.  in cam_iosched_iops_tick()
485 	new_ios = (int)((ios->current * (uint64_t)ios->softc->this_frac) >> 16);  in cam_iosched_iops_tick()
486 	if (new_ios < 1 && ios->l_value2 < ios->current) {  in cam_iosched_iops_tick()
488 		ios->l_value2++;  in cam_iosched_iops_tick()
492 	 * If this a new accounting interval, discard any "unspent" ios  in cam_iosched_iops_tick()
493 	 * granted in the previous interval.  Otherwise add the new ios to  in cam_iosched_iops_tick()
496 	if ((ios->softc->total_ticks % ios->softc->quanta) == 0) {  in cam_iosched_iops_tick()
497 		ios->l_value1 = new_ios;  in cam_iosched_iops_tick()
498 		ios->l_value2 = 1;  in cam_iosched_iops_tick()
500 		ios->l_value1 += new_ios;  in cam_iosched_iops_tick()
507 cam_iosched_iops_caniop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_iops_caniop()  argument
515 	if (ios->current > 0 && ios->l_value1 <= 0)  in cam_iosched_iops_caniop()
521 cam_iosched_iops_iop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_iops_iop()  argument
525 	rv = cam_iosched_limiter_caniop(ios, bp);  in cam_iosched_iops_iop()
527 		ios->l_value1--;  in cam_iosched_iops_iop()
533 cam_iosched_bw_init(struct iop_stats *ios)  in cam_iosched_bw_init()  argument
536 	/* ios->current is in kB/s, so scale to bytes */  in cam_iosched_bw_init()
537 	ios->l_value1 = ios->current * 1000 / ios->softc->quanta;  in cam_iosched_bw_init()
543 cam_iosched_bw_tick(struct iop_stats *ios)  in cam_iosched_bw_tick()  argument
551 	 * then too bad, that's lost. Also, ios->current  in cam_iosched_bw_tick()
558 	bw = (int)((ios->current * 1000ull * (uint64_t)ios->softc->this_frac) >> 16);  in cam_iosched_bw_tick()
559 	if (ios->l_value1 < bw * 4)  in cam_iosched_bw_tick()
560 		ios->l_value1 += bw;  in cam_iosched_bw_tick()
566 cam_iosched_bw_caniop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_bw_caniop()  argument
585 	if (ios->current > 0 && ios->l_value1 <= 0)  in cam_iosched_bw_caniop()
592 cam_iosched_bw_iop(struct iop_stats *ios, struct bio *bp)  in cam_iosched_bw_iop()  argument
596 	rv = cam_iosched_limiter_caniop(ios, bp);  in cam_iosched_bw_iop()
598 		ios->l_value1 -= bp->bio_length;  in cam_iosched_bw_iop()
852 cam_iosched_iop_stats_init(struct cam_iosched_softc *isc, struct iop_stats *ios)  in cam_iosched_iop_stats_init()  argument
855 	ios->limiter = none;  in cam_iosched_iop_stats_init()
856 	ios->in = 0;  in cam_iosched_iop_stats_init()
857 	ios->max = ios->current = 300000;  in cam_iosched_iop_stats_init()
858 	ios->min = 1;  in cam_iosched_iop_stats_init()
859 	ios->out = 0;  in cam_iosched_iop_stats_init()
860 	ios->errs = 0;  in cam_iosched_iop_stats_init()
861 	ios->pending = 0;  in cam_iosched_iop_stats_init()
862 	ios->queued = 0;  in cam_iosched_iop_stats_init()
863 	ios->total = 0;  in cam_iosched_iop_stats_init()
864 	ios->ema = 0;  in cam_iosched_iop_stats_init()
865 	ios->emvar = 0;  in cam_iosched_iop_stats_init()
866 	ios->bad_latency = SBT_1S / 2;	/* Default to 500ms */  in cam_iosched_iop_stats_init()
867 	ios->softc = isc;  in cam_iosched_iop_stats_init()
868 	cam_iosched_limiter_init(ios);  in cam_iosched_iop_stats_init()
875 	struct iop_stats *ios;  in cam_iosched_limiter_sysctl()  local
880 	ios = arg1;  in cam_iosched_limiter_sysctl()
881 	isc = ios->softc;  in cam_iosched_limiter_sysctl()
882 	value = ios->limiter;  in cam_iosched_limiter_sysctl()
898 		ios->limiter = i;  in cam_iosched_limiter_sysctl()
899 		error = cam_iosched_limiter_init(ios);  in cam_iosched_limiter_sysctl()
901 			ios->limiter = value;  in cam_iosched_limiter_sysctl()
1013 cam_iosched_iop_stats_sysctl_init(struct cam_iosched_softc *isc, struct iop_stats *ios, char *name)  in cam_iosched_iop_stats_sysctl_init()  argument
1018 	ios->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx,  in cam_iosched_iop_stats_sysctl_init()
1021 	n = SYSCTL_CHILDREN(ios->sysctl_tree);  in cam_iosched_iop_stats_sysctl_init()
1022 	ctx = &ios->sysctl_ctx;  in cam_iosched_iop_stats_sysctl_init()
1026 	    &ios->ema,  in cam_iosched_iop_stats_sysctl_init()
1030 	    &ios->emvar,  in cam_iosched_iop_stats_sysctl_init()
1035 	    &ios->pending, 0,  in cam_iosched_iop_stats_sysctl_init()
1039 	    &ios->total, 0,  in cam_iosched_iop_stats_sysctl_init()
1043 	    &ios->queued, 0,  in cam_iosched_iop_stats_sysctl_init()
1047 	    &ios->in, 0,  in cam_iosched_iop_stats_sysctl_init()
1051 	    &ios->out, 0,  in cam_iosched_iop_stats_sysctl_init()
1055 	    &ios->errs, 0,  in cam_iosched_iop_stats_sysctl_init()
1059 	    &ios->too_long, 0,  in cam_iosched_iop_stats_sysctl_init()
1064 	    &ios->bad_latency, 0, cam_iosched_sbintime_sysctl, "A",  in cam_iosched_iop_stats_sysctl_init()
1070 	    ios, 0, cam_iosched_limiter_sysctl, "A",  in cam_iosched_iop_stats_sysctl_init()
1074 	    &ios->min, 0,  in cam_iosched_iop_stats_sysctl_init()
1078 	    &ios->max, 0,  in cam_iosched_iop_stats_sysctl_init()
1082 	    &ios->current, 0,  in cam_iosched_iop_stats_sysctl_init()
1088 	    &ios->latencies, 0,  in cam_iosched_iop_stats_sysctl_init()
1099 cam_iosched_iop_stats_fini(struct iop_stats *ios)  in cam_iosched_iop_stats_fini()  argument
1101 	if (ios->sysctl_tree)  in cam_iosched_iop_stats_fini()
1102 		if (sysctl_ctx_free(&ios->sysctl_ctx) != 0)  in cam_iosched_iop_stats_fini()
1542 cam_iosched_rate_limited(struct iop_stats *ios)  in cam_iosched_rate_limited()  argument
1544 	return ios->state_flags & IOP_RATE_LIMITED;  in cam_iosched_rate_limited()