1ba6c22ceSWarner Losh /*- 2ba6c22ceSWarner Losh * CAM IO Scheduler Interface 3ba6c22ceSWarner Losh * 4ba6c22ceSWarner Losh * Copyright (c) 2015 Netflix, Inc. 5ba6c22ceSWarner Losh * All rights reserved. 6ba6c22ceSWarner Losh * 7ba6c22ceSWarner Losh * Redistribution and use in source and binary forms, with or without 8ba6c22ceSWarner Losh * modification, are permitted provided that the following conditions 9ba6c22ceSWarner Losh * are met: 10ba6c22ceSWarner Losh * 1. Redistributions of source code must retain the above copyright 11ba6c22ceSWarner Losh * notice, this list of conditions, and the following disclaimer, 12ba6c22ceSWarner Losh * without modification, immediately at the beginning of the file. 13ba6c22ceSWarner Losh * 2. The name of the author may not be used to endorse or promote products 14ba6c22ceSWarner Losh * derived from this software without specific prior written permission. 15ba6c22ceSWarner Losh * 16ba6c22ceSWarner Losh * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17ba6c22ceSWarner Losh * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18ba6c22ceSWarner Losh * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19ba6c22ceSWarner Losh * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 20ba6c22ceSWarner Losh * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21ba6c22ceSWarner Losh * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22ba6c22ceSWarner Losh * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23ba6c22ceSWarner Losh * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24ba6c22ceSWarner Losh * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25ba6c22ceSWarner Losh * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26ba6c22ceSWarner Losh * SUCH DAMAGE. 27ba6c22ceSWarner Losh * 28ba6c22ceSWarner Losh * $FreeBSD$ 29ba6c22ceSWarner Losh */ 30ba6c22ceSWarner Losh 31ba6c22ceSWarner Losh #include "opt_cam.h" 32ba6c22ceSWarner Losh #include "opt_ddb.h" 33ba6c22ceSWarner Losh 34ba6c22ceSWarner Losh #include <sys/cdefs.h> 35ba6c22ceSWarner Losh __FBSDID("$FreeBSD$"); 36ba6c22ceSWarner Losh 37ba6c22ceSWarner Losh #include <sys/param.h> 38ba6c22ceSWarner Losh 39ba6c22ceSWarner Losh #include <sys/systm.h> 40ba6c22ceSWarner Losh #include <sys/kernel.h> 41ba6c22ceSWarner Losh #include <sys/bio.h> 42ba6c22ceSWarner Losh #include <sys/lock.h> 43ba6c22ceSWarner Losh #include <sys/malloc.h> 44ba6c22ceSWarner Losh #include <sys/mutex.h> 45cf3ec151SWarner Losh #include <sys/sbuf.h> 46ba6c22ceSWarner Losh #include <sys/sysctl.h> 47ba6c22ceSWarner Losh 48ba6c22ceSWarner Losh #include <cam/cam.h> 49ba6c22ceSWarner Losh #include <cam/cam_ccb.h> 50ba6c22ceSWarner Losh #include <cam/cam_periph.h> 51ba6c22ceSWarner Losh #include <cam/cam_xpt_periph.h> 52cf3ec151SWarner Losh #include <cam/cam_xpt_internal.h> 53ba6c22ceSWarner Losh #include <cam/cam_iosched.h> 54ba6c22ceSWarner Losh 55ba6c22ceSWarner Losh #include <ddb/ddb.h> 56ba6c22ceSWarner Losh 57ba6c22ceSWarner Losh static MALLOC_DEFINE(M_CAMSCHED, "CAM I/O Scheduler", 58ba6c22ceSWarner Losh "CAM I/O Scheduler buffers"); 59ba6c22ceSWarner Losh 60ba6c22ceSWarner Losh /* 61ba6c22ceSWarner Losh * Default I/O scheduler for FreeBSD. This implementation is just a thin-vineer 62ba6c22ceSWarner Losh * over the bioq_* interface, with notions of separate calls for normal I/O and 63ba6c22ceSWarner Losh * for trims. 64035ec48eSWarner Losh * 65035ec48eSWarner Losh * When CAM_IOSCHED_DYNAMIC is defined, the scheduler is enhanced to dynamically 66035ec48eSWarner Losh * steer the rate of one type of traffic to help other types of traffic (eg 67035ec48eSWarner Losh * limit writes when read latency deteriorates on SSDs). 68ba6c22ceSWarner Losh */ 69ba6c22ceSWarner Losh 70df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 71ba6c22ceSWarner Losh 72035ec48eSWarner Losh static int do_dynamic_iosched = 1; 73035ec48eSWarner Losh TUNABLE_INT("kern.cam.do_dynamic_iosched", &do_dynamic_iosched); 74035ec48eSWarner Losh SYSCTL_INT(_kern_cam, OID_AUTO, do_dynamic_iosched, CTLFLAG_RD, 75035ec48eSWarner Losh &do_dynamic_iosched, 1, 76035ec48eSWarner Losh "Enable Dynamic I/O scheduler optimizations."); 77ba6c22ceSWarner Losh 78cf3ec151SWarner Losh /* 79cf3ec151SWarner Losh * For an EMA, with an alpha of alpha, we know 80cf3ec151SWarner Losh * alpha = 2 / (N + 1) 81cf3ec151SWarner Losh * or 82cf3ec151SWarner Losh * N = 1 + (2 / alpha) 83cf3ec151SWarner Losh * where N is the number of samples that 86% of the current 84cf3ec151SWarner Losh * EMA is derived from. 85cf3ec151SWarner Losh * 86cf3ec151SWarner Losh * So we invent[*] alpha_bits: 87cf3ec151SWarner Losh * alpha_bits = -log_2(alpha) 88cf3ec151SWarner Losh * alpha = 2^-alpha_bits 89cf3ec151SWarner Losh * So 90cf3ec151SWarner Losh * N = 1 + 2^(alpha_bits + 1) 91cf3ec151SWarner Losh * 92cf3ec151SWarner Losh * The default 9 gives a 1025 lookback for 86% of the data. 93cf3ec151SWarner Losh * For a brief intro: https://en.wikipedia.org/wiki/Moving_average 94cf3ec151SWarner Losh * 95cf3ec151SWarner Losh * [*] Steal from the load average code and many other places. 9679d80af2SWarner Losh * Note: See computation of EMA and EMVAR for acceptable ranges of alpha. 97cf3ec151SWarner Losh */ 98ba6c22ceSWarner Losh static int alpha_bits = 9; 99ba6c22ceSWarner Losh TUNABLE_INT("kern.cam.iosched_alpha_bits", &alpha_bits); 100ba6c22ceSWarner Losh SYSCTL_INT(_kern_cam, OID_AUTO, iosched_alpha_bits, CTLFLAG_RW, 101ba6c22ceSWarner Losh &alpha_bits, 1, 102ba6c22ceSWarner Losh "Bits in EMA's alpha."); 103ba6c22ceSWarner Losh 104ba6c22ceSWarner Losh struct iop_stats; 105ba6c22ceSWarner Losh struct cam_iosched_softc; 106ba6c22ceSWarner Losh 107ba6c22ceSWarner Losh int iosched_debug = 0; 108ba6c22ceSWarner Losh 109ba6c22ceSWarner Losh typedef enum { 110ba6c22ceSWarner Losh none = 0, /* No limits */ 111ba6c22ceSWarner Losh queue_depth, /* Limit how many ops we queue to SIM */ 112ba6c22ceSWarner Losh iops, /* Limit # of IOPS to the drive */ 113ba6c22ceSWarner Losh bandwidth, /* Limit bandwidth to the drive */ 114ba6c22ceSWarner Losh limiter_max 115ba6c22ceSWarner Losh } io_limiter; 116ba6c22ceSWarner Losh 117ba6c22ceSWarner Losh static const char *cam_iosched_limiter_names[] = 118ba6c22ceSWarner Losh { "none", "queue_depth", "iops", "bandwidth" }; 119ba6c22ceSWarner Losh 120ba6c22ceSWarner Losh /* 121ba6c22ceSWarner Losh * Called to initialize the bits of the iop_stats structure relevant to the 122ba6c22ceSWarner Losh * limiter. Called just after the limiter is set. 123ba6c22ceSWarner Losh */ 124ba6c22ceSWarner Losh typedef int l_init_t(struct iop_stats *); 125ba6c22ceSWarner Losh 126ba6c22ceSWarner Losh /* 127ba6c22ceSWarner Losh * Called every tick. 128ba6c22ceSWarner Losh */ 129ba6c22ceSWarner Losh typedef int l_tick_t(struct iop_stats *); 130ba6c22ceSWarner Losh 131ba6c22ceSWarner Losh /* 132ba6c22ceSWarner Losh * Called to see if the limiter thinks this IOP can be allowed to 1330b4060b0SEd Maste * proceed. If so, the limiter assumes that the IOP proceeded 134ba6c22ceSWarner Losh * and makes any accounting of it that's needed. 135ba6c22ceSWarner Losh */ 136ba6c22ceSWarner Losh typedef int l_iop_t(struct iop_stats *, struct bio *); 137ba6c22ceSWarner Losh 138ba6c22ceSWarner Losh /* 1390b4060b0SEd Maste * Called when an I/O completes so the limiter can update its 140ba6c22ceSWarner Losh * accounting. Pending I/Os may complete in any order (even when 141ba6c22ceSWarner Losh * sent to the hardware at the same time), so the limiter may not 142ba6c22ceSWarner Losh * make any assumptions other than this I/O has completed. If it 143ba6c22ceSWarner Losh * returns 1, then xpt_schedule() needs to be called again. 144ba6c22ceSWarner Losh */ 145ba6c22ceSWarner Losh typedef int l_iodone_t(struct iop_stats *, struct bio *); 146ba6c22ceSWarner Losh 147ba6c22ceSWarner Losh static l_iop_t cam_iosched_qd_iop; 148ba6c22ceSWarner Losh static l_iop_t cam_iosched_qd_caniop; 149ba6c22ceSWarner Losh static l_iodone_t cam_iosched_qd_iodone; 150ba6c22ceSWarner Losh 151ba6c22ceSWarner Losh static l_init_t cam_iosched_iops_init; 152ba6c22ceSWarner Losh static l_tick_t cam_iosched_iops_tick; 153ba6c22ceSWarner Losh static l_iop_t cam_iosched_iops_caniop; 154ba6c22ceSWarner Losh static l_iop_t cam_iosched_iops_iop; 155ba6c22ceSWarner Losh 156ba6c22ceSWarner Losh static l_init_t cam_iosched_bw_init; 157ba6c22ceSWarner Losh static l_tick_t cam_iosched_bw_tick; 158ba6c22ceSWarner Losh static l_iop_t cam_iosched_bw_caniop; 159ba6c22ceSWarner Losh static l_iop_t cam_iosched_bw_iop; 160ba6c22ceSWarner Losh 161b20c0a07SWarner Losh struct limswitch { 162ba6c22ceSWarner Losh l_init_t *l_init; 163ba6c22ceSWarner Losh l_tick_t *l_tick; 164ba6c22ceSWarner Losh l_iop_t *l_iop; 165ba6c22ceSWarner Losh l_iop_t *l_caniop; 166ba6c22ceSWarner Losh l_iodone_t *l_iodone; 167ba6c22ceSWarner Losh } limsw[] = 168ba6c22ceSWarner Losh { 169ba6c22ceSWarner Losh { /* none */ 170ba6c22ceSWarner Losh .l_init = NULL, 171ba6c22ceSWarner Losh .l_tick = NULL, 172ba6c22ceSWarner Losh .l_iop = NULL, 173ba6c22ceSWarner Losh .l_iodone= NULL, 174ba6c22ceSWarner Losh }, 175ba6c22ceSWarner Losh { /* queue_depth */ 176ba6c22ceSWarner Losh .l_init = NULL, 177ba6c22ceSWarner Losh .l_tick = NULL, 178ba6c22ceSWarner Losh .l_caniop = cam_iosched_qd_caniop, 179ba6c22ceSWarner Losh .l_iop = cam_iosched_qd_iop, 180ba6c22ceSWarner Losh .l_iodone= cam_iosched_qd_iodone, 181ba6c22ceSWarner Losh }, 182ba6c22ceSWarner Losh { /* iops */ 183ba6c22ceSWarner Losh .l_init = cam_iosched_iops_init, 184ba6c22ceSWarner Losh .l_tick = cam_iosched_iops_tick, 185ba6c22ceSWarner Losh .l_caniop = cam_iosched_iops_caniop, 186ba6c22ceSWarner Losh .l_iop = cam_iosched_iops_iop, 187ba6c22ceSWarner Losh .l_iodone= NULL, 188ba6c22ceSWarner Losh }, 189ba6c22ceSWarner Losh { /* bandwidth */ 190ba6c22ceSWarner Losh .l_init = cam_iosched_bw_init, 191ba6c22ceSWarner Losh .l_tick = cam_iosched_bw_tick, 192ba6c22ceSWarner Losh .l_caniop = cam_iosched_bw_caniop, 193ba6c22ceSWarner Losh .l_iop = cam_iosched_bw_iop, 194ba6c22ceSWarner Losh .l_iodone= NULL, 195ba6c22ceSWarner Losh }, 196ba6c22ceSWarner Losh }; 197ba6c22ceSWarner Losh 198b20c0a07SWarner Losh struct iop_stats { 199ba6c22ceSWarner Losh /* 200ba6c22ceSWarner Losh * sysctl state for this subnode. 201ba6c22ceSWarner Losh */ 202ba6c22ceSWarner Losh struct sysctl_ctx_list sysctl_ctx; 203ba6c22ceSWarner Losh struct sysctl_oid *sysctl_tree; 204ba6c22ceSWarner Losh 205ba6c22ceSWarner Losh /* 206ba6c22ceSWarner Losh * Information about the current rate limiters, if any 207ba6c22ceSWarner Losh */ 208ba6c22ceSWarner Losh io_limiter limiter; /* How are I/Os being limited */ 209ba6c22ceSWarner Losh int min; /* Low range of limit */ 210ba6c22ceSWarner Losh int max; /* High range of limit */ 211ba6c22ceSWarner Losh int current; /* Current rate limiter */ 212ba6c22ceSWarner Losh int l_value1; /* per-limiter scratch value 1. */ 213ba6c22ceSWarner Losh int l_value2; /* per-limiter scratch value 2. */ 214ba6c22ceSWarner Losh 215ba6c22ceSWarner Losh /* 216ba6c22ceSWarner Losh * Debug information about counts of I/Os that have gone through the 217ba6c22ceSWarner Losh * scheduler. 218ba6c22ceSWarner Losh */ 219ba6c22ceSWarner Losh int pending; /* I/Os pending in the hardware */ 220ba6c22ceSWarner Losh int queued; /* number currently in the queue */ 221ba6c22ceSWarner Losh int total; /* Total for all time -- wraps */ 222ba6c22ceSWarner Losh int in; /* number queued all time -- wraps */ 223ba6c22ceSWarner Losh int out; /* number completed all time -- wraps */ 224ba6c22ceSWarner Losh 225ba6c22ceSWarner Losh /* 226ba6c22ceSWarner Losh * Statistics on different bits of the process. 227ba6c22ceSWarner Losh */ 228cf3ec151SWarner Losh /* Exp Moving Average, see alpha_bits for more details */ 229ba6c22ceSWarner Losh sbintime_t ema; 23079d80af2SWarner Losh sbintime_t emvar; 231ba6c22ceSWarner Losh sbintime_t sd; /* Last computed sd */ 232ba6c22ceSWarner Losh 233cf3ec151SWarner Losh uint32_t state_flags; 234cf3ec151SWarner Losh #define IOP_RATE_LIMITED 1u 235cf3ec151SWarner Losh 236*08fc2f23SWarner Losh #define LAT_BUCKETS 15 /* < 1ms < 2ms ... < 2^(n-1)ms >= 2^(n-1)ms*/ 237cf3ec151SWarner Losh uint64_t latencies[LAT_BUCKETS]; 238cf3ec151SWarner Losh 239ba6c22ceSWarner Losh struct cam_iosched_softc *softc; 240ba6c22ceSWarner Losh }; 241ba6c22ceSWarner Losh 242ba6c22ceSWarner Losh 243ba6c22ceSWarner Losh typedef enum { 244ba6c22ceSWarner Losh set_max = 0, /* current = max */ 245ba6c22ceSWarner Losh read_latency, /* Steer read latency by throttling writes */ 246ba6c22ceSWarner Losh cl_max /* Keep last */ 247ba6c22ceSWarner Losh } control_type; 248ba6c22ceSWarner Losh 249ba6c22ceSWarner Losh static const char *cam_iosched_control_type_names[] = 250ba6c22ceSWarner Losh { "set_max", "read_latency" }; 251ba6c22ceSWarner Losh 252b20c0a07SWarner Losh struct control_loop { 253ba6c22ceSWarner Losh /* 254ba6c22ceSWarner Losh * sysctl state for this subnode. 255ba6c22ceSWarner Losh */ 256ba6c22ceSWarner Losh struct sysctl_ctx_list sysctl_ctx; 257ba6c22ceSWarner Losh struct sysctl_oid *sysctl_tree; 258ba6c22ceSWarner Losh 259ba6c22ceSWarner Losh sbintime_t next_steer; /* Time of next steer */ 260ba6c22ceSWarner Losh sbintime_t steer_interval; /* How often do we steer? */ 261ba6c22ceSWarner Losh sbintime_t lolat; 262ba6c22ceSWarner Losh sbintime_t hilat; 263ba6c22ceSWarner Losh int alpha; 264ba6c22ceSWarner Losh control_type type; /* What type of control? */ 265ba6c22ceSWarner Losh int last_count; /* Last I/O count */ 266ba6c22ceSWarner Losh 267ba6c22ceSWarner Losh struct cam_iosched_softc *softc; 268ba6c22ceSWarner Losh }; 269ba6c22ceSWarner Losh 270ba6c22ceSWarner Losh #endif 271ba6c22ceSWarner Losh 272b20c0a07SWarner Losh struct cam_iosched_softc { 273ba6c22ceSWarner Losh struct bio_queue_head bio_queue; 274ba6c22ceSWarner Losh struct bio_queue_head trim_queue; 275ba6c22ceSWarner Losh /* scheduler flags < 16, user flags >= 16 */ 276ba6c22ceSWarner Losh uint32_t flags; 277ba6c22ceSWarner Losh int sort_io_queue; 278df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 279ba6c22ceSWarner Losh int read_bias; /* Read bias setting */ 280ba6c22ceSWarner Losh int current_read_bias; /* Current read bias state */ 281ba6c22ceSWarner Losh int total_ticks; 282cf3ec151SWarner Losh int load; /* EMA of 'load average' of disk / 2^16 */ 283ba6c22ceSWarner Losh 284ba6c22ceSWarner Losh struct bio_queue_head write_queue; 285ba6c22ceSWarner Losh struct iop_stats read_stats, write_stats, trim_stats; 286ba6c22ceSWarner Losh struct sysctl_ctx_list sysctl_ctx; 287ba6c22ceSWarner Losh struct sysctl_oid *sysctl_tree; 288ba6c22ceSWarner Losh 289ba6c22ceSWarner Losh int quanta; /* Number of quanta per second */ 290ba6c22ceSWarner Losh struct callout ticker; /* Callout for our quota system */ 291ba6c22ceSWarner Losh struct cam_periph *periph; /* cam periph associated with this device */ 292ba6c22ceSWarner Losh uint32_t this_frac; /* Fraction of a second (1024ths) for this tick */ 293ba6c22ceSWarner Losh sbintime_t last_time; /* Last time we ticked */ 294ba6c22ceSWarner Losh struct control_loop cl; 295ba6c22ceSWarner Losh #endif 296ba6c22ceSWarner Losh }; 297ba6c22ceSWarner Losh 298df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 299ba6c22ceSWarner Losh /* 300ba6c22ceSWarner Losh * helper functions to call the limsw functions. 301ba6c22ceSWarner Losh */ 302ba6c22ceSWarner Losh static int 303ba6c22ceSWarner Losh cam_iosched_limiter_init(struct iop_stats *ios) 304ba6c22ceSWarner Losh { 305ba6c22ceSWarner Losh int lim = ios->limiter; 306ba6c22ceSWarner Losh 307ba6c22ceSWarner Losh /* maybe this should be a kassert */ 308ba6c22ceSWarner Losh if (lim < none || lim >= limiter_max) 309ba6c22ceSWarner Losh return EINVAL; 310ba6c22ceSWarner Losh 311ba6c22ceSWarner Losh if (limsw[lim].l_init) 312ba6c22ceSWarner Losh return limsw[lim].l_init(ios); 313ba6c22ceSWarner Losh 314ba6c22ceSWarner Losh return 0; 315ba6c22ceSWarner Losh } 316ba6c22ceSWarner Losh 317ba6c22ceSWarner Losh static int 318ba6c22ceSWarner Losh cam_iosched_limiter_tick(struct iop_stats *ios) 319ba6c22ceSWarner Losh { 320ba6c22ceSWarner Losh int lim = ios->limiter; 321ba6c22ceSWarner Losh 322ba6c22ceSWarner Losh /* maybe this should be a kassert */ 323ba6c22ceSWarner Losh if (lim < none || lim >= limiter_max) 324ba6c22ceSWarner Losh return EINVAL; 325ba6c22ceSWarner Losh 326ba6c22ceSWarner Losh if (limsw[lim].l_tick) 327ba6c22ceSWarner Losh return limsw[lim].l_tick(ios); 328ba6c22ceSWarner Losh 329ba6c22ceSWarner Losh return 0; 330ba6c22ceSWarner Losh } 331ba6c22ceSWarner Losh 332ba6c22ceSWarner Losh static int 333ba6c22ceSWarner Losh cam_iosched_limiter_iop(struct iop_stats *ios, struct bio *bp) 334ba6c22ceSWarner Losh { 335ba6c22ceSWarner Losh int lim = ios->limiter; 336ba6c22ceSWarner Losh 337ba6c22ceSWarner Losh /* maybe this should be a kassert */ 338ba6c22ceSWarner Losh if (lim < none || lim >= limiter_max) 339ba6c22ceSWarner Losh return EINVAL; 340ba6c22ceSWarner Losh 341ba6c22ceSWarner Losh if (limsw[lim].l_iop) 342ba6c22ceSWarner Losh return limsw[lim].l_iop(ios, bp); 343ba6c22ceSWarner Losh 344ba6c22ceSWarner Losh return 0; 345ba6c22ceSWarner Losh } 346ba6c22ceSWarner Losh 347ba6c22ceSWarner Losh static int 348ba6c22ceSWarner Losh cam_iosched_limiter_caniop(struct iop_stats *ios, struct bio *bp) 349ba6c22ceSWarner Losh { 350ba6c22ceSWarner Losh int lim = ios->limiter; 351ba6c22ceSWarner Losh 352ba6c22ceSWarner Losh /* maybe this should be a kassert */ 353ba6c22ceSWarner Losh if (lim < none || lim >= limiter_max) 354ba6c22ceSWarner Losh return EINVAL; 355ba6c22ceSWarner Losh 356ba6c22ceSWarner Losh if (limsw[lim].l_caniop) 357ba6c22ceSWarner Losh return limsw[lim].l_caniop(ios, bp); 358ba6c22ceSWarner Losh 359ba6c22ceSWarner Losh return 0; 360ba6c22ceSWarner Losh } 361ba6c22ceSWarner Losh 362ba6c22ceSWarner Losh static int 363ba6c22ceSWarner Losh cam_iosched_limiter_iodone(struct iop_stats *ios, struct bio *bp) 364ba6c22ceSWarner Losh { 365ba6c22ceSWarner Losh int lim = ios->limiter; 366ba6c22ceSWarner Losh 367ba6c22ceSWarner Losh /* maybe this should be a kassert */ 368ba6c22ceSWarner Losh if (lim < none || lim >= limiter_max) 369ba6c22ceSWarner Losh return 0; 370ba6c22ceSWarner Losh 371ba6c22ceSWarner Losh if (limsw[lim].l_iodone) 372ba6c22ceSWarner Losh return limsw[lim].l_iodone(ios, bp); 373ba6c22ceSWarner Losh 374ba6c22ceSWarner Losh return 0; 375ba6c22ceSWarner Losh } 376ba6c22ceSWarner Losh 377ba6c22ceSWarner Losh /* 378ba6c22ceSWarner Losh * Functions to implement the different kinds of limiters 379ba6c22ceSWarner Losh */ 380ba6c22ceSWarner Losh 381ba6c22ceSWarner Losh static int 382ba6c22ceSWarner Losh cam_iosched_qd_iop(struct iop_stats *ios, struct bio *bp) 383ba6c22ceSWarner Losh { 384ba6c22ceSWarner Losh 385ba6c22ceSWarner Losh if (ios->current <= 0 || ios->pending < ios->current) 386ba6c22ceSWarner Losh return 0; 387ba6c22ceSWarner Losh 388ba6c22ceSWarner Losh return EAGAIN; 389ba6c22ceSWarner Losh } 390ba6c22ceSWarner Losh 391ba6c22ceSWarner Losh static int 392ba6c22ceSWarner Losh cam_iosched_qd_caniop(struct iop_stats *ios, struct bio *bp) 393ba6c22ceSWarner Losh { 394ba6c22ceSWarner Losh 395ba6c22ceSWarner Losh if (ios->current <= 0 || ios->pending < ios->current) 396ba6c22ceSWarner Losh return 0; 397ba6c22ceSWarner Losh 398ba6c22ceSWarner Losh return EAGAIN; 399ba6c22ceSWarner Losh } 400ba6c22ceSWarner Losh 401ba6c22ceSWarner Losh static int 402ba6c22ceSWarner Losh cam_iosched_qd_iodone(struct iop_stats *ios, struct bio *bp) 403ba6c22ceSWarner Losh { 404ba6c22ceSWarner Losh 405ba6c22ceSWarner Losh if (ios->current <= 0 || ios->pending != ios->current) 406ba6c22ceSWarner Losh return 0; 407ba6c22ceSWarner Losh 408ba6c22ceSWarner Losh return 1; 409ba6c22ceSWarner Losh } 410ba6c22ceSWarner Losh 411ba6c22ceSWarner Losh static int 412ba6c22ceSWarner Losh cam_iosched_iops_init(struct iop_stats *ios) 413ba6c22ceSWarner Losh { 414ba6c22ceSWarner Losh 415ba6c22ceSWarner Losh ios->l_value1 = ios->current / ios->softc->quanta; 416ba6c22ceSWarner Losh if (ios->l_value1 <= 0) 417ba6c22ceSWarner Losh ios->l_value1 = 1; 418ba6c22ceSWarner Losh 419ba6c22ceSWarner Losh return 0; 420ba6c22ceSWarner Losh } 421ba6c22ceSWarner Losh 422ba6c22ceSWarner Losh static int 423ba6c22ceSWarner Losh cam_iosched_iops_tick(struct iop_stats *ios) 424ba6c22ceSWarner Losh { 425ba6c22ceSWarner Losh 426ba6c22ceSWarner Losh ios->l_value1 = (int)((ios->current * (uint64_t)ios->softc->this_frac) >> 16); 427ba6c22ceSWarner Losh if (ios->l_value1 <= 0) 428ba6c22ceSWarner Losh ios->l_value1 = 1; 429ba6c22ceSWarner Losh 430ba6c22ceSWarner Losh return 0; 431ba6c22ceSWarner Losh } 432ba6c22ceSWarner Losh 433ba6c22ceSWarner Losh static int 434ba6c22ceSWarner Losh cam_iosched_iops_caniop(struct iop_stats *ios, struct bio *bp) 435ba6c22ceSWarner Losh { 436ba6c22ceSWarner Losh 437ba6c22ceSWarner Losh /* 438ba6c22ceSWarner Losh * So if we have any more IOPs left, allow it, 439ba6c22ceSWarner Losh * otherwise wait. 440ba6c22ceSWarner Losh */ 441ba6c22ceSWarner Losh if (ios->l_value1 <= 0) 442ba6c22ceSWarner Losh return EAGAIN; 443ba6c22ceSWarner Losh return 0; 444ba6c22ceSWarner Losh } 445ba6c22ceSWarner Losh 446ba6c22ceSWarner Losh static int 447ba6c22ceSWarner Losh cam_iosched_iops_iop(struct iop_stats *ios, struct bio *bp) 448ba6c22ceSWarner Losh { 449ba6c22ceSWarner Losh int rv; 450ba6c22ceSWarner Losh 451ba6c22ceSWarner Losh rv = cam_iosched_limiter_caniop(ios, bp); 452ba6c22ceSWarner Losh if (rv == 0) 453ba6c22ceSWarner Losh ios->l_value1--; 454ba6c22ceSWarner Losh 455ba6c22ceSWarner Losh return rv; 456ba6c22ceSWarner Losh } 457ba6c22ceSWarner Losh 458ba6c22ceSWarner Losh static int 459ba6c22ceSWarner Losh cam_iosched_bw_init(struct iop_stats *ios) 460ba6c22ceSWarner Losh { 461ba6c22ceSWarner Losh 462ba6c22ceSWarner Losh /* ios->current is in kB/s, so scale to bytes */ 463ba6c22ceSWarner Losh ios->l_value1 = ios->current * 1000 / ios->softc->quanta; 464ba6c22ceSWarner Losh 465ba6c22ceSWarner Losh return 0; 466ba6c22ceSWarner Losh } 467ba6c22ceSWarner Losh 468ba6c22ceSWarner Losh static int 469ba6c22ceSWarner Losh cam_iosched_bw_tick(struct iop_stats *ios) 470ba6c22ceSWarner Losh { 471ba6c22ceSWarner Losh int bw; 472ba6c22ceSWarner Losh 473ba6c22ceSWarner Losh /* 474ba6c22ceSWarner Losh * If we're in the hole for available quota from 475ba6c22ceSWarner Losh * the last time, then add the quantum for this. 476ba6c22ceSWarner Losh * If we have any left over from last quantum, 477ba6c22ceSWarner Losh * then too bad, that's lost. Also, ios->current 478ba6c22ceSWarner Losh * is in kB/s, so scale. 479ba6c22ceSWarner Losh * 480ba6c22ceSWarner Losh * We also allow up to 4 quanta of credits to 481ba6c22ceSWarner Losh * accumulate to deal with burstiness. 4 is extremely 482ba6c22ceSWarner Losh * arbitrary. 483ba6c22ceSWarner Losh */ 484ba6c22ceSWarner Losh bw = (int)((ios->current * 1000ull * (uint64_t)ios->softc->this_frac) >> 16); 485ba6c22ceSWarner Losh if (ios->l_value1 < bw * 4) 486ba6c22ceSWarner Losh ios->l_value1 += bw; 487ba6c22ceSWarner Losh 488ba6c22ceSWarner Losh return 0; 489ba6c22ceSWarner Losh } 490ba6c22ceSWarner Losh 491ba6c22ceSWarner Losh static int 492ba6c22ceSWarner Losh cam_iosched_bw_caniop(struct iop_stats *ios, struct bio *bp) 493ba6c22ceSWarner Losh { 494ba6c22ceSWarner Losh /* 495ba6c22ceSWarner Losh * So if we have any more bw quota left, allow it, 4960b4060b0SEd Maste * otherwise wait. Note, we'll go negative and that's 4970b4060b0SEd Maste * OK. We'll just get a little less next quota. 498ba6c22ceSWarner Losh * 499ba6c22ceSWarner Losh * Note on going negative: that allows us to process 500ba6c22ceSWarner Losh * requests in order better, since we won't allow 501ba6c22ceSWarner Losh * shorter reads to get around the long one that we 502ba6c22ceSWarner Losh * don't have the quota to do just yet. It also prevents 503ba6c22ceSWarner Losh * starvation by being a little more permissive about 504ba6c22ceSWarner Losh * what we let through this quantum (to prevent the 505ba6c22ceSWarner Losh * starvation), at the cost of getting a little less 506ba6c22ceSWarner Losh * next quantum. 507ba6c22ceSWarner Losh */ 508ba6c22ceSWarner Losh if (ios->l_value1 <= 0) 509ba6c22ceSWarner Losh return EAGAIN; 510ba6c22ceSWarner Losh 511ba6c22ceSWarner Losh 512ba6c22ceSWarner Losh return 0; 513ba6c22ceSWarner Losh } 514ba6c22ceSWarner Losh 515ba6c22ceSWarner Losh static int 516ba6c22ceSWarner Losh cam_iosched_bw_iop(struct iop_stats *ios, struct bio *bp) 517ba6c22ceSWarner Losh { 518ba6c22ceSWarner Losh int rv; 519ba6c22ceSWarner Losh 520ba6c22ceSWarner Losh rv = cam_iosched_limiter_caniop(ios, bp); 521ba6c22ceSWarner Losh if (rv == 0) 522ba6c22ceSWarner Losh ios->l_value1 -= bp->bio_length; 523ba6c22ceSWarner Losh 524ba6c22ceSWarner Losh return rv; 525ba6c22ceSWarner Losh } 526ba6c22ceSWarner Losh 527ba6c22ceSWarner Losh static void cam_iosched_cl_maybe_steer(struct control_loop *clp); 528ba6c22ceSWarner Losh 529ba6c22ceSWarner Losh static void 530ba6c22ceSWarner Losh cam_iosched_ticker(void *arg) 531ba6c22ceSWarner Losh { 532ba6c22ceSWarner Losh struct cam_iosched_softc *isc = arg; 533ba6c22ceSWarner Losh sbintime_t now, delta; 534cf3ec151SWarner Losh int pending; 535ba6c22ceSWarner Losh 536ba6c22ceSWarner Losh callout_reset(&isc->ticker, hz / isc->quanta - 1, cam_iosched_ticker, isc); 537ba6c22ceSWarner Losh 538ba6c22ceSWarner Losh now = sbinuptime(); 539ba6c22ceSWarner Losh delta = now - isc->last_time; 540ba6c22ceSWarner Losh isc->this_frac = (uint32_t)delta >> 16; /* Note: discards seconds -- should be 0 harmless if not */ 541ba6c22ceSWarner Losh isc->last_time = now; 542ba6c22ceSWarner Losh 543ba6c22ceSWarner Losh cam_iosched_cl_maybe_steer(&isc->cl); 544ba6c22ceSWarner Losh 545ba6c22ceSWarner Losh cam_iosched_limiter_tick(&isc->read_stats); 546ba6c22ceSWarner Losh cam_iosched_limiter_tick(&isc->write_stats); 547ba6c22ceSWarner Losh cam_iosched_limiter_tick(&isc->trim_stats); 548ba6c22ceSWarner Losh 549ba6c22ceSWarner Losh cam_iosched_schedule(isc, isc->periph); 550ba6c22ceSWarner Losh 551cf3ec151SWarner Losh /* 552cf3ec151SWarner Losh * isc->load is an EMA of the pending I/Os at each tick. The number of 553cf3ec151SWarner Losh * pending I/Os is the sum of the I/Os queued to the hardware, and those 554cf3ec151SWarner Losh * in the software queue that could be queued to the hardware if there 555cf3ec151SWarner Losh * were slots. 556cf3ec151SWarner Losh * 557cf3ec151SWarner Losh * ios_stats.pending is a count of requests in the SIM right now for 558cf3ec151SWarner Losh * each of these types of I/O. So the total pending count is the sum of 559cf3ec151SWarner Losh * these I/Os and the sum of the queued I/Os still in the software queue 560cf3ec151SWarner Losh * for those operations that aren't being rate limited at the moment. 561cf3ec151SWarner Losh * 562cf3ec151SWarner Losh * The reason for the rate limiting bit is because those I/Os 563cf3ec151SWarner Losh * aren't part of the software queued load (since we could 564cf3ec151SWarner Losh * give them to hardware, but choose not to). 565cf3ec151SWarner Losh * 566cf3ec151SWarner Losh * Note: due to a bug in counting pending TRIM in the device, we 567cf3ec151SWarner Losh * don't include them in this count. We count each BIO_DELETE in 568cf3ec151SWarner Losh * the pending count, but the periph drivers collapse them down 569cf3ec151SWarner Losh * into one TRIM command. That one trim command gets the completion 570cf3ec151SWarner Losh * so the counts get off. 571cf3ec151SWarner Losh */ 572cf3ec151SWarner Losh pending = isc->read_stats.pending + isc->write_stats.pending /* + isc->trim_stats.pending */; 573cf3ec151SWarner Losh pending += !!(isc->read_stats.state_flags & IOP_RATE_LIMITED) * isc->read_stats.queued + 574cf3ec151SWarner Losh !!(isc->write_stats.state_flags & IOP_RATE_LIMITED) * isc->write_stats.queued /* + 575cf3ec151SWarner Losh !!(isc->trim_stats.state_flags & IOP_RATE_LIMITED) * isc->trim_stats.queued */ ; 576cf3ec151SWarner Losh pending <<= 16; 577cf3ec151SWarner Losh pending /= isc->periph->path->device->ccbq.total_openings; 578cf3ec151SWarner Losh 579cf3ec151SWarner Losh isc->load = (pending + (isc->load << 13) - isc->load) >> 13; /* see above: 13 -> 16139 / 200/s = ~81s ~1 minute */ 580cf3ec151SWarner Losh 581ba6c22ceSWarner Losh isc->total_ticks++; 582ba6c22ceSWarner Losh } 583ba6c22ceSWarner Losh 584ba6c22ceSWarner Losh 585ba6c22ceSWarner Losh static void 586ba6c22ceSWarner Losh cam_iosched_cl_init(struct control_loop *clp, struct cam_iosched_softc *isc) 587ba6c22ceSWarner Losh { 588ba6c22ceSWarner Losh 589ba6c22ceSWarner Losh clp->next_steer = sbinuptime(); 590ba6c22ceSWarner Losh clp->softc = isc; 591ba6c22ceSWarner Losh clp->steer_interval = SBT_1S * 5; /* Let's start out steering every 5s */ 592ba6c22ceSWarner Losh clp->lolat = 5 * SBT_1MS; 593ba6c22ceSWarner Losh clp->hilat = 15 * SBT_1MS; 594ba6c22ceSWarner Losh clp->alpha = 20; /* Alpha == gain. 20 = .2 */ 595ba6c22ceSWarner Losh clp->type = set_max; 596ba6c22ceSWarner Losh } 597ba6c22ceSWarner Losh 598ba6c22ceSWarner Losh static void 599ba6c22ceSWarner Losh cam_iosched_cl_maybe_steer(struct control_loop *clp) 600ba6c22ceSWarner Losh { 601ba6c22ceSWarner Losh struct cam_iosched_softc *isc; 602ba6c22ceSWarner Losh sbintime_t now, lat; 603ba6c22ceSWarner Losh int old; 604ba6c22ceSWarner Losh 605ba6c22ceSWarner Losh isc = clp->softc; 606ba6c22ceSWarner Losh now = isc->last_time; 607ba6c22ceSWarner Losh if (now < clp->next_steer) 608ba6c22ceSWarner Losh return; 609ba6c22ceSWarner Losh 610ba6c22ceSWarner Losh clp->next_steer = now + clp->steer_interval; 611ba6c22ceSWarner Losh switch (clp->type) { 612ba6c22ceSWarner Losh case set_max: 613ba6c22ceSWarner Losh if (isc->write_stats.current != isc->write_stats.max) 614ba6c22ceSWarner Losh printf("Steering write from %d kBps to %d kBps\n", 615ba6c22ceSWarner Losh isc->write_stats.current, isc->write_stats.max); 616ba6c22ceSWarner Losh isc->read_stats.current = isc->read_stats.max; 617ba6c22ceSWarner Losh isc->write_stats.current = isc->write_stats.max; 618ba6c22ceSWarner Losh isc->trim_stats.current = isc->trim_stats.max; 619ba6c22ceSWarner Losh break; 620ba6c22ceSWarner Losh case read_latency: 621ba6c22ceSWarner Losh old = isc->write_stats.current; 622ba6c22ceSWarner Losh lat = isc->read_stats.ema; 623ba6c22ceSWarner Losh /* 624ba6c22ceSWarner Losh * Simple PLL-like engine. Since we're steering to a range for 625ba6c22ceSWarner Losh * the SP (set point) that makes things a little more 626ba6c22ceSWarner Losh * complicated. In addition, we're not directly controlling our 627ba6c22ceSWarner Losh * PV (process variable), the read latency, but instead are 628ba6c22ceSWarner Losh * manipulating the write bandwidth limit for our MV 629ba6c22ceSWarner Losh * (manipulation variable), analysis of this code gets a bit 630ba6c22ceSWarner Losh * messy. Also, the MV is a very noisy control surface for read 631ba6c22ceSWarner Losh * latency since it is affected by many hidden processes inside 632ba6c22ceSWarner Losh * the device which change how responsive read latency will be 633ba6c22ceSWarner Losh * in reaction to changes in write bandwidth. Unlike the classic 634ba6c22ceSWarner Losh * boiler control PLL. this may result in over-steering while 635ba6c22ceSWarner Losh * the SSD takes its time to react to the new, lower load. This 636ba6c22ceSWarner Losh * is why we use a relatively low alpha of between .1 and .25 to 637ba6c22ceSWarner Losh * compensate for this effect. At .1, it takes ~22 steering 638ba6c22ceSWarner Losh * intervals to back off by a factor of 10. At .2 it only takes 639ba6c22ceSWarner Losh * ~10. At .25 it only takes ~8. However some preliminary data 640ba6c22ceSWarner Losh * from the SSD drives suggests a reasponse time in 10's of 641ba6c22ceSWarner Losh * seconds before latency drops regardless of the new write 6420b4060b0SEd Maste * rate. Careful observation will be required to tune this 643ba6c22ceSWarner Losh * effectively. 644ba6c22ceSWarner Losh * 645ba6c22ceSWarner Losh * Also, when there's no read traffic, we jack up the write 646ba6c22ceSWarner Losh * limit too regardless of the last read latency. 10 is 647ba6c22ceSWarner Losh * somewhat arbitrary. 648ba6c22ceSWarner Losh */ 649ba6c22ceSWarner Losh if (lat < clp->lolat || isc->read_stats.total - clp->last_count < 10) 650ba6c22ceSWarner Losh isc->write_stats.current = isc->write_stats.current * 651ba6c22ceSWarner Losh (100 + clp->alpha) / 100; /* Scale up */ 652ba6c22ceSWarner Losh else if (lat > clp->hilat) 653ba6c22ceSWarner Losh isc->write_stats.current = isc->write_stats.current * 654ba6c22ceSWarner Losh (100 - clp->alpha) / 100; /* Scale down */ 655ba6c22ceSWarner Losh clp->last_count = isc->read_stats.total; 656ba6c22ceSWarner Losh 657ba6c22ceSWarner Losh /* 658ba6c22ceSWarner Losh * Even if we don't steer, per se, enforce the min/max limits as 659ba6c22ceSWarner Losh * those may have changed. 660ba6c22ceSWarner Losh */ 661ba6c22ceSWarner Losh if (isc->write_stats.current < isc->write_stats.min) 662ba6c22ceSWarner Losh isc->write_stats.current = isc->write_stats.min; 663ba6c22ceSWarner Losh if (isc->write_stats.current > isc->write_stats.max) 664ba6c22ceSWarner Losh isc->write_stats.current = isc->write_stats.max; 6652b5c19f1SWarner Losh if (old != isc->write_stats.current && iosched_debug) 666cf3ec151SWarner Losh printf("Steering write from %d kBps to %d kBps due to latency of %jdus\n", 667ba6c22ceSWarner Losh old, isc->write_stats.current, 6682b5c19f1SWarner Losh (uintmax_t)((uint64_t)1000000 * (uint32_t)lat) >> 32); 669ba6c22ceSWarner Losh break; 670ba6c22ceSWarner Losh case cl_max: 671ba6c22ceSWarner Losh break; 672ba6c22ceSWarner Losh } 673ba6c22ceSWarner Losh } 674ba6c22ceSWarner Losh #endif 675ba6c22ceSWarner Losh 676ba6c22ceSWarner Losh /* Trim or similar currently pending completion */ 677ba6c22ceSWarner Losh #define CAM_IOSCHED_FLAG_TRIM_ACTIVE (1ul << 0) 678ba6c22ceSWarner Losh /* Callout active, and needs to be torn down */ 679ba6c22ceSWarner Losh #define CAM_IOSCHED_FLAG_CALLOUT_ACTIVE (1ul << 1) 680ba6c22ceSWarner Losh 681ba6c22ceSWarner Losh /* Periph drivers set these flags to indicate work */ 682ba6c22ceSWarner Losh #define CAM_IOSCHED_FLAG_WORK_FLAGS ((0xffffu) << 16) 683ba6c22ceSWarner Losh 684df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 685ba6c22ceSWarner Losh static void 686ba6c22ceSWarner Losh cam_iosched_io_metric_update(struct cam_iosched_softc *isc, 687ba6c22ceSWarner Losh sbintime_t sim_latency, int cmd, size_t size); 6885ede5b8cSWarner Losh #endif 689ba6c22ceSWarner Losh 690ba6c22ceSWarner Losh static inline int 691ba6c22ceSWarner Losh cam_iosched_has_flagged_work(struct cam_iosched_softc *isc) 692ba6c22ceSWarner Losh { 693ba6c22ceSWarner Losh return !!(isc->flags & CAM_IOSCHED_FLAG_WORK_FLAGS); 694ba6c22ceSWarner Losh } 695ba6c22ceSWarner Losh 696ba6c22ceSWarner Losh static inline int 697ba6c22ceSWarner Losh cam_iosched_has_io(struct cam_iosched_softc *isc) 698ba6c22ceSWarner Losh { 699df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 700035ec48eSWarner Losh if (do_dynamic_iosched) { 701ba6c22ceSWarner Losh struct bio *rbp = bioq_first(&isc->bio_queue); 702ba6c22ceSWarner Losh struct bio *wbp = bioq_first(&isc->write_queue); 703ba6c22ceSWarner Losh int can_write = wbp != NULL && 704ba6c22ceSWarner Losh cam_iosched_limiter_caniop(&isc->write_stats, wbp) == 0; 705ba6c22ceSWarner Losh int can_read = rbp != NULL && 706ba6c22ceSWarner Losh cam_iosched_limiter_caniop(&isc->read_stats, rbp) == 0; 707ba6c22ceSWarner Losh if (iosched_debug > 2) { 708ba6c22ceSWarner Losh printf("can write %d: pending_writes %d max_writes %d\n", can_write, isc->write_stats.pending, isc->write_stats.max); 709ba6c22ceSWarner Losh printf("can read %d: read_stats.pending %d max_reads %d\n", can_read, isc->read_stats.pending, isc->read_stats.max); 710ba6c22ceSWarner Losh printf("Queued reads %d writes %d\n", isc->read_stats.queued, isc->write_stats.queued); 711ba6c22ceSWarner Losh } 712ba6c22ceSWarner Losh return can_read || can_write; 713ba6c22ceSWarner Losh } 714ba6c22ceSWarner Losh #endif 715ba6c22ceSWarner Losh return bioq_first(&isc->bio_queue) != NULL; 716ba6c22ceSWarner Losh } 717ba6c22ceSWarner Losh 718ba6c22ceSWarner Losh static inline int 719ba6c22ceSWarner Losh cam_iosched_has_more_trim(struct cam_iosched_softc *isc) 720ba6c22ceSWarner Losh { 721ba6c22ceSWarner Losh return !(isc->flags & CAM_IOSCHED_FLAG_TRIM_ACTIVE) && 722ba6c22ceSWarner Losh bioq_first(&isc->trim_queue); 723ba6c22ceSWarner Losh } 724ba6c22ceSWarner Losh 725ba6c22ceSWarner Losh #define cam_iosched_sort_queue(isc) ((isc)->sort_io_queue >= 0 ? \ 726ba6c22ceSWarner Losh (isc)->sort_io_queue : cam_sort_io_queues) 727ba6c22ceSWarner Losh 728ba6c22ceSWarner Losh 729ba6c22ceSWarner Losh static inline int 730ba6c22ceSWarner Losh cam_iosched_has_work(struct cam_iosched_softc *isc) 731ba6c22ceSWarner Losh { 732df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 733ba6c22ceSWarner Losh if (iosched_debug > 2) 734ba6c22ceSWarner Losh printf("has work: %d %d %d\n", cam_iosched_has_io(isc), 735ba6c22ceSWarner Losh cam_iosched_has_more_trim(isc), 736ba6c22ceSWarner Losh cam_iosched_has_flagged_work(isc)); 737ba6c22ceSWarner Losh #endif 738ba6c22ceSWarner Losh 739ba6c22ceSWarner Losh return cam_iosched_has_io(isc) || 740ba6c22ceSWarner Losh cam_iosched_has_more_trim(isc) || 741ba6c22ceSWarner Losh cam_iosched_has_flagged_work(isc); 742ba6c22ceSWarner Losh } 743ba6c22ceSWarner Losh 744df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 745ba6c22ceSWarner Losh static void 746ba6c22ceSWarner Losh cam_iosched_iop_stats_init(struct cam_iosched_softc *isc, struct iop_stats *ios) 747ba6c22ceSWarner Losh { 748ba6c22ceSWarner Losh 749ba6c22ceSWarner Losh ios->limiter = none; 750ba6c22ceSWarner Losh cam_iosched_limiter_init(ios); 751ba6c22ceSWarner Losh ios->in = 0; 752ba6c22ceSWarner Losh ios->max = 300000; 753ba6c22ceSWarner Losh ios->min = 1; 754ba6c22ceSWarner Losh ios->out = 0; 755ba6c22ceSWarner Losh ios->pending = 0; 756ba6c22ceSWarner Losh ios->queued = 0; 757ba6c22ceSWarner Losh ios->total = 0; 758ba6c22ceSWarner Losh ios->ema = 0; 75979d80af2SWarner Losh ios->emvar = 0; 760ba6c22ceSWarner Losh ios->softc = isc; 761ba6c22ceSWarner Losh } 762ba6c22ceSWarner Losh 763ba6c22ceSWarner Losh static int 764ba6c22ceSWarner Losh cam_iosched_limiter_sysctl(SYSCTL_HANDLER_ARGS) 765ba6c22ceSWarner Losh { 766ba6c22ceSWarner Losh char buf[16]; 767ba6c22ceSWarner Losh struct iop_stats *ios; 768ba6c22ceSWarner Losh struct cam_iosched_softc *isc; 769cf3ec151SWarner Losh int value, i, error; 770ba6c22ceSWarner Losh const char *p; 771ba6c22ceSWarner Losh 772ba6c22ceSWarner Losh ios = arg1; 773ba6c22ceSWarner Losh isc = ios->softc; 774ba6c22ceSWarner Losh value = ios->limiter; 775ba6c22ceSWarner Losh if (value < none || value >= limiter_max) 776ba6c22ceSWarner Losh p = "UNKNOWN"; 777ba6c22ceSWarner Losh else 778ba6c22ceSWarner Losh p = cam_iosched_limiter_names[value]; 779ba6c22ceSWarner Losh 780ba6c22ceSWarner Losh strlcpy(buf, p, sizeof(buf)); 781ba6c22ceSWarner Losh error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 782ba6c22ceSWarner Losh if (error != 0 || req->newptr == NULL) 783ba6c22ceSWarner Losh return error; 784ba6c22ceSWarner Losh 785ba6c22ceSWarner Losh cam_periph_lock(isc->periph); 786ba6c22ceSWarner Losh 787ba6c22ceSWarner Losh for (i = none; i < limiter_max; i++) { 788ba6c22ceSWarner Losh if (strcmp(buf, cam_iosched_limiter_names[i]) != 0) 789ba6c22ceSWarner Losh continue; 790ba6c22ceSWarner Losh ios->limiter = i; 791ba6c22ceSWarner Losh error = cam_iosched_limiter_init(ios); 792ba6c22ceSWarner Losh if (error != 0) { 793ba6c22ceSWarner Losh ios->limiter = value; 794ba6c22ceSWarner Losh cam_periph_unlock(isc->periph); 795ba6c22ceSWarner Losh return error; 796ba6c22ceSWarner Losh } 797cf3ec151SWarner Losh /* Note: disk load averate requires ticker to be always running */ 798ba6c22ceSWarner Losh callout_reset(&isc->ticker, hz / isc->quanta - 1, cam_iosched_ticker, isc); 799ba6c22ceSWarner Losh isc->flags |= CAM_IOSCHED_FLAG_CALLOUT_ACTIVE; 800ba6c22ceSWarner Losh 801ba6c22ceSWarner Losh cam_periph_unlock(isc->periph); 802ba6c22ceSWarner Losh return 0; 803ba6c22ceSWarner Losh } 804ba6c22ceSWarner Losh 805ba6c22ceSWarner Losh cam_periph_unlock(isc->periph); 806ba6c22ceSWarner Losh return EINVAL; 807ba6c22ceSWarner Losh } 808ba6c22ceSWarner Losh 809ba6c22ceSWarner Losh static int 810ba6c22ceSWarner Losh cam_iosched_control_type_sysctl(SYSCTL_HANDLER_ARGS) 811ba6c22ceSWarner Losh { 812ba6c22ceSWarner Losh char buf[16]; 813ba6c22ceSWarner Losh struct control_loop *clp; 814ba6c22ceSWarner Losh struct cam_iosched_softc *isc; 815ba6c22ceSWarner Losh int value, i, error; 816ba6c22ceSWarner Losh const char *p; 817ba6c22ceSWarner Losh 818ba6c22ceSWarner Losh clp = arg1; 819ba6c22ceSWarner Losh isc = clp->softc; 820ba6c22ceSWarner Losh value = clp->type; 821ba6c22ceSWarner Losh if (value < none || value >= cl_max) 822ba6c22ceSWarner Losh p = "UNKNOWN"; 823ba6c22ceSWarner Losh else 824ba6c22ceSWarner Losh p = cam_iosched_control_type_names[value]; 825ba6c22ceSWarner Losh 826ba6c22ceSWarner Losh strlcpy(buf, p, sizeof(buf)); 827ba6c22ceSWarner Losh error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 828ba6c22ceSWarner Losh if (error != 0 || req->newptr == NULL) 829ba6c22ceSWarner Losh return error; 830ba6c22ceSWarner Losh 831ba6c22ceSWarner Losh for (i = set_max; i < cl_max; i++) { 832ba6c22ceSWarner Losh if (strcmp(buf, cam_iosched_control_type_names[i]) != 0) 833ba6c22ceSWarner Losh continue; 834ba6c22ceSWarner Losh cam_periph_lock(isc->periph); 835ba6c22ceSWarner Losh clp->type = i; 836ba6c22ceSWarner Losh cam_periph_unlock(isc->periph); 837ba6c22ceSWarner Losh return 0; 838ba6c22ceSWarner Losh } 839ba6c22ceSWarner Losh 840ba6c22ceSWarner Losh return EINVAL; 841ba6c22ceSWarner Losh } 842ba6c22ceSWarner Losh 843ba6c22ceSWarner Losh static int 844ba6c22ceSWarner Losh cam_iosched_sbintime_sysctl(SYSCTL_HANDLER_ARGS) 845ba6c22ceSWarner Losh { 846ba6c22ceSWarner Losh char buf[16]; 847ba6c22ceSWarner Losh sbintime_t value; 848ba6c22ceSWarner Losh int error; 849ba6c22ceSWarner Losh uint64_t us; 850ba6c22ceSWarner Losh 851ba6c22ceSWarner Losh value = *(sbintime_t *)arg1; 852ba6c22ceSWarner Losh us = (uint64_t)value / SBT_1US; 853ba6c22ceSWarner Losh snprintf(buf, sizeof(buf), "%ju", (intmax_t)us); 854ba6c22ceSWarner Losh error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 855ba6c22ceSWarner Losh if (error != 0 || req->newptr == NULL) 856ba6c22ceSWarner Losh return error; 857ba6c22ceSWarner Losh us = strtoul(buf, NULL, 10); 858ba6c22ceSWarner Losh if (us == 0) 859ba6c22ceSWarner Losh return EINVAL; 860ba6c22ceSWarner Losh *(sbintime_t *)arg1 = us * SBT_1US; 861ba6c22ceSWarner Losh return 0; 862ba6c22ceSWarner Losh } 863ba6c22ceSWarner Losh 864cf3ec151SWarner Losh static int 865cf3ec151SWarner Losh cam_iosched_sysctl_latencies(SYSCTL_HANDLER_ARGS) 866cf3ec151SWarner Losh { 867cf3ec151SWarner Losh int i, error; 868cf3ec151SWarner Losh struct sbuf sb; 869cf3ec151SWarner Losh uint64_t *latencies; 870cf3ec151SWarner Losh 871cf3ec151SWarner Losh latencies = arg1; 872cf3ec151SWarner Losh sbuf_new_for_sysctl(&sb, NULL, LAT_BUCKETS * 16, req); 873cf3ec151SWarner Losh 874cf3ec151SWarner Losh for (i = 0; i < LAT_BUCKETS - 1; i++) 875cf3ec151SWarner Losh sbuf_printf(&sb, "%jd,", (intmax_t)latencies[i]); 876cf3ec151SWarner Losh sbuf_printf(&sb, "%jd", (intmax_t)latencies[LAT_BUCKETS - 1]); 877cf3ec151SWarner Losh error = sbuf_finish(&sb); 878cf3ec151SWarner Losh sbuf_delete(&sb); 879cf3ec151SWarner Losh 880cf3ec151SWarner Losh return (error); 881cf3ec151SWarner Losh } 882cf3ec151SWarner Losh 883ba6c22ceSWarner Losh static void 884ba6c22ceSWarner Losh cam_iosched_iop_stats_sysctl_init(struct cam_iosched_softc *isc, struct iop_stats *ios, char *name) 885ba6c22ceSWarner Losh { 886ba6c22ceSWarner Losh struct sysctl_oid_list *n; 887ba6c22ceSWarner Losh struct sysctl_ctx_list *ctx; 888ba6c22ceSWarner Losh 889ba6c22ceSWarner Losh ios->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx, 890ba6c22ceSWarner Losh SYSCTL_CHILDREN(isc->sysctl_tree), OID_AUTO, name, 891ba6c22ceSWarner Losh CTLFLAG_RD, 0, name); 892ba6c22ceSWarner Losh n = SYSCTL_CHILDREN(ios->sysctl_tree); 893ba6c22ceSWarner Losh ctx = &ios->sysctl_ctx; 894ba6c22ceSWarner Losh 895ba6c22ceSWarner Losh SYSCTL_ADD_UQUAD(ctx, n, 896ba6c22ceSWarner Losh OID_AUTO, "ema", CTLFLAG_RD, 897ba6c22ceSWarner Losh &ios->ema, 898ba6c22ceSWarner Losh "Fast Exponentially Weighted Moving Average"); 899ba6c22ceSWarner Losh SYSCTL_ADD_UQUAD(ctx, n, 90079d80af2SWarner Losh OID_AUTO, "emvar", CTLFLAG_RD, 90179d80af2SWarner Losh &ios->emvar, 90279d80af2SWarner Losh "Fast Exponentially Weighted Moving Variance"); 903ba6c22ceSWarner Losh 904ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 905ba6c22ceSWarner Losh OID_AUTO, "pending", CTLFLAG_RD, 906ba6c22ceSWarner Losh &ios->pending, 0, 907ba6c22ceSWarner Losh "Instantaneous # of pending transactions"); 908ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 909ba6c22ceSWarner Losh OID_AUTO, "count", CTLFLAG_RD, 910ba6c22ceSWarner Losh &ios->total, 0, 911ba6c22ceSWarner Losh "# of transactions submitted to hardware"); 912ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 913ba6c22ceSWarner Losh OID_AUTO, "queued", CTLFLAG_RD, 914ba6c22ceSWarner Losh &ios->queued, 0, 915ba6c22ceSWarner Losh "# of transactions in the queue"); 916ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 917ba6c22ceSWarner Losh OID_AUTO, "in", CTLFLAG_RD, 918ba6c22ceSWarner Losh &ios->in, 0, 919ba6c22ceSWarner Losh "# of transactions queued to driver"); 920ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 921ba6c22ceSWarner Losh OID_AUTO, "out", CTLFLAG_RD, 922ba6c22ceSWarner Losh &ios->out, 0, 923ba6c22ceSWarner Losh "# of transactions completed"); 924ba6c22ceSWarner Losh 925ba6c22ceSWarner Losh SYSCTL_ADD_PROC(ctx, n, 926ba6c22ceSWarner Losh OID_AUTO, "limiter", CTLTYPE_STRING | CTLFLAG_RW, 927ba6c22ceSWarner Losh ios, 0, cam_iosched_limiter_sysctl, "A", 928ba6c22ceSWarner Losh "Current limiting type."); 929ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 930ba6c22ceSWarner Losh OID_AUTO, "min", CTLFLAG_RW, 931ba6c22ceSWarner Losh &ios->min, 0, 932ba6c22ceSWarner Losh "min resource"); 933ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 934ba6c22ceSWarner Losh OID_AUTO, "max", CTLFLAG_RW, 935ba6c22ceSWarner Losh &ios->max, 0, 936ba6c22ceSWarner Losh "max resource"); 937ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 938ba6c22ceSWarner Losh OID_AUTO, "current", CTLFLAG_RW, 939ba6c22ceSWarner Losh &ios->current, 0, 940ba6c22ceSWarner Losh "current resource"); 941ba6c22ceSWarner Losh 942cf3ec151SWarner Losh SYSCTL_ADD_PROC(ctx, n, 943cf3ec151SWarner Losh OID_AUTO, "latencies", CTLTYPE_STRING | CTLFLAG_RD, 944cf3ec151SWarner Losh &ios->latencies, 0, 945cf3ec151SWarner Losh cam_iosched_sysctl_latencies, "A", 946cf3ec151SWarner Losh "Array of power of 2 latency from 1ms to 1.024s"); 947ba6c22ceSWarner Losh } 948ba6c22ceSWarner Losh 949ba6c22ceSWarner Losh static void 950ba6c22ceSWarner Losh cam_iosched_iop_stats_fini(struct iop_stats *ios) 951ba6c22ceSWarner Losh { 952ba6c22ceSWarner Losh if (ios->sysctl_tree) 953ba6c22ceSWarner Losh if (sysctl_ctx_free(&ios->sysctl_ctx) != 0) 954ba6c22ceSWarner Losh printf("can't remove iosched sysctl stats context\n"); 955ba6c22ceSWarner Losh } 956ba6c22ceSWarner Losh 957ba6c22ceSWarner Losh static void 958ba6c22ceSWarner Losh cam_iosched_cl_sysctl_init(struct cam_iosched_softc *isc) 959ba6c22ceSWarner Losh { 960ba6c22ceSWarner Losh struct sysctl_oid_list *n; 961ba6c22ceSWarner Losh struct sysctl_ctx_list *ctx; 962ba6c22ceSWarner Losh struct control_loop *clp; 963ba6c22ceSWarner Losh 964ba6c22ceSWarner Losh clp = &isc->cl; 965ba6c22ceSWarner Losh clp->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx, 966ba6c22ceSWarner Losh SYSCTL_CHILDREN(isc->sysctl_tree), OID_AUTO, "control", 967ba6c22ceSWarner Losh CTLFLAG_RD, 0, "Control loop info"); 968ba6c22ceSWarner Losh n = SYSCTL_CHILDREN(clp->sysctl_tree); 969ba6c22ceSWarner Losh ctx = &clp->sysctl_ctx; 970ba6c22ceSWarner Losh 971ba6c22ceSWarner Losh SYSCTL_ADD_PROC(ctx, n, 972ba6c22ceSWarner Losh OID_AUTO, "type", CTLTYPE_STRING | CTLFLAG_RW, 973ba6c22ceSWarner Losh clp, 0, cam_iosched_control_type_sysctl, "A", 974ba6c22ceSWarner Losh "Control loop algorithm"); 975ba6c22ceSWarner Losh SYSCTL_ADD_PROC(ctx, n, 976ba6c22ceSWarner Losh OID_AUTO, "steer_interval", CTLTYPE_STRING | CTLFLAG_RW, 977ba6c22ceSWarner Losh &clp->steer_interval, 0, cam_iosched_sbintime_sysctl, "A", 978ba6c22ceSWarner Losh "How often to steer (in us)"); 979ba6c22ceSWarner Losh SYSCTL_ADD_PROC(ctx, n, 980ba6c22ceSWarner Losh OID_AUTO, "lolat", CTLTYPE_STRING | CTLFLAG_RW, 981ba6c22ceSWarner Losh &clp->lolat, 0, cam_iosched_sbintime_sysctl, "A", 982ba6c22ceSWarner Losh "Low water mark for Latency (in us)"); 983ba6c22ceSWarner Losh SYSCTL_ADD_PROC(ctx, n, 984ba6c22ceSWarner Losh OID_AUTO, "hilat", CTLTYPE_STRING | CTLFLAG_RW, 985ba6c22ceSWarner Losh &clp->hilat, 0, cam_iosched_sbintime_sysctl, "A", 986ba6c22ceSWarner Losh "Hi water mark for Latency (in us)"); 987ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 988ba6c22ceSWarner Losh OID_AUTO, "alpha", CTLFLAG_RW, 989ba6c22ceSWarner Losh &clp->alpha, 0, 990ba6c22ceSWarner Losh "Alpha for PLL (x100) aka gain"); 991ba6c22ceSWarner Losh } 992ba6c22ceSWarner Losh 993ba6c22ceSWarner Losh static void 994ba6c22ceSWarner Losh cam_iosched_cl_sysctl_fini(struct control_loop *clp) 995ba6c22ceSWarner Losh { 996ba6c22ceSWarner Losh if (clp->sysctl_tree) 997ba6c22ceSWarner Losh if (sysctl_ctx_free(&clp->sysctl_ctx) != 0) 998ba6c22ceSWarner Losh printf("can't remove iosched sysctl control loop context\n"); 999ba6c22ceSWarner Losh } 1000ba6c22ceSWarner Losh #endif 1001ba6c22ceSWarner Losh 1002ba6c22ceSWarner Losh /* 1003ba6c22ceSWarner Losh * Allocate the iosched structure. This also insulates callers from knowing 1004ba6c22ceSWarner Losh * sizeof struct cam_iosched_softc. 1005ba6c22ceSWarner Losh */ 1006ba6c22ceSWarner Losh int 1007ba6c22ceSWarner Losh cam_iosched_init(struct cam_iosched_softc **iscp, struct cam_periph *periph) 1008ba6c22ceSWarner Losh { 1009ba6c22ceSWarner Losh 1010ba6c22ceSWarner Losh *iscp = malloc(sizeof(**iscp), M_CAMSCHED, M_NOWAIT | M_ZERO); 1011ba6c22ceSWarner Losh if (*iscp == NULL) 1012ba6c22ceSWarner Losh return ENOMEM; 1013df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1014ba6c22ceSWarner Losh if (iosched_debug) 1015ba6c22ceSWarner Losh printf("CAM IOSCHEDULER Allocating entry at %p\n", *iscp); 1016ba6c22ceSWarner Losh #endif 1017ba6c22ceSWarner Losh (*iscp)->sort_io_queue = -1; 1018ba6c22ceSWarner Losh bioq_init(&(*iscp)->bio_queue); 1019ba6c22ceSWarner Losh bioq_init(&(*iscp)->trim_queue); 1020df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1021035ec48eSWarner Losh if (do_dynamic_iosched) { 1022ba6c22ceSWarner Losh bioq_init(&(*iscp)->write_queue); 1023ba6c22ceSWarner Losh (*iscp)->read_bias = 100; 1024ba6c22ceSWarner Losh (*iscp)->current_read_bias = 100; 1025ba6c22ceSWarner Losh (*iscp)->quanta = 200; 1026ba6c22ceSWarner Losh cam_iosched_iop_stats_init(*iscp, &(*iscp)->read_stats); 1027ba6c22ceSWarner Losh cam_iosched_iop_stats_init(*iscp, &(*iscp)->write_stats); 1028ba6c22ceSWarner Losh cam_iosched_iop_stats_init(*iscp, &(*iscp)->trim_stats); 1029ba6c22ceSWarner Losh (*iscp)->trim_stats.max = 1; /* Trims are special: one at a time for now */ 1030ba6c22ceSWarner Losh (*iscp)->last_time = sbinuptime(); 1031ba6c22ceSWarner Losh callout_init_mtx(&(*iscp)->ticker, cam_periph_mtx(periph), 0); 1032ba6c22ceSWarner Losh (*iscp)->periph = periph; 1033ba6c22ceSWarner Losh cam_iosched_cl_init(&(*iscp)->cl, *iscp); 1034ba6c22ceSWarner Losh callout_reset(&(*iscp)->ticker, hz / (*iscp)->quanta - 1, cam_iosched_ticker, *iscp); 1035ba6c22ceSWarner Losh (*iscp)->flags |= CAM_IOSCHED_FLAG_CALLOUT_ACTIVE; 1036ba6c22ceSWarner Losh } 1037ba6c22ceSWarner Losh #endif 1038ba6c22ceSWarner Losh 1039ba6c22ceSWarner Losh return 0; 1040ba6c22ceSWarner Losh } 1041ba6c22ceSWarner Losh 1042ba6c22ceSWarner Losh /* 1043ba6c22ceSWarner Losh * Reclaim all used resources. This assumes that other folks have 1044ba6c22ceSWarner Losh * drained the requests in the hardware. Maybe an unwise assumption. 1045ba6c22ceSWarner Losh */ 1046ba6c22ceSWarner Losh void 1047ba6c22ceSWarner Losh cam_iosched_fini(struct cam_iosched_softc *isc) 1048ba6c22ceSWarner Losh { 1049ba6c22ceSWarner Losh if (isc) { 1050ba6c22ceSWarner Losh cam_iosched_flush(isc, NULL, ENXIO); 1051df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1052ba6c22ceSWarner Losh cam_iosched_iop_stats_fini(&isc->read_stats); 1053ba6c22ceSWarner Losh cam_iosched_iop_stats_fini(&isc->write_stats); 1054ba6c22ceSWarner Losh cam_iosched_iop_stats_fini(&isc->trim_stats); 1055ba6c22ceSWarner Losh cam_iosched_cl_sysctl_fini(&isc->cl); 1056ba6c22ceSWarner Losh if (isc->sysctl_tree) 1057ba6c22ceSWarner Losh if (sysctl_ctx_free(&isc->sysctl_ctx) != 0) 1058ba6c22ceSWarner Losh printf("can't remove iosched sysctl stats context\n"); 1059ba6c22ceSWarner Losh if (isc->flags & CAM_IOSCHED_FLAG_CALLOUT_ACTIVE) { 1060ba6c22ceSWarner Losh callout_drain(&isc->ticker); 1061ba6c22ceSWarner Losh isc->flags &= ~ CAM_IOSCHED_FLAG_CALLOUT_ACTIVE; 1062ba6c22ceSWarner Losh } 1063ba6c22ceSWarner Losh #endif 1064ba6c22ceSWarner Losh free(isc, M_CAMSCHED); 1065ba6c22ceSWarner Losh } 1066ba6c22ceSWarner Losh } 1067ba6c22ceSWarner Losh 1068ba6c22ceSWarner Losh /* 1069ba6c22ceSWarner Losh * After we're sure we're attaching a device, go ahead and add 1070ba6c22ceSWarner Losh * hooks for any sysctl we may wish to honor. 1071ba6c22ceSWarner Losh */ 1072ba6c22ceSWarner Losh void cam_iosched_sysctl_init(struct cam_iosched_softc *isc, 1073ba6c22ceSWarner Losh struct sysctl_ctx_list *ctx, struct sysctl_oid *node) 1074ba6c22ceSWarner Losh { 1075df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1076ba6c22ceSWarner Losh struct sysctl_oid_list *n; 1077ba6c22ceSWarner Losh #endif 1078ba6c22ceSWarner Losh 1079ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(node), 1080ba6c22ceSWarner Losh OID_AUTO, "sort_io_queue", CTLFLAG_RW | CTLFLAG_MPSAFE, 1081ba6c22ceSWarner Losh &isc->sort_io_queue, 0, 1082ba6c22ceSWarner Losh "Sort IO queue to try and optimise disk access patterns"); 1083ba6c22ceSWarner Losh 1084df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1085035ec48eSWarner Losh if (!do_dynamic_iosched) 1086ba6c22ceSWarner Losh return; 1087ba6c22ceSWarner Losh 1088ba6c22ceSWarner Losh isc->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx, 1089ba6c22ceSWarner Losh SYSCTL_CHILDREN(node), OID_AUTO, "iosched", 1090ba6c22ceSWarner Losh CTLFLAG_RD, 0, "I/O scheduler statistics"); 1091ba6c22ceSWarner Losh n = SYSCTL_CHILDREN(isc->sysctl_tree); 1092ba6c22ceSWarner Losh ctx = &isc->sysctl_ctx; 1093ba6c22ceSWarner Losh 1094ba6c22ceSWarner Losh cam_iosched_iop_stats_sysctl_init(isc, &isc->read_stats, "read"); 1095ba6c22ceSWarner Losh cam_iosched_iop_stats_sysctl_init(isc, &isc->write_stats, "write"); 1096ba6c22ceSWarner Losh cam_iosched_iop_stats_sysctl_init(isc, &isc->trim_stats, "trim"); 1097ba6c22ceSWarner Losh cam_iosched_cl_sysctl_init(isc); 1098ba6c22ceSWarner Losh 1099ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 1100ba6c22ceSWarner Losh OID_AUTO, "read_bias", CTLFLAG_RW, 1101ba6c22ceSWarner Losh &isc->read_bias, 100, 1102ba6c22ceSWarner Losh "How biased towards read should we be independent of limits"); 1103ba6c22ceSWarner Losh 1104ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 1105ba6c22ceSWarner Losh OID_AUTO, "quanta", CTLFLAG_RW, 1106ba6c22ceSWarner Losh &isc->quanta, 200, 1107ba6c22ceSWarner Losh "How many quanta per second do we slice the I/O up into"); 1108ba6c22ceSWarner Losh 1109ba6c22ceSWarner Losh SYSCTL_ADD_INT(ctx, n, 1110ba6c22ceSWarner Losh OID_AUTO, "total_ticks", CTLFLAG_RD, 1111ba6c22ceSWarner Losh &isc->total_ticks, 0, 1112ba6c22ceSWarner Losh "Total number of ticks we've done"); 1113cf3ec151SWarner Losh 1114cf3ec151SWarner Losh SYSCTL_ADD_INT(ctx, n, 1115cf3ec151SWarner Losh OID_AUTO, "load", CTLFLAG_RD, 1116cf3ec151SWarner Losh &isc->load, 0, 1117cf3ec151SWarner Losh "scaled load average / 100"); 1118ba6c22ceSWarner Losh #endif 1119ba6c22ceSWarner Losh } 1120ba6c22ceSWarner Losh 1121ba6c22ceSWarner Losh /* 1122ba6c22ceSWarner Losh * Flush outstanding I/O. Consumers of this library don't know all the 1123ba6c22ceSWarner Losh * queues we may keep, so this allows all I/O to be flushed in one 1124ba6c22ceSWarner Losh * convenient call. 1125ba6c22ceSWarner Losh */ 1126ba6c22ceSWarner Losh void 1127ba6c22ceSWarner Losh cam_iosched_flush(struct cam_iosched_softc *isc, struct devstat *stp, int err) 1128ba6c22ceSWarner Losh { 1129ba6c22ceSWarner Losh bioq_flush(&isc->bio_queue, stp, err); 1130ba6c22ceSWarner Losh bioq_flush(&isc->trim_queue, stp, err); 1131df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1132035ec48eSWarner Losh if (do_dynamic_iosched) 1133ba6c22ceSWarner Losh bioq_flush(&isc->write_queue, stp, err); 1134ba6c22ceSWarner Losh #endif 1135ba6c22ceSWarner Losh } 1136ba6c22ceSWarner Losh 1137df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1138ba6c22ceSWarner Losh static struct bio * 1139ba6c22ceSWarner Losh cam_iosched_get_write(struct cam_iosched_softc *isc) 1140ba6c22ceSWarner Losh { 1141ba6c22ceSWarner Losh struct bio *bp; 1142ba6c22ceSWarner Losh 1143ba6c22ceSWarner Losh /* 1144ba6c22ceSWarner Losh * We control the write rate by controlling how many requests we send 1145ba6c22ceSWarner Losh * down to the drive at any one time. Fewer requests limits the 1146ba6c22ceSWarner Losh * effects of both starvation when the requests take a while and write 1147ba6c22ceSWarner Losh * amplification when each request is causing more than one write to 1148ba6c22ceSWarner Losh * the NAND media. Limiting the queue depth like this will also limit 1149ba6c22ceSWarner Losh * the write throughput and give and reads that want to compete to 1150ba6c22ceSWarner Losh * compete unfairly. 1151ba6c22ceSWarner Losh */ 1152ba6c22ceSWarner Losh bp = bioq_first(&isc->write_queue); 1153ba6c22ceSWarner Losh if (bp == NULL) { 1154ba6c22ceSWarner Losh if (iosched_debug > 3) 1155ba6c22ceSWarner Losh printf("No writes present in write_queue\n"); 1156ba6c22ceSWarner Losh return NULL; 1157ba6c22ceSWarner Losh } 1158ba6c22ceSWarner Losh 1159ba6c22ceSWarner Losh /* 1160ba6c22ceSWarner Losh * If pending read, prefer that based on current read bias 1161ba6c22ceSWarner Losh * setting. 1162ba6c22ceSWarner Losh */ 1163ba6c22ceSWarner Losh if (bioq_first(&isc->bio_queue) && isc->current_read_bias) { 1164ba6c22ceSWarner Losh if (iosched_debug) 1165ba6c22ceSWarner Losh printf("Reads present and current_read_bias is %d queued writes %d queued reads %d\n", isc->current_read_bias, isc->write_stats.queued, isc->read_stats.queued); 1166ba6c22ceSWarner Losh isc->current_read_bias--; 1167cf3ec151SWarner Losh /* We're not limiting writes, per se, just doing reads first */ 1168ba6c22ceSWarner Losh return NULL; 1169ba6c22ceSWarner Losh } 1170ba6c22ceSWarner Losh 1171ba6c22ceSWarner Losh /* 1172ba6c22ceSWarner Losh * See if our current limiter allows this I/O. 1173ba6c22ceSWarner Losh */ 1174ba6c22ceSWarner Losh if (cam_iosched_limiter_iop(&isc->write_stats, bp) != 0) { 1175ba6c22ceSWarner Losh if (iosched_debug) 1176ba6c22ceSWarner Losh printf("Can't write because limiter says no.\n"); 1177cf3ec151SWarner Losh isc->write_stats.state_flags |= IOP_RATE_LIMITED; 1178ba6c22ceSWarner Losh return NULL; 1179ba6c22ceSWarner Losh } 1180ba6c22ceSWarner Losh 1181ba6c22ceSWarner Losh /* 1182ba6c22ceSWarner Losh * Let's do this: We've passed all the gates and we're a go 1183ba6c22ceSWarner Losh * to schedule the I/O in the SIM. 1184ba6c22ceSWarner Losh */ 1185ba6c22ceSWarner Losh isc->current_read_bias = isc->read_bias; 1186ba6c22ceSWarner Losh bioq_remove(&isc->write_queue, bp); 1187ba6c22ceSWarner Losh if (bp->bio_cmd == BIO_WRITE) { 1188ba6c22ceSWarner Losh isc->write_stats.queued--; 1189ba6c22ceSWarner Losh isc->write_stats.total++; 1190ba6c22ceSWarner Losh isc->write_stats.pending++; 1191ba6c22ceSWarner Losh } 1192ba6c22ceSWarner Losh if (iosched_debug > 9) 1193ba6c22ceSWarner Losh printf("HWQ : %p %#x\n", bp, bp->bio_cmd); 1194cf3ec151SWarner Losh isc->write_stats.state_flags &= ~IOP_RATE_LIMITED; 1195ba6c22ceSWarner Losh return bp; 1196ba6c22ceSWarner Losh } 1197ba6c22ceSWarner Losh #endif 1198ba6c22ceSWarner Losh 1199ba6c22ceSWarner Losh /* 1200ba6c22ceSWarner Losh * Put back a trim that you weren't able to actually schedule this time. 1201ba6c22ceSWarner Losh */ 1202ba6c22ceSWarner Losh void 1203ba6c22ceSWarner Losh cam_iosched_put_back_trim(struct cam_iosched_softc *isc, struct bio *bp) 1204ba6c22ceSWarner Losh { 1205ba6c22ceSWarner Losh bioq_insert_head(&isc->trim_queue, bp); 1206df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1207ba6c22ceSWarner Losh isc->trim_stats.queued++; 1208ba6c22ceSWarner Losh isc->trim_stats.total--; /* since we put it back, don't double count */ 1209ba6c22ceSWarner Losh isc->trim_stats.pending--; 1210ba6c22ceSWarner Losh #endif 1211ba6c22ceSWarner Losh } 1212ba6c22ceSWarner Losh 1213ba6c22ceSWarner Losh /* 1214ba6c22ceSWarner Losh * gets the next trim from the trim queue. 1215ba6c22ceSWarner Losh * 1216ba6c22ceSWarner Losh * Assumes we're called with the periph lock held. It removes this 12170b4060b0SEd Maste * trim from the queue and the device must explicitly reinsert it 1218ba6c22ceSWarner Losh * should the need arise. 1219ba6c22ceSWarner Losh */ 1220ba6c22ceSWarner Losh struct bio * 1221ba6c22ceSWarner Losh cam_iosched_next_trim(struct cam_iosched_softc *isc) 1222ba6c22ceSWarner Losh { 1223ba6c22ceSWarner Losh struct bio *bp; 1224ba6c22ceSWarner Losh 1225ba6c22ceSWarner Losh bp = bioq_first(&isc->trim_queue); 1226ba6c22ceSWarner Losh if (bp == NULL) 1227ba6c22ceSWarner Losh return NULL; 1228ba6c22ceSWarner Losh bioq_remove(&isc->trim_queue, bp); 1229df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1230ba6c22ceSWarner Losh isc->trim_stats.queued--; 1231ba6c22ceSWarner Losh isc->trim_stats.total++; 1232ba6c22ceSWarner Losh isc->trim_stats.pending++; 1233ba6c22ceSWarner Losh #endif 1234ba6c22ceSWarner Losh return bp; 1235ba6c22ceSWarner Losh } 1236ba6c22ceSWarner Losh 1237ba6c22ceSWarner Losh /* 12380b4060b0SEd Maste * gets an available trim from the trim queue, if there's no trim 1239ba6c22ceSWarner Losh * already pending. It removes this trim from the queue and the device 12400b4060b0SEd Maste * must explicitly reinsert it should the need arise. 1241ba6c22ceSWarner Losh * 1242ba6c22ceSWarner Losh * Assumes we're called with the periph lock held. 1243ba6c22ceSWarner Losh */ 1244ba6c22ceSWarner Losh struct bio * 1245ba6c22ceSWarner Losh cam_iosched_get_trim(struct cam_iosched_softc *isc) 1246ba6c22ceSWarner Losh { 1247ba6c22ceSWarner Losh 1248ba6c22ceSWarner Losh if (!cam_iosched_has_more_trim(isc)) 1249ba6c22ceSWarner Losh return NULL; 1250ba6c22ceSWarner Losh 1251ba6c22ceSWarner Losh return cam_iosched_next_trim(isc); 1252ba6c22ceSWarner Losh } 1253ba6c22ceSWarner Losh 1254ba6c22ceSWarner Losh /* 1255ba6c22ceSWarner Losh * Determine what the next bit of work to do is for the periph. The 1256ba6c22ceSWarner Losh * default implementation looks to see if we have trims to do, but no 1257ba6c22ceSWarner Losh * trims outstanding. If so, we do that. Otherwise we see if we have 1258ba6c22ceSWarner Losh * other work. If we do, then we do that. Otherwise why were we called? 1259ba6c22ceSWarner Losh */ 1260ba6c22ceSWarner Losh struct bio * 1261ba6c22ceSWarner Losh cam_iosched_next_bio(struct cam_iosched_softc *isc) 1262ba6c22ceSWarner Losh { 1263ba6c22ceSWarner Losh struct bio *bp; 1264ba6c22ceSWarner Losh 1265ba6c22ceSWarner Losh /* 1266ba6c22ceSWarner Losh * See if we have a trim that can be scheduled. We can only send one 1267ba6c22ceSWarner Losh * at a time down, so this takes that into account. 1268ba6c22ceSWarner Losh * 1269ba6c22ceSWarner Losh * XXX newer TRIM commands are queueable. Revisit this when we 1270ba6c22ceSWarner Losh * implement them. 1271ba6c22ceSWarner Losh */ 1272ba6c22ceSWarner Losh if ((bp = cam_iosched_get_trim(isc)) != NULL) 1273ba6c22ceSWarner Losh return bp; 1274ba6c22ceSWarner Losh 1275df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1276ba6c22ceSWarner Losh /* 1277ba6c22ceSWarner Losh * See if we have any pending writes, and room in the queue for them, 1278ba6c22ceSWarner Losh * and if so, those are next. 1279ba6c22ceSWarner Losh */ 1280035ec48eSWarner Losh if (do_dynamic_iosched) { 1281ba6c22ceSWarner Losh if ((bp = cam_iosched_get_write(isc)) != NULL) 1282ba6c22ceSWarner Losh return bp; 1283ba6c22ceSWarner Losh } 1284ba6c22ceSWarner Losh #endif 1285ba6c22ceSWarner Losh 1286ba6c22ceSWarner Losh /* 1287ba6c22ceSWarner Losh * next, see if there's other, normal I/O waiting. If so return that. 1288ba6c22ceSWarner Losh */ 1289ba6c22ceSWarner Losh if ((bp = bioq_first(&isc->bio_queue)) == NULL) 1290ba6c22ceSWarner Losh return NULL; 1291ba6c22ceSWarner Losh 1292df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1293ba6c22ceSWarner Losh /* 1294cf3ec151SWarner Losh * For the dynamic scheduler, bio_queue is only for reads, so enforce 1295ba6c22ceSWarner Losh * the limits here. Enforce only for reads. 1296ba6c22ceSWarner Losh */ 1297035ec48eSWarner Losh if (do_dynamic_iosched) { 1298ba6c22ceSWarner Losh if (bp->bio_cmd == BIO_READ && 1299cf3ec151SWarner Losh cam_iosched_limiter_iop(&isc->read_stats, bp) != 0) { 1300cf3ec151SWarner Losh isc->read_stats.state_flags |= IOP_RATE_LIMITED; 1301ba6c22ceSWarner Losh return NULL; 1302ba6c22ceSWarner Losh } 1303cf3ec151SWarner Losh } 1304cf3ec151SWarner Losh isc->read_stats.state_flags &= ~IOP_RATE_LIMITED; 1305ba6c22ceSWarner Losh #endif 1306ba6c22ceSWarner Losh bioq_remove(&isc->bio_queue, bp); 1307df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1308035ec48eSWarner Losh if (do_dynamic_iosched) { 1309ba6c22ceSWarner Losh if (bp->bio_cmd == BIO_READ) { 1310ba6c22ceSWarner Losh isc->read_stats.queued--; 1311ba6c22ceSWarner Losh isc->read_stats.total++; 1312ba6c22ceSWarner Losh isc->read_stats.pending++; 1313ba6c22ceSWarner Losh } else 1314ba6c22ceSWarner Losh printf("Found bio_cmd = %#x\n", bp->bio_cmd); 1315ba6c22ceSWarner Losh } 1316ba6c22ceSWarner Losh if (iosched_debug > 9) 1317ba6c22ceSWarner Losh printf("HWQ : %p %#x\n", bp, bp->bio_cmd); 1318ba6c22ceSWarner Losh #endif 1319ba6c22ceSWarner Losh return bp; 1320ba6c22ceSWarner Losh } 1321ba6c22ceSWarner Losh 1322ba6c22ceSWarner Losh /* 1323ba6c22ceSWarner Losh * Driver has been given some work to do by the block layer. Tell the 1324ba6c22ceSWarner Losh * scheduler about it and have it queue the work up. The scheduler module 1325ba6c22ceSWarner Losh * will then return the currently most useful bit of work later, possibly 1326ba6c22ceSWarner Losh * deferring work for various reasons. 1327ba6c22ceSWarner Losh */ 1328ba6c22ceSWarner Losh void 1329ba6c22ceSWarner Losh cam_iosched_queue_work(struct cam_iosched_softc *isc, struct bio *bp) 1330ba6c22ceSWarner Losh { 1331ba6c22ceSWarner Losh 1332ba6c22ceSWarner Losh /* 1333ba6c22ceSWarner Losh * Put all trims on the trim queue sorted, since we know 1334ba6c22ceSWarner Losh * that the collapsing code requires this. Otherwise put 1335ba6c22ceSWarner Losh * the work on the bio queue. 1336ba6c22ceSWarner Losh */ 1337ba6c22ceSWarner Losh if (bp->bio_cmd == BIO_DELETE) { 1338ba6c22ceSWarner Losh bioq_disksort(&isc->trim_queue, bp); 1339df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1340ba6c22ceSWarner Losh isc->trim_stats.in++; 1341ba6c22ceSWarner Losh isc->trim_stats.queued++; 1342ba6c22ceSWarner Losh #endif 1343ba6c22ceSWarner Losh } 1344df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1345035ec48eSWarner Losh else if (do_dynamic_iosched && 1346ba6c22ceSWarner Losh (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) { 1347ba6c22ceSWarner Losh if (cam_iosched_sort_queue(isc)) 1348ba6c22ceSWarner Losh bioq_disksort(&isc->write_queue, bp); 1349ba6c22ceSWarner Losh else 1350ba6c22ceSWarner Losh bioq_insert_tail(&isc->write_queue, bp); 1351ba6c22ceSWarner Losh if (iosched_debug > 9) 1352ba6c22ceSWarner Losh printf("Qw : %p %#x\n", bp, bp->bio_cmd); 1353ba6c22ceSWarner Losh if (bp->bio_cmd == BIO_WRITE) { 1354ba6c22ceSWarner Losh isc->write_stats.in++; 1355ba6c22ceSWarner Losh isc->write_stats.queued++; 1356ba6c22ceSWarner Losh } 1357ba6c22ceSWarner Losh } 1358ba6c22ceSWarner Losh #endif 1359ba6c22ceSWarner Losh else { 1360ba6c22ceSWarner Losh if (cam_iosched_sort_queue(isc)) 1361ba6c22ceSWarner Losh bioq_disksort(&isc->bio_queue, bp); 1362ba6c22ceSWarner Losh else 1363ba6c22ceSWarner Losh bioq_insert_tail(&isc->bio_queue, bp); 1364df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1365ba6c22ceSWarner Losh if (iosched_debug > 9) 1366ba6c22ceSWarner Losh printf("Qr : %p %#x\n", bp, bp->bio_cmd); 1367ba6c22ceSWarner Losh if (bp->bio_cmd == BIO_READ) { 1368ba6c22ceSWarner Losh isc->read_stats.in++; 1369ba6c22ceSWarner Losh isc->read_stats.queued++; 1370ba6c22ceSWarner Losh } else if (bp->bio_cmd == BIO_WRITE) { 1371ba6c22ceSWarner Losh isc->write_stats.in++; 1372ba6c22ceSWarner Losh isc->write_stats.queued++; 1373ba6c22ceSWarner Losh } 1374ba6c22ceSWarner Losh #endif 1375ba6c22ceSWarner Losh } 1376ba6c22ceSWarner Losh } 1377ba6c22ceSWarner Losh 1378ba6c22ceSWarner Losh /* 1379ba6c22ceSWarner Losh * If we have work, get it scheduled. Called with the periph lock held. 1380ba6c22ceSWarner Losh */ 1381ba6c22ceSWarner Losh void 1382ba6c22ceSWarner Losh cam_iosched_schedule(struct cam_iosched_softc *isc, struct cam_periph *periph) 1383ba6c22ceSWarner Losh { 1384ba6c22ceSWarner Losh 1385ba6c22ceSWarner Losh if (cam_iosched_has_work(isc)) 1386ba6c22ceSWarner Losh xpt_schedule(periph, CAM_PRIORITY_NORMAL); 1387ba6c22ceSWarner Losh } 1388ba6c22ceSWarner Losh 1389ba6c22ceSWarner Losh /* 1390ba6c22ceSWarner Losh * Complete a trim request 1391ba6c22ceSWarner Losh */ 1392ba6c22ceSWarner Losh void 1393ba6c22ceSWarner Losh cam_iosched_trim_done(struct cam_iosched_softc *isc) 1394ba6c22ceSWarner Losh { 1395ba6c22ceSWarner Losh 1396ba6c22ceSWarner Losh isc->flags &= ~CAM_IOSCHED_FLAG_TRIM_ACTIVE; 1397ba6c22ceSWarner Losh } 1398ba6c22ceSWarner Losh 1399ba6c22ceSWarner Losh /* 1400ba6c22ceSWarner Losh * Complete a bio. Called before we release the ccb with xpt_release_ccb so we 1401ba6c22ceSWarner Losh * might use notes in the ccb for statistics. 1402ba6c22ceSWarner Losh */ 1403ba6c22ceSWarner Losh int 1404ba6c22ceSWarner Losh cam_iosched_bio_complete(struct cam_iosched_softc *isc, struct bio *bp, 1405ba6c22ceSWarner Losh union ccb *done_ccb) 1406ba6c22ceSWarner Losh { 1407ba6c22ceSWarner Losh int retval = 0; 1408df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1409035ec48eSWarner Losh if (!do_dynamic_iosched) 1410ba6c22ceSWarner Losh return retval; 1411ba6c22ceSWarner Losh 1412ba6c22ceSWarner Losh if (iosched_debug > 10) 1413ba6c22ceSWarner Losh printf("done: %p %#x\n", bp, bp->bio_cmd); 1414ba6c22ceSWarner Losh if (bp->bio_cmd == BIO_WRITE) { 1415ba6c22ceSWarner Losh retval = cam_iosched_limiter_iodone(&isc->write_stats, bp); 1416ba6c22ceSWarner Losh isc->write_stats.out++; 1417ba6c22ceSWarner Losh isc->write_stats.pending--; 1418ba6c22ceSWarner Losh } else if (bp->bio_cmd == BIO_READ) { 1419ba6c22ceSWarner Losh retval = cam_iosched_limiter_iodone(&isc->read_stats, bp); 1420ba6c22ceSWarner Losh isc->read_stats.out++; 1421ba6c22ceSWarner Losh isc->read_stats.pending--; 1422ba6c22ceSWarner Losh } else if (bp->bio_cmd == BIO_DELETE) { 1423ba6c22ceSWarner Losh isc->trim_stats.out++; 1424ba6c22ceSWarner Losh isc->trim_stats.pending--; 1425ba6c22ceSWarner Losh } else if (bp->bio_cmd != BIO_FLUSH) { 1426ba6c22ceSWarner Losh if (iosched_debug) 1427ba6c22ceSWarner Losh printf("Completing command with bio_cmd == %#x\n", bp->bio_cmd); 1428ba6c22ceSWarner Losh } 1429ba6c22ceSWarner Losh 1430ba6c22ceSWarner Losh if (!(bp->bio_flags & BIO_ERROR)) 1431e4c9cba7SWarner Losh cam_iosched_io_metric_update(isc, 1432e4c9cba7SWarner Losh cam_iosched_sbintime_t(done_ccb->ccb_h.qos.periph_data), 1433ba6c22ceSWarner Losh bp->bio_cmd, bp->bio_bcount); 1434ba6c22ceSWarner Losh #endif 1435ba6c22ceSWarner Losh return retval; 1436ba6c22ceSWarner Losh } 1437ba6c22ceSWarner Losh 1438ba6c22ceSWarner Losh /* 1439ba6c22ceSWarner Losh * Tell the io scheduler that you've pushed a trim down into the sim. 1440ba6c22ceSWarner Losh * xxx better place for this? 1441ba6c22ceSWarner Losh */ 1442ba6c22ceSWarner Losh void 1443ba6c22ceSWarner Losh cam_iosched_submit_trim(struct cam_iosched_softc *isc) 1444ba6c22ceSWarner Losh { 1445ba6c22ceSWarner Losh 1446ba6c22ceSWarner Losh isc->flags |= CAM_IOSCHED_FLAG_TRIM_ACTIVE; 1447ba6c22ceSWarner Losh } 1448ba6c22ceSWarner Losh 1449ba6c22ceSWarner Losh /* 1450ba6c22ceSWarner Losh * Change the sorting policy hint for I/O transactions for this device. 1451ba6c22ceSWarner Losh */ 1452ba6c22ceSWarner Losh void 1453ba6c22ceSWarner Losh cam_iosched_set_sort_queue(struct cam_iosched_softc *isc, int val) 1454ba6c22ceSWarner Losh { 1455ba6c22ceSWarner Losh 1456ba6c22ceSWarner Losh isc->sort_io_queue = val; 1457ba6c22ceSWarner Losh } 1458ba6c22ceSWarner Losh 1459ba6c22ceSWarner Losh int 1460ba6c22ceSWarner Losh cam_iosched_has_work_flags(struct cam_iosched_softc *isc, uint32_t flags) 1461ba6c22ceSWarner Losh { 1462ba6c22ceSWarner Losh return isc->flags & flags; 1463ba6c22ceSWarner Losh } 1464ba6c22ceSWarner Losh 1465ba6c22ceSWarner Losh void 1466ba6c22ceSWarner Losh cam_iosched_set_work_flags(struct cam_iosched_softc *isc, uint32_t flags) 1467ba6c22ceSWarner Losh { 1468ba6c22ceSWarner Losh isc->flags |= flags; 1469ba6c22ceSWarner Losh } 1470ba6c22ceSWarner Losh 1471ba6c22ceSWarner Losh void 1472ba6c22ceSWarner Losh cam_iosched_clr_work_flags(struct cam_iosched_softc *isc, uint32_t flags) 1473ba6c22ceSWarner Losh { 1474ba6c22ceSWarner Losh isc->flags &= ~flags; 1475ba6c22ceSWarner Losh } 1476ba6c22ceSWarner Losh 1477df236247SWarner Losh #ifdef CAM_IOSCHED_DYNAMIC 1478ba6c22ceSWarner Losh /* 1479ba6c22ceSWarner Losh * After the method presented in Jack Crenshaw's 1998 article "Integer 14800b4060b0SEd Maste * Square Roots," reprinted at 1481ba6c22ceSWarner Losh * http://www.embedded.com/electronics-blogs/programmer-s-toolbox/4219659/Integer-Square-Roots 1482ba6c22ceSWarner Losh * and well worth the read. Briefly, we find the power of 4 that's the 1483ba6c22ceSWarner Losh * largest smaller than val. We then check each smaller power of 4 to 1484ba6c22ceSWarner Losh * see if val is still bigger. The right shifts at each step divide 1485ba6c22ceSWarner Losh * the result by 2 which after successive application winds up 1486ba6c22ceSWarner Losh * accumulating the right answer. It could also have been accumulated 1487ba6c22ceSWarner Losh * using a separate root counter, but this code is smaller and faster 1488ba6c22ceSWarner Losh * than that method. This method is also integer size invariant. 14890b4060b0SEd Maste * It returns floor(sqrt((float)val)), or the largest integer less than 1490ba6c22ceSWarner Losh * or equal to the square root. 1491ba6c22ceSWarner Losh */ 1492ba6c22ceSWarner Losh static uint64_t 1493ba6c22ceSWarner Losh isqrt64(uint64_t val) 1494ba6c22ceSWarner Losh { 1495ba6c22ceSWarner Losh uint64_t res = 0; 1496ba6c22ceSWarner Losh uint64_t bit = 1ULL << (sizeof(uint64_t) * NBBY - 2); 1497ba6c22ceSWarner Losh 1498ba6c22ceSWarner Losh /* 1499ba6c22ceSWarner Losh * Find the largest power of 4 smaller than val. 1500ba6c22ceSWarner Losh */ 1501ba6c22ceSWarner Losh while (bit > val) 1502ba6c22ceSWarner Losh bit >>= 2; 1503ba6c22ceSWarner Losh 1504ba6c22ceSWarner Losh /* 1505ba6c22ceSWarner Losh * Accumulate the answer, one bit at a time (we keep moving 1506ba6c22ceSWarner Losh * them over since 2 is the square root of 4 and we test 1507ba6c22ceSWarner Losh * powers of 4). We accumulate where we find the bit, but 1508ba6c22ceSWarner Losh * the successive shifts land the bit in the right place 1509ba6c22ceSWarner Losh * by the end. 1510ba6c22ceSWarner Losh */ 1511ba6c22ceSWarner Losh while (bit != 0) { 1512ba6c22ceSWarner Losh if (val >= res + bit) { 1513ba6c22ceSWarner Losh val -= res + bit; 1514ba6c22ceSWarner Losh res = (res >> 1) + bit; 1515ba6c22ceSWarner Losh } else 1516ba6c22ceSWarner Losh res >>= 1; 1517ba6c22ceSWarner Losh bit >>= 2; 1518ba6c22ceSWarner Losh } 1519ba6c22ceSWarner Losh 1520ba6c22ceSWarner Losh return res; 1521ba6c22ceSWarner Losh } 1522ba6c22ceSWarner Losh 1523*08fc2f23SWarner Losh static sbintime_t latencies[LAT_BUCKETS - 1] = { 1524cf3ec151SWarner Losh SBT_1MS << 0, 1525cf3ec151SWarner Losh SBT_1MS << 1, 1526cf3ec151SWarner Losh SBT_1MS << 2, 1527cf3ec151SWarner Losh SBT_1MS << 3, 1528cf3ec151SWarner Losh SBT_1MS << 4, 1529cf3ec151SWarner Losh SBT_1MS << 5, 1530cf3ec151SWarner Losh SBT_1MS << 6, 1531cf3ec151SWarner Losh SBT_1MS << 7, 1532cf3ec151SWarner Losh SBT_1MS << 8, 1533cf3ec151SWarner Losh SBT_1MS << 9, 1534*08fc2f23SWarner Losh SBT_1MS << 10, 1535*08fc2f23SWarner Losh SBT_1MS << 11, 1536*08fc2f23SWarner Losh SBT_1MS << 12, 1537*08fc2f23SWarner Losh SBT_1MS << 13 /* 8.192s */ 1538cf3ec151SWarner Losh }; 1539cf3ec151SWarner Losh 1540ba6c22ceSWarner Losh static void 1541ba6c22ceSWarner Losh cam_iosched_update(struct iop_stats *iop, sbintime_t sim_latency) 1542ba6c22ceSWarner Losh { 154379d80af2SWarner Losh sbintime_t y, deltasq, delta; 1544cf3ec151SWarner Losh int i; 1545cf3ec151SWarner Losh 1546cf3ec151SWarner Losh /* 1547cf3ec151SWarner Losh * Keep counts for latency. We do it by power of two buckets. 1548cf3ec151SWarner Losh * This helps us spot outlier behavior obscured by averages. 1549cf3ec151SWarner Losh */ 1550cf3ec151SWarner Losh for (i = 0; i < LAT_BUCKETS - 1; i++) { 1551cf3ec151SWarner Losh if (sim_latency < latencies[i]) { 1552cf3ec151SWarner Losh iop->latencies[i]++; 1553cf3ec151SWarner Losh break; 1554cf3ec151SWarner Losh } 1555cf3ec151SWarner Losh } 1556cf3ec151SWarner Losh if (i == LAT_BUCKETS - 1) 1557cf3ec151SWarner Losh iop->latencies[i]++; /* Put all > 1024ms values into the last bucket. */ 1558ba6c22ceSWarner Losh 1559ba6c22ceSWarner Losh /* 15600b4060b0SEd Maste * Classic exponentially decaying average with a tiny alpha 1561ba6c22ceSWarner Losh * (2 ^ -alpha_bits). For more info see the NIST statistical 1562ba6c22ceSWarner Losh * handbook. 1563ba6c22ceSWarner Losh * 156479d80af2SWarner Losh * ema_t = y_t * alpha + ema_t-1 * (1 - alpha) [nist] 156579d80af2SWarner Losh * ema_t = y_t * alpha + ema_t-1 - alpha * ema_t-1 156679d80af2SWarner Losh * ema_t = alpha * y_t - alpha * ema_t-1 + ema_t-1 1567ba6c22ceSWarner Losh * alpha = 1 / (1 << alpha_bits) 156879d80af2SWarner Losh * sub e == ema_t-1, b == 1/alpha (== 1 << alpha_bits), d == y_t - ema_t-1 156979d80af2SWarner Losh * = y_t/b - e/b + be/b 157079d80af2SWarner Losh * = (y_t - e + be) / b 157179d80af2SWarner Losh * = (e + d) / b 1572ba6c22ceSWarner Losh * 1573ba6c22ceSWarner Losh * Since alpha is a power of two, we can compute this w/o any mult or 1574ba6c22ceSWarner Losh * division. 157579d80af2SWarner Losh * 157679d80af2SWarner Losh * Variance can also be computed. Usually, it would be expressed as follows: 157779d80af2SWarner Losh * diff_t = y_t - ema_t-1 157879d80af2SWarner Losh * emvar_t = (1 - alpha) * (emavar_t-1 + diff_t^2 * alpha) 157979d80af2SWarner Losh * = emavar_t-1 - alpha * emavar_t-1 + delta_t^2 * alpha - (delta_t * alpha)^2 158079d80af2SWarner Losh * sub b == 1/alpha (== 1 << alpha_bits), e == emavar_t-1, d = delta_t^2 158179d80af2SWarner Losh * = e - e/b + dd/b + dd/bb 158279d80af2SWarner Losh * = (bbe - be + bdd + dd) / bb 158379d80af2SWarner Losh * = (bbe + b(dd-e) + dd) / bb (which is expanded below bb = 1<<(2*alpha_bits)) 158479d80af2SWarner Losh */ 158579d80af2SWarner Losh /* 158679d80af2SWarner Losh * XXX possible numeric issues 158779d80af2SWarner Losh * o We assume right shifted integers do the right thing, since that's 158879d80af2SWarner Losh * implementation defined. You can change the right shifts to / (1LL << alpha). 158979d80af2SWarner Losh * o alpha_bits = 9 gives ema ceiling of 23 bits of seconds for ema and 14 bits 159079d80af2SWarner Losh * for emvar. This puts a ceiling of 13 bits on alpha since we need a 159179d80af2SWarner Losh * few tens of seconds of representation. 159279d80af2SWarner Losh * o We mitigate alpha issues by never setting it too high. 1593ba6c22ceSWarner Losh */ 1594ba6c22ceSWarner Losh y = sim_latency; 159579d80af2SWarner Losh delta = (y - iop->ema); /* d */ 159679d80af2SWarner Losh iop->ema = ((iop->ema << alpha_bits) + delta) >> alpha_bits; 1597ba6c22ceSWarner Losh 1598ba6c22ceSWarner Losh /* 159979d80af2SWarner Losh * Were we to naively plow ahead at this point, we wind up with many numerical 160079d80af2SWarner Losh * issues making any SD > ~3ms unreliable. So, we shift right by 12. This leaves 160179d80af2SWarner Losh * us with microsecond level precision in the input, so the same in the 160279d80af2SWarner Losh * output. It means we can't overflow deltasq unless delta > 4k seconds. It 160379d80af2SWarner Losh * also means that emvar can be up 46 bits 40 of which are fraction, which 160479d80af2SWarner Losh * gives us a way to measure up to ~8s in the SD before the computation goes 160579d80af2SWarner Losh * unstable. Even the worst hard disk rarely has > 1s service time in the 160679d80af2SWarner Losh * drive. It does mean we have to shift left 12 bits after taking the 160779d80af2SWarner Losh * square root to compute the actual standard deviation estimate. This loss of 160879d80af2SWarner Losh * precision is preferable to needing int128 types to work. The above numbers 160979d80af2SWarner Losh * assume alpha=9. 10 or 11 are ok, but we start to run into issues at 12, 161079d80af2SWarner Losh * so 12 or 13 is OK for EMA, EMVAR and SD will be wrong in those cases. 1611ba6c22ceSWarner Losh */ 161279d80af2SWarner Losh delta >>= 12; 161379d80af2SWarner Losh deltasq = delta * delta; /* dd */ 161479d80af2SWarner Losh iop->emvar = ((iop->emvar << (2 * alpha_bits)) + /* bbe */ 161579d80af2SWarner Losh ((deltasq - iop->emvar) << alpha_bits) + /* b(dd-e) */ 161679d80af2SWarner Losh deltasq) /* dd */ 161779d80af2SWarner Losh >> (2 * alpha_bits); /* div bb */ 161879d80af2SWarner Losh iop->sd = (sbintime_t)isqrt64((uint64_t)iop->emvar) << 12; 1619ba6c22ceSWarner Losh } 1620ba6c22ceSWarner Losh 1621ba6c22ceSWarner Losh static void 1622ba6c22ceSWarner Losh cam_iosched_io_metric_update(struct cam_iosched_softc *isc, 1623ba6c22ceSWarner Losh sbintime_t sim_latency, int cmd, size_t size) 1624ba6c22ceSWarner Losh { 1625ba6c22ceSWarner Losh /* xxx Do we need to scale based on the size of the I/O ? */ 1626ba6c22ceSWarner Losh switch (cmd) { 1627ba6c22ceSWarner Losh case BIO_READ: 1628ba6c22ceSWarner Losh cam_iosched_update(&isc->read_stats, sim_latency); 1629ba6c22ceSWarner Losh break; 1630ba6c22ceSWarner Losh case BIO_WRITE: 1631ba6c22ceSWarner Losh cam_iosched_update(&isc->write_stats, sim_latency); 1632ba6c22ceSWarner Losh break; 1633ba6c22ceSWarner Losh case BIO_DELETE: 1634ba6c22ceSWarner Losh cam_iosched_update(&isc->trim_stats, sim_latency); 1635ba6c22ceSWarner Losh break; 1636ba6c22ceSWarner Losh default: 1637ba6c22ceSWarner Losh break; 1638ba6c22ceSWarner Losh } 1639ba6c22ceSWarner Losh } 1640ba6c22ceSWarner Losh 1641ba6c22ceSWarner Losh #ifdef DDB 1642ba6c22ceSWarner Losh static int biolen(struct bio_queue_head *bq) 1643ba6c22ceSWarner Losh { 1644ba6c22ceSWarner Losh int i = 0; 1645ba6c22ceSWarner Losh struct bio *bp; 1646ba6c22ceSWarner Losh 1647ba6c22ceSWarner Losh TAILQ_FOREACH(bp, &bq->queue, bio_queue) { 1648ba6c22ceSWarner Losh i++; 1649ba6c22ceSWarner Losh } 1650ba6c22ceSWarner Losh return i; 1651ba6c22ceSWarner Losh } 1652ba6c22ceSWarner Losh 1653ba6c22ceSWarner Losh /* 1654ba6c22ceSWarner Losh * Show the internal state of the I/O scheduler. 1655ba6c22ceSWarner Losh */ 1656ba6c22ceSWarner Losh DB_SHOW_COMMAND(iosched, cam_iosched_db_show) 1657ba6c22ceSWarner Losh { 1658ba6c22ceSWarner Losh struct cam_iosched_softc *isc; 1659ba6c22ceSWarner Losh 1660ba6c22ceSWarner Losh if (!have_addr) { 1661ba6c22ceSWarner Losh db_printf("Need addr\n"); 1662ba6c22ceSWarner Losh return; 1663ba6c22ceSWarner Losh } 1664ba6c22ceSWarner Losh isc = (struct cam_iosched_softc *)addr; 1665ba6c22ceSWarner Losh db_printf("pending_reads: %d\n", isc->read_stats.pending); 1666ba6c22ceSWarner Losh db_printf("min_reads: %d\n", isc->read_stats.min); 1667ba6c22ceSWarner Losh db_printf("max_reads: %d\n", isc->read_stats.max); 1668ba6c22ceSWarner Losh db_printf("reads: %d\n", isc->read_stats.total); 1669ba6c22ceSWarner Losh db_printf("in_reads: %d\n", isc->read_stats.in); 1670ba6c22ceSWarner Losh db_printf("out_reads: %d\n", isc->read_stats.out); 1671ba6c22ceSWarner Losh db_printf("queued_reads: %d\n", isc->read_stats.queued); 1672ba6c22ceSWarner Losh db_printf("Current Q len %d\n", biolen(&isc->bio_queue)); 1673ba6c22ceSWarner Losh db_printf("pending_writes: %d\n", isc->write_stats.pending); 1674ba6c22ceSWarner Losh db_printf("min_writes: %d\n", isc->write_stats.min); 1675ba6c22ceSWarner Losh db_printf("max_writes: %d\n", isc->write_stats.max); 1676ba6c22ceSWarner Losh db_printf("writes: %d\n", isc->write_stats.total); 1677ba6c22ceSWarner Losh db_printf("in_writes: %d\n", isc->write_stats.in); 1678ba6c22ceSWarner Losh db_printf("out_writes: %d\n", isc->write_stats.out); 1679ba6c22ceSWarner Losh db_printf("queued_writes: %d\n", isc->write_stats.queued); 1680ba6c22ceSWarner Losh db_printf("Current Q len %d\n", biolen(&isc->write_queue)); 1681ba6c22ceSWarner Losh db_printf("pending_trims: %d\n", isc->trim_stats.pending); 1682ba6c22ceSWarner Losh db_printf("min_trims: %d\n", isc->trim_stats.min); 1683ba6c22ceSWarner Losh db_printf("max_trims: %d\n", isc->trim_stats.max); 1684ba6c22ceSWarner Losh db_printf("trims: %d\n", isc->trim_stats.total); 1685ba6c22ceSWarner Losh db_printf("in_trims: %d\n", isc->trim_stats.in); 1686ba6c22ceSWarner Losh db_printf("out_trims: %d\n", isc->trim_stats.out); 1687ba6c22ceSWarner Losh db_printf("queued_trims: %d\n", isc->trim_stats.queued); 1688ba6c22ceSWarner Losh db_printf("Current Q len %d\n", biolen(&isc->trim_queue)); 1689ba6c22ceSWarner Losh db_printf("read_bias: %d\n", isc->read_bias); 1690ba6c22ceSWarner Losh db_printf("current_read_bias: %d\n", isc->current_read_bias); 1691ba6c22ceSWarner Losh db_printf("Trim active? %s\n", 1692ba6c22ceSWarner Losh (isc->flags & CAM_IOSCHED_FLAG_TRIM_ACTIVE) ? "yes" : "no"); 1693ba6c22ceSWarner Losh } 1694ba6c22ceSWarner Losh #endif 1695ba6c22ceSWarner Losh #endif 1696