1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 14 * Use is subject to license terms. 15 */ 16 17 /* 18 * Copyright (c) 2012, 2018 by Delphix. All rights reserved. 19 */ 20 21 #include <sys/zfs_context.h> 22 #include <sys/vdev_impl.h> 23 #include <sys/spa_impl.h> 24 #include <sys/zio.h> 25 #include <sys/avl.h> 26 #include <sys/dsl_pool.h> 27 #include <sys/metaslab_impl.h> 28 #include <sys/spa.h> 29 #include <sys/abd.h> 30 31 /* 32 * ZFS I/O Scheduler 33 * --------------- 34 * 35 * ZFS issues I/O operations to leaf vdevs to satisfy and complete zios. The 36 * I/O scheduler determines when and in what order those operations are 37 * issued. The I/O scheduler divides operations into five I/O classes 38 * prioritized in the following order: sync read, sync write, async read, 39 * async write, and scrub/resilver. Each queue defines the minimum and 40 * maximum number of concurrent operations that may be issued to the device. 41 * In addition, the device has an aggregate maximum. Note that the sum of the 42 * per-queue minimums must not exceed the aggregate maximum. If the 43 * sum of the per-queue maximums exceeds the aggregate maximum, then the 44 * number of active i/os may reach zfs_vdev_max_active, in which case no 45 * further i/os will be issued regardless of whether all per-queue 46 * minimums have been met. 47 * 48 * For many physical devices, throughput increases with the number of 49 * concurrent operations, but latency typically suffers. Further, physical 50 * devices typically have a limit at which more concurrent operations have no 51 * effect on throughput or can actually cause it to decrease. 52 * 53 * The scheduler selects the next operation to issue by first looking for an 54 * I/O class whose minimum has not been satisfied. Once all are satisfied and 55 * the aggregate maximum has not been hit, the scheduler looks for classes 56 * whose maximum has not been satisfied. Iteration through the I/O classes is 57 * done in the order specified above. No further operations are issued if the 58 * aggregate maximum number of concurrent operations has been hit or if there 59 * are no operations queued for an I/O class that has not hit its maximum. 60 * Every time an i/o is queued or an operation completes, the I/O scheduler 61 * looks for new operations to issue. 62 * 63 * All I/O classes have a fixed maximum number of outstanding operations 64 * except for the async write class. Asynchronous writes represent the data 65 * that is committed to stable storage during the syncing stage for 66 * transaction groups (see txg.c). Transaction groups enter the syncing state 67 * periodically so the number of queued async writes will quickly burst up and 68 * then bleed down to zero. Rather than servicing them as quickly as possible, 69 * the I/O scheduler changes the maximum number of active async write i/os 70 * according to the amount of dirty data in the pool (see dsl_pool.c). Since 71 * both throughput and latency typically increase with the number of 72 * concurrent operations issued to physical devices, reducing the burstiness 73 * in the number of concurrent operations also stabilizes the response time of 74 * operations from other -- and in particular synchronous -- queues. In broad 75 * strokes, the I/O scheduler will issue more concurrent operations from the 76 * async write queue as there's more dirty data in the pool. 77 * 78 * Async Writes 79 * 80 * The number of concurrent operations issued for the async write I/O class 81 * follows a piece-wise linear function defined by a few adjustable points. 82 * 83 * | o---------| <-- zfs_vdev_async_write_max_active 84 * ^ | /^ | 85 * | | / | | 86 * active | / | | 87 * I/O | / | | 88 * count | / | | 89 * | / | | 90 * |------------o | | <-- zfs_vdev_async_write_min_active 91 * 0|____________^______|_________| 92 * 0% | | 100% of zfs_dirty_data_max 93 * | | 94 * | `-- zfs_vdev_async_write_active_max_dirty_percent 95 * `--------- zfs_vdev_async_write_active_min_dirty_percent 96 * 97 * Until the amount of dirty data exceeds a minimum percentage of the dirty 98 * data allowed in the pool, the I/O scheduler will limit the number of 99 * concurrent operations to the minimum. As that threshold is crossed, the 100 * number of concurrent operations issued increases linearly to the maximum at 101 * the specified maximum percentage of the dirty data allowed in the pool. 102 * 103 * Ideally, the amount of dirty data on a busy pool will stay in the sloped 104 * part of the function between zfs_vdev_async_write_active_min_dirty_percent 105 * and zfs_vdev_async_write_active_max_dirty_percent. If it exceeds the 106 * maximum percentage, this indicates that the rate of incoming data is 107 * greater than the rate that the backend storage can handle. In this case, we 108 * must further throttle incoming writes (see dmu_tx_delay() for details). 109 */ 110 111 /* 112 * The maximum number of i/os active to each device. Ideally, this will be >= 113 * the sum of each queue's max_active. 114 */ 115 uint_t zfs_vdev_max_active = 1000; 116 117 /* 118 * Per-queue limits on the number of i/os active to each device. If the 119 * number of active i/os is < zfs_vdev_max_active, then the min_active comes 120 * into play. We will send min_active from each queue round-robin, and then 121 * send from queues in the order defined by zio_priority_t up to max_active. 122 * Some queues have additional mechanisms to limit number of active I/Os in 123 * addition to min_active and max_active, see below. 124 * 125 * In general, smaller max_active's will lead to lower latency of synchronous 126 * operations. Larger max_active's may lead to higher overall throughput, 127 * depending on underlying storage. 128 * 129 * The ratio of the queues' max_actives determines the balance of performance 130 * between reads, writes, and scrubs. E.g., increasing 131 * zfs_vdev_scrub_max_active will cause the scrub or resilver to complete 132 * more quickly, but reads and writes to have higher latency and lower 133 * throughput. 134 */ 135 static uint_t zfs_vdev_sync_read_min_active = 10; 136 static uint_t zfs_vdev_sync_read_max_active = 10; 137 static uint_t zfs_vdev_sync_write_min_active = 10; 138 static uint_t zfs_vdev_sync_write_max_active = 10; 139 static uint_t zfs_vdev_async_read_min_active = 1; 140 /* */ uint_t zfs_vdev_async_read_max_active = 3; 141 static uint_t zfs_vdev_async_write_min_active = 2; 142 static uint_t zfs_vdev_async_write_max_active = 10; 143 static uint_t zfs_vdev_scrub_min_active = 1; 144 static uint_t zfs_vdev_scrub_max_active = 3; 145 static uint_t zfs_vdev_removal_min_active = 1; 146 static uint_t zfs_vdev_removal_max_active = 2; 147 static uint_t zfs_vdev_initializing_min_active = 1; 148 static uint_t zfs_vdev_initializing_max_active = 1; 149 static uint_t zfs_vdev_trim_min_active = 1; 150 static uint_t zfs_vdev_trim_max_active = 2; 151 static uint_t zfs_vdev_rebuild_min_active = 1; 152 static uint_t zfs_vdev_rebuild_max_active = 3; 153 154 /* 155 * When the pool has less than zfs_vdev_async_write_active_min_dirty_percent 156 * dirty data, use zfs_vdev_async_write_min_active. When it has more than 157 * zfs_vdev_async_write_active_max_dirty_percent, use 158 * zfs_vdev_async_write_max_active. The value is linearly interpolated 159 * between min and max. 160 */ 161 uint_t zfs_vdev_async_write_active_min_dirty_percent = 30; 162 uint_t zfs_vdev_async_write_active_max_dirty_percent = 60; 163 164 /* 165 * For non-interactive I/O (scrub, resilver, removal, initialize and rebuild), 166 * the number of concurrently-active I/O's is limited to *_min_active, unless 167 * the vdev is "idle". When there are no interactive I/Os active (sync or 168 * async), and zfs_vdev_nia_delay I/Os have completed since the last 169 * interactive I/O, then the vdev is considered to be "idle", and the number 170 * of concurrently-active non-interactive I/O's is increased to *_max_active. 171 */ 172 static uint_t zfs_vdev_nia_delay = 5; 173 174 /* 175 * Some HDDs tend to prioritize sequential I/O so high that concurrent 176 * random I/O latency reaches several seconds. On some HDDs it happens 177 * even if sequential I/Os are submitted one at a time, and so setting 178 * *_max_active to 1 does not help. To prevent non-interactive I/Os, like 179 * scrub, from monopolizing the device no more than zfs_vdev_nia_credit 180 * I/Os can be sent while there are outstanding incomplete interactive 181 * I/Os. This enforced wait ensures the HDD services the interactive I/O 182 * within a reasonable amount of time. 183 */ 184 static uint_t zfs_vdev_nia_credit = 5; 185 186 /* 187 * To reduce IOPs, we aggregate small adjacent I/Os into one large I/O. 188 * For read I/Os, we also aggregate across small adjacency gaps; for writes 189 * we include spans of optional I/Os to aid aggregation at the disk even when 190 * they aren't able to help us aggregate at this level. 191 */ 192 static uint_t zfs_vdev_aggregation_limit = 1 << 20; 193 static uint_t zfs_vdev_aggregation_limit_non_rotating = SPA_OLD_MAXBLOCKSIZE; 194 static uint_t zfs_vdev_read_gap_limit = 32 << 10; 195 static uint_t zfs_vdev_write_gap_limit = 4 << 10; 196 197 static int 198 vdev_queue_offset_compare(const void *x1, const void *x2) 199 { 200 const zio_t *z1 = (const zio_t *)x1; 201 const zio_t *z2 = (const zio_t *)x2; 202 203 int cmp = TREE_CMP(z1->io_offset, z2->io_offset); 204 205 if (likely(cmp)) 206 return (cmp); 207 208 return (TREE_PCMP(z1, z2)); 209 } 210 211 #define VDQ_T_SHIFT 29 212 213 static int 214 vdev_queue_to_compare(const void *x1, const void *x2) 215 { 216 const zio_t *z1 = (const zio_t *)x1; 217 const zio_t *z2 = (const zio_t *)x2; 218 219 int cmp = TREE_CMP(z1->io_timestamp >> VDQ_T_SHIFT, 220 z2->io_timestamp >> VDQ_T_SHIFT); 221 if (cmp == 0) 222 cmp = TREE_CMP(z1->io_offset, z2->io_offset); 223 224 if (likely(cmp | (z1->io_queue_state == ZIO_QS_NONE))) 225 return (cmp); 226 227 return (TREE_PCMP(z1, z2)); 228 } 229 230 static inline boolean_t 231 vdev_queue_class_fifo(zio_priority_t p) 232 { 233 return (p == ZIO_PRIORITY_SYNC_READ || p == ZIO_PRIORITY_SYNC_WRITE || 234 p == ZIO_PRIORITY_TRIM); 235 } 236 237 static void 238 vdev_queue_class_add(vdev_queue_t *vq, zio_t *zio) 239 { 240 zio_priority_t p = zio->io_priority; 241 vq->vq_cqueued |= 1U << p; 242 if (vdev_queue_class_fifo(p)) { 243 list_insert_tail(&vq->vq_class[p].vqc_list, zio); 244 vq->vq_class[p].vqc_list_numnodes++; 245 } 246 else 247 avl_add(&vq->vq_class[p].vqc_tree, zio); 248 } 249 250 static void 251 vdev_queue_class_remove(vdev_queue_t *vq, zio_t *zio) 252 { 253 zio_priority_t p = zio->io_priority; 254 uint32_t empty; 255 if (vdev_queue_class_fifo(p)) { 256 list_t *list = &vq->vq_class[p].vqc_list; 257 list_remove(list, zio); 258 empty = list_is_empty(list); 259 vq->vq_class[p].vqc_list_numnodes--; 260 } else { 261 avl_tree_t *tree = &vq->vq_class[p].vqc_tree; 262 avl_remove(tree, zio); 263 empty = avl_is_empty(tree); 264 } 265 vq->vq_cqueued &= ~(empty << p); 266 } 267 268 static uint_t 269 vdev_queue_class_min_active(vdev_queue_t *vq, zio_priority_t p) 270 { 271 switch (p) { 272 case ZIO_PRIORITY_SYNC_READ: 273 return (zfs_vdev_sync_read_min_active); 274 case ZIO_PRIORITY_SYNC_WRITE: 275 return (zfs_vdev_sync_write_min_active); 276 case ZIO_PRIORITY_ASYNC_READ: 277 return (zfs_vdev_async_read_min_active); 278 case ZIO_PRIORITY_ASYNC_WRITE: 279 return (zfs_vdev_async_write_min_active); 280 case ZIO_PRIORITY_SCRUB: 281 return (vq->vq_ia_active == 0 ? zfs_vdev_scrub_min_active : 282 MIN(vq->vq_nia_credit, zfs_vdev_scrub_min_active)); 283 case ZIO_PRIORITY_REMOVAL: 284 return (vq->vq_ia_active == 0 ? zfs_vdev_removal_min_active : 285 MIN(vq->vq_nia_credit, zfs_vdev_removal_min_active)); 286 case ZIO_PRIORITY_INITIALIZING: 287 return (vq->vq_ia_active == 0 ?zfs_vdev_initializing_min_active: 288 MIN(vq->vq_nia_credit, zfs_vdev_initializing_min_active)); 289 case ZIO_PRIORITY_TRIM: 290 return (zfs_vdev_trim_min_active); 291 case ZIO_PRIORITY_REBUILD: 292 return (vq->vq_ia_active == 0 ? zfs_vdev_rebuild_min_active : 293 MIN(vq->vq_nia_credit, zfs_vdev_rebuild_min_active)); 294 default: 295 panic("invalid priority %u", p); 296 return (0); 297 } 298 } 299 300 static uint_t 301 vdev_queue_max_async_writes(spa_t *spa) 302 { 303 uint_t writes; 304 uint64_t dirty = 0; 305 dsl_pool_t *dp = spa_get_dsl(spa); 306 uint64_t min_bytes = zfs_dirty_data_max * 307 zfs_vdev_async_write_active_min_dirty_percent / 100; 308 uint64_t max_bytes = zfs_dirty_data_max * 309 zfs_vdev_async_write_active_max_dirty_percent / 100; 310 311 /* 312 * Async writes may occur before the assignment of the spa's 313 * dsl_pool_t if a self-healing zio is issued prior to the 314 * completion of dmu_objset_open_impl(). 315 */ 316 if (dp == NULL) 317 return (zfs_vdev_async_write_max_active); 318 319 /* 320 * Sync tasks correspond to interactive user actions. To reduce the 321 * execution time of those actions we push data out as fast as possible. 322 */ 323 dirty = dp->dp_dirty_total; 324 if (dirty > max_bytes || spa_has_pending_synctask(spa)) 325 return (zfs_vdev_async_write_max_active); 326 327 if (dirty < min_bytes) 328 return (zfs_vdev_async_write_min_active); 329 330 /* 331 * linear interpolation: 332 * slope = (max_writes - min_writes) / (max_bytes - min_bytes) 333 * move right by min_bytes 334 * move up by min_writes 335 */ 336 writes = (dirty - min_bytes) * 337 (zfs_vdev_async_write_max_active - 338 zfs_vdev_async_write_min_active) / 339 (max_bytes - min_bytes) + 340 zfs_vdev_async_write_min_active; 341 ASSERT3U(writes, >=, zfs_vdev_async_write_min_active); 342 ASSERT3U(writes, <=, zfs_vdev_async_write_max_active); 343 return (writes); 344 } 345 346 static uint_t 347 vdev_queue_class_max_active(vdev_queue_t *vq, zio_priority_t p) 348 { 349 switch (p) { 350 case ZIO_PRIORITY_SYNC_READ: 351 return (zfs_vdev_sync_read_max_active); 352 case ZIO_PRIORITY_SYNC_WRITE: 353 return (zfs_vdev_sync_write_max_active); 354 case ZIO_PRIORITY_ASYNC_READ: 355 return (zfs_vdev_async_read_max_active); 356 case ZIO_PRIORITY_ASYNC_WRITE: 357 return (vdev_queue_max_async_writes(vq->vq_vdev->vdev_spa)); 358 case ZIO_PRIORITY_SCRUB: 359 if (vq->vq_ia_active > 0) { 360 return (MIN(vq->vq_nia_credit, 361 zfs_vdev_scrub_min_active)); 362 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay) 363 return (MAX(1, zfs_vdev_scrub_min_active)); 364 return (zfs_vdev_scrub_max_active); 365 case ZIO_PRIORITY_REMOVAL: 366 if (vq->vq_ia_active > 0) { 367 return (MIN(vq->vq_nia_credit, 368 zfs_vdev_removal_min_active)); 369 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay) 370 return (MAX(1, zfs_vdev_removal_min_active)); 371 return (zfs_vdev_removal_max_active); 372 case ZIO_PRIORITY_INITIALIZING: 373 if (vq->vq_ia_active > 0) { 374 return (MIN(vq->vq_nia_credit, 375 zfs_vdev_initializing_min_active)); 376 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay) 377 return (MAX(1, zfs_vdev_initializing_min_active)); 378 return (zfs_vdev_initializing_max_active); 379 case ZIO_PRIORITY_TRIM: 380 return (zfs_vdev_trim_max_active); 381 case ZIO_PRIORITY_REBUILD: 382 if (vq->vq_ia_active > 0) { 383 return (MIN(vq->vq_nia_credit, 384 zfs_vdev_rebuild_min_active)); 385 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay) 386 return (MAX(1, zfs_vdev_rebuild_min_active)); 387 return (zfs_vdev_rebuild_max_active); 388 default: 389 panic("invalid priority %u", p); 390 return (0); 391 } 392 } 393 394 /* 395 * Return the i/o class to issue from, or ZIO_PRIORITY_NUM_QUEUEABLE if 396 * there is no eligible class. 397 */ 398 static zio_priority_t 399 vdev_queue_class_to_issue(vdev_queue_t *vq) 400 { 401 uint32_t cq = vq->vq_cqueued; 402 zio_priority_t p, p1; 403 404 if (cq == 0 || vq->vq_active >= zfs_vdev_max_active) 405 return (ZIO_PRIORITY_NUM_QUEUEABLE); 406 407 /* 408 * Find a queue that has not reached its minimum # outstanding i/os. 409 * Do round-robin to reduce starvation due to zfs_vdev_max_active 410 * and vq_nia_credit limits. 411 */ 412 p1 = vq->vq_last_prio + 1; 413 if (p1 >= ZIO_PRIORITY_NUM_QUEUEABLE) 414 p1 = 0; 415 for (p = p1; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) { 416 if ((cq & (1U << p)) != 0 && vq->vq_cactive[p] < 417 vdev_queue_class_min_active(vq, p)) 418 goto found; 419 } 420 for (p = 0; p < p1; p++) { 421 if ((cq & (1U << p)) != 0 && vq->vq_cactive[p] < 422 vdev_queue_class_min_active(vq, p)) 423 goto found; 424 } 425 426 /* 427 * If we haven't found a queue, look for one that hasn't reached its 428 * maximum # outstanding i/os. 429 */ 430 for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) { 431 if ((cq & (1U << p)) != 0 && vq->vq_cactive[p] < 432 vdev_queue_class_max_active(vq, p)) 433 break; 434 } 435 436 found: 437 vq->vq_last_prio = p; 438 return (p); 439 } 440 441 void 442 vdev_queue_init(vdev_t *vd) 443 { 444 vdev_queue_t *vq = &vd->vdev_queue; 445 zio_priority_t p; 446 447 vq->vq_vdev = vd; 448 449 for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) { 450 if (vdev_queue_class_fifo(p)) { 451 list_create(&vq->vq_class[p].vqc_list, 452 sizeof (zio_t), 453 offsetof(struct zio, io_queue_node.l)); 454 } else { 455 avl_create(&vq->vq_class[p].vqc_tree, 456 vdev_queue_to_compare, sizeof (zio_t), 457 offsetof(struct zio, io_queue_node.a)); 458 } 459 } 460 avl_create(&vq->vq_read_offset_tree, 461 vdev_queue_offset_compare, sizeof (zio_t), 462 offsetof(struct zio, io_offset_node)); 463 avl_create(&vq->vq_write_offset_tree, 464 vdev_queue_offset_compare, sizeof (zio_t), 465 offsetof(struct zio, io_offset_node)); 466 467 vq->vq_last_offset = 0; 468 list_create(&vq->vq_active_list, sizeof (struct zio), 469 offsetof(struct zio, io_queue_node.l)); 470 mutex_init(&vq->vq_lock, NULL, MUTEX_DEFAULT, NULL); 471 } 472 473 void 474 vdev_queue_fini(vdev_t *vd) 475 { 476 vdev_queue_t *vq = &vd->vdev_queue; 477 478 for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) { 479 if (vdev_queue_class_fifo(p)) 480 list_destroy(&vq->vq_class[p].vqc_list); 481 else 482 avl_destroy(&vq->vq_class[p].vqc_tree); 483 } 484 avl_destroy(&vq->vq_read_offset_tree); 485 avl_destroy(&vq->vq_write_offset_tree); 486 487 list_destroy(&vq->vq_active_list); 488 mutex_destroy(&vq->vq_lock); 489 } 490 491 static void 492 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio) 493 { 494 zio->io_queue_state = ZIO_QS_QUEUED; 495 vdev_queue_class_add(vq, zio); 496 if (zio->io_type == ZIO_TYPE_READ) 497 avl_add(&vq->vq_read_offset_tree, zio); 498 else if (zio->io_type == ZIO_TYPE_WRITE) 499 avl_add(&vq->vq_write_offset_tree, zio); 500 } 501 502 static void 503 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio) 504 { 505 vdev_queue_class_remove(vq, zio); 506 if (zio->io_type == ZIO_TYPE_READ) 507 avl_remove(&vq->vq_read_offset_tree, zio); 508 else if (zio->io_type == ZIO_TYPE_WRITE) 509 avl_remove(&vq->vq_write_offset_tree, zio); 510 zio->io_queue_state = ZIO_QS_NONE; 511 } 512 513 static boolean_t 514 vdev_queue_is_interactive(zio_priority_t p) 515 { 516 switch (p) { 517 case ZIO_PRIORITY_SCRUB: 518 case ZIO_PRIORITY_REMOVAL: 519 case ZIO_PRIORITY_INITIALIZING: 520 case ZIO_PRIORITY_REBUILD: 521 return (B_FALSE); 522 default: 523 return (B_TRUE); 524 } 525 } 526 527 static void 528 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio) 529 { 530 ASSERT(MUTEX_HELD(&vq->vq_lock)); 531 ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); 532 vq->vq_cactive[zio->io_priority]++; 533 vq->vq_active++; 534 if (vdev_queue_is_interactive(zio->io_priority)) { 535 if (++vq->vq_ia_active == 1) 536 vq->vq_nia_credit = 1; 537 } else if (vq->vq_ia_active > 0) { 538 vq->vq_nia_credit--; 539 } 540 zio->io_queue_state = ZIO_QS_ACTIVE; 541 list_insert_tail(&vq->vq_active_list, zio); 542 } 543 544 static void 545 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio) 546 { 547 ASSERT(MUTEX_HELD(&vq->vq_lock)); 548 ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); 549 vq->vq_cactive[zio->io_priority]--; 550 vq->vq_active--; 551 if (vdev_queue_is_interactive(zio->io_priority)) { 552 if (--vq->vq_ia_active == 0) 553 vq->vq_nia_credit = 0; 554 else 555 vq->vq_nia_credit = zfs_vdev_nia_credit; 556 } else if (vq->vq_ia_active == 0) 557 vq->vq_nia_credit++; 558 list_remove(&vq->vq_active_list, zio); 559 zio->io_queue_state = ZIO_QS_NONE; 560 } 561 562 static void 563 vdev_queue_agg_io_done(zio_t *aio) 564 { 565 abd_free(aio->io_abd); 566 } 567 568 /* 569 * Compute the range spanned by two i/os, which is the endpoint of the last 570 * (lio->io_offset + lio->io_size) minus start of the first (fio->io_offset). 571 * Conveniently, the gap between fio and lio is given by -IO_SPAN(lio, fio); 572 * thus fio and lio are adjacent if and only if IO_SPAN(lio, fio) == 0. 573 */ 574 #define IO_SPAN(fio, lio) ((lio)->io_offset + (lio)->io_size - (fio)->io_offset) 575 #define IO_GAP(fio, lio) (-IO_SPAN(lio, fio)) 576 577 /* 578 * Sufficiently adjacent io_offset's in ZIOs will be aggregated. We do this 579 * by creating a gang ABD from the adjacent ZIOs io_abd's. By using 580 * a gang ABD we avoid doing memory copies to and from the parent, 581 * child ZIOs. The gang ABD also accounts for gaps between adjacent 582 * io_offsets by simply getting the zero ABD for writes or allocating 583 * a new ABD for reads and placing them in the gang ABD as well. 584 */ 585 static zio_t * 586 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio) 587 { 588 zio_t *first, *last, *aio, *dio, *mandatory, *nio; 589 uint64_t maxgap = 0; 590 uint64_t size; 591 uint64_t limit; 592 boolean_t stretch = B_FALSE; 593 uint64_t next_offset; 594 abd_t *abd; 595 avl_tree_t *t; 596 597 /* 598 * TRIM aggregation should not be needed since code in zfs_trim.c can 599 * submit TRIM I/O for extents up to zfs_trim_extent_bytes_max (128M). 600 */ 601 if (zio->io_type == ZIO_TYPE_TRIM) 602 return (NULL); 603 604 if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE) 605 return (NULL); 606 607 if (vq->vq_vdev->vdev_nonrot) 608 limit = zfs_vdev_aggregation_limit_non_rotating; 609 else 610 limit = zfs_vdev_aggregation_limit; 611 if (limit == 0) 612 return (NULL); 613 limit = MIN(limit, SPA_MAXBLOCKSIZE); 614 615 /* 616 * I/Os to distributed spares are directly dispatched to the dRAID 617 * leaf vdevs for aggregation. See the comment at the end of the 618 * zio_vdev_io_start() function. 619 */ 620 ASSERT(vq->vq_vdev->vdev_ops != &vdev_draid_spare_ops); 621 622 first = last = zio; 623 624 if (zio->io_type == ZIO_TYPE_READ) { 625 maxgap = zfs_vdev_read_gap_limit; 626 t = &vq->vq_read_offset_tree; 627 } else { 628 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE); 629 t = &vq->vq_write_offset_tree; 630 } 631 632 /* 633 * We can aggregate I/Os that are sufficiently adjacent and of 634 * the same flavor, as expressed by the AGG_INHERIT flags. 635 * The latter requirement is necessary so that certain 636 * attributes of the I/O, such as whether it's a normal I/O 637 * or a scrub/resilver, can be preserved in the aggregate. 638 * We can include optional I/Os, but don't allow them 639 * to begin a range as they add no benefit in that situation. 640 */ 641 642 /* 643 * We keep track of the last non-optional I/O. 644 */ 645 mandatory = (first->io_flags & ZIO_FLAG_OPTIONAL) ? NULL : first; 646 647 /* 648 * Walk backwards through sufficiently contiguous I/Os 649 * recording the last non-optional I/O. 650 */ 651 zio_flag_t flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT; 652 while ((dio = AVL_PREV(t, first)) != NULL && 653 (dio->io_flags & ZIO_FLAG_AGG_INHERIT) == flags && 654 IO_SPAN(dio, last) <= limit && 655 IO_GAP(dio, first) <= maxgap && 656 dio->io_type == zio->io_type) { 657 first = dio; 658 if (mandatory == NULL && !(first->io_flags & ZIO_FLAG_OPTIONAL)) 659 mandatory = first; 660 } 661 662 /* 663 * Skip any initial optional I/Os. 664 */ 665 while ((first->io_flags & ZIO_FLAG_OPTIONAL) && first != last) { 666 first = AVL_NEXT(t, first); 667 ASSERT(first != NULL); 668 } 669 670 671 /* 672 * Walk forward through sufficiently contiguous I/Os. 673 * The aggregation limit does not apply to optional i/os, so that 674 * we can issue contiguous writes even if they are larger than the 675 * aggregation limit. 676 */ 677 while ((dio = AVL_NEXT(t, last)) != NULL && 678 (dio->io_flags & ZIO_FLAG_AGG_INHERIT) == flags && 679 (IO_SPAN(first, dio) <= limit || 680 (dio->io_flags & ZIO_FLAG_OPTIONAL)) && 681 IO_SPAN(first, dio) <= SPA_MAXBLOCKSIZE && 682 IO_GAP(last, dio) <= maxgap && 683 dio->io_type == zio->io_type) { 684 last = dio; 685 if (!(last->io_flags & ZIO_FLAG_OPTIONAL)) 686 mandatory = last; 687 } 688 689 /* 690 * Now that we've established the range of the I/O aggregation 691 * we must decide what to do with trailing optional I/Os. 692 * For reads, there's nothing to do. While we are unable to 693 * aggregate further, it's possible that a trailing optional 694 * I/O would allow the underlying device to aggregate with 695 * subsequent I/Os. We must therefore determine if the next 696 * non-optional I/O is close enough to make aggregation 697 * worthwhile. 698 */ 699 if (zio->io_type == ZIO_TYPE_WRITE && mandatory != NULL) { 700 zio_t *nio = last; 701 while ((dio = AVL_NEXT(t, nio)) != NULL && 702 IO_GAP(nio, dio) == 0 && 703 IO_GAP(mandatory, dio) <= zfs_vdev_write_gap_limit) { 704 nio = dio; 705 if (!(nio->io_flags & ZIO_FLAG_OPTIONAL)) { 706 stretch = B_TRUE; 707 break; 708 } 709 } 710 } 711 712 if (stretch) { 713 /* 714 * We are going to include an optional io in our aggregated 715 * span, thus closing the write gap. Only mandatory i/os can 716 * start aggregated spans, so make sure that the next i/o 717 * after our span is mandatory. 718 */ 719 dio = AVL_NEXT(t, last); 720 ASSERT3P(dio, !=, NULL); 721 dio->io_flags &= ~ZIO_FLAG_OPTIONAL; 722 } else { 723 /* do not include the optional i/o */ 724 while (last != mandatory && last != first) { 725 ASSERT(last->io_flags & ZIO_FLAG_OPTIONAL); 726 last = AVL_PREV(t, last); 727 ASSERT(last != NULL); 728 } 729 } 730 731 if (first == last) 732 return (NULL); 733 734 size = IO_SPAN(first, last); 735 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 736 737 abd = abd_alloc_gang(); 738 if (abd == NULL) 739 return (NULL); 740 741 aio = zio_vdev_delegated_io(first->io_vd, first->io_offset, 742 abd, size, first->io_type, zio->io_priority, 743 flags | ZIO_FLAG_DONT_QUEUE, vdev_queue_agg_io_done, NULL); 744 aio->io_timestamp = first->io_timestamp; 745 746 nio = first; 747 next_offset = first->io_offset; 748 do { 749 dio = nio; 750 nio = AVL_NEXT(t, dio); 751 ASSERT3P(dio, !=, NULL); 752 zio_add_child(dio, aio); 753 vdev_queue_io_remove(vq, dio); 754 755 if (dio->io_offset != next_offset) { 756 /* allocate a buffer for a read gap */ 757 ASSERT3U(dio->io_type, ==, ZIO_TYPE_READ); 758 ASSERT3U(dio->io_offset, >, next_offset); 759 abd = abd_alloc_for_io( 760 dio->io_offset - next_offset, B_TRUE); 761 abd_gang_add(aio->io_abd, abd, B_TRUE); 762 } 763 if (dio->io_abd && 764 (dio->io_size != abd_get_size(dio->io_abd))) { 765 /* abd size not the same as IO size */ 766 ASSERT3U(abd_get_size(dio->io_abd), >, dio->io_size); 767 abd = abd_get_offset_size(dio->io_abd, 0, dio->io_size); 768 abd_gang_add(aio->io_abd, abd, B_TRUE); 769 } else { 770 if (dio->io_flags & ZIO_FLAG_NODATA) { 771 /* allocate a buffer for a write gap */ 772 ASSERT3U(dio->io_type, ==, ZIO_TYPE_WRITE); 773 ASSERT0P(dio->io_abd); 774 abd_gang_add(aio->io_abd, 775 abd_get_zeros(dio->io_size), B_TRUE); 776 } else { 777 /* 778 * We pass B_FALSE to abd_gang_add() 779 * because we did not allocate a new 780 * ABD, so it is assumed the caller 781 * will free this ABD. 782 */ 783 abd_gang_add(aio->io_abd, dio->io_abd, 784 B_FALSE); 785 } 786 } 787 next_offset = dio->io_offset + dio->io_size; 788 } while (dio != last); 789 ASSERT3U(abd_get_size(aio->io_abd), ==, aio->io_size); 790 791 /* 792 * Callers must call zio_vdev_io_bypass() and zio_execute() for 793 * aggregated (parent) I/Os so that we could avoid dropping the 794 * queue's lock here to avoid a deadlock that we could encounter 795 * due to lock order reversal between vq_lock and io_lock in 796 * zio_change_priority(). 797 */ 798 return (aio); 799 } 800 801 static zio_t * 802 vdev_queue_io_to_issue(vdev_queue_t *vq) 803 { 804 zio_t *zio, *aio; 805 zio_priority_t p; 806 avl_index_t idx; 807 avl_tree_t *tree; 808 809 again: 810 ASSERT(MUTEX_HELD(&vq->vq_lock)); 811 812 p = vdev_queue_class_to_issue(vq); 813 814 if (p == ZIO_PRIORITY_NUM_QUEUEABLE) { 815 /* No eligible queued i/os */ 816 return (NULL); 817 } 818 819 if (vdev_queue_class_fifo(p)) { 820 zio = list_head(&vq->vq_class[p].vqc_list); 821 } else { 822 /* 823 * For LBA-ordered queues (async / scrub / initializing), 824 * issue the I/O which follows the most recently issued I/O 825 * in LBA (offset) order, but to avoid starvation only within 826 * the same 0.5 second interval as the first I/O. 827 */ 828 tree = &vq->vq_class[p].vqc_tree; 829 zio = aio = avl_first(tree); 830 if (zio->io_offset < vq->vq_last_offset) { 831 vq->vq_io_search.io_timestamp = zio->io_timestamp; 832 vq->vq_io_search.io_offset = vq->vq_last_offset; 833 zio = avl_find(tree, &vq->vq_io_search, &idx); 834 if (zio == NULL) { 835 zio = avl_nearest(tree, idx, AVL_AFTER); 836 if (zio == NULL || 837 (zio->io_timestamp >> VDQ_T_SHIFT) != 838 (aio->io_timestamp >> VDQ_T_SHIFT)) 839 zio = aio; 840 } 841 } 842 } 843 ASSERT3U(zio->io_priority, ==, p); 844 845 aio = vdev_queue_aggregate(vq, zio); 846 if (aio != NULL) { 847 zio = aio; 848 } else { 849 vdev_queue_io_remove(vq, zio); 850 851 /* 852 * If the I/O is or was optional and therefore has no data, we 853 * need to simply discard it. We need to drop the vdev queue's 854 * lock to avoid a deadlock that we could encounter since this 855 * I/O will complete immediately. 856 */ 857 if (zio->io_flags & ZIO_FLAG_NODATA) { 858 mutex_exit(&vq->vq_lock); 859 zio_vdev_io_bypass(zio); 860 zio_execute(zio); 861 mutex_enter(&vq->vq_lock); 862 goto again; 863 } 864 } 865 866 vdev_queue_pending_add(vq, zio); 867 vq->vq_last_offset = zio->io_offset + zio->io_size; 868 869 return (zio); 870 } 871 872 boolean_t 873 vdev_should_queue_io(zio_t *zio) 874 { 875 vdev_t *vd = zio->io_vd; 876 boolean_t should_queue = B_TRUE; 877 878 /* 879 * Add zio with ZIO_FLAG_NODATA to queue as bypass code 880 * currently does not handle certain cases (gang abd, raidz 881 * write aggregation). 882 */ 883 if (zio->io_flags & ZIO_FLAG_NODATA) 884 return (B_TRUE); 885 886 switch (vd->vdev_scheduler) { 887 case VDEV_SCHEDULER_AUTO: 888 if (vd->vdev_nonrot && vd->vdev_is_blkdev) 889 should_queue = B_FALSE; 890 break; 891 case VDEV_SCHEDULER_ON: 892 should_queue = B_TRUE; 893 break; 894 case VDEV_SCHEDULER_OFF: 895 should_queue = B_FALSE; 896 break; 897 default: 898 should_queue = B_TRUE; 899 break; 900 } 901 return (should_queue); 902 } 903 904 zio_t * 905 vdev_queue_io(zio_t *zio) 906 { 907 vdev_queue_t *vq = &zio->io_vd->vdev_queue; 908 zio_t *dio, *nio; 909 zio_link_t *zl = NULL; 910 911 if (zio->io_flags & ZIO_FLAG_DONT_QUEUE) 912 return (zio); 913 914 /* 915 * Children i/os inherent their parent's priority, which might 916 * not match the child's i/o type. Fix it up here. 917 */ 918 if (zio->io_type == ZIO_TYPE_READ) { 919 ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM); 920 921 if (zio->io_priority != ZIO_PRIORITY_SYNC_READ && 922 zio->io_priority != ZIO_PRIORITY_ASYNC_READ && 923 zio->io_priority != ZIO_PRIORITY_SCRUB && 924 zio->io_priority != ZIO_PRIORITY_REMOVAL && 925 zio->io_priority != ZIO_PRIORITY_INITIALIZING && 926 zio->io_priority != ZIO_PRIORITY_REBUILD) { 927 zio->io_priority = ZIO_PRIORITY_ASYNC_READ; 928 } 929 } else if (zio->io_type == ZIO_TYPE_WRITE) { 930 ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM); 931 932 if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE && 933 zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE && 934 zio->io_priority != ZIO_PRIORITY_REMOVAL && 935 zio->io_priority != ZIO_PRIORITY_INITIALIZING && 936 zio->io_priority != ZIO_PRIORITY_REBUILD) { 937 zio->io_priority = ZIO_PRIORITY_ASYNC_WRITE; 938 } 939 } else { 940 ASSERT(zio->io_type == ZIO_TYPE_TRIM); 941 ASSERT(zio->io_priority == ZIO_PRIORITY_TRIM); 942 } 943 944 zio->io_flags |= ZIO_FLAG_DONT_QUEUE; 945 zio->io_timestamp = gethrtime(); 946 947 if (!vdev_should_queue_io(zio)) { 948 zio->io_queue_state = ZIO_QS_NONE; 949 zio->io_flags |= ZIO_FLAG_BYPASSED_QUEUE; 950 return (zio); 951 } 952 953 /* 954 * A zio holding a queue slot can not be part of a completion batch, 955 * since the slot would then be released only once the whole batch is 956 * complete. 957 */ 958 zio_t *batch = zio_batch_leave(zio); 959 if (batch != NULL) 960 zio_execute(batch); 961 962 mutex_enter(&vq->vq_lock); 963 vdev_queue_io_add(vq, zio); 964 nio = vdev_queue_io_to_issue(vq); 965 mutex_exit(&vq->vq_lock); 966 967 if (nio == NULL) 968 return (NULL); 969 970 if (nio->io_done == vdev_queue_agg_io_done) { 971 while ((dio = zio_walk_parents(nio, &zl)) != NULL) { 972 ASSERT3U(dio->io_type, ==, nio->io_type); 973 zio_vdev_io_bypass(dio); 974 zio_execute(dio); 975 } 976 zio_nowait(nio); 977 return (NULL); 978 } 979 980 return (nio); 981 } 982 983 void 984 vdev_queue_io_done(zio_t *zio) 985 { 986 vdev_queue_t *vq = &zio->io_vd->vdev_queue; 987 zio_t *dio, *nio; 988 zio_link_t *zl = NULL; 989 990 /* 991 * io_delta is already set if this zio's completion was deferred, so 992 * that the service time excludes however long it waited. Batched 993 * completions are not processed in arrival order, so the resulting 994 * completion time may predate what vq_io_complete_ts holds. 995 */ 996 if (zio->io_delta == 0) 997 zio->io_delta = gethrtime() - zio->io_timestamp; 998 hrtime_t now = zio->io_timestamp + zio->io_delta; 999 if (now > vq->vq_io_complete_ts) { 1000 vq->vq_io_complete_ts = now; 1001 vq->vq_io_delta_ts = zio->io_delta; 1002 } 1003 1004 if (zio->io_queue_state == ZIO_QS_NONE) 1005 return; 1006 1007 mutex_enter(&vq->vq_lock); 1008 vdev_queue_pending_remove(vq, zio); 1009 1010 while ((nio = vdev_queue_io_to_issue(vq)) != NULL) { 1011 mutex_exit(&vq->vq_lock); 1012 if (nio->io_done == vdev_queue_agg_io_done) { 1013 while ((dio = zio_walk_parents(nio, &zl)) != NULL) { 1014 ASSERT3U(dio->io_type, ==, nio->io_type); 1015 zio_vdev_io_bypass(dio); 1016 zio_execute(dio); 1017 } 1018 zio_nowait(nio); 1019 } else { 1020 zio_vdev_io_reissue(nio); 1021 zio_execute(nio); 1022 } 1023 mutex_enter(&vq->vq_lock); 1024 } 1025 1026 mutex_exit(&vq->vq_lock); 1027 } 1028 1029 void 1030 vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority) 1031 { 1032 vdev_queue_t *vq = &zio->io_vd->vdev_queue; 1033 1034 /* 1035 * ZIO_PRIORITY_NOW is used by the vdev cache code and the aggregate zio 1036 * code to issue IOs without adding them to the vdev queue. In this 1037 * case, the zio is already going to be issued as quickly as possible 1038 * and so it doesn't need any reprioritization to help. 1039 */ 1040 if (zio->io_priority == ZIO_PRIORITY_NOW) 1041 return; 1042 1043 ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); 1044 ASSERT3U(priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); 1045 1046 if (zio->io_type == ZIO_TYPE_READ) { 1047 if (priority != ZIO_PRIORITY_SYNC_READ && 1048 priority != ZIO_PRIORITY_ASYNC_READ && 1049 priority != ZIO_PRIORITY_SCRUB) 1050 priority = ZIO_PRIORITY_ASYNC_READ; 1051 } else { 1052 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 1053 if (priority != ZIO_PRIORITY_SYNC_WRITE && 1054 priority != ZIO_PRIORITY_ASYNC_WRITE) 1055 priority = ZIO_PRIORITY_ASYNC_WRITE; 1056 } 1057 1058 mutex_enter(&vq->vq_lock); 1059 1060 /* 1061 * If the zio is in none of the queues we can simply change 1062 * the priority. If the zio is waiting to be submitted we must 1063 * remove it from the queue and re-insert it with the new priority. 1064 * Otherwise, the zio is currently active and we cannot change its 1065 * priority. 1066 */ 1067 if (zio->io_queue_state == ZIO_QS_QUEUED) { 1068 vdev_queue_class_remove(vq, zio); 1069 zio->io_priority = priority; 1070 vdev_queue_class_add(vq, zio); 1071 } else if (zio->io_queue_state == ZIO_QS_NONE) { 1072 zio->io_priority = priority; 1073 } 1074 1075 mutex_exit(&vq->vq_lock); 1076 } 1077 1078 boolean_t 1079 vdev_queue_pool_busy(spa_t *spa) 1080 { 1081 dsl_pool_t *dp = spa_get_dsl(spa); 1082 uint64_t min_bytes = zfs_dirty_data_max * 1083 zfs_vdev_async_write_active_min_dirty_percent / 100; 1084 1085 return (dp->dp_dirty_total > min_bytes); 1086 } 1087 1088 /* 1089 * As these two methods are only used for load calculations we're not 1090 * concerned if we get an incorrect value on 32bit platforms due to lack of 1091 * vq_lock mutex use here, instead we prefer to keep it lock free for 1092 * performance. 1093 */ 1094 uint32_t 1095 vdev_queue_length(vdev_t *vd) 1096 { 1097 return (vd->vdev_queue.vq_active); 1098 } 1099 1100 uint64_t 1101 vdev_queue_last_offset(vdev_t *vd) 1102 { 1103 return (vd->vdev_queue.vq_last_offset); 1104 } 1105 1106 uint64_t 1107 vdev_queue_class_length(vdev_t *vd, zio_priority_t p) 1108 { 1109 vdev_queue_t *vq = &vd->vdev_queue; 1110 if (vdev_queue_class_fifo(p)) 1111 return (vq->vq_class[p].vqc_list_numnodes); 1112 else 1113 return (avl_numnodes(&vq->vq_class[p].vqc_tree)); 1114 } 1115 1116 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, aggregation_limit, UINT, ZMOD_RW, 1117 "Max vdev I/O aggregation size"); 1118 1119 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, aggregation_limit_non_rotating, UINT, 1120 ZMOD_RW, "Max vdev I/O aggregation size for non-rotating media"); 1121 1122 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, read_gap_limit, UINT, ZMOD_RW, 1123 "Aggregate read I/O over gap"); 1124 1125 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, write_gap_limit, UINT, ZMOD_RW, 1126 "Aggregate write I/O over gap"); 1127 1128 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, max_active, UINT, ZMOD_RW, 1129 "Maximum number of active I/Os per vdev"); 1130 1131 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, async_write_active_max_dirty_percent, 1132 UINT, ZMOD_RW, "Async write concurrency max threshold"); 1133 1134 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, async_write_active_min_dirty_percent, 1135 UINT, ZMOD_RW, "Async write concurrency min threshold"); 1136 1137 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, async_read_max_active, UINT, ZMOD_RW, 1138 "Max active async read I/Os per vdev"); 1139 1140 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, async_read_min_active, UINT, ZMOD_RW, 1141 "Min active async read I/Os per vdev"); 1142 1143 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, async_write_max_active, UINT, ZMOD_RW, 1144 "Max active async write I/Os per vdev"); 1145 1146 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, async_write_min_active, UINT, ZMOD_RW, 1147 "Min active async write I/Os per vdev"); 1148 1149 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, initializing_max_active, UINT, ZMOD_RW, 1150 "Max active initializing I/Os per vdev"); 1151 1152 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, initializing_min_active, UINT, ZMOD_RW, 1153 "Min active initializing I/Os per vdev"); 1154 1155 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, removal_max_active, UINT, ZMOD_RW, 1156 "Max active removal I/Os per vdev"); 1157 1158 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, removal_min_active, UINT, ZMOD_RW, 1159 "Min active removal I/Os per vdev"); 1160 1161 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, scrub_max_active, UINT, ZMOD_RW, 1162 "Max active scrub I/Os per vdev"); 1163 1164 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, scrub_min_active, UINT, ZMOD_RW, 1165 "Min active scrub I/Os per vdev"); 1166 1167 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, sync_read_max_active, UINT, ZMOD_RW, 1168 "Max active sync read I/Os per vdev"); 1169 1170 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, sync_read_min_active, UINT, ZMOD_RW, 1171 "Min active sync read I/Os per vdev"); 1172 1173 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, sync_write_max_active, UINT, ZMOD_RW, 1174 "Max active sync write I/Os per vdev"); 1175 1176 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, sync_write_min_active, UINT, ZMOD_RW, 1177 "Min active sync write I/Os per vdev"); 1178 1179 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, trim_max_active, UINT, ZMOD_RW, 1180 "Max active trim/discard I/Os per vdev"); 1181 1182 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, trim_min_active, UINT, ZMOD_RW, 1183 "Min active trim/discard I/Os per vdev"); 1184 1185 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, rebuild_max_active, UINT, ZMOD_RW, 1186 "Max active rebuild I/Os per vdev"); 1187 1188 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, rebuild_min_active, UINT, ZMOD_RW, 1189 "Min active rebuild I/Os per vdev"); 1190 1191 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, nia_credit, UINT, ZMOD_RW, 1192 "Number of non-interactive I/Os to allow in sequence"); 1193 1194 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, nia_delay, UINT, ZMOD_RW, 1195 "Number of non-interactive I/Os before _max_active"); 1196