1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to setting various queue properties from drivers 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/init.h> 8 #include <linux/bio.h> 9 #include <linux/blk-integrity.h> 10 #include <linux/pagemap.h> 11 #include <linux/backing-dev-defs.h> 12 #include <linux/gcd.h> 13 #include <linux/lcm.h> 14 #include <linux/jiffies.h> 15 #include <linux/gfp.h> 16 #include <linux/dma-mapping.h> 17 18 #include "blk.h" 19 #include "blk-rq-qos.h" 20 #include "blk-wbt.h" 21 22 void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout) 23 { 24 WRITE_ONCE(q->rq_timeout, timeout); 25 } 26 EXPORT_SYMBOL_GPL(blk_queue_rq_timeout); 27 28 /** 29 * blk_set_stacking_limits - set default limits for stacking devices 30 * @lim: the queue_limits structure to reset 31 * 32 * Prepare queue limits for applying limits from underlying devices using 33 * blk_stack_limits(). 34 */ 35 void blk_set_stacking_limits(struct queue_limits *lim) 36 { 37 memset(lim, 0, sizeof(*lim)); 38 lim->logical_block_size = SECTOR_SIZE; 39 lim->physical_block_size = SECTOR_SIZE; 40 lim->io_min = SECTOR_SIZE; 41 lim->discard_granularity = SECTOR_SIZE; 42 lim->dma_alignment = SECTOR_SIZE - 1; 43 lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK; 44 45 /* Inherit limits from component devices */ 46 lim->max_segments = USHRT_MAX; 47 lim->max_discard_segments = USHRT_MAX; 48 lim->max_hw_sectors = UINT_MAX; 49 lim->max_segment_size = UINT_MAX; 50 lim->max_sectors = UINT_MAX; 51 lim->max_dev_sectors = UINT_MAX; 52 lim->max_write_zeroes_sectors = UINT_MAX; 53 lim->max_hw_zone_append_sectors = UINT_MAX; 54 lim->max_user_discard_sectors = UINT_MAX; 55 } 56 EXPORT_SYMBOL(blk_set_stacking_limits); 57 58 void blk_apply_bdi_limits(struct backing_dev_info *bdi, 59 struct queue_limits *lim) 60 { 61 /* 62 * For read-ahead of large files to be effective, we need to read ahead 63 * at least twice the optimal I/O size. 64 * 65 * There is no hardware limitation for the read-ahead size and the user 66 * might have increased the read-ahead size through sysfs, so don't ever 67 * decrease it. 68 */ 69 bdi->ra_pages = max3(bdi->ra_pages, 70 lim->io_opt * 2 / PAGE_SIZE, 71 VM_READAHEAD_PAGES); 72 bdi->io_pages = lim->max_sectors >> PAGE_SECTORS_SHIFT; 73 } 74 75 static int blk_validate_zoned_limits(struct queue_limits *lim) 76 { 77 if (!(lim->features & BLK_FEAT_ZONED)) { 78 if (WARN_ON_ONCE(lim->max_open_zones) || 79 WARN_ON_ONCE(lim->max_active_zones) || 80 WARN_ON_ONCE(lim->zone_write_granularity) || 81 WARN_ON_ONCE(lim->max_zone_append_sectors)) 82 return -EINVAL; 83 return 0; 84 } 85 86 if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_BLK_DEV_ZONED))) 87 return -EINVAL; 88 89 /* 90 * Given that active zones include open zones, the maximum number of 91 * open zones cannot be larger than the maximum number of active zones. 92 */ 93 if (lim->max_active_zones && 94 lim->max_open_zones > lim->max_active_zones) 95 return -EINVAL; 96 97 if (lim->zone_write_granularity < lim->logical_block_size) 98 lim->zone_write_granularity = lim->logical_block_size; 99 100 /* 101 * The Zone Append size is limited by the maximum I/O size and the zone 102 * size given that it can't span zones. 103 * 104 * If no max_hw_zone_append_sectors limit is provided, the block layer 105 * will emulated it, else we're also bound by the hardware limit. 106 */ 107 lim->max_zone_append_sectors = 108 min_not_zero(lim->max_hw_zone_append_sectors, 109 min(lim->chunk_sectors, lim->max_hw_sectors)); 110 return 0; 111 } 112 113 static int blk_validate_integrity_limits(struct queue_limits *lim) 114 { 115 struct blk_integrity *bi = &lim->integrity; 116 117 if (!bi->tuple_size) { 118 if (bi->csum_type != BLK_INTEGRITY_CSUM_NONE || 119 bi->tag_size || ((bi->flags & BLK_INTEGRITY_REF_TAG))) { 120 pr_warn("invalid PI settings.\n"); 121 return -EINVAL; 122 } 123 bi->flags |= BLK_INTEGRITY_NOGENERATE | BLK_INTEGRITY_NOVERIFY; 124 return 0; 125 } 126 127 if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) { 128 pr_warn("integrity support disabled.\n"); 129 return -EINVAL; 130 } 131 132 if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE && 133 (bi->flags & BLK_INTEGRITY_REF_TAG)) { 134 pr_warn("ref tag not support without checksum.\n"); 135 return -EINVAL; 136 } 137 138 if (!bi->interval_exp) 139 bi->interval_exp = ilog2(lim->logical_block_size); 140 141 return 0; 142 } 143 144 /* 145 * Returns max guaranteed bytes which we can fit in a bio. 146 * 147 * We request that an atomic_write is ITER_UBUF iov_iter (so a single vector), 148 * so we assume that we can fit in at least PAGE_SIZE in a segment, apart from 149 * the first and last segments. 150 */ 151 static unsigned int blk_queue_max_guaranteed_bio(struct queue_limits *lim) 152 { 153 unsigned int max_segments = min(BIO_MAX_VECS, lim->max_segments); 154 unsigned int length; 155 156 length = min(max_segments, 2) * lim->logical_block_size; 157 if (max_segments > 2) 158 length += (max_segments - 2) * PAGE_SIZE; 159 160 return length; 161 } 162 163 static void blk_atomic_writes_update_limits(struct queue_limits *lim) 164 { 165 unsigned int unit_limit = min(lim->max_hw_sectors << SECTOR_SHIFT, 166 blk_queue_max_guaranteed_bio(lim)); 167 168 unit_limit = rounddown_pow_of_two(unit_limit); 169 170 lim->atomic_write_max_sectors = 171 min(lim->atomic_write_hw_max >> SECTOR_SHIFT, 172 lim->max_hw_sectors); 173 lim->atomic_write_unit_min = 174 min(lim->atomic_write_hw_unit_min, unit_limit); 175 lim->atomic_write_unit_max = 176 min(lim->atomic_write_hw_unit_max, unit_limit); 177 lim->atomic_write_boundary_sectors = 178 lim->atomic_write_hw_boundary >> SECTOR_SHIFT; 179 } 180 181 static void blk_validate_atomic_write_limits(struct queue_limits *lim) 182 { 183 unsigned int boundary_sectors; 184 unsigned int atomic_write_hw_max_sectors = 185 lim->atomic_write_hw_max >> SECTOR_SHIFT; 186 187 if (!(lim->features & BLK_FEAT_ATOMIC_WRITES)) 188 goto unsupported; 189 190 if (!lim->atomic_write_hw_max) 191 goto unsupported; 192 193 if (WARN_ON_ONCE(!is_power_of_2(lim->atomic_write_hw_unit_min))) 194 goto unsupported; 195 196 if (WARN_ON_ONCE(!is_power_of_2(lim->atomic_write_hw_unit_max))) 197 goto unsupported; 198 199 if (WARN_ON_ONCE(lim->atomic_write_hw_unit_min > 200 lim->atomic_write_hw_unit_max)) 201 goto unsupported; 202 203 if (WARN_ON_ONCE(lim->atomic_write_hw_unit_max > 204 lim->atomic_write_hw_max)) 205 goto unsupported; 206 207 if (WARN_ON_ONCE(lim->chunk_sectors && 208 atomic_write_hw_max_sectors > lim->chunk_sectors)) 209 goto unsupported; 210 211 boundary_sectors = lim->atomic_write_hw_boundary >> SECTOR_SHIFT; 212 213 if (boundary_sectors) { 214 if (WARN_ON_ONCE(lim->atomic_write_hw_max > 215 lim->atomic_write_hw_boundary)) 216 goto unsupported; 217 /* 218 * A feature of boundary support is that it disallows bios to 219 * be merged which would result in a merged request which 220 * crosses either a chunk sector or atomic write HW boundary, 221 * even though chunk sectors may be just set for performance. 222 * For simplicity, disallow atomic writes for a chunk sector 223 * which is non-zero and smaller than atomic write HW boundary. 224 * Furthermore, chunk sectors must be a multiple of atomic 225 * write HW boundary. Otherwise boundary support becomes 226 * complicated. 227 * Devices which do not conform to these rules can be dealt 228 * with if and when they show up. 229 */ 230 if (WARN_ON_ONCE(lim->chunk_sectors % boundary_sectors)) 231 goto unsupported; 232 233 /* 234 * The boundary size just needs to be a multiple of unit_max 235 * (and not necessarily a power-of-2), so this following check 236 * could be relaxed in future. 237 * Furthermore, if needed, unit_max could even be reduced so 238 * that it is compliant with a !power-of-2 boundary. 239 */ 240 if (!is_power_of_2(boundary_sectors)) 241 goto unsupported; 242 } 243 244 blk_atomic_writes_update_limits(lim); 245 return; 246 247 unsupported: 248 lim->atomic_write_max_sectors = 0; 249 lim->atomic_write_boundary_sectors = 0; 250 lim->atomic_write_unit_min = 0; 251 lim->atomic_write_unit_max = 0; 252 } 253 254 /* 255 * Check that the limits in lim are valid, initialize defaults for unset 256 * values, and cap values based on others where needed. 257 */ 258 int blk_validate_limits(struct queue_limits *lim) 259 { 260 unsigned int max_hw_sectors; 261 unsigned int logical_block_sectors; 262 unsigned long seg_size; 263 int err; 264 265 /* 266 * Unless otherwise specified, default to 512 byte logical blocks and a 267 * physical block size equal to the logical block size. 268 */ 269 if (!lim->logical_block_size) 270 lim->logical_block_size = SECTOR_SIZE; 271 else if (blk_validate_block_size(lim->logical_block_size)) { 272 pr_warn("Invalid logical block size (%d)\n", lim->logical_block_size); 273 return -EINVAL; 274 } 275 if (lim->physical_block_size < lim->logical_block_size) 276 lim->physical_block_size = lim->logical_block_size; 277 278 /* 279 * The minimum I/O size defaults to the physical block size unless 280 * explicitly overridden. 281 */ 282 if (lim->io_min < lim->physical_block_size) 283 lim->io_min = lim->physical_block_size; 284 285 /* 286 * The optimal I/O size may not be aligned to physical block size 287 * (because it may be limited by dma engines which have no clue about 288 * block size of the disks attached to them), so we round it down here. 289 */ 290 lim->io_opt = round_down(lim->io_opt, lim->physical_block_size); 291 292 /* 293 * max_hw_sectors has a somewhat weird default for historical reason, 294 * but driver really should set their own instead of relying on this 295 * value. 296 * 297 * The block layer relies on the fact that every driver can 298 * handle at lest a page worth of data per I/O, and needs the value 299 * aligned to the logical block size. 300 */ 301 if (!lim->max_hw_sectors) 302 lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS; 303 if (WARN_ON_ONCE(lim->max_hw_sectors < PAGE_SECTORS)) 304 return -EINVAL; 305 logical_block_sectors = lim->logical_block_size >> SECTOR_SHIFT; 306 if (WARN_ON_ONCE(logical_block_sectors > lim->max_hw_sectors)) 307 return -EINVAL; 308 lim->max_hw_sectors = round_down(lim->max_hw_sectors, 309 logical_block_sectors); 310 311 /* 312 * The actual max_sectors value is a complex beast and also takes the 313 * max_dev_sectors value (set by SCSI ULPs) and a user configurable 314 * value into account. The ->max_sectors value is always calculated 315 * from these, so directly setting it won't have any effect. 316 */ 317 max_hw_sectors = min_not_zero(lim->max_hw_sectors, 318 lim->max_dev_sectors); 319 if (lim->max_user_sectors) { 320 if (lim->max_user_sectors < BLK_MIN_SEGMENT_SIZE / SECTOR_SIZE) 321 return -EINVAL; 322 lim->max_sectors = min(max_hw_sectors, lim->max_user_sectors); 323 } else if (lim->io_opt > (BLK_DEF_MAX_SECTORS_CAP << SECTOR_SHIFT)) { 324 lim->max_sectors = 325 min(max_hw_sectors, lim->io_opt >> SECTOR_SHIFT); 326 } else if (lim->io_min > (BLK_DEF_MAX_SECTORS_CAP << SECTOR_SHIFT)) { 327 lim->max_sectors = 328 min(max_hw_sectors, lim->io_min >> SECTOR_SHIFT); 329 } else { 330 lim->max_sectors = min(max_hw_sectors, BLK_DEF_MAX_SECTORS_CAP); 331 } 332 lim->max_sectors = round_down(lim->max_sectors, 333 logical_block_sectors); 334 335 /* 336 * Random default for the maximum number of segments. Driver should not 337 * rely on this and set their own. 338 */ 339 if (!lim->max_segments) 340 lim->max_segments = BLK_MAX_SEGMENTS; 341 342 lim->max_discard_sectors = 343 min(lim->max_hw_discard_sectors, lim->max_user_discard_sectors); 344 345 if (!lim->max_discard_segments) 346 lim->max_discard_segments = 1; 347 348 if (lim->discard_granularity < lim->physical_block_size) 349 lim->discard_granularity = lim->physical_block_size; 350 351 /* 352 * By default there is no limit on the segment boundary alignment, 353 * but if there is one it can't be smaller than the page size as 354 * that would break all the normal I/O patterns. 355 */ 356 if (!lim->seg_boundary_mask) 357 lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK; 358 if (WARN_ON_ONCE(lim->seg_boundary_mask < BLK_MIN_SEGMENT_SIZE - 1)) 359 return -EINVAL; 360 361 /* 362 * Stacking device may have both virtual boundary and max segment 363 * size limit, so allow this setting now, and long-term the two 364 * might need to move out of stacking limits since we have immutable 365 * bvec and lower layer bio splitting is supposed to handle the two 366 * correctly. 367 */ 368 if (lim->virt_boundary_mask) { 369 if (!lim->max_segment_size) 370 lim->max_segment_size = UINT_MAX; 371 } else { 372 /* 373 * The maximum segment size has an odd historic 64k default that 374 * drivers probably should override. Just like the I/O size we 375 * require drivers to at least handle a full page per segment. 376 */ 377 if (!lim->max_segment_size) 378 lim->max_segment_size = BLK_MAX_SEGMENT_SIZE; 379 if (WARN_ON_ONCE(lim->max_segment_size < BLK_MIN_SEGMENT_SIZE)) 380 return -EINVAL; 381 } 382 383 /* setup min segment size for building new segment in fast path */ 384 if (lim->seg_boundary_mask > lim->max_segment_size - 1) 385 seg_size = lim->max_segment_size; 386 else 387 seg_size = lim->seg_boundary_mask + 1; 388 lim->min_segment_size = min_t(unsigned int, seg_size, PAGE_SIZE); 389 390 /* 391 * We require drivers to at least do logical block aligned I/O, but 392 * historically could not check for that due to the separate calls 393 * to set the limits. Once the transition is finished the check 394 * below should be narrowed down to check the logical block size. 395 */ 396 if (!lim->dma_alignment) 397 lim->dma_alignment = SECTOR_SIZE - 1; 398 if (WARN_ON_ONCE(lim->dma_alignment > PAGE_SIZE)) 399 return -EINVAL; 400 401 if (lim->alignment_offset) { 402 lim->alignment_offset &= (lim->physical_block_size - 1); 403 lim->flags &= ~BLK_FLAG_MISALIGNED; 404 } 405 406 if (!(lim->features & BLK_FEAT_WRITE_CACHE)) 407 lim->features &= ~BLK_FEAT_FUA; 408 409 blk_validate_atomic_write_limits(lim); 410 411 err = blk_validate_integrity_limits(lim); 412 if (err) 413 return err; 414 return blk_validate_zoned_limits(lim); 415 } 416 EXPORT_SYMBOL_GPL(blk_validate_limits); 417 418 /* 419 * Set the default limits for a newly allocated queue. @lim contains the 420 * initial limits set by the driver, which could be no limit in which case 421 * all fields are cleared to zero. 422 */ 423 int blk_set_default_limits(struct queue_limits *lim) 424 { 425 /* 426 * Most defaults are set by capping the bounds in blk_validate_limits, 427 * but max_user_discard_sectors is special and needs an explicit 428 * initialization to the max value here. 429 */ 430 lim->max_user_discard_sectors = UINT_MAX; 431 return blk_validate_limits(lim); 432 } 433 434 /** 435 * queue_limits_commit_update - commit an atomic update of queue limits 436 * @q: queue to update 437 * @lim: limits to apply 438 * 439 * Apply the limits in @lim that were obtained from queue_limits_start_update() 440 * and updated by the caller to @q. The caller must have frozen the queue or 441 * ensure that there are no outstanding I/Os by other means. 442 * 443 * Returns 0 if successful, else a negative error code. 444 */ 445 int queue_limits_commit_update(struct request_queue *q, 446 struct queue_limits *lim) 447 { 448 int error; 449 450 error = blk_validate_limits(lim); 451 if (error) 452 goto out_unlock; 453 454 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 455 if (q->crypto_profile && lim->integrity.tag_size) { 456 pr_warn("blk-integrity: Integrity and hardware inline encryption are not supported together.\n"); 457 error = -EINVAL; 458 goto out_unlock; 459 } 460 #endif 461 462 q->limits = *lim; 463 if (q->disk) 464 blk_apply_bdi_limits(q->disk->bdi, lim); 465 out_unlock: 466 mutex_unlock(&q->limits_lock); 467 return error; 468 } 469 EXPORT_SYMBOL_GPL(queue_limits_commit_update); 470 471 /** 472 * queue_limits_commit_update_frozen - commit an atomic update of queue limits 473 * @q: queue to update 474 * @lim: limits to apply 475 * 476 * Apply the limits in @lim that were obtained from queue_limits_start_update() 477 * and updated with the new values by the caller to @q. Freezes the queue 478 * before the update and unfreezes it after. 479 * 480 * Returns 0 if successful, else a negative error code. 481 */ 482 int queue_limits_commit_update_frozen(struct request_queue *q, 483 struct queue_limits *lim) 484 { 485 unsigned int memflags; 486 int ret; 487 488 memflags = blk_mq_freeze_queue(q); 489 ret = queue_limits_commit_update(q, lim); 490 blk_mq_unfreeze_queue(q, memflags); 491 492 return ret; 493 } 494 EXPORT_SYMBOL_GPL(queue_limits_commit_update_frozen); 495 496 /** 497 * queue_limits_set - apply queue limits to queue 498 * @q: queue to update 499 * @lim: limits to apply 500 * 501 * Apply the limits in @lim that were freshly initialized to @q. 502 * To update existing limits use queue_limits_start_update() and 503 * queue_limits_commit_update() instead. 504 * 505 * Returns 0 if successful, else a negative error code. 506 */ 507 int queue_limits_set(struct request_queue *q, struct queue_limits *lim) 508 { 509 mutex_lock(&q->limits_lock); 510 return queue_limits_commit_update(q, lim); 511 } 512 EXPORT_SYMBOL_GPL(queue_limits_set); 513 514 static int queue_limit_alignment_offset(const struct queue_limits *lim, 515 sector_t sector) 516 { 517 unsigned int granularity = max(lim->physical_block_size, lim->io_min); 518 unsigned int alignment = sector_div(sector, granularity >> SECTOR_SHIFT) 519 << SECTOR_SHIFT; 520 521 return (granularity + lim->alignment_offset - alignment) % granularity; 522 } 523 524 static unsigned int queue_limit_discard_alignment( 525 const struct queue_limits *lim, sector_t sector) 526 { 527 unsigned int alignment, granularity, offset; 528 529 if (!lim->max_discard_sectors) 530 return 0; 531 532 /* Why are these in bytes, not sectors? */ 533 alignment = lim->discard_alignment >> SECTOR_SHIFT; 534 granularity = lim->discard_granularity >> SECTOR_SHIFT; 535 536 /* Offset of the partition start in 'granularity' sectors */ 537 offset = sector_div(sector, granularity); 538 539 /* And why do we do this modulus *again* in blkdev_issue_discard()? */ 540 offset = (granularity + alignment - offset) % granularity; 541 542 /* Turn it back into bytes, gaah */ 543 return offset << SECTOR_SHIFT; 544 } 545 546 static unsigned int blk_round_down_sectors(unsigned int sectors, unsigned int lbs) 547 { 548 sectors = round_down(sectors, lbs >> SECTOR_SHIFT); 549 if (sectors < PAGE_SIZE >> SECTOR_SHIFT) 550 sectors = PAGE_SIZE >> SECTOR_SHIFT; 551 return sectors; 552 } 553 554 /* Check if second and later bottom devices are compliant */ 555 static bool blk_stack_atomic_writes_tail(struct queue_limits *t, 556 struct queue_limits *b) 557 { 558 /* We're not going to support different boundary sizes.. yet */ 559 if (t->atomic_write_hw_boundary != b->atomic_write_hw_boundary) 560 return false; 561 562 /* Can't support this */ 563 if (t->atomic_write_hw_unit_min > b->atomic_write_hw_unit_max) 564 return false; 565 566 /* Or this */ 567 if (t->atomic_write_hw_unit_max < b->atomic_write_hw_unit_min) 568 return false; 569 570 t->atomic_write_hw_max = min(t->atomic_write_hw_max, 571 b->atomic_write_hw_max); 572 t->atomic_write_hw_unit_min = max(t->atomic_write_hw_unit_min, 573 b->atomic_write_hw_unit_min); 574 t->atomic_write_hw_unit_max = min(t->atomic_write_hw_unit_max, 575 b->atomic_write_hw_unit_max); 576 return true; 577 } 578 579 /* Check for valid boundary of first bottom device */ 580 static bool blk_stack_atomic_writes_boundary_head(struct queue_limits *t, 581 struct queue_limits *b) 582 { 583 /* 584 * Ensure atomic write boundary is aligned with chunk sectors. Stacked 585 * devices store chunk sectors in t->io_min. 586 */ 587 if (b->atomic_write_hw_boundary > t->io_min && 588 b->atomic_write_hw_boundary % t->io_min) 589 return false; 590 if (t->io_min > b->atomic_write_hw_boundary && 591 t->io_min % b->atomic_write_hw_boundary) 592 return false; 593 594 t->atomic_write_hw_boundary = b->atomic_write_hw_boundary; 595 return true; 596 } 597 598 static void blk_stack_atomic_writes_chunk_sectors(struct queue_limits *t) 599 { 600 unsigned int chunk_bytes; 601 602 if (!t->chunk_sectors) 603 return; 604 605 /* 606 * If chunk sectors is so large that its value in bytes overflows 607 * UINT_MAX, then just shift it down so it definitely will fit. 608 * We don't support atomic writes of such a large size anyway. 609 */ 610 if (check_shl_overflow(t->chunk_sectors, SECTOR_SHIFT, &chunk_bytes)) 611 chunk_bytes = t->chunk_sectors; 612 613 /* 614 * Find values for limits which work for chunk size. 615 * b->atomic_write_hw_unit_{min, max} may not be aligned with chunk 616 * size, as the chunk size is not restricted to a power-of-2. 617 * So we need to find highest power-of-2 which works for the chunk 618 * size. 619 * As an example scenario, we could have t->unit_max = 16K and 620 * t->chunk_sectors = 24KB. For this case, reduce t->unit_max to a 621 * value aligned with both limits, i.e. 8K in this example. 622 */ 623 t->atomic_write_hw_unit_max = min(t->atomic_write_hw_unit_max, 624 max_pow_of_two_factor(chunk_bytes)); 625 626 t->atomic_write_hw_unit_min = min(t->atomic_write_hw_unit_min, 627 t->atomic_write_hw_unit_max); 628 t->atomic_write_hw_max = min(t->atomic_write_hw_max, chunk_bytes); 629 } 630 631 /* Check stacking of first bottom device */ 632 static bool blk_stack_atomic_writes_head(struct queue_limits *t, 633 struct queue_limits *b) 634 { 635 if (b->atomic_write_hw_boundary && 636 !blk_stack_atomic_writes_boundary_head(t, b)) 637 return false; 638 639 t->atomic_write_hw_unit_max = b->atomic_write_hw_unit_max; 640 t->atomic_write_hw_unit_min = b->atomic_write_hw_unit_min; 641 t->atomic_write_hw_max = b->atomic_write_hw_max; 642 return true; 643 } 644 645 static void blk_stack_atomic_writes_limits(struct queue_limits *t, 646 struct queue_limits *b, sector_t start) 647 { 648 if (!(b->features & BLK_FEAT_ATOMIC_WRITES)) 649 goto unsupported; 650 651 if (!b->atomic_write_hw_unit_min) 652 goto unsupported; 653 654 if (!blk_atomic_write_start_sect_aligned(start, b)) 655 goto unsupported; 656 657 /* 658 * If atomic_write_hw_max is set, we have already stacked 1x bottom 659 * device, so check for compliance. 660 */ 661 if (t->atomic_write_hw_max) { 662 if (!blk_stack_atomic_writes_tail(t, b)) 663 goto unsupported; 664 return; 665 } 666 667 if (!blk_stack_atomic_writes_head(t, b)) 668 goto unsupported; 669 blk_stack_atomic_writes_chunk_sectors(t); 670 return; 671 672 unsupported: 673 t->atomic_write_hw_max = 0; 674 t->atomic_write_hw_unit_max = 0; 675 t->atomic_write_hw_unit_min = 0; 676 t->atomic_write_hw_boundary = 0; 677 } 678 679 /** 680 * blk_stack_limits - adjust queue_limits for stacked devices 681 * @t: the stacking driver limits (top device) 682 * @b: the underlying queue limits (bottom, component device) 683 * @start: first data sector within component device 684 * 685 * Description: 686 * This function is used by stacking drivers like MD and DM to ensure 687 * that all component devices have compatible block sizes and 688 * alignments. The stacking driver must provide a queue_limits 689 * struct (top) and then iteratively call the stacking function for 690 * all component (bottom) devices. The stacking function will 691 * attempt to combine the values and ensure proper alignment. 692 * 693 * Returns 0 if the top and bottom queue_limits are compatible. The 694 * top device's block sizes and alignment offsets may be adjusted to 695 * ensure alignment with the bottom device. If no compatible sizes 696 * and alignments exist, -1 is returned and the resulting top 697 * queue_limits will have the misaligned flag set to indicate that 698 * the alignment_offset is undefined. 699 */ 700 int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, 701 sector_t start) 702 { 703 unsigned int top, bottom, alignment, ret = 0; 704 705 t->features |= (b->features & BLK_FEAT_INHERIT_MASK); 706 707 /* 708 * Some feaures need to be supported both by the stacking driver and all 709 * underlying devices. The stacking driver sets these flags before 710 * stacking the limits, and this will clear the flags if any of the 711 * underlying devices does not support it. 712 */ 713 if (!(b->features & BLK_FEAT_NOWAIT)) 714 t->features &= ~BLK_FEAT_NOWAIT; 715 if (!(b->features & BLK_FEAT_POLL)) 716 t->features &= ~BLK_FEAT_POLL; 717 718 t->flags |= (b->flags & BLK_FLAG_MISALIGNED); 719 720 t->max_sectors = min_not_zero(t->max_sectors, b->max_sectors); 721 t->max_user_sectors = min_not_zero(t->max_user_sectors, 722 b->max_user_sectors); 723 t->max_hw_sectors = min_not_zero(t->max_hw_sectors, b->max_hw_sectors); 724 t->max_dev_sectors = min_not_zero(t->max_dev_sectors, b->max_dev_sectors); 725 t->max_write_zeroes_sectors = min(t->max_write_zeroes_sectors, 726 b->max_write_zeroes_sectors); 727 t->max_hw_zone_append_sectors = min(t->max_hw_zone_append_sectors, 728 b->max_hw_zone_append_sectors); 729 730 t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask, 731 b->seg_boundary_mask); 732 t->virt_boundary_mask = min_not_zero(t->virt_boundary_mask, 733 b->virt_boundary_mask); 734 735 t->max_segments = min_not_zero(t->max_segments, b->max_segments); 736 t->max_discard_segments = min_not_zero(t->max_discard_segments, 737 b->max_discard_segments); 738 t->max_integrity_segments = min_not_zero(t->max_integrity_segments, 739 b->max_integrity_segments); 740 741 t->max_segment_size = min_not_zero(t->max_segment_size, 742 b->max_segment_size); 743 744 alignment = queue_limit_alignment_offset(b, start); 745 746 /* Bottom device has different alignment. Check that it is 747 * compatible with the current top alignment. 748 */ 749 if (t->alignment_offset != alignment) { 750 751 top = max(t->physical_block_size, t->io_min) 752 + t->alignment_offset; 753 bottom = max(b->physical_block_size, b->io_min) + alignment; 754 755 /* Verify that top and bottom intervals line up */ 756 if (max(top, bottom) % min(top, bottom)) { 757 t->flags |= BLK_FLAG_MISALIGNED; 758 ret = -1; 759 } 760 } 761 762 t->logical_block_size = max(t->logical_block_size, 763 b->logical_block_size); 764 765 t->physical_block_size = max(t->physical_block_size, 766 b->physical_block_size); 767 768 t->io_min = max(t->io_min, b->io_min); 769 t->io_opt = lcm_not_zero(t->io_opt, b->io_opt); 770 t->dma_alignment = max(t->dma_alignment, b->dma_alignment); 771 772 /* Set non-power-of-2 compatible chunk_sectors boundary */ 773 if (b->chunk_sectors) 774 t->chunk_sectors = gcd(t->chunk_sectors, b->chunk_sectors); 775 776 /* Physical block size a multiple of the logical block size? */ 777 if (t->physical_block_size & (t->logical_block_size - 1)) { 778 t->physical_block_size = t->logical_block_size; 779 t->flags |= BLK_FLAG_MISALIGNED; 780 ret = -1; 781 } 782 783 /* Minimum I/O a multiple of the physical block size? */ 784 if (t->io_min & (t->physical_block_size - 1)) { 785 t->io_min = t->physical_block_size; 786 t->flags |= BLK_FLAG_MISALIGNED; 787 ret = -1; 788 } 789 790 /* Optimal I/O a multiple of the physical block size? */ 791 if (t->io_opt & (t->physical_block_size - 1)) { 792 t->io_opt = 0; 793 t->flags |= BLK_FLAG_MISALIGNED; 794 ret = -1; 795 } 796 797 /* chunk_sectors a multiple of the physical block size? */ 798 if ((t->chunk_sectors << 9) & (t->physical_block_size - 1)) { 799 t->chunk_sectors = 0; 800 t->flags |= BLK_FLAG_MISALIGNED; 801 ret = -1; 802 } 803 804 /* Find lowest common alignment_offset */ 805 t->alignment_offset = lcm_not_zero(t->alignment_offset, alignment) 806 % max(t->physical_block_size, t->io_min); 807 808 /* Verify that new alignment_offset is on a logical block boundary */ 809 if (t->alignment_offset & (t->logical_block_size - 1)) { 810 t->flags |= BLK_FLAG_MISALIGNED; 811 ret = -1; 812 } 813 814 t->max_sectors = blk_round_down_sectors(t->max_sectors, t->logical_block_size); 815 t->max_hw_sectors = blk_round_down_sectors(t->max_hw_sectors, t->logical_block_size); 816 t->max_dev_sectors = blk_round_down_sectors(t->max_dev_sectors, t->logical_block_size); 817 818 /* Discard alignment and granularity */ 819 if (b->discard_granularity) { 820 alignment = queue_limit_discard_alignment(b, start); 821 822 t->max_discard_sectors = min_not_zero(t->max_discard_sectors, 823 b->max_discard_sectors); 824 t->max_hw_discard_sectors = min_not_zero(t->max_hw_discard_sectors, 825 b->max_hw_discard_sectors); 826 t->discard_granularity = max(t->discard_granularity, 827 b->discard_granularity); 828 t->discard_alignment = lcm_not_zero(t->discard_alignment, alignment) % 829 t->discard_granularity; 830 } 831 t->max_secure_erase_sectors = min_not_zero(t->max_secure_erase_sectors, 832 b->max_secure_erase_sectors); 833 t->zone_write_granularity = max(t->zone_write_granularity, 834 b->zone_write_granularity); 835 if (!(t->features & BLK_FEAT_ZONED)) { 836 t->zone_write_granularity = 0; 837 t->max_zone_append_sectors = 0; 838 } 839 blk_stack_atomic_writes_limits(t, b, start); 840 841 return ret; 842 } 843 EXPORT_SYMBOL(blk_stack_limits); 844 845 /** 846 * queue_limits_stack_bdev - adjust queue_limits for stacked devices 847 * @t: the stacking driver limits (top device) 848 * @bdev: the underlying block device (bottom) 849 * @offset: offset to beginning of data within component device 850 * @pfx: prefix to use for warnings logged 851 * 852 * Description: 853 * This function is used by stacking drivers like MD and DM to ensure 854 * that all component devices have compatible block sizes and 855 * alignments. The stacking driver must provide a queue_limits 856 * struct (top) and then iteratively call the stacking function for 857 * all component (bottom) devices. The stacking function will 858 * attempt to combine the values and ensure proper alignment. 859 */ 860 void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev, 861 sector_t offset, const char *pfx) 862 { 863 if (blk_stack_limits(t, bdev_limits(bdev), 864 get_start_sect(bdev) + offset)) 865 pr_notice("%s: Warning: Device %pg is misaligned\n", 866 pfx, bdev); 867 } 868 EXPORT_SYMBOL_GPL(queue_limits_stack_bdev); 869 870 /** 871 * queue_limits_stack_integrity - stack integrity profile 872 * @t: target queue limits 873 * @b: base queue limits 874 * 875 * Check if the integrity profile in the @b can be stacked into the 876 * target @t. Stacking is possible if either: 877 * 878 * a) does not have any integrity information stacked into it yet 879 * b) the integrity profile in @b is identical to the one in @t 880 * 881 * If @b can be stacked into @t, return %true. Else return %false and clear the 882 * integrity information in @t. 883 */ 884 bool queue_limits_stack_integrity(struct queue_limits *t, 885 struct queue_limits *b) 886 { 887 struct blk_integrity *ti = &t->integrity; 888 struct blk_integrity *bi = &b->integrity; 889 890 if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) 891 return true; 892 893 if (ti->flags & BLK_INTEGRITY_STACKED) { 894 if (ti->tuple_size != bi->tuple_size) 895 goto incompatible; 896 if (ti->interval_exp != bi->interval_exp) 897 goto incompatible; 898 if (ti->tag_size != bi->tag_size) 899 goto incompatible; 900 if (ti->csum_type != bi->csum_type) 901 goto incompatible; 902 if ((ti->flags & BLK_INTEGRITY_REF_TAG) != 903 (bi->flags & BLK_INTEGRITY_REF_TAG)) 904 goto incompatible; 905 } else { 906 ti->flags = BLK_INTEGRITY_STACKED; 907 ti->flags |= (bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE) | 908 (bi->flags & BLK_INTEGRITY_REF_TAG); 909 ti->csum_type = bi->csum_type; 910 ti->tuple_size = bi->tuple_size; 911 ti->pi_offset = bi->pi_offset; 912 ti->interval_exp = bi->interval_exp; 913 ti->tag_size = bi->tag_size; 914 } 915 return true; 916 917 incompatible: 918 memset(ti, 0, sizeof(*ti)); 919 return false; 920 } 921 EXPORT_SYMBOL_GPL(queue_limits_stack_integrity); 922 923 /** 924 * blk_set_queue_depth - tell the block layer about the device queue depth 925 * @q: the request queue for the device 926 * @depth: queue depth 927 * 928 */ 929 void blk_set_queue_depth(struct request_queue *q, unsigned int depth) 930 { 931 q->queue_depth = depth; 932 rq_qos_queue_depth_changed(q); 933 } 934 EXPORT_SYMBOL(blk_set_queue_depth); 935 936 int bdev_alignment_offset(struct block_device *bdev) 937 { 938 struct request_queue *q = bdev_get_queue(bdev); 939 940 if (q->limits.flags & BLK_FLAG_MISALIGNED) 941 return -1; 942 if (bdev_is_partition(bdev)) 943 return queue_limit_alignment_offset(&q->limits, 944 bdev->bd_start_sect); 945 return q->limits.alignment_offset; 946 } 947 EXPORT_SYMBOL_GPL(bdev_alignment_offset); 948 949 unsigned int bdev_discard_alignment(struct block_device *bdev) 950 { 951 struct request_queue *q = bdev_get_queue(bdev); 952 953 if (bdev_is_partition(bdev)) 954 return queue_limit_discard_alignment(&q->limits, 955 bdev->bd_start_sect); 956 return q->limits.discard_alignment; 957 } 958 EXPORT_SYMBOL_GPL(bdev_discard_alignment); 959