1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC. 14 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 15 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>. 16 * LLNL-CODE-403049. 17 * Copyright (c) 2012, 2019 by Delphix. All rights reserved. 18 * Copyright (c) 2023, 2024, 2025, Klara, Inc. 19 * Copyright (c) 2026, TrueNAS. 20 */ 21 22 #include <sys/zfs_context.h> 23 #include <sys/spa_impl.h> 24 #include <sys/vdev_disk.h> 25 #include <sys/vdev_impl.h> 26 #include <sys/vdev_trim.h> 27 #include <sys/abd.h> 28 #include <sys/fs/zfs.h> 29 #include <sys/zio.h> 30 #include <linux/blkpg.h> 31 #include <linux/msdos_fs.h> 32 #include <linux/vfs_compat.h> 33 #include <linux/blk-cgroup.h> 34 35 /* 36 * Linux 6.8.x uses a bdev_handle as an instance/refcount for an underlying 37 * block_device. Since it carries the block_device inside, its convenient to 38 * just use the handle as a proxy. 39 * 40 * Linux 6.9.x uses a file for the same purpose. 41 * 42 * For pre-6.8, we just emulate this with a cast, since we don't need any of 43 * the other fields inside the handle. 44 */ 45 #if defined(HAVE_BDEV_OPEN_BY_PATH) 46 typedef struct bdev_handle zfs_bdev_handle_t; 47 #define BDH_BDEV(bdh) ((bdh)->bdev) 48 #define BDH_IS_ERR(bdh) (IS_ERR(bdh)) 49 #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh)) 50 #define BDH_ERR_PTR(err) (ERR_PTR(err)) 51 #elif defined(HAVE_BDEV_FILE_OPEN_BY_PATH) 52 typedef struct file zfs_bdev_handle_t; 53 #define BDH_BDEV(bdh) (file_bdev(bdh)) 54 #define BDH_IS_ERR(bdh) (IS_ERR(bdh)) 55 #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh)) 56 #define BDH_ERR_PTR(err) (ERR_PTR(err)) 57 #else 58 typedef void zfs_bdev_handle_t; 59 #define BDH_BDEV(bdh) ((struct block_device *)bdh) 60 #define BDH_IS_ERR(bdh) (IS_ERR(BDH_BDEV(bdh))) 61 #define BDH_PTR_ERR(bdh) (PTR_ERR(BDH_BDEV(bdh))) 62 #define BDH_ERR_PTR(err) (ERR_PTR(err)) 63 #endif 64 65 typedef struct vdev_disk { 66 zfs_bdev_handle_t *vd_bdh; 67 krwlock_t vd_lock; 68 } vdev_disk_t; 69 70 /* 71 * Maximum number of segments to add to a bio (min 4). If this is higher than 72 * the maximum allowed by the device queue or the kernel itself, it will be 73 * clamped. Setting it to zero will cause the kernel's ideal size to be used. 74 */ 75 uint_t zfs_vdev_disk_max_segs = 0; 76 77 /* 78 * Unique identifier for the exclusive vdev holder. 79 */ 80 static void *zfs_vdev_holder = VDEV_HOLDER; 81 82 /* 83 * Wait up to zfs_vdev_open_timeout_ms milliseconds before determining the 84 * device is missing. The missing path may be transient since the links 85 * can be briefly removed and recreated in response to udev events. 86 */ 87 static uint_t zfs_vdev_open_timeout_ms = 1000; 88 89 /* 90 * Size of the "reserved" partition, in blocks. 91 */ 92 #define EFI_MIN_RESV_SIZE (16 * 1024) 93 94 /* 95 * BIO request failfast mask. 96 */ 97 98 static unsigned int zfs_vdev_failfast_mask = 1; 99 100 /* 101 * Whether we wait for bio to complete. Also requires that 102 * zio has bypassed the vdev queue. May lead to performance 103 * improvements when backing vdev devices are fast and low 104 * latency. May impact performance with certain workloads 105 * when enabled on raidz or draid zpool configurations. 106 */ 107 static unsigned int zfs_vdev_disk_calling_thread_io = 0; 108 109 /* 110 * Convert SPA mode flags into bdev open mode flags. 111 */ 112 #ifdef HAVE_BLK_MODE_T 113 typedef blk_mode_t vdev_bdev_mode_t; 114 #define VDEV_BDEV_MODE_READ BLK_OPEN_READ 115 #define VDEV_BDEV_MODE_WRITE BLK_OPEN_WRITE 116 #define VDEV_BDEV_MODE_EXCL BLK_OPEN_EXCL 117 #define VDEV_BDEV_MODE_MASK (BLK_OPEN_READ|BLK_OPEN_WRITE|BLK_OPEN_EXCL) 118 #else 119 typedef fmode_t vdev_bdev_mode_t; 120 #define VDEV_BDEV_MODE_READ FMODE_READ 121 #define VDEV_BDEV_MODE_WRITE FMODE_WRITE 122 #define VDEV_BDEV_MODE_EXCL FMODE_EXCL 123 #define VDEV_BDEV_MODE_MASK (FMODE_READ|FMODE_WRITE|FMODE_EXCL) 124 #endif 125 126 static vdev_bdev_mode_t 127 vdev_bdev_mode(spa_mode_t smode) 128 { 129 ASSERT3U(smode, !=, SPA_MODE_UNINIT); 130 ASSERT0(smode & ~(SPA_MODE_READ|SPA_MODE_WRITE)); 131 132 vdev_bdev_mode_t bmode = VDEV_BDEV_MODE_EXCL; 133 134 if (smode & SPA_MODE_READ) 135 bmode |= VDEV_BDEV_MODE_READ; 136 137 if (smode & SPA_MODE_WRITE) 138 bmode |= VDEV_BDEV_MODE_WRITE; 139 140 ASSERT(bmode & VDEV_BDEV_MODE_MASK); 141 ASSERT0(bmode & ~VDEV_BDEV_MODE_MASK); 142 143 return (bmode); 144 } 145 146 /* 147 * Returns the usable capacity (in bytes) for the partition or disk. 148 */ 149 static uint64_t 150 bdev_capacity(struct block_device *bdev) 151 { 152 #ifdef HAVE_BDEV_NR_BYTES 153 return (bdev_nr_bytes(bdev)); 154 #else 155 return (i_size_read(bdev->bd_inode)); 156 #endif 157 } 158 159 #if !defined(HAVE_BDEV_WHOLE) 160 static inline struct block_device * 161 bdev_whole(struct block_device *bdev) 162 { 163 return (bdev->bd_contains); 164 } 165 #endif 166 167 #if defined(HAVE_BDEVNAME) 168 #define vdev_bdevname(bdev, name) bdevname(bdev, name) 169 #else 170 static inline void 171 vdev_bdevname(struct block_device *bdev, char *name) 172 { 173 snprintf(name, BDEVNAME_SIZE, "%pg", bdev); 174 } 175 #endif 176 177 /* 178 * Returns the maximum expansion capacity of the block device (in bytes). 179 * 180 * It is possible to expand a vdev when it has been created as a wholedisk 181 * and the containing block device has increased in capacity. Or when the 182 * partition containing the pool has been manually increased in size. 183 * 184 * This function is only responsible for calculating the potential expansion 185 * size so it can be reported by 'zpool list'. The efi_use_whole_disk() is 186 * responsible for verifying the expected partition layout in the wholedisk 187 * case, and updating the partition table if appropriate. Once the partition 188 * size has been increased the additional capacity will be visible using 189 * bdev_capacity(). 190 * 191 * The returned maximum expansion capacity is always expected to be larger, or 192 * at the very least equal, to its usable capacity to prevent overestimating 193 * the pool expandsize. 194 */ 195 static uint64_t 196 bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk) 197 { 198 uint64_t psize; 199 int64_t available; 200 201 if (wholedisk && bdev != bdev_whole(bdev)) { 202 /* 203 * When reporting maximum expansion capacity for a wholedisk 204 * deduct any capacity which is expected to be lost due to 205 * alignment restrictions. Over reporting this value isn't 206 * harmful and would only result in slightly less capacity 207 * than expected post expansion. 208 * The estimated available space may be slightly smaller than 209 * bdev_capacity() for devices where the number of sectors is 210 * not a multiple of the alignment size and the partition layout 211 * is keeping less than PARTITION_END_ALIGNMENT bytes after the 212 * "reserved" EFI partition: in such cases return the device 213 * usable capacity. 214 */ 215 available = bdev_capacity(bdev_whole(bdev)) - 216 ((EFI_MIN_RESV_SIZE + NEW_START_BLOCK + 217 PARTITION_END_ALIGNMENT) << SECTOR_BITS); 218 psize = MAX(available, bdev_capacity(bdev)); 219 } else { 220 psize = bdev_capacity(bdev); 221 } 222 223 return (psize); 224 } 225 226 static void 227 vdev_disk_error(zio_t *zio) 228 { 229 /* 230 * This function can be called in interrupt context, for instance while 231 * handling IRQs coming from a misbehaving disk device; use printk() 232 * which is safe from any context. 233 */ 234 printk(KERN_WARNING "zio pool=%s vdev=%s error=%d type=%d " 235 "offset=%llu size=%llu flags=%llu\n", spa_name(zio->io_spa), 236 zio->io_vd->vdev_path, zio->io_error, zio->io_type, 237 (u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size, 238 zio->io_flags); 239 } 240 241 static void 242 vdev_disk_kobj_evt_post(vdev_t *v) 243 { 244 vdev_disk_t *vd = v->vdev_tsd; 245 if (vd && vd->vd_bdh) { 246 spl_signal_kobj_evt(BDH_BDEV(vd->vd_bdh)); 247 } else { 248 vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n", 249 v->vdev_path); 250 } 251 } 252 253 static zfs_bdev_handle_t * 254 vdev_blkdev_get_by_path(const char *path, spa_mode_t smode, void *holder) 255 { 256 vdev_bdev_mode_t bmode = vdev_bdev_mode(smode); 257 258 #if defined(HAVE_BDEV_FILE_OPEN_BY_PATH) 259 return (bdev_file_open_by_path(path, bmode, holder, NULL)); 260 #elif defined(HAVE_BDEV_OPEN_BY_PATH) 261 return (bdev_open_by_path(path, bmode, holder, NULL)); 262 #elif defined(HAVE_BLKDEV_GET_BY_PATH_4ARG) 263 return (blkdev_get_by_path(path, bmode, holder, NULL)); 264 #else 265 return (blkdev_get_by_path(path, bmode, holder)); 266 #endif 267 } 268 269 static void 270 vdev_blkdev_put(zfs_bdev_handle_t *bdh, spa_mode_t smode, void *holder) 271 { 272 #if defined(HAVE_BDEV_RELEASE) 273 return (bdev_release(bdh)); 274 #elif defined(HAVE_BLKDEV_PUT_HOLDER) 275 return (blkdev_put(BDH_BDEV(bdh), holder)); 276 #elif defined(HAVE_BLKDEV_PUT) 277 return (blkdev_put(BDH_BDEV(bdh), vdev_bdev_mode(smode))); 278 #else 279 fput(bdh); 280 #endif 281 } 282 283 static int 284 vdev_path_backing_permission(struct path *path, int mask) 285 { 286 #if defined(HAVE_IDMAP_MNTIDMAP) 287 return (inode_permission(mnt_idmap(path->mnt), 288 d_backing_inode(path->dentry), mask)); 289 #elif defined(HAVE_IDMAP_USERNS) 290 return (inode_permission(mnt_user_ns(path->mnt), 291 d_backing_inode(path->dentry), mask)); 292 #else 293 return (inode_permission(d_backing_inode(path->dentry), mask)); 294 #endif 295 } 296 297 static int 298 vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize, 299 uint64_t *logical_ashift, uint64_t *physical_ashift, cred_t *cr) 300 { 301 zfs_bdev_handle_t *bdh; 302 spa_mode_t smode = spa_mode(v->vdev_spa); 303 hrtime_t timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms); 304 vdev_disk_t *vd; 305 const cred_t *oldcr = NULL; 306 307 /* Must have a pathname and it must be absolute. */ 308 if (v->vdev_path == NULL || v->vdev_path[0] != '/') { 309 v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL; 310 vdev_dbgmsg(v, "invalid vdev_path"); 311 return (SET_ERROR(EINVAL)); 312 } 313 314 /* 315 * Reopen the device if it is currently open. When expanding a 316 * partition force re-scanning the partition table if userland 317 * did not take care of this already. We need to do this while closed 318 * in order to get an accurate updated block device size. Then 319 * since udev may need to recreate the device links increase the 320 * open retry timeout before reporting the device as unavailable. 321 */ 322 vd = v->vdev_tsd; 323 if (vd) { 324 char disk_name[BDEVNAME_SIZE + 6] = "/dev/"; 325 boolean_t reread_part = B_FALSE; 326 327 /* 328 * Reopening an already-open device, so the caller credential 329 * is irrelevant - we had access to it before, we can assume 330 * we still do. 331 */ 332 oldcr = override_creds(kcred); 333 334 rw_enter(&vd->vd_lock, RW_WRITER); 335 bdh = vd->vd_bdh; 336 vd->vd_bdh = NULL; 337 338 if (bdh) { 339 struct block_device *bdev = BDH_BDEV(bdh); 340 if (v->vdev_expanding && bdev != bdev_whole(bdev)) { 341 vdev_bdevname(bdev_whole(bdev), disk_name + 5); 342 /* 343 * If userland has BLKPG_RESIZE_PARTITION, 344 * then it should have updated the partition 345 * table already. We can detect this by 346 * comparing our current physical size 347 * with that of the device. If they are 348 * the same, then we must not have 349 * BLKPG_RESIZE_PARTITION or it failed to 350 * update the partition table online. We 351 * fallback to rescanning the partition 352 * table from the kernel below. However, 353 * if the capacity already reflects the 354 * updated partition, then we skip 355 * rescanning the partition table here. 356 */ 357 if (v->vdev_psize == bdev_capacity(bdev)) 358 reread_part = B_TRUE; 359 } 360 361 vdev_blkdev_put(bdh, smode, zfs_vdev_holder); 362 } 363 364 if (reread_part) { 365 bdh = vdev_blkdev_get_by_path(disk_name, smode, 366 zfs_vdev_holder); 367 if (!BDH_IS_ERR(bdh)) { 368 int error = 369 vdev_bdev_reread_part(BDH_BDEV(bdh)); 370 vdev_blkdev_put(bdh, smode, zfs_vdev_holder); 371 if (error == 0) { 372 timeout = MSEC2NSEC( 373 zfs_vdev_open_timeout_ms * 2); 374 } 375 } 376 } 377 } else { 378 vd = kmem_zalloc(sizeof (vdev_disk_t), KM_SLEEP); 379 380 rw_init(&vd->vd_lock, NULL, RW_DEFAULT, NULL); 381 rw_enter(&vd->vd_lock, RW_WRITER); 382 383 /* Restrict device access below to caller's permissions. */ 384 oldcr = override_creds(cr); 385 } 386 387 /* 388 * Devices are always opened by the path provided at configuration 389 * time. This means that if the provided path is a udev by-id path 390 * then drives may be re-cabled without an issue. If the provided 391 * path is a udev by-path path, then the physical location information 392 * will be preserved. This can be critical for more complicated 393 * configurations where drives are located in specific physical 394 * locations to maximize the systems tolerance to component failure. 395 * 396 * Alternatively, you can provide your own udev rule to flexibly map 397 * the drives as you see fit. It is not advised that you use the 398 * /dev/[hd]d devices which may be reordered due to probing order. 399 * Devices in the wrong locations will be detected by the higher 400 * level vdev validation. 401 * 402 * The specified paths may be briefly removed and recreated in 403 * response to udev events. This should be exceptionally unlikely 404 * because the zpool command makes every effort to verify these paths 405 * have already settled prior to reaching this point. Therefore, 406 * a ENOENT failure at this point is highly likely to be transient 407 * and it is reasonable to sleep and retry before giving up. In 408 * practice delays have been observed to be on the order of 100ms. 409 * 410 * When ERESTARTSYS is returned it indicates the block device is 411 * a zvol which could not be opened due to the deadlock detection 412 * logic in zvol_open(). Extend the timeout and retry the open 413 * subsequent attempts are expected to eventually succeed. 414 */ 415 416 hrtime_t start = gethrtime(); 417 int err = -ENXIO; 418 while (err != 0 && ((gethrtime() - start) < timeout)) { 419 420 /* 421 * Ensure the caller credential (made live by override_creds() 422 * above) has access to the device node. This will include 423 * checking file permissions and considering the 424 * CAP_DAC_OVERRIDE capability. 425 */ 426 struct path devpath; 427 err = kern_path(v->vdev_path, LOOKUP_FOLLOW, &devpath); 428 if (likely(err == 0)) { 429 int mask = 430 (smode & SPA_MODE_READ ? MAY_READ : 0) | 431 (smode & SPA_MODE_WRITE ? MAY_WRITE : 0); 432 err = vdev_path_backing_permission(&devpath, mask); 433 path_put(&devpath); 434 } 435 436 if (likely(err == 0)) { 437 /* 438 * Device node exists and we have access to it, so try 439 * to open it. 440 */ 441 bdh = vdev_blkdev_get_by_path(v->vdev_path, smode, 442 zfs_vdev_holder); 443 444 if (likely(!BDH_IS_ERR(bdh))) 445 /* Device open, nothing more to consider. */ 446 break; 447 448 err = BDH_PTR_ERR(bdh); 449 } 450 451 ASSERT3U(err, !=, 0); 452 453 if (err == -ENOENT) { 454 /* Device node disappeared, see above comment. */ 455 456 if (v->vdev_removed) { 457 /* 458 * There is no point of waiting since device is 459 * removed explicitly 460 */ 461 break; 462 } 463 464 /* Wait a moment, then retry. */ 465 schedule_timeout_interruptible(MSEC_TO_TICK(10)); 466 continue; 467 } 468 469 if (err == -ERESTARTSYS) { 470 /* zvol deadlock avoided, extend timeout and retry. */ 471 timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10); 472 continue; 473 } 474 475 /* Consider all other errors permanent, no retry. */ 476 break; 477 } 478 479 revert_creds(oldcr); 480 481 if (err != 0) { 482 vdev_dbgmsg(v, "open error=%d timeout=%llu/%llu", -err, 483 (u_longlong_t)(gethrtime() - start), 484 (u_longlong_t)timeout); 485 vd->vd_bdh = NULL; 486 v->vdev_tsd = vd; 487 rw_exit(&vd->vd_lock); 488 return (SET_ERROR(-err)); 489 } 490 491 vd->vd_bdh = bdh; 492 v->vdev_tsd = vd; 493 rw_exit(&vd->vd_lock); 494 495 struct block_device *bdev = BDH_BDEV(vd->vd_bdh); 496 497 /* Determine the physical block size */ 498 int physical_block_size = bdev_physical_block_size(bdev); 499 500 /* Determine the logical block size */ 501 int logical_block_size = bdev_logical_block_size(bdev); 502 503 /* 504 * If the device has a write cache, clear the nowritecache flag, 505 * so that we start issuing flush requests again. 506 */ 507 v->vdev_nowritecache = !zfs_bdev_has_write_cache(bdev); 508 509 /* Set when device reports it supports TRIM. */ 510 v->vdev_has_trim = bdev_discard_supported(bdev); 511 512 /* Set when device reports it supports secure TRIM. */ 513 v->vdev_has_securetrim = bdev_secure_discard_supported(bdev); 514 515 /* Inform the ZIO pipeline that we are non-rotational */ 516 #ifdef HAVE_BLK_QUEUE_ROT 517 v->vdev_nonrot = !blk_queue_rot(bdev_get_queue(bdev)); 518 #else 519 v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev)); 520 #endif 521 522 /* Is backed by a block device. */ 523 v->vdev_is_blkdev = B_TRUE; 524 525 /* Physical volume size in bytes for the partition */ 526 *psize = bdev_capacity(bdev); 527 528 /* Physical volume size in bytes including possible expansion space */ 529 *max_psize = bdev_max_capacity(bdev, v->vdev_wholedisk); 530 531 /* Based on the minimum sector size set the block size */ 532 *physical_ashift = highbit64(MAX(physical_block_size, 533 SPA_MINBLOCKSIZE)) - 1; 534 535 *logical_ashift = highbit64(MAX(logical_block_size, 536 SPA_MINBLOCKSIZE)) - 1; 537 538 return (0); 539 } 540 541 static void 542 vdev_disk_close(vdev_t *v) 543 { 544 vdev_disk_t *vd = v->vdev_tsd; 545 546 if (v->vdev_reopening || vd == NULL) 547 return; 548 549 rw_enter(&vd->vd_lock, RW_WRITER); 550 551 if (vd->vd_bdh != NULL) 552 vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa), 553 zfs_vdev_holder); 554 555 v->vdev_tsd = NULL; 556 557 rw_exit(&vd->vd_lock); 558 rw_destroy(&vd->vd_lock); 559 kmem_free(vd, sizeof (vdev_disk_t)); 560 } 561 562 /* 563 * preempt_schedule_notrace is GPL-only which breaks the ZFS build, so 564 * replace it with preempt_schedule under the following condition: 565 */ 566 #if defined(CONFIG_ARM64) && \ 567 defined(CONFIG_PREEMPTION) && \ 568 defined(CONFIG_BLK_CGROUP) 569 #define preempt_schedule_notrace(x) preempt_schedule(x) 570 #endif 571 572 /* 573 * As for the Linux 5.18 kernel bio_alloc() expects a block_device struct 574 * as an argument removing the need to set it with bio_set_dev(). This 575 * removes the need for all of the following compatibility code. 576 */ 577 #if !defined(HAVE_BIO_ALLOC_4ARG) 578 579 #if defined(CONFIG_BLK_CGROUP) && defined(HAVE_BIO_SET_DEV_GPL_ONLY) 580 /* 581 * The Linux 5.5 kernel updated percpu_ref_tryget() which is inlined by 582 * blkg_tryget() to use rcu_read_lock() instead of rcu_read_lock_sched(). 583 * As a side effect the function was converted to GPL-only. Define our 584 * own version when needed which uses rcu_read_lock_sched(). 585 * 586 * The Linux 5.17 kernel split linux/blk-cgroup.h into a private and a public 587 * part, moving blkg_tryget into the private one. Define our own version. 588 */ 589 #if defined(HAVE_BLKG_TRYGET_GPL_ONLY) || !defined(HAVE_BLKG_TRYGET) 590 static inline bool 591 vdev_blkg_tryget(struct blkcg_gq *blkg) 592 { 593 struct percpu_ref *ref = &blkg->refcnt; 594 unsigned long __percpu *count; 595 bool rc; 596 597 rcu_read_lock_sched(); 598 599 if (__ref_is_percpu(ref, &count)) { 600 this_cpu_inc(*count); 601 rc = true; 602 } else { 603 #ifdef ZFS_PERCPU_REF_COUNT_IN_DATA 604 rc = atomic_long_inc_not_zero(&ref->data->count); 605 #else 606 rc = atomic_long_inc_not_zero(&ref->count); 607 #endif 608 } 609 610 rcu_read_unlock_sched(); 611 612 return (rc); 613 } 614 #else 615 #define vdev_blkg_tryget(bg) blkg_tryget(bg) 616 #endif 617 #ifdef HAVE_BIO_SET_DEV_MACRO 618 /* 619 * The Linux 5.0 kernel updated the bio_set_dev() macro so it calls the 620 * GPL-only bio_associate_blkg() symbol thus inadvertently converting 621 * the entire macro. Provide a minimal version which always assigns the 622 * request queue's root_blkg to the bio. 623 */ 624 static inline void 625 vdev_bio_associate_blkg(struct bio *bio) 626 { 627 #if defined(HAVE_BIO_BDEV_DISK) 628 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 629 #else 630 struct request_queue *q = bio->bi_disk->queue; 631 #endif 632 633 ASSERT3P(q, !=, NULL); 634 ASSERT0P(bio->bi_blkg); 635 636 if (q->root_blkg && vdev_blkg_tryget(q->root_blkg)) 637 bio->bi_blkg = q->root_blkg; 638 } 639 640 #define bio_associate_blkg vdev_bio_associate_blkg 641 #else 642 static inline void 643 vdev_bio_set_dev(struct bio *bio, struct block_device *bdev) 644 { 645 #if defined(HAVE_BIO_BDEV_DISK) 646 struct request_queue *q = bdev->bd_disk->queue; 647 #else 648 struct request_queue *q = bio->bi_disk->queue; 649 #endif 650 bio_clear_flag(bio, BIO_REMAPPED); 651 if (bio->bi_bdev != bdev) 652 bio_clear_flag(bio, BIO_THROTTLED); 653 bio->bi_bdev = bdev; 654 655 ASSERT3P(q, !=, NULL); 656 ASSERT0P(bio->bi_blkg); 657 658 if (q->root_blkg && vdev_blkg_tryget(q->root_blkg)) 659 bio->bi_blkg = q->root_blkg; 660 } 661 #define bio_set_dev vdev_bio_set_dev 662 #endif 663 #endif 664 #endif /* !HAVE_BIO_ALLOC_4ARG */ 665 666 static inline void 667 vdev_submit_bio(struct bio *bio) 668 { 669 struct bio_list *bio_list = current->bio_list; 670 current->bio_list = NULL; 671 (void) submit_bio(bio); 672 current->bio_list = bio_list; 673 } 674 675 static inline void 676 vdev_submit_bio_wait(struct bio *bio) 677 { 678 struct bio_list *bio_list = current->bio_list; 679 current->bio_list = NULL; 680 (void) submit_bio_wait(bio); 681 current->bio_list = bio_list; 682 } 683 684 static inline struct bio * 685 vdev_bio_alloc(struct block_device *bdev, gfp_t gfp_mask, 686 unsigned short nr_vecs) 687 { 688 struct bio *bio; 689 690 #ifdef HAVE_BIO_ALLOC_4ARG 691 bio = bio_alloc(bdev, nr_vecs, 0, gfp_mask); 692 #else 693 bio = bio_alloc(gfp_mask, nr_vecs); 694 if (likely(bio != NULL)) 695 bio_set_dev(bio, bdev); 696 #endif 697 698 return (bio); 699 } 700 701 static inline uint_t 702 vdev_bio_max_segs(struct block_device *bdev) 703 { 704 /* 705 * Smallest of the device max segs and the tunable max segs. Minimum 706 * 4, so there's room to finish split pages if they come up. 707 */ 708 const uint_t dev_max_segs = queue_max_segments(bdev_get_queue(bdev)); 709 const uint_t tune_max_segs = (zfs_vdev_disk_max_segs > 0) ? 710 MAX(4, zfs_vdev_disk_max_segs) : dev_max_segs; 711 const uint_t max_segs = MIN(tune_max_segs, dev_max_segs); 712 713 #ifdef HAVE_BIO_MAX_SEGS 714 return (bio_max_segs(max_segs)); 715 #else 716 return (MIN(max_segs, BIO_MAX_PAGES)); 717 #endif 718 } 719 720 static inline uint_t 721 vdev_bio_max_bytes(struct block_device *bdev) 722 { 723 return (queue_max_sectors(bdev_get_queue(bdev)) << 9); 724 } 725 726 727 /* 728 * Virtual block IO object (VBIO) 729 * 730 * Linux block IO (BIO) objects have a limit on how many data segments (pages) 731 * they can hold. Depending on how they're allocated and structured, a large 732 * ZIO can require more than one BIO to be submitted to the kernel, which then 733 * all have to complete before we can return the completed ZIO back to ZFS. 734 * 735 * A VBIO is a wrapper around multiple BIOs, carrying everything needed to 736 * translate a ZIO down into the kernel block layer and back again. 737 * 738 * Note that these are only used for data ZIOs (read/write). Meta-operations 739 * (flush/trim) don't need multiple BIOs and so can just make the call 740 * directly. 741 */ 742 typedef struct { 743 zio_t *vbio_zio; /* parent zio */ 744 745 struct block_device *vbio_bdev; /* blockdev to submit bios to */ 746 747 abd_t *vbio_abd; /* abd carrying borrowed linear buf */ 748 749 uint_t vbio_max_segs; /* max segs per bio */ 750 751 uint_t vbio_max_bytes; /* max bytes per bio */ 752 uint_t vbio_lbs_mask; /* logical block size mask */ 753 754 uint64_t vbio_offset; /* start offset of next bio */ 755 756 struct bio *vbio_bio; /* pointer to the current bio */ 757 int vbio_flags; /* bio flags */ 758 boolean_t vbio_wait; /* wait for completion */ 759 } vbio_t; 760 761 static vbio_t * 762 vbio_alloc(zio_t *zio, struct block_device *bdev, int flags) 763 { 764 vbio_t *vbio = kmem_zalloc(sizeof (vbio_t), KM_SLEEP); 765 766 vbio->vbio_zio = zio; 767 vbio->vbio_bdev = bdev; 768 vbio->vbio_abd = NULL; 769 vbio->vbio_max_segs = vdev_bio_max_segs(bdev); 770 vbio->vbio_max_bytes = vdev_bio_max_bytes(bdev); 771 vbio->vbio_lbs_mask = ~(bdev_logical_block_size(bdev)-1); 772 vbio->vbio_offset = zio->io_offset; 773 vbio->vbio_bio = NULL; 774 vbio->vbio_flags = flags; 775 vbio->vbio_wait = B_FALSE; 776 777 return (vbio); 778 } 779 780 static void vbio_completion(struct bio *bio); 781 782 static int 783 vbio_add_page(vbio_t *vbio, struct page *page, uint_t size, uint_t offset) 784 { 785 struct bio *bio = vbio->vbio_bio; 786 uint_t ssize; 787 788 while (size > 0) { 789 if (bio == NULL) { 790 /* New BIO, allocate and set up */ 791 bio = vdev_bio_alloc(vbio->vbio_bdev, GFP_NOIO, 792 vbio->vbio_max_segs); 793 VERIFY(bio); 794 795 BIO_BI_SECTOR(bio) = vbio->vbio_offset >> 9; 796 bio_set_op_attrs(bio, 797 vbio->vbio_zio->io_type == ZIO_TYPE_WRITE ? 798 WRITE : READ, vbio->vbio_flags); 799 800 if (vbio->vbio_bio) { 801 bio_chain(vbio->vbio_bio, bio); 802 vdev_submit_bio(vbio->vbio_bio); 803 } 804 vbio->vbio_bio = bio; 805 } 806 807 /* 808 * Only load as much of the current page data as will fit in 809 * the space left in the BIO, respecting lbs alignment. Older 810 * kernels will error if we try to overfill the BIO, while 811 * newer ones will accept it and split the BIO. This ensures 812 * everything works on older kernels, and avoids an additional 813 * overhead on the new. 814 */ 815 ssize = MIN(size, (vbio->vbio_max_bytes - BIO_BI_SIZE(bio)) & 816 vbio->vbio_lbs_mask); 817 if (ssize > 0 && 818 bio_add_page(bio, page, ssize, offset) == ssize) { 819 /* Accepted, adjust and load any remaining. */ 820 size -= ssize; 821 offset += ssize; 822 continue; 823 } 824 825 /* No room, set up for a new BIO and loop */ 826 vbio->vbio_offset += BIO_BI_SIZE(bio); 827 828 /* Signal new BIO allocation wanted */ 829 bio = NULL; 830 } 831 832 return (0); 833 } 834 835 /* Iterator callback to submit ABD pages to the vbio. */ 836 static int 837 vbio_fill_cb(struct page *page, size_t off, size_t len, void *priv) 838 { 839 vbio_t *vbio = priv; 840 return (vbio_add_page(vbio, page, len, off)); 841 } 842 843 /* Create some BIOs, fill them with data and submit them */ 844 static void 845 vbio_submit(vbio_t *vbio, abd_t *abd, uint64_t size) 846 { 847 /* 848 * We plug so we can submit the BIOs as we go and only unplug them when 849 * they are fully created and submitted. This is important; if we don't 850 * plug, then the kernel may start executing earlier BIOs while we're 851 * still creating and executing later ones, and if the device goes 852 * away while that's happening, older kernels can get confused and 853 * trample memory. 854 */ 855 struct blk_plug plug; 856 blk_start_plug(&plug); 857 858 (void) abd_iterate_page_func(abd, 0, size, vbio_fill_cb, vbio); 859 ASSERT(vbio->vbio_bio); 860 861 /* 862 * Once submitted, vbio_bio now owns vbio (through bi_private) and we 863 * can't touch it again. The bio may complete and vbio_completion() be 864 * called and free the vbio before this task is run again, so we must 865 * consider it invalid from this point. 866 */ 867 868 if (vbio->vbio_wait) { 869 vdev_submit_bio_wait(vbio->vbio_bio); 870 } else { 871 vbio->vbio_bio->bi_end_io = vbio_completion; 872 vbio->vbio_bio->bi_private = vbio; 873 vdev_submit_bio(vbio->vbio_bio); 874 } 875 876 blk_finish_plug(&plug); 877 } 878 879 /* IO completion callback */ 880 static void 881 vbio_completion(struct bio *bio) 882 { 883 vbio_t *vbio = bio->bi_private; 884 zio_t *zio = vbio->vbio_zio; 885 886 ASSERT(zio); 887 888 /* Capture and log any errors */ 889 zio->io_error = bi_status_to_errno(bio->bi_status); 890 ASSERT3U(zio->io_error, >=, 0); 891 892 if (zio->io_error) 893 vdev_disk_error(zio); 894 895 /* Return the BIO to the kernel */ 896 bio_put(bio); 897 898 /* 899 * We're likely in an interrupt context so we can't do ABD/memory work 900 * here; instead we stash vbio on the zio and take care of it in the 901 * done callback. 902 */ 903 ASSERT0P(zio->io_bio); 904 zio->io_bio = vbio; 905 906 /* Using calling thread io, don't dispatch zio. */ 907 if (vbio->vbio_wait) 908 zio_execute(zio); 909 else 910 zio_delay_interrupt(zio); 911 912 } 913 914 /* 915 * Iterator callback to count ABD pages and check their size & alignment. 916 * 917 * On Linux, each BIO segment can take a page pointer, and an offset+length of 918 * the data within that page. A page can be arbitrarily large ("compound" 919 * pages) but we still have to ensure the data portion is correctly sized and 920 * aligned to the logical block size, to ensure that if the kernel wants to 921 * split the BIO, the two halves will still be properly aligned. 922 * 923 * NOTE: if you change this function, change the copy in 924 * tests/zfs-tests/tests/functional/vdev_disk/page_alignment.c, and add test 925 * data there to validate the change you're making. 926 */ 927 typedef struct { 928 size_t blocksize; 929 int seen_first; 930 int seen_last; 931 } vdev_disk_check_alignment_t; 932 933 static int 934 vdev_disk_check_alignment_cb(struct page *page, size_t off, size_t len, 935 void *priv) 936 { 937 (void) page; 938 vdev_disk_check_alignment_t *s = priv; 939 940 /* 941 * The cardinal rule: a single on-disk block must never cross an 942 * physical (order-0) page boundary, as the kernel expects to be able 943 * to split at both LBS and page boundaries. 944 * 945 * This implies various alignment rules for the blocks in this 946 * (possibly compound) page, which we can check for. 947 */ 948 949 /* 950 * If the previous page did not end on a page boundary, then we 951 * can't proceed without creating a hole. 952 */ 953 if (s->seen_last) 954 return (1); 955 956 /* This page must contain only whole LBS-sized blocks. */ 957 if (!IS_P2ALIGNED(len, s->blocksize)) 958 return (1); 959 960 /* 961 * If this is not the first page in the ABD, then the data must start 962 * on a page-aligned boundary (so the kernel can split on page 963 * boundaries without having to deal with a hole). If it is, then 964 * it can start on LBS-alignment. 965 */ 966 if (s->seen_first) { 967 if (!IS_P2ALIGNED(off, PAGESIZE)) 968 return (1); 969 } else { 970 if (!IS_P2ALIGNED(off, s->blocksize)) 971 return (1); 972 s->seen_first = 1; 973 } 974 975 /* 976 * If this data does not end on a page-aligned boundary, then this 977 * must be the last page in the ABD, for the same reason. 978 */ 979 s->seen_last = !IS_P2ALIGNED(off+len, PAGESIZE); 980 981 return (0); 982 } 983 984 /* 985 * Check if we can submit the pages in this ABD to the kernel as-is. Returns 986 * the number of pages, or 0 if it can't be submitted like this. 987 */ 988 static boolean_t 989 vdev_disk_check_alignment(abd_t *abd, uint64_t size, struct block_device *bdev) 990 { 991 vdev_disk_check_alignment_t s = { 992 .blocksize = bdev_logical_block_size(bdev), 993 }; 994 995 if (abd_iterate_page_func(abd, 0, size, 996 vdev_disk_check_alignment_cb, &s)) 997 return (B_FALSE); 998 999 return (B_TRUE); 1000 } 1001 1002 static int 1003 vdev_disk_io_rw(zio_t *zio) 1004 { 1005 vdev_t *v = zio->io_vd; 1006 vdev_disk_t *vd = v->vdev_tsd; 1007 struct block_device *bdev = BDH_BDEV(vd->vd_bdh); 1008 int flags = 0; 1009 1010 /* 1011 * Accessing outside the block device is never allowed. 1012 */ 1013 if (zio->io_offset + zio->io_size > bdev_capacity(bdev)) { 1014 vdev_dbgmsg(zio->io_vd, 1015 "Illegal access %llu size %llu, device size %llu", 1016 (u_longlong_t)zio->io_offset, 1017 (u_longlong_t)zio->io_size, 1018 (u_longlong_t)bdev_capacity(bdev)); 1019 return (SET_ERROR(EIO)); 1020 } 1021 1022 vdev_t *iter = v; 1023 while (iter != NULL && iter->vdev_failfast == ZPROP_BOOLEAN_INHERIT) 1024 iter = iter->vdev_parent; 1025 1026 boolean_t failfast = iter ? iter->vdev_failfast == 1 : 1027 vdev_prop_default_numeric(VDEV_PROP_FAILFAST); 1028 if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)) && 1029 failfast) { 1030 bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1, 1031 zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4); 1032 } 1033 1034 /* 1035 * Check alignment of the incoming ABD. If any part of it would require 1036 * submitting a page that is not aligned to both the logical block size 1037 * and the page size, then we take a copy into a new memory region with 1038 * correct alignment. This should be impossible on a 512b LBS. On 1039 * larger blocks, this can happen at least when a small number of 1040 * blocks (usually 1) are allocated from a shared slab, or when 1041 * abnormally-small data regions (eg gang headers) are mixed into the 1042 * same ABD as larger allocations (eg aggregations). 1043 */ 1044 abd_t *abd = zio->io_abd; 1045 if (!vdev_disk_check_alignment(abd, zio->io_size, bdev)) { 1046 /* Allocate a new memory region with guaranteed alignment */ 1047 abd = abd_alloc_for_io(zio->io_size, 1048 zio->io_abd->abd_flags & ABD_FLAG_META); 1049 1050 /* If we're writing copy our data into it */ 1051 if (zio->io_type == ZIO_TYPE_WRITE) 1052 abd_copy(abd, zio->io_abd, zio->io_size); 1053 1054 /* 1055 * False here would mean the new allocation has an invalid 1056 * alignment too, which would mean that abd_alloc() is not 1057 * guaranteeing this, or our logic in 1058 * vdev_disk_check_alignment() is wrong. In either case, 1059 * something in seriously wrong and its not safe to continue. 1060 */ 1061 VERIFY(vdev_disk_check_alignment(abd, zio->io_size, bdev)); 1062 } 1063 1064 /* Allocate vbio, with a pointer to the borrowed ABD if necessary */ 1065 vbio_t *vbio = vbio_alloc(zio, bdev, flags); 1066 if (abd != zio->io_abd) 1067 vbio->vbio_abd = abd; 1068 1069 boolean_t bio_wait = B_FALSE; 1070 if (zfs_vdev_disk_calling_thread_io && 1071 (zio->io_flags & ZIO_FLAG_BYPASSED_QUEUE)) { 1072 vbio->vbio_wait = bio_wait = B_TRUE; 1073 } 1074 /* Fill it with data pages and submit it to the kernel */ 1075 vbio_submit(vbio, abd, zio->io_size); 1076 1077 if (bio_wait) { 1078 vbio->vbio_bio->bi_private = vbio; 1079 vbio_completion(vbio->vbio_bio); 1080 } 1081 1082 return (0); 1083 } 1084 1085 static void 1086 vdev_disk_io_flush_completion(struct bio *bio) 1087 { 1088 zio_t *zio = bio->bi_private; 1089 zio->io_error = bi_status_to_errno(bio->bi_status); 1090 if (zio->io_error == EOPNOTSUPP || zio->io_error == ENOTTY) 1091 zio->io_error = SET_ERROR(ENOTSUP); 1092 1093 bio_put(bio); 1094 ASSERT3S(zio->io_error, >=, 0); 1095 if (zio->io_error) 1096 vdev_disk_error(zio); 1097 zio_interrupt(zio); 1098 } 1099 1100 static int 1101 vdev_disk_io_flush(struct block_device *bdev, zio_t *zio) 1102 { 1103 struct request_queue *q; 1104 struct bio *bio; 1105 1106 q = bdev_get_queue(bdev); 1107 if (!q) 1108 return (SET_ERROR(ENXIO)); 1109 1110 bio = vdev_bio_alloc(bdev, GFP_NOIO, 0); 1111 if (unlikely(bio == NULL)) 1112 return (SET_ERROR(ENOMEM)); 1113 1114 bio->bi_end_io = vdev_disk_io_flush_completion; 1115 bio->bi_private = zio; 1116 bio_set_flush(bio); 1117 vdev_submit_bio(bio); 1118 invalidate_bdev(bdev); 1119 1120 return (0); 1121 } 1122 1123 static void 1124 vdev_disk_discard_end_io(struct bio *bio) 1125 { 1126 zio_t *zio = bio->bi_private; 1127 zio->io_error = bi_status_to_errno(bio->bi_status); 1128 1129 bio_put(bio); 1130 if (zio->io_error) 1131 vdev_disk_error(zio); 1132 zio_interrupt(zio); 1133 } 1134 1135 /* 1136 * Wrappers for the different secure erase and discard APIs. We use async 1137 * when available; in this case, *biop is set to the last bio in the chain. 1138 */ 1139 static int 1140 vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector, 1141 sector_t nsect, struct bio **biop) 1142 { 1143 *biop = NULL; 1144 int error; 1145 1146 #if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) 1147 error = blkdev_issue_secure_erase(BDH_BDEV(bdh), 1148 sector, nsect, GFP_NOFS); 1149 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS) 1150 error = __blkdev_issue_discard(BDH_BDEV(bdh), 1151 sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop); 1152 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS) 1153 error = blkdev_issue_discard(BDH_BDEV(bdh), 1154 sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE); 1155 #else 1156 #error "unsupported kernel" 1157 #endif 1158 1159 return (error); 1160 } 1161 1162 static int 1163 vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector, 1164 sector_t nsect, struct bio **biop) 1165 { 1166 *biop = NULL; 1167 int error; 1168 1169 #if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS) 1170 error = __blkdev_issue_discard(BDH_BDEV(bdh), 1171 sector, nsect, GFP_NOFS, 0, biop); 1172 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS) 1173 error = __blkdev_issue_discard(BDH_BDEV(bdh), 1174 sector, nsect, GFP_NOFS, biop); 1175 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS) 1176 error = blkdev_issue_discard(BDH_BDEV(bdh), 1177 sector, nsect, GFP_NOFS, 0); 1178 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS) 1179 error = blkdev_issue_discard(BDH_BDEV(bdh), 1180 sector, nsect, GFP_NOFS); 1181 #else 1182 #error "unsupported kernel" 1183 #endif 1184 1185 return (error); 1186 } 1187 1188 /* 1189 * Entry point for TRIM ops. This calls the right wrapper for secure erase or 1190 * discard, and then does the appropriate finishing work for error vs success 1191 * and async vs sync. 1192 */ 1193 static int 1194 vdev_disk_io_trim(zio_t *zio) 1195 { 1196 int error; 1197 struct bio *bio; 1198 1199 zfs_bdev_handle_t *bdh = ((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh; 1200 sector_t sector = zio->io_offset >> 9; 1201 sector_t nsects = zio->io_size >> 9; 1202 1203 if (zio->io_trim_flags & ZIO_TRIM_SECURE) 1204 error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio); 1205 else 1206 error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio); 1207 1208 if (error != 0) 1209 return (SET_ERROR(-error)); 1210 1211 if (bio == NULL) { 1212 /* 1213 * This was a synchronous op that completed successfully, so 1214 * return it to ZFS immediately. 1215 */ 1216 zio_interrupt(zio); 1217 } else { 1218 /* 1219 * This was an asynchronous op; set up completion callback and 1220 * issue it. 1221 */ 1222 bio->bi_private = zio; 1223 bio->bi_end_io = vdev_disk_discard_end_io; 1224 vdev_submit_bio(bio); 1225 } 1226 1227 return (0); 1228 } 1229 1230 static void 1231 vdev_disk_io_start(zio_t *zio) 1232 { 1233 vdev_t *v = zio->io_vd; 1234 vdev_disk_t *vd = v->vdev_tsd; 1235 int error; 1236 1237 /* 1238 * If the vdev is closed, it's likely in the REMOVED or FAULTED state. 1239 * Nothing to be done here but return failure. 1240 */ 1241 if (vd == NULL) { 1242 zio->io_error = ENXIO; 1243 zio_interrupt(zio); 1244 return; 1245 } 1246 1247 rw_enter(&vd->vd_lock, RW_READER); 1248 1249 /* 1250 * If the vdev is closed, it's likely due to a failed reopen and is 1251 * in the UNAVAIL state. Nothing to be done here but return failure. 1252 */ 1253 if (vd->vd_bdh == NULL) { 1254 rw_exit(&vd->vd_lock); 1255 zio->io_error = ENXIO; 1256 zio_interrupt(zio); 1257 return; 1258 } 1259 1260 switch (zio->io_type) { 1261 case ZIO_TYPE_FLUSH: 1262 1263 if (!vdev_readable(v)) { 1264 /* Drive not there, can't flush */ 1265 error = SET_ERROR(ENXIO); 1266 } else if (zfs_nocacheflush) { 1267 /* Flushing disabled by operator, declare success */ 1268 error = 0; 1269 } else if (v->vdev_nowritecache) { 1270 /* This vdev not capable of flushing */ 1271 error = SET_ERROR(ENOTSUP); 1272 } else { 1273 /* 1274 * Issue the flush. If successful, the response will 1275 * be handled in the completion callback, so we're done. 1276 */ 1277 error = vdev_disk_io_flush(BDH_BDEV(vd->vd_bdh), zio); 1278 if (error == 0) { 1279 rw_exit(&vd->vd_lock); 1280 return; 1281 } 1282 } 1283 1284 /* Couldn't issue the flush, so set the error and return it */ 1285 rw_exit(&vd->vd_lock); 1286 zio->io_error = error; 1287 zio_execute(zio); 1288 return; 1289 1290 case ZIO_TYPE_TRIM: 1291 error = vdev_disk_io_trim(zio); 1292 rw_exit(&vd->vd_lock); 1293 if (error) { 1294 zio->io_error = error; 1295 zio_execute(zio); 1296 } 1297 return; 1298 1299 case ZIO_TYPE_READ: 1300 case ZIO_TYPE_WRITE: 1301 zio->io_target_timestamp = zio_handle_io_delay(zio); 1302 error = vdev_disk_io_rw(zio); 1303 rw_exit(&vd->vd_lock); 1304 if (error) { 1305 zio->io_error = error; 1306 zio_interrupt(zio); 1307 } 1308 return; 1309 1310 default: 1311 /* 1312 * Getting here means our parent vdev has made a very strange 1313 * request of us, and shouldn't happen. Assert here to force a 1314 * crash in dev builds, but in production return the IO 1315 * unhandled. The pool will likely suspend anyway but that's 1316 * nicer than crashing the kernel. 1317 */ 1318 ASSERT3S(zio->io_type, ==, -1); 1319 1320 rw_exit(&vd->vd_lock); 1321 zio->io_error = SET_ERROR(ENOTSUP); 1322 zio_interrupt(zio); 1323 return; 1324 } 1325 1326 __builtin_unreachable(); 1327 } 1328 1329 static void 1330 vdev_disk_io_done(zio_t *zio) 1331 { 1332 /* If this was a read or write, we need to clean up the vbio */ 1333 if (zio->io_bio != NULL) { 1334 vbio_t *vbio = zio->io_bio; 1335 zio->io_bio = NULL; 1336 1337 /* 1338 * If we copied the ABD before issuing it, clean up and return 1339 * the copy to the ADB, with changes if appropriate. 1340 */ 1341 if (vbio->vbio_abd != NULL) { 1342 if (zio->io_type == ZIO_TYPE_READ) 1343 abd_copy(zio->io_abd, vbio->vbio_abd, 1344 zio->io_size); 1345 1346 abd_free(vbio->vbio_abd); 1347 vbio->vbio_abd = NULL; 1348 } 1349 1350 /* Final cleanup */ 1351 kmem_free(vbio, sizeof (vbio_t)); 1352 } 1353 1354 /* 1355 * If the device returned EIO, we revalidate the media. If it is 1356 * determined the media has changed this triggers the asynchronous 1357 * removal of the device from the configuration. 1358 */ 1359 if (zio->io_error == EIO) { 1360 vdev_t *v = zio->io_vd; 1361 vdev_disk_t *vd = v->vdev_tsd; 1362 1363 if (!zfs_check_disk_status(BDH_BDEV(vd->vd_bdh))) { 1364 invalidate_bdev(BDH_BDEV(vd->vd_bdh)); 1365 v->vdev_remove_wanted = B_TRUE; 1366 spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE); 1367 } 1368 } 1369 } 1370 1371 static void 1372 vdev_disk_hold(vdev_t *vd) 1373 { 1374 ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER)); 1375 1376 /* We must have a pathname, and it must be absolute. */ 1377 if (vd->vdev_path == NULL || vd->vdev_path[0] != '/') 1378 return; 1379 1380 /* 1381 * Only prefetch path and devid info if the device has 1382 * never been opened. 1383 */ 1384 if (vd->vdev_tsd != NULL) 1385 return; 1386 1387 } 1388 1389 static void 1390 vdev_disk_rele(vdev_t *vd) 1391 { 1392 ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER)); 1393 1394 /* XXX: Implement me as a vnode rele for the device */ 1395 } 1396 1397 vdev_ops_t vdev_disk_ops = { 1398 .vdev_op_init = NULL, 1399 .vdev_op_fini = NULL, 1400 .vdev_op_open = vdev_disk_open, 1401 .vdev_op_close = vdev_disk_close, 1402 .vdev_op_asize_to_psize = vdev_default_psize, 1403 .vdev_op_psize_to_asize = vdev_default_asize, 1404 .vdev_op_min_asize = vdev_default_min_asize, 1405 .vdev_op_min_alloc = NULL, 1406 .vdev_op_io_start = vdev_disk_io_start, 1407 .vdev_op_io_done = vdev_disk_io_done, 1408 .vdev_op_state_change = NULL, 1409 .vdev_op_need_resilver = NULL, 1410 .vdev_op_hold = vdev_disk_hold, 1411 .vdev_op_rele = vdev_disk_rele, 1412 .vdev_op_remap = NULL, 1413 .vdev_op_xlate = vdev_default_xlate, 1414 .vdev_op_rebuild_asize = NULL, 1415 .vdev_op_metaslab_init = NULL, 1416 .vdev_op_config_generate = NULL, 1417 .vdev_op_nparity = NULL, 1418 .vdev_op_ndisks = NULL, 1419 .vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */ 1420 .vdev_op_leaf = B_TRUE, /* leaf vdev */ 1421 .vdev_op_kobj_evt_post = vdev_disk_kobj_evt_post 1422 }; 1423 1424 int 1425 param_set_min_auto_ashift(const char *buf, zfs_kernel_param_t *kp) 1426 { 1427 uint_t val; 1428 int error; 1429 1430 error = kstrtouint(buf, 0, &val); 1431 if (error < 0) 1432 return (SET_ERROR(error)); 1433 1434 if (val < ASHIFT_MIN || val > zfs_vdev_max_auto_ashift) 1435 return (SET_ERROR(-EINVAL)); 1436 1437 error = param_set_uint(buf, kp); 1438 if (error < 0) 1439 return (SET_ERROR(error)); 1440 1441 return (0); 1442 } 1443 1444 int 1445 param_set_max_auto_ashift(const char *buf, zfs_kernel_param_t *kp) 1446 { 1447 uint_t val; 1448 int error; 1449 1450 error = kstrtouint(buf, 0, &val); 1451 if (error < 0) 1452 return (SET_ERROR(error)); 1453 1454 if (val > ASHIFT_MAX || val < zfs_vdev_min_auto_ashift) 1455 return (SET_ERROR(-EINVAL)); 1456 1457 error = param_set_uint(buf, kp); 1458 if (error < 0) 1459 return (SET_ERROR(error)); 1460 1461 return (0); 1462 } 1463 1464 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, open_timeout_ms, UINT, ZMOD_RW, 1465 "Timeout before determining that a device is missing"); 1466 1467 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, failfast_mask, UINT, ZMOD_RW, 1468 "Defines failfast mask: 1 - device, 2 - transport, 4 - driver"); 1469 1470 ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, max_segs, UINT, ZMOD_RW, 1471 "Maximum number of data segments to add to an IO request (min 4)"); 1472 1473 ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, calling_thread_io, UINT, 1474 ZMOD_RW, "Enable calling thread io"); 1475