1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
14 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
15 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
16 * LLNL-CODE-403049.
17 * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
18 * Copyright (c) 2023, 2024, 2025, Klara, Inc.
19 * Copyright (c) 2026, TrueNAS.
20 */
21
22 #include <sys/zfs_context.h>
23 #include <sys/spa_impl.h>
24 #include <sys/vdev_disk.h>
25 #include <sys/vdev_impl.h>
26 #include <sys/vdev_trim.h>
27 #include <sys/abd.h>
28 #include <sys/fs/zfs.h>
29 #include <sys/zio.h>
30 #include <linux/blkpg.h>
31 #include <linux/msdos_fs.h>
32 #include <linux/vfs_compat.h>
33 #include <linux/blk-cgroup.h>
34
35 /*
36 * Linux 6.8.x uses a bdev_handle as an instance/refcount for an underlying
37 * block_device. Since it carries the block_device inside, its convenient to
38 * just use the handle as a proxy.
39 *
40 * Linux 6.9.x uses a file for the same purpose.
41 *
42 * For pre-6.8, we just emulate this with a cast, since we don't need any of
43 * the other fields inside the handle.
44 */
45 #if defined(HAVE_BDEV_OPEN_BY_PATH)
46 typedef struct bdev_handle zfs_bdev_handle_t;
47 #define BDH_BDEV(bdh) ((bdh)->bdev)
48 #define BDH_IS_ERR(bdh) (IS_ERR(bdh))
49 #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
50 #define BDH_ERR_PTR(err) (ERR_PTR(err))
51 #elif defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
52 typedef struct file zfs_bdev_handle_t;
53 #define BDH_BDEV(bdh) (file_bdev(bdh))
54 #define BDH_IS_ERR(bdh) (IS_ERR(bdh))
55 #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
56 #define BDH_ERR_PTR(err) (ERR_PTR(err))
57 #else
58 typedef void zfs_bdev_handle_t;
59 #define BDH_BDEV(bdh) ((struct block_device *)bdh)
60 #define BDH_IS_ERR(bdh) (IS_ERR(BDH_BDEV(bdh)))
61 #define BDH_PTR_ERR(bdh) (PTR_ERR(BDH_BDEV(bdh)))
62 #define BDH_ERR_PTR(err) (ERR_PTR(err))
63 #endif
64
65 typedef struct vdev_disk {
66 zfs_bdev_handle_t *vd_bdh;
67 krwlock_t vd_lock;
68 } vdev_disk_t;
69
70 /*
71 * Maximum number of segments to add to a bio (min 4). If this is higher than
72 * the maximum allowed by the device queue or the kernel itself, it will be
73 * clamped. Setting it to zero will cause the kernel's ideal size to be used.
74 */
75 uint_t zfs_vdev_disk_max_segs = 0;
76
77 /*
78 * Unique identifier for the exclusive vdev holder.
79 */
80 static void *zfs_vdev_holder = VDEV_HOLDER;
81
82 /*
83 * Wait up to zfs_vdev_open_timeout_ms milliseconds before determining the
84 * device is missing. The missing path may be transient since the links
85 * can be briefly removed and recreated in response to udev events.
86 */
87 static uint_t zfs_vdev_open_timeout_ms = 1000;
88
89 /*
90 * Size of the "reserved" partition, in blocks.
91 */
92 #define EFI_MIN_RESV_SIZE (16 * 1024)
93
94 /*
95 * BIO request failfast mask.
96 */
97
98 static unsigned int zfs_vdev_failfast_mask = 1;
99
100 /*
101 * Whether we wait for bio to complete. Also requires that
102 * zio has bypassed the vdev queue. May lead to performance
103 * improvements when backing vdev devices are fast and low
104 * latency. May impact performance with certain workloads
105 * when enabled on raidz or draid zpool configurations.
106 */
107 static unsigned int zfs_vdev_disk_calling_thread_io = 0;
108
109 /*
110 * Convert SPA mode flags into bdev open mode flags.
111 */
112 #ifdef HAVE_BLK_MODE_T
113 typedef blk_mode_t vdev_bdev_mode_t;
114 #define VDEV_BDEV_MODE_READ BLK_OPEN_READ
115 #define VDEV_BDEV_MODE_WRITE BLK_OPEN_WRITE
116 #define VDEV_BDEV_MODE_EXCL BLK_OPEN_EXCL
117 #define VDEV_BDEV_MODE_MASK (BLK_OPEN_READ|BLK_OPEN_WRITE|BLK_OPEN_EXCL)
118 #else
119 typedef fmode_t vdev_bdev_mode_t;
120 #define VDEV_BDEV_MODE_READ FMODE_READ
121 #define VDEV_BDEV_MODE_WRITE FMODE_WRITE
122 #define VDEV_BDEV_MODE_EXCL FMODE_EXCL
123 #define VDEV_BDEV_MODE_MASK (FMODE_READ|FMODE_WRITE|FMODE_EXCL)
124 #endif
125
126 static vdev_bdev_mode_t
vdev_bdev_mode(spa_mode_t smode)127 vdev_bdev_mode(spa_mode_t smode)
128 {
129 ASSERT3U(smode, !=, SPA_MODE_UNINIT);
130 ASSERT0(smode & ~(SPA_MODE_READ|SPA_MODE_WRITE));
131
132 vdev_bdev_mode_t bmode = VDEV_BDEV_MODE_EXCL;
133
134 if (smode & SPA_MODE_READ)
135 bmode |= VDEV_BDEV_MODE_READ;
136
137 if (smode & SPA_MODE_WRITE)
138 bmode |= VDEV_BDEV_MODE_WRITE;
139
140 ASSERT(bmode & VDEV_BDEV_MODE_MASK);
141 ASSERT0(bmode & ~VDEV_BDEV_MODE_MASK);
142
143 return (bmode);
144 }
145
146 /*
147 * Returns the usable capacity (in bytes) for the partition or disk.
148 */
149 static uint64_t
bdev_capacity(struct block_device * bdev)150 bdev_capacity(struct block_device *bdev)
151 {
152 #ifdef HAVE_BDEV_NR_BYTES
153 return (bdev_nr_bytes(bdev));
154 #else
155 return (i_size_read(bdev->bd_inode));
156 #endif
157 }
158
159 #if !defined(HAVE_BDEV_WHOLE)
160 static inline struct block_device *
bdev_whole(struct block_device * bdev)161 bdev_whole(struct block_device *bdev)
162 {
163 return (bdev->bd_contains);
164 }
165 #endif
166
167 #if defined(HAVE_BDEVNAME)
168 #define vdev_bdevname(bdev, name) bdevname(bdev, name)
169 #else
170 static inline void
vdev_bdevname(struct block_device * bdev,char * name)171 vdev_bdevname(struct block_device *bdev, char *name)
172 {
173 snprintf(name, BDEVNAME_SIZE, "%pg", bdev);
174 }
175 #endif
176
177 /*
178 * Returns the maximum expansion capacity of the block device (in bytes).
179 *
180 * It is possible to expand a vdev when it has been created as a wholedisk
181 * and the containing block device has increased in capacity. Or when the
182 * partition containing the pool has been manually increased in size.
183 *
184 * This function is only responsible for calculating the potential expansion
185 * size so it can be reported by 'zpool list'. The efi_use_whole_disk() is
186 * responsible for verifying the expected partition layout in the wholedisk
187 * case, and updating the partition table if appropriate. Once the partition
188 * size has been increased the additional capacity will be visible using
189 * bdev_capacity().
190 *
191 * The returned maximum expansion capacity is always expected to be larger, or
192 * at the very least equal, to its usable capacity to prevent overestimating
193 * the pool expandsize.
194 */
195 static uint64_t
bdev_max_capacity(struct block_device * bdev,uint64_t wholedisk)196 bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk)
197 {
198 uint64_t psize;
199 int64_t available;
200
201 if (wholedisk && bdev != bdev_whole(bdev)) {
202 /*
203 * When reporting maximum expansion capacity for a wholedisk
204 * deduct any capacity which is expected to be lost due to
205 * alignment restrictions. Over reporting this value isn't
206 * harmful and would only result in slightly less capacity
207 * than expected post expansion.
208 * The estimated available space may be slightly smaller than
209 * bdev_capacity() for devices where the number of sectors is
210 * not a multiple of the alignment size and the partition layout
211 * is keeping less than PARTITION_END_ALIGNMENT bytes after the
212 * "reserved" EFI partition: in such cases return the device
213 * usable capacity.
214 */
215 available = bdev_capacity(bdev_whole(bdev)) -
216 ((EFI_MIN_RESV_SIZE + NEW_START_BLOCK +
217 PARTITION_END_ALIGNMENT) << SECTOR_BITS);
218 psize = MAX(available, bdev_capacity(bdev));
219 } else {
220 psize = bdev_capacity(bdev);
221 }
222
223 return (psize);
224 }
225
226 static void
vdev_disk_error(zio_t * zio)227 vdev_disk_error(zio_t *zio)
228 {
229 /*
230 * This function can be called in interrupt context, for instance while
231 * handling IRQs coming from a misbehaving disk device; use printk()
232 * which is safe from any context.
233 */
234 printk(KERN_WARNING "zio pool=%s vdev=%s error=%d type=%d "
235 "offset=%llu size=%llu flags=%llu\n", spa_name(zio->io_spa),
236 zio->io_vd->vdev_path, zio->io_error, zio->io_type,
237 (u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size,
238 zio->io_flags);
239 }
240
241 static void
vdev_disk_kobj_evt_post(vdev_t * v)242 vdev_disk_kobj_evt_post(vdev_t *v)
243 {
244 vdev_disk_t *vd = v->vdev_tsd;
245 if (vd && vd->vd_bdh) {
246 spl_signal_kobj_evt(BDH_BDEV(vd->vd_bdh));
247 } else {
248 vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n",
249 v->vdev_path);
250 }
251 }
252
253 static zfs_bdev_handle_t *
vdev_blkdev_get_by_path(const char * path,spa_mode_t smode,void * holder)254 vdev_blkdev_get_by_path(const char *path, spa_mode_t smode, void *holder)
255 {
256 vdev_bdev_mode_t bmode = vdev_bdev_mode(smode);
257
258 #if defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
259 return (bdev_file_open_by_path(path, bmode, holder, NULL));
260 #elif defined(HAVE_BDEV_OPEN_BY_PATH)
261 return (bdev_open_by_path(path, bmode, holder, NULL));
262 #elif defined(HAVE_BLKDEV_GET_BY_PATH_4ARG)
263 return (blkdev_get_by_path(path, bmode, holder, NULL));
264 #else
265 return (blkdev_get_by_path(path, bmode, holder));
266 #endif
267 }
268
269 static void
vdev_blkdev_put(zfs_bdev_handle_t * bdh,spa_mode_t smode,void * holder)270 vdev_blkdev_put(zfs_bdev_handle_t *bdh, spa_mode_t smode, void *holder)
271 {
272 #if defined(HAVE_BDEV_RELEASE)
273 return (bdev_release(bdh));
274 #elif defined(HAVE_BLKDEV_PUT_HOLDER)
275 return (blkdev_put(BDH_BDEV(bdh), holder));
276 #elif defined(HAVE_BLKDEV_PUT)
277 return (blkdev_put(BDH_BDEV(bdh), vdev_bdev_mode(smode)));
278 #else
279 fput(bdh);
280 #endif
281 }
282
283 static int
vdev_path_backing_permission(struct path * path,int mask)284 vdev_path_backing_permission(struct path *path, int mask)
285 {
286 #if defined(HAVE_IDMAP_MNTIDMAP)
287 return (inode_permission(mnt_idmap(path->mnt),
288 d_backing_inode(path->dentry), mask));
289 #elif defined(HAVE_IDMAP_USERNS)
290 return (inode_permission(mnt_user_ns(path->mnt),
291 d_backing_inode(path->dentry), mask));
292 #else
293 return (inode_permission(d_backing_inode(path->dentry), mask));
294 #endif
295 }
296
297 static int
vdev_disk_open(vdev_t * v,uint64_t * psize,uint64_t * max_psize,uint64_t * logical_ashift,uint64_t * physical_ashift,cred_t * cr)298 vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
299 uint64_t *logical_ashift, uint64_t *physical_ashift, cred_t *cr)
300 {
301 zfs_bdev_handle_t *bdh;
302 spa_mode_t smode = spa_mode(v->vdev_spa);
303 hrtime_t timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms);
304 vdev_disk_t *vd;
305 const cred_t *oldcr = NULL;
306
307 /* Must have a pathname and it must be absolute. */
308 if (v->vdev_path == NULL || v->vdev_path[0] != '/') {
309 v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
310 vdev_dbgmsg(v, "invalid vdev_path");
311 return (SET_ERROR(EINVAL));
312 }
313
314 /*
315 * Reopen the device if it is currently open. When expanding a
316 * partition force re-scanning the partition table if userland
317 * did not take care of this already. We need to do this while closed
318 * in order to get an accurate updated block device size. Then
319 * since udev may need to recreate the device links increase the
320 * open retry timeout before reporting the device as unavailable.
321 */
322 vd = v->vdev_tsd;
323 if (vd) {
324 char disk_name[BDEVNAME_SIZE + 6] = "/dev/";
325 boolean_t reread_part = B_FALSE;
326
327 /*
328 * Reopening an already-open device, so the caller credential
329 * is irrelevant - we had access to it before, we can assume
330 * we still do.
331 */
332 oldcr = override_creds(kcred);
333
334 rw_enter(&vd->vd_lock, RW_WRITER);
335 bdh = vd->vd_bdh;
336 vd->vd_bdh = NULL;
337
338 if (bdh) {
339 struct block_device *bdev = BDH_BDEV(bdh);
340 if (v->vdev_expanding && bdev != bdev_whole(bdev)) {
341 vdev_bdevname(bdev_whole(bdev), disk_name + 5);
342 /*
343 * If userland has BLKPG_RESIZE_PARTITION,
344 * then it should have updated the partition
345 * table already. We can detect this by
346 * comparing our current physical size
347 * with that of the device. If they are
348 * the same, then we must not have
349 * BLKPG_RESIZE_PARTITION or it failed to
350 * update the partition table online. We
351 * fallback to rescanning the partition
352 * table from the kernel below. However,
353 * if the capacity already reflects the
354 * updated partition, then we skip
355 * rescanning the partition table here.
356 */
357 if (v->vdev_psize == bdev_capacity(bdev))
358 reread_part = B_TRUE;
359 }
360
361 vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
362 }
363
364 if (reread_part) {
365 bdh = vdev_blkdev_get_by_path(disk_name, smode,
366 zfs_vdev_holder);
367 if (!BDH_IS_ERR(bdh)) {
368 int error =
369 vdev_bdev_reread_part(BDH_BDEV(bdh));
370 vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
371 if (error == 0) {
372 timeout = MSEC2NSEC(
373 zfs_vdev_open_timeout_ms * 2);
374 }
375 }
376 }
377 } else {
378 vd = kmem_zalloc(sizeof (vdev_disk_t), KM_SLEEP);
379
380 rw_init(&vd->vd_lock, NULL, RW_DEFAULT, NULL);
381 rw_enter(&vd->vd_lock, RW_WRITER);
382
383 /* Restrict device access below to caller's permissions. */
384 oldcr = override_creds(cr);
385 }
386
387 /*
388 * Devices are always opened by the path provided at configuration
389 * time. This means that if the provided path is a udev by-id path
390 * then drives may be re-cabled without an issue. If the provided
391 * path is a udev by-path path, then the physical location information
392 * will be preserved. This can be critical for more complicated
393 * configurations where drives are located in specific physical
394 * locations to maximize the systems tolerance to component failure.
395 *
396 * Alternatively, you can provide your own udev rule to flexibly map
397 * the drives as you see fit. It is not advised that you use the
398 * /dev/[hd]d devices which may be reordered due to probing order.
399 * Devices in the wrong locations will be detected by the higher
400 * level vdev validation.
401 *
402 * The specified paths may be briefly removed and recreated in
403 * response to udev events. This should be exceptionally unlikely
404 * because the zpool command makes every effort to verify these paths
405 * have already settled prior to reaching this point. Therefore,
406 * a ENOENT failure at this point is highly likely to be transient
407 * and it is reasonable to sleep and retry before giving up. In
408 * practice delays have been observed to be on the order of 100ms.
409 *
410 * When ERESTARTSYS is returned it indicates the block device is
411 * a zvol which could not be opened due to the deadlock detection
412 * logic in zvol_open(). Extend the timeout and retry the open
413 * subsequent attempts are expected to eventually succeed.
414 */
415
416 hrtime_t start = gethrtime();
417 int err = ENXIO;
418 while (err != 0 && ((gethrtime() - start) < timeout)) {
419
420 /*
421 * Ensure the caller credential (made live by override_creds()
422 * above) has access to the device node. This will include
423 * checking file permissions and considering the
424 * CAP_DAC_OVERRIDE capability.
425 */
426 struct path devpath;
427 err = kern_path(v->vdev_path, LOOKUP_FOLLOW, &devpath);
428 if (err == 0) {
429 int mask =
430 (smode & SPA_MODE_READ ? MAY_READ : 0) |
431 (smode & SPA_MODE_WRITE ? MAY_WRITE : 0);
432 err = vdev_path_backing_permission(&devpath, mask);
433 path_put(&devpath);
434 }
435
436 if (err == 0) {
437 bdh = vdev_blkdev_get_by_path(v->vdev_path, smode,
438 zfs_vdev_holder);
439 err = BDH_IS_ERR(bdh) ? BDH_PTR_ERR(bdh) : 0;
440 }
441
442 if (err == 0)
443 break;
444
445 if (unlikely(err == -ENOENT)) {
446 /*
447 * There is no point of waiting since device is removed
448 * explicitly
449 */
450 if (v->vdev_removed)
451 break;
452
453 schedule_timeout_interruptible(MSEC_TO_TICK(10));
454 } else if (unlikely(err == -ERESTARTSYS)) {
455 timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10);
456 continue;
457 }
458
459 break;
460 }
461
462 revert_creds(oldcr);
463
464 if (err != 0) {
465 vdev_dbgmsg(v, "open error=%d timeout=%llu/%llu", -err,
466 (u_longlong_t)(gethrtime() - start),
467 (u_longlong_t)timeout);
468 vd->vd_bdh = NULL;
469 v->vdev_tsd = vd;
470 rw_exit(&vd->vd_lock);
471 return (SET_ERROR(-err));
472 }
473
474 vd->vd_bdh = bdh;
475 v->vdev_tsd = vd;
476 rw_exit(&vd->vd_lock);
477
478 struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
479
480 /* Determine the physical block size */
481 int physical_block_size = bdev_physical_block_size(bdev);
482
483 /* Determine the logical block size */
484 int logical_block_size = bdev_logical_block_size(bdev);
485
486 /*
487 * If the device has a write cache, clear the nowritecache flag,
488 * so that we start issuing flush requests again.
489 */
490 v->vdev_nowritecache = !zfs_bdev_has_write_cache(bdev);
491
492 /* Set when device reports it supports TRIM. */
493 v->vdev_has_trim = bdev_discard_supported(bdev);
494
495 /* Set when device reports it supports secure TRIM. */
496 v->vdev_has_securetrim = bdev_secure_discard_supported(bdev);
497
498 /* Inform the ZIO pipeline that we are non-rotational */
499 #ifdef HAVE_BLK_QUEUE_ROT
500 v->vdev_nonrot = !blk_queue_rot(bdev_get_queue(bdev));
501 #else
502 v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev));
503 #endif
504
505 /* Is backed by a block device. */
506 v->vdev_is_blkdev = B_TRUE;
507
508 /* Physical volume size in bytes for the partition */
509 *psize = bdev_capacity(bdev);
510
511 /* Physical volume size in bytes including possible expansion space */
512 *max_psize = bdev_max_capacity(bdev, v->vdev_wholedisk);
513
514 /* Based on the minimum sector size set the block size */
515 *physical_ashift = highbit64(MAX(physical_block_size,
516 SPA_MINBLOCKSIZE)) - 1;
517
518 *logical_ashift = highbit64(MAX(logical_block_size,
519 SPA_MINBLOCKSIZE)) - 1;
520
521 return (0);
522 }
523
524 static void
vdev_disk_close(vdev_t * v)525 vdev_disk_close(vdev_t *v)
526 {
527 vdev_disk_t *vd = v->vdev_tsd;
528
529 if (v->vdev_reopening || vd == NULL)
530 return;
531
532 rw_enter(&vd->vd_lock, RW_WRITER);
533
534 if (vd->vd_bdh != NULL)
535 vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa),
536 zfs_vdev_holder);
537
538 v->vdev_tsd = NULL;
539
540 rw_exit(&vd->vd_lock);
541 rw_destroy(&vd->vd_lock);
542 kmem_free(vd, sizeof (vdev_disk_t));
543 }
544
545 /*
546 * preempt_schedule_notrace is GPL-only which breaks the ZFS build, so
547 * replace it with preempt_schedule under the following condition:
548 */
549 #if defined(CONFIG_ARM64) && \
550 defined(CONFIG_PREEMPTION) && \
551 defined(CONFIG_BLK_CGROUP)
552 #define preempt_schedule_notrace(x) preempt_schedule(x)
553 #endif
554
555 /*
556 * As for the Linux 5.18 kernel bio_alloc() expects a block_device struct
557 * as an argument removing the need to set it with bio_set_dev(). This
558 * removes the need for all of the following compatibility code.
559 */
560 #if !defined(HAVE_BIO_ALLOC_4ARG)
561
562 #if defined(CONFIG_BLK_CGROUP) && defined(HAVE_BIO_SET_DEV_GPL_ONLY)
563 /*
564 * The Linux 5.5 kernel updated percpu_ref_tryget() which is inlined by
565 * blkg_tryget() to use rcu_read_lock() instead of rcu_read_lock_sched().
566 * As a side effect the function was converted to GPL-only. Define our
567 * own version when needed which uses rcu_read_lock_sched().
568 *
569 * The Linux 5.17 kernel split linux/blk-cgroup.h into a private and a public
570 * part, moving blkg_tryget into the private one. Define our own version.
571 */
572 #if defined(HAVE_BLKG_TRYGET_GPL_ONLY) || !defined(HAVE_BLKG_TRYGET)
573 static inline bool
vdev_blkg_tryget(struct blkcg_gq * blkg)574 vdev_blkg_tryget(struct blkcg_gq *blkg)
575 {
576 struct percpu_ref *ref = &blkg->refcnt;
577 unsigned long __percpu *count;
578 bool rc;
579
580 rcu_read_lock_sched();
581
582 if (__ref_is_percpu(ref, &count)) {
583 this_cpu_inc(*count);
584 rc = true;
585 } else {
586 #ifdef ZFS_PERCPU_REF_COUNT_IN_DATA
587 rc = atomic_long_inc_not_zero(&ref->data->count);
588 #else
589 rc = atomic_long_inc_not_zero(&ref->count);
590 #endif
591 }
592
593 rcu_read_unlock_sched();
594
595 return (rc);
596 }
597 #else
598 #define vdev_blkg_tryget(bg) blkg_tryget(bg)
599 #endif
600 #ifdef HAVE_BIO_SET_DEV_MACRO
601 /*
602 * The Linux 5.0 kernel updated the bio_set_dev() macro so it calls the
603 * GPL-only bio_associate_blkg() symbol thus inadvertently converting
604 * the entire macro. Provide a minimal version which always assigns the
605 * request queue's root_blkg to the bio.
606 */
607 static inline void
vdev_bio_associate_blkg(struct bio * bio)608 vdev_bio_associate_blkg(struct bio *bio)
609 {
610 #if defined(HAVE_BIO_BDEV_DISK)
611 struct request_queue *q = bio->bi_bdev->bd_disk->queue;
612 #else
613 struct request_queue *q = bio->bi_disk->queue;
614 #endif
615
616 ASSERT3P(q, !=, NULL);
617 ASSERT0P(bio->bi_blkg);
618
619 if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
620 bio->bi_blkg = q->root_blkg;
621 }
622
623 #define bio_associate_blkg vdev_bio_associate_blkg
624 #else
625 static inline void
vdev_bio_set_dev(struct bio * bio,struct block_device * bdev)626 vdev_bio_set_dev(struct bio *bio, struct block_device *bdev)
627 {
628 #if defined(HAVE_BIO_BDEV_DISK)
629 struct request_queue *q = bdev->bd_disk->queue;
630 #else
631 struct request_queue *q = bio->bi_disk->queue;
632 #endif
633 bio_clear_flag(bio, BIO_REMAPPED);
634 if (bio->bi_bdev != bdev)
635 bio_clear_flag(bio, BIO_THROTTLED);
636 bio->bi_bdev = bdev;
637
638 ASSERT3P(q, !=, NULL);
639 ASSERT0P(bio->bi_blkg);
640
641 if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
642 bio->bi_blkg = q->root_blkg;
643 }
644 #define bio_set_dev vdev_bio_set_dev
645 #endif
646 #endif
647 #endif /* !HAVE_BIO_ALLOC_4ARG */
648
649 static inline void
vdev_submit_bio(struct bio * bio)650 vdev_submit_bio(struct bio *bio)
651 {
652 struct bio_list *bio_list = current->bio_list;
653 current->bio_list = NULL;
654 (void) submit_bio(bio);
655 current->bio_list = bio_list;
656 }
657
658 static inline void
vdev_submit_bio_wait(struct bio * bio)659 vdev_submit_bio_wait(struct bio *bio)
660 {
661 struct bio_list *bio_list = current->bio_list;
662 current->bio_list = NULL;
663 (void) submit_bio_wait(bio);
664 current->bio_list = bio_list;
665 }
666
667 static inline struct bio *
vdev_bio_alloc(struct block_device * bdev,gfp_t gfp_mask,unsigned short nr_vecs)668 vdev_bio_alloc(struct block_device *bdev, gfp_t gfp_mask,
669 unsigned short nr_vecs)
670 {
671 struct bio *bio;
672
673 #ifdef HAVE_BIO_ALLOC_4ARG
674 bio = bio_alloc(bdev, nr_vecs, 0, gfp_mask);
675 #else
676 bio = bio_alloc(gfp_mask, nr_vecs);
677 if (likely(bio != NULL))
678 bio_set_dev(bio, bdev);
679 #endif
680
681 return (bio);
682 }
683
684 static inline uint_t
vdev_bio_max_segs(struct block_device * bdev)685 vdev_bio_max_segs(struct block_device *bdev)
686 {
687 /*
688 * Smallest of the device max segs and the tunable max segs. Minimum
689 * 4, so there's room to finish split pages if they come up.
690 */
691 const uint_t dev_max_segs = queue_max_segments(bdev_get_queue(bdev));
692 const uint_t tune_max_segs = (zfs_vdev_disk_max_segs > 0) ?
693 MAX(4, zfs_vdev_disk_max_segs) : dev_max_segs;
694 const uint_t max_segs = MIN(tune_max_segs, dev_max_segs);
695
696 #ifdef HAVE_BIO_MAX_SEGS
697 return (bio_max_segs(max_segs));
698 #else
699 return (MIN(max_segs, BIO_MAX_PAGES));
700 #endif
701 }
702
703 static inline uint_t
vdev_bio_max_bytes(struct block_device * bdev)704 vdev_bio_max_bytes(struct block_device *bdev)
705 {
706 return (queue_max_sectors(bdev_get_queue(bdev)) << 9);
707 }
708
709
710 /*
711 * Virtual block IO object (VBIO)
712 *
713 * Linux block IO (BIO) objects have a limit on how many data segments (pages)
714 * they can hold. Depending on how they're allocated and structured, a large
715 * ZIO can require more than one BIO to be submitted to the kernel, which then
716 * all have to complete before we can return the completed ZIO back to ZFS.
717 *
718 * A VBIO is a wrapper around multiple BIOs, carrying everything needed to
719 * translate a ZIO down into the kernel block layer and back again.
720 *
721 * Note that these are only used for data ZIOs (read/write). Meta-operations
722 * (flush/trim) don't need multiple BIOs and so can just make the call
723 * directly.
724 */
725 typedef struct {
726 zio_t *vbio_zio; /* parent zio */
727
728 struct block_device *vbio_bdev; /* blockdev to submit bios to */
729
730 abd_t *vbio_abd; /* abd carrying borrowed linear buf */
731
732 uint_t vbio_max_segs; /* max segs per bio */
733
734 uint_t vbio_max_bytes; /* max bytes per bio */
735 uint_t vbio_lbs_mask; /* logical block size mask */
736
737 uint64_t vbio_offset; /* start offset of next bio */
738
739 struct bio *vbio_bio; /* pointer to the current bio */
740 int vbio_flags; /* bio flags */
741 boolean_t vbio_wait; /* wait for completion */
742 } vbio_t;
743
744 static vbio_t *
vbio_alloc(zio_t * zio,struct block_device * bdev,int flags)745 vbio_alloc(zio_t *zio, struct block_device *bdev, int flags)
746 {
747 vbio_t *vbio = kmem_zalloc(sizeof (vbio_t), KM_SLEEP);
748
749 vbio->vbio_zio = zio;
750 vbio->vbio_bdev = bdev;
751 vbio->vbio_abd = NULL;
752 vbio->vbio_max_segs = vdev_bio_max_segs(bdev);
753 vbio->vbio_max_bytes = vdev_bio_max_bytes(bdev);
754 vbio->vbio_lbs_mask = ~(bdev_logical_block_size(bdev)-1);
755 vbio->vbio_offset = zio->io_offset;
756 vbio->vbio_bio = NULL;
757 vbio->vbio_flags = flags;
758 vbio->vbio_wait = B_FALSE;
759
760 return (vbio);
761 }
762
763 static void vbio_completion(struct bio *bio);
764
765 static int
vbio_add_page(vbio_t * vbio,struct page * page,uint_t size,uint_t offset)766 vbio_add_page(vbio_t *vbio, struct page *page, uint_t size, uint_t offset)
767 {
768 struct bio *bio = vbio->vbio_bio;
769 uint_t ssize;
770
771 while (size > 0) {
772 if (bio == NULL) {
773 /* New BIO, allocate and set up */
774 bio = vdev_bio_alloc(vbio->vbio_bdev, GFP_NOIO,
775 vbio->vbio_max_segs);
776 VERIFY(bio);
777
778 BIO_BI_SECTOR(bio) = vbio->vbio_offset >> 9;
779 bio_set_op_attrs(bio,
780 vbio->vbio_zio->io_type == ZIO_TYPE_WRITE ?
781 WRITE : READ, vbio->vbio_flags);
782
783 if (vbio->vbio_bio) {
784 bio_chain(vbio->vbio_bio, bio);
785 vdev_submit_bio(vbio->vbio_bio);
786 }
787 vbio->vbio_bio = bio;
788 }
789
790 /*
791 * Only load as much of the current page data as will fit in
792 * the space left in the BIO, respecting lbs alignment. Older
793 * kernels will error if we try to overfill the BIO, while
794 * newer ones will accept it and split the BIO. This ensures
795 * everything works on older kernels, and avoids an additional
796 * overhead on the new.
797 */
798 ssize = MIN(size, (vbio->vbio_max_bytes - BIO_BI_SIZE(bio)) &
799 vbio->vbio_lbs_mask);
800 if (ssize > 0 &&
801 bio_add_page(bio, page, ssize, offset) == ssize) {
802 /* Accepted, adjust and load any remaining. */
803 size -= ssize;
804 offset += ssize;
805 continue;
806 }
807
808 /* No room, set up for a new BIO and loop */
809 vbio->vbio_offset += BIO_BI_SIZE(bio);
810
811 /* Signal new BIO allocation wanted */
812 bio = NULL;
813 }
814
815 return (0);
816 }
817
818 /* Iterator callback to submit ABD pages to the vbio. */
819 static int
vbio_fill_cb(struct page * page,size_t off,size_t len,void * priv)820 vbio_fill_cb(struct page *page, size_t off, size_t len, void *priv)
821 {
822 vbio_t *vbio = priv;
823 return (vbio_add_page(vbio, page, len, off));
824 }
825
826 /* Create some BIOs, fill them with data and submit them */
827 static void
vbio_submit(vbio_t * vbio,abd_t * abd,uint64_t size)828 vbio_submit(vbio_t *vbio, abd_t *abd, uint64_t size)
829 {
830 /*
831 * We plug so we can submit the BIOs as we go and only unplug them when
832 * they are fully created and submitted. This is important; if we don't
833 * plug, then the kernel may start executing earlier BIOs while we're
834 * still creating and executing later ones, and if the device goes
835 * away while that's happening, older kernels can get confused and
836 * trample memory.
837 */
838 struct blk_plug plug;
839 blk_start_plug(&plug);
840
841 (void) abd_iterate_page_func(abd, 0, size, vbio_fill_cb, vbio);
842 ASSERT(vbio->vbio_bio);
843
844 /*
845 * Once submitted, vbio_bio now owns vbio (through bi_private) and we
846 * can't touch it again. The bio may complete and vbio_completion() be
847 * called and free the vbio before this task is run again, so we must
848 * consider it invalid from this point.
849 */
850
851 if (vbio->vbio_wait) {
852 vdev_submit_bio_wait(vbio->vbio_bio);
853 } else {
854 vbio->vbio_bio->bi_end_io = vbio_completion;
855 vbio->vbio_bio->bi_private = vbio;
856 vdev_submit_bio(vbio->vbio_bio);
857 }
858
859 blk_finish_plug(&plug);
860 }
861
862 /* IO completion callback */
863 static void
vbio_completion(struct bio * bio)864 vbio_completion(struct bio *bio)
865 {
866 vbio_t *vbio = bio->bi_private;
867 zio_t *zio = vbio->vbio_zio;
868
869 ASSERT(zio);
870
871 /* Capture and log any errors */
872 zio->io_error = bi_status_to_errno(bio->bi_status);
873 ASSERT3U(zio->io_error, >=, 0);
874
875 if (zio->io_error)
876 vdev_disk_error(zio);
877
878 /* Return the BIO to the kernel */
879 bio_put(bio);
880
881 /*
882 * We're likely in an interrupt context so we can't do ABD/memory work
883 * here; instead we stash vbio on the zio and take care of it in the
884 * done callback.
885 */
886 ASSERT0P(zio->io_bio);
887 zio->io_bio = vbio;
888
889 /* Using calling thread io, don't dispatch zio. */
890 if (vbio->vbio_wait)
891 zio_execute(zio);
892 else
893 zio_delay_interrupt(zio);
894
895 }
896
897 /*
898 * Iterator callback to count ABD pages and check their size & alignment.
899 *
900 * On Linux, each BIO segment can take a page pointer, and an offset+length of
901 * the data within that page. A page can be arbitrarily large ("compound"
902 * pages) but we still have to ensure the data portion is correctly sized and
903 * aligned to the logical block size, to ensure that if the kernel wants to
904 * split the BIO, the two halves will still be properly aligned.
905 *
906 * NOTE: if you change this function, change the copy in
907 * tests/zfs-tests/tests/functional/vdev_disk/page_alignment.c, and add test
908 * data there to validate the change you're making.
909 */
910 typedef struct {
911 size_t blocksize;
912 int seen_first;
913 int seen_last;
914 } vdev_disk_check_alignment_t;
915
916 static int
vdev_disk_check_alignment_cb(struct page * page,size_t off,size_t len,void * priv)917 vdev_disk_check_alignment_cb(struct page *page, size_t off, size_t len,
918 void *priv)
919 {
920 (void) page;
921 vdev_disk_check_alignment_t *s = priv;
922
923 /*
924 * The cardinal rule: a single on-disk block must never cross an
925 * physical (order-0) page boundary, as the kernel expects to be able
926 * to split at both LBS and page boundaries.
927 *
928 * This implies various alignment rules for the blocks in this
929 * (possibly compound) page, which we can check for.
930 */
931
932 /*
933 * If the previous page did not end on a page boundary, then we
934 * can't proceed without creating a hole.
935 */
936 if (s->seen_last)
937 return (1);
938
939 /* This page must contain only whole LBS-sized blocks. */
940 if (!IS_P2ALIGNED(len, s->blocksize))
941 return (1);
942
943 /*
944 * If this is not the first page in the ABD, then the data must start
945 * on a page-aligned boundary (so the kernel can split on page
946 * boundaries without having to deal with a hole). If it is, then
947 * it can start on LBS-alignment.
948 */
949 if (s->seen_first) {
950 if (!IS_P2ALIGNED(off, PAGESIZE))
951 return (1);
952 } else {
953 if (!IS_P2ALIGNED(off, s->blocksize))
954 return (1);
955 s->seen_first = 1;
956 }
957
958 /*
959 * If this data does not end on a page-aligned boundary, then this
960 * must be the last page in the ABD, for the same reason.
961 */
962 s->seen_last = !IS_P2ALIGNED(off+len, PAGESIZE);
963
964 return (0);
965 }
966
967 /*
968 * Check if we can submit the pages in this ABD to the kernel as-is. Returns
969 * the number of pages, or 0 if it can't be submitted like this.
970 */
971 static boolean_t
vdev_disk_check_alignment(abd_t * abd,uint64_t size,struct block_device * bdev)972 vdev_disk_check_alignment(abd_t *abd, uint64_t size, struct block_device *bdev)
973 {
974 vdev_disk_check_alignment_t s = {
975 .blocksize = bdev_logical_block_size(bdev),
976 };
977
978 if (abd_iterate_page_func(abd, 0, size,
979 vdev_disk_check_alignment_cb, &s))
980 return (B_FALSE);
981
982 return (B_TRUE);
983 }
984
985 static int
vdev_disk_io_rw(zio_t * zio)986 vdev_disk_io_rw(zio_t *zio)
987 {
988 vdev_t *v = zio->io_vd;
989 vdev_disk_t *vd = v->vdev_tsd;
990 struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
991 int flags = 0;
992
993 /*
994 * Accessing outside the block device is never allowed.
995 */
996 if (zio->io_offset + zio->io_size > bdev_capacity(bdev)) {
997 vdev_dbgmsg(zio->io_vd,
998 "Illegal access %llu size %llu, device size %llu",
999 (u_longlong_t)zio->io_offset,
1000 (u_longlong_t)zio->io_size,
1001 (u_longlong_t)bdev_capacity(bdev));
1002 return (SET_ERROR(EIO));
1003 }
1004
1005 vdev_t *iter = v;
1006 while (iter != NULL && iter->vdev_failfast == ZPROP_BOOLEAN_INHERIT)
1007 iter = iter->vdev_parent;
1008
1009 boolean_t failfast = iter ? iter->vdev_failfast == 1 :
1010 vdev_prop_default_numeric(VDEV_PROP_FAILFAST);
1011 if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)) &&
1012 failfast) {
1013 bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
1014 zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
1015 }
1016
1017 /*
1018 * Check alignment of the incoming ABD. If any part of it would require
1019 * submitting a page that is not aligned to both the logical block size
1020 * and the page size, then we take a copy into a new memory region with
1021 * correct alignment. This should be impossible on a 512b LBS. On
1022 * larger blocks, this can happen at least when a small number of
1023 * blocks (usually 1) are allocated from a shared slab, or when
1024 * abnormally-small data regions (eg gang headers) are mixed into the
1025 * same ABD as larger allocations (eg aggregations).
1026 */
1027 abd_t *abd = zio->io_abd;
1028 if (!vdev_disk_check_alignment(abd, zio->io_size, bdev)) {
1029 /* Allocate a new memory region with guaranteed alignment */
1030 abd = abd_alloc_for_io(zio->io_size,
1031 zio->io_abd->abd_flags & ABD_FLAG_META);
1032
1033 /* If we're writing copy our data into it */
1034 if (zio->io_type == ZIO_TYPE_WRITE)
1035 abd_copy(abd, zio->io_abd, zio->io_size);
1036
1037 /*
1038 * False here would mean the new allocation has an invalid
1039 * alignment too, which would mean that abd_alloc() is not
1040 * guaranteeing this, or our logic in
1041 * vdev_disk_check_alignment() is wrong. In either case,
1042 * something in seriously wrong and its not safe to continue.
1043 */
1044 VERIFY(vdev_disk_check_alignment(abd, zio->io_size, bdev));
1045 }
1046
1047 /* Allocate vbio, with a pointer to the borrowed ABD if necessary */
1048 vbio_t *vbio = vbio_alloc(zio, bdev, flags);
1049 if (abd != zio->io_abd)
1050 vbio->vbio_abd = abd;
1051
1052 boolean_t bio_wait = B_FALSE;
1053 if (zfs_vdev_disk_calling_thread_io &&
1054 (zio->io_flags & ZIO_FLAG_BYPASSED_QUEUE)) {
1055 vbio->vbio_wait = bio_wait = B_TRUE;
1056 }
1057 /* Fill it with data pages and submit it to the kernel */
1058 vbio_submit(vbio, abd, zio->io_size);
1059
1060 if (bio_wait) {
1061 vbio->vbio_bio->bi_private = vbio;
1062 vbio_completion(vbio->vbio_bio);
1063 }
1064
1065 return (0);
1066 }
1067
1068 static void
vdev_disk_io_flush_completion(struct bio * bio)1069 vdev_disk_io_flush_completion(struct bio *bio)
1070 {
1071 zio_t *zio = bio->bi_private;
1072 zio->io_error = bi_status_to_errno(bio->bi_status);
1073 if (zio->io_error == EOPNOTSUPP || zio->io_error == ENOTTY)
1074 zio->io_error = SET_ERROR(ENOTSUP);
1075
1076 bio_put(bio);
1077 ASSERT3S(zio->io_error, >=, 0);
1078 if (zio->io_error)
1079 vdev_disk_error(zio);
1080 zio_interrupt(zio);
1081 }
1082
1083 static int
vdev_disk_io_flush(struct block_device * bdev,zio_t * zio)1084 vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
1085 {
1086 struct request_queue *q;
1087 struct bio *bio;
1088
1089 q = bdev_get_queue(bdev);
1090 if (!q)
1091 return (SET_ERROR(ENXIO));
1092
1093 bio = vdev_bio_alloc(bdev, GFP_NOIO, 0);
1094 if (unlikely(bio == NULL))
1095 return (SET_ERROR(ENOMEM));
1096
1097 bio->bi_end_io = vdev_disk_io_flush_completion;
1098 bio->bi_private = zio;
1099 bio_set_flush(bio);
1100 vdev_submit_bio(bio);
1101 invalidate_bdev(bdev);
1102
1103 return (0);
1104 }
1105
1106 static void
vdev_disk_discard_end_io(struct bio * bio)1107 vdev_disk_discard_end_io(struct bio *bio)
1108 {
1109 zio_t *zio = bio->bi_private;
1110 zio->io_error = bi_status_to_errno(bio->bi_status);
1111
1112 bio_put(bio);
1113 if (zio->io_error)
1114 vdev_disk_error(zio);
1115 zio_interrupt(zio);
1116 }
1117
1118 /*
1119 * Wrappers for the different secure erase and discard APIs. We use async
1120 * when available; in this case, *biop is set to the last bio in the chain.
1121 */
1122 static int
vdev_bdev_issue_secure_erase(zfs_bdev_handle_t * bdh,sector_t sector,sector_t nsect,struct bio ** biop)1123 vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector,
1124 sector_t nsect, struct bio **biop)
1125 {
1126 *biop = NULL;
1127 int error;
1128
1129 #if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
1130 error = blkdev_issue_secure_erase(BDH_BDEV(bdh),
1131 sector, nsect, GFP_NOFS);
1132 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
1133 error = __blkdev_issue_discard(BDH_BDEV(bdh),
1134 sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop);
1135 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
1136 error = blkdev_issue_discard(BDH_BDEV(bdh),
1137 sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE);
1138 #else
1139 #error "unsupported kernel"
1140 #endif
1141
1142 return (error);
1143 }
1144
1145 static int
vdev_bdev_issue_discard(zfs_bdev_handle_t * bdh,sector_t sector,sector_t nsect,struct bio ** biop)1146 vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector,
1147 sector_t nsect, struct bio **biop)
1148 {
1149 *biop = NULL;
1150 int error;
1151
1152 #if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
1153 error = __blkdev_issue_discard(BDH_BDEV(bdh),
1154 sector, nsect, GFP_NOFS, 0, biop);
1155 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS)
1156 error = __blkdev_issue_discard(BDH_BDEV(bdh),
1157 sector, nsect, GFP_NOFS, biop);
1158 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
1159 error = blkdev_issue_discard(BDH_BDEV(bdh),
1160 sector, nsect, GFP_NOFS, 0);
1161 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS)
1162 error = blkdev_issue_discard(BDH_BDEV(bdh),
1163 sector, nsect, GFP_NOFS);
1164 #else
1165 #error "unsupported kernel"
1166 #endif
1167
1168 return (error);
1169 }
1170
1171 /*
1172 * Entry point for TRIM ops. This calls the right wrapper for secure erase or
1173 * discard, and then does the appropriate finishing work for error vs success
1174 * and async vs sync.
1175 */
1176 static int
vdev_disk_io_trim(zio_t * zio)1177 vdev_disk_io_trim(zio_t *zio)
1178 {
1179 int error;
1180 struct bio *bio;
1181
1182 zfs_bdev_handle_t *bdh = ((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh;
1183 sector_t sector = zio->io_offset >> 9;
1184 sector_t nsects = zio->io_size >> 9;
1185
1186 if (zio->io_trim_flags & ZIO_TRIM_SECURE)
1187 error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);
1188 else
1189 error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);
1190
1191 if (error != 0)
1192 return (SET_ERROR(-error));
1193
1194 if (bio == NULL) {
1195 /*
1196 * This was a synchronous op that completed successfully, so
1197 * return it to ZFS immediately.
1198 */
1199 zio_interrupt(zio);
1200 } else {
1201 /*
1202 * This was an asynchronous op; set up completion callback and
1203 * issue it.
1204 */
1205 bio->bi_private = zio;
1206 bio->bi_end_io = vdev_disk_discard_end_io;
1207 vdev_submit_bio(bio);
1208 }
1209
1210 return (0);
1211 }
1212
1213 static void
vdev_disk_io_start(zio_t * zio)1214 vdev_disk_io_start(zio_t *zio)
1215 {
1216 vdev_t *v = zio->io_vd;
1217 vdev_disk_t *vd = v->vdev_tsd;
1218 int error;
1219
1220 /*
1221 * If the vdev is closed, it's likely in the REMOVED or FAULTED state.
1222 * Nothing to be done here but return failure.
1223 */
1224 if (vd == NULL) {
1225 zio->io_error = ENXIO;
1226 zio_interrupt(zio);
1227 return;
1228 }
1229
1230 rw_enter(&vd->vd_lock, RW_READER);
1231
1232 /*
1233 * If the vdev is closed, it's likely due to a failed reopen and is
1234 * in the UNAVAIL state. Nothing to be done here but return failure.
1235 */
1236 if (vd->vd_bdh == NULL) {
1237 rw_exit(&vd->vd_lock);
1238 zio->io_error = ENXIO;
1239 zio_interrupt(zio);
1240 return;
1241 }
1242
1243 switch (zio->io_type) {
1244 case ZIO_TYPE_FLUSH:
1245
1246 if (!vdev_readable(v)) {
1247 /* Drive not there, can't flush */
1248 error = SET_ERROR(ENXIO);
1249 } else if (zfs_nocacheflush) {
1250 /* Flushing disabled by operator, declare success */
1251 error = 0;
1252 } else if (v->vdev_nowritecache) {
1253 /* This vdev not capable of flushing */
1254 error = SET_ERROR(ENOTSUP);
1255 } else {
1256 /*
1257 * Issue the flush. If successful, the response will
1258 * be handled in the completion callback, so we're done.
1259 */
1260 error = vdev_disk_io_flush(BDH_BDEV(vd->vd_bdh), zio);
1261 if (error == 0) {
1262 rw_exit(&vd->vd_lock);
1263 return;
1264 }
1265 }
1266
1267 /* Couldn't issue the flush, so set the error and return it */
1268 rw_exit(&vd->vd_lock);
1269 zio->io_error = error;
1270 zio_execute(zio);
1271 return;
1272
1273 case ZIO_TYPE_TRIM:
1274 error = vdev_disk_io_trim(zio);
1275 rw_exit(&vd->vd_lock);
1276 if (error) {
1277 zio->io_error = error;
1278 zio_execute(zio);
1279 }
1280 return;
1281
1282 case ZIO_TYPE_READ:
1283 case ZIO_TYPE_WRITE:
1284 zio->io_target_timestamp = zio_handle_io_delay(zio);
1285 error = vdev_disk_io_rw(zio);
1286 rw_exit(&vd->vd_lock);
1287 if (error) {
1288 zio->io_error = error;
1289 zio_interrupt(zio);
1290 }
1291 return;
1292
1293 default:
1294 /*
1295 * Getting here means our parent vdev has made a very strange
1296 * request of us, and shouldn't happen. Assert here to force a
1297 * crash in dev builds, but in production return the IO
1298 * unhandled. The pool will likely suspend anyway but that's
1299 * nicer than crashing the kernel.
1300 */
1301 ASSERT3S(zio->io_type, ==, -1);
1302
1303 rw_exit(&vd->vd_lock);
1304 zio->io_error = SET_ERROR(ENOTSUP);
1305 zio_interrupt(zio);
1306 return;
1307 }
1308
1309 __builtin_unreachable();
1310 }
1311
1312 static void
vdev_disk_io_done(zio_t * zio)1313 vdev_disk_io_done(zio_t *zio)
1314 {
1315 /* If this was a read or write, we need to clean up the vbio */
1316 if (zio->io_bio != NULL) {
1317 vbio_t *vbio = zio->io_bio;
1318 zio->io_bio = NULL;
1319
1320 /*
1321 * If we copied the ABD before issuing it, clean up and return
1322 * the copy to the ADB, with changes if appropriate.
1323 */
1324 if (vbio->vbio_abd != NULL) {
1325 if (zio->io_type == ZIO_TYPE_READ)
1326 abd_copy(zio->io_abd, vbio->vbio_abd,
1327 zio->io_size);
1328
1329 abd_free(vbio->vbio_abd);
1330 vbio->vbio_abd = NULL;
1331 }
1332
1333 /* Final cleanup */
1334 kmem_free(vbio, sizeof (vbio_t));
1335 }
1336
1337 /*
1338 * If the device returned EIO, we revalidate the media. If it is
1339 * determined the media has changed this triggers the asynchronous
1340 * removal of the device from the configuration.
1341 */
1342 if (zio->io_error == EIO) {
1343 vdev_t *v = zio->io_vd;
1344 vdev_disk_t *vd = v->vdev_tsd;
1345
1346 if (!zfs_check_disk_status(BDH_BDEV(vd->vd_bdh))) {
1347 invalidate_bdev(BDH_BDEV(vd->vd_bdh));
1348 v->vdev_remove_wanted = B_TRUE;
1349 spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
1350 }
1351 }
1352 }
1353
1354 static void
vdev_disk_hold(vdev_t * vd)1355 vdev_disk_hold(vdev_t *vd)
1356 {
1357 ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
1358
1359 /* We must have a pathname, and it must be absolute. */
1360 if (vd->vdev_path == NULL || vd->vdev_path[0] != '/')
1361 return;
1362
1363 /*
1364 * Only prefetch path and devid info if the device has
1365 * never been opened.
1366 */
1367 if (vd->vdev_tsd != NULL)
1368 return;
1369
1370 }
1371
1372 static void
vdev_disk_rele(vdev_t * vd)1373 vdev_disk_rele(vdev_t *vd)
1374 {
1375 ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
1376
1377 /* XXX: Implement me as a vnode rele for the device */
1378 }
1379
1380 vdev_ops_t vdev_disk_ops = {
1381 .vdev_op_init = NULL,
1382 .vdev_op_fini = NULL,
1383 .vdev_op_open = vdev_disk_open,
1384 .vdev_op_close = vdev_disk_close,
1385 .vdev_op_asize_to_psize = vdev_default_psize,
1386 .vdev_op_psize_to_asize = vdev_default_asize,
1387 .vdev_op_min_asize = vdev_default_min_asize,
1388 .vdev_op_min_alloc = NULL,
1389 .vdev_op_io_start = vdev_disk_io_start,
1390 .vdev_op_io_done = vdev_disk_io_done,
1391 .vdev_op_state_change = NULL,
1392 .vdev_op_need_resilver = NULL,
1393 .vdev_op_hold = vdev_disk_hold,
1394 .vdev_op_rele = vdev_disk_rele,
1395 .vdev_op_remap = NULL,
1396 .vdev_op_xlate = vdev_default_xlate,
1397 .vdev_op_rebuild_asize = NULL,
1398 .vdev_op_metaslab_init = NULL,
1399 .vdev_op_config_generate = NULL,
1400 .vdev_op_nparity = NULL,
1401 .vdev_op_ndisks = NULL,
1402 .vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */
1403 .vdev_op_leaf = B_TRUE, /* leaf vdev */
1404 .vdev_op_kobj_evt_post = vdev_disk_kobj_evt_post
1405 };
1406
1407 int
param_set_min_auto_ashift(const char * buf,zfs_kernel_param_t * kp)1408 param_set_min_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
1409 {
1410 uint_t val;
1411 int error;
1412
1413 error = kstrtouint(buf, 0, &val);
1414 if (error < 0)
1415 return (SET_ERROR(error));
1416
1417 if (val < ASHIFT_MIN || val > zfs_vdev_max_auto_ashift)
1418 return (SET_ERROR(-EINVAL));
1419
1420 error = param_set_uint(buf, kp);
1421 if (error < 0)
1422 return (SET_ERROR(error));
1423
1424 return (0);
1425 }
1426
1427 int
param_set_max_auto_ashift(const char * buf,zfs_kernel_param_t * kp)1428 param_set_max_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
1429 {
1430 uint_t val;
1431 int error;
1432
1433 error = kstrtouint(buf, 0, &val);
1434 if (error < 0)
1435 return (SET_ERROR(error));
1436
1437 if (val > ASHIFT_MAX || val < zfs_vdev_min_auto_ashift)
1438 return (SET_ERROR(-EINVAL));
1439
1440 error = param_set_uint(buf, kp);
1441 if (error < 0)
1442 return (SET_ERROR(error));
1443
1444 return (0);
1445 }
1446
1447 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, open_timeout_ms, UINT, ZMOD_RW,
1448 "Timeout before determining that a device is missing");
1449
1450 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, failfast_mask, UINT, ZMOD_RW,
1451 "Defines failfast mask: 1 - device, 2 - transport, 4 - driver");
1452
1453 ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, max_segs, UINT, ZMOD_RW,
1454 "Maximum number of data segments to add to an IO request (min 4)");
1455
1456 ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, calling_thread_io, UINT,
1457 ZMOD_RW, "Enable calling thread io");
1458