1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
14 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
15 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
16 * LLNL-CODE-403049.
17 * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
18 * Copyright (c) 2023, 2024, 2025, Klara, Inc.
19 * Copyright (c) 2026, TrueNAS.
20 */
21
22 #include <sys/zfs_context.h>
23 #include <sys/spa_impl.h>
24 #include <sys/vdev_disk.h>
25 #include <sys/vdev_impl.h>
26 #include <sys/vdev_trim.h>
27 #include <sys/abd.h>
28 #include <sys/fs/zfs.h>
29 #include <sys/zio.h>
30 #include <linux/blkpg.h>
31 #include <linux/msdos_fs.h>
32 #include <linux/vfs_compat.h>
33 #include <linux/blk-cgroup.h>
34
35 /*
36 * Linux 6.8.x uses a bdev_handle as an instance/refcount for an underlying
37 * block_device. Since it carries the block_device inside, its convenient to
38 * just use the handle as a proxy.
39 *
40 * Linux 6.9.x uses a file for the same purpose.
41 *
42 * For pre-6.8, we just emulate this with a cast, since we don't need any of
43 * the other fields inside the handle.
44 */
45 #if defined(HAVE_BDEV_OPEN_BY_PATH)
46 typedef struct bdev_handle zfs_bdev_handle_t;
47 #define BDH_BDEV(bdh) ((bdh)->bdev)
48 #define BDH_IS_ERR(bdh) (IS_ERR(bdh))
49 #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
50 #define BDH_ERR_PTR(err) (ERR_PTR(err))
51 #elif defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
52 typedef struct file zfs_bdev_handle_t;
53 #define BDH_BDEV(bdh) (file_bdev(bdh))
54 #define BDH_IS_ERR(bdh) (IS_ERR(bdh))
55 #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
56 #define BDH_ERR_PTR(err) (ERR_PTR(err))
57 #else
58 typedef void zfs_bdev_handle_t;
59 #define BDH_BDEV(bdh) ((struct block_device *)bdh)
60 #define BDH_IS_ERR(bdh) (IS_ERR(BDH_BDEV(bdh)))
61 #define BDH_PTR_ERR(bdh) (PTR_ERR(BDH_BDEV(bdh)))
62 #define BDH_ERR_PTR(err) (ERR_PTR(err))
63 #endif
64
65 typedef struct vdev_disk {
66 zfs_bdev_handle_t *vd_bdh;
67 krwlock_t vd_lock;
68 } vdev_disk_t;
69
70 /*
71 * Maximum number of segments to add to a bio (min 4). If this is higher than
72 * the maximum allowed by the device queue or the kernel itself, it will be
73 * clamped. Setting it to zero will cause the kernel's ideal size to be used.
74 */
75 uint_t zfs_vdev_disk_max_segs = 0;
76
77 /*
78 * Unique identifier for the exclusive vdev holder.
79 */
80 static void *zfs_vdev_holder = VDEV_HOLDER;
81
82 /*
83 * Wait up to zfs_vdev_open_timeout_ms milliseconds before determining the
84 * device is missing. The missing path may be transient since the links
85 * can be briefly removed and recreated in response to udev events.
86 */
87 static uint_t zfs_vdev_open_timeout_ms = 1000;
88
89 /*
90 * Size of the "reserved" partition, in blocks.
91 */
92 #define EFI_MIN_RESV_SIZE (16 * 1024)
93
94 /*
95 * BIO request failfast mask.
96 */
97
98 static unsigned int zfs_vdev_failfast_mask = 1;
99
100 /*
101 * Whether we wait for bio to complete. Also requires that
102 * zio has bypassed the vdev queue. May lead to performance
103 * improvements when backing vdev devices are fast and low
104 * latency. May impact performance with certain workloads
105 * when enabled on raidz or draid zpool configurations.
106 */
107 static unsigned int zfs_vdev_disk_calling_thread_io = 0;
108
109 /*
110 * Convert SPA mode flags into bdev open mode flags.
111 */
112 #ifdef HAVE_BLK_MODE_T
113 typedef blk_mode_t vdev_bdev_mode_t;
114 #define VDEV_BDEV_MODE_READ BLK_OPEN_READ
115 #define VDEV_BDEV_MODE_WRITE BLK_OPEN_WRITE
116 #define VDEV_BDEV_MODE_EXCL BLK_OPEN_EXCL
117 #define VDEV_BDEV_MODE_MASK (BLK_OPEN_READ|BLK_OPEN_WRITE|BLK_OPEN_EXCL)
118 #else
119 typedef fmode_t vdev_bdev_mode_t;
120 #define VDEV_BDEV_MODE_READ FMODE_READ
121 #define VDEV_BDEV_MODE_WRITE FMODE_WRITE
122 #define VDEV_BDEV_MODE_EXCL FMODE_EXCL
123 #define VDEV_BDEV_MODE_MASK (FMODE_READ|FMODE_WRITE|FMODE_EXCL)
124 #endif
125
126 static vdev_bdev_mode_t
vdev_bdev_mode(spa_mode_t smode)127 vdev_bdev_mode(spa_mode_t smode)
128 {
129 ASSERT3U(smode, !=, SPA_MODE_UNINIT);
130 ASSERT0(smode & ~(SPA_MODE_READ|SPA_MODE_WRITE));
131
132 vdev_bdev_mode_t bmode = VDEV_BDEV_MODE_EXCL;
133
134 if (smode & SPA_MODE_READ)
135 bmode |= VDEV_BDEV_MODE_READ;
136
137 if (smode & SPA_MODE_WRITE)
138 bmode |= VDEV_BDEV_MODE_WRITE;
139
140 ASSERT(bmode & VDEV_BDEV_MODE_MASK);
141 ASSERT0(bmode & ~VDEV_BDEV_MODE_MASK);
142
143 return (bmode);
144 }
145
146 /*
147 * Returns the usable capacity (in bytes) for the partition or disk.
148 */
149 static uint64_t
bdev_capacity(struct block_device * bdev)150 bdev_capacity(struct block_device *bdev)
151 {
152 #ifdef HAVE_BDEV_NR_BYTES
153 return (bdev_nr_bytes(bdev));
154 #else
155 return (i_size_read(bdev->bd_inode));
156 #endif
157 }
158
159 #if !defined(HAVE_BDEV_WHOLE)
160 static inline struct block_device *
bdev_whole(struct block_device * bdev)161 bdev_whole(struct block_device *bdev)
162 {
163 return (bdev->bd_contains);
164 }
165 #endif
166
167 #if defined(HAVE_BDEVNAME)
168 #define vdev_bdevname(bdev, name) bdevname(bdev, name)
169 #else
170 static inline void
vdev_bdevname(struct block_device * bdev,char * name)171 vdev_bdevname(struct block_device *bdev, char *name)
172 {
173 snprintf(name, BDEVNAME_SIZE, "%pg", bdev);
174 }
175 #endif
176
177 /*
178 * Returns the maximum expansion capacity of the block device (in bytes).
179 *
180 * It is possible to expand a vdev when it has been created as a wholedisk
181 * and the containing block device has increased in capacity. Or when the
182 * partition containing the pool has been manually increased in size.
183 *
184 * This function is only responsible for calculating the potential expansion
185 * size so it can be reported by 'zpool list'. The efi_use_whole_disk() is
186 * responsible for verifying the expected partition layout in the wholedisk
187 * case, and updating the partition table if appropriate. Once the partition
188 * size has been increased the additional capacity will be visible using
189 * bdev_capacity().
190 *
191 * The returned maximum expansion capacity is always expected to be larger, or
192 * at the very least equal, to its usable capacity to prevent overestimating
193 * the pool expandsize.
194 */
195 static uint64_t
bdev_max_capacity(struct block_device * bdev,uint64_t wholedisk)196 bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk)
197 {
198 uint64_t psize;
199 int64_t available;
200
201 if (wholedisk && bdev != bdev_whole(bdev)) {
202 /*
203 * When reporting maximum expansion capacity for a wholedisk
204 * deduct any capacity which is expected to be lost due to
205 * alignment restrictions. Over reporting this value isn't
206 * harmful and would only result in slightly less capacity
207 * than expected post expansion.
208 * The estimated available space may be slightly smaller than
209 * bdev_capacity() for devices where the number of sectors is
210 * not a multiple of the alignment size and the partition layout
211 * is keeping less than PARTITION_END_ALIGNMENT bytes after the
212 * "reserved" EFI partition: in such cases return the device
213 * usable capacity.
214 */
215 available = bdev_capacity(bdev_whole(bdev)) -
216 ((EFI_MIN_RESV_SIZE + NEW_START_BLOCK +
217 PARTITION_END_ALIGNMENT) << SECTOR_BITS);
218 psize = MAX(available, bdev_capacity(bdev));
219 } else {
220 psize = bdev_capacity(bdev);
221 }
222
223 return (psize);
224 }
225
226 static void
vdev_disk_error(zio_t * zio)227 vdev_disk_error(zio_t *zio)
228 {
229 /*
230 * This function can be called in interrupt context, for instance while
231 * handling IRQs coming from a misbehaving disk device; use printk()
232 * which is safe from any context.
233 */
234 printk(KERN_WARNING "zio pool=%s vdev=%s error=%d type=%d "
235 "offset=%llu size=%llu flags=%llu\n", spa_name(zio->io_spa),
236 zio->io_vd->vdev_path, zio->io_error, zio->io_type,
237 (u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size,
238 zio->io_flags);
239 }
240
241 static void
vdev_disk_kobj_evt_post(vdev_t * v)242 vdev_disk_kobj_evt_post(vdev_t *v)
243 {
244 vdev_disk_t *vd = v->vdev_tsd;
245 if (vd && vd->vd_bdh) {
246 spl_signal_kobj_evt(BDH_BDEV(vd->vd_bdh));
247 } else {
248 vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n",
249 v->vdev_path);
250 }
251 }
252
253 static zfs_bdev_handle_t *
vdev_blkdev_get_by_path(const char * path,spa_mode_t smode,void * holder)254 vdev_blkdev_get_by_path(const char *path, spa_mode_t smode, void *holder)
255 {
256 vdev_bdev_mode_t bmode = vdev_bdev_mode(smode);
257
258 #if defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
259 return (bdev_file_open_by_path(path, bmode, holder, NULL));
260 #elif defined(HAVE_BDEV_OPEN_BY_PATH)
261 return (bdev_open_by_path(path, bmode, holder, NULL));
262 #elif defined(HAVE_BLKDEV_GET_BY_PATH_4ARG)
263 return (blkdev_get_by_path(path, bmode, holder, NULL));
264 #else
265 return (blkdev_get_by_path(path, bmode, holder));
266 #endif
267 }
268
269 static void
vdev_blkdev_put(zfs_bdev_handle_t * bdh,spa_mode_t smode,void * holder)270 vdev_blkdev_put(zfs_bdev_handle_t *bdh, spa_mode_t smode, void *holder)
271 {
272 #if defined(HAVE_BDEV_RELEASE)
273 return (bdev_release(bdh));
274 #elif defined(HAVE_BLKDEV_PUT_HOLDER)
275 return (blkdev_put(BDH_BDEV(bdh), holder));
276 #elif defined(HAVE_BLKDEV_PUT)
277 return (blkdev_put(BDH_BDEV(bdh), vdev_bdev_mode(smode)));
278 #else
279 fput(bdh);
280 #endif
281 }
282
283 static int
vdev_path_backing_permission(struct path * path,int mask)284 vdev_path_backing_permission(struct path *path, int mask)
285 {
286 #if defined(HAVE_IDMAP_MNTIDMAP)
287 return (inode_permission(mnt_idmap(path->mnt),
288 d_backing_inode(path->dentry), mask));
289 #elif defined(HAVE_IDMAP_USERNS)
290 return (inode_permission(mnt_user_ns(path->mnt),
291 d_backing_inode(path->dentry), mask));
292 #else
293 return (inode_permission(d_backing_inode(path->dentry), mask));
294 #endif
295 }
296
297 static int
vdev_disk_open(vdev_t * v,uint64_t * psize,uint64_t * max_psize,uint64_t * logical_ashift,uint64_t * physical_ashift,cred_t * cr)298 vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
299 uint64_t *logical_ashift, uint64_t *physical_ashift, cred_t *cr)
300 {
301 zfs_bdev_handle_t *bdh;
302 spa_mode_t smode = spa_mode(v->vdev_spa);
303 hrtime_t timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms);
304 vdev_disk_t *vd;
305 const cred_t *oldcr = NULL;
306
307 /* Must have a pathname and it must be absolute. */
308 if (v->vdev_path == NULL || v->vdev_path[0] != '/') {
309 v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
310 vdev_dbgmsg(v, "invalid vdev_path");
311 return (SET_ERROR(EINVAL));
312 }
313
314 /*
315 * Reopen the device if it is currently open. When expanding a
316 * partition force re-scanning the partition table if userland
317 * did not take care of this already. We need to do this while closed
318 * in order to get an accurate updated block device size. Then
319 * since udev may need to recreate the device links increase the
320 * open retry timeout before reporting the device as unavailable.
321 */
322 vd = v->vdev_tsd;
323 if (vd) {
324 char disk_name[BDEVNAME_SIZE + 6] = "/dev/";
325 boolean_t reread_part = B_FALSE;
326
327 /*
328 * Reopening an already-open device, so the caller credential
329 * is irrelevant - we had access to it before, we can assume
330 * we still do.
331 */
332 oldcr = override_creds(kcred);
333
334 rw_enter(&vd->vd_lock, RW_WRITER);
335 bdh = vd->vd_bdh;
336 vd->vd_bdh = NULL;
337
338 if (bdh) {
339 struct block_device *bdev = BDH_BDEV(bdh);
340 if (v->vdev_expanding && bdev != bdev_whole(bdev)) {
341 vdev_bdevname(bdev_whole(bdev), disk_name + 5);
342 /*
343 * If userland has BLKPG_RESIZE_PARTITION,
344 * then it should have updated the partition
345 * table already. We can detect this by
346 * comparing our current physical size
347 * with that of the device. If they are
348 * the same, then we must not have
349 * BLKPG_RESIZE_PARTITION or it failed to
350 * update the partition table online. We
351 * fallback to rescanning the partition
352 * table from the kernel below. However,
353 * if the capacity already reflects the
354 * updated partition, then we skip
355 * rescanning the partition table here.
356 */
357 if (v->vdev_psize == bdev_capacity(bdev))
358 reread_part = B_TRUE;
359 }
360
361 vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
362 }
363
364 if (reread_part) {
365 bdh = vdev_blkdev_get_by_path(disk_name, smode,
366 zfs_vdev_holder);
367 if (!BDH_IS_ERR(bdh)) {
368 int error =
369 vdev_bdev_reread_part(BDH_BDEV(bdh));
370 vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
371 if (error == 0) {
372 timeout = MSEC2NSEC(
373 zfs_vdev_open_timeout_ms * 2);
374 }
375 }
376 }
377 } else {
378 vd = kmem_zalloc(sizeof (vdev_disk_t), KM_SLEEP);
379
380 rw_init(&vd->vd_lock, NULL, RW_DEFAULT, NULL);
381 rw_enter(&vd->vd_lock, RW_WRITER);
382
383 /* Restrict device access below to caller's permissions. */
384 oldcr = override_creds(cr);
385 }
386
387 /*
388 * Devices are always opened by the path provided at configuration
389 * time. This means that if the provided path is a udev by-id path
390 * then drives may be re-cabled without an issue. If the provided
391 * path is a udev by-path path, then the physical location information
392 * will be preserved. This can be critical for more complicated
393 * configurations where drives are located in specific physical
394 * locations to maximize the systems tolerance to component failure.
395 *
396 * Alternatively, you can provide your own udev rule to flexibly map
397 * the drives as you see fit. It is not advised that you use the
398 * /dev/[hd]d devices which may be reordered due to probing order.
399 * Devices in the wrong locations will be detected by the higher
400 * level vdev validation.
401 *
402 * The specified paths may be briefly removed and recreated in
403 * response to udev events. This should be exceptionally unlikely
404 * because the zpool command makes every effort to verify these paths
405 * have already settled prior to reaching this point. Therefore,
406 * a ENOENT failure at this point is highly likely to be transient
407 * and it is reasonable to sleep and retry before giving up. In
408 * practice delays have been observed to be on the order of 100ms.
409 *
410 * When ERESTARTSYS is returned it indicates the block device is
411 * a zvol which could not be opened due to the deadlock detection
412 * logic in zvol_open(). Extend the timeout and retry the open
413 * subsequent attempts are expected to eventually succeed.
414 */
415
416 hrtime_t start = gethrtime();
417 int err = -ENXIO;
418 while (err != 0 && ((gethrtime() - start) < timeout)) {
419
420 /*
421 * Ensure the caller credential (made live by override_creds()
422 * above) has access to the device node. This will include
423 * checking file permissions and considering the
424 * CAP_DAC_OVERRIDE capability.
425 */
426 struct path devpath;
427 err = kern_path(v->vdev_path, LOOKUP_FOLLOW, &devpath);
428 if (likely(err == 0)) {
429 int mask =
430 (smode & SPA_MODE_READ ? MAY_READ : 0) |
431 (smode & SPA_MODE_WRITE ? MAY_WRITE : 0);
432 err = vdev_path_backing_permission(&devpath, mask);
433 path_put(&devpath);
434 }
435
436 if (likely(err == 0)) {
437 /*
438 * Device node exists and we have access to it, so try
439 * to open it.
440 */
441 bdh = vdev_blkdev_get_by_path(v->vdev_path, smode,
442 zfs_vdev_holder);
443
444 if (likely(!BDH_IS_ERR(bdh)))
445 /* Device open, nothing more to consider. */
446 break;
447
448 err = BDH_PTR_ERR(bdh);
449 }
450
451 ASSERT3U(err, !=, 0);
452
453 if (err == -ENOENT) {
454 /* Device node disappeared, see above comment. */
455
456 if (v->vdev_removed) {
457 /*
458 * There is no point of waiting since device is
459 * removed explicitly
460 */
461 break;
462 }
463
464 /* Wait a moment, then retry. */
465 schedule_timeout_interruptible(MSEC_TO_TICK(10));
466 continue;
467 }
468
469 if (err == -ERESTARTSYS) {
470 /* zvol deadlock avoided, extend timeout and retry. */
471 timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10);
472 continue;
473 }
474
475 /* Consider all other errors permanent, no retry. */
476 break;
477 }
478
479 revert_creds(oldcr);
480
481 if (err != 0) {
482 vdev_dbgmsg(v, "open error=%d timeout=%llu/%llu", -err,
483 (u_longlong_t)(gethrtime() - start),
484 (u_longlong_t)timeout);
485 vd->vd_bdh = NULL;
486 v->vdev_tsd = vd;
487 rw_exit(&vd->vd_lock);
488 return (SET_ERROR(-err));
489 }
490
491 vd->vd_bdh = bdh;
492 v->vdev_tsd = vd;
493 rw_exit(&vd->vd_lock);
494
495 struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
496
497 /* Determine the physical block size */
498 int physical_block_size = bdev_physical_block_size(bdev);
499
500 /* Determine the logical block size */
501 int logical_block_size = bdev_logical_block_size(bdev);
502
503 /*
504 * If the device has a write cache, clear the nowritecache flag,
505 * so that we start issuing flush requests again.
506 */
507 v->vdev_nowritecache = !zfs_bdev_has_write_cache(bdev);
508
509 /* Set when device reports it supports TRIM. */
510 v->vdev_has_trim = bdev_discard_supported(bdev);
511
512 /* Set when device reports it supports secure TRIM. */
513 v->vdev_has_securetrim = bdev_secure_discard_supported(bdev);
514
515 /* Inform the ZIO pipeline that we are non-rotational */
516 #ifdef HAVE_BLK_QUEUE_ROT
517 v->vdev_nonrot = !blk_queue_rot(bdev_get_queue(bdev));
518 #else
519 v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev));
520 #endif
521
522 /* Is backed by a block device. */
523 v->vdev_is_blkdev = B_TRUE;
524
525 /* Physical volume size in bytes for the partition */
526 *psize = bdev_capacity(bdev);
527
528 /* Physical volume size in bytes including possible expansion space */
529 *max_psize = bdev_max_capacity(bdev, v->vdev_wholedisk);
530
531 /* Based on the minimum sector size set the block size */
532 *physical_ashift = highbit64(MAX(physical_block_size,
533 SPA_MINBLOCKSIZE)) - 1;
534
535 *logical_ashift = highbit64(MAX(logical_block_size,
536 SPA_MINBLOCKSIZE)) - 1;
537
538 return (0);
539 }
540
541 static void
vdev_disk_close(vdev_t * v)542 vdev_disk_close(vdev_t *v)
543 {
544 vdev_disk_t *vd = v->vdev_tsd;
545
546 if (v->vdev_reopening || vd == NULL)
547 return;
548
549 rw_enter(&vd->vd_lock, RW_WRITER);
550
551 if (vd->vd_bdh != NULL)
552 vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa),
553 zfs_vdev_holder);
554
555 v->vdev_tsd = NULL;
556
557 rw_exit(&vd->vd_lock);
558 rw_destroy(&vd->vd_lock);
559 kmem_free(vd, sizeof (vdev_disk_t));
560 }
561
562 /*
563 * preempt_schedule_notrace is GPL-only which breaks the ZFS build, so
564 * replace it with preempt_schedule under the following condition:
565 */
566 #if defined(CONFIG_ARM64) && \
567 defined(CONFIG_PREEMPTION) && \
568 defined(CONFIG_BLK_CGROUP)
569 #define preempt_schedule_notrace(x) preempt_schedule(x)
570 #endif
571
572 /*
573 * As for the Linux 5.18 kernel bio_alloc() expects a block_device struct
574 * as an argument removing the need to set it with bio_set_dev(). This
575 * removes the need for all of the following compatibility code.
576 */
577 #if !defined(HAVE_BIO_ALLOC_4ARG)
578
579 #if defined(CONFIG_BLK_CGROUP) && defined(HAVE_BIO_SET_DEV_GPL_ONLY)
580 /*
581 * The Linux 5.5 kernel updated percpu_ref_tryget() which is inlined by
582 * blkg_tryget() to use rcu_read_lock() instead of rcu_read_lock_sched().
583 * As a side effect the function was converted to GPL-only. Define our
584 * own version when needed which uses rcu_read_lock_sched().
585 *
586 * The Linux 5.17 kernel split linux/blk-cgroup.h into a private and a public
587 * part, moving blkg_tryget into the private one. Define our own version.
588 */
589 #if defined(HAVE_BLKG_TRYGET_GPL_ONLY) || !defined(HAVE_BLKG_TRYGET)
590 static inline bool
vdev_blkg_tryget(struct blkcg_gq * blkg)591 vdev_blkg_tryget(struct blkcg_gq *blkg)
592 {
593 struct percpu_ref *ref = &blkg->refcnt;
594 unsigned long __percpu *count;
595 bool rc;
596
597 rcu_read_lock_sched();
598
599 if (__ref_is_percpu(ref, &count)) {
600 this_cpu_inc(*count);
601 rc = true;
602 } else {
603 #ifdef ZFS_PERCPU_REF_COUNT_IN_DATA
604 rc = atomic_long_inc_not_zero(&ref->data->count);
605 #else
606 rc = atomic_long_inc_not_zero(&ref->count);
607 #endif
608 }
609
610 rcu_read_unlock_sched();
611
612 return (rc);
613 }
614 #else
615 #define vdev_blkg_tryget(bg) blkg_tryget(bg)
616 #endif
617 #ifdef HAVE_BIO_SET_DEV_MACRO
618 /*
619 * The Linux 5.0 kernel updated the bio_set_dev() macro so it calls the
620 * GPL-only bio_associate_blkg() symbol thus inadvertently converting
621 * the entire macro. Provide a minimal version which always assigns the
622 * request queue's root_blkg to the bio.
623 */
624 static inline void
vdev_bio_associate_blkg(struct bio * bio)625 vdev_bio_associate_blkg(struct bio *bio)
626 {
627 #if defined(HAVE_BIO_BDEV_DISK)
628 struct request_queue *q = bio->bi_bdev->bd_disk->queue;
629 #else
630 struct request_queue *q = bio->bi_disk->queue;
631 #endif
632
633 ASSERT3P(q, !=, NULL);
634 ASSERT0P(bio->bi_blkg);
635
636 if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
637 bio->bi_blkg = q->root_blkg;
638 }
639
640 #define bio_associate_blkg vdev_bio_associate_blkg
641 #else
642 static inline void
vdev_bio_set_dev(struct bio * bio,struct block_device * bdev)643 vdev_bio_set_dev(struct bio *bio, struct block_device *bdev)
644 {
645 #if defined(HAVE_BIO_BDEV_DISK)
646 struct request_queue *q = bdev->bd_disk->queue;
647 #else
648 struct request_queue *q = bio->bi_disk->queue;
649 #endif
650 bio_clear_flag(bio, BIO_REMAPPED);
651 if (bio->bi_bdev != bdev)
652 bio_clear_flag(bio, BIO_THROTTLED);
653 bio->bi_bdev = bdev;
654
655 ASSERT3P(q, !=, NULL);
656 ASSERT0P(bio->bi_blkg);
657
658 if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
659 bio->bi_blkg = q->root_blkg;
660 }
661 #define bio_set_dev vdev_bio_set_dev
662 #endif
663 #endif
664 #endif /* !HAVE_BIO_ALLOC_4ARG */
665
666 static inline void
vdev_submit_bio(struct bio * bio)667 vdev_submit_bio(struct bio *bio)
668 {
669 struct bio_list *bio_list = current->bio_list;
670 current->bio_list = NULL;
671 (void) submit_bio(bio);
672 current->bio_list = bio_list;
673 }
674
675 static inline void
vdev_submit_bio_wait(struct bio * bio)676 vdev_submit_bio_wait(struct bio *bio)
677 {
678 struct bio_list *bio_list = current->bio_list;
679 current->bio_list = NULL;
680 (void) submit_bio_wait(bio);
681 current->bio_list = bio_list;
682 }
683
684 static inline struct bio *
vdev_bio_alloc(struct block_device * bdev,gfp_t gfp_mask,unsigned short nr_vecs)685 vdev_bio_alloc(struct block_device *bdev, gfp_t gfp_mask,
686 unsigned short nr_vecs)
687 {
688 struct bio *bio;
689
690 #ifdef HAVE_BIO_ALLOC_4ARG
691 bio = bio_alloc(bdev, nr_vecs, 0, gfp_mask);
692 #else
693 bio = bio_alloc(gfp_mask, nr_vecs);
694 if (likely(bio != NULL))
695 bio_set_dev(bio, bdev);
696 #endif
697
698 return (bio);
699 }
700
701 static inline uint_t
vdev_bio_max_segs(struct block_device * bdev)702 vdev_bio_max_segs(struct block_device *bdev)
703 {
704 /*
705 * Smallest of the device max segs and the tunable max segs. Minimum
706 * 4, so there's room to finish split pages if they come up.
707 */
708 const uint_t dev_max_segs = queue_max_segments(bdev_get_queue(bdev));
709 const uint_t tune_max_segs = (zfs_vdev_disk_max_segs > 0) ?
710 MAX(4, zfs_vdev_disk_max_segs) : dev_max_segs;
711 const uint_t max_segs = MIN(tune_max_segs, dev_max_segs);
712
713 #ifdef HAVE_BIO_MAX_SEGS
714 return (bio_max_segs(max_segs));
715 #else
716 return (MIN(max_segs, BIO_MAX_PAGES));
717 #endif
718 }
719
720 static inline uint_t
vdev_bio_max_bytes(struct block_device * bdev)721 vdev_bio_max_bytes(struct block_device *bdev)
722 {
723 return (queue_max_sectors(bdev_get_queue(bdev)) << 9);
724 }
725
726
727 /*
728 * Virtual block IO object (VBIO)
729 *
730 * Linux block IO (BIO) objects have a limit on how many data segments (pages)
731 * they can hold. Depending on how they're allocated and structured, a large
732 * ZIO can require more than one BIO to be submitted to the kernel, which then
733 * all have to complete before we can return the completed ZIO back to ZFS.
734 *
735 * A VBIO is a wrapper around multiple BIOs, carrying everything needed to
736 * translate a ZIO down into the kernel block layer and back again.
737 *
738 * Note that these are only used for data ZIOs (read/write). Meta-operations
739 * (flush/trim) don't need multiple BIOs and so can just make the call
740 * directly.
741 */
742 typedef struct {
743 zio_t *vbio_zio; /* parent zio */
744
745 struct block_device *vbio_bdev; /* blockdev to submit bios to */
746
747 abd_t *vbio_abd; /* abd carrying borrowed linear buf */
748
749 uint_t vbio_max_segs; /* max segs per bio */
750
751 uint_t vbio_max_bytes; /* max bytes per bio */
752 uint_t vbio_lbs_mask; /* logical block size mask */
753
754 uint64_t vbio_offset; /* start offset of next bio */
755
756 struct bio *vbio_bio; /* pointer to the current bio */
757 int vbio_flags; /* bio flags */
758 boolean_t vbio_wait; /* wait for completion */
759 } vbio_t;
760
761 static vbio_t *
vbio_alloc(zio_t * zio,struct block_device * bdev,int flags)762 vbio_alloc(zio_t *zio, struct block_device *bdev, int flags)
763 {
764 vbio_t *vbio = kmem_zalloc(sizeof (vbio_t), KM_SLEEP);
765
766 vbio->vbio_zio = zio;
767 vbio->vbio_bdev = bdev;
768 vbio->vbio_abd = NULL;
769 vbio->vbio_max_segs = vdev_bio_max_segs(bdev);
770 vbio->vbio_max_bytes = vdev_bio_max_bytes(bdev);
771 vbio->vbio_lbs_mask = ~(bdev_logical_block_size(bdev)-1);
772 vbio->vbio_offset = zio->io_offset;
773 vbio->vbio_bio = NULL;
774 vbio->vbio_flags = flags;
775 vbio->vbio_wait = B_FALSE;
776
777 return (vbio);
778 }
779
780 static void vbio_completion(struct bio *bio);
781
782 static int
vbio_add_page(vbio_t * vbio,struct page * page,uint_t size,uint_t offset)783 vbio_add_page(vbio_t *vbio, struct page *page, uint_t size, uint_t offset)
784 {
785 struct bio *bio = vbio->vbio_bio;
786 uint_t ssize;
787
788 while (size > 0) {
789 if (bio == NULL) {
790 /* New BIO, allocate and set up */
791 bio = vdev_bio_alloc(vbio->vbio_bdev, GFP_NOIO,
792 vbio->vbio_max_segs);
793 VERIFY(bio);
794
795 BIO_BI_SECTOR(bio) = vbio->vbio_offset >> 9;
796 bio_set_op_attrs(bio,
797 vbio->vbio_zio->io_type == ZIO_TYPE_WRITE ?
798 WRITE : READ, vbio->vbio_flags);
799
800 if (vbio->vbio_bio) {
801 bio_chain(vbio->vbio_bio, bio);
802 vdev_submit_bio(vbio->vbio_bio);
803 }
804 vbio->vbio_bio = bio;
805 }
806
807 /*
808 * Only load as much of the current page data as will fit in
809 * the space left in the BIO, respecting lbs alignment. Older
810 * kernels will error if we try to overfill the BIO, while
811 * newer ones will accept it and split the BIO. This ensures
812 * everything works on older kernels, and avoids an additional
813 * overhead on the new.
814 */
815 ssize = MIN(size, (vbio->vbio_max_bytes - BIO_BI_SIZE(bio)) &
816 vbio->vbio_lbs_mask);
817 if (ssize > 0 &&
818 bio_add_page(bio, page, ssize, offset) == ssize) {
819 /* Accepted, adjust and load any remaining. */
820 size -= ssize;
821 offset += ssize;
822 continue;
823 }
824
825 /* No room, set up for a new BIO and loop */
826 vbio->vbio_offset += BIO_BI_SIZE(bio);
827
828 /* Signal new BIO allocation wanted */
829 bio = NULL;
830 }
831
832 return (0);
833 }
834
835 /* Iterator callback to submit ABD pages to the vbio. */
836 static int
vbio_fill_cb(struct page * page,size_t off,size_t len,void * priv)837 vbio_fill_cb(struct page *page, size_t off, size_t len, void *priv)
838 {
839 vbio_t *vbio = priv;
840 return (vbio_add_page(vbio, page, len, off));
841 }
842
843 /* Create some BIOs, fill them with data and submit them */
844 static void
vbio_submit(vbio_t * vbio,abd_t * abd,uint64_t size)845 vbio_submit(vbio_t *vbio, abd_t *abd, uint64_t size)
846 {
847 /*
848 * We plug so we can submit the BIOs as we go and only unplug them when
849 * they are fully created and submitted. This is important; if we don't
850 * plug, then the kernel may start executing earlier BIOs while we're
851 * still creating and executing later ones, and if the device goes
852 * away while that's happening, older kernels can get confused and
853 * trample memory.
854 */
855 struct blk_plug plug;
856 blk_start_plug(&plug);
857
858 (void) abd_iterate_page_func(abd, 0, size, vbio_fill_cb, vbio);
859 ASSERT(vbio->vbio_bio);
860
861 /*
862 * Once submitted, vbio_bio now owns vbio (through bi_private) and we
863 * can't touch it again. The bio may complete and vbio_completion() be
864 * called and free the vbio before this task is run again, so we must
865 * consider it invalid from this point.
866 */
867
868 if (vbio->vbio_wait) {
869 vdev_submit_bio_wait(vbio->vbio_bio);
870 } else {
871 vbio->vbio_bio->bi_end_io = vbio_completion;
872 vbio->vbio_bio->bi_private = vbio;
873 vdev_submit_bio(vbio->vbio_bio);
874 }
875
876 blk_finish_plug(&plug);
877 }
878
879 /* IO completion callback */
880 static void
vbio_completion(struct bio * bio)881 vbio_completion(struct bio *bio)
882 {
883 vbio_t *vbio = bio->bi_private;
884 zio_t *zio = vbio->vbio_zio;
885
886 ASSERT(zio);
887
888 /* Capture and log any errors */
889 zio->io_error = bi_status_to_errno(bio->bi_status);
890 ASSERT3U(zio->io_error, >=, 0);
891
892 if (zio->io_error)
893 vdev_disk_error(zio);
894
895 /* Return the BIO to the kernel */
896 bio_put(bio);
897
898 /*
899 * We're likely in an interrupt context so we can't do ABD/memory work
900 * here; instead we stash vbio on the zio and take care of it in the
901 * done callback.
902 */
903 ASSERT0P(zio->io_bio);
904 zio->io_bio = vbio;
905
906 /* Using calling thread io, don't dispatch zio. */
907 if (vbio->vbio_wait)
908 zio_execute(zio);
909 else
910 zio_delay_interrupt(zio);
911
912 }
913
914 /*
915 * Iterator callback to count ABD pages and check their size & alignment.
916 *
917 * On Linux, each BIO segment can take a page pointer, and an offset+length of
918 * the data within that page. A page can be arbitrarily large ("compound"
919 * pages) but we still have to ensure the data portion is correctly sized and
920 * aligned to the logical block size, to ensure that if the kernel wants to
921 * split the BIO, the two halves will still be properly aligned.
922 *
923 * NOTE: if you change this function, change the copy in
924 * tests/zfs-tests/tests/functional/vdev_disk/page_alignment.c, and add test
925 * data there to validate the change you're making.
926 */
927 typedef struct {
928 size_t blocksize;
929 int seen_first;
930 int seen_last;
931 } vdev_disk_check_alignment_t;
932
933 static int
vdev_disk_check_alignment_cb(struct page * page,size_t off,size_t len,void * priv)934 vdev_disk_check_alignment_cb(struct page *page, size_t off, size_t len,
935 void *priv)
936 {
937 (void) page;
938 vdev_disk_check_alignment_t *s = priv;
939
940 /*
941 * The cardinal rule: a single on-disk block must never cross an
942 * physical (order-0) page boundary, as the kernel expects to be able
943 * to split at both LBS and page boundaries.
944 *
945 * This implies various alignment rules for the blocks in this
946 * (possibly compound) page, which we can check for.
947 */
948
949 /*
950 * If the previous page did not end on a page boundary, then we
951 * can't proceed without creating a hole.
952 */
953 if (s->seen_last)
954 return (1);
955
956 /* This page must contain only whole LBS-sized blocks. */
957 if (!IS_P2ALIGNED(len, s->blocksize))
958 return (1);
959
960 /*
961 * If this is not the first page in the ABD, then the data must start
962 * on a page-aligned boundary (so the kernel can split on page
963 * boundaries without having to deal with a hole). If it is, then
964 * it can start on LBS-alignment.
965 */
966 if (s->seen_first) {
967 if (!IS_P2ALIGNED(off, PAGESIZE))
968 return (1);
969 } else {
970 if (!IS_P2ALIGNED(off, s->blocksize))
971 return (1);
972 s->seen_first = 1;
973 }
974
975 /*
976 * If this data does not end on a page-aligned boundary, then this
977 * must be the last page in the ABD, for the same reason.
978 */
979 s->seen_last = !IS_P2ALIGNED(off+len, PAGESIZE);
980
981 return (0);
982 }
983
984 /*
985 * Check if we can submit the pages in this ABD to the kernel as-is. Returns
986 * the number of pages, or 0 if it can't be submitted like this.
987 */
988 static boolean_t
vdev_disk_check_alignment(abd_t * abd,uint64_t size,struct block_device * bdev)989 vdev_disk_check_alignment(abd_t *abd, uint64_t size, struct block_device *bdev)
990 {
991 vdev_disk_check_alignment_t s = {
992 .blocksize = bdev_logical_block_size(bdev),
993 };
994
995 if (abd_iterate_page_func(abd, 0, size,
996 vdev_disk_check_alignment_cb, &s))
997 return (B_FALSE);
998
999 return (B_TRUE);
1000 }
1001
1002 static int
vdev_disk_io_rw(zio_t * zio)1003 vdev_disk_io_rw(zio_t *zio)
1004 {
1005 vdev_t *v = zio->io_vd;
1006 vdev_disk_t *vd = v->vdev_tsd;
1007 struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
1008 int flags = 0;
1009
1010 /*
1011 * Accessing outside the block device is never allowed.
1012 */
1013 if (zio->io_offset + zio->io_size > bdev_capacity(bdev)) {
1014 vdev_dbgmsg(zio->io_vd,
1015 "Illegal access %llu size %llu, device size %llu",
1016 (u_longlong_t)zio->io_offset,
1017 (u_longlong_t)zio->io_size,
1018 (u_longlong_t)bdev_capacity(bdev));
1019 return (SET_ERROR(EIO));
1020 }
1021
1022 vdev_t *iter = v;
1023 while (iter != NULL && iter->vdev_failfast == ZPROP_BOOLEAN_INHERIT)
1024 iter = iter->vdev_parent;
1025
1026 boolean_t failfast = iter ? iter->vdev_failfast == 1 :
1027 vdev_prop_default_numeric(VDEV_PROP_FAILFAST);
1028 if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)) &&
1029 failfast) {
1030 bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
1031 zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
1032 }
1033
1034 /*
1035 * Check alignment of the incoming ABD. If any part of it would require
1036 * submitting a page that is not aligned to both the logical block size
1037 * and the page size, then we take a copy into a new memory region with
1038 * correct alignment. This should be impossible on a 512b LBS. On
1039 * larger blocks, this can happen at least when a small number of
1040 * blocks (usually 1) are allocated from a shared slab, or when
1041 * abnormally-small data regions (eg gang headers) are mixed into the
1042 * same ABD as larger allocations (eg aggregations).
1043 */
1044 abd_t *abd = zio->io_abd;
1045 if (!vdev_disk_check_alignment(abd, zio->io_size, bdev)) {
1046 /* Allocate a new memory region with guaranteed alignment */
1047 abd = abd_alloc_for_io(zio->io_size,
1048 zio->io_abd->abd_flags & ABD_FLAG_META);
1049
1050 /* If we're writing copy our data into it */
1051 if (zio->io_type == ZIO_TYPE_WRITE)
1052 abd_copy(abd, zio->io_abd, zio->io_size);
1053
1054 /*
1055 * False here would mean the new allocation has an invalid
1056 * alignment too, which would mean that abd_alloc() is not
1057 * guaranteeing this, or our logic in
1058 * vdev_disk_check_alignment() is wrong. In either case,
1059 * something in seriously wrong and its not safe to continue.
1060 */
1061 VERIFY(vdev_disk_check_alignment(abd, zio->io_size, bdev));
1062 }
1063
1064 /* Allocate vbio, with a pointer to the borrowed ABD if necessary */
1065 vbio_t *vbio = vbio_alloc(zio, bdev, flags);
1066 if (abd != zio->io_abd)
1067 vbio->vbio_abd = abd;
1068
1069 boolean_t bio_wait = B_FALSE;
1070 if (zfs_vdev_disk_calling_thread_io &&
1071 (zio->io_flags & ZIO_FLAG_BYPASSED_QUEUE)) {
1072 vbio->vbio_wait = bio_wait = B_TRUE;
1073 }
1074 /* Fill it with data pages and submit it to the kernel */
1075 vbio_submit(vbio, abd, zio->io_size);
1076
1077 if (bio_wait) {
1078 vbio->vbio_bio->bi_private = vbio;
1079 vbio_completion(vbio->vbio_bio);
1080 }
1081
1082 return (0);
1083 }
1084
1085 static void
vdev_disk_io_flush_completion(struct bio * bio)1086 vdev_disk_io_flush_completion(struct bio *bio)
1087 {
1088 zio_t *zio = bio->bi_private;
1089 zio->io_error = bi_status_to_errno(bio->bi_status);
1090 if (zio->io_error == EOPNOTSUPP || zio->io_error == ENOTTY)
1091 zio->io_error = SET_ERROR(ENOTSUP);
1092
1093 bio_put(bio);
1094 ASSERT3S(zio->io_error, >=, 0);
1095 if (zio->io_error)
1096 vdev_disk_error(zio);
1097 zio_interrupt(zio);
1098 }
1099
1100 static int
vdev_disk_io_flush(struct block_device * bdev,zio_t * zio)1101 vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
1102 {
1103 struct request_queue *q;
1104 struct bio *bio;
1105
1106 q = bdev_get_queue(bdev);
1107 if (!q)
1108 return (SET_ERROR(ENXIO));
1109
1110 bio = vdev_bio_alloc(bdev, GFP_NOIO, 0);
1111 if (unlikely(bio == NULL))
1112 return (SET_ERROR(ENOMEM));
1113
1114 bio->bi_end_io = vdev_disk_io_flush_completion;
1115 bio->bi_private = zio;
1116 bio_set_flush(bio);
1117 vdev_submit_bio(bio);
1118 invalidate_bdev(bdev);
1119
1120 return (0);
1121 }
1122
1123 static void
vdev_disk_discard_end_io(struct bio * bio)1124 vdev_disk_discard_end_io(struct bio *bio)
1125 {
1126 zio_t *zio = bio->bi_private;
1127 zio->io_error = bi_status_to_errno(bio->bi_status);
1128
1129 bio_put(bio);
1130 if (zio->io_error)
1131 vdev_disk_error(zio);
1132 zio_interrupt(zio);
1133 }
1134
1135 /*
1136 * Wrappers for the different secure erase and discard APIs. We use async
1137 * when available; in this case, *biop is set to the last bio in the chain.
1138 */
1139 static int
vdev_bdev_issue_secure_erase(zfs_bdev_handle_t * bdh,sector_t sector,sector_t nsect,struct bio ** biop)1140 vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector,
1141 sector_t nsect, struct bio **biop)
1142 {
1143 *biop = NULL;
1144 int error;
1145
1146 #if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
1147 error = blkdev_issue_secure_erase(BDH_BDEV(bdh),
1148 sector, nsect, GFP_NOFS);
1149 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
1150 error = __blkdev_issue_discard(BDH_BDEV(bdh),
1151 sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop);
1152 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
1153 error = blkdev_issue_discard(BDH_BDEV(bdh),
1154 sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE);
1155 #else
1156 #error "unsupported kernel"
1157 #endif
1158
1159 return (error);
1160 }
1161
1162 static int
vdev_bdev_issue_discard(zfs_bdev_handle_t * bdh,sector_t sector,sector_t nsect,struct bio ** biop)1163 vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector,
1164 sector_t nsect, struct bio **biop)
1165 {
1166 *biop = NULL;
1167 int error;
1168
1169 #if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
1170 error = __blkdev_issue_discard(BDH_BDEV(bdh),
1171 sector, nsect, GFP_NOFS, 0, biop);
1172 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS)
1173 error = __blkdev_issue_discard(BDH_BDEV(bdh),
1174 sector, nsect, GFP_NOFS, biop);
1175 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
1176 error = blkdev_issue_discard(BDH_BDEV(bdh),
1177 sector, nsect, GFP_NOFS, 0);
1178 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS)
1179 error = blkdev_issue_discard(BDH_BDEV(bdh),
1180 sector, nsect, GFP_NOFS);
1181 #else
1182 #error "unsupported kernel"
1183 #endif
1184
1185 return (error);
1186 }
1187
1188 /*
1189 * Entry point for TRIM ops. This calls the right wrapper for secure erase or
1190 * discard, and then does the appropriate finishing work for error vs success
1191 * and async vs sync.
1192 */
1193 static int
vdev_disk_io_trim(zio_t * zio)1194 vdev_disk_io_trim(zio_t *zio)
1195 {
1196 int error;
1197 struct bio *bio;
1198
1199 zfs_bdev_handle_t *bdh = ((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh;
1200 sector_t sector = zio->io_offset >> 9;
1201 sector_t nsects = zio->io_size >> 9;
1202
1203 if (zio->io_trim_flags & ZIO_TRIM_SECURE)
1204 error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);
1205 else
1206 error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);
1207
1208 if (error != 0)
1209 return (SET_ERROR(-error));
1210
1211 if (bio == NULL) {
1212 /*
1213 * This was a synchronous op that completed successfully, so
1214 * return it to ZFS immediately.
1215 */
1216 zio_interrupt(zio);
1217 } else {
1218 /*
1219 * This was an asynchronous op; set up completion callback and
1220 * issue it.
1221 */
1222 bio->bi_private = zio;
1223 bio->bi_end_io = vdev_disk_discard_end_io;
1224 vdev_submit_bio(bio);
1225 }
1226
1227 return (0);
1228 }
1229
1230 static void
vdev_disk_io_start(zio_t * zio)1231 vdev_disk_io_start(zio_t *zio)
1232 {
1233 vdev_t *v = zio->io_vd;
1234 vdev_disk_t *vd = v->vdev_tsd;
1235 int error;
1236
1237 /*
1238 * If the vdev is closed, it's likely in the REMOVED or FAULTED state.
1239 * Nothing to be done here but return failure.
1240 */
1241 if (vd == NULL) {
1242 zio->io_error = ENXIO;
1243 zio_interrupt(zio);
1244 return;
1245 }
1246
1247 rw_enter(&vd->vd_lock, RW_READER);
1248
1249 /*
1250 * If the vdev is closed, it's likely due to a failed reopen and is
1251 * in the UNAVAIL state. Nothing to be done here but return failure.
1252 */
1253 if (vd->vd_bdh == NULL) {
1254 rw_exit(&vd->vd_lock);
1255 zio->io_error = ENXIO;
1256 zio_interrupt(zio);
1257 return;
1258 }
1259
1260 switch (zio->io_type) {
1261 case ZIO_TYPE_FLUSH:
1262
1263 if (!vdev_readable(v)) {
1264 /* Drive not there, can't flush */
1265 error = SET_ERROR(ENXIO);
1266 } else if (zfs_nocacheflush) {
1267 /* Flushing disabled by operator, declare success */
1268 error = 0;
1269 } else if (v->vdev_nowritecache) {
1270 /* This vdev not capable of flushing */
1271 error = SET_ERROR(ENOTSUP);
1272 } else {
1273 /*
1274 * Issue the flush. If successful, the response will
1275 * be handled in the completion callback, so we're done.
1276 */
1277 error = vdev_disk_io_flush(BDH_BDEV(vd->vd_bdh), zio);
1278 if (error == 0) {
1279 rw_exit(&vd->vd_lock);
1280 return;
1281 }
1282 }
1283
1284 /* Couldn't issue the flush, so set the error and return it */
1285 rw_exit(&vd->vd_lock);
1286 zio->io_error = error;
1287 zio_execute(zio);
1288 return;
1289
1290 case ZIO_TYPE_TRIM:
1291 error = vdev_disk_io_trim(zio);
1292 rw_exit(&vd->vd_lock);
1293 if (error) {
1294 zio->io_error = error;
1295 zio_execute(zio);
1296 }
1297 return;
1298
1299 case ZIO_TYPE_READ:
1300 case ZIO_TYPE_WRITE:
1301 zio->io_target_timestamp = zio_handle_io_delay(zio);
1302 error = vdev_disk_io_rw(zio);
1303 rw_exit(&vd->vd_lock);
1304 if (error) {
1305 zio->io_error = error;
1306 zio_interrupt(zio);
1307 }
1308 return;
1309
1310 default:
1311 /*
1312 * Getting here means our parent vdev has made a very strange
1313 * request of us, and shouldn't happen. Assert here to force a
1314 * crash in dev builds, but in production return the IO
1315 * unhandled. The pool will likely suspend anyway but that's
1316 * nicer than crashing the kernel.
1317 */
1318 ASSERT3S(zio->io_type, ==, -1);
1319
1320 rw_exit(&vd->vd_lock);
1321 zio->io_error = SET_ERROR(ENOTSUP);
1322 zio_interrupt(zio);
1323 return;
1324 }
1325
1326 __builtin_unreachable();
1327 }
1328
1329 static void
vdev_disk_io_done(zio_t * zio)1330 vdev_disk_io_done(zio_t *zio)
1331 {
1332 /* If this was a read or write, we need to clean up the vbio */
1333 if (zio->io_bio != NULL) {
1334 vbio_t *vbio = zio->io_bio;
1335 zio->io_bio = NULL;
1336
1337 /*
1338 * If we copied the ABD before issuing it, clean up and return
1339 * the copy to the ADB, with changes if appropriate.
1340 */
1341 if (vbio->vbio_abd != NULL) {
1342 if (zio->io_type == ZIO_TYPE_READ)
1343 abd_copy(zio->io_abd, vbio->vbio_abd,
1344 zio->io_size);
1345
1346 abd_free(vbio->vbio_abd);
1347 vbio->vbio_abd = NULL;
1348 }
1349
1350 /* Final cleanup */
1351 kmem_free(vbio, sizeof (vbio_t));
1352 }
1353
1354 /*
1355 * If the device returned EIO, we revalidate the media. If it is
1356 * determined the media has changed this triggers the asynchronous
1357 * removal of the device from the configuration.
1358 */
1359 if (zio->io_error == EIO) {
1360 vdev_t *v = zio->io_vd;
1361 vdev_disk_t *vd = v->vdev_tsd;
1362
1363 if (!zfs_check_disk_status(BDH_BDEV(vd->vd_bdh))) {
1364 invalidate_bdev(BDH_BDEV(vd->vd_bdh));
1365 v->vdev_remove_wanted = B_TRUE;
1366 spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
1367 }
1368 }
1369 }
1370
1371 static void
vdev_disk_hold(vdev_t * vd)1372 vdev_disk_hold(vdev_t *vd)
1373 {
1374 ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
1375
1376 /* We must have a pathname, and it must be absolute. */
1377 if (vd->vdev_path == NULL || vd->vdev_path[0] != '/')
1378 return;
1379
1380 /*
1381 * Only prefetch path and devid info if the device has
1382 * never been opened.
1383 */
1384 if (vd->vdev_tsd != NULL)
1385 return;
1386
1387 }
1388
1389 static void
vdev_disk_rele(vdev_t * vd)1390 vdev_disk_rele(vdev_t *vd)
1391 {
1392 ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
1393
1394 /* XXX: Implement me as a vnode rele for the device */
1395 }
1396
1397 vdev_ops_t vdev_disk_ops = {
1398 .vdev_op_init = NULL,
1399 .vdev_op_fini = NULL,
1400 .vdev_op_open = vdev_disk_open,
1401 .vdev_op_close = vdev_disk_close,
1402 .vdev_op_asize_to_psize = vdev_default_psize,
1403 .vdev_op_psize_to_asize = vdev_default_asize,
1404 .vdev_op_min_asize = vdev_default_min_asize,
1405 .vdev_op_min_alloc = NULL,
1406 .vdev_op_io_start = vdev_disk_io_start,
1407 .vdev_op_io_done = vdev_disk_io_done,
1408 .vdev_op_state_change = NULL,
1409 .vdev_op_need_resilver = NULL,
1410 .vdev_op_hold = vdev_disk_hold,
1411 .vdev_op_rele = vdev_disk_rele,
1412 .vdev_op_remap = NULL,
1413 .vdev_op_xlate = vdev_default_xlate,
1414 .vdev_op_rebuild_asize = NULL,
1415 .vdev_op_metaslab_init = NULL,
1416 .vdev_op_config_generate = NULL,
1417 .vdev_op_nparity = NULL,
1418 .vdev_op_ndisks = NULL,
1419 .vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */
1420 .vdev_op_leaf = B_TRUE, /* leaf vdev */
1421 .vdev_op_kobj_evt_post = vdev_disk_kobj_evt_post
1422 };
1423
1424 int
param_set_min_auto_ashift(const char * buf,zfs_kernel_param_t * kp)1425 param_set_min_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
1426 {
1427 uint_t val;
1428 int error;
1429
1430 error = kstrtouint(buf, 0, &val);
1431 if (error < 0)
1432 return (SET_ERROR(error));
1433
1434 if (val < ASHIFT_MIN || val > zfs_vdev_max_auto_ashift)
1435 return (SET_ERROR(-EINVAL));
1436
1437 error = param_set_uint(buf, kp);
1438 if (error < 0)
1439 return (SET_ERROR(error));
1440
1441 return (0);
1442 }
1443
1444 int
param_set_max_auto_ashift(const char * buf,zfs_kernel_param_t * kp)1445 param_set_max_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
1446 {
1447 uint_t val;
1448 int error;
1449
1450 error = kstrtouint(buf, 0, &val);
1451 if (error < 0)
1452 return (SET_ERROR(error));
1453
1454 if (val > ASHIFT_MAX || val < zfs_vdev_min_auto_ashift)
1455 return (SET_ERROR(-EINVAL));
1456
1457 error = param_set_uint(buf, kp);
1458 if (error < 0)
1459 return (SET_ERROR(error));
1460
1461 return (0);
1462 }
1463
1464 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, open_timeout_ms, UINT, ZMOD_RW,
1465 "Timeout before determining that a device is missing");
1466
1467 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, failfast_mask, UINT, ZMOD_RW,
1468 "Defines failfast mask: 1 - device, 2 - transport, 4 - driver");
1469
1470 ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, max_segs, UINT, ZMOD_RW,
1471 "Maximum number of data segments to add to an IO request (min 4)");
1472
1473 ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, calling_thread_io, UINT,
1474 ZMOD_RW, "Enable calling thread io");
1475