1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
15 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
16 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
17 * LLNL-CODE-403049.
18 */
19
20 #ifndef _ZFS_BLKDEV_H
21 #define _ZFS_BLKDEV_H
22
23 #include <linux/blkdev.h>
24 #include <linux/backing-dev.h>
25 #include <linux/hdreg.h>
26 #include <linux/major.h>
27 #include <linux/msdos_fs.h> /* for SECTOR_* */
28 #include <linux/bio.h>
29 #include <linux/blk-mq.h>
30
31 /*
32 * 6.11 API
33 * Setting the flush flags directly is no longer possible; flush flags are set
34 * on the queue_limits structure and passed to blk_disk_alloc(). In this case
35 * we remove this function entirely.
36 */
37 #if !defined(HAVE_BLK_ALLOC_DISK_2ARG) || \
38 !defined(HAVE_BLKDEV_QUEUE_LIMITS_FEATURES)
39 static inline void
blk_queue_set_write_cache(struct request_queue * q,bool on)40 blk_queue_set_write_cache(struct request_queue *q, bool on)
41 {
42 if (on) {
43 blk_queue_flag_set(QUEUE_FLAG_WC, q);
44 blk_queue_flag_set(QUEUE_FLAG_FUA, q);
45 } else {
46 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
47 blk_queue_flag_clear(QUEUE_FLAG_FUA, q);
48 }
49 }
50 #endif /* !HAVE_BLK_ALLOC_DISK_2ARG || !HAVE_BLKDEV_QUEUE_LIMITS_FEATURES */
51
52 /*
53 * Detect if a device has a write cache. Used to set the intial value for the
54 * vdev nowritecache flag.
55 *
56 * 4.10: QUEUE_FLAG_WC added. Initialised by the driver, but can be changed
57 * later by the operator. If not set, kernel will return flush requests
58 * immediately without doing anything.
59 * 6.6: QUEUE_FLAG_HW_WC added. Initialised by the driver, can't be changed.
60 * Only controls if the operator is allowed to change _WC. Initial version
61 * buggy; aliased to QUEUE_FLAG_FUA, so unuseable.
62 * 6.6.10, 6.7: QUEUE_FLAG_HW_WC fixed.
63 *
64 * Older than 4.10 we just assume write cache, and let the normal flush fail
65 * detection apply.
66 */
67 static inline boolean_t
zfs_bdev_has_write_cache(struct block_device * bdev)68 zfs_bdev_has_write_cache(struct block_device *bdev)
69 {
70 #if defined(QUEUE_FLAG_HW_WC) && QUEUE_FLAG_HW_WC != QUEUE_FLAG_FUA
71 return (test_bit(QUEUE_FLAG_HW_WC, &bdev_get_queue(bdev)->queue_flags));
72 #elif defined(QUEUE_FLAG_WC)
73 return (test_bit(QUEUE_FLAG_WC, &bdev_get_queue(bdev)->queue_flags));
74 #else
75 return (B_TRUE);
76 #endif
77 }
78
79 static inline void
blk_queue_set_read_ahead(struct request_queue * q,unsigned long ra_pages)80 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
81 {
82 #if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \
83 !defined(HAVE_DISK_UPDATE_READAHEAD)
84 #if defined(HAVE_BLK_QUEUE_BDI_DYNAMIC)
85 q->backing_dev_info->ra_pages = ra_pages;
86 #elif defined(HAVE_BLK_QUEUE_DISK_BDI)
87 q->disk->bdi->ra_pages = ra_pages;
88 #else
89 q->backing_dev_info.ra_pages = ra_pages;
90 #endif
91 #endif
92 }
93
94 #define BIO_BI_SECTOR(bio) (bio)->bi_iter.bi_sector
95 #define BIO_BI_SIZE(bio) (bio)->bi_iter.bi_size
96 #define BIO_BI_IDX(bio) (bio)->bi_iter.bi_idx
97 #define BIO_BI_SKIP(bio) (bio)->bi_iter.bi_bvec_done
98 #define bio_for_each_segment4(bv, bvp, b, i) \
99 bio_for_each_segment((bv), (b), (i))
100 typedef struct bvec_iter bvec_iterator_t;
101
102 static inline void
bio_set_flags_failfast(struct block_device * bdev,int * flags,bool dev,bool transport,bool driver)103 bio_set_flags_failfast(struct block_device *bdev, int *flags, bool dev,
104 bool transport, bool driver)
105 {
106 #ifdef CONFIG_BUG
107 /*
108 * Disable FAILFAST for loopback devices because of the
109 * following incorrect BUG_ON() in loop_make_request().
110 * This support is also disabled for md devices because the
111 * test suite layers md devices on top of loopback devices.
112 * This may be removed when the loopback driver is fixed.
113 *
114 * BUG_ON(!lo || (rw != READ && rw != WRITE));
115 */
116 if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) ||
117 (MAJOR(bdev->bd_dev) == MD_MAJOR))
118 return;
119
120 #ifdef BLOCK_EXT_MAJOR
121 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
122 return;
123 #endif /* BLOCK_EXT_MAJOR */
124 #endif /* CONFIG_BUG */
125
126 if (dev)
127 *flags |= REQ_FAILFAST_DEV;
128 if (transport)
129 *flags |= REQ_FAILFAST_TRANSPORT;
130 if (driver)
131 *flags |= REQ_FAILFAST_DRIVER;
132 }
133
134 /*
135 * Maximum disk label length, it may be undefined for some kernels.
136 */
137 #if !defined(DISK_NAME_LEN)
138 #define DISK_NAME_LEN 32
139 #endif /* DISK_NAME_LEN */
140
141 static inline int
bi_status_to_errno(blk_status_t status)142 bi_status_to_errno(blk_status_t status)
143 {
144 switch (status) {
145 case BLK_STS_OK:
146 return (0);
147 case BLK_STS_NOTSUPP:
148 return (EOPNOTSUPP);
149 case BLK_STS_TIMEOUT:
150 return (ETIMEDOUT);
151 case BLK_STS_NOSPC:
152 return (ENOSPC);
153 case BLK_STS_TRANSPORT:
154 return (ENOLINK);
155 case BLK_STS_TARGET:
156 return (EREMOTEIO);
157 #ifdef HAVE_BLK_STS_RESV_CONFLICT
158 case BLK_STS_RESV_CONFLICT:
159 #else
160 case BLK_STS_NEXUS:
161 #endif
162 return (EBADE);
163 case BLK_STS_MEDIUM:
164 return (ENODATA);
165 case BLK_STS_PROTECTION:
166 return (EILSEQ);
167 case BLK_STS_RESOURCE:
168 return (ENOMEM);
169 case BLK_STS_AGAIN:
170 return (EAGAIN);
171 case BLK_STS_IOERR:
172 return (EIO);
173 default:
174 return (EIO);
175 }
176 }
177
178 static inline blk_status_t
errno_to_bi_status(int error)179 errno_to_bi_status(int error)
180 {
181 switch (error) {
182 case 0:
183 return (BLK_STS_OK);
184 case EOPNOTSUPP:
185 return (BLK_STS_NOTSUPP);
186 case ETIMEDOUT:
187 return (BLK_STS_TIMEOUT);
188 case ENOSPC:
189 return (BLK_STS_NOSPC);
190 case ENOLINK:
191 return (BLK_STS_TRANSPORT);
192 case EREMOTEIO:
193 return (BLK_STS_TARGET);
194 case EBADE:
195 #ifdef HAVE_BLK_STS_RESV_CONFLICT
196 return (BLK_STS_RESV_CONFLICT);
197 #else
198 return (BLK_STS_NEXUS);
199 #endif
200 case ENODATA:
201 return (BLK_STS_MEDIUM);
202 case EILSEQ:
203 return (BLK_STS_PROTECTION);
204 case ENOMEM:
205 return (BLK_STS_RESOURCE);
206 case EAGAIN:
207 return (BLK_STS_AGAIN);
208 case EIO:
209 return (BLK_STS_IOERR);
210 default:
211 return (BLK_STS_IOERR);
212 }
213 }
214
215 /*
216 * 5.15 MACRO,
217 * GD_DEAD
218 *
219 * 2.6.36 - 5.14 MACRO,
220 * GENHD_FL_UP
221 *
222 * Check the disk status and return B_TRUE if alive
223 * otherwise B_FALSE
224 */
225 static inline boolean_t
zfs_check_disk_status(struct block_device * bdev)226 zfs_check_disk_status(struct block_device *bdev)
227 {
228 #if defined(GENHD_FL_UP)
229 return (!!(bdev->bd_disk->flags & GENHD_FL_UP));
230 #elif defined(GD_DEAD)
231 return (!test_bit(GD_DEAD, &bdev->bd_disk->state));
232 #else
233 /*
234 * This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in
235 * the kernel - likely due to an MACRO change that needs to be chased down.
236 */
237 #error "Unsupported kernel: no usable disk status check"
238 #endif
239 }
240
241 /*
242 * 5.17 API change
243 *
244 * GENHD_FL_EXT_DEVT flag removed
245 * GENHD_FL_NO_PART_SCAN renamed GENHD_FL_NO_PART
246 */
247 #ifndef HAVE_GENHD_FL_EXT_DEVT
248 #define GENHD_FL_EXT_DEVT (0)
249 #endif
250 #ifndef HAVE_GENHD_FL_NO_PART
251 #define GENHD_FL_NO_PART (GENHD_FL_NO_PART_SCAN)
252 #endif
253
254 /*
255 * 4.1 API,
256 * 3.10.0 CentOS 7.x API,
257 * blkdev_reread_part()
258 *
259 * For older kernels trigger a re-reading of the partition table by calling
260 * check_disk_change() which calls flush_disk() to invalidate the device.
261 *
262 * For newer kernels (as of 5.10), bdev_check_media_change is used, in favor of
263 * check_disk_change(), with the modification that invalidation is no longer
264 * forced.
265 */
266 #ifdef HAVE_CHECK_DISK_CHANGE
267 #define zfs_check_media_change(bdev) check_disk_change(bdev)
268 #ifdef HAVE_BLKDEV_REREAD_PART
269 #define vdev_bdev_reread_part(bdev) blkdev_reread_part(bdev)
270 #else
271 #define vdev_bdev_reread_part(bdev) check_disk_change(bdev)
272 #endif /* HAVE_BLKDEV_REREAD_PART */
273 #else
274 #ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE
275 static inline int
zfs_check_media_change(struct block_device * bdev)276 zfs_check_media_change(struct block_device *bdev)
277 {
278 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
279 struct gendisk *gd = bdev->bd_disk;
280 const struct block_device_operations *bdo = gd->fops;
281 #endif
282
283 if (!bdev_check_media_change(bdev))
284 return (0);
285
286 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
287 /*
288 * Force revalidation, to mimic the old behavior of
289 * check_disk_change()
290 */
291 if (bdo->revalidate_disk)
292 bdo->revalidate_disk(gd);
293 #endif
294
295 return (0);
296 }
297 #define vdev_bdev_reread_part(bdev) zfs_check_media_change(bdev)
298 #elif defined(HAVE_DISK_CHECK_MEDIA_CHANGE)
299 #define vdev_bdev_reread_part(bdev) disk_check_media_change(bdev->bd_disk)
300 #define zfs_check_media_change(bdev) disk_check_media_change(bdev->bd_disk)
301 #else
302 /*
303 * This is encountered if check_disk_change() and bdev_check_media_change()
304 * are not available in the kernel - likely due to an API change that needs
305 * to be chased down.
306 */
307 #error "Unsupported kernel: no usable disk change check"
308 #endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */
309 #endif /* HAVE_CHECK_DISK_CHANGE */
310
311 /*
312 * 2.6.27 API change
313 * The function was exported for use, prior to this it existed but the
314 * symbol was not exported.
315 *
316 * 5.11 API change
317 * Changed to take a dev_t argument which is set on success and return a
318 * non-zero error code on failure.
319 */
320 static inline int
vdev_lookup_bdev(const char * path,dev_t * dev)321 vdev_lookup_bdev(const char *path, dev_t *dev)
322 {
323 #if defined(HAVE_DEVT_LOOKUP_BDEV)
324 return (lookup_bdev(path, dev));
325 #elif defined(HAVE_1ARG_LOOKUP_BDEV)
326 struct block_device *bdev = lookup_bdev(path);
327 if (IS_ERR(bdev))
328 return (PTR_ERR(bdev));
329
330 *dev = bdev->bd_dev;
331 bdput(bdev);
332
333 return (0);
334 #else
335 #error "Unsupported kernel"
336 #endif
337 }
338
339 #if defined(HAVE_BLK_MODE_T)
340 #define blk_mode_is_open_write(flag) ((flag) & BLK_OPEN_WRITE)
341 #else
342 #define blk_mode_is_open_write(flag) ((flag) & FMODE_WRITE)
343 #endif
344
345 /*
346 * Kernels without bio_set_op_attrs use bi_rw for the bio flags.
347 */
348 #if !defined(HAVE_BIO_SET_OP_ATTRS)
349 static inline void
bio_set_op_attrs(struct bio * bio,unsigned rw,unsigned flags)350 bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags)
351 {
352 bio->bi_opf = rw | flags;
353 }
354 #endif
355
356 /*
357 * bio_set_flush - Set the appropriate flags in a bio to guarantee
358 * data are on non-volatile media on completion.
359 */
360 static inline void
bio_set_flush(struct bio * bio)361 bio_set_flush(struct bio *bio)
362 {
363 bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE);
364 }
365
366 /*
367 * 4.8 API,
368 * REQ_OP_FLUSH
369 *
370 * in all cases but may have a performance impact for some kernels. It
371 * has the advantage of minimizing kernel specific changes in the zvol code.
372 *
373 */
374 static inline boolean_t
bio_is_flush(struct bio * bio)375 bio_is_flush(struct bio *bio)
376 {
377 return (bio_op(bio) == REQ_OP_FLUSH || op_is_flush(bio->bi_opf));
378 }
379
380 /*
381 * 4.8 API,
382 * REQ_FUA flag moved to bio->bi_opf
383 */
384 static inline boolean_t
bio_is_fua(struct bio * bio)385 bio_is_fua(struct bio *bio)
386 {
387 return (bio->bi_opf & REQ_FUA);
388 }
389
390 /*
391 * 4.8 API,
392 * REQ_OP_DISCARD
393 *
394 * In all cases the normal I/O path is used for discards. The only
395 * difference is how the kernel tags individual I/Os as discards.
396 */
397 static inline boolean_t
bio_is_discard(struct bio * bio)398 bio_is_discard(struct bio *bio)
399 {
400 return (bio_op(bio) == REQ_OP_DISCARD);
401 }
402
403 /*
404 * 4.8 API,
405 * REQ_OP_SECURE_ERASE
406 */
407 static inline boolean_t
bio_is_secure_erase(struct bio * bio)408 bio_is_secure_erase(struct bio *bio)
409 {
410 return (bio_op(bio) == REQ_OP_SECURE_ERASE);
411 }
412
413 /*
414 * 2.6.33 API change
415 * Discard granularity and alignment restrictions may now be set. For
416 * older kernels which do not support this it is safe to skip it.
417 */
418 static inline void
blk_queue_discard_granularity(struct request_queue * q,unsigned int dg)419 blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
420 {
421 q->limits.discard_granularity = dg;
422 }
423
424 /*
425 * 5.19 API,
426 * bdev_max_discard_sectors()
427 *
428 * 2.6.32 API,
429 * blk_queue_discard()
430 */
431 static inline boolean_t
bdev_discard_supported(struct block_device * bdev)432 bdev_discard_supported(struct block_device *bdev)
433 {
434 #if defined(HAVE_BDEV_MAX_DISCARD_SECTORS)
435 return (bdev_max_discard_sectors(bdev) > 0 &&
436 bdev_discard_granularity(bdev) > 0);
437 #elif defined(HAVE_BLK_QUEUE_DISCARD)
438 return (blk_queue_discard(bdev_get_queue(bdev)) > 0 &&
439 bdev_get_queue(bdev)->limits.discard_granularity > 0);
440 #else
441 #error "Unsupported kernel"
442 #endif
443 }
444
445 /*
446 * 5.19 API,
447 * bdev_max_secure_erase_sectors()
448 *
449 * 4.8 API,
450 * blk_queue_secure_erase()
451 */
452 static inline boolean_t
bdev_secure_discard_supported(struct block_device * bdev)453 bdev_secure_discard_supported(struct block_device *bdev)
454 {
455 #if defined(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS)
456 return (!!bdev_max_secure_erase_sectors(bdev));
457 #elif defined(HAVE_BLK_QUEUE_SECURE_ERASE)
458 return (!!blk_queue_secure_erase(bdev_get_queue(bdev)));
459 #else
460 #error "Unsupported kernel"
461 #endif
462 }
463
464 /*
465 * A common holder for vdev_bdev_open() is used to relax the exclusive open
466 * semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to
467 * allow them to open the device multiple times. Other kernel callers and
468 * user space processes which don't pass this value will get EBUSY. This is
469 * currently required for the correct operation of hot spares.
470 */
471 #define VDEV_HOLDER ((void *)0x2401de7)
472
473 static inline unsigned long
blk_generic_start_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio)474 blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
475 struct gendisk *disk __attribute__((unused)),
476 int rw __attribute__((unused)), struct bio *bio)
477 {
478 #if defined(HAVE_BDEV_IO_ACCT_63)
479 return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
480 jiffies));
481 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
482 return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio),
483 bio_op(bio), jiffies));
484 #elif defined(HAVE_DISK_IO_ACCT)
485 return (disk_start_io_acct(disk, bio_sectors(bio), bio_op(bio)));
486 #elif defined(HAVE_BIO_IO_ACCT)
487 return (bio_start_io_acct(bio));
488 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
489 unsigned long start_time = jiffies;
490 generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0);
491 return (start_time);
492 #else
493 /* Unsupported */
494 return (0);
495 #endif
496 }
497
498 static inline void
blk_generic_end_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio,unsigned long start_time)499 blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
500 struct gendisk *disk __attribute__((unused)),
501 int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
502 {
503 #if defined(HAVE_BDEV_IO_ACCT_63)
504 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio),
505 start_time);
506 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
507 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
508 #elif defined(HAVE_DISK_IO_ACCT)
509 disk_end_io_acct(disk, bio_op(bio), start_time);
510 #elif defined(HAVE_BIO_IO_ACCT)
511 bio_end_io_acct(bio, start_time);
512 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
513 generic_end_io_acct(q, rw, &disk->part0, start_time);
514 #endif
515 }
516
517 #ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
518 static inline struct request_queue *
blk_generic_alloc_queue(make_request_fn make_request,int node_id)519 blk_generic_alloc_queue(make_request_fn make_request, int node_id)
520 {
521 #if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN)
522 return (blk_alloc_queue(make_request, node_id));
523 #elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH)
524 return (blk_alloc_queue_rh(make_request, node_id));
525 #else
526 struct request_queue *q = blk_alloc_queue(GFP_KERNEL);
527 if (q != NULL)
528 blk_queue_make_request(q, make_request);
529
530 return (q);
531 #endif
532 }
533 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
534
535 static inline int
io_is_flush(struct bio * bio,struct request * rq)536 io_is_flush(struct bio *bio, struct request *rq)
537 {
538 if (rq != NULL)
539 return (req_op(rq) == REQ_OP_FLUSH);
540 return (bio_is_flush(bio));
541 }
542
543 static inline int
io_is_discard(struct bio * bio,struct request * rq)544 io_is_discard(struct bio *bio, struct request *rq)
545 {
546 if (rq != NULL)
547 return (req_op(rq) == REQ_OP_DISCARD);
548 return (bio_is_discard(bio));
549 }
550
551 static inline int
io_is_secure_erase(struct bio * bio,struct request * rq)552 io_is_secure_erase(struct bio *bio, struct request *rq)
553 {
554 if (rq != NULL)
555 return (req_op(rq) == REQ_OP_SECURE_ERASE);
556 return (bio_is_secure_erase(bio));
557 }
558
559 static inline int
io_is_fua(struct bio * bio,struct request * rq)560 io_is_fua(struct bio *bio, struct request *rq)
561 {
562 if (rq != NULL)
563 return (rq->cmd_flags & REQ_FUA);
564 return (bio_is_fua(bio));
565 }
566
567
568 static inline uint64_t
io_offset(struct bio * bio,struct request * rq)569 io_offset(struct bio *bio, struct request *rq)
570 {
571 if (rq != NULL)
572 return (blk_rq_pos(rq) << 9);
573 return (BIO_BI_SECTOR(bio) << 9);
574 }
575
576 static inline uint64_t
io_size(struct bio * bio,struct request * rq)577 io_size(struct bio *bio, struct request *rq)
578 {
579 if (rq != NULL)
580 return (blk_rq_bytes(rq));
581 return (BIO_BI_SIZE(bio));
582 }
583
584 static inline int
io_has_data(struct bio * bio,struct request * rq)585 io_has_data(struct bio *bio, struct request *rq)
586 {
587 if (rq != NULL)
588 return (bio_has_data(rq->bio));
589 return (bio_has_data(bio));
590 }
591 #endif /* _ZFS_BLKDEV_H */
592