1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Portions Copyright (C) 1992 Drew Eckhardt
4 */
5 #ifndef _LINUX_BLKDEV_H
6 #define _LINUX_BLKDEV_H
7
8 #include <linux/types.h>
9 #include <linux/blk_types.h>
10 #include <linux/device.h>
11 #include <linux/list.h>
12 #include <linux/llist.h>
13 #include <linux/minmax.h>
14 #include <linux/timer.h>
15 #include <linux/workqueue.h>
16 #include <linux/wait.h>
17 #include <linux/bio.h>
18 #include <linux/gfp.h>
19 #include <linux/kdev_t.h>
20 #include <linux/rcupdate.h>
21 #include <linux/percpu-refcount.h>
22 #include <linux/blkzoned.h>
23 #include <linux/sched.h>
24 #include <linux/sbitmap.h>
25 #include <linux/uuid.h>
26 #include <linux/xarray.h>
27 #include <linux/file.h>
28 #include <linux/lockdep.h>
29
30 struct module;
31 struct request_queue;
32 struct elevator_queue;
33 struct blk_trace;
34 struct request;
35 struct sg_io_hdr;
36 struct blkcg_gq;
37 struct blk_flush_queue;
38 struct kiocb;
39 struct pr_ops;
40 struct rq_qos;
41 struct blk_queue_stats;
42 struct blk_stat_callback;
43 struct blk_crypto_profile;
44
45 extern const struct device_type disk_type;
46 extern const struct device_type part_type;
47 extern const struct class block_class;
48
49 /*
50 * Maximum number of blkcg policies allowed to be registered concurrently.
51 * Defined here to simplify include dependency.
52 */
53 #define BLKCG_MAX_POLS 6
54
55 #define DISK_MAX_PARTS 256
56 #define DISK_NAME_LEN 32
57
58 #define PARTITION_META_INFO_VOLNAMELTH 64
59 /*
60 * Enough for the string representation of any kind of UUID plus NULL.
61 * EFI UUID is 36 characters. MSDOS UUID is 11 characters.
62 */
63 #define PARTITION_META_INFO_UUIDLTH (UUID_STRING_LEN + 1)
64
65 struct partition_meta_info {
66 char uuid[PARTITION_META_INFO_UUIDLTH];
67 u8 volname[PARTITION_META_INFO_VOLNAMELTH];
68 };
69
70 /**
71 * DOC: genhd capability flags
72 *
73 * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to
74 * removable media. When set, the device remains present even when media is not
75 * inserted. Shall not be set for devices which are removed entirely when the
76 * media is removed.
77 *
78 * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events,
79 * doesn't appear in sysfs, and can't be opened from userspace or using
80 * blkdev_get*. Used for the underlying components of multipath devices.
81 *
82 * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not
83 * scan for partitions from add_disk, and users can't add partitions manually.
84 *
85 */
86 enum {
87 GENHD_FL_REMOVABLE = 1 << 0,
88 GENHD_FL_HIDDEN = 1 << 1,
89 GENHD_FL_NO_PART = 1 << 2,
90 };
91
92 enum {
93 DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */
94 DISK_EVENT_EJECT_REQUEST = 1 << 1, /* eject requested */
95 };
96
97 enum {
98 /* Poll even if events_poll_msecs is unset */
99 DISK_EVENT_FLAG_POLL = 1 << 0,
100 /* Forward events to udev */
101 DISK_EVENT_FLAG_UEVENT = 1 << 1,
102 /* Block event polling when open for exclusive write */
103 DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2,
104 };
105
106 struct disk_events;
107 struct badblocks;
108
109 enum blk_integrity_checksum {
110 BLK_INTEGRITY_CSUM_NONE = 0,
111 BLK_INTEGRITY_CSUM_IP = 1,
112 BLK_INTEGRITY_CSUM_CRC = 2,
113 BLK_INTEGRITY_CSUM_CRC64 = 3,
114 } __packed ;
115
116 struct blk_integrity {
117 unsigned char flags;
118 enum blk_integrity_checksum csum_type;
119 unsigned char tuple_size;
120 unsigned char pi_offset;
121 unsigned char interval_exp;
122 unsigned char tag_size;
123 };
124
125 typedef unsigned int __bitwise blk_mode_t;
126
127 /* open for reading */
128 #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0))
129 /* open for writing */
130 #define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1))
131 /* open exclusively (vs other exclusive openers */
132 #define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2))
133 /* opened with O_NDELAY */
134 #define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3))
135 /* open for "writes" only for ioctls (specialy hack for floppy.c) */
136 #define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4))
137 /* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */
138 #define BLK_OPEN_RESTRICT_WRITES ((__force blk_mode_t)(1 << 5))
139 /* return partition scanning errors */
140 #define BLK_OPEN_STRICT_SCAN ((__force blk_mode_t)(1 << 6))
141
142 struct gendisk {
143 /*
144 * major/first_minor/minors should not be set by any new driver, the
145 * block core will take care of allocating them automatically.
146 */
147 int major;
148 int first_minor;
149 int minors;
150
151 char disk_name[DISK_NAME_LEN]; /* name of major driver */
152
153 unsigned short events; /* supported events */
154 unsigned short event_flags; /* flags related to event processing */
155
156 struct xarray part_tbl;
157 struct block_device *part0;
158
159 const struct block_device_operations *fops;
160 struct request_queue *queue;
161 void *private_data;
162
163 struct bio_set bio_split;
164
165 int flags;
166 unsigned long state;
167 #define GD_NEED_PART_SCAN 0
168 #define GD_READ_ONLY 1
169 #define GD_DEAD 2
170 #define GD_NATIVE_CAPACITY 3
171 #define GD_ADDED 4
172 #define GD_SUPPRESS_PART_SCAN 5
173 #define GD_OWNS_QUEUE 6
174
175 struct mutex open_mutex; /* open/close mutex */
176 unsigned open_partitions; /* number of open partitions */
177
178 struct backing_dev_info *bdi;
179 struct kobject queue_kobj; /* the queue/ directory */
180 struct kobject *slave_dir;
181 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
182 struct list_head slave_bdevs;
183 #endif
184 struct timer_rand_state *random;
185 atomic_t sync_io; /* RAID */
186 struct disk_events *ev;
187
188 #ifdef CONFIG_BLK_DEV_ZONED
189 /*
190 * Zoned block device information. Reads of this information must be
191 * protected with blk_queue_enter() / blk_queue_exit(). Modifying this
192 * information is only allowed while no requests are being processed.
193 * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue().
194 */
195 unsigned int nr_zones;
196 unsigned int zone_capacity;
197 unsigned int last_zone_capacity;
198 unsigned long __rcu *conv_zones_bitmap;
199 unsigned int zone_wplugs_hash_bits;
200 atomic_t nr_zone_wplugs;
201 spinlock_t zone_wplugs_lock;
202 struct mempool_s *zone_wplugs_pool;
203 struct hlist_head *zone_wplugs_hash;
204 struct workqueue_struct *zone_wplugs_wq;
205 #endif /* CONFIG_BLK_DEV_ZONED */
206
207 #if IS_ENABLED(CONFIG_CDROM)
208 struct cdrom_device_info *cdi;
209 #endif
210 int node_id;
211 struct badblocks *bb;
212 struct lockdep_map lockdep_map;
213 u64 diskseq;
214 blk_mode_t open_mode;
215
216 /*
217 * Independent sector access ranges. This is always NULL for
218 * devices that do not have multiple independent access ranges.
219 */
220 struct blk_independent_access_ranges *ia_ranges;
221 };
222
223 /**
224 * disk_openers - returns how many openers are there for a disk
225 * @disk: disk to check
226 *
227 * This returns the number of openers for a disk. Note that this value is only
228 * stable if disk->open_mutex is held.
229 *
230 * Note: Due to a quirk in the block layer open code, each open partition is
231 * only counted once even if there are multiple openers.
232 */
disk_openers(struct gendisk * disk)233 static inline unsigned int disk_openers(struct gendisk *disk)
234 {
235 return atomic_read(&disk->part0->bd_openers);
236 }
237
238 /**
239 * disk_has_partscan - return %true if partition scanning is enabled on a disk
240 * @disk: disk to check
241 *
242 * Returns %true if partitions scanning is enabled for @disk, or %false if
243 * partition scanning is disabled either permanently or temporarily.
244 */
disk_has_partscan(struct gendisk * disk)245 static inline bool disk_has_partscan(struct gendisk *disk)
246 {
247 return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) &&
248 !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
249 }
250
251 /*
252 * The gendisk is refcounted by the part0 block_device, and the bd_device
253 * therein is also used for device model presentation in sysfs.
254 */
255 #define dev_to_disk(device) \
256 (dev_to_bdev(device)->bd_disk)
257 #define disk_to_dev(disk) \
258 (&((disk)->part0->bd_device))
259
260 #if IS_REACHABLE(CONFIG_CDROM)
261 #define disk_to_cdi(disk) ((disk)->cdi)
262 #else
263 #define disk_to_cdi(disk) NULL
264 #endif
265
disk_devt(struct gendisk * disk)266 static inline dev_t disk_devt(struct gendisk *disk)
267 {
268 return MKDEV(disk->major, disk->first_minor);
269 }
270
271 /*
272 * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER)
273 * however we constrain this to what we can validate and test.
274 */
275 #define BLK_MAX_BLOCK_SIZE SZ_64K
276
277 /* blk_validate_limits() validates bsize, so drivers don't usually need to */
blk_validate_block_size(unsigned long bsize)278 static inline int blk_validate_block_size(unsigned long bsize)
279 {
280 if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize))
281 return -EINVAL;
282
283 return 0;
284 }
285
blk_op_is_passthrough(blk_opf_t op)286 static inline bool blk_op_is_passthrough(blk_opf_t op)
287 {
288 op &= REQ_OP_MASK;
289 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
290 }
291
292 /* flags set by the driver in queue_limits.features */
293 typedef unsigned int __bitwise blk_features_t;
294
295 /* supports a volatile write cache */
296 #define BLK_FEAT_WRITE_CACHE ((__force blk_features_t)(1u << 0))
297
298 /* supports passing on the FUA bit */
299 #define BLK_FEAT_FUA ((__force blk_features_t)(1u << 1))
300
301 /* rotational device (hard drive or floppy) */
302 #define BLK_FEAT_ROTATIONAL ((__force blk_features_t)(1u << 2))
303
304 /* contributes to the random number pool */
305 #define BLK_FEAT_ADD_RANDOM ((__force blk_features_t)(1u << 3))
306
307 /* do disk/partitions IO accounting */
308 #define BLK_FEAT_IO_STAT ((__force blk_features_t)(1u << 4))
309
310 /* don't modify data until writeback is done */
311 #define BLK_FEAT_STABLE_WRITES ((__force blk_features_t)(1u << 5))
312
313 /* always completes in submit context */
314 #define BLK_FEAT_SYNCHRONOUS ((__force blk_features_t)(1u << 6))
315
316 /* supports REQ_NOWAIT */
317 #define BLK_FEAT_NOWAIT ((__force blk_features_t)(1u << 7))
318
319 /* supports DAX */
320 #define BLK_FEAT_DAX ((__force blk_features_t)(1u << 8))
321
322 /* supports I/O polling */
323 #define BLK_FEAT_POLL ((__force blk_features_t)(1u << 9))
324
325 /* is a zoned device */
326 #define BLK_FEAT_ZONED ((__force blk_features_t)(1u << 10))
327
328 /* supports PCI(e) p2p requests */
329 #define BLK_FEAT_PCI_P2PDMA ((__force blk_features_t)(1u << 12))
330
331 /* skip this queue in blk_mq_(un)quiesce_tagset */
332 #define BLK_FEAT_SKIP_TAGSET_QUIESCE ((__force blk_features_t)(1u << 13))
333
334 /* bounce all highmem pages */
335 #define BLK_FEAT_BOUNCE_HIGH ((__force blk_features_t)(1u << 14))
336
337 /* undocumented magic for bcache */
338 #define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \
339 ((__force blk_features_t)(1u << 15))
340
341 /* atomic writes enabled */
342 #define BLK_FEAT_ATOMIC_WRITES \
343 ((__force blk_features_t)(1u << 16))
344
345 /*
346 * Flags automatically inherited when stacking limits.
347 */
348 #define BLK_FEAT_INHERIT_MASK \
349 (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \
350 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | BLK_FEAT_BOUNCE_HIGH | \
351 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE)
352
353 /* internal flags in queue_limits.flags */
354 typedef unsigned int __bitwise blk_flags_t;
355
356 /* do not send FLUSH/FUA commands despite advertising a write cache */
357 #define BLK_FLAG_WRITE_CACHE_DISABLED ((__force blk_flags_t)(1u << 0))
358
359 /* I/O topology is misaligned */
360 #define BLK_FLAG_MISALIGNED ((__force blk_flags_t)(1u << 1))
361
362 /* passthrough command IO accounting */
363 #define BLK_FLAG_IOSTATS_PASSTHROUGH ((__force blk_flags_t)(1u << 2))
364
365 struct queue_limits {
366 blk_features_t features;
367 blk_flags_t flags;
368 unsigned long seg_boundary_mask;
369 unsigned long virt_boundary_mask;
370
371 unsigned int max_hw_sectors;
372 unsigned int max_dev_sectors;
373 unsigned int chunk_sectors;
374 unsigned int max_sectors;
375 unsigned int max_user_sectors;
376 unsigned int max_segment_size;
377 unsigned int min_segment_size;
378 unsigned int physical_block_size;
379 unsigned int logical_block_size;
380 unsigned int alignment_offset;
381 unsigned int io_min;
382 unsigned int io_opt;
383 unsigned int max_discard_sectors;
384 unsigned int max_hw_discard_sectors;
385 unsigned int max_user_discard_sectors;
386 unsigned int max_secure_erase_sectors;
387 unsigned int max_write_zeroes_sectors;
388 unsigned int max_hw_zone_append_sectors;
389 unsigned int max_zone_append_sectors;
390 unsigned int discard_granularity;
391 unsigned int discard_alignment;
392 unsigned int zone_write_granularity;
393
394 /* atomic write limits */
395 unsigned int atomic_write_hw_max;
396 unsigned int atomic_write_max_sectors;
397 unsigned int atomic_write_hw_boundary;
398 unsigned int atomic_write_boundary_sectors;
399 unsigned int atomic_write_hw_unit_min;
400 unsigned int atomic_write_unit_min;
401 unsigned int atomic_write_hw_unit_max;
402 unsigned int atomic_write_unit_max;
403
404 unsigned short max_segments;
405 unsigned short max_integrity_segments;
406 unsigned short max_discard_segments;
407
408 unsigned int max_open_zones;
409 unsigned int max_active_zones;
410
411 /*
412 * Drivers that set dma_alignment to less than 511 must be prepared to
413 * handle individual bvec's that are not a multiple of a SECTOR_SIZE
414 * due to possible offsets.
415 */
416 unsigned int dma_alignment;
417 unsigned int dma_pad_mask;
418
419 struct blk_integrity integrity;
420 };
421
422 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
423 void *data);
424
425 #define BLK_ALL_ZONES ((unsigned int)-1)
426 int blkdev_report_zones(struct block_device *bdev, sector_t sector,
427 unsigned int nr_zones, report_zones_cb cb, void *data);
428 int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
429 sector_t sectors, sector_t nr_sectors);
430 int blk_revalidate_disk_zones(struct gendisk *disk);
431
432 /*
433 * Independent access ranges: struct blk_independent_access_range describes
434 * a range of contiguous sectors that can be accessed using device command
435 * execution resources that are independent from the resources used for
436 * other access ranges. This is typically found with single-LUN multi-actuator
437 * HDDs where each access range is served by a different set of heads.
438 * The set of independent ranges supported by the device is defined using
439 * struct blk_independent_access_ranges. The independent ranges must not overlap
440 * and must include all sectors within the disk capacity (no sector holes
441 * allowed).
442 * For a device with multiple ranges, requests targeting sectors in different
443 * ranges can be executed in parallel. A request can straddle an access range
444 * boundary.
445 */
446 struct blk_independent_access_range {
447 struct kobject kobj;
448 sector_t sector;
449 sector_t nr_sectors;
450 };
451
452 struct blk_independent_access_ranges {
453 struct kobject kobj;
454 bool sysfs_registered;
455 unsigned int nr_ia_ranges;
456 struct blk_independent_access_range ia_range[];
457 };
458
459 struct request_queue {
460 /*
461 * The queue owner gets to use this for whatever they like.
462 * ll_rw_blk doesn't touch it.
463 */
464 void *queuedata;
465
466 struct elevator_queue *elevator;
467
468 const struct blk_mq_ops *mq_ops;
469
470 /* sw queues */
471 struct blk_mq_ctx __percpu *queue_ctx;
472
473 /*
474 * various queue flags, see QUEUE_* below
475 */
476 unsigned long queue_flags;
477
478 unsigned int rq_timeout;
479
480 unsigned int queue_depth;
481
482 refcount_t refs;
483
484 /* hw dispatch queues */
485 unsigned int nr_hw_queues;
486 struct xarray hctx_table;
487
488 struct percpu_ref q_usage_counter;
489 struct lock_class_key io_lock_cls_key;
490 struct lockdep_map io_lockdep_map;
491
492 struct lock_class_key q_lock_cls_key;
493 struct lockdep_map q_lockdep_map;
494
495 struct request *last_merge;
496
497 spinlock_t queue_lock;
498
499 int quiesce_depth;
500
501 struct gendisk *disk;
502
503 /*
504 * mq queue kobject
505 */
506 struct kobject *mq_kobj;
507
508 struct queue_limits limits;
509
510 #ifdef CONFIG_PM
511 struct device *dev;
512 enum rpm_status rpm_status;
513 #endif
514
515 /*
516 * Number of contexts that have called blk_set_pm_only(). If this
517 * counter is above zero then only RQF_PM requests are processed.
518 */
519 atomic_t pm_only;
520
521 struct blk_queue_stats *stats;
522 struct rq_qos *rq_qos;
523 struct mutex rq_qos_mutex;
524
525 /*
526 * ida allocated id for this queue. Used to index queues from
527 * ioctx.
528 */
529 int id;
530
531 /*
532 * queue settings
533 */
534 unsigned long nr_requests; /* Max # of requests */
535
536 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
537 struct blk_crypto_profile *crypto_profile;
538 struct kobject *crypto_kobject;
539 #endif
540
541 struct timer_list timeout;
542 struct work_struct timeout_work;
543
544 atomic_t nr_active_requests_shared_tags;
545
546 struct blk_mq_tags *sched_shared_tags;
547
548 struct list_head icq_list;
549 #ifdef CONFIG_BLK_CGROUP
550 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
551 struct blkcg_gq *root_blkg;
552 struct list_head blkg_list;
553 struct mutex blkcg_mutex;
554 #endif
555
556 int node;
557
558 spinlock_t requeue_lock;
559 struct list_head requeue_list;
560 struct delayed_work requeue_work;
561
562 #ifdef CONFIG_BLK_DEV_IO_TRACE
563 struct blk_trace __rcu *blk_trace;
564 #endif
565 /*
566 * for flush operations
567 */
568 struct blk_flush_queue *fq;
569 struct list_head flush_list;
570
571 /*
572 * Protects against I/O scheduler switching, particularly when updating
573 * q->elevator. Since the elevator update code path may also modify q->
574 * nr_requests and wbt latency, this lock also protects the sysfs attrs
575 * nr_requests and wbt_lat_usec. Additionally the nr_hw_queues update
576 * may modify hctx tags, reserved-tags and cpumask, so this lock also
577 * helps protect the hctx sysfs/debugfs attrs. To ensure proper locking
578 * order during an elevator or nr_hw_queue update, first freeze the
579 * queue, then acquire ->elevator_lock.
580 */
581 struct mutex elevator_lock;
582
583 struct mutex sysfs_lock;
584 /*
585 * Protects queue limits and also sysfs attribute read_ahead_kb.
586 */
587 struct mutex limits_lock;
588
589 /*
590 * for reusing dead hctx instance in case of updating
591 * nr_hw_queues
592 */
593 struct list_head unused_hctx_list;
594 spinlock_t unused_hctx_lock;
595
596 int mq_freeze_depth;
597
598 #ifdef CONFIG_BLK_DEV_THROTTLING
599 /* Throttle data */
600 struct throtl_data *td;
601 #endif
602 struct rcu_head rcu_head;
603 #ifdef CONFIG_LOCKDEP
604 struct task_struct *mq_freeze_owner;
605 int mq_freeze_owner_depth;
606 /*
607 * Records disk & queue state in current context, used in unfreeze
608 * queue
609 */
610 bool mq_freeze_disk_dead;
611 bool mq_freeze_queue_dying;
612 #endif
613 wait_queue_head_t mq_freeze_wq;
614 /*
615 * Protect concurrent access to q_usage_counter by
616 * percpu_ref_kill() and percpu_ref_reinit().
617 */
618 struct mutex mq_freeze_lock;
619
620 struct blk_mq_tag_set *tag_set;
621 struct list_head tag_set_list;
622
623 struct dentry *debugfs_dir;
624 struct dentry *sched_debugfs_dir;
625 struct dentry *rqos_debugfs_dir;
626 /*
627 * Serializes all debugfs metadata operations using the above dentries.
628 */
629 struct mutex debugfs_mutex;
630 };
631
632 /* Keep blk_queue_flag_name[] in sync with the definitions below */
633 enum {
634 QUEUE_FLAG_DYING, /* queue being torn down */
635 QUEUE_FLAG_NOMERGES, /* disable merge attempts */
636 QUEUE_FLAG_SAME_COMP, /* complete on same CPU-group */
637 QUEUE_FLAG_FAIL_IO, /* fake timeout */
638 QUEUE_FLAG_NOXMERGES, /* No extended merges */
639 QUEUE_FLAG_SAME_FORCE, /* force complete on same CPU */
640 QUEUE_FLAG_INIT_DONE, /* queue is initialized */
641 QUEUE_FLAG_STATS, /* track IO start and completion times */
642 QUEUE_FLAG_REGISTERED, /* queue has been registered to a disk */
643 QUEUE_FLAG_QUIESCED, /* queue has been quiesced */
644 QUEUE_FLAG_RQ_ALLOC_TIME, /* record rq->alloc_time_ns */
645 QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */
646 QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */
647 QUEUE_FLAG_MAX
648 };
649
650 #define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP)
651
652 void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
653 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
654
655 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
656 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
657 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
658 #define blk_queue_noxmerges(q) \
659 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
660 #define blk_queue_nonrot(q) (!((q)->limits.features & BLK_FEAT_ROTATIONAL))
661 #define blk_queue_io_stat(q) ((q)->limits.features & BLK_FEAT_IO_STAT)
662 #define blk_queue_passthrough_stat(q) \
663 ((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH)
664 #define blk_queue_dax(q) ((q)->limits.features & BLK_FEAT_DAX)
665 #define blk_queue_pci_p2pdma(q) ((q)->limits.features & BLK_FEAT_PCI_P2PDMA)
666 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
667 #define blk_queue_rq_alloc_time(q) \
668 test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
669 #else
670 #define blk_queue_rq_alloc_time(q) false
671 #endif
672
673 #define blk_noretry_request(rq) \
674 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
675 REQ_FAILFAST_DRIVER))
676 #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
677 #define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
678 #define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
679 #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
680 #define blk_queue_skip_tagset_quiesce(q) \
681 ((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE)
682
683 extern void blk_set_pm_only(struct request_queue *q);
684 extern void blk_clear_pm_only(struct request_queue *q);
685
686 #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
687
688 #define dma_map_bvec(dev, bv, dir, attrs) \
689 dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
690 (dir), (attrs))
691
queue_is_mq(struct request_queue * q)692 static inline bool queue_is_mq(struct request_queue *q)
693 {
694 return q->mq_ops;
695 }
696
697 #ifdef CONFIG_PM
queue_rpm_status(struct request_queue * q)698 static inline enum rpm_status queue_rpm_status(struct request_queue *q)
699 {
700 return q->rpm_status;
701 }
702 #else
queue_rpm_status(struct request_queue * q)703 static inline enum rpm_status queue_rpm_status(struct request_queue *q)
704 {
705 return RPM_ACTIVE;
706 }
707 #endif
708
blk_queue_is_zoned(struct request_queue * q)709 static inline bool blk_queue_is_zoned(struct request_queue *q)
710 {
711 return IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
712 (q->limits.features & BLK_FEAT_ZONED);
713 }
714
715 #ifdef CONFIG_BLK_DEV_ZONED
disk_nr_zones(struct gendisk * disk)716 static inline unsigned int disk_nr_zones(struct gendisk *disk)
717 {
718 return disk->nr_zones;
719 }
720 bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs);
721 #else /* CONFIG_BLK_DEV_ZONED */
disk_nr_zones(struct gendisk * disk)722 static inline unsigned int disk_nr_zones(struct gendisk *disk)
723 {
724 return 0;
725 }
blk_zone_plug_bio(struct bio * bio,unsigned int nr_segs)726 static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs)
727 {
728 return false;
729 }
730 #endif /* CONFIG_BLK_DEV_ZONED */
731
disk_zone_no(struct gendisk * disk,sector_t sector)732 static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
733 {
734 if (!blk_queue_is_zoned(disk->queue))
735 return 0;
736 return sector >> ilog2(disk->queue->limits.chunk_sectors);
737 }
738
bdev_nr_zones(struct block_device * bdev)739 static inline unsigned int bdev_nr_zones(struct block_device *bdev)
740 {
741 return disk_nr_zones(bdev->bd_disk);
742 }
743
bdev_max_open_zones(struct block_device * bdev)744 static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
745 {
746 return bdev->bd_disk->queue->limits.max_open_zones;
747 }
748
bdev_max_active_zones(struct block_device * bdev)749 static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
750 {
751 return bdev->bd_disk->queue->limits.max_active_zones;
752 }
753
blk_queue_depth(struct request_queue * q)754 static inline unsigned int blk_queue_depth(struct request_queue *q)
755 {
756 if (q->queue_depth)
757 return q->queue_depth;
758
759 return q->nr_requests;
760 }
761
762 /*
763 * default timeout for SG_IO if none specified
764 */
765 #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
766 #define BLK_MIN_SG_TIMEOUT (7 * HZ)
767
768 /* This should not be used directly - use rq_for_each_segment */
769 #define for_each_bio(_bio) \
770 for (; _bio; _bio = _bio->bi_next)
771
772 int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk,
773 const struct attribute_group **groups,
774 struct fwnode_handle *fwnode);
775 int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
776 const struct attribute_group **groups);
add_disk(struct gendisk * disk)777 static inline int __must_check add_disk(struct gendisk *disk)
778 {
779 return device_add_disk(NULL, disk, NULL);
780 }
781 void del_gendisk(struct gendisk *gp);
782 void invalidate_disk(struct gendisk *disk);
783 void set_disk_ro(struct gendisk *disk, bool read_only);
784 void disk_uevent(struct gendisk *disk, enum kobject_action action);
785
bdev_partno(const struct block_device * bdev)786 static inline u8 bdev_partno(const struct block_device *bdev)
787 {
788 return atomic_read(&bdev->__bd_flags) & BD_PARTNO;
789 }
790
bdev_test_flag(const struct block_device * bdev,unsigned flag)791 static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag)
792 {
793 return atomic_read(&bdev->__bd_flags) & flag;
794 }
795
bdev_set_flag(struct block_device * bdev,unsigned flag)796 static inline void bdev_set_flag(struct block_device *bdev, unsigned flag)
797 {
798 atomic_or(flag, &bdev->__bd_flags);
799 }
800
bdev_clear_flag(struct block_device * bdev,unsigned flag)801 static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag)
802 {
803 atomic_andnot(flag, &bdev->__bd_flags);
804 }
805
get_disk_ro(struct gendisk * disk)806 static inline bool get_disk_ro(struct gendisk *disk)
807 {
808 return bdev_test_flag(disk->part0, BD_READ_ONLY) ||
809 test_bit(GD_READ_ONLY, &disk->state);
810 }
811
bdev_read_only(struct block_device * bdev)812 static inline bool bdev_read_only(struct block_device *bdev)
813 {
814 return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk);
815 }
816
817 bool set_capacity_and_notify(struct gendisk *disk, sector_t size);
818 void disk_force_media_change(struct gendisk *disk);
819 void bdev_mark_dead(struct block_device *bdev, bool surprise);
820
821 void add_disk_randomness(struct gendisk *disk) __latent_entropy;
822 void rand_initialize_disk(struct gendisk *disk);
823
get_start_sect(struct block_device * bdev)824 static inline sector_t get_start_sect(struct block_device *bdev)
825 {
826 return bdev->bd_start_sect;
827 }
828
bdev_nr_sectors(struct block_device * bdev)829 static inline sector_t bdev_nr_sectors(struct block_device *bdev)
830 {
831 return bdev->bd_nr_sectors;
832 }
833
bdev_nr_bytes(struct block_device * bdev)834 static inline loff_t bdev_nr_bytes(struct block_device *bdev)
835 {
836 return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT;
837 }
838
get_capacity(struct gendisk * disk)839 static inline sector_t get_capacity(struct gendisk *disk)
840 {
841 return bdev_nr_sectors(disk->part0);
842 }
843
sb_bdev_nr_blocks(struct super_block * sb)844 static inline u64 sb_bdev_nr_blocks(struct super_block *sb)
845 {
846 return bdev_nr_sectors(sb->s_bdev) >>
847 (sb->s_blocksize_bits - SECTOR_SHIFT);
848 }
849
850 int bdev_disk_changed(struct gendisk *disk, bool invalidate);
851
852 void put_disk(struct gendisk *disk);
853 struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node,
854 struct lock_class_key *lkclass);
855
856 /**
857 * blk_alloc_disk - allocate a gendisk structure
858 * @lim: queue limits to be used for this disk.
859 * @node_id: numa node to allocate on
860 *
861 * Allocate and pre-initialize a gendisk structure for use with BIO based
862 * drivers.
863 *
864 * Returns an ERR_PTR on error, else the allocated disk.
865 *
866 * Context: can sleep
867 */
868 #define blk_alloc_disk(lim, node_id) \
869 ({ \
870 static struct lock_class_key __key; \
871 \
872 __blk_alloc_disk(lim, node_id, &__key); \
873 })
874
875 int __register_blkdev(unsigned int major, const char *name,
876 void (*probe)(dev_t devt));
877 #define register_blkdev(major, name) \
878 __register_blkdev(major, name, NULL)
879 void unregister_blkdev(unsigned int major, const char *name);
880
881 bool disk_check_media_change(struct gendisk *disk);
882 void set_capacity(struct gendisk *disk, sector_t size);
883
884 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
885 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk);
886 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk);
887 #else
bd_link_disk_holder(struct block_device * bdev,struct gendisk * disk)888 static inline int bd_link_disk_holder(struct block_device *bdev,
889 struct gendisk *disk)
890 {
891 return 0;
892 }
bd_unlink_disk_holder(struct block_device * bdev,struct gendisk * disk)893 static inline void bd_unlink_disk_holder(struct block_device *bdev,
894 struct gendisk *disk)
895 {
896 }
897 #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */
898
899 dev_t part_devt(struct gendisk *disk, u8 partno);
900 void inc_diskseq(struct gendisk *disk);
901 void blk_request_module(dev_t devt);
902
903 extern int blk_register_queue(struct gendisk *disk);
904 extern void blk_unregister_queue(struct gendisk *disk);
905 void submit_bio_noacct(struct bio *bio);
906 struct bio *bio_split_to_limits(struct bio *bio);
907
908 extern int blk_lld_busy(struct request_queue *q);
909 extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
910 extern void blk_queue_exit(struct request_queue *q);
911 extern void blk_sync_queue(struct request_queue *q);
912
913 /* Helper to convert REQ_OP_XXX to its string format XXX */
914 extern const char *blk_op_str(enum req_op op);
915
916 int blk_status_to_errno(blk_status_t status);
917 blk_status_t errno_to_blk_status(int errno);
918 const char *blk_status_to_str(blk_status_t status);
919
920 /* only poll the hardware once, don't continue until a completion was found */
921 #define BLK_POLL_ONESHOT (1 << 0)
922 int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
923 int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
924 unsigned int flags);
925
bdev_get_queue(struct block_device * bdev)926 static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
927 {
928 return bdev->bd_queue; /* this is never NULL */
929 }
930
931 /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */
932 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
933
bio_zone_no(struct bio * bio)934 static inline unsigned int bio_zone_no(struct bio *bio)
935 {
936 return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
937 }
938
bio_straddles_zones(struct bio * bio)939 static inline bool bio_straddles_zones(struct bio *bio)
940 {
941 return bio_sectors(bio) &&
942 bio_zone_no(bio) !=
943 disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1);
944 }
945
946 /*
947 * Return how much within the boundary is left to be used for I/O at a given
948 * offset.
949 */
blk_boundary_sectors_left(sector_t offset,unsigned int boundary_sectors)950 static inline unsigned int blk_boundary_sectors_left(sector_t offset,
951 unsigned int boundary_sectors)
952 {
953 if (unlikely(!is_power_of_2(boundary_sectors)))
954 return boundary_sectors - sector_div(offset, boundary_sectors);
955 return boundary_sectors - (offset & (boundary_sectors - 1));
956 }
957
958 /**
959 * queue_limits_start_update - start an atomic update of queue limits
960 * @q: queue to update
961 *
962 * This functions starts an atomic update of the queue limits. It takes a lock
963 * to prevent other updates and returns a snapshot of the current limits that
964 * the caller can modify. The caller must call queue_limits_commit_update()
965 * to finish the update.
966 *
967 * Context: process context.
968 */
969 static inline struct queue_limits
queue_limits_start_update(struct request_queue * q)970 queue_limits_start_update(struct request_queue *q)
971 {
972 mutex_lock(&q->limits_lock);
973 return q->limits;
974 }
975 int queue_limits_commit_update_frozen(struct request_queue *q,
976 struct queue_limits *lim);
977 int queue_limits_commit_update(struct request_queue *q,
978 struct queue_limits *lim);
979 int queue_limits_set(struct request_queue *q, struct queue_limits *lim);
980 int blk_validate_limits(struct queue_limits *lim);
981
982 /**
983 * queue_limits_cancel_update - cancel an atomic update of queue limits
984 * @q: queue to update
985 *
986 * This functions cancels an atomic update of the queue limits started by
987 * queue_limits_start_update() and should be used when an error occurs after
988 * starting update.
989 */
queue_limits_cancel_update(struct request_queue * q)990 static inline void queue_limits_cancel_update(struct request_queue *q)
991 {
992 mutex_unlock(&q->limits_lock);
993 }
994
995 /*
996 * These helpers are for drivers that have sloppy feature negotiation and might
997 * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O
998 * completion handler when the device returned an indicator that the respective
999 * feature is not actually supported. They are racy and the driver needs to
1000 * cope with that. Try to avoid this scheme if you can.
1001 */
blk_queue_disable_discard(struct request_queue * q)1002 static inline void blk_queue_disable_discard(struct request_queue *q)
1003 {
1004 q->limits.max_discard_sectors = 0;
1005 }
1006
blk_queue_disable_secure_erase(struct request_queue * q)1007 static inline void blk_queue_disable_secure_erase(struct request_queue *q)
1008 {
1009 q->limits.max_secure_erase_sectors = 0;
1010 }
1011
blk_queue_disable_write_zeroes(struct request_queue * q)1012 static inline void blk_queue_disable_write_zeroes(struct request_queue *q)
1013 {
1014 q->limits.max_write_zeroes_sectors = 0;
1015 }
1016
1017 /*
1018 * Access functions for manipulating queue properties
1019 */
1020 extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1021 extern void blk_set_stacking_limits(struct queue_limits *lim);
1022 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
1023 sector_t offset);
1024 void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev,
1025 sector_t offset, const char *pfx);
1026 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1027
1028 struct blk_independent_access_ranges *
1029 disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges);
1030 void disk_set_independent_access_ranges(struct gendisk *disk,
1031 struct blk_independent_access_ranges *iars);
1032
1033 bool __must_check blk_get_queue(struct request_queue *);
1034 extern void blk_put_queue(struct request_queue *);
1035
1036 void blk_mark_disk_dead(struct gendisk *disk);
1037
1038 struct rq_list {
1039 struct request *head;
1040 struct request *tail;
1041 };
1042
1043 #ifdef CONFIG_BLOCK
1044 /*
1045 * blk_plug permits building a queue of related requests by holding the I/O
1046 * fragments for a short period. This allows merging of sequential requests
1047 * into single larger request. As the requests are moved from a per-task list to
1048 * the device's request_queue in a batch, this results in improved scalability
1049 * as the lock contention for request_queue lock is reduced.
1050 *
1051 * It is ok not to disable preemption when adding the request to the plug list
1052 * or when attempting a merge. For details, please see schedule() where
1053 * blk_flush_plug() is called.
1054 */
1055 struct blk_plug {
1056 struct rq_list mq_list; /* blk-mq requests */
1057
1058 /* if ios_left is > 1, we can batch tag/rq allocations */
1059 struct rq_list cached_rqs;
1060 u64 cur_ktime;
1061 unsigned short nr_ios;
1062
1063 unsigned short rq_count;
1064
1065 bool multiple_queues;
1066 bool has_elevator;
1067
1068 struct list_head cb_list; /* md requires an unplug callback */
1069 };
1070
1071 struct blk_plug_cb;
1072 typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1073 struct blk_plug_cb {
1074 struct list_head list;
1075 blk_plug_cb_fn callback;
1076 void *data;
1077 };
1078 extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1079 void *data, int size);
1080 extern void blk_start_plug(struct blk_plug *);
1081 extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short);
1082 extern void blk_finish_plug(struct blk_plug *);
1083
1084 void __blk_flush_plug(struct blk_plug *plug, bool from_schedule);
blk_flush_plug(struct blk_plug * plug,bool async)1085 static inline void blk_flush_plug(struct blk_plug *plug, bool async)
1086 {
1087 if (plug)
1088 __blk_flush_plug(plug, async);
1089 }
1090
1091 /*
1092 * tsk == current here
1093 */
blk_plug_invalidate_ts(struct task_struct * tsk)1094 static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
1095 {
1096 struct blk_plug *plug = tsk->plug;
1097
1098 if (plug)
1099 plug->cur_ktime = 0;
1100 current->flags &= ~PF_BLOCK_TS;
1101 }
1102
1103 int blkdev_issue_flush(struct block_device *bdev);
1104 long nr_blockdev_pages(void);
1105 #else /* CONFIG_BLOCK */
1106 struct blk_plug {
1107 };
1108
blk_start_plug_nr_ios(struct blk_plug * plug,unsigned short nr_ios)1109 static inline void blk_start_plug_nr_ios(struct blk_plug *plug,
1110 unsigned short nr_ios)
1111 {
1112 }
1113
blk_start_plug(struct blk_plug * plug)1114 static inline void blk_start_plug(struct blk_plug *plug)
1115 {
1116 }
1117
blk_finish_plug(struct blk_plug * plug)1118 static inline void blk_finish_plug(struct blk_plug *plug)
1119 {
1120 }
1121
blk_flush_plug(struct blk_plug * plug,bool async)1122 static inline void blk_flush_plug(struct blk_plug *plug, bool async)
1123 {
1124 }
1125
blk_plug_invalidate_ts(struct task_struct * tsk)1126 static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
1127 {
1128 }
1129
blkdev_issue_flush(struct block_device * bdev)1130 static inline int blkdev_issue_flush(struct block_device *bdev)
1131 {
1132 return 0;
1133 }
1134
nr_blockdev_pages(void)1135 static inline long nr_blockdev_pages(void)
1136 {
1137 return 0;
1138 }
1139 #endif /* CONFIG_BLOCK */
1140
1141 extern void blk_io_schedule(void);
1142
1143 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1144 sector_t nr_sects, gfp_t gfp_mask);
1145 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1146 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop);
1147 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
1148 sector_t nr_sects, gfp_t gfp);
1149
1150 #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */
1151 #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */
1152 #define BLKDEV_ZERO_KILLABLE (1 << 2) /* interruptible by fatal signals */
1153
1154 extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1155 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1156 unsigned flags);
1157 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1158 sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1159
sb_issue_discard(struct super_block * sb,sector_t block,sector_t nr_blocks,gfp_t gfp_mask,unsigned long flags)1160 static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1161 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1162 {
1163 return blkdev_issue_discard(sb->s_bdev,
1164 block << (sb->s_blocksize_bits -
1165 SECTOR_SHIFT),
1166 nr_blocks << (sb->s_blocksize_bits -
1167 SECTOR_SHIFT),
1168 gfp_mask);
1169 }
sb_issue_zeroout(struct super_block * sb,sector_t block,sector_t nr_blocks,gfp_t gfp_mask)1170 static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1171 sector_t nr_blocks, gfp_t gfp_mask)
1172 {
1173 return blkdev_issue_zeroout(sb->s_bdev,
1174 block << (sb->s_blocksize_bits -
1175 SECTOR_SHIFT),
1176 nr_blocks << (sb->s_blocksize_bits -
1177 SECTOR_SHIFT),
1178 gfp_mask, 0);
1179 }
1180
bdev_is_partition(struct block_device * bdev)1181 static inline bool bdev_is_partition(struct block_device *bdev)
1182 {
1183 return bdev_partno(bdev) != 0;
1184 }
1185
1186 enum blk_default_limits {
1187 BLK_MAX_SEGMENTS = 128,
1188 BLK_SAFE_MAX_SECTORS = 255,
1189 BLK_MAX_SEGMENT_SIZE = 65536,
1190 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1191 };
1192
1193 /*
1194 * Default upper limit for the software max_sectors limit used for
1195 * regular file system I/O. This can be increased through sysfs.
1196 *
1197 * Not to be confused with the max_hw_sector limit that is entirely
1198 * controlled by the driver, usually based on hardware limits.
1199 */
1200 #define BLK_DEF_MAX_SECTORS_CAP 2560u
1201
bdev_limits(struct block_device * bdev)1202 static inline struct queue_limits *bdev_limits(struct block_device *bdev)
1203 {
1204 return &bdev_get_queue(bdev)->limits;
1205 }
1206
queue_segment_boundary(const struct request_queue * q)1207 static inline unsigned long queue_segment_boundary(const struct request_queue *q)
1208 {
1209 return q->limits.seg_boundary_mask;
1210 }
1211
queue_virt_boundary(const struct request_queue * q)1212 static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1213 {
1214 return q->limits.virt_boundary_mask;
1215 }
1216
queue_max_sectors(const struct request_queue * q)1217 static inline unsigned int queue_max_sectors(const struct request_queue *q)
1218 {
1219 return q->limits.max_sectors;
1220 }
1221
queue_max_bytes(struct request_queue * q)1222 static inline unsigned int queue_max_bytes(struct request_queue *q)
1223 {
1224 return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9;
1225 }
1226
queue_max_hw_sectors(const struct request_queue * q)1227 static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1228 {
1229 return q->limits.max_hw_sectors;
1230 }
1231
queue_max_segments(const struct request_queue * q)1232 static inline unsigned short queue_max_segments(const struct request_queue *q)
1233 {
1234 return q->limits.max_segments;
1235 }
1236
queue_max_discard_segments(const struct request_queue * q)1237 static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1238 {
1239 return q->limits.max_discard_segments;
1240 }
1241
queue_max_segment_size(const struct request_queue * q)1242 static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1243 {
1244 return q->limits.max_segment_size;
1245 }
1246
queue_emulates_zone_append(struct request_queue * q)1247 static inline bool queue_emulates_zone_append(struct request_queue *q)
1248 {
1249 return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors;
1250 }
1251
bdev_emulates_zone_append(struct block_device * bdev)1252 static inline bool bdev_emulates_zone_append(struct block_device *bdev)
1253 {
1254 return queue_emulates_zone_append(bdev_get_queue(bdev));
1255 }
1256
1257 static inline unsigned int
bdev_max_zone_append_sectors(struct block_device * bdev)1258 bdev_max_zone_append_sectors(struct block_device *bdev)
1259 {
1260 return bdev_limits(bdev)->max_zone_append_sectors;
1261 }
1262
bdev_max_segments(struct block_device * bdev)1263 static inline unsigned int bdev_max_segments(struct block_device *bdev)
1264 {
1265 return queue_max_segments(bdev_get_queue(bdev));
1266 }
1267
queue_logical_block_size(const struct request_queue * q)1268 static inline unsigned queue_logical_block_size(const struct request_queue *q)
1269 {
1270 return q->limits.logical_block_size;
1271 }
1272
bdev_logical_block_size(struct block_device * bdev)1273 static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
1274 {
1275 return queue_logical_block_size(bdev_get_queue(bdev));
1276 }
1277
queue_physical_block_size(const struct request_queue * q)1278 static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1279 {
1280 return q->limits.physical_block_size;
1281 }
1282
bdev_physical_block_size(struct block_device * bdev)1283 static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1284 {
1285 return queue_physical_block_size(bdev_get_queue(bdev));
1286 }
1287
queue_io_min(const struct request_queue * q)1288 static inline unsigned int queue_io_min(const struct request_queue *q)
1289 {
1290 return q->limits.io_min;
1291 }
1292
bdev_io_min(struct block_device * bdev)1293 static inline unsigned int bdev_io_min(struct block_device *bdev)
1294 {
1295 return queue_io_min(bdev_get_queue(bdev));
1296 }
1297
queue_io_opt(const struct request_queue * q)1298 static inline unsigned int queue_io_opt(const struct request_queue *q)
1299 {
1300 return q->limits.io_opt;
1301 }
1302
bdev_io_opt(struct block_device * bdev)1303 static inline unsigned int bdev_io_opt(struct block_device *bdev)
1304 {
1305 return queue_io_opt(bdev_get_queue(bdev));
1306 }
1307
1308 static inline unsigned int
queue_zone_write_granularity(const struct request_queue * q)1309 queue_zone_write_granularity(const struct request_queue *q)
1310 {
1311 return q->limits.zone_write_granularity;
1312 }
1313
1314 static inline unsigned int
bdev_zone_write_granularity(struct block_device * bdev)1315 bdev_zone_write_granularity(struct block_device *bdev)
1316 {
1317 return queue_zone_write_granularity(bdev_get_queue(bdev));
1318 }
1319
1320 int bdev_alignment_offset(struct block_device *bdev);
1321 unsigned int bdev_discard_alignment(struct block_device *bdev);
1322
bdev_max_discard_sectors(struct block_device * bdev)1323 static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev)
1324 {
1325 return bdev_limits(bdev)->max_discard_sectors;
1326 }
1327
bdev_discard_granularity(struct block_device * bdev)1328 static inline unsigned int bdev_discard_granularity(struct block_device *bdev)
1329 {
1330 return bdev_limits(bdev)->discard_granularity;
1331 }
1332
1333 static inline unsigned int
bdev_max_secure_erase_sectors(struct block_device * bdev)1334 bdev_max_secure_erase_sectors(struct block_device *bdev)
1335 {
1336 return bdev_limits(bdev)->max_secure_erase_sectors;
1337 }
1338
bdev_write_zeroes_sectors(struct block_device * bdev)1339 static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1340 {
1341 return bdev_limits(bdev)->max_write_zeroes_sectors;
1342 }
1343
bdev_nonrot(struct block_device * bdev)1344 static inline bool bdev_nonrot(struct block_device *bdev)
1345 {
1346 return blk_queue_nonrot(bdev_get_queue(bdev));
1347 }
1348
bdev_synchronous(struct block_device * bdev)1349 static inline bool bdev_synchronous(struct block_device *bdev)
1350 {
1351 return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
1352 }
1353
bdev_stable_writes(struct block_device * bdev)1354 static inline bool bdev_stable_writes(struct block_device *bdev)
1355 {
1356 struct request_queue *q = bdev_get_queue(bdev);
1357
1358 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1359 q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE)
1360 return true;
1361 return q->limits.features & BLK_FEAT_STABLE_WRITES;
1362 }
1363
blk_queue_write_cache(struct request_queue * q)1364 static inline bool blk_queue_write_cache(struct request_queue *q)
1365 {
1366 return (q->limits.features & BLK_FEAT_WRITE_CACHE) &&
1367 !(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED);
1368 }
1369
bdev_write_cache(struct block_device * bdev)1370 static inline bool bdev_write_cache(struct block_device *bdev)
1371 {
1372 return blk_queue_write_cache(bdev_get_queue(bdev));
1373 }
1374
bdev_fua(struct block_device * bdev)1375 static inline bool bdev_fua(struct block_device *bdev)
1376 {
1377 return bdev_limits(bdev)->features & BLK_FEAT_FUA;
1378 }
1379
bdev_nowait(struct block_device * bdev)1380 static inline bool bdev_nowait(struct block_device *bdev)
1381 {
1382 return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT;
1383 }
1384
bdev_is_zoned(struct block_device * bdev)1385 static inline bool bdev_is_zoned(struct block_device *bdev)
1386 {
1387 return blk_queue_is_zoned(bdev_get_queue(bdev));
1388 }
1389
bdev_zone_no(struct block_device * bdev,sector_t sec)1390 static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec)
1391 {
1392 return disk_zone_no(bdev->bd_disk, sec);
1393 }
1394
bdev_zone_sectors(struct block_device * bdev)1395 static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1396 {
1397 struct request_queue *q = bdev_get_queue(bdev);
1398
1399 if (!blk_queue_is_zoned(q))
1400 return 0;
1401 return q->limits.chunk_sectors;
1402 }
1403
bdev_offset_from_zone_start(struct block_device * bdev,sector_t sector)1404 static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev,
1405 sector_t sector)
1406 {
1407 return sector & (bdev_zone_sectors(bdev) - 1);
1408 }
1409
bio_offset_from_zone_start(struct bio * bio)1410 static inline sector_t bio_offset_from_zone_start(struct bio *bio)
1411 {
1412 return bdev_offset_from_zone_start(bio->bi_bdev,
1413 bio->bi_iter.bi_sector);
1414 }
1415
bdev_is_zone_start(struct block_device * bdev,sector_t sector)1416 static inline bool bdev_is_zone_start(struct block_device *bdev,
1417 sector_t sector)
1418 {
1419 return bdev_offset_from_zone_start(bdev, sector) == 0;
1420 }
1421
1422 /**
1423 * bdev_zone_is_seq - check if a sector belongs to a sequential write zone
1424 * @bdev: block device to check
1425 * @sector: sector number
1426 *
1427 * Check if @sector on @bdev is contained in a sequential write required zone.
1428 */
bdev_zone_is_seq(struct block_device * bdev,sector_t sector)1429 static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector)
1430 {
1431 bool is_seq = false;
1432
1433 #if IS_ENABLED(CONFIG_BLK_DEV_ZONED)
1434 if (bdev_is_zoned(bdev)) {
1435 struct gendisk *disk = bdev->bd_disk;
1436 unsigned long *bitmap;
1437
1438 rcu_read_lock();
1439 bitmap = rcu_dereference(disk->conv_zones_bitmap);
1440 is_seq = !bitmap ||
1441 !test_bit(disk_zone_no(disk, sector), bitmap);
1442 rcu_read_unlock();
1443 }
1444 #endif
1445
1446 return is_seq;
1447 }
1448
1449 int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector,
1450 sector_t nr_sects, gfp_t gfp_mask);
1451
queue_dma_alignment(const struct request_queue * q)1452 static inline unsigned int queue_dma_alignment(const struct request_queue *q)
1453 {
1454 return q->limits.dma_alignment;
1455 }
1456
1457 static inline unsigned int
queue_atomic_write_unit_max_bytes(const struct request_queue * q)1458 queue_atomic_write_unit_max_bytes(const struct request_queue *q)
1459 {
1460 return q->limits.atomic_write_unit_max;
1461 }
1462
1463 static inline unsigned int
queue_atomic_write_unit_min_bytes(const struct request_queue * q)1464 queue_atomic_write_unit_min_bytes(const struct request_queue *q)
1465 {
1466 return q->limits.atomic_write_unit_min;
1467 }
1468
1469 static inline unsigned int
queue_atomic_write_boundary_bytes(const struct request_queue * q)1470 queue_atomic_write_boundary_bytes(const struct request_queue *q)
1471 {
1472 return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT;
1473 }
1474
1475 static inline unsigned int
queue_atomic_write_max_bytes(const struct request_queue * q)1476 queue_atomic_write_max_bytes(const struct request_queue *q)
1477 {
1478 return q->limits.atomic_write_max_sectors << SECTOR_SHIFT;
1479 }
1480
bdev_dma_alignment(struct block_device * bdev)1481 static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
1482 {
1483 return queue_dma_alignment(bdev_get_queue(bdev));
1484 }
1485
bdev_iter_is_aligned(struct block_device * bdev,struct iov_iter * iter)1486 static inline bool bdev_iter_is_aligned(struct block_device *bdev,
1487 struct iov_iter *iter)
1488 {
1489 return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev),
1490 bdev_logical_block_size(bdev) - 1);
1491 }
1492
1493 static inline unsigned int
blk_lim_dma_alignment_and_pad(struct queue_limits * lim)1494 blk_lim_dma_alignment_and_pad(struct queue_limits *lim)
1495 {
1496 return lim->dma_alignment | lim->dma_pad_mask;
1497 }
1498
blk_rq_aligned(struct request_queue * q,unsigned long addr,unsigned int len)1499 static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr,
1500 unsigned int len)
1501 {
1502 unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits);
1503
1504 return !(addr & alignment) && !(len & alignment);
1505 }
1506
1507 /* assumes size > 256 */
blksize_bits(unsigned int size)1508 static inline unsigned int blksize_bits(unsigned int size)
1509 {
1510 return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT;
1511 }
1512
1513 int kblockd_schedule_work(struct work_struct *work);
1514 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1515
1516 #define MODULE_ALIAS_BLOCKDEV(major,minor) \
1517 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1518 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1519 MODULE_ALIAS("block-major-" __stringify(major) "-*")
1520
1521 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1522
1523 bool blk_crypto_register(struct blk_crypto_profile *profile,
1524 struct request_queue *q);
1525
1526 #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1527
blk_crypto_register(struct blk_crypto_profile * profile,struct request_queue * q)1528 static inline bool blk_crypto_register(struct blk_crypto_profile *profile,
1529 struct request_queue *q)
1530 {
1531 return true;
1532 }
1533
1534 #endif /* CONFIG_BLK_INLINE_ENCRYPTION */
1535
1536 enum blk_unique_id {
1537 /* these match the Designator Types specified in SPC */
1538 BLK_UID_T10 = 1,
1539 BLK_UID_EUI64 = 2,
1540 BLK_UID_NAA = 3,
1541 };
1542
1543 struct block_device_operations {
1544 void (*submit_bio)(struct bio *bio);
1545 int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob,
1546 unsigned int flags);
1547 int (*open)(struct gendisk *disk, blk_mode_t mode);
1548 void (*release)(struct gendisk *disk);
1549 int (*ioctl)(struct block_device *bdev, blk_mode_t mode,
1550 unsigned cmd, unsigned long arg);
1551 int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode,
1552 unsigned cmd, unsigned long arg);
1553 unsigned int (*check_events) (struct gendisk *disk,
1554 unsigned int clearing);
1555 void (*unlock_native_capacity) (struct gendisk *);
1556 int (*getgeo)(struct block_device *, struct hd_geometry *);
1557 int (*set_read_only)(struct block_device *bdev, bool ro);
1558 void (*free_disk)(struct gendisk *disk);
1559 /* this callback is with swap_lock and sometimes page table lock held */
1560 void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1561 int (*report_zones)(struct gendisk *, sector_t sector,
1562 unsigned int nr_zones, report_zones_cb cb, void *data);
1563 char *(*devnode)(struct gendisk *disk, umode_t *mode);
1564 /* returns the length of the identifier or a negative errno: */
1565 int (*get_unique_id)(struct gendisk *disk, u8 id[16],
1566 enum blk_unique_id id_type);
1567 struct module *owner;
1568 const struct pr_ops *pr_ops;
1569
1570 /*
1571 * Special callback for probing GPT entry at a given sector.
1572 * Needed by Android devices, used by GPT scanner and MMC blk
1573 * driver.
1574 */
1575 int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector);
1576 };
1577
1578 #ifdef CONFIG_COMPAT
1579 extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t,
1580 unsigned int, unsigned long);
1581 #else
1582 #define blkdev_compat_ptr_ioctl NULL
1583 #endif
1584
blk_wake_io_task(struct task_struct * waiter)1585 static inline void blk_wake_io_task(struct task_struct *waiter)
1586 {
1587 /*
1588 * If we're polling, the task itself is doing the completions. For
1589 * that case, we don't need to signal a wakeup, it's enough to just
1590 * mark us as RUNNING.
1591 */
1592 if (waiter == current)
1593 __set_current_state(TASK_RUNNING);
1594 else
1595 wake_up_process(waiter);
1596 }
1597
1598 unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
1599 unsigned long start_time);
1600 void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
1601 unsigned int sectors, unsigned long start_time);
1602
1603 unsigned long bio_start_io_acct(struct bio *bio);
1604 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1605 struct block_device *orig_bdev);
1606
1607 /**
1608 * bio_end_io_acct - end I/O accounting for bio based drivers
1609 * @bio: bio to end account for
1610 * @start_time: start time returned by bio_start_io_acct()
1611 */
bio_end_io_acct(struct bio * bio,unsigned long start_time)1612 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
1613 {
1614 return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
1615 }
1616
1617 int set_blocksize(struct file *file, int size);
1618
1619 int lookup_bdev(const char *pathname, dev_t *dev);
1620
1621 void blkdev_show(struct seq_file *seqf, off_t offset);
1622
1623 #define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */
1624 #define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */
1625 #ifdef CONFIG_BLOCK
1626 #define BLKDEV_MAJOR_MAX 512
1627 #else
1628 #define BLKDEV_MAJOR_MAX 0
1629 #endif
1630
1631 struct blk_holder_ops {
1632 void (*mark_dead)(struct block_device *bdev, bool surprise);
1633
1634 /*
1635 * Sync the file system mounted on the block device.
1636 */
1637 void (*sync)(struct block_device *bdev);
1638
1639 /*
1640 * Freeze the file system mounted on the block device.
1641 */
1642 int (*freeze)(struct block_device *bdev);
1643
1644 /*
1645 * Thaw the file system mounted on the block device.
1646 */
1647 int (*thaw)(struct block_device *bdev);
1648 };
1649
1650 /*
1651 * For filesystems using @fs_holder_ops, the @holder argument passed to
1652 * helpers used to open and claim block devices via
1653 * bd_prepare_to_claim() must point to a superblock.
1654 */
1655 extern const struct blk_holder_ops fs_holder_ops;
1656
1657 /*
1658 * Return the correct open flags for blkdev_get_by_* for super block flags
1659 * as stored in sb->s_flags.
1660 */
1661 #define sb_open_mode(flags) \
1662 (BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \
1663 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
1664
1665 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
1666 const struct blk_holder_ops *hops);
1667 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,
1668 void *holder, const struct blk_holder_ops *hops);
1669 int bd_prepare_to_claim(struct block_device *bdev, void *holder,
1670 const struct blk_holder_ops *hops);
1671 void bd_abort_claiming(struct block_device *bdev, void *holder);
1672
1673 /* just for blk-cgroup, don't use elsewhere */
1674 struct block_device *blkdev_get_no_open(dev_t dev);
1675 void blkdev_put_no_open(struct block_device *bdev);
1676
1677 struct block_device *I_BDEV(struct inode *inode);
1678 struct block_device *file_bdev(struct file *bdev_file);
1679 bool disk_live(struct gendisk *disk);
1680 unsigned int block_size(struct block_device *bdev);
1681
1682 #ifdef CONFIG_BLOCK
1683 void invalidate_bdev(struct block_device *bdev);
1684 int sync_blockdev(struct block_device *bdev);
1685 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
1686 int sync_blockdev_nowait(struct block_device *bdev);
1687 void sync_bdevs(bool wait);
1688 void bdev_statx(struct path *, struct kstat *, u32);
1689 void printk_all_partitions(void);
1690 int __init early_lookup_bdev(const char *pathname, dev_t *dev);
1691 #else
invalidate_bdev(struct block_device * bdev)1692 static inline void invalidate_bdev(struct block_device *bdev)
1693 {
1694 }
sync_blockdev(struct block_device * bdev)1695 static inline int sync_blockdev(struct block_device *bdev)
1696 {
1697 return 0;
1698 }
sync_blockdev_nowait(struct block_device * bdev)1699 static inline int sync_blockdev_nowait(struct block_device *bdev)
1700 {
1701 return 0;
1702 }
sync_bdevs(bool wait)1703 static inline void sync_bdevs(bool wait)
1704 {
1705 }
bdev_statx(struct path * path,struct kstat * stat,u32 request_mask)1706 static inline void bdev_statx(struct path *path, struct kstat *stat,
1707 u32 request_mask)
1708 {
1709 }
printk_all_partitions(void)1710 static inline void printk_all_partitions(void)
1711 {
1712 }
early_lookup_bdev(const char * pathname,dev_t * dev)1713 static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
1714 {
1715 return -EINVAL;
1716 }
1717 #endif /* CONFIG_BLOCK */
1718
1719 int bdev_freeze(struct block_device *bdev);
1720 int bdev_thaw(struct block_device *bdev);
1721 void bdev_fput(struct file *bdev_file);
1722
1723 struct io_comp_batch {
1724 struct rq_list req_list;
1725 bool need_ts;
1726 void (*complete)(struct io_comp_batch *);
1727 };
1728
blk_atomic_write_start_sect_aligned(sector_t sector,struct queue_limits * limits)1729 static inline bool blk_atomic_write_start_sect_aligned(sector_t sector,
1730 struct queue_limits *limits)
1731 {
1732 unsigned int alignment = max(limits->atomic_write_hw_unit_min,
1733 limits->atomic_write_hw_boundary);
1734
1735 return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT);
1736 }
1737
bdev_can_atomic_write(struct block_device * bdev)1738 static inline bool bdev_can_atomic_write(struct block_device *bdev)
1739 {
1740 struct request_queue *bd_queue = bdev->bd_queue;
1741 struct queue_limits *limits = &bd_queue->limits;
1742
1743 if (!limits->atomic_write_unit_min)
1744 return false;
1745
1746 if (bdev_is_partition(bdev))
1747 return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect,
1748 limits);
1749
1750 return true;
1751 }
1752
1753 static inline unsigned int
bdev_atomic_write_unit_min_bytes(struct block_device * bdev)1754 bdev_atomic_write_unit_min_bytes(struct block_device *bdev)
1755 {
1756 if (!bdev_can_atomic_write(bdev))
1757 return 0;
1758 return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev));
1759 }
1760
1761 static inline unsigned int
bdev_atomic_write_unit_max_bytes(struct block_device * bdev)1762 bdev_atomic_write_unit_max_bytes(struct block_device *bdev)
1763 {
1764 if (!bdev_can_atomic_write(bdev))
1765 return 0;
1766 return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev));
1767 }
1768
1769 #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { }
1770
1771 #endif /* _LINUX_BLKDEV_H */
1772