xref: /linux/include/linux/blkdev.h (revision 603523fd54664f17630f742229166278568c3ce8)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Portions Copyright (C) 1992 Drew Eckhardt
4  */
5 #ifndef _LINUX_BLKDEV_H
6 #define _LINUX_BLKDEV_H
7 
8 #include <linux/types.h>
9 #include <linux/blk_types.h>
10 #include <linux/blk_plug.h>
11 #include <linux/device.h>
12 #include <linux/list.h>
13 #include <linux/llist.h>
14 #include <linux/minmax.h>
15 #include <linux/timer.h>
16 #include <linux/workqueue.h>
17 #include <linux/completion.h>
18 #include <linux/wait.h>
19 #include <linux/bio.h>
20 #include <linux/gfp.h>
21 #include <linux/kdev_t.h>
22 #include <linux/rcupdate.h>
23 #include <linux/percpu-refcount.h>
24 #include <linux/blkzoned.h>
25 #include <linux/sbitmap.h>
26 #include <linux/uuid.h>
27 #include <linux/xarray.h>
28 #include <linux/file.h>
29 #include <linux/lockdep.h>
30 
31 struct module;
32 struct request_queue;
33 struct elevator_queue;
34 struct blk_trace;
35 struct request;
36 struct sg_io_hdr;
37 struct blkcg_gq;
38 struct blk_flush_queue;
39 struct kiocb;
40 struct pr_ops;
41 struct rq_qos;
42 struct hd_geometry;
43 struct blk_report_zones_args;
44 struct blk_queue_stats;
45 struct blk_stat_callback;
46 struct blk_crypto_profile;
47 
48 extern const struct device_type disk_type;
49 extern const struct device_type part_type;
50 extern const struct class block_class;
51 
52 /*
53  * Maximum number of blkcg policies allowed to be registered concurrently.
54  * Defined here to simplify include dependency.
55  */
56 #define BLKCG_MAX_POLS		6
57 
58 #define DISK_MAX_PARTS			256
59 #define DISK_NAME_LEN			32
60 
61 #define PARTITION_META_INFO_VOLNAMELTH	64
62 /*
63  * Enough for the string representation of any kind of UUID plus NULL.
64  * EFI UUID is 36 characters. MSDOS UUID is 11 characters.
65  */
66 #define PARTITION_META_INFO_UUIDLTH	(UUID_STRING_LEN + 1)
67 
68 struct partition_meta_info {
69 	char uuid[PARTITION_META_INFO_UUIDLTH];
70 	u8 volname[PARTITION_META_INFO_VOLNAMELTH];
71 };
72 
73 /**
74  * DOC: genhd capability flags
75  *
76  * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to
77  * removable media.  When set, the device remains present even when media is not
78  * inserted.  Shall not be set for devices which are removed entirely when the
79  * media is removed.
80  *
81  * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events,
82  * doesn't appear in sysfs, and can't be opened from userspace or using
83  * blkdev_get*. Used for the underlying components of multipath devices.
84  *
85  * ``GENHD_FL_NO_PART``: partition support is disabled.  The kernel will not
86  * scan for partitions from add_disk, and users can't add partitions manually.
87  *
88  */
89 enum {
90 	GENHD_FL_REMOVABLE			= 1 << 0,
91 	GENHD_FL_HIDDEN				= 1 << 1,
92 	GENHD_FL_NO_PART			= 1 << 2,
93 };
94 
95 enum {
96 	DISK_EVENT_MEDIA_CHANGE			= 1 << 0, /* media changed */
97 	DISK_EVENT_EJECT_REQUEST		= 1 << 1, /* eject requested */
98 };
99 
100 enum {
101 	/* Poll even if events_poll_msecs is unset */
102 	DISK_EVENT_FLAG_POLL			= 1 << 0,
103 	/* Forward events to udev */
104 	DISK_EVENT_FLAG_UEVENT			= 1 << 1,
105 	/* Block event polling when open for exclusive write */
106 	DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE	= 1 << 2,
107 };
108 
109 struct disk_events;
110 struct badblocks;
111 
112 enum blk_integrity_checksum {
113 	BLK_INTEGRITY_CSUM_NONE		= 0,
114 	BLK_INTEGRITY_CSUM_IP		= 1,
115 	BLK_INTEGRITY_CSUM_CRC		= 2,
116 	BLK_INTEGRITY_CSUM_CRC64	= 3,
117 } __packed ;
118 
119 struct blk_integrity {
120 	unsigned char				flags;
121 	enum blk_integrity_checksum		csum_type;
122 	unsigned char				metadata_size;
123 	unsigned char				pi_offset;
124 	unsigned char				interval_exp;
125 	unsigned char				tag_size;
126 	unsigned char				pi_tuple_size;
127 };
128 
129 /* open for reading */
130 #define BLK_OPEN_READ		((__force blk_mode_t)(1 << 0))
131 /* open for writing */
132 #define BLK_OPEN_WRITE		((__force blk_mode_t)(1 << 1))
133 /* open exclusively (vs other exclusive openers */
134 #define BLK_OPEN_EXCL		((__force blk_mode_t)(1 << 2))
135 /* opened with O_NDELAY */
136 #define BLK_OPEN_NDELAY		((__force blk_mode_t)(1 << 3))
137 /* open for "writes" only for ioctls (specialy hack for floppy.c) */
138 #define BLK_OPEN_WRITE_IOCTL	((__force blk_mode_t)(1 << 4))
139 /* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */
140 #define BLK_OPEN_RESTRICT_WRITES	((__force blk_mode_t)(1 << 5))
141 /* return partition scanning errors */
142 #define BLK_OPEN_STRICT_SCAN	((__force blk_mode_t)(1 << 6))
143 
144 struct gendisk {
145 	/*
146 	 * major/first_minor/minors should not be set by any new driver, the
147 	 * block core will take care of allocating them automatically.
148 	 */
149 	int major;
150 	int first_minor;
151 	int minors;
152 
153 	char disk_name[DISK_NAME_LEN];	/* name of major driver */
154 
155 	unsigned short events;		/* supported events */
156 	unsigned short event_flags;	/* flags related to event processing */
157 
158 	struct xarray part_tbl;
159 	struct block_device *part0;
160 
161 	const struct block_device_operations *fops;
162 	struct request_queue *queue;
163 	void *private_data;
164 
165 	struct bio_set bio_split;
166 
167 	int flags;
168 	unsigned long state;
169 #define GD_NEED_PART_SCAN		0
170 #define GD_READ_ONLY			1
171 #define GD_DEAD				2
172 #define GD_NATIVE_CAPACITY		3
173 #define GD_ADDED			4
174 #define GD_SUPPRESS_PART_SCAN		5
175 #define GD_OWNS_QUEUE			6
176 #define GD_ZONE_APPEND_USED		7
177 #define GD_ERROR_INJECT			8
178 
179 	struct mutex open_mutex;	/* open/close mutex */
180 	unsigned open_partitions;	/* number of open partitions */
181 
182 	struct backing_dev_info	*bdi;
183 	struct kobject queue_kobj;	/* the queue/ directory */
184 	struct kobject *slave_dir;
185 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
186 	struct list_head slave_bdevs;
187 #endif
188 	struct timer_rand_state *random;
189 	struct disk_events *ev;
190 
191 #ifdef CONFIG_BLK_DEV_ZONED
192 	/*
193 	 * Zoned block device information. Reads of this information must be
194 	 * protected with blk_queue_enter() / blk_queue_exit() or by holding a
195 	 * lock on zone_revalidate_mutex. blk_revalidate_disk_zones() may modify
196 	 * this information while no requests are being processed (disk queue
197 	 * frozen with blk_mq_freeze_queue()) and while holding a lock on
198 	 * zone_revalidate_mutex.
199 	 */
200 	struct mutex		zone_revalidate_mutex;
201 	unsigned int		nr_zones;
202 	unsigned int		zone_capacity;
203 	unsigned int		last_zone_capacity;
204 	u8 __rcu		*zones_state;
205 	unsigned int		zone_wplugs_hash_bits;
206 	atomic_t		nr_zone_wplugs;
207 	spinlock_t		zone_wplugs_hash_lock;
208 	struct mempool		*zone_wplugs_pool;
209 	struct hlist_head	*zone_wplugs_hash;
210 	struct workqueue_struct *zone_wplugs_wq;
211 	spinlock_t		zone_wplugs_list_lock;
212 	struct list_head	zone_wplugs_list;
213 	struct task_struct	*zone_wplugs_worker;
214 	struct completion	zone_wplugs_worker_bio_done;
215 #endif /* CONFIG_BLK_DEV_ZONED */
216 
217 #if IS_ENABLED(CONFIG_CDROM)
218 	struct cdrom_device_info *cdi;
219 #endif
220 	int node_id;
221 	struct badblocks *bb;
222 	struct lockdep_map lockdep_map;
223 	u64 diskseq;
224 	blk_mode_t open_mode;
225 
226 	/*
227 	 * Independent sector access ranges. This is always NULL for
228 	 * devices that do not have multiple independent access ranges.
229 	 */
230 	struct blk_independent_access_ranges *ia_ranges;
231 
232 #ifdef CONFIG_BLK_ERROR_INJECTION
233 	struct mutex		error_injection_lock;
234 	struct list_head	error_injection_list;
235 #endif
236 
237 	struct mutex rqos_state_mutex;	/* rqos state change mutex */
238 };
239 
240 /**
241  * disk_openers - returns how many openers are there for a disk
242  * @disk: disk to check
243  *
244  * This returns the number of openers for a disk.  Note that this value is only
245  * stable if disk->open_mutex is held.
246  *
247  * Note: Due to a quirk in the block layer open code, each open partition is
248  * only counted once even if there are multiple openers.
249  */
250 static inline unsigned int disk_openers(struct gendisk *disk)
251 {
252 	return atomic_read(&disk->part0->bd_openers);
253 }
254 
255 /**
256  * disk_has_partscan - return %true if partition scanning is enabled on a disk
257  * @disk: disk to check
258  *
259  * Returns %true if partitions scanning is enabled for @disk, or %false if
260  * partition scanning is disabled either permanently or temporarily.
261  */
262 static inline bool disk_has_partscan(struct gendisk *disk)
263 {
264 	return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) &&
265 		!test_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
266 }
267 
268 /*
269  * The gendisk is refcounted by the part0 block_device, and the bd_device
270  * therein is also used for device model presentation in sysfs.
271  */
272 #define dev_to_disk(device) \
273 	(dev_to_bdev(device)->bd_disk)
274 #define disk_to_dev(disk) \
275 	(&((disk)->part0->bd_device))
276 
277 #if IS_REACHABLE(CONFIG_CDROM)
278 #define disk_to_cdi(disk)	((disk)->cdi)
279 #else
280 #define disk_to_cdi(disk)	NULL
281 #endif
282 
283 static inline dev_t disk_devt(struct gendisk *disk)
284 {
285 	return MKDEV(disk->major, disk->first_minor);
286 }
287 
288 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
289 /*
290  * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER)
291  * however we constrain this to what we can validate and test.
292  */
293 #define BLK_MAX_BLOCK_SIZE      SZ_64K
294 #else
295 #define BLK_MAX_BLOCK_SIZE      PAGE_SIZE
296 #endif
297 
298 
299 /* blk_validate_limits() validates bsize, so drivers don't usually need to */
300 static inline int blk_validate_block_size(unsigned long bsize)
301 {
302 	if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize))
303 		return -EINVAL;
304 
305 	return 0;
306 }
307 
308 static inline bool blk_op_is_passthrough(blk_opf_t op)
309 {
310 	op &= REQ_OP_MASK;
311 	return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
312 }
313 
314 /* flags set by the driver in queue_limits.features */
315 typedef unsigned int __bitwise blk_features_t;
316 
317 /* supports a volatile write cache */
318 #define BLK_FEAT_WRITE_CACHE		((__force blk_features_t)(1u << 0))
319 
320 /* supports passing on the FUA bit */
321 #define BLK_FEAT_FUA			((__force blk_features_t)(1u << 1))
322 
323 /* rotational device (hard drive or floppy) */
324 #define BLK_FEAT_ROTATIONAL		((__force blk_features_t)(1u << 2))
325 
326 /* contributes to the random number pool */
327 #define BLK_FEAT_ADD_RANDOM		((__force blk_features_t)(1u << 3))
328 
329 /* do disk/partitions IO accounting */
330 #define BLK_FEAT_IO_STAT		((__force blk_features_t)(1u << 4))
331 
332 /* don't modify data until writeback is done */
333 #define BLK_FEAT_STABLE_WRITES		((__force blk_features_t)(1u << 5))
334 
335 /* always completes in submit context */
336 #define BLK_FEAT_SYNCHRONOUS		((__force blk_features_t)(1u << 6))
337 
338 /* supports REQ_NOWAIT */
339 #define BLK_FEAT_NOWAIT			((__force blk_features_t)(1u << 7))
340 
341 /* supports DAX */
342 #define BLK_FEAT_DAX			((__force blk_features_t)(1u << 8))
343 
344 /* supports I/O polling */
345 #define BLK_FEAT_POLL			((__force blk_features_t)(1u << 9))
346 
347 /* is a zoned device */
348 #define BLK_FEAT_ZONED			((__force blk_features_t)(1u << 10))
349 
350 /* supports PCI(e) p2p requests */
351 #define BLK_FEAT_PCI_P2PDMA		((__force blk_features_t)(1u << 12))
352 
353 /* skip this queue in blk_mq_(un)quiesce_tagset */
354 #define BLK_FEAT_SKIP_TAGSET_QUIESCE	((__force blk_features_t)(1u << 13))
355 
356 /* atomic writes enabled */
357 #define BLK_FEAT_ATOMIC_WRITES		((__force blk_features_t)(1u << 14))
358 
359 /* undocumented magic for bcache */
360 #define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \
361 	((__force blk_features_t)(1u << 15))
362 
363 /*
364  * Flags automatically inherited when stacking limits.
365  */
366 #define BLK_FEAT_INHERIT_MASK \
367 	(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \
368 	 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | \
369 	 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE)
370 
371 /* internal flags in queue_limits.flags */
372 typedef unsigned int __bitwise blk_flags_t;
373 
374 /* do not send FLUSH/FUA commands despite advertising a write cache */
375 #define BLK_FLAG_WRITE_CACHE_DISABLED	((__force blk_flags_t)(1u << 0))
376 
377 /* I/O topology is misaligned */
378 #define BLK_FLAG_MISALIGNED		((__force blk_flags_t)(1u << 1))
379 
380 /* passthrough command IO accounting */
381 #define BLK_FLAG_IOSTATS_PASSTHROUGH	((__force blk_flags_t)(1u << 2))
382 
383 struct queue_limits {
384 	blk_features_t		features;
385 	blk_flags_t		flags;
386 	unsigned long		seg_boundary_mask;
387 	unsigned long		virt_boundary_mask;
388 
389 	unsigned int		max_hw_sectors;
390 	unsigned int		max_dev_sectors;
391 	unsigned int		chunk_sectors;
392 	unsigned int		max_sectors;
393 	unsigned int		max_user_sectors;
394 	unsigned int		max_segment_size;
395 	unsigned int		max_fast_segment_size;
396 	unsigned int		physical_block_size;
397 	unsigned int		logical_block_size;
398 	unsigned int		alignment_offset;
399 	unsigned int		io_min;
400 	unsigned int		io_opt;
401 	unsigned int		max_discard_sectors;
402 	unsigned int		max_hw_discard_sectors;
403 	unsigned int		max_user_discard_sectors;
404 	unsigned int		max_secure_erase_sectors;
405 	unsigned int		max_write_zeroes_sectors;
406 	unsigned int		max_wzeroes_unmap_sectors;
407 	unsigned int		max_hw_wzeroes_unmap_sectors;
408 	unsigned int		max_user_wzeroes_unmap_sectors;
409 	unsigned int		max_hw_zone_append_sectors;
410 	unsigned int		max_zone_append_sectors;
411 	unsigned int		discard_granularity;
412 	unsigned int		discard_alignment;
413 	unsigned int		zone_write_granularity;
414 
415 	/* atomic write limits */
416 	unsigned int		atomic_write_hw_max;
417 	unsigned int		atomic_write_max_sectors;
418 	unsigned int		atomic_write_hw_boundary;
419 	unsigned int		atomic_write_boundary_sectors;
420 	unsigned int		atomic_write_hw_unit_min;
421 	unsigned int		atomic_write_unit_min;
422 	unsigned int		atomic_write_hw_unit_max;
423 	unsigned int		atomic_write_unit_max;
424 
425 	unsigned short		max_segments;
426 	unsigned short		max_integrity_segments;
427 	unsigned short		max_discard_segments;
428 
429 	unsigned short		max_write_streams;
430 	unsigned int		write_stream_granularity;
431 
432 	unsigned int		max_open_zones;
433 	unsigned int		max_active_zones;
434 
435 	/*
436 	 * Drivers that set dma_alignment to less than 511 must be prepared to
437 	 * handle individual bvec's that are not a multiple of a SECTOR_SIZE
438 	 * due to possible offsets.
439 	 */
440 	unsigned int		dma_alignment;
441 	unsigned int		dma_pad_mask;
442 
443 	struct blk_integrity	integrity;
444 };
445 
446 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
447 			       void *data);
448 
449 int disk_report_zone(struct gendisk *disk, struct blk_zone *zone,
450 		     unsigned int idx, struct blk_report_zones_args *args);
451 
452 int blkdev_get_zone_info(struct block_device *bdev, sector_t sector,
453 			 struct blk_zone *zone);
454 
455 #define BLK_ALL_ZONES  ((unsigned int)-1)
456 int blkdev_report_zones(struct block_device *bdev, sector_t sector,
457 		unsigned int nr_zones, report_zones_cb cb, void *data);
458 int blkdev_report_zones_cached(struct block_device *bdev, sector_t sector,
459 		unsigned int nr_zones, report_zones_cb cb, void *data);
460 int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
461 		sector_t sectors, sector_t nr_sectors);
462 int blk_revalidate_disk_zones(struct gendisk *disk);
463 
464 /*
465  * Independent access ranges: struct blk_independent_access_range describes
466  * a range of contiguous sectors that can be accessed using device command
467  * execution resources that are independent from the resources used for
468  * other access ranges. This is typically found with single-LUN multi-actuator
469  * HDDs where each access range is served by a different set of heads.
470  * The set of independent ranges supported by the device is defined using
471  * struct blk_independent_access_ranges. The independent ranges must not overlap
472  * and must include all sectors within the disk capacity (no sector holes
473  * allowed).
474  * For a device with multiple ranges, requests targeting sectors in different
475  * ranges can be executed in parallel. A request can straddle an access range
476  * boundary.
477  */
478 struct blk_independent_access_range {
479 	struct kobject		kobj;
480 	sector_t		sector;
481 	sector_t		nr_sectors;
482 };
483 
484 struct blk_independent_access_ranges {
485 	struct kobject				kobj;
486 	bool					sysfs_registered;
487 	unsigned int				nr_ia_ranges;
488 	struct blk_independent_access_range	ia_range[];
489 };
490 
491 struct request_queue {
492 	/*
493 	 * The queue owner gets to use this for whatever they like.
494 	 * ll_rw_blk doesn't touch it.
495 	 */
496 	void			*queuedata;
497 
498 	struct elevator_queue	*elevator;
499 
500 	const struct blk_mq_ops	*mq_ops;
501 
502 	/* sw queues */
503 	struct blk_mq_ctx __percpu	*queue_ctx;
504 
505 	/*
506 	 * various queue flags, see QUEUE_* below
507 	 */
508 	unsigned long		queue_flags;
509 
510 	unsigned int __data_racy rq_timeout;
511 
512 	unsigned int		queue_depth;
513 
514 	refcount_t		refs;
515 
516 	/* hw dispatch queues */
517 	unsigned int		nr_hw_queues;
518 	struct blk_mq_hw_ctx * __rcu *queue_hw_ctx __counted_by_ptr(nr_hw_queues);
519 
520 	struct percpu_ref	q_usage_counter;
521 	struct lock_class_key	io_lock_cls_key;
522 	struct lockdep_map	io_lockdep_map;
523 
524 	struct lock_class_key	q_lock_cls_key;
525 	struct lockdep_map	q_lockdep_map;
526 
527 	struct request		*last_merge;
528 
529 	spinlock_t		queue_lock;
530 
531 	int			quiesce_depth;
532 
533 	struct gendisk		*disk;
534 
535 	/*
536 	 * mq queue kobject
537 	 */
538 	struct kobject *mq_kobj;
539 
540 	struct queue_limits	limits;
541 
542 #ifdef CONFIG_PM
543 	struct device		*dev;
544 	enum rpm_status		rpm_status;
545 #endif
546 
547 	/*
548 	 * Number of contexts that have called blk_set_pm_only(). If this
549 	 * counter is above zero then only RQF_PM requests are processed.
550 	 */
551 	atomic_t		pm_only;
552 
553 	struct blk_queue_stats	*stats;
554 	struct rq_qos		*rq_qos;
555 	struct mutex		rq_qos_mutex;
556 
557 	/*
558 	 * ida allocated id for this queue.  Used to index queues from
559 	 * ioctx.
560 	 */
561 	int			id;
562 
563 	/*
564 	 * queue settings
565 	 */
566 	unsigned int		nr_requests;	/* Max # of requests */
567 	unsigned int		async_depth;	/* Max # of async requests */
568 
569 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
570 	struct blk_crypto_profile *crypto_profile;
571 	struct kobject *crypto_kobject;
572 #endif
573 
574 	struct timer_list	timeout;
575 	struct work_struct	timeout_work;
576 
577 	atomic_t		nr_active_requests_shared_tags;
578 
579 	struct blk_mq_tags	*sched_shared_tags;
580 
581 	struct list_head	icq_list;
582 #ifdef CONFIG_BLK_CGROUP
583 	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
584 	struct blkcg_gq		*root_blkg;
585 	struct list_head	blkg_list;
586 	struct mutex		blkcg_mutex;
587 #endif
588 
589 	int			node;
590 
591 	spinlock_t		requeue_lock;
592 	struct list_head	requeue_list;
593 	struct delayed_work	requeue_work;
594 
595 #ifdef CONFIG_BLK_DEV_IO_TRACE
596 	struct blk_trace __rcu	*blk_trace;
597 #endif
598 	/*
599 	 * for flush operations
600 	 */
601 	struct blk_flush_queue	*fq;
602 	struct list_head	flush_list;
603 
604 	/*
605 	 * Protects against I/O scheduler switching, particularly when updating
606 	 * q->elevator. Since the elevator update code path may also modify q->
607 	 * nr_requests and wbt latency, this lock also protects the sysfs attrs
608 	 * nr_requests and wbt_lat_usec. Additionally the nr_hw_queues update
609 	 * may modify hctx tags, reserved-tags and cpumask, so this lock also
610 	 * helps protect the hctx sysfs/debugfs attrs. To ensure proper locking
611 	 * order during an elevator or nr_hw_queue update, first freeze the
612 	 * queue, then acquire ->elevator_lock.
613 	 */
614 	struct mutex		elevator_lock;
615 
616 	struct mutex		sysfs_lock;
617 	/*
618 	 * Protects queue limits and also sysfs attribute read_ahead_kb.
619 	 */
620 	struct mutex		limits_lock;
621 
622 	/*
623 	 * for reusing dead hctx instance in case of updating
624 	 * nr_hw_queues
625 	 */
626 	struct list_head	unused_hctx_list;
627 	spinlock_t		unused_hctx_lock;
628 
629 	int			mq_freeze_depth;
630 
631 #ifdef CONFIG_BLK_DEV_THROTTLING
632 	/* Throttle data */
633 	struct throtl_data *td;
634 #endif
635 	struct rcu_head		rcu_head;
636 #ifdef CONFIG_LOCKDEP
637 	struct task_struct	*mq_freeze_owner;
638 	int			mq_freeze_owner_depth;
639 	/*
640 	 * Records disk & queue state in current context, used in unfreeze
641 	 * queue
642 	 */
643 	bool			mq_freeze_disk_dead;
644 	bool			mq_freeze_queue_dying;
645 #endif
646 	wait_queue_head_t	mq_freeze_wq;
647 	/*
648 	 * Protect concurrent access to q_usage_counter by
649 	 * percpu_ref_kill() and percpu_ref_reinit().
650 	 */
651 	struct mutex		mq_freeze_lock;
652 
653 	struct blk_mq_tag_set	*tag_set;
654 	struct list_head	tag_set_list;
655 
656 	struct dentry		*debugfs_dir;
657 	struct dentry		*sched_debugfs_dir;
658 	struct dentry		*rqos_debugfs_dir;
659 	/*
660 	 * Serializes all debugfs metadata operations using the above dentries.
661 	 */
662 	struct mutex		debugfs_mutex;
663 };
664 
665 /* Keep blk_queue_flag_name[] in sync with the definitions below */
666 enum {
667 	QUEUE_FLAG_DYING,		/* queue being torn down */
668 	QUEUE_FLAG_NOMERGES,		/* disable merge attempts */
669 	QUEUE_FLAG_SAME_COMP,		/* complete on same CPU-group */
670 	QUEUE_FLAG_FAIL_IO,		/* fake timeout */
671 	QUEUE_FLAG_NOXMERGES,		/* No extended merges */
672 	QUEUE_FLAG_SAME_FORCE,		/* force complete on same CPU */
673 	QUEUE_FLAG_INIT_DONE,		/* queue is initialized */
674 	QUEUE_FLAG_STATS,		/* track IO start and completion times */
675 	QUEUE_FLAG_REGISTERED,		/* queue has been registered to a disk */
676 	QUEUE_FLAG_QUIESCED,		/* queue has been quiesced */
677 	QUEUE_FLAG_RQ_ALLOC_TIME,	/* record rq->alloc_time_ns */
678 	QUEUE_FLAG_HCTX_ACTIVE,		/* at least one blk-mq hctx is active */
679 	QUEUE_FLAG_SQ_SCHED,		/* single queue style io dispatch */
680 	QUEUE_FLAG_DISABLE_WBT_DEF,	/* for sched to disable/enable wbt */
681 	QUEUE_FLAG_NO_ELV_SWITCH,	/* a switch is running or disk is going away */
682 	QUEUE_FLAG_QOS_ENABLED,		/* qos is enabled */
683 	QUEUE_FLAG_BIO_ISSUE_TIME,	/* record bio->issue_time_ns */
684 	QUEUE_FLAG_ZONED_QD1_WRITES,	/* Limit zoned devices writes to QD=1 */
685 	QUEUE_FLAG_MAX
686 };
687 
688 #define QUEUE_FLAG_MQ_DEFAULT	(1UL << QUEUE_FLAG_SAME_COMP)
689 
690 void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
691 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
692 
693 #define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
694 #define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
695 #define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
696 #define blk_queue_noxmerges(q)	\
697 	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
698 #define blk_queue_rot(q)	((q)->limits.features & BLK_FEAT_ROTATIONAL)
699 #define blk_queue_io_stat(q)	((q)->limits.features & BLK_FEAT_IO_STAT)
700 #define blk_queue_passthrough_stat(q)	\
701 	((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH)
702 #define blk_queue_dax(q)	((q)->limits.features & BLK_FEAT_DAX)
703 #define blk_queue_pci_p2pdma(q)	((q)->limits.features & BLK_FEAT_PCI_P2PDMA)
704 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
705 #define blk_queue_rq_alloc_time(q)	\
706 	test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
707 #else
708 #define blk_queue_rq_alloc_time(q)	false
709 #endif
710 
711 #define blk_noretry_request(rq) \
712 	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
713 			     REQ_FAILFAST_DRIVER))
714 #define blk_queue_quiesced(q)	test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
715 #define blk_queue_pm_only(q)	atomic_read(&(q)->pm_only)
716 #define blk_queue_registered(q)	test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
717 #define blk_queue_sq_sched(q)	test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
718 #define blk_queue_skip_tagset_quiesce(q) \
719 	((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE)
720 #define blk_queue_disable_wbt(q)	\
721 	test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags)
722 #define blk_queue_zoned_qd1_writes(q)	\
723 	test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags)
724 
725 extern void blk_set_pm_only(struct request_queue *q);
726 extern void blk_clear_pm_only(struct request_queue *q);
727 
728 #define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)
729 
730 #define dma_map_bvec(dev, bv, dir, attrs) \
731 	dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
732 	(dir), (attrs))
733 
734 static inline bool queue_is_mq(struct request_queue *q)
735 {
736 	return q->mq_ops;
737 }
738 
739 #ifdef CONFIG_PM
740 static inline enum rpm_status queue_rpm_status(struct request_queue *q)
741 {
742 	return q->rpm_status;
743 }
744 #else
745 static inline enum rpm_status queue_rpm_status(struct request_queue *q)
746 {
747 	return RPM_ACTIVE;
748 }
749 #endif
750 
751 static inline bool blk_queue_is_zoned(struct request_queue *q)
752 {
753 	return IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
754 		(q->limits.features & BLK_FEAT_ZONED);
755 }
756 
757 static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
758 {
759 	if (!blk_queue_is_zoned(disk->queue))
760 		return 0;
761 	return sector >> ilog2(disk->queue->limits.chunk_sectors);
762 }
763 
764 static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
765 {
766 	return bdev->bd_disk->queue->limits.max_open_zones;
767 }
768 
769 static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
770 {
771 	return bdev->bd_disk->queue->limits.max_active_zones;
772 }
773 
774 static inline unsigned int blk_queue_depth(struct request_queue *q)
775 {
776 	if (q->queue_depth)
777 		return q->queue_depth;
778 
779 	return q->nr_requests;
780 }
781 
782 /*
783  * default timeout for SG_IO if none specified
784  */
785 #define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
786 #define BLK_MIN_SG_TIMEOUT	(7 * HZ)
787 
788 /* This should not be used directly - use rq_for_each_segment */
789 #define for_each_bio(_bio)		\
790 	for (; _bio; _bio = _bio->bi_next)
791 
792 int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk,
793 				 const struct attribute_group **groups,
794 				 struct fwnode_handle *fwnode);
795 int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
796 				 const struct attribute_group **groups);
797 static inline int __must_check add_disk(struct gendisk *disk)
798 {
799 	return device_add_disk(NULL, disk, NULL);
800 }
801 void del_gendisk(struct gendisk *gp);
802 void invalidate_disk(struct gendisk *disk);
803 void set_disk_ro(struct gendisk *disk, bool read_only);
804 void disk_uevent(struct gendisk *disk, enum kobject_action action);
805 
806 static inline u8 bdev_partno(const struct block_device *bdev)
807 {
808 	return atomic_read(&bdev->__bd_flags) & BD_PARTNO;
809 }
810 
811 static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag)
812 {
813 	return atomic_read(&bdev->__bd_flags) & flag;
814 }
815 
816 static inline void bdev_set_flag(struct block_device *bdev, unsigned flag)
817 {
818 	atomic_or(flag, &bdev->__bd_flags);
819 }
820 
821 static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag)
822 {
823 	atomic_andnot(flag, &bdev->__bd_flags);
824 }
825 
826 static inline bool get_disk_ro(struct gendisk *disk)
827 {
828 	return bdev_test_flag(disk->part0, BD_READ_ONLY) ||
829 		test_bit(GD_READ_ONLY, &disk->state);
830 }
831 
832 static inline bool bdev_read_only(struct block_device *bdev)
833 {
834 	return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk);
835 }
836 
837 bool set_capacity_and_notify(struct gendisk *disk, sector_t size);
838 void disk_force_media_change(struct gendisk *disk);
839 void bdev_mark_dead(struct block_device *bdev, bool surprise);
840 
841 void add_disk_randomness(struct gendisk *disk) __latent_entropy;
842 void rand_initialize_disk(struct gendisk *disk);
843 
844 static inline sector_t get_start_sect(struct block_device *bdev)
845 {
846 	return bdev->bd_start_sect;
847 }
848 
849 static inline sector_t bdev_nr_sectors(struct block_device *bdev)
850 {
851 	return bdev->bd_nr_sectors;
852 }
853 
854 static inline loff_t bdev_nr_bytes(struct block_device *bdev)
855 {
856 	return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT;
857 }
858 
859 static inline sector_t get_capacity(struct gendisk *disk)
860 {
861 	return bdev_nr_sectors(disk->part0);
862 }
863 
864 static inline u64 sb_bdev_nr_blocks(struct super_block *sb)
865 {
866 	return bdev_nr_sectors(sb->s_bdev) >>
867 		(sb->s_blocksize_bits - SECTOR_SHIFT);
868 }
869 
870 #ifdef CONFIG_BLK_DEV_ZONED
871 static inline unsigned int disk_nr_zones(struct gendisk *disk)
872 {
873 	return disk->nr_zones;
874 }
875 
876 /**
877  * bio_needs_zone_write_plugging - Check if a BIO needs to be handled with zone
878  *				   write plugging
879  * @bio: The BIO being submitted
880  *
881  * Return true whenever @bio execution needs to be handled through zone
882  * write plugging (using blk_zone_plug_bio()). Return false otherwise.
883  */
884 static inline bool bio_needs_zone_write_plugging(struct bio *bio)
885 {
886 	enum req_op op = bio_op(bio);
887 
888 	/*
889 	 * Only zoned block devices have a zone write plug hash table. But not
890 	 * all of them have one (e.g. DM devices may not need one).
891 	 */
892 	if (!bio->bi_bdev->bd_disk->zone_wplugs_hash)
893 		return false;
894 
895 	/* Only write operations need zone write plugging. */
896 	if (!op_is_write(op))
897 		return false;
898 
899 	/* Ignore empty flush */
900 	if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
901 		return false;
902 
903 	/* Ignore BIOs that already have been handled by zone write plugging. */
904 	if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING))
905 		return false;
906 
907 	/*
908 	 * All zone write operations must be handled through zone write plugging
909 	 * using blk_zone_plug_bio().
910 	 */
911 	switch (op) {
912 	case REQ_OP_ZONE_APPEND:
913 	case REQ_OP_WRITE:
914 	case REQ_OP_WRITE_ZEROES:
915 	case REQ_OP_ZONE_FINISH:
916 	case REQ_OP_ZONE_RESET:
917 	case REQ_OP_ZONE_RESET_ALL:
918 		return true;
919 	default:
920 		return false;
921 	}
922 }
923 
924 bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs);
925 
926 /**
927  * disk_zone_capacity - returns the zone capacity of zone containing @sector
928  * @disk:	disk to work with
929  * @sector:	sector number within the querying zone
930  *
931  * Returns the zone capacity of a zone containing @sector. @sector can be any
932  * sector in the zone.
933  */
934 static inline unsigned int disk_zone_capacity(struct gendisk *disk,
935 					      sector_t sector)
936 {
937 	sector_t zone_sectors = disk->queue->limits.chunk_sectors;
938 
939 	if (sector + zone_sectors >= get_capacity(disk))
940 		return disk->last_zone_capacity;
941 	return disk->zone_capacity;
942 }
943 static inline unsigned int bdev_zone_capacity(struct block_device *bdev,
944 					      sector_t pos)
945 {
946 	return disk_zone_capacity(bdev->bd_disk, pos);
947 }
948 
949 bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector);
950 
951 #else /* CONFIG_BLK_DEV_ZONED */
952 static inline unsigned int disk_nr_zones(struct gendisk *disk)
953 {
954 	return 0;
955 }
956 
957 static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector)
958 {
959 	return false;
960 }
961 
962 static inline bool bio_needs_zone_write_plugging(struct bio *bio)
963 {
964 	return false;
965 }
966 
967 static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs)
968 {
969 	return false;
970 }
971 #endif /* CONFIG_BLK_DEV_ZONED */
972 
973 static inline unsigned int bdev_nr_zones(struct block_device *bdev)
974 {
975 	return disk_nr_zones(bdev->bd_disk);
976 }
977 
978 int bdev_disk_changed(struct gendisk *disk, bool invalidate);
979 
980 void put_disk(struct gendisk *disk);
981 struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node,
982 		struct lock_class_key *lkclass);
983 
984 /**
985  * blk_alloc_disk - allocate a gendisk structure
986  * @lim: queue limits to be used for this disk.
987  * @node_id: numa node to allocate on
988  *
989  * Allocate and pre-initialize a gendisk structure for use with BIO based
990  * drivers.
991  *
992  * Returns an ERR_PTR on error, else the allocated disk.
993  *
994  * Context: can sleep
995  */
996 #define blk_alloc_disk(lim, node_id)					\
997 ({									\
998 	static struct lock_class_key __key;				\
999 									\
1000 	__blk_alloc_disk(lim, node_id, &__key);				\
1001 })
1002 
1003 int __register_blkdev(unsigned int major, const char *name,
1004 		void (*probe)(dev_t devt));
1005 #define register_blkdev(major, name) \
1006 	__register_blkdev(major, name, NULL)
1007 void unregister_blkdev(unsigned int major, const char *name);
1008 
1009 bool disk_check_media_change(struct gendisk *disk);
1010 void set_capacity(struct gendisk *disk, sector_t size);
1011 
1012 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
1013 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk);
1014 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk);
1015 #else
1016 static inline int bd_link_disk_holder(struct block_device *bdev,
1017 				      struct gendisk *disk)
1018 {
1019 	return 0;
1020 }
1021 static inline void bd_unlink_disk_holder(struct block_device *bdev,
1022 					 struct gendisk *disk)
1023 {
1024 }
1025 #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */
1026 
1027 dev_t part_devt(struct gendisk *disk, u8 partno);
1028 void inc_diskseq(struct gendisk *disk);
1029 void blk_request_module(dev_t devt);
1030 
1031 extern int blk_register_queue(struct gendisk *disk);
1032 extern void blk_unregister_queue(struct gendisk *disk);
1033 void submit_bio_noacct(struct bio *bio);
1034 struct bio *bio_split_to_limits(struct bio *bio);
1035 struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors,
1036 				    struct bio_set *bs);
1037 
1038 extern int blk_lld_busy(struct request_queue *q);
1039 extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
1040 extern void blk_queue_exit(struct request_queue *q);
1041 extern void blk_sync_queue(struct request_queue *q);
1042 
1043 /* Convert a request operation REQ_OP_name into the string "name" */
1044 extern const char *blk_op_str(enum req_op op);
1045 
1046 int blk_status_to_errno(blk_status_t status);
1047 blk_status_t errno_to_blk_status(int errno);
1048 
1049 /* only poll the hardware once, don't continue until a completion was found */
1050 #define BLK_POLL_ONESHOT		(1 << 0)
1051 int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
1052 int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
1053 			unsigned int flags);
1054 
1055 static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
1056 {
1057 	return bdev->bd_queue;	/* this is never NULL */
1058 }
1059 
1060 /* Convert a zone condition BLK_ZONE_COND_name into the string "name" */
1061 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
1062 
1063 static inline unsigned int bio_zone_no(struct bio *bio)
1064 {
1065 	return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
1066 }
1067 
1068 static inline bool bio_straddles_zones(struct bio *bio)
1069 {
1070 	return bio_sectors(bio) &&
1071 		bio_zone_no(bio) !=
1072 		disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1);
1073 }
1074 
1075 /*
1076  * Return how much within the boundary is left to be used for I/O at a given
1077  * offset.
1078  */
1079 static inline unsigned int blk_boundary_sectors_left(sector_t offset,
1080 		unsigned int boundary_sectors)
1081 {
1082 	if (unlikely(!is_power_of_2(boundary_sectors)))
1083 		return boundary_sectors - sector_div(offset, boundary_sectors);
1084 	return boundary_sectors - (offset & (boundary_sectors - 1));
1085 }
1086 
1087 /**
1088  * queue_limits_start_update - start an atomic update of queue limits
1089  * @q:		queue to update
1090  *
1091  * This functions starts an atomic update of the queue limits.  It takes a lock
1092  * to prevent other updates and returns a snapshot of the current limits that
1093  * the caller can modify.  The caller must call queue_limits_commit_update()
1094  * to finish the update.
1095  *
1096  * Context: process context.
1097  */
1098 static inline struct queue_limits
1099 queue_limits_start_update(struct request_queue *q)
1100 	__acquires(&q->limits_lock)
1101 {
1102 	mutex_lock(&q->limits_lock);
1103 	return q->limits;
1104 }
1105 int queue_limits_commit_update_frozen(struct request_queue *q,
1106 		struct queue_limits *lim) __releases(&q->limits_lock);
1107 int queue_limits_commit_update(struct request_queue *q,
1108 		struct queue_limits *lim) __releases(&q->limits_lock);
1109 int queue_limits_set(struct request_queue *q, struct queue_limits *lim)
1110 	__must_not_hold(&q->limits_lock);
1111 int blk_validate_limits(struct queue_limits *lim);
1112 
1113 /**
1114  * queue_limits_cancel_update - cancel an atomic update of queue limits
1115  * @q:		queue to update
1116  *
1117  * This functions cancels an atomic update of the queue limits started by
1118  * queue_limits_start_update() and should be used when an error occurs after
1119  * starting update.
1120  */
1121 static inline void queue_limits_cancel_update(struct request_queue *q)
1122 	__releases(&q->limits_lock)
1123 {
1124 	mutex_unlock(&q->limits_lock);
1125 }
1126 
1127 /*
1128  * These helpers are for drivers that have sloppy feature negotiation and might
1129  * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O
1130  * completion handler when the device returned an indicator that the respective
1131  * feature is not actually supported.  They are racy and the driver needs to
1132  * cope with that.  Try to avoid this scheme if you can.
1133  */
1134 static inline void blk_queue_disable_discard(struct request_queue *q)
1135 {
1136 	q->limits.max_discard_sectors = 0;
1137 }
1138 
1139 static inline void blk_queue_disable_secure_erase(struct request_queue *q)
1140 {
1141 	q->limits.max_secure_erase_sectors = 0;
1142 }
1143 
1144 static inline void blk_queue_disable_write_zeroes(struct request_queue *q)
1145 {
1146 	q->limits.max_write_zeroes_sectors = 0;
1147 	q->limits.max_wzeroes_unmap_sectors = 0;
1148 }
1149 
1150 /*
1151  * Access functions for manipulating queue properties
1152  */
1153 extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1154 extern void blk_set_stacking_limits(struct queue_limits *lim);
1155 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
1156 			    sector_t offset);
1157 void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev,
1158 		sector_t offset, const char *pfx);
1159 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1160 
1161 struct blk_independent_access_ranges *
1162 disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges);
1163 void disk_set_independent_access_ranges(struct gendisk *disk,
1164 				struct blk_independent_access_ranges *iars);
1165 
1166 bool __must_check blk_get_queue(struct request_queue *);
1167 extern void blk_put_queue(struct request_queue *);
1168 
1169 void blk_mark_disk_dead(struct gendisk *disk);
1170 
1171 #ifdef CONFIG_BLOCK
1172 int blkdev_issue_flush(struct block_device *bdev);
1173 long nr_blockdev_pages(void);
1174 #else /* CONFIG_BLOCK */
1175 static inline int blkdev_issue_flush(struct block_device *bdev)
1176 {
1177 	return 0;
1178 }
1179 
1180 static inline long nr_blockdev_pages(void)
1181 {
1182 	return 0;
1183 }
1184 #endif /* CONFIG_BLOCK */
1185 
1186 extern void blk_io_schedule(void);
1187 
1188 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1189 		sector_t nr_sects, gfp_t gfp_mask);
1190 void __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1191 		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop);
1192 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
1193 		sector_t nr_sects, gfp_t gfp);
1194 
1195 #define BLKDEV_ZERO_NOUNMAP	(1 << 0)  /* do not free blocks */
1196 #define BLKDEV_ZERO_NOFALLBACK	(1 << 1)  /* don't write explicit zeroes */
1197 #define BLKDEV_ZERO_KILLABLE	(1 << 2)  /* interruptible by fatal signals */
1198 
1199 extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1200 		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1201 		unsigned flags);
1202 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1203 		sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1204 
1205 static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1206 		sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1207 {
1208 	return blkdev_issue_discard(sb->s_bdev,
1209 				    block << (sb->s_blocksize_bits -
1210 					      SECTOR_SHIFT),
1211 				    nr_blocks << (sb->s_blocksize_bits -
1212 						  SECTOR_SHIFT),
1213 				    gfp_mask);
1214 }
1215 static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1216 		sector_t nr_blocks, gfp_t gfp_mask)
1217 {
1218 	return blkdev_issue_zeroout(sb->s_bdev,
1219 				    block << (sb->s_blocksize_bits -
1220 					      SECTOR_SHIFT),
1221 				    nr_blocks << (sb->s_blocksize_bits -
1222 						  SECTOR_SHIFT),
1223 				    gfp_mask, 0);
1224 }
1225 
1226 static inline bool bdev_is_partition(struct block_device *bdev)
1227 {
1228 	return bdev_partno(bdev) != 0;
1229 }
1230 
1231 enum blk_default_limits {
1232 	BLK_MAX_SEGMENTS	= 128,
1233 	BLK_SAFE_MAX_SECTORS	= 255,
1234 	BLK_MAX_SEGMENT_SIZE	= 65536,
1235 	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
1236 };
1237 
1238 static inline struct queue_limits *bdev_limits(struct block_device *bdev)
1239 {
1240 	return &bdev_get_queue(bdev)->limits;
1241 }
1242 
1243 static inline unsigned long queue_segment_boundary(const struct request_queue *q)
1244 {
1245 	return q->limits.seg_boundary_mask;
1246 }
1247 
1248 static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1249 {
1250 	return q->limits.virt_boundary_mask;
1251 }
1252 
1253 static inline unsigned int queue_max_sectors(const struct request_queue *q)
1254 {
1255 	return q->limits.max_sectors;
1256 }
1257 
1258 static inline unsigned int queue_max_bytes(struct request_queue *q)
1259 {
1260 	return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9;
1261 }
1262 
1263 static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1264 {
1265 	return q->limits.max_hw_sectors;
1266 }
1267 
1268 static inline unsigned short queue_max_segments(const struct request_queue *q)
1269 {
1270 	return q->limits.max_segments;
1271 }
1272 
1273 static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1274 {
1275 	return q->limits.max_discard_segments;
1276 }
1277 
1278 static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1279 {
1280 	return q->limits.max_segment_size;
1281 }
1282 
1283 static inline bool queue_emulates_zone_append(struct request_queue *q)
1284 {
1285 	return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors;
1286 }
1287 
1288 static inline bool bdev_emulates_zone_append(struct block_device *bdev)
1289 {
1290 	return queue_emulates_zone_append(bdev_get_queue(bdev));
1291 }
1292 
1293 static inline unsigned int
1294 bdev_max_zone_append_sectors(struct block_device *bdev)
1295 {
1296 	return bdev_limits(bdev)->max_zone_append_sectors;
1297 }
1298 
1299 static inline unsigned int bdev_max_segments(struct block_device *bdev)
1300 {
1301 	return queue_max_segments(bdev_get_queue(bdev));
1302 }
1303 
1304 static inline unsigned short bdev_max_write_streams(struct block_device *bdev)
1305 {
1306 	if (bdev_is_partition(bdev))
1307 		return 0;
1308 	return bdev_limits(bdev)->max_write_streams;
1309 }
1310 
1311 static inline unsigned queue_logical_block_size(const struct request_queue *q)
1312 {
1313 	return q->limits.logical_block_size;
1314 }
1315 
1316 static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
1317 {
1318 	return queue_logical_block_size(bdev_get_queue(bdev));
1319 }
1320 
1321 static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1322 {
1323 	return q->limits.physical_block_size;
1324 }
1325 
1326 static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1327 {
1328 	return queue_physical_block_size(bdev_get_queue(bdev));
1329 }
1330 
1331 static inline unsigned int queue_io_min(const struct request_queue *q)
1332 {
1333 	return q->limits.io_min;
1334 }
1335 
1336 static inline unsigned int bdev_io_min(struct block_device *bdev)
1337 {
1338 	return queue_io_min(bdev_get_queue(bdev));
1339 }
1340 
1341 static inline unsigned int queue_io_opt(const struct request_queue *q)
1342 {
1343 	return q->limits.io_opt;
1344 }
1345 
1346 static inline unsigned int bdev_io_opt(struct block_device *bdev)
1347 {
1348 	return queue_io_opt(bdev_get_queue(bdev));
1349 }
1350 
1351 static inline unsigned int
1352 queue_zone_write_granularity(const struct request_queue *q)
1353 {
1354 	return q->limits.zone_write_granularity;
1355 }
1356 
1357 static inline unsigned int
1358 bdev_zone_write_granularity(struct block_device *bdev)
1359 {
1360 	return queue_zone_write_granularity(bdev_get_queue(bdev));
1361 }
1362 
1363 int bdev_alignment_offset(struct block_device *bdev);
1364 unsigned int bdev_discard_alignment(struct block_device *bdev);
1365 
1366 static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev)
1367 {
1368 	return bdev_limits(bdev)->max_discard_sectors;
1369 }
1370 
1371 static inline unsigned int bdev_discard_granularity(struct block_device *bdev)
1372 {
1373 	return bdev_limits(bdev)->discard_granularity;
1374 }
1375 
1376 static inline unsigned int
1377 bdev_max_secure_erase_sectors(struct block_device *bdev)
1378 {
1379 	return bdev_limits(bdev)->max_secure_erase_sectors;
1380 }
1381 
1382 static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1383 {
1384 	return bdev_limits(bdev)->max_write_zeroes_sectors;
1385 }
1386 
1387 static inline unsigned int
1388 bdev_write_zeroes_unmap_sectors(struct block_device *bdev)
1389 {
1390 	return bdev_limits(bdev)->max_wzeroes_unmap_sectors;
1391 }
1392 
1393 static inline bool bdev_rot(struct block_device *bdev)
1394 {
1395 	return blk_queue_rot(bdev_get_queue(bdev));
1396 }
1397 
1398 static inline bool bdev_synchronous(struct block_device *bdev)
1399 {
1400 	return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
1401 }
1402 
1403 static inline bool bdev_has_integrity_csum(struct block_device *bdev)
1404 {
1405 	struct queue_limits *lim = bdev_limits(bdev);
1406 
1407 	return IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1408 		lim->integrity.csum_type != BLK_INTEGRITY_CSUM_NONE;
1409 }
1410 
1411 static inline bool bdev_stable_writes(struct block_device *bdev)
1412 {
1413 	return bdev_has_integrity_csum(bdev) ||
1414 		(bdev_limits(bdev)->features & BLK_FEAT_STABLE_WRITES);
1415 }
1416 
1417 static inline bool blk_queue_write_cache(struct request_queue *q)
1418 {
1419 	return (q->limits.features & BLK_FEAT_WRITE_CACHE) &&
1420 		!(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED);
1421 }
1422 
1423 static inline bool bdev_write_cache(struct block_device *bdev)
1424 {
1425 	return blk_queue_write_cache(bdev_get_queue(bdev));
1426 }
1427 
1428 static inline bool bdev_fua(struct block_device *bdev)
1429 {
1430 	return bdev_limits(bdev)->features & BLK_FEAT_FUA;
1431 }
1432 
1433 static inline bool bdev_nowait(struct block_device *bdev)
1434 {
1435 	return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT;
1436 }
1437 
1438 static inline bool bdev_is_zoned(struct block_device *bdev)
1439 {
1440 	return blk_queue_is_zoned(bdev_get_queue(bdev));
1441 }
1442 
1443 static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec)
1444 {
1445 	return disk_zone_no(bdev->bd_disk, sec);
1446 }
1447 
1448 static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1449 {
1450 	struct request_queue *q = bdev_get_queue(bdev);
1451 
1452 	if (!blk_queue_is_zoned(q))
1453 		return 0;
1454 	return q->limits.chunk_sectors;
1455 }
1456 
1457 static inline sector_t bdev_zone_start(struct block_device *bdev,
1458 				       sector_t sector)
1459 {
1460 	return sector & ~(bdev_zone_sectors(bdev) - 1);
1461 }
1462 
1463 static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev,
1464 						   sector_t sector)
1465 {
1466 	return sector & (bdev_zone_sectors(bdev) - 1);
1467 }
1468 
1469 static inline sector_t bio_offset_from_zone_start(struct bio *bio)
1470 {
1471 	return bdev_offset_from_zone_start(bio->bi_bdev,
1472 					   bio->bi_iter.bi_sector);
1473 }
1474 
1475 static inline bool bdev_is_zone_start(struct block_device *bdev,
1476 				      sector_t sector)
1477 {
1478 	return bdev_offset_from_zone_start(bdev, sector) == 0;
1479 }
1480 
1481 /* Check whether @sector is a multiple of the zone size. */
1482 static inline bool bdev_is_zone_aligned(struct block_device *bdev,
1483 					sector_t sector)
1484 {
1485 	return bdev_is_zone_start(bdev, sector);
1486 }
1487 
1488 int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector,
1489 			   sector_t nr_sects, gfp_t gfp_mask);
1490 
1491 static inline unsigned int queue_dma_alignment(const struct request_queue *q)
1492 {
1493 	return q->limits.dma_alignment;
1494 }
1495 
1496 static inline unsigned int
1497 queue_atomic_write_unit_max_bytes(const struct request_queue *q)
1498 {
1499 	return q->limits.atomic_write_unit_max;
1500 }
1501 
1502 static inline unsigned int
1503 queue_atomic_write_unit_min_bytes(const struct request_queue *q)
1504 {
1505 	return q->limits.atomic_write_unit_min;
1506 }
1507 
1508 static inline unsigned int
1509 queue_atomic_write_boundary_bytes(const struct request_queue *q)
1510 {
1511 	return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT;
1512 }
1513 
1514 static inline unsigned int
1515 queue_atomic_write_max_bytes(const struct request_queue *q)
1516 {
1517 	return q->limits.atomic_write_max_sectors << SECTOR_SHIFT;
1518 }
1519 
1520 static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
1521 {
1522 	return queue_dma_alignment(bdev_get_queue(bdev));
1523 }
1524 
1525 static inline unsigned int
1526 blk_lim_dma_alignment_and_pad(struct queue_limits *lim)
1527 {
1528 	return lim->dma_alignment | lim->dma_pad_mask;
1529 }
1530 
1531 static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr,
1532 				 unsigned int len)
1533 {
1534 	unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits);
1535 
1536 	return !(addr & alignment) && !(len & alignment);
1537 }
1538 
1539 /* assumes size > 256 */
1540 static inline unsigned int blksize_bits(unsigned int size)
1541 {
1542 	return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT;
1543 }
1544 
1545 int kblockd_schedule_work(struct work_struct *work);
1546 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1547 
1548 #define MODULE_ALIAS_BLOCKDEV(major,minor) \
1549 	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1550 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1551 	MODULE_ALIAS("block-major-" __stringify(major) "-*")
1552 
1553 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1554 
1555 bool blk_crypto_register(struct blk_crypto_profile *profile,
1556 			 struct request_queue *q);
1557 
1558 #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1559 
1560 static inline bool blk_crypto_register(struct blk_crypto_profile *profile,
1561 				       struct request_queue *q)
1562 {
1563 	return true;
1564 }
1565 
1566 #endif /* CONFIG_BLK_INLINE_ENCRYPTION */
1567 
1568 enum blk_unique_id {
1569 	/* these match the Designator Types specified in SPC */
1570 	BLK_UID_T10	= 1,
1571 	BLK_UID_EUI64	= 2,
1572 	BLK_UID_NAA	= 3,
1573 };
1574 
1575 struct block_device_operations {
1576 	void (*submit_bio)(struct bio *bio);
1577 	int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob,
1578 			unsigned int flags);
1579 	int (*open)(struct gendisk *disk, blk_mode_t mode);
1580 	void (*release)(struct gendisk *disk);
1581 	int (*ioctl)(struct block_device *bdev, blk_mode_t mode,
1582 			unsigned cmd, unsigned long arg);
1583 	int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode,
1584 			unsigned cmd, unsigned long arg);
1585 	unsigned int (*check_events) (struct gendisk *disk,
1586 				      unsigned int clearing);
1587 	void (*unlock_native_capacity) (struct gendisk *);
1588 	int (*getgeo)(struct gendisk *, struct hd_geometry *);
1589 	int (*set_read_only)(struct block_device *bdev, bool ro);
1590 	void (*free_disk)(struct gendisk *disk);
1591 	/* this callback is with swap_lock and sometimes page table lock held */
1592 	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1593 	int (*report_zones)(struct gendisk *, sector_t sector,
1594 			    unsigned int nr_zones,
1595 			    struct blk_report_zones_args *args);
1596 	char *(*devnode)(struct gendisk *disk, umode_t *mode);
1597 	/* returns the length of the identifier or a negative errno: */
1598 	int (*get_unique_id)(struct gendisk *disk, u8 id[16],
1599 			enum blk_unique_id id_type);
1600 	int (*init_dma_buf_io_ctx)(struct block_device *,
1601 				   struct dma_buf_io_ctx *);
1602 	struct module *owner;
1603 	const struct pr_ops *pr_ops;
1604 
1605 	/*
1606 	 * Special callback for probing GPT entry at a given sector.
1607 	 * Needed by Android devices, used by GPT scanner and MMC blk
1608 	 * driver.
1609 	 */
1610 	int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector);
1611 };
1612 
1613 #ifdef CONFIG_COMPAT
1614 extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t,
1615 				      unsigned int, unsigned long);
1616 #else
1617 #define blkdev_compat_ptr_ioctl NULL
1618 #endif
1619 
1620 static inline void blk_wake_io_task(struct task_struct *waiter)
1621 {
1622 	/*
1623 	 * If we're polling, the task itself is doing the completions. For
1624 	 * that case, we don't need to signal a wakeup, it's enough to just
1625 	 * mark us as RUNNING.
1626 	 */
1627 	if (waiter == current)
1628 		__set_current_state(TASK_RUNNING);
1629 	else
1630 		wake_up_process(waiter);
1631 }
1632 
1633 unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
1634 				 unsigned long start_time);
1635 void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
1636 		      unsigned int sectors, unsigned long start_time);
1637 
1638 unsigned long bio_start_io_acct(struct bio *bio);
1639 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1640 		struct block_device *orig_bdev);
1641 
1642 /**
1643  * bio_end_io_acct - end I/O accounting for bio based drivers
1644  * @bio:	bio to end account for
1645  * @start_time:	start time returned by bio_start_io_acct()
1646  */
1647 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
1648 {
1649 	return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
1650 }
1651 
1652 int bdev_validate_blocksize(struct block_device *bdev, int block_size);
1653 int set_blocksize(struct file *file, int size);
1654 
1655 int lookup_bdev(const char *pathname, dev_t *dev);
1656 
1657 void blkdev_show(struct seq_file *seqf, off_t offset);
1658 
1659 #define BDEVNAME_SIZE	32	/* Largest string for a blockdev identifier */
1660 #define BDEVT_SIZE	10	/* Largest string for MAJ:MIN for blkdev */
1661 #ifdef CONFIG_BLOCK
1662 #define BLKDEV_MAJOR_MAX	512
1663 #else
1664 #define BLKDEV_MAJOR_MAX	0
1665 #endif
1666 
1667 struct blk_holder_ops {
1668 	void (*mark_dead)(struct block_device *bdev, bool surprise)
1669 		__releases(&bdev->bd_holder_lock);
1670 
1671 	/*
1672 	 * Sync the file system mounted on the block device.
1673 	 */
1674 	void (*sync)(struct block_device *bdev)
1675 		__releases(&bdev->bd_holder_lock);
1676 
1677 	/*
1678 	 * Freeze the file system mounted on the block device.
1679 	 */
1680 	int (*freeze)(struct block_device *bdev)
1681 		__releases(&bdev->bd_holder_lock);
1682 
1683 	/*
1684 	 * Thaw the file system mounted on the block device.
1685 	 */
1686 	int (*thaw)(struct block_device *bdev)
1687 		__releases(&bdev->bd_holder_lock);
1688 };
1689 
1690 /*
1691  * Return the correct open flags for blkdev_get_by_* for super block flags
1692  * as stored in sb->s_flags.
1693  */
1694 #define sb_open_mode(flags) \
1695 	(BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \
1696 	 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
1697 
1698 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
1699 		const struct blk_holder_ops *hops);
1700 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,
1701 		void *holder, const struct blk_holder_ops *hops);
1702 int bd_prepare_to_claim(struct block_device *bdev, void *holder,
1703 		const struct blk_holder_ops *hops);
1704 void bd_abort_claiming(struct block_device *bdev, void *holder);
1705 
1706 struct block_device *I_BDEV(struct inode *inode);
1707 struct block_device *file_bdev(struct file *bdev_file);
1708 bool disk_live(struct gendisk *disk);
1709 unsigned int block_size(struct block_device *bdev);
1710 
1711 #ifdef CONFIG_BLOCK
1712 void invalidate_bdev(struct block_device *bdev);
1713 int sync_blockdev(struct block_device *bdev);
1714 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
1715 int sync_blockdev_nowait(struct block_device *bdev);
1716 void sync_bdevs(bool wait);
1717 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask);
1718 void printk_all_partitions(void);
1719 int __init early_lookup_bdev(const char *pathname, dev_t *dev);
1720 #else
1721 static inline void invalidate_bdev(struct block_device *bdev)
1722 {
1723 }
1724 static inline int sync_blockdev(struct block_device *bdev)
1725 {
1726 	return 0;
1727 }
1728 static inline int sync_blockdev_nowait(struct block_device *bdev)
1729 {
1730 	return 0;
1731 }
1732 static inline void sync_bdevs(bool wait)
1733 {
1734 }
1735 static inline void bdev_statx(const struct path *path, struct kstat *stat,
1736 		u32 request_mask)
1737 {
1738 }
1739 static inline void printk_all_partitions(void)
1740 {
1741 }
1742 static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
1743 {
1744 	return -EINVAL;
1745 }
1746 #endif /* CONFIG_BLOCK */
1747 
1748 int bdev_freeze(struct block_device *bdev);
1749 int bdev_thaw(struct block_device *bdev);
1750 int bdev_deny_freeze(struct block_device *bdev);
1751 void bdev_allow_freeze(struct block_device *bdev);
1752 void bdev_fput(struct file *bdev_file);
1753 void bdev_yield_claim(struct file *bdev_file);
1754 
1755 struct io_comp_batch {
1756 	struct rq_list req_list;
1757 	bool need_ts;
1758 	void (*complete)(struct io_comp_batch *);
1759 	void *poll_ctx;
1760 };
1761 
1762 static inline bool blk_atomic_write_start_sect_aligned(sector_t sector,
1763 						struct queue_limits *limits)
1764 {
1765 	unsigned int alignment = max(limits->atomic_write_hw_unit_min,
1766 				limits->atomic_write_hw_boundary);
1767 
1768 	return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT);
1769 }
1770 
1771 static inline bool bdev_can_atomic_write(struct block_device *bdev)
1772 {
1773 	struct request_queue *bd_queue = bdev->bd_queue;
1774 	struct queue_limits *limits = &bd_queue->limits;
1775 
1776 	if (!limits->atomic_write_unit_min)
1777 		return false;
1778 
1779 	if (bdev_is_partition(bdev))
1780 		return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect,
1781 							limits);
1782 
1783 	return true;
1784 }
1785 
1786 static inline unsigned int
1787 bdev_atomic_write_unit_min_bytes(struct block_device *bdev)
1788 {
1789 	if (!bdev_can_atomic_write(bdev))
1790 		return 0;
1791 	return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev));
1792 }
1793 
1794 static inline unsigned int
1795 bdev_atomic_write_unit_max_bytes(struct block_device *bdev)
1796 {
1797 	if (!bdev_can_atomic_write(bdev))
1798 		return 0;
1799 	return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev));
1800 }
1801 
1802 static inline int bio_split_rw_at(struct bio *bio,
1803 		const struct queue_limits *lim,
1804 		unsigned *segs, unsigned max_bytes)
1805 {
1806 	return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment);
1807 }
1808 
1809 /*
1810  * Maximum contiguous integrity buffer allocation.
1811  */
1812 #define BLK_INTEGRITY_MAX_SIZE		SZ_2M
1813 
1814 /*
1815  * Maximum size of I/O that needs a block layer integrity buffer.  Limited
1816  * by the number of intervals for which we can fit the integrity buffer into
1817  * the buffer size.  Because the buffer is a single segment it is also limited
1818  * by the maximum segment size.
1819  */
1820 static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
1821 {
1822 	unsigned int buf_size = min_t(unsigned int, lim->max_segment_size,
1823 				      BLK_INTEGRITY_MAX_SIZE);
1824 
1825 	return min_t(u64, UINT_MAX,
1826 		(u64)(buf_size / lim->integrity.metadata_size) <<
1827 			lim->integrity.interval_exp);
1828 }
1829 
1830 #define DEFINE_IO_COMP_BATCH(name)	struct io_comp_batch name = { }
1831 
1832 #endif /* _LINUX_BLKDEV_H */
1833