xref: /linux/include/linux/blk_types.h (revision d88e44f43e939e6ac0ca8302c6f3c957b33aa5fe)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Block data types and constants.  Directly include this file only to
4  * break include dependency loop.
5  */
6 #ifndef __LINUX_BLK_TYPES_H
7 #define __LINUX_BLK_TYPES_H
8 
9 #include <linux/types.h>
10 #include <linux/bvec.h>
11 #include <linux/device.h>
12 #include <linux/ktime.h>
13 #include <linux/rw_hint.h>
14 
15 struct bio_set;
16 struct bio;
17 struct bio_integrity_payload;
18 struct page;
19 struct io_context;
20 struct cgroup_subsys_state;
21 typedef void (bio_end_io_t) (struct bio *);
22 struct bio_crypt_ctx;
23 
24 /*
25  * The basic unit of block I/O is a sector. It is used in a number of contexts
26  * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
27  * bytes. Variables of type sector_t represent an offset or size that is a
28  * multiple of 512 bytes. Hence these two constants.
29  */
30 #ifndef SECTOR_SHIFT
31 #define SECTOR_SHIFT 9
32 #endif
33 #ifndef SECTOR_SIZE
34 #define SECTOR_SIZE (1 << SECTOR_SHIFT)
35 #endif
36 
37 #define PAGE_SECTORS_SHIFT	(PAGE_SHIFT - SECTOR_SHIFT)
38 #define PAGE_SECTORS		(1 << PAGE_SECTORS_SHIFT)
39 #define SECTOR_MASK		(PAGE_SECTORS - 1)
40 
41 struct block_device {
42 	sector_t		bd_start_sect;
43 	sector_t		bd_nr_sectors;
44 	struct gendisk *	bd_disk;
45 	struct request_queue *	bd_queue;
46 	struct disk_stats __percpu *bd_stats;
47 	unsigned long		bd_stamp;
48 	atomic_t		__bd_flags;	// partition number + flags
49 #define BD_PARTNO		255	// lower 8 bits; assign-once
50 #define BD_READ_ONLY		(1u<<8) // read-only policy
51 #define BD_WRITE_HOLDER		(1u<<9)
52 #define BD_HAS_SUBMIT_BIO	(1u<<10)
53 #define BD_RO_WARNED		(1u<<11)
54 #ifdef CONFIG_FAIL_MAKE_REQUEST
55 #define BD_MAKE_IT_FAIL		(1u<<12)
56 #endif
57 	dev_t			bd_dev;
58 	struct address_space	*bd_mapping;	/* page cache */
59 
60 	atomic_t		bd_openers;
61 	spinlock_t		bd_size_lock; /* for bd_inode->i_size updates */
62 	void *			bd_claiming;
63 	void *			bd_holder;
64 	const struct blk_holder_ops *bd_holder_ops;
65 	struct mutex		bd_holder_lock;
66 	int			bd_holders;
67 	struct kobject		*bd_holder_dir;
68 
69 	atomic_t		bd_fsfreeze_count; /* >0 freeze requests, <0 freeze deniers */
70 	struct mutex		bd_fsfreeze_mutex; /* serialize freeze/thaw */
71 
72 	struct partition_meta_info *bd_meta_info;
73 	int			bd_writers;
74 #ifdef CONFIG_SECURITY
75 	void			*bd_security;
76 #endif
77 	/*
78 	 * keep this out-of-line as it's both big and not needed in the fast
79 	 * path
80 	 */
81 	struct device		bd_device;
82 } __randomize_layout;
83 
84 #define bdev_whole(_bdev) \
85 	((_bdev)->bd_disk->part0)
86 
87 #define dev_to_bdev(device) \
88 	container_of((device), struct block_device, bd_device)
89 
90 #define bdev_kobj(_bdev) \
91 	(&((_bdev)->bd_device.kobj))
92 
93 /*
94  * Block error status values.  See block/blk-core:blk_errors for the details.
95  */
96 typedef u8 __bitwise blk_status_t;
97 typedef u16 blk_short_t;
98 #define	BLK_STS_OK 0
99 #define BLK_STS_NOTSUPP		((__force blk_status_t)1)
100 #define BLK_STS_TIMEOUT		((__force blk_status_t)2)
101 #define BLK_STS_NOSPC		((__force blk_status_t)3)
102 #define BLK_STS_TRANSPORT	((__force blk_status_t)4)
103 #define BLK_STS_TARGET		((__force blk_status_t)5)
104 #define BLK_STS_RESV_CONFLICT	((__force blk_status_t)6)
105 #define BLK_STS_MEDIUM		((__force blk_status_t)7)
106 #define BLK_STS_PROTECTION	((__force blk_status_t)8)
107 #define BLK_STS_RESOURCE	((__force blk_status_t)9)
108 #define BLK_STS_IOERR		((__force blk_status_t)10)
109 
110 /* hack for device mapper, don't use elsewhere: */
111 #define BLK_STS_DM_REQUEUE    ((__force blk_status_t)11)
112 
113 /*
114  * BLK_STS_AGAIN should only be returned if RQF_NOWAIT is set
115  * and the bio would block (cf bio_wouldblock_error())
116  */
117 #define BLK_STS_AGAIN		((__force blk_status_t)12)
118 
119 /*
120  * BLK_STS_DEV_RESOURCE is returned from the driver to the block layer if
121  * device related resources are unavailable, but the driver can guarantee
122  * that the queue will be rerun in the future once resources become
123  * available again. This is typically the case for device specific
124  * resources that are consumed for IO. If the driver fails allocating these
125  * resources, we know that inflight (or pending) IO will free these
126  * resource upon completion.
127  *
128  * This is different from BLK_STS_RESOURCE in that it explicitly references
129  * a device specific resource. For resources of wider scope, allocation
130  * failure can happen without having pending IO. This means that we can't
131  * rely on request completions freeing these resources, as IO may not be in
132  * flight. Examples of that are kernel memory allocations, DMA mappings, or
133  * any other system wide resources.
134  */
135 #define BLK_STS_DEV_RESOURCE	((__force blk_status_t)13)
136 
137 /*
138  * BLK_STS_ZONE_OPEN_RESOURCE is returned from the driver in the completion
139  * path if the device returns a status indicating that too many zone resources
140  * are currently open. The same command should be successful if resubmitted
141  * after the number of open zones decreases below the device's limits, which is
142  * reported in the request_queue's max_open_zones.
143  */
144 #define BLK_STS_ZONE_OPEN_RESOURCE	((__force blk_status_t)14)
145 
146 /*
147  * BLK_STS_ZONE_ACTIVE_RESOURCE is returned from the driver in the completion
148  * path if the device returns a status indicating that too many zone resources
149  * are currently active. The same command should be successful if resubmitted
150  * after the number of active zones decreases below the device's limits, which
151  * is reported in the request_queue's max_active_zones.
152  */
153 #define BLK_STS_ZONE_ACTIVE_RESOURCE	((__force blk_status_t)15)
154 
155 /*
156  * BLK_STS_OFFLINE is returned from the driver when the target device is offline
157  * or is being taken offline. This could help differentiate the case where a
158  * device is intentionally being shut down from a real I/O error.
159  */
160 #define BLK_STS_OFFLINE		((__force blk_status_t)16)
161 
162 /*
163  * BLK_STS_DURATION_LIMIT is returned from the driver when the target device
164  * aborted the command because it exceeded one of its Command Duration Limits.
165  */
166 #define BLK_STS_DURATION_LIMIT	((__force blk_status_t)17)
167 
168 /*
169  * Invalid size or alignment.
170  */
171 #define BLK_STS_INVAL	((__force blk_status_t)19)
172 
173 /**
174  * blk_path_error - returns true if error may be path related
175  * @error: status the request was completed with
176  *
177  * Description:
178  *     This classifies block error status into non-retryable errors and ones
179  *     that may be successful if retried on a failover path.
180  *
181  * Return:
182  *     %false - retrying failover path will not help
183  *     %true  - may succeed if retried
184  */
185 static inline bool blk_path_error(blk_status_t error)
186 {
187 	switch (error) {
188 	case BLK_STS_NOTSUPP:
189 	case BLK_STS_NOSPC:
190 	case BLK_STS_TARGET:
191 	case BLK_STS_RESV_CONFLICT:
192 	case BLK_STS_MEDIUM:
193 	case BLK_STS_PROTECTION:
194 		return false;
195 	}
196 
197 	/* Anything else could be a path failure, so should be retried */
198 	return true;
199 }
200 
201 typedef __u32 __bitwise blk_opf_t;
202 
203 typedef unsigned int blk_qc_t;
204 #define BLK_QC_T_NONE		-1U
205 
206 /*
207  * main unit of I/O for the block layer and lower layers (ie drivers and
208  * stacking drivers)
209  */
210 struct bio {
211 	struct bio		*bi_next;	/* request queue link */
212 	struct block_device	*bi_bdev;
213 	blk_opf_t		bi_opf;		/* bottom bits REQ_OP, top bits
214 						 * req_flags.
215 						 */
216 	unsigned short		bi_flags;	/* BIO_* below */
217 	unsigned short		bi_ioprio;
218 	enum rw_hint		bi_write_hint;
219 	u8			bi_write_stream;
220 	blk_status_t		bi_status;
221 
222 	/*
223 	 * The bvec gap bit indicates the lowest set bit in any address offset
224 	 * between all bi_io_vecs. This field is initialized only after the bio
225 	 * is split to the hardware limits (see bio_split_io_at()). The value
226 	 * may be used to consider DMA optimization when performing that
227 	 * mapping. The value is compared to a power of two mask where the
228 	 * result depends on any bit set within the mask, so saving the lowest
229 	 * bit is sufficient to know if any segment gap collides with the mask.
230 	 */
231 	u8			bi_bvec_gap_bit;
232 
233 	atomic_t		__bi_remaining;
234 
235 	/* The actual vec list, preserved by bio_reset() */
236 	union {
237 		struct bio_vec		*bi_io_vec;
238 		/* Driver specific dma map, valid IFF REQ_DMABUF is set */
239 		struct dma_buf_io_map	*bi_dmabuf_map;
240 	};
241 
242 	struct bvec_iter	bi_iter;
243 
244 	union {
245 		/* for polled bios: */
246 		blk_qc_t		bi_cookie;
247 		/* for plugged zoned writes only: */
248 		unsigned int		__bi_nr_segments;
249 	};
250 	bio_end_io_t		*bi_end_io;
251 	void			*bi_private;
252 #ifdef CONFIG_BLK_CGROUP
253 	/*
254 	 * Represents the association of the css and request_queue for the bio.
255 	 * If a bio goes direct to device, it will not have a blkg as it will
256 	 * not have a request_queue associated with it.  The reference is put
257 	 * on release of the bio.
258 	 */
259 	struct blkcg_gq		*bi_blkg;
260 	/* Time that this bio was issued. */
261 	u64			issue_time_ns;
262 #ifdef CONFIG_BLK_CGROUP_IOCOST
263 	u64			bi_iocost_cost;
264 #endif
265 #endif
266 
267 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
268 	struct bio_crypt_ctx	*bi_crypt_context;
269 #endif
270 
271 #if defined(CONFIG_BLK_DEV_INTEGRITY)
272 	struct bio_integrity_payload *bi_integrity; /* data integrity */
273 #endif
274 
275 	unsigned short		bi_vcnt;	/* how many bio_vec's */
276 
277 	/*
278 	 * Everything starting with bi_max_vecs will be preserved by bio_reset()
279 	 */
280 
281 	/*
282 	 * Number of elements in `bi_io_vec` that were allocated for this bio.
283 	 * Only used by the bio submitter to make `bio_add_page` fail once full
284 	 * and to free the `bi_io_vec` allocation. Must not be used in drivers
285 	 * and does not hold a useful value for cloned bios.
286 	 */
287 	unsigned short		bi_max_vecs;
288 
289 	atomic_t		__bi_cnt;	/* pin count */
290 
291 	struct bio_set		*bi_pool;
292 };
293 
294 #define BIO_RESET_BYTES		offsetof(struct bio, bi_max_vecs)
295 #define BIO_MAX_SIZE		UINT_MAX /* max value of bi_iter.bi_size */
296 #define BIO_MAX_SECTORS		(BIO_MAX_SIZE >> SECTOR_SHIFT)
297 
298 static inline struct bio_vec *bio_inline_vecs(struct bio *bio)
299 {
300 	return (struct bio_vec *)(bio + 1);
301 }
302 
303 /*
304  * bio flags
305  */
306 enum {
307 	BIO_PAGE_PINNED,	/* Unpin pages in bio_release_pages() */
308 	BIO_CLONED,		/* doesn't own data */
309 	BIO_QUIET,		/* Make BIO Quiet */
310 	BIO_CHAIN,		/* chained bio, ->bi_remaining in effect */
311 	BIO_REFFED,		/* bio has elevated ->bi_cnt */
312 	BIO_BPS_THROTTLED,	/* This bio has already been subjected to
313 				 * throttling rules. Don't do it again. */
314 	BIO_TRACE_COMPLETION,	/* bio_endio() should trace the final completion
315 				 * of this bio. */
316 	BIO_CGROUP_ACCT,	/* has been accounted to a cgroup */
317 	BIO_QOS_THROTTLED,	/* bio went through rq_qos throttle path */
318 	/*
319 	 * This bio has completed bps throttling at the single tg granularity,
320 	 * which is different from BIO_BPS_THROTTLED. When the bio is enqueued
321 	 * into the sq->queued of the upper tg, or is about to be dispatched,
322 	 * this flag needs to be cleared. Since blk-throttle and rq_qos are not
323 	 * on the same hierarchical level, reuse the value.
324 	 */
325 	BIO_TG_BPS_THROTTLED = BIO_QOS_THROTTLED,
326 	BIO_QOS_MERGED,		/* but went through rq_qos merge path */
327 	BIO_REMAPPED,
328 	BIO_ZONE_WRITE_PLUGGING, /* bio handled through zone write plugging */
329 	BIO_EMULATES_ZONE_APPEND, /* bio emulates a zone append operation */
330 	BIO_COMPLETE_IN_TASK, /* complete bi_end_io() in task context */
331 	BIO_FLAG_LAST
332 };
333 
334 typedef __u32 __bitwise blk_mq_req_flags_t;
335 
336 #define REQ_OP_BITS	8
337 #define REQ_OP_MASK	(__force blk_opf_t)((1 << REQ_OP_BITS) - 1)
338 #define REQ_FLAG_BITS	24
339 
340 /**
341  * enum req_op - Operations common to the bio and request structures.
342  * We use 8 bits for encoding the operation, and the remaining 24 for flags.
343  *
344  * The least significant bit of the operation number indicates the data
345  * transfer direction:
346  *
347  *   - if the least significant bit is set transfers are TO the device
348  *   - if the least significant bit is not set transfers are FROM the device
349  *
350  * If a operation does not transfer data the least significant bit has no
351  * meaning.
352  */
353 enum req_op {
354 	/** @REQ_OP_READ: read sectors from the device */
355 	REQ_OP_READ		= (__force blk_opf_t)0,
356 	/** @REQ_OP_WRITE: write sectors to the device */
357 	REQ_OP_WRITE		= (__force blk_opf_t)1,
358 	/** @REQ_OP_FLUSH: flush the volatile write cache */
359 	REQ_OP_FLUSH		= (__force blk_opf_t)2,
360 	/** @REQ_OP_DISCARD: discard sectors */
361 	REQ_OP_DISCARD		= (__force blk_opf_t)3,
362 	/** @REQ_OP_SECURE_ERASE: securely erase sectors */
363 	REQ_OP_SECURE_ERASE	= (__force blk_opf_t)5,
364 	/** @REQ_OP_ZONE_APPEND: write data at the current zone write pointer */
365 	REQ_OP_ZONE_APPEND	= (__force blk_opf_t)7,
366 	/** @REQ_OP_WRITE_ZEROES: write the zero filled sector many times */
367 	REQ_OP_WRITE_ZEROES	= (__force blk_opf_t)9,
368 	/** @REQ_OP_ZONE_OPEN: Open a zone */
369 	REQ_OP_ZONE_OPEN	= (__force blk_opf_t)11,
370 	/** @REQ_OP_ZONE_CLOSE: Close a zone */
371 	REQ_OP_ZONE_CLOSE	= (__force blk_opf_t)13,
372 	/** @REQ_OP_ZONE_FINISH: Transition a zone to full */
373 	REQ_OP_ZONE_FINISH	= (__force blk_opf_t)15,
374 	/** @REQ_OP_ZONE_RESET: reset a zone write pointer */
375 	REQ_OP_ZONE_RESET	= (__force blk_opf_t)17,
376 	/** @REQ_OP_ZONE_RESET_ALL: reset all the zone present on the device */
377 	REQ_OP_ZONE_RESET_ALL	= (__force blk_opf_t)19,
378 
379 	/* Driver private requests */
380 	/* private: */
381 	REQ_OP_DRV_IN		= (__force blk_opf_t)34,
382 	REQ_OP_DRV_OUT		= (__force blk_opf_t)35,
383 
384 	REQ_OP_LAST		= (__force blk_opf_t)36,
385 };
386 
387 /* Keep cmd_flag_name[] in sync with the definitions below */
388 enum req_flag_bits {
389 	__REQ_FAILFAST_DEV =	/* no driver retries of device errors */
390 		REQ_OP_BITS,
391 	__REQ_FAILFAST_TRANSPORT, /* no driver retries of transport errors */
392 	__REQ_FAILFAST_DRIVER,	/* no driver retries of driver errors */
393 	__REQ_SYNC,		/* request is sync (sync write or read) */
394 	__REQ_META,		/* metadata io request */
395 	__REQ_PRIO,		/* boost priority in cfq */
396 	__REQ_NOMERGE,		/* don't touch this for merging */
397 	__REQ_IDLE,		/* anticipate more IO after this one */
398 	__REQ_INTEGRITY,	/* I/O includes block integrity payload */
399 	__REQ_FUA,		/* forced unit access */
400 	__REQ_PREFLUSH,		/* request for cache flush */
401 	__REQ_RAHEAD,		/* read ahead, can fail anytime */
402 	__REQ_BACKGROUND,	/* background IO */
403 	__REQ_NOWAIT,           /* Don't wait if request will block */
404 	__REQ_POLLED,		/* caller polls for completion using bio_poll */
405 	__REQ_ALLOC_CACHE,	/* allocate IO from cache if available */
406 	__REQ_SWAP,		/* swap I/O */
407 	__REQ_DRV,		/* for driver use */
408 	__REQ_FS_PRIVATE,	/* for file system (submitter) use */
409 	__REQ_ATOMIC,		/* for atomic write operations */
410 	__REQ_DMABUF,		/* Using premmaped dma buffers */
411 	/*
412 	 * Command specific flags, keep last:
413 	 */
414 	/* for REQ_OP_WRITE_ZEROES: */
415 	__REQ_NOUNMAP,		/* do not free blocks when zeroing */
416 
417 	__REQ_NR_BITS,		/* stops here */
418 };
419 
420 #define REQ_FAILFAST_DEV	\
421 			(__force blk_opf_t)(1ULL << __REQ_FAILFAST_DEV)
422 #define REQ_FAILFAST_TRANSPORT	\
423 			(__force blk_opf_t)(1ULL << __REQ_FAILFAST_TRANSPORT)
424 #define REQ_FAILFAST_DRIVER	\
425 			(__force blk_opf_t)(1ULL << __REQ_FAILFAST_DRIVER)
426 #define REQ_SYNC	(__force blk_opf_t)(1ULL << __REQ_SYNC)
427 #define REQ_META	(__force blk_opf_t)(1ULL << __REQ_META)
428 #define REQ_PRIO	(__force blk_opf_t)(1ULL << __REQ_PRIO)
429 #define REQ_NOMERGE	(__force blk_opf_t)(1ULL << __REQ_NOMERGE)
430 #define REQ_IDLE	(__force blk_opf_t)(1ULL << __REQ_IDLE)
431 #define REQ_INTEGRITY	(__force blk_opf_t)(1ULL << __REQ_INTEGRITY)
432 #define REQ_FUA		(__force blk_opf_t)(1ULL << __REQ_FUA)
433 #define REQ_PREFLUSH	(__force blk_opf_t)(1ULL << __REQ_PREFLUSH)
434 #define REQ_RAHEAD	(__force blk_opf_t)(1ULL << __REQ_RAHEAD)
435 #define REQ_BACKGROUND	(__force blk_opf_t)(1ULL << __REQ_BACKGROUND)
436 #define REQ_NOWAIT	(__force blk_opf_t)(1ULL << __REQ_NOWAIT)
437 #define REQ_POLLED	(__force blk_opf_t)(1ULL << __REQ_POLLED)
438 #define REQ_ALLOC_CACHE	(__force blk_opf_t)(1ULL << __REQ_ALLOC_CACHE)
439 #define REQ_SWAP	(__force blk_opf_t)(1ULL << __REQ_SWAP)
440 #define REQ_DRV		(__force blk_opf_t)(1ULL << __REQ_DRV)
441 #define REQ_FS_PRIVATE	(__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE)
442 #define REQ_ATOMIC	(__force blk_opf_t)(1ULL << __REQ_ATOMIC)
443 #define REQ_DMABUF	(__force blk_opf_t)(1ULL << __REQ_DMABUF)
444 
445 #define REQ_NOUNMAP	(__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
446 
447 #define REQ_FAILFAST_MASK \
448 	(REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
449 
450 #define REQ_NOMERGE_FLAGS \
451 	(REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA)
452 
453 enum stat_group {
454 	STAT_READ,
455 	STAT_WRITE,
456 	STAT_DISCARD,
457 	STAT_FLUSH,
458 
459 	NR_STAT_GROUPS
460 };
461 
462 static inline enum req_op bio_op(const struct bio *bio)
463 {
464 	return bio->bi_opf & REQ_OP_MASK;
465 }
466 
467 static inline bool op_is_write(blk_opf_t op)
468 {
469 	return !!(op & (__force blk_opf_t)1);
470 }
471 
472 /*
473  * Check if the bio or request is one that needs special treatment in the
474  * flush state machine.
475  */
476 static inline bool op_is_flush(blk_opf_t op)
477 {
478 	return op & (REQ_FUA | REQ_PREFLUSH);
479 }
480 
481 /*
482  * Reads are always treated as synchronous, as are requests with the FUA or
483  * PREFLUSH flag.  Other operations may be marked as synchronous using the
484  * REQ_SYNC flag.
485  */
486 static inline bool op_is_sync(blk_opf_t op)
487 {
488 	return (op & REQ_OP_MASK) == REQ_OP_READ ||
489 		(op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH));
490 }
491 
492 static inline bool op_is_discard(blk_opf_t op)
493 {
494 	return (op & REQ_OP_MASK) == REQ_OP_DISCARD;
495 }
496 
497 static inline bool op_is_dmabuf(blk_opf_t op)
498 {
499 	return op & REQ_DMABUF;
500 }
501 
502 /*
503  * Check if a bio or request operation is a zone management operation.
504  */
505 static inline bool op_is_zone_mgmt(enum req_op op)
506 {
507 	switch (op & REQ_OP_MASK) {
508 	case REQ_OP_ZONE_RESET:
509 	case REQ_OP_ZONE_RESET_ALL:
510 	case REQ_OP_ZONE_OPEN:
511 	case REQ_OP_ZONE_CLOSE:
512 	case REQ_OP_ZONE_FINISH:
513 		return true;
514 	default:
515 		return false;
516 	}
517 }
518 
519 static inline int op_stat_group(enum req_op op)
520 {
521 	if (op_is_discard(op))
522 		return STAT_DISCARD;
523 	return op_is_write(op);
524 }
525 
526 struct blk_rq_stat {
527 	u64 mean;
528 	u64 min;
529 	u64 max;
530 	u32 nr_samples;
531 	u64 batch;
532 };
533 
534 #endif /* __LINUX_BLK_TYPES_H */
535