xref: /linux/include/linux/bio.h (revision ed02f4f7ad9b5dabae2118e642cbbafe7dc9bb62)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
4  */
5 #ifndef __LINUX_BIO_H
6 #define __LINUX_BIO_H
7 
8 #include <linux/mempool.h>
9 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
10 #include <linux/blk_types.h>
11 #include <linux/uio.h>
12 
13 #define BIO_MAX_VECS		256U
14 #define BIO_MAX_INLINE_VECS	UIO_MAXIOV
15 
16 struct queue_limits;
17 
18 static inline unsigned int bio_max_segs(unsigned int nr_segs)
19 {
20 	return min(nr_segs, BIO_MAX_VECS);
21 }
22 
23 #define bio_iter_iovec(bio, iter)				\
24 	bvec_iter_bvec((bio)->bi_io_vec, (iter))
25 
26 #define bio_iter_page(bio, iter)				\
27 	bvec_iter_page((bio)->bi_io_vec, (iter))
28 #define bio_iter_len(bio, iter)					\
29 	bvec_iter_len((bio)->bi_io_vec, (iter))
30 #define bio_iter_offset(bio, iter)				\
31 	bvec_iter_offset((bio)->bi_io_vec, (iter))
32 
33 #define bio_page(bio)		bio_iter_page((bio), (bio)->bi_iter)
34 #define bio_offset(bio)		bio_iter_offset((bio), (bio)->bi_iter)
35 #define bio_iovec(bio)		bio_iter_iovec((bio), (bio)->bi_iter)
36 
37 #define bvec_iter_sectors(iter)	((iter).bi_size >> 9)
38 #define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter)))
39 
40 #define bio_sectors(bio)	bvec_iter_sectors((bio)->bi_iter)
41 #define bio_end_sector(bio)	bvec_iter_end_sector((bio)->bi_iter)
42 
43 /*
44  * Return the data direction, READ or WRITE.
45  */
46 #define bio_data_dir(bio) \
47 	(op_is_write(bio_op(bio)) ? WRITE : READ)
48 
49 static inline bool bio_flagged(const struct bio *bio, unsigned int bit)
50 {
51 	return bio->bi_flags & (1U << bit);
52 }
53 
54 static inline void bio_set_flag(struct bio *bio, unsigned int bit)
55 {
56 	bio->bi_flags |= (1U << bit);
57 }
58 
59 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
60 {
61 	bio->bi_flags &= ~(1U << bit);
62 }
63 
64 /*
65  * Check whether this bio carries any data or not. A NULL bio is allowed.
66  */
67 static inline bool bio_has_data(struct bio *bio)
68 {
69 	if (bio &&
70 	    bio->bi_iter.bi_size &&
71 	    bio_op(bio) != REQ_OP_DISCARD &&
72 	    bio_op(bio) != REQ_OP_SECURE_ERASE &&
73 	    bio_op(bio) != REQ_OP_WRITE_ZEROES)
74 		return true;
75 
76 	return false;
77 }
78 
79 static inline bool bio_no_advance_iter(const struct bio *bio)
80 {
81 	return bio_op(bio) == REQ_OP_DISCARD ||
82 	       bio_op(bio) == REQ_OP_SECURE_ERASE ||
83 	       bio_op(bio) == REQ_OP_WRITE_ZEROES ||
84 	       op_is_dmabuf(bio->bi_opf);
85 }
86 
87 static inline void *bio_data(struct bio *bio)
88 {
89 	if (bio_has_data(bio))
90 		return page_address(bio_page(bio)) + bio_offset(bio);
91 
92 	return NULL;
93 }
94 
95 static inline bool bio_next_segment(const struct bio *bio,
96 				    struct bvec_iter_all *iter)
97 {
98 	if (iter->idx >= bio->bi_vcnt)
99 		return false;
100 
101 	bvec_advance(&bio->bi_io_vec[iter->idx], iter);
102 	return true;
103 }
104 
105 /*
106  * drivers should _never_ use the all version - the bio may have been split
107  * before it got to the driver and the driver won't own all of it
108  */
109 #define bio_for_each_segment_all(bvl, bio, iter) \
110 	for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
111 
112 static inline void bio_advance_iter(const struct bio *bio,
113 				    struct bvec_iter *iter, unsigned int bytes)
114 {
115 	iter->bi_sector += bytes >> 9;
116 
117 	if (bio_no_advance_iter(bio)) {
118 		iter->bi_size -= bytes;
119 		iter->bi_offset += bytes;
120 	} else {
121 		bvec_iter_advance(bio->bi_io_vec, iter, bytes);
122 		/* TODO: It is reasonable to complete bio with error here. */
123 	}
124 }
125 
126 /* @bytes should be less or equal to bvec[i->bi_idx].bv_len */
127 static inline void bio_advance_iter_single(const struct bio *bio,
128 					   struct bvec_iter *iter,
129 					   unsigned int bytes)
130 {
131 	iter->bi_sector += bytes >> 9;
132 
133 	if (bio_no_advance_iter(bio)) {
134 		iter->bi_size -= bytes;
135 		iter->bi_offset += bytes;
136 	} else {
137 		bvec_iter_advance_single(bio->bi_io_vec, iter, bytes);
138 	}
139 }
140 
141 void __bio_advance(struct bio *, unsigned bytes);
142 
143 /**
144  * bio_advance - increment/complete a bio by some number of bytes
145  * @bio:	bio to advance
146  * @nbytes:	number of bytes to complete
147  *
148  * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
149  * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
150  * be updated on the last bvec as well.
151  *
152  * @bio will then represent the remaining, uncompleted portion of the io.
153  */
154 static inline void bio_advance(struct bio *bio, unsigned int nbytes)
155 {
156 	if (nbytes == bio->bi_iter.bi_size) {
157 		bio->bi_iter.bi_size = 0;
158 		return;
159 	}
160 	__bio_advance(bio, nbytes);
161 }
162 
163 #define __bio_for_each_segment(bvl, bio, iter, start)			\
164 	for (iter = (start);						\
165 	     (iter).bi_size &&						\
166 		((bvl = bio_iter_iovec((bio), (iter))), 1);		\
167 	     bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
168 
169 #define bio_for_each_segment(bvl, bio, iter)				\
170 	__bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
171 
172 #define __bio_for_each_bvec(bvl, bio, iter, start)		\
173 	for (iter = (start);						\
174 	     (iter).bi_size &&						\
175 		((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
176 	     bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
177 
178 /* iterate over multi-page bvec */
179 #define bio_for_each_bvec(bvl, bio, iter)			\
180 	__bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
181 
182 /*
183  * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the
184  * same reasons as bio_for_each_segment_all().
185  */
186 #define bio_for_each_bvec_all(bvl, bio, i)		\
187 	for (i = 0, bvl = bio_first_bvec_all(bio);	\
188 	     i < (bio)->bi_vcnt; i++, bvl++)
189 
190 #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
191 
192 static inline unsigned bio_segments(struct bio *bio)
193 {
194 	unsigned segs = 0;
195 	struct bio_vec bv;
196 	struct bvec_iter iter;
197 
198 	/*
199 	 * We special case discard/write same/write zeroes, because they
200 	 * interpret bi_size differently:
201 	 */
202 
203 	switch (bio_op(bio)) {
204 	case REQ_OP_DISCARD:
205 	case REQ_OP_SECURE_ERASE:
206 	case REQ_OP_WRITE_ZEROES:
207 		return 0;
208 	default:
209 		break;
210 	}
211 
212 	bio_for_each_segment(bv, bio, iter)
213 		segs++;
214 
215 	return segs;
216 }
217 
218 /*
219  * get a reference to a bio, so it won't disappear. the intended use is
220  * something like:
221  *
222  * bio_get(bio);
223  * submit_bio(rw, bio);
224  * if (bio->bi_flags ...)
225  *	do_something
226  * bio_put(bio);
227  *
228  * without the bio_get(), it could potentially complete I/O before submit_bio
229  * returns. and then bio would be freed memory when if (bio->bi_flags ...)
230  * runs
231  */
232 static inline void bio_get(struct bio *bio)
233 {
234 	bio->bi_flags |= (1 << BIO_REFFED);
235 	smp_mb__before_atomic();
236 	atomic_inc(&bio->__bi_cnt);
237 }
238 
239 static inline void bio_cnt_set(struct bio *bio, unsigned int count)
240 {
241 	if (count != 1) {
242 		bio->bi_flags |= (1 << BIO_REFFED);
243 		smp_mb();
244 	}
245 	atomic_set(&bio->__bi_cnt, count);
246 }
247 
248 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)
249 {
250 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
251 	return bio->bi_io_vec;
252 }
253 
254 static inline struct page *bio_first_page_all(struct bio *bio)
255 {
256 	return bio_first_bvec_all(bio)->bv_page;
257 }
258 
259 static inline struct folio *bio_first_folio_all(struct bio *bio)
260 {
261 	return page_folio(bio_first_page_all(bio));
262 }
263 
264 /**
265  * struct folio_iter - State for iterating all folios in a bio.
266  * @folio: The current folio we're iterating.  NULL after the last folio.
267  * @offset: The byte offset within the current folio.
268  * @length: The number of bytes in this iteration (will not cross folio
269  *	boundary).
270  */
271 struct folio_iter {
272 	struct folio *folio;
273 	size_t offset;
274 	size_t length;
275 	/* private: for use by the iterator */
276 	struct folio *_next;
277 	size_t _seg_count;
278 	int _i;
279 };
280 
281 static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio,
282 				   int i)
283 {
284 	struct bio_vec *bvec = bio_first_bvec_all(bio) + i;
285 
286 	if (unlikely(i >= bio->bi_vcnt)) {
287 		fi->folio = NULL;
288 		return;
289 	}
290 
291 	fi->folio = bvec_folio(bvec);
292 	fi->offset = bvec->bv_offset +
293 			PAGE_SIZE * folio_page_idx(fi->folio, bvec->bv_page);
294 	fi->_seg_count = bvec->bv_len;
295 	fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count);
296 	fi->_next = folio_next(fi->folio);
297 	fi->_i = i;
298 }
299 
300 static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)
301 {
302 	fi->_seg_count -= fi->length;
303 	if (fi->_seg_count) {
304 		fi->folio = fi->_next;
305 		fi->offset = 0;
306 		fi->length = min(folio_size(fi->folio), fi->_seg_count);
307 		fi->_next = folio_next(fi->folio);
308 	} else {
309 		bio_first_folio(fi, bio, fi->_i + 1);
310 	}
311 }
312 
313 /**
314  * bio_for_each_folio_all - Iterate over each folio in a bio.
315  * @fi: struct folio_iter which is updated for each folio.
316  * @bio: struct bio to iterate over.
317  */
318 #define bio_for_each_folio_all(fi, bio)				\
319 	for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio))
320 
321 void bio_trim(struct bio *bio, sector_t offset, sector_t size);
322 extern struct bio *bio_split(struct bio *bio, int sectors,
323 			     gfp_t gfp, struct bio_set *bs);
324 int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
325 		unsigned *segs, unsigned max_bytes, unsigned len_align);
326 u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next,
327 		u8 gaps_bit);
328 
329 /**
330  * bio_next_split - get next @sectors from a bio, splitting if necessary
331  * @bio:	bio to split
332  * @sectors:	number of sectors to split from the front of @bio
333  * @gfp:	gfp mask
334  * @bs:		bio set to allocate from
335  *
336  * Return: a bio representing the next @sectors of @bio - if the bio is smaller
337  * than @sectors, returns the original bio unchanged.
338  */
339 static inline struct bio *bio_next_split(struct bio *bio, int sectors,
340 					 gfp_t gfp, struct bio_set *bs)
341 {
342 	if (sectors >= bio_sectors(bio))
343 		return bio;
344 
345 	return bio_split(bio, sectors, gfp, bs);
346 }
347 
348 enum {
349 	BIOSET_NEED_BVECS = BIT(0),
350 	BIOSET_NEED_RESCUER = BIT(1),
351 	BIOSET_PERCPU_CACHE = BIT(2),
352 };
353 extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags);
354 extern void bioset_exit(struct bio_set *);
355 
356 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
357 			     blk_opf_t opf, gfp_t gfp, struct bio_set *bs);
358 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask);
359 extern void bio_put(struct bio *);
360 
361 struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
362 		gfp_t gfp, struct bio_set *bs);
363 int bio_init_clone(struct block_device *bdev, struct bio *bio,
364 		struct bio *bio_src, gfp_t gfp);
365 
366 extern struct bio_set fs_bio_set;
367 
368 static inline struct bio *bio_alloc(struct block_device *bdev,
369 		unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask)
370 {
371 	return bio_alloc_bioset(bdev, nr_vecs, opf, gfp_mask, &fs_bio_set);
372 }
373 
374 void submit_bio(struct bio *bio);
375 
376 /**
377  * bio_in_atomic - check if the current context is unsafe for bio completion
378  *
379  * Return: %true in atomic contexts (e.g. hard/soft IRQ, preempt-disabled);
380  * %false when a bio can be safely completed in the current context.
381  */
382 static inline bool bio_in_atomic(void)
383 {
384 	if (IS_ENABLED(CONFIG_PREEMPTION) && rcu_preempt_depth())
385 		return true;
386 	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
387 		return true;
388 	return !preemptible();
389 }
390 
391 void __bio_complete_in_task(struct bio *bio);
392 
393 /**
394  * bio_complete_in_task - ensure a bio is completed in preemptible task context
395  * @bio: bio to complete
396  *
397  * If called from non-task context, offload the bio completion to a worker
398  * thread and return %true. Else return %false and do nothing.
399  *
400  * Uses BIO_COMPLETE_IN_TASK as a sentinel: if set, the bio was already
401  * deferred and we are running in the worker — return %false so the
402  * callback proceeds instead of re-deferring.
403  */
404 static inline bool bio_complete_in_task(struct bio *bio)
405 {
406 	if (bio_flagged(bio, BIO_COMPLETE_IN_TASK))
407 		return false;
408 	if (!bio_in_atomic())
409 		return false;
410 	bio_set_flag(bio, BIO_COMPLETE_IN_TASK);
411 	__bio_complete_in_task(bio);
412 	return true;
413 }
414 
415 extern void bio_endio(struct bio *);
416 
417 /**
418  * bio_endio_status - end I/O on a bio with a specific status
419  * @bio:	bio
420  * @status:	status to set
421  *
422  * Set @bio->bi_status to @status and call bio_endio().
423  **/
424 static inline void bio_endio_status(struct bio *bio, blk_status_t status)
425 {
426 	bio->bi_status = status;
427 	bio_endio(bio);
428 }
429 
430 static inline void bio_io_error(struct bio *bio)
431 {
432 	bio_endio_status(bio, BLK_STS_IOERR);
433 }
434 
435 static inline void bio_wouldblock_error(struct bio *bio)
436 {
437 	bio_endio_status(bio, BLK_STS_AGAIN);
438 }
439 
440 /*
441  * Calculate number of bvec segments that should be allocated to fit data
442  * pointed by @iter. If @iter is backed by a bvec or a dmabuf, the bvec array /
443  * the dma map are going to be reused, and so no extra allocation is required.
444  */
445 static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
446 {
447 	if (iov_iter_is_bvec(iter) || iov_iter_is_dmabuf_map(iter))
448 		return 0;
449 	return iov_iter_npages(iter, max_segs);
450 }
451 
452 /**
453  * bio_iov_bounce_nr_vecs - calculate number of bvecs for a bounce bio
454  * @iter:	iter to bounce from
455  * @op:		REQ_OP_* for the bio
456  *
457  * Calculates how many bvecs are needed for the next bio to bounce from/to
458  * @iter.
459  */
460 static inline unsigned short
461 bio_iov_bounce_nr_vecs(struct iov_iter *iter, blk_opf_t op)
462 {
463 	/*
464 	 * We still need to bounce bvec iters, so don't special case them
465 	 * here unlike in bio_iov_vecs_to_alloc.
466 	 *
467 	 * For reads we need to use a vector for the bounce buffer, account
468 	 * for that here.
469 	 */
470 	if (op_is_write(op))
471 		return iov_iter_npages(iter, BIO_MAX_VECS);
472 	return iov_iter_npages(iter, BIO_MAX_VECS - 1) + 1;
473 }
474 
475 struct request_queue;
476 
477 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
478 	      unsigned short max_vecs, blk_opf_t opf);
479 static inline void bio_init_inline(struct bio *bio, struct block_device *bdev,
480 	      unsigned short max_vecs, blk_opf_t opf)
481 {
482 	bio_init(bio, bdev, bio_inline_vecs(bio), max_vecs, opf);
483 }
484 extern void bio_uninit(struct bio *);
485 void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf);
486 void bio_reuse(struct bio *bio, blk_opf_t opf);
487 void bio_prepare_reissue(struct bio *bio, struct block_device *bdev);
488 void bio_chain(struct bio *, struct bio *);
489 void bio_await(struct bio *bio, void *priv,
490 	       void (*submit)(struct bio *bio, void *priv));
491 
492 int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len,
493 			      unsigned off);
494 bool __must_check bio_add_folio(struct bio *bio, struct folio *folio,
495 				size_t len, size_t off);
496 void __bio_add_page(struct bio *bio, struct page *page,
497 		unsigned int len, unsigned int off);
498 void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
499 			  size_t off);
500 void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len);
501 
502 /**
503  * bio_add_max_vecs - number of bio_vecs needed to add data to a bio
504  * @kaddr: kernel virtual address to add
505  * @len: length in bytes to add
506  *
507  * Calculate how many bio_vecs need to be allocated to add the kernel virtual
508  * address range in [@kaddr:@len] in the worse case.
509  */
510 static inline unsigned int bio_add_max_vecs(void *kaddr, unsigned int len)
511 {
512 	if (is_vmalloc_addr(kaddr))
513 		return DIV_ROUND_UP(offset_in_page(kaddr) + len, PAGE_SIZE);
514 	return 1;
515 }
516 
517 unsigned int bio_add_vmalloc_chunk(struct bio *bio, void *vaddr, unsigned len);
518 bool bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned int len);
519 
520 int submit_bio_wait(struct bio *bio);
521 int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data,
522 		size_t len, enum req_op op);
523 
524 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
525 		unsigned maxlen, unsigned mem_align_mask,
526 		unsigned len_align_mask);
527 
528 bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter);
529 void __bio_release_pages(struct bio *bio, bool mark_dirty);
530 extern void bio_set_pages_dirty(struct bio *bio);
531 extern void bio_check_pages_dirty(struct bio *bio);
532 
533 int bio_alloc_bounce_folios(struct bio *bio, size_t total_len, size_t minsize);
534 void bio_free_folios(struct bio *bio);
535 int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
536 		size_t maxlen, size_t minsize);
537 
538 extern void bio_copy_data(struct bio *dst, struct bio *src);
539 extern void bio_free_pages(struct bio *bio);
540 void zero_fill_bio(struct bio *bio);
541 void guard_bio_eod(struct bio *bio);
542 
543 static inline void bio_release_pages(struct bio *bio, bool mark_dirty)
544 {
545 	if (bio_flagged(bio, BIO_PAGE_PINNED))
546 		__bio_release_pages(bio, mark_dirty);
547 }
548 
549 #define bio_dev(bio) \
550 	disk_devt((bio)->bi_bdev->bd_disk)
551 
552 #ifdef CONFIG_BLK_CGROUP
553 void bio_associate_blkg(struct bio *bio);
554 void bio_associate_blkg_from_css(struct bio *bio,
555 				 struct cgroup_subsys_state *css);
556 void bio_clone_blkg_association(struct bio *dst, struct bio *src);
557 void blkcg_punt_bio_submit(struct bio *bio);
558 #else	/* CONFIG_BLK_CGROUP */
559 static inline void bio_associate_blkg(struct bio *bio) { }
560 static inline void bio_associate_blkg_from_css(struct bio *bio,
561 					       struct cgroup_subsys_state *css)
562 { }
563 static inline void bio_clone_blkg_association(struct bio *dst,
564 					      struct bio *src) { }
565 static inline void blkcg_punt_bio_submit(struct bio *bio)
566 {
567 	submit_bio(bio);
568 }
569 #endif	/* CONFIG_BLK_CGROUP */
570 
571 static inline void bio_set_dev(struct bio *bio, struct block_device *bdev)
572 {
573 	bio_clear_flag(bio, BIO_REMAPPED);
574 	if (bio->bi_bdev != bdev)
575 		bio_clear_flag(bio, BIO_BPS_THROTTLED);
576 	bio->bi_bdev = bdev;
577 	bio_associate_blkg(bio);
578 }
579 
580 /*
581  * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
582  *
583  * A bio_list anchors a singly-linked list of bios chained through the bi_next
584  * member of the bio.  The bio_list also caches the last list member to allow
585  * fast access to the tail.
586  */
587 struct bio_list {
588 	struct bio *head;
589 	struct bio *tail;
590 };
591 
592 static inline int bio_list_empty(const struct bio_list *bl)
593 {
594 	return bl->head == NULL;
595 }
596 
597 static inline void bio_list_init(struct bio_list *bl)
598 {
599 	bl->head = bl->tail = NULL;
600 }
601 
602 #define BIO_EMPTY_LIST	{ NULL, NULL }
603 
604 #define bio_list_for_each(bio, bl) \
605 	for (bio = (bl)->head; bio; bio = bio->bi_next)
606 
607 static inline unsigned bio_list_size(const struct bio_list *bl)
608 {
609 	unsigned sz = 0;
610 	struct bio *bio;
611 
612 	bio_list_for_each(bio, bl)
613 		sz++;
614 
615 	return sz;
616 }
617 
618 static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
619 {
620 	bio->bi_next = NULL;
621 
622 	if (bl->tail)
623 		bl->tail->bi_next = bio;
624 	else
625 		bl->head = bio;
626 
627 	bl->tail = bio;
628 }
629 
630 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
631 {
632 	bio->bi_next = bl->head;
633 
634 	bl->head = bio;
635 
636 	if (!bl->tail)
637 		bl->tail = bio;
638 }
639 
640 static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
641 {
642 	if (!bl2->head)
643 		return;
644 
645 	if (bl->tail)
646 		bl->tail->bi_next = bl2->head;
647 	else
648 		bl->head = bl2->head;
649 
650 	bl->tail = bl2->tail;
651 }
652 
653 static inline void bio_list_merge_init(struct bio_list *bl,
654 		struct bio_list *bl2)
655 {
656 	bio_list_merge(bl, bl2);
657 	bio_list_init(bl2);
658 }
659 
660 static inline void bio_list_merge_head(struct bio_list *bl,
661 				       struct bio_list *bl2)
662 {
663 	if (!bl2->head)
664 		return;
665 
666 	if (bl->head)
667 		bl2->tail->bi_next = bl->head;
668 	else
669 		bl->tail = bl2->tail;
670 
671 	bl->head = bl2->head;
672 }
673 
674 static inline struct bio *bio_list_peek(struct bio_list *bl)
675 {
676 	return bl->head;
677 }
678 
679 static inline struct bio *bio_list_pop(struct bio_list *bl)
680 {
681 	struct bio *bio = bl->head;
682 
683 	if (bio) {
684 		bl->head = bl->head->bi_next;
685 		if (!bl->head)
686 			bl->tail = NULL;
687 
688 		bio->bi_next = NULL;
689 	}
690 
691 	return bio;
692 }
693 
694 static inline struct bio *bio_list_get(struct bio_list *bl)
695 {
696 	struct bio *bio = bl->head;
697 
698 	bl->head = bl->tail = NULL;
699 
700 	return bio;
701 }
702 
703 /*
704  * Increment chain count for the bio. Make sure the CHAIN flag update
705  * is visible before the raised count.
706  */
707 static inline void bio_inc_remaining(struct bio *bio)
708 {
709 	bio_set_flag(bio, BIO_CHAIN);
710 	smp_mb__before_atomic();
711 	atomic_inc(&bio->__bi_remaining);
712 }
713 
714 /*
715  * bio_set is used to allow other portions of the IO system to
716  * allocate their own private memory pools for bio and iovec structures.
717  * These memory pools in turn all allocate from the bio_slab
718  * and the bvec_slabs[].
719  */
720 #define BIO_POOL_SIZE 2
721 
722 struct bio_set {
723 	struct kmem_cache *bio_slab;
724 	unsigned int front_pad;
725 
726 	/*
727 	 * per-cpu bio alloc cache
728 	 */
729 	struct bio_alloc_cache __percpu *cache;
730 
731 	mempool_t bio_pool;
732 	mempool_t bvec_pool;
733 
734 	unsigned int back_pad;
735 	/*
736 	 * Deadlock avoidance for stacking block drivers: see comments in
737 	 * bio_alloc_bioset() for details
738 	 */
739 	spinlock_t		rescue_lock;
740 	struct bio_list		rescue_list;
741 	struct work_struct	rescue_work;
742 	struct workqueue_struct	*rescue_workqueue;
743 
744 	/*
745 	 * Hot un-plug notifier for the per-cpu cache, if used
746 	 */
747 	struct hlist_node cpuhp_dead;
748 };
749 
750 static inline bool bioset_initialized(struct bio_set *bs)
751 {
752 	return bs->bio_slab != NULL;
753 }
754 
755 static inline void bio_clear_polled(struct bio *bio)
756 {
757 	bio->bi_opf &= ~REQ_POLLED;
758 }
759 
760 /**
761  * bio_is_zone_append - is this a zone append bio?
762  * @bio:	bio to check
763  *
764  * Check if @bio is a zone append operation.  Core block layer code and end_io
765  * handlers must use this instead of an open coded REQ_OP_ZONE_APPEND check
766  * because the block layer can rewrite REQ_OP_ZONE_APPEND to REQ_OP_WRITE if
767  * it is not natively supported.
768  */
769 static inline bool bio_is_zone_append(struct bio *bio)
770 {
771 	if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED))
772 		return false;
773 	return bio_op(bio) == REQ_OP_ZONE_APPEND ||
774 		bio_flagged(bio, BIO_EMULATES_ZONE_APPEND);
775 }
776 
777 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
778 		unsigned int nr_pages, blk_opf_t opf, gfp_t gfp);
779 struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new);
780 
781 struct bio *blk_alloc_discard_bio(struct block_device *bdev,
782 		sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask);
783 
784 #endif /* __LINUX_BIO_H */
785