xref: /linux/block/bio.c (revision 55d0969c451159cff86949b38c39171cab962069)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
4  */
5 #include <linux/mm.h>
6 #include <linux/swap.h>
7 #include <linux/bio-integrity.h>
8 #include <linux/blkdev.h>
9 #include <linux/uio.h>
10 #include <linux/iocontext.h>
11 #include <linux/slab.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/export.h>
15 #include <linux/mempool.h>
16 #include <linux/workqueue.h>
17 #include <linux/cgroup.h>
18 #include <linux/highmem.h>
19 #include <linux/blk-crypto.h>
20 #include <linux/xarray.h>
21 
22 #include <trace/events/block.h>
23 #include "blk.h"
24 #include "blk-rq-qos.h"
25 #include "blk-cgroup.h"
26 
27 #define ALLOC_CACHE_THRESHOLD	16
28 #define ALLOC_CACHE_MAX		256
29 
30 struct bio_alloc_cache {
31 	struct bio		*free_list;
32 	struct bio		*free_list_irq;
33 	unsigned int		nr;
34 	unsigned int		nr_irq;
35 };
36 
37 static struct biovec_slab {
38 	int nr_vecs;
39 	char *name;
40 	struct kmem_cache *slab;
41 } bvec_slabs[] __read_mostly = {
42 	{ .nr_vecs = 16, .name = "biovec-16" },
43 	{ .nr_vecs = 64, .name = "biovec-64" },
44 	{ .nr_vecs = 128, .name = "biovec-128" },
45 	{ .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" },
46 };
47 
48 static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
49 {
50 	switch (nr_vecs) {
51 	/* smaller bios use inline vecs */
52 	case 5 ... 16:
53 		return &bvec_slabs[0];
54 	case 17 ... 64:
55 		return &bvec_slabs[1];
56 	case 65 ... 128:
57 		return &bvec_slabs[2];
58 	case 129 ... BIO_MAX_VECS:
59 		return &bvec_slabs[3];
60 	default:
61 		BUG();
62 		return NULL;
63 	}
64 }
65 
66 /*
67  * fs_bio_set is the bio_set containing bio and iovec memory pools used by
68  * IO code that does not need private memory pools.
69  */
70 struct bio_set fs_bio_set;
71 EXPORT_SYMBOL(fs_bio_set);
72 
73 /*
74  * Our slab pool management
75  */
76 struct bio_slab {
77 	struct kmem_cache *slab;
78 	unsigned int slab_ref;
79 	unsigned int slab_size;
80 	char name[8];
81 };
82 static DEFINE_MUTEX(bio_slab_lock);
83 static DEFINE_XARRAY(bio_slabs);
84 
85 static struct bio_slab *create_bio_slab(unsigned int size)
86 {
87 	struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL);
88 
89 	if (!bslab)
90 		return NULL;
91 
92 	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size);
93 	bslab->slab = kmem_cache_create(bslab->name, size,
94 			ARCH_KMALLOC_MINALIGN,
95 			SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL);
96 	if (!bslab->slab)
97 		goto fail_alloc_slab;
98 
99 	bslab->slab_ref = 1;
100 	bslab->slab_size = size;
101 
102 	if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL)))
103 		return bslab;
104 
105 	kmem_cache_destroy(bslab->slab);
106 
107 fail_alloc_slab:
108 	kfree(bslab);
109 	return NULL;
110 }
111 
112 static inline unsigned int bs_bio_slab_size(struct bio_set *bs)
113 {
114 	return bs->front_pad + sizeof(struct bio) + bs->back_pad;
115 }
116 
117 static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs)
118 {
119 	unsigned int size = bs_bio_slab_size(bs);
120 	struct bio_slab *bslab;
121 
122 	mutex_lock(&bio_slab_lock);
123 	bslab = xa_load(&bio_slabs, size);
124 	if (bslab)
125 		bslab->slab_ref++;
126 	else
127 		bslab = create_bio_slab(size);
128 	mutex_unlock(&bio_slab_lock);
129 
130 	if (bslab)
131 		return bslab->slab;
132 	return NULL;
133 }
134 
135 static void bio_put_slab(struct bio_set *bs)
136 {
137 	struct bio_slab *bslab = NULL;
138 	unsigned int slab_size = bs_bio_slab_size(bs);
139 
140 	mutex_lock(&bio_slab_lock);
141 
142 	bslab = xa_load(&bio_slabs, slab_size);
143 	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
144 		goto out;
145 
146 	WARN_ON_ONCE(bslab->slab != bs->bio_slab);
147 
148 	WARN_ON(!bslab->slab_ref);
149 
150 	if (--bslab->slab_ref)
151 		goto out;
152 
153 	xa_erase(&bio_slabs, slab_size);
154 
155 	kmem_cache_destroy(bslab->slab);
156 	kfree(bslab);
157 
158 out:
159 	mutex_unlock(&bio_slab_lock);
160 }
161 
162 void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)
163 {
164 	BUG_ON(nr_vecs > BIO_MAX_VECS);
165 
166 	if (nr_vecs == BIO_MAX_VECS)
167 		mempool_free(bv, pool);
168 	else if (nr_vecs > BIO_INLINE_VECS)
169 		kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
170 }
171 
172 /*
173  * Make the first allocation restricted and don't dump info on allocation
174  * failures, since we'll fall back to the mempool in case of failure.
175  */
176 static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
177 {
178 	return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) |
179 		__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
180 }
181 
182 struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
183 		gfp_t gfp_mask)
184 {
185 	struct biovec_slab *bvs = biovec_slab(*nr_vecs);
186 
187 	if (WARN_ON_ONCE(!bvs))
188 		return NULL;
189 
190 	/*
191 	 * Upgrade the nr_vecs request to take full advantage of the allocation.
192 	 * We also rely on this in the bvec_free path.
193 	 */
194 	*nr_vecs = bvs->nr_vecs;
195 
196 	/*
197 	 * Try a slab allocation first for all smaller allocations.  If that
198 	 * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
199 	 * The mempool is sized to handle up to BIO_MAX_VECS entries.
200 	 */
201 	if (*nr_vecs < BIO_MAX_VECS) {
202 		struct bio_vec *bvl;
203 
204 		bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
205 		if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM))
206 			return bvl;
207 		*nr_vecs = BIO_MAX_VECS;
208 	}
209 
210 	return mempool_alloc(pool, gfp_mask);
211 }
212 
213 void bio_uninit(struct bio *bio)
214 {
215 #ifdef CONFIG_BLK_CGROUP
216 	if (bio->bi_blkg) {
217 		blkg_put(bio->bi_blkg);
218 		bio->bi_blkg = NULL;
219 	}
220 #endif
221 	if (bio_integrity(bio))
222 		bio_integrity_free(bio);
223 
224 	bio_crypt_free_ctx(bio);
225 }
226 EXPORT_SYMBOL(bio_uninit);
227 
228 static void bio_free(struct bio *bio)
229 {
230 	struct bio_set *bs = bio->bi_pool;
231 	void *p = bio;
232 
233 	WARN_ON_ONCE(!bs);
234 
235 	bio_uninit(bio);
236 	bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
237 	mempool_free(p - bs->front_pad, &bs->bio_pool);
238 }
239 
240 /*
241  * Users of this function have their own bio allocation. Subsequently,
242  * they must remember to pair any call to bio_init() with bio_uninit()
243  * when IO has completed, or when the bio is released.
244  */
245 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
246 	      unsigned short max_vecs, blk_opf_t opf)
247 {
248 	bio->bi_next = NULL;
249 	bio->bi_bdev = bdev;
250 	bio->bi_opf = opf;
251 	bio->bi_flags = 0;
252 	bio->bi_ioprio = 0;
253 	bio->bi_write_hint = 0;
254 	bio->bi_status = 0;
255 	bio->bi_iter.bi_sector = 0;
256 	bio->bi_iter.bi_size = 0;
257 	bio->bi_iter.bi_idx = 0;
258 	bio->bi_iter.bi_bvec_done = 0;
259 	bio->bi_end_io = NULL;
260 	bio->bi_private = NULL;
261 #ifdef CONFIG_BLK_CGROUP
262 	bio->bi_blkg = NULL;
263 	bio->bi_issue.value = 0;
264 	if (bdev)
265 		bio_associate_blkg(bio);
266 #ifdef CONFIG_BLK_CGROUP_IOCOST
267 	bio->bi_iocost_cost = 0;
268 #endif
269 #endif
270 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
271 	bio->bi_crypt_context = NULL;
272 #endif
273 #ifdef CONFIG_BLK_DEV_INTEGRITY
274 	bio->bi_integrity = NULL;
275 #endif
276 	bio->bi_vcnt = 0;
277 
278 	atomic_set(&bio->__bi_remaining, 1);
279 	atomic_set(&bio->__bi_cnt, 1);
280 	bio->bi_cookie = BLK_QC_T_NONE;
281 
282 	bio->bi_max_vecs = max_vecs;
283 	bio->bi_io_vec = table;
284 	bio->bi_pool = NULL;
285 }
286 EXPORT_SYMBOL(bio_init);
287 
288 /**
289  * bio_reset - reinitialize a bio
290  * @bio:	bio to reset
291  * @bdev:	block device to use the bio for
292  * @opf:	operation and flags for bio
293  *
294  * Description:
295  *   After calling bio_reset(), @bio will be in the same state as a freshly
296  *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
297  *   preserved are the ones that are initialized by bio_alloc_bioset(). See
298  *   comment in struct bio.
299  */
300 void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf)
301 {
302 	bio_uninit(bio);
303 	memset(bio, 0, BIO_RESET_BYTES);
304 	atomic_set(&bio->__bi_remaining, 1);
305 	bio->bi_bdev = bdev;
306 	if (bio->bi_bdev)
307 		bio_associate_blkg(bio);
308 	bio->bi_opf = opf;
309 }
310 EXPORT_SYMBOL(bio_reset);
311 
312 static struct bio *__bio_chain_endio(struct bio *bio)
313 {
314 	struct bio *parent = bio->bi_private;
315 
316 	if (bio->bi_status && !parent->bi_status)
317 		parent->bi_status = bio->bi_status;
318 	bio_put(bio);
319 	return parent;
320 }
321 
322 static void bio_chain_endio(struct bio *bio)
323 {
324 	bio_endio(__bio_chain_endio(bio));
325 }
326 
327 /**
328  * bio_chain - chain bio completions
329  * @bio: the target bio
330  * @parent: the parent bio of @bio
331  *
332  * The caller won't have a bi_end_io called when @bio completes - instead,
333  * @parent's bi_end_io won't be called until both @parent and @bio have
334  * completed; the chained bio will also be freed when it completes.
335  *
336  * The caller must not set bi_private or bi_end_io in @bio.
337  */
338 void bio_chain(struct bio *bio, struct bio *parent)
339 {
340 	BUG_ON(bio->bi_private || bio->bi_end_io);
341 
342 	bio->bi_private = parent;
343 	bio->bi_end_io	= bio_chain_endio;
344 	bio_inc_remaining(parent);
345 }
346 EXPORT_SYMBOL(bio_chain);
347 
348 /**
349  * bio_chain_and_submit - submit a bio after chaining it to another one
350  * @prev: bio to chain and submit
351  * @new: bio to chain to
352  *
353  * If @prev is non-NULL, chain it to @new and submit it.
354  *
355  * Return: @new.
356  */
357 struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new)
358 {
359 	if (prev) {
360 		bio_chain(prev, new);
361 		submit_bio(prev);
362 	}
363 	return new;
364 }
365 
366 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
367 		unsigned int nr_pages, blk_opf_t opf, gfp_t gfp)
368 {
369 	return bio_chain_and_submit(bio, bio_alloc(bdev, nr_pages, opf, gfp));
370 }
371 EXPORT_SYMBOL_GPL(blk_next_bio);
372 
373 static void bio_alloc_rescue(struct work_struct *work)
374 {
375 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
376 	struct bio *bio;
377 
378 	while (1) {
379 		spin_lock(&bs->rescue_lock);
380 		bio = bio_list_pop(&bs->rescue_list);
381 		spin_unlock(&bs->rescue_lock);
382 
383 		if (!bio)
384 			break;
385 
386 		submit_bio_noacct(bio);
387 	}
388 }
389 
390 static void punt_bios_to_rescuer(struct bio_set *bs)
391 {
392 	struct bio_list punt, nopunt;
393 	struct bio *bio;
394 
395 	if (WARN_ON_ONCE(!bs->rescue_workqueue))
396 		return;
397 	/*
398 	 * In order to guarantee forward progress we must punt only bios that
399 	 * were allocated from this bio_set; otherwise, if there was a bio on
400 	 * there for a stacking driver higher up in the stack, processing it
401 	 * could require allocating bios from this bio_set, and doing that from
402 	 * our own rescuer would be bad.
403 	 *
404 	 * Since bio lists are singly linked, pop them all instead of trying to
405 	 * remove from the middle of the list:
406 	 */
407 
408 	bio_list_init(&punt);
409 	bio_list_init(&nopunt);
410 
411 	while ((bio = bio_list_pop(&current->bio_list[0])))
412 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
413 	current->bio_list[0] = nopunt;
414 
415 	bio_list_init(&nopunt);
416 	while ((bio = bio_list_pop(&current->bio_list[1])))
417 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
418 	current->bio_list[1] = nopunt;
419 
420 	spin_lock(&bs->rescue_lock);
421 	bio_list_merge(&bs->rescue_list, &punt);
422 	spin_unlock(&bs->rescue_lock);
423 
424 	queue_work(bs->rescue_workqueue, &bs->rescue_work);
425 }
426 
427 static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache)
428 {
429 	unsigned long flags;
430 
431 	/* cache->free_list must be empty */
432 	if (WARN_ON_ONCE(cache->free_list))
433 		return;
434 
435 	local_irq_save(flags);
436 	cache->free_list = cache->free_list_irq;
437 	cache->free_list_irq = NULL;
438 	cache->nr += cache->nr_irq;
439 	cache->nr_irq = 0;
440 	local_irq_restore(flags);
441 }
442 
443 static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
444 		unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp,
445 		struct bio_set *bs)
446 {
447 	struct bio_alloc_cache *cache;
448 	struct bio *bio;
449 
450 	cache = per_cpu_ptr(bs->cache, get_cpu());
451 	if (!cache->free_list) {
452 		if (READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD)
453 			bio_alloc_irq_cache_splice(cache);
454 		if (!cache->free_list) {
455 			put_cpu();
456 			return NULL;
457 		}
458 	}
459 	bio = cache->free_list;
460 	cache->free_list = bio->bi_next;
461 	cache->nr--;
462 	put_cpu();
463 
464 	bio_init(bio, bdev, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs, opf);
465 	bio->bi_pool = bs;
466 	return bio;
467 }
468 
469 /**
470  * bio_alloc_bioset - allocate a bio for I/O
471  * @bdev:	block device to allocate the bio for (can be %NULL)
472  * @nr_vecs:	number of bvecs to pre-allocate
473  * @opf:	operation and flags for bio
474  * @gfp_mask:   the GFP_* mask given to the slab allocator
475  * @bs:		the bio_set to allocate from.
476  *
477  * Allocate a bio from the mempools in @bs.
478  *
479  * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to
480  * allocate a bio.  This is due to the mempool guarantees.  To make this work,
481  * callers must never allocate more than 1 bio at a time from the general pool.
482  * Callers that need to allocate more than 1 bio must always submit the
483  * previously allocated bio for IO before attempting to allocate a new one.
484  * Failure to do so can cause deadlocks under memory pressure.
485  *
486  * Note that when running under submit_bio_noacct() (i.e. any block driver),
487  * bios are not submitted until after you return - see the code in
488  * submit_bio_noacct() that converts recursion into iteration, to prevent
489  * stack overflows.
490  *
491  * This would normally mean allocating multiple bios under submit_bio_noacct()
492  * would be susceptible to deadlocks, but we have
493  * deadlock avoidance code that resubmits any blocked bios from a rescuer
494  * thread.
495  *
496  * However, we do not guarantee forward progress for allocations from other
497  * mempools. Doing multiple allocations from the same mempool under
498  * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
499  * for per bio allocations.
500  *
501  * Returns: Pointer to new bio on success, NULL on failure.
502  */
503 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
504 			     blk_opf_t opf, gfp_t gfp_mask,
505 			     struct bio_set *bs)
506 {
507 	gfp_t saved_gfp = gfp_mask;
508 	struct bio *bio;
509 	void *p;
510 
511 	/* should not use nobvec bioset for nr_vecs > 0 */
512 	if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0))
513 		return NULL;
514 
515 	if (opf & REQ_ALLOC_CACHE) {
516 		if (bs->cache && nr_vecs <= BIO_INLINE_VECS) {
517 			bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf,
518 						     gfp_mask, bs);
519 			if (bio)
520 				return bio;
521 			/*
522 			 * No cached bio available, bio returned below marked with
523 			 * REQ_ALLOC_CACHE to particpate in per-cpu alloc cache.
524 			 */
525 		} else {
526 			opf &= ~REQ_ALLOC_CACHE;
527 		}
528 	}
529 
530 	/*
531 	 * submit_bio_noacct() converts recursion to iteration; this means if
532 	 * we're running beneath it, any bios we allocate and submit will not be
533 	 * submitted (and thus freed) until after we return.
534 	 *
535 	 * This exposes us to a potential deadlock if we allocate multiple bios
536 	 * from the same bio_set() while running underneath submit_bio_noacct().
537 	 * If we were to allocate multiple bios (say a stacking block driver
538 	 * that was splitting bios), we would deadlock if we exhausted the
539 	 * mempool's reserve.
540 	 *
541 	 * We solve this, and guarantee forward progress, with a rescuer
542 	 * workqueue per bio_set. If we go to allocate and there are bios on
543 	 * current->bio_list, we first try the allocation without
544 	 * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be
545 	 * blocking to the rescuer workqueue before we retry with the original
546 	 * gfp_flags.
547 	 */
548 	if (current->bio_list &&
549 	    (!bio_list_empty(&current->bio_list[0]) ||
550 	     !bio_list_empty(&current->bio_list[1])) &&
551 	    bs->rescue_workqueue)
552 		gfp_mask &= ~__GFP_DIRECT_RECLAIM;
553 
554 	p = mempool_alloc(&bs->bio_pool, gfp_mask);
555 	if (!p && gfp_mask != saved_gfp) {
556 		punt_bios_to_rescuer(bs);
557 		gfp_mask = saved_gfp;
558 		p = mempool_alloc(&bs->bio_pool, gfp_mask);
559 	}
560 	if (unlikely(!p))
561 		return NULL;
562 	if (!mempool_is_saturated(&bs->bio_pool))
563 		opf &= ~REQ_ALLOC_CACHE;
564 
565 	bio = p + bs->front_pad;
566 	if (nr_vecs > BIO_INLINE_VECS) {
567 		struct bio_vec *bvl = NULL;
568 
569 		bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
570 		if (!bvl && gfp_mask != saved_gfp) {
571 			punt_bios_to_rescuer(bs);
572 			gfp_mask = saved_gfp;
573 			bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
574 		}
575 		if (unlikely(!bvl))
576 			goto err_free;
577 
578 		bio_init(bio, bdev, bvl, nr_vecs, opf);
579 	} else if (nr_vecs) {
580 		bio_init(bio, bdev, bio->bi_inline_vecs, BIO_INLINE_VECS, opf);
581 	} else {
582 		bio_init(bio, bdev, NULL, 0, opf);
583 	}
584 
585 	bio->bi_pool = bs;
586 	return bio;
587 
588 err_free:
589 	mempool_free(p, &bs->bio_pool);
590 	return NULL;
591 }
592 EXPORT_SYMBOL(bio_alloc_bioset);
593 
594 /**
595  * bio_kmalloc - kmalloc a bio
596  * @nr_vecs:	number of bio_vecs to allocate
597  * @gfp_mask:   the GFP_* mask given to the slab allocator
598  *
599  * Use kmalloc to allocate a bio (including bvecs).  The bio must be initialized
600  * using bio_init() before use.  To free a bio returned from this function use
601  * kfree() after calling bio_uninit().  A bio returned from this function can
602  * be reused by calling bio_uninit() before calling bio_init() again.
603  *
604  * Note that unlike bio_alloc() or bio_alloc_bioset() allocations from this
605  * function are not backed by a mempool can fail.  Do not use this function
606  * for allocations in the file system I/O path.
607  *
608  * Returns: Pointer to new bio on success, NULL on failure.
609  */
610 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask)
611 {
612 	struct bio *bio;
613 
614 	if (nr_vecs > UIO_MAXIOV)
615 		return NULL;
616 	return kmalloc(struct_size(bio, bi_inline_vecs, nr_vecs), gfp_mask);
617 }
618 EXPORT_SYMBOL(bio_kmalloc);
619 
620 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
621 {
622 	struct bio_vec bv;
623 	struct bvec_iter iter;
624 
625 	__bio_for_each_segment(bv, bio, iter, start)
626 		memzero_bvec(&bv);
627 }
628 EXPORT_SYMBOL(zero_fill_bio_iter);
629 
630 /**
631  * bio_truncate - truncate the bio to small size of @new_size
632  * @bio:	the bio to be truncated
633  * @new_size:	new size for truncating the bio
634  *
635  * Description:
636  *   Truncate the bio to new size of @new_size. If bio_op(bio) is
637  *   REQ_OP_READ, zero the truncated part. This function should only
638  *   be used for handling corner cases, such as bio eod.
639  */
640 static void bio_truncate(struct bio *bio, unsigned new_size)
641 {
642 	struct bio_vec bv;
643 	struct bvec_iter iter;
644 	unsigned int done = 0;
645 	bool truncated = false;
646 
647 	if (new_size >= bio->bi_iter.bi_size)
648 		return;
649 
650 	if (bio_op(bio) != REQ_OP_READ)
651 		goto exit;
652 
653 	bio_for_each_segment(bv, bio, iter) {
654 		if (done + bv.bv_len > new_size) {
655 			unsigned offset;
656 
657 			if (!truncated)
658 				offset = new_size - done;
659 			else
660 				offset = 0;
661 			zero_user(bv.bv_page, bv.bv_offset + offset,
662 				  bv.bv_len - offset);
663 			truncated = true;
664 		}
665 		done += bv.bv_len;
666 	}
667 
668  exit:
669 	/*
670 	 * Don't touch bvec table here and make it really immutable, since
671 	 * fs bio user has to retrieve all pages via bio_for_each_segment_all
672 	 * in its .end_bio() callback.
673 	 *
674 	 * It is enough to truncate bio by updating .bi_size since we can make
675 	 * correct bvec with the updated .bi_size for drivers.
676 	 */
677 	bio->bi_iter.bi_size = new_size;
678 }
679 
680 /**
681  * guard_bio_eod - truncate a BIO to fit the block device
682  * @bio:	bio to truncate
683  *
684  * This allows us to do IO even on the odd last sectors of a device, even if the
685  * block size is some multiple of the physical sector size.
686  *
687  * We'll just truncate the bio to the size of the device, and clear the end of
688  * the buffer head manually.  Truly out-of-range accesses will turn into actual
689  * I/O errors, this only handles the "we need to be able to do I/O at the final
690  * sector" case.
691  */
692 void guard_bio_eod(struct bio *bio)
693 {
694 	sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
695 
696 	if (!maxsector)
697 		return;
698 
699 	/*
700 	 * If the *whole* IO is past the end of the device,
701 	 * let it through, and the IO layer will turn it into
702 	 * an EIO.
703 	 */
704 	if (unlikely(bio->bi_iter.bi_sector >= maxsector))
705 		return;
706 
707 	maxsector -= bio->bi_iter.bi_sector;
708 	if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
709 		return;
710 
711 	bio_truncate(bio, maxsector << 9);
712 }
713 
714 static int __bio_alloc_cache_prune(struct bio_alloc_cache *cache,
715 				   unsigned int nr)
716 {
717 	unsigned int i = 0;
718 	struct bio *bio;
719 
720 	while ((bio = cache->free_list) != NULL) {
721 		cache->free_list = bio->bi_next;
722 		cache->nr--;
723 		bio_free(bio);
724 		if (++i == nr)
725 			break;
726 	}
727 	return i;
728 }
729 
730 static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
731 				  unsigned int nr)
732 {
733 	nr -= __bio_alloc_cache_prune(cache, nr);
734 	if (!READ_ONCE(cache->free_list)) {
735 		bio_alloc_irq_cache_splice(cache);
736 		__bio_alloc_cache_prune(cache, nr);
737 	}
738 }
739 
740 static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node)
741 {
742 	struct bio_set *bs;
743 
744 	bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead);
745 	if (bs->cache) {
746 		struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);
747 
748 		bio_alloc_cache_prune(cache, -1U);
749 	}
750 	return 0;
751 }
752 
753 static void bio_alloc_cache_destroy(struct bio_set *bs)
754 {
755 	int cpu;
756 
757 	if (!bs->cache)
758 		return;
759 
760 	cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
761 	for_each_possible_cpu(cpu) {
762 		struct bio_alloc_cache *cache;
763 
764 		cache = per_cpu_ptr(bs->cache, cpu);
765 		bio_alloc_cache_prune(cache, -1U);
766 	}
767 	free_percpu(bs->cache);
768 	bs->cache = NULL;
769 }
770 
771 static inline void bio_put_percpu_cache(struct bio *bio)
772 {
773 	struct bio_alloc_cache *cache;
774 
775 	cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
776 	if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX)
777 		goto out_free;
778 
779 	if (in_task()) {
780 		bio_uninit(bio);
781 		bio->bi_next = cache->free_list;
782 		/* Not necessary but helps not to iopoll already freed bios */
783 		bio->bi_bdev = NULL;
784 		cache->free_list = bio;
785 		cache->nr++;
786 	} else if (in_hardirq()) {
787 		lockdep_assert_irqs_disabled();
788 
789 		bio_uninit(bio);
790 		bio->bi_next = cache->free_list_irq;
791 		cache->free_list_irq = bio;
792 		cache->nr_irq++;
793 	} else {
794 		goto out_free;
795 	}
796 	put_cpu();
797 	return;
798 out_free:
799 	put_cpu();
800 	bio_free(bio);
801 }
802 
803 /**
804  * bio_put - release a reference to a bio
805  * @bio:   bio to release reference to
806  *
807  * Description:
808  *   Put a reference to a &struct bio, either one you have gotten with
809  *   bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
810  **/
811 void bio_put(struct bio *bio)
812 {
813 	if (unlikely(bio_flagged(bio, BIO_REFFED))) {
814 		BUG_ON(!atomic_read(&bio->__bi_cnt));
815 		if (!atomic_dec_and_test(&bio->__bi_cnt))
816 			return;
817 	}
818 	if (bio->bi_opf & REQ_ALLOC_CACHE)
819 		bio_put_percpu_cache(bio);
820 	else
821 		bio_free(bio);
822 }
823 EXPORT_SYMBOL(bio_put);
824 
825 static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
826 {
827 	bio_set_flag(bio, BIO_CLONED);
828 	bio->bi_ioprio = bio_src->bi_ioprio;
829 	bio->bi_write_hint = bio_src->bi_write_hint;
830 	bio->bi_iter = bio_src->bi_iter;
831 
832 	if (bio->bi_bdev) {
833 		if (bio->bi_bdev == bio_src->bi_bdev &&
834 		    bio_flagged(bio_src, BIO_REMAPPED))
835 			bio_set_flag(bio, BIO_REMAPPED);
836 		bio_clone_blkg_association(bio, bio_src);
837 	}
838 
839 	if (bio_crypt_clone(bio, bio_src, gfp) < 0)
840 		return -ENOMEM;
841 	if (bio_integrity(bio_src) &&
842 	    bio_integrity_clone(bio, bio_src, gfp) < 0)
843 		return -ENOMEM;
844 	return 0;
845 }
846 
847 /**
848  * bio_alloc_clone - clone a bio that shares the original bio's biovec
849  * @bdev: block_device to clone onto
850  * @bio_src: bio to clone from
851  * @gfp: allocation priority
852  * @bs: bio_set to allocate from
853  *
854  * Allocate a new bio that is a clone of @bio_src. The caller owns the returned
855  * bio, but not the actual data it points to.
856  *
857  * The caller must ensure that the return bio is not freed before @bio_src.
858  */
859 struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
860 		gfp_t gfp, struct bio_set *bs)
861 {
862 	struct bio *bio;
863 
864 	bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs);
865 	if (!bio)
866 		return NULL;
867 
868 	if (__bio_clone(bio, bio_src, gfp) < 0) {
869 		bio_put(bio);
870 		return NULL;
871 	}
872 	bio->bi_io_vec = bio_src->bi_io_vec;
873 
874 	return bio;
875 }
876 EXPORT_SYMBOL(bio_alloc_clone);
877 
878 /**
879  * bio_init_clone - clone a bio that shares the original bio's biovec
880  * @bdev: block_device to clone onto
881  * @bio: bio to clone into
882  * @bio_src: bio to clone from
883  * @gfp: allocation priority
884  *
885  * Initialize a new bio in caller provided memory that is a clone of @bio_src.
886  * The caller owns the returned bio, but not the actual data it points to.
887  *
888  * The caller must ensure that @bio_src is not freed before @bio.
889  */
890 int bio_init_clone(struct block_device *bdev, struct bio *bio,
891 		struct bio *bio_src, gfp_t gfp)
892 {
893 	int ret;
894 
895 	bio_init(bio, bdev, bio_src->bi_io_vec, 0, bio_src->bi_opf);
896 	ret = __bio_clone(bio, bio_src, gfp);
897 	if (ret)
898 		bio_uninit(bio);
899 	return ret;
900 }
901 EXPORT_SYMBOL(bio_init_clone);
902 
903 /**
904  * bio_full - check if the bio is full
905  * @bio:	bio to check
906  * @len:	length of one segment to be added
907  *
908  * Return true if @bio is full and one segment with @len bytes can't be
909  * added to the bio, otherwise return false
910  */
911 static inline bool bio_full(struct bio *bio, unsigned len)
912 {
913 	if (bio->bi_vcnt >= bio->bi_max_vecs)
914 		return true;
915 	if (bio->bi_iter.bi_size > UINT_MAX - len)
916 		return true;
917 	return false;
918 }
919 
920 static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page,
921 		unsigned int len, unsigned int off, bool *same_page)
922 {
923 	size_t bv_end = bv->bv_offset + bv->bv_len;
924 	phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1;
925 	phys_addr_t page_addr = page_to_phys(page);
926 
927 	if (vec_end_addr + 1 != page_addr + off)
928 		return false;
929 	if (xen_domain() && !xen_biovec_phys_mergeable(bv, page))
930 		return false;
931 	if (!zone_device_pages_have_same_pgmap(bv->bv_page, page))
932 		return false;
933 
934 	*same_page = ((vec_end_addr & PAGE_MASK) == ((page_addr + off) &
935 		     PAGE_MASK));
936 	if (!*same_page) {
937 		if (IS_ENABLED(CONFIG_KMSAN))
938 			return false;
939 		if (bv->bv_page + bv_end / PAGE_SIZE != page + off / PAGE_SIZE)
940 			return false;
941 	}
942 
943 	bv->bv_len += len;
944 	return true;
945 }
946 
947 /*
948  * Try to merge a page into a segment, while obeying the hardware segment
949  * size limit.  This is not for normal read/write bios, but for passthrough
950  * or Zone Append operations that we can't split.
951  */
952 bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv,
953 		struct page *page, unsigned len, unsigned offset,
954 		bool *same_page)
955 {
956 	unsigned long mask = queue_segment_boundary(q);
957 	phys_addr_t addr1 = bvec_phys(bv);
958 	phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
959 
960 	if ((addr1 | mask) != (addr2 | mask))
961 		return false;
962 	if (len > queue_max_segment_size(q) - bv->bv_len)
963 		return false;
964 	return bvec_try_merge_page(bv, page, len, offset, same_page);
965 }
966 
967 /**
968  * bio_add_hw_page - attempt to add a page to a bio with hw constraints
969  * @q: the target queue
970  * @bio: destination bio
971  * @page: page to add
972  * @len: vec entry length
973  * @offset: vec entry offset
974  * @max_sectors: maximum number of sectors that can be added
975  * @same_page: return if the segment has been merged inside the same page
976  *
977  * Add a page to a bio while respecting the hardware max_sectors, max_segment
978  * and gap limitations.
979  */
980 int bio_add_hw_page(struct request_queue *q, struct bio *bio,
981 		struct page *page, unsigned int len, unsigned int offset,
982 		unsigned int max_sectors, bool *same_page)
983 {
984 	unsigned int max_size = max_sectors << SECTOR_SHIFT;
985 
986 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
987 		return 0;
988 
989 	len = min3(len, max_size, queue_max_segment_size(q));
990 	if (len > max_size - bio->bi_iter.bi_size)
991 		return 0;
992 
993 	if (bio->bi_vcnt > 0) {
994 		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
995 
996 		if (bvec_try_merge_hw_page(q, bv, page, len, offset,
997 				same_page)) {
998 			bio->bi_iter.bi_size += len;
999 			return len;
1000 		}
1001 
1002 		if (bio->bi_vcnt >=
1003 		    min(bio->bi_max_vecs, queue_max_segments(q)))
1004 			return 0;
1005 
1006 		/*
1007 		 * If the queue doesn't support SG gaps and adding this segment
1008 		 * would create a gap, disallow it.
1009 		 */
1010 		if (bvec_gap_to_prev(&q->limits, bv, offset))
1011 			return 0;
1012 	}
1013 
1014 	bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, offset);
1015 	bio->bi_vcnt++;
1016 	bio->bi_iter.bi_size += len;
1017 	return len;
1018 }
1019 
1020 /**
1021  * bio_add_hw_folio - attempt to add a folio to a bio with hw constraints
1022  * @q: the target queue
1023  * @bio: destination bio
1024  * @folio: folio to add
1025  * @len: vec entry length
1026  * @offset: vec entry offset in the folio
1027  * @max_sectors: maximum number of sectors that can be added
1028  * @same_page: return if the segment has been merged inside the same folio
1029  *
1030  * Add a folio to a bio while respecting the hardware max_sectors, max_segment
1031  * and gap limitations.
1032  */
1033 int bio_add_hw_folio(struct request_queue *q, struct bio *bio,
1034 		struct folio *folio, size_t len, size_t offset,
1035 		unsigned int max_sectors, bool *same_page)
1036 {
1037 	if (len > UINT_MAX || offset > UINT_MAX)
1038 		return 0;
1039 	return bio_add_hw_page(q, bio, folio_page(folio, 0), len, offset,
1040 			       max_sectors, same_page);
1041 }
1042 
1043 /**
1044  * bio_add_pc_page	- attempt to add page to passthrough bio
1045  * @q: the target queue
1046  * @bio: destination bio
1047  * @page: page to add
1048  * @len: vec entry length
1049  * @offset: vec entry offset
1050  *
1051  * Attempt to add a page to the bio_vec maplist. This can fail for a
1052  * number of reasons, such as the bio being full or target block device
1053  * limitations. The target block device must allow bio's up to PAGE_SIZE,
1054  * so it is always possible to add a single page to an empty bio.
1055  *
1056  * This should only be used by passthrough bios.
1057  */
1058 int bio_add_pc_page(struct request_queue *q, struct bio *bio,
1059 		struct page *page, unsigned int len, unsigned int offset)
1060 {
1061 	bool same_page = false;
1062 	return bio_add_hw_page(q, bio, page, len, offset,
1063 			queue_max_hw_sectors(q), &same_page);
1064 }
1065 EXPORT_SYMBOL(bio_add_pc_page);
1066 
1067 /**
1068  * bio_add_zone_append_page - attempt to add page to zone-append bio
1069  * @bio: destination bio
1070  * @page: page to add
1071  * @len: vec entry length
1072  * @offset: vec entry offset
1073  *
1074  * Attempt to add a page to the bio_vec maplist of a bio that will be submitted
1075  * for a zone-append request. This can fail for a number of reasons, such as the
1076  * bio being full or the target block device is not a zoned block device or
1077  * other limitations of the target block device. The target block device must
1078  * allow bio's up to PAGE_SIZE, so it is always possible to add a single page
1079  * to an empty bio.
1080  *
1081  * Returns: number of bytes added to the bio, or 0 in case of a failure.
1082  */
1083 int bio_add_zone_append_page(struct bio *bio, struct page *page,
1084 			     unsigned int len, unsigned int offset)
1085 {
1086 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1087 	bool same_page = false;
1088 
1089 	if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND))
1090 		return 0;
1091 
1092 	if (WARN_ON_ONCE(!bdev_is_zoned(bio->bi_bdev)))
1093 		return 0;
1094 
1095 	return bio_add_hw_page(q, bio, page, len, offset,
1096 			       queue_max_zone_append_sectors(q), &same_page);
1097 }
1098 EXPORT_SYMBOL_GPL(bio_add_zone_append_page);
1099 
1100 /**
1101  * __bio_add_page - add page(s) to a bio in a new segment
1102  * @bio: destination bio
1103  * @page: start page to add
1104  * @len: length of the data to add, may cross pages
1105  * @off: offset of the data relative to @page, may cross pages
1106  *
1107  * Add the data at @page + @off to @bio as a new bvec.  The caller must ensure
1108  * that @bio has space for another bvec.
1109  */
1110 void __bio_add_page(struct bio *bio, struct page *page,
1111 		unsigned int len, unsigned int off)
1112 {
1113 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
1114 	WARN_ON_ONCE(bio_full(bio, len));
1115 
1116 	bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off);
1117 	bio->bi_iter.bi_size += len;
1118 	bio->bi_vcnt++;
1119 }
1120 EXPORT_SYMBOL_GPL(__bio_add_page);
1121 
1122 /**
1123  *	bio_add_page	-	attempt to add page(s) to bio
1124  *	@bio: destination bio
1125  *	@page: start page to add
1126  *	@len: vec entry length, may cross pages
1127  *	@offset: vec entry offset relative to @page, may cross pages
1128  *
1129  *	Attempt to add page(s) to the bio_vec maplist. This will only fail
1130  *	if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
1131  */
1132 int bio_add_page(struct bio *bio, struct page *page,
1133 		 unsigned int len, unsigned int offset)
1134 {
1135 	bool same_page = false;
1136 
1137 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1138 		return 0;
1139 	if (bio->bi_iter.bi_size > UINT_MAX - len)
1140 		return 0;
1141 
1142 	if (bio->bi_vcnt > 0 &&
1143 	    bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1],
1144 				page, len, offset, &same_page)) {
1145 		bio->bi_iter.bi_size += len;
1146 		return len;
1147 	}
1148 
1149 	if (bio->bi_vcnt >= bio->bi_max_vecs)
1150 		return 0;
1151 	__bio_add_page(bio, page, len, offset);
1152 	return len;
1153 }
1154 EXPORT_SYMBOL(bio_add_page);
1155 
1156 void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
1157 			  size_t off)
1158 {
1159 	WARN_ON_ONCE(len > UINT_MAX);
1160 	WARN_ON_ONCE(off > UINT_MAX);
1161 	__bio_add_page(bio, &folio->page, len, off);
1162 }
1163 EXPORT_SYMBOL_GPL(bio_add_folio_nofail);
1164 
1165 /**
1166  * bio_add_folio - Attempt to add part of a folio to a bio.
1167  * @bio: BIO to add to.
1168  * @folio: Folio to add.
1169  * @len: How many bytes from the folio to add.
1170  * @off: First byte in this folio to add.
1171  *
1172  * Filesystems that use folios can call this function instead of calling
1173  * bio_add_page() for each page in the folio.  If @off is bigger than
1174  * PAGE_SIZE, this function can create a bio_vec that starts in a page
1175  * after the bv_page.  BIOs do not support folios that are 4GiB or larger.
1176  *
1177  * Return: Whether the addition was successful.
1178  */
1179 bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len,
1180 		   size_t off)
1181 {
1182 	if (len > UINT_MAX || off > UINT_MAX)
1183 		return false;
1184 	return bio_add_page(bio, &folio->page, len, off) > 0;
1185 }
1186 EXPORT_SYMBOL(bio_add_folio);
1187 
1188 void __bio_release_pages(struct bio *bio, bool mark_dirty)
1189 {
1190 	struct folio_iter fi;
1191 
1192 	bio_for_each_folio_all(fi, bio) {
1193 		size_t nr_pages;
1194 
1195 		if (mark_dirty) {
1196 			folio_lock(fi.folio);
1197 			folio_mark_dirty(fi.folio);
1198 			folio_unlock(fi.folio);
1199 		}
1200 		nr_pages = (fi.offset + fi.length - 1) / PAGE_SIZE -
1201 			   fi.offset / PAGE_SIZE + 1;
1202 		unpin_user_folio(fi.folio, nr_pages);
1203 	}
1204 }
1205 EXPORT_SYMBOL_GPL(__bio_release_pages);
1206 
1207 void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
1208 {
1209 	size_t size = iov_iter_count(iter);
1210 
1211 	WARN_ON_ONCE(bio->bi_max_vecs);
1212 
1213 	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
1214 		struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1215 		size_t max_sectors = queue_max_zone_append_sectors(q);
1216 
1217 		size = min(size, max_sectors << SECTOR_SHIFT);
1218 	}
1219 
1220 	bio->bi_vcnt = iter->nr_segs;
1221 	bio->bi_io_vec = (struct bio_vec *)iter->bvec;
1222 	bio->bi_iter.bi_bvec_done = iter->iov_offset;
1223 	bio->bi_iter.bi_size = size;
1224 	bio_set_flag(bio, BIO_CLONED);
1225 }
1226 
1227 static int bio_iov_add_folio(struct bio *bio, struct folio *folio, size_t len,
1228 			     size_t offset)
1229 {
1230 	bool same_page = false;
1231 
1232 	if (WARN_ON_ONCE(bio->bi_iter.bi_size > UINT_MAX - len))
1233 		return -EIO;
1234 
1235 	if (bio->bi_vcnt > 0 &&
1236 	    bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1],
1237 				folio_page(folio, 0), len, offset,
1238 				&same_page)) {
1239 		bio->bi_iter.bi_size += len;
1240 		if (same_page && bio_flagged(bio, BIO_PAGE_PINNED))
1241 			unpin_user_folio(folio, 1);
1242 		return 0;
1243 	}
1244 	bio_add_folio_nofail(bio, folio, len, offset);
1245 	return 0;
1246 }
1247 
1248 static int bio_iov_add_zone_append_folio(struct bio *bio, struct folio *folio,
1249 					 size_t len, size_t offset)
1250 {
1251 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1252 	bool same_page = false;
1253 
1254 	if (bio_add_hw_folio(q, bio, folio, len, offset,
1255 			queue_max_zone_append_sectors(q), &same_page) != len)
1256 		return -EINVAL;
1257 	if (same_page && bio_flagged(bio, BIO_PAGE_PINNED))
1258 		unpin_user_folio(folio, 1);
1259 	return 0;
1260 }
1261 
1262 static unsigned int get_contig_folio_len(unsigned int *num_pages,
1263 					 struct page **pages, unsigned int i,
1264 					 struct folio *folio, size_t left,
1265 					 size_t offset)
1266 {
1267 	size_t bytes = left;
1268 	size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, bytes);
1269 	unsigned int j;
1270 
1271 	/*
1272 	 * We might COW a single page in the middle of
1273 	 * a large folio, so we have to check that all
1274 	 * pages belong to the same folio.
1275 	 */
1276 	bytes -= contig_sz;
1277 	for (j = i + 1; j < i + *num_pages; j++) {
1278 		size_t next = min_t(size_t, PAGE_SIZE, bytes);
1279 
1280 		if (page_folio(pages[j]) != folio ||
1281 		    pages[j] != pages[j - 1] + 1) {
1282 			break;
1283 		}
1284 		contig_sz += next;
1285 		bytes -= next;
1286 	}
1287 	*num_pages = j - i;
1288 
1289 	return contig_sz;
1290 }
1291 
1292 #define PAGE_PTRS_PER_BVEC     (sizeof(struct bio_vec) / sizeof(struct page *))
1293 
1294 /**
1295  * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
1296  * @bio: bio to add pages to
1297  * @iter: iov iterator describing the region to be mapped
1298  *
1299  * Extracts pages from *iter and appends them to @bio's bvec array.  The pages
1300  * will have to be cleaned up in the way indicated by the BIO_PAGE_PINNED flag.
1301  * For a multi-segment *iter, this function only adds pages from the next
1302  * non-empty segment of the iov iterator.
1303  */
1304 static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
1305 {
1306 	iov_iter_extraction_t extraction_flags = 0;
1307 	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
1308 	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
1309 	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
1310 	struct page **pages = (struct page **)bv;
1311 	ssize_t size;
1312 	unsigned int num_pages, i = 0;
1313 	size_t offset, folio_offset, left, len;
1314 	int ret = 0;
1315 
1316 	/*
1317 	 * Move page array up in the allocated memory for the bio vecs as far as
1318 	 * possible so that we can start filling biovecs from the beginning
1319 	 * without overwriting the temporary page array.
1320 	 */
1321 	BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
1322 	pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
1323 
1324 	if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue))
1325 		extraction_flags |= ITER_ALLOW_P2PDMA;
1326 
1327 	/*
1328 	 * Each segment in the iov is required to be a block size multiple.
1329 	 * However, we may not be able to get the entire segment if it spans
1330 	 * more pages than bi_max_vecs allows, so we have to ALIGN_DOWN the
1331 	 * result to ensure the bio's total size is correct. The remainder of
1332 	 * the iov data will be picked up in the next bio iteration.
1333 	 */
1334 	size = iov_iter_extract_pages(iter, &pages,
1335 				      UINT_MAX - bio->bi_iter.bi_size,
1336 				      nr_pages, extraction_flags, &offset);
1337 	if (unlikely(size <= 0))
1338 		return size ? size : -EFAULT;
1339 
1340 	nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE);
1341 
1342 	if (bio->bi_bdev) {
1343 		size_t trim = size & (bdev_logical_block_size(bio->bi_bdev) - 1);
1344 		iov_iter_revert(iter, trim);
1345 		size -= trim;
1346 	}
1347 
1348 	if (unlikely(!size)) {
1349 		ret = -EFAULT;
1350 		goto out;
1351 	}
1352 
1353 	for (left = size, i = 0; left > 0; left -= len, i += num_pages) {
1354 		struct page *page = pages[i];
1355 		struct folio *folio = page_folio(page);
1356 
1357 		folio_offset = ((size_t)folio_page_idx(folio, page) <<
1358 			       PAGE_SHIFT) + offset;
1359 
1360 		len = min(folio_size(folio) - folio_offset, left);
1361 
1362 		num_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
1363 
1364 		if (num_pages > 1)
1365 			len = get_contig_folio_len(&num_pages, pages, i,
1366 						   folio, left, offset);
1367 
1368 		if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
1369 			ret = bio_iov_add_zone_append_folio(bio, folio, len,
1370 					folio_offset);
1371 			if (ret)
1372 				break;
1373 		} else
1374 			bio_iov_add_folio(bio, folio, len, folio_offset);
1375 
1376 		offset = 0;
1377 	}
1378 
1379 	iov_iter_revert(iter, left);
1380 out:
1381 	while (i < nr_pages)
1382 		bio_release_page(bio, pages[i++]);
1383 
1384 	return ret;
1385 }
1386 
1387 /**
1388  * bio_iov_iter_get_pages - add user or kernel pages to a bio
1389  * @bio: bio to add pages to
1390  * @iter: iov iterator describing the region to be added
1391  *
1392  * This takes either an iterator pointing to user memory, or one pointing to
1393  * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
1394  * map them into the kernel. On IO completion, the caller should put those
1395  * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided
1396  * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs
1397  * to ensure the bvecs and pages stay referenced until the submitted I/O is
1398  * completed by a call to ->ki_complete() or returns with an error other than
1399  * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF
1400  * on IO completion. If it isn't, then pages should be released.
1401  *
1402  * The function tries, but does not guarantee, to pin as many pages as
1403  * fit into the bio, or are requested in @iter, whatever is smaller. If
1404  * MM encounters an error pinning the requested pages, it stops. Error
1405  * is returned only if 0 pages could be pinned.
1406  */
1407 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
1408 {
1409 	int ret = 0;
1410 
1411 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1412 		return -EIO;
1413 
1414 	if (iov_iter_is_bvec(iter)) {
1415 		bio_iov_bvec_set(bio, iter);
1416 		iov_iter_advance(iter, bio->bi_iter.bi_size);
1417 		return 0;
1418 	}
1419 
1420 	if (iov_iter_extract_will_pin(iter))
1421 		bio_set_flag(bio, BIO_PAGE_PINNED);
1422 	do {
1423 		ret = __bio_iov_iter_get_pages(bio, iter);
1424 	} while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
1425 
1426 	return bio->bi_vcnt ? 0 : ret;
1427 }
1428 EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
1429 
1430 static void submit_bio_wait_endio(struct bio *bio)
1431 {
1432 	complete(bio->bi_private);
1433 }
1434 
1435 /**
1436  * submit_bio_wait - submit a bio, and wait until it completes
1437  * @bio: The &struct bio which describes the I/O
1438  *
1439  * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
1440  * bio_endio() on failure.
1441  *
1442  * WARNING: Unlike to how submit_bio() is usually used, this function does not
1443  * result in bio reference to be consumed. The caller must drop the reference
1444  * on his own.
1445  */
1446 int submit_bio_wait(struct bio *bio)
1447 {
1448 	DECLARE_COMPLETION_ONSTACK_MAP(done,
1449 			bio->bi_bdev->bd_disk->lockdep_map);
1450 
1451 	bio->bi_private = &done;
1452 	bio->bi_end_io = submit_bio_wait_endio;
1453 	bio->bi_opf |= REQ_SYNC;
1454 	submit_bio(bio);
1455 	blk_wait_io(&done);
1456 
1457 	return blk_status_to_errno(bio->bi_status);
1458 }
1459 EXPORT_SYMBOL(submit_bio_wait);
1460 
1461 static void bio_wait_end_io(struct bio *bio)
1462 {
1463 	complete(bio->bi_private);
1464 	bio_put(bio);
1465 }
1466 
1467 /*
1468  * bio_await_chain - ends @bio and waits for every chained bio to complete
1469  */
1470 void bio_await_chain(struct bio *bio)
1471 {
1472 	DECLARE_COMPLETION_ONSTACK_MAP(done,
1473 			bio->bi_bdev->bd_disk->lockdep_map);
1474 
1475 	bio->bi_private = &done;
1476 	bio->bi_end_io = bio_wait_end_io;
1477 	bio_endio(bio);
1478 	blk_wait_io(&done);
1479 }
1480 
1481 void __bio_advance(struct bio *bio, unsigned bytes)
1482 {
1483 	if (bio_integrity(bio))
1484 		bio_integrity_advance(bio, bytes);
1485 
1486 	bio_crypt_advance(bio, bytes);
1487 	bio_advance_iter(bio, &bio->bi_iter, bytes);
1488 }
1489 EXPORT_SYMBOL(__bio_advance);
1490 
1491 void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
1492 			struct bio *src, struct bvec_iter *src_iter)
1493 {
1494 	while (src_iter->bi_size && dst_iter->bi_size) {
1495 		struct bio_vec src_bv = bio_iter_iovec(src, *src_iter);
1496 		struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter);
1497 		unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len);
1498 		void *src_buf = bvec_kmap_local(&src_bv);
1499 		void *dst_buf = bvec_kmap_local(&dst_bv);
1500 
1501 		memcpy(dst_buf, src_buf, bytes);
1502 
1503 		kunmap_local(dst_buf);
1504 		kunmap_local(src_buf);
1505 
1506 		bio_advance_iter_single(src, src_iter, bytes);
1507 		bio_advance_iter_single(dst, dst_iter, bytes);
1508 	}
1509 }
1510 EXPORT_SYMBOL(bio_copy_data_iter);
1511 
1512 /**
1513  * bio_copy_data - copy contents of data buffers from one bio to another
1514  * @src: source bio
1515  * @dst: destination bio
1516  *
1517  * Stops when it reaches the end of either @src or @dst - that is, copies
1518  * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
1519  */
1520 void bio_copy_data(struct bio *dst, struct bio *src)
1521 {
1522 	struct bvec_iter src_iter = src->bi_iter;
1523 	struct bvec_iter dst_iter = dst->bi_iter;
1524 
1525 	bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
1526 }
1527 EXPORT_SYMBOL(bio_copy_data);
1528 
1529 void bio_free_pages(struct bio *bio)
1530 {
1531 	struct bio_vec *bvec;
1532 	struct bvec_iter_all iter_all;
1533 
1534 	bio_for_each_segment_all(bvec, bio, iter_all)
1535 		__free_page(bvec->bv_page);
1536 }
1537 EXPORT_SYMBOL(bio_free_pages);
1538 
1539 /*
1540  * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1541  * for performing direct-IO in BIOs.
1542  *
1543  * The problem is that we cannot run folio_mark_dirty() from interrupt context
1544  * because the required locks are not interrupt-safe.  So what we can do is to
1545  * mark the pages dirty _before_ performing IO.  And in interrupt context,
1546  * check that the pages are still dirty.   If so, fine.  If not, redirty them
1547  * in process context.
1548  *
1549  * Note that this code is very hard to test under normal circumstances because
1550  * direct-io pins the pages with get_user_pages().  This makes
1551  * is_page_cache_freeable return false, and the VM will not clean the pages.
1552  * But other code (eg, flusher threads) could clean the pages if they are mapped
1553  * pagecache.
1554  *
1555  * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1556  * deferred bio dirtying paths.
1557  */
1558 
1559 /*
1560  * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1561  */
1562 void bio_set_pages_dirty(struct bio *bio)
1563 {
1564 	struct folio_iter fi;
1565 
1566 	bio_for_each_folio_all(fi, bio) {
1567 		folio_lock(fi.folio);
1568 		folio_mark_dirty(fi.folio);
1569 		folio_unlock(fi.folio);
1570 	}
1571 }
1572 EXPORT_SYMBOL_GPL(bio_set_pages_dirty);
1573 
1574 /*
1575  * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1576  * If they are, then fine.  If, however, some pages are clean then they must
1577  * have been written out during the direct-IO read.  So we take another ref on
1578  * the BIO and re-dirty the pages in process context.
1579  *
1580  * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1581  * here on.  It will unpin each page and will run one bio_put() against the
1582  * BIO.
1583  */
1584 
1585 static void bio_dirty_fn(struct work_struct *work);
1586 
1587 static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1588 static DEFINE_SPINLOCK(bio_dirty_lock);
1589 static struct bio *bio_dirty_list;
1590 
1591 /*
1592  * This runs in process context
1593  */
1594 static void bio_dirty_fn(struct work_struct *work)
1595 {
1596 	struct bio *bio, *next;
1597 
1598 	spin_lock_irq(&bio_dirty_lock);
1599 	next = bio_dirty_list;
1600 	bio_dirty_list = NULL;
1601 	spin_unlock_irq(&bio_dirty_lock);
1602 
1603 	while ((bio = next) != NULL) {
1604 		next = bio->bi_private;
1605 
1606 		bio_release_pages(bio, true);
1607 		bio_put(bio);
1608 	}
1609 }
1610 
1611 void bio_check_pages_dirty(struct bio *bio)
1612 {
1613 	struct folio_iter fi;
1614 	unsigned long flags;
1615 
1616 	bio_for_each_folio_all(fi, bio) {
1617 		if (!folio_test_dirty(fi.folio))
1618 			goto defer;
1619 	}
1620 
1621 	bio_release_pages(bio, false);
1622 	bio_put(bio);
1623 	return;
1624 defer:
1625 	spin_lock_irqsave(&bio_dirty_lock, flags);
1626 	bio->bi_private = bio_dirty_list;
1627 	bio_dirty_list = bio;
1628 	spin_unlock_irqrestore(&bio_dirty_lock, flags);
1629 	schedule_work(&bio_dirty_work);
1630 }
1631 EXPORT_SYMBOL_GPL(bio_check_pages_dirty);
1632 
1633 static inline bool bio_remaining_done(struct bio *bio)
1634 {
1635 	/*
1636 	 * If we're not chaining, then ->__bi_remaining is always 1 and
1637 	 * we always end io on the first invocation.
1638 	 */
1639 	if (!bio_flagged(bio, BIO_CHAIN))
1640 		return true;
1641 
1642 	BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1643 
1644 	if (atomic_dec_and_test(&bio->__bi_remaining)) {
1645 		bio_clear_flag(bio, BIO_CHAIN);
1646 		return true;
1647 	}
1648 
1649 	return false;
1650 }
1651 
1652 /**
1653  * bio_endio - end I/O on a bio
1654  * @bio:	bio
1655  *
1656  * Description:
1657  *   bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1658  *   way to end I/O on a bio. No one should call bi_end_io() directly on a
1659  *   bio unless they own it and thus know that it has an end_io function.
1660  *
1661  *   bio_endio() can be called several times on a bio that has been chained
1662  *   using bio_chain().  The ->bi_end_io() function will only be called the
1663  *   last time.
1664  **/
1665 void bio_endio(struct bio *bio)
1666 {
1667 again:
1668 	if (!bio_remaining_done(bio))
1669 		return;
1670 	if (!bio_integrity_endio(bio))
1671 		return;
1672 
1673 	blk_zone_bio_endio(bio);
1674 
1675 	rq_qos_done_bio(bio);
1676 
1677 	if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1678 		trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio);
1679 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1680 	}
1681 
1682 	/*
1683 	 * Need to have a real endio function for chained bios, otherwise
1684 	 * various corner cases will break (like stacking block devices that
1685 	 * save/restore bi_end_io) - however, we want to avoid unbounded
1686 	 * recursion and blowing the stack. Tail call optimization would
1687 	 * handle this, but compiling with frame pointers also disables
1688 	 * gcc's sibling call optimization.
1689 	 */
1690 	if (bio->bi_end_io == bio_chain_endio) {
1691 		bio = __bio_chain_endio(bio);
1692 		goto again;
1693 	}
1694 
1695 #ifdef CONFIG_BLK_CGROUP
1696 	/*
1697 	 * Release cgroup info.  We shouldn't have to do this here, but quite
1698 	 * a few callers of bio_init fail to call bio_uninit, so we cover up
1699 	 * for that here at least for now.
1700 	 */
1701 	if (bio->bi_blkg) {
1702 		blkg_put(bio->bi_blkg);
1703 		bio->bi_blkg = NULL;
1704 	}
1705 #endif
1706 
1707 	if (bio->bi_end_io)
1708 		bio->bi_end_io(bio);
1709 }
1710 EXPORT_SYMBOL(bio_endio);
1711 
1712 /**
1713  * bio_split - split a bio
1714  * @bio:	bio to split
1715  * @sectors:	number of sectors to split from the front of @bio
1716  * @gfp:	gfp mask
1717  * @bs:		bio set to allocate from
1718  *
1719  * Allocates and returns a new bio which represents @sectors from the start of
1720  * @bio, and updates @bio to represent the remaining sectors.
1721  *
1722  * Unless this is a discard request the newly allocated bio will point
1723  * to @bio's bi_io_vec. It is the caller's responsibility to ensure that
1724  * neither @bio nor @bs are freed before the split bio.
1725  */
1726 struct bio *bio_split(struct bio *bio, int sectors,
1727 		      gfp_t gfp, struct bio_set *bs)
1728 {
1729 	struct bio *split;
1730 
1731 	BUG_ON(sectors <= 0);
1732 	BUG_ON(sectors >= bio_sectors(bio));
1733 
1734 	/* Zone append commands cannot be split */
1735 	if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
1736 		return NULL;
1737 
1738 	split = bio_alloc_clone(bio->bi_bdev, bio, gfp, bs);
1739 	if (!split)
1740 		return NULL;
1741 
1742 	split->bi_iter.bi_size = sectors << 9;
1743 
1744 	if (bio_integrity(split))
1745 		bio_integrity_trim(split);
1746 
1747 	bio_advance(bio, split->bi_iter.bi_size);
1748 
1749 	if (bio_flagged(bio, BIO_TRACE_COMPLETION))
1750 		bio_set_flag(split, BIO_TRACE_COMPLETION);
1751 
1752 	return split;
1753 }
1754 EXPORT_SYMBOL(bio_split);
1755 
1756 /**
1757  * bio_trim - trim a bio
1758  * @bio:	bio to trim
1759  * @offset:	number of sectors to trim from the front of @bio
1760  * @size:	size we want to trim @bio to, in sectors
1761  *
1762  * This function is typically used for bios that are cloned and submitted
1763  * to the underlying device in parts.
1764  */
1765 void bio_trim(struct bio *bio, sector_t offset, sector_t size)
1766 {
1767 	if (WARN_ON_ONCE(offset > BIO_MAX_SECTORS || size > BIO_MAX_SECTORS ||
1768 			 offset + size > bio_sectors(bio)))
1769 		return;
1770 
1771 	size <<= 9;
1772 	if (offset == 0 && size == bio->bi_iter.bi_size)
1773 		return;
1774 
1775 	bio_advance(bio, offset << 9);
1776 	bio->bi_iter.bi_size = size;
1777 
1778 	if (bio_integrity(bio))
1779 		bio_integrity_trim(bio);
1780 }
1781 EXPORT_SYMBOL_GPL(bio_trim);
1782 
1783 /*
1784  * create memory pools for biovec's in a bio_set.
1785  * use the global biovec slabs created for general use.
1786  */
1787 int biovec_init_pool(mempool_t *pool, int pool_entries)
1788 {
1789 	struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
1790 
1791 	return mempool_init_slab_pool(pool, pool_entries, bp->slab);
1792 }
1793 
1794 /*
1795  * bioset_exit - exit a bioset initialized with bioset_init()
1796  *
1797  * May be called on a zeroed but uninitialized bioset (i.e. allocated with
1798  * kzalloc()).
1799  */
1800 void bioset_exit(struct bio_set *bs)
1801 {
1802 	bio_alloc_cache_destroy(bs);
1803 	if (bs->rescue_workqueue)
1804 		destroy_workqueue(bs->rescue_workqueue);
1805 	bs->rescue_workqueue = NULL;
1806 
1807 	mempool_exit(&bs->bio_pool);
1808 	mempool_exit(&bs->bvec_pool);
1809 
1810 	bioset_integrity_free(bs);
1811 	if (bs->bio_slab)
1812 		bio_put_slab(bs);
1813 	bs->bio_slab = NULL;
1814 }
1815 EXPORT_SYMBOL(bioset_exit);
1816 
1817 /**
1818  * bioset_init - Initialize a bio_set
1819  * @bs:		pool to initialize
1820  * @pool_size:	Number of bio and bio_vecs to cache in the mempool
1821  * @front_pad:	Number of bytes to allocate in front of the returned bio
1822  * @flags:	Flags to modify behavior, currently %BIOSET_NEED_BVECS
1823  *              and %BIOSET_NEED_RESCUER
1824  *
1825  * Description:
1826  *    Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1827  *    to ask for a number of bytes to be allocated in front of the bio.
1828  *    Front pad allocation is useful for embedding the bio inside
1829  *    another structure, to avoid allocating extra data to go with the bio.
1830  *    Note that the bio must be embedded at the END of that structure always,
1831  *    or things will break badly.
1832  *    If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
1833  *    for allocating iovecs.  This pool is not needed e.g. for bio_init_clone().
1834  *    If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used
1835  *    to dispatch queued requests when the mempool runs out of space.
1836  *
1837  */
1838 int bioset_init(struct bio_set *bs,
1839 		unsigned int pool_size,
1840 		unsigned int front_pad,
1841 		int flags)
1842 {
1843 	bs->front_pad = front_pad;
1844 	if (flags & BIOSET_NEED_BVECS)
1845 		bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1846 	else
1847 		bs->back_pad = 0;
1848 
1849 	spin_lock_init(&bs->rescue_lock);
1850 	bio_list_init(&bs->rescue_list);
1851 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1852 
1853 	bs->bio_slab = bio_find_or_create_slab(bs);
1854 	if (!bs->bio_slab)
1855 		return -ENOMEM;
1856 
1857 	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
1858 		goto bad;
1859 
1860 	if ((flags & BIOSET_NEED_BVECS) &&
1861 	    biovec_init_pool(&bs->bvec_pool, pool_size))
1862 		goto bad;
1863 
1864 	if (flags & BIOSET_NEED_RESCUER) {
1865 		bs->rescue_workqueue = alloc_workqueue("bioset",
1866 							WQ_MEM_RECLAIM, 0);
1867 		if (!bs->rescue_workqueue)
1868 			goto bad;
1869 	}
1870 	if (flags & BIOSET_PERCPU_CACHE) {
1871 		bs->cache = alloc_percpu(struct bio_alloc_cache);
1872 		if (!bs->cache)
1873 			goto bad;
1874 		cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
1875 	}
1876 
1877 	return 0;
1878 bad:
1879 	bioset_exit(bs);
1880 	return -ENOMEM;
1881 }
1882 EXPORT_SYMBOL(bioset_init);
1883 
1884 static int __init init_bio(void)
1885 {
1886 	int i;
1887 
1888 	BUILD_BUG_ON(BIO_FLAG_LAST > 8 * sizeof_field(struct bio, bi_flags));
1889 
1890 	bio_integrity_init();
1891 
1892 	for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
1893 		struct biovec_slab *bvs = bvec_slabs + i;
1894 
1895 		bvs->slab = kmem_cache_create(bvs->name,
1896 				bvs->nr_vecs * sizeof(struct bio_vec), 0,
1897 				SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
1898 	}
1899 
1900 	cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL,
1901 					bio_cpu_dead);
1902 
1903 	if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0,
1904 			BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE))
1905 		panic("bio: can't allocate bios\n");
1906 
1907 	if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
1908 		panic("bio: can't create integrity pool\n");
1909 
1910 	return 0;
1911 }
1912 subsys_initcall(init_bio);
1913