1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2001 Jens Axboe <axboe@suse.de> 4 */ 5 #ifndef __LINUX_BIO_H 6 #define __LINUX_BIO_H 7 8 #include <linux/mempool.h> 9 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ 10 #include <linux/blk_types.h> 11 #include <linux/uio.h> 12 13 #define BIO_MAX_VECS 256U 14 #define BIO_MAX_INLINE_VECS UIO_MAXIOV 15 16 struct queue_limits; 17 18 static inline unsigned int bio_max_segs(unsigned int nr_segs) 19 { 20 return min(nr_segs, BIO_MAX_VECS); 21 } 22 23 #define bio_iter_iovec(bio, iter) \ 24 bvec_iter_bvec((bio)->bi_io_vec, (iter)) 25 26 #define bio_iter_page(bio, iter) \ 27 bvec_iter_page((bio)->bi_io_vec, (iter)) 28 #define bio_iter_len(bio, iter) \ 29 bvec_iter_len((bio)->bi_io_vec, (iter)) 30 #define bio_iter_offset(bio, iter) \ 31 bvec_iter_offset((bio)->bi_io_vec, (iter)) 32 33 #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter) 34 #define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter) 35 #define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter) 36 37 #define bvec_iter_sectors(iter) ((iter).bi_size >> 9) 38 #define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter))) 39 40 #define bio_sectors(bio) bvec_iter_sectors((bio)->bi_iter) 41 #define bio_end_sector(bio) bvec_iter_end_sector((bio)->bi_iter) 42 43 /* 44 * Return the data direction, READ or WRITE. 45 */ 46 #define bio_data_dir(bio) \ 47 (op_is_write(bio_op(bio)) ? WRITE : READ) 48 49 static inline bool bio_flagged(const struct bio *bio, unsigned int bit) 50 { 51 return bio->bi_flags & (1U << bit); 52 } 53 54 static inline void bio_set_flag(struct bio *bio, unsigned int bit) 55 { 56 bio->bi_flags |= (1U << bit); 57 } 58 59 static inline void bio_clear_flag(struct bio *bio, unsigned int bit) 60 { 61 bio->bi_flags &= ~(1U << bit); 62 } 63 64 /* 65 * Check whether this bio carries any data or not. A NULL bio is allowed. 66 */ 67 static inline bool bio_has_data(struct bio *bio) 68 { 69 if (bio && 70 bio->bi_iter.bi_size && 71 bio_op(bio) != REQ_OP_DISCARD && 72 bio_op(bio) != REQ_OP_SECURE_ERASE && 73 bio_op(bio) != REQ_OP_WRITE_ZEROES) 74 return true; 75 76 return false; 77 } 78 79 static inline bool bio_no_advance_iter(const struct bio *bio) 80 { 81 return bio_op(bio) == REQ_OP_DISCARD || 82 bio_op(bio) == REQ_OP_SECURE_ERASE || 83 bio_op(bio) == REQ_OP_WRITE_ZEROES || 84 op_is_dmabuf(bio->bi_opf); 85 } 86 87 static inline void *bio_data(struct bio *bio) 88 { 89 if (bio_has_data(bio)) 90 return page_address(bio_page(bio)) + bio_offset(bio); 91 92 return NULL; 93 } 94 95 static inline bool bio_next_segment(const struct bio *bio, 96 struct bvec_iter_all *iter) 97 { 98 if (iter->idx >= bio->bi_vcnt) 99 return false; 100 101 bvec_advance(&bio->bi_io_vec[iter->idx], iter); 102 return true; 103 } 104 105 /* 106 * drivers should _never_ use the all version - the bio may have been split 107 * before it got to the driver and the driver won't own all of it 108 */ 109 #define bio_for_each_segment_all(bvl, bio, iter) \ 110 for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); ) 111 112 static inline void bio_advance_iter(const struct bio *bio, 113 struct bvec_iter *iter, unsigned int bytes) 114 { 115 iter->bi_sector += bytes >> 9; 116 117 if (bio_no_advance_iter(bio)) { 118 iter->bi_size -= bytes; 119 iter->bi_offset += bytes; 120 } else { 121 bvec_iter_advance(bio->bi_io_vec, iter, bytes); 122 /* TODO: It is reasonable to complete bio with error here. */ 123 } 124 } 125 126 /* @bytes should be less or equal to bvec[i->bi_idx].bv_len */ 127 static inline void bio_advance_iter_single(const struct bio *bio, 128 struct bvec_iter *iter, 129 unsigned int bytes) 130 { 131 iter->bi_sector += bytes >> 9; 132 133 if (bio_no_advance_iter(bio)) { 134 iter->bi_size -= bytes; 135 iter->bi_offset += bytes; 136 } else { 137 bvec_iter_advance_single(bio->bi_io_vec, iter, bytes); 138 } 139 } 140 141 void __bio_advance(struct bio *, unsigned bytes); 142 143 /** 144 * bio_advance - increment/complete a bio by some number of bytes 145 * @bio: bio to advance 146 * @nbytes: number of bytes to complete 147 * 148 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to 149 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will 150 * be updated on the last bvec as well. 151 * 152 * @bio will then represent the remaining, uncompleted portion of the io. 153 */ 154 static inline void bio_advance(struct bio *bio, unsigned int nbytes) 155 { 156 if (nbytes == bio->bi_iter.bi_size) { 157 bio->bi_iter.bi_size = 0; 158 return; 159 } 160 __bio_advance(bio, nbytes); 161 } 162 163 #define __bio_for_each_segment(bvl, bio, iter, start) \ 164 for (iter = (start); \ 165 (iter).bi_size && \ 166 ((bvl = bio_iter_iovec((bio), (iter))), 1); \ 167 bio_advance_iter_single((bio), &(iter), (bvl).bv_len)) 168 169 #define bio_for_each_segment(bvl, bio, iter) \ 170 __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter) 171 172 #define __bio_for_each_bvec(bvl, bio, iter, start) \ 173 for (iter = (start); \ 174 (iter).bi_size && \ 175 ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \ 176 bio_advance_iter_single((bio), &(iter), (bvl).bv_len)) 177 178 /* iterate over multi-page bvec */ 179 #define bio_for_each_bvec(bvl, bio, iter) \ 180 __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter) 181 182 /* 183 * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the 184 * same reasons as bio_for_each_segment_all(). 185 */ 186 #define bio_for_each_bvec_all(bvl, bio, i) \ 187 for (i = 0, bvl = bio_first_bvec_all(bio); \ 188 i < (bio)->bi_vcnt; i++, bvl++) 189 190 #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len) 191 192 static inline unsigned bio_segments(struct bio *bio) 193 { 194 unsigned segs = 0; 195 struct bio_vec bv; 196 struct bvec_iter iter; 197 198 /* 199 * We special case discard/write same/write zeroes, because they 200 * interpret bi_size differently: 201 */ 202 203 switch (bio_op(bio)) { 204 case REQ_OP_DISCARD: 205 case REQ_OP_SECURE_ERASE: 206 case REQ_OP_WRITE_ZEROES: 207 return 0; 208 default: 209 break; 210 } 211 212 bio_for_each_segment(bv, bio, iter) 213 segs++; 214 215 return segs; 216 } 217 218 /* 219 * get a reference to a bio, so it won't disappear. the intended use is 220 * something like: 221 * 222 * bio_get(bio); 223 * submit_bio(rw, bio); 224 * if (bio->bi_flags ...) 225 * do_something 226 * bio_put(bio); 227 * 228 * without the bio_get(), it could potentially complete I/O before submit_bio 229 * returns. and then bio would be freed memory when if (bio->bi_flags ...) 230 * runs 231 */ 232 static inline void bio_get(struct bio *bio) 233 { 234 bio->bi_flags |= (1 << BIO_REFFED); 235 smp_mb__before_atomic(); 236 atomic_inc(&bio->__bi_cnt); 237 } 238 239 static inline void bio_cnt_set(struct bio *bio, unsigned int count) 240 { 241 if (count != 1) { 242 bio->bi_flags |= (1 << BIO_REFFED); 243 smp_mb(); 244 } 245 atomic_set(&bio->__bi_cnt, count); 246 } 247 248 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio) 249 { 250 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 251 return bio->bi_io_vec; 252 } 253 254 static inline struct page *bio_first_page_all(struct bio *bio) 255 { 256 return bio_first_bvec_all(bio)->bv_page; 257 } 258 259 static inline struct folio *bio_first_folio_all(struct bio *bio) 260 { 261 return page_folio(bio_first_page_all(bio)); 262 } 263 264 /** 265 * struct folio_iter - State for iterating all folios in a bio. 266 * @folio: The current folio we're iterating. NULL after the last folio. 267 * @offset: The byte offset within the current folio. 268 * @length: The number of bytes in this iteration (will not cross folio 269 * boundary). 270 */ 271 struct folio_iter { 272 struct folio *folio; 273 size_t offset; 274 size_t length; 275 /* private: for use by the iterator */ 276 struct folio *_next; 277 size_t _seg_count; 278 int _i; 279 }; 280 281 static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio, 282 int i) 283 { 284 struct bio_vec *bvec = bio_first_bvec_all(bio) + i; 285 286 if (unlikely(i >= bio->bi_vcnt)) { 287 fi->folio = NULL; 288 return; 289 } 290 291 fi->folio = bvec_folio(bvec); 292 fi->offset = bvec->bv_offset + 293 PAGE_SIZE * folio_page_idx(fi->folio, bvec->bv_page); 294 fi->_seg_count = bvec->bv_len; 295 fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count); 296 fi->_next = folio_next(fi->folio); 297 fi->_i = i; 298 } 299 300 static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio) 301 { 302 fi->_seg_count -= fi->length; 303 if (fi->_seg_count) { 304 fi->folio = fi->_next; 305 fi->offset = 0; 306 fi->length = min(folio_size(fi->folio), fi->_seg_count); 307 fi->_next = folio_next(fi->folio); 308 } else { 309 bio_first_folio(fi, bio, fi->_i + 1); 310 } 311 } 312 313 /** 314 * bio_for_each_folio_all - Iterate over each folio in a bio. 315 * @fi: struct folio_iter which is updated for each folio. 316 * @bio: struct bio to iterate over. 317 */ 318 #define bio_for_each_folio_all(fi, bio) \ 319 for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio)) 320 321 void bio_trim(struct bio *bio, sector_t offset, sector_t size); 322 extern struct bio *bio_split(struct bio *bio, int sectors, 323 gfp_t gfp, struct bio_set *bs); 324 int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, 325 unsigned *segs, unsigned max_bytes, unsigned len_align); 326 u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next, 327 u8 gaps_bit); 328 329 /** 330 * bio_next_split - get next @sectors from a bio, splitting if necessary 331 * @bio: bio to split 332 * @sectors: number of sectors to split from the front of @bio 333 * @gfp: gfp mask 334 * @bs: bio set to allocate from 335 * 336 * Return: a bio representing the next @sectors of @bio - if the bio is smaller 337 * than @sectors, returns the original bio unchanged. 338 */ 339 static inline struct bio *bio_next_split(struct bio *bio, int sectors, 340 gfp_t gfp, struct bio_set *bs) 341 { 342 if (sectors >= bio_sectors(bio)) 343 return bio; 344 345 return bio_split(bio, sectors, gfp, bs); 346 } 347 348 enum { 349 BIOSET_NEED_BVECS = BIT(0), 350 BIOSET_NEED_RESCUER = BIT(1), 351 BIOSET_PERCPU_CACHE = BIT(2), 352 }; 353 extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags); 354 extern void bioset_exit(struct bio_set *); 355 356 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, 357 blk_opf_t opf, gfp_t gfp, struct bio_set *bs); 358 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask); 359 extern void bio_put(struct bio *); 360 361 struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, 362 gfp_t gfp, struct bio_set *bs); 363 int bio_init_clone(struct block_device *bdev, struct bio *bio, 364 struct bio *bio_src, gfp_t gfp); 365 366 extern struct bio_set fs_bio_set; 367 368 static inline struct bio *bio_alloc(struct block_device *bdev, 369 unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask) 370 { 371 return bio_alloc_bioset(bdev, nr_vecs, opf, gfp_mask, &fs_bio_set); 372 } 373 374 void submit_bio(struct bio *bio); 375 376 /** 377 * bio_in_atomic - check if the current context is unsafe for bio completion 378 * 379 * Return: %true in atomic contexts (e.g. hard/soft IRQ, preempt-disabled); 380 * %false when a bio can be safely completed in the current context. 381 */ 382 static inline bool bio_in_atomic(void) 383 { 384 if (IS_ENABLED(CONFIG_PREEMPTION) && rcu_preempt_depth()) 385 return true; 386 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 387 return true; 388 return !preemptible(); 389 } 390 391 void __bio_complete_in_task(struct bio *bio); 392 393 /** 394 * bio_complete_in_task - ensure a bio is completed in preemptible task context 395 * @bio: bio to complete 396 * 397 * If called from non-task context, offload the bio completion to a worker 398 * thread and return %true. Else return %false and do nothing. 399 * 400 * Uses BIO_COMPLETE_IN_TASK as a sentinel: if set, the bio was already 401 * deferred and we are running in the worker — return %false so the 402 * callback proceeds instead of re-deferring. 403 */ 404 static inline bool bio_complete_in_task(struct bio *bio) 405 { 406 if (bio_flagged(bio, BIO_COMPLETE_IN_TASK)) 407 return false; 408 if (!bio_in_atomic()) 409 return false; 410 bio_set_flag(bio, BIO_COMPLETE_IN_TASK); 411 __bio_complete_in_task(bio); 412 return true; 413 } 414 415 extern void bio_endio(struct bio *); 416 417 /** 418 * bio_endio_status - end I/O on a bio with a specific status 419 * @bio: bio 420 * @status: status to set 421 * 422 * Set @bio->bi_status to @status and call bio_endio(). 423 **/ 424 static inline void bio_endio_status(struct bio *bio, blk_status_t status) 425 { 426 bio->bi_status = status; 427 bio_endio(bio); 428 } 429 430 static inline void bio_io_error(struct bio *bio) 431 { 432 bio_endio_status(bio, BLK_STS_IOERR); 433 } 434 435 static inline void bio_wouldblock_error(struct bio *bio) 436 { 437 bio_endio_status(bio, BLK_STS_AGAIN); 438 } 439 440 /* 441 * Calculate number of bvec segments that should be allocated to fit data 442 * pointed by @iter. If @iter is backed by a bvec or a dmabuf, the bvec array / 443 * the dma map are going to be reused, and so no extra allocation is required. 444 */ 445 static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) 446 { 447 if (iov_iter_is_bvec(iter) || iov_iter_is_dmabuf_map(iter)) 448 return 0; 449 return iov_iter_npages(iter, max_segs); 450 } 451 452 /** 453 * bio_iov_bounce_nr_vecs - calculate number of bvecs for a bounce bio 454 * @iter: iter to bounce from 455 * @op: REQ_OP_* for the bio 456 * 457 * Calculates how many bvecs are needed for the next bio to bounce from/to 458 * @iter. 459 */ 460 static inline unsigned short 461 bio_iov_bounce_nr_vecs(struct iov_iter *iter, blk_opf_t op) 462 { 463 /* 464 * We still need to bounce bvec iters, so don't special case them 465 * here unlike in bio_iov_vecs_to_alloc. 466 * 467 * For reads we need to use a vector for the bounce buffer, account 468 * for that here. 469 */ 470 if (op_is_write(op)) 471 return iov_iter_npages(iter, BIO_MAX_VECS); 472 return iov_iter_npages(iter, BIO_MAX_VECS - 1) + 1; 473 } 474 475 struct request_queue; 476 477 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, 478 unsigned short max_vecs, blk_opf_t opf); 479 static inline void bio_init_inline(struct bio *bio, struct block_device *bdev, 480 unsigned short max_vecs, blk_opf_t opf) 481 { 482 bio_init(bio, bdev, bio_inline_vecs(bio), max_vecs, opf); 483 } 484 extern void bio_uninit(struct bio *); 485 void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf); 486 void bio_reuse(struct bio *bio, blk_opf_t opf); 487 void bio_prepare_reissue(struct bio *bio, struct block_device *bdev); 488 void bio_chain(struct bio *, struct bio *); 489 void bio_await(struct bio *bio, void *priv, 490 void (*submit)(struct bio *bio, void *priv)); 491 492 int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len, 493 unsigned off); 494 bool __must_check bio_add_folio(struct bio *bio, struct folio *folio, 495 size_t len, size_t off); 496 void __bio_add_page(struct bio *bio, struct page *page, 497 unsigned int len, unsigned int off); 498 void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len, 499 size_t off); 500 void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len); 501 502 /** 503 * bio_add_max_vecs - number of bio_vecs needed to add data to a bio 504 * @kaddr: kernel virtual address to add 505 * @len: length in bytes to add 506 * 507 * Calculate how many bio_vecs need to be allocated to add the kernel virtual 508 * address range in [@kaddr:@len] in the worse case. 509 */ 510 static inline unsigned int bio_add_max_vecs(void *kaddr, unsigned int len) 511 { 512 if (is_vmalloc_addr(kaddr)) 513 return DIV_ROUND_UP(offset_in_page(kaddr) + len, PAGE_SIZE); 514 return 1; 515 } 516 517 unsigned int bio_add_vmalloc_chunk(struct bio *bio, void *vaddr, unsigned len); 518 bool bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned int len); 519 520 int submit_bio_wait(struct bio *bio); 521 int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, 522 size_t len, enum req_op op); 523 524 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, 525 unsigned maxlen, unsigned mem_align_mask, 526 unsigned len_align_mask); 527 528 bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter); 529 void __bio_release_pages(struct bio *bio, bool mark_dirty); 530 extern void bio_set_pages_dirty(struct bio *bio); 531 extern void bio_check_pages_dirty(struct bio *bio); 532 533 int bio_alloc_bounce_folios(struct bio *bio, size_t total_len, size_t minsize); 534 void bio_free_folios(struct bio *bio); 535 int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, 536 size_t maxlen, size_t minsize); 537 538 extern void bio_copy_data(struct bio *dst, struct bio *src); 539 extern void bio_free_pages(struct bio *bio); 540 void zero_fill_bio(struct bio *bio); 541 void guard_bio_eod(struct bio *bio); 542 543 static inline void bio_release_pages(struct bio *bio, bool mark_dirty) 544 { 545 if (bio_flagged(bio, BIO_PAGE_PINNED)) 546 __bio_release_pages(bio, mark_dirty); 547 } 548 549 #define bio_dev(bio) \ 550 disk_devt((bio)->bi_bdev->bd_disk) 551 552 #ifdef CONFIG_BLK_CGROUP 553 void bio_associate_blkg(struct bio *bio); 554 void bio_associate_blkg_from_css(struct bio *bio, 555 struct cgroup_subsys_state *css); 556 void bio_clone_blkg_association(struct bio *dst, struct bio *src); 557 void blkcg_punt_bio_submit(struct bio *bio); 558 #else /* CONFIG_BLK_CGROUP */ 559 static inline void bio_associate_blkg(struct bio *bio) { } 560 static inline void bio_associate_blkg_from_css(struct bio *bio, 561 struct cgroup_subsys_state *css) 562 { } 563 static inline void bio_clone_blkg_association(struct bio *dst, 564 struct bio *src) { } 565 static inline void blkcg_punt_bio_submit(struct bio *bio) 566 { 567 submit_bio(bio); 568 } 569 #endif /* CONFIG_BLK_CGROUP */ 570 571 static inline void bio_set_dev(struct bio *bio, struct block_device *bdev) 572 { 573 bio_clear_flag(bio, BIO_REMAPPED); 574 if (bio->bi_bdev != bdev) 575 bio_clear_flag(bio, BIO_BPS_THROTTLED); 576 bio->bi_bdev = bdev; 577 bio_associate_blkg(bio); 578 } 579 580 /* 581 * BIO list management for use by remapping drivers (e.g. DM or MD) and loop. 582 * 583 * A bio_list anchors a singly-linked list of bios chained through the bi_next 584 * member of the bio. The bio_list also caches the last list member to allow 585 * fast access to the tail. 586 */ 587 struct bio_list { 588 struct bio *head; 589 struct bio *tail; 590 }; 591 592 static inline int bio_list_empty(const struct bio_list *bl) 593 { 594 return bl->head == NULL; 595 } 596 597 static inline void bio_list_init(struct bio_list *bl) 598 { 599 bl->head = bl->tail = NULL; 600 } 601 602 #define BIO_EMPTY_LIST { NULL, NULL } 603 604 #define bio_list_for_each(bio, bl) \ 605 for (bio = (bl)->head; bio; bio = bio->bi_next) 606 607 static inline unsigned bio_list_size(const struct bio_list *bl) 608 { 609 unsigned sz = 0; 610 struct bio *bio; 611 612 bio_list_for_each(bio, bl) 613 sz++; 614 615 return sz; 616 } 617 618 static inline void bio_list_add(struct bio_list *bl, struct bio *bio) 619 { 620 bio->bi_next = NULL; 621 622 if (bl->tail) 623 bl->tail->bi_next = bio; 624 else 625 bl->head = bio; 626 627 bl->tail = bio; 628 } 629 630 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio) 631 { 632 bio->bi_next = bl->head; 633 634 bl->head = bio; 635 636 if (!bl->tail) 637 bl->tail = bio; 638 } 639 640 static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2) 641 { 642 if (!bl2->head) 643 return; 644 645 if (bl->tail) 646 bl->tail->bi_next = bl2->head; 647 else 648 bl->head = bl2->head; 649 650 bl->tail = bl2->tail; 651 } 652 653 static inline void bio_list_merge_init(struct bio_list *bl, 654 struct bio_list *bl2) 655 { 656 bio_list_merge(bl, bl2); 657 bio_list_init(bl2); 658 } 659 660 static inline void bio_list_merge_head(struct bio_list *bl, 661 struct bio_list *bl2) 662 { 663 if (!bl2->head) 664 return; 665 666 if (bl->head) 667 bl2->tail->bi_next = bl->head; 668 else 669 bl->tail = bl2->tail; 670 671 bl->head = bl2->head; 672 } 673 674 static inline struct bio *bio_list_peek(struct bio_list *bl) 675 { 676 return bl->head; 677 } 678 679 static inline struct bio *bio_list_pop(struct bio_list *bl) 680 { 681 struct bio *bio = bl->head; 682 683 if (bio) { 684 bl->head = bl->head->bi_next; 685 if (!bl->head) 686 bl->tail = NULL; 687 688 bio->bi_next = NULL; 689 } 690 691 return bio; 692 } 693 694 static inline struct bio *bio_list_get(struct bio_list *bl) 695 { 696 struct bio *bio = bl->head; 697 698 bl->head = bl->tail = NULL; 699 700 return bio; 701 } 702 703 /* 704 * Increment chain count for the bio. Make sure the CHAIN flag update 705 * is visible before the raised count. 706 */ 707 static inline void bio_inc_remaining(struct bio *bio) 708 { 709 bio_set_flag(bio, BIO_CHAIN); 710 smp_mb__before_atomic(); 711 atomic_inc(&bio->__bi_remaining); 712 } 713 714 /* 715 * bio_set is used to allow other portions of the IO system to 716 * allocate their own private memory pools for bio and iovec structures. 717 * These memory pools in turn all allocate from the bio_slab 718 * and the bvec_slabs[]. 719 */ 720 #define BIO_POOL_SIZE 2 721 722 struct bio_set { 723 struct kmem_cache *bio_slab; 724 unsigned int front_pad; 725 726 /* 727 * per-cpu bio alloc cache 728 */ 729 struct bio_alloc_cache __percpu *cache; 730 731 mempool_t bio_pool; 732 mempool_t bvec_pool; 733 734 unsigned int back_pad; 735 /* 736 * Deadlock avoidance for stacking block drivers: see comments in 737 * bio_alloc_bioset() for details 738 */ 739 spinlock_t rescue_lock; 740 struct bio_list rescue_list; 741 struct work_struct rescue_work; 742 struct workqueue_struct *rescue_workqueue; 743 744 /* 745 * Hot un-plug notifier for the per-cpu cache, if used 746 */ 747 struct hlist_node cpuhp_dead; 748 }; 749 750 static inline bool bioset_initialized(struct bio_set *bs) 751 { 752 return bs->bio_slab != NULL; 753 } 754 755 static inline void bio_clear_polled(struct bio *bio) 756 { 757 bio->bi_opf &= ~REQ_POLLED; 758 } 759 760 /** 761 * bio_is_zone_append - is this a zone append bio? 762 * @bio: bio to check 763 * 764 * Check if @bio is a zone append operation. Core block layer code and end_io 765 * handlers must use this instead of an open coded REQ_OP_ZONE_APPEND check 766 * because the block layer can rewrite REQ_OP_ZONE_APPEND to REQ_OP_WRITE if 767 * it is not natively supported. 768 */ 769 static inline bool bio_is_zone_append(struct bio *bio) 770 { 771 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) 772 return false; 773 return bio_op(bio) == REQ_OP_ZONE_APPEND || 774 bio_flagged(bio, BIO_EMULATES_ZONE_APPEND); 775 } 776 777 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, 778 unsigned int nr_pages, blk_opf_t opf, gfp_t gfp); 779 struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new); 780 781 struct bio *blk_alloc_discard_bio(struct block_device *bdev, 782 sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask); 783 784 #endif /* __LINUX_BIO_H */ 785