bio.c (0cf41e5e9bafc185490624c3e321c915885a91f3) bio.c (c42bca92be928ce7dece5fc04cf68d0e37ee6718)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
4 */
5#include <linux/mm.h>
6#include <linux/swap.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>

--- 928 unchanged lines hidden (view full) ---

937 bio_for_each_segment_all(bvec, bio, iter_all) {
938 if (mark_dirty && !PageCompound(bvec->bv_page))
939 set_page_dirty_lock(bvec->bv_page);
940 put_page(bvec->bv_page);
941 }
942}
943EXPORT_SYMBOL_GPL(bio_release_pages);
944
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
4 */
5#include <linux/mm.h>
6#include <linux/swap.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>

--- 928 unchanged lines hidden (view full) ---

937 bio_for_each_segment_all(bvec, bio, iter_all) {
938 if (mark_dirty && !PageCompound(bvec->bv_page))
939 set_page_dirty_lock(bvec->bv_page);
940 put_page(bvec->bv_page);
941 }
942}
943EXPORT_SYMBOL_GPL(bio_release_pages);
944
945static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
945static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
946{
946{
947 const struct bio_vec *bv = iter->bvec;
948 unsigned int len;
949 size_t size;
947 WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0);
950
948
951 if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len))
952 return -EINVAL;
949 bio->bi_vcnt = iter->nr_segs;
950 bio->bi_max_vecs = iter->nr_segs;
951 bio->bi_io_vec = (struct bio_vec *)iter->bvec;
952 bio->bi_iter.bi_bvec_done = iter->iov_offset;
953 bio->bi_iter.bi_size = iter->count;
953
954
954 len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count);
955 size = bio_add_page(bio, bv->bv_page, len,
956 bv->bv_offset + iter->iov_offset);
957 if (unlikely(size != len))
958 return -EINVAL;
959 iov_iter_advance(iter, size);
955 iov_iter_advance(iter, iter->count);
960 return 0;
961}
962
963#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
964
965/**
966 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
967 * @bio: bio to add pages to

--- 97 unchanged lines hidden (view full) ---

1065/**
1066 * bio_iov_iter_get_pages - add user or kernel pages to a bio
1067 * @bio: bio to add pages to
1068 * @iter: iov iterator describing the region to be added
1069 *
1070 * This takes either an iterator pointing to user memory, or one pointing to
1071 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
1072 * map them into the kernel. On IO completion, the caller should put those
956 return 0;
957}
958
959#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
960
961/**
962 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
963 * @bio: bio to add pages to

--- 97 unchanged lines hidden (view full) ---

1061/**
1062 * bio_iov_iter_get_pages - add user or kernel pages to a bio
1063 * @bio: bio to add pages to
1064 * @iter: iov iterator describing the region to be added
1065 *
1066 * This takes either an iterator pointing to user memory, or one pointing to
1067 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
1068 * map them into the kernel. On IO completion, the caller should put those
1073 * pages. If we're adding kernel pages, and the caller told us it's safe to
1074 * do so, we just have to add the pages to the bio directly. We don't grab an
1075 * extra reference to those pages (the user should already have that), and we
1076 * don't put the page on IO completion. The caller needs to check if the bio is
1077 * flagged BIO_NO_PAGE_REF on IO completion. If it isn't, then pages should be
1078 * released.
1069 * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided
1070 * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs
1071 * to ensure the bvecs and pages stay referenced until the submitted I/O is
1072 * completed by a call to ->ki_complete() or returns with an error other than
1073 * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF
1074 * on IO completion. If it isn't, then pages should be released.
1079 *
1080 * The function tries, but does not guarantee, to pin as many pages as
1081 * fit into the bio, or are requested in @iter, whatever is smaller. If
1082 * MM encounters an error pinning the requested pages, it stops. Error
1083 * is returned only if 0 pages could be pinned.
1084 *
1085 * It's intended for direct IO, so doesn't do PSI tracking, the caller is
1086 * responsible for setting BIO_WORKINGSET if necessary.
1087 */
1088int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
1089{
1075 *
1076 * The function tries, but does not guarantee, to pin as many pages as
1077 * fit into the bio, or are requested in @iter, whatever is smaller. If
1078 * MM encounters an error pinning the requested pages, it stops. Error
1079 * is returned only if 0 pages could be pinned.
1080 *
1081 * It's intended for direct IO, so doesn't do PSI tracking, the caller is
1082 * responsible for setting BIO_WORKINGSET if necessary.
1083 */
1084int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
1085{
1090 const bool is_bvec = iov_iter_is_bvec(iter);
1091 int ret;
1086 int ret = 0;
1092
1087
1093 if (WARN_ON_ONCE(bio->bi_vcnt))
1094 return -EINVAL;
1095
1096 do {
1097 if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
1098 if (WARN_ON_ONCE(is_bvec))
1099 return -EINVAL;
1100 ret = __bio_iov_append_get_pages(bio, iter);
1101 } else {
1102 if (is_bvec)
1103 ret = __bio_iov_bvec_add_pages(bio, iter);
1088 if (iov_iter_is_bvec(iter)) {
1089 if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
1090 return -EINVAL;
1091 bio_iov_bvec_set(bio, iter);
1092 bio_set_flag(bio, BIO_NO_PAGE_REF);
1093 return 0;
1094 } else {
1095 do {
1096 if (bio_op(bio) == REQ_OP_ZONE_APPEND)
1097 ret = __bio_iov_append_get_pages(bio, iter);
1104 else
1105 ret = __bio_iov_iter_get_pages(bio, iter);
1098 else
1099 ret = __bio_iov_iter_get_pages(bio, iter);
1106 }
1107 } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
1100 } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
1101 }
1108
1102
1109 if (is_bvec)
1110 bio_set_flag(bio, BIO_NO_PAGE_REF);
1111
1112 /* don't account direct I/O as memory stall */
1113 bio_clear_flag(bio, BIO_WORKINGSET);
1114 return bio->bi_vcnt ? 0 : ret;
1115}
1116EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
1117
1118static void submit_bio_wait_endio(struct bio *bio)
1119{

--- 539 unchanged lines hidden --- 		1103 /* don't account direct I/O as memory stall */
1104 bio_clear_flag(bio, BIO_WORKINGSET);
1105 return bio->bi_vcnt ? 0 : ret;
1106}
1107EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
1108
1109static void submit_bio_wait_endio(struct bio *bio)
1110{

--- 539 unchanged lines hidden ---