1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Iterator helpers. 3 * 4 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/export.h> 9 #include <linux/slab.h> 10 #include <linux/mm.h> 11 #include <linux/uio.h> 12 #include <linux/scatterlist.h> 13 #include <linux/netfs.h> 14 #include "internal.h" 15 16 /** 17 * netfs_extract_user_iter - Extract the pages from a user iterator into a bvec 18 * @orig: The original iterator 19 * @orig_len: The amount of iterator to copy 20 * @new: The iterator to be set up 21 * @extraction_flags: Flags to qualify the request 22 * 23 * Extract the page fragments from the given amount of the source iterator and 24 * build up a second iterator that refers to all of those bits. This allows 25 * the original iterator to disposed of. 26 * 27 * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be 28 * allowed on the pages extracted. 29 * 30 * On success, the number of elements in the bvec is returned, the original 31 * iterator will have been advanced by the amount extracted. 32 * 33 * The iov_iter_extract_mode() function should be used to query how cleanup 34 * should be performed. 35 */ 36 ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, 37 struct iov_iter *new, 38 iov_iter_extraction_t extraction_flags) 39 { 40 struct bio_vec *bv = NULL; 41 struct page **pages; 42 unsigned int cur_npages; 43 unsigned int max_pages; 44 unsigned int npages = 0; 45 unsigned int i; 46 ssize_t ret; 47 size_t count = orig_len, offset, len; 48 size_t bv_size, pg_size; 49 50 if (WARN_ON_ONCE(!iter_is_ubuf(orig) && !iter_is_iovec(orig))) 51 return -EIO; 52 53 max_pages = iov_iter_npages(orig, INT_MAX); 54 bv_size = array_size(max_pages, sizeof(*bv)); 55 bv = kvmalloc(bv_size, GFP_KERNEL); 56 if (!bv) 57 return -ENOMEM; 58 59 /* Put the page list at the end of the bvec list storage. bvec 60 * elements are larger than page pointers, so as long as we work 61 * 0->last, we should be fine. 62 */ 63 pg_size = array_size(max_pages, sizeof(*pages)); 64 pages = (void *)bv + bv_size - pg_size; 65 66 while (count && npages < max_pages) { 67 ret = iov_iter_extract_pages(orig, &pages, count, 68 max_pages - npages, extraction_flags, 69 &offset); 70 if (ret < 0) { 71 pr_err("Couldn't get user pages (rc=%zd)\n", ret); 72 break; 73 } 74 75 if (ret > count) { 76 pr_err("get_pages rc=%zd more than %zu\n", ret, count); 77 break; 78 } 79 80 count -= ret; 81 ret += offset; 82 cur_npages = DIV_ROUND_UP(ret, PAGE_SIZE); 83 84 if (npages + cur_npages > max_pages) { 85 pr_err("Out of bvec array capacity (%u vs %u)\n", 86 npages + cur_npages, max_pages); 87 break; 88 } 89 90 for (i = 0; i < cur_npages; i++) { 91 len = ret > PAGE_SIZE ? PAGE_SIZE : ret; 92 bvec_set_page(bv + npages + i, *pages++, len - offset, offset); 93 ret -= len; 94 offset = 0; 95 } 96 97 npages += cur_npages; 98 } 99 100 iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count); 101 return npages; 102 } 103 EXPORT_SYMBOL_GPL(netfs_extract_user_iter); 104 105 /* 106 * Select the span of a bvec iterator we're going to use. Limit it by both maximum 107 * size and maximum number of segments. Returns the size of the span in bytes. 108 */ 109 static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset, 110 size_t max_size, size_t max_segs) 111 { 112 const struct bio_vec *bvecs = iter->bvec; 113 unsigned int nbv = iter->nr_segs, ix = 0, nsegs = 0; 114 size_t len, span = 0, n = iter->count; 115 size_t skip = iter->iov_offset + start_offset; 116 117 if (WARN_ON(!iov_iter_is_bvec(iter)) || 118 WARN_ON(start_offset > n) || 119 n == 0) 120 return 0; 121 122 while (n && ix < nbv && skip) { 123 len = bvecs[ix].bv_len; 124 if (skip < len) 125 break; 126 skip -= len; 127 n -= len; 128 ix++; 129 } 130 131 while (n && ix < nbv) { 132 len = min3(n, bvecs[ix].bv_len - skip, max_size); 133 span += len; 134 nsegs++; 135 ix++; 136 if (span >= max_size || nsegs >= max_segs) 137 break; 138 skip = 0; 139 n -= len; 140 } 141 142 return min(span, max_size); 143 } 144 145 /* 146 * Select the span of an xarray iterator we're going to use. Limit it by both 147 * maximum size and maximum number of segments. It is assumed that segments 148 * can be larger than a page in size, provided they're physically contiguous. 149 * Returns the size of the span in bytes. 150 */ 151 static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offset, 152 size_t max_size, size_t max_segs) 153 { 154 struct folio *folio; 155 unsigned int nsegs = 0; 156 loff_t pos = iter->xarray_start + iter->iov_offset; 157 pgoff_t index = pos / PAGE_SIZE; 158 size_t span = 0, n = iter->count; 159 160 XA_STATE(xas, iter->xarray, index); 161 162 if (WARN_ON(!iov_iter_is_xarray(iter)) || 163 WARN_ON(start_offset > n) || 164 n == 0) 165 return 0; 166 max_size = min(max_size, n - start_offset); 167 168 rcu_read_lock(); 169 xas_for_each(&xas, folio, ULONG_MAX) { 170 size_t offset, flen, len; 171 if (xas_retry(&xas, folio)) 172 continue; 173 if (WARN_ON(xa_is_value(folio))) 174 break; 175 if (WARN_ON(folio_test_hugetlb(folio))) 176 break; 177 178 flen = folio_size(folio); 179 offset = offset_in_folio(folio, pos); 180 len = min(max_size, flen - offset); 181 span += len; 182 nsegs++; 183 if (span >= max_size || nsegs >= max_segs) 184 break; 185 } 186 187 rcu_read_unlock(); 188 return min(span, max_size); 189 } 190 191 size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, 192 size_t max_size, size_t max_segs) 193 { 194 if (iov_iter_is_bvec(iter)) 195 return netfs_limit_bvec(iter, start_offset, max_size, max_segs); 196 if (iov_iter_is_xarray(iter)) 197 return netfs_limit_xarray(iter, start_offset, max_size, max_segs); 198 BUG(); 199 } 200 EXPORT_SYMBOL(netfs_limit_iter); 201