1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Iterator helpers.
3 *
4 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/export.h>
9 #include <linux/slab.h>
10 #include <linux/mm.h>
11 #include <linux/uio.h>
12 #include <linux/scatterlist.h>
13 #include <linux/netfs.h>
14 #include "internal.h"
15
16 /**
17 * netfs_extract_user_iter - Extract the pages from a user iterator into a bvec
18 * @orig: The original iterator
19 * @orig_len: The amount of iterator to copy
20 * @new: The iterator to be set up
21 * @extraction_flags: Flags to qualify the request
22 *
23 * Extract the page fragments from the given amount of the source iterator and
24 * build up a second iterator that refers to all of those bits. This allows
25 * the original iterator to disposed of.
26 *
27 * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be
28 * allowed on the pages extracted.
29 *
30 * On success, the number of elements in the bvec is returned, the original
31 * iterator will have been advanced by the amount extracted.
32 *
33 * The iov_iter_extract_mode() function should be used to query how cleanup
34 * should be performed.
35 */
netfs_extract_user_iter(struct iov_iter * orig,size_t orig_len,struct iov_iter * new,iov_iter_extraction_t extraction_flags)36 ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
37 struct iov_iter *new,
38 iov_iter_extraction_t extraction_flags)
39 {
40 struct bio_vec *bv = NULL;
41 struct page **pages;
42 unsigned int cur_npages;
43 unsigned int max_pages;
44 unsigned int npages = 0;
45 unsigned int i;
46 ssize_t ret;
47 size_t count = orig_len, offset, len;
48 size_t bv_size, pg_size;
49
50 if (WARN_ON_ONCE(!iter_is_ubuf(orig) && !iter_is_iovec(orig)))
51 return -EIO;
52
53 max_pages = iov_iter_npages(orig, INT_MAX);
54 bv_size = array_size(max_pages, sizeof(*bv));
55 bv = kvmalloc(bv_size, GFP_KERNEL);
56 if (!bv)
57 return -ENOMEM;
58
59 /* Put the page list at the end of the bvec list storage. bvec
60 * elements are larger than page pointers, so as long as we work
61 * 0->last, we should be fine.
62 */
63 pg_size = array_size(max_pages, sizeof(*pages));
64 pages = (void *)bv + bv_size - pg_size;
65
66 while (count && npages < max_pages) {
67 ret = iov_iter_extract_pages(orig, &pages, count,
68 max_pages - npages, extraction_flags,
69 &offset);
70 if (ret < 0) {
71 pr_err("Couldn't get user pages (rc=%zd)\n", ret);
72 break;
73 }
74
75 if (ret > count) {
76 pr_err("get_pages rc=%zd more than %zu\n", ret, count);
77 break;
78 }
79
80 count -= ret;
81 ret += offset;
82 cur_npages = DIV_ROUND_UP(ret, PAGE_SIZE);
83
84 if (npages + cur_npages > max_pages) {
85 pr_err("Out of bvec array capacity (%u vs %u)\n",
86 npages + cur_npages, max_pages);
87 break;
88 }
89
90 for (i = 0; i < cur_npages; i++) {
91 len = ret > PAGE_SIZE ? PAGE_SIZE : ret;
92 bvec_set_page(bv + npages + i, *pages++, len - offset, offset);
93 ret -= len;
94 offset = 0;
95 }
96
97 npages += cur_npages;
98 }
99
100 iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count);
101 return npages;
102 }
103 EXPORT_SYMBOL_GPL(netfs_extract_user_iter);
104
105 /*
106 * Select the span of a bvec iterator we're going to use. Limit it by both maximum
107 * size and maximum number of segments. Returns the size of the span in bytes.
108 */
netfs_limit_bvec(const struct iov_iter * iter,size_t start_offset,size_t max_size,size_t max_segs)109 static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset,
110 size_t max_size, size_t max_segs)
111 {
112 const struct bio_vec *bvecs = iter->bvec;
113 unsigned int nbv = iter->nr_segs, ix = 0, nsegs = 0;
114 size_t len, span = 0, n = iter->count;
115 size_t skip = iter->iov_offset + start_offset;
116
117 if (WARN_ON(!iov_iter_is_bvec(iter)) ||
118 WARN_ON(start_offset > n) ||
119 n == 0)
120 return 0;
121
122 while (n && ix < nbv && skip) {
123 len = bvecs[ix].bv_len;
124 if (skip < len)
125 break;
126 skip -= len;
127 n -= len;
128 ix++;
129 }
130
131 while (n && ix < nbv) {
132 len = min3(n, bvecs[ix].bv_len - skip, max_size);
133 span += len;
134 nsegs++;
135 ix++;
136 if (span >= max_size || nsegs >= max_segs)
137 break;
138 skip = 0;
139 n -= len;
140 }
141
142 return min(span, max_size);
143 }
144
145 /*
146 * Select the span of a kvec iterator we're going to use. Limit it by both
147 * maximum size and maximum number of segments. Returns the size of the span
148 * in bytes.
149 */
netfs_limit_kvec(const struct iov_iter * iter,size_t start_offset,size_t max_size,size_t max_segs)150 static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset,
151 size_t max_size, size_t max_segs)
152 {
153 const struct kvec *kvecs = iter->kvec;
154 unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0;
155 size_t len, span = 0, n = iter->count;
156 size_t skip = iter->iov_offset + start_offset;
157
158 if (WARN_ON(!iov_iter_is_kvec(iter)) ||
159 WARN_ON(start_offset > n) ||
160 n == 0)
161 return 0;
162
163 while (n && ix < nkv && skip) {
164 len = kvecs[ix].iov_len;
165 if (skip < len)
166 break;
167 skip -= len;
168 n -= len;
169 ix++;
170 }
171
172 while (n && ix < nkv) {
173 len = min3(n, kvecs[ix].iov_len - skip, max_size);
174 span += len;
175 nsegs++;
176 ix++;
177 if (span >= max_size || nsegs >= max_segs)
178 break;
179 skip = 0;
180 n -= len;
181 }
182
183 return min(span, max_size);
184 }
185
186 /*
187 * Select the span of an xarray iterator we're going to use. Limit it by both
188 * maximum size and maximum number of segments. It is assumed that segments
189 * can be larger than a page in size, provided they're physically contiguous.
190 * Returns the size of the span in bytes.
191 */
netfs_limit_xarray(const struct iov_iter * iter,size_t start_offset,size_t max_size,size_t max_segs)192 static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offset,
193 size_t max_size, size_t max_segs)
194 {
195 struct folio *folio;
196 unsigned int nsegs = 0;
197 loff_t pos = iter->xarray_start + iter->iov_offset;
198 pgoff_t index = pos / PAGE_SIZE;
199 size_t span = 0, n = iter->count;
200
201 XA_STATE(xas, iter->xarray, index);
202
203 if (WARN_ON(!iov_iter_is_xarray(iter)) ||
204 WARN_ON(start_offset > n) ||
205 n == 0)
206 return 0;
207 max_size = min(max_size, n - start_offset);
208
209 rcu_read_lock();
210 xas_for_each(&xas, folio, ULONG_MAX) {
211 size_t offset, flen, len;
212 if (xas_retry(&xas, folio))
213 continue;
214 if (WARN_ON(xa_is_value(folio)))
215 break;
216 if (WARN_ON(folio_test_hugetlb(folio)))
217 break;
218
219 flen = folio_size(folio);
220 offset = offset_in_folio(folio, pos);
221 len = min(max_size, flen - offset);
222 span += len;
223 nsegs++;
224 if (span >= max_size || nsegs >= max_segs)
225 break;
226 }
227
228 rcu_read_unlock();
229 return min(span, max_size);
230 }
231
232 /*
233 * Select the span of a folio queue iterator we're going to use. Limit it by
234 * both maximum size and maximum number of segments. Returns the size of the
235 * span in bytes.
236 */
netfs_limit_folioq(const struct iov_iter * iter,size_t start_offset,size_t max_size,size_t max_segs)237 static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset,
238 size_t max_size, size_t max_segs)
239 {
240 const struct folio_queue *folioq = iter->folioq;
241 unsigned int nsegs = 0;
242 unsigned int slot = iter->folioq_slot;
243 size_t span = 0, n = iter->count;
244
245 if (WARN_ON(!iov_iter_is_folioq(iter)) ||
246 WARN_ON(start_offset > n) ||
247 n == 0)
248 return 0;
249 max_size = umin(max_size, n - start_offset);
250
251 if (slot >= folioq_nr_slots(folioq)) {
252 folioq = folioq->next;
253 slot = 0;
254 }
255
256 start_offset += iter->iov_offset;
257 do {
258 size_t flen = folioq_folio_size(folioq, slot);
259
260 if (start_offset < flen) {
261 span += flen - start_offset;
262 nsegs++;
263 start_offset = 0;
264 } else {
265 start_offset -= flen;
266 }
267 if (span >= max_size || nsegs >= max_segs)
268 break;
269
270 slot++;
271 if (slot >= folioq_nr_slots(folioq)) {
272 folioq = folioq->next;
273 slot = 0;
274 }
275 } while (folioq);
276
277 return umin(span, max_size);
278 }
279
netfs_limit_iter(const struct iov_iter * iter,size_t start_offset,size_t max_size,size_t max_segs)280 size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
281 size_t max_size, size_t max_segs)
282 {
283 if (iov_iter_is_folioq(iter))
284 return netfs_limit_folioq(iter, start_offset, max_size, max_segs);
285 if (iov_iter_is_bvec(iter))
286 return netfs_limit_bvec(iter, start_offset, max_size, max_segs);
287 if (iov_iter_is_xarray(iter))
288 return netfs_limit_xarray(iter, start_offset, max_size, max_segs);
289 if (iov_iter_is_kvec(iter))
290 return netfs_limit_kvec(iter, start_offset, max_size, max_segs);
291 BUG();
292 }
293 EXPORT_SYMBOL(netfs_limit_iter);
294