1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* I/O iterator iteration building functions.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #ifndef _LINUX_IOV_ITER_H
9 #define _LINUX_IOV_ITER_H
10
11 #include <linux/uio.h>
12 #include <linux/bvec.h>
13 #include <linux/folio_queue.h>
14
15 typedef size_t (*iov_step_f)(void *iter_base, size_t progress, size_t len,
16 void *priv, void *priv2);
17 typedef size_t (*iov_ustep_f)(void __user *iter_base, size_t progress, size_t len,
18 void *priv, void *priv2);
19
20 /*
21 * Handle ITER_UBUF.
22 */
23 static __always_inline
iterate_ubuf(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_ustep_f step)24 size_t iterate_ubuf(struct iov_iter *iter, size_t len, void *priv, void *priv2,
25 iov_ustep_f step)
26 {
27 void __user *base = iter->ubuf;
28 size_t progress = 0, remain;
29
30 remain = step(base + iter->iov_offset, 0, len, priv, priv2);
31 progress = len - remain;
32 iter->iov_offset += progress;
33 iter->count -= progress;
34 return progress;
35 }
36
37 /*
38 * Handle ITER_IOVEC.
39 */
40 static __always_inline
iterate_iovec(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_ustep_f step)41 size_t iterate_iovec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
42 iov_ustep_f step)
43 {
44 const struct iovec *p = iter->__iov;
45 size_t progress = 0, skip = iter->iov_offset;
46
47 do {
48 size_t remain, consumed;
49 size_t part = min(len, p->iov_len - skip);
50
51 if (likely(part)) {
52 remain = step(p->iov_base + skip, progress, part, priv, priv2);
53 consumed = part - remain;
54 progress += consumed;
55 skip += consumed;
56 len -= consumed;
57 if (skip < p->iov_len)
58 break;
59 }
60 p++;
61 skip = 0;
62 } while (len);
63
64 iter->nr_segs -= p - iter->__iov;
65 iter->__iov = p;
66 iter->iov_offset = skip;
67 iter->count -= progress;
68 return progress;
69 }
70
71 /*
72 * Handle ITER_KVEC.
73 */
74 static __always_inline
iterate_kvec(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_step_f step)75 size_t iterate_kvec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
76 iov_step_f step)
77 {
78 const struct kvec *p = iter->kvec;
79 size_t progress = 0, skip = iter->iov_offset;
80
81 do {
82 size_t remain, consumed;
83 size_t part = min(len, p->iov_len - skip);
84
85 if (likely(part)) {
86 remain = step(p->iov_base + skip, progress, part, priv, priv2);
87 consumed = part - remain;
88 progress += consumed;
89 skip += consumed;
90 len -= consumed;
91 if (skip < p->iov_len)
92 break;
93 }
94 p++;
95 skip = 0;
96 } while (len);
97
98 iter->nr_segs -= p - iter->kvec;
99 iter->kvec = p;
100 iter->iov_offset = skip;
101 iter->count -= progress;
102 return progress;
103 }
104
105 /*
106 * Handle ITER_BVEC.
107 */
108 static __always_inline
iterate_bvec(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_step_f step)109 size_t iterate_bvec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
110 iov_step_f step)
111 {
112 const struct bio_vec *p = iter->bvec;
113 size_t progress = 0, skip = iter->iov_offset;
114
115 do {
116 size_t remain, consumed;
117 size_t offset = p->bv_offset + skip, part;
118 void *kaddr = kmap_local_page(p->bv_page + offset / PAGE_SIZE);
119
120 part = min3(len,
121 (size_t)(p->bv_len - skip),
122 (size_t)(PAGE_SIZE - offset % PAGE_SIZE));
123 remain = step(kaddr + offset % PAGE_SIZE, progress, part, priv, priv2);
124 kunmap_local(kaddr);
125 consumed = part - remain;
126 len -= consumed;
127 progress += consumed;
128 skip += consumed;
129 if (skip >= p->bv_len) {
130 skip = 0;
131 p++;
132 }
133 if (remain)
134 break;
135 } while (len);
136
137 iter->nr_segs -= p - iter->bvec;
138 iter->bvec = p;
139 iter->iov_offset = skip;
140 iter->count -= progress;
141 return progress;
142 }
143
144 /*
145 * Handle ITER_FOLIOQ.
146 */
147 static __always_inline
iterate_folioq(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_step_f step)148 size_t iterate_folioq(struct iov_iter *iter, size_t len, void *priv, void *priv2,
149 iov_step_f step)
150 {
151 const struct folio_queue *folioq = iter->folioq;
152 unsigned int slot = iter->folioq_slot;
153 size_t progress = 0, skip = iter->iov_offset;
154
155 if (slot == folioq_nr_slots(folioq)) {
156 /* The iterator may have been extended. */
157 folioq = folioq->next;
158 slot = 0;
159 }
160
161 do {
162 struct folio *folio = folioq_folio(folioq, slot);
163 size_t part, remain, consumed;
164 size_t fsize;
165 void *base;
166
167 if (!folio)
168 break;
169
170 fsize = folioq_folio_size(folioq, slot);
171 base = kmap_local_folio(folio, skip);
172 part = umin(len, PAGE_SIZE - skip % PAGE_SIZE);
173 remain = step(base, progress, part, priv, priv2);
174 kunmap_local(base);
175 consumed = part - remain;
176 len -= consumed;
177 progress += consumed;
178 skip += consumed;
179 if (skip >= fsize) {
180 skip = 0;
181 slot++;
182 if (slot == folioq_nr_slots(folioq) && folioq->next) {
183 folioq = folioq->next;
184 slot = 0;
185 }
186 }
187 if (remain)
188 break;
189 } while (len);
190
191 iter->folioq_slot = slot;
192 iter->folioq = folioq;
193 iter->iov_offset = skip;
194 iter->count -= progress;
195 return progress;
196 }
197
198 /*
199 * Handle ITER_XARRAY.
200 */
201 static __always_inline
iterate_xarray(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_step_f step)202 size_t iterate_xarray(struct iov_iter *iter, size_t len, void *priv, void *priv2,
203 iov_step_f step)
204 {
205 struct folio *folio;
206 size_t progress = 0;
207 loff_t start = iter->xarray_start + iter->iov_offset;
208 pgoff_t index = start / PAGE_SIZE;
209 XA_STATE(xas, iter->xarray, index);
210
211 rcu_read_lock();
212 xas_for_each(&xas, folio, ULONG_MAX) {
213 size_t remain, consumed, offset, part, flen;
214
215 if (xas_retry(&xas, folio))
216 continue;
217 if (WARN_ON(xa_is_value(folio)))
218 break;
219 if (WARN_ON(folio_test_hugetlb(folio)))
220 break;
221
222 offset = offset_in_folio(folio, start + progress);
223 flen = min(folio_size(folio) - offset, len);
224
225 while (flen) {
226 void *base = kmap_local_folio(folio, offset);
227
228 part = min_t(size_t, flen,
229 PAGE_SIZE - offset_in_page(offset));
230 remain = step(base, progress, part, priv, priv2);
231 kunmap_local(base);
232
233 consumed = part - remain;
234 progress += consumed;
235 len -= consumed;
236
237 if (remain || len == 0)
238 goto out;
239 flen -= consumed;
240 offset += consumed;
241 }
242 }
243
244 out:
245 rcu_read_unlock();
246 iter->iov_offset += progress;
247 iter->count -= progress;
248 return progress;
249 }
250
251 /*
252 * Handle ITER_DISCARD.
253 */
254 static __always_inline
iterate_discard(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_step_f step)255 size_t iterate_discard(struct iov_iter *iter, size_t len, void *priv, void *priv2,
256 iov_step_f step)
257 {
258 size_t progress = len;
259
260 iter->count -= progress;
261 return progress;
262 }
263
264 /**
265 * iterate_and_advance2 - Iterate over an iterator
266 * @iter: The iterator to iterate over.
267 * @len: The amount to iterate over.
268 * @priv: Data for the step functions.
269 * @priv2: More data for the step functions.
270 * @ustep: Function for UBUF/IOVEC iterators; given __user addresses.
271 * @step: Function for other iterators; given kernel addresses.
272 *
273 * Iterate over the next part of an iterator, up to the specified length. The
274 * buffer is presented in segments, which for kernel iteration are broken up by
275 * physical pages and mapped, with the mapped address being presented.
276 *
277 * Two step functions, @step and @ustep, must be provided, one for handling
278 * mapped kernel addresses and the other is given user addresses which have the
279 * potential to fault since no pinning is performed.
280 *
281 * The step functions are passed the address and length of the segment, @priv,
282 * @priv2 and the amount of data so far iterated over (which can, for example,
283 * be added to @priv to point to the right part of a second buffer). The step
284 * functions should return the amount of the segment they didn't process (ie. 0
285 * indicates complete processsing).
286 *
287 * This function returns the amount of data processed (ie. 0 means nothing was
288 * processed and the value of @len means processes to completion).
289 */
290 static __always_inline
iterate_and_advance2(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_ustep_f ustep,iov_step_f step)291 size_t iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv,
292 void *priv2, iov_ustep_f ustep, iov_step_f step)
293 {
294 if (unlikely(iter->count < len))
295 len = iter->count;
296 if (unlikely(!len))
297 return 0;
298
299 if (likely(iter_is_ubuf(iter)))
300 return iterate_ubuf(iter, len, priv, priv2, ustep);
301 if (likely(iter_is_iovec(iter)))
302 return iterate_iovec(iter, len, priv, priv2, ustep);
303 if (iov_iter_is_bvec(iter))
304 return iterate_bvec(iter, len, priv, priv2, step);
305 if (iov_iter_is_kvec(iter))
306 return iterate_kvec(iter, len, priv, priv2, step);
307 if (iov_iter_is_folioq(iter))
308 return iterate_folioq(iter, len, priv, priv2, step);
309 if (iov_iter_is_xarray(iter))
310 return iterate_xarray(iter, len, priv, priv2, step);
311 return iterate_discard(iter, len, priv, priv2, step);
312 }
313
314 /**
315 * iterate_and_advance - Iterate over an iterator
316 * @iter: The iterator to iterate over.
317 * @len: The amount to iterate over.
318 * @priv: Data for the step functions.
319 * @ustep: Function for UBUF/IOVEC iterators; given __user addresses.
320 * @step: Function for other iterators; given kernel addresses.
321 *
322 * As iterate_and_advance2(), but priv2 is always NULL.
323 */
324 static __always_inline
iterate_and_advance(struct iov_iter * iter,size_t len,void * priv,iov_ustep_f ustep,iov_step_f step)325 size_t iterate_and_advance(struct iov_iter *iter, size_t len, void *priv,
326 iov_ustep_f ustep, iov_step_f step)
327 {
328 return iterate_and_advance2(iter, len, priv, NULL, ustep, step);
329 }
330
331 /**
332 * iterate_and_advance_kernel - Iterate over a kernel-internal iterator
333 * @iter: The iterator to iterate over.
334 * @len: The amount to iterate over.
335 * @priv: Data for the step functions.
336 * @priv2: More data for the step functions.
337 * @step: Function for other iterators; given kernel addresses.
338 *
339 * Iterate over the next part of an iterator, up to the specified length. The
340 * buffer is presented in segments, which for kernel iteration are broken up by
341 * physical pages and mapped, with the mapped address being presented.
342 *
343 * [!] Note This will only handle BVEC, KVEC, FOLIOQ, XARRAY and DISCARD-type
344 * iterators; it will not handle UBUF or IOVEC-type iterators.
345 *
346 * A step functions, @step, must be provided, one for handling mapped kernel
347 * addresses and the other is given user addresses which have the potential to
348 * fault since no pinning is performed.
349 *
350 * The step functions are passed the address and length of the segment, @priv,
351 * @priv2 and the amount of data so far iterated over (which can, for example,
352 * be added to @priv to point to the right part of a second buffer). The step
353 * functions should return the amount of the segment they didn't process (ie. 0
354 * indicates complete processsing).
355 *
356 * This function returns the amount of data processed (ie. 0 means nothing was
357 * processed and the value of @len means processes to completion).
358 */
359 static __always_inline
iterate_and_advance_kernel(struct iov_iter * iter,size_t len,void * priv,void * priv2,iov_step_f step)360 size_t iterate_and_advance_kernel(struct iov_iter *iter, size_t len, void *priv,
361 void *priv2, iov_step_f step)
362 {
363 if (unlikely(iter->count < len))
364 len = iter->count;
365 if (unlikely(!len))
366 return 0;
367 if (iov_iter_is_bvec(iter))
368 return iterate_bvec(iter, len, priv, priv2, step);
369 if (iov_iter_is_kvec(iter))
370 return iterate_kvec(iter, len, priv, priv2, step);
371 if (iov_iter_is_folioq(iter))
372 return iterate_folioq(iter, len, priv, priv2, step);
373 if (iov_iter_is_xarray(iter))
374 return iterate_xarray(iter, len, priv, priv2, step);
375 return iterate_discard(iter, len, priv, priv2, step);
376 }
377
378 #endif /* _LINUX_IOV_ITER_H */
379