1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
23 * Copyright (c) 2019 by Delphix. All rights reserved.
24 * Copyright (c) 2023, 2024, Klara Inc.
25 */
26
27 #include <sys/abd_impl.h>
28 #include <sys/param.h>
29 #include <sys/zio.h>
30 #include <sys/arc.h>
31 #include <sys/zfs_context.h>
32 #include <sys/zfs_znode.h>
33
34 /*
35 * We're simulating scatter/gather with 4K allocations, since that's more like
36 * what a typical kernel does.
37 */
38 #define ABD_PAGESIZE (4096)
39 #define ABD_PAGESHIFT (12)
40 #define ABD_PAGEMASK (ABD_PAGESIZE-1)
41
42 /*
43 * See rationale in module/os/linux/zfs/abd_os.c, but in userspace this is
44 * mostly useful to get a mix of linear and scatter ABDs for testing.
45 */
46 #define ABD_SCATTER_MIN_SIZE (512 * 3)
47
48 abd_t *abd_zero_scatter = NULL;
49
50 static uint_t
abd_iovcnt_for_bytes(size_t size)51 abd_iovcnt_for_bytes(size_t size)
52 {
53 /*
54 * Each iovec points to a 4K page. There's no real reason to do this
55 * in userspace, but our whole point here is to make it feel a bit
56 * more like a real paged memory model.
57 */
58 return (P2ROUNDUP(size, ABD_PAGESIZE) / ABD_PAGESIZE);
59 }
60
61 abd_t *
abd_alloc_struct_impl(size_t size)62 abd_alloc_struct_impl(size_t size)
63 {
64 /*
65 * Zero-sized means it will be used for a linear or gang abd, so just
66 * allocate the abd itself and return.
67 */
68 if (size == 0)
69 return (umem_alloc(sizeof (abd_t), UMEM_NOFAIL));
70
71 /*
72 * Allocating for a scatter abd, so compute how many ABD_PAGESIZE
73 * iovecs we will need to hold this size. Append that allocation to the
74 * end. Note that struct abd_scatter has includes abd_iov[1], so we
75 * allocate one less iovec than we need.
76 *
77 * Note we're not allocating the pages proper, just the iovec pointers.
78 * That's down in abd_alloc_chunks. We _could_ do it here in a single
79 * allocation, but it's fiddly and harder to read for no real gain.
80 */
81 uint_t n = abd_iovcnt_for_bytes(size);
82 abd_t *abd = umem_alloc(sizeof (abd_t) + (n-1) * sizeof (struct iovec),
83 UMEM_NOFAIL);
84 ABD_SCATTER(abd).abd_offset = 0;
85 ABD_SCATTER(abd).abd_iovcnt = n;
86 return (abd);
87 }
88
89 void
abd_free_struct_impl(abd_t * abd)90 abd_free_struct_impl(abd_t *abd)
91 {
92 /* For scatter, compute the extra amount we need to free */
93 uint_t iovcnt =
94 abd_is_linear(abd) || abd_is_gang(abd) ?
95 0 : (ABD_SCATTER(abd).abd_iovcnt - 1);
96 umem_free(abd, sizeof (abd_t) + iovcnt * sizeof (struct iovec));
97 }
98
99 void
abd_alloc_chunks(abd_t * abd,size_t size)100 abd_alloc_chunks(abd_t *abd, size_t size)
101 {
102 /*
103 * We've already allocated the iovec array; ensure that the wanted size
104 * actually matches, otherwise the caller has made a mistake somewhere.
105 */
106 uint_t n = ABD_SCATTER(abd).abd_iovcnt;
107 ASSERT3U(n, ==, abd_iovcnt_for_bytes(size));
108
109 /*
110 * Allocate a ABD_PAGESIZE region for each iovec.
111 */
112 struct iovec *iov = ABD_SCATTER(abd).abd_iov;
113 for (int i = 0; i < n; i++) {
114 iov[i].iov_base =
115 umem_alloc_aligned(ABD_PAGESIZE, ABD_PAGESIZE, UMEM_NOFAIL);
116 iov[i].iov_len = ABD_PAGESIZE;
117 }
118 }
119
120 void
abd_free_chunks(abd_t * abd)121 abd_free_chunks(abd_t *abd)
122 {
123 uint_t n = ABD_SCATTER(abd).abd_iovcnt;
124 struct iovec *iov = ABD_SCATTER(abd).abd_iov;
125 for (int i = 0; i < n; i++)
126 umem_free_aligned(iov[i].iov_base, ABD_PAGESIZE);
127 }
128
129 boolean_t
abd_size_alloc_linear(size_t size)130 abd_size_alloc_linear(size_t size)
131 {
132 return (size < ABD_SCATTER_MIN_SIZE);
133 }
134
135 void
abd_update_scatter_stats(abd_t * abd,abd_stats_op_t op)136 abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op)
137 {
138 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
139 int waste = P2ROUNDUP(abd->abd_size, ABD_PAGESIZE) - abd->abd_size;
140 if (op == ABDSTAT_INCR) {
141 arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE);
142 } else {
143 arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE);
144 }
145 }
146
147 void
abd_update_linear_stats(abd_t * abd,abd_stats_op_t op)148 abd_update_linear_stats(abd_t *abd, abd_stats_op_t op)
149 {
150 (void) abd;
151 (void) op;
152 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
153 }
154
155 void
abd_verify_scatter(abd_t * abd)156 abd_verify_scatter(abd_t *abd)
157 {
158 #ifdef ZFS_DEBUG
159 /*
160 * scatter abds shall have:
161 * - at least one iovec
162 * - all iov_base point somewhere
163 * - all iov_len are ABD_PAGESIZE
164 * - offset set within the abd pages somewhere
165 */
166 uint_t n = ABD_SCATTER(abd).abd_iovcnt;
167 ASSERT3U(n, >, 0);
168
169 uint_t len = 0;
170 for (int i = 0; i < n; i++) {
171 ASSERT3P(ABD_SCATTER(abd).abd_iov[i].iov_base, !=, NULL);
172 ASSERT3U(ABD_SCATTER(abd).abd_iov[i].iov_len, ==, ABD_PAGESIZE);
173 len += ABD_PAGESIZE;
174 }
175
176 ASSERT3U(ABD_SCATTER(abd).abd_offset, <, len);
177 #endif
178 }
179
180 void
abd_init(void)181 abd_init(void)
182 {
183 /*
184 * Create the "zero" scatter abd. This is always the size of the
185 * largest possible block, but only actually has a single allocated
186 * page, which all iovecs in the abd point to.
187 */
188 abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
189 abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER;
190 abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;
191
192 void *zero =
193 umem_alloc_aligned(ABD_PAGESIZE, ABD_PAGESIZE, UMEM_NOFAIL);
194 memset(zero, 0, ABD_PAGESIZE);
195
196 uint_t n = abd_iovcnt_for_bytes(SPA_MAXBLOCKSIZE);
197 struct iovec *iov = ABD_SCATTER(abd_zero_scatter).abd_iov;
198 for (int i = 0; i < n; i++) {
199 iov[i].iov_base = zero;
200 iov[i].iov_len = ABD_PAGESIZE;
201 }
202 }
203
204 void
abd_fini(void)205 abd_fini(void)
206 {
207 umem_free_aligned(
208 ABD_SCATTER(abd_zero_scatter).abd_iov[0].iov_base, ABD_PAGESIZE);
209 abd_free_struct(abd_zero_scatter);
210 abd_zero_scatter = NULL;
211 }
212
213 void
abd_free_linear_page(abd_t * abd)214 abd_free_linear_page(abd_t *abd)
215 {
216 /*
217 * LINEAR_PAGE is specific to the Linux kernel; we never set this
218 * flag, so this will never be called.
219 */
220 (void) abd;
221 PANIC("unreachable");
222 }
223
224 abd_t *
abd_alloc_for_io(size_t size,boolean_t is_metadata)225 abd_alloc_for_io(size_t size, boolean_t is_metadata)
226 {
227 return (abd_alloc(size, is_metadata));
228 }
229
230 abd_t *
abd_get_offset_scatter(abd_t * dabd,abd_t * sabd,size_t off,size_t size)231 abd_get_offset_scatter(abd_t *dabd, abd_t *sabd, size_t off, size_t size)
232 {
233
234 /*
235 * Create a new scatter dabd by borrowing data pages from sabd to cover
236 * off+size.
237 *
238 * sabd is an existing scatter abd with a set of iovecs, each covering
239 * an ABD_PAGESIZE (4K) allocation. It's "zero" is at abd_offset.
240 *
241 * [........][........][........][........]
242 * ^- sabd_offset
243 *
244 * We want to produce a new abd, referencing those allocations at the
245 * given offset.
246 *
247 * [........][........][........][........]
248 * ^- dabd_offset = sabd_offset + off
249 * ^- dabd_offset + size
250 *
251 * In this example, dabd needs three iovecs. The first iovec is offset
252 * 0, so the final dabd_offset is masked back into the first iovec.
253 *
254 * [........][........][........]
255 * ^- dabd_offset
256 */
257 size_t soff = ABD_SCATTER(sabd).abd_offset + off;
258 size_t doff = soff & ABD_PAGEMASK;
259 size_t iovcnt = abd_iovcnt_for_bytes(doff + size);
260
261 /*
262 * If the passed-in abd has enough allocated iovecs already, reuse it.
263 * Otherwise, make a new one. The caller will free the original if the
264 * one it gets back is not the same.
265 *
266 * Note that it's ok if we reuse an abd with more iovecs than we need.
267 * abd_size has the usable amount of data, and the abd does not own the
268 * pages referenced by the iovecs. At worst, they're holding dangling
269 * pointers that we'll never use anyway.
270 */
271 if (dabd == NULL || ABD_SCATTER(dabd).abd_iovcnt < iovcnt)
272 dabd = abd_alloc_struct(iovcnt << ABD_PAGESHIFT);
273
274 /* Set offset into first page in view */
275 ABD_SCATTER(dabd).abd_offset = doff;
276
277 /* Copy the wanted iovecs from the source to the dest */
278 memcpy(&ABD_SCATTER(dabd).abd_iov[0],
279 &ABD_SCATTER(sabd).abd_iov[soff >> ABD_PAGESHIFT],
280 iovcnt * sizeof (struct iovec));
281
282 return (dabd);
283 }
284
285 void
abd_iter_init(struct abd_iter * aiter,abd_t * abd)286 abd_iter_init(struct abd_iter *aiter, abd_t *abd)
287 {
288 ASSERT(!abd_is_gang(abd));
289 abd_verify(abd);
290 memset(aiter, 0, sizeof (struct abd_iter));
291 aiter->iter_abd = abd;
292 }
293
294 boolean_t
abd_iter_at_end(struct abd_iter * aiter)295 abd_iter_at_end(struct abd_iter *aiter)
296 {
297 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size);
298 return (aiter->iter_pos == aiter->iter_abd->abd_size);
299 }
300
301 void
abd_iter_advance(struct abd_iter * aiter,size_t amount)302 abd_iter_advance(struct abd_iter *aiter, size_t amount)
303 {
304 ASSERT3P(aiter->iter_mapaddr, ==, NULL);
305 ASSERT0(aiter->iter_mapsize);
306
307 if (abd_iter_at_end(aiter))
308 return;
309
310 aiter->iter_pos += amount;
311 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size);
312 }
313
314 void
abd_iter_map(struct abd_iter * aiter)315 abd_iter_map(struct abd_iter *aiter)
316 {
317 ASSERT3P(aiter->iter_mapaddr, ==, NULL);
318 ASSERT0(aiter->iter_mapsize);
319
320 if (abd_iter_at_end(aiter))
321 return;
322
323 if (abd_is_linear(aiter->iter_abd)) {
324 aiter->iter_mapaddr =
325 ABD_LINEAR_BUF(aiter->iter_abd) + aiter->iter_pos;
326 aiter->iter_mapsize =
327 aiter->iter_abd->abd_size - aiter->iter_pos;
328 return;
329 }
330
331 /*
332 * For scatter, we index into the appropriate iovec, and return the
333 * smaller of the amount requested, or up to the end of the page.
334 */
335 size_t poff = aiter->iter_pos + ABD_SCATTER(aiter->iter_abd).abd_offset;
336
337 ASSERT3U(poff >> ABD_PAGESHIFT, <=,
338 ABD_SCATTER(aiter->iter_abd).abd_iovcnt);
339 struct iovec *iov = &ABD_SCATTER(aiter->iter_abd).
340 abd_iov[poff >> ABD_PAGESHIFT];
341
342 aiter->iter_mapsize = MIN(ABD_PAGESIZE - (poff & ABD_PAGEMASK),
343 aiter->iter_abd->abd_size - aiter->iter_pos);
344 ASSERT3U(aiter->iter_mapsize, <=, ABD_PAGESIZE);
345
346 aiter->iter_mapaddr = iov->iov_base + (poff & ABD_PAGEMASK);
347 }
348
349 void
abd_iter_unmap(struct abd_iter * aiter)350 abd_iter_unmap(struct abd_iter *aiter)
351 {
352 if (abd_iter_at_end(aiter))
353 return;
354
355 ASSERT3P(aiter->iter_mapaddr, !=, NULL);
356 ASSERT3U(aiter->iter_mapsize, >, 0);
357
358 aiter->iter_mapaddr = NULL;
359 aiter->iter_mapsize = 0;
360 }
361
362 void
abd_cache_reap_now(void)363 abd_cache_reap_now(void)
364 {
365 }
366
367 /*
368 * Borrow a raw buffer from an ABD without copying the contents of the ABD
369 * into the buffer. If the ABD is scattered, this will alloate a raw buffer
370 * whose contents are undefined. To copy over the existing data in the ABD, use
371 * abd_borrow_buf_copy() instead.
372 */
373 void *
abd_borrow_buf(abd_t * abd,size_t n)374 abd_borrow_buf(abd_t *abd, size_t n)
375 {
376 void *buf;
377 abd_verify(abd);
378 ASSERT3U(abd->abd_size, >=, 0);
379 if (abd_is_linear(abd)) {
380 buf = abd_to_buf(abd);
381 } else {
382 buf = zio_buf_alloc(n);
383 }
384 #ifdef ZFS_DEBUG
385 (void) zfs_refcount_add_many(&abd->abd_children, n, buf);
386 #endif
387 return (buf);
388 }
389
390 void *
abd_borrow_buf_copy(abd_t * abd,size_t n)391 abd_borrow_buf_copy(abd_t *abd, size_t n)
392 {
393 void *buf = abd_borrow_buf(abd, n);
394 if (!abd_is_linear(abd)) {
395 abd_copy_to_buf(buf, abd, n);
396 }
397 return (buf);
398 }
399
400 /*
401 * Return a borrowed raw buffer to an ABD. If the ABD is scattered, this will
402 * no change the contents of the ABD and will ASSERT that you didn't modify
403 * the buffer since it was borrowed. If you want any changes you made to buf to
404 * be copied back to abd, use abd_return_buf_copy() instead.
405 */
406 void
abd_return_buf(abd_t * abd,void * buf,size_t n)407 abd_return_buf(abd_t *abd, void *buf, size_t n)
408 {
409 abd_verify(abd);
410 ASSERT3U(abd->abd_size, >=, n);
411 #ifdef ZFS_DEBUG
412 (void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
413 #endif
414 if (abd_is_linear(abd)) {
415 ASSERT3P(buf, ==, abd_to_buf(abd));
416 } else {
417 ASSERT0(abd_cmp_buf(abd, buf, n));
418 zio_buf_free(buf, n);
419 }
420 }
421
422 void
abd_return_buf_copy(abd_t * abd,void * buf,size_t n)423 abd_return_buf_copy(abd_t *abd, void *buf, size_t n)
424 {
425 if (!abd_is_linear(abd)) {
426 abd_copy_from_buf(abd, buf, n);
427 }
428 abd_return_buf(abd, buf, n);
429 }
430