1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 24 * Copyright (c) 2019 by Delphix. All rights reserved. 25 * Copyright (c) 2023, 2024, Klara Inc. 26 */ 27 28 #include <sys/abd_impl.h> 29 #include <sys/param.h> 30 #include <sys/zio.h> 31 #include <sys/arc.h> 32 #include <sys/zfs_context.h> 33 #include <sys/zfs_znode.h> 34 35 /* 36 * We're simulating scatter/gather with 4K allocations, since that's more like 37 * what a typical kernel does. 38 */ 39 #define ABD_PAGESIZE (4096) 40 #define ABD_PAGESHIFT (12) 41 #define ABD_PAGEMASK (ABD_PAGESIZE-1) 42 43 /* 44 * See rationale in module/os/linux/zfs/abd_os.c, but in userspace this is 45 * mostly useful to get a mix of linear and scatter ABDs for testing. 46 */ 47 #define ABD_SCATTER_MIN_SIZE (512 * 3) 48 49 abd_t *abd_zero_scatter = NULL; 50 51 static uint_t 52 abd_iovcnt_for_bytes(size_t size) 53 { 54 /* 55 * Each iovec points to a 4K page. There's no real reason to do this 56 * in userspace, but our whole point here is to make it feel a bit 57 * more like a real paged memory model. 58 */ 59 return (P2ROUNDUP(size, ABD_PAGESIZE) / ABD_PAGESIZE); 60 } 61 62 abd_t * 63 abd_alloc_struct_impl(size_t size) 64 { 65 /* 66 * Zero-sized means it will be used for a linear or gang abd, so just 67 * allocate the abd itself and return. 68 */ 69 if (size == 0) 70 return (umem_alloc(sizeof (abd_t), UMEM_NOFAIL)); 71 72 /* 73 * Allocating for a scatter abd, so compute how many ABD_PAGESIZE 74 * iovecs we will need to hold this size. Append that allocation to the 75 * end. Note that struct abd_scatter has includes abd_iov[1], so we 76 * allocate one less iovec than we need. 77 * 78 * Note we're not allocating the pages proper, just the iovec pointers. 79 * That's down in abd_alloc_chunks. We _could_ do it here in a single 80 * allocation, but it's fiddly and harder to read for no real gain. 81 */ 82 uint_t n = abd_iovcnt_for_bytes(size); 83 abd_t *abd = umem_alloc(sizeof (abd_t) + (n-1) * sizeof (struct iovec), 84 UMEM_NOFAIL); 85 ABD_SCATTER(abd).abd_offset = 0; 86 ABD_SCATTER(abd).abd_iovcnt = n; 87 return (abd); 88 } 89 90 void 91 abd_free_struct_impl(abd_t *abd) 92 { 93 /* For scatter, compute the extra amount we need to free */ 94 uint_t iovcnt = 95 abd_is_linear(abd) || abd_is_gang(abd) ? 96 0 : (ABD_SCATTER(abd).abd_iovcnt - 1); 97 umem_free(abd, sizeof (abd_t) + iovcnt * sizeof (struct iovec)); 98 } 99 100 void 101 abd_alloc_chunks(abd_t *abd, size_t size) 102 { 103 /* 104 * We've already allocated the iovec array; ensure that the wanted size 105 * actually matches, otherwise the caller has made a mistake somewhere. 106 */ 107 uint_t n = ABD_SCATTER(abd).abd_iovcnt; 108 ASSERT3U(n, ==, abd_iovcnt_for_bytes(size)); 109 110 /* 111 * Allocate a ABD_PAGESIZE region for each iovec. 112 */ 113 struct iovec *iov = ABD_SCATTER(abd).abd_iov; 114 for (int i = 0; i < n; i++) { 115 iov[i].iov_base = 116 umem_alloc_aligned(ABD_PAGESIZE, ABD_PAGESIZE, UMEM_NOFAIL); 117 iov[i].iov_len = ABD_PAGESIZE; 118 } 119 } 120 121 void 122 abd_free_chunks(abd_t *abd) 123 { 124 uint_t n = ABD_SCATTER(abd).abd_iovcnt; 125 struct iovec *iov = ABD_SCATTER(abd).abd_iov; 126 for (int i = 0; i < n; i++) 127 umem_free_aligned(iov[i].iov_base, ABD_PAGESIZE); 128 } 129 130 boolean_t 131 abd_size_alloc_linear(size_t size) 132 { 133 return (size < ABD_SCATTER_MIN_SIZE); 134 } 135 136 void 137 abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op) 138 { 139 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR); 140 int waste = P2ROUNDUP(abd->abd_size, ABD_PAGESIZE) - abd->abd_size; 141 if (op == ABDSTAT_INCR) { 142 arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE); 143 } else { 144 arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE); 145 } 146 } 147 148 void 149 abd_update_linear_stats(abd_t *abd, abd_stats_op_t op) 150 { 151 (void) abd; 152 (void) op; 153 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR); 154 } 155 156 void 157 abd_verify_scatter(abd_t *abd) 158 { 159 #ifdef ZFS_DEBUG 160 /* 161 * scatter abds shall have: 162 * - at least one iovec 163 * - all iov_base point somewhere 164 * - all iov_len are ABD_PAGESIZE 165 * - offset set within the abd pages somewhere 166 */ 167 uint_t n = ABD_SCATTER(abd).abd_iovcnt; 168 ASSERT3U(n, >, 0); 169 170 uint_t len = 0; 171 for (int i = 0; i < n; i++) { 172 ASSERT3P(ABD_SCATTER(abd).abd_iov[i].iov_base, !=, NULL); 173 ASSERT3U(ABD_SCATTER(abd).abd_iov[i].iov_len, ==, ABD_PAGESIZE); 174 len += ABD_PAGESIZE; 175 } 176 177 ASSERT3U(ABD_SCATTER(abd).abd_offset, <, len); 178 #endif 179 } 180 181 void 182 abd_init(void) 183 { 184 /* 185 * Create the "zero" scatter abd. This is always the size of the 186 * largest possible block, but only actually has a single allocated 187 * page, which all iovecs in the abd point to. 188 */ 189 abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE); 190 abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER; 191 abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE; 192 193 void *zero = 194 umem_alloc_aligned(ABD_PAGESIZE, ABD_PAGESIZE, UMEM_NOFAIL); 195 memset(zero, 0, ABD_PAGESIZE); 196 197 uint_t n = abd_iovcnt_for_bytes(SPA_MAXBLOCKSIZE); 198 struct iovec *iov = ABD_SCATTER(abd_zero_scatter).abd_iov; 199 for (int i = 0; i < n; i++) { 200 iov[i].iov_base = zero; 201 iov[i].iov_len = ABD_PAGESIZE; 202 } 203 } 204 205 void 206 abd_fini(void) 207 { 208 umem_free_aligned( 209 ABD_SCATTER(abd_zero_scatter).abd_iov[0].iov_base, ABD_PAGESIZE); 210 abd_free_struct(abd_zero_scatter); 211 abd_zero_scatter = NULL; 212 } 213 214 void 215 abd_free_linear_page(abd_t *abd) 216 { 217 /* 218 * LINEAR_PAGE is specific to the Linux kernel; we never set this 219 * flag, so this will never be called. 220 */ 221 (void) abd; 222 PANIC("unreachable"); 223 } 224 225 abd_t * 226 abd_alloc_for_io(size_t size, boolean_t is_metadata) 227 { 228 return (abd_alloc(size, is_metadata)); 229 } 230 231 abd_t * 232 abd_get_offset_scatter(abd_t *dabd, abd_t *sabd, size_t off, size_t size) 233 { 234 235 /* 236 * Create a new scatter dabd by borrowing data pages from sabd to cover 237 * off+size. 238 * 239 * sabd is an existing scatter abd with a set of iovecs, each covering 240 * an ABD_PAGESIZE (4K) allocation. It's "zero" is at abd_offset. 241 * 242 * [........][........][........][........] 243 * ^- sabd_offset 244 * 245 * We want to produce a new abd, referencing those allocations at the 246 * given offset. 247 * 248 * [........][........][........][........] 249 * ^- dabd_offset = sabd_offset + off 250 * ^- dabd_offset + size 251 * 252 * In this example, dabd needs three iovecs. The first iovec is offset 253 * 0, so the final dabd_offset is masked back into the first iovec. 254 * 255 * [........][........][........] 256 * ^- dabd_offset 257 */ 258 size_t soff = ABD_SCATTER(sabd).abd_offset + off; 259 size_t doff = soff & ABD_PAGEMASK; 260 size_t iovcnt = abd_iovcnt_for_bytes(doff + size); 261 262 /* 263 * If the passed-in abd has enough allocated iovecs already, reuse it. 264 * Otherwise, make a new one. The caller will free the original if the 265 * one it gets back is not the same. 266 * 267 * Note that it's ok if we reuse an abd with more iovecs than we need. 268 * abd_size has the usable amount of data, and the abd does not own the 269 * pages referenced by the iovecs. At worst, they're holding dangling 270 * pointers that we'll never use anyway. 271 */ 272 if (dabd == NULL || ABD_SCATTER(dabd).abd_iovcnt < iovcnt) 273 dabd = abd_alloc_struct(iovcnt << ABD_PAGESHIFT); 274 275 /* Set offset into first page in view */ 276 ABD_SCATTER(dabd).abd_offset = doff; 277 278 /* Copy the wanted iovecs from the source to the dest */ 279 memcpy(&ABD_SCATTER(dabd).abd_iov[0], 280 &ABD_SCATTER(sabd).abd_iov[soff >> ABD_PAGESHIFT], 281 iovcnt * sizeof (struct iovec)); 282 283 return (dabd); 284 } 285 286 void 287 abd_iter_init(struct abd_iter *aiter, abd_t *abd) 288 { 289 ASSERT(!abd_is_gang(abd)); 290 abd_verify(abd); 291 memset(aiter, 0, sizeof (struct abd_iter)); 292 aiter->iter_abd = abd; 293 } 294 295 boolean_t 296 abd_iter_at_end(struct abd_iter *aiter) 297 { 298 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size); 299 return (aiter->iter_pos == aiter->iter_abd->abd_size); 300 } 301 302 void 303 abd_iter_advance(struct abd_iter *aiter, size_t amount) 304 { 305 ASSERT0P(aiter->iter_mapaddr); 306 ASSERT0(aiter->iter_mapsize); 307 308 if (abd_iter_at_end(aiter)) 309 return; 310 311 aiter->iter_pos += amount; 312 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size); 313 } 314 315 void 316 abd_iter_map(struct abd_iter *aiter) 317 { 318 ASSERT0P(aiter->iter_mapaddr); 319 ASSERT0(aiter->iter_mapsize); 320 321 if (abd_iter_at_end(aiter)) 322 return; 323 324 if (abd_is_linear(aiter->iter_abd)) { 325 aiter->iter_mapaddr = 326 ABD_LINEAR_BUF(aiter->iter_abd) + aiter->iter_pos; 327 aiter->iter_mapsize = 328 aiter->iter_abd->abd_size - aiter->iter_pos; 329 return; 330 } 331 332 /* 333 * For scatter, we index into the appropriate iovec, and return the 334 * smaller of the amount requested, or up to the end of the page. 335 */ 336 size_t poff = aiter->iter_pos + ABD_SCATTER(aiter->iter_abd).abd_offset; 337 338 ASSERT3U(poff >> ABD_PAGESHIFT, <=, 339 ABD_SCATTER(aiter->iter_abd).abd_iovcnt); 340 struct iovec *iov = &ABD_SCATTER(aiter->iter_abd). 341 abd_iov[poff >> ABD_PAGESHIFT]; 342 343 aiter->iter_mapsize = MIN(ABD_PAGESIZE - (poff & ABD_PAGEMASK), 344 aiter->iter_abd->abd_size - aiter->iter_pos); 345 ASSERT3U(aiter->iter_mapsize, <=, ABD_PAGESIZE); 346 347 aiter->iter_mapaddr = iov->iov_base + (poff & ABD_PAGEMASK); 348 } 349 350 void 351 abd_iter_unmap(struct abd_iter *aiter) 352 { 353 if (abd_iter_at_end(aiter)) 354 return; 355 356 ASSERT3P(aiter->iter_mapaddr, !=, NULL); 357 ASSERT3U(aiter->iter_mapsize, >, 0); 358 359 aiter->iter_mapaddr = NULL; 360 aiter->iter_mapsize = 0; 361 } 362 363 void 364 abd_cache_reap_now(void) 365 { 366 } 367 368 /* 369 * Borrow a raw buffer from an ABD without copying the contents of the ABD 370 * into the buffer. If the ABD is scattered, this will alloate a raw buffer 371 * whose contents are undefined. To copy over the existing data in the ABD, use 372 * abd_borrow_buf_copy() instead. 373 */ 374 void * 375 abd_borrow_buf(abd_t *abd, size_t n) 376 { 377 void *buf; 378 abd_verify(abd); 379 ASSERT3U(abd->abd_size, >=, 0); 380 if (abd_is_linear(abd)) { 381 buf = abd_to_buf(abd); 382 } else { 383 buf = zio_buf_alloc(n); 384 } 385 #ifdef ZFS_DEBUG 386 (void) zfs_refcount_add_many(&abd->abd_children, n, buf); 387 #endif 388 return (buf); 389 } 390 391 void * 392 abd_borrow_buf_copy(abd_t *abd, size_t n) 393 { 394 void *buf = abd_borrow_buf(abd, n); 395 if (!abd_is_linear(abd)) { 396 abd_copy_to_buf(buf, abd, n); 397 } 398 return (buf); 399 } 400 401 /* 402 * Return a borrowed raw buffer to an ABD. If the ABD is scattered, this will 403 * no change the contents of the ABD and will ASSERT that you didn't modify 404 * the buffer since it was borrowed. If you want any changes you made to buf to 405 * be copied back to abd, use abd_return_buf_copy() instead. 406 */ 407 void 408 abd_return_buf(abd_t *abd, void *buf, size_t n) 409 { 410 abd_verify(abd); 411 ASSERT3U(abd->abd_size, >=, n); 412 #ifdef ZFS_DEBUG 413 (void) zfs_refcount_remove_many(&abd->abd_children, n, buf); 414 #endif 415 if (abd_is_linear(abd)) { 416 ASSERT3P(buf, ==, abd_to_buf(abd)); 417 } else { 418 ASSERT0(abd_cmp_buf(abd, buf, n)); 419 zio_buf_free(buf, n); 420 } 421 } 422 423 void 424 abd_return_buf_copy(abd_t *abd, void *buf, size_t n) 425 { 426 if (!abd_is_linear(abd)) { 427 abd_copy_from_buf(abd, buf, n); 428 } 429 abd_return_buf(abd, buf, n); 430 } 431