1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 23 * Copyright (c) 2019 by Delphix. All rights reserved. 24 * Copyright (c) 2023, 2024, Klara Inc. 25 */ 26 27 #include <sys/abd_impl.h> 28 #include <sys/param.h> 29 #include <sys/zio.h> 30 #include <sys/arc.h> 31 #include <sys/zfs_context.h> 32 #include <sys/zfs_znode.h> 33 34 /* 35 * We're simulating scatter/gather with 4K allocations, since that's more like 36 * what a typical kernel does. 37 */ 38 #define ABD_PAGESIZE (4096) 39 #define ABD_PAGESHIFT (12) 40 #define ABD_PAGEMASK (ABD_PAGESIZE-1) 41 42 /* 43 * See rationale in module/os/linux/zfs/abd_os.c, but in userspace this is 44 * mostly useful to get a mix of linear and scatter ABDs for testing. 45 */ 46 #define ABD_SCATTER_MIN_SIZE (512 * 3) 47 48 abd_t *abd_zero_scatter = NULL; 49 50 static uint_t 51 abd_iovcnt_for_bytes(size_t size) 52 { 53 /* 54 * Each iovec points to a 4K page. There's no real reason to do this 55 * in userspace, but our whole point here is to make it feel a bit 56 * more like a real paged memory model. 57 */ 58 return (P2ROUNDUP(size, ABD_PAGESIZE) / ABD_PAGESIZE); 59 } 60 61 abd_t * 62 abd_alloc_struct_impl(size_t size) 63 { 64 /* 65 * Zero-sized means it will be used for a linear or gang abd, so just 66 * allocate the abd itself and return. 67 */ 68 if (size == 0) 69 return (umem_alloc(sizeof (abd_t), UMEM_NOFAIL)); 70 71 /* 72 * Allocating for a scatter abd, so compute how many ABD_PAGESIZE 73 * iovecs we will need to hold this size. Append that allocation to the 74 * end. Note that struct abd_scatter has includes abd_iov[1], so we 75 * allocate one less iovec than we need. 76 * 77 * Note we're not allocating the pages proper, just the iovec pointers. 78 * That's down in abd_alloc_chunks. We _could_ do it here in a single 79 * allocation, but it's fiddly and harder to read for no real gain. 80 */ 81 uint_t n = abd_iovcnt_for_bytes(size); 82 abd_t *abd = umem_alloc(sizeof (abd_t) + (n-1) * sizeof (struct iovec), 83 UMEM_NOFAIL); 84 ABD_SCATTER(abd).abd_offset = 0; 85 ABD_SCATTER(abd).abd_iovcnt = n; 86 return (abd); 87 } 88 89 void 90 abd_free_struct_impl(abd_t *abd) 91 { 92 /* For scatter, compute the extra amount we need to free */ 93 uint_t iovcnt = 94 abd_is_linear(abd) || abd_is_gang(abd) ? 95 0 : (ABD_SCATTER(abd).abd_iovcnt - 1); 96 umem_free(abd, sizeof (abd_t) + iovcnt * sizeof (struct iovec)); 97 } 98 99 void 100 abd_alloc_chunks(abd_t *abd, size_t size) 101 { 102 /* 103 * We've already allocated the iovec array; ensure that the wanted size 104 * actually matches, otherwise the caller has made a mistake somewhere. 105 */ 106 uint_t n = ABD_SCATTER(abd).abd_iovcnt; 107 ASSERT3U(n, ==, abd_iovcnt_for_bytes(size)); 108 109 /* 110 * Allocate a ABD_PAGESIZE region for each iovec. 111 */ 112 struct iovec *iov = ABD_SCATTER(abd).abd_iov; 113 for (int i = 0; i < n; i++) { 114 iov[i].iov_base = 115 umem_alloc_aligned(ABD_PAGESIZE, ABD_PAGESIZE, UMEM_NOFAIL); 116 iov[i].iov_len = ABD_PAGESIZE; 117 } 118 } 119 120 void 121 abd_free_chunks(abd_t *abd) 122 { 123 uint_t n = ABD_SCATTER(abd).abd_iovcnt; 124 struct iovec *iov = ABD_SCATTER(abd).abd_iov; 125 for (int i = 0; i < n; i++) 126 umem_free_aligned(iov[i].iov_base, ABD_PAGESIZE); 127 } 128 129 boolean_t 130 abd_size_alloc_linear(size_t size) 131 { 132 return (size < ABD_SCATTER_MIN_SIZE); 133 } 134 135 void 136 abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op) 137 { 138 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR); 139 int waste = P2ROUNDUP(abd->abd_size, ABD_PAGESIZE) - abd->abd_size; 140 if (op == ABDSTAT_INCR) { 141 arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE); 142 } else { 143 arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE); 144 } 145 } 146 147 void 148 abd_update_linear_stats(abd_t *abd, abd_stats_op_t op) 149 { 150 (void) abd; 151 (void) op; 152 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR); 153 } 154 155 void 156 abd_verify_scatter(abd_t *abd) 157 { 158 #ifdef ZFS_DEBUG 159 /* 160 * scatter abds shall have: 161 * - at least one iovec 162 * - all iov_base point somewhere 163 * - all iov_len are ABD_PAGESIZE 164 * - offset set within the abd pages somewhere 165 */ 166 uint_t n = ABD_SCATTER(abd).abd_iovcnt; 167 ASSERT3U(n, >, 0); 168 169 uint_t len = 0; 170 for (int i = 0; i < n; i++) { 171 ASSERT3P(ABD_SCATTER(abd).abd_iov[i].iov_base, !=, NULL); 172 ASSERT3U(ABD_SCATTER(abd).abd_iov[i].iov_len, ==, ABD_PAGESIZE); 173 len += ABD_PAGESIZE; 174 } 175 176 ASSERT3U(ABD_SCATTER(abd).abd_offset, <, len); 177 #endif 178 } 179 180 void 181 abd_init(void) 182 { 183 /* 184 * Create the "zero" scatter abd. This is always the size of the 185 * largest possible block, but only actually has a single allocated 186 * page, which all iovecs in the abd point to. 187 */ 188 abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE); 189 abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER; 190 abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE; 191 192 void *zero = 193 umem_alloc_aligned(ABD_PAGESIZE, ABD_PAGESIZE, UMEM_NOFAIL); 194 memset(zero, 0, ABD_PAGESIZE); 195 196 uint_t n = abd_iovcnt_for_bytes(SPA_MAXBLOCKSIZE); 197 struct iovec *iov = ABD_SCATTER(abd_zero_scatter).abd_iov; 198 for (int i = 0; i < n; i++) { 199 iov[i].iov_base = zero; 200 iov[i].iov_len = ABD_PAGESIZE; 201 } 202 } 203 204 void 205 abd_fini(void) 206 { 207 umem_free_aligned( 208 ABD_SCATTER(abd_zero_scatter).abd_iov[0].iov_base, ABD_PAGESIZE); 209 abd_free_struct(abd_zero_scatter); 210 abd_zero_scatter = NULL; 211 } 212 213 void 214 abd_free_linear_page(abd_t *abd) 215 { 216 /* 217 * LINEAR_PAGE is specific to the Linux kernel; we never set this 218 * flag, so this will never be called. 219 */ 220 (void) abd; 221 PANIC("unreachable"); 222 } 223 224 abd_t * 225 abd_alloc_for_io(size_t size, boolean_t is_metadata) 226 { 227 return (abd_alloc(size, is_metadata)); 228 } 229 230 abd_t * 231 abd_get_offset_scatter(abd_t *dabd, abd_t *sabd, size_t off, size_t size) 232 { 233 234 /* 235 * Create a new scatter dabd by borrowing data pages from sabd to cover 236 * off+size. 237 * 238 * sabd is an existing scatter abd with a set of iovecs, each covering 239 * an ABD_PAGESIZE (4K) allocation. It's "zero" is at abd_offset. 240 * 241 * [........][........][........][........] 242 * ^- sabd_offset 243 * 244 * We want to produce a new abd, referencing those allocations at the 245 * given offset. 246 * 247 * [........][........][........][........] 248 * ^- dabd_offset = sabd_offset + off 249 * ^- dabd_offset + size 250 * 251 * In this example, dabd needs three iovecs. The first iovec is offset 252 * 0, so the final dabd_offset is masked back into the first iovec. 253 * 254 * [........][........][........] 255 * ^- dabd_offset 256 */ 257 size_t soff = ABD_SCATTER(sabd).abd_offset + off; 258 size_t doff = soff & ABD_PAGEMASK; 259 size_t iovcnt = abd_iovcnt_for_bytes(doff + size); 260 261 /* 262 * If the passed-in abd has enough allocated iovecs already, reuse it. 263 * Otherwise, make a new one. The caller will free the original if the 264 * one it gets back is not the same. 265 * 266 * Note that it's ok if we reuse an abd with more iovecs than we need. 267 * abd_size has the usable amount of data, and the abd does not own the 268 * pages referenced by the iovecs. At worst, they're holding dangling 269 * pointers that we'll never use anyway. 270 */ 271 if (dabd == NULL || ABD_SCATTER(dabd).abd_iovcnt < iovcnt) 272 dabd = abd_alloc_struct(iovcnt << ABD_PAGESHIFT); 273 274 /* Set offset into first page in view */ 275 ABD_SCATTER(dabd).abd_offset = doff; 276 277 /* Copy the wanted iovecs from the source to the dest */ 278 memcpy(&ABD_SCATTER(dabd).abd_iov[0], 279 &ABD_SCATTER(sabd).abd_iov[soff >> ABD_PAGESHIFT], 280 iovcnt * sizeof (struct iovec)); 281 282 return (dabd); 283 } 284 285 void 286 abd_iter_init(struct abd_iter *aiter, abd_t *abd) 287 { 288 ASSERT(!abd_is_gang(abd)); 289 abd_verify(abd); 290 memset(aiter, 0, sizeof (struct abd_iter)); 291 aiter->iter_abd = abd; 292 } 293 294 boolean_t 295 abd_iter_at_end(struct abd_iter *aiter) 296 { 297 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size); 298 return (aiter->iter_pos == aiter->iter_abd->abd_size); 299 } 300 301 void 302 abd_iter_advance(struct abd_iter *aiter, size_t amount) 303 { 304 ASSERT3P(aiter->iter_mapaddr, ==, NULL); 305 ASSERT0(aiter->iter_mapsize); 306 307 if (abd_iter_at_end(aiter)) 308 return; 309 310 aiter->iter_pos += amount; 311 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size); 312 } 313 314 void 315 abd_iter_map(struct abd_iter *aiter) 316 { 317 ASSERT3P(aiter->iter_mapaddr, ==, NULL); 318 ASSERT0(aiter->iter_mapsize); 319 320 if (abd_iter_at_end(aiter)) 321 return; 322 323 if (abd_is_linear(aiter->iter_abd)) { 324 aiter->iter_mapaddr = 325 ABD_LINEAR_BUF(aiter->iter_abd) + aiter->iter_pos; 326 aiter->iter_mapsize = 327 aiter->iter_abd->abd_size - aiter->iter_pos; 328 return; 329 } 330 331 /* 332 * For scatter, we index into the appropriate iovec, and return the 333 * smaller of the amount requested, or up to the end of the page. 334 */ 335 size_t poff = aiter->iter_pos + ABD_SCATTER(aiter->iter_abd).abd_offset; 336 337 ASSERT3U(poff >> ABD_PAGESHIFT, <=, 338 ABD_SCATTER(aiter->iter_abd).abd_iovcnt); 339 struct iovec *iov = &ABD_SCATTER(aiter->iter_abd). 340 abd_iov[poff >> ABD_PAGESHIFT]; 341 342 aiter->iter_mapsize = MIN(ABD_PAGESIZE - (poff & ABD_PAGEMASK), 343 aiter->iter_abd->abd_size - aiter->iter_pos); 344 ASSERT3U(aiter->iter_mapsize, <=, ABD_PAGESIZE); 345 346 aiter->iter_mapaddr = iov->iov_base + (poff & ABD_PAGEMASK); 347 } 348 349 void 350 abd_iter_unmap(struct abd_iter *aiter) 351 { 352 if (abd_iter_at_end(aiter)) 353 return; 354 355 ASSERT3P(aiter->iter_mapaddr, !=, NULL); 356 ASSERT3U(aiter->iter_mapsize, >, 0); 357 358 aiter->iter_mapaddr = NULL; 359 aiter->iter_mapsize = 0; 360 } 361 362 void 363 abd_cache_reap_now(void) 364 { 365 } 366 367 /* 368 * Borrow a raw buffer from an ABD without copying the contents of the ABD 369 * into the buffer. If the ABD is scattered, this will alloate a raw buffer 370 * whose contents are undefined. To copy over the existing data in the ABD, use 371 * abd_borrow_buf_copy() instead. 372 */ 373 void * 374 abd_borrow_buf(abd_t *abd, size_t n) 375 { 376 void *buf; 377 abd_verify(abd); 378 ASSERT3U(abd->abd_size, >=, 0); 379 if (abd_is_linear(abd)) { 380 buf = abd_to_buf(abd); 381 } else { 382 buf = zio_buf_alloc(n); 383 } 384 #ifdef ZFS_DEBUG 385 (void) zfs_refcount_add_many(&abd->abd_children, n, buf); 386 #endif 387 return (buf); 388 } 389 390 void * 391 abd_borrow_buf_copy(abd_t *abd, size_t n) 392 { 393 void *buf = abd_borrow_buf(abd, n); 394 if (!abd_is_linear(abd)) { 395 abd_copy_to_buf(buf, abd, n); 396 } 397 return (buf); 398 } 399 400 /* 401 * Return a borrowed raw buffer to an ABD. If the ABD is scattered, this will 402 * no change the contents of the ABD and will ASSERT that you didn't modify 403 * the buffer since it was borrowed. If you want any changes you made to buf to 404 * be copied back to abd, use abd_return_buf_copy() instead. 405 */ 406 void 407 abd_return_buf(abd_t *abd, void *buf, size_t n) 408 { 409 abd_verify(abd); 410 ASSERT3U(abd->abd_size, >=, n); 411 #ifdef ZFS_DEBUG 412 (void) zfs_refcount_remove_many(&abd->abd_children, n, buf); 413 #endif 414 if (abd_is_linear(abd)) { 415 ASSERT3P(buf, ==, abd_to_buf(abd)); 416 } else { 417 ASSERT0(abd_cmp_buf(abd, buf, n)); 418 zio_buf_free(buf, n); 419 } 420 } 421 422 void 423 abd_return_buf_copy(abd_t *abd, void *buf, size_t n) 424 { 425 if (!abd_is_linear(abd)) { 426 abd_copy_from_buf(abd, buf, n); 427 } 428 abd_return_buf(abd, buf, n); 429 } 430