1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2008 Yahoo!, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the author nor the names of any co-contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/bio.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/proc.h> 42 #include <sys/sglist.h> 43 #include <sys/uio.h> 44 45 #include <vm/vm.h> 46 #include <vm/vm_page.h> 47 #include <vm/pmap.h> 48 #include <vm/vm_map.h> 49 50 #include <sys/ktr.h> 51 52 static MALLOC_DEFINE(M_SGLIST, "sglist", "scatter/gather lists"); 53 54 /* 55 * Convenience macros to save the state of an sglist so it can be restored 56 * if an append attempt fails. Since sglist's only grow we only need to 57 * save the current count of segments and the length of the ending segment. 58 * Earlier segments will not be changed by an append, and the only change 59 * that can occur to the ending segment is that it can be extended. 60 */ 61 struct sgsave { 62 u_short sg_nseg; 63 size_t ss_len; 64 }; 65 66 #define SGLIST_SAVE(sg, sgsave) do { \ 67 (sgsave).sg_nseg = (sg)->sg_nseg; \ 68 if ((sgsave).sg_nseg > 0) \ 69 (sgsave).ss_len = (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len; \ 70 else \ 71 (sgsave).ss_len = 0; \ 72 } while (0) 73 74 #define SGLIST_RESTORE(sg, sgsave) do { \ 75 (sg)->sg_nseg = (sgsave).sg_nseg; \ 76 if ((sgsave).sg_nseg > 0) \ 77 (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len = (sgsave).ss_len; \ 78 } while (0) 79 80 /* 81 * Append a single (paddr, len) to a sglist. sg is the list and ss is 82 * the current segment in the list. If we run out of segments then 83 * EFBIG will be returned. 84 */ 85 static __inline int 86 _sglist_append_range(struct sglist *sg, struct sglist_seg **ssp, 87 vm_paddr_t paddr, size_t len) 88 { 89 struct sglist_seg *ss; 90 91 ss = *ssp; 92 if (ss->ss_paddr + ss->ss_len == paddr) 93 ss->ss_len += len; 94 else { 95 if (sg->sg_nseg == sg->sg_maxseg) 96 return (EFBIG); 97 ss++; 98 ss->ss_paddr = paddr; 99 ss->ss_len = len; 100 sg->sg_nseg++; 101 *ssp = ss; 102 } 103 return (0); 104 } 105 106 /* 107 * Worker routine to append a virtual address range (either kernel or 108 * user) to a scatter/gather list. 109 */ 110 static __inline int 111 _sglist_append_buf(struct sglist *sg, void *buf, size_t len, pmap_t pmap, 112 size_t *donep) 113 { 114 struct sglist_seg *ss; 115 vm_offset_t vaddr, offset; 116 vm_paddr_t paddr; 117 size_t seglen; 118 int error; 119 120 if (donep) 121 *donep = 0; 122 if (len == 0) 123 return (0); 124 125 /* Do the first page. It may have an offset. */ 126 vaddr = (vm_offset_t)buf; 127 offset = vaddr & PAGE_MASK; 128 if (pmap != NULL) 129 paddr = pmap_extract(pmap, vaddr); 130 else 131 paddr = pmap_kextract(vaddr); 132 seglen = MIN(len, PAGE_SIZE - offset); 133 if (sg->sg_nseg == 0) { 134 ss = sg->sg_segs; 135 ss->ss_paddr = paddr; 136 ss->ss_len = seglen; 137 sg->sg_nseg = 1; 138 } else { 139 ss = &sg->sg_segs[sg->sg_nseg - 1]; 140 error = _sglist_append_range(sg, &ss, paddr, seglen); 141 if (error) 142 return (error); 143 } 144 vaddr += seglen; 145 len -= seglen; 146 if (donep) 147 *donep += seglen; 148 149 while (len > 0) { 150 seglen = MIN(len, PAGE_SIZE); 151 if (pmap != NULL) 152 paddr = pmap_extract(pmap, vaddr); 153 else 154 paddr = pmap_kextract(vaddr); 155 error = _sglist_append_range(sg, &ss, paddr, seglen); 156 if (error) 157 return (error); 158 vaddr += seglen; 159 len -= seglen; 160 if (donep) 161 *donep += seglen; 162 } 163 164 return (0); 165 } 166 167 /* 168 * Determine the number of scatter/gather list elements needed to 169 * describe a kernel virtual address range. 170 */ 171 int 172 sglist_count(void *buf, size_t len) 173 { 174 vm_offset_t vaddr, vendaddr; 175 vm_paddr_t lastaddr, paddr; 176 int nsegs; 177 178 if (len == 0) 179 return (0); 180 181 vaddr = trunc_page((vm_offset_t)buf); 182 vendaddr = (vm_offset_t)buf + len; 183 nsegs = 1; 184 lastaddr = pmap_kextract(vaddr); 185 vaddr += PAGE_SIZE; 186 while (vaddr < vendaddr) { 187 paddr = pmap_kextract(vaddr); 188 if (lastaddr + PAGE_SIZE != paddr) 189 nsegs++; 190 lastaddr = paddr; 191 vaddr += PAGE_SIZE; 192 } 193 return (nsegs); 194 } 195 196 /* 197 * Determine the number of scatter/gather list elements needed to 198 * describe a buffer backed by an array of VM pages. 199 */ 200 int 201 sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len) 202 { 203 vm_paddr_t lastaddr, paddr; 204 int i, nsegs; 205 206 if (len == 0) 207 return (0); 208 209 len += pgoff; 210 nsegs = 1; 211 lastaddr = VM_PAGE_TO_PHYS(m[0]); 212 for (i = 1; len > PAGE_SIZE; len -= PAGE_SIZE, i++) { 213 paddr = VM_PAGE_TO_PHYS(m[i]); 214 if (lastaddr + PAGE_SIZE != paddr) 215 nsegs++; 216 lastaddr = paddr; 217 } 218 return (nsegs); 219 } 220 221 /* 222 * Allocate a scatter/gather list along with 'nsegs' segments. The 223 * 'mflags' parameters are the same as passed to malloc(9). The caller 224 * should use sglist_free() to free this list. 225 */ 226 struct sglist * 227 sglist_alloc(int nsegs, int mflags) 228 { 229 struct sglist *sg; 230 231 sg = malloc(sizeof(struct sglist) + nsegs * sizeof(struct sglist_seg), 232 M_SGLIST, mflags); 233 if (sg == NULL) 234 return (NULL); 235 sglist_init(sg, nsegs, (struct sglist_seg *)(sg + 1)); 236 return (sg); 237 } 238 239 /* 240 * Free a scatter/gather list allocated via sglist_allc(). 241 */ 242 void 243 sglist_free(struct sglist *sg) 244 { 245 246 if (sg == NULL) 247 return; 248 249 if (refcount_release(&sg->sg_refs)) 250 free(sg, M_SGLIST); 251 } 252 253 /* 254 * Append the segments to describe a single kernel virtual address 255 * range to a scatter/gather list. If there are insufficient 256 * segments, then this fails with EFBIG. 257 */ 258 int 259 sglist_append(struct sglist *sg, void *buf, size_t len) 260 { 261 struct sgsave save; 262 int error; 263 264 if (sg->sg_maxseg == 0) 265 return (EINVAL); 266 SGLIST_SAVE(sg, save); 267 error = _sglist_append_buf(sg, buf, len, NULL, NULL); 268 if (error) 269 SGLIST_RESTORE(sg, save); 270 return (error); 271 } 272 273 /* 274 * Append the segments to describe a bio's data to a scatter/gather list. 275 * If there are insufficient segments, then this fails with EFBIG. 276 * 277 * NOTE: This function expects bio_bcount to be initialized. 278 */ 279 int 280 sglist_append_bio(struct sglist *sg, struct bio *bp) 281 { 282 int error; 283 284 if ((bp->bio_flags & BIO_UNMAPPED) == 0) 285 error = sglist_append(sg, bp->bio_data, bp->bio_bcount); 286 else 287 error = sglist_append_vmpages(sg, bp->bio_ma, 288 bp->bio_ma_offset, bp->bio_bcount); 289 return (error); 290 } 291 292 /* 293 * Append a single physical address range to a scatter/gather list. 294 * If there are insufficient segments, then this fails with EFBIG. 295 */ 296 int 297 sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len) 298 { 299 struct sglist_seg *ss; 300 struct sgsave save; 301 int error; 302 303 if (sg->sg_maxseg == 0) 304 return (EINVAL); 305 if (len == 0) 306 return (0); 307 308 if (sg->sg_nseg == 0) { 309 sg->sg_segs[0].ss_paddr = paddr; 310 sg->sg_segs[0].ss_len = len; 311 sg->sg_nseg = 1; 312 return (0); 313 } 314 ss = &sg->sg_segs[sg->sg_nseg - 1]; 315 SGLIST_SAVE(sg, save); 316 error = _sglist_append_range(sg, &ss, paddr, len); 317 if (error) 318 SGLIST_RESTORE(sg, save); 319 return (error); 320 } 321 322 /* 323 * Append the segments that describe a single mbuf chain to a 324 * scatter/gather list. If there are insufficient segments, then this 325 * fails with EFBIG. 326 */ 327 int 328 sglist_append_mbuf(struct sglist *sg, struct mbuf *m0) 329 { 330 struct sgsave save; 331 struct mbuf *m; 332 int error; 333 334 if (sg->sg_maxseg == 0) 335 return (EINVAL); 336 337 error = 0; 338 SGLIST_SAVE(sg, save); 339 for (m = m0; m != NULL; m = m->m_next) { 340 if (m->m_len > 0) { 341 error = sglist_append(sg, m->m_data, m->m_len); 342 if (error) { 343 SGLIST_RESTORE(sg, save); 344 return (error); 345 } 346 } 347 } 348 return (0); 349 } 350 351 /* 352 * Append the segments that describe a buffer spanning an array of VM 353 * pages. The buffer begins at an offset of 'pgoff' in the first 354 * page. 355 */ 356 int 357 sglist_append_vmpages(struct sglist *sg, vm_page_t *m, size_t pgoff, 358 size_t len) 359 { 360 struct sgsave save; 361 struct sglist_seg *ss; 362 vm_paddr_t paddr; 363 size_t seglen; 364 int error, i; 365 366 if (sg->sg_maxseg == 0) 367 return (EINVAL); 368 if (len == 0) 369 return (0); 370 371 SGLIST_SAVE(sg, save); 372 i = 0; 373 if (sg->sg_nseg == 0) { 374 seglen = min(PAGE_SIZE - pgoff, len); 375 sg->sg_segs[0].ss_paddr = VM_PAGE_TO_PHYS(m[0]) + pgoff; 376 sg->sg_segs[0].ss_len = seglen; 377 sg->sg_nseg = 1; 378 pgoff = 0; 379 len -= seglen; 380 i++; 381 } 382 ss = &sg->sg_segs[sg->sg_nseg - 1]; 383 for (; len > 0; i++, len -= seglen) { 384 seglen = min(PAGE_SIZE - pgoff, len); 385 paddr = VM_PAGE_TO_PHYS(m[i]) + pgoff; 386 error = _sglist_append_range(sg, &ss, paddr, seglen); 387 if (error) { 388 SGLIST_RESTORE(sg, save); 389 return (error); 390 } 391 pgoff = 0; 392 } 393 return (0); 394 } 395 396 /* 397 * Append the segments that describe a single user address range to a 398 * scatter/gather list. If there are insufficient segments, then this 399 * fails with EFBIG. 400 */ 401 int 402 sglist_append_user(struct sglist *sg, void *buf, size_t len, struct thread *td) 403 { 404 struct sgsave save; 405 int error; 406 407 if (sg->sg_maxseg == 0) 408 return (EINVAL); 409 SGLIST_SAVE(sg, save); 410 error = _sglist_append_buf(sg, buf, len, 411 vmspace_pmap(td->td_proc->p_vmspace), NULL); 412 if (error) 413 SGLIST_RESTORE(sg, save); 414 return (error); 415 } 416 417 /* 418 * Append a subset of an existing scatter/gather list 'source' to a 419 * the scatter/gather list 'sg'. If there are insufficient segments, 420 * then this fails with EFBIG. 421 */ 422 int 423 sglist_append_sglist(struct sglist *sg, struct sglist *source, size_t offset, 424 size_t length) 425 { 426 struct sgsave save; 427 struct sglist_seg *ss; 428 size_t seglen; 429 int error, i; 430 431 if (sg->sg_maxseg == 0 || length == 0) 432 return (EINVAL); 433 SGLIST_SAVE(sg, save); 434 error = EINVAL; 435 ss = &sg->sg_segs[sg->sg_nseg - 1]; 436 for (i = 0; i < source->sg_nseg; i++) { 437 if (offset >= source->sg_segs[i].ss_len) { 438 offset -= source->sg_segs[i].ss_len; 439 continue; 440 } 441 seglen = source->sg_segs[i].ss_len - offset; 442 if (seglen > length) 443 seglen = length; 444 error = _sglist_append_range(sg, &ss, 445 source->sg_segs[i].ss_paddr + offset, seglen); 446 if (error) 447 break; 448 offset = 0; 449 length -= seglen; 450 if (length == 0) 451 break; 452 } 453 if (length != 0) 454 error = EINVAL; 455 if (error) 456 SGLIST_RESTORE(sg, save); 457 return (error); 458 } 459 460 /* 461 * Append the segments that describe a single uio to a scatter/gather 462 * list. If there are insufficient segments, then this fails with 463 * EFBIG. 464 */ 465 int 466 sglist_append_uio(struct sglist *sg, struct uio *uio) 467 { 468 struct iovec *iov; 469 struct sgsave save; 470 size_t resid, minlen; 471 pmap_t pmap; 472 int error, i; 473 474 if (sg->sg_maxseg == 0) 475 return (EINVAL); 476 477 resid = uio->uio_resid; 478 iov = uio->uio_iov; 479 480 if (uio->uio_segflg == UIO_USERSPACE) { 481 KASSERT(uio->uio_td != NULL, 482 ("sglist_append_uio: USERSPACE but no thread")); 483 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 484 } else 485 pmap = NULL; 486 487 error = 0; 488 SGLIST_SAVE(sg, save); 489 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 490 /* 491 * Now at the first iovec to load. Load each iovec 492 * until we have exhausted the residual count. 493 */ 494 minlen = MIN(resid, iov[i].iov_len); 495 if (minlen > 0) { 496 error = _sglist_append_buf(sg, iov[i].iov_base, minlen, 497 pmap, NULL); 498 if (error) { 499 SGLIST_RESTORE(sg, save); 500 return (error); 501 } 502 resid -= minlen; 503 } 504 } 505 return (0); 506 } 507 508 /* 509 * Append the segments that describe at most 'resid' bytes from a 510 * single uio to a scatter/gather list. If there are insufficient 511 * segments, then only the amount that fits is appended. 512 */ 513 int 514 sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid) 515 { 516 struct iovec *iov; 517 size_t done; 518 pmap_t pmap; 519 int error, len; 520 521 if (sg->sg_maxseg == 0) 522 return (EINVAL); 523 524 if (uio->uio_segflg == UIO_USERSPACE) { 525 KASSERT(uio->uio_td != NULL, 526 ("sglist_consume_uio: USERSPACE but no thread")); 527 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 528 } else 529 pmap = NULL; 530 531 error = 0; 532 while (resid > 0 && uio->uio_resid) { 533 iov = uio->uio_iov; 534 len = iov->iov_len; 535 if (len == 0) { 536 uio->uio_iov++; 537 uio->uio_iovcnt--; 538 continue; 539 } 540 if (len > resid) 541 len = resid; 542 543 /* 544 * Try to append this iovec. If we run out of room, 545 * then break out of the loop. 546 */ 547 error = _sglist_append_buf(sg, iov->iov_base, len, pmap, &done); 548 iov->iov_base = (char *)iov->iov_base + done; 549 iov->iov_len -= done; 550 uio->uio_resid -= done; 551 uio->uio_offset += done; 552 resid -= done; 553 if (error) 554 break; 555 } 556 return (0); 557 } 558 559 /* 560 * Allocate and populate a scatter/gather list to describe a single 561 * kernel virtual address range. 562 */ 563 struct sglist * 564 sglist_build(void *buf, size_t len, int mflags) 565 { 566 struct sglist *sg; 567 int nsegs; 568 569 if (len == 0) 570 return (NULL); 571 572 nsegs = sglist_count(buf, len); 573 sg = sglist_alloc(nsegs, mflags); 574 if (sg == NULL) 575 return (NULL); 576 if (sglist_append(sg, buf, len) != 0) { 577 sglist_free(sg); 578 return (NULL); 579 } 580 return (sg); 581 } 582 583 /* 584 * Clone a new copy of a scatter/gather list. 585 */ 586 struct sglist * 587 sglist_clone(struct sglist *sg, int mflags) 588 { 589 struct sglist *new; 590 591 if (sg == NULL) 592 return (NULL); 593 new = sglist_alloc(sg->sg_maxseg, mflags); 594 if (new == NULL) 595 return (NULL); 596 new->sg_nseg = sg->sg_nseg; 597 bcopy(sg->sg_segs, new->sg_segs, sizeof(struct sglist_seg) * 598 sg->sg_nseg); 599 return (new); 600 } 601 602 /* 603 * Calculate the total length of the segments described in a 604 * scatter/gather list. 605 */ 606 size_t 607 sglist_length(struct sglist *sg) 608 { 609 size_t space; 610 int i; 611 612 space = 0; 613 for (i = 0; i < sg->sg_nseg; i++) 614 space += sg->sg_segs[i].ss_len; 615 return (space); 616 } 617 618 /* 619 * Split a scatter/gather list into two lists. The scatter/gather 620 * entries for the first 'length' bytes of the 'original' list are 621 * stored in the '*head' list and are removed from 'original'. 622 * 623 * If '*head' is NULL, then a new list will be allocated using 624 * 'mflags'. If M_NOWAIT is specified and the allocation fails, 625 * ENOMEM will be returned. 626 * 627 * If '*head' is not NULL, it should point to an empty sglist. If it 628 * does not have enough room for the remaining space, then EFBIG will 629 * be returned. If '*head' is not empty, then EINVAL will be 630 * returned. 631 * 632 * If 'original' is shared (refcount > 1), then EDOOFUS will be 633 * returned. 634 */ 635 int 636 sglist_split(struct sglist *original, struct sglist **head, size_t length, 637 int mflags) 638 { 639 struct sglist *sg; 640 size_t space, split; 641 int count, i; 642 643 if (original->sg_refs > 1) 644 return (EDOOFUS); 645 646 /* Figure out how big of a sglist '*head' has to hold. */ 647 count = 0; 648 space = 0; 649 split = 0; 650 for (i = 0; i < original->sg_nseg; i++) { 651 space += original->sg_segs[i].ss_len; 652 count++; 653 if (space >= length) { 654 /* 655 * If 'length' falls in the middle of a 656 * scatter/gather list entry, then 'split' 657 * holds how much of that entry will remain in 658 * 'original'. 659 */ 660 split = space - length; 661 break; 662 } 663 } 664 665 /* Nothing to do, so leave head empty. */ 666 if (count == 0) 667 return (0); 668 669 if (*head == NULL) { 670 sg = sglist_alloc(count, mflags); 671 if (sg == NULL) 672 return (ENOMEM); 673 *head = sg; 674 } else { 675 sg = *head; 676 if (sg->sg_maxseg < count) 677 return (EFBIG); 678 if (sg->sg_nseg != 0) 679 return (EINVAL); 680 } 681 682 /* Copy 'count' entries to 'sg' from 'original'. */ 683 bcopy(original->sg_segs, sg->sg_segs, count * 684 sizeof(struct sglist_seg)); 685 sg->sg_nseg = count; 686 687 /* 688 * If we had to split a list entry, fixup the last entry in 689 * 'sg' and the new first entry in 'original'. We also 690 * decrement 'count' by 1 since we will only be removing 691 * 'count - 1' segments from 'original' now. 692 */ 693 if (split != 0) { 694 count--; 695 sg->sg_segs[count].ss_len -= split; 696 original->sg_segs[count].ss_paddr = 697 sg->sg_segs[count].ss_paddr + split; 698 original->sg_segs[count].ss_len = split; 699 } 700 701 /* Trim 'count' entries from the front of 'original'. */ 702 original->sg_nseg -= count; 703 bcopy(original->sg_segs + count, original->sg_segs, count * 704 sizeof(struct sglist_seg)); 705 return (0); 706 } 707 708 /* 709 * Append the scatter/gather list elements in 'second' to the 710 * scatter/gather list 'first'. If there is not enough space in 711 * 'first', EFBIG is returned. 712 */ 713 int 714 sglist_join(struct sglist *first, struct sglist *second) 715 { 716 struct sglist_seg *flast, *sfirst; 717 int append; 718 719 /* If 'second' is empty, there is nothing to do. */ 720 if (second->sg_nseg == 0) 721 return (0); 722 723 /* 724 * If the first entry in 'second' can be appended to the last entry 725 * in 'first' then set append to '1'. 726 */ 727 append = 0; 728 flast = &first->sg_segs[first->sg_nseg - 1]; 729 sfirst = &second->sg_segs[0]; 730 if (first->sg_nseg != 0 && 731 flast->ss_paddr + flast->ss_len == sfirst->ss_paddr) 732 append = 1; 733 734 /* Make sure 'first' has enough room. */ 735 if (first->sg_nseg + second->sg_nseg - append > first->sg_maxseg) 736 return (EFBIG); 737 738 /* Merge last in 'first' and first in 'second' if needed. */ 739 if (append) 740 flast->ss_len += sfirst->ss_len; 741 742 /* Append new segments from 'second' to 'first'. */ 743 bcopy(first->sg_segs + first->sg_nseg, second->sg_segs + append, 744 (second->sg_nseg - append) * sizeof(struct sglist_seg)); 745 first->sg_nseg += second->sg_nseg - append; 746 sglist_reset(second); 747 return (0); 748 } 749 750 /* 751 * Generate a new scatter/gather list from a range of an existing 752 * scatter/gather list. The 'offset' and 'length' parameters specify 753 * the logical range of the 'original' list to extract. If that range 754 * is not a subset of the length of 'original', then EINVAL is 755 * returned. The new scatter/gather list is stored in '*slice'. 756 * 757 * If '*slice' is NULL, then a new list will be allocated using 758 * 'mflags'. If M_NOWAIT is specified and the allocation fails, 759 * ENOMEM will be returned. 760 * 761 * If '*slice' is not NULL, it should point to an empty sglist. If it 762 * does not have enough room for the remaining space, then EFBIG will 763 * be returned. If '*slice' is not empty, then EINVAL will be 764 * returned. 765 */ 766 int 767 sglist_slice(struct sglist *original, struct sglist **slice, size_t offset, 768 size_t length, int mflags) 769 { 770 struct sglist *sg; 771 size_t space, end, foffs, loffs; 772 int count, i, fseg; 773 774 /* Nothing to do. */ 775 if (length == 0) 776 return (0); 777 778 /* Figure out how many segments '*slice' needs to have. */ 779 end = offset + length; 780 space = 0; 781 count = 0; 782 fseg = 0; 783 foffs = loffs = 0; 784 for (i = 0; i < original->sg_nseg; i++) { 785 space += original->sg_segs[i].ss_len; 786 if (space > offset) { 787 /* 788 * When we hit the first segment, store its index 789 * in 'fseg' and the offset into the first segment 790 * of 'offset' in 'foffs'. 791 */ 792 if (count == 0) { 793 fseg = i; 794 foffs = offset - (space - 795 original->sg_segs[i].ss_len); 796 CTR1(KTR_DEV, "sglist_slice: foffs = %08lx", 797 foffs); 798 } 799 count++; 800 801 /* 802 * When we hit the last segment, break out of 803 * the loop. Store the amount of extra space 804 * at the end of this segment in 'loffs'. 805 */ 806 if (space >= end) { 807 loffs = space - end; 808 CTR1(KTR_DEV, "sglist_slice: loffs = %08lx", 809 loffs); 810 break; 811 } 812 } 813 } 814 815 /* If we never hit 'end', then 'length' ran off the end, so fail. */ 816 if (space < end) 817 return (EINVAL); 818 819 if (*slice == NULL) { 820 sg = sglist_alloc(count, mflags); 821 if (sg == NULL) 822 return (ENOMEM); 823 *slice = sg; 824 } else { 825 sg = *slice; 826 if (sg->sg_maxseg < count) 827 return (EFBIG); 828 if (sg->sg_nseg != 0) 829 return (EINVAL); 830 } 831 832 /* 833 * Copy over 'count' segments from 'original' starting at 834 * 'fseg' to 'sg'. 835 */ 836 bcopy(original->sg_segs + fseg, sg->sg_segs, 837 count * sizeof(struct sglist_seg)); 838 sg->sg_nseg = count; 839 840 /* Fixup first and last segments if needed. */ 841 if (foffs != 0) { 842 sg->sg_segs[0].ss_paddr += foffs; 843 sg->sg_segs[0].ss_len -= foffs; 844 CTR2(KTR_DEV, "sglist_slice seg[0]: %08lx:%08lx", 845 (long)sg->sg_segs[0].ss_paddr, sg->sg_segs[0].ss_len); 846 } 847 if (loffs != 0) { 848 sg->sg_segs[count - 1].ss_len -= loffs; 849 CTR2(KTR_DEV, "sglist_slice seg[%d]: len %08x", count - 1, 850 sg->sg_segs[count - 1].ss_len); 851 } 852 return (0); 853 } 854