1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * Copyright (c) 2016 Matthew Macy 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #ifndef _LINUXKPI_LINUX_SCATTERLIST_H_ 32 #define _LINUXKPI_LINUX_SCATTERLIST_H_ 33 34 #include <sys/types.h> 35 #include <sys/proc.h> 36 #include <sys/sched.h> 37 #include <sys/sf_buf.h> 38 39 #include <linux/err.h> 40 #include <linux/page.h> 41 #include <linux/slab.h> 42 #include <linux/mm.h> 43 44 struct bus_dmamap; 45 struct scatterlist { 46 unsigned long page_link; 47 #define SG_PAGE_LINK_CHAIN 0x1UL 48 #define SG_PAGE_LINK_LAST 0x2UL 49 #define SG_PAGE_LINK_MASK 0x3UL 50 unsigned int offset; 51 unsigned int length; 52 dma_addr_t dma_address; 53 unsigned int dma_length; 54 struct bus_dmamap *dma_map; /* FreeBSD specific */ 55 }; 56 57 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 58 59 struct sg_table { 60 struct scatterlist *sgl; 61 unsigned int nents; 62 unsigned int orig_nents; 63 }; 64 65 struct sg_page_iter { 66 struct scatterlist *sg; 67 unsigned int sg_pgoffset; 68 unsigned int maxents; 69 struct { 70 unsigned int nents; 71 int pg_advance; 72 } internal; 73 }; 74 75 struct sg_dma_page_iter { 76 struct sg_page_iter base; 77 }; 78 79 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 80 81 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 82 83 #define SG_MAGIC 0x87654321UL 84 #define SG_CHAIN SG_PAGE_LINK_CHAIN 85 #define SG_END SG_PAGE_LINK_LAST 86 87 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 88 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 89 #define sg_chain_ptr(sg) \ 90 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 91 92 #define sg_dma_address(sg) (sg)->dma_address 93 #ifdef CONFIG_NEED_SG_DMA_LENGTH 94 #define sg_dma_len(sg) (sg)->dma_length 95 #else 96 #define sg_dma_len(sg) (sg)->length 97 #endif 98 99 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 100 for (_sg_iter_init(sgl, iter, nents, pgoffset, false); \ 101 (iter)->sg; _sg_iter_next(iter, false)) 102 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 103 for (_sg_iter_init(sgl, &(iter)->base, nents, pgoffset, true); \ 104 (iter)->base.sg; _sg_iter_next(&(iter)->base, true)) 105 106 #define for_each_sg(sglist, sg, sgmax, iter) \ 107 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 108 109 #define for_each_sgtable_sg(sgt, sg, i) \ 110 for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i) 111 112 #define for_each_sgtable_page(sgt, iter, pgoffset) \ 113 for_each_sg_page((sgt)->sgl, iter, (sgt)->orig_nents, pgoffset) 114 115 #define for_each_sgtable_dma_sg(sgt, sg, iter) \ 116 for_each_sg((sgt)->sgl, sg, (sgt)->nents, iter) 117 118 #define for_each_sgtable_dma_page(sgt, iter, pgoffset) \ 119 for_each_sg_dma_page((sgt)->sgl, iter, (sgt)->nents, pgoffset) 120 121 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 122 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 123 124 static inline void 125 sg_assign_page(struct scatterlist *sg, struct page *page) 126 { 127 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 128 129 sg->page_link = page_link | (unsigned long)page; 130 } 131 132 static inline void 133 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 134 unsigned int offset) 135 { 136 sg_assign_page(sg, page); 137 sg->offset = offset; 138 sg->length = len; 139 } 140 141 static inline struct page * 142 sg_page(struct scatterlist *sg) 143 { 144 return ((struct page *)(sg->page_link & ~SG_PAGE_LINK_MASK)); 145 } 146 147 static inline void 148 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 149 { 150 sg_set_page(sg, virt_to_page(buf), buflen, 151 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 152 } 153 154 static inline struct scatterlist * 155 sg_next(struct scatterlist *sg) 156 { 157 if (sg_is_last(sg)) 158 return (NULL); 159 sg++; 160 if (sg_is_chain(sg)) 161 sg = sg_chain_ptr(sg); 162 return (sg); 163 } 164 165 static inline vm_paddr_t 166 sg_phys(struct scatterlist *sg) 167 { 168 return (page_to_phys(sg_page(sg)) + sg->offset); 169 } 170 171 static inline void * 172 sg_virt(struct scatterlist *sg) 173 { 174 175 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 176 } 177 178 static inline void 179 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 180 struct scatterlist *sgl) 181 { 182 struct scatterlist *sg = &prv[prv_nents - 1]; 183 184 sg->offset = 0; 185 sg->length = 0; 186 sg->page_link = ((unsigned long)sgl | 187 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 188 } 189 190 static inline void 191 sg_mark_end(struct scatterlist *sg) 192 { 193 sg->page_link |= SG_PAGE_LINK_LAST; 194 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 195 } 196 197 static inline void 198 sg_init_marker(struct scatterlist *sg, uint32_t num_sgs) 199 { 200 sg_mark_end(&sg[num_sgs - 1]); 201 } 202 203 static inline void 204 sg_init_table(struct scatterlist *sg, unsigned int nents) 205 { 206 bzero(sg, sizeof(*sg) * nents); 207 sg_mark_end(&sg[nents - 1]); 208 } 209 210 static inline void 211 sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen) 212 { 213 sg_init_table(sg, 1); 214 sg_set_buf(sg, buf, buflen); 215 } 216 217 static struct scatterlist * 218 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 219 { 220 if (nents == SG_MAX_SINGLE_ALLOC) { 221 return ((void *)__get_free_page(gfp_mask)); 222 } else 223 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 224 } 225 226 static inline void 227 sg_kfree(struct scatterlist *sg, unsigned int nents) 228 { 229 if (nents == SG_MAX_SINGLE_ALLOC) { 230 free_page((unsigned long)sg); 231 } else 232 kfree(sg); 233 } 234 235 static inline void 236 __sg_free_table(struct sg_table *table, unsigned int max_ents, 237 bool skip_first_chunk, sg_free_fn * free_fn) 238 { 239 struct scatterlist *sgl, *next; 240 241 if (unlikely(!table->sgl)) 242 return; 243 244 sgl = table->sgl; 245 while (table->orig_nents) { 246 unsigned int alloc_size = table->orig_nents; 247 unsigned int sg_size; 248 249 if (alloc_size > max_ents) { 250 next = sg_chain_ptr(&sgl[max_ents - 1]); 251 alloc_size = max_ents; 252 sg_size = alloc_size - 1; 253 } else { 254 sg_size = alloc_size; 255 next = NULL; 256 } 257 258 table->orig_nents -= sg_size; 259 if (skip_first_chunk) 260 skip_first_chunk = 0; 261 else 262 free_fn(sgl, alloc_size); 263 sgl = next; 264 } 265 266 table->sgl = NULL; 267 } 268 269 static inline void 270 sg_free_table(struct sg_table *table) 271 { 272 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 273 } 274 275 static inline int 276 __sg_alloc_table(struct sg_table *table, unsigned int nents, 277 unsigned int max_ents, struct scatterlist *first_chunk, 278 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 279 { 280 struct scatterlist *sg, *prv; 281 unsigned int left; 282 283 memset(table, 0, sizeof(*table)); 284 285 if (nents == 0) 286 return (-EINVAL); 287 left = nents; 288 prv = NULL; 289 do { 290 unsigned int sg_size; 291 unsigned int alloc_size = left; 292 293 if (alloc_size > max_ents) { 294 alloc_size = max_ents; 295 sg_size = alloc_size - 1; 296 } else 297 sg_size = alloc_size; 298 299 left -= sg_size; 300 301 if (first_chunk) { 302 sg = first_chunk; 303 first_chunk = NULL; 304 } else { 305 sg = alloc_fn(alloc_size, gfp_mask); 306 } 307 if (unlikely(!sg)) { 308 if (prv) 309 table->nents = ++table->orig_nents; 310 311 return (-ENOMEM); 312 } 313 sg_init_table(sg, alloc_size); 314 table->nents = table->orig_nents += sg_size; 315 316 if (prv) 317 sg_chain(prv, max_ents, sg); 318 else 319 table->sgl = sg; 320 321 if (!left) 322 sg_mark_end(&sg[sg_size - 1]); 323 324 prv = sg; 325 } while (left); 326 327 return (0); 328 } 329 330 static inline int 331 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 332 { 333 int ret; 334 335 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 336 NULL, gfp_mask, sg_kmalloc); 337 if (unlikely(ret)) 338 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 339 340 return (ret); 341 } 342 343 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 344 static inline struct scatterlist * 345 __sg_alloc_table_from_pages(struct sg_table *sgt, 346 struct page **pages, unsigned int count, 347 unsigned long off, unsigned long size, 348 unsigned int max_segment, 349 struct scatterlist *prv, unsigned int left_pages, 350 gfp_t gfp_mask) 351 #else 352 static inline int 353 __sg_alloc_table_from_pages(struct sg_table *sgt, 354 struct page **pages, unsigned int count, 355 unsigned long off, unsigned long size, 356 unsigned int max_segment, gfp_t gfp_mask) 357 #endif 358 { 359 unsigned int i, segs, cur, len; 360 int rc; 361 struct scatterlist *s, *sg_iter; 362 363 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 364 if (prv != NULL) { 365 panic( 366 "Support for prv != NULL not implemented in " 367 "__sg_alloc_table_from_pages()"); 368 } 369 #endif 370 371 if (__predict_false(!max_segment || offset_in_page(max_segment))) 372 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 373 return (ERR_PTR(-EINVAL)); 374 #else 375 return (-EINVAL); 376 #endif 377 378 len = 0; 379 for (segs = i = 1; i < count; ++i) { 380 len += PAGE_SIZE; 381 if (len >= max_segment || 382 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 383 ++segs; 384 len = 0; 385 } 386 } 387 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 388 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 389 return (ERR_PTR(rc)); 390 #else 391 return (rc); 392 #endif 393 394 cur = 0; 395 for_each_sg(sgt->sgl, sg_iter, sgt->orig_nents, i) { 396 unsigned long seg_size; 397 unsigned int j; 398 399 /* 400 * We need to make sure that when we exit this loop "s" has the 401 * last sg in the chain so we can call sg_mark_end() on it. 402 * Only set this inside the loop since sg_iter will be iterated 403 * until it is NULL. 404 */ 405 s = sg_iter; 406 407 len = 0; 408 for (j = cur + 1; j < count; ++j) { 409 len += PAGE_SIZE; 410 if (len >= max_segment || page_to_pfn(pages[j]) != 411 page_to_pfn(pages[j - 1]) + 1) 412 break; 413 } 414 415 seg_size = ((j - cur) << PAGE_SHIFT) - off; 416 sg_set_page(s, pages[cur], MIN(size, seg_size), off); 417 size -= seg_size; 418 off = 0; 419 cur = j; 420 } 421 KASSERT(s != NULL, ("s is NULL after loop in __sg_alloc_table_from_pages()")); 422 423 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 424 if (left_pages == 0) 425 sg_mark_end(s); 426 427 return (s); 428 #else 429 return (0); 430 #endif 431 } 432 433 static inline int 434 sg_alloc_table_from_pages(struct sg_table *sgt, 435 struct page **pages, unsigned int count, 436 unsigned long off, unsigned long size, 437 gfp_t gfp_mask) 438 { 439 440 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 441 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, 442 SCATTERLIST_MAX_SEGMENT, NULL, 0, gfp_mask))); 443 #else 444 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 445 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 446 #endif 447 } 448 449 static inline int 450 sg_alloc_table_from_pages_segment(struct sg_table *sgt, 451 struct page **pages, unsigned int count, unsigned int off, 452 unsigned long size, unsigned int max_segment, gfp_t gfp_mask) 453 { 454 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 455 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, 456 max_segment, NULL, 0, gfp_mask))); 457 #else 458 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 459 max_segment, gfp_mask)); 460 #endif 461 } 462 463 static inline int 464 sg_nents(struct scatterlist *sg) 465 { 466 int nents; 467 468 for (nents = 0; sg; sg = sg_next(sg)) 469 nents++; 470 return (nents); 471 } 472 473 static inline void 474 __sg_page_iter_start(struct sg_page_iter *piter, 475 struct scatterlist *sglist, unsigned int nents, 476 unsigned long pgoffset) 477 { 478 piter->internal.pg_advance = 0; 479 piter->internal.nents = nents; 480 481 piter->sg = sglist; 482 piter->sg_pgoffset = pgoffset; 483 } 484 485 static inline void 486 _sg_iter_next(struct sg_page_iter *iter, bool is_dma) 487 { 488 struct scatterlist *sg; 489 unsigned int pgcount, sglength; 490 491 sg = iter->sg; 492 sglength = is_dma ? sg_dma_len(sg) : sg->length; 493 pgcount = (sg->offset + sglength + PAGE_SIZE - 1) >> PAGE_SHIFT; 494 495 ++iter->sg_pgoffset; 496 while (iter->sg_pgoffset >= pgcount) { 497 iter->sg_pgoffset -= pgcount; 498 sg = sg_next(sg); 499 --iter->maxents; 500 if (sg == NULL || iter->maxents == 0) 501 break; 502 sglength = is_dma ? sg_dma_len(sg) : sg->length; 503 pgcount = (sg->offset + sglength + PAGE_SIZE - 1) >> PAGE_SHIFT; 504 } 505 iter->sg = sg; 506 } 507 508 static inline int 509 sg_page_count(struct scatterlist *sg) 510 { 511 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 512 } 513 #define sg_dma_page_count(sg) \ 514 sg_page_count(sg) 515 516 static inline bool 517 __sg_page_iter_next(struct sg_page_iter *piter) 518 { 519 unsigned int pgcount; 520 521 if (piter->internal.nents == 0) 522 return (0); 523 if (piter->sg == NULL) 524 return (0); 525 526 piter->sg_pgoffset += piter->internal.pg_advance; 527 piter->internal.pg_advance = 1; 528 529 while (1) { 530 pgcount = sg_page_count(piter->sg); 531 if (likely(piter->sg_pgoffset < pgcount)) 532 break; 533 piter->sg_pgoffset -= pgcount; 534 piter->sg = sg_next(piter->sg); 535 if (--piter->internal.nents == 0) 536 return (0); 537 if (piter->sg == NULL) 538 return (0); 539 } 540 return (1); 541 } 542 #define __sg_page_iter_dma_next(itr) \ 543 __sg_page_iter_next(&(itr)->base) 544 545 static inline void 546 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 547 unsigned int nents, unsigned long pgoffset, bool is_dma) 548 { 549 if (nents) { 550 iter->sg = sgl; 551 iter->sg_pgoffset = pgoffset - 1; 552 iter->maxents = nents; 553 _sg_iter_next(iter, is_dma); 554 } else { 555 iter->sg = NULL; 556 iter->sg_pgoffset = 0; 557 iter->maxents = 0; 558 } 559 } 560 561 /* 562 * sg_page_iter_dma_address() is implemented as a macro because it 563 * needs to accept two different and identical structure types. This 564 * allows both old and new code to co-exist. The compile time assert 565 * adds some safety, that the structure sizes match. 566 */ 567 #define sg_page_iter_dma_address(spi) ({ \ 568 struct sg_page_iter *__spi = (void *)(spi); \ 569 dma_addr_t __dma_address; \ 570 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 571 __dma_address = __spi->sg->dma_address + \ 572 (__spi->sg_pgoffset << PAGE_SHIFT); \ 573 __dma_address; \ 574 }) 575 576 static inline struct page * 577 sg_page_iter_page(struct sg_page_iter *piter) 578 { 579 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 580 } 581 582 static __inline size_t 583 sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 584 const void *buf, size_t buflen, off_t skip) 585 { 586 struct sg_page_iter piter; 587 struct page *page; 588 struct sf_buf *sf; 589 size_t len, copied; 590 char *p, *b; 591 592 if (buflen == 0) 593 return (0); 594 595 b = __DECONST(char *, buf); 596 copied = 0; 597 sched_pin(); 598 for_each_sg_page(sgl, &piter, nents, 0) { 599 600 /* Skip to the start. */ 601 if (piter.sg->length <= skip) { 602 skip -= piter.sg->length; 603 continue; 604 } 605 606 /* See how much to copy. */ 607 KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)), 608 ("%s: sg len %u - skip %ju || buflen %zu is 0\n", 609 __func__, piter.sg->length, (uintmax_t)skip, buflen)); 610 len = min(piter.sg->length - skip, buflen); 611 612 page = sg_page_iter_page(&piter); 613 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 614 if (sf == NULL) 615 break; 616 p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; 617 memcpy(p, b, len); 618 sf_buf_free(sf); 619 620 /* We copied so nothing more to skip. */ 621 skip = 0; 622 copied += len; 623 /* Either we exactly filled the page, or we are done. */ 624 buflen -= len; 625 if (buflen == 0) 626 break; 627 b += len; 628 } 629 sched_unpin(); 630 631 return (copied); 632 } 633 634 static inline size_t 635 sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, 636 const void *buf, size_t buflen) 637 { 638 return (sg_pcopy_from_buffer(sgl, nents, buf, buflen, 0)); 639 } 640 641 static inline size_t 642 sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, 643 void *buf, size_t buflen, off_t offset) 644 { 645 struct sg_page_iter iter; 646 struct scatterlist *sg; 647 struct page *page; 648 struct sf_buf *sf; 649 char *vaddr; 650 size_t total = 0; 651 size_t len; 652 653 if (!PMAP_HAS_DMAP) 654 sched_pin(); 655 for_each_sg_page(sgl, &iter, nents, 0) { 656 sg = iter.sg; 657 658 if (offset >= sg->length) { 659 offset -= sg->length; 660 continue; 661 } 662 len = ulmin(buflen, sg->length - offset); 663 if (len == 0) 664 break; 665 666 page = sg_page_iter_page(&iter); 667 if (!PMAP_HAS_DMAP) { 668 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 669 if (sf == NULL) 670 break; 671 vaddr = sf_buf_kva(sf); 672 } else 673 vaddr = PHYS_TO_DMAP(page_to_phys(page)); 674 memcpy(buf, vaddr + sg->offset + offset, len); 675 if (!PMAP_HAS_DMAP) 676 sf_buf_free(sf); 677 678 /* start at beginning of next page */ 679 offset = 0; 680 681 /* advance buffer */ 682 buf = (char *)buf + len; 683 buflen -= len; 684 total += len; 685 } 686 if (!PMAP_HAS_DMAP) 687 sched_unpin(); 688 return (total); 689 } 690 691 static inline void 692 sg_set_folio(struct scatterlist *sg, struct folio *folio, size_t len, 693 size_t offset) 694 { 695 sg_set_page(sg, &folio->page, len, offset); 696 } 697 698 #endif /* _LINUXKPI_LINUX_SCATTERLIST_H_ */ 699