1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * Copyright (c) 2016 Matthew Macy 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #ifndef _LINUXKPI_LINUX_SCATTERLIST_H_ 32 #define _LINUXKPI_LINUX_SCATTERLIST_H_ 33 34 #include <sys/types.h> 35 #include <sys/proc.h> 36 #include <sys/sched.h> 37 #include <sys/sf_buf.h> 38 39 #include <linux/err.h> 40 #include <linux/page.h> 41 #include <linux/slab.h> 42 #include <linux/mm.h> 43 44 struct bus_dmamap; 45 struct scatterlist { 46 unsigned long page_link; 47 #define SG_PAGE_LINK_CHAIN 0x1UL 48 #define SG_PAGE_LINK_LAST 0x2UL 49 #define SG_PAGE_LINK_MASK 0x3UL 50 unsigned int offset; 51 unsigned int length; 52 dma_addr_t dma_address; 53 struct bus_dmamap *dma_map; /* FreeBSD specific */ 54 }; 55 56 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 57 58 struct sg_table { 59 struct scatterlist *sgl; 60 unsigned int nents; 61 unsigned int orig_nents; 62 }; 63 64 struct sg_page_iter { 65 struct scatterlist *sg; 66 unsigned int sg_pgoffset; 67 unsigned int maxents; 68 struct { 69 unsigned int nents; 70 int pg_advance; 71 } internal; 72 }; 73 74 struct sg_dma_page_iter { 75 struct sg_page_iter base; 76 }; 77 78 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 79 80 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 81 82 #define SG_MAGIC 0x87654321UL 83 #define SG_CHAIN SG_PAGE_LINK_CHAIN 84 #define SG_END SG_PAGE_LINK_LAST 85 86 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 87 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 88 #define sg_chain_ptr(sg) \ 89 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 90 91 #define sg_dma_address(sg) (sg)->dma_address 92 #define sg_dma_len(sg) (sg)->length 93 94 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 95 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 96 (iter)->sg; _sg_iter_next(iter)) 97 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 98 for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) 99 100 #define for_each_sg(sglist, sg, sgmax, iter) \ 101 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 102 103 #define for_each_sgtable_sg(sgt, sg, i) \ 104 for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i) 105 106 #define for_each_sgtable_page(sgt, iter, pgoffset) \ 107 for_each_sg_page((sgt)->sgl, iter, (sgt)->orig_nents, pgoffset) 108 109 #define for_each_sgtable_dma_sg(sgt, sg, iter) \ 110 for_each_sg((sgt)->sgl, sg, (sgt)->nents, iter) 111 112 #define for_each_sgtable_dma_page(sgt, iter, pgoffset) \ 113 for_each_sg_dma_page((sgt)->sgl, iter, (sgt)->nents, pgoffset) 114 115 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 116 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 117 118 static inline void 119 sg_assign_page(struct scatterlist *sg, struct page *page) 120 { 121 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 122 123 sg->page_link = page_link | (unsigned long)page; 124 } 125 126 static inline void 127 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 128 unsigned int offset) 129 { 130 sg_assign_page(sg, page); 131 sg->offset = offset; 132 sg->length = len; 133 } 134 135 static inline struct page * 136 sg_page(struct scatterlist *sg) 137 { 138 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 139 } 140 141 static inline void 142 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 143 { 144 sg_set_page(sg, virt_to_page(buf), buflen, 145 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 146 } 147 148 static inline struct scatterlist * 149 sg_next(struct scatterlist *sg) 150 { 151 if (sg_is_last(sg)) 152 return (NULL); 153 sg++; 154 if (sg_is_chain(sg)) 155 sg = sg_chain_ptr(sg); 156 return (sg); 157 } 158 159 static inline vm_paddr_t 160 sg_phys(struct scatterlist *sg) 161 { 162 return (page_to_phys(sg_page(sg)) + sg->offset); 163 } 164 165 static inline void * 166 sg_virt(struct scatterlist *sg) 167 { 168 169 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 170 } 171 172 static inline void 173 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 174 struct scatterlist *sgl) 175 { 176 struct scatterlist *sg = &prv[prv_nents - 1]; 177 178 sg->offset = 0; 179 sg->length = 0; 180 sg->page_link = ((unsigned long)sgl | 181 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 182 } 183 184 static inline void 185 sg_mark_end(struct scatterlist *sg) 186 { 187 sg->page_link |= SG_PAGE_LINK_LAST; 188 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 189 } 190 191 static inline void 192 sg_init_marker(struct scatterlist *sg, uint32_t num_sgs) 193 { 194 sg_mark_end(&sg[num_sgs - 1]); 195 } 196 197 static inline void 198 sg_init_table(struct scatterlist *sg, unsigned int nents) 199 { 200 bzero(sg, sizeof(*sg) * nents); 201 sg_mark_end(&sg[nents - 1]); 202 } 203 204 static inline void 205 sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen) 206 { 207 sg_init_table(sg, 1); 208 sg_set_buf(sg, buf, buflen); 209 } 210 211 static struct scatterlist * 212 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 213 { 214 if (nents == SG_MAX_SINGLE_ALLOC) { 215 return ((void *)__get_free_page(gfp_mask)); 216 } else 217 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 218 } 219 220 static inline void 221 sg_kfree(struct scatterlist *sg, unsigned int nents) 222 { 223 if (nents == SG_MAX_SINGLE_ALLOC) { 224 free_page((unsigned long)sg); 225 } else 226 kfree(sg); 227 } 228 229 static inline void 230 __sg_free_table(struct sg_table *table, unsigned int max_ents, 231 bool skip_first_chunk, sg_free_fn * free_fn) 232 { 233 struct scatterlist *sgl, *next; 234 235 if (unlikely(!table->sgl)) 236 return; 237 238 sgl = table->sgl; 239 while (table->orig_nents) { 240 unsigned int alloc_size = table->orig_nents; 241 unsigned int sg_size; 242 243 if (alloc_size > max_ents) { 244 next = sg_chain_ptr(&sgl[max_ents - 1]); 245 alloc_size = max_ents; 246 sg_size = alloc_size - 1; 247 } else { 248 sg_size = alloc_size; 249 next = NULL; 250 } 251 252 table->orig_nents -= sg_size; 253 if (skip_first_chunk) 254 skip_first_chunk = 0; 255 else 256 free_fn(sgl, alloc_size); 257 sgl = next; 258 } 259 260 table->sgl = NULL; 261 } 262 263 static inline void 264 sg_free_table(struct sg_table *table) 265 { 266 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 267 } 268 269 static inline int 270 __sg_alloc_table(struct sg_table *table, unsigned int nents, 271 unsigned int max_ents, struct scatterlist *first_chunk, 272 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 273 { 274 struct scatterlist *sg, *prv; 275 unsigned int left; 276 277 memset(table, 0, sizeof(*table)); 278 279 if (nents == 0) 280 return (-EINVAL); 281 left = nents; 282 prv = NULL; 283 do { 284 unsigned int sg_size; 285 unsigned int alloc_size = left; 286 287 if (alloc_size > max_ents) { 288 alloc_size = max_ents; 289 sg_size = alloc_size - 1; 290 } else 291 sg_size = alloc_size; 292 293 left -= sg_size; 294 295 if (first_chunk) { 296 sg = first_chunk; 297 first_chunk = NULL; 298 } else { 299 sg = alloc_fn(alloc_size, gfp_mask); 300 } 301 if (unlikely(!sg)) { 302 if (prv) 303 table->nents = ++table->orig_nents; 304 305 return (-ENOMEM); 306 } 307 sg_init_table(sg, alloc_size); 308 table->nents = table->orig_nents += sg_size; 309 310 if (prv) 311 sg_chain(prv, max_ents, sg); 312 else 313 table->sgl = sg; 314 315 if (!left) 316 sg_mark_end(&sg[sg_size - 1]); 317 318 prv = sg; 319 } while (left); 320 321 return (0); 322 } 323 324 static inline int 325 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 326 { 327 int ret; 328 329 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 330 NULL, gfp_mask, sg_kmalloc); 331 if (unlikely(ret)) 332 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 333 334 return (ret); 335 } 336 337 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 338 static inline struct scatterlist * 339 __sg_alloc_table_from_pages(struct sg_table *sgt, 340 struct page **pages, unsigned int count, 341 unsigned long off, unsigned long size, 342 unsigned int max_segment, 343 struct scatterlist *prv, unsigned int left_pages, 344 gfp_t gfp_mask) 345 #else 346 static inline int 347 __sg_alloc_table_from_pages(struct sg_table *sgt, 348 struct page **pages, unsigned int count, 349 unsigned long off, unsigned long size, 350 unsigned int max_segment, gfp_t gfp_mask) 351 #endif 352 { 353 unsigned int i, segs, cur, len; 354 int rc; 355 struct scatterlist *s, *sg_iter; 356 357 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 358 if (prv != NULL) { 359 panic( 360 "Support for prv != NULL not implemented in " 361 "__sg_alloc_table_from_pages()"); 362 } 363 #endif 364 365 if (__predict_false(!max_segment || offset_in_page(max_segment))) 366 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 367 return (ERR_PTR(-EINVAL)); 368 #else 369 return (-EINVAL); 370 #endif 371 372 len = 0; 373 for (segs = i = 1; i < count; ++i) { 374 len += PAGE_SIZE; 375 if (len >= max_segment || 376 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 377 ++segs; 378 len = 0; 379 } 380 } 381 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 382 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 383 return (ERR_PTR(rc)); 384 #else 385 return (rc); 386 #endif 387 388 cur = 0; 389 for_each_sg(sgt->sgl, sg_iter, sgt->orig_nents, i) { 390 unsigned long seg_size; 391 unsigned int j; 392 393 /* 394 * We need to make sure that when we exit this loop "s" has the 395 * last sg in the chain so we can call sg_mark_end() on it. 396 * Only set this inside the loop since sg_iter will be iterated 397 * until it is NULL. 398 */ 399 s = sg_iter; 400 401 len = 0; 402 for (j = cur + 1; j < count; ++j) { 403 len += PAGE_SIZE; 404 if (len >= max_segment || page_to_pfn(pages[j]) != 405 page_to_pfn(pages[j - 1]) + 1) 406 break; 407 } 408 409 seg_size = ((j - cur) << PAGE_SHIFT) - off; 410 sg_set_page(s, pages[cur], MIN(size, seg_size), off); 411 size -= seg_size; 412 off = 0; 413 cur = j; 414 } 415 KASSERT(s != NULL, ("s is NULL after loop in __sg_alloc_table_from_pages()")); 416 417 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 418 if (left_pages == 0) 419 sg_mark_end(s); 420 421 return (s); 422 #else 423 return (0); 424 #endif 425 } 426 427 static inline int 428 sg_alloc_table_from_pages(struct sg_table *sgt, 429 struct page **pages, unsigned int count, 430 unsigned long off, unsigned long size, 431 gfp_t gfp_mask) 432 { 433 434 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 435 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, 436 SCATTERLIST_MAX_SEGMENT, NULL, 0, gfp_mask))); 437 #else 438 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 439 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 440 #endif 441 } 442 443 static inline int 444 sg_alloc_table_from_pages_segment(struct sg_table *sgt, 445 struct page **pages, unsigned int count, unsigned int off, 446 unsigned long size, unsigned int max_segment, gfp_t gfp_mask) 447 { 448 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 449 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, 450 max_segment, NULL, 0, gfp_mask))); 451 #else 452 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 453 max_segment, gfp_mask)); 454 #endif 455 } 456 457 static inline int 458 sg_nents(struct scatterlist *sg) 459 { 460 int nents; 461 462 for (nents = 0; sg; sg = sg_next(sg)) 463 nents++; 464 return (nents); 465 } 466 467 static inline void 468 __sg_page_iter_start(struct sg_page_iter *piter, 469 struct scatterlist *sglist, unsigned int nents, 470 unsigned long pgoffset) 471 { 472 piter->internal.pg_advance = 0; 473 piter->internal.nents = nents; 474 475 piter->sg = sglist; 476 piter->sg_pgoffset = pgoffset; 477 } 478 479 static inline void 480 _sg_iter_next(struct sg_page_iter *iter) 481 { 482 struct scatterlist *sg; 483 unsigned int pgcount; 484 485 sg = iter->sg; 486 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 487 488 ++iter->sg_pgoffset; 489 while (iter->sg_pgoffset >= pgcount) { 490 iter->sg_pgoffset -= pgcount; 491 sg = sg_next(sg); 492 --iter->maxents; 493 if (sg == NULL || iter->maxents == 0) 494 break; 495 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 496 } 497 iter->sg = sg; 498 } 499 500 static inline int 501 sg_page_count(struct scatterlist *sg) 502 { 503 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 504 } 505 #define sg_dma_page_count(sg) \ 506 sg_page_count(sg) 507 508 static inline bool 509 __sg_page_iter_next(struct sg_page_iter *piter) 510 { 511 unsigned int pgcount; 512 513 if (piter->internal.nents == 0) 514 return (0); 515 if (piter->sg == NULL) 516 return (0); 517 518 piter->sg_pgoffset += piter->internal.pg_advance; 519 piter->internal.pg_advance = 1; 520 521 while (1) { 522 pgcount = sg_page_count(piter->sg); 523 if (likely(piter->sg_pgoffset < pgcount)) 524 break; 525 piter->sg_pgoffset -= pgcount; 526 piter->sg = sg_next(piter->sg); 527 if (--piter->internal.nents == 0) 528 return (0); 529 if (piter->sg == NULL) 530 return (0); 531 } 532 return (1); 533 } 534 #define __sg_page_iter_dma_next(itr) \ 535 __sg_page_iter_next(&(itr)->base) 536 537 static inline void 538 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 539 unsigned int nents, unsigned long pgoffset) 540 { 541 if (nents) { 542 iter->sg = sgl; 543 iter->sg_pgoffset = pgoffset - 1; 544 iter->maxents = nents; 545 _sg_iter_next(iter); 546 } else { 547 iter->sg = NULL; 548 iter->sg_pgoffset = 0; 549 iter->maxents = 0; 550 } 551 } 552 553 /* 554 * sg_page_iter_dma_address() is implemented as a macro because it 555 * needs to accept two different and identical structure types. This 556 * allows both old and new code to co-exist. The compile time assert 557 * adds some safety, that the structure sizes match. 558 */ 559 #define sg_page_iter_dma_address(spi) ({ \ 560 struct sg_page_iter *__spi = (void *)(spi); \ 561 dma_addr_t __dma_address; \ 562 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 563 __dma_address = __spi->sg->dma_address + \ 564 (__spi->sg_pgoffset << PAGE_SHIFT); \ 565 __dma_address; \ 566 }) 567 568 static inline struct page * 569 sg_page_iter_page(struct sg_page_iter *piter) 570 { 571 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 572 } 573 574 static __inline size_t 575 sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 576 const void *buf, size_t buflen, off_t skip) 577 { 578 struct sg_page_iter piter; 579 struct page *page; 580 struct sf_buf *sf; 581 size_t len, copied; 582 char *p, *b; 583 584 if (buflen == 0) 585 return (0); 586 587 b = __DECONST(char *, buf); 588 copied = 0; 589 sched_pin(); 590 for_each_sg_page(sgl, &piter, nents, 0) { 591 592 /* Skip to the start. */ 593 if (piter.sg->length <= skip) { 594 skip -= piter.sg->length; 595 continue; 596 } 597 598 /* See how much to copy. */ 599 KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)), 600 ("%s: sg len %u - skip %ju || buflen %zu is 0\n", 601 __func__, piter.sg->length, (uintmax_t)skip, buflen)); 602 len = min(piter.sg->length - skip, buflen); 603 604 page = sg_page_iter_page(&piter); 605 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 606 if (sf == NULL) 607 break; 608 p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; 609 memcpy(p, b, len); 610 sf_buf_free(sf); 611 612 /* We copied so nothing more to skip. */ 613 skip = 0; 614 copied += len; 615 /* Either we exactly filled the page, or we are done. */ 616 buflen -= len; 617 if (buflen == 0) 618 break; 619 b += len; 620 } 621 sched_unpin(); 622 623 return (copied); 624 } 625 626 static inline size_t 627 sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, 628 const void *buf, size_t buflen) 629 { 630 return (sg_pcopy_from_buffer(sgl, nents, buf, buflen, 0)); 631 } 632 633 static inline size_t 634 sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, 635 void *buf, size_t buflen, off_t offset) 636 { 637 struct sg_page_iter iter; 638 struct scatterlist *sg; 639 struct page *page; 640 struct sf_buf *sf; 641 char *vaddr; 642 size_t total = 0; 643 size_t len; 644 645 if (!PMAP_HAS_DMAP) 646 sched_pin(); 647 for_each_sg_page(sgl, &iter, nents, 0) { 648 sg = iter.sg; 649 650 if (offset >= sg->length) { 651 offset -= sg->length; 652 continue; 653 } 654 len = ulmin(buflen, sg->length - offset); 655 if (len == 0) 656 break; 657 658 page = sg_page_iter_page(&iter); 659 if (!PMAP_HAS_DMAP) { 660 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 661 if (sf == NULL) 662 break; 663 vaddr = sf_buf_kva(sf); 664 } else 665 vaddr = PHYS_TO_DMAP(page_to_phys(page)); 666 memcpy(buf, vaddr + sg->offset + offset, len); 667 if (!PMAP_HAS_DMAP) 668 sf_buf_free(sf); 669 670 /* start at beginning of next page */ 671 offset = 0; 672 673 /* advance buffer */ 674 buf = (char *)buf + len; 675 buflen -= len; 676 total += len; 677 } 678 if (!PMAP_HAS_DMAP) 679 sched_unpin(); 680 return (total); 681 } 682 683 static inline void 684 sg_set_folio(struct scatterlist *sg, struct folio *folio, size_t len, 685 size_t offset) 686 { 687 sg_set_page(sg, &folio->page, len, offset); 688 } 689 690 #endif /* _LINUXKPI_LINUX_SCATTERLIST_H_ */ 691