1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * Copyright (c) 2016 Matthew Macy 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #ifndef _LINUXKPI_LINUX_SCATTERLIST_H_ 32 #define _LINUXKPI_LINUX_SCATTERLIST_H_ 33 34 #include <sys/types.h> 35 #include <sys/proc.h> 36 #include <sys/sched.h> 37 #include <sys/sf_buf.h> 38 39 #include <linux/err.h> 40 #include <linux/page.h> 41 #include <linux/slab.h> 42 #include <linux/mm.h> 43 44 struct bus_dmamap; 45 struct scatterlist { 46 unsigned long page_link; 47 #define SG_PAGE_LINK_CHAIN 0x1UL 48 #define SG_PAGE_LINK_LAST 0x2UL 49 #define SG_PAGE_LINK_MASK 0x3UL 50 unsigned int offset; 51 unsigned int length; 52 dma_addr_t dma_address; 53 struct bus_dmamap *dma_map; /* FreeBSD specific */ 54 }; 55 56 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 57 58 struct sg_table { 59 struct scatterlist *sgl; 60 unsigned int nents; 61 unsigned int orig_nents; 62 }; 63 64 struct sg_page_iter { 65 struct scatterlist *sg; 66 unsigned int sg_pgoffset; 67 unsigned int maxents; 68 struct { 69 unsigned int nents; 70 int pg_advance; 71 } internal; 72 }; 73 74 struct sg_dma_page_iter { 75 struct sg_page_iter base; 76 }; 77 78 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 79 80 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 81 82 #define SG_MAGIC 0x87654321UL 83 #define SG_CHAIN SG_PAGE_LINK_CHAIN 84 #define SG_END SG_PAGE_LINK_LAST 85 86 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 87 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 88 #define sg_chain_ptr(sg) \ 89 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 90 91 #define sg_dma_address(sg) (sg)->dma_address 92 #define sg_dma_len(sg) (sg)->length 93 94 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 95 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 96 (iter)->sg; _sg_iter_next(iter)) 97 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 98 for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) 99 100 #define for_each_sg(sglist, sg, sgmax, iter) \ 101 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 102 103 #define for_each_sgtable_sg(sgt, sg, i) \ 104 for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i) 105 106 #define for_each_sgtable_page(sgt, iter, pgoffset) \ 107 for_each_sg_page((sgt)->sgl, iter, (sgt)->orig_nents, pgoffset) 108 109 #define for_each_sgtable_dma_sg(sgt, sg, iter) \ 110 for_each_sg((sgt)->sgl, sg, (sgt)->nents, iter) 111 112 #define for_each_sgtable_dma_page(sgt, iter, pgoffset) \ 113 for_each_sg_dma_page((sgt)->sgl, iter, (sgt)->nents, pgoffset) 114 115 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 116 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 117 118 static inline void 119 sg_assign_page(struct scatterlist *sg, struct page *page) 120 { 121 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 122 123 sg->page_link = page_link | (unsigned long)page; 124 } 125 126 static inline void 127 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 128 unsigned int offset) 129 { 130 sg_assign_page(sg, page); 131 sg->offset = offset; 132 sg->length = len; 133 } 134 135 static inline struct page * 136 sg_page(struct scatterlist *sg) 137 { 138 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 139 } 140 141 static inline void 142 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 143 { 144 sg_set_page(sg, virt_to_page(buf), buflen, 145 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 146 } 147 148 static inline struct scatterlist * 149 sg_next(struct scatterlist *sg) 150 { 151 if (sg_is_last(sg)) 152 return (NULL); 153 sg++; 154 if (sg_is_chain(sg)) 155 sg = sg_chain_ptr(sg); 156 return (sg); 157 } 158 159 static inline vm_paddr_t 160 sg_phys(struct scatterlist *sg) 161 { 162 return (page_to_phys(sg_page(sg)) + sg->offset); 163 } 164 165 static inline void * 166 sg_virt(struct scatterlist *sg) 167 { 168 169 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 170 } 171 172 static inline void 173 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 174 struct scatterlist *sgl) 175 { 176 struct scatterlist *sg = &prv[prv_nents - 1]; 177 178 sg->offset = 0; 179 sg->length = 0; 180 sg->page_link = ((unsigned long)sgl | 181 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 182 } 183 184 static inline void 185 sg_mark_end(struct scatterlist *sg) 186 { 187 sg->page_link |= SG_PAGE_LINK_LAST; 188 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 189 } 190 191 static inline void 192 sg_init_table(struct scatterlist *sg, unsigned int nents) 193 { 194 bzero(sg, sizeof(*sg) * nents); 195 sg_mark_end(&sg[nents - 1]); 196 } 197 198 static inline void 199 sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen) 200 { 201 sg_init_table(sg, 1); 202 sg_set_buf(sg, buf, buflen); 203 } 204 205 static struct scatterlist * 206 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 207 { 208 if (nents == SG_MAX_SINGLE_ALLOC) { 209 return ((void *)__get_free_page(gfp_mask)); 210 } else 211 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 212 } 213 214 static inline void 215 sg_kfree(struct scatterlist *sg, unsigned int nents) 216 { 217 if (nents == SG_MAX_SINGLE_ALLOC) { 218 free_page((unsigned long)sg); 219 } else 220 kfree(sg); 221 } 222 223 static inline void 224 __sg_free_table(struct sg_table *table, unsigned int max_ents, 225 bool skip_first_chunk, sg_free_fn * free_fn) 226 { 227 struct scatterlist *sgl, *next; 228 229 if (unlikely(!table->sgl)) 230 return; 231 232 sgl = table->sgl; 233 while (table->orig_nents) { 234 unsigned int alloc_size = table->orig_nents; 235 unsigned int sg_size; 236 237 if (alloc_size > max_ents) { 238 next = sg_chain_ptr(&sgl[max_ents - 1]); 239 alloc_size = max_ents; 240 sg_size = alloc_size - 1; 241 } else { 242 sg_size = alloc_size; 243 next = NULL; 244 } 245 246 table->orig_nents -= sg_size; 247 if (skip_first_chunk) 248 skip_first_chunk = 0; 249 else 250 free_fn(sgl, alloc_size); 251 sgl = next; 252 } 253 254 table->sgl = NULL; 255 } 256 257 static inline void 258 sg_free_table(struct sg_table *table) 259 { 260 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 261 } 262 263 static inline int 264 __sg_alloc_table(struct sg_table *table, unsigned int nents, 265 unsigned int max_ents, struct scatterlist *first_chunk, 266 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 267 { 268 struct scatterlist *sg, *prv; 269 unsigned int left; 270 271 memset(table, 0, sizeof(*table)); 272 273 if (nents == 0) 274 return (-EINVAL); 275 left = nents; 276 prv = NULL; 277 do { 278 unsigned int sg_size; 279 unsigned int alloc_size = left; 280 281 if (alloc_size > max_ents) { 282 alloc_size = max_ents; 283 sg_size = alloc_size - 1; 284 } else 285 sg_size = alloc_size; 286 287 left -= sg_size; 288 289 if (first_chunk) { 290 sg = first_chunk; 291 first_chunk = NULL; 292 } else { 293 sg = alloc_fn(alloc_size, gfp_mask); 294 } 295 if (unlikely(!sg)) { 296 if (prv) 297 table->nents = ++table->orig_nents; 298 299 return (-ENOMEM); 300 } 301 sg_init_table(sg, alloc_size); 302 table->nents = table->orig_nents += sg_size; 303 304 if (prv) 305 sg_chain(prv, max_ents, sg); 306 else 307 table->sgl = sg; 308 309 if (!left) 310 sg_mark_end(&sg[sg_size - 1]); 311 312 prv = sg; 313 } while (left); 314 315 return (0); 316 } 317 318 static inline int 319 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 320 { 321 int ret; 322 323 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 324 NULL, gfp_mask, sg_kmalloc); 325 if (unlikely(ret)) 326 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 327 328 return (ret); 329 } 330 331 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 332 static inline struct scatterlist * 333 __sg_alloc_table_from_pages(struct sg_table *sgt, 334 struct page **pages, unsigned int count, 335 unsigned long off, unsigned long size, 336 unsigned int max_segment, 337 struct scatterlist *prv, unsigned int left_pages, 338 gfp_t gfp_mask) 339 #else 340 static inline int 341 __sg_alloc_table_from_pages(struct sg_table *sgt, 342 struct page **pages, unsigned int count, 343 unsigned long off, unsigned long size, 344 unsigned int max_segment, gfp_t gfp_mask) 345 #endif 346 { 347 unsigned int i, segs, cur, len; 348 int rc; 349 struct scatterlist *s, *sg_iter; 350 351 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 352 if (prv != NULL) { 353 panic( 354 "Support for prv != NULL not implemented in " 355 "__sg_alloc_table_from_pages()"); 356 } 357 #endif 358 359 if (__predict_false(!max_segment || offset_in_page(max_segment))) 360 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 361 return (ERR_PTR(-EINVAL)); 362 #else 363 return (-EINVAL); 364 #endif 365 366 len = 0; 367 for (segs = i = 1; i < count; ++i) { 368 len += PAGE_SIZE; 369 if (len >= max_segment || 370 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 371 ++segs; 372 len = 0; 373 } 374 } 375 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 376 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 377 return (ERR_PTR(rc)); 378 #else 379 return (rc); 380 #endif 381 382 cur = 0; 383 for_each_sg(sgt->sgl, sg_iter, sgt->orig_nents, i) { 384 unsigned long seg_size; 385 unsigned int j; 386 387 /* 388 * We need to make sure that when we exit this loop "s" has the 389 * last sg in the chain so we can call sg_mark_end() on it. 390 * Only set this inside the loop since sg_iter will be iterated 391 * until it is NULL. 392 */ 393 s = sg_iter; 394 395 len = 0; 396 for (j = cur + 1; j < count; ++j) { 397 len += PAGE_SIZE; 398 if (len >= max_segment || page_to_pfn(pages[j]) != 399 page_to_pfn(pages[j - 1]) + 1) 400 break; 401 } 402 403 seg_size = ((j - cur) << PAGE_SHIFT) - off; 404 sg_set_page(s, pages[cur], MIN(size, seg_size), off); 405 size -= seg_size; 406 off = 0; 407 cur = j; 408 } 409 KASSERT(s != NULL, ("s is NULL after loop in __sg_alloc_table_from_pages()")); 410 411 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 412 if (left_pages == 0) 413 sg_mark_end(s); 414 415 return (s); 416 #else 417 return (0); 418 #endif 419 } 420 421 static inline int 422 sg_alloc_table_from_pages(struct sg_table *sgt, 423 struct page **pages, unsigned int count, 424 unsigned long off, unsigned long size, 425 gfp_t gfp_mask) 426 { 427 428 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 429 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, 430 SCATTERLIST_MAX_SEGMENT, NULL, 0, gfp_mask))); 431 #else 432 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 433 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 434 #endif 435 } 436 437 static inline int 438 sg_alloc_table_from_pages_segment(struct sg_table *sgt, 439 struct page **pages, unsigned int count, unsigned int off, 440 unsigned long size, unsigned int max_segment, gfp_t gfp_mask) 441 { 442 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 443 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, 444 max_segment, NULL, 0, gfp_mask))); 445 #else 446 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 447 max_segment, gfp_mask)); 448 #endif 449 } 450 451 static inline int 452 sg_nents(struct scatterlist *sg) 453 { 454 int nents; 455 456 for (nents = 0; sg; sg = sg_next(sg)) 457 nents++; 458 return (nents); 459 } 460 461 static inline void 462 __sg_page_iter_start(struct sg_page_iter *piter, 463 struct scatterlist *sglist, unsigned int nents, 464 unsigned long pgoffset) 465 { 466 piter->internal.pg_advance = 0; 467 piter->internal.nents = nents; 468 469 piter->sg = sglist; 470 piter->sg_pgoffset = pgoffset; 471 } 472 473 static inline void 474 _sg_iter_next(struct sg_page_iter *iter) 475 { 476 struct scatterlist *sg; 477 unsigned int pgcount; 478 479 sg = iter->sg; 480 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 481 482 ++iter->sg_pgoffset; 483 while (iter->sg_pgoffset >= pgcount) { 484 iter->sg_pgoffset -= pgcount; 485 sg = sg_next(sg); 486 --iter->maxents; 487 if (sg == NULL || iter->maxents == 0) 488 break; 489 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 490 } 491 iter->sg = sg; 492 } 493 494 static inline int 495 sg_page_count(struct scatterlist *sg) 496 { 497 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 498 } 499 #define sg_dma_page_count(sg) \ 500 sg_page_count(sg) 501 502 static inline bool 503 __sg_page_iter_next(struct sg_page_iter *piter) 504 { 505 unsigned int pgcount; 506 507 if (piter->internal.nents == 0) 508 return (0); 509 if (piter->sg == NULL) 510 return (0); 511 512 piter->sg_pgoffset += piter->internal.pg_advance; 513 piter->internal.pg_advance = 1; 514 515 while (1) { 516 pgcount = sg_page_count(piter->sg); 517 if (likely(piter->sg_pgoffset < pgcount)) 518 break; 519 piter->sg_pgoffset -= pgcount; 520 piter->sg = sg_next(piter->sg); 521 if (--piter->internal.nents == 0) 522 return (0); 523 if (piter->sg == NULL) 524 return (0); 525 } 526 return (1); 527 } 528 #define __sg_page_iter_dma_next(itr) \ 529 __sg_page_iter_next(&(itr)->base) 530 531 static inline void 532 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 533 unsigned int nents, unsigned long pgoffset) 534 { 535 if (nents) { 536 iter->sg = sgl; 537 iter->sg_pgoffset = pgoffset - 1; 538 iter->maxents = nents; 539 _sg_iter_next(iter); 540 } else { 541 iter->sg = NULL; 542 iter->sg_pgoffset = 0; 543 iter->maxents = 0; 544 } 545 } 546 547 /* 548 * sg_page_iter_dma_address() is implemented as a macro because it 549 * needs to accept two different and identical structure types. This 550 * allows both old and new code to co-exist. The compile time assert 551 * adds some safety, that the structure sizes match. 552 */ 553 #define sg_page_iter_dma_address(spi) ({ \ 554 struct sg_page_iter *__spi = (void *)(spi); \ 555 dma_addr_t __dma_address; \ 556 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 557 __dma_address = __spi->sg->dma_address + \ 558 (__spi->sg_pgoffset << PAGE_SHIFT); \ 559 __dma_address; \ 560 }) 561 562 static inline struct page * 563 sg_page_iter_page(struct sg_page_iter *piter) 564 { 565 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 566 } 567 568 static __inline size_t 569 sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 570 const void *buf, size_t buflen, off_t skip) 571 { 572 struct sg_page_iter piter; 573 struct page *page; 574 struct sf_buf *sf; 575 size_t len, copied; 576 char *p, *b; 577 578 if (buflen == 0) 579 return (0); 580 581 b = __DECONST(char *, buf); 582 copied = 0; 583 sched_pin(); 584 for_each_sg_page(sgl, &piter, nents, 0) { 585 586 /* Skip to the start. */ 587 if (piter.sg->length <= skip) { 588 skip -= piter.sg->length; 589 continue; 590 } 591 592 /* See how much to copy. */ 593 KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)), 594 ("%s: sg len %u - skip %ju || buflen %zu is 0\n", 595 __func__, piter.sg->length, (uintmax_t)skip, buflen)); 596 len = min(piter.sg->length - skip, buflen); 597 598 page = sg_page_iter_page(&piter); 599 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 600 if (sf == NULL) 601 break; 602 p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; 603 memcpy(p, b, len); 604 sf_buf_free(sf); 605 606 /* We copied so nothing more to skip. */ 607 skip = 0; 608 copied += len; 609 /* Either we exactly filled the page, or we are done. */ 610 buflen -= len; 611 if (buflen == 0) 612 break; 613 b += len; 614 } 615 sched_unpin(); 616 617 return (copied); 618 } 619 620 static inline size_t 621 sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, 622 const void *buf, size_t buflen) 623 { 624 return (sg_pcopy_from_buffer(sgl, nents, buf, buflen, 0)); 625 } 626 627 static inline size_t 628 sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, 629 void *buf, size_t buflen, off_t offset) 630 { 631 struct sg_page_iter iter; 632 struct scatterlist *sg; 633 struct page *page; 634 struct sf_buf *sf; 635 char *vaddr; 636 size_t total = 0; 637 size_t len; 638 639 if (!PMAP_HAS_DMAP) 640 sched_pin(); 641 for_each_sg_page(sgl, &iter, nents, 0) { 642 sg = iter.sg; 643 644 if (offset >= sg->length) { 645 offset -= sg->length; 646 continue; 647 } 648 len = ulmin(buflen, sg->length - offset); 649 if (len == 0) 650 break; 651 652 page = sg_page_iter_page(&iter); 653 if (!PMAP_HAS_DMAP) { 654 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 655 if (sf == NULL) 656 break; 657 vaddr = (char *)sf_buf_kva(sf); 658 } else 659 vaddr = (char *)PHYS_TO_DMAP(page_to_phys(page)); 660 memcpy(buf, vaddr + sg->offset + offset, len); 661 if (!PMAP_HAS_DMAP) 662 sf_buf_free(sf); 663 664 /* start at beginning of next page */ 665 offset = 0; 666 667 /* advance buffer */ 668 buf = (char *)buf + len; 669 buflen -= len; 670 total += len; 671 } 672 if (!PMAP_HAS_DMAP) 673 sched_unpin(); 674 return (total); 675 } 676 677 #endif /* _LINUXKPI_LINUX_SCATTERLIST_H_ */ 678