1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * Copyright (c) 2016 Matthew Macy 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 #ifndef _LINUXKPI_LINUX_SCATTERLIST_H_ 34 #define _LINUXKPI_LINUX_SCATTERLIST_H_ 35 36 #include <sys/types.h> 37 #include <sys/sf_buf.h> 38 39 #include <linux/page.h> 40 #include <linux/slab.h> 41 #include <linux/mm.h> 42 43 struct bus_dmamap; 44 struct scatterlist { 45 unsigned long page_link; 46 #define SG_PAGE_LINK_CHAIN 0x1UL 47 #define SG_PAGE_LINK_LAST 0x2UL 48 #define SG_PAGE_LINK_MASK 0x3UL 49 unsigned int offset; 50 unsigned int length; 51 dma_addr_t dma_address; 52 struct bus_dmamap *dma_map; /* FreeBSD specific */ 53 }; 54 55 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 56 57 struct sg_table { 58 struct scatterlist *sgl; 59 unsigned int nents; 60 unsigned int orig_nents; 61 }; 62 63 struct sg_page_iter { 64 struct scatterlist *sg; 65 unsigned int sg_pgoffset; 66 unsigned int maxents; 67 struct { 68 unsigned int nents; 69 int pg_advance; 70 } internal; 71 }; 72 73 struct sg_dma_page_iter { 74 struct sg_page_iter base; 75 }; 76 77 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 78 79 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 80 81 #define SG_MAGIC 0x87654321UL 82 #define SG_CHAIN SG_PAGE_LINK_CHAIN 83 #define SG_END SG_PAGE_LINK_LAST 84 85 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 86 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 87 #define sg_chain_ptr(sg) \ 88 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 89 90 #define sg_dma_address(sg) (sg)->dma_address 91 #define sg_dma_len(sg) (sg)->length 92 93 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 94 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 95 (iter)->sg; _sg_iter_next(iter)) 96 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 97 for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) 98 99 #define for_each_sg(sglist, sg, sgmax, iter) \ 100 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 101 102 #define for_each_sgtable_dma_sg(sgt, sg, iter) \ 103 for_each_sg((sgt)->sgl, sg, (sgt)->nents, iter) 104 105 #define for_each_sgtable_dma_page(sgt, iter, pgoffset) \ 106 for_each_sg_dma_page((sgt)->sgl, iter, (sgt)->nents, pgoffset) 107 108 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 109 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 110 111 static inline void 112 sg_assign_page(struct scatterlist *sg, struct page *page) 113 { 114 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 115 116 sg->page_link = page_link | (unsigned long)page; 117 } 118 119 static inline void 120 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 121 unsigned int offset) 122 { 123 sg_assign_page(sg, page); 124 sg->offset = offset; 125 sg->length = len; 126 } 127 128 static inline struct page * 129 sg_page(struct scatterlist *sg) 130 { 131 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 132 } 133 134 static inline void 135 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 136 { 137 sg_set_page(sg, virt_to_page(buf), buflen, 138 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 139 } 140 141 static inline struct scatterlist * 142 sg_next(struct scatterlist *sg) 143 { 144 if (sg_is_last(sg)) 145 return (NULL); 146 sg++; 147 if (sg_is_chain(sg)) 148 sg = sg_chain_ptr(sg); 149 return (sg); 150 } 151 152 static inline vm_paddr_t 153 sg_phys(struct scatterlist *sg) 154 { 155 return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 156 } 157 158 static inline void * 159 sg_virt(struct scatterlist *sg) 160 { 161 162 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 163 } 164 165 static inline void 166 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 167 struct scatterlist *sgl) 168 { 169 struct scatterlist *sg = &prv[prv_nents - 1]; 170 171 sg->offset = 0; 172 sg->length = 0; 173 sg->page_link = ((unsigned long)sgl | 174 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 175 } 176 177 static inline void 178 sg_mark_end(struct scatterlist *sg) 179 { 180 sg->page_link |= SG_PAGE_LINK_LAST; 181 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 182 } 183 184 static inline void 185 sg_init_table(struct scatterlist *sg, unsigned int nents) 186 { 187 bzero(sg, sizeof(*sg) * nents); 188 sg_mark_end(&sg[nents - 1]); 189 } 190 191 static inline void 192 sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen) 193 { 194 sg_init_table(sg, 1); 195 sg_set_buf(sg, buf, buflen); 196 } 197 198 static struct scatterlist * 199 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 200 { 201 if (nents == SG_MAX_SINGLE_ALLOC) { 202 return ((void *)__get_free_page(gfp_mask)); 203 } else 204 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 205 } 206 207 static inline void 208 sg_kfree(struct scatterlist *sg, unsigned int nents) 209 { 210 if (nents == SG_MAX_SINGLE_ALLOC) { 211 free_page((unsigned long)sg); 212 } else 213 kfree(sg); 214 } 215 216 static inline void 217 __sg_free_table(struct sg_table *table, unsigned int max_ents, 218 bool skip_first_chunk, sg_free_fn * free_fn) 219 { 220 struct scatterlist *sgl, *next; 221 222 if (unlikely(!table->sgl)) 223 return; 224 225 sgl = table->sgl; 226 while (table->orig_nents) { 227 unsigned int alloc_size = table->orig_nents; 228 unsigned int sg_size; 229 230 if (alloc_size > max_ents) { 231 next = sg_chain_ptr(&sgl[max_ents - 1]); 232 alloc_size = max_ents; 233 sg_size = alloc_size - 1; 234 } else { 235 sg_size = alloc_size; 236 next = NULL; 237 } 238 239 table->orig_nents -= sg_size; 240 if (skip_first_chunk) 241 skip_first_chunk = 0; 242 else 243 free_fn(sgl, alloc_size); 244 sgl = next; 245 } 246 247 table->sgl = NULL; 248 } 249 250 static inline void 251 sg_free_table(struct sg_table *table) 252 { 253 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 254 } 255 256 static inline int 257 __sg_alloc_table(struct sg_table *table, unsigned int nents, 258 unsigned int max_ents, struct scatterlist *first_chunk, 259 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 260 { 261 struct scatterlist *sg, *prv; 262 unsigned int left; 263 264 memset(table, 0, sizeof(*table)); 265 266 if (nents == 0) 267 return (-EINVAL); 268 left = nents; 269 prv = NULL; 270 do { 271 unsigned int sg_size; 272 unsigned int alloc_size = left; 273 274 if (alloc_size > max_ents) { 275 alloc_size = max_ents; 276 sg_size = alloc_size - 1; 277 } else 278 sg_size = alloc_size; 279 280 left -= sg_size; 281 282 if (first_chunk) { 283 sg = first_chunk; 284 first_chunk = NULL; 285 } else { 286 sg = alloc_fn(alloc_size, gfp_mask); 287 } 288 if (unlikely(!sg)) { 289 if (prv) 290 table->nents = ++table->orig_nents; 291 292 return (-ENOMEM); 293 } 294 sg_init_table(sg, alloc_size); 295 table->nents = table->orig_nents += sg_size; 296 297 if (prv) 298 sg_chain(prv, max_ents, sg); 299 else 300 table->sgl = sg; 301 302 if (!left) 303 sg_mark_end(&sg[sg_size - 1]); 304 305 prv = sg; 306 } while (left); 307 308 return (0); 309 } 310 311 static inline int 312 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 313 { 314 int ret; 315 316 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 317 NULL, gfp_mask, sg_kmalloc); 318 if (unlikely(ret)) 319 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 320 321 return (ret); 322 } 323 324 static inline int 325 __sg_alloc_table_from_pages(struct sg_table *sgt, 326 struct page **pages, unsigned int count, 327 unsigned long off, unsigned long size, 328 unsigned int max_segment, gfp_t gfp_mask) 329 { 330 unsigned int i, segs, cur, len; 331 int rc; 332 struct scatterlist *s; 333 334 if (__predict_false(!max_segment || offset_in_page(max_segment))) 335 return (-EINVAL); 336 337 len = 0; 338 for (segs = i = 1; i < count; ++i) { 339 len += PAGE_SIZE; 340 if (len >= max_segment || 341 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 342 ++segs; 343 len = 0; 344 } 345 } 346 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 347 return (rc); 348 349 cur = 0; 350 for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 351 unsigned long seg_size; 352 unsigned int j; 353 354 len = 0; 355 for (j = cur + 1; j < count; ++j) { 356 len += PAGE_SIZE; 357 if (len >= max_segment || page_to_pfn(pages[j]) != 358 page_to_pfn(pages[j - 1]) + 1) 359 break; 360 } 361 362 seg_size = ((j - cur) << PAGE_SHIFT) - off; 363 sg_set_page(s, pages[cur], MIN(size, seg_size), off); 364 size -= seg_size; 365 off = 0; 366 cur = j; 367 } 368 return (0); 369 } 370 371 static inline int 372 sg_alloc_table_from_pages(struct sg_table *sgt, 373 struct page **pages, unsigned int count, 374 unsigned long off, unsigned long size, 375 gfp_t gfp_mask) 376 { 377 378 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 379 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 380 } 381 382 static inline int 383 sg_nents(struct scatterlist *sg) 384 { 385 int nents; 386 387 for (nents = 0; sg; sg = sg_next(sg)) 388 nents++; 389 return (nents); 390 } 391 392 static inline void 393 __sg_page_iter_start(struct sg_page_iter *piter, 394 struct scatterlist *sglist, unsigned int nents, 395 unsigned long pgoffset) 396 { 397 piter->internal.pg_advance = 0; 398 piter->internal.nents = nents; 399 400 piter->sg = sglist; 401 piter->sg_pgoffset = pgoffset; 402 } 403 404 static inline void 405 _sg_iter_next(struct sg_page_iter *iter) 406 { 407 struct scatterlist *sg; 408 unsigned int pgcount; 409 410 sg = iter->sg; 411 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 412 413 ++iter->sg_pgoffset; 414 while (iter->sg_pgoffset >= pgcount) { 415 iter->sg_pgoffset -= pgcount; 416 sg = sg_next(sg); 417 --iter->maxents; 418 if (sg == NULL || iter->maxents == 0) 419 break; 420 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 421 } 422 iter->sg = sg; 423 } 424 425 static inline int 426 sg_page_count(struct scatterlist *sg) 427 { 428 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 429 } 430 #define sg_dma_page_count(sg) \ 431 sg_page_count(sg) 432 433 static inline bool 434 __sg_page_iter_next(struct sg_page_iter *piter) 435 { 436 unsigned int pgcount; 437 438 if (piter->internal.nents == 0) 439 return (0); 440 if (piter->sg == NULL) 441 return (0); 442 443 piter->sg_pgoffset += piter->internal.pg_advance; 444 piter->internal.pg_advance = 1; 445 446 while (1) { 447 pgcount = sg_page_count(piter->sg); 448 if (likely(piter->sg_pgoffset < pgcount)) 449 break; 450 piter->sg_pgoffset -= pgcount; 451 piter->sg = sg_next(piter->sg); 452 if (--piter->internal.nents == 0) 453 return (0); 454 if (piter->sg == NULL) 455 return (0); 456 } 457 return (1); 458 } 459 #define __sg_page_iter_dma_next(itr) \ 460 __sg_page_iter_next(&(itr)->base) 461 462 static inline void 463 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 464 unsigned int nents, unsigned long pgoffset) 465 { 466 if (nents) { 467 iter->sg = sgl; 468 iter->sg_pgoffset = pgoffset - 1; 469 iter->maxents = nents; 470 _sg_iter_next(iter); 471 } else { 472 iter->sg = NULL; 473 iter->sg_pgoffset = 0; 474 iter->maxents = 0; 475 } 476 } 477 478 /* 479 * sg_page_iter_dma_address() is implemented as a macro because it 480 * needs to accept two different and identical structure types. This 481 * allows both old and new code to co-exist. The compile time assert 482 * adds some safety, that the structure sizes match. 483 */ 484 #define sg_page_iter_dma_address(spi) ({ \ 485 struct sg_page_iter *__spi = (void *)(spi); \ 486 dma_addr_t __dma_address; \ 487 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 488 __dma_address = __spi->sg->dma_address + \ 489 (__spi->sg_pgoffset << PAGE_SHIFT); \ 490 __dma_address; \ 491 }) 492 493 static inline struct page * 494 sg_page_iter_page(struct sg_page_iter *piter) 495 { 496 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 497 } 498 499 static __inline size_t 500 sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 501 const void *buf, size_t buflen, off_t skip) 502 { 503 struct sg_page_iter piter; 504 struct page *page; 505 struct sf_buf *sf; 506 size_t len, copied; 507 char *p, *b; 508 509 if (buflen == 0) 510 return (0); 511 512 b = __DECONST(char *, buf); 513 copied = 0; 514 sched_pin(); 515 for_each_sg_page(sgl, &piter, nents, 0) { 516 517 /* Skip to the start. */ 518 if (piter.sg->length <= skip) { 519 skip -= piter.sg->length; 520 continue; 521 } 522 523 /* See how much to copy. */ 524 KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)), 525 ("%s: sg len %u - skip %ju || buflen %zu is 0\n", 526 __func__, piter.sg->length, (uintmax_t)skip, buflen)); 527 len = min(piter.sg->length - skip, buflen); 528 529 page = sg_page_iter_page(&piter); 530 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 531 if (sf == NULL) 532 break; 533 p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; 534 memcpy(p, b, len); 535 sf_buf_free(sf); 536 537 /* We copied so nothing more to skip. */ 538 skip = 0; 539 copied += len; 540 /* Either we exactly filled the page, or we are done. */ 541 buflen -= len; 542 if (buflen == 0) 543 break; 544 b += len; 545 } 546 sched_unpin(); 547 548 return (copied); 549 } 550 551 static inline size_t 552 sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, 553 const void *buf, size_t buflen) 554 { 555 return (sg_pcopy_from_buffer(sgl, nents, buf, buflen, 0)); 556 } 557 558 static inline size_t 559 sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, 560 void *buf, size_t buflen, off_t offset) 561 { 562 struct sg_page_iter iter; 563 struct scatterlist *sg; 564 struct page *page; 565 struct sf_buf *sf; 566 char *vaddr; 567 size_t total = 0; 568 size_t len; 569 570 if (!PMAP_HAS_DMAP) 571 sched_pin(); 572 for_each_sg_page(sgl, &iter, nents, 0) { 573 sg = iter.sg; 574 575 if (offset >= sg->length) { 576 offset -= sg->length; 577 continue; 578 } 579 len = ulmin(buflen, sg->length - offset); 580 if (len == 0) 581 break; 582 583 page = sg_page_iter_page(&iter); 584 if (!PMAP_HAS_DMAP) { 585 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 586 if (sf == NULL) 587 break; 588 vaddr = (char *)sf_buf_kva(sf); 589 } else 590 vaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page)); 591 memcpy(buf, vaddr + sg->offset + offset, len); 592 if (!PMAP_HAS_DMAP) 593 sf_buf_free(sf); 594 595 /* start at beginning of next page */ 596 offset = 0; 597 598 /* advance buffer */ 599 buf = (char *)buf + len; 600 buflen -= len; 601 total += len; 602 } 603 if (!PMAP_HAS_DMAP) 604 sched_unpin(); 605 return (total); 606 } 607 608 #endif /* _LINUXKPI_LINUX_SCATTERLIST_H_ */ 609