1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * Copyright (c) 2016 Matthew Macy 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 #ifndef _LINUX_SCATTERLIST_H_ 34 #define _LINUX_SCATTERLIST_H_ 35 36 #include <sys/types.h> 37 #include <sys/sf_buf.h> 38 39 #include <linux/page.h> 40 #include <linux/slab.h> 41 #include <linux/mm.h> 42 43 struct bus_dmamap; 44 struct scatterlist { 45 unsigned long page_link; 46 #define SG_PAGE_LINK_CHAIN 0x1UL 47 #define SG_PAGE_LINK_LAST 0x2UL 48 #define SG_PAGE_LINK_MASK 0x3UL 49 unsigned int offset; 50 unsigned int length; 51 dma_addr_t dma_address; 52 struct bus_dmamap *dma_map; /* FreeBSD specific */ 53 }; 54 55 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 56 57 struct sg_table { 58 struct scatterlist *sgl; 59 unsigned int nents; 60 unsigned int orig_nents; 61 }; 62 63 struct sg_page_iter { 64 struct scatterlist *sg; 65 unsigned int sg_pgoffset; 66 unsigned int maxents; 67 struct { 68 unsigned int nents; 69 int pg_advance; 70 } internal; 71 }; 72 73 struct sg_dma_page_iter { 74 struct sg_page_iter base; 75 }; 76 77 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 78 79 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 80 81 #define SG_MAGIC 0x87654321UL 82 #define SG_CHAIN SG_PAGE_LINK_CHAIN 83 #define SG_END SG_PAGE_LINK_LAST 84 85 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 86 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 87 #define sg_chain_ptr(sg) \ 88 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 89 90 #define sg_dma_address(sg) (sg)->dma_address 91 #define sg_dma_len(sg) (sg)->length 92 93 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 94 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 95 (iter)->sg; _sg_iter_next(iter)) 96 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 97 for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) 98 99 #define for_each_sg(sglist, sg, sgmax, iter) \ 100 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 101 102 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 103 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 104 105 static inline void 106 sg_assign_page(struct scatterlist *sg, struct page *page) 107 { 108 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 109 110 sg->page_link = page_link | (unsigned long)page; 111 } 112 113 static inline void 114 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 115 unsigned int offset) 116 { 117 sg_assign_page(sg, page); 118 sg->offset = offset; 119 sg->length = len; 120 } 121 122 static inline struct page * 123 sg_page(struct scatterlist *sg) 124 { 125 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 126 } 127 128 static inline void 129 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 130 { 131 sg_set_page(sg, virt_to_page(buf), buflen, 132 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 133 } 134 135 static inline struct scatterlist * 136 sg_next(struct scatterlist *sg) 137 { 138 if (sg_is_last(sg)) 139 return (NULL); 140 sg++; 141 if (sg_is_chain(sg)) 142 sg = sg_chain_ptr(sg); 143 return (sg); 144 } 145 146 static inline vm_paddr_t 147 sg_phys(struct scatterlist *sg) 148 { 149 return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 150 } 151 152 static inline void * 153 sg_virt(struct scatterlist *sg) 154 { 155 156 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 157 } 158 159 static inline void 160 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 161 struct scatterlist *sgl) 162 { 163 struct scatterlist *sg = &prv[prv_nents - 1]; 164 165 sg->offset = 0; 166 sg->length = 0; 167 sg->page_link = ((unsigned long)sgl | 168 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 169 } 170 171 static inline void 172 sg_mark_end(struct scatterlist *sg) 173 { 174 sg->page_link |= SG_PAGE_LINK_LAST; 175 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 176 } 177 178 static inline void 179 sg_init_table(struct scatterlist *sg, unsigned int nents) 180 { 181 bzero(sg, sizeof(*sg) * nents); 182 sg_mark_end(&sg[nents - 1]); 183 } 184 185 static struct scatterlist * 186 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 187 { 188 if (nents == SG_MAX_SINGLE_ALLOC) { 189 return ((void *)__get_free_page(gfp_mask)); 190 } else 191 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 192 } 193 194 static inline void 195 sg_kfree(struct scatterlist *sg, unsigned int nents) 196 { 197 if (nents == SG_MAX_SINGLE_ALLOC) { 198 free_page((unsigned long)sg); 199 } else 200 kfree(sg); 201 } 202 203 static inline void 204 __sg_free_table(struct sg_table *table, unsigned int max_ents, 205 bool skip_first_chunk, sg_free_fn * free_fn) 206 { 207 struct scatterlist *sgl, *next; 208 209 if (unlikely(!table->sgl)) 210 return; 211 212 sgl = table->sgl; 213 while (table->orig_nents) { 214 unsigned int alloc_size = table->orig_nents; 215 unsigned int sg_size; 216 217 if (alloc_size > max_ents) { 218 next = sg_chain_ptr(&sgl[max_ents - 1]); 219 alloc_size = max_ents; 220 sg_size = alloc_size - 1; 221 } else { 222 sg_size = alloc_size; 223 next = NULL; 224 } 225 226 table->orig_nents -= sg_size; 227 if (skip_first_chunk) 228 skip_first_chunk = 0; 229 else 230 free_fn(sgl, alloc_size); 231 sgl = next; 232 } 233 234 table->sgl = NULL; 235 } 236 237 static inline void 238 sg_free_table(struct sg_table *table) 239 { 240 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 241 } 242 243 static inline int 244 __sg_alloc_table(struct sg_table *table, unsigned int nents, 245 unsigned int max_ents, struct scatterlist *first_chunk, 246 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 247 { 248 struct scatterlist *sg, *prv; 249 unsigned int left; 250 251 memset(table, 0, sizeof(*table)); 252 253 if (nents == 0) 254 return (-EINVAL); 255 left = nents; 256 prv = NULL; 257 do { 258 unsigned int sg_size; 259 unsigned int alloc_size = left; 260 261 if (alloc_size > max_ents) { 262 alloc_size = max_ents; 263 sg_size = alloc_size - 1; 264 } else 265 sg_size = alloc_size; 266 267 left -= sg_size; 268 269 if (first_chunk) { 270 sg = first_chunk; 271 first_chunk = NULL; 272 } else { 273 sg = alloc_fn(alloc_size, gfp_mask); 274 } 275 if (unlikely(!sg)) { 276 if (prv) 277 table->nents = ++table->orig_nents; 278 279 return (-ENOMEM); 280 } 281 sg_init_table(sg, alloc_size); 282 table->nents = table->orig_nents += sg_size; 283 284 if (prv) 285 sg_chain(prv, max_ents, sg); 286 else 287 table->sgl = sg; 288 289 if (!left) 290 sg_mark_end(&sg[sg_size - 1]); 291 292 prv = sg; 293 } while (left); 294 295 return (0); 296 } 297 298 static inline int 299 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 300 { 301 int ret; 302 303 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 304 NULL, gfp_mask, sg_kmalloc); 305 if (unlikely(ret)) 306 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 307 308 return (ret); 309 } 310 311 static inline int 312 __sg_alloc_table_from_pages(struct sg_table *sgt, 313 struct page **pages, unsigned int count, 314 unsigned long off, unsigned long size, 315 unsigned int max_segment, gfp_t gfp_mask) 316 { 317 unsigned int i, segs, cur, len; 318 int rc; 319 struct scatterlist *s; 320 321 if (__predict_false(!max_segment || offset_in_page(max_segment))) 322 return (-EINVAL); 323 324 len = 0; 325 for (segs = i = 1; i < count; ++i) { 326 len += PAGE_SIZE; 327 if (len >= max_segment || 328 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 329 ++segs; 330 len = 0; 331 } 332 } 333 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 334 return (rc); 335 336 cur = 0; 337 for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 338 unsigned long seg_size; 339 unsigned int j; 340 341 len = 0; 342 for (j = cur + 1; j < count; ++j) { 343 len += PAGE_SIZE; 344 if (len >= max_segment || page_to_pfn(pages[j]) != 345 page_to_pfn(pages[j - 1]) + 1) 346 break; 347 } 348 349 seg_size = ((j - cur) << PAGE_SHIFT) - off; 350 sg_set_page(s, pages[cur], MIN(size, seg_size), off); 351 size -= seg_size; 352 off = 0; 353 cur = j; 354 } 355 return (0); 356 } 357 358 static inline int 359 sg_alloc_table_from_pages(struct sg_table *sgt, 360 struct page **pages, unsigned int count, 361 unsigned long off, unsigned long size, 362 gfp_t gfp_mask) 363 { 364 365 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 366 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 367 } 368 369 static inline int 370 sg_nents(struct scatterlist *sg) 371 { 372 int nents; 373 374 for (nents = 0; sg; sg = sg_next(sg)) 375 nents++; 376 return (nents); 377 } 378 379 static inline void 380 __sg_page_iter_start(struct sg_page_iter *piter, 381 struct scatterlist *sglist, unsigned int nents, 382 unsigned long pgoffset) 383 { 384 piter->internal.pg_advance = 0; 385 piter->internal.nents = nents; 386 387 piter->sg = sglist; 388 piter->sg_pgoffset = pgoffset; 389 } 390 391 static inline void 392 _sg_iter_next(struct sg_page_iter *iter) 393 { 394 struct scatterlist *sg; 395 unsigned int pgcount; 396 397 sg = iter->sg; 398 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 399 400 ++iter->sg_pgoffset; 401 while (iter->sg_pgoffset >= pgcount) { 402 iter->sg_pgoffset -= pgcount; 403 sg = sg_next(sg); 404 --iter->maxents; 405 if (sg == NULL || iter->maxents == 0) 406 break; 407 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 408 } 409 iter->sg = sg; 410 } 411 412 static inline int 413 sg_page_count(struct scatterlist *sg) 414 { 415 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 416 } 417 #define sg_dma_page_count(sg) \ 418 sg_page_count(sg) 419 420 static inline bool 421 __sg_page_iter_next(struct sg_page_iter *piter) 422 { 423 unsigned int pgcount; 424 425 if (piter->internal.nents == 0) 426 return (0); 427 if (piter->sg == NULL) 428 return (0); 429 430 piter->sg_pgoffset += piter->internal.pg_advance; 431 piter->internal.pg_advance = 1; 432 433 while (1) { 434 pgcount = sg_page_count(piter->sg); 435 if (likely(piter->sg_pgoffset < pgcount)) 436 break; 437 piter->sg_pgoffset -= pgcount; 438 piter->sg = sg_next(piter->sg); 439 if (--piter->internal.nents == 0) 440 return (0); 441 if (piter->sg == NULL) 442 return (0); 443 } 444 return (1); 445 } 446 #define __sg_page_iter_dma_next(itr) \ 447 __sg_page_iter_next(&(itr)->base) 448 449 static inline void 450 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 451 unsigned int nents, unsigned long pgoffset) 452 { 453 if (nents) { 454 iter->sg = sgl; 455 iter->sg_pgoffset = pgoffset - 1; 456 iter->maxents = nents; 457 _sg_iter_next(iter); 458 } else { 459 iter->sg = NULL; 460 iter->sg_pgoffset = 0; 461 iter->maxents = 0; 462 } 463 } 464 465 /* 466 * sg_page_iter_dma_address() is implemented as a macro because it 467 * needs to accept two different and identical structure types. This 468 * allows both old and new code to co-exist. The compile time assert 469 * adds some safety, that the structure sizes match. 470 */ 471 #define sg_page_iter_dma_address(spi) ({ \ 472 struct sg_page_iter *__spi = (void *)(spi); \ 473 dma_addr_t __dma_address; \ 474 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 475 __dma_address = __spi->sg->dma_address + \ 476 (__spi->sg_pgoffset << PAGE_SHIFT); \ 477 __dma_address; \ 478 }) 479 480 static inline struct page * 481 sg_page_iter_page(struct sg_page_iter *piter) 482 { 483 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 484 } 485 486 static __inline size_t 487 sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 488 const void *buf, size_t buflen, off_t skip) 489 { 490 struct sg_page_iter piter; 491 struct page *page; 492 struct sf_buf *sf; 493 size_t len, copied; 494 char *p, *b; 495 496 if (buflen == 0) 497 return (0); 498 499 b = __DECONST(char *, buf); 500 copied = 0; 501 sched_pin(); 502 for_each_sg_page(sgl, &piter, nents, 0) { 503 504 /* Skip to the start. */ 505 if (piter.sg->length <= skip) { 506 skip -= piter.sg->length; 507 continue; 508 } 509 510 /* See how much to copy. */ 511 KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)), 512 ("%s: sg len %u - skip %ju || buflen %zu is 0\n", 513 __func__, piter.sg->length, (uintmax_t)skip, buflen)); 514 len = min(piter.sg->length - skip, buflen); 515 516 page = sg_page_iter_page(&piter); 517 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 518 if (sf == NULL) 519 break; 520 p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; 521 memcpy(p, b, len); 522 sf_buf_free(sf); 523 524 /* We copied so nothing more to skip. */ 525 skip = 0; 526 copied += len; 527 /* Either we exactly filled the page, or we are done. */ 528 buflen -= len; 529 if (buflen == 0) 530 break; 531 b += len; 532 } 533 sched_unpin(); 534 535 return (copied); 536 } 537 538 static inline size_t 539 sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, 540 const void *buf, size_t buflen) 541 { 542 return (sg_pcopy_from_buffer(sgl, nents, buf, buflen, 0)); 543 } 544 545 static inline size_t 546 sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, 547 void *buf, size_t buflen, off_t offset) 548 { 549 struct sg_page_iter iter; 550 struct scatterlist *sg; 551 struct page *page; 552 struct sf_buf *sf; 553 char *vaddr; 554 size_t total = 0; 555 size_t len; 556 557 if (!PMAP_HAS_DMAP) 558 sched_pin(); 559 for_each_sg_page(sgl, &iter, nents, 0) { 560 sg = iter.sg; 561 562 if (offset >= sg->length) { 563 offset -= sg->length; 564 continue; 565 } 566 len = ulmin(buflen, sg->length - offset); 567 if (len == 0) 568 break; 569 570 page = sg_page_iter_page(&iter); 571 if (!PMAP_HAS_DMAP) { 572 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 573 if (sf == NULL) 574 break; 575 vaddr = (char *)sf_buf_kva(sf); 576 } else 577 vaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page)); 578 memcpy(buf, vaddr + sg->offset + offset, len); 579 if (!PMAP_HAS_DMAP) 580 sf_buf_free(sf); 581 582 /* start at beginning of next page */ 583 offset = 0; 584 585 /* advance buffer */ 586 buf = (char *)buf + len; 587 buflen -= len; 588 total += len; 589 } 590 if (!PMAP_HAS_DMAP) 591 sched_unpin(); 592 return (total); 593 } 594 595 #endif /* _LINUX_SCATTERLIST_H_ */ 596