1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 #ifndef _LINUX_SCATTERLIST_H_ 33 #define _LINUX_SCATTERLIST_H_ 34 35 #include <sys/types.h> 36 #include <sys/sf_buf.h> 37 38 #include <linux/page.h> 39 #include <linux/slab.h> 40 #include <linux/mm.h> 41 42 struct bus_dmamap; 43 struct scatterlist { 44 unsigned long page_link; 45 #define SG_PAGE_LINK_CHAIN 0x1UL 46 #define SG_PAGE_LINK_LAST 0x2UL 47 #define SG_PAGE_LINK_MASK 0x3UL 48 unsigned int offset; 49 unsigned int length; 50 dma_addr_t dma_address; 51 struct bus_dmamap *dma_map; /* FreeBSD specific */ 52 }; 53 54 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 55 56 struct sg_table { 57 struct scatterlist *sgl; 58 unsigned int nents; 59 unsigned int orig_nents; 60 }; 61 62 struct sg_page_iter { 63 struct scatterlist *sg; 64 unsigned int sg_pgoffset; 65 unsigned int maxents; 66 struct { 67 unsigned int nents; 68 int pg_advance; 69 } internal; 70 }; 71 72 struct sg_dma_page_iter { 73 struct sg_page_iter base; 74 }; 75 76 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 77 78 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 79 80 #define SG_MAGIC 0x87654321UL 81 #define SG_CHAIN SG_PAGE_LINK_CHAIN 82 #define SG_END SG_PAGE_LINK_LAST 83 84 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 85 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 86 #define sg_chain_ptr(sg) \ 87 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 88 89 #define sg_dma_address(sg) (sg)->dma_address 90 #define sg_dma_len(sg) (sg)->length 91 92 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 93 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 94 (iter)->sg; _sg_iter_next(iter)) 95 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 96 for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) 97 98 #define for_each_sg(sglist, sg, sgmax, iter) \ 99 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 100 101 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 102 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 103 104 static inline void 105 sg_assign_page(struct scatterlist *sg, struct page *page) 106 { 107 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 108 109 sg->page_link = page_link | (unsigned long)page; 110 } 111 112 static inline void 113 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 114 unsigned int offset) 115 { 116 sg_assign_page(sg, page); 117 sg->offset = offset; 118 sg->length = len; 119 } 120 121 static inline struct page * 122 sg_page(struct scatterlist *sg) 123 { 124 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 125 } 126 127 static inline void 128 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 129 { 130 sg_set_page(sg, virt_to_page(buf), buflen, 131 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 132 } 133 134 static inline struct scatterlist * 135 sg_next(struct scatterlist *sg) 136 { 137 if (sg_is_last(sg)) 138 return (NULL); 139 sg++; 140 if (sg_is_chain(sg)) 141 sg = sg_chain_ptr(sg); 142 return (sg); 143 } 144 145 static inline vm_paddr_t 146 sg_phys(struct scatterlist *sg) 147 { 148 return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 149 } 150 151 static inline void * 152 sg_virt(struct scatterlist *sg) 153 { 154 155 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 156 } 157 158 static inline void 159 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 160 struct scatterlist *sgl) 161 { 162 struct scatterlist *sg = &prv[prv_nents - 1]; 163 164 sg->offset = 0; 165 sg->length = 0; 166 sg->page_link = ((unsigned long)sgl | 167 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 168 } 169 170 static inline void 171 sg_mark_end(struct scatterlist *sg) 172 { 173 sg->page_link |= SG_PAGE_LINK_LAST; 174 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 175 } 176 177 static inline void 178 sg_init_table(struct scatterlist *sg, unsigned int nents) 179 { 180 bzero(sg, sizeof(*sg) * nents); 181 sg_mark_end(&sg[nents - 1]); 182 } 183 184 static struct scatterlist * 185 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 186 { 187 if (nents == SG_MAX_SINGLE_ALLOC) { 188 return ((void *)__get_free_page(gfp_mask)); 189 } else 190 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 191 } 192 193 static inline void 194 sg_kfree(struct scatterlist *sg, unsigned int nents) 195 { 196 if (nents == SG_MAX_SINGLE_ALLOC) { 197 free_page((unsigned long)sg); 198 } else 199 kfree(sg); 200 } 201 202 static inline void 203 __sg_free_table(struct sg_table *table, unsigned int max_ents, 204 bool skip_first_chunk, sg_free_fn * free_fn) 205 { 206 struct scatterlist *sgl, *next; 207 208 if (unlikely(!table->sgl)) 209 return; 210 211 sgl = table->sgl; 212 while (table->orig_nents) { 213 unsigned int alloc_size = table->orig_nents; 214 unsigned int sg_size; 215 216 if (alloc_size > max_ents) { 217 next = sg_chain_ptr(&sgl[max_ents - 1]); 218 alloc_size = max_ents; 219 sg_size = alloc_size - 1; 220 } else { 221 sg_size = alloc_size; 222 next = NULL; 223 } 224 225 table->orig_nents -= sg_size; 226 if (skip_first_chunk) 227 skip_first_chunk = 0; 228 else 229 free_fn(sgl, alloc_size); 230 sgl = next; 231 } 232 233 table->sgl = NULL; 234 } 235 236 static inline void 237 sg_free_table(struct sg_table *table) 238 { 239 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 240 } 241 242 static inline int 243 __sg_alloc_table(struct sg_table *table, unsigned int nents, 244 unsigned int max_ents, struct scatterlist *first_chunk, 245 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 246 { 247 struct scatterlist *sg, *prv; 248 unsigned int left; 249 250 memset(table, 0, sizeof(*table)); 251 252 if (nents == 0) 253 return (-EINVAL); 254 left = nents; 255 prv = NULL; 256 do { 257 unsigned int sg_size; 258 unsigned int alloc_size = left; 259 260 if (alloc_size > max_ents) { 261 alloc_size = max_ents; 262 sg_size = alloc_size - 1; 263 } else 264 sg_size = alloc_size; 265 266 left -= sg_size; 267 268 if (first_chunk) { 269 sg = first_chunk; 270 first_chunk = NULL; 271 } else { 272 sg = alloc_fn(alloc_size, gfp_mask); 273 } 274 if (unlikely(!sg)) { 275 if (prv) 276 table->nents = ++table->orig_nents; 277 278 return (-ENOMEM); 279 } 280 sg_init_table(sg, alloc_size); 281 table->nents = table->orig_nents += sg_size; 282 283 if (prv) 284 sg_chain(prv, max_ents, sg); 285 else 286 table->sgl = sg; 287 288 if (!left) 289 sg_mark_end(&sg[sg_size - 1]); 290 291 prv = sg; 292 } while (left); 293 294 return (0); 295 } 296 297 static inline int 298 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 299 { 300 int ret; 301 302 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 303 NULL, gfp_mask, sg_kmalloc); 304 if (unlikely(ret)) 305 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 306 307 return (ret); 308 } 309 310 static inline int 311 __sg_alloc_table_from_pages(struct sg_table *sgt, 312 struct page **pages, unsigned int count, 313 unsigned long off, unsigned long size, 314 unsigned int max_segment, gfp_t gfp_mask) 315 { 316 unsigned int i, segs, cur, len; 317 int rc; 318 struct scatterlist *s; 319 320 if (__predict_false(!max_segment || offset_in_page(max_segment))) 321 return (-EINVAL); 322 323 len = 0; 324 for (segs = i = 1; i < count; ++i) { 325 len += PAGE_SIZE; 326 if (len >= max_segment || 327 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 328 ++segs; 329 len = 0; 330 } 331 } 332 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 333 return (rc); 334 335 cur = 0; 336 for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 337 unsigned long seg_size; 338 unsigned int j; 339 340 len = 0; 341 for (j = cur + 1; j < count; ++j) { 342 len += PAGE_SIZE; 343 if (len >= max_segment || page_to_pfn(pages[j]) != 344 page_to_pfn(pages[j - 1]) + 1) 345 break; 346 } 347 348 seg_size = ((j - cur) << PAGE_SHIFT) - off; 349 sg_set_page(s, pages[cur], MIN(size, seg_size), off); 350 size -= seg_size; 351 off = 0; 352 cur = j; 353 } 354 return (0); 355 } 356 357 static inline int 358 sg_alloc_table_from_pages(struct sg_table *sgt, 359 struct page **pages, unsigned int count, 360 unsigned long off, unsigned long size, 361 gfp_t gfp_mask) 362 { 363 364 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 365 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 366 } 367 368 static inline int 369 sg_nents(struct scatterlist *sg) 370 { 371 int nents; 372 373 for (nents = 0; sg; sg = sg_next(sg)) 374 nents++; 375 return (nents); 376 } 377 378 static inline void 379 __sg_page_iter_start(struct sg_page_iter *piter, 380 struct scatterlist *sglist, unsigned int nents, 381 unsigned long pgoffset) 382 { 383 piter->internal.pg_advance = 0; 384 piter->internal.nents = nents; 385 386 piter->sg = sglist; 387 piter->sg_pgoffset = pgoffset; 388 } 389 390 static inline void 391 _sg_iter_next(struct sg_page_iter *iter) 392 { 393 struct scatterlist *sg; 394 unsigned int pgcount; 395 396 sg = iter->sg; 397 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 398 399 ++iter->sg_pgoffset; 400 while (iter->sg_pgoffset >= pgcount) { 401 iter->sg_pgoffset -= pgcount; 402 sg = sg_next(sg); 403 --iter->maxents; 404 if (sg == NULL || iter->maxents == 0) 405 break; 406 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 407 } 408 iter->sg = sg; 409 } 410 411 static inline int 412 sg_page_count(struct scatterlist *sg) 413 { 414 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 415 } 416 #define sg_dma_page_count(sg) \ 417 sg_page_count(sg) 418 419 static inline bool 420 __sg_page_iter_next(struct sg_page_iter *piter) 421 { 422 unsigned int pgcount; 423 424 if (piter->internal.nents == 0) 425 return (0); 426 if (piter->sg == NULL) 427 return (0); 428 429 piter->sg_pgoffset += piter->internal.pg_advance; 430 piter->internal.pg_advance = 1; 431 432 while (1) { 433 pgcount = sg_page_count(piter->sg); 434 if (likely(piter->sg_pgoffset < pgcount)) 435 break; 436 piter->sg_pgoffset -= pgcount; 437 piter->sg = sg_next(piter->sg); 438 if (--piter->internal.nents == 0) 439 return (0); 440 if (piter->sg == NULL) 441 return (0); 442 } 443 return (1); 444 } 445 #define __sg_page_iter_dma_next(itr) \ 446 __sg_page_iter_next(&(itr)->base) 447 448 static inline void 449 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 450 unsigned int nents, unsigned long pgoffset) 451 { 452 if (nents) { 453 iter->sg = sgl; 454 iter->sg_pgoffset = pgoffset - 1; 455 iter->maxents = nents; 456 _sg_iter_next(iter); 457 } else { 458 iter->sg = NULL; 459 iter->sg_pgoffset = 0; 460 iter->maxents = 0; 461 } 462 } 463 464 /* 465 * sg_page_iter_dma_address() is implemented as a macro because it 466 * needs to accept two different and identical structure types. This 467 * allows both old and new code to co-exist. The compile time assert 468 * adds some safety, that the structure sizes match. 469 */ 470 #define sg_page_iter_dma_address(spi) ({ \ 471 struct sg_page_iter *__spi = (void *)(spi); \ 472 dma_addr_t __dma_address; \ 473 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 474 __dma_address = __spi->sg->dma_address + \ 475 (__spi->sg_pgoffset << PAGE_SHIFT); \ 476 __dma_address; \ 477 }) 478 479 static inline struct page * 480 sg_page_iter_page(struct sg_page_iter *piter) 481 { 482 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 483 } 484 485 static __inline size_t 486 sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 487 const void *buf, size_t buflen, off_t skip) 488 { 489 struct sg_page_iter piter; 490 struct page *page; 491 struct sf_buf *sf; 492 size_t len, copied; 493 char *p, *b; 494 495 if (buflen == 0) 496 return (0); 497 498 b = __DECONST(char *, buf); 499 copied = 0; 500 sched_pin(); 501 for_each_sg_page(sgl, &piter, nents, 0) { 502 503 /* Skip to the start. */ 504 if (piter.sg->length <= skip) { 505 skip -= piter.sg->length; 506 continue; 507 } 508 509 /* See how much to copy. */ 510 KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)), 511 ("%s: sg len %u - skip %ju || buflen %zu is 0\n", 512 __func__, piter.sg->length, (uintmax_t)skip, buflen)); 513 len = min(piter.sg->length - skip, buflen); 514 515 page = sg_page_iter_page(&piter); 516 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); 517 if (sf == NULL) 518 break; 519 p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; 520 memcpy(p, b, len); 521 sf_buf_free(sf); 522 523 copied += len; 524 /* Either we exactly filled the page, or we are done. */ 525 buflen -= len; 526 if (buflen == 0) 527 break; 528 skip -= len; 529 b += len; 530 } 531 sched_unpin(); 532 533 return (copied); 534 } 535 536 #endif /* _LINUX_SCATTERLIST_H_ */ 537