1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 #ifndef _LINUX_SCATTERLIST_H_ 33 #define _LINUX_SCATTERLIST_H_ 34 35 #include <linux/page.h> 36 #include <linux/slab.h> 37 #include <linux/mm.h> 38 39 struct bus_dmamap; 40 struct scatterlist { 41 unsigned long page_link; 42 #define SG_PAGE_LINK_CHAIN 0x1UL 43 #define SG_PAGE_LINK_LAST 0x2UL 44 #define SG_PAGE_LINK_MASK 0x3UL 45 unsigned int offset; 46 unsigned int length; 47 dma_addr_t dma_address; 48 struct bus_dmamap *dma_map; /* FreeBSD specific */ 49 }; 50 51 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 52 53 struct sg_table { 54 struct scatterlist *sgl; 55 unsigned int nents; 56 unsigned int orig_nents; 57 }; 58 59 struct sg_page_iter { 60 struct scatterlist *sg; 61 unsigned int sg_pgoffset; 62 unsigned int maxents; 63 struct { 64 unsigned int nents; 65 int pg_advance; 66 } internal; 67 }; 68 69 struct sg_dma_page_iter { 70 struct sg_page_iter base; 71 }; 72 73 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 74 75 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 76 77 #define SG_MAGIC 0x87654321UL 78 #define SG_CHAIN SG_PAGE_LINK_CHAIN 79 #define SG_END SG_PAGE_LINK_LAST 80 81 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 82 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 83 #define sg_chain_ptr(sg) \ 84 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 85 86 #define sg_dma_address(sg) (sg)->dma_address 87 #define sg_dma_len(sg) (sg)->length 88 89 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 90 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 91 (iter)->sg; _sg_iter_next(iter)) 92 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 93 for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) 94 95 #define for_each_sg(sglist, sg, sgmax, iter) \ 96 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 97 98 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 99 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 100 101 static inline void 102 sg_assign_page(struct scatterlist *sg, struct page *page) 103 { 104 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 105 106 sg->page_link = page_link | (unsigned long)page; 107 } 108 109 static inline void 110 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 111 unsigned int offset) 112 { 113 sg_assign_page(sg, page); 114 sg->offset = offset; 115 sg->length = len; 116 } 117 118 static inline struct page * 119 sg_page(struct scatterlist *sg) 120 { 121 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 122 } 123 124 static inline void 125 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 126 { 127 sg_set_page(sg, virt_to_page(buf), buflen, 128 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 129 } 130 131 static inline struct scatterlist * 132 sg_next(struct scatterlist *sg) 133 { 134 if (sg_is_last(sg)) 135 return (NULL); 136 sg++; 137 if (sg_is_chain(sg)) 138 sg = sg_chain_ptr(sg); 139 return (sg); 140 } 141 142 static inline vm_paddr_t 143 sg_phys(struct scatterlist *sg) 144 { 145 return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 146 } 147 148 static inline void * 149 sg_virt(struct scatterlist *sg) 150 { 151 152 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 153 } 154 155 static inline void 156 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 157 struct scatterlist *sgl) 158 { 159 struct scatterlist *sg = &prv[prv_nents - 1]; 160 161 sg->offset = 0; 162 sg->length = 0; 163 sg->page_link = ((unsigned long)sgl | 164 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 165 } 166 167 static inline void 168 sg_mark_end(struct scatterlist *sg) 169 { 170 sg->page_link |= SG_PAGE_LINK_LAST; 171 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 172 } 173 174 static inline void 175 sg_init_table(struct scatterlist *sg, unsigned int nents) 176 { 177 bzero(sg, sizeof(*sg) * nents); 178 sg_mark_end(&sg[nents - 1]); 179 } 180 181 static struct scatterlist * 182 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 183 { 184 if (nents == SG_MAX_SINGLE_ALLOC) { 185 return ((void *)__get_free_page(gfp_mask)); 186 } else 187 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 188 } 189 190 static inline void 191 sg_kfree(struct scatterlist *sg, unsigned int nents) 192 { 193 if (nents == SG_MAX_SINGLE_ALLOC) { 194 free_page((unsigned long)sg); 195 } else 196 kfree(sg); 197 } 198 199 static inline void 200 __sg_free_table(struct sg_table *table, unsigned int max_ents, 201 bool skip_first_chunk, sg_free_fn * free_fn) 202 { 203 struct scatterlist *sgl, *next; 204 205 if (unlikely(!table->sgl)) 206 return; 207 208 sgl = table->sgl; 209 while (table->orig_nents) { 210 unsigned int alloc_size = table->orig_nents; 211 unsigned int sg_size; 212 213 if (alloc_size > max_ents) { 214 next = sg_chain_ptr(&sgl[max_ents - 1]); 215 alloc_size = max_ents; 216 sg_size = alloc_size - 1; 217 } else { 218 sg_size = alloc_size; 219 next = NULL; 220 } 221 222 table->orig_nents -= sg_size; 223 if (skip_first_chunk) 224 skip_first_chunk = 0; 225 else 226 free_fn(sgl, alloc_size); 227 sgl = next; 228 } 229 230 table->sgl = NULL; 231 } 232 233 static inline void 234 sg_free_table(struct sg_table *table) 235 { 236 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 237 } 238 239 static inline int 240 __sg_alloc_table(struct sg_table *table, unsigned int nents, 241 unsigned int max_ents, struct scatterlist *first_chunk, 242 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 243 { 244 struct scatterlist *sg, *prv; 245 unsigned int left; 246 247 memset(table, 0, sizeof(*table)); 248 249 if (nents == 0) 250 return (-EINVAL); 251 left = nents; 252 prv = NULL; 253 do { 254 unsigned int sg_size; 255 unsigned int alloc_size = left; 256 257 if (alloc_size > max_ents) { 258 alloc_size = max_ents; 259 sg_size = alloc_size - 1; 260 } else 261 sg_size = alloc_size; 262 263 left -= sg_size; 264 265 if (first_chunk) { 266 sg = first_chunk; 267 first_chunk = NULL; 268 } else { 269 sg = alloc_fn(alloc_size, gfp_mask); 270 } 271 if (unlikely(!sg)) { 272 if (prv) 273 table->nents = ++table->orig_nents; 274 275 return (-ENOMEM); 276 } 277 sg_init_table(sg, alloc_size); 278 table->nents = table->orig_nents += sg_size; 279 280 if (prv) 281 sg_chain(prv, max_ents, sg); 282 else 283 table->sgl = sg; 284 285 if (!left) 286 sg_mark_end(&sg[sg_size - 1]); 287 288 prv = sg; 289 } while (left); 290 291 return (0); 292 } 293 294 static inline int 295 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 296 { 297 int ret; 298 299 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 300 NULL, gfp_mask, sg_kmalloc); 301 if (unlikely(ret)) 302 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 303 304 return (ret); 305 } 306 307 static inline int 308 __sg_alloc_table_from_pages(struct sg_table *sgt, 309 struct page **pages, unsigned int count, 310 unsigned long off, unsigned long size, 311 unsigned int max_segment, gfp_t gfp_mask) 312 { 313 unsigned int i, segs, cur, len; 314 int rc; 315 struct scatterlist *s; 316 317 if (__predict_false(!max_segment || offset_in_page(max_segment))) 318 return (-EINVAL); 319 320 len = 0; 321 for (segs = i = 1; i < count; ++i) { 322 len += PAGE_SIZE; 323 if (len >= max_segment || 324 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 325 ++segs; 326 len = 0; 327 } 328 } 329 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 330 return (rc); 331 332 cur = 0; 333 for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 334 unsigned long seg_size; 335 unsigned int j; 336 337 len = 0; 338 for (j = cur + 1; j < count; ++j) { 339 len += PAGE_SIZE; 340 if (len >= max_segment || page_to_pfn(pages[j]) != 341 page_to_pfn(pages[j - 1]) + 1) 342 break; 343 } 344 345 seg_size = ((j - cur) << PAGE_SHIFT) - off; 346 sg_set_page(s, pages[cur], min(size, seg_size), off); 347 size -= seg_size; 348 off = 0; 349 cur = j; 350 } 351 return (0); 352 } 353 354 static inline int 355 sg_alloc_table_from_pages(struct sg_table *sgt, 356 struct page **pages, unsigned int count, 357 unsigned long off, unsigned long size, 358 gfp_t gfp_mask) 359 { 360 361 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 362 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 363 } 364 365 static inline int 366 sg_nents(struct scatterlist *sg) 367 { 368 int nents; 369 370 for (nents = 0; sg; sg = sg_next(sg)) 371 nents++; 372 return (nents); 373 } 374 375 static inline void 376 __sg_page_iter_start(struct sg_page_iter *piter, 377 struct scatterlist *sglist, unsigned int nents, 378 unsigned long pgoffset) 379 { 380 piter->internal.pg_advance = 0; 381 piter->internal.nents = nents; 382 383 piter->sg = sglist; 384 piter->sg_pgoffset = pgoffset; 385 } 386 387 static inline void 388 _sg_iter_next(struct sg_page_iter *iter) 389 { 390 struct scatterlist *sg; 391 unsigned int pgcount; 392 393 sg = iter->sg; 394 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 395 396 ++iter->sg_pgoffset; 397 while (iter->sg_pgoffset >= pgcount) { 398 iter->sg_pgoffset -= pgcount; 399 sg = sg_next(sg); 400 --iter->maxents; 401 if (sg == NULL || iter->maxents == 0) 402 break; 403 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 404 } 405 iter->sg = sg; 406 } 407 408 static inline int 409 sg_page_count(struct scatterlist *sg) 410 { 411 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 412 } 413 #define sg_dma_page_count(sg) \ 414 sg_page_count(sg) 415 416 static inline bool 417 __sg_page_iter_next(struct sg_page_iter *piter) 418 { 419 unsigned int pgcount; 420 421 if (piter->internal.nents == 0) 422 return (0); 423 if (piter->sg == NULL) 424 return (0); 425 426 piter->sg_pgoffset += piter->internal.pg_advance; 427 piter->internal.pg_advance = 1; 428 429 while (1) { 430 pgcount = sg_page_count(piter->sg); 431 if (likely(piter->sg_pgoffset < pgcount)) 432 break; 433 piter->sg_pgoffset -= pgcount; 434 piter->sg = sg_next(piter->sg); 435 if (--piter->internal.nents == 0) 436 return (0); 437 if (piter->sg == NULL) 438 return (0); 439 } 440 return (1); 441 } 442 #define __sg_page_iter_dma_next(itr) \ 443 __sg_page_iter_next(&(itr)->base) 444 445 static inline void 446 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 447 unsigned int nents, unsigned long pgoffset) 448 { 449 if (nents) { 450 iter->sg = sgl; 451 iter->sg_pgoffset = pgoffset - 1; 452 iter->maxents = nents; 453 _sg_iter_next(iter); 454 } else { 455 iter->sg = NULL; 456 iter->sg_pgoffset = 0; 457 iter->maxents = 0; 458 } 459 } 460 461 /* 462 * sg_page_iter_dma_address() is implemented as a macro because it 463 * needs to accept two different and identical structure types. This 464 * allows both old and new code to co-exist. The compile time assert 465 * adds some safety, that the structure sizes match. 466 */ 467 #define sg_page_iter_dma_address(spi) ({ \ 468 struct sg_page_iter *__spi = (void *)(spi); \ 469 dma_addr_t __dma_address; \ 470 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 471 __dma_address = __spi->sg->dma_address + \ 472 (__spi->sg_pgoffset << PAGE_SHIFT); \ 473 __dma_address; \ 474 }) 475 476 static inline struct page * 477 sg_page_iter_page(struct sg_page_iter *piter) 478 { 479 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 480 } 481 482 #endif /* _LINUX_SCATTERLIST_H_ */ 483