1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 #ifndef _LINUX_SCATTERLIST_H_ 33 #define _LINUX_SCATTERLIST_H_ 34 35 #include <linux/page.h> 36 #include <linux/slab.h> 37 #include <linux/mm.h> 38 39 struct scatterlist { 40 unsigned long page_link; 41 #define SG_PAGE_LINK_CHAIN 0x1UL 42 #define SG_PAGE_LINK_LAST 0x2UL 43 #define SG_PAGE_LINK_MASK 0x3UL 44 unsigned int offset; 45 unsigned int length; 46 dma_addr_t dma_address; 47 unsigned int dma_length; 48 }; 49 50 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 51 52 struct sg_table { 53 struct scatterlist *sgl; 54 unsigned int nents; 55 unsigned int orig_nents; 56 }; 57 58 struct sg_page_iter { 59 struct scatterlist *sg; 60 unsigned int sg_pgoffset; 61 unsigned int maxents; 62 struct { 63 unsigned int nents; 64 int pg_advance; 65 } internal; 66 }; 67 68 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 69 70 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 71 72 #define SG_MAGIC 0x87654321UL 73 #define SG_CHAIN SG_PAGE_LINK_CHAIN 74 #define SG_END SG_PAGE_LINK_LAST 75 76 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 77 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 78 #define sg_chain_ptr(sg) \ 79 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 80 81 #define sg_dma_address(sg) (sg)->dma_address 82 #define sg_dma_len(sg) (sg)->dma_length 83 84 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 85 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 86 (iter)->sg; _sg_iter_next(iter)) 87 88 #define for_each_sg(sglist, sg, sgmax, iter) \ 89 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 90 91 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 92 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 93 94 static inline void 95 sg_assign_page(struct scatterlist *sg, struct page *page) 96 { 97 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 98 99 sg->page_link = page_link | (unsigned long)page; 100 } 101 102 static inline void 103 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 104 unsigned int offset) 105 { 106 sg_assign_page(sg, page); 107 sg->offset = offset; 108 sg->length = len; 109 } 110 111 static inline struct page * 112 sg_page(struct scatterlist *sg) 113 { 114 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 115 } 116 117 static inline void 118 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 119 { 120 sg_set_page(sg, virt_to_page(buf), buflen, 121 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 122 } 123 124 static inline struct scatterlist * 125 sg_next(struct scatterlist *sg) 126 { 127 if (sg_is_last(sg)) 128 return (NULL); 129 sg++; 130 if (sg_is_chain(sg)) 131 sg = sg_chain_ptr(sg); 132 return (sg); 133 } 134 135 static inline vm_paddr_t 136 sg_phys(struct scatterlist *sg) 137 { 138 return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 139 } 140 141 static inline void * 142 sg_virt(struct scatterlist *sg) 143 { 144 145 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 146 } 147 148 static inline void 149 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 150 struct scatterlist *sgl) 151 { 152 struct scatterlist *sg = &prv[prv_nents - 1]; 153 154 sg->offset = 0; 155 sg->length = 0; 156 sg->page_link = ((unsigned long)sgl | 157 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 158 } 159 160 static inline void 161 sg_mark_end(struct scatterlist *sg) 162 { 163 sg->page_link |= SG_PAGE_LINK_LAST; 164 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 165 } 166 167 static inline void 168 sg_init_table(struct scatterlist *sg, unsigned int nents) 169 { 170 bzero(sg, sizeof(*sg) * nents); 171 sg_mark_end(&sg[nents - 1]); 172 } 173 174 static struct scatterlist * 175 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 176 { 177 if (nents == SG_MAX_SINGLE_ALLOC) { 178 return ((void *)__get_free_page(gfp_mask)); 179 } else 180 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 181 } 182 183 static inline void 184 sg_kfree(struct scatterlist *sg, unsigned int nents) 185 { 186 if (nents == SG_MAX_SINGLE_ALLOC) { 187 free_page((unsigned long)sg); 188 } else 189 kfree(sg); 190 } 191 192 static inline void 193 __sg_free_table(struct sg_table *table, unsigned int max_ents, 194 bool skip_first_chunk, sg_free_fn * free_fn) 195 { 196 struct scatterlist *sgl, *next; 197 198 if (unlikely(!table->sgl)) 199 return; 200 201 sgl = table->sgl; 202 while (table->orig_nents) { 203 unsigned int alloc_size = table->orig_nents; 204 unsigned int sg_size; 205 206 if (alloc_size > max_ents) { 207 next = sg_chain_ptr(&sgl[max_ents - 1]); 208 alloc_size = max_ents; 209 sg_size = alloc_size - 1; 210 } else { 211 sg_size = alloc_size; 212 next = NULL; 213 } 214 215 table->orig_nents -= sg_size; 216 if (skip_first_chunk) 217 skip_first_chunk = 0; 218 else 219 free_fn(sgl, alloc_size); 220 sgl = next; 221 } 222 223 table->sgl = NULL; 224 } 225 226 static inline void 227 sg_free_table(struct sg_table *table) 228 { 229 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 230 } 231 232 static inline int 233 __sg_alloc_table(struct sg_table *table, unsigned int nents, 234 unsigned int max_ents, struct scatterlist *first_chunk, 235 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 236 { 237 struct scatterlist *sg, *prv; 238 unsigned int left; 239 240 memset(table, 0, sizeof(*table)); 241 242 if (nents == 0) 243 return (-EINVAL); 244 left = nents; 245 prv = NULL; 246 do { 247 unsigned int sg_size; 248 unsigned int alloc_size = left; 249 250 if (alloc_size > max_ents) { 251 alloc_size = max_ents; 252 sg_size = alloc_size - 1; 253 } else 254 sg_size = alloc_size; 255 256 left -= sg_size; 257 258 if (first_chunk) { 259 sg = first_chunk; 260 first_chunk = NULL; 261 } else { 262 sg = alloc_fn(alloc_size, gfp_mask); 263 } 264 if (unlikely(!sg)) { 265 if (prv) 266 table->nents = ++table->orig_nents; 267 268 return (-ENOMEM); 269 } 270 sg_init_table(sg, alloc_size); 271 table->nents = table->orig_nents += sg_size; 272 273 if (prv) 274 sg_chain(prv, max_ents, sg); 275 else 276 table->sgl = sg; 277 278 if (!left) 279 sg_mark_end(&sg[sg_size - 1]); 280 281 prv = sg; 282 } while (left); 283 284 return (0); 285 } 286 287 static inline int 288 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 289 { 290 int ret; 291 292 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 293 NULL, gfp_mask, sg_kmalloc); 294 if (unlikely(ret)) 295 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 296 297 return (ret); 298 } 299 300 static inline int 301 __sg_alloc_table_from_pages(struct sg_table *sgt, 302 struct page **pages, unsigned int count, 303 unsigned long off, unsigned long size, 304 unsigned int max_segment, gfp_t gfp_mask) 305 { 306 unsigned int i, segs, cur, len; 307 int rc; 308 struct scatterlist *s; 309 310 if (__predict_false(!max_segment || offset_in_page(max_segment))) 311 return (-EINVAL); 312 313 len = 0; 314 for (segs = i = 1; i < count; ++i) { 315 len += PAGE_SIZE; 316 if (len >= max_segment || 317 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 318 ++segs; 319 len = 0; 320 } 321 } 322 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 323 return (rc); 324 325 cur = 0; 326 for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 327 unsigned long seg_size; 328 unsigned int j; 329 330 len = 0; 331 for (j = cur + 1; j < count; ++j) { 332 len += PAGE_SIZE; 333 if (len >= max_segment || page_to_pfn(pages[j]) != 334 page_to_pfn(pages[j - 1]) + 1) 335 break; 336 } 337 338 seg_size = ((j - cur) << PAGE_SHIFT) - off; 339 sg_set_page(s, pages[cur], min(size, seg_size), off); 340 size -= seg_size; 341 off = 0; 342 cur = j; 343 } 344 return (0); 345 } 346 347 static inline int 348 sg_alloc_table_from_pages(struct sg_table *sgt, 349 struct page **pages, unsigned int count, 350 unsigned long off, unsigned long size, 351 gfp_t gfp_mask) 352 { 353 354 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 355 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 356 } 357 358 static inline int 359 sg_nents(struct scatterlist *sg) 360 { 361 int nents; 362 363 for (nents = 0; sg; sg = sg_next(sg)) 364 nents++; 365 return (nents); 366 } 367 368 static inline void 369 __sg_page_iter_start(struct sg_page_iter *piter, 370 struct scatterlist *sglist, unsigned int nents, 371 unsigned long pgoffset) 372 { 373 piter->internal.pg_advance = 0; 374 piter->internal.nents = nents; 375 376 piter->sg = sglist; 377 piter->sg_pgoffset = pgoffset; 378 } 379 380 static inline void 381 _sg_iter_next(struct sg_page_iter *iter) 382 { 383 struct scatterlist *sg; 384 unsigned int pgcount; 385 386 sg = iter->sg; 387 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 388 389 ++iter->sg_pgoffset; 390 while (iter->sg_pgoffset >= pgcount) { 391 iter->sg_pgoffset -= pgcount; 392 sg = sg_next(sg); 393 --iter->maxents; 394 if (sg == NULL || iter->maxents == 0) 395 break; 396 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 397 } 398 iter->sg = sg; 399 } 400 401 static inline int 402 sg_page_count(struct scatterlist *sg) 403 { 404 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 405 } 406 407 static inline bool 408 __sg_page_iter_next(struct sg_page_iter *piter) 409 { 410 if (piter->internal.nents == 0) 411 return (0); 412 if (piter->sg == NULL) 413 return (0); 414 415 piter->sg_pgoffset += piter->internal.pg_advance; 416 piter->internal.pg_advance = 1; 417 418 while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { 419 piter->sg_pgoffset -= sg_page_count(piter->sg); 420 piter->sg = sg_next(piter->sg); 421 if (--piter->internal.nents == 0) 422 return (0); 423 if (piter->sg == NULL) 424 return (0); 425 } 426 return (1); 427 } 428 429 static inline void 430 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 431 unsigned int nents, unsigned long pgoffset) 432 { 433 if (nents) { 434 iter->sg = sgl; 435 iter->sg_pgoffset = pgoffset - 1; 436 iter->maxents = nents; 437 _sg_iter_next(iter); 438 } else { 439 iter->sg = NULL; 440 iter->sg_pgoffset = 0; 441 iter->maxents = 0; 442 } 443 } 444 445 static inline dma_addr_t 446 sg_page_iter_dma_address(struct sg_page_iter *spi) 447 { 448 return (spi->sg->dma_address + (spi->sg_pgoffset << PAGE_SHIFT)); 449 } 450 451 static inline struct page * 452 sg_page_iter_page(struct sg_page_iter *piter) 453 { 454 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 455 } 456 457 458 #endif /* _LINUX_SCATTERLIST_H_ */ 459