1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 #ifndef _LINUX_SCATTERLIST_H_ 33 #define _LINUX_SCATTERLIST_H_ 34 35 #include <linux/page.h> 36 #include <linux/slab.h> 37 #include <linux/mm.h> 38 39 struct scatterlist { 40 unsigned long page_link; 41 #define SG_PAGE_LINK_CHAIN 0x1UL 42 #define SG_PAGE_LINK_LAST 0x2UL 43 #define SG_PAGE_LINK_MASK 0x3UL 44 unsigned int offset; 45 unsigned int length; 46 dma_addr_t address; 47 }; 48 49 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 50 51 struct sg_table { 52 struct scatterlist *sgl; 53 unsigned int nents; 54 unsigned int orig_nents; 55 }; 56 57 struct sg_page_iter { 58 struct scatterlist *sg; 59 unsigned int sg_pgoffset; 60 unsigned int maxents; 61 struct { 62 unsigned int nents; 63 int pg_advance; 64 } internal; 65 }; 66 67 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 68 69 #define SG_MAGIC 0x87654321UL 70 71 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 72 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 73 #define sg_chain_ptr(sg) \ 74 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 75 76 #define sg_dma_address(sg) (sg)->address 77 #define sg_dma_len(sg) (sg)->length 78 79 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 80 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 81 (iter)->sg; _sg_iter_next(iter)) 82 83 #define for_each_sg(sglist, sg, sgmax, iter) \ 84 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 85 86 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 87 typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 88 89 static inline void 90 sg_assign_page(struct scatterlist *sg, struct page *page) 91 { 92 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 93 94 sg->page_link = page_link | (unsigned long)page; 95 } 96 97 static inline void 98 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 99 unsigned int offset) 100 { 101 sg_assign_page(sg, page); 102 sg->offset = offset; 103 sg->length = len; 104 } 105 106 static inline struct page * 107 sg_page(struct scatterlist *sg) 108 { 109 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 110 } 111 112 static inline void 113 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 114 { 115 sg_set_page(sg, virt_to_page(buf), buflen, 116 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 117 } 118 119 static inline struct scatterlist * 120 sg_next(struct scatterlist *sg) 121 { 122 if (sg_is_last(sg)) 123 return (NULL); 124 sg++; 125 if (sg_is_chain(sg)) 126 sg = sg_chain_ptr(sg); 127 return (sg); 128 } 129 130 static inline vm_paddr_t 131 sg_phys(struct scatterlist *sg) 132 { 133 return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 134 } 135 136 static inline void 137 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 138 struct scatterlist *sgl) 139 { 140 struct scatterlist *sg = &prv[prv_nents - 1]; 141 142 sg->offset = 0; 143 sg->length = 0; 144 sg->page_link = ((unsigned long)sgl | 145 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 146 } 147 148 static inline void 149 sg_mark_end(struct scatterlist *sg) 150 { 151 sg->page_link |= SG_PAGE_LINK_LAST; 152 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 153 } 154 155 static inline void 156 sg_init_table(struct scatterlist *sg, unsigned int nents) 157 { 158 bzero(sg, sizeof(*sg) * nents); 159 sg_mark_end(&sg[nents - 1]); 160 } 161 162 static struct scatterlist * 163 sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 164 { 165 if (nents == SG_MAX_SINGLE_ALLOC) { 166 return ((void *)__get_free_page(gfp_mask)); 167 } else 168 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 169 } 170 171 static inline void 172 sg_kfree(struct scatterlist *sg, unsigned int nents) 173 { 174 if (nents == SG_MAX_SINGLE_ALLOC) { 175 free_page((unsigned long)sg); 176 } else 177 kfree(sg); 178 } 179 180 static inline void 181 __sg_free_table(struct sg_table *table, unsigned int max_ents, 182 bool skip_first_chunk, sg_free_fn * free_fn) 183 { 184 struct scatterlist *sgl, *next; 185 186 if (unlikely(!table->sgl)) 187 return; 188 189 sgl = table->sgl; 190 while (table->orig_nents) { 191 unsigned int alloc_size = table->orig_nents; 192 unsigned int sg_size; 193 194 if (alloc_size > max_ents) { 195 next = sg_chain_ptr(&sgl[max_ents - 1]); 196 alloc_size = max_ents; 197 sg_size = alloc_size - 1; 198 } else { 199 sg_size = alloc_size; 200 next = NULL; 201 } 202 203 table->orig_nents -= sg_size; 204 if (skip_first_chunk) 205 skip_first_chunk = 0; 206 else 207 free_fn(sgl, alloc_size); 208 sgl = next; 209 } 210 211 table->sgl = NULL; 212 } 213 214 static inline void 215 sg_free_table(struct sg_table *table) 216 { 217 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 218 } 219 220 static inline int 221 __sg_alloc_table(struct sg_table *table, unsigned int nents, 222 unsigned int max_ents, struct scatterlist *first_chunk, 223 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 224 { 225 struct scatterlist *sg, *prv; 226 unsigned int left; 227 228 memset(table, 0, sizeof(*table)); 229 230 if (nents == 0) 231 return (-EINVAL); 232 left = nents; 233 prv = NULL; 234 do { 235 unsigned int sg_size; 236 unsigned int alloc_size = left; 237 238 if (alloc_size > max_ents) { 239 alloc_size = max_ents; 240 sg_size = alloc_size - 1; 241 } else 242 sg_size = alloc_size; 243 244 left -= sg_size; 245 246 if (first_chunk) { 247 sg = first_chunk; 248 first_chunk = NULL; 249 } else { 250 sg = alloc_fn(alloc_size, gfp_mask); 251 } 252 if (unlikely(!sg)) { 253 if (prv) 254 table->nents = ++table->orig_nents; 255 256 return (-ENOMEM); 257 } 258 sg_init_table(sg, alloc_size); 259 table->nents = table->orig_nents += sg_size; 260 261 if (prv) 262 sg_chain(prv, max_ents, sg); 263 else 264 table->sgl = sg; 265 266 if (!left) 267 sg_mark_end(&sg[sg_size - 1]); 268 269 prv = sg; 270 } while (left); 271 272 return (0); 273 } 274 275 static inline int 276 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 277 { 278 int ret; 279 280 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 281 NULL, gfp_mask, sg_kmalloc); 282 if (unlikely(ret)) 283 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 284 285 return (ret); 286 } 287 288 static inline int 289 sg_alloc_table_from_pages(struct sg_table *sgt, 290 struct page **pages, unsigned int count, 291 unsigned long off, unsigned long size, 292 gfp_t gfp_mask) 293 { 294 unsigned int i, segs, cur; 295 int rc; 296 struct scatterlist *s; 297 298 for (segs = i = 1; i < count; ++i) { 299 if (page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) 300 ++segs; 301 } 302 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 303 return (rc); 304 305 cur = 0; 306 for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 307 unsigned long seg_size; 308 unsigned int j; 309 310 for (j = cur + 1; j < count; ++j) 311 if (page_to_pfn(pages[j]) != 312 page_to_pfn(pages[j - 1]) + 1) 313 break; 314 315 seg_size = ((j - cur) << PAGE_SHIFT) - off; 316 sg_set_page(s, pages[cur], min(size, seg_size), off); 317 size -= seg_size; 318 off = 0; 319 cur = j; 320 } 321 return (0); 322 } 323 324 325 static inline int 326 sg_nents(struct scatterlist *sg) 327 { 328 int nents; 329 330 for (nents = 0; sg; sg = sg_next(sg)) 331 nents++; 332 return (nents); 333 } 334 335 static inline void 336 __sg_page_iter_start(struct sg_page_iter *piter, 337 struct scatterlist *sglist, unsigned int nents, 338 unsigned long pgoffset) 339 { 340 piter->internal.pg_advance = 0; 341 piter->internal.nents = nents; 342 343 piter->sg = sglist; 344 piter->sg_pgoffset = pgoffset; 345 } 346 347 static inline void 348 _sg_iter_next(struct sg_page_iter *iter) 349 { 350 struct scatterlist *sg; 351 unsigned int pgcount; 352 353 sg = iter->sg; 354 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 355 356 ++iter->sg_pgoffset; 357 while (iter->sg_pgoffset >= pgcount) { 358 iter->sg_pgoffset -= pgcount; 359 sg = sg_next(sg); 360 --iter->maxents; 361 if (sg == NULL || iter->maxents == 0) 362 break; 363 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 364 } 365 iter->sg = sg; 366 } 367 368 static inline int 369 sg_page_count(struct scatterlist *sg) 370 { 371 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 372 } 373 374 static inline bool 375 __sg_page_iter_next(struct sg_page_iter *piter) 376 { 377 if (piter->internal.nents == 0) 378 return (0); 379 if (piter->sg == NULL) 380 return (0); 381 382 piter->sg_pgoffset += piter->internal.pg_advance; 383 piter->internal.pg_advance = 1; 384 385 while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { 386 piter->sg_pgoffset -= sg_page_count(piter->sg); 387 piter->sg = sg_next(piter->sg); 388 if (--piter->internal.nents == 0) 389 return (0); 390 if (piter->sg == NULL) 391 return (0); 392 } 393 return (1); 394 } 395 396 static inline void 397 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 398 unsigned int nents, unsigned long pgoffset) 399 { 400 if (nents) { 401 iter->sg = sgl; 402 iter->sg_pgoffset = pgoffset - 1; 403 iter->maxents = nents; 404 _sg_iter_next(iter); 405 } else { 406 iter->sg = NULL; 407 iter->sg_pgoffset = 0; 408 iter->maxents = 0; 409 } 410 } 411 412 static inline dma_addr_t 413 sg_page_iter_dma_address(struct sg_page_iter *spi) 414 { 415 return (spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT)); 416 } 417 418 static inline struct page * 419 sg_page_iter_page(struct sg_page_iter *piter) 420 { 421 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 422 } 423 424 425 #endif /* _LINUX_SCATTERLIST_H_ */ 426