1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 #ifndef _LINUX_SCATTERLIST_H_ 33 #define _LINUX_SCATTERLIST_H_ 34 35 #include <linux/page.h> 36 #include <linux/slab.h> 37 38 struct scatterlist { 39 union { 40 struct page *page; 41 struct scatterlist *sg; 42 } sl_un; 43 dma_addr_t address; 44 unsigned long offset; 45 uint32_t length; 46 uint32_t flags; 47 }; 48 49 struct sg_table { 50 struct scatterlist *sgl; 51 unsigned int nents; 52 unsigned int orig_nents; 53 }; 54 55 struct sg_page_iter { 56 struct scatterlist *sg; 57 unsigned int sg_pgoffset; 58 unsigned int maxents; 59 }; 60 61 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 62 63 #define sg_dma_address(sg) (sg)->address 64 #define sg_dma_len(sg) (sg)->length 65 #define sg_page(sg) (sg)->sl_un.page 66 #define sg_scatternext(sg) (sg)->sl_un.sg 67 68 #define SG_END 0x01 69 #define SG_CHAIN 0x02 70 71 static inline void 72 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 73 unsigned int offset) 74 { 75 sg_page(sg) = page; 76 sg_dma_len(sg) = len; 77 sg->offset = offset; 78 if (offset > PAGE_SIZE) 79 panic("sg_set_page: Invalid offset %d\n", offset); 80 } 81 82 static inline void 83 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 84 { 85 sg_set_page(sg, virt_to_page(buf), buflen, 86 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 87 } 88 89 static inline void 90 sg_init_table(struct scatterlist *sg, unsigned int nents) 91 { 92 bzero(sg, sizeof(*sg) * nents); 93 sg[nents - 1].flags = SG_END; 94 } 95 96 static inline struct scatterlist * 97 sg_next(struct scatterlist *sg) 98 { 99 if (sg->flags & SG_END) 100 return (NULL); 101 sg++; 102 if (sg->flags & SG_CHAIN) 103 sg = sg_scatternext(sg); 104 return (sg); 105 } 106 107 static inline vm_paddr_t 108 sg_phys(struct scatterlist *sg) 109 { 110 return sg_page(sg)->phys_addr + sg->offset; 111 } 112 113 static inline void 114 sg_chain(struct scatterlist *prv, unsigned int prv_nents, 115 struct scatterlist *sgl) 116 { 117 struct scatterlist *sg = &prv[prv_nents - 1]; 118 119 sg->offset = 0; 120 sg->length = 0; 121 sg->flags = SG_CHAIN; 122 sg->sl_un.sg = sgl; 123 } 124 125 static inline void 126 sg_mark_end(struct scatterlist *sg) 127 { 128 sg->flags = SG_END; 129 } 130 131 static inline void 132 __sg_free_table(struct sg_table *table, unsigned int max_ents) 133 { 134 struct scatterlist *sgl, *next; 135 136 if (unlikely(!table->sgl)) 137 return; 138 139 sgl = table->sgl; 140 while (table->orig_nents) { 141 unsigned int alloc_size = table->orig_nents; 142 unsigned int sg_size; 143 144 if (alloc_size > max_ents) { 145 next = sgl[max_ents - 1].sl_un.sg; 146 alloc_size = max_ents; 147 sg_size = alloc_size - 1; 148 } else { 149 sg_size = alloc_size; 150 next = NULL; 151 } 152 153 table->orig_nents -= sg_size; 154 kfree(sgl); 155 sgl = next; 156 } 157 158 table->sgl = NULL; 159 } 160 161 static inline void 162 sg_free_table(struct sg_table *table) 163 { 164 __sg_free_table(table, SG_MAX_SINGLE_ALLOC); 165 } 166 167 static inline int 168 __sg_alloc_table(struct sg_table *table, unsigned int nents, 169 unsigned int max_ents, gfp_t gfp_mask) 170 { 171 struct scatterlist *sg, *prv; 172 unsigned int left; 173 174 memset(table, 0, sizeof(*table)); 175 176 if (nents == 0) 177 return -EINVAL; 178 left = nents; 179 prv = NULL; 180 do { 181 unsigned int sg_size; 182 unsigned int alloc_size = left; 183 184 if (alloc_size > max_ents) { 185 alloc_size = max_ents; 186 sg_size = alloc_size - 1; 187 } else 188 sg_size = alloc_size; 189 190 left -= sg_size; 191 192 sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); 193 if (unlikely(!sg)) { 194 if (prv) 195 table->nents = ++table->orig_nents; 196 197 return -ENOMEM; 198 } 199 sg_init_table(sg, alloc_size); 200 table->nents = table->orig_nents += sg_size; 201 202 if (prv) 203 sg_chain(prv, max_ents, sg); 204 else 205 table->sgl = sg; 206 207 if (!left) 208 sg_mark_end(&sg[sg_size - 1]); 209 210 prv = sg; 211 } while (left); 212 213 return 0; 214 } 215 216 static inline int 217 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 218 { 219 int ret; 220 221 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 222 gfp_mask); 223 if (unlikely(ret)) 224 __sg_free_table(table, SG_MAX_SINGLE_ALLOC); 225 226 return ret; 227 } 228 229 static inline void 230 _sg_iter_next(struct sg_page_iter *iter) 231 { 232 struct scatterlist *sg; 233 unsigned int pgcount; 234 235 sg = iter->sg; 236 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 237 238 ++iter->sg_pgoffset; 239 while (iter->sg_pgoffset >= pgcount) { 240 iter->sg_pgoffset -= pgcount; 241 sg = sg_next(sg); 242 --iter->maxents; 243 if (sg == NULL || iter->maxents == 0) 244 break; 245 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 246 } 247 iter->sg = sg; 248 } 249 250 static inline void 251 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 252 unsigned int nents, unsigned long pgoffset) 253 { 254 if (nents) { 255 iter->sg = sgl; 256 iter->sg_pgoffset = pgoffset - 1; 257 iter->maxents = nents; 258 _sg_iter_next(iter); 259 } else { 260 iter->sg = NULL; 261 iter->sg_pgoffset = 0; 262 iter->maxents = 0; 263 } 264 } 265 266 static inline dma_addr_t 267 sg_page_iter_dma_address(struct sg_page_iter *spi) 268 { 269 return spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT); 270 } 271 272 #define for_each_sg_page(sgl, iter, nents, pgoffset) \ 273 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 274 (iter)->sg; _sg_iter_next(iter)) 275 276 #define for_each_sg(sglist, sg, sgmax, _itr) \ 277 for (_itr = 0, sg = (sglist); _itr < (sgmax); _itr++, sg = sg_next(sg)) 278 279 #endif /* _LINUX_SCATTERLIST_H_ */ 280