1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * bvec iterator 4 * 5 * Copyright (C) 2001 Ming Lei <ming.lei@canonical.com> 6 */ 7 #ifndef __LINUX_BVEC_H 8 #define __LINUX_BVEC_H 9 10 #include <linux/highmem.h> 11 #include <linux/bug.h> 12 #include <linux/errno.h> 13 #include <linux/limits.h> 14 #include <linux/minmax.h> 15 #include <linux/types.h> 16 17 struct page; 18 19 /** 20 * struct bio_vec - a contiguous range of physical memory addresses 21 * @bv_page: First page associated with the address range. 22 * @bv_len: Number of bytes in the address range. 23 * @bv_offset: Start of the address range relative to the start of @bv_page. 24 * 25 * All pages within a bio_vec starting from @bv_page are contiguous and 26 * can simply be iterated (see bvec_advance()). 27 */ 28 struct bio_vec { 29 struct page *bv_page; 30 unsigned int bv_len; 31 unsigned int bv_offset; 32 }; 33 34 /** 35 * bvec_set_page - initialize a bvec based off a struct page 36 * @bv: bvec to initialize 37 * @page: page the bvec should point to 38 * @len: length of the bvec 39 * @offset: offset into the page 40 */ 41 static inline void bvec_set_page(struct bio_vec *bv, struct page *page, 42 unsigned int len, unsigned int offset) 43 { 44 bv->bv_page = page; 45 bv->bv_len = len; 46 bv->bv_offset = offset; 47 } 48 49 /** 50 * bvec_set_folio - initialize a bvec based off a struct folio 51 * @bv: bvec to initialize 52 * @folio: folio the bvec should point to 53 * @len: length of the bvec 54 * @offset: offset into the folio 55 */ 56 static inline void bvec_set_folio(struct bio_vec *bv, struct folio *folio, 57 size_t len, size_t offset) 58 { 59 unsigned long nr = offset / PAGE_SIZE; 60 61 WARN_ON_ONCE(len > UINT_MAX); 62 bvec_set_page(bv, folio_page(folio, nr), len, offset % PAGE_SIZE); 63 } 64 65 /** 66 * bvec_set_virt - initialize a bvec based on a virtual address 67 * @bv: bvec to initialize 68 * @vaddr: virtual address to set the bvec to 69 * @len: length of the bvec 70 */ 71 static inline void bvec_set_virt(struct bio_vec *bv, void *vaddr, 72 unsigned int len) 73 { 74 bvec_set_page(bv, virt_to_page(vaddr), len, offset_in_page(vaddr)); 75 } 76 77 /** 78 * bvec_folio - Return the first folio referenced by this bvec 79 * @bv: bvec to access 80 * 81 * A bvec can contain non-folio memory, so this should only be called by 82 * the creator of the bvec; drivers have no business looking at the owner 83 * of the memory. It may not even be the right interface for the caller 84 * to use as a bvec can span multiple folios. You may be better off using 85 * something like bio_for_each_folio_all() which iterates over all folios. 86 */ 87 static inline struct folio *bvec_folio(const struct bio_vec *bv) 88 { 89 return page_folio(bv->bv_page); 90 } 91 92 struct bvec_iter { 93 /* 94 * Current device address in 512 byte sectors. Only updated by the bio 95 * iter wrappers and not the bvec iterator helpers themselves. 96 */ 97 sector_t bi_sector; 98 99 /* 100 * Remaining size in bytes. 101 */ 102 unsigned int bi_size; 103 104 /* 105 * Current index into the bvec array. This indexes into `bi_io_vec` when 106 * iterating a bvec array that is part of a `bio`. 107 */ 108 unsigned int bi_idx; 109 110 /* 111 * Current offset in the bvec entry pointed to by `bi_idx` or into 112 * a dma-buf map. 113 */ 114 unsigned int bi_offset; 115 } __packed __aligned(4); 116 117 struct bvec_iter_all { 118 struct bio_vec bv; 119 int idx; 120 unsigned done; 121 }; 122 123 static __always_inline const struct bio_vec * 124 __bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) 125 { 126 return bvecs + iter.bi_idx; 127 } 128 129 /* multi-page (mp_bvec) helpers */ 130 static __always_inline struct page * 131 mp_bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) 132 { 133 return __bvec_iter_bvec(bvecs, iter)->bv_page; 134 } 135 136 static __always_inline unsigned int 137 mp_bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) 138 { 139 return min(__bvec_iter_bvec(bvecs, iter)->bv_len - iter.bi_offset, 140 iter.bi_size); 141 } 142 143 static __always_inline unsigned int 144 mp_bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) 145 { 146 return __bvec_iter_bvec(bvecs, iter)->bv_offset + iter.bi_offset; 147 } 148 149 static __always_inline unsigned int 150 mp_bvec_iter_page_idx(const struct bio_vec *bvecs, const struct bvec_iter iter) 151 { 152 return mp_bvec_iter_offset(bvecs, iter) / PAGE_SIZE; 153 } 154 155 static __always_inline struct bio_vec 156 mp_bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) 157 { 158 return (struct bio_vec) { 159 .bv_page = mp_bvec_iter_page(bvecs, iter), 160 .bv_len = mp_bvec_iter_len(bvecs, iter), 161 .bv_offset = mp_bvec_iter_offset(bvecs, iter), 162 }; 163 } 164 165 /* For building single-page bvec in flight */ 166 static __always_inline unsigned int 167 bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) 168 { 169 return mp_bvec_iter_offset(bvecs, iter) % PAGE_SIZE; 170 } 171 172 static __always_inline unsigned int 173 bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) 174 { 175 return min(mp_bvec_iter_len(bvecs, iter), 176 PAGE_SIZE - bvec_iter_offset(bvecs, iter)); 177 } 178 179 static __always_inline struct page * 180 bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) 181 { 182 return mp_bvec_iter_page(bvecs, iter) + 183 mp_bvec_iter_page_idx(bvecs, iter); 184 } 185 186 static __always_inline struct bio_vec 187 bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) 188 { 189 return (struct bio_vec) { 190 .bv_page = bvec_iter_page(bvecs, iter), 191 .bv_len = bvec_iter_len(bvecs, iter), 192 .bv_offset = bvec_iter_offset(bvecs, iter), 193 }; 194 } 195 196 static inline bool bvec_iter_advance(const struct bio_vec *bv, 197 struct bvec_iter *iter, unsigned bytes) 198 { 199 unsigned int idx = iter->bi_idx; 200 201 if (WARN_ONCE(bytes > iter->bi_size, 202 "Attempted to advance past end of bvec iter\n")) { 203 iter->bi_size = 0; 204 return false; 205 } 206 207 iter->bi_size -= bytes; 208 bytes += iter->bi_offset; 209 210 while (bytes && bytes >= bv[idx].bv_len) { 211 bytes -= bv[idx].bv_len; 212 idx++; 213 } 214 215 iter->bi_idx = idx; 216 iter->bi_offset = bytes; 217 return true; 218 } 219 220 /* 221 * A simpler version of bvec_iter_advance(), @bytes should not span 222 * across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len 223 */ 224 static inline void bvec_iter_advance_single(const struct bio_vec *bv, 225 struct bvec_iter *iter, unsigned int bytes) 226 { 227 unsigned int done = iter->bi_offset + bytes; 228 229 if (done == bv[iter->bi_idx].bv_len) { 230 done = 0; 231 iter->bi_idx++; 232 } 233 iter->bi_offset = done; 234 iter->bi_size -= bytes; 235 } 236 237 #define for_each_bvec(bvl, bio_vec, iter, start) \ 238 for (iter = (start); \ 239 (iter).bi_size && \ 240 ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \ 241 bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) 242 243 #define for_each_mp_bvec(bvl, bio_vec, iter, start) \ 244 for (iter = (start); \ 245 (iter).bi_size && \ 246 ((bvl = mp_bvec_iter_bvec((bio_vec), (iter))), 1); \ 247 bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) 248 249 static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all) 250 { 251 iter_all->done = 0; 252 iter_all->idx = 0; 253 254 return &iter_all->bv; 255 } 256 257 static inline void bvec_advance(const struct bio_vec *bvec, 258 struct bvec_iter_all *iter_all) 259 { 260 struct bio_vec *bv = &iter_all->bv; 261 262 if (iter_all->done) { 263 bv->bv_page++; 264 bv->bv_offset = 0; 265 } else { 266 bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); 267 bv->bv_offset = bvec->bv_offset & ~PAGE_MASK; 268 } 269 bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset, 270 bvec->bv_len - iter_all->done); 271 iter_all->done += bv->bv_len; 272 273 if (iter_all->done == bvec->bv_len) { 274 iter_all->idx++; 275 iter_all->done = 0; 276 } 277 } 278 279 /** 280 * bvec_kmap_local - map a bvec into the kernel virtual address space 281 * @bvec: bvec to map 282 * 283 * Must be called on single-page bvecs only. Call kunmap_local on the returned 284 * address to unmap. 285 */ 286 static inline void *bvec_kmap_local(struct bio_vec *bvec) 287 { 288 return kmap_local_page(bvec->bv_page) + bvec->bv_offset; 289 } 290 291 /** 292 * memcpy_from_bvec - copy data from a bvec 293 * @to: Kernel virtual address to copy to. 294 * @bvec: bvec to copy from 295 * 296 * Must be called on single-page bvecs only. 297 */ 298 static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec) 299 { 300 memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, bvec->bv_len); 301 } 302 303 /** 304 * memcpy_to_bvec - copy data to a bvec 305 * @bvec: bvec to copy to 306 * @from: Kernel virtual address to copy from. 307 * 308 * Must be called on single-page bvecs only. 309 */ 310 static inline void memcpy_to_bvec(struct bio_vec *bvec, const char *from) 311 { 312 memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, bvec->bv_len); 313 } 314 315 /** 316 * memzero_bvec - zero all data in a bvec 317 * @bvec: bvec to zero 318 * 319 * Must be called on single-page bvecs only. 320 */ 321 static inline void memzero_bvec(struct bio_vec *bvec) 322 { 323 memzero_page(bvec->bv_page, bvec->bv_offset, bvec->bv_len); 324 } 325 326 /** 327 * bvec_virt - return the virtual address for a bvec 328 * @bvec: bvec to return the virtual address for 329 * 330 * Note: the caller must ensure that @bvec->bv_page is not a highmem page. 331 */ 332 static inline void *bvec_virt(struct bio_vec *bvec) 333 { 334 WARN_ON_ONCE(PageHighMem(bvec->bv_page)); 335 return page_address(bvec->bv_page) + bvec->bv_offset; 336 } 337 338 /** 339 * bvec_phys - return the physical address for a bvec 340 * @bvec: bvec to return the physical address for 341 */ 342 static inline phys_addr_t bvec_phys(const struct bio_vec *bvec) 343 { 344 return page_to_phys(bvec->bv_page) + bvec->bv_offset; 345 } 346 347 #endif /* __LINUX_BVEC_H */ 348