1 #include <linux/export.h> 2 #include <linux/uio.h> 3 #include <linux/pagemap.h> 4 #include <linux/slab.h> 5 #include <linux/vmalloc.h> 6 #include <net/checksum.h> 7 8 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ 9 size_t left; \ 10 size_t wanted = n; \ 11 __p = i->iov; \ 12 __v.iov_len = min(n, __p->iov_len - skip); \ 13 if (likely(__v.iov_len)) { \ 14 __v.iov_base = __p->iov_base + skip; \ 15 left = (STEP); \ 16 __v.iov_len -= left; \ 17 skip += __v.iov_len; \ 18 n -= __v.iov_len; \ 19 } else { \ 20 left = 0; \ 21 } \ 22 while (unlikely(!left && n)) { \ 23 __p++; \ 24 __v.iov_len = min(n, __p->iov_len); \ 25 if (unlikely(!__v.iov_len)) \ 26 continue; \ 27 __v.iov_base = __p->iov_base; \ 28 left = (STEP); \ 29 __v.iov_len -= left; \ 30 skip = __v.iov_len; \ 31 n -= __v.iov_len; \ 32 } \ 33 n = wanted - n; \ 34 } 35 36 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ 37 size_t wanted = n; \ 38 __p = i->kvec; \ 39 __v.iov_len = min(n, __p->iov_len - skip); \ 40 if (likely(__v.iov_len)) { \ 41 __v.iov_base = __p->iov_base + skip; \ 42 (void)(STEP); \ 43 skip += __v.iov_len; \ 44 n -= __v.iov_len; \ 45 } \ 46 while (unlikely(n)) { \ 47 __p++; \ 48 __v.iov_len = min(n, __p->iov_len); \ 49 if (unlikely(!__v.iov_len)) \ 50 continue; \ 51 __v.iov_base = __p->iov_base; \ 52 (void)(STEP); \ 53 skip = __v.iov_len; \ 54 n -= __v.iov_len; \ 55 } \ 56 n = wanted; \ 57 } 58 59 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \ 60 size_t wanted = n; \ 61 __p = i->bvec; \ 62 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \ 63 if (likely(__v.bv_len)) { \ 64 __v.bv_page = __p->bv_page; \ 65 __v.bv_offset = __p->bv_offset + skip; \ 66 (void)(STEP); \ 67 skip += __v.bv_len; \ 68 n -= __v.bv_len; \ 69 } \ 70 while (unlikely(n)) { \ 71 __p++; \ 72 __v.bv_len = min_t(size_t, n, __p->bv_len); \ 73 if (unlikely(!__v.bv_len)) \ 74 continue; \ 75 __v.bv_page = __p->bv_page; \ 76 __v.bv_offset = __p->bv_offset; \ 77 (void)(STEP); \ 78 skip = __v.bv_len; \ 79 n -= __v.bv_len; \ 80 } \ 81 n = wanted; \ 82 } 83 84 #define iterate_all_kinds(i, n, v, I, B, K) { \ 85 size_t skip = i->iov_offset; \ 86 if (unlikely(i->type & ITER_BVEC)) { \ 87 const struct bio_vec *bvec; \ 88 struct bio_vec v; \ 89 iterate_bvec(i, n, v, bvec, skip, (B)) \ 90 } else if (unlikely(i->type & ITER_KVEC)) { \ 91 const struct kvec *kvec; \ 92 struct kvec v; \ 93 iterate_kvec(i, n, v, kvec, skip, (K)) \ 94 } else { \ 95 const struct iovec *iov; \ 96 struct iovec v; \ 97 iterate_iovec(i, n, v, iov, skip, (I)) \ 98 } \ 99 } 100 101 #define iterate_and_advance(i, n, v, I, B, K) { \ 102 size_t skip = i->iov_offset; \ 103 if (unlikely(i->type & ITER_BVEC)) { \ 104 const struct bio_vec *bvec; \ 105 struct bio_vec v; \ 106 iterate_bvec(i, n, v, bvec, skip, (B)) \ 107 if (skip == bvec->bv_len) { \ 108 bvec++; \ 109 skip = 0; \ 110 } \ 111 i->nr_segs -= bvec - i->bvec; \ 112 i->bvec = bvec; \ 113 } else if (unlikely(i->type & ITER_KVEC)) { \ 114 const struct kvec *kvec; \ 115 struct kvec v; \ 116 iterate_kvec(i, n, v, kvec, skip, (K)) \ 117 if (skip == kvec->iov_len) { \ 118 kvec++; \ 119 skip = 0; \ 120 } \ 121 i->nr_segs -= kvec - i->kvec; \ 122 i->kvec = kvec; \ 123 } else { \ 124 const struct iovec *iov; \ 125 struct iovec v; \ 126 iterate_iovec(i, n, v, iov, skip, (I)) \ 127 if (skip == iov->iov_len) { \ 128 iov++; \ 129 skip = 0; \ 130 } \ 131 i->nr_segs -= iov - i->iov; \ 132 i->iov = iov; \ 133 } \ 134 i->count -= n; \ 135 i->iov_offset = skip; \ 136 } 137 138 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 139 struct iov_iter *i) 140 { 141 size_t skip, copy, left, wanted; 142 const struct iovec *iov; 143 char __user *buf; 144 void *kaddr, *from; 145 146 if (unlikely(bytes > i->count)) 147 bytes = i->count; 148 149 if (unlikely(!bytes)) 150 return 0; 151 152 wanted = bytes; 153 iov = i->iov; 154 skip = i->iov_offset; 155 buf = iov->iov_base + skip; 156 copy = min(bytes, iov->iov_len - skip); 157 158 if (!fault_in_pages_writeable(buf, copy)) { 159 kaddr = kmap_atomic(page); 160 from = kaddr + offset; 161 162 /* first chunk, usually the only one */ 163 left = __copy_to_user_inatomic(buf, from, copy); 164 copy -= left; 165 skip += copy; 166 from += copy; 167 bytes -= copy; 168 169 while (unlikely(!left && bytes)) { 170 iov++; 171 buf = iov->iov_base; 172 copy = min(bytes, iov->iov_len); 173 left = __copy_to_user_inatomic(buf, from, copy); 174 copy -= left; 175 skip = copy; 176 from += copy; 177 bytes -= copy; 178 } 179 if (likely(!bytes)) { 180 kunmap_atomic(kaddr); 181 goto done; 182 } 183 offset = from - kaddr; 184 buf += copy; 185 kunmap_atomic(kaddr); 186 copy = min(bytes, iov->iov_len - skip); 187 } 188 /* Too bad - revert to non-atomic kmap */ 189 kaddr = kmap(page); 190 from = kaddr + offset; 191 left = __copy_to_user(buf, from, copy); 192 copy -= left; 193 skip += copy; 194 from += copy; 195 bytes -= copy; 196 while (unlikely(!left && bytes)) { 197 iov++; 198 buf = iov->iov_base; 199 copy = min(bytes, iov->iov_len); 200 left = __copy_to_user(buf, from, copy); 201 copy -= left; 202 skip = copy; 203 from += copy; 204 bytes -= copy; 205 } 206 kunmap(page); 207 done: 208 if (skip == iov->iov_len) { 209 iov++; 210 skip = 0; 211 } 212 i->count -= wanted - bytes; 213 i->nr_segs -= iov - i->iov; 214 i->iov = iov; 215 i->iov_offset = skip; 216 return wanted - bytes; 217 } 218 219 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, 220 struct iov_iter *i) 221 { 222 size_t skip, copy, left, wanted; 223 const struct iovec *iov; 224 char __user *buf; 225 void *kaddr, *to; 226 227 if (unlikely(bytes > i->count)) 228 bytes = i->count; 229 230 if (unlikely(!bytes)) 231 return 0; 232 233 wanted = bytes; 234 iov = i->iov; 235 skip = i->iov_offset; 236 buf = iov->iov_base + skip; 237 copy = min(bytes, iov->iov_len - skip); 238 239 if (!fault_in_pages_readable(buf, copy)) { 240 kaddr = kmap_atomic(page); 241 to = kaddr + offset; 242 243 /* first chunk, usually the only one */ 244 left = __copy_from_user_inatomic(to, buf, copy); 245 copy -= left; 246 skip += copy; 247 to += copy; 248 bytes -= copy; 249 250 while (unlikely(!left && bytes)) { 251 iov++; 252 buf = iov->iov_base; 253 copy = min(bytes, iov->iov_len); 254 left = __copy_from_user_inatomic(to, buf, copy); 255 copy -= left; 256 skip = copy; 257 to += copy; 258 bytes -= copy; 259 } 260 if (likely(!bytes)) { 261 kunmap_atomic(kaddr); 262 goto done; 263 } 264 offset = to - kaddr; 265 buf += copy; 266 kunmap_atomic(kaddr); 267 copy = min(bytes, iov->iov_len - skip); 268 } 269 /* Too bad - revert to non-atomic kmap */ 270 kaddr = kmap(page); 271 to = kaddr + offset; 272 left = __copy_from_user(to, buf, copy); 273 copy -= left; 274 skip += copy; 275 to += copy; 276 bytes -= copy; 277 while (unlikely(!left && bytes)) { 278 iov++; 279 buf = iov->iov_base; 280 copy = min(bytes, iov->iov_len); 281 left = __copy_from_user(to, buf, copy); 282 copy -= left; 283 skip = copy; 284 to += copy; 285 bytes -= copy; 286 } 287 kunmap(page); 288 done: 289 if (skip == iov->iov_len) { 290 iov++; 291 skip = 0; 292 } 293 i->count -= wanted - bytes; 294 i->nr_segs -= iov - i->iov; 295 i->iov = iov; 296 i->iov_offset = skip; 297 return wanted - bytes; 298 } 299 300 /* 301 * Fault in the first iovec of the given iov_iter, to a maximum length 302 * of bytes. Returns 0 on success, or non-zero if the memory could not be 303 * accessed (ie. because it is an invalid address). 304 * 305 * writev-intensive code may want this to prefault several iovecs -- that 306 * would be possible (callers must not rely on the fact that _only_ the 307 * first iovec will be faulted with the current implementation). 308 */ 309 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 310 { 311 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 312 char __user *buf = i->iov->iov_base + i->iov_offset; 313 bytes = min(bytes, i->iov->iov_len - i->iov_offset); 314 return fault_in_pages_readable(buf, bytes); 315 } 316 return 0; 317 } 318 EXPORT_SYMBOL(iov_iter_fault_in_readable); 319 320 /* 321 * Fault in one or more iovecs of the given iov_iter, to a maximum length of 322 * bytes. For each iovec, fault in each page that constitutes the iovec. 323 * 324 * Return 0 on success, or non-zero if the memory could not be accessed (i.e. 325 * because it is an invalid address). 326 */ 327 int iov_iter_fault_in_multipages_readable(struct iov_iter *i, size_t bytes) 328 { 329 size_t skip = i->iov_offset; 330 const struct iovec *iov; 331 int err; 332 struct iovec v; 333 334 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 335 iterate_iovec(i, bytes, v, iov, skip, ({ 336 err = fault_in_multipages_readable(v.iov_base, 337 v.iov_len); 338 if (unlikely(err)) 339 return err; 340 0;})) 341 } 342 return 0; 343 } 344 EXPORT_SYMBOL(iov_iter_fault_in_multipages_readable); 345 346 void iov_iter_init(struct iov_iter *i, int direction, 347 const struct iovec *iov, unsigned long nr_segs, 348 size_t count) 349 { 350 /* It will get better. Eventually... */ 351 if (segment_eq(get_fs(), KERNEL_DS)) { 352 direction |= ITER_KVEC; 353 i->type = direction; 354 i->kvec = (struct kvec *)iov; 355 } else { 356 i->type = direction; 357 i->iov = iov; 358 } 359 i->nr_segs = nr_segs; 360 i->iov_offset = 0; 361 i->count = count; 362 } 363 EXPORT_SYMBOL(iov_iter_init); 364 365 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) 366 { 367 char *from = kmap_atomic(page); 368 memcpy(to, from + offset, len); 369 kunmap_atomic(from); 370 } 371 372 static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len) 373 { 374 char *to = kmap_atomic(page); 375 memcpy(to + offset, from, len); 376 kunmap_atomic(to); 377 } 378 379 static void memzero_page(struct page *page, size_t offset, size_t len) 380 { 381 char *addr = kmap_atomic(page); 382 memset(addr + offset, 0, len); 383 kunmap_atomic(addr); 384 } 385 386 size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i) 387 { 388 char *from = addr; 389 if (unlikely(bytes > i->count)) 390 bytes = i->count; 391 392 if (unlikely(!bytes)) 393 return 0; 394 395 iterate_and_advance(i, bytes, v, 396 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len, 397 v.iov_len), 398 memcpy_to_page(v.bv_page, v.bv_offset, 399 (from += v.bv_len) - v.bv_len, v.bv_len), 400 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) 401 ) 402 403 return bytes; 404 } 405 EXPORT_SYMBOL(copy_to_iter); 406 407 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 408 { 409 char *to = addr; 410 if (unlikely(bytes > i->count)) 411 bytes = i->count; 412 413 if (unlikely(!bytes)) 414 return 0; 415 416 iterate_and_advance(i, bytes, v, 417 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base, 418 v.iov_len), 419 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 420 v.bv_offset, v.bv_len), 421 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 422 ) 423 424 return bytes; 425 } 426 EXPORT_SYMBOL(copy_from_iter); 427 428 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 429 { 430 char *to = addr; 431 if (unlikely(bytes > i->count)) 432 bytes = i->count; 433 434 if (unlikely(!bytes)) 435 return 0; 436 437 iterate_and_advance(i, bytes, v, 438 __copy_from_user_nocache((to += v.iov_len) - v.iov_len, 439 v.iov_base, v.iov_len), 440 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 441 v.bv_offset, v.bv_len), 442 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 443 ) 444 445 return bytes; 446 } 447 EXPORT_SYMBOL(copy_from_iter_nocache); 448 449 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 450 struct iov_iter *i) 451 { 452 if (i->type & (ITER_BVEC|ITER_KVEC)) { 453 void *kaddr = kmap_atomic(page); 454 size_t wanted = copy_to_iter(kaddr + offset, bytes, i); 455 kunmap_atomic(kaddr); 456 return wanted; 457 } else 458 return copy_page_to_iter_iovec(page, offset, bytes, i); 459 } 460 EXPORT_SYMBOL(copy_page_to_iter); 461 462 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 463 struct iov_iter *i) 464 { 465 if (i->type & (ITER_BVEC|ITER_KVEC)) { 466 void *kaddr = kmap_atomic(page); 467 size_t wanted = copy_from_iter(kaddr + offset, bytes, i); 468 kunmap_atomic(kaddr); 469 return wanted; 470 } else 471 return copy_page_from_iter_iovec(page, offset, bytes, i); 472 } 473 EXPORT_SYMBOL(copy_page_from_iter); 474 475 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 476 { 477 if (unlikely(bytes > i->count)) 478 bytes = i->count; 479 480 if (unlikely(!bytes)) 481 return 0; 482 483 iterate_and_advance(i, bytes, v, 484 __clear_user(v.iov_base, v.iov_len), 485 memzero_page(v.bv_page, v.bv_offset, v.bv_len), 486 memset(v.iov_base, 0, v.iov_len) 487 ) 488 489 return bytes; 490 } 491 EXPORT_SYMBOL(iov_iter_zero); 492 493 size_t iov_iter_copy_from_user_atomic(struct page *page, 494 struct iov_iter *i, unsigned long offset, size_t bytes) 495 { 496 char *kaddr = kmap_atomic(page), *p = kaddr + offset; 497 iterate_all_kinds(i, bytes, v, 498 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len, 499 v.iov_base, v.iov_len), 500 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, 501 v.bv_offset, v.bv_len), 502 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 503 ) 504 kunmap_atomic(kaddr); 505 return bytes; 506 } 507 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 508 509 void iov_iter_advance(struct iov_iter *i, size_t size) 510 { 511 iterate_and_advance(i, size, v, 0, 0, 0) 512 } 513 EXPORT_SYMBOL(iov_iter_advance); 514 515 /* 516 * Return the count of just the current iov_iter segment. 517 */ 518 size_t iov_iter_single_seg_count(const struct iov_iter *i) 519 { 520 if (i->nr_segs == 1) 521 return i->count; 522 else if (i->type & ITER_BVEC) 523 return min(i->count, i->bvec->bv_len - i->iov_offset); 524 else 525 return min(i->count, i->iov->iov_len - i->iov_offset); 526 } 527 EXPORT_SYMBOL(iov_iter_single_seg_count); 528 529 void iov_iter_kvec(struct iov_iter *i, int direction, 530 const struct kvec *kvec, unsigned long nr_segs, 531 size_t count) 532 { 533 BUG_ON(!(direction & ITER_KVEC)); 534 i->type = direction; 535 i->kvec = kvec; 536 i->nr_segs = nr_segs; 537 i->iov_offset = 0; 538 i->count = count; 539 } 540 EXPORT_SYMBOL(iov_iter_kvec); 541 542 void iov_iter_bvec(struct iov_iter *i, int direction, 543 const struct bio_vec *bvec, unsigned long nr_segs, 544 size_t count) 545 { 546 BUG_ON(!(direction & ITER_BVEC)); 547 i->type = direction; 548 i->bvec = bvec; 549 i->nr_segs = nr_segs; 550 i->iov_offset = 0; 551 i->count = count; 552 } 553 EXPORT_SYMBOL(iov_iter_bvec); 554 555 unsigned long iov_iter_alignment(const struct iov_iter *i) 556 { 557 unsigned long res = 0; 558 size_t size = i->count; 559 560 if (!size) 561 return 0; 562 563 iterate_all_kinds(i, size, v, 564 (res |= (unsigned long)v.iov_base | v.iov_len, 0), 565 res |= v.bv_offset | v.bv_len, 566 res |= (unsigned long)v.iov_base | v.iov_len 567 ) 568 return res; 569 } 570 EXPORT_SYMBOL(iov_iter_alignment); 571 572 ssize_t iov_iter_get_pages(struct iov_iter *i, 573 struct page **pages, size_t maxsize, unsigned maxpages, 574 size_t *start) 575 { 576 if (maxsize > i->count) 577 maxsize = i->count; 578 579 if (!maxsize) 580 return 0; 581 582 iterate_all_kinds(i, maxsize, v, ({ 583 unsigned long addr = (unsigned long)v.iov_base; 584 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 585 int n; 586 int res; 587 588 if (len > maxpages * PAGE_SIZE) 589 len = maxpages * PAGE_SIZE; 590 addr &= ~(PAGE_SIZE - 1); 591 n = DIV_ROUND_UP(len, PAGE_SIZE); 592 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); 593 if (unlikely(res < 0)) 594 return res; 595 return (res == n ? len : res * PAGE_SIZE) - *start; 596 0;}),({ 597 /* can't be more than PAGE_SIZE */ 598 *start = v.bv_offset; 599 get_page(*pages = v.bv_page); 600 return v.bv_len; 601 }),({ 602 return -EFAULT; 603 }) 604 ) 605 return 0; 606 } 607 EXPORT_SYMBOL(iov_iter_get_pages); 608 609 static struct page **get_pages_array(size_t n) 610 { 611 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL); 612 if (!p) 613 p = vmalloc(n * sizeof(struct page *)); 614 return p; 615 } 616 617 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, 618 struct page ***pages, size_t maxsize, 619 size_t *start) 620 { 621 struct page **p; 622 623 if (maxsize > i->count) 624 maxsize = i->count; 625 626 if (!maxsize) 627 return 0; 628 629 iterate_all_kinds(i, maxsize, v, ({ 630 unsigned long addr = (unsigned long)v.iov_base; 631 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 632 int n; 633 int res; 634 635 addr &= ~(PAGE_SIZE - 1); 636 n = DIV_ROUND_UP(len, PAGE_SIZE); 637 p = get_pages_array(n); 638 if (!p) 639 return -ENOMEM; 640 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); 641 if (unlikely(res < 0)) { 642 kvfree(p); 643 return res; 644 } 645 *pages = p; 646 return (res == n ? len : res * PAGE_SIZE) - *start; 647 0;}),({ 648 /* can't be more than PAGE_SIZE */ 649 *start = v.bv_offset; 650 *pages = p = get_pages_array(1); 651 if (!p) 652 return -ENOMEM; 653 get_page(*p = v.bv_page); 654 return v.bv_len; 655 }),({ 656 return -EFAULT; 657 }) 658 ) 659 return 0; 660 } 661 EXPORT_SYMBOL(iov_iter_get_pages_alloc); 662 663 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, 664 struct iov_iter *i) 665 { 666 char *to = addr; 667 __wsum sum, next; 668 size_t off = 0; 669 if (unlikely(bytes > i->count)) 670 bytes = i->count; 671 672 if (unlikely(!bytes)) 673 return 0; 674 675 sum = *csum; 676 iterate_and_advance(i, bytes, v, ({ 677 int err = 0; 678 next = csum_and_copy_from_user(v.iov_base, 679 (to += v.iov_len) - v.iov_len, 680 v.iov_len, 0, &err); 681 if (!err) { 682 sum = csum_block_add(sum, next, off); 683 off += v.iov_len; 684 } 685 err ? v.iov_len : 0; 686 }), ({ 687 char *p = kmap_atomic(v.bv_page); 688 next = csum_partial_copy_nocheck(p + v.bv_offset, 689 (to += v.bv_len) - v.bv_len, 690 v.bv_len, 0); 691 kunmap_atomic(p); 692 sum = csum_block_add(sum, next, off); 693 off += v.bv_len; 694 }),({ 695 next = csum_partial_copy_nocheck(v.iov_base, 696 (to += v.iov_len) - v.iov_len, 697 v.iov_len, 0); 698 sum = csum_block_add(sum, next, off); 699 off += v.iov_len; 700 }) 701 ) 702 *csum = sum; 703 return bytes; 704 } 705 EXPORT_SYMBOL(csum_and_copy_from_iter); 706 707 size_t csum_and_copy_to_iter(void *addr, size_t bytes, __wsum *csum, 708 struct iov_iter *i) 709 { 710 char *from = addr; 711 __wsum sum, next; 712 size_t off = 0; 713 if (unlikely(bytes > i->count)) 714 bytes = i->count; 715 716 if (unlikely(!bytes)) 717 return 0; 718 719 sum = *csum; 720 iterate_and_advance(i, bytes, v, ({ 721 int err = 0; 722 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, 723 v.iov_base, 724 v.iov_len, 0, &err); 725 if (!err) { 726 sum = csum_block_add(sum, next, off); 727 off += v.iov_len; 728 } 729 err ? v.iov_len : 0; 730 }), ({ 731 char *p = kmap_atomic(v.bv_page); 732 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len, 733 p + v.bv_offset, 734 v.bv_len, 0); 735 kunmap_atomic(p); 736 sum = csum_block_add(sum, next, off); 737 off += v.bv_len; 738 }),({ 739 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len, 740 v.iov_base, 741 v.iov_len, 0); 742 sum = csum_block_add(sum, next, off); 743 off += v.iov_len; 744 }) 745 ) 746 *csum = sum; 747 return bytes; 748 } 749 EXPORT_SYMBOL(csum_and_copy_to_iter); 750 751 int iov_iter_npages(const struct iov_iter *i, int maxpages) 752 { 753 size_t size = i->count; 754 int npages = 0; 755 756 if (!size) 757 return 0; 758 759 iterate_all_kinds(i, size, v, ({ 760 unsigned long p = (unsigned long)v.iov_base; 761 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 762 - p / PAGE_SIZE; 763 if (npages >= maxpages) 764 return maxpages; 765 0;}),({ 766 npages++; 767 if (npages >= maxpages) 768 return maxpages; 769 }),({ 770 unsigned long p = (unsigned long)v.iov_base; 771 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 772 - p / PAGE_SIZE; 773 if (npages >= maxpages) 774 return maxpages; 775 }) 776 ) 777 return npages; 778 } 779 EXPORT_SYMBOL(iov_iter_npages); 780 781 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 782 { 783 *new = *old; 784 if (new->type & ITER_BVEC) 785 return new->bvec = kmemdup(new->bvec, 786 new->nr_segs * sizeof(struct bio_vec), 787 flags); 788 else 789 /* iovec and kvec have identical layout */ 790 return new->iov = kmemdup(new->iov, 791 new->nr_segs * sizeof(struct iovec), 792 flags); 793 } 794 EXPORT_SYMBOL(dup_iter); 795 796 int import_iovec(int type, const struct iovec __user * uvector, 797 unsigned nr_segs, unsigned fast_segs, 798 struct iovec **iov, struct iov_iter *i) 799 { 800 ssize_t n; 801 struct iovec *p; 802 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 803 *iov, &p); 804 if (n < 0) { 805 if (p != *iov) 806 kfree(p); 807 *iov = NULL; 808 return n; 809 } 810 iov_iter_init(i, type, p, nr_segs, n); 811 *iov = p == *iov ? NULL : p; 812 return 0; 813 } 814 EXPORT_SYMBOL(import_iovec); 815 816 #ifdef CONFIG_COMPAT 817 #include <linux/compat.h> 818 819 int compat_import_iovec(int type, const struct compat_iovec __user * uvector, 820 unsigned nr_segs, unsigned fast_segs, 821 struct iovec **iov, struct iov_iter *i) 822 { 823 ssize_t n; 824 struct iovec *p; 825 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 826 *iov, &p); 827 if (n < 0) { 828 if (p != *iov) 829 kfree(p); 830 *iov = NULL; 831 return n; 832 } 833 iov_iter_init(i, type, p, nr_segs, n); 834 *iov = p == *iov ? NULL : p; 835 return 0; 836 } 837 #endif 838 839 int import_single_range(int rw, void __user *buf, size_t len, 840 struct iovec *iov, struct iov_iter *i) 841 { 842 if (len > MAX_RW_COUNT) 843 len = MAX_RW_COUNT; 844 if (unlikely(!access_ok(!rw, buf, len))) 845 return -EFAULT; 846 847 iov->iov_base = buf; 848 iov->iov_len = len; 849 iov_iter_init(i, rw, iov, 1, len); 850 return 0; 851 } 852