Lines Matching refs:i
102 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) in fault_in_iov_iter_readable() argument
104 if (iter_is_ubuf(i)) { in fault_in_iov_iter_readable()
105 size_t n = min(size, iov_iter_count(i)); in fault_in_iov_iter_readable()
106 n -= fault_in_readable(i->ubuf + i->iov_offset, n); in fault_in_iov_iter_readable()
108 } else if (iter_is_iovec(i)) { in fault_in_iov_iter_readable()
109 size_t count = min(size, iov_iter_count(i)); in fault_in_iov_iter_readable()
114 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) { in fault_in_iov_iter_readable()
145 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) in fault_in_iov_iter_writeable() argument
147 if (iter_is_ubuf(i)) { in fault_in_iov_iter_writeable()
148 size_t n = min(size, iov_iter_count(i)); in fault_in_iov_iter_writeable()
149 n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n); in fault_in_iov_iter_writeable()
151 } else if (iter_is_iovec(i)) { in fault_in_iov_iter_writeable()
152 size_t count = min(size, iov_iter_count(i)); in fault_in_iov_iter_writeable()
157 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) { in fault_in_iov_iter_writeable()
174 void iov_iter_init(struct iov_iter *i, unsigned int direction, in iov_iter_init() argument
179 *i = (struct iov_iter) { in iov_iter_init()
191 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_to_iter() argument
193 if (WARN_ON_ONCE(i->data_source)) in _copy_to_iter()
195 if (user_backed_iter(i)) in _copy_to_iter()
197 return iterate_and_advance(i, bytes, (void *)addr, in _copy_to_iter()
246 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_mc_to_iter() argument
248 if (WARN_ON_ONCE(i->data_source)) in _copy_mc_to_iter()
250 if (user_backed_iter(i)) in _copy_mc_to_iter()
252 return iterate_and_advance(i, bytes, (void *)addr, in _copy_mc_to_iter()
259 size_t __copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) in __copy_from_iter() argument
261 return iterate_and_advance(i, bytes, addr, in __copy_from_iter()
265 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter() argument
267 if (WARN_ON_ONCE(!i->data_source)) in _copy_from_iter()
270 if (user_backed_iter(i)) in _copy_from_iter()
272 return __copy_from_iter(addr, bytes, i); in _copy_from_iter()
283 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_nocache() argument
285 if (WARN_ON_ONCE(!i->data_source)) in _copy_from_iter_nocache()
288 return iterate_and_advance(i, bytes, addr, in _copy_from_iter_nocache()
326 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_flushcache() argument
328 if (WARN_ON_ONCE(!i->data_source)) in _copy_from_iter_flushcache()
331 return iterate_and_advance(i, bytes, addr, in _copy_from_iter_flushcache()
362 struct iov_iter *i) in copy_page_to_iter() argument
367 if (WARN_ON_ONCE(i->data_source)) in copy_page_to_iter()
374 n = _copy_to_iter(kaddr + offset, n, i); in copy_page_to_iter()
391 struct iov_iter *i) in copy_page_to_iter_nofault() argument
397 if (WARN_ON_ONCE(i->data_source)) in copy_page_to_iter_nofault()
405 n = iterate_and_advance(i, n, kaddr + offset, in copy_page_to_iter_nofault()
424 struct iov_iter *i) in copy_page_from_iter() argument
434 n = _copy_from_iter(kaddr + offset, n, i); in copy_page_from_iter()
465 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) in iov_iter_zero() argument
467 return iterate_and_advance(i, bytes, NULL, in iov_iter_zero()
473 size_t bytes, struct iov_iter *i) in copy_folio_from_iter_atomic() argument
479 if (WARN_ON_ONCE(!i->data_source)) in copy_folio_from_iter_atomic()
491 n = __copy_from_iter(to, n, i); in copy_folio_from_iter_atomic()
502 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) in iov_iter_bvec_advance() argument
506 if (!i->count) in iov_iter_bvec_advance()
508 i->count -= size; in iov_iter_bvec_advance()
510 size += i->iov_offset; in iov_iter_bvec_advance()
512 for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) { in iov_iter_bvec_advance()
517 i->iov_offset = size; in iov_iter_bvec_advance()
518 i->nr_segs -= bvec - i->bvec; in iov_iter_bvec_advance()
519 i->bvec = bvec; in iov_iter_bvec_advance()
522 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) in iov_iter_iovec_advance() argument
526 if (!i->count) in iov_iter_iovec_advance()
528 i->count -= size; in iov_iter_iovec_advance()
530 size += i->iov_offset; // from beginning of current segment in iov_iter_iovec_advance()
531 for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) { in iov_iter_iovec_advance()
536 i->iov_offset = size; in iov_iter_iovec_advance()
537 i->nr_segs -= iov - iter_iov(i); in iov_iter_iovec_advance()
538 i->__iov = iov; in iov_iter_iovec_advance()
541 static void iov_iter_folioq_advance(struct iov_iter *i, size_t size) in iov_iter_folioq_advance() argument
543 const struct folio_queue *folioq = i->folioq; in iov_iter_folioq_advance()
544 unsigned int slot = i->folioq_slot; in iov_iter_folioq_advance()
546 if (!i->count) in iov_iter_folioq_advance()
548 i->count -= size; in iov_iter_folioq_advance()
555 size += i->iov_offset; /* From beginning of current segment. */ in iov_iter_folioq_advance()
569 i->iov_offset = size; in iov_iter_folioq_advance()
570 i->folioq_slot = slot; in iov_iter_folioq_advance()
571 i->folioq = folioq; in iov_iter_folioq_advance()
574 void iov_iter_advance(struct iov_iter *i, size_t size) in iov_iter_advance() argument
576 if (unlikely(i->count < size)) in iov_iter_advance()
577 size = i->count; in iov_iter_advance()
578 if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) { in iov_iter_advance()
579 i->iov_offset += size; in iov_iter_advance()
580 i->count -= size; in iov_iter_advance()
581 } else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { in iov_iter_advance()
583 iov_iter_iovec_advance(i, size); in iov_iter_advance()
584 } else if (iov_iter_is_bvec(i)) { in iov_iter_advance()
585 iov_iter_bvec_advance(i, size); in iov_iter_advance()
586 } else if (iov_iter_is_folioq(i)) { in iov_iter_advance()
587 iov_iter_folioq_advance(i, size); in iov_iter_advance()
588 } else if (iov_iter_is_discard(i)) { in iov_iter_advance()
589 i->count -= size; in iov_iter_advance()
594 static void iov_iter_folioq_revert(struct iov_iter *i, size_t unroll) in iov_iter_folioq_revert() argument
596 const struct folio_queue *folioq = i->folioq; in iov_iter_folioq_revert()
597 unsigned int slot = i->folioq_slot; in iov_iter_folioq_revert()
610 i->iov_offset = fsize - unroll; in iov_iter_folioq_revert()
616 i->folioq_slot = slot; in iov_iter_folioq_revert()
617 i->folioq = folioq; in iov_iter_folioq_revert()
620 void iov_iter_revert(struct iov_iter *i, size_t unroll) in iov_iter_revert() argument
626 i->count += unroll; in iov_iter_revert()
627 if (unlikely(iov_iter_is_discard(i))) in iov_iter_revert()
629 if (unroll <= i->iov_offset) { in iov_iter_revert()
630 i->iov_offset -= unroll; in iov_iter_revert()
633 unroll -= i->iov_offset; in iov_iter_revert()
634 if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) { in iov_iter_revert()
639 } else if (iov_iter_is_bvec(i)) { in iov_iter_revert()
640 const struct bio_vec *bvec = i->bvec; in iov_iter_revert()
643 i->nr_segs++; in iov_iter_revert()
645 i->bvec = bvec; in iov_iter_revert()
646 i->iov_offset = n - unroll; in iov_iter_revert()
651 } else if (iov_iter_is_folioq(i)) { in iov_iter_revert()
652 i->iov_offset = 0; in iov_iter_revert()
653 iov_iter_folioq_revert(i, unroll); in iov_iter_revert()
655 const struct iovec *iov = iter_iov(i); in iov_iter_revert()
658 i->nr_segs++; in iov_iter_revert()
660 i->__iov = iov; in iov_iter_revert()
661 i->iov_offset = n - unroll; in iov_iter_revert()
673 size_t iov_iter_single_seg_count(const struct iov_iter *i) in iov_iter_single_seg_count() argument
675 if (i->nr_segs > 1) { in iov_iter_single_seg_count()
676 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_single_seg_count()
677 return min(i->count, iter_iov(i)->iov_len - i->iov_offset); in iov_iter_single_seg_count()
678 if (iov_iter_is_bvec(i)) in iov_iter_single_seg_count()
679 return min(i->count, i->bvec->bv_len - i->iov_offset); in iov_iter_single_seg_count()
681 if (unlikely(iov_iter_is_folioq(i))) in iov_iter_single_seg_count()
682 return !i->count ? 0 : in iov_iter_single_seg_count()
683 umin(folioq_folio_size(i->folioq, i->folioq_slot), i->count); in iov_iter_single_seg_count()
684 return i->count; in iov_iter_single_seg_count()
688 void iov_iter_kvec(struct iov_iter *i, unsigned int direction, in iov_iter_kvec() argument
693 *i = (struct iov_iter){ in iov_iter_kvec()
704 void iov_iter_bvec(struct iov_iter *i, unsigned int direction, in iov_iter_bvec() argument
709 *i = (struct iov_iter){ in iov_iter_bvec()
734 void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction, in iov_iter_folio_queue() argument
739 *i = (struct iov_iter) { in iov_iter_folio_queue()
763 void iov_iter_xarray(struct iov_iter *i, unsigned int direction, in iov_iter_xarray() argument
767 *i = (struct iov_iter) { in iov_iter_xarray()
787 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) in iov_iter_discard() argument
790 *i = (struct iov_iter){ in iov_iter_discard()
799 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) in iov_iter_alignment_iovec() argument
801 const struct iovec *iov = iter_iov(i); in iov_iter_alignment_iovec()
803 size_t size = i->count; in iov_iter_alignment_iovec()
804 size_t skip = i->iov_offset; in iov_iter_alignment_iovec()
821 static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) in iov_iter_alignment_bvec() argument
823 const struct bio_vec *bvec = i->bvec; in iov_iter_alignment_bvec()
825 size_t size = i->count; in iov_iter_alignment_bvec()
826 unsigned skip = i->iov_offset; in iov_iter_alignment_bvec()
842 unsigned long iov_iter_alignment(const struct iov_iter *i) in iov_iter_alignment() argument
844 if (likely(iter_is_ubuf(i))) { in iov_iter_alignment()
845 size_t size = i->count; in iov_iter_alignment()
847 return ((unsigned long)i->ubuf + i->iov_offset) | size; in iov_iter_alignment()
852 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_alignment()
853 return iov_iter_alignment_iovec(i); in iov_iter_alignment()
855 if (iov_iter_is_bvec(i)) in iov_iter_alignment()
856 return iov_iter_alignment_bvec(i); in iov_iter_alignment()
859 if (iov_iter_is_folioq(i)) in iov_iter_alignment()
860 return i->iov_offset | i->count; in iov_iter_alignment()
861 if (iov_iter_is_xarray(i)) in iov_iter_alignment()
862 return (i->xarray_start + i->iov_offset) | i->count; in iov_iter_alignment()
868 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) in iov_iter_gap_alignment() argument
872 size_t size = i->count; in iov_iter_gap_alignment()
875 if (iter_is_ubuf(i)) in iov_iter_gap_alignment()
878 if (WARN_ON(!iter_is_iovec(i))) in iov_iter_gap_alignment()
881 for (k = 0; k < i->nr_segs; k++) { in iov_iter_gap_alignment()
882 const struct iovec *iov = iter_iov(i) + k; in iov_iter_gap_alignment()
999 static ssize_t iter_xarray_get_pages(struct iov_iter *i, in iter_xarray_get_pages() argument
1007 pos = i->xarray_start + i->iov_offset; in iter_xarray_get_pages()
1015 nr = iter_xarray_populate_pages(*pages, i->xarray, index, count); in iter_xarray_get_pages()
1020 i->iov_offset += maxsize; in iter_xarray_get_pages()
1021 i->count -= maxsize; in iter_xarray_get_pages()
1026 static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size) in first_iovec_segment() argument
1031 if (iter_is_ubuf(i)) in first_iovec_segment()
1032 return (unsigned long)i->ubuf + i->iov_offset; in first_iovec_segment()
1034 for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { in first_iovec_segment()
1035 const struct iovec *iov = iter_iov(i) + k; in first_iovec_segment()
1048 static struct page *first_bvec_segment(const struct iov_iter *i, in first_bvec_segment() argument
1052 size_t skip = i->iov_offset, len; in first_bvec_segment()
1054 len = i->bvec->bv_len - skip; in first_bvec_segment()
1057 skip += i->bvec->bv_offset; in first_bvec_segment()
1058 page = i->bvec->bv_page + skip / PAGE_SIZE; in first_bvec_segment()
1063 static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i, in __iov_iter_get_pages_alloc() argument
1069 if (maxsize > i->count) in __iov_iter_get_pages_alloc()
1070 maxsize = i->count; in __iov_iter_get_pages_alloc()
1076 if (likely(user_backed_iter(i))) { in __iov_iter_get_pages_alloc()
1080 if (iov_iter_rw(i) != WRITE) in __iov_iter_get_pages_alloc()
1082 if (i->nofault) in __iov_iter_get_pages_alloc()
1085 addr = first_iovec_segment(i, &maxsize); in __iov_iter_get_pages_alloc()
1095 iov_iter_advance(i, maxsize); in __iov_iter_get_pages_alloc()
1098 if (iov_iter_is_bvec(i)) { in __iov_iter_get_pages_alloc()
1102 page = first_bvec_segment(i, &maxsize, start); in __iov_iter_get_pages_alloc()
1114 i->count -= maxsize; in __iov_iter_get_pages_alloc()
1115 i->iov_offset += maxsize; in __iov_iter_get_pages_alloc()
1116 if (i->iov_offset == i->bvec->bv_len) { in __iov_iter_get_pages_alloc()
1117 i->iov_offset = 0; in __iov_iter_get_pages_alloc()
1118 i->bvec++; in __iov_iter_get_pages_alloc()
1119 i->nr_segs--; in __iov_iter_get_pages_alloc()
1123 if (iov_iter_is_folioq(i)) in __iov_iter_get_pages_alloc()
1124 return iter_folioq_get_pages(i, pages, maxsize, maxpages, start); in __iov_iter_get_pages_alloc()
1125 if (iov_iter_is_xarray(i)) in __iov_iter_get_pages_alloc()
1126 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); in __iov_iter_get_pages_alloc()
1130 ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages, in iov_iter_get_pages2() argument
1137 return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start); in iov_iter_get_pages2()
1141 ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, in iov_iter_get_pages_alloc2() argument
1148 len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start); in iov_iter_get_pages_alloc2()
1157 static int iov_npages(const struct iov_iter *i, int maxpages) in iov_npages() argument
1159 size_t skip = i->iov_offset, size = i->count; in iov_npages()
1163 for (p = iter_iov(i); size; skip = 0, p++) { in iov_npages()
1177 static int bvec_npages(const struct iov_iter *i, int maxpages) in bvec_npages() argument
1179 size_t skip = i->iov_offset, size = i->count; in bvec_npages()
1183 for (p = i->bvec; size; skip = 0, p++) { in bvec_npages()
1195 int iov_iter_npages(const struct iov_iter *i, int maxpages) in iov_iter_npages() argument
1197 if (unlikely(!i->count)) in iov_iter_npages()
1199 if (likely(iter_is_ubuf(i))) { in iov_iter_npages()
1200 unsigned offs = offset_in_page(i->ubuf + i->iov_offset); in iov_iter_npages()
1201 int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE); in iov_iter_npages()
1205 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_npages()
1206 return iov_npages(i, maxpages); in iov_iter_npages()
1207 if (iov_iter_is_bvec(i)) in iov_iter_npages()
1208 return bvec_npages(i, maxpages); in iov_iter_npages()
1209 if (iov_iter_is_folioq(i)) { in iov_iter_npages()
1210 unsigned offset = i->iov_offset % PAGE_SIZE; in iov_iter_npages()
1211 int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); in iov_iter_npages()
1214 if (iov_iter_is_xarray(i)) { in iov_iter_npages()
1215 unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; in iov_iter_npages()
1216 int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); in iov_iter_npages()
1245 u32 i; in copy_compat_iovec_from_user() local
1250 for (i = 0; i < nr_segs; i++) { in copy_compat_iovec_from_user()
1254 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end); in copy_compat_iovec_from_user()
1255 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end); in copy_compat_iovec_from_user()
1262 iov[i].iov_base = compat_ptr(buf); in copy_compat_iovec_from_user()
1263 iov[i].iov_len = len; in copy_compat_iovec_from_user()
1343 struct iovec **iovp, struct iov_iter *i, in __import_iovec_ubuf() argument
1358 ret = import_ubuf(type, iov->iov_base, iov->iov_len, i); in __import_iovec_ubuf()
1361 return i->count; in __import_iovec_ubuf()
1366 struct iov_iter *i, bool compat) in __import_iovec() argument
1373 return __import_iovec_ubuf(type, uvec, iovp, i, compat); in __import_iovec()
1406 iov_iter_init(i, type, iov, nr_segs, total_len); in __import_iovec()
1438 struct iovec **iovp, struct iov_iter *i) in import_iovec() argument
1440 return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, in import_iovec()
1445 int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) in import_ubuf() argument
1452 iov_iter_ubuf(i, rw, buf, len); in import_ubuf()
1469 void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) in iov_iter_restore() argument
1471 if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) && in iov_iter_restore()
1472 !iter_is_ubuf(i)) && !iov_iter_is_kvec(i)) in iov_iter_restore()
1474 i->iov_offset = state->iov_offset; in iov_iter_restore()
1475 i->count = state->count; in iov_iter_restore()
1476 if (iter_is_ubuf(i)) in iov_iter_restore()
1488 if (iov_iter_is_bvec(i)) in iov_iter_restore()
1489 i->bvec -= state->nr_segs - i->nr_segs; in iov_iter_restore()
1491 i->__iov -= state->nr_segs - i->nr_segs; in iov_iter_restore()
1492 i->nr_segs = state->nr_segs; in iov_iter_restore()
1500 static ssize_t iov_iter_extract_folioq_pages(struct iov_iter *i, in iov_iter_extract_folioq_pages() argument
1506 const struct folio_queue *folioq = i->folioq; in iov_iter_extract_folioq_pages()
1509 size_t extracted = 0, offset, slot = i->folioq_slot; in iov_iter_extract_folioq_pages()
1514 if (WARN_ON(i->iov_offset != 0)) in iov_iter_extract_folioq_pages()
1518 offset = i->iov_offset & ~PAGE_MASK; in iov_iter_extract_folioq_pages()
1528 size_t offset = i->iov_offset, fsize = folioq_folio_size(folioq, slot); in iov_iter_extract_folioq_pages()
1533 i->count -= part; in iov_iter_extract_folioq_pages()
1534 i->iov_offset += part; in iov_iter_extract_folioq_pages()
1543 if (i->iov_offset >= fsize) { in iov_iter_extract_folioq_pages()
1544 i->iov_offset = 0; in iov_iter_extract_folioq_pages()
1553 i->folioq = folioq; in iov_iter_extract_folioq_pages()
1554 i->folioq_slot = slot; in iov_iter_extract_folioq_pages()
1562 static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, in iov_iter_extract_xarray_pages() argument
1571 loff_t pos = i->xarray_start + i->iov_offset; in iov_iter_extract_xarray_pages()
1573 XA_STATE(xas, i->xarray, pos >> PAGE_SHIFT); in iov_iter_extract_xarray_pages()
1609 iov_iter_advance(i, maxsize); in iov_iter_extract_xarray_pages()
1617 static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, in iov_iter_extract_bvec_pages() argument
1623 size_t skip = i->iov_offset, size = 0; in iov_iter_extract_bvec_pages()
1627 if (i->nr_segs == 0) in iov_iter_extract_bvec_pages()
1630 if (i->iov_offset == i->bvec->bv_len) { in iov_iter_extract_bvec_pages()
1631 i->iov_offset = 0; in iov_iter_extract_bvec_pages()
1632 i->nr_segs--; in iov_iter_extract_bvec_pages()
1633 i->bvec++; in iov_iter_extract_bvec_pages()
1644 while (bi.bi_size && bi.bi_idx < i->nr_segs) { in iov_iter_extract_bvec_pages()
1645 struct bio_vec bv = bvec_iter_bvec(i->bvec, bi); in iov_iter_extract_bvec_pages()
1675 bvec_iter_advance_single(i->bvec, &bi, bv.bv_len); in iov_iter_extract_bvec_pages()
1678 iov_iter_advance(i, size); in iov_iter_extract_bvec_pages()
1686 static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i, in iov_iter_extract_kvec_pages() argument
1694 size_t skip = i->iov_offset, offset, len, size; in iov_iter_extract_kvec_pages()
1698 if (i->nr_segs == 0) in iov_iter_extract_kvec_pages()
1700 size = min(maxsize, i->kvec->iov_len - skip); in iov_iter_extract_kvec_pages()
1703 i->iov_offset = 0; in iov_iter_extract_kvec_pages()
1704 i->nr_segs--; in iov_iter_extract_kvec_pages()
1705 i->kvec++; in iov_iter_extract_kvec_pages()
1709 kaddr = i->kvec->iov_base + skip; in iov_iter_extract_kvec_pages()
1734 iov_iter_advance(i, size); in iov_iter_extract_kvec_pages()
1750 static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, in iov_iter_extract_user_pages() argument
1763 if (i->data_source == ITER_DEST) in iov_iter_extract_user_pages()
1767 if (i->nofault) in iov_iter_extract_user_pages()
1770 addr = first_iovec_segment(i, &maxsize); in iov_iter_extract_user_pages()
1786 iov_iter_advance(i, maxsize); in iov_iter_extract_user_pages()
1833 ssize_t iov_iter_extract_pages(struct iov_iter *i, in iov_iter_extract_pages() argument
1840 maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT); in iov_iter_extract_pages()
1844 if (likely(user_backed_iter(i))) in iov_iter_extract_pages()
1845 return iov_iter_extract_user_pages(i, pages, maxsize, in iov_iter_extract_pages()
1848 if (iov_iter_is_kvec(i)) in iov_iter_extract_pages()
1849 return iov_iter_extract_kvec_pages(i, pages, maxsize, in iov_iter_extract_pages()
1852 if (iov_iter_is_bvec(i)) in iov_iter_extract_pages()
1853 return iov_iter_extract_bvec_pages(i, pages, maxsize, in iov_iter_extract_pages()
1856 if (iov_iter_is_folioq(i)) in iov_iter_extract_pages()
1857 return iov_iter_extract_folioq_pages(i, pages, maxsize, in iov_iter_extract_pages()
1860 if (iov_iter_is_xarray(i)) in iov_iter_extract_pages()
1861 return iov_iter_extract_xarray_pages(i, pages, maxsize, in iov_iter_extract_pages()
1873 unsigned int max_pages, i; in get_contig_folio_len() local
1885 for (i = 1; i < max_pages; i++) { in get_contig_folio_len()
1888 if (page_folio(pages[i]) != folio || in get_contig_folio_len()
1889 pages[i] != pages[i - 1] + 1) in get_contig_folio_len()
1895 *num_pages = i; in get_contig_folio_len()
1926 unsigned short nr_pages, i = 0; in iov_iter_extract_bvecs() local
1954 pages[i])) in iov_iter_extract_bvecs()
1957 len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset); in iov_iter_extract_bvecs()
1958 bvec_set_page(&bv[*nr_vecs], pages[i], len, offset); in iov_iter_extract_bvecs()
1959 i += nr_to_add; in iov_iter_extract_bvecs()
1966 while (i < nr_pages) in iov_iter_extract_bvecs()
1967 unpin_user_page(pages[i++]); in iov_iter_extract_bvecs()