17 size_t copy_to_user_iter(void __user *iter_to, size_t progress,  in copy_to_user_iter()
18 			 size_t len, void *from, void *priv2)  in copy_to_user_iter()
31 size_t copy_to_user_iter_nofault(void __user *iter_to, size_t progress,  in copy_to_user_iter_nofault()
32 				 size_t len, void *from, void *priv2)  in copy_to_user_iter_nofault()
45 size_t copy_from_user_iter(void __user *iter_from, size_t progress,  in copy_from_user_iter()
46 			   size_t len, void *to, void *priv2)  in copy_from_user_iter()
48 	size_t res = len;  in copy_from_user_iter()
74 size_t memcpy_to_iter(void *iter_to, size_t progress,  in memcpy_to_iter()
75 		      size_t len, void *from, void *priv2)  in memcpy_to_iter()
82 size_t memcpy_from_iter(void *iter_from, size_t progress,  in memcpy_from_iter()
83 			size_t len, void *to, void *priv2)  in memcpy_from_iter()
102 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)  in fault_in_iov_iter_readable()
105 		size_t n = min(size, iov_iter_count(i));  in fault_in_iov_iter_readable()
109 		size_t count = min(size, iov_iter_count(i));  in fault_in_iov_iter_readable()
111 		size_t skip;  in fault_in_iov_iter_readable()
115 			size_t len = min(count, p->iov_len - skip);  in fault_in_iov_iter_readable()
116 			size_t ret;  in fault_in_iov_iter_readable()
145 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)  in fault_in_iov_iter_writeable()
148 		size_t n = min(size, iov_iter_count(i));  in fault_in_iov_iter_writeable()
152 		size_t count = min(size, iov_iter_count(i));  in fault_in_iov_iter_writeable()
154 		size_t skip;  in fault_in_iov_iter_writeable()
158 			size_t len = min(count, p->iov_len - skip);  in fault_in_iov_iter_writeable()
159 			size_t ret;  in fault_in_iov_iter_writeable()
176 			size_t count)  in iov_iter_init()
191 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()
204 size_t copy_to_user_iter_mc(void __user *iter_to, size_t progress,  in copy_to_user_iter_mc()
205 			    size_t len, void *from, void *priv2)  in copy_to_user_iter_mc()
216 size_t memcpy_to_iter_mc(void *iter_to, size_t progress,  in memcpy_to_iter_mc()
217 			 size_t len, void *from, void *priv2)  in memcpy_to_iter_mc()
246 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_mc_to_iter()
259 size_t __copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in __copy_from_iter()
265 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()
277 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,  in copy_from_user_iter_nocache()
278 				   size_t len, void *to, void *priv2)  in copy_from_user_iter_nocache()
283 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()
296 size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress,  in copy_from_user_iter_flushcache()
297 				      size_t len, void *to, void *priv2)  in copy_from_user_iter_flushcache()
303 size_t memcpy_from_iter_flushcache(void *iter_from, size_t progress,  in memcpy_from_iter_flushcache()
304 				   size_t len, void *to, void *priv2)  in memcpy_from_iter_flushcache()
326 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()
338 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)  in page_copy_sane()
341 	size_t v = n + offset;  in page_copy_sane()
361 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()
364 	size_t res = 0;  in copy_page_to_iter()
373 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_to_iter()
390 size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes,  in copy_page_to_iter_nofault()
393 	size_t res = 0;  in copy_page_to_iter_nofault()
403 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_to_iter_nofault()
423 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()
426 	size_t res = 0;  in copy_page_from_iter()
433 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_from_iter()
451 size_t zero_to_user_iter(void __user *iter_to, size_t progress,  in zero_to_user_iter()
452 			 size_t len, void *priv, void *priv2)  in zero_to_user_iter()
458 size_t zero_to_iter(void *iter_to, size_t progress,  in zero_to_iter()
459 		    size_t len, void *priv, void *priv2)  in zero_to_iter()
465 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()
472 size_t copy_folio_from_iter_atomic(struct folio *folio, size_t offset,  in copy_folio_from_iter_atomic()
473 		size_t bytes, struct iov_iter *i)  in copy_folio_from_iter_atomic()
475 	size_t n, copied = 0;  in copy_folio_from_iter_atomic()
502 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)  in iov_iter_bvec_advance()
522 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)  in iov_iter_iovec_advance()
541 static void iov_iter_folioq_advance(struct iov_iter *i, size_t size)  in iov_iter_folioq_advance()
557 		size_t fsize = folioq_folio_size(folioq, slot);  in iov_iter_folioq_advance()
574 void iov_iter_advance(struct iov_iter *i, size_t size)  in iov_iter_advance()
594 static void iov_iter_folioq_revert(struct iov_iter *i, size_t unroll)  in iov_iter_folioq_revert()
600 		size_t fsize;  in iov_iter_folioq_revert()
620 void iov_iter_revert(struct iov_iter *i, size_t unroll)  in iov_iter_revert()
642 			size_t n = (--bvec)->bv_len;  in iov_iter_revert()
657 			size_t n = (--iov)->iov_len;  in iov_iter_revert()
673 size_t iov_iter_single_seg_count(const struct iov_iter *i)  in iov_iter_single_seg_count()
690 			size_t count)  in iov_iter_kvec()
706 			size_t count)  in iov_iter_bvec()
736 			  unsigned int offset, size_t count)  in iov_iter_folio_queue()
764 		     struct xarray *xarray, loff_t start, size_t count)  in iov_iter_xarray()
787 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)  in iov_iter_discard()
803 	size_t size = i->count;  in iov_iter_alignment_iovec()
804 	size_t skip = i->iov_offset;  in iov_iter_alignment_iovec()
807 		size_t len = iov->iov_len - skip;  in iov_iter_alignment_iovec()
825 	size_t size = i->count;  in iov_iter_alignment_bvec()
829 		size_t len = bvec->bv_len - skip;  in iov_iter_alignment_bvec()
845 		size_t size = i->count;  in iov_iter_alignment()
872 	size_t size = i->count;  in iov_iter_gap_alignment()
897 static int want_pages_array(struct page ***res, size_t size,  in want_pages_array()
898 			    size_t start, unsigned int maxpages)  in want_pages_array()
914 				     struct page ***ppages, size_t maxsize,  in iter_folioq_get_pages()
915 				     unsigned maxpages, size_t *_start_offset)  in iter_folioq_get_pages()
920 	size_t extracted = 0, count = iter->count, iov_offset = iter->iov_offset;  in iter_folioq_get_pages()
937 		size_t offset = iov_offset, fsize = folioq_folio_size(folioq, slot);  in iter_folioq_get_pages()
938 		size_t part = PAGE_SIZE - offset % PAGE_SIZE;  in iter_folioq_get_pages()
1000 				     struct page ***pages, size_t maxsize,  in iter_xarray_get_pages()
1001 				     unsigned maxpages, size_t *_start_offset)  in iter_xarray_get_pages()
1019 	maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);  in iter_xarray_get_pages()
1026 static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)  in first_iovec_segment()
1028 	size_t skip;  in first_iovec_segment()
1036 		size_t len = iov->iov_len - skip;  in first_iovec_segment()
1049 				       size_t *size, size_t *start)  in first_bvec_segment()
1052 	size_t skip = i->iov_offset, len;  in first_bvec_segment()
1064 		   struct page ***pages, size_t maxsize,  in __iov_iter_get_pages_alloc()
1065 		   unsigned int maxpages, size_t *start)  in __iov_iter_get_pages_alloc()
1094 		maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start);  in __iov_iter_get_pages_alloc()
1113 		maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start);  in __iov_iter_get_pages_alloc()
1131 		size_t maxsize, unsigned maxpages, size_t *start)  in iov_iter_get_pages2()
1142 		struct page ***pages, size_t maxsize, size_t *start)  in iov_iter_get_pages_alloc2()
1159 	size_t skip = i->iov_offset, size = i->count;  in iov_npages()
1165 		size_t len = min(p->iov_len - skip, size);  in iov_npages()
1179 	size_t skip = i->iov_offset, size = i->count;  in bvec_npages()
1185 		size_t len = min(p->bv_len - skip, size);  in bvec_npages()
1445 int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i)  in import_ubuf()
1500 					     struct page ***pages, size_t maxsize,  in iov_iter_extract_folioq_pages()
1503 					     size_t *offset0)  in iov_iter_extract_folioq_pages()
1508 	size_t extracted = 0, offset, slot = i->folioq_slot;  in iov_iter_extract_folioq_pages()
1527 		size_t offset = i->iov_offset, fsize = folioq_folio_size(folioq, slot);  in iov_iter_extract_folioq_pages()
1528 		size_t part = PAGE_SIZE - offset % PAGE_SIZE;  in iov_iter_extract_folioq_pages()
1562 					     struct page ***pages, size_t maxsize,  in iov_iter_extract_xarray_pages()
1565 					     size_t *offset0)  in iov_iter_extract_xarray_pages()
1598 	maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);  in iov_iter_extract_xarray_pages()
1608 					   struct page ***pages, size_t maxsize,  in iov_iter_extract_bvec_pages()
1611 					   size_t *offset0)  in iov_iter_extract_bvec_pages()
1613 	size_t skip = i->iov_offset, size = 0;  in iov_iter_extract_bvec_pages()
1675 					   struct page ***pages, size_t maxsize,  in iov_iter_extract_kvec_pages()
1678 					   size_t *offset0)  in iov_iter_extract_kvec_pages()
1682 	size_t skip = i->iov_offset, offset, len, size;  in iov_iter_extract_kvec_pages()
1709 		size_t seg = min_t(size_t, len, PAGE_SIZE);  in iov_iter_extract_kvec_pages()
1721 	size = min_t(size_t, size, maxpages * PAGE_SIZE - offset);  in iov_iter_extract_kvec_pages()
1740 					   size_t maxsize,  in iov_iter_extract_user_pages()
1743 					   size_t *offset0)  in iov_iter_extract_user_pages()
1747 	size_t offset;  in iov_iter_extract_user_pages()
1766 	maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset);  in iov_iter_extract_user_pages()
1816 			       size_t maxsize,  in iov_iter_extract_pages()
1819 			       size_t *offset0)  in iov_iter_extract_pages()
1821 	maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT);  in iov_iter_extract_pages()
1850 		unsigned int *num_pages, size_t left, size_t offset)  in get_contig_folio_len()
1853 	size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, left);  in get_contig_folio_len()
1855 	size_t folio_offset, len;  in get_contig_folio_len()
1867 		size_t next = min_t(size_t, PAGE_SIZE, left);  in get_contig_folio_len()
1899 		size_t max_size, unsigned short *nr_vecs,  in iov_iter_extract_bvecs()
1904 	size_t left, offset, len;  in iov_iter_extract_bvecs()