xref: /freebsd/sys/contrib/openzfs/module/os/linux/zfs/zfs_uio.c (revision cc1a53bc1aea0675d64e9547cdca241612906592)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
27 /*	  All Rights Reserved	*/
28 
29 /*
30  * University Copyright- Copyright (c) 1982, 1986, 1988
31  * The Regents of the University of California
32  * All Rights Reserved
33  *
34  * University Acknowledgment- Portions of this document are derived from
35  * software developed by the University of California, Berkeley, and its
36  * contributors.
37  */
38 /*
39  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
40  */
41 
42 #ifdef _KERNEL
43 
44 #include <sys/types.h>
45 #include <sys/uio_impl.h>
46 #include <sys/sysmacros.h>
47 #include <sys/string.h>
48 #include <linux/kmap_compat.h>
49 #include <linux/uaccess.h>
50 
51 /*
52  * Move "n" bytes at byte address "p"; "rw" indicates the direction
53  * of the move, and the I/O parameters are provided in "uio", which is
54  * update to reflect the data which was moved.  Returns 0 on success or
55  * a non-zero errno on failure.
56  */
57 static int
58 zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
59 {
60 	const struct iovec *iov = uio->uio_iov;
61 	size_t skip = uio->uio_skip;
62 	ulong_t cnt;
63 
64 	while (n && uio->uio_resid) {
65 		cnt = MIN(iov->iov_len - skip, n);
66 		switch (uio->uio_segflg) {
67 		case UIO_USERSPACE:
68 			/*
69 			 * p = kernel data pointer
70 			 * iov->iov_base = user data pointer
71 			 */
72 			if (rw == UIO_READ) {
73 				if (copy_to_user(iov->iov_base+skip, p, cnt))
74 					return (EFAULT);
75 			} else {
76 				unsigned long b_left = 0;
77 				if (uio->uio_fault_disable) {
78 					if (!zfs_access_ok(VERIFY_READ,
79 					    (iov->iov_base + skip), cnt)) {
80 						return (EFAULT);
81 					}
82 					pagefault_disable();
83 					b_left =
84 					    __copy_from_user_inatomic(p,
85 					    (iov->iov_base + skip), cnt);
86 					pagefault_enable();
87 				} else {
88 					b_left =
89 					    copy_from_user(p,
90 					    (iov->iov_base + skip), cnt);
91 				}
92 				if (b_left > 0) {
93 					unsigned long c_bytes =
94 					    cnt - b_left;
95 					uio->uio_skip += c_bytes;
96 					ASSERT3U(uio->uio_skip, <,
97 					    iov->iov_len);
98 					uio->uio_resid -= c_bytes;
99 					uio->uio_loffset += c_bytes;
100 					return (EFAULT);
101 				}
102 			}
103 			break;
104 		case UIO_SYSSPACE:
105 			if (rw == UIO_READ)
106 				memcpy(iov->iov_base + skip, p, cnt);
107 			else
108 				memcpy(p, iov->iov_base + skip, cnt);
109 			break;
110 		default:
111 			ASSERT(0);
112 		}
113 		skip += cnt;
114 		if (skip == iov->iov_len) {
115 			skip = 0;
116 			uio->uio_iov = (++iov);
117 			uio->uio_iovcnt--;
118 		}
119 		uio->uio_skip = skip;
120 		uio->uio_resid -= cnt;
121 		uio->uio_loffset += cnt;
122 		p = (caddr_t)p + cnt;
123 		n -= cnt;
124 	}
125 	return (0);
126 }
127 
128 static int
129 zfs_uiomove_bvec(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
130 {
131 	const struct bio_vec *bv = uio->uio_bvec;
132 	size_t skip = uio->uio_skip;
133 	ulong_t cnt;
134 
135 	while (n && uio->uio_resid) {
136 		void *paddr;
137 		cnt = MIN(bv->bv_len - skip, n);
138 
139 		paddr = zfs_kmap_atomic(bv->bv_page);
140 		if (rw == UIO_READ)
141 			memcpy(paddr + bv->bv_offset + skip, p, cnt);
142 		else
143 			memcpy(p, paddr + bv->bv_offset + skip, cnt);
144 		zfs_kunmap_atomic(paddr);
145 
146 		skip += cnt;
147 		if (skip == bv->bv_len) {
148 			skip = 0;
149 			uio->uio_bvec = (++bv);
150 			uio->uio_iovcnt--;
151 		}
152 		uio->uio_skip = skip;
153 		uio->uio_resid -= cnt;
154 		uio->uio_loffset += cnt;
155 		p = (caddr_t)p + cnt;
156 		n -= cnt;
157 	}
158 	return (0);
159 }
160 
161 #if defined(HAVE_VFS_IOV_ITER)
162 static int
163 zfs_uiomove_iter(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio,
164     boolean_t revert)
165 {
166 	size_t cnt = MIN(n, uio->uio_resid);
167 
168 	if (uio->uio_skip)
169 		iov_iter_advance(uio->uio_iter, uio->uio_skip);
170 
171 	if (rw == UIO_READ)
172 		cnt = copy_to_iter(p, cnt, uio->uio_iter);
173 	else
174 		cnt = copy_from_iter(p, cnt, uio->uio_iter);
175 
176 	/*
177 	 * When operating on a full pipe no bytes are processed.
178 	 * In which case return EFAULT which is converted to EAGAIN
179 	 * by the kernel's generic_file_splice_read() function.
180 	 */
181 	if (cnt == 0)
182 		return (EFAULT);
183 
184 	/*
185 	 * Revert advancing the uio_iter.  This is set by zfs_uiocopy()
186 	 * to avoid consuming the uio and its iov_iter structure.
187 	 */
188 	if (revert)
189 		iov_iter_revert(uio->uio_iter, cnt);
190 
191 	uio->uio_resid -= cnt;
192 	uio->uio_loffset += cnt;
193 
194 	return (0);
195 }
196 #endif
197 
198 int
199 zfs_uiomove(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
200 {
201 	if (uio->uio_segflg == UIO_BVEC)
202 		return (zfs_uiomove_bvec(p, n, rw, uio));
203 #if defined(HAVE_VFS_IOV_ITER)
204 	else if (uio->uio_segflg == UIO_ITER)
205 		return (zfs_uiomove_iter(p, n, rw, uio, B_FALSE));
206 #endif
207 	else
208 		return (zfs_uiomove_iov(p, n, rw, uio));
209 }
210 EXPORT_SYMBOL(zfs_uiomove);
211 
212 /*
213  * Fault in the pages of the first n bytes specified by the uio structure.
214  * 1 byte in each page is touched and the uio struct is unmodified. Any
215  * error will terminate the process as this is only a best attempt to get
216  * the pages resident.
217  */
218 int
219 zfs_uio_prefaultpages(ssize_t n, zfs_uio_t *uio)
220 {
221 	if (uio->uio_segflg == UIO_SYSSPACE || uio->uio_segflg == UIO_BVEC) {
222 		/* There's never a need to fault in kernel pages */
223 		return (0);
224 #if defined(HAVE_VFS_IOV_ITER)
225 	} else if (uio->uio_segflg == UIO_ITER) {
226 		/*
227 		 * At least a Linux 4.9 kernel, iov_iter_fault_in_readable()
228 		 * can be relied on to fault in user pages when referenced.
229 		 */
230 		if (iov_iter_fault_in_readable(uio->uio_iter, n))
231 			return (EFAULT);
232 #endif
233 	} else {
234 		/* Fault in all user pages */
235 		ASSERT3S(uio->uio_segflg, ==, UIO_USERSPACE);
236 		const struct iovec *iov = uio->uio_iov;
237 		int iovcnt = uio->uio_iovcnt;
238 		size_t skip = uio->uio_skip;
239 		uint8_t tmp;
240 		caddr_t p;
241 
242 		for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
243 			ulong_t cnt = MIN(iov->iov_len - skip, n);
244 			/* empty iov */
245 			if (cnt == 0)
246 				continue;
247 			n -= cnt;
248 			/* touch each page in this segment. */
249 			p = iov->iov_base + skip;
250 			while (cnt) {
251 				if (get_user(tmp, (uint8_t *)p))
252 					return (EFAULT);
253 				ulong_t incr = MIN(cnt, PAGESIZE);
254 				p += incr;
255 				cnt -= incr;
256 			}
257 			/* touch the last byte in case it straddles a page. */
258 			p--;
259 			if (get_user(tmp, (uint8_t *)p))
260 				return (EFAULT);
261 		}
262 	}
263 
264 	return (0);
265 }
266 EXPORT_SYMBOL(zfs_uio_prefaultpages);
267 
268 /*
269  * The same as zfs_uiomove() but doesn't modify uio structure.
270  * return in cbytes how many bytes were copied.
271  */
272 int
273 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes)
274 {
275 	zfs_uio_t uio_copy;
276 	int ret;
277 
278 	memcpy(&uio_copy, uio, sizeof (zfs_uio_t));
279 
280 	if (uio->uio_segflg == UIO_BVEC)
281 		ret = zfs_uiomove_bvec(p, n, rw, &uio_copy);
282 #if defined(HAVE_VFS_IOV_ITER)
283 	else if (uio->uio_segflg == UIO_ITER)
284 		ret = zfs_uiomove_iter(p, n, rw, &uio_copy, B_TRUE);
285 #endif
286 	else
287 		ret = zfs_uiomove_iov(p, n, rw, &uio_copy);
288 
289 	*cbytes = uio->uio_resid - uio_copy.uio_resid;
290 
291 	return (ret);
292 }
293 EXPORT_SYMBOL(zfs_uiocopy);
294 
295 /*
296  * Drop the next n chars out of *uio.
297  */
298 void
299 zfs_uioskip(zfs_uio_t *uio, size_t n)
300 {
301 	if (n > uio->uio_resid)
302 		return;
303 
304 	if (uio->uio_segflg == UIO_BVEC) {
305 		uio->uio_skip += n;
306 		while (uio->uio_iovcnt &&
307 		    uio->uio_skip >= uio->uio_bvec->bv_len) {
308 			uio->uio_skip -= uio->uio_bvec->bv_len;
309 			uio->uio_bvec++;
310 			uio->uio_iovcnt--;
311 		}
312 #if defined(HAVE_VFS_IOV_ITER)
313 	} else if (uio->uio_segflg == UIO_ITER) {
314 		iov_iter_advance(uio->uio_iter, n);
315 #endif
316 	} else {
317 		uio->uio_skip += n;
318 		while (uio->uio_iovcnt &&
319 		    uio->uio_skip >= uio->uio_iov->iov_len) {
320 			uio->uio_skip -= uio->uio_iov->iov_len;
321 			uio->uio_iov++;
322 			uio->uio_iovcnt--;
323 		}
324 	}
325 	uio->uio_loffset += n;
326 	uio->uio_resid -= n;
327 }
328 EXPORT_SYMBOL(zfs_uioskip);
329 
330 #endif /* _KERNEL */
331