1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
27 /* All Rights Reserved */
28
29 /*
30 * University Copyright- Copyright (c) 1982, 1986, 1988
31 * The Regents of the University of California
32 * All Rights Reserved
33 *
34 * University Acknowledgment- Portions of this document are derived from
35 * software developed by the University of California, Berkeley, and its
36 * contributors.
37 */
38
39 /*
40 * $FreeBSD$
41 */
42
43 #include <sys/param.h>
44 #include <sys/uio_impl.h>
45 #include <sys/vnode.h>
46 #include <sys/zfs_znode.h>
47 #include <sys/byteorder.h>
48 #include <sys/lock.h>
49 #include <sys/vm.h>
50 #include <vm/vm_map.h>
51
52 static void
zfs_freeuio(struct uio * uio)53 zfs_freeuio(struct uio *uio)
54 {
55 #if __FreeBSD_version > 1500013
56 freeuio(uio);
57 #else
58 free(uio, M_IOV);
59 #endif
60 }
61
62 int
zfs_uiomove(void * cp,size_t n,zfs_uio_rw_t dir,zfs_uio_t * uio)63 zfs_uiomove(void *cp, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio)
64 {
65 ASSERT3U(zfs_uio_rw(uio), ==, dir);
66 return (uiomove(cp, (int)n, GET_UIO_STRUCT(uio)));
67 }
68
69 /*
70 * same as zfs_uiomove() but doesn't modify uio structure.
71 * return in cbytes how many bytes were copied.
72 */
73 int
zfs_uiocopy(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio,size_t * cbytes)74 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes)
75 {
76 struct iovec small_iovec[1];
77 struct uio small_uio_clone;
78 struct uio *uio_clone;
79 int error;
80
81 ASSERT3U(zfs_uio_rw(uio), ==, rw);
82 if (zfs_uio_iovcnt(uio) == 1) {
83 small_uio_clone = *(GET_UIO_STRUCT(uio));
84 small_iovec[0] = *(GET_UIO_STRUCT(uio)->uio_iov);
85 small_uio_clone.uio_iov = small_iovec;
86 uio_clone = &small_uio_clone;
87 } else {
88 uio_clone = cloneuio(GET_UIO_STRUCT(uio));
89 }
90
91 error = vn_io_fault_uiomove(p, n, uio_clone);
92 *cbytes = zfs_uio_resid(uio) - uio_clone->uio_resid;
93 if (uio_clone != &small_uio_clone)
94 zfs_freeuio(uio_clone);
95 return (error);
96 }
97
98 /*
99 * Drop the next n chars out of *uiop.
100 */
101 void
zfs_uioskip(zfs_uio_t * uio,size_t n)102 zfs_uioskip(zfs_uio_t *uio, size_t n)
103 {
104 zfs_uio_seg_t segflg;
105
106 /* For the full compatibility with illumos. */
107 if (n > zfs_uio_resid(uio))
108 return;
109
110 segflg = zfs_uio_segflg(uio);
111 zfs_uio_segflg(uio) = UIO_NOCOPY;
112 zfs_uiomove(NULL, n, zfs_uio_rw(uio), uio);
113 zfs_uio_segflg(uio) = segflg;
114 }
115
116 int
zfs_uio_fault_move(void * p,size_t n,zfs_uio_rw_t dir,zfs_uio_t * uio)117 zfs_uio_fault_move(void *p, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio)
118 {
119 ASSERT3U(zfs_uio_rw(uio), ==, dir);
120 return (vn_io_fault_uiomove(p, n, GET_UIO_STRUCT(uio)));
121 }
122
123 /*
124 * Check if the uio is page-aligned in memory.
125 */
126 boolean_t
zfs_uio_page_aligned(zfs_uio_t * uio)127 zfs_uio_page_aligned(zfs_uio_t *uio)
128 {
129 const struct iovec *iov = GET_UIO_STRUCT(uio)->uio_iov;
130
131 for (int i = zfs_uio_iovcnt(uio); i > 0; iov++, i--) {
132 uintptr_t addr = (uintptr_t)iov->iov_base;
133 size_t size = iov->iov_len;
134 if ((addr & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
135 return (B_FALSE);
136 }
137 }
138
139 return (B_TRUE);
140 }
141
142 static void
zfs_uio_set_pages_to_stable(zfs_uio_t * uio)143 zfs_uio_set_pages_to_stable(zfs_uio_t *uio)
144 {
145 ASSERT3P(uio->uio_dio.pages, !=, NULL);
146 ASSERT3S(uio->uio_dio.npages, >, 0);
147
148 for (int i = 0; i < uio->uio_dio.npages; i++) {
149 vm_page_t page = uio->uio_dio.pages[i];
150 ASSERT3P(page, !=, NULL);
151
152 MPASS(page == PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(page)));
153 vm_page_busy_acquire(page, VM_ALLOC_SBUSY);
154 pmap_remove_write(page);
155 }
156 }
157
158 static void
zfs_uio_release_stable_pages(zfs_uio_t * uio)159 zfs_uio_release_stable_pages(zfs_uio_t *uio)
160 {
161 ASSERT3P(uio->uio_dio.pages, !=, NULL);
162 for (int i = 0; i < uio->uio_dio.npages; i++) {
163 vm_page_t page = uio->uio_dio.pages[i];
164
165 ASSERT3P(page, !=, NULL);
166 vm_page_sunbusy(page);
167 }
168 }
169
170 /*
171 * If the operation is marked as read, then we are stating the pages will be
172 * written to and must be given write access.
173 */
174 static int
zfs_uio_hold_pages(unsigned long start,size_t len,int nr_pages,zfs_uio_rw_t rw,vm_page_t * pages)175 zfs_uio_hold_pages(unsigned long start, size_t len, int nr_pages,
176 zfs_uio_rw_t rw, vm_page_t *pages)
177 {
178 vm_map_t map;
179 vm_prot_t prot;
180 int count;
181
182 map = &curthread->td_proc->p_vmspace->vm_map;
183 ASSERT3S(len, >, 0);
184
185 prot = rw == UIO_READ ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ;
186 count = vm_fault_quick_hold_pages(map, start, len, prot, pages,
187 nr_pages);
188
189 return (count);
190 }
191
192 void
zfs_uio_free_dio_pages(zfs_uio_t * uio,zfs_uio_rw_t rw)193 zfs_uio_free_dio_pages(zfs_uio_t *uio, zfs_uio_rw_t rw)
194 {
195 ASSERT(uio->uio_extflg & UIO_DIRECT);
196 ASSERT3P(uio->uio_dio.pages, !=, NULL);
197 ASSERT(zfs_uio_rw(uio) == rw);
198
199 if (rw == UIO_WRITE)
200 zfs_uio_release_stable_pages(uio);
201
202 vm_page_unhold_pages(&uio->uio_dio.pages[0],
203 uio->uio_dio.npages);
204
205 kmem_free(uio->uio_dio.pages,
206 uio->uio_dio.npages * sizeof (vm_page_t));
207 }
208
209 static int
zfs_uio_get_user_pages(unsigned long start,int nr_pages,size_t len,zfs_uio_rw_t rw,vm_page_t * pages)210 zfs_uio_get_user_pages(unsigned long start, int nr_pages,
211 size_t len, zfs_uio_rw_t rw, vm_page_t *pages)
212 {
213 int count;
214
215 count = zfs_uio_hold_pages(start, len, nr_pages, rw, pages);
216
217 if (count != nr_pages) {
218 if (count > 0)
219 vm_page_unhold_pages(pages, count);
220 return (0);
221 }
222
223 ASSERT3S(count, ==, nr_pages);
224
225 return (count);
226 }
227
228 static int
zfs_uio_iov_step(struct iovec v,zfs_uio_t * uio,int * numpages)229 zfs_uio_iov_step(struct iovec v, zfs_uio_t *uio, int *numpages)
230 {
231 unsigned long addr = (unsigned long)(v.iov_base);
232 size_t len = v.iov_len;
233 int n = DIV_ROUND_UP(len, PAGE_SIZE);
234
235 int res = zfs_uio_get_user_pages(
236 P2ALIGN_TYPED(addr, PAGE_SIZE, unsigned long), n, len,
237 zfs_uio_rw(uio), &uio->uio_dio.pages[uio->uio_dio.npages]);
238
239 if (res != n)
240 return (SET_ERROR(EFAULT));
241
242 ASSERT3U(len, ==, res * PAGE_SIZE);
243 *numpages = res;
244 return (0);
245 }
246
247 static int
zfs_uio_get_dio_pages_impl(zfs_uio_t * uio)248 zfs_uio_get_dio_pages_impl(zfs_uio_t *uio)
249 {
250 const struct iovec *iovp = GET_UIO_STRUCT(uio)->uio_iov;
251 size_t len = zfs_uio_resid(uio);
252
253 for (int i = 0; i < zfs_uio_iovcnt(uio); i++) {
254 struct iovec iov;
255 int numpages = 0;
256
257 if (iovp->iov_len == 0) {
258 iovp++;
259 continue;
260 }
261 iov.iov_len = MIN(len, iovp->iov_len);
262 iov.iov_base = iovp->iov_base;
263 int error = zfs_uio_iov_step(iov, uio, &numpages);
264
265 if (error)
266 return (error);
267
268 uio->uio_dio.npages += numpages;
269 len -= iov.iov_len;
270 iovp++;
271 }
272
273 ASSERT0(len);
274
275 return (0);
276 }
277
278 /*
279 * This function holds user pages into the kernel. In the event that the user
280 * pages are not successfully held an error value is returned.
281 *
282 * On success, 0 is returned.
283 */
284 int
zfs_uio_get_dio_pages_alloc(zfs_uio_t * uio,zfs_uio_rw_t rw)285 zfs_uio_get_dio_pages_alloc(zfs_uio_t *uio, zfs_uio_rw_t rw)
286 {
287 int error = 0;
288 int npages = DIV_ROUND_UP(zfs_uio_resid(uio), PAGE_SIZE);
289 size_t size = npages * sizeof (vm_page_t);
290
291 ASSERT(zfs_uio_rw(uio) == rw);
292
293 uio->uio_dio.pages = kmem_alloc(size, KM_SLEEP);
294
295 error = zfs_uio_get_dio_pages_impl(uio);
296
297 if (error) {
298 vm_page_unhold_pages(&uio->uio_dio.pages[0],
299 uio->uio_dio.npages);
300 kmem_free(uio->uio_dio.pages, size);
301 return (error);
302 }
303
304 ASSERT3S(uio->uio_dio.npages, >, 0);
305
306 /*
307 * Since we will be writing the user pages we must make sure that
308 * they are stable. That way the contents of the pages can not change
309 * while we are doing: compression, checksumming, encryption, parity
310 * calculations or deduplication.
311 */
312 if (zfs_uio_rw(uio) == UIO_WRITE)
313 zfs_uio_set_pages_to_stable(uio);
314
315 uio->uio_extflg |= UIO_DIRECT;
316
317 return (0);
318 }
319