1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 /*
31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 * The Regents of the University of California
33 * All Rights Reserved
34 *
35 * University Acknowledgment- Portions of this document are derived from
36 * software developed by the University of California, Berkeley, and its
37 * contributors.
38 */
39
40 /*
41 * $FreeBSD$
42 */
43
44 #include <sys/param.h>
45 #include <sys/uio_impl.h>
46 #include <sys/vnode.h>
47 #include <sys/zfs_znode.h>
48 #include <sys/byteorder.h>
49 #include <sys/lock.h>
50 #include <sys/vm.h>
51 #include <vm/vm_map.h>
52
53 static void
zfs_freeuio(struct uio * uio)54 zfs_freeuio(struct uio *uio)
55 {
56 #if __FreeBSD_version > 1500013
57 freeuio(uio);
58 #else
59 free(uio, M_IOV);
60 #endif
61 }
62
63 int
zfs_uiomove(void * cp,size_t n,zfs_uio_rw_t dir,zfs_uio_t * uio)64 zfs_uiomove(void *cp, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio)
65 {
66 ASSERT3U(zfs_uio_rw(uio), ==, dir);
67 return (uiomove(cp, (int)n, GET_UIO_STRUCT(uio)));
68 }
69
70 /*
71 * same as zfs_uiomove() but doesn't modify uio structure.
72 * return in cbytes how many bytes were copied.
73 */
74 int
zfs_uiocopy(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio,size_t * cbytes)75 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes)
76 {
77 struct iovec small_iovec[1];
78 struct uio small_uio_clone;
79 struct uio *uio_clone;
80 int error;
81
82 ASSERT3U(zfs_uio_rw(uio), ==, rw);
83 if (zfs_uio_iovcnt(uio) == 1) {
84 small_uio_clone = *(GET_UIO_STRUCT(uio));
85 small_iovec[0] = *(GET_UIO_STRUCT(uio)->uio_iov);
86 small_uio_clone.uio_iov = small_iovec;
87 uio_clone = &small_uio_clone;
88 } else {
89 uio_clone = cloneuio(GET_UIO_STRUCT(uio));
90 }
91
92 error = vn_io_fault_uiomove(p, n, uio_clone);
93 *cbytes = zfs_uio_resid(uio) - uio_clone->uio_resid;
94 if (uio_clone != &small_uio_clone)
95 zfs_freeuio(uio_clone);
96 return (error);
97 }
98
99 /*
100 * Drop the next n chars out of *uiop.
101 */
102 void
zfs_uioskip(zfs_uio_t * uio,size_t n)103 zfs_uioskip(zfs_uio_t *uio, size_t n)
104 {
105 zfs_uio_seg_t segflg;
106
107 /* For the full compatibility with illumos. */
108 if (n > zfs_uio_resid(uio))
109 return;
110
111 segflg = zfs_uio_segflg(uio);
112 zfs_uio_segflg(uio) = UIO_NOCOPY;
113 zfs_uiomove(NULL, n, zfs_uio_rw(uio), uio);
114 zfs_uio_segflg(uio) = segflg;
115 }
116
117 int
zfs_uio_fault_move(void * p,size_t n,zfs_uio_rw_t dir,zfs_uio_t * uio)118 zfs_uio_fault_move(void *p, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio)
119 {
120 ASSERT3U(zfs_uio_rw(uio), ==, dir);
121 return (vn_io_fault_uiomove(p, n, GET_UIO_STRUCT(uio)));
122 }
123
124 /*
125 * Check if the uio is page-aligned in memory.
126 */
127 boolean_t
zfs_uio_page_aligned(zfs_uio_t * uio)128 zfs_uio_page_aligned(zfs_uio_t *uio)
129 {
130 const struct iovec *iov = GET_UIO_STRUCT(uio)->uio_iov;
131
132 for (int i = zfs_uio_iovcnt(uio); i > 0; iov++, i--) {
133 uintptr_t addr = (uintptr_t)iov->iov_base;
134 size_t size = iov->iov_len;
135 if ((addr & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
136 return (B_FALSE);
137 }
138 }
139
140 return (B_TRUE);
141 }
142
143 static void
zfs_uio_set_pages_to_stable(zfs_uio_t * uio)144 zfs_uio_set_pages_to_stable(zfs_uio_t *uio)
145 {
146 ASSERT3P(uio->uio_dio.pages, !=, NULL);
147 ASSERT3S(uio->uio_dio.npages, >, 0);
148
149 for (int i = 0; i < uio->uio_dio.npages; i++) {
150 vm_page_t page = uio->uio_dio.pages[i];
151 ASSERT3P(page, !=, NULL);
152
153 MPASS(page == PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(page)));
154 vm_page_busy_acquire(page, VM_ALLOC_SBUSY);
155 pmap_remove_write(page);
156 }
157 }
158
159 static void
zfs_uio_release_stable_pages(zfs_uio_t * uio)160 zfs_uio_release_stable_pages(zfs_uio_t *uio)
161 {
162 ASSERT3P(uio->uio_dio.pages, !=, NULL);
163 for (int i = 0; i < uio->uio_dio.npages; i++) {
164 vm_page_t page = uio->uio_dio.pages[i];
165
166 ASSERT3P(page, !=, NULL);
167 vm_page_sunbusy(page);
168 }
169 }
170
171 /*
172 * If the operation is marked as read, then we are stating the pages will be
173 * written to and must be given write access.
174 */
175 static int
zfs_uio_hold_pages(unsigned long start,size_t len,int nr_pages,zfs_uio_rw_t rw,vm_page_t * pages)176 zfs_uio_hold_pages(unsigned long start, size_t len, int nr_pages,
177 zfs_uio_rw_t rw, vm_page_t *pages)
178 {
179 vm_map_t map;
180 vm_prot_t prot;
181 int count;
182
183 map = &curthread->td_proc->p_vmspace->vm_map;
184 ASSERT3S(len, >, 0);
185
186 prot = rw == UIO_READ ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ;
187 count = vm_fault_quick_hold_pages(map, start, len, prot, pages,
188 nr_pages);
189
190 return (count);
191 }
192
193 void
zfs_uio_free_dio_pages(zfs_uio_t * uio,zfs_uio_rw_t rw)194 zfs_uio_free_dio_pages(zfs_uio_t *uio, zfs_uio_rw_t rw)
195 {
196 ASSERT(uio->uio_extflg & UIO_DIRECT);
197 ASSERT3P(uio->uio_dio.pages, !=, NULL);
198 ASSERT(zfs_uio_rw(uio) == rw);
199
200 if (rw == UIO_WRITE)
201 zfs_uio_release_stable_pages(uio);
202
203 vm_page_unhold_pages(&uio->uio_dio.pages[0],
204 uio->uio_dio.npages);
205
206 kmem_free(uio->uio_dio.pages,
207 uio->uio_dio.npages * sizeof (vm_page_t));
208 }
209
210 static int
zfs_uio_get_user_pages(unsigned long start,int nr_pages,size_t len,zfs_uio_rw_t rw,vm_page_t * pages)211 zfs_uio_get_user_pages(unsigned long start, int nr_pages,
212 size_t len, zfs_uio_rw_t rw, vm_page_t *pages)
213 {
214 int count;
215
216 count = zfs_uio_hold_pages(start, len, nr_pages, rw, pages);
217
218 if (count != nr_pages) {
219 if (count > 0)
220 vm_page_unhold_pages(pages, count);
221 return (0);
222 }
223
224 ASSERT3S(count, ==, nr_pages);
225
226 return (count);
227 }
228
229 static int
zfs_uio_iov_step(struct iovec v,zfs_uio_t * uio,int * numpages)230 zfs_uio_iov_step(struct iovec v, zfs_uio_t *uio, int *numpages)
231 {
232 unsigned long addr = (unsigned long)(v.iov_base);
233 size_t len = v.iov_len;
234 int n = DIV_ROUND_UP(len, PAGE_SIZE);
235
236 int res = zfs_uio_get_user_pages(
237 P2ALIGN_TYPED(addr, PAGE_SIZE, unsigned long), n, len,
238 zfs_uio_rw(uio), &uio->uio_dio.pages[uio->uio_dio.npages]);
239
240 if (res != n)
241 return (SET_ERROR(EFAULT));
242
243 ASSERT3U(len, ==, res * PAGE_SIZE);
244 *numpages = res;
245 return (0);
246 }
247
248 static int
zfs_uio_get_dio_pages_impl(zfs_uio_t * uio)249 zfs_uio_get_dio_pages_impl(zfs_uio_t *uio)
250 {
251 const struct iovec *iovp = GET_UIO_STRUCT(uio)->uio_iov;
252 size_t len = zfs_uio_resid(uio);
253
254 for (int i = 0; i < zfs_uio_iovcnt(uio); i++) {
255 struct iovec iov;
256 int numpages = 0;
257
258 if (iovp->iov_len == 0) {
259 iovp++;
260 continue;
261 }
262 iov.iov_len = MIN(len, iovp->iov_len);
263 iov.iov_base = iovp->iov_base;
264 int error = zfs_uio_iov_step(iov, uio, &numpages);
265
266 if (error)
267 return (error);
268
269 uio->uio_dio.npages += numpages;
270 len -= iov.iov_len;
271 iovp++;
272 }
273
274 ASSERT0(len);
275
276 return (0);
277 }
278
279 /*
280 * This function holds user pages into the kernel. In the event that the user
281 * pages are not successfully held an error value is returned.
282 *
283 * On success, 0 is returned.
284 */
285 int
zfs_uio_get_dio_pages_alloc(zfs_uio_t * uio,zfs_uio_rw_t rw)286 zfs_uio_get_dio_pages_alloc(zfs_uio_t *uio, zfs_uio_rw_t rw)
287 {
288 int error = 0;
289 int npages = DIV_ROUND_UP(zfs_uio_resid(uio), PAGE_SIZE);
290 size_t size = npages * sizeof (vm_page_t);
291
292 ASSERT(zfs_uio_rw(uio) == rw);
293
294 uio->uio_dio.pages = kmem_alloc(size, KM_SLEEP);
295
296 error = zfs_uio_get_dio_pages_impl(uio);
297
298 if (error) {
299 vm_page_unhold_pages(&uio->uio_dio.pages[0],
300 uio->uio_dio.npages);
301 kmem_free(uio->uio_dio.pages, size);
302 return (error);
303 }
304
305 ASSERT3S(uio->uio_dio.npages, >, 0);
306
307 /*
308 * Since we will be writing the user pages we must make sure that
309 * they are stable. That way the contents of the pages can not change
310 * while we are doing: compression, checksumming, encryption, parity
311 * calculations or deduplication.
312 */
313 if (zfs_uio_rw(uio) == UIO_WRITE)
314 zfs_uio_set_pages_to_stable(uio);
315
316 uio->uio_extflg |= UIO_DIRECT;
317
318 return (0);
319 }
320