xref: /freebsd/sys/kern/subr_uio.c (revision b670c9bafc0e31c7609969bf374b2e80bdc00211) 
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  * (c) UNIX System Laboratories, Inc.
7  * All or some portions of this file are derived from material licensed
8  * to the University of California by American Telephone and Telegraph
9  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10  * the permission of UNIX System Laboratories, Inc.
11  *
12  * Copyright (c) 2014 The FreeBSD Foundation
13  *
14  * Portions of this software were developed by Konstantin Belousov
15  * under sponsorship from the FreeBSD Foundation.
16  *
17  * Redistribution and use in source and binary forms, with or without
18  * modification, are permitted provided that the following conditions
19  * are met:
20  * 1. Redistributions of source code must retain the above copyright
21  *    notice, this list of conditions and the following disclaimer.
22  * 2. Redistributions in binary form must reproduce the above copyright
23  *    notice, this list of conditions and the following disclaimer in the
24  *    documentation and/or other materials provided with the distribution.
25  * 3. Neither the name of the University nor the names of its contributors
26  *    may be used to endorse or promote products derived from this software
27  *    without specific prior written permission.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39  * SUCH DAMAGE.
40  */
41 
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/limits.h>
46 #include <sys/lock.h>
47 #include <sys/mman.h>
48 #include <sys/proc.h>
49 #include <sys/resourcevar.h>
50 #include <sys/rwlock.h>
51 #include <sys/sched.h>
52 #include <sys/sysctl.h>
53 #include <sys/vnode.h>
54 
55 #include <vm/vm.h>
56 #include <vm/vm_param.h>
57 #include <vm/vm_extern.h>
58 #include <vm/vm_page.h>
59 #include <vm/vm_pageout.h>
60 #include <vm/vm_map.h>
61 
62 #include <machine/bus.h>
63 
64 SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, UIO_MAXIOV,
65 	"Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
66 
67 static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
68 
69 int
70 copyin_nofault(const void *udaddr, void *kaddr, size_t len)
71 {
72 	int error, save;
73 
74 	save = vm_fault_disable_pagefaults();
75 	error = copyin(udaddr, kaddr, len);
76 	vm_fault_enable_pagefaults(save);
77 	return (error);
78 }
79 
80 int
81 copyout_nofault(const void *kaddr, void *udaddr, size_t len)
82 {
83 	int error, save;
84 
85 	save = vm_fault_disable_pagefaults();
86 	error = copyout(kaddr, udaddr, len);
87 	vm_fault_enable_pagefaults(save);
88 	return (error);
89 }
90 
91 #define	PHYS_PAGE_COUNT(len)	(howmany(len, PAGE_SIZE) + 1)
92 
93 int
94 physcopyin(void *src, vm_paddr_t dst, size_t len)
95 {
96 	vm_page_t m[PHYS_PAGE_COUNT(len)];
97 	struct iovec iov[1];
98 	struct uio uio;
99 	int i;
100 
101 	iov[0].iov_base = src;
102 	iov[0].iov_len = len;
103 	uio.uio_iov = iov;
104 	uio.uio_iovcnt = 1;
105 	uio.uio_offset = 0;
106 	uio.uio_resid = len;
107 	uio.uio_segflg = UIO_SYSSPACE;
108 	uio.uio_rw = UIO_WRITE;
109 	for (i = 0; i < PHYS_PAGE_COUNT(len); i++, dst += PAGE_SIZE)
110 		m[i] = PHYS_TO_VM_PAGE(dst);
111 	return (uiomove_fromphys(m, dst & PAGE_MASK, len, &uio));
112 }
113 
114 int
115 physcopyout(vm_paddr_t src, void *dst, size_t len)
116 {
117 	vm_page_t m[PHYS_PAGE_COUNT(len)];
118 	struct iovec iov[1];
119 	struct uio uio;
120 	int i;
121 
122 	iov[0].iov_base = dst;
123 	iov[0].iov_len = len;
124 	uio.uio_iov = iov;
125 	uio.uio_iovcnt = 1;
126 	uio.uio_offset = 0;
127 	uio.uio_resid = len;
128 	uio.uio_segflg = UIO_SYSSPACE;
129 	uio.uio_rw = UIO_READ;
130 	for (i = 0; i < PHYS_PAGE_COUNT(len); i++, src += PAGE_SIZE)
131 		m[i] = PHYS_TO_VM_PAGE(src);
132 	return (uiomove_fromphys(m, src & PAGE_MASK, len, &uio));
133 }
134 
135 #undef PHYS_PAGE_COUNT
136 
137 int
138 physcopyin_vlist(bus_dma_segment_t *src, off_t offset, vm_paddr_t dst,
139     size_t len)
140 {
141 	size_t seg_len;
142 	int error;
143 
144 	error = 0;
145 	while (offset >= src->ds_len) {
146 		offset -= src->ds_len;
147 		src++;
148 	}
149 
150 	while (len > 0 && error == 0) {
151 		seg_len = MIN(src->ds_len - offset, len);
152 		error = physcopyin((void *)(uintptr_t)(src->ds_addr + offset),
153 		    dst, seg_len);
154 		offset = 0;
155 		src++;
156 		len -= seg_len;
157 		dst += seg_len;
158 	}
159 
160 	return (error);
161 }
162 
163 int
164 physcopyout_vlist(vm_paddr_t src, bus_dma_segment_t *dst, off_t offset,
165     size_t len)
166 {
167 	size_t seg_len;
168 	int error;
169 
170 	error = 0;
171 	while (offset >= dst->ds_len) {
172 		offset -= dst->ds_len;
173 		dst++;
174 	}
175 
176 	while (len > 0 && error == 0) {
177 		seg_len = MIN(dst->ds_len - offset, len);
178 		error = physcopyout(src, (void *)(uintptr_t)(dst->ds_addr +
179 		    offset), seg_len);
180 		offset = 0;
181 		dst++;
182 		len -= seg_len;
183 		src += seg_len;
184 	}
185 
186 	return (error);
187 }
188 
189 int
190 uiomove(void *cp, int n, struct uio *uio)
191 {
192 
193 	return (uiomove_faultflag(cp, n, uio, 0));
194 }
195 
196 int
197 uiomove_nofault(void *cp, int n, struct uio *uio)
198 {
199 
200 	return (uiomove_faultflag(cp, n, uio, 1));
201 }
202 
203 static int
204 uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
205 {
206 	struct iovec *iov;
207 	size_t cnt;
208 	int error, newflags, save;
209 
210 	save = error = 0;
211 
212 	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
213 	    ("uiomove: mode"));
214 	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
215 	    ("uiomove proc"));
216 	KASSERT(uio->uio_resid >= 0,
217 	    ("%s: uio %p resid underflow", __func__, uio));
218 
219 	if (uio->uio_segflg == UIO_USERSPACE) {
220 		newflags = TDP_DEADLKTREAT;
221 		if (nofault) {
222 			/*
223 			 * Fail if a non-spurious page fault occurs.
224 			 */
225 			newflags |= TDP_NOFAULTING | TDP_RESETSPUR;
226 		} else {
227 			WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
228 			    "Calling uiomove()");
229 		}
230 		save = curthread_pflags_set(newflags);
231 	} else {
232 		KASSERT(nofault == 0, ("uiomove: nofault"));
233 	}
234 
235 	while (n > 0 && uio->uio_resid) {
236 		KASSERT(uio->uio_iovcnt > 0,
237 		    ("%s: uio %p iovcnt underflow", __func__, uio));
238 
239 		iov = uio->uio_iov;
240 		cnt = iov->iov_len;
241 		if (cnt == 0) {
242 			uio->uio_iov++;
243 			uio->uio_iovcnt--;
244 			continue;
245 		}
246 		if (cnt > n)
247 			cnt = n;
248 
249 		switch (uio->uio_segflg) {
250 		case UIO_USERSPACE:
251 			maybe_yield();
252 			switch (uio->uio_rw) {
253 			case UIO_READ:
254 				error = copyout(cp, iov->iov_base, cnt);
255 				break;
256 			case UIO_WRITE:
257 				error = copyin(iov->iov_base, cp, cnt);
258 				break;
259 			}
260 			if (error)
261 				goto out;
262 			break;
263 
264 		case UIO_SYSSPACE:
265 			switch (uio->uio_rw) {
266 			case UIO_READ:
267 				bcopy(cp, iov->iov_base, cnt);
268 				break;
269 			case UIO_WRITE:
270 				bcopy(iov->iov_base, cp, cnt);
271 				break;
272 			}
273 			break;
274 		case UIO_NOCOPY:
275 			break;
276 		}
277 		iov->iov_base = (char *)iov->iov_base + cnt;
278 		iov->iov_len -= cnt;
279 		uio->uio_resid -= cnt;
280 		uio->uio_offset += cnt;
281 		cp = (char *)cp + cnt;
282 		n -= cnt;
283 	}
284 out:
285 	if (save)
286 		curthread_pflags_restore(save);
287 	return (error);
288 }
289 
290 /*
291  * Advance the pointer in the uio by offset.
292  */
293 void
294 uioadvance(struct uio *uio, size_t offset)
295 {
296 
297 	while (offset > 0) {
298 		struct iovec *iov;
299 		size_t cnt;
300 
301 		MPASS(uio->uio_resid >= 0);
302 		MPASS((size_t)uio->uio_resid >= offset);
303 		MPASS(uio->uio_iovcnt > 0);
304 
305 		iov = uio->uio_iov;
306 		if ((cnt = iov->iov_len) == 0) {
307 			uio->uio_iov++;
308 			uio->uio_iovcnt--;
309 			continue;
310 		}
311 		if (cnt > offset)
312 			cnt = offset;
313 		iov->iov_base = (char *)iov->iov_base + cnt;
314 		iov->iov_len -= cnt;
315 		uio->uio_resid -= cnt;
316 		uio->uio_offset += cnt;
317 		offset -= cnt;
318 	}
319 }
320 
321 /*
322  * Wrapper for uiomove() that validates the arguments against a known-good
323  * kernel buffer.  Currently, uiomove accepts a signed (n) argument, which
324  * is almost definitely a bad thing, so we catch that here as well.  We
325  * return a runtime failure, but it might be desirable to generate a runtime
326  * assertion failure instead.
327  */
328 int
329 uiomove_frombuf(void *buf, int buflen, struct uio *uio)
330 {
331 	size_t offset, n;
332 
333 	if (uio->uio_offset < 0 || uio->uio_resid < 0 ||
334 	    (offset = uio->uio_offset) != uio->uio_offset)
335 		return (EINVAL);
336 	if (buflen <= 0 || offset >= buflen)
337 		return (0);
338 	if ((n = buflen - offset) > IOSIZE_MAX)
339 		return (EINVAL);
340 	return (uiomove((char *)buf + offset, n, uio));
341 }
342 
343 /*
344  * Give next character to user as result of read.
345  */
346 int
347 ureadc(int c, struct uio *uio)
348 {
349 	struct iovec *iov;
350 	char *iov_base;
351 
352 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
353 	    "Calling ureadc()");
354 
355 again:
356 	if (uio->uio_iovcnt == 0 || uio->uio_resid == 0)
357 		panic("ureadc");
358 	iov = uio->uio_iov;
359 	if (iov->iov_len == 0) {
360 		uio->uio_iovcnt--;
361 		uio->uio_iov++;
362 		goto again;
363 	}
364 	switch (uio->uio_segflg) {
365 	case UIO_USERSPACE:
366 		if (subyte(iov->iov_base, c) < 0)
367 			return (EFAULT);
368 		break;
369 
370 	case UIO_SYSSPACE:
371 		iov_base = iov->iov_base;
372 		*iov_base = c;
373 		break;
374 
375 	case UIO_NOCOPY:
376 		break;
377 	}
378 	iov->iov_base = (char *)iov->iov_base + 1;
379 	iov->iov_len--;
380 	uio->uio_resid--;
381 	uio->uio_offset++;
382 	return (0);
383 }
384 
385 int
386 copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
387 {
388 	u_int iovlen;
389 
390 	*iov = NULL;
391 	if (iovcnt > UIO_MAXIOV)
392 		return (error);
393 	iovlen = iovcnt * sizeof(struct iovec);
394 	*iov = malloc(iovlen, M_IOV, M_WAITOK);
395 	error = copyin(iovp, *iov, iovlen);
396 	if (error) {
397 		free(*iov, M_IOV);
398 		*iov = NULL;
399 	}
400 	return (error);
401 }
402 
403 int
404 copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop)
405 {
406 	struct iovec *iov;
407 	struct uio *uio;
408 	u_int iovlen;
409 	int error, i;
410 
411 	*uiop = NULL;
412 	if (iovcnt > UIO_MAXIOV)
413 		return (EINVAL);
414 	iovlen = iovcnt * sizeof(struct iovec);
415 	uio = allocuio(iovcnt);
416 	iov = uio->uio_iov;
417 	error = copyin(iovp, iov, iovlen);
418 	if (error != 0) {
419 		freeuio(uio);
420 		return (error);
421 	}
422 	uio->uio_iovcnt = iovcnt;
423 	uio->uio_segflg = UIO_USERSPACE;
424 	uio->uio_offset = -1;
425 	uio->uio_resid = 0;
426 	for (i = 0; i < iovcnt; i++) {
427 		if (iov->iov_len > IOSIZE_MAX - uio->uio_resid) {
428 			freeuio(uio);
429 			return (EINVAL);
430 		}
431 		uio->uio_resid += iov->iov_len;
432 		iov++;
433 	}
434 	*uiop = uio;
435 	return (0);
436 }
437 
438 struct uio *
439 allocuio(u_int iovcnt)
440 {
441 	struct uio *uio;
442 	int iovlen;
443 
444 	KASSERT(iovcnt <= UIO_MAXIOV,
445 	    ("Requested %u iovecs exceed UIO_MAXIOV", iovcnt));
446 	iovlen = iovcnt * sizeof(struct iovec);
447 	uio = malloc(iovlen + sizeof(*uio), M_IOV, M_WAITOK);
448 	uio->uio_iov = (struct iovec *)(uio + 1);
449 
450 	return (uio);
451 }
452 
453 void
454 freeuio(struct uio *uio)
455 {
456 	free(uio, M_IOV);
457 }
458 
459 struct uio *
460 cloneuio(struct uio *uiop)
461 {
462 	struct iovec *iov;
463 	struct uio *uio;
464 	int iovlen;
465 
466 	iovlen = uiop->uio_iovcnt * sizeof(struct iovec);
467 	uio = allocuio(uiop->uio_iovcnt);
468 	iov = uio->uio_iov;
469 	*uio = *uiop;
470 	uio->uio_iov = iov;
471 	bcopy(uiop->uio_iov, uio->uio_iov, iovlen);
472 	return (uio);
473 }
474 
475 /*
476  * Map some anonymous memory in user space of size sz, rounded up to the page
477  * boundary.
478  */
479 int
480 copyout_map(struct thread *td, vm_offset_t *addr, size_t sz)
481 {
482 	struct vmspace *vms;
483 	int error;
484 	vm_size_t size;
485 
486 	vms = td->td_proc->p_vmspace;
487 
488 	/*
489 	 * Map somewhere after heap in process memory.
490 	 */
491 	*addr = round_page((vm_offset_t)vms->vm_daddr +
492 	    lim_max(td, RLIMIT_DATA));
493 
494 	/* round size up to page boundary */
495 	size = (vm_size_t)round_page(sz);
496 	if (size == 0)
497 		return (EINVAL);
498 	error = vm_mmap_object(&vms->vm_map, addr, size, VM_PROT_READ |
499 	    VM_PROT_WRITE, VM_PROT_ALL, MAP_PRIVATE | MAP_ANON, NULL, 0,
500 	    FALSE, td);
501 	return (error);
502 }
503 
504 /*
505  * Unmap memory in user space.
506  */
507 int
508 copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz)
509 {
510 	vm_map_t map;
511 	vm_size_t size;
512 
513 	if (sz == 0)
514 		return (0);
515 
516 	map = &td->td_proc->p_vmspace->vm_map;
517 	size = (vm_size_t)round_page(sz);
518 
519 	if (vm_map_remove(map, addr, addr + size) != KERN_SUCCESS)
520 		return (EINVAL);
521 
522 	return (0);
523 }
524 
525 int32_t
526 fuword32(volatile const void *addr)
527 {
528 	int rv;
529 	int32_t val;
530 
531 	rv = fueword32(addr, &val);
532 	return (rv == -1 ? -1 : val);
533 }
534 
535 #ifdef _LP64
536 int64_t
537 fuword64(volatile const void *addr)
538 {
539 	int rv;
540 	int64_t val;
541 
542 	rv = fueword64(addr, &val);
543 	return (rv == -1 ? -1 : val);
544 }
545 #endif /* _LP64 */
546 
547 long
548 fuword(volatile const void *addr)
549 {
550 	long val;
551 	int rv;
552 
553 	rv = fueword(addr, &val);
554 	return (rv == -1 ? -1 : val);
555 }
556 
557 uint32_t
558 casuword32(volatile uint32_t *addr, uint32_t old, uint32_t new)
559 {
560 	int rv;
561 	uint32_t val;
562 
563 	rv = casueword32(addr, old, &val, new);
564 	return (rv == -1 ? -1 : val);
565 }
566 
567 u_long
568 casuword(volatile u_long *addr, u_long old, u_long new)
569 {
570 	int rv;
571 	u_long val;
572 
573 	rv = casueword(addr, old, &val, new);
574 	return (rv == -1 ? -1 : val);
575 }
576