xref: /freebsd/sys/kern/subr_uio.c (revision f5f7c05209ca2c3748fd8b27c5e80ffad49120eb) 
1 /*-
2  * Copyright (c) 1982, 1986, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * (c) UNIX System Laboratories, Inc.
5  * All or some portions of this file are derived from material licensed
6  * to the University of California by American Telephone and Telegraph
7  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8  * the permission of UNIX System Laboratories, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	@(#)kern_subr.c	8.3 (Berkeley) 1/21/94
35  */
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include "opt_zero.h"
41 
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/limits.h>
46 #include <sys/lock.h>
47 #include <sys/mman.h>
48 #include <sys/mutex.h>
49 #include <sys/proc.h>
50 #include <sys/resourcevar.h>
51 #include <sys/sched.h>
52 #include <sys/sysctl.h>
53 #include <sys/vnode.h>
54 
55 #include <vm/vm.h>
56 #include <vm/vm_param.h>
57 #include <vm/vm_extern.h>
58 #include <vm/vm_page.h>
59 #include <vm/vm_map.h>
60 #ifdef SOCKET_SEND_COW
61 #include <vm/vm_object.h>
62 #endif
63 
64 SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV,
65 	"Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
66 
67 static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
68 
69 #ifdef SOCKET_SEND_COW
70 /* Declared in uipc_socket.c */
71 extern int so_zero_copy_receive;
72 
73 /*
74  * Identify the physical page mapped at the given kernel virtual
75  * address.  Insert this physical page into the given address space at
76  * the given virtual address, replacing the physical page, if any,
77  * that already exists there.
78  */
79 static int
80 vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
81 {
82 	vm_map_t map = mapa;
83 	vm_page_t kern_pg, user_pg;
84 	vm_object_t uobject;
85 	vm_map_entry_t entry;
86 	vm_pindex_t upindex;
87 	vm_prot_t prot;
88 	boolean_t wired;
89 
90 	KASSERT((uaddr & PAGE_MASK) == 0,
91 	    ("vm_pgmoveco: uaddr is not page aligned"));
92 
93 	/*
94 	 * Herein the physical page is validated and dirtied.  It is
95 	 * unwired in sf_buf_mext().
96 	 */
97 	kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
98 	kern_pg->valid = VM_PAGE_BITS_ALL;
99 	KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1,
100 	    ("vm_pgmoveco: kern_pg is not correctly wired"));
101 
102 	if ((vm_map_lookup(&map, uaddr,
103 			   VM_PROT_WRITE, &entry, &uobject,
104 			   &upindex, &prot, &wired)) != KERN_SUCCESS) {
105 		return(EFAULT);
106 	}
107 	VM_OBJECT_LOCK(uobject);
108 retry:
109 	if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
110 		if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco"))
111 			goto retry;
112 		vm_page_lock(user_pg);
113 		pmap_remove_all(user_pg);
114 		vm_page_free(user_pg);
115 		vm_page_unlock(user_pg);
116 	} else {
117 		/*
118 		 * Even if a physical page does not exist in the
119 		 * object chain's first object, a physical page from a
120 		 * backing object may be mapped read only.
121 		 */
122 		if (uobject->backing_object != NULL)
123 			pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE);
124 	}
125 	vm_page_insert(kern_pg, uobject, upindex);
126 	vm_page_dirty(kern_pg);
127 	VM_OBJECT_UNLOCK(uobject);
128 	vm_map_lookup_done(map, entry);
129 	return(KERN_SUCCESS);
130 }
131 #endif /* SOCKET_SEND_COW */
132 
133 int
134 copyin_nofault(const void *udaddr, void *kaddr, size_t len)
135 {
136 	int error, save;
137 
138 	save = vm_fault_disable_pagefaults();
139 	error = copyin(udaddr, kaddr, len);
140 	vm_fault_enable_pagefaults(save);
141 	return (error);
142 }
143 
144 int
145 copyout_nofault(const void *kaddr, void *udaddr, size_t len)
146 {
147 	int error, save;
148 
149 	save = vm_fault_disable_pagefaults();
150 	error = copyout(kaddr, udaddr, len);
151 	vm_fault_enable_pagefaults(save);
152 	return (error);
153 }
154 
155 int
156 uiomove(void *cp, int n, struct uio *uio)
157 {
158 
159 	return (uiomove_faultflag(cp, n, uio, 0));
160 }
161 
162 int
163 uiomove_nofault(void *cp, int n, struct uio *uio)
164 {
165 
166 	return (uiomove_faultflag(cp, n, uio, 1));
167 }
168 
169 static int
170 uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
171 {
172 	struct thread *td;
173 	struct iovec *iov;
174 	size_t cnt;
175 	int error, newflags, save;
176 
177 	td = curthread;
178 	error = 0;
179 
180 	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
181 	    ("uiomove: mode"));
182 	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == td,
183 	    ("uiomove proc"));
184 	if (!nofault)
185 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
186 		    "Calling uiomove()");
187 
188 	/* XXX does it make a sense to set TDP_DEADLKTREAT for UIO_SYSSPACE ? */
189 	newflags = TDP_DEADLKTREAT;
190 	if (uio->uio_segflg == UIO_USERSPACE && nofault) {
191 		/*
192 		 * Fail if a non-spurious page fault occurs.
193 		 */
194 		newflags |= TDP_NOFAULTING | TDP_RESETSPUR;
195 	}
196 	save = curthread_pflags_set(newflags);
197 
198 	while (n > 0 && uio->uio_resid) {
199 		iov = uio->uio_iov;
200 		cnt = iov->iov_len;
201 		if (cnt == 0) {
202 			uio->uio_iov++;
203 			uio->uio_iovcnt--;
204 			continue;
205 		}
206 		if (cnt > n)
207 			cnt = n;
208 
209 		switch (uio->uio_segflg) {
210 
211 		case UIO_USERSPACE:
212 			maybe_yield();
213 			if (uio->uio_rw == UIO_READ)
214 				error = copyout(cp, iov->iov_base, cnt);
215 			else
216 				error = copyin(iov->iov_base, cp, cnt);
217 			if (error)
218 				goto out;
219 			break;
220 
221 		case UIO_SYSSPACE:
222 			if (uio->uio_rw == UIO_READ)
223 				bcopy(cp, iov->iov_base, cnt);
224 			else
225 				bcopy(iov->iov_base, cp, cnt);
226 			break;
227 		case UIO_NOCOPY:
228 			break;
229 		}
230 		iov->iov_base = (char *)iov->iov_base + cnt;
231 		iov->iov_len -= cnt;
232 		uio->uio_resid -= cnt;
233 		uio->uio_offset += cnt;
234 		cp = (char *)cp + cnt;
235 		n -= cnt;
236 	}
237 out:
238 	curthread_pflags_restore(save);
239 	return (error);
240 }
241 
242 /*
243  * Wrapper for uiomove() that validates the arguments against a known-good
244  * kernel buffer.  Currently, uiomove accepts a signed (n) argument, which
245  * is almost definitely a bad thing, so we catch that here as well.  We
246  * return a runtime failure, but it might be desirable to generate a runtime
247  * assertion failure instead.
248  */
249 int
250 uiomove_frombuf(void *buf, int buflen, struct uio *uio)
251 {
252 	size_t offset, n;
253 
254 	if (uio->uio_offset < 0 || uio->uio_resid < 0 ||
255 	    (offset = uio->uio_offset) != uio->uio_offset)
256 		return (EINVAL);
257 	if (buflen <= 0 || offset >= buflen)
258 		return (0);
259 	if ((n = buflen - offset) > IOSIZE_MAX)
260 		return (EINVAL);
261 	return (uiomove((char *)buf + offset, n, uio));
262 }
263 
264 #ifdef SOCKET_RECV_PFLIP
265 /*
266  * Experimental support for zero-copy I/O
267  */
268 static int
269 userspaceco(void *cp, u_int cnt, struct uio *uio, int disposable)
270 {
271 	struct iovec *iov;
272 	int error;
273 
274 	iov = uio->uio_iov;
275 	if (uio->uio_rw == UIO_READ) {
276 		if ((so_zero_copy_receive != 0)
277 		 && ((cnt & PAGE_MASK) == 0)
278 		 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
279 		 && ((uio->uio_offset & PAGE_MASK) == 0)
280 		 && ((((intptr_t) cp) & PAGE_MASK) == 0)
281 		 && (disposable != 0)) {
282 			/* SOCKET: use page-trading */
283 			/*
284 			 * We only want to call vm_pgmoveco() on
285 			 * disposeable pages, since it gives the
286 			 * kernel page to the userland process.
287 			 */
288 			error =	vm_pgmoveco(&curproc->p_vmspace->vm_map,
289 			    (vm_offset_t)cp, (vm_offset_t)iov->iov_base);
290 
291 			/*
292 			 * If we get an error back, attempt
293 			 * to use copyout() instead.  The
294 			 * disposable page should be freed
295 			 * automatically if we weren't able to move
296 			 * it into userland.
297 			 */
298 			if (error != 0)
299 				error = copyout(cp, iov->iov_base, cnt);
300 		} else {
301 			error = copyout(cp, iov->iov_base, cnt);
302 		}
303 	} else {
304 		error = copyin(iov->iov_base, cp, cnt);
305 	}
306 	return (error);
307 }
308 
309 int
310 uiomoveco(void *cp, int n, struct uio *uio, int disposable)
311 {
312 	struct iovec *iov;
313 	u_int cnt;
314 	int error;
315 
316 	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
317 	    ("uiomoveco: mode"));
318 	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
319 	    ("uiomoveco proc"));
320 
321 	while (n > 0 && uio->uio_resid) {
322 		iov = uio->uio_iov;
323 		cnt = iov->iov_len;
324 		if (cnt == 0) {
325 			uio->uio_iov++;
326 			uio->uio_iovcnt--;
327 			continue;
328 		}
329 		if (cnt > n)
330 			cnt = n;
331 
332 		switch (uio->uio_segflg) {
333 
334 		case UIO_USERSPACE:
335 			maybe_yield();
336 			error = userspaceco(cp, cnt, uio, disposable);
337 			if (error)
338 				return (error);
339 			break;
340 
341 		case UIO_SYSSPACE:
342 			if (uio->uio_rw == UIO_READ)
343 				bcopy(cp, iov->iov_base, cnt);
344 			else
345 				bcopy(iov->iov_base, cp, cnt);
346 			break;
347 		case UIO_NOCOPY:
348 			break;
349 		}
350 		iov->iov_base = (char *)iov->iov_base + cnt;
351 		iov->iov_len -= cnt;
352 		uio->uio_resid -= cnt;
353 		uio->uio_offset += cnt;
354 		cp = (char *)cp + cnt;
355 		n -= cnt;
356 	}
357 	return (0);
358 }
359 #endif /* SOCKET_RECV_PFLIP */
360 
361 /*
362  * Give next character to user as result of read.
363  */
364 int
365 ureadc(int c, struct uio *uio)
366 {
367 	struct iovec *iov;
368 	char *iov_base;
369 
370 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
371 	    "Calling ureadc()");
372 
373 again:
374 	if (uio->uio_iovcnt == 0 || uio->uio_resid == 0)
375 		panic("ureadc");
376 	iov = uio->uio_iov;
377 	if (iov->iov_len == 0) {
378 		uio->uio_iovcnt--;
379 		uio->uio_iov++;
380 		goto again;
381 	}
382 	switch (uio->uio_segflg) {
383 
384 	case UIO_USERSPACE:
385 		if (subyte(iov->iov_base, c) < 0)
386 			return (EFAULT);
387 		break;
388 
389 	case UIO_SYSSPACE:
390 		iov_base = iov->iov_base;
391 		*iov_base = c;
392 		break;
393 
394 	case UIO_NOCOPY:
395 		break;
396 	}
397 	iov->iov_base = (char *)iov->iov_base + 1;
398 	iov->iov_len--;
399 	uio->uio_resid--;
400 	uio->uio_offset++;
401 	return (0);
402 }
403 
404 int
405 copyinfrom(const void * __restrict src, void * __restrict dst, size_t len,
406     int seg)
407 {
408 	int error = 0;
409 
410 	switch (seg) {
411 	case UIO_USERSPACE:
412 		error = copyin(src, dst, len);
413 		break;
414 	case UIO_SYSSPACE:
415 		bcopy(src, dst, len);
416 		break;
417 	default:
418 		panic("copyinfrom: bad seg %d\n", seg);
419 	}
420 	return (error);
421 }
422 
423 int
424 copyinstrfrom(const void * __restrict src, void * __restrict dst, size_t len,
425     size_t * __restrict copied, int seg)
426 {
427 	int error = 0;
428 
429 	switch (seg) {
430 	case UIO_USERSPACE:
431 		error = copyinstr(src, dst, len, copied);
432 		break;
433 	case UIO_SYSSPACE:
434 		error = copystr(src, dst, len, copied);
435 		break;
436 	default:
437 		panic("copyinstrfrom: bad seg %d\n", seg);
438 	}
439 	return (error);
440 }
441 
442 int
443 copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
444 {
445 	u_int iovlen;
446 
447 	*iov = NULL;
448 	if (iovcnt > UIO_MAXIOV)
449 		return (error);
450 	iovlen = iovcnt * sizeof (struct iovec);
451 	*iov = malloc(iovlen, M_IOV, M_WAITOK);
452 	error = copyin(iovp, *iov, iovlen);
453 	if (error) {
454 		free(*iov, M_IOV);
455 		*iov = NULL;
456 	}
457 	return (error);
458 }
459 
460 int
461 copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop)
462 {
463 	struct iovec *iov;
464 	struct uio *uio;
465 	u_int iovlen;
466 	int error, i;
467 
468 	*uiop = NULL;
469 	if (iovcnt > UIO_MAXIOV)
470 		return (EINVAL);
471 	iovlen = iovcnt * sizeof (struct iovec);
472 	uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK);
473 	iov = (struct iovec *)(uio + 1);
474 	error = copyin(iovp, iov, iovlen);
475 	if (error) {
476 		free(uio, M_IOV);
477 		return (error);
478 	}
479 	uio->uio_iov = iov;
480 	uio->uio_iovcnt = iovcnt;
481 	uio->uio_segflg = UIO_USERSPACE;
482 	uio->uio_offset = -1;
483 	uio->uio_resid = 0;
484 	for (i = 0; i < iovcnt; i++) {
485 		if (iov->iov_len > IOSIZE_MAX - uio->uio_resid) {
486 			free(uio, M_IOV);
487 			return (EINVAL);
488 		}
489 		uio->uio_resid += iov->iov_len;
490 		iov++;
491 	}
492 	*uiop = uio;
493 	return (0);
494 }
495 
496 struct uio *
497 cloneuio(struct uio *uiop)
498 {
499 	struct uio *uio;
500 	int iovlen;
501 
502 	iovlen = uiop->uio_iovcnt * sizeof (struct iovec);
503 	uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK);
504 	*uio = *uiop;
505 	uio->uio_iov = (struct iovec *)(uio + 1);
506 	bcopy(uiop->uio_iov, uio->uio_iov, iovlen);
507 	return (uio);
508 }
509 
510 /*
511  * Map some anonymous memory in user space of size sz, rounded up to the page
512  * boundary.
513  */
514 int
515 copyout_map(struct thread *td, vm_offset_t *addr, size_t sz)
516 {
517 	struct vmspace *vms;
518 	int error;
519 	vm_size_t size;
520 
521 	vms = td->td_proc->p_vmspace;
522 
523 	/*
524 	 * Map somewhere after heap in process memory.
525 	 */
526 	PROC_LOCK(td->td_proc);
527 	*addr = round_page((vm_offset_t)vms->vm_daddr +
528 	    lim_max(td->td_proc, RLIMIT_DATA));
529 	PROC_UNLOCK(td->td_proc);
530 
531 	/* round size up to page boundry */
532 	size = (vm_size_t)round_page(sz);
533 
534 	error = vm_mmap(&vms->vm_map, addr, size, PROT_READ | PROT_WRITE,
535 	    VM_PROT_ALL, MAP_PRIVATE | MAP_ANON, OBJT_DEFAULT, NULL, 0);
536 
537 	return (error);
538 }
539 
540 /*
541  * Unmap memory in user space.
542  */
543 int
544 copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz)
545 {
546 	vm_map_t map;
547 	vm_size_t size;
548 
549 	if (sz == 0)
550 		return (0);
551 
552 	map = &td->td_proc->p_vmspace->vm_map;
553 	size = (vm_size_t)round_page(sz);
554 
555 	if (vm_map_remove(map, addr, addr + size) != KERN_SUCCESS)
556 		return (EINVAL);
557 
558 	return (0);
559 }
560