1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 * (c) UNIX System Laboratories, Inc.
7 * All or some portions of this file are derived from material licensed
8 * to the University of California by American Telephone and Telegraph
9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10 * the permission of UNIX System Laboratories, Inc.
11 *
12 * Copyright (c) 2014 The FreeBSD Foundation
13 *
14 * Portions of this software were developed by Konstantin Belousov
15 * under sponsorship from the FreeBSD Foundation.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 * 3. Neither the name of the University nor the names of its contributors
26 * may be used to endorse or promote products derived from this software
27 * without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * SUCH DAMAGE.
40 */
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/limits.h>
46 #include <sys/lock.h>
47 #include <sys/mman.h>
48 #include <sys/proc.h>
49 #include <sys/resourcevar.h>
50 #include <sys/rwlock.h>
51 #include <sys/sched.h>
52 #include <sys/sysctl.h>
53 #include <sys/vnode.h>
54
55 #include <vm/vm.h>
56 #include <vm/vm_param.h>
57 #include <vm/vm_extern.h>
58 #include <vm/vm_page.h>
59 #include <vm/vm_pageout.h>
60 #include <vm/vm_map.h>
61
62 #include <machine/bus.h>
63
64 SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, UIO_MAXIOV,
65 "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
66
67 static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
68
69 int
copyin_nofault(const void * udaddr,void * kaddr,size_t len)70 copyin_nofault(const void *udaddr, void *kaddr, size_t len)
71 {
72 int error, save;
73
74 save = vm_fault_disable_pagefaults();
75 error = copyin(udaddr, kaddr, len);
76 vm_fault_enable_pagefaults(save);
77 return (error);
78 }
79
80 int
copyout_nofault(const void * kaddr,void * udaddr,size_t len)81 copyout_nofault(const void *kaddr, void *udaddr, size_t len)
82 {
83 int error, save;
84
85 save = vm_fault_disable_pagefaults();
86 error = copyout(kaddr, udaddr, len);
87 vm_fault_enable_pagefaults(save);
88 return (error);
89 }
90
91 #define PHYS_PAGE_COUNT(len) (howmany(len, PAGE_SIZE) + 1)
92
93 int
physcopyin(void * src,vm_paddr_t dst,size_t len)94 physcopyin(void *src, vm_paddr_t dst, size_t len)
95 {
96 vm_page_t m[PHYS_PAGE_COUNT(len)];
97 struct iovec iov[1];
98 struct uio uio;
99 int i;
100
101 iov[0].iov_base = src;
102 iov[0].iov_len = len;
103 uio.uio_iov = iov;
104 uio.uio_iovcnt = 1;
105 uio.uio_offset = 0;
106 uio.uio_resid = len;
107 uio.uio_segflg = UIO_SYSSPACE;
108 uio.uio_rw = UIO_WRITE;
109 for (i = 0; i < PHYS_PAGE_COUNT(len); i++, dst += PAGE_SIZE)
110 m[i] = PHYS_TO_VM_PAGE(dst);
111 return (uiomove_fromphys(m, dst & PAGE_MASK, len, &uio));
112 }
113
114 int
physcopyout(vm_paddr_t src,void * dst,size_t len)115 physcopyout(vm_paddr_t src, void *dst, size_t len)
116 {
117 vm_page_t m[PHYS_PAGE_COUNT(len)];
118 struct iovec iov[1];
119 struct uio uio;
120 int i;
121
122 iov[0].iov_base = dst;
123 iov[0].iov_len = len;
124 uio.uio_iov = iov;
125 uio.uio_iovcnt = 1;
126 uio.uio_offset = 0;
127 uio.uio_resid = len;
128 uio.uio_segflg = UIO_SYSSPACE;
129 uio.uio_rw = UIO_READ;
130 for (i = 0; i < PHYS_PAGE_COUNT(len); i++, src += PAGE_SIZE)
131 m[i] = PHYS_TO_VM_PAGE(src);
132 return (uiomove_fromphys(m, src & PAGE_MASK, len, &uio));
133 }
134
135 #undef PHYS_PAGE_COUNT
136
137 int
physcopyin_vlist(bus_dma_segment_t * src,off_t offset,vm_paddr_t dst,size_t len)138 physcopyin_vlist(bus_dma_segment_t *src, off_t offset, vm_paddr_t dst,
139 size_t len)
140 {
141 size_t seg_len;
142 int error;
143
144 error = 0;
145 while (offset >= src->ds_len) {
146 offset -= src->ds_len;
147 src++;
148 }
149
150 while (len > 0 && error == 0) {
151 seg_len = MIN(src->ds_len - offset, len);
152 error = physcopyin((void *)(uintptr_t)(src->ds_addr + offset),
153 dst, seg_len);
154 offset = 0;
155 src++;
156 len -= seg_len;
157 dst += seg_len;
158 }
159
160 return (error);
161 }
162
163 int
physcopyout_vlist(vm_paddr_t src,bus_dma_segment_t * dst,off_t offset,size_t len)164 physcopyout_vlist(vm_paddr_t src, bus_dma_segment_t *dst, off_t offset,
165 size_t len)
166 {
167 size_t seg_len;
168 int error;
169
170 error = 0;
171 while (offset >= dst->ds_len) {
172 offset -= dst->ds_len;
173 dst++;
174 }
175
176 while (len > 0 && error == 0) {
177 seg_len = MIN(dst->ds_len - offset, len);
178 error = physcopyout(src, (void *)(uintptr_t)(dst->ds_addr +
179 offset), seg_len);
180 offset = 0;
181 dst++;
182 len -= seg_len;
183 src += seg_len;
184 }
185
186 return (error);
187 }
188
189 int
uiomove(void * cp,int n,struct uio * uio)190 uiomove(void *cp, int n, struct uio *uio)
191 {
192
193 return (uiomove_faultflag(cp, n, uio, 0));
194 }
195
196 int
uiomove_nofault(void * cp,int n,struct uio * uio)197 uiomove_nofault(void *cp, int n, struct uio *uio)
198 {
199
200 return (uiomove_faultflag(cp, n, uio, 1));
201 }
202
203 static int
uiomove_faultflag(void * cp,int n,struct uio * uio,int nofault)204 uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
205 {
206 struct iovec *iov;
207 size_t cnt;
208 int error, newflags, save;
209
210 save = error = 0;
211
212 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
213 ("uiomove: mode"));
214 KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
215 ("uiomove proc"));
216 KASSERT(uio->uio_resid >= 0,
217 ("%s: uio %p resid underflow", __func__, uio));
218
219 if (uio->uio_segflg == UIO_USERSPACE) {
220 newflags = TDP_DEADLKTREAT;
221 if (nofault) {
222 /*
223 * Fail if a non-spurious page fault occurs.
224 */
225 newflags |= TDP_NOFAULTING | TDP_RESETSPUR;
226 } else {
227 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
228 "Calling uiomove()");
229 }
230 save = curthread_pflags_set(newflags);
231 } else {
232 KASSERT(nofault == 0, ("uiomove: nofault"));
233 }
234
235 while (n > 0 && uio->uio_resid) {
236 KASSERT(uio->uio_iovcnt > 0,
237 ("%s: uio %p iovcnt underflow", __func__, uio));
238
239 iov = uio->uio_iov;
240 cnt = iov->iov_len;
241 if (cnt == 0) {
242 uio->uio_iov++;
243 uio->uio_iovcnt--;
244 continue;
245 }
246 if (cnt > n)
247 cnt = n;
248
249 switch (uio->uio_segflg) {
250 case UIO_USERSPACE:
251 maybe_yield();
252 switch (uio->uio_rw) {
253 case UIO_READ:
254 error = copyout(cp, iov->iov_base, cnt);
255 break;
256 case UIO_WRITE:
257 error = copyin(iov->iov_base, cp, cnt);
258 break;
259 }
260 if (error)
261 goto out;
262 break;
263
264 case UIO_SYSSPACE:
265 switch (uio->uio_rw) {
266 case UIO_READ:
267 bcopy(cp, iov->iov_base, cnt);
268 break;
269 case UIO_WRITE:
270 bcopy(iov->iov_base, cp, cnt);
271 break;
272 }
273 break;
274 case UIO_NOCOPY:
275 break;
276 }
277 iov->iov_base = (char *)iov->iov_base + cnt;
278 iov->iov_len -= cnt;
279 uio->uio_resid -= cnt;
280 uio->uio_offset += cnt;
281 cp = (char *)cp + cnt;
282 n -= cnt;
283 }
284 out:
285 if (save)
286 curthread_pflags_restore(save);
287 return (error);
288 }
289
290 /*
291 * Advance the pointer in the uio by offset.
292 */
293 void
uioadvance(struct uio * uio,size_t offset)294 uioadvance(struct uio *uio, size_t offset)
295 {
296
297 while (offset > 0) {
298 struct iovec *iov;
299 size_t cnt;
300
301 MPASS(uio->uio_resid >= 0);
302 MPASS((size_t)uio->uio_resid >= offset);
303 MPASS(uio->uio_iovcnt > 0);
304
305 iov = uio->uio_iov;
306 if ((cnt = iov->iov_len) == 0) {
307 uio->uio_iov++;
308 uio->uio_iovcnt--;
309 continue;
310 }
311 if (cnt > offset)
312 cnt = offset;
313 iov->iov_base = (char *)iov->iov_base + cnt;
314 iov->iov_len -= cnt;
315 uio->uio_resid -= cnt;
316 uio->uio_offset += cnt;
317 offset -= cnt;
318 }
319 }
320
321 /*
322 * Wrapper for uiomove() that validates the arguments against a known-good
323 * kernel buffer. Currently, uiomove accepts a signed (n) argument, which
324 * is almost definitely a bad thing, so we catch that here as well. We
325 * return a runtime failure, but it might be desirable to generate a runtime
326 * assertion failure instead.
327 */
328 int
uiomove_frombuf(void * buf,int buflen,struct uio * uio)329 uiomove_frombuf(void *buf, int buflen, struct uio *uio)
330 {
331 size_t offset, n;
332
333 if (uio->uio_offset < 0 || uio->uio_resid < 0 ||
334 (offset = uio->uio_offset) != uio->uio_offset)
335 return (EINVAL);
336 if (buflen <= 0 || offset >= buflen)
337 return (0);
338 if ((n = buflen - offset) > IOSIZE_MAX)
339 return (EINVAL);
340 return (uiomove((char *)buf + offset, n, uio));
341 }
342
343 /*
344 * Give next character to user as result of read.
345 */
346 int
ureadc(int c,struct uio * uio)347 ureadc(int c, struct uio *uio)
348 {
349 struct iovec *iov;
350 char *iov_base;
351
352 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
353 "Calling ureadc()");
354
355 again:
356 if (uio->uio_iovcnt == 0 || uio->uio_resid == 0)
357 panic("ureadc");
358 iov = uio->uio_iov;
359 if (iov->iov_len == 0) {
360 uio->uio_iovcnt--;
361 uio->uio_iov++;
362 goto again;
363 }
364 switch (uio->uio_segflg) {
365 case UIO_USERSPACE:
366 if (subyte(iov->iov_base, c) < 0)
367 return (EFAULT);
368 break;
369
370 case UIO_SYSSPACE:
371 iov_base = iov->iov_base;
372 *iov_base = c;
373 break;
374
375 case UIO_NOCOPY:
376 break;
377 }
378 iov->iov_base = (char *)iov->iov_base + 1;
379 iov->iov_len--;
380 uio->uio_resid--;
381 uio->uio_offset++;
382 return (0);
383 }
384
385 int
copyiniov(const struct iovec * iovp,u_int iovcnt,struct iovec ** iov,int error)386 copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
387 {
388 u_int iovlen;
389
390 *iov = NULL;
391 if (iovcnt > UIO_MAXIOV)
392 return (error);
393 iovlen = iovcnt * sizeof(struct iovec);
394 *iov = malloc(iovlen, M_IOV, M_WAITOK);
395 error = copyin(iovp, *iov, iovlen);
396 if (error) {
397 free(*iov, M_IOV);
398 *iov = NULL;
399 }
400 return (error);
401 }
402
403 int
copyinuio(const struct iovec * iovp,u_int iovcnt,struct uio ** uiop)404 copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop)
405 {
406 struct iovec *iov;
407 struct uio *uio;
408 u_int iovlen;
409 int error, i;
410
411 *uiop = NULL;
412 if (iovcnt > UIO_MAXIOV)
413 return (EINVAL);
414 iovlen = iovcnt * sizeof(struct iovec);
415 uio = allocuio(iovcnt);
416 iov = uio->uio_iov;
417 error = copyin(iovp, iov, iovlen);
418 if (error != 0) {
419 freeuio(uio);
420 return (error);
421 }
422 uio->uio_iovcnt = iovcnt;
423 uio->uio_segflg = UIO_USERSPACE;
424 uio->uio_offset = -1;
425 uio->uio_resid = 0;
426 for (i = 0; i < iovcnt; i++) {
427 if (iov->iov_len > IOSIZE_MAX - uio->uio_resid) {
428 freeuio(uio);
429 return (EINVAL);
430 }
431 uio->uio_resid += iov->iov_len;
432 iov++;
433 }
434 *uiop = uio;
435 return (0);
436 }
437
438 struct uio *
allocuio(u_int iovcnt)439 allocuio(u_int iovcnt)
440 {
441 struct uio *uio;
442 int iovlen;
443
444 KASSERT(iovcnt <= UIO_MAXIOV,
445 ("Requested %u iovecs exceed UIO_MAXIOV", iovcnt));
446 iovlen = iovcnt * sizeof(struct iovec);
447 uio = malloc(iovlen + sizeof(*uio), M_IOV, M_WAITOK);
448 uio->uio_iov = (struct iovec *)(uio + 1);
449
450 return (uio);
451 }
452
453 void
freeuio(struct uio * uio)454 freeuio(struct uio *uio)
455 {
456 free(uio, M_IOV);
457 }
458
459 struct uio *
cloneuio(struct uio * uiop)460 cloneuio(struct uio *uiop)
461 {
462 struct iovec *iov;
463 struct uio *uio;
464 int iovlen;
465
466 iovlen = uiop->uio_iovcnt * sizeof(struct iovec);
467 uio = allocuio(uiop->uio_iovcnt);
468 iov = uio->uio_iov;
469 *uio = *uiop;
470 uio->uio_iov = iov;
471 bcopy(uiop->uio_iov, uio->uio_iov, iovlen);
472 return (uio);
473 }
474
475 /*
476 * Map some anonymous memory in user space of size sz, rounded up to the page
477 * boundary.
478 */
479 int
copyout_map(struct thread * td,vm_offset_t * addr,size_t sz)480 copyout_map(struct thread *td, vm_offset_t *addr, size_t sz)
481 {
482 struct vmspace *vms;
483 int error;
484 vm_size_t size;
485
486 vms = td->td_proc->p_vmspace;
487
488 /*
489 * Map somewhere after heap in process memory.
490 */
491 *addr = round_page((vm_offset_t)vms->vm_daddr +
492 lim_max(td, RLIMIT_DATA));
493
494 /* round size up to page boundary */
495 size = (vm_size_t)round_page(sz);
496 if (size == 0)
497 return (EINVAL);
498 error = vm_mmap_object(&vms->vm_map, addr, size, VM_PROT_READ |
499 VM_PROT_WRITE, VM_PROT_ALL, MAP_PRIVATE | MAP_ANON, NULL, 0,
500 FALSE, td);
501 return (error);
502 }
503
504 /*
505 * Unmap memory in user space.
506 */
507 int
copyout_unmap(struct thread * td,vm_offset_t addr,size_t sz)508 copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz)
509 {
510 vm_map_t map;
511 vm_size_t size;
512
513 if (sz == 0)
514 return (0);
515
516 map = &td->td_proc->p_vmspace->vm_map;
517 size = (vm_size_t)round_page(sz);
518
519 if (vm_map_remove(map, addr, addr + size) != KERN_SUCCESS)
520 return (EINVAL);
521
522 return (0);
523 }
524
525 int32_t
fuword32(volatile const void * addr)526 fuword32(volatile const void *addr)
527 {
528 int rv;
529 int32_t val;
530
531 rv = fueword32(addr, &val);
532 return (rv == -1 ? -1 : val);
533 }
534
535 #ifdef _LP64
536 int64_t
fuword64(volatile const void * addr)537 fuword64(volatile const void *addr)
538 {
539 int rv;
540 int64_t val;
541
542 rv = fueword64(addr, &val);
543 return (rv == -1 ? -1 : val);
544 }
545 #endif /* _LP64 */
546
547 long
fuword(volatile const void * addr)548 fuword(volatile const void *addr)
549 {
550 long val;
551 int rv;
552
553 rv = fueword(addr, &val);
554 return (rv == -1 ? -1 : val);
555 }
556
557 uint32_t
casuword32(volatile uint32_t * addr,uint32_t old,uint32_t new)558 casuword32(volatile uint32_t *addr, uint32_t old, uint32_t new)
559 {
560 int rv;
561 uint32_t val;
562
563 rv = casueword32(addr, old, &val, new);
564 return (rv == -1 ? -1 : val);
565 }
566
567 u_long
casuword(volatile u_long * addr,u_long old,u_long new)568 casuword(volatile u_long *addr, u_long old, u_long new)
569 {
570 int rv;
571 u_long val;
572
573 rv = casueword(addr, old, &val, new);
574 return (rv == -1 ? -1 : val);
575 }
576