1 /*- 2 * Copyright (c) 1982, 1986, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * (c) UNIX System Laboratories, Inc. 5 * All or some portions of this file are derived from material licensed 6 * to the University of California by American Telephone and Telegraph 7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 8 * the permission of UNIX System Laboratories, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94 35 */ 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 40 #include "opt_zero.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/limits.h> 46 #include <sys/lock.h> 47 #include <sys/mutex.h> 48 #include <sys/proc.h> 49 #include <sys/sched.h> 50 #include <sys/sysctl.h> 51 #include <sys/vnode.h> 52 53 #include <vm/vm.h> 54 #include <vm/vm_page.h> 55 #include <vm/vm_map.h> 56 #ifdef ZERO_COPY_SOCKETS 57 #include <vm/vm_param.h> 58 #include <vm/vm_object.h> 59 #endif 60 61 SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV, 62 "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)"); 63 64 #ifdef ZERO_COPY_SOCKETS 65 /* Declared in uipc_socket.c */ 66 extern int so_zero_copy_receive; 67 68 /* 69 * Identify the physical page mapped at the given kernel virtual 70 * address. Insert this physical page into the given address space at 71 * the given virtual address, replacing the physical page, if any, 72 * that already exists there. 73 */ 74 static int 75 vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr) 76 { 77 vm_map_t map = mapa; 78 vm_page_t kern_pg, user_pg; 79 vm_object_t uobject; 80 vm_map_entry_t entry; 81 vm_pindex_t upindex; 82 vm_prot_t prot; 83 boolean_t wired; 84 85 KASSERT((uaddr & PAGE_MASK) == 0, 86 ("vm_pgmoveco: uaddr is not page aligned")); 87 88 /* 89 * Herein the physical page is validated and dirtied. It is 90 * unwired in sf_buf_mext(). 91 */ 92 kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr)); 93 kern_pg->valid = VM_PAGE_BITS_ALL; 94 KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1, 95 ("vm_pgmoveco: kern_pg is not correctly wired")); 96 97 if ((vm_map_lookup(&map, uaddr, 98 VM_PROT_WRITE, &entry, &uobject, 99 &upindex, &prot, &wired)) != KERN_SUCCESS) { 100 return(EFAULT); 101 } 102 VM_OBJECT_LOCK(uobject); 103 retry: 104 if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { 105 if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) 106 goto retry; 107 vm_page_lock(user_pg); 108 pmap_remove_all(user_pg); 109 vm_page_free(user_pg); 110 vm_page_unlock(user_pg); 111 } else { 112 /* 113 * Even if a physical page does not exist in the 114 * object chain's first object, a physical page from a 115 * backing object may be mapped read only. 116 */ 117 if (uobject->backing_object != NULL) 118 pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE); 119 } 120 vm_page_insert(kern_pg, uobject, upindex); 121 vm_page_dirty(kern_pg); 122 VM_OBJECT_UNLOCK(uobject); 123 vm_map_lookup_done(map, entry); 124 return(KERN_SUCCESS); 125 } 126 #endif /* ZERO_COPY_SOCKETS */ 127 128 int 129 uiomove(void *cp, int n, struct uio *uio) 130 { 131 struct thread *td = curthread; 132 struct iovec *iov; 133 u_int cnt; 134 int error = 0; 135 int save = 0; 136 137 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE, 138 ("uiomove: mode")); 139 KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread, 140 ("uiomove proc")); 141 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 142 "Calling uiomove()"); 143 144 save = td->td_pflags & TDP_DEADLKTREAT; 145 td->td_pflags |= TDP_DEADLKTREAT; 146 147 while (n > 0 && uio->uio_resid) { 148 iov = uio->uio_iov; 149 cnt = iov->iov_len; 150 if (cnt == 0) { 151 uio->uio_iov++; 152 uio->uio_iovcnt--; 153 continue; 154 } 155 if (cnt > n) 156 cnt = n; 157 158 switch (uio->uio_segflg) { 159 160 case UIO_USERSPACE: 161 maybe_yield(); 162 if (uio->uio_rw == UIO_READ) 163 error = copyout(cp, iov->iov_base, cnt); 164 else 165 error = copyin(iov->iov_base, cp, cnt); 166 if (error) 167 goto out; 168 break; 169 170 case UIO_SYSSPACE: 171 if (uio->uio_rw == UIO_READ) 172 bcopy(cp, iov->iov_base, cnt); 173 else 174 bcopy(iov->iov_base, cp, cnt); 175 break; 176 case UIO_NOCOPY: 177 break; 178 } 179 iov->iov_base = (char *)iov->iov_base + cnt; 180 iov->iov_len -= cnt; 181 uio->uio_resid -= cnt; 182 uio->uio_offset += cnt; 183 cp = (char *)cp + cnt; 184 n -= cnt; 185 } 186 out: 187 if (save == 0) 188 td->td_pflags &= ~TDP_DEADLKTREAT; 189 return (error); 190 } 191 192 /* 193 * Wrapper for uiomove() that validates the arguments against a known-good 194 * kernel buffer. Currently, uiomove accepts a signed (n) argument, which 195 * is almost definitely a bad thing, so we catch that here as well. We 196 * return a runtime failure, but it might be desirable to generate a runtime 197 * assertion failure instead. 198 */ 199 int 200 uiomove_frombuf(void *buf, int buflen, struct uio *uio) 201 { 202 unsigned int offset, n; 203 204 if (uio->uio_offset < 0 || uio->uio_resid < 0 || 205 (offset = uio->uio_offset) != uio->uio_offset) 206 return (EINVAL); 207 if (buflen <= 0 || offset >= buflen) 208 return (0); 209 if ((n = buflen - offset) > INT_MAX) 210 return (EINVAL); 211 return (uiomove((char *)buf + offset, n, uio)); 212 } 213 214 #ifdef ZERO_COPY_SOCKETS 215 /* 216 * Experimental support for zero-copy I/O 217 */ 218 static int 219 userspaceco(void *cp, u_int cnt, struct uio *uio, int disposable) 220 { 221 struct iovec *iov; 222 int error; 223 224 iov = uio->uio_iov; 225 if (uio->uio_rw == UIO_READ) { 226 if ((so_zero_copy_receive != 0) 227 && ((cnt & PAGE_MASK) == 0) 228 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) 229 && ((uio->uio_offset & PAGE_MASK) == 0) 230 && ((((intptr_t) cp) & PAGE_MASK) == 0) 231 && (disposable != 0)) { 232 /* SOCKET: use page-trading */ 233 /* 234 * We only want to call vm_pgmoveco() on 235 * disposeable pages, since it gives the 236 * kernel page to the userland process. 237 */ 238 error = vm_pgmoveco(&curproc->p_vmspace->vm_map, 239 (vm_offset_t)cp, (vm_offset_t)iov->iov_base); 240 241 /* 242 * If we get an error back, attempt 243 * to use copyout() instead. The 244 * disposable page should be freed 245 * automatically if we weren't able to move 246 * it into userland. 247 */ 248 if (error != 0) 249 error = copyout(cp, iov->iov_base, cnt); 250 } else { 251 error = copyout(cp, iov->iov_base, cnt); 252 } 253 } else { 254 error = copyin(iov->iov_base, cp, cnt); 255 } 256 return (error); 257 } 258 259 int 260 uiomoveco(void *cp, int n, struct uio *uio, int disposable) 261 { 262 struct iovec *iov; 263 u_int cnt; 264 int error; 265 266 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE, 267 ("uiomoveco: mode")); 268 KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread, 269 ("uiomoveco proc")); 270 271 while (n > 0 && uio->uio_resid) { 272 iov = uio->uio_iov; 273 cnt = iov->iov_len; 274 if (cnt == 0) { 275 uio->uio_iov++; 276 uio->uio_iovcnt--; 277 continue; 278 } 279 if (cnt > n) 280 cnt = n; 281 282 switch (uio->uio_segflg) { 283 284 case UIO_USERSPACE: 285 maybe_yield(); 286 error = userspaceco(cp, cnt, uio, disposable); 287 if (error) 288 return (error); 289 break; 290 291 case UIO_SYSSPACE: 292 if (uio->uio_rw == UIO_READ) 293 bcopy(cp, iov->iov_base, cnt); 294 else 295 bcopy(iov->iov_base, cp, cnt); 296 break; 297 case UIO_NOCOPY: 298 break; 299 } 300 iov->iov_base = (char *)iov->iov_base + cnt; 301 iov->iov_len -= cnt; 302 uio->uio_resid -= cnt; 303 uio->uio_offset += cnt; 304 cp = (char *)cp + cnt; 305 n -= cnt; 306 } 307 return (0); 308 } 309 #endif /* ZERO_COPY_SOCKETS */ 310 311 /* 312 * Give next character to user as result of read. 313 */ 314 int 315 ureadc(int c, struct uio *uio) 316 { 317 struct iovec *iov; 318 char *iov_base; 319 320 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 321 "Calling ureadc()"); 322 323 again: 324 if (uio->uio_iovcnt == 0 || uio->uio_resid == 0) 325 panic("ureadc"); 326 iov = uio->uio_iov; 327 if (iov->iov_len == 0) { 328 uio->uio_iovcnt--; 329 uio->uio_iov++; 330 goto again; 331 } 332 switch (uio->uio_segflg) { 333 334 case UIO_USERSPACE: 335 if (subyte(iov->iov_base, c) < 0) 336 return (EFAULT); 337 break; 338 339 case UIO_SYSSPACE: 340 iov_base = iov->iov_base; 341 *iov_base = c; 342 iov->iov_base = iov_base; 343 break; 344 345 case UIO_NOCOPY: 346 break; 347 } 348 iov->iov_base = (char *)iov->iov_base + 1; 349 iov->iov_len--; 350 uio->uio_resid--; 351 uio->uio_offset++; 352 return (0); 353 } 354 355 int 356 copyinfrom(const void * __restrict src, void * __restrict dst, size_t len, 357 int seg) 358 { 359 int error = 0; 360 361 switch (seg) { 362 case UIO_USERSPACE: 363 error = copyin(src, dst, len); 364 break; 365 case UIO_SYSSPACE: 366 bcopy(src, dst, len); 367 break; 368 default: 369 panic("copyinfrom: bad seg %d\n", seg); 370 } 371 return (error); 372 } 373 374 int 375 copyinstrfrom(const void * __restrict src, void * __restrict dst, size_t len, 376 size_t * __restrict copied, int seg) 377 { 378 int error = 0; 379 380 switch (seg) { 381 case UIO_USERSPACE: 382 error = copyinstr(src, dst, len, copied); 383 break; 384 case UIO_SYSSPACE: 385 error = copystr(src, dst, len, copied); 386 break; 387 default: 388 panic("copyinstrfrom: bad seg %d\n", seg); 389 } 390 return (error); 391 } 392 393 int 394 copyiniov(struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error) 395 { 396 u_int iovlen; 397 398 *iov = NULL; 399 if (iovcnt > UIO_MAXIOV) 400 return (error); 401 iovlen = iovcnt * sizeof (struct iovec); 402 *iov = malloc(iovlen, M_IOV, M_WAITOK); 403 error = copyin(iovp, *iov, iovlen); 404 if (error) { 405 free(*iov, M_IOV); 406 *iov = NULL; 407 } 408 return (error); 409 } 410 411 int 412 copyinuio(struct iovec *iovp, u_int iovcnt, struct uio **uiop) 413 { 414 struct iovec *iov; 415 struct uio *uio; 416 u_int iovlen; 417 int error, i; 418 419 *uiop = NULL; 420 if (iovcnt > UIO_MAXIOV) 421 return (EINVAL); 422 iovlen = iovcnt * sizeof (struct iovec); 423 uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK); 424 iov = (struct iovec *)(uio + 1); 425 error = copyin(iovp, iov, iovlen); 426 if (error) { 427 free(uio, M_IOV); 428 return (error); 429 } 430 uio->uio_iov = iov; 431 uio->uio_iovcnt = iovcnt; 432 uio->uio_segflg = UIO_USERSPACE; 433 uio->uio_offset = -1; 434 uio->uio_resid = 0; 435 for (i = 0; i < iovcnt; i++) { 436 if (iov->iov_len > INT_MAX - uio->uio_resid) { 437 free(uio, M_IOV); 438 return (EINVAL); 439 } 440 uio->uio_resid += iov->iov_len; 441 iov++; 442 } 443 *uiop = uio; 444 return (0); 445 } 446 447 struct uio * 448 cloneuio(struct uio *uiop) 449 { 450 struct uio *uio; 451 int iovlen; 452 453 iovlen = uiop->uio_iovcnt * sizeof (struct iovec); 454 uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK); 455 *uio = *uiop; 456 uio->uio_iov = (struct iovec *)(uio + 1); 457 bcopy(uiop->uio_iov, uio->uio_iov, iovlen); 458 return (uio); 459 } 460