xref: /freebsd/sys/kern/vfs_vnops.c (revision 9e5787d2284e187abb5b654d924394a65772e004)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1989, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  * (c) UNIX System Laboratories, Inc.
7  * All or some portions of this file are derived from material licensed
8  * to the University of California by American Telephone and Telegraph
9  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10  * the permission of UNIX System Laboratories, Inc.
11  *
12  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
13  * Copyright (c) 2013, 2014 The FreeBSD Foundation
14  *
15  * Portions of this software were developed by Konstantin Belousov
16  * under sponsorship from the FreeBSD Foundation.
17  *
18  * Redistribution and use in source and binary forms, with or without
19  * modification, are permitted provided that the following conditions
20  * are met:
21  * 1. Redistributions of source code must retain the above copyright
22  *    notice, this list of conditions and the following disclaimer.
23  * 2. Redistributions in binary form must reproduce the above copyright
24  *    notice, this list of conditions and the following disclaimer in the
25  *    documentation and/or other materials provided with the distribution.
26  * 3. Neither the name of the University nor the names of its contributors
27  *    may be used to endorse or promote products derived from this software
28  *    without specific prior written permission.
29  *
30  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
31  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
34  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
36  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
37  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
38  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
39  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40  * SUCH DAMAGE.
41  *
42  *	@(#)vfs_vnops.c	8.2 (Berkeley) 1/21/94
43  */
44 
45 #include <sys/cdefs.h>
46 __FBSDID("$FreeBSD$");
47 
48 #include "opt_hwpmc_hooks.h"
49 
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/disk.h>
53 #include <sys/fail.h>
54 #include <sys/fcntl.h>
55 #include <sys/file.h>
56 #include <sys/kdb.h>
57 #include <sys/ktr.h>
58 #include <sys/stat.h>
59 #include <sys/priv.h>
60 #include <sys/proc.h>
61 #include <sys/limits.h>
62 #include <sys/lock.h>
63 #include <sys/mman.h>
64 #include <sys/mount.h>
65 #include <sys/mutex.h>
66 #include <sys/namei.h>
67 #include <sys/vnode.h>
68 #include <sys/bio.h>
69 #include <sys/buf.h>
70 #include <sys/filio.h>
71 #include <sys/resourcevar.h>
72 #include <sys/rwlock.h>
73 #include <sys/sx.h>
74 #include <sys/sleepqueue.h>
75 #include <sys/sysctl.h>
76 #include <sys/ttycom.h>
77 #include <sys/conf.h>
78 #include <sys/syslog.h>
79 #include <sys/unistd.h>
80 #include <sys/user.h>
81 
82 #include <security/audit/audit.h>
83 #include <security/mac/mac_framework.h>
84 
85 #include <vm/vm.h>
86 #include <vm/vm_extern.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_object.h>
90 #include <vm/vm_page.h>
91 #include <vm/vm_pager.h>
92 
93 #ifdef HWPMC_HOOKS
94 #include <sys/pmckern.h>
95 #endif
96 
97 static fo_rdwr_t	vn_read;
98 static fo_rdwr_t	vn_write;
99 static fo_rdwr_t	vn_io_fault;
100 static fo_truncate_t	vn_truncate;
101 static fo_ioctl_t	vn_ioctl;
102 static fo_poll_t	vn_poll;
103 static fo_kqfilter_t	vn_kqfilter;
104 static fo_stat_t	vn_statfile;
105 static fo_close_t	vn_closefile;
106 static fo_mmap_t	vn_mmap;
107 static fo_fallocate_t	vn_fallocate;
108 
109 struct 	fileops vnops = {
110 	.fo_read = vn_io_fault,
111 	.fo_write = vn_io_fault,
112 	.fo_truncate = vn_truncate,
113 	.fo_ioctl = vn_ioctl,
114 	.fo_poll = vn_poll,
115 	.fo_kqfilter = vn_kqfilter,
116 	.fo_stat = vn_statfile,
117 	.fo_close = vn_closefile,
118 	.fo_chmod = vn_chmod,
119 	.fo_chown = vn_chown,
120 	.fo_sendfile = vn_sendfile,
121 	.fo_seek = vn_seek,
122 	.fo_fill_kinfo = vn_fill_kinfo,
123 	.fo_mmap = vn_mmap,
124 	.fo_fallocate = vn_fallocate,
125 	.fo_flags = DFLAG_PASSABLE | DFLAG_SEEKABLE
126 };
127 
128 static const int io_hold_cnt = 16;
129 static int vn_io_fault_enable = 1;
130 SYSCTL_INT(_debug, OID_AUTO, vn_io_fault_enable, CTLFLAG_RWTUN,
131     &vn_io_fault_enable, 0, "Enable vn_io_fault lock avoidance");
132 static int vn_io_fault_prefault = 0;
133 SYSCTL_INT(_debug, OID_AUTO, vn_io_fault_prefault, CTLFLAG_RWTUN,
134     &vn_io_fault_prefault, 0, "Enable vn_io_fault prefaulting");
135 static int vn_io_pgcache_read_enable = 1;
136 SYSCTL_INT(_debug, OID_AUTO, vn_io_pgcache_read_enable, CTLFLAG_RWTUN,
137     &vn_io_pgcache_read_enable, 0,
138     "Enable copying from page cache for reads, avoiding fs");
139 static u_long vn_io_faults_cnt;
140 SYSCTL_ULONG(_debug, OID_AUTO, vn_io_faults, CTLFLAG_RD,
141     &vn_io_faults_cnt, 0, "Count of vn_io_fault lock avoidance triggers");
142 
143 static int vfs_allow_read_dir = 0;
144 SYSCTL_INT(_security_bsd, OID_AUTO, allow_read_dir, CTLFLAG_RW,
145     &vfs_allow_read_dir, 0,
146     "Enable read(2) of directory by root for filesystems that support it");
147 
148 /*
149  * Returns true if vn_io_fault mode of handling the i/o request should
150  * be used.
151  */
152 static bool
153 do_vn_io_fault(struct vnode *vp, struct uio *uio)
154 {
155 	struct mount *mp;
156 
157 	return (uio->uio_segflg == UIO_USERSPACE && vp->v_type == VREG &&
158 	    (mp = vp->v_mount) != NULL &&
159 	    (mp->mnt_kern_flag & MNTK_NO_IOPF) != 0 && vn_io_fault_enable);
160 }
161 
162 /*
163  * Structure used to pass arguments to vn_io_fault1(), to do either
164  * file- or vnode-based I/O calls.
165  */
166 struct vn_io_fault_args {
167 	enum {
168 		VN_IO_FAULT_FOP,
169 		VN_IO_FAULT_VOP
170 	} kind;
171 	struct ucred *cred;
172 	int flags;
173 	union {
174 		struct fop_args_tag {
175 			struct file *fp;
176 			fo_rdwr_t *doio;
177 		} fop_args;
178 		struct vop_args_tag {
179 			struct vnode *vp;
180 		} vop_args;
181 	} args;
182 };
183 
184 static int vn_io_fault1(struct vnode *vp, struct uio *uio,
185     struct vn_io_fault_args *args, struct thread *td);
186 
187 int
188 vn_open(struct nameidata *ndp, int *flagp, int cmode, struct file *fp)
189 {
190 	struct thread *td = ndp->ni_cnd.cn_thread;
191 
192 	return (vn_open_cred(ndp, flagp, cmode, 0, td->td_ucred, fp));
193 }
194 
195 /*
196  * Common code for vnode open operations via a name lookup.
197  * Lookup the vnode and invoke VOP_CREATE if needed.
198  * Check permissions, and call the VOP_OPEN or VOP_CREATE routine.
199  *
200  * Note that this does NOT free nameidata for the successful case,
201  * due to the NDINIT being done elsewhere.
202  */
203 int
204 vn_open_cred(struct nameidata *ndp, int *flagp, int cmode, u_int vn_open_flags,
205     struct ucred *cred, struct file *fp)
206 {
207 	struct vnode *vp;
208 	struct mount *mp;
209 	struct thread *td = ndp->ni_cnd.cn_thread;
210 	struct vattr vat;
211 	struct vattr *vap = &vat;
212 	int fmode, error;
213 
214 restart:
215 	fmode = *flagp;
216 	if ((fmode & (O_CREAT | O_EXCL | O_DIRECTORY)) == (O_CREAT |
217 	    O_EXCL | O_DIRECTORY))
218 		return (EINVAL);
219 	else if ((fmode & (O_CREAT | O_DIRECTORY)) == O_CREAT) {
220 		ndp->ni_cnd.cn_nameiop = CREATE;
221 		/*
222 		 * Set NOCACHE to avoid flushing the cache when
223 		 * rolling in many files at once.
224 		*/
225 		ndp->ni_cnd.cn_flags = ISOPEN | LOCKPARENT | LOCKLEAF | NOCACHE;
226 		if ((fmode & O_EXCL) == 0 && (fmode & O_NOFOLLOW) == 0)
227 			ndp->ni_cnd.cn_flags |= FOLLOW;
228 		if ((fmode & O_BENEATH) != 0)
229 			ndp->ni_cnd.cn_flags |= BENEATH;
230 		if (!(vn_open_flags & VN_OPEN_NOAUDIT))
231 			ndp->ni_cnd.cn_flags |= AUDITVNODE1;
232 		if (vn_open_flags & VN_OPEN_NOCAPCHECK)
233 			ndp->ni_cnd.cn_flags |= NOCAPCHECK;
234 		if ((vn_open_flags & VN_OPEN_INVFS) == 0)
235 			bwillwrite();
236 		if ((error = namei(ndp)) != 0)
237 			return (error);
238 		if (ndp->ni_vp == NULL) {
239 			VATTR_NULL(vap);
240 			vap->va_type = VREG;
241 			vap->va_mode = cmode;
242 			if (fmode & O_EXCL)
243 				vap->va_vaflags |= VA_EXCLUSIVE;
244 			if (vn_start_write(ndp->ni_dvp, &mp, V_NOWAIT) != 0) {
245 				NDFREE(ndp, NDF_ONLY_PNBUF);
246 				vput(ndp->ni_dvp);
247 				if ((error = vn_start_write(NULL, &mp,
248 				    V_XSLEEP | PCATCH)) != 0)
249 					return (error);
250 				goto restart;
251 			}
252 			if ((vn_open_flags & VN_OPEN_NAMECACHE) != 0)
253 				ndp->ni_cnd.cn_flags |= MAKEENTRY;
254 #ifdef MAC
255 			error = mac_vnode_check_create(cred, ndp->ni_dvp,
256 			    &ndp->ni_cnd, vap);
257 			if (error == 0)
258 #endif
259 				error = VOP_CREATE(ndp->ni_dvp, &ndp->ni_vp,
260 						   &ndp->ni_cnd, vap);
261 			vput(ndp->ni_dvp);
262 			vn_finished_write(mp);
263 			if (error) {
264 				NDFREE(ndp, NDF_ONLY_PNBUF);
265 				return (error);
266 			}
267 			fmode &= ~O_TRUNC;
268 			vp = ndp->ni_vp;
269 		} else {
270 			if (ndp->ni_dvp == ndp->ni_vp)
271 				vrele(ndp->ni_dvp);
272 			else
273 				vput(ndp->ni_dvp);
274 			ndp->ni_dvp = NULL;
275 			vp = ndp->ni_vp;
276 			if (fmode & O_EXCL) {
277 				error = EEXIST;
278 				goto bad;
279 			}
280 			if (vp->v_type == VDIR) {
281 				error = EISDIR;
282 				goto bad;
283 			}
284 			fmode &= ~O_CREAT;
285 		}
286 	} else {
287 		ndp->ni_cnd.cn_nameiop = LOOKUP;
288 		ndp->ni_cnd.cn_flags = ISOPEN |
289 		    ((fmode & O_NOFOLLOW) ? NOFOLLOW : FOLLOW) | LOCKLEAF;
290 		if (!(fmode & FWRITE))
291 			ndp->ni_cnd.cn_flags |= LOCKSHARED;
292 		if ((fmode & O_BENEATH) != 0)
293 			ndp->ni_cnd.cn_flags |= BENEATH;
294 		if (!(vn_open_flags & VN_OPEN_NOAUDIT))
295 			ndp->ni_cnd.cn_flags |= AUDITVNODE1;
296 		if (vn_open_flags & VN_OPEN_NOCAPCHECK)
297 			ndp->ni_cnd.cn_flags |= NOCAPCHECK;
298 		if ((error = namei(ndp)) != 0)
299 			return (error);
300 		vp = ndp->ni_vp;
301 	}
302 	error = vn_open_vnode(vp, fmode, cred, td, fp);
303 	if (error)
304 		goto bad;
305 	*flagp = fmode;
306 	return (0);
307 bad:
308 	NDFREE(ndp, NDF_ONLY_PNBUF);
309 	vput(vp);
310 	*flagp = fmode;
311 	ndp->ni_vp = NULL;
312 	return (error);
313 }
314 
315 static int
316 vn_open_vnode_advlock(struct vnode *vp, int fmode, struct file *fp)
317 {
318 	struct flock lf;
319 	int error, lock_flags, type;
320 
321 	ASSERT_VOP_LOCKED(vp, "vn_open_vnode_advlock");
322 	if ((fmode & (O_EXLOCK | O_SHLOCK)) == 0)
323 		return (0);
324 	KASSERT(fp != NULL, ("open with flock requires fp"));
325 	if (fp->f_type != DTYPE_NONE && fp->f_type != DTYPE_VNODE)
326 		return (EOPNOTSUPP);
327 
328 	lock_flags = VOP_ISLOCKED(vp);
329 	VOP_UNLOCK(vp);
330 
331 	lf.l_whence = SEEK_SET;
332 	lf.l_start = 0;
333 	lf.l_len = 0;
334 	lf.l_type = (fmode & O_EXLOCK) != 0 ? F_WRLCK : F_RDLCK;
335 	type = F_FLOCK;
336 	if ((fmode & FNONBLOCK) == 0)
337 		type |= F_WAIT;
338 	error = VOP_ADVLOCK(vp, (caddr_t)fp, F_SETLK, &lf, type);
339 	if (error == 0)
340 		fp->f_flag |= FHASLOCK;
341 
342 	vn_lock(vp, lock_flags | LK_RETRY);
343 	if (error == 0 && VN_IS_DOOMED(vp))
344 		error = ENOENT;
345 	return (error);
346 }
347 
348 /*
349  * Common code for vnode open operations once a vnode is located.
350  * Check permissions, and call the VOP_OPEN routine.
351  */
352 int
353 vn_open_vnode(struct vnode *vp, int fmode, struct ucred *cred,
354     struct thread *td, struct file *fp)
355 {
356 	accmode_t accmode;
357 	int error;
358 
359 	if (vp->v_type == VLNK)
360 		return (EMLINK);
361 	if (vp->v_type == VSOCK)
362 		return (EOPNOTSUPP);
363 	if (vp->v_type != VDIR && fmode & O_DIRECTORY)
364 		return (ENOTDIR);
365 	accmode = 0;
366 	if (fmode & (FWRITE | O_TRUNC)) {
367 		if (vp->v_type == VDIR)
368 			return (EISDIR);
369 		accmode |= VWRITE;
370 	}
371 	if (fmode & FREAD)
372 		accmode |= VREAD;
373 	if (fmode & FEXEC)
374 		accmode |= VEXEC;
375 	if ((fmode & O_APPEND) && (fmode & FWRITE))
376 		accmode |= VAPPEND;
377 #ifdef MAC
378 	if (fmode & O_CREAT)
379 		accmode |= VCREAT;
380 	if (fmode & O_VERIFY)
381 		accmode |= VVERIFY;
382 	error = mac_vnode_check_open(cred, vp, accmode);
383 	if (error)
384 		return (error);
385 
386 	accmode &= ~(VCREAT | VVERIFY);
387 #endif
388 	if ((fmode & O_CREAT) == 0 && accmode != 0) {
389 		error = VOP_ACCESS(vp, accmode, cred, td);
390 		if (error != 0)
391 			return (error);
392 	}
393 	if (vp->v_type == VFIFO && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
394 		vn_lock(vp, LK_UPGRADE | LK_RETRY);
395 	error = VOP_OPEN(vp, fmode, cred, td, fp);
396 	if (error != 0)
397 		return (error);
398 
399 	error = vn_open_vnode_advlock(vp, fmode, fp);
400 	if (error == 0 && (fmode & FWRITE) != 0) {
401 		error = VOP_ADD_WRITECOUNT(vp, 1);
402 		if (error == 0) {
403 			CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d",
404 			     __func__, vp, vp->v_writecount);
405 		}
406 	}
407 
408 	/*
409 	 * Error from advlock or VOP_ADD_WRITECOUNT() still requires
410 	 * calling VOP_CLOSE() to pair with earlier VOP_OPEN().
411 	 * Arrange for that by having fdrop() to use vn_closefile().
412 	 */
413 	if (error != 0) {
414 		fp->f_flag |= FOPENFAILED;
415 		fp->f_vnode = vp;
416 		if (fp->f_ops == &badfileops) {
417 			fp->f_type = DTYPE_VNODE;
418 			fp->f_ops = &vnops;
419 		}
420 		vref(vp);
421 	}
422 
423 	ASSERT_VOP_LOCKED(vp, "vn_open_vnode");
424 	return (error);
425 
426 }
427 
428 /*
429  * Check for write permissions on the specified vnode.
430  * Prototype text segments cannot be written.
431  * It is racy.
432  */
433 int
434 vn_writechk(struct vnode *vp)
435 {
436 
437 	ASSERT_VOP_LOCKED(vp, "vn_writechk");
438 	/*
439 	 * If there's shared text associated with
440 	 * the vnode, try to free it up once.  If
441 	 * we fail, we can't allow writing.
442 	 */
443 	if (VOP_IS_TEXT(vp))
444 		return (ETXTBSY);
445 
446 	return (0);
447 }
448 
449 /*
450  * Vnode close call
451  */
452 static int
453 vn_close1(struct vnode *vp, int flags, struct ucred *file_cred,
454     struct thread *td, bool keep_ref)
455 {
456 	struct mount *mp;
457 	int error, lock_flags;
458 
459 	if (vp->v_type != VFIFO && (flags & FWRITE) == 0 &&
460 	    MNT_EXTENDED_SHARED(vp->v_mount))
461 		lock_flags = LK_SHARED;
462 	else
463 		lock_flags = LK_EXCLUSIVE;
464 
465 	vn_start_write(vp, &mp, V_WAIT);
466 	vn_lock(vp, lock_flags | LK_RETRY);
467 	AUDIT_ARG_VNODE1(vp);
468 	if ((flags & (FWRITE | FOPENFAILED)) == FWRITE) {
469 		VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
470 		CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d",
471 		    __func__, vp, vp->v_writecount);
472 	}
473 	error = VOP_CLOSE(vp, flags, file_cred, td);
474 	if (keep_ref)
475 		VOP_UNLOCK(vp);
476 	else
477 		vput(vp);
478 	vn_finished_write(mp);
479 	return (error);
480 }
481 
482 int
483 vn_close(struct vnode *vp, int flags, struct ucred *file_cred,
484     struct thread *td)
485 {
486 
487 	return (vn_close1(vp, flags, file_cred, td, false));
488 }
489 
490 /*
491  * Heuristic to detect sequential operation.
492  */
493 static int
494 sequential_heuristic(struct uio *uio, struct file *fp)
495 {
496 	enum uio_rw rw;
497 
498 	ASSERT_VOP_LOCKED(fp->f_vnode, __func__);
499 
500 	rw = uio->uio_rw;
501 	if (fp->f_flag & FRDAHEAD)
502 		return (fp->f_seqcount[rw] << IO_SEQSHIFT);
503 
504 	/*
505 	 * Offset 0 is handled specially.  open() sets f_seqcount to 1 so
506 	 * that the first I/O is normally considered to be slightly
507 	 * sequential.  Seeking to offset 0 doesn't change sequentiality
508 	 * unless previous seeks have reduced f_seqcount to 0, in which
509 	 * case offset 0 is not special.
510 	 */
511 	if ((uio->uio_offset == 0 && fp->f_seqcount[rw] > 0) ||
512 	    uio->uio_offset == fp->f_nextoff[rw]) {
513 		/*
514 		 * f_seqcount is in units of fixed-size blocks so that it
515 		 * depends mainly on the amount of sequential I/O and not
516 		 * much on the number of sequential I/O's.  The fixed size
517 		 * of 16384 is hard-coded here since it is (not quite) just
518 		 * a magic size that works well here.  This size is more
519 		 * closely related to the best I/O size for real disks than
520 		 * to any block size used by software.
521 		 */
522 		if (uio->uio_resid >= IO_SEQMAX * 16384)
523 			fp->f_seqcount[rw] = IO_SEQMAX;
524 		else {
525 			fp->f_seqcount[rw] += howmany(uio->uio_resid, 16384);
526 			if (fp->f_seqcount[rw] > IO_SEQMAX)
527 				fp->f_seqcount[rw] = IO_SEQMAX;
528 		}
529 		return (fp->f_seqcount[rw] << IO_SEQSHIFT);
530 	}
531 
532 	/* Not sequential.  Quickly draw-down sequentiality. */
533 	if (fp->f_seqcount[rw] > 1)
534 		fp->f_seqcount[rw] = 1;
535 	else
536 		fp->f_seqcount[rw] = 0;
537 	return (0);
538 }
539 
540 /*
541  * Package up an I/O request on a vnode into a uio and do it.
542  */
543 int
544 vn_rdwr(enum uio_rw rw, struct vnode *vp, void *base, int len, off_t offset,
545     enum uio_seg segflg, int ioflg, struct ucred *active_cred,
546     struct ucred *file_cred, ssize_t *aresid, struct thread *td)
547 {
548 	struct uio auio;
549 	struct iovec aiov;
550 	struct mount *mp;
551 	struct ucred *cred;
552 	void *rl_cookie;
553 	struct vn_io_fault_args args;
554 	int error, lock_flags;
555 
556 	if (offset < 0 && vp->v_type != VCHR)
557 		return (EINVAL);
558 	auio.uio_iov = &aiov;
559 	auio.uio_iovcnt = 1;
560 	aiov.iov_base = base;
561 	aiov.iov_len = len;
562 	auio.uio_resid = len;
563 	auio.uio_offset = offset;
564 	auio.uio_segflg = segflg;
565 	auio.uio_rw = rw;
566 	auio.uio_td = td;
567 	error = 0;
568 
569 	if ((ioflg & IO_NODELOCKED) == 0) {
570 		if ((ioflg & IO_RANGELOCKED) == 0) {
571 			if (rw == UIO_READ) {
572 				rl_cookie = vn_rangelock_rlock(vp, offset,
573 				    offset + len);
574 			} else {
575 				rl_cookie = vn_rangelock_wlock(vp, offset,
576 				    offset + len);
577 			}
578 		} else
579 			rl_cookie = NULL;
580 		mp = NULL;
581 		if (rw == UIO_WRITE) {
582 			if (vp->v_type != VCHR &&
583 			    (error = vn_start_write(vp, &mp, V_WAIT | PCATCH))
584 			    != 0)
585 				goto out;
586 			if (MNT_SHARED_WRITES(mp) ||
587 			    ((mp == NULL) && MNT_SHARED_WRITES(vp->v_mount)))
588 				lock_flags = LK_SHARED;
589 			else
590 				lock_flags = LK_EXCLUSIVE;
591 		} else
592 			lock_flags = LK_SHARED;
593 		vn_lock(vp, lock_flags | LK_RETRY);
594 	} else
595 		rl_cookie = NULL;
596 
597 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
598 #ifdef MAC
599 	if ((ioflg & IO_NOMACCHECK) == 0) {
600 		if (rw == UIO_READ)
601 			error = mac_vnode_check_read(active_cred, file_cred,
602 			    vp);
603 		else
604 			error = mac_vnode_check_write(active_cred, file_cred,
605 			    vp);
606 	}
607 #endif
608 	if (error == 0) {
609 		if (file_cred != NULL)
610 			cred = file_cred;
611 		else
612 			cred = active_cred;
613 		if (do_vn_io_fault(vp, &auio)) {
614 			args.kind = VN_IO_FAULT_VOP;
615 			args.cred = cred;
616 			args.flags = ioflg;
617 			args.args.vop_args.vp = vp;
618 			error = vn_io_fault1(vp, &auio, &args, td);
619 		} else if (rw == UIO_READ) {
620 			error = VOP_READ(vp, &auio, ioflg, cred);
621 		} else /* if (rw == UIO_WRITE) */ {
622 			error = VOP_WRITE(vp, &auio, ioflg, cred);
623 		}
624 	}
625 	if (aresid)
626 		*aresid = auio.uio_resid;
627 	else
628 		if (auio.uio_resid && error == 0)
629 			error = EIO;
630 	if ((ioflg & IO_NODELOCKED) == 0) {
631 		VOP_UNLOCK(vp);
632 		if (mp != NULL)
633 			vn_finished_write(mp);
634 	}
635  out:
636 	if (rl_cookie != NULL)
637 		vn_rangelock_unlock(vp, rl_cookie);
638 	return (error);
639 }
640 
641 /*
642  * Package up an I/O request on a vnode into a uio and do it.  The I/O
643  * request is split up into smaller chunks and we try to avoid saturating
644  * the buffer cache while potentially holding a vnode locked, so we
645  * check bwillwrite() before calling vn_rdwr().  We also call kern_yield()
646  * to give other processes a chance to lock the vnode (either other processes
647  * core'ing the same binary, or unrelated processes scanning the directory).
648  */
649 int
650 vn_rdwr_inchunks(enum uio_rw rw, struct vnode *vp, void *base, size_t len,
651     off_t offset, enum uio_seg segflg, int ioflg, struct ucred *active_cred,
652     struct ucred *file_cred, size_t *aresid, struct thread *td)
653 {
654 	int error = 0;
655 	ssize_t iaresid;
656 
657 	do {
658 		int chunk;
659 
660 		/*
661 		 * Force `offset' to a multiple of MAXBSIZE except possibly
662 		 * for the first chunk, so that filesystems only need to
663 		 * write full blocks except possibly for the first and last
664 		 * chunks.
665 		 */
666 		chunk = MAXBSIZE - (uoff_t)offset % MAXBSIZE;
667 
668 		if (chunk > len)
669 			chunk = len;
670 		if (rw != UIO_READ && vp->v_type == VREG)
671 			bwillwrite();
672 		iaresid = 0;
673 		error = vn_rdwr(rw, vp, base, chunk, offset, segflg,
674 		    ioflg, active_cred, file_cred, &iaresid, td);
675 		len -= chunk;	/* aresid calc already includes length */
676 		if (error)
677 			break;
678 		offset += chunk;
679 		base = (char *)base + chunk;
680 		kern_yield(PRI_USER);
681 	} while (len);
682 	if (aresid)
683 		*aresid = len + iaresid;
684 	return (error);
685 }
686 
687 #if OFF_MAX <= LONG_MAX
688 off_t
689 foffset_lock(struct file *fp, int flags)
690 {
691 	volatile short *flagsp;
692 	off_t res;
693 	short state;
694 
695 	KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed"));
696 
697 	if ((flags & FOF_NOLOCK) != 0)
698 		return (atomic_load_long(&fp->f_offset));
699 
700 	/*
701 	 * According to McKusick the vn lock was protecting f_offset here.
702 	 * It is now protected by the FOFFSET_LOCKED flag.
703 	 */
704 	flagsp = &fp->f_vnread_flags;
705 	if (atomic_cmpset_acq_16(flagsp, 0, FOFFSET_LOCKED))
706 		return (atomic_load_long(&fp->f_offset));
707 
708 	sleepq_lock(&fp->f_vnread_flags);
709 	state = atomic_load_16(flagsp);
710 	for (;;) {
711 		if ((state & FOFFSET_LOCKED) == 0) {
712 			if (!atomic_fcmpset_acq_16(flagsp, &state,
713 			    FOFFSET_LOCKED))
714 				continue;
715 			break;
716 		}
717 		if ((state & FOFFSET_LOCK_WAITING) == 0) {
718 			if (!atomic_fcmpset_acq_16(flagsp, &state,
719 			    state | FOFFSET_LOCK_WAITING))
720 				continue;
721 		}
722 		DROP_GIANT();
723 		sleepq_add(&fp->f_vnread_flags, NULL, "vofflock", 0, 0);
724 		sleepq_wait(&fp->f_vnread_flags, PUSER -1);
725 		PICKUP_GIANT();
726 		sleepq_lock(&fp->f_vnread_flags);
727 		state = atomic_load_16(flagsp);
728 	}
729 	res = atomic_load_long(&fp->f_offset);
730 	sleepq_release(&fp->f_vnread_flags);
731 	return (res);
732 }
733 
734 void
735 foffset_unlock(struct file *fp, off_t val, int flags)
736 {
737 	volatile short *flagsp;
738 	short state;
739 
740 	KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed"));
741 
742 	if ((flags & FOF_NOUPDATE) == 0)
743 		atomic_store_long(&fp->f_offset, val);
744 	if ((flags & FOF_NEXTOFF_R) != 0)
745 		fp->f_nextoff[UIO_READ] = val;
746 	if ((flags & FOF_NEXTOFF_W) != 0)
747 		fp->f_nextoff[UIO_WRITE] = val;
748 
749 	if ((flags & FOF_NOLOCK) != 0)
750 		return;
751 
752 	flagsp = &fp->f_vnread_flags;
753 	state = atomic_load_16(flagsp);
754 	if ((state & FOFFSET_LOCK_WAITING) == 0 &&
755 	    atomic_cmpset_rel_16(flagsp, state, 0))
756 		return;
757 
758 	sleepq_lock(&fp->f_vnread_flags);
759 	MPASS((fp->f_vnread_flags & FOFFSET_LOCKED) != 0);
760 	MPASS((fp->f_vnread_flags & FOFFSET_LOCK_WAITING) != 0);
761 	fp->f_vnread_flags = 0;
762 	sleepq_broadcast(&fp->f_vnread_flags, SLEEPQ_SLEEP, 0, 0);
763 	sleepq_release(&fp->f_vnread_flags);
764 }
765 #else
766 off_t
767 foffset_lock(struct file *fp, int flags)
768 {
769 	struct mtx *mtxp;
770 	off_t res;
771 
772 	KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed"));
773 
774 	mtxp = mtx_pool_find(mtxpool_sleep, fp);
775 	mtx_lock(mtxp);
776 	if ((flags & FOF_NOLOCK) == 0) {
777 		while (fp->f_vnread_flags & FOFFSET_LOCKED) {
778 			fp->f_vnread_flags |= FOFFSET_LOCK_WAITING;
779 			msleep(&fp->f_vnread_flags, mtxp, PUSER -1,
780 			    "vofflock", 0);
781 		}
782 		fp->f_vnread_flags |= FOFFSET_LOCKED;
783 	}
784 	res = fp->f_offset;
785 	mtx_unlock(mtxp);
786 	return (res);
787 }
788 
789 void
790 foffset_unlock(struct file *fp, off_t val, int flags)
791 {
792 	struct mtx *mtxp;
793 
794 	KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed"));
795 
796 	mtxp = mtx_pool_find(mtxpool_sleep, fp);
797 	mtx_lock(mtxp);
798 	if ((flags & FOF_NOUPDATE) == 0)
799 		fp->f_offset = val;
800 	if ((flags & FOF_NEXTOFF_R) != 0)
801 		fp->f_nextoff[UIO_READ] = val;
802 	if ((flags & FOF_NEXTOFF_W) != 0)
803 		fp->f_nextoff[UIO_WRITE] = val;
804 	if ((flags & FOF_NOLOCK) == 0) {
805 		KASSERT((fp->f_vnread_flags & FOFFSET_LOCKED) != 0,
806 		    ("Lost FOFFSET_LOCKED"));
807 		if (fp->f_vnread_flags & FOFFSET_LOCK_WAITING)
808 			wakeup(&fp->f_vnread_flags);
809 		fp->f_vnread_flags = 0;
810 	}
811 	mtx_unlock(mtxp);
812 }
813 #endif
814 
815 void
816 foffset_lock_uio(struct file *fp, struct uio *uio, int flags)
817 {
818 
819 	if ((flags & FOF_OFFSET) == 0)
820 		uio->uio_offset = foffset_lock(fp, flags);
821 }
822 
823 void
824 foffset_unlock_uio(struct file *fp, struct uio *uio, int flags)
825 {
826 
827 	if ((flags & FOF_OFFSET) == 0)
828 		foffset_unlock(fp, uio->uio_offset, flags);
829 }
830 
831 static int
832 get_advice(struct file *fp, struct uio *uio)
833 {
834 	struct mtx *mtxp;
835 	int ret;
836 
837 	ret = POSIX_FADV_NORMAL;
838 	if (fp->f_advice == NULL || fp->f_vnode->v_type != VREG)
839 		return (ret);
840 
841 	mtxp = mtx_pool_find(mtxpool_sleep, fp);
842 	mtx_lock(mtxp);
843 	if (fp->f_advice != NULL &&
844 	    uio->uio_offset >= fp->f_advice->fa_start &&
845 	    uio->uio_offset + uio->uio_resid <= fp->f_advice->fa_end)
846 		ret = fp->f_advice->fa_advice;
847 	mtx_unlock(mtxp);
848 	return (ret);
849 }
850 
851 static int
852 vn_read_from_obj(struct vnode *vp, struct uio *uio)
853 {
854 	vm_object_t obj;
855 	vm_page_t ma[io_hold_cnt + 2];
856 	off_t off, vsz;
857 	ssize_t resid;
858 	int error, i, j;
859 
860 	obj = vp->v_object;
861 	MPASS(uio->uio_resid <= ptoa(io_hold_cnt + 2));
862 	MPASS(obj != NULL);
863 	MPASS(obj->type == OBJT_VNODE);
864 
865 	/*
866 	 * Depends on type stability of vm_objects.
867 	 */
868 	vm_object_pip_add(obj, 1);
869 	if ((obj->flags & OBJ_DEAD) != 0) {
870 		/*
871 		 * Note that object might be already reused from the
872 		 * vnode, and the OBJ_DEAD flag cleared.  This is fine,
873 		 * we recheck for DOOMED vnode state after all pages
874 		 * are busied, and retract then.
875 		 *
876 		 * But we check for OBJ_DEAD to ensure that we do not
877 		 * busy pages while vm_object_terminate_pages()
878 		 * processes the queue.
879 		 */
880 		error = EJUSTRETURN;
881 		goto out_pip;
882 	}
883 
884 	resid = uio->uio_resid;
885 	off = uio->uio_offset;
886 	for (i = 0; resid > 0; i++) {
887 		MPASS(i < io_hold_cnt + 2);
888 		ma[i] = vm_page_grab_unlocked(obj, atop(off),
889 		    VM_ALLOC_NOCREAT | VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY |
890 		    VM_ALLOC_NOWAIT);
891 		if (ma[i] == NULL)
892 			break;
893 
894 		/*
895 		 * Skip invalid pages.  Valid mask can be partial only
896 		 * at EOF, and we clip later.
897 		 */
898 		if (vm_page_none_valid(ma[i])) {
899 			vm_page_sunbusy(ma[i]);
900 			break;
901 		}
902 
903 		resid -= PAGE_SIZE;
904 		off += PAGE_SIZE;
905 	}
906 	if (i == 0) {
907 		error = EJUSTRETURN;
908 		goto out_pip;
909 	}
910 
911 	/*
912 	 * Check VIRF_DOOMED after we busied our pages.  Since
913 	 * vgonel() terminates the vnode' vm_object, it cannot
914 	 * process past pages busied by us.
915 	 */
916 	if (VN_IS_DOOMED(vp)) {
917 		error = EJUSTRETURN;
918 		goto out;
919 	}
920 
921 	resid = PAGE_SIZE - (uio->uio_offset & PAGE_MASK) + ptoa(i - 1);
922 	if (resid > uio->uio_resid)
923 		resid = uio->uio_resid;
924 
925 	/*
926 	 * Unlocked read of vnp_size is safe because truncation cannot
927 	 * pass busied page.  But we load vnp_size into a local
928 	 * variable so that possible concurrent extension does not
929 	 * break calculation.
930 	 */
931 #if defined(__powerpc__) && !defined(__powerpc64__)
932 	vsz = obj->un_pager.vnp.vnp_size;
933 #else
934 	vsz = atomic_load_64(&obj->un_pager.vnp.vnp_size);
935 #endif
936 	if (uio->uio_offset + resid > vsz)
937 		resid = vsz - uio->uio_offset;
938 
939 	error = vn_io_fault_pgmove(ma, uio->uio_offset & PAGE_MASK, resid, uio);
940 
941 out:
942 	for (j = 0; j < i; j++) {
943 		if (error == 0)
944 			vm_page_reference(ma[j]);
945 		vm_page_sunbusy(ma[j]);
946 	}
947 out_pip:
948 	vm_object_pip_wakeup(obj);
949 	if (error != 0)
950 		return (error);
951 	return (uio->uio_resid == 0 ? 0 : EJUSTRETURN);
952 }
953 
954 static bool
955 do_vn_read_from_pgcache(struct vnode *vp, struct uio *uio, struct file *fp)
956 {
957 	return ((vp->v_irflag & (VIRF_DOOMED | VIRF_PGREAD)) == VIRF_PGREAD &&
958 	    !mac_vnode_check_read_enabled() &&
959 	    uio->uio_resid <= ptoa(io_hold_cnt) && uio->uio_offset >= 0 &&
960 	    (fp->f_flag & O_DIRECT) == 0 && vn_io_pgcache_read_enable);
961 }
962 
963 /*
964  * File table vnode read routine.
965  */
966 static int
967 vn_read(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags,
968     struct thread *td)
969 {
970 	struct vnode *vp;
971 	off_t orig_offset;
972 	int error, ioflag;
973 	int advice;
974 
975 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
976 	    uio->uio_td, td));
977 	KASSERT(flags & FOF_OFFSET, ("No FOF_OFFSET"));
978 	vp = fp->f_vnode;
979 	if (do_vn_read_from_pgcache(vp, uio, fp)) {
980 		error = vn_read_from_obj(vp, uio);
981 		if (error == 0) {
982 			fp->f_nextoff[UIO_READ] = uio->uio_offset;
983 			return (0);
984 		}
985 		if (error != EJUSTRETURN)
986 			return (error);
987 	}
988 	ioflag = 0;
989 	if (fp->f_flag & FNONBLOCK)
990 		ioflag |= IO_NDELAY;
991 	if (fp->f_flag & O_DIRECT)
992 		ioflag |= IO_DIRECT;
993 	advice = get_advice(fp, uio);
994 	vn_lock(vp, LK_SHARED | LK_RETRY);
995 
996 	switch (advice) {
997 	case POSIX_FADV_NORMAL:
998 	case POSIX_FADV_SEQUENTIAL:
999 	case POSIX_FADV_NOREUSE:
1000 		ioflag |= sequential_heuristic(uio, fp);
1001 		break;
1002 	case POSIX_FADV_RANDOM:
1003 		/* Disable read-ahead for random I/O. */
1004 		break;
1005 	}
1006 	orig_offset = uio->uio_offset;
1007 
1008 #ifdef MAC
1009 	error = mac_vnode_check_read(active_cred, fp->f_cred, vp);
1010 	if (error == 0)
1011 #endif
1012 		error = VOP_READ(vp, uio, ioflag, fp->f_cred);
1013 	fp->f_nextoff[UIO_READ] = uio->uio_offset;
1014 	VOP_UNLOCK(vp);
1015 	if (error == 0 && advice == POSIX_FADV_NOREUSE &&
1016 	    orig_offset != uio->uio_offset)
1017 		/*
1018 		 * Use POSIX_FADV_DONTNEED to flush pages and buffers
1019 		 * for the backing file after a POSIX_FADV_NOREUSE
1020 		 * read(2).
1021 		 */
1022 		error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
1023 		    POSIX_FADV_DONTNEED);
1024 	return (error);
1025 }
1026 
1027 /*
1028  * File table vnode write routine.
1029  */
1030 static int
1031 vn_write(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags,
1032     struct thread *td)
1033 {
1034 	struct vnode *vp;
1035 	struct mount *mp;
1036 	off_t orig_offset;
1037 	int error, ioflag, lock_flags;
1038 	int advice;
1039 
1040 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
1041 	    uio->uio_td, td));
1042 	KASSERT(flags & FOF_OFFSET, ("No FOF_OFFSET"));
1043 	vp = fp->f_vnode;
1044 	if (vp->v_type == VREG)
1045 		bwillwrite();
1046 	ioflag = IO_UNIT;
1047 	if (vp->v_type == VREG && (fp->f_flag & O_APPEND))
1048 		ioflag |= IO_APPEND;
1049 	if (fp->f_flag & FNONBLOCK)
1050 		ioflag |= IO_NDELAY;
1051 	if (fp->f_flag & O_DIRECT)
1052 		ioflag |= IO_DIRECT;
1053 	if ((fp->f_flag & O_FSYNC) ||
1054 	    (vp->v_mount && (vp->v_mount->mnt_flag & MNT_SYNCHRONOUS)))
1055 		ioflag |= IO_SYNC;
1056 	mp = NULL;
1057 	if (vp->v_type != VCHR &&
1058 	    (error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
1059 		goto unlock;
1060 
1061 	advice = get_advice(fp, uio);
1062 
1063 	if (MNT_SHARED_WRITES(mp) ||
1064 	    (mp == NULL && MNT_SHARED_WRITES(vp->v_mount))) {
1065 		lock_flags = LK_SHARED;
1066 	} else {
1067 		lock_flags = LK_EXCLUSIVE;
1068 	}
1069 
1070 	vn_lock(vp, lock_flags | LK_RETRY);
1071 	switch (advice) {
1072 	case POSIX_FADV_NORMAL:
1073 	case POSIX_FADV_SEQUENTIAL:
1074 	case POSIX_FADV_NOREUSE:
1075 		ioflag |= sequential_heuristic(uio, fp);
1076 		break;
1077 	case POSIX_FADV_RANDOM:
1078 		/* XXX: Is this correct? */
1079 		break;
1080 	}
1081 	orig_offset = uio->uio_offset;
1082 
1083 #ifdef MAC
1084 	error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
1085 	if (error == 0)
1086 #endif
1087 		error = VOP_WRITE(vp, uio, ioflag, fp->f_cred);
1088 	fp->f_nextoff[UIO_WRITE] = uio->uio_offset;
1089 	VOP_UNLOCK(vp);
1090 	if (vp->v_type != VCHR)
1091 		vn_finished_write(mp);
1092 	if (error == 0 && advice == POSIX_FADV_NOREUSE &&
1093 	    orig_offset != uio->uio_offset)
1094 		/*
1095 		 * Use POSIX_FADV_DONTNEED to flush pages and buffers
1096 		 * for the backing file after a POSIX_FADV_NOREUSE
1097 		 * write(2).
1098 		 */
1099 		error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
1100 		    POSIX_FADV_DONTNEED);
1101 unlock:
1102 	return (error);
1103 }
1104 
1105 /*
1106  * The vn_io_fault() is a wrapper around vn_read() and vn_write() to
1107  * prevent the following deadlock:
1108  *
1109  * Assume that the thread A reads from the vnode vp1 into userspace
1110  * buffer buf1 backed by the pages of vnode vp2.  If a page in buf1 is
1111  * currently not resident, then system ends up with the call chain
1112  *   vn_read() -> VOP_READ(vp1) -> uiomove() -> [Page Fault] ->
1113  *     vm_fault(buf1) -> vnode_pager_getpages(vp2) -> VOP_GETPAGES(vp2)
1114  * which establishes lock order vp1->vn_lock, then vp2->vn_lock.
1115  * If, at the same time, thread B reads from vnode vp2 into buffer buf2
1116  * backed by the pages of vnode vp1, and some page in buf2 is not
1117  * resident, we get a reversed order vp2->vn_lock, then vp1->vn_lock.
1118  *
1119  * To prevent the lock order reversal and deadlock, vn_io_fault() does
1120  * not allow page faults to happen during VOP_READ() or VOP_WRITE().
1121  * Instead, it first tries to do the whole range i/o with pagefaults
1122  * disabled. If all pages in the i/o buffer are resident and mapped,
1123  * VOP will succeed (ignoring the genuine filesystem errors).
1124  * Otherwise, we get back EFAULT, and vn_io_fault() falls back to do
1125  * i/o in chunks, with all pages in the chunk prefaulted and held
1126  * using vm_fault_quick_hold_pages().
1127  *
1128  * Filesystems using this deadlock avoidance scheme should use the
1129  * array of the held pages from uio, saved in the curthread->td_ma,
1130  * instead of doing uiomove().  A helper function
1131  * vn_io_fault_uiomove() converts uiomove request into
1132  * uiomove_fromphys() over td_ma array.
1133  *
1134  * Since vnode locks do not cover the whole i/o anymore, rangelocks
1135  * make the current i/o request atomic with respect to other i/os and
1136  * truncations.
1137  */
1138 
1139 /*
1140  * Decode vn_io_fault_args and perform the corresponding i/o.
1141  */
1142 static int
1143 vn_io_fault_doio(struct vn_io_fault_args *args, struct uio *uio,
1144     struct thread *td)
1145 {
1146 	int error, save;
1147 
1148 	error = 0;
1149 	save = vm_fault_disable_pagefaults();
1150 	switch (args->kind) {
1151 	case VN_IO_FAULT_FOP:
1152 		error = (args->args.fop_args.doio)(args->args.fop_args.fp,
1153 		    uio, args->cred, args->flags, td);
1154 		break;
1155 	case VN_IO_FAULT_VOP:
1156 		if (uio->uio_rw == UIO_READ) {
1157 			error = VOP_READ(args->args.vop_args.vp, uio,
1158 			    args->flags, args->cred);
1159 		} else if (uio->uio_rw == UIO_WRITE) {
1160 			error = VOP_WRITE(args->args.vop_args.vp, uio,
1161 			    args->flags, args->cred);
1162 		}
1163 		break;
1164 	default:
1165 		panic("vn_io_fault_doio: unknown kind of io %d %d",
1166 		    args->kind, uio->uio_rw);
1167 	}
1168 	vm_fault_enable_pagefaults(save);
1169 	return (error);
1170 }
1171 
1172 static int
1173 vn_io_fault_touch(char *base, const struct uio *uio)
1174 {
1175 	int r;
1176 
1177 	r = fubyte(base);
1178 	if (r == -1 || (uio->uio_rw == UIO_READ && subyte(base, r) == -1))
1179 		return (EFAULT);
1180 	return (0);
1181 }
1182 
1183 static int
1184 vn_io_fault_prefault_user(const struct uio *uio)
1185 {
1186 	char *base;
1187 	const struct iovec *iov;
1188 	size_t len;
1189 	ssize_t resid;
1190 	int error, i;
1191 
1192 	KASSERT(uio->uio_segflg == UIO_USERSPACE,
1193 	    ("vn_io_fault_prefault userspace"));
1194 
1195 	error = i = 0;
1196 	iov = uio->uio_iov;
1197 	resid = uio->uio_resid;
1198 	base = iov->iov_base;
1199 	len = iov->iov_len;
1200 	while (resid > 0) {
1201 		error = vn_io_fault_touch(base, uio);
1202 		if (error != 0)
1203 			break;
1204 		if (len < PAGE_SIZE) {
1205 			if (len != 0) {
1206 				error = vn_io_fault_touch(base + len - 1, uio);
1207 				if (error != 0)
1208 					break;
1209 				resid -= len;
1210 			}
1211 			if (++i >= uio->uio_iovcnt)
1212 				break;
1213 			iov = uio->uio_iov + i;
1214 			base = iov->iov_base;
1215 			len = iov->iov_len;
1216 		} else {
1217 			len -= PAGE_SIZE;
1218 			base += PAGE_SIZE;
1219 			resid -= PAGE_SIZE;
1220 		}
1221 	}
1222 	return (error);
1223 }
1224 
1225 /*
1226  * Common code for vn_io_fault(), agnostic to the kind of i/o request.
1227  * Uses vn_io_fault_doio() to make the call to an actual i/o function.
1228  * Used from vn_rdwr() and vn_io_fault(), which encode the i/o request
1229  * into args and call vn_io_fault1() to handle faults during the user
1230  * mode buffer accesses.
1231  */
1232 static int
1233 vn_io_fault1(struct vnode *vp, struct uio *uio, struct vn_io_fault_args *args,
1234     struct thread *td)
1235 {
1236 	vm_page_t ma[io_hold_cnt + 2];
1237 	struct uio *uio_clone, short_uio;
1238 	struct iovec short_iovec[1];
1239 	vm_page_t *prev_td_ma;
1240 	vm_prot_t prot;
1241 	vm_offset_t addr, end;
1242 	size_t len, resid;
1243 	ssize_t adv;
1244 	int error, cnt, saveheld, prev_td_ma_cnt;
1245 
1246 	if (vn_io_fault_prefault) {
1247 		error = vn_io_fault_prefault_user(uio);
1248 		if (error != 0)
1249 			return (error); /* Or ignore ? */
1250 	}
1251 
1252 	prot = uio->uio_rw == UIO_READ ? VM_PROT_WRITE : VM_PROT_READ;
1253 
1254 	/*
1255 	 * The UFS follows IO_UNIT directive and replays back both
1256 	 * uio_offset and uio_resid if an error is encountered during the
1257 	 * operation.  But, since the iovec may be already advanced,
1258 	 * uio is still in an inconsistent state.
1259 	 *
1260 	 * Cache a copy of the original uio, which is advanced to the redo
1261 	 * point using UIO_NOCOPY below.
1262 	 */
1263 	uio_clone = cloneuio(uio);
1264 	resid = uio->uio_resid;
1265 
1266 	short_uio.uio_segflg = UIO_USERSPACE;
1267 	short_uio.uio_rw = uio->uio_rw;
1268 	short_uio.uio_td = uio->uio_td;
1269 
1270 	error = vn_io_fault_doio(args, uio, td);
1271 	if (error != EFAULT)
1272 		goto out;
1273 
1274 	atomic_add_long(&vn_io_faults_cnt, 1);
1275 	uio_clone->uio_segflg = UIO_NOCOPY;
1276 	uiomove(NULL, resid - uio->uio_resid, uio_clone);
1277 	uio_clone->uio_segflg = uio->uio_segflg;
1278 
1279 	saveheld = curthread_pflags_set(TDP_UIOHELD);
1280 	prev_td_ma = td->td_ma;
1281 	prev_td_ma_cnt = td->td_ma_cnt;
1282 
1283 	while (uio_clone->uio_resid != 0) {
1284 		len = uio_clone->uio_iov->iov_len;
1285 		if (len == 0) {
1286 			KASSERT(uio_clone->uio_iovcnt >= 1,
1287 			    ("iovcnt underflow"));
1288 			uio_clone->uio_iov++;
1289 			uio_clone->uio_iovcnt--;
1290 			continue;
1291 		}
1292 		if (len > ptoa(io_hold_cnt))
1293 			len = ptoa(io_hold_cnt);
1294 		addr = (uintptr_t)uio_clone->uio_iov->iov_base;
1295 		end = round_page(addr + len);
1296 		if (end < addr) {
1297 			error = EFAULT;
1298 			break;
1299 		}
1300 		cnt = atop(end - trunc_page(addr));
1301 		/*
1302 		 * A perfectly misaligned address and length could cause
1303 		 * both the start and the end of the chunk to use partial
1304 		 * page.  +2 accounts for such a situation.
1305 		 */
1306 		cnt = vm_fault_quick_hold_pages(&td->td_proc->p_vmspace->vm_map,
1307 		    addr, len, prot, ma, io_hold_cnt + 2);
1308 		if (cnt == -1) {
1309 			error = EFAULT;
1310 			break;
1311 		}
1312 		short_uio.uio_iov = &short_iovec[0];
1313 		short_iovec[0].iov_base = (void *)addr;
1314 		short_uio.uio_iovcnt = 1;
1315 		short_uio.uio_resid = short_iovec[0].iov_len = len;
1316 		short_uio.uio_offset = uio_clone->uio_offset;
1317 		td->td_ma = ma;
1318 		td->td_ma_cnt = cnt;
1319 
1320 		error = vn_io_fault_doio(args, &short_uio, td);
1321 		vm_page_unhold_pages(ma, cnt);
1322 		adv = len - short_uio.uio_resid;
1323 
1324 		uio_clone->uio_iov->iov_base =
1325 		    (char *)uio_clone->uio_iov->iov_base + adv;
1326 		uio_clone->uio_iov->iov_len -= adv;
1327 		uio_clone->uio_resid -= adv;
1328 		uio_clone->uio_offset += adv;
1329 
1330 		uio->uio_resid -= adv;
1331 		uio->uio_offset += adv;
1332 
1333 		if (error != 0 || adv == 0)
1334 			break;
1335 	}
1336 	td->td_ma = prev_td_ma;
1337 	td->td_ma_cnt = prev_td_ma_cnt;
1338 	curthread_pflags_restore(saveheld);
1339 out:
1340 	free(uio_clone, M_IOV);
1341 	return (error);
1342 }
1343 
1344 static int
1345 vn_io_fault(struct file *fp, struct uio *uio, struct ucred *active_cred,
1346     int flags, struct thread *td)
1347 {
1348 	fo_rdwr_t *doio;
1349 	struct vnode *vp;
1350 	void *rl_cookie;
1351 	struct vn_io_fault_args args;
1352 	int error;
1353 
1354 	doio = uio->uio_rw == UIO_READ ? vn_read : vn_write;
1355 	vp = fp->f_vnode;
1356 
1357 	/*
1358 	 * The ability to read(2) on a directory has historically been
1359 	 * allowed for all users, but this can and has been the source of
1360 	 * at least one security issue in the past.  As such, it is now hidden
1361 	 * away behind a sysctl for those that actually need it to use it, and
1362 	 * restricted to root when it's turned on to make it relatively safe to
1363 	 * leave on for longer sessions of need.
1364 	 */
1365 	if (vp->v_type == VDIR) {
1366 		KASSERT(uio->uio_rw == UIO_READ,
1367 		    ("illegal write attempted on a directory"));
1368 		if (!vfs_allow_read_dir)
1369 			return (EISDIR);
1370 		if ((error = priv_check(td, PRIV_VFS_READ_DIR)) != 0)
1371 			return (EISDIR);
1372 	}
1373 
1374 	foffset_lock_uio(fp, uio, flags);
1375 	if (do_vn_io_fault(vp, uio)) {
1376 		args.kind = VN_IO_FAULT_FOP;
1377 		args.args.fop_args.fp = fp;
1378 		args.args.fop_args.doio = doio;
1379 		args.cred = active_cred;
1380 		args.flags = flags | FOF_OFFSET;
1381 		if (uio->uio_rw == UIO_READ) {
1382 			rl_cookie = vn_rangelock_rlock(vp, uio->uio_offset,
1383 			    uio->uio_offset + uio->uio_resid);
1384 		} else if ((fp->f_flag & O_APPEND) != 0 ||
1385 		    (flags & FOF_OFFSET) == 0) {
1386 			/* For appenders, punt and lock the whole range. */
1387 			rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX);
1388 		} else {
1389 			rl_cookie = vn_rangelock_wlock(vp, uio->uio_offset,
1390 			    uio->uio_offset + uio->uio_resid);
1391 		}
1392 		error = vn_io_fault1(vp, uio, &args, td);
1393 		vn_rangelock_unlock(vp, rl_cookie);
1394 	} else {
1395 		error = doio(fp, uio, active_cred, flags | FOF_OFFSET, td);
1396 	}
1397 	foffset_unlock_uio(fp, uio, flags);
1398 	return (error);
1399 }
1400 
1401 /*
1402  * Helper function to perform the requested uiomove operation using
1403  * the held pages for io->uio_iov[0].iov_base buffer instead of
1404  * copyin/copyout.  Access to the pages with uiomove_fromphys()
1405  * instead of iov_base prevents page faults that could occur due to
1406  * pmap_collect() invalidating the mapping created by
1407  * vm_fault_quick_hold_pages(), or pageout daemon, page laundry or
1408  * object cleanup revoking the write access from page mappings.
1409  *
1410  * Filesystems specified MNTK_NO_IOPF shall use vn_io_fault_uiomove()
1411  * instead of plain uiomove().
1412  */
1413 int
1414 vn_io_fault_uiomove(char *data, int xfersize, struct uio *uio)
1415 {
1416 	struct uio transp_uio;
1417 	struct iovec transp_iov[1];
1418 	struct thread *td;
1419 	size_t adv;
1420 	int error, pgadv;
1421 
1422 	td = curthread;
1423 	if ((td->td_pflags & TDP_UIOHELD) == 0 ||
1424 	    uio->uio_segflg != UIO_USERSPACE)
1425 		return (uiomove(data, xfersize, uio));
1426 
1427 	KASSERT(uio->uio_iovcnt == 1, ("uio_iovcnt %d", uio->uio_iovcnt));
1428 	transp_iov[0].iov_base = data;
1429 	transp_uio.uio_iov = &transp_iov[0];
1430 	transp_uio.uio_iovcnt = 1;
1431 	if (xfersize > uio->uio_resid)
1432 		xfersize = uio->uio_resid;
1433 	transp_uio.uio_resid = transp_iov[0].iov_len = xfersize;
1434 	transp_uio.uio_offset = 0;
1435 	transp_uio.uio_segflg = UIO_SYSSPACE;
1436 	/*
1437 	 * Since transp_iov points to data, and td_ma page array
1438 	 * corresponds to original uio->uio_iov, we need to invert the
1439 	 * direction of the i/o operation as passed to
1440 	 * uiomove_fromphys().
1441 	 */
1442 	switch (uio->uio_rw) {
1443 	case UIO_WRITE:
1444 		transp_uio.uio_rw = UIO_READ;
1445 		break;
1446 	case UIO_READ:
1447 		transp_uio.uio_rw = UIO_WRITE;
1448 		break;
1449 	}
1450 	transp_uio.uio_td = uio->uio_td;
1451 	error = uiomove_fromphys(td->td_ma,
1452 	    ((vm_offset_t)uio->uio_iov->iov_base) & PAGE_MASK,
1453 	    xfersize, &transp_uio);
1454 	adv = xfersize - transp_uio.uio_resid;
1455 	pgadv =
1456 	    (((vm_offset_t)uio->uio_iov->iov_base + adv) >> PAGE_SHIFT) -
1457 	    (((vm_offset_t)uio->uio_iov->iov_base) >> PAGE_SHIFT);
1458 	td->td_ma += pgadv;
1459 	KASSERT(td->td_ma_cnt >= pgadv, ("consumed pages %d %d", td->td_ma_cnt,
1460 	    pgadv));
1461 	td->td_ma_cnt -= pgadv;
1462 	uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + adv;
1463 	uio->uio_iov->iov_len -= adv;
1464 	uio->uio_resid -= adv;
1465 	uio->uio_offset += adv;
1466 	return (error);
1467 }
1468 
1469 int
1470 vn_io_fault_pgmove(vm_page_t ma[], vm_offset_t offset, int xfersize,
1471     struct uio *uio)
1472 {
1473 	struct thread *td;
1474 	vm_offset_t iov_base;
1475 	int cnt, pgadv;
1476 
1477 	td = curthread;
1478 	if ((td->td_pflags & TDP_UIOHELD) == 0 ||
1479 	    uio->uio_segflg != UIO_USERSPACE)
1480 		return (uiomove_fromphys(ma, offset, xfersize, uio));
1481 
1482 	KASSERT(uio->uio_iovcnt == 1, ("uio_iovcnt %d", uio->uio_iovcnt));
1483 	cnt = xfersize > uio->uio_resid ? uio->uio_resid : xfersize;
1484 	iov_base = (vm_offset_t)uio->uio_iov->iov_base;
1485 	switch (uio->uio_rw) {
1486 	case UIO_WRITE:
1487 		pmap_copy_pages(td->td_ma, iov_base & PAGE_MASK, ma,
1488 		    offset, cnt);
1489 		break;
1490 	case UIO_READ:
1491 		pmap_copy_pages(ma, offset, td->td_ma, iov_base & PAGE_MASK,
1492 		    cnt);
1493 		break;
1494 	}
1495 	pgadv = ((iov_base + cnt) >> PAGE_SHIFT) - (iov_base >> PAGE_SHIFT);
1496 	td->td_ma += pgadv;
1497 	KASSERT(td->td_ma_cnt >= pgadv, ("consumed pages %d %d", td->td_ma_cnt,
1498 	    pgadv));
1499 	td->td_ma_cnt -= pgadv;
1500 	uio->uio_iov->iov_base = (char *)(iov_base + cnt);
1501 	uio->uio_iov->iov_len -= cnt;
1502 	uio->uio_resid -= cnt;
1503 	uio->uio_offset += cnt;
1504 	return (0);
1505 }
1506 
1507 /*
1508  * File table truncate routine.
1509  */
1510 static int
1511 vn_truncate(struct file *fp, off_t length, struct ucred *active_cred,
1512     struct thread *td)
1513 {
1514 	struct mount *mp;
1515 	struct vnode *vp;
1516 	void *rl_cookie;
1517 	int error;
1518 
1519 	vp = fp->f_vnode;
1520 
1521 	/*
1522 	 * Lock the whole range for truncation.  Otherwise split i/o
1523 	 * might happen partly before and partly after the truncation.
1524 	 */
1525 	rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX);
1526 	error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
1527 	if (error)
1528 		goto out1;
1529 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1530 	AUDIT_ARG_VNODE1(vp);
1531 	if (vp->v_type == VDIR) {
1532 		error = EISDIR;
1533 		goto out;
1534 	}
1535 #ifdef MAC
1536 	error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
1537 	if (error)
1538 		goto out;
1539 #endif
1540 	error = vn_truncate_locked(vp, length, (fp->f_flag & O_FSYNC) != 0,
1541 	    fp->f_cred);
1542 out:
1543 	VOP_UNLOCK(vp);
1544 	vn_finished_write(mp);
1545 out1:
1546 	vn_rangelock_unlock(vp, rl_cookie);
1547 	return (error);
1548 }
1549 
1550 /*
1551  * Truncate a file that is already locked.
1552  */
1553 int
1554 vn_truncate_locked(struct vnode *vp, off_t length, bool sync,
1555     struct ucred *cred)
1556 {
1557 	struct vattr vattr;
1558 	int error;
1559 
1560 	error = VOP_ADD_WRITECOUNT(vp, 1);
1561 	if (error == 0) {
1562 		VATTR_NULL(&vattr);
1563 		vattr.va_size = length;
1564 		if (sync)
1565 			vattr.va_vaflags |= VA_SYNC;
1566 		error = VOP_SETATTR(vp, &vattr, cred);
1567 		VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
1568 	}
1569 	return (error);
1570 }
1571 
1572 /*
1573  * File table vnode stat routine.
1574  */
1575 static int
1576 vn_statfile(struct file *fp, struct stat *sb, struct ucred *active_cred,
1577     struct thread *td)
1578 {
1579 	struct vnode *vp = fp->f_vnode;
1580 	int error;
1581 
1582 	vn_lock(vp, LK_SHARED | LK_RETRY);
1583 	error = VOP_STAT(vp, sb, active_cred, fp->f_cred, td);
1584 	VOP_UNLOCK(vp);
1585 
1586 	return (error);
1587 }
1588 
1589 /*
1590  * File table vnode ioctl routine.
1591  */
1592 static int
1593 vn_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred,
1594     struct thread *td)
1595 {
1596 	struct vattr vattr;
1597 	struct vnode *vp;
1598 	struct fiobmap2_arg *bmarg;
1599 	int error;
1600 
1601 	vp = fp->f_vnode;
1602 	switch (vp->v_type) {
1603 	case VDIR:
1604 	case VREG:
1605 		switch (com) {
1606 		case FIONREAD:
1607 			vn_lock(vp, LK_SHARED | LK_RETRY);
1608 			error = VOP_GETATTR(vp, &vattr, active_cred);
1609 			VOP_UNLOCK(vp);
1610 			if (error == 0)
1611 				*(int *)data = vattr.va_size - fp->f_offset;
1612 			return (error);
1613 		case FIOBMAP2:
1614 			bmarg = (struct fiobmap2_arg *)data;
1615 			vn_lock(vp, LK_SHARED | LK_RETRY);
1616 #ifdef MAC
1617 			error = mac_vnode_check_read(active_cred, fp->f_cred,
1618 			    vp);
1619 			if (error == 0)
1620 #endif
1621 				error = VOP_BMAP(vp, bmarg->bn, NULL,
1622 				    &bmarg->bn, &bmarg->runp, &bmarg->runb);
1623 			VOP_UNLOCK(vp);
1624 			return (error);
1625 		case FIONBIO:
1626 		case FIOASYNC:
1627 			return (0);
1628 		default:
1629 			return (VOP_IOCTL(vp, com, data, fp->f_flag,
1630 			    active_cred, td));
1631 		}
1632 		break;
1633 	case VCHR:
1634 		return (VOP_IOCTL(vp, com, data, fp->f_flag,
1635 		    active_cred, td));
1636 	default:
1637 		return (ENOTTY);
1638 	}
1639 }
1640 
1641 /*
1642  * File table vnode poll routine.
1643  */
1644 static int
1645 vn_poll(struct file *fp, int events, struct ucred *active_cred,
1646     struct thread *td)
1647 {
1648 	struct vnode *vp;
1649 	int error;
1650 
1651 	vp = fp->f_vnode;
1652 #if defined(MAC) || defined(AUDIT)
1653 	if (AUDITING_TD(td) || mac_vnode_check_poll_enabled()) {
1654 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1655 		AUDIT_ARG_VNODE1(vp);
1656 		error = mac_vnode_check_poll(active_cred, fp->f_cred, vp);
1657 		VOP_UNLOCK(vp);
1658 		if (error != 0)
1659 			return (error);
1660 	}
1661 #endif
1662 	error = VOP_POLL(vp, events, fp->f_cred, td);
1663 	return (error);
1664 }
1665 
1666 /*
1667  * Acquire the requested lock and then check for validity.  LK_RETRY
1668  * permits vn_lock to return doomed vnodes.
1669  */
1670 static int __noinline
1671 _vn_lock_fallback(struct vnode *vp, int flags, const char *file, int line,
1672     int error)
1673 {
1674 
1675 	KASSERT((flags & LK_RETRY) == 0 || error == 0,
1676 	    ("vn_lock: error %d incompatible with flags %#x", error, flags));
1677 
1678 	if (error == 0)
1679 		VNASSERT(VN_IS_DOOMED(vp), vp, ("vnode not doomed"));
1680 
1681 	if ((flags & LK_RETRY) == 0) {
1682 		if (error == 0) {
1683 			VOP_UNLOCK(vp);
1684 			error = ENOENT;
1685 		}
1686 		return (error);
1687 	}
1688 
1689 	/*
1690 	 * LK_RETRY case.
1691 	 *
1692 	 * Nothing to do if we got the lock.
1693 	 */
1694 	if (error == 0)
1695 		return (0);
1696 
1697 	/*
1698 	 * Interlock was dropped by the call in _vn_lock.
1699 	 */
1700 	flags &= ~LK_INTERLOCK;
1701 	do {
1702 		error = VOP_LOCK1(vp, flags, file, line);
1703 	} while (error != 0);
1704 	return (0);
1705 }
1706 
1707 int
1708 _vn_lock(struct vnode *vp, int flags, const char *file, int line)
1709 {
1710 	int error;
1711 
1712 	VNASSERT((flags & LK_TYPE_MASK) != 0, vp,
1713 	    ("vn_lock: no locktype (%d passed)", flags));
1714 	VNPASS(vp->v_holdcnt > 0, vp);
1715 	error = VOP_LOCK1(vp, flags, file, line);
1716 	if (__predict_false(error != 0 || VN_IS_DOOMED(vp)))
1717 		return (_vn_lock_fallback(vp, flags, file, line, error));
1718 	return (0);
1719 }
1720 
1721 /*
1722  * File table vnode close routine.
1723  */
1724 static int
1725 vn_closefile(struct file *fp, struct thread *td)
1726 {
1727 	struct vnode *vp;
1728 	struct flock lf;
1729 	int error;
1730 	bool ref;
1731 
1732 	vp = fp->f_vnode;
1733 	fp->f_ops = &badfileops;
1734 	ref= (fp->f_flag & FHASLOCK) != 0 && fp->f_type == DTYPE_VNODE;
1735 
1736 	error = vn_close1(vp, fp->f_flag, fp->f_cred, td, ref);
1737 
1738 	if (__predict_false(ref)) {
1739 		lf.l_whence = SEEK_SET;
1740 		lf.l_start = 0;
1741 		lf.l_len = 0;
1742 		lf.l_type = F_UNLCK;
1743 		(void) VOP_ADVLOCK(vp, fp, F_UNLCK, &lf, F_FLOCK);
1744 		vrele(vp);
1745 	}
1746 	return (error);
1747 }
1748 
1749 /*
1750  * Preparing to start a filesystem write operation. If the operation is
1751  * permitted, then we bump the count of operations in progress and
1752  * proceed. If a suspend request is in progress, we wait until the
1753  * suspension is over, and then proceed.
1754  */
1755 static int
1756 vn_start_write_refed(struct mount *mp, int flags, bool mplocked)
1757 {
1758 	int error, mflags;
1759 
1760 	if (__predict_true(!mplocked) && (flags & V_XSLEEP) == 0 &&
1761 	    vfs_op_thread_enter(mp)) {
1762 		MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0);
1763 		vfs_mp_count_add_pcpu(mp, writeopcount, 1);
1764 		vfs_op_thread_exit(mp);
1765 		return (0);
1766 	}
1767 
1768 	if (mplocked)
1769 		mtx_assert(MNT_MTX(mp), MA_OWNED);
1770 	else
1771 		MNT_ILOCK(mp);
1772 
1773 	error = 0;
1774 
1775 	/*
1776 	 * Check on status of suspension.
1777 	 */
1778 	if ((curthread->td_pflags & TDP_IGNSUSP) == 0 ||
1779 	    mp->mnt_susp_owner != curthread) {
1780 		mflags = ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ?
1781 		    (flags & PCATCH) : 0) | (PUSER - 1);
1782 		while ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) {
1783 			if (flags & V_NOWAIT) {
1784 				error = EWOULDBLOCK;
1785 				goto unlock;
1786 			}
1787 			error = msleep(&mp->mnt_flag, MNT_MTX(mp), mflags,
1788 			    "suspfs", 0);
1789 			if (error)
1790 				goto unlock;
1791 		}
1792 	}
1793 	if (flags & V_XSLEEP)
1794 		goto unlock;
1795 	mp->mnt_writeopcount++;
1796 unlock:
1797 	if (error != 0 || (flags & V_XSLEEP) != 0)
1798 		MNT_REL(mp);
1799 	MNT_IUNLOCK(mp);
1800 	return (error);
1801 }
1802 
1803 int
1804 vn_start_write(struct vnode *vp, struct mount **mpp, int flags)
1805 {
1806 	struct mount *mp;
1807 	int error;
1808 
1809 	KASSERT((flags & V_MNTREF) == 0 || (*mpp != NULL && vp == NULL),
1810 	    ("V_MNTREF requires mp"));
1811 
1812 	error = 0;
1813 	/*
1814 	 * If a vnode is provided, get and return the mount point that
1815 	 * to which it will write.
1816 	 */
1817 	if (vp != NULL) {
1818 		if ((error = VOP_GETWRITEMOUNT(vp, mpp)) != 0) {
1819 			*mpp = NULL;
1820 			if (error != EOPNOTSUPP)
1821 				return (error);
1822 			return (0);
1823 		}
1824 	}
1825 	if ((mp = *mpp) == NULL)
1826 		return (0);
1827 
1828 	/*
1829 	 * VOP_GETWRITEMOUNT() returns with the mp refcount held through
1830 	 * a vfs_ref().
1831 	 * As long as a vnode is not provided we need to acquire a
1832 	 * refcount for the provided mountpoint too, in order to
1833 	 * emulate a vfs_ref().
1834 	 */
1835 	if (vp == NULL && (flags & V_MNTREF) == 0)
1836 		vfs_ref(mp);
1837 
1838 	return (vn_start_write_refed(mp, flags, false));
1839 }
1840 
1841 /*
1842  * Secondary suspension. Used by operations such as vop_inactive
1843  * routines that are needed by the higher level functions. These
1844  * are allowed to proceed until all the higher level functions have
1845  * completed (indicated by mnt_writeopcount dropping to zero). At that
1846  * time, these operations are halted until the suspension is over.
1847  */
1848 int
1849 vn_start_secondary_write(struct vnode *vp, struct mount **mpp, int flags)
1850 {
1851 	struct mount *mp;
1852 	int error;
1853 
1854 	KASSERT((flags & V_MNTREF) == 0 || (*mpp != NULL && vp == NULL),
1855 	    ("V_MNTREF requires mp"));
1856 
1857  retry:
1858 	if (vp != NULL) {
1859 		if ((error = VOP_GETWRITEMOUNT(vp, mpp)) != 0) {
1860 			*mpp = NULL;
1861 			if (error != EOPNOTSUPP)
1862 				return (error);
1863 			return (0);
1864 		}
1865 	}
1866 	/*
1867 	 * If we are not suspended or have not yet reached suspended
1868 	 * mode, then let the operation proceed.
1869 	 */
1870 	if ((mp = *mpp) == NULL)
1871 		return (0);
1872 
1873 	/*
1874 	 * VOP_GETWRITEMOUNT() returns with the mp refcount held through
1875 	 * a vfs_ref().
1876 	 * As long as a vnode is not provided we need to acquire a
1877 	 * refcount for the provided mountpoint too, in order to
1878 	 * emulate a vfs_ref().
1879 	 */
1880 	MNT_ILOCK(mp);
1881 	if (vp == NULL && (flags & V_MNTREF) == 0)
1882 		MNT_REF(mp);
1883 	if ((mp->mnt_kern_flag & (MNTK_SUSPENDED | MNTK_SUSPEND2)) == 0) {
1884 		mp->mnt_secondary_writes++;
1885 		mp->mnt_secondary_accwrites++;
1886 		MNT_IUNLOCK(mp);
1887 		return (0);
1888 	}
1889 	if (flags & V_NOWAIT) {
1890 		MNT_REL(mp);
1891 		MNT_IUNLOCK(mp);
1892 		return (EWOULDBLOCK);
1893 	}
1894 	/*
1895 	 * Wait for the suspension to finish.
1896 	 */
1897 	error = msleep(&mp->mnt_flag, MNT_MTX(mp), (PUSER - 1) | PDROP |
1898 	    ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ? (flags & PCATCH) : 0),
1899 	    "suspfs", 0);
1900 	vfs_rel(mp);
1901 	if (error == 0)
1902 		goto retry;
1903 	return (error);
1904 }
1905 
1906 /*
1907  * Filesystem write operation has completed. If we are suspending and this
1908  * operation is the last one, notify the suspender that the suspension is
1909  * now in effect.
1910  */
1911 void
1912 vn_finished_write(struct mount *mp)
1913 {
1914 	int c;
1915 
1916 	if (mp == NULL)
1917 		return;
1918 
1919 	if (vfs_op_thread_enter(mp)) {
1920 		vfs_mp_count_sub_pcpu(mp, writeopcount, 1);
1921 		vfs_mp_count_sub_pcpu(mp, ref, 1);
1922 		vfs_op_thread_exit(mp);
1923 		return;
1924 	}
1925 
1926 	MNT_ILOCK(mp);
1927 	vfs_assert_mount_counters(mp);
1928 	MNT_REL(mp);
1929 	c = --mp->mnt_writeopcount;
1930 	if (mp->mnt_vfs_ops == 0) {
1931 		MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0);
1932 		MNT_IUNLOCK(mp);
1933 		return;
1934 	}
1935 	if (c < 0)
1936 		vfs_dump_mount_counters(mp);
1937 	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 && c == 0)
1938 		wakeup(&mp->mnt_writeopcount);
1939 	MNT_IUNLOCK(mp);
1940 }
1941 
1942 /*
1943  * Filesystem secondary write operation has completed. If we are
1944  * suspending and this operation is the last one, notify the suspender
1945  * that the suspension is now in effect.
1946  */
1947 void
1948 vn_finished_secondary_write(struct mount *mp)
1949 {
1950 	if (mp == NULL)
1951 		return;
1952 	MNT_ILOCK(mp);
1953 	MNT_REL(mp);
1954 	mp->mnt_secondary_writes--;
1955 	if (mp->mnt_secondary_writes < 0)
1956 		panic("vn_finished_secondary_write: neg cnt");
1957 	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 &&
1958 	    mp->mnt_secondary_writes <= 0)
1959 		wakeup(&mp->mnt_secondary_writes);
1960 	MNT_IUNLOCK(mp);
1961 }
1962 
1963 /*
1964  * Request a filesystem to suspend write operations.
1965  */
1966 int
1967 vfs_write_suspend(struct mount *mp, int flags)
1968 {
1969 	int error;
1970 
1971 	vfs_op_enter(mp);
1972 
1973 	MNT_ILOCK(mp);
1974 	vfs_assert_mount_counters(mp);
1975 	if (mp->mnt_susp_owner == curthread) {
1976 		vfs_op_exit_locked(mp);
1977 		MNT_IUNLOCK(mp);
1978 		return (EALREADY);
1979 	}
1980 	while (mp->mnt_kern_flag & MNTK_SUSPEND)
1981 		msleep(&mp->mnt_flag, MNT_MTX(mp), PUSER - 1, "wsuspfs", 0);
1982 
1983 	/*
1984 	 * Unmount holds a write reference on the mount point.  If we
1985 	 * own busy reference and drain for writers, we deadlock with
1986 	 * the reference draining in the unmount path.  Callers of
1987 	 * vfs_write_suspend() must specify VS_SKIP_UNMOUNT if
1988 	 * vfs_busy() reference is owned and caller is not in the
1989 	 * unmount context.
1990 	 */
1991 	if ((flags & VS_SKIP_UNMOUNT) != 0 &&
1992 	    (mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) {
1993 		vfs_op_exit_locked(mp);
1994 		MNT_IUNLOCK(mp);
1995 		return (EBUSY);
1996 	}
1997 
1998 	mp->mnt_kern_flag |= MNTK_SUSPEND;
1999 	mp->mnt_susp_owner = curthread;
2000 	if (mp->mnt_writeopcount > 0)
2001 		(void) msleep(&mp->mnt_writeopcount,
2002 		    MNT_MTX(mp), (PUSER - 1)|PDROP, "suspwt", 0);
2003 	else
2004 		MNT_IUNLOCK(mp);
2005 	if ((error = VFS_SYNC(mp, MNT_SUSPEND)) != 0) {
2006 		vfs_write_resume(mp, 0);
2007 		/* vfs_write_resume does vfs_op_exit() for us */
2008 	}
2009 	return (error);
2010 }
2011 
2012 /*
2013  * Request a filesystem to resume write operations.
2014  */
2015 void
2016 vfs_write_resume(struct mount *mp, int flags)
2017 {
2018 
2019 	MNT_ILOCK(mp);
2020 	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) {
2021 		KASSERT(mp->mnt_susp_owner == curthread, ("mnt_susp_owner"));
2022 		mp->mnt_kern_flag &= ~(MNTK_SUSPEND | MNTK_SUSPEND2 |
2023 				       MNTK_SUSPENDED);
2024 		mp->mnt_susp_owner = NULL;
2025 		wakeup(&mp->mnt_writeopcount);
2026 		wakeup(&mp->mnt_flag);
2027 		curthread->td_pflags &= ~TDP_IGNSUSP;
2028 		if ((flags & VR_START_WRITE) != 0) {
2029 			MNT_REF(mp);
2030 			mp->mnt_writeopcount++;
2031 		}
2032 		MNT_IUNLOCK(mp);
2033 		if ((flags & VR_NO_SUSPCLR) == 0)
2034 			VFS_SUSP_CLEAN(mp);
2035 		vfs_op_exit(mp);
2036 	} else if ((flags & VR_START_WRITE) != 0) {
2037 		MNT_REF(mp);
2038 		vn_start_write_refed(mp, 0, true);
2039 	} else {
2040 		MNT_IUNLOCK(mp);
2041 	}
2042 }
2043 
2044 /*
2045  * Helper loop around vfs_write_suspend() for filesystem unmount VFS
2046  * methods.
2047  */
2048 int
2049 vfs_write_suspend_umnt(struct mount *mp)
2050 {
2051 	int error;
2052 
2053 	KASSERT((curthread->td_pflags & TDP_IGNSUSP) == 0,
2054 	    ("vfs_write_suspend_umnt: recursed"));
2055 
2056 	/* dounmount() already called vn_start_write(). */
2057 	for (;;) {
2058 		vn_finished_write(mp);
2059 		error = vfs_write_suspend(mp, 0);
2060 		if (error != 0) {
2061 			vn_start_write(NULL, &mp, V_WAIT);
2062 			return (error);
2063 		}
2064 		MNT_ILOCK(mp);
2065 		if ((mp->mnt_kern_flag & MNTK_SUSPENDED) != 0)
2066 			break;
2067 		MNT_IUNLOCK(mp);
2068 		vn_start_write(NULL, &mp, V_WAIT);
2069 	}
2070 	mp->mnt_kern_flag &= ~(MNTK_SUSPENDED | MNTK_SUSPEND2);
2071 	wakeup(&mp->mnt_flag);
2072 	MNT_IUNLOCK(mp);
2073 	curthread->td_pflags |= TDP_IGNSUSP;
2074 	return (0);
2075 }
2076 
2077 /*
2078  * Implement kqueues for files by translating it to vnode operation.
2079  */
2080 static int
2081 vn_kqfilter(struct file *fp, struct knote *kn)
2082 {
2083 
2084 	return (VOP_KQFILTER(fp->f_vnode, kn));
2085 }
2086 
2087 /*
2088  * Simplified in-kernel wrapper calls for extended attribute access.
2089  * Both calls pass in a NULL credential, authorizing as "kernel" access.
2090  * Set IO_NODELOCKED in ioflg if the vnode is already locked.
2091  */
2092 int
2093 vn_extattr_get(struct vnode *vp, int ioflg, int attrnamespace,
2094     const char *attrname, int *buflen, char *buf, struct thread *td)
2095 {
2096 	struct uio	auio;
2097 	struct iovec	iov;
2098 	int	error;
2099 
2100 	iov.iov_len = *buflen;
2101 	iov.iov_base = buf;
2102 
2103 	auio.uio_iov = &iov;
2104 	auio.uio_iovcnt = 1;
2105 	auio.uio_rw = UIO_READ;
2106 	auio.uio_segflg = UIO_SYSSPACE;
2107 	auio.uio_td = td;
2108 	auio.uio_offset = 0;
2109 	auio.uio_resid = *buflen;
2110 
2111 	if ((ioflg & IO_NODELOCKED) == 0)
2112 		vn_lock(vp, LK_SHARED | LK_RETRY);
2113 
2114 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
2115 
2116 	/* authorize attribute retrieval as kernel */
2117 	error = VOP_GETEXTATTR(vp, attrnamespace, attrname, &auio, NULL, NULL,
2118 	    td);
2119 
2120 	if ((ioflg & IO_NODELOCKED) == 0)
2121 		VOP_UNLOCK(vp);
2122 
2123 	if (error == 0) {
2124 		*buflen = *buflen - auio.uio_resid;
2125 	}
2126 
2127 	return (error);
2128 }
2129 
2130 /*
2131  * XXX failure mode if partially written?
2132  */
2133 int
2134 vn_extattr_set(struct vnode *vp, int ioflg, int attrnamespace,
2135     const char *attrname, int buflen, char *buf, struct thread *td)
2136 {
2137 	struct uio	auio;
2138 	struct iovec	iov;
2139 	struct mount	*mp;
2140 	int	error;
2141 
2142 	iov.iov_len = buflen;
2143 	iov.iov_base = buf;
2144 
2145 	auio.uio_iov = &iov;
2146 	auio.uio_iovcnt = 1;
2147 	auio.uio_rw = UIO_WRITE;
2148 	auio.uio_segflg = UIO_SYSSPACE;
2149 	auio.uio_td = td;
2150 	auio.uio_offset = 0;
2151 	auio.uio_resid = buflen;
2152 
2153 	if ((ioflg & IO_NODELOCKED) == 0) {
2154 		if ((error = vn_start_write(vp, &mp, V_WAIT)) != 0)
2155 			return (error);
2156 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2157 	}
2158 
2159 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
2160 
2161 	/* authorize attribute setting as kernel */
2162 	error = VOP_SETEXTATTR(vp, attrnamespace, attrname, &auio, NULL, td);
2163 
2164 	if ((ioflg & IO_NODELOCKED) == 0) {
2165 		vn_finished_write(mp);
2166 		VOP_UNLOCK(vp);
2167 	}
2168 
2169 	return (error);
2170 }
2171 
2172 int
2173 vn_extattr_rm(struct vnode *vp, int ioflg, int attrnamespace,
2174     const char *attrname, struct thread *td)
2175 {
2176 	struct mount	*mp;
2177 	int	error;
2178 
2179 	if ((ioflg & IO_NODELOCKED) == 0) {
2180 		if ((error = vn_start_write(vp, &mp, V_WAIT)) != 0)
2181 			return (error);
2182 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2183 	}
2184 
2185 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
2186 
2187 	/* authorize attribute removal as kernel */
2188 	error = VOP_DELETEEXTATTR(vp, attrnamespace, attrname, NULL, td);
2189 	if (error == EOPNOTSUPP)
2190 		error = VOP_SETEXTATTR(vp, attrnamespace, attrname, NULL,
2191 		    NULL, td);
2192 
2193 	if ((ioflg & IO_NODELOCKED) == 0) {
2194 		vn_finished_write(mp);
2195 		VOP_UNLOCK(vp);
2196 	}
2197 
2198 	return (error);
2199 }
2200 
2201 static int
2202 vn_get_ino_alloc_vget(struct mount *mp, void *arg, int lkflags,
2203     struct vnode **rvp)
2204 {
2205 
2206 	return (VFS_VGET(mp, *(ino_t *)arg, lkflags, rvp));
2207 }
2208 
2209 int
2210 vn_vget_ino(struct vnode *vp, ino_t ino, int lkflags, struct vnode **rvp)
2211 {
2212 
2213 	return (vn_vget_ino_gen(vp, vn_get_ino_alloc_vget, &ino,
2214 	    lkflags, rvp));
2215 }
2216 
2217 int
2218 vn_vget_ino_gen(struct vnode *vp, vn_get_ino_t alloc, void *alloc_arg,
2219     int lkflags, struct vnode **rvp)
2220 {
2221 	struct mount *mp;
2222 	int ltype, error;
2223 
2224 	ASSERT_VOP_LOCKED(vp, "vn_vget_ino_get");
2225 	mp = vp->v_mount;
2226 	ltype = VOP_ISLOCKED(vp);
2227 	KASSERT(ltype == LK_EXCLUSIVE || ltype == LK_SHARED,
2228 	    ("vn_vget_ino: vp not locked"));
2229 	error = vfs_busy(mp, MBF_NOWAIT);
2230 	if (error != 0) {
2231 		vfs_ref(mp);
2232 		VOP_UNLOCK(vp);
2233 		error = vfs_busy(mp, 0);
2234 		vn_lock(vp, ltype | LK_RETRY);
2235 		vfs_rel(mp);
2236 		if (error != 0)
2237 			return (ENOENT);
2238 		if (VN_IS_DOOMED(vp)) {
2239 			vfs_unbusy(mp);
2240 			return (ENOENT);
2241 		}
2242 	}
2243 	VOP_UNLOCK(vp);
2244 	error = alloc(mp, alloc_arg, lkflags, rvp);
2245 	vfs_unbusy(mp);
2246 	if (error != 0 || *rvp != vp)
2247 		vn_lock(vp, ltype | LK_RETRY);
2248 	if (VN_IS_DOOMED(vp)) {
2249 		if (error == 0) {
2250 			if (*rvp == vp)
2251 				vunref(vp);
2252 			else
2253 				vput(*rvp);
2254 		}
2255 		error = ENOENT;
2256 	}
2257 	return (error);
2258 }
2259 
2260 int
2261 vn_rlimit_fsize(const struct vnode *vp, const struct uio *uio,
2262     struct thread *td)
2263 {
2264 
2265 	if (vp->v_type != VREG || td == NULL)
2266 		return (0);
2267 	if ((uoff_t)uio->uio_offset + uio->uio_resid >
2268 	    lim_cur(td, RLIMIT_FSIZE)) {
2269 		PROC_LOCK(td->td_proc);
2270 		kern_psignal(td->td_proc, SIGXFSZ);
2271 		PROC_UNLOCK(td->td_proc);
2272 		return (EFBIG);
2273 	}
2274 	return (0);
2275 }
2276 
2277 int
2278 vn_chmod(struct file *fp, mode_t mode, struct ucred *active_cred,
2279     struct thread *td)
2280 {
2281 	struct vnode *vp;
2282 
2283 	vp = fp->f_vnode;
2284 #ifdef AUDIT
2285 	vn_lock(vp, LK_SHARED | LK_RETRY);
2286 	AUDIT_ARG_VNODE1(vp);
2287 	VOP_UNLOCK(vp);
2288 #endif
2289 	return (setfmode(td, active_cred, vp, mode));
2290 }
2291 
2292 int
2293 vn_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
2294     struct thread *td)
2295 {
2296 	struct vnode *vp;
2297 
2298 	vp = fp->f_vnode;
2299 #ifdef AUDIT
2300 	vn_lock(vp, LK_SHARED | LK_RETRY);
2301 	AUDIT_ARG_VNODE1(vp);
2302 	VOP_UNLOCK(vp);
2303 #endif
2304 	return (setfown(td, active_cred, vp, uid, gid));
2305 }
2306 
2307 void
2308 vn_pages_remove(struct vnode *vp, vm_pindex_t start, vm_pindex_t end)
2309 {
2310 	vm_object_t object;
2311 
2312 	if ((object = vp->v_object) == NULL)
2313 		return;
2314 	VM_OBJECT_WLOCK(object);
2315 	vm_object_page_remove(object, start, end, 0);
2316 	VM_OBJECT_WUNLOCK(object);
2317 }
2318 
2319 int
2320 vn_bmap_seekhole(struct vnode *vp, u_long cmd, off_t *off, struct ucred *cred)
2321 {
2322 	struct vattr va;
2323 	daddr_t bn, bnp;
2324 	uint64_t bsize;
2325 	off_t noff;
2326 	int error;
2327 
2328 	KASSERT(cmd == FIOSEEKHOLE || cmd == FIOSEEKDATA,
2329 	    ("Wrong command %lu", cmd));
2330 
2331 	if (vn_lock(vp, LK_SHARED) != 0)
2332 		return (EBADF);
2333 	if (vp->v_type != VREG) {
2334 		error = ENOTTY;
2335 		goto unlock;
2336 	}
2337 	error = VOP_GETATTR(vp, &va, cred);
2338 	if (error != 0)
2339 		goto unlock;
2340 	noff = *off;
2341 	if (noff >= va.va_size) {
2342 		error = ENXIO;
2343 		goto unlock;
2344 	}
2345 	bsize = vp->v_mount->mnt_stat.f_iosize;
2346 	for (bn = noff / bsize; noff < va.va_size; bn++, noff += bsize -
2347 	    noff % bsize) {
2348 		error = VOP_BMAP(vp, bn, NULL, &bnp, NULL, NULL);
2349 		if (error == EOPNOTSUPP) {
2350 			error = ENOTTY;
2351 			goto unlock;
2352 		}
2353 		if ((bnp == -1 && cmd == FIOSEEKHOLE) ||
2354 		    (bnp != -1 && cmd == FIOSEEKDATA)) {
2355 			noff = bn * bsize;
2356 			if (noff < *off)
2357 				noff = *off;
2358 			goto unlock;
2359 		}
2360 	}
2361 	if (noff > va.va_size)
2362 		noff = va.va_size;
2363 	/* noff == va.va_size. There is an implicit hole at the end of file. */
2364 	if (cmd == FIOSEEKDATA)
2365 		error = ENXIO;
2366 unlock:
2367 	VOP_UNLOCK(vp);
2368 	if (error == 0)
2369 		*off = noff;
2370 	return (error);
2371 }
2372 
2373 int
2374 vn_seek(struct file *fp, off_t offset, int whence, struct thread *td)
2375 {
2376 	struct ucred *cred;
2377 	struct vnode *vp;
2378 	struct vattr vattr;
2379 	off_t foffset, size;
2380 	int error, noneg;
2381 
2382 	cred = td->td_ucred;
2383 	vp = fp->f_vnode;
2384 	foffset = foffset_lock(fp, 0);
2385 	noneg = (vp->v_type != VCHR);
2386 	error = 0;
2387 	switch (whence) {
2388 	case L_INCR:
2389 		if (noneg &&
2390 		    (foffset < 0 ||
2391 		    (offset > 0 && foffset > OFF_MAX - offset))) {
2392 			error = EOVERFLOW;
2393 			break;
2394 		}
2395 		offset += foffset;
2396 		break;
2397 	case L_XTND:
2398 		vn_lock(vp, LK_SHARED | LK_RETRY);
2399 		error = VOP_GETATTR(vp, &vattr, cred);
2400 		VOP_UNLOCK(vp);
2401 		if (error)
2402 			break;
2403 
2404 		/*
2405 		 * If the file references a disk device, then fetch
2406 		 * the media size and use that to determine the ending
2407 		 * offset.
2408 		 */
2409 		if (vattr.va_size == 0 && vp->v_type == VCHR &&
2410 		    fo_ioctl(fp, DIOCGMEDIASIZE, &size, cred, td) == 0)
2411 			vattr.va_size = size;
2412 		if (noneg &&
2413 		    (vattr.va_size > OFF_MAX ||
2414 		    (offset > 0 && vattr.va_size > OFF_MAX - offset))) {
2415 			error = EOVERFLOW;
2416 			break;
2417 		}
2418 		offset += vattr.va_size;
2419 		break;
2420 	case L_SET:
2421 		break;
2422 	case SEEK_DATA:
2423 		error = fo_ioctl(fp, FIOSEEKDATA, &offset, cred, td);
2424 		if (error == ENOTTY)
2425 			error = EINVAL;
2426 		break;
2427 	case SEEK_HOLE:
2428 		error = fo_ioctl(fp, FIOSEEKHOLE, &offset, cred, td);
2429 		if (error == ENOTTY)
2430 			error = EINVAL;
2431 		break;
2432 	default:
2433 		error = EINVAL;
2434 	}
2435 	if (error == 0 && noneg && offset < 0)
2436 		error = EINVAL;
2437 	if (error != 0)
2438 		goto drop;
2439 	VFS_KNOTE_UNLOCKED(vp, 0);
2440 	td->td_uretoff.tdu_off = offset;
2441 drop:
2442 	foffset_unlock(fp, offset, error != 0 ? FOF_NOUPDATE : 0);
2443 	return (error);
2444 }
2445 
2446 int
2447 vn_utimes_perm(struct vnode *vp, struct vattr *vap, struct ucred *cred,
2448     struct thread *td)
2449 {
2450 	int error;
2451 
2452 	/*
2453 	 * Grant permission if the caller is the owner of the file, or
2454 	 * the super-user, or has ACL_WRITE_ATTRIBUTES permission on
2455 	 * on the file.  If the time pointer is null, then write
2456 	 * permission on the file is also sufficient.
2457 	 *
2458 	 * From NFSv4.1, draft 21, 6.2.1.3.1, Discussion of Mask Attributes:
2459 	 * A user having ACL_WRITE_DATA or ACL_WRITE_ATTRIBUTES
2460 	 * will be allowed to set the times [..] to the current
2461 	 * server time.
2462 	 */
2463 	error = VOP_ACCESSX(vp, VWRITE_ATTRIBUTES, cred, td);
2464 	if (error != 0 && (vap->va_vaflags & VA_UTIMES_NULL) != 0)
2465 		error = VOP_ACCESS(vp, VWRITE, cred, td);
2466 	return (error);
2467 }
2468 
2469 int
2470 vn_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
2471 {
2472 	struct vnode *vp;
2473 	int error;
2474 
2475 	if (fp->f_type == DTYPE_FIFO)
2476 		kif->kf_type = KF_TYPE_FIFO;
2477 	else
2478 		kif->kf_type = KF_TYPE_VNODE;
2479 	vp = fp->f_vnode;
2480 	vref(vp);
2481 	FILEDESC_SUNLOCK(fdp);
2482 	error = vn_fill_kinfo_vnode(vp, kif);
2483 	vrele(vp);
2484 	FILEDESC_SLOCK(fdp);
2485 	return (error);
2486 }
2487 
2488 static inline void
2489 vn_fill_junk(struct kinfo_file *kif)
2490 {
2491 	size_t len, olen;
2492 
2493 	/*
2494 	 * Simulate vn_fullpath returning changing values for a given
2495 	 * vp during e.g. coredump.
2496 	 */
2497 	len = (arc4random() % (sizeof(kif->kf_path) - 2)) + 1;
2498 	olen = strlen(kif->kf_path);
2499 	if (len < olen)
2500 		strcpy(&kif->kf_path[len - 1], "$");
2501 	else
2502 		for (; olen < len; olen++)
2503 			strcpy(&kif->kf_path[olen], "A");
2504 }
2505 
2506 int
2507 vn_fill_kinfo_vnode(struct vnode *vp, struct kinfo_file *kif)
2508 {
2509 	struct vattr va;
2510 	char *fullpath, *freepath;
2511 	int error;
2512 
2513 	kif->kf_un.kf_file.kf_file_type = vntype_to_kinfo(vp->v_type);
2514 	freepath = NULL;
2515 	fullpath = "-";
2516 	error = vn_fullpath(vp, &fullpath, &freepath);
2517 	if (error == 0) {
2518 		strlcpy(kif->kf_path, fullpath, sizeof(kif->kf_path));
2519 	}
2520 	if (freepath != NULL)
2521 		free(freepath, M_TEMP);
2522 
2523 	KFAIL_POINT_CODE(DEBUG_FP, fill_kinfo_vnode__random_path,
2524 		vn_fill_junk(kif);
2525 	);
2526 
2527 	/*
2528 	 * Retrieve vnode attributes.
2529 	 */
2530 	va.va_fsid = VNOVAL;
2531 	va.va_rdev = NODEV;
2532 	vn_lock(vp, LK_SHARED | LK_RETRY);
2533 	error = VOP_GETATTR(vp, &va, curthread->td_ucred);
2534 	VOP_UNLOCK(vp);
2535 	if (error != 0)
2536 		return (error);
2537 	if (va.va_fsid != VNOVAL)
2538 		kif->kf_un.kf_file.kf_file_fsid = va.va_fsid;
2539 	else
2540 		kif->kf_un.kf_file.kf_file_fsid =
2541 		    vp->v_mount->mnt_stat.f_fsid.val[0];
2542 	kif->kf_un.kf_file.kf_file_fsid_freebsd11 =
2543 	    kif->kf_un.kf_file.kf_file_fsid; /* truncate */
2544 	kif->kf_un.kf_file.kf_file_fileid = va.va_fileid;
2545 	kif->kf_un.kf_file.kf_file_mode = MAKEIMODE(va.va_type, va.va_mode);
2546 	kif->kf_un.kf_file.kf_file_size = va.va_size;
2547 	kif->kf_un.kf_file.kf_file_rdev = va.va_rdev;
2548 	kif->kf_un.kf_file.kf_file_rdev_freebsd11 =
2549 	    kif->kf_un.kf_file.kf_file_rdev; /* truncate */
2550 	return (0);
2551 }
2552 
2553 int
2554 vn_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size,
2555     vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff,
2556     struct thread *td)
2557 {
2558 #ifdef HWPMC_HOOKS
2559 	struct pmckern_map_in pkm;
2560 #endif
2561 	struct mount *mp;
2562 	struct vnode *vp;
2563 	vm_object_t object;
2564 	vm_prot_t maxprot;
2565 	boolean_t writecounted;
2566 	int error;
2567 
2568 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \
2569     defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4)
2570 	/*
2571 	 * POSIX shared-memory objects are defined to have
2572 	 * kernel persistence, and are not defined to support
2573 	 * read(2)/write(2) -- or even open(2).  Thus, we can
2574 	 * use MAP_ASYNC to trade on-disk coherence for speed.
2575 	 * The shm_open(3) library routine turns on the FPOSIXSHM
2576 	 * flag to request this behavior.
2577 	 */
2578 	if ((fp->f_flag & FPOSIXSHM) != 0)
2579 		flags |= MAP_NOSYNC;
2580 #endif
2581 	vp = fp->f_vnode;
2582 
2583 	/*
2584 	 * Ensure that file and memory protections are
2585 	 * compatible.  Note that we only worry about
2586 	 * writability if mapping is shared; in this case,
2587 	 * current and max prot are dictated by the open file.
2588 	 * XXX use the vnode instead?  Problem is: what
2589 	 * credentials do we use for determination? What if
2590 	 * proc does a setuid?
2591 	 */
2592 	mp = vp->v_mount;
2593 	if (mp != NULL && (mp->mnt_flag & MNT_NOEXEC) != 0) {
2594 		maxprot = VM_PROT_NONE;
2595 		if ((prot & VM_PROT_EXECUTE) != 0)
2596 			return (EACCES);
2597 	} else
2598 		maxprot = VM_PROT_EXECUTE;
2599 	if ((fp->f_flag & FREAD) != 0)
2600 		maxprot |= VM_PROT_READ;
2601 	else if ((prot & VM_PROT_READ) != 0)
2602 		return (EACCES);
2603 
2604 	/*
2605 	 * If we are sharing potential changes via MAP_SHARED and we
2606 	 * are trying to get write permission although we opened it
2607 	 * without asking for it, bail out.
2608 	 */
2609 	if ((flags & MAP_SHARED) != 0) {
2610 		if ((fp->f_flag & FWRITE) != 0)
2611 			maxprot |= VM_PROT_WRITE;
2612 		else if ((prot & VM_PROT_WRITE) != 0)
2613 			return (EACCES);
2614 	} else {
2615 		maxprot |= VM_PROT_WRITE;
2616 		cap_maxprot |= VM_PROT_WRITE;
2617 	}
2618 	maxprot &= cap_maxprot;
2619 
2620 	/*
2621 	 * For regular files and shared memory, POSIX requires that
2622 	 * the value of foff be a legitimate offset within the data
2623 	 * object.  In particular, negative offsets are invalid.
2624 	 * Blocking negative offsets and overflows here avoids
2625 	 * possible wraparound or user-level access into reserved
2626 	 * ranges of the data object later.  In contrast, POSIX does
2627 	 * not dictate how offsets are used by device drivers, so in
2628 	 * the case of a device mapping a negative offset is passed
2629 	 * on.
2630 	 */
2631 	if (
2632 #ifdef _LP64
2633 	    size > OFF_MAX ||
2634 #endif
2635 	    foff < 0 || foff > OFF_MAX - size)
2636 		return (EINVAL);
2637 
2638 	writecounted = FALSE;
2639 	error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, vp,
2640 	    &foff, &object, &writecounted);
2641 	if (error != 0)
2642 		return (error);
2643 	error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object,
2644 	    foff, writecounted, td);
2645 	if (error != 0) {
2646 		/*
2647 		 * If this mapping was accounted for in the vnode's
2648 		 * writecount, then undo that now.
2649 		 */
2650 		if (writecounted)
2651 			vm_pager_release_writecount(object, 0, size);
2652 		vm_object_deallocate(object);
2653 	}
2654 #ifdef HWPMC_HOOKS
2655 	/* Inform hwpmc(4) if an executable is being mapped. */
2656 	if (PMC_HOOK_INSTALLED(PMC_FN_MMAP)) {
2657 		if ((prot & VM_PROT_EXECUTE) != 0 && error == 0) {
2658 			pkm.pm_file = vp;
2659 			pkm.pm_address = (uintptr_t) *addr;
2660 			PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_MMAP, (void *) &pkm);
2661 		}
2662 	}
2663 #endif
2664 	return (error);
2665 }
2666 
2667 void
2668 vn_fsid(struct vnode *vp, struct vattr *va)
2669 {
2670 	fsid_t *f;
2671 
2672 	f = &vp->v_mount->mnt_stat.f_fsid;
2673 	va->va_fsid = (uint32_t)f->val[1];
2674 	va->va_fsid <<= sizeof(f->val[1]) * NBBY;
2675 	va->va_fsid += (uint32_t)f->val[0];
2676 }
2677 
2678 int
2679 vn_fsync_buf(struct vnode *vp, int waitfor)
2680 {
2681 	struct buf *bp, *nbp;
2682 	struct bufobj *bo;
2683 	struct mount *mp;
2684 	int error, maxretry;
2685 
2686 	error = 0;
2687 	maxretry = 10000;     /* large, arbitrarily chosen */
2688 	mp = NULL;
2689 	if (vp->v_type == VCHR) {
2690 		VI_LOCK(vp);
2691 		mp = vp->v_rdev->si_mountpt;
2692 		VI_UNLOCK(vp);
2693 	}
2694 	bo = &vp->v_bufobj;
2695 	BO_LOCK(bo);
2696 loop1:
2697 	/*
2698 	 * MARK/SCAN initialization to avoid infinite loops.
2699 	 */
2700         TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) {
2701 		bp->b_vflags &= ~BV_SCANNED;
2702 		bp->b_error = 0;
2703 	}
2704 
2705 	/*
2706 	 * Flush all dirty buffers associated with a vnode.
2707 	 */
2708 loop2:
2709 	TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
2710 		if ((bp->b_vflags & BV_SCANNED) != 0)
2711 			continue;
2712 		bp->b_vflags |= BV_SCANNED;
2713 		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) {
2714 			if (waitfor != MNT_WAIT)
2715 				continue;
2716 			if (BUF_LOCK(bp,
2717 			    LK_EXCLUSIVE | LK_INTERLOCK | LK_SLEEPFAIL,
2718 			    BO_LOCKPTR(bo)) != 0) {
2719 				BO_LOCK(bo);
2720 				goto loop1;
2721 			}
2722 			BO_LOCK(bo);
2723 		}
2724 		BO_UNLOCK(bo);
2725 		KASSERT(bp->b_bufobj == bo,
2726 		    ("bp %p wrong b_bufobj %p should be %p",
2727 		    bp, bp->b_bufobj, bo));
2728 		if ((bp->b_flags & B_DELWRI) == 0)
2729 			panic("fsync: not dirty");
2730 		if ((vp->v_object != NULL) && (bp->b_flags & B_CLUSTEROK)) {
2731 			vfs_bio_awrite(bp);
2732 		} else {
2733 			bremfree(bp);
2734 			bawrite(bp);
2735 		}
2736 		if (maxretry < 1000)
2737 			pause("dirty", hz < 1000 ? 1 : hz / 1000);
2738 		BO_LOCK(bo);
2739 		goto loop2;
2740 	}
2741 
2742 	/*
2743 	 * If synchronous the caller expects us to completely resolve all
2744 	 * dirty buffers in the system.  Wait for in-progress I/O to
2745 	 * complete (which could include background bitmap writes), then
2746 	 * retry if dirty blocks still exist.
2747 	 */
2748 	if (waitfor == MNT_WAIT) {
2749 		bufobj_wwait(bo, 0, 0);
2750 		if (bo->bo_dirty.bv_cnt > 0) {
2751 			/*
2752 			 * If we are unable to write any of these buffers
2753 			 * then we fail now rather than trying endlessly
2754 			 * to write them out.
2755 			 */
2756 			TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs)
2757 				if ((error = bp->b_error) != 0)
2758 					break;
2759 			if ((mp != NULL && mp->mnt_secondary_writes > 0) ||
2760 			    (error == 0 && --maxretry >= 0))
2761 				goto loop1;
2762 			if (error == 0)
2763 				error = EAGAIN;
2764 		}
2765 	}
2766 	BO_UNLOCK(bo);
2767 	if (error != 0)
2768 		vn_printf(vp, "fsync: giving up on dirty (error = %d) ", error);
2769 
2770 	return (error);
2771 }
2772 
2773 /*
2774  * Copies a byte range from invp to outvp.  Calls VOP_COPY_FILE_RANGE()
2775  * or vn_generic_copy_file_range() after rangelocking the byte ranges,
2776  * to do the actual copy.
2777  * vn_generic_copy_file_range() is factored out, so it can be called
2778  * from a VOP_COPY_FILE_RANGE() call as well, but handles vnodes from
2779  * different file systems.
2780  */
2781 int
2782 vn_copy_file_range(struct vnode *invp, off_t *inoffp, struct vnode *outvp,
2783     off_t *outoffp, size_t *lenp, unsigned int flags, struct ucred *incred,
2784     struct ucred *outcred, struct thread *fsize_td)
2785 {
2786 	int error;
2787 	size_t len;
2788 	uint64_t uvalin, uvalout;
2789 
2790 	len = *lenp;
2791 	*lenp = 0;		/* For error returns. */
2792 	error = 0;
2793 
2794 	/* Do some sanity checks on the arguments. */
2795 	uvalin = *inoffp;
2796 	uvalin += len;
2797 	uvalout = *outoffp;
2798 	uvalout += len;
2799 	if (invp->v_type == VDIR || outvp->v_type == VDIR)
2800 		error = EISDIR;
2801 	else if (*inoffp < 0 || uvalin > INT64_MAX || uvalin <
2802 	    (uint64_t)*inoffp || *outoffp < 0 || uvalout > INT64_MAX ||
2803 	    uvalout < (uint64_t)*outoffp || invp->v_type != VREG ||
2804 	    outvp->v_type != VREG)
2805 		error = EINVAL;
2806 	if (error != 0)
2807 		goto out;
2808 
2809 	/*
2810 	 * If the two vnode are for the same file system, call
2811 	 * VOP_COPY_FILE_RANGE(), otherwise call vn_generic_copy_file_range()
2812 	 * which can handle copies across multiple file systems.
2813 	 */
2814 	*lenp = len;
2815 	if (invp->v_mount == outvp->v_mount)
2816 		error = VOP_COPY_FILE_RANGE(invp, inoffp, outvp, outoffp,
2817 		    lenp, flags, incred, outcred, fsize_td);
2818 	else
2819 		error = vn_generic_copy_file_range(invp, inoffp, outvp,
2820 		    outoffp, lenp, flags, incred, outcred, fsize_td);
2821 out:
2822 	return (error);
2823 }
2824 
2825 /*
2826  * Test len bytes of data starting at dat for all bytes == 0.
2827  * Return true if all bytes are zero, false otherwise.
2828  * Expects dat to be well aligned.
2829  */
2830 static bool
2831 mem_iszero(void *dat, int len)
2832 {
2833 	int i;
2834 	const u_int *p;
2835 	const char *cp;
2836 
2837 	for (p = dat; len > 0; len -= sizeof(*p), p++) {
2838 		if (len >= sizeof(*p)) {
2839 			if (*p != 0)
2840 				return (false);
2841 		} else {
2842 			cp = (const char *)p;
2843 			for (i = 0; i < len; i++, cp++)
2844 				if (*cp != '\0')
2845 					return (false);
2846 		}
2847 	}
2848 	return (true);
2849 }
2850 
2851 /*
2852  * Look for a hole in the output file and, if found, adjust *outoffp
2853  * and *xferp to skip past the hole.
2854  * *xferp is the entire hole length to be written and xfer2 is how many bytes
2855  * to be written as 0's upon return.
2856  */
2857 static off_t
2858 vn_skip_hole(struct vnode *outvp, off_t xfer2, off_t *outoffp, off_t *xferp,
2859     off_t *dataoffp, off_t *holeoffp, struct ucred *cred)
2860 {
2861 	int error;
2862 	off_t delta;
2863 
2864 	if (*holeoffp == 0 || *holeoffp <= *outoffp) {
2865 		*dataoffp = *outoffp;
2866 		error = VOP_IOCTL(outvp, FIOSEEKDATA, dataoffp, 0, cred,
2867 		    curthread);
2868 		if (error == 0) {
2869 			*holeoffp = *dataoffp;
2870 			error = VOP_IOCTL(outvp, FIOSEEKHOLE, holeoffp, 0, cred,
2871 			    curthread);
2872 		}
2873 		if (error != 0 || *holeoffp == *dataoffp) {
2874 			/*
2875 			 * Since outvp is unlocked, it may be possible for
2876 			 * another thread to do a truncate(), lseek(), write()
2877 			 * creating a hole at startoff between the above
2878 			 * VOP_IOCTL() calls, if the other thread does not do
2879 			 * rangelocking.
2880 			 * If that happens, *holeoffp == *dataoffp and finding
2881 			 * the hole has failed, so disable vn_skip_hole().
2882 			 */
2883 			*holeoffp = -1;	/* Disable use of vn_skip_hole(). */
2884 			return (xfer2);
2885 		}
2886 		KASSERT(*dataoffp >= *outoffp,
2887 		    ("vn_skip_hole: dataoff=%jd < outoff=%jd",
2888 		    (intmax_t)*dataoffp, (intmax_t)*outoffp));
2889 		KASSERT(*holeoffp > *dataoffp,
2890 		    ("vn_skip_hole: holeoff=%jd <= dataoff=%jd",
2891 		    (intmax_t)*holeoffp, (intmax_t)*dataoffp));
2892 	}
2893 
2894 	/*
2895 	 * If there is a hole before the data starts, advance *outoffp and
2896 	 * *xferp past the hole.
2897 	 */
2898 	if (*dataoffp > *outoffp) {
2899 		delta = *dataoffp - *outoffp;
2900 		if (delta >= *xferp) {
2901 			/* Entire *xferp is a hole. */
2902 			*outoffp += *xferp;
2903 			*xferp = 0;
2904 			return (0);
2905 		}
2906 		*xferp -= delta;
2907 		*outoffp += delta;
2908 		xfer2 = MIN(xfer2, *xferp);
2909 	}
2910 
2911 	/*
2912 	 * If a hole starts before the end of this xfer2, reduce this xfer2 so
2913 	 * that the write ends at the start of the hole.
2914 	 * *holeoffp should always be greater than *outoffp, but for the
2915 	 * non-INVARIANTS case, check this to make sure xfer2 remains a sane
2916 	 * value.
2917 	 */
2918 	if (*holeoffp > *outoffp && *holeoffp < *outoffp + xfer2)
2919 		xfer2 = *holeoffp - *outoffp;
2920 	return (xfer2);
2921 }
2922 
2923 /*
2924  * Write an xfer sized chunk to outvp in blksize blocks from dat.
2925  * dat is a maximum of blksize in length and can be written repeatedly in
2926  * the chunk.
2927  * If growfile == true, just grow the file via vn_truncate_locked() instead
2928  * of doing actual writes.
2929  * If checkhole == true, a hole is being punched, so skip over any hole
2930  * already in the output file.
2931  */
2932 static int
2933 vn_write_outvp(struct vnode *outvp, char *dat, off_t outoff, off_t xfer,
2934     u_long blksize, bool growfile, bool checkhole, struct ucred *cred)
2935 {
2936 	struct mount *mp;
2937 	off_t dataoff, holeoff, xfer2;
2938 	int error, lckf;
2939 
2940 	/*
2941 	 * Loop around doing writes of blksize until write has been completed.
2942 	 * Lock/unlock on each loop iteration so that a bwillwrite() can be
2943 	 * done for each iteration, since the xfer argument can be very
2944 	 * large if there is a large hole to punch in the output file.
2945 	 */
2946 	error = 0;
2947 	holeoff = 0;
2948 	do {
2949 		xfer2 = MIN(xfer, blksize);
2950 		if (checkhole) {
2951 			/*
2952 			 * Punching a hole.  Skip writing if there is
2953 			 * already a hole in the output file.
2954 			 */
2955 			xfer2 = vn_skip_hole(outvp, xfer2, &outoff, &xfer,
2956 			    &dataoff, &holeoff, cred);
2957 			if (xfer == 0)
2958 				break;
2959 			if (holeoff < 0)
2960 				checkhole = false;
2961 			KASSERT(xfer2 > 0, ("vn_write_outvp: xfer2=%jd",
2962 			    (intmax_t)xfer2));
2963 		}
2964 		bwillwrite();
2965 		mp = NULL;
2966 		error = vn_start_write(outvp, &mp, V_WAIT);
2967 		if (error == 0) {
2968 			if (MNT_SHARED_WRITES(mp))
2969 				lckf = LK_SHARED;
2970 			else
2971 				lckf = LK_EXCLUSIVE;
2972 			error = vn_lock(outvp, lckf);
2973 		}
2974 		if (error == 0) {
2975 			if (growfile)
2976 				error = vn_truncate_locked(outvp, outoff + xfer,
2977 				    false, cred);
2978 			else {
2979 				error = vn_rdwr(UIO_WRITE, outvp, dat, xfer2,
2980 				    outoff, UIO_SYSSPACE, IO_NODELOCKED,
2981 				    curthread->td_ucred, cred, NULL, curthread);
2982 				outoff += xfer2;
2983 				xfer -= xfer2;
2984 			}
2985 			VOP_UNLOCK(outvp);
2986 		}
2987 		if (mp != NULL)
2988 			vn_finished_write(mp);
2989 	} while (!growfile && xfer > 0 && error == 0);
2990 	return (error);
2991 }
2992 
2993 /*
2994  * Copy a byte range of one file to another.  This function can handle the
2995  * case where invp and outvp are on different file systems.
2996  * It can also be called by a VOP_COPY_FILE_RANGE() to do the work, if there
2997  * is no better file system specific way to do it.
2998  */
2999 int
3000 vn_generic_copy_file_range(struct vnode *invp, off_t *inoffp,
3001     struct vnode *outvp, off_t *outoffp, size_t *lenp, unsigned int flags,
3002     struct ucred *incred, struct ucred *outcred, struct thread *fsize_td)
3003 {
3004 	struct vattr va;
3005 	struct mount *mp;
3006 	struct uio io;
3007 	off_t startoff, endoff, xfer, xfer2;
3008 	u_long blksize;
3009 	int error;
3010 	bool cantseek, readzeros, eof, lastblock;
3011 	ssize_t aresid;
3012 	size_t copylen, len, savlen;
3013 	char *dat;
3014 	long holein, holeout;
3015 
3016 	holein = holeout = 0;
3017 	savlen = len = *lenp;
3018 	error = 0;
3019 	dat = NULL;
3020 
3021 	error = vn_lock(invp, LK_SHARED);
3022 	if (error != 0)
3023 		goto out;
3024 	if (VOP_PATHCONF(invp, _PC_MIN_HOLE_SIZE, &holein) != 0)
3025 		holein = 0;
3026 	VOP_UNLOCK(invp);
3027 
3028 	mp = NULL;
3029 	error = vn_start_write(outvp, &mp, V_WAIT);
3030 	if (error == 0)
3031 		error = vn_lock(outvp, LK_EXCLUSIVE);
3032 	if (error == 0) {
3033 		/*
3034 		 * If fsize_td != NULL, do a vn_rlimit_fsize() call,
3035 		 * now that outvp is locked.
3036 		 */
3037 		if (fsize_td != NULL) {
3038 			io.uio_offset = *outoffp;
3039 			io.uio_resid = len;
3040 			error = vn_rlimit_fsize(outvp, &io, fsize_td);
3041 			if (error != 0)
3042 				error = EFBIG;
3043 		}
3044 		if (VOP_PATHCONF(outvp, _PC_MIN_HOLE_SIZE, &holeout) != 0)
3045 			holeout = 0;
3046 		/*
3047 		 * Holes that are past EOF do not need to be written as a block
3048 		 * of zero bytes.  So, truncate the output file as far as
3049 		 * possible and then use va.va_size to decide if writing 0
3050 		 * bytes is necessary in the loop below.
3051 		 */
3052 		if (error == 0)
3053 			error = VOP_GETATTR(outvp, &va, outcred);
3054 		if (error == 0 && va.va_size > *outoffp && va.va_size <=
3055 		    *outoffp + len) {
3056 #ifdef MAC
3057 			error = mac_vnode_check_write(curthread->td_ucred,
3058 			    outcred, outvp);
3059 			if (error == 0)
3060 #endif
3061 				error = vn_truncate_locked(outvp, *outoffp,
3062 				    false, outcred);
3063 			if (error == 0)
3064 				va.va_size = *outoffp;
3065 		}
3066 		VOP_UNLOCK(outvp);
3067 	}
3068 	if (mp != NULL)
3069 		vn_finished_write(mp);
3070 	if (error != 0)
3071 		goto out;
3072 
3073 	/*
3074 	 * Set the blksize to the larger of the hole sizes for invp and outvp.
3075 	 * If hole sizes aren't available, set the blksize to the larger
3076 	 * f_iosize of invp and outvp.
3077 	 * This code expects the hole sizes and f_iosizes to be powers of 2.
3078 	 * This value is clipped at 4Kbytes and 1Mbyte.
3079 	 */
3080 	blksize = MAX(holein, holeout);
3081 	if (blksize == 0)
3082 		blksize = MAX(invp->v_mount->mnt_stat.f_iosize,
3083 		    outvp->v_mount->mnt_stat.f_iosize);
3084 	if (blksize < 4096)
3085 		blksize = 4096;
3086 	else if (blksize > 1024 * 1024)
3087 		blksize = 1024 * 1024;
3088 	dat = malloc(blksize, M_TEMP, M_WAITOK);
3089 
3090 	/*
3091 	 * If VOP_IOCTL(FIOSEEKHOLE) works for invp, use it and FIOSEEKDATA
3092 	 * to find holes.  Otherwise, just scan the read block for all 0s
3093 	 * in the inner loop where the data copying is done.
3094 	 * Note that some file systems such as NFSv3, NFSv4.0 and NFSv4.1 may
3095 	 * support holes on the server, but do not support FIOSEEKHOLE.
3096 	 */
3097 	eof = false;
3098 	while (len > 0 && error == 0 && !eof) {
3099 		endoff = 0;			/* To shut up compilers. */
3100 		cantseek = true;
3101 		startoff = *inoffp;
3102 		copylen = len;
3103 
3104 		/*
3105 		 * Find the next data area.  If there is just a hole to EOF,
3106 		 * FIOSEEKDATA should fail and then we drop down into the
3107 		 * inner loop and create the hole on the outvp file.
3108 		 * (I do not know if any file system will report a hole to
3109 		 *  EOF via FIOSEEKHOLE, but I am pretty sure FIOSEEKDATA
3110 		 *  will fail for those file systems.)
3111 		 *
3112 		 * For input files that don't support FIOSEEKDATA/FIOSEEKHOLE,
3113 		 * the code just falls through to the inner copy loop.
3114 		 */
3115 		error = EINVAL;
3116 		if (holein > 0)
3117 			error = VOP_IOCTL(invp, FIOSEEKDATA, &startoff, 0,
3118 			    incred, curthread);
3119 		if (error == 0) {
3120 			endoff = startoff;
3121 			error = VOP_IOCTL(invp, FIOSEEKHOLE, &endoff, 0,
3122 			    incred, curthread);
3123 			/*
3124 			 * Since invp is unlocked, it may be possible for
3125 			 * another thread to do a truncate(), lseek(), write()
3126 			 * creating a hole at startoff between the above
3127 			 * VOP_IOCTL() calls, if the other thread does not do
3128 			 * rangelocking.
3129 			 * If that happens, startoff == endoff and finding
3130 			 * the hole has failed, so set an error.
3131 			 */
3132 			if (error == 0 && startoff == endoff)
3133 				error = EINVAL; /* Any error. Reset to 0. */
3134 		}
3135 		if (error == 0) {
3136 			if (startoff > *inoffp) {
3137 				/* Found hole before data block. */
3138 				xfer = MIN(startoff - *inoffp, len);
3139 				if (*outoffp < va.va_size) {
3140 					/* Must write 0s to punch hole. */
3141 					xfer2 = MIN(va.va_size - *outoffp,
3142 					    xfer);
3143 					memset(dat, 0, MIN(xfer2, blksize));
3144 					error = vn_write_outvp(outvp, dat,
3145 					    *outoffp, xfer2, blksize, false,
3146 					    holeout > 0, outcred);
3147 				}
3148 
3149 				if (error == 0 && *outoffp + xfer >
3150 				    va.va_size && xfer == len)
3151 					/* Grow last block. */
3152 					error = vn_write_outvp(outvp, dat,
3153 					    *outoffp, xfer, blksize, true,
3154 					    false, outcred);
3155 				if (error == 0) {
3156 					*inoffp += xfer;
3157 					*outoffp += xfer;
3158 					len -= xfer;
3159 				}
3160 			}
3161 			copylen = MIN(len, endoff - startoff);
3162 			cantseek = false;
3163 		} else {
3164 			cantseek = true;
3165 			startoff = *inoffp;
3166 			copylen = len;
3167 			error = 0;
3168 		}
3169 
3170 		xfer = blksize;
3171 		if (cantseek) {
3172 			/*
3173 			 * Set first xfer to end at a block boundary, so that
3174 			 * holes are more likely detected in the loop below via
3175 			 * the for all bytes 0 method.
3176 			 */
3177 			xfer -= (*inoffp % blksize);
3178 		}
3179 		/* Loop copying the data block. */
3180 		while (copylen > 0 && error == 0 && !eof) {
3181 			if (copylen < xfer)
3182 				xfer = copylen;
3183 			error = vn_lock(invp, LK_SHARED);
3184 			if (error != 0)
3185 				goto out;
3186 			error = vn_rdwr(UIO_READ, invp, dat, xfer,
3187 			    startoff, UIO_SYSSPACE, IO_NODELOCKED,
3188 			    curthread->td_ucred, incred, &aresid,
3189 			    curthread);
3190 			VOP_UNLOCK(invp);
3191 			lastblock = false;
3192 			if (error == 0 && aresid > 0) {
3193 				/* Stop the copy at EOF on the input file. */
3194 				xfer -= aresid;
3195 				eof = true;
3196 				lastblock = true;
3197 			}
3198 			if (error == 0) {
3199 				/*
3200 				 * Skip the write for holes past the initial EOF
3201 				 * of the output file, unless this is the last
3202 				 * write of the output file at EOF.
3203 				 */
3204 				readzeros = cantseek ? mem_iszero(dat, xfer) :
3205 				    false;
3206 				if (xfer == len)
3207 					lastblock = true;
3208 				if (!cantseek || *outoffp < va.va_size ||
3209 				    lastblock || !readzeros)
3210 					error = vn_write_outvp(outvp, dat,
3211 					    *outoffp, xfer, blksize,
3212 					    readzeros && lastblock &&
3213 					    *outoffp >= va.va_size, false,
3214 					    outcred);
3215 				if (error == 0) {
3216 					*inoffp += xfer;
3217 					startoff += xfer;
3218 					*outoffp += xfer;
3219 					copylen -= xfer;
3220 					len -= xfer;
3221 				}
3222 			}
3223 			xfer = blksize;
3224 		}
3225 	}
3226 out:
3227 	*lenp = savlen - len;
3228 	free(dat, M_TEMP);
3229 	return (error);
3230 }
3231 
3232 static int
3233 vn_fallocate(struct file *fp, off_t offset, off_t len, struct thread *td)
3234 {
3235 	struct mount *mp;
3236 	struct vnode *vp;
3237 	off_t olen, ooffset;
3238 	int error;
3239 #ifdef AUDIT
3240 	int audited_vnode1 = 0;
3241 #endif
3242 
3243 	vp = fp->f_vnode;
3244 	if (vp->v_type != VREG)
3245 		return (ENODEV);
3246 
3247 	/* Allocating blocks may take a long time, so iterate. */
3248 	for (;;) {
3249 		olen = len;
3250 		ooffset = offset;
3251 
3252 		bwillwrite();
3253 		mp = NULL;
3254 		error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
3255 		if (error != 0)
3256 			break;
3257 		error = vn_lock(vp, LK_EXCLUSIVE);
3258 		if (error != 0) {
3259 			vn_finished_write(mp);
3260 			break;
3261 		}
3262 #ifdef AUDIT
3263 		if (!audited_vnode1) {
3264 			AUDIT_ARG_VNODE1(vp);
3265 			audited_vnode1 = 1;
3266 		}
3267 #endif
3268 #ifdef MAC
3269 		error = mac_vnode_check_write(td->td_ucred, fp->f_cred, vp);
3270 		if (error == 0)
3271 #endif
3272 			error = VOP_ALLOCATE(vp, &offset, &len);
3273 		VOP_UNLOCK(vp);
3274 		vn_finished_write(mp);
3275 
3276 		if (olen + ooffset != offset + len) {
3277 			panic("offset + len changed from %jx/%jx to %jx/%jx",
3278 			    ooffset, olen, offset, len);
3279 		}
3280 		if (error != 0 || len == 0)
3281 			break;
3282 		KASSERT(olen > len, ("Iteration did not make progress?"));
3283 		maybe_yield();
3284 	}
3285 
3286 	return (error);
3287 }
3288