xref: /freebsd/sys/kern/vfs_vnops.c (revision 20bd59416dcacbd2b776fe49dfa193900f303287)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1989, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  * (c) UNIX System Laboratories, Inc.
7  * All or some portions of this file are derived from material licensed
8  * to the University of California by American Telephone and Telegraph
9  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10  * the permission of UNIX System Laboratories, Inc.
11  *
12  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
13  * Copyright (c) 2013, 2014 The FreeBSD Foundation
14  *
15  * Portions of this software were developed by Konstantin Belousov
16  * under sponsorship from the FreeBSD Foundation.
17  *
18  * Redistribution and use in source and binary forms, with or without
19  * modification, are permitted provided that the following conditions
20  * are met:
21  * 1. Redistributions of source code must retain the above copyright
22  *    notice, this list of conditions and the following disclaimer.
23  * 2. Redistributions in binary form must reproduce the above copyright
24  *    notice, this list of conditions and the following disclaimer in the
25  *    documentation and/or other materials provided with the distribution.
26  * 3. Neither the name of the University nor the names of its contributors
27  *    may be used to endorse or promote products derived from this software
28  *    without specific prior written permission.
29  *
30  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
31  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
34  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
36  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
37  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
38  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
39  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40  * SUCH DAMAGE.
41  *
42  *	@(#)vfs_vnops.c	8.2 (Berkeley) 1/21/94
43  */
44 
45 #include <sys/cdefs.h>
46 __FBSDID("$FreeBSD$");
47 
48 #include "opt_hwpmc_hooks.h"
49 
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/disk.h>
53 #include <sys/fail.h>
54 #include <sys/fcntl.h>
55 #include <sys/file.h>
56 #include <sys/kdb.h>
57 #include <sys/ktr.h>
58 #include <sys/stat.h>
59 #include <sys/priv.h>
60 #include <sys/proc.h>
61 #include <sys/limits.h>
62 #include <sys/lock.h>
63 #include <sys/mman.h>
64 #include <sys/mount.h>
65 #include <sys/mutex.h>
66 #include <sys/namei.h>
67 #include <sys/vnode.h>
68 #include <sys/bio.h>
69 #include <sys/buf.h>
70 #include <sys/filio.h>
71 #include <sys/resourcevar.h>
72 #include <sys/rwlock.h>
73 #include <sys/sx.h>
74 #include <sys/sysctl.h>
75 #include <sys/ttycom.h>
76 #include <sys/conf.h>
77 #include <sys/syslog.h>
78 #include <sys/unistd.h>
79 #include <sys/user.h>
80 
81 #include <security/audit/audit.h>
82 #include <security/mac/mac_framework.h>
83 
84 #include <vm/vm.h>
85 #include <vm/vm_extern.h>
86 #include <vm/pmap.h>
87 #include <vm/vm_map.h>
88 #include <vm/vm_object.h>
89 #include <vm/vm_page.h>
90 #include <vm/vm_pager.h>
91 
92 #ifdef HWPMC_HOOKS
93 #include <sys/pmckern.h>
94 #endif
95 
96 static fo_rdwr_t	vn_read;
97 static fo_rdwr_t	vn_write;
98 static fo_rdwr_t	vn_io_fault;
99 static fo_truncate_t	vn_truncate;
100 static fo_ioctl_t	vn_ioctl;
101 static fo_poll_t	vn_poll;
102 static fo_kqfilter_t	vn_kqfilter;
103 static fo_stat_t	vn_statfile;
104 static fo_close_t	vn_closefile;
105 static fo_mmap_t	vn_mmap;
106 
107 struct 	fileops vnops = {
108 	.fo_read = vn_io_fault,
109 	.fo_write = vn_io_fault,
110 	.fo_truncate = vn_truncate,
111 	.fo_ioctl = vn_ioctl,
112 	.fo_poll = vn_poll,
113 	.fo_kqfilter = vn_kqfilter,
114 	.fo_stat = vn_statfile,
115 	.fo_close = vn_closefile,
116 	.fo_chmod = vn_chmod,
117 	.fo_chown = vn_chown,
118 	.fo_sendfile = vn_sendfile,
119 	.fo_seek = vn_seek,
120 	.fo_fill_kinfo = vn_fill_kinfo,
121 	.fo_mmap = vn_mmap,
122 	.fo_flags = DFLAG_PASSABLE | DFLAG_SEEKABLE
123 };
124 
125 static const int io_hold_cnt = 16;
126 static int vn_io_fault_enable = 1;
127 SYSCTL_INT(_debug, OID_AUTO, vn_io_fault_enable, CTLFLAG_RW,
128     &vn_io_fault_enable, 0, "Enable vn_io_fault lock avoidance");
129 static int vn_io_fault_prefault = 0;
130 SYSCTL_INT(_debug, OID_AUTO, vn_io_fault_prefault, CTLFLAG_RW,
131     &vn_io_fault_prefault, 0, "Enable vn_io_fault prefaulting");
132 static u_long vn_io_faults_cnt;
133 SYSCTL_ULONG(_debug, OID_AUTO, vn_io_faults, CTLFLAG_RD,
134     &vn_io_faults_cnt, 0, "Count of vn_io_fault lock avoidance triggers");
135 
136 /*
137  * Returns true if vn_io_fault mode of handling the i/o request should
138  * be used.
139  */
140 static bool
141 do_vn_io_fault(struct vnode *vp, struct uio *uio)
142 {
143 	struct mount *mp;
144 
145 	return (uio->uio_segflg == UIO_USERSPACE && vp->v_type == VREG &&
146 	    (mp = vp->v_mount) != NULL &&
147 	    (mp->mnt_kern_flag & MNTK_NO_IOPF) != 0 && vn_io_fault_enable);
148 }
149 
150 /*
151  * Structure used to pass arguments to vn_io_fault1(), to do either
152  * file- or vnode-based I/O calls.
153  */
154 struct vn_io_fault_args {
155 	enum {
156 		VN_IO_FAULT_FOP,
157 		VN_IO_FAULT_VOP
158 	} kind;
159 	struct ucred *cred;
160 	int flags;
161 	union {
162 		struct fop_args_tag {
163 			struct file *fp;
164 			fo_rdwr_t *doio;
165 		} fop_args;
166 		struct vop_args_tag {
167 			struct vnode *vp;
168 		} vop_args;
169 	} args;
170 };
171 
172 static int vn_io_fault1(struct vnode *vp, struct uio *uio,
173     struct vn_io_fault_args *args, struct thread *td);
174 
175 int
176 vn_open(struct nameidata *ndp, int *flagp, int cmode, struct file *fp)
177 {
178 	struct thread *td = ndp->ni_cnd.cn_thread;
179 
180 	return (vn_open_cred(ndp, flagp, cmode, 0, td->td_ucred, fp));
181 }
182 
183 /*
184  * Common code for vnode open operations via a name lookup.
185  * Lookup the vnode and invoke VOP_CREATE if needed.
186  * Check permissions, and call the VOP_OPEN or VOP_CREATE routine.
187  *
188  * Note that this does NOT free nameidata for the successful case,
189  * due to the NDINIT being done elsewhere.
190  */
191 int
192 vn_open_cred(struct nameidata *ndp, int *flagp, int cmode, u_int vn_open_flags,
193     struct ucred *cred, struct file *fp)
194 {
195 	struct vnode *vp;
196 	struct mount *mp;
197 	struct thread *td = ndp->ni_cnd.cn_thread;
198 	struct vattr vat;
199 	struct vattr *vap = &vat;
200 	int fmode, error;
201 
202 restart:
203 	fmode = *flagp;
204 	if ((fmode & (O_CREAT | O_EXCL | O_DIRECTORY)) == (O_CREAT |
205 	    O_EXCL | O_DIRECTORY))
206 		return (EINVAL);
207 	else if ((fmode & (O_CREAT | O_DIRECTORY)) == O_CREAT) {
208 		ndp->ni_cnd.cn_nameiop = CREATE;
209 		/*
210 		 * Set NOCACHE to avoid flushing the cache when
211 		 * rolling in many files at once.
212 		*/
213 		ndp->ni_cnd.cn_flags = ISOPEN | LOCKPARENT | LOCKLEAF | NOCACHE;
214 		if ((fmode & O_EXCL) == 0 && (fmode & O_NOFOLLOW) == 0)
215 			ndp->ni_cnd.cn_flags |= FOLLOW;
216 		if ((fmode & O_BENEATH) != 0)
217 			ndp->ni_cnd.cn_flags |= BENEATH;
218 		if (!(vn_open_flags & VN_OPEN_NOAUDIT))
219 			ndp->ni_cnd.cn_flags |= AUDITVNODE1;
220 		if (vn_open_flags & VN_OPEN_NOCAPCHECK)
221 			ndp->ni_cnd.cn_flags |= NOCAPCHECK;
222 		bwillwrite();
223 		if ((error = namei(ndp)) != 0)
224 			return (error);
225 		if (ndp->ni_vp == NULL) {
226 			VATTR_NULL(vap);
227 			vap->va_type = VREG;
228 			vap->va_mode = cmode;
229 			if (fmode & O_EXCL)
230 				vap->va_vaflags |= VA_EXCLUSIVE;
231 			if (vn_start_write(ndp->ni_dvp, &mp, V_NOWAIT) != 0) {
232 				NDFREE(ndp, NDF_ONLY_PNBUF);
233 				vput(ndp->ni_dvp);
234 				if ((error = vn_start_write(NULL, &mp,
235 				    V_XSLEEP | PCATCH)) != 0)
236 					return (error);
237 				goto restart;
238 			}
239 			if ((vn_open_flags & VN_OPEN_NAMECACHE) != 0)
240 				ndp->ni_cnd.cn_flags |= MAKEENTRY;
241 #ifdef MAC
242 			error = mac_vnode_check_create(cred, ndp->ni_dvp,
243 			    &ndp->ni_cnd, vap);
244 			if (error == 0)
245 #endif
246 				error = VOP_CREATE(ndp->ni_dvp, &ndp->ni_vp,
247 						   &ndp->ni_cnd, vap);
248 			vput(ndp->ni_dvp);
249 			vn_finished_write(mp);
250 			if (error) {
251 				NDFREE(ndp, NDF_ONLY_PNBUF);
252 				return (error);
253 			}
254 			fmode &= ~O_TRUNC;
255 			vp = ndp->ni_vp;
256 		} else {
257 			if (ndp->ni_dvp == ndp->ni_vp)
258 				vrele(ndp->ni_dvp);
259 			else
260 				vput(ndp->ni_dvp);
261 			ndp->ni_dvp = NULL;
262 			vp = ndp->ni_vp;
263 			if (fmode & O_EXCL) {
264 				error = EEXIST;
265 				goto bad;
266 			}
267 			if (vp->v_type == VDIR) {
268 				error = EISDIR;
269 				goto bad;
270 			}
271 			fmode &= ~O_CREAT;
272 		}
273 	} else {
274 		ndp->ni_cnd.cn_nameiop = LOOKUP;
275 		ndp->ni_cnd.cn_flags = ISOPEN |
276 		    ((fmode & O_NOFOLLOW) ? NOFOLLOW : FOLLOW) | LOCKLEAF;
277 		if (!(fmode & FWRITE))
278 			ndp->ni_cnd.cn_flags |= LOCKSHARED;
279 		if ((fmode & O_BENEATH) != 0)
280 			ndp->ni_cnd.cn_flags |= BENEATH;
281 		if (!(vn_open_flags & VN_OPEN_NOAUDIT))
282 			ndp->ni_cnd.cn_flags |= AUDITVNODE1;
283 		if (vn_open_flags & VN_OPEN_NOCAPCHECK)
284 			ndp->ni_cnd.cn_flags |= NOCAPCHECK;
285 		if ((error = namei(ndp)) != 0)
286 			return (error);
287 		vp = ndp->ni_vp;
288 	}
289 	error = vn_open_vnode(vp, fmode, cred, td, fp);
290 	if (error)
291 		goto bad;
292 	*flagp = fmode;
293 	return (0);
294 bad:
295 	NDFREE(ndp, NDF_ONLY_PNBUF);
296 	vput(vp);
297 	*flagp = fmode;
298 	ndp->ni_vp = NULL;
299 	return (error);
300 }
301 
302 static int
303 vn_open_vnode_advlock(struct vnode *vp, int fmode, struct file *fp)
304 {
305 	struct flock lf;
306 	int error, lock_flags, type;
307 
308 	ASSERT_VOP_LOCKED(vp, "vn_open_vnode_advlock");
309 	if ((fmode & (O_EXLOCK | O_SHLOCK)) == 0)
310 		return (0);
311 	KASSERT(fp != NULL, ("open with flock requires fp"));
312 	if (fp->f_type != DTYPE_NONE && fp->f_type != DTYPE_VNODE)
313 		return (EOPNOTSUPP);
314 
315 	lock_flags = VOP_ISLOCKED(vp);
316 	VOP_UNLOCK(vp, 0);
317 
318 	lf.l_whence = SEEK_SET;
319 	lf.l_start = 0;
320 	lf.l_len = 0;
321 	lf.l_type = (fmode & O_EXLOCK) != 0 ? F_WRLCK : F_RDLCK;
322 	type = F_FLOCK;
323 	if ((fmode & FNONBLOCK) == 0)
324 		type |= F_WAIT;
325 	error = VOP_ADVLOCK(vp, (caddr_t)fp, F_SETLK, &lf, type);
326 	if (error == 0)
327 		fp->f_flag |= FHASLOCK;
328 
329 	vn_lock(vp, lock_flags | LK_RETRY);
330 	if (error == 0 && (vp->v_iflag & VI_DOOMED) != 0)
331 		error = ENOENT;
332 	return (error);
333 }
334 
335 /*
336  * Common code for vnode open operations once a vnode is located.
337  * Check permissions, and call the VOP_OPEN routine.
338  */
339 int
340 vn_open_vnode(struct vnode *vp, int fmode, struct ucred *cred,
341     struct thread *td, struct file *fp)
342 {
343 	accmode_t accmode;
344 	int error;
345 
346 	if (vp->v_type == VLNK)
347 		return (EMLINK);
348 	if (vp->v_type == VSOCK)
349 		return (EOPNOTSUPP);
350 	if (vp->v_type != VDIR && fmode & O_DIRECTORY)
351 		return (ENOTDIR);
352 	accmode = 0;
353 	if (fmode & (FWRITE | O_TRUNC)) {
354 		if (vp->v_type == VDIR)
355 			return (EISDIR);
356 		accmode |= VWRITE;
357 	}
358 	if (fmode & FREAD)
359 		accmode |= VREAD;
360 	if (fmode & FEXEC)
361 		accmode |= VEXEC;
362 	if ((fmode & O_APPEND) && (fmode & FWRITE))
363 		accmode |= VAPPEND;
364 #ifdef MAC
365 	if (fmode & O_CREAT)
366 		accmode |= VCREAT;
367 	if (fmode & O_VERIFY)
368 		accmode |= VVERIFY;
369 	error = mac_vnode_check_open(cred, vp, accmode);
370 	if (error)
371 		return (error);
372 
373 	accmode &= ~(VCREAT | VVERIFY);
374 #endif
375 	if ((fmode & O_CREAT) == 0 && accmode != 0) {
376 		error = VOP_ACCESS(vp, accmode, cred, td);
377 		if (error != 0)
378 			return (error);
379 	}
380 	if (vp->v_type == VFIFO && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
381 		vn_lock(vp, LK_UPGRADE | LK_RETRY);
382 	error = VOP_OPEN(vp, fmode, cred, td, fp);
383 	if (error != 0)
384 		return (error);
385 
386 	error = vn_open_vnode_advlock(vp, fmode, fp);
387 	if (error == 0 && (fmode & FWRITE) != 0) {
388 		error = VOP_ADD_WRITECOUNT(vp, 1);
389 		if (error == 0) {
390 			CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d",
391 			     __func__, vp, vp->v_writecount);
392 		}
393 	}
394 
395 	/*
396 	 * Error from advlock or VOP_ADD_WRITECOUNT() still requires
397 	 * calling VOP_CLOSE() to pair with earlier VOP_OPEN().
398 	 * Arrange for that by having fdrop() to use vn_closefile().
399 	 */
400 	if (error != 0) {
401 		fp->f_flag |= FOPENFAILED;
402 		fp->f_vnode = vp;
403 		if (fp->f_ops == &badfileops) {
404 			fp->f_type = DTYPE_VNODE;
405 			fp->f_ops = &vnops;
406 		}
407 		vref(vp);
408 	}
409 
410 	ASSERT_VOP_LOCKED(vp, "vn_open_vnode");
411 	return (error);
412 
413 }
414 
415 /*
416  * Check for write permissions on the specified vnode.
417  * Prototype text segments cannot be written.
418  * It is racy.
419  */
420 int
421 vn_writechk(struct vnode *vp)
422 {
423 
424 	ASSERT_VOP_LOCKED(vp, "vn_writechk");
425 	/*
426 	 * If there's shared text associated with
427 	 * the vnode, try to free it up once.  If
428 	 * we fail, we can't allow writing.
429 	 */
430 	if (VOP_IS_TEXT(vp))
431 		return (ETXTBSY);
432 
433 	return (0);
434 }
435 
436 /*
437  * Vnode close call
438  */
439 static int
440 vn_close1(struct vnode *vp, int flags, struct ucred *file_cred,
441     struct thread *td, bool keep_ref)
442 {
443 	struct mount *mp;
444 	int error, lock_flags;
445 
446 	if (vp->v_type != VFIFO && (flags & FWRITE) == 0 &&
447 	    MNT_EXTENDED_SHARED(vp->v_mount))
448 		lock_flags = LK_SHARED;
449 	else
450 		lock_flags = LK_EXCLUSIVE;
451 
452 	vn_start_write(vp, &mp, V_WAIT);
453 	vn_lock(vp, lock_flags | LK_RETRY);
454 	AUDIT_ARG_VNODE1(vp);
455 	if ((flags & (FWRITE | FOPENFAILED)) == FWRITE) {
456 		VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
457 		CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d",
458 		    __func__, vp, vp->v_writecount);
459 	}
460 	error = VOP_CLOSE(vp, flags, file_cred, td);
461 	if (keep_ref)
462 		VOP_UNLOCK(vp, 0);
463 	else
464 		vput(vp);
465 	vn_finished_write(mp);
466 	return (error);
467 }
468 
469 int
470 vn_close(struct vnode *vp, int flags, struct ucred *file_cred,
471     struct thread *td)
472 {
473 
474 	return (vn_close1(vp, flags, file_cred, td, false));
475 }
476 
477 /*
478  * Heuristic to detect sequential operation.
479  */
480 static int
481 sequential_heuristic(struct uio *uio, struct file *fp)
482 {
483 
484 	ASSERT_VOP_LOCKED(fp->f_vnode, __func__);
485 	if (fp->f_flag & FRDAHEAD)
486 		return (fp->f_seqcount << IO_SEQSHIFT);
487 
488 	/*
489 	 * Offset 0 is handled specially.  open() sets f_seqcount to 1 so
490 	 * that the first I/O is normally considered to be slightly
491 	 * sequential.  Seeking to offset 0 doesn't change sequentiality
492 	 * unless previous seeks have reduced f_seqcount to 0, in which
493 	 * case offset 0 is not special.
494 	 */
495 	if ((uio->uio_offset == 0 && fp->f_seqcount > 0) ||
496 	    uio->uio_offset == fp->f_nextoff) {
497 		/*
498 		 * f_seqcount is in units of fixed-size blocks so that it
499 		 * depends mainly on the amount of sequential I/O and not
500 		 * much on the number of sequential I/O's.  The fixed size
501 		 * of 16384 is hard-coded here since it is (not quite) just
502 		 * a magic size that works well here.  This size is more
503 		 * closely related to the best I/O size for real disks than
504 		 * to any block size used by software.
505 		 */
506 		if (uio->uio_resid >= IO_SEQMAX * 16384)
507 			fp->f_seqcount = IO_SEQMAX;
508 		else {
509 			fp->f_seqcount += howmany(uio->uio_resid, 16384);
510 			if (fp->f_seqcount > IO_SEQMAX)
511 				fp->f_seqcount = IO_SEQMAX;
512 		}
513 		return (fp->f_seqcount << IO_SEQSHIFT);
514 	}
515 
516 	/* Not sequential.  Quickly draw-down sequentiality. */
517 	if (fp->f_seqcount > 1)
518 		fp->f_seqcount = 1;
519 	else
520 		fp->f_seqcount = 0;
521 	return (0);
522 }
523 
524 /*
525  * Package up an I/O request on a vnode into a uio and do it.
526  */
527 int
528 vn_rdwr(enum uio_rw rw, struct vnode *vp, void *base, int len, off_t offset,
529     enum uio_seg segflg, int ioflg, struct ucred *active_cred,
530     struct ucred *file_cred, ssize_t *aresid, struct thread *td)
531 {
532 	struct uio auio;
533 	struct iovec aiov;
534 	struct mount *mp;
535 	struct ucred *cred;
536 	void *rl_cookie;
537 	struct vn_io_fault_args args;
538 	int error, lock_flags;
539 
540 	if (offset < 0 && vp->v_type != VCHR)
541 		return (EINVAL);
542 	auio.uio_iov = &aiov;
543 	auio.uio_iovcnt = 1;
544 	aiov.iov_base = base;
545 	aiov.iov_len = len;
546 	auio.uio_resid = len;
547 	auio.uio_offset = offset;
548 	auio.uio_segflg = segflg;
549 	auio.uio_rw = rw;
550 	auio.uio_td = td;
551 	error = 0;
552 
553 	if ((ioflg & IO_NODELOCKED) == 0) {
554 		if ((ioflg & IO_RANGELOCKED) == 0) {
555 			if (rw == UIO_READ) {
556 				rl_cookie = vn_rangelock_rlock(vp, offset,
557 				    offset + len);
558 			} else {
559 				rl_cookie = vn_rangelock_wlock(vp, offset,
560 				    offset + len);
561 			}
562 		} else
563 			rl_cookie = NULL;
564 		mp = NULL;
565 		if (rw == UIO_WRITE) {
566 			if (vp->v_type != VCHR &&
567 			    (error = vn_start_write(vp, &mp, V_WAIT | PCATCH))
568 			    != 0)
569 				goto out;
570 			if (MNT_SHARED_WRITES(mp) ||
571 			    ((mp == NULL) && MNT_SHARED_WRITES(vp->v_mount)))
572 				lock_flags = LK_SHARED;
573 			else
574 				lock_flags = LK_EXCLUSIVE;
575 		} else
576 			lock_flags = LK_SHARED;
577 		vn_lock(vp, lock_flags | LK_RETRY);
578 	} else
579 		rl_cookie = NULL;
580 
581 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
582 #ifdef MAC
583 	if ((ioflg & IO_NOMACCHECK) == 0) {
584 		if (rw == UIO_READ)
585 			error = mac_vnode_check_read(active_cred, file_cred,
586 			    vp);
587 		else
588 			error = mac_vnode_check_write(active_cred, file_cred,
589 			    vp);
590 	}
591 #endif
592 	if (error == 0) {
593 		if (file_cred != NULL)
594 			cred = file_cred;
595 		else
596 			cred = active_cred;
597 		if (do_vn_io_fault(vp, &auio)) {
598 			args.kind = VN_IO_FAULT_VOP;
599 			args.cred = cred;
600 			args.flags = ioflg;
601 			args.args.vop_args.vp = vp;
602 			error = vn_io_fault1(vp, &auio, &args, td);
603 		} else if (rw == UIO_READ) {
604 			error = VOP_READ(vp, &auio, ioflg, cred);
605 		} else /* if (rw == UIO_WRITE) */ {
606 			error = VOP_WRITE(vp, &auio, ioflg, cred);
607 		}
608 	}
609 	if (aresid)
610 		*aresid = auio.uio_resid;
611 	else
612 		if (auio.uio_resid && error == 0)
613 			error = EIO;
614 	if ((ioflg & IO_NODELOCKED) == 0) {
615 		VOP_UNLOCK(vp, 0);
616 		if (mp != NULL)
617 			vn_finished_write(mp);
618 	}
619  out:
620 	if (rl_cookie != NULL)
621 		vn_rangelock_unlock(vp, rl_cookie);
622 	return (error);
623 }
624 
625 /*
626  * Package up an I/O request on a vnode into a uio and do it.  The I/O
627  * request is split up into smaller chunks and we try to avoid saturating
628  * the buffer cache while potentially holding a vnode locked, so we
629  * check bwillwrite() before calling vn_rdwr().  We also call kern_yield()
630  * to give other processes a chance to lock the vnode (either other processes
631  * core'ing the same binary, or unrelated processes scanning the directory).
632  */
633 int
634 vn_rdwr_inchunks(enum uio_rw rw, struct vnode *vp, void *base, size_t len,
635     off_t offset, enum uio_seg segflg, int ioflg, struct ucred *active_cred,
636     struct ucred *file_cred, size_t *aresid, struct thread *td)
637 {
638 	int error = 0;
639 	ssize_t iaresid;
640 
641 	do {
642 		int chunk;
643 
644 		/*
645 		 * Force `offset' to a multiple of MAXBSIZE except possibly
646 		 * for the first chunk, so that filesystems only need to
647 		 * write full blocks except possibly for the first and last
648 		 * chunks.
649 		 */
650 		chunk = MAXBSIZE - (uoff_t)offset % MAXBSIZE;
651 
652 		if (chunk > len)
653 			chunk = len;
654 		if (rw != UIO_READ && vp->v_type == VREG)
655 			bwillwrite();
656 		iaresid = 0;
657 		error = vn_rdwr(rw, vp, base, chunk, offset, segflg,
658 		    ioflg, active_cred, file_cred, &iaresid, td);
659 		len -= chunk;	/* aresid calc already includes length */
660 		if (error)
661 			break;
662 		offset += chunk;
663 		base = (char *)base + chunk;
664 		kern_yield(PRI_USER);
665 	} while (len);
666 	if (aresid)
667 		*aresid = len + iaresid;
668 	return (error);
669 }
670 
671 off_t
672 foffset_lock(struct file *fp, int flags)
673 {
674 	struct mtx *mtxp;
675 	off_t res;
676 
677 	KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed"));
678 
679 #if OFF_MAX <= LONG_MAX
680 	/*
681 	 * Caller only wants the current f_offset value.  Assume that
682 	 * the long and shorter integer types reads are atomic.
683 	 */
684 	if ((flags & FOF_NOLOCK) != 0)
685 		return (fp->f_offset);
686 #endif
687 
688 	/*
689 	 * According to McKusick the vn lock was protecting f_offset here.
690 	 * It is now protected by the FOFFSET_LOCKED flag.
691 	 */
692 	mtxp = mtx_pool_find(mtxpool_sleep, fp);
693 	mtx_lock(mtxp);
694 	if ((flags & FOF_NOLOCK) == 0) {
695 		while (fp->f_vnread_flags & FOFFSET_LOCKED) {
696 			fp->f_vnread_flags |= FOFFSET_LOCK_WAITING;
697 			msleep(&fp->f_vnread_flags, mtxp, PUSER -1,
698 			    "vofflock", 0);
699 		}
700 		fp->f_vnread_flags |= FOFFSET_LOCKED;
701 	}
702 	res = fp->f_offset;
703 	mtx_unlock(mtxp);
704 	return (res);
705 }
706 
707 void
708 foffset_unlock(struct file *fp, off_t val, int flags)
709 {
710 	struct mtx *mtxp;
711 
712 	KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed"));
713 
714 #if OFF_MAX <= LONG_MAX
715 	if ((flags & FOF_NOLOCK) != 0) {
716 		if ((flags & FOF_NOUPDATE) == 0)
717 			fp->f_offset = val;
718 		if ((flags & FOF_NEXTOFF) != 0)
719 			fp->f_nextoff = val;
720 		return;
721 	}
722 #endif
723 
724 	mtxp = mtx_pool_find(mtxpool_sleep, fp);
725 	mtx_lock(mtxp);
726 	if ((flags & FOF_NOUPDATE) == 0)
727 		fp->f_offset = val;
728 	if ((flags & FOF_NEXTOFF) != 0)
729 		fp->f_nextoff = val;
730 	if ((flags & FOF_NOLOCK) == 0) {
731 		KASSERT((fp->f_vnread_flags & FOFFSET_LOCKED) != 0,
732 		    ("Lost FOFFSET_LOCKED"));
733 		if (fp->f_vnread_flags & FOFFSET_LOCK_WAITING)
734 			wakeup(&fp->f_vnread_flags);
735 		fp->f_vnread_flags = 0;
736 	}
737 	mtx_unlock(mtxp);
738 }
739 
740 void
741 foffset_lock_uio(struct file *fp, struct uio *uio, int flags)
742 {
743 
744 	if ((flags & FOF_OFFSET) == 0)
745 		uio->uio_offset = foffset_lock(fp, flags);
746 }
747 
748 void
749 foffset_unlock_uio(struct file *fp, struct uio *uio, int flags)
750 {
751 
752 	if ((flags & FOF_OFFSET) == 0)
753 		foffset_unlock(fp, uio->uio_offset, flags);
754 }
755 
756 static int
757 get_advice(struct file *fp, struct uio *uio)
758 {
759 	struct mtx *mtxp;
760 	int ret;
761 
762 	ret = POSIX_FADV_NORMAL;
763 	if (fp->f_advice == NULL || fp->f_vnode->v_type != VREG)
764 		return (ret);
765 
766 	mtxp = mtx_pool_find(mtxpool_sleep, fp);
767 	mtx_lock(mtxp);
768 	if (fp->f_advice != NULL &&
769 	    uio->uio_offset >= fp->f_advice->fa_start &&
770 	    uio->uio_offset + uio->uio_resid <= fp->f_advice->fa_end)
771 		ret = fp->f_advice->fa_advice;
772 	mtx_unlock(mtxp);
773 	return (ret);
774 }
775 
776 /*
777  * File table vnode read routine.
778  */
779 static int
780 vn_read(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags,
781     struct thread *td)
782 {
783 	struct vnode *vp;
784 	off_t orig_offset;
785 	int error, ioflag;
786 	int advice;
787 
788 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
789 	    uio->uio_td, td));
790 	KASSERT(flags & FOF_OFFSET, ("No FOF_OFFSET"));
791 	vp = fp->f_vnode;
792 	ioflag = 0;
793 	if (fp->f_flag & FNONBLOCK)
794 		ioflag |= IO_NDELAY;
795 	if (fp->f_flag & O_DIRECT)
796 		ioflag |= IO_DIRECT;
797 	advice = get_advice(fp, uio);
798 	vn_lock(vp, LK_SHARED | LK_RETRY);
799 
800 	switch (advice) {
801 	case POSIX_FADV_NORMAL:
802 	case POSIX_FADV_SEQUENTIAL:
803 	case POSIX_FADV_NOREUSE:
804 		ioflag |= sequential_heuristic(uio, fp);
805 		break;
806 	case POSIX_FADV_RANDOM:
807 		/* Disable read-ahead for random I/O. */
808 		break;
809 	}
810 	orig_offset = uio->uio_offset;
811 
812 #ifdef MAC
813 	error = mac_vnode_check_read(active_cred, fp->f_cred, vp);
814 	if (error == 0)
815 #endif
816 		error = VOP_READ(vp, uio, ioflag, fp->f_cred);
817 	fp->f_nextoff = uio->uio_offset;
818 	VOP_UNLOCK(vp, 0);
819 	if (error == 0 && advice == POSIX_FADV_NOREUSE &&
820 	    orig_offset != uio->uio_offset)
821 		/*
822 		 * Use POSIX_FADV_DONTNEED to flush pages and buffers
823 		 * for the backing file after a POSIX_FADV_NOREUSE
824 		 * read(2).
825 		 */
826 		error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
827 		    POSIX_FADV_DONTNEED);
828 	return (error);
829 }
830 
831 /*
832  * File table vnode write routine.
833  */
834 static int
835 vn_write(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags,
836     struct thread *td)
837 {
838 	struct vnode *vp;
839 	struct mount *mp;
840 	off_t orig_offset;
841 	int error, ioflag, lock_flags;
842 	int advice;
843 
844 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
845 	    uio->uio_td, td));
846 	KASSERT(flags & FOF_OFFSET, ("No FOF_OFFSET"));
847 	vp = fp->f_vnode;
848 	if (vp->v_type == VREG)
849 		bwillwrite();
850 	ioflag = IO_UNIT;
851 	if (vp->v_type == VREG && (fp->f_flag & O_APPEND))
852 		ioflag |= IO_APPEND;
853 	if (fp->f_flag & FNONBLOCK)
854 		ioflag |= IO_NDELAY;
855 	if (fp->f_flag & O_DIRECT)
856 		ioflag |= IO_DIRECT;
857 	if ((fp->f_flag & O_FSYNC) ||
858 	    (vp->v_mount && (vp->v_mount->mnt_flag & MNT_SYNCHRONOUS)))
859 		ioflag |= IO_SYNC;
860 	mp = NULL;
861 	if (vp->v_type != VCHR &&
862 	    (error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
863 		goto unlock;
864 
865 	advice = get_advice(fp, uio);
866 
867 	if (MNT_SHARED_WRITES(mp) ||
868 	    (mp == NULL && MNT_SHARED_WRITES(vp->v_mount))) {
869 		lock_flags = LK_SHARED;
870 	} else {
871 		lock_flags = LK_EXCLUSIVE;
872 	}
873 
874 	vn_lock(vp, lock_flags | LK_RETRY);
875 	switch (advice) {
876 	case POSIX_FADV_NORMAL:
877 	case POSIX_FADV_SEQUENTIAL:
878 	case POSIX_FADV_NOREUSE:
879 		ioflag |= sequential_heuristic(uio, fp);
880 		break;
881 	case POSIX_FADV_RANDOM:
882 		/* XXX: Is this correct? */
883 		break;
884 	}
885 	orig_offset = uio->uio_offset;
886 
887 #ifdef MAC
888 	error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
889 	if (error == 0)
890 #endif
891 		error = VOP_WRITE(vp, uio, ioflag, fp->f_cred);
892 	fp->f_nextoff = uio->uio_offset;
893 	VOP_UNLOCK(vp, 0);
894 	if (vp->v_type != VCHR)
895 		vn_finished_write(mp);
896 	if (error == 0 && advice == POSIX_FADV_NOREUSE &&
897 	    orig_offset != uio->uio_offset)
898 		/*
899 		 * Use POSIX_FADV_DONTNEED to flush pages and buffers
900 		 * for the backing file after a POSIX_FADV_NOREUSE
901 		 * write(2).
902 		 */
903 		error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
904 		    POSIX_FADV_DONTNEED);
905 unlock:
906 	return (error);
907 }
908 
909 /*
910  * The vn_io_fault() is a wrapper around vn_read() and vn_write() to
911  * prevent the following deadlock:
912  *
913  * Assume that the thread A reads from the vnode vp1 into userspace
914  * buffer buf1 backed by the pages of vnode vp2.  If a page in buf1 is
915  * currently not resident, then system ends up with the call chain
916  *   vn_read() -> VOP_READ(vp1) -> uiomove() -> [Page Fault] ->
917  *     vm_fault(buf1) -> vnode_pager_getpages(vp2) -> VOP_GETPAGES(vp2)
918  * which establishes lock order vp1->vn_lock, then vp2->vn_lock.
919  * If, at the same time, thread B reads from vnode vp2 into buffer buf2
920  * backed by the pages of vnode vp1, and some page in buf2 is not
921  * resident, we get a reversed order vp2->vn_lock, then vp1->vn_lock.
922  *
923  * To prevent the lock order reversal and deadlock, vn_io_fault() does
924  * not allow page faults to happen during VOP_READ() or VOP_WRITE().
925  * Instead, it first tries to do the whole range i/o with pagefaults
926  * disabled. If all pages in the i/o buffer are resident and mapped,
927  * VOP will succeed (ignoring the genuine filesystem errors).
928  * Otherwise, we get back EFAULT, and vn_io_fault() falls back to do
929  * i/o in chunks, with all pages in the chunk prefaulted and held
930  * using vm_fault_quick_hold_pages().
931  *
932  * Filesystems using this deadlock avoidance scheme should use the
933  * array of the held pages from uio, saved in the curthread->td_ma,
934  * instead of doing uiomove().  A helper function
935  * vn_io_fault_uiomove() converts uiomove request into
936  * uiomove_fromphys() over td_ma array.
937  *
938  * Since vnode locks do not cover the whole i/o anymore, rangelocks
939  * make the current i/o request atomic with respect to other i/os and
940  * truncations.
941  */
942 
943 /*
944  * Decode vn_io_fault_args and perform the corresponding i/o.
945  */
946 static int
947 vn_io_fault_doio(struct vn_io_fault_args *args, struct uio *uio,
948     struct thread *td)
949 {
950 	int error, save;
951 
952 	error = 0;
953 	save = vm_fault_disable_pagefaults();
954 	switch (args->kind) {
955 	case VN_IO_FAULT_FOP:
956 		error = (args->args.fop_args.doio)(args->args.fop_args.fp,
957 		    uio, args->cred, args->flags, td);
958 		break;
959 	case VN_IO_FAULT_VOP:
960 		if (uio->uio_rw == UIO_READ) {
961 			error = VOP_READ(args->args.vop_args.vp, uio,
962 			    args->flags, args->cred);
963 		} else if (uio->uio_rw == UIO_WRITE) {
964 			error = VOP_WRITE(args->args.vop_args.vp, uio,
965 			    args->flags, args->cred);
966 		}
967 		break;
968 	default:
969 		panic("vn_io_fault_doio: unknown kind of io %d %d",
970 		    args->kind, uio->uio_rw);
971 	}
972 	vm_fault_enable_pagefaults(save);
973 	return (error);
974 }
975 
976 static int
977 vn_io_fault_touch(char *base, const struct uio *uio)
978 {
979 	int r;
980 
981 	r = fubyte(base);
982 	if (r == -1 || (uio->uio_rw == UIO_READ && subyte(base, r) == -1))
983 		return (EFAULT);
984 	return (0);
985 }
986 
987 static int
988 vn_io_fault_prefault_user(const struct uio *uio)
989 {
990 	char *base;
991 	const struct iovec *iov;
992 	size_t len;
993 	ssize_t resid;
994 	int error, i;
995 
996 	KASSERT(uio->uio_segflg == UIO_USERSPACE,
997 	    ("vn_io_fault_prefault userspace"));
998 
999 	error = i = 0;
1000 	iov = uio->uio_iov;
1001 	resid = uio->uio_resid;
1002 	base = iov->iov_base;
1003 	len = iov->iov_len;
1004 	while (resid > 0) {
1005 		error = vn_io_fault_touch(base, uio);
1006 		if (error != 0)
1007 			break;
1008 		if (len < PAGE_SIZE) {
1009 			if (len != 0) {
1010 				error = vn_io_fault_touch(base + len - 1, uio);
1011 				if (error != 0)
1012 					break;
1013 				resid -= len;
1014 			}
1015 			if (++i >= uio->uio_iovcnt)
1016 				break;
1017 			iov = uio->uio_iov + i;
1018 			base = iov->iov_base;
1019 			len = iov->iov_len;
1020 		} else {
1021 			len -= PAGE_SIZE;
1022 			base += PAGE_SIZE;
1023 			resid -= PAGE_SIZE;
1024 		}
1025 	}
1026 	return (error);
1027 }
1028 
1029 /*
1030  * Common code for vn_io_fault(), agnostic to the kind of i/o request.
1031  * Uses vn_io_fault_doio() to make the call to an actual i/o function.
1032  * Used from vn_rdwr() and vn_io_fault(), which encode the i/o request
1033  * into args and call vn_io_fault1() to handle faults during the user
1034  * mode buffer accesses.
1035  */
1036 static int
1037 vn_io_fault1(struct vnode *vp, struct uio *uio, struct vn_io_fault_args *args,
1038     struct thread *td)
1039 {
1040 	vm_page_t ma[io_hold_cnt + 2];
1041 	struct uio *uio_clone, short_uio;
1042 	struct iovec short_iovec[1];
1043 	vm_page_t *prev_td_ma;
1044 	vm_prot_t prot;
1045 	vm_offset_t addr, end;
1046 	size_t len, resid;
1047 	ssize_t adv;
1048 	int error, cnt, saveheld, prev_td_ma_cnt;
1049 
1050 	if (vn_io_fault_prefault) {
1051 		error = vn_io_fault_prefault_user(uio);
1052 		if (error != 0)
1053 			return (error); /* Or ignore ? */
1054 	}
1055 
1056 	prot = uio->uio_rw == UIO_READ ? VM_PROT_WRITE : VM_PROT_READ;
1057 
1058 	/*
1059 	 * The UFS follows IO_UNIT directive and replays back both
1060 	 * uio_offset and uio_resid if an error is encountered during the
1061 	 * operation.  But, since the iovec may be already advanced,
1062 	 * uio is still in an inconsistent state.
1063 	 *
1064 	 * Cache a copy of the original uio, which is advanced to the redo
1065 	 * point using UIO_NOCOPY below.
1066 	 */
1067 	uio_clone = cloneuio(uio);
1068 	resid = uio->uio_resid;
1069 
1070 	short_uio.uio_segflg = UIO_USERSPACE;
1071 	short_uio.uio_rw = uio->uio_rw;
1072 	short_uio.uio_td = uio->uio_td;
1073 
1074 	error = vn_io_fault_doio(args, uio, td);
1075 	if (error != EFAULT)
1076 		goto out;
1077 
1078 	atomic_add_long(&vn_io_faults_cnt, 1);
1079 	uio_clone->uio_segflg = UIO_NOCOPY;
1080 	uiomove(NULL, resid - uio->uio_resid, uio_clone);
1081 	uio_clone->uio_segflg = uio->uio_segflg;
1082 
1083 	saveheld = curthread_pflags_set(TDP_UIOHELD);
1084 	prev_td_ma = td->td_ma;
1085 	prev_td_ma_cnt = td->td_ma_cnt;
1086 
1087 	while (uio_clone->uio_resid != 0) {
1088 		len = uio_clone->uio_iov->iov_len;
1089 		if (len == 0) {
1090 			KASSERT(uio_clone->uio_iovcnt >= 1,
1091 			    ("iovcnt underflow"));
1092 			uio_clone->uio_iov++;
1093 			uio_clone->uio_iovcnt--;
1094 			continue;
1095 		}
1096 		if (len > io_hold_cnt * PAGE_SIZE)
1097 			len = io_hold_cnt * PAGE_SIZE;
1098 		addr = (uintptr_t)uio_clone->uio_iov->iov_base;
1099 		end = round_page(addr + len);
1100 		if (end < addr) {
1101 			error = EFAULT;
1102 			break;
1103 		}
1104 		cnt = atop(end - trunc_page(addr));
1105 		/*
1106 		 * A perfectly misaligned address and length could cause
1107 		 * both the start and the end of the chunk to use partial
1108 		 * page.  +2 accounts for such a situation.
1109 		 */
1110 		cnt = vm_fault_quick_hold_pages(&td->td_proc->p_vmspace->vm_map,
1111 		    addr, len, prot, ma, io_hold_cnt + 2);
1112 		if (cnt == -1) {
1113 			error = EFAULT;
1114 			break;
1115 		}
1116 		short_uio.uio_iov = &short_iovec[0];
1117 		short_iovec[0].iov_base = (void *)addr;
1118 		short_uio.uio_iovcnt = 1;
1119 		short_uio.uio_resid = short_iovec[0].iov_len = len;
1120 		short_uio.uio_offset = uio_clone->uio_offset;
1121 		td->td_ma = ma;
1122 		td->td_ma_cnt = cnt;
1123 
1124 		error = vn_io_fault_doio(args, &short_uio, td);
1125 		vm_page_unhold_pages(ma, cnt);
1126 		adv = len - short_uio.uio_resid;
1127 
1128 		uio_clone->uio_iov->iov_base =
1129 		    (char *)uio_clone->uio_iov->iov_base + adv;
1130 		uio_clone->uio_iov->iov_len -= adv;
1131 		uio_clone->uio_resid -= adv;
1132 		uio_clone->uio_offset += adv;
1133 
1134 		uio->uio_resid -= adv;
1135 		uio->uio_offset += adv;
1136 
1137 		if (error != 0 || adv == 0)
1138 			break;
1139 	}
1140 	td->td_ma = prev_td_ma;
1141 	td->td_ma_cnt = prev_td_ma_cnt;
1142 	curthread_pflags_restore(saveheld);
1143 out:
1144 	free(uio_clone, M_IOV);
1145 	return (error);
1146 }
1147 
1148 static int
1149 vn_io_fault(struct file *fp, struct uio *uio, struct ucred *active_cred,
1150     int flags, struct thread *td)
1151 {
1152 	fo_rdwr_t *doio;
1153 	struct vnode *vp;
1154 	void *rl_cookie;
1155 	struct vn_io_fault_args args;
1156 	int error;
1157 
1158 	doio = uio->uio_rw == UIO_READ ? vn_read : vn_write;
1159 	vp = fp->f_vnode;
1160 	foffset_lock_uio(fp, uio, flags);
1161 	if (do_vn_io_fault(vp, uio)) {
1162 		args.kind = VN_IO_FAULT_FOP;
1163 		args.args.fop_args.fp = fp;
1164 		args.args.fop_args.doio = doio;
1165 		args.cred = active_cred;
1166 		args.flags = flags | FOF_OFFSET;
1167 		if (uio->uio_rw == UIO_READ) {
1168 			rl_cookie = vn_rangelock_rlock(vp, uio->uio_offset,
1169 			    uio->uio_offset + uio->uio_resid);
1170 		} else if ((fp->f_flag & O_APPEND) != 0 ||
1171 		    (flags & FOF_OFFSET) == 0) {
1172 			/* For appenders, punt and lock the whole range. */
1173 			rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX);
1174 		} else {
1175 			rl_cookie = vn_rangelock_wlock(vp, uio->uio_offset,
1176 			    uio->uio_offset + uio->uio_resid);
1177 		}
1178 		error = vn_io_fault1(vp, uio, &args, td);
1179 		vn_rangelock_unlock(vp, rl_cookie);
1180 	} else {
1181 		error = doio(fp, uio, active_cred, flags | FOF_OFFSET, td);
1182 	}
1183 	foffset_unlock_uio(fp, uio, flags);
1184 	return (error);
1185 }
1186 
1187 /*
1188  * Helper function to perform the requested uiomove operation using
1189  * the held pages for io->uio_iov[0].iov_base buffer instead of
1190  * copyin/copyout.  Access to the pages with uiomove_fromphys()
1191  * instead of iov_base prevents page faults that could occur due to
1192  * pmap_collect() invalidating the mapping created by
1193  * vm_fault_quick_hold_pages(), or pageout daemon, page laundry or
1194  * object cleanup revoking the write access from page mappings.
1195  *
1196  * Filesystems specified MNTK_NO_IOPF shall use vn_io_fault_uiomove()
1197  * instead of plain uiomove().
1198  */
1199 int
1200 vn_io_fault_uiomove(char *data, int xfersize, struct uio *uio)
1201 {
1202 	struct uio transp_uio;
1203 	struct iovec transp_iov[1];
1204 	struct thread *td;
1205 	size_t adv;
1206 	int error, pgadv;
1207 
1208 	td = curthread;
1209 	if ((td->td_pflags & TDP_UIOHELD) == 0 ||
1210 	    uio->uio_segflg != UIO_USERSPACE)
1211 		return (uiomove(data, xfersize, uio));
1212 
1213 	KASSERT(uio->uio_iovcnt == 1, ("uio_iovcnt %d", uio->uio_iovcnt));
1214 	transp_iov[0].iov_base = data;
1215 	transp_uio.uio_iov = &transp_iov[0];
1216 	transp_uio.uio_iovcnt = 1;
1217 	if (xfersize > uio->uio_resid)
1218 		xfersize = uio->uio_resid;
1219 	transp_uio.uio_resid = transp_iov[0].iov_len = xfersize;
1220 	transp_uio.uio_offset = 0;
1221 	transp_uio.uio_segflg = UIO_SYSSPACE;
1222 	/*
1223 	 * Since transp_iov points to data, and td_ma page array
1224 	 * corresponds to original uio->uio_iov, we need to invert the
1225 	 * direction of the i/o operation as passed to
1226 	 * uiomove_fromphys().
1227 	 */
1228 	switch (uio->uio_rw) {
1229 	case UIO_WRITE:
1230 		transp_uio.uio_rw = UIO_READ;
1231 		break;
1232 	case UIO_READ:
1233 		transp_uio.uio_rw = UIO_WRITE;
1234 		break;
1235 	}
1236 	transp_uio.uio_td = uio->uio_td;
1237 	error = uiomove_fromphys(td->td_ma,
1238 	    ((vm_offset_t)uio->uio_iov->iov_base) & PAGE_MASK,
1239 	    xfersize, &transp_uio);
1240 	adv = xfersize - transp_uio.uio_resid;
1241 	pgadv =
1242 	    (((vm_offset_t)uio->uio_iov->iov_base + adv) >> PAGE_SHIFT) -
1243 	    (((vm_offset_t)uio->uio_iov->iov_base) >> PAGE_SHIFT);
1244 	td->td_ma += pgadv;
1245 	KASSERT(td->td_ma_cnt >= pgadv, ("consumed pages %d %d", td->td_ma_cnt,
1246 	    pgadv));
1247 	td->td_ma_cnt -= pgadv;
1248 	uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + adv;
1249 	uio->uio_iov->iov_len -= adv;
1250 	uio->uio_resid -= adv;
1251 	uio->uio_offset += adv;
1252 	return (error);
1253 }
1254 
1255 int
1256 vn_io_fault_pgmove(vm_page_t ma[], vm_offset_t offset, int xfersize,
1257     struct uio *uio)
1258 {
1259 	struct thread *td;
1260 	vm_offset_t iov_base;
1261 	int cnt, pgadv;
1262 
1263 	td = curthread;
1264 	if ((td->td_pflags & TDP_UIOHELD) == 0 ||
1265 	    uio->uio_segflg != UIO_USERSPACE)
1266 		return (uiomove_fromphys(ma, offset, xfersize, uio));
1267 
1268 	KASSERT(uio->uio_iovcnt == 1, ("uio_iovcnt %d", uio->uio_iovcnt));
1269 	cnt = xfersize > uio->uio_resid ? uio->uio_resid : xfersize;
1270 	iov_base = (vm_offset_t)uio->uio_iov->iov_base;
1271 	switch (uio->uio_rw) {
1272 	case UIO_WRITE:
1273 		pmap_copy_pages(td->td_ma, iov_base & PAGE_MASK, ma,
1274 		    offset, cnt);
1275 		break;
1276 	case UIO_READ:
1277 		pmap_copy_pages(ma, offset, td->td_ma, iov_base & PAGE_MASK,
1278 		    cnt);
1279 		break;
1280 	}
1281 	pgadv = ((iov_base + cnt) >> PAGE_SHIFT) - (iov_base >> PAGE_SHIFT);
1282 	td->td_ma += pgadv;
1283 	KASSERT(td->td_ma_cnt >= pgadv, ("consumed pages %d %d", td->td_ma_cnt,
1284 	    pgadv));
1285 	td->td_ma_cnt -= pgadv;
1286 	uio->uio_iov->iov_base = (char *)(iov_base + cnt);
1287 	uio->uio_iov->iov_len -= cnt;
1288 	uio->uio_resid -= cnt;
1289 	uio->uio_offset += cnt;
1290 	return (0);
1291 }
1292 
1293 
1294 /*
1295  * File table truncate routine.
1296  */
1297 static int
1298 vn_truncate(struct file *fp, off_t length, struct ucred *active_cred,
1299     struct thread *td)
1300 {
1301 	struct mount *mp;
1302 	struct vnode *vp;
1303 	void *rl_cookie;
1304 	int error;
1305 
1306 	vp = fp->f_vnode;
1307 
1308 	/*
1309 	 * Lock the whole range for truncation.  Otherwise split i/o
1310 	 * might happen partly before and partly after the truncation.
1311 	 */
1312 	rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX);
1313 	error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
1314 	if (error)
1315 		goto out1;
1316 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1317 	AUDIT_ARG_VNODE1(vp);
1318 	if (vp->v_type == VDIR) {
1319 		error = EISDIR;
1320 		goto out;
1321 	}
1322 #ifdef MAC
1323 	error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
1324 	if (error)
1325 		goto out;
1326 #endif
1327 	error = vn_truncate_locked(vp, length, (fp->f_flag & O_FSYNC) != 0,
1328 	    fp->f_cred);
1329 out:
1330 	VOP_UNLOCK(vp, 0);
1331 	vn_finished_write(mp);
1332 out1:
1333 	vn_rangelock_unlock(vp, rl_cookie);
1334 	return (error);
1335 }
1336 
1337 /*
1338  * Truncate a file that is already locked.
1339  */
1340 int
1341 vn_truncate_locked(struct vnode *vp, off_t length, bool sync,
1342     struct ucred *cred)
1343 {
1344 	struct vattr vattr;
1345 	int error;
1346 
1347 	error = VOP_ADD_WRITECOUNT(vp, 1);
1348 	if (error == 0) {
1349 		VATTR_NULL(&vattr);
1350 		vattr.va_size = length;
1351 		if (sync)
1352 			vattr.va_vaflags |= VA_SYNC;
1353 		error = VOP_SETATTR(vp, &vattr, cred);
1354 		VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
1355 	}
1356 	return (error);
1357 }
1358 
1359 /*
1360  * File table vnode stat routine.
1361  */
1362 static int
1363 vn_statfile(struct file *fp, struct stat *sb, struct ucred *active_cred,
1364     struct thread *td)
1365 {
1366 	struct vnode *vp = fp->f_vnode;
1367 	int error;
1368 
1369 	vn_lock(vp, LK_SHARED | LK_RETRY);
1370 	error = vn_stat(vp, sb, active_cred, fp->f_cred, td);
1371 	VOP_UNLOCK(vp, 0);
1372 
1373 	return (error);
1374 }
1375 
1376 /*
1377  * Stat a vnode; implementation for the stat syscall
1378  */
1379 int
1380 vn_stat(struct vnode *vp, struct stat *sb, struct ucred *active_cred,
1381     struct ucred *file_cred, struct thread *td)
1382 {
1383 	struct vattr vattr;
1384 	struct vattr *vap;
1385 	int error;
1386 	u_short mode;
1387 
1388 	AUDIT_ARG_VNODE1(vp);
1389 #ifdef MAC
1390 	error = mac_vnode_check_stat(active_cred, file_cred, vp);
1391 	if (error)
1392 		return (error);
1393 #endif
1394 
1395 	vap = &vattr;
1396 
1397 	/*
1398 	 * Initialize defaults for new and unusual fields, so that file
1399 	 * systems which don't support these fields don't need to know
1400 	 * about them.
1401 	 */
1402 	vap->va_birthtime.tv_sec = -1;
1403 	vap->va_birthtime.tv_nsec = 0;
1404 	vap->va_fsid = VNOVAL;
1405 	vap->va_rdev = NODEV;
1406 
1407 	error = VOP_GETATTR(vp, vap, active_cred);
1408 	if (error)
1409 		return (error);
1410 
1411 	/*
1412 	 * Zero the spare stat fields
1413 	 */
1414 	bzero(sb, sizeof *sb);
1415 
1416 	/*
1417 	 * Copy from vattr table
1418 	 */
1419 	if (vap->va_fsid != VNOVAL)
1420 		sb->st_dev = vap->va_fsid;
1421 	else
1422 		sb->st_dev = vp->v_mount->mnt_stat.f_fsid.val[0];
1423 	sb->st_ino = vap->va_fileid;
1424 	mode = vap->va_mode;
1425 	switch (vap->va_type) {
1426 	case VREG:
1427 		mode |= S_IFREG;
1428 		break;
1429 	case VDIR:
1430 		mode |= S_IFDIR;
1431 		break;
1432 	case VBLK:
1433 		mode |= S_IFBLK;
1434 		break;
1435 	case VCHR:
1436 		mode |= S_IFCHR;
1437 		break;
1438 	case VLNK:
1439 		mode |= S_IFLNK;
1440 		break;
1441 	case VSOCK:
1442 		mode |= S_IFSOCK;
1443 		break;
1444 	case VFIFO:
1445 		mode |= S_IFIFO;
1446 		break;
1447 	default:
1448 		return (EBADF);
1449 	}
1450 	sb->st_mode = mode;
1451 	sb->st_nlink = vap->va_nlink;
1452 	sb->st_uid = vap->va_uid;
1453 	sb->st_gid = vap->va_gid;
1454 	sb->st_rdev = vap->va_rdev;
1455 	if (vap->va_size > OFF_MAX)
1456 		return (EOVERFLOW);
1457 	sb->st_size = vap->va_size;
1458 	sb->st_atim = vap->va_atime;
1459 	sb->st_mtim = vap->va_mtime;
1460 	sb->st_ctim = vap->va_ctime;
1461 	sb->st_birthtim = vap->va_birthtime;
1462 
1463         /*
1464 	 * According to www.opengroup.org, the meaning of st_blksize is
1465 	 *   "a filesystem-specific preferred I/O block size for this
1466 	 *    object.  In some filesystem types, this may vary from file
1467 	 *    to file"
1468 	 * Use miminum/default of PAGE_SIZE (e.g. for VCHR).
1469 	 */
1470 
1471 	sb->st_blksize = max(PAGE_SIZE, vap->va_blocksize);
1472 
1473 	sb->st_flags = vap->va_flags;
1474 	if (priv_check(td, PRIV_VFS_GENERATION))
1475 		sb->st_gen = 0;
1476 	else
1477 		sb->st_gen = vap->va_gen;
1478 
1479 	sb->st_blocks = vap->va_bytes / S_BLKSIZE;
1480 	return (0);
1481 }
1482 
1483 /*
1484  * File table vnode ioctl routine.
1485  */
1486 static int
1487 vn_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred,
1488     struct thread *td)
1489 {
1490 	struct vattr vattr;
1491 	struct vnode *vp;
1492 	struct fiobmap2_arg *bmarg;
1493 	int error;
1494 
1495 	vp = fp->f_vnode;
1496 	switch (vp->v_type) {
1497 	case VDIR:
1498 	case VREG:
1499 		switch (com) {
1500 		case FIONREAD:
1501 			vn_lock(vp, LK_SHARED | LK_RETRY);
1502 			error = VOP_GETATTR(vp, &vattr, active_cred);
1503 			VOP_UNLOCK(vp, 0);
1504 			if (error == 0)
1505 				*(int *)data = vattr.va_size - fp->f_offset;
1506 			return (error);
1507 		case FIOBMAP2:
1508 			bmarg = (struct fiobmap2_arg *)data;
1509 			vn_lock(vp, LK_SHARED | LK_RETRY);
1510 #ifdef MAC
1511 			error = mac_vnode_check_read(active_cred, fp->f_cred,
1512 			    vp);
1513 			if (error == 0)
1514 #endif
1515 				error = VOP_BMAP(vp, bmarg->bn, NULL,
1516 				    &bmarg->bn, &bmarg->runp, &bmarg->runb);
1517 			VOP_UNLOCK(vp, 0);
1518 			return (error);
1519 		case FIONBIO:
1520 		case FIOASYNC:
1521 			return (0);
1522 		default:
1523 			return (VOP_IOCTL(vp, com, data, fp->f_flag,
1524 			    active_cred, td));
1525 		}
1526 		break;
1527 	case VCHR:
1528 		return (VOP_IOCTL(vp, com, data, fp->f_flag,
1529 		    active_cred, td));
1530 	default:
1531 		return (ENOTTY);
1532 	}
1533 }
1534 
1535 /*
1536  * File table vnode poll routine.
1537  */
1538 static int
1539 vn_poll(struct file *fp, int events, struct ucred *active_cred,
1540     struct thread *td)
1541 {
1542 	struct vnode *vp;
1543 	int error;
1544 
1545 	vp = fp->f_vnode;
1546 #ifdef MAC
1547 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1548 	AUDIT_ARG_VNODE1(vp);
1549 	error = mac_vnode_check_poll(active_cred, fp->f_cred, vp);
1550 	VOP_UNLOCK(vp, 0);
1551 	if (!error)
1552 #endif
1553 
1554 	error = VOP_POLL(vp, events, fp->f_cred, td);
1555 	return (error);
1556 }
1557 
1558 /*
1559  * Acquire the requested lock and then check for validity.  LK_RETRY
1560  * permits vn_lock to return doomed vnodes.
1561  */
1562 int
1563 _vn_lock(struct vnode *vp, int flags, char *file, int line)
1564 {
1565 	int error;
1566 
1567 	VNASSERT((flags & LK_TYPE_MASK) != 0, vp,
1568 	    ("vn_lock: no locktype"));
1569 	VNASSERT(vp->v_holdcnt != 0, vp, ("vn_lock: zero hold count"));
1570 retry:
1571 	error = VOP_LOCK1(vp, flags, file, line);
1572 	flags &= ~LK_INTERLOCK;	/* Interlock is always dropped. */
1573 	KASSERT((flags & LK_RETRY) == 0 || error == 0,
1574 	    ("vn_lock: error %d incompatible with flags %#x", error, flags));
1575 
1576 	if ((flags & LK_RETRY) == 0) {
1577 		if (error == 0 && (vp->v_iflag & VI_DOOMED) != 0) {
1578 			VOP_UNLOCK(vp, 0);
1579 			error = ENOENT;
1580 		}
1581 	} else if (error != 0)
1582 		goto retry;
1583 	return (error);
1584 }
1585 
1586 /*
1587  * File table vnode close routine.
1588  */
1589 static int
1590 vn_closefile(struct file *fp, struct thread *td)
1591 {
1592 	struct vnode *vp;
1593 	struct flock lf;
1594 	int error;
1595 	bool ref;
1596 
1597 	vp = fp->f_vnode;
1598 	fp->f_ops = &badfileops;
1599 	ref= (fp->f_flag & FHASLOCK) != 0 && fp->f_type == DTYPE_VNODE;
1600 
1601 	error = vn_close1(vp, fp->f_flag, fp->f_cred, td, ref);
1602 
1603 	if (__predict_false(ref)) {
1604 		lf.l_whence = SEEK_SET;
1605 		lf.l_start = 0;
1606 		lf.l_len = 0;
1607 		lf.l_type = F_UNLCK;
1608 		(void) VOP_ADVLOCK(vp, fp, F_UNLCK, &lf, F_FLOCK);
1609 		vrele(vp);
1610 	}
1611 	return (error);
1612 }
1613 
1614 static bool
1615 vn_suspendable(struct mount *mp)
1616 {
1617 
1618 	return (mp->mnt_op->vfs_susp_clean != NULL);
1619 }
1620 
1621 /*
1622  * Preparing to start a filesystem write operation. If the operation is
1623  * permitted, then we bump the count of operations in progress and
1624  * proceed. If a suspend request is in progress, we wait until the
1625  * suspension is over, and then proceed.
1626  */
1627 static int
1628 vn_start_write_refed(struct mount *mp, int flags, bool mplocked)
1629 {
1630 	int error, mflags;
1631 
1632 	if (__predict_true(!mplocked) && (flags & V_XSLEEP) == 0 &&
1633 	    vfs_op_thread_enter(mp)) {
1634 		MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0);
1635 		vfs_mp_count_add_pcpu(mp, writeopcount, 1);
1636 		vfs_op_thread_exit(mp);
1637 		return (0);
1638 	}
1639 
1640 	if (mplocked)
1641 		mtx_assert(MNT_MTX(mp), MA_OWNED);
1642 	else
1643 		MNT_ILOCK(mp);
1644 
1645 	error = 0;
1646 
1647 	/*
1648 	 * Check on status of suspension.
1649 	 */
1650 	if ((curthread->td_pflags & TDP_IGNSUSP) == 0 ||
1651 	    mp->mnt_susp_owner != curthread) {
1652 		mflags = ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ?
1653 		    (flags & PCATCH) : 0) | (PUSER - 1);
1654 		while ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) {
1655 			if (flags & V_NOWAIT) {
1656 				error = EWOULDBLOCK;
1657 				goto unlock;
1658 			}
1659 			error = msleep(&mp->mnt_flag, MNT_MTX(mp), mflags,
1660 			    "suspfs", 0);
1661 			if (error)
1662 				goto unlock;
1663 		}
1664 	}
1665 	if (flags & V_XSLEEP)
1666 		goto unlock;
1667 	mp->mnt_writeopcount++;
1668 unlock:
1669 	if (error != 0 || (flags & V_XSLEEP) != 0)
1670 		MNT_REL(mp);
1671 	MNT_IUNLOCK(mp);
1672 	return (error);
1673 }
1674 
1675 int
1676 vn_start_write(struct vnode *vp, struct mount **mpp, int flags)
1677 {
1678 	struct mount *mp;
1679 	int error;
1680 
1681 	KASSERT((flags & V_MNTREF) == 0 || (*mpp != NULL && vp == NULL),
1682 	    ("V_MNTREF requires mp"));
1683 
1684 	error = 0;
1685 	/*
1686 	 * If a vnode is provided, get and return the mount point that
1687 	 * to which it will write.
1688 	 */
1689 	if (vp != NULL) {
1690 		if ((error = VOP_GETWRITEMOUNT(vp, mpp)) != 0) {
1691 			*mpp = NULL;
1692 			if (error != EOPNOTSUPP)
1693 				return (error);
1694 			return (0);
1695 		}
1696 	}
1697 	if ((mp = *mpp) == NULL)
1698 		return (0);
1699 
1700 	if (!vn_suspendable(mp)) {
1701 		if (vp != NULL || (flags & V_MNTREF) != 0)
1702 			vfs_rel(mp);
1703 		return (0);
1704 	}
1705 
1706 	/*
1707 	 * VOP_GETWRITEMOUNT() returns with the mp refcount held through
1708 	 * a vfs_ref().
1709 	 * As long as a vnode is not provided we need to acquire a
1710 	 * refcount for the provided mountpoint too, in order to
1711 	 * emulate a vfs_ref().
1712 	 */
1713 	if (vp == NULL && (flags & V_MNTREF) == 0)
1714 		vfs_ref(mp);
1715 
1716 	return (vn_start_write_refed(mp, flags, false));
1717 }
1718 
1719 /*
1720  * Secondary suspension. Used by operations such as vop_inactive
1721  * routines that are needed by the higher level functions. These
1722  * are allowed to proceed until all the higher level functions have
1723  * completed (indicated by mnt_writeopcount dropping to zero). At that
1724  * time, these operations are halted until the suspension is over.
1725  */
1726 int
1727 vn_start_secondary_write(struct vnode *vp, struct mount **mpp, int flags)
1728 {
1729 	struct mount *mp;
1730 	int error;
1731 
1732 	KASSERT((flags & V_MNTREF) == 0 || (*mpp != NULL && vp == NULL),
1733 	    ("V_MNTREF requires mp"));
1734 
1735  retry:
1736 	if (vp != NULL) {
1737 		if ((error = VOP_GETWRITEMOUNT(vp, mpp)) != 0) {
1738 			*mpp = NULL;
1739 			if (error != EOPNOTSUPP)
1740 				return (error);
1741 			return (0);
1742 		}
1743 	}
1744 	/*
1745 	 * If we are not suspended or have not yet reached suspended
1746 	 * mode, then let the operation proceed.
1747 	 */
1748 	if ((mp = *mpp) == NULL)
1749 		return (0);
1750 
1751 	if (!vn_suspendable(mp)) {
1752 		if (vp != NULL || (flags & V_MNTREF) != 0)
1753 			vfs_rel(mp);
1754 		return (0);
1755 	}
1756 
1757 	/*
1758 	 * VOP_GETWRITEMOUNT() returns with the mp refcount held through
1759 	 * a vfs_ref().
1760 	 * As long as a vnode is not provided we need to acquire a
1761 	 * refcount for the provided mountpoint too, in order to
1762 	 * emulate a vfs_ref().
1763 	 */
1764 	MNT_ILOCK(mp);
1765 	if (vp == NULL && (flags & V_MNTREF) == 0)
1766 		MNT_REF(mp);
1767 	if ((mp->mnt_kern_flag & (MNTK_SUSPENDED | MNTK_SUSPEND2)) == 0) {
1768 		mp->mnt_secondary_writes++;
1769 		mp->mnt_secondary_accwrites++;
1770 		MNT_IUNLOCK(mp);
1771 		return (0);
1772 	}
1773 	if (flags & V_NOWAIT) {
1774 		MNT_REL(mp);
1775 		MNT_IUNLOCK(mp);
1776 		return (EWOULDBLOCK);
1777 	}
1778 	/*
1779 	 * Wait for the suspension to finish.
1780 	 */
1781 	error = msleep(&mp->mnt_flag, MNT_MTX(mp), (PUSER - 1) | PDROP |
1782 	    ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ? (flags & PCATCH) : 0),
1783 	    "suspfs", 0);
1784 	vfs_rel(mp);
1785 	if (error == 0)
1786 		goto retry;
1787 	return (error);
1788 }
1789 
1790 /*
1791  * Filesystem write operation has completed. If we are suspending and this
1792  * operation is the last one, notify the suspender that the suspension is
1793  * now in effect.
1794  */
1795 void
1796 vn_finished_write(struct mount *mp)
1797 {
1798 	int c;
1799 
1800 	if (mp == NULL || !vn_suspendable(mp))
1801 		return;
1802 
1803 	if (vfs_op_thread_enter(mp)) {
1804 		vfs_mp_count_sub_pcpu(mp, writeopcount, 1);
1805 		vfs_mp_count_sub_pcpu(mp, ref, 1);
1806 		vfs_op_thread_exit(mp);
1807 		return;
1808 	}
1809 
1810 	MNT_ILOCK(mp);
1811 	vfs_assert_mount_counters(mp);
1812 	MNT_REL(mp);
1813 	c = --mp->mnt_writeopcount;
1814 	if (mp->mnt_vfs_ops == 0) {
1815 		MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0);
1816 		MNT_IUNLOCK(mp);
1817 		return;
1818 	}
1819 	if (c < 0)
1820 		vfs_dump_mount_counters(mp);
1821 	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 && c == 0)
1822 		wakeup(&mp->mnt_writeopcount);
1823 	MNT_IUNLOCK(mp);
1824 }
1825 
1826 
1827 /*
1828  * Filesystem secondary write operation has completed. If we are
1829  * suspending and this operation is the last one, notify the suspender
1830  * that the suspension is now in effect.
1831  */
1832 void
1833 vn_finished_secondary_write(struct mount *mp)
1834 {
1835 	if (mp == NULL || !vn_suspendable(mp))
1836 		return;
1837 	MNT_ILOCK(mp);
1838 	MNT_REL(mp);
1839 	mp->mnt_secondary_writes--;
1840 	if (mp->mnt_secondary_writes < 0)
1841 		panic("vn_finished_secondary_write: neg cnt");
1842 	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 &&
1843 	    mp->mnt_secondary_writes <= 0)
1844 		wakeup(&mp->mnt_secondary_writes);
1845 	MNT_IUNLOCK(mp);
1846 }
1847 
1848 
1849 
1850 /*
1851  * Request a filesystem to suspend write operations.
1852  */
1853 int
1854 vfs_write_suspend(struct mount *mp, int flags)
1855 {
1856 	int error;
1857 
1858 	MPASS(vn_suspendable(mp));
1859 
1860 	vfs_op_enter(mp);
1861 
1862 	MNT_ILOCK(mp);
1863 	vfs_assert_mount_counters(mp);
1864 	if (mp->mnt_susp_owner == curthread) {
1865 		vfs_op_exit_locked(mp);
1866 		MNT_IUNLOCK(mp);
1867 		return (EALREADY);
1868 	}
1869 	while (mp->mnt_kern_flag & MNTK_SUSPEND)
1870 		msleep(&mp->mnt_flag, MNT_MTX(mp), PUSER - 1, "wsuspfs", 0);
1871 
1872 	/*
1873 	 * Unmount holds a write reference on the mount point.  If we
1874 	 * own busy reference and drain for writers, we deadlock with
1875 	 * the reference draining in the unmount path.  Callers of
1876 	 * vfs_write_suspend() must specify VS_SKIP_UNMOUNT if
1877 	 * vfs_busy() reference is owned and caller is not in the
1878 	 * unmount context.
1879 	 */
1880 	if ((flags & VS_SKIP_UNMOUNT) != 0 &&
1881 	    (mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) {
1882 		vfs_op_exit_locked(mp);
1883 		MNT_IUNLOCK(mp);
1884 		return (EBUSY);
1885 	}
1886 
1887 	mp->mnt_kern_flag |= MNTK_SUSPEND;
1888 	mp->mnt_susp_owner = curthread;
1889 	if (mp->mnt_writeopcount > 0)
1890 		(void) msleep(&mp->mnt_writeopcount,
1891 		    MNT_MTX(mp), (PUSER - 1)|PDROP, "suspwt", 0);
1892 	else
1893 		MNT_IUNLOCK(mp);
1894 	if ((error = VFS_SYNC(mp, MNT_SUSPEND)) != 0) {
1895 		vfs_write_resume(mp, 0);
1896 		vfs_op_exit(mp);
1897 	}
1898 	return (error);
1899 }
1900 
1901 /*
1902  * Request a filesystem to resume write operations.
1903  */
1904 void
1905 vfs_write_resume(struct mount *mp, int flags)
1906 {
1907 
1908 	MPASS(vn_suspendable(mp));
1909 
1910 	MNT_ILOCK(mp);
1911 	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) {
1912 		KASSERT(mp->mnt_susp_owner == curthread, ("mnt_susp_owner"));
1913 		mp->mnt_kern_flag &= ~(MNTK_SUSPEND | MNTK_SUSPEND2 |
1914 				       MNTK_SUSPENDED);
1915 		mp->mnt_susp_owner = NULL;
1916 		wakeup(&mp->mnt_writeopcount);
1917 		wakeup(&mp->mnt_flag);
1918 		curthread->td_pflags &= ~TDP_IGNSUSP;
1919 		if ((flags & VR_START_WRITE) != 0) {
1920 			MNT_REF(mp);
1921 			mp->mnt_writeopcount++;
1922 		}
1923 		MNT_IUNLOCK(mp);
1924 		if ((flags & VR_NO_SUSPCLR) == 0)
1925 			VFS_SUSP_CLEAN(mp);
1926 		vfs_op_exit(mp);
1927 	} else if ((flags & VR_START_WRITE) != 0) {
1928 		MNT_REF(mp);
1929 		vn_start_write_refed(mp, 0, true);
1930 	} else {
1931 		MNT_IUNLOCK(mp);
1932 	}
1933 }
1934 
1935 /*
1936  * Helper loop around vfs_write_suspend() for filesystem unmount VFS
1937  * methods.
1938  */
1939 int
1940 vfs_write_suspend_umnt(struct mount *mp)
1941 {
1942 	int error;
1943 
1944 	MPASS(vn_suspendable(mp));
1945 	KASSERT((curthread->td_pflags & TDP_IGNSUSP) == 0,
1946 	    ("vfs_write_suspend_umnt: recursed"));
1947 
1948 	/* dounmount() already called vn_start_write(). */
1949 	for (;;) {
1950 		vn_finished_write(mp);
1951 		error = vfs_write_suspend(mp, 0);
1952 		if (error != 0) {
1953 			vn_start_write(NULL, &mp, V_WAIT);
1954 			return (error);
1955 		}
1956 		MNT_ILOCK(mp);
1957 		if ((mp->mnt_kern_flag & MNTK_SUSPENDED) != 0)
1958 			break;
1959 		MNT_IUNLOCK(mp);
1960 		vn_start_write(NULL, &mp, V_WAIT);
1961 	}
1962 	mp->mnt_kern_flag &= ~(MNTK_SUSPENDED | MNTK_SUSPEND2);
1963 	wakeup(&mp->mnt_flag);
1964 	MNT_IUNLOCK(mp);
1965 	curthread->td_pflags |= TDP_IGNSUSP;
1966 	return (0);
1967 }
1968 
1969 /*
1970  * Implement kqueues for files by translating it to vnode operation.
1971  */
1972 static int
1973 vn_kqfilter(struct file *fp, struct knote *kn)
1974 {
1975 
1976 	return (VOP_KQFILTER(fp->f_vnode, kn));
1977 }
1978 
1979 /*
1980  * Simplified in-kernel wrapper calls for extended attribute access.
1981  * Both calls pass in a NULL credential, authorizing as "kernel" access.
1982  * Set IO_NODELOCKED in ioflg if the vnode is already locked.
1983  */
1984 int
1985 vn_extattr_get(struct vnode *vp, int ioflg, int attrnamespace,
1986     const char *attrname, int *buflen, char *buf, struct thread *td)
1987 {
1988 	struct uio	auio;
1989 	struct iovec	iov;
1990 	int	error;
1991 
1992 	iov.iov_len = *buflen;
1993 	iov.iov_base = buf;
1994 
1995 	auio.uio_iov = &iov;
1996 	auio.uio_iovcnt = 1;
1997 	auio.uio_rw = UIO_READ;
1998 	auio.uio_segflg = UIO_SYSSPACE;
1999 	auio.uio_td = td;
2000 	auio.uio_offset = 0;
2001 	auio.uio_resid = *buflen;
2002 
2003 	if ((ioflg & IO_NODELOCKED) == 0)
2004 		vn_lock(vp, LK_SHARED | LK_RETRY);
2005 
2006 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
2007 
2008 	/* authorize attribute retrieval as kernel */
2009 	error = VOP_GETEXTATTR(vp, attrnamespace, attrname, &auio, NULL, NULL,
2010 	    td);
2011 
2012 	if ((ioflg & IO_NODELOCKED) == 0)
2013 		VOP_UNLOCK(vp, 0);
2014 
2015 	if (error == 0) {
2016 		*buflen = *buflen - auio.uio_resid;
2017 	}
2018 
2019 	return (error);
2020 }
2021 
2022 /*
2023  * XXX failure mode if partially written?
2024  */
2025 int
2026 vn_extattr_set(struct vnode *vp, int ioflg, int attrnamespace,
2027     const char *attrname, int buflen, char *buf, struct thread *td)
2028 {
2029 	struct uio	auio;
2030 	struct iovec	iov;
2031 	struct mount	*mp;
2032 	int	error;
2033 
2034 	iov.iov_len = buflen;
2035 	iov.iov_base = buf;
2036 
2037 	auio.uio_iov = &iov;
2038 	auio.uio_iovcnt = 1;
2039 	auio.uio_rw = UIO_WRITE;
2040 	auio.uio_segflg = UIO_SYSSPACE;
2041 	auio.uio_td = td;
2042 	auio.uio_offset = 0;
2043 	auio.uio_resid = buflen;
2044 
2045 	if ((ioflg & IO_NODELOCKED) == 0) {
2046 		if ((error = vn_start_write(vp, &mp, V_WAIT)) != 0)
2047 			return (error);
2048 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2049 	}
2050 
2051 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
2052 
2053 	/* authorize attribute setting as kernel */
2054 	error = VOP_SETEXTATTR(vp, attrnamespace, attrname, &auio, NULL, td);
2055 
2056 	if ((ioflg & IO_NODELOCKED) == 0) {
2057 		vn_finished_write(mp);
2058 		VOP_UNLOCK(vp, 0);
2059 	}
2060 
2061 	return (error);
2062 }
2063 
2064 int
2065 vn_extattr_rm(struct vnode *vp, int ioflg, int attrnamespace,
2066     const char *attrname, struct thread *td)
2067 {
2068 	struct mount	*mp;
2069 	int	error;
2070 
2071 	if ((ioflg & IO_NODELOCKED) == 0) {
2072 		if ((error = vn_start_write(vp, &mp, V_WAIT)) != 0)
2073 			return (error);
2074 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2075 	}
2076 
2077 	ASSERT_VOP_LOCKED(vp, "IO_NODELOCKED with no vp lock held");
2078 
2079 	/* authorize attribute removal as kernel */
2080 	error = VOP_DELETEEXTATTR(vp, attrnamespace, attrname, NULL, td);
2081 	if (error == EOPNOTSUPP)
2082 		error = VOP_SETEXTATTR(vp, attrnamespace, attrname, NULL,
2083 		    NULL, td);
2084 
2085 	if ((ioflg & IO_NODELOCKED) == 0) {
2086 		vn_finished_write(mp);
2087 		VOP_UNLOCK(vp, 0);
2088 	}
2089 
2090 	return (error);
2091 }
2092 
2093 static int
2094 vn_get_ino_alloc_vget(struct mount *mp, void *arg, int lkflags,
2095     struct vnode **rvp)
2096 {
2097 
2098 	return (VFS_VGET(mp, *(ino_t *)arg, lkflags, rvp));
2099 }
2100 
2101 int
2102 vn_vget_ino(struct vnode *vp, ino_t ino, int lkflags, struct vnode **rvp)
2103 {
2104 
2105 	return (vn_vget_ino_gen(vp, vn_get_ino_alloc_vget, &ino,
2106 	    lkflags, rvp));
2107 }
2108 
2109 int
2110 vn_vget_ino_gen(struct vnode *vp, vn_get_ino_t alloc, void *alloc_arg,
2111     int lkflags, struct vnode **rvp)
2112 {
2113 	struct mount *mp;
2114 	int ltype, error;
2115 
2116 	ASSERT_VOP_LOCKED(vp, "vn_vget_ino_get");
2117 	mp = vp->v_mount;
2118 	ltype = VOP_ISLOCKED(vp);
2119 	KASSERT(ltype == LK_EXCLUSIVE || ltype == LK_SHARED,
2120 	    ("vn_vget_ino: vp not locked"));
2121 	error = vfs_busy(mp, MBF_NOWAIT);
2122 	if (error != 0) {
2123 		vfs_ref(mp);
2124 		VOP_UNLOCK(vp, 0);
2125 		error = vfs_busy(mp, 0);
2126 		vn_lock(vp, ltype | LK_RETRY);
2127 		vfs_rel(mp);
2128 		if (error != 0)
2129 			return (ENOENT);
2130 		if (vp->v_iflag & VI_DOOMED) {
2131 			vfs_unbusy(mp);
2132 			return (ENOENT);
2133 		}
2134 	}
2135 	VOP_UNLOCK(vp, 0);
2136 	error = alloc(mp, alloc_arg, lkflags, rvp);
2137 	vfs_unbusy(mp);
2138 	if (error != 0 || *rvp != vp)
2139 		vn_lock(vp, ltype | LK_RETRY);
2140 	if (vp->v_iflag & VI_DOOMED) {
2141 		if (error == 0) {
2142 			if (*rvp == vp)
2143 				vunref(vp);
2144 			else
2145 				vput(*rvp);
2146 		}
2147 		error = ENOENT;
2148 	}
2149 	return (error);
2150 }
2151 
2152 int
2153 vn_rlimit_fsize(const struct vnode *vp, const struct uio *uio,
2154     struct thread *td)
2155 {
2156 
2157 	if (vp->v_type != VREG || td == NULL)
2158 		return (0);
2159 	if ((uoff_t)uio->uio_offset + uio->uio_resid >
2160 	    lim_cur(td, RLIMIT_FSIZE)) {
2161 		PROC_LOCK(td->td_proc);
2162 		kern_psignal(td->td_proc, SIGXFSZ);
2163 		PROC_UNLOCK(td->td_proc);
2164 		return (EFBIG);
2165 	}
2166 	return (0);
2167 }
2168 
2169 int
2170 vn_chmod(struct file *fp, mode_t mode, struct ucred *active_cred,
2171     struct thread *td)
2172 {
2173 	struct vnode *vp;
2174 
2175 	vp = fp->f_vnode;
2176 #ifdef AUDIT
2177 	vn_lock(vp, LK_SHARED | LK_RETRY);
2178 	AUDIT_ARG_VNODE1(vp);
2179 	VOP_UNLOCK(vp, 0);
2180 #endif
2181 	return (setfmode(td, active_cred, vp, mode));
2182 }
2183 
2184 int
2185 vn_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
2186     struct thread *td)
2187 {
2188 	struct vnode *vp;
2189 
2190 	vp = fp->f_vnode;
2191 #ifdef AUDIT
2192 	vn_lock(vp, LK_SHARED | LK_RETRY);
2193 	AUDIT_ARG_VNODE1(vp);
2194 	VOP_UNLOCK(vp, 0);
2195 #endif
2196 	return (setfown(td, active_cred, vp, uid, gid));
2197 }
2198 
2199 void
2200 vn_pages_remove(struct vnode *vp, vm_pindex_t start, vm_pindex_t end)
2201 {
2202 	vm_object_t object;
2203 
2204 	if ((object = vp->v_object) == NULL)
2205 		return;
2206 	VM_OBJECT_WLOCK(object);
2207 	vm_object_page_remove(object, start, end, 0);
2208 	VM_OBJECT_WUNLOCK(object);
2209 }
2210 
2211 int
2212 vn_bmap_seekhole(struct vnode *vp, u_long cmd, off_t *off, struct ucred *cred)
2213 {
2214 	struct vattr va;
2215 	daddr_t bn, bnp;
2216 	uint64_t bsize;
2217 	off_t noff;
2218 	int error;
2219 
2220 	KASSERT(cmd == FIOSEEKHOLE || cmd == FIOSEEKDATA,
2221 	    ("Wrong command %lu", cmd));
2222 
2223 	if (vn_lock(vp, LK_SHARED) != 0)
2224 		return (EBADF);
2225 	if (vp->v_type != VREG) {
2226 		error = ENOTTY;
2227 		goto unlock;
2228 	}
2229 	error = VOP_GETATTR(vp, &va, cred);
2230 	if (error != 0)
2231 		goto unlock;
2232 	noff = *off;
2233 	if (noff >= va.va_size) {
2234 		error = ENXIO;
2235 		goto unlock;
2236 	}
2237 	bsize = vp->v_mount->mnt_stat.f_iosize;
2238 	for (bn = noff / bsize; noff < va.va_size; bn++, noff += bsize -
2239 	    noff % bsize) {
2240 		error = VOP_BMAP(vp, bn, NULL, &bnp, NULL, NULL);
2241 		if (error == EOPNOTSUPP) {
2242 			error = ENOTTY;
2243 			goto unlock;
2244 		}
2245 		if ((bnp == -1 && cmd == FIOSEEKHOLE) ||
2246 		    (bnp != -1 && cmd == FIOSEEKDATA)) {
2247 			noff = bn * bsize;
2248 			if (noff < *off)
2249 				noff = *off;
2250 			goto unlock;
2251 		}
2252 	}
2253 	if (noff > va.va_size)
2254 		noff = va.va_size;
2255 	/* noff == va.va_size. There is an implicit hole at the end of file. */
2256 	if (cmd == FIOSEEKDATA)
2257 		error = ENXIO;
2258 unlock:
2259 	VOP_UNLOCK(vp, 0);
2260 	if (error == 0)
2261 		*off = noff;
2262 	return (error);
2263 }
2264 
2265 int
2266 vn_seek(struct file *fp, off_t offset, int whence, struct thread *td)
2267 {
2268 	struct ucred *cred;
2269 	struct vnode *vp;
2270 	struct vattr vattr;
2271 	off_t foffset, size;
2272 	int error, noneg;
2273 
2274 	cred = td->td_ucred;
2275 	vp = fp->f_vnode;
2276 	foffset = foffset_lock(fp, 0);
2277 	noneg = (vp->v_type != VCHR);
2278 	error = 0;
2279 	switch (whence) {
2280 	case L_INCR:
2281 		if (noneg &&
2282 		    (foffset < 0 ||
2283 		    (offset > 0 && foffset > OFF_MAX - offset))) {
2284 			error = EOVERFLOW;
2285 			break;
2286 		}
2287 		offset += foffset;
2288 		break;
2289 	case L_XTND:
2290 		vn_lock(vp, LK_SHARED | LK_RETRY);
2291 		error = VOP_GETATTR(vp, &vattr, cred);
2292 		VOP_UNLOCK(vp, 0);
2293 		if (error)
2294 			break;
2295 
2296 		/*
2297 		 * If the file references a disk device, then fetch
2298 		 * the media size and use that to determine the ending
2299 		 * offset.
2300 		 */
2301 		if (vattr.va_size == 0 && vp->v_type == VCHR &&
2302 		    fo_ioctl(fp, DIOCGMEDIASIZE, &size, cred, td) == 0)
2303 			vattr.va_size = size;
2304 		if (noneg &&
2305 		    (vattr.va_size > OFF_MAX ||
2306 		    (offset > 0 && vattr.va_size > OFF_MAX - offset))) {
2307 			error = EOVERFLOW;
2308 			break;
2309 		}
2310 		offset += vattr.va_size;
2311 		break;
2312 	case L_SET:
2313 		break;
2314 	case SEEK_DATA:
2315 		error = fo_ioctl(fp, FIOSEEKDATA, &offset, cred, td);
2316 		if (error == ENOTTY)
2317 			error = EINVAL;
2318 		break;
2319 	case SEEK_HOLE:
2320 		error = fo_ioctl(fp, FIOSEEKHOLE, &offset, cred, td);
2321 		if (error == ENOTTY)
2322 			error = EINVAL;
2323 		break;
2324 	default:
2325 		error = EINVAL;
2326 	}
2327 	if (error == 0 && noneg && offset < 0)
2328 		error = EINVAL;
2329 	if (error != 0)
2330 		goto drop;
2331 	VFS_KNOTE_UNLOCKED(vp, 0);
2332 	td->td_uretoff.tdu_off = offset;
2333 drop:
2334 	foffset_unlock(fp, offset, error != 0 ? FOF_NOUPDATE : 0);
2335 	return (error);
2336 }
2337 
2338 int
2339 vn_utimes_perm(struct vnode *vp, struct vattr *vap, struct ucred *cred,
2340     struct thread *td)
2341 {
2342 	int error;
2343 
2344 	/*
2345 	 * Grant permission if the caller is the owner of the file, or
2346 	 * the super-user, or has ACL_WRITE_ATTRIBUTES permission on
2347 	 * on the file.  If the time pointer is null, then write
2348 	 * permission on the file is also sufficient.
2349 	 *
2350 	 * From NFSv4.1, draft 21, 6.2.1.3.1, Discussion of Mask Attributes:
2351 	 * A user having ACL_WRITE_DATA or ACL_WRITE_ATTRIBUTES
2352 	 * will be allowed to set the times [..] to the current
2353 	 * server time.
2354 	 */
2355 	error = VOP_ACCESSX(vp, VWRITE_ATTRIBUTES, cred, td);
2356 	if (error != 0 && (vap->va_vaflags & VA_UTIMES_NULL) != 0)
2357 		error = VOP_ACCESS(vp, VWRITE, cred, td);
2358 	return (error);
2359 }
2360 
2361 int
2362 vn_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
2363 {
2364 	struct vnode *vp;
2365 	int error;
2366 
2367 	if (fp->f_type == DTYPE_FIFO)
2368 		kif->kf_type = KF_TYPE_FIFO;
2369 	else
2370 		kif->kf_type = KF_TYPE_VNODE;
2371 	vp = fp->f_vnode;
2372 	vref(vp);
2373 	FILEDESC_SUNLOCK(fdp);
2374 	error = vn_fill_kinfo_vnode(vp, kif);
2375 	vrele(vp);
2376 	FILEDESC_SLOCK(fdp);
2377 	return (error);
2378 }
2379 
2380 static inline void
2381 vn_fill_junk(struct kinfo_file *kif)
2382 {
2383 	size_t len, olen;
2384 
2385 	/*
2386 	 * Simulate vn_fullpath returning changing values for a given
2387 	 * vp during e.g. coredump.
2388 	 */
2389 	len = (arc4random() % (sizeof(kif->kf_path) - 2)) + 1;
2390 	olen = strlen(kif->kf_path);
2391 	if (len < olen)
2392 		strcpy(&kif->kf_path[len - 1], "$");
2393 	else
2394 		for (; olen < len; olen++)
2395 			strcpy(&kif->kf_path[olen], "A");
2396 }
2397 
2398 int
2399 vn_fill_kinfo_vnode(struct vnode *vp, struct kinfo_file *kif)
2400 {
2401 	struct vattr va;
2402 	char *fullpath, *freepath;
2403 	int error;
2404 
2405 	kif->kf_un.kf_file.kf_file_type = vntype_to_kinfo(vp->v_type);
2406 	freepath = NULL;
2407 	fullpath = "-";
2408 	error = vn_fullpath(curthread, vp, &fullpath, &freepath);
2409 	if (error == 0) {
2410 		strlcpy(kif->kf_path, fullpath, sizeof(kif->kf_path));
2411 	}
2412 	if (freepath != NULL)
2413 		free(freepath, M_TEMP);
2414 
2415 	KFAIL_POINT_CODE(DEBUG_FP, fill_kinfo_vnode__random_path,
2416 		vn_fill_junk(kif);
2417 	);
2418 
2419 	/*
2420 	 * Retrieve vnode attributes.
2421 	 */
2422 	va.va_fsid = VNOVAL;
2423 	va.va_rdev = NODEV;
2424 	vn_lock(vp, LK_SHARED | LK_RETRY);
2425 	error = VOP_GETATTR(vp, &va, curthread->td_ucred);
2426 	VOP_UNLOCK(vp, 0);
2427 	if (error != 0)
2428 		return (error);
2429 	if (va.va_fsid != VNOVAL)
2430 		kif->kf_un.kf_file.kf_file_fsid = va.va_fsid;
2431 	else
2432 		kif->kf_un.kf_file.kf_file_fsid =
2433 		    vp->v_mount->mnt_stat.f_fsid.val[0];
2434 	kif->kf_un.kf_file.kf_file_fsid_freebsd11 =
2435 	    kif->kf_un.kf_file.kf_file_fsid; /* truncate */
2436 	kif->kf_un.kf_file.kf_file_fileid = va.va_fileid;
2437 	kif->kf_un.kf_file.kf_file_mode = MAKEIMODE(va.va_type, va.va_mode);
2438 	kif->kf_un.kf_file.kf_file_size = va.va_size;
2439 	kif->kf_un.kf_file.kf_file_rdev = va.va_rdev;
2440 	kif->kf_un.kf_file.kf_file_rdev_freebsd11 =
2441 	    kif->kf_un.kf_file.kf_file_rdev; /* truncate */
2442 	return (0);
2443 }
2444 
2445 int
2446 vn_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size,
2447     vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff,
2448     struct thread *td)
2449 {
2450 #ifdef HWPMC_HOOKS
2451 	struct pmckern_map_in pkm;
2452 #endif
2453 	struct mount *mp;
2454 	struct vnode *vp;
2455 	vm_object_t object;
2456 	vm_prot_t maxprot;
2457 	boolean_t writecounted;
2458 	int error;
2459 
2460 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \
2461     defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4)
2462 	/*
2463 	 * POSIX shared-memory objects are defined to have
2464 	 * kernel persistence, and are not defined to support
2465 	 * read(2)/write(2) -- or even open(2).  Thus, we can
2466 	 * use MAP_ASYNC to trade on-disk coherence for speed.
2467 	 * The shm_open(3) library routine turns on the FPOSIXSHM
2468 	 * flag to request this behavior.
2469 	 */
2470 	if ((fp->f_flag & FPOSIXSHM) != 0)
2471 		flags |= MAP_NOSYNC;
2472 #endif
2473 	vp = fp->f_vnode;
2474 
2475 	/*
2476 	 * Ensure that file and memory protections are
2477 	 * compatible.  Note that we only worry about
2478 	 * writability if mapping is shared; in this case,
2479 	 * current and max prot are dictated by the open file.
2480 	 * XXX use the vnode instead?  Problem is: what
2481 	 * credentials do we use for determination? What if
2482 	 * proc does a setuid?
2483 	 */
2484 	mp = vp->v_mount;
2485 	if (mp != NULL && (mp->mnt_flag & MNT_NOEXEC) != 0) {
2486 		maxprot = VM_PROT_NONE;
2487 		if ((prot & VM_PROT_EXECUTE) != 0)
2488 			return (EACCES);
2489 	} else
2490 		maxprot = VM_PROT_EXECUTE;
2491 	if ((fp->f_flag & FREAD) != 0)
2492 		maxprot |= VM_PROT_READ;
2493 	else if ((prot & VM_PROT_READ) != 0)
2494 		return (EACCES);
2495 
2496 	/*
2497 	 * If we are sharing potential changes via MAP_SHARED and we
2498 	 * are trying to get write permission although we opened it
2499 	 * without asking for it, bail out.
2500 	 */
2501 	if ((flags & MAP_SHARED) != 0) {
2502 		if ((fp->f_flag & FWRITE) != 0)
2503 			maxprot |= VM_PROT_WRITE;
2504 		else if ((prot & VM_PROT_WRITE) != 0)
2505 			return (EACCES);
2506 	} else {
2507 		maxprot |= VM_PROT_WRITE;
2508 		cap_maxprot |= VM_PROT_WRITE;
2509 	}
2510 	maxprot &= cap_maxprot;
2511 
2512 	/*
2513 	 * For regular files and shared memory, POSIX requires that
2514 	 * the value of foff be a legitimate offset within the data
2515 	 * object.  In particular, negative offsets are invalid.
2516 	 * Blocking negative offsets and overflows here avoids
2517 	 * possible wraparound or user-level access into reserved
2518 	 * ranges of the data object later.  In contrast, POSIX does
2519 	 * not dictate how offsets are used by device drivers, so in
2520 	 * the case of a device mapping a negative offset is passed
2521 	 * on.
2522 	 */
2523 	if (
2524 #ifdef _LP64
2525 	    size > OFF_MAX ||
2526 #endif
2527 	    foff < 0 || foff > OFF_MAX - size)
2528 		return (EINVAL);
2529 
2530 	writecounted = FALSE;
2531 	error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, vp,
2532 	    &foff, &object, &writecounted);
2533 	if (error != 0)
2534 		return (error);
2535 	error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object,
2536 	    foff, writecounted, td);
2537 	if (error != 0) {
2538 		/*
2539 		 * If this mapping was accounted for in the vnode's
2540 		 * writecount, then undo that now.
2541 		 */
2542 		if (writecounted)
2543 			vm_pager_release_writecount(object, 0, size);
2544 		vm_object_deallocate(object);
2545 	}
2546 #ifdef HWPMC_HOOKS
2547 	/* Inform hwpmc(4) if an executable is being mapped. */
2548 	if (PMC_HOOK_INSTALLED(PMC_FN_MMAP)) {
2549 		if ((prot & VM_PROT_EXECUTE) != 0 && error == 0) {
2550 			pkm.pm_file = vp;
2551 			pkm.pm_address = (uintptr_t) *addr;
2552 			PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_MMAP, (void *) &pkm);
2553 		}
2554 	}
2555 #endif
2556 	return (error);
2557 }
2558 
2559 void
2560 vn_fsid(struct vnode *vp, struct vattr *va)
2561 {
2562 	fsid_t *f;
2563 
2564 	f = &vp->v_mount->mnt_stat.f_fsid;
2565 	va->va_fsid = (uint32_t)f->val[1];
2566 	va->va_fsid <<= sizeof(f->val[1]) * NBBY;
2567 	va->va_fsid += (uint32_t)f->val[0];
2568 }
2569 
2570 int
2571 vn_fsync_buf(struct vnode *vp, int waitfor)
2572 {
2573 	struct buf *bp, *nbp;
2574 	struct bufobj *bo;
2575 	struct mount *mp;
2576 	int error, maxretry;
2577 
2578 	error = 0;
2579 	maxretry = 10000;     /* large, arbitrarily chosen */
2580 	mp = NULL;
2581 	if (vp->v_type == VCHR) {
2582 		VI_LOCK(vp);
2583 		mp = vp->v_rdev->si_mountpt;
2584 		VI_UNLOCK(vp);
2585 	}
2586 	bo = &vp->v_bufobj;
2587 	BO_LOCK(bo);
2588 loop1:
2589 	/*
2590 	 * MARK/SCAN initialization to avoid infinite loops.
2591 	 */
2592         TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) {
2593 		bp->b_vflags &= ~BV_SCANNED;
2594 		bp->b_error = 0;
2595 	}
2596 
2597 	/*
2598 	 * Flush all dirty buffers associated with a vnode.
2599 	 */
2600 loop2:
2601 	TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
2602 		if ((bp->b_vflags & BV_SCANNED) != 0)
2603 			continue;
2604 		bp->b_vflags |= BV_SCANNED;
2605 		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) {
2606 			if (waitfor != MNT_WAIT)
2607 				continue;
2608 			if (BUF_LOCK(bp,
2609 			    LK_EXCLUSIVE | LK_INTERLOCK | LK_SLEEPFAIL,
2610 			    BO_LOCKPTR(bo)) != 0) {
2611 				BO_LOCK(bo);
2612 				goto loop1;
2613 			}
2614 			BO_LOCK(bo);
2615 		}
2616 		BO_UNLOCK(bo);
2617 		KASSERT(bp->b_bufobj == bo,
2618 		    ("bp %p wrong b_bufobj %p should be %p",
2619 		    bp, bp->b_bufobj, bo));
2620 		if ((bp->b_flags & B_DELWRI) == 0)
2621 			panic("fsync: not dirty");
2622 		if ((vp->v_object != NULL) && (bp->b_flags & B_CLUSTEROK)) {
2623 			vfs_bio_awrite(bp);
2624 		} else {
2625 			bremfree(bp);
2626 			bawrite(bp);
2627 		}
2628 		if (maxretry < 1000)
2629 			pause("dirty", hz < 1000 ? 1 : hz / 1000);
2630 		BO_LOCK(bo);
2631 		goto loop2;
2632 	}
2633 
2634 	/*
2635 	 * If synchronous the caller expects us to completely resolve all
2636 	 * dirty buffers in the system.  Wait for in-progress I/O to
2637 	 * complete (which could include background bitmap writes), then
2638 	 * retry if dirty blocks still exist.
2639 	 */
2640 	if (waitfor == MNT_WAIT) {
2641 		bufobj_wwait(bo, 0, 0);
2642 		if (bo->bo_dirty.bv_cnt > 0) {
2643 			/*
2644 			 * If we are unable to write any of these buffers
2645 			 * then we fail now rather than trying endlessly
2646 			 * to write them out.
2647 			 */
2648 			TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs)
2649 				if ((error = bp->b_error) != 0)
2650 					break;
2651 			if ((mp != NULL && mp->mnt_secondary_writes > 0) ||
2652 			    (error == 0 && --maxretry >= 0))
2653 				goto loop1;
2654 			if (error == 0)
2655 				error = EAGAIN;
2656 		}
2657 	}
2658 	BO_UNLOCK(bo);
2659 	if (error != 0)
2660 		vn_printf(vp, "fsync: giving up on dirty (error = %d) ", error);
2661 
2662 	return (error);
2663 }
2664 
2665 /*
2666  * Copies a byte range from invp to outvp.  Calls VOP_COPY_FILE_RANGE()
2667  * or vn_generic_copy_file_range() after rangelocking the byte ranges,
2668  * to do the actual copy.
2669  * vn_generic_copy_file_range() is factored out, so it can be called
2670  * from a VOP_COPY_FILE_RANGE() call as well, but handles vnodes from
2671  * different file systems.
2672  */
2673 int
2674 vn_copy_file_range(struct vnode *invp, off_t *inoffp, struct vnode *outvp,
2675     off_t *outoffp, size_t *lenp, unsigned int flags, struct ucred *incred,
2676     struct ucred *outcred, struct thread *fsize_td)
2677 {
2678 	struct vattr va;
2679 	int error;
2680 	size_t len;
2681 	uint64_t uvalin, uvalout;
2682 
2683 	len = *lenp;
2684 	*lenp = 0;		/* For error returns. */
2685 	error = 0;
2686 
2687 	/* Do some sanity checks on the arguments. */
2688 	uvalin = *inoffp;
2689 	uvalin += len;
2690 	uvalout = *outoffp;
2691 	uvalout += len;
2692 	if (invp->v_type == VDIR || outvp->v_type == VDIR)
2693 		error = EISDIR;
2694 	else if (*inoffp < 0 || uvalin > INT64_MAX || uvalin <
2695 	    (uint64_t)*inoffp || *outoffp < 0 || uvalout > INT64_MAX ||
2696 	    uvalout < (uint64_t)*outoffp || invp->v_type != VREG ||
2697 	    outvp->v_type != VREG)
2698 		error = EINVAL;
2699 	else if (invp == outvp)
2700 		error = EBADF;
2701 	if (error != 0)
2702 		goto out;
2703 
2704 	error = vn_lock(invp, LK_SHARED);
2705 	if (error != 0)
2706 		goto out;
2707 	/* Check that the offset + len does not go past EOF of invp. */
2708 	error = VOP_GETATTR(invp, &va, incred);
2709 	if (error == 0 && va.va_size < *inoffp + len)
2710 		error = EINVAL;
2711 	VOP_UNLOCK(invp, 0);
2712 	if (error != 0)
2713 		goto out;
2714 
2715 	/*
2716 	 * If the two vnode are for the same file system, call
2717 	 * VOP_COPY_FILE_RANGE(), otherwise call vn_generic_copy_file_range()
2718 	 * which can handle copies across multiple file systems.
2719 	 */
2720 	*lenp = len;
2721 	if (invp->v_mount == outvp->v_mount)
2722 		error = VOP_COPY_FILE_RANGE(invp, inoffp, outvp, outoffp,
2723 		    lenp, flags, incred, outcred, fsize_td);
2724 	else
2725 		error = vn_generic_copy_file_range(invp, inoffp, outvp,
2726 		    outoffp, lenp, flags, incred, outcred, fsize_td);
2727 out:
2728 	return (error);
2729 }
2730 
2731 /*
2732  * Test len bytes of data starting at dat for all bytes == 0.
2733  * Return true if all bytes are zero, false otherwise.
2734  * Expects dat to be well aligned.
2735  */
2736 static bool
2737 mem_iszero(void *dat, int len)
2738 {
2739 	int i;
2740 	const u_int *p;
2741 	const char *cp;
2742 
2743 	for (p = dat; len > 0; len -= sizeof(*p), p++) {
2744 		if (len >= sizeof(*p)) {
2745 			if (*p != 0)
2746 				return (false);
2747 		} else {
2748 			cp = (const char *)p;
2749 			for (i = 0; i < len; i++, cp++)
2750 				if (*cp != '\0')
2751 					return (false);
2752 		}
2753 	}
2754 	return (true);
2755 }
2756 
2757 /*
2758  * Look for a hole in the output file and, if found, adjust *outoffp
2759  * and *xferp to skip past the hole.
2760  * *xferp is the entire hole length to be written and xfer2 is how many bytes
2761  * to be written as 0's upon return.
2762  */
2763 static off_t
2764 vn_skip_hole(struct vnode *outvp, off_t xfer2, off_t *outoffp, off_t *xferp,
2765     off_t *dataoffp, off_t *holeoffp, struct ucred *cred)
2766 {
2767 	int error;
2768 	off_t delta;
2769 
2770 	if (*holeoffp == 0 || *holeoffp <= *outoffp) {
2771 		*dataoffp = *outoffp;
2772 		error = VOP_IOCTL(outvp, FIOSEEKDATA, dataoffp, 0, cred,
2773 		    curthread);
2774 		if (error == 0) {
2775 			*holeoffp = *dataoffp;
2776 			error = VOP_IOCTL(outvp, FIOSEEKHOLE, holeoffp, 0, cred,
2777 			    curthread);
2778 		}
2779 		if (error != 0 || *holeoffp == *dataoffp) {
2780 			/*
2781 			 * Since outvp is unlocked, it may be possible for
2782 			 * another thread to do a truncate(), lseek(), write()
2783 			 * creating a hole at startoff between the above
2784 			 * VOP_IOCTL() calls, if the other thread does not do
2785 			 * rangelocking.
2786 			 * If that happens, *holeoffp == *dataoffp and finding
2787 			 * the hole has failed, so disable vn_skip_hole().
2788 			 */
2789 			*holeoffp = -1;	/* Disable use of vn_skip_hole(). */
2790 			return (xfer2);
2791 		}
2792 		KASSERT(*dataoffp >= *outoffp,
2793 		    ("vn_skip_hole: dataoff=%jd < outoff=%jd",
2794 		    (intmax_t)*dataoffp, (intmax_t)*outoffp));
2795 		KASSERT(*holeoffp > *dataoffp,
2796 		    ("vn_skip_hole: holeoff=%jd <= dataoff=%jd",
2797 		    (intmax_t)*holeoffp, (intmax_t)*dataoffp));
2798 	}
2799 
2800 	/*
2801 	 * If there is a hole before the data starts, advance *outoffp and
2802 	 * *xferp past the hole.
2803 	 */
2804 	if (*dataoffp > *outoffp) {
2805 		delta = *dataoffp - *outoffp;
2806 		if (delta >= *xferp) {
2807 			/* Entire *xferp is a hole. */
2808 			*outoffp += *xferp;
2809 			*xferp = 0;
2810 			return (0);
2811 		}
2812 		*xferp -= delta;
2813 		*outoffp += delta;
2814 		xfer2 = MIN(xfer2, *xferp);
2815 	}
2816 
2817 	/*
2818 	 * If a hole starts before the end of this xfer2, reduce this xfer2 so
2819 	 * that the write ends at the start of the hole.
2820 	 * *holeoffp should always be greater than *outoffp, but for the
2821 	 * non-INVARIANTS case, check this to make sure xfer2 remains a sane
2822 	 * value.
2823 	 */
2824 	if (*holeoffp > *outoffp && *holeoffp < *outoffp + xfer2)
2825 		xfer2 = *holeoffp - *outoffp;
2826 	return (xfer2);
2827 }
2828 
2829 /*
2830  * Write an xfer sized chunk to outvp in blksize blocks from dat.
2831  * dat is a maximum of blksize in length and can be written repeatedly in
2832  * the chunk.
2833  * If growfile == true, just grow the file via vn_truncate_locked() instead
2834  * of doing actual writes.
2835  * If checkhole == true, a hole is being punched, so skip over any hole
2836  * already in the output file.
2837  */
2838 static int
2839 vn_write_outvp(struct vnode *outvp, char *dat, off_t outoff, off_t xfer,
2840     u_long blksize, bool growfile, bool checkhole, struct ucred *cred)
2841 {
2842 	struct mount *mp;
2843 	off_t dataoff, holeoff, xfer2;
2844 	int error, lckf;
2845 
2846 	/*
2847 	 * Loop around doing writes of blksize until write has been completed.
2848 	 * Lock/unlock on each loop iteration so that a bwillwrite() can be
2849 	 * done for each iteration, since the xfer argument can be very
2850 	 * large if there is a large hole to punch in the output file.
2851 	 */
2852 	error = 0;
2853 	holeoff = 0;
2854 	do {
2855 		xfer2 = MIN(xfer, blksize);
2856 		if (checkhole) {
2857 			/*
2858 			 * Punching a hole.  Skip writing if there is
2859 			 * already a hole in the output file.
2860 			 */
2861 			xfer2 = vn_skip_hole(outvp, xfer2, &outoff, &xfer,
2862 			    &dataoff, &holeoff, cred);
2863 			if (xfer == 0)
2864 				break;
2865 			if (holeoff < 0)
2866 				checkhole = false;
2867 			KASSERT(xfer2 > 0, ("vn_write_outvp: xfer2=%jd",
2868 			    (intmax_t)xfer2));
2869 		}
2870 		bwillwrite();
2871 		mp = NULL;
2872 		error = vn_start_write(outvp, &mp, V_WAIT);
2873 		if (error == 0) {
2874 			if (MNT_SHARED_WRITES(mp))
2875 				lckf = LK_SHARED;
2876 			else
2877 				lckf = LK_EXCLUSIVE;
2878 			error = vn_lock(outvp, lckf);
2879 		}
2880 		if (error == 0) {
2881 			if (growfile)
2882 				error = vn_truncate_locked(outvp, outoff + xfer,
2883 				    false, cred);
2884 			else {
2885 				error = vn_rdwr(UIO_WRITE, outvp, dat, xfer2,
2886 				    outoff, UIO_SYSSPACE, IO_NODELOCKED,
2887 				    curthread->td_ucred, cred, NULL, curthread);
2888 				outoff += xfer2;
2889 				xfer -= xfer2;
2890 			}
2891 			VOP_UNLOCK(outvp, 0);
2892 		}
2893 		if (mp != NULL)
2894 			vn_finished_write(mp);
2895 	} while (!growfile && xfer > 0 && error == 0);
2896 	return (error);
2897 }
2898 
2899 /*
2900  * Copy a byte range of one file to another.  This function can handle the
2901  * case where invp and outvp are on different file systems.
2902  * It can also be called by a VOP_COPY_FILE_RANGE() to do the work, if there
2903  * is no better file system specific way to do it.
2904  */
2905 int
2906 vn_generic_copy_file_range(struct vnode *invp, off_t *inoffp,
2907     struct vnode *outvp, off_t *outoffp, size_t *lenp, unsigned int flags,
2908     struct ucred *incred, struct ucred *outcred, struct thread *fsize_td)
2909 {
2910 	struct vattr va;
2911 	struct mount *mp;
2912 	struct uio io;
2913 	off_t startoff, endoff, xfer, xfer2;
2914 	u_long blksize;
2915 	int error;
2916 	bool cantseek, readzeros;
2917 	ssize_t aresid;
2918 	size_t copylen, len, savlen;
2919 	char *dat;
2920 	long holein, holeout;
2921 
2922 	holein = holeout = 0;
2923 	savlen = len = *lenp;
2924 	error = 0;
2925 	dat = NULL;
2926 
2927 	error = vn_lock(invp, LK_SHARED);
2928 	if (error != 0)
2929 		goto out;
2930 	if (VOP_PATHCONF(invp, _PC_MIN_HOLE_SIZE, &holein) != 0)
2931 		holein = 0;
2932 	VOP_UNLOCK(invp, 0);
2933 
2934 	mp = NULL;
2935 	error = vn_start_write(outvp, &mp, V_WAIT);
2936 	if (error == 0)
2937 		error = vn_lock(outvp, LK_EXCLUSIVE);
2938 	if (error == 0) {
2939 		/*
2940 		 * If fsize_td != NULL, do a vn_rlimit_fsize() call,
2941 		 * now that outvp is locked.
2942 		 */
2943 		if (fsize_td != NULL) {
2944 			io.uio_offset = *outoffp;
2945 			io.uio_resid = len;
2946 			error = vn_rlimit_fsize(outvp, &io, fsize_td);
2947 			if (error != 0)
2948 				error = EFBIG;
2949 		}
2950 		if (VOP_PATHCONF(outvp, _PC_MIN_HOLE_SIZE, &holeout) != 0)
2951 			holeout = 0;
2952 		/*
2953 		 * Holes that are past EOF do not need to be written as a block
2954 		 * of zero bytes.  So, truncate the output file as far as
2955 		 * possible and then use va.va_size to decide if writing 0
2956 		 * bytes is necessary in the loop below.
2957 		 */
2958 		if (error == 0)
2959 			error = VOP_GETATTR(outvp, &va, outcred);
2960 		if (error == 0 && va.va_size > *outoffp && va.va_size <=
2961 		    *outoffp + len) {
2962 #ifdef MAC
2963 			error = mac_vnode_check_write(curthread->td_ucred,
2964 			    outcred, outvp);
2965 			if (error == 0)
2966 #endif
2967 				error = vn_truncate_locked(outvp, *outoffp,
2968 				    false, outcred);
2969 			if (error == 0)
2970 				va.va_size = *outoffp;
2971 		}
2972 		VOP_UNLOCK(outvp, 0);
2973 	}
2974 	if (mp != NULL)
2975 		vn_finished_write(mp);
2976 	if (error != 0)
2977 		goto out;
2978 
2979 	/*
2980 	 * Set the blksize to the larger of the hole sizes for invp and outvp.
2981 	 * If hole sizes aren't available, set the blksize to the larger
2982 	 * f_iosize of invp and outvp.
2983 	 * This code expects the hole sizes and f_iosizes to be powers of 2.
2984 	 * This value is clipped at 4Kbytes and 1Mbyte.
2985 	 */
2986 	blksize = MAX(holein, holeout);
2987 	if (blksize == 0)
2988 		blksize = MAX(invp->v_mount->mnt_stat.f_iosize,
2989 		    outvp->v_mount->mnt_stat.f_iosize);
2990 	if (blksize < 4096)
2991 		blksize = 4096;
2992 	else if (blksize > 1024 * 1024)
2993 		blksize = 1024 * 1024;
2994 	dat = malloc(blksize, M_TEMP, M_WAITOK);
2995 
2996 	/*
2997 	 * If VOP_IOCTL(FIOSEEKHOLE) works for invp, use it and FIOSEEKDATA
2998 	 * to find holes.  Otherwise, just scan the read block for all 0s
2999 	 * in the inner loop where the data copying is done.
3000 	 * Note that some file systems such as NFSv3, NFSv4.0 and NFSv4.1 may
3001 	 * support holes on the server, but do not support FIOSEEKHOLE.
3002 	 */
3003 	while (len > 0 && error == 0) {
3004 		endoff = 0;			/* To shut up compilers. */
3005 		cantseek = true;
3006 		startoff = *inoffp;
3007 		copylen = len;
3008 
3009 		/*
3010 		 * Find the next data area.  If there is just a hole to EOF,
3011 		 * FIOSEEKDATA should fail and then we drop down into the
3012 		 * inner loop and create the hole on the outvp file.
3013 		 * (I do not know if any file system will report a hole to
3014 		 *  EOF via FIOSEEKHOLE, but I am pretty sure FIOSEEKDATA
3015 		 *  will fail for those file systems.)
3016 		 *
3017 		 * For input files that don't support FIOSEEKDATA/FIOSEEKHOLE,
3018 		 * the code just falls through to the inner copy loop.
3019 		 */
3020 		error = EINVAL;
3021 		if (holein > 0)
3022 			error = VOP_IOCTL(invp, FIOSEEKDATA, &startoff, 0,
3023 			    incred, curthread);
3024 		if (error == 0) {
3025 			endoff = startoff;
3026 			error = VOP_IOCTL(invp, FIOSEEKHOLE, &endoff, 0,
3027 			    incred, curthread);
3028 			/*
3029 			 * Since invp is unlocked, it may be possible for
3030 			 * another thread to do a truncate(), lseek(), write()
3031 			 * creating a hole at startoff between the above
3032 			 * VOP_IOCTL() calls, if the other thread does not do
3033 			 * rangelocking.
3034 			 * If that happens, startoff == endoff and finding
3035 			 * the hole has failed, so set an error.
3036 			 */
3037 			if (error == 0 && startoff == endoff)
3038 				error = EINVAL; /* Any error. Reset to 0. */
3039 		}
3040 		if (error == 0) {
3041 			if (startoff > *inoffp) {
3042 				/* Found hole before data block. */
3043 				xfer = MIN(startoff - *inoffp, len);
3044 				if (*outoffp < va.va_size) {
3045 					/* Must write 0s to punch hole. */
3046 					xfer2 = MIN(va.va_size - *outoffp,
3047 					    xfer);
3048 					memset(dat, 0, MIN(xfer2, blksize));
3049 					error = vn_write_outvp(outvp, dat,
3050 					    *outoffp, xfer2, blksize, false,
3051 					    holeout > 0, outcred);
3052 				}
3053 
3054 				if (error == 0 && *outoffp + xfer >
3055 				    va.va_size && xfer == len)
3056 					/* Grow last block. */
3057 					error = vn_write_outvp(outvp, dat,
3058 					    *outoffp, xfer, blksize, true,
3059 					    false, outcred);
3060 				if (error == 0) {
3061 					*inoffp += xfer;
3062 					*outoffp += xfer;
3063 					len -= xfer;
3064 				}
3065 			}
3066 			copylen = MIN(len, endoff - startoff);
3067 			cantseek = false;
3068 		} else {
3069 			cantseek = true;
3070 			startoff = *inoffp;
3071 			copylen = len;
3072 			error = 0;
3073 		}
3074 
3075 		xfer = blksize;
3076 		if (cantseek) {
3077 			/*
3078 			 * Set first xfer to end at a block boundary, so that
3079 			 * holes are more likely detected in the loop below via
3080 			 * the for all bytes 0 method.
3081 			 */
3082 			xfer -= (*inoffp % blksize);
3083 		}
3084 		/* Loop copying the data block. */
3085 		while (copylen > 0 && error == 0) {
3086 			if (copylen < xfer)
3087 				xfer = copylen;
3088 			error = vn_lock(invp, LK_SHARED);
3089 			if (error != 0)
3090 				goto out;
3091 			error = vn_rdwr(UIO_READ, invp, dat, xfer,
3092 			    startoff, UIO_SYSSPACE, IO_NODELOCKED,
3093 			    curthread->td_ucred, incred, &aresid,
3094 			    curthread);
3095 			VOP_UNLOCK(invp, 0);
3096 			/*
3097 			 * Linux considers a range that exceeds EOF to
3098 			 * be an error, so we will too.
3099 			 */
3100 			if (error == 0 && aresid > 0)
3101 				error = EINVAL;
3102 			if (error == 0) {
3103 				/*
3104 				 * Skip the write for holes past the initial EOF
3105 				 * of the output file, unless this is the last
3106 				 * write of the output file at EOF.
3107 				 */
3108 				readzeros = cantseek ? mem_iszero(dat, xfer) :
3109 				    false;
3110 				if (!cantseek || *outoffp < va.va_size ||
3111 				    xfer == len || !readzeros)
3112 					error = vn_write_outvp(outvp, dat,
3113 					    *outoffp, xfer, blksize,
3114 					    readzeros && xfer == len &&
3115 					    *outoffp >= va.va_size, false,
3116 					    outcred);
3117 				if (error == 0) {
3118 					*inoffp += xfer;
3119 					startoff += xfer;
3120 					*outoffp += xfer;
3121 					copylen -= xfer;
3122 					len -= xfer;
3123 				}
3124 			}
3125 			xfer = blksize;
3126 		}
3127 	}
3128 out:
3129 	*lenp = savlen - len;
3130 	free(dat, M_TEMP);
3131 	return (error);
3132 }
3133