xref: /freebsd/sys/kern/vfs_subr.c (revision aa77200569e397d6ff1fdb4d255d0fa254d0a128)
1 /*-
2  * Copyright (c) 1989, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * (c) UNIX System Laboratories, Inc.
5  * All or some portions of this file are derived from material licensed
6  * to the University of California by American Telephone and Telegraph
7  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8  * the permission of UNIX System Laboratories, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	@(#)vfs_subr.c	8.31 (Berkeley) 5/26/95
35  */
36 
37 /*
38  * External virtual filesystem routines
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43 
44 #include "opt_compat.h"
45 #include "opt_ddb.h"
46 #include "opt_watchdog.h"
47 
48 #include <sys/param.h>
49 #include <sys/systm.h>
50 #include <sys/bio.h>
51 #include <sys/buf.h>
52 #include <sys/condvar.h>
53 #include <sys/conf.h>
54 #include <sys/dirent.h>
55 #include <sys/event.h>
56 #include <sys/eventhandler.h>
57 #include <sys/extattr.h>
58 #include <sys/file.h>
59 #include <sys/fcntl.h>
60 #include <sys/jail.h>
61 #include <sys/kdb.h>
62 #include <sys/kernel.h>
63 #include <sys/kthread.h>
64 #include <sys/lockf.h>
65 #include <sys/malloc.h>
66 #include <sys/mount.h>
67 #include <sys/namei.h>
68 #include <sys/priv.h>
69 #include <sys/reboot.h>
70 #include <sys/sched.h>
71 #include <sys/sleepqueue.h>
72 #include <sys/stat.h>
73 #include <sys/sysctl.h>
74 #include <sys/syslog.h>
75 #include <sys/vmmeter.h>
76 #include <sys/vnode.h>
77 #include <sys/watchdog.h>
78 
79 #include <machine/stdarg.h>
80 
81 #include <security/mac/mac_framework.h>
82 
83 #include <vm/vm.h>
84 #include <vm/vm_object.h>
85 #include <vm/vm_extern.h>
86 #include <vm/pmap.h>
87 #include <vm/vm_map.h>
88 #include <vm/vm_page.h>
89 #include <vm/vm_kern.h>
90 #include <vm/uma.h>
91 
92 #ifdef DDB
93 #include <ddb/ddb.h>
94 #endif
95 
96 static void	delmntque(struct vnode *vp);
97 static int	flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo,
98 		    int slpflag, int slptimeo);
99 static void	syncer_shutdown(void *arg, int howto);
100 static int	vtryrecycle(struct vnode *vp);
101 static void	v_incr_usecount(struct vnode *);
102 static void	v_decr_usecount(struct vnode *);
103 static void	v_decr_useonly(struct vnode *);
104 static void	v_upgrade_usecount(struct vnode *);
105 static void	vnlru_free(int);
106 static void	vgonel(struct vnode *);
107 static void	vfs_knllock(void *arg);
108 static void	vfs_knlunlock(void *arg);
109 static void	vfs_knl_assert_locked(void *arg);
110 static void	vfs_knl_assert_unlocked(void *arg);
111 static void	destroy_vpollinfo(struct vpollinfo *vi);
112 
113 /*
114  * Number of vnodes in existence.  Increased whenever getnewvnode()
115  * allocates a new vnode, decreased in vdropl() for VI_DOOMED vnode.
116  */
117 static unsigned long	numvnodes;
118 
119 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0,
120     "Number of vnodes in existence");
121 
122 /*
123  * Conversion tables for conversion from vnode types to inode formats
124  * and back.
125  */
126 enum vtype iftovt_tab[16] = {
127 	VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
128 	VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VBAD,
129 };
130 int vttoif_tab[10] = {
131 	0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK,
132 	S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT
133 };
134 
135 /*
136  * List of vnodes that are ready for recycling.
137  */
138 static TAILQ_HEAD(freelst, vnode) vnode_free_list;
139 
140 /*
141  * Free vnode target.  Free vnodes may simply be files which have been stat'd
142  * but not read.  This is somewhat common, and a small cache of such files
143  * should be kept to avoid recreation costs.
144  */
145 static u_long wantfreevnodes;
146 SYSCTL_ULONG(_vfs, OID_AUTO, wantfreevnodes, CTLFLAG_RW, &wantfreevnodes, 0, "");
147 /* Number of vnodes in the free list. */
148 static u_long freevnodes;
149 SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD, &freevnodes, 0,
150     "Number of vnodes in the free list");
151 
152 static int vlru_allow_cache_src;
153 SYSCTL_INT(_vfs, OID_AUTO, vlru_allow_cache_src, CTLFLAG_RW,
154     &vlru_allow_cache_src, 0, "Allow vlru to reclaim source vnode");
155 
156 /*
157  * Various variables used for debugging the new implementation of
158  * reassignbuf().
159  * XXX these are probably of (very) limited utility now.
160  */
161 static int reassignbufcalls;
162 SYSCTL_INT(_vfs, OID_AUTO, reassignbufcalls, CTLFLAG_RW, &reassignbufcalls, 0,
163     "Number of calls to reassignbuf");
164 
165 /*
166  * Cache for the mount type id assigned to NFS.  This is used for
167  * special checks in nfs/nfs_nqlease.c and vm/vnode_pager.c.
168  */
169 int	nfs_mount_type = -1;
170 
171 /* To keep more than one thread at a time from running vfs_getnewfsid */
172 static struct mtx mntid_mtx;
173 
174 /*
175  * Lock for any access to the following:
176  *	vnode_free_list
177  *	numvnodes
178  *	freevnodes
179  */
180 static struct mtx vnode_free_list_mtx;
181 
182 /* Publicly exported FS */
183 struct nfs_public nfs_pub;
184 
185 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */
186 static uma_zone_t vnode_zone;
187 static uma_zone_t vnodepoll_zone;
188 
189 /*
190  * The workitem queue.
191  *
192  * It is useful to delay writes of file data and filesystem metadata
193  * for tens of seconds so that quickly created and deleted files need
194  * not waste disk bandwidth being created and removed. To realize this,
195  * we append vnodes to a "workitem" queue. When running with a soft
196  * updates implementation, most pending metadata dependencies should
197  * not wait for more than a few seconds. Thus, mounted on block devices
198  * are delayed only about a half the time that file data is delayed.
199  * Similarly, directory updates are more critical, so are only delayed
200  * about a third the time that file data is delayed. Thus, there are
201  * SYNCER_MAXDELAY queues that are processed round-robin at a rate of
202  * one each second (driven off the filesystem syncer process). The
203  * syncer_delayno variable indicates the next queue that is to be processed.
204  * Items that need to be processed soon are placed in this queue:
205  *
206  *	syncer_workitem_pending[syncer_delayno]
207  *
208  * A delay of fifteen seconds is done by placing the request fifteen
209  * entries later in the queue:
210  *
211  *	syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask]
212  *
213  */
214 static int syncer_delayno;
215 static long syncer_mask;
216 LIST_HEAD(synclist, bufobj);
217 static struct synclist *syncer_workitem_pending;
218 /*
219  * The sync_mtx protects:
220  *	bo->bo_synclist
221  *	sync_vnode_count
222  *	syncer_delayno
223  *	syncer_state
224  *	syncer_workitem_pending
225  *	syncer_worklist_len
226  *	rushjob
227  */
228 static struct mtx sync_mtx;
229 static struct cv sync_wakeup;
230 
231 #define SYNCER_MAXDELAY		32
232 static int syncer_maxdelay = SYNCER_MAXDELAY;	/* maximum delay time */
233 static int syncdelay = 30;		/* max time to delay syncing data */
234 static int filedelay = 30;		/* time to delay syncing files */
235 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0,
236     "Time to delay syncing files (in seconds)");
237 static int dirdelay = 29;		/* time to delay syncing directories */
238 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0,
239     "Time to delay syncing directories (in seconds)");
240 static int metadelay = 28;		/* time to delay syncing metadata */
241 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0,
242     "Time to delay syncing metadata (in seconds)");
243 static int rushjob;		/* number of slots to run ASAP */
244 static int stat_rush_requests;	/* number of times I/O speeded up */
245 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0,
246     "Number of times I/O speeded up (rush requests)");
247 
248 /*
249  * When shutting down the syncer, run it at four times normal speed.
250  */
251 #define SYNCER_SHUTDOWN_SPEEDUP		4
252 static int sync_vnode_count;
253 static int syncer_worklist_len;
254 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY }
255     syncer_state;
256 
257 /*
258  * Number of vnodes we want to exist at any one time.  This is mostly used
259  * to size hash tables in vnode-related code.  It is normally not used in
260  * getnewvnode(), as wantfreevnodes is normally nonzero.)
261  *
262  * XXX desiredvnodes is historical cruft and should not exist.
263  */
264 int desiredvnodes;
265 SYSCTL_INT(_kern, KERN_MAXVNODES, maxvnodes, CTLFLAG_RW,
266     &desiredvnodes, 0, "Maximum number of vnodes");
267 SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW,
268     &wantfreevnodes, 0, "Minimum number of vnodes (legacy)");
269 static int vnlru_nowhere;
270 SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW,
271     &vnlru_nowhere, 0, "Number of times the vnlru process ran without success");
272 
273 /*
274  * Macros to control when a vnode is freed and recycled.  All require
275  * the vnode interlock.
276  */
277 #define VCANRECYCLE(vp) (((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt)
278 #define VSHOULDFREE(vp) (!((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt)
279 #define VSHOULDBUSY(vp) (((vp)->v_iflag & VI_FREE) && (vp)->v_holdcnt)
280 
281 
282 /*
283  * Initialize the vnode management data structures.
284  *
285  * Reevaluate the following cap on the number of vnodes after the physical
286  * memory size exceeds 512GB.  In the limit, as the physical memory size
287  * grows, the ratio of physical pages to vnodes approaches sixteen to one.
288  */
289 #ifndef	MAXVNODES_MAX
290 #define	MAXVNODES_MAX	(512 * (1024 * 1024 * 1024 / (int)PAGE_SIZE / 16))
291 #endif
292 static void
293 vntblinit(void *dummy __unused)
294 {
295 	int physvnodes, virtvnodes;
296 
297 	/*
298 	 * Desiredvnodes is a function of the physical memory size and the
299 	 * kernel's heap size.  Generally speaking, it scales with the
300 	 * physical memory size.  The ratio of desiredvnodes to physical pages
301 	 * is one to four until desiredvnodes exceeds 98,304.  Thereafter, the
302 	 * marginal ratio of desiredvnodes to physical pages is one to
303 	 * sixteen.  However, desiredvnodes is limited by the kernel's heap
304 	 * size.  The memory required by desiredvnodes vnodes and vm objects
305 	 * may not exceed one seventh of the kernel's heap size.
306 	 */
307 	physvnodes = maxproc + cnt.v_page_count / 16 + 3 * min(98304 * 4,
308 	    cnt.v_page_count) / 16;
309 	virtvnodes = vm_kmem_size / (7 * (sizeof(struct vm_object) +
310 	    sizeof(struct vnode)));
311 	desiredvnodes = min(physvnodes, virtvnodes);
312 	if (desiredvnodes > MAXVNODES_MAX) {
313 		if (bootverbose)
314 			printf("Reducing kern.maxvnodes %d -> %d\n",
315 			    desiredvnodes, MAXVNODES_MAX);
316 		desiredvnodes = MAXVNODES_MAX;
317 	}
318 	wantfreevnodes = desiredvnodes / 4;
319 	mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF);
320 	TAILQ_INIT(&vnode_free_list);
321 	mtx_init(&vnode_free_list_mtx, "vnode_free_list", NULL, MTX_DEF);
322 	vnode_zone = uma_zcreate("VNODE", sizeof (struct vnode), NULL, NULL,
323 	    NULL, NULL, UMA_ALIGN_PTR, 0);
324 	vnodepoll_zone = uma_zcreate("VNODEPOLL", sizeof (struct vpollinfo),
325 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
326 	/*
327 	 * Initialize the filesystem syncer.
328 	 */
329 	syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE,
330 	    &syncer_mask);
331 	syncer_maxdelay = syncer_mask + 1;
332 	mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF);
333 	cv_init(&sync_wakeup, "syncer");
334 }
335 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);
336 
337 
338 /*
339  * Mark a mount point as busy. Used to synchronize access and to delay
340  * unmounting. Eventually, mountlist_mtx is not released on failure.
341  *
342  * vfs_busy() is a custom lock, it can block the caller.
343  * vfs_busy() only sleeps if the unmount is active on the mount point.
344  * For a mountpoint mp, vfs_busy-enforced lock is before lock of any
345  * vnode belonging to mp.
346  *
347  * Lookup uses vfs_busy() to traverse mount points.
348  * root fs			var fs
349  * / vnode lock		A	/ vnode lock (/var)		D
350  * /var vnode lock	B	/log vnode lock(/var/log)	E
351  * vfs_busy lock	C	vfs_busy lock			F
352  *
353  * Within each file system, the lock order is C->A->B and F->D->E.
354  *
355  * When traversing across mounts, the system follows that lock order:
356  *
357  *        C->A->B
358  *              |
359  *              +->F->D->E
360  *
361  * The lookup() process for namei("/var") illustrates the process:
362  *  VOP_LOOKUP() obtains B while A is held
363  *  vfs_busy() obtains a shared lock on F while A and B are held
364  *  vput() releases lock on B
365  *  vput() releases lock on A
366  *  VFS_ROOT() obtains lock on D while shared lock on F is held
367  *  vfs_unbusy() releases shared lock on F
368  *  vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A.
369  *    Attempt to lock A (instead of vp_crossmp) while D is held would
370  *    violate the global order, causing deadlocks.
371  *
372  * dounmount() locks B while F is drained.
373  */
374 int
375 vfs_busy(struct mount *mp, int flags)
376 {
377 
378 	MPASS((flags & ~MBF_MASK) == 0);
379 	CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags);
380 
381 	MNT_ILOCK(mp);
382 	MNT_REF(mp);
383 	/*
384 	 * If mount point is currenly being unmounted, sleep until the
385 	 * mount point fate is decided.  If thread doing the unmounting fails,
386 	 * it will clear MNTK_UNMOUNT flag before waking us up, indicating
387 	 * that this mount point has survived the unmount attempt and vfs_busy
388 	 * should retry.  Otherwise the unmounter thread will set MNTK_REFEXPIRE
389 	 * flag in addition to MNTK_UNMOUNT, indicating that mount point is
390 	 * about to be really destroyed.  vfs_busy needs to release its
391 	 * reference on the mount point in this case and return with ENOENT,
392 	 * telling the caller that mount mount it tried to busy is no longer
393 	 * valid.
394 	 */
395 	while (mp->mnt_kern_flag & MNTK_UNMOUNT) {
396 		if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) {
397 			MNT_REL(mp);
398 			MNT_IUNLOCK(mp);
399 			CTR1(KTR_VFS, "%s: failed busying before sleeping",
400 			    __func__);
401 			return (ENOENT);
402 		}
403 		if (flags & MBF_MNTLSTLOCK)
404 			mtx_unlock(&mountlist_mtx);
405 		mp->mnt_kern_flag |= MNTK_MWAIT;
406 		msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0);
407 		if (flags & MBF_MNTLSTLOCK)
408 			mtx_lock(&mountlist_mtx);
409 		MNT_ILOCK(mp);
410 	}
411 	if (flags & MBF_MNTLSTLOCK)
412 		mtx_unlock(&mountlist_mtx);
413 	mp->mnt_lockref++;
414 	MNT_IUNLOCK(mp);
415 	return (0);
416 }
417 
418 /*
419  * Free a busy filesystem.
420  */
421 void
422 vfs_unbusy(struct mount *mp)
423 {
424 
425 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
426 	MNT_ILOCK(mp);
427 	MNT_REL(mp);
428 	KASSERT(mp->mnt_lockref > 0, ("negative mnt_lockref"));
429 	mp->mnt_lockref--;
430 	if (mp->mnt_lockref == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) {
431 		MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT);
432 		CTR1(KTR_VFS, "%s: waking up waiters", __func__);
433 		mp->mnt_kern_flag &= ~MNTK_DRAINING;
434 		wakeup(&mp->mnt_lockref);
435 	}
436 	MNT_IUNLOCK(mp);
437 }
438 
439 /*
440  * Lookup a mount point by filesystem identifier.
441  */
442 struct mount *
443 vfs_getvfs(fsid_t *fsid)
444 {
445 	struct mount *mp;
446 
447 	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
448 	mtx_lock(&mountlist_mtx);
449 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
450 		if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
451 		    mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
452 			vfs_ref(mp);
453 			mtx_unlock(&mountlist_mtx);
454 			return (mp);
455 		}
456 	}
457 	mtx_unlock(&mountlist_mtx);
458 	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
459 	return ((struct mount *) 0);
460 }
461 
462 /*
463  * Lookup a mount point by filesystem identifier, busying it before
464  * returning.
465  */
466 struct mount *
467 vfs_busyfs(fsid_t *fsid)
468 {
469 	struct mount *mp;
470 	int error;
471 
472 	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
473 	mtx_lock(&mountlist_mtx);
474 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
475 		if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
476 		    mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
477 			error = vfs_busy(mp, MBF_MNTLSTLOCK);
478 			if (error) {
479 				mtx_unlock(&mountlist_mtx);
480 				return (NULL);
481 			}
482 			return (mp);
483 		}
484 	}
485 	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
486 	mtx_unlock(&mountlist_mtx);
487 	return ((struct mount *) 0);
488 }
489 
490 /*
491  * Check if a user can access privileged mount options.
492  */
493 int
494 vfs_suser(struct mount *mp, struct thread *td)
495 {
496 	int error;
497 
498 	/*
499 	 * If the thread is jailed, but this is not a jail-friendly file
500 	 * system, deny immediately.
501 	 */
502 	if (!(mp->mnt_vfc->vfc_flags & VFCF_JAIL) && jailed(td->td_ucred))
503 		return (EPERM);
504 
505 	/*
506 	 * If the file system was mounted outside the jail of the calling
507 	 * thread, deny immediately.
508 	 */
509 	if (prison_check(td->td_ucred, mp->mnt_cred) != 0)
510 		return (EPERM);
511 
512 	/*
513 	 * If file system supports delegated administration, we don't check
514 	 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified
515 	 * by the file system itself.
516 	 * If this is not the user that did original mount, we check for
517 	 * the PRIV_VFS_MOUNT_OWNER privilege.
518 	 */
519 	if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) &&
520 	    mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) {
521 		if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0)
522 			return (error);
523 	}
524 	return (0);
525 }
526 
527 /*
528  * Get a new unique fsid.  Try to make its val[0] unique, since this value
529  * will be used to create fake device numbers for stat().  Also try (but
530  * not so hard) make its val[0] unique mod 2^16, since some emulators only
531  * support 16-bit device numbers.  We end up with unique val[0]'s for the
532  * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls.
533  *
534  * Keep in mind that several mounts may be running in parallel.  Starting
535  * the search one past where the previous search terminated is both a
536  * micro-optimization and a defense against returning the same fsid to
537  * different mounts.
538  */
539 void
540 vfs_getnewfsid(struct mount *mp)
541 {
542 	static uint16_t mntid_base;
543 	struct mount *nmp;
544 	fsid_t tfsid;
545 	int mtype;
546 
547 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
548 	mtx_lock(&mntid_mtx);
549 	mtype = mp->mnt_vfc->vfc_typenum;
550 	tfsid.val[1] = mtype;
551 	mtype = (mtype & 0xFF) << 24;
552 	for (;;) {
553 		tfsid.val[0] = makedev(255,
554 		    mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF));
555 		mntid_base++;
556 		if ((nmp = vfs_getvfs(&tfsid)) == NULL)
557 			break;
558 		vfs_rel(nmp);
559 	}
560 	mp->mnt_stat.f_fsid.val[0] = tfsid.val[0];
561 	mp->mnt_stat.f_fsid.val[1] = tfsid.val[1];
562 	mtx_unlock(&mntid_mtx);
563 }
564 
565 /*
566  * Knob to control the precision of file timestamps:
567  *
568  *   0 = seconds only; nanoseconds zeroed.
569  *   1 = seconds and nanoseconds, accurate within 1/HZ.
570  *   2 = seconds and nanoseconds, truncated to microseconds.
571  * >=3 = seconds and nanoseconds, maximum precision.
572  */
573 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC };
574 
575 static int timestamp_precision = TSP_SEC;
576 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW,
577     &timestamp_precision, 0, "File timestamp precision (0: seconds, "
578     "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to ms, "
579     "3+: sec + ns (max. precision))");
580 
581 /*
582  * Get a current timestamp.
583  */
584 void
585 vfs_timestamp(struct timespec *tsp)
586 {
587 	struct timeval tv;
588 
589 	switch (timestamp_precision) {
590 	case TSP_SEC:
591 		tsp->tv_sec = time_second;
592 		tsp->tv_nsec = 0;
593 		break;
594 	case TSP_HZ:
595 		getnanotime(tsp);
596 		break;
597 	case TSP_USEC:
598 		microtime(&tv);
599 		TIMEVAL_TO_TIMESPEC(&tv, tsp);
600 		break;
601 	case TSP_NSEC:
602 	default:
603 		nanotime(tsp);
604 		break;
605 	}
606 }
607 
608 /*
609  * Set vnode attributes to VNOVAL
610  */
611 void
612 vattr_null(struct vattr *vap)
613 {
614 
615 	vap->va_type = VNON;
616 	vap->va_size = VNOVAL;
617 	vap->va_bytes = VNOVAL;
618 	vap->va_mode = VNOVAL;
619 	vap->va_nlink = VNOVAL;
620 	vap->va_uid = VNOVAL;
621 	vap->va_gid = VNOVAL;
622 	vap->va_fsid = VNOVAL;
623 	vap->va_fileid = VNOVAL;
624 	vap->va_blocksize = VNOVAL;
625 	vap->va_rdev = VNOVAL;
626 	vap->va_atime.tv_sec = VNOVAL;
627 	vap->va_atime.tv_nsec = VNOVAL;
628 	vap->va_mtime.tv_sec = VNOVAL;
629 	vap->va_mtime.tv_nsec = VNOVAL;
630 	vap->va_ctime.tv_sec = VNOVAL;
631 	vap->va_ctime.tv_nsec = VNOVAL;
632 	vap->va_birthtime.tv_sec = VNOVAL;
633 	vap->va_birthtime.tv_nsec = VNOVAL;
634 	vap->va_flags = VNOVAL;
635 	vap->va_gen = VNOVAL;
636 	vap->va_vaflags = 0;
637 }
638 
639 /*
640  * This routine is called when we have too many vnodes.  It attempts
641  * to free <count> vnodes and will potentially free vnodes that still
642  * have VM backing store (VM backing store is typically the cause
643  * of a vnode blowout so we want to do this).  Therefore, this operation
644  * is not considered cheap.
645  *
646  * A number of conditions may prevent a vnode from being reclaimed.
647  * the buffer cache may have references on the vnode, a directory
648  * vnode may still have references due to the namei cache representing
649  * underlying files, or the vnode may be in active use.   It is not
650  * desireable to reuse such vnodes.  These conditions may cause the
651  * number of vnodes to reach some minimum value regardless of what
652  * you set kern.maxvnodes to.  Do not set kern.maxvnodes too low.
653  */
654 static int
655 vlrureclaim(struct mount *mp)
656 {
657 	struct vnode *vp;
658 	int done;
659 	int trigger;
660 	int usevnodes;
661 	int count;
662 
663 	/*
664 	 * Calculate the trigger point, don't allow user
665 	 * screwups to blow us up.   This prevents us from
666 	 * recycling vnodes with lots of resident pages.  We
667 	 * aren't trying to free memory, we are trying to
668 	 * free vnodes.
669 	 */
670 	usevnodes = desiredvnodes;
671 	if (usevnodes <= 0)
672 		usevnodes = 1;
673 	trigger = cnt.v_page_count * 2 / usevnodes;
674 	done = 0;
675 	vn_start_write(NULL, &mp, V_WAIT);
676 	MNT_ILOCK(mp);
677 	count = mp->mnt_nvnodelistsize / 10 + 1;
678 	while (count != 0) {
679 		vp = TAILQ_FIRST(&mp->mnt_nvnodelist);
680 		while (vp != NULL && vp->v_type == VMARKER)
681 			vp = TAILQ_NEXT(vp, v_nmntvnodes);
682 		if (vp == NULL)
683 			break;
684 		TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
685 		TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
686 		--count;
687 		if (!VI_TRYLOCK(vp))
688 			goto next_iter;
689 		/*
690 		 * If it's been deconstructed already, it's still
691 		 * referenced, or it exceeds the trigger, skip it.
692 		 */
693 		if (vp->v_usecount ||
694 		    (!vlru_allow_cache_src &&
695 			!LIST_EMPTY(&(vp)->v_cache_src)) ||
696 		    (vp->v_iflag & VI_DOOMED) != 0 || (vp->v_object != NULL &&
697 		    vp->v_object->resident_page_count > trigger)) {
698 			VI_UNLOCK(vp);
699 			goto next_iter;
700 		}
701 		MNT_IUNLOCK(mp);
702 		vholdl(vp);
703 		if (VOP_LOCK(vp, LK_INTERLOCK|LK_EXCLUSIVE|LK_NOWAIT)) {
704 			vdrop(vp);
705 			goto next_iter_mntunlocked;
706 		}
707 		VI_LOCK(vp);
708 		/*
709 		 * v_usecount may have been bumped after VOP_LOCK() dropped
710 		 * the vnode interlock and before it was locked again.
711 		 *
712 		 * It is not necessary to recheck VI_DOOMED because it can
713 		 * only be set by another thread that holds both the vnode
714 		 * lock and vnode interlock.  If another thread has the
715 		 * vnode lock before we get to VOP_LOCK() and obtains the
716 		 * vnode interlock after VOP_LOCK() drops the vnode
717 		 * interlock, the other thread will be unable to drop the
718 		 * vnode lock before our VOP_LOCK() call fails.
719 		 */
720 		if (vp->v_usecount ||
721 		    (!vlru_allow_cache_src &&
722 			!LIST_EMPTY(&(vp)->v_cache_src)) ||
723 		    (vp->v_object != NULL &&
724 		    vp->v_object->resident_page_count > trigger)) {
725 			VOP_UNLOCK(vp, LK_INTERLOCK);
726 			goto next_iter_mntunlocked;
727 		}
728 		KASSERT((vp->v_iflag & VI_DOOMED) == 0,
729 		    ("VI_DOOMED unexpectedly detected in vlrureclaim()"));
730 		vgonel(vp);
731 		VOP_UNLOCK(vp, 0);
732 		vdropl(vp);
733 		done++;
734 next_iter_mntunlocked:
735 		if (!should_yield())
736 			goto relock_mnt;
737 		goto yield;
738 next_iter:
739 		if (!should_yield())
740 			continue;
741 		MNT_IUNLOCK(mp);
742 yield:
743 		kern_yield(PRI_UNCHANGED);
744 relock_mnt:
745 		MNT_ILOCK(mp);
746 	}
747 	MNT_IUNLOCK(mp);
748 	vn_finished_write(mp);
749 	return done;
750 }
751 
752 /*
753  * Attempt to keep the free list at wantfreevnodes length.
754  */
755 static void
756 vnlru_free(int count)
757 {
758 	struct vnode *vp;
759 
760 	mtx_assert(&vnode_free_list_mtx, MA_OWNED);
761 	for (; count > 0; count--) {
762 		vp = TAILQ_FIRST(&vnode_free_list);
763 		/*
764 		 * The list can be modified while the free_list_mtx
765 		 * has been dropped and vp could be NULL here.
766 		 */
767 		if (!vp)
768 			break;
769 		VNASSERT(vp->v_op != NULL, vp,
770 		    ("vnlru_free: vnode already reclaimed."));
771 		KASSERT((vp->v_iflag & VI_FREE) != 0,
772 		    ("Removing vnode not on freelist"));
773 		KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
774 		    ("Mangling active vnode"));
775 		TAILQ_REMOVE(&vnode_free_list, vp, v_actfreelist);
776 		/*
777 		 * Don't recycle if we can't get the interlock.
778 		 */
779 		if (!VI_TRYLOCK(vp)) {
780 			TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_actfreelist);
781 			continue;
782 		}
783 		VNASSERT(VCANRECYCLE(vp), vp,
784 		    ("vp inconsistent on freelist"));
785 		freevnodes--;
786 		vp->v_iflag &= ~VI_FREE;
787 		vholdl(vp);
788 		mtx_unlock(&vnode_free_list_mtx);
789 		VI_UNLOCK(vp);
790 		vtryrecycle(vp);
791 		/*
792 		 * If the recycled succeeded this vdrop will actually free
793 		 * the vnode.  If not it will simply place it back on
794 		 * the free list.
795 		 */
796 		vdrop(vp);
797 		mtx_lock(&vnode_free_list_mtx);
798 	}
799 }
800 /*
801  * Attempt to recycle vnodes in a context that is always safe to block.
802  * Calling vlrurecycle() from the bowels of filesystem code has some
803  * interesting deadlock problems.
804  */
805 static struct proc *vnlruproc;
806 static int vnlruproc_sig;
807 
808 static void
809 vnlru_proc(void)
810 {
811 	struct mount *mp, *nmp;
812 	int done;
813 	struct proc *p = vnlruproc;
814 
815 	EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, p,
816 	    SHUTDOWN_PRI_FIRST);
817 
818 	for (;;) {
819 		kproc_suspend_check(p);
820 		mtx_lock(&vnode_free_list_mtx);
821 		if (freevnodes > wantfreevnodes)
822 			vnlru_free(freevnodes - wantfreevnodes);
823 		if (numvnodes <= desiredvnodes * 9 / 10) {
824 			vnlruproc_sig = 0;
825 			wakeup(&vnlruproc_sig);
826 			msleep(vnlruproc, &vnode_free_list_mtx,
827 			    PVFS|PDROP, "vlruwt", hz);
828 			continue;
829 		}
830 		mtx_unlock(&vnode_free_list_mtx);
831 		done = 0;
832 		mtx_lock(&mountlist_mtx);
833 		for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
834 			if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) {
835 				nmp = TAILQ_NEXT(mp, mnt_list);
836 				continue;
837 			}
838 			done += vlrureclaim(mp);
839 			mtx_lock(&mountlist_mtx);
840 			nmp = TAILQ_NEXT(mp, mnt_list);
841 			vfs_unbusy(mp);
842 		}
843 		mtx_unlock(&mountlist_mtx);
844 		if (done == 0) {
845 #if 0
846 			/* These messages are temporary debugging aids */
847 			if (vnlru_nowhere < 5)
848 				printf("vnlru process getting nowhere..\n");
849 			else if (vnlru_nowhere == 5)
850 				printf("vnlru process messages stopped.\n");
851 #endif
852 			vnlru_nowhere++;
853 			tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3);
854 		} else
855 			kern_yield(PRI_UNCHANGED);
856 	}
857 }
858 
859 static struct kproc_desc vnlru_kp = {
860 	"vnlru",
861 	vnlru_proc,
862 	&vnlruproc
863 };
864 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start,
865     &vnlru_kp);
866 
867 /*
868  * Routines having to do with the management of the vnode table.
869  */
870 
871 /*
872  * Try to recycle a freed vnode.  We abort if anyone picks up a reference
873  * before we actually vgone().  This function must be called with the vnode
874  * held to prevent the vnode from being returned to the free list midway
875  * through vgone().
876  */
877 static int
878 vtryrecycle(struct vnode *vp)
879 {
880 	struct mount *vnmp;
881 
882 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
883 	VNASSERT(vp->v_holdcnt, vp,
884 	    ("vtryrecycle: Recycling vp %p without a reference.", vp));
885 	/*
886 	 * This vnode may found and locked via some other list, if so we
887 	 * can't recycle it yet.
888 	 */
889 	if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) {
890 		CTR2(KTR_VFS,
891 		    "%s: impossible to recycle, vp %p lock is already held",
892 		    __func__, vp);
893 		return (EWOULDBLOCK);
894 	}
895 	/*
896 	 * Don't recycle if its filesystem is being suspended.
897 	 */
898 	if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) {
899 		VOP_UNLOCK(vp, 0);
900 		CTR2(KTR_VFS,
901 		    "%s: impossible to recycle, cannot start the write for %p",
902 		    __func__, vp);
903 		return (EBUSY);
904 	}
905 	/*
906 	 * If we got this far, we need to acquire the interlock and see if
907 	 * anyone picked up this vnode from another list.  If not, we will
908 	 * mark it with DOOMED via vgonel() so that anyone who does find it
909 	 * will skip over it.
910 	 */
911 	VI_LOCK(vp);
912 	if (vp->v_usecount) {
913 		VOP_UNLOCK(vp, LK_INTERLOCK);
914 		vn_finished_write(vnmp);
915 		CTR2(KTR_VFS,
916 		    "%s: impossible to recycle, %p is already referenced",
917 		    __func__, vp);
918 		return (EBUSY);
919 	}
920 	if ((vp->v_iflag & VI_DOOMED) == 0)
921 		vgonel(vp);
922 	VOP_UNLOCK(vp, LK_INTERLOCK);
923 	vn_finished_write(vnmp);
924 	return (0);
925 }
926 
927 /*
928  * Wait for available vnodes.
929  */
930 static int
931 getnewvnode_wait(int suspended)
932 {
933 
934 	mtx_assert(&vnode_free_list_mtx, MA_OWNED);
935 	if (numvnodes > desiredvnodes) {
936 		if (suspended) {
937 			/*
938 			 * File system is beeing suspended, we cannot risk a
939 			 * deadlock here, so allocate new vnode anyway.
940 			 */
941 			if (freevnodes > wantfreevnodes)
942 				vnlru_free(freevnodes - wantfreevnodes);
943 			return (0);
944 		}
945 		if (vnlruproc_sig == 0) {
946 			vnlruproc_sig = 1;	/* avoid unnecessary wakeups */
947 			wakeup(vnlruproc);
948 		}
949 		msleep(&vnlruproc_sig, &vnode_free_list_mtx, PVFS,
950 		    "vlruwk", hz);
951 	}
952 	return (numvnodes > desiredvnodes ? ENFILE : 0);
953 }
954 
955 void
956 getnewvnode_reserve(u_int count)
957 {
958 	struct thread *td;
959 
960 	td = curthread;
961 	mtx_lock(&vnode_free_list_mtx);
962 	while (count > 0) {
963 		if (getnewvnode_wait(0) == 0) {
964 			count--;
965 			td->td_vp_reserv++;
966 			numvnodes++;
967 		}
968 	}
969 	mtx_unlock(&vnode_free_list_mtx);
970 }
971 
972 void
973 getnewvnode_drop_reserve(void)
974 {
975 	struct thread *td;
976 
977 	td = curthread;
978 	mtx_lock(&vnode_free_list_mtx);
979 	KASSERT(numvnodes >= td->td_vp_reserv, ("reserve too large"));
980 	numvnodes -= td->td_vp_reserv;
981 	mtx_unlock(&vnode_free_list_mtx);
982 	td->td_vp_reserv = 0;
983 }
984 
985 /*
986  * Return the next vnode from the free list.
987  */
988 int
989 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops,
990     struct vnode **vpp)
991 {
992 	struct vnode *vp;
993 	struct bufobj *bo;
994 	struct thread *td;
995 	int error;
996 
997 	CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag);
998 	vp = NULL;
999 	td = curthread;
1000 	if (td->td_vp_reserv > 0) {
1001 		td->td_vp_reserv -= 1;
1002 		goto alloc;
1003 	}
1004 	mtx_lock(&vnode_free_list_mtx);
1005 	/*
1006 	 * Lend our context to reclaim vnodes if they've exceeded the max.
1007 	 */
1008 	if (freevnodes > wantfreevnodes)
1009 		vnlru_free(1);
1010 	error = getnewvnode_wait(mp != NULL && (mp->mnt_kern_flag &
1011 	    MNTK_SUSPEND));
1012 #if 0	/* XXX Not all VFS_VGET/ffs_vget callers check returns. */
1013 	if (error != 0) {
1014 		mtx_unlock(&vnode_free_list_mtx);
1015 		return (error);
1016 	}
1017 #endif
1018 	numvnodes++;
1019 	mtx_unlock(&vnode_free_list_mtx);
1020 alloc:
1021 	vp = (struct vnode *) uma_zalloc(vnode_zone, M_WAITOK|M_ZERO);
1022 	/*
1023 	 * Setup locks.
1024 	 */
1025 	vp->v_vnlock = &vp->v_lock;
1026 	mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF);
1027 	/*
1028 	 * By default, don't allow shared locks unless filesystems
1029 	 * opt-in.
1030 	 */
1031 	lockinit(vp->v_vnlock, PVFS, tag, VLKTIMEOUT, LK_NOSHARE);
1032 	/*
1033 	 * Initialize bufobj.
1034 	 */
1035 	bo = &vp->v_bufobj;
1036 	bo->__bo_vnode = vp;
1037 	mtx_init(BO_MTX(bo), "bufobj interlock", NULL, MTX_DEF);
1038 	bo->bo_ops = &buf_ops_bio;
1039 	bo->bo_private = vp;
1040 	TAILQ_INIT(&bo->bo_clean.bv_hd);
1041 	TAILQ_INIT(&bo->bo_dirty.bv_hd);
1042 	/*
1043 	 * Initialize namecache.
1044 	 */
1045 	LIST_INIT(&vp->v_cache_src);
1046 	TAILQ_INIT(&vp->v_cache_dst);
1047 	/*
1048 	 * Finalize various vnode identity bits.
1049 	 */
1050 	vp->v_type = VNON;
1051 	vp->v_tag = tag;
1052 	vp->v_op = vops;
1053 	v_incr_usecount(vp);
1054 	vp->v_data = NULL;
1055 #ifdef MAC
1056 	mac_vnode_init(vp);
1057 	if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0)
1058 		mac_vnode_associate_singlelabel(mp, vp);
1059 	else if (mp == NULL && vops != &dead_vnodeops)
1060 		printf("NULL mp in getnewvnode()\n");
1061 #endif
1062 	if (mp != NULL) {
1063 		bo->bo_bsize = mp->mnt_stat.f_iosize;
1064 		if ((mp->mnt_kern_flag & MNTK_NOKNOTE) != 0)
1065 			vp->v_vflag |= VV_NOKNOTE;
1066 	}
1067 	rangelock_init(&vp->v_rl);
1068 
1069 	*vpp = vp;
1070 	return (0);
1071 }
1072 
1073 /*
1074  * Delete from old mount point vnode list, if on one.
1075  */
1076 static void
1077 delmntque(struct vnode *vp)
1078 {
1079 	struct mount *mp;
1080 	int active;
1081 
1082 	mp = vp->v_mount;
1083 	if (mp == NULL)
1084 		return;
1085 	MNT_ILOCK(mp);
1086 	VI_LOCK(vp);
1087 	KASSERT(mp->mnt_activevnodelistsize <= mp->mnt_nvnodelistsize,
1088 	    ("Active vnode list size %d > Vnode list size %d",
1089 	     mp->mnt_activevnodelistsize, mp->mnt_nvnodelistsize));
1090 	active = vp->v_iflag & VI_ACTIVE;
1091 	vp->v_iflag &= ~VI_ACTIVE;
1092 	if (active) {
1093 		mtx_lock(&vnode_free_list_mtx);
1094 		TAILQ_REMOVE(&mp->mnt_activevnodelist, vp, v_actfreelist);
1095 		mp->mnt_activevnodelistsize--;
1096 		mtx_unlock(&vnode_free_list_mtx);
1097 	}
1098 	vp->v_mount = NULL;
1099 	VI_UNLOCK(vp);
1100 	VNASSERT(mp->mnt_nvnodelistsize > 0, vp,
1101 		("bad mount point vnode list size"));
1102 	TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1103 	mp->mnt_nvnodelistsize--;
1104 	MNT_REL(mp);
1105 	MNT_IUNLOCK(mp);
1106 }
1107 
1108 static void
1109 insmntque_stddtr(struct vnode *vp, void *dtr_arg)
1110 {
1111 
1112 	vp->v_data = NULL;
1113 	vp->v_op = &dead_vnodeops;
1114 	vgone(vp);
1115 	vput(vp);
1116 }
1117 
1118 /*
1119  * Insert into list of vnodes for the new mount point, if available.
1120  */
1121 int
1122 insmntque1(struct vnode *vp, struct mount *mp,
1123 	void (*dtr)(struct vnode *, void *), void *dtr_arg)
1124 {
1125 
1126 	KASSERT(vp->v_mount == NULL,
1127 		("insmntque: vnode already on per mount vnode list"));
1128 	VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)"));
1129 	ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp");
1130 
1131 	/*
1132 	 * We acquire the vnode interlock early to ensure that the
1133 	 * vnode cannot be recycled by another process releasing a
1134 	 * holdcnt on it before we get it on both the vnode list
1135 	 * and the active vnode list. The mount mutex protects only
1136 	 * manipulation of the vnode list and the vnode freelist
1137 	 * mutex protects only manipulation of the active vnode list.
1138 	 * Hence the need to hold the vnode interlock throughout.
1139 	 */
1140 	MNT_ILOCK(mp);
1141 	VI_LOCK(vp);
1142 	if (((mp->mnt_kern_flag & MNTK_NOINSMNTQ) != 0 &&
1143 	    ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 ||
1144 	    mp->mnt_nvnodelistsize == 0)) &&
1145 	    (vp->v_vflag & VV_FORCEINSMQ) == 0) {
1146 		VI_UNLOCK(vp);
1147 		MNT_IUNLOCK(mp);
1148 		if (dtr != NULL)
1149 			dtr(vp, dtr_arg);
1150 		return (EBUSY);
1151 	}
1152 	vp->v_mount = mp;
1153 	MNT_REF(mp);
1154 	TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1155 	VNASSERT(mp->mnt_nvnodelistsize >= 0, vp,
1156 		("neg mount point vnode list size"));
1157 	mp->mnt_nvnodelistsize++;
1158 	KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
1159 	    ("Activating already active vnode"));
1160 	vp->v_iflag |= VI_ACTIVE;
1161 	mtx_lock(&vnode_free_list_mtx);
1162 	TAILQ_INSERT_HEAD(&mp->mnt_activevnodelist, vp, v_actfreelist);
1163 	mp->mnt_activevnodelistsize++;
1164 	mtx_unlock(&vnode_free_list_mtx);
1165 	VI_UNLOCK(vp);
1166 	MNT_IUNLOCK(mp);
1167 	return (0);
1168 }
1169 
1170 int
1171 insmntque(struct vnode *vp, struct mount *mp)
1172 {
1173 
1174 	return (insmntque1(vp, mp, insmntque_stddtr, NULL));
1175 }
1176 
1177 /*
1178  * Flush out and invalidate all buffers associated with a bufobj
1179  * Called with the underlying object locked.
1180  */
1181 int
1182 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
1183 {
1184 	int error;
1185 
1186 	BO_LOCK(bo);
1187 	if (flags & V_SAVE) {
1188 		error = bufobj_wwait(bo, slpflag, slptimeo);
1189 		if (error) {
1190 			BO_UNLOCK(bo);
1191 			return (error);
1192 		}
1193 		if (bo->bo_dirty.bv_cnt > 0) {
1194 			BO_UNLOCK(bo);
1195 			if ((error = BO_SYNC(bo, MNT_WAIT)) != 0)
1196 				return (error);
1197 			/*
1198 			 * XXX We could save a lock/unlock if this was only
1199 			 * enabled under INVARIANTS
1200 			 */
1201 			BO_LOCK(bo);
1202 			if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0)
1203 				panic("vinvalbuf: dirty bufs");
1204 		}
1205 	}
1206 	/*
1207 	 * If you alter this loop please notice that interlock is dropped and
1208 	 * reacquired in flushbuflist.  Special care is needed to ensure that
1209 	 * no race conditions occur from this.
1210 	 */
1211 	do {
1212 		error = flushbuflist(&bo->bo_clean,
1213 		    flags, bo, slpflag, slptimeo);
1214 		if (error == 0 && !(flags & V_CLEANONLY))
1215 			error = flushbuflist(&bo->bo_dirty,
1216 			    flags, bo, slpflag, slptimeo);
1217 		if (error != 0 && error != EAGAIN) {
1218 			BO_UNLOCK(bo);
1219 			return (error);
1220 		}
1221 	} while (error != 0);
1222 
1223 	/*
1224 	 * Wait for I/O to complete.  XXX needs cleaning up.  The vnode can
1225 	 * have write I/O in-progress but if there is a VM object then the
1226 	 * VM object can also have read-I/O in-progress.
1227 	 */
1228 	do {
1229 		bufobj_wwait(bo, 0, 0);
1230 		BO_UNLOCK(bo);
1231 		if (bo->bo_object != NULL) {
1232 			VM_OBJECT_LOCK(bo->bo_object);
1233 			vm_object_pip_wait(bo->bo_object, "bovlbx");
1234 			VM_OBJECT_UNLOCK(bo->bo_object);
1235 		}
1236 		BO_LOCK(bo);
1237 	} while (bo->bo_numoutput > 0);
1238 	BO_UNLOCK(bo);
1239 
1240 	/*
1241 	 * Destroy the copy in the VM cache, too.
1242 	 */
1243 	if (bo->bo_object != NULL &&
1244 	    (flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0) {
1245 		VM_OBJECT_LOCK(bo->bo_object);
1246 		vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ?
1247 		    OBJPR_CLEANONLY : 0);
1248 		VM_OBJECT_UNLOCK(bo->bo_object);
1249 	}
1250 
1251 #ifdef INVARIANTS
1252 	BO_LOCK(bo);
1253 	if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0 &&
1254 	    (bo->bo_dirty.bv_cnt > 0 || bo->bo_clean.bv_cnt > 0))
1255 		panic("vinvalbuf: flush failed");
1256 	BO_UNLOCK(bo);
1257 #endif
1258 	return (0);
1259 }
1260 
1261 /*
1262  * Flush out and invalidate all buffers associated with a vnode.
1263  * Called with the underlying object locked.
1264  */
1265 int
1266 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
1267 {
1268 
1269 	CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
1270 	ASSERT_VOP_LOCKED(vp, "vinvalbuf");
1271 	return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo));
1272 }
1273 
1274 /*
1275  * Flush out buffers on the specified list.
1276  *
1277  */
1278 static int
1279 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag,
1280     int slptimeo)
1281 {
1282 	struct buf *bp, *nbp;
1283 	int retval, error;
1284 	daddr_t lblkno;
1285 	b_xflags_t xflags;
1286 
1287 	ASSERT_BO_LOCKED(bo);
1288 
1289 	retval = 0;
1290 	TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) {
1291 		if (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA)) ||
1292 		    ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0)) {
1293 			continue;
1294 		}
1295 		lblkno = 0;
1296 		xflags = 0;
1297 		if (nbp != NULL) {
1298 			lblkno = nbp->b_lblkno;
1299 			xflags = nbp->b_xflags &
1300 				(BX_BKGRDMARKER | BX_VNDIRTY | BX_VNCLEAN);
1301 		}
1302 		retval = EAGAIN;
1303 		error = BUF_TIMELOCK(bp,
1304 		    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_MTX(bo),
1305 		    "flushbuf", slpflag, slptimeo);
1306 		if (error) {
1307 			BO_LOCK(bo);
1308 			return (error != ENOLCK ? error : EAGAIN);
1309 		}
1310 		KASSERT(bp->b_bufobj == bo,
1311 		    ("bp %p wrong b_bufobj %p should be %p",
1312 		    bp, bp->b_bufobj, bo));
1313 		if (bp->b_bufobj != bo) {	/* XXX: necessary ? */
1314 			BUF_UNLOCK(bp);
1315 			BO_LOCK(bo);
1316 			return (EAGAIN);
1317 		}
1318 		/*
1319 		 * XXX Since there are no node locks for NFS, I
1320 		 * believe there is a slight chance that a delayed
1321 		 * write will occur while sleeping just above, so
1322 		 * check for it.
1323 		 */
1324 		if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
1325 		    (flags & V_SAVE)) {
1326 			BO_LOCK(bo);
1327 			bremfree(bp);
1328 			BO_UNLOCK(bo);
1329 			bp->b_flags |= B_ASYNC;
1330 			bwrite(bp);
1331 			BO_LOCK(bo);
1332 			return (EAGAIN);	/* XXX: why not loop ? */
1333 		}
1334 		BO_LOCK(bo);
1335 		bremfree(bp);
1336 		BO_UNLOCK(bo);
1337 		bp->b_flags |= (B_INVAL | B_RELBUF);
1338 		bp->b_flags &= ~B_ASYNC;
1339 		brelse(bp);
1340 		BO_LOCK(bo);
1341 		if (nbp != NULL &&
1342 		    (nbp->b_bufobj != bo ||
1343 		     nbp->b_lblkno != lblkno ||
1344 		     (nbp->b_xflags &
1345 		      (BX_BKGRDMARKER | BX_VNDIRTY | BX_VNCLEAN)) != xflags))
1346 			break;			/* nbp invalid */
1347 	}
1348 	return (retval);
1349 }
1350 
1351 /*
1352  * Truncate a file's buffer and pages to a specified length.  This
1353  * is in lieu of the old vinvalbuf mechanism, which performed unneeded
1354  * sync activity.
1355  */
1356 int
1357 vtruncbuf(struct vnode *vp, struct ucred *cred, off_t length, int blksize)
1358 {
1359 	struct buf *bp, *nbp;
1360 	int anyfreed;
1361 	int trunclbn;
1362 	struct bufobj *bo;
1363 
1364 	CTR5(KTR_VFS, "%s: vp %p with cred %p and block %d:%ju", __func__,
1365 	    vp, cred, blksize, (uintmax_t)length);
1366 
1367 	/*
1368 	 * Round up to the *next* lbn.
1369 	 */
1370 	trunclbn = (length + blksize - 1) / blksize;
1371 
1372 	ASSERT_VOP_LOCKED(vp, "vtruncbuf");
1373 restart:
1374 	bo = &vp->v_bufobj;
1375 	BO_LOCK(bo);
1376 	anyfreed = 1;
1377 	for (;anyfreed;) {
1378 		anyfreed = 0;
1379 		TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) {
1380 			if (bp->b_lblkno < trunclbn)
1381 				continue;
1382 			if (BUF_LOCK(bp,
1383 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1384 			    BO_MTX(bo)) == ENOLCK)
1385 				goto restart;
1386 
1387 			BO_LOCK(bo);
1388 			bremfree(bp);
1389 			BO_UNLOCK(bo);
1390 			bp->b_flags |= (B_INVAL | B_RELBUF);
1391 			bp->b_flags &= ~B_ASYNC;
1392 			brelse(bp);
1393 			anyfreed = 1;
1394 
1395 			BO_LOCK(bo);
1396 			if (nbp != NULL &&
1397 			    (((nbp->b_xflags & BX_VNCLEAN) == 0) ||
1398 			    (nbp->b_vp != vp) ||
1399 			    (nbp->b_flags & B_DELWRI))) {
1400 				BO_UNLOCK(bo);
1401 				goto restart;
1402 			}
1403 		}
1404 
1405 		TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
1406 			if (bp->b_lblkno < trunclbn)
1407 				continue;
1408 			if (BUF_LOCK(bp,
1409 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1410 			    BO_MTX(bo)) == ENOLCK)
1411 				goto restart;
1412 			BO_LOCK(bo);
1413 			bremfree(bp);
1414 			BO_UNLOCK(bo);
1415 			bp->b_flags |= (B_INVAL | B_RELBUF);
1416 			bp->b_flags &= ~B_ASYNC;
1417 			brelse(bp);
1418 			anyfreed = 1;
1419 
1420 			BO_LOCK(bo);
1421 			if (nbp != NULL &&
1422 			    (((nbp->b_xflags & BX_VNDIRTY) == 0) ||
1423 			    (nbp->b_vp != vp) ||
1424 			    (nbp->b_flags & B_DELWRI) == 0)) {
1425 				BO_UNLOCK(bo);
1426 				goto restart;
1427 			}
1428 		}
1429 	}
1430 
1431 	if (length > 0) {
1432 restartsync:
1433 		TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
1434 			if (bp->b_lblkno > 0)
1435 				continue;
1436 			/*
1437 			 * Since we hold the vnode lock this should only
1438 			 * fail if we're racing with the buf daemon.
1439 			 */
1440 			if (BUF_LOCK(bp,
1441 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1442 			    BO_MTX(bo)) == ENOLCK) {
1443 				goto restart;
1444 			}
1445 			VNASSERT((bp->b_flags & B_DELWRI), vp,
1446 			    ("buf(%p) on dirty queue without DELWRI", bp));
1447 
1448 			BO_LOCK(bo);
1449 			bremfree(bp);
1450 			BO_UNLOCK(bo);
1451 			bawrite(bp);
1452 			BO_LOCK(bo);
1453 			goto restartsync;
1454 		}
1455 	}
1456 
1457 	bufobj_wwait(bo, 0, 0);
1458 	BO_UNLOCK(bo);
1459 	vnode_pager_setsize(vp, length);
1460 
1461 	return (0);
1462 }
1463 
1464 /*
1465  * buf_splay() - splay tree core for the clean/dirty list of buffers in
1466  *		 a vnode.
1467  *
1468  *	NOTE: We have to deal with the special case of a background bitmap
1469  *	buffer, a situation where two buffers will have the same logical
1470  *	block offset.  We want (1) only the foreground buffer to be accessed
1471  *	in a lookup and (2) must differentiate between the foreground and
1472  *	background buffer in the splay tree algorithm because the splay
1473  *	tree cannot normally handle multiple entities with the same 'index'.
1474  *	We accomplish this by adding differentiating flags to the splay tree's
1475  *	numerical domain.
1476  */
1477 static
1478 struct buf *
1479 buf_splay(daddr_t lblkno, b_xflags_t xflags, struct buf *root)
1480 {
1481 	struct buf dummy;
1482 	struct buf *lefttreemax, *righttreemin, *y;
1483 
1484 	if (root == NULL)
1485 		return (NULL);
1486 	lefttreemax = righttreemin = &dummy;
1487 	for (;;) {
1488 		if (lblkno < root->b_lblkno ||
1489 		    (lblkno == root->b_lblkno &&
1490 		    (xflags & BX_BKGRDMARKER) < (root->b_xflags & BX_BKGRDMARKER))) {
1491 			if ((y = root->b_left) == NULL)
1492 				break;
1493 			if (lblkno < y->b_lblkno) {
1494 				/* Rotate right. */
1495 				root->b_left = y->b_right;
1496 				y->b_right = root;
1497 				root = y;
1498 				if ((y = root->b_left) == NULL)
1499 					break;
1500 			}
1501 			/* Link into the new root's right tree. */
1502 			righttreemin->b_left = root;
1503 			righttreemin = root;
1504 		} else if (lblkno > root->b_lblkno ||
1505 		    (lblkno == root->b_lblkno &&
1506 		    (xflags & BX_BKGRDMARKER) > (root->b_xflags & BX_BKGRDMARKER))) {
1507 			if ((y = root->b_right) == NULL)
1508 				break;
1509 			if (lblkno > y->b_lblkno) {
1510 				/* Rotate left. */
1511 				root->b_right = y->b_left;
1512 				y->b_left = root;
1513 				root = y;
1514 				if ((y = root->b_right) == NULL)
1515 					break;
1516 			}
1517 			/* Link into the new root's left tree. */
1518 			lefttreemax->b_right = root;
1519 			lefttreemax = root;
1520 		} else {
1521 			break;
1522 		}
1523 		root = y;
1524 	}
1525 	/* Assemble the new root. */
1526 	lefttreemax->b_right = root->b_left;
1527 	righttreemin->b_left = root->b_right;
1528 	root->b_left = dummy.b_right;
1529 	root->b_right = dummy.b_left;
1530 	return (root);
1531 }
1532 
1533 static void
1534 buf_vlist_remove(struct buf *bp)
1535 {
1536 	struct buf *root;
1537 	struct bufv *bv;
1538 
1539 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
1540 	ASSERT_BO_LOCKED(bp->b_bufobj);
1541 	KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) !=
1542 	    (BX_VNDIRTY|BX_VNCLEAN),
1543 	    ("buf_vlist_remove: Buf %p is on two lists", bp));
1544 	if (bp->b_xflags & BX_VNDIRTY)
1545 		bv = &bp->b_bufobj->bo_dirty;
1546 	else
1547 		bv = &bp->b_bufobj->bo_clean;
1548 	if (bp != bv->bv_root) {
1549 		root = buf_splay(bp->b_lblkno, bp->b_xflags, bv->bv_root);
1550 		KASSERT(root == bp, ("splay lookup failed in remove"));
1551 	}
1552 	if (bp->b_left == NULL) {
1553 		root = bp->b_right;
1554 	} else {
1555 		root = buf_splay(bp->b_lblkno, bp->b_xflags, bp->b_left);
1556 		root->b_right = bp->b_right;
1557 	}
1558 	bv->bv_root = root;
1559 	TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs);
1560 	bv->bv_cnt--;
1561 	bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN);
1562 }
1563 
1564 /*
1565  * Add the buffer to the sorted clean or dirty block list using a
1566  * splay tree algorithm.
1567  *
1568  * NOTE: xflags is passed as a constant, optimizing this inline function!
1569  */
1570 static void
1571 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags)
1572 {
1573 	struct buf *root;
1574 	struct bufv *bv;
1575 
1576 	ASSERT_BO_LOCKED(bo);
1577 	KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0,
1578 	    ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags));
1579 	bp->b_xflags |= xflags;
1580 	if (xflags & BX_VNDIRTY)
1581 		bv = &bo->bo_dirty;
1582 	else
1583 		bv = &bo->bo_clean;
1584 
1585 	root = buf_splay(bp->b_lblkno, bp->b_xflags, bv->bv_root);
1586 	if (root == NULL) {
1587 		bp->b_left = NULL;
1588 		bp->b_right = NULL;
1589 		TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs);
1590 	} else if (bp->b_lblkno < root->b_lblkno ||
1591 	    (bp->b_lblkno == root->b_lblkno &&
1592 	    (bp->b_xflags & BX_BKGRDMARKER) < (root->b_xflags & BX_BKGRDMARKER))) {
1593 		bp->b_left = root->b_left;
1594 		bp->b_right = root;
1595 		root->b_left = NULL;
1596 		TAILQ_INSERT_BEFORE(root, bp, b_bobufs);
1597 	} else {
1598 		bp->b_right = root->b_right;
1599 		bp->b_left = root;
1600 		root->b_right = NULL;
1601 		TAILQ_INSERT_AFTER(&bv->bv_hd, root, bp, b_bobufs);
1602 	}
1603 	bv->bv_cnt++;
1604 	bv->bv_root = bp;
1605 }
1606 
1607 /*
1608  * Lookup a buffer using the splay tree.  Note that we specifically avoid
1609  * shadow buffers used in background bitmap writes.
1610  *
1611  * This code isn't quite efficient as it could be because we are maintaining
1612  * two sorted lists and do not know which list the block resides in.
1613  *
1614  * During a "make buildworld" the desired buffer is found at one of
1615  * the roots more than 60% of the time.  Thus, checking both roots
1616  * before performing either splay eliminates unnecessary splays on the
1617  * first tree splayed.
1618  */
1619 struct buf *
1620 gbincore(struct bufobj *bo, daddr_t lblkno)
1621 {
1622 	struct buf *bp;
1623 
1624 	ASSERT_BO_LOCKED(bo);
1625 	if ((bp = bo->bo_clean.bv_root) != NULL &&
1626 	    bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1627 		return (bp);
1628 	if ((bp = bo->bo_dirty.bv_root) != NULL &&
1629 	    bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1630 		return (bp);
1631 	if ((bp = bo->bo_clean.bv_root) != NULL) {
1632 		bo->bo_clean.bv_root = bp = buf_splay(lblkno, 0, bp);
1633 		if (bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1634 			return (bp);
1635 	}
1636 	if ((bp = bo->bo_dirty.bv_root) != NULL) {
1637 		bo->bo_dirty.bv_root = bp = buf_splay(lblkno, 0, bp);
1638 		if (bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1639 			return (bp);
1640 	}
1641 	return (NULL);
1642 }
1643 
1644 /*
1645  * Associate a buffer with a vnode.
1646  */
1647 void
1648 bgetvp(struct vnode *vp, struct buf *bp)
1649 {
1650 	struct bufobj *bo;
1651 
1652 	bo = &vp->v_bufobj;
1653 	ASSERT_BO_LOCKED(bo);
1654 	VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free"));
1655 
1656 	CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags);
1657 	VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp,
1658 	    ("bgetvp: bp already attached! %p", bp));
1659 
1660 	vhold(vp);
1661 	bp->b_vp = vp;
1662 	bp->b_bufobj = bo;
1663 	/*
1664 	 * Insert onto list for new vnode.
1665 	 */
1666 	buf_vlist_add(bp, bo, BX_VNCLEAN);
1667 }
1668 
1669 /*
1670  * Disassociate a buffer from a vnode.
1671  */
1672 void
1673 brelvp(struct buf *bp)
1674 {
1675 	struct bufobj *bo;
1676 	struct vnode *vp;
1677 
1678 	CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1679 	KASSERT(bp->b_vp != NULL, ("brelvp: NULL"));
1680 
1681 	/*
1682 	 * Delete from old vnode list, if on one.
1683 	 */
1684 	vp = bp->b_vp;		/* XXX */
1685 	bo = bp->b_bufobj;
1686 	BO_LOCK(bo);
1687 	if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
1688 		buf_vlist_remove(bp);
1689 	else
1690 		panic("brelvp: Buffer %p not on queue.", bp);
1691 	if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
1692 		bo->bo_flag &= ~BO_ONWORKLST;
1693 		mtx_lock(&sync_mtx);
1694 		LIST_REMOVE(bo, bo_synclist);
1695 		syncer_worklist_len--;
1696 		mtx_unlock(&sync_mtx);
1697 	}
1698 	bp->b_vp = NULL;
1699 	bp->b_bufobj = NULL;
1700 	BO_UNLOCK(bo);
1701 	vdrop(vp);
1702 }
1703 
1704 /*
1705  * Add an item to the syncer work queue.
1706  */
1707 static void
1708 vn_syncer_add_to_worklist(struct bufobj *bo, int delay)
1709 {
1710 	int slot;
1711 
1712 	ASSERT_BO_LOCKED(bo);
1713 
1714 	mtx_lock(&sync_mtx);
1715 	if (bo->bo_flag & BO_ONWORKLST)
1716 		LIST_REMOVE(bo, bo_synclist);
1717 	else {
1718 		bo->bo_flag |= BO_ONWORKLST;
1719 		syncer_worklist_len++;
1720 	}
1721 
1722 	if (delay > syncer_maxdelay - 2)
1723 		delay = syncer_maxdelay - 2;
1724 	slot = (syncer_delayno + delay) & syncer_mask;
1725 
1726 	LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist);
1727 	mtx_unlock(&sync_mtx);
1728 }
1729 
1730 static int
1731 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS)
1732 {
1733 	int error, len;
1734 
1735 	mtx_lock(&sync_mtx);
1736 	len = syncer_worklist_len - sync_vnode_count;
1737 	mtx_unlock(&sync_mtx);
1738 	error = SYSCTL_OUT(req, &len, sizeof(len));
1739 	return (error);
1740 }
1741 
1742 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, CTLTYPE_INT | CTLFLAG_RD, NULL, 0,
1743     sysctl_vfs_worklist_len, "I", "Syncer thread worklist length");
1744 
1745 static struct proc *updateproc;
1746 static void sched_sync(void);
1747 static struct kproc_desc up_kp = {
1748 	"syncer",
1749 	sched_sync,
1750 	&updateproc
1751 };
1752 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp);
1753 
1754 static int
1755 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td)
1756 {
1757 	struct vnode *vp;
1758 	struct mount *mp;
1759 
1760 	*bo = LIST_FIRST(slp);
1761 	if (*bo == NULL)
1762 		return (0);
1763 	vp = (*bo)->__bo_vnode;	/* XXX */
1764 	if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0)
1765 		return (1);
1766 	/*
1767 	 * We use vhold in case the vnode does not
1768 	 * successfully sync.  vhold prevents the vnode from
1769 	 * going away when we unlock the sync_mtx so that
1770 	 * we can acquire the vnode interlock.
1771 	 */
1772 	vholdl(vp);
1773 	mtx_unlock(&sync_mtx);
1774 	VI_UNLOCK(vp);
1775 	if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
1776 		vdrop(vp);
1777 		mtx_lock(&sync_mtx);
1778 		return (*bo == LIST_FIRST(slp));
1779 	}
1780 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1781 	(void) VOP_FSYNC(vp, MNT_LAZY, td);
1782 	VOP_UNLOCK(vp, 0);
1783 	vn_finished_write(mp);
1784 	BO_LOCK(*bo);
1785 	if (((*bo)->bo_flag & BO_ONWORKLST) != 0) {
1786 		/*
1787 		 * Put us back on the worklist.  The worklist
1788 		 * routine will remove us from our current
1789 		 * position and then add us back in at a later
1790 		 * position.
1791 		 */
1792 		vn_syncer_add_to_worklist(*bo, syncdelay);
1793 	}
1794 	BO_UNLOCK(*bo);
1795 	vdrop(vp);
1796 	mtx_lock(&sync_mtx);
1797 	return (0);
1798 }
1799 
1800 /*
1801  * System filesystem synchronizer daemon.
1802  */
1803 static void
1804 sched_sync(void)
1805 {
1806 	struct synclist *next, *slp;
1807 	struct bufobj *bo;
1808 	long starttime;
1809 	struct thread *td = curthread;
1810 	int last_work_seen;
1811 	int net_worklist_len;
1812 	int syncer_final_iter;
1813 	int first_printf;
1814 	int error;
1815 
1816 	last_work_seen = 0;
1817 	syncer_final_iter = 0;
1818 	first_printf = 1;
1819 	syncer_state = SYNCER_RUNNING;
1820 	starttime = time_uptime;
1821 	td->td_pflags |= TDP_NORUNNINGBUF;
1822 
1823 	EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc,
1824 	    SHUTDOWN_PRI_LAST);
1825 
1826 	mtx_lock(&sync_mtx);
1827 	for (;;) {
1828 		if (syncer_state == SYNCER_FINAL_DELAY &&
1829 		    syncer_final_iter == 0) {
1830 			mtx_unlock(&sync_mtx);
1831 			kproc_suspend_check(td->td_proc);
1832 			mtx_lock(&sync_mtx);
1833 		}
1834 		net_worklist_len = syncer_worklist_len - sync_vnode_count;
1835 		if (syncer_state != SYNCER_RUNNING &&
1836 		    starttime != time_uptime) {
1837 			if (first_printf) {
1838 				printf("\nSyncing disks, vnodes remaining...");
1839 				first_printf = 0;
1840 			}
1841 			printf("%d ", net_worklist_len);
1842 		}
1843 		starttime = time_uptime;
1844 
1845 		/*
1846 		 * Push files whose dirty time has expired.  Be careful
1847 		 * of interrupt race on slp queue.
1848 		 *
1849 		 * Skip over empty worklist slots when shutting down.
1850 		 */
1851 		do {
1852 			slp = &syncer_workitem_pending[syncer_delayno];
1853 			syncer_delayno += 1;
1854 			if (syncer_delayno == syncer_maxdelay)
1855 				syncer_delayno = 0;
1856 			next = &syncer_workitem_pending[syncer_delayno];
1857 			/*
1858 			 * If the worklist has wrapped since the
1859 			 * it was emptied of all but syncer vnodes,
1860 			 * switch to the FINAL_DELAY state and run
1861 			 * for one more second.
1862 			 */
1863 			if (syncer_state == SYNCER_SHUTTING_DOWN &&
1864 			    net_worklist_len == 0 &&
1865 			    last_work_seen == syncer_delayno) {
1866 				syncer_state = SYNCER_FINAL_DELAY;
1867 				syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP;
1868 			}
1869 		} while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) &&
1870 		    syncer_worklist_len > 0);
1871 
1872 		/*
1873 		 * Keep track of the last time there was anything
1874 		 * on the worklist other than syncer vnodes.
1875 		 * Return to the SHUTTING_DOWN state if any
1876 		 * new work appears.
1877 		 */
1878 		if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING)
1879 			last_work_seen = syncer_delayno;
1880 		if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY)
1881 			syncer_state = SYNCER_SHUTTING_DOWN;
1882 		while (!LIST_EMPTY(slp)) {
1883 			error = sync_vnode(slp, &bo, td);
1884 			if (error == 1) {
1885 				LIST_REMOVE(bo, bo_synclist);
1886 				LIST_INSERT_HEAD(next, bo, bo_synclist);
1887 				continue;
1888 			}
1889 
1890 			if (first_printf == 0)
1891 				wdog_kern_pat(WD_LASTVAL);
1892 
1893 		}
1894 		if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0)
1895 			syncer_final_iter--;
1896 		/*
1897 		 * The variable rushjob allows the kernel to speed up the
1898 		 * processing of the filesystem syncer process. A rushjob
1899 		 * value of N tells the filesystem syncer to process the next
1900 		 * N seconds worth of work on its queue ASAP. Currently rushjob
1901 		 * is used by the soft update code to speed up the filesystem
1902 		 * syncer process when the incore state is getting so far
1903 		 * ahead of the disk that the kernel memory pool is being
1904 		 * threatened with exhaustion.
1905 		 */
1906 		if (rushjob > 0) {
1907 			rushjob -= 1;
1908 			continue;
1909 		}
1910 		/*
1911 		 * Just sleep for a short period of time between
1912 		 * iterations when shutting down to allow some I/O
1913 		 * to happen.
1914 		 *
1915 		 * If it has taken us less than a second to process the
1916 		 * current work, then wait. Otherwise start right over
1917 		 * again. We can still lose time if any single round
1918 		 * takes more than two seconds, but it does not really
1919 		 * matter as we are just trying to generally pace the
1920 		 * filesystem activity.
1921 		 */
1922 		if (syncer_state != SYNCER_RUNNING ||
1923 		    time_uptime == starttime) {
1924 			thread_lock(td);
1925 			sched_prio(td, PPAUSE);
1926 			thread_unlock(td);
1927 		}
1928 		if (syncer_state != SYNCER_RUNNING)
1929 			cv_timedwait(&sync_wakeup, &sync_mtx,
1930 			    hz / SYNCER_SHUTDOWN_SPEEDUP);
1931 		else if (time_uptime == starttime)
1932 			cv_timedwait(&sync_wakeup, &sync_mtx, hz);
1933 	}
1934 }
1935 
1936 /*
1937  * Request the syncer daemon to speed up its work.
1938  * We never push it to speed up more than half of its
1939  * normal turn time, otherwise it could take over the cpu.
1940  */
1941 int
1942 speedup_syncer(void)
1943 {
1944 	int ret = 0;
1945 
1946 	mtx_lock(&sync_mtx);
1947 	if (rushjob < syncdelay / 2) {
1948 		rushjob += 1;
1949 		stat_rush_requests += 1;
1950 		ret = 1;
1951 	}
1952 	mtx_unlock(&sync_mtx);
1953 	cv_broadcast(&sync_wakeup);
1954 	return (ret);
1955 }
1956 
1957 /*
1958  * Tell the syncer to speed up its work and run though its work
1959  * list several times, then tell it to shut down.
1960  */
1961 static void
1962 syncer_shutdown(void *arg, int howto)
1963 {
1964 
1965 	if (howto & RB_NOSYNC)
1966 		return;
1967 	mtx_lock(&sync_mtx);
1968 	syncer_state = SYNCER_SHUTTING_DOWN;
1969 	rushjob = 0;
1970 	mtx_unlock(&sync_mtx);
1971 	cv_broadcast(&sync_wakeup);
1972 	kproc_shutdown(arg, howto);
1973 }
1974 
1975 /*
1976  * Reassign a buffer from one vnode to another.
1977  * Used to assign file specific control information
1978  * (indirect blocks) to the vnode to which they belong.
1979  */
1980 void
1981 reassignbuf(struct buf *bp)
1982 {
1983 	struct vnode *vp;
1984 	struct bufobj *bo;
1985 	int delay;
1986 #ifdef INVARIANTS
1987 	struct bufv *bv;
1988 #endif
1989 
1990 	vp = bp->b_vp;
1991 	bo = bp->b_bufobj;
1992 	++reassignbufcalls;
1993 
1994 	CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X",
1995 	    bp, bp->b_vp, bp->b_flags);
1996 	/*
1997 	 * B_PAGING flagged buffers cannot be reassigned because their vp
1998 	 * is not fully linked in.
1999 	 */
2000 	if (bp->b_flags & B_PAGING)
2001 		panic("cannot reassign paging buffer");
2002 
2003 	/*
2004 	 * Delete from old vnode list, if on one.
2005 	 */
2006 	BO_LOCK(bo);
2007 	if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
2008 		buf_vlist_remove(bp);
2009 	else
2010 		panic("reassignbuf: Buffer %p not on queue.", bp);
2011 	/*
2012 	 * If dirty, put on list of dirty buffers; otherwise insert onto list
2013 	 * of clean buffers.
2014 	 */
2015 	if (bp->b_flags & B_DELWRI) {
2016 		if ((bo->bo_flag & BO_ONWORKLST) == 0) {
2017 			switch (vp->v_type) {
2018 			case VDIR:
2019 				delay = dirdelay;
2020 				break;
2021 			case VCHR:
2022 				delay = metadelay;
2023 				break;
2024 			default:
2025 				delay = filedelay;
2026 			}
2027 			vn_syncer_add_to_worklist(bo, delay);
2028 		}
2029 		buf_vlist_add(bp, bo, BX_VNDIRTY);
2030 	} else {
2031 		buf_vlist_add(bp, bo, BX_VNCLEAN);
2032 
2033 		if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
2034 			mtx_lock(&sync_mtx);
2035 			LIST_REMOVE(bo, bo_synclist);
2036 			syncer_worklist_len--;
2037 			mtx_unlock(&sync_mtx);
2038 			bo->bo_flag &= ~BO_ONWORKLST;
2039 		}
2040 	}
2041 #ifdef INVARIANTS
2042 	bv = &bo->bo_clean;
2043 	bp = TAILQ_FIRST(&bv->bv_hd);
2044 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2045 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2046 	bp = TAILQ_LAST(&bv->bv_hd, buflists);
2047 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2048 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2049 	bv = &bo->bo_dirty;
2050 	bp = TAILQ_FIRST(&bv->bv_hd);
2051 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2052 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2053 	bp = TAILQ_LAST(&bv->bv_hd, buflists);
2054 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2055 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2056 #endif
2057 	BO_UNLOCK(bo);
2058 }
2059 
2060 /*
2061  * Increment the use and hold counts on the vnode, taking care to reference
2062  * the driver's usecount if this is a chardev.  The vholdl() will remove
2063  * the vnode from the free list if it is presently free.  Requires the
2064  * vnode interlock and returns with it held.
2065  */
2066 static void
2067 v_incr_usecount(struct vnode *vp)
2068 {
2069 
2070 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2071 	vp->v_usecount++;
2072 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2073 		dev_lock();
2074 		vp->v_rdev->si_usecount++;
2075 		dev_unlock();
2076 	}
2077 	vholdl(vp);
2078 }
2079 
2080 /*
2081  * Turn a holdcnt into a use+holdcnt such that only one call to
2082  * v_decr_usecount is needed.
2083  */
2084 static void
2085 v_upgrade_usecount(struct vnode *vp)
2086 {
2087 
2088 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2089 	vp->v_usecount++;
2090 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2091 		dev_lock();
2092 		vp->v_rdev->si_usecount++;
2093 		dev_unlock();
2094 	}
2095 }
2096 
2097 /*
2098  * Decrement the vnode use and hold count along with the driver's usecount
2099  * if this is a chardev.  The vdropl() below releases the vnode interlock
2100  * as it may free the vnode.
2101  */
2102 static void
2103 v_decr_usecount(struct vnode *vp)
2104 {
2105 
2106 	ASSERT_VI_LOCKED(vp, __FUNCTION__);
2107 	VNASSERT(vp->v_usecount > 0, vp,
2108 	    ("v_decr_usecount: negative usecount"));
2109 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2110 	vp->v_usecount--;
2111 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2112 		dev_lock();
2113 		vp->v_rdev->si_usecount--;
2114 		dev_unlock();
2115 	}
2116 	vdropl(vp);
2117 }
2118 
2119 /*
2120  * Decrement only the use count and driver use count.  This is intended to
2121  * be paired with a follow on vdropl() to release the remaining hold count.
2122  * In this way we may vgone() a vnode with a 0 usecount without risk of
2123  * having it end up on a free list because the hold count is kept above 0.
2124  */
2125 static void
2126 v_decr_useonly(struct vnode *vp)
2127 {
2128 
2129 	ASSERT_VI_LOCKED(vp, __FUNCTION__);
2130 	VNASSERT(vp->v_usecount > 0, vp,
2131 	    ("v_decr_useonly: negative usecount"));
2132 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2133 	vp->v_usecount--;
2134 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2135 		dev_lock();
2136 		vp->v_rdev->si_usecount--;
2137 		dev_unlock();
2138 	}
2139 }
2140 
2141 /*
2142  * Grab a particular vnode from the free list, increment its
2143  * reference count and lock it.  VI_DOOMED is set if the vnode
2144  * is being destroyed.  Only callers who specify LK_RETRY will
2145  * see doomed vnodes.  If inactive processing was delayed in
2146  * vput try to do it here.
2147  */
2148 int
2149 vget(struct vnode *vp, int flags, struct thread *td)
2150 {
2151 	int error;
2152 
2153 	error = 0;
2154 	VNASSERT((flags & LK_TYPE_MASK) != 0, vp,
2155 	    ("vget: invalid lock operation"));
2156 	CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
2157 
2158 	if ((flags & LK_INTERLOCK) == 0)
2159 		VI_LOCK(vp);
2160 	vholdl(vp);
2161 	if ((error = vn_lock(vp, flags | LK_INTERLOCK)) != 0) {
2162 		vdrop(vp);
2163 		CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__,
2164 		    vp);
2165 		return (error);
2166 	}
2167 	if (vp->v_iflag & VI_DOOMED && (flags & LK_RETRY) == 0)
2168 		panic("vget: vn_lock failed to return ENOENT\n");
2169 	VI_LOCK(vp);
2170 	/* Upgrade our holdcnt to a usecount. */
2171 	v_upgrade_usecount(vp);
2172 	/*
2173 	 * We don't guarantee that any particular close will
2174 	 * trigger inactive processing so just make a best effort
2175 	 * here at preventing a reference to a removed file.  If
2176 	 * we don't succeed no harm is done.
2177 	 */
2178 	if (vp->v_iflag & VI_OWEINACT) {
2179 		if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE &&
2180 		    (flags & LK_NOWAIT) == 0)
2181 			vinactive(vp, td);
2182 		vp->v_iflag &= ~VI_OWEINACT;
2183 	}
2184 	VI_UNLOCK(vp);
2185 	return (0);
2186 }
2187 
2188 /*
2189  * Increase the reference count of a vnode.
2190  */
2191 void
2192 vref(struct vnode *vp)
2193 {
2194 
2195 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2196 	VI_LOCK(vp);
2197 	v_incr_usecount(vp);
2198 	VI_UNLOCK(vp);
2199 }
2200 
2201 /*
2202  * Return reference count of a vnode.
2203  *
2204  * The results of this call are only guaranteed when some mechanism other
2205  * than the VI lock is used to stop other processes from gaining references
2206  * to the vnode.  This may be the case if the caller holds the only reference.
2207  * This is also useful when stale data is acceptable as race conditions may
2208  * be accounted for by some other means.
2209  */
2210 int
2211 vrefcnt(struct vnode *vp)
2212 {
2213 	int usecnt;
2214 
2215 	VI_LOCK(vp);
2216 	usecnt = vp->v_usecount;
2217 	VI_UNLOCK(vp);
2218 
2219 	return (usecnt);
2220 }
2221 
2222 #define	VPUTX_VRELE	1
2223 #define	VPUTX_VPUT	2
2224 #define	VPUTX_VUNREF	3
2225 
2226 static void
2227 vputx(struct vnode *vp, int func)
2228 {
2229 	int error;
2230 
2231 	KASSERT(vp != NULL, ("vputx: null vp"));
2232 	if (func == VPUTX_VUNREF)
2233 		ASSERT_VOP_LOCKED(vp, "vunref");
2234 	else if (func == VPUTX_VPUT)
2235 		ASSERT_VOP_LOCKED(vp, "vput");
2236 	else
2237 		KASSERT(func == VPUTX_VRELE, ("vputx: wrong func"));
2238 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2239 	VI_LOCK(vp);
2240 
2241 	/* Skip this v_writecount check if we're going to panic below. */
2242 	VNASSERT(vp->v_writecount < vp->v_usecount || vp->v_usecount < 1, vp,
2243 	    ("vputx: missed vn_close"));
2244 	error = 0;
2245 
2246 	if (vp->v_usecount > 1 || ((vp->v_iflag & VI_DOINGINACT) &&
2247 	    vp->v_usecount == 1)) {
2248 		if (func == VPUTX_VPUT)
2249 			VOP_UNLOCK(vp, 0);
2250 		v_decr_usecount(vp);
2251 		return;
2252 	}
2253 
2254 	if (vp->v_usecount != 1) {
2255 		vprint("vputx: negative ref count", vp);
2256 		panic("vputx: negative ref cnt");
2257 	}
2258 	CTR2(KTR_VFS, "%s: return vnode %p to the freelist", __func__, vp);
2259 	/*
2260 	 * We want to hold the vnode until the inactive finishes to
2261 	 * prevent vgone() races.  We drop the use count here and the
2262 	 * hold count below when we're done.
2263 	 */
2264 	v_decr_useonly(vp);
2265 	/*
2266 	 * We must call VOP_INACTIVE with the node locked. Mark
2267 	 * as VI_DOINGINACT to avoid recursion.
2268 	 */
2269 	vp->v_iflag |= VI_OWEINACT;
2270 	switch (func) {
2271 	case VPUTX_VRELE:
2272 		error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK);
2273 		VI_LOCK(vp);
2274 		break;
2275 	case VPUTX_VPUT:
2276 		if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
2277 			error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK |
2278 			    LK_NOWAIT);
2279 			VI_LOCK(vp);
2280 		}
2281 		break;
2282 	case VPUTX_VUNREF:
2283 		if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
2284 			error = EBUSY;
2285 		break;
2286 	}
2287 	if (vp->v_usecount > 0)
2288 		vp->v_iflag &= ~VI_OWEINACT;
2289 	if (error == 0) {
2290 		if (vp->v_iflag & VI_OWEINACT)
2291 			vinactive(vp, curthread);
2292 		if (func != VPUTX_VUNREF)
2293 			VOP_UNLOCK(vp, 0);
2294 	}
2295 	vdropl(vp);
2296 }
2297 
2298 /*
2299  * Vnode put/release.
2300  * If count drops to zero, call inactive routine and return to freelist.
2301  */
2302 void
2303 vrele(struct vnode *vp)
2304 {
2305 
2306 	vputx(vp, VPUTX_VRELE);
2307 }
2308 
2309 /*
2310  * Release an already locked vnode.  This give the same effects as
2311  * unlock+vrele(), but takes less time and avoids releasing and
2312  * re-aquiring the lock (as vrele() acquires the lock internally.)
2313  */
2314 void
2315 vput(struct vnode *vp)
2316 {
2317 
2318 	vputx(vp, VPUTX_VPUT);
2319 }
2320 
2321 /*
2322  * Release an exclusively locked vnode. Do not unlock the vnode lock.
2323  */
2324 void
2325 vunref(struct vnode *vp)
2326 {
2327 
2328 	vputx(vp, VPUTX_VUNREF);
2329 }
2330 
2331 /*
2332  * Somebody doesn't want the vnode recycled.
2333  */
2334 void
2335 vhold(struct vnode *vp)
2336 {
2337 
2338 	VI_LOCK(vp);
2339 	vholdl(vp);
2340 	VI_UNLOCK(vp);
2341 }
2342 
2343 /*
2344  * Increase the hold count and activate if this is the first reference.
2345  */
2346 void
2347 vholdl(struct vnode *vp)
2348 {
2349 	struct mount *mp;
2350 
2351 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2352 	vp->v_holdcnt++;
2353 	if (!VSHOULDBUSY(vp))
2354 		return;
2355 	ASSERT_VI_LOCKED(vp, "vholdl");
2356 	VNASSERT((vp->v_iflag & VI_FREE) != 0, vp, ("vnode not free"));
2357 	VNASSERT(vp->v_op != NULL, vp, ("vholdl: vnode already reclaimed."));
2358 	/*
2359 	 * Remove a vnode from the free list, mark it as in use,
2360 	 * and put it on the active list.
2361 	 */
2362 	mtx_lock(&vnode_free_list_mtx);
2363 	TAILQ_REMOVE(&vnode_free_list, vp, v_actfreelist);
2364 	freevnodes--;
2365 	vp->v_iflag &= ~(VI_FREE|VI_AGE);
2366 	KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
2367 	    ("Activating already active vnode"));
2368 	vp->v_iflag |= VI_ACTIVE;
2369 	mp = vp->v_mount;
2370 	TAILQ_INSERT_HEAD(&mp->mnt_activevnodelist, vp, v_actfreelist);
2371 	mp->mnt_activevnodelistsize++;
2372 	mtx_unlock(&vnode_free_list_mtx);
2373 }
2374 
2375 /*
2376  * Note that there is one less who cares about this vnode.
2377  * vdrop() is the opposite of vhold().
2378  */
2379 void
2380 vdrop(struct vnode *vp)
2381 {
2382 
2383 	VI_LOCK(vp);
2384 	vdropl(vp);
2385 }
2386 
2387 /*
2388  * Drop the hold count of the vnode.  If this is the last reference to
2389  * the vnode we place it on the free list unless it has been vgone'd
2390  * (marked VI_DOOMED) in which case we will free it.
2391  */
2392 void
2393 vdropl(struct vnode *vp)
2394 {
2395 	struct bufobj *bo;
2396 	struct mount *mp;
2397 	int active;
2398 
2399 	ASSERT_VI_LOCKED(vp, "vdropl");
2400 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2401 	if (vp->v_holdcnt <= 0)
2402 		panic("vdrop: holdcnt %d", vp->v_holdcnt);
2403 	vp->v_holdcnt--;
2404 	if (vp->v_holdcnt > 0) {
2405 		VI_UNLOCK(vp);
2406 		return;
2407 	}
2408 	if ((vp->v_iflag & VI_DOOMED) == 0) {
2409 		/*
2410 		 * Mark a vnode as free: remove it from its active list
2411 		 * and put it up for recycling on the freelist.
2412 		 */
2413 		VNASSERT(vp->v_op != NULL, vp,
2414 		    ("vdropl: vnode already reclaimed."));
2415 		VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
2416 		    ("vnode already free"));
2417 		VNASSERT(VSHOULDFREE(vp), vp,
2418 		    ("vdropl: freeing when we shouldn't"));
2419 		active = vp->v_iflag & VI_ACTIVE;
2420 		vp->v_iflag &= ~VI_ACTIVE;
2421 		mp = vp->v_mount;
2422 		mtx_lock(&vnode_free_list_mtx);
2423 		if (active) {
2424 			TAILQ_REMOVE(&mp->mnt_activevnodelist, vp,
2425 			    v_actfreelist);
2426 			mp->mnt_activevnodelistsize--;
2427 		}
2428 		if (vp->v_iflag & VI_AGE) {
2429 			TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_actfreelist);
2430 		} else {
2431 			TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_actfreelist);
2432 		}
2433 		freevnodes++;
2434 		vp->v_iflag &= ~VI_AGE;
2435 		vp->v_iflag |= VI_FREE;
2436 		mtx_unlock(&vnode_free_list_mtx);
2437 		VI_UNLOCK(vp);
2438 		return;
2439 	}
2440 	/*
2441 	 * The vnode has been marked for destruction, so free it.
2442 	 */
2443 	CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp);
2444 	mtx_lock(&vnode_free_list_mtx);
2445 	numvnodes--;
2446 	mtx_unlock(&vnode_free_list_mtx);
2447 	bo = &vp->v_bufobj;
2448 	VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
2449 	    ("cleaned vnode still on the free list."));
2450 	VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't"));
2451 	VNASSERT(vp->v_holdcnt == 0, vp, ("Non-zero hold count"));
2452 	VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count"));
2453 	VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count"));
2454 	VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's"));
2455 	VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0"));
2456 	VNASSERT(bo->bo_clean.bv_root == NULL, vp, ("cleanblkroot not NULL"));
2457 	VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0"));
2458 	VNASSERT(bo->bo_dirty.bv_root == NULL, vp, ("dirtyblkroot not NULL"));
2459 	VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst"));
2460 	VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src"));
2461 	VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for .."));
2462 	VI_UNLOCK(vp);
2463 #ifdef MAC
2464 	mac_vnode_destroy(vp);
2465 #endif
2466 	if (vp->v_pollinfo != NULL)
2467 		destroy_vpollinfo(vp->v_pollinfo);
2468 #ifdef INVARIANTS
2469 	/* XXX Elsewhere we detect an already freed vnode via NULL v_op. */
2470 	vp->v_op = NULL;
2471 #endif
2472 	rangelock_destroy(&vp->v_rl);
2473 	lockdestroy(vp->v_vnlock);
2474 	mtx_destroy(&vp->v_interlock);
2475 	mtx_destroy(BO_MTX(bo));
2476 	uma_zfree(vnode_zone, vp);
2477 }
2478 
2479 /*
2480  * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT
2481  * flags.  DOINGINACT prevents us from recursing in calls to vinactive.
2482  * OWEINACT tracks whether a vnode missed a call to inactive due to a
2483  * failed lock upgrade.
2484  */
2485 void
2486 vinactive(struct vnode *vp, struct thread *td)
2487 {
2488 	struct vm_object *obj;
2489 
2490 	ASSERT_VOP_ELOCKED(vp, "vinactive");
2491 	ASSERT_VI_LOCKED(vp, "vinactive");
2492 	VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp,
2493 	    ("vinactive: recursed on VI_DOINGINACT"));
2494 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2495 	vp->v_iflag |= VI_DOINGINACT;
2496 	vp->v_iflag &= ~VI_OWEINACT;
2497 	VI_UNLOCK(vp);
2498 	/*
2499 	 * Before moving off the active list, we must be sure that any
2500 	 * modified pages are on the vnode's dirty list since these will
2501 	 * no longer be checked once the vnode is on the inactive list.
2502 	 * Because the vnode vm object keeps a hold reference on the vnode
2503 	 * if there is at least one resident non-cached page, the vnode
2504 	 * cannot leave the active list without the page cleanup done.
2505 	 */
2506 	obj = vp->v_object;
2507 	if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0) {
2508 		VM_OBJECT_LOCK(obj);
2509 		vm_object_page_clean(obj, 0, 0, OBJPC_NOSYNC);
2510 		VM_OBJECT_UNLOCK(obj);
2511 	}
2512 	VOP_INACTIVE(vp, td);
2513 	VI_LOCK(vp);
2514 	VNASSERT(vp->v_iflag & VI_DOINGINACT, vp,
2515 	    ("vinactive: lost VI_DOINGINACT"));
2516 	vp->v_iflag &= ~VI_DOINGINACT;
2517 }
2518 
2519 /*
2520  * Remove any vnodes in the vnode table belonging to mount point mp.
2521  *
2522  * If FORCECLOSE is not specified, there should not be any active ones,
2523  * return error if any are found (nb: this is a user error, not a
2524  * system error). If FORCECLOSE is specified, detach any active vnodes
2525  * that are found.
2526  *
2527  * If WRITECLOSE is set, only flush out regular file vnodes open for
2528  * writing.
2529  *
2530  * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped.
2531  *
2532  * `rootrefs' specifies the base reference count for the root vnode
2533  * of this filesystem. The root vnode is considered busy if its
2534  * v_usecount exceeds this value. On a successful return, vflush(, td)
2535  * will call vrele() on the root vnode exactly rootrefs times.
2536  * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must
2537  * be zero.
2538  */
2539 #ifdef DIAGNOSTIC
2540 static int busyprt = 0;		/* print out busy vnodes */
2541 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes");
2542 #endif
2543 
2544 int
2545 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td)
2546 {
2547 	struct vnode *vp, *mvp, *rootvp = NULL;
2548 	struct vattr vattr;
2549 	int busy = 0, error;
2550 
2551 	CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp,
2552 	    rootrefs, flags);
2553 	if (rootrefs > 0) {
2554 		KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0,
2555 		    ("vflush: bad args"));
2556 		/*
2557 		 * Get the filesystem root vnode. We can vput() it
2558 		 * immediately, since with rootrefs > 0, it won't go away.
2559 		 */
2560 		if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) {
2561 			CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d",
2562 			    __func__, error);
2563 			return (error);
2564 		}
2565 		vput(rootvp);
2566 	}
2567 loop:
2568 	MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
2569 		vholdl(vp);
2570 		error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE);
2571 		if (error) {
2572 			vdrop(vp);
2573 			MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
2574 			goto loop;
2575 		}
2576 		/*
2577 		 * Skip over a vnodes marked VV_SYSTEM.
2578 		 */
2579 		if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) {
2580 			VOP_UNLOCK(vp, 0);
2581 			vdrop(vp);
2582 			continue;
2583 		}
2584 		/*
2585 		 * If WRITECLOSE is set, flush out unlinked but still open
2586 		 * files (even if open only for reading) and regular file
2587 		 * vnodes open for writing.
2588 		 */
2589 		if (flags & WRITECLOSE) {
2590 			if (vp->v_object != NULL) {
2591 				VM_OBJECT_LOCK(vp->v_object);
2592 				vm_object_page_clean(vp->v_object, 0, 0, 0);
2593 				VM_OBJECT_UNLOCK(vp->v_object);
2594 			}
2595 			error = VOP_FSYNC(vp, MNT_WAIT, td);
2596 			if (error != 0) {
2597 				VOP_UNLOCK(vp, 0);
2598 				vdrop(vp);
2599 				MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
2600 				return (error);
2601 			}
2602 			error = VOP_GETATTR(vp, &vattr, td->td_ucred);
2603 			VI_LOCK(vp);
2604 
2605 			if ((vp->v_type == VNON ||
2606 			    (error == 0 && vattr.va_nlink > 0)) &&
2607 			    (vp->v_writecount == 0 || vp->v_type != VREG)) {
2608 				VOP_UNLOCK(vp, 0);
2609 				vdropl(vp);
2610 				continue;
2611 			}
2612 		} else
2613 			VI_LOCK(vp);
2614 		/*
2615 		 * With v_usecount == 0, all we need to do is clear out the
2616 		 * vnode data structures and we are done.
2617 		 *
2618 		 * If FORCECLOSE is set, forcibly close the vnode.
2619 		 */
2620 		if (vp->v_usecount == 0 || (flags & FORCECLOSE)) {
2621 			VNASSERT(vp->v_usecount == 0 ||
2622 			    (vp->v_type != VCHR && vp->v_type != VBLK), vp,
2623 			    ("device VNODE %p is FORCECLOSED", vp));
2624 			vgonel(vp);
2625 		} else {
2626 			busy++;
2627 #ifdef DIAGNOSTIC
2628 			if (busyprt)
2629 				vprint("vflush: busy vnode", vp);
2630 #endif
2631 		}
2632 		VOP_UNLOCK(vp, 0);
2633 		vdropl(vp);
2634 	}
2635 	if (rootrefs > 0 && (flags & FORCECLOSE) == 0) {
2636 		/*
2637 		 * If just the root vnode is busy, and if its refcount
2638 		 * is equal to `rootrefs', then go ahead and kill it.
2639 		 */
2640 		VI_LOCK(rootvp);
2641 		KASSERT(busy > 0, ("vflush: not busy"));
2642 		VNASSERT(rootvp->v_usecount >= rootrefs, rootvp,
2643 		    ("vflush: usecount %d < rootrefs %d",
2644 		     rootvp->v_usecount, rootrefs));
2645 		if (busy == 1 && rootvp->v_usecount == rootrefs) {
2646 			VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK);
2647 			vgone(rootvp);
2648 			VOP_UNLOCK(rootvp, 0);
2649 			busy = 0;
2650 		} else
2651 			VI_UNLOCK(rootvp);
2652 	}
2653 	if (busy) {
2654 		CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__,
2655 		    busy);
2656 		return (EBUSY);
2657 	}
2658 	for (; rootrefs > 0; rootrefs--)
2659 		vrele(rootvp);
2660 	return (0);
2661 }
2662 
2663 /*
2664  * Recycle an unused vnode to the front of the free list.
2665  */
2666 int
2667 vrecycle(struct vnode *vp)
2668 {
2669 	int recycled;
2670 
2671 	ASSERT_VOP_ELOCKED(vp, "vrecycle");
2672 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2673 	recycled = 0;
2674 	VI_LOCK(vp);
2675 	if (vp->v_usecount == 0) {
2676 		recycled = 1;
2677 		vgonel(vp);
2678 	}
2679 	VI_UNLOCK(vp);
2680 	return (recycled);
2681 }
2682 
2683 /*
2684  * Eliminate all activity associated with a vnode
2685  * in preparation for reuse.
2686  */
2687 void
2688 vgone(struct vnode *vp)
2689 {
2690 	VI_LOCK(vp);
2691 	vgonel(vp);
2692 	VI_UNLOCK(vp);
2693 }
2694 
2695 static void
2696 vgonel_reclaim_lowervp_vfs(struct mount *mp __unused,
2697     struct vnode *lowervp __unused)
2698 {
2699 }
2700 
2701 /*
2702  * Notify upper mounts about reclaimed vnode.
2703  */
2704 static void
2705 vgonel_reclaim_lowervp(struct vnode *vp)
2706 {
2707 	static struct vfsops vgonel_vfsops = {
2708 		.vfs_reclaim_lowervp = vgonel_reclaim_lowervp_vfs
2709 	};
2710 	struct mount *mp, *ump, *mmp;
2711 
2712 	mp = vp->v_mount;
2713 	if (mp == NULL)
2714 		return;
2715 
2716 	MNT_ILOCK(mp);
2717 	if (TAILQ_EMPTY(&mp->mnt_uppers))
2718 		goto unlock;
2719 	MNT_IUNLOCK(mp);
2720 	mmp = malloc(sizeof(struct mount), M_TEMP, M_WAITOK | M_ZERO);
2721 	mmp->mnt_op = &vgonel_vfsops;
2722 	mmp->mnt_kern_flag |= MNTK_MARKER;
2723 	MNT_ILOCK(mp);
2724 	mp->mnt_kern_flag |= MNTK_VGONE_UPPER;
2725 	for (ump = TAILQ_FIRST(&mp->mnt_uppers); ump != NULL;) {
2726 		if ((ump->mnt_kern_flag & MNTK_MARKER) != 0) {
2727 			ump = TAILQ_NEXT(ump, mnt_upper_link);
2728 			continue;
2729 		}
2730 		TAILQ_INSERT_AFTER(&mp->mnt_uppers, ump, mmp, mnt_upper_link);
2731 		MNT_IUNLOCK(mp);
2732 		VFS_RECLAIM_LOWERVP(ump, vp);
2733 		MNT_ILOCK(mp);
2734 		ump = TAILQ_NEXT(mmp, mnt_upper_link);
2735 		TAILQ_REMOVE(&mp->mnt_uppers, mmp, mnt_upper_link);
2736 	}
2737 	free(mmp, M_TEMP);
2738 	mp->mnt_kern_flag &= ~MNTK_VGONE_UPPER;
2739 	if ((mp->mnt_kern_flag & MNTK_VGONE_WAITER) != 0) {
2740 		mp->mnt_kern_flag &= ~MNTK_VGONE_WAITER;
2741 		wakeup(&mp->mnt_uppers);
2742 	}
2743 unlock:
2744 	MNT_IUNLOCK(mp);
2745 }
2746 
2747 /*
2748  * vgone, with the vp interlock held.
2749  */
2750 void
2751 vgonel(struct vnode *vp)
2752 {
2753 	struct thread *td;
2754 	int oweinact;
2755 	int active;
2756 	struct mount *mp;
2757 
2758 	ASSERT_VOP_ELOCKED(vp, "vgonel");
2759 	ASSERT_VI_LOCKED(vp, "vgonel");
2760 	VNASSERT(vp->v_holdcnt, vp,
2761 	    ("vgonel: vp %p has no reference.", vp));
2762 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2763 	td = curthread;
2764 
2765 	/*
2766 	 * Don't vgonel if we're already doomed.
2767 	 */
2768 	if (vp->v_iflag & VI_DOOMED)
2769 		return;
2770 	vp->v_iflag |= VI_DOOMED;
2771 
2772 	/*
2773 	 * Check to see if the vnode is in use.  If so, we have to call
2774 	 * VOP_CLOSE() and VOP_INACTIVE().
2775 	 */
2776 	active = vp->v_usecount;
2777 	oweinact = (vp->v_iflag & VI_OWEINACT);
2778 	VI_UNLOCK(vp);
2779 	vgonel_reclaim_lowervp(vp);
2780 
2781 	/*
2782 	 * Clean out any buffers associated with the vnode.
2783 	 * If the flush fails, just toss the buffers.
2784 	 */
2785 	mp = NULL;
2786 	if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd))
2787 		(void) vn_start_secondary_write(vp, &mp, V_WAIT);
2788 	if (vinvalbuf(vp, V_SAVE, 0, 0) != 0)
2789 		vinvalbuf(vp, 0, 0, 0);
2790 
2791 	/*
2792 	 * If purging an active vnode, it must be closed and
2793 	 * deactivated before being reclaimed.
2794 	 */
2795 	if (active)
2796 		VOP_CLOSE(vp, FNONBLOCK, NOCRED, td);
2797 	if (oweinact || active) {
2798 		VI_LOCK(vp);
2799 		if ((vp->v_iflag & VI_DOINGINACT) == 0)
2800 			vinactive(vp, td);
2801 		VI_UNLOCK(vp);
2802 	}
2803 	if (vp->v_type == VSOCK)
2804 		vfs_unp_reclaim(vp);
2805 	/*
2806 	 * Reclaim the vnode.
2807 	 */
2808 	if (VOP_RECLAIM(vp, td))
2809 		panic("vgone: cannot reclaim");
2810 	if (mp != NULL)
2811 		vn_finished_secondary_write(mp);
2812 	VNASSERT(vp->v_object == NULL, vp,
2813 	    ("vop_reclaim left v_object vp=%p, tag=%s", vp, vp->v_tag));
2814 	/*
2815 	 * Clear the advisory locks and wake up waiting threads.
2816 	 */
2817 	(void)VOP_ADVLOCKPURGE(vp);
2818 	/*
2819 	 * Delete from old mount point vnode list.
2820 	 */
2821 	delmntque(vp);
2822 	cache_purge(vp);
2823 	/*
2824 	 * Done with purge, reset to the standard lock and invalidate
2825 	 * the vnode.
2826 	 */
2827 	VI_LOCK(vp);
2828 	vp->v_vnlock = &vp->v_lock;
2829 	vp->v_op = &dead_vnodeops;
2830 	vp->v_tag = "none";
2831 	vp->v_type = VBAD;
2832 }
2833 
2834 /*
2835  * Calculate the total number of references to a special device.
2836  */
2837 int
2838 vcount(struct vnode *vp)
2839 {
2840 	int count;
2841 
2842 	dev_lock();
2843 	count = vp->v_rdev->si_usecount;
2844 	dev_unlock();
2845 	return (count);
2846 }
2847 
2848 /*
2849  * Same as above, but using the struct cdev *as argument
2850  */
2851 int
2852 count_dev(struct cdev *dev)
2853 {
2854 	int count;
2855 
2856 	dev_lock();
2857 	count = dev->si_usecount;
2858 	dev_unlock();
2859 	return(count);
2860 }
2861 
2862 /*
2863  * Print out a description of a vnode.
2864  */
2865 static char *typename[] =
2866 {"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD",
2867  "VMARKER"};
2868 
2869 void
2870 vn_printf(struct vnode *vp, const char *fmt, ...)
2871 {
2872 	va_list ap;
2873 	char buf[256], buf2[16];
2874 	u_long flags;
2875 
2876 	va_start(ap, fmt);
2877 	vprintf(fmt, ap);
2878 	va_end(ap);
2879 	printf("%p: ", (void *)vp);
2880 	printf("tag %s, type %s\n", vp->v_tag, typename[vp->v_type]);
2881 	printf("    usecount %d, writecount %d, refcount %d mountedhere %p\n",
2882 	    vp->v_usecount, vp->v_writecount, vp->v_holdcnt, vp->v_mountedhere);
2883 	buf[0] = '\0';
2884 	buf[1] = '\0';
2885 	if (vp->v_vflag & VV_ROOT)
2886 		strlcat(buf, "|VV_ROOT", sizeof(buf));
2887 	if (vp->v_vflag & VV_ISTTY)
2888 		strlcat(buf, "|VV_ISTTY", sizeof(buf));
2889 	if (vp->v_vflag & VV_NOSYNC)
2890 		strlcat(buf, "|VV_NOSYNC", sizeof(buf));
2891 	if (vp->v_vflag & VV_ETERNALDEV)
2892 		strlcat(buf, "|VV_ETERNALDEV", sizeof(buf));
2893 	if (vp->v_vflag & VV_CACHEDLABEL)
2894 		strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf));
2895 	if (vp->v_vflag & VV_TEXT)
2896 		strlcat(buf, "|VV_TEXT", sizeof(buf));
2897 	if (vp->v_vflag & VV_COPYONWRITE)
2898 		strlcat(buf, "|VV_COPYONWRITE", sizeof(buf));
2899 	if (vp->v_vflag & VV_SYSTEM)
2900 		strlcat(buf, "|VV_SYSTEM", sizeof(buf));
2901 	if (vp->v_vflag & VV_PROCDEP)
2902 		strlcat(buf, "|VV_PROCDEP", sizeof(buf));
2903 	if (vp->v_vflag & VV_NOKNOTE)
2904 		strlcat(buf, "|VV_NOKNOTE", sizeof(buf));
2905 	if (vp->v_vflag & VV_DELETED)
2906 		strlcat(buf, "|VV_DELETED", sizeof(buf));
2907 	if (vp->v_vflag & VV_MD)
2908 		strlcat(buf, "|VV_MD", sizeof(buf));
2909 	if (vp->v_vflag & VV_FORCEINSMQ)
2910 		strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf));
2911 	flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV |
2912 	    VV_CACHEDLABEL | VV_TEXT | VV_COPYONWRITE | VV_SYSTEM | VV_PROCDEP |
2913 	    VV_NOKNOTE | VV_DELETED | VV_MD | VV_FORCEINSMQ);
2914 	if (flags != 0) {
2915 		snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags);
2916 		strlcat(buf, buf2, sizeof(buf));
2917 	}
2918 	if (vp->v_iflag & VI_MOUNT)
2919 		strlcat(buf, "|VI_MOUNT", sizeof(buf));
2920 	if (vp->v_iflag & VI_AGE)
2921 		strlcat(buf, "|VI_AGE", sizeof(buf));
2922 	if (vp->v_iflag & VI_DOOMED)
2923 		strlcat(buf, "|VI_DOOMED", sizeof(buf));
2924 	if (vp->v_iflag & VI_FREE)
2925 		strlcat(buf, "|VI_FREE", sizeof(buf));
2926 	if (vp->v_iflag & VI_ACTIVE)
2927 		strlcat(buf, "|VI_ACTIVE", sizeof(buf));
2928 	if (vp->v_iflag & VI_DOINGINACT)
2929 		strlcat(buf, "|VI_DOINGINACT", sizeof(buf));
2930 	if (vp->v_iflag & VI_OWEINACT)
2931 		strlcat(buf, "|VI_OWEINACT", sizeof(buf));
2932 	flags = vp->v_iflag & ~(VI_MOUNT | VI_AGE | VI_DOOMED | VI_FREE |
2933 	    VI_ACTIVE | VI_DOINGINACT | VI_OWEINACT);
2934 	if (flags != 0) {
2935 		snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags);
2936 		strlcat(buf, buf2, sizeof(buf));
2937 	}
2938 	printf("    flags (%s)\n", buf + 1);
2939 	if (mtx_owned(VI_MTX(vp)))
2940 		printf(" VI_LOCKed");
2941 	if (vp->v_object != NULL)
2942 		printf("    v_object %p ref %d pages %d\n",
2943 		    vp->v_object, vp->v_object->ref_count,
2944 		    vp->v_object->resident_page_count);
2945 	printf("    ");
2946 	lockmgr_printinfo(vp->v_vnlock);
2947 	if (vp->v_data != NULL)
2948 		VOP_PRINT(vp);
2949 }
2950 
2951 #ifdef DDB
2952 /*
2953  * List all of the locked vnodes in the system.
2954  * Called when debugging the kernel.
2955  */
2956 DB_SHOW_COMMAND(lockedvnods, lockedvnodes)
2957 {
2958 	struct mount *mp, *nmp;
2959 	struct vnode *vp;
2960 
2961 	/*
2962 	 * Note: because this is DDB, we can't obey the locking semantics
2963 	 * for these structures, which means we could catch an inconsistent
2964 	 * state and dereference a nasty pointer.  Not much to be done
2965 	 * about that.
2966 	 */
2967 	db_printf("Locked vnodes\n");
2968 	for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
2969 		nmp = TAILQ_NEXT(mp, mnt_list);
2970 		TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
2971 			if (vp->v_type != VMARKER &&
2972 			    VOP_ISLOCKED(vp))
2973 				vprint("", vp);
2974 		}
2975 		nmp = TAILQ_NEXT(mp, mnt_list);
2976 	}
2977 }
2978 
2979 /*
2980  * Show details about the given vnode.
2981  */
2982 DB_SHOW_COMMAND(vnode, db_show_vnode)
2983 {
2984 	struct vnode *vp;
2985 
2986 	if (!have_addr)
2987 		return;
2988 	vp = (struct vnode *)addr;
2989 	vn_printf(vp, "vnode ");
2990 }
2991 
2992 /*
2993  * Show details about the given mount point.
2994  */
2995 DB_SHOW_COMMAND(mount, db_show_mount)
2996 {
2997 	struct mount *mp;
2998 	struct vfsopt *opt;
2999 	struct statfs *sp;
3000 	struct vnode *vp;
3001 	char buf[512];
3002 	uint64_t mflags;
3003 	u_int flags;
3004 
3005 	if (!have_addr) {
3006 		/* No address given, print short info about all mount points. */
3007 		TAILQ_FOREACH(mp, &mountlist, mnt_list) {
3008 			db_printf("%p %s on %s (%s)\n", mp,
3009 			    mp->mnt_stat.f_mntfromname,
3010 			    mp->mnt_stat.f_mntonname,
3011 			    mp->mnt_stat.f_fstypename);
3012 			if (db_pager_quit)
3013 				break;
3014 		}
3015 		db_printf("\nMore info: show mount <addr>\n");
3016 		return;
3017 	}
3018 
3019 	mp = (struct mount *)addr;
3020 	db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname,
3021 	    mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename);
3022 
3023 	buf[0] = '\0';
3024 	mflags = mp->mnt_flag;
3025 #define	MNT_FLAG(flag)	do {						\
3026 	if (mflags & (flag)) {						\
3027 		if (buf[0] != '\0')					\
3028 			strlcat(buf, ", ", sizeof(buf));		\
3029 		strlcat(buf, (#flag) + 4, sizeof(buf));			\
3030 		mflags &= ~(flag);					\
3031 	}								\
3032 } while (0)
3033 	MNT_FLAG(MNT_RDONLY);
3034 	MNT_FLAG(MNT_SYNCHRONOUS);
3035 	MNT_FLAG(MNT_NOEXEC);
3036 	MNT_FLAG(MNT_NOSUID);
3037 	MNT_FLAG(MNT_NFS4ACLS);
3038 	MNT_FLAG(MNT_UNION);
3039 	MNT_FLAG(MNT_ASYNC);
3040 	MNT_FLAG(MNT_SUIDDIR);
3041 	MNT_FLAG(MNT_SOFTDEP);
3042 	MNT_FLAG(MNT_NOSYMFOLLOW);
3043 	MNT_FLAG(MNT_GJOURNAL);
3044 	MNT_FLAG(MNT_MULTILABEL);
3045 	MNT_FLAG(MNT_ACLS);
3046 	MNT_FLAG(MNT_NOATIME);
3047 	MNT_FLAG(MNT_NOCLUSTERR);
3048 	MNT_FLAG(MNT_NOCLUSTERW);
3049 	MNT_FLAG(MNT_SUJ);
3050 	MNT_FLAG(MNT_EXRDONLY);
3051 	MNT_FLAG(MNT_EXPORTED);
3052 	MNT_FLAG(MNT_DEFEXPORTED);
3053 	MNT_FLAG(MNT_EXPORTANON);
3054 	MNT_FLAG(MNT_EXKERB);
3055 	MNT_FLAG(MNT_EXPUBLIC);
3056 	MNT_FLAG(MNT_LOCAL);
3057 	MNT_FLAG(MNT_QUOTA);
3058 	MNT_FLAG(MNT_ROOTFS);
3059 	MNT_FLAG(MNT_USER);
3060 	MNT_FLAG(MNT_IGNORE);
3061 	MNT_FLAG(MNT_UPDATE);
3062 	MNT_FLAG(MNT_DELEXPORT);
3063 	MNT_FLAG(MNT_RELOAD);
3064 	MNT_FLAG(MNT_FORCE);
3065 	MNT_FLAG(MNT_SNAPSHOT);
3066 	MNT_FLAG(MNT_BYFSID);
3067 #undef MNT_FLAG
3068 	if (mflags != 0) {
3069 		if (buf[0] != '\0')
3070 			strlcat(buf, ", ", sizeof(buf));
3071 		snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
3072 		    "0x%016jx", mflags);
3073 	}
3074 	db_printf("    mnt_flag = %s\n", buf);
3075 
3076 	buf[0] = '\0';
3077 	flags = mp->mnt_kern_flag;
3078 #define	MNT_KERN_FLAG(flag)	do {					\
3079 	if (flags & (flag)) {						\
3080 		if (buf[0] != '\0')					\
3081 			strlcat(buf, ", ", sizeof(buf));		\
3082 		strlcat(buf, (#flag) + 5, sizeof(buf));			\
3083 		flags &= ~(flag);					\
3084 	}								\
3085 } while (0)
3086 	MNT_KERN_FLAG(MNTK_UNMOUNTF);
3087 	MNT_KERN_FLAG(MNTK_ASYNC);
3088 	MNT_KERN_FLAG(MNTK_SOFTDEP);
3089 	MNT_KERN_FLAG(MNTK_NOINSMNTQ);
3090 	MNT_KERN_FLAG(MNTK_DRAINING);
3091 	MNT_KERN_FLAG(MNTK_REFEXPIRE);
3092 	MNT_KERN_FLAG(MNTK_EXTENDED_SHARED);
3093 	MNT_KERN_FLAG(MNTK_SHARED_WRITES);
3094 	MNT_KERN_FLAG(MNTK_NO_IOPF);
3095 	MNT_KERN_FLAG(MNTK_VGONE_UPPER);
3096 	MNT_KERN_FLAG(MNTK_VGONE_WAITER);
3097 	MNT_KERN_FLAG(MNTK_LOOKUP_EXCL_DOTDOT);
3098 	MNT_KERN_FLAG(MNTK_MARKER);
3099 	MNT_KERN_FLAG(MNTK_NOASYNC);
3100 	MNT_KERN_FLAG(MNTK_UNMOUNT);
3101 	MNT_KERN_FLAG(MNTK_MWAIT);
3102 	MNT_KERN_FLAG(MNTK_SUSPEND);
3103 	MNT_KERN_FLAG(MNTK_SUSPEND2);
3104 	MNT_KERN_FLAG(MNTK_SUSPENDED);
3105 	MNT_KERN_FLAG(MNTK_LOOKUP_SHARED);
3106 	MNT_KERN_FLAG(MNTK_NOKNOTE);
3107 #undef MNT_KERN_FLAG
3108 	if (flags != 0) {
3109 		if (buf[0] != '\0')
3110 			strlcat(buf, ", ", sizeof(buf));
3111 		snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
3112 		    "0x%08x", flags);
3113 	}
3114 	db_printf("    mnt_kern_flag = %s\n", buf);
3115 
3116 	db_printf("    mnt_opt = ");
3117 	opt = TAILQ_FIRST(mp->mnt_opt);
3118 	if (opt != NULL) {
3119 		db_printf("%s", opt->name);
3120 		opt = TAILQ_NEXT(opt, link);
3121 		while (opt != NULL) {
3122 			db_printf(", %s", opt->name);
3123 			opt = TAILQ_NEXT(opt, link);
3124 		}
3125 	}
3126 	db_printf("\n");
3127 
3128 	sp = &mp->mnt_stat;
3129 	db_printf("    mnt_stat = { version=%u type=%u flags=0x%016jx "
3130 	    "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju "
3131 	    "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju "
3132 	    "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n",
3133 	    (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags,
3134 	    (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize,
3135 	    (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree,
3136 	    (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files,
3137 	    (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites,
3138 	    (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads,
3139 	    (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax,
3140 	    (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]);
3141 
3142 	db_printf("    mnt_cred = { uid=%u ruid=%u",
3143 	    (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid);
3144 	if (jailed(mp->mnt_cred))
3145 		db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id);
3146 	db_printf(" }\n");
3147 	db_printf("    mnt_ref = %d\n", mp->mnt_ref);
3148 	db_printf("    mnt_gen = %d\n", mp->mnt_gen);
3149 	db_printf("    mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize);
3150 	db_printf("    mnt_activevnodelistsize = %d\n",
3151 	    mp->mnt_activevnodelistsize);
3152 	db_printf("    mnt_writeopcount = %d\n", mp->mnt_writeopcount);
3153 	db_printf("    mnt_maxsymlinklen = %d\n", mp->mnt_maxsymlinklen);
3154 	db_printf("    mnt_iosize_max = %d\n", mp->mnt_iosize_max);
3155 	db_printf("    mnt_hashseed = %u\n", mp->mnt_hashseed);
3156 	db_printf("    mnt_secondary_writes = %d\n", mp->mnt_secondary_writes);
3157 	db_printf("    mnt_secondary_accwrites = %d\n",
3158 	    mp->mnt_secondary_accwrites);
3159 	db_printf("    mnt_gjprovider = %s\n",
3160 	    mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL");
3161 
3162 	db_printf("\n\nList of active vnodes\n");
3163 	TAILQ_FOREACH(vp, &mp->mnt_activevnodelist, v_actfreelist) {
3164 		if (vp->v_type != VMARKER) {
3165 			vn_printf(vp, "vnode ");
3166 			if (db_pager_quit)
3167 				break;
3168 		}
3169 	}
3170 	db_printf("\n\nList of inactive vnodes\n");
3171 	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
3172 		if (vp->v_type != VMARKER && (vp->v_iflag & VI_ACTIVE) == 0) {
3173 			vn_printf(vp, "vnode ");
3174 			if (db_pager_quit)
3175 				break;
3176 		}
3177 	}
3178 }
3179 #endif	/* DDB */
3180 
3181 /*
3182  * Fill in a struct xvfsconf based on a struct vfsconf.
3183  */
3184 static int
3185 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp)
3186 {
3187 	struct xvfsconf xvfsp;
3188 
3189 	bzero(&xvfsp, sizeof(xvfsp));
3190 	strcpy(xvfsp.vfc_name, vfsp->vfc_name);
3191 	xvfsp.vfc_typenum = vfsp->vfc_typenum;
3192 	xvfsp.vfc_refcount = vfsp->vfc_refcount;
3193 	xvfsp.vfc_flags = vfsp->vfc_flags;
3194 	/*
3195 	 * These are unused in userland, we keep them
3196 	 * to not break binary compatibility.
3197 	 */
3198 	xvfsp.vfc_vfsops = NULL;
3199 	xvfsp.vfc_next = NULL;
3200 	return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
3201 }
3202 
3203 #ifdef COMPAT_FREEBSD32
3204 struct xvfsconf32 {
3205 	uint32_t	vfc_vfsops;
3206 	char		vfc_name[MFSNAMELEN];
3207 	int32_t		vfc_typenum;
3208 	int32_t		vfc_refcount;
3209 	int32_t		vfc_flags;
3210 	uint32_t	vfc_next;
3211 };
3212 
3213 static int
3214 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp)
3215 {
3216 	struct xvfsconf32 xvfsp;
3217 
3218 	strcpy(xvfsp.vfc_name, vfsp->vfc_name);
3219 	xvfsp.vfc_typenum = vfsp->vfc_typenum;
3220 	xvfsp.vfc_refcount = vfsp->vfc_refcount;
3221 	xvfsp.vfc_flags = vfsp->vfc_flags;
3222 	xvfsp.vfc_vfsops = 0;
3223 	xvfsp.vfc_next = 0;
3224 	return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
3225 }
3226 #endif
3227 
3228 /*
3229  * Top level filesystem related information gathering.
3230  */
3231 static int
3232 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS)
3233 {
3234 	struct vfsconf *vfsp;
3235 	int error;
3236 
3237 	error = 0;
3238 	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
3239 #ifdef COMPAT_FREEBSD32
3240 		if (req->flags & SCTL_MASK32)
3241 			error = vfsconf2x32(req, vfsp);
3242 		else
3243 #endif
3244 			error = vfsconf2x(req, vfsp);
3245 		if (error)
3246 			break;
3247 	}
3248 	return (error);
3249 }
3250 
3251 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD,
3252     NULL, 0, sysctl_vfs_conflist,
3253     "S,xvfsconf", "List of all configured filesystems");
3254 
3255 #ifndef BURN_BRIDGES
3256 static int	sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS);
3257 
3258 static int
3259 vfs_sysctl(SYSCTL_HANDLER_ARGS)
3260 {
3261 	int *name = (int *)arg1 - 1;	/* XXX */
3262 	u_int namelen = arg2 + 1;	/* XXX */
3263 	struct vfsconf *vfsp;
3264 
3265 	log(LOG_WARNING, "userland calling deprecated sysctl, "
3266 	    "please rebuild world\n");
3267 
3268 #if 1 || defined(COMPAT_PRELITE2)
3269 	/* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */
3270 	if (namelen == 1)
3271 		return (sysctl_ovfs_conf(oidp, arg1, arg2, req));
3272 #endif
3273 
3274 	switch (name[1]) {
3275 	case VFS_MAXTYPENUM:
3276 		if (namelen != 2)
3277 			return (ENOTDIR);
3278 		return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int)));
3279 	case VFS_CONF:
3280 		if (namelen != 3)
3281 			return (ENOTDIR);	/* overloaded */
3282 		TAILQ_FOREACH(vfsp, &vfsconf, vfc_list)
3283 			if (vfsp->vfc_typenum == name[2])
3284 				break;
3285 		if (vfsp == NULL)
3286 			return (EOPNOTSUPP);
3287 #ifdef COMPAT_FREEBSD32
3288 		if (req->flags & SCTL_MASK32)
3289 			return (vfsconf2x32(req, vfsp));
3290 		else
3291 #endif
3292 			return (vfsconf2x(req, vfsp));
3293 	}
3294 	return (EOPNOTSUPP);
3295 }
3296 
3297 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP,
3298     vfs_sysctl, "Generic filesystem");
3299 
3300 #if 1 || defined(COMPAT_PRELITE2)
3301 
3302 static int
3303 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS)
3304 {
3305 	int error;
3306 	struct vfsconf *vfsp;
3307 	struct ovfsconf ovfs;
3308 
3309 	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
3310 		bzero(&ovfs, sizeof(ovfs));
3311 		ovfs.vfc_vfsops = vfsp->vfc_vfsops;	/* XXX used as flag */
3312 		strcpy(ovfs.vfc_name, vfsp->vfc_name);
3313 		ovfs.vfc_index = vfsp->vfc_typenum;
3314 		ovfs.vfc_refcount = vfsp->vfc_refcount;
3315 		ovfs.vfc_flags = vfsp->vfc_flags;
3316 		error = SYSCTL_OUT(req, &ovfs, sizeof ovfs);
3317 		if (error)
3318 			return error;
3319 	}
3320 	return 0;
3321 }
3322 
3323 #endif /* 1 || COMPAT_PRELITE2 */
3324 #endif /* !BURN_BRIDGES */
3325 
3326 #define KINFO_VNODESLOP		10
3327 #ifdef notyet
3328 /*
3329  * Dump vnode list (via sysctl).
3330  */
3331 /* ARGSUSED */
3332 static int
3333 sysctl_vnode(SYSCTL_HANDLER_ARGS)
3334 {
3335 	struct xvnode *xvn;
3336 	struct mount *mp;
3337 	struct vnode *vp;
3338 	int error, len, n;
3339 
3340 	/*
3341 	 * Stale numvnodes access is not fatal here.
3342 	 */
3343 	req->lock = 0;
3344 	len = (numvnodes + KINFO_VNODESLOP) * sizeof *xvn;
3345 	if (!req->oldptr)
3346 		/* Make an estimate */
3347 		return (SYSCTL_OUT(req, 0, len));
3348 
3349 	error = sysctl_wire_old_buffer(req, 0);
3350 	if (error != 0)
3351 		return (error);
3352 	xvn = malloc(len, M_TEMP, M_ZERO | M_WAITOK);
3353 	n = 0;
3354 	mtx_lock(&mountlist_mtx);
3355 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
3356 		if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK))
3357 			continue;
3358 		MNT_ILOCK(mp);
3359 		TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
3360 			if (n == len)
3361 				break;
3362 			vref(vp);
3363 			xvn[n].xv_size = sizeof *xvn;
3364 			xvn[n].xv_vnode = vp;
3365 			xvn[n].xv_id = 0;	/* XXX compat */
3366 #define XV_COPY(field) xvn[n].xv_##field = vp->v_##field
3367 			XV_COPY(usecount);
3368 			XV_COPY(writecount);
3369 			XV_COPY(holdcnt);
3370 			XV_COPY(mount);
3371 			XV_COPY(numoutput);
3372 			XV_COPY(type);
3373 #undef XV_COPY
3374 			xvn[n].xv_flag = vp->v_vflag;
3375 
3376 			switch (vp->v_type) {
3377 			case VREG:
3378 			case VDIR:
3379 			case VLNK:
3380 				break;
3381 			case VBLK:
3382 			case VCHR:
3383 				if (vp->v_rdev == NULL) {
3384 					vrele(vp);
3385 					continue;
3386 				}
3387 				xvn[n].xv_dev = dev2udev(vp->v_rdev);
3388 				break;
3389 			case VSOCK:
3390 				xvn[n].xv_socket = vp->v_socket;
3391 				break;
3392 			case VFIFO:
3393 				xvn[n].xv_fifo = vp->v_fifoinfo;
3394 				break;
3395 			case VNON:
3396 			case VBAD:
3397 			default:
3398 				/* shouldn't happen? */
3399 				vrele(vp);
3400 				continue;
3401 			}
3402 			vrele(vp);
3403 			++n;
3404 		}
3405 		MNT_IUNLOCK(mp);
3406 		mtx_lock(&mountlist_mtx);
3407 		vfs_unbusy(mp);
3408 		if (n == len)
3409 			break;
3410 	}
3411 	mtx_unlock(&mountlist_mtx);
3412 
3413 	error = SYSCTL_OUT(req, xvn, n * sizeof *xvn);
3414 	free(xvn, M_TEMP);
3415 	return (error);
3416 }
3417 
3418 SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE|CTLFLAG_RD,
3419     0, 0, sysctl_vnode, "S,xvnode", "");
3420 #endif
3421 
3422 /*
3423  * Unmount all filesystems. The list is traversed in reverse order
3424  * of mounting to avoid dependencies.
3425  */
3426 void
3427 vfs_unmountall(void)
3428 {
3429 	struct mount *mp;
3430 	struct thread *td;
3431 	int error;
3432 
3433 	CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__);
3434 	td = curthread;
3435 
3436 	/*
3437 	 * Since this only runs when rebooting, it is not interlocked.
3438 	 */
3439 	while(!TAILQ_EMPTY(&mountlist)) {
3440 		mp = TAILQ_LAST(&mountlist, mntlist);
3441 		error = dounmount(mp, MNT_FORCE, td);
3442 		if (error) {
3443 			TAILQ_REMOVE(&mountlist, mp, mnt_list);
3444 			/*
3445 			 * XXX: Due to the way in which we mount the root
3446 			 * file system off of devfs, devfs will generate a
3447 			 * "busy" warning when we try to unmount it before
3448 			 * the root.  Don't print a warning as a result in
3449 			 * order to avoid false positive errors that may
3450 			 * cause needless upset.
3451 			 */
3452 			if (strcmp(mp->mnt_vfc->vfc_name, "devfs") != 0) {
3453 				printf("unmount of %s failed (",
3454 				    mp->mnt_stat.f_mntonname);
3455 				if (error == EBUSY)
3456 					printf("BUSY)\n");
3457 				else
3458 					printf("%d)\n", error);
3459 			}
3460 		} else {
3461 			/* The unmount has removed mp from the mountlist */
3462 		}
3463 	}
3464 }
3465 
3466 /*
3467  * perform msync on all vnodes under a mount point
3468  * the mount point must be locked.
3469  */
3470 void
3471 vfs_msync(struct mount *mp, int flags)
3472 {
3473 	struct vnode *vp, *mvp;
3474 	struct vm_object *obj;
3475 
3476 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
3477 	MNT_VNODE_FOREACH_ACTIVE(vp, mp, mvp) {
3478 		obj = vp->v_object;
3479 		if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 &&
3480 		    (flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) {
3481 			if (!vget(vp,
3482 			    LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK,
3483 			    curthread)) {
3484 				if (vp->v_vflag & VV_NOSYNC) {	/* unlinked */
3485 					vput(vp);
3486 					continue;
3487 				}
3488 
3489 				obj = vp->v_object;
3490 				if (obj != NULL) {
3491 					VM_OBJECT_LOCK(obj);
3492 					vm_object_page_clean(obj, 0, 0,
3493 					    flags == MNT_WAIT ?
3494 					    OBJPC_SYNC : OBJPC_NOSYNC);
3495 					VM_OBJECT_UNLOCK(obj);
3496 				}
3497 				vput(vp);
3498 			}
3499 		} else
3500 			VI_UNLOCK(vp);
3501 	}
3502 }
3503 
3504 static void
3505 destroy_vpollinfo(struct vpollinfo *vi)
3506 {
3507 	seldrain(&vi->vpi_selinfo);
3508 	knlist_destroy(&vi->vpi_selinfo.si_note);
3509 	mtx_destroy(&vi->vpi_lock);
3510 	uma_zfree(vnodepoll_zone, vi);
3511 }
3512 
3513 /*
3514  * Initalize per-vnode helper structure to hold poll-related state.
3515  */
3516 void
3517 v_addpollinfo(struct vnode *vp)
3518 {
3519 	struct vpollinfo *vi;
3520 
3521 	if (vp->v_pollinfo != NULL)
3522 		return;
3523 	vi = uma_zalloc(vnodepoll_zone, M_WAITOK);
3524 	mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF);
3525 	knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock,
3526 	    vfs_knlunlock, vfs_knl_assert_locked, vfs_knl_assert_unlocked);
3527 	VI_LOCK(vp);
3528 	if (vp->v_pollinfo != NULL) {
3529 		VI_UNLOCK(vp);
3530 		destroy_vpollinfo(vi);
3531 		return;
3532 	}
3533 	vp->v_pollinfo = vi;
3534 	VI_UNLOCK(vp);
3535 }
3536 
3537 /*
3538  * Record a process's interest in events which might happen to
3539  * a vnode.  Because poll uses the historic select-style interface
3540  * internally, this routine serves as both the ``check for any
3541  * pending events'' and the ``record my interest in future events''
3542  * functions.  (These are done together, while the lock is held,
3543  * to avoid race conditions.)
3544  */
3545 int
3546 vn_pollrecord(struct vnode *vp, struct thread *td, int events)
3547 {
3548 
3549 	v_addpollinfo(vp);
3550 	mtx_lock(&vp->v_pollinfo->vpi_lock);
3551 	if (vp->v_pollinfo->vpi_revents & events) {
3552 		/*
3553 		 * This leaves events we are not interested
3554 		 * in available for the other process which
3555 		 * which presumably had requested them
3556 		 * (otherwise they would never have been
3557 		 * recorded).
3558 		 */
3559 		events &= vp->v_pollinfo->vpi_revents;
3560 		vp->v_pollinfo->vpi_revents &= ~events;
3561 
3562 		mtx_unlock(&vp->v_pollinfo->vpi_lock);
3563 		return (events);
3564 	}
3565 	vp->v_pollinfo->vpi_events |= events;
3566 	selrecord(td, &vp->v_pollinfo->vpi_selinfo);
3567 	mtx_unlock(&vp->v_pollinfo->vpi_lock);
3568 	return (0);
3569 }
3570 
3571 /*
3572  * Routine to create and manage a filesystem syncer vnode.
3573  */
3574 #define sync_close ((int (*)(struct  vop_close_args *))nullop)
3575 static int	sync_fsync(struct  vop_fsync_args *);
3576 static int	sync_inactive(struct  vop_inactive_args *);
3577 static int	sync_reclaim(struct  vop_reclaim_args *);
3578 
3579 static struct vop_vector sync_vnodeops = {
3580 	.vop_bypass =	VOP_EOPNOTSUPP,
3581 	.vop_close =	sync_close,		/* close */
3582 	.vop_fsync =	sync_fsync,		/* fsync */
3583 	.vop_inactive =	sync_inactive,	/* inactive */
3584 	.vop_reclaim =	sync_reclaim,	/* reclaim */
3585 	.vop_lock1 =	vop_stdlock,	/* lock */
3586 	.vop_unlock =	vop_stdunlock,	/* unlock */
3587 	.vop_islocked =	vop_stdislocked,	/* islocked */
3588 };
3589 
3590 /*
3591  * Create a new filesystem syncer vnode for the specified mount point.
3592  */
3593 void
3594 vfs_allocate_syncvnode(struct mount *mp)
3595 {
3596 	struct vnode *vp;
3597 	struct bufobj *bo;
3598 	static long start, incr, next;
3599 	int error;
3600 
3601 	/* Allocate a new vnode */
3602 	error = getnewvnode("syncer", mp, &sync_vnodeops, &vp);
3603 	if (error != 0)
3604 		panic("vfs_allocate_syncvnode: getnewvnode() failed");
3605 	vp->v_type = VNON;
3606 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
3607 	vp->v_vflag |= VV_FORCEINSMQ;
3608 	error = insmntque(vp, mp);
3609 	if (error != 0)
3610 		panic("vfs_allocate_syncvnode: insmntque() failed");
3611 	vp->v_vflag &= ~VV_FORCEINSMQ;
3612 	VOP_UNLOCK(vp, 0);
3613 	/*
3614 	 * Place the vnode onto the syncer worklist. We attempt to
3615 	 * scatter them about on the list so that they will go off
3616 	 * at evenly distributed times even if all the filesystems
3617 	 * are mounted at once.
3618 	 */
3619 	next += incr;
3620 	if (next == 0 || next > syncer_maxdelay) {
3621 		start /= 2;
3622 		incr /= 2;
3623 		if (start == 0) {
3624 			start = syncer_maxdelay / 2;
3625 			incr = syncer_maxdelay;
3626 		}
3627 		next = start;
3628 	}
3629 	bo = &vp->v_bufobj;
3630 	BO_LOCK(bo);
3631 	vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0);
3632 	/* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */
3633 	mtx_lock(&sync_mtx);
3634 	sync_vnode_count++;
3635 	if (mp->mnt_syncer == NULL) {
3636 		mp->mnt_syncer = vp;
3637 		vp = NULL;
3638 	}
3639 	mtx_unlock(&sync_mtx);
3640 	BO_UNLOCK(bo);
3641 	if (vp != NULL) {
3642 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
3643 		vgone(vp);
3644 		vput(vp);
3645 	}
3646 }
3647 
3648 void
3649 vfs_deallocate_syncvnode(struct mount *mp)
3650 {
3651 	struct vnode *vp;
3652 
3653 	mtx_lock(&sync_mtx);
3654 	vp = mp->mnt_syncer;
3655 	if (vp != NULL)
3656 		mp->mnt_syncer = NULL;
3657 	mtx_unlock(&sync_mtx);
3658 	if (vp != NULL)
3659 		vrele(vp);
3660 }
3661 
3662 /*
3663  * Do a lazy sync of the filesystem.
3664  */
3665 static int
3666 sync_fsync(struct vop_fsync_args *ap)
3667 {
3668 	struct vnode *syncvp = ap->a_vp;
3669 	struct mount *mp = syncvp->v_mount;
3670 	int error, save;
3671 	struct bufobj *bo;
3672 
3673 	/*
3674 	 * We only need to do something if this is a lazy evaluation.
3675 	 */
3676 	if (ap->a_waitfor != MNT_LAZY)
3677 		return (0);
3678 
3679 	/*
3680 	 * Move ourselves to the back of the sync list.
3681 	 */
3682 	bo = &syncvp->v_bufobj;
3683 	BO_LOCK(bo);
3684 	vn_syncer_add_to_worklist(bo, syncdelay);
3685 	BO_UNLOCK(bo);
3686 
3687 	/*
3688 	 * Walk the list of vnodes pushing all that are dirty and
3689 	 * not already on the sync list.
3690 	 */
3691 	mtx_lock(&mountlist_mtx);
3692 	if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK) != 0) {
3693 		mtx_unlock(&mountlist_mtx);
3694 		return (0);
3695 	}
3696 	if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) {
3697 		vfs_unbusy(mp);
3698 		return (0);
3699 	}
3700 	save = curthread_pflags_set(TDP_SYNCIO);
3701 	vfs_msync(mp, MNT_NOWAIT);
3702 	error = VFS_SYNC(mp, MNT_LAZY);
3703 	curthread_pflags_restore(save);
3704 	vn_finished_write(mp);
3705 	vfs_unbusy(mp);
3706 	return (error);
3707 }
3708 
3709 /*
3710  * The syncer vnode is no referenced.
3711  */
3712 static int
3713 sync_inactive(struct vop_inactive_args *ap)
3714 {
3715 
3716 	vgone(ap->a_vp);
3717 	return (0);
3718 }
3719 
3720 /*
3721  * The syncer vnode is no longer needed and is being decommissioned.
3722  *
3723  * Modifications to the worklist must be protected by sync_mtx.
3724  */
3725 static int
3726 sync_reclaim(struct vop_reclaim_args *ap)
3727 {
3728 	struct vnode *vp = ap->a_vp;
3729 	struct bufobj *bo;
3730 
3731 	bo = &vp->v_bufobj;
3732 	BO_LOCK(bo);
3733 	mtx_lock(&sync_mtx);
3734 	if (vp->v_mount->mnt_syncer == vp)
3735 		vp->v_mount->mnt_syncer = NULL;
3736 	if (bo->bo_flag & BO_ONWORKLST) {
3737 		LIST_REMOVE(bo, bo_synclist);
3738 		syncer_worklist_len--;
3739 		sync_vnode_count--;
3740 		bo->bo_flag &= ~BO_ONWORKLST;
3741 	}
3742 	mtx_unlock(&sync_mtx);
3743 	BO_UNLOCK(bo);
3744 
3745 	return (0);
3746 }
3747 
3748 /*
3749  * Check if vnode represents a disk device
3750  */
3751 int
3752 vn_isdisk(struct vnode *vp, int *errp)
3753 {
3754 	int error;
3755 
3756 	error = 0;
3757 	dev_lock();
3758 	if (vp->v_type != VCHR)
3759 		error = ENOTBLK;
3760 	else if (vp->v_rdev == NULL)
3761 		error = ENXIO;
3762 	else if (vp->v_rdev->si_devsw == NULL)
3763 		error = ENXIO;
3764 	else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK))
3765 		error = ENOTBLK;
3766 	dev_unlock();
3767 	if (errp != NULL)
3768 		*errp = error;
3769 	return (error == 0);
3770 }
3771 
3772 /*
3773  * Common filesystem object access control check routine.  Accepts a
3774  * vnode's type, "mode", uid and gid, requested access mode, credentials,
3775  * and optional call-by-reference privused argument allowing vaccess()
3776  * to indicate to the caller whether privilege was used to satisfy the
3777  * request (obsoleted).  Returns 0 on success, or an errno on failure.
3778  */
3779 int
3780 vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid,
3781     accmode_t accmode, struct ucred *cred, int *privused)
3782 {
3783 	accmode_t dac_granted;
3784 	accmode_t priv_granted;
3785 
3786 	KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0,
3787 	    ("invalid bit in accmode"));
3788 	KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE),
3789 	    ("VAPPEND without VWRITE"));
3790 
3791 	/*
3792 	 * Look for a normal, non-privileged way to access the file/directory
3793 	 * as requested.  If it exists, go with that.
3794 	 */
3795 
3796 	if (privused != NULL)
3797 		*privused = 0;
3798 
3799 	dac_granted = 0;
3800 
3801 	/* Check the owner. */
3802 	if (cred->cr_uid == file_uid) {
3803 		dac_granted |= VADMIN;
3804 		if (file_mode & S_IXUSR)
3805 			dac_granted |= VEXEC;
3806 		if (file_mode & S_IRUSR)
3807 			dac_granted |= VREAD;
3808 		if (file_mode & S_IWUSR)
3809 			dac_granted |= (VWRITE | VAPPEND);
3810 
3811 		if ((accmode & dac_granted) == accmode)
3812 			return (0);
3813 
3814 		goto privcheck;
3815 	}
3816 
3817 	/* Otherwise, check the groups (first match) */
3818 	if (groupmember(file_gid, cred)) {
3819 		if (file_mode & S_IXGRP)
3820 			dac_granted |= VEXEC;
3821 		if (file_mode & S_IRGRP)
3822 			dac_granted |= VREAD;
3823 		if (file_mode & S_IWGRP)
3824 			dac_granted |= (VWRITE | VAPPEND);
3825 
3826 		if ((accmode & dac_granted) == accmode)
3827 			return (0);
3828 
3829 		goto privcheck;
3830 	}
3831 
3832 	/* Otherwise, check everyone else. */
3833 	if (file_mode & S_IXOTH)
3834 		dac_granted |= VEXEC;
3835 	if (file_mode & S_IROTH)
3836 		dac_granted |= VREAD;
3837 	if (file_mode & S_IWOTH)
3838 		dac_granted |= (VWRITE | VAPPEND);
3839 	if ((accmode & dac_granted) == accmode)
3840 		return (0);
3841 
3842 privcheck:
3843 	/*
3844 	 * Build a privilege mask to determine if the set of privileges
3845 	 * satisfies the requirements when combined with the granted mask
3846 	 * from above.  For each privilege, if the privilege is required,
3847 	 * bitwise or the request type onto the priv_granted mask.
3848 	 */
3849 	priv_granted = 0;
3850 
3851 	if (type == VDIR) {
3852 		/*
3853 		 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC
3854 		 * requests, instead of PRIV_VFS_EXEC.
3855 		 */
3856 		if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
3857 		    !priv_check_cred(cred, PRIV_VFS_LOOKUP, 0))
3858 			priv_granted |= VEXEC;
3859 	} else {
3860 		/*
3861 		 * Ensure that at least one execute bit is on. Otherwise,
3862 		 * a privileged user will always succeed, and we don't want
3863 		 * this to happen unless the file really is executable.
3864 		 */
3865 		if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
3866 		    (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 &&
3867 		    !priv_check_cred(cred, PRIV_VFS_EXEC, 0))
3868 			priv_granted |= VEXEC;
3869 	}
3870 
3871 	if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) &&
3872 	    !priv_check_cred(cred, PRIV_VFS_READ, 0))
3873 		priv_granted |= VREAD;
3874 
3875 	if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) &&
3876 	    !priv_check_cred(cred, PRIV_VFS_WRITE, 0))
3877 		priv_granted |= (VWRITE | VAPPEND);
3878 
3879 	if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) &&
3880 	    !priv_check_cred(cred, PRIV_VFS_ADMIN, 0))
3881 		priv_granted |= VADMIN;
3882 
3883 	if ((accmode & (priv_granted | dac_granted)) == accmode) {
3884 		/* XXX audit: privilege used */
3885 		if (privused != NULL)
3886 			*privused = 1;
3887 		return (0);
3888 	}
3889 
3890 	return ((accmode & VADMIN) ? EPERM : EACCES);
3891 }
3892 
3893 /*
3894  * Credential check based on process requesting service, and per-attribute
3895  * permissions.
3896  */
3897 int
3898 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred,
3899     struct thread *td, accmode_t accmode)
3900 {
3901 
3902 	/*
3903 	 * Kernel-invoked always succeeds.
3904 	 */
3905 	if (cred == NOCRED)
3906 		return (0);
3907 
3908 	/*
3909 	 * Do not allow privileged processes in jail to directly manipulate
3910 	 * system attributes.
3911 	 */
3912 	switch (attrnamespace) {
3913 	case EXTATTR_NAMESPACE_SYSTEM:
3914 		/* Potentially should be: return (EPERM); */
3915 		return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM, 0));
3916 	case EXTATTR_NAMESPACE_USER:
3917 		return (VOP_ACCESS(vp, accmode, cred, td));
3918 	default:
3919 		return (EPERM);
3920 	}
3921 }
3922 
3923 #ifdef DEBUG_VFS_LOCKS
3924 /*
3925  * This only exists to supress warnings from unlocked specfs accesses.  It is
3926  * no longer ok to have an unlocked VFS.
3927  */
3928 #define	IGNORE_LOCK(vp) (panicstr != NULL || (vp) == NULL ||		\
3929 	(vp)->v_type == VCHR ||	(vp)->v_type == VBAD)
3930 
3931 int vfs_badlock_ddb = 1;	/* Drop into debugger on violation. */
3932 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0,
3933     "Drop into debugger on lock violation");
3934 
3935 int vfs_badlock_mutex = 1;	/* Check for interlock across VOPs. */
3936 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex,
3937     0, "Check for interlock across VOPs");
3938 
3939 int vfs_badlock_print = 1;	/* Print lock violations. */
3940 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print,
3941     0, "Print lock violations");
3942 
3943 #ifdef KDB
3944 int vfs_badlock_backtrace = 1;	/* Print backtrace at lock violations. */
3945 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW,
3946     &vfs_badlock_backtrace, 0, "Print backtrace at lock violations");
3947 #endif
3948 
3949 static void
3950 vfs_badlock(const char *msg, const char *str, struct vnode *vp)
3951 {
3952 
3953 #ifdef KDB
3954 	if (vfs_badlock_backtrace)
3955 		kdb_backtrace();
3956 #endif
3957 	if (vfs_badlock_print)
3958 		printf("%s: %p %s\n", str, (void *)vp, msg);
3959 	if (vfs_badlock_ddb)
3960 		kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
3961 }
3962 
3963 void
3964 assert_vi_locked(struct vnode *vp, const char *str)
3965 {
3966 
3967 	if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp)))
3968 		vfs_badlock("interlock is not locked but should be", str, vp);
3969 }
3970 
3971 void
3972 assert_vi_unlocked(struct vnode *vp, const char *str)
3973 {
3974 
3975 	if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp)))
3976 		vfs_badlock("interlock is locked but should not be", str, vp);
3977 }
3978 
3979 void
3980 assert_vop_locked(struct vnode *vp, const char *str)
3981 {
3982 	int locked;
3983 
3984 	if (!IGNORE_LOCK(vp)) {
3985 		locked = VOP_ISLOCKED(vp);
3986 		if (locked == 0 || locked == LK_EXCLOTHER)
3987 			vfs_badlock("is not locked but should be", str, vp);
3988 	}
3989 }
3990 
3991 void
3992 assert_vop_unlocked(struct vnode *vp, const char *str)
3993 {
3994 
3995 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
3996 		vfs_badlock("is locked but should not be", str, vp);
3997 }
3998 
3999 void
4000 assert_vop_elocked(struct vnode *vp, const char *str)
4001 {
4002 
4003 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
4004 		vfs_badlock("is not exclusive locked but should be", str, vp);
4005 }
4006 
4007 #if 0
4008 void
4009 assert_vop_elocked_other(struct vnode *vp, const char *str)
4010 {
4011 
4012 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLOTHER)
4013 		vfs_badlock("is not exclusive locked by another thread",
4014 		    str, vp);
4015 }
4016 
4017 void
4018 assert_vop_slocked(struct vnode *vp, const char *str)
4019 {
4020 
4021 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_SHARED)
4022 		vfs_badlock("is not locked shared but should be", str, vp);
4023 }
4024 #endif /* 0 */
4025 #endif /* DEBUG_VFS_LOCKS */
4026 
4027 void
4028 vop_rename_fail(struct vop_rename_args *ap)
4029 {
4030 
4031 	if (ap->a_tvp != NULL)
4032 		vput(ap->a_tvp);
4033 	if (ap->a_tdvp == ap->a_tvp)
4034 		vrele(ap->a_tdvp);
4035 	else
4036 		vput(ap->a_tdvp);
4037 	vrele(ap->a_fdvp);
4038 	vrele(ap->a_fvp);
4039 }
4040 
4041 void
4042 vop_rename_pre(void *ap)
4043 {
4044 	struct vop_rename_args *a = ap;
4045 
4046 #ifdef DEBUG_VFS_LOCKS
4047 	if (a->a_tvp)
4048 		ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME");
4049 	ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME");
4050 	ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME");
4051 	ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME");
4052 
4053 	/* Check the source (from). */
4054 	if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock &&
4055 	    (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock))
4056 		ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked");
4057 	if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock)
4058 		ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked");
4059 
4060 	/* Check the target. */
4061 	if (a->a_tvp)
4062 		ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked");
4063 	ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked");
4064 #endif
4065 	if (a->a_tdvp != a->a_fdvp)
4066 		vhold(a->a_fdvp);
4067 	if (a->a_tvp != a->a_fvp)
4068 		vhold(a->a_fvp);
4069 	vhold(a->a_tdvp);
4070 	if (a->a_tvp)
4071 		vhold(a->a_tvp);
4072 }
4073 
4074 void
4075 vop_strategy_pre(void *ap)
4076 {
4077 #ifdef DEBUG_VFS_LOCKS
4078 	struct vop_strategy_args *a;
4079 	struct buf *bp;
4080 
4081 	a = ap;
4082 	bp = a->a_bp;
4083 
4084 	/*
4085 	 * Cluster ops lock their component buffers but not the IO container.
4086 	 */
4087 	if ((bp->b_flags & B_CLUSTER) != 0)
4088 		return;
4089 
4090 	if (panicstr == NULL && !BUF_ISLOCKED(bp)) {
4091 		if (vfs_badlock_print)
4092 			printf(
4093 			    "VOP_STRATEGY: bp is not locked but should be\n");
4094 		if (vfs_badlock_ddb)
4095 			kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
4096 	}
4097 #endif
4098 }
4099 
4100 void
4101 vop_lock_pre(void *ap)
4102 {
4103 #ifdef DEBUG_VFS_LOCKS
4104 	struct vop_lock1_args *a = ap;
4105 
4106 	if ((a->a_flags & LK_INTERLOCK) == 0)
4107 		ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
4108 	else
4109 		ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK");
4110 #endif
4111 }
4112 
4113 void
4114 vop_lock_post(void *ap, int rc)
4115 {
4116 #ifdef DEBUG_VFS_LOCKS
4117 	struct vop_lock1_args *a = ap;
4118 
4119 	ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
4120 	if (rc == 0)
4121 		ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK");
4122 #endif
4123 }
4124 
4125 void
4126 vop_unlock_pre(void *ap)
4127 {
4128 #ifdef DEBUG_VFS_LOCKS
4129 	struct vop_unlock_args *a = ap;
4130 
4131 	if (a->a_flags & LK_INTERLOCK)
4132 		ASSERT_VI_LOCKED(a->a_vp, "VOP_UNLOCK");
4133 	ASSERT_VOP_LOCKED(a->a_vp, "VOP_UNLOCK");
4134 #endif
4135 }
4136 
4137 void
4138 vop_unlock_post(void *ap, int rc)
4139 {
4140 #ifdef DEBUG_VFS_LOCKS
4141 	struct vop_unlock_args *a = ap;
4142 
4143 	if (a->a_flags & LK_INTERLOCK)
4144 		ASSERT_VI_UNLOCKED(a->a_vp, "VOP_UNLOCK");
4145 #endif
4146 }
4147 
4148 void
4149 vop_create_post(void *ap, int rc)
4150 {
4151 	struct vop_create_args *a = ap;
4152 
4153 	if (!rc)
4154 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4155 }
4156 
4157 void
4158 vop_deleteextattr_post(void *ap, int rc)
4159 {
4160 	struct vop_deleteextattr_args *a = ap;
4161 
4162 	if (!rc)
4163 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
4164 }
4165 
4166 void
4167 vop_link_post(void *ap, int rc)
4168 {
4169 	struct vop_link_args *a = ap;
4170 
4171 	if (!rc) {
4172 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_LINK);
4173 		VFS_KNOTE_LOCKED(a->a_tdvp, NOTE_WRITE);
4174 	}
4175 }
4176 
4177 void
4178 vop_mkdir_post(void *ap, int rc)
4179 {
4180 	struct vop_mkdir_args *a = ap;
4181 
4182 	if (!rc)
4183 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
4184 }
4185 
4186 void
4187 vop_mknod_post(void *ap, int rc)
4188 {
4189 	struct vop_mknod_args *a = ap;
4190 
4191 	if (!rc)
4192 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4193 }
4194 
4195 void
4196 vop_remove_post(void *ap, int rc)
4197 {
4198 	struct vop_remove_args *a = ap;
4199 
4200 	if (!rc) {
4201 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4202 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
4203 	}
4204 }
4205 
4206 void
4207 vop_rename_post(void *ap, int rc)
4208 {
4209 	struct vop_rename_args *a = ap;
4210 
4211 	if (!rc) {
4212 		VFS_KNOTE_UNLOCKED(a->a_fdvp, NOTE_WRITE);
4213 		VFS_KNOTE_UNLOCKED(a->a_tdvp, NOTE_WRITE);
4214 		VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME);
4215 		if (a->a_tvp)
4216 			VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE);
4217 	}
4218 	if (a->a_tdvp != a->a_fdvp)
4219 		vdrop(a->a_fdvp);
4220 	if (a->a_tvp != a->a_fvp)
4221 		vdrop(a->a_fvp);
4222 	vdrop(a->a_tdvp);
4223 	if (a->a_tvp)
4224 		vdrop(a->a_tvp);
4225 }
4226 
4227 void
4228 vop_rmdir_post(void *ap, int rc)
4229 {
4230 	struct vop_rmdir_args *a = ap;
4231 
4232 	if (!rc) {
4233 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
4234 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
4235 	}
4236 }
4237 
4238 void
4239 vop_setattr_post(void *ap, int rc)
4240 {
4241 	struct vop_setattr_args *a = ap;
4242 
4243 	if (!rc)
4244 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
4245 }
4246 
4247 void
4248 vop_setextattr_post(void *ap, int rc)
4249 {
4250 	struct vop_setextattr_args *a = ap;
4251 
4252 	if (!rc)
4253 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
4254 }
4255 
4256 void
4257 vop_symlink_post(void *ap, int rc)
4258 {
4259 	struct vop_symlink_args *a = ap;
4260 
4261 	if (!rc)
4262 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4263 }
4264 
4265 static struct knlist fs_knlist;
4266 
4267 static void
4268 vfs_event_init(void *arg)
4269 {
4270 	knlist_init_mtx(&fs_knlist, NULL);
4271 }
4272 /* XXX - correct order? */
4273 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL);
4274 
4275 void
4276 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused)
4277 {
4278 
4279 	KNOTE_UNLOCKED(&fs_knlist, event);
4280 }
4281 
4282 static int	filt_fsattach(struct knote *kn);
4283 static void	filt_fsdetach(struct knote *kn);
4284 static int	filt_fsevent(struct knote *kn, long hint);
4285 
4286 struct filterops fs_filtops = {
4287 	.f_isfd = 0,
4288 	.f_attach = filt_fsattach,
4289 	.f_detach = filt_fsdetach,
4290 	.f_event = filt_fsevent
4291 };
4292 
4293 static int
4294 filt_fsattach(struct knote *kn)
4295 {
4296 
4297 	kn->kn_flags |= EV_CLEAR;
4298 	knlist_add(&fs_knlist, kn, 0);
4299 	return (0);
4300 }
4301 
4302 static void
4303 filt_fsdetach(struct knote *kn)
4304 {
4305 
4306 	knlist_remove(&fs_knlist, kn, 0);
4307 }
4308 
4309 static int
4310 filt_fsevent(struct knote *kn, long hint)
4311 {
4312 
4313 	kn->kn_fflags |= hint;
4314 	return (kn->kn_fflags != 0);
4315 }
4316 
4317 static int
4318 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS)
4319 {
4320 	struct vfsidctl vc;
4321 	int error;
4322 	struct mount *mp;
4323 
4324 	error = SYSCTL_IN(req, &vc, sizeof(vc));
4325 	if (error)
4326 		return (error);
4327 	if (vc.vc_vers != VFS_CTL_VERS1)
4328 		return (EINVAL);
4329 	mp = vfs_getvfs(&vc.vc_fsid);
4330 	if (mp == NULL)
4331 		return (ENOENT);
4332 	/* ensure that a specific sysctl goes to the right filesystem. */
4333 	if (strcmp(vc.vc_fstypename, "*") != 0 &&
4334 	    strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) {
4335 		vfs_rel(mp);
4336 		return (EINVAL);
4337 	}
4338 	VCTLTOREQ(&vc, req);
4339 	error = VFS_SYSCTL(mp, vc.vc_op, req);
4340 	vfs_rel(mp);
4341 	return (error);
4342 }
4343 
4344 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_WR,
4345     NULL, 0, sysctl_vfs_ctl, "",
4346     "Sysctl by fsid");
4347 
4348 /*
4349  * Function to initialize a va_filerev field sensibly.
4350  * XXX: Wouldn't a random number make a lot more sense ??
4351  */
4352 u_quad_t
4353 init_va_filerev(void)
4354 {
4355 	struct bintime bt;
4356 
4357 	getbinuptime(&bt);
4358 	return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL));
4359 }
4360 
4361 static int	filt_vfsread(struct knote *kn, long hint);
4362 static int	filt_vfswrite(struct knote *kn, long hint);
4363 static int	filt_vfsvnode(struct knote *kn, long hint);
4364 static void	filt_vfsdetach(struct knote *kn);
4365 static struct filterops vfsread_filtops = {
4366 	.f_isfd = 1,
4367 	.f_detach = filt_vfsdetach,
4368 	.f_event = filt_vfsread
4369 };
4370 static struct filterops vfswrite_filtops = {
4371 	.f_isfd = 1,
4372 	.f_detach = filt_vfsdetach,
4373 	.f_event = filt_vfswrite
4374 };
4375 static struct filterops vfsvnode_filtops = {
4376 	.f_isfd = 1,
4377 	.f_detach = filt_vfsdetach,
4378 	.f_event = filt_vfsvnode
4379 };
4380 
4381 static void
4382 vfs_knllock(void *arg)
4383 {
4384 	struct vnode *vp = arg;
4385 
4386 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
4387 }
4388 
4389 static void
4390 vfs_knlunlock(void *arg)
4391 {
4392 	struct vnode *vp = arg;
4393 
4394 	VOP_UNLOCK(vp, 0);
4395 }
4396 
4397 static void
4398 vfs_knl_assert_locked(void *arg)
4399 {
4400 #ifdef DEBUG_VFS_LOCKS
4401 	struct vnode *vp = arg;
4402 
4403 	ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked");
4404 #endif
4405 }
4406 
4407 static void
4408 vfs_knl_assert_unlocked(void *arg)
4409 {
4410 #ifdef DEBUG_VFS_LOCKS
4411 	struct vnode *vp = arg;
4412 
4413 	ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked");
4414 #endif
4415 }
4416 
4417 int
4418 vfs_kqfilter(struct vop_kqfilter_args *ap)
4419 {
4420 	struct vnode *vp = ap->a_vp;
4421 	struct knote *kn = ap->a_kn;
4422 	struct knlist *knl;
4423 
4424 	switch (kn->kn_filter) {
4425 	case EVFILT_READ:
4426 		kn->kn_fop = &vfsread_filtops;
4427 		break;
4428 	case EVFILT_WRITE:
4429 		kn->kn_fop = &vfswrite_filtops;
4430 		break;
4431 	case EVFILT_VNODE:
4432 		kn->kn_fop = &vfsvnode_filtops;
4433 		break;
4434 	default:
4435 		return (EINVAL);
4436 	}
4437 
4438 	kn->kn_hook = (caddr_t)vp;
4439 
4440 	v_addpollinfo(vp);
4441 	if (vp->v_pollinfo == NULL)
4442 		return (ENOMEM);
4443 	knl = &vp->v_pollinfo->vpi_selinfo.si_note;
4444 	knlist_add(knl, kn, 0);
4445 
4446 	return (0);
4447 }
4448 
4449 /*
4450  * Detach knote from vnode
4451  */
4452 static void
4453 filt_vfsdetach(struct knote *kn)
4454 {
4455 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4456 
4457 	KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo"));
4458 	knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0);
4459 }
4460 
4461 /*ARGSUSED*/
4462 static int
4463 filt_vfsread(struct knote *kn, long hint)
4464 {
4465 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4466 	struct vattr va;
4467 	int res;
4468 
4469 	/*
4470 	 * filesystem is gone, so set the EOF flag and schedule
4471 	 * the knote for deletion.
4472 	 */
4473 	if (hint == NOTE_REVOKE) {
4474 		VI_LOCK(vp);
4475 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
4476 		VI_UNLOCK(vp);
4477 		return (1);
4478 	}
4479 
4480 	if (VOP_GETATTR(vp, &va, curthread->td_ucred))
4481 		return (0);
4482 
4483 	VI_LOCK(vp);
4484 	kn->kn_data = va.va_size - kn->kn_fp->f_offset;
4485 	res = (kn->kn_data != 0);
4486 	VI_UNLOCK(vp);
4487 	return (res);
4488 }
4489 
4490 /*ARGSUSED*/
4491 static int
4492 filt_vfswrite(struct knote *kn, long hint)
4493 {
4494 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4495 
4496 	VI_LOCK(vp);
4497 
4498 	/*
4499 	 * filesystem is gone, so set the EOF flag and schedule
4500 	 * the knote for deletion.
4501 	 */
4502 	if (hint == NOTE_REVOKE)
4503 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
4504 
4505 	kn->kn_data = 0;
4506 	VI_UNLOCK(vp);
4507 	return (1);
4508 }
4509 
4510 static int
4511 filt_vfsvnode(struct knote *kn, long hint)
4512 {
4513 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4514 	int res;
4515 
4516 	VI_LOCK(vp);
4517 	if (kn->kn_sfflags & hint)
4518 		kn->kn_fflags |= hint;
4519 	if (hint == NOTE_REVOKE) {
4520 		kn->kn_flags |= EV_EOF;
4521 		VI_UNLOCK(vp);
4522 		return (1);
4523 	}
4524 	res = (kn->kn_fflags != 0);
4525 	VI_UNLOCK(vp);
4526 	return (res);
4527 }
4528 
4529 int
4530 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off)
4531 {
4532 	int error;
4533 
4534 	if (dp->d_reclen > ap->a_uio->uio_resid)
4535 		return (ENAMETOOLONG);
4536 	error = uiomove(dp, dp->d_reclen, ap->a_uio);
4537 	if (error) {
4538 		if (ap->a_ncookies != NULL) {
4539 			if (ap->a_cookies != NULL)
4540 				free(ap->a_cookies, M_TEMP);
4541 			ap->a_cookies = NULL;
4542 			*ap->a_ncookies = 0;
4543 		}
4544 		return (error);
4545 	}
4546 	if (ap->a_ncookies == NULL)
4547 		return (0);
4548 
4549 	KASSERT(ap->a_cookies,
4550 	    ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!"));
4551 
4552 	*ap->a_cookies = realloc(*ap->a_cookies,
4553 	    (*ap->a_ncookies + 1) * sizeof(u_long), M_TEMP, M_WAITOK | M_ZERO);
4554 	(*ap->a_cookies)[*ap->a_ncookies] = off;
4555 	return (0);
4556 }
4557 
4558 /*
4559  * Mark for update the access time of the file if the filesystem
4560  * supports VOP_MARKATIME.  This functionality is used by execve and
4561  * mmap, so we want to avoid the I/O implied by directly setting
4562  * va_atime for the sake of efficiency.
4563  */
4564 void
4565 vfs_mark_atime(struct vnode *vp, struct ucred *cred)
4566 {
4567 	struct mount *mp;
4568 
4569 	mp = vp->v_mount;
4570 	ASSERT_VOP_LOCKED(vp, "vfs_mark_atime");
4571 	if (mp != NULL && (mp->mnt_flag & (MNT_NOATIME | MNT_RDONLY)) == 0)
4572 		(void)VOP_MARKATIME(vp);
4573 }
4574 
4575 /*
4576  * The purpose of this routine is to remove granularity from accmode_t,
4577  * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE,
4578  * VADMIN and VAPPEND.
4579  *
4580  * If it returns 0, the caller is supposed to continue with the usual
4581  * access checks using 'accmode' as modified by this routine.  If it
4582  * returns nonzero value, the caller is supposed to return that value
4583  * as errno.
4584  *
4585  * Note that after this routine runs, accmode may be zero.
4586  */
4587 int
4588 vfs_unixify_accmode(accmode_t *accmode)
4589 {
4590 	/*
4591 	 * There is no way to specify explicit "deny" rule using
4592 	 * file mode or POSIX.1e ACLs.
4593 	 */
4594 	if (*accmode & VEXPLICIT_DENY) {
4595 		*accmode = 0;
4596 		return (0);
4597 	}
4598 
4599 	/*
4600 	 * None of these can be translated into usual access bits.
4601 	 * Also, the common case for NFSv4 ACLs is to not contain
4602 	 * either of these bits. Caller should check for VWRITE
4603 	 * on the containing directory instead.
4604 	 */
4605 	if (*accmode & (VDELETE_CHILD | VDELETE))
4606 		return (EPERM);
4607 
4608 	if (*accmode & VADMIN_PERMS) {
4609 		*accmode &= ~VADMIN_PERMS;
4610 		*accmode |= VADMIN;
4611 	}
4612 
4613 	/*
4614 	 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL
4615 	 * or VSYNCHRONIZE using file mode or POSIX.1e ACL.
4616 	 */
4617 	*accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE);
4618 
4619 	return (0);
4620 }
4621 
4622 /*
4623  * These are helper functions for filesystems to traverse all
4624  * their vnodes.  See MNT_VNODE_FOREACH_ALL() in sys/mount.h.
4625  *
4626  * This interface replaces MNT_VNODE_FOREACH.
4627  */
4628 
4629 MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker");
4630 
4631 struct vnode *
4632 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp)
4633 {
4634 	struct vnode *vp;
4635 
4636 	if (should_yield())
4637 		kern_yield(PRI_UNCHANGED);
4638 	MNT_ILOCK(mp);
4639 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4640 	vp = TAILQ_NEXT(*mvp, v_nmntvnodes);
4641 	while (vp != NULL && (vp->v_type == VMARKER ||
4642 	    (vp->v_iflag & VI_DOOMED) != 0))
4643 		vp = TAILQ_NEXT(vp, v_nmntvnodes);
4644 
4645 	/* Check if we are done */
4646 	if (vp == NULL) {
4647 		__mnt_vnode_markerfree_all(mvp, mp);
4648 		/* MNT_IUNLOCK(mp); -- done in above function */
4649 		mtx_assert(MNT_MTX(mp), MA_NOTOWNED);
4650 		return (NULL);
4651 	}
4652 	TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
4653 	TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
4654 	VI_LOCK(vp);
4655 	MNT_IUNLOCK(mp);
4656 	return (vp);
4657 }
4658 
4659 struct vnode *
4660 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp)
4661 {
4662 	struct vnode *vp;
4663 
4664 	*mvp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO);
4665 	MNT_ILOCK(mp);
4666 	MNT_REF(mp);
4667 	(*mvp)->v_type = VMARKER;
4668 
4669 	vp = TAILQ_FIRST(&mp->mnt_nvnodelist);
4670 	while (vp != NULL && (vp->v_type == VMARKER ||
4671 	    (vp->v_iflag & VI_DOOMED) != 0))
4672 		vp = TAILQ_NEXT(vp, v_nmntvnodes);
4673 
4674 	/* Check if we are done */
4675 	if (vp == NULL) {
4676 		MNT_REL(mp);
4677 		MNT_IUNLOCK(mp);
4678 		free(*mvp, M_VNODE_MARKER);
4679 		*mvp = NULL;
4680 		return (NULL);
4681 	}
4682 	(*mvp)->v_mount = mp;
4683 	TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
4684 	VI_LOCK(vp);
4685 	MNT_IUNLOCK(mp);
4686 	return (vp);
4687 }
4688 
4689 
4690 void
4691 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp)
4692 {
4693 
4694 	if (*mvp == NULL) {
4695 		MNT_IUNLOCK(mp);
4696 		return;
4697 	}
4698 
4699 	mtx_assert(MNT_MTX(mp), MA_OWNED);
4700 
4701 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4702 	TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
4703 	MNT_REL(mp);
4704 	MNT_IUNLOCK(mp);
4705 	free(*mvp, M_VNODE_MARKER);
4706 	*mvp = NULL;
4707 }
4708 
4709 /*
4710  * These are helper functions for filesystems to traverse their
4711  * active vnodes.  See MNT_VNODE_FOREACH_ACTIVE() in sys/mount.h
4712  */
4713 struct vnode *
4714 __mnt_vnode_next_active(struct vnode **mvp, struct mount *mp)
4715 {
4716 	struct vnode *vp, *nvp;
4717 
4718 	if (should_yield())
4719 		kern_yield(PRI_UNCHANGED);
4720 	MNT_ILOCK(mp);
4721 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4722 	vp = TAILQ_NEXT(*mvp, v_actfreelist);
4723 	while (vp != NULL) {
4724 		VI_LOCK(vp);
4725 		if (vp->v_mount == mp && vp->v_type != VMARKER &&
4726 		    (vp->v_iflag & VI_DOOMED) == 0)
4727 			break;
4728 		nvp = TAILQ_NEXT(vp, v_actfreelist);
4729 		VI_UNLOCK(vp);
4730 		vp = nvp;
4731 	}
4732 
4733 	/* Check if we are done */
4734 	if (vp == NULL) {
4735 		__mnt_vnode_markerfree_active(mvp, mp);
4736 		/* MNT_IUNLOCK(mp); -- done in above function */
4737 		mtx_assert(MNT_MTX(mp), MA_NOTOWNED);
4738 		return (NULL);
4739 	}
4740 	mtx_lock(&vnode_free_list_mtx);
4741 	TAILQ_REMOVE(&mp->mnt_activevnodelist, *mvp, v_actfreelist);
4742 	TAILQ_INSERT_AFTER(&mp->mnt_activevnodelist, vp, *mvp, v_actfreelist);
4743 	mtx_unlock(&vnode_free_list_mtx);
4744 	MNT_IUNLOCK(mp);
4745 	return (vp);
4746 }
4747 
4748 struct vnode *
4749 __mnt_vnode_first_active(struct vnode **mvp, struct mount *mp)
4750 {
4751 	struct vnode *vp, *nvp;
4752 
4753 	*mvp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO);
4754 	MNT_ILOCK(mp);
4755 	MNT_REF(mp);
4756 	(*mvp)->v_type = VMARKER;
4757 
4758 	vp = TAILQ_FIRST(&mp->mnt_activevnodelist);
4759 	while (vp != NULL) {
4760 		VI_LOCK(vp);
4761 		if (vp->v_mount == mp && vp->v_type != VMARKER &&
4762 		    (vp->v_iflag & VI_DOOMED) == 0)
4763 			break;
4764 		nvp = TAILQ_NEXT(vp, v_actfreelist);
4765 		VI_UNLOCK(vp);
4766 		vp = nvp;
4767 	}
4768 
4769 	/* Check if we are done */
4770 	if (vp == NULL) {
4771 		MNT_REL(mp);
4772 		MNT_IUNLOCK(mp);
4773 		free(*mvp, M_VNODE_MARKER);
4774 		*mvp = NULL;
4775 		return (NULL);
4776 	}
4777 	(*mvp)->v_mount = mp;
4778 	mtx_lock(&vnode_free_list_mtx);
4779 	TAILQ_INSERT_AFTER(&mp->mnt_activevnodelist, vp, *mvp, v_actfreelist);
4780 	mtx_unlock(&vnode_free_list_mtx);
4781 	MNT_IUNLOCK(mp);
4782 	return (vp);
4783 }
4784 
4785 void
4786 __mnt_vnode_markerfree_active(struct vnode **mvp, struct mount *mp)
4787 {
4788 
4789 	if (*mvp == NULL) {
4790 		MNT_IUNLOCK(mp);
4791 		return;
4792 	}
4793 
4794 	mtx_assert(MNT_MTX(mp), MA_OWNED);
4795 
4796 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4797 	mtx_lock(&vnode_free_list_mtx);
4798 	TAILQ_REMOVE(&mp->mnt_activevnodelist, *mvp, v_actfreelist);
4799 	mtx_unlock(&vnode_free_list_mtx);
4800 	MNT_REL(mp);
4801 	MNT_IUNLOCK(mp);
4802 	free(*mvp, M_VNODE_MARKER);
4803 	*mvp = NULL;
4804 }
4805