xref: /freebsd/sys/kern/vfs_subr.c (revision f6a3b357e9be4c6423c85eff9a847163a0d307c8)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1989, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  * (c) UNIX System Laboratories, Inc.
7  * All or some portions of this file are derived from material licensed
8  * to the University of California by American Telephone and Telegraph
9  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10  * the permission of UNIX System Laboratories, Inc.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. Neither the name of the University nor the names of its contributors
21  *    may be used to endorse or promote products derived from this software
22  *    without specific prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34  * SUCH DAMAGE.
35  *
36  *	@(#)vfs_subr.c	8.31 (Berkeley) 5/26/95
37  */
38 
39 /*
40  * External virtual filesystem routines
41  */
42 
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD$");
45 
46 #include "opt_ddb.h"
47 #include "opt_watchdog.h"
48 
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/bio.h>
52 #include <sys/buf.h>
53 #include <sys/capsicum.h>
54 #include <sys/condvar.h>
55 #include <sys/conf.h>
56 #include <sys/counter.h>
57 #include <sys/dirent.h>
58 #include <sys/event.h>
59 #include <sys/eventhandler.h>
60 #include <sys/extattr.h>
61 #include <sys/file.h>
62 #include <sys/fcntl.h>
63 #include <sys/jail.h>
64 #include <sys/kdb.h>
65 #include <sys/kernel.h>
66 #include <sys/kthread.h>
67 #include <sys/ktr.h>
68 #include <sys/lockf.h>
69 #include <sys/malloc.h>
70 #include <sys/mount.h>
71 #include <sys/namei.h>
72 #include <sys/pctrie.h>
73 #include <sys/priv.h>
74 #include <sys/reboot.h>
75 #include <sys/refcount.h>
76 #include <sys/rwlock.h>
77 #include <sys/sched.h>
78 #include <sys/sleepqueue.h>
79 #include <sys/smp.h>
80 #include <sys/stat.h>
81 #include <sys/sysctl.h>
82 #include <sys/syslog.h>
83 #include <sys/vmmeter.h>
84 #include <sys/vnode.h>
85 #include <sys/watchdog.h>
86 
87 #include <machine/stdarg.h>
88 
89 #include <security/mac/mac_framework.h>
90 
91 #include <vm/vm.h>
92 #include <vm/vm_object.h>
93 #include <vm/vm_extern.h>
94 #include <vm/pmap.h>
95 #include <vm/vm_map.h>
96 #include <vm/vm_page.h>
97 #include <vm/vm_kern.h>
98 #include <vm/uma.h>
99 
100 #ifdef DDB
101 #include <ddb/ddb.h>
102 #endif
103 
104 static void	delmntque(struct vnode *vp);
105 static int	flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo,
106 		    int slpflag, int slptimeo);
107 static void	syncer_shutdown(void *arg, int howto);
108 static int	vtryrecycle(struct vnode *vp);
109 static void	v_init_counters(struct vnode *);
110 static void	v_incr_devcount(struct vnode *);
111 static void	v_decr_devcount(struct vnode *);
112 static void	vgonel(struct vnode *);
113 static void	vfs_knllock(void *arg);
114 static void	vfs_knlunlock(void *arg);
115 static void	vfs_knl_assert_locked(void *arg);
116 static void	vfs_knl_assert_unlocked(void *arg);
117 static void	vnlru_return_batches(struct vfsops *mnt_op);
118 static void	destroy_vpollinfo(struct vpollinfo *vi);
119 static int	v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo,
120 		    daddr_t startlbn, daddr_t endlbn);
121 
122 /*
123  * These fences are intended for cases where some synchronization is
124  * needed between access of v_iflags and lockless vnode refcount (v_holdcnt
125  * and v_usecount) updates.  Access to v_iflags is generally synchronized
126  * by the interlock, but we have some internal assertions that check vnode
127  * flags without acquiring the lock.  Thus, these fences are INVARIANTS-only
128  * for now.
129  */
130 #ifdef INVARIANTS
131 #define	VNODE_REFCOUNT_FENCE_ACQ()	atomic_thread_fence_acq()
132 #define	VNODE_REFCOUNT_FENCE_REL()	atomic_thread_fence_rel()
133 #else
134 #define	VNODE_REFCOUNT_FENCE_ACQ()
135 #define	VNODE_REFCOUNT_FENCE_REL()
136 #endif
137 
138 /*
139  * Number of vnodes in existence.  Increased whenever getnewvnode()
140  * allocates a new vnode, decreased in vdropl() for VI_DOOMED vnode.
141  */
142 static unsigned long	numvnodes;
143 
144 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0,
145     "Number of vnodes in existence");
146 
147 static counter_u64_t vnodes_created;
148 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created,
149     "Number of vnodes created by getnewvnode");
150 
151 static u_long mnt_free_list_batch = 128;
152 SYSCTL_ULONG(_vfs, OID_AUTO, mnt_free_list_batch, CTLFLAG_RW,
153     &mnt_free_list_batch, 0, "Limit of vnodes held on mnt's free list");
154 
155 /*
156  * Conversion tables for conversion from vnode types to inode formats
157  * and back.
158  */
159 enum vtype iftovt_tab[16] = {
160 	VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
161 	VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON
162 };
163 int vttoif_tab[10] = {
164 	0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK,
165 	S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT
166 };
167 
168 /*
169  * List of vnodes that are ready for recycling.
170  */
171 static TAILQ_HEAD(freelst, vnode) vnode_free_list;
172 
173 /*
174  * "Free" vnode target.  Free vnodes are rarely completely free, but are
175  * just ones that are cheap to recycle.  Usually they are for files which
176  * have been stat'd but not read; these usually have inode and namecache
177  * data attached to them.  This target is the preferred minimum size of a
178  * sub-cache consisting mostly of such files. The system balances the size
179  * of this sub-cache with its complement to try to prevent either from
180  * thrashing while the other is relatively inactive.  The targets express
181  * a preference for the best balance.
182  *
183  * "Above" this target there are 2 further targets (watermarks) related
184  * to recyling of free vnodes.  In the best-operating case, the cache is
185  * exactly full, the free list has size between vlowat and vhiwat above the
186  * free target, and recycling from it and normal use maintains this state.
187  * Sometimes the free list is below vlowat or even empty, but this state
188  * is even better for immediate use provided the cache is not full.
189  * Otherwise, vnlru_proc() runs to reclaim enough vnodes (usually non-free
190  * ones) to reach one of these states.  The watermarks are currently hard-
191  * coded as 4% and 9% of the available space higher.  These and the default
192  * of 25% for wantfreevnodes are too large if the memory size is large.
193  * E.g., 9% of 75% of MAXVNODES is more than 566000 vnodes to reclaim
194  * whenever vnlru_proc() becomes active.
195  */
196 static u_long wantfreevnodes;
197 SYSCTL_ULONG(_vfs, OID_AUTO, wantfreevnodes, CTLFLAG_RW,
198     &wantfreevnodes, 0, "Target for minimum number of \"free\" vnodes");
199 static u_long freevnodes;
200 SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD,
201     &freevnodes, 0, "Number of \"free\" vnodes");
202 
203 static counter_u64_t recycles_count;
204 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles, CTLFLAG_RD, &recycles_count,
205     "Number of vnodes recycled to meet vnode cache targets");
206 
207 /*
208  * Various variables used for debugging the new implementation of
209  * reassignbuf().
210  * XXX these are probably of (very) limited utility now.
211  */
212 static int reassignbufcalls;
213 SYSCTL_INT(_vfs, OID_AUTO, reassignbufcalls, CTLFLAG_RW, &reassignbufcalls, 0,
214     "Number of calls to reassignbuf");
215 
216 static counter_u64_t free_owe_inact;
217 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, free_owe_inact, CTLFLAG_RD, &free_owe_inact,
218     "Number of times free vnodes kept on active list due to VFS "
219     "owing inactivation");
220 
221 /* To keep more than one thread at a time from running vfs_getnewfsid */
222 static struct mtx mntid_mtx;
223 
224 /*
225  * Lock for any access to the following:
226  *	vnode_free_list
227  *	numvnodes
228  *	freevnodes
229  */
230 static struct mtx vnode_free_list_mtx;
231 
232 /* Publicly exported FS */
233 struct nfs_public nfs_pub;
234 
235 static uma_zone_t buf_trie_zone;
236 
237 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */
238 static uma_zone_t vnode_zone;
239 static uma_zone_t vnodepoll_zone;
240 
241 /*
242  * The workitem queue.
243  *
244  * It is useful to delay writes of file data and filesystem metadata
245  * for tens of seconds so that quickly created and deleted files need
246  * not waste disk bandwidth being created and removed. To realize this,
247  * we append vnodes to a "workitem" queue. When running with a soft
248  * updates implementation, most pending metadata dependencies should
249  * not wait for more than a few seconds. Thus, mounted on block devices
250  * are delayed only about a half the time that file data is delayed.
251  * Similarly, directory updates are more critical, so are only delayed
252  * about a third the time that file data is delayed. Thus, there are
253  * SYNCER_MAXDELAY queues that are processed round-robin at a rate of
254  * one each second (driven off the filesystem syncer process). The
255  * syncer_delayno variable indicates the next queue that is to be processed.
256  * Items that need to be processed soon are placed in this queue:
257  *
258  *	syncer_workitem_pending[syncer_delayno]
259  *
260  * A delay of fifteen seconds is done by placing the request fifteen
261  * entries later in the queue:
262  *
263  *	syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask]
264  *
265  */
266 static int syncer_delayno;
267 static long syncer_mask;
268 LIST_HEAD(synclist, bufobj);
269 static struct synclist *syncer_workitem_pending;
270 /*
271  * The sync_mtx protects:
272  *	bo->bo_synclist
273  *	sync_vnode_count
274  *	syncer_delayno
275  *	syncer_state
276  *	syncer_workitem_pending
277  *	syncer_worklist_len
278  *	rushjob
279  */
280 static struct mtx sync_mtx;
281 static struct cv sync_wakeup;
282 
283 #define SYNCER_MAXDELAY		32
284 static int syncer_maxdelay = SYNCER_MAXDELAY;	/* maximum delay time */
285 static int syncdelay = 30;		/* max time to delay syncing data */
286 static int filedelay = 30;		/* time to delay syncing files */
287 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0,
288     "Time to delay syncing files (in seconds)");
289 static int dirdelay = 29;		/* time to delay syncing directories */
290 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0,
291     "Time to delay syncing directories (in seconds)");
292 static int metadelay = 28;		/* time to delay syncing metadata */
293 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0,
294     "Time to delay syncing metadata (in seconds)");
295 static int rushjob;		/* number of slots to run ASAP */
296 static int stat_rush_requests;	/* number of times I/O speeded up */
297 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0,
298     "Number of times I/O speeded up (rush requests)");
299 
300 /*
301  * When shutting down the syncer, run it at four times normal speed.
302  */
303 #define SYNCER_SHUTDOWN_SPEEDUP		4
304 static int sync_vnode_count;
305 static int syncer_worklist_len;
306 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY }
307     syncer_state;
308 
309 /* Target for maximum number of vnodes. */
310 int desiredvnodes;
311 static int gapvnodes;		/* gap between wanted and desired */
312 static int vhiwat;		/* enough extras after expansion */
313 static int vlowat;		/* minimal extras before expansion */
314 static int vstir;		/* nonzero to stir non-free vnodes */
315 static volatile int vsmalltrigger = 8;	/* pref to keep if > this many pages */
316 
317 static int
318 sysctl_update_desiredvnodes(SYSCTL_HANDLER_ARGS)
319 {
320 	int error, old_desiredvnodes;
321 
322 	old_desiredvnodes = desiredvnodes;
323 	if ((error = sysctl_handle_int(oidp, arg1, arg2, req)) != 0)
324 		return (error);
325 	if (old_desiredvnodes != desiredvnodes) {
326 		wantfreevnodes = desiredvnodes / 4;
327 		/* XXX locking seems to be incomplete. */
328 		vfs_hash_changesize(desiredvnodes);
329 		cache_changesize(desiredvnodes);
330 	}
331 	return (0);
332 }
333 
334 SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes,
335     CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, &desiredvnodes, 0,
336     sysctl_update_desiredvnodes, "I", "Target for maximum number of vnodes");
337 SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW,
338     &wantfreevnodes, 0, "Old name for vfs.wantfreevnodes (legacy)");
339 static int vnlru_nowhere;
340 SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW,
341     &vnlru_nowhere, 0, "Number of times the vnlru process ran without success");
342 
343 static int
344 sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS)
345 {
346 	struct vnode *vp;
347 	struct nameidata nd;
348 	char *buf;
349 	unsigned long ndflags;
350 	int error;
351 
352 	if (req->newptr == NULL)
353 		return (EINVAL);
354 	if (req->newlen > PATH_MAX)
355 		return (E2BIG);
356 
357 	buf = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK);
358 	error = SYSCTL_IN(req, buf, req->newlen);
359 	if (error != 0)
360 		goto out;
361 
362 	buf[req->newlen] = '\0';
363 
364 	ndflags = LOCKLEAF | NOFOLLOW | AUDITVNODE1 | NOCACHE | SAVENAME;
365 	NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE, buf, curthread);
366 	if ((error = namei(&nd)) != 0)
367 		goto out;
368 	vp = nd.ni_vp;
369 
370 	if ((vp->v_iflag & VI_DOOMED) != 0) {
371 		/*
372 		 * This vnode is being recycled.  Return != 0 to let the caller
373 		 * know that the sysctl had no effect.  Return EAGAIN because a
374 		 * subsequent call will likely succeed (since namei will create
375 		 * a new vnode if necessary)
376 		 */
377 		error = EAGAIN;
378 		goto putvnode;
379 	}
380 
381 	counter_u64_add(recycles_count, 1);
382 	vgone(vp);
383 putvnode:
384 	NDFREE(&nd, 0);
385 out:
386 	free(buf, M_TEMP);
387 	return (error);
388 }
389 
390 static int
391 sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS)
392 {
393 	struct thread *td = curthread;
394 	struct vnode *vp;
395 	struct file *fp;
396 	int error;
397 	int fd;
398 
399 	if (req->newptr == NULL)
400 		return (EBADF);
401 
402         error = sysctl_handle_int(oidp, &fd, 0, req);
403         if (error != 0)
404                 return (error);
405 	error = getvnode(curthread, fd, &cap_fcntl_rights, &fp);
406 	if (error != 0)
407 		return (error);
408 	vp = fp->f_vnode;
409 
410 	error = vn_lock(vp, LK_EXCLUSIVE);
411 	if (error != 0)
412 		goto drop;
413 
414 	counter_u64_add(recycles_count, 1);
415 	vgone(vp);
416 	VOP_UNLOCK(vp, 0);
417 drop:
418 	fdrop(fp, td);
419 	return (error);
420 }
421 
422 SYSCTL_PROC(_debug, OID_AUTO, try_reclaim_vnode,
423     CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0,
424     sysctl_try_reclaim_vnode, "A", "Try to reclaim a vnode by its pathname");
425 SYSCTL_PROC(_debug, OID_AUTO, ftry_reclaim_vnode,
426     CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0,
427     sysctl_ftry_reclaim_vnode, "I",
428     "Try to reclaim a vnode by its file descriptor");
429 
430 /* Shift count for (uintptr_t)vp to initialize vp->v_hash. */
431 static int vnsz2log;
432 
433 /*
434  * Support for the bufobj clean & dirty pctrie.
435  */
436 static void *
437 buf_trie_alloc(struct pctrie *ptree)
438 {
439 
440 	return uma_zalloc(buf_trie_zone, M_NOWAIT);
441 }
442 
443 static void
444 buf_trie_free(struct pctrie *ptree, void *node)
445 {
446 
447 	uma_zfree(buf_trie_zone, node);
448 }
449 PCTRIE_DEFINE(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free);
450 
451 /*
452  * Initialize the vnode management data structures.
453  *
454  * Reevaluate the following cap on the number of vnodes after the physical
455  * memory size exceeds 512GB.  In the limit, as the physical memory size
456  * grows, the ratio of the memory size in KB to vnodes approaches 64:1.
457  */
458 #ifndef	MAXVNODES_MAX
459 #define	MAXVNODES_MAX	(512 * 1024 * 1024 / 64)	/* 8M */
460 #endif
461 
462 /*
463  * Initialize a vnode as it first enters the zone.
464  */
465 static int
466 vnode_init(void *mem, int size, int flags)
467 {
468 	struct vnode *vp;
469 
470 	vp = mem;
471 	bzero(vp, size);
472 	/*
473 	 * Setup locks.
474 	 */
475 	vp->v_vnlock = &vp->v_lock;
476 	mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF);
477 	/*
478 	 * By default, don't allow shared locks unless filesystems opt-in.
479 	 */
480 	lockinit(vp->v_vnlock, PVFS, "vnode", VLKTIMEOUT,
481 	    LK_NOSHARE | LK_IS_VNODE);
482 	/*
483 	 * Initialize bufobj.
484 	 */
485 	bufobj_init(&vp->v_bufobj, vp);
486 	/*
487 	 * Initialize namecache.
488 	 */
489 	LIST_INIT(&vp->v_cache_src);
490 	TAILQ_INIT(&vp->v_cache_dst);
491 	/*
492 	 * Initialize rangelocks.
493 	 */
494 	rangelock_init(&vp->v_rl);
495 	return (0);
496 }
497 
498 /*
499  * Free a vnode when it is cleared from the zone.
500  */
501 static void
502 vnode_fini(void *mem, int size)
503 {
504 	struct vnode *vp;
505 	struct bufobj *bo;
506 
507 	vp = mem;
508 	rangelock_destroy(&vp->v_rl);
509 	lockdestroy(vp->v_vnlock);
510 	mtx_destroy(&vp->v_interlock);
511 	bo = &vp->v_bufobj;
512 	rw_destroy(BO_LOCKPTR(bo));
513 }
514 
515 /*
516  * Provide the size of NFS nclnode and NFS fh for calculation of the
517  * vnode memory consumption.  The size is specified directly to
518  * eliminate dependency on NFS-private header.
519  *
520  * Other filesystems may use bigger or smaller (like UFS and ZFS)
521  * private inode data, but the NFS-based estimation is ample enough.
522  * Still, we care about differences in the size between 64- and 32-bit
523  * platforms.
524  *
525  * Namecache structure size is heuristically
526  * sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1.
527  */
528 #ifdef _LP64
529 #define	NFS_NCLNODE_SZ	(528 + 64)
530 #define	NC_SZ		148
531 #else
532 #define	NFS_NCLNODE_SZ	(360 + 32)
533 #define	NC_SZ		92
534 #endif
535 
536 static void
537 vntblinit(void *dummy __unused)
538 {
539 	u_int i;
540 	int physvnodes, virtvnodes;
541 
542 	/*
543 	 * Desiredvnodes is a function of the physical memory size and the
544 	 * kernel's heap size.  Generally speaking, it scales with the
545 	 * physical memory size.  The ratio of desiredvnodes to the physical
546 	 * memory size is 1:16 until desiredvnodes exceeds 98,304.
547 	 * Thereafter, the
548 	 * marginal ratio of desiredvnodes to the physical memory size is
549 	 * 1:64.  However, desiredvnodes is limited by the kernel's heap
550 	 * size.  The memory required by desiredvnodes vnodes and vm objects
551 	 * must not exceed 1/10th of the kernel's heap size.
552 	 */
553 	physvnodes = maxproc + pgtok(vm_cnt.v_page_count) / 64 +
554 	    3 * min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 64;
555 	virtvnodes = vm_kmem_size / (10 * (sizeof(struct vm_object) +
556 	    sizeof(struct vnode) + NC_SZ * ncsizefactor + NFS_NCLNODE_SZ));
557 	desiredvnodes = min(physvnodes, virtvnodes);
558 	if (desiredvnodes > MAXVNODES_MAX) {
559 		if (bootverbose)
560 			printf("Reducing kern.maxvnodes %d -> %d\n",
561 			    desiredvnodes, MAXVNODES_MAX);
562 		desiredvnodes = MAXVNODES_MAX;
563 	}
564 	wantfreevnodes = desiredvnodes / 4;
565 	mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF);
566 	TAILQ_INIT(&vnode_free_list);
567 	mtx_init(&vnode_free_list_mtx, "vnode_free_list", NULL, MTX_DEF);
568 	vnode_zone = uma_zcreate("VNODE", sizeof (struct vnode), NULL, NULL,
569 	    vnode_init, vnode_fini, UMA_ALIGN_PTR, 0);
570 	vnodepoll_zone = uma_zcreate("VNODEPOLL", sizeof (struct vpollinfo),
571 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
572 	/*
573 	 * Preallocate enough nodes to support one-per buf so that
574 	 * we can not fail an insert.  reassignbuf() callers can not
575 	 * tolerate the insertion failure.
576 	 */
577 	buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(),
578 	    NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR,
579 	    UMA_ZONE_NOFREE | UMA_ZONE_VM);
580 	uma_prealloc(buf_trie_zone, nbuf);
581 
582 	vnodes_created = counter_u64_alloc(M_WAITOK);
583 	recycles_count = counter_u64_alloc(M_WAITOK);
584 	free_owe_inact = counter_u64_alloc(M_WAITOK);
585 
586 	/*
587 	 * Initialize the filesystem syncer.
588 	 */
589 	syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE,
590 	    &syncer_mask);
591 	syncer_maxdelay = syncer_mask + 1;
592 	mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF);
593 	cv_init(&sync_wakeup, "syncer");
594 	for (i = 1; i <= sizeof(struct vnode); i <<= 1)
595 		vnsz2log++;
596 	vnsz2log--;
597 }
598 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);
599 
600 
601 /*
602  * Mark a mount point as busy. Used to synchronize access and to delay
603  * unmounting. Eventually, mountlist_mtx is not released on failure.
604  *
605  * vfs_busy() is a custom lock, it can block the caller.
606  * vfs_busy() only sleeps if the unmount is active on the mount point.
607  * For a mountpoint mp, vfs_busy-enforced lock is before lock of any
608  * vnode belonging to mp.
609  *
610  * Lookup uses vfs_busy() to traverse mount points.
611  * root fs			var fs
612  * / vnode lock		A	/ vnode lock (/var)		D
613  * /var vnode lock	B	/log vnode lock(/var/log)	E
614  * vfs_busy lock	C	vfs_busy lock			F
615  *
616  * Within each file system, the lock order is C->A->B and F->D->E.
617  *
618  * When traversing across mounts, the system follows that lock order:
619  *
620  *        C->A->B
621  *              |
622  *              +->F->D->E
623  *
624  * The lookup() process for namei("/var") illustrates the process:
625  *  VOP_LOOKUP() obtains B while A is held
626  *  vfs_busy() obtains a shared lock on F while A and B are held
627  *  vput() releases lock on B
628  *  vput() releases lock on A
629  *  VFS_ROOT() obtains lock on D while shared lock on F is held
630  *  vfs_unbusy() releases shared lock on F
631  *  vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A.
632  *    Attempt to lock A (instead of vp_crossmp) while D is held would
633  *    violate the global order, causing deadlocks.
634  *
635  * dounmount() locks B while F is drained.
636  */
637 int
638 vfs_busy(struct mount *mp, int flags)
639 {
640 
641 	MPASS((flags & ~MBF_MASK) == 0);
642 	CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags);
643 
644 	MNT_ILOCK(mp);
645 	MNT_REF(mp);
646 	/*
647 	 * If mount point is currently being unmounted, sleep until the
648 	 * mount point fate is decided.  If thread doing the unmounting fails,
649 	 * it will clear MNTK_UNMOUNT flag before waking us up, indicating
650 	 * that this mount point has survived the unmount attempt and vfs_busy
651 	 * should retry.  Otherwise the unmounter thread will set MNTK_REFEXPIRE
652 	 * flag in addition to MNTK_UNMOUNT, indicating that mount point is
653 	 * about to be really destroyed.  vfs_busy needs to release its
654 	 * reference on the mount point in this case and return with ENOENT,
655 	 * telling the caller that mount mount it tried to busy is no longer
656 	 * valid.
657 	 */
658 	while (mp->mnt_kern_flag & MNTK_UNMOUNT) {
659 		if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) {
660 			MNT_REL(mp);
661 			MNT_IUNLOCK(mp);
662 			CTR1(KTR_VFS, "%s: failed busying before sleeping",
663 			    __func__);
664 			return (ENOENT);
665 		}
666 		if (flags & MBF_MNTLSTLOCK)
667 			mtx_unlock(&mountlist_mtx);
668 		mp->mnt_kern_flag |= MNTK_MWAIT;
669 		msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0);
670 		if (flags & MBF_MNTLSTLOCK)
671 			mtx_lock(&mountlist_mtx);
672 		MNT_ILOCK(mp);
673 	}
674 	if (flags & MBF_MNTLSTLOCK)
675 		mtx_unlock(&mountlist_mtx);
676 	mp->mnt_lockref++;
677 	MNT_IUNLOCK(mp);
678 	return (0);
679 }
680 
681 /*
682  * Free a busy filesystem.
683  */
684 void
685 vfs_unbusy(struct mount *mp)
686 {
687 
688 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
689 	MNT_ILOCK(mp);
690 	MNT_REL(mp);
691 	KASSERT(mp->mnt_lockref > 0, ("negative mnt_lockref"));
692 	mp->mnt_lockref--;
693 	if (mp->mnt_lockref == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) {
694 		MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT);
695 		CTR1(KTR_VFS, "%s: waking up waiters", __func__);
696 		mp->mnt_kern_flag &= ~MNTK_DRAINING;
697 		wakeup(&mp->mnt_lockref);
698 	}
699 	MNT_IUNLOCK(mp);
700 }
701 
702 /*
703  * Lookup a mount point by filesystem identifier.
704  */
705 struct mount *
706 vfs_getvfs(fsid_t *fsid)
707 {
708 	struct mount *mp;
709 
710 	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
711 	mtx_lock(&mountlist_mtx);
712 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
713 		if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
714 		    mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
715 			vfs_ref(mp);
716 			mtx_unlock(&mountlist_mtx);
717 			return (mp);
718 		}
719 	}
720 	mtx_unlock(&mountlist_mtx);
721 	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
722 	return ((struct mount *) 0);
723 }
724 
725 /*
726  * Lookup a mount point by filesystem identifier, busying it before
727  * returning.
728  *
729  * To avoid congestion on mountlist_mtx, implement simple direct-mapped
730  * cache for popular filesystem identifiers.  The cache is lockess, using
731  * the fact that struct mount's are never freed.  In worst case we may
732  * get pointer to unmounted or even different filesystem, so we have to
733  * check what we got, and go slow way if so.
734  */
735 struct mount *
736 vfs_busyfs(fsid_t *fsid)
737 {
738 #define	FSID_CACHE_SIZE	256
739 	typedef struct mount * volatile vmp_t;
740 	static vmp_t cache[FSID_CACHE_SIZE];
741 	struct mount *mp;
742 	int error;
743 	uint32_t hash;
744 
745 	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
746 	hash = fsid->val[0] ^ fsid->val[1];
747 	hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1);
748 	mp = cache[hash];
749 	if (mp == NULL ||
750 	    mp->mnt_stat.f_fsid.val[0] != fsid->val[0] ||
751 	    mp->mnt_stat.f_fsid.val[1] != fsid->val[1])
752 		goto slow;
753 	if (vfs_busy(mp, 0) != 0) {
754 		cache[hash] = NULL;
755 		goto slow;
756 	}
757 	if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
758 	    mp->mnt_stat.f_fsid.val[1] == fsid->val[1])
759 		return (mp);
760 	else
761 	    vfs_unbusy(mp);
762 
763 slow:
764 	mtx_lock(&mountlist_mtx);
765 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
766 		if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
767 		    mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
768 			error = vfs_busy(mp, MBF_MNTLSTLOCK);
769 			if (error) {
770 				cache[hash] = NULL;
771 				mtx_unlock(&mountlist_mtx);
772 				return (NULL);
773 			}
774 			cache[hash] = mp;
775 			return (mp);
776 		}
777 	}
778 	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
779 	mtx_unlock(&mountlist_mtx);
780 	return ((struct mount *) 0);
781 }
782 
783 /*
784  * Check if a user can access privileged mount options.
785  */
786 int
787 vfs_suser(struct mount *mp, struct thread *td)
788 {
789 	int error;
790 
791 	if (jailed(td->td_ucred)) {
792 		/*
793 		 * If the jail of the calling thread lacks permission for
794 		 * this type of file system, deny immediately.
795 		 */
796 		if (!prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag))
797 			return (EPERM);
798 
799 		/*
800 		 * If the file system was mounted outside the jail of the
801 		 * calling thread, deny immediately.
802 		 */
803 		if (prison_check(td->td_ucred, mp->mnt_cred) != 0)
804 			return (EPERM);
805 	}
806 
807 	/*
808 	 * If file system supports delegated administration, we don't check
809 	 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified
810 	 * by the file system itself.
811 	 * If this is not the user that did original mount, we check for
812 	 * the PRIV_VFS_MOUNT_OWNER privilege.
813 	 */
814 	if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) &&
815 	    mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) {
816 		if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0)
817 			return (error);
818 	}
819 	return (0);
820 }
821 
822 /*
823  * Get a new unique fsid.  Try to make its val[0] unique, since this value
824  * will be used to create fake device numbers for stat().  Also try (but
825  * not so hard) make its val[0] unique mod 2^16, since some emulators only
826  * support 16-bit device numbers.  We end up with unique val[0]'s for the
827  * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls.
828  *
829  * Keep in mind that several mounts may be running in parallel.  Starting
830  * the search one past where the previous search terminated is both a
831  * micro-optimization and a defense against returning the same fsid to
832  * different mounts.
833  */
834 void
835 vfs_getnewfsid(struct mount *mp)
836 {
837 	static uint16_t mntid_base;
838 	struct mount *nmp;
839 	fsid_t tfsid;
840 	int mtype;
841 
842 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
843 	mtx_lock(&mntid_mtx);
844 	mtype = mp->mnt_vfc->vfc_typenum;
845 	tfsid.val[1] = mtype;
846 	mtype = (mtype & 0xFF) << 24;
847 	for (;;) {
848 		tfsid.val[0] = makedev(255,
849 		    mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF));
850 		mntid_base++;
851 		if ((nmp = vfs_getvfs(&tfsid)) == NULL)
852 			break;
853 		vfs_rel(nmp);
854 	}
855 	mp->mnt_stat.f_fsid.val[0] = tfsid.val[0];
856 	mp->mnt_stat.f_fsid.val[1] = tfsid.val[1];
857 	mtx_unlock(&mntid_mtx);
858 }
859 
860 /*
861  * Knob to control the precision of file timestamps:
862  *
863  *   0 = seconds only; nanoseconds zeroed.
864  *   1 = seconds and nanoseconds, accurate within 1/HZ.
865  *   2 = seconds and nanoseconds, truncated to microseconds.
866  * >=3 = seconds and nanoseconds, maximum precision.
867  */
868 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC };
869 
870 static int timestamp_precision = TSP_USEC;
871 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW,
872     &timestamp_precision, 0, "File timestamp precision (0: seconds, "
873     "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, "
874     "3+: sec + ns (max. precision))");
875 
876 /*
877  * Get a current timestamp.
878  */
879 void
880 vfs_timestamp(struct timespec *tsp)
881 {
882 	struct timeval tv;
883 
884 	switch (timestamp_precision) {
885 	case TSP_SEC:
886 		tsp->tv_sec = time_second;
887 		tsp->tv_nsec = 0;
888 		break;
889 	case TSP_HZ:
890 		getnanotime(tsp);
891 		break;
892 	case TSP_USEC:
893 		microtime(&tv);
894 		TIMEVAL_TO_TIMESPEC(&tv, tsp);
895 		break;
896 	case TSP_NSEC:
897 	default:
898 		nanotime(tsp);
899 		break;
900 	}
901 }
902 
903 /*
904  * Set vnode attributes to VNOVAL
905  */
906 void
907 vattr_null(struct vattr *vap)
908 {
909 
910 	vap->va_type = VNON;
911 	vap->va_size = VNOVAL;
912 	vap->va_bytes = VNOVAL;
913 	vap->va_mode = VNOVAL;
914 	vap->va_nlink = VNOVAL;
915 	vap->va_uid = VNOVAL;
916 	vap->va_gid = VNOVAL;
917 	vap->va_fsid = VNOVAL;
918 	vap->va_fileid = VNOVAL;
919 	vap->va_blocksize = VNOVAL;
920 	vap->va_rdev = VNOVAL;
921 	vap->va_atime.tv_sec = VNOVAL;
922 	vap->va_atime.tv_nsec = VNOVAL;
923 	vap->va_mtime.tv_sec = VNOVAL;
924 	vap->va_mtime.tv_nsec = VNOVAL;
925 	vap->va_ctime.tv_sec = VNOVAL;
926 	vap->va_ctime.tv_nsec = VNOVAL;
927 	vap->va_birthtime.tv_sec = VNOVAL;
928 	vap->va_birthtime.tv_nsec = VNOVAL;
929 	vap->va_flags = VNOVAL;
930 	vap->va_gen = VNOVAL;
931 	vap->va_vaflags = 0;
932 }
933 
934 /*
935  * This routine is called when we have too many vnodes.  It attempts
936  * to free <count> vnodes and will potentially free vnodes that still
937  * have VM backing store (VM backing store is typically the cause
938  * of a vnode blowout so we want to do this).  Therefore, this operation
939  * is not considered cheap.
940  *
941  * A number of conditions may prevent a vnode from being reclaimed.
942  * the buffer cache may have references on the vnode, a directory
943  * vnode may still have references due to the namei cache representing
944  * underlying files, or the vnode may be in active use.   It is not
945  * desirable to reuse such vnodes.  These conditions may cause the
946  * number of vnodes to reach some minimum value regardless of what
947  * you set kern.maxvnodes to.  Do not set kern.maxvnodes too low.
948  *
949  * @param mp		 Try to reclaim vnodes from this mountpoint
950  * @param reclaim_nc_src Only reclaim directories with outgoing namecache
951  * 			 entries if this argument is strue
952  * @param trigger	 Only reclaim vnodes with fewer than this many resident
953  *			 pages.
954  * @return		 The number of vnodes that were reclaimed.
955  */
956 static int
957 vlrureclaim(struct mount *mp, bool reclaim_nc_src, int trigger)
958 {
959 	struct vnode *vp;
960 	int count, done, target;
961 
962 	done = 0;
963 	vn_start_write(NULL, &mp, V_WAIT);
964 	MNT_ILOCK(mp);
965 	count = mp->mnt_nvnodelistsize;
966 	target = count * (int64_t)gapvnodes / imax(desiredvnodes, 1);
967 	target = target / 10 + 1;
968 	while (count != 0 && done < target) {
969 		vp = TAILQ_FIRST(&mp->mnt_nvnodelist);
970 		while (vp != NULL && vp->v_type == VMARKER)
971 			vp = TAILQ_NEXT(vp, v_nmntvnodes);
972 		if (vp == NULL)
973 			break;
974 		/*
975 		 * XXX LRU is completely broken for non-free vnodes.  First
976 		 * by calling here in mountpoint order, then by moving
977 		 * unselected vnodes to the end here, and most grossly by
978 		 * removing the vlruvp() function that was supposed to
979 		 * maintain the order.  (This function was born broken
980 		 * since syncer problems prevented it doing anything.)  The
981 		 * order is closer to LRC (C = Created).
982 		 *
983 		 * LRU reclaiming of vnodes seems to have last worked in
984 		 * FreeBSD-3 where LRU wasn't mentioned under any spelling.
985 		 * Then there was no hold count, and inactive vnodes were
986 		 * simply put on the free list in LRU order.  The separate
987 		 * lists also break LRU.  We prefer to reclaim from the
988 		 * free list for technical reasons.  This tends to thrash
989 		 * the free list to keep very unrecently used held vnodes.
990 		 * The problem is mitigated by keeping the free list large.
991 		 */
992 		TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
993 		TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
994 		--count;
995 		if (!VI_TRYLOCK(vp))
996 			goto next_iter;
997 		/*
998 		 * If it's been deconstructed already, it's still
999 		 * referenced, or it exceeds the trigger, skip it.
1000 		 * Also skip free vnodes.  We are trying to make space
1001 		 * to expand the free list, not reduce it.
1002 		 */
1003 		if (vp->v_usecount ||
1004 		    (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) ||
1005 		    ((vp->v_iflag & VI_FREE) != 0) ||
1006 		    (vp->v_iflag & VI_DOOMED) != 0 || (vp->v_object != NULL &&
1007 		    vp->v_object->resident_page_count > trigger)) {
1008 			VI_UNLOCK(vp);
1009 			goto next_iter;
1010 		}
1011 		MNT_IUNLOCK(mp);
1012 		vholdl(vp);
1013 		if (VOP_LOCK(vp, LK_INTERLOCK|LK_EXCLUSIVE|LK_NOWAIT)) {
1014 			vdrop(vp);
1015 			goto next_iter_mntunlocked;
1016 		}
1017 		VI_LOCK(vp);
1018 		/*
1019 		 * v_usecount may have been bumped after VOP_LOCK() dropped
1020 		 * the vnode interlock and before it was locked again.
1021 		 *
1022 		 * It is not necessary to recheck VI_DOOMED because it can
1023 		 * only be set by another thread that holds both the vnode
1024 		 * lock and vnode interlock.  If another thread has the
1025 		 * vnode lock before we get to VOP_LOCK() and obtains the
1026 		 * vnode interlock after VOP_LOCK() drops the vnode
1027 		 * interlock, the other thread will be unable to drop the
1028 		 * vnode lock before our VOP_LOCK() call fails.
1029 		 */
1030 		if (vp->v_usecount ||
1031 		    (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) ||
1032 		    (vp->v_iflag & VI_FREE) != 0 ||
1033 		    (vp->v_object != NULL &&
1034 		    vp->v_object->resident_page_count > trigger)) {
1035 			VOP_UNLOCK(vp, 0);
1036 			vdropl(vp);
1037 			goto next_iter_mntunlocked;
1038 		}
1039 		KASSERT((vp->v_iflag & VI_DOOMED) == 0,
1040 		    ("VI_DOOMED unexpectedly detected in vlrureclaim()"));
1041 		counter_u64_add(recycles_count, 1);
1042 		vgonel(vp);
1043 		VOP_UNLOCK(vp, 0);
1044 		vdropl(vp);
1045 		done++;
1046 next_iter_mntunlocked:
1047 		if (!should_yield())
1048 			goto relock_mnt;
1049 		goto yield;
1050 next_iter:
1051 		if (!should_yield())
1052 			continue;
1053 		MNT_IUNLOCK(mp);
1054 yield:
1055 		kern_yield(PRI_USER);
1056 relock_mnt:
1057 		MNT_ILOCK(mp);
1058 	}
1059 	MNT_IUNLOCK(mp);
1060 	vn_finished_write(mp);
1061 	return done;
1062 }
1063 
1064 static int max_vnlru_free = 10000; /* limit on vnode free requests per call */
1065 SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_vnlru_free,
1066     0,
1067     "limit on vnode free requests per call to the vnlru_free routine");
1068 
1069 /*
1070  * Attempt to reduce the free list by the requested amount.
1071  */
1072 static void
1073 vnlru_free_locked(int count, struct vfsops *mnt_op)
1074 {
1075 	struct vnode *vp;
1076 	struct mount *mp;
1077 	bool tried_batches;
1078 
1079 	tried_batches = false;
1080 	mtx_assert(&vnode_free_list_mtx, MA_OWNED);
1081 	if (count > max_vnlru_free)
1082 		count = max_vnlru_free;
1083 	for (; count > 0; count--) {
1084 		vp = TAILQ_FIRST(&vnode_free_list);
1085 		/*
1086 		 * The list can be modified while the free_list_mtx
1087 		 * has been dropped and vp could be NULL here.
1088 		 */
1089 		if (vp == NULL) {
1090 			if (tried_batches)
1091 				break;
1092 			mtx_unlock(&vnode_free_list_mtx);
1093 			vnlru_return_batches(mnt_op);
1094 			tried_batches = true;
1095 			mtx_lock(&vnode_free_list_mtx);
1096 			continue;
1097 		}
1098 
1099 		VNASSERT(vp->v_op != NULL, vp,
1100 		    ("vnlru_free: vnode already reclaimed."));
1101 		KASSERT((vp->v_iflag & VI_FREE) != 0,
1102 		    ("Removing vnode not on freelist"));
1103 		KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
1104 		    ("Mangling active vnode"));
1105 		TAILQ_REMOVE(&vnode_free_list, vp, v_actfreelist);
1106 
1107 		/*
1108 		 * Don't recycle if our vnode is from different type
1109 		 * of mount point.  Note that mp is type-safe, the
1110 		 * check does not reach unmapped address even if
1111 		 * vnode is reclaimed.
1112 		 * Don't recycle if we can't get the interlock without
1113 		 * blocking.
1114 		 */
1115 		if ((mnt_op != NULL && (mp = vp->v_mount) != NULL &&
1116 		    mp->mnt_op != mnt_op) || !VI_TRYLOCK(vp)) {
1117 			TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_actfreelist);
1118 			continue;
1119 		}
1120 		VNASSERT((vp->v_iflag & VI_FREE) != 0 && vp->v_holdcnt == 0,
1121 		    vp, ("vp inconsistent on freelist"));
1122 
1123 		/*
1124 		 * The clear of VI_FREE prevents activation of the
1125 		 * vnode.  There is no sense in putting the vnode on
1126 		 * the mount point active list, only to remove it
1127 		 * later during recycling.  Inline the relevant part
1128 		 * of vholdl(), to avoid triggering assertions or
1129 		 * activating.
1130 		 */
1131 		freevnodes--;
1132 		vp->v_iflag &= ~VI_FREE;
1133 		VNODE_REFCOUNT_FENCE_REL();
1134 		refcount_acquire(&vp->v_holdcnt);
1135 
1136 		mtx_unlock(&vnode_free_list_mtx);
1137 		VI_UNLOCK(vp);
1138 		vtryrecycle(vp);
1139 		/*
1140 		 * If the recycled succeeded this vdrop will actually free
1141 		 * the vnode.  If not it will simply place it back on
1142 		 * the free list.
1143 		 */
1144 		vdrop(vp);
1145 		mtx_lock(&vnode_free_list_mtx);
1146 	}
1147 }
1148 
1149 void
1150 vnlru_free(int count, struct vfsops *mnt_op)
1151 {
1152 
1153 	mtx_lock(&vnode_free_list_mtx);
1154 	vnlru_free_locked(count, mnt_op);
1155 	mtx_unlock(&vnode_free_list_mtx);
1156 }
1157 
1158 
1159 /* XXX some names and initialization are bad for limits and watermarks. */
1160 static int
1161 vspace(void)
1162 {
1163 	int space;
1164 
1165 	gapvnodes = imax(desiredvnodes - wantfreevnodes, 100);
1166 	vhiwat = gapvnodes / 11; /* 9% -- just under the 10% in vlrureclaim() */
1167 	vlowat = vhiwat / 2;
1168 	if (numvnodes > desiredvnodes)
1169 		return (0);
1170 	space = desiredvnodes - numvnodes;
1171 	if (freevnodes > wantfreevnodes)
1172 		space += freevnodes - wantfreevnodes;
1173 	return (space);
1174 }
1175 
1176 static void
1177 vnlru_return_batch_locked(struct mount *mp)
1178 {
1179 	struct vnode *vp;
1180 
1181 	mtx_assert(&mp->mnt_listmtx, MA_OWNED);
1182 
1183 	if (mp->mnt_tmpfreevnodelistsize == 0)
1184 		return;
1185 
1186 	TAILQ_FOREACH(vp, &mp->mnt_tmpfreevnodelist, v_actfreelist) {
1187 		VNASSERT((vp->v_mflag & VMP_TMPMNTFREELIST) != 0, vp,
1188 		    ("vnode without VMP_TMPMNTFREELIST on mnt_tmpfreevnodelist"));
1189 		vp->v_mflag &= ~VMP_TMPMNTFREELIST;
1190 	}
1191 	mtx_lock(&vnode_free_list_mtx);
1192 	TAILQ_CONCAT(&vnode_free_list, &mp->mnt_tmpfreevnodelist, v_actfreelist);
1193 	freevnodes += mp->mnt_tmpfreevnodelistsize;
1194 	mtx_unlock(&vnode_free_list_mtx);
1195 	mp->mnt_tmpfreevnodelistsize = 0;
1196 }
1197 
1198 static void
1199 vnlru_return_batch(struct mount *mp)
1200 {
1201 
1202 	mtx_lock(&mp->mnt_listmtx);
1203 	vnlru_return_batch_locked(mp);
1204 	mtx_unlock(&mp->mnt_listmtx);
1205 }
1206 
1207 static void
1208 vnlru_return_batches(struct vfsops *mnt_op)
1209 {
1210 	struct mount *mp, *nmp;
1211 	bool need_unbusy;
1212 
1213 	mtx_lock(&mountlist_mtx);
1214 	for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
1215 		need_unbusy = false;
1216 		if (mnt_op != NULL && mp->mnt_op != mnt_op)
1217 			goto next;
1218 		if (mp->mnt_tmpfreevnodelistsize == 0)
1219 			goto next;
1220 		if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK) == 0) {
1221 			vnlru_return_batch(mp);
1222 			need_unbusy = true;
1223 			mtx_lock(&mountlist_mtx);
1224 		}
1225 next:
1226 		nmp = TAILQ_NEXT(mp, mnt_list);
1227 		if (need_unbusy)
1228 			vfs_unbusy(mp);
1229 	}
1230 	mtx_unlock(&mountlist_mtx);
1231 }
1232 
1233 /*
1234  * Attempt to recycle vnodes in a context that is always safe to block.
1235  * Calling vlrurecycle() from the bowels of filesystem code has some
1236  * interesting deadlock problems.
1237  */
1238 static struct proc *vnlruproc;
1239 static int vnlruproc_sig;
1240 
1241 static void
1242 vnlru_proc(void)
1243 {
1244 	struct mount *mp, *nmp;
1245 	unsigned long onumvnodes;
1246 	int done, force, trigger, usevnodes;
1247 	bool reclaim_nc_src;
1248 
1249 	EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, vnlruproc,
1250 	    SHUTDOWN_PRI_FIRST);
1251 
1252 	force = 0;
1253 	for (;;) {
1254 		kproc_suspend_check(vnlruproc);
1255 		mtx_lock(&vnode_free_list_mtx);
1256 		/*
1257 		 * If numvnodes is too large (due to desiredvnodes being
1258 		 * adjusted using its sysctl, or emergency growth), first
1259 		 * try to reduce it by discarding from the free list.
1260 		 */
1261 		if (numvnodes > desiredvnodes)
1262 			vnlru_free_locked(numvnodes - desiredvnodes, NULL);
1263 		/*
1264 		 * Sleep if the vnode cache is in a good state.  This is
1265 		 * when it is not over-full and has space for about a 4%
1266 		 * or 9% expansion (by growing its size or inexcessively
1267 		 * reducing its free list).  Otherwise, try to reclaim
1268 		 * space for a 10% expansion.
1269 		 */
1270 		if (vstir && force == 0) {
1271 			force = 1;
1272 			vstir = 0;
1273 		}
1274 		if (vspace() >= vlowat && force == 0) {
1275 			vnlruproc_sig = 0;
1276 			wakeup(&vnlruproc_sig);
1277 			msleep(vnlruproc, &vnode_free_list_mtx,
1278 			    PVFS|PDROP, "vlruwt", hz);
1279 			continue;
1280 		}
1281 		mtx_unlock(&vnode_free_list_mtx);
1282 		done = 0;
1283 		onumvnodes = numvnodes;
1284 		/*
1285 		 * Calculate parameters for recycling.  These are the same
1286 		 * throughout the loop to give some semblance of fairness.
1287 		 * The trigger point is to avoid recycling vnodes with lots
1288 		 * of resident pages.  We aren't trying to free memory; we
1289 		 * are trying to recycle or at least free vnodes.
1290 		 */
1291 		if (numvnodes <= desiredvnodes)
1292 			usevnodes = numvnodes - freevnodes;
1293 		else
1294 			usevnodes = numvnodes;
1295 		if (usevnodes <= 0)
1296 			usevnodes = 1;
1297 		/*
1298 		 * The trigger value is is chosen to give a conservatively
1299 		 * large value to ensure that it alone doesn't prevent
1300 		 * making progress.  The value can easily be so large that
1301 		 * it is effectively infinite in some congested and
1302 		 * misconfigured cases, and this is necessary.  Normally
1303 		 * it is about 8 to 100 (pages), which is quite large.
1304 		 */
1305 		trigger = vm_cnt.v_page_count * 2 / usevnodes;
1306 		if (force < 2)
1307 			trigger = vsmalltrigger;
1308 		reclaim_nc_src = force >= 3;
1309 		mtx_lock(&mountlist_mtx);
1310 		for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
1311 			if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) {
1312 				nmp = TAILQ_NEXT(mp, mnt_list);
1313 				continue;
1314 			}
1315 			done += vlrureclaim(mp, reclaim_nc_src, trigger);
1316 			mtx_lock(&mountlist_mtx);
1317 			nmp = TAILQ_NEXT(mp, mnt_list);
1318 			vfs_unbusy(mp);
1319 		}
1320 		mtx_unlock(&mountlist_mtx);
1321 		if (onumvnodes > desiredvnodes && numvnodes <= desiredvnodes)
1322 			uma_reclaim(UMA_RECLAIM_DRAIN);
1323 		if (done == 0) {
1324 			if (force == 0 || force == 1) {
1325 				force = 2;
1326 				continue;
1327 			}
1328 			if (force == 2) {
1329 				force = 3;
1330 				continue;
1331 			}
1332 			force = 0;
1333 			vnlru_nowhere++;
1334 			tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3);
1335 		} else
1336 			kern_yield(PRI_USER);
1337 		/*
1338 		 * After becoming active to expand above low water, keep
1339 		 * active until above high water.
1340 		 */
1341 		force = vspace() < vhiwat;
1342 	}
1343 }
1344 
1345 static struct kproc_desc vnlru_kp = {
1346 	"vnlru",
1347 	vnlru_proc,
1348 	&vnlruproc
1349 };
1350 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start,
1351     &vnlru_kp);
1352 
1353 /*
1354  * Routines having to do with the management of the vnode table.
1355  */
1356 
1357 /*
1358  * Try to recycle a freed vnode.  We abort if anyone picks up a reference
1359  * before we actually vgone().  This function must be called with the vnode
1360  * held to prevent the vnode from being returned to the free list midway
1361  * through vgone().
1362  */
1363 static int
1364 vtryrecycle(struct vnode *vp)
1365 {
1366 	struct mount *vnmp;
1367 
1368 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
1369 	VNASSERT(vp->v_holdcnt, vp,
1370 	    ("vtryrecycle: Recycling vp %p without a reference.", vp));
1371 	/*
1372 	 * This vnode may found and locked via some other list, if so we
1373 	 * can't recycle it yet.
1374 	 */
1375 	if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) {
1376 		CTR2(KTR_VFS,
1377 		    "%s: impossible to recycle, vp %p lock is already held",
1378 		    __func__, vp);
1379 		return (EWOULDBLOCK);
1380 	}
1381 	/*
1382 	 * Don't recycle if its filesystem is being suspended.
1383 	 */
1384 	if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) {
1385 		VOP_UNLOCK(vp, 0);
1386 		CTR2(KTR_VFS,
1387 		    "%s: impossible to recycle, cannot start the write for %p",
1388 		    __func__, vp);
1389 		return (EBUSY);
1390 	}
1391 	/*
1392 	 * If we got this far, we need to acquire the interlock and see if
1393 	 * anyone picked up this vnode from another list.  If not, we will
1394 	 * mark it with DOOMED via vgonel() so that anyone who does find it
1395 	 * will skip over it.
1396 	 */
1397 	VI_LOCK(vp);
1398 	if (vp->v_usecount) {
1399 		VOP_UNLOCK(vp, 0);
1400 		VI_UNLOCK(vp);
1401 		vn_finished_write(vnmp);
1402 		CTR2(KTR_VFS,
1403 		    "%s: impossible to recycle, %p is already referenced",
1404 		    __func__, vp);
1405 		return (EBUSY);
1406 	}
1407 	if ((vp->v_iflag & VI_DOOMED) == 0) {
1408 		counter_u64_add(recycles_count, 1);
1409 		vgonel(vp);
1410 	}
1411 	VOP_UNLOCK(vp, 0);
1412 	VI_UNLOCK(vp);
1413 	vn_finished_write(vnmp);
1414 	return (0);
1415 }
1416 
1417 static void
1418 vcheckspace(void)
1419 {
1420 
1421 	if (vspace() < vlowat && vnlruproc_sig == 0) {
1422 		vnlruproc_sig = 1;
1423 		wakeup(vnlruproc);
1424 	}
1425 }
1426 
1427 /*
1428  * Wait if necessary for space for a new vnode.
1429  */
1430 static int
1431 getnewvnode_wait(int suspended)
1432 {
1433 
1434 	mtx_assert(&vnode_free_list_mtx, MA_OWNED);
1435 	if (numvnodes >= desiredvnodes) {
1436 		if (suspended) {
1437 			/*
1438 			 * The file system is being suspended.  We cannot
1439 			 * risk a deadlock here, so allow allocation of
1440 			 * another vnode even if this would give too many.
1441 			 */
1442 			return (0);
1443 		}
1444 		if (vnlruproc_sig == 0) {
1445 			vnlruproc_sig = 1;	/* avoid unnecessary wakeups */
1446 			wakeup(vnlruproc);
1447 		}
1448 		msleep(&vnlruproc_sig, &vnode_free_list_mtx, PVFS,
1449 		    "vlruwk", hz);
1450 	}
1451 	/* Post-adjust like the pre-adjust in getnewvnode(). */
1452 	if (numvnodes + 1 > desiredvnodes && freevnodes > 1)
1453 		vnlru_free_locked(1, NULL);
1454 	return (numvnodes >= desiredvnodes ? ENFILE : 0);
1455 }
1456 
1457 /*
1458  * This hack is fragile, and probably not needed any more now that the
1459  * watermark handling works.
1460  */
1461 void
1462 getnewvnode_reserve(u_int count)
1463 {
1464 	struct thread *td;
1465 
1466 	/* Pre-adjust like the pre-adjust in getnewvnode(), with any count. */
1467 	/* XXX no longer so quick, but this part is not racy. */
1468 	mtx_lock(&vnode_free_list_mtx);
1469 	if (numvnodes + count > desiredvnodes && freevnodes > wantfreevnodes)
1470 		vnlru_free_locked(ulmin(numvnodes + count - desiredvnodes,
1471 		    freevnodes - wantfreevnodes), NULL);
1472 	mtx_unlock(&vnode_free_list_mtx);
1473 
1474 	td = curthread;
1475 	/* First try to be quick and racy. */
1476 	if (atomic_fetchadd_long(&numvnodes, count) + count <= desiredvnodes) {
1477 		td->td_vp_reserv += count;
1478 		vcheckspace();	/* XXX no longer so quick, but more racy */
1479 		return;
1480 	} else
1481 		atomic_subtract_long(&numvnodes, count);
1482 
1483 	mtx_lock(&vnode_free_list_mtx);
1484 	while (count > 0) {
1485 		if (getnewvnode_wait(0) == 0) {
1486 			count--;
1487 			td->td_vp_reserv++;
1488 			atomic_add_long(&numvnodes, 1);
1489 		}
1490 	}
1491 	vcheckspace();
1492 	mtx_unlock(&vnode_free_list_mtx);
1493 }
1494 
1495 /*
1496  * This hack is fragile, especially if desiredvnodes or wantvnodes are
1497  * misconfgured or changed significantly.  Reducing desiredvnodes below
1498  * the reserved amount should cause bizarre behaviour like reducing it
1499  * below the number of active vnodes -- the system will try to reduce
1500  * numvnodes to match, but should fail, so the subtraction below should
1501  * not overflow.
1502  */
1503 void
1504 getnewvnode_drop_reserve(void)
1505 {
1506 	struct thread *td;
1507 
1508 	td = curthread;
1509 	atomic_subtract_long(&numvnodes, td->td_vp_reserv);
1510 	td->td_vp_reserv = 0;
1511 }
1512 
1513 /*
1514  * Return the next vnode from the free list.
1515  */
1516 int
1517 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops,
1518     struct vnode **vpp)
1519 {
1520 	struct vnode *vp;
1521 	struct thread *td;
1522 	struct lock_object *lo;
1523 	static int cyclecount;
1524 	int error __unused;
1525 
1526 	CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag);
1527 	vp = NULL;
1528 	td = curthread;
1529 	if (td->td_vp_reserv > 0) {
1530 		td->td_vp_reserv -= 1;
1531 		goto alloc;
1532 	}
1533 	mtx_lock(&vnode_free_list_mtx);
1534 	if (numvnodes < desiredvnodes)
1535 		cyclecount = 0;
1536 	else if (cyclecount++ >= freevnodes) {
1537 		cyclecount = 0;
1538 		vstir = 1;
1539 	}
1540 	/*
1541 	 * Grow the vnode cache if it will not be above its target max
1542 	 * after growing.  Otherwise, if the free list is nonempty, try
1543 	 * to reclaim 1 item from it before growing the cache (possibly
1544 	 * above its target max if the reclamation failed or is delayed).
1545 	 * Otherwise, wait for some space.  In all cases, schedule
1546 	 * vnlru_proc() if we are getting short of space.  The watermarks
1547 	 * should be chosen so that we never wait or even reclaim from
1548 	 * the free list to below its target minimum.
1549 	 */
1550 	if (numvnodes + 1 <= desiredvnodes)
1551 		;
1552 	else if (freevnodes > 0)
1553 		vnlru_free_locked(1, NULL);
1554 	else {
1555 		error = getnewvnode_wait(mp != NULL && (mp->mnt_kern_flag &
1556 		    MNTK_SUSPEND));
1557 #if 0	/* XXX Not all VFS_VGET/ffs_vget callers check returns. */
1558 		if (error != 0) {
1559 			mtx_unlock(&vnode_free_list_mtx);
1560 			return (error);
1561 		}
1562 #endif
1563 	}
1564 	vcheckspace();
1565 	atomic_add_long(&numvnodes, 1);
1566 	mtx_unlock(&vnode_free_list_mtx);
1567 alloc:
1568 	counter_u64_add(vnodes_created, 1);
1569 	vp = (struct vnode *) uma_zalloc(vnode_zone, M_WAITOK);
1570 	/*
1571 	 * Locks are given the generic name "vnode" when created.
1572 	 * Follow the historic practice of using the filesystem
1573 	 * name when they allocated, e.g., "zfs", "ufs", "nfs, etc.
1574 	 *
1575 	 * Locks live in a witness group keyed on their name. Thus,
1576 	 * when a lock is renamed, it must also move from the witness
1577 	 * group of its old name to the witness group of its new name.
1578 	 *
1579 	 * The change only needs to be made when the vnode moves
1580 	 * from one filesystem type to another. We ensure that each
1581 	 * filesystem use a single static name pointer for its tag so
1582 	 * that we can compare pointers rather than doing a strcmp().
1583 	 */
1584 	lo = &vp->v_vnlock->lock_object;
1585 	if (lo->lo_name != tag) {
1586 		lo->lo_name = tag;
1587 		WITNESS_DESTROY(lo);
1588 		WITNESS_INIT(lo, tag);
1589 	}
1590 	/*
1591 	 * By default, don't allow shared locks unless filesystems opt-in.
1592 	 */
1593 	vp->v_vnlock->lock_object.lo_flags |= LK_NOSHARE;
1594 	/*
1595 	 * Finalize various vnode identity bits.
1596 	 */
1597 	KASSERT(vp->v_object == NULL, ("stale v_object %p", vp));
1598 	KASSERT(vp->v_lockf == NULL, ("stale v_lockf %p", vp));
1599 	KASSERT(vp->v_pollinfo == NULL, ("stale v_pollinfo %p", vp));
1600 	vp->v_type = VNON;
1601 	vp->v_tag = tag;
1602 	vp->v_op = vops;
1603 	v_init_counters(vp);
1604 	vp->v_bufobj.bo_ops = &buf_ops_bio;
1605 #ifdef DIAGNOSTIC
1606 	if (mp == NULL && vops != &dead_vnodeops)
1607 		printf("NULL mp in getnewvnode(9), tag %s\n", tag);
1608 #endif
1609 #ifdef MAC
1610 	mac_vnode_init(vp);
1611 	if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0)
1612 		mac_vnode_associate_singlelabel(mp, vp);
1613 #endif
1614 	if (mp != NULL) {
1615 		vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize;
1616 		if ((mp->mnt_kern_flag & MNTK_NOKNOTE) != 0)
1617 			vp->v_vflag |= VV_NOKNOTE;
1618 	}
1619 
1620 	/*
1621 	 * For the filesystems which do not use vfs_hash_insert(),
1622 	 * still initialize v_hash to have vfs_hash_index() useful.
1623 	 * E.g., nullfs uses vfs_hash_index() on the lower vnode for
1624 	 * its own hashing.
1625 	 */
1626 	vp->v_hash = (uintptr_t)vp >> vnsz2log;
1627 
1628 	*vpp = vp;
1629 	return (0);
1630 }
1631 
1632 /*
1633  * Delete from old mount point vnode list, if on one.
1634  */
1635 static void
1636 delmntque(struct vnode *vp)
1637 {
1638 	struct mount *mp;
1639 	int active;
1640 
1641 	mp = vp->v_mount;
1642 	if (mp == NULL)
1643 		return;
1644 	MNT_ILOCK(mp);
1645 	VI_LOCK(vp);
1646 	KASSERT(mp->mnt_activevnodelistsize <= mp->mnt_nvnodelistsize,
1647 	    ("Active vnode list size %d > Vnode list size %d",
1648 	     mp->mnt_activevnodelistsize, mp->mnt_nvnodelistsize));
1649 	active = vp->v_iflag & VI_ACTIVE;
1650 	vp->v_iflag &= ~VI_ACTIVE;
1651 	if (active) {
1652 		mtx_lock(&mp->mnt_listmtx);
1653 		TAILQ_REMOVE(&mp->mnt_activevnodelist, vp, v_actfreelist);
1654 		mp->mnt_activevnodelistsize--;
1655 		mtx_unlock(&mp->mnt_listmtx);
1656 	}
1657 	vp->v_mount = NULL;
1658 	VI_UNLOCK(vp);
1659 	VNASSERT(mp->mnt_nvnodelistsize > 0, vp,
1660 		("bad mount point vnode list size"));
1661 	TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1662 	mp->mnt_nvnodelistsize--;
1663 	MNT_REL(mp);
1664 	MNT_IUNLOCK(mp);
1665 }
1666 
1667 static void
1668 insmntque_stddtr(struct vnode *vp, void *dtr_arg)
1669 {
1670 
1671 	vp->v_data = NULL;
1672 	vp->v_op = &dead_vnodeops;
1673 	vgone(vp);
1674 	vput(vp);
1675 }
1676 
1677 /*
1678  * Insert into list of vnodes for the new mount point, if available.
1679  */
1680 int
1681 insmntque1(struct vnode *vp, struct mount *mp,
1682 	void (*dtr)(struct vnode *, void *), void *dtr_arg)
1683 {
1684 
1685 	KASSERT(vp->v_mount == NULL,
1686 		("insmntque: vnode already on per mount vnode list"));
1687 	VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)"));
1688 	ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp");
1689 
1690 	/*
1691 	 * We acquire the vnode interlock early to ensure that the
1692 	 * vnode cannot be recycled by another process releasing a
1693 	 * holdcnt on it before we get it on both the vnode list
1694 	 * and the active vnode list. The mount mutex protects only
1695 	 * manipulation of the vnode list and the vnode freelist
1696 	 * mutex protects only manipulation of the active vnode list.
1697 	 * Hence the need to hold the vnode interlock throughout.
1698 	 */
1699 	MNT_ILOCK(mp);
1700 	VI_LOCK(vp);
1701 	if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 &&
1702 	    ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 ||
1703 	    mp->mnt_nvnodelistsize == 0)) &&
1704 	    (vp->v_vflag & VV_FORCEINSMQ) == 0) {
1705 		VI_UNLOCK(vp);
1706 		MNT_IUNLOCK(mp);
1707 		if (dtr != NULL)
1708 			dtr(vp, dtr_arg);
1709 		return (EBUSY);
1710 	}
1711 	vp->v_mount = mp;
1712 	MNT_REF(mp);
1713 	TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1714 	VNASSERT(mp->mnt_nvnodelistsize >= 0, vp,
1715 		("neg mount point vnode list size"));
1716 	mp->mnt_nvnodelistsize++;
1717 	KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
1718 	    ("Activating already active vnode"));
1719 	vp->v_iflag |= VI_ACTIVE;
1720 	mtx_lock(&mp->mnt_listmtx);
1721 	TAILQ_INSERT_HEAD(&mp->mnt_activevnodelist, vp, v_actfreelist);
1722 	mp->mnt_activevnodelistsize++;
1723 	mtx_unlock(&mp->mnt_listmtx);
1724 	VI_UNLOCK(vp);
1725 	MNT_IUNLOCK(mp);
1726 	return (0);
1727 }
1728 
1729 int
1730 insmntque(struct vnode *vp, struct mount *mp)
1731 {
1732 
1733 	return (insmntque1(vp, mp, insmntque_stddtr, NULL));
1734 }
1735 
1736 /*
1737  * Flush out and invalidate all buffers associated with a bufobj
1738  * Called with the underlying object locked.
1739  */
1740 int
1741 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
1742 {
1743 	int error;
1744 
1745 	BO_LOCK(bo);
1746 	if (flags & V_SAVE) {
1747 		error = bufobj_wwait(bo, slpflag, slptimeo);
1748 		if (error) {
1749 			BO_UNLOCK(bo);
1750 			return (error);
1751 		}
1752 		if (bo->bo_dirty.bv_cnt > 0) {
1753 			BO_UNLOCK(bo);
1754 			if ((error = BO_SYNC(bo, MNT_WAIT)) != 0)
1755 				return (error);
1756 			/*
1757 			 * XXX We could save a lock/unlock if this was only
1758 			 * enabled under INVARIANTS
1759 			 */
1760 			BO_LOCK(bo);
1761 			if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0)
1762 				panic("vinvalbuf: dirty bufs");
1763 		}
1764 	}
1765 	/*
1766 	 * If you alter this loop please notice that interlock is dropped and
1767 	 * reacquired in flushbuflist.  Special care is needed to ensure that
1768 	 * no race conditions occur from this.
1769 	 */
1770 	do {
1771 		error = flushbuflist(&bo->bo_clean,
1772 		    flags, bo, slpflag, slptimeo);
1773 		if (error == 0 && !(flags & V_CLEANONLY))
1774 			error = flushbuflist(&bo->bo_dirty,
1775 			    flags, bo, slpflag, slptimeo);
1776 		if (error != 0 && error != EAGAIN) {
1777 			BO_UNLOCK(bo);
1778 			return (error);
1779 		}
1780 	} while (error != 0);
1781 
1782 	/*
1783 	 * Wait for I/O to complete.  XXX needs cleaning up.  The vnode can
1784 	 * have write I/O in-progress but if there is a VM object then the
1785 	 * VM object can also have read-I/O in-progress.
1786 	 */
1787 	do {
1788 		bufobj_wwait(bo, 0, 0);
1789 		if ((flags & V_VMIO) == 0 && bo->bo_object != NULL) {
1790 			BO_UNLOCK(bo);
1791 			vm_object_pip_wait_unlocked(bo->bo_object, "bovlbx");
1792 			BO_LOCK(bo);
1793 		}
1794 	} while (bo->bo_numoutput > 0);
1795 	BO_UNLOCK(bo);
1796 
1797 	/*
1798 	 * Destroy the copy in the VM cache, too.
1799 	 */
1800 	if (bo->bo_object != NULL &&
1801 	    (flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0) {
1802 		VM_OBJECT_WLOCK(bo->bo_object);
1803 		vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ?
1804 		    OBJPR_CLEANONLY : 0);
1805 		VM_OBJECT_WUNLOCK(bo->bo_object);
1806 	}
1807 
1808 #ifdef INVARIANTS
1809 	BO_LOCK(bo);
1810 	if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO |
1811 	    V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 ||
1812 	    bo->bo_clean.bv_cnt > 0))
1813 		panic("vinvalbuf: flush failed");
1814 	if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0 &&
1815 	    bo->bo_dirty.bv_cnt > 0)
1816 		panic("vinvalbuf: flush dirty failed");
1817 	BO_UNLOCK(bo);
1818 #endif
1819 	return (0);
1820 }
1821 
1822 /*
1823  * Flush out and invalidate all buffers associated with a vnode.
1824  * Called with the underlying object locked.
1825  */
1826 int
1827 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
1828 {
1829 
1830 	CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
1831 	ASSERT_VOP_LOCKED(vp, "vinvalbuf");
1832 	if (vp->v_object != NULL && vp->v_object->handle != vp)
1833 		return (0);
1834 	return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo));
1835 }
1836 
1837 /*
1838  * Flush out buffers on the specified list.
1839  *
1840  */
1841 static int
1842 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag,
1843     int slptimeo)
1844 {
1845 	struct buf *bp, *nbp;
1846 	int retval, error;
1847 	daddr_t lblkno;
1848 	b_xflags_t xflags;
1849 
1850 	ASSERT_BO_WLOCKED(bo);
1851 
1852 	retval = 0;
1853 	TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) {
1854 		/*
1855 		 * If we are flushing both V_NORMAL and V_ALT buffers then
1856 		 * do not skip any buffers. If we are flushing only V_NORMAL
1857 		 * buffers then skip buffers marked as BX_ALTDATA. If we are
1858 		 * flushing only V_ALT buffers then skip buffers not marked
1859 		 * as BX_ALTDATA.
1860 		 */
1861 		if (((flags & (V_NORMAL | V_ALT)) != (V_NORMAL | V_ALT)) &&
1862 		   (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) ||
1863 		    ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) {
1864 			continue;
1865 		}
1866 		if (nbp != NULL) {
1867 			lblkno = nbp->b_lblkno;
1868 			xflags = nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN);
1869 		}
1870 		retval = EAGAIN;
1871 		error = BUF_TIMELOCK(bp,
1872 		    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo),
1873 		    "flushbuf", slpflag, slptimeo);
1874 		if (error) {
1875 			BO_LOCK(bo);
1876 			return (error != ENOLCK ? error : EAGAIN);
1877 		}
1878 		KASSERT(bp->b_bufobj == bo,
1879 		    ("bp %p wrong b_bufobj %p should be %p",
1880 		    bp, bp->b_bufobj, bo));
1881 		/*
1882 		 * XXX Since there are no node locks for NFS, I
1883 		 * believe there is a slight chance that a delayed
1884 		 * write will occur while sleeping just above, so
1885 		 * check for it.
1886 		 */
1887 		if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
1888 		    (flags & V_SAVE)) {
1889 			bremfree(bp);
1890 			bp->b_flags |= B_ASYNC;
1891 			bwrite(bp);
1892 			BO_LOCK(bo);
1893 			return (EAGAIN);	/* XXX: why not loop ? */
1894 		}
1895 		bremfree(bp);
1896 		bp->b_flags |= (B_INVAL | B_RELBUF);
1897 		bp->b_flags &= ~B_ASYNC;
1898 		brelse(bp);
1899 		BO_LOCK(bo);
1900 		if (nbp == NULL)
1901 			break;
1902 		nbp = gbincore(bo, lblkno);
1903 		if (nbp == NULL || (nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
1904 		    != xflags)
1905 			break;			/* nbp invalid */
1906 	}
1907 	return (retval);
1908 }
1909 
1910 int
1911 bnoreuselist(struct bufv *bufv, struct bufobj *bo, daddr_t startn, daddr_t endn)
1912 {
1913 	struct buf *bp;
1914 	int error;
1915 	daddr_t lblkno;
1916 
1917 	ASSERT_BO_LOCKED(bo);
1918 
1919 	for (lblkno = startn;;) {
1920 again:
1921 		bp = BUF_PCTRIE_LOOKUP_GE(&bufv->bv_root, lblkno);
1922 		if (bp == NULL || bp->b_lblkno >= endn ||
1923 		    bp->b_lblkno < startn)
1924 			break;
1925 		error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL |
1926 		    LK_INTERLOCK, BO_LOCKPTR(bo), "brlsfl", 0, 0);
1927 		if (error != 0) {
1928 			BO_RLOCK(bo);
1929 			if (error == ENOLCK)
1930 				goto again;
1931 			return (error);
1932 		}
1933 		KASSERT(bp->b_bufobj == bo,
1934 		    ("bp %p wrong b_bufobj %p should be %p",
1935 		    bp, bp->b_bufobj, bo));
1936 		lblkno = bp->b_lblkno + 1;
1937 		if ((bp->b_flags & B_MANAGED) == 0)
1938 			bremfree(bp);
1939 		bp->b_flags |= B_RELBUF;
1940 		/*
1941 		 * In the VMIO case, use the B_NOREUSE flag to hint that the
1942 		 * pages backing each buffer in the range are unlikely to be
1943 		 * reused.  Dirty buffers will have the hint applied once
1944 		 * they've been written.
1945 		 */
1946 		if ((bp->b_flags & B_VMIO) != 0)
1947 			bp->b_flags |= B_NOREUSE;
1948 		brelse(bp);
1949 		BO_RLOCK(bo);
1950 	}
1951 	return (0);
1952 }
1953 
1954 /*
1955  * Truncate a file's buffer and pages to a specified length.  This
1956  * is in lieu of the old vinvalbuf mechanism, which performed unneeded
1957  * sync activity.
1958  */
1959 int
1960 vtruncbuf(struct vnode *vp, off_t length, int blksize)
1961 {
1962 	struct buf *bp, *nbp;
1963 	struct bufobj *bo;
1964 	daddr_t startlbn;
1965 
1966 	CTR4(KTR_VFS, "%s: vp %p with block %d:%ju", __func__,
1967 	    vp, blksize, (uintmax_t)length);
1968 
1969 	/*
1970 	 * Round up to the *next* lbn.
1971 	 */
1972 	startlbn = howmany(length, blksize);
1973 
1974 	ASSERT_VOP_LOCKED(vp, "vtruncbuf");
1975 
1976 	bo = &vp->v_bufobj;
1977 restart_unlocked:
1978 	BO_LOCK(bo);
1979 
1980 	while (v_inval_buf_range_locked(vp, bo, startlbn, INT64_MAX) == EAGAIN)
1981 		;
1982 
1983 	if (length > 0) {
1984 restartsync:
1985 		TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
1986 			if (bp->b_lblkno > 0)
1987 				continue;
1988 			/*
1989 			 * Since we hold the vnode lock this should only
1990 			 * fail if we're racing with the buf daemon.
1991 			 */
1992 			if (BUF_LOCK(bp,
1993 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1994 			    BO_LOCKPTR(bo)) == ENOLCK)
1995 				goto restart_unlocked;
1996 
1997 			VNASSERT((bp->b_flags & B_DELWRI), vp,
1998 			    ("buf(%p) on dirty queue without DELWRI", bp));
1999 
2000 			bremfree(bp);
2001 			bawrite(bp);
2002 			BO_LOCK(bo);
2003 			goto restartsync;
2004 		}
2005 	}
2006 
2007 	bufobj_wwait(bo, 0, 0);
2008 	BO_UNLOCK(bo);
2009 	vnode_pager_setsize(vp, length);
2010 
2011 	return (0);
2012 }
2013 
2014 /*
2015  * Invalidate the cached pages of a file's buffer within the range of block
2016  * numbers [startlbn, endlbn).
2017  */
2018 void
2019 v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn,
2020     int blksize)
2021 {
2022 	struct bufobj *bo;
2023 	off_t start, end;
2024 
2025 	ASSERT_VOP_LOCKED(vp, "v_inval_buf_range");
2026 
2027 	start = blksize * startlbn;
2028 	end = blksize * endlbn;
2029 
2030 	bo = &vp->v_bufobj;
2031 	BO_LOCK(bo);
2032 	MPASS(blksize == bo->bo_bsize);
2033 
2034 	while (v_inval_buf_range_locked(vp, bo, startlbn, endlbn) == EAGAIN)
2035 		;
2036 
2037 	BO_UNLOCK(bo);
2038 	vn_pages_remove(vp, OFF_TO_IDX(start), OFF_TO_IDX(end + PAGE_SIZE - 1));
2039 }
2040 
2041 static int
2042 v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo,
2043     daddr_t startlbn, daddr_t endlbn)
2044 {
2045 	struct buf *bp, *nbp;
2046 	bool anyfreed;
2047 
2048 	ASSERT_VOP_LOCKED(vp, "v_inval_buf_range_locked");
2049 	ASSERT_BO_LOCKED(bo);
2050 
2051 	do {
2052 		anyfreed = false;
2053 		TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) {
2054 			if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn)
2055 				continue;
2056 			if (BUF_LOCK(bp,
2057 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
2058 			    BO_LOCKPTR(bo)) == ENOLCK) {
2059 				BO_LOCK(bo);
2060 				return (EAGAIN);
2061 			}
2062 
2063 			bremfree(bp);
2064 			bp->b_flags |= B_INVAL | B_RELBUF;
2065 			bp->b_flags &= ~B_ASYNC;
2066 			brelse(bp);
2067 			anyfreed = true;
2068 
2069 			BO_LOCK(bo);
2070 			if (nbp != NULL &&
2071 			    (((nbp->b_xflags & BX_VNCLEAN) == 0) ||
2072 			    nbp->b_vp != vp ||
2073 			    (nbp->b_flags & B_DELWRI) != 0))
2074 				return (EAGAIN);
2075 		}
2076 
2077 		TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
2078 			if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn)
2079 				continue;
2080 			if (BUF_LOCK(bp,
2081 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
2082 			    BO_LOCKPTR(bo)) == ENOLCK) {
2083 				BO_LOCK(bo);
2084 				return (EAGAIN);
2085 			}
2086 			bremfree(bp);
2087 			bp->b_flags |= B_INVAL | B_RELBUF;
2088 			bp->b_flags &= ~B_ASYNC;
2089 			brelse(bp);
2090 			anyfreed = true;
2091 
2092 			BO_LOCK(bo);
2093 			if (nbp != NULL &&
2094 			    (((nbp->b_xflags & BX_VNDIRTY) == 0) ||
2095 			    (nbp->b_vp != vp) ||
2096 			    (nbp->b_flags & B_DELWRI) == 0))
2097 				return (EAGAIN);
2098 		}
2099 	} while (anyfreed);
2100 	return (0);
2101 }
2102 
2103 static void
2104 buf_vlist_remove(struct buf *bp)
2105 {
2106 	struct bufv *bv;
2107 
2108 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2109 	ASSERT_BO_WLOCKED(bp->b_bufobj);
2110 	KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) !=
2111 	    (BX_VNDIRTY|BX_VNCLEAN),
2112 	    ("buf_vlist_remove: Buf %p is on two lists", bp));
2113 	if (bp->b_xflags & BX_VNDIRTY)
2114 		bv = &bp->b_bufobj->bo_dirty;
2115 	else
2116 		bv = &bp->b_bufobj->bo_clean;
2117 	BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno);
2118 	TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs);
2119 	bv->bv_cnt--;
2120 	bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN);
2121 }
2122 
2123 /*
2124  * Add the buffer to the sorted clean or dirty block list.
2125  *
2126  * NOTE: xflags is passed as a constant, optimizing this inline function!
2127  */
2128 static void
2129 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags)
2130 {
2131 	struct bufv *bv;
2132 	struct buf *n;
2133 	int error;
2134 
2135 	ASSERT_BO_WLOCKED(bo);
2136 	KASSERT((xflags & BX_VNDIRTY) == 0 || (bo->bo_flag & BO_DEAD) == 0,
2137 	    ("dead bo %p", bo));
2138 	KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0,
2139 	    ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags));
2140 	bp->b_xflags |= xflags;
2141 	if (xflags & BX_VNDIRTY)
2142 		bv = &bo->bo_dirty;
2143 	else
2144 		bv = &bo->bo_clean;
2145 
2146 	/*
2147 	 * Keep the list ordered.  Optimize empty list insertion.  Assume
2148 	 * we tend to grow at the tail so lookup_le should usually be cheaper
2149 	 * than _ge.
2150 	 */
2151 	if (bv->bv_cnt == 0 ||
2152 	    bp->b_lblkno > TAILQ_LAST(&bv->bv_hd, buflists)->b_lblkno)
2153 		TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs);
2154 	else if ((n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, bp->b_lblkno)) == NULL)
2155 		TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs);
2156 	else
2157 		TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs);
2158 	error = BUF_PCTRIE_INSERT(&bv->bv_root, bp);
2159 	if (error)
2160 		panic("buf_vlist_add:  Preallocated nodes insufficient.");
2161 	bv->bv_cnt++;
2162 }
2163 
2164 /*
2165  * Look up a buffer using the buffer tries.
2166  */
2167 struct buf *
2168 gbincore(struct bufobj *bo, daddr_t lblkno)
2169 {
2170 	struct buf *bp;
2171 
2172 	ASSERT_BO_LOCKED(bo);
2173 	bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno);
2174 	if (bp != NULL)
2175 		return (bp);
2176 	return BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno);
2177 }
2178 
2179 /*
2180  * Associate a buffer with a vnode.
2181  */
2182 void
2183 bgetvp(struct vnode *vp, struct buf *bp)
2184 {
2185 	struct bufobj *bo;
2186 
2187 	bo = &vp->v_bufobj;
2188 	ASSERT_BO_WLOCKED(bo);
2189 	VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free"));
2190 
2191 	CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags);
2192 	VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp,
2193 	    ("bgetvp: bp already attached! %p", bp));
2194 
2195 	vhold(vp);
2196 	bp->b_vp = vp;
2197 	bp->b_bufobj = bo;
2198 	/*
2199 	 * Insert onto list for new vnode.
2200 	 */
2201 	buf_vlist_add(bp, bo, BX_VNCLEAN);
2202 }
2203 
2204 /*
2205  * Disassociate a buffer from a vnode.
2206  */
2207 void
2208 brelvp(struct buf *bp)
2209 {
2210 	struct bufobj *bo;
2211 	struct vnode *vp;
2212 
2213 	CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2214 	KASSERT(bp->b_vp != NULL, ("brelvp: NULL"));
2215 
2216 	/*
2217 	 * Delete from old vnode list, if on one.
2218 	 */
2219 	vp = bp->b_vp;		/* XXX */
2220 	bo = bp->b_bufobj;
2221 	BO_LOCK(bo);
2222 	if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
2223 		buf_vlist_remove(bp);
2224 	else
2225 		panic("brelvp: Buffer %p not on queue.", bp);
2226 	if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
2227 		bo->bo_flag &= ~BO_ONWORKLST;
2228 		mtx_lock(&sync_mtx);
2229 		LIST_REMOVE(bo, bo_synclist);
2230 		syncer_worklist_len--;
2231 		mtx_unlock(&sync_mtx);
2232 	}
2233 	bp->b_vp = NULL;
2234 	bp->b_bufobj = NULL;
2235 	BO_UNLOCK(bo);
2236 	vdrop(vp);
2237 }
2238 
2239 /*
2240  * Add an item to the syncer work queue.
2241  */
2242 static void
2243 vn_syncer_add_to_worklist(struct bufobj *bo, int delay)
2244 {
2245 	int slot;
2246 
2247 	ASSERT_BO_WLOCKED(bo);
2248 
2249 	mtx_lock(&sync_mtx);
2250 	if (bo->bo_flag & BO_ONWORKLST)
2251 		LIST_REMOVE(bo, bo_synclist);
2252 	else {
2253 		bo->bo_flag |= BO_ONWORKLST;
2254 		syncer_worklist_len++;
2255 	}
2256 
2257 	if (delay > syncer_maxdelay - 2)
2258 		delay = syncer_maxdelay - 2;
2259 	slot = (syncer_delayno + delay) & syncer_mask;
2260 
2261 	LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist);
2262 	mtx_unlock(&sync_mtx);
2263 }
2264 
2265 static int
2266 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS)
2267 {
2268 	int error, len;
2269 
2270 	mtx_lock(&sync_mtx);
2271 	len = syncer_worklist_len - sync_vnode_count;
2272 	mtx_unlock(&sync_mtx);
2273 	error = SYSCTL_OUT(req, &len, sizeof(len));
2274 	return (error);
2275 }
2276 
2277 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, CTLTYPE_INT | CTLFLAG_RD, NULL, 0,
2278     sysctl_vfs_worklist_len, "I", "Syncer thread worklist length");
2279 
2280 static struct proc *updateproc;
2281 static void sched_sync(void);
2282 static struct kproc_desc up_kp = {
2283 	"syncer",
2284 	sched_sync,
2285 	&updateproc
2286 };
2287 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp);
2288 
2289 static int
2290 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td)
2291 {
2292 	struct vnode *vp;
2293 	struct mount *mp;
2294 
2295 	*bo = LIST_FIRST(slp);
2296 	if (*bo == NULL)
2297 		return (0);
2298 	vp = bo2vnode(*bo);
2299 	if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0)
2300 		return (1);
2301 	/*
2302 	 * We use vhold in case the vnode does not
2303 	 * successfully sync.  vhold prevents the vnode from
2304 	 * going away when we unlock the sync_mtx so that
2305 	 * we can acquire the vnode interlock.
2306 	 */
2307 	vholdl(vp);
2308 	mtx_unlock(&sync_mtx);
2309 	VI_UNLOCK(vp);
2310 	if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
2311 		vdrop(vp);
2312 		mtx_lock(&sync_mtx);
2313 		return (*bo == LIST_FIRST(slp));
2314 	}
2315 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2316 	(void) VOP_FSYNC(vp, MNT_LAZY, td);
2317 	VOP_UNLOCK(vp, 0);
2318 	vn_finished_write(mp);
2319 	BO_LOCK(*bo);
2320 	if (((*bo)->bo_flag & BO_ONWORKLST) != 0) {
2321 		/*
2322 		 * Put us back on the worklist.  The worklist
2323 		 * routine will remove us from our current
2324 		 * position and then add us back in at a later
2325 		 * position.
2326 		 */
2327 		vn_syncer_add_to_worklist(*bo, syncdelay);
2328 	}
2329 	BO_UNLOCK(*bo);
2330 	vdrop(vp);
2331 	mtx_lock(&sync_mtx);
2332 	return (0);
2333 }
2334 
2335 static int first_printf = 1;
2336 
2337 /*
2338  * System filesystem synchronizer daemon.
2339  */
2340 static void
2341 sched_sync(void)
2342 {
2343 	struct synclist *next, *slp;
2344 	struct bufobj *bo;
2345 	long starttime;
2346 	struct thread *td = curthread;
2347 	int last_work_seen;
2348 	int net_worklist_len;
2349 	int syncer_final_iter;
2350 	int error;
2351 
2352 	last_work_seen = 0;
2353 	syncer_final_iter = 0;
2354 	syncer_state = SYNCER_RUNNING;
2355 	starttime = time_uptime;
2356 	td->td_pflags |= TDP_NORUNNINGBUF;
2357 
2358 	EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc,
2359 	    SHUTDOWN_PRI_LAST);
2360 
2361 	mtx_lock(&sync_mtx);
2362 	for (;;) {
2363 		if (syncer_state == SYNCER_FINAL_DELAY &&
2364 		    syncer_final_iter == 0) {
2365 			mtx_unlock(&sync_mtx);
2366 			kproc_suspend_check(td->td_proc);
2367 			mtx_lock(&sync_mtx);
2368 		}
2369 		net_worklist_len = syncer_worklist_len - sync_vnode_count;
2370 		if (syncer_state != SYNCER_RUNNING &&
2371 		    starttime != time_uptime) {
2372 			if (first_printf) {
2373 				printf("\nSyncing disks, vnodes remaining... ");
2374 				first_printf = 0;
2375 			}
2376 			printf("%d ", net_worklist_len);
2377 		}
2378 		starttime = time_uptime;
2379 
2380 		/*
2381 		 * Push files whose dirty time has expired.  Be careful
2382 		 * of interrupt race on slp queue.
2383 		 *
2384 		 * Skip over empty worklist slots when shutting down.
2385 		 */
2386 		do {
2387 			slp = &syncer_workitem_pending[syncer_delayno];
2388 			syncer_delayno += 1;
2389 			if (syncer_delayno == syncer_maxdelay)
2390 				syncer_delayno = 0;
2391 			next = &syncer_workitem_pending[syncer_delayno];
2392 			/*
2393 			 * If the worklist has wrapped since the
2394 			 * it was emptied of all but syncer vnodes,
2395 			 * switch to the FINAL_DELAY state and run
2396 			 * for one more second.
2397 			 */
2398 			if (syncer_state == SYNCER_SHUTTING_DOWN &&
2399 			    net_worklist_len == 0 &&
2400 			    last_work_seen == syncer_delayno) {
2401 				syncer_state = SYNCER_FINAL_DELAY;
2402 				syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP;
2403 			}
2404 		} while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) &&
2405 		    syncer_worklist_len > 0);
2406 
2407 		/*
2408 		 * Keep track of the last time there was anything
2409 		 * on the worklist other than syncer vnodes.
2410 		 * Return to the SHUTTING_DOWN state if any
2411 		 * new work appears.
2412 		 */
2413 		if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING)
2414 			last_work_seen = syncer_delayno;
2415 		if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY)
2416 			syncer_state = SYNCER_SHUTTING_DOWN;
2417 		while (!LIST_EMPTY(slp)) {
2418 			error = sync_vnode(slp, &bo, td);
2419 			if (error == 1) {
2420 				LIST_REMOVE(bo, bo_synclist);
2421 				LIST_INSERT_HEAD(next, bo, bo_synclist);
2422 				continue;
2423 			}
2424 
2425 			if (first_printf == 0) {
2426 				/*
2427 				 * Drop the sync mutex, because some watchdog
2428 				 * drivers need to sleep while patting
2429 				 */
2430 				mtx_unlock(&sync_mtx);
2431 				wdog_kern_pat(WD_LASTVAL);
2432 				mtx_lock(&sync_mtx);
2433 			}
2434 
2435 		}
2436 		if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0)
2437 			syncer_final_iter--;
2438 		/*
2439 		 * The variable rushjob allows the kernel to speed up the
2440 		 * processing of the filesystem syncer process. A rushjob
2441 		 * value of N tells the filesystem syncer to process the next
2442 		 * N seconds worth of work on its queue ASAP. Currently rushjob
2443 		 * is used by the soft update code to speed up the filesystem
2444 		 * syncer process when the incore state is getting so far
2445 		 * ahead of the disk that the kernel memory pool is being
2446 		 * threatened with exhaustion.
2447 		 */
2448 		if (rushjob > 0) {
2449 			rushjob -= 1;
2450 			continue;
2451 		}
2452 		/*
2453 		 * Just sleep for a short period of time between
2454 		 * iterations when shutting down to allow some I/O
2455 		 * to happen.
2456 		 *
2457 		 * If it has taken us less than a second to process the
2458 		 * current work, then wait. Otherwise start right over
2459 		 * again. We can still lose time if any single round
2460 		 * takes more than two seconds, but it does not really
2461 		 * matter as we are just trying to generally pace the
2462 		 * filesystem activity.
2463 		 */
2464 		if (syncer_state != SYNCER_RUNNING ||
2465 		    time_uptime == starttime) {
2466 			thread_lock(td);
2467 			sched_prio(td, PPAUSE);
2468 			thread_unlock(td);
2469 		}
2470 		if (syncer_state != SYNCER_RUNNING)
2471 			cv_timedwait(&sync_wakeup, &sync_mtx,
2472 			    hz / SYNCER_SHUTDOWN_SPEEDUP);
2473 		else if (time_uptime == starttime)
2474 			cv_timedwait(&sync_wakeup, &sync_mtx, hz);
2475 	}
2476 }
2477 
2478 /*
2479  * Request the syncer daemon to speed up its work.
2480  * We never push it to speed up more than half of its
2481  * normal turn time, otherwise it could take over the cpu.
2482  */
2483 int
2484 speedup_syncer(void)
2485 {
2486 	int ret = 0;
2487 
2488 	mtx_lock(&sync_mtx);
2489 	if (rushjob < syncdelay / 2) {
2490 		rushjob += 1;
2491 		stat_rush_requests += 1;
2492 		ret = 1;
2493 	}
2494 	mtx_unlock(&sync_mtx);
2495 	cv_broadcast(&sync_wakeup);
2496 	return (ret);
2497 }
2498 
2499 /*
2500  * Tell the syncer to speed up its work and run though its work
2501  * list several times, then tell it to shut down.
2502  */
2503 static void
2504 syncer_shutdown(void *arg, int howto)
2505 {
2506 
2507 	if (howto & RB_NOSYNC)
2508 		return;
2509 	mtx_lock(&sync_mtx);
2510 	syncer_state = SYNCER_SHUTTING_DOWN;
2511 	rushjob = 0;
2512 	mtx_unlock(&sync_mtx);
2513 	cv_broadcast(&sync_wakeup);
2514 	kproc_shutdown(arg, howto);
2515 }
2516 
2517 void
2518 syncer_suspend(void)
2519 {
2520 
2521 	syncer_shutdown(updateproc, 0);
2522 }
2523 
2524 void
2525 syncer_resume(void)
2526 {
2527 
2528 	mtx_lock(&sync_mtx);
2529 	first_printf = 1;
2530 	syncer_state = SYNCER_RUNNING;
2531 	mtx_unlock(&sync_mtx);
2532 	cv_broadcast(&sync_wakeup);
2533 	kproc_resume(updateproc);
2534 }
2535 
2536 /*
2537  * Reassign a buffer from one vnode to another.
2538  * Used to assign file specific control information
2539  * (indirect blocks) to the vnode to which they belong.
2540  */
2541 void
2542 reassignbuf(struct buf *bp)
2543 {
2544 	struct vnode *vp;
2545 	struct bufobj *bo;
2546 	int delay;
2547 #ifdef INVARIANTS
2548 	struct bufv *bv;
2549 #endif
2550 
2551 	vp = bp->b_vp;
2552 	bo = bp->b_bufobj;
2553 	++reassignbufcalls;
2554 
2555 	CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X",
2556 	    bp, bp->b_vp, bp->b_flags);
2557 	/*
2558 	 * B_PAGING flagged buffers cannot be reassigned because their vp
2559 	 * is not fully linked in.
2560 	 */
2561 	if (bp->b_flags & B_PAGING)
2562 		panic("cannot reassign paging buffer");
2563 
2564 	/*
2565 	 * Delete from old vnode list, if on one.
2566 	 */
2567 	BO_LOCK(bo);
2568 	if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
2569 		buf_vlist_remove(bp);
2570 	else
2571 		panic("reassignbuf: Buffer %p not on queue.", bp);
2572 	/*
2573 	 * If dirty, put on list of dirty buffers; otherwise insert onto list
2574 	 * of clean buffers.
2575 	 */
2576 	if (bp->b_flags & B_DELWRI) {
2577 		if ((bo->bo_flag & BO_ONWORKLST) == 0) {
2578 			switch (vp->v_type) {
2579 			case VDIR:
2580 				delay = dirdelay;
2581 				break;
2582 			case VCHR:
2583 				delay = metadelay;
2584 				break;
2585 			default:
2586 				delay = filedelay;
2587 			}
2588 			vn_syncer_add_to_worklist(bo, delay);
2589 		}
2590 		buf_vlist_add(bp, bo, BX_VNDIRTY);
2591 	} else {
2592 		buf_vlist_add(bp, bo, BX_VNCLEAN);
2593 
2594 		if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
2595 			mtx_lock(&sync_mtx);
2596 			LIST_REMOVE(bo, bo_synclist);
2597 			syncer_worklist_len--;
2598 			mtx_unlock(&sync_mtx);
2599 			bo->bo_flag &= ~BO_ONWORKLST;
2600 		}
2601 	}
2602 #ifdef INVARIANTS
2603 	bv = &bo->bo_clean;
2604 	bp = TAILQ_FIRST(&bv->bv_hd);
2605 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2606 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2607 	bp = TAILQ_LAST(&bv->bv_hd, buflists);
2608 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2609 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2610 	bv = &bo->bo_dirty;
2611 	bp = TAILQ_FIRST(&bv->bv_hd);
2612 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2613 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2614 	bp = TAILQ_LAST(&bv->bv_hd, buflists);
2615 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2616 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2617 #endif
2618 	BO_UNLOCK(bo);
2619 }
2620 
2621 static void
2622 v_init_counters(struct vnode *vp)
2623 {
2624 
2625 	VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0,
2626 	    vp, ("%s called for an initialized vnode", __FUNCTION__));
2627 	ASSERT_VI_UNLOCKED(vp, __FUNCTION__);
2628 
2629 	refcount_init(&vp->v_holdcnt, 1);
2630 	refcount_init(&vp->v_usecount, 1);
2631 }
2632 
2633 /*
2634  * Increment si_usecount of the associated device, if any.
2635  */
2636 static void
2637 v_incr_devcount(struct vnode *vp)
2638 {
2639 
2640 	ASSERT_VI_LOCKED(vp, __FUNCTION__);
2641 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2642 		dev_lock();
2643 		vp->v_rdev->si_usecount++;
2644 		dev_unlock();
2645 	}
2646 }
2647 
2648 /*
2649  * Decrement si_usecount of the associated device, if any.
2650  */
2651 static void
2652 v_decr_devcount(struct vnode *vp)
2653 {
2654 
2655 	ASSERT_VI_LOCKED(vp, __FUNCTION__);
2656 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2657 		dev_lock();
2658 		vp->v_rdev->si_usecount--;
2659 		dev_unlock();
2660 	}
2661 }
2662 
2663 /*
2664  * Grab a particular vnode from the free list, increment its
2665  * reference count and lock it.  VI_DOOMED is set if the vnode
2666  * is being destroyed.  Only callers who specify LK_RETRY will
2667  * see doomed vnodes.  If inactive processing was delayed in
2668  * vput try to do it here.
2669  *
2670  * Notes on lockless counter manipulation:
2671  * _vhold, vputx and other routines make various decisions based
2672  * on either holdcnt or usecount being 0. As long as either counter
2673  * is not transitioning 0->1 nor 1->0, the manipulation can be done
2674  * with atomic operations. Otherwise the interlock is taken covering
2675  * both the atomic and additional actions.
2676  */
2677 static enum vgetstate
2678 _vget_prep(struct vnode *vp, bool interlock)
2679 {
2680 	enum vgetstate vs;
2681 
2682 	if (__predict_true(vp->v_type != VCHR)) {
2683 		if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
2684 			vs = VGET_USECOUNT;
2685 		} else {
2686 			_vhold(vp, interlock);
2687 			vs = VGET_HOLDCNT;
2688 		}
2689 	} else {
2690 		if (!interlock)
2691 			VI_LOCK(vp);
2692 		if (vp->v_usecount == 0) {
2693 			vholdl(vp);
2694 			vs = VGET_HOLDCNT;
2695 		} else {
2696 			v_incr_devcount(vp);
2697 			refcount_acquire(&vp->v_usecount);
2698 			vs = VGET_USECOUNT;
2699 		}
2700 		if (!interlock)
2701 			VI_UNLOCK(vp);
2702 	}
2703 	return (vs);
2704 }
2705 
2706 enum vgetstate
2707 vget_prep(struct vnode *vp)
2708 {
2709 
2710 	return (_vget_prep(vp, false));
2711 }
2712 
2713 int
2714 vget(struct vnode *vp, int flags, struct thread *td)
2715 {
2716 	enum vgetstate vs;
2717 
2718 	MPASS(td == curthread);
2719 
2720 	vs = _vget_prep(vp, (flags & LK_INTERLOCK) != 0);
2721 	return (vget_finish(vp, flags, vs));
2722 }
2723 
2724 int
2725 vget_finish(struct vnode *vp, int flags, enum vgetstate vs)
2726 {
2727 	int error, oweinact;
2728 
2729 	VNASSERT((flags & LK_TYPE_MASK) != 0, vp,
2730 	    ("%s: invalid lock operation", __func__));
2731 
2732 	if ((flags & LK_INTERLOCK) != 0)
2733 		ASSERT_VI_LOCKED(vp, __func__);
2734 	else
2735 		ASSERT_VI_UNLOCKED(vp, __func__);
2736 	VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__));
2737 	if (vs == VGET_USECOUNT) {
2738 		VNASSERT(vp->v_usecount > 0, vp,
2739 		    ("%s: vnode without usecount when VGET_USECOUNT was passed",
2740 		    __func__));
2741 	}
2742 
2743 	if ((error = vn_lock(vp, flags)) != 0) {
2744 		if (vs == VGET_USECOUNT)
2745 			vrele(vp);
2746 		else
2747 			vdrop(vp);
2748 		CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__,
2749 		    vp);
2750 		return (error);
2751 	}
2752 
2753 	if (vs == VGET_USECOUNT) {
2754 		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
2755 		    ("%s: vnode with usecount and VI_OWEINACT set", __func__));
2756 		return (0);
2757 	}
2758 
2759 	/*
2760 	 * We hold the vnode. If the usecount is 0 it will be utilized to keep
2761 	 * the vnode around. Otherwise someone else lended their hold count and
2762 	 * we have to drop ours.
2763 	 */
2764 	if (vp->v_type != VCHR &&
2765 	    refcount_acquire_if_not_zero(&vp->v_usecount)) {
2766 #ifdef INVARIANTS
2767 		int old = atomic_fetchadd_int(&vp->v_holdcnt, -1) - 1;
2768 		VNASSERT(old > 0, vp, ("%s: wrong hold count", __func__));
2769 #else
2770 		refcount_release(&vp->v_holdcnt);
2771 #endif
2772 		VNODE_REFCOUNT_FENCE_ACQ();
2773 		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
2774 		    ("%s: vnode with usecount and VI_OWEINACT set", __func__));
2775 		return (0);
2776 	}
2777 
2778 	/*
2779 	 * We don't guarantee that any particular close will
2780 	 * trigger inactive processing so just make a best effort
2781 	 * here at preventing a reference to a removed file.  If
2782 	 * we don't succeed no harm is done.
2783 	 *
2784 	 * Upgrade our holdcnt to a usecount.
2785 	 */
2786 	VI_LOCK(vp);
2787 	if ((vp->v_iflag & VI_OWEINACT) == 0) {
2788 		oweinact = 0;
2789 	} else {
2790 		oweinact = 1;
2791 		vp->v_iflag &= ~VI_OWEINACT;
2792 		VNODE_REFCOUNT_FENCE_REL();
2793 	}
2794 	if (vp->v_usecount > 0)
2795 		refcount_release(&vp->v_holdcnt);
2796 	v_incr_devcount(vp);
2797 	refcount_acquire(&vp->v_usecount);
2798 	if (oweinact && VOP_ISLOCKED(vp) == LK_EXCLUSIVE &&
2799 	    (flags & LK_NOWAIT) == 0)
2800 		vinactive(vp, curthread);
2801 	VI_UNLOCK(vp);
2802 	return (0);
2803 }
2804 
2805 /*
2806  * Increase the reference (use) and hold count of a vnode.
2807  * This will also remove the vnode from the free list if it is presently free.
2808  */
2809 void
2810 vref(struct vnode *vp)
2811 {
2812 
2813 	ASSERT_VI_UNLOCKED(vp, __func__);
2814 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2815 	if (vp->v_type != VCHR &&
2816 	    refcount_acquire_if_not_zero(&vp->v_usecount)) {
2817 		VNODE_REFCOUNT_FENCE_ACQ();
2818 		VNASSERT(vp->v_holdcnt > 0, vp,
2819 		    ("%s: active vnode not held", __func__));
2820 		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
2821 		    ("%s: vnode with usecount and VI_OWEINACT set", __func__));
2822 		return;
2823 	}
2824 	VI_LOCK(vp);
2825 	vrefl(vp);
2826 	VI_UNLOCK(vp);
2827 }
2828 
2829 void
2830 vrefl(struct vnode *vp)
2831 {
2832 
2833 	ASSERT_VI_LOCKED(vp, __func__);
2834 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2835 	if (vp->v_usecount == 0)
2836 		vholdl(vp);
2837 	if ((vp->v_iflag & VI_OWEINACT) != 0) {
2838 		vp->v_iflag &= ~VI_OWEINACT;
2839 		VNODE_REFCOUNT_FENCE_REL();
2840 	}
2841 	v_incr_devcount(vp);
2842 	refcount_acquire(&vp->v_usecount);
2843 }
2844 
2845 void
2846 vrefact(struct vnode *vp)
2847 {
2848 
2849 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2850 	if (__predict_false(vp->v_type == VCHR)) {
2851 		VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp,
2852 		    ("%s: wrong ref counts", __func__));
2853 		vref(vp);
2854 		return;
2855 	}
2856 #ifdef INVARIANTS
2857 	int old = atomic_fetchadd_int(&vp->v_usecount, 1);
2858 	VNASSERT(old > 0, vp, ("%s: wrong use count", __func__));
2859 #else
2860 	refcount_acquire(&vp->v_usecount);
2861 #endif
2862 }
2863 
2864 /*
2865  * Return reference count of a vnode.
2866  *
2867  * The results of this call are only guaranteed when some mechanism is used to
2868  * stop other processes from gaining references to the vnode.  This may be the
2869  * case if the caller holds the only reference.  This is also useful when stale
2870  * data is acceptable as race conditions may be accounted for by some other
2871  * means.
2872  */
2873 int
2874 vrefcnt(struct vnode *vp)
2875 {
2876 
2877 	return (vp->v_usecount);
2878 }
2879 
2880 #define	VPUTX_VRELE	1
2881 #define	VPUTX_VPUT	2
2882 #define	VPUTX_VUNREF	3
2883 
2884 /*
2885  * Decrement the use and hold counts for a vnode.
2886  *
2887  * See an explanation near vget() as to why atomic operation is safe.
2888  */
2889 static void
2890 vputx(struct vnode *vp, int func)
2891 {
2892 	int error;
2893 
2894 	KASSERT(vp != NULL, ("vputx: null vp"));
2895 	if (func == VPUTX_VUNREF)
2896 		ASSERT_VOP_LOCKED(vp, "vunref");
2897 	else if (func == VPUTX_VPUT)
2898 		ASSERT_VOP_LOCKED(vp, "vput");
2899 	else
2900 		KASSERT(func == VPUTX_VRELE, ("vputx: wrong func"));
2901 	ASSERT_VI_UNLOCKED(vp, __func__);
2902 	VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp,
2903 	    ("%s: wrong ref counts", __func__));
2904 
2905 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2906 
2907 	/*
2908 	 * It is an invariant that all VOP_* calls operate on a held vnode.
2909 	 * We may be only having an implicit hold stemming from our usecount,
2910 	 * which we are about to release. If we unlock the vnode afterwards we
2911 	 * open a time window where someone else dropped the last usecount and
2912 	 * proceeded to free the vnode before our unlock finished. For this
2913 	 * reason we unlock the vnode early. This is a little bit wasteful as
2914 	 * it may be the vnode is exclusively locked and inactive processing is
2915 	 * needed, in which case we are adding work.
2916 	 */
2917 	if (func == VPUTX_VPUT)
2918 		VOP_UNLOCK(vp, 0);
2919 
2920 	if (vp->v_type != VCHR &&
2921 	    refcount_release_if_not_last(&vp->v_usecount))
2922 		return;
2923 
2924 	VI_LOCK(vp);
2925 
2926 	/*
2927 	 * We want to hold the vnode until the inactive finishes to
2928 	 * prevent vgone() races.  We drop the use count here and the
2929 	 * hold count below when we're done.
2930 	 */
2931 	v_decr_devcount(vp);
2932 	if (!refcount_release(&vp->v_usecount)) {
2933 		VI_UNLOCK(vp);
2934 		return;
2935 	}
2936 	if (vp->v_iflag & VI_DOINGINACT) {
2937 		vdropl(vp);
2938 		return;
2939 	}
2940 
2941 	error = 0;
2942 
2943 	if (vp->v_usecount != 0) {
2944 		vn_printf(vp, "vputx: usecount not zero for vnode ");
2945 		panic("vputx: usecount not zero");
2946 	}
2947 
2948 	CTR2(KTR_VFS, "%s: return vnode %p to the freelist", __func__, vp);
2949 
2950 	/*
2951 	 * Check if the fs wants to perform inactive processing. Note we
2952 	 * may be only holding the interlock, in which case it is possible
2953 	 * someone else called vgone on the vnode and ->v_data is now NULL.
2954 	 * Since vgone performs inactive on its own there is nothing to do
2955 	 * here but to drop our hold count.
2956 	 */
2957 	if (__predict_false(vp->v_iflag & VI_DOOMED) ||
2958 	    VOP_NEED_INACTIVE(vp) == 0) {
2959 		vdropl(vp);
2960 		return;
2961 	}
2962 
2963 	/*
2964 	 * We must call VOP_INACTIVE with the node locked. Mark
2965 	 * as VI_DOINGINACT to avoid recursion.
2966 	 */
2967 	vp->v_iflag |= VI_OWEINACT;
2968 	switch (func) {
2969 	case VPUTX_VRELE:
2970 		error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK);
2971 		VI_LOCK(vp);
2972 		break;
2973 	case VPUTX_VPUT:
2974 		error = VOP_LOCK(vp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT);
2975 		VI_LOCK(vp);
2976 		break;
2977 	case VPUTX_VUNREF:
2978 		if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
2979 			error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK);
2980 			VI_LOCK(vp);
2981 		}
2982 		break;
2983 	}
2984 	VNASSERT(vp->v_usecount == 0 || (vp->v_iflag & VI_OWEINACT) == 0, vp,
2985 	    ("vnode with usecount and VI_OWEINACT set"));
2986 	if (error == 0) {
2987 		if (vp->v_iflag & VI_OWEINACT)
2988 			vinactive(vp, curthread);
2989 		if (func != VPUTX_VUNREF)
2990 			VOP_UNLOCK(vp, 0);
2991 	}
2992 	vdropl(vp);
2993 }
2994 
2995 /*
2996  * Vnode put/release.
2997  * If count drops to zero, call inactive routine and return to freelist.
2998  */
2999 void
3000 vrele(struct vnode *vp)
3001 {
3002 
3003 	vputx(vp, VPUTX_VRELE);
3004 }
3005 
3006 /*
3007  * Release an already locked vnode.  This give the same effects as
3008  * unlock+vrele(), but takes less time and avoids releasing and
3009  * re-aquiring the lock (as vrele() acquires the lock internally.)
3010  */
3011 void
3012 vput(struct vnode *vp)
3013 {
3014 
3015 	vputx(vp, VPUTX_VPUT);
3016 }
3017 
3018 /*
3019  * Release an exclusively locked vnode. Do not unlock the vnode lock.
3020  */
3021 void
3022 vunref(struct vnode *vp)
3023 {
3024 
3025 	vputx(vp, VPUTX_VUNREF);
3026 }
3027 
3028 /*
3029  * Increase the hold count and activate if this is the first reference.
3030  */
3031 void
3032 _vhold(struct vnode *vp, bool locked)
3033 {
3034 	struct mount *mp;
3035 
3036 	if (locked)
3037 		ASSERT_VI_LOCKED(vp, __func__);
3038 	else
3039 		ASSERT_VI_UNLOCKED(vp, __func__);
3040 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3041 	if (!locked) {
3042 		if (refcount_acquire_if_not_zero(&vp->v_holdcnt)) {
3043 			VNODE_REFCOUNT_FENCE_ACQ();
3044 			VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
3045 			    ("_vhold: vnode with holdcnt is free"));
3046 			return;
3047 		}
3048 		VI_LOCK(vp);
3049 	}
3050 	if ((vp->v_iflag & VI_FREE) == 0) {
3051 		refcount_acquire(&vp->v_holdcnt);
3052 		if (!locked)
3053 			VI_UNLOCK(vp);
3054 		return;
3055 	}
3056 	VNASSERT(vp->v_holdcnt == 0, vp,
3057 	    ("%s: wrong hold count", __func__));
3058 	VNASSERT(vp->v_op != NULL, vp,
3059 	    ("%s: vnode already reclaimed.", __func__));
3060 	/*
3061 	 * Remove a vnode from the free list, mark it as in use,
3062 	 * and put it on the active list.
3063 	 */
3064 	VNASSERT(vp->v_mount != NULL, vp,
3065 	    ("_vhold: vnode not on per mount vnode list"));
3066 	mp = vp->v_mount;
3067 	mtx_lock(&mp->mnt_listmtx);
3068 	if ((vp->v_mflag & VMP_TMPMNTFREELIST) != 0) {
3069 		TAILQ_REMOVE(&mp->mnt_tmpfreevnodelist, vp, v_actfreelist);
3070 		mp->mnt_tmpfreevnodelistsize--;
3071 		vp->v_mflag &= ~VMP_TMPMNTFREELIST;
3072 	} else {
3073 		mtx_lock(&vnode_free_list_mtx);
3074 		TAILQ_REMOVE(&vnode_free_list, vp, v_actfreelist);
3075 		freevnodes--;
3076 		mtx_unlock(&vnode_free_list_mtx);
3077 	}
3078 	KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
3079 	    ("Activating already active vnode"));
3080 	vp->v_iflag &= ~VI_FREE;
3081 	vp->v_iflag |= VI_ACTIVE;
3082 	TAILQ_INSERT_HEAD(&mp->mnt_activevnodelist, vp, v_actfreelist);
3083 	mp->mnt_activevnodelistsize++;
3084 	mtx_unlock(&mp->mnt_listmtx);
3085 	refcount_acquire(&vp->v_holdcnt);
3086 	if (!locked)
3087 		VI_UNLOCK(vp);
3088 }
3089 
3090 void
3091 vholdnz(struct vnode *vp)
3092 {
3093 
3094 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3095 #ifdef INVARIANTS
3096 	int old = atomic_fetchadd_int(&vp->v_holdcnt, 1);
3097 	VNASSERT(old > 0, vp, ("%s: wrong hold count", __func__));
3098 #else
3099 	atomic_add_int(&vp->v_holdcnt, 1);
3100 #endif
3101 }
3102 
3103 /*
3104  * Drop the hold count of the vnode.  If this is the last reference to
3105  * the vnode we place it on the free list unless it has been vgone'd
3106  * (marked VI_DOOMED) in which case we will free it.
3107  *
3108  * Because the vnode vm object keeps a hold reference on the vnode if
3109  * there is at least one resident non-cached page, the vnode cannot
3110  * leave the active list without the page cleanup done.
3111  */
3112 void
3113 _vdrop(struct vnode *vp, bool locked)
3114 {
3115 	struct bufobj *bo;
3116 	struct mount *mp;
3117 	int active;
3118 
3119 	if (locked)
3120 		ASSERT_VI_LOCKED(vp, __func__);
3121 	else
3122 		ASSERT_VI_UNLOCKED(vp, __func__);
3123 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3124 	if (__predict_false((int)vp->v_holdcnt <= 0)) {
3125 		vn_printf(vp, "vdrop: holdcnt %d", vp->v_holdcnt);
3126 		panic("vdrop: wrong holdcnt");
3127 	}
3128 	if (!locked) {
3129 		if (refcount_release_if_not_last(&vp->v_holdcnt))
3130 			return;
3131 		VI_LOCK(vp);
3132 	}
3133 	if (refcount_release(&vp->v_holdcnt) == 0) {
3134 		VI_UNLOCK(vp);
3135 		return;
3136 	}
3137 	if ((vp->v_iflag & VI_DOOMED) == 0) {
3138 		/*
3139 		 * Mark a vnode as free: remove it from its active list
3140 		 * and put it up for recycling on the freelist.
3141 		 */
3142 		VNASSERT(vp->v_op != NULL, vp,
3143 		    ("vdropl: vnode already reclaimed."));
3144 		VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
3145 		    ("vnode already free"));
3146 		VNASSERT(vp->v_holdcnt == 0, vp,
3147 		    ("vdropl: freeing when we shouldn't"));
3148 		active = vp->v_iflag & VI_ACTIVE;
3149 		if ((vp->v_iflag & VI_OWEINACT) == 0) {
3150 			vp->v_iflag &= ~VI_ACTIVE;
3151 			mp = vp->v_mount;
3152 			if (mp != NULL) {
3153 				mtx_lock(&mp->mnt_listmtx);
3154 				if (active) {
3155 					TAILQ_REMOVE(&mp->mnt_activevnodelist,
3156 					    vp, v_actfreelist);
3157 					mp->mnt_activevnodelistsize--;
3158 				}
3159 				TAILQ_INSERT_TAIL(&mp->mnt_tmpfreevnodelist,
3160 				    vp, v_actfreelist);
3161 				mp->mnt_tmpfreevnodelistsize++;
3162 				vp->v_iflag |= VI_FREE;
3163 				vp->v_mflag |= VMP_TMPMNTFREELIST;
3164 				VI_UNLOCK(vp);
3165 				if (mp->mnt_tmpfreevnodelistsize >=
3166 				    mnt_free_list_batch)
3167 					vnlru_return_batch_locked(mp);
3168 				mtx_unlock(&mp->mnt_listmtx);
3169 			} else {
3170 				VNASSERT(active == 0, vp,
3171 				    ("vdropl: active vnode not on per mount "
3172 				    "vnode list"));
3173 				mtx_lock(&vnode_free_list_mtx);
3174 				TAILQ_INSERT_TAIL(&vnode_free_list, vp,
3175 				    v_actfreelist);
3176 				freevnodes++;
3177 				vp->v_iflag |= VI_FREE;
3178 				VI_UNLOCK(vp);
3179 				mtx_unlock(&vnode_free_list_mtx);
3180 			}
3181 		} else {
3182 			VI_UNLOCK(vp);
3183 			counter_u64_add(free_owe_inact, 1);
3184 		}
3185 		return;
3186 	}
3187 	/*
3188 	 * The vnode has been marked for destruction, so free it.
3189 	 *
3190 	 * The vnode will be returned to the zone where it will
3191 	 * normally remain until it is needed for another vnode. We
3192 	 * need to cleanup (or verify that the cleanup has already
3193 	 * been done) any residual data left from its current use
3194 	 * so as not to contaminate the freshly allocated vnode.
3195 	 */
3196 	CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp);
3197 	atomic_subtract_long(&numvnodes, 1);
3198 	bo = &vp->v_bufobj;
3199 	VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
3200 	    ("cleaned vnode still on the free list."));
3201 	VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't"));
3202 	VNASSERT(vp->v_holdcnt == 0, vp, ("Non-zero hold count"));
3203 	VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count"));
3204 	VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count"));
3205 	VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's"));
3206 	VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0"));
3207 	VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp,
3208 	    ("clean blk trie not empty"));
3209 	VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0"));
3210 	VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp,
3211 	    ("dirty blk trie not empty"));
3212 	VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst"));
3213 	VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src"));
3214 	VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for .."));
3215 	VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp,
3216 	    ("Dangling rangelock waiters"));
3217 	VI_UNLOCK(vp);
3218 #ifdef MAC
3219 	mac_vnode_destroy(vp);
3220 #endif
3221 	if (vp->v_pollinfo != NULL) {
3222 		destroy_vpollinfo(vp->v_pollinfo);
3223 		vp->v_pollinfo = NULL;
3224 	}
3225 #ifdef INVARIANTS
3226 	/* XXX Elsewhere we detect an already freed vnode via NULL v_op. */
3227 	vp->v_op = NULL;
3228 #endif
3229 	vp->v_mountedhere = NULL;
3230 	vp->v_unpcb = NULL;
3231 	vp->v_rdev = NULL;
3232 	vp->v_fifoinfo = NULL;
3233 	vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0;
3234 	vp->v_iflag = 0;
3235 	vp->v_vflag = 0;
3236 	bo->bo_flag = 0;
3237 	uma_zfree(vnode_zone, vp);
3238 }
3239 
3240 /*
3241  * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT
3242  * flags.  DOINGINACT prevents us from recursing in calls to vinactive.
3243  * OWEINACT tracks whether a vnode missed a call to inactive due to a
3244  * failed lock upgrade.
3245  */
3246 void
3247 vinactive(struct vnode *vp, struct thread *td)
3248 {
3249 	struct vm_object *obj;
3250 
3251 	ASSERT_VOP_ELOCKED(vp, "vinactive");
3252 	ASSERT_VI_LOCKED(vp, "vinactive");
3253 	VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp,
3254 	    ("vinactive: recursed on VI_DOINGINACT"));
3255 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3256 	vp->v_iflag |= VI_DOINGINACT;
3257 	vp->v_iflag &= ~VI_OWEINACT;
3258 	VI_UNLOCK(vp);
3259 	/*
3260 	 * Before moving off the active list, we must be sure that any
3261 	 * modified pages are converted into the vnode's dirty
3262 	 * buffers, since these will no longer be checked once the
3263 	 * vnode is on the inactive list.
3264 	 *
3265 	 * The write-out of the dirty pages is asynchronous.  At the
3266 	 * point that VOP_INACTIVE() is called, there could still be
3267 	 * pending I/O and dirty pages in the object.
3268 	 */
3269 	if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
3270 	    (obj->flags & OBJ_MIGHTBEDIRTY) != 0) {
3271 		VM_OBJECT_WLOCK(obj);
3272 		vm_object_page_clean(obj, 0, 0, 0);
3273 		VM_OBJECT_WUNLOCK(obj);
3274 	}
3275 	VOP_INACTIVE(vp, td);
3276 	VI_LOCK(vp);
3277 	VNASSERT(vp->v_iflag & VI_DOINGINACT, vp,
3278 	    ("vinactive: lost VI_DOINGINACT"));
3279 	vp->v_iflag &= ~VI_DOINGINACT;
3280 }
3281 
3282 /*
3283  * Remove any vnodes in the vnode table belonging to mount point mp.
3284  *
3285  * If FORCECLOSE is not specified, there should not be any active ones,
3286  * return error if any are found (nb: this is a user error, not a
3287  * system error). If FORCECLOSE is specified, detach any active vnodes
3288  * that are found.
3289  *
3290  * If WRITECLOSE is set, only flush out regular file vnodes open for
3291  * writing.
3292  *
3293  * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped.
3294  *
3295  * `rootrefs' specifies the base reference count for the root vnode
3296  * of this filesystem. The root vnode is considered busy if its
3297  * v_usecount exceeds this value. On a successful return, vflush(, td)
3298  * will call vrele() on the root vnode exactly rootrefs times.
3299  * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must
3300  * be zero.
3301  */
3302 #ifdef DIAGNOSTIC
3303 static int busyprt = 0;		/* print out busy vnodes */
3304 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes");
3305 #endif
3306 
3307 int
3308 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td)
3309 {
3310 	struct vnode *vp, *mvp, *rootvp = NULL;
3311 	struct vattr vattr;
3312 	int busy = 0, error;
3313 
3314 	CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp,
3315 	    rootrefs, flags);
3316 	if (rootrefs > 0) {
3317 		KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0,
3318 		    ("vflush: bad args"));
3319 		/*
3320 		 * Get the filesystem root vnode. We can vput() it
3321 		 * immediately, since with rootrefs > 0, it won't go away.
3322 		 */
3323 		if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) {
3324 			CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d",
3325 			    __func__, error);
3326 			return (error);
3327 		}
3328 		vput(rootvp);
3329 	}
3330 loop:
3331 	MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
3332 		vholdl(vp);
3333 		error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE);
3334 		if (error) {
3335 			vdrop(vp);
3336 			MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
3337 			goto loop;
3338 		}
3339 		/*
3340 		 * Skip over a vnodes marked VV_SYSTEM.
3341 		 */
3342 		if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) {
3343 			VOP_UNLOCK(vp, 0);
3344 			vdrop(vp);
3345 			continue;
3346 		}
3347 		/*
3348 		 * If WRITECLOSE is set, flush out unlinked but still open
3349 		 * files (even if open only for reading) and regular file
3350 		 * vnodes open for writing.
3351 		 */
3352 		if (flags & WRITECLOSE) {
3353 			if (vp->v_object != NULL) {
3354 				VM_OBJECT_WLOCK(vp->v_object);
3355 				vm_object_page_clean(vp->v_object, 0, 0, 0);
3356 				VM_OBJECT_WUNLOCK(vp->v_object);
3357 			}
3358 			error = VOP_FSYNC(vp, MNT_WAIT, td);
3359 			if (error != 0) {
3360 				VOP_UNLOCK(vp, 0);
3361 				vdrop(vp);
3362 				MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
3363 				return (error);
3364 			}
3365 			error = VOP_GETATTR(vp, &vattr, td->td_ucred);
3366 			VI_LOCK(vp);
3367 
3368 			if ((vp->v_type == VNON ||
3369 			    (error == 0 && vattr.va_nlink > 0)) &&
3370 			    (vp->v_writecount <= 0 || vp->v_type != VREG)) {
3371 				VOP_UNLOCK(vp, 0);
3372 				vdropl(vp);
3373 				continue;
3374 			}
3375 		} else
3376 			VI_LOCK(vp);
3377 		/*
3378 		 * With v_usecount == 0, all we need to do is clear out the
3379 		 * vnode data structures and we are done.
3380 		 *
3381 		 * If FORCECLOSE is set, forcibly close the vnode.
3382 		 */
3383 		if (vp->v_usecount == 0 || (flags & FORCECLOSE)) {
3384 			vgonel(vp);
3385 		} else {
3386 			busy++;
3387 #ifdef DIAGNOSTIC
3388 			if (busyprt)
3389 				vn_printf(vp, "vflush: busy vnode ");
3390 #endif
3391 		}
3392 		VOP_UNLOCK(vp, 0);
3393 		vdropl(vp);
3394 	}
3395 	if (rootrefs > 0 && (flags & FORCECLOSE) == 0) {
3396 		/*
3397 		 * If just the root vnode is busy, and if its refcount
3398 		 * is equal to `rootrefs', then go ahead and kill it.
3399 		 */
3400 		VI_LOCK(rootvp);
3401 		KASSERT(busy > 0, ("vflush: not busy"));
3402 		VNASSERT(rootvp->v_usecount >= rootrefs, rootvp,
3403 		    ("vflush: usecount %d < rootrefs %d",
3404 		     rootvp->v_usecount, rootrefs));
3405 		if (busy == 1 && rootvp->v_usecount == rootrefs) {
3406 			VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK);
3407 			vgone(rootvp);
3408 			VOP_UNLOCK(rootvp, 0);
3409 			busy = 0;
3410 		} else
3411 			VI_UNLOCK(rootvp);
3412 	}
3413 	if (busy) {
3414 		CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__,
3415 		    busy);
3416 		return (EBUSY);
3417 	}
3418 	for (; rootrefs > 0; rootrefs--)
3419 		vrele(rootvp);
3420 	return (0);
3421 }
3422 
3423 /*
3424  * Recycle an unused vnode to the front of the free list.
3425  */
3426 int
3427 vrecycle(struct vnode *vp)
3428 {
3429 	int recycled;
3430 
3431 	VI_LOCK(vp);
3432 	recycled = vrecyclel(vp);
3433 	VI_UNLOCK(vp);
3434 	return (recycled);
3435 }
3436 
3437 /*
3438  * vrecycle, with the vp interlock held.
3439  */
3440 int
3441 vrecyclel(struct vnode *vp)
3442 {
3443 	int recycled;
3444 
3445 	ASSERT_VOP_ELOCKED(vp, __func__);
3446 	ASSERT_VI_LOCKED(vp, __func__);
3447 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3448 	recycled = 0;
3449 	if (vp->v_usecount == 0) {
3450 		recycled = 1;
3451 		vgonel(vp);
3452 	}
3453 	return (recycled);
3454 }
3455 
3456 /*
3457  * Eliminate all activity associated with a vnode
3458  * in preparation for reuse.
3459  */
3460 void
3461 vgone(struct vnode *vp)
3462 {
3463 	VI_LOCK(vp);
3464 	vgonel(vp);
3465 	VI_UNLOCK(vp);
3466 }
3467 
3468 static void
3469 notify_lowervp_vfs_dummy(struct mount *mp __unused,
3470     struct vnode *lowervp __unused)
3471 {
3472 }
3473 
3474 /*
3475  * Notify upper mounts about reclaimed or unlinked vnode.
3476  */
3477 void
3478 vfs_notify_upper(struct vnode *vp, int event)
3479 {
3480 	static struct vfsops vgonel_vfsops = {
3481 		.vfs_reclaim_lowervp = notify_lowervp_vfs_dummy,
3482 		.vfs_unlink_lowervp = notify_lowervp_vfs_dummy,
3483 	};
3484 	struct mount *mp, *ump, *mmp;
3485 
3486 	mp = vp->v_mount;
3487 	if (mp == NULL)
3488 		return;
3489 
3490 	MNT_ILOCK(mp);
3491 	if (TAILQ_EMPTY(&mp->mnt_uppers))
3492 		goto unlock;
3493 	MNT_IUNLOCK(mp);
3494 	mmp = malloc(sizeof(struct mount), M_TEMP, M_WAITOK | M_ZERO);
3495 	mmp->mnt_op = &vgonel_vfsops;
3496 	mmp->mnt_kern_flag |= MNTK_MARKER;
3497 	MNT_ILOCK(mp);
3498 	mp->mnt_kern_flag |= MNTK_VGONE_UPPER;
3499 	for (ump = TAILQ_FIRST(&mp->mnt_uppers); ump != NULL;) {
3500 		if ((ump->mnt_kern_flag & MNTK_MARKER) != 0) {
3501 			ump = TAILQ_NEXT(ump, mnt_upper_link);
3502 			continue;
3503 		}
3504 		TAILQ_INSERT_AFTER(&mp->mnt_uppers, ump, mmp, mnt_upper_link);
3505 		MNT_IUNLOCK(mp);
3506 		switch (event) {
3507 		case VFS_NOTIFY_UPPER_RECLAIM:
3508 			VFS_RECLAIM_LOWERVP(ump, vp);
3509 			break;
3510 		case VFS_NOTIFY_UPPER_UNLINK:
3511 			VFS_UNLINK_LOWERVP(ump, vp);
3512 			break;
3513 		default:
3514 			KASSERT(0, ("invalid event %d", event));
3515 			break;
3516 		}
3517 		MNT_ILOCK(mp);
3518 		ump = TAILQ_NEXT(mmp, mnt_upper_link);
3519 		TAILQ_REMOVE(&mp->mnt_uppers, mmp, mnt_upper_link);
3520 	}
3521 	free(mmp, M_TEMP);
3522 	mp->mnt_kern_flag &= ~MNTK_VGONE_UPPER;
3523 	if ((mp->mnt_kern_flag & MNTK_VGONE_WAITER) != 0) {
3524 		mp->mnt_kern_flag &= ~MNTK_VGONE_WAITER;
3525 		wakeup(&mp->mnt_uppers);
3526 	}
3527 unlock:
3528 	MNT_IUNLOCK(mp);
3529 }
3530 
3531 /*
3532  * vgone, with the vp interlock held.
3533  */
3534 static void
3535 vgonel(struct vnode *vp)
3536 {
3537 	struct thread *td;
3538 	struct mount *mp;
3539 	vm_object_t object;
3540 	bool active, oweinact;
3541 
3542 	ASSERT_VOP_ELOCKED(vp, "vgonel");
3543 	ASSERT_VI_LOCKED(vp, "vgonel");
3544 	VNASSERT(vp->v_holdcnt, vp,
3545 	    ("vgonel: vp %p has no reference.", vp));
3546 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3547 	td = curthread;
3548 
3549 	/*
3550 	 * Don't vgonel if we're already doomed.
3551 	 */
3552 	if (vp->v_iflag & VI_DOOMED)
3553 		return;
3554 	vp->v_iflag |= VI_DOOMED;
3555 
3556 	/*
3557 	 * Check to see if the vnode is in use.  If so, we have to call
3558 	 * VOP_CLOSE() and VOP_INACTIVE().
3559 	 */
3560 	active = vp->v_usecount > 0;
3561 	oweinact = (vp->v_iflag & VI_OWEINACT) != 0;
3562 	VI_UNLOCK(vp);
3563 	vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM);
3564 
3565 	/*
3566 	 * If purging an active vnode, it must be closed and
3567 	 * deactivated before being reclaimed.
3568 	 */
3569 	if (active)
3570 		VOP_CLOSE(vp, FNONBLOCK, NOCRED, td);
3571 	if (oweinact || active) {
3572 		VI_LOCK(vp);
3573 		if ((vp->v_iflag & VI_DOINGINACT) == 0)
3574 			vinactive(vp, td);
3575 		VI_UNLOCK(vp);
3576 	}
3577 	if (vp->v_type == VSOCK)
3578 		vfs_unp_reclaim(vp);
3579 
3580 	/*
3581 	 * Clean out any buffers associated with the vnode.
3582 	 * If the flush fails, just toss the buffers.
3583 	 */
3584 	mp = NULL;
3585 	if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd))
3586 		(void) vn_start_secondary_write(vp, &mp, V_WAIT);
3587 	if (vinvalbuf(vp, V_SAVE, 0, 0) != 0) {
3588 		while (vinvalbuf(vp, 0, 0, 0) != 0)
3589 			;
3590 	}
3591 
3592 	BO_LOCK(&vp->v_bufobj);
3593 	KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) &&
3594 	    vp->v_bufobj.bo_dirty.bv_cnt == 0 &&
3595 	    TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) &&
3596 	    vp->v_bufobj.bo_clean.bv_cnt == 0,
3597 	    ("vp %p bufobj not invalidated", vp));
3598 
3599 	/*
3600 	 * For VMIO bufobj, BO_DEAD is set later, or in
3601 	 * vm_object_terminate() after the object's page queue is
3602 	 * flushed.
3603 	 */
3604 	object = vp->v_bufobj.bo_object;
3605 	if (object == NULL)
3606 		vp->v_bufobj.bo_flag |= BO_DEAD;
3607 	BO_UNLOCK(&vp->v_bufobj);
3608 
3609 	/*
3610 	 * Handle the VM part.  Tmpfs handles v_object on its own (the
3611 	 * OBJT_VNODE check).  Nullfs or other bypassing filesystems
3612 	 * should not touch the object borrowed from the lower vnode
3613 	 * (the handle check).
3614 	 */
3615 	if (object != NULL && object->type == OBJT_VNODE &&
3616 	    object->handle == vp)
3617 		vnode_destroy_vobject(vp);
3618 
3619 	/*
3620 	 * Reclaim the vnode.
3621 	 */
3622 	if (VOP_RECLAIM(vp, td))
3623 		panic("vgone: cannot reclaim");
3624 	if (mp != NULL)
3625 		vn_finished_secondary_write(mp);
3626 	VNASSERT(vp->v_object == NULL, vp,
3627 	    ("vop_reclaim left v_object vp=%p, tag=%s", vp, vp->v_tag));
3628 	/*
3629 	 * Clear the advisory locks and wake up waiting threads.
3630 	 */
3631 	(void)VOP_ADVLOCKPURGE(vp);
3632 	vp->v_lockf = NULL;
3633 	/*
3634 	 * Delete from old mount point vnode list.
3635 	 */
3636 	delmntque(vp);
3637 	cache_purge(vp);
3638 	/*
3639 	 * Done with purge, reset to the standard lock and invalidate
3640 	 * the vnode.
3641 	 */
3642 	VI_LOCK(vp);
3643 	vp->v_vnlock = &vp->v_lock;
3644 	vp->v_op = &dead_vnodeops;
3645 	vp->v_tag = "none";
3646 	vp->v_type = VBAD;
3647 }
3648 
3649 /*
3650  * Calculate the total number of references to a special device.
3651  */
3652 int
3653 vcount(struct vnode *vp)
3654 {
3655 	int count;
3656 
3657 	dev_lock();
3658 	count = vp->v_rdev->si_usecount;
3659 	dev_unlock();
3660 	return (count);
3661 }
3662 
3663 /*
3664  * Same as above, but using the struct cdev *as argument
3665  */
3666 int
3667 count_dev(struct cdev *dev)
3668 {
3669 	int count;
3670 
3671 	dev_lock();
3672 	count = dev->si_usecount;
3673 	dev_unlock();
3674 	return(count);
3675 }
3676 
3677 /*
3678  * Print out a description of a vnode.
3679  */
3680 static char *typename[] =
3681 {"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD",
3682  "VMARKER"};
3683 
3684 void
3685 vn_printf(struct vnode *vp, const char *fmt, ...)
3686 {
3687 	va_list ap;
3688 	char buf[256], buf2[16];
3689 	u_long flags;
3690 
3691 	va_start(ap, fmt);
3692 	vprintf(fmt, ap);
3693 	va_end(ap);
3694 	printf("%p: ", (void *)vp);
3695 	printf("tag %s, type %s\n", vp->v_tag, typename[vp->v_type]);
3696 	printf("    usecount %d, writecount %d, refcount %d",
3697 	    vp->v_usecount, vp->v_writecount, vp->v_holdcnt);
3698 	switch (vp->v_type) {
3699 	case VDIR:
3700 		printf(" mountedhere %p\n", vp->v_mountedhere);
3701 		break;
3702 	case VCHR:
3703 		printf(" rdev %p\n", vp->v_rdev);
3704 		break;
3705 	case VSOCK:
3706 		printf(" socket %p\n", vp->v_unpcb);
3707 		break;
3708 	case VFIFO:
3709 		printf(" fifoinfo %p\n", vp->v_fifoinfo);
3710 		break;
3711 	default:
3712 		printf("\n");
3713 		break;
3714 	}
3715 	buf[0] = '\0';
3716 	buf[1] = '\0';
3717 	if (vp->v_vflag & VV_ROOT)
3718 		strlcat(buf, "|VV_ROOT", sizeof(buf));
3719 	if (vp->v_vflag & VV_ISTTY)
3720 		strlcat(buf, "|VV_ISTTY", sizeof(buf));
3721 	if (vp->v_vflag & VV_NOSYNC)
3722 		strlcat(buf, "|VV_NOSYNC", sizeof(buf));
3723 	if (vp->v_vflag & VV_ETERNALDEV)
3724 		strlcat(buf, "|VV_ETERNALDEV", sizeof(buf));
3725 	if (vp->v_vflag & VV_CACHEDLABEL)
3726 		strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf));
3727 	if (vp->v_vflag & VV_COPYONWRITE)
3728 		strlcat(buf, "|VV_COPYONWRITE", sizeof(buf));
3729 	if (vp->v_vflag & VV_SYSTEM)
3730 		strlcat(buf, "|VV_SYSTEM", sizeof(buf));
3731 	if (vp->v_vflag & VV_PROCDEP)
3732 		strlcat(buf, "|VV_PROCDEP", sizeof(buf));
3733 	if (vp->v_vflag & VV_NOKNOTE)
3734 		strlcat(buf, "|VV_NOKNOTE", sizeof(buf));
3735 	if (vp->v_vflag & VV_DELETED)
3736 		strlcat(buf, "|VV_DELETED", sizeof(buf));
3737 	if (vp->v_vflag & VV_MD)
3738 		strlcat(buf, "|VV_MD", sizeof(buf));
3739 	if (vp->v_vflag & VV_FORCEINSMQ)
3740 		strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf));
3741 	flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV |
3742 	    VV_CACHEDLABEL | VV_COPYONWRITE | VV_SYSTEM | VV_PROCDEP |
3743 	    VV_NOKNOTE | VV_DELETED | VV_MD | VV_FORCEINSMQ);
3744 	if (flags != 0) {
3745 		snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags);
3746 		strlcat(buf, buf2, sizeof(buf));
3747 	}
3748 	if (vp->v_iflag & VI_MOUNT)
3749 		strlcat(buf, "|VI_MOUNT", sizeof(buf));
3750 	if (vp->v_iflag & VI_DOOMED)
3751 		strlcat(buf, "|VI_DOOMED", sizeof(buf));
3752 	if (vp->v_iflag & VI_FREE)
3753 		strlcat(buf, "|VI_FREE", sizeof(buf));
3754 	if (vp->v_iflag & VI_ACTIVE)
3755 		strlcat(buf, "|VI_ACTIVE", sizeof(buf));
3756 	if (vp->v_iflag & VI_DOINGINACT)
3757 		strlcat(buf, "|VI_DOINGINACT", sizeof(buf));
3758 	if (vp->v_iflag & VI_OWEINACT)
3759 		strlcat(buf, "|VI_OWEINACT", sizeof(buf));
3760 	flags = vp->v_iflag & ~(VI_MOUNT | VI_DOOMED | VI_FREE |
3761 	    VI_ACTIVE | VI_DOINGINACT | VI_OWEINACT);
3762 	if (flags != 0) {
3763 		snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags);
3764 		strlcat(buf, buf2, sizeof(buf));
3765 	}
3766 	printf("    flags (%s)\n", buf + 1);
3767 	if (mtx_owned(VI_MTX(vp)))
3768 		printf(" VI_LOCKed");
3769 	if (vp->v_object != NULL)
3770 		printf("    v_object %p ref %d pages %d "
3771 		    "cleanbuf %d dirtybuf %d\n",
3772 		    vp->v_object, vp->v_object->ref_count,
3773 		    vp->v_object->resident_page_count,
3774 		    vp->v_bufobj.bo_clean.bv_cnt,
3775 		    vp->v_bufobj.bo_dirty.bv_cnt);
3776 	printf("    ");
3777 	lockmgr_printinfo(vp->v_vnlock);
3778 	if (vp->v_data != NULL)
3779 		VOP_PRINT(vp);
3780 }
3781 
3782 #ifdef DDB
3783 /*
3784  * List all of the locked vnodes in the system.
3785  * Called when debugging the kernel.
3786  */
3787 DB_SHOW_COMMAND(lockedvnods, lockedvnodes)
3788 {
3789 	struct mount *mp;
3790 	struct vnode *vp;
3791 
3792 	/*
3793 	 * Note: because this is DDB, we can't obey the locking semantics
3794 	 * for these structures, which means we could catch an inconsistent
3795 	 * state and dereference a nasty pointer.  Not much to be done
3796 	 * about that.
3797 	 */
3798 	db_printf("Locked vnodes\n");
3799 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
3800 		TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
3801 			if (vp->v_type != VMARKER && VOP_ISLOCKED(vp))
3802 				vn_printf(vp, "vnode ");
3803 		}
3804 	}
3805 }
3806 
3807 /*
3808  * Show details about the given vnode.
3809  */
3810 DB_SHOW_COMMAND(vnode, db_show_vnode)
3811 {
3812 	struct vnode *vp;
3813 
3814 	if (!have_addr)
3815 		return;
3816 	vp = (struct vnode *)addr;
3817 	vn_printf(vp, "vnode ");
3818 }
3819 
3820 /*
3821  * Show details about the given mount point.
3822  */
3823 DB_SHOW_COMMAND(mount, db_show_mount)
3824 {
3825 	struct mount *mp;
3826 	struct vfsopt *opt;
3827 	struct statfs *sp;
3828 	struct vnode *vp;
3829 	char buf[512];
3830 	uint64_t mflags;
3831 	u_int flags;
3832 
3833 	if (!have_addr) {
3834 		/* No address given, print short info about all mount points. */
3835 		TAILQ_FOREACH(mp, &mountlist, mnt_list) {
3836 			db_printf("%p %s on %s (%s)\n", mp,
3837 			    mp->mnt_stat.f_mntfromname,
3838 			    mp->mnt_stat.f_mntonname,
3839 			    mp->mnt_stat.f_fstypename);
3840 			if (db_pager_quit)
3841 				break;
3842 		}
3843 		db_printf("\nMore info: show mount <addr>\n");
3844 		return;
3845 	}
3846 
3847 	mp = (struct mount *)addr;
3848 	db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname,
3849 	    mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename);
3850 
3851 	buf[0] = '\0';
3852 	mflags = mp->mnt_flag;
3853 #define	MNT_FLAG(flag)	do {						\
3854 	if (mflags & (flag)) {						\
3855 		if (buf[0] != '\0')					\
3856 			strlcat(buf, ", ", sizeof(buf));		\
3857 		strlcat(buf, (#flag) + 4, sizeof(buf));			\
3858 		mflags &= ~(flag);					\
3859 	}								\
3860 } while (0)
3861 	MNT_FLAG(MNT_RDONLY);
3862 	MNT_FLAG(MNT_SYNCHRONOUS);
3863 	MNT_FLAG(MNT_NOEXEC);
3864 	MNT_FLAG(MNT_NOSUID);
3865 	MNT_FLAG(MNT_NFS4ACLS);
3866 	MNT_FLAG(MNT_UNION);
3867 	MNT_FLAG(MNT_ASYNC);
3868 	MNT_FLAG(MNT_SUIDDIR);
3869 	MNT_FLAG(MNT_SOFTDEP);
3870 	MNT_FLAG(MNT_NOSYMFOLLOW);
3871 	MNT_FLAG(MNT_GJOURNAL);
3872 	MNT_FLAG(MNT_MULTILABEL);
3873 	MNT_FLAG(MNT_ACLS);
3874 	MNT_FLAG(MNT_NOATIME);
3875 	MNT_FLAG(MNT_NOCLUSTERR);
3876 	MNT_FLAG(MNT_NOCLUSTERW);
3877 	MNT_FLAG(MNT_SUJ);
3878 	MNT_FLAG(MNT_EXRDONLY);
3879 	MNT_FLAG(MNT_EXPORTED);
3880 	MNT_FLAG(MNT_DEFEXPORTED);
3881 	MNT_FLAG(MNT_EXPORTANON);
3882 	MNT_FLAG(MNT_EXKERB);
3883 	MNT_FLAG(MNT_EXPUBLIC);
3884 	MNT_FLAG(MNT_LOCAL);
3885 	MNT_FLAG(MNT_QUOTA);
3886 	MNT_FLAG(MNT_ROOTFS);
3887 	MNT_FLAG(MNT_USER);
3888 	MNT_FLAG(MNT_IGNORE);
3889 	MNT_FLAG(MNT_UPDATE);
3890 	MNT_FLAG(MNT_DELEXPORT);
3891 	MNT_FLAG(MNT_RELOAD);
3892 	MNT_FLAG(MNT_FORCE);
3893 	MNT_FLAG(MNT_SNAPSHOT);
3894 	MNT_FLAG(MNT_BYFSID);
3895 #undef MNT_FLAG
3896 	if (mflags != 0) {
3897 		if (buf[0] != '\0')
3898 			strlcat(buf, ", ", sizeof(buf));
3899 		snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
3900 		    "0x%016jx", mflags);
3901 	}
3902 	db_printf("    mnt_flag = %s\n", buf);
3903 
3904 	buf[0] = '\0';
3905 	flags = mp->mnt_kern_flag;
3906 #define	MNT_KERN_FLAG(flag)	do {					\
3907 	if (flags & (flag)) {						\
3908 		if (buf[0] != '\0')					\
3909 			strlcat(buf, ", ", sizeof(buf));		\
3910 		strlcat(buf, (#flag) + 5, sizeof(buf));			\
3911 		flags &= ~(flag);					\
3912 	}								\
3913 } while (0)
3914 	MNT_KERN_FLAG(MNTK_UNMOUNTF);
3915 	MNT_KERN_FLAG(MNTK_ASYNC);
3916 	MNT_KERN_FLAG(MNTK_SOFTDEP);
3917 	MNT_KERN_FLAG(MNTK_DRAINING);
3918 	MNT_KERN_FLAG(MNTK_REFEXPIRE);
3919 	MNT_KERN_FLAG(MNTK_EXTENDED_SHARED);
3920 	MNT_KERN_FLAG(MNTK_SHARED_WRITES);
3921 	MNT_KERN_FLAG(MNTK_NO_IOPF);
3922 	MNT_KERN_FLAG(MNTK_VGONE_UPPER);
3923 	MNT_KERN_FLAG(MNTK_VGONE_WAITER);
3924 	MNT_KERN_FLAG(MNTK_LOOKUP_EXCL_DOTDOT);
3925 	MNT_KERN_FLAG(MNTK_MARKER);
3926 	MNT_KERN_FLAG(MNTK_USES_BCACHE);
3927 	MNT_KERN_FLAG(MNTK_NOASYNC);
3928 	MNT_KERN_FLAG(MNTK_UNMOUNT);
3929 	MNT_KERN_FLAG(MNTK_MWAIT);
3930 	MNT_KERN_FLAG(MNTK_SUSPEND);
3931 	MNT_KERN_FLAG(MNTK_SUSPEND2);
3932 	MNT_KERN_FLAG(MNTK_SUSPENDED);
3933 	MNT_KERN_FLAG(MNTK_LOOKUP_SHARED);
3934 	MNT_KERN_FLAG(MNTK_NOKNOTE);
3935 #undef MNT_KERN_FLAG
3936 	if (flags != 0) {
3937 		if (buf[0] != '\0')
3938 			strlcat(buf, ", ", sizeof(buf));
3939 		snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
3940 		    "0x%08x", flags);
3941 	}
3942 	db_printf("    mnt_kern_flag = %s\n", buf);
3943 
3944 	db_printf("    mnt_opt = ");
3945 	opt = TAILQ_FIRST(mp->mnt_opt);
3946 	if (opt != NULL) {
3947 		db_printf("%s", opt->name);
3948 		opt = TAILQ_NEXT(opt, link);
3949 		while (opt != NULL) {
3950 			db_printf(", %s", opt->name);
3951 			opt = TAILQ_NEXT(opt, link);
3952 		}
3953 	}
3954 	db_printf("\n");
3955 
3956 	sp = &mp->mnt_stat;
3957 	db_printf("    mnt_stat = { version=%u type=%u flags=0x%016jx "
3958 	    "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju "
3959 	    "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju "
3960 	    "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n",
3961 	    (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags,
3962 	    (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize,
3963 	    (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree,
3964 	    (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files,
3965 	    (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites,
3966 	    (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads,
3967 	    (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax,
3968 	    (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]);
3969 
3970 	db_printf("    mnt_cred = { uid=%u ruid=%u",
3971 	    (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid);
3972 	if (jailed(mp->mnt_cred))
3973 		db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id);
3974 	db_printf(" }\n");
3975 	db_printf("    mnt_ref = %d\n", mp->mnt_ref);
3976 	db_printf("    mnt_gen = %d\n", mp->mnt_gen);
3977 	db_printf("    mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize);
3978 	db_printf("    mnt_activevnodelistsize = %d\n",
3979 	    mp->mnt_activevnodelistsize);
3980 	db_printf("    mnt_writeopcount = %d\n", mp->mnt_writeopcount);
3981 	db_printf("    mnt_maxsymlinklen = %d\n", mp->mnt_maxsymlinklen);
3982 	db_printf("    mnt_iosize_max = %d\n", mp->mnt_iosize_max);
3983 	db_printf("    mnt_hashseed = %u\n", mp->mnt_hashseed);
3984 	db_printf("    mnt_lockref = %d\n", mp->mnt_lockref);
3985 	db_printf("    mnt_secondary_writes = %d\n", mp->mnt_secondary_writes);
3986 	db_printf("    mnt_secondary_accwrites = %d\n",
3987 	    mp->mnt_secondary_accwrites);
3988 	db_printf("    mnt_gjprovider = %s\n",
3989 	    mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL");
3990 
3991 	db_printf("\n\nList of active vnodes\n");
3992 	TAILQ_FOREACH(vp, &mp->mnt_activevnodelist, v_actfreelist) {
3993 		if (vp->v_type != VMARKER) {
3994 			vn_printf(vp, "vnode ");
3995 			if (db_pager_quit)
3996 				break;
3997 		}
3998 	}
3999 	db_printf("\n\nList of inactive vnodes\n");
4000 	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
4001 		if (vp->v_type != VMARKER && (vp->v_iflag & VI_ACTIVE) == 0) {
4002 			vn_printf(vp, "vnode ");
4003 			if (db_pager_quit)
4004 				break;
4005 		}
4006 	}
4007 }
4008 #endif	/* DDB */
4009 
4010 /*
4011  * Fill in a struct xvfsconf based on a struct vfsconf.
4012  */
4013 static int
4014 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp)
4015 {
4016 	struct xvfsconf xvfsp;
4017 
4018 	bzero(&xvfsp, sizeof(xvfsp));
4019 	strcpy(xvfsp.vfc_name, vfsp->vfc_name);
4020 	xvfsp.vfc_typenum = vfsp->vfc_typenum;
4021 	xvfsp.vfc_refcount = vfsp->vfc_refcount;
4022 	xvfsp.vfc_flags = vfsp->vfc_flags;
4023 	/*
4024 	 * These are unused in userland, we keep them
4025 	 * to not break binary compatibility.
4026 	 */
4027 	xvfsp.vfc_vfsops = NULL;
4028 	xvfsp.vfc_next = NULL;
4029 	return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
4030 }
4031 
4032 #ifdef COMPAT_FREEBSD32
4033 struct xvfsconf32 {
4034 	uint32_t	vfc_vfsops;
4035 	char		vfc_name[MFSNAMELEN];
4036 	int32_t		vfc_typenum;
4037 	int32_t		vfc_refcount;
4038 	int32_t		vfc_flags;
4039 	uint32_t	vfc_next;
4040 };
4041 
4042 static int
4043 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp)
4044 {
4045 	struct xvfsconf32 xvfsp;
4046 
4047 	bzero(&xvfsp, sizeof(xvfsp));
4048 	strcpy(xvfsp.vfc_name, vfsp->vfc_name);
4049 	xvfsp.vfc_typenum = vfsp->vfc_typenum;
4050 	xvfsp.vfc_refcount = vfsp->vfc_refcount;
4051 	xvfsp.vfc_flags = vfsp->vfc_flags;
4052 	return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
4053 }
4054 #endif
4055 
4056 /*
4057  * Top level filesystem related information gathering.
4058  */
4059 static int
4060 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS)
4061 {
4062 	struct vfsconf *vfsp;
4063 	int error;
4064 
4065 	error = 0;
4066 	vfsconf_slock();
4067 	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
4068 #ifdef COMPAT_FREEBSD32
4069 		if (req->flags & SCTL_MASK32)
4070 			error = vfsconf2x32(req, vfsp);
4071 		else
4072 #endif
4073 			error = vfsconf2x(req, vfsp);
4074 		if (error)
4075 			break;
4076 	}
4077 	vfsconf_sunlock();
4078 	return (error);
4079 }
4080 
4081 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD |
4082     CTLFLAG_MPSAFE, NULL, 0, sysctl_vfs_conflist,
4083     "S,xvfsconf", "List of all configured filesystems");
4084 
4085 #ifndef BURN_BRIDGES
4086 static int	sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS);
4087 
4088 static int
4089 vfs_sysctl(SYSCTL_HANDLER_ARGS)
4090 {
4091 	int *name = (int *)arg1 - 1;	/* XXX */
4092 	u_int namelen = arg2 + 1;	/* XXX */
4093 	struct vfsconf *vfsp;
4094 
4095 	log(LOG_WARNING, "userland calling deprecated sysctl, "
4096 	    "please rebuild world\n");
4097 
4098 #if 1 || defined(COMPAT_PRELITE2)
4099 	/* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */
4100 	if (namelen == 1)
4101 		return (sysctl_ovfs_conf(oidp, arg1, arg2, req));
4102 #endif
4103 
4104 	switch (name[1]) {
4105 	case VFS_MAXTYPENUM:
4106 		if (namelen != 2)
4107 			return (ENOTDIR);
4108 		return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int)));
4109 	case VFS_CONF:
4110 		if (namelen != 3)
4111 			return (ENOTDIR);	/* overloaded */
4112 		vfsconf_slock();
4113 		TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
4114 			if (vfsp->vfc_typenum == name[2])
4115 				break;
4116 		}
4117 		vfsconf_sunlock();
4118 		if (vfsp == NULL)
4119 			return (EOPNOTSUPP);
4120 #ifdef COMPAT_FREEBSD32
4121 		if (req->flags & SCTL_MASK32)
4122 			return (vfsconf2x32(req, vfsp));
4123 		else
4124 #endif
4125 			return (vfsconf2x(req, vfsp));
4126 	}
4127 	return (EOPNOTSUPP);
4128 }
4129 
4130 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP |
4131     CTLFLAG_MPSAFE, vfs_sysctl,
4132     "Generic filesystem");
4133 
4134 #if 1 || defined(COMPAT_PRELITE2)
4135 
4136 static int
4137 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS)
4138 {
4139 	int error;
4140 	struct vfsconf *vfsp;
4141 	struct ovfsconf ovfs;
4142 
4143 	vfsconf_slock();
4144 	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
4145 		bzero(&ovfs, sizeof(ovfs));
4146 		ovfs.vfc_vfsops = vfsp->vfc_vfsops;	/* XXX used as flag */
4147 		strcpy(ovfs.vfc_name, vfsp->vfc_name);
4148 		ovfs.vfc_index = vfsp->vfc_typenum;
4149 		ovfs.vfc_refcount = vfsp->vfc_refcount;
4150 		ovfs.vfc_flags = vfsp->vfc_flags;
4151 		error = SYSCTL_OUT(req, &ovfs, sizeof ovfs);
4152 		if (error != 0) {
4153 			vfsconf_sunlock();
4154 			return (error);
4155 		}
4156 	}
4157 	vfsconf_sunlock();
4158 	return (0);
4159 }
4160 
4161 #endif /* 1 || COMPAT_PRELITE2 */
4162 #endif /* !BURN_BRIDGES */
4163 
4164 #define KINFO_VNODESLOP		10
4165 #ifdef notyet
4166 /*
4167  * Dump vnode list (via sysctl).
4168  */
4169 /* ARGSUSED */
4170 static int
4171 sysctl_vnode(SYSCTL_HANDLER_ARGS)
4172 {
4173 	struct xvnode *xvn;
4174 	struct mount *mp;
4175 	struct vnode *vp;
4176 	int error, len, n;
4177 
4178 	/*
4179 	 * Stale numvnodes access is not fatal here.
4180 	 */
4181 	req->lock = 0;
4182 	len = (numvnodes + KINFO_VNODESLOP) * sizeof *xvn;
4183 	if (!req->oldptr)
4184 		/* Make an estimate */
4185 		return (SYSCTL_OUT(req, 0, len));
4186 
4187 	error = sysctl_wire_old_buffer(req, 0);
4188 	if (error != 0)
4189 		return (error);
4190 	xvn = malloc(len, M_TEMP, M_ZERO | M_WAITOK);
4191 	n = 0;
4192 	mtx_lock(&mountlist_mtx);
4193 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
4194 		if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK))
4195 			continue;
4196 		MNT_ILOCK(mp);
4197 		TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
4198 			if (n == len)
4199 				break;
4200 			vref(vp);
4201 			xvn[n].xv_size = sizeof *xvn;
4202 			xvn[n].xv_vnode = vp;
4203 			xvn[n].xv_id = 0;	/* XXX compat */
4204 #define XV_COPY(field) xvn[n].xv_##field = vp->v_##field
4205 			XV_COPY(usecount);
4206 			XV_COPY(writecount);
4207 			XV_COPY(holdcnt);
4208 			XV_COPY(mount);
4209 			XV_COPY(numoutput);
4210 			XV_COPY(type);
4211 #undef XV_COPY
4212 			xvn[n].xv_flag = vp->v_vflag;
4213 
4214 			switch (vp->v_type) {
4215 			case VREG:
4216 			case VDIR:
4217 			case VLNK:
4218 				break;
4219 			case VBLK:
4220 			case VCHR:
4221 				if (vp->v_rdev == NULL) {
4222 					vrele(vp);
4223 					continue;
4224 				}
4225 				xvn[n].xv_dev = dev2udev(vp->v_rdev);
4226 				break;
4227 			case VSOCK:
4228 				xvn[n].xv_socket = vp->v_socket;
4229 				break;
4230 			case VFIFO:
4231 				xvn[n].xv_fifo = vp->v_fifoinfo;
4232 				break;
4233 			case VNON:
4234 			case VBAD:
4235 			default:
4236 				/* shouldn't happen? */
4237 				vrele(vp);
4238 				continue;
4239 			}
4240 			vrele(vp);
4241 			++n;
4242 		}
4243 		MNT_IUNLOCK(mp);
4244 		mtx_lock(&mountlist_mtx);
4245 		vfs_unbusy(mp);
4246 		if (n == len)
4247 			break;
4248 	}
4249 	mtx_unlock(&mountlist_mtx);
4250 
4251 	error = SYSCTL_OUT(req, xvn, n * sizeof *xvn);
4252 	free(xvn, M_TEMP);
4253 	return (error);
4254 }
4255 
4256 SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE | CTLFLAG_RD |
4257     CTLFLAG_MPSAFE, 0, 0, sysctl_vnode, "S,xvnode",
4258     "");
4259 #endif
4260 
4261 static void
4262 unmount_or_warn(struct mount *mp)
4263 {
4264 	int error;
4265 
4266 	error = dounmount(mp, MNT_FORCE, curthread);
4267 	if (error != 0) {
4268 		printf("unmount of %s failed (", mp->mnt_stat.f_mntonname);
4269 		if (error == EBUSY)
4270 			printf("BUSY)\n");
4271 		else
4272 			printf("%d)\n", error);
4273 	}
4274 }
4275 
4276 /*
4277  * Unmount all filesystems. The list is traversed in reverse order
4278  * of mounting to avoid dependencies.
4279  */
4280 void
4281 vfs_unmountall(void)
4282 {
4283 	struct mount *mp, *tmp;
4284 
4285 	CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__);
4286 
4287 	/*
4288 	 * Since this only runs when rebooting, it is not interlocked.
4289 	 */
4290 	TAILQ_FOREACH_REVERSE_SAFE(mp, &mountlist, mntlist, mnt_list, tmp) {
4291 		vfs_ref(mp);
4292 
4293 		/*
4294 		 * Forcibly unmounting "/dev" before "/" would prevent clean
4295 		 * unmount of the latter.
4296 		 */
4297 		if (mp == rootdevmp)
4298 			continue;
4299 
4300 		unmount_or_warn(mp);
4301 	}
4302 
4303 	if (rootdevmp != NULL)
4304 		unmount_or_warn(rootdevmp);
4305 }
4306 
4307 /*
4308  * perform msync on all vnodes under a mount point
4309  * the mount point must be locked.
4310  */
4311 void
4312 vfs_msync(struct mount *mp, int flags)
4313 {
4314 	struct vnode *vp, *mvp;
4315 	struct vm_object *obj;
4316 
4317 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
4318 
4319 	vnlru_return_batch(mp);
4320 
4321 	MNT_VNODE_FOREACH_ACTIVE(vp, mp, mvp) {
4322 		obj = vp->v_object;
4323 		if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 &&
4324 		    (flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) {
4325 			if (!vget(vp,
4326 			    LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK,
4327 			    curthread)) {
4328 				if (vp->v_vflag & VV_NOSYNC) {	/* unlinked */
4329 					vput(vp);
4330 					continue;
4331 				}
4332 
4333 				obj = vp->v_object;
4334 				if (obj != NULL) {
4335 					VM_OBJECT_WLOCK(obj);
4336 					vm_object_page_clean(obj, 0, 0,
4337 					    flags == MNT_WAIT ?
4338 					    OBJPC_SYNC : OBJPC_NOSYNC);
4339 					VM_OBJECT_WUNLOCK(obj);
4340 				}
4341 				vput(vp);
4342 			}
4343 		} else
4344 			VI_UNLOCK(vp);
4345 	}
4346 }
4347 
4348 static void
4349 destroy_vpollinfo_free(struct vpollinfo *vi)
4350 {
4351 
4352 	knlist_destroy(&vi->vpi_selinfo.si_note);
4353 	mtx_destroy(&vi->vpi_lock);
4354 	uma_zfree(vnodepoll_zone, vi);
4355 }
4356 
4357 static void
4358 destroy_vpollinfo(struct vpollinfo *vi)
4359 {
4360 
4361 	knlist_clear(&vi->vpi_selinfo.si_note, 1);
4362 	seldrain(&vi->vpi_selinfo);
4363 	destroy_vpollinfo_free(vi);
4364 }
4365 
4366 /*
4367  * Initialize per-vnode helper structure to hold poll-related state.
4368  */
4369 void
4370 v_addpollinfo(struct vnode *vp)
4371 {
4372 	struct vpollinfo *vi;
4373 
4374 	if (vp->v_pollinfo != NULL)
4375 		return;
4376 	vi = uma_zalloc(vnodepoll_zone, M_WAITOK | M_ZERO);
4377 	mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF);
4378 	knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock,
4379 	    vfs_knlunlock, vfs_knl_assert_locked, vfs_knl_assert_unlocked);
4380 	VI_LOCK(vp);
4381 	if (vp->v_pollinfo != NULL) {
4382 		VI_UNLOCK(vp);
4383 		destroy_vpollinfo_free(vi);
4384 		return;
4385 	}
4386 	vp->v_pollinfo = vi;
4387 	VI_UNLOCK(vp);
4388 }
4389 
4390 /*
4391  * Record a process's interest in events which might happen to
4392  * a vnode.  Because poll uses the historic select-style interface
4393  * internally, this routine serves as both the ``check for any
4394  * pending events'' and the ``record my interest in future events''
4395  * functions.  (These are done together, while the lock is held,
4396  * to avoid race conditions.)
4397  */
4398 int
4399 vn_pollrecord(struct vnode *vp, struct thread *td, int events)
4400 {
4401 
4402 	v_addpollinfo(vp);
4403 	mtx_lock(&vp->v_pollinfo->vpi_lock);
4404 	if (vp->v_pollinfo->vpi_revents & events) {
4405 		/*
4406 		 * This leaves events we are not interested
4407 		 * in available for the other process which
4408 		 * which presumably had requested them
4409 		 * (otherwise they would never have been
4410 		 * recorded).
4411 		 */
4412 		events &= vp->v_pollinfo->vpi_revents;
4413 		vp->v_pollinfo->vpi_revents &= ~events;
4414 
4415 		mtx_unlock(&vp->v_pollinfo->vpi_lock);
4416 		return (events);
4417 	}
4418 	vp->v_pollinfo->vpi_events |= events;
4419 	selrecord(td, &vp->v_pollinfo->vpi_selinfo);
4420 	mtx_unlock(&vp->v_pollinfo->vpi_lock);
4421 	return (0);
4422 }
4423 
4424 /*
4425  * Routine to create and manage a filesystem syncer vnode.
4426  */
4427 #define sync_close ((int (*)(struct  vop_close_args *))nullop)
4428 static int	sync_fsync(struct  vop_fsync_args *);
4429 static int	sync_inactive(struct  vop_inactive_args *);
4430 static int	sync_reclaim(struct  vop_reclaim_args *);
4431 
4432 static struct vop_vector sync_vnodeops = {
4433 	.vop_bypass =	VOP_EOPNOTSUPP,
4434 	.vop_close =	sync_close,		/* close */
4435 	.vop_fsync =	sync_fsync,		/* fsync */
4436 	.vop_inactive =	sync_inactive,	/* inactive */
4437 	.vop_need_inactive = vop_stdneed_inactive, /* need_inactive */
4438 	.vop_reclaim =	sync_reclaim,	/* reclaim */
4439 	.vop_lock1 =	vop_stdlock,	/* lock */
4440 	.vop_unlock =	vop_stdunlock,	/* unlock */
4441 	.vop_islocked =	vop_stdislocked,	/* islocked */
4442 };
4443 
4444 /*
4445  * Create a new filesystem syncer vnode for the specified mount point.
4446  */
4447 void
4448 vfs_allocate_syncvnode(struct mount *mp)
4449 {
4450 	struct vnode *vp;
4451 	struct bufobj *bo;
4452 	static long start, incr, next;
4453 	int error;
4454 
4455 	/* Allocate a new vnode */
4456 	error = getnewvnode("syncer", mp, &sync_vnodeops, &vp);
4457 	if (error != 0)
4458 		panic("vfs_allocate_syncvnode: getnewvnode() failed");
4459 	vp->v_type = VNON;
4460 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
4461 	vp->v_vflag |= VV_FORCEINSMQ;
4462 	error = insmntque(vp, mp);
4463 	if (error != 0)
4464 		panic("vfs_allocate_syncvnode: insmntque() failed");
4465 	vp->v_vflag &= ~VV_FORCEINSMQ;
4466 	VOP_UNLOCK(vp, 0);
4467 	/*
4468 	 * Place the vnode onto the syncer worklist. We attempt to
4469 	 * scatter them about on the list so that they will go off
4470 	 * at evenly distributed times even if all the filesystems
4471 	 * are mounted at once.
4472 	 */
4473 	next += incr;
4474 	if (next == 0 || next > syncer_maxdelay) {
4475 		start /= 2;
4476 		incr /= 2;
4477 		if (start == 0) {
4478 			start = syncer_maxdelay / 2;
4479 			incr = syncer_maxdelay;
4480 		}
4481 		next = start;
4482 	}
4483 	bo = &vp->v_bufobj;
4484 	BO_LOCK(bo);
4485 	vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0);
4486 	/* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */
4487 	mtx_lock(&sync_mtx);
4488 	sync_vnode_count++;
4489 	if (mp->mnt_syncer == NULL) {
4490 		mp->mnt_syncer = vp;
4491 		vp = NULL;
4492 	}
4493 	mtx_unlock(&sync_mtx);
4494 	BO_UNLOCK(bo);
4495 	if (vp != NULL) {
4496 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
4497 		vgone(vp);
4498 		vput(vp);
4499 	}
4500 }
4501 
4502 void
4503 vfs_deallocate_syncvnode(struct mount *mp)
4504 {
4505 	struct vnode *vp;
4506 
4507 	mtx_lock(&sync_mtx);
4508 	vp = mp->mnt_syncer;
4509 	if (vp != NULL)
4510 		mp->mnt_syncer = NULL;
4511 	mtx_unlock(&sync_mtx);
4512 	if (vp != NULL)
4513 		vrele(vp);
4514 }
4515 
4516 /*
4517  * Do a lazy sync of the filesystem.
4518  */
4519 static int
4520 sync_fsync(struct vop_fsync_args *ap)
4521 {
4522 	struct vnode *syncvp = ap->a_vp;
4523 	struct mount *mp = syncvp->v_mount;
4524 	int error, save;
4525 	struct bufobj *bo;
4526 
4527 	/*
4528 	 * We only need to do something if this is a lazy evaluation.
4529 	 */
4530 	if (ap->a_waitfor != MNT_LAZY)
4531 		return (0);
4532 
4533 	/*
4534 	 * Move ourselves to the back of the sync list.
4535 	 */
4536 	bo = &syncvp->v_bufobj;
4537 	BO_LOCK(bo);
4538 	vn_syncer_add_to_worklist(bo, syncdelay);
4539 	BO_UNLOCK(bo);
4540 
4541 	/*
4542 	 * Walk the list of vnodes pushing all that are dirty and
4543 	 * not already on the sync list.
4544 	 */
4545 	if (vfs_busy(mp, MBF_NOWAIT) != 0)
4546 		return (0);
4547 	if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) {
4548 		vfs_unbusy(mp);
4549 		return (0);
4550 	}
4551 	save = curthread_pflags_set(TDP_SYNCIO);
4552 	vfs_msync(mp, MNT_NOWAIT);
4553 	error = VFS_SYNC(mp, MNT_LAZY);
4554 	curthread_pflags_restore(save);
4555 	vn_finished_write(mp);
4556 	vfs_unbusy(mp);
4557 	return (error);
4558 }
4559 
4560 /*
4561  * The syncer vnode is no referenced.
4562  */
4563 static int
4564 sync_inactive(struct vop_inactive_args *ap)
4565 {
4566 
4567 	vgone(ap->a_vp);
4568 	return (0);
4569 }
4570 
4571 /*
4572  * The syncer vnode is no longer needed and is being decommissioned.
4573  *
4574  * Modifications to the worklist must be protected by sync_mtx.
4575  */
4576 static int
4577 sync_reclaim(struct vop_reclaim_args *ap)
4578 {
4579 	struct vnode *vp = ap->a_vp;
4580 	struct bufobj *bo;
4581 
4582 	bo = &vp->v_bufobj;
4583 	BO_LOCK(bo);
4584 	mtx_lock(&sync_mtx);
4585 	if (vp->v_mount->mnt_syncer == vp)
4586 		vp->v_mount->mnt_syncer = NULL;
4587 	if (bo->bo_flag & BO_ONWORKLST) {
4588 		LIST_REMOVE(bo, bo_synclist);
4589 		syncer_worklist_len--;
4590 		sync_vnode_count--;
4591 		bo->bo_flag &= ~BO_ONWORKLST;
4592 	}
4593 	mtx_unlock(&sync_mtx);
4594 	BO_UNLOCK(bo);
4595 
4596 	return (0);
4597 }
4598 
4599 int
4600 vn_need_pageq_flush(struct vnode *vp)
4601 {
4602 	struct vm_object *obj;
4603 	int need;
4604 
4605 	MPASS(mtx_owned(VI_MTX(vp)));
4606 	need = 0;
4607 	if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
4608 	    (obj->flags & OBJ_MIGHTBEDIRTY) != 0)
4609 		need = 1;
4610 	return (need);
4611 }
4612 
4613 /*
4614  * Check if vnode represents a disk device
4615  */
4616 int
4617 vn_isdisk(struct vnode *vp, int *errp)
4618 {
4619 	int error;
4620 
4621 	if (vp->v_type != VCHR) {
4622 		error = ENOTBLK;
4623 		goto out;
4624 	}
4625 	error = 0;
4626 	dev_lock();
4627 	if (vp->v_rdev == NULL)
4628 		error = ENXIO;
4629 	else if (vp->v_rdev->si_devsw == NULL)
4630 		error = ENXIO;
4631 	else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK))
4632 		error = ENOTBLK;
4633 	dev_unlock();
4634 out:
4635 	if (errp != NULL)
4636 		*errp = error;
4637 	return (error == 0);
4638 }
4639 
4640 /*
4641  * Common filesystem object access control check routine.  Accepts a
4642  * vnode's type, "mode", uid and gid, requested access mode, credentials,
4643  * and optional call-by-reference privused argument allowing vaccess()
4644  * to indicate to the caller whether privilege was used to satisfy the
4645  * request (obsoleted).  Returns 0 on success, or an errno on failure.
4646  */
4647 int
4648 vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid,
4649     accmode_t accmode, struct ucred *cred, int *privused)
4650 {
4651 	accmode_t dac_granted;
4652 	accmode_t priv_granted;
4653 
4654 	KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0,
4655 	    ("invalid bit in accmode"));
4656 	KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE),
4657 	    ("VAPPEND without VWRITE"));
4658 
4659 	/*
4660 	 * Look for a normal, non-privileged way to access the file/directory
4661 	 * as requested.  If it exists, go with that.
4662 	 */
4663 
4664 	if (privused != NULL)
4665 		*privused = 0;
4666 
4667 	dac_granted = 0;
4668 
4669 	/* Check the owner. */
4670 	if (cred->cr_uid == file_uid) {
4671 		dac_granted |= VADMIN;
4672 		if (file_mode & S_IXUSR)
4673 			dac_granted |= VEXEC;
4674 		if (file_mode & S_IRUSR)
4675 			dac_granted |= VREAD;
4676 		if (file_mode & S_IWUSR)
4677 			dac_granted |= (VWRITE | VAPPEND);
4678 
4679 		if ((accmode & dac_granted) == accmode)
4680 			return (0);
4681 
4682 		goto privcheck;
4683 	}
4684 
4685 	/* Otherwise, check the groups (first match) */
4686 	if (groupmember(file_gid, cred)) {
4687 		if (file_mode & S_IXGRP)
4688 			dac_granted |= VEXEC;
4689 		if (file_mode & S_IRGRP)
4690 			dac_granted |= VREAD;
4691 		if (file_mode & S_IWGRP)
4692 			dac_granted |= (VWRITE | VAPPEND);
4693 
4694 		if ((accmode & dac_granted) == accmode)
4695 			return (0);
4696 
4697 		goto privcheck;
4698 	}
4699 
4700 	/* Otherwise, check everyone else. */
4701 	if (file_mode & S_IXOTH)
4702 		dac_granted |= VEXEC;
4703 	if (file_mode & S_IROTH)
4704 		dac_granted |= VREAD;
4705 	if (file_mode & S_IWOTH)
4706 		dac_granted |= (VWRITE | VAPPEND);
4707 	if ((accmode & dac_granted) == accmode)
4708 		return (0);
4709 
4710 privcheck:
4711 	/*
4712 	 * Build a privilege mask to determine if the set of privileges
4713 	 * satisfies the requirements when combined with the granted mask
4714 	 * from above.  For each privilege, if the privilege is required,
4715 	 * bitwise or the request type onto the priv_granted mask.
4716 	 */
4717 	priv_granted = 0;
4718 
4719 	if (type == VDIR) {
4720 		/*
4721 		 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC
4722 		 * requests, instead of PRIV_VFS_EXEC.
4723 		 */
4724 		if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
4725 		    !priv_check_cred(cred, PRIV_VFS_LOOKUP))
4726 			priv_granted |= VEXEC;
4727 	} else {
4728 		/*
4729 		 * Ensure that at least one execute bit is on. Otherwise,
4730 		 * a privileged user will always succeed, and we don't want
4731 		 * this to happen unless the file really is executable.
4732 		 */
4733 		if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
4734 		    (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 &&
4735 		    !priv_check_cred(cred, PRIV_VFS_EXEC))
4736 			priv_granted |= VEXEC;
4737 	}
4738 
4739 	if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) &&
4740 	    !priv_check_cred(cred, PRIV_VFS_READ))
4741 		priv_granted |= VREAD;
4742 
4743 	if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) &&
4744 	    !priv_check_cred(cred, PRIV_VFS_WRITE))
4745 		priv_granted |= (VWRITE | VAPPEND);
4746 
4747 	if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) &&
4748 	    !priv_check_cred(cred, PRIV_VFS_ADMIN))
4749 		priv_granted |= VADMIN;
4750 
4751 	if ((accmode & (priv_granted | dac_granted)) == accmode) {
4752 		/* XXX audit: privilege used */
4753 		if (privused != NULL)
4754 			*privused = 1;
4755 		return (0);
4756 	}
4757 
4758 	return ((accmode & VADMIN) ? EPERM : EACCES);
4759 }
4760 
4761 /*
4762  * Credential check based on process requesting service, and per-attribute
4763  * permissions.
4764  */
4765 int
4766 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred,
4767     struct thread *td, accmode_t accmode)
4768 {
4769 
4770 	/*
4771 	 * Kernel-invoked always succeeds.
4772 	 */
4773 	if (cred == NOCRED)
4774 		return (0);
4775 
4776 	/*
4777 	 * Do not allow privileged processes in jail to directly manipulate
4778 	 * system attributes.
4779 	 */
4780 	switch (attrnamespace) {
4781 	case EXTATTR_NAMESPACE_SYSTEM:
4782 		/* Potentially should be: return (EPERM); */
4783 		return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM));
4784 	case EXTATTR_NAMESPACE_USER:
4785 		return (VOP_ACCESS(vp, accmode, cred, td));
4786 	default:
4787 		return (EPERM);
4788 	}
4789 }
4790 
4791 #ifdef DEBUG_VFS_LOCKS
4792 /*
4793  * This only exists to suppress warnings from unlocked specfs accesses.  It is
4794  * no longer ok to have an unlocked VFS.
4795  */
4796 #define	IGNORE_LOCK(vp) (panicstr != NULL || (vp) == NULL ||		\
4797 	(vp)->v_type == VCHR ||	(vp)->v_type == VBAD)
4798 
4799 int vfs_badlock_ddb = 1;	/* Drop into debugger on violation. */
4800 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0,
4801     "Drop into debugger on lock violation");
4802 
4803 int vfs_badlock_mutex = 1;	/* Check for interlock across VOPs. */
4804 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex,
4805     0, "Check for interlock across VOPs");
4806 
4807 int vfs_badlock_print = 1;	/* Print lock violations. */
4808 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print,
4809     0, "Print lock violations");
4810 
4811 int vfs_badlock_vnode = 1;	/* Print vnode details on lock violations. */
4812 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_vnode, CTLFLAG_RW, &vfs_badlock_vnode,
4813     0, "Print vnode details on lock violations");
4814 
4815 #ifdef KDB
4816 int vfs_badlock_backtrace = 1;	/* Print backtrace at lock violations. */
4817 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW,
4818     &vfs_badlock_backtrace, 0, "Print backtrace at lock violations");
4819 #endif
4820 
4821 static void
4822 vfs_badlock(const char *msg, const char *str, struct vnode *vp)
4823 {
4824 
4825 #ifdef KDB
4826 	if (vfs_badlock_backtrace)
4827 		kdb_backtrace();
4828 #endif
4829 	if (vfs_badlock_vnode)
4830 		vn_printf(vp, "vnode ");
4831 	if (vfs_badlock_print)
4832 		printf("%s: %p %s\n", str, (void *)vp, msg);
4833 	if (vfs_badlock_ddb)
4834 		kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
4835 }
4836 
4837 void
4838 assert_vi_locked(struct vnode *vp, const char *str)
4839 {
4840 
4841 	if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp)))
4842 		vfs_badlock("interlock is not locked but should be", str, vp);
4843 }
4844 
4845 void
4846 assert_vi_unlocked(struct vnode *vp, const char *str)
4847 {
4848 
4849 	if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp)))
4850 		vfs_badlock("interlock is locked but should not be", str, vp);
4851 }
4852 
4853 void
4854 assert_vop_locked(struct vnode *vp, const char *str)
4855 {
4856 	int locked;
4857 
4858 	if (!IGNORE_LOCK(vp)) {
4859 		locked = VOP_ISLOCKED(vp);
4860 		if (locked == 0 || locked == LK_EXCLOTHER)
4861 			vfs_badlock("is not locked but should be", str, vp);
4862 	}
4863 }
4864 
4865 void
4866 assert_vop_unlocked(struct vnode *vp, const char *str)
4867 {
4868 
4869 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
4870 		vfs_badlock("is locked but should not be", str, vp);
4871 }
4872 
4873 void
4874 assert_vop_elocked(struct vnode *vp, const char *str)
4875 {
4876 
4877 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
4878 		vfs_badlock("is not exclusive locked but should be", str, vp);
4879 }
4880 #endif /* DEBUG_VFS_LOCKS */
4881 
4882 void
4883 vop_rename_fail(struct vop_rename_args *ap)
4884 {
4885 
4886 	if (ap->a_tvp != NULL)
4887 		vput(ap->a_tvp);
4888 	if (ap->a_tdvp == ap->a_tvp)
4889 		vrele(ap->a_tdvp);
4890 	else
4891 		vput(ap->a_tdvp);
4892 	vrele(ap->a_fdvp);
4893 	vrele(ap->a_fvp);
4894 }
4895 
4896 void
4897 vop_rename_pre(void *ap)
4898 {
4899 	struct vop_rename_args *a = ap;
4900 
4901 #ifdef DEBUG_VFS_LOCKS
4902 	if (a->a_tvp)
4903 		ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME");
4904 	ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME");
4905 	ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME");
4906 	ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME");
4907 
4908 	/* Check the source (from). */
4909 	if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock &&
4910 	    (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock))
4911 		ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked");
4912 	if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock)
4913 		ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked");
4914 
4915 	/* Check the target. */
4916 	if (a->a_tvp)
4917 		ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked");
4918 	ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked");
4919 #endif
4920 	if (a->a_tdvp != a->a_fdvp)
4921 		vhold(a->a_fdvp);
4922 	if (a->a_tvp != a->a_fvp)
4923 		vhold(a->a_fvp);
4924 	vhold(a->a_tdvp);
4925 	if (a->a_tvp)
4926 		vhold(a->a_tvp);
4927 }
4928 
4929 #ifdef DEBUG_VFS_LOCKS
4930 void
4931 vop_strategy_pre(void *ap)
4932 {
4933 	struct vop_strategy_args *a;
4934 	struct buf *bp;
4935 
4936 	a = ap;
4937 	bp = a->a_bp;
4938 
4939 	/*
4940 	 * Cluster ops lock their component buffers but not the IO container.
4941 	 */
4942 	if ((bp->b_flags & B_CLUSTER) != 0)
4943 		return;
4944 
4945 	if (panicstr == NULL && !BUF_ISLOCKED(bp)) {
4946 		if (vfs_badlock_print)
4947 			printf(
4948 			    "VOP_STRATEGY: bp is not locked but should be\n");
4949 		if (vfs_badlock_ddb)
4950 			kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
4951 	}
4952 }
4953 
4954 void
4955 vop_lock_pre(void *ap)
4956 {
4957 	struct vop_lock1_args *a = ap;
4958 
4959 	if ((a->a_flags & LK_INTERLOCK) == 0)
4960 		ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
4961 	else
4962 		ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK");
4963 }
4964 
4965 void
4966 vop_lock_post(void *ap, int rc)
4967 {
4968 	struct vop_lock1_args *a = ap;
4969 
4970 	ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
4971 	if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0)
4972 		ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK");
4973 }
4974 
4975 void
4976 vop_unlock_pre(void *ap)
4977 {
4978 	struct vop_unlock_args *a = ap;
4979 
4980 	if (a->a_flags & LK_INTERLOCK)
4981 		ASSERT_VI_LOCKED(a->a_vp, "VOP_UNLOCK");
4982 	ASSERT_VOP_LOCKED(a->a_vp, "VOP_UNLOCK");
4983 }
4984 
4985 void
4986 vop_unlock_post(void *ap, int rc)
4987 {
4988 	struct vop_unlock_args *a = ap;
4989 
4990 	if (a->a_flags & LK_INTERLOCK)
4991 		ASSERT_VI_UNLOCKED(a->a_vp, "VOP_UNLOCK");
4992 }
4993 
4994 void
4995 vop_need_inactive_pre(void *ap)
4996 {
4997 	struct vop_need_inactive_args *a = ap;
4998 
4999 	ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE");
5000 }
5001 
5002 void
5003 vop_need_inactive_post(void *ap, int rc)
5004 {
5005 	struct vop_need_inactive_args *a = ap;
5006 
5007 	ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE");
5008 }
5009 #endif
5010 
5011 void
5012 vop_create_post(void *ap, int rc)
5013 {
5014 	struct vop_create_args *a = ap;
5015 
5016 	if (!rc)
5017 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
5018 }
5019 
5020 void
5021 vop_deleteextattr_post(void *ap, int rc)
5022 {
5023 	struct vop_deleteextattr_args *a = ap;
5024 
5025 	if (!rc)
5026 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
5027 }
5028 
5029 void
5030 vop_link_post(void *ap, int rc)
5031 {
5032 	struct vop_link_args *a = ap;
5033 
5034 	if (!rc) {
5035 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_LINK);
5036 		VFS_KNOTE_LOCKED(a->a_tdvp, NOTE_WRITE);
5037 	}
5038 }
5039 
5040 void
5041 vop_mkdir_post(void *ap, int rc)
5042 {
5043 	struct vop_mkdir_args *a = ap;
5044 
5045 	if (!rc)
5046 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
5047 }
5048 
5049 void
5050 vop_mknod_post(void *ap, int rc)
5051 {
5052 	struct vop_mknod_args *a = ap;
5053 
5054 	if (!rc)
5055 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
5056 }
5057 
5058 void
5059 vop_reclaim_post(void *ap, int rc)
5060 {
5061 	struct vop_reclaim_args *a = ap;
5062 
5063 	if (!rc)
5064 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_REVOKE);
5065 }
5066 
5067 void
5068 vop_remove_post(void *ap, int rc)
5069 {
5070 	struct vop_remove_args *a = ap;
5071 
5072 	if (!rc) {
5073 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
5074 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
5075 	}
5076 }
5077 
5078 void
5079 vop_rename_post(void *ap, int rc)
5080 {
5081 	struct vop_rename_args *a = ap;
5082 	long hint;
5083 
5084 	if (!rc) {
5085 		hint = NOTE_WRITE;
5086 		if (a->a_fdvp == a->a_tdvp) {
5087 			if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR)
5088 				hint |= NOTE_LINK;
5089 			VFS_KNOTE_UNLOCKED(a->a_fdvp, hint);
5090 			VFS_KNOTE_UNLOCKED(a->a_tdvp, hint);
5091 		} else {
5092 			hint |= NOTE_EXTEND;
5093 			if (a->a_fvp->v_type == VDIR)
5094 				hint |= NOTE_LINK;
5095 			VFS_KNOTE_UNLOCKED(a->a_fdvp, hint);
5096 
5097 			if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL &&
5098 			    a->a_tvp->v_type == VDIR)
5099 				hint &= ~NOTE_LINK;
5100 			VFS_KNOTE_UNLOCKED(a->a_tdvp, hint);
5101 		}
5102 
5103 		VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME);
5104 		if (a->a_tvp)
5105 			VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE);
5106 	}
5107 	if (a->a_tdvp != a->a_fdvp)
5108 		vdrop(a->a_fdvp);
5109 	if (a->a_tvp != a->a_fvp)
5110 		vdrop(a->a_fvp);
5111 	vdrop(a->a_tdvp);
5112 	if (a->a_tvp)
5113 		vdrop(a->a_tvp);
5114 }
5115 
5116 void
5117 vop_rmdir_post(void *ap, int rc)
5118 {
5119 	struct vop_rmdir_args *a = ap;
5120 
5121 	if (!rc) {
5122 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
5123 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
5124 	}
5125 }
5126 
5127 void
5128 vop_setattr_post(void *ap, int rc)
5129 {
5130 	struct vop_setattr_args *a = ap;
5131 
5132 	if (!rc)
5133 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
5134 }
5135 
5136 void
5137 vop_setextattr_post(void *ap, int rc)
5138 {
5139 	struct vop_setextattr_args *a = ap;
5140 
5141 	if (!rc)
5142 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
5143 }
5144 
5145 void
5146 vop_symlink_post(void *ap, int rc)
5147 {
5148 	struct vop_symlink_args *a = ap;
5149 
5150 	if (!rc)
5151 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
5152 }
5153 
5154 void
5155 vop_open_post(void *ap, int rc)
5156 {
5157 	struct vop_open_args *a = ap;
5158 
5159 	if (!rc)
5160 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN);
5161 }
5162 
5163 void
5164 vop_close_post(void *ap, int rc)
5165 {
5166 	struct vop_close_args *a = ap;
5167 
5168 	if (!rc && (a->a_cred != NOCRED || /* filter out revokes */
5169 	    (a->a_vp->v_iflag & VI_DOOMED) == 0)) {
5170 		VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ?
5171 		    NOTE_CLOSE_WRITE : NOTE_CLOSE);
5172 	}
5173 }
5174 
5175 void
5176 vop_read_post(void *ap, int rc)
5177 {
5178 	struct vop_read_args *a = ap;
5179 
5180 	if (!rc)
5181 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ);
5182 }
5183 
5184 void
5185 vop_readdir_post(void *ap, int rc)
5186 {
5187 	struct vop_readdir_args *a = ap;
5188 
5189 	if (!rc)
5190 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ);
5191 }
5192 
5193 static struct knlist fs_knlist;
5194 
5195 static void
5196 vfs_event_init(void *arg)
5197 {
5198 	knlist_init_mtx(&fs_knlist, NULL);
5199 }
5200 /* XXX - correct order? */
5201 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL);
5202 
5203 void
5204 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused)
5205 {
5206 
5207 	KNOTE_UNLOCKED(&fs_knlist, event);
5208 }
5209 
5210 static int	filt_fsattach(struct knote *kn);
5211 static void	filt_fsdetach(struct knote *kn);
5212 static int	filt_fsevent(struct knote *kn, long hint);
5213 
5214 struct filterops fs_filtops = {
5215 	.f_isfd = 0,
5216 	.f_attach = filt_fsattach,
5217 	.f_detach = filt_fsdetach,
5218 	.f_event = filt_fsevent
5219 };
5220 
5221 static int
5222 filt_fsattach(struct knote *kn)
5223 {
5224 
5225 	kn->kn_flags |= EV_CLEAR;
5226 	knlist_add(&fs_knlist, kn, 0);
5227 	return (0);
5228 }
5229 
5230 static void
5231 filt_fsdetach(struct knote *kn)
5232 {
5233 
5234 	knlist_remove(&fs_knlist, kn, 0);
5235 }
5236 
5237 static int
5238 filt_fsevent(struct knote *kn, long hint)
5239 {
5240 
5241 	kn->kn_fflags |= hint;
5242 	return (kn->kn_fflags != 0);
5243 }
5244 
5245 static int
5246 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS)
5247 {
5248 	struct vfsidctl vc;
5249 	int error;
5250 	struct mount *mp;
5251 
5252 	error = SYSCTL_IN(req, &vc, sizeof(vc));
5253 	if (error)
5254 		return (error);
5255 	if (vc.vc_vers != VFS_CTL_VERS1)
5256 		return (EINVAL);
5257 	mp = vfs_getvfs(&vc.vc_fsid);
5258 	if (mp == NULL)
5259 		return (ENOENT);
5260 	/* ensure that a specific sysctl goes to the right filesystem. */
5261 	if (strcmp(vc.vc_fstypename, "*") != 0 &&
5262 	    strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) {
5263 		vfs_rel(mp);
5264 		return (EINVAL);
5265 	}
5266 	VCTLTOREQ(&vc, req);
5267 	error = VFS_SYSCTL(mp, vc.vc_op, req);
5268 	vfs_rel(mp);
5269 	return (error);
5270 }
5271 
5272 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_WR,
5273     NULL, 0, sysctl_vfs_ctl, "",
5274     "Sysctl by fsid");
5275 
5276 /*
5277  * Function to initialize a va_filerev field sensibly.
5278  * XXX: Wouldn't a random number make a lot more sense ??
5279  */
5280 u_quad_t
5281 init_va_filerev(void)
5282 {
5283 	struct bintime bt;
5284 
5285 	getbinuptime(&bt);
5286 	return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL));
5287 }
5288 
5289 static int	filt_vfsread(struct knote *kn, long hint);
5290 static int	filt_vfswrite(struct knote *kn, long hint);
5291 static int	filt_vfsvnode(struct knote *kn, long hint);
5292 static void	filt_vfsdetach(struct knote *kn);
5293 static struct filterops vfsread_filtops = {
5294 	.f_isfd = 1,
5295 	.f_detach = filt_vfsdetach,
5296 	.f_event = filt_vfsread
5297 };
5298 static struct filterops vfswrite_filtops = {
5299 	.f_isfd = 1,
5300 	.f_detach = filt_vfsdetach,
5301 	.f_event = filt_vfswrite
5302 };
5303 static struct filterops vfsvnode_filtops = {
5304 	.f_isfd = 1,
5305 	.f_detach = filt_vfsdetach,
5306 	.f_event = filt_vfsvnode
5307 };
5308 
5309 static void
5310 vfs_knllock(void *arg)
5311 {
5312 	struct vnode *vp = arg;
5313 
5314 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
5315 }
5316 
5317 static void
5318 vfs_knlunlock(void *arg)
5319 {
5320 	struct vnode *vp = arg;
5321 
5322 	VOP_UNLOCK(vp, 0);
5323 }
5324 
5325 static void
5326 vfs_knl_assert_locked(void *arg)
5327 {
5328 #ifdef DEBUG_VFS_LOCKS
5329 	struct vnode *vp = arg;
5330 
5331 	ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked");
5332 #endif
5333 }
5334 
5335 static void
5336 vfs_knl_assert_unlocked(void *arg)
5337 {
5338 #ifdef DEBUG_VFS_LOCKS
5339 	struct vnode *vp = arg;
5340 
5341 	ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked");
5342 #endif
5343 }
5344 
5345 int
5346 vfs_kqfilter(struct vop_kqfilter_args *ap)
5347 {
5348 	struct vnode *vp = ap->a_vp;
5349 	struct knote *kn = ap->a_kn;
5350 	struct knlist *knl;
5351 
5352 	switch (kn->kn_filter) {
5353 	case EVFILT_READ:
5354 		kn->kn_fop = &vfsread_filtops;
5355 		break;
5356 	case EVFILT_WRITE:
5357 		kn->kn_fop = &vfswrite_filtops;
5358 		break;
5359 	case EVFILT_VNODE:
5360 		kn->kn_fop = &vfsvnode_filtops;
5361 		break;
5362 	default:
5363 		return (EINVAL);
5364 	}
5365 
5366 	kn->kn_hook = (caddr_t)vp;
5367 
5368 	v_addpollinfo(vp);
5369 	if (vp->v_pollinfo == NULL)
5370 		return (ENOMEM);
5371 	knl = &vp->v_pollinfo->vpi_selinfo.si_note;
5372 	vhold(vp);
5373 	knlist_add(knl, kn, 0);
5374 
5375 	return (0);
5376 }
5377 
5378 /*
5379  * Detach knote from vnode
5380  */
5381 static void
5382 filt_vfsdetach(struct knote *kn)
5383 {
5384 	struct vnode *vp = (struct vnode *)kn->kn_hook;
5385 
5386 	KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo"));
5387 	knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0);
5388 	vdrop(vp);
5389 }
5390 
5391 /*ARGSUSED*/
5392 static int
5393 filt_vfsread(struct knote *kn, long hint)
5394 {
5395 	struct vnode *vp = (struct vnode *)kn->kn_hook;
5396 	struct vattr va;
5397 	int res;
5398 
5399 	/*
5400 	 * filesystem is gone, so set the EOF flag and schedule
5401 	 * the knote for deletion.
5402 	 */
5403 	if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) {
5404 		VI_LOCK(vp);
5405 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
5406 		VI_UNLOCK(vp);
5407 		return (1);
5408 	}
5409 
5410 	if (VOP_GETATTR(vp, &va, curthread->td_ucred))
5411 		return (0);
5412 
5413 	VI_LOCK(vp);
5414 	kn->kn_data = va.va_size - kn->kn_fp->f_offset;
5415 	res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 || kn->kn_data != 0;
5416 	VI_UNLOCK(vp);
5417 	return (res);
5418 }
5419 
5420 /*ARGSUSED*/
5421 static int
5422 filt_vfswrite(struct knote *kn, long hint)
5423 {
5424 	struct vnode *vp = (struct vnode *)kn->kn_hook;
5425 
5426 	VI_LOCK(vp);
5427 
5428 	/*
5429 	 * filesystem is gone, so set the EOF flag and schedule
5430 	 * the knote for deletion.
5431 	 */
5432 	if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD))
5433 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
5434 
5435 	kn->kn_data = 0;
5436 	VI_UNLOCK(vp);
5437 	return (1);
5438 }
5439 
5440 static int
5441 filt_vfsvnode(struct knote *kn, long hint)
5442 {
5443 	struct vnode *vp = (struct vnode *)kn->kn_hook;
5444 	int res;
5445 
5446 	VI_LOCK(vp);
5447 	if (kn->kn_sfflags & hint)
5448 		kn->kn_fflags |= hint;
5449 	if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) {
5450 		kn->kn_flags |= EV_EOF;
5451 		VI_UNLOCK(vp);
5452 		return (1);
5453 	}
5454 	res = (kn->kn_fflags != 0);
5455 	VI_UNLOCK(vp);
5456 	return (res);
5457 }
5458 
5459 int
5460 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off)
5461 {
5462 	int error;
5463 
5464 	if (dp->d_reclen > ap->a_uio->uio_resid)
5465 		return (ENAMETOOLONG);
5466 	error = uiomove(dp, dp->d_reclen, ap->a_uio);
5467 	if (error) {
5468 		if (ap->a_ncookies != NULL) {
5469 			if (ap->a_cookies != NULL)
5470 				free(ap->a_cookies, M_TEMP);
5471 			ap->a_cookies = NULL;
5472 			*ap->a_ncookies = 0;
5473 		}
5474 		return (error);
5475 	}
5476 	if (ap->a_ncookies == NULL)
5477 		return (0);
5478 
5479 	KASSERT(ap->a_cookies,
5480 	    ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!"));
5481 
5482 	*ap->a_cookies = realloc(*ap->a_cookies,
5483 	    (*ap->a_ncookies + 1) * sizeof(u_long), M_TEMP, M_WAITOK | M_ZERO);
5484 	(*ap->a_cookies)[*ap->a_ncookies] = off;
5485 	*ap->a_ncookies += 1;
5486 	return (0);
5487 }
5488 
5489 /*
5490  * Mark for update the access time of the file if the filesystem
5491  * supports VOP_MARKATIME.  This functionality is used by execve and
5492  * mmap, so we want to avoid the I/O implied by directly setting
5493  * va_atime for the sake of efficiency.
5494  */
5495 void
5496 vfs_mark_atime(struct vnode *vp, struct ucred *cred)
5497 {
5498 	struct mount *mp;
5499 
5500 	mp = vp->v_mount;
5501 	ASSERT_VOP_LOCKED(vp, "vfs_mark_atime");
5502 	if (mp != NULL && (mp->mnt_flag & (MNT_NOATIME | MNT_RDONLY)) == 0)
5503 		(void)VOP_MARKATIME(vp);
5504 }
5505 
5506 /*
5507  * The purpose of this routine is to remove granularity from accmode_t,
5508  * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE,
5509  * VADMIN and VAPPEND.
5510  *
5511  * If it returns 0, the caller is supposed to continue with the usual
5512  * access checks using 'accmode' as modified by this routine.  If it
5513  * returns nonzero value, the caller is supposed to return that value
5514  * as errno.
5515  *
5516  * Note that after this routine runs, accmode may be zero.
5517  */
5518 int
5519 vfs_unixify_accmode(accmode_t *accmode)
5520 {
5521 	/*
5522 	 * There is no way to specify explicit "deny" rule using
5523 	 * file mode or POSIX.1e ACLs.
5524 	 */
5525 	if (*accmode & VEXPLICIT_DENY) {
5526 		*accmode = 0;
5527 		return (0);
5528 	}
5529 
5530 	/*
5531 	 * None of these can be translated into usual access bits.
5532 	 * Also, the common case for NFSv4 ACLs is to not contain
5533 	 * either of these bits. Caller should check for VWRITE
5534 	 * on the containing directory instead.
5535 	 */
5536 	if (*accmode & (VDELETE_CHILD | VDELETE))
5537 		return (EPERM);
5538 
5539 	if (*accmode & VADMIN_PERMS) {
5540 		*accmode &= ~VADMIN_PERMS;
5541 		*accmode |= VADMIN;
5542 	}
5543 
5544 	/*
5545 	 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL
5546 	 * or VSYNCHRONIZE using file mode or POSIX.1e ACL.
5547 	 */
5548 	*accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE);
5549 
5550 	return (0);
5551 }
5552 
5553 /*
5554  * These are helper functions for filesystems to traverse all
5555  * their vnodes.  See MNT_VNODE_FOREACH_ALL() in sys/mount.h.
5556  *
5557  * This interface replaces MNT_VNODE_FOREACH.
5558  */
5559 
5560 MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker");
5561 
5562 struct vnode *
5563 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp)
5564 {
5565 	struct vnode *vp;
5566 
5567 	if (should_yield())
5568 		kern_yield(PRI_USER);
5569 	MNT_ILOCK(mp);
5570 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
5571 	for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL;
5572 	    vp = TAILQ_NEXT(vp, v_nmntvnodes)) {
5573 		/* Allow a racy peek at VI_DOOMED to save a lock acquisition. */
5574 		if (vp->v_type == VMARKER || (vp->v_iflag & VI_DOOMED) != 0)
5575 			continue;
5576 		VI_LOCK(vp);
5577 		if ((vp->v_iflag & VI_DOOMED) != 0) {
5578 			VI_UNLOCK(vp);
5579 			continue;
5580 		}
5581 		break;
5582 	}
5583 	if (vp == NULL) {
5584 		__mnt_vnode_markerfree_all(mvp, mp);
5585 		/* MNT_IUNLOCK(mp); -- done in above function */
5586 		mtx_assert(MNT_MTX(mp), MA_NOTOWNED);
5587 		return (NULL);
5588 	}
5589 	TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
5590 	TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
5591 	MNT_IUNLOCK(mp);
5592 	return (vp);
5593 }
5594 
5595 struct vnode *
5596 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp)
5597 {
5598 	struct vnode *vp;
5599 
5600 	*mvp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO);
5601 	MNT_ILOCK(mp);
5602 	MNT_REF(mp);
5603 	(*mvp)->v_mount = mp;
5604 	(*mvp)->v_type = VMARKER;
5605 
5606 	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
5607 		/* Allow a racy peek at VI_DOOMED to save a lock acquisition. */
5608 		if (vp->v_type == VMARKER || (vp->v_iflag & VI_DOOMED) != 0)
5609 			continue;
5610 		VI_LOCK(vp);
5611 		if ((vp->v_iflag & VI_DOOMED) != 0) {
5612 			VI_UNLOCK(vp);
5613 			continue;
5614 		}
5615 		break;
5616 	}
5617 	if (vp == NULL) {
5618 		MNT_REL(mp);
5619 		MNT_IUNLOCK(mp);
5620 		free(*mvp, M_VNODE_MARKER);
5621 		*mvp = NULL;
5622 		return (NULL);
5623 	}
5624 	TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
5625 	MNT_IUNLOCK(mp);
5626 	return (vp);
5627 }
5628 
5629 void
5630 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp)
5631 {
5632 
5633 	if (*mvp == NULL) {
5634 		MNT_IUNLOCK(mp);
5635 		return;
5636 	}
5637 
5638 	mtx_assert(MNT_MTX(mp), MA_OWNED);
5639 
5640 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
5641 	TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
5642 	MNT_REL(mp);
5643 	MNT_IUNLOCK(mp);
5644 	free(*mvp, M_VNODE_MARKER);
5645 	*mvp = NULL;
5646 }
5647 
5648 /*
5649  * These are helper functions for filesystems to traverse their
5650  * active vnodes.  See MNT_VNODE_FOREACH_ACTIVE() in sys/mount.h
5651  */
5652 static void
5653 mnt_vnode_markerfree_active(struct vnode **mvp, struct mount *mp)
5654 {
5655 
5656 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
5657 
5658 	MNT_ILOCK(mp);
5659 	MNT_REL(mp);
5660 	MNT_IUNLOCK(mp);
5661 	free(*mvp, M_VNODE_MARKER);
5662 	*mvp = NULL;
5663 }
5664 
5665 /*
5666  * Relock the mp mount vnode list lock with the vp vnode interlock in the
5667  * conventional lock order during mnt_vnode_next_active iteration.
5668  *
5669  * On entry, the mount vnode list lock is held and the vnode interlock is not.
5670  * The list lock is dropped and reacquired.  On success, both locks are held.
5671  * On failure, the mount vnode list lock is held but the vnode interlock is
5672  * not, and the procedure may have yielded.
5673  */
5674 static bool
5675 mnt_vnode_next_active_relock(struct vnode *mvp, struct mount *mp,
5676     struct vnode *vp)
5677 {
5678 	const struct vnode *tmp;
5679 	bool held, ret;
5680 
5681 	VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER &&
5682 	    TAILQ_NEXT(mvp, v_actfreelist) != NULL, mvp,
5683 	    ("%s: bad marker", __func__));
5684 	VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp,
5685 	    ("%s: inappropriate vnode", __func__));
5686 	ASSERT_VI_UNLOCKED(vp, __func__);
5687 	mtx_assert(&mp->mnt_listmtx, MA_OWNED);
5688 
5689 	ret = false;
5690 
5691 	TAILQ_REMOVE(&mp->mnt_activevnodelist, mvp, v_actfreelist);
5692 	TAILQ_INSERT_BEFORE(vp, mvp, v_actfreelist);
5693 
5694 	/*
5695 	 * Use a hold to prevent vp from disappearing while the mount vnode
5696 	 * list lock is dropped and reacquired.  Normally a hold would be
5697 	 * acquired with vhold(), but that might try to acquire the vnode
5698 	 * interlock, which would be a LOR with the mount vnode list lock.
5699 	 */
5700 	held = refcount_acquire_if_not_zero(&vp->v_holdcnt);
5701 	mtx_unlock(&mp->mnt_listmtx);
5702 	if (!held)
5703 		goto abort;
5704 	VI_LOCK(vp);
5705 	if (!refcount_release_if_not_last(&vp->v_holdcnt)) {
5706 		vdropl(vp);
5707 		goto abort;
5708 	}
5709 	mtx_lock(&mp->mnt_listmtx);
5710 
5711 	/*
5712 	 * Determine whether the vnode is still the next one after the marker,
5713 	 * excepting any other markers.  If the vnode has not been doomed by
5714 	 * vgone() then the hold should have ensured that it remained on the
5715 	 * active list.  If it has been doomed but is still on the active list,
5716 	 * don't abort, but rather skip over it (avoid spinning on doomed
5717 	 * vnodes).
5718 	 */
5719 	tmp = mvp;
5720 	do {
5721 		tmp = TAILQ_NEXT(tmp, v_actfreelist);
5722 	} while (tmp != NULL && tmp->v_type == VMARKER);
5723 	if (tmp != vp) {
5724 		mtx_unlock(&mp->mnt_listmtx);
5725 		VI_UNLOCK(vp);
5726 		goto abort;
5727 	}
5728 
5729 	ret = true;
5730 	goto out;
5731 abort:
5732 	maybe_yield();
5733 	mtx_lock(&mp->mnt_listmtx);
5734 out:
5735 	if (ret)
5736 		ASSERT_VI_LOCKED(vp, __func__);
5737 	else
5738 		ASSERT_VI_UNLOCKED(vp, __func__);
5739 	mtx_assert(&mp->mnt_listmtx, MA_OWNED);
5740 	return (ret);
5741 }
5742 
5743 static struct vnode *
5744 mnt_vnode_next_active(struct vnode **mvp, struct mount *mp)
5745 {
5746 	struct vnode *vp, *nvp;
5747 
5748 	mtx_assert(&mp->mnt_listmtx, MA_OWNED);
5749 	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
5750 restart:
5751 	vp = TAILQ_NEXT(*mvp, v_actfreelist);
5752 	while (vp != NULL) {
5753 		if (vp->v_type == VMARKER) {
5754 			vp = TAILQ_NEXT(vp, v_actfreelist);
5755 			continue;
5756 		}
5757 		/*
5758 		 * Try-lock because this is the wrong lock order.  If that does
5759 		 * not succeed, drop the mount vnode list lock and try to
5760 		 * reacquire it and the vnode interlock in the right order.
5761 		 */
5762 		if (!VI_TRYLOCK(vp) &&
5763 		    !mnt_vnode_next_active_relock(*mvp, mp, vp))
5764 			goto restart;
5765 		KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp));
5766 		KASSERT(vp->v_mount == mp || vp->v_mount == NULL,
5767 		    ("alien vnode on the active list %p %p", vp, mp));
5768 		if (vp->v_mount == mp && (vp->v_iflag & VI_DOOMED) == 0)
5769 			break;
5770 		nvp = TAILQ_NEXT(vp, v_actfreelist);
5771 		VI_UNLOCK(vp);
5772 		vp = nvp;
5773 	}
5774 	TAILQ_REMOVE(&mp->mnt_activevnodelist, *mvp, v_actfreelist);
5775 
5776 	/* Check if we are done */
5777 	if (vp == NULL) {
5778 		mtx_unlock(&mp->mnt_listmtx);
5779 		mnt_vnode_markerfree_active(mvp, mp);
5780 		return (NULL);
5781 	}
5782 	TAILQ_INSERT_AFTER(&mp->mnt_activevnodelist, vp, *mvp, v_actfreelist);
5783 	mtx_unlock(&mp->mnt_listmtx);
5784 	ASSERT_VI_LOCKED(vp, "active iter");
5785 	KASSERT((vp->v_iflag & VI_ACTIVE) != 0, ("Non-active vp %p", vp));
5786 	return (vp);
5787 }
5788 
5789 struct vnode *
5790 __mnt_vnode_next_active(struct vnode **mvp, struct mount *mp)
5791 {
5792 
5793 	if (should_yield())
5794 		kern_yield(PRI_USER);
5795 	mtx_lock(&mp->mnt_listmtx);
5796 	return (mnt_vnode_next_active(mvp, mp));
5797 }
5798 
5799 struct vnode *
5800 __mnt_vnode_first_active(struct vnode **mvp, struct mount *mp)
5801 {
5802 	struct vnode *vp;
5803 
5804 	*mvp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO);
5805 	MNT_ILOCK(mp);
5806 	MNT_REF(mp);
5807 	MNT_IUNLOCK(mp);
5808 	(*mvp)->v_type = VMARKER;
5809 	(*mvp)->v_mount = mp;
5810 
5811 	mtx_lock(&mp->mnt_listmtx);
5812 	vp = TAILQ_FIRST(&mp->mnt_activevnodelist);
5813 	if (vp == NULL) {
5814 		mtx_unlock(&mp->mnt_listmtx);
5815 		mnt_vnode_markerfree_active(mvp, mp);
5816 		return (NULL);
5817 	}
5818 	TAILQ_INSERT_BEFORE(vp, *mvp, v_actfreelist);
5819 	return (mnt_vnode_next_active(mvp, mp));
5820 }
5821 
5822 void
5823 __mnt_vnode_markerfree_active(struct vnode **mvp, struct mount *mp)
5824 {
5825 
5826 	if (*mvp == NULL)
5827 		return;
5828 
5829 	mtx_lock(&mp->mnt_listmtx);
5830 	TAILQ_REMOVE(&mp->mnt_activevnodelist, *mvp, v_actfreelist);
5831 	mtx_unlock(&mp->mnt_listmtx);
5832 	mnt_vnode_markerfree_active(mvp, mp);
5833 }
5834