xref: /freebsd/sys/kern/vfs_subr.c (revision 6472ac3d8a86336899b6cfb789a4cd9897e3fab5)
1 /*-
2  * Copyright (c) 1989, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * (c) UNIX System Laboratories, Inc.
5  * All or some portions of this file are derived from material licensed
6  * to the University of California by American Telephone and Telegraph
7  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8  * the permission of UNIX System Laboratories, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	@(#)vfs_subr.c	8.31 (Berkeley) 5/26/95
35  */
36 
37 /*
38  * External virtual filesystem routines
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43 
44 #include "opt_ddb.h"
45 #include "opt_watchdog.h"
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/bio.h>
50 #include <sys/buf.h>
51 #include <sys/condvar.h>
52 #include <sys/conf.h>
53 #include <sys/dirent.h>
54 #include <sys/event.h>
55 #include <sys/eventhandler.h>
56 #include <sys/extattr.h>
57 #include <sys/file.h>
58 #include <sys/fcntl.h>
59 #include <sys/jail.h>
60 #include <sys/kdb.h>
61 #include <sys/kernel.h>
62 #include <sys/kthread.h>
63 #include <sys/lockf.h>
64 #include <sys/malloc.h>
65 #include <sys/mount.h>
66 #include <sys/namei.h>
67 #include <sys/priv.h>
68 #include <sys/reboot.h>
69 #include <sys/sched.h>
70 #include <sys/sleepqueue.h>
71 #include <sys/stat.h>
72 #include <sys/sysctl.h>
73 #include <sys/syslog.h>
74 #include <sys/vmmeter.h>
75 #include <sys/vnode.h>
76 #ifdef SW_WATCHDOG
77 #include <sys/watchdog.h>
78 #endif
79 
80 #include <machine/stdarg.h>
81 
82 #include <security/mac/mac_framework.h>
83 
84 #include <vm/vm.h>
85 #include <vm/vm_object.h>
86 #include <vm/vm_extern.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_page.h>
90 #include <vm/vm_kern.h>
91 #include <vm/uma.h>
92 
93 #ifdef DDB
94 #include <ddb/ddb.h>
95 #endif
96 
97 #define	WI_MPSAFEQ	0
98 #define	WI_GIANTQ	1
99 
100 static void	delmntque(struct vnode *vp);
101 static int	flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo,
102 		    int slpflag, int slptimeo);
103 static void	syncer_shutdown(void *arg, int howto);
104 static int	vtryrecycle(struct vnode *vp);
105 static void	vbusy(struct vnode *vp);
106 static void	vinactive(struct vnode *, struct thread *);
107 static void	v_incr_usecount(struct vnode *);
108 static void	v_decr_usecount(struct vnode *);
109 static void	v_decr_useonly(struct vnode *);
110 static void	v_upgrade_usecount(struct vnode *);
111 static void	vfree(struct vnode *);
112 static void	vnlru_free(int);
113 static void	vgonel(struct vnode *);
114 static void	vfs_knllock(void *arg);
115 static void	vfs_knlunlock(void *arg);
116 static void	vfs_knl_assert_locked(void *arg);
117 static void	vfs_knl_assert_unlocked(void *arg);
118 static void	destroy_vpollinfo(struct vpollinfo *vi);
119 
120 /*
121  * Number of vnodes in existence.  Increased whenever getnewvnode()
122  * allocates a new vnode, decreased on vdestroy() called on VI_DOOMed
123  * vnode.
124  */
125 static unsigned long	numvnodes;
126 
127 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0,
128     "Number of vnodes in existence");
129 
130 /*
131  * Conversion tables for conversion from vnode types to inode formats
132  * and back.
133  */
134 enum vtype iftovt_tab[16] = {
135 	VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
136 	VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VBAD,
137 };
138 int vttoif_tab[10] = {
139 	0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK,
140 	S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT
141 };
142 
143 /*
144  * List of vnodes that are ready for recycling.
145  */
146 static TAILQ_HEAD(freelst, vnode) vnode_free_list;
147 
148 /*
149  * Free vnode target.  Free vnodes may simply be files which have been stat'd
150  * but not read.  This is somewhat common, and a small cache of such files
151  * should be kept to avoid recreation costs.
152  */
153 static u_long wantfreevnodes;
154 SYSCTL_ULONG(_vfs, OID_AUTO, wantfreevnodes, CTLFLAG_RW, &wantfreevnodes, 0, "");
155 /* Number of vnodes in the free list. */
156 static u_long freevnodes;
157 SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD, &freevnodes, 0,
158     "Number of vnodes in the free list");
159 
160 static int vlru_allow_cache_src;
161 SYSCTL_INT(_vfs, OID_AUTO, vlru_allow_cache_src, CTLFLAG_RW,
162     &vlru_allow_cache_src, 0, "Allow vlru to reclaim source vnode");
163 
164 /*
165  * Various variables used for debugging the new implementation of
166  * reassignbuf().
167  * XXX these are probably of (very) limited utility now.
168  */
169 static int reassignbufcalls;
170 SYSCTL_INT(_vfs, OID_AUTO, reassignbufcalls, CTLFLAG_RW, &reassignbufcalls, 0,
171     "Number of calls to reassignbuf");
172 
173 /*
174  * Cache for the mount type id assigned to NFS.  This is used for
175  * special checks in nfs/nfs_nqlease.c and vm/vnode_pager.c.
176  */
177 int	nfs_mount_type = -1;
178 
179 /* To keep more than one thread at a time from running vfs_getnewfsid */
180 static struct mtx mntid_mtx;
181 
182 /*
183  * Lock for any access to the following:
184  *	vnode_free_list
185  *	numvnodes
186  *	freevnodes
187  */
188 static struct mtx vnode_free_list_mtx;
189 
190 /* Publicly exported FS */
191 struct nfs_public nfs_pub;
192 
193 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */
194 static uma_zone_t vnode_zone;
195 static uma_zone_t vnodepoll_zone;
196 
197 /*
198  * The workitem queue.
199  *
200  * It is useful to delay writes of file data and filesystem metadata
201  * for tens of seconds so that quickly created and deleted files need
202  * not waste disk bandwidth being created and removed. To realize this,
203  * we append vnodes to a "workitem" queue. When running with a soft
204  * updates implementation, most pending metadata dependencies should
205  * not wait for more than a few seconds. Thus, mounted on block devices
206  * are delayed only about a half the time that file data is delayed.
207  * Similarly, directory updates are more critical, so are only delayed
208  * about a third the time that file data is delayed. Thus, there are
209  * SYNCER_MAXDELAY queues that are processed round-robin at a rate of
210  * one each second (driven off the filesystem syncer process). The
211  * syncer_delayno variable indicates the next queue that is to be processed.
212  * Items that need to be processed soon are placed in this queue:
213  *
214  *	syncer_workitem_pending[syncer_delayno]
215  *
216  * A delay of fifteen seconds is done by placing the request fifteen
217  * entries later in the queue:
218  *
219  *	syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask]
220  *
221  */
222 static int syncer_delayno;
223 static long syncer_mask;
224 LIST_HEAD(synclist, bufobj);
225 static struct synclist *syncer_workitem_pending[2];
226 /*
227  * The sync_mtx protects:
228  *	bo->bo_synclist
229  *	sync_vnode_count
230  *	syncer_delayno
231  *	syncer_state
232  *	syncer_workitem_pending
233  *	syncer_worklist_len
234  *	rushjob
235  */
236 static struct mtx sync_mtx;
237 static struct cv sync_wakeup;
238 
239 #define SYNCER_MAXDELAY		32
240 static int syncer_maxdelay = SYNCER_MAXDELAY;	/* maximum delay time */
241 static int syncdelay = 30;		/* max time to delay syncing data */
242 static int filedelay = 30;		/* time to delay syncing files */
243 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0,
244     "Time to delay syncing files (in seconds)");
245 static int dirdelay = 29;		/* time to delay syncing directories */
246 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0,
247     "Time to delay syncing directories (in seconds)");
248 static int metadelay = 28;		/* time to delay syncing metadata */
249 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0,
250     "Time to delay syncing metadata (in seconds)");
251 static int rushjob;		/* number of slots to run ASAP */
252 static int stat_rush_requests;	/* number of times I/O speeded up */
253 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0,
254     "Number of times I/O speeded up (rush requests)");
255 
256 /*
257  * When shutting down the syncer, run it at four times normal speed.
258  */
259 #define SYNCER_SHUTDOWN_SPEEDUP		4
260 static int sync_vnode_count;
261 static int syncer_worklist_len;
262 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY }
263     syncer_state;
264 
265 /*
266  * Number of vnodes we want to exist at any one time.  This is mostly used
267  * to size hash tables in vnode-related code.  It is normally not used in
268  * getnewvnode(), as wantfreevnodes is normally nonzero.)
269  *
270  * XXX desiredvnodes is historical cruft and should not exist.
271  */
272 int desiredvnodes;
273 SYSCTL_INT(_kern, KERN_MAXVNODES, maxvnodes, CTLFLAG_RW,
274     &desiredvnodes, 0, "Maximum number of vnodes");
275 SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW,
276     &wantfreevnodes, 0, "Minimum number of vnodes (legacy)");
277 static int vnlru_nowhere;
278 SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW,
279     &vnlru_nowhere, 0, "Number of times the vnlru process ran without success");
280 
281 /*
282  * Macros to control when a vnode is freed and recycled.  All require
283  * the vnode interlock.
284  */
285 #define VCANRECYCLE(vp) (((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt)
286 #define VSHOULDFREE(vp) (!((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt)
287 #define VSHOULDBUSY(vp) (((vp)->v_iflag & VI_FREE) && (vp)->v_holdcnt)
288 
289 
290 /*
291  * Initialize the vnode management data structures.
292  *
293  * Reevaluate the following cap on the number of vnodes after the physical
294  * memory size exceeds 512GB.  In the limit, as the physical memory size
295  * grows, the ratio of physical pages to vnodes approaches sixteen to one.
296  */
297 #ifndef	MAXVNODES_MAX
298 #define	MAXVNODES_MAX	(512 * (1024 * 1024 * 1024 / (int)PAGE_SIZE / 16))
299 #endif
300 static void
301 vntblinit(void *dummy __unused)
302 {
303 	int physvnodes, virtvnodes;
304 
305 	/*
306 	 * Desiredvnodes is a function of the physical memory size and the
307 	 * kernel's heap size.  Generally speaking, it scales with the
308 	 * physical memory size.  The ratio of desiredvnodes to physical pages
309 	 * is one to four until desiredvnodes exceeds 98,304.  Thereafter, the
310 	 * marginal ratio of desiredvnodes to physical pages is one to
311 	 * sixteen.  However, desiredvnodes is limited by the kernel's heap
312 	 * size.  The memory required by desiredvnodes vnodes and vm objects
313 	 * may not exceed one seventh of the kernel's heap size.
314 	 */
315 	physvnodes = maxproc + cnt.v_page_count / 16 + 3 * min(98304 * 4,
316 	    cnt.v_page_count) / 16;
317 	virtvnodes = vm_kmem_size / (7 * (sizeof(struct vm_object) +
318 	    sizeof(struct vnode)));
319 	desiredvnodes = min(physvnodes, virtvnodes);
320 	if (desiredvnodes > MAXVNODES_MAX) {
321 		if (bootverbose)
322 			printf("Reducing kern.maxvnodes %d -> %d\n",
323 			    desiredvnodes, MAXVNODES_MAX);
324 		desiredvnodes = MAXVNODES_MAX;
325 	}
326 	wantfreevnodes = desiredvnodes / 4;
327 	mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF);
328 	TAILQ_INIT(&vnode_free_list);
329 	mtx_init(&vnode_free_list_mtx, "vnode_free_list", NULL, MTX_DEF);
330 	vnode_zone = uma_zcreate("VNODE", sizeof (struct vnode), NULL, NULL,
331 	    NULL, NULL, UMA_ALIGN_PTR, 0);
332 	vnodepoll_zone = uma_zcreate("VNODEPOLL", sizeof (struct vpollinfo),
333 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
334 	/*
335 	 * Initialize the filesystem syncer.
336 	 */
337 	syncer_workitem_pending[WI_MPSAFEQ] = hashinit(syncer_maxdelay, M_VNODE,
338 	    &syncer_mask);
339 	syncer_workitem_pending[WI_GIANTQ] = hashinit(syncer_maxdelay, M_VNODE,
340 	    &syncer_mask);
341 	syncer_maxdelay = syncer_mask + 1;
342 	mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF);
343 	cv_init(&sync_wakeup, "syncer");
344 }
345 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);
346 
347 
348 /*
349  * Mark a mount point as busy. Used to synchronize access and to delay
350  * unmounting. Eventually, mountlist_mtx is not released on failure.
351  *
352  * vfs_busy() is a custom lock, it can block the caller.
353  * vfs_busy() only sleeps if the unmount is active on the mount point.
354  * For a mountpoint mp, vfs_busy-enforced lock is before lock of any
355  * vnode belonging to mp.
356  *
357  * Lookup uses vfs_busy() to traverse mount points.
358  * root fs			var fs
359  * / vnode lock		A	/ vnode lock (/var)		D
360  * /var vnode lock	B	/log vnode lock(/var/log)	E
361  * vfs_busy lock	C	vfs_busy lock			F
362  *
363  * Within each file system, the lock order is C->A->B and F->D->E.
364  *
365  * When traversing across mounts, the system follows that lock order:
366  *
367  *        C->A->B
368  *              |
369  *              +->F->D->E
370  *
371  * The lookup() process for namei("/var") illustrates the process:
372  *  VOP_LOOKUP() obtains B while A is held
373  *  vfs_busy() obtains a shared lock on F while A and B are held
374  *  vput() releases lock on B
375  *  vput() releases lock on A
376  *  VFS_ROOT() obtains lock on D while shared lock on F is held
377  *  vfs_unbusy() releases shared lock on F
378  *  vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A.
379  *    Attempt to lock A (instead of vp_crossmp) while D is held would
380  *    violate the global order, causing deadlocks.
381  *
382  * dounmount() locks B while F is drained.
383  */
384 int
385 vfs_busy(struct mount *mp, int flags)
386 {
387 
388 	MPASS((flags & ~MBF_MASK) == 0);
389 	CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags);
390 
391 	MNT_ILOCK(mp);
392 	MNT_REF(mp);
393 	/*
394 	 * If mount point is currenly being unmounted, sleep until the
395 	 * mount point fate is decided.  If thread doing the unmounting fails,
396 	 * it will clear MNTK_UNMOUNT flag before waking us up, indicating
397 	 * that this mount point has survived the unmount attempt and vfs_busy
398 	 * should retry.  Otherwise the unmounter thread will set MNTK_REFEXPIRE
399 	 * flag in addition to MNTK_UNMOUNT, indicating that mount point is
400 	 * about to be really destroyed.  vfs_busy needs to release its
401 	 * reference on the mount point in this case and return with ENOENT,
402 	 * telling the caller that mount mount it tried to busy is no longer
403 	 * valid.
404 	 */
405 	while (mp->mnt_kern_flag & MNTK_UNMOUNT) {
406 		if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) {
407 			MNT_REL(mp);
408 			MNT_IUNLOCK(mp);
409 			CTR1(KTR_VFS, "%s: failed busying before sleeping",
410 			    __func__);
411 			return (ENOENT);
412 		}
413 		if (flags & MBF_MNTLSTLOCK)
414 			mtx_unlock(&mountlist_mtx);
415 		mp->mnt_kern_flag |= MNTK_MWAIT;
416 		msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0);
417 		if (flags & MBF_MNTLSTLOCK)
418 			mtx_lock(&mountlist_mtx);
419 		MNT_ILOCK(mp);
420 	}
421 	if (flags & MBF_MNTLSTLOCK)
422 		mtx_unlock(&mountlist_mtx);
423 	mp->mnt_lockref++;
424 	MNT_IUNLOCK(mp);
425 	return (0);
426 }
427 
428 /*
429  * Free a busy filesystem.
430  */
431 void
432 vfs_unbusy(struct mount *mp)
433 {
434 
435 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
436 	MNT_ILOCK(mp);
437 	MNT_REL(mp);
438 	KASSERT(mp->mnt_lockref > 0, ("negative mnt_lockref"));
439 	mp->mnt_lockref--;
440 	if (mp->mnt_lockref == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) {
441 		MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT);
442 		CTR1(KTR_VFS, "%s: waking up waiters", __func__);
443 		mp->mnt_kern_flag &= ~MNTK_DRAINING;
444 		wakeup(&mp->mnt_lockref);
445 	}
446 	MNT_IUNLOCK(mp);
447 }
448 
449 /*
450  * Lookup a mount point by filesystem identifier.
451  */
452 struct mount *
453 vfs_getvfs(fsid_t *fsid)
454 {
455 	struct mount *mp;
456 
457 	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
458 	mtx_lock(&mountlist_mtx);
459 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
460 		if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
461 		    mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
462 			vfs_ref(mp);
463 			mtx_unlock(&mountlist_mtx);
464 			return (mp);
465 		}
466 	}
467 	mtx_unlock(&mountlist_mtx);
468 	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
469 	return ((struct mount *) 0);
470 }
471 
472 /*
473  * Lookup a mount point by filesystem identifier, busying it before
474  * returning.
475  */
476 struct mount *
477 vfs_busyfs(fsid_t *fsid)
478 {
479 	struct mount *mp;
480 	int error;
481 
482 	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
483 	mtx_lock(&mountlist_mtx);
484 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
485 		if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
486 		    mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
487 			error = vfs_busy(mp, MBF_MNTLSTLOCK);
488 			if (error) {
489 				mtx_unlock(&mountlist_mtx);
490 				return (NULL);
491 			}
492 			return (mp);
493 		}
494 	}
495 	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
496 	mtx_unlock(&mountlist_mtx);
497 	return ((struct mount *) 0);
498 }
499 
500 /*
501  * Check if a user can access privileged mount options.
502  */
503 int
504 vfs_suser(struct mount *mp, struct thread *td)
505 {
506 	int error;
507 
508 	/*
509 	 * If the thread is jailed, but this is not a jail-friendly file
510 	 * system, deny immediately.
511 	 */
512 	if (!(mp->mnt_vfc->vfc_flags & VFCF_JAIL) && jailed(td->td_ucred))
513 		return (EPERM);
514 
515 	/*
516 	 * If the file system was mounted outside the jail of the calling
517 	 * thread, deny immediately.
518 	 */
519 	if (prison_check(td->td_ucred, mp->mnt_cred) != 0)
520 		return (EPERM);
521 
522 	/*
523 	 * If file system supports delegated administration, we don't check
524 	 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified
525 	 * by the file system itself.
526 	 * If this is not the user that did original mount, we check for
527 	 * the PRIV_VFS_MOUNT_OWNER privilege.
528 	 */
529 	if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) &&
530 	    mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) {
531 		if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0)
532 			return (error);
533 	}
534 	return (0);
535 }
536 
537 /*
538  * Get a new unique fsid.  Try to make its val[0] unique, since this value
539  * will be used to create fake device numbers for stat().  Also try (but
540  * not so hard) make its val[0] unique mod 2^16, since some emulators only
541  * support 16-bit device numbers.  We end up with unique val[0]'s for the
542  * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls.
543  *
544  * Keep in mind that several mounts may be running in parallel.  Starting
545  * the search one past where the previous search terminated is both a
546  * micro-optimization and a defense against returning the same fsid to
547  * different mounts.
548  */
549 void
550 vfs_getnewfsid(struct mount *mp)
551 {
552 	static uint16_t mntid_base;
553 	struct mount *nmp;
554 	fsid_t tfsid;
555 	int mtype;
556 
557 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
558 	mtx_lock(&mntid_mtx);
559 	mtype = mp->mnt_vfc->vfc_typenum;
560 	tfsid.val[1] = mtype;
561 	mtype = (mtype & 0xFF) << 24;
562 	for (;;) {
563 		tfsid.val[0] = makedev(255,
564 		    mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF));
565 		mntid_base++;
566 		if ((nmp = vfs_getvfs(&tfsid)) == NULL)
567 			break;
568 		vfs_rel(nmp);
569 	}
570 	mp->mnt_stat.f_fsid.val[0] = tfsid.val[0];
571 	mp->mnt_stat.f_fsid.val[1] = tfsid.val[1];
572 	mtx_unlock(&mntid_mtx);
573 }
574 
575 /*
576  * Knob to control the precision of file timestamps:
577  *
578  *   0 = seconds only; nanoseconds zeroed.
579  *   1 = seconds and nanoseconds, accurate within 1/HZ.
580  *   2 = seconds and nanoseconds, truncated to microseconds.
581  * >=3 = seconds and nanoseconds, maximum precision.
582  */
583 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC };
584 
585 static int timestamp_precision = TSP_SEC;
586 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW,
587     &timestamp_precision, 0, "File timestamp precision (0: seconds, "
588     "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to ms, "
589     "3+: sec + ns (max. precision))");
590 
591 /*
592  * Get a current timestamp.
593  */
594 void
595 vfs_timestamp(struct timespec *tsp)
596 {
597 	struct timeval tv;
598 
599 	switch (timestamp_precision) {
600 	case TSP_SEC:
601 		tsp->tv_sec = time_second;
602 		tsp->tv_nsec = 0;
603 		break;
604 	case TSP_HZ:
605 		getnanotime(tsp);
606 		break;
607 	case TSP_USEC:
608 		microtime(&tv);
609 		TIMEVAL_TO_TIMESPEC(&tv, tsp);
610 		break;
611 	case TSP_NSEC:
612 	default:
613 		nanotime(tsp);
614 		break;
615 	}
616 }
617 
618 /*
619  * Set vnode attributes to VNOVAL
620  */
621 void
622 vattr_null(struct vattr *vap)
623 {
624 
625 	vap->va_type = VNON;
626 	vap->va_size = VNOVAL;
627 	vap->va_bytes = VNOVAL;
628 	vap->va_mode = VNOVAL;
629 	vap->va_nlink = VNOVAL;
630 	vap->va_uid = VNOVAL;
631 	vap->va_gid = VNOVAL;
632 	vap->va_fsid = VNOVAL;
633 	vap->va_fileid = VNOVAL;
634 	vap->va_blocksize = VNOVAL;
635 	vap->va_rdev = VNOVAL;
636 	vap->va_atime.tv_sec = VNOVAL;
637 	vap->va_atime.tv_nsec = VNOVAL;
638 	vap->va_mtime.tv_sec = VNOVAL;
639 	vap->va_mtime.tv_nsec = VNOVAL;
640 	vap->va_ctime.tv_sec = VNOVAL;
641 	vap->va_ctime.tv_nsec = VNOVAL;
642 	vap->va_birthtime.tv_sec = VNOVAL;
643 	vap->va_birthtime.tv_nsec = VNOVAL;
644 	vap->va_flags = VNOVAL;
645 	vap->va_gen = VNOVAL;
646 	vap->va_vaflags = 0;
647 }
648 
649 /*
650  * This routine is called when we have too many vnodes.  It attempts
651  * to free <count> vnodes and will potentially free vnodes that still
652  * have VM backing store (VM backing store is typically the cause
653  * of a vnode blowout so we want to do this).  Therefore, this operation
654  * is not considered cheap.
655  *
656  * A number of conditions may prevent a vnode from being reclaimed.
657  * the buffer cache may have references on the vnode, a directory
658  * vnode may still have references due to the namei cache representing
659  * underlying files, or the vnode may be in active use.   It is not
660  * desireable to reuse such vnodes.  These conditions may cause the
661  * number of vnodes to reach some minimum value regardless of what
662  * you set kern.maxvnodes to.  Do not set kern.maxvnodes too low.
663  */
664 static int
665 vlrureclaim(struct mount *mp)
666 {
667 	struct vnode *vp;
668 	int done;
669 	int trigger;
670 	int usevnodes;
671 	int count;
672 
673 	/*
674 	 * Calculate the trigger point, don't allow user
675 	 * screwups to blow us up.   This prevents us from
676 	 * recycling vnodes with lots of resident pages.  We
677 	 * aren't trying to free memory, we are trying to
678 	 * free vnodes.
679 	 */
680 	usevnodes = desiredvnodes;
681 	if (usevnodes <= 0)
682 		usevnodes = 1;
683 	trigger = cnt.v_page_count * 2 / usevnodes;
684 	done = 0;
685 	vn_start_write(NULL, &mp, V_WAIT);
686 	MNT_ILOCK(mp);
687 	count = mp->mnt_nvnodelistsize / 10 + 1;
688 	while (count != 0) {
689 		vp = TAILQ_FIRST(&mp->mnt_nvnodelist);
690 		while (vp != NULL && vp->v_type == VMARKER)
691 			vp = TAILQ_NEXT(vp, v_nmntvnodes);
692 		if (vp == NULL)
693 			break;
694 		TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
695 		TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
696 		--count;
697 		if (!VI_TRYLOCK(vp))
698 			goto next_iter;
699 		/*
700 		 * If it's been deconstructed already, it's still
701 		 * referenced, or it exceeds the trigger, skip it.
702 		 */
703 		if (vp->v_usecount ||
704 		    (!vlru_allow_cache_src &&
705 			!LIST_EMPTY(&(vp)->v_cache_src)) ||
706 		    (vp->v_iflag & VI_DOOMED) != 0 || (vp->v_object != NULL &&
707 		    vp->v_object->resident_page_count > trigger)) {
708 			VI_UNLOCK(vp);
709 			goto next_iter;
710 		}
711 		MNT_IUNLOCK(mp);
712 		vholdl(vp);
713 		if (VOP_LOCK(vp, LK_INTERLOCK|LK_EXCLUSIVE|LK_NOWAIT)) {
714 			vdrop(vp);
715 			goto next_iter_mntunlocked;
716 		}
717 		VI_LOCK(vp);
718 		/*
719 		 * v_usecount may have been bumped after VOP_LOCK() dropped
720 		 * the vnode interlock and before it was locked again.
721 		 *
722 		 * It is not necessary to recheck VI_DOOMED because it can
723 		 * only be set by another thread that holds both the vnode
724 		 * lock and vnode interlock.  If another thread has the
725 		 * vnode lock before we get to VOP_LOCK() and obtains the
726 		 * vnode interlock after VOP_LOCK() drops the vnode
727 		 * interlock, the other thread will be unable to drop the
728 		 * vnode lock before our VOP_LOCK() call fails.
729 		 */
730 		if (vp->v_usecount ||
731 		    (!vlru_allow_cache_src &&
732 			!LIST_EMPTY(&(vp)->v_cache_src)) ||
733 		    (vp->v_object != NULL &&
734 		    vp->v_object->resident_page_count > trigger)) {
735 			VOP_UNLOCK(vp, LK_INTERLOCK);
736 			goto next_iter_mntunlocked;
737 		}
738 		KASSERT((vp->v_iflag & VI_DOOMED) == 0,
739 		    ("VI_DOOMED unexpectedly detected in vlrureclaim()"));
740 		vgonel(vp);
741 		VOP_UNLOCK(vp, 0);
742 		vdropl(vp);
743 		done++;
744 next_iter_mntunlocked:
745 		if (!should_yield())
746 			goto relock_mnt;
747 		goto yield;
748 next_iter:
749 		if (!should_yield())
750 			continue;
751 		MNT_IUNLOCK(mp);
752 yield:
753 		kern_yield(PRI_UNCHANGED);
754 relock_mnt:
755 		MNT_ILOCK(mp);
756 	}
757 	MNT_IUNLOCK(mp);
758 	vn_finished_write(mp);
759 	return done;
760 }
761 
762 /*
763  * Attempt to keep the free list at wantfreevnodes length.
764  */
765 static void
766 vnlru_free(int count)
767 {
768 	struct vnode *vp;
769 	int vfslocked;
770 
771 	mtx_assert(&vnode_free_list_mtx, MA_OWNED);
772 	for (; count > 0; count--) {
773 		vp = TAILQ_FIRST(&vnode_free_list);
774 		/*
775 		 * The list can be modified while the free_list_mtx
776 		 * has been dropped and vp could be NULL here.
777 		 */
778 		if (!vp)
779 			break;
780 		VNASSERT(vp->v_op != NULL, vp,
781 		    ("vnlru_free: vnode already reclaimed."));
782 		TAILQ_REMOVE(&vnode_free_list, vp, v_freelist);
783 		/*
784 		 * Don't recycle if we can't get the interlock.
785 		 */
786 		if (!VI_TRYLOCK(vp)) {
787 			TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
788 			continue;
789 		}
790 		VNASSERT(VCANRECYCLE(vp), vp,
791 		    ("vp inconsistent on freelist"));
792 		freevnodes--;
793 		vp->v_iflag &= ~VI_FREE;
794 		vholdl(vp);
795 		mtx_unlock(&vnode_free_list_mtx);
796 		VI_UNLOCK(vp);
797 		vfslocked = VFS_LOCK_GIANT(vp->v_mount);
798 		vtryrecycle(vp);
799 		VFS_UNLOCK_GIANT(vfslocked);
800 		/*
801 		 * If the recycled succeeded this vdrop will actually free
802 		 * the vnode.  If not it will simply place it back on
803 		 * the free list.
804 		 */
805 		vdrop(vp);
806 		mtx_lock(&vnode_free_list_mtx);
807 	}
808 }
809 /*
810  * Attempt to recycle vnodes in a context that is always safe to block.
811  * Calling vlrurecycle() from the bowels of filesystem code has some
812  * interesting deadlock problems.
813  */
814 static struct proc *vnlruproc;
815 static int vnlruproc_sig;
816 
817 static void
818 vnlru_proc(void)
819 {
820 	struct mount *mp, *nmp;
821 	int done, vfslocked;
822 	struct proc *p = vnlruproc;
823 
824 	EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, p,
825 	    SHUTDOWN_PRI_FIRST);
826 
827 	for (;;) {
828 		kproc_suspend_check(p);
829 		mtx_lock(&vnode_free_list_mtx);
830 		if (freevnodes > wantfreevnodes)
831 			vnlru_free(freevnodes - wantfreevnodes);
832 		if (numvnodes <= desiredvnodes * 9 / 10) {
833 			vnlruproc_sig = 0;
834 			wakeup(&vnlruproc_sig);
835 			msleep(vnlruproc, &vnode_free_list_mtx,
836 			    PVFS|PDROP, "vlruwt", hz);
837 			continue;
838 		}
839 		mtx_unlock(&vnode_free_list_mtx);
840 		done = 0;
841 		mtx_lock(&mountlist_mtx);
842 		for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
843 			if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) {
844 				nmp = TAILQ_NEXT(mp, mnt_list);
845 				continue;
846 			}
847 			vfslocked = VFS_LOCK_GIANT(mp);
848 			done += vlrureclaim(mp);
849 			VFS_UNLOCK_GIANT(vfslocked);
850 			mtx_lock(&mountlist_mtx);
851 			nmp = TAILQ_NEXT(mp, mnt_list);
852 			vfs_unbusy(mp);
853 		}
854 		mtx_unlock(&mountlist_mtx);
855 		if (done == 0) {
856 #if 0
857 			/* These messages are temporary debugging aids */
858 			if (vnlru_nowhere < 5)
859 				printf("vnlru process getting nowhere..\n");
860 			else if (vnlru_nowhere == 5)
861 				printf("vnlru process messages stopped.\n");
862 #endif
863 			vnlru_nowhere++;
864 			tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3);
865 		} else
866 			kern_yield(PRI_UNCHANGED);
867 	}
868 }
869 
870 static struct kproc_desc vnlru_kp = {
871 	"vnlru",
872 	vnlru_proc,
873 	&vnlruproc
874 };
875 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start,
876     &vnlru_kp);
877 
878 /*
879  * Routines having to do with the management of the vnode table.
880  */
881 
882 void
883 vdestroy(struct vnode *vp)
884 {
885 	struct bufobj *bo;
886 
887 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
888 	mtx_lock(&vnode_free_list_mtx);
889 	numvnodes--;
890 	mtx_unlock(&vnode_free_list_mtx);
891 	bo = &vp->v_bufobj;
892 	VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
893 	    ("cleaned vnode still on the free list."));
894 	VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't"));
895 	VNASSERT(vp->v_holdcnt == 0, vp, ("Non-zero hold count"));
896 	VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count"));
897 	VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count"));
898 	VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's"));
899 	VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0"));
900 	VNASSERT(bo->bo_clean.bv_root == NULL, vp, ("cleanblkroot not NULL"));
901 	VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0"));
902 	VNASSERT(bo->bo_dirty.bv_root == NULL, vp, ("dirtyblkroot not NULL"));
903 	VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst"));
904 	VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src"));
905 	VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for .."));
906 	VI_UNLOCK(vp);
907 #ifdef MAC
908 	mac_vnode_destroy(vp);
909 #endif
910 	if (vp->v_pollinfo != NULL)
911 		destroy_vpollinfo(vp->v_pollinfo);
912 #ifdef INVARIANTS
913 	/* XXX Elsewhere we can detect an already freed vnode via NULL v_op. */
914 	vp->v_op = NULL;
915 #endif
916 	lockdestroy(vp->v_vnlock);
917 	mtx_destroy(&vp->v_interlock);
918 	mtx_destroy(BO_MTX(bo));
919 	uma_zfree(vnode_zone, vp);
920 }
921 
922 /*
923  * Try to recycle a freed vnode.  We abort if anyone picks up a reference
924  * before we actually vgone().  This function must be called with the vnode
925  * held to prevent the vnode from being returned to the free list midway
926  * through vgone().
927  */
928 static int
929 vtryrecycle(struct vnode *vp)
930 {
931 	struct mount *vnmp;
932 
933 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
934 	VNASSERT(vp->v_holdcnt, vp,
935 	    ("vtryrecycle: Recycling vp %p without a reference.", vp));
936 	/*
937 	 * This vnode may found and locked via some other list, if so we
938 	 * can't recycle it yet.
939 	 */
940 	if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) {
941 		CTR2(KTR_VFS,
942 		    "%s: impossible to recycle, vp %p lock is already held",
943 		    __func__, vp);
944 		return (EWOULDBLOCK);
945 	}
946 	/*
947 	 * Don't recycle if its filesystem is being suspended.
948 	 */
949 	if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) {
950 		VOP_UNLOCK(vp, 0);
951 		CTR2(KTR_VFS,
952 		    "%s: impossible to recycle, cannot start the write for %p",
953 		    __func__, vp);
954 		return (EBUSY);
955 	}
956 	/*
957 	 * If we got this far, we need to acquire the interlock and see if
958 	 * anyone picked up this vnode from another list.  If not, we will
959 	 * mark it with DOOMED via vgonel() so that anyone who does find it
960 	 * will skip over it.
961 	 */
962 	VI_LOCK(vp);
963 	if (vp->v_usecount) {
964 		VOP_UNLOCK(vp, LK_INTERLOCK);
965 		vn_finished_write(vnmp);
966 		CTR2(KTR_VFS,
967 		    "%s: impossible to recycle, %p is already referenced",
968 		    __func__, vp);
969 		return (EBUSY);
970 	}
971 	if ((vp->v_iflag & VI_DOOMED) == 0)
972 		vgonel(vp);
973 	VOP_UNLOCK(vp, LK_INTERLOCK);
974 	vn_finished_write(vnmp);
975 	return (0);
976 }
977 
978 /*
979  * Return the next vnode from the free list.
980  */
981 int
982 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops,
983     struct vnode **vpp)
984 {
985 	struct vnode *vp = NULL;
986 	struct bufobj *bo;
987 
988 	CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag);
989 	mtx_lock(&vnode_free_list_mtx);
990 	/*
991 	 * Lend our context to reclaim vnodes if they've exceeded the max.
992 	 */
993 	if (freevnodes > wantfreevnodes)
994 		vnlru_free(1);
995 	/*
996 	 * Wait for available vnodes.
997 	 */
998 	if (numvnodes > desiredvnodes) {
999 		if (mp != NULL && (mp->mnt_kern_flag & MNTK_SUSPEND)) {
1000 			/*
1001 			 * File system is beeing suspended, we cannot risk a
1002 			 * deadlock here, so allocate new vnode anyway.
1003 			 */
1004 			if (freevnodes > wantfreevnodes)
1005 				vnlru_free(freevnodes - wantfreevnodes);
1006 			goto alloc;
1007 		}
1008 		if (vnlruproc_sig == 0) {
1009 			vnlruproc_sig = 1;	/* avoid unnecessary wakeups */
1010 			wakeup(vnlruproc);
1011 		}
1012 		msleep(&vnlruproc_sig, &vnode_free_list_mtx, PVFS,
1013 		    "vlruwk", hz);
1014 #if 0	/* XXX Not all VFS_VGET/ffs_vget callers check returns. */
1015 		if (numvnodes > desiredvnodes) {
1016 			mtx_unlock(&vnode_free_list_mtx);
1017 			return (ENFILE);
1018 		}
1019 #endif
1020 	}
1021 alloc:
1022 	numvnodes++;
1023 	mtx_unlock(&vnode_free_list_mtx);
1024 	vp = (struct vnode *) uma_zalloc(vnode_zone, M_WAITOK|M_ZERO);
1025 	/*
1026 	 * Setup locks.
1027 	 */
1028 	vp->v_vnlock = &vp->v_lock;
1029 	mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF);
1030 	/*
1031 	 * By default, don't allow shared locks unless filesystems
1032 	 * opt-in.
1033 	 */
1034 	lockinit(vp->v_vnlock, PVFS, tag, VLKTIMEOUT, LK_NOSHARE);
1035 	/*
1036 	 * Initialize bufobj.
1037 	 */
1038 	bo = &vp->v_bufobj;
1039 	bo->__bo_vnode = vp;
1040 	mtx_init(BO_MTX(bo), "bufobj interlock", NULL, MTX_DEF);
1041 	bo->bo_ops = &buf_ops_bio;
1042 	bo->bo_private = vp;
1043 	TAILQ_INIT(&bo->bo_clean.bv_hd);
1044 	TAILQ_INIT(&bo->bo_dirty.bv_hd);
1045 	/*
1046 	 * Initialize namecache.
1047 	 */
1048 	LIST_INIT(&vp->v_cache_src);
1049 	TAILQ_INIT(&vp->v_cache_dst);
1050 	/*
1051 	 * Finalize various vnode identity bits.
1052 	 */
1053 	vp->v_type = VNON;
1054 	vp->v_tag = tag;
1055 	vp->v_op = vops;
1056 	v_incr_usecount(vp);
1057 	vp->v_data = NULL;
1058 #ifdef MAC
1059 	mac_vnode_init(vp);
1060 	if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0)
1061 		mac_vnode_associate_singlelabel(mp, vp);
1062 	else if (mp == NULL && vops != &dead_vnodeops)
1063 		printf("NULL mp in getnewvnode()\n");
1064 #endif
1065 	if (mp != NULL) {
1066 		bo->bo_bsize = mp->mnt_stat.f_iosize;
1067 		if ((mp->mnt_kern_flag & MNTK_NOKNOTE) != 0)
1068 			vp->v_vflag |= VV_NOKNOTE;
1069 	}
1070 
1071 	*vpp = vp;
1072 	return (0);
1073 }
1074 
1075 /*
1076  * Delete from old mount point vnode list, if on one.
1077  */
1078 static void
1079 delmntque(struct vnode *vp)
1080 {
1081 	struct mount *mp;
1082 
1083 	mp = vp->v_mount;
1084 	if (mp == NULL)
1085 		return;
1086 	MNT_ILOCK(mp);
1087 	vp->v_mount = NULL;
1088 	VNASSERT(mp->mnt_nvnodelistsize > 0, vp,
1089 		("bad mount point vnode list size"));
1090 	TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1091 	mp->mnt_nvnodelistsize--;
1092 	MNT_REL(mp);
1093 	MNT_IUNLOCK(mp);
1094 }
1095 
1096 static void
1097 insmntque_stddtr(struct vnode *vp, void *dtr_arg)
1098 {
1099 
1100 	vp->v_data = NULL;
1101 	vp->v_op = &dead_vnodeops;
1102 	/* XXX non mp-safe fs may still call insmntque with vnode
1103 	   unlocked */
1104 	if (!VOP_ISLOCKED(vp))
1105 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1106 	vgone(vp);
1107 	vput(vp);
1108 }
1109 
1110 /*
1111  * Insert into list of vnodes for the new mount point, if available.
1112  */
1113 int
1114 insmntque1(struct vnode *vp, struct mount *mp,
1115 	void (*dtr)(struct vnode *, void *), void *dtr_arg)
1116 {
1117 	int locked;
1118 
1119 	KASSERT(vp->v_mount == NULL,
1120 		("insmntque: vnode already on per mount vnode list"));
1121 	VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)"));
1122 #ifdef DEBUG_VFS_LOCKS
1123 	if (!VFS_NEEDSGIANT(mp))
1124 		ASSERT_VOP_ELOCKED(vp,
1125 		    "insmntque: mp-safe fs and non-locked vp");
1126 #endif
1127 	MNT_ILOCK(mp);
1128 	if ((mp->mnt_kern_flag & MNTK_NOINSMNTQ) != 0 &&
1129 	    ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 ||
1130 	     mp->mnt_nvnodelistsize == 0)) {
1131 		locked = VOP_ISLOCKED(vp);
1132 		if (!locked || (locked == LK_EXCLUSIVE &&
1133 		     (vp->v_vflag & VV_FORCEINSMQ) == 0)) {
1134 			MNT_IUNLOCK(mp);
1135 			if (dtr != NULL)
1136 				dtr(vp, dtr_arg);
1137 			return (EBUSY);
1138 		}
1139 	}
1140 	vp->v_mount = mp;
1141 	MNT_REF(mp);
1142 	TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1143 	VNASSERT(mp->mnt_nvnodelistsize >= 0, vp,
1144 		("neg mount point vnode list size"));
1145 	mp->mnt_nvnodelistsize++;
1146 	MNT_IUNLOCK(mp);
1147 	return (0);
1148 }
1149 
1150 int
1151 insmntque(struct vnode *vp, struct mount *mp)
1152 {
1153 
1154 	return (insmntque1(vp, mp, insmntque_stddtr, NULL));
1155 }
1156 
1157 /*
1158  * Flush out and invalidate all buffers associated with a bufobj
1159  * Called with the underlying object locked.
1160  */
1161 int
1162 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
1163 {
1164 	int error;
1165 
1166 	BO_LOCK(bo);
1167 	if (flags & V_SAVE) {
1168 		error = bufobj_wwait(bo, slpflag, slptimeo);
1169 		if (error) {
1170 			BO_UNLOCK(bo);
1171 			return (error);
1172 		}
1173 		if (bo->bo_dirty.bv_cnt > 0) {
1174 			BO_UNLOCK(bo);
1175 			if ((error = BO_SYNC(bo, MNT_WAIT)) != 0)
1176 				return (error);
1177 			/*
1178 			 * XXX We could save a lock/unlock if this was only
1179 			 * enabled under INVARIANTS
1180 			 */
1181 			BO_LOCK(bo);
1182 			if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0)
1183 				panic("vinvalbuf: dirty bufs");
1184 		}
1185 	}
1186 	/*
1187 	 * If you alter this loop please notice that interlock is dropped and
1188 	 * reacquired in flushbuflist.  Special care is needed to ensure that
1189 	 * no race conditions occur from this.
1190 	 */
1191 	do {
1192 		error = flushbuflist(&bo->bo_clean,
1193 		    flags, bo, slpflag, slptimeo);
1194 		if (error == 0 && !(flags & V_CLEANONLY))
1195 			error = flushbuflist(&bo->bo_dirty,
1196 			    flags, bo, slpflag, slptimeo);
1197 		if (error != 0 && error != EAGAIN) {
1198 			BO_UNLOCK(bo);
1199 			return (error);
1200 		}
1201 	} while (error != 0);
1202 
1203 	/*
1204 	 * Wait for I/O to complete.  XXX needs cleaning up.  The vnode can
1205 	 * have write I/O in-progress but if there is a VM object then the
1206 	 * VM object can also have read-I/O in-progress.
1207 	 */
1208 	do {
1209 		bufobj_wwait(bo, 0, 0);
1210 		BO_UNLOCK(bo);
1211 		if (bo->bo_object != NULL) {
1212 			VM_OBJECT_LOCK(bo->bo_object);
1213 			vm_object_pip_wait(bo->bo_object, "bovlbx");
1214 			VM_OBJECT_UNLOCK(bo->bo_object);
1215 		}
1216 		BO_LOCK(bo);
1217 	} while (bo->bo_numoutput > 0);
1218 	BO_UNLOCK(bo);
1219 
1220 	/*
1221 	 * Destroy the copy in the VM cache, too.
1222 	 */
1223 	if (bo->bo_object != NULL &&
1224 	    (flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0) {
1225 		VM_OBJECT_LOCK(bo->bo_object);
1226 		vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ?
1227 		    OBJPR_CLEANONLY : 0);
1228 		VM_OBJECT_UNLOCK(bo->bo_object);
1229 	}
1230 
1231 #ifdef INVARIANTS
1232 	BO_LOCK(bo);
1233 	if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0 &&
1234 	    (bo->bo_dirty.bv_cnt > 0 || bo->bo_clean.bv_cnt > 0))
1235 		panic("vinvalbuf: flush failed");
1236 	BO_UNLOCK(bo);
1237 #endif
1238 	return (0);
1239 }
1240 
1241 /*
1242  * Flush out and invalidate all buffers associated with a vnode.
1243  * Called with the underlying object locked.
1244  */
1245 int
1246 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
1247 {
1248 
1249 	CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
1250 	ASSERT_VOP_LOCKED(vp, "vinvalbuf");
1251 	return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo));
1252 }
1253 
1254 /*
1255  * Flush out buffers on the specified list.
1256  *
1257  */
1258 static int
1259 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag,
1260     int slptimeo)
1261 {
1262 	struct buf *bp, *nbp;
1263 	int retval, error;
1264 	daddr_t lblkno;
1265 	b_xflags_t xflags;
1266 
1267 	ASSERT_BO_LOCKED(bo);
1268 
1269 	retval = 0;
1270 	TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) {
1271 		if (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA)) ||
1272 		    ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0)) {
1273 			continue;
1274 		}
1275 		lblkno = 0;
1276 		xflags = 0;
1277 		if (nbp != NULL) {
1278 			lblkno = nbp->b_lblkno;
1279 			xflags = nbp->b_xflags &
1280 				(BX_BKGRDMARKER | BX_VNDIRTY | BX_VNCLEAN);
1281 		}
1282 		retval = EAGAIN;
1283 		error = BUF_TIMELOCK(bp,
1284 		    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_MTX(bo),
1285 		    "flushbuf", slpflag, slptimeo);
1286 		if (error) {
1287 			BO_LOCK(bo);
1288 			return (error != ENOLCK ? error : EAGAIN);
1289 		}
1290 		KASSERT(bp->b_bufobj == bo,
1291 		    ("bp %p wrong b_bufobj %p should be %p",
1292 		    bp, bp->b_bufobj, bo));
1293 		if (bp->b_bufobj != bo) {	/* XXX: necessary ? */
1294 			BUF_UNLOCK(bp);
1295 			BO_LOCK(bo);
1296 			return (EAGAIN);
1297 		}
1298 		/*
1299 		 * XXX Since there are no node locks for NFS, I
1300 		 * believe there is a slight chance that a delayed
1301 		 * write will occur while sleeping just above, so
1302 		 * check for it.
1303 		 */
1304 		if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
1305 		    (flags & V_SAVE)) {
1306 			BO_LOCK(bo);
1307 			bremfree(bp);
1308 			BO_UNLOCK(bo);
1309 			bp->b_flags |= B_ASYNC;
1310 			bwrite(bp);
1311 			BO_LOCK(bo);
1312 			return (EAGAIN);	/* XXX: why not loop ? */
1313 		}
1314 		BO_LOCK(bo);
1315 		bremfree(bp);
1316 		BO_UNLOCK(bo);
1317 		bp->b_flags |= (B_INVAL | B_RELBUF);
1318 		bp->b_flags &= ~B_ASYNC;
1319 		brelse(bp);
1320 		BO_LOCK(bo);
1321 		if (nbp != NULL &&
1322 		    (nbp->b_bufobj != bo ||
1323 		     nbp->b_lblkno != lblkno ||
1324 		     (nbp->b_xflags &
1325 		      (BX_BKGRDMARKER | BX_VNDIRTY | BX_VNCLEAN)) != xflags))
1326 			break;			/* nbp invalid */
1327 	}
1328 	return (retval);
1329 }
1330 
1331 /*
1332  * Truncate a file's buffer and pages to a specified length.  This
1333  * is in lieu of the old vinvalbuf mechanism, which performed unneeded
1334  * sync activity.
1335  */
1336 int
1337 vtruncbuf(struct vnode *vp, struct ucred *cred, struct thread *td,
1338     off_t length, int blksize)
1339 {
1340 	struct buf *bp, *nbp;
1341 	int anyfreed;
1342 	int trunclbn;
1343 	struct bufobj *bo;
1344 
1345 	CTR5(KTR_VFS, "%s: vp %p with cred %p and block %d:%ju", __func__,
1346 	    vp, cred, blksize, (uintmax_t)length);
1347 
1348 	/*
1349 	 * Round up to the *next* lbn.
1350 	 */
1351 	trunclbn = (length + blksize - 1) / blksize;
1352 
1353 	ASSERT_VOP_LOCKED(vp, "vtruncbuf");
1354 restart:
1355 	bo = &vp->v_bufobj;
1356 	BO_LOCK(bo);
1357 	anyfreed = 1;
1358 	for (;anyfreed;) {
1359 		anyfreed = 0;
1360 		TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) {
1361 			if (bp->b_lblkno < trunclbn)
1362 				continue;
1363 			if (BUF_LOCK(bp,
1364 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1365 			    BO_MTX(bo)) == ENOLCK)
1366 				goto restart;
1367 
1368 			BO_LOCK(bo);
1369 			bremfree(bp);
1370 			BO_UNLOCK(bo);
1371 			bp->b_flags |= (B_INVAL | B_RELBUF);
1372 			bp->b_flags &= ~B_ASYNC;
1373 			brelse(bp);
1374 			anyfreed = 1;
1375 
1376 			BO_LOCK(bo);
1377 			if (nbp != NULL &&
1378 			    (((nbp->b_xflags & BX_VNCLEAN) == 0) ||
1379 			    (nbp->b_vp != vp) ||
1380 			    (nbp->b_flags & B_DELWRI))) {
1381 				BO_UNLOCK(bo);
1382 				goto restart;
1383 			}
1384 		}
1385 
1386 		TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
1387 			if (bp->b_lblkno < trunclbn)
1388 				continue;
1389 			if (BUF_LOCK(bp,
1390 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1391 			    BO_MTX(bo)) == ENOLCK)
1392 				goto restart;
1393 			BO_LOCK(bo);
1394 			bremfree(bp);
1395 			BO_UNLOCK(bo);
1396 			bp->b_flags |= (B_INVAL | B_RELBUF);
1397 			bp->b_flags &= ~B_ASYNC;
1398 			brelse(bp);
1399 			anyfreed = 1;
1400 
1401 			BO_LOCK(bo);
1402 			if (nbp != NULL &&
1403 			    (((nbp->b_xflags & BX_VNDIRTY) == 0) ||
1404 			    (nbp->b_vp != vp) ||
1405 			    (nbp->b_flags & B_DELWRI) == 0)) {
1406 				BO_UNLOCK(bo);
1407 				goto restart;
1408 			}
1409 		}
1410 	}
1411 
1412 	if (length > 0) {
1413 restartsync:
1414 		TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
1415 			if (bp->b_lblkno > 0)
1416 				continue;
1417 			/*
1418 			 * Since we hold the vnode lock this should only
1419 			 * fail if we're racing with the buf daemon.
1420 			 */
1421 			if (BUF_LOCK(bp,
1422 			    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1423 			    BO_MTX(bo)) == ENOLCK) {
1424 				goto restart;
1425 			}
1426 			VNASSERT((bp->b_flags & B_DELWRI), vp,
1427 			    ("buf(%p) on dirty queue without DELWRI", bp));
1428 
1429 			BO_LOCK(bo);
1430 			bremfree(bp);
1431 			BO_UNLOCK(bo);
1432 			bawrite(bp);
1433 			BO_LOCK(bo);
1434 			goto restartsync;
1435 		}
1436 	}
1437 
1438 	bufobj_wwait(bo, 0, 0);
1439 	BO_UNLOCK(bo);
1440 	vnode_pager_setsize(vp, length);
1441 
1442 	return (0);
1443 }
1444 
1445 /*
1446  * buf_splay() - splay tree core for the clean/dirty list of buffers in
1447  *		 a vnode.
1448  *
1449  *	NOTE: We have to deal with the special case of a background bitmap
1450  *	buffer, a situation where two buffers will have the same logical
1451  *	block offset.  We want (1) only the foreground buffer to be accessed
1452  *	in a lookup and (2) must differentiate between the foreground and
1453  *	background buffer in the splay tree algorithm because the splay
1454  *	tree cannot normally handle multiple entities with the same 'index'.
1455  *	We accomplish this by adding differentiating flags to the splay tree's
1456  *	numerical domain.
1457  */
1458 static
1459 struct buf *
1460 buf_splay(daddr_t lblkno, b_xflags_t xflags, struct buf *root)
1461 {
1462 	struct buf dummy;
1463 	struct buf *lefttreemax, *righttreemin, *y;
1464 
1465 	if (root == NULL)
1466 		return (NULL);
1467 	lefttreemax = righttreemin = &dummy;
1468 	for (;;) {
1469 		if (lblkno < root->b_lblkno ||
1470 		    (lblkno == root->b_lblkno &&
1471 		    (xflags & BX_BKGRDMARKER) < (root->b_xflags & BX_BKGRDMARKER))) {
1472 			if ((y = root->b_left) == NULL)
1473 				break;
1474 			if (lblkno < y->b_lblkno) {
1475 				/* Rotate right. */
1476 				root->b_left = y->b_right;
1477 				y->b_right = root;
1478 				root = y;
1479 				if ((y = root->b_left) == NULL)
1480 					break;
1481 			}
1482 			/* Link into the new root's right tree. */
1483 			righttreemin->b_left = root;
1484 			righttreemin = root;
1485 		} else if (lblkno > root->b_lblkno ||
1486 		    (lblkno == root->b_lblkno &&
1487 		    (xflags & BX_BKGRDMARKER) > (root->b_xflags & BX_BKGRDMARKER))) {
1488 			if ((y = root->b_right) == NULL)
1489 				break;
1490 			if (lblkno > y->b_lblkno) {
1491 				/* Rotate left. */
1492 				root->b_right = y->b_left;
1493 				y->b_left = root;
1494 				root = y;
1495 				if ((y = root->b_right) == NULL)
1496 					break;
1497 			}
1498 			/* Link into the new root's left tree. */
1499 			lefttreemax->b_right = root;
1500 			lefttreemax = root;
1501 		} else {
1502 			break;
1503 		}
1504 		root = y;
1505 	}
1506 	/* Assemble the new root. */
1507 	lefttreemax->b_right = root->b_left;
1508 	righttreemin->b_left = root->b_right;
1509 	root->b_left = dummy.b_right;
1510 	root->b_right = dummy.b_left;
1511 	return (root);
1512 }
1513 
1514 static void
1515 buf_vlist_remove(struct buf *bp)
1516 {
1517 	struct buf *root;
1518 	struct bufv *bv;
1519 
1520 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
1521 	ASSERT_BO_LOCKED(bp->b_bufobj);
1522 	KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) !=
1523 	    (BX_VNDIRTY|BX_VNCLEAN),
1524 	    ("buf_vlist_remove: Buf %p is on two lists", bp));
1525 	if (bp->b_xflags & BX_VNDIRTY)
1526 		bv = &bp->b_bufobj->bo_dirty;
1527 	else
1528 		bv = &bp->b_bufobj->bo_clean;
1529 	if (bp != bv->bv_root) {
1530 		root = buf_splay(bp->b_lblkno, bp->b_xflags, bv->bv_root);
1531 		KASSERT(root == bp, ("splay lookup failed in remove"));
1532 	}
1533 	if (bp->b_left == NULL) {
1534 		root = bp->b_right;
1535 	} else {
1536 		root = buf_splay(bp->b_lblkno, bp->b_xflags, bp->b_left);
1537 		root->b_right = bp->b_right;
1538 	}
1539 	bv->bv_root = root;
1540 	TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs);
1541 	bv->bv_cnt--;
1542 	bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN);
1543 }
1544 
1545 /*
1546  * Add the buffer to the sorted clean or dirty block list using a
1547  * splay tree algorithm.
1548  *
1549  * NOTE: xflags is passed as a constant, optimizing this inline function!
1550  */
1551 static void
1552 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags)
1553 {
1554 	struct buf *root;
1555 	struct bufv *bv;
1556 
1557 	ASSERT_BO_LOCKED(bo);
1558 	KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0,
1559 	    ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags));
1560 	bp->b_xflags |= xflags;
1561 	if (xflags & BX_VNDIRTY)
1562 		bv = &bo->bo_dirty;
1563 	else
1564 		bv = &bo->bo_clean;
1565 
1566 	root = buf_splay(bp->b_lblkno, bp->b_xflags, bv->bv_root);
1567 	if (root == NULL) {
1568 		bp->b_left = NULL;
1569 		bp->b_right = NULL;
1570 		TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs);
1571 	} else if (bp->b_lblkno < root->b_lblkno ||
1572 	    (bp->b_lblkno == root->b_lblkno &&
1573 	    (bp->b_xflags & BX_BKGRDMARKER) < (root->b_xflags & BX_BKGRDMARKER))) {
1574 		bp->b_left = root->b_left;
1575 		bp->b_right = root;
1576 		root->b_left = NULL;
1577 		TAILQ_INSERT_BEFORE(root, bp, b_bobufs);
1578 	} else {
1579 		bp->b_right = root->b_right;
1580 		bp->b_left = root;
1581 		root->b_right = NULL;
1582 		TAILQ_INSERT_AFTER(&bv->bv_hd, root, bp, b_bobufs);
1583 	}
1584 	bv->bv_cnt++;
1585 	bv->bv_root = bp;
1586 }
1587 
1588 /*
1589  * Lookup a buffer using the splay tree.  Note that we specifically avoid
1590  * shadow buffers used in background bitmap writes.
1591  *
1592  * This code isn't quite efficient as it could be because we are maintaining
1593  * two sorted lists and do not know which list the block resides in.
1594  *
1595  * During a "make buildworld" the desired buffer is found at one of
1596  * the roots more than 60% of the time.  Thus, checking both roots
1597  * before performing either splay eliminates unnecessary splays on the
1598  * first tree splayed.
1599  */
1600 struct buf *
1601 gbincore(struct bufobj *bo, daddr_t lblkno)
1602 {
1603 	struct buf *bp;
1604 
1605 	ASSERT_BO_LOCKED(bo);
1606 	if ((bp = bo->bo_clean.bv_root) != NULL &&
1607 	    bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1608 		return (bp);
1609 	if ((bp = bo->bo_dirty.bv_root) != NULL &&
1610 	    bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1611 		return (bp);
1612 	if ((bp = bo->bo_clean.bv_root) != NULL) {
1613 		bo->bo_clean.bv_root = bp = buf_splay(lblkno, 0, bp);
1614 		if (bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1615 			return (bp);
1616 	}
1617 	if ((bp = bo->bo_dirty.bv_root) != NULL) {
1618 		bo->bo_dirty.bv_root = bp = buf_splay(lblkno, 0, bp);
1619 		if (bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1620 			return (bp);
1621 	}
1622 	return (NULL);
1623 }
1624 
1625 /*
1626  * Associate a buffer with a vnode.
1627  */
1628 void
1629 bgetvp(struct vnode *vp, struct buf *bp)
1630 {
1631 	struct bufobj *bo;
1632 
1633 	bo = &vp->v_bufobj;
1634 	ASSERT_BO_LOCKED(bo);
1635 	VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free"));
1636 
1637 	CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags);
1638 	VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp,
1639 	    ("bgetvp: bp already attached! %p", bp));
1640 
1641 	vhold(vp);
1642 	if (VFS_NEEDSGIANT(vp->v_mount) || bo->bo_flag & BO_NEEDSGIANT)
1643 		bp->b_flags |= B_NEEDSGIANT;
1644 	bp->b_vp = vp;
1645 	bp->b_bufobj = bo;
1646 	/*
1647 	 * Insert onto list for new vnode.
1648 	 */
1649 	buf_vlist_add(bp, bo, BX_VNCLEAN);
1650 }
1651 
1652 /*
1653  * Disassociate a buffer from a vnode.
1654  */
1655 void
1656 brelvp(struct buf *bp)
1657 {
1658 	struct bufobj *bo;
1659 	struct vnode *vp;
1660 
1661 	CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1662 	KASSERT(bp->b_vp != NULL, ("brelvp: NULL"));
1663 
1664 	/*
1665 	 * Delete from old vnode list, if on one.
1666 	 */
1667 	vp = bp->b_vp;		/* XXX */
1668 	bo = bp->b_bufobj;
1669 	BO_LOCK(bo);
1670 	if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
1671 		buf_vlist_remove(bp);
1672 	else
1673 		panic("brelvp: Buffer %p not on queue.", bp);
1674 	if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
1675 		bo->bo_flag &= ~BO_ONWORKLST;
1676 		mtx_lock(&sync_mtx);
1677 		LIST_REMOVE(bo, bo_synclist);
1678 		syncer_worklist_len--;
1679 		mtx_unlock(&sync_mtx);
1680 	}
1681 	bp->b_flags &= ~B_NEEDSGIANT;
1682 	bp->b_vp = NULL;
1683 	bp->b_bufobj = NULL;
1684 	BO_UNLOCK(bo);
1685 	vdrop(vp);
1686 }
1687 
1688 /*
1689  * Add an item to the syncer work queue.
1690  */
1691 static void
1692 vn_syncer_add_to_worklist(struct bufobj *bo, int delay)
1693 {
1694 	int queue, slot;
1695 
1696 	ASSERT_BO_LOCKED(bo);
1697 
1698 	mtx_lock(&sync_mtx);
1699 	if (bo->bo_flag & BO_ONWORKLST)
1700 		LIST_REMOVE(bo, bo_synclist);
1701 	else {
1702 		bo->bo_flag |= BO_ONWORKLST;
1703 		syncer_worklist_len++;
1704 	}
1705 
1706 	if (delay > syncer_maxdelay - 2)
1707 		delay = syncer_maxdelay - 2;
1708 	slot = (syncer_delayno + delay) & syncer_mask;
1709 
1710 	queue = VFS_NEEDSGIANT(bo->__bo_vnode->v_mount) ? WI_GIANTQ :
1711 	    WI_MPSAFEQ;
1712 	LIST_INSERT_HEAD(&syncer_workitem_pending[queue][slot], bo,
1713 	    bo_synclist);
1714 	mtx_unlock(&sync_mtx);
1715 }
1716 
1717 static int
1718 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS)
1719 {
1720 	int error, len;
1721 
1722 	mtx_lock(&sync_mtx);
1723 	len = syncer_worklist_len - sync_vnode_count;
1724 	mtx_unlock(&sync_mtx);
1725 	error = SYSCTL_OUT(req, &len, sizeof(len));
1726 	return (error);
1727 }
1728 
1729 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, CTLTYPE_INT | CTLFLAG_RD, NULL, 0,
1730     sysctl_vfs_worklist_len, "I", "Syncer thread worklist length");
1731 
1732 static struct proc *updateproc;
1733 static void sched_sync(void);
1734 static struct kproc_desc up_kp = {
1735 	"syncer",
1736 	sched_sync,
1737 	&updateproc
1738 };
1739 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp);
1740 
1741 static int
1742 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td)
1743 {
1744 	struct vnode *vp;
1745 	struct mount *mp;
1746 
1747 	*bo = LIST_FIRST(slp);
1748 	if (*bo == NULL)
1749 		return (0);
1750 	vp = (*bo)->__bo_vnode;	/* XXX */
1751 	if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0)
1752 		return (1);
1753 	/*
1754 	 * We use vhold in case the vnode does not
1755 	 * successfully sync.  vhold prevents the vnode from
1756 	 * going away when we unlock the sync_mtx so that
1757 	 * we can acquire the vnode interlock.
1758 	 */
1759 	vholdl(vp);
1760 	mtx_unlock(&sync_mtx);
1761 	VI_UNLOCK(vp);
1762 	if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
1763 		vdrop(vp);
1764 		mtx_lock(&sync_mtx);
1765 		return (*bo == LIST_FIRST(slp));
1766 	}
1767 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1768 	(void) VOP_FSYNC(vp, MNT_LAZY, td);
1769 	VOP_UNLOCK(vp, 0);
1770 	vn_finished_write(mp);
1771 	BO_LOCK(*bo);
1772 	if (((*bo)->bo_flag & BO_ONWORKLST) != 0) {
1773 		/*
1774 		 * Put us back on the worklist.  The worklist
1775 		 * routine will remove us from our current
1776 		 * position and then add us back in at a later
1777 		 * position.
1778 		 */
1779 		vn_syncer_add_to_worklist(*bo, syncdelay);
1780 	}
1781 	BO_UNLOCK(*bo);
1782 	vdrop(vp);
1783 	mtx_lock(&sync_mtx);
1784 	return (0);
1785 }
1786 
1787 /*
1788  * System filesystem synchronizer daemon.
1789  */
1790 static void
1791 sched_sync(void)
1792 {
1793 	struct synclist *gnext, *next;
1794 	struct synclist *gslp, *slp;
1795 	struct bufobj *bo;
1796 	long starttime;
1797 	struct thread *td = curthread;
1798 	int last_work_seen;
1799 	int net_worklist_len;
1800 	int syncer_final_iter;
1801 	int first_printf;
1802 	int error;
1803 
1804 	last_work_seen = 0;
1805 	syncer_final_iter = 0;
1806 	first_printf = 1;
1807 	syncer_state = SYNCER_RUNNING;
1808 	starttime = time_uptime;
1809 	td->td_pflags |= TDP_NORUNNINGBUF;
1810 
1811 	EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc,
1812 	    SHUTDOWN_PRI_LAST);
1813 
1814 	mtx_lock(&sync_mtx);
1815 	for (;;) {
1816 		if (syncer_state == SYNCER_FINAL_DELAY &&
1817 		    syncer_final_iter == 0) {
1818 			mtx_unlock(&sync_mtx);
1819 			kproc_suspend_check(td->td_proc);
1820 			mtx_lock(&sync_mtx);
1821 		}
1822 		net_worklist_len = syncer_worklist_len - sync_vnode_count;
1823 		if (syncer_state != SYNCER_RUNNING &&
1824 		    starttime != time_uptime) {
1825 			if (first_printf) {
1826 				printf("\nSyncing disks, vnodes remaining...");
1827 				first_printf = 0;
1828 			}
1829 			printf("%d ", net_worklist_len);
1830 		}
1831 		starttime = time_uptime;
1832 
1833 		/*
1834 		 * Push files whose dirty time has expired.  Be careful
1835 		 * of interrupt race on slp queue.
1836 		 *
1837 		 * Skip over empty worklist slots when shutting down.
1838 		 */
1839 		do {
1840 			slp = &syncer_workitem_pending[WI_MPSAFEQ][syncer_delayno];
1841 			gslp = &syncer_workitem_pending[WI_GIANTQ][syncer_delayno];
1842 			syncer_delayno += 1;
1843 			if (syncer_delayno == syncer_maxdelay)
1844 				syncer_delayno = 0;
1845 			next = &syncer_workitem_pending[WI_MPSAFEQ][syncer_delayno];
1846 			gnext = &syncer_workitem_pending[WI_GIANTQ][syncer_delayno];
1847 			/*
1848 			 * If the worklist has wrapped since the
1849 			 * it was emptied of all but syncer vnodes,
1850 			 * switch to the FINAL_DELAY state and run
1851 			 * for one more second.
1852 			 */
1853 			if (syncer_state == SYNCER_SHUTTING_DOWN &&
1854 			    net_worklist_len == 0 &&
1855 			    last_work_seen == syncer_delayno) {
1856 				syncer_state = SYNCER_FINAL_DELAY;
1857 				syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP;
1858 			}
1859 		} while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) &&
1860 		    LIST_EMPTY(gslp) && syncer_worklist_len > 0);
1861 
1862 		/*
1863 		 * Keep track of the last time there was anything
1864 		 * on the worklist other than syncer vnodes.
1865 		 * Return to the SHUTTING_DOWN state if any
1866 		 * new work appears.
1867 		 */
1868 		if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING)
1869 			last_work_seen = syncer_delayno;
1870 		if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY)
1871 			syncer_state = SYNCER_SHUTTING_DOWN;
1872 		while (!LIST_EMPTY(slp)) {
1873 			error = sync_vnode(slp, &bo, td);
1874 			if (error == 1) {
1875 				LIST_REMOVE(bo, bo_synclist);
1876 				LIST_INSERT_HEAD(next, bo, bo_synclist);
1877 				continue;
1878 			}
1879 #ifdef SW_WATCHDOG
1880 			if (first_printf == 0)
1881 				wdog_kern_pat(WD_LASTVAL);
1882 #endif
1883 		}
1884 		if (!LIST_EMPTY(gslp)) {
1885 			mtx_unlock(&sync_mtx);
1886 			mtx_lock(&Giant);
1887 			mtx_lock(&sync_mtx);
1888 			while (!LIST_EMPTY(gslp)) {
1889 				error = sync_vnode(gslp, &bo, td);
1890 				if (error == 1) {
1891 					LIST_REMOVE(bo, bo_synclist);
1892 					LIST_INSERT_HEAD(gnext, bo,
1893 					    bo_synclist);
1894 					continue;
1895 				}
1896 			}
1897 			mtx_unlock(&Giant);
1898 		}
1899 		if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0)
1900 			syncer_final_iter--;
1901 		/*
1902 		 * The variable rushjob allows the kernel to speed up the
1903 		 * processing of the filesystem syncer process. A rushjob
1904 		 * value of N tells the filesystem syncer to process the next
1905 		 * N seconds worth of work on its queue ASAP. Currently rushjob
1906 		 * is used by the soft update code to speed up the filesystem
1907 		 * syncer process when the incore state is getting so far
1908 		 * ahead of the disk that the kernel memory pool is being
1909 		 * threatened with exhaustion.
1910 		 */
1911 		if (rushjob > 0) {
1912 			rushjob -= 1;
1913 			continue;
1914 		}
1915 		/*
1916 		 * Just sleep for a short period of time between
1917 		 * iterations when shutting down to allow some I/O
1918 		 * to happen.
1919 		 *
1920 		 * If it has taken us less than a second to process the
1921 		 * current work, then wait. Otherwise start right over
1922 		 * again. We can still lose time if any single round
1923 		 * takes more than two seconds, but it does not really
1924 		 * matter as we are just trying to generally pace the
1925 		 * filesystem activity.
1926 		 */
1927 		if (syncer_state != SYNCER_RUNNING ||
1928 		    time_uptime == starttime) {
1929 			thread_lock(td);
1930 			sched_prio(td, PPAUSE);
1931 			thread_unlock(td);
1932 		}
1933 		if (syncer_state != SYNCER_RUNNING)
1934 			cv_timedwait(&sync_wakeup, &sync_mtx,
1935 			    hz / SYNCER_SHUTDOWN_SPEEDUP);
1936 		else if (time_uptime == starttime)
1937 			cv_timedwait(&sync_wakeup, &sync_mtx, hz);
1938 	}
1939 }
1940 
1941 /*
1942  * Request the syncer daemon to speed up its work.
1943  * We never push it to speed up more than half of its
1944  * normal turn time, otherwise it could take over the cpu.
1945  */
1946 int
1947 speedup_syncer(void)
1948 {
1949 	int ret = 0;
1950 
1951 	mtx_lock(&sync_mtx);
1952 	if (rushjob < syncdelay / 2) {
1953 		rushjob += 1;
1954 		stat_rush_requests += 1;
1955 		ret = 1;
1956 	}
1957 	mtx_unlock(&sync_mtx);
1958 	cv_broadcast(&sync_wakeup);
1959 	return (ret);
1960 }
1961 
1962 /*
1963  * Tell the syncer to speed up its work and run though its work
1964  * list several times, then tell it to shut down.
1965  */
1966 static void
1967 syncer_shutdown(void *arg, int howto)
1968 {
1969 
1970 	if (howto & RB_NOSYNC)
1971 		return;
1972 	mtx_lock(&sync_mtx);
1973 	syncer_state = SYNCER_SHUTTING_DOWN;
1974 	rushjob = 0;
1975 	mtx_unlock(&sync_mtx);
1976 	cv_broadcast(&sync_wakeup);
1977 	kproc_shutdown(arg, howto);
1978 }
1979 
1980 /*
1981  * Reassign a buffer from one vnode to another.
1982  * Used to assign file specific control information
1983  * (indirect blocks) to the vnode to which they belong.
1984  */
1985 void
1986 reassignbuf(struct buf *bp)
1987 {
1988 	struct vnode *vp;
1989 	struct bufobj *bo;
1990 	int delay;
1991 #ifdef INVARIANTS
1992 	struct bufv *bv;
1993 #endif
1994 
1995 	vp = bp->b_vp;
1996 	bo = bp->b_bufobj;
1997 	++reassignbufcalls;
1998 
1999 	CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X",
2000 	    bp, bp->b_vp, bp->b_flags);
2001 	/*
2002 	 * B_PAGING flagged buffers cannot be reassigned because their vp
2003 	 * is not fully linked in.
2004 	 */
2005 	if (bp->b_flags & B_PAGING)
2006 		panic("cannot reassign paging buffer");
2007 
2008 	/*
2009 	 * Delete from old vnode list, if on one.
2010 	 */
2011 	BO_LOCK(bo);
2012 	if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
2013 		buf_vlist_remove(bp);
2014 	else
2015 		panic("reassignbuf: Buffer %p not on queue.", bp);
2016 	/*
2017 	 * If dirty, put on list of dirty buffers; otherwise insert onto list
2018 	 * of clean buffers.
2019 	 */
2020 	if (bp->b_flags & B_DELWRI) {
2021 		if ((bo->bo_flag & BO_ONWORKLST) == 0) {
2022 			switch (vp->v_type) {
2023 			case VDIR:
2024 				delay = dirdelay;
2025 				break;
2026 			case VCHR:
2027 				delay = metadelay;
2028 				break;
2029 			default:
2030 				delay = filedelay;
2031 			}
2032 			vn_syncer_add_to_worklist(bo, delay);
2033 		}
2034 		buf_vlist_add(bp, bo, BX_VNDIRTY);
2035 	} else {
2036 		buf_vlist_add(bp, bo, BX_VNCLEAN);
2037 
2038 		if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
2039 			mtx_lock(&sync_mtx);
2040 			LIST_REMOVE(bo, bo_synclist);
2041 			syncer_worklist_len--;
2042 			mtx_unlock(&sync_mtx);
2043 			bo->bo_flag &= ~BO_ONWORKLST;
2044 		}
2045 	}
2046 #ifdef INVARIANTS
2047 	bv = &bo->bo_clean;
2048 	bp = TAILQ_FIRST(&bv->bv_hd);
2049 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2050 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2051 	bp = TAILQ_LAST(&bv->bv_hd, buflists);
2052 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2053 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2054 	bv = &bo->bo_dirty;
2055 	bp = TAILQ_FIRST(&bv->bv_hd);
2056 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2057 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2058 	bp = TAILQ_LAST(&bv->bv_hd, buflists);
2059 	KASSERT(bp == NULL || bp->b_bufobj == bo,
2060 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2061 #endif
2062 	BO_UNLOCK(bo);
2063 }
2064 
2065 /*
2066  * Increment the use and hold counts on the vnode, taking care to reference
2067  * the driver's usecount if this is a chardev.  The vholdl() will remove
2068  * the vnode from the free list if it is presently free.  Requires the
2069  * vnode interlock and returns with it held.
2070  */
2071 static void
2072 v_incr_usecount(struct vnode *vp)
2073 {
2074 
2075 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2076 	vp->v_usecount++;
2077 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2078 		dev_lock();
2079 		vp->v_rdev->si_usecount++;
2080 		dev_unlock();
2081 	}
2082 	vholdl(vp);
2083 }
2084 
2085 /*
2086  * Turn a holdcnt into a use+holdcnt such that only one call to
2087  * v_decr_usecount is needed.
2088  */
2089 static void
2090 v_upgrade_usecount(struct vnode *vp)
2091 {
2092 
2093 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2094 	vp->v_usecount++;
2095 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2096 		dev_lock();
2097 		vp->v_rdev->si_usecount++;
2098 		dev_unlock();
2099 	}
2100 }
2101 
2102 /*
2103  * Decrement the vnode use and hold count along with the driver's usecount
2104  * if this is a chardev.  The vdropl() below releases the vnode interlock
2105  * as it may free the vnode.
2106  */
2107 static void
2108 v_decr_usecount(struct vnode *vp)
2109 {
2110 
2111 	ASSERT_VI_LOCKED(vp, __FUNCTION__);
2112 	VNASSERT(vp->v_usecount > 0, vp,
2113 	    ("v_decr_usecount: negative usecount"));
2114 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2115 	vp->v_usecount--;
2116 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2117 		dev_lock();
2118 		vp->v_rdev->si_usecount--;
2119 		dev_unlock();
2120 	}
2121 	vdropl(vp);
2122 }
2123 
2124 /*
2125  * Decrement only the use count and driver use count.  This is intended to
2126  * be paired with a follow on vdropl() to release the remaining hold count.
2127  * In this way we may vgone() a vnode with a 0 usecount without risk of
2128  * having it end up on a free list because the hold count is kept above 0.
2129  */
2130 static void
2131 v_decr_useonly(struct vnode *vp)
2132 {
2133 
2134 	ASSERT_VI_LOCKED(vp, __FUNCTION__);
2135 	VNASSERT(vp->v_usecount > 0, vp,
2136 	    ("v_decr_useonly: negative usecount"));
2137 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2138 	vp->v_usecount--;
2139 	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2140 		dev_lock();
2141 		vp->v_rdev->si_usecount--;
2142 		dev_unlock();
2143 	}
2144 }
2145 
2146 /*
2147  * Grab a particular vnode from the free list, increment its
2148  * reference count and lock it.  VI_DOOMED is set if the vnode
2149  * is being destroyed.  Only callers who specify LK_RETRY will
2150  * see doomed vnodes.  If inactive processing was delayed in
2151  * vput try to do it here.
2152  */
2153 int
2154 vget(struct vnode *vp, int flags, struct thread *td)
2155 {
2156 	int error;
2157 
2158 	error = 0;
2159 	VFS_ASSERT_GIANT(vp->v_mount);
2160 	VNASSERT((flags & LK_TYPE_MASK) != 0, vp,
2161 	    ("vget: invalid lock operation"));
2162 	CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
2163 
2164 	if ((flags & LK_INTERLOCK) == 0)
2165 		VI_LOCK(vp);
2166 	vholdl(vp);
2167 	if ((error = vn_lock(vp, flags | LK_INTERLOCK)) != 0) {
2168 		vdrop(vp);
2169 		CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__,
2170 		    vp);
2171 		return (error);
2172 	}
2173 	if (vp->v_iflag & VI_DOOMED && (flags & LK_RETRY) == 0)
2174 		panic("vget: vn_lock failed to return ENOENT\n");
2175 	VI_LOCK(vp);
2176 	/* Upgrade our holdcnt to a usecount. */
2177 	v_upgrade_usecount(vp);
2178 	/*
2179 	 * We don't guarantee that any particular close will
2180 	 * trigger inactive processing so just make a best effort
2181 	 * here at preventing a reference to a removed file.  If
2182 	 * we don't succeed no harm is done.
2183 	 */
2184 	if (vp->v_iflag & VI_OWEINACT) {
2185 		if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE &&
2186 		    (flags & LK_NOWAIT) == 0)
2187 			vinactive(vp, td);
2188 		vp->v_iflag &= ~VI_OWEINACT;
2189 	}
2190 	VI_UNLOCK(vp);
2191 	return (0);
2192 }
2193 
2194 /*
2195  * Increase the reference count of a vnode.
2196  */
2197 void
2198 vref(struct vnode *vp)
2199 {
2200 
2201 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2202 	VI_LOCK(vp);
2203 	v_incr_usecount(vp);
2204 	VI_UNLOCK(vp);
2205 }
2206 
2207 /*
2208  * Return reference count of a vnode.
2209  *
2210  * The results of this call are only guaranteed when some mechanism other
2211  * than the VI lock is used to stop other processes from gaining references
2212  * to the vnode.  This may be the case if the caller holds the only reference.
2213  * This is also useful when stale data is acceptable as race conditions may
2214  * be accounted for by some other means.
2215  */
2216 int
2217 vrefcnt(struct vnode *vp)
2218 {
2219 	int usecnt;
2220 
2221 	VI_LOCK(vp);
2222 	usecnt = vp->v_usecount;
2223 	VI_UNLOCK(vp);
2224 
2225 	return (usecnt);
2226 }
2227 
2228 #define	VPUTX_VRELE	1
2229 #define	VPUTX_VPUT	2
2230 #define	VPUTX_VUNREF	3
2231 
2232 static void
2233 vputx(struct vnode *vp, int func)
2234 {
2235 	int error;
2236 
2237 	KASSERT(vp != NULL, ("vputx: null vp"));
2238 	if (func == VPUTX_VUNREF)
2239 		ASSERT_VOP_LOCKED(vp, "vunref");
2240 	else if (func == VPUTX_VPUT)
2241 		ASSERT_VOP_LOCKED(vp, "vput");
2242 	else
2243 		KASSERT(func == VPUTX_VRELE, ("vputx: wrong func"));
2244 	VFS_ASSERT_GIANT(vp->v_mount);
2245 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2246 	VI_LOCK(vp);
2247 
2248 	/* Skip this v_writecount check if we're going to panic below. */
2249 	VNASSERT(vp->v_writecount < vp->v_usecount || vp->v_usecount < 1, vp,
2250 	    ("vputx: missed vn_close"));
2251 	error = 0;
2252 
2253 	if (vp->v_usecount > 1 || ((vp->v_iflag & VI_DOINGINACT) &&
2254 	    vp->v_usecount == 1)) {
2255 		if (func == VPUTX_VPUT)
2256 			VOP_UNLOCK(vp, 0);
2257 		v_decr_usecount(vp);
2258 		return;
2259 	}
2260 
2261 	if (vp->v_usecount != 1) {
2262 		vprint("vputx: negative ref count", vp);
2263 		panic("vputx: negative ref cnt");
2264 	}
2265 	CTR2(KTR_VFS, "%s: return vnode %p to the freelist", __func__, vp);
2266 	/*
2267 	 * We want to hold the vnode until the inactive finishes to
2268 	 * prevent vgone() races.  We drop the use count here and the
2269 	 * hold count below when we're done.
2270 	 */
2271 	v_decr_useonly(vp);
2272 	/*
2273 	 * We must call VOP_INACTIVE with the node locked. Mark
2274 	 * as VI_DOINGINACT to avoid recursion.
2275 	 */
2276 	vp->v_iflag |= VI_OWEINACT;
2277 	switch (func) {
2278 	case VPUTX_VRELE:
2279 		error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK);
2280 		VI_LOCK(vp);
2281 		break;
2282 	case VPUTX_VPUT:
2283 		if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
2284 			error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK |
2285 			    LK_NOWAIT);
2286 			VI_LOCK(vp);
2287 		}
2288 		break;
2289 	case VPUTX_VUNREF:
2290 		if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
2291 			error = EBUSY;
2292 		break;
2293 	}
2294 	if (vp->v_usecount > 0)
2295 		vp->v_iflag &= ~VI_OWEINACT;
2296 	if (error == 0) {
2297 		if (vp->v_iflag & VI_OWEINACT)
2298 			vinactive(vp, curthread);
2299 		if (func != VPUTX_VUNREF)
2300 			VOP_UNLOCK(vp, 0);
2301 	}
2302 	vdropl(vp);
2303 }
2304 
2305 /*
2306  * Vnode put/release.
2307  * If count drops to zero, call inactive routine and return to freelist.
2308  */
2309 void
2310 vrele(struct vnode *vp)
2311 {
2312 
2313 	vputx(vp, VPUTX_VRELE);
2314 }
2315 
2316 /*
2317  * Release an already locked vnode.  This give the same effects as
2318  * unlock+vrele(), but takes less time and avoids releasing and
2319  * re-aquiring the lock (as vrele() acquires the lock internally.)
2320  */
2321 void
2322 vput(struct vnode *vp)
2323 {
2324 
2325 	vputx(vp, VPUTX_VPUT);
2326 }
2327 
2328 /*
2329  * Release an exclusively locked vnode. Do not unlock the vnode lock.
2330  */
2331 void
2332 vunref(struct vnode *vp)
2333 {
2334 
2335 	vputx(vp, VPUTX_VUNREF);
2336 }
2337 
2338 /*
2339  * Somebody doesn't want the vnode recycled.
2340  */
2341 void
2342 vhold(struct vnode *vp)
2343 {
2344 
2345 	VI_LOCK(vp);
2346 	vholdl(vp);
2347 	VI_UNLOCK(vp);
2348 }
2349 
2350 void
2351 vholdl(struct vnode *vp)
2352 {
2353 
2354 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2355 	vp->v_holdcnt++;
2356 	if (VSHOULDBUSY(vp))
2357 		vbusy(vp);
2358 }
2359 
2360 /*
2361  * Note that there is one less who cares about this vnode.  vdrop() is the
2362  * opposite of vhold().
2363  */
2364 void
2365 vdrop(struct vnode *vp)
2366 {
2367 
2368 	VI_LOCK(vp);
2369 	vdropl(vp);
2370 }
2371 
2372 /*
2373  * Drop the hold count of the vnode.  If this is the last reference to
2374  * the vnode we will free it if it has been vgone'd otherwise it is
2375  * placed on the free list.
2376  */
2377 void
2378 vdropl(struct vnode *vp)
2379 {
2380 
2381 	ASSERT_VI_LOCKED(vp, "vdropl");
2382 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2383 	if (vp->v_holdcnt <= 0)
2384 		panic("vdrop: holdcnt %d", vp->v_holdcnt);
2385 	vp->v_holdcnt--;
2386 	if (vp->v_holdcnt == 0) {
2387 		if (vp->v_iflag & VI_DOOMED) {
2388 			CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__,
2389 			    vp);
2390 			vdestroy(vp);
2391 			return;
2392 		} else
2393 			vfree(vp);
2394 	}
2395 	VI_UNLOCK(vp);
2396 }
2397 
2398 /*
2399  * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT
2400  * flags.  DOINGINACT prevents us from recursing in calls to vinactive.
2401  * OWEINACT tracks whether a vnode missed a call to inactive due to a
2402  * failed lock upgrade.
2403  */
2404 static void
2405 vinactive(struct vnode *vp, struct thread *td)
2406 {
2407 
2408 	ASSERT_VOP_ELOCKED(vp, "vinactive");
2409 	ASSERT_VI_LOCKED(vp, "vinactive");
2410 	VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp,
2411 	    ("vinactive: recursed on VI_DOINGINACT"));
2412 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2413 	vp->v_iflag |= VI_DOINGINACT;
2414 	vp->v_iflag &= ~VI_OWEINACT;
2415 	VI_UNLOCK(vp);
2416 	VOP_INACTIVE(vp, td);
2417 	VI_LOCK(vp);
2418 	VNASSERT(vp->v_iflag & VI_DOINGINACT, vp,
2419 	    ("vinactive: lost VI_DOINGINACT"));
2420 	vp->v_iflag &= ~VI_DOINGINACT;
2421 }
2422 
2423 /*
2424  * Remove any vnodes in the vnode table belonging to mount point mp.
2425  *
2426  * If FORCECLOSE is not specified, there should not be any active ones,
2427  * return error if any are found (nb: this is a user error, not a
2428  * system error). If FORCECLOSE is specified, detach any active vnodes
2429  * that are found.
2430  *
2431  * If WRITECLOSE is set, only flush out regular file vnodes open for
2432  * writing.
2433  *
2434  * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped.
2435  *
2436  * `rootrefs' specifies the base reference count for the root vnode
2437  * of this filesystem. The root vnode is considered busy if its
2438  * v_usecount exceeds this value. On a successful return, vflush(, td)
2439  * will call vrele() on the root vnode exactly rootrefs times.
2440  * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must
2441  * be zero.
2442  */
2443 #ifdef DIAGNOSTIC
2444 static int busyprt = 0;		/* print out busy vnodes */
2445 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes");
2446 #endif
2447 
2448 int
2449 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td)
2450 {
2451 	struct vnode *vp, *mvp, *rootvp = NULL;
2452 	struct vattr vattr;
2453 	int busy = 0, error;
2454 
2455 	CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp,
2456 	    rootrefs, flags);
2457 	if (rootrefs > 0) {
2458 		KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0,
2459 		    ("vflush: bad args"));
2460 		/*
2461 		 * Get the filesystem root vnode. We can vput() it
2462 		 * immediately, since with rootrefs > 0, it won't go away.
2463 		 */
2464 		if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) {
2465 			CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d",
2466 			    __func__, error);
2467 			return (error);
2468 		}
2469 		vput(rootvp);
2470 	}
2471 	MNT_ILOCK(mp);
2472 loop:
2473 	MNT_VNODE_FOREACH(vp, mp, mvp) {
2474 		VI_LOCK(vp);
2475 		vholdl(vp);
2476 		MNT_IUNLOCK(mp);
2477 		error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE);
2478 		if (error) {
2479 			vdrop(vp);
2480 			MNT_ILOCK(mp);
2481 			MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp);
2482 			goto loop;
2483 		}
2484 		/*
2485 		 * Skip over a vnodes marked VV_SYSTEM.
2486 		 */
2487 		if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) {
2488 			VOP_UNLOCK(vp, 0);
2489 			vdrop(vp);
2490 			MNT_ILOCK(mp);
2491 			continue;
2492 		}
2493 		/*
2494 		 * If WRITECLOSE is set, flush out unlinked but still open
2495 		 * files (even if open only for reading) and regular file
2496 		 * vnodes open for writing.
2497 		 */
2498 		if (flags & WRITECLOSE) {
2499 			error = VOP_GETATTR(vp, &vattr, td->td_ucred);
2500 			VI_LOCK(vp);
2501 
2502 			if ((vp->v_type == VNON ||
2503 			    (error == 0 && vattr.va_nlink > 0)) &&
2504 			    (vp->v_writecount == 0 || vp->v_type != VREG)) {
2505 				VOP_UNLOCK(vp, 0);
2506 				vdropl(vp);
2507 				MNT_ILOCK(mp);
2508 				continue;
2509 			}
2510 		} else
2511 			VI_LOCK(vp);
2512 		/*
2513 		 * With v_usecount == 0, all we need to do is clear out the
2514 		 * vnode data structures and we are done.
2515 		 *
2516 		 * If FORCECLOSE is set, forcibly close the vnode.
2517 		 */
2518 		if (vp->v_usecount == 0 || (flags & FORCECLOSE)) {
2519 			VNASSERT(vp->v_usecount == 0 ||
2520 			    (vp->v_type != VCHR && vp->v_type != VBLK), vp,
2521 			    ("device VNODE %p is FORCECLOSED", vp));
2522 			vgonel(vp);
2523 		} else {
2524 			busy++;
2525 #ifdef DIAGNOSTIC
2526 			if (busyprt)
2527 				vprint("vflush: busy vnode", vp);
2528 #endif
2529 		}
2530 		VOP_UNLOCK(vp, 0);
2531 		vdropl(vp);
2532 		MNT_ILOCK(mp);
2533 	}
2534 	MNT_IUNLOCK(mp);
2535 	if (rootrefs > 0 && (flags & FORCECLOSE) == 0) {
2536 		/*
2537 		 * If just the root vnode is busy, and if its refcount
2538 		 * is equal to `rootrefs', then go ahead and kill it.
2539 		 */
2540 		VI_LOCK(rootvp);
2541 		KASSERT(busy > 0, ("vflush: not busy"));
2542 		VNASSERT(rootvp->v_usecount >= rootrefs, rootvp,
2543 		    ("vflush: usecount %d < rootrefs %d",
2544 		     rootvp->v_usecount, rootrefs));
2545 		if (busy == 1 && rootvp->v_usecount == rootrefs) {
2546 			VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK);
2547 			vgone(rootvp);
2548 			VOP_UNLOCK(rootvp, 0);
2549 			busy = 0;
2550 		} else
2551 			VI_UNLOCK(rootvp);
2552 	}
2553 	if (busy) {
2554 		CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__,
2555 		    busy);
2556 		return (EBUSY);
2557 	}
2558 	for (; rootrefs > 0; rootrefs--)
2559 		vrele(rootvp);
2560 	return (0);
2561 }
2562 
2563 /*
2564  * Recycle an unused vnode to the front of the free list.
2565  */
2566 int
2567 vrecycle(struct vnode *vp, struct thread *td)
2568 {
2569 	int recycled;
2570 
2571 	ASSERT_VOP_ELOCKED(vp, "vrecycle");
2572 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2573 	recycled = 0;
2574 	VI_LOCK(vp);
2575 	if (vp->v_usecount == 0) {
2576 		recycled = 1;
2577 		vgonel(vp);
2578 	}
2579 	VI_UNLOCK(vp);
2580 	return (recycled);
2581 }
2582 
2583 /*
2584  * Eliminate all activity associated with a vnode
2585  * in preparation for reuse.
2586  */
2587 void
2588 vgone(struct vnode *vp)
2589 {
2590 	VI_LOCK(vp);
2591 	vgonel(vp);
2592 	VI_UNLOCK(vp);
2593 }
2594 
2595 /*
2596  * vgone, with the vp interlock held.
2597  */
2598 void
2599 vgonel(struct vnode *vp)
2600 {
2601 	struct thread *td;
2602 	int oweinact;
2603 	int active;
2604 	struct mount *mp;
2605 
2606 	ASSERT_VOP_ELOCKED(vp, "vgonel");
2607 	ASSERT_VI_LOCKED(vp, "vgonel");
2608 	VNASSERT(vp->v_holdcnt, vp,
2609 	    ("vgonel: vp %p has no reference.", vp));
2610 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2611 	td = curthread;
2612 
2613 	/*
2614 	 * Don't vgonel if we're already doomed.
2615 	 */
2616 	if (vp->v_iflag & VI_DOOMED)
2617 		return;
2618 	vp->v_iflag |= VI_DOOMED;
2619 	/*
2620 	 * Check to see if the vnode is in use.  If so, we have to call
2621 	 * VOP_CLOSE() and VOP_INACTIVE().
2622 	 */
2623 	active = vp->v_usecount;
2624 	oweinact = (vp->v_iflag & VI_OWEINACT);
2625 	VI_UNLOCK(vp);
2626 	/*
2627 	 * Clean out any buffers associated with the vnode.
2628 	 * If the flush fails, just toss the buffers.
2629 	 */
2630 	mp = NULL;
2631 	if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd))
2632 		(void) vn_start_secondary_write(vp, &mp, V_WAIT);
2633 	if (vinvalbuf(vp, V_SAVE, 0, 0) != 0)
2634 		vinvalbuf(vp, 0, 0, 0);
2635 
2636 	/*
2637 	 * If purging an active vnode, it must be closed and
2638 	 * deactivated before being reclaimed.
2639 	 */
2640 	if (active)
2641 		VOP_CLOSE(vp, FNONBLOCK, NOCRED, td);
2642 	if (oweinact || active) {
2643 		VI_LOCK(vp);
2644 		if ((vp->v_iflag & VI_DOINGINACT) == 0)
2645 			vinactive(vp, td);
2646 		VI_UNLOCK(vp);
2647 	}
2648 	/*
2649 	 * Reclaim the vnode.
2650 	 */
2651 	if (VOP_RECLAIM(vp, td))
2652 		panic("vgone: cannot reclaim");
2653 	if (mp != NULL)
2654 		vn_finished_secondary_write(mp);
2655 	VNASSERT(vp->v_object == NULL, vp,
2656 	    ("vop_reclaim left v_object vp=%p, tag=%s", vp, vp->v_tag));
2657 	/*
2658 	 * Clear the advisory locks and wake up waiting threads.
2659 	 */
2660 	(void)VOP_ADVLOCKPURGE(vp);
2661 	/*
2662 	 * Delete from old mount point vnode list.
2663 	 */
2664 	delmntque(vp);
2665 	cache_purge(vp);
2666 	/*
2667 	 * Done with purge, reset to the standard lock and invalidate
2668 	 * the vnode.
2669 	 */
2670 	VI_LOCK(vp);
2671 	vp->v_vnlock = &vp->v_lock;
2672 	vp->v_op = &dead_vnodeops;
2673 	vp->v_tag = "none";
2674 	vp->v_type = VBAD;
2675 }
2676 
2677 /*
2678  * Calculate the total number of references to a special device.
2679  */
2680 int
2681 vcount(struct vnode *vp)
2682 {
2683 	int count;
2684 
2685 	dev_lock();
2686 	count = vp->v_rdev->si_usecount;
2687 	dev_unlock();
2688 	return (count);
2689 }
2690 
2691 /*
2692  * Same as above, but using the struct cdev *as argument
2693  */
2694 int
2695 count_dev(struct cdev *dev)
2696 {
2697 	int count;
2698 
2699 	dev_lock();
2700 	count = dev->si_usecount;
2701 	dev_unlock();
2702 	return(count);
2703 }
2704 
2705 /*
2706  * Print out a description of a vnode.
2707  */
2708 static char *typename[] =
2709 {"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD",
2710  "VMARKER"};
2711 
2712 void
2713 vn_printf(struct vnode *vp, const char *fmt, ...)
2714 {
2715 	va_list ap;
2716 	char buf[256], buf2[16];
2717 	u_long flags;
2718 
2719 	va_start(ap, fmt);
2720 	vprintf(fmt, ap);
2721 	va_end(ap);
2722 	printf("%p: ", (void *)vp);
2723 	printf("tag %s, type %s\n", vp->v_tag, typename[vp->v_type]);
2724 	printf("    usecount %d, writecount %d, refcount %d mountedhere %p\n",
2725 	    vp->v_usecount, vp->v_writecount, vp->v_holdcnt, vp->v_mountedhere);
2726 	buf[0] = '\0';
2727 	buf[1] = '\0';
2728 	if (vp->v_vflag & VV_ROOT)
2729 		strlcat(buf, "|VV_ROOT", sizeof(buf));
2730 	if (vp->v_vflag & VV_ISTTY)
2731 		strlcat(buf, "|VV_ISTTY", sizeof(buf));
2732 	if (vp->v_vflag & VV_NOSYNC)
2733 		strlcat(buf, "|VV_NOSYNC", sizeof(buf));
2734 	if (vp->v_vflag & VV_CACHEDLABEL)
2735 		strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf));
2736 	if (vp->v_vflag & VV_TEXT)
2737 		strlcat(buf, "|VV_TEXT", sizeof(buf));
2738 	if (vp->v_vflag & VV_COPYONWRITE)
2739 		strlcat(buf, "|VV_COPYONWRITE", sizeof(buf));
2740 	if (vp->v_vflag & VV_SYSTEM)
2741 		strlcat(buf, "|VV_SYSTEM", sizeof(buf));
2742 	if (vp->v_vflag & VV_PROCDEP)
2743 		strlcat(buf, "|VV_PROCDEP", sizeof(buf));
2744 	if (vp->v_vflag & VV_NOKNOTE)
2745 		strlcat(buf, "|VV_NOKNOTE", sizeof(buf));
2746 	if (vp->v_vflag & VV_DELETED)
2747 		strlcat(buf, "|VV_DELETED", sizeof(buf));
2748 	if (vp->v_vflag & VV_MD)
2749 		strlcat(buf, "|VV_MD", sizeof(buf));
2750 	flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC |
2751 	    VV_CACHEDLABEL | VV_TEXT | VV_COPYONWRITE | VV_SYSTEM | VV_PROCDEP |
2752 	    VV_NOKNOTE | VV_DELETED | VV_MD);
2753 	if (flags != 0) {
2754 		snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags);
2755 		strlcat(buf, buf2, sizeof(buf));
2756 	}
2757 	if (vp->v_iflag & VI_MOUNT)
2758 		strlcat(buf, "|VI_MOUNT", sizeof(buf));
2759 	if (vp->v_iflag & VI_AGE)
2760 		strlcat(buf, "|VI_AGE", sizeof(buf));
2761 	if (vp->v_iflag & VI_DOOMED)
2762 		strlcat(buf, "|VI_DOOMED", sizeof(buf));
2763 	if (vp->v_iflag & VI_FREE)
2764 		strlcat(buf, "|VI_FREE", sizeof(buf));
2765 	if (vp->v_iflag & VI_DOINGINACT)
2766 		strlcat(buf, "|VI_DOINGINACT", sizeof(buf));
2767 	if (vp->v_iflag & VI_OWEINACT)
2768 		strlcat(buf, "|VI_OWEINACT", sizeof(buf));
2769 	flags = vp->v_iflag & ~(VI_MOUNT | VI_AGE | VI_DOOMED | VI_FREE |
2770 	    VI_DOINGINACT | VI_OWEINACT);
2771 	if (flags != 0) {
2772 		snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags);
2773 		strlcat(buf, buf2, sizeof(buf));
2774 	}
2775 	printf("    flags (%s)\n", buf + 1);
2776 	if (mtx_owned(VI_MTX(vp)))
2777 		printf(" VI_LOCKed");
2778 	if (vp->v_object != NULL)
2779 		printf("    v_object %p ref %d pages %d\n",
2780 		    vp->v_object, vp->v_object->ref_count,
2781 		    vp->v_object->resident_page_count);
2782 	printf("    ");
2783 	lockmgr_printinfo(vp->v_vnlock);
2784 	if (vp->v_data != NULL)
2785 		VOP_PRINT(vp);
2786 }
2787 
2788 #ifdef DDB
2789 /*
2790  * List all of the locked vnodes in the system.
2791  * Called when debugging the kernel.
2792  */
2793 DB_SHOW_COMMAND(lockedvnods, lockedvnodes)
2794 {
2795 	struct mount *mp, *nmp;
2796 	struct vnode *vp;
2797 
2798 	/*
2799 	 * Note: because this is DDB, we can't obey the locking semantics
2800 	 * for these structures, which means we could catch an inconsistent
2801 	 * state and dereference a nasty pointer.  Not much to be done
2802 	 * about that.
2803 	 */
2804 	db_printf("Locked vnodes\n");
2805 	for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
2806 		nmp = TAILQ_NEXT(mp, mnt_list);
2807 		TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
2808 			if (vp->v_type != VMARKER &&
2809 			    VOP_ISLOCKED(vp))
2810 				vprint("", vp);
2811 		}
2812 		nmp = TAILQ_NEXT(mp, mnt_list);
2813 	}
2814 }
2815 
2816 /*
2817  * Show details about the given vnode.
2818  */
2819 DB_SHOW_COMMAND(vnode, db_show_vnode)
2820 {
2821 	struct vnode *vp;
2822 
2823 	if (!have_addr)
2824 		return;
2825 	vp = (struct vnode *)addr;
2826 	vn_printf(vp, "vnode ");
2827 }
2828 
2829 /*
2830  * Show details about the given mount point.
2831  */
2832 DB_SHOW_COMMAND(mount, db_show_mount)
2833 {
2834 	struct mount *mp;
2835 	struct vfsopt *opt;
2836 	struct statfs *sp;
2837 	struct vnode *vp;
2838 	char buf[512];
2839 	u_int flags;
2840 
2841 	if (!have_addr) {
2842 		/* No address given, print short info about all mount points. */
2843 		TAILQ_FOREACH(mp, &mountlist, mnt_list) {
2844 			db_printf("%p %s on %s (%s)\n", mp,
2845 			    mp->mnt_stat.f_mntfromname,
2846 			    mp->mnt_stat.f_mntonname,
2847 			    mp->mnt_stat.f_fstypename);
2848 			if (db_pager_quit)
2849 				break;
2850 		}
2851 		db_printf("\nMore info: show mount <addr>\n");
2852 		return;
2853 	}
2854 
2855 	mp = (struct mount *)addr;
2856 	db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname,
2857 	    mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename);
2858 
2859 	buf[0] = '\0';
2860 	flags = mp->mnt_flag;
2861 #define	MNT_FLAG(flag)	do {						\
2862 	if (flags & (flag)) {						\
2863 		if (buf[0] != '\0')					\
2864 			strlcat(buf, ", ", sizeof(buf));		\
2865 		strlcat(buf, (#flag) + 4, sizeof(buf));			\
2866 		flags &= ~(flag);					\
2867 	}								\
2868 } while (0)
2869 	MNT_FLAG(MNT_RDONLY);
2870 	MNT_FLAG(MNT_SYNCHRONOUS);
2871 	MNT_FLAG(MNT_NOEXEC);
2872 	MNT_FLAG(MNT_NOSUID);
2873 	MNT_FLAG(MNT_UNION);
2874 	MNT_FLAG(MNT_ASYNC);
2875 	MNT_FLAG(MNT_SUIDDIR);
2876 	MNT_FLAG(MNT_SOFTDEP);
2877 	MNT_FLAG(MNT_SUJ);
2878 	MNT_FLAG(MNT_NOSYMFOLLOW);
2879 	MNT_FLAG(MNT_GJOURNAL);
2880 	MNT_FLAG(MNT_MULTILABEL);
2881 	MNT_FLAG(MNT_ACLS);
2882 	MNT_FLAG(MNT_NOATIME);
2883 	MNT_FLAG(MNT_NOCLUSTERR);
2884 	MNT_FLAG(MNT_NOCLUSTERW);
2885 	MNT_FLAG(MNT_NFS4ACLS);
2886 	MNT_FLAG(MNT_EXRDONLY);
2887 	MNT_FLAG(MNT_EXPORTED);
2888 	MNT_FLAG(MNT_DEFEXPORTED);
2889 	MNT_FLAG(MNT_EXPORTANON);
2890 	MNT_FLAG(MNT_EXKERB);
2891 	MNT_FLAG(MNT_EXPUBLIC);
2892 	MNT_FLAG(MNT_LOCAL);
2893 	MNT_FLAG(MNT_QUOTA);
2894 	MNT_FLAG(MNT_ROOTFS);
2895 	MNT_FLAG(MNT_USER);
2896 	MNT_FLAG(MNT_IGNORE);
2897 	MNT_FLAG(MNT_UPDATE);
2898 	MNT_FLAG(MNT_DELEXPORT);
2899 	MNT_FLAG(MNT_RELOAD);
2900 	MNT_FLAG(MNT_FORCE);
2901 	MNT_FLAG(MNT_SNAPSHOT);
2902 	MNT_FLAG(MNT_BYFSID);
2903 #undef MNT_FLAG
2904 	if (flags != 0) {
2905 		if (buf[0] != '\0')
2906 			strlcat(buf, ", ", sizeof(buf));
2907 		snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
2908 		    "0x%08x", flags);
2909 	}
2910 	db_printf("    mnt_flag = %s\n", buf);
2911 
2912 	buf[0] = '\0';
2913 	flags = mp->mnt_kern_flag;
2914 #define	MNT_KERN_FLAG(flag)	do {					\
2915 	if (flags & (flag)) {						\
2916 		if (buf[0] != '\0')					\
2917 			strlcat(buf, ", ", sizeof(buf));		\
2918 		strlcat(buf, (#flag) + 5, sizeof(buf));			\
2919 		flags &= ~(flag);					\
2920 	}								\
2921 } while (0)
2922 	MNT_KERN_FLAG(MNTK_UNMOUNTF);
2923 	MNT_KERN_FLAG(MNTK_ASYNC);
2924 	MNT_KERN_FLAG(MNTK_SOFTDEP);
2925 	MNT_KERN_FLAG(MNTK_NOINSMNTQ);
2926 	MNT_KERN_FLAG(MNTK_DRAINING);
2927 	MNT_KERN_FLAG(MNTK_REFEXPIRE);
2928 	MNT_KERN_FLAG(MNTK_EXTENDED_SHARED);
2929 	MNT_KERN_FLAG(MNTK_SHARED_WRITES);
2930 	MNT_KERN_FLAG(MNTK_UNMOUNT);
2931 	MNT_KERN_FLAG(MNTK_MWAIT);
2932 	MNT_KERN_FLAG(MNTK_SUSPEND);
2933 	MNT_KERN_FLAG(MNTK_SUSPEND2);
2934 	MNT_KERN_FLAG(MNTK_SUSPENDED);
2935 	MNT_KERN_FLAG(MNTK_MPSAFE);
2936 	MNT_KERN_FLAG(MNTK_LOOKUP_SHARED);
2937 	MNT_KERN_FLAG(MNTK_NOKNOTE);
2938 #undef MNT_KERN_FLAG
2939 	if (flags != 0) {
2940 		if (buf[0] != '\0')
2941 			strlcat(buf, ", ", sizeof(buf));
2942 		snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
2943 		    "0x%08x", flags);
2944 	}
2945 	db_printf("    mnt_kern_flag = %s\n", buf);
2946 
2947 	db_printf("    mnt_opt = ");
2948 	opt = TAILQ_FIRST(mp->mnt_opt);
2949 	if (opt != NULL) {
2950 		db_printf("%s", opt->name);
2951 		opt = TAILQ_NEXT(opt, link);
2952 		while (opt != NULL) {
2953 			db_printf(", %s", opt->name);
2954 			opt = TAILQ_NEXT(opt, link);
2955 		}
2956 	}
2957 	db_printf("\n");
2958 
2959 	sp = &mp->mnt_stat;
2960 	db_printf("    mnt_stat = { version=%u type=%u flags=0x%016jx "
2961 	    "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju "
2962 	    "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju "
2963 	    "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n",
2964 	    (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags,
2965 	    (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize,
2966 	    (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree,
2967 	    (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files,
2968 	    (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites,
2969 	    (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads,
2970 	    (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax,
2971 	    (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]);
2972 
2973 	db_printf("    mnt_cred = { uid=%u ruid=%u",
2974 	    (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid);
2975 	if (jailed(mp->mnt_cred))
2976 		db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id);
2977 	db_printf(" }\n");
2978 	db_printf("    mnt_ref = %d\n", mp->mnt_ref);
2979 	db_printf("    mnt_gen = %d\n", mp->mnt_gen);
2980 	db_printf("    mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize);
2981 	db_printf("    mnt_writeopcount = %d\n", mp->mnt_writeopcount);
2982 	db_printf("    mnt_noasync = %u\n", mp->mnt_noasync);
2983 	db_printf("    mnt_maxsymlinklen = %d\n", mp->mnt_maxsymlinklen);
2984 	db_printf("    mnt_iosize_max = %d\n", mp->mnt_iosize_max);
2985 	db_printf("    mnt_hashseed = %u\n", mp->mnt_hashseed);
2986 	db_printf("    mnt_secondary_writes = %d\n", mp->mnt_secondary_writes);
2987 	db_printf("    mnt_secondary_accwrites = %d\n",
2988 	    mp->mnt_secondary_accwrites);
2989 	db_printf("    mnt_gjprovider = %s\n",
2990 	    mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL");
2991 	db_printf("\n");
2992 
2993 	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
2994 		if (vp->v_type != VMARKER) {
2995 			vn_printf(vp, "vnode ");
2996 			if (db_pager_quit)
2997 				break;
2998 		}
2999 	}
3000 }
3001 #endif	/* DDB */
3002 
3003 /*
3004  * Fill in a struct xvfsconf based on a struct vfsconf.
3005  */
3006 static void
3007 vfsconf2x(struct vfsconf *vfsp, struct xvfsconf *xvfsp)
3008 {
3009 
3010 	strcpy(xvfsp->vfc_name, vfsp->vfc_name);
3011 	xvfsp->vfc_typenum = vfsp->vfc_typenum;
3012 	xvfsp->vfc_refcount = vfsp->vfc_refcount;
3013 	xvfsp->vfc_flags = vfsp->vfc_flags;
3014 	/*
3015 	 * These are unused in userland, we keep them
3016 	 * to not break binary compatibility.
3017 	 */
3018 	xvfsp->vfc_vfsops = NULL;
3019 	xvfsp->vfc_next = NULL;
3020 }
3021 
3022 /*
3023  * Top level filesystem related information gathering.
3024  */
3025 static int
3026 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS)
3027 {
3028 	struct vfsconf *vfsp;
3029 	struct xvfsconf xvfsp;
3030 	int error;
3031 
3032 	error = 0;
3033 	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
3034 		bzero(&xvfsp, sizeof(xvfsp));
3035 		vfsconf2x(vfsp, &xvfsp);
3036 		error = SYSCTL_OUT(req, &xvfsp, sizeof xvfsp);
3037 		if (error)
3038 			break;
3039 	}
3040 	return (error);
3041 }
3042 
3043 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD,
3044     NULL, 0, sysctl_vfs_conflist,
3045     "S,xvfsconf", "List of all configured filesystems");
3046 
3047 #ifndef BURN_BRIDGES
3048 static int	sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS);
3049 
3050 static int
3051 vfs_sysctl(SYSCTL_HANDLER_ARGS)
3052 {
3053 	int *name = (int *)arg1 - 1;	/* XXX */
3054 	u_int namelen = arg2 + 1;	/* XXX */
3055 	struct vfsconf *vfsp;
3056 	struct xvfsconf xvfsp;
3057 
3058 	log(LOG_WARNING, "userland calling deprecated sysctl, "
3059 	    "please rebuild world\n");
3060 
3061 #if 1 || defined(COMPAT_PRELITE2)
3062 	/* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */
3063 	if (namelen == 1)
3064 		return (sysctl_ovfs_conf(oidp, arg1, arg2, req));
3065 #endif
3066 
3067 	switch (name[1]) {
3068 	case VFS_MAXTYPENUM:
3069 		if (namelen != 2)
3070 			return (ENOTDIR);
3071 		return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int)));
3072 	case VFS_CONF:
3073 		if (namelen != 3)
3074 			return (ENOTDIR);	/* overloaded */
3075 		TAILQ_FOREACH(vfsp, &vfsconf, vfc_list)
3076 			if (vfsp->vfc_typenum == name[2])
3077 				break;
3078 		if (vfsp == NULL)
3079 			return (EOPNOTSUPP);
3080 		bzero(&xvfsp, sizeof(xvfsp));
3081 		vfsconf2x(vfsp, &xvfsp);
3082 		return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
3083 	}
3084 	return (EOPNOTSUPP);
3085 }
3086 
3087 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP,
3088     vfs_sysctl, "Generic filesystem");
3089 
3090 #if 1 || defined(COMPAT_PRELITE2)
3091 
3092 static int
3093 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS)
3094 {
3095 	int error;
3096 	struct vfsconf *vfsp;
3097 	struct ovfsconf ovfs;
3098 
3099 	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
3100 		bzero(&ovfs, sizeof(ovfs));
3101 		ovfs.vfc_vfsops = vfsp->vfc_vfsops;	/* XXX used as flag */
3102 		strcpy(ovfs.vfc_name, vfsp->vfc_name);
3103 		ovfs.vfc_index = vfsp->vfc_typenum;
3104 		ovfs.vfc_refcount = vfsp->vfc_refcount;
3105 		ovfs.vfc_flags = vfsp->vfc_flags;
3106 		error = SYSCTL_OUT(req, &ovfs, sizeof ovfs);
3107 		if (error)
3108 			return error;
3109 	}
3110 	return 0;
3111 }
3112 
3113 #endif /* 1 || COMPAT_PRELITE2 */
3114 #endif /* !BURN_BRIDGES */
3115 
3116 #define KINFO_VNODESLOP		10
3117 #ifdef notyet
3118 /*
3119  * Dump vnode list (via sysctl).
3120  */
3121 /* ARGSUSED */
3122 static int
3123 sysctl_vnode(SYSCTL_HANDLER_ARGS)
3124 {
3125 	struct xvnode *xvn;
3126 	struct mount *mp;
3127 	struct vnode *vp;
3128 	int error, len, n;
3129 
3130 	/*
3131 	 * Stale numvnodes access is not fatal here.
3132 	 */
3133 	req->lock = 0;
3134 	len = (numvnodes + KINFO_VNODESLOP) * sizeof *xvn;
3135 	if (!req->oldptr)
3136 		/* Make an estimate */
3137 		return (SYSCTL_OUT(req, 0, len));
3138 
3139 	error = sysctl_wire_old_buffer(req, 0);
3140 	if (error != 0)
3141 		return (error);
3142 	xvn = malloc(len, M_TEMP, M_ZERO | M_WAITOK);
3143 	n = 0;
3144 	mtx_lock(&mountlist_mtx);
3145 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
3146 		if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK))
3147 			continue;
3148 		MNT_ILOCK(mp);
3149 		TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
3150 			if (n == len)
3151 				break;
3152 			vref(vp);
3153 			xvn[n].xv_size = sizeof *xvn;
3154 			xvn[n].xv_vnode = vp;
3155 			xvn[n].xv_id = 0;	/* XXX compat */
3156 #define XV_COPY(field) xvn[n].xv_##field = vp->v_##field
3157 			XV_COPY(usecount);
3158 			XV_COPY(writecount);
3159 			XV_COPY(holdcnt);
3160 			XV_COPY(mount);
3161 			XV_COPY(numoutput);
3162 			XV_COPY(type);
3163 #undef XV_COPY
3164 			xvn[n].xv_flag = vp->v_vflag;
3165 
3166 			switch (vp->v_type) {
3167 			case VREG:
3168 			case VDIR:
3169 			case VLNK:
3170 				break;
3171 			case VBLK:
3172 			case VCHR:
3173 				if (vp->v_rdev == NULL) {
3174 					vrele(vp);
3175 					continue;
3176 				}
3177 				xvn[n].xv_dev = dev2udev(vp->v_rdev);
3178 				break;
3179 			case VSOCK:
3180 				xvn[n].xv_socket = vp->v_socket;
3181 				break;
3182 			case VFIFO:
3183 				xvn[n].xv_fifo = vp->v_fifoinfo;
3184 				break;
3185 			case VNON:
3186 			case VBAD:
3187 			default:
3188 				/* shouldn't happen? */
3189 				vrele(vp);
3190 				continue;
3191 			}
3192 			vrele(vp);
3193 			++n;
3194 		}
3195 		MNT_IUNLOCK(mp);
3196 		mtx_lock(&mountlist_mtx);
3197 		vfs_unbusy(mp);
3198 		if (n == len)
3199 			break;
3200 	}
3201 	mtx_unlock(&mountlist_mtx);
3202 
3203 	error = SYSCTL_OUT(req, xvn, n * sizeof *xvn);
3204 	free(xvn, M_TEMP);
3205 	return (error);
3206 }
3207 
3208 SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE|CTLFLAG_RD,
3209     0, 0, sysctl_vnode, "S,xvnode", "");
3210 #endif
3211 
3212 /*
3213  * Unmount all filesystems. The list is traversed in reverse order
3214  * of mounting to avoid dependencies.
3215  */
3216 void
3217 vfs_unmountall(void)
3218 {
3219 	struct mount *mp;
3220 	struct thread *td;
3221 	int error;
3222 
3223 	KASSERT(curthread != NULL, ("vfs_unmountall: NULL curthread"));
3224 	CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__);
3225 	td = curthread;
3226 
3227 	/*
3228 	 * Since this only runs when rebooting, it is not interlocked.
3229 	 */
3230 	while(!TAILQ_EMPTY(&mountlist)) {
3231 		mp = TAILQ_LAST(&mountlist, mntlist);
3232 		error = dounmount(mp, MNT_FORCE, td);
3233 		if (error) {
3234 			TAILQ_REMOVE(&mountlist, mp, mnt_list);
3235 			/*
3236 			 * XXX: Due to the way in which we mount the root
3237 			 * file system off of devfs, devfs will generate a
3238 			 * "busy" warning when we try to unmount it before
3239 			 * the root.  Don't print a warning as a result in
3240 			 * order to avoid false positive errors that may
3241 			 * cause needless upset.
3242 			 */
3243 			if (strcmp(mp->mnt_vfc->vfc_name, "devfs") != 0) {
3244 				printf("unmount of %s failed (",
3245 				    mp->mnt_stat.f_mntonname);
3246 				if (error == EBUSY)
3247 					printf("BUSY)\n");
3248 				else
3249 					printf("%d)\n", error);
3250 			}
3251 		} else {
3252 			/* The unmount has removed mp from the mountlist */
3253 		}
3254 	}
3255 }
3256 
3257 /*
3258  * perform msync on all vnodes under a mount point
3259  * the mount point must be locked.
3260  */
3261 void
3262 vfs_msync(struct mount *mp, int flags)
3263 {
3264 	struct vnode *vp, *mvp;
3265 	struct vm_object *obj;
3266 
3267 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
3268 	MNT_ILOCK(mp);
3269 	MNT_VNODE_FOREACH(vp, mp, mvp) {
3270 		VI_LOCK(vp);
3271 		obj = vp->v_object;
3272 		if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 &&
3273 		    (flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) {
3274 			MNT_IUNLOCK(mp);
3275 			if (!vget(vp,
3276 			    LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK,
3277 			    curthread)) {
3278 				if (vp->v_vflag & VV_NOSYNC) {	/* unlinked */
3279 					vput(vp);
3280 					MNT_ILOCK(mp);
3281 					continue;
3282 				}
3283 
3284 				obj = vp->v_object;
3285 				if (obj != NULL) {
3286 					VM_OBJECT_LOCK(obj);
3287 					vm_object_page_clean(obj, 0, 0,
3288 					    flags == MNT_WAIT ?
3289 					    OBJPC_SYNC : OBJPC_NOSYNC);
3290 					VM_OBJECT_UNLOCK(obj);
3291 				}
3292 				vput(vp);
3293 			}
3294 			MNT_ILOCK(mp);
3295 		} else
3296 			VI_UNLOCK(vp);
3297 	}
3298 	MNT_IUNLOCK(mp);
3299 }
3300 
3301 /*
3302  * Mark a vnode as free, putting it up for recycling.
3303  */
3304 static void
3305 vfree(struct vnode *vp)
3306 {
3307 
3308 	ASSERT_VI_LOCKED(vp, "vfree");
3309 	mtx_lock(&vnode_free_list_mtx);
3310 	VNASSERT(vp->v_op != NULL, vp, ("vfree: vnode already reclaimed."));
3311 	VNASSERT((vp->v_iflag & VI_FREE) == 0, vp, ("vnode already free"));
3312 	VNASSERT(VSHOULDFREE(vp), vp, ("vfree: freeing when we shouldn't"));
3313 	VNASSERT((vp->v_iflag & VI_DOOMED) == 0, vp,
3314 	    ("vfree: Freeing doomed vnode"));
3315 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3316 	if (vp->v_iflag & VI_AGE) {
3317 		TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_freelist);
3318 	} else {
3319 		TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
3320 	}
3321 	freevnodes++;
3322 	vp->v_iflag &= ~VI_AGE;
3323 	vp->v_iflag |= VI_FREE;
3324 	mtx_unlock(&vnode_free_list_mtx);
3325 }
3326 
3327 /*
3328  * Opposite of vfree() - mark a vnode as in use.
3329  */
3330 static void
3331 vbusy(struct vnode *vp)
3332 {
3333 	ASSERT_VI_LOCKED(vp, "vbusy");
3334 	VNASSERT((vp->v_iflag & VI_FREE) != 0, vp, ("vnode not free"));
3335 	VNASSERT(vp->v_op != NULL, vp, ("vbusy: vnode already reclaimed."));
3336 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
3337 
3338 	mtx_lock(&vnode_free_list_mtx);
3339 	TAILQ_REMOVE(&vnode_free_list, vp, v_freelist);
3340 	freevnodes--;
3341 	vp->v_iflag &= ~(VI_FREE|VI_AGE);
3342 	mtx_unlock(&vnode_free_list_mtx);
3343 }
3344 
3345 static void
3346 destroy_vpollinfo(struct vpollinfo *vi)
3347 {
3348 	seldrain(&vi->vpi_selinfo);
3349 	knlist_destroy(&vi->vpi_selinfo.si_note);
3350 	mtx_destroy(&vi->vpi_lock);
3351 	uma_zfree(vnodepoll_zone, vi);
3352 }
3353 
3354 /*
3355  * Initalize per-vnode helper structure to hold poll-related state.
3356  */
3357 void
3358 v_addpollinfo(struct vnode *vp)
3359 {
3360 	struct vpollinfo *vi;
3361 
3362 	if (vp->v_pollinfo != NULL)
3363 		return;
3364 	vi = uma_zalloc(vnodepoll_zone, M_WAITOK);
3365 	mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF);
3366 	knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock,
3367 	    vfs_knlunlock, vfs_knl_assert_locked, vfs_knl_assert_unlocked);
3368 	VI_LOCK(vp);
3369 	if (vp->v_pollinfo != NULL) {
3370 		VI_UNLOCK(vp);
3371 		destroy_vpollinfo(vi);
3372 		return;
3373 	}
3374 	vp->v_pollinfo = vi;
3375 	VI_UNLOCK(vp);
3376 }
3377 
3378 /*
3379  * Record a process's interest in events which might happen to
3380  * a vnode.  Because poll uses the historic select-style interface
3381  * internally, this routine serves as both the ``check for any
3382  * pending events'' and the ``record my interest in future events''
3383  * functions.  (These are done together, while the lock is held,
3384  * to avoid race conditions.)
3385  */
3386 int
3387 vn_pollrecord(struct vnode *vp, struct thread *td, int events)
3388 {
3389 
3390 	v_addpollinfo(vp);
3391 	mtx_lock(&vp->v_pollinfo->vpi_lock);
3392 	if (vp->v_pollinfo->vpi_revents & events) {
3393 		/*
3394 		 * This leaves events we are not interested
3395 		 * in available for the other process which
3396 		 * which presumably had requested them
3397 		 * (otherwise they would never have been
3398 		 * recorded).
3399 		 */
3400 		events &= vp->v_pollinfo->vpi_revents;
3401 		vp->v_pollinfo->vpi_revents &= ~events;
3402 
3403 		mtx_unlock(&vp->v_pollinfo->vpi_lock);
3404 		return (events);
3405 	}
3406 	vp->v_pollinfo->vpi_events |= events;
3407 	selrecord(td, &vp->v_pollinfo->vpi_selinfo);
3408 	mtx_unlock(&vp->v_pollinfo->vpi_lock);
3409 	return (0);
3410 }
3411 
3412 /*
3413  * Routine to create and manage a filesystem syncer vnode.
3414  */
3415 #define sync_close ((int (*)(struct  vop_close_args *))nullop)
3416 static int	sync_fsync(struct  vop_fsync_args *);
3417 static int	sync_inactive(struct  vop_inactive_args *);
3418 static int	sync_reclaim(struct  vop_reclaim_args *);
3419 
3420 static struct vop_vector sync_vnodeops = {
3421 	.vop_bypass =	VOP_EOPNOTSUPP,
3422 	.vop_close =	sync_close,		/* close */
3423 	.vop_fsync =	sync_fsync,		/* fsync */
3424 	.vop_inactive =	sync_inactive,	/* inactive */
3425 	.vop_reclaim =	sync_reclaim,	/* reclaim */
3426 	.vop_lock1 =	vop_stdlock,	/* lock */
3427 	.vop_unlock =	vop_stdunlock,	/* unlock */
3428 	.vop_islocked =	vop_stdislocked,	/* islocked */
3429 };
3430 
3431 /*
3432  * Create a new filesystem syncer vnode for the specified mount point.
3433  */
3434 void
3435 vfs_allocate_syncvnode(struct mount *mp)
3436 {
3437 	struct vnode *vp;
3438 	struct bufobj *bo;
3439 	static long start, incr, next;
3440 	int error;
3441 
3442 	/* Allocate a new vnode */
3443 	error = getnewvnode("syncer", mp, &sync_vnodeops, &vp);
3444 	if (error != 0)
3445 		panic("vfs_allocate_syncvnode: getnewvnode() failed");
3446 	vp->v_type = VNON;
3447 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
3448 	vp->v_vflag |= VV_FORCEINSMQ;
3449 	error = insmntque(vp, mp);
3450 	if (error != 0)
3451 		panic("vfs_allocate_syncvnode: insmntque() failed");
3452 	vp->v_vflag &= ~VV_FORCEINSMQ;
3453 	VOP_UNLOCK(vp, 0);
3454 	/*
3455 	 * Place the vnode onto the syncer worklist. We attempt to
3456 	 * scatter them about on the list so that they will go off
3457 	 * at evenly distributed times even if all the filesystems
3458 	 * are mounted at once.
3459 	 */
3460 	next += incr;
3461 	if (next == 0 || next > syncer_maxdelay) {
3462 		start /= 2;
3463 		incr /= 2;
3464 		if (start == 0) {
3465 			start = syncer_maxdelay / 2;
3466 			incr = syncer_maxdelay;
3467 		}
3468 		next = start;
3469 	}
3470 	bo = &vp->v_bufobj;
3471 	BO_LOCK(bo);
3472 	vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0);
3473 	/* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */
3474 	mtx_lock(&sync_mtx);
3475 	sync_vnode_count++;
3476 	if (mp->mnt_syncer == NULL) {
3477 		mp->mnt_syncer = vp;
3478 		vp = NULL;
3479 	}
3480 	mtx_unlock(&sync_mtx);
3481 	BO_UNLOCK(bo);
3482 	if (vp != NULL) {
3483 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
3484 		vgone(vp);
3485 		vput(vp);
3486 	}
3487 }
3488 
3489 void
3490 vfs_deallocate_syncvnode(struct mount *mp)
3491 {
3492 	struct vnode *vp;
3493 
3494 	mtx_lock(&sync_mtx);
3495 	vp = mp->mnt_syncer;
3496 	if (vp != NULL)
3497 		mp->mnt_syncer = NULL;
3498 	mtx_unlock(&sync_mtx);
3499 	if (vp != NULL)
3500 		vrele(vp);
3501 }
3502 
3503 /*
3504  * Do a lazy sync of the filesystem.
3505  */
3506 static int
3507 sync_fsync(struct vop_fsync_args *ap)
3508 {
3509 	struct vnode *syncvp = ap->a_vp;
3510 	struct mount *mp = syncvp->v_mount;
3511 	int error;
3512 	struct bufobj *bo;
3513 
3514 	/*
3515 	 * We only need to do something if this is a lazy evaluation.
3516 	 */
3517 	if (ap->a_waitfor != MNT_LAZY)
3518 		return (0);
3519 
3520 	/*
3521 	 * Move ourselves to the back of the sync list.
3522 	 */
3523 	bo = &syncvp->v_bufobj;
3524 	BO_LOCK(bo);
3525 	vn_syncer_add_to_worklist(bo, syncdelay);
3526 	BO_UNLOCK(bo);
3527 
3528 	/*
3529 	 * Walk the list of vnodes pushing all that are dirty and
3530 	 * not already on the sync list.
3531 	 */
3532 	mtx_lock(&mountlist_mtx);
3533 	if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK) != 0) {
3534 		mtx_unlock(&mountlist_mtx);
3535 		return (0);
3536 	}
3537 	if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) {
3538 		vfs_unbusy(mp);
3539 		return (0);
3540 	}
3541 	MNT_ILOCK(mp);
3542 	mp->mnt_noasync++;
3543 	mp->mnt_kern_flag &= ~MNTK_ASYNC;
3544 	MNT_IUNLOCK(mp);
3545 	vfs_msync(mp, MNT_NOWAIT);
3546 	error = VFS_SYNC(mp, MNT_LAZY);
3547 	MNT_ILOCK(mp);
3548 	mp->mnt_noasync--;
3549 	if ((mp->mnt_flag & MNT_ASYNC) != 0 && mp->mnt_noasync == 0)
3550 		mp->mnt_kern_flag |= MNTK_ASYNC;
3551 	MNT_IUNLOCK(mp);
3552 	vn_finished_write(mp);
3553 	vfs_unbusy(mp);
3554 	return (error);
3555 }
3556 
3557 /*
3558  * The syncer vnode is no referenced.
3559  */
3560 static int
3561 sync_inactive(struct vop_inactive_args *ap)
3562 {
3563 
3564 	vgone(ap->a_vp);
3565 	return (0);
3566 }
3567 
3568 /*
3569  * The syncer vnode is no longer needed and is being decommissioned.
3570  *
3571  * Modifications to the worklist must be protected by sync_mtx.
3572  */
3573 static int
3574 sync_reclaim(struct vop_reclaim_args *ap)
3575 {
3576 	struct vnode *vp = ap->a_vp;
3577 	struct bufobj *bo;
3578 
3579 	bo = &vp->v_bufobj;
3580 	BO_LOCK(bo);
3581 	mtx_lock(&sync_mtx);
3582 	if (vp->v_mount->mnt_syncer == vp)
3583 		vp->v_mount->mnt_syncer = NULL;
3584 	if (bo->bo_flag & BO_ONWORKLST) {
3585 		LIST_REMOVE(bo, bo_synclist);
3586 		syncer_worklist_len--;
3587 		sync_vnode_count--;
3588 		bo->bo_flag &= ~BO_ONWORKLST;
3589 	}
3590 	mtx_unlock(&sync_mtx);
3591 	BO_UNLOCK(bo);
3592 
3593 	return (0);
3594 }
3595 
3596 /*
3597  * Check if vnode represents a disk device
3598  */
3599 int
3600 vn_isdisk(struct vnode *vp, int *errp)
3601 {
3602 	int error;
3603 
3604 	error = 0;
3605 	dev_lock();
3606 	if (vp->v_type != VCHR)
3607 		error = ENOTBLK;
3608 	else if (vp->v_rdev == NULL)
3609 		error = ENXIO;
3610 	else if (vp->v_rdev->si_devsw == NULL)
3611 		error = ENXIO;
3612 	else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK))
3613 		error = ENOTBLK;
3614 	dev_unlock();
3615 	if (errp != NULL)
3616 		*errp = error;
3617 	return (error == 0);
3618 }
3619 
3620 /*
3621  * Common filesystem object access control check routine.  Accepts a
3622  * vnode's type, "mode", uid and gid, requested access mode, credentials,
3623  * and optional call-by-reference privused argument allowing vaccess()
3624  * to indicate to the caller whether privilege was used to satisfy the
3625  * request (obsoleted).  Returns 0 on success, or an errno on failure.
3626  */
3627 int
3628 vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid,
3629     accmode_t accmode, struct ucred *cred, int *privused)
3630 {
3631 	accmode_t dac_granted;
3632 	accmode_t priv_granted;
3633 
3634 	KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0,
3635 	    ("invalid bit in accmode"));
3636 	KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE),
3637 	    ("VAPPEND without VWRITE"));
3638 
3639 	/*
3640 	 * Look for a normal, non-privileged way to access the file/directory
3641 	 * as requested.  If it exists, go with that.
3642 	 */
3643 
3644 	if (privused != NULL)
3645 		*privused = 0;
3646 
3647 	dac_granted = 0;
3648 
3649 	/* Check the owner. */
3650 	if (cred->cr_uid == file_uid) {
3651 		dac_granted |= VADMIN;
3652 		if (file_mode & S_IXUSR)
3653 			dac_granted |= VEXEC;
3654 		if (file_mode & S_IRUSR)
3655 			dac_granted |= VREAD;
3656 		if (file_mode & S_IWUSR)
3657 			dac_granted |= (VWRITE | VAPPEND);
3658 
3659 		if ((accmode & dac_granted) == accmode)
3660 			return (0);
3661 
3662 		goto privcheck;
3663 	}
3664 
3665 	/* Otherwise, check the groups (first match) */
3666 	if (groupmember(file_gid, cred)) {
3667 		if (file_mode & S_IXGRP)
3668 			dac_granted |= VEXEC;
3669 		if (file_mode & S_IRGRP)
3670 			dac_granted |= VREAD;
3671 		if (file_mode & S_IWGRP)
3672 			dac_granted |= (VWRITE | VAPPEND);
3673 
3674 		if ((accmode & dac_granted) == accmode)
3675 			return (0);
3676 
3677 		goto privcheck;
3678 	}
3679 
3680 	/* Otherwise, check everyone else. */
3681 	if (file_mode & S_IXOTH)
3682 		dac_granted |= VEXEC;
3683 	if (file_mode & S_IROTH)
3684 		dac_granted |= VREAD;
3685 	if (file_mode & S_IWOTH)
3686 		dac_granted |= (VWRITE | VAPPEND);
3687 	if ((accmode & dac_granted) == accmode)
3688 		return (0);
3689 
3690 privcheck:
3691 	/*
3692 	 * Build a privilege mask to determine if the set of privileges
3693 	 * satisfies the requirements when combined with the granted mask
3694 	 * from above.  For each privilege, if the privilege is required,
3695 	 * bitwise or the request type onto the priv_granted mask.
3696 	 */
3697 	priv_granted = 0;
3698 
3699 	if (type == VDIR) {
3700 		/*
3701 		 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC
3702 		 * requests, instead of PRIV_VFS_EXEC.
3703 		 */
3704 		if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
3705 		    !priv_check_cred(cred, PRIV_VFS_LOOKUP, 0))
3706 			priv_granted |= VEXEC;
3707 	} else {
3708 		/*
3709 		 * Ensure that at least one execute bit is on. Otherwise,
3710 		 * a privileged user will always succeed, and we don't want
3711 		 * this to happen unless the file really is executable.
3712 		 */
3713 		if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
3714 		    (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 &&
3715 		    !priv_check_cred(cred, PRIV_VFS_EXEC, 0))
3716 			priv_granted |= VEXEC;
3717 	}
3718 
3719 	if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) &&
3720 	    !priv_check_cred(cred, PRIV_VFS_READ, 0))
3721 		priv_granted |= VREAD;
3722 
3723 	if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) &&
3724 	    !priv_check_cred(cred, PRIV_VFS_WRITE, 0))
3725 		priv_granted |= (VWRITE | VAPPEND);
3726 
3727 	if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) &&
3728 	    !priv_check_cred(cred, PRIV_VFS_ADMIN, 0))
3729 		priv_granted |= VADMIN;
3730 
3731 	if ((accmode & (priv_granted | dac_granted)) == accmode) {
3732 		/* XXX audit: privilege used */
3733 		if (privused != NULL)
3734 			*privused = 1;
3735 		return (0);
3736 	}
3737 
3738 	return ((accmode & VADMIN) ? EPERM : EACCES);
3739 }
3740 
3741 /*
3742  * Credential check based on process requesting service, and per-attribute
3743  * permissions.
3744  */
3745 int
3746 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred,
3747     struct thread *td, accmode_t accmode)
3748 {
3749 
3750 	/*
3751 	 * Kernel-invoked always succeeds.
3752 	 */
3753 	if (cred == NOCRED)
3754 		return (0);
3755 
3756 	/*
3757 	 * Do not allow privileged processes in jail to directly manipulate
3758 	 * system attributes.
3759 	 */
3760 	switch (attrnamespace) {
3761 	case EXTATTR_NAMESPACE_SYSTEM:
3762 		/* Potentially should be: return (EPERM); */
3763 		return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM, 0));
3764 	case EXTATTR_NAMESPACE_USER:
3765 		return (VOP_ACCESS(vp, accmode, cred, td));
3766 	default:
3767 		return (EPERM);
3768 	}
3769 }
3770 
3771 #ifdef DEBUG_VFS_LOCKS
3772 /*
3773  * This only exists to supress warnings from unlocked specfs accesses.  It is
3774  * no longer ok to have an unlocked VFS.
3775  */
3776 #define	IGNORE_LOCK(vp) (panicstr != NULL || (vp) == NULL ||		\
3777 	(vp)->v_type == VCHR ||	(vp)->v_type == VBAD)
3778 
3779 int vfs_badlock_ddb = 1;	/* Drop into debugger on violation. */
3780 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0,
3781     "Drop into debugger on lock violation");
3782 
3783 int vfs_badlock_mutex = 1;	/* Check for interlock across VOPs. */
3784 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex,
3785     0, "Check for interlock across VOPs");
3786 
3787 int vfs_badlock_print = 1;	/* Print lock violations. */
3788 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print,
3789     0, "Print lock violations");
3790 
3791 #ifdef KDB
3792 int vfs_badlock_backtrace = 1;	/* Print backtrace at lock violations. */
3793 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW,
3794     &vfs_badlock_backtrace, 0, "Print backtrace at lock violations");
3795 #endif
3796 
3797 static void
3798 vfs_badlock(const char *msg, const char *str, struct vnode *vp)
3799 {
3800 
3801 #ifdef KDB
3802 	if (vfs_badlock_backtrace)
3803 		kdb_backtrace();
3804 #endif
3805 	if (vfs_badlock_print)
3806 		printf("%s: %p %s\n", str, (void *)vp, msg);
3807 	if (vfs_badlock_ddb)
3808 		kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
3809 }
3810 
3811 void
3812 assert_vi_locked(struct vnode *vp, const char *str)
3813 {
3814 
3815 	if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp)))
3816 		vfs_badlock("interlock is not locked but should be", str, vp);
3817 }
3818 
3819 void
3820 assert_vi_unlocked(struct vnode *vp, const char *str)
3821 {
3822 
3823 	if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp)))
3824 		vfs_badlock("interlock is locked but should not be", str, vp);
3825 }
3826 
3827 void
3828 assert_vop_locked(struct vnode *vp, const char *str)
3829 {
3830 
3831 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) == 0)
3832 		vfs_badlock("is not locked but should be", str, vp);
3833 }
3834 
3835 void
3836 assert_vop_unlocked(struct vnode *vp, const char *str)
3837 {
3838 
3839 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
3840 		vfs_badlock("is locked but should not be", str, vp);
3841 }
3842 
3843 void
3844 assert_vop_elocked(struct vnode *vp, const char *str)
3845 {
3846 
3847 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
3848 		vfs_badlock("is not exclusive locked but should be", str, vp);
3849 }
3850 
3851 #if 0
3852 void
3853 assert_vop_elocked_other(struct vnode *vp, const char *str)
3854 {
3855 
3856 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLOTHER)
3857 		vfs_badlock("is not exclusive locked by another thread",
3858 		    str, vp);
3859 }
3860 
3861 void
3862 assert_vop_slocked(struct vnode *vp, const char *str)
3863 {
3864 
3865 	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_SHARED)
3866 		vfs_badlock("is not locked shared but should be", str, vp);
3867 }
3868 #endif /* 0 */
3869 #endif /* DEBUG_VFS_LOCKS */
3870 
3871 void
3872 vop_rename_fail(struct vop_rename_args *ap)
3873 {
3874 
3875 	if (ap->a_tvp != NULL)
3876 		vput(ap->a_tvp);
3877 	if (ap->a_tdvp == ap->a_tvp)
3878 		vrele(ap->a_tdvp);
3879 	else
3880 		vput(ap->a_tdvp);
3881 	vrele(ap->a_fdvp);
3882 	vrele(ap->a_fvp);
3883 }
3884 
3885 void
3886 vop_rename_pre(void *ap)
3887 {
3888 	struct vop_rename_args *a = ap;
3889 
3890 #ifdef DEBUG_VFS_LOCKS
3891 	if (a->a_tvp)
3892 		ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME");
3893 	ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME");
3894 	ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME");
3895 	ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME");
3896 
3897 	/* Check the source (from). */
3898 	if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock &&
3899 	    (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock))
3900 		ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked");
3901 	if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock)
3902 		ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked");
3903 
3904 	/* Check the target. */
3905 	if (a->a_tvp)
3906 		ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked");
3907 	ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked");
3908 #endif
3909 	if (a->a_tdvp != a->a_fdvp)
3910 		vhold(a->a_fdvp);
3911 	if (a->a_tvp != a->a_fvp)
3912 		vhold(a->a_fvp);
3913 	vhold(a->a_tdvp);
3914 	if (a->a_tvp)
3915 		vhold(a->a_tvp);
3916 }
3917 
3918 void
3919 vop_strategy_pre(void *ap)
3920 {
3921 #ifdef DEBUG_VFS_LOCKS
3922 	struct vop_strategy_args *a;
3923 	struct buf *bp;
3924 
3925 	a = ap;
3926 	bp = a->a_bp;
3927 
3928 	/*
3929 	 * Cluster ops lock their component buffers but not the IO container.
3930 	 */
3931 	if ((bp->b_flags & B_CLUSTER) != 0)
3932 		return;
3933 
3934 	if (panicstr == NULL && !BUF_ISLOCKED(bp)) {
3935 		if (vfs_badlock_print)
3936 			printf(
3937 			    "VOP_STRATEGY: bp is not locked but should be\n");
3938 		if (vfs_badlock_ddb)
3939 			kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
3940 	}
3941 #endif
3942 }
3943 
3944 void
3945 vop_lookup_pre(void *ap)
3946 {
3947 #ifdef DEBUG_VFS_LOCKS
3948 	struct vop_lookup_args *a;
3949 	struct vnode *dvp;
3950 
3951 	a = ap;
3952 	dvp = a->a_dvp;
3953 	ASSERT_VI_UNLOCKED(dvp, "VOP_LOOKUP");
3954 	ASSERT_VOP_LOCKED(dvp, "VOP_LOOKUP");
3955 #endif
3956 }
3957 
3958 void
3959 vop_lookup_post(void *ap, int rc)
3960 {
3961 #ifdef DEBUG_VFS_LOCKS
3962 	struct vop_lookup_args *a;
3963 	struct vnode *dvp;
3964 	struct vnode *vp;
3965 
3966 	a = ap;
3967 	dvp = a->a_dvp;
3968 	vp = *(a->a_vpp);
3969 
3970 	ASSERT_VI_UNLOCKED(dvp, "VOP_LOOKUP");
3971 	ASSERT_VOP_LOCKED(dvp, "VOP_LOOKUP");
3972 
3973 	if (!rc)
3974 		ASSERT_VOP_LOCKED(vp, "VOP_LOOKUP (child)");
3975 #endif
3976 }
3977 
3978 void
3979 vop_lock_pre(void *ap)
3980 {
3981 #ifdef DEBUG_VFS_LOCKS
3982 	struct vop_lock1_args *a = ap;
3983 
3984 	if ((a->a_flags & LK_INTERLOCK) == 0)
3985 		ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
3986 	else
3987 		ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK");
3988 #endif
3989 }
3990 
3991 void
3992 vop_lock_post(void *ap, int rc)
3993 {
3994 #ifdef DEBUG_VFS_LOCKS
3995 	struct vop_lock1_args *a = ap;
3996 
3997 	ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
3998 	if (rc == 0)
3999 		ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK");
4000 #endif
4001 }
4002 
4003 void
4004 vop_unlock_pre(void *ap)
4005 {
4006 #ifdef DEBUG_VFS_LOCKS
4007 	struct vop_unlock_args *a = ap;
4008 
4009 	if (a->a_flags & LK_INTERLOCK)
4010 		ASSERT_VI_LOCKED(a->a_vp, "VOP_UNLOCK");
4011 	ASSERT_VOP_LOCKED(a->a_vp, "VOP_UNLOCK");
4012 #endif
4013 }
4014 
4015 void
4016 vop_unlock_post(void *ap, int rc)
4017 {
4018 #ifdef DEBUG_VFS_LOCKS
4019 	struct vop_unlock_args *a = ap;
4020 
4021 	if (a->a_flags & LK_INTERLOCK)
4022 		ASSERT_VI_UNLOCKED(a->a_vp, "VOP_UNLOCK");
4023 #endif
4024 }
4025 
4026 void
4027 vop_create_post(void *ap, int rc)
4028 {
4029 	struct vop_create_args *a = ap;
4030 
4031 	if (!rc)
4032 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4033 }
4034 
4035 void
4036 vop_link_post(void *ap, int rc)
4037 {
4038 	struct vop_link_args *a = ap;
4039 
4040 	if (!rc) {
4041 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_LINK);
4042 		VFS_KNOTE_LOCKED(a->a_tdvp, NOTE_WRITE);
4043 	}
4044 }
4045 
4046 void
4047 vop_mkdir_post(void *ap, int rc)
4048 {
4049 	struct vop_mkdir_args *a = ap;
4050 
4051 	if (!rc)
4052 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
4053 }
4054 
4055 void
4056 vop_mknod_post(void *ap, int rc)
4057 {
4058 	struct vop_mknod_args *a = ap;
4059 
4060 	if (!rc)
4061 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4062 }
4063 
4064 void
4065 vop_remove_post(void *ap, int rc)
4066 {
4067 	struct vop_remove_args *a = ap;
4068 
4069 	if (!rc) {
4070 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4071 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
4072 	}
4073 }
4074 
4075 void
4076 vop_rename_post(void *ap, int rc)
4077 {
4078 	struct vop_rename_args *a = ap;
4079 
4080 	if (!rc) {
4081 		VFS_KNOTE_UNLOCKED(a->a_fdvp, NOTE_WRITE);
4082 		VFS_KNOTE_UNLOCKED(a->a_tdvp, NOTE_WRITE);
4083 		VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME);
4084 		if (a->a_tvp)
4085 			VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE);
4086 	}
4087 	if (a->a_tdvp != a->a_fdvp)
4088 		vdrop(a->a_fdvp);
4089 	if (a->a_tvp != a->a_fvp)
4090 		vdrop(a->a_fvp);
4091 	vdrop(a->a_tdvp);
4092 	if (a->a_tvp)
4093 		vdrop(a->a_tvp);
4094 }
4095 
4096 void
4097 vop_rmdir_post(void *ap, int rc)
4098 {
4099 	struct vop_rmdir_args *a = ap;
4100 
4101 	if (!rc) {
4102 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
4103 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
4104 	}
4105 }
4106 
4107 void
4108 vop_setattr_post(void *ap, int rc)
4109 {
4110 	struct vop_setattr_args *a = ap;
4111 
4112 	if (!rc)
4113 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
4114 }
4115 
4116 void
4117 vop_symlink_post(void *ap, int rc)
4118 {
4119 	struct vop_symlink_args *a = ap;
4120 
4121 	if (!rc)
4122 		VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4123 }
4124 
4125 static struct knlist fs_knlist;
4126 
4127 static void
4128 vfs_event_init(void *arg)
4129 {
4130 	knlist_init_mtx(&fs_knlist, NULL);
4131 }
4132 /* XXX - correct order? */
4133 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL);
4134 
4135 void
4136 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused)
4137 {
4138 
4139 	KNOTE_UNLOCKED(&fs_knlist, event);
4140 }
4141 
4142 static int	filt_fsattach(struct knote *kn);
4143 static void	filt_fsdetach(struct knote *kn);
4144 static int	filt_fsevent(struct knote *kn, long hint);
4145 
4146 struct filterops fs_filtops = {
4147 	.f_isfd = 0,
4148 	.f_attach = filt_fsattach,
4149 	.f_detach = filt_fsdetach,
4150 	.f_event = filt_fsevent
4151 };
4152 
4153 static int
4154 filt_fsattach(struct knote *kn)
4155 {
4156 
4157 	kn->kn_flags |= EV_CLEAR;
4158 	knlist_add(&fs_knlist, kn, 0);
4159 	return (0);
4160 }
4161 
4162 static void
4163 filt_fsdetach(struct knote *kn)
4164 {
4165 
4166 	knlist_remove(&fs_knlist, kn, 0);
4167 }
4168 
4169 static int
4170 filt_fsevent(struct knote *kn, long hint)
4171 {
4172 
4173 	kn->kn_fflags |= hint;
4174 	return (kn->kn_fflags != 0);
4175 }
4176 
4177 static int
4178 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS)
4179 {
4180 	struct vfsidctl vc;
4181 	int error;
4182 	struct mount *mp;
4183 
4184 	error = SYSCTL_IN(req, &vc, sizeof(vc));
4185 	if (error)
4186 		return (error);
4187 	if (vc.vc_vers != VFS_CTL_VERS1)
4188 		return (EINVAL);
4189 	mp = vfs_getvfs(&vc.vc_fsid);
4190 	if (mp == NULL)
4191 		return (ENOENT);
4192 	/* ensure that a specific sysctl goes to the right filesystem. */
4193 	if (strcmp(vc.vc_fstypename, "*") != 0 &&
4194 	    strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) {
4195 		vfs_rel(mp);
4196 		return (EINVAL);
4197 	}
4198 	VCTLTOREQ(&vc, req);
4199 	error = VFS_SYSCTL(mp, vc.vc_op, req);
4200 	vfs_rel(mp);
4201 	return (error);
4202 }
4203 
4204 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_WR,
4205     NULL, 0, sysctl_vfs_ctl, "",
4206     "Sysctl by fsid");
4207 
4208 /*
4209  * Function to initialize a va_filerev field sensibly.
4210  * XXX: Wouldn't a random number make a lot more sense ??
4211  */
4212 u_quad_t
4213 init_va_filerev(void)
4214 {
4215 	struct bintime bt;
4216 
4217 	getbinuptime(&bt);
4218 	return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL));
4219 }
4220 
4221 static int	filt_vfsread(struct knote *kn, long hint);
4222 static int	filt_vfswrite(struct knote *kn, long hint);
4223 static int	filt_vfsvnode(struct knote *kn, long hint);
4224 static void	filt_vfsdetach(struct knote *kn);
4225 static struct filterops vfsread_filtops = {
4226 	.f_isfd = 1,
4227 	.f_detach = filt_vfsdetach,
4228 	.f_event = filt_vfsread
4229 };
4230 static struct filterops vfswrite_filtops = {
4231 	.f_isfd = 1,
4232 	.f_detach = filt_vfsdetach,
4233 	.f_event = filt_vfswrite
4234 };
4235 static struct filterops vfsvnode_filtops = {
4236 	.f_isfd = 1,
4237 	.f_detach = filt_vfsdetach,
4238 	.f_event = filt_vfsvnode
4239 };
4240 
4241 static void
4242 vfs_knllock(void *arg)
4243 {
4244 	struct vnode *vp = arg;
4245 
4246 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
4247 }
4248 
4249 static void
4250 vfs_knlunlock(void *arg)
4251 {
4252 	struct vnode *vp = arg;
4253 
4254 	VOP_UNLOCK(vp, 0);
4255 }
4256 
4257 static void
4258 vfs_knl_assert_locked(void *arg)
4259 {
4260 #ifdef DEBUG_VFS_LOCKS
4261 	struct vnode *vp = arg;
4262 
4263 	ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked");
4264 #endif
4265 }
4266 
4267 static void
4268 vfs_knl_assert_unlocked(void *arg)
4269 {
4270 #ifdef DEBUG_VFS_LOCKS
4271 	struct vnode *vp = arg;
4272 
4273 	ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked");
4274 #endif
4275 }
4276 
4277 int
4278 vfs_kqfilter(struct vop_kqfilter_args *ap)
4279 {
4280 	struct vnode *vp = ap->a_vp;
4281 	struct knote *kn = ap->a_kn;
4282 	struct knlist *knl;
4283 
4284 	switch (kn->kn_filter) {
4285 	case EVFILT_READ:
4286 		kn->kn_fop = &vfsread_filtops;
4287 		break;
4288 	case EVFILT_WRITE:
4289 		kn->kn_fop = &vfswrite_filtops;
4290 		break;
4291 	case EVFILT_VNODE:
4292 		kn->kn_fop = &vfsvnode_filtops;
4293 		break;
4294 	default:
4295 		return (EINVAL);
4296 	}
4297 
4298 	kn->kn_hook = (caddr_t)vp;
4299 
4300 	v_addpollinfo(vp);
4301 	if (vp->v_pollinfo == NULL)
4302 		return (ENOMEM);
4303 	knl = &vp->v_pollinfo->vpi_selinfo.si_note;
4304 	knlist_add(knl, kn, 0);
4305 
4306 	return (0);
4307 }
4308 
4309 /*
4310  * Detach knote from vnode
4311  */
4312 static void
4313 filt_vfsdetach(struct knote *kn)
4314 {
4315 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4316 
4317 	KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo"));
4318 	knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0);
4319 }
4320 
4321 /*ARGSUSED*/
4322 static int
4323 filt_vfsread(struct knote *kn, long hint)
4324 {
4325 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4326 	struct vattr va;
4327 	int res;
4328 
4329 	/*
4330 	 * filesystem is gone, so set the EOF flag and schedule
4331 	 * the knote for deletion.
4332 	 */
4333 	if (hint == NOTE_REVOKE) {
4334 		VI_LOCK(vp);
4335 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
4336 		VI_UNLOCK(vp);
4337 		return (1);
4338 	}
4339 
4340 	if (VOP_GETATTR(vp, &va, curthread->td_ucred))
4341 		return (0);
4342 
4343 	VI_LOCK(vp);
4344 	kn->kn_data = va.va_size - kn->kn_fp->f_offset;
4345 	res = (kn->kn_data != 0);
4346 	VI_UNLOCK(vp);
4347 	return (res);
4348 }
4349 
4350 /*ARGSUSED*/
4351 static int
4352 filt_vfswrite(struct knote *kn, long hint)
4353 {
4354 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4355 
4356 	VI_LOCK(vp);
4357 
4358 	/*
4359 	 * filesystem is gone, so set the EOF flag and schedule
4360 	 * the knote for deletion.
4361 	 */
4362 	if (hint == NOTE_REVOKE)
4363 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
4364 
4365 	kn->kn_data = 0;
4366 	VI_UNLOCK(vp);
4367 	return (1);
4368 }
4369 
4370 static int
4371 filt_vfsvnode(struct knote *kn, long hint)
4372 {
4373 	struct vnode *vp = (struct vnode *)kn->kn_hook;
4374 	int res;
4375 
4376 	VI_LOCK(vp);
4377 	if (kn->kn_sfflags & hint)
4378 		kn->kn_fflags |= hint;
4379 	if (hint == NOTE_REVOKE) {
4380 		kn->kn_flags |= EV_EOF;
4381 		VI_UNLOCK(vp);
4382 		return (1);
4383 	}
4384 	res = (kn->kn_fflags != 0);
4385 	VI_UNLOCK(vp);
4386 	return (res);
4387 }
4388 
4389 int
4390 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off)
4391 {
4392 	int error;
4393 
4394 	if (dp->d_reclen > ap->a_uio->uio_resid)
4395 		return (ENAMETOOLONG);
4396 	error = uiomove(dp, dp->d_reclen, ap->a_uio);
4397 	if (error) {
4398 		if (ap->a_ncookies != NULL) {
4399 			if (ap->a_cookies != NULL)
4400 				free(ap->a_cookies, M_TEMP);
4401 			ap->a_cookies = NULL;
4402 			*ap->a_ncookies = 0;
4403 		}
4404 		return (error);
4405 	}
4406 	if (ap->a_ncookies == NULL)
4407 		return (0);
4408 
4409 	KASSERT(ap->a_cookies,
4410 	    ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!"));
4411 
4412 	*ap->a_cookies = realloc(*ap->a_cookies,
4413 	    (*ap->a_ncookies + 1) * sizeof(u_long), M_TEMP, M_WAITOK | M_ZERO);
4414 	(*ap->a_cookies)[*ap->a_ncookies] = off;
4415 	return (0);
4416 }
4417 
4418 /*
4419  * Mark for update the access time of the file if the filesystem
4420  * supports VOP_MARKATIME.  This functionality is used by execve and
4421  * mmap, so we want to avoid the I/O implied by directly setting
4422  * va_atime for the sake of efficiency.
4423  */
4424 void
4425 vfs_mark_atime(struct vnode *vp, struct ucred *cred)
4426 {
4427 	struct mount *mp;
4428 
4429 	mp = vp->v_mount;
4430 	VFS_ASSERT_GIANT(mp);
4431 	ASSERT_VOP_LOCKED(vp, "vfs_mark_atime");
4432 	if (mp != NULL && (mp->mnt_flag & (MNT_NOATIME | MNT_RDONLY)) == 0)
4433 		(void)VOP_MARKATIME(vp);
4434 }
4435 
4436 /*
4437  * The purpose of this routine is to remove granularity from accmode_t,
4438  * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE,
4439  * VADMIN and VAPPEND.
4440  *
4441  * If it returns 0, the caller is supposed to continue with the usual
4442  * access checks using 'accmode' as modified by this routine.  If it
4443  * returns nonzero value, the caller is supposed to return that value
4444  * as errno.
4445  *
4446  * Note that after this routine runs, accmode may be zero.
4447  */
4448 int
4449 vfs_unixify_accmode(accmode_t *accmode)
4450 {
4451 	/*
4452 	 * There is no way to specify explicit "deny" rule using
4453 	 * file mode or POSIX.1e ACLs.
4454 	 */
4455 	if (*accmode & VEXPLICIT_DENY) {
4456 		*accmode = 0;
4457 		return (0);
4458 	}
4459 
4460 	/*
4461 	 * None of these can be translated into usual access bits.
4462 	 * Also, the common case for NFSv4 ACLs is to not contain
4463 	 * either of these bits. Caller should check for VWRITE
4464 	 * on the containing directory instead.
4465 	 */
4466 	if (*accmode & (VDELETE_CHILD | VDELETE))
4467 		return (EPERM);
4468 
4469 	if (*accmode & VADMIN_PERMS) {
4470 		*accmode &= ~VADMIN_PERMS;
4471 		*accmode |= VADMIN;
4472 	}
4473 
4474 	/*
4475 	 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL
4476 	 * or VSYNCHRONIZE using file mode or POSIX.1e ACL.
4477 	 */
4478 	*accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE);
4479 
4480 	return (0);
4481 }
4482