1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vfs_subr.c 8.31 (Berkeley) 5/26/95 37 */ 38 39 /* 40 * External virtual filesystem routines 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include "opt_ddb.h" 47 #include "opt_watchdog.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/bio.h> 52 #include <sys/buf.h> 53 #include <sys/capsicum.h> 54 #include <sys/condvar.h> 55 #include <sys/conf.h> 56 #include <sys/counter.h> 57 #include <sys/dirent.h> 58 #include <sys/event.h> 59 #include <sys/eventhandler.h> 60 #include <sys/extattr.h> 61 #include <sys/file.h> 62 #include <sys/fcntl.h> 63 #include <sys/jail.h> 64 #include <sys/kdb.h> 65 #include <sys/kernel.h> 66 #include <sys/kthread.h> 67 #include <sys/ktr.h> 68 #include <sys/lockf.h> 69 #include <sys/malloc.h> 70 #include <sys/mount.h> 71 #include <sys/namei.h> 72 #include <sys/pctrie.h> 73 #include <sys/priv.h> 74 #include <sys/reboot.h> 75 #include <sys/refcount.h> 76 #include <sys/rwlock.h> 77 #include <sys/sched.h> 78 #include <sys/sleepqueue.h> 79 #include <sys/smr.h> 80 #include <sys/smp.h> 81 #include <sys/stat.h> 82 #include <sys/sysctl.h> 83 #include <sys/syslog.h> 84 #include <sys/vmmeter.h> 85 #include <sys/vnode.h> 86 #include <sys/watchdog.h> 87 88 #include <machine/stdarg.h> 89 90 #include <security/mac/mac_framework.h> 91 92 #include <vm/vm.h> 93 #include <vm/vm_object.h> 94 #include <vm/vm_extern.h> 95 #include <vm/pmap.h> 96 #include <vm/vm_map.h> 97 #include <vm/vm_page.h> 98 #include <vm/vm_kern.h> 99 #include <vm/uma.h> 100 101 #ifdef DDB 102 #include <ddb/ddb.h> 103 #endif 104 105 static void delmntque(struct vnode *vp); 106 static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, 107 int slpflag, int slptimeo); 108 static void syncer_shutdown(void *arg, int howto); 109 static int vtryrecycle(struct vnode *vp); 110 static void v_init_counters(struct vnode *); 111 static void vgonel(struct vnode *); 112 static bool vhold_recycle(struct vnode *); 113 static void vfs_knllock(void *arg); 114 static void vfs_knlunlock(void *arg); 115 static void vfs_knl_assert_locked(void *arg); 116 static void vfs_knl_assert_unlocked(void *arg); 117 static void destroy_vpollinfo(struct vpollinfo *vi); 118 static int v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 119 daddr_t startlbn, daddr_t endlbn); 120 static void vnlru_recalc(void); 121 122 /* 123 * These fences are intended for cases where some synchronization is 124 * needed between access of v_iflags and lockless vnode refcount (v_holdcnt 125 * and v_usecount) updates. Access to v_iflags is generally synchronized 126 * by the interlock, but we have some internal assertions that check vnode 127 * flags without acquiring the lock. Thus, these fences are INVARIANTS-only 128 * for now. 129 */ 130 #ifdef INVARIANTS 131 #define VNODE_REFCOUNT_FENCE_ACQ() atomic_thread_fence_acq() 132 #define VNODE_REFCOUNT_FENCE_REL() atomic_thread_fence_rel() 133 #else 134 #define VNODE_REFCOUNT_FENCE_ACQ() 135 #define VNODE_REFCOUNT_FENCE_REL() 136 #endif 137 138 /* 139 * Number of vnodes in existence. Increased whenever getnewvnode() 140 * allocates a new vnode, decreased in vdropl() for VIRF_DOOMED vnode. 141 */ 142 static u_long __exclusive_cache_line numvnodes; 143 144 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0, 145 "Number of vnodes in existence"); 146 147 static counter_u64_t vnodes_created; 148 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created, 149 "Number of vnodes created by getnewvnode"); 150 151 /* 152 * Conversion tables for conversion from vnode types to inode formats 153 * and back. 154 */ 155 enum vtype iftovt_tab[16] = { 156 VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON, 157 VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON 158 }; 159 int vttoif_tab[10] = { 160 0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK, 161 S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT 162 }; 163 164 /* 165 * List of allocates vnodes in the system. 166 */ 167 static TAILQ_HEAD(freelst, vnode) vnode_list; 168 static struct vnode *vnode_list_free_marker; 169 static struct vnode *vnode_list_reclaim_marker; 170 171 /* 172 * "Free" vnode target. Free vnodes are rarely completely free, but are 173 * just ones that are cheap to recycle. Usually they are for files which 174 * have been stat'd but not read; these usually have inode and namecache 175 * data attached to them. This target is the preferred minimum size of a 176 * sub-cache consisting mostly of such files. The system balances the size 177 * of this sub-cache with its complement to try to prevent either from 178 * thrashing while the other is relatively inactive. The targets express 179 * a preference for the best balance. 180 * 181 * "Above" this target there are 2 further targets (watermarks) related 182 * to recyling of free vnodes. In the best-operating case, the cache is 183 * exactly full, the free list has size between vlowat and vhiwat above the 184 * free target, and recycling from it and normal use maintains this state. 185 * Sometimes the free list is below vlowat or even empty, but this state 186 * is even better for immediate use provided the cache is not full. 187 * Otherwise, vnlru_proc() runs to reclaim enough vnodes (usually non-free 188 * ones) to reach one of these states. The watermarks are currently hard- 189 * coded as 4% and 9% of the available space higher. These and the default 190 * of 25% for wantfreevnodes are too large if the memory size is large. 191 * E.g., 9% of 75% of MAXVNODES is more than 566000 vnodes to reclaim 192 * whenever vnlru_proc() becomes active. 193 */ 194 static long wantfreevnodes; 195 static long __exclusive_cache_line freevnodes; 196 SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD, 197 &freevnodes, 0, "Number of \"free\" vnodes"); 198 static long freevnodes_old; 199 200 static counter_u64_t recycles_count; 201 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles, CTLFLAG_RD, &recycles_count, 202 "Number of vnodes recycled to meet vnode cache targets"); 203 204 static counter_u64_t recycles_free_count; 205 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles_free, CTLFLAG_RD, &recycles_free_count, 206 "Number of free vnodes recycled to meet vnode cache targets"); 207 208 static counter_u64_t deferred_inact; 209 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, deferred_inact, CTLFLAG_RD, &deferred_inact, 210 "Number of times inactive processing was deferred"); 211 212 /* To keep more than one thread at a time from running vfs_getnewfsid */ 213 static struct mtx mntid_mtx; 214 215 /* 216 * Lock for any access to the following: 217 * vnode_list 218 * numvnodes 219 * freevnodes 220 */ 221 static struct mtx __exclusive_cache_line vnode_list_mtx; 222 223 /* Publicly exported FS */ 224 struct nfs_public nfs_pub; 225 226 static uma_zone_t buf_trie_zone; 227 static smr_t buf_trie_smr; 228 229 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */ 230 static uma_zone_t vnode_zone; 231 static uma_zone_t vnodepoll_zone; 232 233 __read_frequently smr_t vfs_smr; 234 235 /* 236 * The workitem queue. 237 * 238 * It is useful to delay writes of file data and filesystem metadata 239 * for tens of seconds so that quickly created and deleted files need 240 * not waste disk bandwidth being created and removed. To realize this, 241 * we append vnodes to a "workitem" queue. When running with a soft 242 * updates implementation, most pending metadata dependencies should 243 * not wait for more than a few seconds. Thus, mounted on block devices 244 * are delayed only about a half the time that file data is delayed. 245 * Similarly, directory updates are more critical, so are only delayed 246 * about a third the time that file data is delayed. Thus, there are 247 * SYNCER_MAXDELAY queues that are processed round-robin at a rate of 248 * one each second (driven off the filesystem syncer process). The 249 * syncer_delayno variable indicates the next queue that is to be processed. 250 * Items that need to be processed soon are placed in this queue: 251 * 252 * syncer_workitem_pending[syncer_delayno] 253 * 254 * A delay of fifteen seconds is done by placing the request fifteen 255 * entries later in the queue: 256 * 257 * syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask] 258 * 259 */ 260 static int syncer_delayno; 261 static long syncer_mask; 262 LIST_HEAD(synclist, bufobj); 263 static struct synclist *syncer_workitem_pending; 264 /* 265 * The sync_mtx protects: 266 * bo->bo_synclist 267 * sync_vnode_count 268 * syncer_delayno 269 * syncer_state 270 * syncer_workitem_pending 271 * syncer_worklist_len 272 * rushjob 273 */ 274 static struct mtx sync_mtx; 275 static struct cv sync_wakeup; 276 277 #define SYNCER_MAXDELAY 32 278 static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */ 279 static int syncdelay = 30; /* max time to delay syncing data */ 280 static int filedelay = 30; /* time to delay syncing files */ 281 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0, 282 "Time to delay syncing files (in seconds)"); 283 static int dirdelay = 29; /* time to delay syncing directories */ 284 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0, 285 "Time to delay syncing directories (in seconds)"); 286 static int metadelay = 28; /* time to delay syncing metadata */ 287 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0, 288 "Time to delay syncing metadata (in seconds)"); 289 static int rushjob; /* number of slots to run ASAP */ 290 static int stat_rush_requests; /* number of times I/O speeded up */ 291 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0, 292 "Number of times I/O speeded up (rush requests)"); 293 294 #define VDBATCH_SIZE 8 295 struct vdbatch { 296 u_int index; 297 long freevnodes; 298 struct mtx lock; 299 struct vnode *tab[VDBATCH_SIZE]; 300 }; 301 DPCPU_DEFINE_STATIC(struct vdbatch, vd); 302 303 static void vdbatch_dequeue(struct vnode *vp); 304 305 /* 306 * When shutting down the syncer, run it at four times normal speed. 307 */ 308 #define SYNCER_SHUTDOWN_SPEEDUP 4 309 static int sync_vnode_count; 310 static int syncer_worklist_len; 311 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY } 312 syncer_state; 313 314 /* Target for maximum number of vnodes. */ 315 u_long desiredvnodes; 316 static u_long gapvnodes; /* gap between wanted and desired */ 317 static u_long vhiwat; /* enough extras after expansion */ 318 static u_long vlowat; /* minimal extras before expansion */ 319 static u_long vstir; /* nonzero to stir non-free vnodes */ 320 static volatile int vsmalltrigger = 8; /* pref to keep if > this many pages */ 321 322 static u_long vnlru_read_freevnodes(void); 323 324 /* 325 * Note that no attempt is made to sanitize these parameters. 326 */ 327 static int 328 sysctl_maxvnodes(SYSCTL_HANDLER_ARGS) 329 { 330 u_long val; 331 int error; 332 333 val = desiredvnodes; 334 error = sysctl_handle_long(oidp, &val, 0, req); 335 if (error != 0 || req->newptr == NULL) 336 return (error); 337 338 if (val == desiredvnodes) 339 return (0); 340 mtx_lock(&vnode_list_mtx); 341 desiredvnodes = val; 342 wantfreevnodes = desiredvnodes / 4; 343 vnlru_recalc(); 344 mtx_unlock(&vnode_list_mtx); 345 /* 346 * XXX There is no protection against multiple threads changing 347 * desiredvnodes at the same time. Locking above only helps vnlru and 348 * getnewvnode. 349 */ 350 vfs_hash_changesize(desiredvnodes); 351 cache_changesize(desiredvnodes); 352 return (0); 353 } 354 355 SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes, 356 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, 357 "LU", "Target for maximum number of vnodes"); 358 359 static int 360 sysctl_wantfreevnodes(SYSCTL_HANDLER_ARGS) 361 { 362 u_long val; 363 int error; 364 365 val = wantfreevnodes; 366 error = sysctl_handle_long(oidp, &val, 0, req); 367 if (error != 0 || req->newptr == NULL) 368 return (error); 369 370 if (val == wantfreevnodes) 371 return (0); 372 mtx_lock(&vnode_list_mtx); 373 wantfreevnodes = val; 374 vnlru_recalc(); 375 mtx_unlock(&vnode_list_mtx); 376 return (0); 377 } 378 379 SYSCTL_PROC(_vfs, OID_AUTO, wantfreevnodes, 380 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes, 381 "LU", "Target for minimum number of \"free\" vnodes"); 382 383 SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW, 384 &wantfreevnodes, 0, "Old name for vfs.wantfreevnodes (legacy)"); 385 static int vnlru_nowhere; 386 SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW, 387 &vnlru_nowhere, 0, "Number of times the vnlru process ran without success"); 388 389 static int 390 sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS) 391 { 392 struct vnode *vp; 393 struct nameidata nd; 394 char *buf; 395 unsigned long ndflags; 396 int error; 397 398 if (req->newptr == NULL) 399 return (EINVAL); 400 if (req->newlen >= PATH_MAX) 401 return (E2BIG); 402 403 buf = malloc(PATH_MAX, M_TEMP, M_WAITOK); 404 error = SYSCTL_IN(req, buf, req->newlen); 405 if (error != 0) 406 goto out; 407 408 buf[req->newlen] = '\0'; 409 410 ndflags = LOCKLEAF | NOFOLLOW | AUDITVNODE1 | SAVENAME; 411 NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE, buf, curthread); 412 if ((error = namei(&nd)) != 0) 413 goto out; 414 vp = nd.ni_vp; 415 416 if (VN_IS_DOOMED(vp)) { 417 /* 418 * This vnode is being recycled. Return != 0 to let the caller 419 * know that the sysctl had no effect. Return EAGAIN because a 420 * subsequent call will likely succeed (since namei will create 421 * a new vnode if necessary) 422 */ 423 error = EAGAIN; 424 goto putvnode; 425 } 426 427 counter_u64_add(recycles_count, 1); 428 vgone(vp); 429 putvnode: 430 NDFREE(&nd, 0); 431 out: 432 free(buf, M_TEMP); 433 return (error); 434 } 435 436 static int 437 sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS) 438 { 439 struct thread *td = curthread; 440 struct vnode *vp; 441 struct file *fp; 442 int error; 443 int fd; 444 445 if (req->newptr == NULL) 446 return (EBADF); 447 448 error = sysctl_handle_int(oidp, &fd, 0, req); 449 if (error != 0) 450 return (error); 451 error = getvnode(curthread, fd, &cap_fcntl_rights, &fp); 452 if (error != 0) 453 return (error); 454 vp = fp->f_vnode; 455 456 error = vn_lock(vp, LK_EXCLUSIVE); 457 if (error != 0) 458 goto drop; 459 460 counter_u64_add(recycles_count, 1); 461 vgone(vp); 462 VOP_UNLOCK(vp); 463 drop: 464 fdrop(fp, td); 465 return (error); 466 } 467 468 SYSCTL_PROC(_debug, OID_AUTO, try_reclaim_vnode, 469 CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 470 sysctl_try_reclaim_vnode, "A", "Try to reclaim a vnode by its pathname"); 471 SYSCTL_PROC(_debug, OID_AUTO, ftry_reclaim_vnode, 472 CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 473 sysctl_ftry_reclaim_vnode, "I", 474 "Try to reclaim a vnode by its file descriptor"); 475 476 /* Shift count for (uintptr_t)vp to initialize vp->v_hash. */ 477 static int vnsz2log; 478 479 /* 480 * Support for the bufobj clean & dirty pctrie. 481 */ 482 static void * 483 buf_trie_alloc(struct pctrie *ptree) 484 { 485 return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT)); 486 } 487 488 static void 489 buf_trie_free(struct pctrie *ptree, void *node) 490 { 491 uma_zfree_smr(buf_trie_zone, node); 492 } 493 PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free, 494 buf_trie_smr); 495 496 /* 497 * Initialize the vnode management data structures. 498 * 499 * Reevaluate the following cap on the number of vnodes after the physical 500 * memory size exceeds 512GB. In the limit, as the physical memory size 501 * grows, the ratio of the memory size in KB to vnodes approaches 64:1. 502 */ 503 #ifndef MAXVNODES_MAX 504 #define MAXVNODES_MAX (512UL * 1024 * 1024 / 64) /* 8M */ 505 #endif 506 507 static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); 508 509 static struct vnode * 510 vn_alloc_marker(struct mount *mp) 511 { 512 struct vnode *vp; 513 514 vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO); 515 vp->v_type = VMARKER; 516 vp->v_mount = mp; 517 518 return (vp); 519 } 520 521 static void 522 vn_free_marker(struct vnode *vp) 523 { 524 525 MPASS(vp->v_type == VMARKER); 526 free(vp, M_VNODE_MARKER); 527 } 528 529 /* 530 * Initialize a vnode as it first enters the zone. 531 */ 532 static int 533 vnode_init(void *mem, int size, int flags) 534 { 535 struct vnode *vp; 536 537 vp = mem; 538 bzero(vp, size); 539 /* 540 * Setup locks. 541 */ 542 vp->v_vnlock = &vp->v_lock; 543 mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF); 544 /* 545 * By default, don't allow shared locks unless filesystems opt-in. 546 */ 547 lockinit(vp->v_vnlock, PVFS, "vnode", VLKTIMEOUT, 548 LK_NOSHARE | LK_IS_VNODE); 549 /* 550 * Initialize bufobj. 551 */ 552 bufobj_init(&vp->v_bufobj, vp); 553 /* 554 * Initialize namecache. 555 */ 556 cache_vnode_init(vp); 557 /* 558 * Initialize rangelocks. 559 */ 560 rangelock_init(&vp->v_rl); 561 562 vp->v_dbatchcpu = NOCPU; 563 564 mtx_lock(&vnode_list_mtx); 565 TAILQ_INSERT_BEFORE(vnode_list_free_marker, vp, v_vnodelist); 566 mtx_unlock(&vnode_list_mtx); 567 return (0); 568 } 569 570 /* 571 * Free a vnode when it is cleared from the zone. 572 */ 573 static void 574 vnode_fini(void *mem, int size) 575 { 576 struct vnode *vp; 577 struct bufobj *bo; 578 579 vp = mem; 580 vdbatch_dequeue(vp); 581 mtx_lock(&vnode_list_mtx); 582 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 583 mtx_unlock(&vnode_list_mtx); 584 rangelock_destroy(&vp->v_rl); 585 lockdestroy(vp->v_vnlock); 586 mtx_destroy(&vp->v_interlock); 587 bo = &vp->v_bufobj; 588 rw_destroy(BO_LOCKPTR(bo)); 589 } 590 591 /* 592 * Provide the size of NFS nclnode and NFS fh for calculation of the 593 * vnode memory consumption. The size is specified directly to 594 * eliminate dependency on NFS-private header. 595 * 596 * Other filesystems may use bigger or smaller (like UFS and ZFS) 597 * private inode data, but the NFS-based estimation is ample enough. 598 * Still, we care about differences in the size between 64- and 32-bit 599 * platforms. 600 * 601 * Namecache structure size is heuristically 602 * sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1. 603 */ 604 #ifdef _LP64 605 #define NFS_NCLNODE_SZ (528 + 64) 606 #define NC_SZ 148 607 #else 608 #define NFS_NCLNODE_SZ (360 + 32) 609 #define NC_SZ 92 610 #endif 611 612 static void 613 vntblinit(void *dummy __unused) 614 { 615 struct vdbatch *vd; 616 int cpu, physvnodes, virtvnodes; 617 u_int i; 618 619 /* 620 * Desiredvnodes is a function of the physical memory size and the 621 * kernel's heap size. Generally speaking, it scales with the 622 * physical memory size. The ratio of desiredvnodes to the physical 623 * memory size is 1:16 until desiredvnodes exceeds 98,304. 624 * Thereafter, the 625 * marginal ratio of desiredvnodes to the physical memory size is 626 * 1:64. However, desiredvnodes is limited by the kernel's heap 627 * size. The memory required by desiredvnodes vnodes and vm objects 628 * must not exceed 1/10th of the kernel's heap size. 629 */ 630 physvnodes = maxproc + pgtok(vm_cnt.v_page_count) / 64 + 631 3 * min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 64; 632 virtvnodes = vm_kmem_size / (10 * (sizeof(struct vm_object) + 633 sizeof(struct vnode) + NC_SZ * ncsizefactor + NFS_NCLNODE_SZ)); 634 desiredvnodes = min(physvnodes, virtvnodes); 635 if (desiredvnodes > MAXVNODES_MAX) { 636 if (bootverbose) 637 printf("Reducing kern.maxvnodes %lu -> %lu\n", 638 desiredvnodes, MAXVNODES_MAX); 639 desiredvnodes = MAXVNODES_MAX; 640 } 641 wantfreevnodes = desiredvnodes / 4; 642 mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF); 643 TAILQ_INIT(&vnode_list); 644 mtx_init(&vnode_list_mtx, "vnode_list", NULL, MTX_DEF); 645 /* 646 * The lock is taken to appease WITNESS. 647 */ 648 mtx_lock(&vnode_list_mtx); 649 vnlru_recalc(); 650 mtx_unlock(&vnode_list_mtx); 651 vnode_list_free_marker = vn_alloc_marker(NULL); 652 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist); 653 vnode_list_reclaim_marker = vn_alloc_marker(NULL); 654 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist); 655 vnode_zone = uma_zcreate("VNODE", sizeof (struct vnode), NULL, NULL, 656 vnode_init, vnode_fini, UMA_ALIGN_PTR, 0); 657 uma_zone_set_smr(vnode_zone, vfs_smr); 658 vnodepoll_zone = uma_zcreate("VNODEPOLL", sizeof (struct vpollinfo), 659 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 660 /* 661 * Preallocate enough nodes to support one-per buf so that 662 * we can not fail an insert. reassignbuf() callers can not 663 * tolerate the insertion failure. 664 */ 665 buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(), 666 NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR, 667 UMA_ZONE_NOFREE | UMA_ZONE_SMR); 668 buf_trie_smr = uma_zone_get_smr(buf_trie_zone); 669 uma_prealloc(buf_trie_zone, nbuf); 670 671 vnodes_created = counter_u64_alloc(M_WAITOK); 672 recycles_count = counter_u64_alloc(M_WAITOK); 673 recycles_free_count = counter_u64_alloc(M_WAITOK); 674 deferred_inact = counter_u64_alloc(M_WAITOK); 675 676 /* 677 * Initialize the filesystem syncer. 678 */ 679 syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE, 680 &syncer_mask); 681 syncer_maxdelay = syncer_mask + 1; 682 mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF); 683 cv_init(&sync_wakeup, "syncer"); 684 for (i = 1; i <= sizeof(struct vnode); i <<= 1) 685 vnsz2log++; 686 vnsz2log--; 687 688 CPU_FOREACH(cpu) { 689 vd = DPCPU_ID_PTR((cpu), vd); 690 bzero(vd, sizeof(*vd)); 691 mtx_init(&vd->lock, "vdbatch", NULL, MTX_DEF); 692 } 693 } 694 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL); 695 696 /* 697 * Mark a mount point as busy. Used to synchronize access and to delay 698 * unmounting. Eventually, mountlist_mtx is not released on failure. 699 * 700 * vfs_busy() is a custom lock, it can block the caller. 701 * vfs_busy() only sleeps if the unmount is active on the mount point. 702 * For a mountpoint mp, vfs_busy-enforced lock is before lock of any 703 * vnode belonging to mp. 704 * 705 * Lookup uses vfs_busy() to traverse mount points. 706 * root fs var fs 707 * / vnode lock A / vnode lock (/var) D 708 * /var vnode lock B /log vnode lock(/var/log) E 709 * vfs_busy lock C vfs_busy lock F 710 * 711 * Within each file system, the lock order is C->A->B and F->D->E. 712 * 713 * When traversing across mounts, the system follows that lock order: 714 * 715 * C->A->B 716 * | 717 * +->F->D->E 718 * 719 * The lookup() process for namei("/var") illustrates the process: 720 * VOP_LOOKUP() obtains B while A is held 721 * vfs_busy() obtains a shared lock on F while A and B are held 722 * vput() releases lock on B 723 * vput() releases lock on A 724 * VFS_ROOT() obtains lock on D while shared lock on F is held 725 * vfs_unbusy() releases shared lock on F 726 * vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A. 727 * Attempt to lock A (instead of vp_crossmp) while D is held would 728 * violate the global order, causing deadlocks. 729 * 730 * dounmount() locks B while F is drained. 731 */ 732 int 733 vfs_busy(struct mount *mp, int flags) 734 { 735 736 MPASS((flags & ~MBF_MASK) == 0); 737 CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags); 738 739 if (vfs_op_thread_enter(mp)) { 740 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 741 MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0); 742 MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0); 743 vfs_mp_count_add_pcpu(mp, ref, 1); 744 vfs_mp_count_add_pcpu(mp, lockref, 1); 745 vfs_op_thread_exit(mp); 746 if (flags & MBF_MNTLSTLOCK) 747 mtx_unlock(&mountlist_mtx); 748 return (0); 749 } 750 751 MNT_ILOCK(mp); 752 vfs_assert_mount_counters(mp); 753 MNT_REF(mp); 754 /* 755 * If mount point is currently being unmounted, sleep until the 756 * mount point fate is decided. If thread doing the unmounting fails, 757 * it will clear MNTK_UNMOUNT flag before waking us up, indicating 758 * that this mount point has survived the unmount attempt and vfs_busy 759 * should retry. Otherwise the unmounter thread will set MNTK_REFEXPIRE 760 * flag in addition to MNTK_UNMOUNT, indicating that mount point is 761 * about to be really destroyed. vfs_busy needs to release its 762 * reference on the mount point in this case and return with ENOENT, 763 * telling the caller that mount mount it tried to busy is no longer 764 * valid. 765 */ 766 while (mp->mnt_kern_flag & MNTK_UNMOUNT) { 767 if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) { 768 MNT_REL(mp); 769 MNT_IUNLOCK(mp); 770 CTR1(KTR_VFS, "%s: failed busying before sleeping", 771 __func__); 772 return (ENOENT); 773 } 774 if (flags & MBF_MNTLSTLOCK) 775 mtx_unlock(&mountlist_mtx); 776 mp->mnt_kern_flag |= MNTK_MWAIT; 777 msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0); 778 if (flags & MBF_MNTLSTLOCK) 779 mtx_lock(&mountlist_mtx); 780 MNT_ILOCK(mp); 781 } 782 if (flags & MBF_MNTLSTLOCK) 783 mtx_unlock(&mountlist_mtx); 784 mp->mnt_lockref++; 785 MNT_IUNLOCK(mp); 786 return (0); 787 } 788 789 /* 790 * Free a busy filesystem. 791 */ 792 void 793 vfs_unbusy(struct mount *mp) 794 { 795 int c; 796 797 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 798 799 if (vfs_op_thread_enter(mp)) { 800 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 801 vfs_mp_count_sub_pcpu(mp, lockref, 1); 802 vfs_mp_count_sub_pcpu(mp, ref, 1); 803 vfs_op_thread_exit(mp); 804 return; 805 } 806 807 MNT_ILOCK(mp); 808 vfs_assert_mount_counters(mp); 809 MNT_REL(mp); 810 c = --mp->mnt_lockref; 811 if (mp->mnt_vfs_ops == 0) { 812 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 813 MNT_IUNLOCK(mp); 814 return; 815 } 816 if (c < 0) 817 vfs_dump_mount_counters(mp); 818 if (c == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) { 819 MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT); 820 CTR1(KTR_VFS, "%s: waking up waiters", __func__); 821 mp->mnt_kern_flag &= ~MNTK_DRAINING; 822 wakeup(&mp->mnt_lockref); 823 } 824 MNT_IUNLOCK(mp); 825 } 826 827 /* 828 * Lookup a mount point by filesystem identifier. 829 */ 830 struct mount * 831 vfs_getvfs(fsid_t *fsid) 832 { 833 struct mount *mp; 834 835 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 836 mtx_lock(&mountlist_mtx); 837 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 838 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 839 vfs_ref(mp); 840 mtx_unlock(&mountlist_mtx); 841 return (mp); 842 } 843 } 844 mtx_unlock(&mountlist_mtx); 845 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 846 return ((struct mount *) 0); 847 } 848 849 /* 850 * Lookup a mount point by filesystem identifier, busying it before 851 * returning. 852 * 853 * To avoid congestion on mountlist_mtx, implement simple direct-mapped 854 * cache for popular filesystem identifiers. The cache is lockess, using 855 * the fact that struct mount's are never freed. In worst case we may 856 * get pointer to unmounted or even different filesystem, so we have to 857 * check what we got, and go slow way if so. 858 */ 859 struct mount * 860 vfs_busyfs(fsid_t *fsid) 861 { 862 #define FSID_CACHE_SIZE 256 863 typedef struct mount * volatile vmp_t; 864 static vmp_t cache[FSID_CACHE_SIZE]; 865 struct mount *mp; 866 int error; 867 uint32_t hash; 868 869 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 870 hash = fsid->val[0] ^ fsid->val[1]; 871 hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1); 872 mp = cache[hash]; 873 if (mp == NULL || fsidcmp(&mp->mnt_stat.f_fsid, fsid) != 0) 874 goto slow; 875 if (vfs_busy(mp, 0) != 0) { 876 cache[hash] = NULL; 877 goto slow; 878 } 879 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) 880 return (mp); 881 else 882 vfs_unbusy(mp); 883 884 slow: 885 mtx_lock(&mountlist_mtx); 886 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 887 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 888 error = vfs_busy(mp, MBF_MNTLSTLOCK); 889 if (error) { 890 cache[hash] = NULL; 891 mtx_unlock(&mountlist_mtx); 892 return (NULL); 893 } 894 cache[hash] = mp; 895 return (mp); 896 } 897 } 898 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 899 mtx_unlock(&mountlist_mtx); 900 return ((struct mount *) 0); 901 } 902 903 /* 904 * Check if a user can access privileged mount options. 905 */ 906 int 907 vfs_suser(struct mount *mp, struct thread *td) 908 { 909 int error; 910 911 if (jailed(td->td_ucred)) { 912 /* 913 * If the jail of the calling thread lacks permission for 914 * this type of file system, deny immediately. 915 */ 916 if (!prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag)) 917 return (EPERM); 918 919 /* 920 * If the file system was mounted outside the jail of the 921 * calling thread, deny immediately. 922 */ 923 if (prison_check(td->td_ucred, mp->mnt_cred) != 0) 924 return (EPERM); 925 } 926 927 /* 928 * If file system supports delegated administration, we don't check 929 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified 930 * by the file system itself. 931 * If this is not the user that did original mount, we check for 932 * the PRIV_VFS_MOUNT_OWNER privilege. 933 */ 934 if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) && 935 mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) { 936 if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0) 937 return (error); 938 } 939 return (0); 940 } 941 942 /* 943 * Get a new unique fsid. Try to make its val[0] unique, since this value 944 * will be used to create fake device numbers for stat(). Also try (but 945 * not so hard) make its val[0] unique mod 2^16, since some emulators only 946 * support 16-bit device numbers. We end up with unique val[0]'s for the 947 * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls. 948 * 949 * Keep in mind that several mounts may be running in parallel. Starting 950 * the search one past where the previous search terminated is both a 951 * micro-optimization and a defense against returning the same fsid to 952 * different mounts. 953 */ 954 void 955 vfs_getnewfsid(struct mount *mp) 956 { 957 static uint16_t mntid_base; 958 struct mount *nmp; 959 fsid_t tfsid; 960 int mtype; 961 962 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 963 mtx_lock(&mntid_mtx); 964 mtype = mp->mnt_vfc->vfc_typenum; 965 tfsid.val[1] = mtype; 966 mtype = (mtype & 0xFF) << 24; 967 for (;;) { 968 tfsid.val[0] = makedev(255, 969 mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF)); 970 mntid_base++; 971 if ((nmp = vfs_getvfs(&tfsid)) == NULL) 972 break; 973 vfs_rel(nmp); 974 } 975 mp->mnt_stat.f_fsid.val[0] = tfsid.val[0]; 976 mp->mnt_stat.f_fsid.val[1] = tfsid.val[1]; 977 mtx_unlock(&mntid_mtx); 978 } 979 980 /* 981 * Knob to control the precision of file timestamps: 982 * 983 * 0 = seconds only; nanoseconds zeroed. 984 * 1 = seconds and nanoseconds, accurate within 1/HZ. 985 * 2 = seconds and nanoseconds, truncated to microseconds. 986 * >=3 = seconds and nanoseconds, maximum precision. 987 */ 988 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC }; 989 990 static int timestamp_precision = TSP_USEC; 991 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW, 992 ×tamp_precision, 0, "File timestamp precision (0: seconds, " 993 "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, " 994 "3+: sec + ns (max. precision))"); 995 996 /* 997 * Get a current timestamp. 998 */ 999 void 1000 vfs_timestamp(struct timespec *tsp) 1001 { 1002 struct timeval tv; 1003 1004 switch (timestamp_precision) { 1005 case TSP_SEC: 1006 tsp->tv_sec = time_second; 1007 tsp->tv_nsec = 0; 1008 break; 1009 case TSP_HZ: 1010 getnanotime(tsp); 1011 break; 1012 case TSP_USEC: 1013 microtime(&tv); 1014 TIMEVAL_TO_TIMESPEC(&tv, tsp); 1015 break; 1016 case TSP_NSEC: 1017 default: 1018 nanotime(tsp); 1019 break; 1020 } 1021 } 1022 1023 /* 1024 * Set vnode attributes to VNOVAL 1025 */ 1026 void 1027 vattr_null(struct vattr *vap) 1028 { 1029 1030 vap->va_type = VNON; 1031 vap->va_size = VNOVAL; 1032 vap->va_bytes = VNOVAL; 1033 vap->va_mode = VNOVAL; 1034 vap->va_nlink = VNOVAL; 1035 vap->va_uid = VNOVAL; 1036 vap->va_gid = VNOVAL; 1037 vap->va_fsid = VNOVAL; 1038 vap->va_fileid = VNOVAL; 1039 vap->va_blocksize = VNOVAL; 1040 vap->va_rdev = VNOVAL; 1041 vap->va_atime.tv_sec = VNOVAL; 1042 vap->va_atime.tv_nsec = VNOVAL; 1043 vap->va_mtime.tv_sec = VNOVAL; 1044 vap->va_mtime.tv_nsec = VNOVAL; 1045 vap->va_ctime.tv_sec = VNOVAL; 1046 vap->va_ctime.tv_nsec = VNOVAL; 1047 vap->va_birthtime.tv_sec = VNOVAL; 1048 vap->va_birthtime.tv_nsec = VNOVAL; 1049 vap->va_flags = VNOVAL; 1050 vap->va_gen = VNOVAL; 1051 vap->va_vaflags = 0; 1052 } 1053 1054 /* 1055 * Try to reduce the total number of vnodes. 1056 * 1057 * This routine (and its user) are buggy in at least the following ways: 1058 * - all parameters were picked years ago when RAM sizes were significantly 1059 * smaller 1060 * - it can pick vnodes based on pages used by the vm object, but filesystems 1061 * like ZFS don't use it making the pick broken 1062 * - since ZFS has its own aging policy it gets partially combated by this one 1063 * - a dedicated method should be provided for filesystems to let them decide 1064 * whether the vnode should be recycled 1065 * 1066 * This routine is called when we have too many vnodes. It attempts 1067 * to free <count> vnodes and will potentially free vnodes that still 1068 * have VM backing store (VM backing store is typically the cause 1069 * of a vnode blowout so we want to do this). Therefore, this operation 1070 * is not considered cheap. 1071 * 1072 * A number of conditions may prevent a vnode from being reclaimed. 1073 * the buffer cache may have references on the vnode, a directory 1074 * vnode may still have references due to the namei cache representing 1075 * underlying files, or the vnode may be in active use. It is not 1076 * desirable to reuse such vnodes. These conditions may cause the 1077 * number of vnodes to reach some minimum value regardless of what 1078 * you set kern.maxvnodes to. Do not set kern.maxvnodes too low. 1079 * 1080 * @param reclaim_nc_src Only reclaim directories with outgoing namecache 1081 * entries if this argument is strue 1082 * @param trigger Only reclaim vnodes with fewer than this many resident 1083 * pages. 1084 * @param target How many vnodes to reclaim. 1085 * @return The number of vnodes that were reclaimed. 1086 */ 1087 static int 1088 vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) 1089 { 1090 struct vnode *vp, *mvp; 1091 struct mount *mp; 1092 struct vm_object *object; 1093 u_long done; 1094 bool retried; 1095 1096 mtx_assert(&vnode_list_mtx, MA_OWNED); 1097 1098 retried = false; 1099 done = 0; 1100 1101 mvp = vnode_list_reclaim_marker; 1102 restart: 1103 vp = mvp; 1104 while (done < target) { 1105 vp = TAILQ_NEXT(vp, v_vnodelist); 1106 if (__predict_false(vp == NULL)) 1107 break; 1108 1109 if (__predict_false(vp->v_type == VMARKER)) 1110 continue; 1111 1112 /* 1113 * If it's been deconstructed already, it's still 1114 * referenced, or it exceeds the trigger, skip it. 1115 * Also skip free vnodes. We are trying to make space 1116 * to expand the free list, not reduce it. 1117 */ 1118 if (vp->v_usecount > 0 || vp->v_holdcnt == 0 || 1119 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src))) 1120 goto next_iter; 1121 1122 if (vp->v_type == VBAD || vp->v_type == VNON) 1123 goto next_iter; 1124 1125 object = atomic_load_ptr(&vp->v_object); 1126 if (object == NULL || object->resident_page_count > trigger) { 1127 goto next_iter; 1128 } 1129 1130 if (!vhold_recycle(vp)) 1131 goto next_iter; 1132 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1133 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1134 mtx_unlock(&vnode_list_mtx); 1135 1136 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 1137 vdrop(vp); 1138 goto next_iter_unlocked; 1139 } 1140 if (VOP_LOCK(vp, LK_EXCLUSIVE|LK_NOWAIT) != 0) { 1141 vdrop(vp); 1142 vn_finished_write(mp); 1143 goto next_iter_unlocked; 1144 } 1145 1146 VI_LOCK(vp); 1147 if (vp->v_usecount > 0 || 1148 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) || 1149 (vp->v_object != NULL && 1150 vp->v_object->resident_page_count > trigger)) { 1151 VOP_UNLOCK(vp); 1152 vdropl(vp); 1153 vn_finished_write(mp); 1154 goto next_iter_unlocked; 1155 } 1156 counter_u64_add(recycles_count, 1); 1157 vgonel(vp); 1158 VOP_UNLOCK(vp); 1159 vdropl(vp); 1160 vn_finished_write(mp); 1161 done++; 1162 next_iter_unlocked: 1163 if (should_yield()) 1164 kern_yield(PRI_USER); 1165 mtx_lock(&vnode_list_mtx); 1166 goto restart; 1167 next_iter: 1168 MPASS(vp->v_type != VMARKER); 1169 if (!should_yield()) 1170 continue; 1171 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1172 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1173 mtx_unlock(&vnode_list_mtx); 1174 kern_yield(PRI_USER); 1175 mtx_lock(&vnode_list_mtx); 1176 goto restart; 1177 } 1178 if (done == 0 && !retried) { 1179 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1180 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); 1181 retried = true; 1182 goto restart; 1183 } 1184 return (done); 1185 } 1186 1187 static int max_vnlru_free = 10000; /* limit on vnode free requests per call */ 1188 SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_vnlru_free, 1189 0, 1190 "limit on vnode free requests per call to the vnlru_free routine"); 1191 1192 /* 1193 * Attempt to reduce the free list by the requested amount. 1194 */ 1195 static int 1196 vnlru_free_locked(int count, struct vfsops *mnt_op) 1197 { 1198 struct vnode *vp, *mvp; 1199 struct mount *mp; 1200 int ocount; 1201 1202 mtx_assert(&vnode_list_mtx, MA_OWNED); 1203 if (count > max_vnlru_free) 1204 count = max_vnlru_free; 1205 ocount = count; 1206 mvp = vnode_list_free_marker; 1207 restart: 1208 vp = mvp; 1209 while (count > 0) { 1210 vp = TAILQ_NEXT(vp, v_vnodelist); 1211 if (__predict_false(vp == NULL)) { 1212 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1213 TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist); 1214 break; 1215 } 1216 if (__predict_false(vp->v_type == VMARKER)) 1217 continue; 1218 1219 /* 1220 * Don't recycle if our vnode is from different type 1221 * of mount point. Note that mp is type-safe, the 1222 * check does not reach unmapped address even if 1223 * vnode is reclaimed. 1224 * Don't recycle if we can't get the interlock without 1225 * blocking. 1226 */ 1227 if (vp->v_holdcnt > 0 || (mnt_op != NULL && (mp = vp->v_mount) != NULL && 1228 mp->mnt_op != mnt_op)) { 1229 continue; 1230 } 1231 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1232 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1233 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1234 continue; 1235 } 1236 if (!vhold_recycle(vp)) 1237 continue; 1238 count--; 1239 mtx_unlock(&vnode_list_mtx); 1240 vtryrecycle(vp); 1241 mtx_lock(&vnode_list_mtx); 1242 goto restart; 1243 } 1244 return (ocount - count); 1245 } 1246 1247 void 1248 vnlru_free(int count, struct vfsops *mnt_op) 1249 { 1250 1251 mtx_lock(&vnode_list_mtx); 1252 vnlru_free_locked(count, mnt_op); 1253 mtx_unlock(&vnode_list_mtx); 1254 } 1255 1256 static void 1257 vnlru_recalc(void) 1258 { 1259 1260 mtx_assert(&vnode_list_mtx, MA_OWNED); 1261 gapvnodes = imax(desiredvnodes - wantfreevnodes, 100); 1262 vhiwat = gapvnodes / 11; /* 9% -- just under the 10% in vlrureclaim() */ 1263 vlowat = vhiwat / 2; 1264 } 1265 1266 /* 1267 * Attempt to recycle vnodes in a context that is always safe to block. 1268 * Calling vlrurecycle() from the bowels of filesystem code has some 1269 * interesting deadlock problems. 1270 */ 1271 static struct proc *vnlruproc; 1272 static int vnlruproc_sig; 1273 1274 /* 1275 * The main freevnodes counter is only updated when threads requeue their vnode 1276 * batches. CPUs are conditionally walked to compute a more accurate total. 1277 * 1278 * Limit how much of a slop are we willing to tolerate. Note: the actual value 1279 * at any given moment can still exceed slop, but it should not be by significant 1280 * margin in practice. 1281 */ 1282 #define VNLRU_FREEVNODES_SLOP 128 1283 1284 static __inline void 1285 vn_freevnodes_inc(void) 1286 { 1287 struct vdbatch *vd; 1288 1289 critical_enter(); 1290 vd = DPCPU_PTR(vd); 1291 vd->freevnodes++; 1292 critical_exit(); 1293 } 1294 1295 static __inline void 1296 vn_freevnodes_dec(void) 1297 { 1298 struct vdbatch *vd; 1299 1300 critical_enter(); 1301 vd = DPCPU_PTR(vd); 1302 vd->freevnodes--; 1303 critical_exit(); 1304 } 1305 1306 static u_long 1307 vnlru_read_freevnodes(void) 1308 { 1309 struct vdbatch *vd; 1310 long slop; 1311 int cpu; 1312 1313 mtx_assert(&vnode_list_mtx, MA_OWNED); 1314 if (freevnodes > freevnodes_old) 1315 slop = freevnodes - freevnodes_old; 1316 else 1317 slop = freevnodes_old - freevnodes; 1318 if (slop < VNLRU_FREEVNODES_SLOP) 1319 return (freevnodes >= 0 ? freevnodes : 0); 1320 freevnodes_old = freevnodes; 1321 CPU_FOREACH(cpu) { 1322 vd = DPCPU_ID_PTR((cpu), vd); 1323 freevnodes_old += vd->freevnodes; 1324 } 1325 return (freevnodes_old >= 0 ? freevnodes_old : 0); 1326 } 1327 1328 static bool 1329 vnlru_under(u_long rnumvnodes, u_long limit) 1330 { 1331 u_long rfreevnodes, space; 1332 1333 if (__predict_false(rnumvnodes > desiredvnodes)) 1334 return (true); 1335 1336 space = desiredvnodes - rnumvnodes; 1337 if (space < limit) { 1338 rfreevnodes = vnlru_read_freevnodes(); 1339 if (rfreevnodes > wantfreevnodes) 1340 space += rfreevnodes - wantfreevnodes; 1341 } 1342 return (space < limit); 1343 } 1344 1345 static bool 1346 vnlru_under_unlocked(u_long rnumvnodes, u_long limit) 1347 { 1348 long rfreevnodes, space; 1349 1350 if (__predict_false(rnumvnodes > desiredvnodes)) 1351 return (true); 1352 1353 space = desiredvnodes - rnumvnodes; 1354 if (space < limit) { 1355 rfreevnodes = atomic_load_long(&freevnodes); 1356 if (rfreevnodes > wantfreevnodes) 1357 space += rfreevnodes - wantfreevnodes; 1358 } 1359 return (space < limit); 1360 } 1361 1362 static void 1363 vnlru_kick(void) 1364 { 1365 1366 mtx_assert(&vnode_list_mtx, MA_OWNED); 1367 if (vnlruproc_sig == 0) { 1368 vnlruproc_sig = 1; 1369 wakeup(vnlruproc); 1370 } 1371 } 1372 1373 static void 1374 vnlru_proc(void) 1375 { 1376 u_long rnumvnodes, rfreevnodes, target; 1377 unsigned long onumvnodes; 1378 int done, force, trigger, usevnodes; 1379 bool reclaim_nc_src, want_reread; 1380 1381 EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, vnlruproc, 1382 SHUTDOWN_PRI_FIRST); 1383 1384 force = 0; 1385 want_reread = false; 1386 for (;;) { 1387 kproc_suspend_check(vnlruproc); 1388 mtx_lock(&vnode_list_mtx); 1389 rnumvnodes = atomic_load_long(&numvnodes); 1390 1391 if (want_reread) { 1392 force = vnlru_under(numvnodes, vhiwat) ? 1 : 0; 1393 want_reread = false; 1394 } 1395 1396 /* 1397 * If numvnodes is too large (due to desiredvnodes being 1398 * adjusted using its sysctl, or emergency growth), first 1399 * try to reduce it by discarding from the free list. 1400 */ 1401 if (rnumvnodes > desiredvnodes) { 1402 vnlru_free_locked(rnumvnodes - desiredvnodes, NULL); 1403 rnumvnodes = atomic_load_long(&numvnodes); 1404 } 1405 /* 1406 * Sleep if the vnode cache is in a good state. This is 1407 * when it is not over-full and has space for about a 4% 1408 * or 9% expansion (by growing its size or inexcessively 1409 * reducing its free list). Otherwise, try to reclaim 1410 * space for a 10% expansion. 1411 */ 1412 if (vstir && force == 0) { 1413 force = 1; 1414 vstir = 0; 1415 } 1416 if (force == 0 && !vnlru_under(rnumvnodes, vlowat)) { 1417 vnlruproc_sig = 0; 1418 wakeup(&vnlruproc_sig); 1419 msleep(vnlruproc, &vnode_list_mtx, 1420 PVFS|PDROP, "vlruwt", hz); 1421 continue; 1422 } 1423 rfreevnodes = vnlru_read_freevnodes(); 1424 1425 onumvnodes = rnumvnodes; 1426 /* 1427 * Calculate parameters for recycling. These are the same 1428 * throughout the loop to give some semblance of fairness. 1429 * The trigger point is to avoid recycling vnodes with lots 1430 * of resident pages. We aren't trying to free memory; we 1431 * are trying to recycle or at least free vnodes. 1432 */ 1433 if (rnumvnodes <= desiredvnodes) 1434 usevnodes = rnumvnodes - rfreevnodes; 1435 else 1436 usevnodes = rnumvnodes; 1437 if (usevnodes <= 0) 1438 usevnodes = 1; 1439 /* 1440 * The trigger value is is chosen to give a conservatively 1441 * large value to ensure that it alone doesn't prevent 1442 * making progress. The value can easily be so large that 1443 * it is effectively infinite in some congested and 1444 * misconfigured cases, and this is necessary. Normally 1445 * it is about 8 to 100 (pages), which is quite large. 1446 */ 1447 trigger = vm_cnt.v_page_count * 2 / usevnodes; 1448 if (force < 2) 1449 trigger = vsmalltrigger; 1450 reclaim_nc_src = force >= 3; 1451 target = rnumvnodes * (int64_t)gapvnodes / imax(desiredvnodes, 1); 1452 target = target / 10 + 1; 1453 done = vlrureclaim(reclaim_nc_src, trigger, target); 1454 mtx_unlock(&vnode_list_mtx); 1455 if (onumvnodes > desiredvnodes && numvnodes <= desiredvnodes) 1456 uma_reclaim(UMA_RECLAIM_DRAIN); 1457 if (done == 0) { 1458 if (force == 0 || force == 1) { 1459 force = 2; 1460 continue; 1461 } 1462 if (force == 2) { 1463 force = 3; 1464 continue; 1465 } 1466 want_reread = true; 1467 force = 0; 1468 vnlru_nowhere++; 1469 tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3); 1470 } else { 1471 want_reread = true; 1472 kern_yield(PRI_USER); 1473 } 1474 } 1475 } 1476 1477 static struct kproc_desc vnlru_kp = { 1478 "vnlru", 1479 vnlru_proc, 1480 &vnlruproc 1481 }; 1482 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, 1483 &vnlru_kp); 1484 1485 /* 1486 * Routines having to do with the management of the vnode table. 1487 */ 1488 1489 /* 1490 * Try to recycle a freed vnode. We abort if anyone picks up a reference 1491 * before we actually vgone(). This function must be called with the vnode 1492 * held to prevent the vnode from being returned to the free list midway 1493 * through vgone(). 1494 */ 1495 static int 1496 vtryrecycle(struct vnode *vp) 1497 { 1498 struct mount *vnmp; 1499 1500 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 1501 VNASSERT(vp->v_holdcnt, vp, 1502 ("vtryrecycle: Recycling vp %p without a reference.", vp)); 1503 /* 1504 * This vnode may found and locked via some other list, if so we 1505 * can't recycle it yet. 1506 */ 1507 if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { 1508 CTR2(KTR_VFS, 1509 "%s: impossible to recycle, vp %p lock is already held", 1510 __func__, vp); 1511 vdrop(vp); 1512 return (EWOULDBLOCK); 1513 } 1514 /* 1515 * Don't recycle if its filesystem is being suspended. 1516 */ 1517 if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) { 1518 VOP_UNLOCK(vp); 1519 CTR2(KTR_VFS, 1520 "%s: impossible to recycle, cannot start the write for %p", 1521 __func__, vp); 1522 vdrop(vp); 1523 return (EBUSY); 1524 } 1525 /* 1526 * If we got this far, we need to acquire the interlock and see if 1527 * anyone picked up this vnode from another list. If not, we will 1528 * mark it with DOOMED via vgonel() so that anyone who does find it 1529 * will skip over it. 1530 */ 1531 VI_LOCK(vp); 1532 if (vp->v_usecount) { 1533 VOP_UNLOCK(vp); 1534 vdropl(vp); 1535 vn_finished_write(vnmp); 1536 CTR2(KTR_VFS, 1537 "%s: impossible to recycle, %p is already referenced", 1538 __func__, vp); 1539 return (EBUSY); 1540 } 1541 if (!VN_IS_DOOMED(vp)) { 1542 counter_u64_add(recycles_free_count, 1); 1543 vgonel(vp); 1544 } 1545 VOP_UNLOCK(vp); 1546 vdropl(vp); 1547 vn_finished_write(vnmp); 1548 return (0); 1549 } 1550 1551 /* 1552 * Allocate a new vnode. 1553 * 1554 * The operation never returns an error. Returning an error was disabled 1555 * in r145385 (dated 2005) with the following comment: 1556 * 1557 * XXX Not all VFS_VGET/ffs_vget callers check returns. 1558 * 1559 * Given the age of this commit (almost 15 years at the time of writing this 1560 * comment) restoring the ability to fail requires a significant audit of 1561 * all codepaths. 1562 * 1563 * The routine can try to free a vnode or stall for up to 1 second waiting for 1564 * vnlru to clear things up, but ultimately always performs a M_WAITOK allocation. 1565 */ 1566 static u_long vn_alloc_cyclecount; 1567 1568 static struct vnode * __noinline 1569 vn_alloc_hard(struct mount *mp) 1570 { 1571 u_long rnumvnodes, rfreevnodes; 1572 1573 mtx_lock(&vnode_list_mtx); 1574 rnumvnodes = atomic_load_long(&numvnodes); 1575 if (rnumvnodes + 1 < desiredvnodes) { 1576 vn_alloc_cyclecount = 0; 1577 goto alloc; 1578 } 1579 rfreevnodes = vnlru_read_freevnodes(); 1580 if (vn_alloc_cyclecount++ >= rfreevnodes) { 1581 vn_alloc_cyclecount = 0; 1582 vstir = 1; 1583 } 1584 /* 1585 * Grow the vnode cache if it will not be above its target max 1586 * after growing. Otherwise, if the free list is nonempty, try 1587 * to reclaim 1 item from it before growing the cache (possibly 1588 * above its target max if the reclamation failed or is delayed). 1589 * Otherwise, wait for some space. In all cases, schedule 1590 * vnlru_proc() if we are getting short of space. The watermarks 1591 * should be chosen so that we never wait or even reclaim from 1592 * the free list to below its target minimum. 1593 */ 1594 if (vnlru_free_locked(1, NULL) > 0) 1595 goto alloc; 1596 if (mp == NULL || (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 1597 /* 1598 * Wait for space for a new vnode. 1599 */ 1600 vnlru_kick(); 1601 msleep(&vnlruproc_sig, &vnode_list_mtx, PVFS, "vlruwk", hz); 1602 if (atomic_load_long(&numvnodes) + 1 > desiredvnodes && 1603 vnlru_read_freevnodes() > 1) 1604 vnlru_free_locked(1, NULL); 1605 } 1606 alloc: 1607 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 1608 if (vnlru_under(rnumvnodes, vlowat)) 1609 vnlru_kick(); 1610 mtx_unlock(&vnode_list_mtx); 1611 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 1612 } 1613 1614 static struct vnode * 1615 vn_alloc(struct mount *mp) 1616 { 1617 u_long rnumvnodes; 1618 1619 if (__predict_false(vn_alloc_cyclecount != 0)) 1620 return (vn_alloc_hard(mp)); 1621 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 1622 if (__predict_false(vnlru_under_unlocked(rnumvnodes, vlowat))) { 1623 atomic_subtract_long(&numvnodes, 1); 1624 return (vn_alloc_hard(mp)); 1625 } 1626 1627 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 1628 } 1629 1630 static void 1631 vn_free(struct vnode *vp) 1632 { 1633 1634 atomic_subtract_long(&numvnodes, 1); 1635 uma_zfree_smr(vnode_zone, vp); 1636 } 1637 1638 /* 1639 * Return the next vnode from the free list. 1640 */ 1641 int 1642 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops, 1643 struct vnode **vpp) 1644 { 1645 struct vnode *vp; 1646 struct thread *td; 1647 struct lock_object *lo; 1648 1649 CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag); 1650 1651 KASSERT(vops->registered, 1652 ("%s: not registered vector op %p\n", __func__, vops)); 1653 1654 td = curthread; 1655 if (td->td_vp_reserved != NULL) { 1656 vp = td->td_vp_reserved; 1657 td->td_vp_reserved = NULL; 1658 } else { 1659 vp = vn_alloc(mp); 1660 } 1661 counter_u64_add(vnodes_created, 1); 1662 /* 1663 * Locks are given the generic name "vnode" when created. 1664 * Follow the historic practice of using the filesystem 1665 * name when they allocated, e.g., "zfs", "ufs", "nfs, etc. 1666 * 1667 * Locks live in a witness group keyed on their name. Thus, 1668 * when a lock is renamed, it must also move from the witness 1669 * group of its old name to the witness group of its new name. 1670 * 1671 * The change only needs to be made when the vnode moves 1672 * from one filesystem type to another. We ensure that each 1673 * filesystem use a single static name pointer for its tag so 1674 * that we can compare pointers rather than doing a strcmp(). 1675 */ 1676 lo = &vp->v_vnlock->lock_object; 1677 #ifdef WITNESS 1678 if (lo->lo_name != tag) { 1679 #endif 1680 lo->lo_name = tag; 1681 #ifdef WITNESS 1682 WITNESS_DESTROY(lo); 1683 WITNESS_INIT(lo, tag); 1684 } 1685 #endif 1686 /* 1687 * By default, don't allow shared locks unless filesystems opt-in. 1688 */ 1689 vp->v_vnlock->lock_object.lo_flags |= LK_NOSHARE; 1690 /* 1691 * Finalize various vnode identity bits. 1692 */ 1693 KASSERT(vp->v_object == NULL, ("stale v_object %p", vp)); 1694 KASSERT(vp->v_lockf == NULL, ("stale v_lockf %p", vp)); 1695 KASSERT(vp->v_pollinfo == NULL, ("stale v_pollinfo %p", vp)); 1696 vp->v_type = VNON; 1697 vp->v_op = vops; 1698 v_init_counters(vp); 1699 vp->v_bufobj.bo_ops = &buf_ops_bio; 1700 #ifdef DIAGNOSTIC 1701 if (mp == NULL && vops != &dead_vnodeops) 1702 printf("NULL mp in getnewvnode(9), tag %s\n", tag); 1703 #endif 1704 #ifdef MAC 1705 mac_vnode_init(vp); 1706 if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0) 1707 mac_vnode_associate_singlelabel(mp, vp); 1708 #endif 1709 if (mp != NULL) { 1710 vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize; 1711 if ((mp->mnt_kern_flag & MNTK_NOKNOTE) != 0) 1712 vp->v_vflag |= VV_NOKNOTE; 1713 } 1714 1715 /* 1716 * For the filesystems which do not use vfs_hash_insert(), 1717 * still initialize v_hash to have vfs_hash_index() useful. 1718 * E.g., nullfs uses vfs_hash_index() on the lower vnode for 1719 * its own hashing. 1720 */ 1721 vp->v_hash = (uintptr_t)vp >> vnsz2log; 1722 1723 *vpp = vp; 1724 return (0); 1725 } 1726 1727 void 1728 getnewvnode_reserve(void) 1729 { 1730 struct thread *td; 1731 1732 td = curthread; 1733 MPASS(td->td_vp_reserved == NULL); 1734 td->td_vp_reserved = vn_alloc(NULL); 1735 } 1736 1737 void 1738 getnewvnode_drop_reserve(void) 1739 { 1740 struct thread *td; 1741 1742 td = curthread; 1743 if (td->td_vp_reserved != NULL) { 1744 vn_free(td->td_vp_reserved); 1745 td->td_vp_reserved = NULL; 1746 } 1747 } 1748 1749 static void __noinline 1750 freevnode(struct vnode *vp) 1751 { 1752 struct bufobj *bo; 1753 1754 /* 1755 * The vnode has been marked for destruction, so free it. 1756 * 1757 * The vnode will be returned to the zone where it will 1758 * normally remain until it is needed for another vnode. We 1759 * need to cleanup (or verify that the cleanup has already 1760 * been done) any residual data left from its current use 1761 * so as not to contaminate the freshly allocated vnode. 1762 */ 1763 CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); 1764 /* 1765 * Paired with vgone. 1766 */ 1767 vn_seqc_write_end_locked(vp); 1768 VNPASS(vp->v_seqc_users == 0, vp); 1769 1770 bo = &vp->v_bufobj; 1771 VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); 1772 VNPASS(vp->v_holdcnt == VHOLD_NO_SMR, vp); 1773 VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); 1774 VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); 1775 VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); 1776 VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); 1777 VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp, 1778 ("clean blk trie not empty")); 1779 VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0")); 1780 VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp, 1781 ("dirty blk trie not empty")); 1782 VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst")); 1783 VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src")); 1784 VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for ..")); 1785 VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp, 1786 ("Dangling rangelock waiters")); 1787 VNASSERT((vp->v_iflag & (VI_DOINGINACT | VI_OWEINACT)) == 0, vp, 1788 ("Leaked inactivation")); 1789 VI_UNLOCK(vp); 1790 #ifdef MAC 1791 mac_vnode_destroy(vp); 1792 #endif 1793 if (vp->v_pollinfo != NULL) { 1794 destroy_vpollinfo(vp->v_pollinfo); 1795 vp->v_pollinfo = NULL; 1796 } 1797 #ifdef INVARIANTS 1798 /* XXX Elsewhere we detect an already freed vnode via NULL v_op. */ 1799 vp->v_op = NULL; 1800 #endif 1801 vp->v_mountedhere = NULL; 1802 vp->v_unpcb = NULL; 1803 vp->v_rdev = NULL; 1804 vp->v_fifoinfo = NULL; 1805 vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0; 1806 vp->v_irflag = 0; 1807 vp->v_iflag = 0; 1808 vp->v_vflag = 0; 1809 bo->bo_flag = 0; 1810 vn_free(vp); 1811 } 1812 1813 /* 1814 * Delete from old mount point vnode list, if on one. 1815 */ 1816 static void 1817 delmntque(struct vnode *vp) 1818 { 1819 struct mount *mp; 1820 1821 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 1822 1823 mp = vp->v_mount; 1824 if (mp == NULL) 1825 return; 1826 MNT_ILOCK(mp); 1827 VI_LOCK(vp); 1828 vp->v_mount = NULL; 1829 VI_UNLOCK(vp); 1830 VNASSERT(mp->mnt_nvnodelistsize > 0, vp, 1831 ("bad mount point vnode list size")); 1832 TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 1833 mp->mnt_nvnodelistsize--; 1834 MNT_REL(mp); 1835 MNT_IUNLOCK(mp); 1836 } 1837 1838 static void 1839 insmntque_stddtr(struct vnode *vp, void *dtr_arg) 1840 { 1841 1842 vp->v_data = NULL; 1843 vp->v_op = &dead_vnodeops; 1844 vgone(vp); 1845 vput(vp); 1846 } 1847 1848 /* 1849 * Insert into list of vnodes for the new mount point, if available. 1850 */ 1851 int 1852 insmntque1(struct vnode *vp, struct mount *mp, 1853 void (*dtr)(struct vnode *, void *), void *dtr_arg) 1854 { 1855 1856 KASSERT(vp->v_mount == NULL, 1857 ("insmntque: vnode already on per mount vnode list")); 1858 VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)")); 1859 ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp"); 1860 1861 /* 1862 * We acquire the vnode interlock early to ensure that the 1863 * vnode cannot be recycled by another process releasing a 1864 * holdcnt on it before we get it on both the vnode list 1865 * and the active vnode list. The mount mutex protects only 1866 * manipulation of the vnode list and the vnode freelist 1867 * mutex protects only manipulation of the active vnode list. 1868 * Hence the need to hold the vnode interlock throughout. 1869 */ 1870 MNT_ILOCK(mp); 1871 VI_LOCK(vp); 1872 if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 && 1873 ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 || 1874 mp->mnt_nvnodelistsize == 0)) && 1875 (vp->v_vflag & VV_FORCEINSMQ) == 0) { 1876 VI_UNLOCK(vp); 1877 MNT_IUNLOCK(mp); 1878 if (dtr != NULL) 1879 dtr(vp, dtr_arg); 1880 return (EBUSY); 1881 } 1882 vp->v_mount = mp; 1883 MNT_REF(mp); 1884 TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 1885 VNASSERT(mp->mnt_nvnodelistsize >= 0, vp, 1886 ("neg mount point vnode list size")); 1887 mp->mnt_nvnodelistsize++; 1888 VI_UNLOCK(vp); 1889 MNT_IUNLOCK(mp); 1890 return (0); 1891 } 1892 1893 int 1894 insmntque(struct vnode *vp, struct mount *mp) 1895 { 1896 1897 return (insmntque1(vp, mp, insmntque_stddtr, NULL)); 1898 } 1899 1900 /* 1901 * Flush out and invalidate all buffers associated with a bufobj 1902 * Called with the underlying object locked. 1903 */ 1904 int 1905 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo) 1906 { 1907 int error; 1908 1909 BO_LOCK(bo); 1910 if (flags & V_SAVE) { 1911 error = bufobj_wwait(bo, slpflag, slptimeo); 1912 if (error) { 1913 BO_UNLOCK(bo); 1914 return (error); 1915 } 1916 if (bo->bo_dirty.bv_cnt > 0) { 1917 BO_UNLOCK(bo); 1918 if ((error = BO_SYNC(bo, MNT_WAIT)) != 0) 1919 return (error); 1920 /* 1921 * XXX We could save a lock/unlock if this was only 1922 * enabled under INVARIANTS 1923 */ 1924 BO_LOCK(bo); 1925 if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0) 1926 panic("vinvalbuf: dirty bufs"); 1927 } 1928 } 1929 /* 1930 * If you alter this loop please notice that interlock is dropped and 1931 * reacquired in flushbuflist. Special care is needed to ensure that 1932 * no race conditions occur from this. 1933 */ 1934 do { 1935 error = flushbuflist(&bo->bo_clean, 1936 flags, bo, slpflag, slptimeo); 1937 if (error == 0 && !(flags & V_CLEANONLY)) 1938 error = flushbuflist(&bo->bo_dirty, 1939 flags, bo, slpflag, slptimeo); 1940 if (error != 0 && error != EAGAIN) { 1941 BO_UNLOCK(bo); 1942 return (error); 1943 } 1944 } while (error != 0); 1945 1946 /* 1947 * Wait for I/O to complete. XXX needs cleaning up. The vnode can 1948 * have write I/O in-progress but if there is a VM object then the 1949 * VM object can also have read-I/O in-progress. 1950 */ 1951 do { 1952 bufobj_wwait(bo, 0, 0); 1953 if ((flags & V_VMIO) == 0 && bo->bo_object != NULL) { 1954 BO_UNLOCK(bo); 1955 vm_object_pip_wait_unlocked(bo->bo_object, "bovlbx"); 1956 BO_LOCK(bo); 1957 } 1958 } while (bo->bo_numoutput > 0); 1959 BO_UNLOCK(bo); 1960 1961 /* 1962 * Destroy the copy in the VM cache, too. 1963 */ 1964 if (bo->bo_object != NULL && 1965 (flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0) { 1966 VM_OBJECT_WLOCK(bo->bo_object); 1967 vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ? 1968 OBJPR_CLEANONLY : 0); 1969 VM_OBJECT_WUNLOCK(bo->bo_object); 1970 } 1971 1972 #ifdef INVARIANTS 1973 BO_LOCK(bo); 1974 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO | 1975 V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 || 1976 bo->bo_clean.bv_cnt > 0)) 1977 panic("vinvalbuf: flush failed"); 1978 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0 && 1979 bo->bo_dirty.bv_cnt > 0) 1980 panic("vinvalbuf: flush dirty failed"); 1981 BO_UNLOCK(bo); 1982 #endif 1983 return (0); 1984 } 1985 1986 /* 1987 * Flush out and invalidate all buffers associated with a vnode. 1988 * Called with the underlying object locked. 1989 */ 1990 int 1991 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) 1992 { 1993 1994 CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags); 1995 ASSERT_VOP_LOCKED(vp, "vinvalbuf"); 1996 if (vp->v_object != NULL && vp->v_object->handle != vp) 1997 return (0); 1998 return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo)); 1999 } 2000 2001 /* 2002 * Flush out buffers on the specified list. 2003 * 2004 */ 2005 static int 2006 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, 2007 int slptimeo) 2008 { 2009 struct buf *bp, *nbp; 2010 int retval, error; 2011 daddr_t lblkno; 2012 b_xflags_t xflags; 2013 2014 ASSERT_BO_WLOCKED(bo); 2015 2016 retval = 0; 2017 TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) { 2018 /* 2019 * If we are flushing both V_NORMAL and V_ALT buffers then 2020 * do not skip any buffers. If we are flushing only V_NORMAL 2021 * buffers then skip buffers marked as BX_ALTDATA. If we are 2022 * flushing only V_ALT buffers then skip buffers not marked 2023 * as BX_ALTDATA. 2024 */ 2025 if (((flags & (V_NORMAL | V_ALT)) != (V_NORMAL | V_ALT)) && 2026 (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) || 2027 ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) { 2028 continue; 2029 } 2030 if (nbp != NULL) { 2031 lblkno = nbp->b_lblkno; 2032 xflags = nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN); 2033 } 2034 retval = EAGAIN; 2035 error = BUF_TIMELOCK(bp, 2036 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo), 2037 "flushbuf", slpflag, slptimeo); 2038 if (error) { 2039 BO_LOCK(bo); 2040 return (error != ENOLCK ? error : EAGAIN); 2041 } 2042 KASSERT(bp->b_bufobj == bo, 2043 ("bp %p wrong b_bufobj %p should be %p", 2044 bp, bp->b_bufobj, bo)); 2045 /* 2046 * XXX Since there are no node locks for NFS, I 2047 * believe there is a slight chance that a delayed 2048 * write will occur while sleeping just above, so 2049 * check for it. 2050 */ 2051 if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) && 2052 (flags & V_SAVE)) { 2053 bremfree(bp); 2054 bp->b_flags |= B_ASYNC; 2055 bwrite(bp); 2056 BO_LOCK(bo); 2057 return (EAGAIN); /* XXX: why not loop ? */ 2058 } 2059 bremfree(bp); 2060 bp->b_flags |= (B_INVAL | B_RELBUF); 2061 bp->b_flags &= ~B_ASYNC; 2062 brelse(bp); 2063 BO_LOCK(bo); 2064 if (nbp == NULL) 2065 break; 2066 nbp = gbincore(bo, lblkno); 2067 if (nbp == NULL || (nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) 2068 != xflags) 2069 break; /* nbp invalid */ 2070 } 2071 return (retval); 2072 } 2073 2074 int 2075 bnoreuselist(struct bufv *bufv, struct bufobj *bo, daddr_t startn, daddr_t endn) 2076 { 2077 struct buf *bp; 2078 int error; 2079 daddr_t lblkno; 2080 2081 ASSERT_BO_LOCKED(bo); 2082 2083 for (lblkno = startn;;) { 2084 again: 2085 bp = BUF_PCTRIE_LOOKUP_GE(&bufv->bv_root, lblkno); 2086 if (bp == NULL || bp->b_lblkno >= endn || 2087 bp->b_lblkno < startn) 2088 break; 2089 error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 2090 LK_INTERLOCK, BO_LOCKPTR(bo), "brlsfl", 0, 0); 2091 if (error != 0) { 2092 BO_RLOCK(bo); 2093 if (error == ENOLCK) 2094 goto again; 2095 return (error); 2096 } 2097 KASSERT(bp->b_bufobj == bo, 2098 ("bp %p wrong b_bufobj %p should be %p", 2099 bp, bp->b_bufobj, bo)); 2100 lblkno = bp->b_lblkno + 1; 2101 if ((bp->b_flags & B_MANAGED) == 0) 2102 bremfree(bp); 2103 bp->b_flags |= B_RELBUF; 2104 /* 2105 * In the VMIO case, use the B_NOREUSE flag to hint that the 2106 * pages backing each buffer in the range are unlikely to be 2107 * reused. Dirty buffers will have the hint applied once 2108 * they've been written. 2109 */ 2110 if ((bp->b_flags & B_VMIO) != 0) 2111 bp->b_flags |= B_NOREUSE; 2112 brelse(bp); 2113 BO_RLOCK(bo); 2114 } 2115 return (0); 2116 } 2117 2118 /* 2119 * Truncate a file's buffer and pages to a specified length. This 2120 * is in lieu of the old vinvalbuf mechanism, which performed unneeded 2121 * sync activity. 2122 */ 2123 int 2124 vtruncbuf(struct vnode *vp, off_t length, int blksize) 2125 { 2126 struct buf *bp, *nbp; 2127 struct bufobj *bo; 2128 daddr_t startlbn; 2129 2130 CTR4(KTR_VFS, "%s: vp %p with block %d:%ju", __func__, 2131 vp, blksize, (uintmax_t)length); 2132 2133 /* 2134 * Round up to the *next* lbn. 2135 */ 2136 startlbn = howmany(length, blksize); 2137 2138 ASSERT_VOP_LOCKED(vp, "vtruncbuf"); 2139 2140 bo = &vp->v_bufobj; 2141 restart_unlocked: 2142 BO_LOCK(bo); 2143 2144 while (v_inval_buf_range_locked(vp, bo, startlbn, INT64_MAX) == EAGAIN) 2145 ; 2146 2147 if (length > 0) { 2148 restartsync: 2149 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2150 if (bp->b_lblkno > 0) 2151 continue; 2152 /* 2153 * Since we hold the vnode lock this should only 2154 * fail if we're racing with the buf daemon. 2155 */ 2156 if (BUF_LOCK(bp, 2157 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2158 BO_LOCKPTR(bo)) == ENOLCK) 2159 goto restart_unlocked; 2160 2161 VNASSERT((bp->b_flags & B_DELWRI), vp, 2162 ("buf(%p) on dirty queue without DELWRI", bp)); 2163 2164 bremfree(bp); 2165 bawrite(bp); 2166 BO_LOCK(bo); 2167 goto restartsync; 2168 } 2169 } 2170 2171 bufobj_wwait(bo, 0, 0); 2172 BO_UNLOCK(bo); 2173 vnode_pager_setsize(vp, length); 2174 2175 return (0); 2176 } 2177 2178 /* 2179 * Invalidate the cached pages of a file's buffer within the range of block 2180 * numbers [startlbn, endlbn). 2181 */ 2182 void 2183 v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn, 2184 int blksize) 2185 { 2186 struct bufobj *bo; 2187 off_t start, end; 2188 2189 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range"); 2190 2191 start = blksize * startlbn; 2192 end = blksize * endlbn; 2193 2194 bo = &vp->v_bufobj; 2195 BO_LOCK(bo); 2196 MPASS(blksize == bo->bo_bsize); 2197 2198 while (v_inval_buf_range_locked(vp, bo, startlbn, endlbn) == EAGAIN) 2199 ; 2200 2201 BO_UNLOCK(bo); 2202 vn_pages_remove(vp, OFF_TO_IDX(start), OFF_TO_IDX(end + PAGE_SIZE - 1)); 2203 } 2204 2205 static int 2206 v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 2207 daddr_t startlbn, daddr_t endlbn) 2208 { 2209 struct buf *bp, *nbp; 2210 bool anyfreed; 2211 2212 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range_locked"); 2213 ASSERT_BO_LOCKED(bo); 2214 2215 do { 2216 anyfreed = false; 2217 TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) { 2218 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn) 2219 continue; 2220 if (BUF_LOCK(bp, 2221 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2222 BO_LOCKPTR(bo)) == ENOLCK) { 2223 BO_LOCK(bo); 2224 return (EAGAIN); 2225 } 2226 2227 bremfree(bp); 2228 bp->b_flags |= B_INVAL | B_RELBUF; 2229 bp->b_flags &= ~B_ASYNC; 2230 brelse(bp); 2231 anyfreed = true; 2232 2233 BO_LOCK(bo); 2234 if (nbp != NULL && 2235 (((nbp->b_xflags & BX_VNCLEAN) == 0) || 2236 nbp->b_vp != vp || 2237 (nbp->b_flags & B_DELWRI) != 0)) 2238 return (EAGAIN); 2239 } 2240 2241 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2242 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn) 2243 continue; 2244 if (BUF_LOCK(bp, 2245 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2246 BO_LOCKPTR(bo)) == ENOLCK) { 2247 BO_LOCK(bo); 2248 return (EAGAIN); 2249 } 2250 bremfree(bp); 2251 bp->b_flags |= B_INVAL | B_RELBUF; 2252 bp->b_flags &= ~B_ASYNC; 2253 brelse(bp); 2254 anyfreed = true; 2255 2256 BO_LOCK(bo); 2257 if (nbp != NULL && 2258 (((nbp->b_xflags & BX_VNDIRTY) == 0) || 2259 (nbp->b_vp != vp) || 2260 (nbp->b_flags & B_DELWRI) == 0)) 2261 return (EAGAIN); 2262 } 2263 } while (anyfreed); 2264 return (0); 2265 } 2266 2267 static void 2268 buf_vlist_remove(struct buf *bp) 2269 { 2270 struct bufv *bv; 2271 b_xflags_t flags; 2272 2273 flags = bp->b_xflags; 2274 2275 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp)); 2276 ASSERT_BO_WLOCKED(bp->b_bufobj); 2277 KASSERT((flags & (BX_VNDIRTY | BX_VNCLEAN)) != 0 && 2278 (flags & (BX_VNDIRTY | BX_VNCLEAN)) != (BX_VNDIRTY | BX_VNCLEAN), 2279 ("%s: buffer %p has invalid queue state", __func__, bp)); 2280 2281 if ((flags & BX_VNDIRTY) != 0) 2282 bv = &bp->b_bufobj->bo_dirty; 2283 else 2284 bv = &bp->b_bufobj->bo_clean; 2285 BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno); 2286 TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs); 2287 bv->bv_cnt--; 2288 bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN); 2289 } 2290 2291 /* 2292 * Add the buffer to the sorted clean or dirty block list. 2293 * 2294 * NOTE: xflags is passed as a constant, optimizing this inline function! 2295 */ 2296 static void 2297 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags) 2298 { 2299 struct bufv *bv; 2300 struct buf *n; 2301 int error; 2302 2303 ASSERT_BO_WLOCKED(bo); 2304 KASSERT((bo->bo_flag & BO_NOBUFS) == 0, 2305 ("buf_vlist_add: bo %p does not allow bufs", bo)); 2306 KASSERT((xflags & BX_VNDIRTY) == 0 || (bo->bo_flag & BO_DEAD) == 0, 2307 ("dead bo %p", bo)); 2308 KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, 2309 ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags)); 2310 bp->b_xflags |= xflags; 2311 if (xflags & BX_VNDIRTY) 2312 bv = &bo->bo_dirty; 2313 else 2314 bv = &bo->bo_clean; 2315 2316 /* 2317 * Keep the list ordered. Optimize empty list insertion. Assume 2318 * we tend to grow at the tail so lookup_le should usually be cheaper 2319 * than _ge. 2320 */ 2321 if (bv->bv_cnt == 0 || 2322 bp->b_lblkno > TAILQ_LAST(&bv->bv_hd, buflists)->b_lblkno) 2323 TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs); 2324 else if ((n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, bp->b_lblkno)) == NULL) 2325 TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs); 2326 else 2327 TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs); 2328 error = BUF_PCTRIE_INSERT(&bv->bv_root, bp); 2329 if (error) 2330 panic("buf_vlist_add: Preallocated nodes insufficient."); 2331 bv->bv_cnt++; 2332 } 2333 2334 /* 2335 * Look up a buffer using the buffer tries. 2336 */ 2337 struct buf * 2338 gbincore(struct bufobj *bo, daddr_t lblkno) 2339 { 2340 struct buf *bp; 2341 2342 ASSERT_BO_LOCKED(bo); 2343 bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno); 2344 if (bp != NULL) 2345 return (bp); 2346 return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno)); 2347 } 2348 2349 /* 2350 * Look up a buf using the buffer tries, without the bufobj lock. This relies 2351 * on SMR for safe lookup, and bufs being in a no-free zone to provide type 2352 * stability of the result. Like other lockless lookups, the found buf may 2353 * already be invalid by the time this function returns. 2354 */ 2355 struct buf * 2356 gbincore_unlocked(struct bufobj *bo, daddr_t lblkno) 2357 { 2358 struct buf *bp; 2359 2360 ASSERT_BO_UNLOCKED(bo); 2361 bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno); 2362 if (bp != NULL) 2363 return (bp); 2364 return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno)); 2365 } 2366 2367 /* 2368 * Associate a buffer with a vnode. 2369 */ 2370 void 2371 bgetvp(struct vnode *vp, struct buf *bp) 2372 { 2373 struct bufobj *bo; 2374 2375 bo = &vp->v_bufobj; 2376 ASSERT_BO_WLOCKED(bo); 2377 VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free")); 2378 2379 CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags); 2380 VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp, 2381 ("bgetvp: bp already attached! %p", bp)); 2382 2383 vhold(vp); 2384 bp->b_vp = vp; 2385 bp->b_bufobj = bo; 2386 /* 2387 * Insert onto list for new vnode. 2388 */ 2389 buf_vlist_add(bp, bo, BX_VNCLEAN); 2390 } 2391 2392 /* 2393 * Disassociate a buffer from a vnode. 2394 */ 2395 void 2396 brelvp(struct buf *bp) 2397 { 2398 struct bufobj *bo; 2399 struct vnode *vp; 2400 2401 CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags); 2402 KASSERT(bp->b_vp != NULL, ("brelvp: NULL")); 2403 2404 /* 2405 * Delete from old vnode list, if on one. 2406 */ 2407 vp = bp->b_vp; /* XXX */ 2408 bo = bp->b_bufobj; 2409 BO_LOCK(bo); 2410 buf_vlist_remove(bp); 2411 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2412 bo->bo_flag &= ~BO_ONWORKLST; 2413 mtx_lock(&sync_mtx); 2414 LIST_REMOVE(bo, bo_synclist); 2415 syncer_worklist_len--; 2416 mtx_unlock(&sync_mtx); 2417 } 2418 bp->b_vp = NULL; 2419 bp->b_bufobj = NULL; 2420 BO_UNLOCK(bo); 2421 vdrop(vp); 2422 } 2423 2424 /* 2425 * Add an item to the syncer work queue. 2426 */ 2427 static void 2428 vn_syncer_add_to_worklist(struct bufobj *bo, int delay) 2429 { 2430 int slot; 2431 2432 ASSERT_BO_WLOCKED(bo); 2433 2434 mtx_lock(&sync_mtx); 2435 if (bo->bo_flag & BO_ONWORKLST) 2436 LIST_REMOVE(bo, bo_synclist); 2437 else { 2438 bo->bo_flag |= BO_ONWORKLST; 2439 syncer_worklist_len++; 2440 } 2441 2442 if (delay > syncer_maxdelay - 2) 2443 delay = syncer_maxdelay - 2; 2444 slot = (syncer_delayno + delay) & syncer_mask; 2445 2446 LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist); 2447 mtx_unlock(&sync_mtx); 2448 } 2449 2450 static int 2451 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS) 2452 { 2453 int error, len; 2454 2455 mtx_lock(&sync_mtx); 2456 len = syncer_worklist_len - sync_vnode_count; 2457 mtx_unlock(&sync_mtx); 2458 error = SYSCTL_OUT(req, &len, sizeof(len)); 2459 return (error); 2460 } 2461 2462 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, 2463 CTLTYPE_INT | CTLFLAG_MPSAFE| CTLFLAG_RD, NULL, 0, 2464 sysctl_vfs_worklist_len, "I", "Syncer thread worklist length"); 2465 2466 static struct proc *updateproc; 2467 static void sched_sync(void); 2468 static struct kproc_desc up_kp = { 2469 "syncer", 2470 sched_sync, 2471 &updateproc 2472 }; 2473 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp); 2474 2475 static int 2476 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td) 2477 { 2478 struct vnode *vp; 2479 struct mount *mp; 2480 2481 *bo = LIST_FIRST(slp); 2482 if (*bo == NULL) 2483 return (0); 2484 vp = bo2vnode(*bo); 2485 if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0) 2486 return (1); 2487 /* 2488 * We use vhold in case the vnode does not 2489 * successfully sync. vhold prevents the vnode from 2490 * going away when we unlock the sync_mtx so that 2491 * we can acquire the vnode interlock. 2492 */ 2493 vholdl(vp); 2494 mtx_unlock(&sync_mtx); 2495 VI_UNLOCK(vp); 2496 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 2497 vdrop(vp); 2498 mtx_lock(&sync_mtx); 2499 return (*bo == LIST_FIRST(slp)); 2500 } 2501 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2502 (void) VOP_FSYNC(vp, MNT_LAZY, td); 2503 VOP_UNLOCK(vp); 2504 vn_finished_write(mp); 2505 BO_LOCK(*bo); 2506 if (((*bo)->bo_flag & BO_ONWORKLST) != 0) { 2507 /* 2508 * Put us back on the worklist. The worklist 2509 * routine will remove us from our current 2510 * position and then add us back in at a later 2511 * position. 2512 */ 2513 vn_syncer_add_to_worklist(*bo, syncdelay); 2514 } 2515 BO_UNLOCK(*bo); 2516 vdrop(vp); 2517 mtx_lock(&sync_mtx); 2518 return (0); 2519 } 2520 2521 static int first_printf = 1; 2522 2523 /* 2524 * System filesystem synchronizer daemon. 2525 */ 2526 static void 2527 sched_sync(void) 2528 { 2529 struct synclist *next, *slp; 2530 struct bufobj *bo; 2531 long starttime; 2532 struct thread *td = curthread; 2533 int last_work_seen; 2534 int net_worklist_len; 2535 int syncer_final_iter; 2536 int error; 2537 2538 last_work_seen = 0; 2539 syncer_final_iter = 0; 2540 syncer_state = SYNCER_RUNNING; 2541 starttime = time_uptime; 2542 td->td_pflags |= TDP_NORUNNINGBUF; 2543 2544 EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc, 2545 SHUTDOWN_PRI_LAST); 2546 2547 mtx_lock(&sync_mtx); 2548 for (;;) { 2549 if (syncer_state == SYNCER_FINAL_DELAY && 2550 syncer_final_iter == 0) { 2551 mtx_unlock(&sync_mtx); 2552 kproc_suspend_check(td->td_proc); 2553 mtx_lock(&sync_mtx); 2554 } 2555 net_worklist_len = syncer_worklist_len - sync_vnode_count; 2556 if (syncer_state != SYNCER_RUNNING && 2557 starttime != time_uptime) { 2558 if (first_printf) { 2559 printf("\nSyncing disks, vnodes remaining... "); 2560 first_printf = 0; 2561 } 2562 printf("%d ", net_worklist_len); 2563 } 2564 starttime = time_uptime; 2565 2566 /* 2567 * Push files whose dirty time has expired. Be careful 2568 * of interrupt race on slp queue. 2569 * 2570 * Skip over empty worklist slots when shutting down. 2571 */ 2572 do { 2573 slp = &syncer_workitem_pending[syncer_delayno]; 2574 syncer_delayno += 1; 2575 if (syncer_delayno == syncer_maxdelay) 2576 syncer_delayno = 0; 2577 next = &syncer_workitem_pending[syncer_delayno]; 2578 /* 2579 * If the worklist has wrapped since the 2580 * it was emptied of all but syncer vnodes, 2581 * switch to the FINAL_DELAY state and run 2582 * for one more second. 2583 */ 2584 if (syncer_state == SYNCER_SHUTTING_DOWN && 2585 net_worklist_len == 0 && 2586 last_work_seen == syncer_delayno) { 2587 syncer_state = SYNCER_FINAL_DELAY; 2588 syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP; 2589 } 2590 } while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) && 2591 syncer_worklist_len > 0); 2592 2593 /* 2594 * Keep track of the last time there was anything 2595 * on the worklist other than syncer vnodes. 2596 * Return to the SHUTTING_DOWN state if any 2597 * new work appears. 2598 */ 2599 if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING) 2600 last_work_seen = syncer_delayno; 2601 if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY) 2602 syncer_state = SYNCER_SHUTTING_DOWN; 2603 while (!LIST_EMPTY(slp)) { 2604 error = sync_vnode(slp, &bo, td); 2605 if (error == 1) { 2606 LIST_REMOVE(bo, bo_synclist); 2607 LIST_INSERT_HEAD(next, bo, bo_synclist); 2608 continue; 2609 } 2610 2611 if (first_printf == 0) { 2612 /* 2613 * Drop the sync mutex, because some watchdog 2614 * drivers need to sleep while patting 2615 */ 2616 mtx_unlock(&sync_mtx); 2617 wdog_kern_pat(WD_LASTVAL); 2618 mtx_lock(&sync_mtx); 2619 } 2620 } 2621 if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0) 2622 syncer_final_iter--; 2623 /* 2624 * The variable rushjob allows the kernel to speed up the 2625 * processing of the filesystem syncer process. A rushjob 2626 * value of N tells the filesystem syncer to process the next 2627 * N seconds worth of work on its queue ASAP. Currently rushjob 2628 * is used by the soft update code to speed up the filesystem 2629 * syncer process when the incore state is getting so far 2630 * ahead of the disk that the kernel memory pool is being 2631 * threatened with exhaustion. 2632 */ 2633 if (rushjob > 0) { 2634 rushjob -= 1; 2635 continue; 2636 } 2637 /* 2638 * Just sleep for a short period of time between 2639 * iterations when shutting down to allow some I/O 2640 * to happen. 2641 * 2642 * If it has taken us less than a second to process the 2643 * current work, then wait. Otherwise start right over 2644 * again. We can still lose time if any single round 2645 * takes more than two seconds, but it does not really 2646 * matter as we are just trying to generally pace the 2647 * filesystem activity. 2648 */ 2649 if (syncer_state != SYNCER_RUNNING || 2650 time_uptime == starttime) { 2651 thread_lock(td); 2652 sched_prio(td, PPAUSE); 2653 thread_unlock(td); 2654 } 2655 if (syncer_state != SYNCER_RUNNING) 2656 cv_timedwait(&sync_wakeup, &sync_mtx, 2657 hz / SYNCER_SHUTDOWN_SPEEDUP); 2658 else if (time_uptime == starttime) 2659 cv_timedwait(&sync_wakeup, &sync_mtx, hz); 2660 } 2661 } 2662 2663 /* 2664 * Request the syncer daemon to speed up its work. 2665 * We never push it to speed up more than half of its 2666 * normal turn time, otherwise it could take over the cpu. 2667 */ 2668 int 2669 speedup_syncer(void) 2670 { 2671 int ret = 0; 2672 2673 mtx_lock(&sync_mtx); 2674 if (rushjob < syncdelay / 2) { 2675 rushjob += 1; 2676 stat_rush_requests += 1; 2677 ret = 1; 2678 } 2679 mtx_unlock(&sync_mtx); 2680 cv_broadcast(&sync_wakeup); 2681 return (ret); 2682 } 2683 2684 /* 2685 * Tell the syncer to speed up its work and run though its work 2686 * list several times, then tell it to shut down. 2687 */ 2688 static void 2689 syncer_shutdown(void *arg, int howto) 2690 { 2691 2692 if (howto & RB_NOSYNC) 2693 return; 2694 mtx_lock(&sync_mtx); 2695 syncer_state = SYNCER_SHUTTING_DOWN; 2696 rushjob = 0; 2697 mtx_unlock(&sync_mtx); 2698 cv_broadcast(&sync_wakeup); 2699 kproc_shutdown(arg, howto); 2700 } 2701 2702 void 2703 syncer_suspend(void) 2704 { 2705 2706 syncer_shutdown(updateproc, 0); 2707 } 2708 2709 void 2710 syncer_resume(void) 2711 { 2712 2713 mtx_lock(&sync_mtx); 2714 first_printf = 1; 2715 syncer_state = SYNCER_RUNNING; 2716 mtx_unlock(&sync_mtx); 2717 cv_broadcast(&sync_wakeup); 2718 kproc_resume(updateproc); 2719 } 2720 2721 /* 2722 * Move the buffer between the clean and dirty lists of its vnode. 2723 */ 2724 void 2725 reassignbuf(struct buf *bp) 2726 { 2727 struct vnode *vp; 2728 struct bufobj *bo; 2729 int delay; 2730 #ifdef INVARIANTS 2731 struct bufv *bv; 2732 #endif 2733 2734 vp = bp->b_vp; 2735 bo = bp->b_bufobj; 2736 2737 KASSERT((bp->b_flags & B_PAGING) == 0, 2738 ("%s: cannot reassign paging buffer %p", __func__, bp)); 2739 2740 CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X", 2741 bp, bp->b_vp, bp->b_flags); 2742 2743 BO_LOCK(bo); 2744 buf_vlist_remove(bp); 2745 2746 /* 2747 * If dirty, put on list of dirty buffers; otherwise insert onto list 2748 * of clean buffers. 2749 */ 2750 if (bp->b_flags & B_DELWRI) { 2751 if ((bo->bo_flag & BO_ONWORKLST) == 0) { 2752 switch (vp->v_type) { 2753 case VDIR: 2754 delay = dirdelay; 2755 break; 2756 case VCHR: 2757 delay = metadelay; 2758 break; 2759 default: 2760 delay = filedelay; 2761 } 2762 vn_syncer_add_to_worklist(bo, delay); 2763 } 2764 buf_vlist_add(bp, bo, BX_VNDIRTY); 2765 } else { 2766 buf_vlist_add(bp, bo, BX_VNCLEAN); 2767 2768 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2769 mtx_lock(&sync_mtx); 2770 LIST_REMOVE(bo, bo_synclist); 2771 syncer_worklist_len--; 2772 mtx_unlock(&sync_mtx); 2773 bo->bo_flag &= ~BO_ONWORKLST; 2774 } 2775 } 2776 #ifdef INVARIANTS 2777 bv = &bo->bo_clean; 2778 bp = TAILQ_FIRST(&bv->bv_hd); 2779 KASSERT(bp == NULL || bp->b_bufobj == bo, 2780 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2781 bp = TAILQ_LAST(&bv->bv_hd, buflists); 2782 KASSERT(bp == NULL || bp->b_bufobj == bo, 2783 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2784 bv = &bo->bo_dirty; 2785 bp = TAILQ_FIRST(&bv->bv_hd); 2786 KASSERT(bp == NULL || bp->b_bufobj == bo, 2787 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2788 bp = TAILQ_LAST(&bv->bv_hd, buflists); 2789 KASSERT(bp == NULL || bp->b_bufobj == bo, 2790 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2791 #endif 2792 BO_UNLOCK(bo); 2793 } 2794 2795 static void 2796 v_init_counters(struct vnode *vp) 2797 { 2798 2799 VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0, 2800 vp, ("%s called for an initialized vnode", __FUNCTION__)); 2801 ASSERT_VI_UNLOCKED(vp, __FUNCTION__); 2802 2803 refcount_init(&vp->v_holdcnt, 1); 2804 refcount_init(&vp->v_usecount, 1); 2805 } 2806 2807 /* 2808 * Grab a particular vnode from the free list, increment its 2809 * reference count and lock it. VIRF_DOOMED is set if the vnode 2810 * is being destroyed. Only callers who specify LK_RETRY will 2811 * see doomed vnodes. If inactive processing was delayed in 2812 * vput try to do it here. 2813 * 2814 * usecount is manipulated using atomics without holding any locks. 2815 * 2816 * holdcnt can be manipulated using atomics without holding any locks, 2817 * except when transitioning 1<->0, in which case the interlock is held. 2818 * 2819 * Consumers which don't guarantee liveness of the vnode can use SMR to 2820 * try to get a reference. Note this operation can fail since the vnode 2821 * may be awaiting getting freed by the time they get to it. 2822 */ 2823 enum vgetstate 2824 vget_prep_smr(struct vnode *vp) 2825 { 2826 enum vgetstate vs; 2827 2828 VFS_SMR_ASSERT_ENTERED(); 2829 2830 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 2831 vs = VGET_USECOUNT; 2832 } else { 2833 if (vhold_smr(vp)) 2834 vs = VGET_HOLDCNT; 2835 else 2836 vs = VGET_NONE; 2837 } 2838 return (vs); 2839 } 2840 2841 enum vgetstate 2842 vget_prep(struct vnode *vp) 2843 { 2844 enum vgetstate vs; 2845 2846 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 2847 vs = VGET_USECOUNT; 2848 } else { 2849 vhold(vp); 2850 vs = VGET_HOLDCNT; 2851 } 2852 return (vs); 2853 } 2854 2855 void 2856 vget_abort(struct vnode *vp, enum vgetstate vs) 2857 { 2858 2859 switch (vs) { 2860 case VGET_USECOUNT: 2861 vrele(vp); 2862 break; 2863 case VGET_HOLDCNT: 2864 vdrop(vp); 2865 break; 2866 default: 2867 __assert_unreachable(); 2868 } 2869 } 2870 2871 int 2872 vget(struct vnode *vp, int flags) 2873 { 2874 enum vgetstate vs; 2875 2876 vs = vget_prep(vp); 2877 return (vget_finish(vp, flags, vs)); 2878 } 2879 2880 int 2881 vget_finish(struct vnode *vp, int flags, enum vgetstate vs) 2882 { 2883 int error; 2884 2885 if ((flags & LK_INTERLOCK) != 0) 2886 ASSERT_VI_LOCKED(vp, __func__); 2887 else 2888 ASSERT_VI_UNLOCKED(vp, __func__); 2889 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 2890 VNPASS(vp->v_holdcnt > 0, vp); 2891 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 2892 2893 error = vn_lock(vp, flags); 2894 if (__predict_false(error != 0)) { 2895 vget_abort(vp, vs); 2896 CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__, 2897 vp); 2898 return (error); 2899 } 2900 2901 vget_finish_ref(vp, vs); 2902 return (0); 2903 } 2904 2905 void 2906 vget_finish_ref(struct vnode *vp, enum vgetstate vs) 2907 { 2908 int old; 2909 2910 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 2911 VNPASS(vp->v_holdcnt > 0, vp); 2912 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 2913 2914 if (vs == VGET_USECOUNT) 2915 return; 2916 2917 /* 2918 * We hold the vnode. If the usecount is 0 it will be utilized to keep 2919 * the vnode around. Otherwise someone else lended their hold count and 2920 * we have to drop ours. 2921 */ 2922 old = atomic_fetchadd_int(&vp->v_usecount, 1); 2923 VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old)); 2924 if (old != 0) { 2925 #ifdef INVARIANTS 2926 old = atomic_fetchadd_int(&vp->v_holdcnt, -1); 2927 VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); 2928 #else 2929 refcount_release(&vp->v_holdcnt); 2930 #endif 2931 } 2932 } 2933 2934 void 2935 vref(struct vnode *vp) 2936 { 2937 enum vgetstate vs; 2938 2939 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 2940 vs = vget_prep(vp); 2941 vget_finish_ref(vp, vs); 2942 } 2943 2944 void 2945 vrefact(struct vnode *vp) 2946 { 2947 2948 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 2949 #ifdef INVARIANTS 2950 int old = atomic_fetchadd_int(&vp->v_usecount, 1); 2951 VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old)); 2952 #else 2953 refcount_acquire(&vp->v_usecount); 2954 #endif 2955 } 2956 2957 void 2958 vlazy(struct vnode *vp) 2959 { 2960 struct mount *mp; 2961 2962 VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); 2963 2964 if ((vp->v_mflag & VMP_LAZYLIST) != 0) 2965 return; 2966 /* 2967 * We may get here for inactive routines after the vnode got doomed. 2968 */ 2969 if (VN_IS_DOOMED(vp)) 2970 return; 2971 mp = vp->v_mount; 2972 mtx_lock(&mp->mnt_listmtx); 2973 if ((vp->v_mflag & VMP_LAZYLIST) == 0) { 2974 vp->v_mflag |= VMP_LAZYLIST; 2975 TAILQ_INSERT_TAIL(&mp->mnt_lazyvnodelist, vp, v_lazylist); 2976 mp->mnt_lazyvnodelistsize++; 2977 } 2978 mtx_unlock(&mp->mnt_listmtx); 2979 } 2980 2981 /* 2982 * This routine is only meant to be called from vgonel prior to dooming 2983 * the vnode. 2984 */ 2985 static void 2986 vunlazy_gone(struct vnode *vp) 2987 { 2988 struct mount *mp; 2989 2990 ASSERT_VOP_ELOCKED(vp, __func__); 2991 ASSERT_VI_LOCKED(vp, __func__); 2992 VNPASS(!VN_IS_DOOMED(vp), vp); 2993 2994 if (vp->v_mflag & VMP_LAZYLIST) { 2995 mp = vp->v_mount; 2996 mtx_lock(&mp->mnt_listmtx); 2997 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 2998 vp->v_mflag &= ~VMP_LAZYLIST; 2999 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3000 mp->mnt_lazyvnodelistsize--; 3001 mtx_unlock(&mp->mnt_listmtx); 3002 } 3003 } 3004 3005 static void 3006 vdefer_inactive(struct vnode *vp) 3007 { 3008 3009 ASSERT_VI_LOCKED(vp, __func__); 3010 VNASSERT(vp->v_holdcnt > 0, vp, 3011 ("%s: vnode without hold count", __func__)); 3012 if (VN_IS_DOOMED(vp)) { 3013 vdropl(vp); 3014 return; 3015 } 3016 if (vp->v_iflag & VI_DEFINACT) { 3017 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 3018 vdropl(vp); 3019 return; 3020 } 3021 if (vp->v_usecount > 0) { 3022 vp->v_iflag &= ~VI_OWEINACT; 3023 vdropl(vp); 3024 return; 3025 } 3026 vlazy(vp); 3027 vp->v_iflag |= VI_DEFINACT; 3028 VI_UNLOCK(vp); 3029 counter_u64_add(deferred_inact, 1); 3030 } 3031 3032 static void 3033 vdefer_inactive_unlocked(struct vnode *vp) 3034 { 3035 3036 VI_LOCK(vp); 3037 if ((vp->v_iflag & VI_OWEINACT) == 0) { 3038 vdropl(vp); 3039 return; 3040 } 3041 vdefer_inactive(vp); 3042 } 3043 3044 enum vput_op { VRELE, VPUT, VUNREF }; 3045 3046 /* 3047 * Handle ->v_usecount transitioning to 0. 3048 * 3049 * By releasing the last usecount we take ownership of the hold count which 3050 * provides liveness of the vnode, meaning we have to vdrop. 3051 * 3052 * For all vnodes we may need to perform inactive processing. It requires an 3053 * exclusive lock on the vnode, while it is legal to call here with only a 3054 * shared lock (or no locks). If locking the vnode in an expected manner fails, 3055 * inactive processing gets deferred to the syncer. 3056 * 3057 * XXX Some filesystems pass in an exclusively locked vnode and strongly depend 3058 * on the lock being held all the way until VOP_INACTIVE. This in particular 3059 * happens with UFS which adds half-constructed vnodes to the hash, where they 3060 * can be found by other code. 3061 */ 3062 static void 3063 vput_final(struct vnode *vp, enum vput_op func) 3064 { 3065 int error; 3066 bool want_unlock; 3067 3068 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3069 VNPASS(vp->v_holdcnt > 0, vp); 3070 3071 VI_LOCK(vp); 3072 3073 /* 3074 * By the time we got here someone else might have transitioned 3075 * the count back to > 0. 3076 */ 3077 if (vp->v_usecount > 0) 3078 goto out; 3079 3080 /* 3081 * If the vnode is doomed vgone already performed inactive processing 3082 * (if needed). 3083 */ 3084 if (VN_IS_DOOMED(vp)) 3085 goto out; 3086 3087 if (__predict_true(VOP_NEED_INACTIVE(vp) == 0)) 3088 goto out; 3089 3090 if (vp->v_iflag & VI_DOINGINACT) 3091 goto out; 3092 3093 /* 3094 * Locking operations here will drop the interlock and possibly the 3095 * vnode lock, opening a window where the vnode can get doomed all the 3096 * while ->v_usecount is 0. Set VI_OWEINACT to let vgone know to 3097 * perform inactive. 3098 */ 3099 vp->v_iflag |= VI_OWEINACT; 3100 want_unlock = false; 3101 error = 0; 3102 switch (func) { 3103 case VRELE: 3104 switch (VOP_ISLOCKED(vp)) { 3105 case LK_EXCLUSIVE: 3106 break; 3107 case LK_EXCLOTHER: 3108 case 0: 3109 want_unlock = true; 3110 error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); 3111 VI_LOCK(vp); 3112 break; 3113 default: 3114 /* 3115 * The lock has at least one sharer, but we have no way 3116 * to conclude whether this is us. Play it safe and 3117 * defer processing. 3118 */ 3119 error = EAGAIN; 3120 break; 3121 } 3122 break; 3123 case VPUT: 3124 want_unlock = true; 3125 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3126 error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK | 3127 LK_NOWAIT); 3128 VI_LOCK(vp); 3129 } 3130 break; 3131 case VUNREF: 3132 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3133 error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK); 3134 VI_LOCK(vp); 3135 } 3136 break; 3137 } 3138 if (error == 0) { 3139 vinactive(vp); 3140 if (want_unlock) 3141 VOP_UNLOCK(vp); 3142 vdropl(vp); 3143 } else { 3144 vdefer_inactive(vp); 3145 } 3146 return; 3147 out: 3148 if (func == VPUT) 3149 VOP_UNLOCK(vp); 3150 vdropl(vp); 3151 } 3152 3153 /* 3154 * Decrement ->v_usecount for a vnode. 3155 * 3156 * Releasing the last use count requires additional processing, see vput_final 3157 * above for details. 3158 * 3159 * Comment above each variant denotes lock state on entry and exit. 3160 */ 3161 3162 /* 3163 * in: any 3164 * out: same as passed in 3165 */ 3166 void 3167 vrele(struct vnode *vp) 3168 { 3169 3170 ASSERT_VI_UNLOCKED(vp, __func__); 3171 if (!refcount_release(&vp->v_usecount)) 3172 return; 3173 vput_final(vp, VRELE); 3174 } 3175 3176 /* 3177 * in: locked 3178 * out: unlocked 3179 */ 3180 void 3181 vput(struct vnode *vp) 3182 { 3183 3184 ASSERT_VOP_LOCKED(vp, __func__); 3185 ASSERT_VI_UNLOCKED(vp, __func__); 3186 if (!refcount_release(&vp->v_usecount)) { 3187 VOP_UNLOCK(vp); 3188 return; 3189 } 3190 vput_final(vp, VPUT); 3191 } 3192 3193 /* 3194 * in: locked 3195 * out: locked 3196 */ 3197 void 3198 vunref(struct vnode *vp) 3199 { 3200 3201 ASSERT_VOP_LOCKED(vp, __func__); 3202 ASSERT_VI_UNLOCKED(vp, __func__); 3203 if (!refcount_release(&vp->v_usecount)) 3204 return; 3205 vput_final(vp, VUNREF); 3206 } 3207 3208 void 3209 vhold(struct vnode *vp) 3210 { 3211 int old; 3212 3213 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3214 old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3215 VNASSERT(old >= 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3216 ("%s: wrong hold count %d", __func__, old)); 3217 if (old == 0) 3218 vn_freevnodes_dec(); 3219 } 3220 3221 void 3222 vholdnz(struct vnode *vp) 3223 { 3224 3225 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3226 #ifdef INVARIANTS 3227 int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3228 VNASSERT(old > 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3229 ("%s: wrong hold count %d", __func__, old)); 3230 #else 3231 atomic_add_int(&vp->v_holdcnt, 1); 3232 #endif 3233 } 3234 3235 /* 3236 * Grab a hold count unless the vnode is freed. 3237 * 3238 * Only use this routine if vfs smr is the only protection you have against 3239 * freeing the vnode. 3240 * 3241 * The code loops trying to add a hold count as long as the VHOLD_NO_SMR flag 3242 * is not set. After the flag is set the vnode becomes immutable to anyone but 3243 * the thread which managed to set the flag. 3244 * 3245 * It may be tempting to replace the loop with: 3246 * count = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3247 * if (count & VHOLD_NO_SMR) { 3248 * backpedal and error out; 3249 * } 3250 * 3251 * However, while this is more performant, it hinders debugging by eliminating 3252 * the previously mentioned invariant. 3253 */ 3254 static bool __always_inline 3255 _vhold_cond(struct vnode *vp) 3256 { 3257 int count; 3258 3259 count = atomic_load_int(&vp->v_holdcnt); 3260 for (;;) { 3261 if (count & VHOLD_NO_SMR) { 3262 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3263 ("non-zero hold count with flags %d\n", count)); 3264 return (false); 3265 } 3266 3267 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3268 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3269 if (count == 0) 3270 vn_freevnodes_dec(); 3271 return (true); 3272 } 3273 } 3274 } 3275 3276 bool 3277 vhold_smr(struct vnode *vp) 3278 { 3279 3280 VFS_SMR_ASSERT_ENTERED(); 3281 return (_vhold_cond(vp)); 3282 } 3283 3284 /* 3285 * Special case for vnode recycling. 3286 * 3287 * Vnodes are present on the global list until UMA takes them out. 3288 * Attempts to recycle only need the relevant lock and have no use for SMR. 3289 */ 3290 static bool 3291 vhold_recycle(struct vnode *vp) 3292 { 3293 3294 mtx_assert(&vnode_list_mtx, MA_OWNED); 3295 return (_vhold_cond(vp)); 3296 } 3297 3298 static void __noinline 3299 vdbatch_process(struct vdbatch *vd) 3300 { 3301 struct vnode *vp; 3302 int i; 3303 3304 mtx_assert(&vd->lock, MA_OWNED); 3305 MPASS(curthread->td_pinned > 0); 3306 MPASS(vd->index == VDBATCH_SIZE); 3307 3308 mtx_lock(&vnode_list_mtx); 3309 critical_enter(); 3310 freevnodes += vd->freevnodes; 3311 for (i = 0; i < VDBATCH_SIZE; i++) { 3312 vp = vd->tab[i]; 3313 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 3314 TAILQ_INSERT_TAIL(&vnode_list, vp, v_vnodelist); 3315 MPASS(vp->v_dbatchcpu != NOCPU); 3316 vp->v_dbatchcpu = NOCPU; 3317 } 3318 mtx_unlock(&vnode_list_mtx); 3319 vd->freevnodes = 0; 3320 bzero(vd->tab, sizeof(vd->tab)); 3321 vd->index = 0; 3322 critical_exit(); 3323 } 3324 3325 static void 3326 vdbatch_enqueue(struct vnode *vp) 3327 { 3328 struct vdbatch *vd; 3329 3330 ASSERT_VI_LOCKED(vp, __func__); 3331 VNASSERT(!VN_IS_DOOMED(vp), vp, 3332 ("%s: deferring requeue of a doomed vnode", __func__)); 3333 3334 if (vp->v_dbatchcpu != NOCPU) { 3335 VI_UNLOCK(vp); 3336 return; 3337 } 3338 3339 sched_pin(); 3340 vd = DPCPU_PTR(vd); 3341 mtx_lock(&vd->lock); 3342 MPASS(vd->index < VDBATCH_SIZE); 3343 MPASS(vd->tab[vd->index] == NULL); 3344 /* 3345 * A hack: we depend on being pinned so that we know what to put in 3346 * ->v_dbatchcpu. 3347 */ 3348 vp->v_dbatchcpu = curcpu; 3349 vd->tab[vd->index] = vp; 3350 vd->index++; 3351 VI_UNLOCK(vp); 3352 if (vd->index == VDBATCH_SIZE) 3353 vdbatch_process(vd); 3354 mtx_unlock(&vd->lock); 3355 sched_unpin(); 3356 } 3357 3358 /* 3359 * This routine must only be called for vnodes which are about to be 3360 * deallocated. Supporting dequeue for arbitrary vndoes would require 3361 * validating that the locked batch matches. 3362 */ 3363 static void 3364 vdbatch_dequeue(struct vnode *vp) 3365 { 3366 struct vdbatch *vd; 3367 int i; 3368 short cpu; 3369 3370 VNASSERT(vp->v_type == VBAD || vp->v_type == VNON, vp, 3371 ("%s: called for a used vnode\n", __func__)); 3372 3373 cpu = vp->v_dbatchcpu; 3374 if (cpu == NOCPU) 3375 return; 3376 3377 vd = DPCPU_ID_PTR(cpu, vd); 3378 mtx_lock(&vd->lock); 3379 for (i = 0; i < vd->index; i++) { 3380 if (vd->tab[i] != vp) 3381 continue; 3382 vp->v_dbatchcpu = NOCPU; 3383 vd->index--; 3384 vd->tab[i] = vd->tab[vd->index]; 3385 vd->tab[vd->index] = NULL; 3386 break; 3387 } 3388 mtx_unlock(&vd->lock); 3389 /* 3390 * Either we dequeued the vnode above or the target CPU beat us to it. 3391 */ 3392 MPASS(vp->v_dbatchcpu == NOCPU); 3393 } 3394 3395 /* 3396 * Drop the hold count of the vnode. If this is the last reference to 3397 * the vnode we place it on the free list unless it has been vgone'd 3398 * (marked VIRF_DOOMED) in which case we will free it. 3399 * 3400 * Because the vnode vm object keeps a hold reference on the vnode if 3401 * there is at least one resident non-cached page, the vnode cannot 3402 * leave the active list without the page cleanup done. 3403 */ 3404 static void 3405 vdrop_deactivate(struct vnode *vp) 3406 { 3407 struct mount *mp; 3408 3409 ASSERT_VI_LOCKED(vp, __func__); 3410 /* 3411 * Mark a vnode as free: remove it from its active list 3412 * and put it up for recycling on the freelist. 3413 */ 3414 VNASSERT(!VN_IS_DOOMED(vp), vp, 3415 ("vdrop: returning doomed vnode")); 3416 VNASSERT(vp->v_op != NULL, vp, 3417 ("vdrop: vnode already reclaimed.")); 3418 VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp, 3419 ("vnode with VI_OWEINACT set")); 3420 VNASSERT((vp->v_iflag & VI_DEFINACT) == 0, vp, 3421 ("vnode with VI_DEFINACT set")); 3422 if (vp->v_mflag & VMP_LAZYLIST) { 3423 mp = vp->v_mount; 3424 mtx_lock(&mp->mnt_listmtx); 3425 VNASSERT(vp->v_mflag & VMP_LAZYLIST, vp, ("lost VMP_LAZYLIST")); 3426 /* 3427 * Don't remove the vnode from the lazy list if another thread 3428 * has increased the hold count. It may have re-enqueued the 3429 * vnode to the lazy list and is now responsible for its 3430 * removal. 3431 */ 3432 if (vp->v_holdcnt == 0) { 3433 vp->v_mflag &= ~VMP_LAZYLIST; 3434 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3435 mp->mnt_lazyvnodelistsize--; 3436 } 3437 mtx_unlock(&mp->mnt_listmtx); 3438 } 3439 vdbatch_enqueue(vp); 3440 } 3441 3442 static void __noinline 3443 vdropl_final(struct vnode *vp) 3444 { 3445 3446 ASSERT_VI_LOCKED(vp, __func__); 3447 VNPASS(VN_IS_DOOMED(vp), vp); 3448 /* 3449 * Set the VHOLD_NO_SMR flag. 3450 * 3451 * We may be racing against vhold_smr. If they win we can just pretend 3452 * we never got this far, they will vdrop later. 3453 */ 3454 if (__predict_false(!atomic_cmpset_int(&vp->v_holdcnt, 0, VHOLD_NO_SMR))) { 3455 vn_freevnodes_inc(); 3456 VI_UNLOCK(vp); 3457 /* 3458 * We lost the aforementioned race. Any subsequent access is 3459 * invalid as they might have managed to vdropl on their own. 3460 */ 3461 return; 3462 } 3463 /* 3464 * Don't bump freevnodes as this one is going away. 3465 */ 3466 freevnode(vp); 3467 } 3468 3469 void 3470 vdrop(struct vnode *vp) 3471 { 3472 3473 ASSERT_VI_UNLOCKED(vp, __func__); 3474 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3475 if (refcount_release_if_not_last(&vp->v_holdcnt)) 3476 return; 3477 VI_LOCK(vp); 3478 vdropl(vp); 3479 } 3480 3481 void 3482 vdropl(struct vnode *vp) 3483 { 3484 3485 ASSERT_VI_LOCKED(vp, __func__); 3486 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3487 if (!refcount_release(&vp->v_holdcnt)) { 3488 VI_UNLOCK(vp); 3489 return; 3490 } 3491 if (!VN_IS_DOOMED(vp)) { 3492 vn_freevnodes_inc(); 3493 vdrop_deactivate(vp); 3494 /* 3495 * Also unlocks the interlock. We can't assert on it as we 3496 * released our hold and by now the vnode might have been 3497 * freed. 3498 */ 3499 return; 3500 } 3501 vdropl_final(vp); 3502 } 3503 3504 /* 3505 * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT 3506 * flags. DOINGINACT prevents us from recursing in calls to vinactive. 3507 */ 3508 static void 3509 vinactivef(struct vnode *vp) 3510 { 3511 struct vm_object *obj; 3512 3513 ASSERT_VOP_ELOCKED(vp, "vinactive"); 3514 ASSERT_VI_LOCKED(vp, "vinactive"); 3515 VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp, 3516 ("vinactive: recursed on VI_DOINGINACT")); 3517 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3518 vp->v_iflag |= VI_DOINGINACT; 3519 vp->v_iflag &= ~VI_OWEINACT; 3520 VI_UNLOCK(vp); 3521 /* 3522 * Before moving off the active list, we must be sure that any 3523 * modified pages are converted into the vnode's dirty 3524 * buffers, since these will no longer be checked once the 3525 * vnode is on the inactive list. 3526 * 3527 * The write-out of the dirty pages is asynchronous. At the 3528 * point that VOP_INACTIVE() is called, there could still be 3529 * pending I/O and dirty pages in the object. 3530 */ 3531 if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 3532 vm_object_mightbedirty(obj)) { 3533 VM_OBJECT_WLOCK(obj); 3534 vm_object_page_clean(obj, 0, 0, 0); 3535 VM_OBJECT_WUNLOCK(obj); 3536 } 3537 VOP_INACTIVE(vp); 3538 VI_LOCK(vp); 3539 VNASSERT(vp->v_iflag & VI_DOINGINACT, vp, 3540 ("vinactive: lost VI_DOINGINACT")); 3541 vp->v_iflag &= ~VI_DOINGINACT; 3542 } 3543 3544 void 3545 vinactive(struct vnode *vp) 3546 { 3547 3548 ASSERT_VOP_ELOCKED(vp, "vinactive"); 3549 ASSERT_VI_LOCKED(vp, "vinactive"); 3550 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3551 3552 if ((vp->v_iflag & VI_OWEINACT) == 0) 3553 return; 3554 if (vp->v_iflag & VI_DOINGINACT) 3555 return; 3556 if (vp->v_usecount > 0) { 3557 vp->v_iflag &= ~VI_OWEINACT; 3558 return; 3559 } 3560 vinactivef(vp); 3561 } 3562 3563 /* 3564 * Remove any vnodes in the vnode table belonging to mount point mp. 3565 * 3566 * If FORCECLOSE is not specified, there should not be any active ones, 3567 * return error if any are found (nb: this is a user error, not a 3568 * system error). If FORCECLOSE is specified, detach any active vnodes 3569 * that are found. 3570 * 3571 * If WRITECLOSE is set, only flush out regular file vnodes open for 3572 * writing. 3573 * 3574 * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped. 3575 * 3576 * `rootrefs' specifies the base reference count for the root vnode 3577 * of this filesystem. The root vnode is considered busy if its 3578 * v_usecount exceeds this value. On a successful return, vflush(, td) 3579 * will call vrele() on the root vnode exactly rootrefs times. 3580 * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must 3581 * be zero. 3582 */ 3583 #ifdef DIAGNOSTIC 3584 static int busyprt = 0; /* print out busy vnodes */ 3585 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes"); 3586 #endif 3587 3588 int 3589 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td) 3590 { 3591 struct vnode *vp, *mvp, *rootvp = NULL; 3592 struct vattr vattr; 3593 int busy = 0, error; 3594 3595 CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp, 3596 rootrefs, flags); 3597 if (rootrefs > 0) { 3598 KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0, 3599 ("vflush: bad args")); 3600 /* 3601 * Get the filesystem root vnode. We can vput() it 3602 * immediately, since with rootrefs > 0, it won't go away. 3603 */ 3604 if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) { 3605 CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d", 3606 __func__, error); 3607 return (error); 3608 } 3609 vput(rootvp); 3610 } 3611 loop: 3612 MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { 3613 vholdl(vp); 3614 error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE); 3615 if (error) { 3616 vdrop(vp); 3617 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 3618 goto loop; 3619 } 3620 /* 3621 * Skip over a vnodes marked VV_SYSTEM. 3622 */ 3623 if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) { 3624 VOP_UNLOCK(vp); 3625 vdrop(vp); 3626 continue; 3627 } 3628 /* 3629 * If WRITECLOSE is set, flush out unlinked but still open 3630 * files (even if open only for reading) and regular file 3631 * vnodes open for writing. 3632 */ 3633 if (flags & WRITECLOSE) { 3634 if (vp->v_object != NULL) { 3635 VM_OBJECT_WLOCK(vp->v_object); 3636 vm_object_page_clean(vp->v_object, 0, 0, 0); 3637 VM_OBJECT_WUNLOCK(vp->v_object); 3638 } 3639 error = VOP_FSYNC(vp, MNT_WAIT, td); 3640 if (error != 0) { 3641 VOP_UNLOCK(vp); 3642 vdrop(vp); 3643 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 3644 return (error); 3645 } 3646 error = VOP_GETATTR(vp, &vattr, td->td_ucred); 3647 VI_LOCK(vp); 3648 3649 if ((vp->v_type == VNON || 3650 (error == 0 && vattr.va_nlink > 0)) && 3651 (vp->v_writecount <= 0 || vp->v_type != VREG)) { 3652 VOP_UNLOCK(vp); 3653 vdropl(vp); 3654 continue; 3655 } 3656 } else 3657 VI_LOCK(vp); 3658 /* 3659 * With v_usecount == 0, all we need to do is clear out the 3660 * vnode data structures and we are done. 3661 * 3662 * If FORCECLOSE is set, forcibly close the vnode. 3663 */ 3664 if (vp->v_usecount == 0 || (flags & FORCECLOSE)) { 3665 vgonel(vp); 3666 } else { 3667 busy++; 3668 #ifdef DIAGNOSTIC 3669 if (busyprt) 3670 vn_printf(vp, "vflush: busy vnode "); 3671 #endif 3672 } 3673 VOP_UNLOCK(vp); 3674 vdropl(vp); 3675 } 3676 if (rootrefs > 0 && (flags & FORCECLOSE) == 0) { 3677 /* 3678 * If just the root vnode is busy, and if its refcount 3679 * is equal to `rootrefs', then go ahead and kill it. 3680 */ 3681 VI_LOCK(rootvp); 3682 KASSERT(busy > 0, ("vflush: not busy")); 3683 VNASSERT(rootvp->v_usecount >= rootrefs, rootvp, 3684 ("vflush: usecount %d < rootrefs %d", 3685 rootvp->v_usecount, rootrefs)); 3686 if (busy == 1 && rootvp->v_usecount == rootrefs) { 3687 VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK); 3688 vgone(rootvp); 3689 VOP_UNLOCK(rootvp); 3690 busy = 0; 3691 } else 3692 VI_UNLOCK(rootvp); 3693 } 3694 if (busy) { 3695 CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__, 3696 busy); 3697 return (EBUSY); 3698 } 3699 for (; rootrefs > 0; rootrefs--) 3700 vrele(rootvp); 3701 return (0); 3702 } 3703 3704 /* 3705 * Recycle an unused vnode to the front of the free list. 3706 */ 3707 int 3708 vrecycle(struct vnode *vp) 3709 { 3710 int recycled; 3711 3712 VI_LOCK(vp); 3713 recycled = vrecyclel(vp); 3714 VI_UNLOCK(vp); 3715 return (recycled); 3716 } 3717 3718 /* 3719 * vrecycle, with the vp interlock held. 3720 */ 3721 int 3722 vrecyclel(struct vnode *vp) 3723 { 3724 int recycled; 3725 3726 ASSERT_VOP_ELOCKED(vp, __func__); 3727 ASSERT_VI_LOCKED(vp, __func__); 3728 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3729 recycled = 0; 3730 if (vp->v_usecount == 0) { 3731 recycled = 1; 3732 vgonel(vp); 3733 } 3734 return (recycled); 3735 } 3736 3737 /* 3738 * Eliminate all activity associated with a vnode 3739 * in preparation for reuse. 3740 */ 3741 void 3742 vgone(struct vnode *vp) 3743 { 3744 VI_LOCK(vp); 3745 vgonel(vp); 3746 VI_UNLOCK(vp); 3747 } 3748 3749 static void 3750 notify_lowervp_vfs_dummy(struct mount *mp __unused, 3751 struct vnode *lowervp __unused) 3752 { 3753 } 3754 3755 /* 3756 * Notify upper mounts about reclaimed or unlinked vnode. 3757 */ 3758 void 3759 vfs_notify_upper(struct vnode *vp, int event) 3760 { 3761 static struct vfsops vgonel_vfsops = { 3762 .vfs_reclaim_lowervp = notify_lowervp_vfs_dummy, 3763 .vfs_unlink_lowervp = notify_lowervp_vfs_dummy, 3764 }; 3765 struct mount *mp, *ump, *mmp; 3766 3767 mp = vp->v_mount; 3768 if (mp == NULL) 3769 return; 3770 if (TAILQ_EMPTY(&mp->mnt_uppers)) 3771 return; 3772 3773 mmp = malloc(sizeof(struct mount), M_TEMP, M_WAITOK | M_ZERO); 3774 mmp->mnt_op = &vgonel_vfsops; 3775 mmp->mnt_kern_flag |= MNTK_MARKER; 3776 MNT_ILOCK(mp); 3777 mp->mnt_kern_flag |= MNTK_VGONE_UPPER; 3778 for (ump = TAILQ_FIRST(&mp->mnt_uppers); ump != NULL;) { 3779 if ((ump->mnt_kern_flag & MNTK_MARKER) != 0) { 3780 ump = TAILQ_NEXT(ump, mnt_upper_link); 3781 continue; 3782 } 3783 TAILQ_INSERT_AFTER(&mp->mnt_uppers, ump, mmp, mnt_upper_link); 3784 MNT_IUNLOCK(mp); 3785 switch (event) { 3786 case VFS_NOTIFY_UPPER_RECLAIM: 3787 VFS_RECLAIM_LOWERVP(ump, vp); 3788 break; 3789 case VFS_NOTIFY_UPPER_UNLINK: 3790 VFS_UNLINK_LOWERVP(ump, vp); 3791 break; 3792 default: 3793 KASSERT(0, ("invalid event %d", event)); 3794 break; 3795 } 3796 MNT_ILOCK(mp); 3797 ump = TAILQ_NEXT(mmp, mnt_upper_link); 3798 TAILQ_REMOVE(&mp->mnt_uppers, mmp, mnt_upper_link); 3799 } 3800 free(mmp, M_TEMP); 3801 mp->mnt_kern_flag &= ~MNTK_VGONE_UPPER; 3802 if ((mp->mnt_kern_flag & MNTK_VGONE_WAITER) != 0) { 3803 mp->mnt_kern_flag &= ~MNTK_VGONE_WAITER; 3804 wakeup(&mp->mnt_uppers); 3805 } 3806 MNT_IUNLOCK(mp); 3807 } 3808 3809 /* 3810 * vgone, with the vp interlock held. 3811 */ 3812 static void 3813 vgonel(struct vnode *vp) 3814 { 3815 struct thread *td; 3816 struct mount *mp; 3817 vm_object_t object; 3818 bool active, doinginact, oweinact; 3819 3820 ASSERT_VOP_ELOCKED(vp, "vgonel"); 3821 ASSERT_VI_LOCKED(vp, "vgonel"); 3822 VNASSERT(vp->v_holdcnt, vp, 3823 ("vgonel: vp %p has no reference.", vp)); 3824 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3825 td = curthread; 3826 3827 /* 3828 * Don't vgonel if we're already doomed. 3829 */ 3830 if (vp->v_irflag & VIRF_DOOMED) 3831 return; 3832 /* 3833 * Paired with freevnode. 3834 */ 3835 vn_seqc_write_begin_locked(vp); 3836 vunlazy_gone(vp); 3837 vp->v_irflag |= VIRF_DOOMED; 3838 3839 /* 3840 * Check to see if the vnode is in use. If so, we have to 3841 * call VOP_CLOSE() and VOP_INACTIVE(). 3842 * 3843 * It could be that VOP_INACTIVE() requested reclamation, in 3844 * which case we should avoid recursion, so check 3845 * VI_DOINGINACT. This is not precise but good enough. 3846 */ 3847 active = vp->v_usecount > 0; 3848 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 3849 doinginact = (vp->v_iflag & VI_DOINGINACT) != 0; 3850 3851 /* 3852 * If we need to do inactive VI_OWEINACT will be set. 3853 */ 3854 if (vp->v_iflag & VI_DEFINACT) { 3855 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 3856 vp->v_iflag &= ~VI_DEFINACT; 3857 vdropl(vp); 3858 } else { 3859 VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count")); 3860 VI_UNLOCK(vp); 3861 } 3862 cache_purge_vgone(vp); 3863 vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM); 3864 3865 /* 3866 * If purging an active vnode, it must be closed and 3867 * deactivated before being reclaimed. 3868 */ 3869 if (active) 3870 VOP_CLOSE(vp, FNONBLOCK, NOCRED, td); 3871 if ((oweinact || active) && !doinginact) { 3872 VI_LOCK(vp); 3873 vinactivef(vp); 3874 VI_UNLOCK(vp); 3875 } 3876 if (vp->v_type == VSOCK) 3877 vfs_unp_reclaim(vp); 3878 3879 /* 3880 * Clean out any buffers associated with the vnode. 3881 * If the flush fails, just toss the buffers. 3882 */ 3883 mp = NULL; 3884 if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd)) 3885 (void) vn_start_secondary_write(vp, &mp, V_WAIT); 3886 if (vinvalbuf(vp, V_SAVE, 0, 0) != 0) { 3887 while (vinvalbuf(vp, 0, 0, 0) != 0) 3888 ; 3889 } 3890 3891 BO_LOCK(&vp->v_bufobj); 3892 KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) && 3893 vp->v_bufobj.bo_dirty.bv_cnt == 0 && 3894 TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) && 3895 vp->v_bufobj.bo_clean.bv_cnt == 0, 3896 ("vp %p bufobj not invalidated", vp)); 3897 3898 /* 3899 * For VMIO bufobj, BO_DEAD is set later, or in 3900 * vm_object_terminate() after the object's page queue is 3901 * flushed. 3902 */ 3903 object = vp->v_bufobj.bo_object; 3904 if (object == NULL) 3905 vp->v_bufobj.bo_flag |= BO_DEAD; 3906 BO_UNLOCK(&vp->v_bufobj); 3907 3908 /* 3909 * Handle the VM part. Tmpfs handles v_object on its own (the 3910 * OBJT_VNODE check). Nullfs or other bypassing filesystems 3911 * should not touch the object borrowed from the lower vnode 3912 * (the handle check). 3913 */ 3914 if (object != NULL && object->type == OBJT_VNODE && 3915 object->handle == vp) 3916 vnode_destroy_vobject(vp); 3917 3918 /* 3919 * Reclaim the vnode. 3920 */ 3921 if (VOP_RECLAIM(vp)) 3922 panic("vgone: cannot reclaim"); 3923 if (mp != NULL) 3924 vn_finished_secondary_write(mp); 3925 VNASSERT(vp->v_object == NULL, vp, 3926 ("vop_reclaim left v_object vp=%p", vp)); 3927 /* 3928 * Clear the advisory locks and wake up waiting threads. 3929 */ 3930 (void)VOP_ADVLOCKPURGE(vp); 3931 vp->v_lockf = NULL; 3932 /* 3933 * Delete from old mount point vnode list. 3934 */ 3935 delmntque(vp); 3936 /* 3937 * Done with purge, reset to the standard lock and invalidate 3938 * the vnode. 3939 */ 3940 VI_LOCK(vp); 3941 vp->v_vnlock = &vp->v_lock; 3942 vp->v_op = &dead_vnodeops; 3943 vp->v_type = VBAD; 3944 } 3945 3946 /* 3947 * Print out a description of a vnode. 3948 */ 3949 static const char * const typename[] = 3950 {"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD", 3951 "VMARKER"}; 3952 3953 _Static_assert((VHOLD_ALL_FLAGS & ~VHOLD_NO_SMR) == 0, 3954 "new hold count flag not added to vn_printf"); 3955 3956 void 3957 vn_printf(struct vnode *vp, const char *fmt, ...) 3958 { 3959 va_list ap; 3960 char buf[256], buf2[16]; 3961 u_long flags; 3962 u_int holdcnt; 3963 3964 va_start(ap, fmt); 3965 vprintf(fmt, ap); 3966 va_end(ap); 3967 printf("%p: ", (void *)vp); 3968 printf("type %s\n", typename[vp->v_type]); 3969 holdcnt = atomic_load_int(&vp->v_holdcnt); 3970 printf(" usecount %d, writecount %d, refcount %d seqc users %d", 3971 vp->v_usecount, vp->v_writecount, holdcnt & ~VHOLD_ALL_FLAGS, 3972 vp->v_seqc_users); 3973 switch (vp->v_type) { 3974 case VDIR: 3975 printf(" mountedhere %p\n", vp->v_mountedhere); 3976 break; 3977 case VCHR: 3978 printf(" rdev %p\n", vp->v_rdev); 3979 break; 3980 case VSOCK: 3981 printf(" socket %p\n", vp->v_unpcb); 3982 break; 3983 case VFIFO: 3984 printf(" fifoinfo %p\n", vp->v_fifoinfo); 3985 break; 3986 default: 3987 printf("\n"); 3988 break; 3989 } 3990 buf[0] = '\0'; 3991 buf[1] = '\0'; 3992 if (holdcnt & VHOLD_NO_SMR) 3993 strlcat(buf, "|VHOLD_NO_SMR", sizeof(buf)); 3994 printf(" hold count flags (%s)\n", buf + 1); 3995 3996 buf[0] = '\0'; 3997 buf[1] = '\0'; 3998 if (vp->v_irflag & VIRF_DOOMED) 3999 strlcat(buf, "|VIRF_DOOMED", sizeof(buf)); 4000 if (vp->v_irflag & VIRF_PGREAD) 4001 strlcat(buf, "|VIRF_PGREAD", sizeof(buf)); 4002 flags = vp->v_irflag & ~(VIRF_DOOMED | VIRF_PGREAD); 4003 if (flags != 0) { 4004 snprintf(buf2, sizeof(buf2), "|VIRF(0x%lx)", flags); 4005 strlcat(buf, buf2, sizeof(buf)); 4006 } 4007 if (vp->v_vflag & VV_ROOT) 4008 strlcat(buf, "|VV_ROOT", sizeof(buf)); 4009 if (vp->v_vflag & VV_ISTTY) 4010 strlcat(buf, "|VV_ISTTY", sizeof(buf)); 4011 if (vp->v_vflag & VV_NOSYNC) 4012 strlcat(buf, "|VV_NOSYNC", sizeof(buf)); 4013 if (vp->v_vflag & VV_ETERNALDEV) 4014 strlcat(buf, "|VV_ETERNALDEV", sizeof(buf)); 4015 if (vp->v_vflag & VV_CACHEDLABEL) 4016 strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf)); 4017 if (vp->v_vflag & VV_VMSIZEVNLOCK) 4018 strlcat(buf, "|VV_VMSIZEVNLOCK", sizeof(buf)); 4019 if (vp->v_vflag & VV_COPYONWRITE) 4020 strlcat(buf, "|VV_COPYONWRITE", sizeof(buf)); 4021 if (vp->v_vflag & VV_SYSTEM) 4022 strlcat(buf, "|VV_SYSTEM", sizeof(buf)); 4023 if (vp->v_vflag & VV_PROCDEP) 4024 strlcat(buf, "|VV_PROCDEP", sizeof(buf)); 4025 if (vp->v_vflag & VV_NOKNOTE) 4026 strlcat(buf, "|VV_NOKNOTE", sizeof(buf)); 4027 if (vp->v_vflag & VV_DELETED) 4028 strlcat(buf, "|VV_DELETED", sizeof(buf)); 4029 if (vp->v_vflag & VV_MD) 4030 strlcat(buf, "|VV_MD", sizeof(buf)); 4031 if (vp->v_vflag & VV_FORCEINSMQ) 4032 strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf)); 4033 if (vp->v_vflag & VV_READLINK) 4034 strlcat(buf, "|VV_READLINK", sizeof(buf)); 4035 flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV | 4036 VV_CACHEDLABEL | VV_COPYONWRITE | VV_SYSTEM | VV_PROCDEP | 4037 VV_NOKNOTE | VV_DELETED | VV_MD | VV_FORCEINSMQ); 4038 if (flags != 0) { 4039 snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags); 4040 strlcat(buf, buf2, sizeof(buf)); 4041 } 4042 if (vp->v_iflag & VI_TEXT_REF) 4043 strlcat(buf, "|VI_TEXT_REF", sizeof(buf)); 4044 if (vp->v_iflag & VI_MOUNT) 4045 strlcat(buf, "|VI_MOUNT", sizeof(buf)); 4046 if (vp->v_iflag & VI_DOINGINACT) 4047 strlcat(buf, "|VI_DOINGINACT", sizeof(buf)); 4048 if (vp->v_iflag & VI_OWEINACT) 4049 strlcat(buf, "|VI_OWEINACT", sizeof(buf)); 4050 if (vp->v_iflag & VI_DEFINACT) 4051 strlcat(buf, "|VI_DEFINACT", sizeof(buf)); 4052 flags = vp->v_iflag & ~(VI_TEXT_REF | VI_MOUNT | VI_DOINGINACT | 4053 VI_OWEINACT | VI_DEFINACT); 4054 if (flags != 0) { 4055 snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags); 4056 strlcat(buf, buf2, sizeof(buf)); 4057 } 4058 if (vp->v_mflag & VMP_LAZYLIST) 4059 strlcat(buf, "|VMP_LAZYLIST", sizeof(buf)); 4060 flags = vp->v_mflag & ~(VMP_LAZYLIST); 4061 if (flags != 0) { 4062 snprintf(buf2, sizeof(buf2), "|VMP(0x%lx)", flags); 4063 strlcat(buf, buf2, sizeof(buf)); 4064 } 4065 printf(" flags (%s)\n", buf + 1); 4066 if (mtx_owned(VI_MTX(vp))) 4067 printf(" VI_LOCKed"); 4068 if (vp->v_object != NULL) 4069 printf(" v_object %p ref %d pages %d " 4070 "cleanbuf %d dirtybuf %d\n", 4071 vp->v_object, vp->v_object->ref_count, 4072 vp->v_object->resident_page_count, 4073 vp->v_bufobj.bo_clean.bv_cnt, 4074 vp->v_bufobj.bo_dirty.bv_cnt); 4075 printf(" "); 4076 lockmgr_printinfo(vp->v_vnlock); 4077 if (vp->v_data != NULL) 4078 VOP_PRINT(vp); 4079 } 4080 4081 #ifdef DDB 4082 /* 4083 * List all of the locked vnodes in the system. 4084 * Called when debugging the kernel. 4085 */ 4086 DB_SHOW_COMMAND(lockedvnods, lockedvnodes) 4087 { 4088 struct mount *mp; 4089 struct vnode *vp; 4090 4091 /* 4092 * Note: because this is DDB, we can't obey the locking semantics 4093 * for these structures, which means we could catch an inconsistent 4094 * state and dereference a nasty pointer. Not much to be done 4095 * about that. 4096 */ 4097 db_printf("Locked vnodes\n"); 4098 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4099 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4100 if (vp->v_type != VMARKER && VOP_ISLOCKED(vp)) 4101 vn_printf(vp, "vnode "); 4102 } 4103 } 4104 } 4105 4106 /* 4107 * Show details about the given vnode. 4108 */ 4109 DB_SHOW_COMMAND(vnode, db_show_vnode) 4110 { 4111 struct vnode *vp; 4112 4113 if (!have_addr) 4114 return; 4115 vp = (struct vnode *)addr; 4116 vn_printf(vp, "vnode "); 4117 } 4118 4119 /* 4120 * Show details about the given mount point. 4121 */ 4122 DB_SHOW_COMMAND(mount, db_show_mount) 4123 { 4124 struct mount *mp; 4125 struct vfsopt *opt; 4126 struct statfs *sp; 4127 struct vnode *vp; 4128 char buf[512]; 4129 uint64_t mflags; 4130 u_int flags; 4131 4132 if (!have_addr) { 4133 /* No address given, print short info about all mount points. */ 4134 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4135 db_printf("%p %s on %s (%s)\n", mp, 4136 mp->mnt_stat.f_mntfromname, 4137 mp->mnt_stat.f_mntonname, 4138 mp->mnt_stat.f_fstypename); 4139 if (db_pager_quit) 4140 break; 4141 } 4142 db_printf("\nMore info: show mount <addr>\n"); 4143 return; 4144 } 4145 4146 mp = (struct mount *)addr; 4147 db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname, 4148 mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename); 4149 4150 buf[0] = '\0'; 4151 mflags = mp->mnt_flag; 4152 #define MNT_FLAG(flag) do { \ 4153 if (mflags & (flag)) { \ 4154 if (buf[0] != '\0') \ 4155 strlcat(buf, ", ", sizeof(buf)); \ 4156 strlcat(buf, (#flag) + 4, sizeof(buf)); \ 4157 mflags &= ~(flag); \ 4158 } \ 4159 } while (0) 4160 MNT_FLAG(MNT_RDONLY); 4161 MNT_FLAG(MNT_SYNCHRONOUS); 4162 MNT_FLAG(MNT_NOEXEC); 4163 MNT_FLAG(MNT_NOSUID); 4164 MNT_FLAG(MNT_NFS4ACLS); 4165 MNT_FLAG(MNT_UNION); 4166 MNT_FLAG(MNT_ASYNC); 4167 MNT_FLAG(MNT_SUIDDIR); 4168 MNT_FLAG(MNT_SOFTDEP); 4169 MNT_FLAG(MNT_NOSYMFOLLOW); 4170 MNT_FLAG(MNT_GJOURNAL); 4171 MNT_FLAG(MNT_MULTILABEL); 4172 MNT_FLAG(MNT_ACLS); 4173 MNT_FLAG(MNT_NOATIME); 4174 MNT_FLAG(MNT_NOCLUSTERR); 4175 MNT_FLAG(MNT_NOCLUSTERW); 4176 MNT_FLAG(MNT_SUJ); 4177 MNT_FLAG(MNT_EXRDONLY); 4178 MNT_FLAG(MNT_EXPORTED); 4179 MNT_FLAG(MNT_DEFEXPORTED); 4180 MNT_FLAG(MNT_EXPORTANON); 4181 MNT_FLAG(MNT_EXKERB); 4182 MNT_FLAG(MNT_EXPUBLIC); 4183 MNT_FLAG(MNT_LOCAL); 4184 MNT_FLAG(MNT_QUOTA); 4185 MNT_FLAG(MNT_ROOTFS); 4186 MNT_FLAG(MNT_USER); 4187 MNT_FLAG(MNT_IGNORE); 4188 MNT_FLAG(MNT_UPDATE); 4189 MNT_FLAG(MNT_DELEXPORT); 4190 MNT_FLAG(MNT_RELOAD); 4191 MNT_FLAG(MNT_FORCE); 4192 MNT_FLAG(MNT_SNAPSHOT); 4193 MNT_FLAG(MNT_BYFSID); 4194 #undef MNT_FLAG 4195 if (mflags != 0) { 4196 if (buf[0] != '\0') 4197 strlcat(buf, ", ", sizeof(buf)); 4198 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4199 "0x%016jx", mflags); 4200 } 4201 db_printf(" mnt_flag = %s\n", buf); 4202 4203 buf[0] = '\0'; 4204 flags = mp->mnt_kern_flag; 4205 #define MNT_KERN_FLAG(flag) do { \ 4206 if (flags & (flag)) { \ 4207 if (buf[0] != '\0') \ 4208 strlcat(buf, ", ", sizeof(buf)); \ 4209 strlcat(buf, (#flag) + 5, sizeof(buf)); \ 4210 flags &= ~(flag); \ 4211 } \ 4212 } while (0) 4213 MNT_KERN_FLAG(MNTK_UNMOUNTF); 4214 MNT_KERN_FLAG(MNTK_ASYNC); 4215 MNT_KERN_FLAG(MNTK_SOFTDEP); 4216 MNT_KERN_FLAG(MNTK_DRAINING); 4217 MNT_KERN_FLAG(MNTK_REFEXPIRE); 4218 MNT_KERN_FLAG(MNTK_EXTENDED_SHARED); 4219 MNT_KERN_FLAG(MNTK_SHARED_WRITES); 4220 MNT_KERN_FLAG(MNTK_NO_IOPF); 4221 MNT_KERN_FLAG(MNTK_VGONE_UPPER); 4222 MNT_KERN_FLAG(MNTK_VGONE_WAITER); 4223 MNT_KERN_FLAG(MNTK_LOOKUP_EXCL_DOTDOT); 4224 MNT_KERN_FLAG(MNTK_MARKER); 4225 MNT_KERN_FLAG(MNTK_USES_BCACHE); 4226 MNT_KERN_FLAG(MNTK_FPLOOKUP); 4227 MNT_KERN_FLAG(MNTK_NOASYNC); 4228 MNT_KERN_FLAG(MNTK_UNMOUNT); 4229 MNT_KERN_FLAG(MNTK_MWAIT); 4230 MNT_KERN_FLAG(MNTK_SUSPEND); 4231 MNT_KERN_FLAG(MNTK_SUSPEND2); 4232 MNT_KERN_FLAG(MNTK_SUSPENDED); 4233 MNT_KERN_FLAG(MNTK_LOOKUP_SHARED); 4234 MNT_KERN_FLAG(MNTK_NOKNOTE); 4235 #undef MNT_KERN_FLAG 4236 if (flags != 0) { 4237 if (buf[0] != '\0') 4238 strlcat(buf, ", ", sizeof(buf)); 4239 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4240 "0x%08x", flags); 4241 } 4242 db_printf(" mnt_kern_flag = %s\n", buf); 4243 4244 db_printf(" mnt_opt = "); 4245 opt = TAILQ_FIRST(mp->mnt_opt); 4246 if (opt != NULL) { 4247 db_printf("%s", opt->name); 4248 opt = TAILQ_NEXT(opt, link); 4249 while (opt != NULL) { 4250 db_printf(", %s", opt->name); 4251 opt = TAILQ_NEXT(opt, link); 4252 } 4253 } 4254 db_printf("\n"); 4255 4256 sp = &mp->mnt_stat; 4257 db_printf(" mnt_stat = { version=%u type=%u flags=0x%016jx " 4258 "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju " 4259 "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju " 4260 "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n", 4261 (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags, 4262 (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize, 4263 (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree, 4264 (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files, 4265 (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites, 4266 (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads, 4267 (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax, 4268 (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]); 4269 4270 db_printf(" mnt_cred = { uid=%u ruid=%u", 4271 (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid); 4272 if (jailed(mp->mnt_cred)) 4273 db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id); 4274 db_printf(" }\n"); 4275 db_printf(" mnt_ref = %d (with %d in the struct)\n", 4276 vfs_mount_fetch_counter(mp, MNT_COUNT_REF), mp->mnt_ref); 4277 db_printf(" mnt_gen = %d\n", mp->mnt_gen); 4278 db_printf(" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize); 4279 db_printf(" mnt_lazyvnodelistsize = %d\n", 4280 mp->mnt_lazyvnodelistsize); 4281 db_printf(" mnt_writeopcount = %d (with %d in the struct)\n", 4282 vfs_mount_fetch_counter(mp, MNT_COUNT_WRITEOPCOUNT), mp->mnt_writeopcount); 4283 db_printf(" mnt_maxsymlinklen = %d\n", mp->mnt_maxsymlinklen); 4284 db_printf(" mnt_iosize_max = %d\n", mp->mnt_iosize_max); 4285 db_printf(" mnt_hashseed = %u\n", mp->mnt_hashseed); 4286 db_printf(" mnt_lockref = %d (with %d in the struct)\n", 4287 vfs_mount_fetch_counter(mp, MNT_COUNT_LOCKREF), mp->mnt_lockref); 4288 db_printf(" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes); 4289 db_printf(" mnt_secondary_accwrites = %d\n", 4290 mp->mnt_secondary_accwrites); 4291 db_printf(" mnt_gjprovider = %s\n", 4292 mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL"); 4293 db_printf(" mnt_vfs_ops = %d\n", mp->mnt_vfs_ops); 4294 4295 db_printf("\n\nList of active vnodes\n"); 4296 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4297 if (vp->v_type != VMARKER && vp->v_holdcnt > 0) { 4298 vn_printf(vp, "vnode "); 4299 if (db_pager_quit) 4300 break; 4301 } 4302 } 4303 db_printf("\n\nList of inactive vnodes\n"); 4304 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4305 if (vp->v_type != VMARKER && vp->v_holdcnt == 0) { 4306 vn_printf(vp, "vnode "); 4307 if (db_pager_quit) 4308 break; 4309 } 4310 } 4311 } 4312 #endif /* DDB */ 4313 4314 /* 4315 * Fill in a struct xvfsconf based on a struct vfsconf. 4316 */ 4317 static int 4318 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp) 4319 { 4320 struct xvfsconf xvfsp; 4321 4322 bzero(&xvfsp, sizeof(xvfsp)); 4323 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4324 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4325 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4326 xvfsp.vfc_flags = vfsp->vfc_flags; 4327 /* 4328 * These are unused in userland, we keep them 4329 * to not break binary compatibility. 4330 */ 4331 xvfsp.vfc_vfsops = NULL; 4332 xvfsp.vfc_next = NULL; 4333 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4334 } 4335 4336 #ifdef COMPAT_FREEBSD32 4337 struct xvfsconf32 { 4338 uint32_t vfc_vfsops; 4339 char vfc_name[MFSNAMELEN]; 4340 int32_t vfc_typenum; 4341 int32_t vfc_refcount; 4342 int32_t vfc_flags; 4343 uint32_t vfc_next; 4344 }; 4345 4346 static int 4347 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp) 4348 { 4349 struct xvfsconf32 xvfsp; 4350 4351 bzero(&xvfsp, sizeof(xvfsp)); 4352 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4353 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4354 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4355 xvfsp.vfc_flags = vfsp->vfc_flags; 4356 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4357 } 4358 #endif 4359 4360 /* 4361 * Top level filesystem related information gathering. 4362 */ 4363 static int 4364 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS) 4365 { 4366 struct vfsconf *vfsp; 4367 int error; 4368 4369 error = 0; 4370 vfsconf_slock(); 4371 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4372 #ifdef COMPAT_FREEBSD32 4373 if (req->flags & SCTL_MASK32) 4374 error = vfsconf2x32(req, vfsp); 4375 else 4376 #endif 4377 error = vfsconf2x(req, vfsp); 4378 if (error) 4379 break; 4380 } 4381 vfsconf_sunlock(); 4382 return (error); 4383 } 4384 4385 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD | 4386 CTLFLAG_MPSAFE, NULL, 0, sysctl_vfs_conflist, 4387 "S,xvfsconf", "List of all configured filesystems"); 4388 4389 #ifndef BURN_BRIDGES 4390 static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS); 4391 4392 static int 4393 vfs_sysctl(SYSCTL_HANDLER_ARGS) 4394 { 4395 int *name = (int *)arg1 - 1; /* XXX */ 4396 u_int namelen = arg2 + 1; /* XXX */ 4397 struct vfsconf *vfsp; 4398 4399 log(LOG_WARNING, "userland calling deprecated sysctl, " 4400 "please rebuild world\n"); 4401 4402 #if 1 || defined(COMPAT_PRELITE2) 4403 /* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */ 4404 if (namelen == 1) 4405 return (sysctl_ovfs_conf(oidp, arg1, arg2, req)); 4406 #endif 4407 4408 switch (name[1]) { 4409 case VFS_MAXTYPENUM: 4410 if (namelen != 2) 4411 return (ENOTDIR); 4412 return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int))); 4413 case VFS_CONF: 4414 if (namelen != 3) 4415 return (ENOTDIR); /* overloaded */ 4416 vfsconf_slock(); 4417 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4418 if (vfsp->vfc_typenum == name[2]) 4419 break; 4420 } 4421 vfsconf_sunlock(); 4422 if (vfsp == NULL) 4423 return (EOPNOTSUPP); 4424 #ifdef COMPAT_FREEBSD32 4425 if (req->flags & SCTL_MASK32) 4426 return (vfsconf2x32(req, vfsp)); 4427 else 4428 #endif 4429 return (vfsconf2x(req, vfsp)); 4430 } 4431 return (EOPNOTSUPP); 4432 } 4433 4434 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP | 4435 CTLFLAG_MPSAFE, vfs_sysctl, 4436 "Generic filesystem"); 4437 4438 #if 1 || defined(COMPAT_PRELITE2) 4439 4440 static int 4441 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS) 4442 { 4443 int error; 4444 struct vfsconf *vfsp; 4445 struct ovfsconf ovfs; 4446 4447 vfsconf_slock(); 4448 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4449 bzero(&ovfs, sizeof(ovfs)); 4450 ovfs.vfc_vfsops = vfsp->vfc_vfsops; /* XXX used as flag */ 4451 strcpy(ovfs.vfc_name, vfsp->vfc_name); 4452 ovfs.vfc_index = vfsp->vfc_typenum; 4453 ovfs.vfc_refcount = vfsp->vfc_refcount; 4454 ovfs.vfc_flags = vfsp->vfc_flags; 4455 error = SYSCTL_OUT(req, &ovfs, sizeof ovfs); 4456 if (error != 0) { 4457 vfsconf_sunlock(); 4458 return (error); 4459 } 4460 } 4461 vfsconf_sunlock(); 4462 return (0); 4463 } 4464 4465 #endif /* 1 || COMPAT_PRELITE2 */ 4466 #endif /* !BURN_BRIDGES */ 4467 4468 #define KINFO_VNODESLOP 10 4469 #ifdef notyet 4470 /* 4471 * Dump vnode list (via sysctl). 4472 */ 4473 /* ARGSUSED */ 4474 static int 4475 sysctl_vnode(SYSCTL_HANDLER_ARGS) 4476 { 4477 struct xvnode *xvn; 4478 struct mount *mp; 4479 struct vnode *vp; 4480 int error, len, n; 4481 4482 /* 4483 * Stale numvnodes access is not fatal here. 4484 */ 4485 req->lock = 0; 4486 len = (numvnodes + KINFO_VNODESLOP) * sizeof *xvn; 4487 if (!req->oldptr) 4488 /* Make an estimate */ 4489 return (SYSCTL_OUT(req, 0, len)); 4490 4491 error = sysctl_wire_old_buffer(req, 0); 4492 if (error != 0) 4493 return (error); 4494 xvn = malloc(len, M_TEMP, M_ZERO | M_WAITOK); 4495 n = 0; 4496 mtx_lock(&mountlist_mtx); 4497 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4498 if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) 4499 continue; 4500 MNT_ILOCK(mp); 4501 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4502 if (n == len) 4503 break; 4504 vref(vp); 4505 xvn[n].xv_size = sizeof *xvn; 4506 xvn[n].xv_vnode = vp; 4507 xvn[n].xv_id = 0; /* XXX compat */ 4508 #define XV_COPY(field) xvn[n].xv_##field = vp->v_##field 4509 XV_COPY(usecount); 4510 XV_COPY(writecount); 4511 XV_COPY(holdcnt); 4512 XV_COPY(mount); 4513 XV_COPY(numoutput); 4514 XV_COPY(type); 4515 #undef XV_COPY 4516 xvn[n].xv_flag = vp->v_vflag; 4517 4518 switch (vp->v_type) { 4519 case VREG: 4520 case VDIR: 4521 case VLNK: 4522 break; 4523 case VBLK: 4524 case VCHR: 4525 if (vp->v_rdev == NULL) { 4526 vrele(vp); 4527 continue; 4528 } 4529 xvn[n].xv_dev = dev2udev(vp->v_rdev); 4530 break; 4531 case VSOCK: 4532 xvn[n].xv_socket = vp->v_socket; 4533 break; 4534 case VFIFO: 4535 xvn[n].xv_fifo = vp->v_fifoinfo; 4536 break; 4537 case VNON: 4538 case VBAD: 4539 default: 4540 /* shouldn't happen? */ 4541 vrele(vp); 4542 continue; 4543 } 4544 vrele(vp); 4545 ++n; 4546 } 4547 MNT_IUNLOCK(mp); 4548 mtx_lock(&mountlist_mtx); 4549 vfs_unbusy(mp); 4550 if (n == len) 4551 break; 4552 } 4553 mtx_unlock(&mountlist_mtx); 4554 4555 error = SYSCTL_OUT(req, xvn, n * sizeof *xvn); 4556 free(xvn, M_TEMP); 4557 return (error); 4558 } 4559 4560 SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE | CTLFLAG_RD | 4561 CTLFLAG_MPSAFE, 0, 0, sysctl_vnode, "S,xvnode", 4562 ""); 4563 #endif 4564 4565 static void 4566 unmount_or_warn(struct mount *mp) 4567 { 4568 int error; 4569 4570 error = dounmount(mp, MNT_FORCE, curthread); 4571 if (error != 0) { 4572 printf("unmount of %s failed (", mp->mnt_stat.f_mntonname); 4573 if (error == EBUSY) 4574 printf("BUSY)\n"); 4575 else 4576 printf("%d)\n", error); 4577 } 4578 } 4579 4580 /* 4581 * Unmount all filesystems. The list is traversed in reverse order 4582 * of mounting to avoid dependencies. 4583 */ 4584 void 4585 vfs_unmountall(void) 4586 { 4587 struct mount *mp, *tmp; 4588 4589 CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__); 4590 4591 /* 4592 * Since this only runs when rebooting, it is not interlocked. 4593 */ 4594 TAILQ_FOREACH_REVERSE_SAFE(mp, &mountlist, mntlist, mnt_list, tmp) { 4595 vfs_ref(mp); 4596 4597 /* 4598 * Forcibly unmounting "/dev" before "/" would prevent clean 4599 * unmount of the latter. 4600 */ 4601 if (mp == rootdevmp) 4602 continue; 4603 4604 unmount_or_warn(mp); 4605 } 4606 4607 if (rootdevmp != NULL) 4608 unmount_or_warn(rootdevmp); 4609 } 4610 4611 static void 4612 vfs_deferred_inactive(struct vnode *vp, int lkflags) 4613 { 4614 4615 ASSERT_VI_LOCKED(vp, __func__); 4616 VNASSERT((vp->v_iflag & VI_DEFINACT) == 0, vp, ("VI_DEFINACT still set")); 4617 if ((vp->v_iflag & VI_OWEINACT) == 0) { 4618 vdropl(vp); 4619 return; 4620 } 4621 if (vn_lock(vp, lkflags) == 0) { 4622 VI_LOCK(vp); 4623 vinactive(vp); 4624 VOP_UNLOCK(vp); 4625 vdropl(vp); 4626 return; 4627 } 4628 vdefer_inactive_unlocked(vp); 4629 } 4630 4631 static int 4632 vfs_periodic_inactive_filter(struct vnode *vp, void *arg) 4633 { 4634 4635 return (vp->v_iflag & VI_DEFINACT); 4636 } 4637 4638 static void __noinline 4639 vfs_periodic_inactive(struct mount *mp, int flags) 4640 { 4641 struct vnode *vp, *mvp; 4642 int lkflags; 4643 4644 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 4645 if (flags != MNT_WAIT) 4646 lkflags |= LK_NOWAIT; 4647 4648 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_inactive_filter, NULL) { 4649 if ((vp->v_iflag & VI_DEFINACT) == 0) { 4650 VI_UNLOCK(vp); 4651 continue; 4652 } 4653 vp->v_iflag &= ~VI_DEFINACT; 4654 vfs_deferred_inactive(vp, lkflags); 4655 } 4656 } 4657 4658 static inline bool 4659 vfs_want_msync(struct vnode *vp) 4660 { 4661 struct vm_object *obj; 4662 4663 /* 4664 * This test may be performed without any locks held. 4665 * We rely on vm_object's type stability. 4666 */ 4667 if (vp->v_vflag & VV_NOSYNC) 4668 return (false); 4669 obj = vp->v_object; 4670 return (obj != NULL && vm_object_mightbedirty(obj)); 4671 } 4672 4673 static int 4674 vfs_periodic_msync_inactive_filter(struct vnode *vp, void *arg __unused) 4675 { 4676 4677 if (vp->v_vflag & VV_NOSYNC) 4678 return (false); 4679 if (vp->v_iflag & VI_DEFINACT) 4680 return (true); 4681 return (vfs_want_msync(vp)); 4682 } 4683 4684 static void __noinline 4685 vfs_periodic_msync_inactive(struct mount *mp, int flags) 4686 { 4687 struct vnode *vp, *mvp; 4688 struct vm_object *obj; 4689 int lkflags, objflags; 4690 bool seen_defer; 4691 4692 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 4693 if (flags != MNT_WAIT) { 4694 lkflags |= LK_NOWAIT; 4695 objflags = OBJPC_NOSYNC; 4696 } else { 4697 objflags = OBJPC_SYNC; 4698 } 4699 4700 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_msync_inactive_filter, NULL) { 4701 seen_defer = false; 4702 if (vp->v_iflag & VI_DEFINACT) { 4703 vp->v_iflag &= ~VI_DEFINACT; 4704 seen_defer = true; 4705 } 4706 if (!vfs_want_msync(vp)) { 4707 if (seen_defer) 4708 vfs_deferred_inactive(vp, lkflags); 4709 else 4710 VI_UNLOCK(vp); 4711 continue; 4712 } 4713 if (vget(vp, lkflags) == 0) { 4714 obj = vp->v_object; 4715 if (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0) { 4716 VM_OBJECT_WLOCK(obj); 4717 vm_object_page_clean(obj, 0, 0, objflags); 4718 VM_OBJECT_WUNLOCK(obj); 4719 } 4720 vput(vp); 4721 if (seen_defer) 4722 vdrop(vp); 4723 } else { 4724 if (seen_defer) 4725 vdefer_inactive_unlocked(vp); 4726 } 4727 } 4728 } 4729 4730 void 4731 vfs_periodic(struct mount *mp, int flags) 4732 { 4733 4734 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 4735 4736 if ((mp->mnt_kern_flag & MNTK_NOMSYNC) != 0) 4737 vfs_periodic_inactive(mp, flags); 4738 else 4739 vfs_periodic_msync_inactive(mp, flags); 4740 } 4741 4742 static void 4743 destroy_vpollinfo_free(struct vpollinfo *vi) 4744 { 4745 4746 knlist_destroy(&vi->vpi_selinfo.si_note); 4747 mtx_destroy(&vi->vpi_lock); 4748 uma_zfree(vnodepoll_zone, vi); 4749 } 4750 4751 static void 4752 destroy_vpollinfo(struct vpollinfo *vi) 4753 { 4754 4755 knlist_clear(&vi->vpi_selinfo.si_note, 1); 4756 seldrain(&vi->vpi_selinfo); 4757 destroy_vpollinfo_free(vi); 4758 } 4759 4760 /* 4761 * Initialize per-vnode helper structure to hold poll-related state. 4762 */ 4763 void 4764 v_addpollinfo(struct vnode *vp) 4765 { 4766 struct vpollinfo *vi; 4767 4768 if (vp->v_pollinfo != NULL) 4769 return; 4770 vi = uma_zalloc(vnodepoll_zone, M_WAITOK | M_ZERO); 4771 mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF); 4772 knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock, 4773 vfs_knlunlock, vfs_knl_assert_locked, vfs_knl_assert_unlocked); 4774 VI_LOCK(vp); 4775 if (vp->v_pollinfo != NULL) { 4776 VI_UNLOCK(vp); 4777 destroy_vpollinfo_free(vi); 4778 return; 4779 } 4780 vp->v_pollinfo = vi; 4781 VI_UNLOCK(vp); 4782 } 4783 4784 /* 4785 * Record a process's interest in events which might happen to 4786 * a vnode. Because poll uses the historic select-style interface 4787 * internally, this routine serves as both the ``check for any 4788 * pending events'' and the ``record my interest in future events'' 4789 * functions. (These are done together, while the lock is held, 4790 * to avoid race conditions.) 4791 */ 4792 int 4793 vn_pollrecord(struct vnode *vp, struct thread *td, int events) 4794 { 4795 4796 v_addpollinfo(vp); 4797 mtx_lock(&vp->v_pollinfo->vpi_lock); 4798 if (vp->v_pollinfo->vpi_revents & events) { 4799 /* 4800 * This leaves events we are not interested 4801 * in available for the other process which 4802 * which presumably had requested them 4803 * (otherwise they would never have been 4804 * recorded). 4805 */ 4806 events &= vp->v_pollinfo->vpi_revents; 4807 vp->v_pollinfo->vpi_revents &= ~events; 4808 4809 mtx_unlock(&vp->v_pollinfo->vpi_lock); 4810 return (events); 4811 } 4812 vp->v_pollinfo->vpi_events |= events; 4813 selrecord(td, &vp->v_pollinfo->vpi_selinfo); 4814 mtx_unlock(&vp->v_pollinfo->vpi_lock); 4815 return (0); 4816 } 4817 4818 /* 4819 * Routine to create and manage a filesystem syncer vnode. 4820 */ 4821 #define sync_close ((int (*)(struct vop_close_args *))nullop) 4822 static int sync_fsync(struct vop_fsync_args *); 4823 static int sync_inactive(struct vop_inactive_args *); 4824 static int sync_reclaim(struct vop_reclaim_args *); 4825 4826 static struct vop_vector sync_vnodeops = { 4827 .vop_bypass = VOP_EOPNOTSUPP, 4828 .vop_close = sync_close, /* close */ 4829 .vop_fsync = sync_fsync, /* fsync */ 4830 .vop_inactive = sync_inactive, /* inactive */ 4831 .vop_need_inactive = vop_stdneed_inactive, /* need_inactive */ 4832 .vop_reclaim = sync_reclaim, /* reclaim */ 4833 .vop_lock1 = vop_stdlock, /* lock */ 4834 .vop_unlock = vop_stdunlock, /* unlock */ 4835 .vop_islocked = vop_stdislocked, /* islocked */ 4836 }; 4837 VFS_VOP_VECTOR_REGISTER(sync_vnodeops); 4838 4839 /* 4840 * Create a new filesystem syncer vnode for the specified mount point. 4841 */ 4842 void 4843 vfs_allocate_syncvnode(struct mount *mp) 4844 { 4845 struct vnode *vp; 4846 struct bufobj *bo; 4847 static long start, incr, next; 4848 int error; 4849 4850 /* Allocate a new vnode */ 4851 error = getnewvnode("syncer", mp, &sync_vnodeops, &vp); 4852 if (error != 0) 4853 panic("vfs_allocate_syncvnode: getnewvnode() failed"); 4854 vp->v_type = VNON; 4855 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 4856 vp->v_vflag |= VV_FORCEINSMQ; 4857 error = insmntque(vp, mp); 4858 if (error != 0) 4859 panic("vfs_allocate_syncvnode: insmntque() failed"); 4860 vp->v_vflag &= ~VV_FORCEINSMQ; 4861 VOP_UNLOCK(vp); 4862 /* 4863 * Place the vnode onto the syncer worklist. We attempt to 4864 * scatter them about on the list so that they will go off 4865 * at evenly distributed times even if all the filesystems 4866 * are mounted at once. 4867 */ 4868 next += incr; 4869 if (next == 0 || next > syncer_maxdelay) { 4870 start /= 2; 4871 incr /= 2; 4872 if (start == 0) { 4873 start = syncer_maxdelay / 2; 4874 incr = syncer_maxdelay; 4875 } 4876 next = start; 4877 } 4878 bo = &vp->v_bufobj; 4879 BO_LOCK(bo); 4880 vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0); 4881 /* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */ 4882 mtx_lock(&sync_mtx); 4883 sync_vnode_count++; 4884 if (mp->mnt_syncer == NULL) { 4885 mp->mnt_syncer = vp; 4886 vp = NULL; 4887 } 4888 mtx_unlock(&sync_mtx); 4889 BO_UNLOCK(bo); 4890 if (vp != NULL) { 4891 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 4892 vgone(vp); 4893 vput(vp); 4894 } 4895 } 4896 4897 void 4898 vfs_deallocate_syncvnode(struct mount *mp) 4899 { 4900 struct vnode *vp; 4901 4902 mtx_lock(&sync_mtx); 4903 vp = mp->mnt_syncer; 4904 if (vp != NULL) 4905 mp->mnt_syncer = NULL; 4906 mtx_unlock(&sync_mtx); 4907 if (vp != NULL) 4908 vrele(vp); 4909 } 4910 4911 /* 4912 * Do a lazy sync of the filesystem. 4913 */ 4914 static int 4915 sync_fsync(struct vop_fsync_args *ap) 4916 { 4917 struct vnode *syncvp = ap->a_vp; 4918 struct mount *mp = syncvp->v_mount; 4919 int error, save; 4920 struct bufobj *bo; 4921 4922 /* 4923 * We only need to do something if this is a lazy evaluation. 4924 */ 4925 if (ap->a_waitfor != MNT_LAZY) 4926 return (0); 4927 4928 /* 4929 * Move ourselves to the back of the sync list. 4930 */ 4931 bo = &syncvp->v_bufobj; 4932 BO_LOCK(bo); 4933 vn_syncer_add_to_worklist(bo, syncdelay); 4934 BO_UNLOCK(bo); 4935 4936 /* 4937 * Walk the list of vnodes pushing all that are dirty and 4938 * not already on the sync list. 4939 */ 4940 if (vfs_busy(mp, MBF_NOWAIT) != 0) 4941 return (0); 4942 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) { 4943 vfs_unbusy(mp); 4944 return (0); 4945 } 4946 save = curthread_pflags_set(TDP_SYNCIO); 4947 /* 4948 * The filesystem at hand may be idle with free vnodes stored in the 4949 * batch. Return them instead of letting them stay there indefinitely. 4950 */ 4951 vfs_periodic(mp, MNT_NOWAIT); 4952 error = VFS_SYNC(mp, MNT_LAZY); 4953 curthread_pflags_restore(save); 4954 vn_finished_write(mp); 4955 vfs_unbusy(mp); 4956 return (error); 4957 } 4958 4959 /* 4960 * The syncer vnode is no referenced. 4961 */ 4962 static int 4963 sync_inactive(struct vop_inactive_args *ap) 4964 { 4965 4966 vgone(ap->a_vp); 4967 return (0); 4968 } 4969 4970 /* 4971 * The syncer vnode is no longer needed and is being decommissioned. 4972 * 4973 * Modifications to the worklist must be protected by sync_mtx. 4974 */ 4975 static int 4976 sync_reclaim(struct vop_reclaim_args *ap) 4977 { 4978 struct vnode *vp = ap->a_vp; 4979 struct bufobj *bo; 4980 4981 bo = &vp->v_bufobj; 4982 BO_LOCK(bo); 4983 mtx_lock(&sync_mtx); 4984 if (vp->v_mount->mnt_syncer == vp) 4985 vp->v_mount->mnt_syncer = NULL; 4986 if (bo->bo_flag & BO_ONWORKLST) { 4987 LIST_REMOVE(bo, bo_synclist); 4988 syncer_worklist_len--; 4989 sync_vnode_count--; 4990 bo->bo_flag &= ~BO_ONWORKLST; 4991 } 4992 mtx_unlock(&sync_mtx); 4993 BO_UNLOCK(bo); 4994 4995 return (0); 4996 } 4997 4998 int 4999 vn_need_pageq_flush(struct vnode *vp) 5000 { 5001 struct vm_object *obj; 5002 int need; 5003 5004 MPASS(mtx_owned(VI_MTX(vp))); 5005 need = 0; 5006 if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 5007 vm_object_mightbedirty(obj)) 5008 need = 1; 5009 return (need); 5010 } 5011 5012 /* 5013 * Check if vnode represents a disk device 5014 */ 5015 bool 5016 vn_isdisk_error(struct vnode *vp, int *errp) 5017 { 5018 int error; 5019 5020 if (vp->v_type != VCHR) { 5021 error = ENOTBLK; 5022 goto out; 5023 } 5024 error = 0; 5025 dev_lock(); 5026 if (vp->v_rdev == NULL) 5027 error = ENXIO; 5028 else if (vp->v_rdev->si_devsw == NULL) 5029 error = ENXIO; 5030 else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK)) 5031 error = ENOTBLK; 5032 dev_unlock(); 5033 out: 5034 *errp = error; 5035 return (error == 0); 5036 } 5037 5038 bool 5039 vn_isdisk(struct vnode *vp) 5040 { 5041 int error; 5042 5043 return (vn_isdisk_error(vp, &error)); 5044 } 5045 5046 /* 5047 * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see 5048 * the comment above cache_fplookup for details. 5049 */ 5050 int 5051 vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid, struct ucred *cred) 5052 { 5053 int error; 5054 5055 VFS_SMR_ASSERT_ENTERED(); 5056 5057 /* Check the owner. */ 5058 if (cred->cr_uid == file_uid) { 5059 if (file_mode & S_IXUSR) 5060 return (0); 5061 goto out_error; 5062 } 5063 5064 /* Otherwise, check the groups (first match) */ 5065 if (groupmember(file_gid, cred)) { 5066 if (file_mode & S_IXGRP) 5067 return (0); 5068 goto out_error; 5069 } 5070 5071 /* Otherwise, check everyone else. */ 5072 if (file_mode & S_IXOTH) 5073 return (0); 5074 out_error: 5075 /* 5076 * Permission check failed, but it is possible denial will get overwritten 5077 * (e.g., when root is traversing through a 700 directory owned by someone 5078 * else). 5079 * 5080 * vaccess() calls priv_check_cred which in turn can descent into MAC 5081 * modules overriding this result. It's quite unclear what semantics 5082 * are allowed for them to operate, thus for safety we don't call them 5083 * from within the SMR section. This also means if any such modules 5084 * are present, we have to let the regular lookup decide. 5085 */ 5086 error = priv_check_cred_vfs_lookup_nomac(cred); 5087 switch (error) { 5088 case 0: 5089 return (0); 5090 case EAGAIN: 5091 /* 5092 * MAC modules present. 5093 */ 5094 return (EAGAIN); 5095 case EPERM: 5096 return (EACCES); 5097 default: 5098 return (error); 5099 } 5100 } 5101 5102 /* 5103 * Common filesystem object access control check routine. Accepts a 5104 * vnode's type, "mode", uid and gid, requested access mode, and credentials. 5105 * Returns 0 on success, or an errno on failure. 5106 */ 5107 int 5108 vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid, 5109 accmode_t accmode, struct ucred *cred) 5110 { 5111 accmode_t dac_granted; 5112 accmode_t priv_granted; 5113 5114 KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0, 5115 ("invalid bit in accmode")); 5116 KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE), 5117 ("VAPPEND without VWRITE")); 5118 5119 /* 5120 * Look for a normal, non-privileged way to access the file/directory 5121 * as requested. If it exists, go with that. 5122 */ 5123 5124 dac_granted = 0; 5125 5126 /* Check the owner. */ 5127 if (cred->cr_uid == file_uid) { 5128 dac_granted |= VADMIN; 5129 if (file_mode & S_IXUSR) 5130 dac_granted |= VEXEC; 5131 if (file_mode & S_IRUSR) 5132 dac_granted |= VREAD; 5133 if (file_mode & S_IWUSR) 5134 dac_granted |= (VWRITE | VAPPEND); 5135 5136 if ((accmode & dac_granted) == accmode) 5137 return (0); 5138 5139 goto privcheck; 5140 } 5141 5142 /* Otherwise, check the groups (first match) */ 5143 if (groupmember(file_gid, cred)) { 5144 if (file_mode & S_IXGRP) 5145 dac_granted |= VEXEC; 5146 if (file_mode & S_IRGRP) 5147 dac_granted |= VREAD; 5148 if (file_mode & S_IWGRP) 5149 dac_granted |= (VWRITE | VAPPEND); 5150 5151 if ((accmode & dac_granted) == accmode) 5152 return (0); 5153 5154 goto privcheck; 5155 } 5156 5157 /* Otherwise, check everyone else. */ 5158 if (file_mode & S_IXOTH) 5159 dac_granted |= VEXEC; 5160 if (file_mode & S_IROTH) 5161 dac_granted |= VREAD; 5162 if (file_mode & S_IWOTH) 5163 dac_granted |= (VWRITE | VAPPEND); 5164 if ((accmode & dac_granted) == accmode) 5165 return (0); 5166 5167 privcheck: 5168 /* 5169 * Build a privilege mask to determine if the set of privileges 5170 * satisfies the requirements when combined with the granted mask 5171 * from above. For each privilege, if the privilege is required, 5172 * bitwise or the request type onto the priv_granted mask. 5173 */ 5174 priv_granted = 0; 5175 5176 if (type == VDIR) { 5177 /* 5178 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC 5179 * requests, instead of PRIV_VFS_EXEC. 5180 */ 5181 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5182 !priv_check_cred(cred, PRIV_VFS_LOOKUP)) 5183 priv_granted |= VEXEC; 5184 } else { 5185 /* 5186 * Ensure that at least one execute bit is on. Otherwise, 5187 * a privileged user will always succeed, and we don't want 5188 * this to happen unless the file really is executable. 5189 */ 5190 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5191 (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && 5192 !priv_check_cred(cred, PRIV_VFS_EXEC)) 5193 priv_granted |= VEXEC; 5194 } 5195 5196 if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) && 5197 !priv_check_cred(cred, PRIV_VFS_READ)) 5198 priv_granted |= VREAD; 5199 5200 if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) && 5201 !priv_check_cred(cred, PRIV_VFS_WRITE)) 5202 priv_granted |= (VWRITE | VAPPEND); 5203 5204 if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) && 5205 !priv_check_cred(cred, PRIV_VFS_ADMIN)) 5206 priv_granted |= VADMIN; 5207 5208 if ((accmode & (priv_granted | dac_granted)) == accmode) { 5209 return (0); 5210 } 5211 5212 return ((accmode & VADMIN) ? EPERM : EACCES); 5213 } 5214 5215 /* 5216 * Credential check based on process requesting service, and per-attribute 5217 * permissions. 5218 */ 5219 int 5220 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred, 5221 struct thread *td, accmode_t accmode) 5222 { 5223 5224 /* 5225 * Kernel-invoked always succeeds. 5226 */ 5227 if (cred == NOCRED) 5228 return (0); 5229 5230 /* 5231 * Do not allow privileged processes in jail to directly manipulate 5232 * system attributes. 5233 */ 5234 switch (attrnamespace) { 5235 case EXTATTR_NAMESPACE_SYSTEM: 5236 /* Potentially should be: return (EPERM); */ 5237 return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM)); 5238 case EXTATTR_NAMESPACE_USER: 5239 return (VOP_ACCESS(vp, accmode, cred, td)); 5240 default: 5241 return (EPERM); 5242 } 5243 } 5244 5245 #ifdef DEBUG_VFS_LOCKS 5246 /* 5247 * This only exists to suppress warnings from unlocked specfs accesses. It is 5248 * no longer ok to have an unlocked VFS. 5249 */ 5250 #define IGNORE_LOCK(vp) (KERNEL_PANICKED() || (vp) == NULL || \ 5251 (vp)->v_type == VCHR || (vp)->v_type == VBAD) 5252 5253 int vfs_badlock_ddb = 1; /* Drop into debugger on violation. */ 5254 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0, 5255 "Drop into debugger on lock violation"); 5256 5257 int vfs_badlock_mutex = 1; /* Check for interlock across VOPs. */ 5258 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex, 5259 0, "Check for interlock across VOPs"); 5260 5261 int vfs_badlock_print = 1; /* Print lock violations. */ 5262 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print, 5263 0, "Print lock violations"); 5264 5265 int vfs_badlock_vnode = 1; /* Print vnode details on lock violations. */ 5266 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_vnode, CTLFLAG_RW, &vfs_badlock_vnode, 5267 0, "Print vnode details on lock violations"); 5268 5269 #ifdef KDB 5270 int vfs_badlock_backtrace = 1; /* Print backtrace at lock violations. */ 5271 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW, 5272 &vfs_badlock_backtrace, 0, "Print backtrace at lock violations"); 5273 #endif 5274 5275 static void 5276 vfs_badlock(const char *msg, const char *str, struct vnode *vp) 5277 { 5278 5279 #ifdef KDB 5280 if (vfs_badlock_backtrace) 5281 kdb_backtrace(); 5282 #endif 5283 if (vfs_badlock_vnode) 5284 vn_printf(vp, "vnode "); 5285 if (vfs_badlock_print) 5286 printf("%s: %p %s\n", str, (void *)vp, msg); 5287 if (vfs_badlock_ddb) 5288 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5289 } 5290 5291 void 5292 assert_vi_locked(struct vnode *vp, const char *str) 5293 { 5294 5295 if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp))) 5296 vfs_badlock("interlock is not locked but should be", str, vp); 5297 } 5298 5299 void 5300 assert_vi_unlocked(struct vnode *vp, const char *str) 5301 { 5302 5303 if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp))) 5304 vfs_badlock("interlock is locked but should not be", str, vp); 5305 } 5306 5307 void 5308 assert_vop_locked(struct vnode *vp, const char *str) 5309 { 5310 int locked; 5311 5312 if (!IGNORE_LOCK(vp)) { 5313 locked = VOP_ISLOCKED(vp); 5314 if (locked == 0 || locked == LK_EXCLOTHER) 5315 vfs_badlock("is not locked but should be", str, vp); 5316 } 5317 } 5318 5319 void 5320 assert_vop_unlocked(struct vnode *vp, const char *str) 5321 { 5322 5323 if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) == LK_EXCLUSIVE) 5324 vfs_badlock("is locked but should not be", str, vp); 5325 } 5326 5327 void 5328 assert_vop_elocked(struct vnode *vp, const char *str) 5329 { 5330 5331 if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLUSIVE) 5332 vfs_badlock("is not exclusive locked but should be", str, vp); 5333 } 5334 #endif /* DEBUG_VFS_LOCKS */ 5335 5336 void 5337 vop_rename_fail(struct vop_rename_args *ap) 5338 { 5339 5340 if (ap->a_tvp != NULL) 5341 vput(ap->a_tvp); 5342 if (ap->a_tdvp == ap->a_tvp) 5343 vrele(ap->a_tdvp); 5344 else 5345 vput(ap->a_tdvp); 5346 vrele(ap->a_fdvp); 5347 vrele(ap->a_fvp); 5348 } 5349 5350 void 5351 vop_rename_pre(void *ap) 5352 { 5353 struct vop_rename_args *a = ap; 5354 5355 #ifdef DEBUG_VFS_LOCKS 5356 if (a->a_tvp) 5357 ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME"); 5358 ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME"); 5359 ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME"); 5360 ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME"); 5361 5362 /* Check the source (from). */ 5363 if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock && 5364 (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock)) 5365 ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked"); 5366 if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock) 5367 ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked"); 5368 5369 /* Check the target. */ 5370 if (a->a_tvp) 5371 ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked"); 5372 ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked"); 5373 #endif 5374 /* 5375 * It may be tempting to add vn_seqc_write_begin/end calls here and 5376 * in vop_rename_post but that's not going to work out since some 5377 * filesystems relookup vnodes mid-rename. This is probably a bug. 5378 * 5379 * For now filesystems are expected to do the relevant calls after they 5380 * decide what vnodes to operate on. 5381 */ 5382 if (a->a_tdvp != a->a_fdvp) 5383 vhold(a->a_fdvp); 5384 if (a->a_tvp != a->a_fvp) 5385 vhold(a->a_fvp); 5386 vhold(a->a_tdvp); 5387 if (a->a_tvp) 5388 vhold(a->a_tvp); 5389 } 5390 5391 #ifdef DEBUG_VFS_LOCKS 5392 void 5393 vop_fplookup_vexec_debugpre(void *ap __unused) 5394 { 5395 5396 VFS_SMR_ASSERT_ENTERED(); 5397 } 5398 5399 void 5400 vop_fplookup_vexec_debugpost(void *ap __unused, int rc __unused) 5401 { 5402 5403 VFS_SMR_ASSERT_ENTERED(); 5404 } 5405 5406 void 5407 vop_strategy_debugpre(void *ap) 5408 { 5409 struct vop_strategy_args *a; 5410 struct buf *bp; 5411 5412 a = ap; 5413 bp = a->a_bp; 5414 5415 /* 5416 * Cluster ops lock their component buffers but not the IO container. 5417 */ 5418 if ((bp->b_flags & B_CLUSTER) != 0) 5419 return; 5420 5421 if (!KERNEL_PANICKED() && !BUF_ISLOCKED(bp)) { 5422 if (vfs_badlock_print) 5423 printf( 5424 "VOP_STRATEGY: bp is not locked but should be\n"); 5425 if (vfs_badlock_ddb) 5426 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5427 } 5428 } 5429 5430 void 5431 vop_lock_debugpre(void *ap) 5432 { 5433 struct vop_lock1_args *a = ap; 5434 5435 if ((a->a_flags & LK_INTERLOCK) == 0) 5436 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 5437 else 5438 ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK"); 5439 } 5440 5441 void 5442 vop_lock_debugpost(void *ap, int rc) 5443 { 5444 struct vop_lock1_args *a = ap; 5445 5446 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 5447 if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0) 5448 ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK"); 5449 } 5450 5451 void 5452 vop_unlock_debugpre(void *ap) 5453 { 5454 struct vop_unlock_args *a = ap; 5455 5456 ASSERT_VOP_LOCKED(a->a_vp, "VOP_UNLOCK"); 5457 } 5458 5459 void 5460 vop_need_inactive_debugpre(void *ap) 5461 { 5462 struct vop_need_inactive_args *a = ap; 5463 5464 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 5465 } 5466 5467 void 5468 vop_need_inactive_debugpost(void *ap, int rc) 5469 { 5470 struct vop_need_inactive_args *a = ap; 5471 5472 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 5473 } 5474 #endif 5475 5476 void 5477 vop_create_pre(void *ap) 5478 { 5479 struct vop_create_args *a; 5480 struct vnode *dvp; 5481 5482 a = ap; 5483 dvp = a->a_dvp; 5484 vn_seqc_write_begin(dvp); 5485 } 5486 5487 void 5488 vop_create_post(void *ap, int rc) 5489 { 5490 struct vop_create_args *a; 5491 struct vnode *dvp; 5492 5493 a = ap; 5494 dvp = a->a_dvp; 5495 vn_seqc_write_end(dvp); 5496 if (!rc) 5497 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5498 } 5499 5500 void 5501 vop_whiteout_pre(void *ap) 5502 { 5503 struct vop_whiteout_args *a; 5504 struct vnode *dvp; 5505 5506 a = ap; 5507 dvp = a->a_dvp; 5508 vn_seqc_write_begin(dvp); 5509 } 5510 5511 void 5512 vop_whiteout_post(void *ap, int rc) 5513 { 5514 struct vop_whiteout_args *a; 5515 struct vnode *dvp; 5516 5517 a = ap; 5518 dvp = a->a_dvp; 5519 vn_seqc_write_end(dvp); 5520 } 5521 5522 void 5523 vop_deleteextattr_pre(void *ap) 5524 { 5525 struct vop_deleteextattr_args *a; 5526 struct vnode *vp; 5527 5528 a = ap; 5529 vp = a->a_vp; 5530 vn_seqc_write_begin(vp); 5531 } 5532 5533 void 5534 vop_deleteextattr_post(void *ap, int rc) 5535 { 5536 struct vop_deleteextattr_args *a; 5537 struct vnode *vp; 5538 5539 a = ap; 5540 vp = a->a_vp; 5541 vn_seqc_write_end(vp); 5542 if (!rc) 5543 VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB); 5544 } 5545 5546 void 5547 vop_link_pre(void *ap) 5548 { 5549 struct vop_link_args *a; 5550 struct vnode *vp, *tdvp; 5551 5552 a = ap; 5553 vp = a->a_vp; 5554 tdvp = a->a_tdvp; 5555 vn_seqc_write_begin(vp); 5556 vn_seqc_write_begin(tdvp); 5557 } 5558 5559 void 5560 vop_link_post(void *ap, int rc) 5561 { 5562 struct vop_link_args *a; 5563 struct vnode *vp, *tdvp; 5564 5565 a = ap; 5566 vp = a->a_vp; 5567 tdvp = a->a_tdvp; 5568 vn_seqc_write_end(vp); 5569 vn_seqc_write_end(tdvp); 5570 if (!rc) { 5571 VFS_KNOTE_LOCKED(vp, NOTE_LINK); 5572 VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE); 5573 } 5574 } 5575 5576 void 5577 vop_mkdir_pre(void *ap) 5578 { 5579 struct vop_mkdir_args *a; 5580 struct vnode *dvp; 5581 5582 a = ap; 5583 dvp = a->a_dvp; 5584 vn_seqc_write_begin(dvp); 5585 } 5586 5587 void 5588 vop_mkdir_post(void *ap, int rc) 5589 { 5590 struct vop_mkdir_args *a; 5591 struct vnode *dvp; 5592 5593 a = ap; 5594 dvp = a->a_dvp; 5595 vn_seqc_write_end(dvp); 5596 if (!rc) 5597 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 5598 } 5599 5600 #ifdef DEBUG_VFS_LOCKS 5601 void 5602 vop_mkdir_debugpost(void *ap, int rc) 5603 { 5604 struct vop_mkdir_args *a; 5605 5606 a = ap; 5607 if (!rc) 5608 cache_validate(a->a_dvp, *a->a_vpp, a->a_cnp); 5609 } 5610 #endif 5611 5612 void 5613 vop_mknod_pre(void *ap) 5614 { 5615 struct vop_mknod_args *a; 5616 struct vnode *dvp; 5617 5618 a = ap; 5619 dvp = a->a_dvp; 5620 vn_seqc_write_begin(dvp); 5621 } 5622 5623 void 5624 vop_mknod_post(void *ap, int rc) 5625 { 5626 struct vop_mknod_args *a; 5627 struct vnode *dvp; 5628 5629 a = ap; 5630 dvp = a->a_dvp; 5631 vn_seqc_write_end(dvp); 5632 if (!rc) 5633 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5634 } 5635 5636 void 5637 vop_reclaim_post(void *ap, int rc) 5638 { 5639 struct vop_reclaim_args *a; 5640 struct vnode *vp; 5641 5642 a = ap; 5643 vp = a->a_vp; 5644 ASSERT_VOP_IN_SEQC(vp); 5645 if (!rc) 5646 VFS_KNOTE_LOCKED(vp, NOTE_REVOKE); 5647 } 5648 5649 void 5650 vop_remove_pre(void *ap) 5651 { 5652 struct vop_remove_args *a; 5653 struct vnode *dvp, *vp; 5654 5655 a = ap; 5656 dvp = a->a_dvp; 5657 vp = a->a_vp; 5658 vn_seqc_write_begin(dvp); 5659 vn_seqc_write_begin(vp); 5660 } 5661 5662 void 5663 vop_remove_post(void *ap, int rc) 5664 { 5665 struct vop_remove_args *a; 5666 struct vnode *dvp, *vp; 5667 5668 a = ap; 5669 dvp = a->a_dvp; 5670 vp = a->a_vp; 5671 vn_seqc_write_end(dvp); 5672 vn_seqc_write_end(vp); 5673 if (!rc) { 5674 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5675 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 5676 } 5677 } 5678 5679 void 5680 vop_rename_post(void *ap, int rc) 5681 { 5682 struct vop_rename_args *a = ap; 5683 long hint; 5684 5685 if (!rc) { 5686 hint = NOTE_WRITE; 5687 if (a->a_fdvp == a->a_tdvp) { 5688 if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR) 5689 hint |= NOTE_LINK; 5690 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 5691 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 5692 } else { 5693 hint |= NOTE_EXTEND; 5694 if (a->a_fvp->v_type == VDIR) 5695 hint |= NOTE_LINK; 5696 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 5697 5698 if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL && 5699 a->a_tvp->v_type == VDIR) 5700 hint &= ~NOTE_LINK; 5701 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 5702 } 5703 5704 VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME); 5705 if (a->a_tvp) 5706 VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE); 5707 } 5708 if (a->a_tdvp != a->a_fdvp) 5709 vdrop(a->a_fdvp); 5710 if (a->a_tvp != a->a_fvp) 5711 vdrop(a->a_fvp); 5712 vdrop(a->a_tdvp); 5713 if (a->a_tvp) 5714 vdrop(a->a_tvp); 5715 } 5716 5717 void 5718 vop_rmdir_pre(void *ap) 5719 { 5720 struct vop_rmdir_args *a; 5721 struct vnode *dvp, *vp; 5722 5723 a = ap; 5724 dvp = a->a_dvp; 5725 vp = a->a_vp; 5726 vn_seqc_write_begin(dvp); 5727 vn_seqc_write_begin(vp); 5728 } 5729 5730 void 5731 vop_rmdir_post(void *ap, int rc) 5732 { 5733 struct vop_rmdir_args *a; 5734 struct vnode *dvp, *vp; 5735 5736 a = ap; 5737 dvp = a->a_dvp; 5738 vp = a->a_vp; 5739 vn_seqc_write_end(dvp); 5740 vn_seqc_write_end(vp); 5741 if (!rc) { 5742 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 5743 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 5744 } 5745 } 5746 5747 void 5748 vop_setattr_pre(void *ap) 5749 { 5750 struct vop_setattr_args *a; 5751 struct vnode *vp; 5752 5753 a = ap; 5754 vp = a->a_vp; 5755 vn_seqc_write_begin(vp); 5756 } 5757 5758 void 5759 vop_setattr_post(void *ap, int rc) 5760 { 5761 struct vop_setattr_args *a; 5762 struct vnode *vp; 5763 5764 a = ap; 5765 vp = a->a_vp; 5766 vn_seqc_write_end(vp); 5767 if (!rc) 5768 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 5769 } 5770 5771 void 5772 vop_setacl_pre(void *ap) 5773 { 5774 struct vop_setacl_args *a; 5775 struct vnode *vp; 5776 5777 a = ap; 5778 vp = a->a_vp; 5779 vn_seqc_write_begin(vp); 5780 } 5781 5782 void 5783 vop_setacl_post(void *ap, int rc __unused) 5784 { 5785 struct vop_setacl_args *a; 5786 struct vnode *vp; 5787 5788 a = ap; 5789 vp = a->a_vp; 5790 vn_seqc_write_end(vp); 5791 } 5792 5793 void 5794 vop_setextattr_pre(void *ap) 5795 { 5796 struct vop_setextattr_args *a; 5797 struct vnode *vp; 5798 5799 a = ap; 5800 vp = a->a_vp; 5801 vn_seqc_write_begin(vp); 5802 } 5803 5804 void 5805 vop_setextattr_post(void *ap, int rc) 5806 { 5807 struct vop_setextattr_args *a; 5808 struct vnode *vp; 5809 5810 a = ap; 5811 vp = a->a_vp; 5812 vn_seqc_write_end(vp); 5813 if (!rc) 5814 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 5815 } 5816 5817 void 5818 vop_symlink_pre(void *ap) 5819 { 5820 struct vop_symlink_args *a; 5821 struct vnode *dvp; 5822 5823 a = ap; 5824 dvp = a->a_dvp; 5825 vn_seqc_write_begin(dvp); 5826 } 5827 5828 void 5829 vop_symlink_post(void *ap, int rc) 5830 { 5831 struct vop_symlink_args *a; 5832 struct vnode *dvp; 5833 5834 a = ap; 5835 dvp = a->a_dvp; 5836 vn_seqc_write_end(dvp); 5837 if (!rc) 5838 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5839 } 5840 5841 void 5842 vop_open_post(void *ap, int rc) 5843 { 5844 struct vop_open_args *a = ap; 5845 5846 if (!rc) 5847 VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN); 5848 } 5849 5850 void 5851 vop_close_post(void *ap, int rc) 5852 { 5853 struct vop_close_args *a = ap; 5854 5855 if (!rc && (a->a_cred != NOCRED || /* filter out revokes */ 5856 !VN_IS_DOOMED(a->a_vp))) { 5857 VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ? 5858 NOTE_CLOSE_WRITE : NOTE_CLOSE); 5859 } 5860 } 5861 5862 void 5863 vop_read_post(void *ap, int rc) 5864 { 5865 struct vop_read_args *a = ap; 5866 5867 if (!rc) 5868 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 5869 } 5870 5871 void 5872 vop_read_pgcache_post(void *ap, int rc) 5873 { 5874 struct vop_read_pgcache_args *a = ap; 5875 5876 if (!rc) 5877 VFS_KNOTE_UNLOCKED(a->a_vp, NOTE_READ); 5878 } 5879 5880 void 5881 vop_readdir_post(void *ap, int rc) 5882 { 5883 struct vop_readdir_args *a = ap; 5884 5885 if (!rc) 5886 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 5887 } 5888 5889 static struct knlist fs_knlist; 5890 5891 static void 5892 vfs_event_init(void *arg) 5893 { 5894 knlist_init_mtx(&fs_knlist, NULL); 5895 } 5896 /* XXX - correct order? */ 5897 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL); 5898 5899 void 5900 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused) 5901 { 5902 5903 KNOTE_UNLOCKED(&fs_knlist, event); 5904 } 5905 5906 static int filt_fsattach(struct knote *kn); 5907 static void filt_fsdetach(struct knote *kn); 5908 static int filt_fsevent(struct knote *kn, long hint); 5909 5910 struct filterops fs_filtops = { 5911 .f_isfd = 0, 5912 .f_attach = filt_fsattach, 5913 .f_detach = filt_fsdetach, 5914 .f_event = filt_fsevent 5915 }; 5916 5917 static int 5918 filt_fsattach(struct knote *kn) 5919 { 5920 5921 kn->kn_flags |= EV_CLEAR; 5922 knlist_add(&fs_knlist, kn, 0); 5923 return (0); 5924 } 5925 5926 static void 5927 filt_fsdetach(struct knote *kn) 5928 { 5929 5930 knlist_remove(&fs_knlist, kn, 0); 5931 } 5932 5933 static int 5934 filt_fsevent(struct knote *kn, long hint) 5935 { 5936 5937 kn->kn_fflags |= hint; 5938 return (kn->kn_fflags != 0); 5939 } 5940 5941 static int 5942 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS) 5943 { 5944 struct vfsidctl vc; 5945 int error; 5946 struct mount *mp; 5947 5948 error = SYSCTL_IN(req, &vc, sizeof(vc)); 5949 if (error) 5950 return (error); 5951 if (vc.vc_vers != VFS_CTL_VERS1) 5952 return (EINVAL); 5953 mp = vfs_getvfs(&vc.vc_fsid); 5954 if (mp == NULL) 5955 return (ENOENT); 5956 /* ensure that a specific sysctl goes to the right filesystem. */ 5957 if (strcmp(vc.vc_fstypename, "*") != 0 && 5958 strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) { 5959 vfs_rel(mp); 5960 return (EINVAL); 5961 } 5962 VCTLTOREQ(&vc, req); 5963 error = VFS_SYSCTL(mp, vc.vc_op, req); 5964 vfs_rel(mp); 5965 return (error); 5966 } 5967 5968 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_MPSAFE | CTLFLAG_WR, 5969 NULL, 0, sysctl_vfs_ctl, "", 5970 "Sysctl by fsid"); 5971 5972 /* 5973 * Function to initialize a va_filerev field sensibly. 5974 * XXX: Wouldn't a random number make a lot more sense ?? 5975 */ 5976 u_quad_t 5977 init_va_filerev(void) 5978 { 5979 struct bintime bt; 5980 5981 getbinuptime(&bt); 5982 return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL)); 5983 } 5984 5985 static int filt_vfsread(struct knote *kn, long hint); 5986 static int filt_vfswrite(struct knote *kn, long hint); 5987 static int filt_vfsvnode(struct knote *kn, long hint); 5988 static void filt_vfsdetach(struct knote *kn); 5989 static struct filterops vfsread_filtops = { 5990 .f_isfd = 1, 5991 .f_detach = filt_vfsdetach, 5992 .f_event = filt_vfsread 5993 }; 5994 static struct filterops vfswrite_filtops = { 5995 .f_isfd = 1, 5996 .f_detach = filt_vfsdetach, 5997 .f_event = filt_vfswrite 5998 }; 5999 static struct filterops vfsvnode_filtops = { 6000 .f_isfd = 1, 6001 .f_detach = filt_vfsdetach, 6002 .f_event = filt_vfsvnode 6003 }; 6004 6005 static void 6006 vfs_knllock(void *arg) 6007 { 6008 struct vnode *vp = arg; 6009 6010 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 6011 } 6012 6013 static void 6014 vfs_knlunlock(void *arg) 6015 { 6016 struct vnode *vp = arg; 6017 6018 VOP_UNLOCK(vp); 6019 } 6020 6021 static void 6022 vfs_knl_assert_locked(void *arg) 6023 { 6024 #ifdef DEBUG_VFS_LOCKS 6025 struct vnode *vp = arg; 6026 6027 ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked"); 6028 #endif 6029 } 6030 6031 static void 6032 vfs_knl_assert_unlocked(void *arg) 6033 { 6034 #ifdef DEBUG_VFS_LOCKS 6035 struct vnode *vp = arg; 6036 6037 ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked"); 6038 #endif 6039 } 6040 6041 int 6042 vfs_kqfilter(struct vop_kqfilter_args *ap) 6043 { 6044 struct vnode *vp = ap->a_vp; 6045 struct knote *kn = ap->a_kn; 6046 struct knlist *knl; 6047 6048 switch (kn->kn_filter) { 6049 case EVFILT_READ: 6050 kn->kn_fop = &vfsread_filtops; 6051 break; 6052 case EVFILT_WRITE: 6053 kn->kn_fop = &vfswrite_filtops; 6054 break; 6055 case EVFILT_VNODE: 6056 kn->kn_fop = &vfsvnode_filtops; 6057 break; 6058 default: 6059 return (EINVAL); 6060 } 6061 6062 kn->kn_hook = (caddr_t)vp; 6063 6064 v_addpollinfo(vp); 6065 if (vp->v_pollinfo == NULL) 6066 return (ENOMEM); 6067 knl = &vp->v_pollinfo->vpi_selinfo.si_note; 6068 vhold(vp); 6069 knlist_add(knl, kn, 0); 6070 6071 return (0); 6072 } 6073 6074 /* 6075 * Detach knote from vnode 6076 */ 6077 static void 6078 filt_vfsdetach(struct knote *kn) 6079 { 6080 struct vnode *vp = (struct vnode *)kn->kn_hook; 6081 6082 KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo")); 6083 knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0); 6084 vdrop(vp); 6085 } 6086 6087 /*ARGSUSED*/ 6088 static int 6089 filt_vfsread(struct knote *kn, long hint) 6090 { 6091 struct vnode *vp = (struct vnode *)kn->kn_hook; 6092 struct vattr va; 6093 int res; 6094 6095 /* 6096 * filesystem is gone, so set the EOF flag and schedule 6097 * the knote for deletion. 6098 */ 6099 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6100 VI_LOCK(vp); 6101 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6102 VI_UNLOCK(vp); 6103 return (1); 6104 } 6105 6106 if (VOP_GETATTR(vp, &va, curthread->td_ucred)) 6107 return (0); 6108 6109 VI_LOCK(vp); 6110 kn->kn_data = va.va_size - kn->kn_fp->f_offset; 6111 res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 || kn->kn_data != 0; 6112 VI_UNLOCK(vp); 6113 return (res); 6114 } 6115 6116 /*ARGSUSED*/ 6117 static int 6118 filt_vfswrite(struct knote *kn, long hint) 6119 { 6120 struct vnode *vp = (struct vnode *)kn->kn_hook; 6121 6122 VI_LOCK(vp); 6123 6124 /* 6125 * filesystem is gone, so set the EOF flag and schedule 6126 * the knote for deletion. 6127 */ 6128 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) 6129 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6130 6131 kn->kn_data = 0; 6132 VI_UNLOCK(vp); 6133 return (1); 6134 } 6135 6136 static int 6137 filt_vfsvnode(struct knote *kn, long hint) 6138 { 6139 struct vnode *vp = (struct vnode *)kn->kn_hook; 6140 int res; 6141 6142 VI_LOCK(vp); 6143 if (kn->kn_sfflags & hint) 6144 kn->kn_fflags |= hint; 6145 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6146 kn->kn_flags |= EV_EOF; 6147 VI_UNLOCK(vp); 6148 return (1); 6149 } 6150 res = (kn->kn_fflags != 0); 6151 VI_UNLOCK(vp); 6152 return (res); 6153 } 6154 6155 /* 6156 * Returns whether the directory is empty or not. 6157 * If it is empty, the return value is 0; otherwise 6158 * the return value is an error value (which may 6159 * be ENOTEMPTY). 6160 */ 6161 int 6162 vfs_emptydir(struct vnode *vp) 6163 { 6164 struct uio uio; 6165 struct iovec iov; 6166 struct dirent *dirent, *dp, *endp; 6167 int error, eof; 6168 6169 error = 0; 6170 eof = 0; 6171 6172 ASSERT_VOP_LOCKED(vp, "vfs_emptydir"); 6173 6174 dirent = malloc(sizeof(struct dirent), M_TEMP, M_WAITOK); 6175 iov.iov_base = dirent; 6176 iov.iov_len = sizeof(struct dirent); 6177 6178 uio.uio_iov = &iov; 6179 uio.uio_iovcnt = 1; 6180 uio.uio_offset = 0; 6181 uio.uio_resid = sizeof(struct dirent); 6182 uio.uio_segflg = UIO_SYSSPACE; 6183 uio.uio_rw = UIO_READ; 6184 uio.uio_td = curthread; 6185 6186 while (eof == 0 && error == 0) { 6187 error = VOP_READDIR(vp, &uio, curthread->td_ucred, &eof, 6188 NULL, NULL); 6189 if (error != 0) 6190 break; 6191 endp = (void *)((uint8_t *)dirent + 6192 sizeof(struct dirent) - uio.uio_resid); 6193 for (dp = dirent; dp < endp; 6194 dp = (void *)((uint8_t *)dp + GENERIC_DIRSIZ(dp))) { 6195 if (dp->d_type == DT_WHT) 6196 continue; 6197 if (dp->d_namlen == 0) 6198 continue; 6199 if (dp->d_type != DT_DIR && 6200 dp->d_type != DT_UNKNOWN) { 6201 error = ENOTEMPTY; 6202 break; 6203 } 6204 if (dp->d_namlen > 2) { 6205 error = ENOTEMPTY; 6206 break; 6207 } 6208 if (dp->d_namlen == 1 && 6209 dp->d_name[0] != '.') { 6210 error = ENOTEMPTY; 6211 break; 6212 } 6213 if (dp->d_namlen == 2 && 6214 dp->d_name[1] != '.') { 6215 error = ENOTEMPTY; 6216 break; 6217 } 6218 uio.uio_resid = sizeof(struct dirent); 6219 } 6220 } 6221 free(dirent, M_TEMP); 6222 return (error); 6223 } 6224 6225 int 6226 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off) 6227 { 6228 int error; 6229 6230 if (dp->d_reclen > ap->a_uio->uio_resid) 6231 return (ENAMETOOLONG); 6232 error = uiomove(dp, dp->d_reclen, ap->a_uio); 6233 if (error) { 6234 if (ap->a_ncookies != NULL) { 6235 if (ap->a_cookies != NULL) 6236 free(ap->a_cookies, M_TEMP); 6237 ap->a_cookies = NULL; 6238 *ap->a_ncookies = 0; 6239 } 6240 return (error); 6241 } 6242 if (ap->a_ncookies == NULL) 6243 return (0); 6244 6245 KASSERT(ap->a_cookies, 6246 ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!")); 6247 6248 *ap->a_cookies = realloc(*ap->a_cookies, 6249 (*ap->a_ncookies + 1) * sizeof(u_long), M_TEMP, M_WAITOK | M_ZERO); 6250 (*ap->a_cookies)[*ap->a_ncookies] = off; 6251 *ap->a_ncookies += 1; 6252 return (0); 6253 } 6254 6255 /* 6256 * The purpose of this routine is to remove granularity from accmode_t, 6257 * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE, 6258 * VADMIN and VAPPEND. 6259 * 6260 * If it returns 0, the caller is supposed to continue with the usual 6261 * access checks using 'accmode' as modified by this routine. If it 6262 * returns nonzero value, the caller is supposed to return that value 6263 * as errno. 6264 * 6265 * Note that after this routine runs, accmode may be zero. 6266 */ 6267 int 6268 vfs_unixify_accmode(accmode_t *accmode) 6269 { 6270 /* 6271 * There is no way to specify explicit "deny" rule using 6272 * file mode or POSIX.1e ACLs. 6273 */ 6274 if (*accmode & VEXPLICIT_DENY) { 6275 *accmode = 0; 6276 return (0); 6277 } 6278 6279 /* 6280 * None of these can be translated into usual access bits. 6281 * Also, the common case for NFSv4 ACLs is to not contain 6282 * either of these bits. Caller should check for VWRITE 6283 * on the containing directory instead. 6284 */ 6285 if (*accmode & (VDELETE_CHILD | VDELETE)) 6286 return (EPERM); 6287 6288 if (*accmode & VADMIN_PERMS) { 6289 *accmode &= ~VADMIN_PERMS; 6290 *accmode |= VADMIN; 6291 } 6292 6293 /* 6294 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL 6295 * or VSYNCHRONIZE using file mode or POSIX.1e ACL. 6296 */ 6297 *accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE); 6298 6299 return (0); 6300 } 6301 6302 /* 6303 * Clear out a doomed vnode (if any) and replace it with a new one as long 6304 * as the fs is not being unmounted. Return the root vnode to the caller. 6305 */ 6306 static int __noinline 6307 vfs_cache_root_fallback(struct mount *mp, int flags, struct vnode **vpp) 6308 { 6309 struct vnode *vp; 6310 int error; 6311 6312 restart: 6313 if (mp->mnt_rootvnode != NULL) { 6314 MNT_ILOCK(mp); 6315 vp = mp->mnt_rootvnode; 6316 if (vp != NULL) { 6317 if (!VN_IS_DOOMED(vp)) { 6318 vrefact(vp); 6319 MNT_IUNLOCK(mp); 6320 error = vn_lock(vp, flags); 6321 if (error == 0) { 6322 *vpp = vp; 6323 return (0); 6324 } 6325 vrele(vp); 6326 goto restart; 6327 } 6328 /* 6329 * Clear the old one. 6330 */ 6331 mp->mnt_rootvnode = NULL; 6332 } 6333 MNT_IUNLOCK(mp); 6334 if (vp != NULL) { 6335 vfs_op_barrier_wait(mp); 6336 vrele(vp); 6337 } 6338 } 6339 error = VFS_CACHEDROOT(mp, flags, vpp); 6340 if (error != 0) 6341 return (error); 6342 if (mp->mnt_vfs_ops == 0) { 6343 MNT_ILOCK(mp); 6344 if (mp->mnt_vfs_ops != 0) { 6345 MNT_IUNLOCK(mp); 6346 return (0); 6347 } 6348 if (mp->mnt_rootvnode == NULL) { 6349 vrefact(*vpp); 6350 mp->mnt_rootvnode = *vpp; 6351 } else { 6352 if (mp->mnt_rootvnode != *vpp) { 6353 if (!VN_IS_DOOMED(mp->mnt_rootvnode)) { 6354 panic("%s: mismatch between vnode returned " 6355 " by VFS_CACHEDROOT and the one cached " 6356 " (%p != %p)", 6357 __func__, *vpp, mp->mnt_rootvnode); 6358 } 6359 } 6360 } 6361 MNT_IUNLOCK(mp); 6362 } 6363 return (0); 6364 } 6365 6366 int 6367 vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp) 6368 { 6369 struct vnode *vp; 6370 int error; 6371 6372 if (!vfs_op_thread_enter(mp)) 6373 return (vfs_cache_root_fallback(mp, flags, vpp)); 6374 vp = atomic_load_ptr(&mp->mnt_rootvnode); 6375 if (vp == NULL || VN_IS_DOOMED(vp)) { 6376 vfs_op_thread_exit(mp); 6377 return (vfs_cache_root_fallback(mp, flags, vpp)); 6378 } 6379 vrefact(vp); 6380 vfs_op_thread_exit(mp); 6381 error = vn_lock(vp, flags); 6382 if (error != 0) { 6383 vrele(vp); 6384 return (vfs_cache_root_fallback(mp, flags, vpp)); 6385 } 6386 *vpp = vp; 6387 return (0); 6388 } 6389 6390 struct vnode * 6391 vfs_cache_root_clear(struct mount *mp) 6392 { 6393 struct vnode *vp; 6394 6395 /* 6396 * ops > 0 guarantees there is nobody who can see this vnode 6397 */ 6398 MPASS(mp->mnt_vfs_ops > 0); 6399 vp = mp->mnt_rootvnode; 6400 if (vp != NULL) 6401 vn_seqc_write_begin(vp); 6402 mp->mnt_rootvnode = NULL; 6403 return (vp); 6404 } 6405 6406 void 6407 vfs_cache_root_set(struct mount *mp, struct vnode *vp) 6408 { 6409 6410 MPASS(mp->mnt_vfs_ops > 0); 6411 vrefact(vp); 6412 mp->mnt_rootvnode = vp; 6413 } 6414 6415 /* 6416 * These are helper functions for filesystems to traverse all 6417 * their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h. 6418 * 6419 * This interface replaces MNT_VNODE_FOREACH. 6420 */ 6421 6422 struct vnode * 6423 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp) 6424 { 6425 struct vnode *vp; 6426 6427 if (should_yield()) 6428 kern_yield(PRI_USER); 6429 MNT_ILOCK(mp); 6430 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6431 for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL; 6432 vp = TAILQ_NEXT(vp, v_nmntvnodes)) { 6433 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6434 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6435 continue; 6436 VI_LOCK(vp); 6437 if (VN_IS_DOOMED(vp)) { 6438 VI_UNLOCK(vp); 6439 continue; 6440 } 6441 break; 6442 } 6443 if (vp == NULL) { 6444 __mnt_vnode_markerfree_all(mvp, mp); 6445 /* MNT_IUNLOCK(mp); -- done in above function */ 6446 mtx_assert(MNT_MTX(mp), MA_NOTOWNED); 6447 return (NULL); 6448 } 6449 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6450 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6451 MNT_IUNLOCK(mp); 6452 return (vp); 6453 } 6454 6455 struct vnode * 6456 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp) 6457 { 6458 struct vnode *vp; 6459 6460 *mvp = vn_alloc_marker(mp); 6461 MNT_ILOCK(mp); 6462 MNT_REF(mp); 6463 6464 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 6465 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6466 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6467 continue; 6468 VI_LOCK(vp); 6469 if (VN_IS_DOOMED(vp)) { 6470 VI_UNLOCK(vp); 6471 continue; 6472 } 6473 break; 6474 } 6475 if (vp == NULL) { 6476 MNT_REL(mp); 6477 MNT_IUNLOCK(mp); 6478 vn_free_marker(*mvp); 6479 *mvp = NULL; 6480 return (NULL); 6481 } 6482 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6483 MNT_IUNLOCK(mp); 6484 return (vp); 6485 } 6486 6487 void 6488 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp) 6489 { 6490 6491 if (*mvp == NULL) { 6492 MNT_IUNLOCK(mp); 6493 return; 6494 } 6495 6496 mtx_assert(MNT_MTX(mp), MA_OWNED); 6497 6498 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6499 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6500 MNT_REL(mp); 6501 MNT_IUNLOCK(mp); 6502 vn_free_marker(*mvp); 6503 *mvp = NULL; 6504 } 6505 6506 /* 6507 * These are helper functions for filesystems to traverse their 6508 * lazy vnodes. See MNT_VNODE_FOREACH_LAZY() in sys/mount.h 6509 */ 6510 static void 6511 mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 6512 { 6513 6514 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6515 6516 MNT_ILOCK(mp); 6517 MNT_REL(mp); 6518 MNT_IUNLOCK(mp); 6519 vn_free_marker(*mvp); 6520 *mvp = NULL; 6521 } 6522 6523 /* 6524 * Relock the mp mount vnode list lock with the vp vnode interlock in the 6525 * conventional lock order during mnt_vnode_next_lazy iteration. 6526 * 6527 * On entry, the mount vnode list lock is held and the vnode interlock is not. 6528 * The list lock is dropped and reacquired. On success, both locks are held. 6529 * On failure, the mount vnode list lock is held but the vnode interlock is 6530 * not, and the procedure may have yielded. 6531 */ 6532 static bool 6533 mnt_vnode_next_lazy_relock(struct vnode *mvp, struct mount *mp, 6534 struct vnode *vp) 6535 { 6536 6537 VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER && 6538 TAILQ_NEXT(mvp, v_lazylist) != NULL, mvp, 6539 ("%s: bad marker", __func__)); 6540 VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp, 6541 ("%s: inappropriate vnode", __func__)); 6542 ASSERT_VI_UNLOCKED(vp, __func__); 6543 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 6544 6545 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, mvp, v_lazylist); 6546 TAILQ_INSERT_BEFORE(vp, mvp, v_lazylist); 6547 6548 /* 6549 * Note we may be racing against vdrop which transitioned the hold 6550 * count to 0 and now waits for the ->mnt_listmtx lock. This is fine, 6551 * if we are the only user after we get the interlock we will just 6552 * vdrop. 6553 */ 6554 vhold(vp); 6555 mtx_unlock(&mp->mnt_listmtx); 6556 VI_LOCK(vp); 6557 if (VN_IS_DOOMED(vp)) { 6558 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 6559 goto out_lost; 6560 } 6561 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 6562 /* 6563 * There is nothing to do if we are the last user. 6564 */ 6565 if (!refcount_release_if_not_last(&vp->v_holdcnt)) 6566 goto out_lost; 6567 mtx_lock(&mp->mnt_listmtx); 6568 return (true); 6569 out_lost: 6570 vdropl(vp); 6571 maybe_yield(); 6572 mtx_lock(&mp->mnt_listmtx); 6573 return (false); 6574 } 6575 6576 static struct vnode * 6577 mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6578 void *cbarg) 6579 { 6580 struct vnode *vp; 6581 6582 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 6583 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6584 restart: 6585 vp = TAILQ_NEXT(*mvp, v_lazylist); 6586 while (vp != NULL) { 6587 if (vp->v_type == VMARKER) { 6588 vp = TAILQ_NEXT(vp, v_lazylist); 6589 continue; 6590 } 6591 /* 6592 * See if we want to process the vnode. Note we may encounter a 6593 * long string of vnodes we don't care about and hog the list 6594 * as a result. Check for it and requeue the marker. 6595 */ 6596 VNPASS(!VN_IS_DOOMED(vp), vp); 6597 if (!cb(vp, cbarg)) { 6598 if (!should_yield()) { 6599 vp = TAILQ_NEXT(vp, v_lazylist); 6600 continue; 6601 } 6602 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, 6603 v_lazylist); 6604 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, 6605 v_lazylist); 6606 mtx_unlock(&mp->mnt_listmtx); 6607 kern_yield(PRI_USER); 6608 mtx_lock(&mp->mnt_listmtx); 6609 goto restart; 6610 } 6611 /* 6612 * Try-lock because this is the wrong lock order. 6613 */ 6614 if (!VI_TRYLOCK(vp) && 6615 !mnt_vnode_next_lazy_relock(*mvp, mp, vp)) 6616 goto restart; 6617 KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp)); 6618 KASSERT(vp->v_mount == mp || vp->v_mount == NULL, 6619 ("alien vnode on the lazy list %p %p", vp, mp)); 6620 VNPASS(vp->v_mount == mp, vp); 6621 VNPASS(!VN_IS_DOOMED(vp), vp); 6622 break; 6623 } 6624 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 6625 6626 /* Check if we are done */ 6627 if (vp == NULL) { 6628 mtx_unlock(&mp->mnt_listmtx); 6629 mnt_vnode_markerfree_lazy(mvp, mp); 6630 return (NULL); 6631 } 6632 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, v_lazylist); 6633 mtx_unlock(&mp->mnt_listmtx); 6634 ASSERT_VI_LOCKED(vp, "lazy iter"); 6635 return (vp); 6636 } 6637 6638 struct vnode * 6639 __mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6640 void *cbarg) 6641 { 6642 6643 if (should_yield()) 6644 kern_yield(PRI_USER); 6645 mtx_lock(&mp->mnt_listmtx); 6646 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 6647 } 6648 6649 struct vnode * 6650 __mnt_vnode_first_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6651 void *cbarg) 6652 { 6653 struct vnode *vp; 6654 6655 if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist)) 6656 return (NULL); 6657 6658 *mvp = vn_alloc_marker(mp); 6659 MNT_ILOCK(mp); 6660 MNT_REF(mp); 6661 MNT_IUNLOCK(mp); 6662 6663 mtx_lock(&mp->mnt_listmtx); 6664 vp = TAILQ_FIRST(&mp->mnt_lazyvnodelist); 6665 if (vp == NULL) { 6666 mtx_unlock(&mp->mnt_listmtx); 6667 mnt_vnode_markerfree_lazy(mvp, mp); 6668 return (NULL); 6669 } 6670 TAILQ_INSERT_BEFORE(vp, *mvp, v_lazylist); 6671 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 6672 } 6673 6674 void 6675 __mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 6676 { 6677 6678 if (*mvp == NULL) 6679 return; 6680 6681 mtx_lock(&mp->mnt_listmtx); 6682 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 6683 mtx_unlock(&mp->mnt_listmtx); 6684 mnt_vnode_markerfree_lazy(mvp, mp); 6685 } 6686 6687 int 6688 vn_dir_check_exec(struct vnode *vp, struct componentname *cnp) 6689 { 6690 6691 if ((cnp->cn_flags & NOEXECCHECK) != 0) { 6692 cnp->cn_flags &= ~NOEXECCHECK; 6693 return (0); 6694 } 6695 6696 return (VOP_ACCESS(vp, VEXEC, cnp->cn_cred, cnp->cn_thread)); 6697 } 6698 6699 /* 6700 * Do not use this variant unless you have means other than the hold count 6701 * to prevent the vnode from getting freed. 6702 */ 6703 void 6704 vn_seqc_write_begin_unheld_locked(struct vnode *vp) 6705 { 6706 6707 ASSERT_VI_LOCKED(vp, __func__); 6708 VNPASS(vp->v_seqc_users >= 0, vp); 6709 vp->v_seqc_users++; 6710 if (vp->v_seqc_users == 1) 6711 seqc_sleepable_write_begin(&vp->v_seqc); 6712 } 6713 6714 void 6715 vn_seqc_write_begin_locked(struct vnode *vp) 6716 { 6717 6718 ASSERT_VI_LOCKED(vp, __func__); 6719 VNPASS(vp->v_holdcnt > 0, vp); 6720 vn_seqc_write_begin_unheld_locked(vp); 6721 } 6722 6723 void 6724 vn_seqc_write_begin(struct vnode *vp) 6725 { 6726 6727 VI_LOCK(vp); 6728 vn_seqc_write_begin_locked(vp); 6729 VI_UNLOCK(vp); 6730 } 6731 6732 void 6733 vn_seqc_write_begin_unheld(struct vnode *vp) 6734 { 6735 6736 VI_LOCK(vp); 6737 vn_seqc_write_begin_unheld_locked(vp); 6738 VI_UNLOCK(vp); 6739 } 6740 6741 void 6742 vn_seqc_write_end_locked(struct vnode *vp) 6743 { 6744 6745 ASSERT_VI_LOCKED(vp, __func__); 6746 VNPASS(vp->v_seqc_users > 0, vp); 6747 vp->v_seqc_users--; 6748 if (vp->v_seqc_users == 0) 6749 seqc_sleepable_write_end(&vp->v_seqc); 6750 } 6751 6752 void 6753 vn_seqc_write_end(struct vnode *vp) 6754 { 6755 6756 VI_LOCK(vp); 6757 vn_seqc_write_end_locked(vp); 6758 VI_UNLOCK(vp); 6759 } 6760