1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vfs_subr.c 8.31 (Berkeley) 5/26/95 37 */ 38 39 /* 40 * External virtual filesystem routines 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include "opt_ddb.h" 47 #include "opt_watchdog.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/asan.h> 52 #include <sys/bio.h> 53 #include <sys/buf.h> 54 #include <sys/capsicum.h> 55 #include <sys/condvar.h> 56 #include <sys/conf.h> 57 #include <sys/counter.h> 58 #include <sys/dirent.h> 59 #include <sys/event.h> 60 #include <sys/eventhandler.h> 61 #include <sys/extattr.h> 62 #include <sys/file.h> 63 #include <sys/fcntl.h> 64 #include <sys/jail.h> 65 #include <sys/kdb.h> 66 #include <sys/kernel.h> 67 #include <sys/kthread.h> 68 #include <sys/ktr.h> 69 #include <sys/lockf.h> 70 #include <sys/malloc.h> 71 #include <sys/mount.h> 72 #include <sys/namei.h> 73 #include <sys/pctrie.h> 74 #include <sys/priv.h> 75 #include <sys/reboot.h> 76 #include <sys/refcount.h> 77 #include <sys/rwlock.h> 78 #include <sys/sched.h> 79 #include <sys/sleepqueue.h> 80 #include <sys/smr.h> 81 #include <sys/smp.h> 82 #include <sys/stat.h> 83 #include <sys/sysctl.h> 84 #include <sys/syslog.h> 85 #include <sys/vmmeter.h> 86 #include <sys/vnode.h> 87 #include <sys/watchdog.h> 88 89 #include <machine/stdarg.h> 90 91 #include <security/mac/mac_framework.h> 92 93 #include <vm/vm.h> 94 #include <vm/vm_object.h> 95 #include <vm/vm_extern.h> 96 #include <vm/pmap.h> 97 #include <vm/vm_map.h> 98 #include <vm/vm_page.h> 99 #include <vm/vm_kern.h> 100 #include <vm/uma.h> 101 102 #ifdef DDB 103 #include <ddb/ddb.h> 104 #endif 105 106 static void delmntque(struct vnode *vp); 107 static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, 108 int slpflag, int slptimeo); 109 static void syncer_shutdown(void *arg, int howto); 110 static int vtryrecycle(struct vnode *vp); 111 static void v_init_counters(struct vnode *); 112 static void vn_seqc_init(struct vnode *); 113 static void vn_seqc_write_end_free(struct vnode *vp); 114 static void vgonel(struct vnode *); 115 static bool vhold_recycle_free(struct vnode *); 116 static void vdropl_recycle(struct vnode *vp); 117 static void vdrop_recycle(struct vnode *vp); 118 static void vfs_knllock(void *arg); 119 static void vfs_knlunlock(void *arg); 120 static void vfs_knl_assert_lock(void *arg, int what); 121 static void destroy_vpollinfo(struct vpollinfo *vi); 122 static int v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 123 daddr_t startlbn, daddr_t endlbn); 124 static void vnlru_recalc(void); 125 126 /* 127 * Number of vnodes in existence. Increased whenever getnewvnode() 128 * allocates a new vnode, decreased in vdropl() for VIRF_DOOMED vnode. 129 */ 130 static u_long __exclusive_cache_line numvnodes; 131 132 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0, 133 "Number of vnodes in existence"); 134 135 static counter_u64_t vnodes_created; 136 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created, 137 "Number of vnodes created by getnewvnode"); 138 139 /* 140 * Conversion tables for conversion from vnode types to inode formats 141 * and back. 142 */ 143 enum vtype iftovt_tab[16] = { 144 VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON, 145 VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON 146 }; 147 int vttoif_tab[10] = { 148 0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK, 149 S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT 150 }; 151 152 /* 153 * List of allocates vnodes in the system. 154 */ 155 static TAILQ_HEAD(freelst, vnode) vnode_list; 156 static struct vnode *vnode_list_free_marker; 157 static struct vnode *vnode_list_reclaim_marker; 158 159 /* 160 * "Free" vnode target. Free vnodes are rarely completely free, but are 161 * just ones that are cheap to recycle. Usually they are for files which 162 * have been stat'd but not read; these usually have inode and namecache 163 * data attached to them. This target is the preferred minimum size of a 164 * sub-cache consisting mostly of such files. The system balances the size 165 * of this sub-cache with its complement to try to prevent either from 166 * thrashing while the other is relatively inactive. The targets express 167 * a preference for the best balance. 168 * 169 * "Above" this target there are 2 further targets (watermarks) related 170 * to recyling of free vnodes. In the best-operating case, the cache is 171 * exactly full, the free list has size between vlowat and vhiwat above the 172 * free target, and recycling from it and normal use maintains this state. 173 * Sometimes the free list is below vlowat or even empty, but this state 174 * is even better for immediate use provided the cache is not full. 175 * Otherwise, vnlru_proc() runs to reclaim enough vnodes (usually non-free 176 * ones) to reach one of these states. The watermarks are currently hard- 177 * coded as 4% and 9% of the available space higher. These and the default 178 * of 25% for wantfreevnodes are too large if the memory size is large. 179 * E.g., 9% of 75% of MAXVNODES is more than 566000 vnodes to reclaim 180 * whenever vnlru_proc() becomes active. 181 */ 182 static long wantfreevnodes; 183 static long __exclusive_cache_line freevnodes; 184 SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD, 185 &freevnodes, 0, "Number of \"free\" vnodes"); 186 static long freevnodes_old; 187 188 static counter_u64_t recycles_count; 189 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles, CTLFLAG_RD, &recycles_count, 190 "Number of vnodes recycled to meet vnode cache targets"); 191 192 static counter_u64_t recycles_free_count; 193 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles_free, CTLFLAG_RD, &recycles_free_count, 194 "Number of free vnodes recycled to meet vnode cache targets"); 195 196 static counter_u64_t deferred_inact; 197 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, deferred_inact, CTLFLAG_RD, &deferred_inact, 198 "Number of times inactive processing was deferred"); 199 200 /* To keep more than one thread at a time from running vfs_getnewfsid */ 201 static struct mtx mntid_mtx; 202 203 /* 204 * Lock for any access to the following: 205 * vnode_list 206 * numvnodes 207 * freevnodes 208 */ 209 static struct mtx __exclusive_cache_line vnode_list_mtx; 210 211 /* Publicly exported FS */ 212 struct nfs_public nfs_pub; 213 214 static uma_zone_t buf_trie_zone; 215 static smr_t buf_trie_smr; 216 217 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */ 218 static uma_zone_t vnode_zone; 219 MALLOC_DEFINE(M_VNODEPOLL, "VN POLL", "vnode poll"); 220 221 __read_frequently smr_t vfs_smr; 222 223 /* 224 * The workitem queue. 225 * 226 * It is useful to delay writes of file data and filesystem metadata 227 * for tens of seconds so that quickly created and deleted files need 228 * not waste disk bandwidth being created and removed. To realize this, 229 * we append vnodes to a "workitem" queue. When running with a soft 230 * updates implementation, most pending metadata dependencies should 231 * not wait for more than a few seconds. Thus, mounted on block devices 232 * are delayed only about a half the time that file data is delayed. 233 * Similarly, directory updates are more critical, so are only delayed 234 * about a third the time that file data is delayed. Thus, there are 235 * SYNCER_MAXDELAY queues that are processed round-robin at a rate of 236 * one each second (driven off the filesystem syncer process). The 237 * syncer_delayno variable indicates the next queue that is to be processed. 238 * Items that need to be processed soon are placed in this queue: 239 * 240 * syncer_workitem_pending[syncer_delayno] 241 * 242 * A delay of fifteen seconds is done by placing the request fifteen 243 * entries later in the queue: 244 * 245 * syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask] 246 * 247 */ 248 static int syncer_delayno; 249 static long syncer_mask; 250 LIST_HEAD(synclist, bufobj); 251 static struct synclist *syncer_workitem_pending; 252 /* 253 * The sync_mtx protects: 254 * bo->bo_synclist 255 * sync_vnode_count 256 * syncer_delayno 257 * syncer_state 258 * syncer_workitem_pending 259 * syncer_worklist_len 260 * rushjob 261 */ 262 static struct mtx sync_mtx; 263 static struct cv sync_wakeup; 264 265 #define SYNCER_MAXDELAY 32 266 static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */ 267 static int syncdelay = 30; /* max time to delay syncing data */ 268 static int filedelay = 30; /* time to delay syncing files */ 269 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0, 270 "Time to delay syncing files (in seconds)"); 271 static int dirdelay = 29; /* time to delay syncing directories */ 272 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0, 273 "Time to delay syncing directories (in seconds)"); 274 static int metadelay = 28; /* time to delay syncing metadata */ 275 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0, 276 "Time to delay syncing metadata (in seconds)"); 277 static int rushjob; /* number of slots to run ASAP */ 278 static int stat_rush_requests; /* number of times I/O speeded up */ 279 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0, 280 "Number of times I/O speeded up (rush requests)"); 281 282 #define VDBATCH_SIZE 8 283 struct vdbatch { 284 u_int index; 285 long freevnodes; 286 struct mtx lock; 287 struct vnode *tab[VDBATCH_SIZE]; 288 }; 289 DPCPU_DEFINE_STATIC(struct vdbatch, vd); 290 291 static void vdbatch_dequeue(struct vnode *vp); 292 293 /* 294 * When shutting down the syncer, run it at four times normal speed. 295 */ 296 #define SYNCER_SHUTDOWN_SPEEDUP 4 297 static int sync_vnode_count; 298 static int syncer_worklist_len; 299 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY } 300 syncer_state; 301 302 /* Target for maximum number of vnodes. */ 303 u_long desiredvnodes; 304 static u_long gapvnodes; /* gap between wanted and desired */ 305 static u_long vhiwat; /* enough extras after expansion */ 306 static u_long vlowat; /* minimal extras before expansion */ 307 static u_long vstir; /* nonzero to stir non-free vnodes */ 308 static volatile int vsmalltrigger = 8; /* pref to keep if > this many pages */ 309 310 static u_long vnlru_read_freevnodes(void); 311 312 /* 313 * Note that no attempt is made to sanitize these parameters. 314 */ 315 static int 316 sysctl_maxvnodes(SYSCTL_HANDLER_ARGS) 317 { 318 u_long val; 319 int error; 320 321 val = desiredvnodes; 322 error = sysctl_handle_long(oidp, &val, 0, req); 323 if (error != 0 || req->newptr == NULL) 324 return (error); 325 326 if (val == desiredvnodes) 327 return (0); 328 mtx_lock(&vnode_list_mtx); 329 desiredvnodes = val; 330 wantfreevnodes = desiredvnodes / 4; 331 vnlru_recalc(); 332 mtx_unlock(&vnode_list_mtx); 333 /* 334 * XXX There is no protection against multiple threads changing 335 * desiredvnodes at the same time. Locking above only helps vnlru and 336 * getnewvnode. 337 */ 338 vfs_hash_changesize(desiredvnodes); 339 cache_changesize(desiredvnodes); 340 return (0); 341 } 342 343 SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes, 344 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, 345 "LU", "Target for maximum number of vnodes"); 346 347 static int 348 sysctl_wantfreevnodes(SYSCTL_HANDLER_ARGS) 349 { 350 u_long val; 351 int error; 352 353 val = wantfreevnodes; 354 error = sysctl_handle_long(oidp, &val, 0, req); 355 if (error != 0 || req->newptr == NULL) 356 return (error); 357 358 if (val == wantfreevnodes) 359 return (0); 360 mtx_lock(&vnode_list_mtx); 361 wantfreevnodes = val; 362 vnlru_recalc(); 363 mtx_unlock(&vnode_list_mtx); 364 return (0); 365 } 366 367 SYSCTL_PROC(_vfs, OID_AUTO, wantfreevnodes, 368 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes, 369 "LU", "Target for minimum number of \"free\" vnodes"); 370 371 SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW, 372 &wantfreevnodes, 0, "Old name for vfs.wantfreevnodes (legacy)"); 373 static int vnlru_nowhere; 374 SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW, 375 &vnlru_nowhere, 0, "Number of times the vnlru process ran without success"); 376 377 static int 378 sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS) 379 { 380 struct vnode *vp; 381 struct nameidata nd; 382 char *buf; 383 unsigned long ndflags; 384 int error; 385 386 if (req->newptr == NULL) 387 return (EINVAL); 388 if (req->newlen >= PATH_MAX) 389 return (E2BIG); 390 391 buf = malloc(PATH_MAX, M_TEMP, M_WAITOK); 392 error = SYSCTL_IN(req, buf, req->newlen); 393 if (error != 0) 394 goto out; 395 396 buf[req->newlen] = '\0'; 397 398 ndflags = LOCKLEAF | NOFOLLOW | AUDITVNODE1; 399 NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE, buf); 400 if ((error = namei(&nd)) != 0) 401 goto out; 402 vp = nd.ni_vp; 403 404 if (VN_IS_DOOMED(vp)) { 405 /* 406 * This vnode is being recycled. Return != 0 to let the caller 407 * know that the sysctl had no effect. Return EAGAIN because a 408 * subsequent call will likely succeed (since namei will create 409 * a new vnode if necessary) 410 */ 411 error = EAGAIN; 412 goto putvnode; 413 } 414 415 counter_u64_add(recycles_count, 1); 416 vgone(vp); 417 putvnode: 418 NDFREE(&nd, 0); 419 out: 420 free(buf, M_TEMP); 421 return (error); 422 } 423 424 static int 425 sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS) 426 { 427 struct thread *td = curthread; 428 struct vnode *vp; 429 struct file *fp; 430 int error; 431 int fd; 432 433 if (req->newptr == NULL) 434 return (EBADF); 435 436 error = sysctl_handle_int(oidp, &fd, 0, req); 437 if (error != 0) 438 return (error); 439 error = getvnode(curthread, fd, &cap_fcntl_rights, &fp); 440 if (error != 0) 441 return (error); 442 vp = fp->f_vnode; 443 444 error = vn_lock(vp, LK_EXCLUSIVE); 445 if (error != 0) 446 goto drop; 447 448 counter_u64_add(recycles_count, 1); 449 vgone(vp); 450 VOP_UNLOCK(vp); 451 drop: 452 fdrop(fp, td); 453 return (error); 454 } 455 456 SYSCTL_PROC(_debug, OID_AUTO, try_reclaim_vnode, 457 CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 458 sysctl_try_reclaim_vnode, "A", "Try to reclaim a vnode by its pathname"); 459 SYSCTL_PROC(_debug, OID_AUTO, ftry_reclaim_vnode, 460 CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 461 sysctl_ftry_reclaim_vnode, "I", 462 "Try to reclaim a vnode by its file descriptor"); 463 464 /* Shift count for (uintptr_t)vp to initialize vp->v_hash. */ 465 #define vnsz2log 8 466 #ifndef DEBUG_LOCKS 467 _Static_assert(sizeof(struct vnode) >= 1UL << vnsz2log && 468 sizeof(struct vnode) < 1UL << (vnsz2log + 1), 469 "vnsz2log needs to be updated"); 470 #endif 471 472 /* 473 * Support for the bufobj clean & dirty pctrie. 474 */ 475 static void * 476 buf_trie_alloc(struct pctrie *ptree) 477 { 478 return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT)); 479 } 480 481 static void 482 buf_trie_free(struct pctrie *ptree, void *node) 483 { 484 uma_zfree_smr(buf_trie_zone, node); 485 } 486 PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free, 487 buf_trie_smr); 488 489 /* 490 * Initialize the vnode management data structures. 491 * 492 * Reevaluate the following cap on the number of vnodes after the physical 493 * memory size exceeds 512GB. In the limit, as the physical memory size 494 * grows, the ratio of the memory size in KB to vnodes approaches 64:1. 495 */ 496 #ifndef MAXVNODES_MAX 497 #define MAXVNODES_MAX (512UL * 1024 * 1024 / 64) /* 8M */ 498 #endif 499 500 static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); 501 502 static struct vnode * 503 vn_alloc_marker(struct mount *mp) 504 { 505 struct vnode *vp; 506 507 vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO); 508 vp->v_type = VMARKER; 509 vp->v_mount = mp; 510 511 return (vp); 512 } 513 514 static void 515 vn_free_marker(struct vnode *vp) 516 { 517 518 MPASS(vp->v_type == VMARKER); 519 free(vp, M_VNODE_MARKER); 520 } 521 522 #ifdef KASAN 523 static int 524 vnode_ctor(void *mem, int size, void *arg __unused, int flags __unused) 525 { 526 kasan_mark(mem, size, roundup2(size, UMA_ALIGN_PTR + 1), 0); 527 return (0); 528 } 529 530 static void 531 vnode_dtor(void *mem, int size, void *arg __unused) 532 { 533 size_t end1, end2, off1, off2; 534 535 _Static_assert(offsetof(struct vnode, v_vnodelist) < 536 offsetof(struct vnode, v_dbatchcpu), 537 "KASAN marks require updating"); 538 539 off1 = offsetof(struct vnode, v_vnodelist); 540 off2 = offsetof(struct vnode, v_dbatchcpu); 541 end1 = off1 + sizeof(((struct vnode *)NULL)->v_vnodelist); 542 end2 = off2 + sizeof(((struct vnode *)NULL)->v_dbatchcpu); 543 544 /* 545 * Access to the v_vnodelist and v_dbatchcpu fields are permitted even 546 * after the vnode has been freed. Try to get some KASAN coverage by 547 * marking everything except those two fields as invalid. Because 548 * KASAN's tracking is not byte-granular, any preceding fields sharing 549 * the same 8-byte aligned word must also be marked valid. 550 */ 551 552 /* Handle the area from the start until v_vnodelist... */ 553 off1 = rounddown2(off1, KASAN_SHADOW_SCALE); 554 kasan_mark(mem, off1, off1, KASAN_UMA_FREED); 555 556 /* ... then the area between v_vnodelist and v_dbatchcpu ... */ 557 off1 = roundup2(end1, KASAN_SHADOW_SCALE); 558 off2 = rounddown2(off2, KASAN_SHADOW_SCALE); 559 if (off2 > off1) 560 kasan_mark((void *)((char *)mem + off1), off2 - off1, 561 off2 - off1, KASAN_UMA_FREED); 562 563 /* ... and finally the area from v_dbatchcpu to the end. */ 564 off2 = roundup2(end2, KASAN_SHADOW_SCALE); 565 kasan_mark((void *)((char *)mem + off2), size - off2, size - off2, 566 KASAN_UMA_FREED); 567 } 568 #endif /* KASAN */ 569 570 /* 571 * Initialize a vnode as it first enters the zone. 572 */ 573 static int 574 vnode_init(void *mem, int size, int flags) 575 { 576 struct vnode *vp; 577 578 vp = mem; 579 bzero(vp, size); 580 /* 581 * Setup locks. 582 */ 583 vp->v_vnlock = &vp->v_lock; 584 mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF); 585 /* 586 * By default, don't allow shared locks unless filesystems opt-in. 587 */ 588 lockinit(vp->v_vnlock, PVFS, "vnode", VLKTIMEOUT, 589 LK_NOSHARE | LK_IS_VNODE); 590 /* 591 * Initialize bufobj. 592 */ 593 bufobj_init(&vp->v_bufobj, vp); 594 /* 595 * Initialize namecache. 596 */ 597 cache_vnode_init(vp); 598 /* 599 * Initialize rangelocks. 600 */ 601 rangelock_init(&vp->v_rl); 602 603 vp->v_dbatchcpu = NOCPU; 604 605 /* 606 * Check vhold_recycle_free for an explanation. 607 */ 608 vp->v_holdcnt = VHOLD_NO_SMR; 609 vp->v_type = VNON; 610 mtx_lock(&vnode_list_mtx); 611 TAILQ_INSERT_BEFORE(vnode_list_free_marker, vp, v_vnodelist); 612 mtx_unlock(&vnode_list_mtx); 613 return (0); 614 } 615 616 /* 617 * Free a vnode when it is cleared from the zone. 618 */ 619 static void 620 vnode_fini(void *mem, int size) 621 { 622 struct vnode *vp; 623 struct bufobj *bo; 624 625 vp = mem; 626 vdbatch_dequeue(vp); 627 mtx_lock(&vnode_list_mtx); 628 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 629 mtx_unlock(&vnode_list_mtx); 630 rangelock_destroy(&vp->v_rl); 631 lockdestroy(vp->v_vnlock); 632 mtx_destroy(&vp->v_interlock); 633 bo = &vp->v_bufobj; 634 rw_destroy(BO_LOCKPTR(bo)); 635 636 kasan_mark(mem, size, size, 0); 637 } 638 639 /* 640 * Provide the size of NFS nclnode and NFS fh for calculation of the 641 * vnode memory consumption. The size is specified directly to 642 * eliminate dependency on NFS-private header. 643 * 644 * Other filesystems may use bigger or smaller (like UFS and ZFS) 645 * private inode data, but the NFS-based estimation is ample enough. 646 * Still, we care about differences in the size between 64- and 32-bit 647 * platforms. 648 * 649 * Namecache structure size is heuristically 650 * sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1. 651 */ 652 #ifdef _LP64 653 #define NFS_NCLNODE_SZ (528 + 64) 654 #define NC_SZ 148 655 #else 656 #define NFS_NCLNODE_SZ (360 + 32) 657 #define NC_SZ 92 658 #endif 659 660 static void 661 vntblinit(void *dummy __unused) 662 { 663 struct vdbatch *vd; 664 uma_ctor ctor; 665 uma_dtor dtor; 666 int cpu, physvnodes, virtvnodes; 667 668 /* 669 * Desiredvnodes is a function of the physical memory size and the 670 * kernel's heap size. Generally speaking, it scales with the 671 * physical memory size. The ratio of desiredvnodes to the physical 672 * memory size is 1:16 until desiredvnodes exceeds 98,304. 673 * Thereafter, the 674 * marginal ratio of desiredvnodes to the physical memory size is 675 * 1:64. However, desiredvnodes is limited by the kernel's heap 676 * size. The memory required by desiredvnodes vnodes and vm objects 677 * must not exceed 1/10th of the kernel's heap size. 678 */ 679 physvnodes = maxproc + pgtok(vm_cnt.v_page_count) / 64 + 680 3 * min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 64; 681 virtvnodes = vm_kmem_size / (10 * (sizeof(struct vm_object) + 682 sizeof(struct vnode) + NC_SZ * ncsizefactor + NFS_NCLNODE_SZ)); 683 desiredvnodes = min(physvnodes, virtvnodes); 684 if (desiredvnodes > MAXVNODES_MAX) { 685 if (bootverbose) 686 printf("Reducing kern.maxvnodes %lu -> %lu\n", 687 desiredvnodes, MAXVNODES_MAX); 688 desiredvnodes = MAXVNODES_MAX; 689 } 690 wantfreevnodes = desiredvnodes / 4; 691 mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF); 692 TAILQ_INIT(&vnode_list); 693 mtx_init(&vnode_list_mtx, "vnode_list", NULL, MTX_DEF); 694 /* 695 * The lock is taken to appease WITNESS. 696 */ 697 mtx_lock(&vnode_list_mtx); 698 vnlru_recalc(); 699 mtx_unlock(&vnode_list_mtx); 700 vnode_list_free_marker = vn_alloc_marker(NULL); 701 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist); 702 vnode_list_reclaim_marker = vn_alloc_marker(NULL); 703 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist); 704 705 #ifdef KASAN 706 ctor = vnode_ctor; 707 dtor = vnode_dtor; 708 #else 709 ctor = NULL; 710 dtor = NULL; 711 #endif 712 vnode_zone = uma_zcreate("VNODE", sizeof(struct vnode), ctor, dtor, 713 vnode_init, vnode_fini, UMA_ALIGN_PTR, UMA_ZONE_NOKASAN); 714 uma_zone_set_smr(vnode_zone, vfs_smr); 715 716 /* 717 * Preallocate enough nodes to support one-per buf so that 718 * we can not fail an insert. reassignbuf() callers can not 719 * tolerate the insertion failure. 720 */ 721 buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(), 722 NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR, 723 UMA_ZONE_NOFREE | UMA_ZONE_SMR); 724 buf_trie_smr = uma_zone_get_smr(buf_trie_zone); 725 uma_prealloc(buf_trie_zone, nbuf); 726 727 vnodes_created = counter_u64_alloc(M_WAITOK); 728 recycles_count = counter_u64_alloc(M_WAITOK); 729 recycles_free_count = counter_u64_alloc(M_WAITOK); 730 deferred_inact = counter_u64_alloc(M_WAITOK); 731 732 /* 733 * Initialize the filesystem syncer. 734 */ 735 syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE, 736 &syncer_mask); 737 syncer_maxdelay = syncer_mask + 1; 738 mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF); 739 cv_init(&sync_wakeup, "syncer"); 740 741 CPU_FOREACH(cpu) { 742 vd = DPCPU_ID_PTR((cpu), vd); 743 bzero(vd, sizeof(*vd)); 744 mtx_init(&vd->lock, "vdbatch", NULL, MTX_DEF); 745 } 746 } 747 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL); 748 749 /* 750 * Mark a mount point as busy. Used to synchronize access and to delay 751 * unmounting. Eventually, mountlist_mtx is not released on failure. 752 * 753 * vfs_busy() is a custom lock, it can block the caller. 754 * vfs_busy() only sleeps if the unmount is active on the mount point. 755 * For a mountpoint mp, vfs_busy-enforced lock is before lock of any 756 * vnode belonging to mp. 757 * 758 * Lookup uses vfs_busy() to traverse mount points. 759 * root fs var fs 760 * / vnode lock A / vnode lock (/var) D 761 * /var vnode lock B /log vnode lock(/var/log) E 762 * vfs_busy lock C vfs_busy lock F 763 * 764 * Within each file system, the lock order is C->A->B and F->D->E. 765 * 766 * When traversing across mounts, the system follows that lock order: 767 * 768 * C->A->B 769 * | 770 * +->F->D->E 771 * 772 * The lookup() process for namei("/var") illustrates the process: 773 * VOP_LOOKUP() obtains B while A is held 774 * vfs_busy() obtains a shared lock on F while A and B are held 775 * vput() releases lock on B 776 * vput() releases lock on A 777 * VFS_ROOT() obtains lock on D while shared lock on F is held 778 * vfs_unbusy() releases shared lock on F 779 * vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A. 780 * Attempt to lock A (instead of vp_crossmp) while D is held would 781 * violate the global order, causing deadlocks. 782 * 783 * dounmount() locks B while F is drained. 784 */ 785 int 786 vfs_busy(struct mount *mp, int flags) 787 { 788 struct mount_pcpu *mpcpu; 789 790 MPASS((flags & ~MBF_MASK) == 0); 791 CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags); 792 793 if (vfs_op_thread_enter(mp, mpcpu)) { 794 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 795 MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0); 796 MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0); 797 vfs_mp_count_add_pcpu(mpcpu, ref, 1); 798 vfs_mp_count_add_pcpu(mpcpu, lockref, 1); 799 vfs_op_thread_exit(mp, mpcpu); 800 if (flags & MBF_MNTLSTLOCK) 801 mtx_unlock(&mountlist_mtx); 802 return (0); 803 } 804 805 MNT_ILOCK(mp); 806 vfs_assert_mount_counters(mp); 807 MNT_REF(mp); 808 /* 809 * If mount point is currently being unmounted, sleep until the 810 * mount point fate is decided. If thread doing the unmounting fails, 811 * it will clear MNTK_UNMOUNT flag before waking us up, indicating 812 * that this mount point has survived the unmount attempt and vfs_busy 813 * should retry. Otherwise the unmounter thread will set MNTK_REFEXPIRE 814 * flag in addition to MNTK_UNMOUNT, indicating that mount point is 815 * about to be really destroyed. vfs_busy needs to release its 816 * reference on the mount point in this case and return with ENOENT, 817 * telling the caller that mount mount it tried to busy is no longer 818 * valid. 819 */ 820 while (mp->mnt_kern_flag & MNTK_UNMOUNT) { 821 KASSERT(TAILQ_EMPTY(&mp->mnt_uppers), 822 ("%s: non-empty upper mount list with pending unmount", 823 __func__)); 824 if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) { 825 MNT_REL(mp); 826 MNT_IUNLOCK(mp); 827 CTR1(KTR_VFS, "%s: failed busying before sleeping", 828 __func__); 829 return (ENOENT); 830 } 831 if (flags & MBF_MNTLSTLOCK) 832 mtx_unlock(&mountlist_mtx); 833 mp->mnt_kern_flag |= MNTK_MWAIT; 834 msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0); 835 if (flags & MBF_MNTLSTLOCK) 836 mtx_lock(&mountlist_mtx); 837 MNT_ILOCK(mp); 838 } 839 if (flags & MBF_MNTLSTLOCK) 840 mtx_unlock(&mountlist_mtx); 841 mp->mnt_lockref++; 842 MNT_IUNLOCK(mp); 843 return (0); 844 } 845 846 /* 847 * Free a busy filesystem. 848 */ 849 void 850 vfs_unbusy(struct mount *mp) 851 { 852 struct mount_pcpu *mpcpu; 853 int c; 854 855 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 856 857 if (vfs_op_thread_enter(mp, mpcpu)) { 858 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 859 vfs_mp_count_sub_pcpu(mpcpu, lockref, 1); 860 vfs_mp_count_sub_pcpu(mpcpu, ref, 1); 861 vfs_op_thread_exit(mp, mpcpu); 862 return; 863 } 864 865 MNT_ILOCK(mp); 866 vfs_assert_mount_counters(mp); 867 MNT_REL(mp); 868 c = --mp->mnt_lockref; 869 if (mp->mnt_vfs_ops == 0) { 870 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 871 MNT_IUNLOCK(mp); 872 return; 873 } 874 if (c < 0) 875 vfs_dump_mount_counters(mp); 876 if (c == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) { 877 MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT); 878 CTR1(KTR_VFS, "%s: waking up waiters", __func__); 879 mp->mnt_kern_flag &= ~MNTK_DRAINING; 880 wakeup(&mp->mnt_lockref); 881 } 882 MNT_IUNLOCK(mp); 883 } 884 885 /* 886 * Lookup a mount point by filesystem identifier. 887 */ 888 struct mount * 889 vfs_getvfs(fsid_t *fsid) 890 { 891 struct mount *mp; 892 893 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 894 mtx_lock(&mountlist_mtx); 895 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 896 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 897 vfs_ref(mp); 898 mtx_unlock(&mountlist_mtx); 899 return (mp); 900 } 901 } 902 mtx_unlock(&mountlist_mtx); 903 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 904 return ((struct mount *) 0); 905 } 906 907 /* 908 * Lookup a mount point by filesystem identifier, busying it before 909 * returning. 910 * 911 * To avoid congestion on mountlist_mtx, implement simple direct-mapped 912 * cache for popular filesystem identifiers. The cache is lockess, using 913 * the fact that struct mount's are never freed. In worst case we may 914 * get pointer to unmounted or even different filesystem, so we have to 915 * check what we got, and go slow way if so. 916 */ 917 struct mount * 918 vfs_busyfs(fsid_t *fsid) 919 { 920 #define FSID_CACHE_SIZE 256 921 typedef struct mount * volatile vmp_t; 922 static vmp_t cache[FSID_CACHE_SIZE]; 923 struct mount *mp; 924 int error; 925 uint32_t hash; 926 927 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 928 hash = fsid->val[0] ^ fsid->val[1]; 929 hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1); 930 mp = cache[hash]; 931 if (mp == NULL || fsidcmp(&mp->mnt_stat.f_fsid, fsid) != 0) 932 goto slow; 933 if (vfs_busy(mp, 0) != 0) { 934 cache[hash] = NULL; 935 goto slow; 936 } 937 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) 938 return (mp); 939 else 940 vfs_unbusy(mp); 941 942 slow: 943 mtx_lock(&mountlist_mtx); 944 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 945 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 946 error = vfs_busy(mp, MBF_MNTLSTLOCK); 947 if (error) { 948 cache[hash] = NULL; 949 mtx_unlock(&mountlist_mtx); 950 return (NULL); 951 } 952 cache[hash] = mp; 953 return (mp); 954 } 955 } 956 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 957 mtx_unlock(&mountlist_mtx); 958 return ((struct mount *) 0); 959 } 960 961 /* 962 * Check if a user can access privileged mount options. 963 */ 964 int 965 vfs_suser(struct mount *mp, struct thread *td) 966 { 967 int error; 968 969 if (jailed(td->td_ucred)) { 970 /* 971 * If the jail of the calling thread lacks permission for 972 * this type of file system, deny immediately. 973 */ 974 if (!prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag)) 975 return (EPERM); 976 977 /* 978 * If the file system was mounted outside the jail of the 979 * calling thread, deny immediately. 980 */ 981 if (prison_check(td->td_ucred, mp->mnt_cred) != 0) 982 return (EPERM); 983 } 984 985 /* 986 * If file system supports delegated administration, we don't check 987 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified 988 * by the file system itself. 989 * If this is not the user that did original mount, we check for 990 * the PRIV_VFS_MOUNT_OWNER privilege. 991 */ 992 if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) && 993 mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) { 994 if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0) 995 return (error); 996 } 997 return (0); 998 } 999 1000 /* 1001 * Get a new unique fsid. Try to make its val[0] unique, since this value 1002 * will be used to create fake device numbers for stat(). Also try (but 1003 * not so hard) make its val[0] unique mod 2^16, since some emulators only 1004 * support 16-bit device numbers. We end up with unique val[0]'s for the 1005 * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls. 1006 * 1007 * Keep in mind that several mounts may be running in parallel. Starting 1008 * the search one past where the previous search terminated is both a 1009 * micro-optimization and a defense against returning the same fsid to 1010 * different mounts. 1011 */ 1012 void 1013 vfs_getnewfsid(struct mount *mp) 1014 { 1015 static uint16_t mntid_base; 1016 struct mount *nmp; 1017 fsid_t tfsid; 1018 int mtype; 1019 1020 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 1021 mtx_lock(&mntid_mtx); 1022 mtype = mp->mnt_vfc->vfc_typenum; 1023 tfsid.val[1] = mtype; 1024 mtype = (mtype & 0xFF) << 24; 1025 for (;;) { 1026 tfsid.val[0] = makedev(255, 1027 mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF)); 1028 mntid_base++; 1029 if ((nmp = vfs_getvfs(&tfsid)) == NULL) 1030 break; 1031 vfs_rel(nmp); 1032 } 1033 mp->mnt_stat.f_fsid.val[0] = tfsid.val[0]; 1034 mp->mnt_stat.f_fsid.val[1] = tfsid.val[1]; 1035 mtx_unlock(&mntid_mtx); 1036 } 1037 1038 /* 1039 * Knob to control the precision of file timestamps: 1040 * 1041 * 0 = seconds only; nanoseconds zeroed. 1042 * 1 = seconds and nanoseconds, accurate within 1/HZ. 1043 * 2 = seconds and nanoseconds, truncated to microseconds. 1044 * >=3 = seconds and nanoseconds, maximum precision. 1045 */ 1046 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC }; 1047 1048 static int timestamp_precision = TSP_USEC; 1049 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW, 1050 ×tamp_precision, 0, "File timestamp precision (0: seconds, " 1051 "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, " 1052 "3+: sec + ns (max. precision))"); 1053 1054 /* 1055 * Get a current timestamp. 1056 */ 1057 void 1058 vfs_timestamp(struct timespec *tsp) 1059 { 1060 struct timeval tv; 1061 1062 switch (timestamp_precision) { 1063 case TSP_SEC: 1064 tsp->tv_sec = time_second; 1065 tsp->tv_nsec = 0; 1066 break; 1067 case TSP_HZ: 1068 getnanotime(tsp); 1069 break; 1070 case TSP_USEC: 1071 microtime(&tv); 1072 TIMEVAL_TO_TIMESPEC(&tv, tsp); 1073 break; 1074 case TSP_NSEC: 1075 default: 1076 nanotime(tsp); 1077 break; 1078 } 1079 } 1080 1081 /* 1082 * Set vnode attributes to VNOVAL 1083 */ 1084 void 1085 vattr_null(struct vattr *vap) 1086 { 1087 1088 vap->va_type = VNON; 1089 vap->va_size = VNOVAL; 1090 vap->va_bytes = VNOVAL; 1091 vap->va_mode = VNOVAL; 1092 vap->va_nlink = VNOVAL; 1093 vap->va_uid = VNOVAL; 1094 vap->va_gid = VNOVAL; 1095 vap->va_fsid = VNOVAL; 1096 vap->va_fileid = VNOVAL; 1097 vap->va_blocksize = VNOVAL; 1098 vap->va_rdev = VNOVAL; 1099 vap->va_atime.tv_sec = VNOVAL; 1100 vap->va_atime.tv_nsec = VNOVAL; 1101 vap->va_mtime.tv_sec = VNOVAL; 1102 vap->va_mtime.tv_nsec = VNOVAL; 1103 vap->va_ctime.tv_sec = VNOVAL; 1104 vap->va_ctime.tv_nsec = VNOVAL; 1105 vap->va_birthtime.tv_sec = VNOVAL; 1106 vap->va_birthtime.tv_nsec = VNOVAL; 1107 vap->va_flags = VNOVAL; 1108 vap->va_gen = VNOVAL; 1109 vap->va_vaflags = 0; 1110 } 1111 1112 /* 1113 * Try to reduce the total number of vnodes. 1114 * 1115 * This routine (and its user) are buggy in at least the following ways: 1116 * - all parameters were picked years ago when RAM sizes were significantly 1117 * smaller 1118 * - it can pick vnodes based on pages used by the vm object, but filesystems 1119 * like ZFS don't use it making the pick broken 1120 * - since ZFS has its own aging policy it gets partially combated by this one 1121 * - a dedicated method should be provided for filesystems to let them decide 1122 * whether the vnode should be recycled 1123 * 1124 * This routine is called when we have too many vnodes. It attempts 1125 * to free <count> vnodes and will potentially free vnodes that still 1126 * have VM backing store (VM backing store is typically the cause 1127 * of a vnode blowout so we want to do this). Therefore, this operation 1128 * is not considered cheap. 1129 * 1130 * A number of conditions may prevent a vnode from being reclaimed. 1131 * the buffer cache may have references on the vnode, a directory 1132 * vnode may still have references due to the namei cache representing 1133 * underlying files, or the vnode may be in active use. It is not 1134 * desirable to reuse such vnodes. These conditions may cause the 1135 * number of vnodes to reach some minimum value regardless of what 1136 * you set kern.maxvnodes to. Do not set kern.maxvnodes too low. 1137 * 1138 * @param reclaim_nc_src Only reclaim directories with outgoing namecache 1139 * entries if this argument is strue 1140 * @param trigger Only reclaim vnodes with fewer than this many resident 1141 * pages. 1142 * @param target How many vnodes to reclaim. 1143 * @return The number of vnodes that were reclaimed. 1144 */ 1145 static int 1146 vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) 1147 { 1148 struct vnode *vp, *mvp; 1149 struct mount *mp; 1150 struct vm_object *object; 1151 u_long done; 1152 bool retried; 1153 1154 mtx_assert(&vnode_list_mtx, MA_OWNED); 1155 1156 retried = false; 1157 done = 0; 1158 1159 mvp = vnode_list_reclaim_marker; 1160 restart: 1161 vp = mvp; 1162 while (done < target) { 1163 vp = TAILQ_NEXT(vp, v_vnodelist); 1164 if (__predict_false(vp == NULL)) 1165 break; 1166 1167 if (__predict_false(vp->v_type == VMARKER)) 1168 continue; 1169 1170 /* 1171 * If it's been deconstructed already, it's still 1172 * referenced, or it exceeds the trigger, skip it. 1173 * Also skip free vnodes. We are trying to make space 1174 * to expand the free list, not reduce it. 1175 */ 1176 if (vp->v_usecount > 0 || vp->v_holdcnt == 0 || 1177 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src))) 1178 goto next_iter; 1179 1180 if (vp->v_type == VBAD || vp->v_type == VNON) 1181 goto next_iter; 1182 1183 object = atomic_load_ptr(&vp->v_object); 1184 if (object == NULL || object->resident_page_count > trigger) { 1185 goto next_iter; 1186 } 1187 1188 /* 1189 * Handle races against vnode allocation. Filesystems lock the 1190 * vnode some time after it gets returned from getnewvnode, 1191 * despite type and hold count being manipulated earlier. 1192 * Resorting to checking v_mount restores guarantees present 1193 * before the global list was reworked to contain all vnodes. 1194 */ 1195 if (!VI_TRYLOCK(vp)) 1196 goto next_iter; 1197 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1198 VI_UNLOCK(vp); 1199 goto next_iter; 1200 } 1201 if (vp->v_mount == NULL) { 1202 VI_UNLOCK(vp); 1203 goto next_iter; 1204 } 1205 vholdl(vp); 1206 VI_UNLOCK(vp); 1207 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1208 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1209 mtx_unlock(&vnode_list_mtx); 1210 1211 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 1212 vdrop_recycle(vp); 1213 goto next_iter_unlocked; 1214 } 1215 if (VOP_LOCK(vp, LK_EXCLUSIVE|LK_NOWAIT) != 0) { 1216 vdrop_recycle(vp); 1217 vn_finished_write(mp); 1218 goto next_iter_unlocked; 1219 } 1220 1221 VI_LOCK(vp); 1222 if (vp->v_usecount > 0 || 1223 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) || 1224 (vp->v_object != NULL && vp->v_object->handle == vp && 1225 vp->v_object->resident_page_count > trigger)) { 1226 VOP_UNLOCK(vp); 1227 vdropl_recycle(vp); 1228 vn_finished_write(mp); 1229 goto next_iter_unlocked; 1230 } 1231 counter_u64_add(recycles_count, 1); 1232 vgonel(vp); 1233 VOP_UNLOCK(vp); 1234 vdropl_recycle(vp); 1235 vn_finished_write(mp); 1236 done++; 1237 next_iter_unlocked: 1238 if (should_yield()) 1239 kern_yield(PRI_USER); 1240 mtx_lock(&vnode_list_mtx); 1241 goto restart; 1242 next_iter: 1243 MPASS(vp->v_type != VMARKER); 1244 if (!should_yield()) 1245 continue; 1246 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1247 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1248 mtx_unlock(&vnode_list_mtx); 1249 kern_yield(PRI_USER); 1250 mtx_lock(&vnode_list_mtx); 1251 goto restart; 1252 } 1253 if (done == 0 && !retried) { 1254 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1255 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); 1256 retried = true; 1257 goto restart; 1258 } 1259 return (done); 1260 } 1261 1262 static int max_vnlru_free = 10000; /* limit on vnode free requests per call */ 1263 SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_vnlru_free, 1264 0, 1265 "limit on vnode free requests per call to the vnlru_free routine"); 1266 1267 /* 1268 * Attempt to reduce the free list by the requested amount. 1269 */ 1270 static int 1271 vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp) 1272 { 1273 struct vnode *vp; 1274 struct mount *mp; 1275 int ocount; 1276 1277 mtx_assert(&vnode_list_mtx, MA_OWNED); 1278 if (count > max_vnlru_free) 1279 count = max_vnlru_free; 1280 ocount = count; 1281 vp = mvp; 1282 for (;;) { 1283 if (count == 0) { 1284 break; 1285 } 1286 vp = TAILQ_NEXT(vp, v_vnodelist); 1287 if (__predict_false(vp == NULL)) { 1288 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1289 TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist); 1290 break; 1291 } 1292 if (__predict_false(vp->v_type == VMARKER)) 1293 continue; 1294 if (vp->v_holdcnt > 0) 1295 continue; 1296 /* 1297 * Don't recycle if our vnode is from different type 1298 * of mount point. Note that mp is type-safe, the 1299 * check does not reach unmapped address even if 1300 * vnode is reclaimed. 1301 */ 1302 if (mnt_op != NULL && (mp = vp->v_mount) != NULL && 1303 mp->mnt_op != mnt_op) { 1304 continue; 1305 } 1306 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1307 continue; 1308 } 1309 if (!vhold_recycle_free(vp)) 1310 continue; 1311 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1312 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1313 mtx_unlock(&vnode_list_mtx); 1314 /* 1315 * FIXME: ignores the return value, meaning it may be nothing 1316 * got recycled but it claims otherwise to the caller. 1317 * 1318 * Originally the value started being ignored in 2005 with 1319 * 114a1006a8204aa156e1f9ad6476cdff89cada7f . 1320 * 1321 * Respecting the value can run into significant stalls if most 1322 * vnodes belong to one file system and it has writes 1323 * suspended. In presence of many threads and millions of 1324 * vnodes they keep contending on the vnode_list_mtx lock only 1325 * to find vnodes they can't recycle. 1326 * 1327 * The solution would be to pre-check if the vnode is likely to 1328 * be recycle-able, but it needs to happen with the 1329 * vnode_list_mtx lock held. This runs into a problem where 1330 * VOP_GETWRITEMOUNT (currently needed to find out about if 1331 * writes are frozen) can take locks which LOR against it. 1332 * 1333 * Check nullfs for one example (null_getwritemount). 1334 */ 1335 vtryrecycle(vp); 1336 count--; 1337 mtx_lock(&vnode_list_mtx); 1338 vp = mvp; 1339 } 1340 return (ocount - count); 1341 } 1342 1343 static int 1344 vnlru_free_locked(int count) 1345 { 1346 1347 mtx_assert(&vnode_list_mtx, MA_OWNED); 1348 return (vnlru_free_impl(count, NULL, vnode_list_free_marker)); 1349 } 1350 1351 void 1352 vnlru_free_vfsops(int count, struct vfsops *mnt_op, struct vnode *mvp) 1353 { 1354 1355 MPASS(mnt_op != NULL); 1356 MPASS(mvp != NULL); 1357 VNPASS(mvp->v_type == VMARKER, mvp); 1358 mtx_lock(&vnode_list_mtx); 1359 vnlru_free_impl(count, mnt_op, mvp); 1360 mtx_unlock(&vnode_list_mtx); 1361 } 1362 1363 struct vnode * 1364 vnlru_alloc_marker(void) 1365 { 1366 struct vnode *mvp; 1367 1368 mvp = vn_alloc_marker(NULL); 1369 mtx_lock(&vnode_list_mtx); 1370 TAILQ_INSERT_BEFORE(vnode_list_free_marker, mvp, v_vnodelist); 1371 mtx_unlock(&vnode_list_mtx); 1372 return (mvp); 1373 } 1374 1375 void 1376 vnlru_free_marker(struct vnode *mvp) 1377 { 1378 mtx_lock(&vnode_list_mtx); 1379 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1380 mtx_unlock(&vnode_list_mtx); 1381 vn_free_marker(mvp); 1382 } 1383 1384 static void 1385 vnlru_recalc(void) 1386 { 1387 1388 mtx_assert(&vnode_list_mtx, MA_OWNED); 1389 gapvnodes = imax(desiredvnodes - wantfreevnodes, 100); 1390 vhiwat = gapvnodes / 11; /* 9% -- just under the 10% in vlrureclaim() */ 1391 vlowat = vhiwat / 2; 1392 } 1393 1394 /* 1395 * Attempt to recycle vnodes in a context that is always safe to block. 1396 * Calling vlrurecycle() from the bowels of filesystem code has some 1397 * interesting deadlock problems. 1398 */ 1399 static struct proc *vnlruproc; 1400 static int vnlruproc_sig; 1401 1402 /* 1403 * The main freevnodes counter is only updated when threads requeue their vnode 1404 * batches. CPUs are conditionally walked to compute a more accurate total. 1405 * 1406 * Limit how much of a slop are we willing to tolerate. Note: the actual value 1407 * at any given moment can still exceed slop, but it should not be by significant 1408 * margin in practice. 1409 */ 1410 #define VNLRU_FREEVNODES_SLOP 128 1411 1412 static __inline void 1413 vfs_freevnodes_inc(void) 1414 { 1415 struct vdbatch *vd; 1416 1417 critical_enter(); 1418 vd = DPCPU_PTR(vd); 1419 vd->freevnodes++; 1420 critical_exit(); 1421 } 1422 1423 static __inline void 1424 vfs_freevnodes_dec(void) 1425 { 1426 struct vdbatch *vd; 1427 1428 critical_enter(); 1429 vd = DPCPU_PTR(vd); 1430 vd->freevnodes--; 1431 critical_exit(); 1432 } 1433 1434 static u_long 1435 vnlru_read_freevnodes(void) 1436 { 1437 struct vdbatch *vd; 1438 long slop; 1439 int cpu; 1440 1441 mtx_assert(&vnode_list_mtx, MA_OWNED); 1442 if (freevnodes > freevnodes_old) 1443 slop = freevnodes - freevnodes_old; 1444 else 1445 slop = freevnodes_old - freevnodes; 1446 if (slop < VNLRU_FREEVNODES_SLOP) 1447 return (freevnodes >= 0 ? freevnodes : 0); 1448 freevnodes_old = freevnodes; 1449 CPU_FOREACH(cpu) { 1450 vd = DPCPU_ID_PTR((cpu), vd); 1451 freevnodes_old += vd->freevnodes; 1452 } 1453 return (freevnodes_old >= 0 ? freevnodes_old : 0); 1454 } 1455 1456 static bool 1457 vnlru_under(u_long rnumvnodes, u_long limit) 1458 { 1459 u_long rfreevnodes, space; 1460 1461 if (__predict_false(rnumvnodes > desiredvnodes)) 1462 return (true); 1463 1464 space = desiredvnodes - rnumvnodes; 1465 if (space < limit) { 1466 rfreevnodes = vnlru_read_freevnodes(); 1467 if (rfreevnodes > wantfreevnodes) 1468 space += rfreevnodes - wantfreevnodes; 1469 } 1470 return (space < limit); 1471 } 1472 1473 static bool 1474 vnlru_under_unlocked(u_long rnumvnodes, u_long limit) 1475 { 1476 long rfreevnodes, space; 1477 1478 if (__predict_false(rnumvnodes > desiredvnodes)) 1479 return (true); 1480 1481 space = desiredvnodes - rnumvnodes; 1482 if (space < limit) { 1483 rfreevnodes = atomic_load_long(&freevnodes); 1484 if (rfreevnodes > wantfreevnodes) 1485 space += rfreevnodes - wantfreevnodes; 1486 } 1487 return (space < limit); 1488 } 1489 1490 static void 1491 vnlru_kick(void) 1492 { 1493 1494 mtx_assert(&vnode_list_mtx, MA_OWNED); 1495 if (vnlruproc_sig == 0) { 1496 vnlruproc_sig = 1; 1497 wakeup(vnlruproc); 1498 } 1499 } 1500 1501 static void 1502 vnlru_proc(void) 1503 { 1504 u_long rnumvnodes, rfreevnodes, target; 1505 unsigned long onumvnodes; 1506 int done, force, trigger, usevnodes; 1507 bool reclaim_nc_src, want_reread; 1508 1509 EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, vnlruproc, 1510 SHUTDOWN_PRI_FIRST); 1511 1512 force = 0; 1513 want_reread = false; 1514 for (;;) { 1515 kproc_suspend_check(vnlruproc); 1516 mtx_lock(&vnode_list_mtx); 1517 rnumvnodes = atomic_load_long(&numvnodes); 1518 1519 if (want_reread) { 1520 force = vnlru_under(numvnodes, vhiwat) ? 1 : 0; 1521 want_reread = false; 1522 } 1523 1524 /* 1525 * If numvnodes is too large (due to desiredvnodes being 1526 * adjusted using its sysctl, or emergency growth), first 1527 * try to reduce it by discarding from the free list. 1528 */ 1529 if (rnumvnodes > desiredvnodes) { 1530 vnlru_free_locked(rnumvnodes - desiredvnodes); 1531 rnumvnodes = atomic_load_long(&numvnodes); 1532 } 1533 /* 1534 * Sleep if the vnode cache is in a good state. This is 1535 * when it is not over-full and has space for about a 4% 1536 * or 9% expansion (by growing its size or inexcessively 1537 * reducing its free list). Otherwise, try to reclaim 1538 * space for a 10% expansion. 1539 */ 1540 if (vstir && force == 0) { 1541 force = 1; 1542 vstir = 0; 1543 } 1544 if (force == 0 && !vnlru_under(rnumvnodes, vlowat)) { 1545 vnlruproc_sig = 0; 1546 wakeup(&vnlruproc_sig); 1547 msleep(vnlruproc, &vnode_list_mtx, 1548 PVFS|PDROP, "vlruwt", hz); 1549 continue; 1550 } 1551 rfreevnodes = vnlru_read_freevnodes(); 1552 1553 onumvnodes = rnumvnodes; 1554 /* 1555 * Calculate parameters for recycling. These are the same 1556 * throughout the loop to give some semblance of fairness. 1557 * The trigger point is to avoid recycling vnodes with lots 1558 * of resident pages. We aren't trying to free memory; we 1559 * are trying to recycle or at least free vnodes. 1560 */ 1561 if (rnumvnodes <= desiredvnodes) 1562 usevnodes = rnumvnodes - rfreevnodes; 1563 else 1564 usevnodes = rnumvnodes; 1565 if (usevnodes <= 0) 1566 usevnodes = 1; 1567 /* 1568 * The trigger value is chosen to give a conservatively 1569 * large value to ensure that it alone doesn't prevent 1570 * making progress. The value can easily be so large that 1571 * it is effectively infinite in some congested and 1572 * misconfigured cases, and this is necessary. Normally 1573 * it is about 8 to 100 (pages), which is quite large. 1574 */ 1575 trigger = vm_cnt.v_page_count * 2 / usevnodes; 1576 if (force < 2) 1577 trigger = vsmalltrigger; 1578 reclaim_nc_src = force >= 3; 1579 target = rnumvnodes * (int64_t)gapvnodes / imax(desiredvnodes, 1); 1580 target = target / 10 + 1; 1581 done = vlrureclaim(reclaim_nc_src, trigger, target); 1582 mtx_unlock(&vnode_list_mtx); 1583 if (onumvnodes > desiredvnodes && numvnodes <= desiredvnodes) 1584 uma_reclaim(UMA_RECLAIM_DRAIN); 1585 if (done == 0) { 1586 if (force == 0 || force == 1) { 1587 force = 2; 1588 continue; 1589 } 1590 if (force == 2) { 1591 force = 3; 1592 continue; 1593 } 1594 want_reread = true; 1595 force = 0; 1596 vnlru_nowhere++; 1597 tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3); 1598 } else { 1599 want_reread = true; 1600 kern_yield(PRI_USER); 1601 } 1602 } 1603 } 1604 1605 static struct kproc_desc vnlru_kp = { 1606 "vnlru", 1607 vnlru_proc, 1608 &vnlruproc 1609 }; 1610 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, 1611 &vnlru_kp); 1612 1613 /* 1614 * Routines having to do with the management of the vnode table. 1615 */ 1616 1617 /* 1618 * Try to recycle a freed vnode. We abort if anyone picks up a reference 1619 * before we actually vgone(). This function must be called with the vnode 1620 * held to prevent the vnode from being returned to the free list midway 1621 * through vgone(). 1622 */ 1623 static int 1624 vtryrecycle(struct vnode *vp) 1625 { 1626 struct mount *vnmp; 1627 1628 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 1629 VNASSERT(vp->v_holdcnt, vp, 1630 ("vtryrecycle: Recycling vp %p without a reference.", vp)); 1631 /* 1632 * This vnode may found and locked via some other list, if so we 1633 * can't recycle it yet. 1634 */ 1635 if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { 1636 CTR2(KTR_VFS, 1637 "%s: impossible to recycle, vp %p lock is already held", 1638 __func__, vp); 1639 vdrop_recycle(vp); 1640 return (EWOULDBLOCK); 1641 } 1642 /* 1643 * Don't recycle if its filesystem is being suspended. 1644 */ 1645 if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) { 1646 VOP_UNLOCK(vp); 1647 CTR2(KTR_VFS, 1648 "%s: impossible to recycle, cannot start the write for %p", 1649 __func__, vp); 1650 vdrop_recycle(vp); 1651 return (EBUSY); 1652 } 1653 /* 1654 * If we got this far, we need to acquire the interlock and see if 1655 * anyone picked up this vnode from another list. If not, we will 1656 * mark it with DOOMED via vgonel() so that anyone who does find it 1657 * will skip over it. 1658 */ 1659 VI_LOCK(vp); 1660 if (vp->v_usecount) { 1661 VOP_UNLOCK(vp); 1662 vdropl_recycle(vp); 1663 vn_finished_write(vnmp); 1664 CTR2(KTR_VFS, 1665 "%s: impossible to recycle, %p is already referenced", 1666 __func__, vp); 1667 return (EBUSY); 1668 } 1669 if (!VN_IS_DOOMED(vp)) { 1670 counter_u64_add(recycles_free_count, 1); 1671 vgonel(vp); 1672 } 1673 VOP_UNLOCK(vp); 1674 vdropl_recycle(vp); 1675 vn_finished_write(vnmp); 1676 return (0); 1677 } 1678 1679 /* 1680 * Allocate a new vnode. 1681 * 1682 * The operation never returns an error. Returning an error was disabled 1683 * in r145385 (dated 2005) with the following comment: 1684 * 1685 * XXX Not all VFS_VGET/ffs_vget callers check returns. 1686 * 1687 * Given the age of this commit (almost 15 years at the time of writing this 1688 * comment) restoring the ability to fail requires a significant audit of 1689 * all codepaths. 1690 * 1691 * The routine can try to free a vnode or stall for up to 1 second waiting for 1692 * vnlru to clear things up, but ultimately always performs a M_WAITOK allocation. 1693 */ 1694 static u_long vn_alloc_cyclecount; 1695 1696 static struct vnode * __noinline 1697 vn_alloc_hard(struct mount *mp) 1698 { 1699 u_long rnumvnodes, rfreevnodes; 1700 1701 mtx_lock(&vnode_list_mtx); 1702 rnumvnodes = atomic_load_long(&numvnodes); 1703 if (rnumvnodes + 1 < desiredvnodes) { 1704 vn_alloc_cyclecount = 0; 1705 goto alloc; 1706 } 1707 rfreevnodes = vnlru_read_freevnodes(); 1708 if (vn_alloc_cyclecount++ >= rfreevnodes) { 1709 vn_alloc_cyclecount = 0; 1710 vstir = 1; 1711 } 1712 /* 1713 * Grow the vnode cache if it will not be above its target max 1714 * after growing. Otherwise, if the free list is nonempty, try 1715 * to reclaim 1 item from it before growing the cache (possibly 1716 * above its target max if the reclamation failed or is delayed). 1717 * Otherwise, wait for some space. In all cases, schedule 1718 * vnlru_proc() if we are getting short of space. The watermarks 1719 * should be chosen so that we never wait or even reclaim from 1720 * the free list to below its target minimum. 1721 */ 1722 if (vnlru_free_locked(1) > 0) 1723 goto alloc; 1724 if (mp == NULL || (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 1725 /* 1726 * Wait for space for a new vnode. 1727 */ 1728 vnlru_kick(); 1729 msleep(&vnlruproc_sig, &vnode_list_mtx, PVFS, "vlruwk", hz); 1730 if (atomic_load_long(&numvnodes) + 1 > desiredvnodes && 1731 vnlru_read_freevnodes() > 1) 1732 vnlru_free_locked(1); 1733 } 1734 alloc: 1735 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 1736 if (vnlru_under(rnumvnodes, vlowat)) 1737 vnlru_kick(); 1738 mtx_unlock(&vnode_list_mtx); 1739 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 1740 } 1741 1742 static struct vnode * 1743 vn_alloc(struct mount *mp) 1744 { 1745 u_long rnumvnodes; 1746 1747 if (__predict_false(vn_alloc_cyclecount != 0)) 1748 return (vn_alloc_hard(mp)); 1749 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 1750 if (__predict_false(vnlru_under_unlocked(rnumvnodes, vlowat))) { 1751 atomic_subtract_long(&numvnodes, 1); 1752 return (vn_alloc_hard(mp)); 1753 } 1754 1755 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 1756 } 1757 1758 static void 1759 vn_free(struct vnode *vp) 1760 { 1761 1762 atomic_subtract_long(&numvnodes, 1); 1763 uma_zfree_smr(vnode_zone, vp); 1764 } 1765 1766 /* 1767 * Return the next vnode from the free list. 1768 */ 1769 int 1770 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops, 1771 struct vnode **vpp) 1772 { 1773 struct vnode *vp; 1774 struct thread *td; 1775 struct lock_object *lo; 1776 1777 CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag); 1778 1779 KASSERT(vops->registered, 1780 ("%s: not registered vector op %p\n", __func__, vops)); 1781 1782 td = curthread; 1783 if (td->td_vp_reserved != NULL) { 1784 vp = td->td_vp_reserved; 1785 td->td_vp_reserved = NULL; 1786 } else { 1787 vp = vn_alloc(mp); 1788 } 1789 counter_u64_add(vnodes_created, 1); 1790 /* 1791 * Locks are given the generic name "vnode" when created. 1792 * Follow the historic practice of using the filesystem 1793 * name when they allocated, e.g., "zfs", "ufs", "nfs, etc. 1794 * 1795 * Locks live in a witness group keyed on their name. Thus, 1796 * when a lock is renamed, it must also move from the witness 1797 * group of its old name to the witness group of its new name. 1798 * 1799 * The change only needs to be made when the vnode moves 1800 * from one filesystem type to another. We ensure that each 1801 * filesystem use a single static name pointer for its tag so 1802 * that we can compare pointers rather than doing a strcmp(). 1803 */ 1804 lo = &vp->v_vnlock->lock_object; 1805 #ifdef WITNESS 1806 if (lo->lo_name != tag) { 1807 #endif 1808 lo->lo_name = tag; 1809 #ifdef WITNESS 1810 WITNESS_DESTROY(lo); 1811 WITNESS_INIT(lo, tag); 1812 } 1813 #endif 1814 /* 1815 * By default, don't allow shared locks unless filesystems opt-in. 1816 */ 1817 vp->v_vnlock->lock_object.lo_flags |= LK_NOSHARE; 1818 /* 1819 * Finalize various vnode identity bits. 1820 */ 1821 KASSERT(vp->v_object == NULL, ("stale v_object %p", vp)); 1822 KASSERT(vp->v_lockf == NULL, ("stale v_lockf %p", vp)); 1823 KASSERT(vp->v_pollinfo == NULL, ("stale v_pollinfo %p", vp)); 1824 vp->v_type = VNON; 1825 vp->v_op = vops; 1826 vp->v_irflag = 0; 1827 v_init_counters(vp); 1828 vn_seqc_init(vp); 1829 vp->v_bufobj.bo_ops = &buf_ops_bio; 1830 #ifdef DIAGNOSTIC 1831 if (mp == NULL && vops != &dead_vnodeops) 1832 printf("NULL mp in getnewvnode(9), tag %s\n", tag); 1833 #endif 1834 #ifdef MAC 1835 mac_vnode_init(vp); 1836 if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0) 1837 mac_vnode_associate_singlelabel(mp, vp); 1838 #endif 1839 if (mp != NULL) { 1840 vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize; 1841 } 1842 1843 /* 1844 * For the filesystems which do not use vfs_hash_insert(), 1845 * still initialize v_hash to have vfs_hash_index() useful. 1846 * E.g., nullfs uses vfs_hash_index() on the lower vnode for 1847 * its own hashing. 1848 */ 1849 vp->v_hash = (uintptr_t)vp >> vnsz2log; 1850 1851 *vpp = vp; 1852 return (0); 1853 } 1854 1855 void 1856 getnewvnode_reserve(void) 1857 { 1858 struct thread *td; 1859 1860 td = curthread; 1861 MPASS(td->td_vp_reserved == NULL); 1862 td->td_vp_reserved = vn_alloc(NULL); 1863 } 1864 1865 void 1866 getnewvnode_drop_reserve(void) 1867 { 1868 struct thread *td; 1869 1870 td = curthread; 1871 if (td->td_vp_reserved != NULL) { 1872 vn_free(td->td_vp_reserved); 1873 td->td_vp_reserved = NULL; 1874 } 1875 } 1876 1877 static void __noinline 1878 freevnode(struct vnode *vp) 1879 { 1880 struct bufobj *bo; 1881 1882 /* 1883 * The vnode has been marked for destruction, so free it. 1884 * 1885 * The vnode will be returned to the zone where it will 1886 * normally remain until it is needed for another vnode. We 1887 * need to cleanup (or verify that the cleanup has already 1888 * been done) any residual data left from its current use 1889 * so as not to contaminate the freshly allocated vnode. 1890 */ 1891 CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); 1892 /* 1893 * Paired with vgone. 1894 */ 1895 vn_seqc_write_end_free(vp); 1896 1897 bo = &vp->v_bufobj; 1898 VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); 1899 VNPASS(vp->v_holdcnt == VHOLD_NO_SMR, vp); 1900 VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); 1901 VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); 1902 VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); 1903 VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); 1904 VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp, 1905 ("clean blk trie not empty")); 1906 VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0")); 1907 VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp, 1908 ("dirty blk trie not empty")); 1909 VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst")); 1910 VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src")); 1911 VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for ..")); 1912 VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp, 1913 ("Dangling rangelock waiters")); 1914 VNASSERT((vp->v_iflag & (VI_DOINGINACT | VI_OWEINACT)) == 0, vp, 1915 ("Leaked inactivation")); 1916 VI_UNLOCK(vp); 1917 #ifdef MAC 1918 mac_vnode_destroy(vp); 1919 #endif 1920 if (vp->v_pollinfo != NULL) { 1921 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1922 destroy_vpollinfo(vp->v_pollinfo); 1923 VOP_UNLOCK(vp); 1924 vp->v_pollinfo = NULL; 1925 } 1926 vp->v_mountedhere = NULL; 1927 vp->v_unpcb = NULL; 1928 vp->v_rdev = NULL; 1929 vp->v_fifoinfo = NULL; 1930 vp->v_iflag = 0; 1931 vp->v_vflag = 0; 1932 bo->bo_flag = 0; 1933 vn_free(vp); 1934 } 1935 1936 /* 1937 * Delete from old mount point vnode list, if on one. 1938 */ 1939 static void 1940 delmntque(struct vnode *vp) 1941 { 1942 struct mount *mp; 1943 1944 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 1945 1946 mp = vp->v_mount; 1947 MNT_ILOCK(mp); 1948 VI_LOCK(vp); 1949 vp->v_mount = NULL; 1950 VNASSERT(mp->mnt_nvnodelistsize > 0, vp, 1951 ("bad mount point vnode list size")); 1952 TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 1953 mp->mnt_nvnodelistsize--; 1954 MNT_REL(mp); 1955 MNT_IUNLOCK(mp); 1956 /* 1957 * The caller expects the interlock to be still held. 1958 */ 1959 ASSERT_VI_LOCKED(vp, __func__); 1960 } 1961 1962 static int 1963 insmntque1_int(struct vnode *vp, struct mount *mp, bool dtr) 1964 { 1965 1966 KASSERT(vp->v_mount == NULL, 1967 ("insmntque: vnode already on per mount vnode list")); 1968 VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)")); 1969 if ((mp->mnt_kern_flag & MNTK_UNLOCKED_INSMNTQUE) == 0) { 1970 ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp"); 1971 } else { 1972 KASSERT(!dtr, 1973 ("%s: can't have MNTK_UNLOCKED_INSMNTQUE and cleanup", 1974 __func__)); 1975 } 1976 1977 /* 1978 * We acquire the vnode interlock early to ensure that the 1979 * vnode cannot be recycled by another process releasing a 1980 * holdcnt on it before we get it on both the vnode list 1981 * and the active vnode list. The mount mutex protects only 1982 * manipulation of the vnode list and the vnode freelist 1983 * mutex protects only manipulation of the active vnode list. 1984 * Hence the need to hold the vnode interlock throughout. 1985 */ 1986 MNT_ILOCK(mp); 1987 VI_LOCK(vp); 1988 if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 && 1989 ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 || 1990 mp->mnt_nvnodelistsize == 0)) && 1991 (vp->v_vflag & VV_FORCEINSMQ) == 0) { 1992 VI_UNLOCK(vp); 1993 MNT_IUNLOCK(mp); 1994 if (dtr) { 1995 vp->v_data = NULL; 1996 vp->v_op = &dead_vnodeops; 1997 vgone(vp); 1998 vput(vp); 1999 } 2000 return (EBUSY); 2001 } 2002 vp->v_mount = mp; 2003 MNT_REF(mp); 2004 TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 2005 VNASSERT(mp->mnt_nvnodelistsize >= 0, vp, 2006 ("neg mount point vnode list size")); 2007 mp->mnt_nvnodelistsize++; 2008 VI_UNLOCK(vp); 2009 MNT_IUNLOCK(mp); 2010 return (0); 2011 } 2012 2013 /* 2014 * Insert into list of vnodes for the new mount point, if available. 2015 * insmntque() reclaims the vnode on insertion failure, insmntque1() 2016 * leaves handling of the vnode to the caller. 2017 */ 2018 int 2019 insmntque(struct vnode *vp, struct mount *mp) 2020 { 2021 return (insmntque1_int(vp, mp, true)); 2022 } 2023 2024 int 2025 insmntque1(struct vnode *vp, struct mount *mp) 2026 { 2027 return (insmntque1_int(vp, mp, false)); 2028 } 2029 2030 /* 2031 * Flush out and invalidate all buffers associated with a bufobj 2032 * Called with the underlying object locked. 2033 */ 2034 int 2035 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo) 2036 { 2037 int error; 2038 2039 BO_LOCK(bo); 2040 if (flags & V_SAVE) { 2041 error = bufobj_wwait(bo, slpflag, slptimeo); 2042 if (error) { 2043 BO_UNLOCK(bo); 2044 return (error); 2045 } 2046 if (bo->bo_dirty.bv_cnt > 0) { 2047 BO_UNLOCK(bo); 2048 do { 2049 error = BO_SYNC(bo, MNT_WAIT); 2050 } while (error == ERELOOKUP); 2051 if (error != 0) 2052 return (error); 2053 BO_LOCK(bo); 2054 if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0) { 2055 BO_UNLOCK(bo); 2056 return (EBUSY); 2057 } 2058 } 2059 } 2060 /* 2061 * If you alter this loop please notice that interlock is dropped and 2062 * reacquired in flushbuflist. Special care is needed to ensure that 2063 * no race conditions occur from this. 2064 */ 2065 do { 2066 error = flushbuflist(&bo->bo_clean, 2067 flags, bo, slpflag, slptimeo); 2068 if (error == 0 && !(flags & V_CLEANONLY)) 2069 error = flushbuflist(&bo->bo_dirty, 2070 flags, bo, slpflag, slptimeo); 2071 if (error != 0 && error != EAGAIN) { 2072 BO_UNLOCK(bo); 2073 return (error); 2074 } 2075 } while (error != 0); 2076 2077 /* 2078 * Wait for I/O to complete. XXX needs cleaning up. The vnode can 2079 * have write I/O in-progress but if there is a VM object then the 2080 * VM object can also have read-I/O in-progress. 2081 */ 2082 do { 2083 bufobj_wwait(bo, 0, 0); 2084 if ((flags & V_VMIO) == 0 && bo->bo_object != NULL) { 2085 BO_UNLOCK(bo); 2086 vm_object_pip_wait_unlocked(bo->bo_object, "bovlbx"); 2087 BO_LOCK(bo); 2088 } 2089 } while (bo->bo_numoutput > 0); 2090 BO_UNLOCK(bo); 2091 2092 /* 2093 * Destroy the copy in the VM cache, too. 2094 */ 2095 if (bo->bo_object != NULL && 2096 (flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0) { 2097 VM_OBJECT_WLOCK(bo->bo_object); 2098 vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ? 2099 OBJPR_CLEANONLY : 0); 2100 VM_OBJECT_WUNLOCK(bo->bo_object); 2101 } 2102 2103 #ifdef INVARIANTS 2104 BO_LOCK(bo); 2105 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO | 2106 V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 || 2107 bo->bo_clean.bv_cnt > 0)) 2108 panic("vinvalbuf: flush failed"); 2109 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0 && 2110 bo->bo_dirty.bv_cnt > 0) 2111 panic("vinvalbuf: flush dirty failed"); 2112 BO_UNLOCK(bo); 2113 #endif 2114 return (0); 2115 } 2116 2117 /* 2118 * Flush out and invalidate all buffers associated with a vnode. 2119 * Called with the underlying object locked. 2120 */ 2121 int 2122 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) 2123 { 2124 2125 CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags); 2126 ASSERT_VOP_LOCKED(vp, "vinvalbuf"); 2127 if (vp->v_object != NULL && vp->v_object->handle != vp) 2128 return (0); 2129 return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo)); 2130 } 2131 2132 /* 2133 * Flush out buffers on the specified list. 2134 * 2135 */ 2136 static int 2137 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, 2138 int slptimeo) 2139 { 2140 struct buf *bp, *nbp; 2141 int retval, error; 2142 daddr_t lblkno; 2143 b_xflags_t xflags; 2144 2145 ASSERT_BO_WLOCKED(bo); 2146 2147 retval = 0; 2148 TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) { 2149 /* 2150 * If we are flushing both V_NORMAL and V_ALT buffers then 2151 * do not skip any buffers. If we are flushing only V_NORMAL 2152 * buffers then skip buffers marked as BX_ALTDATA. If we are 2153 * flushing only V_ALT buffers then skip buffers not marked 2154 * as BX_ALTDATA. 2155 */ 2156 if (((flags & (V_NORMAL | V_ALT)) != (V_NORMAL | V_ALT)) && 2157 (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) || 2158 ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) { 2159 continue; 2160 } 2161 if (nbp != NULL) { 2162 lblkno = nbp->b_lblkno; 2163 xflags = nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN); 2164 } 2165 retval = EAGAIN; 2166 error = BUF_TIMELOCK(bp, 2167 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo), 2168 "flushbuf", slpflag, slptimeo); 2169 if (error) { 2170 BO_LOCK(bo); 2171 return (error != ENOLCK ? error : EAGAIN); 2172 } 2173 KASSERT(bp->b_bufobj == bo, 2174 ("bp %p wrong b_bufobj %p should be %p", 2175 bp, bp->b_bufobj, bo)); 2176 /* 2177 * XXX Since there are no node locks for NFS, I 2178 * believe there is a slight chance that a delayed 2179 * write will occur while sleeping just above, so 2180 * check for it. 2181 */ 2182 if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) && 2183 (flags & V_SAVE)) { 2184 bremfree(bp); 2185 bp->b_flags |= B_ASYNC; 2186 bwrite(bp); 2187 BO_LOCK(bo); 2188 return (EAGAIN); /* XXX: why not loop ? */ 2189 } 2190 bremfree(bp); 2191 bp->b_flags |= (B_INVAL | B_RELBUF); 2192 bp->b_flags &= ~B_ASYNC; 2193 brelse(bp); 2194 BO_LOCK(bo); 2195 if (nbp == NULL) 2196 break; 2197 nbp = gbincore(bo, lblkno); 2198 if (nbp == NULL || (nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) 2199 != xflags) 2200 break; /* nbp invalid */ 2201 } 2202 return (retval); 2203 } 2204 2205 int 2206 bnoreuselist(struct bufv *bufv, struct bufobj *bo, daddr_t startn, daddr_t endn) 2207 { 2208 struct buf *bp; 2209 int error; 2210 daddr_t lblkno; 2211 2212 ASSERT_BO_LOCKED(bo); 2213 2214 for (lblkno = startn;;) { 2215 again: 2216 bp = BUF_PCTRIE_LOOKUP_GE(&bufv->bv_root, lblkno); 2217 if (bp == NULL || bp->b_lblkno >= endn || 2218 bp->b_lblkno < startn) 2219 break; 2220 error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 2221 LK_INTERLOCK, BO_LOCKPTR(bo), "brlsfl", 0, 0); 2222 if (error != 0) { 2223 BO_RLOCK(bo); 2224 if (error == ENOLCK) 2225 goto again; 2226 return (error); 2227 } 2228 KASSERT(bp->b_bufobj == bo, 2229 ("bp %p wrong b_bufobj %p should be %p", 2230 bp, bp->b_bufobj, bo)); 2231 lblkno = bp->b_lblkno + 1; 2232 if ((bp->b_flags & B_MANAGED) == 0) 2233 bremfree(bp); 2234 bp->b_flags |= B_RELBUF; 2235 /* 2236 * In the VMIO case, use the B_NOREUSE flag to hint that the 2237 * pages backing each buffer in the range are unlikely to be 2238 * reused. Dirty buffers will have the hint applied once 2239 * they've been written. 2240 */ 2241 if ((bp->b_flags & B_VMIO) != 0) 2242 bp->b_flags |= B_NOREUSE; 2243 brelse(bp); 2244 BO_RLOCK(bo); 2245 } 2246 return (0); 2247 } 2248 2249 /* 2250 * Truncate a file's buffer and pages to a specified length. This 2251 * is in lieu of the old vinvalbuf mechanism, which performed unneeded 2252 * sync activity. 2253 */ 2254 int 2255 vtruncbuf(struct vnode *vp, off_t length, int blksize) 2256 { 2257 struct buf *bp, *nbp; 2258 struct bufobj *bo; 2259 daddr_t startlbn; 2260 2261 CTR4(KTR_VFS, "%s: vp %p with block %d:%ju", __func__, 2262 vp, blksize, (uintmax_t)length); 2263 2264 /* 2265 * Round up to the *next* lbn. 2266 */ 2267 startlbn = howmany(length, blksize); 2268 2269 ASSERT_VOP_LOCKED(vp, "vtruncbuf"); 2270 2271 bo = &vp->v_bufobj; 2272 restart_unlocked: 2273 BO_LOCK(bo); 2274 2275 while (v_inval_buf_range_locked(vp, bo, startlbn, INT64_MAX) == EAGAIN) 2276 ; 2277 2278 if (length > 0) { 2279 restartsync: 2280 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2281 if (bp->b_lblkno > 0) 2282 continue; 2283 /* 2284 * Since we hold the vnode lock this should only 2285 * fail if we're racing with the buf daemon. 2286 */ 2287 if (BUF_LOCK(bp, 2288 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2289 BO_LOCKPTR(bo)) == ENOLCK) 2290 goto restart_unlocked; 2291 2292 VNASSERT((bp->b_flags & B_DELWRI), vp, 2293 ("buf(%p) on dirty queue without DELWRI", bp)); 2294 2295 bremfree(bp); 2296 bawrite(bp); 2297 BO_LOCK(bo); 2298 goto restartsync; 2299 } 2300 } 2301 2302 bufobj_wwait(bo, 0, 0); 2303 BO_UNLOCK(bo); 2304 vnode_pager_setsize(vp, length); 2305 2306 return (0); 2307 } 2308 2309 /* 2310 * Invalidate the cached pages of a file's buffer within the range of block 2311 * numbers [startlbn, endlbn). 2312 */ 2313 void 2314 v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn, 2315 int blksize) 2316 { 2317 struct bufobj *bo; 2318 off_t start, end; 2319 2320 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range"); 2321 2322 start = blksize * startlbn; 2323 end = blksize * endlbn; 2324 2325 bo = &vp->v_bufobj; 2326 BO_LOCK(bo); 2327 MPASS(blksize == bo->bo_bsize); 2328 2329 while (v_inval_buf_range_locked(vp, bo, startlbn, endlbn) == EAGAIN) 2330 ; 2331 2332 BO_UNLOCK(bo); 2333 vn_pages_remove(vp, OFF_TO_IDX(start), OFF_TO_IDX(end + PAGE_SIZE - 1)); 2334 } 2335 2336 static int 2337 v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 2338 daddr_t startlbn, daddr_t endlbn) 2339 { 2340 struct buf *bp, *nbp; 2341 bool anyfreed; 2342 2343 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range_locked"); 2344 ASSERT_BO_LOCKED(bo); 2345 2346 do { 2347 anyfreed = false; 2348 TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) { 2349 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn) 2350 continue; 2351 if (BUF_LOCK(bp, 2352 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2353 BO_LOCKPTR(bo)) == ENOLCK) { 2354 BO_LOCK(bo); 2355 return (EAGAIN); 2356 } 2357 2358 bremfree(bp); 2359 bp->b_flags |= B_INVAL | B_RELBUF; 2360 bp->b_flags &= ~B_ASYNC; 2361 brelse(bp); 2362 anyfreed = true; 2363 2364 BO_LOCK(bo); 2365 if (nbp != NULL && 2366 (((nbp->b_xflags & BX_VNCLEAN) == 0) || 2367 nbp->b_vp != vp || 2368 (nbp->b_flags & B_DELWRI) != 0)) 2369 return (EAGAIN); 2370 } 2371 2372 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2373 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn) 2374 continue; 2375 if (BUF_LOCK(bp, 2376 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2377 BO_LOCKPTR(bo)) == ENOLCK) { 2378 BO_LOCK(bo); 2379 return (EAGAIN); 2380 } 2381 bremfree(bp); 2382 bp->b_flags |= B_INVAL | B_RELBUF; 2383 bp->b_flags &= ~B_ASYNC; 2384 brelse(bp); 2385 anyfreed = true; 2386 2387 BO_LOCK(bo); 2388 if (nbp != NULL && 2389 (((nbp->b_xflags & BX_VNDIRTY) == 0) || 2390 (nbp->b_vp != vp) || 2391 (nbp->b_flags & B_DELWRI) == 0)) 2392 return (EAGAIN); 2393 } 2394 } while (anyfreed); 2395 return (0); 2396 } 2397 2398 static void 2399 buf_vlist_remove(struct buf *bp) 2400 { 2401 struct bufv *bv; 2402 b_xflags_t flags; 2403 2404 flags = bp->b_xflags; 2405 2406 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp)); 2407 ASSERT_BO_WLOCKED(bp->b_bufobj); 2408 KASSERT((flags & (BX_VNDIRTY | BX_VNCLEAN)) != 0 && 2409 (flags & (BX_VNDIRTY | BX_VNCLEAN)) != (BX_VNDIRTY | BX_VNCLEAN), 2410 ("%s: buffer %p has invalid queue state", __func__, bp)); 2411 2412 if ((flags & BX_VNDIRTY) != 0) 2413 bv = &bp->b_bufobj->bo_dirty; 2414 else 2415 bv = &bp->b_bufobj->bo_clean; 2416 BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno); 2417 TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs); 2418 bv->bv_cnt--; 2419 bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN); 2420 } 2421 2422 /* 2423 * Add the buffer to the sorted clean or dirty block list. 2424 * 2425 * NOTE: xflags is passed as a constant, optimizing this inline function! 2426 */ 2427 static void 2428 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags) 2429 { 2430 struct bufv *bv; 2431 struct buf *n; 2432 int error; 2433 2434 ASSERT_BO_WLOCKED(bo); 2435 KASSERT((bo->bo_flag & BO_NOBUFS) == 0, 2436 ("buf_vlist_add: bo %p does not allow bufs", bo)); 2437 KASSERT((xflags & BX_VNDIRTY) == 0 || (bo->bo_flag & BO_DEAD) == 0, 2438 ("dead bo %p", bo)); 2439 KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, 2440 ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags)); 2441 bp->b_xflags |= xflags; 2442 if (xflags & BX_VNDIRTY) 2443 bv = &bo->bo_dirty; 2444 else 2445 bv = &bo->bo_clean; 2446 2447 /* 2448 * Keep the list ordered. Optimize empty list insertion. Assume 2449 * we tend to grow at the tail so lookup_le should usually be cheaper 2450 * than _ge. 2451 */ 2452 if (bv->bv_cnt == 0 || 2453 bp->b_lblkno > TAILQ_LAST(&bv->bv_hd, buflists)->b_lblkno) 2454 TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs); 2455 else if ((n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, bp->b_lblkno)) == NULL) 2456 TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs); 2457 else 2458 TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs); 2459 error = BUF_PCTRIE_INSERT(&bv->bv_root, bp); 2460 if (error) 2461 panic("buf_vlist_add: Preallocated nodes insufficient."); 2462 bv->bv_cnt++; 2463 } 2464 2465 /* 2466 * Look up a buffer using the buffer tries. 2467 */ 2468 struct buf * 2469 gbincore(struct bufobj *bo, daddr_t lblkno) 2470 { 2471 struct buf *bp; 2472 2473 ASSERT_BO_LOCKED(bo); 2474 bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno); 2475 if (bp != NULL) 2476 return (bp); 2477 return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno)); 2478 } 2479 2480 /* 2481 * Look up a buf using the buffer tries, without the bufobj lock. This relies 2482 * on SMR for safe lookup, and bufs being in a no-free zone to provide type 2483 * stability of the result. Like other lockless lookups, the found buf may 2484 * already be invalid by the time this function returns. 2485 */ 2486 struct buf * 2487 gbincore_unlocked(struct bufobj *bo, daddr_t lblkno) 2488 { 2489 struct buf *bp; 2490 2491 ASSERT_BO_UNLOCKED(bo); 2492 bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno); 2493 if (bp != NULL) 2494 return (bp); 2495 return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno)); 2496 } 2497 2498 /* 2499 * Associate a buffer with a vnode. 2500 */ 2501 void 2502 bgetvp(struct vnode *vp, struct buf *bp) 2503 { 2504 struct bufobj *bo; 2505 2506 bo = &vp->v_bufobj; 2507 ASSERT_BO_WLOCKED(bo); 2508 VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free")); 2509 2510 CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags); 2511 VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp, 2512 ("bgetvp: bp already attached! %p", bp)); 2513 2514 vhold(vp); 2515 bp->b_vp = vp; 2516 bp->b_bufobj = bo; 2517 /* 2518 * Insert onto list for new vnode. 2519 */ 2520 buf_vlist_add(bp, bo, BX_VNCLEAN); 2521 } 2522 2523 /* 2524 * Disassociate a buffer from a vnode. 2525 */ 2526 void 2527 brelvp(struct buf *bp) 2528 { 2529 struct bufobj *bo; 2530 struct vnode *vp; 2531 2532 CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags); 2533 KASSERT(bp->b_vp != NULL, ("brelvp: NULL")); 2534 2535 /* 2536 * Delete from old vnode list, if on one. 2537 */ 2538 vp = bp->b_vp; /* XXX */ 2539 bo = bp->b_bufobj; 2540 BO_LOCK(bo); 2541 buf_vlist_remove(bp); 2542 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2543 bo->bo_flag &= ~BO_ONWORKLST; 2544 mtx_lock(&sync_mtx); 2545 LIST_REMOVE(bo, bo_synclist); 2546 syncer_worklist_len--; 2547 mtx_unlock(&sync_mtx); 2548 } 2549 bp->b_vp = NULL; 2550 bp->b_bufobj = NULL; 2551 BO_UNLOCK(bo); 2552 vdrop(vp); 2553 } 2554 2555 /* 2556 * Add an item to the syncer work queue. 2557 */ 2558 static void 2559 vn_syncer_add_to_worklist(struct bufobj *bo, int delay) 2560 { 2561 int slot; 2562 2563 ASSERT_BO_WLOCKED(bo); 2564 2565 mtx_lock(&sync_mtx); 2566 if (bo->bo_flag & BO_ONWORKLST) 2567 LIST_REMOVE(bo, bo_synclist); 2568 else { 2569 bo->bo_flag |= BO_ONWORKLST; 2570 syncer_worklist_len++; 2571 } 2572 2573 if (delay > syncer_maxdelay - 2) 2574 delay = syncer_maxdelay - 2; 2575 slot = (syncer_delayno + delay) & syncer_mask; 2576 2577 LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist); 2578 mtx_unlock(&sync_mtx); 2579 } 2580 2581 static int 2582 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS) 2583 { 2584 int error, len; 2585 2586 mtx_lock(&sync_mtx); 2587 len = syncer_worklist_len - sync_vnode_count; 2588 mtx_unlock(&sync_mtx); 2589 error = SYSCTL_OUT(req, &len, sizeof(len)); 2590 return (error); 2591 } 2592 2593 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, 2594 CTLTYPE_INT | CTLFLAG_MPSAFE| CTLFLAG_RD, NULL, 0, 2595 sysctl_vfs_worklist_len, "I", "Syncer thread worklist length"); 2596 2597 static struct proc *updateproc; 2598 static void sched_sync(void); 2599 static struct kproc_desc up_kp = { 2600 "syncer", 2601 sched_sync, 2602 &updateproc 2603 }; 2604 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp); 2605 2606 static int 2607 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td) 2608 { 2609 struct vnode *vp; 2610 struct mount *mp; 2611 2612 *bo = LIST_FIRST(slp); 2613 if (*bo == NULL) 2614 return (0); 2615 vp = bo2vnode(*bo); 2616 if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0) 2617 return (1); 2618 /* 2619 * We use vhold in case the vnode does not 2620 * successfully sync. vhold prevents the vnode from 2621 * going away when we unlock the sync_mtx so that 2622 * we can acquire the vnode interlock. 2623 */ 2624 vholdl(vp); 2625 mtx_unlock(&sync_mtx); 2626 VI_UNLOCK(vp); 2627 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 2628 vdrop(vp); 2629 mtx_lock(&sync_mtx); 2630 return (*bo == LIST_FIRST(slp)); 2631 } 2632 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2633 (void) VOP_FSYNC(vp, MNT_LAZY, td); 2634 VOP_UNLOCK(vp); 2635 vn_finished_write(mp); 2636 BO_LOCK(*bo); 2637 if (((*bo)->bo_flag & BO_ONWORKLST) != 0) { 2638 /* 2639 * Put us back on the worklist. The worklist 2640 * routine will remove us from our current 2641 * position and then add us back in at a later 2642 * position. 2643 */ 2644 vn_syncer_add_to_worklist(*bo, syncdelay); 2645 } 2646 BO_UNLOCK(*bo); 2647 vdrop(vp); 2648 mtx_lock(&sync_mtx); 2649 return (0); 2650 } 2651 2652 static int first_printf = 1; 2653 2654 /* 2655 * System filesystem synchronizer daemon. 2656 */ 2657 static void 2658 sched_sync(void) 2659 { 2660 struct synclist *next, *slp; 2661 struct bufobj *bo; 2662 long starttime; 2663 struct thread *td = curthread; 2664 int last_work_seen; 2665 int net_worklist_len; 2666 int syncer_final_iter; 2667 int error; 2668 2669 last_work_seen = 0; 2670 syncer_final_iter = 0; 2671 syncer_state = SYNCER_RUNNING; 2672 starttime = time_uptime; 2673 td->td_pflags |= TDP_NORUNNINGBUF; 2674 2675 EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc, 2676 SHUTDOWN_PRI_LAST); 2677 2678 mtx_lock(&sync_mtx); 2679 for (;;) { 2680 if (syncer_state == SYNCER_FINAL_DELAY && 2681 syncer_final_iter == 0) { 2682 mtx_unlock(&sync_mtx); 2683 kproc_suspend_check(td->td_proc); 2684 mtx_lock(&sync_mtx); 2685 } 2686 net_worklist_len = syncer_worklist_len - sync_vnode_count; 2687 if (syncer_state != SYNCER_RUNNING && 2688 starttime != time_uptime) { 2689 if (first_printf) { 2690 printf("\nSyncing disks, vnodes remaining... "); 2691 first_printf = 0; 2692 } 2693 printf("%d ", net_worklist_len); 2694 } 2695 starttime = time_uptime; 2696 2697 /* 2698 * Push files whose dirty time has expired. Be careful 2699 * of interrupt race on slp queue. 2700 * 2701 * Skip over empty worklist slots when shutting down. 2702 */ 2703 do { 2704 slp = &syncer_workitem_pending[syncer_delayno]; 2705 syncer_delayno += 1; 2706 if (syncer_delayno == syncer_maxdelay) 2707 syncer_delayno = 0; 2708 next = &syncer_workitem_pending[syncer_delayno]; 2709 /* 2710 * If the worklist has wrapped since the 2711 * it was emptied of all but syncer vnodes, 2712 * switch to the FINAL_DELAY state and run 2713 * for one more second. 2714 */ 2715 if (syncer_state == SYNCER_SHUTTING_DOWN && 2716 net_worklist_len == 0 && 2717 last_work_seen == syncer_delayno) { 2718 syncer_state = SYNCER_FINAL_DELAY; 2719 syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP; 2720 } 2721 } while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) && 2722 syncer_worklist_len > 0); 2723 2724 /* 2725 * Keep track of the last time there was anything 2726 * on the worklist other than syncer vnodes. 2727 * Return to the SHUTTING_DOWN state if any 2728 * new work appears. 2729 */ 2730 if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING) 2731 last_work_seen = syncer_delayno; 2732 if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY) 2733 syncer_state = SYNCER_SHUTTING_DOWN; 2734 while (!LIST_EMPTY(slp)) { 2735 error = sync_vnode(slp, &bo, td); 2736 if (error == 1) { 2737 LIST_REMOVE(bo, bo_synclist); 2738 LIST_INSERT_HEAD(next, bo, bo_synclist); 2739 continue; 2740 } 2741 2742 if (first_printf == 0) { 2743 /* 2744 * Drop the sync mutex, because some watchdog 2745 * drivers need to sleep while patting 2746 */ 2747 mtx_unlock(&sync_mtx); 2748 wdog_kern_pat(WD_LASTVAL); 2749 mtx_lock(&sync_mtx); 2750 } 2751 } 2752 if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0) 2753 syncer_final_iter--; 2754 /* 2755 * The variable rushjob allows the kernel to speed up the 2756 * processing of the filesystem syncer process. A rushjob 2757 * value of N tells the filesystem syncer to process the next 2758 * N seconds worth of work on its queue ASAP. Currently rushjob 2759 * is used by the soft update code to speed up the filesystem 2760 * syncer process when the incore state is getting so far 2761 * ahead of the disk that the kernel memory pool is being 2762 * threatened with exhaustion. 2763 */ 2764 if (rushjob > 0) { 2765 rushjob -= 1; 2766 continue; 2767 } 2768 /* 2769 * Just sleep for a short period of time between 2770 * iterations when shutting down to allow some I/O 2771 * to happen. 2772 * 2773 * If it has taken us less than a second to process the 2774 * current work, then wait. Otherwise start right over 2775 * again. We can still lose time if any single round 2776 * takes more than two seconds, but it does not really 2777 * matter as we are just trying to generally pace the 2778 * filesystem activity. 2779 */ 2780 if (syncer_state != SYNCER_RUNNING || 2781 time_uptime == starttime) { 2782 thread_lock(td); 2783 sched_prio(td, PPAUSE); 2784 thread_unlock(td); 2785 } 2786 if (syncer_state != SYNCER_RUNNING) 2787 cv_timedwait(&sync_wakeup, &sync_mtx, 2788 hz / SYNCER_SHUTDOWN_SPEEDUP); 2789 else if (time_uptime == starttime) 2790 cv_timedwait(&sync_wakeup, &sync_mtx, hz); 2791 } 2792 } 2793 2794 /* 2795 * Request the syncer daemon to speed up its work. 2796 * We never push it to speed up more than half of its 2797 * normal turn time, otherwise it could take over the cpu. 2798 */ 2799 int 2800 speedup_syncer(void) 2801 { 2802 int ret = 0; 2803 2804 mtx_lock(&sync_mtx); 2805 if (rushjob < syncdelay / 2) { 2806 rushjob += 1; 2807 stat_rush_requests += 1; 2808 ret = 1; 2809 } 2810 mtx_unlock(&sync_mtx); 2811 cv_broadcast(&sync_wakeup); 2812 return (ret); 2813 } 2814 2815 /* 2816 * Tell the syncer to speed up its work and run though its work 2817 * list several times, then tell it to shut down. 2818 */ 2819 static void 2820 syncer_shutdown(void *arg, int howto) 2821 { 2822 2823 if (howto & RB_NOSYNC) 2824 return; 2825 mtx_lock(&sync_mtx); 2826 syncer_state = SYNCER_SHUTTING_DOWN; 2827 rushjob = 0; 2828 mtx_unlock(&sync_mtx); 2829 cv_broadcast(&sync_wakeup); 2830 kproc_shutdown(arg, howto); 2831 } 2832 2833 void 2834 syncer_suspend(void) 2835 { 2836 2837 syncer_shutdown(updateproc, 0); 2838 } 2839 2840 void 2841 syncer_resume(void) 2842 { 2843 2844 mtx_lock(&sync_mtx); 2845 first_printf = 1; 2846 syncer_state = SYNCER_RUNNING; 2847 mtx_unlock(&sync_mtx); 2848 cv_broadcast(&sync_wakeup); 2849 kproc_resume(updateproc); 2850 } 2851 2852 /* 2853 * Move the buffer between the clean and dirty lists of its vnode. 2854 */ 2855 void 2856 reassignbuf(struct buf *bp) 2857 { 2858 struct vnode *vp; 2859 struct bufobj *bo; 2860 int delay; 2861 #ifdef INVARIANTS 2862 struct bufv *bv; 2863 #endif 2864 2865 vp = bp->b_vp; 2866 bo = bp->b_bufobj; 2867 2868 KASSERT((bp->b_flags & B_PAGING) == 0, 2869 ("%s: cannot reassign paging buffer %p", __func__, bp)); 2870 2871 CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X", 2872 bp, bp->b_vp, bp->b_flags); 2873 2874 BO_LOCK(bo); 2875 buf_vlist_remove(bp); 2876 2877 /* 2878 * If dirty, put on list of dirty buffers; otherwise insert onto list 2879 * of clean buffers. 2880 */ 2881 if (bp->b_flags & B_DELWRI) { 2882 if ((bo->bo_flag & BO_ONWORKLST) == 0) { 2883 switch (vp->v_type) { 2884 case VDIR: 2885 delay = dirdelay; 2886 break; 2887 case VCHR: 2888 delay = metadelay; 2889 break; 2890 default: 2891 delay = filedelay; 2892 } 2893 vn_syncer_add_to_worklist(bo, delay); 2894 } 2895 buf_vlist_add(bp, bo, BX_VNDIRTY); 2896 } else { 2897 buf_vlist_add(bp, bo, BX_VNCLEAN); 2898 2899 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2900 mtx_lock(&sync_mtx); 2901 LIST_REMOVE(bo, bo_synclist); 2902 syncer_worklist_len--; 2903 mtx_unlock(&sync_mtx); 2904 bo->bo_flag &= ~BO_ONWORKLST; 2905 } 2906 } 2907 #ifdef INVARIANTS 2908 bv = &bo->bo_clean; 2909 bp = TAILQ_FIRST(&bv->bv_hd); 2910 KASSERT(bp == NULL || bp->b_bufobj == bo, 2911 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2912 bp = TAILQ_LAST(&bv->bv_hd, buflists); 2913 KASSERT(bp == NULL || bp->b_bufobj == bo, 2914 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2915 bv = &bo->bo_dirty; 2916 bp = TAILQ_FIRST(&bv->bv_hd); 2917 KASSERT(bp == NULL || bp->b_bufobj == bo, 2918 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2919 bp = TAILQ_LAST(&bv->bv_hd, buflists); 2920 KASSERT(bp == NULL || bp->b_bufobj == bo, 2921 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2922 #endif 2923 BO_UNLOCK(bo); 2924 } 2925 2926 static void 2927 v_init_counters(struct vnode *vp) 2928 { 2929 2930 VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0, 2931 vp, ("%s called for an initialized vnode", __FUNCTION__)); 2932 ASSERT_VI_UNLOCKED(vp, __FUNCTION__); 2933 2934 refcount_init(&vp->v_holdcnt, 1); 2935 refcount_init(&vp->v_usecount, 1); 2936 } 2937 2938 /* 2939 * Grab a particular vnode from the free list, increment its 2940 * reference count and lock it. VIRF_DOOMED is set if the vnode 2941 * is being destroyed. Only callers who specify LK_RETRY will 2942 * see doomed vnodes. If inactive processing was delayed in 2943 * vput try to do it here. 2944 * 2945 * usecount is manipulated using atomics without holding any locks. 2946 * 2947 * holdcnt can be manipulated using atomics without holding any locks, 2948 * except when transitioning 1<->0, in which case the interlock is held. 2949 * 2950 * Consumers which don't guarantee liveness of the vnode can use SMR to 2951 * try to get a reference. Note this operation can fail since the vnode 2952 * may be awaiting getting freed by the time they get to it. 2953 */ 2954 enum vgetstate 2955 vget_prep_smr(struct vnode *vp) 2956 { 2957 enum vgetstate vs; 2958 2959 VFS_SMR_ASSERT_ENTERED(); 2960 2961 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 2962 vs = VGET_USECOUNT; 2963 } else { 2964 if (vhold_smr(vp)) 2965 vs = VGET_HOLDCNT; 2966 else 2967 vs = VGET_NONE; 2968 } 2969 return (vs); 2970 } 2971 2972 enum vgetstate 2973 vget_prep(struct vnode *vp) 2974 { 2975 enum vgetstate vs; 2976 2977 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 2978 vs = VGET_USECOUNT; 2979 } else { 2980 vhold(vp); 2981 vs = VGET_HOLDCNT; 2982 } 2983 return (vs); 2984 } 2985 2986 void 2987 vget_abort(struct vnode *vp, enum vgetstate vs) 2988 { 2989 2990 switch (vs) { 2991 case VGET_USECOUNT: 2992 vrele(vp); 2993 break; 2994 case VGET_HOLDCNT: 2995 vdrop(vp); 2996 break; 2997 default: 2998 __assert_unreachable(); 2999 } 3000 } 3001 3002 int 3003 vget(struct vnode *vp, int flags) 3004 { 3005 enum vgetstate vs; 3006 3007 vs = vget_prep(vp); 3008 return (vget_finish(vp, flags, vs)); 3009 } 3010 3011 int 3012 vget_finish(struct vnode *vp, int flags, enum vgetstate vs) 3013 { 3014 int error; 3015 3016 if ((flags & LK_INTERLOCK) != 0) 3017 ASSERT_VI_LOCKED(vp, __func__); 3018 else 3019 ASSERT_VI_UNLOCKED(vp, __func__); 3020 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3021 VNPASS(vp->v_holdcnt > 0, vp); 3022 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3023 3024 error = vn_lock(vp, flags); 3025 if (__predict_false(error != 0)) { 3026 vget_abort(vp, vs); 3027 CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__, 3028 vp); 3029 return (error); 3030 } 3031 3032 vget_finish_ref(vp, vs); 3033 return (0); 3034 } 3035 3036 void 3037 vget_finish_ref(struct vnode *vp, enum vgetstate vs) 3038 { 3039 int old; 3040 3041 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3042 VNPASS(vp->v_holdcnt > 0, vp); 3043 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3044 3045 if (vs == VGET_USECOUNT) 3046 return; 3047 3048 /* 3049 * We hold the vnode. If the usecount is 0 it will be utilized to keep 3050 * the vnode around. Otherwise someone else lended their hold count and 3051 * we have to drop ours. 3052 */ 3053 old = atomic_fetchadd_int(&vp->v_usecount, 1); 3054 VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old)); 3055 if (old != 0) { 3056 #ifdef INVARIANTS 3057 old = atomic_fetchadd_int(&vp->v_holdcnt, -1); 3058 VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); 3059 #else 3060 refcount_release(&vp->v_holdcnt); 3061 #endif 3062 } 3063 } 3064 3065 void 3066 vref(struct vnode *vp) 3067 { 3068 enum vgetstate vs; 3069 3070 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3071 vs = vget_prep(vp); 3072 vget_finish_ref(vp, vs); 3073 } 3074 3075 void 3076 vrefact(struct vnode *vp) 3077 { 3078 3079 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3080 #ifdef INVARIANTS 3081 int old = atomic_fetchadd_int(&vp->v_usecount, 1); 3082 VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old)); 3083 #else 3084 refcount_acquire(&vp->v_usecount); 3085 #endif 3086 } 3087 3088 void 3089 vlazy(struct vnode *vp) 3090 { 3091 struct mount *mp; 3092 3093 VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); 3094 3095 if ((vp->v_mflag & VMP_LAZYLIST) != 0) 3096 return; 3097 /* 3098 * We may get here for inactive routines after the vnode got doomed. 3099 */ 3100 if (VN_IS_DOOMED(vp)) 3101 return; 3102 mp = vp->v_mount; 3103 mtx_lock(&mp->mnt_listmtx); 3104 if ((vp->v_mflag & VMP_LAZYLIST) == 0) { 3105 vp->v_mflag |= VMP_LAZYLIST; 3106 TAILQ_INSERT_TAIL(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3107 mp->mnt_lazyvnodelistsize++; 3108 } 3109 mtx_unlock(&mp->mnt_listmtx); 3110 } 3111 3112 static void 3113 vunlazy(struct vnode *vp) 3114 { 3115 struct mount *mp; 3116 3117 ASSERT_VI_LOCKED(vp, __func__); 3118 VNPASS(!VN_IS_DOOMED(vp), vp); 3119 3120 mp = vp->v_mount; 3121 mtx_lock(&mp->mnt_listmtx); 3122 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3123 /* 3124 * Don't remove the vnode from the lazy list if another thread 3125 * has increased the hold count. It may have re-enqueued the 3126 * vnode to the lazy list and is now responsible for its 3127 * removal. 3128 */ 3129 if (vp->v_holdcnt == 0) { 3130 vp->v_mflag &= ~VMP_LAZYLIST; 3131 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3132 mp->mnt_lazyvnodelistsize--; 3133 } 3134 mtx_unlock(&mp->mnt_listmtx); 3135 } 3136 3137 /* 3138 * This routine is only meant to be called from vgonel prior to dooming 3139 * the vnode. 3140 */ 3141 static void 3142 vunlazy_gone(struct vnode *vp) 3143 { 3144 struct mount *mp; 3145 3146 ASSERT_VOP_ELOCKED(vp, __func__); 3147 ASSERT_VI_LOCKED(vp, __func__); 3148 VNPASS(!VN_IS_DOOMED(vp), vp); 3149 3150 if (vp->v_mflag & VMP_LAZYLIST) { 3151 mp = vp->v_mount; 3152 mtx_lock(&mp->mnt_listmtx); 3153 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3154 vp->v_mflag &= ~VMP_LAZYLIST; 3155 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3156 mp->mnt_lazyvnodelistsize--; 3157 mtx_unlock(&mp->mnt_listmtx); 3158 } 3159 } 3160 3161 static void 3162 vdefer_inactive(struct vnode *vp) 3163 { 3164 3165 ASSERT_VI_LOCKED(vp, __func__); 3166 VNASSERT(vp->v_holdcnt > 0, vp, 3167 ("%s: vnode without hold count", __func__)); 3168 if (VN_IS_DOOMED(vp)) { 3169 vdropl(vp); 3170 return; 3171 } 3172 if (vp->v_iflag & VI_DEFINACT) { 3173 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 3174 vdropl(vp); 3175 return; 3176 } 3177 if (vp->v_usecount > 0) { 3178 vp->v_iflag &= ~VI_OWEINACT; 3179 vdropl(vp); 3180 return; 3181 } 3182 vlazy(vp); 3183 vp->v_iflag |= VI_DEFINACT; 3184 VI_UNLOCK(vp); 3185 counter_u64_add(deferred_inact, 1); 3186 } 3187 3188 static void 3189 vdefer_inactive_unlocked(struct vnode *vp) 3190 { 3191 3192 VI_LOCK(vp); 3193 if ((vp->v_iflag & VI_OWEINACT) == 0) { 3194 vdropl(vp); 3195 return; 3196 } 3197 vdefer_inactive(vp); 3198 } 3199 3200 enum vput_op { VRELE, VPUT, VUNREF }; 3201 3202 /* 3203 * Handle ->v_usecount transitioning to 0. 3204 * 3205 * By releasing the last usecount we take ownership of the hold count which 3206 * provides liveness of the vnode, meaning we have to vdrop. 3207 * 3208 * For all vnodes we may need to perform inactive processing. It requires an 3209 * exclusive lock on the vnode, while it is legal to call here with only a 3210 * shared lock (or no locks). If locking the vnode in an expected manner fails, 3211 * inactive processing gets deferred to the syncer. 3212 * 3213 * XXX Some filesystems pass in an exclusively locked vnode and strongly depend 3214 * on the lock being held all the way until VOP_INACTIVE. This in particular 3215 * happens with UFS which adds half-constructed vnodes to the hash, where they 3216 * can be found by other code. 3217 */ 3218 static void 3219 vput_final(struct vnode *vp, enum vput_op func) 3220 { 3221 int error; 3222 bool want_unlock; 3223 3224 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3225 VNPASS(vp->v_holdcnt > 0, vp); 3226 3227 VI_LOCK(vp); 3228 3229 /* 3230 * By the time we got here someone else might have transitioned 3231 * the count back to > 0. 3232 */ 3233 if (vp->v_usecount > 0) 3234 goto out; 3235 3236 /* 3237 * If the vnode is doomed vgone already performed inactive processing 3238 * (if needed). 3239 */ 3240 if (VN_IS_DOOMED(vp)) 3241 goto out; 3242 3243 if (__predict_true(VOP_NEED_INACTIVE(vp) == 0)) 3244 goto out; 3245 3246 if (vp->v_iflag & VI_DOINGINACT) 3247 goto out; 3248 3249 /* 3250 * Locking operations here will drop the interlock and possibly the 3251 * vnode lock, opening a window where the vnode can get doomed all the 3252 * while ->v_usecount is 0. Set VI_OWEINACT to let vgone know to 3253 * perform inactive. 3254 */ 3255 vp->v_iflag |= VI_OWEINACT; 3256 want_unlock = false; 3257 error = 0; 3258 switch (func) { 3259 case VRELE: 3260 switch (VOP_ISLOCKED(vp)) { 3261 case LK_EXCLUSIVE: 3262 break; 3263 case LK_EXCLOTHER: 3264 case 0: 3265 want_unlock = true; 3266 error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); 3267 VI_LOCK(vp); 3268 break; 3269 default: 3270 /* 3271 * The lock has at least one sharer, but we have no way 3272 * to conclude whether this is us. Play it safe and 3273 * defer processing. 3274 */ 3275 error = EAGAIN; 3276 break; 3277 } 3278 break; 3279 case VPUT: 3280 want_unlock = true; 3281 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3282 error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK | 3283 LK_NOWAIT); 3284 VI_LOCK(vp); 3285 } 3286 break; 3287 case VUNREF: 3288 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3289 error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK); 3290 VI_LOCK(vp); 3291 } 3292 break; 3293 } 3294 if (error == 0) { 3295 if (func == VUNREF) { 3296 VNASSERT((vp->v_vflag & VV_UNREF) == 0, vp, 3297 ("recursive vunref")); 3298 vp->v_vflag |= VV_UNREF; 3299 } 3300 for (;;) { 3301 error = vinactive(vp); 3302 if (want_unlock) 3303 VOP_UNLOCK(vp); 3304 if (error != ERELOOKUP || !want_unlock) 3305 break; 3306 VOP_LOCK(vp, LK_EXCLUSIVE); 3307 } 3308 if (func == VUNREF) 3309 vp->v_vflag &= ~VV_UNREF; 3310 vdropl(vp); 3311 } else { 3312 vdefer_inactive(vp); 3313 } 3314 return; 3315 out: 3316 if (func == VPUT) 3317 VOP_UNLOCK(vp); 3318 vdropl(vp); 3319 } 3320 3321 /* 3322 * Decrement ->v_usecount for a vnode. 3323 * 3324 * Releasing the last use count requires additional processing, see vput_final 3325 * above for details. 3326 * 3327 * Comment above each variant denotes lock state on entry and exit. 3328 */ 3329 3330 /* 3331 * in: any 3332 * out: same as passed in 3333 */ 3334 void 3335 vrele(struct vnode *vp) 3336 { 3337 3338 ASSERT_VI_UNLOCKED(vp, __func__); 3339 if (!refcount_release(&vp->v_usecount)) 3340 return; 3341 vput_final(vp, VRELE); 3342 } 3343 3344 /* 3345 * in: locked 3346 * out: unlocked 3347 */ 3348 void 3349 vput(struct vnode *vp) 3350 { 3351 3352 ASSERT_VOP_LOCKED(vp, __func__); 3353 ASSERT_VI_UNLOCKED(vp, __func__); 3354 if (!refcount_release(&vp->v_usecount)) { 3355 VOP_UNLOCK(vp); 3356 return; 3357 } 3358 vput_final(vp, VPUT); 3359 } 3360 3361 /* 3362 * in: locked 3363 * out: locked 3364 */ 3365 void 3366 vunref(struct vnode *vp) 3367 { 3368 3369 ASSERT_VOP_LOCKED(vp, __func__); 3370 ASSERT_VI_UNLOCKED(vp, __func__); 3371 if (!refcount_release(&vp->v_usecount)) 3372 return; 3373 vput_final(vp, VUNREF); 3374 } 3375 3376 void 3377 vhold(struct vnode *vp) 3378 { 3379 int old; 3380 3381 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3382 old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3383 VNASSERT(old >= 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3384 ("%s: wrong hold count %d", __func__, old)); 3385 if (old == 0) 3386 vfs_freevnodes_dec(); 3387 } 3388 3389 void 3390 vholdnz(struct vnode *vp) 3391 { 3392 3393 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3394 #ifdef INVARIANTS 3395 int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3396 VNASSERT(old > 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3397 ("%s: wrong hold count %d", __func__, old)); 3398 #else 3399 atomic_add_int(&vp->v_holdcnt, 1); 3400 #endif 3401 } 3402 3403 /* 3404 * Grab a hold count unless the vnode is freed. 3405 * 3406 * Only use this routine if vfs smr is the only protection you have against 3407 * freeing the vnode. 3408 * 3409 * The code loops trying to add a hold count as long as the VHOLD_NO_SMR flag 3410 * is not set. After the flag is set the vnode becomes immutable to anyone but 3411 * the thread which managed to set the flag. 3412 * 3413 * It may be tempting to replace the loop with: 3414 * count = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3415 * if (count & VHOLD_NO_SMR) { 3416 * backpedal and error out; 3417 * } 3418 * 3419 * However, while this is more performant, it hinders debugging by eliminating 3420 * the previously mentioned invariant. 3421 */ 3422 bool 3423 vhold_smr(struct vnode *vp) 3424 { 3425 int count; 3426 3427 VFS_SMR_ASSERT_ENTERED(); 3428 3429 count = atomic_load_int(&vp->v_holdcnt); 3430 for (;;) { 3431 if (count & VHOLD_NO_SMR) { 3432 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3433 ("non-zero hold count with flags %d\n", count)); 3434 return (false); 3435 } 3436 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3437 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3438 if (count == 0) 3439 vfs_freevnodes_dec(); 3440 return (true); 3441 } 3442 } 3443 } 3444 3445 /* 3446 * Hold a free vnode for recycling. 3447 * 3448 * Note: vnode_init references this comment. 3449 * 3450 * Attempts to recycle only need the global vnode list lock and have no use for 3451 * SMR. 3452 * 3453 * However, vnodes get inserted into the global list before they get fully 3454 * initialized and stay there until UMA decides to free the memory. This in 3455 * particular means the target can be found before it becomes usable and after 3456 * it becomes recycled. Picking up such vnodes is guarded with v_holdcnt set to 3457 * VHOLD_NO_SMR. 3458 * 3459 * Note: the vnode may gain more references after we transition the count 0->1. 3460 */ 3461 static bool 3462 vhold_recycle_free(struct vnode *vp) 3463 { 3464 int count; 3465 3466 mtx_assert(&vnode_list_mtx, MA_OWNED); 3467 3468 count = atomic_load_int(&vp->v_holdcnt); 3469 for (;;) { 3470 if (count & VHOLD_NO_SMR) { 3471 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3472 ("non-zero hold count with flags %d\n", count)); 3473 return (false); 3474 } 3475 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3476 if (count > 0) { 3477 return (false); 3478 } 3479 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3480 vfs_freevnodes_dec(); 3481 return (true); 3482 } 3483 } 3484 } 3485 3486 static void __noinline 3487 vdbatch_process(struct vdbatch *vd) 3488 { 3489 struct vnode *vp; 3490 int i; 3491 3492 mtx_assert(&vd->lock, MA_OWNED); 3493 MPASS(curthread->td_pinned > 0); 3494 MPASS(vd->index == VDBATCH_SIZE); 3495 3496 mtx_lock(&vnode_list_mtx); 3497 critical_enter(); 3498 freevnodes += vd->freevnodes; 3499 for (i = 0; i < VDBATCH_SIZE; i++) { 3500 vp = vd->tab[i]; 3501 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 3502 TAILQ_INSERT_TAIL(&vnode_list, vp, v_vnodelist); 3503 MPASS(vp->v_dbatchcpu != NOCPU); 3504 vp->v_dbatchcpu = NOCPU; 3505 } 3506 mtx_unlock(&vnode_list_mtx); 3507 vd->freevnodes = 0; 3508 bzero(vd->tab, sizeof(vd->tab)); 3509 vd->index = 0; 3510 critical_exit(); 3511 } 3512 3513 static void 3514 vdbatch_enqueue(struct vnode *vp) 3515 { 3516 struct vdbatch *vd; 3517 3518 ASSERT_VI_LOCKED(vp, __func__); 3519 VNASSERT(!VN_IS_DOOMED(vp), vp, 3520 ("%s: deferring requeue of a doomed vnode", __func__)); 3521 3522 if (vp->v_dbatchcpu != NOCPU) { 3523 VI_UNLOCK(vp); 3524 return; 3525 } 3526 3527 sched_pin(); 3528 vd = DPCPU_PTR(vd); 3529 mtx_lock(&vd->lock); 3530 MPASS(vd->index < VDBATCH_SIZE); 3531 MPASS(vd->tab[vd->index] == NULL); 3532 /* 3533 * A hack: we depend on being pinned so that we know what to put in 3534 * ->v_dbatchcpu. 3535 */ 3536 vp->v_dbatchcpu = curcpu; 3537 vd->tab[vd->index] = vp; 3538 vd->index++; 3539 VI_UNLOCK(vp); 3540 if (vd->index == VDBATCH_SIZE) 3541 vdbatch_process(vd); 3542 mtx_unlock(&vd->lock); 3543 sched_unpin(); 3544 } 3545 3546 /* 3547 * This routine must only be called for vnodes which are about to be 3548 * deallocated. Supporting dequeue for arbitrary vndoes would require 3549 * validating that the locked batch matches. 3550 */ 3551 static void 3552 vdbatch_dequeue(struct vnode *vp) 3553 { 3554 struct vdbatch *vd; 3555 int i; 3556 short cpu; 3557 3558 VNASSERT(vp->v_type == VBAD || vp->v_type == VNON, vp, 3559 ("%s: called for a used vnode\n", __func__)); 3560 3561 cpu = vp->v_dbatchcpu; 3562 if (cpu == NOCPU) 3563 return; 3564 3565 vd = DPCPU_ID_PTR(cpu, vd); 3566 mtx_lock(&vd->lock); 3567 for (i = 0; i < vd->index; i++) { 3568 if (vd->tab[i] != vp) 3569 continue; 3570 vp->v_dbatchcpu = NOCPU; 3571 vd->index--; 3572 vd->tab[i] = vd->tab[vd->index]; 3573 vd->tab[vd->index] = NULL; 3574 break; 3575 } 3576 mtx_unlock(&vd->lock); 3577 /* 3578 * Either we dequeued the vnode above or the target CPU beat us to it. 3579 */ 3580 MPASS(vp->v_dbatchcpu == NOCPU); 3581 } 3582 3583 /* 3584 * Drop the hold count of the vnode. If this is the last reference to 3585 * the vnode we place it on the free list unless it has been vgone'd 3586 * (marked VIRF_DOOMED) in which case we will free it. 3587 * 3588 * Because the vnode vm object keeps a hold reference on the vnode if 3589 * there is at least one resident non-cached page, the vnode cannot 3590 * leave the active list without the page cleanup done. 3591 */ 3592 static void __noinline 3593 vdropl_final(struct vnode *vp) 3594 { 3595 3596 ASSERT_VI_LOCKED(vp, __func__); 3597 VNPASS(VN_IS_DOOMED(vp), vp); 3598 /* 3599 * Set the VHOLD_NO_SMR flag. 3600 * 3601 * We may be racing against vhold_smr. If they win we can just pretend 3602 * we never got this far, they will vdrop later. 3603 */ 3604 if (__predict_false(!atomic_cmpset_int(&vp->v_holdcnt, 0, VHOLD_NO_SMR))) { 3605 vfs_freevnodes_inc(); 3606 VI_UNLOCK(vp); 3607 /* 3608 * We lost the aforementioned race. Any subsequent access is 3609 * invalid as they might have managed to vdropl on their own. 3610 */ 3611 return; 3612 } 3613 /* 3614 * Don't bump freevnodes as this one is going away. 3615 */ 3616 freevnode(vp); 3617 } 3618 3619 void 3620 vdrop(struct vnode *vp) 3621 { 3622 3623 ASSERT_VI_UNLOCKED(vp, __func__); 3624 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3625 if (refcount_release_if_not_last(&vp->v_holdcnt)) 3626 return; 3627 VI_LOCK(vp); 3628 vdropl(vp); 3629 } 3630 3631 static void __always_inline 3632 vdropl_impl(struct vnode *vp, bool enqueue) 3633 { 3634 3635 ASSERT_VI_LOCKED(vp, __func__); 3636 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3637 if (!refcount_release(&vp->v_holdcnt)) { 3638 VI_UNLOCK(vp); 3639 return; 3640 } 3641 VNPASS((vp->v_iflag & VI_OWEINACT) == 0, vp); 3642 VNPASS((vp->v_iflag & VI_DEFINACT) == 0, vp); 3643 if (VN_IS_DOOMED(vp)) { 3644 vdropl_final(vp); 3645 return; 3646 } 3647 3648 vfs_freevnodes_inc(); 3649 if (vp->v_mflag & VMP_LAZYLIST) { 3650 vunlazy(vp); 3651 } 3652 3653 if (!enqueue) { 3654 VI_UNLOCK(vp); 3655 return; 3656 } 3657 3658 /* 3659 * Also unlocks the interlock. We can't assert on it as we 3660 * released our hold and by now the vnode might have been 3661 * freed. 3662 */ 3663 vdbatch_enqueue(vp); 3664 } 3665 3666 void 3667 vdropl(struct vnode *vp) 3668 { 3669 3670 vdropl_impl(vp, true); 3671 } 3672 3673 /* 3674 * vdrop a vnode when recycling 3675 * 3676 * This is a special case routine only to be used when recycling, differs from 3677 * regular vdrop by not requeieing the vnode on LRU. 3678 * 3679 * Consider a case where vtryrecycle continuously fails with all vnodes (due to 3680 * e.g., frozen writes on the filesystem), filling the batch and causing it to 3681 * be requeued. Then vnlru will end up revisiting the same vnodes. This is a 3682 * loop which can last for as long as writes are frozen. 3683 */ 3684 static void 3685 vdropl_recycle(struct vnode *vp) 3686 { 3687 3688 vdropl_impl(vp, false); 3689 } 3690 3691 static void 3692 vdrop_recycle(struct vnode *vp) 3693 { 3694 3695 VI_LOCK(vp); 3696 vdropl_recycle(vp); 3697 } 3698 3699 /* 3700 * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT 3701 * flags. DOINGINACT prevents us from recursing in calls to vinactive. 3702 */ 3703 static int 3704 vinactivef(struct vnode *vp) 3705 { 3706 struct vm_object *obj; 3707 int error; 3708 3709 ASSERT_VOP_ELOCKED(vp, "vinactive"); 3710 ASSERT_VI_LOCKED(vp, "vinactive"); 3711 VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp, 3712 ("vinactive: recursed on VI_DOINGINACT")); 3713 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3714 vp->v_iflag |= VI_DOINGINACT; 3715 vp->v_iflag &= ~VI_OWEINACT; 3716 VI_UNLOCK(vp); 3717 /* 3718 * Before moving off the active list, we must be sure that any 3719 * modified pages are converted into the vnode's dirty 3720 * buffers, since these will no longer be checked once the 3721 * vnode is on the inactive list. 3722 * 3723 * The write-out of the dirty pages is asynchronous. At the 3724 * point that VOP_INACTIVE() is called, there could still be 3725 * pending I/O and dirty pages in the object. 3726 */ 3727 if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 3728 vm_object_mightbedirty(obj)) { 3729 VM_OBJECT_WLOCK(obj); 3730 vm_object_page_clean(obj, 0, 0, 0); 3731 VM_OBJECT_WUNLOCK(obj); 3732 } 3733 error = VOP_INACTIVE(vp); 3734 VI_LOCK(vp); 3735 VNASSERT(vp->v_iflag & VI_DOINGINACT, vp, 3736 ("vinactive: lost VI_DOINGINACT")); 3737 vp->v_iflag &= ~VI_DOINGINACT; 3738 return (error); 3739 } 3740 3741 int 3742 vinactive(struct vnode *vp) 3743 { 3744 3745 ASSERT_VOP_ELOCKED(vp, "vinactive"); 3746 ASSERT_VI_LOCKED(vp, "vinactive"); 3747 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3748 3749 if ((vp->v_iflag & VI_OWEINACT) == 0) 3750 return (0); 3751 if (vp->v_iflag & VI_DOINGINACT) 3752 return (0); 3753 if (vp->v_usecount > 0) { 3754 vp->v_iflag &= ~VI_OWEINACT; 3755 return (0); 3756 } 3757 return (vinactivef(vp)); 3758 } 3759 3760 /* 3761 * Remove any vnodes in the vnode table belonging to mount point mp. 3762 * 3763 * If FORCECLOSE is not specified, there should not be any active ones, 3764 * return error if any are found (nb: this is a user error, not a 3765 * system error). If FORCECLOSE is specified, detach any active vnodes 3766 * that are found. 3767 * 3768 * If WRITECLOSE is set, only flush out regular file vnodes open for 3769 * writing. 3770 * 3771 * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped. 3772 * 3773 * `rootrefs' specifies the base reference count for the root vnode 3774 * of this filesystem. The root vnode is considered busy if its 3775 * v_usecount exceeds this value. On a successful return, vflush(, td) 3776 * will call vrele() on the root vnode exactly rootrefs times. 3777 * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must 3778 * be zero. 3779 */ 3780 #ifdef DIAGNOSTIC 3781 static int busyprt = 0; /* print out busy vnodes */ 3782 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes"); 3783 #endif 3784 3785 int 3786 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td) 3787 { 3788 struct vnode *vp, *mvp, *rootvp = NULL; 3789 struct vattr vattr; 3790 int busy = 0, error; 3791 3792 CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp, 3793 rootrefs, flags); 3794 if (rootrefs > 0) { 3795 KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0, 3796 ("vflush: bad args")); 3797 /* 3798 * Get the filesystem root vnode. We can vput() it 3799 * immediately, since with rootrefs > 0, it won't go away. 3800 */ 3801 if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) { 3802 CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d", 3803 __func__, error); 3804 return (error); 3805 } 3806 vput(rootvp); 3807 } 3808 loop: 3809 MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { 3810 vholdl(vp); 3811 error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE); 3812 if (error) { 3813 vdrop(vp); 3814 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 3815 goto loop; 3816 } 3817 /* 3818 * Skip over a vnodes marked VV_SYSTEM. 3819 */ 3820 if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) { 3821 VOP_UNLOCK(vp); 3822 vdrop(vp); 3823 continue; 3824 } 3825 /* 3826 * If WRITECLOSE is set, flush out unlinked but still open 3827 * files (even if open only for reading) and regular file 3828 * vnodes open for writing. 3829 */ 3830 if (flags & WRITECLOSE) { 3831 if (vp->v_object != NULL) { 3832 VM_OBJECT_WLOCK(vp->v_object); 3833 vm_object_page_clean(vp->v_object, 0, 0, 0); 3834 VM_OBJECT_WUNLOCK(vp->v_object); 3835 } 3836 do { 3837 error = VOP_FSYNC(vp, MNT_WAIT, td); 3838 } while (error == ERELOOKUP); 3839 if (error != 0) { 3840 VOP_UNLOCK(vp); 3841 vdrop(vp); 3842 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 3843 return (error); 3844 } 3845 error = VOP_GETATTR(vp, &vattr, td->td_ucred); 3846 VI_LOCK(vp); 3847 3848 if ((vp->v_type == VNON || 3849 (error == 0 && vattr.va_nlink > 0)) && 3850 (vp->v_writecount <= 0 || vp->v_type != VREG)) { 3851 VOP_UNLOCK(vp); 3852 vdropl(vp); 3853 continue; 3854 } 3855 } else 3856 VI_LOCK(vp); 3857 /* 3858 * With v_usecount == 0, all we need to do is clear out the 3859 * vnode data structures and we are done. 3860 * 3861 * If FORCECLOSE is set, forcibly close the vnode. 3862 */ 3863 if (vp->v_usecount == 0 || (flags & FORCECLOSE)) { 3864 vgonel(vp); 3865 } else { 3866 busy++; 3867 #ifdef DIAGNOSTIC 3868 if (busyprt) 3869 vn_printf(vp, "vflush: busy vnode "); 3870 #endif 3871 } 3872 VOP_UNLOCK(vp); 3873 vdropl(vp); 3874 } 3875 if (rootrefs > 0 && (flags & FORCECLOSE) == 0) { 3876 /* 3877 * If just the root vnode is busy, and if its refcount 3878 * is equal to `rootrefs', then go ahead and kill it. 3879 */ 3880 VI_LOCK(rootvp); 3881 KASSERT(busy > 0, ("vflush: not busy")); 3882 VNASSERT(rootvp->v_usecount >= rootrefs, rootvp, 3883 ("vflush: usecount %d < rootrefs %d", 3884 rootvp->v_usecount, rootrefs)); 3885 if (busy == 1 && rootvp->v_usecount == rootrefs) { 3886 VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK); 3887 vgone(rootvp); 3888 VOP_UNLOCK(rootvp); 3889 busy = 0; 3890 } else 3891 VI_UNLOCK(rootvp); 3892 } 3893 if (busy) { 3894 CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__, 3895 busy); 3896 return (EBUSY); 3897 } 3898 for (; rootrefs > 0; rootrefs--) 3899 vrele(rootvp); 3900 return (0); 3901 } 3902 3903 /* 3904 * Recycle an unused vnode to the front of the free list. 3905 */ 3906 int 3907 vrecycle(struct vnode *vp) 3908 { 3909 int recycled; 3910 3911 VI_LOCK(vp); 3912 recycled = vrecyclel(vp); 3913 VI_UNLOCK(vp); 3914 return (recycled); 3915 } 3916 3917 /* 3918 * vrecycle, with the vp interlock held. 3919 */ 3920 int 3921 vrecyclel(struct vnode *vp) 3922 { 3923 int recycled; 3924 3925 ASSERT_VOP_ELOCKED(vp, __func__); 3926 ASSERT_VI_LOCKED(vp, __func__); 3927 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3928 recycled = 0; 3929 if (vp->v_usecount == 0) { 3930 recycled = 1; 3931 vgonel(vp); 3932 } 3933 return (recycled); 3934 } 3935 3936 /* 3937 * Eliminate all activity associated with a vnode 3938 * in preparation for reuse. 3939 */ 3940 void 3941 vgone(struct vnode *vp) 3942 { 3943 VI_LOCK(vp); 3944 vgonel(vp); 3945 VI_UNLOCK(vp); 3946 } 3947 3948 /* 3949 * Notify upper mounts about reclaimed or unlinked vnode. 3950 */ 3951 void 3952 vfs_notify_upper(struct vnode *vp, enum vfs_notify_upper_type event) 3953 { 3954 struct mount *mp; 3955 struct mount_upper_node *ump; 3956 3957 mp = atomic_load_ptr(&vp->v_mount); 3958 if (mp == NULL) 3959 return; 3960 if (TAILQ_EMPTY(&mp->mnt_notify)) 3961 return; 3962 3963 MNT_ILOCK(mp); 3964 mp->mnt_upper_pending++; 3965 KASSERT(mp->mnt_upper_pending > 0, 3966 ("%s: mnt_upper_pending %d", __func__, mp->mnt_upper_pending)); 3967 TAILQ_FOREACH(ump, &mp->mnt_notify, mnt_upper_link) { 3968 MNT_IUNLOCK(mp); 3969 switch (event) { 3970 case VFS_NOTIFY_UPPER_RECLAIM: 3971 VFS_RECLAIM_LOWERVP(ump->mp, vp); 3972 break; 3973 case VFS_NOTIFY_UPPER_UNLINK: 3974 VFS_UNLINK_LOWERVP(ump->mp, vp); 3975 break; 3976 } 3977 MNT_ILOCK(mp); 3978 } 3979 mp->mnt_upper_pending--; 3980 if ((mp->mnt_kern_flag & MNTK_UPPER_WAITER) != 0 && 3981 mp->mnt_upper_pending == 0) { 3982 mp->mnt_kern_flag &= ~MNTK_UPPER_WAITER; 3983 wakeup(&mp->mnt_uppers); 3984 } 3985 MNT_IUNLOCK(mp); 3986 } 3987 3988 /* 3989 * vgone, with the vp interlock held. 3990 */ 3991 static void 3992 vgonel(struct vnode *vp) 3993 { 3994 struct thread *td; 3995 struct mount *mp; 3996 vm_object_t object; 3997 bool active, doinginact, oweinact; 3998 3999 ASSERT_VOP_ELOCKED(vp, "vgonel"); 4000 ASSERT_VI_LOCKED(vp, "vgonel"); 4001 VNASSERT(vp->v_holdcnt, vp, 4002 ("vgonel: vp %p has no reference.", vp)); 4003 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4004 td = curthread; 4005 4006 /* 4007 * Don't vgonel if we're already doomed. 4008 */ 4009 if (VN_IS_DOOMED(vp)) 4010 return; 4011 /* 4012 * Paired with freevnode. 4013 */ 4014 vn_seqc_write_begin_locked(vp); 4015 vunlazy_gone(vp); 4016 vn_irflag_set_locked(vp, VIRF_DOOMED); 4017 4018 /* 4019 * Check to see if the vnode is in use. If so, we have to 4020 * call VOP_CLOSE() and VOP_INACTIVE(). 4021 * 4022 * It could be that VOP_INACTIVE() requested reclamation, in 4023 * which case we should avoid recursion, so check 4024 * VI_DOINGINACT. This is not precise but good enough. 4025 */ 4026 active = vp->v_usecount > 0; 4027 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4028 doinginact = (vp->v_iflag & VI_DOINGINACT) != 0; 4029 4030 /* 4031 * If we need to do inactive VI_OWEINACT will be set. 4032 */ 4033 if (vp->v_iflag & VI_DEFINACT) { 4034 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 4035 vp->v_iflag &= ~VI_DEFINACT; 4036 vdropl(vp); 4037 } else { 4038 VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count")); 4039 VI_UNLOCK(vp); 4040 } 4041 cache_purge_vgone(vp); 4042 vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM); 4043 4044 /* 4045 * If purging an active vnode, it must be closed and 4046 * deactivated before being reclaimed. 4047 */ 4048 if (active) 4049 VOP_CLOSE(vp, FNONBLOCK, NOCRED, td); 4050 if (!doinginact) { 4051 do { 4052 if (oweinact || active) { 4053 VI_LOCK(vp); 4054 vinactivef(vp); 4055 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4056 VI_UNLOCK(vp); 4057 } 4058 } while (oweinact); 4059 } 4060 if (vp->v_type == VSOCK) 4061 vfs_unp_reclaim(vp); 4062 4063 /* 4064 * Clean out any buffers associated with the vnode. 4065 * If the flush fails, just toss the buffers. 4066 */ 4067 mp = NULL; 4068 if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd)) 4069 (void) vn_start_secondary_write(vp, &mp, V_WAIT); 4070 if (vinvalbuf(vp, V_SAVE, 0, 0) != 0) { 4071 while (vinvalbuf(vp, 0, 0, 0) != 0) 4072 ; 4073 } 4074 4075 BO_LOCK(&vp->v_bufobj); 4076 KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) && 4077 vp->v_bufobj.bo_dirty.bv_cnt == 0 && 4078 TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) && 4079 vp->v_bufobj.bo_clean.bv_cnt == 0, 4080 ("vp %p bufobj not invalidated", vp)); 4081 4082 /* 4083 * For VMIO bufobj, BO_DEAD is set later, or in 4084 * vm_object_terminate() after the object's page queue is 4085 * flushed. 4086 */ 4087 object = vp->v_bufobj.bo_object; 4088 if (object == NULL) 4089 vp->v_bufobj.bo_flag |= BO_DEAD; 4090 BO_UNLOCK(&vp->v_bufobj); 4091 4092 /* 4093 * Handle the VM part. Tmpfs handles v_object on its own (the 4094 * OBJT_VNODE check). Nullfs or other bypassing filesystems 4095 * should not touch the object borrowed from the lower vnode 4096 * (the handle check). 4097 */ 4098 if (object != NULL && object->type == OBJT_VNODE && 4099 object->handle == vp) 4100 vnode_destroy_vobject(vp); 4101 4102 /* 4103 * Reclaim the vnode. 4104 */ 4105 if (VOP_RECLAIM(vp)) 4106 panic("vgone: cannot reclaim"); 4107 if (mp != NULL) 4108 vn_finished_secondary_write(mp); 4109 VNASSERT(vp->v_object == NULL, vp, 4110 ("vop_reclaim left v_object vp=%p", vp)); 4111 /* 4112 * Clear the advisory locks and wake up waiting threads. 4113 */ 4114 if (vp->v_lockf != NULL) { 4115 (void)VOP_ADVLOCKPURGE(vp); 4116 vp->v_lockf = NULL; 4117 } 4118 /* 4119 * Delete from old mount point vnode list. 4120 */ 4121 if (vp->v_mount == NULL) { 4122 VI_LOCK(vp); 4123 } else { 4124 delmntque(vp); 4125 ASSERT_VI_LOCKED(vp, "vgonel 2"); 4126 } 4127 /* 4128 * Done with purge, reset to the standard lock and invalidate 4129 * the vnode. 4130 */ 4131 vp->v_vnlock = &vp->v_lock; 4132 vp->v_op = &dead_vnodeops; 4133 vp->v_type = VBAD; 4134 } 4135 4136 /* 4137 * Print out a description of a vnode. 4138 */ 4139 static const char * const typename[] = 4140 {"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD", 4141 "VMARKER"}; 4142 4143 _Static_assert((VHOLD_ALL_FLAGS & ~VHOLD_NO_SMR) == 0, 4144 "new hold count flag not added to vn_printf"); 4145 4146 void 4147 vn_printf(struct vnode *vp, const char *fmt, ...) 4148 { 4149 va_list ap; 4150 char buf[256], buf2[16]; 4151 u_long flags; 4152 u_int holdcnt; 4153 short irflag; 4154 4155 va_start(ap, fmt); 4156 vprintf(fmt, ap); 4157 va_end(ap); 4158 printf("%p: ", (void *)vp); 4159 printf("type %s\n", typename[vp->v_type]); 4160 holdcnt = atomic_load_int(&vp->v_holdcnt); 4161 printf(" usecount %d, writecount %d, refcount %d seqc users %d", 4162 vp->v_usecount, vp->v_writecount, holdcnt & ~VHOLD_ALL_FLAGS, 4163 vp->v_seqc_users); 4164 switch (vp->v_type) { 4165 case VDIR: 4166 printf(" mountedhere %p\n", vp->v_mountedhere); 4167 break; 4168 case VCHR: 4169 printf(" rdev %p\n", vp->v_rdev); 4170 break; 4171 case VSOCK: 4172 printf(" socket %p\n", vp->v_unpcb); 4173 break; 4174 case VFIFO: 4175 printf(" fifoinfo %p\n", vp->v_fifoinfo); 4176 break; 4177 default: 4178 printf("\n"); 4179 break; 4180 } 4181 buf[0] = '\0'; 4182 buf[1] = '\0'; 4183 if (holdcnt & VHOLD_NO_SMR) 4184 strlcat(buf, "|VHOLD_NO_SMR", sizeof(buf)); 4185 printf(" hold count flags (%s)\n", buf + 1); 4186 4187 buf[0] = '\0'; 4188 buf[1] = '\0'; 4189 irflag = vn_irflag_read(vp); 4190 if (irflag & VIRF_DOOMED) 4191 strlcat(buf, "|VIRF_DOOMED", sizeof(buf)); 4192 if (irflag & VIRF_PGREAD) 4193 strlcat(buf, "|VIRF_PGREAD", sizeof(buf)); 4194 if (irflag & VIRF_MOUNTPOINT) 4195 strlcat(buf, "|VIRF_MOUNTPOINT", sizeof(buf)); 4196 if (irflag & VIRF_TEXT_REF) 4197 strlcat(buf, "|VIRF_TEXT_REF", sizeof(buf)); 4198 flags = irflag & ~(VIRF_DOOMED | VIRF_PGREAD | VIRF_MOUNTPOINT | VIRF_TEXT_REF); 4199 if (flags != 0) { 4200 snprintf(buf2, sizeof(buf2), "|VIRF(0x%lx)", flags); 4201 strlcat(buf, buf2, sizeof(buf)); 4202 } 4203 if (vp->v_vflag & VV_ROOT) 4204 strlcat(buf, "|VV_ROOT", sizeof(buf)); 4205 if (vp->v_vflag & VV_ISTTY) 4206 strlcat(buf, "|VV_ISTTY", sizeof(buf)); 4207 if (vp->v_vflag & VV_NOSYNC) 4208 strlcat(buf, "|VV_NOSYNC", sizeof(buf)); 4209 if (vp->v_vflag & VV_ETERNALDEV) 4210 strlcat(buf, "|VV_ETERNALDEV", sizeof(buf)); 4211 if (vp->v_vflag & VV_CACHEDLABEL) 4212 strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf)); 4213 if (vp->v_vflag & VV_VMSIZEVNLOCK) 4214 strlcat(buf, "|VV_VMSIZEVNLOCK", sizeof(buf)); 4215 if (vp->v_vflag & VV_COPYONWRITE) 4216 strlcat(buf, "|VV_COPYONWRITE", sizeof(buf)); 4217 if (vp->v_vflag & VV_SYSTEM) 4218 strlcat(buf, "|VV_SYSTEM", sizeof(buf)); 4219 if (vp->v_vflag & VV_PROCDEP) 4220 strlcat(buf, "|VV_PROCDEP", sizeof(buf)); 4221 if (vp->v_vflag & VV_DELETED) 4222 strlcat(buf, "|VV_DELETED", sizeof(buf)); 4223 if (vp->v_vflag & VV_MD) 4224 strlcat(buf, "|VV_MD", sizeof(buf)); 4225 if (vp->v_vflag & VV_FORCEINSMQ) 4226 strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf)); 4227 if (vp->v_vflag & VV_READLINK) 4228 strlcat(buf, "|VV_READLINK", sizeof(buf)); 4229 flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV | 4230 VV_CACHEDLABEL | VV_VMSIZEVNLOCK | VV_COPYONWRITE | VV_SYSTEM | 4231 VV_PROCDEP | VV_DELETED | VV_MD | VV_FORCEINSMQ | VV_READLINK); 4232 if (flags != 0) { 4233 snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags); 4234 strlcat(buf, buf2, sizeof(buf)); 4235 } 4236 if (vp->v_iflag & VI_MOUNT) 4237 strlcat(buf, "|VI_MOUNT", sizeof(buf)); 4238 if (vp->v_iflag & VI_DOINGINACT) 4239 strlcat(buf, "|VI_DOINGINACT", sizeof(buf)); 4240 if (vp->v_iflag & VI_OWEINACT) 4241 strlcat(buf, "|VI_OWEINACT", sizeof(buf)); 4242 if (vp->v_iflag & VI_DEFINACT) 4243 strlcat(buf, "|VI_DEFINACT", sizeof(buf)); 4244 if (vp->v_iflag & VI_FOPENING) 4245 strlcat(buf, "|VI_FOPENING", sizeof(buf)); 4246 flags = vp->v_iflag & ~(VI_MOUNT | VI_DOINGINACT | 4247 VI_OWEINACT | VI_DEFINACT | VI_FOPENING); 4248 if (flags != 0) { 4249 snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags); 4250 strlcat(buf, buf2, sizeof(buf)); 4251 } 4252 if (vp->v_mflag & VMP_LAZYLIST) 4253 strlcat(buf, "|VMP_LAZYLIST", sizeof(buf)); 4254 flags = vp->v_mflag & ~(VMP_LAZYLIST); 4255 if (flags != 0) { 4256 snprintf(buf2, sizeof(buf2), "|VMP(0x%lx)", flags); 4257 strlcat(buf, buf2, sizeof(buf)); 4258 } 4259 printf(" flags (%s)", buf + 1); 4260 if (mtx_owned(VI_MTX(vp))) 4261 printf(" VI_LOCKed"); 4262 printf("\n"); 4263 if (vp->v_object != NULL) 4264 printf(" v_object %p ref %d pages %d " 4265 "cleanbuf %d dirtybuf %d\n", 4266 vp->v_object, vp->v_object->ref_count, 4267 vp->v_object->resident_page_count, 4268 vp->v_bufobj.bo_clean.bv_cnt, 4269 vp->v_bufobj.bo_dirty.bv_cnt); 4270 printf(" "); 4271 lockmgr_printinfo(vp->v_vnlock); 4272 if (vp->v_data != NULL) 4273 VOP_PRINT(vp); 4274 } 4275 4276 #ifdef DDB 4277 /* 4278 * List all of the locked vnodes in the system. 4279 * Called when debugging the kernel. 4280 */ 4281 DB_SHOW_COMMAND_FLAGS(lockedvnods, lockedvnodes, DB_CMD_MEMSAFE) 4282 { 4283 struct mount *mp; 4284 struct vnode *vp; 4285 4286 /* 4287 * Note: because this is DDB, we can't obey the locking semantics 4288 * for these structures, which means we could catch an inconsistent 4289 * state and dereference a nasty pointer. Not much to be done 4290 * about that. 4291 */ 4292 db_printf("Locked vnodes\n"); 4293 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4294 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4295 if (vp->v_type != VMARKER && VOP_ISLOCKED(vp)) 4296 vn_printf(vp, "vnode "); 4297 } 4298 } 4299 } 4300 4301 /* 4302 * Show details about the given vnode. 4303 */ 4304 DB_SHOW_COMMAND(vnode, db_show_vnode) 4305 { 4306 struct vnode *vp; 4307 4308 if (!have_addr) 4309 return; 4310 vp = (struct vnode *)addr; 4311 vn_printf(vp, "vnode "); 4312 } 4313 4314 /* 4315 * Show details about the given mount point. 4316 */ 4317 DB_SHOW_COMMAND(mount, db_show_mount) 4318 { 4319 struct mount *mp; 4320 struct vfsopt *opt; 4321 struct statfs *sp; 4322 struct vnode *vp; 4323 char buf[512]; 4324 uint64_t mflags; 4325 u_int flags; 4326 4327 if (!have_addr) { 4328 /* No address given, print short info about all mount points. */ 4329 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4330 db_printf("%p %s on %s (%s)\n", mp, 4331 mp->mnt_stat.f_mntfromname, 4332 mp->mnt_stat.f_mntonname, 4333 mp->mnt_stat.f_fstypename); 4334 if (db_pager_quit) 4335 break; 4336 } 4337 db_printf("\nMore info: show mount <addr>\n"); 4338 return; 4339 } 4340 4341 mp = (struct mount *)addr; 4342 db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname, 4343 mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename); 4344 4345 buf[0] = '\0'; 4346 mflags = mp->mnt_flag; 4347 #define MNT_FLAG(flag) do { \ 4348 if (mflags & (flag)) { \ 4349 if (buf[0] != '\0') \ 4350 strlcat(buf, ", ", sizeof(buf)); \ 4351 strlcat(buf, (#flag) + 4, sizeof(buf)); \ 4352 mflags &= ~(flag); \ 4353 } \ 4354 } while (0) 4355 MNT_FLAG(MNT_RDONLY); 4356 MNT_FLAG(MNT_SYNCHRONOUS); 4357 MNT_FLAG(MNT_NOEXEC); 4358 MNT_FLAG(MNT_NOSUID); 4359 MNT_FLAG(MNT_NFS4ACLS); 4360 MNT_FLAG(MNT_UNION); 4361 MNT_FLAG(MNT_ASYNC); 4362 MNT_FLAG(MNT_SUIDDIR); 4363 MNT_FLAG(MNT_SOFTDEP); 4364 MNT_FLAG(MNT_NOSYMFOLLOW); 4365 MNT_FLAG(MNT_GJOURNAL); 4366 MNT_FLAG(MNT_MULTILABEL); 4367 MNT_FLAG(MNT_ACLS); 4368 MNT_FLAG(MNT_NOATIME); 4369 MNT_FLAG(MNT_NOCLUSTERR); 4370 MNT_FLAG(MNT_NOCLUSTERW); 4371 MNT_FLAG(MNT_SUJ); 4372 MNT_FLAG(MNT_EXRDONLY); 4373 MNT_FLAG(MNT_EXPORTED); 4374 MNT_FLAG(MNT_DEFEXPORTED); 4375 MNT_FLAG(MNT_EXPORTANON); 4376 MNT_FLAG(MNT_EXKERB); 4377 MNT_FLAG(MNT_EXPUBLIC); 4378 MNT_FLAG(MNT_LOCAL); 4379 MNT_FLAG(MNT_QUOTA); 4380 MNT_FLAG(MNT_ROOTFS); 4381 MNT_FLAG(MNT_USER); 4382 MNT_FLAG(MNT_IGNORE); 4383 MNT_FLAG(MNT_UPDATE); 4384 MNT_FLAG(MNT_DELEXPORT); 4385 MNT_FLAG(MNT_RELOAD); 4386 MNT_FLAG(MNT_FORCE); 4387 MNT_FLAG(MNT_SNAPSHOT); 4388 MNT_FLAG(MNT_BYFSID); 4389 #undef MNT_FLAG 4390 if (mflags != 0) { 4391 if (buf[0] != '\0') 4392 strlcat(buf, ", ", sizeof(buf)); 4393 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4394 "0x%016jx", mflags); 4395 } 4396 db_printf(" mnt_flag = %s\n", buf); 4397 4398 buf[0] = '\0'; 4399 flags = mp->mnt_kern_flag; 4400 #define MNT_KERN_FLAG(flag) do { \ 4401 if (flags & (flag)) { \ 4402 if (buf[0] != '\0') \ 4403 strlcat(buf, ", ", sizeof(buf)); \ 4404 strlcat(buf, (#flag) + 5, sizeof(buf)); \ 4405 flags &= ~(flag); \ 4406 } \ 4407 } while (0) 4408 MNT_KERN_FLAG(MNTK_UNMOUNTF); 4409 MNT_KERN_FLAG(MNTK_ASYNC); 4410 MNT_KERN_FLAG(MNTK_SOFTDEP); 4411 MNT_KERN_FLAG(MNTK_NOMSYNC); 4412 MNT_KERN_FLAG(MNTK_DRAINING); 4413 MNT_KERN_FLAG(MNTK_REFEXPIRE); 4414 MNT_KERN_FLAG(MNTK_EXTENDED_SHARED); 4415 MNT_KERN_FLAG(MNTK_SHARED_WRITES); 4416 MNT_KERN_FLAG(MNTK_NO_IOPF); 4417 MNT_KERN_FLAG(MNTK_RECURSE); 4418 MNT_KERN_FLAG(MNTK_UPPER_WAITER); 4419 MNT_KERN_FLAG(MNTK_UNLOCKED_INSMNTQUE); 4420 MNT_KERN_FLAG(MNTK_USES_BCACHE); 4421 MNT_KERN_FLAG(MNTK_VMSETSIZE_BUG); 4422 MNT_KERN_FLAG(MNTK_FPLOOKUP); 4423 MNT_KERN_FLAG(MNTK_TASKQUEUE_WAITER); 4424 MNT_KERN_FLAG(MNTK_NOASYNC); 4425 MNT_KERN_FLAG(MNTK_UNMOUNT); 4426 MNT_KERN_FLAG(MNTK_MWAIT); 4427 MNT_KERN_FLAG(MNTK_SUSPEND); 4428 MNT_KERN_FLAG(MNTK_SUSPEND2); 4429 MNT_KERN_FLAG(MNTK_SUSPENDED); 4430 MNT_KERN_FLAG(MNTK_NULL_NOCACHE); 4431 MNT_KERN_FLAG(MNTK_LOOKUP_SHARED); 4432 #undef MNT_KERN_FLAG 4433 if (flags != 0) { 4434 if (buf[0] != '\0') 4435 strlcat(buf, ", ", sizeof(buf)); 4436 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4437 "0x%08x", flags); 4438 } 4439 db_printf(" mnt_kern_flag = %s\n", buf); 4440 4441 db_printf(" mnt_opt = "); 4442 opt = TAILQ_FIRST(mp->mnt_opt); 4443 if (opt != NULL) { 4444 db_printf("%s", opt->name); 4445 opt = TAILQ_NEXT(opt, link); 4446 while (opt != NULL) { 4447 db_printf(", %s", opt->name); 4448 opt = TAILQ_NEXT(opt, link); 4449 } 4450 } 4451 db_printf("\n"); 4452 4453 sp = &mp->mnt_stat; 4454 db_printf(" mnt_stat = { version=%u type=%u flags=0x%016jx " 4455 "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju " 4456 "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju " 4457 "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n", 4458 (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags, 4459 (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize, 4460 (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree, 4461 (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files, 4462 (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites, 4463 (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads, 4464 (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax, 4465 (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]); 4466 4467 db_printf(" mnt_cred = { uid=%u ruid=%u", 4468 (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid); 4469 if (jailed(mp->mnt_cred)) 4470 db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id); 4471 db_printf(" }\n"); 4472 db_printf(" mnt_ref = %d (with %d in the struct)\n", 4473 vfs_mount_fetch_counter(mp, MNT_COUNT_REF), mp->mnt_ref); 4474 db_printf(" mnt_gen = %d\n", mp->mnt_gen); 4475 db_printf(" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize); 4476 db_printf(" mnt_lazyvnodelistsize = %d\n", 4477 mp->mnt_lazyvnodelistsize); 4478 db_printf(" mnt_writeopcount = %d (with %d in the struct)\n", 4479 vfs_mount_fetch_counter(mp, MNT_COUNT_WRITEOPCOUNT), mp->mnt_writeopcount); 4480 db_printf(" mnt_iosize_max = %d\n", mp->mnt_iosize_max); 4481 db_printf(" mnt_hashseed = %u\n", mp->mnt_hashseed); 4482 db_printf(" mnt_lockref = %d (with %d in the struct)\n", 4483 vfs_mount_fetch_counter(mp, MNT_COUNT_LOCKREF), mp->mnt_lockref); 4484 db_printf(" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes); 4485 db_printf(" mnt_secondary_accwrites = %d\n", 4486 mp->mnt_secondary_accwrites); 4487 db_printf(" mnt_gjprovider = %s\n", 4488 mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL"); 4489 db_printf(" mnt_vfs_ops = %d\n", mp->mnt_vfs_ops); 4490 4491 db_printf("\n\nList of active vnodes\n"); 4492 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4493 if (vp->v_type != VMARKER && vp->v_holdcnt > 0) { 4494 vn_printf(vp, "vnode "); 4495 if (db_pager_quit) 4496 break; 4497 } 4498 } 4499 db_printf("\n\nList of inactive vnodes\n"); 4500 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4501 if (vp->v_type != VMARKER && vp->v_holdcnt == 0) { 4502 vn_printf(vp, "vnode "); 4503 if (db_pager_quit) 4504 break; 4505 } 4506 } 4507 } 4508 #endif /* DDB */ 4509 4510 /* 4511 * Fill in a struct xvfsconf based on a struct vfsconf. 4512 */ 4513 static int 4514 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp) 4515 { 4516 struct xvfsconf xvfsp; 4517 4518 bzero(&xvfsp, sizeof(xvfsp)); 4519 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4520 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4521 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4522 xvfsp.vfc_flags = vfsp->vfc_flags; 4523 /* 4524 * These are unused in userland, we keep them 4525 * to not break binary compatibility. 4526 */ 4527 xvfsp.vfc_vfsops = NULL; 4528 xvfsp.vfc_next = NULL; 4529 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4530 } 4531 4532 #ifdef COMPAT_FREEBSD32 4533 struct xvfsconf32 { 4534 uint32_t vfc_vfsops; 4535 char vfc_name[MFSNAMELEN]; 4536 int32_t vfc_typenum; 4537 int32_t vfc_refcount; 4538 int32_t vfc_flags; 4539 uint32_t vfc_next; 4540 }; 4541 4542 static int 4543 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp) 4544 { 4545 struct xvfsconf32 xvfsp; 4546 4547 bzero(&xvfsp, sizeof(xvfsp)); 4548 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4549 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4550 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4551 xvfsp.vfc_flags = vfsp->vfc_flags; 4552 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4553 } 4554 #endif 4555 4556 /* 4557 * Top level filesystem related information gathering. 4558 */ 4559 static int 4560 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS) 4561 { 4562 struct vfsconf *vfsp; 4563 int error; 4564 4565 error = 0; 4566 vfsconf_slock(); 4567 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4568 #ifdef COMPAT_FREEBSD32 4569 if (req->flags & SCTL_MASK32) 4570 error = vfsconf2x32(req, vfsp); 4571 else 4572 #endif 4573 error = vfsconf2x(req, vfsp); 4574 if (error) 4575 break; 4576 } 4577 vfsconf_sunlock(); 4578 return (error); 4579 } 4580 4581 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD | 4582 CTLFLAG_MPSAFE, NULL, 0, sysctl_vfs_conflist, 4583 "S,xvfsconf", "List of all configured filesystems"); 4584 4585 #ifndef BURN_BRIDGES 4586 static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS); 4587 4588 static int 4589 vfs_sysctl(SYSCTL_HANDLER_ARGS) 4590 { 4591 int *name = (int *)arg1 - 1; /* XXX */ 4592 u_int namelen = arg2 + 1; /* XXX */ 4593 struct vfsconf *vfsp; 4594 4595 log(LOG_WARNING, "userland calling deprecated sysctl, " 4596 "please rebuild world\n"); 4597 4598 #if 1 || defined(COMPAT_PRELITE2) 4599 /* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */ 4600 if (namelen == 1) 4601 return (sysctl_ovfs_conf(oidp, arg1, arg2, req)); 4602 #endif 4603 4604 switch (name[1]) { 4605 case VFS_MAXTYPENUM: 4606 if (namelen != 2) 4607 return (ENOTDIR); 4608 return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int))); 4609 case VFS_CONF: 4610 if (namelen != 3) 4611 return (ENOTDIR); /* overloaded */ 4612 vfsconf_slock(); 4613 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4614 if (vfsp->vfc_typenum == name[2]) 4615 break; 4616 } 4617 vfsconf_sunlock(); 4618 if (vfsp == NULL) 4619 return (EOPNOTSUPP); 4620 #ifdef COMPAT_FREEBSD32 4621 if (req->flags & SCTL_MASK32) 4622 return (vfsconf2x32(req, vfsp)); 4623 else 4624 #endif 4625 return (vfsconf2x(req, vfsp)); 4626 } 4627 return (EOPNOTSUPP); 4628 } 4629 4630 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP | 4631 CTLFLAG_MPSAFE, vfs_sysctl, 4632 "Generic filesystem"); 4633 4634 #if 1 || defined(COMPAT_PRELITE2) 4635 4636 static int 4637 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS) 4638 { 4639 int error; 4640 struct vfsconf *vfsp; 4641 struct ovfsconf ovfs; 4642 4643 vfsconf_slock(); 4644 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4645 bzero(&ovfs, sizeof(ovfs)); 4646 ovfs.vfc_vfsops = vfsp->vfc_vfsops; /* XXX used as flag */ 4647 strcpy(ovfs.vfc_name, vfsp->vfc_name); 4648 ovfs.vfc_index = vfsp->vfc_typenum; 4649 ovfs.vfc_refcount = vfsp->vfc_refcount; 4650 ovfs.vfc_flags = vfsp->vfc_flags; 4651 error = SYSCTL_OUT(req, &ovfs, sizeof ovfs); 4652 if (error != 0) { 4653 vfsconf_sunlock(); 4654 return (error); 4655 } 4656 } 4657 vfsconf_sunlock(); 4658 return (0); 4659 } 4660 4661 #endif /* 1 || COMPAT_PRELITE2 */ 4662 #endif /* !BURN_BRIDGES */ 4663 4664 #define KINFO_VNODESLOP 10 4665 #ifdef notyet 4666 /* 4667 * Dump vnode list (via sysctl). 4668 */ 4669 /* ARGSUSED */ 4670 static int 4671 sysctl_vnode(SYSCTL_HANDLER_ARGS) 4672 { 4673 struct xvnode *xvn; 4674 struct mount *mp; 4675 struct vnode *vp; 4676 int error, len, n; 4677 4678 /* 4679 * Stale numvnodes access is not fatal here. 4680 */ 4681 req->lock = 0; 4682 len = (numvnodes + KINFO_VNODESLOP) * sizeof *xvn; 4683 if (!req->oldptr) 4684 /* Make an estimate */ 4685 return (SYSCTL_OUT(req, 0, len)); 4686 4687 error = sysctl_wire_old_buffer(req, 0); 4688 if (error != 0) 4689 return (error); 4690 xvn = malloc(len, M_TEMP, M_ZERO | M_WAITOK); 4691 n = 0; 4692 mtx_lock(&mountlist_mtx); 4693 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4694 if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) 4695 continue; 4696 MNT_ILOCK(mp); 4697 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4698 if (n == len) 4699 break; 4700 vref(vp); 4701 xvn[n].xv_size = sizeof *xvn; 4702 xvn[n].xv_vnode = vp; 4703 xvn[n].xv_id = 0; /* XXX compat */ 4704 #define XV_COPY(field) xvn[n].xv_##field = vp->v_##field 4705 XV_COPY(usecount); 4706 XV_COPY(writecount); 4707 XV_COPY(holdcnt); 4708 XV_COPY(mount); 4709 XV_COPY(numoutput); 4710 XV_COPY(type); 4711 #undef XV_COPY 4712 xvn[n].xv_flag = vp->v_vflag; 4713 4714 switch (vp->v_type) { 4715 case VREG: 4716 case VDIR: 4717 case VLNK: 4718 break; 4719 case VBLK: 4720 case VCHR: 4721 if (vp->v_rdev == NULL) { 4722 vrele(vp); 4723 continue; 4724 } 4725 xvn[n].xv_dev = dev2udev(vp->v_rdev); 4726 break; 4727 case VSOCK: 4728 xvn[n].xv_socket = vp->v_socket; 4729 break; 4730 case VFIFO: 4731 xvn[n].xv_fifo = vp->v_fifoinfo; 4732 break; 4733 case VNON: 4734 case VBAD: 4735 default: 4736 /* shouldn't happen? */ 4737 vrele(vp); 4738 continue; 4739 } 4740 vrele(vp); 4741 ++n; 4742 } 4743 MNT_IUNLOCK(mp); 4744 mtx_lock(&mountlist_mtx); 4745 vfs_unbusy(mp); 4746 if (n == len) 4747 break; 4748 } 4749 mtx_unlock(&mountlist_mtx); 4750 4751 error = SYSCTL_OUT(req, xvn, n * sizeof *xvn); 4752 free(xvn, M_TEMP); 4753 return (error); 4754 } 4755 4756 SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE | CTLFLAG_RD | 4757 CTLFLAG_MPSAFE, 0, 0, sysctl_vnode, "S,xvnode", 4758 ""); 4759 #endif 4760 4761 static void 4762 unmount_or_warn(struct mount *mp) 4763 { 4764 int error; 4765 4766 error = dounmount(mp, MNT_FORCE, curthread); 4767 if (error != 0) { 4768 printf("unmount of %s failed (", mp->mnt_stat.f_mntonname); 4769 if (error == EBUSY) 4770 printf("BUSY)\n"); 4771 else 4772 printf("%d)\n", error); 4773 } 4774 } 4775 4776 /* 4777 * Unmount all filesystems. The list is traversed in reverse order 4778 * of mounting to avoid dependencies. 4779 */ 4780 void 4781 vfs_unmountall(void) 4782 { 4783 struct mount *mp, *tmp; 4784 4785 CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__); 4786 4787 /* 4788 * Since this only runs when rebooting, it is not interlocked. 4789 */ 4790 TAILQ_FOREACH_REVERSE_SAFE(mp, &mountlist, mntlist, mnt_list, tmp) { 4791 vfs_ref(mp); 4792 4793 /* 4794 * Forcibly unmounting "/dev" before "/" would prevent clean 4795 * unmount of the latter. 4796 */ 4797 if (mp == rootdevmp) 4798 continue; 4799 4800 unmount_or_warn(mp); 4801 } 4802 4803 if (rootdevmp != NULL) 4804 unmount_or_warn(rootdevmp); 4805 } 4806 4807 static void 4808 vfs_deferred_inactive(struct vnode *vp, int lkflags) 4809 { 4810 4811 ASSERT_VI_LOCKED(vp, __func__); 4812 VNASSERT((vp->v_iflag & VI_DEFINACT) == 0, vp, ("VI_DEFINACT still set")); 4813 if ((vp->v_iflag & VI_OWEINACT) == 0) { 4814 vdropl(vp); 4815 return; 4816 } 4817 if (vn_lock(vp, lkflags) == 0) { 4818 VI_LOCK(vp); 4819 vinactive(vp); 4820 VOP_UNLOCK(vp); 4821 vdropl(vp); 4822 return; 4823 } 4824 vdefer_inactive_unlocked(vp); 4825 } 4826 4827 static int 4828 vfs_periodic_inactive_filter(struct vnode *vp, void *arg) 4829 { 4830 4831 return (vp->v_iflag & VI_DEFINACT); 4832 } 4833 4834 static void __noinline 4835 vfs_periodic_inactive(struct mount *mp, int flags) 4836 { 4837 struct vnode *vp, *mvp; 4838 int lkflags; 4839 4840 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 4841 if (flags != MNT_WAIT) 4842 lkflags |= LK_NOWAIT; 4843 4844 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_inactive_filter, NULL) { 4845 if ((vp->v_iflag & VI_DEFINACT) == 0) { 4846 VI_UNLOCK(vp); 4847 continue; 4848 } 4849 vp->v_iflag &= ~VI_DEFINACT; 4850 vfs_deferred_inactive(vp, lkflags); 4851 } 4852 } 4853 4854 static inline bool 4855 vfs_want_msync(struct vnode *vp) 4856 { 4857 struct vm_object *obj; 4858 4859 /* 4860 * This test may be performed without any locks held. 4861 * We rely on vm_object's type stability. 4862 */ 4863 if (vp->v_vflag & VV_NOSYNC) 4864 return (false); 4865 obj = vp->v_object; 4866 return (obj != NULL && vm_object_mightbedirty(obj)); 4867 } 4868 4869 static int 4870 vfs_periodic_msync_inactive_filter(struct vnode *vp, void *arg __unused) 4871 { 4872 4873 if (vp->v_vflag & VV_NOSYNC) 4874 return (false); 4875 if (vp->v_iflag & VI_DEFINACT) 4876 return (true); 4877 return (vfs_want_msync(vp)); 4878 } 4879 4880 static void __noinline 4881 vfs_periodic_msync_inactive(struct mount *mp, int flags) 4882 { 4883 struct vnode *vp, *mvp; 4884 struct vm_object *obj; 4885 int lkflags, objflags; 4886 bool seen_defer; 4887 4888 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 4889 if (flags != MNT_WAIT) { 4890 lkflags |= LK_NOWAIT; 4891 objflags = OBJPC_NOSYNC; 4892 } else { 4893 objflags = OBJPC_SYNC; 4894 } 4895 4896 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_msync_inactive_filter, NULL) { 4897 seen_defer = false; 4898 if (vp->v_iflag & VI_DEFINACT) { 4899 vp->v_iflag &= ~VI_DEFINACT; 4900 seen_defer = true; 4901 } 4902 if (!vfs_want_msync(vp)) { 4903 if (seen_defer) 4904 vfs_deferred_inactive(vp, lkflags); 4905 else 4906 VI_UNLOCK(vp); 4907 continue; 4908 } 4909 if (vget(vp, lkflags) == 0) { 4910 obj = vp->v_object; 4911 if (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0) { 4912 VM_OBJECT_WLOCK(obj); 4913 vm_object_page_clean(obj, 0, 0, objflags); 4914 VM_OBJECT_WUNLOCK(obj); 4915 } 4916 vput(vp); 4917 if (seen_defer) 4918 vdrop(vp); 4919 } else { 4920 if (seen_defer) 4921 vdefer_inactive_unlocked(vp); 4922 } 4923 } 4924 } 4925 4926 void 4927 vfs_periodic(struct mount *mp, int flags) 4928 { 4929 4930 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 4931 4932 if ((mp->mnt_kern_flag & MNTK_NOMSYNC) != 0) 4933 vfs_periodic_inactive(mp, flags); 4934 else 4935 vfs_periodic_msync_inactive(mp, flags); 4936 } 4937 4938 static void 4939 destroy_vpollinfo_free(struct vpollinfo *vi) 4940 { 4941 4942 knlist_destroy(&vi->vpi_selinfo.si_note); 4943 mtx_destroy(&vi->vpi_lock); 4944 free(vi, M_VNODEPOLL); 4945 } 4946 4947 static void 4948 destroy_vpollinfo(struct vpollinfo *vi) 4949 { 4950 4951 knlist_clear(&vi->vpi_selinfo.si_note, 1); 4952 seldrain(&vi->vpi_selinfo); 4953 destroy_vpollinfo_free(vi); 4954 } 4955 4956 /* 4957 * Initialize per-vnode helper structure to hold poll-related state. 4958 */ 4959 void 4960 v_addpollinfo(struct vnode *vp) 4961 { 4962 struct vpollinfo *vi; 4963 4964 if (vp->v_pollinfo != NULL) 4965 return; 4966 vi = malloc(sizeof(*vi), M_VNODEPOLL, M_WAITOK | M_ZERO); 4967 mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF); 4968 knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock, 4969 vfs_knlunlock, vfs_knl_assert_lock); 4970 VI_LOCK(vp); 4971 if (vp->v_pollinfo != NULL) { 4972 VI_UNLOCK(vp); 4973 destroy_vpollinfo_free(vi); 4974 return; 4975 } 4976 vp->v_pollinfo = vi; 4977 VI_UNLOCK(vp); 4978 } 4979 4980 /* 4981 * Record a process's interest in events which might happen to 4982 * a vnode. Because poll uses the historic select-style interface 4983 * internally, this routine serves as both the ``check for any 4984 * pending events'' and the ``record my interest in future events'' 4985 * functions. (These are done together, while the lock is held, 4986 * to avoid race conditions.) 4987 */ 4988 int 4989 vn_pollrecord(struct vnode *vp, struct thread *td, int events) 4990 { 4991 4992 v_addpollinfo(vp); 4993 mtx_lock(&vp->v_pollinfo->vpi_lock); 4994 if (vp->v_pollinfo->vpi_revents & events) { 4995 /* 4996 * This leaves events we are not interested 4997 * in available for the other process which 4998 * which presumably had requested them 4999 * (otherwise they would never have been 5000 * recorded). 5001 */ 5002 events &= vp->v_pollinfo->vpi_revents; 5003 vp->v_pollinfo->vpi_revents &= ~events; 5004 5005 mtx_unlock(&vp->v_pollinfo->vpi_lock); 5006 return (events); 5007 } 5008 vp->v_pollinfo->vpi_events |= events; 5009 selrecord(td, &vp->v_pollinfo->vpi_selinfo); 5010 mtx_unlock(&vp->v_pollinfo->vpi_lock); 5011 return (0); 5012 } 5013 5014 /* 5015 * Routine to create and manage a filesystem syncer vnode. 5016 */ 5017 #define sync_close ((int (*)(struct vop_close_args *))nullop) 5018 static int sync_fsync(struct vop_fsync_args *); 5019 static int sync_inactive(struct vop_inactive_args *); 5020 static int sync_reclaim(struct vop_reclaim_args *); 5021 5022 static struct vop_vector sync_vnodeops = { 5023 .vop_bypass = VOP_EOPNOTSUPP, 5024 .vop_close = sync_close, /* close */ 5025 .vop_fsync = sync_fsync, /* fsync */ 5026 .vop_inactive = sync_inactive, /* inactive */ 5027 .vop_need_inactive = vop_stdneed_inactive, /* need_inactive */ 5028 .vop_reclaim = sync_reclaim, /* reclaim */ 5029 .vop_lock1 = vop_stdlock, /* lock */ 5030 .vop_unlock = vop_stdunlock, /* unlock */ 5031 .vop_islocked = vop_stdislocked, /* islocked */ 5032 }; 5033 VFS_VOP_VECTOR_REGISTER(sync_vnodeops); 5034 5035 /* 5036 * Create a new filesystem syncer vnode for the specified mount point. 5037 */ 5038 void 5039 vfs_allocate_syncvnode(struct mount *mp) 5040 { 5041 struct vnode *vp; 5042 struct bufobj *bo; 5043 static long start, incr, next; 5044 int error; 5045 5046 /* Allocate a new vnode */ 5047 error = getnewvnode("syncer", mp, &sync_vnodeops, &vp); 5048 if (error != 0) 5049 panic("vfs_allocate_syncvnode: getnewvnode() failed"); 5050 vp->v_type = VNON; 5051 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5052 vp->v_vflag |= VV_FORCEINSMQ; 5053 error = insmntque1(vp, mp); 5054 if (error != 0) 5055 panic("vfs_allocate_syncvnode: insmntque() failed"); 5056 vp->v_vflag &= ~VV_FORCEINSMQ; 5057 VOP_UNLOCK(vp); 5058 /* 5059 * Place the vnode onto the syncer worklist. We attempt to 5060 * scatter them about on the list so that they will go off 5061 * at evenly distributed times even if all the filesystems 5062 * are mounted at once. 5063 */ 5064 next += incr; 5065 if (next == 0 || next > syncer_maxdelay) { 5066 start /= 2; 5067 incr /= 2; 5068 if (start == 0) { 5069 start = syncer_maxdelay / 2; 5070 incr = syncer_maxdelay; 5071 } 5072 next = start; 5073 } 5074 bo = &vp->v_bufobj; 5075 BO_LOCK(bo); 5076 vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0); 5077 /* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */ 5078 mtx_lock(&sync_mtx); 5079 sync_vnode_count++; 5080 if (mp->mnt_syncer == NULL) { 5081 mp->mnt_syncer = vp; 5082 vp = NULL; 5083 } 5084 mtx_unlock(&sync_mtx); 5085 BO_UNLOCK(bo); 5086 if (vp != NULL) { 5087 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5088 vgone(vp); 5089 vput(vp); 5090 } 5091 } 5092 5093 void 5094 vfs_deallocate_syncvnode(struct mount *mp) 5095 { 5096 struct vnode *vp; 5097 5098 mtx_lock(&sync_mtx); 5099 vp = mp->mnt_syncer; 5100 if (vp != NULL) 5101 mp->mnt_syncer = NULL; 5102 mtx_unlock(&sync_mtx); 5103 if (vp != NULL) 5104 vrele(vp); 5105 } 5106 5107 /* 5108 * Do a lazy sync of the filesystem. 5109 */ 5110 static int 5111 sync_fsync(struct vop_fsync_args *ap) 5112 { 5113 struct vnode *syncvp = ap->a_vp; 5114 struct mount *mp = syncvp->v_mount; 5115 int error, save; 5116 struct bufobj *bo; 5117 5118 /* 5119 * We only need to do something if this is a lazy evaluation. 5120 */ 5121 if (ap->a_waitfor != MNT_LAZY) 5122 return (0); 5123 5124 /* 5125 * Move ourselves to the back of the sync list. 5126 */ 5127 bo = &syncvp->v_bufobj; 5128 BO_LOCK(bo); 5129 vn_syncer_add_to_worklist(bo, syncdelay); 5130 BO_UNLOCK(bo); 5131 5132 /* 5133 * Walk the list of vnodes pushing all that are dirty and 5134 * not already on the sync list. 5135 */ 5136 if (vfs_busy(mp, MBF_NOWAIT) != 0) 5137 return (0); 5138 VOP_UNLOCK(syncvp); 5139 save = curthread_pflags_set(TDP_SYNCIO); 5140 /* 5141 * The filesystem at hand may be idle with free vnodes stored in the 5142 * batch. Return them instead of letting them stay there indefinitely. 5143 */ 5144 vfs_periodic(mp, MNT_NOWAIT); 5145 error = VFS_SYNC(mp, MNT_LAZY); 5146 curthread_pflags_restore(save); 5147 vn_lock(syncvp, LK_EXCLUSIVE | LK_RETRY); 5148 vfs_unbusy(mp); 5149 return (error); 5150 } 5151 5152 /* 5153 * The syncer vnode is no referenced. 5154 */ 5155 static int 5156 sync_inactive(struct vop_inactive_args *ap) 5157 { 5158 5159 vgone(ap->a_vp); 5160 return (0); 5161 } 5162 5163 /* 5164 * The syncer vnode is no longer needed and is being decommissioned. 5165 * 5166 * Modifications to the worklist must be protected by sync_mtx. 5167 */ 5168 static int 5169 sync_reclaim(struct vop_reclaim_args *ap) 5170 { 5171 struct vnode *vp = ap->a_vp; 5172 struct bufobj *bo; 5173 5174 bo = &vp->v_bufobj; 5175 BO_LOCK(bo); 5176 mtx_lock(&sync_mtx); 5177 if (vp->v_mount->mnt_syncer == vp) 5178 vp->v_mount->mnt_syncer = NULL; 5179 if (bo->bo_flag & BO_ONWORKLST) { 5180 LIST_REMOVE(bo, bo_synclist); 5181 syncer_worklist_len--; 5182 sync_vnode_count--; 5183 bo->bo_flag &= ~BO_ONWORKLST; 5184 } 5185 mtx_unlock(&sync_mtx); 5186 BO_UNLOCK(bo); 5187 5188 return (0); 5189 } 5190 5191 int 5192 vn_need_pageq_flush(struct vnode *vp) 5193 { 5194 struct vm_object *obj; 5195 5196 obj = vp->v_object; 5197 return (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 5198 vm_object_mightbedirty(obj)); 5199 } 5200 5201 /* 5202 * Check if vnode represents a disk device 5203 */ 5204 bool 5205 vn_isdisk_error(struct vnode *vp, int *errp) 5206 { 5207 int error; 5208 5209 if (vp->v_type != VCHR) { 5210 error = ENOTBLK; 5211 goto out; 5212 } 5213 error = 0; 5214 dev_lock(); 5215 if (vp->v_rdev == NULL) 5216 error = ENXIO; 5217 else if (vp->v_rdev->si_devsw == NULL) 5218 error = ENXIO; 5219 else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK)) 5220 error = ENOTBLK; 5221 dev_unlock(); 5222 out: 5223 *errp = error; 5224 return (error == 0); 5225 } 5226 5227 bool 5228 vn_isdisk(struct vnode *vp) 5229 { 5230 int error; 5231 5232 return (vn_isdisk_error(vp, &error)); 5233 } 5234 5235 /* 5236 * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see 5237 * the comment above cache_fplookup for details. 5238 */ 5239 int 5240 vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid, struct ucred *cred) 5241 { 5242 int error; 5243 5244 VFS_SMR_ASSERT_ENTERED(); 5245 5246 /* Check the owner. */ 5247 if (cred->cr_uid == file_uid) { 5248 if (file_mode & S_IXUSR) 5249 return (0); 5250 goto out_error; 5251 } 5252 5253 /* Otherwise, check the groups (first match) */ 5254 if (groupmember(file_gid, cred)) { 5255 if (file_mode & S_IXGRP) 5256 return (0); 5257 goto out_error; 5258 } 5259 5260 /* Otherwise, check everyone else. */ 5261 if (file_mode & S_IXOTH) 5262 return (0); 5263 out_error: 5264 /* 5265 * Permission check failed, but it is possible denial will get overwritten 5266 * (e.g., when root is traversing through a 700 directory owned by someone 5267 * else). 5268 * 5269 * vaccess() calls priv_check_cred which in turn can descent into MAC 5270 * modules overriding this result. It's quite unclear what semantics 5271 * are allowed for them to operate, thus for safety we don't call them 5272 * from within the SMR section. This also means if any such modules 5273 * are present, we have to let the regular lookup decide. 5274 */ 5275 error = priv_check_cred_vfs_lookup_nomac(cred); 5276 switch (error) { 5277 case 0: 5278 return (0); 5279 case EAGAIN: 5280 /* 5281 * MAC modules present. 5282 */ 5283 return (EAGAIN); 5284 case EPERM: 5285 return (EACCES); 5286 default: 5287 return (error); 5288 } 5289 } 5290 5291 /* 5292 * Common filesystem object access control check routine. Accepts a 5293 * vnode's type, "mode", uid and gid, requested access mode, and credentials. 5294 * Returns 0 on success, or an errno on failure. 5295 */ 5296 int 5297 vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid, 5298 accmode_t accmode, struct ucred *cred) 5299 { 5300 accmode_t dac_granted; 5301 accmode_t priv_granted; 5302 5303 KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0, 5304 ("invalid bit in accmode")); 5305 KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE), 5306 ("VAPPEND without VWRITE")); 5307 5308 /* 5309 * Look for a normal, non-privileged way to access the file/directory 5310 * as requested. If it exists, go with that. 5311 */ 5312 5313 dac_granted = 0; 5314 5315 /* Check the owner. */ 5316 if (cred->cr_uid == file_uid) { 5317 dac_granted |= VADMIN; 5318 if (file_mode & S_IXUSR) 5319 dac_granted |= VEXEC; 5320 if (file_mode & S_IRUSR) 5321 dac_granted |= VREAD; 5322 if (file_mode & S_IWUSR) 5323 dac_granted |= (VWRITE | VAPPEND); 5324 5325 if ((accmode & dac_granted) == accmode) 5326 return (0); 5327 5328 goto privcheck; 5329 } 5330 5331 /* Otherwise, check the groups (first match) */ 5332 if (groupmember(file_gid, cred)) { 5333 if (file_mode & S_IXGRP) 5334 dac_granted |= VEXEC; 5335 if (file_mode & S_IRGRP) 5336 dac_granted |= VREAD; 5337 if (file_mode & S_IWGRP) 5338 dac_granted |= (VWRITE | VAPPEND); 5339 5340 if ((accmode & dac_granted) == accmode) 5341 return (0); 5342 5343 goto privcheck; 5344 } 5345 5346 /* Otherwise, check everyone else. */ 5347 if (file_mode & S_IXOTH) 5348 dac_granted |= VEXEC; 5349 if (file_mode & S_IROTH) 5350 dac_granted |= VREAD; 5351 if (file_mode & S_IWOTH) 5352 dac_granted |= (VWRITE | VAPPEND); 5353 if ((accmode & dac_granted) == accmode) 5354 return (0); 5355 5356 privcheck: 5357 /* 5358 * Build a privilege mask to determine if the set of privileges 5359 * satisfies the requirements when combined with the granted mask 5360 * from above. For each privilege, if the privilege is required, 5361 * bitwise or the request type onto the priv_granted mask. 5362 */ 5363 priv_granted = 0; 5364 5365 if (type == VDIR) { 5366 /* 5367 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC 5368 * requests, instead of PRIV_VFS_EXEC. 5369 */ 5370 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5371 !priv_check_cred(cred, PRIV_VFS_LOOKUP)) 5372 priv_granted |= VEXEC; 5373 } else { 5374 /* 5375 * Ensure that at least one execute bit is on. Otherwise, 5376 * a privileged user will always succeed, and we don't want 5377 * this to happen unless the file really is executable. 5378 */ 5379 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5380 (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && 5381 !priv_check_cred(cred, PRIV_VFS_EXEC)) 5382 priv_granted |= VEXEC; 5383 } 5384 5385 if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) && 5386 !priv_check_cred(cred, PRIV_VFS_READ)) 5387 priv_granted |= VREAD; 5388 5389 if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) && 5390 !priv_check_cred(cred, PRIV_VFS_WRITE)) 5391 priv_granted |= (VWRITE | VAPPEND); 5392 5393 if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) && 5394 !priv_check_cred(cred, PRIV_VFS_ADMIN)) 5395 priv_granted |= VADMIN; 5396 5397 if ((accmode & (priv_granted | dac_granted)) == accmode) { 5398 return (0); 5399 } 5400 5401 return ((accmode & VADMIN) ? EPERM : EACCES); 5402 } 5403 5404 /* 5405 * Credential check based on process requesting service, and per-attribute 5406 * permissions. 5407 */ 5408 int 5409 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred, 5410 struct thread *td, accmode_t accmode) 5411 { 5412 5413 /* 5414 * Kernel-invoked always succeeds. 5415 */ 5416 if (cred == NOCRED) 5417 return (0); 5418 5419 /* 5420 * Do not allow privileged processes in jail to directly manipulate 5421 * system attributes. 5422 */ 5423 switch (attrnamespace) { 5424 case EXTATTR_NAMESPACE_SYSTEM: 5425 /* Potentially should be: return (EPERM); */ 5426 return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM)); 5427 case EXTATTR_NAMESPACE_USER: 5428 return (VOP_ACCESS(vp, accmode, cred, td)); 5429 default: 5430 return (EPERM); 5431 } 5432 } 5433 5434 #ifdef DEBUG_VFS_LOCKS 5435 int vfs_badlock_ddb = 1; /* Drop into debugger on violation. */ 5436 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0, 5437 "Drop into debugger on lock violation"); 5438 5439 int vfs_badlock_mutex = 1; /* Check for interlock across VOPs. */ 5440 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex, 5441 0, "Check for interlock across VOPs"); 5442 5443 int vfs_badlock_print = 1; /* Print lock violations. */ 5444 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print, 5445 0, "Print lock violations"); 5446 5447 int vfs_badlock_vnode = 1; /* Print vnode details on lock violations. */ 5448 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_vnode, CTLFLAG_RW, &vfs_badlock_vnode, 5449 0, "Print vnode details on lock violations"); 5450 5451 #ifdef KDB 5452 int vfs_badlock_backtrace = 1; /* Print backtrace at lock violations. */ 5453 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW, 5454 &vfs_badlock_backtrace, 0, "Print backtrace at lock violations"); 5455 #endif 5456 5457 static void 5458 vfs_badlock(const char *msg, const char *str, struct vnode *vp) 5459 { 5460 5461 #ifdef KDB 5462 if (vfs_badlock_backtrace) 5463 kdb_backtrace(); 5464 #endif 5465 if (vfs_badlock_vnode) 5466 vn_printf(vp, "vnode "); 5467 if (vfs_badlock_print) 5468 printf("%s: %p %s\n", str, (void *)vp, msg); 5469 if (vfs_badlock_ddb) 5470 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5471 } 5472 5473 void 5474 assert_vi_locked(struct vnode *vp, const char *str) 5475 { 5476 5477 if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp))) 5478 vfs_badlock("interlock is not locked but should be", str, vp); 5479 } 5480 5481 void 5482 assert_vi_unlocked(struct vnode *vp, const char *str) 5483 { 5484 5485 if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp))) 5486 vfs_badlock("interlock is locked but should not be", str, vp); 5487 } 5488 5489 void 5490 assert_vop_locked(struct vnode *vp, const char *str) 5491 { 5492 int locked; 5493 5494 if (KERNEL_PANICKED() || vp == NULL) 5495 return; 5496 5497 locked = VOP_ISLOCKED(vp); 5498 if (locked == 0 || locked == LK_EXCLOTHER) 5499 vfs_badlock("is not locked but should be", str, vp); 5500 } 5501 5502 void 5503 assert_vop_unlocked(struct vnode *vp, const char *str) 5504 { 5505 if (KERNEL_PANICKED() || vp == NULL) 5506 return; 5507 5508 if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE) 5509 vfs_badlock("is locked but should not be", str, vp); 5510 } 5511 5512 void 5513 assert_vop_elocked(struct vnode *vp, const char *str) 5514 { 5515 if (KERNEL_PANICKED() || vp == NULL) 5516 return; 5517 5518 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) 5519 vfs_badlock("is not exclusive locked but should be", str, vp); 5520 } 5521 #endif /* DEBUG_VFS_LOCKS */ 5522 5523 void 5524 vop_rename_fail(struct vop_rename_args *ap) 5525 { 5526 5527 if (ap->a_tvp != NULL) 5528 vput(ap->a_tvp); 5529 if (ap->a_tdvp == ap->a_tvp) 5530 vrele(ap->a_tdvp); 5531 else 5532 vput(ap->a_tdvp); 5533 vrele(ap->a_fdvp); 5534 vrele(ap->a_fvp); 5535 } 5536 5537 void 5538 vop_rename_pre(void *ap) 5539 { 5540 struct vop_rename_args *a = ap; 5541 5542 #ifdef DEBUG_VFS_LOCKS 5543 if (a->a_tvp) 5544 ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME"); 5545 ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME"); 5546 ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME"); 5547 ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME"); 5548 5549 /* Check the source (from). */ 5550 if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock && 5551 (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock)) 5552 ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked"); 5553 if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock) 5554 ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked"); 5555 5556 /* Check the target. */ 5557 if (a->a_tvp) 5558 ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked"); 5559 ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked"); 5560 #endif 5561 /* 5562 * It may be tempting to add vn_seqc_write_begin/end calls here and 5563 * in vop_rename_post but that's not going to work out since some 5564 * filesystems relookup vnodes mid-rename. This is probably a bug. 5565 * 5566 * For now filesystems are expected to do the relevant calls after they 5567 * decide what vnodes to operate on. 5568 */ 5569 if (a->a_tdvp != a->a_fdvp) 5570 vhold(a->a_fdvp); 5571 if (a->a_tvp != a->a_fvp) 5572 vhold(a->a_fvp); 5573 vhold(a->a_tdvp); 5574 if (a->a_tvp) 5575 vhold(a->a_tvp); 5576 } 5577 5578 #ifdef DEBUG_VFS_LOCKS 5579 void 5580 vop_fplookup_vexec_debugpre(void *ap __unused) 5581 { 5582 5583 VFS_SMR_ASSERT_ENTERED(); 5584 } 5585 5586 void 5587 vop_fplookup_vexec_debugpost(void *ap __unused, int rc __unused) 5588 { 5589 5590 VFS_SMR_ASSERT_ENTERED(); 5591 } 5592 5593 void 5594 vop_fplookup_symlink_debugpre(void *ap __unused) 5595 { 5596 5597 VFS_SMR_ASSERT_ENTERED(); 5598 } 5599 5600 void 5601 vop_fplookup_symlink_debugpost(void *ap __unused, int rc __unused) 5602 { 5603 5604 VFS_SMR_ASSERT_ENTERED(); 5605 } 5606 5607 static void 5608 vop_fsync_debugprepost(struct vnode *vp, const char *name) 5609 { 5610 if (vp->v_type == VCHR) 5611 ; 5612 else if (MNT_EXTENDED_SHARED(vp->v_mount)) 5613 ASSERT_VOP_LOCKED(vp, name); 5614 else 5615 ASSERT_VOP_ELOCKED(vp, name); 5616 } 5617 5618 void 5619 vop_fsync_debugpre(void *a) 5620 { 5621 struct vop_fsync_args *ap; 5622 5623 ap = a; 5624 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5625 } 5626 5627 void 5628 vop_fsync_debugpost(void *a, int rc __unused) 5629 { 5630 struct vop_fsync_args *ap; 5631 5632 ap = a; 5633 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5634 } 5635 5636 void 5637 vop_fdatasync_debugpre(void *a) 5638 { 5639 struct vop_fdatasync_args *ap; 5640 5641 ap = a; 5642 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5643 } 5644 5645 void 5646 vop_fdatasync_debugpost(void *a, int rc __unused) 5647 { 5648 struct vop_fdatasync_args *ap; 5649 5650 ap = a; 5651 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5652 } 5653 5654 void 5655 vop_strategy_debugpre(void *ap) 5656 { 5657 struct vop_strategy_args *a; 5658 struct buf *bp; 5659 5660 a = ap; 5661 bp = a->a_bp; 5662 5663 /* 5664 * Cluster ops lock their component buffers but not the IO container. 5665 */ 5666 if ((bp->b_flags & B_CLUSTER) != 0) 5667 return; 5668 5669 if (!KERNEL_PANICKED() && !BUF_ISLOCKED(bp)) { 5670 if (vfs_badlock_print) 5671 printf( 5672 "VOP_STRATEGY: bp is not locked but should be\n"); 5673 if (vfs_badlock_ddb) 5674 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5675 } 5676 } 5677 5678 void 5679 vop_lock_debugpre(void *ap) 5680 { 5681 struct vop_lock1_args *a = ap; 5682 5683 if ((a->a_flags & LK_INTERLOCK) == 0) 5684 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 5685 else 5686 ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK"); 5687 } 5688 5689 void 5690 vop_lock_debugpost(void *ap, int rc) 5691 { 5692 struct vop_lock1_args *a = ap; 5693 5694 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 5695 if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0) 5696 ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK"); 5697 } 5698 5699 void 5700 vop_unlock_debugpre(void *ap) 5701 { 5702 struct vop_unlock_args *a = ap; 5703 5704 ASSERT_VOP_LOCKED(a->a_vp, "VOP_UNLOCK"); 5705 } 5706 5707 void 5708 vop_need_inactive_debugpre(void *ap) 5709 { 5710 struct vop_need_inactive_args *a = ap; 5711 5712 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 5713 } 5714 5715 void 5716 vop_need_inactive_debugpost(void *ap, int rc) 5717 { 5718 struct vop_need_inactive_args *a = ap; 5719 5720 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 5721 } 5722 #endif 5723 5724 void 5725 vop_create_pre(void *ap) 5726 { 5727 struct vop_create_args *a; 5728 struct vnode *dvp; 5729 5730 a = ap; 5731 dvp = a->a_dvp; 5732 vn_seqc_write_begin(dvp); 5733 } 5734 5735 void 5736 vop_create_post(void *ap, int rc) 5737 { 5738 struct vop_create_args *a; 5739 struct vnode *dvp; 5740 5741 a = ap; 5742 dvp = a->a_dvp; 5743 vn_seqc_write_end(dvp); 5744 if (!rc) 5745 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5746 } 5747 5748 void 5749 vop_whiteout_pre(void *ap) 5750 { 5751 struct vop_whiteout_args *a; 5752 struct vnode *dvp; 5753 5754 a = ap; 5755 dvp = a->a_dvp; 5756 vn_seqc_write_begin(dvp); 5757 } 5758 5759 void 5760 vop_whiteout_post(void *ap, int rc) 5761 { 5762 struct vop_whiteout_args *a; 5763 struct vnode *dvp; 5764 5765 a = ap; 5766 dvp = a->a_dvp; 5767 vn_seqc_write_end(dvp); 5768 } 5769 5770 void 5771 vop_deleteextattr_pre(void *ap) 5772 { 5773 struct vop_deleteextattr_args *a; 5774 struct vnode *vp; 5775 5776 a = ap; 5777 vp = a->a_vp; 5778 vn_seqc_write_begin(vp); 5779 } 5780 5781 void 5782 vop_deleteextattr_post(void *ap, int rc) 5783 { 5784 struct vop_deleteextattr_args *a; 5785 struct vnode *vp; 5786 5787 a = ap; 5788 vp = a->a_vp; 5789 vn_seqc_write_end(vp); 5790 if (!rc) 5791 VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB); 5792 } 5793 5794 void 5795 vop_link_pre(void *ap) 5796 { 5797 struct vop_link_args *a; 5798 struct vnode *vp, *tdvp; 5799 5800 a = ap; 5801 vp = a->a_vp; 5802 tdvp = a->a_tdvp; 5803 vn_seqc_write_begin(vp); 5804 vn_seqc_write_begin(tdvp); 5805 } 5806 5807 void 5808 vop_link_post(void *ap, int rc) 5809 { 5810 struct vop_link_args *a; 5811 struct vnode *vp, *tdvp; 5812 5813 a = ap; 5814 vp = a->a_vp; 5815 tdvp = a->a_tdvp; 5816 vn_seqc_write_end(vp); 5817 vn_seqc_write_end(tdvp); 5818 if (!rc) { 5819 VFS_KNOTE_LOCKED(vp, NOTE_LINK); 5820 VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE); 5821 } 5822 } 5823 5824 void 5825 vop_mkdir_pre(void *ap) 5826 { 5827 struct vop_mkdir_args *a; 5828 struct vnode *dvp; 5829 5830 a = ap; 5831 dvp = a->a_dvp; 5832 vn_seqc_write_begin(dvp); 5833 } 5834 5835 void 5836 vop_mkdir_post(void *ap, int rc) 5837 { 5838 struct vop_mkdir_args *a; 5839 struct vnode *dvp; 5840 5841 a = ap; 5842 dvp = a->a_dvp; 5843 vn_seqc_write_end(dvp); 5844 if (!rc) 5845 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 5846 } 5847 5848 #ifdef DEBUG_VFS_LOCKS 5849 void 5850 vop_mkdir_debugpost(void *ap, int rc) 5851 { 5852 struct vop_mkdir_args *a; 5853 5854 a = ap; 5855 if (!rc) 5856 cache_validate(a->a_dvp, *a->a_vpp, a->a_cnp); 5857 } 5858 #endif 5859 5860 void 5861 vop_mknod_pre(void *ap) 5862 { 5863 struct vop_mknod_args *a; 5864 struct vnode *dvp; 5865 5866 a = ap; 5867 dvp = a->a_dvp; 5868 vn_seqc_write_begin(dvp); 5869 } 5870 5871 void 5872 vop_mknod_post(void *ap, int rc) 5873 { 5874 struct vop_mknod_args *a; 5875 struct vnode *dvp; 5876 5877 a = ap; 5878 dvp = a->a_dvp; 5879 vn_seqc_write_end(dvp); 5880 if (!rc) 5881 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5882 } 5883 5884 void 5885 vop_reclaim_post(void *ap, int rc) 5886 { 5887 struct vop_reclaim_args *a; 5888 struct vnode *vp; 5889 5890 a = ap; 5891 vp = a->a_vp; 5892 ASSERT_VOP_IN_SEQC(vp); 5893 if (!rc) 5894 VFS_KNOTE_LOCKED(vp, NOTE_REVOKE); 5895 } 5896 5897 void 5898 vop_remove_pre(void *ap) 5899 { 5900 struct vop_remove_args *a; 5901 struct vnode *dvp, *vp; 5902 5903 a = ap; 5904 dvp = a->a_dvp; 5905 vp = a->a_vp; 5906 vn_seqc_write_begin(dvp); 5907 vn_seqc_write_begin(vp); 5908 } 5909 5910 void 5911 vop_remove_post(void *ap, int rc) 5912 { 5913 struct vop_remove_args *a; 5914 struct vnode *dvp, *vp; 5915 5916 a = ap; 5917 dvp = a->a_dvp; 5918 vp = a->a_vp; 5919 vn_seqc_write_end(dvp); 5920 vn_seqc_write_end(vp); 5921 if (!rc) { 5922 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5923 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 5924 } 5925 } 5926 5927 void 5928 vop_rename_post(void *ap, int rc) 5929 { 5930 struct vop_rename_args *a = ap; 5931 long hint; 5932 5933 if (!rc) { 5934 hint = NOTE_WRITE; 5935 if (a->a_fdvp == a->a_tdvp) { 5936 if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR) 5937 hint |= NOTE_LINK; 5938 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 5939 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 5940 } else { 5941 hint |= NOTE_EXTEND; 5942 if (a->a_fvp->v_type == VDIR) 5943 hint |= NOTE_LINK; 5944 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 5945 5946 if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL && 5947 a->a_tvp->v_type == VDIR) 5948 hint &= ~NOTE_LINK; 5949 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 5950 } 5951 5952 VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME); 5953 if (a->a_tvp) 5954 VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE); 5955 } 5956 if (a->a_tdvp != a->a_fdvp) 5957 vdrop(a->a_fdvp); 5958 if (a->a_tvp != a->a_fvp) 5959 vdrop(a->a_fvp); 5960 vdrop(a->a_tdvp); 5961 if (a->a_tvp) 5962 vdrop(a->a_tvp); 5963 } 5964 5965 void 5966 vop_rmdir_pre(void *ap) 5967 { 5968 struct vop_rmdir_args *a; 5969 struct vnode *dvp, *vp; 5970 5971 a = ap; 5972 dvp = a->a_dvp; 5973 vp = a->a_vp; 5974 vn_seqc_write_begin(dvp); 5975 vn_seqc_write_begin(vp); 5976 } 5977 5978 void 5979 vop_rmdir_post(void *ap, int rc) 5980 { 5981 struct vop_rmdir_args *a; 5982 struct vnode *dvp, *vp; 5983 5984 a = ap; 5985 dvp = a->a_dvp; 5986 vp = a->a_vp; 5987 vn_seqc_write_end(dvp); 5988 vn_seqc_write_end(vp); 5989 if (!rc) { 5990 vp->v_vflag |= VV_UNLINKED; 5991 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 5992 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 5993 } 5994 } 5995 5996 void 5997 vop_setattr_pre(void *ap) 5998 { 5999 struct vop_setattr_args *a; 6000 struct vnode *vp; 6001 6002 a = ap; 6003 vp = a->a_vp; 6004 vn_seqc_write_begin(vp); 6005 } 6006 6007 void 6008 vop_setattr_post(void *ap, int rc) 6009 { 6010 struct vop_setattr_args *a; 6011 struct vnode *vp; 6012 6013 a = ap; 6014 vp = a->a_vp; 6015 vn_seqc_write_end(vp); 6016 if (!rc) 6017 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 6018 } 6019 6020 void 6021 vop_setacl_pre(void *ap) 6022 { 6023 struct vop_setacl_args *a; 6024 struct vnode *vp; 6025 6026 a = ap; 6027 vp = a->a_vp; 6028 vn_seqc_write_begin(vp); 6029 } 6030 6031 void 6032 vop_setacl_post(void *ap, int rc __unused) 6033 { 6034 struct vop_setacl_args *a; 6035 struct vnode *vp; 6036 6037 a = ap; 6038 vp = a->a_vp; 6039 vn_seqc_write_end(vp); 6040 } 6041 6042 void 6043 vop_setextattr_pre(void *ap) 6044 { 6045 struct vop_setextattr_args *a; 6046 struct vnode *vp; 6047 6048 a = ap; 6049 vp = a->a_vp; 6050 vn_seqc_write_begin(vp); 6051 } 6052 6053 void 6054 vop_setextattr_post(void *ap, int rc) 6055 { 6056 struct vop_setextattr_args *a; 6057 struct vnode *vp; 6058 6059 a = ap; 6060 vp = a->a_vp; 6061 vn_seqc_write_end(vp); 6062 if (!rc) 6063 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 6064 } 6065 6066 void 6067 vop_symlink_pre(void *ap) 6068 { 6069 struct vop_symlink_args *a; 6070 struct vnode *dvp; 6071 6072 a = ap; 6073 dvp = a->a_dvp; 6074 vn_seqc_write_begin(dvp); 6075 } 6076 6077 void 6078 vop_symlink_post(void *ap, int rc) 6079 { 6080 struct vop_symlink_args *a; 6081 struct vnode *dvp; 6082 6083 a = ap; 6084 dvp = a->a_dvp; 6085 vn_seqc_write_end(dvp); 6086 if (!rc) 6087 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6088 } 6089 6090 void 6091 vop_open_post(void *ap, int rc) 6092 { 6093 struct vop_open_args *a = ap; 6094 6095 if (!rc) 6096 VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN); 6097 } 6098 6099 void 6100 vop_close_post(void *ap, int rc) 6101 { 6102 struct vop_close_args *a = ap; 6103 6104 if (!rc && (a->a_cred != NOCRED || /* filter out revokes */ 6105 !VN_IS_DOOMED(a->a_vp))) { 6106 VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ? 6107 NOTE_CLOSE_WRITE : NOTE_CLOSE); 6108 } 6109 } 6110 6111 void 6112 vop_read_post(void *ap, int rc) 6113 { 6114 struct vop_read_args *a = ap; 6115 6116 if (!rc) 6117 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 6118 } 6119 6120 void 6121 vop_read_pgcache_post(void *ap, int rc) 6122 { 6123 struct vop_read_pgcache_args *a = ap; 6124 6125 if (!rc) 6126 VFS_KNOTE_UNLOCKED(a->a_vp, NOTE_READ); 6127 } 6128 6129 void 6130 vop_readdir_post(void *ap, int rc) 6131 { 6132 struct vop_readdir_args *a = ap; 6133 6134 if (!rc) 6135 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 6136 } 6137 6138 static struct knlist fs_knlist; 6139 6140 static void 6141 vfs_event_init(void *arg) 6142 { 6143 knlist_init_mtx(&fs_knlist, NULL); 6144 } 6145 /* XXX - correct order? */ 6146 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL); 6147 6148 void 6149 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused) 6150 { 6151 6152 KNOTE_UNLOCKED(&fs_knlist, event); 6153 } 6154 6155 static int filt_fsattach(struct knote *kn); 6156 static void filt_fsdetach(struct knote *kn); 6157 static int filt_fsevent(struct knote *kn, long hint); 6158 6159 struct filterops fs_filtops = { 6160 .f_isfd = 0, 6161 .f_attach = filt_fsattach, 6162 .f_detach = filt_fsdetach, 6163 .f_event = filt_fsevent 6164 }; 6165 6166 static int 6167 filt_fsattach(struct knote *kn) 6168 { 6169 6170 kn->kn_flags |= EV_CLEAR; 6171 knlist_add(&fs_knlist, kn, 0); 6172 return (0); 6173 } 6174 6175 static void 6176 filt_fsdetach(struct knote *kn) 6177 { 6178 6179 knlist_remove(&fs_knlist, kn, 0); 6180 } 6181 6182 static int 6183 filt_fsevent(struct knote *kn, long hint) 6184 { 6185 6186 kn->kn_fflags |= kn->kn_sfflags & hint; 6187 6188 return (kn->kn_fflags != 0); 6189 } 6190 6191 static int 6192 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS) 6193 { 6194 struct vfsidctl vc; 6195 int error; 6196 struct mount *mp; 6197 6198 error = SYSCTL_IN(req, &vc, sizeof(vc)); 6199 if (error) 6200 return (error); 6201 if (vc.vc_vers != VFS_CTL_VERS1) 6202 return (EINVAL); 6203 mp = vfs_getvfs(&vc.vc_fsid); 6204 if (mp == NULL) 6205 return (ENOENT); 6206 /* ensure that a specific sysctl goes to the right filesystem. */ 6207 if (strcmp(vc.vc_fstypename, "*") != 0 && 6208 strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) { 6209 vfs_rel(mp); 6210 return (EINVAL); 6211 } 6212 VCTLTOREQ(&vc, req); 6213 error = VFS_SYSCTL(mp, vc.vc_op, req); 6214 vfs_rel(mp); 6215 return (error); 6216 } 6217 6218 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_MPSAFE | CTLFLAG_WR, 6219 NULL, 0, sysctl_vfs_ctl, "", 6220 "Sysctl by fsid"); 6221 6222 /* 6223 * Function to initialize a va_filerev field sensibly. 6224 * XXX: Wouldn't a random number make a lot more sense ?? 6225 */ 6226 u_quad_t 6227 init_va_filerev(void) 6228 { 6229 struct bintime bt; 6230 6231 getbinuptime(&bt); 6232 return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL)); 6233 } 6234 6235 static int filt_vfsread(struct knote *kn, long hint); 6236 static int filt_vfswrite(struct knote *kn, long hint); 6237 static int filt_vfsvnode(struct knote *kn, long hint); 6238 static void filt_vfsdetach(struct knote *kn); 6239 static struct filterops vfsread_filtops = { 6240 .f_isfd = 1, 6241 .f_detach = filt_vfsdetach, 6242 .f_event = filt_vfsread 6243 }; 6244 static struct filterops vfswrite_filtops = { 6245 .f_isfd = 1, 6246 .f_detach = filt_vfsdetach, 6247 .f_event = filt_vfswrite 6248 }; 6249 static struct filterops vfsvnode_filtops = { 6250 .f_isfd = 1, 6251 .f_detach = filt_vfsdetach, 6252 .f_event = filt_vfsvnode 6253 }; 6254 6255 static void 6256 vfs_knllock(void *arg) 6257 { 6258 struct vnode *vp = arg; 6259 6260 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 6261 } 6262 6263 static void 6264 vfs_knlunlock(void *arg) 6265 { 6266 struct vnode *vp = arg; 6267 6268 VOP_UNLOCK(vp); 6269 } 6270 6271 static void 6272 vfs_knl_assert_lock(void *arg, int what) 6273 { 6274 #ifdef DEBUG_VFS_LOCKS 6275 struct vnode *vp = arg; 6276 6277 if (what == LA_LOCKED) 6278 ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked"); 6279 else 6280 ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked"); 6281 #endif 6282 } 6283 6284 int 6285 vfs_kqfilter(struct vop_kqfilter_args *ap) 6286 { 6287 struct vnode *vp = ap->a_vp; 6288 struct knote *kn = ap->a_kn; 6289 struct knlist *knl; 6290 6291 KASSERT(vp->v_type != VFIFO || (kn->kn_filter != EVFILT_READ && 6292 kn->kn_filter != EVFILT_WRITE), 6293 ("READ/WRITE filter on a FIFO leaked through")); 6294 switch (kn->kn_filter) { 6295 case EVFILT_READ: 6296 kn->kn_fop = &vfsread_filtops; 6297 break; 6298 case EVFILT_WRITE: 6299 kn->kn_fop = &vfswrite_filtops; 6300 break; 6301 case EVFILT_VNODE: 6302 kn->kn_fop = &vfsvnode_filtops; 6303 break; 6304 default: 6305 return (EINVAL); 6306 } 6307 6308 kn->kn_hook = (caddr_t)vp; 6309 6310 v_addpollinfo(vp); 6311 if (vp->v_pollinfo == NULL) 6312 return (ENOMEM); 6313 knl = &vp->v_pollinfo->vpi_selinfo.si_note; 6314 vhold(vp); 6315 knlist_add(knl, kn, 0); 6316 6317 return (0); 6318 } 6319 6320 /* 6321 * Detach knote from vnode 6322 */ 6323 static void 6324 filt_vfsdetach(struct knote *kn) 6325 { 6326 struct vnode *vp = (struct vnode *)kn->kn_hook; 6327 6328 KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo")); 6329 knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0); 6330 vdrop(vp); 6331 } 6332 6333 /*ARGSUSED*/ 6334 static int 6335 filt_vfsread(struct knote *kn, long hint) 6336 { 6337 struct vnode *vp = (struct vnode *)kn->kn_hook; 6338 struct vattr va; 6339 int res; 6340 6341 /* 6342 * filesystem is gone, so set the EOF flag and schedule 6343 * the knote for deletion. 6344 */ 6345 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6346 VI_LOCK(vp); 6347 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6348 VI_UNLOCK(vp); 6349 return (1); 6350 } 6351 6352 if (VOP_GETATTR(vp, &va, curthread->td_ucred)) 6353 return (0); 6354 6355 VI_LOCK(vp); 6356 kn->kn_data = va.va_size - kn->kn_fp->f_offset; 6357 res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 || kn->kn_data != 0; 6358 VI_UNLOCK(vp); 6359 return (res); 6360 } 6361 6362 /*ARGSUSED*/ 6363 static int 6364 filt_vfswrite(struct knote *kn, long hint) 6365 { 6366 struct vnode *vp = (struct vnode *)kn->kn_hook; 6367 6368 VI_LOCK(vp); 6369 6370 /* 6371 * filesystem is gone, so set the EOF flag and schedule 6372 * the knote for deletion. 6373 */ 6374 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) 6375 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6376 6377 kn->kn_data = 0; 6378 VI_UNLOCK(vp); 6379 return (1); 6380 } 6381 6382 static int 6383 filt_vfsvnode(struct knote *kn, long hint) 6384 { 6385 struct vnode *vp = (struct vnode *)kn->kn_hook; 6386 int res; 6387 6388 VI_LOCK(vp); 6389 if (kn->kn_sfflags & hint) 6390 kn->kn_fflags |= hint; 6391 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6392 kn->kn_flags |= EV_EOF; 6393 VI_UNLOCK(vp); 6394 return (1); 6395 } 6396 res = (kn->kn_fflags != 0); 6397 VI_UNLOCK(vp); 6398 return (res); 6399 } 6400 6401 /* 6402 * Returns whether the directory is empty or not. 6403 * If it is empty, the return value is 0; otherwise 6404 * the return value is an error value (which may 6405 * be ENOTEMPTY). 6406 */ 6407 int 6408 vfs_emptydir(struct vnode *vp) 6409 { 6410 struct uio uio; 6411 struct iovec iov; 6412 struct dirent *dirent, *dp, *endp; 6413 int error, eof; 6414 6415 error = 0; 6416 eof = 0; 6417 6418 ASSERT_VOP_LOCKED(vp, "vfs_emptydir"); 6419 VNASSERT(vp->v_type == VDIR, vp, ("vp is not a directory")); 6420 6421 dirent = malloc(sizeof(struct dirent), M_TEMP, M_WAITOK); 6422 iov.iov_base = dirent; 6423 iov.iov_len = sizeof(struct dirent); 6424 6425 uio.uio_iov = &iov; 6426 uio.uio_iovcnt = 1; 6427 uio.uio_offset = 0; 6428 uio.uio_resid = sizeof(struct dirent); 6429 uio.uio_segflg = UIO_SYSSPACE; 6430 uio.uio_rw = UIO_READ; 6431 uio.uio_td = curthread; 6432 6433 while (eof == 0 && error == 0) { 6434 error = VOP_READDIR(vp, &uio, curthread->td_ucred, &eof, 6435 NULL, NULL); 6436 if (error != 0) 6437 break; 6438 endp = (void *)((uint8_t *)dirent + 6439 sizeof(struct dirent) - uio.uio_resid); 6440 for (dp = dirent; dp < endp; 6441 dp = (void *)((uint8_t *)dp + GENERIC_DIRSIZ(dp))) { 6442 if (dp->d_type == DT_WHT) 6443 continue; 6444 if (dp->d_namlen == 0) 6445 continue; 6446 if (dp->d_type != DT_DIR && 6447 dp->d_type != DT_UNKNOWN) { 6448 error = ENOTEMPTY; 6449 break; 6450 } 6451 if (dp->d_namlen > 2) { 6452 error = ENOTEMPTY; 6453 break; 6454 } 6455 if (dp->d_namlen == 1 && 6456 dp->d_name[0] != '.') { 6457 error = ENOTEMPTY; 6458 break; 6459 } 6460 if (dp->d_namlen == 2 && 6461 dp->d_name[1] != '.') { 6462 error = ENOTEMPTY; 6463 break; 6464 } 6465 uio.uio_resid = sizeof(struct dirent); 6466 } 6467 } 6468 free(dirent, M_TEMP); 6469 return (error); 6470 } 6471 6472 int 6473 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off) 6474 { 6475 int error; 6476 6477 if (dp->d_reclen > ap->a_uio->uio_resid) 6478 return (ENAMETOOLONG); 6479 error = uiomove(dp, dp->d_reclen, ap->a_uio); 6480 if (error) { 6481 if (ap->a_ncookies != NULL) { 6482 if (ap->a_cookies != NULL) 6483 free(ap->a_cookies, M_TEMP); 6484 ap->a_cookies = NULL; 6485 *ap->a_ncookies = 0; 6486 } 6487 return (error); 6488 } 6489 if (ap->a_ncookies == NULL) 6490 return (0); 6491 6492 KASSERT(ap->a_cookies, 6493 ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!")); 6494 6495 *ap->a_cookies = realloc(*ap->a_cookies, 6496 (*ap->a_ncookies + 1) * sizeof(uint64_t), M_TEMP, M_WAITOK | M_ZERO); 6497 (*ap->a_cookies)[*ap->a_ncookies] = off; 6498 *ap->a_ncookies += 1; 6499 return (0); 6500 } 6501 6502 /* 6503 * The purpose of this routine is to remove granularity from accmode_t, 6504 * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE, 6505 * VADMIN and VAPPEND. 6506 * 6507 * If it returns 0, the caller is supposed to continue with the usual 6508 * access checks using 'accmode' as modified by this routine. If it 6509 * returns nonzero value, the caller is supposed to return that value 6510 * as errno. 6511 * 6512 * Note that after this routine runs, accmode may be zero. 6513 */ 6514 int 6515 vfs_unixify_accmode(accmode_t *accmode) 6516 { 6517 /* 6518 * There is no way to specify explicit "deny" rule using 6519 * file mode or POSIX.1e ACLs. 6520 */ 6521 if (*accmode & VEXPLICIT_DENY) { 6522 *accmode = 0; 6523 return (0); 6524 } 6525 6526 /* 6527 * None of these can be translated into usual access bits. 6528 * Also, the common case for NFSv4 ACLs is to not contain 6529 * either of these bits. Caller should check for VWRITE 6530 * on the containing directory instead. 6531 */ 6532 if (*accmode & (VDELETE_CHILD | VDELETE)) 6533 return (EPERM); 6534 6535 if (*accmode & VADMIN_PERMS) { 6536 *accmode &= ~VADMIN_PERMS; 6537 *accmode |= VADMIN; 6538 } 6539 6540 /* 6541 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL 6542 * or VSYNCHRONIZE using file mode or POSIX.1e ACL. 6543 */ 6544 *accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE); 6545 6546 return (0); 6547 } 6548 6549 /* 6550 * Clear out a doomed vnode (if any) and replace it with a new one as long 6551 * as the fs is not being unmounted. Return the root vnode to the caller. 6552 */ 6553 static int __noinline 6554 vfs_cache_root_fallback(struct mount *mp, int flags, struct vnode **vpp) 6555 { 6556 struct vnode *vp; 6557 int error; 6558 6559 restart: 6560 if (mp->mnt_rootvnode != NULL) { 6561 MNT_ILOCK(mp); 6562 vp = mp->mnt_rootvnode; 6563 if (vp != NULL) { 6564 if (!VN_IS_DOOMED(vp)) { 6565 vrefact(vp); 6566 MNT_IUNLOCK(mp); 6567 error = vn_lock(vp, flags); 6568 if (error == 0) { 6569 *vpp = vp; 6570 return (0); 6571 } 6572 vrele(vp); 6573 goto restart; 6574 } 6575 /* 6576 * Clear the old one. 6577 */ 6578 mp->mnt_rootvnode = NULL; 6579 } 6580 MNT_IUNLOCK(mp); 6581 if (vp != NULL) { 6582 vfs_op_barrier_wait(mp); 6583 vrele(vp); 6584 } 6585 } 6586 error = VFS_CACHEDROOT(mp, flags, vpp); 6587 if (error != 0) 6588 return (error); 6589 if (mp->mnt_vfs_ops == 0) { 6590 MNT_ILOCK(mp); 6591 if (mp->mnt_vfs_ops != 0) { 6592 MNT_IUNLOCK(mp); 6593 return (0); 6594 } 6595 if (mp->mnt_rootvnode == NULL) { 6596 vrefact(*vpp); 6597 mp->mnt_rootvnode = *vpp; 6598 } else { 6599 if (mp->mnt_rootvnode != *vpp) { 6600 if (!VN_IS_DOOMED(mp->mnt_rootvnode)) { 6601 panic("%s: mismatch between vnode returned " 6602 " by VFS_CACHEDROOT and the one cached " 6603 " (%p != %p)", 6604 __func__, *vpp, mp->mnt_rootvnode); 6605 } 6606 } 6607 } 6608 MNT_IUNLOCK(mp); 6609 } 6610 return (0); 6611 } 6612 6613 int 6614 vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp) 6615 { 6616 struct mount_pcpu *mpcpu; 6617 struct vnode *vp; 6618 int error; 6619 6620 if (!vfs_op_thread_enter(mp, mpcpu)) 6621 return (vfs_cache_root_fallback(mp, flags, vpp)); 6622 vp = atomic_load_ptr(&mp->mnt_rootvnode); 6623 if (vp == NULL || VN_IS_DOOMED(vp)) { 6624 vfs_op_thread_exit(mp, mpcpu); 6625 return (vfs_cache_root_fallback(mp, flags, vpp)); 6626 } 6627 vrefact(vp); 6628 vfs_op_thread_exit(mp, mpcpu); 6629 error = vn_lock(vp, flags); 6630 if (error != 0) { 6631 vrele(vp); 6632 return (vfs_cache_root_fallback(mp, flags, vpp)); 6633 } 6634 *vpp = vp; 6635 return (0); 6636 } 6637 6638 struct vnode * 6639 vfs_cache_root_clear(struct mount *mp) 6640 { 6641 struct vnode *vp; 6642 6643 /* 6644 * ops > 0 guarantees there is nobody who can see this vnode 6645 */ 6646 MPASS(mp->mnt_vfs_ops > 0); 6647 vp = mp->mnt_rootvnode; 6648 if (vp != NULL) 6649 vn_seqc_write_begin(vp); 6650 mp->mnt_rootvnode = NULL; 6651 return (vp); 6652 } 6653 6654 void 6655 vfs_cache_root_set(struct mount *mp, struct vnode *vp) 6656 { 6657 6658 MPASS(mp->mnt_vfs_ops > 0); 6659 vrefact(vp); 6660 mp->mnt_rootvnode = vp; 6661 } 6662 6663 /* 6664 * These are helper functions for filesystems to traverse all 6665 * their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h. 6666 * 6667 * This interface replaces MNT_VNODE_FOREACH. 6668 */ 6669 6670 struct vnode * 6671 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp) 6672 { 6673 struct vnode *vp; 6674 6675 if (should_yield()) 6676 kern_yield(PRI_USER); 6677 MNT_ILOCK(mp); 6678 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6679 for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL; 6680 vp = TAILQ_NEXT(vp, v_nmntvnodes)) { 6681 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6682 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6683 continue; 6684 VI_LOCK(vp); 6685 if (VN_IS_DOOMED(vp)) { 6686 VI_UNLOCK(vp); 6687 continue; 6688 } 6689 break; 6690 } 6691 if (vp == NULL) { 6692 __mnt_vnode_markerfree_all(mvp, mp); 6693 /* MNT_IUNLOCK(mp); -- done in above function */ 6694 mtx_assert(MNT_MTX(mp), MA_NOTOWNED); 6695 return (NULL); 6696 } 6697 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6698 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6699 MNT_IUNLOCK(mp); 6700 return (vp); 6701 } 6702 6703 struct vnode * 6704 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp) 6705 { 6706 struct vnode *vp; 6707 6708 *mvp = vn_alloc_marker(mp); 6709 MNT_ILOCK(mp); 6710 MNT_REF(mp); 6711 6712 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 6713 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6714 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6715 continue; 6716 VI_LOCK(vp); 6717 if (VN_IS_DOOMED(vp)) { 6718 VI_UNLOCK(vp); 6719 continue; 6720 } 6721 break; 6722 } 6723 if (vp == NULL) { 6724 MNT_REL(mp); 6725 MNT_IUNLOCK(mp); 6726 vn_free_marker(*mvp); 6727 *mvp = NULL; 6728 return (NULL); 6729 } 6730 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6731 MNT_IUNLOCK(mp); 6732 return (vp); 6733 } 6734 6735 void 6736 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp) 6737 { 6738 6739 if (*mvp == NULL) { 6740 MNT_IUNLOCK(mp); 6741 return; 6742 } 6743 6744 mtx_assert(MNT_MTX(mp), MA_OWNED); 6745 6746 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6747 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6748 MNT_REL(mp); 6749 MNT_IUNLOCK(mp); 6750 vn_free_marker(*mvp); 6751 *mvp = NULL; 6752 } 6753 6754 /* 6755 * These are helper functions for filesystems to traverse their 6756 * lazy vnodes. See MNT_VNODE_FOREACH_LAZY() in sys/mount.h 6757 */ 6758 static void 6759 mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 6760 { 6761 6762 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6763 6764 MNT_ILOCK(mp); 6765 MNT_REL(mp); 6766 MNT_IUNLOCK(mp); 6767 vn_free_marker(*mvp); 6768 *mvp = NULL; 6769 } 6770 6771 /* 6772 * Relock the mp mount vnode list lock with the vp vnode interlock in the 6773 * conventional lock order during mnt_vnode_next_lazy iteration. 6774 * 6775 * On entry, the mount vnode list lock is held and the vnode interlock is not. 6776 * The list lock is dropped and reacquired. On success, both locks are held. 6777 * On failure, the mount vnode list lock is held but the vnode interlock is 6778 * not, and the procedure may have yielded. 6779 */ 6780 static bool 6781 mnt_vnode_next_lazy_relock(struct vnode *mvp, struct mount *mp, 6782 struct vnode *vp) 6783 { 6784 6785 VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER && 6786 TAILQ_NEXT(mvp, v_lazylist) != NULL, mvp, 6787 ("%s: bad marker", __func__)); 6788 VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp, 6789 ("%s: inappropriate vnode", __func__)); 6790 ASSERT_VI_UNLOCKED(vp, __func__); 6791 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 6792 6793 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, mvp, v_lazylist); 6794 TAILQ_INSERT_BEFORE(vp, mvp, v_lazylist); 6795 6796 /* 6797 * Note we may be racing against vdrop which transitioned the hold 6798 * count to 0 and now waits for the ->mnt_listmtx lock. This is fine, 6799 * if we are the only user after we get the interlock we will just 6800 * vdrop. 6801 */ 6802 vhold(vp); 6803 mtx_unlock(&mp->mnt_listmtx); 6804 VI_LOCK(vp); 6805 if (VN_IS_DOOMED(vp)) { 6806 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 6807 goto out_lost; 6808 } 6809 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 6810 /* 6811 * There is nothing to do if we are the last user. 6812 */ 6813 if (!refcount_release_if_not_last(&vp->v_holdcnt)) 6814 goto out_lost; 6815 mtx_lock(&mp->mnt_listmtx); 6816 return (true); 6817 out_lost: 6818 vdropl(vp); 6819 maybe_yield(); 6820 mtx_lock(&mp->mnt_listmtx); 6821 return (false); 6822 } 6823 6824 static struct vnode * 6825 mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6826 void *cbarg) 6827 { 6828 struct vnode *vp; 6829 6830 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 6831 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6832 restart: 6833 vp = TAILQ_NEXT(*mvp, v_lazylist); 6834 while (vp != NULL) { 6835 if (vp->v_type == VMARKER) { 6836 vp = TAILQ_NEXT(vp, v_lazylist); 6837 continue; 6838 } 6839 /* 6840 * See if we want to process the vnode. Note we may encounter a 6841 * long string of vnodes we don't care about and hog the list 6842 * as a result. Check for it and requeue the marker. 6843 */ 6844 VNPASS(!VN_IS_DOOMED(vp), vp); 6845 if (!cb(vp, cbarg)) { 6846 if (!should_yield()) { 6847 vp = TAILQ_NEXT(vp, v_lazylist); 6848 continue; 6849 } 6850 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, 6851 v_lazylist); 6852 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, 6853 v_lazylist); 6854 mtx_unlock(&mp->mnt_listmtx); 6855 kern_yield(PRI_USER); 6856 mtx_lock(&mp->mnt_listmtx); 6857 goto restart; 6858 } 6859 /* 6860 * Try-lock because this is the wrong lock order. 6861 */ 6862 if (!VI_TRYLOCK(vp) && 6863 !mnt_vnode_next_lazy_relock(*mvp, mp, vp)) 6864 goto restart; 6865 KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp)); 6866 KASSERT(vp->v_mount == mp || vp->v_mount == NULL, 6867 ("alien vnode on the lazy list %p %p", vp, mp)); 6868 VNPASS(vp->v_mount == mp, vp); 6869 VNPASS(!VN_IS_DOOMED(vp), vp); 6870 break; 6871 } 6872 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 6873 6874 /* Check if we are done */ 6875 if (vp == NULL) { 6876 mtx_unlock(&mp->mnt_listmtx); 6877 mnt_vnode_markerfree_lazy(mvp, mp); 6878 return (NULL); 6879 } 6880 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, v_lazylist); 6881 mtx_unlock(&mp->mnt_listmtx); 6882 ASSERT_VI_LOCKED(vp, "lazy iter"); 6883 return (vp); 6884 } 6885 6886 struct vnode * 6887 __mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6888 void *cbarg) 6889 { 6890 6891 if (should_yield()) 6892 kern_yield(PRI_USER); 6893 mtx_lock(&mp->mnt_listmtx); 6894 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 6895 } 6896 6897 struct vnode * 6898 __mnt_vnode_first_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6899 void *cbarg) 6900 { 6901 struct vnode *vp; 6902 6903 if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist)) 6904 return (NULL); 6905 6906 *mvp = vn_alloc_marker(mp); 6907 MNT_ILOCK(mp); 6908 MNT_REF(mp); 6909 MNT_IUNLOCK(mp); 6910 6911 mtx_lock(&mp->mnt_listmtx); 6912 vp = TAILQ_FIRST(&mp->mnt_lazyvnodelist); 6913 if (vp == NULL) { 6914 mtx_unlock(&mp->mnt_listmtx); 6915 mnt_vnode_markerfree_lazy(mvp, mp); 6916 return (NULL); 6917 } 6918 TAILQ_INSERT_BEFORE(vp, *mvp, v_lazylist); 6919 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 6920 } 6921 6922 void 6923 __mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 6924 { 6925 6926 if (*mvp == NULL) 6927 return; 6928 6929 mtx_lock(&mp->mnt_listmtx); 6930 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 6931 mtx_unlock(&mp->mnt_listmtx); 6932 mnt_vnode_markerfree_lazy(mvp, mp); 6933 } 6934 6935 int 6936 vn_dir_check_exec(struct vnode *vp, struct componentname *cnp) 6937 { 6938 6939 if ((cnp->cn_flags & NOEXECCHECK) != 0) { 6940 cnp->cn_flags &= ~NOEXECCHECK; 6941 return (0); 6942 } 6943 6944 return (VOP_ACCESS(vp, VEXEC, cnp->cn_cred, curthread)); 6945 } 6946 6947 /* 6948 * Do not use this variant unless you have means other than the hold count 6949 * to prevent the vnode from getting freed. 6950 */ 6951 void 6952 vn_seqc_write_begin_locked(struct vnode *vp) 6953 { 6954 6955 ASSERT_VI_LOCKED(vp, __func__); 6956 VNPASS(vp->v_holdcnt > 0, vp); 6957 VNPASS(vp->v_seqc_users >= 0, vp); 6958 vp->v_seqc_users++; 6959 if (vp->v_seqc_users == 1) 6960 seqc_sleepable_write_begin(&vp->v_seqc); 6961 } 6962 6963 void 6964 vn_seqc_write_begin(struct vnode *vp) 6965 { 6966 6967 VI_LOCK(vp); 6968 vn_seqc_write_begin_locked(vp); 6969 VI_UNLOCK(vp); 6970 } 6971 6972 void 6973 vn_seqc_write_end_locked(struct vnode *vp) 6974 { 6975 6976 ASSERT_VI_LOCKED(vp, __func__); 6977 VNPASS(vp->v_seqc_users > 0, vp); 6978 vp->v_seqc_users--; 6979 if (vp->v_seqc_users == 0) 6980 seqc_sleepable_write_end(&vp->v_seqc); 6981 } 6982 6983 void 6984 vn_seqc_write_end(struct vnode *vp) 6985 { 6986 6987 VI_LOCK(vp); 6988 vn_seqc_write_end_locked(vp); 6989 VI_UNLOCK(vp); 6990 } 6991 6992 /* 6993 * Special case handling for allocating and freeing vnodes. 6994 * 6995 * The counter remains unchanged on free so that a doomed vnode will 6996 * keep testing as in modify as long as it is accessible with SMR. 6997 */ 6998 static void 6999 vn_seqc_init(struct vnode *vp) 7000 { 7001 7002 vp->v_seqc = 0; 7003 vp->v_seqc_users = 0; 7004 } 7005 7006 static void 7007 vn_seqc_write_end_free(struct vnode *vp) 7008 { 7009 7010 VNPASS(seqc_in_modify(vp->v_seqc), vp); 7011 VNPASS(vp->v_seqc_users == 1, vp); 7012 } 7013 7014 void 7015 vn_irflag_set_locked(struct vnode *vp, short toset) 7016 { 7017 short flags; 7018 7019 ASSERT_VI_LOCKED(vp, __func__); 7020 flags = vn_irflag_read(vp); 7021 VNASSERT((flags & toset) == 0, vp, 7022 ("%s: some of the passed flags already set (have %d, passed %d)\n", 7023 __func__, flags, toset)); 7024 atomic_store_short(&vp->v_irflag, flags | toset); 7025 } 7026 7027 void 7028 vn_irflag_set(struct vnode *vp, short toset) 7029 { 7030 7031 VI_LOCK(vp); 7032 vn_irflag_set_locked(vp, toset); 7033 VI_UNLOCK(vp); 7034 } 7035 7036 void 7037 vn_irflag_set_cond_locked(struct vnode *vp, short toset) 7038 { 7039 short flags; 7040 7041 ASSERT_VI_LOCKED(vp, __func__); 7042 flags = vn_irflag_read(vp); 7043 atomic_store_short(&vp->v_irflag, flags | toset); 7044 } 7045 7046 void 7047 vn_irflag_set_cond(struct vnode *vp, short toset) 7048 { 7049 7050 VI_LOCK(vp); 7051 vn_irflag_set_cond_locked(vp, toset); 7052 VI_UNLOCK(vp); 7053 } 7054 7055 void 7056 vn_irflag_unset_locked(struct vnode *vp, short tounset) 7057 { 7058 short flags; 7059 7060 ASSERT_VI_LOCKED(vp, __func__); 7061 flags = vn_irflag_read(vp); 7062 VNASSERT((flags & tounset) == tounset, vp, 7063 ("%s: some of the passed flags not set (have %d, passed %d)\n", 7064 __func__, flags, tounset)); 7065 atomic_store_short(&vp->v_irflag, flags & ~tounset); 7066 } 7067 7068 void 7069 vn_irflag_unset(struct vnode *vp, short tounset) 7070 { 7071 7072 VI_LOCK(vp); 7073 vn_irflag_unset_locked(vp, tounset); 7074 VI_UNLOCK(vp); 7075 } 7076