1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vfs_subr.c 8.31 (Berkeley) 5/26/95 37 */ 38 39 /* 40 * External virtual filesystem routines 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include "opt_ddb.h" 47 #include "opt_watchdog.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/asan.h> 52 #include <sys/bio.h> 53 #include <sys/buf.h> 54 #include <sys/capsicum.h> 55 #include <sys/condvar.h> 56 #include <sys/conf.h> 57 #include <sys/counter.h> 58 #include <sys/dirent.h> 59 #include <sys/event.h> 60 #include <sys/eventhandler.h> 61 #include <sys/extattr.h> 62 #include <sys/file.h> 63 #include <sys/fcntl.h> 64 #include <sys/jail.h> 65 #include <sys/kdb.h> 66 #include <sys/kernel.h> 67 #include <sys/kthread.h> 68 #include <sys/ktr.h> 69 #include <sys/limits.h> 70 #include <sys/lockf.h> 71 #include <sys/malloc.h> 72 #include <sys/mount.h> 73 #include <sys/namei.h> 74 #include <sys/pctrie.h> 75 #include <sys/priv.h> 76 #include <sys/reboot.h> 77 #include <sys/refcount.h> 78 #include <sys/rwlock.h> 79 #include <sys/sched.h> 80 #include <sys/sleepqueue.h> 81 #include <sys/smr.h> 82 #include <sys/smp.h> 83 #include <sys/stat.h> 84 #include <sys/sysctl.h> 85 #include <sys/syslog.h> 86 #include <sys/vmmeter.h> 87 #include <sys/vnode.h> 88 #include <sys/watchdog.h> 89 90 #include <machine/stdarg.h> 91 92 #include <security/mac/mac_framework.h> 93 94 #include <vm/vm.h> 95 #include <vm/vm_object.h> 96 #include <vm/vm_extern.h> 97 #include <vm/pmap.h> 98 #include <vm/vm_map.h> 99 #include <vm/vm_page.h> 100 #include <vm/vm_kern.h> 101 #include <vm/uma.h> 102 103 #if defined(DEBUG_VFS_LOCKS) && (!defined(INVARIANTS) || !defined(WITNESS)) 104 #error DEBUG_VFS_LOCKS requires INVARIANTS and WITNESS 105 #endif 106 107 #ifdef DDB 108 #include <ddb/ddb.h> 109 #endif 110 111 static void delmntque(struct vnode *vp); 112 static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, 113 int slpflag, int slptimeo); 114 static void syncer_shutdown(void *arg, int howto); 115 static int vtryrecycle(struct vnode *vp); 116 static void v_init_counters(struct vnode *); 117 static void vn_seqc_init(struct vnode *); 118 static void vn_seqc_write_end_free(struct vnode *vp); 119 static void vgonel(struct vnode *); 120 static bool vhold_recycle_free(struct vnode *); 121 static void vdropl_recycle(struct vnode *vp); 122 static void vdrop_recycle(struct vnode *vp); 123 static void vfs_knllock(void *arg); 124 static void vfs_knlunlock(void *arg); 125 static void vfs_knl_assert_lock(void *arg, int what); 126 static void destroy_vpollinfo(struct vpollinfo *vi); 127 static int v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 128 daddr_t startlbn, daddr_t endlbn); 129 static void vnlru_recalc(void); 130 131 /* 132 * Number of vnodes in existence. Increased whenever getnewvnode() 133 * allocates a new vnode, decreased in vdropl() for VIRF_DOOMED vnode. 134 */ 135 static u_long __exclusive_cache_line numvnodes; 136 137 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0, 138 "Number of vnodes in existence"); 139 140 static counter_u64_t vnodes_created; 141 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created, 142 "Number of vnodes created by getnewvnode"); 143 144 /* 145 * Conversion tables for conversion from vnode types to inode formats 146 * and back. 147 */ 148 enum vtype iftovt_tab[16] = { 149 VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON, 150 VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON 151 }; 152 int vttoif_tab[10] = { 153 0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK, 154 S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT 155 }; 156 157 /* 158 * List of allocates vnodes in the system. 159 */ 160 static TAILQ_HEAD(freelst, vnode) vnode_list; 161 static struct vnode *vnode_list_free_marker; 162 static struct vnode *vnode_list_reclaim_marker; 163 164 /* 165 * "Free" vnode target. Free vnodes are rarely completely free, but are 166 * just ones that are cheap to recycle. Usually they are for files which 167 * have been stat'd but not read; these usually have inode and namecache 168 * data attached to them. This target is the preferred minimum size of a 169 * sub-cache consisting mostly of such files. The system balances the size 170 * of this sub-cache with its complement to try to prevent either from 171 * thrashing while the other is relatively inactive. The targets express 172 * a preference for the best balance. 173 * 174 * "Above" this target there are 2 further targets (watermarks) related 175 * to recyling of free vnodes. In the best-operating case, the cache is 176 * exactly full, the free list has size between vlowat and vhiwat above the 177 * free target, and recycling from it and normal use maintains this state. 178 * Sometimes the free list is below vlowat or even empty, but this state 179 * is even better for immediate use provided the cache is not full. 180 * Otherwise, vnlru_proc() runs to reclaim enough vnodes (usually non-free 181 * ones) to reach one of these states. The watermarks are currently hard- 182 * coded as 4% and 9% of the available space higher. These and the default 183 * of 25% for wantfreevnodes are too large if the memory size is large. 184 * E.g., 9% of 75% of MAXVNODES is more than 566000 vnodes to reclaim 185 * whenever vnlru_proc() becomes active. 186 */ 187 static long wantfreevnodes; 188 static long __exclusive_cache_line freevnodes; 189 SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD, 190 &freevnodes, 0, "Number of \"free\" vnodes"); 191 static long freevnodes_old; 192 193 static counter_u64_t recycles_count; 194 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles, CTLFLAG_RD, &recycles_count, 195 "Number of vnodes recycled to meet vnode cache targets"); 196 197 static counter_u64_t recycles_free_count; 198 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles_free, CTLFLAG_RD, &recycles_free_count, 199 "Number of free vnodes recycled to meet vnode cache targets"); 200 201 static counter_u64_t deferred_inact; 202 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, deferred_inact, CTLFLAG_RD, &deferred_inact, 203 "Number of times inactive processing was deferred"); 204 205 /* To keep more than one thread at a time from running vfs_getnewfsid */ 206 static struct mtx mntid_mtx; 207 208 /* 209 * Lock for any access to the following: 210 * vnode_list 211 * numvnodes 212 * freevnodes 213 */ 214 static struct mtx __exclusive_cache_line vnode_list_mtx; 215 216 /* Publicly exported FS */ 217 struct nfs_public nfs_pub; 218 219 static uma_zone_t buf_trie_zone; 220 static smr_t buf_trie_smr; 221 222 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */ 223 static uma_zone_t vnode_zone; 224 MALLOC_DEFINE(M_VNODEPOLL, "VN POLL", "vnode poll"); 225 226 __read_frequently smr_t vfs_smr; 227 228 /* 229 * The workitem queue. 230 * 231 * It is useful to delay writes of file data and filesystem metadata 232 * for tens of seconds so that quickly created and deleted files need 233 * not waste disk bandwidth being created and removed. To realize this, 234 * we append vnodes to a "workitem" queue. When running with a soft 235 * updates implementation, most pending metadata dependencies should 236 * not wait for more than a few seconds. Thus, mounted on block devices 237 * are delayed only about a half the time that file data is delayed. 238 * Similarly, directory updates are more critical, so are only delayed 239 * about a third the time that file data is delayed. Thus, there are 240 * SYNCER_MAXDELAY queues that are processed round-robin at a rate of 241 * one each second (driven off the filesystem syncer process). The 242 * syncer_delayno variable indicates the next queue that is to be processed. 243 * Items that need to be processed soon are placed in this queue: 244 * 245 * syncer_workitem_pending[syncer_delayno] 246 * 247 * A delay of fifteen seconds is done by placing the request fifteen 248 * entries later in the queue: 249 * 250 * syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask] 251 * 252 */ 253 static int syncer_delayno; 254 static long syncer_mask; 255 LIST_HEAD(synclist, bufobj); 256 static struct synclist *syncer_workitem_pending; 257 /* 258 * The sync_mtx protects: 259 * bo->bo_synclist 260 * sync_vnode_count 261 * syncer_delayno 262 * syncer_state 263 * syncer_workitem_pending 264 * syncer_worklist_len 265 * rushjob 266 */ 267 static struct mtx sync_mtx; 268 static struct cv sync_wakeup; 269 270 #define SYNCER_MAXDELAY 32 271 static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */ 272 static int syncdelay = 30; /* max time to delay syncing data */ 273 static int filedelay = 30; /* time to delay syncing files */ 274 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0, 275 "Time to delay syncing files (in seconds)"); 276 static int dirdelay = 29; /* time to delay syncing directories */ 277 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0, 278 "Time to delay syncing directories (in seconds)"); 279 static int metadelay = 28; /* time to delay syncing metadata */ 280 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0, 281 "Time to delay syncing metadata (in seconds)"); 282 static int rushjob; /* number of slots to run ASAP */ 283 static int stat_rush_requests; /* number of times I/O speeded up */ 284 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0, 285 "Number of times I/O speeded up (rush requests)"); 286 287 #define VDBATCH_SIZE 8 288 struct vdbatch { 289 u_int index; 290 struct mtx lock; 291 struct vnode *tab[VDBATCH_SIZE]; 292 }; 293 DPCPU_DEFINE_STATIC(struct vdbatch, vd); 294 295 static void vdbatch_dequeue(struct vnode *vp); 296 297 /* 298 * When shutting down the syncer, run it at four times normal speed. 299 */ 300 #define SYNCER_SHUTDOWN_SPEEDUP 4 301 static int sync_vnode_count; 302 static int syncer_worklist_len; 303 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY } 304 syncer_state; 305 306 /* Target for maximum number of vnodes. */ 307 u_long desiredvnodes; 308 static u_long gapvnodes; /* gap between wanted and desired */ 309 static u_long vhiwat; /* enough extras after expansion */ 310 static u_long vlowat; /* minimal extras before expansion */ 311 static u_long vstir; /* nonzero to stir non-free vnodes */ 312 static volatile int vsmalltrigger = 8; /* pref to keep if > this many pages */ 313 314 static u_long vnlru_read_freevnodes(void); 315 316 /* 317 * Note that no attempt is made to sanitize these parameters. 318 */ 319 static int 320 sysctl_maxvnodes(SYSCTL_HANDLER_ARGS) 321 { 322 u_long val; 323 int error; 324 325 val = desiredvnodes; 326 error = sysctl_handle_long(oidp, &val, 0, req); 327 if (error != 0 || req->newptr == NULL) 328 return (error); 329 330 if (val == desiredvnodes) 331 return (0); 332 mtx_lock(&vnode_list_mtx); 333 desiredvnodes = val; 334 wantfreevnodes = desiredvnodes / 4; 335 vnlru_recalc(); 336 mtx_unlock(&vnode_list_mtx); 337 /* 338 * XXX There is no protection against multiple threads changing 339 * desiredvnodes at the same time. Locking above only helps vnlru and 340 * getnewvnode. 341 */ 342 vfs_hash_changesize(desiredvnodes); 343 cache_changesize(desiredvnodes); 344 return (0); 345 } 346 347 SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes, 348 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, 349 "LU", "Target for maximum number of vnodes"); 350 351 static int 352 sysctl_wantfreevnodes(SYSCTL_HANDLER_ARGS) 353 { 354 u_long val; 355 int error; 356 357 val = wantfreevnodes; 358 error = sysctl_handle_long(oidp, &val, 0, req); 359 if (error != 0 || req->newptr == NULL) 360 return (error); 361 362 if (val == wantfreevnodes) 363 return (0); 364 mtx_lock(&vnode_list_mtx); 365 wantfreevnodes = val; 366 vnlru_recalc(); 367 mtx_unlock(&vnode_list_mtx); 368 return (0); 369 } 370 371 SYSCTL_PROC(_vfs, OID_AUTO, wantfreevnodes, 372 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes, 373 "LU", "Target for minimum number of \"free\" vnodes"); 374 375 SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW, 376 &wantfreevnodes, 0, "Old name for vfs.wantfreevnodes (legacy)"); 377 static int vnlru_nowhere; 378 SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW | CTLFLAG_STATS, 379 &vnlru_nowhere, 0, "Number of times the vnlru process ran without success"); 380 381 static int 382 sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS) 383 { 384 struct vnode *vp; 385 struct nameidata nd; 386 char *buf; 387 unsigned long ndflags; 388 int error; 389 390 if (req->newptr == NULL) 391 return (EINVAL); 392 if (req->newlen >= PATH_MAX) 393 return (E2BIG); 394 395 buf = malloc(PATH_MAX, M_TEMP, M_WAITOK); 396 error = SYSCTL_IN(req, buf, req->newlen); 397 if (error != 0) 398 goto out; 399 400 buf[req->newlen] = '\0'; 401 402 ndflags = LOCKLEAF | NOFOLLOW | AUDITVNODE1; 403 NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE, buf); 404 if ((error = namei(&nd)) != 0) 405 goto out; 406 vp = nd.ni_vp; 407 408 if (VN_IS_DOOMED(vp)) { 409 /* 410 * This vnode is being recycled. Return != 0 to let the caller 411 * know that the sysctl had no effect. Return EAGAIN because a 412 * subsequent call will likely succeed (since namei will create 413 * a new vnode if necessary) 414 */ 415 error = EAGAIN; 416 goto putvnode; 417 } 418 419 counter_u64_add(recycles_count, 1); 420 vgone(vp); 421 putvnode: 422 vput(vp); 423 NDFREE_PNBUF(&nd); 424 out: 425 free(buf, M_TEMP); 426 return (error); 427 } 428 429 static int 430 sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS) 431 { 432 struct thread *td = curthread; 433 struct vnode *vp; 434 struct file *fp; 435 int error; 436 int fd; 437 438 if (req->newptr == NULL) 439 return (EBADF); 440 441 error = sysctl_handle_int(oidp, &fd, 0, req); 442 if (error != 0) 443 return (error); 444 error = getvnode(curthread, fd, &cap_fcntl_rights, &fp); 445 if (error != 0) 446 return (error); 447 vp = fp->f_vnode; 448 449 error = vn_lock(vp, LK_EXCLUSIVE); 450 if (error != 0) 451 goto drop; 452 453 counter_u64_add(recycles_count, 1); 454 vgone(vp); 455 VOP_UNLOCK(vp); 456 drop: 457 fdrop(fp, td); 458 return (error); 459 } 460 461 SYSCTL_PROC(_debug, OID_AUTO, try_reclaim_vnode, 462 CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 463 sysctl_try_reclaim_vnode, "A", "Try to reclaim a vnode by its pathname"); 464 SYSCTL_PROC(_debug, OID_AUTO, ftry_reclaim_vnode, 465 CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 466 sysctl_ftry_reclaim_vnode, "I", 467 "Try to reclaim a vnode by its file descriptor"); 468 469 /* Shift count for (uintptr_t)vp to initialize vp->v_hash. */ 470 #define vnsz2log 8 471 #ifndef DEBUG_LOCKS 472 _Static_assert(sizeof(struct vnode) >= 1UL << vnsz2log && 473 sizeof(struct vnode) < 1UL << (vnsz2log + 1), 474 "vnsz2log needs to be updated"); 475 #endif 476 477 /* 478 * Support for the bufobj clean & dirty pctrie. 479 */ 480 static void * 481 buf_trie_alloc(struct pctrie *ptree) 482 { 483 return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT)); 484 } 485 486 static void 487 buf_trie_free(struct pctrie *ptree, void *node) 488 { 489 uma_zfree_smr(buf_trie_zone, node); 490 } 491 PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free, 492 buf_trie_smr); 493 494 /* 495 * Initialize the vnode management data structures. 496 * 497 * Reevaluate the following cap on the number of vnodes after the physical 498 * memory size exceeds 512GB. In the limit, as the physical memory size 499 * grows, the ratio of the memory size in KB to vnodes approaches 64:1. 500 */ 501 #ifndef MAXVNODES_MAX 502 #define MAXVNODES_MAX (512UL * 1024 * 1024 / 64) /* 8M */ 503 #endif 504 505 static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); 506 507 static struct vnode * 508 vn_alloc_marker(struct mount *mp) 509 { 510 struct vnode *vp; 511 512 vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO); 513 vp->v_type = VMARKER; 514 vp->v_mount = mp; 515 516 return (vp); 517 } 518 519 static void 520 vn_free_marker(struct vnode *vp) 521 { 522 523 MPASS(vp->v_type == VMARKER); 524 free(vp, M_VNODE_MARKER); 525 } 526 527 #ifdef KASAN 528 static int 529 vnode_ctor(void *mem, int size, void *arg __unused, int flags __unused) 530 { 531 kasan_mark(mem, size, roundup2(size, UMA_ALIGN_PTR + 1), 0); 532 return (0); 533 } 534 535 static void 536 vnode_dtor(void *mem, int size, void *arg __unused) 537 { 538 size_t end1, end2, off1, off2; 539 540 _Static_assert(offsetof(struct vnode, v_vnodelist) < 541 offsetof(struct vnode, v_dbatchcpu), 542 "KASAN marks require updating"); 543 544 off1 = offsetof(struct vnode, v_vnodelist); 545 off2 = offsetof(struct vnode, v_dbatchcpu); 546 end1 = off1 + sizeof(((struct vnode *)NULL)->v_vnodelist); 547 end2 = off2 + sizeof(((struct vnode *)NULL)->v_dbatchcpu); 548 549 /* 550 * Access to the v_vnodelist and v_dbatchcpu fields are permitted even 551 * after the vnode has been freed. Try to get some KASAN coverage by 552 * marking everything except those two fields as invalid. Because 553 * KASAN's tracking is not byte-granular, any preceding fields sharing 554 * the same 8-byte aligned word must also be marked valid. 555 */ 556 557 /* Handle the area from the start until v_vnodelist... */ 558 off1 = rounddown2(off1, KASAN_SHADOW_SCALE); 559 kasan_mark(mem, off1, off1, KASAN_UMA_FREED); 560 561 /* ... then the area between v_vnodelist and v_dbatchcpu ... */ 562 off1 = roundup2(end1, KASAN_SHADOW_SCALE); 563 off2 = rounddown2(off2, KASAN_SHADOW_SCALE); 564 if (off2 > off1) 565 kasan_mark((void *)((char *)mem + off1), off2 - off1, 566 off2 - off1, KASAN_UMA_FREED); 567 568 /* ... and finally the area from v_dbatchcpu to the end. */ 569 off2 = roundup2(end2, KASAN_SHADOW_SCALE); 570 kasan_mark((void *)((char *)mem + off2), size - off2, size - off2, 571 KASAN_UMA_FREED); 572 } 573 #endif /* KASAN */ 574 575 /* 576 * Initialize a vnode as it first enters the zone. 577 */ 578 static int 579 vnode_init(void *mem, int size, int flags) 580 { 581 struct vnode *vp; 582 583 vp = mem; 584 bzero(vp, size); 585 /* 586 * Setup locks. 587 */ 588 vp->v_vnlock = &vp->v_lock; 589 mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF); 590 /* 591 * By default, don't allow shared locks unless filesystems opt-in. 592 */ 593 lockinit(vp->v_vnlock, PVFS, "vnode", VLKTIMEOUT, 594 LK_NOSHARE | LK_IS_VNODE); 595 /* 596 * Initialize bufobj. 597 */ 598 bufobj_init(&vp->v_bufobj, vp); 599 /* 600 * Initialize namecache. 601 */ 602 cache_vnode_init(vp); 603 /* 604 * Initialize rangelocks. 605 */ 606 rangelock_init(&vp->v_rl); 607 608 vp->v_dbatchcpu = NOCPU; 609 610 vp->v_state = VSTATE_DEAD; 611 612 /* 613 * Check vhold_recycle_free for an explanation. 614 */ 615 vp->v_holdcnt = VHOLD_NO_SMR; 616 vp->v_type = VNON; 617 mtx_lock(&vnode_list_mtx); 618 TAILQ_INSERT_BEFORE(vnode_list_free_marker, vp, v_vnodelist); 619 mtx_unlock(&vnode_list_mtx); 620 return (0); 621 } 622 623 /* 624 * Free a vnode when it is cleared from the zone. 625 */ 626 static void 627 vnode_fini(void *mem, int size) 628 { 629 struct vnode *vp; 630 struct bufobj *bo; 631 632 vp = mem; 633 vdbatch_dequeue(vp); 634 mtx_lock(&vnode_list_mtx); 635 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 636 mtx_unlock(&vnode_list_mtx); 637 rangelock_destroy(&vp->v_rl); 638 lockdestroy(vp->v_vnlock); 639 mtx_destroy(&vp->v_interlock); 640 bo = &vp->v_bufobj; 641 rw_destroy(BO_LOCKPTR(bo)); 642 643 kasan_mark(mem, size, size, 0); 644 } 645 646 /* 647 * Provide the size of NFS nclnode and NFS fh for calculation of the 648 * vnode memory consumption. The size is specified directly to 649 * eliminate dependency on NFS-private header. 650 * 651 * Other filesystems may use bigger or smaller (like UFS and ZFS) 652 * private inode data, but the NFS-based estimation is ample enough. 653 * Still, we care about differences in the size between 64- and 32-bit 654 * platforms. 655 * 656 * Namecache structure size is heuristically 657 * sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1. 658 */ 659 #ifdef _LP64 660 #define NFS_NCLNODE_SZ (528 + 64) 661 #define NC_SZ 148 662 #else 663 #define NFS_NCLNODE_SZ (360 + 32) 664 #define NC_SZ 92 665 #endif 666 667 static void 668 vntblinit(void *dummy __unused) 669 { 670 struct vdbatch *vd; 671 uma_ctor ctor; 672 uma_dtor dtor; 673 int cpu, physvnodes, virtvnodes; 674 675 /* 676 * Desiredvnodes is a function of the physical memory size and the 677 * kernel's heap size. Generally speaking, it scales with the 678 * physical memory size. The ratio of desiredvnodes to the physical 679 * memory size is 1:16 until desiredvnodes exceeds 98,304. 680 * Thereafter, the 681 * marginal ratio of desiredvnodes to the physical memory size is 682 * 1:64. However, desiredvnodes is limited by the kernel's heap 683 * size. The memory required by desiredvnodes vnodes and vm objects 684 * must not exceed 1/10th of the kernel's heap size. 685 */ 686 physvnodes = maxproc + pgtok(vm_cnt.v_page_count) / 64 + 687 3 * min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 64; 688 virtvnodes = vm_kmem_size / (10 * (sizeof(struct vm_object) + 689 sizeof(struct vnode) + NC_SZ * ncsizefactor + NFS_NCLNODE_SZ)); 690 desiredvnodes = min(physvnodes, virtvnodes); 691 if (desiredvnodes > MAXVNODES_MAX) { 692 if (bootverbose) 693 printf("Reducing kern.maxvnodes %lu -> %lu\n", 694 desiredvnodes, MAXVNODES_MAX); 695 desiredvnodes = MAXVNODES_MAX; 696 } 697 wantfreevnodes = desiredvnodes / 4; 698 mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF); 699 TAILQ_INIT(&vnode_list); 700 mtx_init(&vnode_list_mtx, "vnode_list", NULL, MTX_DEF); 701 /* 702 * The lock is taken to appease WITNESS. 703 */ 704 mtx_lock(&vnode_list_mtx); 705 vnlru_recalc(); 706 mtx_unlock(&vnode_list_mtx); 707 vnode_list_free_marker = vn_alloc_marker(NULL); 708 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist); 709 vnode_list_reclaim_marker = vn_alloc_marker(NULL); 710 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist); 711 712 #ifdef KASAN 713 ctor = vnode_ctor; 714 dtor = vnode_dtor; 715 #else 716 ctor = NULL; 717 dtor = NULL; 718 #endif 719 vnode_zone = uma_zcreate("VNODE", sizeof(struct vnode), ctor, dtor, 720 vnode_init, vnode_fini, UMA_ALIGN_PTR, UMA_ZONE_NOKASAN); 721 uma_zone_set_smr(vnode_zone, vfs_smr); 722 723 /* 724 * Preallocate enough nodes to support one-per buf so that 725 * we can not fail an insert. reassignbuf() callers can not 726 * tolerate the insertion failure. 727 */ 728 buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(), 729 NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR, 730 UMA_ZONE_NOFREE | UMA_ZONE_SMR); 731 buf_trie_smr = uma_zone_get_smr(buf_trie_zone); 732 uma_prealloc(buf_trie_zone, nbuf); 733 734 vnodes_created = counter_u64_alloc(M_WAITOK); 735 recycles_count = counter_u64_alloc(M_WAITOK); 736 recycles_free_count = counter_u64_alloc(M_WAITOK); 737 deferred_inact = counter_u64_alloc(M_WAITOK); 738 739 /* 740 * Initialize the filesystem syncer. 741 */ 742 syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE, 743 &syncer_mask); 744 syncer_maxdelay = syncer_mask + 1; 745 mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF); 746 cv_init(&sync_wakeup, "syncer"); 747 748 CPU_FOREACH(cpu) { 749 vd = DPCPU_ID_PTR((cpu), vd); 750 bzero(vd, sizeof(*vd)); 751 mtx_init(&vd->lock, "vdbatch", NULL, MTX_DEF); 752 } 753 } 754 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL); 755 756 /* 757 * Mark a mount point as busy. Used to synchronize access and to delay 758 * unmounting. Eventually, mountlist_mtx is not released on failure. 759 * 760 * vfs_busy() is a custom lock, it can block the caller. 761 * vfs_busy() only sleeps if the unmount is active on the mount point. 762 * For a mountpoint mp, vfs_busy-enforced lock is before lock of any 763 * vnode belonging to mp. 764 * 765 * Lookup uses vfs_busy() to traverse mount points. 766 * root fs var fs 767 * / vnode lock A / vnode lock (/var) D 768 * /var vnode lock B /log vnode lock(/var/log) E 769 * vfs_busy lock C vfs_busy lock F 770 * 771 * Within each file system, the lock order is C->A->B and F->D->E. 772 * 773 * When traversing across mounts, the system follows that lock order: 774 * 775 * C->A->B 776 * | 777 * +->F->D->E 778 * 779 * The lookup() process for namei("/var") illustrates the process: 780 * 1. VOP_LOOKUP() obtains B while A is held 781 * 2. vfs_busy() obtains a shared lock on F while A and B are held 782 * 3. vput() releases lock on B 783 * 4. vput() releases lock on A 784 * 5. VFS_ROOT() obtains lock on D while shared lock on F is held 785 * 6. vfs_unbusy() releases shared lock on F 786 * 7. vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A. 787 * Attempt to lock A (instead of vp_crossmp) while D is held would 788 * violate the global order, causing deadlocks. 789 * 790 * dounmount() locks B while F is drained. Note that for stacked 791 * filesystems, D and B in the example above may be the same lock, 792 * which introdues potential lock order reversal deadlock between 793 * dounmount() and step 5 above. These filesystems may avoid the LOR 794 * by setting VV_CROSSLOCK on the covered vnode so that lock B will 795 * remain held until after step 5. 796 */ 797 int 798 vfs_busy(struct mount *mp, int flags) 799 { 800 struct mount_pcpu *mpcpu; 801 802 MPASS((flags & ~MBF_MASK) == 0); 803 CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags); 804 805 if (vfs_op_thread_enter(mp, mpcpu)) { 806 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 807 MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0); 808 MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0); 809 vfs_mp_count_add_pcpu(mpcpu, ref, 1); 810 vfs_mp_count_add_pcpu(mpcpu, lockref, 1); 811 vfs_op_thread_exit(mp, mpcpu); 812 if (flags & MBF_MNTLSTLOCK) 813 mtx_unlock(&mountlist_mtx); 814 return (0); 815 } 816 817 MNT_ILOCK(mp); 818 vfs_assert_mount_counters(mp); 819 MNT_REF(mp); 820 /* 821 * If mount point is currently being unmounted, sleep until the 822 * mount point fate is decided. If thread doing the unmounting fails, 823 * it will clear MNTK_UNMOUNT flag before waking us up, indicating 824 * that this mount point has survived the unmount attempt and vfs_busy 825 * should retry. Otherwise the unmounter thread will set MNTK_REFEXPIRE 826 * flag in addition to MNTK_UNMOUNT, indicating that mount point is 827 * about to be really destroyed. vfs_busy needs to release its 828 * reference on the mount point in this case and return with ENOENT, 829 * telling the caller the mount it tried to busy is no longer valid. 830 */ 831 while (mp->mnt_kern_flag & MNTK_UNMOUNT) { 832 KASSERT(TAILQ_EMPTY(&mp->mnt_uppers), 833 ("%s: non-empty upper mount list with pending unmount", 834 __func__)); 835 if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) { 836 MNT_REL(mp); 837 MNT_IUNLOCK(mp); 838 CTR1(KTR_VFS, "%s: failed busying before sleeping", 839 __func__); 840 return (ENOENT); 841 } 842 if (flags & MBF_MNTLSTLOCK) 843 mtx_unlock(&mountlist_mtx); 844 mp->mnt_kern_flag |= MNTK_MWAIT; 845 msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0); 846 if (flags & MBF_MNTLSTLOCK) 847 mtx_lock(&mountlist_mtx); 848 MNT_ILOCK(mp); 849 } 850 if (flags & MBF_MNTLSTLOCK) 851 mtx_unlock(&mountlist_mtx); 852 mp->mnt_lockref++; 853 MNT_IUNLOCK(mp); 854 return (0); 855 } 856 857 /* 858 * Free a busy filesystem. 859 */ 860 void 861 vfs_unbusy(struct mount *mp) 862 { 863 struct mount_pcpu *mpcpu; 864 int c; 865 866 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 867 868 if (vfs_op_thread_enter(mp, mpcpu)) { 869 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 870 vfs_mp_count_sub_pcpu(mpcpu, lockref, 1); 871 vfs_mp_count_sub_pcpu(mpcpu, ref, 1); 872 vfs_op_thread_exit(mp, mpcpu); 873 return; 874 } 875 876 MNT_ILOCK(mp); 877 vfs_assert_mount_counters(mp); 878 MNT_REL(mp); 879 c = --mp->mnt_lockref; 880 if (mp->mnt_vfs_ops == 0) { 881 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 882 MNT_IUNLOCK(mp); 883 return; 884 } 885 if (c < 0) 886 vfs_dump_mount_counters(mp); 887 if (c == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) { 888 MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT); 889 CTR1(KTR_VFS, "%s: waking up waiters", __func__); 890 mp->mnt_kern_flag &= ~MNTK_DRAINING; 891 wakeup(&mp->mnt_lockref); 892 } 893 MNT_IUNLOCK(mp); 894 } 895 896 /* 897 * Lookup a mount point by filesystem identifier. 898 */ 899 struct mount * 900 vfs_getvfs(fsid_t *fsid) 901 { 902 struct mount *mp; 903 904 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 905 mtx_lock(&mountlist_mtx); 906 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 907 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 908 vfs_ref(mp); 909 mtx_unlock(&mountlist_mtx); 910 return (mp); 911 } 912 } 913 mtx_unlock(&mountlist_mtx); 914 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 915 return ((struct mount *) 0); 916 } 917 918 /* 919 * Lookup a mount point by filesystem identifier, busying it before 920 * returning. 921 * 922 * To avoid congestion on mountlist_mtx, implement simple direct-mapped 923 * cache for popular filesystem identifiers. The cache is lockess, using 924 * the fact that struct mount's are never freed. In worst case we may 925 * get pointer to unmounted or even different filesystem, so we have to 926 * check what we got, and go slow way if so. 927 */ 928 struct mount * 929 vfs_busyfs(fsid_t *fsid) 930 { 931 #define FSID_CACHE_SIZE 256 932 typedef struct mount * volatile vmp_t; 933 static vmp_t cache[FSID_CACHE_SIZE]; 934 struct mount *mp; 935 int error; 936 uint32_t hash; 937 938 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 939 hash = fsid->val[0] ^ fsid->val[1]; 940 hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1); 941 mp = cache[hash]; 942 if (mp == NULL || fsidcmp(&mp->mnt_stat.f_fsid, fsid) != 0) 943 goto slow; 944 if (vfs_busy(mp, 0) != 0) { 945 cache[hash] = NULL; 946 goto slow; 947 } 948 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) 949 return (mp); 950 else 951 vfs_unbusy(mp); 952 953 slow: 954 mtx_lock(&mountlist_mtx); 955 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 956 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 957 error = vfs_busy(mp, MBF_MNTLSTLOCK); 958 if (error) { 959 cache[hash] = NULL; 960 mtx_unlock(&mountlist_mtx); 961 return (NULL); 962 } 963 cache[hash] = mp; 964 return (mp); 965 } 966 } 967 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 968 mtx_unlock(&mountlist_mtx); 969 return ((struct mount *) 0); 970 } 971 972 /* 973 * Check if a user can access privileged mount options. 974 */ 975 int 976 vfs_suser(struct mount *mp, struct thread *td) 977 { 978 int error; 979 980 if (jailed(td->td_ucred)) { 981 /* 982 * If the jail of the calling thread lacks permission for 983 * this type of file system, deny immediately. 984 */ 985 if (!prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag)) 986 return (EPERM); 987 988 /* 989 * If the file system was mounted outside the jail of the 990 * calling thread, deny immediately. 991 */ 992 if (prison_check(td->td_ucred, mp->mnt_cred) != 0) 993 return (EPERM); 994 } 995 996 /* 997 * If file system supports delegated administration, we don't check 998 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified 999 * by the file system itself. 1000 * If this is not the user that did original mount, we check for 1001 * the PRIV_VFS_MOUNT_OWNER privilege. 1002 */ 1003 if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) && 1004 mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) { 1005 if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0) 1006 return (error); 1007 } 1008 return (0); 1009 } 1010 1011 /* 1012 * Get a new unique fsid. Try to make its val[0] unique, since this value 1013 * will be used to create fake device numbers for stat(). Also try (but 1014 * not so hard) make its val[0] unique mod 2^16, since some emulators only 1015 * support 16-bit device numbers. We end up with unique val[0]'s for the 1016 * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls. 1017 * 1018 * Keep in mind that several mounts may be running in parallel. Starting 1019 * the search one past where the previous search terminated is both a 1020 * micro-optimization and a defense against returning the same fsid to 1021 * different mounts. 1022 */ 1023 void 1024 vfs_getnewfsid(struct mount *mp) 1025 { 1026 static uint16_t mntid_base; 1027 struct mount *nmp; 1028 fsid_t tfsid; 1029 int mtype; 1030 1031 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 1032 mtx_lock(&mntid_mtx); 1033 mtype = mp->mnt_vfc->vfc_typenum; 1034 tfsid.val[1] = mtype; 1035 mtype = (mtype & 0xFF) << 24; 1036 for (;;) { 1037 tfsid.val[0] = makedev(255, 1038 mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF)); 1039 mntid_base++; 1040 if ((nmp = vfs_getvfs(&tfsid)) == NULL) 1041 break; 1042 vfs_rel(nmp); 1043 } 1044 mp->mnt_stat.f_fsid.val[0] = tfsid.val[0]; 1045 mp->mnt_stat.f_fsid.val[1] = tfsid.val[1]; 1046 mtx_unlock(&mntid_mtx); 1047 } 1048 1049 /* 1050 * Knob to control the precision of file timestamps: 1051 * 1052 * 0 = seconds only; nanoseconds zeroed. 1053 * 1 = seconds and nanoseconds, accurate within 1/HZ. 1054 * 2 = seconds and nanoseconds, truncated to microseconds. 1055 * >=3 = seconds and nanoseconds, maximum precision. 1056 */ 1057 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC }; 1058 1059 static int timestamp_precision = TSP_USEC; 1060 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW, 1061 ×tamp_precision, 0, "File timestamp precision (0: seconds, " 1062 "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, " 1063 "3+: sec + ns (max. precision))"); 1064 1065 /* 1066 * Get a current timestamp. 1067 */ 1068 void 1069 vfs_timestamp(struct timespec *tsp) 1070 { 1071 struct timeval tv; 1072 1073 switch (timestamp_precision) { 1074 case TSP_SEC: 1075 tsp->tv_sec = time_second; 1076 tsp->tv_nsec = 0; 1077 break; 1078 case TSP_HZ: 1079 getnanotime(tsp); 1080 break; 1081 case TSP_USEC: 1082 microtime(&tv); 1083 TIMEVAL_TO_TIMESPEC(&tv, tsp); 1084 break; 1085 case TSP_NSEC: 1086 default: 1087 nanotime(tsp); 1088 break; 1089 } 1090 } 1091 1092 /* 1093 * Set vnode attributes to VNOVAL 1094 */ 1095 void 1096 vattr_null(struct vattr *vap) 1097 { 1098 1099 vap->va_type = VNON; 1100 vap->va_size = VNOVAL; 1101 vap->va_bytes = VNOVAL; 1102 vap->va_mode = VNOVAL; 1103 vap->va_nlink = VNOVAL; 1104 vap->va_uid = VNOVAL; 1105 vap->va_gid = VNOVAL; 1106 vap->va_fsid = VNOVAL; 1107 vap->va_fileid = VNOVAL; 1108 vap->va_blocksize = VNOVAL; 1109 vap->va_rdev = VNOVAL; 1110 vap->va_atime.tv_sec = VNOVAL; 1111 vap->va_atime.tv_nsec = VNOVAL; 1112 vap->va_mtime.tv_sec = VNOVAL; 1113 vap->va_mtime.tv_nsec = VNOVAL; 1114 vap->va_ctime.tv_sec = VNOVAL; 1115 vap->va_ctime.tv_nsec = VNOVAL; 1116 vap->va_birthtime.tv_sec = VNOVAL; 1117 vap->va_birthtime.tv_nsec = VNOVAL; 1118 vap->va_flags = VNOVAL; 1119 vap->va_gen = VNOVAL; 1120 vap->va_vaflags = 0; 1121 } 1122 1123 /* 1124 * Try to reduce the total number of vnodes. 1125 * 1126 * This routine (and its user) are buggy in at least the following ways: 1127 * - all parameters were picked years ago when RAM sizes were significantly 1128 * smaller 1129 * - it can pick vnodes based on pages used by the vm object, but filesystems 1130 * like ZFS don't use it making the pick broken 1131 * - since ZFS has its own aging policy it gets partially combated by this one 1132 * - a dedicated method should be provided for filesystems to let them decide 1133 * whether the vnode should be recycled 1134 * 1135 * This routine is called when we have too many vnodes. It attempts 1136 * to free <count> vnodes and will potentially free vnodes that still 1137 * have VM backing store (VM backing store is typically the cause 1138 * of a vnode blowout so we want to do this). Therefore, this operation 1139 * is not considered cheap. 1140 * 1141 * A number of conditions may prevent a vnode from being reclaimed. 1142 * the buffer cache may have references on the vnode, a directory 1143 * vnode may still have references due to the namei cache representing 1144 * underlying files, or the vnode may be in active use. It is not 1145 * desirable to reuse such vnodes. These conditions may cause the 1146 * number of vnodes to reach some minimum value regardless of what 1147 * you set kern.maxvnodes to. Do not set kern.maxvnodes too low. 1148 * 1149 * @param reclaim_nc_src Only reclaim directories with outgoing namecache 1150 * entries if this argument is strue 1151 * @param trigger Only reclaim vnodes with fewer than this many resident 1152 * pages. 1153 * @param target How many vnodes to reclaim. 1154 * @return The number of vnodes that were reclaimed. 1155 */ 1156 static int 1157 vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) 1158 { 1159 struct vnode *vp, *mvp; 1160 struct mount *mp; 1161 struct vm_object *object; 1162 u_long done; 1163 bool retried; 1164 1165 mtx_assert(&vnode_list_mtx, MA_OWNED); 1166 1167 retried = false; 1168 done = 0; 1169 1170 mvp = vnode_list_reclaim_marker; 1171 restart: 1172 vp = mvp; 1173 while (done < target) { 1174 vp = TAILQ_NEXT(vp, v_vnodelist); 1175 if (__predict_false(vp == NULL)) 1176 break; 1177 1178 if (__predict_false(vp->v_type == VMARKER)) 1179 continue; 1180 1181 /* 1182 * If it's been deconstructed already, it's still 1183 * referenced, or it exceeds the trigger, skip it. 1184 * Also skip free vnodes. We are trying to make space 1185 * to expand the free list, not reduce it. 1186 */ 1187 if (vp->v_usecount > 0 || vp->v_holdcnt == 0 || 1188 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src))) 1189 goto next_iter; 1190 1191 if (vp->v_type == VBAD || vp->v_type == VNON) 1192 goto next_iter; 1193 1194 object = atomic_load_ptr(&vp->v_object); 1195 if (object == NULL || object->resident_page_count > trigger) { 1196 goto next_iter; 1197 } 1198 1199 /* 1200 * Handle races against vnode allocation. Filesystems lock the 1201 * vnode some time after it gets returned from getnewvnode, 1202 * despite type and hold count being manipulated earlier. 1203 * Resorting to checking v_mount restores guarantees present 1204 * before the global list was reworked to contain all vnodes. 1205 */ 1206 if (!VI_TRYLOCK(vp)) 1207 goto next_iter; 1208 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1209 VI_UNLOCK(vp); 1210 goto next_iter; 1211 } 1212 if (vp->v_mount == NULL) { 1213 VI_UNLOCK(vp); 1214 goto next_iter; 1215 } 1216 vholdl(vp); 1217 VI_UNLOCK(vp); 1218 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1219 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1220 mtx_unlock(&vnode_list_mtx); 1221 1222 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 1223 vdrop_recycle(vp); 1224 goto next_iter_unlocked; 1225 } 1226 if (VOP_LOCK(vp, LK_EXCLUSIVE|LK_NOWAIT) != 0) { 1227 vdrop_recycle(vp); 1228 vn_finished_write(mp); 1229 goto next_iter_unlocked; 1230 } 1231 1232 VI_LOCK(vp); 1233 if (vp->v_usecount > 0 || 1234 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) || 1235 (vp->v_object != NULL && vp->v_object->handle == vp && 1236 vp->v_object->resident_page_count > trigger)) { 1237 VOP_UNLOCK(vp); 1238 vdropl_recycle(vp); 1239 vn_finished_write(mp); 1240 goto next_iter_unlocked; 1241 } 1242 counter_u64_add(recycles_count, 1); 1243 vgonel(vp); 1244 VOP_UNLOCK(vp); 1245 vdropl_recycle(vp); 1246 vn_finished_write(mp); 1247 done++; 1248 next_iter_unlocked: 1249 maybe_yield(); 1250 mtx_lock(&vnode_list_mtx); 1251 goto restart; 1252 next_iter: 1253 MPASS(vp->v_type != VMARKER); 1254 if (!should_yield()) 1255 continue; 1256 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1257 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1258 mtx_unlock(&vnode_list_mtx); 1259 kern_yield(PRI_USER); 1260 mtx_lock(&vnode_list_mtx); 1261 goto restart; 1262 } 1263 if (done == 0 && !retried) { 1264 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1265 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); 1266 retried = true; 1267 goto restart; 1268 } 1269 return (done); 1270 } 1271 1272 static int max_vnlru_free = 10000; /* limit on vnode free requests per call */ 1273 SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_vnlru_free, 1274 0, 1275 "limit on vnode free requests per call to the vnlru_free routine"); 1276 1277 /* 1278 * Attempt to reduce the free list by the requested amount. 1279 */ 1280 static int 1281 vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp) 1282 { 1283 struct vnode *vp; 1284 struct mount *mp; 1285 int ocount; 1286 1287 mtx_assert(&vnode_list_mtx, MA_OWNED); 1288 if (count > max_vnlru_free) 1289 count = max_vnlru_free; 1290 ocount = count; 1291 vp = mvp; 1292 for (;;) { 1293 if (count == 0) { 1294 break; 1295 } 1296 vp = TAILQ_NEXT(vp, v_vnodelist); 1297 if (__predict_false(vp == NULL)) { 1298 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1299 TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist); 1300 break; 1301 } 1302 if (__predict_false(vp->v_type == VMARKER)) 1303 continue; 1304 if (vp->v_holdcnt > 0) 1305 continue; 1306 /* 1307 * Don't recycle if our vnode is from different type 1308 * of mount point. Note that mp is type-safe, the 1309 * check does not reach unmapped address even if 1310 * vnode is reclaimed. 1311 */ 1312 if (mnt_op != NULL && (mp = vp->v_mount) != NULL && 1313 mp->mnt_op != mnt_op) { 1314 continue; 1315 } 1316 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1317 continue; 1318 } 1319 if (!vhold_recycle_free(vp)) 1320 continue; 1321 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1322 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1323 mtx_unlock(&vnode_list_mtx); 1324 /* 1325 * FIXME: ignores the return value, meaning it may be nothing 1326 * got recycled but it claims otherwise to the caller. 1327 * 1328 * Originally the value started being ignored in 2005 with 1329 * 114a1006a8204aa156e1f9ad6476cdff89cada7f . 1330 * 1331 * Respecting the value can run into significant stalls if most 1332 * vnodes belong to one file system and it has writes 1333 * suspended. In presence of many threads and millions of 1334 * vnodes they keep contending on the vnode_list_mtx lock only 1335 * to find vnodes they can't recycle. 1336 * 1337 * The solution would be to pre-check if the vnode is likely to 1338 * be recycle-able, but it needs to happen with the 1339 * vnode_list_mtx lock held. This runs into a problem where 1340 * VOP_GETWRITEMOUNT (currently needed to find out about if 1341 * writes are frozen) can take locks which LOR against it. 1342 * 1343 * Check nullfs for one example (null_getwritemount). 1344 */ 1345 vtryrecycle(vp); 1346 count--; 1347 mtx_lock(&vnode_list_mtx); 1348 vp = mvp; 1349 } 1350 return (ocount - count); 1351 } 1352 1353 static int 1354 vnlru_free_locked(int count) 1355 { 1356 1357 mtx_assert(&vnode_list_mtx, MA_OWNED); 1358 return (vnlru_free_impl(count, NULL, vnode_list_free_marker)); 1359 } 1360 1361 void 1362 vnlru_free_vfsops(int count, struct vfsops *mnt_op, struct vnode *mvp) 1363 { 1364 1365 MPASS(mnt_op != NULL); 1366 MPASS(mvp != NULL); 1367 VNPASS(mvp->v_type == VMARKER, mvp); 1368 mtx_lock(&vnode_list_mtx); 1369 vnlru_free_impl(count, mnt_op, mvp); 1370 mtx_unlock(&vnode_list_mtx); 1371 } 1372 1373 struct vnode * 1374 vnlru_alloc_marker(void) 1375 { 1376 struct vnode *mvp; 1377 1378 mvp = vn_alloc_marker(NULL); 1379 mtx_lock(&vnode_list_mtx); 1380 TAILQ_INSERT_BEFORE(vnode_list_free_marker, mvp, v_vnodelist); 1381 mtx_unlock(&vnode_list_mtx); 1382 return (mvp); 1383 } 1384 1385 void 1386 vnlru_free_marker(struct vnode *mvp) 1387 { 1388 mtx_lock(&vnode_list_mtx); 1389 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1390 mtx_unlock(&vnode_list_mtx); 1391 vn_free_marker(mvp); 1392 } 1393 1394 static void 1395 vnlru_recalc(void) 1396 { 1397 1398 mtx_assert(&vnode_list_mtx, MA_OWNED); 1399 gapvnodes = imax(desiredvnodes - wantfreevnodes, 100); 1400 vhiwat = gapvnodes / 11; /* 9% -- just under the 10% in vlrureclaim() */ 1401 vlowat = vhiwat / 2; 1402 } 1403 1404 /* 1405 * Attempt to recycle vnodes in a context that is always safe to block. 1406 * Calling vlrurecycle() from the bowels of filesystem code has some 1407 * interesting deadlock problems. 1408 */ 1409 static struct proc *vnlruproc; 1410 static int vnlruproc_sig; 1411 1412 /* 1413 * The main freevnodes counter is only updated when threads requeue their vnode 1414 * batches. CPUs are conditionally walked to compute a more accurate total. 1415 * 1416 * Limit how much of a slop are we willing to tolerate. Note: the actual value 1417 * at any given moment can still exceed slop, but it should not be by significant 1418 * margin in practice. 1419 */ 1420 #define VNLRU_FREEVNODES_SLOP 126 1421 1422 static void __noinline 1423 vfs_freevnodes_rollup(int8_t *lfreevnodes) 1424 { 1425 1426 atomic_add_long(&freevnodes, *lfreevnodes); 1427 *lfreevnodes = 0; 1428 critical_exit(); 1429 } 1430 1431 static __inline void 1432 vfs_freevnodes_inc(void) 1433 { 1434 int8_t *lfreevnodes; 1435 1436 critical_enter(); 1437 lfreevnodes = PCPU_PTR(vfs_freevnodes); 1438 (*lfreevnodes)++; 1439 if (__predict_false(*lfreevnodes == VNLRU_FREEVNODES_SLOP)) 1440 vfs_freevnodes_rollup(lfreevnodes); 1441 else 1442 critical_exit(); 1443 } 1444 1445 static __inline void 1446 vfs_freevnodes_dec(void) 1447 { 1448 int8_t *lfreevnodes; 1449 1450 critical_enter(); 1451 lfreevnodes = PCPU_PTR(vfs_freevnodes); 1452 (*lfreevnodes)--; 1453 if (__predict_false(*lfreevnodes == -VNLRU_FREEVNODES_SLOP)) 1454 vfs_freevnodes_rollup(lfreevnodes); 1455 else 1456 critical_exit(); 1457 } 1458 1459 static u_long 1460 vnlru_read_freevnodes(void) 1461 { 1462 long slop, rfreevnodes; 1463 int cpu; 1464 1465 rfreevnodes = atomic_load_long(&freevnodes); 1466 1467 if (rfreevnodes > freevnodes_old) 1468 slop = rfreevnodes - freevnodes_old; 1469 else 1470 slop = freevnodes_old - rfreevnodes; 1471 if (slop < VNLRU_FREEVNODES_SLOP) 1472 return (rfreevnodes >= 0 ? rfreevnodes : 0); 1473 freevnodes_old = rfreevnodes; 1474 CPU_FOREACH(cpu) { 1475 freevnodes_old += cpuid_to_pcpu[cpu]->pc_vfs_freevnodes; 1476 } 1477 return (freevnodes_old >= 0 ? freevnodes_old : 0); 1478 } 1479 1480 static bool 1481 vnlru_under(u_long rnumvnodes, u_long limit) 1482 { 1483 u_long rfreevnodes, space; 1484 1485 if (__predict_false(rnumvnodes > desiredvnodes)) 1486 return (true); 1487 1488 space = desiredvnodes - rnumvnodes; 1489 if (space < limit) { 1490 rfreevnodes = vnlru_read_freevnodes(); 1491 if (rfreevnodes > wantfreevnodes) 1492 space += rfreevnodes - wantfreevnodes; 1493 } 1494 return (space < limit); 1495 } 1496 1497 static bool 1498 vnlru_under_unlocked(u_long rnumvnodes, u_long limit) 1499 { 1500 long rfreevnodes, space; 1501 1502 if (__predict_false(rnumvnodes > desiredvnodes)) 1503 return (true); 1504 1505 space = desiredvnodes - rnumvnodes; 1506 if (space < limit) { 1507 rfreevnodes = atomic_load_long(&freevnodes); 1508 if (rfreevnodes > wantfreevnodes) 1509 space += rfreevnodes - wantfreevnodes; 1510 } 1511 return (space < limit); 1512 } 1513 1514 static void 1515 vnlru_kick(void) 1516 { 1517 1518 mtx_assert(&vnode_list_mtx, MA_OWNED); 1519 if (vnlruproc_sig == 0) { 1520 vnlruproc_sig = 1; 1521 wakeup(vnlruproc); 1522 } 1523 } 1524 1525 static void 1526 vnlru_proc(void) 1527 { 1528 u_long rnumvnodes, rfreevnodes, target; 1529 unsigned long onumvnodes; 1530 int done, force, trigger, usevnodes; 1531 bool reclaim_nc_src, want_reread; 1532 1533 EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, vnlruproc, 1534 SHUTDOWN_PRI_FIRST); 1535 1536 force = 0; 1537 want_reread = false; 1538 for (;;) { 1539 kproc_suspend_check(vnlruproc); 1540 mtx_lock(&vnode_list_mtx); 1541 rnumvnodes = atomic_load_long(&numvnodes); 1542 1543 if (want_reread) { 1544 force = vnlru_under(numvnodes, vhiwat) ? 1 : 0; 1545 want_reread = false; 1546 } 1547 1548 /* 1549 * If numvnodes is too large (due to desiredvnodes being 1550 * adjusted using its sysctl, or emergency growth), first 1551 * try to reduce it by discarding from the free list. 1552 */ 1553 if (rnumvnodes > desiredvnodes) { 1554 vnlru_free_locked(rnumvnodes - desiredvnodes); 1555 rnumvnodes = atomic_load_long(&numvnodes); 1556 } 1557 /* 1558 * Sleep if the vnode cache is in a good state. This is 1559 * when it is not over-full and has space for about a 4% 1560 * or 9% expansion (by growing its size or inexcessively 1561 * reducing its free list). Otherwise, try to reclaim 1562 * space for a 10% expansion. 1563 */ 1564 if (vstir && force == 0) { 1565 force = 1; 1566 vstir = 0; 1567 } 1568 if (force == 0 && !vnlru_under(rnumvnodes, vlowat)) { 1569 vnlruproc_sig = 0; 1570 wakeup(&vnlruproc_sig); 1571 msleep(vnlruproc, &vnode_list_mtx, 1572 PVFS|PDROP, "vlruwt", hz); 1573 continue; 1574 } 1575 rfreevnodes = vnlru_read_freevnodes(); 1576 1577 onumvnodes = rnumvnodes; 1578 /* 1579 * Calculate parameters for recycling. These are the same 1580 * throughout the loop to give some semblance of fairness. 1581 * The trigger point is to avoid recycling vnodes with lots 1582 * of resident pages. We aren't trying to free memory; we 1583 * are trying to recycle or at least free vnodes. 1584 */ 1585 if (rnumvnodes <= desiredvnodes) 1586 usevnodes = rnumvnodes - rfreevnodes; 1587 else 1588 usevnodes = rnumvnodes; 1589 if (usevnodes <= 0) 1590 usevnodes = 1; 1591 /* 1592 * The trigger value is chosen to give a conservatively 1593 * large value to ensure that it alone doesn't prevent 1594 * making progress. The value can easily be so large that 1595 * it is effectively infinite in some congested and 1596 * misconfigured cases, and this is necessary. Normally 1597 * it is about 8 to 100 (pages), which is quite large. 1598 */ 1599 trigger = vm_cnt.v_page_count * 2 / usevnodes; 1600 if (force < 2) 1601 trigger = vsmalltrigger; 1602 reclaim_nc_src = force >= 3; 1603 target = rnumvnodes * (int64_t)gapvnodes / imax(desiredvnodes, 1); 1604 target = target / 10 + 1; 1605 done = vlrureclaim(reclaim_nc_src, trigger, target); 1606 mtx_unlock(&vnode_list_mtx); 1607 if (onumvnodes > desiredvnodes && numvnodes <= desiredvnodes) 1608 uma_reclaim(UMA_RECLAIM_DRAIN); 1609 if (done == 0) { 1610 if (force == 0 || force == 1) { 1611 force = 2; 1612 continue; 1613 } 1614 if (force == 2) { 1615 force = 3; 1616 continue; 1617 } 1618 want_reread = true; 1619 force = 0; 1620 vnlru_nowhere++; 1621 tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3); 1622 } else { 1623 want_reread = true; 1624 kern_yield(PRI_USER); 1625 } 1626 } 1627 } 1628 1629 static struct kproc_desc vnlru_kp = { 1630 "vnlru", 1631 vnlru_proc, 1632 &vnlruproc 1633 }; 1634 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, 1635 &vnlru_kp); 1636 1637 /* 1638 * Routines having to do with the management of the vnode table. 1639 */ 1640 1641 /* 1642 * Try to recycle a freed vnode. We abort if anyone picks up a reference 1643 * before we actually vgone(). This function must be called with the vnode 1644 * held to prevent the vnode from being returned to the free list midway 1645 * through vgone(). 1646 */ 1647 static int 1648 vtryrecycle(struct vnode *vp) 1649 { 1650 struct mount *vnmp; 1651 1652 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 1653 VNASSERT(vp->v_holdcnt, vp, 1654 ("vtryrecycle: Recycling vp %p without a reference.", vp)); 1655 /* 1656 * This vnode may found and locked via some other list, if so we 1657 * can't recycle it yet. 1658 */ 1659 if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { 1660 CTR2(KTR_VFS, 1661 "%s: impossible to recycle, vp %p lock is already held", 1662 __func__, vp); 1663 vdrop_recycle(vp); 1664 return (EWOULDBLOCK); 1665 } 1666 /* 1667 * Don't recycle if its filesystem is being suspended. 1668 */ 1669 if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) { 1670 VOP_UNLOCK(vp); 1671 CTR2(KTR_VFS, 1672 "%s: impossible to recycle, cannot start the write for %p", 1673 __func__, vp); 1674 vdrop_recycle(vp); 1675 return (EBUSY); 1676 } 1677 /* 1678 * If we got this far, we need to acquire the interlock and see if 1679 * anyone picked up this vnode from another list. If not, we will 1680 * mark it with DOOMED via vgonel() so that anyone who does find it 1681 * will skip over it. 1682 */ 1683 VI_LOCK(vp); 1684 if (vp->v_usecount) { 1685 VOP_UNLOCK(vp); 1686 vdropl_recycle(vp); 1687 vn_finished_write(vnmp); 1688 CTR2(KTR_VFS, 1689 "%s: impossible to recycle, %p is already referenced", 1690 __func__, vp); 1691 return (EBUSY); 1692 } 1693 if (!VN_IS_DOOMED(vp)) { 1694 counter_u64_add(recycles_free_count, 1); 1695 vgonel(vp); 1696 } 1697 VOP_UNLOCK(vp); 1698 vdropl_recycle(vp); 1699 vn_finished_write(vnmp); 1700 return (0); 1701 } 1702 1703 /* 1704 * Allocate a new vnode. 1705 * 1706 * The operation never returns an error. Returning an error was disabled 1707 * in r145385 (dated 2005) with the following comment: 1708 * 1709 * XXX Not all VFS_VGET/ffs_vget callers check returns. 1710 * 1711 * Given the age of this commit (almost 15 years at the time of writing this 1712 * comment) restoring the ability to fail requires a significant audit of 1713 * all codepaths. 1714 * 1715 * The routine can try to free a vnode or stall for up to 1 second waiting for 1716 * vnlru to clear things up, but ultimately always performs a M_WAITOK allocation. 1717 */ 1718 static u_long vn_alloc_cyclecount; 1719 1720 static struct vnode * __noinline 1721 vn_alloc_hard(struct mount *mp) 1722 { 1723 u_long rnumvnodes, rfreevnodes; 1724 1725 mtx_lock(&vnode_list_mtx); 1726 rnumvnodes = atomic_load_long(&numvnodes); 1727 if (rnumvnodes + 1 < desiredvnodes) { 1728 vn_alloc_cyclecount = 0; 1729 goto alloc; 1730 } 1731 rfreevnodes = vnlru_read_freevnodes(); 1732 if (vn_alloc_cyclecount++ >= rfreevnodes) { 1733 vn_alloc_cyclecount = 0; 1734 vstir = 1; 1735 } 1736 /* 1737 * Grow the vnode cache if it will not be above its target max 1738 * after growing. Otherwise, if the free list is nonempty, try 1739 * to reclaim 1 item from it before growing the cache (possibly 1740 * above its target max if the reclamation failed or is delayed). 1741 * Otherwise, wait for some space. In all cases, schedule 1742 * vnlru_proc() if we are getting short of space. The watermarks 1743 * should be chosen so that we never wait or even reclaim from 1744 * the free list to below its target minimum. 1745 */ 1746 if (vnlru_free_locked(1) > 0) 1747 goto alloc; 1748 if (mp == NULL || (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 1749 /* 1750 * Wait for space for a new vnode. 1751 */ 1752 vnlru_kick(); 1753 msleep(&vnlruproc_sig, &vnode_list_mtx, PVFS, "vlruwk", hz); 1754 if (atomic_load_long(&numvnodes) + 1 > desiredvnodes && 1755 vnlru_read_freevnodes() > 1) 1756 vnlru_free_locked(1); 1757 } 1758 alloc: 1759 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 1760 if (vnlru_under(rnumvnodes, vlowat)) 1761 vnlru_kick(); 1762 mtx_unlock(&vnode_list_mtx); 1763 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 1764 } 1765 1766 static struct vnode * 1767 vn_alloc(struct mount *mp) 1768 { 1769 u_long rnumvnodes; 1770 1771 if (__predict_false(vn_alloc_cyclecount != 0)) 1772 return (vn_alloc_hard(mp)); 1773 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 1774 if (__predict_false(vnlru_under_unlocked(rnumvnodes, vlowat))) { 1775 atomic_subtract_long(&numvnodes, 1); 1776 return (vn_alloc_hard(mp)); 1777 } 1778 1779 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 1780 } 1781 1782 static void 1783 vn_free(struct vnode *vp) 1784 { 1785 1786 atomic_subtract_long(&numvnodes, 1); 1787 uma_zfree_smr(vnode_zone, vp); 1788 } 1789 1790 /* 1791 * Return the next vnode from the free list. 1792 */ 1793 int 1794 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops, 1795 struct vnode **vpp) 1796 { 1797 struct vnode *vp; 1798 struct thread *td; 1799 struct lock_object *lo; 1800 1801 CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag); 1802 1803 KASSERT(vops->registered, 1804 ("%s: not registered vector op %p\n", __func__, vops)); 1805 1806 td = curthread; 1807 if (td->td_vp_reserved != NULL) { 1808 vp = td->td_vp_reserved; 1809 td->td_vp_reserved = NULL; 1810 } else { 1811 vp = vn_alloc(mp); 1812 } 1813 counter_u64_add(vnodes_created, 1); 1814 1815 vn_set_state(vp, VSTATE_UNINITIALIZED); 1816 1817 /* 1818 * Locks are given the generic name "vnode" when created. 1819 * Follow the historic practice of using the filesystem 1820 * name when they allocated, e.g., "zfs", "ufs", "nfs, etc. 1821 * 1822 * Locks live in a witness group keyed on their name. Thus, 1823 * when a lock is renamed, it must also move from the witness 1824 * group of its old name to the witness group of its new name. 1825 * 1826 * The change only needs to be made when the vnode moves 1827 * from one filesystem type to another. We ensure that each 1828 * filesystem use a single static name pointer for its tag so 1829 * that we can compare pointers rather than doing a strcmp(). 1830 */ 1831 lo = &vp->v_vnlock->lock_object; 1832 #ifdef WITNESS 1833 if (lo->lo_name != tag) { 1834 #endif 1835 lo->lo_name = tag; 1836 #ifdef WITNESS 1837 WITNESS_DESTROY(lo); 1838 WITNESS_INIT(lo, tag); 1839 } 1840 #endif 1841 /* 1842 * By default, don't allow shared locks unless filesystems opt-in. 1843 */ 1844 vp->v_vnlock->lock_object.lo_flags |= LK_NOSHARE; 1845 /* 1846 * Finalize various vnode identity bits. 1847 */ 1848 KASSERT(vp->v_object == NULL, ("stale v_object %p", vp)); 1849 KASSERT(vp->v_lockf == NULL, ("stale v_lockf %p", vp)); 1850 KASSERT(vp->v_pollinfo == NULL, ("stale v_pollinfo %p", vp)); 1851 vp->v_type = VNON; 1852 vp->v_op = vops; 1853 vp->v_irflag = 0; 1854 v_init_counters(vp); 1855 vn_seqc_init(vp); 1856 vp->v_bufobj.bo_ops = &buf_ops_bio; 1857 #ifdef DIAGNOSTIC 1858 if (mp == NULL && vops != &dead_vnodeops) 1859 printf("NULL mp in getnewvnode(9), tag %s\n", tag); 1860 #endif 1861 #ifdef MAC 1862 mac_vnode_init(vp); 1863 if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0) 1864 mac_vnode_associate_singlelabel(mp, vp); 1865 #endif 1866 if (mp != NULL) { 1867 vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize; 1868 } 1869 1870 /* 1871 * For the filesystems which do not use vfs_hash_insert(), 1872 * still initialize v_hash to have vfs_hash_index() useful. 1873 * E.g., nullfs uses vfs_hash_index() on the lower vnode for 1874 * its own hashing. 1875 */ 1876 vp->v_hash = (uintptr_t)vp >> vnsz2log; 1877 1878 *vpp = vp; 1879 return (0); 1880 } 1881 1882 void 1883 getnewvnode_reserve(void) 1884 { 1885 struct thread *td; 1886 1887 td = curthread; 1888 MPASS(td->td_vp_reserved == NULL); 1889 td->td_vp_reserved = vn_alloc(NULL); 1890 } 1891 1892 void 1893 getnewvnode_drop_reserve(void) 1894 { 1895 struct thread *td; 1896 1897 td = curthread; 1898 if (td->td_vp_reserved != NULL) { 1899 vn_free(td->td_vp_reserved); 1900 td->td_vp_reserved = NULL; 1901 } 1902 } 1903 1904 static void __noinline 1905 freevnode(struct vnode *vp) 1906 { 1907 struct bufobj *bo; 1908 1909 /* 1910 * The vnode has been marked for destruction, so free it. 1911 * 1912 * The vnode will be returned to the zone where it will 1913 * normally remain until it is needed for another vnode. We 1914 * need to cleanup (or verify that the cleanup has already 1915 * been done) any residual data left from its current use 1916 * so as not to contaminate the freshly allocated vnode. 1917 */ 1918 CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); 1919 /* 1920 * Paired with vgone. 1921 */ 1922 vn_seqc_write_end_free(vp); 1923 1924 bo = &vp->v_bufobj; 1925 VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); 1926 VNPASS(vp->v_holdcnt == VHOLD_NO_SMR, vp); 1927 VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); 1928 VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); 1929 VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); 1930 VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); 1931 VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp, 1932 ("clean blk trie not empty")); 1933 VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0")); 1934 VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp, 1935 ("dirty blk trie not empty")); 1936 VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp, 1937 ("Dangling rangelock waiters")); 1938 VNASSERT((vp->v_iflag & (VI_DOINGINACT | VI_OWEINACT)) == 0, vp, 1939 ("Leaked inactivation")); 1940 VI_UNLOCK(vp); 1941 cache_assert_no_entries(vp); 1942 1943 #ifdef MAC 1944 mac_vnode_destroy(vp); 1945 #endif 1946 if (vp->v_pollinfo != NULL) { 1947 /* 1948 * Use LK_NOWAIT to shut up witness about the lock. We may get 1949 * here while having another vnode locked when trying to 1950 * satisfy a lookup and needing to recycle. 1951 */ 1952 VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT); 1953 destroy_vpollinfo(vp->v_pollinfo); 1954 VOP_UNLOCK(vp); 1955 vp->v_pollinfo = NULL; 1956 } 1957 vp->v_mountedhere = NULL; 1958 vp->v_unpcb = NULL; 1959 vp->v_rdev = NULL; 1960 vp->v_fifoinfo = NULL; 1961 vp->v_iflag = 0; 1962 vp->v_vflag = 0; 1963 bo->bo_flag = 0; 1964 vn_free(vp); 1965 } 1966 1967 /* 1968 * Delete from old mount point vnode list, if on one. 1969 */ 1970 static void 1971 delmntque(struct vnode *vp) 1972 { 1973 struct mount *mp; 1974 1975 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 1976 1977 mp = vp->v_mount; 1978 MNT_ILOCK(mp); 1979 VI_LOCK(vp); 1980 vp->v_mount = NULL; 1981 VNASSERT(mp->mnt_nvnodelistsize > 0, vp, 1982 ("bad mount point vnode list size")); 1983 TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 1984 mp->mnt_nvnodelistsize--; 1985 MNT_REL(mp); 1986 MNT_IUNLOCK(mp); 1987 /* 1988 * The caller expects the interlock to be still held. 1989 */ 1990 ASSERT_VI_LOCKED(vp, __func__); 1991 } 1992 1993 static int 1994 insmntque1_int(struct vnode *vp, struct mount *mp, bool dtr) 1995 { 1996 1997 KASSERT(vp->v_mount == NULL, 1998 ("insmntque: vnode already on per mount vnode list")); 1999 VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)")); 2000 if ((mp->mnt_kern_flag & MNTK_UNLOCKED_INSMNTQUE) == 0) { 2001 ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp"); 2002 } else { 2003 KASSERT(!dtr, 2004 ("%s: can't have MNTK_UNLOCKED_INSMNTQUE and cleanup", 2005 __func__)); 2006 } 2007 2008 /* 2009 * We acquire the vnode interlock early to ensure that the 2010 * vnode cannot be recycled by another process releasing a 2011 * holdcnt on it before we get it on both the vnode list 2012 * and the active vnode list. The mount mutex protects only 2013 * manipulation of the vnode list and the vnode freelist 2014 * mutex protects only manipulation of the active vnode list. 2015 * Hence the need to hold the vnode interlock throughout. 2016 */ 2017 MNT_ILOCK(mp); 2018 VI_LOCK(vp); 2019 if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 && 2020 ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 || 2021 mp->mnt_nvnodelistsize == 0)) && 2022 (vp->v_vflag & VV_FORCEINSMQ) == 0) { 2023 VI_UNLOCK(vp); 2024 MNT_IUNLOCK(mp); 2025 if (dtr) { 2026 vp->v_data = NULL; 2027 vp->v_op = &dead_vnodeops; 2028 vgone(vp); 2029 vput(vp); 2030 } 2031 return (EBUSY); 2032 } 2033 vp->v_mount = mp; 2034 MNT_REF(mp); 2035 TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 2036 VNASSERT(mp->mnt_nvnodelistsize >= 0, vp, 2037 ("neg mount point vnode list size")); 2038 mp->mnt_nvnodelistsize++; 2039 VI_UNLOCK(vp); 2040 MNT_IUNLOCK(mp); 2041 return (0); 2042 } 2043 2044 /* 2045 * Insert into list of vnodes for the new mount point, if available. 2046 * insmntque() reclaims the vnode on insertion failure, insmntque1() 2047 * leaves handling of the vnode to the caller. 2048 */ 2049 int 2050 insmntque(struct vnode *vp, struct mount *mp) 2051 { 2052 return (insmntque1_int(vp, mp, true)); 2053 } 2054 2055 int 2056 insmntque1(struct vnode *vp, struct mount *mp) 2057 { 2058 return (insmntque1_int(vp, mp, false)); 2059 } 2060 2061 /* 2062 * Flush out and invalidate all buffers associated with a bufobj 2063 * Called with the underlying object locked. 2064 */ 2065 int 2066 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo) 2067 { 2068 int error; 2069 2070 BO_LOCK(bo); 2071 if (flags & V_SAVE) { 2072 error = bufobj_wwait(bo, slpflag, slptimeo); 2073 if (error) { 2074 BO_UNLOCK(bo); 2075 return (error); 2076 } 2077 if (bo->bo_dirty.bv_cnt > 0) { 2078 BO_UNLOCK(bo); 2079 do { 2080 error = BO_SYNC(bo, MNT_WAIT); 2081 } while (error == ERELOOKUP); 2082 if (error != 0) 2083 return (error); 2084 BO_LOCK(bo); 2085 if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0) { 2086 BO_UNLOCK(bo); 2087 return (EBUSY); 2088 } 2089 } 2090 } 2091 /* 2092 * If you alter this loop please notice that interlock is dropped and 2093 * reacquired in flushbuflist. Special care is needed to ensure that 2094 * no race conditions occur from this. 2095 */ 2096 do { 2097 error = flushbuflist(&bo->bo_clean, 2098 flags, bo, slpflag, slptimeo); 2099 if (error == 0 && !(flags & V_CLEANONLY)) 2100 error = flushbuflist(&bo->bo_dirty, 2101 flags, bo, slpflag, slptimeo); 2102 if (error != 0 && error != EAGAIN) { 2103 BO_UNLOCK(bo); 2104 return (error); 2105 } 2106 } while (error != 0); 2107 2108 /* 2109 * Wait for I/O to complete. XXX needs cleaning up. The vnode can 2110 * have write I/O in-progress but if there is a VM object then the 2111 * VM object can also have read-I/O in-progress. 2112 */ 2113 do { 2114 bufobj_wwait(bo, 0, 0); 2115 if ((flags & V_VMIO) == 0 && bo->bo_object != NULL) { 2116 BO_UNLOCK(bo); 2117 vm_object_pip_wait_unlocked(bo->bo_object, "bovlbx"); 2118 BO_LOCK(bo); 2119 } 2120 } while (bo->bo_numoutput > 0); 2121 BO_UNLOCK(bo); 2122 2123 /* 2124 * Destroy the copy in the VM cache, too. 2125 */ 2126 if (bo->bo_object != NULL && 2127 (flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0) { 2128 VM_OBJECT_WLOCK(bo->bo_object); 2129 vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ? 2130 OBJPR_CLEANONLY : 0); 2131 VM_OBJECT_WUNLOCK(bo->bo_object); 2132 } 2133 2134 #ifdef INVARIANTS 2135 BO_LOCK(bo); 2136 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO | 2137 V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 || 2138 bo->bo_clean.bv_cnt > 0)) 2139 panic("vinvalbuf: flush failed"); 2140 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0 && 2141 bo->bo_dirty.bv_cnt > 0) 2142 panic("vinvalbuf: flush dirty failed"); 2143 BO_UNLOCK(bo); 2144 #endif 2145 return (0); 2146 } 2147 2148 /* 2149 * Flush out and invalidate all buffers associated with a vnode. 2150 * Called with the underlying object locked. 2151 */ 2152 int 2153 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) 2154 { 2155 2156 CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags); 2157 ASSERT_VOP_LOCKED(vp, "vinvalbuf"); 2158 if (vp->v_object != NULL && vp->v_object->handle != vp) 2159 return (0); 2160 return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo)); 2161 } 2162 2163 /* 2164 * Flush out buffers on the specified list. 2165 * 2166 */ 2167 static int 2168 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, 2169 int slptimeo) 2170 { 2171 struct buf *bp, *nbp; 2172 int retval, error; 2173 daddr_t lblkno; 2174 b_xflags_t xflags; 2175 2176 ASSERT_BO_WLOCKED(bo); 2177 2178 retval = 0; 2179 TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) { 2180 /* 2181 * If we are flushing both V_NORMAL and V_ALT buffers then 2182 * do not skip any buffers. If we are flushing only V_NORMAL 2183 * buffers then skip buffers marked as BX_ALTDATA. If we are 2184 * flushing only V_ALT buffers then skip buffers not marked 2185 * as BX_ALTDATA. 2186 */ 2187 if (((flags & (V_NORMAL | V_ALT)) != (V_NORMAL | V_ALT)) && 2188 (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) || 2189 ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) { 2190 continue; 2191 } 2192 if (nbp != NULL) { 2193 lblkno = nbp->b_lblkno; 2194 xflags = nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN); 2195 } 2196 retval = EAGAIN; 2197 error = BUF_TIMELOCK(bp, 2198 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo), 2199 "flushbuf", slpflag, slptimeo); 2200 if (error) { 2201 BO_LOCK(bo); 2202 return (error != ENOLCK ? error : EAGAIN); 2203 } 2204 KASSERT(bp->b_bufobj == bo, 2205 ("bp %p wrong b_bufobj %p should be %p", 2206 bp, bp->b_bufobj, bo)); 2207 /* 2208 * XXX Since there are no node locks for NFS, I 2209 * believe there is a slight chance that a delayed 2210 * write will occur while sleeping just above, so 2211 * check for it. 2212 */ 2213 if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) && 2214 (flags & V_SAVE)) { 2215 bremfree(bp); 2216 bp->b_flags |= B_ASYNC; 2217 bwrite(bp); 2218 BO_LOCK(bo); 2219 return (EAGAIN); /* XXX: why not loop ? */ 2220 } 2221 bremfree(bp); 2222 bp->b_flags |= (B_INVAL | B_RELBUF); 2223 bp->b_flags &= ~B_ASYNC; 2224 brelse(bp); 2225 BO_LOCK(bo); 2226 if (nbp == NULL) 2227 break; 2228 nbp = gbincore(bo, lblkno); 2229 if (nbp == NULL || (nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) 2230 != xflags) 2231 break; /* nbp invalid */ 2232 } 2233 return (retval); 2234 } 2235 2236 int 2237 bnoreuselist(struct bufv *bufv, struct bufobj *bo, daddr_t startn, daddr_t endn) 2238 { 2239 struct buf *bp; 2240 int error; 2241 daddr_t lblkno; 2242 2243 ASSERT_BO_LOCKED(bo); 2244 2245 for (lblkno = startn;;) { 2246 again: 2247 bp = BUF_PCTRIE_LOOKUP_GE(&bufv->bv_root, lblkno); 2248 if (bp == NULL || bp->b_lblkno >= endn || 2249 bp->b_lblkno < startn) 2250 break; 2251 error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 2252 LK_INTERLOCK, BO_LOCKPTR(bo), "brlsfl", 0, 0); 2253 if (error != 0) { 2254 BO_RLOCK(bo); 2255 if (error == ENOLCK) 2256 goto again; 2257 return (error); 2258 } 2259 KASSERT(bp->b_bufobj == bo, 2260 ("bp %p wrong b_bufobj %p should be %p", 2261 bp, bp->b_bufobj, bo)); 2262 lblkno = bp->b_lblkno + 1; 2263 if ((bp->b_flags & B_MANAGED) == 0) 2264 bremfree(bp); 2265 bp->b_flags |= B_RELBUF; 2266 /* 2267 * In the VMIO case, use the B_NOREUSE flag to hint that the 2268 * pages backing each buffer in the range are unlikely to be 2269 * reused. Dirty buffers will have the hint applied once 2270 * they've been written. 2271 */ 2272 if ((bp->b_flags & B_VMIO) != 0) 2273 bp->b_flags |= B_NOREUSE; 2274 brelse(bp); 2275 BO_RLOCK(bo); 2276 } 2277 return (0); 2278 } 2279 2280 /* 2281 * Truncate a file's buffer and pages to a specified length. This 2282 * is in lieu of the old vinvalbuf mechanism, which performed unneeded 2283 * sync activity. 2284 */ 2285 int 2286 vtruncbuf(struct vnode *vp, off_t length, int blksize) 2287 { 2288 struct buf *bp, *nbp; 2289 struct bufobj *bo; 2290 daddr_t startlbn; 2291 2292 CTR4(KTR_VFS, "%s: vp %p with block %d:%ju", __func__, 2293 vp, blksize, (uintmax_t)length); 2294 2295 /* 2296 * Round up to the *next* lbn. 2297 */ 2298 startlbn = howmany(length, blksize); 2299 2300 ASSERT_VOP_LOCKED(vp, "vtruncbuf"); 2301 2302 bo = &vp->v_bufobj; 2303 restart_unlocked: 2304 BO_LOCK(bo); 2305 2306 while (v_inval_buf_range_locked(vp, bo, startlbn, INT64_MAX) == EAGAIN) 2307 ; 2308 2309 if (length > 0) { 2310 restartsync: 2311 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2312 if (bp->b_lblkno > 0) 2313 continue; 2314 /* 2315 * Since we hold the vnode lock this should only 2316 * fail if we're racing with the buf daemon. 2317 */ 2318 if (BUF_LOCK(bp, 2319 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2320 BO_LOCKPTR(bo)) == ENOLCK) 2321 goto restart_unlocked; 2322 2323 VNASSERT((bp->b_flags & B_DELWRI), vp, 2324 ("buf(%p) on dirty queue without DELWRI", bp)); 2325 2326 bremfree(bp); 2327 bawrite(bp); 2328 BO_LOCK(bo); 2329 goto restartsync; 2330 } 2331 } 2332 2333 bufobj_wwait(bo, 0, 0); 2334 BO_UNLOCK(bo); 2335 vnode_pager_setsize(vp, length); 2336 2337 return (0); 2338 } 2339 2340 /* 2341 * Invalidate the cached pages of a file's buffer within the range of block 2342 * numbers [startlbn, endlbn). 2343 */ 2344 void 2345 v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn, 2346 int blksize) 2347 { 2348 struct bufobj *bo; 2349 off_t start, end; 2350 2351 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range"); 2352 2353 start = blksize * startlbn; 2354 end = blksize * endlbn; 2355 2356 bo = &vp->v_bufobj; 2357 BO_LOCK(bo); 2358 MPASS(blksize == bo->bo_bsize); 2359 2360 while (v_inval_buf_range_locked(vp, bo, startlbn, endlbn) == EAGAIN) 2361 ; 2362 2363 BO_UNLOCK(bo); 2364 vn_pages_remove(vp, OFF_TO_IDX(start), OFF_TO_IDX(end + PAGE_SIZE - 1)); 2365 } 2366 2367 static int 2368 v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 2369 daddr_t startlbn, daddr_t endlbn) 2370 { 2371 struct buf *bp, *nbp; 2372 bool anyfreed; 2373 2374 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range_locked"); 2375 ASSERT_BO_LOCKED(bo); 2376 2377 do { 2378 anyfreed = false; 2379 TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) { 2380 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn) 2381 continue; 2382 if (BUF_LOCK(bp, 2383 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2384 BO_LOCKPTR(bo)) == ENOLCK) { 2385 BO_LOCK(bo); 2386 return (EAGAIN); 2387 } 2388 2389 bremfree(bp); 2390 bp->b_flags |= B_INVAL | B_RELBUF; 2391 bp->b_flags &= ~B_ASYNC; 2392 brelse(bp); 2393 anyfreed = true; 2394 2395 BO_LOCK(bo); 2396 if (nbp != NULL && 2397 (((nbp->b_xflags & BX_VNCLEAN) == 0) || 2398 nbp->b_vp != vp || 2399 (nbp->b_flags & B_DELWRI) != 0)) 2400 return (EAGAIN); 2401 } 2402 2403 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2404 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn) 2405 continue; 2406 if (BUF_LOCK(bp, 2407 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2408 BO_LOCKPTR(bo)) == ENOLCK) { 2409 BO_LOCK(bo); 2410 return (EAGAIN); 2411 } 2412 bremfree(bp); 2413 bp->b_flags |= B_INVAL | B_RELBUF; 2414 bp->b_flags &= ~B_ASYNC; 2415 brelse(bp); 2416 anyfreed = true; 2417 2418 BO_LOCK(bo); 2419 if (nbp != NULL && 2420 (((nbp->b_xflags & BX_VNDIRTY) == 0) || 2421 (nbp->b_vp != vp) || 2422 (nbp->b_flags & B_DELWRI) == 0)) 2423 return (EAGAIN); 2424 } 2425 } while (anyfreed); 2426 return (0); 2427 } 2428 2429 static void 2430 buf_vlist_remove(struct buf *bp) 2431 { 2432 struct bufv *bv; 2433 b_xflags_t flags; 2434 2435 flags = bp->b_xflags; 2436 2437 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp)); 2438 ASSERT_BO_WLOCKED(bp->b_bufobj); 2439 KASSERT((flags & (BX_VNDIRTY | BX_VNCLEAN)) != 0 && 2440 (flags & (BX_VNDIRTY | BX_VNCLEAN)) != (BX_VNDIRTY | BX_VNCLEAN), 2441 ("%s: buffer %p has invalid queue state", __func__, bp)); 2442 2443 if ((flags & BX_VNDIRTY) != 0) 2444 bv = &bp->b_bufobj->bo_dirty; 2445 else 2446 bv = &bp->b_bufobj->bo_clean; 2447 BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno); 2448 TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs); 2449 bv->bv_cnt--; 2450 bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN); 2451 } 2452 2453 /* 2454 * Add the buffer to the sorted clean or dirty block list. 2455 * 2456 * NOTE: xflags is passed as a constant, optimizing this inline function! 2457 */ 2458 static void 2459 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags) 2460 { 2461 struct bufv *bv; 2462 struct buf *n; 2463 int error; 2464 2465 ASSERT_BO_WLOCKED(bo); 2466 KASSERT((bo->bo_flag & BO_NOBUFS) == 0, 2467 ("buf_vlist_add: bo %p does not allow bufs", bo)); 2468 KASSERT((xflags & BX_VNDIRTY) == 0 || (bo->bo_flag & BO_DEAD) == 0, 2469 ("dead bo %p", bo)); 2470 KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, 2471 ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags)); 2472 bp->b_xflags |= xflags; 2473 if (xflags & BX_VNDIRTY) 2474 bv = &bo->bo_dirty; 2475 else 2476 bv = &bo->bo_clean; 2477 2478 /* 2479 * Keep the list ordered. Optimize empty list insertion. Assume 2480 * we tend to grow at the tail so lookup_le should usually be cheaper 2481 * than _ge. 2482 */ 2483 if (bv->bv_cnt == 0 || 2484 bp->b_lblkno > TAILQ_LAST(&bv->bv_hd, buflists)->b_lblkno) 2485 TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs); 2486 else if ((n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, bp->b_lblkno)) == NULL) 2487 TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs); 2488 else 2489 TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs); 2490 error = BUF_PCTRIE_INSERT(&bv->bv_root, bp); 2491 if (error) 2492 panic("buf_vlist_add: Preallocated nodes insufficient."); 2493 bv->bv_cnt++; 2494 } 2495 2496 /* 2497 * Look up a buffer using the buffer tries. 2498 */ 2499 struct buf * 2500 gbincore(struct bufobj *bo, daddr_t lblkno) 2501 { 2502 struct buf *bp; 2503 2504 ASSERT_BO_LOCKED(bo); 2505 bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno); 2506 if (bp != NULL) 2507 return (bp); 2508 return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno)); 2509 } 2510 2511 /* 2512 * Look up a buf using the buffer tries, without the bufobj lock. This relies 2513 * on SMR for safe lookup, and bufs being in a no-free zone to provide type 2514 * stability of the result. Like other lockless lookups, the found buf may 2515 * already be invalid by the time this function returns. 2516 */ 2517 struct buf * 2518 gbincore_unlocked(struct bufobj *bo, daddr_t lblkno) 2519 { 2520 struct buf *bp; 2521 2522 ASSERT_BO_UNLOCKED(bo); 2523 bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno); 2524 if (bp != NULL) 2525 return (bp); 2526 return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno)); 2527 } 2528 2529 /* 2530 * Associate a buffer with a vnode. 2531 */ 2532 void 2533 bgetvp(struct vnode *vp, struct buf *bp) 2534 { 2535 struct bufobj *bo; 2536 2537 bo = &vp->v_bufobj; 2538 ASSERT_BO_WLOCKED(bo); 2539 VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free")); 2540 2541 CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags); 2542 VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp, 2543 ("bgetvp: bp already attached! %p", bp)); 2544 2545 vhold(vp); 2546 bp->b_vp = vp; 2547 bp->b_bufobj = bo; 2548 /* 2549 * Insert onto list for new vnode. 2550 */ 2551 buf_vlist_add(bp, bo, BX_VNCLEAN); 2552 } 2553 2554 /* 2555 * Disassociate a buffer from a vnode. 2556 */ 2557 void 2558 brelvp(struct buf *bp) 2559 { 2560 struct bufobj *bo; 2561 struct vnode *vp; 2562 2563 CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags); 2564 KASSERT(bp->b_vp != NULL, ("brelvp: NULL")); 2565 2566 /* 2567 * Delete from old vnode list, if on one. 2568 */ 2569 vp = bp->b_vp; /* XXX */ 2570 bo = bp->b_bufobj; 2571 BO_LOCK(bo); 2572 buf_vlist_remove(bp); 2573 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2574 bo->bo_flag &= ~BO_ONWORKLST; 2575 mtx_lock(&sync_mtx); 2576 LIST_REMOVE(bo, bo_synclist); 2577 syncer_worklist_len--; 2578 mtx_unlock(&sync_mtx); 2579 } 2580 bp->b_vp = NULL; 2581 bp->b_bufobj = NULL; 2582 BO_UNLOCK(bo); 2583 vdrop(vp); 2584 } 2585 2586 /* 2587 * Add an item to the syncer work queue. 2588 */ 2589 static void 2590 vn_syncer_add_to_worklist(struct bufobj *bo, int delay) 2591 { 2592 int slot; 2593 2594 ASSERT_BO_WLOCKED(bo); 2595 2596 mtx_lock(&sync_mtx); 2597 if (bo->bo_flag & BO_ONWORKLST) 2598 LIST_REMOVE(bo, bo_synclist); 2599 else { 2600 bo->bo_flag |= BO_ONWORKLST; 2601 syncer_worklist_len++; 2602 } 2603 2604 if (delay > syncer_maxdelay - 2) 2605 delay = syncer_maxdelay - 2; 2606 slot = (syncer_delayno + delay) & syncer_mask; 2607 2608 LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist); 2609 mtx_unlock(&sync_mtx); 2610 } 2611 2612 static int 2613 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS) 2614 { 2615 int error, len; 2616 2617 mtx_lock(&sync_mtx); 2618 len = syncer_worklist_len - sync_vnode_count; 2619 mtx_unlock(&sync_mtx); 2620 error = SYSCTL_OUT(req, &len, sizeof(len)); 2621 return (error); 2622 } 2623 2624 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, 2625 CTLTYPE_INT | CTLFLAG_MPSAFE| CTLFLAG_RD, NULL, 0, 2626 sysctl_vfs_worklist_len, "I", "Syncer thread worklist length"); 2627 2628 static struct proc *updateproc; 2629 static void sched_sync(void); 2630 static struct kproc_desc up_kp = { 2631 "syncer", 2632 sched_sync, 2633 &updateproc 2634 }; 2635 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp); 2636 2637 static int 2638 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td) 2639 { 2640 struct vnode *vp; 2641 struct mount *mp; 2642 2643 *bo = LIST_FIRST(slp); 2644 if (*bo == NULL) 2645 return (0); 2646 vp = bo2vnode(*bo); 2647 if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0) 2648 return (1); 2649 /* 2650 * We use vhold in case the vnode does not 2651 * successfully sync. vhold prevents the vnode from 2652 * going away when we unlock the sync_mtx so that 2653 * we can acquire the vnode interlock. 2654 */ 2655 vholdl(vp); 2656 mtx_unlock(&sync_mtx); 2657 VI_UNLOCK(vp); 2658 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 2659 vdrop(vp); 2660 mtx_lock(&sync_mtx); 2661 return (*bo == LIST_FIRST(slp)); 2662 } 2663 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2664 (void) VOP_FSYNC(vp, MNT_LAZY, td); 2665 VOP_UNLOCK(vp); 2666 vn_finished_write(mp); 2667 BO_LOCK(*bo); 2668 if (((*bo)->bo_flag & BO_ONWORKLST) != 0) { 2669 /* 2670 * Put us back on the worklist. The worklist 2671 * routine will remove us from our current 2672 * position and then add us back in at a later 2673 * position. 2674 */ 2675 vn_syncer_add_to_worklist(*bo, syncdelay); 2676 } 2677 BO_UNLOCK(*bo); 2678 vdrop(vp); 2679 mtx_lock(&sync_mtx); 2680 return (0); 2681 } 2682 2683 static int first_printf = 1; 2684 2685 /* 2686 * System filesystem synchronizer daemon. 2687 */ 2688 static void 2689 sched_sync(void) 2690 { 2691 struct synclist *next, *slp; 2692 struct bufobj *bo; 2693 long starttime; 2694 struct thread *td = curthread; 2695 int last_work_seen; 2696 int net_worklist_len; 2697 int syncer_final_iter; 2698 int error; 2699 2700 last_work_seen = 0; 2701 syncer_final_iter = 0; 2702 syncer_state = SYNCER_RUNNING; 2703 starttime = time_uptime; 2704 td->td_pflags |= TDP_NORUNNINGBUF; 2705 2706 EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc, 2707 SHUTDOWN_PRI_LAST); 2708 2709 mtx_lock(&sync_mtx); 2710 for (;;) { 2711 if (syncer_state == SYNCER_FINAL_DELAY && 2712 syncer_final_iter == 0) { 2713 mtx_unlock(&sync_mtx); 2714 kproc_suspend_check(td->td_proc); 2715 mtx_lock(&sync_mtx); 2716 } 2717 net_worklist_len = syncer_worklist_len - sync_vnode_count; 2718 if (syncer_state != SYNCER_RUNNING && 2719 starttime != time_uptime) { 2720 if (first_printf) { 2721 printf("\nSyncing disks, vnodes remaining... "); 2722 first_printf = 0; 2723 } 2724 printf("%d ", net_worklist_len); 2725 } 2726 starttime = time_uptime; 2727 2728 /* 2729 * Push files whose dirty time has expired. Be careful 2730 * of interrupt race on slp queue. 2731 * 2732 * Skip over empty worklist slots when shutting down. 2733 */ 2734 do { 2735 slp = &syncer_workitem_pending[syncer_delayno]; 2736 syncer_delayno += 1; 2737 if (syncer_delayno == syncer_maxdelay) 2738 syncer_delayno = 0; 2739 next = &syncer_workitem_pending[syncer_delayno]; 2740 /* 2741 * If the worklist has wrapped since the 2742 * it was emptied of all but syncer vnodes, 2743 * switch to the FINAL_DELAY state and run 2744 * for one more second. 2745 */ 2746 if (syncer_state == SYNCER_SHUTTING_DOWN && 2747 net_worklist_len == 0 && 2748 last_work_seen == syncer_delayno) { 2749 syncer_state = SYNCER_FINAL_DELAY; 2750 syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP; 2751 } 2752 } while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) && 2753 syncer_worklist_len > 0); 2754 2755 /* 2756 * Keep track of the last time there was anything 2757 * on the worklist other than syncer vnodes. 2758 * Return to the SHUTTING_DOWN state if any 2759 * new work appears. 2760 */ 2761 if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING) 2762 last_work_seen = syncer_delayno; 2763 if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY) 2764 syncer_state = SYNCER_SHUTTING_DOWN; 2765 while (!LIST_EMPTY(slp)) { 2766 error = sync_vnode(slp, &bo, td); 2767 if (error == 1) { 2768 LIST_REMOVE(bo, bo_synclist); 2769 LIST_INSERT_HEAD(next, bo, bo_synclist); 2770 continue; 2771 } 2772 2773 if (first_printf == 0) { 2774 /* 2775 * Drop the sync mutex, because some watchdog 2776 * drivers need to sleep while patting 2777 */ 2778 mtx_unlock(&sync_mtx); 2779 wdog_kern_pat(WD_LASTVAL); 2780 mtx_lock(&sync_mtx); 2781 } 2782 } 2783 if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0) 2784 syncer_final_iter--; 2785 /* 2786 * The variable rushjob allows the kernel to speed up the 2787 * processing of the filesystem syncer process. A rushjob 2788 * value of N tells the filesystem syncer to process the next 2789 * N seconds worth of work on its queue ASAP. Currently rushjob 2790 * is used by the soft update code to speed up the filesystem 2791 * syncer process when the incore state is getting so far 2792 * ahead of the disk that the kernel memory pool is being 2793 * threatened with exhaustion. 2794 */ 2795 if (rushjob > 0) { 2796 rushjob -= 1; 2797 continue; 2798 } 2799 /* 2800 * Just sleep for a short period of time between 2801 * iterations when shutting down to allow some I/O 2802 * to happen. 2803 * 2804 * If it has taken us less than a second to process the 2805 * current work, then wait. Otherwise start right over 2806 * again. We can still lose time if any single round 2807 * takes more than two seconds, but it does not really 2808 * matter as we are just trying to generally pace the 2809 * filesystem activity. 2810 */ 2811 if (syncer_state != SYNCER_RUNNING || 2812 time_uptime == starttime) { 2813 thread_lock(td); 2814 sched_prio(td, PPAUSE); 2815 thread_unlock(td); 2816 } 2817 if (syncer_state != SYNCER_RUNNING) 2818 cv_timedwait(&sync_wakeup, &sync_mtx, 2819 hz / SYNCER_SHUTDOWN_SPEEDUP); 2820 else if (time_uptime == starttime) 2821 cv_timedwait(&sync_wakeup, &sync_mtx, hz); 2822 } 2823 } 2824 2825 /* 2826 * Request the syncer daemon to speed up its work. 2827 * We never push it to speed up more than half of its 2828 * normal turn time, otherwise it could take over the cpu. 2829 */ 2830 int 2831 speedup_syncer(void) 2832 { 2833 int ret = 0; 2834 2835 mtx_lock(&sync_mtx); 2836 if (rushjob < syncdelay / 2) { 2837 rushjob += 1; 2838 stat_rush_requests += 1; 2839 ret = 1; 2840 } 2841 mtx_unlock(&sync_mtx); 2842 cv_broadcast(&sync_wakeup); 2843 return (ret); 2844 } 2845 2846 /* 2847 * Tell the syncer to speed up its work and run though its work 2848 * list several times, then tell it to shut down. 2849 */ 2850 static void 2851 syncer_shutdown(void *arg, int howto) 2852 { 2853 2854 if (howto & RB_NOSYNC) 2855 return; 2856 mtx_lock(&sync_mtx); 2857 syncer_state = SYNCER_SHUTTING_DOWN; 2858 rushjob = 0; 2859 mtx_unlock(&sync_mtx); 2860 cv_broadcast(&sync_wakeup); 2861 kproc_shutdown(arg, howto); 2862 } 2863 2864 void 2865 syncer_suspend(void) 2866 { 2867 2868 syncer_shutdown(updateproc, 0); 2869 } 2870 2871 void 2872 syncer_resume(void) 2873 { 2874 2875 mtx_lock(&sync_mtx); 2876 first_printf = 1; 2877 syncer_state = SYNCER_RUNNING; 2878 mtx_unlock(&sync_mtx); 2879 cv_broadcast(&sync_wakeup); 2880 kproc_resume(updateproc); 2881 } 2882 2883 /* 2884 * Move the buffer between the clean and dirty lists of its vnode. 2885 */ 2886 void 2887 reassignbuf(struct buf *bp) 2888 { 2889 struct vnode *vp; 2890 struct bufobj *bo; 2891 int delay; 2892 #ifdef INVARIANTS 2893 struct bufv *bv; 2894 #endif 2895 2896 vp = bp->b_vp; 2897 bo = bp->b_bufobj; 2898 2899 KASSERT((bp->b_flags & B_PAGING) == 0, 2900 ("%s: cannot reassign paging buffer %p", __func__, bp)); 2901 2902 CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X", 2903 bp, bp->b_vp, bp->b_flags); 2904 2905 BO_LOCK(bo); 2906 buf_vlist_remove(bp); 2907 2908 /* 2909 * If dirty, put on list of dirty buffers; otherwise insert onto list 2910 * of clean buffers. 2911 */ 2912 if (bp->b_flags & B_DELWRI) { 2913 if ((bo->bo_flag & BO_ONWORKLST) == 0) { 2914 switch (vp->v_type) { 2915 case VDIR: 2916 delay = dirdelay; 2917 break; 2918 case VCHR: 2919 delay = metadelay; 2920 break; 2921 default: 2922 delay = filedelay; 2923 } 2924 vn_syncer_add_to_worklist(bo, delay); 2925 } 2926 buf_vlist_add(bp, bo, BX_VNDIRTY); 2927 } else { 2928 buf_vlist_add(bp, bo, BX_VNCLEAN); 2929 2930 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2931 mtx_lock(&sync_mtx); 2932 LIST_REMOVE(bo, bo_synclist); 2933 syncer_worklist_len--; 2934 mtx_unlock(&sync_mtx); 2935 bo->bo_flag &= ~BO_ONWORKLST; 2936 } 2937 } 2938 #ifdef INVARIANTS 2939 bv = &bo->bo_clean; 2940 bp = TAILQ_FIRST(&bv->bv_hd); 2941 KASSERT(bp == NULL || bp->b_bufobj == bo, 2942 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2943 bp = TAILQ_LAST(&bv->bv_hd, buflists); 2944 KASSERT(bp == NULL || bp->b_bufobj == bo, 2945 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2946 bv = &bo->bo_dirty; 2947 bp = TAILQ_FIRST(&bv->bv_hd); 2948 KASSERT(bp == NULL || bp->b_bufobj == bo, 2949 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2950 bp = TAILQ_LAST(&bv->bv_hd, buflists); 2951 KASSERT(bp == NULL || bp->b_bufobj == bo, 2952 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 2953 #endif 2954 BO_UNLOCK(bo); 2955 } 2956 2957 static void 2958 v_init_counters(struct vnode *vp) 2959 { 2960 2961 VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0, 2962 vp, ("%s called for an initialized vnode", __FUNCTION__)); 2963 ASSERT_VI_UNLOCKED(vp, __FUNCTION__); 2964 2965 refcount_init(&vp->v_holdcnt, 1); 2966 refcount_init(&vp->v_usecount, 1); 2967 } 2968 2969 /* 2970 * Grab a particular vnode from the free list, increment its 2971 * reference count and lock it. VIRF_DOOMED is set if the vnode 2972 * is being destroyed. Only callers who specify LK_RETRY will 2973 * see doomed vnodes. If inactive processing was delayed in 2974 * vput try to do it here. 2975 * 2976 * usecount is manipulated using atomics without holding any locks. 2977 * 2978 * holdcnt can be manipulated using atomics without holding any locks, 2979 * except when transitioning 1<->0, in which case the interlock is held. 2980 * 2981 * Consumers which don't guarantee liveness of the vnode can use SMR to 2982 * try to get a reference. Note this operation can fail since the vnode 2983 * may be awaiting getting freed by the time they get to it. 2984 */ 2985 enum vgetstate 2986 vget_prep_smr(struct vnode *vp) 2987 { 2988 enum vgetstate vs; 2989 2990 VFS_SMR_ASSERT_ENTERED(); 2991 2992 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 2993 vs = VGET_USECOUNT; 2994 } else { 2995 if (vhold_smr(vp)) 2996 vs = VGET_HOLDCNT; 2997 else 2998 vs = VGET_NONE; 2999 } 3000 return (vs); 3001 } 3002 3003 enum vgetstate 3004 vget_prep(struct vnode *vp) 3005 { 3006 enum vgetstate vs; 3007 3008 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 3009 vs = VGET_USECOUNT; 3010 } else { 3011 vhold(vp); 3012 vs = VGET_HOLDCNT; 3013 } 3014 return (vs); 3015 } 3016 3017 void 3018 vget_abort(struct vnode *vp, enum vgetstate vs) 3019 { 3020 3021 switch (vs) { 3022 case VGET_USECOUNT: 3023 vrele(vp); 3024 break; 3025 case VGET_HOLDCNT: 3026 vdrop(vp); 3027 break; 3028 default: 3029 __assert_unreachable(); 3030 } 3031 } 3032 3033 int 3034 vget(struct vnode *vp, int flags) 3035 { 3036 enum vgetstate vs; 3037 3038 vs = vget_prep(vp); 3039 return (vget_finish(vp, flags, vs)); 3040 } 3041 3042 int 3043 vget_finish(struct vnode *vp, int flags, enum vgetstate vs) 3044 { 3045 int error; 3046 3047 if ((flags & LK_INTERLOCK) != 0) 3048 ASSERT_VI_LOCKED(vp, __func__); 3049 else 3050 ASSERT_VI_UNLOCKED(vp, __func__); 3051 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3052 VNPASS(vp->v_holdcnt > 0, vp); 3053 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3054 3055 error = vn_lock(vp, flags); 3056 if (__predict_false(error != 0)) { 3057 vget_abort(vp, vs); 3058 CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__, 3059 vp); 3060 return (error); 3061 } 3062 3063 vget_finish_ref(vp, vs); 3064 return (0); 3065 } 3066 3067 void 3068 vget_finish_ref(struct vnode *vp, enum vgetstate vs) 3069 { 3070 int old; 3071 3072 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3073 VNPASS(vp->v_holdcnt > 0, vp); 3074 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3075 3076 if (vs == VGET_USECOUNT) 3077 return; 3078 3079 /* 3080 * We hold the vnode. If the usecount is 0 it will be utilized to keep 3081 * the vnode around. Otherwise someone else lended their hold count and 3082 * we have to drop ours. 3083 */ 3084 old = atomic_fetchadd_int(&vp->v_usecount, 1); 3085 VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old)); 3086 if (old != 0) { 3087 #ifdef INVARIANTS 3088 old = atomic_fetchadd_int(&vp->v_holdcnt, -1); 3089 VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); 3090 #else 3091 refcount_release(&vp->v_holdcnt); 3092 #endif 3093 } 3094 } 3095 3096 void 3097 vref(struct vnode *vp) 3098 { 3099 enum vgetstate vs; 3100 3101 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3102 vs = vget_prep(vp); 3103 vget_finish_ref(vp, vs); 3104 } 3105 3106 void 3107 vrefact(struct vnode *vp) 3108 { 3109 3110 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3111 #ifdef INVARIANTS 3112 int old = atomic_fetchadd_int(&vp->v_usecount, 1); 3113 VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old)); 3114 #else 3115 refcount_acquire(&vp->v_usecount); 3116 #endif 3117 } 3118 3119 void 3120 vlazy(struct vnode *vp) 3121 { 3122 struct mount *mp; 3123 3124 VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); 3125 3126 if ((vp->v_mflag & VMP_LAZYLIST) != 0) 3127 return; 3128 /* 3129 * We may get here for inactive routines after the vnode got doomed. 3130 */ 3131 if (VN_IS_DOOMED(vp)) 3132 return; 3133 mp = vp->v_mount; 3134 mtx_lock(&mp->mnt_listmtx); 3135 if ((vp->v_mflag & VMP_LAZYLIST) == 0) { 3136 vp->v_mflag |= VMP_LAZYLIST; 3137 TAILQ_INSERT_TAIL(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3138 mp->mnt_lazyvnodelistsize++; 3139 } 3140 mtx_unlock(&mp->mnt_listmtx); 3141 } 3142 3143 static void 3144 vunlazy(struct vnode *vp) 3145 { 3146 struct mount *mp; 3147 3148 ASSERT_VI_LOCKED(vp, __func__); 3149 VNPASS(!VN_IS_DOOMED(vp), vp); 3150 3151 mp = vp->v_mount; 3152 mtx_lock(&mp->mnt_listmtx); 3153 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3154 /* 3155 * Don't remove the vnode from the lazy list if another thread 3156 * has increased the hold count. It may have re-enqueued the 3157 * vnode to the lazy list and is now responsible for its 3158 * removal. 3159 */ 3160 if (vp->v_holdcnt == 0) { 3161 vp->v_mflag &= ~VMP_LAZYLIST; 3162 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3163 mp->mnt_lazyvnodelistsize--; 3164 } 3165 mtx_unlock(&mp->mnt_listmtx); 3166 } 3167 3168 /* 3169 * This routine is only meant to be called from vgonel prior to dooming 3170 * the vnode. 3171 */ 3172 static void 3173 vunlazy_gone(struct vnode *vp) 3174 { 3175 struct mount *mp; 3176 3177 ASSERT_VOP_ELOCKED(vp, __func__); 3178 ASSERT_VI_LOCKED(vp, __func__); 3179 VNPASS(!VN_IS_DOOMED(vp), vp); 3180 3181 if (vp->v_mflag & VMP_LAZYLIST) { 3182 mp = vp->v_mount; 3183 mtx_lock(&mp->mnt_listmtx); 3184 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3185 vp->v_mflag &= ~VMP_LAZYLIST; 3186 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3187 mp->mnt_lazyvnodelistsize--; 3188 mtx_unlock(&mp->mnt_listmtx); 3189 } 3190 } 3191 3192 static void 3193 vdefer_inactive(struct vnode *vp) 3194 { 3195 3196 ASSERT_VI_LOCKED(vp, __func__); 3197 VNASSERT(vp->v_holdcnt > 0, vp, 3198 ("%s: vnode without hold count", __func__)); 3199 if (VN_IS_DOOMED(vp)) { 3200 vdropl(vp); 3201 return; 3202 } 3203 if (vp->v_iflag & VI_DEFINACT) { 3204 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 3205 vdropl(vp); 3206 return; 3207 } 3208 if (vp->v_usecount > 0) { 3209 vp->v_iflag &= ~VI_OWEINACT; 3210 vdropl(vp); 3211 return; 3212 } 3213 vlazy(vp); 3214 vp->v_iflag |= VI_DEFINACT; 3215 VI_UNLOCK(vp); 3216 counter_u64_add(deferred_inact, 1); 3217 } 3218 3219 static void 3220 vdefer_inactive_unlocked(struct vnode *vp) 3221 { 3222 3223 VI_LOCK(vp); 3224 if ((vp->v_iflag & VI_OWEINACT) == 0) { 3225 vdropl(vp); 3226 return; 3227 } 3228 vdefer_inactive(vp); 3229 } 3230 3231 enum vput_op { VRELE, VPUT, VUNREF }; 3232 3233 /* 3234 * Handle ->v_usecount transitioning to 0. 3235 * 3236 * By releasing the last usecount we take ownership of the hold count which 3237 * provides liveness of the vnode, meaning we have to vdrop. 3238 * 3239 * For all vnodes we may need to perform inactive processing. It requires an 3240 * exclusive lock on the vnode, while it is legal to call here with only a 3241 * shared lock (or no locks). If locking the vnode in an expected manner fails, 3242 * inactive processing gets deferred to the syncer. 3243 * 3244 * XXX Some filesystems pass in an exclusively locked vnode and strongly depend 3245 * on the lock being held all the way until VOP_INACTIVE. This in particular 3246 * happens with UFS which adds half-constructed vnodes to the hash, where they 3247 * can be found by other code. 3248 */ 3249 static void 3250 vput_final(struct vnode *vp, enum vput_op func) 3251 { 3252 int error; 3253 bool want_unlock; 3254 3255 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3256 VNPASS(vp->v_holdcnt > 0, vp); 3257 3258 VI_LOCK(vp); 3259 3260 /* 3261 * By the time we got here someone else might have transitioned 3262 * the count back to > 0. 3263 */ 3264 if (vp->v_usecount > 0) 3265 goto out; 3266 3267 /* 3268 * If the vnode is doomed vgone already performed inactive processing 3269 * (if needed). 3270 */ 3271 if (VN_IS_DOOMED(vp)) 3272 goto out; 3273 3274 if (__predict_true(VOP_NEED_INACTIVE(vp) == 0)) 3275 goto out; 3276 3277 if (vp->v_iflag & VI_DOINGINACT) 3278 goto out; 3279 3280 /* 3281 * Locking operations here will drop the interlock and possibly the 3282 * vnode lock, opening a window where the vnode can get doomed all the 3283 * while ->v_usecount is 0. Set VI_OWEINACT to let vgone know to 3284 * perform inactive. 3285 */ 3286 vp->v_iflag |= VI_OWEINACT; 3287 want_unlock = false; 3288 error = 0; 3289 switch (func) { 3290 case VRELE: 3291 switch (VOP_ISLOCKED(vp)) { 3292 case LK_EXCLUSIVE: 3293 break; 3294 case LK_EXCLOTHER: 3295 case 0: 3296 want_unlock = true; 3297 error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); 3298 VI_LOCK(vp); 3299 break; 3300 default: 3301 /* 3302 * The lock has at least one sharer, but we have no way 3303 * to conclude whether this is us. Play it safe and 3304 * defer processing. 3305 */ 3306 error = EAGAIN; 3307 break; 3308 } 3309 break; 3310 case VPUT: 3311 want_unlock = true; 3312 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3313 error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK | 3314 LK_NOWAIT); 3315 VI_LOCK(vp); 3316 } 3317 break; 3318 case VUNREF: 3319 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3320 error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK); 3321 VI_LOCK(vp); 3322 } 3323 break; 3324 } 3325 if (error == 0) { 3326 if (func == VUNREF) { 3327 VNASSERT((vp->v_vflag & VV_UNREF) == 0, vp, 3328 ("recursive vunref")); 3329 vp->v_vflag |= VV_UNREF; 3330 } 3331 for (;;) { 3332 error = vinactive(vp); 3333 if (want_unlock) 3334 VOP_UNLOCK(vp); 3335 if (error != ERELOOKUP || !want_unlock) 3336 break; 3337 VOP_LOCK(vp, LK_EXCLUSIVE); 3338 } 3339 if (func == VUNREF) 3340 vp->v_vflag &= ~VV_UNREF; 3341 vdropl(vp); 3342 } else { 3343 vdefer_inactive(vp); 3344 } 3345 return; 3346 out: 3347 if (func == VPUT) 3348 VOP_UNLOCK(vp); 3349 vdropl(vp); 3350 } 3351 3352 /* 3353 * Decrement ->v_usecount for a vnode. 3354 * 3355 * Releasing the last use count requires additional processing, see vput_final 3356 * above for details. 3357 * 3358 * Comment above each variant denotes lock state on entry and exit. 3359 */ 3360 3361 /* 3362 * in: any 3363 * out: same as passed in 3364 */ 3365 void 3366 vrele(struct vnode *vp) 3367 { 3368 3369 ASSERT_VI_UNLOCKED(vp, __func__); 3370 if (!refcount_release(&vp->v_usecount)) 3371 return; 3372 vput_final(vp, VRELE); 3373 } 3374 3375 /* 3376 * in: locked 3377 * out: unlocked 3378 */ 3379 void 3380 vput(struct vnode *vp) 3381 { 3382 3383 ASSERT_VOP_LOCKED(vp, __func__); 3384 ASSERT_VI_UNLOCKED(vp, __func__); 3385 if (!refcount_release(&vp->v_usecount)) { 3386 VOP_UNLOCK(vp); 3387 return; 3388 } 3389 vput_final(vp, VPUT); 3390 } 3391 3392 /* 3393 * in: locked 3394 * out: locked 3395 */ 3396 void 3397 vunref(struct vnode *vp) 3398 { 3399 3400 ASSERT_VOP_LOCKED(vp, __func__); 3401 ASSERT_VI_UNLOCKED(vp, __func__); 3402 if (!refcount_release(&vp->v_usecount)) 3403 return; 3404 vput_final(vp, VUNREF); 3405 } 3406 3407 void 3408 vhold(struct vnode *vp) 3409 { 3410 int old; 3411 3412 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3413 old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3414 VNASSERT(old >= 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3415 ("%s: wrong hold count %d", __func__, old)); 3416 if (old == 0) 3417 vfs_freevnodes_dec(); 3418 } 3419 3420 void 3421 vholdnz(struct vnode *vp) 3422 { 3423 3424 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3425 #ifdef INVARIANTS 3426 int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3427 VNASSERT(old > 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3428 ("%s: wrong hold count %d", __func__, old)); 3429 #else 3430 atomic_add_int(&vp->v_holdcnt, 1); 3431 #endif 3432 } 3433 3434 /* 3435 * Grab a hold count unless the vnode is freed. 3436 * 3437 * Only use this routine if vfs smr is the only protection you have against 3438 * freeing the vnode. 3439 * 3440 * The code loops trying to add a hold count as long as the VHOLD_NO_SMR flag 3441 * is not set. After the flag is set the vnode becomes immutable to anyone but 3442 * the thread which managed to set the flag. 3443 * 3444 * It may be tempting to replace the loop with: 3445 * count = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3446 * if (count & VHOLD_NO_SMR) { 3447 * backpedal and error out; 3448 * } 3449 * 3450 * However, while this is more performant, it hinders debugging by eliminating 3451 * the previously mentioned invariant. 3452 */ 3453 bool 3454 vhold_smr(struct vnode *vp) 3455 { 3456 int count; 3457 3458 VFS_SMR_ASSERT_ENTERED(); 3459 3460 count = atomic_load_int(&vp->v_holdcnt); 3461 for (;;) { 3462 if (count & VHOLD_NO_SMR) { 3463 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3464 ("non-zero hold count with flags %d\n", count)); 3465 return (false); 3466 } 3467 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3468 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3469 if (count == 0) 3470 vfs_freevnodes_dec(); 3471 return (true); 3472 } 3473 } 3474 } 3475 3476 /* 3477 * Hold a free vnode for recycling. 3478 * 3479 * Note: vnode_init references this comment. 3480 * 3481 * Attempts to recycle only need the global vnode list lock and have no use for 3482 * SMR. 3483 * 3484 * However, vnodes get inserted into the global list before they get fully 3485 * initialized and stay there until UMA decides to free the memory. This in 3486 * particular means the target can be found before it becomes usable and after 3487 * it becomes recycled. Picking up such vnodes is guarded with v_holdcnt set to 3488 * VHOLD_NO_SMR. 3489 * 3490 * Note: the vnode may gain more references after we transition the count 0->1. 3491 */ 3492 static bool 3493 vhold_recycle_free(struct vnode *vp) 3494 { 3495 int count; 3496 3497 mtx_assert(&vnode_list_mtx, MA_OWNED); 3498 3499 count = atomic_load_int(&vp->v_holdcnt); 3500 for (;;) { 3501 if (count & VHOLD_NO_SMR) { 3502 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3503 ("non-zero hold count with flags %d\n", count)); 3504 return (false); 3505 } 3506 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3507 if (count > 0) { 3508 return (false); 3509 } 3510 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3511 vfs_freevnodes_dec(); 3512 return (true); 3513 } 3514 } 3515 } 3516 3517 static void __noinline 3518 vdbatch_process(struct vdbatch *vd) 3519 { 3520 struct vnode *vp; 3521 int i; 3522 3523 mtx_assert(&vd->lock, MA_OWNED); 3524 MPASS(curthread->td_pinned > 0); 3525 MPASS(vd->index == VDBATCH_SIZE); 3526 3527 mtx_lock(&vnode_list_mtx); 3528 critical_enter(); 3529 for (i = 0; i < VDBATCH_SIZE; i++) { 3530 vp = vd->tab[i]; 3531 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 3532 TAILQ_INSERT_TAIL(&vnode_list, vp, v_vnodelist); 3533 MPASS(vp->v_dbatchcpu != NOCPU); 3534 vp->v_dbatchcpu = NOCPU; 3535 } 3536 mtx_unlock(&vnode_list_mtx); 3537 bzero(vd->tab, sizeof(vd->tab)); 3538 vd->index = 0; 3539 critical_exit(); 3540 } 3541 3542 static void 3543 vdbatch_enqueue(struct vnode *vp) 3544 { 3545 struct vdbatch *vd; 3546 3547 ASSERT_VI_LOCKED(vp, __func__); 3548 VNASSERT(!VN_IS_DOOMED(vp), vp, 3549 ("%s: deferring requeue of a doomed vnode", __func__)); 3550 3551 if (vp->v_dbatchcpu != NOCPU) { 3552 VI_UNLOCK(vp); 3553 return; 3554 } 3555 3556 sched_pin(); 3557 vd = DPCPU_PTR(vd); 3558 mtx_lock(&vd->lock); 3559 MPASS(vd->index < VDBATCH_SIZE); 3560 MPASS(vd->tab[vd->index] == NULL); 3561 /* 3562 * A hack: we depend on being pinned so that we know what to put in 3563 * ->v_dbatchcpu. 3564 */ 3565 vp->v_dbatchcpu = curcpu; 3566 vd->tab[vd->index] = vp; 3567 vd->index++; 3568 VI_UNLOCK(vp); 3569 if (vd->index == VDBATCH_SIZE) 3570 vdbatch_process(vd); 3571 mtx_unlock(&vd->lock); 3572 sched_unpin(); 3573 } 3574 3575 /* 3576 * This routine must only be called for vnodes which are about to be 3577 * deallocated. Supporting dequeue for arbitrary vndoes would require 3578 * validating that the locked batch matches. 3579 */ 3580 static void 3581 vdbatch_dequeue(struct vnode *vp) 3582 { 3583 struct vdbatch *vd; 3584 int i; 3585 short cpu; 3586 3587 VNASSERT(vp->v_type == VBAD || vp->v_type == VNON, vp, 3588 ("%s: called for a used vnode\n", __func__)); 3589 3590 cpu = vp->v_dbatchcpu; 3591 if (cpu == NOCPU) 3592 return; 3593 3594 vd = DPCPU_ID_PTR(cpu, vd); 3595 mtx_lock(&vd->lock); 3596 for (i = 0; i < vd->index; i++) { 3597 if (vd->tab[i] != vp) 3598 continue; 3599 vp->v_dbatchcpu = NOCPU; 3600 vd->index--; 3601 vd->tab[i] = vd->tab[vd->index]; 3602 vd->tab[vd->index] = NULL; 3603 break; 3604 } 3605 mtx_unlock(&vd->lock); 3606 /* 3607 * Either we dequeued the vnode above or the target CPU beat us to it. 3608 */ 3609 MPASS(vp->v_dbatchcpu == NOCPU); 3610 } 3611 3612 /* 3613 * Drop the hold count of the vnode. If this is the last reference to 3614 * the vnode we place it on the free list unless it has been vgone'd 3615 * (marked VIRF_DOOMED) in which case we will free it. 3616 * 3617 * Because the vnode vm object keeps a hold reference on the vnode if 3618 * there is at least one resident non-cached page, the vnode cannot 3619 * leave the active list without the page cleanup done. 3620 */ 3621 static void __noinline 3622 vdropl_final(struct vnode *vp) 3623 { 3624 3625 ASSERT_VI_LOCKED(vp, __func__); 3626 VNPASS(VN_IS_DOOMED(vp), vp); 3627 /* 3628 * Set the VHOLD_NO_SMR flag. 3629 * 3630 * We may be racing against vhold_smr. If they win we can just pretend 3631 * we never got this far, they will vdrop later. 3632 */ 3633 if (__predict_false(!atomic_cmpset_int(&vp->v_holdcnt, 0, VHOLD_NO_SMR))) { 3634 vfs_freevnodes_inc(); 3635 VI_UNLOCK(vp); 3636 /* 3637 * We lost the aforementioned race. Any subsequent access is 3638 * invalid as they might have managed to vdropl on their own. 3639 */ 3640 return; 3641 } 3642 /* 3643 * Don't bump freevnodes as this one is going away. 3644 */ 3645 freevnode(vp); 3646 } 3647 3648 void 3649 vdrop(struct vnode *vp) 3650 { 3651 3652 ASSERT_VI_UNLOCKED(vp, __func__); 3653 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3654 if (refcount_release_if_not_last(&vp->v_holdcnt)) 3655 return; 3656 VI_LOCK(vp); 3657 vdropl(vp); 3658 } 3659 3660 static void __always_inline 3661 vdropl_impl(struct vnode *vp, bool enqueue) 3662 { 3663 3664 ASSERT_VI_LOCKED(vp, __func__); 3665 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3666 if (!refcount_release(&vp->v_holdcnt)) { 3667 VI_UNLOCK(vp); 3668 return; 3669 } 3670 VNPASS((vp->v_iflag & VI_OWEINACT) == 0, vp); 3671 VNPASS((vp->v_iflag & VI_DEFINACT) == 0, vp); 3672 if (VN_IS_DOOMED(vp)) { 3673 vdropl_final(vp); 3674 return; 3675 } 3676 3677 vfs_freevnodes_inc(); 3678 if (vp->v_mflag & VMP_LAZYLIST) { 3679 vunlazy(vp); 3680 } 3681 3682 if (!enqueue) { 3683 VI_UNLOCK(vp); 3684 return; 3685 } 3686 3687 /* 3688 * Also unlocks the interlock. We can't assert on it as we 3689 * released our hold and by now the vnode might have been 3690 * freed. 3691 */ 3692 vdbatch_enqueue(vp); 3693 } 3694 3695 void 3696 vdropl(struct vnode *vp) 3697 { 3698 3699 vdropl_impl(vp, true); 3700 } 3701 3702 /* 3703 * vdrop a vnode when recycling 3704 * 3705 * This is a special case routine only to be used when recycling, differs from 3706 * regular vdrop by not requeieing the vnode on LRU. 3707 * 3708 * Consider a case where vtryrecycle continuously fails with all vnodes (due to 3709 * e.g., frozen writes on the filesystem), filling the batch and causing it to 3710 * be requeued. Then vnlru will end up revisiting the same vnodes. This is a 3711 * loop which can last for as long as writes are frozen. 3712 */ 3713 static void 3714 vdropl_recycle(struct vnode *vp) 3715 { 3716 3717 vdropl_impl(vp, false); 3718 } 3719 3720 static void 3721 vdrop_recycle(struct vnode *vp) 3722 { 3723 3724 VI_LOCK(vp); 3725 vdropl_recycle(vp); 3726 } 3727 3728 /* 3729 * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT 3730 * flags. DOINGINACT prevents us from recursing in calls to vinactive. 3731 */ 3732 static int 3733 vinactivef(struct vnode *vp) 3734 { 3735 struct vm_object *obj; 3736 int error; 3737 3738 ASSERT_VOP_ELOCKED(vp, "vinactive"); 3739 ASSERT_VI_LOCKED(vp, "vinactive"); 3740 VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp, 3741 ("vinactive: recursed on VI_DOINGINACT")); 3742 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3743 vp->v_iflag |= VI_DOINGINACT; 3744 vp->v_iflag &= ~VI_OWEINACT; 3745 VI_UNLOCK(vp); 3746 /* 3747 * Before moving off the active list, we must be sure that any 3748 * modified pages are converted into the vnode's dirty 3749 * buffers, since these will no longer be checked once the 3750 * vnode is on the inactive list. 3751 * 3752 * The write-out of the dirty pages is asynchronous. At the 3753 * point that VOP_INACTIVE() is called, there could still be 3754 * pending I/O and dirty pages in the object. 3755 */ 3756 if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 3757 vm_object_mightbedirty(obj)) { 3758 VM_OBJECT_WLOCK(obj); 3759 vm_object_page_clean(obj, 0, 0, 0); 3760 VM_OBJECT_WUNLOCK(obj); 3761 } 3762 error = VOP_INACTIVE(vp); 3763 VI_LOCK(vp); 3764 VNASSERT(vp->v_iflag & VI_DOINGINACT, vp, 3765 ("vinactive: lost VI_DOINGINACT")); 3766 vp->v_iflag &= ~VI_DOINGINACT; 3767 return (error); 3768 } 3769 3770 int 3771 vinactive(struct vnode *vp) 3772 { 3773 3774 ASSERT_VOP_ELOCKED(vp, "vinactive"); 3775 ASSERT_VI_LOCKED(vp, "vinactive"); 3776 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3777 3778 if ((vp->v_iflag & VI_OWEINACT) == 0) 3779 return (0); 3780 if (vp->v_iflag & VI_DOINGINACT) 3781 return (0); 3782 if (vp->v_usecount > 0) { 3783 vp->v_iflag &= ~VI_OWEINACT; 3784 return (0); 3785 } 3786 return (vinactivef(vp)); 3787 } 3788 3789 /* 3790 * Remove any vnodes in the vnode table belonging to mount point mp. 3791 * 3792 * If FORCECLOSE is not specified, there should not be any active ones, 3793 * return error if any are found (nb: this is a user error, not a 3794 * system error). If FORCECLOSE is specified, detach any active vnodes 3795 * that are found. 3796 * 3797 * If WRITECLOSE is set, only flush out regular file vnodes open for 3798 * writing. 3799 * 3800 * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped. 3801 * 3802 * `rootrefs' specifies the base reference count for the root vnode 3803 * of this filesystem. The root vnode is considered busy if its 3804 * v_usecount exceeds this value. On a successful return, vflush(, td) 3805 * will call vrele() on the root vnode exactly rootrefs times. 3806 * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must 3807 * be zero. 3808 */ 3809 #ifdef DIAGNOSTIC 3810 static int busyprt = 0; /* print out busy vnodes */ 3811 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes"); 3812 #endif 3813 3814 int 3815 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td) 3816 { 3817 struct vnode *vp, *mvp, *rootvp = NULL; 3818 struct vattr vattr; 3819 int busy = 0, error; 3820 3821 CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp, 3822 rootrefs, flags); 3823 if (rootrefs > 0) { 3824 KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0, 3825 ("vflush: bad args")); 3826 /* 3827 * Get the filesystem root vnode. We can vput() it 3828 * immediately, since with rootrefs > 0, it won't go away. 3829 */ 3830 if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) { 3831 CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d", 3832 __func__, error); 3833 return (error); 3834 } 3835 vput(rootvp); 3836 } 3837 loop: 3838 MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { 3839 vholdl(vp); 3840 error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE); 3841 if (error) { 3842 vdrop(vp); 3843 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 3844 goto loop; 3845 } 3846 /* 3847 * Skip over a vnodes marked VV_SYSTEM. 3848 */ 3849 if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) { 3850 VOP_UNLOCK(vp); 3851 vdrop(vp); 3852 continue; 3853 } 3854 /* 3855 * If WRITECLOSE is set, flush out unlinked but still open 3856 * files (even if open only for reading) and regular file 3857 * vnodes open for writing. 3858 */ 3859 if (flags & WRITECLOSE) { 3860 if (vp->v_object != NULL) { 3861 VM_OBJECT_WLOCK(vp->v_object); 3862 vm_object_page_clean(vp->v_object, 0, 0, 0); 3863 VM_OBJECT_WUNLOCK(vp->v_object); 3864 } 3865 do { 3866 error = VOP_FSYNC(vp, MNT_WAIT, td); 3867 } while (error == ERELOOKUP); 3868 if (error != 0) { 3869 VOP_UNLOCK(vp); 3870 vdrop(vp); 3871 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 3872 return (error); 3873 } 3874 error = VOP_GETATTR(vp, &vattr, td->td_ucred); 3875 VI_LOCK(vp); 3876 3877 if ((vp->v_type == VNON || 3878 (error == 0 && vattr.va_nlink > 0)) && 3879 (vp->v_writecount <= 0 || vp->v_type != VREG)) { 3880 VOP_UNLOCK(vp); 3881 vdropl(vp); 3882 continue; 3883 } 3884 } else 3885 VI_LOCK(vp); 3886 /* 3887 * With v_usecount == 0, all we need to do is clear out the 3888 * vnode data structures and we are done. 3889 * 3890 * If FORCECLOSE is set, forcibly close the vnode. 3891 */ 3892 if (vp->v_usecount == 0 || (flags & FORCECLOSE)) { 3893 vgonel(vp); 3894 } else { 3895 busy++; 3896 #ifdef DIAGNOSTIC 3897 if (busyprt) 3898 vn_printf(vp, "vflush: busy vnode "); 3899 #endif 3900 } 3901 VOP_UNLOCK(vp); 3902 vdropl(vp); 3903 } 3904 if (rootrefs > 0 && (flags & FORCECLOSE) == 0) { 3905 /* 3906 * If just the root vnode is busy, and if its refcount 3907 * is equal to `rootrefs', then go ahead and kill it. 3908 */ 3909 VI_LOCK(rootvp); 3910 KASSERT(busy > 0, ("vflush: not busy")); 3911 VNASSERT(rootvp->v_usecount >= rootrefs, rootvp, 3912 ("vflush: usecount %d < rootrefs %d", 3913 rootvp->v_usecount, rootrefs)); 3914 if (busy == 1 && rootvp->v_usecount == rootrefs) { 3915 VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK); 3916 vgone(rootvp); 3917 VOP_UNLOCK(rootvp); 3918 busy = 0; 3919 } else 3920 VI_UNLOCK(rootvp); 3921 } 3922 if (busy) { 3923 CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__, 3924 busy); 3925 return (EBUSY); 3926 } 3927 for (; rootrefs > 0; rootrefs--) 3928 vrele(rootvp); 3929 return (0); 3930 } 3931 3932 /* 3933 * Recycle an unused vnode to the front of the free list. 3934 */ 3935 int 3936 vrecycle(struct vnode *vp) 3937 { 3938 int recycled; 3939 3940 VI_LOCK(vp); 3941 recycled = vrecyclel(vp); 3942 VI_UNLOCK(vp); 3943 return (recycled); 3944 } 3945 3946 /* 3947 * vrecycle, with the vp interlock held. 3948 */ 3949 int 3950 vrecyclel(struct vnode *vp) 3951 { 3952 int recycled; 3953 3954 ASSERT_VOP_ELOCKED(vp, __func__); 3955 ASSERT_VI_LOCKED(vp, __func__); 3956 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3957 recycled = 0; 3958 if (vp->v_usecount == 0) { 3959 recycled = 1; 3960 vgonel(vp); 3961 } 3962 return (recycled); 3963 } 3964 3965 /* 3966 * Eliminate all activity associated with a vnode 3967 * in preparation for reuse. 3968 */ 3969 void 3970 vgone(struct vnode *vp) 3971 { 3972 VI_LOCK(vp); 3973 vgonel(vp); 3974 VI_UNLOCK(vp); 3975 } 3976 3977 /* 3978 * Notify upper mounts about reclaimed or unlinked vnode. 3979 */ 3980 void 3981 vfs_notify_upper(struct vnode *vp, enum vfs_notify_upper_type event) 3982 { 3983 struct mount *mp; 3984 struct mount_upper_node *ump; 3985 3986 mp = atomic_load_ptr(&vp->v_mount); 3987 if (mp == NULL) 3988 return; 3989 if (TAILQ_EMPTY(&mp->mnt_notify)) 3990 return; 3991 3992 MNT_ILOCK(mp); 3993 mp->mnt_upper_pending++; 3994 KASSERT(mp->mnt_upper_pending > 0, 3995 ("%s: mnt_upper_pending %d", __func__, mp->mnt_upper_pending)); 3996 TAILQ_FOREACH(ump, &mp->mnt_notify, mnt_upper_link) { 3997 MNT_IUNLOCK(mp); 3998 switch (event) { 3999 case VFS_NOTIFY_UPPER_RECLAIM: 4000 VFS_RECLAIM_LOWERVP(ump->mp, vp); 4001 break; 4002 case VFS_NOTIFY_UPPER_UNLINK: 4003 VFS_UNLINK_LOWERVP(ump->mp, vp); 4004 break; 4005 } 4006 MNT_ILOCK(mp); 4007 } 4008 mp->mnt_upper_pending--; 4009 if ((mp->mnt_kern_flag & MNTK_UPPER_WAITER) != 0 && 4010 mp->mnt_upper_pending == 0) { 4011 mp->mnt_kern_flag &= ~MNTK_UPPER_WAITER; 4012 wakeup(&mp->mnt_uppers); 4013 } 4014 MNT_IUNLOCK(mp); 4015 } 4016 4017 /* 4018 * vgone, with the vp interlock held. 4019 */ 4020 static void 4021 vgonel(struct vnode *vp) 4022 { 4023 struct thread *td; 4024 struct mount *mp; 4025 vm_object_t object; 4026 bool active, doinginact, oweinact; 4027 4028 ASSERT_VOP_ELOCKED(vp, "vgonel"); 4029 ASSERT_VI_LOCKED(vp, "vgonel"); 4030 VNASSERT(vp->v_holdcnt, vp, 4031 ("vgonel: vp %p has no reference.", vp)); 4032 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4033 td = curthread; 4034 4035 /* 4036 * Don't vgonel if we're already doomed. 4037 */ 4038 if (VN_IS_DOOMED(vp)) { 4039 VNPASS(vn_get_state(vp) == VSTATE_DESTROYING || \ 4040 vn_get_state(vp) == VSTATE_DEAD, vp); 4041 return; 4042 } 4043 /* 4044 * Paired with freevnode. 4045 */ 4046 vn_seqc_write_begin_locked(vp); 4047 vunlazy_gone(vp); 4048 vn_irflag_set_locked(vp, VIRF_DOOMED); 4049 vn_set_state(vp, VSTATE_DESTROYING); 4050 4051 /* 4052 * Check to see if the vnode is in use. If so, we have to 4053 * call VOP_CLOSE() and VOP_INACTIVE(). 4054 * 4055 * It could be that VOP_INACTIVE() requested reclamation, in 4056 * which case we should avoid recursion, so check 4057 * VI_DOINGINACT. This is not precise but good enough. 4058 */ 4059 active = vp->v_usecount > 0; 4060 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4061 doinginact = (vp->v_iflag & VI_DOINGINACT) != 0; 4062 4063 /* 4064 * If we need to do inactive VI_OWEINACT will be set. 4065 */ 4066 if (vp->v_iflag & VI_DEFINACT) { 4067 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 4068 vp->v_iflag &= ~VI_DEFINACT; 4069 vdropl(vp); 4070 } else { 4071 VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count")); 4072 VI_UNLOCK(vp); 4073 } 4074 cache_purge_vgone(vp); 4075 vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM); 4076 4077 /* 4078 * If purging an active vnode, it must be closed and 4079 * deactivated before being reclaimed. 4080 */ 4081 if (active) 4082 VOP_CLOSE(vp, FNONBLOCK, NOCRED, td); 4083 if (!doinginact) { 4084 do { 4085 if (oweinact || active) { 4086 VI_LOCK(vp); 4087 vinactivef(vp); 4088 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4089 VI_UNLOCK(vp); 4090 } 4091 } while (oweinact); 4092 } 4093 if (vp->v_type == VSOCK) 4094 vfs_unp_reclaim(vp); 4095 4096 /* 4097 * Clean out any buffers associated with the vnode. 4098 * If the flush fails, just toss the buffers. 4099 */ 4100 mp = NULL; 4101 if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd)) 4102 (void) vn_start_secondary_write(vp, &mp, V_WAIT); 4103 if (vinvalbuf(vp, V_SAVE, 0, 0) != 0) { 4104 while (vinvalbuf(vp, 0, 0, 0) != 0) 4105 ; 4106 } 4107 4108 BO_LOCK(&vp->v_bufobj); 4109 KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) && 4110 vp->v_bufobj.bo_dirty.bv_cnt == 0 && 4111 TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) && 4112 vp->v_bufobj.bo_clean.bv_cnt == 0, 4113 ("vp %p bufobj not invalidated", vp)); 4114 4115 /* 4116 * For VMIO bufobj, BO_DEAD is set later, or in 4117 * vm_object_terminate() after the object's page queue is 4118 * flushed. 4119 */ 4120 object = vp->v_bufobj.bo_object; 4121 if (object == NULL) 4122 vp->v_bufobj.bo_flag |= BO_DEAD; 4123 BO_UNLOCK(&vp->v_bufobj); 4124 4125 /* 4126 * Handle the VM part. Tmpfs handles v_object on its own (the 4127 * OBJT_VNODE check). Nullfs or other bypassing filesystems 4128 * should not touch the object borrowed from the lower vnode 4129 * (the handle check). 4130 */ 4131 if (object != NULL && object->type == OBJT_VNODE && 4132 object->handle == vp) 4133 vnode_destroy_vobject(vp); 4134 4135 /* 4136 * Reclaim the vnode. 4137 */ 4138 if (VOP_RECLAIM(vp)) 4139 panic("vgone: cannot reclaim"); 4140 if (mp != NULL) 4141 vn_finished_secondary_write(mp); 4142 VNASSERT(vp->v_object == NULL, vp, 4143 ("vop_reclaim left v_object vp=%p", vp)); 4144 /* 4145 * Clear the advisory locks and wake up waiting threads. 4146 */ 4147 if (vp->v_lockf != NULL) { 4148 (void)VOP_ADVLOCKPURGE(vp); 4149 vp->v_lockf = NULL; 4150 } 4151 /* 4152 * Delete from old mount point vnode list. 4153 */ 4154 if (vp->v_mount == NULL) { 4155 VI_LOCK(vp); 4156 } else { 4157 delmntque(vp); 4158 ASSERT_VI_LOCKED(vp, "vgonel 2"); 4159 } 4160 /* 4161 * Done with purge, reset to the standard lock and invalidate 4162 * the vnode. 4163 */ 4164 vp->v_vnlock = &vp->v_lock; 4165 vp->v_op = &dead_vnodeops; 4166 vp->v_type = VBAD; 4167 vn_set_state(vp, VSTATE_DEAD); 4168 } 4169 4170 /* 4171 * Print out a description of a vnode. 4172 */ 4173 static const char *const vtypename[] = { 4174 [VNON] = "VNON", 4175 [VREG] = "VREG", 4176 [VDIR] = "VDIR", 4177 [VBLK] = "VBLK", 4178 [VCHR] = "VCHR", 4179 [VLNK] = "VLNK", 4180 [VSOCK] = "VSOCK", 4181 [VFIFO] = "VFIFO", 4182 [VBAD] = "VBAD", 4183 [VMARKER] = "VMARKER", 4184 }; 4185 _Static_assert(nitems(vtypename) == VLASTTYPE + 1, 4186 "vnode type name not added to vtypename"); 4187 4188 static const char *const vstatename[] = { 4189 [VSTATE_UNINITIALIZED] = "VSTATE_UNINITIALIZED", 4190 [VSTATE_CONSTRUCTED] = "VSTATE_CONSTRUCTED", 4191 [VSTATE_DESTROYING] = "VSTATE_DESTROYING", 4192 [VSTATE_DEAD] = "VSTATE_DEAD", 4193 }; 4194 _Static_assert(nitems(vstatename) == VLASTSTATE + 1, 4195 "vnode state name not added to vstatename"); 4196 4197 _Static_assert((VHOLD_ALL_FLAGS & ~VHOLD_NO_SMR) == 0, 4198 "new hold count flag not added to vn_printf"); 4199 4200 void 4201 vn_printf(struct vnode *vp, const char *fmt, ...) 4202 { 4203 va_list ap; 4204 char buf[256], buf2[16]; 4205 u_long flags; 4206 u_int holdcnt; 4207 short irflag; 4208 4209 va_start(ap, fmt); 4210 vprintf(fmt, ap); 4211 va_end(ap); 4212 printf("%p: ", (void *)vp); 4213 printf("type %s state %s\n", vtypename[vp->v_type], vstatename[vp->v_state]); 4214 holdcnt = atomic_load_int(&vp->v_holdcnt); 4215 printf(" usecount %d, writecount %d, refcount %d seqc users %d", 4216 vp->v_usecount, vp->v_writecount, holdcnt & ~VHOLD_ALL_FLAGS, 4217 vp->v_seqc_users); 4218 switch (vp->v_type) { 4219 case VDIR: 4220 printf(" mountedhere %p\n", vp->v_mountedhere); 4221 break; 4222 case VCHR: 4223 printf(" rdev %p\n", vp->v_rdev); 4224 break; 4225 case VSOCK: 4226 printf(" socket %p\n", vp->v_unpcb); 4227 break; 4228 case VFIFO: 4229 printf(" fifoinfo %p\n", vp->v_fifoinfo); 4230 break; 4231 default: 4232 printf("\n"); 4233 break; 4234 } 4235 buf[0] = '\0'; 4236 buf[1] = '\0'; 4237 if (holdcnt & VHOLD_NO_SMR) 4238 strlcat(buf, "|VHOLD_NO_SMR", sizeof(buf)); 4239 printf(" hold count flags (%s)\n", buf + 1); 4240 4241 buf[0] = '\0'; 4242 buf[1] = '\0'; 4243 irflag = vn_irflag_read(vp); 4244 if (irflag & VIRF_DOOMED) 4245 strlcat(buf, "|VIRF_DOOMED", sizeof(buf)); 4246 if (irflag & VIRF_PGREAD) 4247 strlcat(buf, "|VIRF_PGREAD", sizeof(buf)); 4248 if (irflag & VIRF_MOUNTPOINT) 4249 strlcat(buf, "|VIRF_MOUNTPOINT", sizeof(buf)); 4250 if (irflag & VIRF_TEXT_REF) 4251 strlcat(buf, "|VIRF_TEXT_REF", sizeof(buf)); 4252 flags = irflag & ~(VIRF_DOOMED | VIRF_PGREAD | VIRF_MOUNTPOINT | VIRF_TEXT_REF); 4253 if (flags != 0) { 4254 snprintf(buf2, sizeof(buf2), "|VIRF(0x%lx)", flags); 4255 strlcat(buf, buf2, sizeof(buf)); 4256 } 4257 if (vp->v_vflag & VV_ROOT) 4258 strlcat(buf, "|VV_ROOT", sizeof(buf)); 4259 if (vp->v_vflag & VV_ISTTY) 4260 strlcat(buf, "|VV_ISTTY", sizeof(buf)); 4261 if (vp->v_vflag & VV_NOSYNC) 4262 strlcat(buf, "|VV_NOSYNC", sizeof(buf)); 4263 if (vp->v_vflag & VV_ETERNALDEV) 4264 strlcat(buf, "|VV_ETERNALDEV", sizeof(buf)); 4265 if (vp->v_vflag & VV_CACHEDLABEL) 4266 strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf)); 4267 if (vp->v_vflag & VV_VMSIZEVNLOCK) 4268 strlcat(buf, "|VV_VMSIZEVNLOCK", sizeof(buf)); 4269 if (vp->v_vflag & VV_COPYONWRITE) 4270 strlcat(buf, "|VV_COPYONWRITE", sizeof(buf)); 4271 if (vp->v_vflag & VV_SYSTEM) 4272 strlcat(buf, "|VV_SYSTEM", sizeof(buf)); 4273 if (vp->v_vflag & VV_PROCDEP) 4274 strlcat(buf, "|VV_PROCDEP", sizeof(buf)); 4275 if (vp->v_vflag & VV_DELETED) 4276 strlcat(buf, "|VV_DELETED", sizeof(buf)); 4277 if (vp->v_vflag & VV_MD) 4278 strlcat(buf, "|VV_MD", sizeof(buf)); 4279 if (vp->v_vflag & VV_FORCEINSMQ) 4280 strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf)); 4281 if (vp->v_vflag & VV_READLINK) 4282 strlcat(buf, "|VV_READLINK", sizeof(buf)); 4283 flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV | 4284 VV_CACHEDLABEL | VV_VMSIZEVNLOCK | VV_COPYONWRITE | VV_SYSTEM | 4285 VV_PROCDEP | VV_DELETED | VV_MD | VV_FORCEINSMQ | VV_READLINK); 4286 if (flags != 0) { 4287 snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags); 4288 strlcat(buf, buf2, sizeof(buf)); 4289 } 4290 if (vp->v_iflag & VI_MOUNT) 4291 strlcat(buf, "|VI_MOUNT", sizeof(buf)); 4292 if (vp->v_iflag & VI_DOINGINACT) 4293 strlcat(buf, "|VI_DOINGINACT", sizeof(buf)); 4294 if (vp->v_iflag & VI_OWEINACT) 4295 strlcat(buf, "|VI_OWEINACT", sizeof(buf)); 4296 if (vp->v_iflag & VI_DEFINACT) 4297 strlcat(buf, "|VI_DEFINACT", sizeof(buf)); 4298 if (vp->v_iflag & VI_FOPENING) 4299 strlcat(buf, "|VI_FOPENING", sizeof(buf)); 4300 flags = vp->v_iflag & ~(VI_MOUNT | VI_DOINGINACT | 4301 VI_OWEINACT | VI_DEFINACT | VI_FOPENING); 4302 if (flags != 0) { 4303 snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags); 4304 strlcat(buf, buf2, sizeof(buf)); 4305 } 4306 if (vp->v_mflag & VMP_LAZYLIST) 4307 strlcat(buf, "|VMP_LAZYLIST", sizeof(buf)); 4308 flags = vp->v_mflag & ~(VMP_LAZYLIST); 4309 if (flags != 0) { 4310 snprintf(buf2, sizeof(buf2), "|VMP(0x%lx)", flags); 4311 strlcat(buf, buf2, sizeof(buf)); 4312 } 4313 printf(" flags (%s)", buf + 1); 4314 if (mtx_owned(VI_MTX(vp))) 4315 printf(" VI_LOCKed"); 4316 printf("\n"); 4317 if (vp->v_object != NULL) 4318 printf(" v_object %p ref %d pages %d " 4319 "cleanbuf %d dirtybuf %d\n", 4320 vp->v_object, vp->v_object->ref_count, 4321 vp->v_object->resident_page_count, 4322 vp->v_bufobj.bo_clean.bv_cnt, 4323 vp->v_bufobj.bo_dirty.bv_cnt); 4324 printf(" "); 4325 lockmgr_printinfo(vp->v_vnlock); 4326 if (vp->v_data != NULL) 4327 VOP_PRINT(vp); 4328 } 4329 4330 #ifdef DDB 4331 /* 4332 * List all of the locked vnodes in the system. 4333 * Called when debugging the kernel. 4334 */ 4335 DB_SHOW_COMMAND_FLAGS(lockedvnods, lockedvnodes, DB_CMD_MEMSAFE) 4336 { 4337 struct mount *mp; 4338 struct vnode *vp; 4339 4340 /* 4341 * Note: because this is DDB, we can't obey the locking semantics 4342 * for these structures, which means we could catch an inconsistent 4343 * state and dereference a nasty pointer. Not much to be done 4344 * about that. 4345 */ 4346 db_printf("Locked vnodes\n"); 4347 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4348 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4349 if (vp->v_type != VMARKER && VOP_ISLOCKED(vp)) 4350 vn_printf(vp, "vnode "); 4351 } 4352 } 4353 } 4354 4355 /* 4356 * Show details about the given vnode. 4357 */ 4358 DB_SHOW_COMMAND(vnode, db_show_vnode) 4359 { 4360 struct vnode *vp; 4361 4362 if (!have_addr) 4363 return; 4364 vp = (struct vnode *)addr; 4365 vn_printf(vp, "vnode "); 4366 } 4367 4368 /* 4369 * Show details about the given mount point. 4370 */ 4371 DB_SHOW_COMMAND(mount, db_show_mount) 4372 { 4373 struct mount *mp; 4374 struct vfsopt *opt; 4375 struct statfs *sp; 4376 struct vnode *vp; 4377 char buf[512]; 4378 uint64_t mflags; 4379 u_int flags; 4380 4381 if (!have_addr) { 4382 /* No address given, print short info about all mount points. */ 4383 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4384 db_printf("%p %s on %s (%s)\n", mp, 4385 mp->mnt_stat.f_mntfromname, 4386 mp->mnt_stat.f_mntonname, 4387 mp->mnt_stat.f_fstypename); 4388 if (db_pager_quit) 4389 break; 4390 } 4391 db_printf("\nMore info: show mount <addr>\n"); 4392 return; 4393 } 4394 4395 mp = (struct mount *)addr; 4396 db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname, 4397 mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename); 4398 4399 buf[0] = '\0'; 4400 mflags = mp->mnt_flag; 4401 #define MNT_FLAG(flag) do { \ 4402 if (mflags & (flag)) { \ 4403 if (buf[0] != '\0') \ 4404 strlcat(buf, ", ", sizeof(buf)); \ 4405 strlcat(buf, (#flag) + 4, sizeof(buf)); \ 4406 mflags &= ~(flag); \ 4407 } \ 4408 } while (0) 4409 MNT_FLAG(MNT_RDONLY); 4410 MNT_FLAG(MNT_SYNCHRONOUS); 4411 MNT_FLAG(MNT_NOEXEC); 4412 MNT_FLAG(MNT_NOSUID); 4413 MNT_FLAG(MNT_NFS4ACLS); 4414 MNT_FLAG(MNT_UNION); 4415 MNT_FLAG(MNT_ASYNC); 4416 MNT_FLAG(MNT_SUIDDIR); 4417 MNT_FLAG(MNT_SOFTDEP); 4418 MNT_FLAG(MNT_NOSYMFOLLOW); 4419 MNT_FLAG(MNT_GJOURNAL); 4420 MNT_FLAG(MNT_MULTILABEL); 4421 MNT_FLAG(MNT_ACLS); 4422 MNT_FLAG(MNT_NOATIME); 4423 MNT_FLAG(MNT_NOCLUSTERR); 4424 MNT_FLAG(MNT_NOCLUSTERW); 4425 MNT_FLAG(MNT_SUJ); 4426 MNT_FLAG(MNT_EXRDONLY); 4427 MNT_FLAG(MNT_EXPORTED); 4428 MNT_FLAG(MNT_DEFEXPORTED); 4429 MNT_FLAG(MNT_EXPORTANON); 4430 MNT_FLAG(MNT_EXKERB); 4431 MNT_FLAG(MNT_EXPUBLIC); 4432 MNT_FLAG(MNT_LOCAL); 4433 MNT_FLAG(MNT_QUOTA); 4434 MNT_FLAG(MNT_ROOTFS); 4435 MNT_FLAG(MNT_USER); 4436 MNT_FLAG(MNT_IGNORE); 4437 MNT_FLAG(MNT_UPDATE); 4438 MNT_FLAG(MNT_DELEXPORT); 4439 MNT_FLAG(MNT_RELOAD); 4440 MNT_FLAG(MNT_FORCE); 4441 MNT_FLAG(MNT_SNAPSHOT); 4442 MNT_FLAG(MNT_BYFSID); 4443 #undef MNT_FLAG 4444 if (mflags != 0) { 4445 if (buf[0] != '\0') 4446 strlcat(buf, ", ", sizeof(buf)); 4447 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4448 "0x%016jx", mflags); 4449 } 4450 db_printf(" mnt_flag = %s\n", buf); 4451 4452 buf[0] = '\0'; 4453 flags = mp->mnt_kern_flag; 4454 #define MNT_KERN_FLAG(flag) do { \ 4455 if (flags & (flag)) { \ 4456 if (buf[0] != '\0') \ 4457 strlcat(buf, ", ", sizeof(buf)); \ 4458 strlcat(buf, (#flag) + 5, sizeof(buf)); \ 4459 flags &= ~(flag); \ 4460 } \ 4461 } while (0) 4462 MNT_KERN_FLAG(MNTK_UNMOUNTF); 4463 MNT_KERN_FLAG(MNTK_ASYNC); 4464 MNT_KERN_FLAG(MNTK_SOFTDEP); 4465 MNT_KERN_FLAG(MNTK_NOMSYNC); 4466 MNT_KERN_FLAG(MNTK_DRAINING); 4467 MNT_KERN_FLAG(MNTK_REFEXPIRE); 4468 MNT_KERN_FLAG(MNTK_EXTENDED_SHARED); 4469 MNT_KERN_FLAG(MNTK_SHARED_WRITES); 4470 MNT_KERN_FLAG(MNTK_NO_IOPF); 4471 MNT_KERN_FLAG(MNTK_RECURSE); 4472 MNT_KERN_FLAG(MNTK_UPPER_WAITER); 4473 MNT_KERN_FLAG(MNTK_UNLOCKED_INSMNTQUE); 4474 MNT_KERN_FLAG(MNTK_USES_BCACHE); 4475 MNT_KERN_FLAG(MNTK_VMSETSIZE_BUG); 4476 MNT_KERN_FLAG(MNTK_FPLOOKUP); 4477 MNT_KERN_FLAG(MNTK_TASKQUEUE_WAITER); 4478 MNT_KERN_FLAG(MNTK_NOASYNC); 4479 MNT_KERN_FLAG(MNTK_UNMOUNT); 4480 MNT_KERN_FLAG(MNTK_MWAIT); 4481 MNT_KERN_FLAG(MNTK_SUSPEND); 4482 MNT_KERN_FLAG(MNTK_SUSPEND2); 4483 MNT_KERN_FLAG(MNTK_SUSPENDED); 4484 MNT_KERN_FLAG(MNTK_NULL_NOCACHE); 4485 MNT_KERN_FLAG(MNTK_LOOKUP_SHARED); 4486 #undef MNT_KERN_FLAG 4487 if (flags != 0) { 4488 if (buf[0] != '\0') 4489 strlcat(buf, ", ", sizeof(buf)); 4490 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4491 "0x%08x", flags); 4492 } 4493 db_printf(" mnt_kern_flag = %s\n", buf); 4494 4495 db_printf(" mnt_opt = "); 4496 opt = TAILQ_FIRST(mp->mnt_opt); 4497 if (opt != NULL) { 4498 db_printf("%s", opt->name); 4499 opt = TAILQ_NEXT(opt, link); 4500 while (opt != NULL) { 4501 db_printf(", %s", opt->name); 4502 opt = TAILQ_NEXT(opt, link); 4503 } 4504 } 4505 db_printf("\n"); 4506 4507 sp = &mp->mnt_stat; 4508 db_printf(" mnt_stat = { version=%u type=%u flags=0x%016jx " 4509 "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju " 4510 "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju " 4511 "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n", 4512 (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags, 4513 (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize, 4514 (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree, 4515 (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files, 4516 (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites, 4517 (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads, 4518 (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax, 4519 (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]); 4520 4521 db_printf(" mnt_cred = { uid=%u ruid=%u", 4522 (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid); 4523 if (jailed(mp->mnt_cred)) 4524 db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id); 4525 db_printf(" }\n"); 4526 db_printf(" mnt_ref = %d (with %d in the struct)\n", 4527 vfs_mount_fetch_counter(mp, MNT_COUNT_REF), mp->mnt_ref); 4528 db_printf(" mnt_gen = %d\n", mp->mnt_gen); 4529 db_printf(" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize); 4530 db_printf(" mnt_lazyvnodelistsize = %d\n", 4531 mp->mnt_lazyvnodelistsize); 4532 db_printf(" mnt_writeopcount = %d (with %d in the struct)\n", 4533 vfs_mount_fetch_counter(mp, MNT_COUNT_WRITEOPCOUNT), mp->mnt_writeopcount); 4534 db_printf(" mnt_iosize_max = %d\n", mp->mnt_iosize_max); 4535 db_printf(" mnt_hashseed = %u\n", mp->mnt_hashseed); 4536 db_printf(" mnt_lockref = %d (with %d in the struct)\n", 4537 vfs_mount_fetch_counter(mp, MNT_COUNT_LOCKREF), mp->mnt_lockref); 4538 db_printf(" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes); 4539 db_printf(" mnt_secondary_accwrites = %d\n", 4540 mp->mnt_secondary_accwrites); 4541 db_printf(" mnt_gjprovider = %s\n", 4542 mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL"); 4543 db_printf(" mnt_vfs_ops = %d\n", mp->mnt_vfs_ops); 4544 4545 db_printf("\n\nList of active vnodes\n"); 4546 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4547 if (vp->v_type != VMARKER && vp->v_holdcnt > 0) { 4548 vn_printf(vp, "vnode "); 4549 if (db_pager_quit) 4550 break; 4551 } 4552 } 4553 db_printf("\n\nList of inactive vnodes\n"); 4554 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4555 if (vp->v_type != VMARKER && vp->v_holdcnt == 0) { 4556 vn_printf(vp, "vnode "); 4557 if (db_pager_quit) 4558 break; 4559 } 4560 } 4561 } 4562 #endif /* DDB */ 4563 4564 /* 4565 * Fill in a struct xvfsconf based on a struct vfsconf. 4566 */ 4567 static int 4568 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp) 4569 { 4570 struct xvfsconf xvfsp; 4571 4572 bzero(&xvfsp, sizeof(xvfsp)); 4573 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4574 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4575 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4576 xvfsp.vfc_flags = vfsp->vfc_flags; 4577 /* 4578 * These are unused in userland, we keep them 4579 * to not break binary compatibility. 4580 */ 4581 xvfsp.vfc_vfsops = NULL; 4582 xvfsp.vfc_next = NULL; 4583 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4584 } 4585 4586 #ifdef COMPAT_FREEBSD32 4587 struct xvfsconf32 { 4588 uint32_t vfc_vfsops; 4589 char vfc_name[MFSNAMELEN]; 4590 int32_t vfc_typenum; 4591 int32_t vfc_refcount; 4592 int32_t vfc_flags; 4593 uint32_t vfc_next; 4594 }; 4595 4596 static int 4597 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp) 4598 { 4599 struct xvfsconf32 xvfsp; 4600 4601 bzero(&xvfsp, sizeof(xvfsp)); 4602 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4603 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4604 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4605 xvfsp.vfc_flags = vfsp->vfc_flags; 4606 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4607 } 4608 #endif 4609 4610 /* 4611 * Top level filesystem related information gathering. 4612 */ 4613 static int 4614 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS) 4615 { 4616 struct vfsconf *vfsp; 4617 int error; 4618 4619 error = 0; 4620 vfsconf_slock(); 4621 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4622 #ifdef COMPAT_FREEBSD32 4623 if (req->flags & SCTL_MASK32) 4624 error = vfsconf2x32(req, vfsp); 4625 else 4626 #endif 4627 error = vfsconf2x(req, vfsp); 4628 if (error) 4629 break; 4630 } 4631 vfsconf_sunlock(); 4632 return (error); 4633 } 4634 4635 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD | 4636 CTLFLAG_MPSAFE, NULL, 0, sysctl_vfs_conflist, 4637 "S,xvfsconf", "List of all configured filesystems"); 4638 4639 #ifndef BURN_BRIDGES 4640 static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS); 4641 4642 static int 4643 vfs_sysctl(SYSCTL_HANDLER_ARGS) 4644 { 4645 int *name = (int *)arg1 - 1; /* XXX */ 4646 u_int namelen = arg2 + 1; /* XXX */ 4647 struct vfsconf *vfsp; 4648 4649 log(LOG_WARNING, "userland calling deprecated sysctl, " 4650 "please rebuild world\n"); 4651 4652 #if 1 || defined(COMPAT_PRELITE2) 4653 /* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */ 4654 if (namelen == 1) 4655 return (sysctl_ovfs_conf(oidp, arg1, arg2, req)); 4656 #endif 4657 4658 switch (name[1]) { 4659 case VFS_MAXTYPENUM: 4660 if (namelen != 2) 4661 return (ENOTDIR); 4662 return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int))); 4663 case VFS_CONF: 4664 if (namelen != 3) 4665 return (ENOTDIR); /* overloaded */ 4666 vfsconf_slock(); 4667 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4668 if (vfsp->vfc_typenum == name[2]) 4669 break; 4670 } 4671 vfsconf_sunlock(); 4672 if (vfsp == NULL) 4673 return (EOPNOTSUPP); 4674 #ifdef COMPAT_FREEBSD32 4675 if (req->flags & SCTL_MASK32) 4676 return (vfsconf2x32(req, vfsp)); 4677 else 4678 #endif 4679 return (vfsconf2x(req, vfsp)); 4680 } 4681 return (EOPNOTSUPP); 4682 } 4683 4684 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP | 4685 CTLFLAG_MPSAFE, vfs_sysctl, 4686 "Generic filesystem"); 4687 4688 #if 1 || defined(COMPAT_PRELITE2) 4689 4690 static int 4691 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS) 4692 { 4693 int error; 4694 struct vfsconf *vfsp; 4695 struct ovfsconf ovfs; 4696 4697 vfsconf_slock(); 4698 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4699 bzero(&ovfs, sizeof(ovfs)); 4700 ovfs.vfc_vfsops = vfsp->vfc_vfsops; /* XXX used as flag */ 4701 strcpy(ovfs.vfc_name, vfsp->vfc_name); 4702 ovfs.vfc_index = vfsp->vfc_typenum; 4703 ovfs.vfc_refcount = vfsp->vfc_refcount; 4704 ovfs.vfc_flags = vfsp->vfc_flags; 4705 error = SYSCTL_OUT(req, &ovfs, sizeof ovfs); 4706 if (error != 0) { 4707 vfsconf_sunlock(); 4708 return (error); 4709 } 4710 } 4711 vfsconf_sunlock(); 4712 return (0); 4713 } 4714 4715 #endif /* 1 || COMPAT_PRELITE2 */ 4716 #endif /* !BURN_BRIDGES */ 4717 4718 static void 4719 unmount_or_warn(struct mount *mp) 4720 { 4721 int error; 4722 4723 error = dounmount(mp, MNT_FORCE, curthread); 4724 if (error != 0) { 4725 printf("unmount of %s failed (", mp->mnt_stat.f_mntonname); 4726 if (error == EBUSY) 4727 printf("BUSY)\n"); 4728 else 4729 printf("%d)\n", error); 4730 } 4731 } 4732 4733 /* 4734 * Unmount all filesystems. The list is traversed in reverse order 4735 * of mounting to avoid dependencies. 4736 */ 4737 void 4738 vfs_unmountall(void) 4739 { 4740 struct mount *mp, *tmp; 4741 4742 CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__); 4743 4744 /* 4745 * Since this only runs when rebooting, it is not interlocked. 4746 */ 4747 TAILQ_FOREACH_REVERSE_SAFE(mp, &mountlist, mntlist, mnt_list, tmp) { 4748 vfs_ref(mp); 4749 4750 /* 4751 * Forcibly unmounting "/dev" before "/" would prevent clean 4752 * unmount of the latter. 4753 */ 4754 if (mp == rootdevmp) 4755 continue; 4756 4757 unmount_or_warn(mp); 4758 } 4759 4760 if (rootdevmp != NULL) 4761 unmount_or_warn(rootdevmp); 4762 } 4763 4764 static void 4765 vfs_deferred_inactive(struct vnode *vp, int lkflags) 4766 { 4767 4768 ASSERT_VI_LOCKED(vp, __func__); 4769 VNASSERT((vp->v_iflag & VI_DEFINACT) == 0, vp, ("VI_DEFINACT still set")); 4770 if ((vp->v_iflag & VI_OWEINACT) == 0) { 4771 vdropl(vp); 4772 return; 4773 } 4774 if (vn_lock(vp, lkflags) == 0) { 4775 VI_LOCK(vp); 4776 vinactive(vp); 4777 VOP_UNLOCK(vp); 4778 vdropl(vp); 4779 return; 4780 } 4781 vdefer_inactive_unlocked(vp); 4782 } 4783 4784 static int 4785 vfs_periodic_inactive_filter(struct vnode *vp, void *arg) 4786 { 4787 4788 return (vp->v_iflag & VI_DEFINACT); 4789 } 4790 4791 static void __noinline 4792 vfs_periodic_inactive(struct mount *mp, int flags) 4793 { 4794 struct vnode *vp, *mvp; 4795 int lkflags; 4796 4797 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 4798 if (flags != MNT_WAIT) 4799 lkflags |= LK_NOWAIT; 4800 4801 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_inactive_filter, NULL) { 4802 if ((vp->v_iflag & VI_DEFINACT) == 0) { 4803 VI_UNLOCK(vp); 4804 continue; 4805 } 4806 vp->v_iflag &= ~VI_DEFINACT; 4807 vfs_deferred_inactive(vp, lkflags); 4808 } 4809 } 4810 4811 static inline bool 4812 vfs_want_msync(struct vnode *vp) 4813 { 4814 struct vm_object *obj; 4815 4816 /* 4817 * This test may be performed without any locks held. 4818 * We rely on vm_object's type stability. 4819 */ 4820 if (vp->v_vflag & VV_NOSYNC) 4821 return (false); 4822 obj = vp->v_object; 4823 return (obj != NULL && vm_object_mightbedirty(obj)); 4824 } 4825 4826 static int 4827 vfs_periodic_msync_inactive_filter(struct vnode *vp, void *arg __unused) 4828 { 4829 4830 if (vp->v_vflag & VV_NOSYNC) 4831 return (false); 4832 if (vp->v_iflag & VI_DEFINACT) 4833 return (true); 4834 return (vfs_want_msync(vp)); 4835 } 4836 4837 static void __noinline 4838 vfs_periodic_msync_inactive(struct mount *mp, int flags) 4839 { 4840 struct vnode *vp, *mvp; 4841 struct vm_object *obj; 4842 int lkflags, objflags; 4843 bool seen_defer; 4844 4845 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 4846 if (flags != MNT_WAIT) { 4847 lkflags |= LK_NOWAIT; 4848 objflags = OBJPC_NOSYNC; 4849 } else { 4850 objflags = OBJPC_SYNC; 4851 } 4852 4853 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_msync_inactive_filter, NULL) { 4854 seen_defer = false; 4855 if (vp->v_iflag & VI_DEFINACT) { 4856 vp->v_iflag &= ~VI_DEFINACT; 4857 seen_defer = true; 4858 } 4859 if (!vfs_want_msync(vp)) { 4860 if (seen_defer) 4861 vfs_deferred_inactive(vp, lkflags); 4862 else 4863 VI_UNLOCK(vp); 4864 continue; 4865 } 4866 if (vget(vp, lkflags) == 0) { 4867 obj = vp->v_object; 4868 if (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0) { 4869 VM_OBJECT_WLOCK(obj); 4870 vm_object_page_clean(obj, 0, 0, objflags); 4871 VM_OBJECT_WUNLOCK(obj); 4872 } 4873 vput(vp); 4874 if (seen_defer) 4875 vdrop(vp); 4876 } else { 4877 if (seen_defer) 4878 vdefer_inactive_unlocked(vp); 4879 } 4880 } 4881 } 4882 4883 void 4884 vfs_periodic(struct mount *mp, int flags) 4885 { 4886 4887 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 4888 4889 if ((mp->mnt_kern_flag & MNTK_NOMSYNC) != 0) 4890 vfs_periodic_inactive(mp, flags); 4891 else 4892 vfs_periodic_msync_inactive(mp, flags); 4893 } 4894 4895 static void 4896 destroy_vpollinfo_free(struct vpollinfo *vi) 4897 { 4898 4899 knlist_destroy(&vi->vpi_selinfo.si_note); 4900 mtx_destroy(&vi->vpi_lock); 4901 free(vi, M_VNODEPOLL); 4902 } 4903 4904 static void 4905 destroy_vpollinfo(struct vpollinfo *vi) 4906 { 4907 4908 knlist_clear(&vi->vpi_selinfo.si_note, 1); 4909 seldrain(&vi->vpi_selinfo); 4910 destroy_vpollinfo_free(vi); 4911 } 4912 4913 /* 4914 * Initialize per-vnode helper structure to hold poll-related state. 4915 */ 4916 void 4917 v_addpollinfo(struct vnode *vp) 4918 { 4919 struct vpollinfo *vi; 4920 4921 if (vp->v_pollinfo != NULL) 4922 return; 4923 vi = malloc(sizeof(*vi), M_VNODEPOLL, M_WAITOK | M_ZERO); 4924 mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF); 4925 knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock, 4926 vfs_knlunlock, vfs_knl_assert_lock); 4927 VI_LOCK(vp); 4928 if (vp->v_pollinfo != NULL) { 4929 VI_UNLOCK(vp); 4930 destroy_vpollinfo_free(vi); 4931 return; 4932 } 4933 vp->v_pollinfo = vi; 4934 VI_UNLOCK(vp); 4935 } 4936 4937 /* 4938 * Record a process's interest in events which might happen to 4939 * a vnode. Because poll uses the historic select-style interface 4940 * internally, this routine serves as both the ``check for any 4941 * pending events'' and the ``record my interest in future events'' 4942 * functions. (These are done together, while the lock is held, 4943 * to avoid race conditions.) 4944 */ 4945 int 4946 vn_pollrecord(struct vnode *vp, struct thread *td, int events) 4947 { 4948 4949 v_addpollinfo(vp); 4950 mtx_lock(&vp->v_pollinfo->vpi_lock); 4951 if (vp->v_pollinfo->vpi_revents & events) { 4952 /* 4953 * This leaves events we are not interested 4954 * in available for the other process which 4955 * which presumably had requested them 4956 * (otherwise they would never have been 4957 * recorded). 4958 */ 4959 events &= vp->v_pollinfo->vpi_revents; 4960 vp->v_pollinfo->vpi_revents &= ~events; 4961 4962 mtx_unlock(&vp->v_pollinfo->vpi_lock); 4963 return (events); 4964 } 4965 vp->v_pollinfo->vpi_events |= events; 4966 selrecord(td, &vp->v_pollinfo->vpi_selinfo); 4967 mtx_unlock(&vp->v_pollinfo->vpi_lock); 4968 return (0); 4969 } 4970 4971 /* 4972 * Routine to create and manage a filesystem syncer vnode. 4973 */ 4974 #define sync_close ((int (*)(struct vop_close_args *))nullop) 4975 static int sync_fsync(struct vop_fsync_args *); 4976 static int sync_inactive(struct vop_inactive_args *); 4977 static int sync_reclaim(struct vop_reclaim_args *); 4978 4979 static struct vop_vector sync_vnodeops = { 4980 .vop_bypass = VOP_EOPNOTSUPP, 4981 .vop_close = sync_close, /* close */ 4982 .vop_fsync = sync_fsync, /* fsync */ 4983 .vop_inactive = sync_inactive, /* inactive */ 4984 .vop_need_inactive = vop_stdneed_inactive, /* need_inactive */ 4985 .vop_reclaim = sync_reclaim, /* reclaim */ 4986 .vop_lock1 = vop_stdlock, /* lock */ 4987 .vop_unlock = vop_stdunlock, /* unlock */ 4988 .vop_islocked = vop_stdislocked, /* islocked */ 4989 }; 4990 VFS_VOP_VECTOR_REGISTER(sync_vnodeops); 4991 4992 /* 4993 * Create a new filesystem syncer vnode for the specified mount point. 4994 */ 4995 void 4996 vfs_allocate_syncvnode(struct mount *mp) 4997 { 4998 struct vnode *vp; 4999 struct bufobj *bo; 5000 static long start, incr, next; 5001 int error; 5002 5003 /* Allocate a new vnode */ 5004 error = getnewvnode("syncer", mp, &sync_vnodeops, &vp); 5005 if (error != 0) 5006 panic("vfs_allocate_syncvnode: getnewvnode() failed"); 5007 vp->v_type = VNON; 5008 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5009 vp->v_vflag |= VV_FORCEINSMQ; 5010 error = insmntque1(vp, mp); 5011 if (error != 0) 5012 panic("vfs_allocate_syncvnode: insmntque() failed"); 5013 vp->v_vflag &= ~VV_FORCEINSMQ; 5014 vn_set_state(vp, VSTATE_CONSTRUCTED); 5015 VOP_UNLOCK(vp); 5016 /* 5017 * Place the vnode onto the syncer worklist. We attempt to 5018 * scatter them about on the list so that they will go off 5019 * at evenly distributed times even if all the filesystems 5020 * are mounted at once. 5021 */ 5022 next += incr; 5023 if (next == 0 || next > syncer_maxdelay) { 5024 start /= 2; 5025 incr /= 2; 5026 if (start == 0) { 5027 start = syncer_maxdelay / 2; 5028 incr = syncer_maxdelay; 5029 } 5030 next = start; 5031 } 5032 bo = &vp->v_bufobj; 5033 BO_LOCK(bo); 5034 vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0); 5035 /* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */ 5036 mtx_lock(&sync_mtx); 5037 sync_vnode_count++; 5038 if (mp->mnt_syncer == NULL) { 5039 mp->mnt_syncer = vp; 5040 vp = NULL; 5041 } 5042 mtx_unlock(&sync_mtx); 5043 BO_UNLOCK(bo); 5044 if (vp != NULL) { 5045 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5046 vgone(vp); 5047 vput(vp); 5048 } 5049 } 5050 5051 void 5052 vfs_deallocate_syncvnode(struct mount *mp) 5053 { 5054 struct vnode *vp; 5055 5056 mtx_lock(&sync_mtx); 5057 vp = mp->mnt_syncer; 5058 if (vp != NULL) 5059 mp->mnt_syncer = NULL; 5060 mtx_unlock(&sync_mtx); 5061 if (vp != NULL) 5062 vrele(vp); 5063 } 5064 5065 /* 5066 * Do a lazy sync of the filesystem. 5067 */ 5068 static int 5069 sync_fsync(struct vop_fsync_args *ap) 5070 { 5071 struct vnode *syncvp = ap->a_vp; 5072 struct mount *mp = syncvp->v_mount; 5073 int error, save; 5074 struct bufobj *bo; 5075 5076 /* 5077 * We only need to do something if this is a lazy evaluation. 5078 */ 5079 if (ap->a_waitfor != MNT_LAZY) 5080 return (0); 5081 5082 /* 5083 * Move ourselves to the back of the sync list. 5084 */ 5085 bo = &syncvp->v_bufobj; 5086 BO_LOCK(bo); 5087 vn_syncer_add_to_worklist(bo, syncdelay); 5088 BO_UNLOCK(bo); 5089 5090 /* 5091 * Walk the list of vnodes pushing all that are dirty and 5092 * not already on the sync list. 5093 */ 5094 if (vfs_busy(mp, MBF_NOWAIT) != 0) 5095 return (0); 5096 VOP_UNLOCK(syncvp); 5097 save = curthread_pflags_set(TDP_SYNCIO); 5098 /* 5099 * The filesystem at hand may be idle with free vnodes stored in the 5100 * batch. Return them instead of letting them stay there indefinitely. 5101 */ 5102 vfs_periodic(mp, MNT_NOWAIT); 5103 error = VFS_SYNC(mp, MNT_LAZY); 5104 curthread_pflags_restore(save); 5105 vn_lock(syncvp, LK_EXCLUSIVE | LK_RETRY); 5106 vfs_unbusy(mp); 5107 return (error); 5108 } 5109 5110 /* 5111 * The syncer vnode is no referenced. 5112 */ 5113 static int 5114 sync_inactive(struct vop_inactive_args *ap) 5115 { 5116 5117 vgone(ap->a_vp); 5118 return (0); 5119 } 5120 5121 /* 5122 * The syncer vnode is no longer needed and is being decommissioned. 5123 * 5124 * Modifications to the worklist must be protected by sync_mtx. 5125 */ 5126 static int 5127 sync_reclaim(struct vop_reclaim_args *ap) 5128 { 5129 struct vnode *vp = ap->a_vp; 5130 struct bufobj *bo; 5131 5132 bo = &vp->v_bufobj; 5133 BO_LOCK(bo); 5134 mtx_lock(&sync_mtx); 5135 if (vp->v_mount->mnt_syncer == vp) 5136 vp->v_mount->mnt_syncer = NULL; 5137 if (bo->bo_flag & BO_ONWORKLST) { 5138 LIST_REMOVE(bo, bo_synclist); 5139 syncer_worklist_len--; 5140 sync_vnode_count--; 5141 bo->bo_flag &= ~BO_ONWORKLST; 5142 } 5143 mtx_unlock(&sync_mtx); 5144 BO_UNLOCK(bo); 5145 5146 return (0); 5147 } 5148 5149 int 5150 vn_need_pageq_flush(struct vnode *vp) 5151 { 5152 struct vm_object *obj; 5153 5154 obj = vp->v_object; 5155 return (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 5156 vm_object_mightbedirty(obj)); 5157 } 5158 5159 /* 5160 * Check if vnode represents a disk device 5161 */ 5162 bool 5163 vn_isdisk_error(struct vnode *vp, int *errp) 5164 { 5165 int error; 5166 5167 if (vp->v_type != VCHR) { 5168 error = ENOTBLK; 5169 goto out; 5170 } 5171 error = 0; 5172 dev_lock(); 5173 if (vp->v_rdev == NULL) 5174 error = ENXIO; 5175 else if (vp->v_rdev->si_devsw == NULL) 5176 error = ENXIO; 5177 else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK)) 5178 error = ENOTBLK; 5179 dev_unlock(); 5180 out: 5181 *errp = error; 5182 return (error == 0); 5183 } 5184 5185 bool 5186 vn_isdisk(struct vnode *vp) 5187 { 5188 int error; 5189 5190 return (vn_isdisk_error(vp, &error)); 5191 } 5192 5193 /* 5194 * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see 5195 * the comment above cache_fplookup for details. 5196 */ 5197 int 5198 vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid, struct ucred *cred) 5199 { 5200 int error; 5201 5202 VFS_SMR_ASSERT_ENTERED(); 5203 5204 /* Check the owner. */ 5205 if (cred->cr_uid == file_uid) { 5206 if (file_mode & S_IXUSR) 5207 return (0); 5208 goto out_error; 5209 } 5210 5211 /* Otherwise, check the groups (first match) */ 5212 if (groupmember(file_gid, cred)) { 5213 if (file_mode & S_IXGRP) 5214 return (0); 5215 goto out_error; 5216 } 5217 5218 /* Otherwise, check everyone else. */ 5219 if (file_mode & S_IXOTH) 5220 return (0); 5221 out_error: 5222 /* 5223 * Permission check failed, but it is possible denial will get overwritten 5224 * (e.g., when root is traversing through a 700 directory owned by someone 5225 * else). 5226 * 5227 * vaccess() calls priv_check_cred which in turn can descent into MAC 5228 * modules overriding this result. It's quite unclear what semantics 5229 * are allowed for them to operate, thus for safety we don't call them 5230 * from within the SMR section. This also means if any such modules 5231 * are present, we have to let the regular lookup decide. 5232 */ 5233 error = priv_check_cred_vfs_lookup_nomac(cred); 5234 switch (error) { 5235 case 0: 5236 return (0); 5237 case EAGAIN: 5238 /* 5239 * MAC modules present. 5240 */ 5241 return (EAGAIN); 5242 case EPERM: 5243 return (EACCES); 5244 default: 5245 return (error); 5246 } 5247 } 5248 5249 /* 5250 * Common filesystem object access control check routine. Accepts a 5251 * vnode's type, "mode", uid and gid, requested access mode, and credentials. 5252 * Returns 0 on success, or an errno on failure. 5253 */ 5254 int 5255 vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid, 5256 accmode_t accmode, struct ucred *cred) 5257 { 5258 accmode_t dac_granted; 5259 accmode_t priv_granted; 5260 5261 KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0, 5262 ("invalid bit in accmode")); 5263 KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE), 5264 ("VAPPEND without VWRITE")); 5265 5266 /* 5267 * Look for a normal, non-privileged way to access the file/directory 5268 * as requested. If it exists, go with that. 5269 */ 5270 5271 dac_granted = 0; 5272 5273 /* Check the owner. */ 5274 if (cred->cr_uid == file_uid) { 5275 dac_granted |= VADMIN; 5276 if (file_mode & S_IXUSR) 5277 dac_granted |= VEXEC; 5278 if (file_mode & S_IRUSR) 5279 dac_granted |= VREAD; 5280 if (file_mode & S_IWUSR) 5281 dac_granted |= (VWRITE | VAPPEND); 5282 5283 if ((accmode & dac_granted) == accmode) 5284 return (0); 5285 5286 goto privcheck; 5287 } 5288 5289 /* Otherwise, check the groups (first match) */ 5290 if (groupmember(file_gid, cred)) { 5291 if (file_mode & S_IXGRP) 5292 dac_granted |= VEXEC; 5293 if (file_mode & S_IRGRP) 5294 dac_granted |= VREAD; 5295 if (file_mode & S_IWGRP) 5296 dac_granted |= (VWRITE | VAPPEND); 5297 5298 if ((accmode & dac_granted) == accmode) 5299 return (0); 5300 5301 goto privcheck; 5302 } 5303 5304 /* Otherwise, check everyone else. */ 5305 if (file_mode & S_IXOTH) 5306 dac_granted |= VEXEC; 5307 if (file_mode & S_IROTH) 5308 dac_granted |= VREAD; 5309 if (file_mode & S_IWOTH) 5310 dac_granted |= (VWRITE | VAPPEND); 5311 if ((accmode & dac_granted) == accmode) 5312 return (0); 5313 5314 privcheck: 5315 /* 5316 * Build a privilege mask to determine if the set of privileges 5317 * satisfies the requirements when combined with the granted mask 5318 * from above. For each privilege, if the privilege is required, 5319 * bitwise or the request type onto the priv_granted mask. 5320 */ 5321 priv_granted = 0; 5322 5323 if (type == VDIR) { 5324 /* 5325 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC 5326 * requests, instead of PRIV_VFS_EXEC. 5327 */ 5328 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5329 !priv_check_cred(cred, PRIV_VFS_LOOKUP)) 5330 priv_granted |= VEXEC; 5331 } else { 5332 /* 5333 * Ensure that at least one execute bit is on. Otherwise, 5334 * a privileged user will always succeed, and we don't want 5335 * this to happen unless the file really is executable. 5336 */ 5337 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5338 (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && 5339 !priv_check_cred(cred, PRIV_VFS_EXEC)) 5340 priv_granted |= VEXEC; 5341 } 5342 5343 if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) && 5344 !priv_check_cred(cred, PRIV_VFS_READ)) 5345 priv_granted |= VREAD; 5346 5347 if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) && 5348 !priv_check_cred(cred, PRIV_VFS_WRITE)) 5349 priv_granted |= (VWRITE | VAPPEND); 5350 5351 if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) && 5352 !priv_check_cred(cred, PRIV_VFS_ADMIN)) 5353 priv_granted |= VADMIN; 5354 5355 if ((accmode & (priv_granted | dac_granted)) == accmode) { 5356 return (0); 5357 } 5358 5359 return ((accmode & VADMIN) ? EPERM : EACCES); 5360 } 5361 5362 /* 5363 * Credential check based on process requesting service, and per-attribute 5364 * permissions. 5365 */ 5366 int 5367 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred, 5368 struct thread *td, accmode_t accmode) 5369 { 5370 5371 /* 5372 * Kernel-invoked always succeeds. 5373 */ 5374 if (cred == NOCRED) 5375 return (0); 5376 5377 /* 5378 * Do not allow privileged processes in jail to directly manipulate 5379 * system attributes. 5380 */ 5381 switch (attrnamespace) { 5382 case EXTATTR_NAMESPACE_SYSTEM: 5383 /* Potentially should be: return (EPERM); */ 5384 return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM)); 5385 case EXTATTR_NAMESPACE_USER: 5386 return (VOP_ACCESS(vp, accmode, cred, td)); 5387 default: 5388 return (EPERM); 5389 } 5390 } 5391 5392 #ifdef DEBUG_VFS_LOCKS 5393 int vfs_badlock_ddb = 1; /* Drop into debugger on violation. */ 5394 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0, 5395 "Drop into debugger on lock violation"); 5396 5397 int vfs_badlock_mutex = 1; /* Check for interlock across VOPs. */ 5398 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex, 5399 0, "Check for interlock across VOPs"); 5400 5401 int vfs_badlock_print = 1; /* Print lock violations. */ 5402 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print, 5403 0, "Print lock violations"); 5404 5405 int vfs_badlock_vnode = 1; /* Print vnode details on lock violations. */ 5406 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_vnode, CTLFLAG_RW, &vfs_badlock_vnode, 5407 0, "Print vnode details on lock violations"); 5408 5409 #ifdef KDB 5410 int vfs_badlock_backtrace = 1; /* Print backtrace at lock violations. */ 5411 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW, 5412 &vfs_badlock_backtrace, 0, "Print backtrace at lock violations"); 5413 #endif 5414 5415 static void 5416 vfs_badlock(const char *msg, const char *str, struct vnode *vp) 5417 { 5418 5419 #ifdef KDB 5420 if (vfs_badlock_backtrace) 5421 kdb_backtrace(); 5422 #endif 5423 if (vfs_badlock_vnode) 5424 vn_printf(vp, "vnode "); 5425 if (vfs_badlock_print) 5426 printf("%s: %p %s\n", str, (void *)vp, msg); 5427 if (vfs_badlock_ddb) 5428 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5429 } 5430 5431 void 5432 assert_vi_locked(struct vnode *vp, const char *str) 5433 { 5434 5435 if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp))) 5436 vfs_badlock("interlock is not locked but should be", str, vp); 5437 } 5438 5439 void 5440 assert_vi_unlocked(struct vnode *vp, const char *str) 5441 { 5442 5443 if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp))) 5444 vfs_badlock("interlock is locked but should not be", str, vp); 5445 } 5446 5447 void 5448 assert_vop_locked(struct vnode *vp, const char *str) 5449 { 5450 int locked; 5451 5452 if (KERNEL_PANICKED() || vp == NULL) 5453 return; 5454 5455 locked = VOP_ISLOCKED(vp); 5456 if (locked == 0 || locked == LK_EXCLOTHER) 5457 vfs_badlock("is not locked but should be", str, vp); 5458 } 5459 5460 void 5461 assert_vop_unlocked(struct vnode *vp, const char *str) 5462 { 5463 if (KERNEL_PANICKED() || vp == NULL) 5464 return; 5465 5466 if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE) 5467 vfs_badlock("is locked but should not be", str, vp); 5468 } 5469 5470 void 5471 assert_vop_elocked(struct vnode *vp, const char *str) 5472 { 5473 if (KERNEL_PANICKED() || vp == NULL) 5474 return; 5475 5476 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) 5477 vfs_badlock("is not exclusive locked but should be", str, vp); 5478 } 5479 #endif /* DEBUG_VFS_LOCKS */ 5480 5481 void 5482 vop_rename_fail(struct vop_rename_args *ap) 5483 { 5484 5485 if (ap->a_tvp != NULL) 5486 vput(ap->a_tvp); 5487 if (ap->a_tdvp == ap->a_tvp) 5488 vrele(ap->a_tdvp); 5489 else 5490 vput(ap->a_tdvp); 5491 vrele(ap->a_fdvp); 5492 vrele(ap->a_fvp); 5493 } 5494 5495 void 5496 vop_rename_pre(void *ap) 5497 { 5498 struct vop_rename_args *a = ap; 5499 5500 #ifdef DEBUG_VFS_LOCKS 5501 if (a->a_tvp) 5502 ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME"); 5503 ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME"); 5504 ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME"); 5505 ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME"); 5506 5507 /* Check the source (from). */ 5508 if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock && 5509 (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock)) 5510 ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked"); 5511 if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock) 5512 ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked"); 5513 5514 /* Check the target. */ 5515 if (a->a_tvp) 5516 ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked"); 5517 ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked"); 5518 #endif 5519 /* 5520 * It may be tempting to add vn_seqc_write_begin/end calls here and 5521 * in vop_rename_post but that's not going to work out since some 5522 * filesystems relookup vnodes mid-rename. This is probably a bug. 5523 * 5524 * For now filesystems are expected to do the relevant calls after they 5525 * decide what vnodes to operate on. 5526 */ 5527 if (a->a_tdvp != a->a_fdvp) 5528 vhold(a->a_fdvp); 5529 if (a->a_tvp != a->a_fvp) 5530 vhold(a->a_fvp); 5531 vhold(a->a_tdvp); 5532 if (a->a_tvp) 5533 vhold(a->a_tvp); 5534 } 5535 5536 #ifdef DEBUG_VFS_LOCKS 5537 void 5538 vop_fplookup_vexec_debugpre(void *ap __unused) 5539 { 5540 5541 VFS_SMR_ASSERT_ENTERED(); 5542 } 5543 5544 void 5545 vop_fplookup_vexec_debugpost(void *ap __unused, int rc __unused) 5546 { 5547 5548 VFS_SMR_ASSERT_ENTERED(); 5549 } 5550 5551 void 5552 vop_fplookup_symlink_debugpre(void *ap __unused) 5553 { 5554 5555 VFS_SMR_ASSERT_ENTERED(); 5556 } 5557 5558 void 5559 vop_fplookup_symlink_debugpost(void *ap __unused, int rc __unused) 5560 { 5561 5562 VFS_SMR_ASSERT_ENTERED(); 5563 } 5564 5565 static void 5566 vop_fsync_debugprepost(struct vnode *vp, const char *name) 5567 { 5568 if (vp->v_type == VCHR) 5569 ; 5570 else if (MNT_EXTENDED_SHARED(vp->v_mount)) 5571 ASSERT_VOP_LOCKED(vp, name); 5572 else 5573 ASSERT_VOP_ELOCKED(vp, name); 5574 } 5575 5576 void 5577 vop_fsync_debugpre(void *a) 5578 { 5579 struct vop_fsync_args *ap; 5580 5581 ap = a; 5582 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5583 } 5584 5585 void 5586 vop_fsync_debugpost(void *a, int rc __unused) 5587 { 5588 struct vop_fsync_args *ap; 5589 5590 ap = a; 5591 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5592 } 5593 5594 void 5595 vop_fdatasync_debugpre(void *a) 5596 { 5597 struct vop_fdatasync_args *ap; 5598 5599 ap = a; 5600 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5601 } 5602 5603 void 5604 vop_fdatasync_debugpost(void *a, int rc __unused) 5605 { 5606 struct vop_fdatasync_args *ap; 5607 5608 ap = a; 5609 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5610 } 5611 5612 void 5613 vop_strategy_debugpre(void *ap) 5614 { 5615 struct vop_strategy_args *a; 5616 struct buf *bp; 5617 5618 a = ap; 5619 bp = a->a_bp; 5620 5621 /* 5622 * Cluster ops lock their component buffers but not the IO container. 5623 */ 5624 if ((bp->b_flags & B_CLUSTER) != 0) 5625 return; 5626 5627 if (!KERNEL_PANICKED() && !BUF_ISLOCKED(bp)) { 5628 if (vfs_badlock_print) 5629 printf( 5630 "VOP_STRATEGY: bp is not locked but should be\n"); 5631 if (vfs_badlock_ddb) 5632 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5633 } 5634 } 5635 5636 void 5637 vop_lock_debugpre(void *ap) 5638 { 5639 struct vop_lock1_args *a = ap; 5640 5641 if ((a->a_flags & LK_INTERLOCK) == 0) 5642 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 5643 else 5644 ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK"); 5645 } 5646 5647 void 5648 vop_lock_debugpost(void *ap, int rc) 5649 { 5650 struct vop_lock1_args *a = ap; 5651 5652 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 5653 if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0) 5654 ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK"); 5655 } 5656 5657 void 5658 vop_unlock_debugpre(void *ap) 5659 { 5660 struct vop_unlock_args *a = ap; 5661 struct vnode *vp = a->a_vp; 5662 5663 VNPASS(vn_get_state(vp) != VSTATE_UNINITIALIZED, vp); 5664 ASSERT_VOP_LOCKED(vp, "VOP_UNLOCK"); 5665 } 5666 5667 void 5668 vop_need_inactive_debugpre(void *ap) 5669 { 5670 struct vop_need_inactive_args *a = ap; 5671 5672 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 5673 } 5674 5675 void 5676 vop_need_inactive_debugpost(void *ap, int rc) 5677 { 5678 struct vop_need_inactive_args *a = ap; 5679 5680 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 5681 } 5682 #endif 5683 5684 void 5685 vop_create_pre(void *ap) 5686 { 5687 struct vop_create_args *a; 5688 struct vnode *dvp; 5689 5690 a = ap; 5691 dvp = a->a_dvp; 5692 vn_seqc_write_begin(dvp); 5693 } 5694 5695 void 5696 vop_create_post(void *ap, int rc) 5697 { 5698 struct vop_create_args *a; 5699 struct vnode *dvp; 5700 5701 a = ap; 5702 dvp = a->a_dvp; 5703 vn_seqc_write_end(dvp); 5704 if (!rc) 5705 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5706 } 5707 5708 void 5709 vop_whiteout_pre(void *ap) 5710 { 5711 struct vop_whiteout_args *a; 5712 struct vnode *dvp; 5713 5714 a = ap; 5715 dvp = a->a_dvp; 5716 vn_seqc_write_begin(dvp); 5717 } 5718 5719 void 5720 vop_whiteout_post(void *ap, int rc) 5721 { 5722 struct vop_whiteout_args *a; 5723 struct vnode *dvp; 5724 5725 a = ap; 5726 dvp = a->a_dvp; 5727 vn_seqc_write_end(dvp); 5728 } 5729 5730 void 5731 vop_deleteextattr_pre(void *ap) 5732 { 5733 struct vop_deleteextattr_args *a; 5734 struct vnode *vp; 5735 5736 a = ap; 5737 vp = a->a_vp; 5738 vn_seqc_write_begin(vp); 5739 } 5740 5741 void 5742 vop_deleteextattr_post(void *ap, int rc) 5743 { 5744 struct vop_deleteextattr_args *a; 5745 struct vnode *vp; 5746 5747 a = ap; 5748 vp = a->a_vp; 5749 vn_seqc_write_end(vp); 5750 if (!rc) 5751 VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB); 5752 } 5753 5754 void 5755 vop_link_pre(void *ap) 5756 { 5757 struct vop_link_args *a; 5758 struct vnode *vp, *tdvp; 5759 5760 a = ap; 5761 vp = a->a_vp; 5762 tdvp = a->a_tdvp; 5763 vn_seqc_write_begin(vp); 5764 vn_seqc_write_begin(tdvp); 5765 } 5766 5767 void 5768 vop_link_post(void *ap, int rc) 5769 { 5770 struct vop_link_args *a; 5771 struct vnode *vp, *tdvp; 5772 5773 a = ap; 5774 vp = a->a_vp; 5775 tdvp = a->a_tdvp; 5776 vn_seqc_write_end(vp); 5777 vn_seqc_write_end(tdvp); 5778 if (!rc) { 5779 VFS_KNOTE_LOCKED(vp, NOTE_LINK); 5780 VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE); 5781 } 5782 } 5783 5784 void 5785 vop_mkdir_pre(void *ap) 5786 { 5787 struct vop_mkdir_args *a; 5788 struct vnode *dvp; 5789 5790 a = ap; 5791 dvp = a->a_dvp; 5792 vn_seqc_write_begin(dvp); 5793 } 5794 5795 void 5796 vop_mkdir_post(void *ap, int rc) 5797 { 5798 struct vop_mkdir_args *a; 5799 struct vnode *dvp; 5800 5801 a = ap; 5802 dvp = a->a_dvp; 5803 vn_seqc_write_end(dvp); 5804 if (!rc) 5805 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 5806 } 5807 5808 #ifdef DEBUG_VFS_LOCKS 5809 void 5810 vop_mkdir_debugpost(void *ap, int rc) 5811 { 5812 struct vop_mkdir_args *a; 5813 5814 a = ap; 5815 if (!rc) 5816 cache_validate(a->a_dvp, *a->a_vpp, a->a_cnp); 5817 } 5818 #endif 5819 5820 void 5821 vop_mknod_pre(void *ap) 5822 { 5823 struct vop_mknod_args *a; 5824 struct vnode *dvp; 5825 5826 a = ap; 5827 dvp = a->a_dvp; 5828 vn_seqc_write_begin(dvp); 5829 } 5830 5831 void 5832 vop_mknod_post(void *ap, int rc) 5833 { 5834 struct vop_mknod_args *a; 5835 struct vnode *dvp; 5836 5837 a = ap; 5838 dvp = a->a_dvp; 5839 vn_seqc_write_end(dvp); 5840 if (!rc) 5841 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5842 } 5843 5844 void 5845 vop_reclaim_post(void *ap, int rc) 5846 { 5847 struct vop_reclaim_args *a; 5848 struct vnode *vp; 5849 5850 a = ap; 5851 vp = a->a_vp; 5852 ASSERT_VOP_IN_SEQC(vp); 5853 if (!rc) 5854 VFS_KNOTE_LOCKED(vp, NOTE_REVOKE); 5855 } 5856 5857 void 5858 vop_remove_pre(void *ap) 5859 { 5860 struct vop_remove_args *a; 5861 struct vnode *dvp, *vp; 5862 5863 a = ap; 5864 dvp = a->a_dvp; 5865 vp = a->a_vp; 5866 vn_seqc_write_begin(dvp); 5867 vn_seqc_write_begin(vp); 5868 } 5869 5870 void 5871 vop_remove_post(void *ap, int rc) 5872 { 5873 struct vop_remove_args *a; 5874 struct vnode *dvp, *vp; 5875 5876 a = ap; 5877 dvp = a->a_dvp; 5878 vp = a->a_vp; 5879 vn_seqc_write_end(dvp); 5880 vn_seqc_write_end(vp); 5881 if (!rc) { 5882 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 5883 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 5884 } 5885 } 5886 5887 void 5888 vop_rename_post(void *ap, int rc) 5889 { 5890 struct vop_rename_args *a = ap; 5891 long hint; 5892 5893 if (!rc) { 5894 hint = NOTE_WRITE; 5895 if (a->a_fdvp == a->a_tdvp) { 5896 if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR) 5897 hint |= NOTE_LINK; 5898 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 5899 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 5900 } else { 5901 hint |= NOTE_EXTEND; 5902 if (a->a_fvp->v_type == VDIR) 5903 hint |= NOTE_LINK; 5904 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 5905 5906 if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL && 5907 a->a_tvp->v_type == VDIR) 5908 hint &= ~NOTE_LINK; 5909 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 5910 } 5911 5912 VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME); 5913 if (a->a_tvp) 5914 VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE); 5915 } 5916 if (a->a_tdvp != a->a_fdvp) 5917 vdrop(a->a_fdvp); 5918 if (a->a_tvp != a->a_fvp) 5919 vdrop(a->a_fvp); 5920 vdrop(a->a_tdvp); 5921 if (a->a_tvp) 5922 vdrop(a->a_tvp); 5923 } 5924 5925 void 5926 vop_rmdir_pre(void *ap) 5927 { 5928 struct vop_rmdir_args *a; 5929 struct vnode *dvp, *vp; 5930 5931 a = ap; 5932 dvp = a->a_dvp; 5933 vp = a->a_vp; 5934 vn_seqc_write_begin(dvp); 5935 vn_seqc_write_begin(vp); 5936 } 5937 5938 void 5939 vop_rmdir_post(void *ap, int rc) 5940 { 5941 struct vop_rmdir_args *a; 5942 struct vnode *dvp, *vp; 5943 5944 a = ap; 5945 dvp = a->a_dvp; 5946 vp = a->a_vp; 5947 vn_seqc_write_end(dvp); 5948 vn_seqc_write_end(vp); 5949 if (!rc) { 5950 vp->v_vflag |= VV_UNLINKED; 5951 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 5952 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 5953 } 5954 } 5955 5956 void 5957 vop_setattr_pre(void *ap) 5958 { 5959 struct vop_setattr_args *a; 5960 struct vnode *vp; 5961 5962 a = ap; 5963 vp = a->a_vp; 5964 vn_seqc_write_begin(vp); 5965 } 5966 5967 void 5968 vop_setattr_post(void *ap, int rc) 5969 { 5970 struct vop_setattr_args *a; 5971 struct vnode *vp; 5972 5973 a = ap; 5974 vp = a->a_vp; 5975 vn_seqc_write_end(vp); 5976 if (!rc) 5977 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 5978 } 5979 5980 void 5981 vop_setacl_pre(void *ap) 5982 { 5983 struct vop_setacl_args *a; 5984 struct vnode *vp; 5985 5986 a = ap; 5987 vp = a->a_vp; 5988 vn_seqc_write_begin(vp); 5989 } 5990 5991 void 5992 vop_setacl_post(void *ap, int rc __unused) 5993 { 5994 struct vop_setacl_args *a; 5995 struct vnode *vp; 5996 5997 a = ap; 5998 vp = a->a_vp; 5999 vn_seqc_write_end(vp); 6000 } 6001 6002 void 6003 vop_setextattr_pre(void *ap) 6004 { 6005 struct vop_setextattr_args *a; 6006 struct vnode *vp; 6007 6008 a = ap; 6009 vp = a->a_vp; 6010 vn_seqc_write_begin(vp); 6011 } 6012 6013 void 6014 vop_setextattr_post(void *ap, int rc) 6015 { 6016 struct vop_setextattr_args *a; 6017 struct vnode *vp; 6018 6019 a = ap; 6020 vp = a->a_vp; 6021 vn_seqc_write_end(vp); 6022 if (!rc) 6023 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 6024 } 6025 6026 void 6027 vop_symlink_pre(void *ap) 6028 { 6029 struct vop_symlink_args *a; 6030 struct vnode *dvp; 6031 6032 a = ap; 6033 dvp = a->a_dvp; 6034 vn_seqc_write_begin(dvp); 6035 } 6036 6037 void 6038 vop_symlink_post(void *ap, int rc) 6039 { 6040 struct vop_symlink_args *a; 6041 struct vnode *dvp; 6042 6043 a = ap; 6044 dvp = a->a_dvp; 6045 vn_seqc_write_end(dvp); 6046 if (!rc) 6047 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6048 } 6049 6050 void 6051 vop_open_post(void *ap, int rc) 6052 { 6053 struct vop_open_args *a = ap; 6054 6055 if (!rc) 6056 VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN); 6057 } 6058 6059 void 6060 vop_close_post(void *ap, int rc) 6061 { 6062 struct vop_close_args *a = ap; 6063 6064 if (!rc && (a->a_cred != NOCRED || /* filter out revokes */ 6065 !VN_IS_DOOMED(a->a_vp))) { 6066 VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ? 6067 NOTE_CLOSE_WRITE : NOTE_CLOSE); 6068 } 6069 } 6070 6071 void 6072 vop_read_post(void *ap, int rc) 6073 { 6074 struct vop_read_args *a = ap; 6075 6076 if (!rc) 6077 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 6078 } 6079 6080 void 6081 vop_read_pgcache_post(void *ap, int rc) 6082 { 6083 struct vop_read_pgcache_args *a = ap; 6084 6085 if (!rc) 6086 VFS_KNOTE_UNLOCKED(a->a_vp, NOTE_READ); 6087 } 6088 6089 void 6090 vop_readdir_post(void *ap, int rc) 6091 { 6092 struct vop_readdir_args *a = ap; 6093 6094 if (!rc) 6095 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 6096 } 6097 6098 static struct knlist fs_knlist; 6099 6100 static void 6101 vfs_event_init(void *arg) 6102 { 6103 knlist_init_mtx(&fs_knlist, NULL); 6104 } 6105 /* XXX - correct order? */ 6106 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL); 6107 6108 void 6109 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused) 6110 { 6111 6112 KNOTE_UNLOCKED(&fs_knlist, event); 6113 } 6114 6115 static int filt_fsattach(struct knote *kn); 6116 static void filt_fsdetach(struct knote *kn); 6117 static int filt_fsevent(struct knote *kn, long hint); 6118 6119 struct filterops fs_filtops = { 6120 .f_isfd = 0, 6121 .f_attach = filt_fsattach, 6122 .f_detach = filt_fsdetach, 6123 .f_event = filt_fsevent 6124 }; 6125 6126 static int 6127 filt_fsattach(struct knote *kn) 6128 { 6129 6130 kn->kn_flags |= EV_CLEAR; 6131 knlist_add(&fs_knlist, kn, 0); 6132 return (0); 6133 } 6134 6135 static void 6136 filt_fsdetach(struct knote *kn) 6137 { 6138 6139 knlist_remove(&fs_knlist, kn, 0); 6140 } 6141 6142 static int 6143 filt_fsevent(struct knote *kn, long hint) 6144 { 6145 6146 kn->kn_fflags |= kn->kn_sfflags & hint; 6147 6148 return (kn->kn_fflags != 0); 6149 } 6150 6151 static int 6152 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS) 6153 { 6154 struct vfsidctl vc; 6155 int error; 6156 struct mount *mp; 6157 6158 error = SYSCTL_IN(req, &vc, sizeof(vc)); 6159 if (error) 6160 return (error); 6161 if (vc.vc_vers != VFS_CTL_VERS1) 6162 return (EINVAL); 6163 mp = vfs_getvfs(&vc.vc_fsid); 6164 if (mp == NULL) 6165 return (ENOENT); 6166 /* ensure that a specific sysctl goes to the right filesystem. */ 6167 if (strcmp(vc.vc_fstypename, "*") != 0 && 6168 strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) { 6169 vfs_rel(mp); 6170 return (EINVAL); 6171 } 6172 VCTLTOREQ(&vc, req); 6173 error = VFS_SYSCTL(mp, vc.vc_op, req); 6174 vfs_rel(mp); 6175 return (error); 6176 } 6177 6178 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_MPSAFE | CTLFLAG_WR, 6179 NULL, 0, sysctl_vfs_ctl, "", 6180 "Sysctl by fsid"); 6181 6182 /* 6183 * Function to initialize a va_filerev field sensibly. 6184 * XXX: Wouldn't a random number make a lot more sense ?? 6185 */ 6186 u_quad_t 6187 init_va_filerev(void) 6188 { 6189 struct bintime bt; 6190 6191 getbinuptime(&bt); 6192 return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL)); 6193 } 6194 6195 static int filt_vfsread(struct knote *kn, long hint); 6196 static int filt_vfswrite(struct knote *kn, long hint); 6197 static int filt_vfsvnode(struct knote *kn, long hint); 6198 static void filt_vfsdetach(struct knote *kn); 6199 static struct filterops vfsread_filtops = { 6200 .f_isfd = 1, 6201 .f_detach = filt_vfsdetach, 6202 .f_event = filt_vfsread 6203 }; 6204 static struct filterops vfswrite_filtops = { 6205 .f_isfd = 1, 6206 .f_detach = filt_vfsdetach, 6207 .f_event = filt_vfswrite 6208 }; 6209 static struct filterops vfsvnode_filtops = { 6210 .f_isfd = 1, 6211 .f_detach = filt_vfsdetach, 6212 .f_event = filt_vfsvnode 6213 }; 6214 6215 static void 6216 vfs_knllock(void *arg) 6217 { 6218 struct vnode *vp = arg; 6219 6220 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 6221 } 6222 6223 static void 6224 vfs_knlunlock(void *arg) 6225 { 6226 struct vnode *vp = arg; 6227 6228 VOP_UNLOCK(vp); 6229 } 6230 6231 static void 6232 vfs_knl_assert_lock(void *arg, int what) 6233 { 6234 #ifdef DEBUG_VFS_LOCKS 6235 struct vnode *vp = arg; 6236 6237 if (what == LA_LOCKED) 6238 ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked"); 6239 else 6240 ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked"); 6241 #endif 6242 } 6243 6244 int 6245 vfs_kqfilter(struct vop_kqfilter_args *ap) 6246 { 6247 struct vnode *vp = ap->a_vp; 6248 struct knote *kn = ap->a_kn; 6249 struct knlist *knl; 6250 6251 KASSERT(vp->v_type != VFIFO || (kn->kn_filter != EVFILT_READ && 6252 kn->kn_filter != EVFILT_WRITE), 6253 ("READ/WRITE filter on a FIFO leaked through")); 6254 switch (kn->kn_filter) { 6255 case EVFILT_READ: 6256 kn->kn_fop = &vfsread_filtops; 6257 break; 6258 case EVFILT_WRITE: 6259 kn->kn_fop = &vfswrite_filtops; 6260 break; 6261 case EVFILT_VNODE: 6262 kn->kn_fop = &vfsvnode_filtops; 6263 break; 6264 default: 6265 return (EINVAL); 6266 } 6267 6268 kn->kn_hook = (caddr_t)vp; 6269 6270 v_addpollinfo(vp); 6271 if (vp->v_pollinfo == NULL) 6272 return (ENOMEM); 6273 knl = &vp->v_pollinfo->vpi_selinfo.si_note; 6274 vhold(vp); 6275 knlist_add(knl, kn, 0); 6276 6277 return (0); 6278 } 6279 6280 /* 6281 * Detach knote from vnode 6282 */ 6283 static void 6284 filt_vfsdetach(struct knote *kn) 6285 { 6286 struct vnode *vp = (struct vnode *)kn->kn_hook; 6287 6288 KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo")); 6289 knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0); 6290 vdrop(vp); 6291 } 6292 6293 /*ARGSUSED*/ 6294 static int 6295 filt_vfsread(struct knote *kn, long hint) 6296 { 6297 struct vnode *vp = (struct vnode *)kn->kn_hook; 6298 off_t size; 6299 int res; 6300 6301 /* 6302 * filesystem is gone, so set the EOF flag and schedule 6303 * the knote for deletion. 6304 */ 6305 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6306 VI_LOCK(vp); 6307 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6308 VI_UNLOCK(vp); 6309 return (1); 6310 } 6311 6312 if (vn_getsize_locked(vp, &size, curthread->td_ucred) != 0) 6313 return (0); 6314 6315 VI_LOCK(vp); 6316 kn->kn_data = size - kn->kn_fp->f_offset; 6317 res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 || kn->kn_data != 0; 6318 VI_UNLOCK(vp); 6319 return (res); 6320 } 6321 6322 /*ARGSUSED*/ 6323 static int 6324 filt_vfswrite(struct knote *kn, long hint) 6325 { 6326 struct vnode *vp = (struct vnode *)kn->kn_hook; 6327 6328 VI_LOCK(vp); 6329 6330 /* 6331 * filesystem is gone, so set the EOF flag and schedule 6332 * the knote for deletion. 6333 */ 6334 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) 6335 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6336 6337 kn->kn_data = 0; 6338 VI_UNLOCK(vp); 6339 return (1); 6340 } 6341 6342 static int 6343 filt_vfsvnode(struct knote *kn, long hint) 6344 { 6345 struct vnode *vp = (struct vnode *)kn->kn_hook; 6346 int res; 6347 6348 VI_LOCK(vp); 6349 if (kn->kn_sfflags & hint) 6350 kn->kn_fflags |= hint; 6351 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6352 kn->kn_flags |= EV_EOF; 6353 VI_UNLOCK(vp); 6354 return (1); 6355 } 6356 res = (kn->kn_fflags != 0); 6357 VI_UNLOCK(vp); 6358 return (res); 6359 } 6360 6361 /* 6362 * Returns whether the directory is empty or not. 6363 * If it is empty, the return value is 0; otherwise 6364 * the return value is an error value (which may 6365 * be ENOTEMPTY). 6366 */ 6367 int 6368 vfs_emptydir(struct vnode *vp) 6369 { 6370 struct uio uio; 6371 struct iovec iov; 6372 struct dirent *dirent, *dp, *endp; 6373 int error, eof; 6374 6375 error = 0; 6376 eof = 0; 6377 6378 ASSERT_VOP_LOCKED(vp, "vfs_emptydir"); 6379 VNASSERT(vp->v_type == VDIR, vp, ("vp is not a directory")); 6380 6381 dirent = malloc(sizeof(struct dirent), M_TEMP, M_WAITOK); 6382 iov.iov_base = dirent; 6383 iov.iov_len = sizeof(struct dirent); 6384 6385 uio.uio_iov = &iov; 6386 uio.uio_iovcnt = 1; 6387 uio.uio_offset = 0; 6388 uio.uio_resid = sizeof(struct dirent); 6389 uio.uio_segflg = UIO_SYSSPACE; 6390 uio.uio_rw = UIO_READ; 6391 uio.uio_td = curthread; 6392 6393 while (eof == 0 && error == 0) { 6394 error = VOP_READDIR(vp, &uio, curthread->td_ucred, &eof, 6395 NULL, NULL); 6396 if (error != 0) 6397 break; 6398 endp = (void *)((uint8_t *)dirent + 6399 sizeof(struct dirent) - uio.uio_resid); 6400 for (dp = dirent; dp < endp; 6401 dp = (void *)((uint8_t *)dp + GENERIC_DIRSIZ(dp))) { 6402 if (dp->d_type == DT_WHT) 6403 continue; 6404 if (dp->d_namlen == 0) 6405 continue; 6406 if (dp->d_type != DT_DIR && 6407 dp->d_type != DT_UNKNOWN) { 6408 error = ENOTEMPTY; 6409 break; 6410 } 6411 if (dp->d_namlen > 2) { 6412 error = ENOTEMPTY; 6413 break; 6414 } 6415 if (dp->d_namlen == 1 && 6416 dp->d_name[0] != '.') { 6417 error = ENOTEMPTY; 6418 break; 6419 } 6420 if (dp->d_namlen == 2 && 6421 dp->d_name[1] != '.') { 6422 error = ENOTEMPTY; 6423 break; 6424 } 6425 uio.uio_resid = sizeof(struct dirent); 6426 } 6427 } 6428 free(dirent, M_TEMP); 6429 return (error); 6430 } 6431 6432 int 6433 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off) 6434 { 6435 int error; 6436 6437 if (dp->d_reclen > ap->a_uio->uio_resid) 6438 return (ENAMETOOLONG); 6439 error = uiomove(dp, dp->d_reclen, ap->a_uio); 6440 if (error) { 6441 if (ap->a_ncookies != NULL) { 6442 if (ap->a_cookies != NULL) 6443 free(ap->a_cookies, M_TEMP); 6444 ap->a_cookies = NULL; 6445 *ap->a_ncookies = 0; 6446 } 6447 return (error); 6448 } 6449 if (ap->a_ncookies == NULL) 6450 return (0); 6451 6452 KASSERT(ap->a_cookies, 6453 ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!")); 6454 6455 *ap->a_cookies = realloc(*ap->a_cookies, 6456 (*ap->a_ncookies + 1) * sizeof(uint64_t), M_TEMP, M_WAITOK | M_ZERO); 6457 (*ap->a_cookies)[*ap->a_ncookies] = off; 6458 *ap->a_ncookies += 1; 6459 return (0); 6460 } 6461 6462 /* 6463 * The purpose of this routine is to remove granularity from accmode_t, 6464 * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE, 6465 * VADMIN and VAPPEND. 6466 * 6467 * If it returns 0, the caller is supposed to continue with the usual 6468 * access checks using 'accmode' as modified by this routine. If it 6469 * returns nonzero value, the caller is supposed to return that value 6470 * as errno. 6471 * 6472 * Note that after this routine runs, accmode may be zero. 6473 */ 6474 int 6475 vfs_unixify_accmode(accmode_t *accmode) 6476 { 6477 /* 6478 * There is no way to specify explicit "deny" rule using 6479 * file mode or POSIX.1e ACLs. 6480 */ 6481 if (*accmode & VEXPLICIT_DENY) { 6482 *accmode = 0; 6483 return (0); 6484 } 6485 6486 /* 6487 * None of these can be translated into usual access bits. 6488 * Also, the common case for NFSv4 ACLs is to not contain 6489 * either of these bits. Caller should check for VWRITE 6490 * on the containing directory instead. 6491 */ 6492 if (*accmode & (VDELETE_CHILD | VDELETE)) 6493 return (EPERM); 6494 6495 if (*accmode & VADMIN_PERMS) { 6496 *accmode &= ~VADMIN_PERMS; 6497 *accmode |= VADMIN; 6498 } 6499 6500 /* 6501 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL 6502 * or VSYNCHRONIZE using file mode or POSIX.1e ACL. 6503 */ 6504 *accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE); 6505 6506 return (0); 6507 } 6508 6509 /* 6510 * Clear out a doomed vnode (if any) and replace it with a new one as long 6511 * as the fs is not being unmounted. Return the root vnode to the caller. 6512 */ 6513 static int __noinline 6514 vfs_cache_root_fallback(struct mount *mp, int flags, struct vnode **vpp) 6515 { 6516 struct vnode *vp; 6517 int error; 6518 6519 restart: 6520 if (mp->mnt_rootvnode != NULL) { 6521 MNT_ILOCK(mp); 6522 vp = mp->mnt_rootvnode; 6523 if (vp != NULL) { 6524 if (!VN_IS_DOOMED(vp)) { 6525 vrefact(vp); 6526 MNT_IUNLOCK(mp); 6527 error = vn_lock(vp, flags); 6528 if (error == 0) { 6529 *vpp = vp; 6530 return (0); 6531 } 6532 vrele(vp); 6533 goto restart; 6534 } 6535 /* 6536 * Clear the old one. 6537 */ 6538 mp->mnt_rootvnode = NULL; 6539 } 6540 MNT_IUNLOCK(mp); 6541 if (vp != NULL) { 6542 vfs_op_barrier_wait(mp); 6543 vrele(vp); 6544 } 6545 } 6546 error = VFS_CACHEDROOT(mp, flags, vpp); 6547 if (error != 0) 6548 return (error); 6549 if (mp->mnt_vfs_ops == 0) { 6550 MNT_ILOCK(mp); 6551 if (mp->mnt_vfs_ops != 0) { 6552 MNT_IUNLOCK(mp); 6553 return (0); 6554 } 6555 if (mp->mnt_rootvnode == NULL) { 6556 vrefact(*vpp); 6557 mp->mnt_rootvnode = *vpp; 6558 } else { 6559 if (mp->mnt_rootvnode != *vpp) { 6560 if (!VN_IS_DOOMED(mp->mnt_rootvnode)) { 6561 panic("%s: mismatch between vnode returned " 6562 " by VFS_CACHEDROOT and the one cached " 6563 " (%p != %p)", 6564 __func__, *vpp, mp->mnt_rootvnode); 6565 } 6566 } 6567 } 6568 MNT_IUNLOCK(mp); 6569 } 6570 return (0); 6571 } 6572 6573 int 6574 vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp) 6575 { 6576 struct mount_pcpu *mpcpu; 6577 struct vnode *vp; 6578 int error; 6579 6580 if (!vfs_op_thread_enter(mp, mpcpu)) 6581 return (vfs_cache_root_fallback(mp, flags, vpp)); 6582 vp = atomic_load_ptr(&mp->mnt_rootvnode); 6583 if (vp == NULL || VN_IS_DOOMED(vp)) { 6584 vfs_op_thread_exit(mp, mpcpu); 6585 return (vfs_cache_root_fallback(mp, flags, vpp)); 6586 } 6587 vrefact(vp); 6588 vfs_op_thread_exit(mp, mpcpu); 6589 error = vn_lock(vp, flags); 6590 if (error != 0) { 6591 vrele(vp); 6592 return (vfs_cache_root_fallback(mp, flags, vpp)); 6593 } 6594 *vpp = vp; 6595 return (0); 6596 } 6597 6598 struct vnode * 6599 vfs_cache_root_clear(struct mount *mp) 6600 { 6601 struct vnode *vp; 6602 6603 /* 6604 * ops > 0 guarantees there is nobody who can see this vnode 6605 */ 6606 MPASS(mp->mnt_vfs_ops > 0); 6607 vp = mp->mnt_rootvnode; 6608 if (vp != NULL) 6609 vn_seqc_write_begin(vp); 6610 mp->mnt_rootvnode = NULL; 6611 return (vp); 6612 } 6613 6614 void 6615 vfs_cache_root_set(struct mount *mp, struct vnode *vp) 6616 { 6617 6618 MPASS(mp->mnt_vfs_ops > 0); 6619 vrefact(vp); 6620 mp->mnt_rootvnode = vp; 6621 } 6622 6623 /* 6624 * These are helper functions for filesystems to traverse all 6625 * their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h. 6626 * 6627 * This interface replaces MNT_VNODE_FOREACH. 6628 */ 6629 6630 struct vnode * 6631 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp) 6632 { 6633 struct vnode *vp; 6634 6635 maybe_yield(); 6636 MNT_ILOCK(mp); 6637 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6638 for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL; 6639 vp = TAILQ_NEXT(vp, v_nmntvnodes)) { 6640 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6641 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6642 continue; 6643 VI_LOCK(vp); 6644 if (VN_IS_DOOMED(vp)) { 6645 VI_UNLOCK(vp); 6646 continue; 6647 } 6648 break; 6649 } 6650 if (vp == NULL) { 6651 __mnt_vnode_markerfree_all(mvp, mp); 6652 /* MNT_IUNLOCK(mp); -- done in above function */ 6653 mtx_assert(MNT_MTX(mp), MA_NOTOWNED); 6654 return (NULL); 6655 } 6656 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6657 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6658 MNT_IUNLOCK(mp); 6659 return (vp); 6660 } 6661 6662 struct vnode * 6663 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp) 6664 { 6665 struct vnode *vp; 6666 6667 *mvp = vn_alloc_marker(mp); 6668 MNT_ILOCK(mp); 6669 MNT_REF(mp); 6670 6671 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 6672 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6673 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6674 continue; 6675 VI_LOCK(vp); 6676 if (VN_IS_DOOMED(vp)) { 6677 VI_UNLOCK(vp); 6678 continue; 6679 } 6680 break; 6681 } 6682 if (vp == NULL) { 6683 MNT_REL(mp); 6684 MNT_IUNLOCK(mp); 6685 vn_free_marker(*mvp); 6686 *mvp = NULL; 6687 return (NULL); 6688 } 6689 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6690 MNT_IUNLOCK(mp); 6691 return (vp); 6692 } 6693 6694 void 6695 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp) 6696 { 6697 6698 if (*mvp == NULL) { 6699 MNT_IUNLOCK(mp); 6700 return; 6701 } 6702 6703 mtx_assert(MNT_MTX(mp), MA_OWNED); 6704 6705 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6706 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6707 MNT_REL(mp); 6708 MNT_IUNLOCK(mp); 6709 vn_free_marker(*mvp); 6710 *mvp = NULL; 6711 } 6712 6713 /* 6714 * These are helper functions for filesystems to traverse their 6715 * lazy vnodes. See MNT_VNODE_FOREACH_LAZY() in sys/mount.h 6716 */ 6717 static void 6718 mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 6719 { 6720 6721 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6722 6723 MNT_ILOCK(mp); 6724 MNT_REL(mp); 6725 MNT_IUNLOCK(mp); 6726 vn_free_marker(*mvp); 6727 *mvp = NULL; 6728 } 6729 6730 /* 6731 * Relock the mp mount vnode list lock with the vp vnode interlock in the 6732 * conventional lock order during mnt_vnode_next_lazy iteration. 6733 * 6734 * On entry, the mount vnode list lock is held and the vnode interlock is not. 6735 * The list lock is dropped and reacquired. On success, both locks are held. 6736 * On failure, the mount vnode list lock is held but the vnode interlock is 6737 * not, and the procedure may have yielded. 6738 */ 6739 static bool 6740 mnt_vnode_next_lazy_relock(struct vnode *mvp, struct mount *mp, 6741 struct vnode *vp) 6742 { 6743 6744 VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER && 6745 TAILQ_NEXT(mvp, v_lazylist) != NULL, mvp, 6746 ("%s: bad marker", __func__)); 6747 VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp, 6748 ("%s: inappropriate vnode", __func__)); 6749 ASSERT_VI_UNLOCKED(vp, __func__); 6750 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 6751 6752 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, mvp, v_lazylist); 6753 TAILQ_INSERT_BEFORE(vp, mvp, v_lazylist); 6754 6755 /* 6756 * Note we may be racing against vdrop which transitioned the hold 6757 * count to 0 and now waits for the ->mnt_listmtx lock. This is fine, 6758 * if we are the only user after we get the interlock we will just 6759 * vdrop. 6760 */ 6761 vhold(vp); 6762 mtx_unlock(&mp->mnt_listmtx); 6763 VI_LOCK(vp); 6764 if (VN_IS_DOOMED(vp)) { 6765 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 6766 goto out_lost; 6767 } 6768 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 6769 /* 6770 * There is nothing to do if we are the last user. 6771 */ 6772 if (!refcount_release_if_not_last(&vp->v_holdcnt)) 6773 goto out_lost; 6774 mtx_lock(&mp->mnt_listmtx); 6775 return (true); 6776 out_lost: 6777 vdropl(vp); 6778 maybe_yield(); 6779 mtx_lock(&mp->mnt_listmtx); 6780 return (false); 6781 } 6782 6783 static struct vnode * 6784 mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6785 void *cbarg) 6786 { 6787 struct vnode *vp; 6788 6789 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 6790 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6791 restart: 6792 vp = TAILQ_NEXT(*mvp, v_lazylist); 6793 while (vp != NULL) { 6794 if (vp->v_type == VMARKER) { 6795 vp = TAILQ_NEXT(vp, v_lazylist); 6796 continue; 6797 } 6798 /* 6799 * See if we want to process the vnode. Note we may encounter a 6800 * long string of vnodes we don't care about and hog the list 6801 * as a result. Check for it and requeue the marker. 6802 */ 6803 VNPASS(!VN_IS_DOOMED(vp), vp); 6804 if (!cb(vp, cbarg)) { 6805 if (!should_yield()) { 6806 vp = TAILQ_NEXT(vp, v_lazylist); 6807 continue; 6808 } 6809 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, 6810 v_lazylist); 6811 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, 6812 v_lazylist); 6813 mtx_unlock(&mp->mnt_listmtx); 6814 kern_yield(PRI_USER); 6815 mtx_lock(&mp->mnt_listmtx); 6816 goto restart; 6817 } 6818 /* 6819 * Try-lock because this is the wrong lock order. 6820 */ 6821 if (!VI_TRYLOCK(vp) && 6822 !mnt_vnode_next_lazy_relock(*mvp, mp, vp)) 6823 goto restart; 6824 KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp)); 6825 KASSERT(vp->v_mount == mp || vp->v_mount == NULL, 6826 ("alien vnode on the lazy list %p %p", vp, mp)); 6827 VNPASS(vp->v_mount == mp, vp); 6828 VNPASS(!VN_IS_DOOMED(vp), vp); 6829 break; 6830 } 6831 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 6832 6833 /* Check if we are done */ 6834 if (vp == NULL) { 6835 mtx_unlock(&mp->mnt_listmtx); 6836 mnt_vnode_markerfree_lazy(mvp, mp); 6837 return (NULL); 6838 } 6839 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, v_lazylist); 6840 mtx_unlock(&mp->mnt_listmtx); 6841 ASSERT_VI_LOCKED(vp, "lazy iter"); 6842 return (vp); 6843 } 6844 6845 struct vnode * 6846 __mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6847 void *cbarg) 6848 { 6849 6850 maybe_yield(); 6851 mtx_lock(&mp->mnt_listmtx); 6852 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 6853 } 6854 6855 struct vnode * 6856 __mnt_vnode_first_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 6857 void *cbarg) 6858 { 6859 struct vnode *vp; 6860 6861 if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist)) 6862 return (NULL); 6863 6864 *mvp = vn_alloc_marker(mp); 6865 MNT_ILOCK(mp); 6866 MNT_REF(mp); 6867 MNT_IUNLOCK(mp); 6868 6869 mtx_lock(&mp->mnt_listmtx); 6870 vp = TAILQ_FIRST(&mp->mnt_lazyvnodelist); 6871 if (vp == NULL) { 6872 mtx_unlock(&mp->mnt_listmtx); 6873 mnt_vnode_markerfree_lazy(mvp, mp); 6874 return (NULL); 6875 } 6876 TAILQ_INSERT_BEFORE(vp, *mvp, v_lazylist); 6877 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 6878 } 6879 6880 void 6881 __mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 6882 { 6883 6884 if (*mvp == NULL) 6885 return; 6886 6887 mtx_lock(&mp->mnt_listmtx); 6888 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 6889 mtx_unlock(&mp->mnt_listmtx); 6890 mnt_vnode_markerfree_lazy(mvp, mp); 6891 } 6892 6893 int 6894 vn_dir_check_exec(struct vnode *vp, struct componentname *cnp) 6895 { 6896 6897 if ((cnp->cn_flags & NOEXECCHECK) != 0) { 6898 cnp->cn_flags &= ~NOEXECCHECK; 6899 return (0); 6900 } 6901 6902 return (VOP_ACCESS(vp, VEXEC, cnp->cn_cred, curthread)); 6903 } 6904 6905 /* 6906 * Do not use this variant unless you have means other than the hold count 6907 * to prevent the vnode from getting freed. 6908 */ 6909 void 6910 vn_seqc_write_begin_locked(struct vnode *vp) 6911 { 6912 6913 ASSERT_VI_LOCKED(vp, __func__); 6914 VNPASS(vp->v_holdcnt > 0, vp); 6915 VNPASS(vp->v_seqc_users >= 0, vp); 6916 vp->v_seqc_users++; 6917 if (vp->v_seqc_users == 1) 6918 seqc_sleepable_write_begin(&vp->v_seqc); 6919 } 6920 6921 void 6922 vn_seqc_write_begin(struct vnode *vp) 6923 { 6924 6925 VI_LOCK(vp); 6926 vn_seqc_write_begin_locked(vp); 6927 VI_UNLOCK(vp); 6928 } 6929 6930 void 6931 vn_seqc_write_end_locked(struct vnode *vp) 6932 { 6933 6934 ASSERT_VI_LOCKED(vp, __func__); 6935 VNPASS(vp->v_seqc_users > 0, vp); 6936 vp->v_seqc_users--; 6937 if (vp->v_seqc_users == 0) 6938 seqc_sleepable_write_end(&vp->v_seqc); 6939 } 6940 6941 void 6942 vn_seqc_write_end(struct vnode *vp) 6943 { 6944 6945 VI_LOCK(vp); 6946 vn_seqc_write_end_locked(vp); 6947 VI_UNLOCK(vp); 6948 } 6949 6950 /* 6951 * Special case handling for allocating and freeing vnodes. 6952 * 6953 * The counter remains unchanged on free so that a doomed vnode will 6954 * keep testing as in modify as long as it is accessible with SMR. 6955 */ 6956 static void 6957 vn_seqc_init(struct vnode *vp) 6958 { 6959 6960 vp->v_seqc = 0; 6961 vp->v_seqc_users = 0; 6962 } 6963 6964 static void 6965 vn_seqc_write_end_free(struct vnode *vp) 6966 { 6967 6968 VNPASS(seqc_in_modify(vp->v_seqc), vp); 6969 VNPASS(vp->v_seqc_users == 1, vp); 6970 } 6971 6972 void 6973 vn_irflag_set_locked(struct vnode *vp, short toset) 6974 { 6975 short flags; 6976 6977 ASSERT_VI_LOCKED(vp, __func__); 6978 flags = vn_irflag_read(vp); 6979 VNASSERT((flags & toset) == 0, vp, 6980 ("%s: some of the passed flags already set (have %d, passed %d)\n", 6981 __func__, flags, toset)); 6982 atomic_store_short(&vp->v_irflag, flags | toset); 6983 } 6984 6985 void 6986 vn_irflag_set(struct vnode *vp, short toset) 6987 { 6988 6989 VI_LOCK(vp); 6990 vn_irflag_set_locked(vp, toset); 6991 VI_UNLOCK(vp); 6992 } 6993 6994 void 6995 vn_irflag_set_cond_locked(struct vnode *vp, short toset) 6996 { 6997 short flags; 6998 6999 ASSERT_VI_LOCKED(vp, __func__); 7000 flags = vn_irflag_read(vp); 7001 atomic_store_short(&vp->v_irflag, flags | toset); 7002 } 7003 7004 void 7005 vn_irflag_set_cond(struct vnode *vp, short toset) 7006 { 7007 7008 VI_LOCK(vp); 7009 vn_irflag_set_cond_locked(vp, toset); 7010 VI_UNLOCK(vp); 7011 } 7012 7013 void 7014 vn_irflag_unset_locked(struct vnode *vp, short tounset) 7015 { 7016 short flags; 7017 7018 ASSERT_VI_LOCKED(vp, __func__); 7019 flags = vn_irflag_read(vp); 7020 VNASSERT((flags & tounset) == tounset, vp, 7021 ("%s: some of the passed flags not set (have %d, passed %d)\n", 7022 __func__, flags, tounset)); 7023 atomic_store_short(&vp->v_irflag, flags & ~tounset); 7024 } 7025 7026 void 7027 vn_irflag_unset(struct vnode *vp, short tounset) 7028 { 7029 7030 VI_LOCK(vp); 7031 vn_irflag_unset_locked(vp, tounset); 7032 VI_UNLOCK(vp); 7033 } 7034 7035 int 7036 vn_getsize_locked(struct vnode *vp, off_t *size, struct ucred *cred) 7037 { 7038 struct vattr vattr; 7039 int error; 7040 7041 ASSERT_VOP_LOCKED(vp, __func__); 7042 error = VOP_GETATTR(vp, &vattr, cred); 7043 if (__predict_true(error == 0)) { 7044 if (vattr.va_size <= OFF_MAX) 7045 *size = vattr.va_size; 7046 else 7047 error = EFBIG; 7048 } 7049 return (error); 7050 } 7051 7052 int 7053 vn_getsize(struct vnode *vp, off_t *size, struct ucred *cred) 7054 { 7055 int error; 7056 7057 VOP_LOCK(vp, LK_SHARED); 7058 error = vn_getsize_locked(vp, size, cred); 7059 VOP_UNLOCK(vp); 7060 return (error); 7061 } 7062 7063 #ifdef INVARIANTS 7064 void 7065 vn_set_state_validate(struct vnode *vp, enum vstate state) 7066 { 7067 7068 switch (vp->v_state) { 7069 case VSTATE_UNINITIALIZED: 7070 switch (state) { 7071 case VSTATE_CONSTRUCTED: 7072 case VSTATE_DESTROYING: 7073 return; 7074 default: 7075 break; 7076 } 7077 break; 7078 case VSTATE_CONSTRUCTED: 7079 ASSERT_VOP_ELOCKED(vp, __func__); 7080 switch (state) { 7081 case VSTATE_DESTROYING: 7082 return; 7083 default: 7084 break; 7085 } 7086 break; 7087 case VSTATE_DESTROYING: 7088 ASSERT_VOP_ELOCKED(vp, __func__); 7089 switch (state) { 7090 case VSTATE_DEAD: 7091 return; 7092 default: 7093 break; 7094 } 7095 break; 7096 case VSTATE_DEAD: 7097 switch (state) { 7098 case VSTATE_UNINITIALIZED: 7099 return; 7100 default: 7101 break; 7102 } 7103 break; 7104 } 7105 7106 vn_printf(vp, "invalid state transition %d -> %d\n", vp->v_state, state); 7107 panic("invalid state transition %d -> %d\n", vp->v_state, state); 7108 } 7109 #endif 7110