1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 /* 38 * External virtual filesystem routines 39 */ 40 41 #include <sys/cdefs.h> 42 #include "opt_ddb.h" 43 #include "opt_watchdog.h" 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/asan.h> 48 #include <sys/bio.h> 49 #include <sys/buf.h> 50 #include <sys/capsicum.h> 51 #include <sys/condvar.h> 52 #include <sys/conf.h> 53 #include <sys/counter.h> 54 #include <sys/dirent.h> 55 #include <sys/event.h> 56 #include <sys/eventhandler.h> 57 #include <sys/extattr.h> 58 #include <sys/file.h> 59 #include <sys/fcntl.h> 60 #include <sys/jail.h> 61 #include <sys/kdb.h> 62 #include <sys/kernel.h> 63 #include <sys/kthread.h> 64 #include <sys/ktr.h> 65 #include <sys/limits.h> 66 #include <sys/lockf.h> 67 #include <sys/malloc.h> 68 #include <sys/mount.h> 69 #include <sys/namei.h> 70 #include <sys/pctrie.h> 71 #include <sys/priv.h> 72 #include <sys/reboot.h> 73 #include <sys/refcount.h> 74 #include <sys/rwlock.h> 75 #include <sys/sched.h> 76 #include <sys/sleepqueue.h> 77 #include <sys/smr.h> 78 #include <sys/smp.h> 79 #include <sys/stat.h> 80 #include <sys/sysctl.h> 81 #include <sys/syslog.h> 82 #include <sys/vmmeter.h> 83 #include <sys/vnode.h> 84 #include <sys/watchdog.h> 85 86 #include <machine/stdarg.h> 87 88 #include <security/mac/mac_framework.h> 89 90 #include <vm/vm.h> 91 #include <vm/vm_object.h> 92 #include <vm/vm_extern.h> 93 #include <vm/pmap.h> 94 #include <vm/vm_map.h> 95 #include <vm/vm_page.h> 96 #include <vm/vm_kern.h> 97 #include <vm/vnode_pager.h> 98 #include <vm/uma.h> 99 100 #if defined(DEBUG_VFS_LOCKS) && (!defined(INVARIANTS) || !defined(WITNESS)) 101 #error DEBUG_VFS_LOCKS requires INVARIANTS and WITNESS 102 #endif 103 104 #ifdef DDB 105 #include <ddb/ddb.h> 106 #endif 107 108 static void delmntque(struct vnode *vp); 109 static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, 110 int slpflag, int slptimeo); 111 static void syncer_shutdown(void *arg, int howto); 112 static int vtryrecycle(struct vnode *vp, bool isvnlru); 113 static void v_init_counters(struct vnode *); 114 static void vn_seqc_init(struct vnode *); 115 static void vn_seqc_write_end_free(struct vnode *vp); 116 static void vgonel(struct vnode *); 117 static bool vhold_recycle_free(struct vnode *); 118 static void vdropl_recycle(struct vnode *vp); 119 static void vdrop_recycle(struct vnode *vp); 120 static void vfs_knllock(void *arg); 121 static void vfs_knlunlock(void *arg); 122 static void vfs_knl_assert_lock(void *arg, int what); 123 static void destroy_vpollinfo(struct vpollinfo *vi); 124 static int v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 125 daddr_t startlbn, daddr_t endlbn); 126 static void vnlru_recalc(void); 127 128 static SYSCTL_NODE(_vfs, OID_AUTO, vnode, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 129 "vnode configuration and statistics"); 130 static SYSCTL_NODE(_vfs_vnode, OID_AUTO, param, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 131 "vnode configuration"); 132 static SYSCTL_NODE(_vfs_vnode, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 133 "vnode statistics"); 134 static SYSCTL_NODE(_vfs_vnode, OID_AUTO, vnlru, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 135 "vnode recycling"); 136 137 /* 138 * Number of vnodes in existence. Increased whenever getnewvnode() 139 * allocates a new vnode, decreased in vdropl() for VIRF_DOOMED vnode. 140 */ 141 static u_long __exclusive_cache_line numvnodes; 142 143 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0, 144 "Number of vnodes in existence (legacy)"); 145 SYSCTL_ULONG(_vfs_vnode_stats, OID_AUTO, count, CTLFLAG_RD, &numvnodes, 0, 146 "Number of vnodes in existence"); 147 148 static counter_u64_t vnodes_created; 149 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created, 150 "Number of vnodes created by getnewvnode (legacy)"); 151 SYSCTL_COUNTER_U64(_vfs_vnode_stats, OID_AUTO, created, CTLFLAG_RD, &vnodes_created, 152 "Number of vnodes created by getnewvnode"); 153 154 /* 155 * Conversion tables for conversion from vnode types to inode formats 156 * and back. 157 */ 158 __enum_uint8(vtype) iftovt_tab[16] = { 159 VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON, 160 VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON 161 }; 162 int vttoif_tab[10] = { 163 0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK, 164 S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT 165 }; 166 167 /* 168 * List of allocates vnodes in the system. 169 */ 170 static TAILQ_HEAD(freelst, vnode) vnode_list; 171 static struct vnode *vnode_list_free_marker; 172 static struct vnode *vnode_list_reclaim_marker; 173 174 /* 175 * "Free" vnode target. Free vnodes are rarely completely free, but are 176 * just ones that are cheap to recycle. Usually they are for files which 177 * have been stat'd but not read; these usually have inode and namecache 178 * data attached to them. This target is the preferred minimum size of a 179 * sub-cache consisting mostly of such files. The system balances the size 180 * of this sub-cache with its complement to try to prevent either from 181 * thrashing while the other is relatively inactive. The targets express 182 * a preference for the best balance. 183 * 184 * "Above" this target there are 2 further targets (watermarks) related 185 * to recyling of free vnodes. In the best-operating case, the cache is 186 * exactly full, the free list has size between vlowat and vhiwat above the 187 * free target, and recycling from it and normal use maintains this state. 188 * Sometimes the free list is below vlowat or even empty, but this state 189 * is even better for immediate use provided the cache is not full. 190 * Otherwise, vnlru_proc() runs to reclaim enough vnodes (usually non-free 191 * ones) to reach one of these states. The watermarks are currently hard- 192 * coded as 4% and 9% of the available space higher. These and the default 193 * of 25% for wantfreevnodes are too large if the memory size is large. 194 * E.g., 9% of 75% of MAXVNODES is more than 566000 vnodes to reclaim 195 * whenever vnlru_proc() becomes active. 196 */ 197 static long wantfreevnodes; 198 static long __exclusive_cache_line freevnodes; 199 static long freevnodes_old; 200 201 static u_long recycles_count; 202 SYSCTL_ULONG(_vfs, OID_AUTO, recycles, CTLFLAG_RD | CTLFLAG_STATS, &recycles_count, 0, 203 "Number of vnodes recycled to meet vnode cache targets (legacy)"); 204 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, recycles, CTLFLAG_RD | CTLFLAG_STATS, 205 &recycles_count, 0, 206 "Number of vnodes recycled to meet vnode cache targets"); 207 208 static u_long recycles_free_count; 209 SYSCTL_ULONG(_vfs, OID_AUTO, recycles_free, CTLFLAG_RD | CTLFLAG_STATS, 210 &recycles_free_count, 0, 211 "Number of free vnodes recycled to meet vnode cache targets (legacy)"); 212 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, recycles_free, CTLFLAG_RD | CTLFLAG_STATS, 213 &recycles_free_count, 0, 214 "Number of free vnodes recycled to meet vnode cache targets"); 215 216 static counter_u64_t direct_recycles_free_count; 217 SYSCTL_COUNTER_U64(_vfs_vnode_vnlru, OID_AUTO, direct_recycles_free, CTLFLAG_RD, 218 &direct_recycles_free_count, 219 "Number of free vnodes recycled by vn_alloc callers to meet vnode cache targets"); 220 221 static counter_u64_t vnode_skipped_requeues; 222 SYSCTL_COUNTER_U64(_vfs_vnode_stats, OID_AUTO, skipped_requeues, CTLFLAG_RD, &vnode_skipped_requeues, 223 "Number of times LRU requeue was skipped due to lock contention"); 224 225 static __read_mostly bool vnode_can_skip_requeue; 226 SYSCTL_BOOL(_vfs_vnode_param, OID_AUTO, can_skip_requeue, CTLFLAG_RW, 227 &vnode_can_skip_requeue, 0, "Is LRU requeue skippable"); 228 229 static u_long deferred_inact; 230 SYSCTL_ULONG(_vfs, OID_AUTO, deferred_inact, CTLFLAG_RD, 231 &deferred_inact, 0, "Number of times inactive processing was deferred"); 232 233 /* To keep more than one thread at a time from running vfs_getnewfsid */ 234 static struct mtx mntid_mtx; 235 236 /* 237 * Lock for any access to the following: 238 * vnode_list 239 * numvnodes 240 * freevnodes 241 */ 242 static struct mtx __exclusive_cache_line vnode_list_mtx; 243 244 /* Publicly exported FS */ 245 struct nfs_public nfs_pub; 246 247 static uma_zone_t buf_trie_zone; 248 static smr_t buf_trie_smr; 249 250 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */ 251 static uma_zone_t vnode_zone; 252 MALLOC_DEFINE(M_VNODEPOLL, "VN POLL", "vnode poll"); 253 254 __read_frequently smr_t vfs_smr; 255 256 /* 257 * The workitem queue. 258 * 259 * It is useful to delay writes of file data and filesystem metadata 260 * for tens of seconds so that quickly created and deleted files need 261 * not waste disk bandwidth being created and removed. To realize this, 262 * we append vnodes to a "workitem" queue. When running with a soft 263 * updates implementation, most pending metadata dependencies should 264 * not wait for more than a few seconds. Thus, mounted on block devices 265 * are delayed only about a half the time that file data is delayed. 266 * Similarly, directory updates are more critical, so are only delayed 267 * about a third the time that file data is delayed. Thus, there are 268 * SYNCER_MAXDELAY queues that are processed round-robin at a rate of 269 * one each second (driven off the filesystem syncer process). The 270 * syncer_delayno variable indicates the next queue that is to be processed. 271 * Items that need to be processed soon are placed in this queue: 272 * 273 * syncer_workitem_pending[syncer_delayno] 274 * 275 * A delay of fifteen seconds is done by placing the request fifteen 276 * entries later in the queue: 277 * 278 * syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask] 279 * 280 */ 281 static int syncer_delayno; 282 static long syncer_mask; 283 LIST_HEAD(synclist, bufobj); 284 static struct synclist *syncer_workitem_pending; 285 /* 286 * The sync_mtx protects: 287 * bo->bo_synclist 288 * sync_vnode_count 289 * syncer_delayno 290 * syncer_state 291 * syncer_workitem_pending 292 * syncer_worklist_len 293 * rushjob 294 */ 295 static struct mtx sync_mtx; 296 static struct cv sync_wakeup; 297 298 #define SYNCER_MAXDELAY 32 299 static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */ 300 static int syncdelay = 30; /* max time to delay syncing data */ 301 static int filedelay = 30; /* time to delay syncing files */ 302 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0, 303 "Time to delay syncing files (in seconds)"); 304 static int dirdelay = 29; /* time to delay syncing directories */ 305 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0, 306 "Time to delay syncing directories (in seconds)"); 307 static int metadelay = 28; /* time to delay syncing metadata */ 308 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0, 309 "Time to delay syncing metadata (in seconds)"); 310 static int rushjob; /* number of slots to run ASAP */ 311 static int stat_rush_requests; /* number of times I/O speeded up */ 312 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0, 313 "Number of times I/O speeded up (rush requests)"); 314 315 #define VDBATCH_SIZE 8 316 struct vdbatch { 317 u_int index; 318 struct mtx lock; 319 struct vnode *tab[VDBATCH_SIZE]; 320 }; 321 DPCPU_DEFINE_STATIC(struct vdbatch, vd); 322 323 static void vdbatch_dequeue(struct vnode *vp); 324 325 /* 326 * When shutting down the syncer, run it at four times normal speed. 327 */ 328 #define SYNCER_SHUTDOWN_SPEEDUP 4 329 static int sync_vnode_count; 330 static int syncer_worklist_len; 331 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY } 332 syncer_state; 333 334 /* Target for maximum number of vnodes. */ 335 u_long desiredvnodes; 336 static u_long gapvnodes; /* gap between wanted and desired */ 337 static u_long vhiwat; /* enough extras after expansion */ 338 static u_long vlowat; /* minimal extras before expansion */ 339 static bool vstir; /* nonzero to stir non-free vnodes */ 340 static volatile int vsmalltrigger = 8; /* pref to keep if > this many pages */ 341 342 static u_long vnlru_read_freevnodes(void); 343 344 /* 345 * Note that no attempt is made to sanitize these parameters. 346 */ 347 static int 348 sysctl_maxvnodes(SYSCTL_HANDLER_ARGS) 349 { 350 u_long val; 351 int error; 352 353 val = desiredvnodes; 354 error = sysctl_handle_long(oidp, &val, 0, req); 355 if (error != 0 || req->newptr == NULL) 356 return (error); 357 358 if (val == desiredvnodes) 359 return (0); 360 mtx_lock(&vnode_list_mtx); 361 desiredvnodes = val; 362 wantfreevnodes = desiredvnodes / 4; 363 vnlru_recalc(); 364 mtx_unlock(&vnode_list_mtx); 365 /* 366 * XXX There is no protection against multiple threads changing 367 * desiredvnodes at the same time. Locking above only helps vnlru and 368 * getnewvnode. 369 */ 370 vfs_hash_changesize(desiredvnodes); 371 cache_changesize(desiredvnodes); 372 return (0); 373 } 374 375 SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes, 376 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, 377 "LU", "Target for maximum number of vnodes (legacy)"); 378 SYSCTL_PROC(_vfs_vnode_param, OID_AUTO, limit, 379 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, 380 "LU", "Target for maximum number of vnodes"); 381 382 static int 383 sysctl_freevnodes(SYSCTL_HANDLER_ARGS) 384 { 385 u_long rfreevnodes; 386 387 rfreevnodes = vnlru_read_freevnodes(); 388 return (sysctl_handle_long(oidp, &rfreevnodes, 0, req)); 389 } 390 391 SYSCTL_PROC(_vfs, OID_AUTO, freevnodes, 392 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0, sysctl_freevnodes, 393 "LU", "Number of \"free\" vnodes (legacy)"); 394 SYSCTL_PROC(_vfs_vnode_stats, OID_AUTO, free, 395 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0, sysctl_freevnodes, 396 "LU", "Number of \"free\" vnodes"); 397 398 static int 399 sysctl_wantfreevnodes(SYSCTL_HANDLER_ARGS) 400 { 401 u_long val; 402 int error; 403 404 val = wantfreevnodes; 405 error = sysctl_handle_long(oidp, &val, 0, req); 406 if (error != 0 || req->newptr == NULL) 407 return (error); 408 409 if (val == wantfreevnodes) 410 return (0); 411 mtx_lock(&vnode_list_mtx); 412 wantfreevnodes = val; 413 vnlru_recalc(); 414 mtx_unlock(&vnode_list_mtx); 415 return (0); 416 } 417 418 SYSCTL_PROC(_vfs, OID_AUTO, wantfreevnodes, 419 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes, 420 "LU", "Target for minimum number of \"free\" vnodes (legacy)"); 421 SYSCTL_PROC(_vfs_vnode_param, OID_AUTO, wantfree, 422 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes, 423 "LU", "Target for minimum number of \"free\" vnodes"); 424 425 static int vnlru_nowhere; 426 SYSCTL_INT(_vfs_vnode_vnlru, OID_AUTO, failed_runs, CTLFLAG_RD | CTLFLAG_STATS, 427 &vnlru_nowhere, 0, "Number of times the vnlru process ran without success"); 428 429 static int 430 sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS) 431 { 432 struct vnode *vp; 433 struct nameidata nd; 434 char *buf; 435 unsigned long ndflags; 436 int error; 437 438 if (req->newptr == NULL) 439 return (EINVAL); 440 if (req->newlen >= PATH_MAX) 441 return (E2BIG); 442 443 buf = malloc(PATH_MAX, M_TEMP, M_WAITOK); 444 error = SYSCTL_IN(req, buf, req->newlen); 445 if (error != 0) 446 goto out; 447 448 buf[req->newlen] = '\0'; 449 450 ndflags = LOCKLEAF | NOFOLLOW | AUDITVNODE1; 451 NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE, buf); 452 if ((error = namei(&nd)) != 0) 453 goto out; 454 vp = nd.ni_vp; 455 456 if (VN_IS_DOOMED(vp)) { 457 /* 458 * This vnode is being recycled. Return != 0 to let the caller 459 * know that the sysctl had no effect. Return EAGAIN because a 460 * subsequent call will likely succeed (since namei will create 461 * a new vnode if necessary) 462 */ 463 error = EAGAIN; 464 goto putvnode; 465 } 466 467 vgone(vp); 468 putvnode: 469 vput(vp); 470 NDFREE_PNBUF(&nd); 471 out: 472 free(buf, M_TEMP); 473 return (error); 474 } 475 476 static int 477 sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS) 478 { 479 struct thread *td = curthread; 480 struct vnode *vp; 481 struct file *fp; 482 int error; 483 int fd; 484 485 if (req->newptr == NULL) 486 return (EBADF); 487 488 error = sysctl_handle_int(oidp, &fd, 0, req); 489 if (error != 0) 490 return (error); 491 error = getvnode(curthread, fd, &cap_fcntl_rights, &fp); 492 if (error != 0) 493 return (error); 494 vp = fp->f_vnode; 495 496 error = vn_lock(vp, LK_EXCLUSIVE); 497 if (error != 0) 498 goto drop; 499 500 vgone(vp); 501 VOP_UNLOCK(vp); 502 drop: 503 fdrop(fp, td); 504 return (error); 505 } 506 507 SYSCTL_PROC(_debug, OID_AUTO, try_reclaim_vnode, 508 CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 509 sysctl_try_reclaim_vnode, "A", "Try to reclaim a vnode by its pathname"); 510 SYSCTL_PROC(_debug, OID_AUTO, ftry_reclaim_vnode, 511 CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 512 sysctl_ftry_reclaim_vnode, "I", 513 "Try to reclaim a vnode by its file descriptor"); 514 515 /* Shift count for (uintptr_t)vp to initialize vp->v_hash. */ 516 #define vnsz2log 8 517 #ifndef DEBUG_LOCKS 518 _Static_assert(sizeof(struct vnode) >= 1UL << vnsz2log && 519 sizeof(struct vnode) < 1UL << (vnsz2log + 1), 520 "vnsz2log needs to be updated"); 521 #endif 522 523 /* 524 * Support for the bufobj clean & dirty pctrie. 525 */ 526 static void * 527 buf_trie_alloc(struct pctrie *ptree) 528 { 529 return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT)); 530 } 531 532 static void 533 buf_trie_free(struct pctrie *ptree, void *node) 534 { 535 uma_zfree_smr(buf_trie_zone, node); 536 } 537 PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free, 538 buf_trie_smr); 539 540 /* 541 * Lookup the next element greater than or equal to lblkno, accounting for the 542 * fact that, for pctries, negative values are greater than nonnegative ones. 543 */ 544 static struct buf * 545 buf_lookup_ge(struct bufv *bv, daddr_t lblkno) 546 { 547 struct buf *bp; 548 549 bp = BUF_PCTRIE_LOOKUP_GE(&bv->bv_root, lblkno); 550 if (bp == NULL && lblkno < 0) 551 bp = BUF_PCTRIE_LOOKUP_GE(&bv->bv_root, 0); 552 if (bp != NULL && bp->b_lblkno < lblkno) 553 bp = NULL; 554 return (bp); 555 } 556 557 /* 558 * Insert bp, and find the next element smaller than bp, accounting for the fact 559 * that, for pctries, negative values are greater than nonnegative ones. 560 */ 561 static int 562 buf_insert_lookup_le(struct bufv *bv, struct buf *bp, struct buf **n) 563 { 564 int error; 565 566 error = BUF_PCTRIE_INSERT_LOOKUP_LE(&bv->bv_root, bp, n); 567 if (error != EEXIST) { 568 if (*n == NULL && bp->b_lblkno >= 0) 569 *n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, ~0L); 570 if (*n != NULL && (*n)->b_lblkno >= bp->b_lblkno) 571 *n = NULL; 572 } 573 return (error); 574 } 575 576 /* 577 * Initialize the vnode management data structures. 578 * 579 * Reevaluate the following cap on the number of vnodes after the physical 580 * memory size exceeds 512GB. In the limit, as the physical memory size 581 * grows, the ratio of the memory size in KB to vnodes approaches 64:1. 582 */ 583 #ifndef MAXVNODES_MAX 584 #define MAXVNODES_MAX (512UL * 1024 * 1024 / 64) /* 8M */ 585 #endif 586 587 static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); 588 589 static struct vnode * 590 vn_alloc_marker(struct mount *mp) 591 { 592 struct vnode *vp; 593 594 vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO); 595 vp->v_type = VMARKER; 596 vp->v_mount = mp; 597 598 return (vp); 599 } 600 601 static void 602 vn_free_marker(struct vnode *vp) 603 { 604 605 MPASS(vp->v_type == VMARKER); 606 free(vp, M_VNODE_MARKER); 607 } 608 609 #ifdef KASAN 610 static int 611 vnode_ctor(void *mem, int size, void *arg __unused, int flags __unused) 612 { 613 kasan_mark(mem, size, roundup2(size, UMA_ALIGN_PTR + 1), 0); 614 return (0); 615 } 616 617 static void 618 vnode_dtor(void *mem, int size, void *arg __unused) 619 { 620 size_t end1, end2, off1, off2; 621 622 _Static_assert(offsetof(struct vnode, v_vnodelist) < 623 offsetof(struct vnode, v_dbatchcpu), 624 "KASAN marks require updating"); 625 626 off1 = offsetof(struct vnode, v_vnodelist); 627 off2 = offsetof(struct vnode, v_dbatchcpu); 628 end1 = off1 + sizeof(((struct vnode *)NULL)->v_vnodelist); 629 end2 = off2 + sizeof(((struct vnode *)NULL)->v_dbatchcpu); 630 631 /* 632 * Access to the v_vnodelist and v_dbatchcpu fields are permitted even 633 * after the vnode has been freed. Try to get some KASAN coverage by 634 * marking everything except those two fields as invalid. Because 635 * KASAN's tracking is not byte-granular, any preceding fields sharing 636 * the same 8-byte aligned word must also be marked valid. 637 */ 638 639 /* Handle the area from the start until v_vnodelist... */ 640 off1 = rounddown2(off1, KASAN_SHADOW_SCALE); 641 kasan_mark(mem, off1, off1, KASAN_UMA_FREED); 642 643 /* ... then the area between v_vnodelist and v_dbatchcpu ... */ 644 off1 = roundup2(end1, KASAN_SHADOW_SCALE); 645 off2 = rounddown2(off2, KASAN_SHADOW_SCALE); 646 if (off2 > off1) 647 kasan_mark((void *)((char *)mem + off1), off2 - off1, 648 off2 - off1, KASAN_UMA_FREED); 649 650 /* ... and finally the area from v_dbatchcpu to the end. */ 651 off2 = roundup2(end2, KASAN_SHADOW_SCALE); 652 kasan_mark((void *)((char *)mem + off2), size - off2, size - off2, 653 KASAN_UMA_FREED); 654 } 655 #endif /* KASAN */ 656 657 /* 658 * Initialize a vnode as it first enters the zone. 659 */ 660 static int 661 vnode_init(void *mem, int size, int flags) 662 { 663 struct vnode *vp; 664 665 vp = mem; 666 bzero(vp, size); 667 /* 668 * Setup locks. 669 */ 670 vp->v_vnlock = &vp->v_lock; 671 mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF); 672 /* 673 * By default, don't allow shared locks unless filesystems opt-in. 674 */ 675 lockinit(vp->v_vnlock, PVFS, "vnode", VLKTIMEOUT, 676 LK_NOSHARE | LK_IS_VNODE); 677 /* 678 * Initialize bufobj. 679 */ 680 bufobj_init(&vp->v_bufobj, vp); 681 /* 682 * Initialize namecache. 683 */ 684 cache_vnode_init(vp); 685 /* 686 * Initialize rangelocks. 687 */ 688 rangelock_init(&vp->v_rl); 689 690 vp->v_dbatchcpu = NOCPU; 691 692 vp->v_state = VSTATE_DEAD; 693 694 /* 695 * Check vhold_recycle_free for an explanation. 696 */ 697 vp->v_holdcnt = VHOLD_NO_SMR; 698 vp->v_type = VNON; 699 mtx_lock(&vnode_list_mtx); 700 TAILQ_INSERT_BEFORE(vnode_list_free_marker, vp, v_vnodelist); 701 mtx_unlock(&vnode_list_mtx); 702 return (0); 703 } 704 705 /* 706 * Free a vnode when it is cleared from the zone. 707 */ 708 static void 709 vnode_fini(void *mem, int size) 710 { 711 struct vnode *vp; 712 struct bufobj *bo; 713 714 vp = mem; 715 vdbatch_dequeue(vp); 716 mtx_lock(&vnode_list_mtx); 717 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 718 mtx_unlock(&vnode_list_mtx); 719 rangelock_destroy(&vp->v_rl); 720 lockdestroy(vp->v_vnlock); 721 mtx_destroy(&vp->v_interlock); 722 bo = &vp->v_bufobj; 723 rw_destroy(BO_LOCKPTR(bo)); 724 725 kasan_mark(mem, size, size, 0); 726 } 727 728 /* 729 * Provide the size of NFS nclnode and NFS fh for calculation of the 730 * vnode memory consumption. The size is specified directly to 731 * eliminate dependency on NFS-private header. 732 * 733 * Other filesystems may use bigger or smaller (like UFS and ZFS) 734 * private inode data, but the NFS-based estimation is ample enough. 735 * Still, we care about differences in the size between 64- and 32-bit 736 * platforms. 737 * 738 * Namecache structure size is heuristically 739 * sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1. 740 */ 741 #ifdef _LP64 742 #define NFS_NCLNODE_SZ (528 + 64) 743 #define NC_SZ 148 744 #else 745 #define NFS_NCLNODE_SZ (360 + 32) 746 #define NC_SZ 92 747 #endif 748 749 static void 750 vntblinit(void *dummy __unused) 751 { 752 struct vdbatch *vd; 753 uma_ctor ctor; 754 uma_dtor dtor; 755 int cpu, physvnodes, virtvnodes; 756 757 /* 758 * Desiredvnodes is a function of the physical memory size and the 759 * kernel's heap size. Generally speaking, it scales with the 760 * physical memory size. The ratio of desiredvnodes to the physical 761 * memory size is 1:16 until desiredvnodes exceeds 98,304. 762 * Thereafter, the 763 * marginal ratio of desiredvnodes to the physical memory size is 764 * 1:64. However, desiredvnodes is limited by the kernel's heap 765 * size. The memory required by desiredvnodes vnodes and vm objects 766 * must not exceed 1/10th of the kernel's heap size. 767 */ 768 physvnodes = maxproc + pgtok(vm_cnt.v_page_count) / 64 + 769 3 * min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 64; 770 virtvnodes = vm_kmem_size / (10 * (sizeof(struct vm_object) + 771 sizeof(struct vnode) + NC_SZ * ncsizefactor + NFS_NCLNODE_SZ)); 772 desiredvnodes = min(physvnodes, virtvnodes); 773 if (desiredvnodes > MAXVNODES_MAX) { 774 if (bootverbose) 775 printf("Reducing kern.maxvnodes %lu -> %lu\n", 776 desiredvnodes, MAXVNODES_MAX); 777 desiredvnodes = MAXVNODES_MAX; 778 } 779 wantfreevnodes = desiredvnodes / 4; 780 mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF); 781 TAILQ_INIT(&vnode_list); 782 mtx_init(&vnode_list_mtx, "vnode_list", NULL, MTX_DEF); 783 /* 784 * The lock is taken to appease WITNESS. 785 */ 786 mtx_lock(&vnode_list_mtx); 787 vnlru_recalc(); 788 mtx_unlock(&vnode_list_mtx); 789 vnode_list_free_marker = vn_alloc_marker(NULL); 790 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist); 791 vnode_list_reclaim_marker = vn_alloc_marker(NULL); 792 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist); 793 794 #ifdef KASAN 795 ctor = vnode_ctor; 796 dtor = vnode_dtor; 797 #else 798 ctor = NULL; 799 dtor = NULL; 800 #endif 801 vnode_zone = uma_zcreate("VNODE", sizeof(struct vnode), ctor, dtor, 802 vnode_init, vnode_fini, UMA_ALIGN_PTR, UMA_ZONE_NOKASAN); 803 uma_zone_set_smr(vnode_zone, vfs_smr); 804 805 /* 806 * Preallocate enough nodes to support one-per buf so that 807 * we can not fail an insert. reassignbuf() callers can not 808 * tolerate the insertion failure. 809 */ 810 buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(), 811 NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR, 812 UMA_ZONE_NOFREE | UMA_ZONE_SMR); 813 buf_trie_smr = uma_zone_get_smr(buf_trie_zone); 814 uma_prealloc(buf_trie_zone, nbuf); 815 816 vnodes_created = counter_u64_alloc(M_WAITOK); 817 direct_recycles_free_count = counter_u64_alloc(M_WAITOK); 818 vnode_skipped_requeues = counter_u64_alloc(M_WAITOK); 819 820 /* 821 * Initialize the filesystem syncer. 822 */ 823 syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE, 824 &syncer_mask); 825 syncer_maxdelay = syncer_mask + 1; 826 mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF); 827 cv_init(&sync_wakeup, "syncer"); 828 829 CPU_FOREACH(cpu) { 830 vd = DPCPU_ID_PTR((cpu), vd); 831 bzero(vd, sizeof(*vd)); 832 mtx_init(&vd->lock, "vdbatch", NULL, MTX_DEF); 833 } 834 } 835 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL); 836 837 /* 838 * Mark a mount point as busy. Used to synchronize access and to delay 839 * unmounting. Eventually, mountlist_mtx is not released on failure. 840 * 841 * vfs_busy() is a custom lock, it can block the caller. 842 * vfs_busy() only sleeps if the unmount is active on the mount point. 843 * For a mountpoint mp, vfs_busy-enforced lock is before lock of any 844 * vnode belonging to mp. 845 * 846 * Lookup uses vfs_busy() to traverse mount points. 847 * root fs var fs 848 * / vnode lock A / vnode lock (/var) D 849 * /var vnode lock B /log vnode lock(/var/log) E 850 * vfs_busy lock C vfs_busy lock F 851 * 852 * Within each file system, the lock order is C->A->B and F->D->E. 853 * 854 * When traversing across mounts, the system follows that lock order: 855 * 856 * C->A->B 857 * | 858 * +->F->D->E 859 * 860 * The lookup() process for namei("/var") illustrates the process: 861 * 1. VOP_LOOKUP() obtains B while A is held 862 * 2. vfs_busy() obtains a shared lock on F while A and B are held 863 * 3. vput() releases lock on B 864 * 4. vput() releases lock on A 865 * 5. VFS_ROOT() obtains lock on D while shared lock on F is held 866 * 6. vfs_unbusy() releases shared lock on F 867 * 7. vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A. 868 * Attempt to lock A (instead of vp_crossmp) while D is held would 869 * violate the global order, causing deadlocks. 870 * 871 * dounmount() locks B while F is drained. Note that for stacked 872 * filesystems, D and B in the example above may be the same lock, 873 * which introdues potential lock order reversal deadlock between 874 * dounmount() and step 5 above. These filesystems may avoid the LOR 875 * by setting VV_CROSSLOCK on the covered vnode so that lock B will 876 * remain held until after step 5. 877 */ 878 int 879 vfs_busy(struct mount *mp, int flags) 880 { 881 struct mount_pcpu *mpcpu; 882 883 MPASS((flags & ~MBF_MASK) == 0); 884 CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags); 885 886 if (vfs_op_thread_enter(mp, mpcpu)) { 887 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 888 MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0); 889 MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0); 890 vfs_mp_count_add_pcpu(mpcpu, ref, 1); 891 vfs_mp_count_add_pcpu(mpcpu, lockref, 1); 892 vfs_op_thread_exit(mp, mpcpu); 893 if (flags & MBF_MNTLSTLOCK) 894 mtx_unlock(&mountlist_mtx); 895 return (0); 896 } 897 898 MNT_ILOCK(mp); 899 vfs_assert_mount_counters(mp); 900 MNT_REF(mp); 901 /* 902 * If mount point is currently being unmounted, sleep until the 903 * mount point fate is decided. If thread doing the unmounting fails, 904 * it will clear MNTK_UNMOUNT flag before waking us up, indicating 905 * that this mount point has survived the unmount attempt and vfs_busy 906 * should retry. Otherwise the unmounter thread will set MNTK_REFEXPIRE 907 * flag in addition to MNTK_UNMOUNT, indicating that mount point is 908 * about to be really destroyed. vfs_busy needs to release its 909 * reference on the mount point in this case and return with ENOENT, 910 * telling the caller the mount it tried to busy is no longer valid. 911 */ 912 while (mp->mnt_kern_flag & MNTK_UNMOUNT) { 913 KASSERT(TAILQ_EMPTY(&mp->mnt_uppers), 914 ("%s: non-empty upper mount list with pending unmount", 915 __func__)); 916 if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) { 917 MNT_REL(mp); 918 MNT_IUNLOCK(mp); 919 CTR1(KTR_VFS, "%s: failed busying before sleeping", 920 __func__); 921 return (ENOENT); 922 } 923 if (flags & MBF_MNTLSTLOCK) 924 mtx_unlock(&mountlist_mtx); 925 mp->mnt_kern_flag |= MNTK_MWAIT; 926 msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0); 927 if (flags & MBF_MNTLSTLOCK) 928 mtx_lock(&mountlist_mtx); 929 MNT_ILOCK(mp); 930 } 931 if (flags & MBF_MNTLSTLOCK) 932 mtx_unlock(&mountlist_mtx); 933 mp->mnt_lockref++; 934 MNT_IUNLOCK(mp); 935 return (0); 936 } 937 938 /* 939 * Free a busy filesystem. 940 */ 941 void 942 vfs_unbusy(struct mount *mp) 943 { 944 struct mount_pcpu *mpcpu; 945 int c; 946 947 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 948 949 if (vfs_op_thread_enter(mp, mpcpu)) { 950 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 951 vfs_mp_count_sub_pcpu(mpcpu, lockref, 1); 952 vfs_mp_count_sub_pcpu(mpcpu, ref, 1); 953 vfs_op_thread_exit(mp, mpcpu); 954 return; 955 } 956 957 MNT_ILOCK(mp); 958 vfs_assert_mount_counters(mp); 959 MNT_REL(mp); 960 c = --mp->mnt_lockref; 961 if (mp->mnt_vfs_ops == 0) { 962 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 963 MNT_IUNLOCK(mp); 964 return; 965 } 966 if (c < 0) 967 vfs_dump_mount_counters(mp); 968 if (c == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) { 969 MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT); 970 CTR1(KTR_VFS, "%s: waking up waiters", __func__); 971 mp->mnt_kern_flag &= ~MNTK_DRAINING; 972 wakeup(&mp->mnt_lockref); 973 } 974 MNT_IUNLOCK(mp); 975 } 976 977 /* 978 * Lookup a mount point by filesystem identifier. 979 */ 980 struct mount * 981 vfs_getvfs(fsid_t *fsid) 982 { 983 struct mount *mp; 984 985 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 986 mtx_lock(&mountlist_mtx); 987 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 988 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 989 vfs_ref(mp); 990 mtx_unlock(&mountlist_mtx); 991 return (mp); 992 } 993 } 994 mtx_unlock(&mountlist_mtx); 995 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 996 return ((struct mount *) 0); 997 } 998 999 /* 1000 * Lookup a mount point by filesystem identifier, busying it before 1001 * returning. 1002 * 1003 * To avoid congestion on mountlist_mtx, implement simple direct-mapped 1004 * cache for popular filesystem identifiers. The cache is lockess, using 1005 * the fact that struct mount's are never freed. In worst case we may 1006 * get pointer to unmounted or even different filesystem, so we have to 1007 * check what we got, and go slow way if so. 1008 */ 1009 struct mount * 1010 vfs_busyfs(fsid_t *fsid) 1011 { 1012 #define FSID_CACHE_SIZE 256 1013 typedef struct mount * volatile vmp_t; 1014 static vmp_t cache[FSID_CACHE_SIZE]; 1015 struct mount *mp; 1016 int error; 1017 uint32_t hash; 1018 1019 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 1020 hash = fsid->val[0] ^ fsid->val[1]; 1021 hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1); 1022 mp = cache[hash]; 1023 if (mp == NULL || fsidcmp(&mp->mnt_stat.f_fsid, fsid) != 0) 1024 goto slow; 1025 if (vfs_busy(mp, 0) != 0) { 1026 cache[hash] = NULL; 1027 goto slow; 1028 } 1029 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) 1030 return (mp); 1031 else 1032 vfs_unbusy(mp); 1033 1034 slow: 1035 mtx_lock(&mountlist_mtx); 1036 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 1037 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 1038 error = vfs_busy(mp, MBF_MNTLSTLOCK); 1039 if (error) { 1040 cache[hash] = NULL; 1041 mtx_unlock(&mountlist_mtx); 1042 return (NULL); 1043 } 1044 cache[hash] = mp; 1045 return (mp); 1046 } 1047 } 1048 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 1049 mtx_unlock(&mountlist_mtx); 1050 return ((struct mount *) 0); 1051 } 1052 1053 /* 1054 * Check if a user can access privileged mount options. 1055 */ 1056 int 1057 vfs_suser(struct mount *mp, struct thread *td) 1058 { 1059 int error; 1060 1061 if (jailed(td->td_ucred)) { 1062 /* 1063 * If the jail of the calling thread lacks permission for 1064 * this type of file system, deny immediately. 1065 */ 1066 if (!prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag)) 1067 return (EPERM); 1068 1069 /* 1070 * If the file system was mounted outside the jail of the 1071 * calling thread, deny immediately. 1072 */ 1073 if (prison_check(td->td_ucred, mp->mnt_cred) != 0) 1074 return (EPERM); 1075 } 1076 1077 /* 1078 * If file system supports delegated administration, we don't check 1079 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified 1080 * by the file system itself. 1081 * If this is not the user that did original mount, we check for 1082 * the PRIV_VFS_MOUNT_OWNER privilege. 1083 */ 1084 if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) && 1085 mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) { 1086 if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0) 1087 return (error); 1088 } 1089 return (0); 1090 } 1091 1092 /* 1093 * Get a new unique fsid. Try to make its val[0] unique, since this value 1094 * will be used to create fake device numbers for stat(). Also try (but 1095 * not so hard) make its val[0] unique mod 2^16, since some emulators only 1096 * support 16-bit device numbers. We end up with unique val[0]'s for the 1097 * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls. 1098 * 1099 * Keep in mind that several mounts may be running in parallel. Starting 1100 * the search one past where the previous search terminated is both a 1101 * micro-optimization and a defense against returning the same fsid to 1102 * different mounts. 1103 */ 1104 void 1105 vfs_getnewfsid(struct mount *mp) 1106 { 1107 static uint16_t mntid_base; 1108 struct mount *nmp; 1109 fsid_t tfsid; 1110 int mtype; 1111 1112 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 1113 mtx_lock(&mntid_mtx); 1114 mtype = mp->mnt_vfc->vfc_typenum; 1115 tfsid.val[1] = mtype; 1116 mtype = (mtype & 0xFF) << 24; 1117 for (;;) { 1118 tfsid.val[0] = makedev(255, 1119 mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF)); 1120 mntid_base++; 1121 if ((nmp = vfs_getvfs(&tfsid)) == NULL) 1122 break; 1123 vfs_rel(nmp); 1124 } 1125 mp->mnt_stat.f_fsid.val[0] = tfsid.val[0]; 1126 mp->mnt_stat.f_fsid.val[1] = tfsid.val[1]; 1127 mtx_unlock(&mntid_mtx); 1128 } 1129 1130 /* 1131 * Knob to control the precision of file timestamps: 1132 * 1133 * 0 = seconds only; nanoseconds zeroed. 1134 * 1 = seconds and nanoseconds, accurate within 1/HZ. 1135 * 2 = seconds and nanoseconds, truncated to microseconds. 1136 * >=3 = seconds and nanoseconds, maximum precision. 1137 */ 1138 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC }; 1139 1140 static int timestamp_precision = TSP_USEC; 1141 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW, 1142 ×tamp_precision, 0, "File timestamp precision (0: seconds, " 1143 "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, " 1144 "3+: sec + ns (max. precision))"); 1145 1146 /* 1147 * Get a current timestamp. 1148 */ 1149 void 1150 vfs_timestamp(struct timespec *tsp) 1151 { 1152 struct timeval tv; 1153 1154 switch (timestamp_precision) { 1155 case TSP_SEC: 1156 tsp->tv_sec = time_second; 1157 tsp->tv_nsec = 0; 1158 break; 1159 case TSP_HZ: 1160 getnanotime(tsp); 1161 break; 1162 case TSP_USEC: 1163 microtime(&tv); 1164 TIMEVAL_TO_TIMESPEC(&tv, tsp); 1165 break; 1166 case TSP_NSEC: 1167 default: 1168 nanotime(tsp); 1169 break; 1170 } 1171 } 1172 1173 /* 1174 * Set vnode attributes to VNOVAL 1175 */ 1176 void 1177 vattr_null(struct vattr *vap) 1178 { 1179 1180 vap->va_type = VNON; 1181 vap->va_size = VNOVAL; 1182 vap->va_bytes = VNOVAL; 1183 vap->va_mode = VNOVAL; 1184 vap->va_nlink = VNOVAL; 1185 vap->va_uid = VNOVAL; 1186 vap->va_gid = VNOVAL; 1187 vap->va_fsid = VNOVAL; 1188 vap->va_fileid = VNOVAL; 1189 vap->va_blocksize = VNOVAL; 1190 vap->va_rdev = VNOVAL; 1191 vap->va_atime.tv_sec = VNOVAL; 1192 vap->va_atime.tv_nsec = VNOVAL; 1193 vap->va_mtime.tv_sec = VNOVAL; 1194 vap->va_mtime.tv_nsec = VNOVAL; 1195 vap->va_ctime.tv_sec = VNOVAL; 1196 vap->va_ctime.tv_nsec = VNOVAL; 1197 vap->va_birthtime.tv_sec = VNOVAL; 1198 vap->va_birthtime.tv_nsec = VNOVAL; 1199 vap->va_flags = VNOVAL; 1200 vap->va_gen = VNOVAL; 1201 vap->va_vaflags = 0; 1202 } 1203 1204 /* 1205 * Try to reduce the total number of vnodes. 1206 * 1207 * This routine (and its user) are buggy in at least the following ways: 1208 * - all parameters were picked years ago when RAM sizes were significantly 1209 * smaller 1210 * - it can pick vnodes based on pages used by the vm object, but filesystems 1211 * like ZFS don't use it making the pick broken 1212 * - since ZFS has its own aging policy it gets partially combated by this one 1213 * - a dedicated method should be provided for filesystems to let them decide 1214 * whether the vnode should be recycled 1215 * 1216 * This routine is called when we have too many vnodes. It attempts 1217 * to free <count> vnodes and will potentially free vnodes that still 1218 * have VM backing store (VM backing store is typically the cause 1219 * of a vnode blowout so we want to do this). Therefore, this operation 1220 * is not considered cheap. 1221 * 1222 * A number of conditions may prevent a vnode from being reclaimed. 1223 * the buffer cache may have references on the vnode, a directory 1224 * vnode may still have references due to the namei cache representing 1225 * underlying files, or the vnode may be in active use. It is not 1226 * desirable to reuse such vnodes. These conditions may cause the 1227 * number of vnodes to reach some minimum value regardless of what 1228 * you set kern.maxvnodes to. Do not set kern.maxvnodes too low. 1229 * 1230 * @param reclaim_nc_src Only reclaim directories with outgoing namecache 1231 * entries if this argument is strue 1232 * @param trigger Only reclaim vnodes with fewer than this many resident 1233 * pages. 1234 * @param target How many vnodes to reclaim. 1235 * @return The number of vnodes that were reclaimed. 1236 */ 1237 static int 1238 vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) 1239 { 1240 struct vnode *vp, *mvp; 1241 struct mount *mp; 1242 struct vm_object *object; 1243 u_long done; 1244 bool retried; 1245 1246 mtx_assert(&vnode_list_mtx, MA_OWNED); 1247 1248 retried = false; 1249 done = 0; 1250 1251 mvp = vnode_list_reclaim_marker; 1252 restart: 1253 vp = mvp; 1254 while (done < target) { 1255 vp = TAILQ_NEXT(vp, v_vnodelist); 1256 if (__predict_false(vp == NULL)) 1257 break; 1258 1259 if (__predict_false(vp->v_type == VMARKER)) 1260 continue; 1261 1262 /* 1263 * If it's been deconstructed already, it's still 1264 * referenced, or it exceeds the trigger, skip it. 1265 * Also skip free vnodes. We are trying to make space 1266 * for more free vnodes, not reduce their count. 1267 */ 1268 if (vp->v_usecount > 0 || vp->v_holdcnt == 0 || 1269 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src))) 1270 goto next_iter; 1271 1272 if (vp->v_type == VBAD || vp->v_type == VNON) 1273 goto next_iter; 1274 1275 object = atomic_load_ptr(&vp->v_object); 1276 if (object == NULL || object->resident_page_count > trigger) { 1277 goto next_iter; 1278 } 1279 1280 /* 1281 * Handle races against vnode allocation. Filesystems lock the 1282 * vnode some time after it gets returned from getnewvnode, 1283 * despite type and hold count being manipulated earlier. 1284 * Resorting to checking v_mount restores guarantees present 1285 * before the global list was reworked to contain all vnodes. 1286 */ 1287 if (!VI_TRYLOCK(vp)) 1288 goto next_iter; 1289 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1290 VI_UNLOCK(vp); 1291 goto next_iter; 1292 } 1293 if (vp->v_mount == NULL) { 1294 VI_UNLOCK(vp); 1295 goto next_iter; 1296 } 1297 vholdl(vp); 1298 VI_UNLOCK(vp); 1299 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1300 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1301 mtx_unlock(&vnode_list_mtx); 1302 1303 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 1304 vdrop_recycle(vp); 1305 goto next_iter_unlocked; 1306 } 1307 if (VOP_LOCK(vp, LK_EXCLUSIVE|LK_NOWAIT) != 0) { 1308 vdrop_recycle(vp); 1309 vn_finished_write(mp); 1310 goto next_iter_unlocked; 1311 } 1312 1313 VI_LOCK(vp); 1314 if (vp->v_usecount > 0 || 1315 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) || 1316 (vp->v_object != NULL && vp->v_object->handle == vp && 1317 vp->v_object->resident_page_count > trigger)) { 1318 VOP_UNLOCK(vp); 1319 vdropl_recycle(vp); 1320 vn_finished_write(mp); 1321 goto next_iter_unlocked; 1322 } 1323 recycles_count++; 1324 vgonel(vp); 1325 VOP_UNLOCK(vp); 1326 vdropl_recycle(vp); 1327 vn_finished_write(mp); 1328 done++; 1329 next_iter_unlocked: 1330 maybe_yield(); 1331 mtx_lock(&vnode_list_mtx); 1332 goto restart; 1333 next_iter: 1334 MPASS(vp->v_type != VMARKER); 1335 if (!should_yield()) 1336 continue; 1337 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1338 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1339 mtx_unlock(&vnode_list_mtx); 1340 kern_yield(PRI_USER); 1341 mtx_lock(&vnode_list_mtx); 1342 goto restart; 1343 } 1344 if (done == 0 && !retried) { 1345 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1346 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); 1347 retried = true; 1348 goto restart; 1349 } 1350 return (done); 1351 } 1352 1353 static int max_free_per_call = 10000; 1354 SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_free_per_call, 0, 1355 "limit on vnode free requests per call to the vnlru_free routine (legacy)"); 1356 SYSCTL_INT(_vfs_vnode_vnlru, OID_AUTO, max_free_per_call, CTLFLAG_RW, 1357 &max_free_per_call, 0, 1358 "limit on vnode free requests per call to the vnlru_free routine"); 1359 1360 /* 1361 * Attempt to recycle requested amount of free vnodes. 1362 */ 1363 static int 1364 vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp, bool isvnlru) 1365 { 1366 struct vnode *vp; 1367 struct mount *mp; 1368 int ocount; 1369 bool retried; 1370 1371 mtx_assert(&vnode_list_mtx, MA_OWNED); 1372 if (count > max_free_per_call) 1373 count = max_free_per_call; 1374 if (count == 0) { 1375 mtx_unlock(&vnode_list_mtx); 1376 return (0); 1377 } 1378 ocount = count; 1379 retried = false; 1380 vp = mvp; 1381 for (;;) { 1382 vp = TAILQ_NEXT(vp, v_vnodelist); 1383 if (__predict_false(vp == NULL)) { 1384 /* 1385 * The free vnode marker can be past eligible vnodes: 1386 * 1. if vdbatch_process trylock failed 1387 * 2. if vtryrecycle failed 1388 * 1389 * If so, start the scan from scratch. 1390 */ 1391 if (!retried && vnlru_read_freevnodes() > 0) { 1392 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1393 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); 1394 vp = mvp; 1395 retried = true; 1396 continue; 1397 } 1398 1399 /* 1400 * Give up 1401 */ 1402 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1403 TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist); 1404 mtx_unlock(&vnode_list_mtx); 1405 break; 1406 } 1407 if (__predict_false(vp->v_type == VMARKER)) 1408 continue; 1409 if (vp->v_holdcnt > 0) 1410 continue; 1411 /* 1412 * Don't recycle if our vnode is from different type 1413 * of mount point. Note that mp is type-safe, the 1414 * check does not reach unmapped address even if 1415 * vnode is reclaimed. 1416 */ 1417 if (mnt_op != NULL && (mp = vp->v_mount) != NULL && 1418 mp->mnt_op != mnt_op) { 1419 continue; 1420 } 1421 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1422 continue; 1423 } 1424 if (!vhold_recycle_free(vp)) 1425 continue; 1426 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1427 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1428 mtx_unlock(&vnode_list_mtx); 1429 /* 1430 * FIXME: ignores the return value, meaning it may be nothing 1431 * got recycled but it claims otherwise to the caller. 1432 * 1433 * Originally the value started being ignored in 2005 with 1434 * 114a1006a8204aa156e1f9ad6476cdff89cada7f . 1435 * 1436 * Respecting the value can run into significant stalls if most 1437 * vnodes belong to one file system and it has writes 1438 * suspended. In presence of many threads and millions of 1439 * vnodes they keep contending on the vnode_list_mtx lock only 1440 * to find vnodes they can't recycle. 1441 * 1442 * The solution would be to pre-check if the vnode is likely to 1443 * be recycle-able, but it needs to happen with the 1444 * vnode_list_mtx lock held. This runs into a problem where 1445 * VOP_GETWRITEMOUNT (currently needed to find out about if 1446 * writes are frozen) can take locks which LOR against it. 1447 * 1448 * Check nullfs for one example (null_getwritemount). 1449 */ 1450 vtryrecycle(vp, isvnlru); 1451 count--; 1452 if (count == 0) { 1453 break; 1454 } 1455 mtx_lock(&vnode_list_mtx); 1456 vp = mvp; 1457 } 1458 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1459 return (ocount - count); 1460 } 1461 1462 /* 1463 * XXX: returns without vnode_list_mtx locked! 1464 */ 1465 static int 1466 vnlru_free_locked_direct(int count) 1467 { 1468 int ret; 1469 1470 mtx_assert(&vnode_list_mtx, MA_OWNED); 1471 ret = vnlru_free_impl(count, NULL, vnode_list_free_marker, false); 1472 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1473 return (ret); 1474 } 1475 1476 static int 1477 vnlru_free_locked_vnlru(int count) 1478 { 1479 int ret; 1480 1481 mtx_assert(&vnode_list_mtx, MA_OWNED); 1482 ret = vnlru_free_impl(count, NULL, vnode_list_free_marker, true); 1483 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1484 return (ret); 1485 } 1486 1487 static int 1488 vnlru_free_vnlru(int count) 1489 { 1490 1491 mtx_lock(&vnode_list_mtx); 1492 return (vnlru_free_locked_vnlru(count)); 1493 } 1494 1495 void 1496 vnlru_free_vfsops(int count, struct vfsops *mnt_op, struct vnode *mvp) 1497 { 1498 1499 MPASS(mnt_op != NULL); 1500 MPASS(mvp != NULL); 1501 VNPASS(mvp->v_type == VMARKER, mvp); 1502 mtx_lock(&vnode_list_mtx); 1503 vnlru_free_impl(count, mnt_op, mvp, true); 1504 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1505 } 1506 1507 struct vnode * 1508 vnlru_alloc_marker(void) 1509 { 1510 struct vnode *mvp; 1511 1512 mvp = vn_alloc_marker(NULL); 1513 mtx_lock(&vnode_list_mtx); 1514 TAILQ_INSERT_BEFORE(vnode_list_free_marker, mvp, v_vnodelist); 1515 mtx_unlock(&vnode_list_mtx); 1516 return (mvp); 1517 } 1518 1519 void 1520 vnlru_free_marker(struct vnode *mvp) 1521 { 1522 mtx_lock(&vnode_list_mtx); 1523 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1524 mtx_unlock(&vnode_list_mtx); 1525 vn_free_marker(mvp); 1526 } 1527 1528 static void 1529 vnlru_recalc(void) 1530 { 1531 1532 mtx_assert(&vnode_list_mtx, MA_OWNED); 1533 gapvnodes = imax(desiredvnodes - wantfreevnodes, 100); 1534 vhiwat = gapvnodes / 11; /* 9% -- just under the 10% in vlrureclaim() */ 1535 vlowat = vhiwat / 2; 1536 } 1537 1538 /* 1539 * Attempt to recycle vnodes in a context that is always safe to block. 1540 * Calling vlrurecycle() from the bowels of filesystem code has some 1541 * interesting deadlock problems. 1542 */ 1543 static struct proc *vnlruproc; 1544 static int vnlruproc_sig; 1545 static u_long vnlruproc_kicks; 1546 1547 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, kicks, CTLFLAG_RD, &vnlruproc_kicks, 0, 1548 "Number of times vnlru awakened due to vnode shortage"); 1549 1550 #define VNLRU_COUNT_SLOP 100 1551 1552 /* 1553 * The main freevnodes counter is only updated when a counter local to CPU 1554 * diverges from 0 by more than VNLRU_FREEVNODES_SLOP. CPUs are conditionally 1555 * walked to compute a more accurate total. 1556 * 1557 * Note: the actual value at any given moment can still exceed slop, but it 1558 * should not be by significant margin in practice. 1559 */ 1560 #define VNLRU_FREEVNODES_SLOP 126 1561 1562 static void __noinline 1563 vfs_freevnodes_rollup(int8_t *lfreevnodes) 1564 { 1565 1566 atomic_add_long(&freevnodes, *lfreevnodes); 1567 *lfreevnodes = 0; 1568 critical_exit(); 1569 } 1570 1571 static __inline void 1572 vfs_freevnodes_inc(void) 1573 { 1574 int8_t *lfreevnodes; 1575 1576 critical_enter(); 1577 lfreevnodes = PCPU_PTR(vfs_freevnodes); 1578 (*lfreevnodes)++; 1579 if (__predict_false(*lfreevnodes == VNLRU_FREEVNODES_SLOP)) 1580 vfs_freevnodes_rollup(lfreevnodes); 1581 else 1582 critical_exit(); 1583 } 1584 1585 static __inline void 1586 vfs_freevnodes_dec(void) 1587 { 1588 int8_t *lfreevnodes; 1589 1590 critical_enter(); 1591 lfreevnodes = PCPU_PTR(vfs_freevnodes); 1592 (*lfreevnodes)--; 1593 if (__predict_false(*lfreevnodes == -VNLRU_FREEVNODES_SLOP)) 1594 vfs_freevnodes_rollup(lfreevnodes); 1595 else 1596 critical_exit(); 1597 } 1598 1599 static u_long 1600 vnlru_read_freevnodes(void) 1601 { 1602 long slop, rfreevnodes, rfreevnodes_old; 1603 int cpu; 1604 1605 rfreevnodes = atomic_load_long(&freevnodes); 1606 rfreevnodes_old = atomic_load_long(&freevnodes_old); 1607 1608 if (rfreevnodes > rfreevnodes_old) 1609 slop = rfreevnodes - rfreevnodes_old; 1610 else 1611 slop = rfreevnodes_old - rfreevnodes; 1612 if (slop < VNLRU_FREEVNODES_SLOP) 1613 return (rfreevnodes >= 0 ? rfreevnodes : 0); 1614 CPU_FOREACH(cpu) { 1615 rfreevnodes += cpuid_to_pcpu[cpu]->pc_vfs_freevnodes; 1616 } 1617 atomic_store_long(&freevnodes_old, rfreevnodes); 1618 return (freevnodes_old >= 0 ? freevnodes_old : 0); 1619 } 1620 1621 static bool 1622 vnlru_under(u_long rnumvnodes, u_long limit) 1623 { 1624 u_long rfreevnodes, space; 1625 1626 if (__predict_false(rnumvnodes > desiredvnodes)) 1627 return (true); 1628 1629 space = desiredvnodes - rnumvnodes; 1630 if (space < limit) { 1631 rfreevnodes = vnlru_read_freevnodes(); 1632 if (rfreevnodes > wantfreevnodes) 1633 space += rfreevnodes - wantfreevnodes; 1634 } 1635 return (space < limit); 1636 } 1637 1638 static void 1639 vnlru_kick_locked(void) 1640 { 1641 1642 mtx_assert(&vnode_list_mtx, MA_OWNED); 1643 if (vnlruproc_sig == 0) { 1644 vnlruproc_sig = 1; 1645 vnlruproc_kicks++; 1646 wakeup(vnlruproc); 1647 } 1648 } 1649 1650 static void 1651 vnlru_kick_cond(void) 1652 { 1653 1654 if (vnlru_read_freevnodes() > wantfreevnodes) 1655 return; 1656 1657 if (vnlruproc_sig) 1658 return; 1659 mtx_lock(&vnode_list_mtx); 1660 vnlru_kick_locked(); 1661 mtx_unlock(&vnode_list_mtx); 1662 } 1663 1664 static void 1665 vnlru_proc_sleep(void) 1666 { 1667 1668 if (vnlruproc_sig) { 1669 vnlruproc_sig = 0; 1670 wakeup(&vnlruproc_sig); 1671 } 1672 msleep(vnlruproc, &vnode_list_mtx, PVFS|PDROP, "vlruwt", hz); 1673 } 1674 1675 /* 1676 * A lighter version of the machinery below. 1677 * 1678 * Tries to reach goals only by recycling free vnodes and does not invoke 1679 * uma_reclaim(UMA_RECLAIM_DRAIN). 1680 * 1681 * This works around pathological behavior in vnlru in presence of tons of free 1682 * vnodes, but without having to rewrite the machinery at this time. Said 1683 * behavior boils down to continuously trying to reclaim all kinds of vnodes 1684 * (cycling through all levels of "force") when the count is transiently above 1685 * limit. This happens a lot when all vnodes are used up and vn_alloc 1686 * speculatively increments the counter. 1687 * 1688 * Sample testcase: vnode limit 8388608, 20 separate directory trees each with 1689 * 1 million files in total and 20 find(1) processes stating them in parallel 1690 * (one per each tree). 1691 * 1692 * On a kernel with only stock machinery this needs anywhere between 60 and 120 1693 * seconds to execute (time varies *wildly* between runs). With the workaround 1694 * it consistently stays around 20 seconds [it got further down with later 1695 * changes]. 1696 * 1697 * That is to say the entire thing needs a fundamental redesign (most notably 1698 * to accommodate faster recycling), the above only tries to get it ouf the way. 1699 * 1700 * Return values are: 1701 * -1 -- fallback to regular vnlru loop 1702 * 0 -- do nothing, go to sleep 1703 * >0 -- recycle this many vnodes 1704 */ 1705 static long 1706 vnlru_proc_light_pick(void) 1707 { 1708 u_long rnumvnodes, rfreevnodes; 1709 1710 if (vstir || vnlruproc_sig == 1) 1711 return (-1); 1712 1713 rnumvnodes = atomic_load_long(&numvnodes); 1714 rfreevnodes = vnlru_read_freevnodes(); 1715 1716 /* 1717 * vnode limit might have changed and now we may be at a significant 1718 * excess. Bail if we can't sort it out with free vnodes. 1719 * 1720 * Due to atomic updates the count can legitimately go above 1721 * the limit for a short period, don't bother doing anything in 1722 * that case. 1723 */ 1724 if (rnumvnodes > desiredvnodes + VNLRU_COUNT_SLOP + 10) { 1725 if (rnumvnodes - rfreevnodes >= desiredvnodes || 1726 rfreevnodes <= wantfreevnodes) { 1727 return (-1); 1728 } 1729 1730 return (rnumvnodes - desiredvnodes); 1731 } 1732 1733 /* 1734 * Don't try to reach wantfreevnodes target if there are too few vnodes 1735 * to begin with. 1736 */ 1737 if (rnumvnodes < wantfreevnodes) { 1738 return (0); 1739 } 1740 1741 if (rfreevnodes < wantfreevnodes) { 1742 return (-1); 1743 } 1744 1745 return (0); 1746 } 1747 1748 static bool 1749 vnlru_proc_light(void) 1750 { 1751 long freecount; 1752 1753 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1754 1755 freecount = vnlru_proc_light_pick(); 1756 if (freecount == -1) 1757 return (false); 1758 1759 if (freecount != 0) { 1760 vnlru_free_vnlru(freecount); 1761 } 1762 1763 mtx_lock(&vnode_list_mtx); 1764 vnlru_proc_sleep(); 1765 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1766 return (true); 1767 } 1768 1769 static u_long uma_reclaim_calls; 1770 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, uma_reclaim_calls, CTLFLAG_RD | CTLFLAG_STATS, 1771 &uma_reclaim_calls, 0, "Number of calls to uma_reclaim"); 1772 1773 static void 1774 vnlru_proc(void) 1775 { 1776 u_long rnumvnodes, rfreevnodes, target; 1777 unsigned long onumvnodes; 1778 int done, force, trigger, usevnodes; 1779 bool reclaim_nc_src, want_reread; 1780 1781 EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, vnlruproc, 1782 SHUTDOWN_PRI_FIRST); 1783 1784 force = 0; 1785 want_reread = false; 1786 for (;;) { 1787 kproc_suspend_check(vnlruproc); 1788 1789 if (force == 0 && vnlru_proc_light()) 1790 continue; 1791 1792 mtx_lock(&vnode_list_mtx); 1793 rnumvnodes = atomic_load_long(&numvnodes); 1794 1795 if (want_reread) { 1796 force = vnlru_under(numvnodes, vhiwat) ? 1 : 0; 1797 want_reread = false; 1798 } 1799 1800 /* 1801 * If numvnodes is too large (due to desiredvnodes being 1802 * adjusted using its sysctl, or emergency growth), first 1803 * try to reduce it by discarding free vnodes. 1804 */ 1805 if (rnumvnodes > desiredvnodes + 10) { 1806 vnlru_free_locked_vnlru(rnumvnodes - desiredvnodes); 1807 mtx_lock(&vnode_list_mtx); 1808 rnumvnodes = atomic_load_long(&numvnodes); 1809 } 1810 /* 1811 * Sleep if the vnode cache is in a good state. This is 1812 * when it is not over-full and has space for about a 4% 1813 * or 9% expansion (by growing its size or inexcessively 1814 * reducing free vnode count). Otherwise, try to reclaim 1815 * space for a 10% expansion. 1816 */ 1817 if (vstir && force == 0) { 1818 force = 1; 1819 vstir = false; 1820 } 1821 if (force == 0 && !vnlru_under(rnumvnodes, vlowat)) { 1822 vnlru_proc_sleep(); 1823 continue; 1824 } 1825 rfreevnodes = vnlru_read_freevnodes(); 1826 1827 onumvnodes = rnumvnodes; 1828 /* 1829 * Calculate parameters for recycling. These are the same 1830 * throughout the loop to give some semblance of fairness. 1831 * The trigger point is to avoid recycling vnodes with lots 1832 * of resident pages. We aren't trying to free memory; we 1833 * are trying to recycle or at least free vnodes. 1834 */ 1835 if (rnumvnodes <= desiredvnodes) 1836 usevnodes = rnumvnodes - rfreevnodes; 1837 else 1838 usevnodes = rnumvnodes; 1839 if (usevnodes <= 0) 1840 usevnodes = 1; 1841 /* 1842 * The trigger value is chosen to give a conservatively 1843 * large value to ensure that it alone doesn't prevent 1844 * making progress. The value can easily be so large that 1845 * it is effectively infinite in some congested and 1846 * misconfigured cases, and this is necessary. Normally 1847 * it is about 8 to 100 (pages), which is quite large. 1848 */ 1849 trigger = vm_cnt.v_page_count * 2 / usevnodes; 1850 if (force < 2) 1851 trigger = vsmalltrigger; 1852 reclaim_nc_src = force >= 3; 1853 target = rnumvnodes * (int64_t)gapvnodes / imax(desiredvnodes, 1); 1854 target = target / 10 + 1; 1855 done = vlrureclaim(reclaim_nc_src, trigger, target); 1856 mtx_unlock(&vnode_list_mtx); 1857 /* 1858 * Total number of vnodes can transiently go slightly above the 1859 * limit (see vn_alloc_hard), no need to call uma_reclaim if 1860 * this happens. 1861 */ 1862 if (onumvnodes + VNLRU_COUNT_SLOP + 1000 > desiredvnodes && 1863 numvnodes <= desiredvnodes) { 1864 uma_reclaim_calls++; 1865 uma_reclaim(UMA_RECLAIM_DRAIN); 1866 } 1867 if (done == 0) { 1868 if (force == 0 || force == 1) { 1869 force = 2; 1870 continue; 1871 } 1872 if (force == 2) { 1873 force = 3; 1874 continue; 1875 } 1876 want_reread = true; 1877 force = 0; 1878 vnlru_nowhere++; 1879 tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3); 1880 } else { 1881 want_reread = true; 1882 kern_yield(PRI_USER); 1883 } 1884 } 1885 } 1886 1887 static struct kproc_desc vnlru_kp = { 1888 "vnlru", 1889 vnlru_proc, 1890 &vnlruproc 1891 }; 1892 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, 1893 &vnlru_kp); 1894 1895 /* 1896 * Routines having to do with the management of the vnode table. 1897 */ 1898 1899 /* 1900 * Try to recycle a freed vnode. 1901 */ 1902 static int 1903 vtryrecycle(struct vnode *vp, bool isvnlru) 1904 { 1905 struct mount *vnmp; 1906 1907 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 1908 VNPASS(vp->v_holdcnt > 0, vp); 1909 /* 1910 * This vnode may found and locked via some other list, if so we 1911 * can't recycle it yet. 1912 */ 1913 if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { 1914 CTR2(KTR_VFS, 1915 "%s: impossible to recycle, vp %p lock is already held", 1916 __func__, vp); 1917 vdrop_recycle(vp); 1918 return (EWOULDBLOCK); 1919 } 1920 /* 1921 * Don't recycle if its filesystem is being suspended. 1922 */ 1923 if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) { 1924 VOP_UNLOCK(vp); 1925 CTR2(KTR_VFS, 1926 "%s: impossible to recycle, cannot start the write for %p", 1927 __func__, vp); 1928 vdrop_recycle(vp); 1929 return (EBUSY); 1930 } 1931 /* 1932 * If we got this far, we need to acquire the interlock and see if 1933 * anyone picked up this vnode from another list. If not, we will 1934 * mark it with DOOMED via vgonel() so that anyone who does find it 1935 * will skip over it. 1936 */ 1937 VI_LOCK(vp); 1938 if (vp->v_usecount) { 1939 VOP_UNLOCK(vp); 1940 vdropl_recycle(vp); 1941 vn_finished_write(vnmp); 1942 CTR2(KTR_VFS, 1943 "%s: impossible to recycle, %p is already referenced", 1944 __func__, vp); 1945 return (EBUSY); 1946 } 1947 if (!VN_IS_DOOMED(vp)) { 1948 if (isvnlru) 1949 recycles_free_count++; 1950 else 1951 counter_u64_add(direct_recycles_free_count, 1); 1952 vgonel(vp); 1953 } 1954 VOP_UNLOCK(vp); 1955 vdropl_recycle(vp); 1956 vn_finished_write(vnmp); 1957 return (0); 1958 } 1959 1960 /* 1961 * Allocate a new vnode. 1962 * 1963 * The operation never returns an error. Returning an error was disabled 1964 * in r145385 (dated 2005) with the following comment: 1965 * 1966 * XXX Not all VFS_VGET/ffs_vget callers check returns. 1967 * 1968 * Given the age of this commit (almost 15 years at the time of writing this 1969 * comment) restoring the ability to fail requires a significant audit of 1970 * all codepaths. 1971 * 1972 * The routine can try to free a vnode or stall for up to 1 second waiting for 1973 * vnlru to clear things up, but ultimately always performs a M_WAITOK allocation. 1974 */ 1975 static u_long vn_alloc_cyclecount; 1976 static u_long vn_alloc_sleeps; 1977 1978 SYSCTL_ULONG(_vfs_vnode_stats, OID_AUTO, alloc_sleeps, CTLFLAG_RD, &vn_alloc_sleeps, 0, 1979 "Number of times vnode allocation blocked waiting on vnlru"); 1980 1981 static struct vnode * __noinline 1982 vn_alloc_hard(struct mount *mp, u_long rnumvnodes, bool bumped) 1983 { 1984 u_long rfreevnodes; 1985 1986 if (bumped) { 1987 if (rnumvnodes > desiredvnodes + VNLRU_COUNT_SLOP) { 1988 atomic_subtract_long(&numvnodes, 1); 1989 bumped = false; 1990 } 1991 } 1992 1993 mtx_lock(&vnode_list_mtx); 1994 1995 rfreevnodes = vnlru_read_freevnodes(); 1996 if (vn_alloc_cyclecount++ >= rfreevnodes) { 1997 vn_alloc_cyclecount = 0; 1998 vstir = true; 1999 } 2000 /* 2001 * Grow the vnode cache if it will not be above its target max after 2002 * growing. Otherwise, if there is at least one free vnode, try to 2003 * reclaim 1 item from it before growing the cache (possibly above its 2004 * target max if the reclamation failed or is delayed). 2005 */ 2006 if (vnlru_free_locked_direct(1) > 0) 2007 goto alloc; 2008 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 2009 if (mp == NULL || (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 2010 /* 2011 * Wait for space for a new vnode. 2012 */ 2013 if (bumped) { 2014 atomic_subtract_long(&numvnodes, 1); 2015 bumped = false; 2016 } 2017 mtx_lock(&vnode_list_mtx); 2018 vnlru_kick_locked(); 2019 vn_alloc_sleeps++; 2020 msleep(&vnlruproc_sig, &vnode_list_mtx, PVFS, "vlruwk", hz); 2021 if (atomic_load_long(&numvnodes) + 1 > desiredvnodes && 2022 vnlru_read_freevnodes() > 1) 2023 vnlru_free_locked_direct(1); 2024 else 2025 mtx_unlock(&vnode_list_mtx); 2026 } 2027 alloc: 2028 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 2029 if (!bumped) 2030 atomic_add_long(&numvnodes, 1); 2031 vnlru_kick_cond(); 2032 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 2033 } 2034 2035 static struct vnode * 2036 vn_alloc(struct mount *mp) 2037 { 2038 u_long rnumvnodes; 2039 2040 if (__predict_false(vn_alloc_cyclecount != 0)) 2041 return (vn_alloc_hard(mp, 0, false)); 2042 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 2043 if (__predict_false(vnlru_under(rnumvnodes, vlowat))) { 2044 return (vn_alloc_hard(mp, rnumvnodes, true)); 2045 } 2046 2047 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 2048 } 2049 2050 static void 2051 vn_free(struct vnode *vp) 2052 { 2053 2054 atomic_subtract_long(&numvnodes, 1); 2055 uma_zfree_smr(vnode_zone, vp); 2056 } 2057 2058 /* 2059 * Allocate a new vnode. 2060 */ 2061 int 2062 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops, 2063 struct vnode **vpp) 2064 { 2065 struct vnode *vp; 2066 struct thread *td; 2067 struct lock_object *lo; 2068 2069 CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag); 2070 2071 KASSERT(vops->registered, 2072 ("%s: not registered vector op %p\n", __func__, vops)); 2073 cache_validate_vop_vector(mp, vops); 2074 2075 td = curthread; 2076 if (td->td_vp_reserved != NULL) { 2077 vp = td->td_vp_reserved; 2078 td->td_vp_reserved = NULL; 2079 } else { 2080 vp = vn_alloc(mp); 2081 } 2082 counter_u64_add(vnodes_created, 1); 2083 2084 vn_set_state(vp, VSTATE_UNINITIALIZED); 2085 2086 /* 2087 * Locks are given the generic name "vnode" when created. 2088 * Follow the historic practice of using the filesystem 2089 * name when they allocated, e.g., "zfs", "ufs", "nfs, etc. 2090 * 2091 * Locks live in a witness group keyed on their name. Thus, 2092 * when a lock is renamed, it must also move from the witness 2093 * group of its old name to the witness group of its new name. 2094 * 2095 * The change only needs to be made when the vnode moves 2096 * from one filesystem type to another. We ensure that each 2097 * filesystem use a single static name pointer for its tag so 2098 * that we can compare pointers rather than doing a strcmp(). 2099 */ 2100 lo = &vp->v_vnlock->lock_object; 2101 #ifdef WITNESS 2102 if (lo->lo_name != tag) { 2103 #endif 2104 lo->lo_name = tag; 2105 #ifdef WITNESS 2106 WITNESS_DESTROY(lo); 2107 WITNESS_INIT(lo, tag); 2108 } 2109 #endif 2110 /* 2111 * By default, don't allow shared locks unless filesystems opt-in. 2112 */ 2113 vp->v_vnlock->lock_object.lo_flags |= LK_NOSHARE; 2114 /* 2115 * Finalize various vnode identity bits. 2116 */ 2117 KASSERT(vp->v_object == NULL, ("stale v_object %p", vp)); 2118 KASSERT(vp->v_lockf == NULL, ("stale v_lockf %p", vp)); 2119 KASSERT(vp->v_pollinfo == NULL, ("stale v_pollinfo %p", vp)); 2120 vp->v_type = VNON; 2121 vp->v_op = vops; 2122 vp->v_irflag = 0; 2123 v_init_counters(vp); 2124 vn_seqc_init(vp); 2125 vp->v_bufobj.bo_ops = &buf_ops_bio; 2126 #ifdef DIAGNOSTIC 2127 if (mp == NULL && vops != &dead_vnodeops) 2128 printf("NULL mp in getnewvnode(9), tag %s\n", tag); 2129 #endif 2130 #ifdef MAC 2131 mac_vnode_init(vp); 2132 if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0) 2133 mac_vnode_associate_singlelabel(mp, vp); 2134 #endif 2135 if (mp != NULL) { 2136 vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize; 2137 } 2138 2139 /* 2140 * For the filesystems which do not use vfs_hash_insert(), 2141 * still initialize v_hash to have vfs_hash_index() useful. 2142 * E.g., nullfs uses vfs_hash_index() on the lower vnode for 2143 * its own hashing. 2144 */ 2145 vp->v_hash = (uintptr_t)vp >> vnsz2log; 2146 2147 *vpp = vp; 2148 return (0); 2149 } 2150 2151 void 2152 getnewvnode_reserve(void) 2153 { 2154 struct thread *td; 2155 2156 td = curthread; 2157 MPASS(td->td_vp_reserved == NULL); 2158 td->td_vp_reserved = vn_alloc(NULL); 2159 } 2160 2161 void 2162 getnewvnode_drop_reserve(void) 2163 { 2164 struct thread *td; 2165 2166 td = curthread; 2167 if (td->td_vp_reserved != NULL) { 2168 vn_free(td->td_vp_reserved); 2169 td->td_vp_reserved = NULL; 2170 } 2171 } 2172 2173 static void __noinline 2174 freevnode(struct vnode *vp) 2175 { 2176 struct bufobj *bo; 2177 2178 /* 2179 * The vnode has been marked for destruction, so free it. 2180 * 2181 * The vnode will be returned to the zone where it will 2182 * normally remain until it is needed for another vnode. We 2183 * need to cleanup (or verify that the cleanup has already 2184 * been done) any residual data left from its current use 2185 * so as not to contaminate the freshly allocated vnode. 2186 */ 2187 CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); 2188 /* 2189 * Paired with vgone. 2190 */ 2191 vn_seqc_write_end_free(vp); 2192 2193 bo = &vp->v_bufobj; 2194 VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); 2195 VNPASS(vp->v_holdcnt == VHOLD_NO_SMR, vp); 2196 VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); 2197 VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); 2198 VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); 2199 VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); 2200 VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp, 2201 ("clean blk trie not empty")); 2202 VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0")); 2203 VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp, 2204 ("dirty blk trie not empty")); 2205 VNASSERT((vp->v_iflag & (VI_DOINGINACT | VI_OWEINACT)) == 0, vp, 2206 ("Leaked inactivation")); 2207 VI_UNLOCK(vp); 2208 cache_assert_no_entries(vp); 2209 2210 #ifdef MAC 2211 mac_vnode_destroy(vp); 2212 #endif 2213 if (vp->v_pollinfo != NULL) { 2214 /* 2215 * Use LK_NOWAIT to shut up witness about the lock. We may get 2216 * here while having another vnode locked when trying to 2217 * satisfy a lookup and needing to recycle. 2218 */ 2219 VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT); 2220 destroy_vpollinfo(vp->v_pollinfo); 2221 VOP_UNLOCK(vp); 2222 vp->v_pollinfo = NULL; 2223 } 2224 vp->v_mountedhere = NULL; 2225 vp->v_unpcb = NULL; 2226 vp->v_rdev = NULL; 2227 vp->v_fifoinfo = NULL; 2228 vp->v_iflag = 0; 2229 vp->v_vflag = 0; 2230 bo->bo_flag = 0; 2231 vn_free(vp); 2232 } 2233 2234 /* 2235 * Delete from old mount point vnode list, if on one. 2236 */ 2237 static void 2238 delmntque(struct vnode *vp) 2239 { 2240 struct mount *mp; 2241 2242 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 2243 2244 mp = vp->v_mount; 2245 MNT_ILOCK(mp); 2246 VI_LOCK(vp); 2247 vp->v_mount = NULL; 2248 VNASSERT(mp->mnt_nvnodelistsize > 0, vp, 2249 ("bad mount point vnode list size")); 2250 TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 2251 mp->mnt_nvnodelistsize--; 2252 MNT_REL(mp); 2253 MNT_IUNLOCK(mp); 2254 /* 2255 * The caller expects the interlock to be still held. 2256 */ 2257 ASSERT_VI_LOCKED(vp, __func__); 2258 } 2259 2260 static int 2261 insmntque1_int(struct vnode *vp, struct mount *mp, bool dtr) 2262 { 2263 2264 KASSERT(vp->v_mount == NULL, 2265 ("insmntque: vnode already on per mount vnode list")); 2266 VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)")); 2267 if ((mp->mnt_kern_flag & MNTK_UNLOCKED_INSMNTQUE) == 0) { 2268 ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp"); 2269 } else { 2270 KASSERT(!dtr, 2271 ("%s: can't have MNTK_UNLOCKED_INSMNTQUE and cleanup", 2272 __func__)); 2273 } 2274 2275 /* 2276 * We acquire the vnode interlock early to ensure that the 2277 * vnode cannot be recycled by another process releasing a 2278 * holdcnt on it before we get it on both the vnode list 2279 * and the active vnode list. The mount mutex protects only 2280 * manipulation of the vnode list and the vnode freelist 2281 * mutex protects only manipulation of the active vnode list. 2282 * Hence the need to hold the vnode interlock throughout. 2283 */ 2284 MNT_ILOCK(mp); 2285 VI_LOCK(vp); 2286 if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 && 2287 ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 || 2288 mp->mnt_nvnodelistsize == 0)) && 2289 (vp->v_vflag & VV_FORCEINSMQ) == 0) { 2290 VI_UNLOCK(vp); 2291 MNT_IUNLOCK(mp); 2292 if (dtr) { 2293 vp->v_data = NULL; 2294 vp->v_op = &dead_vnodeops; 2295 vgone(vp); 2296 vput(vp); 2297 } 2298 return (EBUSY); 2299 } 2300 vp->v_mount = mp; 2301 MNT_REF(mp); 2302 TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 2303 VNASSERT(mp->mnt_nvnodelistsize >= 0, vp, 2304 ("neg mount point vnode list size")); 2305 mp->mnt_nvnodelistsize++; 2306 VI_UNLOCK(vp); 2307 MNT_IUNLOCK(mp); 2308 return (0); 2309 } 2310 2311 /* 2312 * Insert into list of vnodes for the new mount point, if available. 2313 * insmntque() reclaims the vnode on insertion failure, insmntque1() 2314 * leaves handling of the vnode to the caller. 2315 */ 2316 int 2317 insmntque(struct vnode *vp, struct mount *mp) 2318 { 2319 return (insmntque1_int(vp, mp, true)); 2320 } 2321 2322 int 2323 insmntque1(struct vnode *vp, struct mount *mp) 2324 { 2325 return (insmntque1_int(vp, mp, false)); 2326 } 2327 2328 /* 2329 * Flush out and invalidate all buffers associated with a bufobj 2330 * Called with the underlying object locked. 2331 */ 2332 int 2333 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo) 2334 { 2335 int error; 2336 2337 BO_LOCK(bo); 2338 if (flags & V_SAVE) { 2339 error = bufobj_wwait(bo, slpflag, slptimeo); 2340 if (error) { 2341 BO_UNLOCK(bo); 2342 return (error); 2343 } 2344 if (bo->bo_dirty.bv_cnt > 0) { 2345 BO_UNLOCK(bo); 2346 do { 2347 error = BO_SYNC(bo, MNT_WAIT); 2348 } while (error == ERELOOKUP); 2349 if (error != 0) 2350 return (error); 2351 BO_LOCK(bo); 2352 if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0) { 2353 BO_UNLOCK(bo); 2354 return (EBUSY); 2355 } 2356 } 2357 } 2358 /* 2359 * If you alter this loop please notice that interlock is dropped and 2360 * reacquired in flushbuflist. Special care is needed to ensure that 2361 * no race conditions occur from this. 2362 */ 2363 do { 2364 error = flushbuflist(&bo->bo_clean, 2365 flags, bo, slpflag, slptimeo); 2366 if (error == 0 && !(flags & V_CLEANONLY)) 2367 error = flushbuflist(&bo->bo_dirty, 2368 flags, bo, slpflag, slptimeo); 2369 if (error != 0 && error != EAGAIN) { 2370 BO_UNLOCK(bo); 2371 return (error); 2372 } 2373 } while (error != 0); 2374 2375 /* 2376 * Wait for I/O to complete. XXX needs cleaning up. The vnode can 2377 * have write I/O in-progress but if there is a VM object then the 2378 * VM object can also have read-I/O in-progress. 2379 */ 2380 do { 2381 bufobj_wwait(bo, 0, 0); 2382 if ((flags & V_VMIO) == 0 && bo->bo_object != NULL) { 2383 BO_UNLOCK(bo); 2384 vm_object_pip_wait_unlocked(bo->bo_object, "bovlbx"); 2385 BO_LOCK(bo); 2386 } 2387 } while (bo->bo_numoutput > 0); 2388 BO_UNLOCK(bo); 2389 2390 /* 2391 * Destroy the copy in the VM cache, too. 2392 */ 2393 if (bo->bo_object != NULL && 2394 (flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0) { 2395 VM_OBJECT_WLOCK(bo->bo_object); 2396 vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ? 2397 OBJPR_CLEANONLY : 0); 2398 VM_OBJECT_WUNLOCK(bo->bo_object); 2399 } 2400 2401 #ifdef INVARIANTS 2402 BO_LOCK(bo); 2403 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO | 2404 V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 || 2405 bo->bo_clean.bv_cnt > 0)) 2406 panic("vinvalbuf: flush failed"); 2407 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0 && 2408 bo->bo_dirty.bv_cnt > 0) 2409 panic("vinvalbuf: flush dirty failed"); 2410 BO_UNLOCK(bo); 2411 #endif 2412 return (0); 2413 } 2414 2415 /* 2416 * Flush out and invalidate all buffers associated with a vnode. 2417 * Called with the underlying object locked. 2418 */ 2419 int 2420 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) 2421 { 2422 2423 CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags); 2424 ASSERT_VOP_LOCKED(vp, "vinvalbuf"); 2425 if (vp->v_object != NULL && vp->v_object->handle != vp) 2426 return (0); 2427 return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo)); 2428 } 2429 2430 /* 2431 * Flush out buffers on the specified list. 2432 * 2433 */ 2434 static int 2435 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, 2436 int slptimeo) 2437 { 2438 struct buf *bp, *nbp; 2439 int retval, error; 2440 daddr_t lblkno; 2441 b_xflags_t xflags; 2442 2443 ASSERT_BO_WLOCKED(bo); 2444 2445 retval = 0; 2446 TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) { 2447 /* 2448 * If we are flushing both V_NORMAL and V_ALT buffers then 2449 * do not skip any buffers. If we are flushing only V_NORMAL 2450 * buffers then skip buffers marked as BX_ALTDATA. If we are 2451 * flushing only V_ALT buffers then skip buffers not marked 2452 * as BX_ALTDATA. 2453 */ 2454 if (((flags & (V_NORMAL | V_ALT)) != (V_NORMAL | V_ALT)) && 2455 (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) || 2456 ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) { 2457 continue; 2458 } 2459 if (nbp != NULL) { 2460 lblkno = nbp->b_lblkno; 2461 xflags = nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN); 2462 } 2463 retval = EAGAIN; 2464 error = BUF_TIMELOCK(bp, 2465 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo), 2466 "flushbuf", slpflag, slptimeo); 2467 if (error) { 2468 BO_LOCK(bo); 2469 return (error != ENOLCK ? error : EAGAIN); 2470 } 2471 KASSERT(bp->b_bufobj == bo, 2472 ("bp %p wrong b_bufobj %p should be %p", 2473 bp, bp->b_bufobj, bo)); 2474 /* 2475 * XXX Since there are no node locks for NFS, I 2476 * believe there is a slight chance that a delayed 2477 * write will occur while sleeping just above, so 2478 * check for it. 2479 */ 2480 if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) && 2481 (flags & V_SAVE)) { 2482 bremfree(bp); 2483 bp->b_flags |= B_ASYNC; 2484 bwrite(bp); 2485 BO_LOCK(bo); 2486 return (EAGAIN); /* XXX: why not loop ? */ 2487 } 2488 bremfree(bp); 2489 bp->b_flags |= (B_INVAL | B_RELBUF); 2490 bp->b_flags &= ~B_ASYNC; 2491 brelse(bp); 2492 BO_LOCK(bo); 2493 if (nbp == NULL) 2494 break; 2495 nbp = gbincore(bo, lblkno); 2496 if (nbp == NULL || (nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) 2497 != xflags) 2498 break; /* nbp invalid */ 2499 } 2500 return (retval); 2501 } 2502 2503 int 2504 bnoreuselist(struct bufv *bufv, struct bufobj *bo, daddr_t startn, daddr_t endn) 2505 { 2506 struct buf *bp; 2507 int error; 2508 daddr_t lblkno; 2509 2510 ASSERT_BO_LOCKED(bo); 2511 2512 for (lblkno = startn;;) { 2513 again: 2514 bp = buf_lookup_ge(bufv, lblkno); 2515 if (bp == NULL || bp->b_lblkno >= endn) 2516 break; 2517 error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 2518 LK_INTERLOCK, BO_LOCKPTR(bo), "brlsfl", 0, 0); 2519 if (error != 0) { 2520 BO_RLOCK(bo); 2521 if (error == ENOLCK) 2522 goto again; 2523 return (error); 2524 } 2525 KASSERT(bp->b_bufobj == bo, 2526 ("bp %p wrong b_bufobj %p should be %p", 2527 bp, bp->b_bufobj, bo)); 2528 lblkno = bp->b_lblkno + 1; 2529 if ((bp->b_flags & B_MANAGED) == 0) 2530 bremfree(bp); 2531 bp->b_flags |= B_RELBUF; 2532 /* 2533 * In the VMIO case, use the B_NOREUSE flag to hint that the 2534 * pages backing each buffer in the range are unlikely to be 2535 * reused. Dirty buffers will have the hint applied once 2536 * they've been written. 2537 */ 2538 if ((bp->b_flags & B_VMIO) != 0) 2539 bp->b_flags |= B_NOREUSE; 2540 brelse(bp); 2541 BO_RLOCK(bo); 2542 } 2543 return (0); 2544 } 2545 2546 /* 2547 * Truncate a file's buffer and pages to a specified length. This 2548 * is in lieu of the old vinvalbuf mechanism, which performed unneeded 2549 * sync activity. 2550 */ 2551 int 2552 vtruncbuf(struct vnode *vp, off_t length, int blksize) 2553 { 2554 struct buf *bp, *nbp; 2555 struct bufobj *bo; 2556 daddr_t startlbn; 2557 2558 CTR4(KTR_VFS, "%s: vp %p with block %d:%ju", __func__, 2559 vp, blksize, (uintmax_t)length); 2560 2561 /* 2562 * Round up to the *next* lbn. 2563 */ 2564 startlbn = howmany(length, blksize); 2565 2566 ASSERT_VOP_LOCKED(vp, "vtruncbuf"); 2567 2568 bo = &vp->v_bufobj; 2569 restart_unlocked: 2570 BO_LOCK(bo); 2571 2572 while (v_inval_buf_range_locked(vp, bo, startlbn, INT64_MAX) == EAGAIN) 2573 ; 2574 2575 if (length > 0) { 2576 /* 2577 * Write out vnode metadata, e.g. indirect blocks. 2578 */ 2579 restartsync: 2580 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2581 if (bp->b_lblkno >= 0) 2582 continue; 2583 /* 2584 * Since we hold the vnode lock this should only 2585 * fail if we're racing with the buf daemon. 2586 */ 2587 if (BUF_LOCK(bp, 2588 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2589 BO_LOCKPTR(bo)) == ENOLCK) 2590 goto restart_unlocked; 2591 2592 VNASSERT((bp->b_flags & B_DELWRI), vp, 2593 ("buf(%p) on dirty queue without DELWRI", bp)); 2594 2595 bremfree(bp); 2596 bawrite(bp); 2597 BO_LOCK(bo); 2598 goto restartsync; 2599 } 2600 } 2601 2602 bufobj_wwait(bo, 0, 0); 2603 BO_UNLOCK(bo); 2604 vnode_pager_setsize(vp, length); 2605 2606 return (0); 2607 } 2608 2609 /* 2610 * Invalidate the cached pages of a file's buffer within the range of block 2611 * numbers [startlbn, endlbn). 2612 */ 2613 void 2614 v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn, 2615 int blksize) 2616 { 2617 struct bufobj *bo; 2618 off_t start, end; 2619 2620 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range"); 2621 2622 start = blksize * startlbn; 2623 end = blksize * endlbn; 2624 2625 bo = &vp->v_bufobj; 2626 BO_LOCK(bo); 2627 MPASS(blksize == bo->bo_bsize); 2628 2629 while (v_inval_buf_range_locked(vp, bo, startlbn, endlbn) == EAGAIN) 2630 ; 2631 2632 BO_UNLOCK(bo); 2633 vn_pages_remove(vp, OFF_TO_IDX(start), OFF_TO_IDX(end + PAGE_SIZE - 1)); 2634 } 2635 2636 static int 2637 v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 2638 daddr_t startlbn, daddr_t endlbn) 2639 { 2640 struct bufv *bv; 2641 struct buf *bp, *nbp; 2642 uint8_t anyfreed; 2643 bool clean; 2644 2645 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range_locked"); 2646 ASSERT_BO_LOCKED(bo); 2647 2648 anyfreed = 1; 2649 clean = true; 2650 do { 2651 bv = clean ? &bo->bo_clean : &bo->bo_dirty; 2652 bp = buf_lookup_ge(bv, startlbn); 2653 if (bp == NULL) 2654 continue; 2655 TAILQ_FOREACH_FROM_SAFE(bp, &bv->bv_hd, b_bobufs, nbp) { 2656 if (bp->b_lblkno >= endlbn) 2657 break; 2658 if (BUF_LOCK(bp, 2659 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2660 BO_LOCKPTR(bo)) == ENOLCK) { 2661 BO_LOCK(bo); 2662 return (EAGAIN); 2663 } 2664 2665 bremfree(bp); 2666 bp->b_flags |= B_INVAL | B_RELBUF; 2667 bp->b_flags &= ~B_ASYNC; 2668 brelse(bp); 2669 anyfreed = 2; 2670 2671 BO_LOCK(bo); 2672 if (nbp != NULL && 2673 (((nbp->b_xflags & 2674 (clean ? BX_VNCLEAN : BX_VNDIRTY)) == 0) || 2675 nbp->b_vp != vp || 2676 (nbp->b_flags & B_DELWRI) == (clean? B_DELWRI: 0))) 2677 return (EAGAIN); 2678 } 2679 } while (clean = !clean, anyfreed-- > 0); 2680 return (0); 2681 } 2682 2683 static void 2684 buf_vlist_remove(struct buf *bp) 2685 { 2686 struct bufv *bv; 2687 b_xflags_t flags; 2688 2689 flags = bp->b_xflags; 2690 2691 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp)); 2692 ASSERT_BO_WLOCKED(bp->b_bufobj); 2693 KASSERT((flags & (BX_VNDIRTY | BX_VNCLEAN)) != 0 && 2694 (flags & (BX_VNDIRTY | BX_VNCLEAN)) != (BX_VNDIRTY | BX_VNCLEAN), 2695 ("%s: buffer %p has invalid queue state", __func__, bp)); 2696 2697 if ((flags & BX_VNDIRTY) != 0) 2698 bv = &bp->b_bufobj->bo_dirty; 2699 else 2700 bv = &bp->b_bufobj->bo_clean; 2701 BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno); 2702 TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs); 2703 bv->bv_cnt--; 2704 bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN); 2705 } 2706 2707 /* 2708 * Add the buffer to the sorted clean or dirty block list. Return zero on 2709 * success, EEXIST if a buffer with this identity already exists, or another 2710 * error on allocation failure. 2711 */ 2712 static inline int 2713 buf_vlist_find_or_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags) 2714 { 2715 struct bufv *bv; 2716 struct buf *n; 2717 int error; 2718 2719 ASSERT_BO_WLOCKED(bo); 2720 KASSERT((bo->bo_flag & BO_NOBUFS) == 0, 2721 ("buf_vlist_add: bo %p does not allow bufs", bo)); 2722 KASSERT((xflags & BX_VNDIRTY) == 0 || (bo->bo_flag & BO_DEAD) == 0, 2723 ("dead bo %p", bo)); 2724 KASSERT((bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) == xflags, 2725 ("buf_vlist_add: b_xflags %#x not set on bp %p", xflags, bp)); 2726 2727 if (xflags & BX_VNDIRTY) 2728 bv = &bo->bo_dirty; 2729 else 2730 bv = &bo->bo_clean; 2731 2732 error = buf_insert_lookup_le(bv, bp, &n); 2733 if (n == NULL) { 2734 KASSERT(error != EEXIST, 2735 ("buf_vlist_add: EEXIST but no existing buf found: bp %p", 2736 bp)); 2737 } else { 2738 KASSERT(n->b_lblkno <= bp->b_lblkno, 2739 ("buf_vlist_add: out of order insert/lookup: bp %p n %p", 2740 bp, n)); 2741 KASSERT((n->b_lblkno == bp->b_lblkno) == (error == EEXIST), 2742 ("buf_vlist_add: inconsistent result for existing buf: " 2743 "error %d bp %p n %p", error, bp, n)); 2744 } 2745 if (error != 0) 2746 return (error); 2747 2748 /* Keep the list ordered. */ 2749 if (n == NULL) { 2750 KASSERT(TAILQ_EMPTY(&bv->bv_hd) || 2751 bp->b_lblkno < TAILQ_FIRST(&bv->bv_hd)->b_lblkno, 2752 ("buf_vlist_add: queue order: " 2753 "%p should be before first %p", 2754 bp, TAILQ_FIRST(&bv->bv_hd))); 2755 TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs); 2756 } else { 2757 KASSERT(TAILQ_NEXT(n, b_bobufs) == NULL || 2758 bp->b_lblkno < TAILQ_NEXT(n, b_bobufs)->b_lblkno, 2759 ("buf_vlist_add: queue order: " 2760 "%p should be before next %p", 2761 bp, TAILQ_NEXT(n, b_bobufs))); 2762 TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs); 2763 } 2764 2765 bv->bv_cnt++; 2766 return (0); 2767 } 2768 2769 /* 2770 * Add the buffer to the sorted clean or dirty block list. 2771 * 2772 * NOTE: xflags is passed as a constant, optimizing this inline function! 2773 */ 2774 static void 2775 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags) 2776 { 2777 int error; 2778 2779 KASSERT((bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) == 0, 2780 ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags)); 2781 bp->b_xflags |= xflags; 2782 error = buf_vlist_find_or_add(bp, bo, xflags); 2783 if (error) 2784 panic("buf_vlist_add: error=%d", error); 2785 } 2786 2787 /* 2788 * Look up a buffer using the buffer tries. 2789 */ 2790 struct buf * 2791 gbincore(struct bufobj *bo, daddr_t lblkno) 2792 { 2793 struct buf *bp; 2794 2795 ASSERT_BO_LOCKED(bo); 2796 bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno); 2797 if (bp != NULL) 2798 return (bp); 2799 return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno)); 2800 } 2801 2802 /* 2803 * Look up a buf using the buffer tries, without the bufobj lock. This relies 2804 * on SMR for safe lookup, and bufs being in a no-free zone to provide type 2805 * stability of the result. Like other lockless lookups, the found buf may 2806 * already be invalid by the time this function returns. 2807 */ 2808 struct buf * 2809 gbincore_unlocked(struct bufobj *bo, daddr_t lblkno) 2810 { 2811 struct buf *bp; 2812 2813 ASSERT_BO_UNLOCKED(bo); 2814 bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno); 2815 if (bp != NULL) 2816 return (bp); 2817 return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno)); 2818 } 2819 2820 /* 2821 * Associate a buffer with a vnode. 2822 */ 2823 int 2824 bgetvp(struct vnode *vp, struct buf *bp) 2825 { 2826 struct bufobj *bo; 2827 int error; 2828 2829 bo = &vp->v_bufobj; 2830 ASSERT_BO_UNLOCKED(bo); 2831 VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free")); 2832 2833 CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags); 2834 VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp, 2835 ("bgetvp: bp already attached! %p", bp)); 2836 2837 /* 2838 * Add the buf to the vnode's clean list unless we lost a race and find 2839 * an existing buf in either dirty or clean. 2840 */ 2841 bp->b_vp = vp; 2842 bp->b_bufobj = bo; 2843 bp->b_xflags |= BX_VNCLEAN; 2844 error = EEXIST; 2845 BO_LOCK(bo); 2846 if (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, bp->b_lblkno) == NULL) 2847 error = buf_vlist_find_or_add(bp, bo, BX_VNCLEAN); 2848 BO_UNLOCK(bo); 2849 if (__predict_true(error == 0)) { 2850 vhold(vp); 2851 return (0); 2852 } 2853 if (error != EEXIST) 2854 panic("bgetvp: buf_vlist_add error: %d", error); 2855 bp->b_vp = NULL; 2856 bp->b_bufobj = NULL; 2857 bp->b_xflags &= ~BX_VNCLEAN; 2858 return (error); 2859 } 2860 2861 /* 2862 * Disassociate a buffer from a vnode. 2863 */ 2864 void 2865 brelvp(struct buf *bp) 2866 { 2867 struct bufobj *bo; 2868 struct vnode *vp; 2869 2870 CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags); 2871 KASSERT(bp->b_vp != NULL, ("brelvp: NULL")); 2872 2873 /* 2874 * Delete from old vnode list, if on one. 2875 */ 2876 vp = bp->b_vp; /* XXX */ 2877 bo = bp->b_bufobj; 2878 BO_LOCK(bo); 2879 buf_vlist_remove(bp); 2880 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2881 bo->bo_flag &= ~BO_ONWORKLST; 2882 mtx_lock(&sync_mtx); 2883 LIST_REMOVE(bo, bo_synclist); 2884 syncer_worklist_len--; 2885 mtx_unlock(&sync_mtx); 2886 } 2887 bp->b_vp = NULL; 2888 bp->b_bufobj = NULL; 2889 BO_UNLOCK(bo); 2890 vdrop(vp); 2891 } 2892 2893 /* 2894 * Add an item to the syncer work queue. 2895 */ 2896 static void 2897 vn_syncer_add_to_worklist(struct bufobj *bo, int delay) 2898 { 2899 int slot; 2900 2901 ASSERT_BO_WLOCKED(bo); 2902 2903 mtx_lock(&sync_mtx); 2904 if (bo->bo_flag & BO_ONWORKLST) 2905 LIST_REMOVE(bo, bo_synclist); 2906 else { 2907 bo->bo_flag |= BO_ONWORKLST; 2908 syncer_worklist_len++; 2909 } 2910 2911 if (delay > syncer_maxdelay - 2) 2912 delay = syncer_maxdelay - 2; 2913 slot = (syncer_delayno + delay) & syncer_mask; 2914 2915 LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist); 2916 mtx_unlock(&sync_mtx); 2917 } 2918 2919 static int 2920 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS) 2921 { 2922 int error, len; 2923 2924 mtx_lock(&sync_mtx); 2925 len = syncer_worklist_len - sync_vnode_count; 2926 mtx_unlock(&sync_mtx); 2927 error = SYSCTL_OUT(req, &len, sizeof(len)); 2928 return (error); 2929 } 2930 2931 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, 2932 CTLTYPE_INT | CTLFLAG_MPSAFE| CTLFLAG_RD, NULL, 0, 2933 sysctl_vfs_worklist_len, "I", "Syncer thread worklist length"); 2934 2935 static struct proc *updateproc; 2936 static void sched_sync(void); 2937 static struct kproc_desc up_kp = { 2938 "syncer", 2939 sched_sync, 2940 &updateproc 2941 }; 2942 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp); 2943 2944 static int 2945 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td) 2946 { 2947 struct vnode *vp; 2948 struct mount *mp; 2949 2950 *bo = LIST_FIRST(slp); 2951 if (*bo == NULL) 2952 return (0); 2953 vp = bo2vnode(*bo); 2954 if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0) 2955 return (1); 2956 /* 2957 * We use vhold in case the vnode does not 2958 * successfully sync. vhold prevents the vnode from 2959 * going away when we unlock the sync_mtx so that 2960 * we can acquire the vnode interlock. 2961 */ 2962 vholdl(vp); 2963 mtx_unlock(&sync_mtx); 2964 VI_UNLOCK(vp); 2965 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 2966 vdrop(vp); 2967 mtx_lock(&sync_mtx); 2968 return (*bo == LIST_FIRST(slp)); 2969 } 2970 MPASSERT(mp == NULL || (curthread->td_pflags & TDP_IGNSUSP) != 0 || 2971 (mp->mnt_kern_flag & MNTK_SUSPENDED) == 0, mp, 2972 ("suspended mp syncing vp %p", vp)); 2973 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2974 (void) VOP_FSYNC(vp, MNT_LAZY, td); 2975 VOP_UNLOCK(vp); 2976 vn_finished_write(mp); 2977 BO_LOCK(*bo); 2978 if (((*bo)->bo_flag & BO_ONWORKLST) != 0) { 2979 /* 2980 * Put us back on the worklist. The worklist 2981 * routine will remove us from our current 2982 * position and then add us back in at a later 2983 * position. 2984 */ 2985 vn_syncer_add_to_worklist(*bo, syncdelay); 2986 } 2987 BO_UNLOCK(*bo); 2988 vdrop(vp); 2989 mtx_lock(&sync_mtx); 2990 return (0); 2991 } 2992 2993 static int first_printf = 1; 2994 2995 /* 2996 * System filesystem synchronizer daemon. 2997 */ 2998 static void 2999 sched_sync(void) 3000 { 3001 struct synclist *next, *slp; 3002 struct bufobj *bo; 3003 long starttime; 3004 struct thread *td = curthread; 3005 int last_work_seen; 3006 int net_worklist_len; 3007 int syncer_final_iter; 3008 int error; 3009 3010 last_work_seen = 0; 3011 syncer_final_iter = 0; 3012 syncer_state = SYNCER_RUNNING; 3013 starttime = time_uptime; 3014 td->td_pflags |= TDP_NORUNNINGBUF; 3015 3016 EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc, 3017 SHUTDOWN_PRI_LAST); 3018 3019 mtx_lock(&sync_mtx); 3020 for (;;) { 3021 if (syncer_state == SYNCER_FINAL_DELAY && 3022 syncer_final_iter == 0) { 3023 mtx_unlock(&sync_mtx); 3024 kproc_suspend_check(td->td_proc); 3025 mtx_lock(&sync_mtx); 3026 } 3027 net_worklist_len = syncer_worklist_len - sync_vnode_count; 3028 if (syncer_state != SYNCER_RUNNING && 3029 starttime != time_uptime) { 3030 if (first_printf) { 3031 printf("\nSyncing disks, vnodes remaining... "); 3032 first_printf = 0; 3033 } 3034 printf("%d ", net_worklist_len); 3035 } 3036 starttime = time_uptime; 3037 3038 /* 3039 * Push files whose dirty time has expired. Be careful 3040 * of interrupt race on slp queue. 3041 * 3042 * Skip over empty worklist slots when shutting down. 3043 */ 3044 do { 3045 slp = &syncer_workitem_pending[syncer_delayno]; 3046 syncer_delayno += 1; 3047 if (syncer_delayno == syncer_maxdelay) 3048 syncer_delayno = 0; 3049 next = &syncer_workitem_pending[syncer_delayno]; 3050 /* 3051 * If the worklist has wrapped since the 3052 * it was emptied of all but syncer vnodes, 3053 * switch to the FINAL_DELAY state and run 3054 * for one more second. 3055 */ 3056 if (syncer_state == SYNCER_SHUTTING_DOWN && 3057 net_worklist_len == 0 && 3058 last_work_seen == syncer_delayno) { 3059 syncer_state = SYNCER_FINAL_DELAY; 3060 syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP; 3061 } 3062 } while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) && 3063 syncer_worklist_len > 0); 3064 3065 /* 3066 * Keep track of the last time there was anything 3067 * on the worklist other than syncer vnodes. 3068 * Return to the SHUTTING_DOWN state if any 3069 * new work appears. 3070 */ 3071 if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING) 3072 last_work_seen = syncer_delayno; 3073 if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY) 3074 syncer_state = SYNCER_SHUTTING_DOWN; 3075 while (!LIST_EMPTY(slp)) { 3076 error = sync_vnode(slp, &bo, td); 3077 if (error == 1) { 3078 LIST_REMOVE(bo, bo_synclist); 3079 LIST_INSERT_HEAD(next, bo, bo_synclist); 3080 continue; 3081 } 3082 3083 if (first_printf == 0) { 3084 /* 3085 * Drop the sync mutex, because some watchdog 3086 * drivers need to sleep while patting 3087 */ 3088 mtx_unlock(&sync_mtx); 3089 wdog_kern_pat(WD_LASTVAL); 3090 mtx_lock(&sync_mtx); 3091 } 3092 } 3093 if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0) 3094 syncer_final_iter--; 3095 /* 3096 * The variable rushjob allows the kernel to speed up the 3097 * processing of the filesystem syncer process. A rushjob 3098 * value of N tells the filesystem syncer to process the next 3099 * N seconds worth of work on its queue ASAP. Currently rushjob 3100 * is used by the soft update code to speed up the filesystem 3101 * syncer process when the incore state is getting so far 3102 * ahead of the disk that the kernel memory pool is being 3103 * threatened with exhaustion. 3104 */ 3105 if (rushjob > 0) { 3106 rushjob -= 1; 3107 continue; 3108 } 3109 /* 3110 * Just sleep for a short period of time between 3111 * iterations when shutting down to allow some I/O 3112 * to happen. 3113 * 3114 * If it has taken us less than a second to process the 3115 * current work, then wait. Otherwise start right over 3116 * again. We can still lose time if any single round 3117 * takes more than two seconds, but it does not really 3118 * matter as we are just trying to generally pace the 3119 * filesystem activity. 3120 */ 3121 if (syncer_state != SYNCER_RUNNING || 3122 time_uptime == starttime) { 3123 thread_lock(td); 3124 sched_prio(td, PPAUSE); 3125 thread_unlock(td); 3126 } 3127 if (syncer_state != SYNCER_RUNNING) 3128 cv_timedwait(&sync_wakeup, &sync_mtx, 3129 hz / SYNCER_SHUTDOWN_SPEEDUP); 3130 else if (time_uptime == starttime) 3131 cv_timedwait(&sync_wakeup, &sync_mtx, hz); 3132 } 3133 } 3134 3135 /* 3136 * Request the syncer daemon to speed up its work. 3137 * We never push it to speed up more than half of its 3138 * normal turn time, otherwise it could take over the cpu. 3139 */ 3140 int 3141 speedup_syncer(void) 3142 { 3143 int ret = 0; 3144 3145 mtx_lock(&sync_mtx); 3146 if (rushjob < syncdelay / 2) { 3147 rushjob += 1; 3148 stat_rush_requests += 1; 3149 ret = 1; 3150 } 3151 mtx_unlock(&sync_mtx); 3152 cv_broadcast(&sync_wakeup); 3153 return (ret); 3154 } 3155 3156 /* 3157 * Tell the syncer to speed up its work and run though its work 3158 * list several times, then tell it to shut down. 3159 */ 3160 static void 3161 syncer_shutdown(void *arg, int howto) 3162 { 3163 3164 if (howto & RB_NOSYNC) 3165 return; 3166 mtx_lock(&sync_mtx); 3167 syncer_state = SYNCER_SHUTTING_DOWN; 3168 rushjob = 0; 3169 mtx_unlock(&sync_mtx); 3170 cv_broadcast(&sync_wakeup); 3171 kproc_shutdown(arg, howto); 3172 } 3173 3174 void 3175 syncer_suspend(void) 3176 { 3177 3178 syncer_shutdown(updateproc, 0); 3179 } 3180 3181 void 3182 syncer_resume(void) 3183 { 3184 3185 mtx_lock(&sync_mtx); 3186 first_printf = 1; 3187 syncer_state = SYNCER_RUNNING; 3188 mtx_unlock(&sync_mtx); 3189 cv_broadcast(&sync_wakeup); 3190 kproc_resume(updateproc); 3191 } 3192 3193 /* 3194 * Move the buffer between the clean and dirty lists of its vnode. 3195 */ 3196 void 3197 reassignbuf(struct buf *bp) 3198 { 3199 struct vnode *vp; 3200 struct bufobj *bo; 3201 int delay; 3202 #ifdef INVARIANTS 3203 struct bufv *bv; 3204 #endif 3205 3206 vp = bp->b_vp; 3207 bo = bp->b_bufobj; 3208 3209 KASSERT((bp->b_flags & B_PAGING) == 0, 3210 ("%s: cannot reassign paging buffer %p", __func__, bp)); 3211 3212 CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X", 3213 bp, bp->b_vp, bp->b_flags); 3214 3215 BO_LOCK(bo); 3216 if ((bo->bo_flag & BO_NONSTERILE) == 0) { 3217 /* 3218 * Coordinate with getblk's unlocked lookup. Make 3219 * BO_NONSTERILE visible before the first reassignbuf produces 3220 * any side effect. This could be outside the bo lock if we 3221 * used a separate atomic flag field. 3222 */ 3223 bo->bo_flag |= BO_NONSTERILE; 3224 atomic_thread_fence_rel(); 3225 } 3226 buf_vlist_remove(bp); 3227 3228 /* 3229 * If dirty, put on list of dirty buffers; otherwise insert onto list 3230 * of clean buffers. 3231 */ 3232 if (bp->b_flags & B_DELWRI) { 3233 if ((bo->bo_flag & BO_ONWORKLST) == 0) { 3234 switch (vp->v_type) { 3235 case VDIR: 3236 delay = dirdelay; 3237 break; 3238 case VCHR: 3239 delay = metadelay; 3240 break; 3241 default: 3242 delay = filedelay; 3243 } 3244 vn_syncer_add_to_worklist(bo, delay); 3245 } 3246 buf_vlist_add(bp, bo, BX_VNDIRTY); 3247 } else { 3248 buf_vlist_add(bp, bo, BX_VNCLEAN); 3249 3250 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 3251 mtx_lock(&sync_mtx); 3252 LIST_REMOVE(bo, bo_synclist); 3253 syncer_worklist_len--; 3254 mtx_unlock(&sync_mtx); 3255 bo->bo_flag &= ~BO_ONWORKLST; 3256 } 3257 } 3258 #ifdef INVARIANTS 3259 bv = &bo->bo_clean; 3260 bp = TAILQ_FIRST(&bv->bv_hd); 3261 KASSERT(bp == NULL || bp->b_bufobj == bo, 3262 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3263 bp = TAILQ_LAST(&bv->bv_hd, buflists); 3264 KASSERT(bp == NULL || bp->b_bufobj == bo, 3265 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3266 bv = &bo->bo_dirty; 3267 bp = TAILQ_FIRST(&bv->bv_hd); 3268 KASSERT(bp == NULL || bp->b_bufobj == bo, 3269 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3270 bp = TAILQ_LAST(&bv->bv_hd, buflists); 3271 KASSERT(bp == NULL || bp->b_bufobj == bo, 3272 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3273 #endif 3274 BO_UNLOCK(bo); 3275 } 3276 3277 static void 3278 v_init_counters(struct vnode *vp) 3279 { 3280 3281 VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0, 3282 vp, ("%s called for an initialized vnode", __FUNCTION__)); 3283 ASSERT_VI_UNLOCKED(vp, __FUNCTION__); 3284 3285 refcount_init(&vp->v_holdcnt, 1); 3286 refcount_init(&vp->v_usecount, 1); 3287 } 3288 3289 /* 3290 * Get a usecount on a vnode. 3291 * 3292 * vget and vget_finish may fail to lock the vnode if they lose a race against 3293 * it being doomed. LK_RETRY can be passed in flags to lock it anyway. 3294 * 3295 * Consumers which don't guarantee liveness of the vnode can use SMR to 3296 * try to get a reference. Note this operation can fail since the vnode 3297 * may be awaiting getting freed by the time they get to it. 3298 */ 3299 enum vgetstate 3300 vget_prep_smr(struct vnode *vp) 3301 { 3302 enum vgetstate vs; 3303 3304 VFS_SMR_ASSERT_ENTERED(); 3305 3306 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 3307 vs = VGET_USECOUNT; 3308 } else { 3309 if (vhold_smr(vp)) 3310 vs = VGET_HOLDCNT; 3311 else 3312 vs = VGET_NONE; 3313 } 3314 return (vs); 3315 } 3316 3317 enum vgetstate 3318 vget_prep(struct vnode *vp) 3319 { 3320 enum vgetstate vs; 3321 3322 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 3323 vs = VGET_USECOUNT; 3324 } else { 3325 vhold(vp); 3326 vs = VGET_HOLDCNT; 3327 } 3328 return (vs); 3329 } 3330 3331 void 3332 vget_abort(struct vnode *vp, enum vgetstate vs) 3333 { 3334 3335 switch (vs) { 3336 case VGET_USECOUNT: 3337 vrele(vp); 3338 break; 3339 case VGET_HOLDCNT: 3340 vdrop(vp); 3341 break; 3342 default: 3343 __assert_unreachable(); 3344 } 3345 } 3346 3347 int 3348 vget(struct vnode *vp, int flags) 3349 { 3350 enum vgetstate vs; 3351 3352 vs = vget_prep(vp); 3353 return (vget_finish(vp, flags, vs)); 3354 } 3355 3356 int 3357 vget_finish(struct vnode *vp, int flags, enum vgetstate vs) 3358 { 3359 int error; 3360 3361 if ((flags & LK_INTERLOCK) != 0) 3362 ASSERT_VI_LOCKED(vp, __func__); 3363 else 3364 ASSERT_VI_UNLOCKED(vp, __func__); 3365 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3366 VNPASS(vp->v_holdcnt > 0, vp); 3367 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3368 3369 error = vn_lock(vp, flags); 3370 if (__predict_false(error != 0)) { 3371 vget_abort(vp, vs); 3372 CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__, 3373 vp); 3374 return (error); 3375 } 3376 3377 vget_finish_ref(vp, vs); 3378 return (0); 3379 } 3380 3381 void 3382 vget_finish_ref(struct vnode *vp, enum vgetstate vs) 3383 { 3384 int old; 3385 3386 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3387 VNPASS(vp->v_holdcnt > 0, vp); 3388 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3389 3390 if (vs == VGET_USECOUNT) 3391 return; 3392 3393 /* 3394 * We hold the vnode. If the usecount is 0 it will be utilized to keep 3395 * the vnode around. Otherwise someone else lended their hold count and 3396 * we have to drop ours. 3397 */ 3398 old = atomic_fetchadd_int(&vp->v_usecount, 1); 3399 VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old)); 3400 if (old != 0) { 3401 #ifdef INVARIANTS 3402 old = atomic_fetchadd_int(&vp->v_holdcnt, -1); 3403 VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); 3404 #else 3405 refcount_release(&vp->v_holdcnt); 3406 #endif 3407 } 3408 } 3409 3410 void 3411 vref(struct vnode *vp) 3412 { 3413 enum vgetstate vs; 3414 3415 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3416 vs = vget_prep(vp); 3417 vget_finish_ref(vp, vs); 3418 } 3419 3420 void 3421 vrefact(struct vnode *vp) 3422 { 3423 int old __diagused; 3424 3425 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3426 old = refcount_acquire(&vp->v_usecount); 3427 VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old)); 3428 } 3429 3430 void 3431 vlazy(struct vnode *vp) 3432 { 3433 struct mount *mp; 3434 3435 VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); 3436 3437 if ((vp->v_mflag & VMP_LAZYLIST) != 0) 3438 return; 3439 /* 3440 * We may get here for inactive routines after the vnode got doomed. 3441 */ 3442 if (VN_IS_DOOMED(vp)) 3443 return; 3444 mp = vp->v_mount; 3445 mtx_lock(&mp->mnt_listmtx); 3446 if ((vp->v_mflag & VMP_LAZYLIST) == 0) { 3447 vp->v_mflag |= VMP_LAZYLIST; 3448 TAILQ_INSERT_TAIL(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3449 mp->mnt_lazyvnodelistsize++; 3450 } 3451 mtx_unlock(&mp->mnt_listmtx); 3452 } 3453 3454 static void 3455 vunlazy(struct vnode *vp) 3456 { 3457 struct mount *mp; 3458 3459 ASSERT_VI_LOCKED(vp, __func__); 3460 VNPASS(!VN_IS_DOOMED(vp), vp); 3461 3462 mp = vp->v_mount; 3463 mtx_lock(&mp->mnt_listmtx); 3464 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3465 /* 3466 * Don't remove the vnode from the lazy list if another thread 3467 * has increased the hold count. It may have re-enqueued the 3468 * vnode to the lazy list and is now responsible for its 3469 * removal. 3470 */ 3471 if (vp->v_holdcnt == 0) { 3472 vp->v_mflag &= ~VMP_LAZYLIST; 3473 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3474 mp->mnt_lazyvnodelistsize--; 3475 } 3476 mtx_unlock(&mp->mnt_listmtx); 3477 } 3478 3479 /* 3480 * This routine is only meant to be called from vgonel prior to dooming 3481 * the vnode. 3482 */ 3483 static void 3484 vunlazy_gone(struct vnode *vp) 3485 { 3486 struct mount *mp; 3487 3488 ASSERT_VOP_ELOCKED(vp, __func__); 3489 ASSERT_VI_LOCKED(vp, __func__); 3490 VNPASS(!VN_IS_DOOMED(vp), vp); 3491 3492 if (vp->v_mflag & VMP_LAZYLIST) { 3493 mp = vp->v_mount; 3494 mtx_lock(&mp->mnt_listmtx); 3495 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3496 vp->v_mflag &= ~VMP_LAZYLIST; 3497 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3498 mp->mnt_lazyvnodelistsize--; 3499 mtx_unlock(&mp->mnt_listmtx); 3500 } 3501 } 3502 3503 static void 3504 vdefer_inactive(struct vnode *vp) 3505 { 3506 3507 ASSERT_VI_LOCKED(vp, __func__); 3508 VNPASS(vp->v_holdcnt > 0, vp); 3509 if (VN_IS_DOOMED(vp)) { 3510 vdropl(vp); 3511 return; 3512 } 3513 if (vp->v_iflag & VI_DEFINACT) { 3514 VNPASS(vp->v_holdcnt > 1, vp); 3515 vdropl(vp); 3516 return; 3517 } 3518 if (vp->v_usecount > 0) { 3519 vp->v_iflag &= ~VI_OWEINACT; 3520 vdropl(vp); 3521 return; 3522 } 3523 vlazy(vp); 3524 vp->v_iflag |= VI_DEFINACT; 3525 VI_UNLOCK(vp); 3526 atomic_add_long(&deferred_inact, 1); 3527 } 3528 3529 static void 3530 vdefer_inactive_unlocked(struct vnode *vp) 3531 { 3532 3533 VI_LOCK(vp); 3534 if ((vp->v_iflag & VI_OWEINACT) == 0) { 3535 vdropl(vp); 3536 return; 3537 } 3538 vdefer_inactive(vp); 3539 } 3540 3541 enum vput_op { VRELE, VPUT, VUNREF }; 3542 3543 /* 3544 * Handle ->v_usecount transitioning to 0. 3545 * 3546 * By releasing the last usecount we take ownership of the hold count which 3547 * provides liveness of the vnode, meaning we have to vdrop. 3548 * 3549 * For all vnodes we may need to perform inactive processing. It requires an 3550 * exclusive lock on the vnode, while it is legal to call here with only a 3551 * shared lock (or no locks). If locking the vnode in an expected manner fails, 3552 * inactive processing gets deferred to the syncer. 3553 * 3554 * XXX Some filesystems pass in an exclusively locked vnode and strongly depend 3555 * on the lock being held all the way until VOP_INACTIVE. This in particular 3556 * happens with UFS which adds half-constructed vnodes to the hash, where they 3557 * can be found by other code. 3558 */ 3559 static void 3560 vput_final(struct vnode *vp, enum vput_op func) 3561 { 3562 int error; 3563 bool want_unlock; 3564 3565 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3566 VNPASS(vp->v_holdcnt > 0, vp); 3567 3568 VI_LOCK(vp); 3569 3570 /* 3571 * By the time we got here someone else might have transitioned 3572 * the count back to > 0. 3573 */ 3574 if (vp->v_usecount > 0) 3575 goto out; 3576 3577 /* 3578 * If the vnode is doomed vgone already performed inactive processing 3579 * (if needed). 3580 */ 3581 if (VN_IS_DOOMED(vp)) 3582 goto out; 3583 3584 if (__predict_true(VOP_NEED_INACTIVE(vp) == 0)) 3585 goto out; 3586 3587 if (vp->v_iflag & VI_DOINGINACT) 3588 goto out; 3589 3590 /* 3591 * Locking operations here will drop the interlock and possibly the 3592 * vnode lock, opening a window where the vnode can get doomed all the 3593 * while ->v_usecount is 0. Set VI_OWEINACT to let vgone know to 3594 * perform inactive. 3595 */ 3596 vp->v_iflag |= VI_OWEINACT; 3597 want_unlock = false; 3598 error = 0; 3599 switch (func) { 3600 case VRELE: 3601 switch (VOP_ISLOCKED(vp)) { 3602 case LK_EXCLUSIVE: 3603 break; 3604 case LK_EXCLOTHER: 3605 case 0: 3606 want_unlock = true; 3607 error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); 3608 VI_LOCK(vp); 3609 break; 3610 default: 3611 /* 3612 * The lock has at least one sharer, but we have no way 3613 * to conclude whether this is us. Play it safe and 3614 * defer processing. 3615 */ 3616 error = EAGAIN; 3617 break; 3618 } 3619 break; 3620 case VPUT: 3621 want_unlock = true; 3622 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3623 error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK | 3624 LK_NOWAIT); 3625 VI_LOCK(vp); 3626 } 3627 break; 3628 case VUNREF: 3629 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3630 error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK); 3631 VI_LOCK(vp); 3632 } 3633 break; 3634 } 3635 if (error == 0) { 3636 if (func == VUNREF) { 3637 VNASSERT((vp->v_vflag & VV_UNREF) == 0, vp, 3638 ("recursive vunref")); 3639 vp->v_vflag |= VV_UNREF; 3640 } 3641 for (;;) { 3642 error = vinactive(vp); 3643 if (want_unlock) 3644 VOP_UNLOCK(vp); 3645 if (error != ERELOOKUP || !want_unlock) 3646 break; 3647 VOP_LOCK(vp, LK_EXCLUSIVE); 3648 } 3649 if (func == VUNREF) 3650 vp->v_vflag &= ~VV_UNREF; 3651 vdropl(vp); 3652 } else { 3653 vdefer_inactive(vp); 3654 } 3655 return; 3656 out: 3657 if (func == VPUT) 3658 VOP_UNLOCK(vp); 3659 vdropl(vp); 3660 } 3661 3662 /* 3663 * Decrement ->v_usecount for a vnode. 3664 * 3665 * Releasing the last use count requires additional processing, see vput_final 3666 * above for details. 3667 * 3668 * Comment above each variant denotes lock state on entry and exit. 3669 */ 3670 3671 /* 3672 * in: any 3673 * out: same as passed in 3674 */ 3675 void 3676 vrele(struct vnode *vp) 3677 { 3678 3679 ASSERT_VI_UNLOCKED(vp, __func__); 3680 if (!refcount_release(&vp->v_usecount)) 3681 return; 3682 vput_final(vp, VRELE); 3683 } 3684 3685 /* 3686 * in: locked 3687 * out: unlocked 3688 */ 3689 void 3690 vput(struct vnode *vp) 3691 { 3692 3693 ASSERT_VOP_LOCKED(vp, __func__); 3694 ASSERT_VI_UNLOCKED(vp, __func__); 3695 if (!refcount_release(&vp->v_usecount)) { 3696 VOP_UNLOCK(vp); 3697 return; 3698 } 3699 vput_final(vp, VPUT); 3700 } 3701 3702 /* 3703 * in: locked 3704 * out: locked 3705 */ 3706 void 3707 vunref(struct vnode *vp) 3708 { 3709 3710 ASSERT_VOP_LOCKED(vp, __func__); 3711 ASSERT_VI_UNLOCKED(vp, __func__); 3712 if (!refcount_release(&vp->v_usecount)) 3713 return; 3714 vput_final(vp, VUNREF); 3715 } 3716 3717 void 3718 vhold(struct vnode *vp) 3719 { 3720 int old; 3721 3722 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3723 old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3724 VNASSERT(old >= 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3725 ("%s: wrong hold count %d", __func__, old)); 3726 if (old == 0) 3727 vfs_freevnodes_dec(); 3728 } 3729 3730 void 3731 vholdnz(struct vnode *vp) 3732 { 3733 3734 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3735 #ifdef INVARIANTS 3736 int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3737 VNASSERT(old > 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3738 ("%s: wrong hold count %d", __func__, old)); 3739 #else 3740 atomic_add_int(&vp->v_holdcnt, 1); 3741 #endif 3742 } 3743 3744 /* 3745 * Grab a hold count unless the vnode is freed. 3746 * 3747 * Only use this routine if vfs smr is the only protection you have against 3748 * freeing the vnode. 3749 * 3750 * The code loops trying to add a hold count as long as the VHOLD_NO_SMR flag 3751 * is not set. After the flag is set the vnode becomes immutable to anyone but 3752 * the thread which managed to set the flag. 3753 * 3754 * It may be tempting to replace the loop with: 3755 * count = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3756 * if (count & VHOLD_NO_SMR) { 3757 * backpedal and error out; 3758 * } 3759 * 3760 * However, while this is more performant, it hinders debugging by eliminating 3761 * the previously mentioned invariant. 3762 */ 3763 bool 3764 vhold_smr(struct vnode *vp) 3765 { 3766 int count; 3767 3768 VFS_SMR_ASSERT_ENTERED(); 3769 3770 count = atomic_load_int(&vp->v_holdcnt); 3771 for (;;) { 3772 if (count & VHOLD_NO_SMR) { 3773 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3774 ("non-zero hold count with flags %d\n", count)); 3775 return (false); 3776 } 3777 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3778 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3779 if (count == 0) 3780 vfs_freevnodes_dec(); 3781 return (true); 3782 } 3783 } 3784 } 3785 3786 /* 3787 * Hold a free vnode for recycling. 3788 * 3789 * Note: vnode_init references this comment. 3790 * 3791 * Attempts to recycle only need the global vnode list lock and have no use for 3792 * SMR. 3793 * 3794 * However, vnodes get inserted into the global list before they get fully 3795 * initialized and stay there until UMA decides to free the memory. This in 3796 * particular means the target can be found before it becomes usable and after 3797 * it becomes recycled. Picking up such vnodes is guarded with v_holdcnt set to 3798 * VHOLD_NO_SMR. 3799 * 3800 * Note: the vnode may gain more references after we transition the count 0->1. 3801 */ 3802 static bool 3803 vhold_recycle_free(struct vnode *vp) 3804 { 3805 int count; 3806 3807 mtx_assert(&vnode_list_mtx, MA_OWNED); 3808 3809 count = atomic_load_int(&vp->v_holdcnt); 3810 for (;;) { 3811 if (count & VHOLD_NO_SMR) { 3812 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3813 ("non-zero hold count with flags %d\n", count)); 3814 return (false); 3815 } 3816 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3817 if (count > 0) { 3818 return (false); 3819 } 3820 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3821 vfs_freevnodes_dec(); 3822 return (true); 3823 } 3824 } 3825 } 3826 3827 static void __noinline 3828 vdbatch_process(struct vdbatch *vd) 3829 { 3830 struct vnode *vp; 3831 int i; 3832 3833 mtx_assert(&vd->lock, MA_OWNED); 3834 MPASS(curthread->td_pinned > 0); 3835 MPASS(vd->index == VDBATCH_SIZE); 3836 3837 /* 3838 * Attempt to requeue the passed batch, but give up easily. 3839 * 3840 * Despite batching the mechanism is prone to transient *significant* 3841 * lock contention, where vnode_list_mtx becomes the primary bottleneck 3842 * if multiple CPUs get here (one real-world example is highly parallel 3843 * do-nothing make , which will stat *tons* of vnodes). Since it is 3844 * quasi-LRU (read: not that great even if fully honoured) provide an 3845 * option to just dodge the problem. Parties which don't like it are 3846 * welcome to implement something better. 3847 */ 3848 if (vnode_can_skip_requeue) { 3849 if (!mtx_trylock(&vnode_list_mtx)) { 3850 counter_u64_add(vnode_skipped_requeues, 1); 3851 critical_enter(); 3852 for (i = 0; i < VDBATCH_SIZE; i++) { 3853 vp = vd->tab[i]; 3854 vd->tab[i] = NULL; 3855 MPASS(vp->v_dbatchcpu != NOCPU); 3856 vp->v_dbatchcpu = NOCPU; 3857 } 3858 vd->index = 0; 3859 critical_exit(); 3860 return; 3861 3862 } 3863 /* fallthrough to locked processing */ 3864 } else { 3865 mtx_lock(&vnode_list_mtx); 3866 } 3867 3868 mtx_assert(&vnode_list_mtx, MA_OWNED); 3869 critical_enter(); 3870 for (i = 0; i < VDBATCH_SIZE; i++) { 3871 vp = vd->tab[i]; 3872 vd->tab[i] = NULL; 3873 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 3874 TAILQ_INSERT_TAIL(&vnode_list, vp, v_vnodelist); 3875 MPASS(vp->v_dbatchcpu != NOCPU); 3876 vp->v_dbatchcpu = NOCPU; 3877 } 3878 mtx_unlock(&vnode_list_mtx); 3879 vd->index = 0; 3880 critical_exit(); 3881 } 3882 3883 static void 3884 vdbatch_enqueue(struct vnode *vp) 3885 { 3886 struct vdbatch *vd; 3887 3888 ASSERT_VI_LOCKED(vp, __func__); 3889 VNPASS(!VN_IS_DOOMED(vp), vp); 3890 3891 if (vp->v_dbatchcpu != NOCPU) { 3892 VI_UNLOCK(vp); 3893 return; 3894 } 3895 3896 sched_pin(); 3897 vd = DPCPU_PTR(vd); 3898 mtx_lock(&vd->lock); 3899 MPASS(vd->index < VDBATCH_SIZE); 3900 MPASS(vd->tab[vd->index] == NULL); 3901 /* 3902 * A hack: we depend on being pinned so that we know what to put in 3903 * ->v_dbatchcpu. 3904 */ 3905 vp->v_dbatchcpu = curcpu; 3906 vd->tab[vd->index] = vp; 3907 vd->index++; 3908 VI_UNLOCK(vp); 3909 if (vd->index == VDBATCH_SIZE) 3910 vdbatch_process(vd); 3911 mtx_unlock(&vd->lock); 3912 sched_unpin(); 3913 } 3914 3915 /* 3916 * This routine must only be called for vnodes which are about to be 3917 * deallocated. Supporting dequeue for arbitrary vndoes would require 3918 * validating that the locked batch matches. 3919 */ 3920 static void 3921 vdbatch_dequeue(struct vnode *vp) 3922 { 3923 struct vdbatch *vd; 3924 int i; 3925 short cpu; 3926 3927 VNPASS(vp->v_type == VBAD || vp->v_type == VNON, vp); 3928 3929 cpu = vp->v_dbatchcpu; 3930 if (cpu == NOCPU) 3931 return; 3932 3933 vd = DPCPU_ID_PTR(cpu, vd); 3934 mtx_lock(&vd->lock); 3935 for (i = 0; i < vd->index; i++) { 3936 if (vd->tab[i] != vp) 3937 continue; 3938 vp->v_dbatchcpu = NOCPU; 3939 vd->index--; 3940 vd->tab[i] = vd->tab[vd->index]; 3941 vd->tab[vd->index] = NULL; 3942 break; 3943 } 3944 mtx_unlock(&vd->lock); 3945 /* 3946 * Either we dequeued the vnode above or the target CPU beat us to it. 3947 */ 3948 MPASS(vp->v_dbatchcpu == NOCPU); 3949 } 3950 3951 /* 3952 * Drop the hold count of the vnode. 3953 * 3954 * It will only get freed if this is the last hold *and* it has been vgone'd. 3955 * 3956 * Because the vnode vm object keeps a hold reference on the vnode if 3957 * there is at least one resident non-cached page, the vnode cannot 3958 * leave the active list without the page cleanup done. 3959 */ 3960 static void __noinline 3961 vdropl_final(struct vnode *vp) 3962 { 3963 3964 ASSERT_VI_LOCKED(vp, __func__); 3965 VNPASS(VN_IS_DOOMED(vp), vp); 3966 /* 3967 * Set the VHOLD_NO_SMR flag. 3968 * 3969 * We may be racing against vhold_smr. If they win we can just pretend 3970 * we never got this far, they will vdrop later. 3971 */ 3972 if (__predict_false(!atomic_cmpset_int(&vp->v_holdcnt, 0, VHOLD_NO_SMR))) { 3973 vfs_freevnodes_inc(); 3974 VI_UNLOCK(vp); 3975 /* 3976 * We lost the aforementioned race. Any subsequent access is 3977 * invalid as they might have managed to vdropl on their own. 3978 */ 3979 return; 3980 } 3981 /* 3982 * Don't bump freevnodes as this one is going away. 3983 */ 3984 freevnode(vp); 3985 } 3986 3987 void 3988 vdrop(struct vnode *vp) 3989 { 3990 3991 ASSERT_VI_UNLOCKED(vp, __func__); 3992 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3993 if (refcount_release_if_not_last(&vp->v_holdcnt)) 3994 return; 3995 VI_LOCK(vp); 3996 vdropl(vp); 3997 } 3998 3999 static __always_inline void 4000 vdropl_impl(struct vnode *vp, bool enqueue) 4001 { 4002 4003 ASSERT_VI_LOCKED(vp, __func__); 4004 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4005 if (!refcount_release(&vp->v_holdcnt)) { 4006 VI_UNLOCK(vp); 4007 return; 4008 } 4009 VNPASS((vp->v_iflag & VI_OWEINACT) == 0, vp); 4010 VNPASS((vp->v_iflag & VI_DEFINACT) == 0, vp); 4011 if (VN_IS_DOOMED(vp)) { 4012 vdropl_final(vp); 4013 return; 4014 } 4015 4016 vfs_freevnodes_inc(); 4017 if (vp->v_mflag & VMP_LAZYLIST) { 4018 vunlazy(vp); 4019 } 4020 4021 if (!enqueue) { 4022 VI_UNLOCK(vp); 4023 return; 4024 } 4025 4026 /* 4027 * Also unlocks the interlock. We can't assert on it as we 4028 * released our hold and by now the vnode might have been 4029 * freed. 4030 */ 4031 vdbatch_enqueue(vp); 4032 } 4033 4034 void 4035 vdropl(struct vnode *vp) 4036 { 4037 4038 vdropl_impl(vp, true); 4039 } 4040 4041 /* 4042 * vdrop a vnode when recycling 4043 * 4044 * This is a special case routine only to be used when recycling, differs from 4045 * regular vdrop by not requeieing the vnode on LRU. 4046 * 4047 * Consider a case where vtryrecycle continuously fails with all vnodes (due to 4048 * e.g., frozen writes on the filesystem), filling the batch and causing it to 4049 * be requeued. Then vnlru will end up revisiting the same vnodes. This is a 4050 * loop which can last for as long as writes are frozen. 4051 */ 4052 static void 4053 vdropl_recycle(struct vnode *vp) 4054 { 4055 4056 vdropl_impl(vp, false); 4057 } 4058 4059 static void 4060 vdrop_recycle(struct vnode *vp) 4061 { 4062 4063 VI_LOCK(vp); 4064 vdropl_recycle(vp); 4065 } 4066 4067 /* 4068 * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT 4069 * flags. DOINGINACT prevents us from recursing in calls to vinactive. 4070 */ 4071 static int 4072 vinactivef(struct vnode *vp) 4073 { 4074 int error; 4075 4076 ASSERT_VOP_ELOCKED(vp, "vinactive"); 4077 ASSERT_VI_LOCKED(vp, "vinactive"); 4078 VNPASS((vp->v_iflag & VI_DOINGINACT) == 0, vp); 4079 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4080 vp->v_iflag |= VI_DOINGINACT; 4081 vp->v_iflag &= ~VI_OWEINACT; 4082 VI_UNLOCK(vp); 4083 4084 /* 4085 * Before moving off the active list, we must be sure that any 4086 * modified pages are converted into the vnode's dirty 4087 * buffers, since these will no longer be checked once the 4088 * vnode is on the inactive list. 4089 * 4090 * The write-out of the dirty pages is asynchronous. At the 4091 * point that VOP_INACTIVE() is called, there could still be 4092 * pending I/O and dirty pages in the object. 4093 */ 4094 if ((vp->v_vflag & VV_NOSYNC) == 0) 4095 vnode_pager_clean_async(vp); 4096 4097 error = VOP_INACTIVE(vp); 4098 VI_LOCK(vp); 4099 VNPASS(vp->v_iflag & VI_DOINGINACT, vp); 4100 vp->v_iflag &= ~VI_DOINGINACT; 4101 return (error); 4102 } 4103 4104 int 4105 vinactive(struct vnode *vp) 4106 { 4107 4108 ASSERT_VOP_ELOCKED(vp, "vinactive"); 4109 ASSERT_VI_LOCKED(vp, "vinactive"); 4110 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4111 4112 if ((vp->v_iflag & VI_OWEINACT) == 0) 4113 return (0); 4114 if (vp->v_iflag & VI_DOINGINACT) 4115 return (0); 4116 if (vp->v_usecount > 0) { 4117 vp->v_iflag &= ~VI_OWEINACT; 4118 return (0); 4119 } 4120 return (vinactivef(vp)); 4121 } 4122 4123 /* 4124 * Remove any vnodes in the vnode table belonging to mount point mp. 4125 * 4126 * If FORCECLOSE is not specified, there should not be any active ones, 4127 * return error if any are found (nb: this is a user error, not a 4128 * system error). If FORCECLOSE is specified, detach any active vnodes 4129 * that are found. 4130 * 4131 * If WRITECLOSE is set, only flush out regular file vnodes open for 4132 * writing. 4133 * 4134 * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped. 4135 * 4136 * `rootrefs' specifies the base reference count for the root vnode 4137 * of this filesystem. The root vnode is considered busy if its 4138 * v_usecount exceeds this value. On a successful return, vflush(, td) 4139 * will call vrele() on the root vnode exactly rootrefs times. 4140 * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must 4141 * be zero. 4142 */ 4143 #ifdef DIAGNOSTIC 4144 static int busyprt = 0; /* print out busy vnodes */ 4145 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes"); 4146 #endif 4147 4148 int 4149 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td) 4150 { 4151 struct vnode *vp, *mvp, *rootvp = NULL; 4152 struct vattr vattr; 4153 int busy = 0, error; 4154 4155 CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp, 4156 rootrefs, flags); 4157 if (rootrefs > 0) { 4158 KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0, 4159 ("vflush: bad args")); 4160 /* 4161 * Get the filesystem root vnode. We can vput() it 4162 * immediately, since with rootrefs > 0, it won't go away. 4163 */ 4164 if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) { 4165 CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d", 4166 __func__, error); 4167 return (error); 4168 } 4169 vput(rootvp); 4170 } 4171 loop: 4172 MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { 4173 vholdl(vp); 4174 error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE); 4175 if (error) { 4176 vdrop(vp); 4177 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 4178 goto loop; 4179 } 4180 /* 4181 * Skip over a vnodes marked VV_SYSTEM. 4182 */ 4183 if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) { 4184 VOP_UNLOCK(vp); 4185 vdrop(vp); 4186 continue; 4187 } 4188 /* 4189 * If WRITECLOSE is set, flush out unlinked but still open 4190 * files (even if open only for reading) and regular file 4191 * vnodes open for writing. 4192 */ 4193 if (flags & WRITECLOSE) { 4194 vnode_pager_clean_async(vp); 4195 do { 4196 error = VOP_FSYNC(vp, MNT_WAIT, td); 4197 } while (error == ERELOOKUP); 4198 if (error != 0) { 4199 VOP_UNLOCK(vp); 4200 vdrop(vp); 4201 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 4202 return (error); 4203 } 4204 error = VOP_GETATTR(vp, &vattr, td->td_ucred); 4205 VI_LOCK(vp); 4206 4207 if ((vp->v_type == VNON || 4208 (error == 0 && vattr.va_nlink > 0)) && 4209 (vp->v_writecount <= 0 || vp->v_type != VREG)) { 4210 VOP_UNLOCK(vp); 4211 vdropl(vp); 4212 continue; 4213 } 4214 } else 4215 VI_LOCK(vp); 4216 /* 4217 * With v_usecount == 0, all we need to do is clear out the 4218 * vnode data structures and we are done. 4219 * 4220 * If FORCECLOSE is set, forcibly close the vnode. 4221 */ 4222 if (vp->v_usecount == 0 || (flags & FORCECLOSE)) { 4223 vgonel(vp); 4224 } else { 4225 busy++; 4226 #ifdef DIAGNOSTIC 4227 if (busyprt) 4228 vn_printf(vp, "vflush: busy vnode "); 4229 #endif 4230 } 4231 VOP_UNLOCK(vp); 4232 vdropl(vp); 4233 } 4234 if (rootrefs > 0 && (flags & FORCECLOSE) == 0) { 4235 /* 4236 * If just the root vnode is busy, and if its refcount 4237 * is equal to `rootrefs', then go ahead and kill it. 4238 */ 4239 VI_LOCK(rootvp); 4240 KASSERT(busy > 0, ("vflush: not busy")); 4241 VNASSERT(rootvp->v_usecount >= rootrefs, rootvp, 4242 ("vflush: usecount %d < rootrefs %d", 4243 rootvp->v_usecount, rootrefs)); 4244 if (busy == 1 && rootvp->v_usecount == rootrefs) { 4245 VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK); 4246 vgone(rootvp); 4247 VOP_UNLOCK(rootvp); 4248 busy = 0; 4249 } else 4250 VI_UNLOCK(rootvp); 4251 } 4252 if (busy) { 4253 CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__, 4254 busy); 4255 return (EBUSY); 4256 } 4257 for (; rootrefs > 0; rootrefs--) 4258 vrele(rootvp); 4259 return (0); 4260 } 4261 4262 /* 4263 * Recycle an unused vnode. 4264 */ 4265 int 4266 vrecycle(struct vnode *vp) 4267 { 4268 int recycled; 4269 4270 VI_LOCK(vp); 4271 recycled = vrecyclel(vp); 4272 VI_UNLOCK(vp); 4273 return (recycled); 4274 } 4275 4276 /* 4277 * vrecycle, with the vp interlock held. 4278 */ 4279 int 4280 vrecyclel(struct vnode *vp) 4281 { 4282 int recycled; 4283 4284 ASSERT_VOP_ELOCKED(vp, __func__); 4285 ASSERT_VI_LOCKED(vp, __func__); 4286 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4287 recycled = 0; 4288 if (vp->v_usecount == 0) { 4289 recycled = 1; 4290 vgonel(vp); 4291 } 4292 return (recycled); 4293 } 4294 4295 /* 4296 * Eliminate all activity associated with a vnode 4297 * in preparation for reuse. 4298 */ 4299 void 4300 vgone(struct vnode *vp) 4301 { 4302 VI_LOCK(vp); 4303 vgonel(vp); 4304 VI_UNLOCK(vp); 4305 } 4306 4307 /* 4308 * Notify upper mounts about reclaimed or unlinked vnode. 4309 */ 4310 void 4311 vfs_notify_upper(struct vnode *vp, enum vfs_notify_upper_type event) 4312 { 4313 struct mount *mp; 4314 struct mount_upper_node *ump; 4315 4316 mp = atomic_load_ptr(&vp->v_mount); 4317 if (mp == NULL) 4318 return; 4319 if (TAILQ_EMPTY(&mp->mnt_notify)) 4320 return; 4321 4322 MNT_ILOCK(mp); 4323 mp->mnt_upper_pending++; 4324 KASSERT(mp->mnt_upper_pending > 0, 4325 ("%s: mnt_upper_pending %d", __func__, mp->mnt_upper_pending)); 4326 TAILQ_FOREACH(ump, &mp->mnt_notify, mnt_upper_link) { 4327 MNT_IUNLOCK(mp); 4328 switch (event) { 4329 case VFS_NOTIFY_UPPER_RECLAIM: 4330 VFS_RECLAIM_LOWERVP(ump->mp, vp); 4331 break; 4332 case VFS_NOTIFY_UPPER_UNLINK: 4333 VFS_UNLINK_LOWERVP(ump->mp, vp); 4334 break; 4335 } 4336 MNT_ILOCK(mp); 4337 } 4338 mp->mnt_upper_pending--; 4339 if ((mp->mnt_kern_flag & MNTK_UPPER_WAITER) != 0 && 4340 mp->mnt_upper_pending == 0) { 4341 mp->mnt_kern_flag &= ~MNTK_UPPER_WAITER; 4342 wakeup(&mp->mnt_uppers); 4343 } 4344 MNT_IUNLOCK(mp); 4345 } 4346 4347 /* 4348 * vgone, with the vp interlock held. 4349 */ 4350 static void 4351 vgonel(struct vnode *vp) 4352 { 4353 struct thread *td; 4354 struct mount *mp; 4355 vm_object_t object; 4356 bool active, doinginact, oweinact; 4357 4358 ASSERT_VOP_ELOCKED(vp, "vgonel"); 4359 ASSERT_VI_LOCKED(vp, "vgonel"); 4360 VNASSERT(vp->v_holdcnt, vp, 4361 ("vgonel: vp %p has no reference.", vp)); 4362 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4363 td = curthread; 4364 4365 /* 4366 * Don't vgonel if we're already doomed. 4367 */ 4368 if (VN_IS_DOOMED(vp)) { 4369 VNPASS(vn_get_state(vp) == VSTATE_DESTROYING || \ 4370 vn_get_state(vp) == VSTATE_DEAD, vp); 4371 return; 4372 } 4373 /* 4374 * Paired with freevnode. 4375 */ 4376 vn_seqc_write_begin_locked(vp); 4377 vunlazy_gone(vp); 4378 vn_irflag_set_locked(vp, VIRF_DOOMED); 4379 vn_set_state(vp, VSTATE_DESTROYING); 4380 4381 /* 4382 * Check to see if the vnode is in use. If so, we have to 4383 * call VOP_CLOSE() and VOP_INACTIVE(). 4384 * 4385 * It could be that VOP_INACTIVE() requested reclamation, in 4386 * which case we should avoid recursion, so check 4387 * VI_DOINGINACT. This is not precise but good enough. 4388 */ 4389 active = vp->v_usecount > 0; 4390 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4391 doinginact = (vp->v_iflag & VI_DOINGINACT) != 0; 4392 4393 /* 4394 * If we need to do inactive VI_OWEINACT will be set. 4395 */ 4396 if (vp->v_iflag & VI_DEFINACT) { 4397 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 4398 vp->v_iflag &= ~VI_DEFINACT; 4399 vdropl(vp); 4400 } else { 4401 VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count")); 4402 VI_UNLOCK(vp); 4403 } 4404 cache_purge_vgone(vp); 4405 vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM); 4406 4407 /* 4408 * If purging an active vnode, it must be closed and 4409 * deactivated before being reclaimed. 4410 */ 4411 if (active) 4412 VOP_CLOSE(vp, FNONBLOCK, NOCRED, td); 4413 if (!doinginact) { 4414 do { 4415 if (oweinact || active) { 4416 VI_LOCK(vp); 4417 vinactivef(vp); 4418 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4419 VI_UNLOCK(vp); 4420 } 4421 } while (oweinact); 4422 } 4423 if (vp->v_type == VSOCK) 4424 vfs_unp_reclaim(vp); 4425 4426 /* 4427 * Clean out any buffers associated with the vnode. 4428 * If the flush fails, just toss the buffers. 4429 */ 4430 mp = NULL; 4431 if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd)) 4432 (void) vn_start_secondary_write(vp, &mp, V_WAIT); 4433 if (vinvalbuf(vp, V_SAVE, 0, 0) != 0) { 4434 while (vinvalbuf(vp, 0, 0, 0) != 0) 4435 ; 4436 } 4437 4438 BO_LOCK(&vp->v_bufobj); 4439 KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) && 4440 vp->v_bufobj.bo_dirty.bv_cnt == 0 && 4441 TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) && 4442 vp->v_bufobj.bo_clean.bv_cnt == 0, 4443 ("vp %p bufobj not invalidated", vp)); 4444 4445 /* 4446 * For VMIO bufobj, BO_DEAD is set later, or in 4447 * vm_object_terminate() after the object's page queue is 4448 * flushed. 4449 */ 4450 object = vp->v_bufobj.bo_object; 4451 if (object == NULL) 4452 vp->v_bufobj.bo_flag |= BO_DEAD; 4453 BO_UNLOCK(&vp->v_bufobj); 4454 4455 /* 4456 * Handle the VM part. Tmpfs handles v_object on its own (the 4457 * OBJT_VNODE check). Nullfs or other bypassing filesystems 4458 * should not touch the object borrowed from the lower vnode 4459 * (the handle check). 4460 */ 4461 if (object != NULL && object->type == OBJT_VNODE && 4462 object->handle == vp) 4463 vnode_destroy_vobject(vp); 4464 4465 /* 4466 * Reclaim the vnode. 4467 */ 4468 if (VOP_RECLAIM(vp)) 4469 panic("vgone: cannot reclaim"); 4470 if (mp != NULL) 4471 vn_finished_secondary_write(mp); 4472 VNASSERT(vp->v_object == NULL, vp, 4473 ("vop_reclaim left v_object vp=%p", vp)); 4474 /* 4475 * Clear the advisory locks and wake up waiting threads. 4476 */ 4477 if (vp->v_lockf != NULL) { 4478 (void)VOP_ADVLOCKPURGE(vp); 4479 vp->v_lockf = NULL; 4480 } 4481 /* 4482 * Delete from old mount point vnode list. 4483 */ 4484 if (vp->v_mount == NULL) { 4485 VI_LOCK(vp); 4486 } else { 4487 delmntque(vp); 4488 ASSERT_VI_LOCKED(vp, "vgonel 2"); 4489 } 4490 /* 4491 * Done with purge, reset to the standard lock and invalidate 4492 * the vnode. 4493 */ 4494 vp->v_vnlock = &vp->v_lock; 4495 vp->v_op = &dead_vnodeops; 4496 vp->v_type = VBAD; 4497 vn_set_state(vp, VSTATE_DEAD); 4498 } 4499 4500 /* 4501 * Print out a description of a vnode. 4502 */ 4503 static const char *const vtypename[] = { 4504 [VNON] = "VNON", 4505 [VREG] = "VREG", 4506 [VDIR] = "VDIR", 4507 [VBLK] = "VBLK", 4508 [VCHR] = "VCHR", 4509 [VLNK] = "VLNK", 4510 [VSOCK] = "VSOCK", 4511 [VFIFO] = "VFIFO", 4512 [VBAD] = "VBAD", 4513 [VMARKER] = "VMARKER", 4514 }; 4515 _Static_assert(nitems(vtypename) == VLASTTYPE + 1, 4516 "vnode type name not added to vtypename"); 4517 4518 static const char *const vstatename[] = { 4519 [VSTATE_UNINITIALIZED] = "VSTATE_UNINITIALIZED", 4520 [VSTATE_CONSTRUCTED] = "VSTATE_CONSTRUCTED", 4521 [VSTATE_DESTROYING] = "VSTATE_DESTROYING", 4522 [VSTATE_DEAD] = "VSTATE_DEAD", 4523 }; 4524 _Static_assert(nitems(vstatename) == VLASTSTATE + 1, 4525 "vnode state name not added to vstatename"); 4526 4527 _Static_assert((VHOLD_ALL_FLAGS & ~VHOLD_NO_SMR) == 0, 4528 "new hold count flag not added to vn_printf"); 4529 4530 void 4531 vn_printf(struct vnode *vp, const char *fmt, ...) 4532 { 4533 va_list ap; 4534 char buf[256], buf2[16]; 4535 u_long flags; 4536 u_int holdcnt; 4537 short irflag; 4538 4539 va_start(ap, fmt); 4540 vprintf(fmt, ap); 4541 va_end(ap); 4542 printf("%p: ", (void *)vp); 4543 printf("type %s state %s op %p\n", vtypename[vp->v_type], 4544 vstatename[vp->v_state], vp->v_op); 4545 holdcnt = atomic_load_int(&vp->v_holdcnt); 4546 printf(" usecount %d, writecount %d, refcount %d seqc users %d", 4547 vp->v_usecount, vp->v_writecount, holdcnt & ~VHOLD_ALL_FLAGS, 4548 vp->v_seqc_users); 4549 switch (vp->v_type) { 4550 case VDIR: 4551 printf(" mountedhere %p\n", vp->v_mountedhere); 4552 break; 4553 case VCHR: 4554 printf(" rdev %p\n", vp->v_rdev); 4555 break; 4556 case VSOCK: 4557 printf(" socket %p\n", vp->v_unpcb); 4558 break; 4559 case VFIFO: 4560 printf(" fifoinfo %p\n", vp->v_fifoinfo); 4561 break; 4562 default: 4563 printf("\n"); 4564 break; 4565 } 4566 buf[0] = '\0'; 4567 buf[1] = '\0'; 4568 if (holdcnt & VHOLD_NO_SMR) 4569 strlcat(buf, "|VHOLD_NO_SMR", sizeof(buf)); 4570 printf(" hold count flags (%s)\n", buf + 1); 4571 4572 buf[0] = '\0'; 4573 buf[1] = '\0'; 4574 irflag = vn_irflag_read(vp); 4575 if (irflag & VIRF_DOOMED) 4576 strlcat(buf, "|VIRF_DOOMED", sizeof(buf)); 4577 if (irflag & VIRF_PGREAD) 4578 strlcat(buf, "|VIRF_PGREAD", sizeof(buf)); 4579 if (irflag & VIRF_MOUNTPOINT) 4580 strlcat(buf, "|VIRF_MOUNTPOINT", sizeof(buf)); 4581 if (irflag & VIRF_TEXT_REF) 4582 strlcat(buf, "|VIRF_TEXT_REF", sizeof(buf)); 4583 flags = irflag & ~(VIRF_DOOMED | VIRF_PGREAD | VIRF_MOUNTPOINT | VIRF_TEXT_REF); 4584 if (flags != 0) { 4585 snprintf(buf2, sizeof(buf2), "|VIRF(0x%lx)", flags); 4586 strlcat(buf, buf2, sizeof(buf)); 4587 } 4588 if (vp->v_vflag & VV_ROOT) 4589 strlcat(buf, "|VV_ROOT", sizeof(buf)); 4590 if (vp->v_vflag & VV_ISTTY) 4591 strlcat(buf, "|VV_ISTTY", sizeof(buf)); 4592 if (vp->v_vflag & VV_NOSYNC) 4593 strlcat(buf, "|VV_NOSYNC", sizeof(buf)); 4594 if (vp->v_vflag & VV_ETERNALDEV) 4595 strlcat(buf, "|VV_ETERNALDEV", sizeof(buf)); 4596 if (vp->v_vflag & VV_CACHEDLABEL) 4597 strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf)); 4598 if (vp->v_vflag & VV_VMSIZEVNLOCK) 4599 strlcat(buf, "|VV_VMSIZEVNLOCK", sizeof(buf)); 4600 if (vp->v_vflag & VV_COPYONWRITE) 4601 strlcat(buf, "|VV_COPYONWRITE", sizeof(buf)); 4602 if (vp->v_vflag & VV_SYSTEM) 4603 strlcat(buf, "|VV_SYSTEM", sizeof(buf)); 4604 if (vp->v_vflag & VV_PROCDEP) 4605 strlcat(buf, "|VV_PROCDEP", sizeof(buf)); 4606 if (vp->v_vflag & VV_DELETED) 4607 strlcat(buf, "|VV_DELETED", sizeof(buf)); 4608 if (vp->v_vflag & VV_MD) 4609 strlcat(buf, "|VV_MD", sizeof(buf)); 4610 if (vp->v_vflag & VV_FORCEINSMQ) 4611 strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf)); 4612 if (vp->v_vflag & VV_READLINK) 4613 strlcat(buf, "|VV_READLINK", sizeof(buf)); 4614 flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV | 4615 VV_CACHEDLABEL | VV_VMSIZEVNLOCK | VV_COPYONWRITE | VV_SYSTEM | 4616 VV_PROCDEP | VV_DELETED | VV_MD | VV_FORCEINSMQ | VV_READLINK); 4617 if (flags != 0) { 4618 snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags); 4619 strlcat(buf, buf2, sizeof(buf)); 4620 } 4621 if (vp->v_iflag & VI_MOUNT) 4622 strlcat(buf, "|VI_MOUNT", sizeof(buf)); 4623 if (vp->v_iflag & VI_DOINGINACT) 4624 strlcat(buf, "|VI_DOINGINACT", sizeof(buf)); 4625 if (vp->v_iflag & VI_OWEINACT) 4626 strlcat(buf, "|VI_OWEINACT", sizeof(buf)); 4627 if (vp->v_iflag & VI_DEFINACT) 4628 strlcat(buf, "|VI_DEFINACT", sizeof(buf)); 4629 if (vp->v_iflag & VI_FOPENING) 4630 strlcat(buf, "|VI_FOPENING", sizeof(buf)); 4631 flags = vp->v_iflag & ~(VI_MOUNT | VI_DOINGINACT | 4632 VI_OWEINACT | VI_DEFINACT | VI_FOPENING); 4633 if (flags != 0) { 4634 snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags); 4635 strlcat(buf, buf2, sizeof(buf)); 4636 } 4637 if (vp->v_mflag & VMP_LAZYLIST) 4638 strlcat(buf, "|VMP_LAZYLIST", sizeof(buf)); 4639 flags = vp->v_mflag & ~(VMP_LAZYLIST); 4640 if (flags != 0) { 4641 snprintf(buf2, sizeof(buf2), "|VMP(0x%lx)", flags); 4642 strlcat(buf, buf2, sizeof(buf)); 4643 } 4644 printf(" flags (%s)", buf + 1); 4645 if (mtx_owned(VI_MTX(vp))) 4646 printf(" VI_LOCKed"); 4647 printf("\n"); 4648 if (vp->v_object != NULL) 4649 printf(" v_object %p ref %d pages %d " 4650 "cleanbuf %d dirtybuf %d\n", 4651 vp->v_object, vp->v_object->ref_count, 4652 vp->v_object->resident_page_count, 4653 vp->v_bufobj.bo_clean.bv_cnt, 4654 vp->v_bufobj.bo_dirty.bv_cnt); 4655 printf(" "); 4656 lockmgr_printinfo(vp->v_vnlock); 4657 if (vp->v_data != NULL) 4658 VOP_PRINT(vp); 4659 } 4660 4661 #ifdef DDB 4662 /* 4663 * List all of the locked vnodes in the system. 4664 * Called when debugging the kernel. 4665 */ 4666 DB_SHOW_COMMAND_FLAGS(lockedvnods, lockedvnodes, DB_CMD_MEMSAFE) 4667 { 4668 struct mount *mp; 4669 struct vnode *vp; 4670 4671 /* 4672 * Note: because this is DDB, we can't obey the locking semantics 4673 * for these structures, which means we could catch an inconsistent 4674 * state and dereference a nasty pointer. Not much to be done 4675 * about that. 4676 */ 4677 db_printf("Locked vnodes\n"); 4678 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4679 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4680 if (vp->v_type != VMARKER && VOP_ISLOCKED(vp)) 4681 vn_printf(vp, "vnode "); 4682 } 4683 } 4684 } 4685 4686 /* 4687 * Show details about the given vnode. 4688 */ 4689 DB_SHOW_COMMAND(vnode, db_show_vnode) 4690 { 4691 struct vnode *vp; 4692 4693 if (!have_addr) 4694 return; 4695 vp = (struct vnode *)addr; 4696 vn_printf(vp, "vnode "); 4697 } 4698 4699 /* 4700 * Show details about the given mount point. 4701 */ 4702 DB_SHOW_COMMAND(mount, db_show_mount) 4703 { 4704 struct mount *mp; 4705 struct vfsopt *opt; 4706 struct statfs *sp; 4707 struct vnode *vp; 4708 char buf[512]; 4709 uint64_t mflags; 4710 u_int flags; 4711 4712 if (!have_addr) { 4713 /* No address given, print short info about all mount points. */ 4714 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4715 db_printf("%p %s on %s (%s)\n", mp, 4716 mp->mnt_stat.f_mntfromname, 4717 mp->mnt_stat.f_mntonname, 4718 mp->mnt_stat.f_fstypename); 4719 if (db_pager_quit) 4720 break; 4721 } 4722 db_printf("\nMore info: show mount <addr>\n"); 4723 return; 4724 } 4725 4726 mp = (struct mount *)addr; 4727 db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname, 4728 mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename); 4729 4730 buf[0] = '\0'; 4731 mflags = mp->mnt_flag; 4732 #define MNT_FLAG(flag) do { \ 4733 if (mflags & (flag)) { \ 4734 if (buf[0] != '\0') \ 4735 strlcat(buf, ", ", sizeof(buf)); \ 4736 strlcat(buf, (#flag) + 4, sizeof(buf)); \ 4737 mflags &= ~(flag); \ 4738 } \ 4739 } while (0) 4740 MNT_FLAG(MNT_RDONLY); 4741 MNT_FLAG(MNT_SYNCHRONOUS); 4742 MNT_FLAG(MNT_NOEXEC); 4743 MNT_FLAG(MNT_NOSUID); 4744 MNT_FLAG(MNT_NFS4ACLS); 4745 MNT_FLAG(MNT_UNION); 4746 MNT_FLAG(MNT_ASYNC); 4747 MNT_FLAG(MNT_SUIDDIR); 4748 MNT_FLAG(MNT_SOFTDEP); 4749 MNT_FLAG(MNT_NOSYMFOLLOW); 4750 MNT_FLAG(MNT_GJOURNAL); 4751 MNT_FLAG(MNT_MULTILABEL); 4752 MNT_FLAG(MNT_ACLS); 4753 MNT_FLAG(MNT_NOATIME); 4754 MNT_FLAG(MNT_NOCLUSTERR); 4755 MNT_FLAG(MNT_NOCLUSTERW); 4756 MNT_FLAG(MNT_SUJ); 4757 MNT_FLAG(MNT_EXRDONLY); 4758 MNT_FLAG(MNT_EXPORTED); 4759 MNT_FLAG(MNT_DEFEXPORTED); 4760 MNT_FLAG(MNT_EXPORTANON); 4761 MNT_FLAG(MNT_EXKERB); 4762 MNT_FLAG(MNT_EXPUBLIC); 4763 MNT_FLAG(MNT_LOCAL); 4764 MNT_FLAG(MNT_QUOTA); 4765 MNT_FLAG(MNT_ROOTFS); 4766 MNT_FLAG(MNT_USER); 4767 MNT_FLAG(MNT_IGNORE); 4768 MNT_FLAG(MNT_UPDATE); 4769 MNT_FLAG(MNT_DELEXPORT); 4770 MNT_FLAG(MNT_RELOAD); 4771 MNT_FLAG(MNT_FORCE); 4772 MNT_FLAG(MNT_SNAPSHOT); 4773 MNT_FLAG(MNT_BYFSID); 4774 #undef MNT_FLAG 4775 if (mflags != 0) { 4776 if (buf[0] != '\0') 4777 strlcat(buf, ", ", sizeof(buf)); 4778 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4779 "0x%016jx", mflags); 4780 } 4781 db_printf(" mnt_flag = %s\n", buf); 4782 4783 buf[0] = '\0'; 4784 flags = mp->mnt_kern_flag; 4785 #define MNT_KERN_FLAG(flag) do { \ 4786 if (flags & (flag)) { \ 4787 if (buf[0] != '\0') \ 4788 strlcat(buf, ", ", sizeof(buf)); \ 4789 strlcat(buf, (#flag) + 5, sizeof(buf)); \ 4790 flags &= ~(flag); \ 4791 } \ 4792 } while (0) 4793 MNT_KERN_FLAG(MNTK_UNMOUNTF); 4794 MNT_KERN_FLAG(MNTK_ASYNC); 4795 MNT_KERN_FLAG(MNTK_SOFTDEP); 4796 MNT_KERN_FLAG(MNTK_NOMSYNC); 4797 MNT_KERN_FLAG(MNTK_DRAINING); 4798 MNT_KERN_FLAG(MNTK_REFEXPIRE); 4799 MNT_KERN_FLAG(MNTK_EXTENDED_SHARED); 4800 MNT_KERN_FLAG(MNTK_SHARED_WRITES); 4801 MNT_KERN_FLAG(MNTK_NO_IOPF); 4802 MNT_KERN_FLAG(MNTK_RECURSE); 4803 MNT_KERN_FLAG(MNTK_UPPER_WAITER); 4804 MNT_KERN_FLAG(MNTK_UNLOCKED_INSMNTQUE); 4805 MNT_KERN_FLAG(MNTK_USES_BCACHE); 4806 MNT_KERN_FLAG(MNTK_VMSETSIZE_BUG); 4807 MNT_KERN_FLAG(MNTK_FPLOOKUP); 4808 MNT_KERN_FLAG(MNTK_TASKQUEUE_WAITER); 4809 MNT_KERN_FLAG(MNTK_NOASYNC); 4810 MNT_KERN_FLAG(MNTK_UNMOUNT); 4811 MNT_KERN_FLAG(MNTK_MWAIT); 4812 MNT_KERN_FLAG(MNTK_SUSPEND); 4813 MNT_KERN_FLAG(MNTK_SUSPEND2); 4814 MNT_KERN_FLAG(MNTK_SUSPENDED); 4815 MNT_KERN_FLAG(MNTK_NULL_NOCACHE); 4816 MNT_KERN_FLAG(MNTK_LOOKUP_SHARED); 4817 #undef MNT_KERN_FLAG 4818 if (flags != 0) { 4819 if (buf[0] != '\0') 4820 strlcat(buf, ", ", sizeof(buf)); 4821 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4822 "0x%08x", flags); 4823 } 4824 db_printf(" mnt_kern_flag = %s\n", buf); 4825 4826 db_printf(" mnt_opt = "); 4827 opt = TAILQ_FIRST(mp->mnt_opt); 4828 if (opt != NULL) { 4829 db_printf("%s", opt->name); 4830 opt = TAILQ_NEXT(opt, link); 4831 while (opt != NULL) { 4832 db_printf(", %s", opt->name); 4833 opt = TAILQ_NEXT(opt, link); 4834 } 4835 } 4836 db_printf("\n"); 4837 4838 sp = &mp->mnt_stat; 4839 db_printf(" mnt_stat = { version=%u type=%u flags=0x%016jx " 4840 "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju " 4841 "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju " 4842 "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n", 4843 (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags, 4844 (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize, 4845 (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree, 4846 (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files, 4847 (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites, 4848 (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads, 4849 (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax, 4850 (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]); 4851 4852 db_printf(" mnt_cred = { uid=%u ruid=%u", 4853 (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid); 4854 if (jailed(mp->mnt_cred)) 4855 db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id); 4856 db_printf(" }\n"); 4857 db_printf(" mnt_ref = %d (with %d in the struct)\n", 4858 vfs_mount_fetch_counter(mp, MNT_COUNT_REF), mp->mnt_ref); 4859 db_printf(" mnt_gen = %d\n", mp->mnt_gen); 4860 db_printf(" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize); 4861 db_printf(" mnt_lazyvnodelistsize = %d\n", 4862 mp->mnt_lazyvnodelistsize); 4863 db_printf(" mnt_writeopcount = %d (with %d in the struct)\n", 4864 vfs_mount_fetch_counter(mp, MNT_COUNT_WRITEOPCOUNT), mp->mnt_writeopcount); 4865 db_printf(" mnt_iosize_max = %d\n", mp->mnt_iosize_max); 4866 db_printf(" mnt_hashseed = %u\n", mp->mnt_hashseed); 4867 db_printf(" mnt_lockref = %d (with %d in the struct)\n", 4868 vfs_mount_fetch_counter(mp, MNT_COUNT_LOCKREF), mp->mnt_lockref); 4869 db_printf(" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes); 4870 db_printf(" mnt_secondary_accwrites = %d\n", 4871 mp->mnt_secondary_accwrites); 4872 db_printf(" mnt_gjprovider = %s\n", 4873 mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL"); 4874 db_printf(" mnt_vfs_ops = %d\n", mp->mnt_vfs_ops); 4875 4876 db_printf("\n\nList of active vnodes\n"); 4877 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4878 if (vp->v_type != VMARKER && vp->v_holdcnt > 0) { 4879 vn_printf(vp, "vnode "); 4880 if (db_pager_quit) 4881 break; 4882 } 4883 } 4884 db_printf("\n\nList of inactive vnodes\n"); 4885 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4886 if (vp->v_type != VMARKER && vp->v_holdcnt == 0) { 4887 vn_printf(vp, "vnode "); 4888 if (db_pager_quit) 4889 break; 4890 } 4891 } 4892 } 4893 #endif /* DDB */ 4894 4895 /* 4896 * Fill in a struct xvfsconf based on a struct vfsconf. 4897 */ 4898 static int 4899 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp) 4900 { 4901 struct xvfsconf xvfsp; 4902 4903 bzero(&xvfsp, sizeof(xvfsp)); 4904 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4905 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4906 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4907 xvfsp.vfc_flags = vfsp->vfc_flags; 4908 /* 4909 * These are unused in userland, we keep them 4910 * to not break binary compatibility. 4911 */ 4912 xvfsp.vfc_vfsops = NULL; 4913 xvfsp.vfc_next = NULL; 4914 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4915 } 4916 4917 #ifdef COMPAT_FREEBSD32 4918 struct xvfsconf32 { 4919 uint32_t vfc_vfsops; 4920 char vfc_name[MFSNAMELEN]; 4921 int32_t vfc_typenum; 4922 int32_t vfc_refcount; 4923 int32_t vfc_flags; 4924 uint32_t vfc_next; 4925 }; 4926 4927 static int 4928 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp) 4929 { 4930 struct xvfsconf32 xvfsp; 4931 4932 bzero(&xvfsp, sizeof(xvfsp)); 4933 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4934 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4935 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4936 xvfsp.vfc_flags = vfsp->vfc_flags; 4937 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4938 } 4939 #endif 4940 4941 /* 4942 * Top level filesystem related information gathering. 4943 */ 4944 static int 4945 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS) 4946 { 4947 struct vfsconf *vfsp; 4948 int error; 4949 4950 error = 0; 4951 vfsconf_slock(); 4952 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4953 #ifdef COMPAT_FREEBSD32 4954 if (req->flags & SCTL_MASK32) 4955 error = vfsconf2x32(req, vfsp); 4956 else 4957 #endif 4958 error = vfsconf2x(req, vfsp); 4959 if (error) 4960 break; 4961 } 4962 vfsconf_sunlock(); 4963 return (error); 4964 } 4965 4966 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD | 4967 CTLFLAG_MPSAFE, NULL, 0, sysctl_vfs_conflist, 4968 "S,xvfsconf", "List of all configured filesystems"); 4969 4970 #ifndef BURN_BRIDGES 4971 static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS); 4972 4973 static int 4974 vfs_sysctl(SYSCTL_HANDLER_ARGS) 4975 { 4976 int *name = (int *)arg1 - 1; /* XXX */ 4977 u_int namelen = arg2 + 1; /* XXX */ 4978 struct vfsconf *vfsp; 4979 4980 log(LOG_WARNING, "userland calling deprecated sysctl, " 4981 "please rebuild world\n"); 4982 4983 #if 1 || defined(COMPAT_PRELITE2) 4984 /* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */ 4985 if (namelen == 1) 4986 return (sysctl_ovfs_conf(oidp, arg1, arg2, req)); 4987 #endif 4988 4989 switch (name[1]) { 4990 case VFS_MAXTYPENUM: 4991 if (namelen != 2) 4992 return (ENOTDIR); 4993 return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int))); 4994 case VFS_CONF: 4995 if (namelen != 3) 4996 return (ENOTDIR); /* overloaded */ 4997 vfsconf_slock(); 4998 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4999 if (vfsp->vfc_typenum == name[2]) 5000 break; 5001 } 5002 vfsconf_sunlock(); 5003 if (vfsp == NULL) 5004 return (EOPNOTSUPP); 5005 #ifdef COMPAT_FREEBSD32 5006 if (req->flags & SCTL_MASK32) 5007 return (vfsconf2x32(req, vfsp)); 5008 else 5009 #endif 5010 return (vfsconf2x(req, vfsp)); 5011 } 5012 return (EOPNOTSUPP); 5013 } 5014 5015 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP | 5016 CTLFLAG_MPSAFE, vfs_sysctl, 5017 "Generic filesystem"); 5018 5019 #if 1 || defined(COMPAT_PRELITE2) 5020 5021 static int 5022 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS) 5023 { 5024 int error; 5025 struct vfsconf *vfsp; 5026 struct ovfsconf ovfs; 5027 5028 vfsconf_slock(); 5029 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 5030 bzero(&ovfs, sizeof(ovfs)); 5031 ovfs.vfc_vfsops = vfsp->vfc_vfsops; /* XXX used as flag */ 5032 strcpy(ovfs.vfc_name, vfsp->vfc_name); 5033 ovfs.vfc_index = vfsp->vfc_typenum; 5034 ovfs.vfc_refcount = vfsp->vfc_refcount; 5035 ovfs.vfc_flags = vfsp->vfc_flags; 5036 error = SYSCTL_OUT(req, &ovfs, sizeof ovfs); 5037 if (error != 0) { 5038 vfsconf_sunlock(); 5039 return (error); 5040 } 5041 } 5042 vfsconf_sunlock(); 5043 return (0); 5044 } 5045 5046 #endif /* 1 || COMPAT_PRELITE2 */ 5047 #endif /* !BURN_BRIDGES */ 5048 5049 static void 5050 unmount_or_warn(struct mount *mp) 5051 { 5052 int error; 5053 5054 error = dounmount(mp, MNT_FORCE, curthread); 5055 if (error != 0) { 5056 printf("unmount of %s failed (", mp->mnt_stat.f_mntonname); 5057 if (error == EBUSY) 5058 printf("BUSY)\n"); 5059 else 5060 printf("%d)\n", error); 5061 } 5062 } 5063 5064 /* 5065 * Unmount all filesystems. The list is traversed in reverse order 5066 * of mounting to avoid dependencies. 5067 */ 5068 void 5069 vfs_unmountall(void) 5070 { 5071 struct mount *mp, *tmp; 5072 5073 CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__); 5074 5075 /* 5076 * Since this only runs when rebooting, it is not interlocked. 5077 */ 5078 TAILQ_FOREACH_REVERSE_SAFE(mp, &mountlist, mntlist, mnt_list, tmp) { 5079 vfs_ref(mp); 5080 5081 /* 5082 * Forcibly unmounting "/dev" before "/" would prevent clean 5083 * unmount of the latter. 5084 */ 5085 if (mp == rootdevmp) 5086 continue; 5087 5088 unmount_or_warn(mp); 5089 } 5090 5091 if (rootdevmp != NULL) 5092 unmount_or_warn(rootdevmp); 5093 } 5094 5095 static void 5096 vfs_deferred_inactive(struct vnode *vp, int lkflags) 5097 { 5098 5099 ASSERT_VI_LOCKED(vp, __func__); 5100 VNPASS((vp->v_iflag & VI_DEFINACT) == 0, vp); 5101 if ((vp->v_iflag & VI_OWEINACT) == 0) { 5102 vdropl(vp); 5103 return; 5104 } 5105 if (vn_lock(vp, lkflags) == 0) { 5106 VI_LOCK(vp); 5107 vinactive(vp); 5108 VOP_UNLOCK(vp); 5109 vdropl(vp); 5110 return; 5111 } 5112 vdefer_inactive_unlocked(vp); 5113 } 5114 5115 static int 5116 vfs_periodic_inactive_filter(struct vnode *vp, void *arg) 5117 { 5118 5119 return (vp->v_iflag & VI_DEFINACT); 5120 } 5121 5122 static void __noinline 5123 vfs_periodic_inactive(struct mount *mp, int flags) 5124 { 5125 struct vnode *vp, *mvp; 5126 int lkflags; 5127 5128 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 5129 if (flags != MNT_WAIT) 5130 lkflags |= LK_NOWAIT; 5131 5132 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_inactive_filter, NULL) { 5133 if ((vp->v_iflag & VI_DEFINACT) == 0) { 5134 VI_UNLOCK(vp); 5135 continue; 5136 } 5137 vp->v_iflag &= ~VI_DEFINACT; 5138 vfs_deferred_inactive(vp, lkflags); 5139 } 5140 } 5141 5142 static inline bool 5143 vfs_want_msync(struct vnode *vp) 5144 { 5145 struct vm_object *obj; 5146 5147 /* 5148 * This test may be performed without any locks held. 5149 * We rely on vm_object's type stability. 5150 */ 5151 if (vp->v_vflag & VV_NOSYNC) 5152 return (false); 5153 obj = vp->v_object; 5154 return (obj != NULL && vm_object_mightbedirty(obj)); 5155 } 5156 5157 static int 5158 vfs_periodic_msync_inactive_filter(struct vnode *vp, void *arg __unused) 5159 { 5160 5161 if (vp->v_vflag & VV_NOSYNC) 5162 return (false); 5163 if (vp->v_iflag & VI_DEFINACT) 5164 return (true); 5165 return (vfs_want_msync(vp)); 5166 } 5167 5168 static void __noinline 5169 vfs_periodic_msync_inactive(struct mount *mp, int flags) 5170 { 5171 struct vnode *vp, *mvp; 5172 int lkflags; 5173 bool seen_defer; 5174 5175 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 5176 if (flags != MNT_WAIT) 5177 lkflags |= LK_NOWAIT; 5178 5179 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_msync_inactive_filter, NULL) { 5180 seen_defer = false; 5181 if (vp->v_iflag & VI_DEFINACT) { 5182 vp->v_iflag &= ~VI_DEFINACT; 5183 seen_defer = true; 5184 } 5185 if (!vfs_want_msync(vp)) { 5186 if (seen_defer) 5187 vfs_deferred_inactive(vp, lkflags); 5188 else 5189 VI_UNLOCK(vp); 5190 continue; 5191 } 5192 if (vget(vp, lkflags) == 0) { 5193 if ((vp->v_vflag & VV_NOSYNC) == 0) { 5194 if (flags == MNT_WAIT) 5195 vnode_pager_clean_sync(vp); 5196 else 5197 vnode_pager_clean_async(vp); 5198 } 5199 vput(vp); 5200 if (seen_defer) 5201 vdrop(vp); 5202 } else { 5203 if (seen_defer) 5204 vdefer_inactive_unlocked(vp); 5205 } 5206 } 5207 } 5208 5209 void 5210 vfs_periodic(struct mount *mp, int flags) 5211 { 5212 5213 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 5214 5215 if ((mp->mnt_kern_flag & MNTK_NOMSYNC) != 0) 5216 vfs_periodic_inactive(mp, flags); 5217 else 5218 vfs_periodic_msync_inactive(mp, flags); 5219 } 5220 5221 static void 5222 destroy_vpollinfo_free(struct vpollinfo *vi) 5223 { 5224 5225 knlist_destroy(&vi->vpi_selinfo.si_note); 5226 mtx_destroy(&vi->vpi_lock); 5227 free(vi, M_VNODEPOLL); 5228 } 5229 5230 static void 5231 destroy_vpollinfo(struct vpollinfo *vi) 5232 { 5233 5234 knlist_clear(&vi->vpi_selinfo.si_note, 1); 5235 seldrain(&vi->vpi_selinfo); 5236 destroy_vpollinfo_free(vi); 5237 } 5238 5239 /* 5240 * Initialize per-vnode helper structure to hold poll-related state. 5241 */ 5242 void 5243 v_addpollinfo(struct vnode *vp) 5244 { 5245 struct vpollinfo *vi; 5246 5247 if (vp->v_pollinfo != NULL) 5248 return; 5249 vi = malloc(sizeof(*vi), M_VNODEPOLL, M_WAITOK | M_ZERO); 5250 mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF); 5251 knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock, 5252 vfs_knlunlock, vfs_knl_assert_lock); 5253 VI_LOCK(vp); 5254 if (vp->v_pollinfo != NULL) { 5255 VI_UNLOCK(vp); 5256 destroy_vpollinfo_free(vi); 5257 return; 5258 } 5259 vp->v_pollinfo = vi; 5260 VI_UNLOCK(vp); 5261 } 5262 5263 /* 5264 * Record a process's interest in events which might happen to 5265 * a vnode. Because poll uses the historic select-style interface 5266 * internally, this routine serves as both the ``check for any 5267 * pending events'' and the ``record my interest in future events'' 5268 * functions. (These are done together, while the lock is held, 5269 * to avoid race conditions.) 5270 */ 5271 int 5272 vn_pollrecord(struct vnode *vp, struct thread *td, int events) 5273 { 5274 5275 v_addpollinfo(vp); 5276 mtx_lock(&vp->v_pollinfo->vpi_lock); 5277 if (vp->v_pollinfo->vpi_revents & events) { 5278 /* 5279 * This leaves events we are not interested 5280 * in available for the other process which 5281 * which presumably had requested them 5282 * (otherwise they would never have been 5283 * recorded). 5284 */ 5285 events &= vp->v_pollinfo->vpi_revents; 5286 vp->v_pollinfo->vpi_revents &= ~events; 5287 5288 mtx_unlock(&vp->v_pollinfo->vpi_lock); 5289 return (events); 5290 } 5291 vp->v_pollinfo->vpi_events |= events; 5292 selrecord(td, &vp->v_pollinfo->vpi_selinfo); 5293 mtx_unlock(&vp->v_pollinfo->vpi_lock); 5294 return (0); 5295 } 5296 5297 /* 5298 * Routine to create and manage a filesystem syncer vnode. 5299 */ 5300 #define sync_close ((int (*)(struct vop_close_args *))nullop) 5301 static int sync_fsync(struct vop_fsync_args *); 5302 static int sync_inactive(struct vop_inactive_args *); 5303 static int sync_reclaim(struct vop_reclaim_args *); 5304 5305 static struct vop_vector sync_vnodeops = { 5306 .vop_bypass = VOP_EOPNOTSUPP, 5307 .vop_close = sync_close, 5308 .vop_fsync = sync_fsync, 5309 .vop_getwritemount = vop_stdgetwritemount, 5310 .vop_inactive = sync_inactive, 5311 .vop_need_inactive = vop_stdneed_inactive, 5312 .vop_reclaim = sync_reclaim, 5313 .vop_lock1 = vop_stdlock, 5314 .vop_unlock = vop_stdunlock, 5315 .vop_islocked = vop_stdislocked, 5316 .vop_fplookup_vexec = VOP_EAGAIN, 5317 .vop_fplookup_symlink = VOP_EAGAIN, 5318 }; 5319 VFS_VOP_VECTOR_REGISTER(sync_vnodeops); 5320 5321 /* 5322 * Create a new filesystem syncer vnode for the specified mount point. 5323 */ 5324 void 5325 vfs_allocate_syncvnode(struct mount *mp) 5326 { 5327 struct vnode *vp; 5328 struct bufobj *bo; 5329 static long start, incr, next; 5330 int error; 5331 5332 /* Allocate a new vnode */ 5333 error = getnewvnode("syncer", mp, &sync_vnodeops, &vp); 5334 if (error != 0) 5335 panic("vfs_allocate_syncvnode: getnewvnode() failed"); 5336 vp->v_type = VNON; 5337 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5338 vp->v_vflag |= VV_FORCEINSMQ; 5339 error = insmntque1(vp, mp); 5340 if (error != 0) 5341 panic("vfs_allocate_syncvnode: insmntque() failed"); 5342 vp->v_vflag &= ~VV_FORCEINSMQ; 5343 vn_set_state(vp, VSTATE_CONSTRUCTED); 5344 VOP_UNLOCK(vp); 5345 /* 5346 * Place the vnode onto the syncer worklist. We attempt to 5347 * scatter them about on the list so that they will go off 5348 * at evenly distributed times even if all the filesystems 5349 * are mounted at once. 5350 */ 5351 next += incr; 5352 if (next == 0 || next > syncer_maxdelay) { 5353 start /= 2; 5354 incr /= 2; 5355 if (start == 0) { 5356 start = syncer_maxdelay / 2; 5357 incr = syncer_maxdelay; 5358 } 5359 next = start; 5360 } 5361 bo = &vp->v_bufobj; 5362 BO_LOCK(bo); 5363 vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0); 5364 /* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */ 5365 mtx_lock(&sync_mtx); 5366 sync_vnode_count++; 5367 if (mp->mnt_syncer == NULL) { 5368 mp->mnt_syncer = vp; 5369 vp = NULL; 5370 } 5371 mtx_unlock(&sync_mtx); 5372 BO_UNLOCK(bo); 5373 if (vp != NULL) { 5374 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5375 vgone(vp); 5376 vput(vp); 5377 } 5378 } 5379 5380 void 5381 vfs_deallocate_syncvnode(struct mount *mp) 5382 { 5383 struct vnode *vp; 5384 5385 mtx_lock(&sync_mtx); 5386 vp = mp->mnt_syncer; 5387 if (vp != NULL) 5388 mp->mnt_syncer = NULL; 5389 mtx_unlock(&sync_mtx); 5390 if (vp != NULL) 5391 vrele(vp); 5392 } 5393 5394 /* 5395 * Do a lazy sync of the filesystem. 5396 */ 5397 static int 5398 sync_fsync(struct vop_fsync_args *ap) 5399 { 5400 struct vnode *syncvp = ap->a_vp; 5401 struct mount *mp = syncvp->v_mount; 5402 int error, save; 5403 struct bufobj *bo; 5404 5405 /* 5406 * We only need to do something if this is a lazy evaluation. 5407 */ 5408 if (ap->a_waitfor != MNT_LAZY) 5409 return (0); 5410 5411 /* 5412 * Move ourselves to the back of the sync list. 5413 */ 5414 bo = &syncvp->v_bufobj; 5415 BO_LOCK(bo); 5416 vn_syncer_add_to_worklist(bo, syncdelay); 5417 BO_UNLOCK(bo); 5418 5419 /* 5420 * Walk the list of vnodes pushing all that are dirty and 5421 * not already on the sync list. 5422 */ 5423 if (vfs_busy(mp, MBF_NOWAIT) != 0) 5424 return (0); 5425 VOP_UNLOCK(syncvp); 5426 save = curthread_pflags_set(TDP_SYNCIO); 5427 /* 5428 * The filesystem at hand may be idle with free vnodes stored in the 5429 * batch. Return them instead of letting them stay there indefinitely. 5430 */ 5431 vfs_periodic(mp, MNT_NOWAIT); 5432 error = VFS_SYNC(mp, MNT_LAZY); 5433 curthread_pflags_restore(save); 5434 vn_lock(syncvp, LK_EXCLUSIVE | LK_RETRY); 5435 vfs_unbusy(mp); 5436 return (error); 5437 } 5438 5439 /* 5440 * The syncer vnode is no referenced. 5441 */ 5442 static int 5443 sync_inactive(struct vop_inactive_args *ap) 5444 { 5445 5446 vgone(ap->a_vp); 5447 return (0); 5448 } 5449 5450 /* 5451 * The syncer vnode is no longer needed and is being decommissioned. 5452 * 5453 * Modifications to the worklist must be protected by sync_mtx. 5454 */ 5455 static int 5456 sync_reclaim(struct vop_reclaim_args *ap) 5457 { 5458 struct vnode *vp = ap->a_vp; 5459 struct bufobj *bo; 5460 5461 bo = &vp->v_bufobj; 5462 BO_LOCK(bo); 5463 mtx_lock(&sync_mtx); 5464 if (vp->v_mount->mnt_syncer == vp) 5465 vp->v_mount->mnt_syncer = NULL; 5466 if (bo->bo_flag & BO_ONWORKLST) { 5467 LIST_REMOVE(bo, bo_synclist); 5468 syncer_worklist_len--; 5469 sync_vnode_count--; 5470 bo->bo_flag &= ~BO_ONWORKLST; 5471 } 5472 mtx_unlock(&sync_mtx); 5473 BO_UNLOCK(bo); 5474 5475 return (0); 5476 } 5477 5478 int 5479 vn_need_pageq_flush(struct vnode *vp) 5480 { 5481 struct vm_object *obj; 5482 5483 obj = vp->v_object; 5484 return (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 5485 vm_object_mightbedirty(obj)); 5486 } 5487 5488 /* 5489 * Check if vnode represents a disk device 5490 */ 5491 bool 5492 vn_isdisk_error(struct vnode *vp, int *errp) 5493 { 5494 int error; 5495 5496 if (vp->v_type != VCHR) { 5497 error = ENOTBLK; 5498 goto out; 5499 } 5500 error = 0; 5501 dev_lock(); 5502 if (vp->v_rdev == NULL) 5503 error = ENXIO; 5504 else if (vp->v_rdev->si_devsw == NULL) 5505 error = ENXIO; 5506 else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK)) 5507 error = ENOTBLK; 5508 dev_unlock(); 5509 out: 5510 *errp = error; 5511 return (error == 0); 5512 } 5513 5514 bool 5515 vn_isdisk(struct vnode *vp) 5516 { 5517 int error; 5518 5519 return (vn_isdisk_error(vp, &error)); 5520 } 5521 5522 /* 5523 * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see 5524 * the comment above cache_fplookup for details. 5525 */ 5526 int 5527 vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid, struct ucred *cred) 5528 { 5529 int error; 5530 5531 VFS_SMR_ASSERT_ENTERED(); 5532 5533 /* Check the owner. */ 5534 if (cred->cr_uid == file_uid) { 5535 if (file_mode & S_IXUSR) 5536 return (0); 5537 goto out_error; 5538 } 5539 5540 /* Otherwise, check the groups (first match) */ 5541 if (groupmember(file_gid, cred)) { 5542 if (file_mode & S_IXGRP) 5543 return (0); 5544 goto out_error; 5545 } 5546 5547 /* Otherwise, check everyone else. */ 5548 if (file_mode & S_IXOTH) 5549 return (0); 5550 out_error: 5551 /* 5552 * Permission check failed, but it is possible denial will get overwritten 5553 * (e.g., when root is traversing through a 700 directory owned by someone 5554 * else). 5555 * 5556 * vaccess() calls priv_check_cred which in turn can descent into MAC 5557 * modules overriding this result. It's quite unclear what semantics 5558 * are allowed for them to operate, thus for safety we don't call them 5559 * from within the SMR section. This also means if any such modules 5560 * are present, we have to let the regular lookup decide. 5561 */ 5562 error = priv_check_cred_vfs_lookup_nomac(cred); 5563 switch (error) { 5564 case 0: 5565 return (0); 5566 case EAGAIN: 5567 /* 5568 * MAC modules present. 5569 */ 5570 return (EAGAIN); 5571 case EPERM: 5572 return (EACCES); 5573 default: 5574 return (error); 5575 } 5576 } 5577 5578 /* 5579 * Common filesystem object access control check routine. Accepts a 5580 * vnode's type, "mode", uid and gid, requested access mode, and credentials. 5581 * Returns 0 on success, or an errno on failure. 5582 */ 5583 int 5584 vaccess(__enum_uint8(vtype) type, mode_t file_mode, uid_t file_uid, gid_t file_gid, 5585 accmode_t accmode, struct ucred *cred) 5586 { 5587 accmode_t dac_granted; 5588 accmode_t priv_granted; 5589 5590 KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0, 5591 ("invalid bit in accmode")); 5592 KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE), 5593 ("VAPPEND without VWRITE")); 5594 5595 /* 5596 * Look for a normal, non-privileged way to access the file/directory 5597 * as requested. If it exists, go with that. 5598 */ 5599 5600 dac_granted = 0; 5601 5602 /* Check the owner. */ 5603 if (cred->cr_uid == file_uid) { 5604 dac_granted |= VADMIN; 5605 if (file_mode & S_IXUSR) 5606 dac_granted |= VEXEC; 5607 if (file_mode & S_IRUSR) 5608 dac_granted |= VREAD; 5609 if (file_mode & S_IWUSR) 5610 dac_granted |= (VWRITE | VAPPEND); 5611 5612 if ((accmode & dac_granted) == accmode) 5613 return (0); 5614 5615 goto privcheck; 5616 } 5617 5618 /* Otherwise, check the groups (first match) */ 5619 if (groupmember(file_gid, cred)) { 5620 if (file_mode & S_IXGRP) 5621 dac_granted |= VEXEC; 5622 if (file_mode & S_IRGRP) 5623 dac_granted |= VREAD; 5624 if (file_mode & S_IWGRP) 5625 dac_granted |= (VWRITE | VAPPEND); 5626 5627 if ((accmode & dac_granted) == accmode) 5628 return (0); 5629 5630 goto privcheck; 5631 } 5632 5633 /* Otherwise, check everyone else. */ 5634 if (file_mode & S_IXOTH) 5635 dac_granted |= VEXEC; 5636 if (file_mode & S_IROTH) 5637 dac_granted |= VREAD; 5638 if (file_mode & S_IWOTH) 5639 dac_granted |= (VWRITE | VAPPEND); 5640 if ((accmode & dac_granted) == accmode) 5641 return (0); 5642 5643 privcheck: 5644 /* 5645 * Build a privilege mask to determine if the set of privileges 5646 * satisfies the requirements when combined with the granted mask 5647 * from above. For each privilege, if the privilege is required, 5648 * bitwise or the request type onto the priv_granted mask. 5649 */ 5650 priv_granted = 0; 5651 5652 if (type == VDIR) { 5653 /* 5654 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC 5655 * requests, instead of PRIV_VFS_EXEC. 5656 */ 5657 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5658 !priv_check_cred(cred, PRIV_VFS_LOOKUP)) 5659 priv_granted |= VEXEC; 5660 } else { 5661 /* 5662 * Ensure that at least one execute bit is on. Otherwise, 5663 * a privileged user will always succeed, and we don't want 5664 * this to happen unless the file really is executable. 5665 */ 5666 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5667 (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && 5668 !priv_check_cred(cred, PRIV_VFS_EXEC)) 5669 priv_granted |= VEXEC; 5670 } 5671 5672 if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) && 5673 !priv_check_cred(cred, PRIV_VFS_READ)) 5674 priv_granted |= VREAD; 5675 5676 if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) && 5677 !priv_check_cred(cred, PRIV_VFS_WRITE)) 5678 priv_granted |= (VWRITE | VAPPEND); 5679 5680 if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) && 5681 !priv_check_cred(cred, PRIV_VFS_ADMIN)) 5682 priv_granted |= VADMIN; 5683 5684 if ((accmode & (priv_granted | dac_granted)) == accmode) { 5685 return (0); 5686 } 5687 5688 return ((accmode & VADMIN) ? EPERM : EACCES); 5689 } 5690 5691 /* 5692 * Credential check based on process requesting service, and per-attribute 5693 * permissions. 5694 */ 5695 int 5696 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred, 5697 struct thread *td, accmode_t accmode) 5698 { 5699 5700 /* 5701 * Kernel-invoked always succeeds. 5702 */ 5703 if (cred == NOCRED) 5704 return (0); 5705 5706 /* 5707 * Do not allow privileged processes in jail to directly manipulate 5708 * system attributes. 5709 */ 5710 switch (attrnamespace) { 5711 case EXTATTR_NAMESPACE_SYSTEM: 5712 /* Potentially should be: return (EPERM); */ 5713 return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM)); 5714 case EXTATTR_NAMESPACE_USER: 5715 return (VOP_ACCESS(vp, accmode, cred, td)); 5716 default: 5717 return (EPERM); 5718 } 5719 } 5720 5721 #ifdef DEBUG_VFS_LOCKS 5722 int vfs_badlock_ddb = 1; /* Drop into debugger on violation. */ 5723 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0, 5724 "Drop into debugger on lock violation"); 5725 5726 int vfs_badlock_mutex = 1; /* Check for interlock across VOPs. */ 5727 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex, 5728 0, "Check for interlock across VOPs"); 5729 5730 int vfs_badlock_print = 1; /* Print lock violations. */ 5731 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print, 5732 0, "Print lock violations"); 5733 5734 int vfs_badlock_vnode = 1; /* Print vnode details on lock violations. */ 5735 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_vnode, CTLFLAG_RW, &vfs_badlock_vnode, 5736 0, "Print vnode details on lock violations"); 5737 5738 #ifdef KDB 5739 int vfs_badlock_backtrace = 1; /* Print backtrace at lock violations. */ 5740 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW, 5741 &vfs_badlock_backtrace, 0, "Print backtrace at lock violations"); 5742 #endif 5743 5744 static void 5745 vfs_badlock(const char *msg, const char *str, struct vnode *vp) 5746 { 5747 5748 #ifdef KDB 5749 if (vfs_badlock_backtrace) 5750 kdb_backtrace(); 5751 #endif 5752 if (vfs_badlock_vnode) 5753 vn_printf(vp, "vnode "); 5754 if (vfs_badlock_print) 5755 printf("%s: %p %s\n", str, (void *)vp, msg); 5756 if (vfs_badlock_ddb) 5757 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5758 } 5759 5760 void 5761 assert_vi_locked(struct vnode *vp, const char *str) 5762 { 5763 5764 if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp))) 5765 vfs_badlock("interlock is not locked but should be", str, vp); 5766 } 5767 5768 void 5769 assert_vi_unlocked(struct vnode *vp, const char *str) 5770 { 5771 5772 if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp))) 5773 vfs_badlock("interlock is locked but should not be", str, vp); 5774 } 5775 5776 void 5777 assert_vop_locked(struct vnode *vp, const char *str) 5778 { 5779 if (KERNEL_PANICKED() || vp == NULL) 5780 return; 5781 5782 #ifdef WITNESS 5783 if ((vp->v_irflag & VIRF_CROSSMP) == 0 && 5784 witness_is_owned(&vp->v_vnlock->lock_object) == -1) 5785 #else 5786 int locked = VOP_ISLOCKED(vp); 5787 if (locked == 0 || locked == LK_EXCLOTHER) 5788 #endif 5789 vfs_badlock("is not locked but should be", str, vp); 5790 } 5791 5792 void 5793 assert_vop_unlocked(struct vnode *vp, const char *str) 5794 { 5795 if (KERNEL_PANICKED() || vp == NULL) 5796 return; 5797 5798 #ifdef WITNESS 5799 if ((vp->v_irflag & VIRF_CROSSMP) == 0 && 5800 witness_is_owned(&vp->v_vnlock->lock_object) == 1) 5801 #else 5802 if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE) 5803 #endif 5804 vfs_badlock("is locked but should not be", str, vp); 5805 } 5806 5807 void 5808 assert_vop_elocked(struct vnode *vp, const char *str) 5809 { 5810 if (KERNEL_PANICKED() || vp == NULL) 5811 return; 5812 5813 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) 5814 vfs_badlock("is not exclusive locked but should be", str, vp); 5815 } 5816 #endif /* DEBUG_VFS_LOCKS */ 5817 5818 void 5819 vop_rename_fail(struct vop_rename_args *ap) 5820 { 5821 5822 if (ap->a_tvp != NULL) 5823 vput(ap->a_tvp); 5824 if (ap->a_tdvp == ap->a_tvp) 5825 vrele(ap->a_tdvp); 5826 else 5827 vput(ap->a_tdvp); 5828 vrele(ap->a_fdvp); 5829 vrele(ap->a_fvp); 5830 } 5831 5832 void 5833 vop_rename_pre(void *ap) 5834 { 5835 struct vop_rename_args *a = ap; 5836 5837 #ifdef DEBUG_VFS_LOCKS 5838 if (a->a_tvp) 5839 ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME"); 5840 ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME"); 5841 ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME"); 5842 ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME"); 5843 5844 /* Check the source (from). */ 5845 if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock && 5846 (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock)) 5847 ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked"); 5848 if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock) 5849 ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked"); 5850 5851 /* Check the target. */ 5852 if (a->a_tvp) 5853 ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked"); 5854 ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked"); 5855 #endif 5856 /* 5857 * It may be tempting to add vn_seqc_write_begin/end calls here and 5858 * in vop_rename_post but that's not going to work out since some 5859 * filesystems relookup vnodes mid-rename. This is probably a bug. 5860 * 5861 * For now filesystems are expected to do the relevant calls after they 5862 * decide what vnodes to operate on. 5863 */ 5864 if (a->a_tdvp != a->a_fdvp) 5865 vhold(a->a_fdvp); 5866 if (a->a_tvp != a->a_fvp) 5867 vhold(a->a_fvp); 5868 vhold(a->a_tdvp); 5869 if (a->a_tvp) 5870 vhold(a->a_tvp); 5871 } 5872 5873 #ifdef DEBUG_VFS_LOCKS 5874 void 5875 vop_fplookup_vexec_debugpre(void *ap __unused) 5876 { 5877 5878 VFS_SMR_ASSERT_ENTERED(); 5879 } 5880 5881 void 5882 vop_fplookup_vexec_debugpost(void *ap, int rc) 5883 { 5884 struct vop_fplookup_vexec_args *a; 5885 struct vnode *vp; 5886 5887 a = ap; 5888 vp = a->a_vp; 5889 5890 VFS_SMR_ASSERT_ENTERED(); 5891 if (rc == EOPNOTSUPP) 5892 VNPASS(VN_IS_DOOMED(vp), vp); 5893 } 5894 5895 void 5896 vop_fplookup_symlink_debugpre(void *ap __unused) 5897 { 5898 5899 VFS_SMR_ASSERT_ENTERED(); 5900 } 5901 5902 void 5903 vop_fplookup_symlink_debugpost(void *ap __unused, int rc __unused) 5904 { 5905 5906 VFS_SMR_ASSERT_ENTERED(); 5907 } 5908 5909 static void 5910 vop_fsync_debugprepost(struct vnode *vp, const char *name) 5911 { 5912 if (vp->v_type == VCHR) 5913 ; 5914 /* 5915 * The shared vs. exclusive locking policy for fsync() 5916 * is actually determined by vp's write mount as indicated 5917 * by VOP_GETWRITEMOUNT(), which for stacked filesystems 5918 * may not be the same as vp->v_mount. However, if the 5919 * underlying filesystem which really handles the fsync() 5920 * supports shared locking, the stacked filesystem must also 5921 * be prepared for its VOP_FSYNC() operation to be called 5922 * with only a shared lock. On the other hand, if the 5923 * stacked filesystem claims support for shared write 5924 * locking but the underlying filesystem does not, and the 5925 * caller incorrectly uses a shared lock, this condition 5926 * should still be caught when the stacked filesystem 5927 * invokes VOP_FSYNC() on the underlying filesystem. 5928 */ 5929 else if (MNT_SHARED_WRITES(vp->v_mount)) 5930 ASSERT_VOP_LOCKED(vp, name); 5931 else 5932 ASSERT_VOP_ELOCKED(vp, name); 5933 } 5934 5935 void 5936 vop_fsync_debugpre(void *a) 5937 { 5938 struct vop_fsync_args *ap; 5939 5940 ap = a; 5941 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5942 } 5943 5944 void 5945 vop_fsync_debugpost(void *a, int rc __unused) 5946 { 5947 struct vop_fsync_args *ap; 5948 5949 ap = a; 5950 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5951 } 5952 5953 void 5954 vop_fdatasync_debugpre(void *a) 5955 { 5956 struct vop_fdatasync_args *ap; 5957 5958 ap = a; 5959 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5960 } 5961 5962 void 5963 vop_fdatasync_debugpost(void *a, int rc __unused) 5964 { 5965 struct vop_fdatasync_args *ap; 5966 5967 ap = a; 5968 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5969 } 5970 5971 void 5972 vop_strategy_debugpre(void *ap) 5973 { 5974 struct vop_strategy_args *a; 5975 struct buf *bp; 5976 5977 a = ap; 5978 bp = a->a_bp; 5979 5980 /* 5981 * Cluster ops lock their component buffers but not the IO container. 5982 */ 5983 if ((bp->b_flags & B_CLUSTER) != 0) 5984 return; 5985 5986 if (!KERNEL_PANICKED() && !BUF_ISLOCKED(bp)) { 5987 if (vfs_badlock_print) 5988 printf( 5989 "VOP_STRATEGY: bp is not locked but should be\n"); 5990 if (vfs_badlock_ddb) 5991 kdb_enter(KDB_WHY_VFSLOCK, "lock violation"); 5992 } 5993 } 5994 5995 void 5996 vop_lock_debugpre(void *ap) 5997 { 5998 struct vop_lock1_args *a = ap; 5999 6000 if ((a->a_flags & LK_INTERLOCK) == 0) 6001 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 6002 else 6003 ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK"); 6004 } 6005 6006 void 6007 vop_lock_debugpost(void *ap, int rc) 6008 { 6009 struct vop_lock1_args *a = ap; 6010 6011 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 6012 if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0) 6013 ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK"); 6014 } 6015 6016 void 6017 vop_unlock_debugpre(void *ap) 6018 { 6019 struct vop_unlock_args *a = ap; 6020 struct vnode *vp = a->a_vp; 6021 6022 VNPASS(vn_get_state(vp) != VSTATE_UNINITIALIZED, vp); 6023 ASSERT_VOP_LOCKED(vp, "VOP_UNLOCK"); 6024 } 6025 6026 void 6027 vop_need_inactive_debugpre(void *ap) 6028 { 6029 struct vop_need_inactive_args *a = ap; 6030 6031 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 6032 } 6033 6034 void 6035 vop_need_inactive_debugpost(void *ap, int rc) 6036 { 6037 struct vop_need_inactive_args *a = ap; 6038 6039 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 6040 } 6041 #endif 6042 6043 void 6044 vop_create_pre(void *ap) 6045 { 6046 struct vop_create_args *a; 6047 struct vnode *dvp; 6048 6049 a = ap; 6050 dvp = a->a_dvp; 6051 vn_seqc_write_begin(dvp); 6052 } 6053 6054 void 6055 vop_create_post(void *ap, int rc) 6056 { 6057 struct vop_create_args *a; 6058 struct vnode *dvp; 6059 6060 a = ap; 6061 dvp = a->a_dvp; 6062 vn_seqc_write_end(dvp); 6063 if (!rc) 6064 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6065 } 6066 6067 void 6068 vop_whiteout_pre(void *ap) 6069 { 6070 struct vop_whiteout_args *a; 6071 struct vnode *dvp; 6072 6073 a = ap; 6074 dvp = a->a_dvp; 6075 vn_seqc_write_begin(dvp); 6076 } 6077 6078 void 6079 vop_whiteout_post(void *ap, int rc) 6080 { 6081 struct vop_whiteout_args *a; 6082 struct vnode *dvp; 6083 6084 a = ap; 6085 dvp = a->a_dvp; 6086 vn_seqc_write_end(dvp); 6087 } 6088 6089 void 6090 vop_deleteextattr_pre(void *ap) 6091 { 6092 struct vop_deleteextattr_args *a; 6093 struct vnode *vp; 6094 6095 a = ap; 6096 vp = a->a_vp; 6097 vn_seqc_write_begin(vp); 6098 } 6099 6100 void 6101 vop_deleteextattr_post(void *ap, int rc) 6102 { 6103 struct vop_deleteextattr_args *a; 6104 struct vnode *vp; 6105 6106 a = ap; 6107 vp = a->a_vp; 6108 vn_seqc_write_end(vp); 6109 if (!rc) 6110 VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB); 6111 } 6112 6113 void 6114 vop_link_pre(void *ap) 6115 { 6116 struct vop_link_args *a; 6117 struct vnode *vp, *tdvp; 6118 6119 a = ap; 6120 vp = a->a_vp; 6121 tdvp = a->a_tdvp; 6122 vn_seqc_write_begin(vp); 6123 vn_seqc_write_begin(tdvp); 6124 } 6125 6126 void 6127 vop_link_post(void *ap, int rc) 6128 { 6129 struct vop_link_args *a; 6130 struct vnode *vp, *tdvp; 6131 6132 a = ap; 6133 vp = a->a_vp; 6134 tdvp = a->a_tdvp; 6135 vn_seqc_write_end(vp); 6136 vn_seqc_write_end(tdvp); 6137 if (!rc) { 6138 VFS_KNOTE_LOCKED(vp, NOTE_LINK); 6139 VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE); 6140 } 6141 } 6142 6143 void 6144 vop_mkdir_pre(void *ap) 6145 { 6146 struct vop_mkdir_args *a; 6147 struct vnode *dvp; 6148 6149 a = ap; 6150 dvp = a->a_dvp; 6151 vn_seqc_write_begin(dvp); 6152 } 6153 6154 void 6155 vop_mkdir_post(void *ap, int rc) 6156 { 6157 struct vop_mkdir_args *a; 6158 struct vnode *dvp; 6159 6160 a = ap; 6161 dvp = a->a_dvp; 6162 vn_seqc_write_end(dvp); 6163 if (!rc) 6164 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 6165 } 6166 6167 #ifdef DEBUG_VFS_LOCKS 6168 void 6169 vop_mkdir_debugpost(void *ap, int rc) 6170 { 6171 struct vop_mkdir_args *a; 6172 6173 a = ap; 6174 if (!rc) 6175 cache_validate(a->a_dvp, *a->a_vpp, a->a_cnp); 6176 } 6177 #endif 6178 6179 void 6180 vop_mknod_pre(void *ap) 6181 { 6182 struct vop_mknod_args *a; 6183 struct vnode *dvp; 6184 6185 a = ap; 6186 dvp = a->a_dvp; 6187 vn_seqc_write_begin(dvp); 6188 } 6189 6190 void 6191 vop_mknod_post(void *ap, int rc) 6192 { 6193 struct vop_mknod_args *a; 6194 struct vnode *dvp; 6195 6196 a = ap; 6197 dvp = a->a_dvp; 6198 vn_seqc_write_end(dvp); 6199 if (!rc) 6200 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6201 } 6202 6203 void 6204 vop_reclaim_post(void *ap, int rc) 6205 { 6206 struct vop_reclaim_args *a; 6207 struct vnode *vp; 6208 6209 a = ap; 6210 vp = a->a_vp; 6211 ASSERT_VOP_IN_SEQC(vp); 6212 if (!rc) 6213 VFS_KNOTE_LOCKED(vp, NOTE_REVOKE); 6214 } 6215 6216 void 6217 vop_remove_pre(void *ap) 6218 { 6219 struct vop_remove_args *a; 6220 struct vnode *dvp, *vp; 6221 6222 a = ap; 6223 dvp = a->a_dvp; 6224 vp = a->a_vp; 6225 vn_seqc_write_begin(dvp); 6226 vn_seqc_write_begin(vp); 6227 } 6228 6229 void 6230 vop_remove_post(void *ap, int rc) 6231 { 6232 struct vop_remove_args *a; 6233 struct vnode *dvp, *vp; 6234 6235 a = ap; 6236 dvp = a->a_dvp; 6237 vp = a->a_vp; 6238 vn_seqc_write_end(dvp); 6239 vn_seqc_write_end(vp); 6240 if (!rc) { 6241 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6242 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 6243 } 6244 } 6245 6246 void 6247 vop_rename_post(void *ap, int rc) 6248 { 6249 struct vop_rename_args *a = ap; 6250 long hint; 6251 6252 if (!rc) { 6253 hint = NOTE_WRITE; 6254 if (a->a_fdvp == a->a_tdvp) { 6255 if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR) 6256 hint |= NOTE_LINK; 6257 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 6258 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 6259 } else { 6260 hint |= NOTE_EXTEND; 6261 if (a->a_fvp->v_type == VDIR) 6262 hint |= NOTE_LINK; 6263 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 6264 6265 if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL && 6266 a->a_tvp->v_type == VDIR) 6267 hint &= ~NOTE_LINK; 6268 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 6269 } 6270 6271 VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME); 6272 if (a->a_tvp) 6273 VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE); 6274 } 6275 if (a->a_tdvp != a->a_fdvp) 6276 vdrop(a->a_fdvp); 6277 if (a->a_tvp != a->a_fvp) 6278 vdrop(a->a_fvp); 6279 vdrop(a->a_tdvp); 6280 if (a->a_tvp) 6281 vdrop(a->a_tvp); 6282 } 6283 6284 void 6285 vop_rmdir_pre(void *ap) 6286 { 6287 struct vop_rmdir_args *a; 6288 struct vnode *dvp, *vp; 6289 6290 a = ap; 6291 dvp = a->a_dvp; 6292 vp = a->a_vp; 6293 vn_seqc_write_begin(dvp); 6294 vn_seqc_write_begin(vp); 6295 } 6296 6297 void 6298 vop_rmdir_post(void *ap, int rc) 6299 { 6300 struct vop_rmdir_args *a; 6301 struct vnode *dvp, *vp; 6302 6303 a = ap; 6304 dvp = a->a_dvp; 6305 vp = a->a_vp; 6306 vn_seqc_write_end(dvp); 6307 vn_seqc_write_end(vp); 6308 if (!rc) { 6309 vp->v_vflag |= VV_UNLINKED; 6310 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 6311 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 6312 } 6313 } 6314 6315 void 6316 vop_setattr_pre(void *ap) 6317 { 6318 struct vop_setattr_args *a; 6319 struct vnode *vp; 6320 6321 a = ap; 6322 vp = a->a_vp; 6323 vn_seqc_write_begin(vp); 6324 } 6325 6326 void 6327 vop_setattr_post(void *ap, int rc) 6328 { 6329 struct vop_setattr_args *a; 6330 struct vnode *vp; 6331 6332 a = ap; 6333 vp = a->a_vp; 6334 vn_seqc_write_end(vp); 6335 if (!rc) 6336 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 6337 } 6338 6339 void 6340 vop_setacl_pre(void *ap) 6341 { 6342 struct vop_setacl_args *a; 6343 struct vnode *vp; 6344 6345 a = ap; 6346 vp = a->a_vp; 6347 vn_seqc_write_begin(vp); 6348 } 6349 6350 void 6351 vop_setacl_post(void *ap, int rc __unused) 6352 { 6353 struct vop_setacl_args *a; 6354 struct vnode *vp; 6355 6356 a = ap; 6357 vp = a->a_vp; 6358 vn_seqc_write_end(vp); 6359 } 6360 6361 void 6362 vop_setextattr_pre(void *ap) 6363 { 6364 struct vop_setextattr_args *a; 6365 struct vnode *vp; 6366 6367 a = ap; 6368 vp = a->a_vp; 6369 vn_seqc_write_begin(vp); 6370 } 6371 6372 void 6373 vop_setextattr_post(void *ap, int rc) 6374 { 6375 struct vop_setextattr_args *a; 6376 struct vnode *vp; 6377 6378 a = ap; 6379 vp = a->a_vp; 6380 vn_seqc_write_end(vp); 6381 if (!rc) 6382 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 6383 } 6384 6385 void 6386 vop_symlink_pre(void *ap) 6387 { 6388 struct vop_symlink_args *a; 6389 struct vnode *dvp; 6390 6391 a = ap; 6392 dvp = a->a_dvp; 6393 vn_seqc_write_begin(dvp); 6394 } 6395 6396 void 6397 vop_symlink_post(void *ap, int rc) 6398 { 6399 struct vop_symlink_args *a; 6400 struct vnode *dvp; 6401 6402 a = ap; 6403 dvp = a->a_dvp; 6404 vn_seqc_write_end(dvp); 6405 if (!rc) 6406 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6407 } 6408 6409 void 6410 vop_open_post(void *ap, int rc) 6411 { 6412 struct vop_open_args *a = ap; 6413 6414 if (!rc) 6415 VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN); 6416 } 6417 6418 void 6419 vop_close_post(void *ap, int rc) 6420 { 6421 struct vop_close_args *a = ap; 6422 6423 if (!rc && (a->a_cred != NOCRED || /* filter out revokes */ 6424 !VN_IS_DOOMED(a->a_vp))) { 6425 VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ? 6426 NOTE_CLOSE_WRITE : NOTE_CLOSE); 6427 } 6428 } 6429 6430 void 6431 vop_read_post(void *ap, int rc) 6432 { 6433 struct vop_read_args *a = ap; 6434 6435 if (!rc) 6436 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 6437 } 6438 6439 void 6440 vop_read_pgcache_post(void *ap, int rc) 6441 { 6442 struct vop_read_pgcache_args *a = ap; 6443 6444 if (!rc) 6445 VFS_KNOTE_UNLOCKED(a->a_vp, NOTE_READ); 6446 } 6447 6448 void 6449 vop_readdir_post(void *ap, int rc) 6450 { 6451 struct vop_readdir_args *a = ap; 6452 6453 if (!rc) 6454 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 6455 } 6456 6457 static struct knlist fs_knlist; 6458 6459 static void 6460 vfs_event_init(void *arg) 6461 { 6462 knlist_init_mtx(&fs_knlist, NULL); 6463 } 6464 /* XXX - correct order? */ 6465 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL); 6466 6467 void 6468 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused) 6469 { 6470 6471 KNOTE_UNLOCKED(&fs_knlist, event); 6472 } 6473 6474 static int filt_fsattach(struct knote *kn); 6475 static void filt_fsdetach(struct knote *kn); 6476 static int filt_fsevent(struct knote *kn, long hint); 6477 6478 struct filterops fs_filtops = { 6479 .f_isfd = 0, 6480 .f_attach = filt_fsattach, 6481 .f_detach = filt_fsdetach, 6482 .f_event = filt_fsevent 6483 }; 6484 6485 static int 6486 filt_fsattach(struct knote *kn) 6487 { 6488 6489 kn->kn_flags |= EV_CLEAR; 6490 knlist_add(&fs_knlist, kn, 0); 6491 return (0); 6492 } 6493 6494 static void 6495 filt_fsdetach(struct knote *kn) 6496 { 6497 6498 knlist_remove(&fs_knlist, kn, 0); 6499 } 6500 6501 static int 6502 filt_fsevent(struct knote *kn, long hint) 6503 { 6504 6505 kn->kn_fflags |= kn->kn_sfflags & hint; 6506 6507 return (kn->kn_fflags != 0); 6508 } 6509 6510 static int 6511 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS) 6512 { 6513 struct vfsidctl vc; 6514 int error; 6515 struct mount *mp; 6516 6517 error = SYSCTL_IN(req, &vc, sizeof(vc)); 6518 if (error) 6519 return (error); 6520 if (vc.vc_vers != VFS_CTL_VERS1) 6521 return (EINVAL); 6522 mp = vfs_getvfs(&vc.vc_fsid); 6523 if (mp == NULL) 6524 return (ENOENT); 6525 /* ensure that a specific sysctl goes to the right filesystem. */ 6526 if (strcmp(vc.vc_fstypename, "*") != 0 && 6527 strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) { 6528 vfs_rel(mp); 6529 return (EINVAL); 6530 } 6531 VCTLTOREQ(&vc, req); 6532 error = VFS_SYSCTL(mp, vc.vc_op, req); 6533 vfs_rel(mp); 6534 return (error); 6535 } 6536 6537 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_MPSAFE | CTLFLAG_WR, 6538 NULL, 0, sysctl_vfs_ctl, "", 6539 "Sysctl by fsid"); 6540 6541 /* 6542 * Function to initialize a va_filerev field sensibly. 6543 * XXX: Wouldn't a random number make a lot more sense ?? 6544 */ 6545 u_quad_t 6546 init_va_filerev(void) 6547 { 6548 struct bintime bt; 6549 6550 getbinuptime(&bt); 6551 return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL)); 6552 } 6553 6554 static int filt_vfsread(struct knote *kn, long hint); 6555 static int filt_vfswrite(struct knote *kn, long hint); 6556 static int filt_vfsvnode(struct knote *kn, long hint); 6557 static void filt_vfsdetach(struct knote *kn); 6558 static struct filterops vfsread_filtops = { 6559 .f_isfd = 1, 6560 .f_detach = filt_vfsdetach, 6561 .f_event = filt_vfsread 6562 }; 6563 static struct filterops vfswrite_filtops = { 6564 .f_isfd = 1, 6565 .f_detach = filt_vfsdetach, 6566 .f_event = filt_vfswrite 6567 }; 6568 static struct filterops vfsvnode_filtops = { 6569 .f_isfd = 1, 6570 .f_detach = filt_vfsdetach, 6571 .f_event = filt_vfsvnode 6572 }; 6573 6574 static void 6575 vfs_knllock(void *arg) 6576 { 6577 struct vnode *vp = arg; 6578 6579 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 6580 } 6581 6582 static void 6583 vfs_knlunlock(void *arg) 6584 { 6585 struct vnode *vp = arg; 6586 6587 VOP_UNLOCK(vp); 6588 } 6589 6590 static void 6591 vfs_knl_assert_lock(void *arg, int what) 6592 { 6593 #ifdef DEBUG_VFS_LOCKS 6594 struct vnode *vp = arg; 6595 6596 if (what == LA_LOCKED) 6597 ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked"); 6598 else 6599 ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked"); 6600 #endif 6601 } 6602 6603 int 6604 vfs_kqfilter(struct vop_kqfilter_args *ap) 6605 { 6606 struct vnode *vp = ap->a_vp; 6607 struct knote *kn = ap->a_kn; 6608 struct knlist *knl; 6609 6610 KASSERT(vp->v_type != VFIFO || (kn->kn_filter != EVFILT_READ && 6611 kn->kn_filter != EVFILT_WRITE), 6612 ("READ/WRITE filter on a FIFO leaked through")); 6613 switch (kn->kn_filter) { 6614 case EVFILT_READ: 6615 kn->kn_fop = &vfsread_filtops; 6616 break; 6617 case EVFILT_WRITE: 6618 kn->kn_fop = &vfswrite_filtops; 6619 break; 6620 case EVFILT_VNODE: 6621 kn->kn_fop = &vfsvnode_filtops; 6622 break; 6623 default: 6624 return (EINVAL); 6625 } 6626 6627 kn->kn_hook = (caddr_t)vp; 6628 6629 v_addpollinfo(vp); 6630 if (vp->v_pollinfo == NULL) 6631 return (ENOMEM); 6632 knl = &vp->v_pollinfo->vpi_selinfo.si_note; 6633 vhold(vp); 6634 knlist_add(knl, kn, 0); 6635 6636 return (0); 6637 } 6638 6639 /* 6640 * Detach knote from vnode 6641 */ 6642 static void 6643 filt_vfsdetach(struct knote *kn) 6644 { 6645 struct vnode *vp = (struct vnode *)kn->kn_hook; 6646 6647 KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo")); 6648 knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0); 6649 vdrop(vp); 6650 } 6651 6652 /*ARGSUSED*/ 6653 static int 6654 filt_vfsread(struct knote *kn, long hint) 6655 { 6656 struct vnode *vp = (struct vnode *)kn->kn_hook; 6657 off_t size; 6658 int res; 6659 6660 /* 6661 * filesystem is gone, so set the EOF flag and schedule 6662 * the knote for deletion. 6663 */ 6664 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6665 VI_LOCK(vp); 6666 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6667 VI_UNLOCK(vp); 6668 return (1); 6669 } 6670 6671 if (vn_getsize_locked(vp, &size, curthread->td_ucred) != 0) 6672 return (0); 6673 6674 VI_LOCK(vp); 6675 kn->kn_data = size - kn->kn_fp->f_offset; 6676 res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 || kn->kn_data != 0; 6677 VI_UNLOCK(vp); 6678 return (res); 6679 } 6680 6681 /*ARGSUSED*/ 6682 static int 6683 filt_vfswrite(struct knote *kn, long hint) 6684 { 6685 struct vnode *vp = (struct vnode *)kn->kn_hook; 6686 6687 VI_LOCK(vp); 6688 6689 /* 6690 * filesystem is gone, so set the EOF flag and schedule 6691 * the knote for deletion. 6692 */ 6693 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) 6694 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6695 6696 kn->kn_data = 0; 6697 VI_UNLOCK(vp); 6698 return (1); 6699 } 6700 6701 static int 6702 filt_vfsvnode(struct knote *kn, long hint) 6703 { 6704 struct vnode *vp = (struct vnode *)kn->kn_hook; 6705 int res; 6706 6707 VI_LOCK(vp); 6708 if (kn->kn_sfflags & hint) 6709 kn->kn_fflags |= hint; 6710 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6711 kn->kn_flags |= EV_EOF; 6712 VI_UNLOCK(vp); 6713 return (1); 6714 } 6715 res = (kn->kn_fflags != 0); 6716 VI_UNLOCK(vp); 6717 return (res); 6718 } 6719 6720 int 6721 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off) 6722 { 6723 int error; 6724 6725 if (dp->d_reclen > ap->a_uio->uio_resid) 6726 return (ENAMETOOLONG); 6727 error = uiomove(dp, dp->d_reclen, ap->a_uio); 6728 if (error) { 6729 if (ap->a_ncookies != NULL) { 6730 if (ap->a_cookies != NULL) 6731 free(ap->a_cookies, M_TEMP); 6732 ap->a_cookies = NULL; 6733 *ap->a_ncookies = 0; 6734 } 6735 return (error); 6736 } 6737 if (ap->a_ncookies == NULL) 6738 return (0); 6739 6740 KASSERT(ap->a_cookies, 6741 ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!")); 6742 6743 *ap->a_cookies = realloc(*ap->a_cookies, 6744 (*ap->a_ncookies + 1) * sizeof(uint64_t), M_TEMP, M_WAITOK | M_ZERO); 6745 (*ap->a_cookies)[*ap->a_ncookies] = off; 6746 *ap->a_ncookies += 1; 6747 return (0); 6748 } 6749 6750 /* 6751 * The purpose of this routine is to remove granularity from accmode_t, 6752 * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE, 6753 * VADMIN and VAPPEND. 6754 * 6755 * If it returns 0, the caller is supposed to continue with the usual 6756 * access checks using 'accmode' as modified by this routine. If it 6757 * returns nonzero value, the caller is supposed to return that value 6758 * as errno. 6759 * 6760 * Note that after this routine runs, accmode may be zero. 6761 */ 6762 int 6763 vfs_unixify_accmode(accmode_t *accmode) 6764 { 6765 /* 6766 * There is no way to specify explicit "deny" rule using 6767 * file mode or POSIX.1e ACLs. 6768 */ 6769 if (*accmode & VEXPLICIT_DENY) { 6770 *accmode = 0; 6771 return (0); 6772 } 6773 6774 /* 6775 * None of these can be translated into usual access bits. 6776 * Also, the common case for NFSv4 ACLs is to not contain 6777 * either of these bits. Caller should check for VWRITE 6778 * on the containing directory instead. 6779 */ 6780 if (*accmode & (VDELETE_CHILD | VDELETE)) 6781 return (EPERM); 6782 6783 if (*accmode & VADMIN_PERMS) { 6784 *accmode &= ~VADMIN_PERMS; 6785 *accmode |= VADMIN; 6786 } 6787 6788 /* 6789 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL 6790 * or VSYNCHRONIZE using file mode or POSIX.1e ACL. 6791 */ 6792 *accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE); 6793 6794 return (0); 6795 } 6796 6797 /* 6798 * Clear out a doomed vnode (if any) and replace it with a new one as long 6799 * as the fs is not being unmounted. Return the root vnode to the caller. 6800 */ 6801 static int __noinline 6802 vfs_cache_root_fallback(struct mount *mp, int flags, struct vnode **vpp) 6803 { 6804 struct vnode *vp; 6805 int error; 6806 6807 restart: 6808 if (mp->mnt_rootvnode != NULL) { 6809 MNT_ILOCK(mp); 6810 vp = mp->mnt_rootvnode; 6811 if (vp != NULL) { 6812 if (!VN_IS_DOOMED(vp)) { 6813 vrefact(vp); 6814 MNT_IUNLOCK(mp); 6815 error = vn_lock(vp, flags); 6816 if (error == 0) { 6817 *vpp = vp; 6818 return (0); 6819 } 6820 vrele(vp); 6821 goto restart; 6822 } 6823 /* 6824 * Clear the old one. 6825 */ 6826 mp->mnt_rootvnode = NULL; 6827 } 6828 MNT_IUNLOCK(mp); 6829 if (vp != NULL) { 6830 vfs_op_barrier_wait(mp); 6831 vrele(vp); 6832 } 6833 } 6834 error = VFS_CACHEDROOT(mp, flags, vpp); 6835 if (error != 0) 6836 return (error); 6837 if (mp->mnt_vfs_ops == 0) { 6838 MNT_ILOCK(mp); 6839 if (mp->mnt_vfs_ops != 0) { 6840 MNT_IUNLOCK(mp); 6841 return (0); 6842 } 6843 if (mp->mnt_rootvnode == NULL) { 6844 vrefact(*vpp); 6845 mp->mnt_rootvnode = *vpp; 6846 } else { 6847 if (mp->mnt_rootvnode != *vpp) { 6848 if (!VN_IS_DOOMED(mp->mnt_rootvnode)) { 6849 panic("%s: mismatch between vnode returned " 6850 " by VFS_CACHEDROOT and the one cached " 6851 " (%p != %p)", 6852 __func__, *vpp, mp->mnt_rootvnode); 6853 } 6854 } 6855 } 6856 MNT_IUNLOCK(mp); 6857 } 6858 return (0); 6859 } 6860 6861 int 6862 vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp) 6863 { 6864 struct mount_pcpu *mpcpu; 6865 struct vnode *vp; 6866 int error; 6867 6868 if (!vfs_op_thread_enter(mp, mpcpu)) 6869 return (vfs_cache_root_fallback(mp, flags, vpp)); 6870 vp = atomic_load_ptr(&mp->mnt_rootvnode); 6871 if (vp == NULL || VN_IS_DOOMED(vp)) { 6872 vfs_op_thread_exit(mp, mpcpu); 6873 return (vfs_cache_root_fallback(mp, flags, vpp)); 6874 } 6875 vrefact(vp); 6876 vfs_op_thread_exit(mp, mpcpu); 6877 error = vn_lock(vp, flags); 6878 if (error != 0) { 6879 vrele(vp); 6880 return (vfs_cache_root_fallback(mp, flags, vpp)); 6881 } 6882 *vpp = vp; 6883 return (0); 6884 } 6885 6886 struct vnode * 6887 vfs_cache_root_clear(struct mount *mp) 6888 { 6889 struct vnode *vp; 6890 6891 /* 6892 * ops > 0 guarantees there is nobody who can see this vnode 6893 */ 6894 MPASS(mp->mnt_vfs_ops > 0); 6895 vp = mp->mnt_rootvnode; 6896 if (vp != NULL) 6897 vn_seqc_write_begin(vp); 6898 mp->mnt_rootvnode = NULL; 6899 return (vp); 6900 } 6901 6902 void 6903 vfs_cache_root_set(struct mount *mp, struct vnode *vp) 6904 { 6905 6906 MPASS(mp->mnt_vfs_ops > 0); 6907 vrefact(vp); 6908 mp->mnt_rootvnode = vp; 6909 } 6910 6911 /* 6912 * These are helper functions for filesystems to traverse all 6913 * their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h. 6914 * 6915 * This interface replaces MNT_VNODE_FOREACH. 6916 */ 6917 6918 struct vnode * 6919 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp) 6920 { 6921 struct vnode *vp; 6922 6923 maybe_yield(); 6924 MNT_ILOCK(mp); 6925 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6926 for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL; 6927 vp = TAILQ_NEXT(vp, v_nmntvnodes)) { 6928 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6929 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6930 continue; 6931 VI_LOCK(vp); 6932 if (VN_IS_DOOMED(vp)) { 6933 VI_UNLOCK(vp); 6934 continue; 6935 } 6936 break; 6937 } 6938 if (vp == NULL) { 6939 __mnt_vnode_markerfree_all(mvp, mp); 6940 /* MNT_IUNLOCK(mp); -- done in above function */ 6941 mtx_assert(MNT_MTX(mp), MA_NOTOWNED); 6942 return (NULL); 6943 } 6944 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6945 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6946 MNT_IUNLOCK(mp); 6947 return (vp); 6948 } 6949 6950 struct vnode * 6951 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp) 6952 { 6953 struct vnode *vp; 6954 6955 *mvp = vn_alloc_marker(mp); 6956 MNT_ILOCK(mp); 6957 MNT_REF(mp); 6958 6959 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 6960 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 6961 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 6962 continue; 6963 VI_LOCK(vp); 6964 if (VN_IS_DOOMED(vp)) { 6965 VI_UNLOCK(vp); 6966 continue; 6967 } 6968 break; 6969 } 6970 if (vp == NULL) { 6971 MNT_REL(mp); 6972 MNT_IUNLOCK(mp); 6973 vn_free_marker(*mvp); 6974 *mvp = NULL; 6975 return (NULL); 6976 } 6977 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 6978 MNT_IUNLOCK(mp); 6979 return (vp); 6980 } 6981 6982 void 6983 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp) 6984 { 6985 6986 if (*mvp == NULL) { 6987 MNT_IUNLOCK(mp); 6988 return; 6989 } 6990 6991 mtx_assert(MNT_MTX(mp), MA_OWNED); 6992 6993 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 6994 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 6995 MNT_REL(mp); 6996 MNT_IUNLOCK(mp); 6997 vn_free_marker(*mvp); 6998 *mvp = NULL; 6999 } 7000 7001 /* 7002 * These are helper functions for filesystems to traverse their 7003 * lazy vnodes. See MNT_VNODE_FOREACH_LAZY() in sys/mount.h 7004 */ 7005 static void 7006 mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 7007 { 7008 7009 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 7010 7011 MNT_ILOCK(mp); 7012 MNT_REL(mp); 7013 MNT_IUNLOCK(mp); 7014 vn_free_marker(*mvp); 7015 *mvp = NULL; 7016 } 7017 7018 /* 7019 * Relock the mp mount vnode list lock with the vp vnode interlock in the 7020 * conventional lock order during mnt_vnode_next_lazy iteration. 7021 * 7022 * On entry, the mount vnode list lock is held and the vnode interlock is not. 7023 * The list lock is dropped and reacquired. On success, both locks are held. 7024 * On failure, the mount vnode list lock is held but the vnode interlock is 7025 * not, and the procedure may have yielded. 7026 */ 7027 static bool 7028 mnt_vnode_next_lazy_relock(struct vnode *mvp, struct mount *mp, 7029 struct vnode *vp) 7030 { 7031 7032 VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER && 7033 TAILQ_NEXT(mvp, v_lazylist) != NULL, mvp, 7034 ("%s: bad marker", __func__)); 7035 VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp, 7036 ("%s: inappropriate vnode", __func__)); 7037 ASSERT_VI_UNLOCKED(vp, __func__); 7038 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 7039 7040 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, mvp, v_lazylist); 7041 TAILQ_INSERT_BEFORE(vp, mvp, v_lazylist); 7042 7043 /* 7044 * Note we may be racing against vdrop which transitioned the hold 7045 * count to 0 and now waits for the ->mnt_listmtx lock. This is fine, 7046 * if we are the only user after we get the interlock we will just 7047 * vdrop. 7048 */ 7049 vhold(vp); 7050 mtx_unlock(&mp->mnt_listmtx); 7051 VI_LOCK(vp); 7052 if (VN_IS_DOOMED(vp)) { 7053 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 7054 goto out_lost; 7055 } 7056 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 7057 /* 7058 * There is nothing to do if we are the last user. 7059 */ 7060 if (!refcount_release_if_not_last(&vp->v_holdcnt)) 7061 goto out_lost; 7062 mtx_lock(&mp->mnt_listmtx); 7063 return (true); 7064 out_lost: 7065 vdropl(vp); 7066 maybe_yield(); 7067 mtx_lock(&mp->mnt_listmtx); 7068 return (false); 7069 } 7070 7071 static struct vnode * 7072 mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 7073 void *cbarg) 7074 { 7075 struct vnode *vp; 7076 7077 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 7078 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 7079 restart: 7080 vp = TAILQ_NEXT(*mvp, v_lazylist); 7081 while (vp != NULL) { 7082 if (vp->v_type == VMARKER) { 7083 vp = TAILQ_NEXT(vp, v_lazylist); 7084 continue; 7085 } 7086 /* 7087 * See if we want to process the vnode. Note we may encounter a 7088 * long string of vnodes we don't care about and hog the list 7089 * as a result. Check for it and requeue the marker. 7090 */ 7091 VNPASS(!VN_IS_DOOMED(vp), vp); 7092 if (!cb(vp, cbarg)) { 7093 if (!should_yield()) { 7094 vp = TAILQ_NEXT(vp, v_lazylist); 7095 continue; 7096 } 7097 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, 7098 v_lazylist); 7099 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, 7100 v_lazylist); 7101 mtx_unlock(&mp->mnt_listmtx); 7102 kern_yield(PRI_USER); 7103 mtx_lock(&mp->mnt_listmtx); 7104 goto restart; 7105 } 7106 /* 7107 * Try-lock because this is the wrong lock order. 7108 */ 7109 if (!VI_TRYLOCK(vp) && 7110 !mnt_vnode_next_lazy_relock(*mvp, mp, vp)) 7111 goto restart; 7112 KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp)); 7113 KASSERT(vp->v_mount == mp || vp->v_mount == NULL, 7114 ("alien vnode on the lazy list %p %p", vp, mp)); 7115 VNPASS(vp->v_mount == mp, vp); 7116 VNPASS(!VN_IS_DOOMED(vp), vp); 7117 break; 7118 } 7119 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 7120 7121 /* Check if we are done */ 7122 if (vp == NULL) { 7123 mtx_unlock(&mp->mnt_listmtx); 7124 mnt_vnode_markerfree_lazy(mvp, mp); 7125 return (NULL); 7126 } 7127 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, v_lazylist); 7128 mtx_unlock(&mp->mnt_listmtx); 7129 ASSERT_VI_LOCKED(vp, "lazy iter"); 7130 return (vp); 7131 } 7132 7133 struct vnode * 7134 __mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 7135 void *cbarg) 7136 { 7137 7138 maybe_yield(); 7139 mtx_lock(&mp->mnt_listmtx); 7140 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 7141 } 7142 7143 struct vnode * 7144 __mnt_vnode_first_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 7145 void *cbarg) 7146 { 7147 struct vnode *vp; 7148 7149 if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist)) 7150 return (NULL); 7151 7152 *mvp = vn_alloc_marker(mp); 7153 MNT_ILOCK(mp); 7154 MNT_REF(mp); 7155 MNT_IUNLOCK(mp); 7156 7157 mtx_lock(&mp->mnt_listmtx); 7158 vp = TAILQ_FIRST(&mp->mnt_lazyvnodelist); 7159 if (vp == NULL) { 7160 mtx_unlock(&mp->mnt_listmtx); 7161 mnt_vnode_markerfree_lazy(mvp, mp); 7162 return (NULL); 7163 } 7164 TAILQ_INSERT_BEFORE(vp, *mvp, v_lazylist); 7165 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 7166 } 7167 7168 void 7169 __mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 7170 { 7171 7172 if (*mvp == NULL) 7173 return; 7174 7175 mtx_lock(&mp->mnt_listmtx); 7176 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 7177 mtx_unlock(&mp->mnt_listmtx); 7178 mnt_vnode_markerfree_lazy(mvp, mp); 7179 } 7180 7181 int 7182 vn_dir_check_exec(struct vnode *vp, struct componentname *cnp) 7183 { 7184 7185 if ((cnp->cn_flags & NOEXECCHECK) != 0) { 7186 cnp->cn_flags &= ~NOEXECCHECK; 7187 return (0); 7188 } 7189 7190 return (VOP_ACCESS(vp, VEXEC, cnp->cn_cred, curthread)); 7191 } 7192 7193 /* 7194 * Do not use this variant unless you have means other than the hold count 7195 * to prevent the vnode from getting freed. 7196 */ 7197 void 7198 vn_seqc_write_begin_locked(struct vnode *vp) 7199 { 7200 7201 ASSERT_VI_LOCKED(vp, __func__); 7202 VNPASS(vp->v_holdcnt > 0, vp); 7203 VNPASS(vp->v_seqc_users >= 0, vp); 7204 vp->v_seqc_users++; 7205 if (vp->v_seqc_users == 1) 7206 seqc_sleepable_write_begin(&vp->v_seqc); 7207 } 7208 7209 void 7210 vn_seqc_write_begin(struct vnode *vp) 7211 { 7212 7213 VI_LOCK(vp); 7214 vn_seqc_write_begin_locked(vp); 7215 VI_UNLOCK(vp); 7216 } 7217 7218 void 7219 vn_seqc_write_end_locked(struct vnode *vp) 7220 { 7221 7222 ASSERT_VI_LOCKED(vp, __func__); 7223 VNPASS(vp->v_seqc_users > 0, vp); 7224 vp->v_seqc_users--; 7225 if (vp->v_seqc_users == 0) 7226 seqc_sleepable_write_end(&vp->v_seqc); 7227 } 7228 7229 void 7230 vn_seqc_write_end(struct vnode *vp) 7231 { 7232 7233 VI_LOCK(vp); 7234 vn_seqc_write_end_locked(vp); 7235 VI_UNLOCK(vp); 7236 } 7237 7238 /* 7239 * Special case handling for allocating and freeing vnodes. 7240 * 7241 * The counter remains unchanged on free so that a doomed vnode will 7242 * keep testing as in modify as long as it is accessible with SMR. 7243 */ 7244 static void 7245 vn_seqc_init(struct vnode *vp) 7246 { 7247 7248 vp->v_seqc = 0; 7249 vp->v_seqc_users = 0; 7250 } 7251 7252 static void 7253 vn_seqc_write_end_free(struct vnode *vp) 7254 { 7255 7256 VNPASS(seqc_in_modify(vp->v_seqc), vp); 7257 VNPASS(vp->v_seqc_users == 1, vp); 7258 } 7259 7260 void 7261 vn_irflag_set_locked(struct vnode *vp, short toset) 7262 { 7263 short flags; 7264 7265 ASSERT_VI_LOCKED(vp, __func__); 7266 flags = vn_irflag_read(vp); 7267 VNASSERT((flags & toset) == 0, vp, 7268 ("%s: some of the passed flags already set (have %d, passed %d)\n", 7269 __func__, flags, toset)); 7270 atomic_store_short(&vp->v_irflag, flags | toset); 7271 } 7272 7273 void 7274 vn_irflag_set(struct vnode *vp, short toset) 7275 { 7276 7277 VI_LOCK(vp); 7278 vn_irflag_set_locked(vp, toset); 7279 VI_UNLOCK(vp); 7280 } 7281 7282 void 7283 vn_irflag_set_cond_locked(struct vnode *vp, short toset) 7284 { 7285 short flags; 7286 7287 ASSERT_VI_LOCKED(vp, __func__); 7288 flags = vn_irflag_read(vp); 7289 atomic_store_short(&vp->v_irflag, flags | toset); 7290 } 7291 7292 void 7293 vn_irflag_set_cond(struct vnode *vp, short toset) 7294 { 7295 7296 VI_LOCK(vp); 7297 vn_irflag_set_cond_locked(vp, toset); 7298 VI_UNLOCK(vp); 7299 } 7300 7301 void 7302 vn_irflag_unset_locked(struct vnode *vp, short tounset) 7303 { 7304 short flags; 7305 7306 ASSERT_VI_LOCKED(vp, __func__); 7307 flags = vn_irflag_read(vp); 7308 VNASSERT((flags & tounset) == tounset, vp, 7309 ("%s: some of the passed flags not set (have %d, passed %d)\n", 7310 __func__, flags, tounset)); 7311 atomic_store_short(&vp->v_irflag, flags & ~tounset); 7312 } 7313 7314 void 7315 vn_irflag_unset(struct vnode *vp, short tounset) 7316 { 7317 7318 VI_LOCK(vp); 7319 vn_irflag_unset_locked(vp, tounset); 7320 VI_UNLOCK(vp); 7321 } 7322 7323 int 7324 vn_getsize_locked(struct vnode *vp, off_t *size, struct ucred *cred) 7325 { 7326 struct vattr vattr; 7327 int error; 7328 7329 ASSERT_VOP_LOCKED(vp, __func__); 7330 error = VOP_GETATTR(vp, &vattr, cred); 7331 if (__predict_true(error == 0)) { 7332 if (vattr.va_size <= OFF_MAX) 7333 *size = vattr.va_size; 7334 else 7335 error = EFBIG; 7336 } 7337 return (error); 7338 } 7339 7340 int 7341 vn_getsize(struct vnode *vp, off_t *size, struct ucred *cred) 7342 { 7343 int error; 7344 7345 VOP_LOCK(vp, LK_SHARED); 7346 error = vn_getsize_locked(vp, size, cred); 7347 VOP_UNLOCK(vp); 7348 return (error); 7349 } 7350 7351 #ifdef INVARIANTS 7352 void 7353 vn_set_state_validate(struct vnode *vp, __enum_uint8(vstate) state) 7354 { 7355 7356 switch (vp->v_state) { 7357 case VSTATE_UNINITIALIZED: 7358 switch (state) { 7359 case VSTATE_CONSTRUCTED: 7360 case VSTATE_DESTROYING: 7361 return; 7362 default: 7363 break; 7364 } 7365 break; 7366 case VSTATE_CONSTRUCTED: 7367 ASSERT_VOP_ELOCKED(vp, __func__); 7368 switch (state) { 7369 case VSTATE_DESTROYING: 7370 return; 7371 default: 7372 break; 7373 } 7374 break; 7375 case VSTATE_DESTROYING: 7376 ASSERT_VOP_ELOCKED(vp, __func__); 7377 switch (state) { 7378 case VSTATE_DEAD: 7379 return; 7380 default: 7381 break; 7382 } 7383 break; 7384 case VSTATE_DEAD: 7385 switch (state) { 7386 case VSTATE_UNINITIALIZED: 7387 return; 7388 default: 7389 break; 7390 } 7391 break; 7392 } 7393 7394 vn_printf(vp, "invalid state transition %d -> %d\n", vp->v_state, state); 7395 panic("invalid state transition %d -> %d\n", vp->v_state, state); 7396 } 7397 #endif 7398