1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 /* 38 * External virtual filesystem routines 39 */ 40 41 #include "opt_ddb.h" 42 #include "opt_watchdog.h" 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/asan.h> 47 #include <sys/bio.h> 48 #include <sys/buf.h> 49 #include <sys/capsicum.h> 50 #include <sys/condvar.h> 51 #include <sys/conf.h> 52 #include <sys/counter.h> 53 #include <sys/dirent.h> 54 #include <sys/event.h> 55 #include <sys/eventhandler.h> 56 #include <sys/extattr.h> 57 #include <sys/file.h> 58 #include <sys/fcntl.h> 59 #include <sys/inotify.h> 60 #include <sys/jail.h> 61 #include <sys/kdb.h> 62 #include <sys/kernel.h> 63 #include <sys/kthread.h> 64 #include <sys/ktr.h> 65 #include <sys/limits.h> 66 #include <sys/lockf.h> 67 #include <sys/malloc.h> 68 #include <sys/mount.h> 69 #include <sys/namei.h> 70 #include <sys/pctrie.h> 71 #include <sys/priv.h> 72 #include <sys/reboot.h> 73 #include <sys/refcount.h> 74 #include <sys/rwlock.h> 75 #include <sys/sched.h> 76 #include <sys/sleepqueue.h> 77 #include <sys/smr.h> 78 #include <sys/smp.h> 79 #include <sys/stat.h> 80 #include <sys/stdarg.h> 81 #include <sys/sysctl.h> 82 #include <sys/syslog.h> 83 #include <sys/user.h> 84 #include <sys/vmmeter.h> 85 #include <sys/vnode.h> 86 #include <sys/watchdog.h> 87 88 #include <security/mac/mac_framework.h> 89 90 #include <vm/vm.h> 91 #include <vm/vm_object.h> 92 #include <vm/vm_extern.h> 93 #include <vm/pmap.h> 94 #include <vm/vm_map.h> 95 #include <vm/vm_page.h> 96 #include <vm/vm_kern.h> 97 #include <vm/vnode_pager.h> 98 #include <vm/uma.h> 99 100 #ifdef DDB 101 #include <ddb/ddb.h> 102 #endif 103 104 static void delmntque(struct vnode *vp); 105 static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, 106 int slpflag, int slptimeo); 107 static void syncer_shutdown(void *arg, int howto); 108 static int vtryrecycle(struct vnode *vp, bool isvnlru); 109 static void v_init_counters(struct vnode *); 110 static void vn_seqc_init(struct vnode *); 111 static void vn_seqc_write_end_free(struct vnode *vp); 112 static void vgonel(struct vnode *); 113 static bool vhold_recycle_free(struct vnode *); 114 static void vdropl_recycle(struct vnode *vp); 115 static void vdrop_recycle(struct vnode *vp); 116 static void vfs_knllock(void *arg); 117 static void vfs_knlunlock(void *arg); 118 static void vfs_knl_assert_lock(void *arg, int what); 119 static void destroy_vpollinfo(struct vpollinfo *vi); 120 static int v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 121 daddr_t startlbn, daddr_t endlbn); 122 static void vnlru_recalc(void); 123 124 static SYSCTL_NODE(_vfs, OID_AUTO, vnode, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 125 "vnode configuration and statistics"); 126 static SYSCTL_NODE(_vfs_vnode, OID_AUTO, param, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 127 "vnode configuration"); 128 static SYSCTL_NODE(_vfs_vnode, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 129 "vnode statistics"); 130 static SYSCTL_NODE(_vfs_vnode, OID_AUTO, vnlru, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 131 "vnode recycling"); 132 133 /* 134 * Number of vnodes in existence. Increased whenever getnewvnode() 135 * allocates a new vnode, decreased in vdropl() for VIRF_DOOMED vnode. 136 */ 137 static u_long __exclusive_cache_line numvnodes; 138 139 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0, 140 "Number of vnodes in existence (legacy)"); 141 SYSCTL_ULONG(_vfs_vnode_stats, OID_AUTO, count, CTLFLAG_RD, &numvnodes, 0, 142 "Number of vnodes in existence"); 143 144 static counter_u64_t vnodes_created; 145 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created, 146 "Number of vnodes created by getnewvnode (legacy)"); 147 SYSCTL_COUNTER_U64(_vfs_vnode_stats, OID_AUTO, created, CTLFLAG_RD, &vnodes_created, 148 "Number of vnodes created by getnewvnode"); 149 150 /* 151 * Conversion tables for conversion from vnode types to inode formats 152 * and back. 153 */ 154 __enum_uint8(vtype) iftovt_tab[16] = { 155 VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON, 156 VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON 157 }; 158 int vttoif_tab[10] = { 159 0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK, 160 S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT 161 }; 162 163 /* 164 * List of allocates vnodes in the system. 165 */ 166 static TAILQ_HEAD(freelst, vnode) vnode_list; 167 static struct vnode *vnode_list_free_marker; 168 static struct vnode *vnode_list_reclaim_marker; 169 170 /* 171 * "Free" vnode target. Free vnodes are rarely completely free, but are 172 * just ones that are cheap to recycle. Usually they are for files which 173 * have been stat'd but not read; these usually have inode and namecache 174 * data attached to them. This target is the preferred minimum size of a 175 * sub-cache consisting mostly of such files. The system balances the size 176 * of this sub-cache with its complement to try to prevent either from 177 * thrashing while the other is relatively inactive. The targets express 178 * a preference for the best balance. 179 * 180 * "Above" this target there are 2 further targets (watermarks) related 181 * to recyling of free vnodes. In the best-operating case, the cache is 182 * exactly full, the free list has size between vlowat and vhiwat above the 183 * free target, and recycling from it and normal use maintains this state. 184 * Sometimes the free list is below vlowat or even empty, but this state 185 * is even better for immediate use provided the cache is not full. 186 * Otherwise, vnlru_proc() runs to reclaim enough vnodes (usually non-free 187 * ones) to reach one of these states. The watermarks are currently hard- 188 * coded as 4% and 9% of the available space higher. These and the default 189 * of 25% for wantfreevnodes are too large if the memory size is large. 190 * E.g., 9% of 75% of MAXVNODES is more than 566000 vnodes to reclaim 191 * whenever vnlru_proc() becomes active. 192 */ 193 static long wantfreevnodes; 194 static long __exclusive_cache_line freevnodes; 195 static long freevnodes_old; 196 197 static u_long recycles_count; 198 SYSCTL_ULONG(_vfs, OID_AUTO, recycles, CTLFLAG_RD | CTLFLAG_STATS, &recycles_count, 0, 199 "Number of vnodes recycled to meet vnode cache targets (legacy)"); 200 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, recycles, CTLFLAG_RD | CTLFLAG_STATS, 201 &recycles_count, 0, 202 "Number of vnodes recycled to meet vnode cache targets"); 203 204 static u_long recycles_free_count; 205 SYSCTL_ULONG(_vfs, OID_AUTO, recycles_free, CTLFLAG_RD | CTLFLAG_STATS, 206 &recycles_free_count, 0, 207 "Number of free vnodes recycled to meet vnode cache targets (legacy)"); 208 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, recycles_free, CTLFLAG_RD | CTLFLAG_STATS, 209 &recycles_free_count, 0, 210 "Number of free vnodes recycled to meet vnode cache targets"); 211 212 static counter_u64_t direct_recycles_free_count; 213 SYSCTL_COUNTER_U64(_vfs_vnode_vnlru, OID_AUTO, direct_recycles_free, CTLFLAG_RD, 214 &direct_recycles_free_count, 215 "Number of free vnodes recycled by vn_alloc callers to meet vnode cache targets"); 216 217 static counter_u64_t vnode_skipped_requeues; 218 SYSCTL_COUNTER_U64(_vfs_vnode_stats, OID_AUTO, skipped_requeues, CTLFLAG_RD, &vnode_skipped_requeues, 219 "Number of times LRU requeue was skipped due to lock contention"); 220 221 static __read_mostly bool vnode_can_skip_requeue; 222 SYSCTL_BOOL(_vfs_vnode_param, OID_AUTO, can_skip_requeue, CTLFLAG_RW, 223 &vnode_can_skip_requeue, 0, "Is LRU requeue skippable"); 224 225 static u_long deferred_inact; 226 SYSCTL_ULONG(_vfs, OID_AUTO, deferred_inact, CTLFLAG_RD, 227 &deferred_inact, 0, "Number of times inactive processing was deferred"); 228 229 /* To keep more than one thread at a time from running vfs_getnewfsid */ 230 static struct mtx mntid_mtx; 231 232 /* 233 * Lock for any access to the following: 234 * vnode_list 235 * numvnodes 236 * freevnodes 237 */ 238 static struct mtx __exclusive_cache_line vnode_list_mtx; 239 240 /* Publicly exported FS */ 241 struct nfs_public nfs_pub; 242 243 static uma_zone_t buf_trie_zone; 244 static smr_t buf_trie_smr; 245 246 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */ 247 static uma_zone_t vnode_zone; 248 MALLOC_DEFINE(M_VNODEPOLL, "VN POLL", "vnode poll"); 249 250 __read_frequently smr_t vfs_smr; 251 252 /* 253 * The workitem queue. 254 * 255 * It is useful to delay writes of file data and filesystem metadata 256 * for tens of seconds so that quickly created and deleted files need 257 * not waste disk bandwidth being created and removed. To realize this, 258 * we append vnodes to a "workitem" queue. When running with a soft 259 * updates implementation, most pending metadata dependencies should 260 * not wait for more than a few seconds. Thus, mounted on block devices 261 * are delayed only about a half the time that file data is delayed. 262 * Similarly, directory updates are more critical, so are only delayed 263 * about a third the time that file data is delayed. Thus, there are 264 * SYNCER_MAXDELAY queues that are processed round-robin at a rate of 265 * one each second (driven off the filesystem syncer process). The 266 * syncer_delayno variable indicates the next queue that is to be processed. 267 * Items that need to be processed soon are placed in this queue: 268 * 269 * syncer_workitem_pending[syncer_delayno] 270 * 271 * A delay of fifteen seconds is done by placing the request fifteen 272 * entries later in the queue: 273 * 274 * syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask] 275 * 276 */ 277 static int syncer_delayno; 278 static long syncer_mask; 279 LIST_HEAD(synclist, bufobj); 280 static struct synclist *syncer_workitem_pending; 281 /* 282 * The sync_mtx protects: 283 * bo->bo_synclist 284 * sync_vnode_count 285 * syncer_delayno 286 * syncer_state 287 * syncer_workitem_pending 288 * syncer_worklist_len 289 * rushjob 290 */ 291 static struct mtx sync_mtx; 292 static struct cv sync_wakeup; 293 294 #define SYNCER_MAXDELAY 32 295 static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */ 296 static int syncdelay = 30; /* max time to delay syncing data */ 297 static int filedelay = 30; /* time to delay syncing files */ 298 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0, 299 "Time to delay syncing files (in seconds)"); 300 static int dirdelay = 29; /* time to delay syncing directories */ 301 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0, 302 "Time to delay syncing directories (in seconds)"); 303 static int metadelay = 28; /* time to delay syncing metadata */ 304 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0, 305 "Time to delay syncing metadata (in seconds)"); 306 static int rushjob; /* number of slots to run ASAP */ 307 static int stat_rush_requests; /* number of times I/O speeded up */ 308 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0, 309 "Number of times I/O speeded up (rush requests)"); 310 311 #define VDBATCH_SIZE 8 312 struct vdbatch { 313 u_int index; 314 struct mtx lock; 315 struct vnode *tab[VDBATCH_SIZE]; 316 }; 317 DPCPU_DEFINE_STATIC(struct vdbatch, vd); 318 319 static void vdbatch_dequeue(struct vnode *vp); 320 321 /* 322 * The syncer will require at least SYNCER_MAXDELAY iterations to shutdown; 323 * we probably don't want to pause for the whole second each time. 324 */ 325 #define SYNCER_SHUTDOWN_SPEEDUP 32 326 static int sync_vnode_count; 327 static int syncer_worklist_len; 328 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY } 329 syncer_state; 330 331 /* Target for maximum number of vnodes. */ 332 u_long desiredvnodes; 333 static u_long gapvnodes; /* gap between wanted and desired */ 334 static u_long vhiwat; /* enough extras after expansion */ 335 static u_long vlowat; /* minimal extras before expansion */ 336 static bool vstir; /* nonzero to stir non-free vnodes */ 337 static volatile int vsmalltrigger = 8; /* pref to keep if > this many pages */ 338 339 static u_long vnlru_read_freevnodes(void); 340 341 /* 342 * Note that no attempt is made to sanitize these parameters. 343 */ 344 static int 345 sysctl_maxvnodes(SYSCTL_HANDLER_ARGS) 346 { 347 u_long val; 348 int error; 349 350 val = desiredvnodes; 351 error = sysctl_handle_long(oidp, &val, 0, req); 352 if (error != 0 || req->newptr == NULL) 353 return (error); 354 355 if (val == desiredvnodes) 356 return (0); 357 mtx_lock(&vnode_list_mtx); 358 desiredvnodes = val; 359 wantfreevnodes = desiredvnodes / 4; 360 vnlru_recalc(); 361 mtx_unlock(&vnode_list_mtx); 362 /* 363 * XXX There is no protection against multiple threads changing 364 * desiredvnodes at the same time. Locking above only helps vnlru and 365 * getnewvnode. 366 */ 367 vfs_hash_changesize(desiredvnodes); 368 cache_changesize(desiredvnodes); 369 return (0); 370 } 371 372 SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes, 373 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, 374 "LU", "Target for maximum number of vnodes (legacy)"); 375 SYSCTL_PROC(_vfs_vnode_param, OID_AUTO, limit, 376 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, 377 "LU", "Target for maximum number of vnodes"); 378 379 static int 380 sysctl_freevnodes(SYSCTL_HANDLER_ARGS) 381 { 382 u_long rfreevnodes; 383 384 rfreevnodes = vnlru_read_freevnodes(); 385 return (sysctl_handle_long(oidp, &rfreevnodes, 0, req)); 386 } 387 388 SYSCTL_PROC(_vfs, OID_AUTO, freevnodes, 389 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0, sysctl_freevnodes, 390 "LU", "Number of \"free\" vnodes (legacy)"); 391 SYSCTL_PROC(_vfs_vnode_stats, OID_AUTO, free, 392 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0, sysctl_freevnodes, 393 "LU", "Number of \"free\" vnodes"); 394 395 static int 396 sysctl_wantfreevnodes(SYSCTL_HANDLER_ARGS) 397 { 398 u_long val; 399 int error; 400 401 val = wantfreevnodes; 402 error = sysctl_handle_long(oidp, &val, 0, req); 403 if (error != 0 || req->newptr == NULL) 404 return (error); 405 406 if (val == wantfreevnodes) 407 return (0); 408 mtx_lock(&vnode_list_mtx); 409 wantfreevnodes = val; 410 vnlru_recalc(); 411 mtx_unlock(&vnode_list_mtx); 412 return (0); 413 } 414 415 SYSCTL_PROC(_vfs, OID_AUTO, wantfreevnodes, 416 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes, 417 "LU", "Target for minimum number of \"free\" vnodes (legacy)"); 418 SYSCTL_PROC(_vfs_vnode_param, OID_AUTO, wantfree, 419 CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes, 420 "LU", "Target for minimum number of \"free\" vnodes"); 421 422 static int vnlru_nowhere; 423 SYSCTL_INT(_vfs_vnode_vnlru, OID_AUTO, failed_runs, CTLFLAG_RD | CTLFLAG_STATS, 424 &vnlru_nowhere, 0, "Number of times the vnlru process ran without success"); 425 426 static int 427 sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS) 428 { 429 struct vnode *vp; 430 struct nameidata nd; 431 char *buf; 432 unsigned long ndflags; 433 int error; 434 435 if (req->newptr == NULL) 436 return (EINVAL); 437 if (req->newlen >= PATH_MAX) 438 return (E2BIG); 439 440 buf = malloc(PATH_MAX, M_TEMP, M_WAITOK); 441 error = SYSCTL_IN(req, buf, req->newlen); 442 if (error != 0) 443 goto out; 444 445 buf[req->newlen] = '\0'; 446 447 ndflags = LOCKLEAF | NOFOLLOW | AUDITVNODE1; 448 NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE, buf); 449 if ((error = namei(&nd)) != 0) 450 goto out; 451 vp = nd.ni_vp; 452 453 if (VN_IS_DOOMED(vp)) { 454 /* 455 * This vnode is being recycled. Return != 0 to let the caller 456 * know that the sysctl had no effect. Return EAGAIN because a 457 * subsequent call will likely succeed (since namei will create 458 * a new vnode if necessary) 459 */ 460 error = EAGAIN; 461 goto putvnode; 462 } 463 464 vgone(vp); 465 putvnode: 466 vput(vp); 467 NDFREE_PNBUF(&nd); 468 out: 469 free(buf, M_TEMP); 470 return (error); 471 } 472 473 static int 474 sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS) 475 { 476 struct thread *td = curthread; 477 struct vnode *vp; 478 struct file *fp; 479 int error; 480 int fd; 481 482 if (req->newptr == NULL) 483 return (EBADF); 484 485 error = sysctl_handle_int(oidp, &fd, 0, req); 486 if (error != 0) 487 return (error); 488 error = getvnode(curthread, fd, &cap_fcntl_rights, &fp); 489 if (error != 0) 490 return (error); 491 vp = fp->f_vnode; 492 493 error = vn_lock(vp, LK_EXCLUSIVE); 494 if (error != 0) 495 goto drop; 496 497 vgone(vp); 498 VOP_UNLOCK(vp); 499 drop: 500 fdrop(fp, td); 501 return (error); 502 } 503 504 SYSCTL_PROC(_debug, OID_AUTO, try_reclaim_vnode, 505 CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 506 sysctl_try_reclaim_vnode, "A", "Try to reclaim a vnode by its pathname"); 507 SYSCTL_PROC(_debug, OID_AUTO, ftry_reclaim_vnode, 508 CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0, 509 sysctl_ftry_reclaim_vnode, "I", 510 "Try to reclaim a vnode by its file descriptor"); 511 512 /* Shift count for (uintptr_t)vp to initialize vp->v_hash. */ 513 #define vnsz2log 8 514 #ifndef DEBUG_LOCKS 515 _Static_assert(sizeof(struct vnode) >= 1UL << vnsz2log && 516 sizeof(struct vnode) < 1UL << (vnsz2log + 1), 517 "vnsz2log needs to be updated"); 518 #endif 519 520 /* 521 * Support for the bufobj clean & dirty pctrie. 522 */ 523 static void * 524 buf_trie_alloc(struct pctrie *ptree) 525 { 526 return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT)); 527 } 528 529 static void 530 buf_trie_free(struct pctrie *ptree, void *node) 531 { 532 uma_zfree_smr(buf_trie_zone, node); 533 } 534 PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free, 535 buf_trie_smr); 536 537 /* 538 * Lookup the next element greater than or equal to lblkno, accounting for the 539 * fact that, for pctries, negative values are greater than nonnegative ones. 540 */ 541 static struct buf * 542 buf_lookup_ge(struct bufv *bv, daddr_t lblkno) 543 { 544 struct buf *bp; 545 546 bp = BUF_PCTRIE_LOOKUP_GE(&bv->bv_root, lblkno); 547 if (bp == NULL && lblkno < 0) 548 bp = BUF_PCTRIE_LOOKUP_GE(&bv->bv_root, 0); 549 if (bp != NULL && bp->b_lblkno < lblkno) 550 bp = NULL; 551 return (bp); 552 } 553 554 /* 555 * Insert bp, and find the next element smaller than bp, accounting for the fact 556 * that, for pctries, negative values are greater than nonnegative ones. 557 */ 558 static int 559 buf_insert_lookup_le(struct bufv *bv, struct buf *bp, struct buf **n) 560 { 561 int error; 562 563 error = BUF_PCTRIE_INSERT_LOOKUP_LE(&bv->bv_root, bp, n); 564 if (error != EEXIST) { 565 if (*n == NULL && bp->b_lblkno >= 0) 566 *n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, ~0L); 567 if (*n != NULL && (*n)->b_lblkno >= bp->b_lblkno) 568 *n = NULL; 569 } 570 return (error); 571 } 572 573 /* 574 * Initialize the vnode management data structures. 575 * 576 * Reevaluate the following cap on the number of vnodes after the physical 577 * memory size exceeds 512GB. In the limit, as the physical memory size 578 * grows, the ratio of the memory size in KB to vnodes approaches 64:1. 579 */ 580 #ifndef MAXVNODES_MAX 581 #define MAXVNODES_MAX (512UL * 1024 * 1024 / 64) /* 8M */ 582 #endif 583 584 static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); 585 586 static struct vnode * 587 vn_alloc_marker(struct mount *mp) 588 { 589 struct vnode *vp; 590 591 vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO); 592 vp->v_type = VMARKER; 593 vp->v_mount = mp; 594 595 return (vp); 596 } 597 598 static void 599 vn_free_marker(struct vnode *vp) 600 { 601 602 MPASS(vp->v_type == VMARKER); 603 free(vp, M_VNODE_MARKER); 604 } 605 606 #ifdef KASAN 607 static int 608 vnode_ctor(void *mem, int size, void *arg __unused, int flags __unused) 609 { 610 kasan_mark(mem, size, roundup2(size, UMA_ALIGN_PTR + 1), 0); 611 return (0); 612 } 613 614 static void 615 vnode_dtor(void *mem, int size, void *arg __unused) 616 { 617 size_t end1, end2, off1, off2; 618 619 _Static_assert(offsetof(struct vnode, v_vnodelist) < 620 offsetof(struct vnode, v_dbatchcpu), 621 "KASAN marks require updating"); 622 623 off1 = offsetof(struct vnode, v_vnodelist); 624 off2 = offsetof(struct vnode, v_dbatchcpu); 625 end1 = off1 + sizeof(((struct vnode *)NULL)->v_vnodelist); 626 end2 = off2 + sizeof(((struct vnode *)NULL)->v_dbatchcpu); 627 628 /* 629 * Access to the v_vnodelist and v_dbatchcpu fields are permitted even 630 * after the vnode has been freed. Try to get some KASAN coverage by 631 * marking everything except those two fields as invalid. Because 632 * KASAN's tracking is not byte-granular, any preceding fields sharing 633 * the same 8-byte aligned word must also be marked valid. 634 */ 635 636 /* Handle the area from the start until v_vnodelist... */ 637 off1 = rounddown2(off1, KASAN_SHADOW_SCALE); 638 kasan_mark(mem, off1, off1, KASAN_UMA_FREED); 639 640 /* ... then the area between v_vnodelist and v_dbatchcpu ... */ 641 off1 = roundup2(end1, KASAN_SHADOW_SCALE); 642 off2 = rounddown2(off2, KASAN_SHADOW_SCALE); 643 if (off2 > off1) 644 kasan_mark((void *)((char *)mem + off1), off2 - off1, 645 off2 - off1, KASAN_UMA_FREED); 646 647 /* ... and finally the area from v_dbatchcpu to the end. */ 648 off2 = roundup2(end2, KASAN_SHADOW_SCALE); 649 kasan_mark((void *)((char *)mem + off2), size - off2, size - off2, 650 KASAN_UMA_FREED); 651 } 652 #endif /* KASAN */ 653 654 /* 655 * Initialize a vnode as it first enters the zone. 656 */ 657 static int 658 vnode_init(void *mem, int size, int flags) 659 { 660 struct vnode *vp; 661 662 vp = mem; 663 bzero(vp, size); 664 /* 665 * Setup locks. 666 */ 667 vp->v_vnlock = &vp->v_lock; 668 mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF); 669 /* 670 * By default, don't allow shared locks unless filesystems opt-in. 671 */ 672 lockinit(vp->v_vnlock, PVFS, "vnode", VLKTIMEOUT, 673 LK_NOSHARE | LK_IS_VNODE); 674 /* 675 * Initialize bufobj. 676 */ 677 bufobj_init(&vp->v_bufobj, vp); 678 /* 679 * Initialize namecache. 680 */ 681 cache_vnode_init(vp); 682 /* 683 * Initialize rangelocks. 684 */ 685 rangelock_init(&vp->v_rl); 686 687 vp->v_dbatchcpu = NOCPU; 688 689 vp->v_state = VSTATE_DEAD; 690 691 /* 692 * Check vhold_recycle_free for an explanation. 693 */ 694 vp->v_holdcnt = VHOLD_NO_SMR; 695 vp->v_type = VNON; 696 mtx_lock(&vnode_list_mtx); 697 TAILQ_INSERT_BEFORE(vnode_list_free_marker, vp, v_vnodelist); 698 mtx_unlock(&vnode_list_mtx); 699 return (0); 700 } 701 702 /* 703 * Free a vnode when it is cleared from the zone. 704 */ 705 static void 706 vnode_fini(void *mem, int size) 707 { 708 struct vnode *vp; 709 struct bufobj *bo; 710 711 vp = mem; 712 vdbatch_dequeue(vp); 713 mtx_lock(&vnode_list_mtx); 714 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 715 mtx_unlock(&vnode_list_mtx); 716 rangelock_destroy(&vp->v_rl); 717 lockdestroy(vp->v_vnlock); 718 mtx_destroy(&vp->v_interlock); 719 bo = &vp->v_bufobj; 720 rw_destroy(BO_LOCKPTR(bo)); 721 722 kasan_mark(mem, size, size, 0); 723 } 724 725 /* 726 * Provide the size of NFS nclnode and NFS fh for calculation of the 727 * vnode memory consumption. The size is specified directly to 728 * eliminate dependency on NFS-private header. 729 * 730 * Other filesystems may use bigger or smaller (like UFS and ZFS) 731 * private inode data, but the NFS-based estimation is ample enough. 732 * Still, we care about differences in the size between 64- and 32-bit 733 * platforms. 734 * 735 * Namecache structure size is heuristically 736 * sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1. 737 */ 738 #ifdef _LP64 739 #define NFS_NCLNODE_SZ (528 + 64) 740 #define NC_SZ 148 741 #else 742 #define NFS_NCLNODE_SZ (360 + 32) 743 #define NC_SZ 92 744 #endif 745 746 static void 747 vntblinit(void *dummy __unused) 748 { 749 struct vdbatch *vd; 750 uma_ctor ctor; 751 uma_dtor dtor; 752 int cpu, physvnodes, virtvnodes; 753 754 /* 755 * 'desiredvnodes' is the minimum of a function of the physical memory 756 * size and another of the kernel heap size (UMA limit, a portion of the 757 * KVA). 758 * 759 * Currently, on 64-bit platforms, 'desiredvnodes' is set to 760 * 'virtvnodes' up to a physical memory cutoff of ~1722MB, after which 761 * 'physvnodes' applies instead. With the current automatic tuning for 762 * 'maxfiles' (32 files/MB), 'desiredvnodes' is always greater than it. 763 */ 764 physvnodes = maxproc + pgtok(vm_cnt.v_page_count) / 32 + 765 min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 32; 766 virtvnodes = vm_kmem_size / (10 * (sizeof(struct vm_object) + 767 sizeof(struct vnode) + NC_SZ * ncsizefactor + NFS_NCLNODE_SZ)); 768 desiredvnodes = min(physvnodes, virtvnodes); 769 if (desiredvnodes > MAXVNODES_MAX) { 770 if (bootverbose) 771 printf("Reducing kern.maxvnodes %lu -> %lu\n", 772 desiredvnodes, MAXVNODES_MAX); 773 desiredvnodes = MAXVNODES_MAX; 774 } 775 wantfreevnodes = desiredvnodes / 4; 776 mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF); 777 TAILQ_INIT(&vnode_list); 778 mtx_init(&vnode_list_mtx, "vnode_list", NULL, MTX_DEF); 779 /* 780 * The lock is taken to appease WITNESS. 781 */ 782 mtx_lock(&vnode_list_mtx); 783 vnlru_recalc(); 784 mtx_unlock(&vnode_list_mtx); 785 vnode_list_free_marker = vn_alloc_marker(NULL); 786 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist); 787 vnode_list_reclaim_marker = vn_alloc_marker(NULL); 788 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist); 789 790 #ifdef KASAN 791 ctor = vnode_ctor; 792 dtor = vnode_dtor; 793 #else 794 ctor = NULL; 795 dtor = NULL; 796 #endif 797 vnode_zone = uma_zcreate("VNODE", sizeof(struct vnode), ctor, dtor, 798 vnode_init, vnode_fini, UMA_ALIGN_PTR, UMA_ZONE_NOKASAN); 799 uma_zone_set_smr(vnode_zone, vfs_smr); 800 801 /* 802 * Preallocate enough nodes to support one-per buf so that 803 * we can not fail an insert. reassignbuf() callers can not 804 * tolerate the insertion failure. 805 */ 806 buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(), 807 NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR, 808 UMA_ZONE_NOFREE | UMA_ZONE_SMR); 809 buf_trie_smr = uma_zone_get_smr(buf_trie_zone); 810 uma_prealloc(buf_trie_zone, nbuf); 811 812 vnodes_created = counter_u64_alloc(M_WAITOK); 813 direct_recycles_free_count = counter_u64_alloc(M_WAITOK); 814 vnode_skipped_requeues = counter_u64_alloc(M_WAITOK); 815 816 /* 817 * Initialize the filesystem syncer. 818 */ 819 syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE, 820 &syncer_mask); 821 syncer_maxdelay = syncer_mask + 1; 822 mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF); 823 cv_init(&sync_wakeup, "syncer"); 824 825 CPU_FOREACH(cpu) { 826 vd = DPCPU_ID_PTR((cpu), vd); 827 bzero(vd, sizeof(*vd)); 828 mtx_init(&vd->lock, "vdbatch", NULL, MTX_DEF); 829 } 830 } 831 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL); 832 833 /* 834 * Mark a mount point as busy. Used to synchronize access and to delay 835 * unmounting. Eventually, mountlist_mtx is not released on failure. 836 * 837 * vfs_busy() is a custom lock, it can block the caller. 838 * vfs_busy() only sleeps if the unmount is active on the mount point. 839 * For a mountpoint mp, vfs_busy-enforced lock is before lock of any 840 * vnode belonging to mp. 841 * 842 * Lookup uses vfs_busy() to traverse mount points. 843 * root fs var fs 844 * / vnode lock A / vnode lock (/var) D 845 * /var vnode lock B /log vnode lock(/var/log) E 846 * vfs_busy lock C vfs_busy lock F 847 * 848 * Within each file system, the lock order is C->A->B and F->D->E. 849 * 850 * When traversing across mounts, the system follows that lock order: 851 * 852 * C->A->B 853 * | 854 * +->F->D->E 855 * 856 * The lookup() process for namei("/var") illustrates the process: 857 * 1. VOP_LOOKUP() obtains B while A is held 858 * 2. vfs_busy() obtains a shared lock on F while A and B are held 859 * 3. vput() releases lock on B 860 * 4. vput() releases lock on A 861 * 5. VFS_ROOT() obtains lock on D while shared lock on F is held 862 * 6. vfs_unbusy() releases shared lock on F 863 * 7. vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A. 864 * Attempt to lock A (instead of vp_crossmp) while D is held would 865 * violate the global order, causing deadlocks. 866 * 867 * dounmount() locks B while F is drained. Note that for stacked 868 * filesystems, D and B in the example above may be the same lock, 869 * which introdues potential lock order reversal deadlock between 870 * dounmount() and step 5 above. These filesystems may avoid the LOR 871 * by setting VV_CROSSLOCK on the covered vnode so that lock B will 872 * remain held until after step 5. 873 */ 874 int 875 vfs_busy(struct mount *mp, int flags) 876 { 877 struct mount_pcpu *mpcpu; 878 879 MPASS((flags & ~MBF_MASK) == 0); 880 CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags); 881 882 if (vfs_op_thread_enter(mp, mpcpu)) { 883 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 884 MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0); 885 MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0); 886 vfs_mp_count_add_pcpu(mpcpu, ref, 1); 887 vfs_mp_count_add_pcpu(mpcpu, lockref, 1); 888 vfs_op_thread_exit(mp, mpcpu); 889 if (flags & MBF_MNTLSTLOCK) 890 mtx_unlock(&mountlist_mtx); 891 return (0); 892 } 893 894 MNT_ILOCK(mp); 895 vfs_assert_mount_counters(mp); 896 MNT_REF(mp); 897 /* 898 * If mount point is currently being unmounted, sleep until the 899 * mount point fate is decided. If thread doing the unmounting fails, 900 * it will clear MNTK_UNMOUNT flag before waking us up, indicating 901 * that this mount point has survived the unmount attempt and vfs_busy 902 * should retry. Otherwise the unmounter thread will set MNTK_REFEXPIRE 903 * flag in addition to MNTK_UNMOUNT, indicating that mount point is 904 * about to be really destroyed. vfs_busy needs to release its 905 * reference on the mount point in this case and return with ENOENT, 906 * telling the caller the mount it tried to busy is no longer valid. 907 */ 908 while (mp->mnt_kern_flag & MNTK_UNMOUNT) { 909 KASSERT(TAILQ_EMPTY(&mp->mnt_uppers), 910 ("%s: non-empty upper mount list with pending unmount", 911 __func__)); 912 if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) { 913 MNT_REL(mp); 914 MNT_IUNLOCK(mp); 915 CTR1(KTR_VFS, "%s: failed busying before sleeping", 916 __func__); 917 return (ENOENT); 918 } 919 if (flags & MBF_MNTLSTLOCK) 920 mtx_unlock(&mountlist_mtx); 921 mp->mnt_kern_flag |= MNTK_MWAIT; 922 msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0); 923 if (flags & MBF_MNTLSTLOCK) 924 mtx_lock(&mountlist_mtx); 925 MNT_ILOCK(mp); 926 } 927 if (flags & MBF_MNTLSTLOCK) 928 mtx_unlock(&mountlist_mtx); 929 mp->mnt_lockref++; 930 MNT_IUNLOCK(mp); 931 return (0); 932 } 933 934 /* 935 * Free a busy filesystem. 936 */ 937 void 938 vfs_unbusy(struct mount *mp) 939 { 940 struct mount_pcpu *mpcpu; 941 int c; 942 943 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 944 945 if (vfs_op_thread_enter(mp, mpcpu)) { 946 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 947 vfs_mp_count_sub_pcpu(mpcpu, lockref, 1); 948 vfs_mp_count_sub_pcpu(mpcpu, ref, 1); 949 vfs_op_thread_exit(mp, mpcpu); 950 return; 951 } 952 953 MNT_ILOCK(mp); 954 vfs_assert_mount_counters(mp); 955 MNT_REL(mp); 956 c = --mp->mnt_lockref; 957 if (mp->mnt_vfs_ops == 0) { 958 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); 959 MNT_IUNLOCK(mp); 960 return; 961 } 962 if (c < 0) 963 vfs_dump_mount_counters(mp); 964 if (c == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) { 965 MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT); 966 CTR1(KTR_VFS, "%s: waking up waiters", __func__); 967 mp->mnt_kern_flag &= ~MNTK_DRAINING; 968 wakeup(&mp->mnt_lockref); 969 } 970 MNT_IUNLOCK(mp); 971 } 972 973 /* 974 * Lookup a mount point by filesystem identifier. 975 */ 976 struct mount * 977 vfs_getvfs(fsid_t *fsid) 978 { 979 struct mount *mp; 980 981 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 982 mtx_lock(&mountlist_mtx); 983 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 984 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 985 vfs_ref(mp); 986 mtx_unlock(&mountlist_mtx); 987 return (mp); 988 } 989 } 990 mtx_unlock(&mountlist_mtx); 991 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 992 return ((struct mount *) 0); 993 } 994 995 /* 996 * Lookup a mount point by filesystem identifier, busying it before 997 * returning. 998 * 999 * To avoid congestion on mountlist_mtx, implement simple direct-mapped 1000 * cache for popular filesystem identifiers. The cache is lockess, using 1001 * the fact that struct mount's are never freed. In worst case we may 1002 * get pointer to unmounted or even different filesystem, so we have to 1003 * check what we got, and go slow way if so. 1004 */ 1005 struct mount * 1006 vfs_busyfs(fsid_t *fsid) 1007 { 1008 #define FSID_CACHE_SIZE 256 1009 typedef struct mount * volatile vmp_t; 1010 static vmp_t cache[FSID_CACHE_SIZE]; 1011 struct mount *mp; 1012 int error; 1013 uint32_t hash; 1014 1015 CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid); 1016 hash = fsid->val[0] ^ fsid->val[1]; 1017 hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1); 1018 mp = cache[hash]; 1019 if (mp == NULL || fsidcmp(&mp->mnt_stat.f_fsid, fsid) != 0) 1020 goto slow; 1021 if (vfs_busy(mp, 0) != 0) { 1022 cache[hash] = NULL; 1023 goto slow; 1024 } 1025 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) 1026 return (mp); 1027 else 1028 vfs_unbusy(mp); 1029 1030 slow: 1031 mtx_lock(&mountlist_mtx); 1032 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 1033 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) { 1034 error = vfs_busy(mp, MBF_MNTLSTLOCK); 1035 if (error) { 1036 cache[hash] = NULL; 1037 mtx_unlock(&mountlist_mtx); 1038 return (NULL); 1039 } 1040 cache[hash] = mp; 1041 return (mp); 1042 } 1043 } 1044 CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid); 1045 mtx_unlock(&mountlist_mtx); 1046 return ((struct mount *) 0); 1047 } 1048 1049 /* 1050 * Check if a user can access privileged mount options. 1051 */ 1052 int 1053 vfs_suser(struct mount *mp, struct thread *td) 1054 { 1055 int error; 1056 1057 if (jailed(td->td_ucred)) { 1058 /* 1059 * If the jail of the calling thread lacks permission for 1060 * this type of file system, deny immediately. 1061 */ 1062 if (!prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag)) 1063 return (EPERM); 1064 1065 /* 1066 * If the file system was mounted outside the jail of the 1067 * calling thread, deny immediately. 1068 */ 1069 if (prison_check(td->td_ucred, mp->mnt_cred) != 0) 1070 return (EPERM); 1071 } 1072 1073 /* 1074 * If file system supports delegated administration, we don't check 1075 * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified 1076 * by the file system itself. 1077 * If this is not the user that did original mount, we check for 1078 * the PRIV_VFS_MOUNT_OWNER privilege. 1079 */ 1080 if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) && 1081 mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) { 1082 if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0) 1083 return (error); 1084 } 1085 return (0); 1086 } 1087 1088 /* 1089 * Get a new unique fsid. Try to make its val[0] unique, since this value 1090 * will be used to create fake device numbers for stat(). Also try (but 1091 * not so hard) make its val[0] unique mod 2^16, since some emulators only 1092 * support 16-bit device numbers. We end up with unique val[0]'s for the 1093 * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls. 1094 * 1095 * Keep in mind that several mounts may be running in parallel. Starting 1096 * the search one past where the previous search terminated is both a 1097 * micro-optimization and a defense against returning the same fsid to 1098 * different mounts. 1099 */ 1100 void 1101 vfs_getnewfsid(struct mount *mp) 1102 { 1103 static uint16_t mntid_base; 1104 struct mount *nmp; 1105 fsid_t tfsid; 1106 int mtype; 1107 1108 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 1109 mtx_lock(&mntid_mtx); 1110 mtype = mp->mnt_vfc->vfc_typenum; 1111 tfsid.val[1] = mtype; 1112 mtype = (mtype & 0xFF) << 24; 1113 for (;;) { 1114 tfsid.val[0] = makedev(255, 1115 mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF)); 1116 mntid_base++; 1117 if ((nmp = vfs_getvfs(&tfsid)) == NULL) 1118 break; 1119 vfs_rel(nmp); 1120 } 1121 mp->mnt_stat.f_fsid.val[0] = tfsid.val[0]; 1122 mp->mnt_stat.f_fsid.val[1] = tfsid.val[1]; 1123 mtx_unlock(&mntid_mtx); 1124 } 1125 1126 /* 1127 * Knob to control the precision of file timestamps: 1128 * 1129 * 0 = seconds only; nanoseconds zeroed. 1130 * 1 = seconds and nanoseconds, accurate within 1/HZ. 1131 * 2 = seconds and nanoseconds, truncated to microseconds. 1132 * >=3 = seconds and nanoseconds, maximum precision. 1133 */ 1134 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC }; 1135 1136 static int timestamp_precision = TSP_USEC; 1137 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW, 1138 ×tamp_precision, 0, "File timestamp precision (0: seconds, " 1139 "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, " 1140 "3+: sec + ns (max. precision))"); 1141 1142 /* 1143 * Get a current timestamp. 1144 */ 1145 void 1146 vfs_timestamp(struct timespec *tsp) 1147 { 1148 struct timeval tv; 1149 1150 switch (timestamp_precision) { 1151 case TSP_SEC: 1152 tsp->tv_sec = time_second; 1153 tsp->tv_nsec = 0; 1154 break; 1155 case TSP_HZ: 1156 getnanotime(tsp); 1157 break; 1158 case TSP_USEC: 1159 microtime(&tv); 1160 TIMEVAL_TO_TIMESPEC(&tv, tsp); 1161 break; 1162 case TSP_NSEC: 1163 default: 1164 nanotime(tsp); 1165 break; 1166 } 1167 } 1168 1169 /* 1170 * Set vnode attributes to VNOVAL 1171 */ 1172 void 1173 vattr_null(struct vattr *vap) 1174 { 1175 1176 vap->va_type = VNON; 1177 vap->va_size = VNOVAL; 1178 vap->va_bytes = VNOVAL; 1179 vap->va_mode = VNOVAL; 1180 vap->va_nlink = VNOVAL; 1181 vap->va_uid = VNOVAL; 1182 vap->va_gid = VNOVAL; 1183 vap->va_fsid = VNOVAL; 1184 vap->va_fileid = VNOVAL; 1185 vap->va_blocksize = VNOVAL; 1186 vap->va_rdev = VNOVAL; 1187 vap->va_atime.tv_sec = VNOVAL; 1188 vap->va_atime.tv_nsec = VNOVAL; 1189 vap->va_mtime.tv_sec = VNOVAL; 1190 vap->va_mtime.tv_nsec = VNOVAL; 1191 vap->va_ctime.tv_sec = VNOVAL; 1192 vap->va_ctime.tv_nsec = VNOVAL; 1193 vap->va_birthtime.tv_sec = VNOVAL; 1194 vap->va_birthtime.tv_nsec = VNOVAL; 1195 vap->va_flags = VNOVAL; 1196 vap->va_gen = VNOVAL; 1197 vap->va_vaflags = 0; 1198 vap->va_filerev = VNOVAL; 1199 vap->va_bsdflags = 0; 1200 } 1201 1202 /* 1203 * Try to reduce the total number of vnodes. 1204 * 1205 * This routine (and its user) are buggy in at least the following ways: 1206 * - all parameters were picked years ago when RAM sizes were significantly 1207 * smaller 1208 * - it can pick vnodes based on pages used by the vm object, but filesystems 1209 * like ZFS don't use it making the pick broken 1210 * - since ZFS has its own aging policy it gets partially combated by this one 1211 * - a dedicated method should be provided for filesystems to let them decide 1212 * whether the vnode should be recycled 1213 * 1214 * This routine is called when we have too many vnodes. It attempts 1215 * to free <count> vnodes and will potentially free vnodes that still 1216 * have VM backing store (VM backing store is typically the cause 1217 * of a vnode blowout so we want to do this). Therefore, this operation 1218 * is not considered cheap. 1219 * 1220 * A number of conditions may prevent a vnode from being reclaimed. 1221 * the buffer cache may have references on the vnode, a directory 1222 * vnode may still have references due to the namei cache representing 1223 * underlying files, or the vnode may be in active use. It is not 1224 * desirable to reuse such vnodes. These conditions may cause the 1225 * number of vnodes to reach some minimum value regardless of what 1226 * you set kern.maxvnodes to. Do not set kern.maxvnodes too low. 1227 * 1228 * @param reclaim_nc_src Only reclaim directories with outgoing namecache 1229 * entries if this argument is strue 1230 * @param trigger Only reclaim vnodes with fewer than this many resident 1231 * pages. 1232 * @param target How many vnodes to reclaim. 1233 * @return The number of vnodes that were reclaimed. 1234 */ 1235 static int 1236 vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) 1237 { 1238 struct vnode *vp, *mvp; 1239 struct mount *mp; 1240 struct vm_object *object; 1241 u_long done; 1242 bool retried; 1243 1244 mtx_assert(&vnode_list_mtx, MA_OWNED); 1245 1246 retried = false; 1247 done = 0; 1248 1249 mvp = vnode_list_reclaim_marker; 1250 restart: 1251 vp = mvp; 1252 while (done < target) { 1253 vp = TAILQ_NEXT(vp, v_vnodelist); 1254 if (__predict_false(vp == NULL)) 1255 break; 1256 1257 if (__predict_false(vp->v_type == VMARKER)) 1258 continue; 1259 1260 /* 1261 * If it's been deconstructed already, it's still 1262 * referenced, or it exceeds the trigger, skip it. 1263 * Also skip free vnodes. We are trying to make space 1264 * for more free vnodes, not reduce their count. 1265 */ 1266 if (vp->v_usecount > 0 || vp->v_holdcnt == 0 || 1267 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src))) 1268 goto next_iter; 1269 1270 if (vp->v_type == VBAD || vp->v_type == VNON) 1271 goto next_iter; 1272 1273 object = atomic_load_ptr(&vp->v_object); 1274 if (object == NULL || object->resident_page_count > trigger) { 1275 goto next_iter; 1276 } 1277 1278 /* 1279 * Handle races against vnode allocation. Filesystems lock the 1280 * vnode some time after it gets returned from getnewvnode, 1281 * despite type and hold count being manipulated earlier. 1282 * Resorting to checking v_mount restores guarantees present 1283 * before the global list was reworked to contain all vnodes. 1284 */ 1285 if (!VI_TRYLOCK(vp)) 1286 goto next_iter; 1287 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1288 VI_UNLOCK(vp); 1289 goto next_iter; 1290 } 1291 if (vp->v_mount == NULL) { 1292 VI_UNLOCK(vp); 1293 goto next_iter; 1294 } 1295 vholdl(vp); 1296 VI_UNLOCK(vp); 1297 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1298 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1299 mtx_unlock(&vnode_list_mtx); 1300 1301 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 1302 vdrop_recycle(vp); 1303 goto next_iter_unlocked; 1304 } 1305 if (VOP_LOCK(vp, LK_EXCLUSIVE|LK_NOWAIT) != 0) { 1306 vdrop_recycle(vp); 1307 vn_finished_write(mp); 1308 goto next_iter_unlocked; 1309 } 1310 1311 VI_LOCK(vp); 1312 if (vp->v_usecount > 0 || 1313 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) || 1314 (vp->v_object != NULL && vp->v_object->handle == vp && 1315 vp->v_object->resident_page_count > trigger)) { 1316 VOP_UNLOCK(vp); 1317 vdropl_recycle(vp); 1318 vn_finished_write(mp); 1319 goto next_iter_unlocked; 1320 } 1321 recycles_count++; 1322 vgonel(vp); 1323 VOP_UNLOCK(vp); 1324 vdropl_recycle(vp); 1325 vn_finished_write(mp); 1326 done++; 1327 next_iter_unlocked: 1328 maybe_yield(); 1329 mtx_lock(&vnode_list_mtx); 1330 goto restart; 1331 next_iter: 1332 MPASS(vp->v_type != VMARKER); 1333 if (!should_yield()) 1334 continue; 1335 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1336 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1337 mtx_unlock(&vnode_list_mtx); 1338 kern_yield(PRI_USER); 1339 mtx_lock(&vnode_list_mtx); 1340 goto restart; 1341 } 1342 if (done == 0 && !retried) { 1343 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1344 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); 1345 retried = true; 1346 goto restart; 1347 } 1348 return (done); 1349 } 1350 1351 static int max_free_per_call = 10000; 1352 SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_free_per_call, 0, 1353 "limit on vnode free requests per call to the vnlru_free routine (legacy)"); 1354 SYSCTL_INT(_vfs_vnode_vnlru, OID_AUTO, max_free_per_call, CTLFLAG_RW, 1355 &max_free_per_call, 0, 1356 "limit on vnode free requests per call to the vnlru_free routine"); 1357 1358 /* 1359 * Attempt to recycle requested amount of free vnodes. 1360 */ 1361 static int 1362 vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp, bool isvnlru) 1363 { 1364 struct vnode *vp; 1365 struct mount *mp; 1366 int ocount; 1367 bool retried; 1368 1369 mtx_assert(&vnode_list_mtx, MA_OWNED); 1370 if (count > max_free_per_call) 1371 count = max_free_per_call; 1372 if (count == 0) { 1373 mtx_unlock(&vnode_list_mtx); 1374 return (0); 1375 } 1376 ocount = count; 1377 retried = false; 1378 vp = mvp; 1379 for (;;) { 1380 vp = TAILQ_NEXT(vp, v_vnodelist); 1381 if (__predict_false(vp == NULL)) { 1382 /* 1383 * The free vnode marker can be past eligible vnodes: 1384 * 1. if vdbatch_process trylock failed 1385 * 2. if vtryrecycle failed 1386 * 1387 * If so, start the scan from scratch. 1388 */ 1389 if (!retried && vnlru_read_freevnodes() > 0) { 1390 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1391 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); 1392 vp = mvp; 1393 retried = true; 1394 continue; 1395 } 1396 1397 /* 1398 * Give up 1399 */ 1400 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1401 TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist); 1402 mtx_unlock(&vnode_list_mtx); 1403 break; 1404 } 1405 if (__predict_false(vp->v_type == VMARKER)) 1406 continue; 1407 if (vp->v_holdcnt > 0) 1408 continue; 1409 /* 1410 * Don't recycle if our vnode is from different type 1411 * of mount point. Note that mp is type-safe, the 1412 * check does not reach unmapped address even if 1413 * vnode is reclaimed. 1414 */ 1415 if (mnt_op != NULL && (mp = vp->v_mount) != NULL && 1416 mp->mnt_op != mnt_op) { 1417 continue; 1418 } 1419 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { 1420 continue; 1421 } 1422 if (!vhold_recycle_free(vp)) 1423 continue; 1424 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1425 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); 1426 mtx_unlock(&vnode_list_mtx); 1427 /* 1428 * FIXME: ignores the return value, meaning it may be nothing 1429 * got recycled but it claims otherwise to the caller. 1430 * 1431 * Originally the value started being ignored in 2005 with 1432 * 114a1006a8204aa156e1f9ad6476cdff89cada7f . 1433 * 1434 * Respecting the value can run into significant stalls if most 1435 * vnodes belong to one file system and it has writes 1436 * suspended. In presence of many threads and millions of 1437 * vnodes they keep contending on the vnode_list_mtx lock only 1438 * to find vnodes they can't recycle. 1439 * 1440 * The solution would be to pre-check if the vnode is likely to 1441 * be recycle-able, but it needs to happen with the 1442 * vnode_list_mtx lock held. This runs into a problem where 1443 * VOP_GETWRITEMOUNT (currently needed to find out about if 1444 * writes are frozen) can take locks which LOR against it. 1445 * 1446 * Check nullfs for one example (null_getwritemount). 1447 */ 1448 vtryrecycle(vp, isvnlru); 1449 count--; 1450 if (count == 0) { 1451 break; 1452 } 1453 mtx_lock(&vnode_list_mtx); 1454 vp = mvp; 1455 } 1456 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1457 return (ocount - count); 1458 } 1459 1460 /* 1461 * XXX: returns without vnode_list_mtx locked! 1462 */ 1463 static int 1464 vnlru_free_locked_direct(int count) 1465 { 1466 int ret; 1467 1468 mtx_assert(&vnode_list_mtx, MA_OWNED); 1469 ret = vnlru_free_impl(count, NULL, vnode_list_free_marker, false); 1470 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1471 return (ret); 1472 } 1473 1474 static int 1475 vnlru_free_locked_vnlru(int count) 1476 { 1477 int ret; 1478 1479 mtx_assert(&vnode_list_mtx, MA_OWNED); 1480 ret = vnlru_free_impl(count, NULL, vnode_list_free_marker, true); 1481 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1482 return (ret); 1483 } 1484 1485 static int 1486 vnlru_free_vnlru(int count) 1487 { 1488 1489 mtx_lock(&vnode_list_mtx); 1490 return (vnlru_free_locked_vnlru(count)); 1491 } 1492 1493 void 1494 vnlru_free_vfsops(int count, struct vfsops *mnt_op, struct vnode *mvp) 1495 { 1496 1497 MPASS(mnt_op != NULL); 1498 MPASS(mvp != NULL); 1499 VNPASS(mvp->v_type == VMARKER, mvp); 1500 mtx_lock(&vnode_list_mtx); 1501 vnlru_free_impl(count, mnt_op, mvp, true); 1502 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1503 } 1504 1505 struct vnode * 1506 vnlru_alloc_marker(void) 1507 { 1508 struct vnode *mvp; 1509 1510 mvp = vn_alloc_marker(NULL); 1511 mtx_lock(&vnode_list_mtx); 1512 TAILQ_INSERT_BEFORE(vnode_list_free_marker, mvp, v_vnodelist); 1513 mtx_unlock(&vnode_list_mtx); 1514 return (mvp); 1515 } 1516 1517 void 1518 vnlru_free_marker(struct vnode *mvp) 1519 { 1520 mtx_lock(&vnode_list_mtx); 1521 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); 1522 mtx_unlock(&vnode_list_mtx); 1523 vn_free_marker(mvp); 1524 } 1525 1526 static void 1527 vnlru_recalc(void) 1528 { 1529 1530 mtx_assert(&vnode_list_mtx, MA_OWNED); 1531 gapvnodes = imax(desiredvnodes - wantfreevnodes, 100); 1532 vhiwat = gapvnodes / 11; /* 9% -- just under the 10% in vlrureclaim() */ 1533 vlowat = vhiwat / 2; 1534 } 1535 1536 /* 1537 * Attempt to recycle vnodes in a context that is always safe to block. 1538 * Calling vlrurecycle() from the bowels of filesystem code has some 1539 * interesting deadlock problems. 1540 */ 1541 static struct proc *vnlruproc; 1542 static int vnlruproc_sig; 1543 static u_long vnlruproc_kicks; 1544 1545 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, kicks, CTLFLAG_RD, &vnlruproc_kicks, 0, 1546 "Number of times vnlru awakened due to vnode shortage"); 1547 1548 #define VNLRU_COUNT_SLOP 100 1549 1550 /* 1551 * The main freevnodes counter is only updated when a counter local to CPU 1552 * diverges from 0 by more than VNLRU_FREEVNODES_SLOP. CPUs are conditionally 1553 * walked to compute a more accurate total. 1554 * 1555 * Note: the actual value at any given moment can still exceed slop, but it 1556 * should not be by significant margin in practice. 1557 */ 1558 #define VNLRU_FREEVNODES_SLOP 126 1559 1560 static void __noinline 1561 vfs_freevnodes_rollup(int8_t *lfreevnodes) 1562 { 1563 1564 atomic_add_long(&freevnodes, *lfreevnodes); 1565 *lfreevnodes = 0; 1566 critical_exit(); 1567 } 1568 1569 static __inline void 1570 vfs_freevnodes_inc(void) 1571 { 1572 int8_t *lfreevnodes; 1573 1574 critical_enter(); 1575 lfreevnodes = PCPU_PTR(vfs_freevnodes); 1576 (*lfreevnodes)++; 1577 if (__predict_false(*lfreevnodes == VNLRU_FREEVNODES_SLOP)) 1578 vfs_freevnodes_rollup(lfreevnodes); 1579 else 1580 critical_exit(); 1581 } 1582 1583 static __inline void 1584 vfs_freevnodes_dec(void) 1585 { 1586 int8_t *lfreevnodes; 1587 1588 critical_enter(); 1589 lfreevnodes = PCPU_PTR(vfs_freevnodes); 1590 (*lfreevnodes)--; 1591 if (__predict_false(*lfreevnodes == -VNLRU_FREEVNODES_SLOP)) 1592 vfs_freevnodes_rollup(lfreevnodes); 1593 else 1594 critical_exit(); 1595 } 1596 1597 static u_long 1598 vnlru_read_freevnodes(void) 1599 { 1600 long slop, rfreevnodes, rfreevnodes_old; 1601 int cpu; 1602 1603 rfreevnodes = atomic_load_long(&freevnodes); 1604 rfreevnodes_old = atomic_load_long(&freevnodes_old); 1605 1606 if (rfreevnodes > rfreevnodes_old) 1607 slop = rfreevnodes - rfreevnodes_old; 1608 else 1609 slop = rfreevnodes_old - rfreevnodes; 1610 if (slop < VNLRU_FREEVNODES_SLOP) 1611 return (rfreevnodes >= 0 ? rfreevnodes : 0); 1612 CPU_FOREACH(cpu) { 1613 rfreevnodes += cpuid_to_pcpu[cpu]->pc_vfs_freevnodes; 1614 } 1615 atomic_store_long(&freevnodes_old, rfreevnodes); 1616 return (freevnodes_old >= 0 ? freevnodes_old : 0); 1617 } 1618 1619 static bool 1620 vnlru_under(u_long rnumvnodes, u_long limit) 1621 { 1622 u_long rfreevnodes, space; 1623 1624 if (__predict_false(rnumvnodes > desiredvnodes)) 1625 return (true); 1626 1627 space = desiredvnodes - rnumvnodes; 1628 if (space < limit) { 1629 rfreevnodes = vnlru_read_freevnodes(); 1630 if (rfreevnodes > wantfreevnodes) 1631 space += rfreevnodes - wantfreevnodes; 1632 } 1633 return (space < limit); 1634 } 1635 1636 static void 1637 vnlru_kick_locked(void) 1638 { 1639 1640 mtx_assert(&vnode_list_mtx, MA_OWNED); 1641 if (vnlruproc_sig == 0) { 1642 vnlruproc_sig = 1; 1643 vnlruproc_kicks++; 1644 wakeup(vnlruproc); 1645 } 1646 } 1647 1648 static void 1649 vnlru_kick_cond(void) 1650 { 1651 1652 if (vnlru_read_freevnodes() > wantfreevnodes) 1653 return; 1654 1655 if (vnlruproc_sig) 1656 return; 1657 mtx_lock(&vnode_list_mtx); 1658 vnlru_kick_locked(); 1659 mtx_unlock(&vnode_list_mtx); 1660 } 1661 1662 static void 1663 vnlru_proc_sleep(void) 1664 { 1665 1666 if (vnlruproc_sig) { 1667 vnlruproc_sig = 0; 1668 wakeup(&vnlruproc_sig); 1669 } 1670 msleep(vnlruproc, &vnode_list_mtx, PVFS|PDROP, "vlruwt", hz); 1671 } 1672 1673 /* 1674 * A lighter version of the machinery below. 1675 * 1676 * Tries to reach goals only by recycling free vnodes and does not invoke 1677 * uma_reclaim(UMA_RECLAIM_DRAIN). 1678 * 1679 * This works around pathological behavior in vnlru in presence of tons of free 1680 * vnodes, but without having to rewrite the machinery at this time. Said 1681 * behavior boils down to continuously trying to reclaim all kinds of vnodes 1682 * (cycling through all levels of "force") when the count is transiently above 1683 * limit. This happens a lot when all vnodes are used up and vn_alloc 1684 * speculatively increments the counter. 1685 * 1686 * Sample testcase: vnode limit 8388608, 20 separate directory trees each with 1687 * 1 million files in total and 20 find(1) processes stating them in parallel 1688 * (one per each tree). 1689 * 1690 * On a kernel with only stock machinery this needs anywhere between 60 and 120 1691 * seconds to execute (time varies *wildly* between runs). With the workaround 1692 * it consistently stays around 20 seconds [it got further down with later 1693 * changes]. 1694 * 1695 * That is to say the entire thing needs a fundamental redesign (most notably 1696 * to accommodate faster recycling), the above only tries to get it ouf the way. 1697 * 1698 * Return values are: 1699 * -1 -- fallback to regular vnlru loop 1700 * 0 -- do nothing, go to sleep 1701 * >0 -- recycle this many vnodes 1702 */ 1703 static long 1704 vnlru_proc_light_pick(void) 1705 { 1706 u_long rnumvnodes, rfreevnodes; 1707 1708 if (vstir || vnlruproc_sig == 1) 1709 return (-1); 1710 1711 rnumvnodes = atomic_load_long(&numvnodes); 1712 rfreevnodes = vnlru_read_freevnodes(); 1713 1714 /* 1715 * vnode limit might have changed and now we may be at a significant 1716 * excess. Bail if we can't sort it out with free vnodes. 1717 * 1718 * Due to atomic updates the count can legitimately go above 1719 * the limit for a short period, don't bother doing anything in 1720 * that case. 1721 */ 1722 if (rnumvnodes > desiredvnodes + VNLRU_COUNT_SLOP + 10) { 1723 if (rnumvnodes - rfreevnodes >= desiredvnodes || 1724 rfreevnodes <= wantfreevnodes) { 1725 return (-1); 1726 } 1727 1728 return (rnumvnodes - desiredvnodes); 1729 } 1730 1731 /* 1732 * Don't try to reach wantfreevnodes target if there are too few vnodes 1733 * to begin with. 1734 */ 1735 if (rnumvnodes < wantfreevnodes) { 1736 return (0); 1737 } 1738 1739 if (rfreevnodes < wantfreevnodes) { 1740 return (-1); 1741 } 1742 1743 return (0); 1744 } 1745 1746 static bool 1747 vnlru_proc_light(void) 1748 { 1749 long freecount; 1750 1751 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1752 1753 freecount = vnlru_proc_light_pick(); 1754 if (freecount == -1) 1755 return (false); 1756 1757 if (freecount != 0) { 1758 vnlru_free_vnlru(freecount); 1759 } 1760 1761 mtx_lock(&vnode_list_mtx); 1762 vnlru_proc_sleep(); 1763 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 1764 return (true); 1765 } 1766 1767 static u_long uma_reclaim_calls; 1768 SYSCTL_ULONG(_vfs_vnode_vnlru, OID_AUTO, uma_reclaim_calls, CTLFLAG_RD | CTLFLAG_STATS, 1769 &uma_reclaim_calls, 0, "Number of calls to uma_reclaim"); 1770 1771 static void 1772 vnlru_proc(void) 1773 { 1774 u_long rnumvnodes, rfreevnodes, target; 1775 unsigned long onumvnodes; 1776 int done, force, trigger, usevnodes; 1777 bool reclaim_nc_src, want_reread; 1778 1779 EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, vnlruproc, 1780 SHUTDOWN_PRI_FIRST); 1781 1782 force = 0; 1783 want_reread = false; 1784 for (;;) { 1785 kproc_suspend_check(vnlruproc); 1786 1787 if (force == 0 && vnlru_proc_light()) 1788 continue; 1789 1790 mtx_lock(&vnode_list_mtx); 1791 rnumvnodes = atomic_load_long(&numvnodes); 1792 1793 if (want_reread) { 1794 force = vnlru_under(numvnodes, vhiwat) ? 1 : 0; 1795 want_reread = false; 1796 } 1797 1798 /* 1799 * If numvnodes is too large (due to desiredvnodes being 1800 * adjusted using its sysctl, or emergency growth), first 1801 * try to reduce it by discarding free vnodes. 1802 */ 1803 if (rnumvnodes > desiredvnodes + 10) { 1804 vnlru_free_locked_vnlru(rnumvnodes - desiredvnodes); 1805 mtx_lock(&vnode_list_mtx); 1806 rnumvnodes = atomic_load_long(&numvnodes); 1807 } 1808 /* 1809 * Sleep if the vnode cache is in a good state. This is 1810 * when it is not over-full and has space for about a 4% 1811 * or 9% expansion (by growing its size or inexcessively 1812 * reducing free vnode count). Otherwise, try to reclaim 1813 * space for a 10% expansion. 1814 */ 1815 if (vstir && force == 0) { 1816 force = 1; 1817 vstir = false; 1818 } 1819 if (force == 0 && !vnlru_under(rnumvnodes, vlowat)) { 1820 vnlru_proc_sleep(); 1821 continue; 1822 } 1823 rfreevnodes = vnlru_read_freevnodes(); 1824 1825 onumvnodes = rnumvnodes; 1826 /* 1827 * Calculate parameters for recycling. These are the same 1828 * throughout the loop to give some semblance of fairness. 1829 * The trigger point is to avoid recycling vnodes with lots 1830 * of resident pages. We aren't trying to free memory; we 1831 * are trying to recycle or at least free vnodes. 1832 */ 1833 if (rnumvnodes <= desiredvnodes) 1834 usevnodes = rnumvnodes - rfreevnodes; 1835 else 1836 usevnodes = rnumvnodes; 1837 if (usevnodes <= 0) 1838 usevnodes = 1; 1839 /* 1840 * The trigger value is chosen to give a conservatively 1841 * large value to ensure that it alone doesn't prevent 1842 * making progress. The value can easily be so large that 1843 * it is effectively infinite in some congested and 1844 * misconfigured cases, and this is necessary. Normally 1845 * it is about 8 to 100 (pages), which is quite large. 1846 */ 1847 trigger = vm_cnt.v_page_count * 2 / usevnodes; 1848 if (force < 2) 1849 trigger = vsmalltrigger; 1850 reclaim_nc_src = force >= 3; 1851 target = rnumvnodes * (int64_t)gapvnodes / imax(desiredvnodes, 1); 1852 target = target / 10 + 1; 1853 done = vlrureclaim(reclaim_nc_src, trigger, target); 1854 mtx_unlock(&vnode_list_mtx); 1855 /* 1856 * Total number of vnodes can transiently go slightly above the 1857 * limit (see vn_alloc_hard), no need to call uma_reclaim if 1858 * this happens. 1859 */ 1860 if (onumvnodes + VNLRU_COUNT_SLOP + 1000 > desiredvnodes && 1861 numvnodes <= desiredvnodes) { 1862 uma_reclaim_calls++; 1863 uma_reclaim(UMA_RECLAIM_DRAIN); 1864 } 1865 if (done == 0) { 1866 if (force == 0 || force == 1) { 1867 force = 2; 1868 continue; 1869 } 1870 if (force == 2) { 1871 force = 3; 1872 continue; 1873 } 1874 want_reread = true; 1875 force = 0; 1876 vnlru_nowhere++; 1877 tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3); 1878 } else { 1879 want_reread = true; 1880 kern_yield(PRI_USER); 1881 } 1882 } 1883 } 1884 1885 static struct kproc_desc vnlru_kp = { 1886 "vnlru", 1887 vnlru_proc, 1888 &vnlruproc 1889 }; 1890 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, 1891 &vnlru_kp); 1892 1893 /* 1894 * Routines having to do with the management of the vnode table. 1895 */ 1896 1897 /* 1898 * Try to recycle a freed vnode. 1899 */ 1900 static int 1901 vtryrecycle(struct vnode *vp, bool isvnlru) 1902 { 1903 struct mount *vnmp; 1904 1905 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 1906 VNPASS(vp->v_holdcnt > 0, vp); 1907 /* 1908 * This vnode may found and locked via some other list, if so we 1909 * can't recycle it yet. 1910 */ 1911 if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { 1912 CTR2(KTR_VFS, 1913 "%s: impossible to recycle, vp %p lock is already held", 1914 __func__, vp); 1915 vdrop_recycle(vp); 1916 return (EWOULDBLOCK); 1917 } 1918 /* 1919 * Don't recycle if its filesystem is being suspended. 1920 */ 1921 if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) { 1922 VOP_UNLOCK(vp); 1923 CTR2(KTR_VFS, 1924 "%s: impossible to recycle, cannot start the write for %p", 1925 __func__, vp); 1926 vdrop_recycle(vp); 1927 return (EBUSY); 1928 } 1929 /* 1930 * If we got this far, we need to acquire the interlock and see if 1931 * anyone picked up this vnode from another list. If not, we will 1932 * mark it with DOOMED via vgonel() so that anyone who does find it 1933 * will skip over it. 1934 */ 1935 VI_LOCK(vp); 1936 if (vp->v_usecount) { 1937 VOP_UNLOCK(vp); 1938 vdropl_recycle(vp); 1939 vn_finished_write(vnmp); 1940 CTR2(KTR_VFS, 1941 "%s: impossible to recycle, %p is already referenced", 1942 __func__, vp); 1943 return (EBUSY); 1944 } 1945 if (!VN_IS_DOOMED(vp)) { 1946 if (isvnlru) 1947 recycles_free_count++; 1948 else 1949 counter_u64_add(direct_recycles_free_count, 1); 1950 vgonel(vp); 1951 } 1952 VOP_UNLOCK(vp); 1953 vdropl_recycle(vp); 1954 vn_finished_write(vnmp); 1955 return (0); 1956 } 1957 1958 /* 1959 * Allocate a new vnode. 1960 * 1961 * The operation never returns an error. Returning an error was disabled 1962 * in r145385 (dated 2005) with the following comment: 1963 * 1964 * XXX Not all VFS_VGET/ffs_vget callers check returns. 1965 * 1966 * Given the age of this commit (almost 15 years at the time of writing this 1967 * comment) restoring the ability to fail requires a significant audit of 1968 * all codepaths. 1969 * 1970 * The routine can try to free a vnode or stall for up to 1 second waiting for 1971 * vnlru to clear things up, but ultimately always performs a M_WAITOK allocation. 1972 */ 1973 static u_long vn_alloc_cyclecount; 1974 static u_long vn_alloc_sleeps; 1975 1976 SYSCTL_ULONG(_vfs_vnode_stats, OID_AUTO, alloc_sleeps, CTLFLAG_RD, &vn_alloc_sleeps, 0, 1977 "Number of times vnode allocation blocked waiting on vnlru"); 1978 1979 static struct vnode * __noinline 1980 vn_alloc_hard(struct mount *mp, u_long rnumvnodes, bool bumped) 1981 { 1982 u_long rfreevnodes; 1983 1984 if (bumped) { 1985 if (rnumvnodes > desiredvnodes + VNLRU_COUNT_SLOP) { 1986 atomic_subtract_long(&numvnodes, 1); 1987 bumped = false; 1988 } 1989 } 1990 1991 mtx_lock(&vnode_list_mtx); 1992 1993 /* 1994 * Reload 'numvnodes', as since we acquired the lock, it may have 1995 * changed significantly if we waited, and 'rnumvnodes' above was only 1996 * actually passed if 'bumped' is true (else it is 0). 1997 */ 1998 rnumvnodes = atomic_load_long(&numvnodes); 1999 if (rnumvnodes + !bumped < desiredvnodes) { 2000 vn_alloc_cyclecount = 0; 2001 mtx_unlock(&vnode_list_mtx); 2002 goto alloc; 2003 } 2004 2005 rfreevnodes = vnlru_read_freevnodes(); 2006 if (vn_alloc_cyclecount++ >= rfreevnodes) { 2007 vn_alloc_cyclecount = 0; 2008 vstir = true; 2009 } 2010 2011 /* 2012 * Grow the vnode cache if it will not be above its target max after 2013 * growing. Otherwise, if there is at least one free vnode, try to 2014 * reclaim 1 item from it before growing the cache (possibly above its 2015 * target max if the reclamation failed or is delayed). 2016 */ 2017 if (vnlru_free_locked_direct(1) > 0) 2018 goto alloc; 2019 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 2020 if (mp == NULL || (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 2021 /* 2022 * Wait for space for a new vnode. 2023 */ 2024 if (bumped) { 2025 atomic_subtract_long(&numvnodes, 1); 2026 bumped = false; 2027 } 2028 mtx_lock(&vnode_list_mtx); 2029 vnlru_kick_locked(); 2030 vn_alloc_sleeps++; 2031 msleep(&vnlruproc_sig, &vnode_list_mtx, PVFS, "vlruwk", hz); 2032 if (atomic_load_long(&numvnodes) + 1 > desiredvnodes && 2033 vnlru_read_freevnodes() > 1) 2034 vnlru_free_locked_direct(1); 2035 else 2036 mtx_unlock(&vnode_list_mtx); 2037 } 2038 alloc: 2039 mtx_assert(&vnode_list_mtx, MA_NOTOWNED); 2040 if (!bumped) 2041 atomic_add_long(&numvnodes, 1); 2042 vnlru_kick_cond(); 2043 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 2044 } 2045 2046 static struct vnode * 2047 vn_alloc(struct mount *mp) 2048 { 2049 u_long rnumvnodes; 2050 2051 if (__predict_false(vn_alloc_cyclecount != 0)) 2052 return (vn_alloc_hard(mp, 0, false)); 2053 rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; 2054 if (__predict_false(vnlru_under(rnumvnodes, vlowat))) { 2055 return (vn_alloc_hard(mp, rnumvnodes, true)); 2056 } 2057 2058 return (uma_zalloc_smr(vnode_zone, M_WAITOK)); 2059 } 2060 2061 static void 2062 vn_free(struct vnode *vp) 2063 { 2064 2065 atomic_subtract_long(&numvnodes, 1); 2066 uma_zfree_smr(vnode_zone, vp); 2067 } 2068 2069 /* 2070 * Allocate a new vnode. 2071 */ 2072 int 2073 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops, 2074 struct vnode **vpp) 2075 { 2076 struct vnode *vp; 2077 struct thread *td; 2078 struct lock_object *lo; 2079 2080 CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag); 2081 2082 KASSERT(vops->registered, 2083 ("%s: not registered vector op %p\n", __func__, vops)); 2084 cache_validate_vop_vector(mp, vops); 2085 2086 td = curthread; 2087 if (td->td_vp_reserved != NULL) { 2088 vp = td->td_vp_reserved; 2089 td->td_vp_reserved = NULL; 2090 } else { 2091 vp = vn_alloc(mp); 2092 } 2093 counter_u64_add(vnodes_created, 1); 2094 2095 vn_set_state(vp, VSTATE_UNINITIALIZED); 2096 2097 /* 2098 * Locks are given the generic name "vnode" when created. 2099 * Follow the historic practice of using the filesystem 2100 * name when they allocated, e.g., "zfs", "ufs", "nfs, etc. 2101 * 2102 * Locks live in a witness group keyed on their name. Thus, 2103 * when a lock is renamed, it must also move from the witness 2104 * group of its old name to the witness group of its new name. 2105 * 2106 * The change only needs to be made when the vnode moves 2107 * from one filesystem type to another. We ensure that each 2108 * filesystem use a single static name pointer for its tag so 2109 * that we can compare pointers rather than doing a strcmp(). 2110 */ 2111 lo = &vp->v_vnlock->lock_object; 2112 #ifdef WITNESS 2113 if (lo->lo_name != tag) { 2114 #endif 2115 lo->lo_name = tag; 2116 #ifdef WITNESS 2117 WITNESS_DESTROY(lo); 2118 WITNESS_INIT(lo, tag); 2119 } 2120 #endif 2121 /* 2122 * By default, don't allow shared locks unless filesystems opt-in. 2123 */ 2124 vp->v_vnlock->lock_object.lo_flags |= LK_NOSHARE; 2125 /* 2126 * Finalize various vnode identity bits. 2127 */ 2128 KASSERT(vp->v_object == NULL, ("stale v_object %p", vp)); 2129 KASSERT(vp->v_lockf == NULL, ("stale v_lockf %p", vp)); 2130 KASSERT(vp->v_pollinfo == NULL, ("stale v_pollinfo %p", vp)); 2131 vp->v_type = VNON; 2132 vp->v_op = vops; 2133 vp->v_irflag = 0; 2134 v_init_counters(vp); 2135 vn_seqc_init(vp); 2136 vp->v_bufobj.bo_ops = &buf_ops_bio; 2137 #ifdef DIAGNOSTIC 2138 if (mp == NULL && vops != &dead_vnodeops) 2139 printf("NULL mp in getnewvnode(9), tag %s\n", tag); 2140 #endif 2141 #ifdef MAC 2142 mac_vnode_init(vp); 2143 if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0) 2144 mac_vnode_associate_singlelabel(mp, vp); 2145 #endif 2146 if (mp != NULL) { 2147 vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize; 2148 } 2149 2150 /* 2151 * For the filesystems which do not use vfs_hash_insert(), 2152 * still initialize v_hash to have vfs_hash_index() useful. 2153 * E.g., nullfs uses vfs_hash_index() on the lower vnode for 2154 * its own hashing. 2155 */ 2156 vp->v_hash = (uintptr_t)vp >> vnsz2log; 2157 2158 *vpp = vp; 2159 return (0); 2160 } 2161 2162 void 2163 getnewvnode_reserve(void) 2164 { 2165 struct thread *td; 2166 2167 td = curthread; 2168 MPASS(td->td_vp_reserved == NULL); 2169 td->td_vp_reserved = vn_alloc(NULL); 2170 } 2171 2172 void 2173 getnewvnode_drop_reserve(void) 2174 { 2175 struct thread *td; 2176 2177 td = curthread; 2178 if (td->td_vp_reserved != NULL) { 2179 vn_free(td->td_vp_reserved); 2180 td->td_vp_reserved = NULL; 2181 } 2182 } 2183 2184 static void __noinline 2185 freevnode(struct vnode *vp) 2186 { 2187 struct bufobj *bo; 2188 2189 /* 2190 * The vnode has been marked for destruction, so free it. 2191 * 2192 * The vnode will be returned to the zone where it will 2193 * normally remain until it is needed for another vnode. We 2194 * need to cleanup (or verify that the cleanup has already 2195 * been done) any residual data left from its current use 2196 * so as not to contaminate the freshly allocated vnode. 2197 */ 2198 CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); 2199 /* 2200 * Paired with vgone. 2201 */ 2202 vn_seqc_write_end_free(vp); 2203 2204 bo = &vp->v_bufobj; 2205 VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); 2206 VNPASS(vp->v_holdcnt == VHOLD_NO_SMR, vp); 2207 VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); 2208 VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); 2209 VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); 2210 VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); 2211 VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp, 2212 ("clean blk trie not empty")); 2213 VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0")); 2214 VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp, 2215 ("dirty blk trie not empty")); 2216 VNASSERT((vp->v_iflag & (VI_DOINGINACT | VI_OWEINACT)) == 0, vp, 2217 ("Leaked inactivation")); 2218 VI_UNLOCK(vp); 2219 cache_assert_no_entries(vp); 2220 2221 #ifdef MAC 2222 mac_vnode_destroy(vp); 2223 #endif 2224 if (vp->v_pollinfo != NULL) { 2225 /* 2226 * Use LK_NOWAIT to shut up witness about the lock. We may get 2227 * here while having another vnode locked when trying to 2228 * satisfy a lookup and needing to recycle. 2229 */ 2230 VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT); 2231 destroy_vpollinfo(vp->v_pollinfo); 2232 VOP_UNLOCK(vp); 2233 vp->v_pollinfo = NULL; 2234 } 2235 vp->v_mountedhere = NULL; 2236 vp->v_unpcb = NULL; 2237 vp->v_rdev = NULL; 2238 vp->v_fifoinfo = NULL; 2239 vp->v_iflag = 0; 2240 vp->v_vflag = 0; 2241 bo->bo_flag = 0; 2242 vn_free(vp); 2243 } 2244 2245 /* 2246 * Delete from old mount point vnode list, if on one. 2247 */ 2248 static void 2249 delmntque(struct vnode *vp) 2250 { 2251 struct mount *mp; 2252 2253 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 2254 2255 mp = vp->v_mount; 2256 MNT_ILOCK(mp); 2257 VI_LOCK(vp); 2258 vp->v_mount = NULL; 2259 VNASSERT(mp->mnt_nvnodelistsize > 0, vp, 2260 ("bad mount point vnode list size")); 2261 TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 2262 mp->mnt_nvnodelistsize--; 2263 MNT_REL(mp); 2264 MNT_IUNLOCK(mp); 2265 /* 2266 * The caller expects the interlock to be still held. 2267 */ 2268 ASSERT_VI_LOCKED(vp, __func__); 2269 } 2270 2271 static int 2272 insmntque1_int(struct vnode *vp, struct mount *mp, bool dtr) 2273 { 2274 2275 KASSERT(vp->v_mount == NULL, 2276 ("insmntque: vnode already on per mount vnode list")); 2277 VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)")); 2278 if ((mp->mnt_kern_flag & MNTK_UNLOCKED_INSMNTQUE) == 0) { 2279 ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp"); 2280 } else { 2281 KASSERT(!dtr, 2282 ("%s: can't have MNTK_UNLOCKED_INSMNTQUE and cleanup", 2283 __func__)); 2284 } 2285 2286 /* 2287 * We acquire the vnode interlock early to ensure that the 2288 * vnode cannot be recycled by another process releasing a 2289 * holdcnt on it before we get it on both the vnode list 2290 * and the active vnode list. The mount mutex protects only 2291 * manipulation of the vnode list and the vnode freelist 2292 * mutex protects only manipulation of the active vnode list. 2293 * Hence the need to hold the vnode interlock throughout. 2294 */ 2295 MNT_ILOCK(mp); 2296 VI_LOCK(vp); 2297 if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 && 2298 ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 || 2299 mp->mnt_nvnodelistsize == 0)) && 2300 (vp->v_vflag & VV_FORCEINSMQ) == 0) { 2301 VI_UNLOCK(vp); 2302 MNT_IUNLOCK(mp); 2303 if (dtr) { 2304 vp->v_data = NULL; 2305 vp->v_op = &dead_vnodeops; 2306 vgone(vp); 2307 vput(vp); 2308 } 2309 return (EBUSY); 2310 } 2311 vp->v_mount = mp; 2312 MNT_REF(mp); 2313 TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes); 2314 VNASSERT(mp->mnt_nvnodelistsize >= 0, vp, 2315 ("neg mount point vnode list size")); 2316 mp->mnt_nvnodelistsize++; 2317 VI_UNLOCK(vp); 2318 MNT_IUNLOCK(mp); 2319 return (0); 2320 } 2321 2322 /* 2323 * Insert into list of vnodes for the new mount point, if available. 2324 * insmntque() reclaims the vnode on insertion failure, insmntque1() 2325 * leaves handling of the vnode to the caller. 2326 */ 2327 int 2328 insmntque(struct vnode *vp, struct mount *mp) 2329 { 2330 return (insmntque1_int(vp, mp, true)); 2331 } 2332 2333 int 2334 insmntque1(struct vnode *vp, struct mount *mp) 2335 { 2336 return (insmntque1_int(vp, mp, false)); 2337 } 2338 2339 /* 2340 * Flush out and invalidate all buffers associated with a bufobj 2341 * Called with the underlying object locked. 2342 */ 2343 int 2344 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo) 2345 { 2346 int error; 2347 2348 BO_LOCK(bo); 2349 if (flags & V_SAVE) { 2350 error = bufobj_wwait(bo, slpflag, slptimeo); 2351 if (error) { 2352 BO_UNLOCK(bo); 2353 return (error); 2354 } 2355 if (bo->bo_dirty.bv_cnt > 0) { 2356 BO_UNLOCK(bo); 2357 do { 2358 error = BO_SYNC(bo, MNT_WAIT); 2359 } while (error == ERELOOKUP); 2360 if (error != 0) 2361 return (error); 2362 BO_LOCK(bo); 2363 if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0) { 2364 BO_UNLOCK(bo); 2365 return (EBUSY); 2366 } 2367 } 2368 } 2369 /* 2370 * If you alter this loop please notice that interlock is dropped and 2371 * reacquired in flushbuflist. Special care is needed to ensure that 2372 * no race conditions occur from this. 2373 */ 2374 do { 2375 error = flushbuflist(&bo->bo_clean, 2376 flags, bo, slpflag, slptimeo); 2377 if (error == 0 && !(flags & V_CLEANONLY)) 2378 error = flushbuflist(&bo->bo_dirty, 2379 flags, bo, slpflag, slptimeo); 2380 if (error != 0 && error != EAGAIN) { 2381 BO_UNLOCK(bo); 2382 return (error); 2383 } 2384 } while (error != 0); 2385 2386 /* 2387 * Wait for I/O to complete. XXX needs cleaning up. The vnode can 2388 * have write I/O in-progress but if there is a VM object then the 2389 * VM object can also have read-I/O in-progress. 2390 */ 2391 do { 2392 bufobj_wwait(bo, 0, 0); 2393 if ((flags & V_VMIO) == 0 && bo->bo_object != NULL) { 2394 BO_UNLOCK(bo); 2395 vm_object_pip_wait_unlocked(bo->bo_object, "bovlbx"); 2396 BO_LOCK(bo); 2397 } 2398 } while (bo->bo_numoutput > 0); 2399 BO_UNLOCK(bo); 2400 2401 /* 2402 * Destroy the copy in the VM cache, too. 2403 */ 2404 if (bo->bo_object != NULL && 2405 (flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0) { 2406 VM_OBJECT_WLOCK(bo->bo_object); 2407 vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ? 2408 OBJPR_CLEANONLY : 0); 2409 VM_OBJECT_WUNLOCK(bo->bo_object); 2410 } 2411 2412 #ifdef INVARIANTS 2413 BO_LOCK(bo); 2414 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO | 2415 V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 || 2416 bo->bo_clean.bv_cnt > 0)) 2417 panic("vinvalbuf: flush failed"); 2418 if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0 && 2419 bo->bo_dirty.bv_cnt > 0) 2420 panic("vinvalbuf: flush dirty failed"); 2421 BO_UNLOCK(bo); 2422 #endif 2423 return (0); 2424 } 2425 2426 /* 2427 * Flush out and invalidate all buffers associated with a vnode. 2428 * Called with the underlying object locked. 2429 */ 2430 int 2431 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo) 2432 { 2433 2434 CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags); 2435 ASSERT_VOP_LOCKED(vp, "vinvalbuf"); 2436 if (vp->v_object != NULL && vp->v_object->handle != vp) 2437 return (0); 2438 return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo)); 2439 } 2440 2441 /* 2442 * Flush out buffers on the specified list. 2443 * 2444 */ 2445 static int 2446 flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, 2447 int slptimeo) 2448 { 2449 struct buf *bp, *nbp; 2450 int retval, error; 2451 daddr_t lblkno; 2452 b_xflags_t xflags; 2453 2454 ASSERT_BO_WLOCKED(bo); 2455 2456 retval = 0; 2457 TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) { 2458 /* 2459 * If we are flushing both V_NORMAL and V_ALT buffers then 2460 * do not skip any buffers. If we are flushing only V_NORMAL 2461 * buffers then skip buffers marked as BX_ALTDATA. If we are 2462 * flushing only V_ALT buffers then skip buffers not marked 2463 * as BX_ALTDATA. 2464 */ 2465 if (((flags & (V_NORMAL | V_ALT)) != (V_NORMAL | V_ALT)) && 2466 (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) || 2467 ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) { 2468 continue; 2469 } 2470 if (nbp != NULL) { 2471 lblkno = nbp->b_lblkno; 2472 xflags = nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN); 2473 } 2474 retval = EAGAIN; 2475 error = BUF_TIMELOCK(bp, 2476 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo), 2477 "flushbuf", slpflag, slptimeo); 2478 if (error) { 2479 BO_LOCK(bo); 2480 return (error != ENOLCK ? error : EAGAIN); 2481 } 2482 KASSERT(bp->b_bufobj == bo, 2483 ("bp %p wrong b_bufobj %p should be %p", 2484 bp, bp->b_bufobj, bo)); 2485 /* 2486 * XXX Since there are no node locks for NFS, I 2487 * believe there is a slight chance that a delayed 2488 * write will occur while sleeping just above, so 2489 * check for it. 2490 */ 2491 if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) && 2492 (flags & V_SAVE)) { 2493 bremfree(bp); 2494 bp->b_flags |= B_ASYNC; 2495 bwrite(bp); 2496 BO_LOCK(bo); 2497 return (EAGAIN); /* XXX: why not loop ? */ 2498 } 2499 bremfree(bp); 2500 bp->b_flags |= (B_INVAL | B_RELBUF); 2501 bp->b_flags &= ~B_ASYNC; 2502 brelse(bp); 2503 BO_LOCK(bo); 2504 if (nbp == NULL) 2505 break; 2506 nbp = gbincore(bo, lblkno); 2507 if (nbp == NULL || (nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) 2508 != xflags) 2509 break; /* nbp invalid */ 2510 } 2511 return (retval); 2512 } 2513 2514 int 2515 bnoreuselist(struct bufv *bufv, struct bufobj *bo, daddr_t startn, daddr_t endn) 2516 { 2517 struct buf *bp; 2518 int error; 2519 daddr_t lblkno; 2520 2521 ASSERT_BO_LOCKED(bo); 2522 2523 for (lblkno = startn;;) { 2524 again: 2525 bp = buf_lookup_ge(bufv, lblkno); 2526 if (bp == NULL || bp->b_lblkno >= endn) 2527 break; 2528 error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 2529 LK_INTERLOCK, BO_LOCKPTR(bo), "brlsfl", 0, 0); 2530 if (error != 0) { 2531 BO_RLOCK(bo); 2532 if (error == ENOLCK) 2533 goto again; 2534 return (error); 2535 } 2536 KASSERT(bp->b_bufobj == bo, 2537 ("bp %p wrong b_bufobj %p should be %p", 2538 bp, bp->b_bufobj, bo)); 2539 lblkno = bp->b_lblkno + 1; 2540 if ((bp->b_flags & B_MANAGED) == 0) 2541 bremfree(bp); 2542 bp->b_flags |= B_RELBUF; 2543 /* 2544 * In the VMIO case, use the B_NOREUSE flag to hint that the 2545 * pages backing each buffer in the range are unlikely to be 2546 * reused. Dirty buffers will have the hint applied once 2547 * they've been written. 2548 */ 2549 if ((bp->b_flags & B_VMIO) != 0) 2550 bp->b_flags |= B_NOREUSE; 2551 brelse(bp); 2552 BO_RLOCK(bo); 2553 } 2554 return (0); 2555 } 2556 2557 /* 2558 * Truncate a file's buffer and pages to a specified length. This 2559 * is in lieu of the old vinvalbuf mechanism, which performed unneeded 2560 * sync activity. 2561 */ 2562 int 2563 vtruncbuf(struct vnode *vp, off_t length, int blksize) 2564 { 2565 struct buf *bp, *nbp; 2566 struct bufobj *bo; 2567 daddr_t startlbn; 2568 2569 CTR4(KTR_VFS, "%s: vp %p with block %d:%ju", __func__, 2570 vp, blksize, (uintmax_t)length); 2571 2572 /* 2573 * Round up to the *next* lbn. 2574 */ 2575 startlbn = howmany(length, blksize); 2576 2577 ASSERT_VOP_LOCKED(vp, "vtruncbuf"); 2578 2579 bo = &vp->v_bufobj; 2580 restart_unlocked: 2581 BO_LOCK(bo); 2582 2583 while (v_inval_buf_range_locked(vp, bo, startlbn, INT64_MAX) == EAGAIN) 2584 ; 2585 2586 if (length > 0) { 2587 /* 2588 * Write out vnode metadata, e.g. indirect blocks. 2589 */ 2590 restartsync: 2591 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 2592 if (bp->b_lblkno >= 0) 2593 continue; 2594 /* 2595 * Since we hold the vnode lock this should only 2596 * fail if we're racing with the buf daemon. 2597 */ 2598 if (BUF_LOCK(bp, 2599 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2600 BO_LOCKPTR(bo)) == ENOLCK) 2601 goto restart_unlocked; 2602 2603 VNASSERT((bp->b_flags & B_DELWRI), vp, 2604 ("buf(%p) on dirty queue without DELWRI", bp)); 2605 2606 bremfree(bp); 2607 bawrite(bp); 2608 BO_LOCK(bo); 2609 goto restartsync; 2610 } 2611 } 2612 2613 bufobj_wwait(bo, 0, 0); 2614 BO_UNLOCK(bo); 2615 vnode_pager_setsize(vp, length); 2616 2617 return (0); 2618 } 2619 2620 /* 2621 * Invalidate the cached pages of a file's buffer within the range of block 2622 * numbers [startlbn, endlbn). 2623 */ 2624 void 2625 v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn, 2626 int blksize) 2627 { 2628 struct bufobj *bo; 2629 off_t start, end; 2630 2631 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range"); 2632 2633 start = blksize * startlbn; 2634 end = blksize * endlbn; 2635 2636 bo = &vp->v_bufobj; 2637 BO_LOCK(bo); 2638 MPASS(blksize == bo->bo_bsize); 2639 2640 while (v_inval_buf_range_locked(vp, bo, startlbn, endlbn) == EAGAIN) 2641 ; 2642 2643 BO_UNLOCK(bo); 2644 vn_pages_remove(vp, OFF_TO_IDX(start), OFF_TO_IDX(end + PAGE_SIZE - 1)); 2645 } 2646 2647 static int 2648 v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, 2649 daddr_t startlbn, daddr_t endlbn) 2650 { 2651 struct bufv *bv; 2652 struct buf *bp, *nbp; 2653 uint8_t anyfreed; 2654 bool clean; 2655 2656 ASSERT_VOP_LOCKED(vp, "v_inval_buf_range_locked"); 2657 ASSERT_BO_LOCKED(bo); 2658 2659 anyfreed = 1; 2660 clean = true; 2661 do { 2662 bv = clean ? &bo->bo_clean : &bo->bo_dirty; 2663 bp = buf_lookup_ge(bv, startlbn); 2664 if (bp == NULL) 2665 continue; 2666 TAILQ_FOREACH_FROM_SAFE(bp, &bv->bv_hd, b_bobufs, nbp) { 2667 if (bp->b_lblkno >= endlbn) 2668 break; 2669 if (BUF_LOCK(bp, 2670 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 2671 BO_LOCKPTR(bo)) == ENOLCK) { 2672 BO_LOCK(bo); 2673 return (EAGAIN); 2674 } 2675 2676 bremfree(bp); 2677 bp->b_flags |= B_INVAL | B_RELBUF; 2678 bp->b_flags &= ~B_ASYNC; 2679 brelse(bp); 2680 anyfreed = 2; 2681 2682 BO_LOCK(bo); 2683 if (nbp != NULL && 2684 (((nbp->b_xflags & 2685 (clean ? BX_VNCLEAN : BX_VNDIRTY)) == 0) || 2686 nbp->b_vp != vp || 2687 (nbp->b_flags & B_DELWRI) == (clean? B_DELWRI: 0))) 2688 return (EAGAIN); 2689 } 2690 } while (clean = !clean, anyfreed-- > 0); 2691 return (0); 2692 } 2693 2694 static void 2695 buf_vlist_remove(struct buf *bp) 2696 { 2697 struct bufv *bv; 2698 b_xflags_t flags; 2699 2700 flags = bp->b_xflags; 2701 2702 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp)); 2703 ASSERT_BO_WLOCKED(bp->b_bufobj); 2704 KASSERT((flags & (BX_VNDIRTY | BX_VNCLEAN)) != 0 && 2705 (flags & (BX_VNDIRTY | BX_VNCLEAN)) != (BX_VNDIRTY | BX_VNCLEAN), 2706 ("%s: buffer %p has invalid queue state", __func__, bp)); 2707 2708 if ((flags & BX_VNDIRTY) != 0) 2709 bv = &bp->b_bufobj->bo_dirty; 2710 else 2711 bv = &bp->b_bufobj->bo_clean; 2712 BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno); 2713 TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs); 2714 bv->bv_cnt--; 2715 bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN); 2716 } 2717 2718 /* 2719 * Add the buffer to the sorted clean or dirty block list. Return zero on 2720 * success, EEXIST if a buffer with this identity already exists, or another 2721 * error on allocation failure. 2722 */ 2723 static inline int 2724 buf_vlist_find_or_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags) 2725 { 2726 struct bufv *bv; 2727 struct buf *n; 2728 int error; 2729 2730 ASSERT_BO_WLOCKED(bo); 2731 KASSERT((bo->bo_flag & BO_NOBUFS) == 0, 2732 ("buf_vlist_add: bo %p does not allow bufs", bo)); 2733 KASSERT((xflags & BX_VNDIRTY) == 0 || (bo->bo_flag & BO_DEAD) == 0, 2734 ("dead bo %p", bo)); 2735 KASSERT((bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) == xflags, 2736 ("buf_vlist_add: b_xflags %#x not set on bp %p", xflags, bp)); 2737 2738 if (xflags & BX_VNDIRTY) 2739 bv = &bo->bo_dirty; 2740 else 2741 bv = &bo->bo_clean; 2742 2743 error = buf_insert_lookup_le(bv, bp, &n); 2744 if (n == NULL) { 2745 KASSERT(error != EEXIST, 2746 ("buf_vlist_add: EEXIST but no existing buf found: bp %p", 2747 bp)); 2748 } else { 2749 KASSERT(n->b_lblkno <= bp->b_lblkno, 2750 ("buf_vlist_add: out of order insert/lookup: bp %p n %p", 2751 bp, n)); 2752 KASSERT((n->b_lblkno == bp->b_lblkno) == (error == EEXIST), 2753 ("buf_vlist_add: inconsistent result for existing buf: " 2754 "error %d bp %p n %p", error, bp, n)); 2755 } 2756 if (error != 0) 2757 return (error); 2758 2759 /* Keep the list ordered. */ 2760 if (n == NULL) { 2761 KASSERT(TAILQ_EMPTY(&bv->bv_hd) || 2762 bp->b_lblkno < TAILQ_FIRST(&bv->bv_hd)->b_lblkno, 2763 ("buf_vlist_add: queue order: " 2764 "%p should be before first %p", 2765 bp, TAILQ_FIRST(&bv->bv_hd))); 2766 TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs); 2767 } else { 2768 KASSERT(TAILQ_NEXT(n, b_bobufs) == NULL || 2769 bp->b_lblkno < TAILQ_NEXT(n, b_bobufs)->b_lblkno, 2770 ("buf_vlist_add: queue order: " 2771 "%p should be before next %p", 2772 bp, TAILQ_NEXT(n, b_bobufs))); 2773 TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs); 2774 } 2775 2776 bv->bv_cnt++; 2777 return (0); 2778 } 2779 2780 /* 2781 * Add the buffer to the sorted clean or dirty block list. 2782 * 2783 * NOTE: xflags is passed as a constant, optimizing this inline function! 2784 */ 2785 static void 2786 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags) 2787 { 2788 int error; 2789 2790 KASSERT((bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) == 0, 2791 ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags)); 2792 bp->b_xflags |= xflags; 2793 error = buf_vlist_find_or_add(bp, bo, xflags); 2794 if (error) 2795 panic("buf_vlist_add: error=%d", error); 2796 } 2797 2798 /* 2799 * Look up a buffer using the buffer tries. 2800 */ 2801 struct buf * 2802 gbincore(struct bufobj *bo, daddr_t lblkno) 2803 { 2804 struct buf *bp; 2805 2806 ASSERT_BO_LOCKED(bo); 2807 bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno); 2808 if (bp != NULL) 2809 return (bp); 2810 return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno)); 2811 } 2812 2813 /* 2814 * Look up a buf using the buffer tries, without the bufobj lock. This relies 2815 * on SMR for safe lookup, and bufs being in a no-free zone to provide type 2816 * stability of the result. Like other lockless lookups, the found buf may 2817 * already be invalid by the time this function returns. 2818 */ 2819 struct buf * 2820 gbincore_unlocked(struct bufobj *bo, daddr_t lblkno) 2821 { 2822 struct buf *bp; 2823 2824 ASSERT_BO_UNLOCKED(bo); 2825 bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno); 2826 if (bp != NULL) 2827 return (bp); 2828 return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno)); 2829 } 2830 2831 /* 2832 * Associate a buffer with a vnode. 2833 */ 2834 int 2835 bgetvp(struct vnode *vp, struct buf *bp) 2836 { 2837 struct bufobj *bo; 2838 int error; 2839 2840 bo = &vp->v_bufobj; 2841 ASSERT_BO_UNLOCKED(bo); 2842 VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free")); 2843 2844 CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags); 2845 VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp, 2846 ("bgetvp: bp already attached! %p", bp)); 2847 2848 /* 2849 * Add the buf to the vnode's clean list unless we lost a race and find 2850 * an existing buf in either dirty or clean. 2851 */ 2852 bp->b_vp = vp; 2853 bp->b_bufobj = bo; 2854 bp->b_xflags |= BX_VNCLEAN; 2855 error = EEXIST; 2856 BO_LOCK(bo); 2857 if (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, bp->b_lblkno) == NULL) 2858 error = buf_vlist_find_or_add(bp, bo, BX_VNCLEAN); 2859 BO_UNLOCK(bo); 2860 if (__predict_true(error == 0)) { 2861 vhold(vp); 2862 return (0); 2863 } 2864 if (error != EEXIST) 2865 panic("bgetvp: buf_vlist_add error: %d", error); 2866 bp->b_vp = NULL; 2867 bp->b_bufobj = NULL; 2868 bp->b_xflags &= ~BX_VNCLEAN; 2869 return (error); 2870 } 2871 2872 /* 2873 * Disassociate a buffer from a vnode. 2874 */ 2875 void 2876 brelvp(struct buf *bp) 2877 { 2878 struct bufobj *bo; 2879 struct vnode *vp; 2880 2881 CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags); 2882 KASSERT(bp->b_vp != NULL, ("brelvp: NULL")); 2883 2884 /* 2885 * Delete from old vnode list, if on one. 2886 */ 2887 vp = bp->b_vp; /* XXX */ 2888 bo = bp->b_bufobj; 2889 BO_LOCK(bo); 2890 buf_vlist_remove(bp); 2891 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 2892 bo->bo_flag &= ~BO_ONWORKLST; 2893 mtx_lock(&sync_mtx); 2894 LIST_REMOVE(bo, bo_synclist); 2895 syncer_worklist_len--; 2896 mtx_unlock(&sync_mtx); 2897 } 2898 bp->b_vp = NULL; 2899 bp->b_bufobj = NULL; 2900 BO_UNLOCK(bo); 2901 vdrop(vp); 2902 } 2903 2904 /* 2905 * Add an item to the syncer work queue. 2906 */ 2907 static void 2908 vn_syncer_add_to_worklist(struct bufobj *bo, int delay) 2909 { 2910 int slot; 2911 2912 ASSERT_BO_WLOCKED(bo); 2913 2914 mtx_lock(&sync_mtx); 2915 if (bo->bo_flag & BO_ONWORKLST) 2916 LIST_REMOVE(bo, bo_synclist); 2917 else { 2918 bo->bo_flag |= BO_ONWORKLST; 2919 syncer_worklist_len++; 2920 } 2921 2922 if (delay > syncer_maxdelay - 2) 2923 delay = syncer_maxdelay - 2; 2924 slot = (syncer_delayno + delay) & syncer_mask; 2925 2926 LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist); 2927 mtx_unlock(&sync_mtx); 2928 } 2929 2930 static int 2931 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS) 2932 { 2933 int error, len; 2934 2935 mtx_lock(&sync_mtx); 2936 len = syncer_worklist_len - sync_vnode_count; 2937 mtx_unlock(&sync_mtx); 2938 error = SYSCTL_OUT(req, &len, sizeof(len)); 2939 return (error); 2940 } 2941 2942 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, 2943 CTLTYPE_INT | CTLFLAG_MPSAFE| CTLFLAG_RD, NULL, 0, 2944 sysctl_vfs_worklist_len, "I", "Syncer thread worklist length"); 2945 2946 static struct proc *updateproc; 2947 static void sched_sync(void); 2948 static struct kproc_desc up_kp = { 2949 "syncer", 2950 sched_sync, 2951 &updateproc 2952 }; 2953 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp); 2954 2955 static int 2956 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td) 2957 { 2958 struct vnode *vp; 2959 struct mount *mp; 2960 2961 *bo = LIST_FIRST(slp); 2962 if (*bo == NULL) 2963 return (0); 2964 vp = bo2vnode(*bo); 2965 if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0) 2966 return (1); 2967 /* 2968 * We use vhold in case the vnode does not 2969 * successfully sync. vhold prevents the vnode from 2970 * going away when we unlock the sync_mtx so that 2971 * we can acquire the vnode interlock. 2972 */ 2973 vholdl(vp); 2974 mtx_unlock(&sync_mtx); 2975 VI_UNLOCK(vp); 2976 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { 2977 vdrop(vp); 2978 mtx_lock(&sync_mtx); 2979 return (*bo == LIST_FIRST(slp)); 2980 } 2981 MPASSERT(mp == NULL || (curthread->td_pflags & TDP_IGNSUSP) != 0 || 2982 (mp->mnt_kern_flag & MNTK_SUSPENDED) == 0, mp, 2983 ("suspended mp syncing vp %p", vp)); 2984 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2985 (void) VOP_FSYNC(vp, MNT_LAZY, td); 2986 VOP_UNLOCK(vp); 2987 vn_finished_write(mp); 2988 BO_LOCK(*bo); 2989 if (((*bo)->bo_flag & BO_ONWORKLST) != 0) { 2990 /* 2991 * Put us back on the worklist. The worklist 2992 * routine will remove us from our current 2993 * position and then add us back in at a later 2994 * position. 2995 */ 2996 vn_syncer_add_to_worklist(*bo, syncdelay); 2997 } 2998 BO_UNLOCK(*bo); 2999 vdrop(vp); 3000 mtx_lock(&sync_mtx); 3001 return (0); 3002 } 3003 3004 static int first_printf = 1; 3005 3006 /* 3007 * System filesystem synchronizer daemon. 3008 */ 3009 static void 3010 sched_sync(void) 3011 { 3012 struct synclist *next, *slp; 3013 struct bufobj *bo; 3014 long starttime; 3015 struct thread *td = curthread; 3016 int last_work_seen; 3017 int net_worklist_len; 3018 int syncer_final_iter; 3019 int error; 3020 3021 last_work_seen = 0; 3022 syncer_final_iter = 0; 3023 syncer_state = SYNCER_RUNNING; 3024 starttime = time_uptime; 3025 td->td_pflags |= TDP_NORUNNINGBUF; 3026 3027 EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc, 3028 SHUTDOWN_PRI_LAST); 3029 3030 mtx_lock(&sync_mtx); 3031 for (;;) { 3032 if (syncer_state == SYNCER_FINAL_DELAY && 3033 syncer_final_iter == 0) { 3034 mtx_unlock(&sync_mtx); 3035 kproc_suspend_check(td->td_proc); 3036 mtx_lock(&sync_mtx); 3037 } 3038 net_worklist_len = syncer_worklist_len - sync_vnode_count; 3039 if (syncer_state != SYNCER_RUNNING && 3040 starttime != time_uptime) { 3041 if (first_printf) { 3042 printf("\nSyncing disks, vnodes remaining... "); 3043 first_printf = 0; 3044 } 3045 printf("%d ", net_worklist_len); 3046 } 3047 starttime = time_uptime; 3048 3049 /* 3050 * Push files whose dirty time has expired. Be careful 3051 * of interrupt race on slp queue. 3052 * 3053 * Skip over empty worklist slots when shutting down. 3054 */ 3055 do { 3056 slp = &syncer_workitem_pending[syncer_delayno]; 3057 syncer_delayno += 1; 3058 if (syncer_delayno == syncer_maxdelay) 3059 syncer_delayno = 0; 3060 next = &syncer_workitem_pending[syncer_delayno]; 3061 /* 3062 * If the worklist has wrapped since the 3063 * it was emptied of all but syncer vnodes, 3064 * switch to the FINAL_DELAY state and run 3065 * for one more second. 3066 */ 3067 if (syncer_state == SYNCER_SHUTTING_DOWN && 3068 net_worklist_len == 0 && 3069 last_work_seen == syncer_delayno) { 3070 syncer_state = SYNCER_FINAL_DELAY; 3071 syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP; 3072 } 3073 } while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) && 3074 syncer_worklist_len > 0); 3075 3076 /* 3077 * Keep track of the last time there was anything 3078 * on the worklist other than syncer vnodes. 3079 * Return to the SHUTTING_DOWN state if any 3080 * new work appears. 3081 */ 3082 if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING) 3083 last_work_seen = syncer_delayno; 3084 if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY) 3085 syncer_state = SYNCER_SHUTTING_DOWN; 3086 while (!LIST_EMPTY(slp)) { 3087 error = sync_vnode(slp, &bo, td); 3088 if (error == 1) { 3089 LIST_REMOVE(bo, bo_synclist); 3090 LIST_INSERT_HEAD(next, bo, bo_synclist); 3091 continue; 3092 } 3093 3094 if (first_printf == 0) { 3095 /* 3096 * Drop the sync mutex, because some watchdog 3097 * drivers need to sleep while patting 3098 */ 3099 mtx_unlock(&sync_mtx); 3100 wdog_kern_pat(WD_LASTVAL); 3101 mtx_lock(&sync_mtx); 3102 } 3103 } 3104 if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0) 3105 syncer_final_iter--; 3106 /* 3107 * The variable rushjob allows the kernel to speed up the 3108 * processing of the filesystem syncer process. A rushjob 3109 * value of N tells the filesystem syncer to process the next 3110 * N seconds worth of work on its queue ASAP. Currently rushjob 3111 * is used by the soft update code to speed up the filesystem 3112 * syncer process when the incore state is getting so far 3113 * ahead of the disk that the kernel memory pool is being 3114 * threatened with exhaustion. 3115 */ 3116 if (rushjob > 0) { 3117 rushjob -= 1; 3118 continue; 3119 } 3120 /* 3121 * Just sleep for a short period of time between 3122 * iterations when shutting down to allow some I/O 3123 * to happen. 3124 * 3125 * If it has taken us less than a second to process the 3126 * current work, then wait. Otherwise start right over 3127 * again. We can still lose time if any single round 3128 * takes more than two seconds, but it does not really 3129 * matter as we are just trying to generally pace the 3130 * filesystem activity. 3131 */ 3132 if (syncer_state != SYNCER_RUNNING || 3133 time_uptime == starttime) { 3134 thread_lock(td); 3135 sched_prio(td, PPAUSE); 3136 thread_unlock(td); 3137 } 3138 if (syncer_state != SYNCER_RUNNING) 3139 cv_timedwait(&sync_wakeup, &sync_mtx, 3140 hz / SYNCER_SHUTDOWN_SPEEDUP); 3141 else if (time_uptime == starttime) 3142 cv_timedwait(&sync_wakeup, &sync_mtx, hz); 3143 } 3144 } 3145 3146 /* 3147 * Request the syncer daemon to speed up its work. 3148 * We never push it to speed up more than half of its 3149 * normal turn time, otherwise it could take over the cpu. 3150 */ 3151 int 3152 speedup_syncer(void) 3153 { 3154 int ret = 0; 3155 3156 mtx_lock(&sync_mtx); 3157 if (rushjob < syncdelay / 2) { 3158 rushjob += 1; 3159 stat_rush_requests += 1; 3160 ret = 1; 3161 } 3162 mtx_unlock(&sync_mtx); 3163 cv_broadcast(&sync_wakeup); 3164 return (ret); 3165 } 3166 3167 /* 3168 * Tell the syncer to speed up its work and run though its work 3169 * list several times, then tell it to shut down. 3170 */ 3171 static void 3172 syncer_shutdown(void *arg, int howto) 3173 { 3174 3175 if (howto & RB_NOSYNC) 3176 return; 3177 mtx_lock(&sync_mtx); 3178 syncer_state = SYNCER_SHUTTING_DOWN; 3179 rushjob = 0; 3180 mtx_unlock(&sync_mtx); 3181 cv_broadcast(&sync_wakeup); 3182 kproc_shutdown(arg, howto); 3183 } 3184 3185 void 3186 syncer_suspend(void) 3187 { 3188 3189 syncer_shutdown(updateproc, 0); 3190 } 3191 3192 void 3193 syncer_resume(void) 3194 { 3195 3196 mtx_lock(&sync_mtx); 3197 first_printf = 1; 3198 syncer_state = SYNCER_RUNNING; 3199 mtx_unlock(&sync_mtx); 3200 cv_broadcast(&sync_wakeup); 3201 kproc_resume(updateproc); 3202 } 3203 3204 /* 3205 * Move the buffer between the clean and dirty lists of its vnode. 3206 */ 3207 void 3208 reassignbuf(struct buf *bp) 3209 { 3210 struct vnode *vp; 3211 struct bufobj *bo; 3212 int delay; 3213 #ifdef INVARIANTS 3214 struct bufv *bv; 3215 #endif 3216 3217 vp = bp->b_vp; 3218 bo = bp->b_bufobj; 3219 3220 KASSERT((bp->b_flags & B_PAGING) == 0, 3221 ("%s: cannot reassign paging buffer %p", __func__, bp)); 3222 3223 CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X", 3224 bp, bp->b_vp, bp->b_flags); 3225 3226 BO_LOCK(bo); 3227 if ((bo->bo_flag & BO_NONSTERILE) == 0) { 3228 /* 3229 * Coordinate with getblk's unlocked lookup. Make 3230 * BO_NONSTERILE visible before the first reassignbuf produces 3231 * any side effect. This could be outside the bo lock if we 3232 * used a separate atomic flag field. 3233 */ 3234 bo->bo_flag |= BO_NONSTERILE; 3235 atomic_thread_fence_rel(); 3236 } 3237 buf_vlist_remove(bp); 3238 3239 /* 3240 * If dirty, put on list of dirty buffers; otherwise insert onto list 3241 * of clean buffers. 3242 */ 3243 if (bp->b_flags & B_DELWRI) { 3244 if ((bo->bo_flag & BO_ONWORKLST) == 0) { 3245 switch (vp->v_type) { 3246 case VDIR: 3247 delay = dirdelay; 3248 break; 3249 case VCHR: 3250 delay = metadelay; 3251 break; 3252 default: 3253 delay = filedelay; 3254 } 3255 vn_syncer_add_to_worklist(bo, delay); 3256 } 3257 buf_vlist_add(bp, bo, BX_VNDIRTY); 3258 } else { 3259 buf_vlist_add(bp, bo, BX_VNCLEAN); 3260 3261 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 3262 mtx_lock(&sync_mtx); 3263 LIST_REMOVE(bo, bo_synclist); 3264 syncer_worklist_len--; 3265 mtx_unlock(&sync_mtx); 3266 bo->bo_flag &= ~BO_ONWORKLST; 3267 } 3268 } 3269 #ifdef INVARIANTS 3270 bv = &bo->bo_clean; 3271 bp = TAILQ_FIRST(&bv->bv_hd); 3272 KASSERT(bp == NULL || bp->b_bufobj == bo, 3273 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3274 bp = TAILQ_LAST(&bv->bv_hd, buflists); 3275 KASSERT(bp == NULL || bp->b_bufobj == bo, 3276 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3277 bv = &bo->bo_dirty; 3278 bp = TAILQ_FIRST(&bv->bv_hd); 3279 KASSERT(bp == NULL || bp->b_bufobj == bo, 3280 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3281 bp = TAILQ_LAST(&bv->bv_hd, buflists); 3282 KASSERT(bp == NULL || bp->b_bufobj == bo, 3283 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); 3284 #endif 3285 BO_UNLOCK(bo); 3286 } 3287 3288 static void 3289 v_init_counters(struct vnode *vp) 3290 { 3291 3292 VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0, 3293 vp, ("%s called for an initialized vnode", __FUNCTION__)); 3294 ASSERT_VI_UNLOCKED(vp, __FUNCTION__); 3295 3296 refcount_init(&vp->v_holdcnt, 1); 3297 refcount_init(&vp->v_usecount, 1); 3298 } 3299 3300 /* 3301 * Get a usecount on a vnode. 3302 * 3303 * vget and vget_finish may fail to lock the vnode if they lose a race against 3304 * it being doomed. LK_RETRY can be passed in flags to lock it anyway. 3305 * 3306 * Consumers which don't guarantee liveness of the vnode can use SMR to 3307 * try to get a reference. Note this operation can fail since the vnode 3308 * may be awaiting getting freed by the time they get to it. 3309 */ 3310 enum vgetstate 3311 vget_prep_smr(struct vnode *vp) 3312 { 3313 enum vgetstate vs; 3314 3315 VFS_SMR_ASSERT_ENTERED(); 3316 3317 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 3318 vs = VGET_USECOUNT; 3319 } else { 3320 if (vhold_smr(vp)) 3321 vs = VGET_HOLDCNT; 3322 else 3323 vs = VGET_NONE; 3324 } 3325 return (vs); 3326 } 3327 3328 enum vgetstate 3329 vget_prep(struct vnode *vp) 3330 { 3331 enum vgetstate vs; 3332 3333 if (refcount_acquire_if_not_zero(&vp->v_usecount)) { 3334 vs = VGET_USECOUNT; 3335 } else { 3336 vhold(vp); 3337 vs = VGET_HOLDCNT; 3338 } 3339 return (vs); 3340 } 3341 3342 void 3343 vget_abort(struct vnode *vp, enum vgetstate vs) 3344 { 3345 3346 switch (vs) { 3347 case VGET_USECOUNT: 3348 vrele(vp); 3349 break; 3350 case VGET_HOLDCNT: 3351 vdrop(vp); 3352 break; 3353 default: 3354 __assert_unreachable(); 3355 } 3356 } 3357 3358 int 3359 vget(struct vnode *vp, int flags) 3360 { 3361 enum vgetstate vs; 3362 3363 vs = vget_prep(vp); 3364 return (vget_finish(vp, flags, vs)); 3365 } 3366 3367 int 3368 vget_finish(struct vnode *vp, int flags, enum vgetstate vs) 3369 { 3370 int error; 3371 3372 if ((flags & LK_INTERLOCK) != 0) 3373 ASSERT_VI_LOCKED(vp, __func__); 3374 else 3375 ASSERT_VI_UNLOCKED(vp, __func__); 3376 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3377 VNPASS(vp->v_holdcnt > 0, vp); 3378 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3379 3380 error = vn_lock(vp, flags); 3381 if (__predict_false(error != 0)) { 3382 vget_abort(vp, vs); 3383 CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__, 3384 vp); 3385 return (error); 3386 } 3387 3388 vget_finish_ref(vp, vs); 3389 return (0); 3390 } 3391 3392 void 3393 vget_finish_ref(struct vnode *vp, enum vgetstate vs) 3394 { 3395 int old; 3396 3397 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp); 3398 VNPASS(vp->v_holdcnt > 0, vp); 3399 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp); 3400 3401 if (vs == VGET_USECOUNT) 3402 return; 3403 3404 /* 3405 * We hold the vnode. If the usecount is 0 it will be utilized to keep 3406 * the vnode around. Otherwise someone else lended their hold count and 3407 * we have to drop ours. 3408 */ 3409 old = atomic_fetchadd_int(&vp->v_usecount, 1); 3410 VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old)); 3411 if (old != 0) { 3412 #ifdef INVARIANTS 3413 old = atomic_fetchadd_int(&vp->v_holdcnt, -1); 3414 VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); 3415 #else 3416 refcount_release(&vp->v_holdcnt); 3417 #endif 3418 } 3419 } 3420 3421 void 3422 vref(struct vnode *vp) 3423 { 3424 enum vgetstate vs; 3425 3426 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3427 vs = vget_prep(vp); 3428 vget_finish_ref(vp, vs); 3429 } 3430 3431 void 3432 vrefact(struct vnode *vp) 3433 { 3434 int old __diagused; 3435 3436 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3437 old = refcount_acquire(&vp->v_usecount); 3438 VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old)); 3439 } 3440 3441 void 3442 vlazy(struct vnode *vp) 3443 { 3444 struct mount *mp; 3445 3446 VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); 3447 3448 if ((vp->v_mflag & VMP_LAZYLIST) != 0) 3449 return; 3450 /* 3451 * We may get here for inactive routines after the vnode got doomed. 3452 */ 3453 if (VN_IS_DOOMED(vp)) 3454 return; 3455 mp = vp->v_mount; 3456 mtx_lock(&mp->mnt_listmtx); 3457 if ((vp->v_mflag & VMP_LAZYLIST) == 0) { 3458 vp->v_mflag |= VMP_LAZYLIST; 3459 TAILQ_INSERT_TAIL(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3460 mp->mnt_lazyvnodelistsize++; 3461 } 3462 mtx_unlock(&mp->mnt_listmtx); 3463 } 3464 3465 static void 3466 vunlazy(struct vnode *vp) 3467 { 3468 struct mount *mp; 3469 3470 ASSERT_VI_LOCKED(vp, __func__); 3471 VNPASS(!VN_IS_DOOMED(vp), vp); 3472 3473 mp = vp->v_mount; 3474 mtx_lock(&mp->mnt_listmtx); 3475 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3476 /* 3477 * Don't remove the vnode from the lazy list if another thread 3478 * has increased the hold count. It may have re-enqueued the 3479 * vnode to the lazy list and is now responsible for its 3480 * removal. 3481 */ 3482 if (vp->v_holdcnt == 0) { 3483 vp->v_mflag &= ~VMP_LAZYLIST; 3484 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3485 mp->mnt_lazyvnodelistsize--; 3486 } 3487 mtx_unlock(&mp->mnt_listmtx); 3488 } 3489 3490 /* 3491 * This routine is only meant to be called from vgonel prior to dooming 3492 * the vnode. 3493 */ 3494 static void 3495 vunlazy_gone(struct vnode *vp) 3496 { 3497 struct mount *mp; 3498 3499 ASSERT_VOP_ELOCKED(vp, __func__); 3500 ASSERT_VI_LOCKED(vp, __func__); 3501 VNPASS(!VN_IS_DOOMED(vp), vp); 3502 3503 if (vp->v_mflag & VMP_LAZYLIST) { 3504 mp = vp->v_mount; 3505 mtx_lock(&mp->mnt_listmtx); 3506 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 3507 vp->v_mflag &= ~VMP_LAZYLIST; 3508 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); 3509 mp->mnt_lazyvnodelistsize--; 3510 mtx_unlock(&mp->mnt_listmtx); 3511 } 3512 } 3513 3514 static void 3515 vdefer_inactive(struct vnode *vp) 3516 { 3517 3518 ASSERT_VI_LOCKED(vp, __func__); 3519 VNPASS(vp->v_holdcnt > 0, vp); 3520 if (VN_IS_DOOMED(vp)) { 3521 vdropl(vp); 3522 return; 3523 } 3524 if (vp->v_iflag & VI_DEFINACT) { 3525 VNPASS(vp->v_holdcnt > 1, vp); 3526 vdropl(vp); 3527 return; 3528 } 3529 if (vp->v_usecount > 0) { 3530 vp->v_iflag &= ~VI_OWEINACT; 3531 vdropl(vp); 3532 return; 3533 } 3534 vlazy(vp); 3535 vp->v_iflag |= VI_DEFINACT; 3536 VI_UNLOCK(vp); 3537 atomic_add_long(&deferred_inact, 1); 3538 } 3539 3540 static void 3541 vdefer_inactive_unlocked(struct vnode *vp) 3542 { 3543 3544 VI_LOCK(vp); 3545 if ((vp->v_iflag & VI_OWEINACT) == 0) { 3546 vdropl(vp); 3547 return; 3548 } 3549 vdefer_inactive(vp); 3550 } 3551 3552 enum vput_op { VRELE, VPUT, VUNREF }; 3553 3554 /* 3555 * Handle ->v_usecount transitioning to 0. 3556 * 3557 * By releasing the last usecount we take ownership of the hold count which 3558 * provides liveness of the vnode, meaning we have to vdrop. 3559 * 3560 * For all vnodes we may need to perform inactive processing. It requires an 3561 * exclusive lock on the vnode, while it is legal to call here with only a 3562 * shared lock (or no locks). If locking the vnode in an expected manner fails, 3563 * inactive processing gets deferred to the syncer. 3564 * 3565 * XXX Some filesystems pass in an exclusively locked vnode and strongly depend 3566 * on the lock being held all the way until VOP_INACTIVE. This in particular 3567 * happens with UFS which adds half-constructed vnodes to the hash, where they 3568 * can be found by other code. 3569 */ 3570 static void 3571 vput_final(struct vnode *vp, enum vput_op func) 3572 { 3573 int error; 3574 bool want_unlock; 3575 3576 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3577 VNPASS(vp->v_holdcnt > 0, vp); 3578 3579 VI_LOCK(vp); 3580 3581 /* 3582 * By the time we got here someone else might have transitioned 3583 * the count back to > 0. 3584 */ 3585 if (vp->v_usecount > 0) 3586 goto out; 3587 3588 /* 3589 * If the vnode is doomed vgone already performed inactive processing 3590 * (if needed). 3591 */ 3592 if (VN_IS_DOOMED(vp)) 3593 goto out; 3594 3595 if (__predict_true(VOP_NEED_INACTIVE(vp) == 0)) 3596 goto out; 3597 3598 if (vp->v_iflag & VI_DOINGINACT) 3599 goto out; 3600 3601 /* 3602 * Locking operations here will drop the interlock and possibly the 3603 * vnode lock, opening a window where the vnode can get doomed all the 3604 * while ->v_usecount is 0. Set VI_OWEINACT to let vgone know to 3605 * perform inactive. 3606 */ 3607 vp->v_iflag |= VI_OWEINACT; 3608 want_unlock = false; 3609 error = 0; 3610 switch (func) { 3611 case VRELE: 3612 switch (VOP_ISLOCKED(vp)) { 3613 case LK_EXCLUSIVE: 3614 break; 3615 case LK_EXCLOTHER: 3616 case 0: 3617 want_unlock = true; 3618 error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); 3619 VI_LOCK(vp); 3620 break; 3621 default: 3622 /* 3623 * The lock has at least one sharer, but we have no way 3624 * to conclude whether this is us. Play it safe and 3625 * defer processing. 3626 */ 3627 error = EAGAIN; 3628 break; 3629 } 3630 break; 3631 case VPUT: 3632 want_unlock = true; 3633 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3634 error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK | 3635 LK_NOWAIT); 3636 VI_LOCK(vp); 3637 } 3638 break; 3639 case VUNREF: 3640 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { 3641 error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK); 3642 VI_LOCK(vp); 3643 } 3644 break; 3645 } 3646 if (error == 0) { 3647 if (func == VUNREF) { 3648 VNASSERT((vp->v_vflag & VV_UNREF) == 0, vp, 3649 ("recursive vunref")); 3650 vp->v_vflag |= VV_UNREF; 3651 } 3652 for (;;) { 3653 error = vinactive(vp); 3654 if (want_unlock) 3655 VOP_UNLOCK(vp); 3656 if (error != ERELOOKUP || !want_unlock) 3657 break; 3658 VOP_LOCK(vp, LK_EXCLUSIVE); 3659 } 3660 if (func == VUNREF) 3661 vp->v_vflag &= ~VV_UNREF; 3662 vdropl(vp); 3663 } else { 3664 vdefer_inactive(vp); 3665 } 3666 return; 3667 out: 3668 if (func == VPUT) 3669 VOP_UNLOCK(vp); 3670 vdropl(vp); 3671 } 3672 3673 /* 3674 * Decrement ->v_usecount for a vnode. 3675 * 3676 * Releasing the last use count requires additional processing, see vput_final 3677 * above for details. 3678 * 3679 * Comment above each variant denotes lock state on entry and exit. 3680 */ 3681 3682 /* 3683 * in: any 3684 * out: same as passed in 3685 */ 3686 void 3687 vrele(struct vnode *vp) 3688 { 3689 3690 ASSERT_VI_UNLOCKED(vp, __func__); 3691 if (!refcount_release(&vp->v_usecount)) 3692 return; 3693 vput_final(vp, VRELE); 3694 } 3695 3696 /* 3697 * in: locked 3698 * out: unlocked 3699 */ 3700 void 3701 vput(struct vnode *vp) 3702 { 3703 3704 ASSERT_VOP_LOCKED(vp, __func__); 3705 ASSERT_VI_UNLOCKED(vp, __func__); 3706 if (!refcount_release(&vp->v_usecount)) { 3707 VOP_UNLOCK(vp); 3708 return; 3709 } 3710 vput_final(vp, VPUT); 3711 } 3712 3713 /* 3714 * in: locked 3715 * out: locked 3716 */ 3717 void 3718 vunref(struct vnode *vp) 3719 { 3720 3721 ASSERT_VOP_LOCKED(vp, __func__); 3722 ASSERT_VI_UNLOCKED(vp, __func__); 3723 if (!refcount_release(&vp->v_usecount)) 3724 return; 3725 vput_final(vp, VUNREF); 3726 } 3727 3728 void 3729 vhold(struct vnode *vp) 3730 { 3731 int old; 3732 3733 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3734 old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3735 VNASSERT(old >= 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3736 ("%s: wrong hold count %d", __func__, old)); 3737 if (old == 0) 3738 vfs_freevnodes_dec(); 3739 } 3740 3741 void 3742 vholdnz(struct vnode *vp) 3743 { 3744 3745 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 3746 #ifdef INVARIANTS 3747 int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3748 VNASSERT(old > 0 && (old & VHOLD_ALL_FLAGS) == 0, vp, 3749 ("%s: wrong hold count %d", __func__, old)); 3750 #else 3751 atomic_add_int(&vp->v_holdcnt, 1); 3752 #endif 3753 } 3754 3755 /* 3756 * Grab a hold count unless the vnode is freed. 3757 * 3758 * Only use this routine if vfs smr is the only protection you have against 3759 * freeing the vnode. 3760 * 3761 * The code loops trying to add a hold count as long as the VHOLD_NO_SMR flag 3762 * is not set. After the flag is set the vnode becomes immutable to anyone but 3763 * the thread which managed to set the flag. 3764 * 3765 * It may be tempting to replace the loop with: 3766 * count = atomic_fetchadd_int(&vp->v_holdcnt, 1); 3767 * if (count & VHOLD_NO_SMR) { 3768 * backpedal and error out; 3769 * } 3770 * 3771 * However, while this is more performant, it hinders debugging by eliminating 3772 * the previously mentioned invariant. 3773 */ 3774 bool 3775 vhold_smr(struct vnode *vp) 3776 { 3777 int count; 3778 3779 VFS_SMR_ASSERT_ENTERED(); 3780 3781 count = atomic_load_int(&vp->v_holdcnt); 3782 for (;;) { 3783 if (count & VHOLD_NO_SMR) { 3784 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3785 ("non-zero hold count with flags %d\n", count)); 3786 return (false); 3787 } 3788 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3789 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3790 if (count == 0) 3791 vfs_freevnodes_dec(); 3792 return (true); 3793 } 3794 } 3795 } 3796 3797 /* 3798 * Hold a free vnode for recycling. 3799 * 3800 * Note: vnode_init references this comment. 3801 * 3802 * Attempts to recycle only need the global vnode list lock and have no use for 3803 * SMR. 3804 * 3805 * However, vnodes get inserted into the global list before they get fully 3806 * initialized and stay there until UMA decides to free the memory. This in 3807 * particular means the target can be found before it becomes usable and after 3808 * it becomes recycled. Picking up such vnodes is guarded with v_holdcnt set to 3809 * VHOLD_NO_SMR. 3810 * 3811 * Note: the vnode may gain more references after we transition the count 0->1. 3812 */ 3813 static bool 3814 vhold_recycle_free(struct vnode *vp) 3815 { 3816 int count; 3817 3818 mtx_assert(&vnode_list_mtx, MA_OWNED); 3819 3820 count = atomic_load_int(&vp->v_holdcnt); 3821 for (;;) { 3822 if (count & VHOLD_NO_SMR) { 3823 VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, 3824 ("non-zero hold count with flags %d\n", count)); 3825 return (false); 3826 } 3827 VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); 3828 if (count > 0) { 3829 return (false); 3830 } 3831 if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { 3832 vfs_freevnodes_dec(); 3833 return (true); 3834 } 3835 } 3836 } 3837 3838 static void __noinline 3839 vdbatch_process(struct vdbatch *vd) 3840 { 3841 struct vnode *vp; 3842 int i; 3843 3844 mtx_assert(&vd->lock, MA_OWNED); 3845 MPASS(curthread->td_pinned > 0); 3846 MPASS(vd->index == VDBATCH_SIZE); 3847 3848 /* 3849 * Attempt to requeue the passed batch, but give up easily. 3850 * 3851 * Despite batching the mechanism is prone to transient *significant* 3852 * lock contention, where vnode_list_mtx becomes the primary bottleneck 3853 * if multiple CPUs get here (one real-world example is highly parallel 3854 * do-nothing make , which will stat *tons* of vnodes). Since it is 3855 * quasi-LRU (read: not that great even if fully honoured) provide an 3856 * option to just dodge the problem. Parties which don't like it are 3857 * welcome to implement something better. 3858 */ 3859 if (vnode_can_skip_requeue) { 3860 if (!mtx_trylock(&vnode_list_mtx)) { 3861 counter_u64_add(vnode_skipped_requeues, 1); 3862 critical_enter(); 3863 for (i = 0; i < VDBATCH_SIZE; i++) { 3864 vp = vd->tab[i]; 3865 vd->tab[i] = NULL; 3866 MPASS(vp->v_dbatchcpu != NOCPU); 3867 vp->v_dbatchcpu = NOCPU; 3868 } 3869 vd->index = 0; 3870 critical_exit(); 3871 return; 3872 3873 } 3874 /* fallthrough to locked processing */ 3875 } else { 3876 mtx_lock(&vnode_list_mtx); 3877 } 3878 3879 mtx_assert(&vnode_list_mtx, MA_OWNED); 3880 critical_enter(); 3881 for (i = 0; i < VDBATCH_SIZE; i++) { 3882 vp = vd->tab[i]; 3883 vd->tab[i] = NULL; 3884 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist); 3885 TAILQ_INSERT_TAIL(&vnode_list, vp, v_vnodelist); 3886 MPASS(vp->v_dbatchcpu != NOCPU); 3887 vp->v_dbatchcpu = NOCPU; 3888 } 3889 mtx_unlock(&vnode_list_mtx); 3890 vd->index = 0; 3891 critical_exit(); 3892 } 3893 3894 static void 3895 vdbatch_enqueue(struct vnode *vp) 3896 { 3897 struct vdbatch *vd; 3898 3899 ASSERT_VI_LOCKED(vp, __func__); 3900 VNPASS(!VN_IS_DOOMED(vp), vp); 3901 3902 if (vp->v_dbatchcpu != NOCPU) { 3903 VI_UNLOCK(vp); 3904 return; 3905 } 3906 3907 sched_pin(); 3908 vd = DPCPU_PTR(vd); 3909 mtx_lock(&vd->lock); 3910 MPASS(vd->index < VDBATCH_SIZE); 3911 MPASS(vd->tab[vd->index] == NULL); 3912 /* 3913 * A hack: we depend on being pinned so that we know what to put in 3914 * ->v_dbatchcpu. 3915 */ 3916 vp->v_dbatchcpu = curcpu; 3917 vd->tab[vd->index] = vp; 3918 vd->index++; 3919 VI_UNLOCK(vp); 3920 if (vd->index == VDBATCH_SIZE) 3921 vdbatch_process(vd); 3922 mtx_unlock(&vd->lock); 3923 sched_unpin(); 3924 } 3925 3926 /* 3927 * This routine must only be called for vnodes which are about to be 3928 * deallocated. Supporting dequeue for arbitrary vndoes would require 3929 * validating that the locked batch matches. 3930 */ 3931 static void 3932 vdbatch_dequeue(struct vnode *vp) 3933 { 3934 struct vdbatch *vd; 3935 int i; 3936 short cpu; 3937 3938 VNPASS(vp->v_type == VBAD || vp->v_type == VNON, vp); 3939 3940 cpu = vp->v_dbatchcpu; 3941 if (cpu == NOCPU) 3942 return; 3943 3944 vd = DPCPU_ID_PTR(cpu, vd); 3945 mtx_lock(&vd->lock); 3946 for (i = 0; i < vd->index; i++) { 3947 if (vd->tab[i] != vp) 3948 continue; 3949 vp->v_dbatchcpu = NOCPU; 3950 vd->index--; 3951 vd->tab[i] = vd->tab[vd->index]; 3952 vd->tab[vd->index] = NULL; 3953 break; 3954 } 3955 mtx_unlock(&vd->lock); 3956 /* 3957 * Either we dequeued the vnode above or the target CPU beat us to it. 3958 */ 3959 MPASS(vp->v_dbatchcpu == NOCPU); 3960 } 3961 3962 /* 3963 * Drop the hold count of the vnode. 3964 * 3965 * It will only get freed if this is the last hold *and* it has been vgone'd. 3966 * 3967 * Because the vnode vm object keeps a hold reference on the vnode if 3968 * there is at least one resident non-cached page, the vnode cannot 3969 * leave the active list without the page cleanup done. 3970 */ 3971 static void __noinline 3972 vdropl_final(struct vnode *vp) 3973 { 3974 3975 ASSERT_VI_LOCKED(vp, __func__); 3976 VNPASS(VN_IS_DOOMED(vp), vp); 3977 /* 3978 * Set the VHOLD_NO_SMR flag. 3979 * 3980 * We may be racing against vhold_smr. If they win we can just pretend 3981 * we never got this far, they will vdrop later. 3982 */ 3983 if (__predict_false(!atomic_cmpset_int(&vp->v_holdcnt, 0, VHOLD_NO_SMR))) { 3984 vfs_freevnodes_inc(); 3985 VI_UNLOCK(vp); 3986 /* 3987 * We lost the aforementioned race. Any subsequent access is 3988 * invalid as they might have managed to vdropl on their own. 3989 */ 3990 return; 3991 } 3992 /* 3993 * Don't bump freevnodes as this one is going away. 3994 */ 3995 freevnode(vp); 3996 } 3997 3998 void 3999 vdrop(struct vnode *vp) 4000 { 4001 4002 ASSERT_VI_UNLOCKED(vp, __func__); 4003 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4004 if (refcount_release_if_not_last(&vp->v_holdcnt)) 4005 return; 4006 VI_LOCK(vp); 4007 vdropl(vp); 4008 } 4009 4010 static __always_inline void 4011 vdropl_impl(struct vnode *vp, bool enqueue) 4012 { 4013 4014 ASSERT_VI_LOCKED(vp, __func__); 4015 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4016 if (!refcount_release(&vp->v_holdcnt)) { 4017 VI_UNLOCK(vp); 4018 return; 4019 } 4020 VNPASS((vp->v_iflag & VI_OWEINACT) == 0, vp); 4021 VNPASS((vp->v_iflag & VI_DEFINACT) == 0, vp); 4022 if (VN_IS_DOOMED(vp)) { 4023 vdropl_final(vp); 4024 return; 4025 } 4026 4027 vfs_freevnodes_inc(); 4028 if (vp->v_mflag & VMP_LAZYLIST) { 4029 vunlazy(vp); 4030 } 4031 4032 if (!enqueue) { 4033 VI_UNLOCK(vp); 4034 return; 4035 } 4036 4037 /* 4038 * Also unlocks the interlock. We can't assert on it as we 4039 * released our hold and by now the vnode might have been 4040 * freed. 4041 */ 4042 vdbatch_enqueue(vp); 4043 } 4044 4045 void 4046 vdropl(struct vnode *vp) 4047 { 4048 4049 vdropl_impl(vp, true); 4050 } 4051 4052 /* 4053 * vdrop a vnode when recycling 4054 * 4055 * This is a special case routine only to be used when recycling, differs from 4056 * regular vdrop by not requeieing the vnode on LRU. 4057 * 4058 * Consider a case where vtryrecycle continuously fails with all vnodes (due to 4059 * e.g., frozen writes on the filesystem), filling the batch and causing it to 4060 * be requeued. Then vnlru will end up revisiting the same vnodes. This is a 4061 * loop which can last for as long as writes are frozen. 4062 */ 4063 static void 4064 vdropl_recycle(struct vnode *vp) 4065 { 4066 4067 vdropl_impl(vp, false); 4068 } 4069 4070 static void 4071 vdrop_recycle(struct vnode *vp) 4072 { 4073 4074 VI_LOCK(vp); 4075 vdropl_recycle(vp); 4076 } 4077 4078 /* 4079 * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT 4080 * flags. DOINGINACT prevents us from recursing in calls to vinactive. 4081 */ 4082 static int 4083 vinactivef(struct vnode *vp) 4084 { 4085 int error; 4086 4087 ASSERT_VOP_ELOCKED(vp, "vinactive"); 4088 ASSERT_VI_LOCKED(vp, "vinactive"); 4089 VNPASS((vp->v_iflag & VI_DOINGINACT) == 0, vp); 4090 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4091 vp->v_iflag |= VI_DOINGINACT; 4092 vp->v_iflag &= ~VI_OWEINACT; 4093 VI_UNLOCK(vp); 4094 4095 /* 4096 * Before moving off the active list, we must be sure that any 4097 * modified pages are converted into the vnode's dirty 4098 * buffers, since these will no longer be checked once the 4099 * vnode is on the inactive list. 4100 * 4101 * The write-out of the dirty pages is asynchronous. At the 4102 * point that VOP_INACTIVE() is called, there could still be 4103 * pending I/O and dirty pages in the object. 4104 */ 4105 if ((vp->v_vflag & VV_NOSYNC) == 0) 4106 vnode_pager_clean_async(vp); 4107 4108 error = VOP_INACTIVE(vp); 4109 VI_LOCK(vp); 4110 VNPASS(vp->v_iflag & VI_DOINGINACT, vp); 4111 vp->v_iflag &= ~VI_DOINGINACT; 4112 return (error); 4113 } 4114 4115 int 4116 vinactive(struct vnode *vp) 4117 { 4118 4119 ASSERT_VOP_ELOCKED(vp, "vinactive"); 4120 ASSERT_VI_LOCKED(vp, "vinactive"); 4121 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4122 4123 if ((vp->v_iflag & VI_OWEINACT) == 0) 4124 return (0); 4125 if (vp->v_iflag & VI_DOINGINACT) 4126 return (0); 4127 if (vp->v_usecount > 0) { 4128 vp->v_iflag &= ~VI_OWEINACT; 4129 return (0); 4130 } 4131 return (vinactivef(vp)); 4132 } 4133 4134 /* 4135 * Remove any vnodes in the vnode table belonging to mount point mp. 4136 * 4137 * If FORCECLOSE is not specified, there should not be any active ones, 4138 * return error if any are found (nb: this is a user error, not a 4139 * system error). If FORCECLOSE is specified, detach any active vnodes 4140 * that are found. 4141 * 4142 * If WRITECLOSE is set, only flush out regular file vnodes open for 4143 * writing. 4144 * 4145 * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped. 4146 * 4147 * `rootrefs' specifies the base reference count for the root vnode 4148 * of this filesystem. The root vnode is considered busy if its 4149 * v_usecount exceeds this value. On a successful return, vflush(, td) 4150 * will call vrele() on the root vnode exactly rootrefs times. 4151 * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must 4152 * be zero. 4153 */ 4154 #ifdef DIAGNOSTIC 4155 static int busyprt = 0; /* print out busy vnodes */ 4156 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes"); 4157 #endif 4158 4159 int 4160 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td) 4161 { 4162 struct vnode *vp, *mvp, *rootvp = NULL; 4163 struct vattr vattr; 4164 int busy = 0, error; 4165 4166 CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp, 4167 rootrefs, flags); 4168 if (rootrefs > 0) { 4169 KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0, 4170 ("vflush: bad args")); 4171 /* 4172 * Get the filesystem root vnode. We can vput() it 4173 * immediately, since with rootrefs > 0, it won't go away. 4174 */ 4175 if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) { 4176 CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d", 4177 __func__, error); 4178 return (error); 4179 } 4180 vput(rootvp); 4181 } 4182 loop: 4183 MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { 4184 vholdl(vp); 4185 error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE); 4186 if (error) { 4187 vdrop(vp); 4188 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 4189 goto loop; 4190 } 4191 /* 4192 * Skip over a vnodes marked VV_SYSTEM. 4193 */ 4194 if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) { 4195 VOP_UNLOCK(vp); 4196 vdrop(vp); 4197 continue; 4198 } 4199 /* 4200 * If WRITECLOSE is set, flush out unlinked but still open 4201 * files (even if open only for reading) and regular file 4202 * vnodes open for writing. 4203 */ 4204 if (flags & WRITECLOSE) { 4205 vnode_pager_clean_async(vp); 4206 do { 4207 error = VOP_FSYNC(vp, MNT_WAIT, td); 4208 } while (error == ERELOOKUP); 4209 if (error != 0) { 4210 VOP_UNLOCK(vp); 4211 vdrop(vp); 4212 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); 4213 return (error); 4214 } 4215 error = VOP_GETATTR(vp, &vattr, td->td_ucred); 4216 VI_LOCK(vp); 4217 4218 if ((vp->v_type == VNON || 4219 (error == 0 && vattr.va_nlink > 0)) && 4220 (vp->v_writecount <= 0 || vp->v_type != VREG)) { 4221 VOP_UNLOCK(vp); 4222 vdropl(vp); 4223 continue; 4224 } 4225 } else 4226 VI_LOCK(vp); 4227 /* 4228 * With v_usecount == 0, all we need to do is clear out the 4229 * vnode data structures and we are done. 4230 * 4231 * If FORCECLOSE is set, forcibly close the vnode. 4232 */ 4233 if (vp->v_usecount == 0 || (flags & FORCECLOSE)) { 4234 vgonel(vp); 4235 } else { 4236 busy++; 4237 #ifdef DIAGNOSTIC 4238 if (busyprt) 4239 vn_printf(vp, "vflush: busy vnode "); 4240 #endif 4241 } 4242 VOP_UNLOCK(vp); 4243 vdropl(vp); 4244 } 4245 if (rootrefs > 0 && (flags & FORCECLOSE) == 0) { 4246 /* 4247 * If just the root vnode is busy, and if its refcount 4248 * is equal to `rootrefs', then go ahead and kill it. 4249 */ 4250 VI_LOCK(rootvp); 4251 KASSERT(busy > 0, ("vflush: not busy")); 4252 VNASSERT(rootvp->v_usecount >= rootrefs, rootvp, 4253 ("vflush: usecount %d < rootrefs %d", 4254 rootvp->v_usecount, rootrefs)); 4255 if (busy == 1 && rootvp->v_usecount == rootrefs) { 4256 VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK); 4257 vgone(rootvp); 4258 VOP_UNLOCK(rootvp); 4259 busy = 0; 4260 } else 4261 VI_UNLOCK(rootvp); 4262 } 4263 if (busy) { 4264 CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__, 4265 busy); 4266 return (EBUSY); 4267 } 4268 for (; rootrefs > 0; rootrefs--) 4269 vrele(rootvp); 4270 return (0); 4271 } 4272 4273 /* 4274 * Recycle an unused vnode. 4275 */ 4276 int 4277 vrecycle(struct vnode *vp) 4278 { 4279 int recycled; 4280 4281 VI_LOCK(vp); 4282 recycled = vrecyclel(vp); 4283 VI_UNLOCK(vp); 4284 return (recycled); 4285 } 4286 4287 /* 4288 * vrecycle, with the vp interlock held. 4289 */ 4290 int 4291 vrecyclel(struct vnode *vp) 4292 { 4293 int recycled; 4294 4295 ASSERT_VOP_ELOCKED(vp, __func__); 4296 ASSERT_VI_LOCKED(vp, __func__); 4297 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4298 recycled = 0; 4299 if (vp->v_usecount == 0) { 4300 recycled = 1; 4301 vgonel(vp); 4302 } 4303 return (recycled); 4304 } 4305 4306 /* 4307 * Eliminate all activity associated with a vnode 4308 * in preparation for reuse. 4309 */ 4310 void 4311 vgone(struct vnode *vp) 4312 { 4313 VI_LOCK(vp); 4314 vgonel(vp); 4315 VI_UNLOCK(vp); 4316 } 4317 4318 /* 4319 * Notify upper mounts about reclaimed or unlinked vnode. 4320 */ 4321 void 4322 vfs_notify_upper(struct vnode *vp, enum vfs_notify_upper_type event) 4323 { 4324 struct mount *mp; 4325 struct mount_upper_node *ump; 4326 4327 mp = atomic_load_ptr(&vp->v_mount); 4328 if (mp == NULL) 4329 return; 4330 if (TAILQ_EMPTY(&mp->mnt_notify)) 4331 return; 4332 4333 MNT_ILOCK(mp); 4334 mp->mnt_upper_pending++; 4335 KASSERT(mp->mnt_upper_pending > 0, 4336 ("%s: mnt_upper_pending %d", __func__, mp->mnt_upper_pending)); 4337 TAILQ_FOREACH(ump, &mp->mnt_notify, mnt_upper_link) { 4338 MNT_IUNLOCK(mp); 4339 switch (event) { 4340 case VFS_NOTIFY_UPPER_RECLAIM: 4341 VFS_RECLAIM_LOWERVP(ump->mp, vp); 4342 break; 4343 case VFS_NOTIFY_UPPER_UNLINK: 4344 VFS_UNLINK_LOWERVP(ump->mp, vp); 4345 break; 4346 } 4347 MNT_ILOCK(mp); 4348 } 4349 mp->mnt_upper_pending--; 4350 if ((mp->mnt_kern_flag & MNTK_UPPER_WAITER) != 0 && 4351 mp->mnt_upper_pending == 0) { 4352 mp->mnt_kern_flag &= ~MNTK_UPPER_WAITER; 4353 wakeup(&mp->mnt_uppers); 4354 } 4355 MNT_IUNLOCK(mp); 4356 } 4357 4358 /* 4359 * vgone, with the vp interlock held. 4360 */ 4361 static void 4362 vgonel(struct vnode *vp) 4363 { 4364 struct thread *td; 4365 struct mount *mp; 4366 vm_object_t object; 4367 bool active, doinginact, oweinact; 4368 4369 ASSERT_VOP_ELOCKED(vp, "vgonel"); 4370 ASSERT_VI_LOCKED(vp, "vgonel"); 4371 VNASSERT(vp->v_holdcnt, vp, 4372 ("vgonel: vp %p has no reference.", vp)); 4373 CTR2(KTR_VFS, "%s: vp %p", __func__, vp); 4374 td = curthread; 4375 4376 /* 4377 * Don't vgonel if we're already doomed. 4378 */ 4379 if (VN_IS_DOOMED(vp)) { 4380 VNPASS(vn_get_state(vp) == VSTATE_DESTROYING || \ 4381 vn_get_state(vp) == VSTATE_DEAD, vp); 4382 return; 4383 } 4384 /* 4385 * Paired with freevnode. 4386 */ 4387 vn_seqc_write_begin_locked(vp); 4388 vunlazy_gone(vp); 4389 vn_irflag_set_locked(vp, VIRF_DOOMED); 4390 vn_set_state(vp, VSTATE_DESTROYING); 4391 4392 /* 4393 * Check to see if the vnode is in use. If so, we have to 4394 * call VOP_CLOSE() and VOP_INACTIVE(). 4395 * 4396 * It could be that VOP_INACTIVE() requested reclamation, in 4397 * which case we should avoid recursion, so check 4398 * VI_DOINGINACT. This is not precise but good enough. 4399 */ 4400 active = vp->v_usecount > 0; 4401 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4402 doinginact = (vp->v_iflag & VI_DOINGINACT) != 0; 4403 4404 /* 4405 * If we need to do inactive VI_OWEINACT will be set. 4406 */ 4407 if (vp->v_iflag & VI_DEFINACT) { 4408 VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); 4409 vp->v_iflag &= ~VI_DEFINACT; 4410 vdropl(vp); 4411 } else { 4412 VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count")); 4413 VI_UNLOCK(vp); 4414 } 4415 cache_purge_vgone(vp); 4416 vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM); 4417 4418 /* 4419 * If purging an active vnode, it must be closed and 4420 * deactivated before being reclaimed. 4421 */ 4422 if (active) 4423 VOP_CLOSE(vp, FNONBLOCK, NOCRED, td); 4424 if (!doinginact) { 4425 do { 4426 if (oweinact || active) { 4427 VI_LOCK(vp); 4428 vinactivef(vp); 4429 oweinact = (vp->v_iflag & VI_OWEINACT) != 0; 4430 VI_UNLOCK(vp); 4431 } 4432 } while (oweinact); 4433 } 4434 if (vp->v_type == VSOCK) 4435 vfs_unp_reclaim(vp); 4436 4437 /* 4438 * Clean out any buffers associated with the vnode. 4439 * If the flush fails, just toss the buffers. 4440 */ 4441 mp = NULL; 4442 if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd)) 4443 (void) vn_start_secondary_write(vp, &mp, V_WAIT); 4444 if (vinvalbuf(vp, V_SAVE, 0, 0) != 0) { 4445 while (vinvalbuf(vp, 0, 0, 0) != 0) 4446 ; 4447 } 4448 4449 BO_LOCK(&vp->v_bufobj); 4450 KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) && 4451 vp->v_bufobj.bo_dirty.bv_cnt == 0 && 4452 TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) && 4453 vp->v_bufobj.bo_clean.bv_cnt == 0, 4454 ("vp %p bufobj not invalidated", vp)); 4455 4456 /* 4457 * For VMIO bufobj, BO_DEAD is set later, or in 4458 * vm_object_terminate() after the object's page queue is 4459 * flushed. 4460 */ 4461 object = vp->v_bufobj.bo_object; 4462 if (object == NULL) 4463 vp->v_bufobj.bo_flag |= BO_DEAD; 4464 BO_UNLOCK(&vp->v_bufobj); 4465 4466 /* 4467 * Handle the VM part. Tmpfs handles v_object on its own (the 4468 * OBJT_VNODE check). Nullfs or other bypassing filesystems 4469 * should not touch the object borrowed from the lower vnode 4470 * (the handle check). 4471 */ 4472 if (object != NULL && object->type == OBJT_VNODE && 4473 object->handle == vp) 4474 vnode_destroy_vobject(vp); 4475 4476 /* 4477 * Reclaim the vnode. 4478 */ 4479 if (VOP_RECLAIM(vp)) 4480 panic("vgone: cannot reclaim"); 4481 if (mp != NULL) 4482 vn_finished_secondary_write(mp); 4483 VNASSERT(vp->v_object == NULL, vp, 4484 ("vop_reclaim left v_object vp=%p", vp)); 4485 /* 4486 * Clear the advisory locks and wake up waiting threads. 4487 */ 4488 if (vp->v_lockf != NULL) { 4489 (void)VOP_ADVLOCKPURGE(vp); 4490 vp->v_lockf = NULL; 4491 } 4492 /* 4493 * Delete from old mount point vnode list. 4494 */ 4495 if (vp->v_mount == NULL) { 4496 VI_LOCK(vp); 4497 } else { 4498 delmntque(vp); 4499 ASSERT_VI_LOCKED(vp, "vgonel 2"); 4500 } 4501 /* 4502 * Done with purge, reset to the standard lock and invalidate 4503 * the vnode. 4504 */ 4505 vp->v_vnlock = &vp->v_lock; 4506 vp->v_op = &dead_vnodeops; 4507 vp->v_type = VBAD; 4508 vn_set_state(vp, VSTATE_DEAD); 4509 } 4510 4511 /* 4512 * Print out a description of a vnode. 4513 */ 4514 static const char *const vtypename[] = { 4515 [VNON] = "VNON", 4516 [VREG] = "VREG", 4517 [VDIR] = "VDIR", 4518 [VBLK] = "VBLK", 4519 [VCHR] = "VCHR", 4520 [VLNK] = "VLNK", 4521 [VSOCK] = "VSOCK", 4522 [VFIFO] = "VFIFO", 4523 [VBAD] = "VBAD", 4524 [VMARKER] = "VMARKER", 4525 }; 4526 _Static_assert(nitems(vtypename) == VLASTTYPE + 1, 4527 "vnode type name not added to vtypename"); 4528 4529 static const char *const vstatename[] = { 4530 [VSTATE_UNINITIALIZED] = "VSTATE_UNINITIALIZED", 4531 [VSTATE_CONSTRUCTED] = "VSTATE_CONSTRUCTED", 4532 [VSTATE_DESTROYING] = "VSTATE_DESTROYING", 4533 [VSTATE_DEAD] = "VSTATE_DEAD", 4534 }; 4535 _Static_assert(nitems(vstatename) == VLASTSTATE + 1, 4536 "vnode state name not added to vstatename"); 4537 4538 _Static_assert((VHOLD_ALL_FLAGS & ~VHOLD_NO_SMR) == 0, 4539 "new hold count flag not added to vn_printf"); 4540 4541 void 4542 vn_printf(struct vnode *vp, const char *fmt, ...) 4543 { 4544 va_list ap; 4545 char buf[256], buf2[16]; 4546 u_long flags; 4547 u_int holdcnt; 4548 short irflag; 4549 4550 va_start(ap, fmt); 4551 vprintf(fmt, ap); 4552 va_end(ap); 4553 printf("%p: ", (void *)vp); 4554 printf("type %s state %s op %p\n", vtypename[vp->v_type], 4555 vstatename[vp->v_state], vp->v_op); 4556 holdcnt = atomic_load_int(&vp->v_holdcnt); 4557 printf(" usecount %d, writecount %d, refcount %d seqc users %d", 4558 vp->v_usecount, vp->v_writecount, holdcnt & ~VHOLD_ALL_FLAGS, 4559 vp->v_seqc_users); 4560 switch (vp->v_type) { 4561 case VDIR: 4562 printf(" mountedhere %p\n", vp->v_mountedhere); 4563 break; 4564 case VCHR: 4565 printf(" rdev %p\n", vp->v_rdev); 4566 break; 4567 case VSOCK: 4568 printf(" socket %p\n", vp->v_unpcb); 4569 break; 4570 case VFIFO: 4571 printf(" fifoinfo %p\n", vp->v_fifoinfo); 4572 break; 4573 default: 4574 printf("\n"); 4575 break; 4576 } 4577 buf[0] = '\0'; 4578 buf[1] = '\0'; 4579 if (holdcnt & VHOLD_NO_SMR) 4580 strlcat(buf, "|VHOLD_NO_SMR", sizeof(buf)); 4581 printf(" hold count flags (%s)\n", buf + 1); 4582 4583 buf[0] = '\0'; 4584 buf[1] = '\0'; 4585 irflag = vn_irflag_read(vp); 4586 if (irflag & VIRF_DOOMED) 4587 strlcat(buf, "|VIRF_DOOMED", sizeof(buf)); 4588 if (irflag & VIRF_PGREAD) 4589 strlcat(buf, "|VIRF_PGREAD", sizeof(buf)); 4590 if (irflag & VIRF_MOUNTPOINT) 4591 strlcat(buf, "|VIRF_MOUNTPOINT", sizeof(buf)); 4592 if (irflag & VIRF_TEXT_REF) 4593 strlcat(buf, "|VIRF_TEXT_REF", sizeof(buf)); 4594 flags = irflag & ~(VIRF_DOOMED | VIRF_PGREAD | VIRF_MOUNTPOINT | VIRF_TEXT_REF); 4595 if (flags != 0) { 4596 snprintf(buf2, sizeof(buf2), "|VIRF(0x%lx)", flags); 4597 strlcat(buf, buf2, sizeof(buf)); 4598 } 4599 if (vp->v_vflag & VV_ROOT) 4600 strlcat(buf, "|VV_ROOT", sizeof(buf)); 4601 if (vp->v_vflag & VV_ISTTY) 4602 strlcat(buf, "|VV_ISTTY", sizeof(buf)); 4603 if (vp->v_vflag & VV_NOSYNC) 4604 strlcat(buf, "|VV_NOSYNC", sizeof(buf)); 4605 if (vp->v_vflag & VV_ETERNALDEV) 4606 strlcat(buf, "|VV_ETERNALDEV", sizeof(buf)); 4607 if (vp->v_vflag & VV_CACHEDLABEL) 4608 strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf)); 4609 if (vp->v_vflag & VV_VMSIZEVNLOCK) 4610 strlcat(buf, "|VV_VMSIZEVNLOCK", sizeof(buf)); 4611 if (vp->v_vflag & VV_COPYONWRITE) 4612 strlcat(buf, "|VV_COPYONWRITE", sizeof(buf)); 4613 if (vp->v_vflag & VV_SYSTEM) 4614 strlcat(buf, "|VV_SYSTEM", sizeof(buf)); 4615 if (vp->v_vflag & VV_PROCDEP) 4616 strlcat(buf, "|VV_PROCDEP", sizeof(buf)); 4617 if (vp->v_vflag & VV_DELETED) 4618 strlcat(buf, "|VV_DELETED", sizeof(buf)); 4619 if (vp->v_vflag & VV_MD) 4620 strlcat(buf, "|VV_MD", sizeof(buf)); 4621 if (vp->v_vflag & VV_FORCEINSMQ) 4622 strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf)); 4623 if (vp->v_vflag & VV_READLINK) 4624 strlcat(buf, "|VV_READLINK", sizeof(buf)); 4625 flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV | 4626 VV_CACHEDLABEL | VV_VMSIZEVNLOCK | VV_COPYONWRITE | VV_SYSTEM | 4627 VV_PROCDEP | VV_DELETED | VV_MD | VV_FORCEINSMQ | VV_READLINK); 4628 if (flags != 0) { 4629 snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags); 4630 strlcat(buf, buf2, sizeof(buf)); 4631 } 4632 if (vp->v_iflag & VI_MOUNT) 4633 strlcat(buf, "|VI_MOUNT", sizeof(buf)); 4634 if (vp->v_iflag & VI_DOINGINACT) 4635 strlcat(buf, "|VI_DOINGINACT", sizeof(buf)); 4636 if (vp->v_iflag & VI_OWEINACT) 4637 strlcat(buf, "|VI_OWEINACT", sizeof(buf)); 4638 if (vp->v_iflag & VI_DEFINACT) 4639 strlcat(buf, "|VI_DEFINACT", sizeof(buf)); 4640 if (vp->v_iflag & VI_FOPENING) 4641 strlcat(buf, "|VI_FOPENING", sizeof(buf)); 4642 flags = vp->v_iflag & ~(VI_MOUNT | VI_DOINGINACT | 4643 VI_OWEINACT | VI_DEFINACT | VI_FOPENING); 4644 if (flags != 0) { 4645 snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags); 4646 strlcat(buf, buf2, sizeof(buf)); 4647 } 4648 if (vp->v_mflag & VMP_LAZYLIST) 4649 strlcat(buf, "|VMP_LAZYLIST", sizeof(buf)); 4650 flags = vp->v_mflag & ~(VMP_LAZYLIST); 4651 if (flags != 0) { 4652 snprintf(buf2, sizeof(buf2), "|VMP(0x%lx)", flags); 4653 strlcat(buf, buf2, sizeof(buf)); 4654 } 4655 printf(" flags (%s)", buf + 1); 4656 if (mtx_owned(VI_MTX(vp))) 4657 printf(" VI_LOCKed"); 4658 printf("\n"); 4659 if (vp->v_object != NULL) 4660 printf(" v_object %p ref %d pages %d " 4661 "cleanbuf %d dirtybuf %d\n", 4662 vp->v_object, vp->v_object->ref_count, 4663 vp->v_object->resident_page_count, 4664 vp->v_bufobj.bo_clean.bv_cnt, 4665 vp->v_bufobj.bo_dirty.bv_cnt); 4666 printf(" "); 4667 lockmgr_printinfo(vp->v_vnlock); 4668 if (vp->v_data != NULL) 4669 VOP_PRINT(vp); 4670 } 4671 4672 #ifdef DDB 4673 /* 4674 * List all of the locked vnodes in the system. 4675 * Called when debugging the kernel. 4676 */ 4677 DB_SHOW_COMMAND_FLAGS(lockedvnods, lockedvnodes, DB_CMD_MEMSAFE) 4678 { 4679 struct mount *mp; 4680 struct vnode *vp; 4681 4682 /* 4683 * Note: because this is DDB, we can't obey the locking semantics 4684 * for these structures, which means we could catch an inconsistent 4685 * state and dereference a nasty pointer. Not much to be done 4686 * about that. 4687 */ 4688 db_printf("Locked vnodes\n"); 4689 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4690 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4691 if (vp->v_type != VMARKER && VOP_ISLOCKED(vp)) 4692 vn_printf(vp, "vnode "); 4693 } 4694 } 4695 } 4696 4697 /* 4698 * Show details about the given vnode. 4699 */ 4700 DB_SHOW_COMMAND(vnode, db_show_vnode) 4701 { 4702 struct vnode *vp; 4703 4704 if (!have_addr) 4705 return; 4706 vp = (struct vnode *)addr; 4707 vn_printf(vp, "vnode "); 4708 } 4709 4710 /* 4711 * Show details about the given mount point. 4712 */ 4713 DB_SHOW_COMMAND(mount, db_show_mount) 4714 { 4715 struct mount *mp; 4716 struct vfsopt *opt; 4717 struct statfs *sp; 4718 struct vnode *vp; 4719 char buf[512]; 4720 uint64_t mflags; 4721 u_int flags; 4722 4723 if (!have_addr) { 4724 /* No address given, print short info about all mount points. */ 4725 TAILQ_FOREACH(mp, &mountlist, mnt_list) { 4726 db_printf("%p %s on %s (%s)\n", mp, 4727 mp->mnt_stat.f_mntfromname, 4728 mp->mnt_stat.f_mntonname, 4729 mp->mnt_stat.f_fstypename); 4730 if (db_pager_quit) 4731 break; 4732 } 4733 db_printf("\nMore info: show mount <addr>\n"); 4734 return; 4735 } 4736 4737 mp = (struct mount *)addr; 4738 db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname, 4739 mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename); 4740 4741 buf[0] = '\0'; 4742 mflags = mp->mnt_flag; 4743 #define MNT_FLAG(flag) do { \ 4744 if (mflags & (flag)) { \ 4745 if (buf[0] != '\0') \ 4746 strlcat(buf, ", ", sizeof(buf)); \ 4747 strlcat(buf, (#flag) + 4, sizeof(buf)); \ 4748 mflags &= ~(flag); \ 4749 } \ 4750 } while (0) 4751 MNT_FLAG(MNT_RDONLY); 4752 MNT_FLAG(MNT_SYNCHRONOUS); 4753 MNT_FLAG(MNT_NOEXEC); 4754 MNT_FLAG(MNT_NOSUID); 4755 MNT_FLAG(MNT_NFS4ACLS); 4756 MNT_FLAG(MNT_UNION); 4757 MNT_FLAG(MNT_ASYNC); 4758 MNT_FLAG(MNT_SUIDDIR); 4759 MNT_FLAG(MNT_SOFTDEP); 4760 MNT_FLAG(MNT_NOSYMFOLLOW); 4761 MNT_FLAG(MNT_GJOURNAL); 4762 MNT_FLAG(MNT_MULTILABEL); 4763 MNT_FLAG(MNT_ACLS); 4764 MNT_FLAG(MNT_NOATIME); 4765 MNT_FLAG(MNT_NOCLUSTERR); 4766 MNT_FLAG(MNT_NOCLUSTERW); 4767 MNT_FLAG(MNT_SUJ); 4768 MNT_FLAG(MNT_EXRDONLY); 4769 MNT_FLAG(MNT_EXPORTED); 4770 MNT_FLAG(MNT_DEFEXPORTED); 4771 MNT_FLAG(MNT_EXPORTANON); 4772 MNT_FLAG(MNT_EXKERB); 4773 MNT_FLAG(MNT_EXPUBLIC); 4774 MNT_FLAG(MNT_LOCAL); 4775 MNT_FLAG(MNT_QUOTA); 4776 MNT_FLAG(MNT_ROOTFS); 4777 MNT_FLAG(MNT_USER); 4778 MNT_FLAG(MNT_IGNORE); 4779 MNT_FLAG(MNT_UPDATE); 4780 MNT_FLAG(MNT_DELEXPORT); 4781 MNT_FLAG(MNT_RELOAD); 4782 MNT_FLAG(MNT_FORCE); 4783 MNT_FLAG(MNT_SNAPSHOT); 4784 MNT_FLAG(MNT_BYFSID); 4785 MNT_FLAG(MNT_NAMEDATTR); 4786 #undef MNT_FLAG 4787 if (mflags != 0) { 4788 if (buf[0] != '\0') 4789 strlcat(buf, ", ", sizeof(buf)); 4790 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4791 "0x%016jx", mflags); 4792 } 4793 db_printf(" mnt_flag = %s\n", buf); 4794 4795 buf[0] = '\0'; 4796 flags = mp->mnt_kern_flag; 4797 #define MNT_KERN_FLAG(flag) do { \ 4798 if (flags & (flag)) { \ 4799 if (buf[0] != '\0') \ 4800 strlcat(buf, ", ", sizeof(buf)); \ 4801 strlcat(buf, (#flag) + 5, sizeof(buf)); \ 4802 flags &= ~(flag); \ 4803 } \ 4804 } while (0) 4805 MNT_KERN_FLAG(MNTK_UNMOUNTF); 4806 MNT_KERN_FLAG(MNTK_ASYNC); 4807 MNT_KERN_FLAG(MNTK_SOFTDEP); 4808 MNT_KERN_FLAG(MNTK_NOMSYNC); 4809 MNT_KERN_FLAG(MNTK_DRAINING); 4810 MNT_KERN_FLAG(MNTK_REFEXPIRE); 4811 MNT_KERN_FLAG(MNTK_EXTENDED_SHARED); 4812 MNT_KERN_FLAG(MNTK_SHARED_WRITES); 4813 MNT_KERN_FLAG(MNTK_NO_IOPF); 4814 MNT_KERN_FLAG(MNTK_RECURSE); 4815 MNT_KERN_FLAG(MNTK_UPPER_WAITER); 4816 MNT_KERN_FLAG(MNTK_UNLOCKED_INSMNTQUE); 4817 MNT_KERN_FLAG(MNTK_USES_BCACHE); 4818 MNT_KERN_FLAG(MNTK_VMSETSIZE_BUG); 4819 MNT_KERN_FLAG(MNTK_FPLOOKUP); 4820 MNT_KERN_FLAG(MNTK_TASKQUEUE_WAITER); 4821 MNT_KERN_FLAG(MNTK_NOASYNC); 4822 MNT_KERN_FLAG(MNTK_UNMOUNT); 4823 MNT_KERN_FLAG(MNTK_MWAIT); 4824 MNT_KERN_FLAG(MNTK_SUSPEND); 4825 MNT_KERN_FLAG(MNTK_SUSPEND2); 4826 MNT_KERN_FLAG(MNTK_SUSPENDED); 4827 MNT_KERN_FLAG(MNTK_NULL_NOCACHE); 4828 MNT_KERN_FLAG(MNTK_LOOKUP_SHARED); 4829 #undef MNT_KERN_FLAG 4830 if (flags != 0) { 4831 if (buf[0] != '\0') 4832 strlcat(buf, ", ", sizeof(buf)); 4833 snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), 4834 "0x%08x", flags); 4835 } 4836 db_printf(" mnt_kern_flag = %s\n", buf); 4837 4838 db_printf(" mnt_opt = "); 4839 opt = TAILQ_FIRST(mp->mnt_opt); 4840 if (opt != NULL) { 4841 db_printf("%s", opt->name); 4842 opt = TAILQ_NEXT(opt, link); 4843 while (opt != NULL) { 4844 db_printf(", %s", opt->name); 4845 opt = TAILQ_NEXT(opt, link); 4846 } 4847 } 4848 db_printf("\n"); 4849 4850 sp = &mp->mnt_stat; 4851 db_printf(" mnt_stat = { version=%u type=%u flags=0x%016jx " 4852 "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju " 4853 "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju " 4854 "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n", 4855 (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags, 4856 (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize, 4857 (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree, 4858 (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files, 4859 (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites, 4860 (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads, 4861 (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax, 4862 (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]); 4863 4864 db_printf(" mnt_cred = { uid=%u ruid=%u", 4865 (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid); 4866 if (jailed(mp->mnt_cred)) 4867 db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id); 4868 db_printf(" }\n"); 4869 db_printf(" mnt_ref = %d (with %d in the struct)\n", 4870 vfs_mount_fetch_counter(mp, MNT_COUNT_REF), mp->mnt_ref); 4871 db_printf(" mnt_gen = %d\n", mp->mnt_gen); 4872 db_printf(" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize); 4873 db_printf(" mnt_lazyvnodelistsize = %d\n", 4874 mp->mnt_lazyvnodelistsize); 4875 db_printf(" mnt_writeopcount = %d (with %d in the struct)\n", 4876 vfs_mount_fetch_counter(mp, MNT_COUNT_WRITEOPCOUNT), mp->mnt_writeopcount); 4877 db_printf(" mnt_iosize_max = %d\n", mp->mnt_iosize_max); 4878 db_printf(" mnt_hashseed = %u\n", mp->mnt_hashseed); 4879 db_printf(" mnt_lockref = %d (with %d in the struct)\n", 4880 vfs_mount_fetch_counter(mp, MNT_COUNT_LOCKREF), mp->mnt_lockref); 4881 db_printf(" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes); 4882 db_printf(" mnt_secondary_accwrites = %d\n", 4883 mp->mnt_secondary_accwrites); 4884 db_printf(" mnt_gjprovider = %s\n", 4885 mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL"); 4886 db_printf(" mnt_vfs_ops = %d\n", mp->mnt_vfs_ops); 4887 4888 db_printf("\n\nList of active vnodes\n"); 4889 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4890 if (vp->v_type != VMARKER && vp->v_holdcnt > 0) { 4891 vn_printf(vp, "vnode "); 4892 if (db_pager_quit) 4893 break; 4894 } 4895 } 4896 db_printf("\n\nList of inactive vnodes\n"); 4897 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 4898 if (vp->v_type != VMARKER && vp->v_holdcnt == 0) { 4899 vn_printf(vp, "vnode "); 4900 if (db_pager_quit) 4901 break; 4902 } 4903 } 4904 } 4905 #endif /* DDB */ 4906 4907 /* 4908 * Fill in a struct xvfsconf based on a struct vfsconf. 4909 */ 4910 static int 4911 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp) 4912 { 4913 struct xvfsconf xvfsp; 4914 4915 bzero(&xvfsp, sizeof(xvfsp)); 4916 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4917 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4918 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4919 xvfsp.vfc_flags = vfsp->vfc_flags; 4920 /* 4921 * These are unused in userland, we keep them 4922 * to not break binary compatibility. 4923 */ 4924 xvfsp.vfc_vfsops = NULL; 4925 xvfsp.vfc_next = NULL; 4926 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4927 } 4928 4929 #ifdef COMPAT_FREEBSD32 4930 struct xvfsconf32 { 4931 uint32_t vfc_vfsops; 4932 char vfc_name[MFSNAMELEN]; 4933 int32_t vfc_typenum; 4934 int32_t vfc_refcount; 4935 int32_t vfc_flags; 4936 uint32_t vfc_next; 4937 }; 4938 4939 static int 4940 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp) 4941 { 4942 struct xvfsconf32 xvfsp; 4943 4944 bzero(&xvfsp, sizeof(xvfsp)); 4945 strcpy(xvfsp.vfc_name, vfsp->vfc_name); 4946 xvfsp.vfc_typenum = vfsp->vfc_typenum; 4947 xvfsp.vfc_refcount = vfsp->vfc_refcount; 4948 xvfsp.vfc_flags = vfsp->vfc_flags; 4949 return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp))); 4950 } 4951 #endif 4952 4953 /* 4954 * Top level filesystem related information gathering. 4955 */ 4956 static int 4957 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS) 4958 { 4959 struct vfsconf *vfsp; 4960 int error; 4961 4962 error = 0; 4963 vfsconf_slock(); 4964 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 4965 #ifdef COMPAT_FREEBSD32 4966 if (req->flags & SCTL_MASK32) 4967 error = vfsconf2x32(req, vfsp); 4968 else 4969 #endif 4970 error = vfsconf2x(req, vfsp); 4971 if (error) 4972 break; 4973 } 4974 vfsconf_sunlock(); 4975 return (error); 4976 } 4977 4978 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD | 4979 CTLFLAG_MPSAFE, NULL, 0, sysctl_vfs_conflist, 4980 "S,xvfsconf", "List of all configured filesystems"); 4981 4982 #ifndef BURN_BRIDGES 4983 static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS); 4984 4985 static int 4986 vfs_sysctl(SYSCTL_HANDLER_ARGS) 4987 { 4988 int *name = (int *)arg1 - 1; /* XXX */ 4989 u_int namelen = arg2 + 1; /* XXX */ 4990 struct vfsconf *vfsp; 4991 4992 log(LOG_WARNING, "userland calling deprecated sysctl, " 4993 "please rebuild world\n"); 4994 4995 #if 1 || defined(COMPAT_PRELITE2) 4996 /* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */ 4997 if (namelen == 1) 4998 return (sysctl_ovfs_conf(oidp, arg1, arg2, req)); 4999 #endif 5000 5001 switch (name[1]) { 5002 case VFS_MAXTYPENUM: 5003 if (namelen != 2) 5004 return (ENOTDIR); 5005 return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int))); 5006 case VFS_CONF: 5007 if (namelen != 3) 5008 return (ENOTDIR); /* overloaded */ 5009 vfsconf_slock(); 5010 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 5011 if (vfsp->vfc_typenum == name[2]) 5012 break; 5013 } 5014 vfsconf_sunlock(); 5015 if (vfsp == NULL) 5016 return (EOPNOTSUPP); 5017 #ifdef COMPAT_FREEBSD32 5018 if (req->flags & SCTL_MASK32) 5019 return (vfsconf2x32(req, vfsp)); 5020 else 5021 #endif 5022 return (vfsconf2x(req, vfsp)); 5023 } 5024 return (EOPNOTSUPP); 5025 } 5026 5027 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP | 5028 CTLFLAG_MPSAFE, vfs_sysctl, 5029 "Generic filesystem"); 5030 5031 #if 1 || defined(COMPAT_PRELITE2) 5032 5033 static int 5034 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS) 5035 { 5036 int error; 5037 struct vfsconf *vfsp; 5038 struct ovfsconf ovfs; 5039 5040 vfsconf_slock(); 5041 TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) { 5042 bzero(&ovfs, sizeof(ovfs)); 5043 ovfs.vfc_vfsops = vfsp->vfc_vfsops; /* XXX used as flag */ 5044 strcpy(ovfs.vfc_name, vfsp->vfc_name); 5045 ovfs.vfc_index = vfsp->vfc_typenum; 5046 ovfs.vfc_refcount = vfsp->vfc_refcount; 5047 ovfs.vfc_flags = vfsp->vfc_flags; 5048 error = SYSCTL_OUT(req, &ovfs, sizeof ovfs); 5049 if (error != 0) { 5050 vfsconf_sunlock(); 5051 return (error); 5052 } 5053 } 5054 vfsconf_sunlock(); 5055 return (0); 5056 } 5057 5058 #endif /* 1 || COMPAT_PRELITE2 */ 5059 #endif /* !BURN_BRIDGES */ 5060 5061 static void 5062 unmount_or_warn(struct mount *mp) 5063 { 5064 int error; 5065 5066 error = dounmount(mp, MNT_FORCE, curthread); 5067 if (error != 0) { 5068 printf("unmount of %s failed (", mp->mnt_stat.f_mntonname); 5069 if (error == EBUSY) 5070 printf("BUSY)\n"); 5071 else 5072 printf("%d)\n", error); 5073 } 5074 } 5075 5076 /* 5077 * Unmount all filesystems. The list is traversed in reverse order 5078 * of mounting to avoid dependencies. 5079 */ 5080 void 5081 vfs_unmountall(void) 5082 { 5083 struct mount *mp, *tmp; 5084 5085 CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__); 5086 5087 /* 5088 * Since this only runs when rebooting, it is not interlocked. 5089 */ 5090 TAILQ_FOREACH_REVERSE_SAFE(mp, &mountlist, mntlist, mnt_list, tmp) { 5091 vfs_ref(mp); 5092 5093 /* 5094 * Forcibly unmounting "/dev" before "/" would prevent clean 5095 * unmount of the latter. 5096 */ 5097 if (mp == rootdevmp) 5098 continue; 5099 5100 unmount_or_warn(mp); 5101 } 5102 5103 if (rootdevmp != NULL) 5104 unmount_or_warn(rootdevmp); 5105 } 5106 5107 static void 5108 vfs_deferred_inactive(struct vnode *vp, int lkflags) 5109 { 5110 5111 ASSERT_VI_LOCKED(vp, __func__); 5112 VNPASS((vp->v_iflag & VI_DEFINACT) == 0, vp); 5113 if ((vp->v_iflag & VI_OWEINACT) == 0) { 5114 vdropl(vp); 5115 return; 5116 } 5117 if (vn_lock(vp, lkflags) == 0) { 5118 VI_LOCK(vp); 5119 vinactive(vp); 5120 VOP_UNLOCK(vp); 5121 vdropl(vp); 5122 return; 5123 } 5124 vdefer_inactive_unlocked(vp); 5125 } 5126 5127 static int 5128 vfs_periodic_inactive_filter(struct vnode *vp, void *arg) 5129 { 5130 5131 return (vp->v_iflag & VI_DEFINACT); 5132 } 5133 5134 static void __noinline 5135 vfs_periodic_inactive(struct mount *mp, int flags) 5136 { 5137 struct vnode *vp, *mvp; 5138 int lkflags; 5139 5140 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 5141 if (flags != MNT_WAIT) 5142 lkflags |= LK_NOWAIT; 5143 5144 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_inactive_filter, NULL) { 5145 if ((vp->v_iflag & VI_DEFINACT) == 0) { 5146 VI_UNLOCK(vp); 5147 continue; 5148 } 5149 vp->v_iflag &= ~VI_DEFINACT; 5150 vfs_deferred_inactive(vp, lkflags); 5151 } 5152 } 5153 5154 static inline bool 5155 vfs_want_msync(struct vnode *vp) 5156 { 5157 struct vm_object *obj; 5158 5159 /* 5160 * This test may be performed without any locks held. 5161 * We rely on vm_object's type stability. 5162 */ 5163 if (vp->v_vflag & VV_NOSYNC) 5164 return (false); 5165 obj = vp->v_object; 5166 return (obj != NULL && vm_object_mightbedirty(obj)); 5167 } 5168 5169 static int 5170 vfs_periodic_msync_inactive_filter(struct vnode *vp, void *arg __unused) 5171 { 5172 5173 if (vp->v_vflag & VV_NOSYNC) 5174 return (false); 5175 if (vp->v_iflag & VI_DEFINACT) 5176 return (true); 5177 return (vfs_want_msync(vp)); 5178 } 5179 5180 static void __noinline 5181 vfs_periodic_msync_inactive(struct mount *mp, int flags) 5182 { 5183 struct vnode *vp, *mvp; 5184 int lkflags; 5185 bool seen_defer; 5186 5187 lkflags = LK_EXCLUSIVE | LK_INTERLOCK; 5188 if (flags != MNT_WAIT) 5189 lkflags |= LK_NOWAIT; 5190 5191 MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_msync_inactive_filter, NULL) { 5192 seen_defer = false; 5193 if (vp->v_iflag & VI_DEFINACT) { 5194 vp->v_iflag &= ~VI_DEFINACT; 5195 seen_defer = true; 5196 } 5197 if (!vfs_want_msync(vp)) { 5198 if (seen_defer) 5199 vfs_deferred_inactive(vp, lkflags); 5200 else 5201 VI_UNLOCK(vp); 5202 continue; 5203 } 5204 if (vget(vp, lkflags) == 0) { 5205 if ((vp->v_vflag & VV_NOSYNC) == 0) { 5206 if (flags == MNT_WAIT) 5207 vnode_pager_clean_sync(vp); 5208 else 5209 vnode_pager_clean_async(vp); 5210 } 5211 vput(vp); 5212 if (seen_defer) 5213 vdrop(vp); 5214 } else { 5215 if (seen_defer) 5216 vdefer_inactive_unlocked(vp); 5217 } 5218 } 5219 } 5220 5221 void 5222 vfs_periodic(struct mount *mp, int flags) 5223 { 5224 5225 CTR2(KTR_VFS, "%s: mp %p", __func__, mp); 5226 5227 if ((mp->mnt_kern_flag & MNTK_NOMSYNC) != 0) 5228 vfs_periodic_inactive(mp, flags); 5229 else 5230 vfs_periodic_msync_inactive(mp, flags); 5231 } 5232 5233 static void 5234 destroy_vpollinfo_free(struct vpollinfo *vi) 5235 { 5236 5237 knlist_destroy(&vi->vpi_selinfo.si_note); 5238 mtx_destroy(&vi->vpi_lock); 5239 free(vi, M_VNODEPOLL); 5240 } 5241 5242 static void 5243 destroy_vpollinfo(struct vpollinfo *vi) 5244 { 5245 KASSERT(TAILQ_EMPTY(&vi->vpi_inotify), 5246 ("%s: pollinfo %p has lingering watches", __func__, vi)); 5247 knlist_clear(&vi->vpi_selinfo.si_note, 1); 5248 seldrain(&vi->vpi_selinfo); 5249 destroy_vpollinfo_free(vi); 5250 } 5251 5252 /* 5253 * Initialize per-vnode helper structure to hold poll-related state. 5254 */ 5255 void 5256 v_addpollinfo(struct vnode *vp) 5257 { 5258 struct vpollinfo *vi; 5259 5260 if (atomic_load_ptr(&vp->v_pollinfo) != NULL) 5261 return; 5262 vi = malloc(sizeof(*vi), M_VNODEPOLL, M_WAITOK | M_ZERO); 5263 mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF); 5264 knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock, 5265 vfs_knlunlock, vfs_knl_assert_lock); 5266 TAILQ_INIT(&vi->vpi_inotify); 5267 VI_LOCK(vp); 5268 if (vp->v_pollinfo != NULL) { 5269 VI_UNLOCK(vp); 5270 destroy_vpollinfo_free(vi); 5271 return; 5272 } 5273 vp->v_pollinfo = vi; 5274 VI_UNLOCK(vp); 5275 } 5276 5277 /* 5278 * Record a process's interest in events which might happen to 5279 * a vnode. Because poll uses the historic select-style interface 5280 * internally, this routine serves as both the ``check for any 5281 * pending events'' and the ``record my interest in future events'' 5282 * functions. (These are done together, while the lock is held, 5283 * to avoid race conditions.) 5284 */ 5285 int 5286 vn_pollrecord(struct vnode *vp, struct thread *td, int events) 5287 { 5288 5289 v_addpollinfo(vp); 5290 mtx_lock(&vp->v_pollinfo->vpi_lock); 5291 if (vp->v_pollinfo->vpi_revents & events) { 5292 /* 5293 * This leaves events we are not interested 5294 * in available for the other process which 5295 * which presumably had requested them 5296 * (otherwise they would never have been 5297 * recorded). 5298 */ 5299 events &= vp->v_pollinfo->vpi_revents; 5300 vp->v_pollinfo->vpi_revents &= ~events; 5301 5302 mtx_unlock(&vp->v_pollinfo->vpi_lock); 5303 return (events); 5304 } 5305 vp->v_pollinfo->vpi_events |= events; 5306 selrecord(td, &vp->v_pollinfo->vpi_selinfo); 5307 mtx_unlock(&vp->v_pollinfo->vpi_lock); 5308 return (0); 5309 } 5310 5311 /* 5312 * Routine to create and manage a filesystem syncer vnode. 5313 */ 5314 #define sync_close ((int (*)(struct vop_close_args *))nullop) 5315 static int sync_fsync(struct vop_fsync_args *); 5316 static int sync_inactive(struct vop_inactive_args *); 5317 static int sync_reclaim(struct vop_reclaim_args *); 5318 5319 static struct vop_vector sync_vnodeops = { 5320 .vop_bypass = VOP_EOPNOTSUPP, 5321 .vop_close = sync_close, 5322 .vop_fsync = sync_fsync, 5323 .vop_getwritemount = vop_stdgetwritemount, 5324 .vop_inactive = sync_inactive, 5325 .vop_need_inactive = vop_stdneed_inactive, 5326 .vop_reclaim = sync_reclaim, 5327 .vop_lock1 = vop_stdlock, 5328 .vop_unlock = vop_stdunlock, 5329 .vop_islocked = vop_stdislocked, 5330 .vop_fplookup_vexec = VOP_EAGAIN, 5331 .vop_fplookup_symlink = VOP_EAGAIN, 5332 }; 5333 VFS_VOP_VECTOR_REGISTER(sync_vnodeops); 5334 5335 /* 5336 * Create a new filesystem syncer vnode for the specified mount point. 5337 */ 5338 void 5339 vfs_allocate_syncvnode(struct mount *mp) 5340 { 5341 struct vnode *vp; 5342 struct bufobj *bo; 5343 static long start, incr, next; 5344 int error; 5345 5346 /* Allocate a new vnode */ 5347 error = getnewvnode("syncer", mp, &sync_vnodeops, &vp); 5348 if (error != 0) 5349 panic("vfs_allocate_syncvnode: getnewvnode() failed"); 5350 vp->v_type = VNON; 5351 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5352 vp->v_vflag |= VV_FORCEINSMQ; 5353 error = insmntque1(vp, mp); 5354 if (error != 0) 5355 panic("vfs_allocate_syncvnode: insmntque() failed"); 5356 vp->v_vflag &= ~VV_FORCEINSMQ; 5357 vn_set_state(vp, VSTATE_CONSTRUCTED); 5358 VOP_UNLOCK(vp); 5359 /* 5360 * Place the vnode onto the syncer worklist. We attempt to 5361 * scatter them about on the list so that they will go off 5362 * at evenly distributed times even if all the filesystems 5363 * are mounted at once. 5364 */ 5365 next += incr; 5366 if (next == 0 || next > syncer_maxdelay) { 5367 start /= 2; 5368 incr /= 2; 5369 if (start == 0) { 5370 start = syncer_maxdelay / 2; 5371 incr = syncer_maxdelay; 5372 } 5373 next = start; 5374 } 5375 bo = &vp->v_bufobj; 5376 BO_LOCK(bo); 5377 vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0); 5378 /* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */ 5379 mtx_lock(&sync_mtx); 5380 sync_vnode_count++; 5381 if (mp->mnt_syncer == NULL) { 5382 mp->mnt_syncer = vp; 5383 vp = NULL; 5384 } 5385 mtx_unlock(&sync_mtx); 5386 BO_UNLOCK(bo); 5387 if (vp != NULL) { 5388 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 5389 vgone(vp); 5390 vput(vp); 5391 } 5392 } 5393 5394 void 5395 vfs_deallocate_syncvnode(struct mount *mp) 5396 { 5397 struct vnode *vp; 5398 5399 mtx_lock(&sync_mtx); 5400 vp = mp->mnt_syncer; 5401 if (vp != NULL) 5402 mp->mnt_syncer = NULL; 5403 mtx_unlock(&sync_mtx); 5404 if (vp != NULL) 5405 vrele(vp); 5406 } 5407 5408 /* 5409 * Do a lazy sync of the filesystem. 5410 */ 5411 static int 5412 sync_fsync(struct vop_fsync_args *ap) 5413 { 5414 struct vnode *syncvp = ap->a_vp; 5415 struct mount *mp = syncvp->v_mount; 5416 int error, save; 5417 struct bufobj *bo; 5418 5419 /* 5420 * We only need to do something if this is a lazy evaluation. 5421 */ 5422 if (ap->a_waitfor != MNT_LAZY) 5423 return (0); 5424 5425 /* 5426 * Move ourselves to the back of the sync list. 5427 */ 5428 bo = &syncvp->v_bufobj; 5429 BO_LOCK(bo); 5430 vn_syncer_add_to_worklist(bo, syncdelay); 5431 BO_UNLOCK(bo); 5432 5433 /* 5434 * Walk the list of vnodes pushing all that are dirty and 5435 * not already on the sync list. 5436 */ 5437 if (vfs_busy(mp, MBF_NOWAIT) != 0) 5438 return (0); 5439 VOP_UNLOCK(syncvp); 5440 save = curthread_pflags_set(TDP_SYNCIO); 5441 /* 5442 * The filesystem at hand may be idle with free vnodes stored in the 5443 * batch. Return them instead of letting them stay there indefinitely. 5444 */ 5445 vfs_periodic(mp, MNT_NOWAIT); 5446 error = VFS_SYNC(mp, MNT_LAZY); 5447 curthread_pflags_restore(save); 5448 vn_lock(syncvp, LK_EXCLUSIVE | LK_RETRY); 5449 vfs_unbusy(mp); 5450 return (error); 5451 } 5452 5453 /* 5454 * The syncer vnode is no referenced. 5455 */ 5456 static int 5457 sync_inactive(struct vop_inactive_args *ap) 5458 { 5459 5460 vgone(ap->a_vp); 5461 return (0); 5462 } 5463 5464 /* 5465 * The syncer vnode is no longer needed and is being decommissioned. 5466 * 5467 * Modifications to the worklist must be protected by sync_mtx. 5468 */ 5469 static int 5470 sync_reclaim(struct vop_reclaim_args *ap) 5471 { 5472 struct vnode *vp = ap->a_vp; 5473 struct bufobj *bo; 5474 5475 bo = &vp->v_bufobj; 5476 BO_LOCK(bo); 5477 mtx_lock(&sync_mtx); 5478 if (vp->v_mount->mnt_syncer == vp) 5479 vp->v_mount->mnt_syncer = NULL; 5480 if (bo->bo_flag & BO_ONWORKLST) { 5481 LIST_REMOVE(bo, bo_synclist); 5482 syncer_worklist_len--; 5483 sync_vnode_count--; 5484 bo->bo_flag &= ~BO_ONWORKLST; 5485 } 5486 mtx_unlock(&sync_mtx); 5487 BO_UNLOCK(bo); 5488 5489 return (0); 5490 } 5491 5492 int 5493 vn_need_pageq_flush(struct vnode *vp) 5494 { 5495 struct vm_object *obj; 5496 5497 obj = vp->v_object; 5498 return (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && 5499 vm_object_mightbedirty(obj)); 5500 } 5501 5502 /* 5503 * Check if vnode represents a disk device 5504 */ 5505 bool 5506 vn_isdisk_error(struct vnode *vp, int *errp) 5507 { 5508 int error; 5509 5510 if (vp->v_type != VCHR) { 5511 error = ENOTBLK; 5512 goto out; 5513 } 5514 error = 0; 5515 dev_lock(); 5516 if (vp->v_rdev == NULL) 5517 error = ENXIO; 5518 else if (vp->v_rdev->si_devsw == NULL) 5519 error = ENXIO; 5520 else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK)) 5521 error = ENOTBLK; 5522 dev_unlock(); 5523 out: 5524 *errp = error; 5525 return (error == 0); 5526 } 5527 5528 bool 5529 vn_isdisk(struct vnode *vp) 5530 { 5531 int error; 5532 5533 return (vn_isdisk_error(vp, &error)); 5534 } 5535 5536 /* 5537 * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see 5538 * the comment above cache_fplookup for details. 5539 */ 5540 int 5541 vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid, struct ucred *cred) 5542 { 5543 int error; 5544 5545 VFS_SMR_ASSERT_ENTERED(); 5546 5547 /* Check the owner. */ 5548 if (cred->cr_uid == file_uid) { 5549 if (file_mode & S_IXUSR) 5550 return (0); 5551 goto out_error; 5552 } 5553 5554 /* Otherwise, check the groups (first match) */ 5555 if (groupmember(file_gid, cred)) { 5556 if (file_mode & S_IXGRP) 5557 return (0); 5558 goto out_error; 5559 } 5560 5561 /* Otherwise, check everyone else. */ 5562 if (file_mode & S_IXOTH) 5563 return (0); 5564 out_error: 5565 /* 5566 * Permission check failed, but it is possible denial will get overwritten 5567 * (e.g., when root is traversing through a 700 directory owned by someone 5568 * else). 5569 * 5570 * vaccess() calls priv_check_cred which in turn can descent into MAC 5571 * modules overriding this result. It's quite unclear what semantics 5572 * are allowed for them to operate, thus for safety we don't call them 5573 * from within the SMR section. This also means if any such modules 5574 * are present, we have to let the regular lookup decide. 5575 */ 5576 error = priv_check_cred_vfs_lookup_nomac(cred); 5577 switch (error) { 5578 case 0: 5579 return (0); 5580 case EAGAIN: 5581 /* 5582 * MAC modules present. 5583 */ 5584 return (EAGAIN); 5585 case EPERM: 5586 return (EACCES); 5587 default: 5588 return (error); 5589 } 5590 } 5591 5592 /* 5593 * Common filesystem object access control check routine. Accepts a 5594 * vnode's type, "mode", uid and gid, requested access mode, and credentials. 5595 * Returns 0 on success, or an errno on failure. 5596 */ 5597 int 5598 vaccess(__enum_uint8(vtype) type, mode_t file_mode, uid_t file_uid, gid_t file_gid, 5599 accmode_t accmode, struct ucred *cred) 5600 { 5601 accmode_t dac_granted; 5602 accmode_t priv_granted; 5603 5604 KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0, 5605 ("invalid bit in accmode")); 5606 KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE), 5607 ("VAPPEND without VWRITE")); 5608 5609 /* 5610 * Look for a normal, non-privileged way to access the file/directory 5611 * as requested. If it exists, go with that. 5612 */ 5613 5614 dac_granted = 0; 5615 5616 /* Check the owner. */ 5617 if (cred->cr_uid == file_uid) { 5618 dac_granted |= VADMIN; 5619 if (file_mode & S_IXUSR) 5620 dac_granted |= VEXEC; 5621 if (file_mode & S_IRUSR) 5622 dac_granted |= VREAD; 5623 if (file_mode & S_IWUSR) 5624 dac_granted |= (VWRITE | VAPPEND); 5625 5626 if ((accmode & dac_granted) == accmode) 5627 return (0); 5628 5629 goto privcheck; 5630 } 5631 5632 /* Otherwise, check the groups (first match) */ 5633 if (groupmember(file_gid, cred)) { 5634 if (file_mode & S_IXGRP) 5635 dac_granted |= VEXEC; 5636 if (file_mode & S_IRGRP) 5637 dac_granted |= VREAD; 5638 if (file_mode & S_IWGRP) 5639 dac_granted |= (VWRITE | VAPPEND); 5640 5641 if ((accmode & dac_granted) == accmode) 5642 return (0); 5643 5644 goto privcheck; 5645 } 5646 5647 /* Otherwise, check everyone else. */ 5648 if (file_mode & S_IXOTH) 5649 dac_granted |= VEXEC; 5650 if (file_mode & S_IROTH) 5651 dac_granted |= VREAD; 5652 if (file_mode & S_IWOTH) 5653 dac_granted |= (VWRITE | VAPPEND); 5654 if ((accmode & dac_granted) == accmode) 5655 return (0); 5656 5657 privcheck: 5658 /* 5659 * Build a privilege mask to determine if the set of privileges 5660 * satisfies the requirements when combined with the granted mask 5661 * from above. For each privilege, if the privilege is required, 5662 * bitwise or the request type onto the priv_granted mask. 5663 */ 5664 priv_granted = 0; 5665 5666 if (type == VDIR) { 5667 /* 5668 * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC 5669 * requests, instead of PRIV_VFS_EXEC. 5670 */ 5671 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5672 !priv_check_cred(cred, PRIV_VFS_LOOKUP)) 5673 priv_granted |= VEXEC; 5674 } else { 5675 /* 5676 * Ensure that at least one execute bit is on. Otherwise, 5677 * a privileged user will always succeed, and we don't want 5678 * this to happen unless the file really is executable. 5679 */ 5680 if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) && 5681 (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && 5682 !priv_check_cred(cred, PRIV_VFS_EXEC)) 5683 priv_granted |= VEXEC; 5684 } 5685 5686 if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) && 5687 !priv_check_cred(cred, PRIV_VFS_READ)) 5688 priv_granted |= VREAD; 5689 5690 if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) && 5691 !priv_check_cred(cred, PRIV_VFS_WRITE)) 5692 priv_granted |= (VWRITE | VAPPEND); 5693 5694 if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) && 5695 !priv_check_cred(cred, PRIV_VFS_ADMIN)) 5696 priv_granted |= VADMIN; 5697 5698 if ((accmode & (priv_granted | dac_granted)) == accmode) { 5699 return (0); 5700 } 5701 5702 return ((accmode & VADMIN) ? EPERM : EACCES); 5703 } 5704 5705 /* 5706 * Credential check based on process requesting service, and per-attribute 5707 * permissions. 5708 */ 5709 int 5710 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred, 5711 struct thread *td, accmode_t accmode) 5712 { 5713 5714 /* 5715 * Kernel-invoked always succeeds. 5716 */ 5717 if (cred == NOCRED) 5718 return (0); 5719 5720 /* 5721 * Do not allow privileged processes in jail to directly manipulate 5722 * system attributes. 5723 */ 5724 switch (attrnamespace) { 5725 case EXTATTR_NAMESPACE_SYSTEM: 5726 /* Potentially should be: return (EPERM); */ 5727 return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM)); 5728 case EXTATTR_NAMESPACE_USER: 5729 return (VOP_ACCESS(vp, accmode, cred, td)); 5730 default: 5731 return (EPERM); 5732 } 5733 } 5734 5735 #ifdef INVARIANTS 5736 void 5737 assert_vi_locked(struct vnode *vp, const char *str) 5738 { 5739 VNASSERT(mtx_owned(VI_MTX(vp)), vp, 5740 ("%s: vnode interlock is not locked but should be", str)); 5741 } 5742 5743 void 5744 assert_vi_unlocked(struct vnode *vp, const char *str) 5745 { 5746 VNASSERT(!mtx_owned(VI_MTX(vp)), vp, 5747 ("%s: vnode interlock is locked but should not be", str)); 5748 } 5749 5750 void 5751 assert_vop_locked(struct vnode *vp, const char *str) 5752 { 5753 bool locked; 5754 5755 if (KERNEL_PANICKED() || vp == NULL) 5756 return; 5757 5758 #ifdef WITNESS 5759 locked = !((vp->v_irflag & VIRF_CROSSMP) == 0 && 5760 witness_is_owned(&vp->v_vnlock->lock_object) == -1); 5761 #else 5762 int state = VOP_ISLOCKED(vp); 5763 locked = state != 0 && state != LK_EXCLOTHER; 5764 #endif 5765 VNASSERT(locked, vp, ("%s: vnode is not locked but should be", str)); 5766 } 5767 5768 void 5769 assert_vop_unlocked(struct vnode *vp, const char *str) 5770 { 5771 bool locked; 5772 5773 if (KERNEL_PANICKED() || vp == NULL) 5774 return; 5775 5776 #ifdef WITNESS 5777 locked = (vp->v_irflag & VIRF_CROSSMP) == 0 && 5778 witness_is_owned(&vp->v_vnlock->lock_object) == 1; 5779 #else 5780 locked = VOP_ISLOCKED(vp) == LK_EXCLUSIVE; 5781 #endif 5782 VNASSERT(!locked, vp, ("%s: vnode is locked but should not be", str)); 5783 } 5784 5785 void 5786 assert_vop_elocked(struct vnode *vp, const char *str) 5787 { 5788 bool locked; 5789 5790 if (KERNEL_PANICKED() || vp == NULL) 5791 return; 5792 5793 locked = VOP_ISLOCKED(vp) == LK_EXCLUSIVE; 5794 VNASSERT(locked, vp, 5795 ("%s: vnode is not exclusive locked but should be", str)); 5796 } 5797 #endif /* INVARIANTS */ 5798 5799 void 5800 vop_rename_fail(struct vop_rename_args *ap) 5801 { 5802 5803 if (ap->a_tvp != NULL) 5804 vput(ap->a_tvp); 5805 if (ap->a_tdvp == ap->a_tvp) 5806 vrele(ap->a_tdvp); 5807 else 5808 vput(ap->a_tdvp); 5809 vrele(ap->a_fdvp); 5810 vrele(ap->a_fvp); 5811 } 5812 5813 void 5814 vop_rename_pre(void *ap) 5815 { 5816 struct vop_rename_args *a = ap; 5817 5818 #ifdef INVARIANTS 5819 struct mount *tmp; 5820 5821 if (a->a_tvp) 5822 ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME"); 5823 ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME"); 5824 ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME"); 5825 ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME"); 5826 5827 /* Check the source (from). */ 5828 if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock && 5829 (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock)) 5830 ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked"); 5831 if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock) 5832 ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked"); 5833 5834 /* Check the target. */ 5835 if (a->a_tvp) 5836 ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked"); 5837 ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked"); 5838 5839 tmp = NULL; 5840 VOP_GETWRITEMOUNT(a->a_tdvp, &tmp); 5841 lockmgr_assert(&tmp->mnt_renamelock, KA_XLOCKED); 5842 vfs_rel(tmp); 5843 #endif 5844 /* 5845 * It may be tempting to add vn_seqc_write_begin/end calls here and 5846 * in vop_rename_post but that's not going to work out since some 5847 * filesystems relookup vnodes mid-rename. This is probably a bug. 5848 * 5849 * For now filesystems are expected to do the relevant calls after they 5850 * decide what vnodes to operate on. 5851 */ 5852 if (a->a_tdvp != a->a_fdvp) 5853 vhold(a->a_fdvp); 5854 if (a->a_tvp != a->a_fvp) 5855 vhold(a->a_fvp); 5856 vhold(a->a_tdvp); 5857 if (a->a_tvp) 5858 vhold(a->a_tvp); 5859 } 5860 5861 #ifdef INVARIANTS 5862 void 5863 vop_fplookup_vexec_debugpre(void *ap __unused) 5864 { 5865 5866 VFS_SMR_ASSERT_ENTERED(); 5867 } 5868 5869 void 5870 vop_fplookup_vexec_debugpost(void *ap, int rc) 5871 { 5872 struct vop_fplookup_vexec_args *a; 5873 struct vnode *vp; 5874 5875 a = ap; 5876 vp = a->a_vp; 5877 5878 VFS_SMR_ASSERT_ENTERED(); 5879 if (rc == EOPNOTSUPP) 5880 VNPASS(VN_IS_DOOMED(vp), vp); 5881 } 5882 5883 void 5884 vop_fplookup_symlink_debugpre(void *ap __unused) 5885 { 5886 5887 VFS_SMR_ASSERT_ENTERED(); 5888 } 5889 5890 void 5891 vop_fplookup_symlink_debugpost(void *ap __unused, int rc __unused) 5892 { 5893 5894 VFS_SMR_ASSERT_ENTERED(); 5895 } 5896 5897 static void 5898 vop_fsync_debugprepost(struct vnode *vp, const char *name) 5899 { 5900 struct mount *mp; 5901 5902 if (vp->v_type == VCHR) 5903 ; 5904 /* 5905 * The shared vs. exclusive locking policy for fsync() 5906 * is actually determined by vp's write mount as indicated 5907 * by VOP_GETWRITEMOUNT(), which for stacked filesystems 5908 * may not be the same as vp->v_mount. However, if the 5909 * underlying filesystem which really handles the fsync() 5910 * supports shared locking, the stacked filesystem must also 5911 * be prepared for its VOP_FSYNC() operation to be called 5912 * with only a shared lock. On the other hand, if the 5913 * stacked filesystem claims support for shared write 5914 * locking but the underlying filesystem does not, and the 5915 * caller incorrectly uses a shared lock, this condition 5916 * should still be caught when the stacked filesystem 5917 * invokes VOP_FSYNC() on the underlying filesystem. 5918 */ 5919 else { 5920 mp = NULL; 5921 VOP_GETWRITEMOUNT(vp, &mp); 5922 if (vn_lktype_write(mp, vp) == LK_SHARED) 5923 ASSERT_VOP_LOCKED(vp, name); 5924 else 5925 ASSERT_VOP_ELOCKED(vp, name); 5926 if (mp != NULL) 5927 vfs_rel(mp); 5928 } 5929 } 5930 5931 void 5932 vop_fsync_debugpre(void *a) 5933 { 5934 struct vop_fsync_args *ap; 5935 5936 ap = a; 5937 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5938 } 5939 5940 void 5941 vop_fsync_debugpost(void *a, int rc __unused) 5942 { 5943 struct vop_fsync_args *ap; 5944 5945 ap = a; 5946 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5947 } 5948 5949 void 5950 vop_fdatasync_debugpre(void *a) 5951 { 5952 struct vop_fdatasync_args *ap; 5953 5954 ap = a; 5955 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5956 } 5957 5958 void 5959 vop_fdatasync_debugpost(void *a, int rc __unused) 5960 { 5961 struct vop_fdatasync_args *ap; 5962 5963 ap = a; 5964 vop_fsync_debugprepost(ap->a_vp, "fsync"); 5965 } 5966 5967 void 5968 vop_strategy_debugpre(void *ap) 5969 { 5970 struct vop_strategy_args *a; 5971 struct buf *bp; 5972 5973 a = ap; 5974 bp = a->a_bp; 5975 5976 /* 5977 * Cluster ops lock their component buffers but not the IO container. 5978 */ 5979 if ((bp->b_flags & B_CLUSTER) != 0) 5980 return; 5981 5982 BUF_ASSERT_LOCKED(bp); 5983 } 5984 5985 void 5986 vop_lock_debugpre(void *ap) 5987 { 5988 struct vop_lock1_args *a = ap; 5989 5990 if ((a->a_flags & LK_INTERLOCK) == 0) 5991 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 5992 else 5993 ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK"); 5994 } 5995 5996 void 5997 vop_lock_debugpost(void *ap, int rc) 5998 { 5999 struct vop_lock1_args *a = ap; 6000 6001 ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK"); 6002 if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0) 6003 ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK"); 6004 } 6005 6006 void 6007 vop_unlock_debugpre(void *ap) 6008 { 6009 struct vop_unlock_args *a = ap; 6010 struct vnode *vp = a->a_vp; 6011 6012 VNPASS(vn_get_state(vp) != VSTATE_UNINITIALIZED, vp); 6013 ASSERT_VOP_LOCKED(vp, "VOP_UNLOCK"); 6014 } 6015 6016 void 6017 vop_need_inactive_debugpre(void *ap) 6018 { 6019 struct vop_need_inactive_args *a = ap; 6020 6021 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 6022 } 6023 6024 void 6025 vop_need_inactive_debugpost(void *ap, int rc) 6026 { 6027 struct vop_need_inactive_args *a = ap; 6028 6029 ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE"); 6030 } 6031 #endif /* INVARIANTS */ 6032 6033 void 6034 vop_allocate_post(void *ap, int rc) 6035 { 6036 struct vop_allocate_args *a; 6037 6038 a = ap; 6039 if (rc == 0) 6040 INOTIFY(a->a_vp, IN_MODIFY); 6041 } 6042 6043 void 6044 vop_copy_file_range_post(void *ap, int rc) 6045 { 6046 struct vop_copy_file_range_args *a; 6047 6048 a = ap; 6049 if (rc == 0) { 6050 INOTIFY(a->a_invp, IN_ACCESS); 6051 INOTIFY(a->a_outvp, IN_MODIFY); 6052 } 6053 } 6054 6055 void 6056 vop_create_pre(void *ap) 6057 { 6058 struct vop_create_args *a; 6059 struct vnode *dvp; 6060 6061 a = ap; 6062 dvp = a->a_dvp; 6063 vn_seqc_write_begin(dvp); 6064 } 6065 6066 void 6067 vop_create_post(void *ap, int rc) 6068 { 6069 struct vop_create_args *a; 6070 struct vnode *dvp; 6071 6072 a = ap; 6073 dvp = a->a_dvp; 6074 vn_seqc_write_end(dvp); 6075 if (!rc) { 6076 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6077 INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE); 6078 } 6079 } 6080 6081 void 6082 vop_deallocate_post(void *ap, int rc) 6083 { 6084 struct vop_deallocate_args *a; 6085 6086 a = ap; 6087 if (rc == 0) 6088 INOTIFY(a->a_vp, IN_MODIFY); 6089 } 6090 6091 void 6092 vop_whiteout_pre(void *ap) 6093 { 6094 struct vop_whiteout_args *a; 6095 struct vnode *dvp; 6096 6097 a = ap; 6098 dvp = a->a_dvp; 6099 vn_seqc_write_begin(dvp); 6100 } 6101 6102 void 6103 vop_whiteout_post(void *ap, int rc) 6104 { 6105 struct vop_whiteout_args *a; 6106 struct vnode *dvp; 6107 6108 a = ap; 6109 dvp = a->a_dvp; 6110 vn_seqc_write_end(dvp); 6111 } 6112 6113 void 6114 vop_deleteextattr_pre(void *ap) 6115 { 6116 struct vop_deleteextattr_args *a; 6117 struct vnode *vp; 6118 6119 a = ap; 6120 vp = a->a_vp; 6121 vn_seqc_write_begin(vp); 6122 } 6123 6124 void 6125 vop_deleteextattr_post(void *ap, int rc) 6126 { 6127 struct vop_deleteextattr_args *a; 6128 struct vnode *vp; 6129 6130 a = ap; 6131 vp = a->a_vp; 6132 vn_seqc_write_end(vp); 6133 if (!rc) { 6134 VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB); 6135 INOTIFY(vp, IN_ATTRIB); 6136 } 6137 } 6138 6139 void 6140 vop_link_pre(void *ap) 6141 { 6142 struct vop_link_args *a; 6143 struct vnode *vp, *tdvp; 6144 6145 a = ap; 6146 vp = a->a_vp; 6147 tdvp = a->a_tdvp; 6148 vn_seqc_write_begin(vp); 6149 vn_seqc_write_begin(tdvp); 6150 } 6151 6152 void 6153 vop_link_post(void *ap, int rc) 6154 { 6155 struct vop_link_args *a; 6156 struct vnode *vp, *tdvp; 6157 6158 a = ap; 6159 vp = a->a_vp; 6160 tdvp = a->a_tdvp; 6161 vn_seqc_write_end(vp); 6162 vn_seqc_write_end(tdvp); 6163 if (!rc) { 6164 VFS_KNOTE_LOCKED(vp, NOTE_LINK); 6165 VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE); 6166 INOTIFY_NAME(vp, tdvp, a->a_cnp, _IN_ATTRIB_LINKCOUNT); 6167 INOTIFY_NAME(vp, tdvp, a->a_cnp, IN_CREATE); 6168 } 6169 } 6170 6171 void 6172 vop_mkdir_pre(void *ap) 6173 { 6174 struct vop_mkdir_args *a; 6175 struct vnode *dvp; 6176 6177 a = ap; 6178 dvp = a->a_dvp; 6179 vn_seqc_write_begin(dvp); 6180 } 6181 6182 void 6183 vop_mkdir_post(void *ap, int rc) 6184 { 6185 struct vop_mkdir_args *a; 6186 struct vnode *dvp; 6187 6188 a = ap; 6189 dvp = a->a_dvp; 6190 vn_seqc_write_end(dvp); 6191 if (!rc) { 6192 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 6193 INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE); 6194 } 6195 } 6196 6197 #ifdef INVARIANTS 6198 void 6199 vop_mkdir_debugpost(void *ap, int rc) 6200 { 6201 struct vop_mkdir_args *a; 6202 6203 a = ap; 6204 if (!rc) 6205 cache_validate(a->a_dvp, *a->a_vpp, a->a_cnp); 6206 } 6207 #endif 6208 6209 void 6210 vop_mknod_pre(void *ap) 6211 { 6212 struct vop_mknod_args *a; 6213 struct vnode *dvp; 6214 6215 a = ap; 6216 dvp = a->a_dvp; 6217 vn_seqc_write_begin(dvp); 6218 } 6219 6220 void 6221 vop_mknod_post(void *ap, int rc) 6222 { 6223 struct vop_mknod_args *a; 6224 struct vnode *dvp; 6225 6226 a = ap; 6227 dvp = a->a_dvp; 6228 vn_seqc_write_end(dvp); 6229 if (!rc) { 6230 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6231 INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE); 6232 } 6233 } 6234 6235 void 6236 vop_reclaim_post(void *ap, int rc) 6237 { 6238 struct vop_reclaim_args *a; 6239 struct vnode *vp; 6240 6241 a = ap; 6242 vp = a->a_vp; 6243 ASSERT_VOP_IN_SEQC(vp); 6244 if (!rc) { 6245 VFS_KNOTE_LOCKED(vp, NOTE_REVOKE); 6246 INOTIFY_REVOKE(vp); 6247 } 6248 } 6249 6250 void 6251 vop_remove_pre(void *ap) 6252 { 6253 struct vop_remove_args *a; 6254 struct vnode *dvp, *vp; 6255 6256 a = ap; 6257 dvp = a->a_dvp; 6258 vp = a->a_vp; 6259 vfs_notify_upper(vp, VFS_NOTIFY_UPPER_UNLINK); 6260 vn_seqc_write_begin(dvp); 6261 vn_seqc_write_begin(vp); 6262 } 6263 6264 void 6265 vop_remove_post(void *ap, int rc) 6266 { 6267 struct vop_remove_args *a; 6268 struct vnode *dvp, *vp; 6269 6270 a = ap; 6271 dvp = a->a_dvp; 6272 vp = a->a_vp; 6273 vn_seqc_write_end(dvp); 6274 vn_seqc_write_end(vp); 6275 if (!rc) { 6276 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6277 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 6278 INOTIFY_NAME(vp, dvp, a->a_cnp, _IN_ATTRIB_LINKCOUNT); 6279 INOTIFY_NAME(vp, dvp, a->a_cnp, IN_DELETE); 6280 } 6281 } 6282 6283 void 6284 vop_rename_post(void *ap, int rc) 6285 { 6286 struct vop_rename_args *a = ap; 6287 long hint; 6288 6289 if (!rc) { 6290 hint = NOTE_WRITE; 6291 if (a->a_fdvp == a->a_tdvp) { 6292 if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR) 6293 hint |= NOTE_LINK; 6294 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 6295 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 6296 } else { 6297 hint |= NOTE_EXTEND; 6298 if (a->a_fvp->v_type == VDIR) 6299 hint |= NOTE_LINK; 6300 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint); 6301 6302 if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL && 6303 a->a_tvp->v_type == VDIR) 6304 hint &= ~NOTE_LINK; 6305 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint); 6306 } 6307 6308 VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME); 6309 if (a->a_tvp) 6310 VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE); 6311 INOTIFY_MOVE(a->a_fvp, a->a_fdvp, a->a_fcnp, a->a_tvp, 6312 a->a_tdvp, a->a_tcnp); 6313 } 6314 if (a->a_tdvp != a->a_fdvp) 6315 vdrop(a->a_fdvp); 6316 if (a->a_tvp != a->a_fvp) 6317 vdrop(a->a_fvp); 6318 vdrop(a->a_tdvp); 6319 if (a->a_tvp) 6320 vdrop(a->a_tvp); 6321 } 6322 6323 void 6324 vop_rmdir_pre(void *ap) 6325 { 6326 struct vop_rmdir_args *a; 6327 struct vnode *dvp, *vp; 6328 6329 a = ap; 6330 dvp = a->a_dvp; 6331 vp = a->a_vp; 6332 vfs_notify_upper(vp, VFS_NOTIFY_UPPER_UNLINK); 6333 vn_seqc_write_begin(dvp); 6334 vn_seqc_write_begin(vp); 6335 } 6336 6337 void 6338 vop_rmdir_post(void *ap, int rc) 6339 { 6340 struct vop_rmdir_args *a; 6341 struct vnode *dvp, *vp; 6342 6343 a = ap; 6344 dvp = a->a_dvp; 6345 vp = a->a_vp; 6346 vn_seqc_write_end(dvp); 6347 vn_seqc_write_end(vp); 6348 if (!rc) { 6349 vp->v_vflag |= VV_UNLINKED; 6350 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK); 6351 VFS_KNOTE_LOCKED(vp, NOTE_DELETE); 6352 INOTIFY_NAME(vp, dvp, a->a_cnp, IN_DELETE); 6353 } 6354 } 6355 6356 void 6357 vop_setattr_pre(void *ap) 6358 { 6359 struct vop_setattr_args *a; 6360 struct vnode *vp; 6361 6362 a = ap; 6363 vp = a->a_vp; 6364 vn_seqc_write_begin(vp); 6365 } 6366 6367 void 6368 vop_setattr_post(void *ap, int rc) 6369 { 6370 struct vop_setattr_args *a; 6371 struct vnode *vp; 6372 6373 a = ap; 6374 vp = a->a_vp; 6375 vn_seqc_write_end(vp); 6376 if (!rc) { 6377 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 6378 INOTIFY(vp, IN_ATTRIB); 6379 } 6380 } 6381 6382 void 6383 vop_setacl_pre(void *ap) 6384 { 6385 struct vop_setacl_args *a; 6386 struct vnode *vp; 6387 6388 a = ap; 6389 vp = a->a_vp; 6390 vn_seqc_write_begin(vp); 6391 } 6392 6393 void 6394 vop_setacl_post(void *ap, int rc __unused) 6395 { 6396 struct vop_setacl_args *a; 6397 struct vnode *vp; 6398 6399 a = ap; 6400 vp = a->a_vp; 6401 vn_seqc_write_end(vp); 6402 } 6403 6404 void 6405 vop_setextattr_pre(void *ap) 6406 { 6407 struct vop_setextattr_args *a; 6408 struct vnode *vp; 6409 6410 a = ap; 6411 vp = a->a_vp; 6412 vn_seqc_write_begin(vp); 6413 } 6414 6415 void 6416 vop_setextattr_post(void *ap, int rc) 6417 { 6418 struct vop_setextattr_args *a; 6419 struct vnode *vp; 6420 6421 a = ap; 6422 vp = a->a_vp; 6423 vn_seqc_write_end(vp); 6424 if (!rc) { 6425 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB); 6426 INOTIFY(vp, IN_ATTRIB); 6427 } 6428 } 6429 6430 void 6431 vop_symlink_pre(void *ap) 6432 { 6433 struct vop_symlink_args *a; 6434 struct vnode *dvp; 6435 6436 a = ap; 6437 dvp = a->a_dvp; 6438 vn_seqc_write_begin(dvp); 6439 } 6440 6441 void 6442 vop_symlink_post(void *ap, int rc) 6443 { 6444 struct vop_symlink_args *a; 6445 struct vnode *dvp; 6446 6447 a = ap; 6448 dvp = a->a_dvp; 6449 vn_seqc_write_end(dvp); 6450 if (!rc) { 6451 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE); 6452 INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE); 6453 } 6454 } 6455 6456 void 6457 vop_open_post(void *ap, int rc) 6458 { 6459 struct vop_open_args *a = ap; 6460 6461 if (!rc) { 6462 VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN); 6463 INOTIFY(a->a_vp, IN_OPEN); 6464 } 6465 } 6466 6467 void 6468 vop_close_post(void *ap, int rc) 6469 { 6470 struct vop_close_args *a = ap; 6471 6472 if (!rc && (a->a_cred != NOCRED || /* filter out revokes */ 6473 !VN_IS_DOOMED(a->a_vp))) { 6474 VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ? 6475 NOTE_CLOSE_WRITE : NOTE_CLOSE); 6476 INOTIFY(a->a_vp, (a->a_fflag & FWRITE) != 0 ? 6477 IN_CLOSE_WRITE : IN_CLOSE_NOWRITE); 6478 } 6479 } 6480 6481 void 6482 vop_read_post(void *ap, int rc) 6483 { 6484 struct vop_read_args *a = ap; 6485 6486 if (!rc) { 6487 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ); 6488 INOTIFY(a->a_vp, IN_ACCESS); 6489 } 6490 } 6491 6492 void 6493 vop_read_pgcache_post(void *ap, int rc) 6494 { 6495 struct vop_read_pgcache_args *a = ap; 6496 6497 if (!rc) 6498 VFS_KNOTE_UNLOCKED(a->a_vp, NOTE_READ); 6499 } 6500 6501 static struct knlist fs_knlist; 6502 6503 static void 6504 vfs_event_init(void *arg) 6505 { 6506 knlist_init_mtx(&fs_knlist, NULL); 6507 } 6508 /* XXX - correct order? */ 6509 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL); 6510 6511 void 6512 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused) 6513 { 6514 6515 KNOTE_UNLOCKED(&fs_knlist, event); 6516 } 6517 6518 static int filt_fsattach(struct knote *kn); 6519 static void filt_fsdetach(struct knote *kn); 6520 static int filt_fsevent(struct knote *kn, long hint); 6521 6522 const struct filterops fs_filtops = { 6523 .f_isfd = 0, 6524 .f_attach = filt_fsattach, 6525 .f_detach = filt_fsdetach, 6526 .f_event = filt_fsevent, 6527 }; 6528 6529 static int 6530 filt_fsattach(struct knote *kn) 6531 { 6532 6533 kn->kn_flags |= EV_CLEAR; 6534 knlist_add(&fs_knlist, kn, 0); 6535 return (0); 6536 } 6537 6538 static void 6539 filt_fsdetach(struct knote *kn) 6540 { 6541 6542 knlist_remove(&fs_knlist, kn, 0); 6543 } 6544 6545 static int 6546 filt_fsevent(struct knote *kn, long hint) 6547 { 6548 6549 kn->kn_fflags |= kn->kn_sfflags & hint; 6550 6551 return (kn->kn_fflags != 0); 6552 } 6553 6554 static int 6555 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS) 6556 { 6557 struct vfsidctl vc; 6558 int error; 6559 struct mount *mp; 6560 6561 if (req->newptr == NULL) 6562 return (EINVAL); 6563 error = SYSCTL_IN(req, &vc, sizeof(vc)); 6564 if (error) 6565 return (error); 6566 if (vc.vc_vers != VFS_CTL_VERS1) 6567 return (EINVAL); 6568 mp = vfs_getvfs(&vc.vc_fsid); 6569 if (mp == NULL) 6570 return (ENOENT); 6571 /* ensure that a specific sysctl goes to the right filesystem. */ 6572 if (strcmp(vc.vc_fstypename, "*") != 0 && 6573 strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) { 6574 vfs_rel(mp); 6575 return (EINVAL); 6576 } 6577 VCTLTOREQ(&vc, req); 6578 error = VFS_SYSCTL(mp, vc.vc_op, req); 6579 vfs_rel(mp); 6580 return (error); 6581 } 6582 6583 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_MPSAFE | CTLFLAG_WR, 6584 NULL, 0, sysctl_vfs_ctl, "", 6585 "Sysctl by fsid"); 6586 6587 /* 6588 * Function to initialize a va_filerev field sensibly. 6589 * XXX: Wouldn't a random number make a lot more sense ?? 6590 */ 6591 u_quad_t 6592 init_va_filerev(void) 6593 { 6594 struct bintime bt; 6595 6596 getbinuptime(&bt); 6597 return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL)); 6598 } 6599 6600 static int filt_vfsread(struct knote *kn, long hint); 6601 static int filt_vfswrite(struct knote *kn, long hint); 6602 static int filt_vfsvnode(struct knote *kn, long hint); 6603 static void filt_vfsdetach(struct knote *kn); 6604 static int filt_vfsdump(struct proc *p, struct knote *kn, 6605 struct kinfo_knote *kin); 6606 6607 static const struct filterops vfsread_filtops = { 6608 .f_isfd = 1, 6609 .f_detach = filt_vfsdetach, 6610 .f_event = filt_vfsread, 6611 .f_userdump = filt_vfsdump, 6612 }; 6613 static const struct filterops vfswrite_filtops = { 6614 .f_isfd = 1, 6615 .f_detach = filt_vfsdetach, 6616 .f_event = filt_vfswrite, 6617 .f_userdump = filt_vfsdump, 6618 }; 6619 static const struct filterops vfsvnode_filtops = { 6620 .f_isfd = 1, 6621 .f_detach = filt_vfsdetach, 6622 .f_event = filt_vfsvnode, 6623 .f_userdump = filt_vfsdump, 6624 }; 6625 6626 static void 6627 vfs_knllock(void *arg) 6628 { 6629 struct vnode *vp = arg; 6630 6631 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 6632 } 6633 6634 static void 6635 vfs_knlunlock(void *arg) 6636 { 6637 struct vnode *vp = arg; 6638 6639 VOP_UNLOCK(vp); 6640 } 6641 6642 static void 6643 vfs_knl_assert_lock(void *arg, int what) 6644 { 6645 #ifdef INVARIANTS 6646 struct vnode *vp = arg; 6647 6648 if (what == LA_LOCKED) 6649 ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked"); 6650 else 6651 ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked"); 6652 #endif 6653 } 6654 6655 int 6656 vfs_kqfilter(struct vop_kqfilter_args *ap) 6657 { 6658 struct vnode *vp = ap->a_vp; 6659 struct knote *kn = ap->a_kn; 6660 struct knlist *knl; 6661 6662 KASSERT(vp->v_type != VFIFO || (kn->kn_filter != EVFILT_READ && 6663 kn->kn_filter != EVFILT_WRITE), 6664 ("READ/WRITE filter on a FIFO leaked through")); 6665 switch (kn->kn_filter) { 6666 case EVFILT_READ: 6667 kn->kn_fop = &vfsread_filtops; 6668 break; 6669 case EVFILT_WRITE: 6670 kn->kn_fop = &vfswrite_filtops; 6671 break; 6672 case EVFILT_VNODE: 6673 kn->kn_fop = &vfsvnode_filtops; 6674 break; 6675 default: 6676 return (EINVAL); 6677 } 6678 6679 kn->kn_hook = (caddr_t)vp; 6680 6681 v_addpollinfo(vp); 6682 if (vp->v_pollinfo == NULL) 6683 return (ENOMEM); 6684 knl = &vp->v_pollinfo->vpi_selinfo.si_note; 6685 vhold(vp); 6686 knlist_add(knl, kn, 0); 6687 6688 return (0); 6689 } 6690 6691 /* 6692 * Detach knote from vnode 6693 */ 6694 static void 6695 filt_vfsdetach(struct knote *kn) 6696 { 6697 struct vnode *vp = (struct vnode *)kn->kn_hook; 6698 6699 KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo")); 6700 knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0); 6701 vdrop(vp); 6702 } 6703 6704 /*ARGSUSED*/ 6705 static int 6706 filt_vfsread(struct knote *kn, long hint) 6707 { 6708 struct vnode *vp = (struct vnode *)kn->kn_hook; 6709 off_t size; 6710 int res; 6711 6712 /* 6713 * filesystem is gone, so set the EOF flag and schedule 6714 * the knote for deletion. 6715 */ 6716 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6717 VI_LOCK(vp); 6718 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6719 VI_UNLOCK(vp); 6720 return (1); 6721 } 6722 6723 if (vn_getsize_locked(vp, &size, curthread->td_ucred) != 0) 6724 return (0); 6725 6726 VI_LOCK(vp); 6727 kn->kn_data = size - kn->kn_fp->f_offset; 6728 res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 || kn->kn_data != 0; 6729 VI_UNLOCK(vp); 6730 return (res); 6731 } 6732 6733 /*ARGSUSED*/ 6734 static int 6735 filt_vfswrite(struct knote *kn, long hint) 6736 { 6737 struct vnode *vp = (struct vnode *)kn->kn_hook; 6738 6739 VI_LOCK(vp); 6740 6741 /* 6742 * filesystem is gone, so set the EOF flag and schedule 6743 * the knote for deletion. 6744 */ 6745 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) 6746 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 6747 6748 kn->kn_data = 0; 6749 VI_UNLOCK(vp); 6750 return (1); 6751 } 6752 6753 static int 6754 filt_vfsvnode(struct knote *kn, long hint) 6755 { 6756 struct vnode *vp = (struct vnode *)kn->kn_hook; 6757 int res; 6758 6759 VI_LOCK(vp); 6760 if (kn->kn_sfflags & hint) 6761 kn->kn_fflags |= hint; 6762 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) { 6763 kn->kn_flags |= EV_EOF; 6764 VI_UNLOCK(vp); 6765 return (1); 6766 } 6767 res = (kn->kn_fflags != 0); 6768 VI_UNLOCK(vp); 6769 return (res); 6770 } 6771 6772 static int 6773 filt_vfsdump(struct proc *p, struct knote *kn, struct kinfo_knote *kin) 6774 { 6775 struct vattr va; 6776 struct vnode *vp; 6777 char *fullpath, *freepath; 6778 int error; 6779 6780 kin->knt_extdata = KNOTE_EXTDATA_VNODE; 6781 6782 vp = kn->kn_fp->f_vnode; 6783 kin->knt_vnode.knt_vnode_type = vntype_to_kinfo(vp->v_type); 6784 6785 va.va_fsid = VNOVAL; 6786 vn_lock(vp, LK_SHARED | LK_RETRY); 6787 error = VOP_GETATTR(vp, &va, curthread->td_ucred); 6788 VOP_UNLOCK(vp); 6789 if (error != 0) 6790 return (error); 6791 kin->knt_vnode.knt_vnode_fsid = va.va_fsid; 6792 kin->knt_vnode.knt_vnode_fileid = va.va_fileid; 6793 6794 freepath = NULL; 6795 fullpath = "-"; 6796 error = vn_fullpath(vp, &fullpath, &freepath); 6797 if (error == 0) { 6798 strlcpy(kin->knt_vnode.knt_vnode_fullpath, fullpath, 6799 sizeof(kin->knt_vnode.knt_vnode_fullpath)); 6800 } 6801 if (freepath != NULL) 6802 free(freepath, M_TEMP); 6803 6804 return (0); 6805 } 6806 6807 int 6808 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off) 6809 { 6810 int error; 6811 6812 if (dp->d_reclen > ap->a_uio->uio_resid) 6813 return (ENAMETOOLONG); 6814 error = uiomove(dp, dp->d_reclen, ap->a_uio); 6815 if (error) { 6816 if (ap->a_ncookies != NULL) { 6817 if (ap->a_cookies != NULL) 6818 free(ap->a_cookies, M_TEMP); 6819 ap->a_cookies = NULL; 6820 *ap->a_ncookies = 0; 6821 } 6822 return (error); 6823 } 6824 if (ap->a_ncookies == NULL) 6825 return (0); 6826 6827 KASSERT(ap->a_cookies, 6828 ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!")); 6829 6830 *ap->a_cookies = realloc(*ap->a_cookies, 6831 (*ap->a_ncookies + 1) * sizeof(uint64_t), M_TEMP, M_WAITOK | M_ZERO); 6832 (*ap->a_cookies)[*ap->a_ncookies] = off; 6833 *ap->a_ncookies += 1; 6834 return (0); 6835 } 6836 6837 /* 6838 * The purpose of this routine is to remove granularity from accmode_t, 6839 * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE, 6840 * VADMIN and VAPPEND. 6841 * 6842 * If it returns 0, the caller is supposed to continue with the usual 6843 * access checks using 'accmode' as modified by this routine. If it 6844 * returns nonzero value, the caller is supposed to return that value 6845 * as errno. 6846 * 6847 * Note that after this routine runs, accmode may be zero. 6848 */ 6849 int 6850 vfs_unixify_accmode(accmode_t *accmode) 6851 { 6852 /* 6853 * There is no way to specify explicit "deny" rule using 6854 * file mode or POSIX.1e ACLs. 6855 */ 6856 if (*accmode & VEXPLICIT_DENY) { 6857 *accmode = 0; 6858 return (0); 6859 } 6860 6861 /* 6862 * None of these can be translated into usual access bits. 6863 * Also, the common case for NFSv4 ACLs is to not contain 6864 * either of these bits. Caller should check for VWRITE 6865 * on the containing directory instead. 6866 */ 6867 if (*accmode & (VDELETE_CHILD | VDELETE)) 6868 return (EPERM); 6869 6870 if (*accmode & VADMIN_PERMS) { 6871 *accmode &= ~VADMIN_PERMS; 6872 *accmode |= VADMIN; 6873 } 6874 6875 /* 6876 * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL 6877 * or VSYNCHRONIZE using file mode or POSIX.1e ACL. 6878 */ 6879 *accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE); 6880 6881 return (0); 6882 } 6883 6884 /* 6885 * Clear out a doomed vnode (if any) and replace it with a new one as long 6886 * as the fs is not being unmounted. Return the root vnode to the caller. 6887 */ 6888 static int __noinline 6889 vfs_cache_root_fallback(struct mount *mp, int flags, struct vnode **vpp) 6890 { 6891 struct vnode *vp; 6892 int error; 6893 6894 restart: 6895 if (mp->mnt_rootvnode != NULL) { 6896 MNT_ILOCK(mp); 6897 vp = mp->mnt_rootvnode; 6898 if (vp != NULL) { 6899 if (!VN_IS_DOOMED(vp)) { 6900 vrefact(vp); 6901 MNT_IUNLOCK(mp); 6902 error = vn_lock(vp, flags); 6903 if (error == 0) { 6904 *vpp = vp; 6905 return (0); 6906 } 6907 vrele(vp); 6908 goto restart; 6909 } 6910 /* 6911 * Clear the old one. 6912 */ 6913 mp->mnt_rootvnode = NULL; 6914 } 6915 MNT_IUNLOCK(mp); 6916 if (vp != NULL) { 6917 vfs_op_barrier_wait(mp); 6918 vrele(vp); 6919 } 6920 } 6921 error = VFS_CACHEDROOT(mp, flags, vpp); 6922 if (error != 0) 6923 return (error); 6924 if (mp->mnt_vfs_ops == 0) { 6925 MNT_ILOCK(mp); 6926 if (mp->mnt_vfs_ops != 0) { 6927 MNT_IUNLOCK(mp); 6928 return (0); 6929 } 6930 if (mp->mnt_rootvnode == NULL) { 6931 vrefact(*vpp); 6932 mp->mnt_rootvnode = *vpp; 6933 } else { 6934 if (mp->mnt_rootvnode != *vpp) { 6935 if (!VN_IS_DOOMED(mp->mnt_rootvnode)) { 6936 panic("%s: mismatch between vnode returned " 6937 " by VFS_CACHEDROOT and the one cached " 6938 " (%p != %p)", 6939 __func__, *vpp, mp->mnt_rootvnode); 6940 } 6941 } 6942 } 6943 MNT_IUNLOCK(mp); 6944 } 6945 return (0); 6946 } 6947 6948 int 6949 vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp) 6950 { 6951 struct mount_pcpu *mpcpu; 6952 struct vnode *vp; 6953 int error; 6954 6955 if (!vfs_op_thread_enter(mp, mpcpu)) 6956 return (vfs_cache_root_fallback(mp, flags, vpp)); 6957 vp = atomic_load_ptr(&mp->mnt_rootvnode); 6958 if (vp == NULL || VN_IS_DOOMED(vp)) { 6959 vfs_op_thread_exit(mp, mpcpu); 6960 return (vfs_cache_root_fallback(mp, flags, vpp)); 6961 } 6962 vrefact(vp); 6963 vfs_op_thread_exit(mp, mpcpu); 6964 error = vn_lock(vp, flags); 6965 if (error != 0) { 6966 vrele(vp); 6967 return (vfs_cache_root_fallback(mp, flags, vpp)); 6968 } 6969 *vpp = vp; 6970 return (0); 6971 } 6972 6973 struct vnode * 6974 vfs_cache_root_clear(struct mount *mp) 6975 { 6976 struct vnode *vp; 6977 6978 /* 6979 * ops > 0 guarantees there is nobody who can see this vnode 6980 */ 6981 MPASS(mp->mnt_vfs_ops > 0); 6982 vp = mp->mnt_rootvnode; 6983 if (vp != NULL) 6984 vn_seqc_write_begin(vp); 6985 mp->mnt_rootvnode = NULL; 6986 return (vp); 6987 } 6988 6989 void 6990 vfs_cache_root_set(struct mount *mp, struct vnode *vp) 6991 { 6992 6993 MPASS(mp->mnt_vfs_ops > 0); 6994 vrefact(vp); 6995 mp->mnt_rootvnode = vp; 6996 } 6997 6998 /* 6999 * These are helper functions for filesystems to traverse all 7000 * their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h. 7001 * 7002 * This interface replaces MNT_VNODE_FOREACH. 7003 */ 7004 7005 struct vnode * 7006 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp) 7007 { 7008 struct vnode *vp; 7009 7010 maybe_yield(); 7011 MNT_ILOCK(mp); 7012 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 7013 for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL; 7014 vp = TAILQ_NEXT(vp, v_nmntvnodes)) { 7015 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 7016 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 7017 continue; 7018 VI_LOCK(vp); 7019 if (VN_IS_DOOMED(vp)) { 7020 VI_UNLOCK(vp); 7021 continue; 7022 } 7023 break; 7024 } 7025 if (vp == NULL) { 7026 __mnt_vnode_markerfree_all(mvp, mp); 7027 /* MNT_IUNLOCK(mp); -- done in above function */ 7028 mtx_assert(MNT_MTX(mp), MA_NOTOWNED); 7029 return (NULL); 7030 } 7031 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 7032 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 7033 MNT_IUNLOCK(mp); 7034 return (vp); 7035 } 7036 7037 struct vnode * 7038 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp) 7039 { 7040 struct vnode *vp; 7041 7042 *mvp = vn_alloc_marker(mp); 7043 MNT_ILOCK(mp); 7044 MNT_REF(mp); 7045 7046 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { 7047 /* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */ 7048 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp)) 7049 continue; 7050 VI_LOCK(vp); 7051 if (VN_IS_DOOMED(vp)) { 7052 VI_UNLOCK(vp); 7053 continue; 7054 } 7055 break; 7056 } 7057 if (vp == NULL) { 7058 MNT_REL(mp); 7059 MNT_IUNLOCK(mp); 7060 vn_free_marker(*mvp); 7061 *mvp = NULL; 7062 return (NULL); 7063 } 7064 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); 7065 MNT_IUNLOCK(mp); 7066 return (vp); 7067 } 7068 7069 void 7070 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp) 7071 { 7072 7073 if (*mvp == NULL) { 7074 MNT_IUNLOCK(mp); 7075 return; 7076 } 7077 7078 mtx_assert(MNT_MTX(mp), MA_OWNED); 7079 7080 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 7081 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); 7082 MNT_REL(mp); 7083 MNT_IUNLOCK(mp); 7084 vn_free_marker(*mvp); 7085 *mvp = NULL; 7086 } 7087 7088 /* 7089 * These are helper functions for filesystems to traverse their 7090 * lazy vnodes. See MNT_VNODE_FOREACH_LAZY() in sys/mount.h 7091 */ 7092 static void 7093 mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 7094 { 7095 7096 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 7097 7098 MNT_ILOCK(mp); 7099 MNT_REL(mp); 7100 MNT_IUNLOCK(mp); 7101 vn_free_marker(*mvp); 7102 *mvp = NULL; 7103 } 7104 7105 /* 7106 * Relock the mp mount vnode list lock with the vp vnode interlock in the 7107 * conventional lock order during mnt_vnode_next_lazy iteration. 7108 * 7109 * On entry, the mount vnode list lock is held and the vnode interlock is not. 7110 * The list lock is dropped and reacquired. On success, both locks are held. 7111 * On failure, the mount vnode list lock is held but the vnode interlock is 7112 * not, and the procedure may have yielded. 7113 */ 7114 static bool 7115 mnt_vnode_next_lazy_relock(struct vnode *mvp, struct mount *mp, 7116 struct vnode *vp) 7117 { 7118 7119 VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER && 7120 TAILQ_NEXT(mvp, v_lazylist) != NULL, mvp, 7121 ("%s: bad marker", __func__)); 7122 VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp, 7123 ("%s: inappropriate vnode", __func__)); 7124 ASSERT_VI_UNLOCKED(vp, __func__); 7125 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 7126 7127 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, mvp, v_lazylist); 7128 TAILQ_INSERT_BEFORE(vp, mvp, v_lazylist); 7129 7130 /* 7131 * Note we may be racing against vdrop which transitioned the hold 7132 * count to 0 and now waits for the ->mnt_listmtx lock. This is fine, 7133 * if we are the only user after we get the interlock we will just 7134 * vdrop. 7135 */ 7136 vhold(vp); 7137 mtx_unlock(&mp->mnt_listmtx); 7138 VI_LOCK(vp); 7139 if (VN_IS_DOOMED(vp)) { 7140 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp); 7141 goto out_lost; 7142 } 7143 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp); 7144 /* 7145 * There is nothing to do if we are the last user. 7146 */ 7147 if (!refcount_release_if_not_last(&vp->v_holdcnt)) 7148 goto out_lost; 7149 mtx_lock(&mp->mnt_listmtx); 7150 return (true); 7151 out_lost: 7152 vdropl(vp); 7153 maybe_yield(); 7154 mtx_lock(&mp->mnt_listmtx); 7155 return (false); 7156 } 7157 7158 static struct vnode * 7159 mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 7160 void *cbarg) 7161 { 7162 struct vnode *vp; 7163 7164 mtx_assert(&mp->mnt_listmtx, MA_OWNED); 7165 KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); 7166 restart: 7167 vp = TAILQ_NEXT(*mvp, v_lazylist); 7168 while (vp != NULL) { 7169 if (vp->v_type == VMARKER) { 7170 vp = TAILQ_NEXT(vp, v_lazylist); 7171 continue; 7172 } 7173 /* 7174 * See if we want to process the vnode. Note we may encounter a 7175 * long string of vnodes we don't care about and hog the list 7176 * as a result. Check for it and requeue the marker. 7177 */ 7178 VNPASS(!VN_IS_DOOMED(vp), vp); 7179 if (!cb(vp, cbarg)) { 7180 if (!should_yield()) { 7181 vp = TAILQ_NEXT(vp, v_lazylist); 7182 continue; 7183 } 7184 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, 7185 v_lazylist); 7186 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, 7187 v_lazylist); 7188 mtx_unlock(&mp->mnt_listmtx); 7189 kern_yield(PRI_USER); 7190 mtx_lock(&mp->mnt_listmtx); 7191 goto restart; 7192 } 7193 /* 7194 * Try-lock because this is the wrong lock order. 7195 */ 7196 if (!VI_TRYLOCK(vp) && 7197 !mnt_vnode_next_lazy_relock(*mvp, mp, vp)) 7198 goto restart; 7199 KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp)); 7200 KASSERT(vp->v_mount == mp || vp->v_mount == NULL, 7201 ("alien vnode on the lazy list %p %p", vp, mp)); 7202 VNPASS(vp->v_mount == mp, vp); 7203 VNPASS(!VN_IS_DOOMED(vp), vp); 7204 break; 7205 } 7206 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 7207 7208 /* Check if we are done */ 7209 if (vp == NULL) { 7210 mtx_unlock(&mp->mnt_listmtx); 7211 mnt_vnode_markerfree_lazy(mvp, mp); 7212 return (NULL); 7213 } 7214 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, v_lazylist); 7215 mtx_unlock(&mp->mnt_listmtx); 7216 ASSERT_VI_LOCKED(vp, "lazy iter"); 7217 return (vp); 7218 } 7219 7220 struct vnode * 7221 __mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 7222 void *cbarg) 7223 { 7224 7225 maybe_yield(); 7226 mtx_lock(&mp->mnt_listmtx); 7227 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 7228 } 7229 7230 struct vnode * 7231 __mnt_vnode_first_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, 7232 void *cbarg) 7233 { 7234 struct vnode *vp; 7235 7236 if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist)) 7237 return (NULL); 7238 7239 *mvp = vn_alloc_marker(mp); 7240 MNT_ILOCK(mp); 7241 MNT_REF(mp); 7242 MNT_IUNLOCK(mp); 7243 7244 mtx_lock(&mp->mnt_listmtx); 7245 vp = TAILQ_FIRST(&mp->mnt_lazyvnodelist); 7246 if (vp == NULL) { 7247 mtx_unlock(&mp->mnt_listmtx); 7248 mnt_vnode_markerfree_lazy(mvp, mp); 7249 return (NULL); 7250 } 7251 TAILQ_INSERT_BEFORE(vp, *mvp, v_lazylist); 7252 return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg)); 7253 } 7254 7255 void 7256 __mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp) 7257 { 7258 7259 if (*mvp == NULL) 7260 return; 7261 7262 mtx_lock(&mp->mnt_listmtx); 7263 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist); 7264 mtx_unlock(&mp->mnt_listmtx); 7265 mnt_vnode_markerfree_lazy(mvp, mp); 7266 } 7267 7268 int 7269 vn_dir_check_exec(struct vnode *vp, struct componentname *cnp) 7270 { 7271 7272 if ((cnp->cn_flags & NOEXECCHECK) != 0) { 7273 cnp->cn_flags &= ~NOEXECCHECK; 7274 return (0); 7275 } 7276 7277 return (VOP_ACCESS(vp, VEXEC, cnp->cn_cred, curthread)); 7278 } 7279 7280 /* 7281 * Do not use this variant unless you have means other than the hold count 7282 * to prevent the vnode from getting freed. 7283 */ 7284 void 7285 vn_seqc_write_begin_locked(struct vnode *vp) 7286 { 7287 7288 ASSERT_VI_LOCKED(vp, __func__); 7289 VNPASS(vp->v_holdcnt > 0, vp); 7290 VNPASS(vp->v_seqc_users >= 0, vp); 7291 vp->v_seqc_users++; 7292 if (vp->v_seqc_users == 1) 7293 seqc_sleepable_write_begin(&vp->v_seqc); 7294 } 7295 7296 void 7297 vn_seqc_write_begin(struct vnode *vp) 7298 { 7299 7300 VI_LOCK(vp); 7301 vn_seqc_write_begin_locked(vp); 7302 VI_UNLOCK(vp); 7303 } 7304 7305 void 7306 vn_seqc_write_end_locked(struct vnode *vp) 7307 { 7308 7309 ASSERT_VI_LOCKED(vp, __func__); 7310 VNPASS(vp->v_seqc_users > 0, vp); 7311 vp->v_seqc_users--; 7312 if (vp->v_seqc_users == 0) 7313 seqc_sleepable_write_end(&vp->v_seqc); 7314 } 7315 7316 void 7317 vn_seqc_write_end(struct vnode *vp) 7318 { 7319 7320 VI_LOCK(vp); 7321 vn_seqc_write_end_locked(vp); 7322 VI_UNLOCK(vp); 7323 } 7324 7325 /* 7326 * Special case handling for allocating and freeing vnodes. 7327 * 7328 * The counter remains unchanged on free so that a doomed vnode will 7329 * keep testing as in modify as long as it is accessible with SMR. 7330 */ 7331 static void 7332 vn_seqc_init(struct vnode *vp) 7333 { 7334 7335 vp->v_seqc = 0; 7336 vp->v_seqc_users = 0; 7337 } 7338 7339 static void 7340 vn_seqc_write_end_free(struct vnode *vp) 7341 { 7342 7343 VNPASS(seqc_in_modify(vp->v_seqc), vp); 7344 VNPASS(vp->v_seqc_users == 1, vp); 7345 } 7346 7347 void 7348 vn_irflag_set_locked(struct vnode *vp, short toset) 7349 { 7350 short flags; 7351 7352 ASSERT_VI_LOCKED(vp, __func__); 7353 flags = vn_irflag_read(vp); 7354 VNASSERT((flags & toset) == 0, vp, 7355 ("%s: some of the passed flags already set (have %d, passed %d)\n", 7356 __func__, flags, toset)); 7357 atomic_store_short(&vp->v_irflag, flags | toset); 7358 } 7359 7360 void 7361 vn_irflag_set(struct vnode *vp, short toset) 7362 { 7363 7364 VI_LOCK(vp); 7365 vn_irflag_set_locked(vp, toset); 7366 VI_UNLOCK(vp); 7367 } 7368 7369 void 7370 vn_irflag_set_cond_locked(struct vnode *vp, short toset) 7371 { 7372 short flags; 7373 7374 ASSERT_VI_LOCKED(vp, __func__); 7375 flags = vn_irflag_read(vp); 7376 atomic_store_short(&vp->v_irflag, flags | toset); 7377 } 7378 7379 void 7380 vn_irflag_set_cond(struct vnode *vp, short toset) 7381 { 7382 7383 VI_LOCK(vp); 7384 vn_irflag_set_cond_locked(vp, toset); 7385 VI_UNLOCK(vp); 7386 } 7387 7388 void 7389 vn_irflag_unset_locked(struct vnode *vp, short tounset) 7390 { 7391 short flags; 7392 7393 ASSERT_VI_LOCKED(vp, __func__); 7394 flags = vn_irflag_read(vp); 7395 VNASSERT((flags & tounset) == tounset, vp, 7396 ("%s: some of the passed flags not set (have %d, passed %d)\n", 7397 __func__, flags, tounset)); 7398 atomic_store_short(&vp->v_irflag, flags & ~tounset); 7399 } 7400 7401 void 7402 vn_irflag_unset(struct vnode *vp, short tounset) 7403 { 7404 7405 VI_LOCK(vp); 7406 vn_irflag_unset_locked(vp, tounset); 7407 VI_UNLOCK(vp); 7408 } 7409 7410 int 7411 vn_getsize_locked(struct vnode *vp, off_t *size, struct ucred *cred) 7412 { 7413 struct vattr vattr; 7414 int error; 7415 7416 ASSERT_VOP_LOCKED(vp, __func__); 7417 error = VOP_GETATTR(vp, &vattr, cred); 7418 if (__predict_true(error == 0)) { 7419 if (vattr.va_size <= OFF_MAX) 7420 *size = vattr.va_size; 7421 else 7422 error = EFBIG; 7423 } 7424 return (error); 7425 } 7426 7427 int 7428 vn_getsize(struct vnode *vp, off_t *size, struct ucred *cred) 7429 { 7430 int error; 7431 7432 VOP_LOCK(vp, LK_SHARED); 7433 error = vn_getsize_locked(vp, size, cred); 7434 VOP_UNLOCK(vp); 7435 return (error); 7436 } 7437 7438 #ifdef INVARIANTS 7439 void 7440 vn_set_state_validate(struct vnode *vp, __enum_uint8(vstate) state) 7441 { 7442 7443 switch (vp->v_state) { 7444 case VSTATE_UNINITIALIZED: 7445 switch (state) { 7446 case VSTATE_CONSTRUCTED: 7447 case VSTATE_DESTROYING: 7448 return; 7449 default: 7450 break; 7451 } 7452 break; 7453 case VSTATE_CONSTRUCTED: 7454 ASSERT_VOP_ELOCKED(vp, __func__); 7455 switch (state) { 7456 case VSTATE_DESTROYING: 7457 return; 7458 default: 7459 break; 7460 } 7461 break; 7462 case VSTATE_DESTROYING: 7463 ASSERT_VOP_ELOCKED(vp, __func__); 7464 switch (state) { 7465 case VSTATE_DEAD: 7466 return; 7467 default: 7468 break; 7469 } 7470 break; 7471 case VSTATE_DEAD: 7472 switch (state) { 7473 case VSTATE_UNINITIALIZED: 7474 return; 7475 default: 7476 break; 7477 } 7478 break; 7479 } 7480 7481 vn_printf(vp, "invalid state transition %d -> %d\n", vp->v_state, state); 7482 panic("invalid state transition %d -> %d\n", vp->v_state, state); 7483 } 7484 #endif 7485