1 /*- 2 * Copyright (c) 1989, 1993, 1995 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * Poul-Henning Kamp of the FreeBSD Project. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)vfs_cache.c 8.5 (Berkeley) 3/22/95 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include "opt_ktrace.h" 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/counter.h> 43 #include <sys/filedesc.h> 44 #include <sys/fnv_hash.h> 45 #include <sys/kernel.h> 46 #include <sys/lock.h> 47 #include <sys/malloc.h> 48 #include <sys/fcntl.h> 49 #include <sys/mount.h> 50 #include <sys/namei.h> 51 #include <sys/proc.h> 52 #include <sys/rwlock.h> 53 #include <sys/sdt.h> 54 #include <sys/smp.h> 55 #include <sys/syscallsubr.h> 56 #include <sys/sysctl.h> 57 #include <sys/sysproto.h> 58 #include <sys/vnode.h> 59 #ifdef KTRACE 60 #include <sys/ktrace.h> 61 #endif 62 63 #include <vm/uma.h> 64 65 SDT_PROVIDER_DECLARE(vfs); 66 SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode *", "char *", 67 "struct vnode *"); 68 SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *", 69 "char *"); 70 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *"); 71 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *", 72 "char *", "struct vnode *"); 73 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *"); 74 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int", 75 "struct vnode *", "char *"); 76 SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode *", "char *", 77 "struct vnode *"); 78 SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit__negative, 79 "struct vnode *", "char *"); 80 SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *", 81 "char *"); 82 SDT_PROBE_DEFINE1(vfs, namecache, purge, done, "struct vnode *"); 83 SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *"); 84 SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *"); 85 SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode *", "char *", 86 "struct vnode *"); 87 SDT_PROBE_DEFINE2(vfs, namecache, zap_negative, done, "struct vnode *", 88 "char *"); 89 90 /* 91 * This structure describes the elements in the cache of recent 92 * names looked up by namei. 93 */ 94 95 struct namecache { 96 LIST_ENTRY(namecache) nc_hash; /* hash chain */ 97 LIST_ENTRY(namecache) nc_src; /* source vnode list */ 98 TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */ 99 struct vnode *nc_dvp; /* vnode of parent of name */ 100 struct vnode *nc_vp; /* vnode the name refers to */ 101 u_char nc_flag; /* flag bits */ 102 u_char nc_nlen; /* length of name */ 103 char nc_name[0]; /* segment name + nul */ 104 }; 105 106 /* 107 * struct namecache_ts repeats struct namecache layout up to the 108 * nc_nlen member. 109 * struct namecache_ts is used in place of struct namecache when time(s) need 110 * to be stored. The nc_dotdottime field is used when a cache entry is mapping 111 * both a non-dotdot directory name plus dotdot for the directory's 112 * parent. 113 */ 114 struct namecache_ts { 115 LIST_ENTRY(namecache) nc_hash; /* hash chain */ 116 LIST_ENTRY(namecache) nc_src; /* source vnode list */ 117 TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */ 118 struct vnode *nc_dvp; /* vnode of parent of name */ 119 struct vnode *nc_vp; /* vnode the name refers to */ 120 u_char nc_flag; /* flag bits */ 121 u_char nc_nlen; /* length of name */ 122 struct timespec nc_time; /* timespec provided by fs */ 123 struct timespec nc_dotdottime; /* dotdot timespec provided by fs */ 124 int nc_ticks; /* ticks value when entry was added */ 125 char nc_name[0]; /* segment name + nul */ 126 }; 127 128 /* 129 * Flags in namecache.nc_flag 130 */ 131 #define NCF_WHITE 0x01 132 #define NCF_ISDOTDOT 0x02 133 #define NCF_TS 0x04 134 #define NCF_DTS 0x08 135 #define NCF_DVDROP 0x10 136 137 /* 138 * Name caching works as follows: 139 * 140 * Names found by directory scans are retained in a cache 141 * for future reference. It is managed LRU, so frequently 142 * used names will hang around. Cache is indexed by hash value 143 * obtained from (vp, name) where vp refers to the directory 144 * containing name. 145 * 146 * If it is a "negative" entry, (i.e. for a name that is known NOT to 147 * exist) the vnode pointer will be NULL. 148 * 149 * Upon reaching the last segment of a path, if the reference 150 * is for DELETE, or NOCACHE is set (rewrite), and the 151 * name is located in the cache, it will be dropped. 152 * 153 * These locks are used (in the order in which they can be taken): 154 * NAME TYPE ROLE 155 * vnodelock mtx vnode lists and v_cache_dd field protection 156 * bucketlock rwlock for access to given set of hash buckets 157 * ncneg_mtx mtx negative entry LRU management 158 * 159 * Additionally, ncneg_shrink_lock mtx is used to have at most one thread 160 * shrinking the LRU list. 161 * 162 * It is legal to take multiple vnodelock and bucketlock locks. The locking 163 * order is lower address first. Both are recursive. 164 * 165 * "." lookups are lockless. 166 * 167 * ".." and vnode -> name lookups require vnodelock. 168 * 169 * name -> vnode lookup requires the relevant bucketlock to be held for reading. 170 * 171 * Insertions and removals of entries require involved vnodes and bucketlocks 172 * to be write-locked to prevent other threads from seeing the entry. 173 * 174 * Some lookups result in removal of the found entry (e.g. getting rid of a 175 * negative entry with the intent to create a positive one), which poses a 176 * problem when multiple threads reach the state. Similarly, two different 177 * threads can purge two different vnodes and try to remove the same name. 178 * 179 * If the already held vnode lock is lower than the second required lock, we 180 * can just take the other lock. However, in the opposite case, this could 181 * deadlock. As such, this is resolved by trylocking and if that fails unlocking 182 * the first node, locking everything in order and revalidating the state. 183 */ 184 185 /* 186 * Structures associated with name caching. 187 */ 188 #define NCHHASH(hash) \ 189 (&nchashtbl[(hash) & nchash]) 190 static LIST_HEAD(nchashhead, namecache) *nchashtbl; /* Hash Table */ 191 static TAILQ_HEAD(, namecache) ncneg; /* Hash Table */ 192 static u_long nchash; /* size of hash table */ 193 SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0, 194 "Size of namecache hash table"); 195 static u_long ncnegfactor = 16; /* ratio of negative entries */ 196 SYSCTL_ULONG(_vfs, OID_AUTO, ncnegfactor, CTLFLAG_RW, &ncnegfactor, 0, 197 "Ratio of negative namecache entries"); 198 static u_long numneg; /* number of negative entries allocated */ 199 SYSCTL_ULONG(_debug, OID_AUTO, numneg, CTLFLAG_RD, &numneg, 0, 200 "Number of negative entries in namecache"); 201 static u_long numcache; /* number of cache entries allocated */ 202 SYSCTL_ULONG(_debug, OID_AUTO, numcache, CTLFLAG_RD, &numcache, 0, 203 "Number of namecache entries"); 204 static u_long numcachehv; /* number of cache entries with vnodes held */ 205 SYSCTL_ULONG(_debug, OID_AUTO, numcachehv, CTLFLAG_RD, &numcachehv, 0, 206 "Number of namecache entries with vnodes held"); 207 u_int ncsizefactor = 2; 208 SYSCTL_UINT(_vfs, OID_AUTO, ncsizefactor, CTLFLAG_RW, &ncsizefactor, 0, 209 "Size factor for namecache"); 210 211 struct nchstats nchstats; /* cache effectiveness statistics */ 212 213 static struct mtx ncneg_shrink_lock; 214 MTX_SYSINIT(vfscache_shrink_neg, &ncneg_shrink_lock, "Name Cache shrink neg", 215 MTX_DEF); 216 217 static struct mtx_padalign ncneg_mtx; 218 MTX_SYSINIT(vfscache_neg, &ncneg_mtx, "ncneg", MTX_DEF); 219 220 static u_int numbucketlocks; 221 static struct rwlock_padalign *bucketlocks; 222 #define HASH2BUCKETLOCK(hash) \ 223 ((struct rwlock *)(&bucketlocks[((hash) % numbucketlocks)])) 224 225 static u_int numvnodelocks; 226 static struct mtx *vnodelocks; 227 static inline struct mtx * 228 VP2VNODELOCK(struct vnode *vp) 229 { 230 struct mtx *vlp; 231 232 if (vp == NULL) 233 return (NULL); 234 vlp = &vnodelocks[(((uintptr_t)(vp) >> 8) % numvnodelocks)]; 235 return (vlp); 236 } 237 238 /* 239 * UMA zones for the VFS cache. 240 * 241 * The small cache is used for entries with short names, which are the 242 * most common. The large cache is used for entries which are too big to 243 * fit in the small cache. 244 */ 245 static uma_zone_t cache_zone_small; 246 static uma_zone_t cache_zone_small_ts; 247 static uma_zone_t cache_zone_large; 248 static uma_zone_t cache_zone_large_ts; 249 250 #define CACHE_PATH_CUTOFF 35 251 252 static struct namecache * 253 cache_alloc(int len, int ts) 254 { 255 256 if (len > CACHE_PATH_CUTOFF) { 257 if (ts) 258 return (uma_zalloc(cache_zone_large_ts, M_WAITOK)); 259 else 260 return (uma_zalloc(cache_zone_large, M_WAITOK)); 261 } 262 if (ts) 263 return (uma_zalloc(cache_zone_small_ts, M_WAITOK)); 264 else 265 return (uma_zalloc(cache_zone_small, M_WAITOK)); 266 } 267 268 static void 269 cache_free(struct namecache *ncp) 270 { 271 int ts; 272 273 if (ncp == NULL) 274 return; 275 ts = ncp->nc_flag & NCF_TS; 276 if ((ncp->nc_flag & NCF_DVDROP) != 0) 277 vdrop(ncp->nc_dvp); 278 if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) { 279 if (ts) 280 uma_zfree(cache_zone_small_ts, ncp); 281 else 282 uma_zfree(cache_zone_small, ncp); 283 } else if (ts) 284 uma_zfree(cache_zone_large_ts, ncp); 285 else 286 uma_zfree(cache_zone_large, ncp); 287 } 288 289 static char * 290 nc_get_name(struct namecache *ncp) 291 { 292 struct namecache_ts *ncp_ts; 293 294 if ((ncp->nc_flag & NCF_TS) == 0) 295 return (ncp->nc_name); 296 ncp_ts = (struct namecache_ts *)ncp; 297 return (ncp_ts->nc_name); 298 } 299 300 static void 301 cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp) 302 { 303 304 KASSERT((ncp->nc_flag & NCF_TS) != 0 || 305 (tsp == NULL && ticksp == NULL), 306 ("No NCF_TS")); 307 308 if (tsp != NULL) 309 *tsp = ((struct namecache_ts *)ncp)->nc_time; 310 if (ticksp != NULL) 311 *ticksp = ((struct namecache_ts *)ncp)->nc_ticks; 312 } 313 314 static int doingcache = 1; /* 1 => enable the cache */ 315 SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0, 316 "VFS namecache enabled"); 317 318 /* Export size information to userland */ 319 SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, 320 sizeof(struct namecache), "sizeof(struct namecache)"); 321 322 /* 323 * The new name cache statistics 324 */ 325 static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW, 0, 326 "Name cache statistics"); 327 #define STATNODE_ULONG(name, descr) \ 328 SYSCTL_ULONG(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, 0, descr); 329 #define STATNODE_COUNTER(name, descr) \ 330 static counter_u64_t name; \ 331 SYSCTL_COUNTER_U64(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, descr); 332 STATNODE_ULONG(numneg, "Number of negative cache entries"); 333 STATNODE_ULONG(numcache, "Number of cache entries"); 334 STATNODE_COUNTER(numcalls, "Number of cache lookups"); 335 STATNODE_COUNTER(dothits, "Number of '.' hits"); 336 STATNODE_COUNTER(dotdothits, "Number of '..' hits"); 337 STATNODE_COUNTER(numchecks, "Number of checks in lookup"); 338 STATNODE_COUNTER(nummiss, "Number of cache misses"); 339 STATNODE_COUNTER(nummisszap, "Number of cache misses we do not want to cache"); 340 STATNODE_COUNTER(numposzaps, 341 "Number of cache hits (positive) we do not want to cache"); 342 STATNODE_COUNTER(numposhits, "Number of cache hits (positive)"); 343 STATNODE_COUNTER(numnegzaps, 344 "Number of cache hits (negative) we do not want to cache"); 345 STATNODE_COUNTER(numneghits, "Number of cache hits (negative)"); 346 /* These count for kern___getcwd(), too. */ 347 STATNODE_COUNTER(numfullpathcalls, "Number of fullpath search calls"); 348 STATNODE_COUNTER(numfullpathfail1, "Number of fullpath search errors (ENOTDIR)"); 349 STATNODE_COUNTER(numfullpathfail2, 350 "Number of fullpath search errors (VOP_VPTOCNP failures)"); 351 STATNODE_COUNTER(numfullpathfail4, "Number of fullpath search errors (ENOMEM)"); 352 STATNODE_COUNTER(numfullpathfound, "Number of successful fullpath calls"); 353 static long zap_and_exit_bucket_fail; STATNODE_ULONG(zap_and_exit_bucket_fail, 354 "Number of times zap_and_exit failed to lock"); 355 static long cache_lock_vnodes_cel_3_failures; 356 STATNODE_ULONG(cache_lock_vnodes_cel_3_failures, 357 "Number of times 3-way vnode locking failed"); 358 359 static void cache_zap_locked(struct namecache *ncp, bool neg_locked); 360 static int vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir, 361 char *buf, char **retbuf, u_int buflen); 362 363 static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries"); 364 365 static int cache_yield; 366 SYSCTL_INT(_vfs_cache, OID_AUTO, yield, CTLFLAG_RD, &cache_yield, 0, 367 "Number of times cache called yield"); 368 369 static void 370 cache_maybe_yield(void) 371 { 372 373 if (should_yield()) { 374 cache_yield++; 375 kern_yield(PRI_USER); 376 } 377 } 378 379 static inline void 380 cache_assert_vlp_locked(struct mtx *vlp) 381 { 382 383 if (vlp != NULL) 384 mtx_assert(vlp, MA_OWNED); 385 } 386 387 static inline void 388 cache_assert_vnode_locked(struct vnode *vp) 389 { 390 struct mtx *vlp; 391 392 vlp = VP2VNODELOCK(vp); 393 cache_assert_vlp_locked(vlp); 394 } 395 396 static uint32_t 397 cache_get_hash(char *name, u_char len, struct vnode *dvp) 398 { 399 uint32_t hash; 400 401 hash = fnv_32_buf(name, len, FNV1_32_INIT); 402 hash = fnv_32_buf(&dvp, sizeof(dvp), hash); 403 return (hash); 404 } 405 406 static inline struct rwlock * 407 NCP2BUCKETLOCK(struct namecache *ncp) 408 { 409 uint32_t hash; 410 411 hash = cache_get_hash(nc_get_name(ncp), ncp->nc_nlen, ncp->nc_dvp); 412 return (HASH2BUCKETLOCK(hash)); 413 } 414 415 #ifdef INVARIANTS 416 static void 417 cache_assert_bucket_locked(struct namecache *ncp, int mode) 418 { 419 struct rwlock *blp; 420 421 blp = NCP2BUCKETLOCK(ncp); 422 rw_assert(blp, mode); 423 } 424 #else 425 #define cache_assert_bucket_locked(x, y) do { } while (0) 426 #endif 427 428 #define cache_sort(x, y) _cache_sort((void **)(x), (void **)(y)) 429 static void 430 _cache_sort(void **p1, void **p2) 431 { 432 void *tmp; 433 434 if (*p1 > *p2) { 435 tmp = *p2; 436 *p2 = *p1; 437 *p1 = tmp; 438 } 439 } 440 441 static void 442 cache_lock_all_buckets(void) 443 { 444 u_int i; 445 446 for (i = 0; i < numbucketlocks; i++) 447 rw_wlock(&bucketlocks[i]); 448 } 449 450 static void 451 cache_unlock_all_buckets(void) 452 { 453 u_int i; 454 455 for (i = 0; i < numbucketlocks; i++) 456 rw_wunlock(&bucketlocks[i]); 457 } 458 459 static void 460 cache_lock_all_vnodes(void) 461 { 462 u_int i; 463 464 for (i = 0; i < numvnodelocks; i++) 465 mtx_lock(&vnodelocks[i]); 466 } 467 468 static void 469 cache_unlock_all_vnodes(void) 470 { 471 u_int i; 472 473 for (i = 0; i < numvnodelocks; i++) 474 mtx_unlock(&vnodelocks[i]); 475 } 476 477 static int 478 cache_trylock_vnodes(struct mtx *vlp1, struct mtx *vlp2) 479 { 480 481 cache_sort(&vlp1, &vlp2); 482 MPASS(vlp2 != NULL); 483 484 if (vlp1 != NULL) { 485 if (!mtx_trylock(vlp1)) 486 return (EAGAIN); 487 } 488 if (!mtx_trylock(vlp2)) { 489 if (vlp1 != NULL) 490 mtx_unlock(vlp1); 491 return (EAGAIN); 492 } 493 494 return (0); 495 } 496 497 static void 498 cache_unlock_vnodes(struct mtx *vlp1, struct mtx *vlp2) 499 { 500 501 MPASS(vlp1 != NULL || vlp2 != NULL); 502 503 if (vlp1 != NULL) 504 mtx_unlock(vlp1); 505 if (vlp2 != NULL) 506 mtx_unlock(vlp2); 507 } 508 509 static int 510 sysctl_nchstats(SYSCTL_HANDLER_ARGS) 511 { 512 struct nchstats snap; 513 514 if (req->oldptr == NULL) 515 return (SYSCTL_OUT(req, 0, sizeof(snap))); 516 517 snap = nchstats; 518 snap.ncs_goodhits = counter_u64_fetch(numposhits); 519 snap.ncs_neghits = counter_u64_fetch(numneghits); 520 snap.ncs_badhits = counter_u64_fetch(numposzaps) + 521 counter_u64_fetch(numnegzaps); 522 snap.ncs_miss = counter_u64_fetch(nummisszap) + 523 counter_u64_fetch(nummiss); 524 525 return (SYSCTL_OUT(req, &snap, sizeof(snap))); 526 } 527 SYSCTL_PROC(_vfs_cache, OID_AUTO, nchstats, CTLTYPE_OPAQUE | CTLFLAG_RD | 528 CTLFLAG_MPSAFE, 0, 0, sysctl_nchstats, "LU", 529 "VFS cache effectiveness statistics"); 530 531 #ifdef DIAGNOSTIC 532 /* 533 * Grab an atomic snapshot of the name cache hash chain lengths 534 */ 535 static SYSCTL_NODE(_debug, OID_AUTO, hashstat, CTLFLAG_RW, NULL, 536 "hash table stats"); 537 538 static int 539 sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS) 540 { 541 struct nchashhead *ncpp; 542 struct namecache *ncp; 543 int i, error, n_nchash, *cntbuf; 544 545 retry: 546 n_nchash = nchash + 1; /* nchash is max index, not count */ 547 if (req->oldptr == NULL) 548 return SYSCTL_OUT(req, 0, n_nchash * sizeof(int)); 549 cntbuf = malloc(n_nchash * sizeof(int), M_TEMP, M_ZERO | M_WAITOK); 550 cache_lock_all_buckets(); 551 if (n_nchash != nchash + 1) { 552 cache_unlock_all_buckets(); 553 free(cntbuf, M_TEMP); 554 goto retry; 555 } 556 /* Scan hash tables counting entries */ 557 for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++) 558 LIST_FOREACH(ncp, ncpp, nc_hash) 559 cntbuf[i]++; 560 cache_unlock_all_buckets(); 561 for (error = 0, i = 0; i < n_nchash; i++) 562 if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0) 563 break; 564 free(cntbuf, M_TEMP); 565 return (error); 566 } 567 SYSCTL_PROC(_debug_hashstat, OID_AUTO, rawnchash, CTLTYPE_INT|CTLFLAG_RD| 568 CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_rawnchash, "S,int", 569 "nchash chain lengths"); 570 571 static int 572 sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS) 573 { 574 int error; 575 struct nchashhead *ncpp; 576 struct namecache *ncp; 577 int n_nchash; 578 int count, maxlength, used, pct; 579 580 if (!req->oldptr) 581 return SYSCTL_OUT(req, 0, 4 * sizeof(int)); 582 583 cache_lock_all_buckets(); 584 n_nchash = nchash + 1; /* nchash is max index, not count */ 585 used = 0; 586 maxlength = 0; 587 588 /* Scan hash tables for applicable entries */ 589 for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) { 590 count = 0; 591 LIST_FOREACH(ncp, ncpp, nc_hash) { 592 count++; 593 } 594 if (count) 595 used++; 596 if (maxlength < count) 597 maxlength = count; 598 } 599 n_nchash = nchash + 1; 600 cache_unlock_all_buckets(); 601 pct = (used * 100) / (n_nchash / 100); 602 error = SYSCTL_OUT(req, &n_nchash, sizeof(n_nchash)); 603 if (error) 604 return (error); 605 error = SYSCTL_OUT(req, &used, sizeof(used)); 606 if (error) 607 return (error); 608 error = SYSCTL_OUT(req, &maxlength, sizeof(maxlength)); 609 if (error) 610 return (error); 611 error = SYSCTL_OUT(req, &pct, sizeof(pct)); 612 if (error) 613 return (error); 614 return (0); 615 } 616 SYSCTL_PROC(_debug_hashstat, OID_AUTO, nchash, CTLTYPE_INT|CTLFLAG_RD| 617 CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_nchash, "I", 618 "nchash statistics (number of total/used buckets, maximum chain length, usage percentage)"); 619 #endif 620 621 /* 622 * Negative entries management 623 */ 624 static void 625 cache_negative_hit(struct namecache *ncp) 626 { 627 628 MPASS(ncp->nc_vp == NULL); 629 mtx_lock(&ncneg_mtx); 630 TAILQ_REMOVE(&ncneg, ncp, nc_dst); 631 TAILQ_INSERT_TAIL(&ncneg, ncp, nc_dst); 632 mtx_unlock(&ncneg_mtx); 633 } 634 635 static void 636 cache_negative_insert(struct namecache *ncp) 637 { 638 639 MPASS(ncp->nc_vp == NULL); 640 cache_assert_bucket_locked(ncp, RA_WLOCKED); 641 mtx_lock(&ncneg_mtx); 642 TAILQ_INSERT_TAIL(&ncneg, ncp, nc_dst); 643 numneg++; 644 mtx_unlock(&ncneg_mtx); 645 } 646 647 static void 648 cache_negative_remove(struct namecache *ncp, bool neg_locked) 649 { 650 651 MPASS(ncp->nc_vp == NULL); 652 cache_assert_bucket_locked(ncp, RA_WLOCKED); 653 if (!neg_locked) 654 mtx_lock(&ncneg_mtx); 655 else 656 mtx_assert(&ncneg_mtx, MA_OWNED); 657 TAILQ_REMOVE(&ncneg, ncp, nc_dst); 658 numneg--; 659 if (!neg_locked) 660 mtx_unlock(&ncneg_mtx); 661 } 662 663 static void 664 cache_negative_zap_one(void) 665 { 666 struct namecache *ncp, *ncp2; 667 struct mtx *dvlp; 668 struct rwlock *blp; 669 670 if (!mtx_trylock(&ncneg_shrink_lock)) 671 return; 672 673 mtx_lock(&ncneg_mtx); 674 ncp = TAILQ_FIRST(&ncneg); 675 if (ncp == NULL) { 676 mtx_unlock(&ncneg_mtx); 677 goto out; 678 } 679 MPASS(ncp->nc_vp == NULL); 680 dvlp = VP2VNODELOCK(ncp->nc_dvp); 681 blp = NCP2BUCKETLOCK(ncp); 682 mtx_unlock(&ncneg_mtx); 683 mtx_lock(dvlp); 684 rw_wlock(blp); 685 mtx_lock(&ncneg_mtx); 686 ncp2 = TAILQ_FIRST(&ncneg); 687 if (ncp != ncp2 || dvlp != VP2VNODELOCK(ncp2->nc_dvp) || 688 blp != NCP2BUCKETLOCK(ncp2) || ncp2->nc_vp != NULL) { 689 ncp = NULL; 690 goto out_unlock_all; 691 } 692 cache_zap_locked(ncp, true); 693 out_unlock_all: 694 mtx_unlock(&ncneg_mtx); 695 rw_wunlock(blp); 696 mtx_unlock(dvlp); 697 out: 698 mtx_unlock(&ncneg_shrink_lock); 699 cache_free(ncp); 700 } 701 702 /* 703 * cache_zap_locked(): 704 * 705 * Removes a namecache entry from cache, whether it contains an actual 706 * pointer to a vnode or if it is just a negative cache entry. 707 */ 708 static void 709 cache_zap_locked(struct namecache *ncp, bool neg_locked) 710 { 711 712 cache_assert_vnode_locked(ncp->nc_vp); 713 cache_assert_vnode_locked(ncp->nc_dvp); 714 cache_assert_bucket_locked(ncp, RA_WLOCKED); 715 716 CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp, ncp->nc_vp); 717 if (ncp->nc_vp != NULL) { 718 SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp, 719 nc_get_name(ncp), ncp->nc_vp); 720 } else { 721 SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp, 722 nc_get_name(ncp)); 723 } 724 LIST_REMOVE(ncp, nc_hash); 725 if (ncp->nc_flag & NCF_ISDOTDOT) { 726 if (ncp == ncp->nc_dvp->v_cache_dd) 727 ncp->nc_dvp->v_cache_dd = NULL; 728 } else { 729 LIST_REMOVE(ncp, nc_src); 730 if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) { 731 ncp->nc_flag |= NCF_DVDROP; 732 atomic_subtract_rel_long(&numcachehv, 1); 733 } 734 } 735 if (ncp->nc_vp) { 736 TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst); 737 if (ncp == ncp->nc_vp->v_cache_dd) 738 ncp->nc_vp->v_cache_dd = NULL; 739 } else { 740 cache_negative_remove(ncp, neg_locked); 741 } 742 atomic_subtract_rel_long(&numcache, 1); 743 } 744 745 static void 746 cache_zap_negative_locked_vnode_kl(struct namecache *ncp, struct vnode *vp) 747 { 748 struct rwlock *blp; 749 750 MPASS(ncp->nc_dvp == vp); 751 MPASS(ncp->nc_vp == NULL); 752 cache_assert_vnode_locked(vp); 753 754 blp = NCP2BUCKETLOCK(ncp); 755 rw_wlock(blp); 756 cache_zap_locked(ncp, false); 757 rw_wunlock(blp); 758 } 759 760 static bool 761 cache_zap_locked_vnode_kl2(struct namecache *ncp, struct vnode *vp, 762 struct mtx **vlpp) 763 { 764 struct mtx *pvlp, *vlp1, *vlp2, *to_unlock; 765 struct rwlock *blp; 766 767 MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp); 768 cache_assert_vnode_locked(vp); 769 770 if (ncp->nc_vp == NULL) { 771 if (*vlpp != NULL) { 772 mtx_unlock(*vlpp); 773 *vlpp = NULL; 774 } 775 cache_zap_negative_locked_vnode_kl(ncp, vp); 776 return (true); 777 } 778 779 pvlp = VP2VNODELOCK(vp); 780 blp = NCP2BUCKETLOCK(ncp); 781 vlp1 = VP2VNODELOCK(ncp->nc_dvp); 782 vlp2 = VP2VNODELOCK(ncp->nc_vp); 783 784 if (*vlpp == vlp1 || *vlpp == vlp2) { 785 to_unlock = *vlpp; 786 *vlpp = NULL; 787 } else { 788 if (*vlpp != NULL) { 789 mtx_unlock(*vlpp); 790 *vlpp = NULL; 791 } 792 cache_sort(&vlp1, &vlp2); 793 if (vlp1 == pvlp) { 794 mtx_lock(vlp2); 795 to_unlock = vlp2; 796 } else { 797 if (!mtx_trylock(vlp1)) 798 goto out_relock; 799 to_unlock = vlp1; 800 } 801 } 802 rw_wlock(blp); 803 cache_zap_locked(ncp, false); 804 rw_wunlock(blp); 805 if (to_unlock != NULL) 806 mtx_unlock(to_unlock); 807 return (true); 808 809 out_relock: 810 mtx_unlock(vlp2); 811 mtx_lock(vlp1); 812 mtx_lock(vlp2); 813 MPASS(*vlpp == NULL); 814 *vlpp = vlp1; 815 return (false); 816 } 817 818 static int 819 cache_zap_locked_vnode(struct namecache *ncp, struct vnode *vp) 820 { 821 struct mtx *pvlp, *vlp1, *vlp2, *to_unlock; 822 struct rwlock *blp; 823 int error = 0; 824 825 MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp); 826 cache_assert_vnode_locked(vp); 827 828 pvlp = VP2VNODELOCK(vp); 829 if (ncp->nc_vp == NULL) { 830 cache_zap_negative_locked_vnode_kl(ncp, vp); 831 goto out; 832 } 833 834 blp = NCP2BUCKETLOCK(ncp); 835 vlp1 = VP2VNODELOCK(ncp->nc_dvp); 836 vlp2 = VP2VNODELOCK(ncp->nc_vp); 837 cache_sort(&vlp1, &vlp2); 838 if (vlp1 == pvlp) { 839 mtx_lock(vlp2); 840 to_unlock = vlp2; 841 } else { 842 if (!mtx_trylock(vlp1)) { 843 error = EAGAIN; 844 goto out; 845 } 846 to_unlock = vlp1; 847 } 848 rw_wlock(blp); 849 cache_zap_locked(ncp, false); 850 rw_wunlock(blp); 851 mtx_unlock(to_unlock); 852 out: 853 mtx_unlock(pvlp); 854 return (error); 855 } 856 857 static int 858 cache_zap_rlocked_bucket(struct namecache *ncp, struct rwlock *blp) 859 { 860 struct mtx *dvlp, *vlp; 861 862 cache_assert_bucket_locked(ncp, RA_RLOCKED); 863 864 dvlp = VP2VNODELOCK(ncp->nc_dvp); 865 vlp = VP2VNODELOCK(ncp->nc_vp); 866 if (cache_trylock_vnodes(dvlp, vlp) == 0) { 867 rw_runlock(blp); 868 rw_wlock(blp); 869 cache_zap_locked(ncp, false); 870 rw_wunlock(blp); 871 cache_unlock_vnodes(dvlp, vlp); 872 return (0); 873 } 874 875 rw_runlock(blp); 876 return (EAGAIN); 877 } 878 879 static int 880 cache_zap_wlocked_bucket_kl(struct namecache *ncp, struct rwlock *blp, 881 struct mtx **vlpp1, struct mtx **vlpp2) 882 { 883 struct mtx *dvlp, *vlp; 884 885 cache_assert_bucket_locked(ncp, RA_WLOCKED); 886 887 dvlp = VP2VNODELOCK(ncp->nc_dvp); 888 vlp = VP2VNODELOCK(ncp->nc_vp); 889 cache_sort(&dvlp, &vlp); 890 891 if (*vlpp1 == dvlp && *vlpp2 == vlp) { 892 cache_zap_locked(ncp, false); 893 cache_unlock_vnodes(dvlp, vlp); 894 *vlpp1 = NULL; 895 *vlpp2 = NULL; 896 return (0); 897 } 898 899 if (*vlpp1 != NULL) 900 mtx_unlock(*vlpp1); 901 if (*vlpp2 != NULL) 902 mtx_unlock(*vlpp2); 903 *vlpp1 = NULL; 904 *vlpp2 = NULL; 905 906 if (cache_trylock_vnodes(dvlp, vlp) == 0) { 907 cache_zap_locked(ncp, false); 908 cache_unlock_vnodes(dvlp, vlp); 909 return (0); 910 } 911 912 rw_wunlock(blp); 913 *vlpp1 = dvlp; 914 *vlpp2 = vlp; 915 if (*vlpp1 != NULL) 916 mtx_lock(*vlpp1); 917 mtx_lock(*vlpp2); 918 rw_wlock(blp); 919 return (EAGAIN); 920 } 921 922 static void 923 cache_lookup_unlock(struct rwlock *blp, struct mtx *vlp) 924 { 925 926 if (blp != NULL) { 927 rw_runlock(blp); 928 mtx_assert(vlp, MA_NOTOWNED); 929 } else { 930 mtx_unlock(vlp); 931 } 932 } 933 934 /* 935 * Lookup an entry in the cache 936 * 937 * Lookup is called with dvp pointing to the directory to search, 938 * cnp pointing to the name of the entry being sought. If the lookup 939 * succeeds, the vnode is returned in *vpp, and a status of -1 is 940 * returned. If the lookup determines that the name does not exist 941 * (negative caching), a status of ENOENT is returned. If the lookup 942 * fails, a status of zero is returned. If the directory vnode is 943 * recycled out from under us due to a forced unmount, a status of 944 * ENOENT is returned. 945 * 946 * vpp is locked and ref'd on return. If we're looking up DOTDOT, dvp is 947 * unlocked. If we're looking up . an extra ref is taken, but the lock is 948 * not recursively acquired. 949 */ 950 951 int 952 cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, 953 struct timespec *tsp, int *ticksp) 954 { 955 struct namecache *ncp; 956 struct rwlock *blp; 957 struct mtx *dvlp, *dvlp2; 958 uint32_t hash; 959 int error, ltype; 960 961 if (!doingcache) { 962 cnp->cn_flags &= ~MAKEENTRY; 963 return (0); 964 } 965 retry: 966 blp = NULL; 967 dvlp = VP2VNODELOCK(dvp); 968 error = 0; 969 counter_u64_add(numcalls, 1); 970 971 if (cnp->cn_nameptr[0] == '.') { 972 if (cnp->cn_namelen == 1) { 973 *vpp = dvp; 974 CTR2(KTR_VFS, "cache_lookup(%p, %s) found via .", 975 dvp, cnp->cn_nameptr); 976 counter_u64_add(dothits, 1); 977 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp); 978 if (tsp != NULL) 979 timespecclear(tsp); 980 if (ticksp != NULL) 981 *ticksp = ticks; 982 VREF(*vpp); 983 /* 984 * When we lookup "." we still can be asked to lock it 985 * differently... 986 */ 987 ltype = cnp->cn_lkflags & LK_TYPE_MASK; 988 if (ltype != VOP_ISLOCKED(*vpp)) { 989 if (ltype == LK_EXCLUSIVE) { 990 vn_lock(*vpp, LK_UPGRADE | LK_RETRY); 991 if ((*vpp)->v_iflag & VI_DOOMED) { 992 /* forced unmount */ 993 vrele(*vpp); 994 *vpp = NULL; 995 return (ENOENT); 996 } 997 } else 998 vn_lock(*vpp, LK_DOWNGRADE | LK_RETRY); 999 } 1000 return (-1); 1001 } 1002 if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') { 1003 counter_u64_add(dotdothits, 1); 1004 dvlp2 = NULL; 1005 mtx_lock(dvlp); 1006 retry_dotdot: 1007 ncp = dvp->v_cache_dd; 1008 if (ncp == NULL) { 1009 SDT_PROBE3(vfs, namecache, lookup, miss, dvp, 1010 "..", NULL); 1011 mtx_unlock(dvlp); 1012 return (0); 1013 } 1014 if ((cnp->cn_flags & MAKEENTRY) == 0) { 1015 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) { 1016 if (ncp->nc_dvp != dvp) 1017 panic("dvp %p v_cache_dd %p\n", dvp, ncp); 1018 if (!cache_zap_locked_vnode_kl2(ncp, 1019 dvp, &dvlp2)) 1020 goto retry_dotdot; 1021 MPASS(dvp->v_cache_dd == NULL); 1022 mtx_unlock(dvlp); 1023 if (dvlp2 != NULL) 1024 mtx_unlock(dvlp2); 1025 cache_free(ncp); 1026 } else { 1027 dvp->v_cache_dd = NULL; 1028 mtx_unlock(dvlp); 1029 if (dvlp2 != NULL) 1030 mtx_unlock(dvlp2); 1031 } 1032 return (0); 1033 } 1034 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) 1035 *vpp = ncp->nc_vp; 1036 else 1037 *vpp = ncp->nc_dvp; 1038 /* Return failure if negative entry was found. */ 1039 if (*vpp == NULL) 1040 goto negative_success; 1041 CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..", 1042 dvp, cnp->cn_nameptr, *vpp); 1043 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..", 1044 *vpp); 1045 cache_out_ts(ncp, tsp, ticksp); 1046 if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) == 1047 NCF_DTS && tsp != NULL) 1048 *tsp = ((struct namecache_ts *)ncp)-> 1049 nc_dotdottime; 1050 goto success; 1051 } 1052 } 1053 1054 hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp); 1055 blp = HASH2BUCKETLOCK(hash); 1056 rw_rlock(blp); 1057 1058 LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { 1059 counter_u64_add(numchecks, 1); 1060 if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && 1061 !bcmp(nc_get_name(ncp), cnp->cn_nameptr, ncp->nc_nlen)) 1062 break; 1063 } 1064 1065 /* We failed to find an entry */ 1066 if (ncp == NULL) { 1067 SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr, 1068 NULL); 1069 if ((cnp->cn_flags & MAKEENTRY) == 0) { 1070 counter_u64_add(nummisszap, 1); 1071 } else { 1072 counter_u64_add(nummiss, 1); 1073 } 1074 goto unlock; 1075 } 1076 1077 /* We don't want to have an entry, so dump it */ 1078 if ((cnp->cn_flags & MAKEENTRY) == 0) { 1079 counter_u64_add(numposzaps, 1); 1080 goto zap_and_exit; 1081 } 1082 1083 /* We found a "positive" match, return the vnode */ 1084 if (ncp->nc_vp) { 1085 counter_u64_add(numposhits, 1); 1086 *vpp = ncp->nc_vp; 1087 CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p", 1088 dvp, cnp->cn_nameptr, *vpp, ncp); 1089 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, nc_get_name(ncp), 1090 *vpp); 1091 cache_out_ts(ncp, tsp, ticksp); 1092 goto success; 1093 } 1094 1095 negative_success: 1096 /* We found a negative match, and want to create it, so purge */ 1097 if (cnp->cn_nameiop == CREATE) { 1098 counter_u64_add(numnegzaps, 1); 1099 goto zap_and_exit; 1100 } 1101 1102 counter_u64_add(numneghits, 1); 1103 cache_negative_hit(ncp); 1104 if (ncp->nc_flag & NCF_WHITE) 1105 cnp->cn_flags |= ISWHITEOUT; 1106 SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, 1107 nc_get_name(ncp)); 1108 cache_out_ts(ncp, tsp, ticksp); 1109 cache_lookup_unlock(blp, dvlp); 1110 return (ENOENT); 1111 1112 success: 1113 /* 1114 * On success we return a locked and ref'd vnode as per the lookup 1115 * protocol. 1116 */ 1117 MPASS(dvp != *vpp); 1118 ltype = 0; /* silence gcc warning */ 1119 if (cnp->cn_flags & ISDOTDOT) { 1120 ltype = VOP_ISLOCKED(dvp); 1121 VOP_UNLOCK(dvp, 0); 1122 } 1123 vhold(*vpp); 1124 cache_lookup_unlock(blp, dvlp); 1125 error = vget(*vpp, cnp->cn_lkflags | LK_VNHELD, cnp->cn_thread); 1126 if (cnp->cn_flags & ISDOTDOT) { 1127 vn_lock(dvp, ltype | LK_RETRY); 1128 if (dvp->v_iflag & VI_DOOMED) { 1129 if (error == 0) 1130 vput(*vpp); 1131 *vpp = NULL; 1132 return (ENOENT); 1133 } 1134 } 1135 if (error) { 1136 *vpp = NULL; 1137 goto retry; 1138 } 1139 if ((cnp->cn_flags & ISLASTCN) && 1140 (cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) { 1141 ASSERT_VOP_ELOCKED(*vpp, "cache_lookup"); 1142 } 1143 return (-1); 1144 1145 unlock: 1146 cache_lookup_unlock(blp, dvlp); 1147 return (0); 1148 1149 zap_and_exit: 1150 if (blp != NULL) 1151 error = cache_zap_rlocked_bucket(ncp, blp); 1152 else 1153 error = cache_zap_locked_vnode(ncp, dvp); 1154 if (error != 0) { 1155 zap_and_exit_bucket_fail++; 1156 cache_maybe_yield(); 1157 goto retry; 1158 } 1159 cache_free(ncp); 1160 return (0); 1161 } 1162 1163 struct celockstate { 1164 struct mtx *vlp[3]; 1165 struct rwlock *blp[2]; 1166 }; 1167 CTASSERT((nitems(((struct celockstate *)0)->vlp) == 3)); 1168 CTASSERT((nitems(((struct celockstate *)0)->blp) == 2)); 1169 1170 static inline void 1171 cache_celockstate_init(struct celockstate *cel) 1172 { 1173 1174 bzero(cel, sizeof(*cel)); 1175 } 1176 1177 static void 1178 cache_lock_vnodes_cel(struct celockstate *cel, struct vnode *vp, 1179 struct vnode *dvp) 1180 { 1181 struct mtx *vlp1, *vlp2; 1182 1183 MPASS(cel->vlp[0] == NULL); 1184 MPASS(cel->vlp[1] == NULL); 1185 MPASS(cel->vlp[2] == NULL); 1186 1187 MPASS(vp != NULL || dvp != NULL); 1188 1189 vlp1 = VP2VNODELOCK(vp); 1190 vlp2 = VP2VNODELOCK(dvp); 1191 cache_sort(&vlp1, &vlp2); 1192 1193 if (vlp1 != NULL) { 1194 mtx_lock(vlp1); 1195 cel->vlp[0] = vlp1; 1196 } 1197 mtx_lock(vlp2); 1198 cel->vlp[1] = vlp2; 1199 } 1200 1201 static void 1202 cache_unlock_vnodes_cel(struct celockstate *cel) 1203 { 1204 1205 MPASS(cel->vlp[0] != NULL || cel->vlp[1] != NULL); 1206 1207 if (cel->vlp[0] != NULL) 1208 mtx_unlock(cel->vlp[0]); 1209 if (cel->vlp[1] != NULL) 1210 mtx_unlock(cel->vlp[1]); 1211 if (cel->vlp[2] != NULL) 1212 mtx_unlock(cel->vlp[2]); 1213 } 1214 1215 static bool 1216 cache_lock_vnodes_cel_3(struct celockstate *cel, struct vnode *vp) 1217 { 1218 struct mtx *vlp; 1219 bool ret; 1220 1221 cache_assert_vlp_locked(cel->vlp[0]); 1222 cache_assert_vlp_locked(cel->vlp[1]); 1223 MPASS(cel->vlp[2] == NULL); 1224 1225 vlp = VP2VNODELOCK(vp); 1226 if (vlp == NULL) 1227 return (true); 1228 1229 ret = true; 1230 if (vlp >= cel->vlp[1]) { 1231 mtx_lock(vlp); 1232 } else { 1233 if (mtx_trylock(vlp)) 1234 goto out; 1235 cache_lock_vnodes_cel_3_failures++; 1236 cache_unlock_vnodes_cel(cel); 1237 if (vlp < cel->vlp[0]) { 1238 mtx_lock(vlp); 1239 mtx_lock(cel->vlp[0]); 1240 mtx_lock(cel->vlp[1]); 1241 } else { 1242 if (cel->vlp[0] != NULL) 1243 mtx_lock(cel->vlp[0]); 1244 mtx_lock(vlp); 1245 mtx_lock(cel->vlp[1]); 1246 } 1247 ret = false; 1248 } 1249 out: 1250 cel->vlp[2] = vlp; 1251 return (ret); 1252 } 1253 1254 static void 1255 cache_lock_buckets_cel(struct celockstate *cel, struct rwlock *blp1, 1256 struct rwlock *blp2) 1257 { 1258 1259 MPASS(cel->blp[0] == NULL); 1260 MPASS(cel->blp[1] == NULL); 1261 1262 cache_sort(&blp1, &blp2); 1263 1264 if (blp1 != NULL) { 1265 rw_wlock(blp1); 1266 cel->blp[0] = blp1; 1267 } 1268 rw_wlock(blp2); 1269 cel->blp[1] = blp2; 1270 } 1271 1272 static void 1273 cache_unlock_buckets_cel(struct celockstate *cel) 1274 { 1275 1276 if (cel->blp[0] != NULL) 1277 rw_wunlock(cel->blp[0]); 1278 rw_wunlock(cel->blp[1]); 1279 } 1280 1281 /* 1282 * Lock part of the cache affected by the insertion. 1283 * 1284 * This means vnodelocks for dvp, vp and the relevant bucketlock. 1285 * However, insertion can result in removal of an old entry. In this 1286 * case we have an additional vnode and bucketlock pair to lock. If the 1287 * entry is negative, ncelock is locked instead of the vnode. 1288 * 1289 * That is, in the worst case we have to lock 3 vnodes and 2 bucketlocks, while 1290 * preserving the locking order (smaller address first). 1291 */ 1292 static void 1293 cache_enter_lock(struct celockstate *cel, struct vnode *dvp, struct vnode *vp, 1294 uint32_t hash) 1295 { 1296 struct namecache *ncp; 1297 struct rwlock *blps[2]; 1298 1299 blps[0] = HASH2BUCKETLOCK(hash); 1300 for (;;) { 1301 blps[1] = NULL; 1302 cache_lock_vnodes_cel(cel, dvp, vp); 1303 if (vp == NULL || vp->v_type != VDIR) 1304 break; 1305 ncp = vp->v_cache_dd; 1306 if (ncp == NULL) 1307 break; 1308 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) 1309 break; 1310 MPASS(ncp->nc_dvp == vp); 1311 blps[1] = NCP2BUCKETLOCK(ncp); 1312 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp)) 1313 break; 1314 /* 1315 * All vnodes got re-locked. Re-validate the state and if 1316 * nothing changed we are done. Otherwise restart. 1317 */ 1318 if (ncp == vp->v_cache_dd && 1319 (ncp->nc_flag & NCF_ISDOTDOT) != 0 && 1320 blps[1] == NCP2BUCKETLOCK(ncp) && 1321 VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2]) 1322 break; 1323 cache_unlock_vnodes_cel(cel); 1324 cel->vlp[0] = NULL; 1325 cel->vlp[1] = NULL; 1326 cel->vlp[2] = NULL; 1327 } 1328 cache_lock_buckets_cel(cel, blps[0], blps[1]); 1329 } 1330 1331 static void 1332 cache_enter_lock_dd(struct celockstate *cel, struct vnode *dvp, struct vnode *vp, 1333 uint32_t hash) 1334 { 1335 struct namecache *ncp; 1336 struct rwlock *blps[2]; 1337 1338 blps[0] = HASH2BUCKETLOCK(hash); 1339 for (;;) { 1340 blps[1] = NULL; 1341 cache_lock_vnodes_cel(cel, dvp, vp); 1342 ncp = dvp->v_cache_dd; 1343 if (ncp == NULL) 1344 break; 1345 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) 1346 break; 1347 MPASS(ncp->nc_dvp == dvp); 1348 blps[1] = NCP2BUCKETLOCK(ncp); 1349 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp)) 1350 break; 1351 if (ncp == dvp->v_cache_dd && 1352 (ncp->nc_flag & NCF_ISDOTDOT) != 0 && 1353 blps[1] == NCP2BUCKETLOCK(ncp) && 1354 VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2]) 1355 break; 1356 cache_unlock_vnodes_cel(cel); 1357 cel->vlp[0] = NULL; 1358 cel->vlp[1] = NULL; 1359 cel->vlp[2] = NULL; 1360 } 1361 cache_lock_buckets_cel(cel, blps[0], blps[1]); 1362 } 1363 1364 static void 1365 cache_enter_unlock(struct celockstate *cel) 1366 { 1367 1368 cache_unlock_buckets_cel(cel); 1369 cache_unlock_vnodes_cel(cel); 1370 } 1371 1372 /* 1373 * Add an entry to the cache. 1374 */ 1375 void 1376 cache_enter_time(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, 1377 struct timespec *tsp, struct timespec *dtsp) 1378 { 1379 struct celockstate cel; 1380 struct namecache *ncp, *n2, *ndd; 1381 struct namecache_ts *n3; 1382 struct nchashhead *ncpp; 1383 uint32_t hash; 1384 int flag; 1385 int len; 1386 1387 CTR3(KTR_VFS, "cache_enter(%p, %p, %s)", dvp, vp, cnp->cn_nameptr); 1388 VNASSERT(vp == NULL || (vp->v_iflag & VI_DOOMED) == 0, vp, 1389 ("cache_enter: Adding a doomed vnode")); 1390 VNASSERT(dvp == NULL || (dvp->v_iflag & VI_DOOMED) == 0, dvp, 1391 ("cache_enter: Doomed vnode used as src")); 1392 1393 if (!doingcache) 1394 return; 1395 1396 /* 1397 * Avoid blowout in namecache entries. 1398 */ 1399 if (numcache >= desiredvnodes * ncsizefactor) 1400 return; 1401 1402 cache_celockstate_init(&cel); 1403 ndd = NULL; 1404 flag = 0; 1405 if (cnp->cn_nameptr[0] == '.') { 1406 if (cnp->cn_namelen == 1) 1407 return; 1408 if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') { 1409 len = cnp->cn_namelen; 1410 hash = cache_get_hash(cnp->cn_nameptr, len, dvp); 1411 cache_enter_lock_dd(&cel, dvp, vp, hash); 1412 /* 1413 * If dotdot entry already exists, just retarget it 1414 * to new parent vnode, otherwise continue with new 1415 * namecache entry allocation. 1416 */ 1417 if ((ncp = dvp->v_cache_dd) != NULL && 1418 ncp->nc_flag & NCF_ISDOTDOT) { 1419 KASSERT(ncp->nc_dvp == dvp, 1420 ("wrong isdotdot parent")); 1421 if (ncp->nc_vp != NULL) { 1422 TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, 1423 ncp, nc_dst); 1424 } else { 1425 cache_negative_remove(ncp, false); 1426 } 1427 if (vp != NULL) { 1428 TAILQ_INSERT_HEAD(&vp->v_cache_dst, 1429 ncp, nc_dst); 1430 } else { 1431 cache_negative_insert(ncp); 1432 } 1433 ncp->nc_vp = vp; 1434 cache_enter_unlock(&cel); 1435 return; 1436 } 1437 dvp->v_cache_dd = NULL; 1438 cache_enter_unlock(&cel); 1439 cache_celockstate_init(&cel); 1440 SDT_PROBE3(vfs, namecache, enter, done, dvp, "..", vp); 1441 flag = NCF_ISDOTDOT; 1442 } 1443 } 1444 1445 /* 1446 * Calculate the hash key and setup as much of the new 1447 * namecache entry as possible before acquiring the lock. 1448 */ 1449 ncp = cache_alloc(cnp->cn_namelen, tsp != NULL); 1450 ncp->nc_vp = vp; 1451 ncp->nc_dvp = dvp; 1452 ncp->nc_flag = flag; 1453 if (tsp != NULL) { 1454 n3 = (struct namecache_ts *)ncp; 1455 n3->nc_time = *tsp; 1456 n3->nc_ticks = ticks; 1457 n3->nc_flag |= NCF_TS; 1458 if (dtsp != NULL) { 1459 n3->nc_dotdottime = *dtsp; 1460 n3->nc_flag |= NCF_DTS; 1461 } 1462 } 1463 len = ncp->nc_nlen = cnp->cn_namelen; 1464 hash = cache_get_hash(cnp->cn_nameptr, len, dvp); 1465 strlcpy(nc_get_name(ncp), cnp->cn_nameptr, len + 1); 1466 cache_enter_lock(&cel, dvp, vp, hash); 1467 1468 /* 1469 * See if this vnode or negative entry is already in the cache 1470 * with this name. This can happen with concurrent lookups of 1471 * the same path name. 1472 */ 1473 ncpp = NCHHASH(hash); 1474 LIST_FOREACH(n2, ncpp, nc_hash) { 1475 if (n2->nc_dvp == dvp && 1476 n2->nc_nlen == cnp->cn_namelen && 1477 !bcmp(nc_get_name(n2), cnp->cn_nameptr, n2->nc_nlen)) { 1478 if (tsp != NULL) { 1479 KASSERT((n2->nc_flag & NCF_TS) != 0, 1480 ("no NCF_TS")); 1481 n3 = (struct namecache_ts *)n2; 1482 n3->nc_time = 1483 ((struct namecache_ts *)ncp)->nc_time; 1484 n3->nc_ticks = 1485 ((struct namecache_ts *)ncp)->nc_ticks; 1486 if (dtsp != NULL) { 1487 n3->nc_dotdottime = 1488 ((struct namecache_ts *)ncp)-> 1489 nc_dotdottime; 1490 n3->nc_flag |= NCF_DTS; 1491 } 1492 } 1493 goto out_unlock_free; 1494 } 1495 } 1496 1497 if (flag == NCF_ISDOTDOT) { 1498 /* 1499 * See if we are trying to add .. entry, but some other lookup 1500 * has populated v_cache_dd pointer already. 1501 */ 1502 if (dvp->v_cache_dd != NULL) 1503 goto out_unlock_free; 1504 KASSERT(vp == NULL || vp->v_type == VDIR, 1505 ("wrong vnode type %p", vp)); 1506 dvp->v_cache_dd = ncp; 1507 } 1508 1509 atomic_add_rel_long(&numcache, 1); 1510 if (vp != NULL) { 1511 if (vp->v_type == VDIR) { 1512 if (flag != NCF_ISDOTDOT) { 1513 /* 1514 * For this case, the cache entry maps both the 1515 * directory name in it and the name ".." for the 1516 * directory's parent. 1517 */ 1518 if ((ndd = vp->v_cache_dd) != NULL) { 1519 if ((ndd->nc_flag & NCF_ISDOTDOT) != 0) 1520 cache_zap_locked(ndd, false); 1521 else 1522 ndd = NULL; 1523 } 1524 vp->v_cache_dd = ncp; 1525 } 1526 } else { 1527 vp->v_cache_dd = NULL; 1528 } 1529 } 1530 1531 if (flag != NCF_ISDOTDOT) { 1532 if (LIST_EMPTY(&dvp->v_cache_src)) { 1533 vhold(dvp); 1534 atomic_add_rel_long(&numcachehv, 1); 1535 } 1536 LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src); 1537 } 1538 1539 /* 1540 * Insert the new namecache entry into the appropriate chain 1541 * within the cache entries table. 1542 */ 1543 LIST_INSERT_HEAD(ncpp, ncp, nc_hash); 1544 1545 /* 1546 * If the entry is "negative", we place it into the 1547 * "negative" cache queue, otherwise, we place it into the 1548 * destination vnode's cache entries queue. 1549 */ 1550 if (vp != NULL) { 1551 TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst); 1552 SDT_PROBE3(vfs, namecache, enter, done, dvp, nc_get_name(ncp), 1553 vp); 1554 } else { 1555 if (cnp->cn_flags & ISWHITEOUT) 1556 ncp->nc_flag |= NCF_WHITE; 1557 cache_negative_insert(ncp); 1558 SDT_PROBE2(vfs, namecache, enter_negative, done, dvp, 1559 nc_get_name(ncp)); 1560 } 1561 cache_enter_unlock(&cel); 1562 if (numneg * ncnegfactor > numcache) 1563 cache_negative_zap_one(); 1564 cache_free(ndd); 1565 return; 1566 out_unlock_free: 1567 cache_enter_unlock(&cel); 1568 cache_free(ncp); 1569 return; 1570 } 1571 1572 static u_int 1573 cache_roundup_2(u_int val) 1574 { 1575 u_int res; 1576 1577 for (res = 1; res <= val; res <<= 1) 1578 continue; 1579 1580 return (res); 1581 } 1582 1583 /* 1584 * Name cache initialization, from vfs_init() when we are booting 1585 */ 1586 static void 1587 nchinit(void *dummy __unused) 1588 { 1589 u_int i; 1590 1591 TAILQ_INIT(&ncneg); 1592 1593 cache_zone_small = uma_zcreate("S VFS Cache", 1594 sizeof(struct namecache) + CACHE_PATH_CUTOFF + 1, 1595 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 1596 cache_zone_small_ts = uma_zcreate("STS VFS Cache", 1597 sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1, 1598 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 1599 cache_zone_large = uma_zcreate("L VFS Cache", 1600 sizeof(struct namecache) + NAME_MAX + 1, 1601 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 1602 cache_zone_large_ts = uma_zcreate("LTS VFS Cache", 1603 sizeof(struct namecache_ts) + NAME_MAX + 1, 1604 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 1605 1606 nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash); 1607 numbucketlocks = cache_roundup_2(mp_ncpus * 64); 1608 bucketlocks = malloc(sizeof(*bucketlocks) * numbucketlocks, M_VFSCACHE, 1609 M_WAITOK | M_ZERO); 1610 for (i = 0; i < numbucketlocks; i++) 1611 rw_init_flags(&bucketlocks[i], "ncbuc", RW_DUPOK | RW_RECURSE); 1612 numvnodelocks = cache_roundup_2(mp_ncpus * 64); 1613 vnodelocks = malloc(sizeof(*vnodelocks) * numvnodelocks, M_VFSCACHE, 1614 M_WAITOK | M_ZERO); 1615 for (i = 0; i < numvnodelocks; i++) 1616 mtx_init(&vnodelocks[i], "ncvn", NULL, MTX_DUPOK | MTX_RECURSE); 1617 1618 numcalls = counter_u64_alloc(M_WAITOK); 1619 dothits = counter_u64_alloc(M_WAITOK); 1620 dotdothits = counter_u64_alloc(M_WAITOK); 1621 numchecks = counter_u64_alloc(M_WAITOK); 1622 nummiss = counter_u64_alloc(M_WAITOK); 1623 nummisszap = counter_u64_alloc(M_WAITOK); 1624 numposzaps = counter_u64_alloc(M_WAITOK); 1625 numposhits = counter_u64_alloc(M_WAITOK); 1626 numnegzaps = counter_u64_alloc(M_WAITOK); 1627 numneghits = counter_u64_alloc(M_WAITOK); 1628 numfullpathcalls = counter_u64_alloc(M_WAITOK); 1629 numfullpathfail1 = counter_u64_alloc(M_WAITOK); 1630 numfullpathfail2 = counter_u64_alloc(M_WAITOK); 1631 numfullpathfail4 = counter_u64_alloc(M_WAITOK); 1632 numfullpathfound = counter_u64_alloc(M_WAITOK); 1633 } 1634 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nchinit, NULL); 1635 1636 void 1637 cache_changesize(int newmaxvnodes) 1638 { 1639 struct nchashhead *new_nchashtbl, *old_nchashtbl; 1640 u_long new_nchash, old_nchash; 1641 struct namecache *ncp; 1642 uint32_t hash; 1643 int i; 1644 1645 new_nchashtbl = hashinit(newmaxvnodes * 2, M_VFSCACHE, &new_nchash); 1646 /* If same hash table size, nothing to do */ 1647 if (nchash == new_nchash) { 1648 free(new_nchashtbl, M_VFSCACHE); 1649 return; 1650 } 1651 /* 1652 * Move everything from the old hash table to the new table. 1653 * None of the namecache entries in the table can be removed 1654 * because to do so, they have to be removed from the hash table. 1655 */ 1656 cache_lock_all_vnodes(); 1657 cache_lock_all_buckets(); 1658 old_nchashtbl = nchashtbl; 1659 old_nchash = nchash; 1660 nchashtbl = new_nchashtbl; 1661 nchash = new_nchash; 1662 for (i = 0; i <= old_nchash; i++) { 1663 while ((ncp = LIST_FIRST(&old_nchashtbl[i])) != NULL) { 1664 hash = cache_get_hash(nc_get_name(ncp), ncp->nc_nlen, 1665 ncp->nc_dvp); 1666 LIST_REMOVE(ncp, nc_hash); 1667 LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash); 1668 } 1669 } 1670 cache_unlock_all_buckets(); 1671 cache_unlock_all_vnodes(); 1672 free(old_nchashtbl, M_VFSCACHE); 1673 } 1674 1675 /* 1676 * Invalidate all entries to a particular vnode. 1677 */ 1678 void 1679 cache_purge(struct vnode *vp) 1680 { 1681 TAILQ_HEAD(, namecache) ncps; 1682 struct namecache *ncp, *nnp; 1683 struct mtx *vlp, *vlp2; 1684 1685 CTR1(KTR_VFS, "cache_purge(%p)", vp); 1686 SDT_PROBE1(vfs, namecache, purge, done, vp); 1687 if (LIST_EMPTY(&vp->v_cache_src) && TAILQ_EMPTY(&vp->v_cache_dst) && 1688 vp->v_cache_dd == NULL) 1689 return; 1690 TAILQ_INIT(&ncps); 1691 vlp = VP2VNODELOCK(vp); 1692 vlp2 = NULL; 1693 mtx_lock(vlp); 1694 retry: 1695 while (!LIST_EMPTY(&vp->v_cache_src)) { 1696 ncp = LIST_FIRST(&vp->v_cache_src); 1697 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2)) 1698 goto retry; 1699 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst); 1700 } 1701 while (!TAILQ_EMPTY(&vp->v_cache_dst)) { 1702 ncp = TAILQ_FIRST(&vp->v_cache_dst); 1703 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2)) 1704 goto retry; 1705 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst); 1706 } 1707 ncp = vp->v_cache_dd; 1708 if (ncp != NULL) { 1709 KASSERT(ncp->nc_flag & NCF_ISDOTDOT, 1710 ("lost dotdot link")); 1711 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2)) 1712 goto retry; 1713 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst); 1714 } 1715 KASSERT(vp->v_cache_dd == NULL, ("incomplete purge")); 1716 mtx_unlock(vlp); 1717 if (vlp2 != NULL) 1718 mtx_unlock(vlp2); 1719 TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) { 1720 cache_free(ncp); 1721 } 1722 } 1723 1724 /* 1725 * Invalidate all negative entries for a particular directory vnode. 1726 */ 1727 void 1728 cache_purge_negative(struct vnode *vp) 1729 { 1730 TAILQ_HEAD(, namecache) ncps; 1731 struct namecache *ncp, *nnp; 1732 struct mtx *vlp; 1733 1734 CTR1(KTR_VFS, "cache_purge_negative(%p)", vp); 1735 SDT_PROBE1(vfs, namecache, purge_negative, done, vp); 1736 TAILQ_INIT(&ncps); 1737 vlp = VP2VNODELOCK(vp); 1738 mtx_lock(vlp); 1739 LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) { 1740 if (ncp->nc_vp != NULL) 1741 continue; 1742 cache_zap_negative_locked_vnode_kl(ncp, vp); 1743 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst); 1744 } 1745 mtx_unlock(vlp); 1746 TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) { 1747 cache_free(ncp); 1748 } 1749 } 1750 1751 /* 1752 * Flush all entries referencing a particular filesystem. 1753 */ 1754 void 1755 cache_purgevfs(struct mount *mp) 1756 { 1757 TAILQ_HEAD(, namecache) ncps; 1758 struct mtx *vlp1, *vlp2; 1759 struct rwlock *blp; 1760 struct nchashhead *bucket; 1761 struct namecache *ncp, *nnp; 1762 u_long i, j, n_nchash; 1763 int error; 1764 1765 /* Scan hash tables for applicable entries */ 1766 SDT_PROBE1(vfs, namecache, purgevfs, done, mp); 1767 TAILQ_INIT(&ncps); 1768 n_nchash = nchash + 1; 1769 vlp1 = vlp2 = NULL; 1770 for (i = 0; i < numbucketlocks; i++) { 1771 blp = (struct rwlock *)&bucketlocks[i]; 1772 rw_wlock(blp); 1773 for (j = i; j < n_nchash; j += numbucketlocks) { 1774 retry: 1775 bucket = &nchashtbl[j]; 1776 LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) { 1777 cache_assert_bucket_locked(ncp, RA_WLOCKED); 1778 if (ncp->nc_dvp->v_mount != mp) 1779 continue; 1780 error = cache_zap_wlocked_bucket_kl(ncp, blp, 1781 &vlp1, &vlp2); 1782 if (error != 0) 1783 goto retry; 1784 TAILQ_INSERT_HEAD(&ncps, ncp, nc_dst); 1785 } 1786 } 1787 rw_wunlock(blp); 1788 if (vlp1 == NULL && vlp2 == NULL) 1789 cache_maybe_yield(); 1790 } 1791 if (vlp1 != NULL) 1792 mtx_unlock(vlp1); 1793 if (vlp2 != NULL) 1794 mtx_unlock(vlp2); 1795 1796 TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) { 1797 cache_free(ncp); 1798 } 1799 } 1800 1801 /* 1802 * Perform canonical checks and cache lookup and pass on to filesystem 1803 * through the vop_cachedlookup only if needed. 1804 */ 1805 1806 int 1807 vfs_cache_lookup(struct vop_lookup_args *ap) 1808 { 1809 struct vnode *dvp; 1810 int error; 1811 struct vnode **vpp = ap->a_vpp; 1812 struct componentname *cnp = ap->a_cnp; 1813 struct ucred *cred = cnp->cn_cred; 1814 int flags = cnp->cn_flags; 1815 struct thread *td = cnp->cn_thread; 1816 1817 *vpp = NULL; 1818 dvp = ap->a_dvp; 1819 1820 if (dvp->v_type != VDIR) 1821 return (ENOTDIR); 1822 1823 if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) && 1824 (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) 1825 return (EROFS); 1826 1827 error = VOP_ACCESS(dvp, VEXEC, cred, td); 1828 if (error) 1829 return (error); 1830 1831 error = cache_lookup(dvp, vpp, cnp, NULL, NULL); 1832 if (error == 0) 1833 return (VOP_CACHEDLOOKUP(dvp, vpp, cnp)); 1834 if (error == -1) 1835 return (0); 1836 return (error); 1837 } 1838 1839 /* 1840 * XXX All of these sysctls would probably be more productive dead. 1841 */ 1842 static int disablecwd; 1843 SYSCTL_INT(_debug, OID_AUTO, disablecwd, CTLFLAG_RW, &disablecwd, 0, 1844 "Disable the getcwd syscall"); 1845 1846 /* Implementation of the getcwd syscall. */ 1847 int 1848 sys___getcwd(struct thread *td, struct __getcwd_args *uap) 1849 { 1850 1851 return (kern___getcwd(td, uap->buf, UIO_USERSPACE, uap->buflen, 1852 MAXPATHLEN)); 1853 } 1854 1855 int 1856 kern___getcwd(struct thread *td, char *buf, enum uio_seg bufseg, u_int buflen, 1857 u_int path_max) 1858 { 1859 char *bp, *tmpbuf; 1860 struct filedesc *fdp; 1861 struct vnode *cdir, *rdir; 1862 int error; 1863 1864 if (disablecwd) 1865 return (ENODEV); 1866 if (buflen < 2) 1867 return (EINVAL); 1868 if (buflen > path_max) 1869 buflen = path_max; 1870 1871 tmpbuf = malloc(buflen, M_TEMP, M_WAITOK); 1872 fdp = td->td_proc->p_fd; 1873 FILEDESC_SLOCK(fdp); 1874 cdir = fdp->fd_cdir; 1875 VREF(cdir); 1876 rdir = fdp->fd_rdir; 1877 VREF(rdir); 1878 FILEDESC_SUNLOCK(fdp); 1879 error = vn_fullpath1(td, cdir, rdir, tmpbuf, &bp, buflen); 1880 vrele(rdir); 1881 vrele(cdir); 1882 1883 if (!error) { 1884 if (bufseg == UIO_SYSSPACE) 1885 bcopy(bp, buf, strlen(bp) + 1); 1886 else 1887 error = copyout(bp, buf, strlen(bp) + 1); 1888 #ifdef KTRACE 1889 if (KTRPOINT(curthread, KTR_NAMEI)) 1890 ktrnamei(bp); 1891 #endif 1892 } 1893 free(tmpbuf, M_TEMP); 1894 return (error); 1895 } 1896 1897 /* 1898 * Thus begins the fullpath magic. 1899 */ 1900 1901 static int disablefullpath; 1902 SYSCTL_INT(_debug, OID_AUTO, disablefullpath, CTLFLAG_RW, &disablefullpath, 0, 1903 "Disable the vn_fullpath function"); 1904 1905 /* 1906 * Retrieve the full filesystem path that correspond to a vnode from the name 1907 * cache (if available) 1908 */ 1909 int 1910 vn_fullpath(struct thread *td, struct vnode *vn, char **retbuf, char **freebuf) 1911 { 1912 char *buf; 1913 struct filedesc *fdp; 1914 struct vnode *rdir; 1915 int error; 1916 1917 if (disablefullpath) 1918 return (ENODEV); 1919 if (vn == NULL) 1920 return (EINVAL); 1921 1922 buf = malloc(MAXPATHLEN, M_TEMP, M_WAITOK); 1923 fdp = td->td_proc->p_fd; 1924 FILEDESC_SLOCK(fdp); 1925 rdir = fdp->fd_rdir; 1926 VREF(rdir); 1927 FILEDESC_SUNLOCK(fdp); 1928 error = vn_fullpath1(td, vn, rdir, buf, retbuf, MAXPATHLEN); 1929 vrele(rdir); 1930 1931 if (!error) 1932 *freebuf = buf; 1933 else 1934 free(buf, M_TEMP); 1935 return (error); 1936 } 1937 1938 /* 1939 * This function is similar to vn_fullpath, but it attempts to lookup the 1940 * pathname relative to the global root mount point. This is required for the 1941 * auditing sub-system, as audited pathnames must be absolute, relative to the 1942 * global root mount point. 1943 */ 1944 int 1945 vn_fullpath_global(struct thread *td, struct vnode *vn, 1946 char **retbuf, char **freebuf) 1947 { 1948 char *buf; 1949 int error; 1950 1951 if (disablefullpath) 1952 return (ENODEV); 1953 if (vn == NULL) 1954 return (EINVAL); 1955 buf = malloc(MAXPATHLEN, M_TEMP, M_WAITOK); 1956 error = vn_fullpath1(td, vn, rootvnode, buf, retbuf, MAXPATHLEN); 1957 if (!error) 1958 *freebuf = buf; 1959 else 1960 free(buf, M_TEMP); 1961 return (error); 1962 } 1963 1964 int 1965 vn_vptocnp(struct vnode **vp, struct ucred *cred, char *buf, u_int *buflen) 1966 { 1967 struct vnode *dvp; 1968 struct namecache *ncp; 1969 struct mtx *vlp; 1970 int error; 1971 1972 vlp = VP2VNODELOCK(*vp); 1973 mtx_lock(vlp); 1974 TAILQ_FOREACH(ncp, &((*vp)->v_cache_dst), nc_dst) { 1975 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) 1976 break; 1977 } 1978 if (ncp != NULL) { 1979 if (*buflen < ncp->nc_nlen) { 1980 mtx_unlock(vlp); 1981 vrele(*vp); 1982 counter_u64_add(numfullpathfail4, 1); 1983 error = ENOMEM; 1984 SDT_PROBE3(vfs, namecache, fullpath, return, error, 1985 vp, NULL); 1986 return (error); 1987 } 1988 *buflen -= ncp->nc_nlen; 1989 memcpy(buf + *buflen, nc_get_name(ncp), ncp->nc_nlen); 1990 SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp, 1991 nc_get_name(ncp), vp); 1992 dvp = *vp; 1993 *vp = ncp->nc_dvp; 1994 vref(*vp); 1995 mtx_unlock(vlp); 1996 vrele(dvp); 1997 return (0); 1998 } 1999 SDT_PROBE1(vfs, namecache, fullpath, miss, vp); 2000 2001 mtx_unlock(vlp); 2002 vn_lock(*vp, LK_SHARED | LK_RETRY); 2003 error = VOP_VPTOCNP(*vp, &dvp, cred, buf, buflen); 2004 vput(*vp); 2005 if (error) { 2006 counter_u64_add(numfullpathfail2, 1); 2007 SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL); 2008 return (error); 2009 } 2010 2011 *vp = dvp; 2012 if (dvp->v_iflag & VI_DOOMED) { 2013 /* forced unmount */ 2014 vrele(dvp); 2015 error = ENOENT; 2016 SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL); 2017 return (error); 2018 } 2019 /* 2020 * *vp has its use count incremented still. 2021 */ 2022 2023 return (0); 2024 } 2025 2026 /* 2027 * The magic behind kern___getcwd() and vn_fullpath(). 2028 */ 2029 static int 2030 vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir, 2031 char *buf, char **retbuf, u_int buflen) 2032 { 2033 int error, slash_prefixed; 2034 #ifdef KDTRACE_HOOKS 2035 struct vnode *startvp = vp; 2036 #endif 2037 struct vnode *vp1; 2038 2039 buflen--; 2040 buf[buflen] = '\0'; 2041 error = 0; 2042 slash_prefixed = 0; 2043 2044 SDT_PROBE1(vfs, namecache, fullpath, entry, vp); 2045 counter_u64_add(numfullpathcalls, 1); 2046 vref(vp); 2047 if (vp->v_type != VDIR) { 2048 error = vn_vptocnp(&vp, td->td_ucred, buf, &buflen); 2049 if (error) 2050 return (error); 2051 if (buflen == 0) { 2052 vrele(vp); 2053 return (ENOMEM); 2054 } 2055 buf[--buflen] = '/'; 2056 slash_prefixed = 1; 2057 } 2058 while (vp != rdir && vp != rootvnode) { 2059 if (vp->v_vflag & VV_ROOT) { 2060 if (vp->v_iflag & VI_DOOMED) { /* forced unmount */ 2061 vrele(vp); 2062 error = ENOENT; 2063 SDT_PROBE3(vfs, namecache, fullpath, return, 2064 error, vp, NULL); 2065 break; 2066 } 2067 vp1 = vp->v_mount->mnt_vnodecovered; 2068 vref(vp1); 2069 vrele(vp); 2070 vp = vp1; 2071 continue; 2072 } 2073 if (vp->v_type != VDIR) { 2074 vrele(vp); 2075 counter_u64_add(numfullpathfail1, 1); 2076 error = ENOTDIR; 2077 SDT_PROBE3(vfs, namecache, fullpath, return, 2078 error, vp, NULL); 2079 break; 2080 } 2081 error = vn_vptocnp(&vp, td->td_ucred, buf, &buflen); 2082 if (error) 2083 break; 2084 if (buflen == 0) { 2085 vrele(vp); 2086 error = ENOMEM; 2087 SDT_PROBE3(vfs, namecache, fullpath, return, error, 2088 startvp, NULL); 2089 break; 2090 } 2091 buf[--buflen] = '/'; 2092 slash_prefixed = 1; 2093 } 2094 if (error) 2095 return (error); 2096 if (!slash_prefixed) { 2097 if (buflen == 0) { 2098 vrele(vp); 2099 counter_u64_add(numfullpathfail4, 1); 2100 SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM, 2101 startvp, NULL); 2102 return (ENOMEM); 2103 } 2104 buf[--buflen] = '/'; 2105 } 2106 counter_u64_add(numfullpathfound, 1); 2107 vrele(vp); 2108 2109 SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, buf + buflen); 2110 *retbuf = buf + buflen; 2111 return (0); 2112 } 2113 2114 struct vnode * 2115 vn_dir_dd_ino(struct vnode *vp) 2116 { 2117 struct namecache *ncp; 2118 struct vnode *ddvp; 2119 struct mtx *vlp; 2120 2121 ASSERT_VOP_LOCKED(vp, "vn_dir_dd_ino"); 2122 vlp = VP2VNODELOCK(vp); 2123 mtx_lock(vlp); 2124 TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) { 2125 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) 2126 continue; 2127 ddvp = ncp->nc_dvp; 2128 vhold(ddvp); 2129 mtx_unlock(vlp); 2130 if (vget(ddvp, LK_SHARED | LK_NOWAIT | LK_VNHELD, curthread)) 2131 return (NULL); 2132 return (ddvp); 2133 } 2134 mtx_unlock(vlp); 2135 return (NULL); 2136 } 2137 2138 int 2139 vn_commname(struct vnode *vp, char *buf, u_int buflen) 2140 { 2141 struct namecache *ncp; 2142 struct mtx *vlp; 2143 int l; 2144 2145 vlp = VP2VNODELOCK(vp); 2146 mtx_lock(vlp); 2147 TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) 2148 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) 2149 break; 2150 if (ncp == NULL) { 2151 mtx_unlock(vlp); 2152 return (ENOENT); 2153 } 2154 l = min(ncp->nc_nlen, buflen - 1); 2155 memcpy(buf, nc_get_name(ncp), l); 2156 mtx_unlock(vlp); 2157 buf[l] = '\0'; 2158 return (0); 2159 } 2160 2161 /* ABI compat shims for old kernel modules. */ 2162 #undef cache_enter 2163 2164 void cache_enter(struct vnode *dvp, struct vnode *vp, 2165 struct componentname *cnp); 2166 2167 void 2168 cache_enter(struct vnode *dvp, struct vnode *vp, struct componentname *cnp) 2169 { 2170 2171 cache_enter_time(dvp, vp, cnp, NULL, NULL); 2172 } 2173 2174 /* 2175 * This function updates path string to vnode's full global path 2176 * and checks the size of the new path string against the pathlen argument. 2177 * 2178 * Requires a locked, referenced vnode. 2179 * Vnode is re-locked on success or ENODEV, otherwise unlocked. 2180 * 2181 * If sysctl debug.disablefullpath is set, ENODEV is returned, 2182 * vnode is left locked and path remain untouched. 2183 * 2184 * If vp is a directory, the call to vn_fullpath_global() always succeeds 2185 * because it falls back to the ".." lookup if the namecache lookup fails. 2186 */ 2187 int 2188 vn_path_to_global_path(struct thread *td, struct vnode *vp, char *path, 2189 u_int pathlen) 2190 { 2191 struct nameidata nd; 2192 struct vnode *vp1; 2193 char *rpath, *fbuf; 2194 int error; 2195 2196 ASSERT_VOP_ELOCKED(vp, __func__); 2197 2198 /* Return ENODEV if sysctl debug.disablefullpath==1 */ 2199 if (disablefullpath) 2200 return (ENODEV); 2201 2202 /* Construct global filesystem path from vp. */ 2203 VOP_UNLOCK(vp, 0); 2204 error = vn_fullpath_global(td, vp, &rpath, &fbuf); 2205 2206 if (error != 0) { 2207 vrele(vp); 2208 return (error); 2209 } 2210 2211 if (strlen(rpath) >= pathlen) { 2212 vrele(vp); 2213 error = ENAMETOOLONG; 2214 goto out; 2215 } 2216 2217 /* 2218 * Re-lookup the vnode by path to detect a possible rename. 2219 * As a side effect, the vnode is relocked. 2220 * If vnode was renamed, return ENOENT. 2221 */ 2222 NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1, 2223 UIO_SYSSPACE, path, td); 2224 error = namei(&nd); 2225 if (error != 0) { 2226 vrele(vp); 2227 goto out; 2228 } 2229 NDFREE(&nd, NDF_ONLY_PNBUF); 2230 vp1 = nd.ni_vp; 2231 vrele(vp); 2232 if (vp1 == vp) 2233 strcpy(path, rpath); 2234 else { 2235 vput(vp1); 2236 error = ENOENT; 2237 } 2238 2239 out: 2240 free(fbuf, M_TEMP); 2241 return (error); 2242 } 2243