Lines Matching +full:south +full:- +full:field
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
8 * Poul-Henning Kamp of the FreeBSD Project.
119 * - set vop_lookup routine to vfs_cache_lookup
120 * - set vop_cachedlookup to a routine which can perform the lookup if the
122 * - if they support lockless lookup (see below), they set vop_fplookup_vexec
125 * - call cache_purge or cache_vop_* routines to eliminate stale entries as
127 * - call cache_enter to add entries depending on the MAKEENTRY flag
130 * - ... -> namei -> cache_fplookup -- this is the default
131 * - ... -> VOP_LOOKUP -> vfs_cache_lookup -- normally only called by namei
135 * ... -> namei -> cache_fplookup -> cache_fplookup_noentry -> VOP_LOOKUP ->
136 * vfs_cache_lookup -> VOP_CACHEDLOOKUP -> ufs_lookup_ino -> cache_enter
143 * from the terminal path component. In other words non-modifying lookups of
150 * At least on amd64 the single-threaded bottleneck for long paths is hashing
156 * introduces significant hit single-threaded and serialization multi-threaded.
158 * Reverse lookup (e.g., "getcwd") fully scales provided it is fully cached --
175 * dtrace -n 'vfs:fplookup:lookup:done { @[arg1, arg2] = count(); }'
178 * dtrace -n 'fbt::cache_enter_time:entry { @[execname] = quantize(args[2]->cn_namelen); }'
181 * dtrace -n 'fbt::cache_enter_time:entry /args[2]->cn_namelen > 64/ { @[execname] = quantize(args[2]->cn_namelen); }'
185 * dtrace -n 'fbt::cache_fplookup_skip_slashes:entry { @[execname, stringof(args[0]->cnp->cn_pnbuf)] = count(); }'
189 * - since it is possible there is no entry for an open file, tools like
190 * "procstat" may fail to resolve fd -> vnode -> path to anything
191 * - even if a filesystem adds an entry, it may get purged (e.g., due to memory
193 * - hardlinks are not tracked, thus if a vnode is reachable in more than one
196 * - by default entries are not added for newly created files
197 * - adding an entry may need to evict negative entry first, which happens in 2
200 * - there is a simple scheme to evict negative entries as the cache is approaching
202 * - vnodes are subject to being recycled even if target inode is left in memory,
204 * names get duplicated -- kept by filesystem itself and namecache separately
205 * - vnode reclamation (see vnlru in kern/vfs_subr.c) defaults to skipping
207 * of vnode LRU. Per the above this is done to avoid breaking vnode -> path
210 * - struct namecache has a fixed size and comes in 2 variants, often wasting
213 * - lack of better integration with the kernel also turns nullfs into a layered
219 * performance left on the table, most notably from single-threaded standpoint.
227 * === SMR-only lookup
234 * single-threaded as it dodges atomics and would provide full scalability for
242 * word-sized instead thanks to the Mycroft trick.
251 * be confused with writing past buffer size) -- this would allow word-sized
292 * ref = atomic_fetchadd_int(&vp->v_count, -1) - 1;
304 * feature. Even if so, handling of v_usecount could be decoupled from it --
305 * vnlru et al would consider the vnode not-freeable if has either hold or
390 * to be stored. The nc_dotdottime field is used when a cache entry is mapping
391 * both a non-dotdot directory name plus dotdot for the directory's
407 * At least mips n32 performs 64-bit accesses to timespec as found
410 * alignment for everyone. Note this is a nop for 64-bit platforms.
418 * was never re-evaluated for suitability. A simple test counting
425 * re-evaluated.
476 KASSERT((ncp->nc_flag & NCF_INVALID) == 0,
478 atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_INVALID);
489 return (ncp->nc_dvp == dvp &&
490 ncp->nc_nlen == cnp->cn_namelen &&
491 bcmp(ncp->nc_name, cnp->cn_nameptr, cnp->cn_namelen) == 0);
505 _nc_flag = atomic_load_char(&_ncp->nc_flag); \
517 _nc_flag = atomic_load_char(&_ncp->nc_flag); \
589 MPASS(atomic_load_char(&ncp->nc_flag) & NCF_NEGATIVE);
590 return (&ncp->nc_neg);
632 KASSERT((ncp->nc_flag & NCF_TS) != 0 ||
640 *tsp = ncp_ts->nc_time;
641 *ticksp = ncp_ts->nc_ticks;
701 "Number of times 3-way vnode locking failed");
743 VNPASS(LIST_EMPTY(&vp->v_cache_src), vp);
757 counter_u64_add(numcachehv, -1);
813 ncp = &ncp_ts->nc_nc;
828 if (__predict_false(ncp->nc_flag & NCF_TS)) {
830 if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
835 if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
876 if ((ncp->nc_flag & NCF_DVDROP) != 0) {
877 cache_drop_vnode(ncp->nc_dvp);
893 if ((ncp->nc_flag & NCF_DVDROP) != 0) {
894 cache_drop_vnode(ncp->nc_dvp);
934 vp->v_nchash = fnv_32_buf(&vp, sizeof(vp), FNV1_32_INIT);
941 return (fnv_32_buf(name, len, dvp->v_nchash));
948 return (dvp->v_nchash);
970 hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
979 hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
1106 if (req->oldptr == NULL)
1158 if (error != 0 || req->newptr == NULL)
1191 if (req->oldptr == NULL)
1224 if (!req->oldptr)
1233 for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) {
1333 ncp->nc_flag |= NCF_NEGATIVE;
1335 ns->neg_flag = 0;
1336 ns->neg_hit = 0;
1349 n = atomic_load_char(&ns->neg_hit);
1353 if (atomic_fcmpset_8(&ns->neg_hit, &n, n + 1))
1370 SDT_PROBE2(vfs, namecache, lookup, hit__negative, ncp->nc_dvp, ncp->nc_name);
1385 mtx_assert(&nl->nl_lock, MA_OWNED);
1386 if ((ns->neg_flag & NEG_HOT) == 0) {
1387 TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst);
1388 TAILQ_INSERT_TAIL(&nl->nl_hotlist, ncp, nc_dst);
1389 nl->nl_hotnum++;
1390 ns->neg_flag |= NEG_HOT;
1405 mtx_assert(&nl->nl_lock, MA_OWNED);
1406 MPASS(ns->neg_flag & NEG_HOT);
1407 TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst);
1408 TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst);
1409 nl->nl_hotnum--;
1410 ns->neg_flag &= ~NEG_HOT;
1411 atomic_store_char(&ns->neg_hit, 0);
1433 mtx_lock(&nl->nl_lock);
1472 nc_flag = atomic_load_char(&ncp->nc_flag);
1484 mtx_unlock(&nl->nl_lock);
1488 mtx_unlock(&nl->nl_lock);
1498 mtx_lock(&nl->nl_lock);
1500 mtx_unlock(&nl->nl_lock);
1508 MPASS(ncp->nc_flag & NCF_NEGATIVE);
1511 mtx_lock(&nl->nl_lock);
1512 TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst);
1513 mtx_unlock(&nl->nl_lock);
1526 mtx_lock(&nl->nl_lock);
1527 if ((ns->neg_flag & NEG_HOT) != 0) {
1528 TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst);
1529 nl->nl_hotnum--;
1531 TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst);
1533 mtx_unlock(&nl->nl_lock);
1545 if (!mtx_trylock(&nl->nl_evict_lock)) {
1559 mtx_assert(&nl->nl_evict_lock, MA_OWNED);
1560 mtx_assert(&nl->nl_lock, MA_OWNED);
1561 ncp = TAILQ_FIRST(&nl->nl_list);
1571 if (ns->neg_hit < lns->neg_hit) {
1596 mtx_lock(&nl->nl_lock);
1597 ncp = TAILQ_FIRST(&nl->nl_hotlist);
1604 mtx_unlock(&nl->nl_lock);
1605 mtx_unlock(&nl->nl_evict_lock);
1608 nlen = ncp->nc_nlen;
1609 dvp = ncp->nc_dvp;
1610 hash = cache_get_hash(ncp->nc_name, nlen, dvp);
1613 mtx_unlock(&nl->nl_lock);
1614 mtx_unlock(&nl->nl_evict_lock);
1622 if (ncp2 == ncp && ncp2->nc_dvp == dvp &&
1623 ncp2->nc_nlen == nlen && (ncp2->nc_flag & NCF_NEGATIVE) != 0)
1631 MPASS(dvlp == VP2VNODELOCK(ncp->nc_dvp));
1633 SDT_PROBE2(vfs, namecache, evict_negative, done, ncp->nc_dvp,
1634 ncp->nc_name);
1663 if (ncsize - 1000 < lnumcache)
1686 dvp = ncp->nc_dvp;
1687 vp = ncp->nc_vp;
1689 if (!(ncp->nc_flag & NCF_NEGATIVE))
1698 if (!(ncp->nc_flag & NCF_NEGATIVE)) {
1699 SDT_PROBE3(vfs, namecache, zap, done, dvp, ncp->nc_name, vp);
1700 TAILQ_REMOVE(&vp->v_cache_dst, ncp, nc_dst);
1701 if (ncp == vp->v_cache_dd) {
1702 atomic_store_ptr(&vp->v_cache_dd, NULL);
1705 SDT_PROBE2(vfs, namecache, zap_negative, done, dvp, ncp->nc_name);
1708 if (ncp->nc_flag & NCF_ISDOTDOT) {
1709 if (ncp == dvp->v_cache_dd) {
1710 atomic_store_ptr(&dvp->v_cache_dd, NULL);
1714 if (LIST_EMPTY(&dvp->v_cache_src)) {
1715 ncp->nc_flag |= NCF_DVDROP;
1725 MPASS(ncp->nc_dvp == vp);
1726 MPASS(ncp->nc_flag & NCF_NEGATIVE);
1742 MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
1745 if (ncp->nc_flag & NCF_NEGATIVE) {
1756 vlp1 = VP2VNODELOCK(ncp->nc_dvp);
1757 vlp2 = VP2VNODELOCK(ncp->nc_vp);
1795 * in the right order and re-lookup the entry.
1817 if (!(ncp->nc_flag & NCF_NEGATIVE))
1818 rvlp = VP2VNODELOCK(rncp->nc_vp);
1845 dvlp = VP2VNODELOCK(ncp->nc_dvp);
1847 if (!(ncp->nc_flag & NCF_NEGATIVE))
1848 vlp = VP2VNODELOCK(ncp->nc_vp);
1856 dvp = ncp->nc_dvp;
1870 if (cnp->cn_namelen == 2 &&
1871 cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
1876 ncp = dvp->v_cache_dd;
1884 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
1887 MPASS(dvp->v_cache_dd == NULL);
1893 atomic_store_ptr(&dvp->v_cache_dd, NULL);
1911 hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
1956 ltype = cnp->cn_lkflags & LK_TYPE_MASK;
1969 return (-1);
1983 MPASS((cnp->cn_flags & ISDOTDOT) != 0);
1985 if ((cnp->cn_flags & MAKEENTRY) == 0) {
1993 ncp = dvp->v_cache_dd;
1999 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
2000 if (ncp->nc_flag & NCF_NEGATIVE)
2003 *vpp = ncp->nc_vp;
2005 *vpp = ncp->nc_dvp;
2010 if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
2013 *tsp = ncp_ts->nc_dotdottime;
2021 error = vget_finish(*vpp, cnp->cn_lkflags, vs);
2033 return (-1);
2035 if (__predict_false(cnp->cn_nameiop == CREATE)) {
2036 if (cnp->cn_flags & ISLASTCN) {
2045 whiteout = (ncp->nc_flag & NCF_WHITE);
2053 cnp->cn_flags |= ISWHITEOUT;
2062 * - dvp: Parent directory in which to search.
2063 * - vpp: Return argument. Will contain desired vnode on cache hit.
2064 * - cnp: Parameters of the name search. The most interesting bits of
2065 * the cn_flags field have the following meanings:
2066 * - MAKEENTRY: If clear, free an entry from the cache rather than look
2068 * - ISDOTDOT: Must be set if and only if cn_nameptr == ".."
2069 * - tsp: Return storage for cache timestamp. On a successful (positive
2073 * - ticks: Return storage for alternate cache timestamp. On a successful
2082 * - -1: A positive cache hit. vpp will contain the desired vnode.
2083 * - ENOENT: A negative cache hit, or dvp was recycled out from under us due
2086 * cnp->cn_flags.
2087 * - 0: A cache miss. vpp will not be modified.
2106 MPASS((cnp->cn_flags & ISDOTDOT) == 0);
2107 MPASS((cnp->cn_flags & (MAKEENTRY | NC_KEEPPOSENTRY)) != 0);
2110 hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
2117 SDT_PROBE2(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr);
2122 if (ncp->nc_flag & NCF_NEGATIVE)
2126 *vpp = ncp->nc_vp;
2127 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp);
2132 error = vget_finish(*vpp, cnp->cn_lkflags, vs);
2137 return (-1);
2142 if (__predict_true(cnp->cn_nameiop == CREATE)) {
2143 if (cnp->cn_flags & ISLASTCN) {
2155 whiteout = (ncp->nc_flag & NCF_WHITE);
2163 cnp->cn_flags |= ISWHITEOUT;
2182 cnp->cn_flags &= ~MAKEENTRY;
2187 if (__predict_false(cnp->cn_nameptr[0] == '.')) {
2188 if (cnp->cn_namelen == 1)
2190 if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.')
2194 MPASS((cnp->cn_flags & ISDOTDOT) == 0);
2196 if ((cnp->cn_flags & (MAKEENTRY | NC_KEEPPOSENTRY)) == 0) {
2201 hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
2207 SDT_PROBE2(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr);
2212 nc_flag = atomic_load_char(&ncp->nc_flag);
2217 *vpp = ncp->nc_vp;
2218 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp);
2232 error = vget_finish(*vpp, cnp->cn_lkflags, vs);
2237 return (-1);
2239 if (cnp->cn_nameiop == CREATE) {
2240 if (cnp->cn_flags & ISLASTCN) {
2247 whiteout = (atomic_load_char(&ncp->nc_flag) & NCF_WHITE);
2263 cnp->cn_flags |= ISWHITEOUT;
2273 CTASSERT((nitems(((struct celockstate *)0)->vlp) == 3));
2274 CTASSERT((nitems(((struct celockstate *)0)->blp) == 2));
2289 MPASS(cel->vlp[0] == NULL);
2290 MPASS(cel->vlp[1] == NULL);
2291 MPASS(cel->vlp[2] == NULL);
2301 cel->vlp[0] = vlp1;
2304 cel->vlp[1] = vlp2;
2311 MPASS(cel->vlp[0] != NULL || cel->vlp[1] != NULL);
2313 if (cel->vlp[0] != NULL)
2314 mtx_unlock(cel->vlp[0]);
2315 if (cel->vlp[1] != NULL)
2316 mtx_unlock(cel->vlp[1]);
2317 if (cel->vlp[2] != NULL)
2318 mtx_unlock(cel->vlp[2]);
2327 cache_assert_vlp_locked(cel->vlp[0]);
2328 cache_assert_vlp_locked(cel->vlp[1]);
2329 MPASS(cel->vlp[2] == NULL);
2335 if (vlp >= cel->vlp[1]) {
2342 if (vlp < cel->vlp[0]) {
2344 mtx_lock(cel->vlp[0]);
2345 mtx_lock(cel->vlp[1]);
2347 if (cel->vlp[0] != NULL)
2348 mtx_lock(cel->vlp[0]);
2350 mtx_lock(cel->vlp[1]);
2355 cel->vlp[2] = vlp;
2364 MPASS(cel->blp[0] == NULL);
2365 MPASS(cel->blp[1] == NULL);
2371 cel->blp[0] = blp1;
2374 cel->blp[1] = blp2;
2381 if (cel->blp[0] != NULL)
2382 mtx_unlock(cel->blp[0]);
2383 mtx_unlock(cel->blp[1]);
2408 if (vp == NULL || vp->v_type != VDIR)
2410 ncp = atomic_load_consume_ptr(&vp->v_cache_dd);
2413 nc_flag = atomic_load_char(&ncp->nc_flag);
2416 MPASS(ncp->nc_dvp == vp);
2420 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
2423 * All vnodes got re-locked. Re-validate the state and if
2426 if (ncp == vp->v_cache_dd &&
2427 (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
2429 VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
2432 cel->vlp[0] = NULL;
2433 cel->vlp[1] = NULL;
2434 cel->vlp[2] = NULL;
2451 ncp = atomic_load_consume_ptr(&dvp->v_cache_dd);
2454 nc_flag = atomic_load_char(&ncp->nc_flag);
2457 MPASS(ncp->nc_dvp == dvp);
2461 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
2463 if (ncp == dvp->v_cache_dd &&
2464 (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
2466 VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
2469 cel->vlp[0] = NULL;
2470 cel->vlp[1] = NULL;
2471 cel->vlp[2] = NULL;
2493 if (atomic_load_ptr(&dvp->v_cache_dd) == NULL)
2495 len = cnp->cn_namelen;
2497 hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
2499 ncp = dvp->v_cache_dd;
2500 if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT)) {
2501 KASSERT(ncp->nc_dvp == dvp, ("wrong isdotdot parent"));
2506 atomic_store_ptr(&dvp->v_cache_dd, NULL);
2526 KASSERT(cnp->cn_namelen <= NAME_MAX,
2527 ("%s: passed len %ld exceeds NAME_MAX (%d)", __func__, cnp->cn_namelen,
2530 VNPASS(dvp->v_type != VNON, dvp);
2533 VNPASS(vp->v_type != VNON, vp);
2535 if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') {
2541 ("%s: same vnode for non-dot entry [%s] (%p)\n", __func__,
2542 cnp->cn_nameptr, dvp));
2551 if (__predict_false(cnp->cn_nameptr[0] == '.')) {
2552 if (cnp->cn_namelen == 1)
2554 if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
2560 ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
2572 ncp->nc_flag = flag | NCF_WIP;
2573 ncp->nc_vp = vp;
2576 ncp->nc_dvp = dvp;
2579 ncp_ts->nc_time = *tsp;
2580 ncp_ts->nc_ticks = ticks;
2581 ncp_ts->nc_nc.nc_flag |= NCF_TS;
2583 ncp_ts->nc_dotdottime = *dtsp;
2584 ncp_ts->nc_nc.nc_flag |= NCF_DTS;
2587 len = ncp->nc_nlen = cnp->cn_namelen;
2588 hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
2589 memcpy(ncp->nc_name, cnp->cn_nameptr, len);
2590 ncp->nc_name[len] = '\0';
2601 if ((n2->nc_flag & NCF_NEGATIVE) != 0)
2605 __func__, NULL, vp, cnp->cn_nameptr));
2607 KASSERT(n2->nc_vp == vp,
2610 __func__, n2->nc_vp, vp, cnp->cn_nameptr));
2620 MPASS((n2->nc_flag & (NCF_TS | NCF_DTS)) ==
2621 (ncp->nc_flag & (NCF_TS | NCF_DTS)));
2624 KASSERT((n2->nc_flag & NCF_TS) != 0,
2627 n2_ts->nc_time = ncp_ts->nc_time;
2628 n2_ts->nc_ticks = ncp_ts->nc_ticks;
2630 n2_ts->nc_dotdottime = ncp_ts->nc_dotdottime;
2631 n2_ts->nc_nc.nc_flag |= NCF_DTS;
2635 SDT_PROBE3(vfs, namecache, enter, duplicate, dvp, ncp->nc_name,
2645 if (dvp->v_cache_dd != NULL)
2647 KASSERT(vp == NULL || vp->v_type == VDIR,
2650 atomic_store_ptr(&dvp->v_cache_dd, ncp);
2665 if ((ndd = vp->v_cache_dd) != NULL) {
2666 if ((ndd->nc_flag & NCF_ISDOTDOT) != 0)
2672 atomic_store_ptr(&vp->v_cache_dd, ncp);
2676 if (LIST_EMPTY(&dvp->v_cache_src)) {
2679 LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
2688 TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
2689 SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name,
2692 if (cnp->cn_flags & ISWHITEOUT)
2693 atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_WHITE);
2696 ncp->nc_name);
2710 atomic_store_char(&ncp->nc_flag, ncp->nc_flag & ~NCF_WIP);
2725 * - VFS_CACHE_DROPOLD -- if a conflicting entry is found, drop it.
2768 *hashmask = hashsize - 1;
2804 ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1;
2832 LIST_INIT(&vp->v_cache_src);
2833 TAILQ_INIT(&vp->v_cache_dst);
2834 vp->v_cache_dd = NULL;
2939 hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen,
2940 ncp->nc_dvp);
2969 while (!LIST_EMPTY(&vp->v_cache_src)) {
2970 ncp = LIST_FIRST(&vp->v_cache_src);
2975 while (!TAILQ_EMPTY(&vp->v_cache_dst)) {
2976 ncp = TAILQ_FIRST(&vp->v_cache_dst);
2981 ncp = vp->v_cache_dd;
2983 KASSERT(ncp->nc_flag & NCF_ISDOTDOT,
2989 KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
3003 if (LIST_EMPTY(&vp->v_cache_src) && TAILQ_EMPTY(&vp->v_cache_dst) &&
3004 atomic_load_ptr(&vp->v_cache_dd) == NULL)
3056 if (LIST_EMPTY(&vp->v_cache_src))
3061 LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) {
3062 if (!(ncp->nc_flag & NCF_NEGATIVE))
3128 hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
3134 if (ncp != NULL && ncp->nc_vp != vp) {
3136 __func__, vp, ncp->nc_vp, ncp, ncp->nc_name, ncp->nc_dvp);
3145 VNPASS(TAILQ_EMPTY(&vp->v_cache_dst), vp);
3146 VNPASS(LIST_EMPTY(&vp->v_cache_src), vp);
3147 VNPASS(vp->v_cache_dd == NULL, vp);
3191 struct vnode **vpp = ap->a_vpp;
3192 struct componentname *cnp = ap->a_cnp;
3193 int flags = cnp->cn_flags;
3196 dvp = ap->a_dvp;
3198 if (dvp->v_type != VDIR)
3201 if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
3202 (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME))
3212 if (error == -1)
3225 buflen = uap->buflen;
3234 error = copyout(retbuf, uap->buf, buflen);
3247 error = vn_fullpath_any_smr(pwd->pwd_cdir, pwd->pwd_rdir, buf, retbuf,
3252 error = vn_fullpath_any(pwd->pwd_cdir, pwd->pwd_rdir, buf,
3268 * - Nothing guarantees the integrity of the entire chain. Consider the case
3290 if (nd.ni_vp->v_type == VREG && nd.ni_dvp->v_type != VDIR &&
3291 (nd.ni_vp->v_vflag & VV_ROOT) != 0) {
3299 * To resolve this, we find the vnode which was mounted on -
3306 covered_vp = nd.ni_vp->v_mount->mnt_vnodecovered;
3339 return (kern___realpathat(td, uap->fd, uap->path, uap->buf, uap->size,
3340 uap->flags, UIO_USERSPACE));
3379 return (pwd->pwd_rdir);
3395 return (pwd->pwd_jdir);
3415 * auditing sub-system, as audited pathnames must be absolute, relative to the
3448 TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) {
3449 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
3465 ncp = (*vp)->v_cache_dd;
3466 if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT) == 0) {
3474 if (*buflen < ncp->nc_nlen) {
3483 *buflen -= ncp->nc_nlen;
3484 memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
3485 SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
3486 ncp->nc_name, vp);
3488 *vp = ncp->nc_dvp;
3542 VNPASS(vp->v_type == VDIR || VN_IS_DOOMED(vp), vp);
3543 VNPASS(vp->v_usecount > 0, vp);
3550 buflen--;
3566 if ((vp->v_vflag & VV_ROOT) != 0) {
3578 (vp1 = vp->v_mount->mnt_vnodecovered) == NULL ||
3579 vp1->v_mountedhere != vp->v_mount) {
3592 VNPASS(vp->v_type == VDIR || VN_IS_DOOMED(vp), vp);
3603 buf[--buflen] = '/';
3616 buf[--buflen] = '/';
3623 *len -= buflen;
3632 * - hardlinks are not tracked, thus if the vnode is not a directory this can
3634 * - namecache is not mandatory, meaning names are not guaranteed to be added
3668 return (-1);
3675 *buflen -= 1;
3681 *buflen -= 1;
3690 error = -1;
3702 if ((vp->v_vflag & VV_ROOT) != 0) {
3703 mp = atomic_load_ptr(&vp->v_mount);
3708 tvp = atomic_load_ptr(&mp->mnt_vnodecovered);
3722 ncp = atomic_load_consume_ptr(&vp->v_cache_dd);
3727 nc_flag = atomic_load_char(&ncp->nc_flag);
3732 if (ncp->nc_nlen >= *buflen) {
3737 *buflen -= ncp->nc_nlen;
3738 memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
3739 *buflen -= 1;
3741 tvp = ncp->nc_dvp;
3754 if (__predict_false(atomic_load_ptr(&vp->v_cache_dd) != ncp)) {
3770 *buflen = orig_buflen - *buflen + addend;
3795 if (vp->v_type != VDIR) {
3796 *buflen -= 1;
3805 *buflen -= 1;
3807 addend = orig_buflen - *buflen;
3821 * - if the found vnode is a directory, the path can be constructed just by
3823 * - otherwise we populate the buffer with the saved name and start resolving
3858 type = atomic_load_8(&vp->v_type);
3869 *buflen -= addend;
3873 tmpbuf[addend - 1] = '\0';
3879 error = vn_fullpath_any_smr(vp, pwd->pwd_rdir, buf, retbuf, buflen,
3885 error = vn_fullpath_dir(vp, pwd->pwd_rdir, buf, retbuf, buflen,
3911 TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
3912 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
3914 ddvp = ncp->nc_dvp;
3934 TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
3935 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
3941 l = min(ncp->nc_nlen, buflen - 1);
3942 memcpy(buf, ncp->nc_name, l);
3953 * Vnode is re-locked on success or ENODEV, otherwise unlocked.
3985 * Re-lookup the vnode by path to detect a possible rename.
4015 * Vnode is re-locked on success or ENODEV, otherwise unlocked.
4051 * Re-lookup the vnode by path to detect a possible rename.
4084 isdir = vp->v_type == VDIR ? IN_ISDIR : 0;
4086 (vp->v_type != VDIR || (event & ~_IN_DIR_EVENTS) != 0);
4103 TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) {
4104 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
4106 if ((vn_irflag_read(ncp->nc_dvp) & VIRF_INOTIFY) != 0) {
4108 * XXX-MJ if the vnode has two links in the same
4111 inotify_log(ncp->nc_dvp, ncp->nc_name, ncp->nc_nlen,
4137 if (vp->v_vflag & VV_ROOT) {
4139 vp = vp->v_mount->mnt_vnodecovered;
4145 ncp = TAILQ_FIRST(&vp->v_cache_dst);
4147 ncn = ncp->nc_name;
4148 for (i = 0; i < ncp->nc_nlen; i++)
4150 vp = ncp->nc_dvp;
4179 #define CACHE_FPL_FAILED -2020
4184 vn_printf(v->a_vp, "no proper vop_fplookup_vexec\n");
4191 vn_printf(v->a_vp, "no proper vop_fplookup_symlink\n");
4201 if (v->vop_fplookup_vexec != NULL) {
4204 if (v->vop_fplookup_symlink != NULL) {
4213 v->vop_fplookup_vexec = cache_vop_bad_vexec;
4214 v->vop_fplookup_symlink = cache_vop_bad_symlink;
4218 printf("%s: invalid vop vector %p -- either all or none fplookup vops "
4220 if (v->vop_fplookup_vexec == NULL) {
4223 if (v->vop_fplookup_symlink == NULL) {
4236 if ((mp->mnt_kern_flag & MNTK_FPLOOKUP) == 0)
4239 if (vops->vop_fplookup_vexec == NULL ||
4240 vops->vop_fplookup_vexec == cache_vop_bad_vexec)
4242 vops, mp->mnt_vfc->vfc_name);
4244 if (vops->vop_fplookup_symlink == NULL ||
4245 vops->vop_fplookup_symlink == cache_vop_bad_symlink)
4247 vops, mp->mnt_vfc->vfc_name);
4279 if (error == 0 && req->newptr && old != atomic_load_int(&cache_fast_lookup))
4344 uma_zfree(namei_zone, cnp->cn_pnbuf);
4345 cnp->cn_pnbuf = NULL;
4346 cnp->cn_nameptr = NULL;
4355 ndp = fpl->ndp;
4356 cnp = fpl->cnp;
4358 MPASS(*(cnp->cn_nameptr) == '/');
4359 cnp->cn_nameptr++;
4362 if (__predict_false(*(cnp->cn_nameptr) == '/')) {
4364 cnp->cn_nameptr++;
4366 } while (*(cnp->cn_nameptr) == '/');
4369 return (ndp->ni_rootdir);
4376 fpl->snd_outer.ni_pathlen = fpl->ndp->ni_pathlen;
4377 fpl->snd_outer.cn_flags = fpl->ndp->ni_cnd.cn_flags;
4385 fpl->snd.cn_nameptr = fpl->ndp->ni_cnd.cn_nameptr;
4386 fpl->snd.ni_pathlen = fpl->debug.ni_pathlen;
4394 fpl->ndp->ni_cnd.cn_flags = fpl->snd_outer.cn_flags;
4396 fpl->debug.ni_pathlen = fpl->snd.ni_pathlen;
4408 fpl->ndp->ni_resflags = 0;
4409 fpl->ndp->ni_cnd.cn_nameptr = fpl->ndp->ni_cnd.cn_pnbuf;
4410 fpl->ndp->ni_pathlen = fpl->snd_outer.ni_pathlen;
4416 MPASS(_fpl->in_smr == true); \
4421 MPASS(_fpl->in_smr == false); \
4428 switch (fpl->status) {
4448 _fpl->in_smr = true; \
4453 MPASS(_fpl->in_smr == false); \
4455 _fpl->in_smr = true; \
4460 MPASS(_fpl->in_smr == true); \
4462 _fpl->in_smr = false; \
4469 if (fpl->status != CACHE_FPL_STATUS_UNSET) {
4470 KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
4472 __func__, fpl->status, line, fpl->line));
4475 fpl->status = CACHE_FPL_STATUS_ABORTED;
4476 fpl->line = line;
4488 ndp = fpl->ndp;
4489 cnp = fpl->cnp;
4491 if (fpl->status != CACHE_FPL_STATUS_UNSET) {
4492 KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
4494 __func__, fpl->status, line, fpl->line));
4496 fpl->status = CACHE_FPL_STATUS_ABORTED;
4497 fpl->line = line;
4498 if (fpl->in_smr)
4505 if (ndp->ni_loopcnt > 0) {
4506 fpl->status = CACHE_FPL_STATUS_DESTROYED;
4518 KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
4520 __func__, line, fpl->status, fpl->line));
4522 fpl->status = CACHE_FPL_STATUS_PARTIAL;
4523 fpl->line = line;
4533 KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
4535 __func__, line, fpl->status, fpl->line));
4537 fpl->status = CACHE_FPL_STATUS_HANDLED;
4538 fpl->line = line;
4548 KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
4550 __func__, line, fpl->status, fpl->line));
4554 fpl->status = CACHE_FPL_STATUS_HANDLED;
4555 fpl->line = line;
4556 fpl->dvp = NULL;
4557 fpl->tvp = NULL;
4567 return (fpl->status != CACHE_FPL_STATUS_UNSET);
4586 return (*ndp->ni_next == 0);
4593 MPASS(fpl->nulchar > fpl->cnp->cn_pnbuf);
4594 return (*(fpl->nulchar - 1) == '/');
4601 if (cnp->cn_namelen == 2 &&
4602 cnp->cn_nameptr[1] == '.' && cnp->cn_nameptr[0] == '.')
4614 ndp = fpl->ndp;
4615 cnp = fpl->cnp;
4622 if ((cnp->cn_flags & ~CACHE_FPL_SUPPORTED_CN_FLAGS) != 0) {
4634 if (ndp->ni_startdir != NULL) {
4648 ndp = fpl->ndp;
4649 cnp = fpl->cnp;
4662 fpl->fsearch = (flags & FSEARCH) != 0;
4663 if ((*vpp)->v_type != VDIR) {
4664 if (!((cnp->cn_flags & EMPTYPATH) != 0 && cnp->cn_pnbuf[0] == '\0')) {
4679 cnp = fpl->cnp;
4680 dvp = fpl->dvp;
4702 ndp = fpl->ndp;
4703 cnp = fpl->cnp;
4704 pwd = *(fpl->pwd);
4705 dvp = fpl->dvp;
4706 dvp_seqc = fpl->dvp_seqc;
4738 if (cnp->cn_nameptr != fpl->snd.cn_nameptr) {
4740 cnp->cn_nameptr, fpl->snd.cn_nameptr, cnp->cn_pnbuf);
4744 ndp->ni_startdir = dvp;
4745 cnp->cn_flags |= MAKEENTRY;
4747 cnp->cn_flags |= ISLASTCN;
4749 cnp->cn_flags |= ISDOTDOT;
4755 if (__predict_false(*(cnp->cn_nameptr) == '/')) {
4757 cnp->cn_nameptr++;
4759 } while (*(cnp->cn_nameptr) == '/');
4762 ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1;
4764 if (ndp->ni_pathlen != fpl->debug.ni_pathlen) {
4766 __func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
4767 cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
4781 cnp = fpl->cnp;
4782 tvp = fpl->tvp;
4783 tvp_seqc = fpl->tvp_seqc;
4785 if ((cnp->cn_flags & LOCKLEAF) != 0) {
4787 if ((cnp->cn_flags & LOCKSHARED) == 0)
4798 if ((cnp->cn_flags & LOCKLEAF) != 0)
4823 ndp = fpl->ndp;
4824 cnp = fpl->cnp;
4825 dvp = fpl->dvp;
4826 dvp_seqc = fpl->dvp_seqc;
4828 MPASS(*(cnp->cn_nameptr) != '/');
4830 if ((cnp->cn_flags & LOCKPARENT) == 0)
4831 MPASS((cnp->cn_flags & WANTPARENT) != 0);
4832 MPASS((cnp->cn_flags & TRAILINGSLASH) == 0);
4833 MPASS(cnp->cn_nameiop == CREATE || cnp->cn_nameiop == DELETE ||
4834 cnp->cn_nameiop == RENAME);
4835 MPASS((cnp->cn_flags & MAKEENTRY) == 0);
4836 MPASS((cnp->cn_flags & ISDOTDOT) == 0);
4838 docache = (cnp->cn_flags & NOCACHE) ^ NOCACHE;
4839 if (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)
4851 mp = atomic_load_ptr(&dvp->v_mount);
4856 if (__predict_false(mp->mnt_flag & MNT_RDONLY)) {
4862 if (cnp->cn_nameiop == CREATE) {
4868 if (fpl->tvp != NULL && (cnp->cn_flags & FAILIFEXISTS) != 0) {
4880 * - the target routine always locks the target vnode, but our caller
4882 * - some of the VOP machinery asserts that the parent is locked, which
4906 cnp->cn_flags |= ISLASTCN;
4908 cnp->cn_flags |= MAKEENTRY;
4910 cnp->cn_flags |= ISDOTDOT;
4911 cnp->cn_lkflags = LK_EXCLUSIVE;
4926 fpl->tvp = tvp;
4930 if ((cnp->cn_flags & LOCKPARENT) == 0) {
4953 if (tvp->v_type == VLNK) {
4954 if ((cnp->cn_flags & FOLLOW) != 0) {
4971 if ((cnp->cn_flags & FAILIFEXISTS) != 0) {
4977 if ((cnp->cn_flags & LOCKLEAF) == 0) {
4981 if ((cnp->cn_flags & LOCKPARENT) == 0) {
4993 ndp = fpl->ndp;
5010 cnp = fpl->cnp;
5011 dvp = fpl->dvp;
5012 dvp_seqc = fpl->dvp_seqc;
5013 tvp = fpl->tvp;
5015 MPASS((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0);
5033 if ((cnp->cn_flags & LOCKPARENT) != 0) {
5045 if ((cnp->cn_flags & LOCKPARENT) != 0)
5054 MPASS(fpl->status == CACHE_FPL_STATUS_ABORTED ||
5055 fpl->status == CACHE_FPL_STATUS_DESTROYED);
5056 if ((cnp->cn_flags & LOCKPARENT) != 0)
5063 MPASS(fpl->status == CACHE_FPL_STATUS_HANDLED);
5075 cnp = fpl->cnp;
5076 dvp = fpl->dvp;
5077 dvp_seqc = fpl->dvp_seqc;
5078 tvp = fpl->tvp;
5080 MPASS(*(cnp->cn_nameptr) != '/');
5082 if (cnp->cn_nameiop != LOOKUP) {
5086 if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0)
5120 fpl->tvp = fpl->dvp;
5121 fpl->tvp_seqc = fpl->dvp_seqc;
5123 cnp = fpl->cnp;
5124 dvp = fpl->dvp;
5127 for (cp = cnp->cn_pnbuf; *cp != '\0'; cp++) {
5129 ("%s: encountered non-slash; string [%s]\n", __func__,
5130 cnp->cn_pnbuf));
5134 if (__predict_false(cnp->cn_nameiop != LOOKUP)) {
5139 if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0) {
5149 if ((cnp->cn_flags & LOCKLEAF) != 0) {
5151 if ((cnp->cn_flags & LOCKSHARED) == 0)
5172 fpl->tvp = fpl->dvp;
5173 fpl->tvp_seqc = fpl->dvp_seqc;
5175 ndp = fpl->ndp;
5176 cnp = fpl->cnp;
5177 tvp = fpl->tvp;
5179 MPASS(*cnp->cn_pnbuf == '\0');
5181 if (__predict_false((cnp->cn_flags & EMPTYPATH) == 0)) {
5186 MPASS((cnp->cn_flags & (LOCKPARENT | WANTPARENT)) == 0);
5194 if ((cnp->cn_flags & LOCKLEAF) != 0) {
5196 if ((cnp->cn_flags & LOCKSHARED) == 0)
5206 ndp->ni_resflags |= NIRES_EMPTYPATH;
5220 ndp = fpl->ndp;
5221 cnp = fpl->cnp;
5222 dvp = fpl->dvp;
5223 dvp_seqc = fpl->dvp_seqc;
5225 MPASS((cnp->cn_flags & MAKEENTRY) == 0);
5226 MPASS((cnp->cn_flags & ISDOTDOT) == 0);
5227 if (cnp->cn_nameiop == LOOKUP)
5228 MPASS((cnp->cn_flags & NOCACHE) == 0);
5234 if (__predict_false(cnp->cn_namelen > NAME_MAX)) {
5239 if (cnp->cn_nameptr[0] == '/') {
5243 if (cnp->cn_pnbuf[0] == '\0') {
5247 if (cnp->cn_nameptr[0] == '\0') {
5248 if (fpl->tvp == NULL) {
5254 if (cnp->cn_nameiop != LOOKUP) {
5255 fpl->tvp = NULL;
5303 cnp->cn_flags |= ISLASTCN | MAKEENTRY;
5304 cnp->cn_lkflags = LK_SHARED;
5305 if ((cnp->cn_flags & LOCKSHARED) == 0) {
5306 cnp->cn_lkflags = LK_EXCLUSIVE;
5322 fpl->tvp = tvp;
5326 if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) {
5328 } else if ((cnp->cn_flags & LOCKPARENT) == 0) {
5334 if (tvp->v_type == VLNK) {
5335 if ((cnp->cn_flags & FOLLOW) != 0) {
5348 if ((cnp->cn_flags & LOCKLEAF) == 0) {
5352 if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) {
5354 } else if ((cnp->cn_flags & LOCKPARENT) == 0) {
5365 MPASS(!seqc_in_modify(fpl->dvp_seqc));
5367 if (__predict_false(fpl->dvp->v_type != VDIR)) {
5373 * Just re-assign the value. seqc will be checked later for the first
5374 * non-dot path component in line and/or before deciding to return the
5377 fpl->tvp = fpl->dvp;
5378 fpl->tvp_seqc = fpl->dvp_seqc;
5380 SDT_PROBE3(vfs, namecache, lookup, hit, fpl->dvp, ".", fpl->dvp);
5397 ndp = fpl->ndp;
5398 dvp = fpl->dvp;
5400 MPASS(cache_fpl_isdotdot(fpl->cnp));
5406 fpl->tvp = dvp;
5407 fpl->tvp_seqc = vn_seqc_read_any(dvp);
5408 if (seqc_in_modify(fpl->tvp_seqc)) {
5414 if ((dvp->v_vflag & VV_ROOT) != 0) {
5422 if (__predict_false(dvp->v_type != VDIR)) {
5427 ncp = atomic_load_consume_ptr(&dvp->v_cache_dd);
5432 nc_flag = atomic_load_char(&ncp->nc_flag);
5436 fpl->tvp = ncp->nc_vp;
5438 fpl->tvp = ncp->nc_dvp;
5441 fpl->tvp_seqc = vn_seqc_read_any(fpl->tvp);
5442 if (seqc_in_modify(fpl->tvp_seqc)) {
5449 if (__predict_false(atomic_load_ptr(&dvp->v_cache_dd) != ncp)) {
5467 nc_flag = atomic_load_char(&ncp->nc_flag);
5473 if (__predict_false(fpl->cnp->cn_nameiop != LOOKUP)) {
5474 fpl->tvp = NULL;
5491 * Resolve a symlink. Called by filesystem-specific routines.
5494 * ... -> cache_fplookup_symlink -> VOP_FPLOOKUP_SYMLINK -> cache_symlink_resolve
5503 ndp = fpl->ndp;
5504 cnp = fpl->cnp;
5510 if (__predict_false(len > MAXPATHLEN - 2)) {
5516 ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr - cnp->cn_namelen + 1;
5518 if (ndp->ni_pathlen != fpl->debug.ni_pathlen) {
5520 __func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
5521 cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
5525 if (__predict_false(len + ndp->ni_pathlen > MAXPATHLEN)) {
5529 if (__predict_false(ndp->ni_loopcnt++ >= MAXSYMLINKS)) {
5534 if (ndp->ni_pathlen > 1) {
5535 bcopy(ndp->ni_next, cnp->cn_pnbuf + len, ndp->ni_pathlen);
5539 cnp->cn_pnbuf[len] = '/';
5540 cnp->cn_pnbuf[len + 1] = '\0';
5542 cnp->cn_pnbuf[len] = '\0';
5545 bcopy(string, cnp->cn_pnbuf, len);
5547 ndp->ni_pathlen += adjust;
5549 cnp->cn_nameptr = cnp->cn_pnbuf;
5550 fpl->nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1];
5551 fpl->tvp = NULL;
5565 ndp = fpl->ndp;
5566 cnp = fpl->cnp;
5567 dvp = fpl->dvp;
5568 tvp = fpl->tvp;
5569 pwd = *(fpl->pwd);
5572 if ((cnp->cn_flags & FOLLOW) == 0) {
5577 mp = atomic_load_ptr(&dvp->v_mount);
5586 if (__predict_false((mp->mnt_flag & MNT_NOSYMFOLLOW) != 0)) {
5606 if (*(cnp->cn_nameptr) == '/') {
5607 fpl->dvp = cache_fpl_handle_root(fpl);
5608 fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
5609 if (seqc_in_modify(fpl->dvp_seqc)) {
5613 * The main loop assumes that ->dvp points to a vnode belonging
5617 mp = atomic_load_ptr(&fpl->dvp->v_mount);
5625 if (__predict_false(pwd->pwd_adir != pwd->pwd_rdir)) {
5642 cnp = fpl->cnp;
5643 dvp = fpl->dvp;
5644 hash = fpl->hash;
5646 if (__predict_false(cnp->cn_nameptr[0] == '.')) {
5647 if (cnp->cn_namelen == 1) {
5650 if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
5662 tvp = atomic_load_ptr(&ncp->nc_vp);
5663 nc_flag = atomic_load_char(&ncp->nc_flag);
5672 fpl->tvp = tvp;
5673 fpl->tvp_seqc = vn_seqc_read_any(tvp);
5674 if (seqc_in_modify(fpl->tvp_seqc)) {
5679 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp);
5693 if ((mp->mnt_kern_flag & MNTK_FPLOOKUP) == 0)
5702 * - all vnodes are protected from freeing with SMR
5703 * - struct mount objects are type stable making them always safe to access
5704 * - stability of the particular mount is provided by busying it
5705 * - relationship between the vnode which is mounted on and the mount is
5707 * - association between root vnode of the mount and the mount is protected
5724 vp = fpl->tvp;
5725 vp_seqc = fpl->tvp_seqc;
5727 VNPASS(vp->v_type == VDIR || vp->v_type == VREG || vp->v_type == VBAD, vp);
5728 mp = atomic_load_ptr(&vp->v_mountedhere);
5750 vp = atomic_load_ptr(&mp->mnt_rootvnode);
5762 mp = atomic_load_ptr(&vp->v_mountedhere);
5768 fpl->tvp = vp;
5769 fpl->tvp_seqc = vp_seqc;
5781 vp = fpl->tvp;
5782 vp_seqc = fpl->tvp_seqc;
5784 VNPASS(vp->v_type == VDIR || vp->v_type == VREG || vp->v_type == VBAD, vp);
5785 mp = atomic_load_ptr(&vp->v_mountedhere);
5801 vp = atomic_load_ptr(&mp->mnt_rootvnode);
5811 mp = atomic_load_ptr(&vp->v_mountedhere);
5821 fpl->tvp = vp;
5822 fpl->tvp_seqc = vp_seqc;
5834 vp = fpl->tvp;
5841 * The code was originally copy-pasted from regular lookup and despite
5855 fpl->debug.ni_pathlen += n;
5856 KASSERT(fpl->debug.ni_pathlen <= PATH_MAX,
5857 ("%s: pathlen overflow to %zd\n", __func__, fpl->debug.ni_pathlen));
5864 fpl->debug.ni_pathlen -= n;
5865 KASSERT(fpl->debug.ni_pathlen <= PATH_MAX,
5866 ("%s: pathlen underflow to %zd\n", __func__, fpl->debug.ni_pathlen));
5913 ndp = fpl->ndp;
5914 cnp = fpl->cnp;
5915 dvp = fpl->dvp;
5924 * TODO: fix this to be word-sized.
5926 MPASS(&cnp->cn_nameptr[fpl->debug.ni_pathlen - 1] >= cnp->cn_pnbuf);
5927 KASSERT(&cnp->cn_nameptr[fpl->debug.ni_pathlen - 1] == fpl->nulchar,
5929 __func__, fpl->debug.ni_pathlen, &cnp->cn_nameptr[fpl->debug.ni_pathlen - 1],
5930 fpl->nulchar, cnp->cn_pnbuf));
5931 KASSERT(*fpl->nulchar == '\0',
5932 ("%s: expected nul at %p; string [%s]\n", __func__, fpl->nulchar,
5933 cnp->cn_pnbuf));
5935 *fpl->nulchar = '/';
5936 for (cp = cnp->cn_nameptr; *cp != '/'; cp++) {
5939 cnp->cn_nameptr));
5943 *fpl->nulchar = '\0';
5944 fpl->hash = cache_get_hash_iter_finish(hash);
5946 cnp->cn_namelen = cp - cnp->cn_nameptr;
5947 cache_fpl_pathlen_sub(fpl, cnp->cn_namelen);
5954 if (cnp->cn_namelen <= NAME_MAX) {
5955 if (fpl->hash != cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp)) {
5957 cnp->cn_nameptr, cnp->cn_namelen);
5965 * be elided in the common case -- if an entry was found in the cache,
5968 ndp->ni_next = cp;
5977 ndp = fpl->ndp;
5978 cnp = fpl->cnp;
5980 cnp->cn_nameptr = ndp->ni_next;
5981 KASSERT(*(cnp->cn_nameptr) == '/',
5983 cnp->cn_nameptr, cnp->cn_pnbuf, cnp->cn_pnbuf));
5984 cnp->cn_nameptr++;
6003 ndp = fpl->ndp;
6004 cnp = fpl->cnp;
6006 MPASS(*(cnp->cn_nameptr) == '/');
6008 cnp->cn_nameptr++;
6010 } while (*(cnp->cn_nameptr) == '/');
6016 cnp->cn_nameptr--;
6020 * cache_fplookup_parse_advance starts from ndp->ni_next
6022 ndp->ni_next = cnp->cn_nameptr;
6027 fpl->tvp = fpl->dvp;
6028 fpl->tvp_seqc = fpl->dvp_seqc;
6042 * manner relying on an invariant that a non-directory vnode will get a miss.
6064 ndp = fpl->ndp;
6065 cnp = fpl->cnp;
6066 tvp = fpl->tvp;
6067 tvp_seqc = fpl->tvp_seqc;
6069 MPASS(fpl->dvp == fpl->tvp);
6071 ("%s: expected trailing slash at %p; string [%s]\n", __func__, fpl->nulchar - 1,
6072 cnp->cn_pnbuf));
6073 KASSERT(cnp->cn_nameptr[0] == '\0',
6074 ("%s: expected nul char at %p; string [%s]\n", __func__, &cnp->cn_nameptr[0],
6075 cnp->cn_pnbuf));
6076 KASSERT(cnp->cn_namelen == 0,
6077 ("%s: namelen 0 but got %ld; string [%s]\n", __func__, cnp->cn_namelen,
6078 cnp->cn_pnbuf));
6079 MPASS(cnp->cn_nameptr > cnp->cn_pnbuf);
6081 if (cnp->cn_nameiop != LOOKUP) {
6085 if (__predict_false(tvp->v_type != VDIR)) {
6096 ndp->ni_next = &cnp->cn_nameptr[0];
6102 cn_nameptr_orig = cnp->cn_nameptr;
6103 while (cnp->cn_nameptr >= cnp->cn_pnbuf) {
6104 cnp->cn_nameptr--;
6105 if (cnp->cn_nameptr[0] != '/') {
6115 cn_nameptr_slash = cnp->cn_nameptr;
6116 while (cnp->cn_nameptr > cnp->cn_pnbuf) {
6117 cnp->cn_nameptr--;
6118 if (cnp->cn_nameptr[0] == '/') {
6122 if (cnp->cn_nameptr[0] == '/') {
6123 cnp->cn_nameptr++;
6126 cnp->cn_namelen = cn_nameptr_slash - cnp->cn_nameptr + 1;
6127 cache_fpl_pathlen_add(fpl, cn_nameptr_orig - cnp->cn_nameptr);
6131 ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1;
6132 if (ni_pathlen != fpl->debug.ni_pathlen) {
6134 __func__, ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
6135 cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
6142 if (cnp->cn_nameptr[0] == '.' && cnp->cn_namelen == 1) {
6149 tvp = fpl->tvp;
6150 ncp = atomic_load_consume_ptr(&tvp->v_cache_dd);
6154 nc_flag = atomic_load_char(&ncp->nc_flag);
6158 fpl->dvp = ncp->nc_dvp;
6159 fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
6160 if (seqc_in_modify(fpl->dvp_seqc)) {
6176 cnp = fpl->cnp;
6177 dvp = fpl->dvp;
6178 dvp_seqc = fpl->dvp_seqc;
6183 if (cnp->cn_pnbuf[0] == '\0') {
6201 if (cnp->cn_nameptr[0] == '\0' && fpl->tvp == NULL) {
6208 if (__predict_false(cnp->cn_namelen > NAME_MAX)) {
6218 if (dvp->v_type != VDIR) {
6246 if (fpl->fsearch) {
6278 ndp = fpl->ndp;
6279 cnp = fpl->cnp;
6288 fpl->dvp = dvp;
6289 fpl->dvp_seqc = vn_seqc_read_notmodify(fpl->dvp);
6291 mp = atomic_load_ptr(&dvp->v_mount);
6296 MPASS(fpl->tvp == NULL);
6301 error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred);
6312 VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp);
6314 if (fpl->tvp->v_type == VLNK) {
6325 if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) {
6330 fpl->dvp = fpl->tvp;
6331 fpl->dvp_seqc = fpl->tvp_seqc;
6357 * little south of 100).
6431 * // at this point we know nothing has changed for any parent<->child pair
6437 * - they are called while within vfs_smr protection which they must never exit
6438 * - EAGAIN can be returned to denote checking could not be performed, it is
6440 * - if the sequence counter has not changed the result must be valid
6441 * - if the sequence counter has changed both false positives and false negatives
6443 * - for simple cases of unix permission checks vaccess_vexec_smr can be used
6446 * - vnodes are passed unlocked and unreferenced with nothing stopping
6447 * VOP_RECLAIM, in turn meaning that ->v_data can become NULL. It is advised
6449 * - the aforementioned object can also get freed, meaning absent other means it
6451 * - either safely checking permissions as they are modified or guaranteeing
6467 fpl.cnp = cnp = &ndp->ni_cnd;
6468 MPASS(ndp->ni_lcf == 0);
6469 KASSERT ((cnp->cn_flags & CACHE_FPL_INTERNAL_CN_FLAGS) == 0,
6471 cnp->cn_flags));
6472 MPASS(cnp->cn_nameptr == cnp->cn_pnbuf);
6473 MPASS(ndp->ni_resflags == 0);
6485 fpl.debug.ni_pathlen = ndp->ni_pathlen;
6487 fpl.nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1];
6494 ndp->ni_topdir = pwd->pwd_jdir;
6496 if (cnp->cn_pnbuf[0] == '/') {
6498 ndp->ni_resflags = NIRES_ABS;
6500 if (ndp->ni_dirfd == AT_FDCWD) {
6501 dvp = pwd->pwd_cdir;
6510 SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true);
6519 SDT_PROBE4(vfs, namei, lookup, return, error, ndp->ni_vp, true,
6526 ndp->ni_dvp = fpl.dvp;
6527 ndp->ni_vp = fpl.tvp;