Lines Matching full:ncp
454 cache_ncp_invalidate(struct namecache *ncp) in cache_ncp_invalidate() argument
457 KASSERT((ncp->nc_flag & NCF_INVALID) == 0, in cache_ncp_invalidate()
458 ("%s: entry %p already invalid", __func__, ncp)); in cache_ncp_invalidate()
459 atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_INVALID); in cache_ncp_invalidate()
467 cache_ncp_match(struct namecache *ncp, struct vnode *dvp, in cache_ncp_match() argument
470 return (ncp->nc_dvp == dvp && in cache_ncp_match()
471 ncp->nc_nlen == cnp->cn_namelen && in cache_ncp_match()
472 bcmp(ncp->nc_name, cnp->cn_nameptr, cnp->cn_namelen) == 0); in cache_ncp_match()
481 #define cache_ncp_canuse(ncp) ({ \ argument
482 struct namecache *_ncp = (ncp); \
493 #define cache_fpl_neg_ncp_canuse(ncp) ({ \ argument
494 struct namecache *_ncp = (ncp); \
560 NCP2NEGLIST(struct namecache *ncp) in NCP2NEGLIST() argument
563 return (&neglists[(((uintptr_t)(ncp) >> 8) & ncneghash)]); in NCP2NEGLIST()
567 NCP2NEGSTATE(struct namecache *ncp) in NCP2NEGSTATE() argument
570 MPASS(atomic_load_char(&ncp->nc_flag) & NCF_NEGATIVE); in NCP2NEGSTATE()
571 return (&ncp->nc_neg); in NCP2NEGSTATE()
597 struct namecache *ncp; in cache_ncp_find() local
601 CK_SLIST_FOREACH(ncp, NCHHASH(hash), nc_hash) { in cache_ncp_find()
602 if (cache_ncp_match(ncp, dvp, cnp)) in cache_ncp_find()
605 return (ncp); in cache_ncp_find()
609 cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp) in cache_out_ts() argument
613 KASSERT((ncp->nc_flag & NCF_TS) != 0 || in cache_out_ts()
620 ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); in cache_out_ts()
684 static void cache_zap_locked(struct namecache *ncp);
787 struct namecache *ncp; in cache_alloc_uma() local
794 ncp = &ncp_ts->nc_nc; in cache_alloc_uma()
797 ncp = uma_zalloc_smr(cache_zone_small, M_WAITOK); in cache_alloc_uma()
799 ncp = uma_zalloc_smr(cache_zone_large, M_WAITOK); in cache_alloc_uma()
801 return (ncp); in cache_alloc_uma()
805 cache_free_uma(struct namecache *ncp) in cache_free_uma() argument
809 if (__predict_false(ncp->nc_flag & NCF_TS)) { in cache_free_uma()
810 ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); in cache_free_uma()
811 if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) in cache_free_uma()
816 if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) in cache_free_uma()
817 uma_zfree_smr(cache_zone_small, ncp); in cache_free_uma()
819 uma_zfree_smr(cache_zone_large, ncp); in cache_free_uma()
853 cache_free(struct namecache *ncp) in cache_free() argument
856 MPASS(ncp != NULL); in cache_free()
857 if ((ncp->nc_flag & NCF_DVDROP) != 0) { in cache_free()
858 cache_drop_vnode(ncp->nc_dvp); in cache_free()
860 cache_free_uma(ncp); in cache_free()
867 struct namecache *ncp, *nnp; in cache_free_batch() local
873 TAILQ_FOREACH_SAFE(ncp, batch, nc_dst, nnp) { in cache_free_batch()
874 if ((ncp->nc_flag & NCF_DVDROP) != 0) { in cache_free_batch()
875 cache_drop_vnode(ncp->nc_dvp); in cache_free_batch()
877 cache_free_uma(ncp); in cache_free_batch()
947 NCP2BUCKET(struct namecache *ncp) in NCP2BUCKET() argument
951 hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp); in NCP2BUCKET()
956 NCP2BUCKETLOCK(struct namecache *ncp) in NCP2BUCKETLOCK() argument
960 hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp); in NCP2BUCKETLOCK()
966 cache_assert_bucket_locked(struct namecache *ncp) in cache_assert_bucket_locked() argument
970 blp = NCP2BUCKETLOCK(ncp); in cache_assert_bucket_locked()
975 cache_assert_bucket_unlocked(struct namecache *ncp) in cache_assert_bucket_unlocked() argument
979 blp = NCP2BUCKETLOCK(ncp); in cache_assert_bucket_unlocked()
1167 struct namecache *ncp; in sysctl_debug_hashstat_rawnchash() local
1183 CK_SLIST_FOREACH(ncp, ncpp, nc_hash) in sysctl_debug_hashstat_rawnchash()
1201 struct namecache *ncp; in sysctl_debug_hashstat_nchash() local
1216 CK_SLIST_FOREACH(ncp, ncpp, nc_hash) { in sysctl_debug_hashstat_nchash()
1310 cache_neg_init(struct namecache *ncp) in cache_neg_init() argument
1314 ncp->nc_flag |= NCF_NEGATIVE; in cache_neg_init()
1315 ns = NCP2NEGSTATE(ncp); in cache_neg_init()
1324 cache_neg_hit_prep(struct namecache *ncp) in cache_neg_hit_prep() argument
1329 ns = NCP2NEGSTATE(ncp); in cache_neg_hit_prep()
1345 #define cache_neg_hit_abort(ncp) do { } while (0) argument
1348 cache_neg_hit_finish(struct namecache *ncp) in cache_neg_hit_finish() argument
1351 SDT_PROBE2(vfs, namecache, lookup, hit__negative, ncp->nc_dvp, ncp->nc_name); in cache_neg_hit_finish()
1359 cache_neg_promote_locked(struct namecache *ncp) in cache_neg_promote_locked() argument
1364 ns = NCP2NEGSTATE(ncp); in cache_neg_promote_locked()
1365 nl = NCP2NEGLIST(ncp); in cache_neg_promote_locked()
1368 TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst); in cache_neg_promote_locked()
1369 TAILQ_INSERT_TAIL(&nl->nl_hotlist, ncp, nc_dst); in cache_neg_promote_locked()
1379 cache_neg_demote_locked(struct namecache *ncp) in cache_neg_demote_locked() argument
1384 ns = NCP2NEGSTATE(ncp); in cache_neg_demote_locked()
1385 nl = NCP2NEGLIST(ncp); in cache_neg_demote_locked()
1388 TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst); in cache_neg_demote_locked()
1389 TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst); in cache_neg_demote_locked()
1408 struct namecache *ncp; in cache_neg_promote_cond() local
1428 * In particular at this point there can be a new ncp which matches the in cache_neg_promote_cond()
1431 CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { in cache_neg_promote_cond()
1432 if (ncp == oncp) in cache_neg_promote_cond()
1439 if (__predict_false(ncp == NULL)) { in cache_neg_promote_cond()
1446 if (!cache_ncp_match(ncp, dvp, cnp)) { in cache_neg_promote_cond()
1453 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_neg_promote_cond()
1458 if (!cache_ncp_canuse(ncp)) { in cache_neg_promote_cond()
1462 cache_neg_promote_locked(ncp); in cache_neg_promote_cond()
1463 cache_neg_hit_finish(ncp); in cache_neg_promote_cond()
1474 cache_neg_promote(struct namecache *ncp) in cache_neg_promote() argument
1478 nl = NCP2NEGLIST(ncp); in cache_neg_promote()
1480 cache_neg_promote_locked(ncp); in cache_neg_promote()
1485 cache_neg_insert(struct namecache *ncp) in cache_neg_insert() argument
1489 MPASS(ncp->nc_flag & NCF_NEGATIVE); in cache_neg_insert()
1490 cache_assert_bucket_locked(ncp); in cache_neg_insert()
1491 nl = NCP2NEGLIST(ncp); in cache_neg_insert()
1493 TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst); in cache_neg_insert()
1499 cache_neg_remove(struct namecache *ncp) in cache_neg_remove() argument
1504 cache_assert_bucket_locked(ncp); in cache_neg_remove()
1505 nl = NCP2NEGLIST(ncp); in cache_neg_remove()
1506 ns = NCP2NEGSTATE(ncp); in cache_neg_remove()
1509 TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst); in cache_neg_remove()
1512 TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst); in cache_neg_remove()
1536 struct namecache *ncp, *lncp; in cache_neg_evict_select_entry() local
1542 ncp = TAILQ_FIRST(&nl->nl_list); in cache_neg_evict_select_entry()
1543 if (ncp == NULL) in cache_neg_evict_select_entry()
1545 lncp = ncp; in cache_neg_evict_select_entry()
1548 ncp = TAILQ_NEXT(ncp, nc_dst); in cache_neg_evict_select_entry()
1549 if (ncp == NULL) in cache_neg_evict_select_entry()
1551 ns = NCP2NEGSTATE(ncp); in cache_neg_evict_select_entry()
1553 lncp = ncp; in cache_neg_evict_select_entry()
1563 struct namecache *ncp, *ncp2; in cache_neg_evict() local
1578 ncp = TAILQ_FIRST(&nl->nl_hotlist); in cache_neg_evict()
1579 if (ncp != NULL) { in cache_neg_evict()
1580 cache_neg_demote_locked(ncp); in cache_neg_evict()
1582 ncp = cache_neg_evict_select_entry(nl); in cache_neg_evict()
1583 if (ncp == NULL) { in cache_neg_evict()
1589 nlen = ncp->nc_nlen; in cache_neg_evict()
1590 dvp = ncp->nc_dvp; in cache_neg_evict()
1591 hash = cache_get_hash(ncp->nc_name, nlen, dvp); in cache_neg_evict()
1603 if (ncp2 == ncp && ncp2->nc_dvp == dvp && in cache_neg_evict()
1609 ncp = NULL; in cache_neg_evict()
1612 MPASS(dvlp == VP2VNODELOCK(ncp->nc_dvp)); in cache_neg_evict()
1613 MPASS(blp == NCP2BUCKETLOCK(ncp)); in cache_neg_evict()
1614 SDT_PROBE2(vfs, namecache, evict_negative, done, ncp->nc_dvp, in cache_neg_evict()
1615 ncp->nc_name); in cache_neg_evict()
1616 cache_zap_locked(ncp); in cache_neg_evict()
1622 if (ncp != NULL) in cache_neg_evict()
1623 cache_free(ncp); in cache_neg_evict()
1662 cache_zap_locked(struct namecache *ncp) in cache_zap_locked() argument
1667 dvp = ncp->nc_dvp; in cache_zap_locked()
1668 vp = ncp->nc_vp; in cache_zap_locked()
1670 if (!(ncp->nc_flag & NCF_NEGATIVE)) in cache_zap_locked()
1673 cache_assert_bucket_locked(ncp); in cache_zap_locked()
1675 cache_ncp_invalidate(ncp); in cache_zap_locked()
1677 ncpp = NCP2BUCKET(ncp); in cache_zap_locked()
1678 CK_SLIST_REMOVE(ncpp, ncp, namecache, nc_hash); in cache_zap_locked()
1679 if (!(ncp->nc_flag & NCF_NEGATIVE)) { in cache_zap_locked()
1680 SDT_PROBE3(vfs, namecache, zap, done, dvp, ncp->nc_name, vp); in cache_zap_locked()
1681 TAILQ_REMOVE(&vp->v_cache_dst, ncp, nc_dst); in cache_zap_locked()
1682 if (ncp == vp->v_cache_dd) { in cache_zap_locked()
1686 SDT_PROBE2(vfs, namecache, zap_negative, done, dvp, ncp->nc_name); in cache_zap_locked()
1687 cache_neg_remove(ncp); in cache_zap_locked()
1689 if (ncp->nc_flag & NCF_ISDOTDOT) { in cache_zap_locked()
1690 if (ncp == dvp->v_cache_dd) { in cache_zap_locked()
1694 LIST_REMOVE(ncp, nc_src); in cache_zap_locked()
1696 ncp->nc_flag |= NCF_DVDROP; in cache_zap_locked()
1702 cache_zap_negative_locked_vnode_kl(struct namecache *ncp, struct vnode *vp) in cache_zap_negative_locked_vnode_kl() argument
1706 MPASS(ncp->nc_dvp == vp); in cache_zap_negative_locked_vnode_kl()
1707 MPASS(ncp->nc_flag & NCF_NEGATIVE); in cache_zap_negative_locked_vnode_kl()
1710 blp = NCP2BUCKETLOCK(ncp); in cache_zap_negative_locked_vnode_kl()
1712 cache_zap_locked(ncp); in cache_zap_negative_locked_vnode_kl()
1717 cache_zap_locked_vnode_kl2(struct namecache *ncp, struct vnode *vp, in cache_zap_locked_vnode_kl2() argument
1723 MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp); in cache_zap_locked_vnode_kl2()
1726 if (ncp->nc_flag & NCF_NEGATIVE) { in cache_zap_locked_vnode_kl2()
1731 cache_zap_negative_locked_vnode_kl(ncp, vp); in cache_zap_locked_vnode_kl2()
1736 blp = NCP2BUCKETLOCK(ncp); in cache_zap_locked_vnode_kl2()
1737 vlp1 = VP2VNODELOCK(ncp->nc_dvp); in cache_zap_locked_vnode_kl2()
1738 vlp2 = VP2VNODELOCK(ncp->nc_vp); in cache_zap_locked_vnode_kl2()
1759 cache_zap_locked(ncp); in cache_zap_locked_vnode_kl2()
1779 cache_zap_unlocked_bucket(struct namecache *ncp, struct componentname *cnp, in cache_zap_unlocked_bucket() argument
1786 cache_assert_bucket_unlocked(ncp); in cache_zap_unlocked_bucket()
1792 if (rncp == ncp && cache_ncp_match(rncp, dvp, cnp)) in cache_zap_unlocked_bucket()
1798 if (!(ncp->nc_flag & NCF_NEGATIVE)) in cache_zap_unlocked_bucket()
1818 cache_zap_locked_bucket(struct namecache *ncp, struct componentname *cnp, in cache_zap_locked_bucket() argument
1824 cache_assert_bucket_locked(ncp); in cache_zap_locked_bucket()
1826 dvlp = VP2VNODELOCK(ncp->nc_dvp); in cache_zap_locked_bucket()
1828 if (!(ncp->nc_flag & NCF_NEGATIVE)) in cache_zap_locked_bucket()
1829 vlp = VP2VNODELOCK(ncp->nc_vp); in cache_zap_locked_bucket()
1831 cache_zap_locked(ncp); in cache_zap_locked_bucket()
1837 dvp = ncp->nc_dvp; in cache_zap_locked_bucket()
1839 return (cache_zap_unlocked_bucket(ncp, cnp, dvp, dvlp, vlp, hash, blp)); in cache_zap_locked_bucket()
1845 struct namecache *ncp; in cache_remove_cnp() local
1857 ncp = dvp->v_cache_dd; in cache_remove_cnp()
1858 if (ncp == NULL) { in cache_remove_cnp()
1865 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) { in cache_remove_cnp()
1866 if (!cache_zap_locked_vnode_kl2(ncp, dvp, &dvlp2)) in cache_remove_cnp()
1872 cache_free(ncp); in cache_remove_cnp()
1899 ncp = cache_ncp_find(dvp, cnp, hash); in cache_remove_cnp()
1900 if (ncp == NULL) { in cache_remove_cnp()
1905 error = cache_zap_locked_bucket(ncp, cnp, hash, blp); in cache_remove_cnp()
1912 cache_free(ncp); in cache_remove_cnp()
1958 struct namecache *ncp; in cache_lookup_dotdot() local
1974 ncp = dvp->v_cache_dd; in cache_lookup_dotdot()
1975 if (ncp == NULL) { in cache_lookup_dotdot()
1980 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) { in cache_lookup_dotdot()
1981 if (ncp->nc_flag & NCF_NEGATIVE) in cache_lookup_dotdot()
1984 *vpp = ncp->nc_vp; in cache_lookup_dotdot()
1986 *vpp = ncp->nc_dvp; in cache_lookup_dotdot()
1990 cache_out_ts(ncp, tsp, ticksp); in cache_lookup_dotdot()
1991 if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) == in cache_lookup_dotdot()
1993 ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); in cache_lookup_dotdot()
2019 cache_zap_negative_locked_vnode_kl(ncp, dvp); in cache_lookup_dotdot()
2021 cache_free(ncp); in cache_lookup_dotdot()
2026 whiteout = (ncp->nc_flag & NCF_WHITE); in cache_lookup_dotdot()
2027 cache_out_ts(ncp, tsp, ticksp); in cache_lookup_dotdot()
2028 if (cache_neg_hit_prep(ncp)) in cache_lookup_dotdot()
2029 cache_neg_promote(ncp); in cache_lookup_dotdot()
2031 cache_neg_hit_finish(ncp); in cache_lookup_dotdot()
2080 struct namecache *ncp; in cache_lookup_fallback() local
2095 ncp = cache_ncp_find(dvp, cnp, hash); in cache_lookup_fallback()
2096 if (__predict_false(ncp == NULL)) { in cache_lookup_fallback()
2103 if (ncp->nc_flag & NCF_NEGATIVE) in cache_lookup_fallback()
2107 *vpp = ncp->nc_vp; in cache_lookup_fallback()
2108 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp); in cache_lookup_fallback()
2109 cache_out_ts(ncp, tsp, ticksp); in cache_lookup_fallback()
2126 error = cache_zap_locked_bucket(ncp, cnp, hash, blp); in cache_lookup_fallback()
2131 cache_free(ncp); in cache_lookup_fallback()
2136 whiteout = (ncp->nc_flag & NCF_WHITE); in cache_lookup_fallback()
2137 cache_out_ts(ncp, tsp, ticksp); in cache_lookup_fallback()
2138 if (cache_neg_hit_prep(ncp)) in cache_lookup_fallback()
2139 cache_neg_promote(ncp); in cache_lookup_fallback()
2141 cache_neg_hit_finish(ncp); in cache_lookup_fallback()
2152 struct namecache *ncp; in cache_lookup() local
2185 ncp = cache_ncp_find(dvp, cnp, hash); in cache_lookup()
2186 if (__predict_false(ncp == NULL)) { in cache_lookup()
2193 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_lookup()
2198 *vpp = ncp->nc_vp; in cache_lookup()
2199 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp); in cache_lookup()
2200 cache_out_ts(ncp, tsp, ticksp); in cache_lookup()
2202 if (!cache_ncp_canuse(ncp)) { in cache_lookup()
2227 cache_out_ts(ncp, tsp, ticksp); in cache_lookup()
2228 whiteout = (atomic_load_char(&ncp->nc_flag) & NCF_WHITE); in cache_lookup()
2229 neg_promote = cache_neg_hit_prep(ncp); in cache_lookup()
2230 if (!cache_ncp_canuse(ncp)) { in cache_lookup()
2231 cache_neg_hit_abort(ncp); in cache_lookup()
2237 if (!cache_neg_promote_cond(dvp, cnp, ncp, hash)) in cache_lookup()
2240 cache_neg_hit_finish(ncp); in cache_lookup()
2381 struct namecache *ncp; in cache_enter_lock() local
2391 ncp = atomic_load_consume_ptr(&vp->v_cache_dd); in cache_enter_lock()
2392 if (ncp == NULL) in cache_enter_lock()
2394 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_enter_lock()
2397 MPASS(ncp->nc_dvp == vp); in cache_enter_lock()
2398 blps[1] = NCP2BUCKETLOCK(ncp); in cache_enter_lock()
2401 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp)) in cache_enter_lock()
2407 if (ncp == vp->v_cache_dd && in cache_enter_lock()
2408 (ncp->nc_flag & NCF_ISDOTDOT) != 0 && in cache_enter_lock()
2409 blps[1] == NCP2BUCKETLOCK(ncp) && in cache_enter_lock()
2410 VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2]) in cache_enter_lock()
2424 struct namecache *ncp; in cache_enter_lock_dd() local
2432 ncp = atomic_load_consume_ptr(&dvp->v_cache_dd); in cache_enter_lock_dd()
2433 if (ncp == NULL) in cache_enter_lock_dd()
2435 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_enter_lock_dd()
2438 MPASS(ncp->nc_dvp == dvp); in cache_enter_lock_dd()
2439 blps[1] = NCP2BUCKETLOCK(ncp); in cache_enter_lock_dd()
2442 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp)) in cache_enter_lock_dd()
2444 if (ncp == dvp->v_cache_dd && in cache_enter_lock_dd()
2445 (ncp->nc_flag & NCF_ISDOTDOT) != 0 && in cache_enter_lock_dd()
2446 blps[1] == NCP2BUCKETLOCK(ncp) && in cache_enter_lock_dd()
2447 VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2]) in cache_enter_lock_dd()
2470 struct namecache *ncp; in cache_enter_dotdot_prep() local
2480 ncp = dvp->v_cache_dd; in cache_enter_dotdot_prep()
2481 if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT)) { in cache_enter_dotdot_prep()
2482 KASSERT(ncp->nc_dvp == dvp, ("wrong isdotdot parent")); in cache_enter_dotdot_prep()
2483 cache_zap_locked(ncp); in cache_enter_dotdot_prep()
2485 ncp = NULL; in cache_enter_dotdot_prep()
2489 if (ncp != NULL) in cache_enter_dotdot_prep()
2490 cache_free(ncp); in cache_enter_dotdot_prep()
2501 struct namecache *ncp, *n2, *ndd; in cache_enter_time() local
2541 ncp = cache_alloc(cnp->cn_namelen, tsp != NULL); in cache_enter_time()
2542 if (ncp == NULL) in cache_enter_time()
2553 ncp->nc_flag = flag | NCF_WIP; in cache_enter_time()
2554 ncp->nc_vp = vp; in cache_enter_time()
2556 cache_neg_init(ncp); in cache_enter_time()
2557 ncp->nc_dvp = dvp; in cache_enter_time()
2559 ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); in cache_enter_time()
2568 len = ncp->nc_nlen = cnp->cn_namelen; in cache_enter_time()
2570 memcpy(ncp->nc_name, cnp->cn_nameptr, len); in cache_enter_time()
2571 ncp->nc_name[len] = '\0'; in cache_enter_time()
2602 (ncp->nc_flag & (NCF_TS | NCF_DTS))); in cache_enter_time()
2616 SDT_PROBE3(vfs, namecache, enter, duplicate, dvp, ncp->nc_name, in cache_enter_time()
2631 atomic_store_ptr(&dvp->v_cache_dd, ncp); in cache_enter_time()
2653 atomic_store_ptr(&vp->v_cache_dd, ncp); in cache_enter_time()
2660 LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src); in cache_enter_time()
2669 TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst); in cache_enter_time()
2670 SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name, in cache_enter_time()
2674 atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_WHITE); in cache_enter_time()
2675 cache_neg_insert(ncp); in cache_enter_time()
2677 ncp->nc_name); in cache_enter_time()
2684 CK_SLIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash); in cache_enter_time()
2691 atomic_store_char(&ncp->nc_flag, ncp->nc_flag & ~NCF_WIP); in cache_enter_time()
2699 cache_free(ncp); in cache_enter_time()
2889 struct namecache *ncp; in cache_changesize() local
2919 while ((ncp = CK_SLIST_FIRST(&old_nchashtbl[i])) != NULL) { in cache_changesize()
2920 hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, in cache_changesize()
2921 ncp->nc_dvp); in cache_changesize()
2922 CK_SLIST_REMOVE(&old_nchashtbl[i], ncp, namecache, nc_hash); in cache_changesize()
2923 CK_SLIST_INSERT_HEAD(&new_nchashtbl[hash & new_nchash], ncp, nc_hash); in cache_changesize()
2942 struct namecache *ncp; in cache_purge_impl() local
2951 ncp = LIST_FIRST(&vp->v_cache_src); in cache_purge_impl()
2952 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2)) in cache_purge_impl()
2954 TAILQ_INSERT_TAIL(&batch, ncp, nc_dst); in cache_purge_impl()
2957 ncp = TAILQ_FIRST(&vp->v_cache_dst); in cache_purge_impl()
2958 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2)) in cache_purge_impl()
2960 TAILQ_INSERT_TAIL(&batch, ncp, nc_dst); in cache_purge_impl()
2962 ncp = vp->v_cache_dd; in cache_purge_impl()
2963 if (ncp != NULL) { in cache_purge_impl()
2964 KASSERT(ncp->nc_flag & NCF_ISDOTDOT, in cache_purge_impl()
2966 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2)) in cache_purge_impl()
2968 TAILQ_INSERT_TAIL(&batch, ncp, nc_dst); in cache_purge_impl()
3033 struct namecache *ncp, *nnp; in cache_purge_negative() local
3042 LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) { in cache_purge_negative()
3043 if (!(ncp->nc_flag & NCF_NEGATIVE)) in cache_purge_negative()
3045 cache_zap_negative_locked_vnode_kl(ncp, vp); in cache_purge_negative()
3046 TAILQ_INSERT_TAIL(&batch, ncp, nc_dst); in cache_purge_negative()
3105 struct namecache *ncp; in cache_validate() local
3114 ncp = cache_ncp_find(dvp, cnp, hash); in cache_validate()
3115 if (ncp != NULL && ncp->nc_vp != vp) { in cache_validate()
3116 panic("%s: mismatch (%p != %p); ncp %p [%s] dvp %p\n", in cache_validate()
3117 __func__, vp, ncp->nc_vp, ncp, ncp->nc_name, ncp->nc_dvp); in cache_validate()
3390 struct namecache *ncp; in vn_dd_from_dst() local
3393 TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) { in vn_dd_from_dst()
3394 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) in vn_dd_from_dst()
3395 return (ncp); in vn_dd_from_dst()
3404 struct namecache *ncp; in vn_vptocnp() local
3410 ncp = (*vp)->v_cache_dd; in vn_vptocnp()
3411 if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT) == 0) { in vn_vptocnp()
3412 KASSERT(ncp == vn_dd_from_dst(*vp), in vn_vptocnp()
3414 ncp, vn_dd_from_dst(*vp))); in vn_vptocnp()
3416 ncp = vn_dd_from_dst(*vp); in vn_vptocnp()
3418 if (ncp != NULL) { in vn_vptocnp()
3419 if (*buflen < ncp->nc_nlen) { in vn_vptocnp()
3428 *buflen -= ncp->nc_nlen; in vn_vptocnp()
3429 memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen); in vn_vptocnp()
3430 SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp, in vn_vptocnp()
3431 ncp->nc_name, vp); in vn_vptocnp()
3433 *vp = ncp->nc_dvp; in vn_vptocnp()
3599 struct namecache *ncp; in vn_fullpath_any_smr() local
3636 ncp = NULL; /* for sdt probe down below */ in vn_fullpath_any_smr()
3667 ncp = atomic_load_consume_ptr(&vp->v_cache_dd); in vn_fullpath_any_smr()
3668 if (ncp == NULL) { in vn_fullpath_any_smr()
3672 nc_flag = atomic_load_char(&ncp->nc_flag); in vn_fullpath_any_smr()
3677 if (ncp->nc_nlen >= *buflen) { in vn_fullpath_any_smr()
3682 *buflen -= ncp->nc_nlen; in vn_fullpath_any_smr()
3683 memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen); in vn_fullpath_any_smr()
3686 tvp = ncp->nc_dvp; in vn_fullpath_any_smr()
3699 if (__predict_false(atomic_load_ptr(&vp->v_cache_dd) != ncp)) { in vn_fullpath_any_smr()
3703 if (!cache_ncp_canuse(ncp)) { in vn_fullpath_any_smr()
3721 SDT_PROBE4(vfs, namecache, fullpath_smr, miss, startvp, ncp, reason, i); in vn_fullpath_any_smr()
3848 struct namecache *ncp; in vn_dir_dd_ino() local
3856 TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) { in vn_dir_dd_ino()
3857 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) in vn_dir_dd_ino()
3859 ddvp = ncp->nc_dvp; in vn_dir_dd_ino()
3873 struct namecache *ncp; in vn_commname() local
3879 TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) in vn_commname()
3880 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) in vn_commname()
3882 if (ncp == NULL) { in vn_commname()
3886 l = min(ncp->nc_nlen, buflen - 1); in vn_commname()
3887 memcpy(buf, ncp->nc_name, l); in vn_commname()
4025 struct namecache *ncp; in cache_vop_inotify() local
4048 TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) { in cache_vop_inotify()
4049 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) in cache_vop_inotify()
4051 if ((vn_irflag_read(ncp->nc_dvp) & VIRF_INOTIFY) != 0) { in cache_vop_inotify()
4056 inotify_log(ncp->nc_dvp, ncp->nc_name, ncp->nc_nlen, in cache_vop_inotify()
4086 struct namecache *ncp; in db_print_vpath() local
4090 ncp = TAILQ_FIRST(&vp->v_cache_dst); in db_print_vpath()
4091 if (ncp != NULL) { in db_print_vpath()
4092 ncn = ncp->nc_name; in db_print_vpath()
4093 for (i = 0; i < ncp->nc_nlen; i++) in db_print_vpath()
4095 vp = ncp->nc_dvp; in db_print_vpath()
5338 struct namecache *ncp; in cache_fplookup_dotdot() local
5372 ncp = atomic_load_consume_ptr(&dvp->v_cache_dd); in cache_fplookup_dotdot()
5373 if (ncp == NULL) { in cache_fplookup_dotdot()
5377 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_fplookup_dotdot()
5381 fpl->tvp = ncp->nc_vp; in cache_fplookup_dotdot()
5383 fpl->tvp = ncp->nc_dvp; in cache_fplookup_dotdot()
5394 if (__predict_false(atomic_load_ptr(&dvp->v_cache_dd) != ncp)) { in cache_fplookup_dotdot()
5398 if (!cache_ncp_canuse(ncp)) { in cache_fplookup_dotdot()
5406 cache_fplookup_neg(struct cache_fpl *fpl, struct namecache *ncp, uint32_t hash) in cache_fplookup_neg() argument
5412 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_fplookup_neg()
5422 neg_promote = cache_neg_hit_prep(ncp); in cache_fplookup_neg()
5423 if (!cache_fpl_neg_ncp_canuse(ncp)) { in cache_fplookup_neg()
5424 cache_neg_hit_abort(ncp); in cache_fplookup_neg()
5428 return (cache_fplookup_negative_promote(fpl, ncp, hash)); in cache_fplookup_neg()
5430 cache_neg_hit_finish(ncp); in cache_fplookup_neg()
5581 struct namecache *ncp; in cache_fplookup_next() local
5602 ncp = cache_ncp_find(dvp, cnp, hash); in cache_fplookup_next()
5603 if (__predict_false(ncp == NULL)) { in cache_fplookup_next()
5607 tvp = atomic_load_ptr(&ncp->nc_vp); in cache_fplookup_next()
5608 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_fplookup_next()
5610 return (cache_fplookup_neg(fpl, ncp, hash)); in cache_fplookup_next()
5613 if (!cache_ncp_canuse(ncp)) { in cache_fplookup_next()
5624 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp); in cache_fplookup_next()
6003 struct namecache *ncp; in cache_fplookup_trailingslash() local
6095 ncp = atomic_load_consume_ptr(&tvp->v_cache_dd); in cache_fplookup_trailingslash()
6096 if (__predict_false(ncp == NULL)) { in cache_fplookup_trailingslash()
6099 nc_flag = atomic_load_char(&ncp->nc_flag); in cache_fplookup_trailingslash()
6103 fpl->dvp = ncp->nc_dvp; in cache_fplookup_trailingslash()