xref: /freebsd/sys/kern/vfs_cache.c (revision c1d255d3ffdbe447de3ab875bf4e7d7accc5bfc5)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1989, 1993, 1995
5  *	The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * Poul-Henning Kamp of the FreeBSD Project.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	@(#)vfs_cache.c	8.5 (Berkeley) 3/22/95
35  */
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include "opt_ddb.h"
41 #include "opt_ktrace.h"
42 
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/capsicum.h>
46 #include <sys/counter.h>
47 #include <sys/filedesc.h>
48 #include <sys/fnv_hash.h>
49 #include <sys/kernel.h>
50 #include <sys/ktr.h>
51 #include <sys/lock.h>
52 #include <sys/malloc.h>
53 #include <sys/fcntl.h>
54 #include <sys/jail.h>
55 #include <sys/mount.h>
56 #include <sys/namei.h>
57 #include <sys/proc.h>
58 #include <sys/seqc.h>
59 #include <sys/sdt.h>
60 #include <sys/smr.h>
61 #include <sys/smp.h>
62 #include <sys/syscallsubr.h>
63 #include <sys/sysctl.h>
64 #include <sys/sysproto.h>
65 #include <sys/vnode.h>
66 #include <ck_queue.h>
67 #ifdef KTRACE
68 #include <sys/ktrace.h>
69 #endif
70 #ifdef INVARIANTS
71 #include <machine/_inttypes.h>
72 #endif
73 
74 #include <sys/capsicum.h>
75 
76 #include <security/audit/audit.h>
77 #include <security/mac/mac_framework.h>
78 
79 #ifdef DDB
80 #include <ddb/ddb.h>
81 #endif
82 
83 #include <vm/uma.h>
84 
85 /*
86  * High level overview of name caching in the VFS layer.
87  *
88  * Originally caching was implemented as part of UFS, later extracted to allow
89  * use by other filesystems. A decision was made to make it optional and
90  * completely detached from the rest of the kernel, which comes with limitations
91  * outlined near the end of this comment block.
92  *
93  * This fundamental choice needs to be revisited. In the meantime, the current
94  * state is described below. Significance of all notable routines is explained
95  * in comments placed above their implementation. Scattered thoroughout the
96  * file are TODO comments indicating shortcomings which can be fixed without
97  * reworking everything (most of the fixes will likely be reusable). Various
98  * details are omitted from this explanation to not clutter the overview, they
99  * have to be checked by reading the code and associated commentary.
100  *
101  * Keep in mind that it's individual path components which are cached, not full
102  * paths. That is, for a fully cached path "foo/bar/baz" there are 3 entries,
103  * one for each name.
104  *
105  * I. Data organization
106  *
107  * Entries are described by "struct namecache" objects and stored in a hash
108  * table. See cache_get_hash for more information.
109  *
110  * "struct vnode" contains pointers to source entries (names which can be found
111  * when traversing through said vnode), destination entries (names of that
112  * vnode (see "Limitations" for a breakdown on the subject) and a pointer to
113  * the parent vnode.
114  *
115  * The (directory vnode; name) tuple reliably determines the target entry if
116  * it exists.
117  *
118  * Since there are no small locks at this time (all are 32 bytes in size on
119  * LP64), the code works around the problem by introducing lock arrays to
120  * protect hash buckets and vnode lists.
121  *
122  * II. Filesystem integration
123  *
124  * Filesystems participating in name caching do the following:
125  * - set vop_lookup routine to vfs_cache_lookup
126  * - set vop_cachedlookup to whatever can perform the lookup if the above fails
127  * - if they support lockless lookup (see below), vop_fplookup_vexec and
128  *   vop_fplookup_symlink are set along with the MNTK_FPLOOKUP flag on the
129  *   mount point
130  * - call cache_purge or cache_vop_* routines to eliminate stale entries as
131  *   applicable
132  * - call cache_enter to add entries depending on the MAKEENTRY flag
133  *
134  * With the above in mind, there are 2 entry points when doing lookups:
135  * - ... -> namei -> cache_fplookup -- this is the default
136  * - ... -> VOP_LOOKUP -> vfs_cache_lookup -- normally only called by namei
137  *   should the above fail
138  *
139  * Example code flow how an entry is added:
140  * ... -> namei -> cache_fplookup -> cache_fplookup_noentry -> VOP_LOOKUP ->
141  * vfs_cache_lookup -> VOP_CACHEDLOOKUP -> ufs_lookup_ino -> cache_enter
142  *
143  * III. Performance considerations
144  *
145  * For lockless case forward lookup avoids any writes to shared areas apart
146  * from the terminal path component. In other words non-modifying lookups of
147  * different files don't suffer any scalability problems in the namecache.
148  * Looking up the same file is limited by VFS and goes beyond the scope of this
149  * file.
150  *
151  * At least on amd64 the single-threaded bottleneck for long paths is hashing
152  * (see cache_get_hash). There are cases where the code issues acquire fence
153  * multiple times, they can be combined on architectures which suffer from it.
154  *
155  * For locked case each encountered vnode has to be referenced and locked in
156  * order to be handed out to the caller (normally that's namei). This
157  * introduces significant hit single-threaded and serialization multi-threaded.
158  *
159  * Reverse lookup (e.g., "getcwd") fully scales provided it is fully cached --
160  * avoids any writes to shared areas to any components.
161  *
162  * Unrelated insertions are partially serialized on updating the global entry
163  * counter and possibly serialized on colliding bucket or vnode locks.
164  *
165  * IV. Observability
166  *
167  * Note not everything has an explicit dtrace probe nor it should have, thus
168  * some of the one-liners below depend on implementation details.
169  *
170  * Examples:
171  *
172  * # Check what lookups failed to be handled in a lockless manner. Column 1 is
173  * # line number, column 2 is status code (see cache_fpl_status)
174  * dtrace -n 'vfs:fplookup:lookup:done { @[arg1, arg2] = count(); }'
175  *
176  * # Lengths of names added by binary name
177  * dtrace -n 'fbt::cache_enter_time:entry { @[execname] = quantize(args[2]->cn_namelen); }'
178  *
179  * # Same as above but only those which exceed 64 characters
180  * dtrace -n 'fbt::cache_enter_time:entry /args[2]->cn_namelen > 64/ { @[execname] = quantize(args[2]->cn_namelen); }'
181  *
182  * # Who is performing lookups with spurious slashes (e.g., "foo//bar") and what
183  * # path is it
184  * dtrace -n 'fbt::cache_fplookup_skip_slashes:entry { @[execname, stringof(args[0]->cnp->cn_pnbuf)] = count(); }'
185  *
186  * V. Limitations and implementation defects
187  *
188  * - since it is possible there is no entry for an open file, tools like
189  *   "procstat" may fail to resolve fd -> vnode -> path to anything
190  * - even if a filesystem adds an entry, it may get purged (e.g., due to memory
191  *   shortage) in which case the above problem applies
192  * - hardlinks are not tracked, thus if a vnode is reachable in more than one
193  *   way, resolving a name may return a different path than the one used to
194  *   open it (even if said path is still valid)
195  * - by default entries are not added for newly created files
196  * - adding an entry may need to evict negative entry first, which happens in 2
197  *   distinct places (evicting on lookup, adding in a later VOP) making it
198  *   impossible to simply reuse it
199  * - there is a simple scheme to evict negative entries as the cache is approaching
200  *   its capacity, but it is very unclear if doing so is a good idea to begin with
201  * - vnodes are subject to being recycled even if target inode is left in memory,
202  *   which loses the name cache entries when it perhaps should not. in case of tmpfs
203  *   names get duplicated -- kept by filesystem itself and namecache separately
204  * - struct namecache has a fixed size and comes in 2 variants, often wasting space.
205  *   now hard to replace with malloc due to dependence on SMR.
206  * - lack of better integration with the kernel also turns nullfs into a layered
207  *   filesystem instead of something which can take advantage of caching
208  */
209 
210 static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
211     "Name cache");
212 
213 SDT_PROVIDER_DECLARE(vfs);
214 SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode *", "char *",
215     "struct vnode *");
216 SDT_PROBE_DEFINE3(vfs, namecache, enter, duplicate, "struct vnode *", "char *",
217     "struct vnode *");
218 SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *",
219     "char *");
220 SDT_PROBE_DEFINE2(vfs, namecache, fullpath_smr, hit, "struct vnode *",
221     "const char *");
222 SDT_PROBE_DEFINE4(vfs, namecache, fullpath_smr, miss, "struct vnode *",
223     "struct namecache *", "int", "int");
224 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *");
225 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *",
226     "char *", "struct vnode *");
227 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *");
228 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int",
229     "struct vnode *", "char *");
230 SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode *", "char *",
231     "struct vnode *");
232 SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit__negative,
233     "struct vnode *", "char *");
234 SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *",
235     "char *");
236 SDT_PROBE_DEFINE2(vfs, namecache, removecnp, hit, "struct vnode *",
237     "struct componentname *");
238 SDT_PROBE_DEFINE2(vfs, namecache, removecnp, miss, "struct vnode *",
239     "struct componentname *");
240 SDT_PROBE_DEFINE1(vfs, namecache, purge, done, "struct vnode *");
241 SDT_PROBE_DEFINE1(vfs, namecache, purge, batch, "int");
242 SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *");
243 SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *");
244 SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode *", "char *",
245     "struct vnode *");
246 SDT_PROBE_DEFINE2(vfs, namecache, zap_negative, done, "struct vnode *",
247     "char *");
248 SDT_PROBE_DEFINE2(vfs, namecache, evict_negative, done, "struct vnode *",
249     "char *");
250 SDT_PROBE_DEFINE1(vfs, namecache, symlink, alloc__fail, "size_t");
251 
252 SDT_PROBE_DEFINE3(vfs, fplookup, lookup, done, "struct nameidata", "int", "bool");
253 SDT_PROBE_DECLARE(vfs, namei, lookup, entry);
254 SDT_PROBE_DECLARE(vfs, namei, lookup, return);
255 
256 static char __read_frequently cache_fast_lookup_enabled = true;
257 
258 /*
259  * This structure describes the elements in the cache of recent
260  * names looked up by namei.
261  */
262 struct negstate {
263 	u_char neg_flag;
264 	u_char neg_hit;
265 };
266 _Static_assert(sizeof(struct negstate) <= sizeof(struct vnode *),
267     "the state must fit in a union with a pointer without growing it");
268 
269 struct	namecache {
270 	LIST_ENTRY(namecache) nc_src;	/* source vnode list */
271 	TAILQ_ENTRY(namecache) nc_dst;	/* destination vnode list */
272 	CK_SLIST_ENTRY(namecache) nc_hash;/* hash chain */
273 	struct	vnode *nc_dvp;		/* vnode of parent of name */
274 	union {
275 		struct	vnode *nu_vp;	/* vnode the name refers to */
276 		struct	negstate nu_neg;/* negative entry state */
277 	} n_un;
278 	u_char	nc_flag;		/* flag bits */
279 	u_char	nc_nlen;		/* length of name */
280 	char	nc_name[0];		/* segment name + nul */
281 };
282 
283 /*
284  * struct namecache_ts repeats struct namecache layout up to the
285  * nc_nlen member.
286  * struct namecache_ts is used in place of struct namecache when time(s) need
287  * to be stored.  The nc_dotdottime field is used when a cache entry is mapping
288  * both a non-dotdot directory name plus dotdot for the directory's
289  * parent.
290  *
291  * See below for alignment requirement.
292  */
293 struct	namecache_ts {
294 	struct	timespec nc_time;	/* timespec provided by fs */
295 	struct	timespec nc_dotdottime;	/* dotdot timespec provided by fs */
296 	int	nc_ticks;		/* ticks value when entry was added */
297 	int	nc_pad;
298 	struct namecache nc_nc;
299 };
300 
301 TAILQ_HEAD(cache_freebatch, namecache);
302 
303 /*
304  * At least mips n32 performs 64-bit accesses to timespec as found
305  * in namecache_ts and requires them to be aligned. Since others
306  * may be in the same spot suffer a little bit and enforce the
307  * alignment for everyone. Note this is a nop for 64-bit platforms.
308  */
309 #define CACHE_ZONE_ALIGNMENT	UMA_ALIGNOF(time_t)
310 
311 /*
312  * TODO: the initial value of CACHE_PATH_CUTOFF was inherited from the
313  * 4.4 BSD codebase. Later on struct namecache was tweaked to become
314  * smaller and the value was bumped to retain the total size, but it
315  * was never re-evaluated for suitability. A simple test counting
316  * lengths during package building shows that the value of 45 covers
317  * about 86% of all added entries, reaching 99% at 65.
318  *
319  * Regardless of the above, use of dedicated zones instead of malloc may be
320  * inducing additional waste. This may be hard to address as said zones are
321  * tied to VFS SMR. Even if retaining them, the current split should be
322  * re-evaluated.
323  */
324 #ifdef __LP64__
325 #define	CACHE_PATH_CUTOFF	45
326 #define	CACHE_LARGE_PAD		6
327 #else
328 #define	CACHE_PATH_CUTOFF	41
329 #define	CACHE_LARGE_PAD		2
330 #endif
331 
332 #define CACHE_ZONE_SMALL_SIZE		(offsetof(struct namecache, nc_name) + CACHE_PATH_CUTOFF + 1)
333 #define CACHE_ZONE_SMALL_TS_SIZE	(offsetof(struct namecache_ts, nc_nc) + CACHE_ZONE_SMALL_SIZE)
334 #define CACHE_ZONE_LARGE_SIZE		(offsetof(struct namecache, nc_name) + NAME_MAX + 1 + CACHE_LARGE_PAD)
335 #define CACHE_ZONE_LARGE_TS_SIZE	(offsetof(struct namecache_ts, nc_nc) + CACHE_ZONE_LARGE_SIZE)
336 
337 _Static_assert((CACHE_ZONE_SMALL_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
338 _Static_assert((CACHE_ZONE_SMALL_TS_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
339 _Static_assert((CACHE_ZONE_LARGE_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
340 _Static_assert((CACHE_ZONE_LARGE_TS_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
341 
342 #define	nc_vp		n_un.nu_vp
343 #define	nc_neg		n_un.nu_neg
344 
345 /*
346  * Flags in namecache.nc_flag
347  */
348 #define NCF_WHITE	0x01
349 #define NCF_ISDOTDOT	0x02
350 #define	NCF_TS		0x04
351 #define	NCF_DTS		0x08
352 #define	NCF_DVDROP	0x10
353 #define	NCF_NEGATIVE	0x20
354 #define	NCF_INVALID	0x40
355 #define	NCF_WIP		0x80
356 
357 /*
358  * Flags in negstate.neg_flag
359  */
360 #define NEG_HOT		0x01
361 
362 static bool	cache_neg_evict_cond(u_long lnumcache);
363 
364 /*
365  * Mark an entry as invalid.
366  *
367  * This is called before it starts getting deconstructed.
368  */
369 static void
370 cache_ncp_invalidate(struct namecache *ncp)
371 {
372 
373 	KASSERT((ncp->nc_flag & NCF_INVALID) == 0,
374 	    ("%s: entry %p already invalid", __func__, ncp));
375 	atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_INVALID);
376 	atomic_thread_fence_rel();
377 }
378 
379 /*
380  * Check whether the entry can be safely used.
381  *
382  * All places which elide locks are supposed to call this after they are
383  * done with reading from an entry.
384  */
385 #define cache_ncp_canuse(ncp)	({					\
386 	struct namecache *_ncp = (ncp);					\
387 	u_char _nc_flag;						\
388 									\
389 	atomic_thread_fence_acq();					\
390 	_nc_flag = atomic_load_char(&_ncp->nc_flag);			\
391 	__predict_true((_nc_flag & (NCF_INVALID | NCF_WIP)) == 0);	\
392 })
393 
394 /*
395  * Like the above but also checks NCF_WHITE.
396  */
397 #define cache_fpl_neg_ncp_canuse(ncp)	({				\
398 	struct namecache *_ncp = (ncp);					\
399 	u_char _nc_flag;						\
400 									\
401 	atomic_thread_fence_acq();					\
402 	_nc_flag = atomic_load_char(&_ncp->nc_flag);			\
403 	__predict_true((_nc_flag & (NCF_INVALID | NCF_WIP | NCF_WHITE)) == 0);	\
404 })
405 
406 VFS_SMR_DECLARE;
407 
408 static SYSCTL_NODE(_vfs_cache, OID_AUTO, param, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
409     "Name cache parameters");
410 
411 static u_int __read_mostly	ncsize; /* the size as computed on creation or resizing */
412 SYSCTL_UINT(_vfs_cache_param, OID_AUTO, size, CTLFLAG_RW, &ncsize, 0,
413     "Total namecache capacity");
414 
415 u_int ncsizefactor = 2;
416 SYSCTL_UINT(_vfs_cache_param, OID_AUTO, sizefactor, CTLFLAG_RW, &ncsizefactor, 0,
417     "Size factor for namecache");
418 
419 static u_long __read_mostly	ncnegfactor = 5; /* ratio of negative entries */
420 SYSCTL_ULONG(_vfs_cache_param, OID_AUTO, negfactor, CTLFLAG_RW, &ncnegfactor, 0,
421     "Ratio of negative namecache entries");
422 
423 /*
424  * Negative entry % of namecache capacity above which automatic eviction is allowed.
425  *
426  * Check cache_neg_evict_cond for details.
427  */
428 static u_int ncnegminpct = 3;
429 
430 static u_int __read_mostly     neg_min; /* the above recomputed against ncsize */
431 SYSCTL_UINT(_vfs_cache_param, OID_AUTO, negmin, CTLFLAG_RD, &neg_min, 0,
432     "Negative entry count above which automatic eviction is allowed");
433 
434 /*
435  * Structures associated with name caching.
436  */
437 #define NCHHASH(hash) \
438 	(&nchashtbl[(hash) & nchash])
439 static __read_mostly CK_SLIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */
440 static u_long __read_mostly	nchash;			/* size of hash table */
441 SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0,
442     "Size of namecache hash table");
443 static u_long __exclusive_cache_line	numneg;	/* number of negative entries allocated */
444 static u_long __exclusive_cache_line	numcache;/* number of cache entries allocated */
445 
446 struct nchstats	nchstats;		/* cache effectiveness statistics */
447 
448 static bool __read_mostly cache_rename_add = true;
449 SYSCTL_BOOL(_vfs, OID_AUTO, cache_rename_add, CTLFLAG_RW,
450     &cache_rename_add, 0, "");
451 
452 static u_int __exclusive_cache_line neg_cycle;
453 
454 #define ncneghash	3
455 #define	numneglists	(ncneghash + 1)
456 
457 struct neglist {
458 	struct mtx		nl_evict_lock;
459 	struct mtx		nl_lock __aligned(CACHE_LINE_SIZE);
460 	TAILQ_HEAD(, namecache) nl_list;
461 	TAILQ_HEAD(, namecache) nl_hotlist;
462 	u_long			nl_hotnum;
463 } __aligned(CACHE_LINE_SIZE);
464 
465 static struct neglist neglists[numneglists];
466 
467 static inline struct neglist *
468 NCP2NEGLIST(struct namecache *ncp)
469 {
470 
471 	return (&neglists[(((uintptr_t)(ncp) >> 8) & ncneghash)]);
472 }
473 
474 static inline struct negstate *
475 NCP2NEGSTATE(struct namecache *ncp)
476 {
477 
478 	MPASS(atomic_load_char(&ncp->nc_flag) & NCF_NEGATIVE);
479 	return (&ncp->nc_neg);
480 }
481 
482 #define	numbucketlocks (ncbuckethash + 1)
483 static u_int __read_mostly  ncbuckethash;
484 static struct mtx_padalign __read_mostly  *bucketlocks;
485 #define	HASH2BUCKETLOCK(hash) \
486 	((struct mtx *)(&bucketlocks[((hash) & ncbuckethash)]))
487 
488 #define	numvnodelocks (ncvnodehash + 1)
489 static u_int __read_mostly  ncvnodehash;
490 static struct mtx __read_mostly *vnodelocks;
491 static inline struct mtx *
492 VP2VNODELOCK(struct vnode *vp)
493 {
494 
495 	return (&vnodelocks[(((uintptr_t)(vp) >> 8) & ncvnodehash)]);
496 }
497 
498 static void
499 cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp)
500 {
501 	struct namecache_ts *ncp_ts;
502 
503 	KASSERT((ncp->nc_flag & NCF_TS) != 0 ||
504 	    (tsp == NULL && ticksp == NULL),
505 	    ("No NCF_TS"));
506 
507 	if (tsp == NULL)
508 		return;
509 
510 	ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
511 	*tsp = ncp_ts->nc_time;
512 	*ticksp = ncp_ts->nc_ticks;
513 }
514 
515 #ifdef DEBUG_CACHE
516 static int __read_mostly	doingcache = 1;	/* 1 => enable the cache */
517 SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0,
518     "VFS namecache enabled");
519 #endif
520 
521 /* Export size information to userland */
522 SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR,
523     sizeof(struct namecache), "sizeof(struct namecache)");
524 
525 /*
526  * The new name cache statistics
527  */
528 static SYSCTL_NODE(_vfs_cache, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
529     "Name cache statistics");
530 
531 #define STATNODE_ULONG(name, varname, descr)					\
532 	SYSCTL_ULONG(_vfs_cache_stats, OID_AUTO, name, CTLFLAG_RD, &varname, 0, descr);
533 #define STATNODE_COUNTER(name, varname, descr)					\
534 	static COUNTER_U64_DEFINE_EARLY(varname);				\
535 	SYSCTL_COUNTER_U64(_vfs_cache_stats, OID_AUTO, name, CTLFLAG_RD, &varname, \
536 	    descr);
537 STATNODE_ULONG(neg, numneg, "Number of negative cache entries");
538 STATNODE_ULONG(count, numcache, "Number of cache entries");
539 STATNODE_COUNTER(heldvnodes, numcachehv, "Number of namecache entries with vnodes held");
540 STATNODE_COUNTER(drops, numdrops, "Number of dropped entries due to reaching the limit");
541 STATNODE_COUNTER(dothits, dothits, "Number of '.' hits");
542 STATNODE_COUNTER(dotdothis, dotdothits, "Number of '..' hits");
543 STATNODE_COUNTER(miss, nummiss, "Number of cache misses");
544 STATNODE_COUNTER(misszap, nummisszap, "Number of cache misses we do not want to cache");
545 STATNODE_COUNTER(posszaps, numposzaps,
546     "Number of cache hits (positive) we do not want to cache");
547 STATNODE_COUNTER(poshits, numposhits, "Number of cache hits (positive)");
548 STATNODE_COUNTER(negzaps, numnegzaps,
549     "Number of cache hits (negative) we do not want to cache");
550 STATNODE_COUNTER(neghits, numneghits, "Number of cache hits (negative)");
551 /* These count for vn_getcwd(), too. */
552 STATNODE_COUNTER(fullpathcalls, numfullpathcalls, "Number of fullpath search calls");
553 STATNODE_COUNTER(fullpathfail1, numfullpathfail1, "Number of fullpath search errors (ENOTDIR)");
554 STATNODE_COUNTER(fullpathfail2, numfullpathfail2,
555     "Number of fullpath search errors (VOP_VPTOCNP failures)");
556 STATNODE_COUNTER(fullpathfail4, numfullpathfail4, "Number of fullpath search errors (ENOMEM)");
557 STATNODE_COUNTER(fullpathfound, numfullpathfound, "Number of successful fullpath calls");
558 STATNODE_COUNTER(symlinktoobig, symlinktoobig, "Number of times symlink did not fit the cache");
559 
560 /*
561  * Debug or developer statistics.
562  */
563 static SYSCTL_NODE(_vfs_cache, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
564     "Name cache debugging");
565 #define DEBUGNODE_ULONG(name, varname, descr)					\
566 	SYSCTL_ULONG(_vfs_cache_debug, OID_AUTO, name, CTLFLAG_RD, &varname, 0, descr);
567 #define DEBUGNODE_COUNTER(name, varname, descr)					\
568 	static COUNTER_U64_DEFINE_EARLY(varname);				\
569 	SYSCTL_COUNTER_U64(_vfs_cache_debug, OID_AUTO, name, CTLFLAG_RD, &varname, \
570 	    descr);
571 DEBUGNODE_COUNTER(zap_bucket_relock_success, zap_bucket_relock_success,
572     "Number of successful removals after relocking");
573 static long zap_bucket_fail;
574 DEBUGNODE_ULONG(zap_bucket_fail, zap_bucket_fail, "");
575 static long zap_bucket_fail2;
576 DEBUGNODE_ULONG(zap_bucket_fail2, zap_bucket_fail2, "");
577 static long cache_lock_vnodes_cel_3_failures;
578 DEBUGNODE_ULONG(vnodes_cel_3_failures, cache_lock_vnodes_cel_3_failures,
579     "Number of times 3-way vnode locking failed");
580 
581 static void cache_zap_locked(struct namecache *ncp);
582 static int vn_fullpath_hardlink(struct nameidata *ndp, char **retbuf,
583     char **freebuf, size_t *buflen);
584 static int vn_fullpath_any_smr(struct vnode *vp, struct vnode *rdir, char *buf,
585     char **retbuf, size_t *buflen, size_t addend);
586 static int vn_fullpath_any(struct vnode *vp, struct vnode *rdir, char *buf,
587     char **retbuf, size_t *buflen);
588 static int vn_fullpath_dir(struct vnode *vp, struct vnode *rdir, char *buf,
589     char **retbuf, size_t *len, size_t addend);
590 
591 static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");
592 
593 static inline void
594 cache_assert_vlp_locked(struct mtx *vlp)
595 {
596 
597 	if (vlp != NULL)
598 		mtx_assert(vlp, MA_OWNED);
599 }
600 
601 static inline void
602 cache_assert_vnode_locked(struct vnode *vp)
603 {
604 	struct mtx *vlp;
605 
606 	vlp = VP2VNODELOCK(vp);
607 	cache_assert_vlp_locked(vlp);
608 }
609 
610 /*
611  * Directory vnodes with entries are held for two reasons:
612  * 1. make them less of a target for reclamation in vnlru
613  * 2. suffer smaller performance penalty in locked lookup as requeieing is avoided
614  *
615  * It will be feasible to stop doing it altogether if all filesystems start
616  * supporting lockless lookup.
617  */
618 static void
619 cache_hold_vnode(struct vnode *vp)
620 {
621 
622 	cache_assert_vnode_locked(vp);
623 	VNPASS(LIST_EMPTY(&vp->v_cache_src), vp);
624 	vhold(vp);
625 	counter_u64_add(numcachehv, 1);
626 }
627 
628 static void
629 cache_drop_vnode(struct vnode *vp)
630 {
631 
632 	/*
633 	 * Called after all locks are dropped, meaning we can't assert
634 	 * on the state of v_cache_src.
635 	 */
636 	vdrop(vp);
637 	counter_u64_add(numcachehv, -1);
638 }
639 
640 /*
641  * UMA zones.
642  */
643 static uma_zone_t __read_mostly cache_zone_small;
644 static uma_zone_t __read_mostly cache_zone_small_ts;
645 static uma_zone_t __read_mostly cache_zone_large;
646 static uma_zone_t __read_mostly cache_zone_large_ts;
647 
648 char *
649 cache_symlink_alloc(size_t size, int flags)
650 {
651 
652 	if (size < CACHE_ZONE_SMALL_SIZE) {
653 		return (uma_zalloc_smr(cache_zone_small, flags));
654 	}
655 	if (size < CACHE_ZONE_LARGE_SIZE) {
656 		return (uma_zalloc_smr(cache_zone_large, flags));
657 	}
658 	counter_u64_add(symlinktoobig, 1);
659 	SDT_PROBE1(vfs, namecache, symlink, alloc__fail, size);
660 	return (NULL);
661 }
662 
663 void
664 cache_symlink_free(char *string, size_t size)
665 {
666 
667 	MPASS(string != NULL);
668 	KASSERT(size < CACHE_ZONE_LARGE_SIZE,
669 	    ("%s: size %zu too big", __func__, size));
670 
671 	if (size < CACHE_ZONE_SMALL_SIZE) {
672 		uma_zfree_smr(cache_zone_small, string);
673 		return;
674 	}
675 	if (size < CACHE_ZONE_LARGE_SIZE) {
676 		uma_zfree_smr(cache_zone_large, string);
677 		return;
678 	}
679 	__assert_unreachable();
680 }
681 
682 static struct namecache *
683 cache_alloc_uma(int len, bool ts)
684 {
685 	struct namecache_ts *ncp_ts;
686 	struct namecache *ncp;
687 
688 	if (__predict_false(ts)) {
689 		if (len <= CACHE_PATH_CUTOFF)
690 			ncp_ts = uma_zalloc_smr(cache_zone_small_ts, M_WAITOK);
691 		else
692 			ncp_ts = uma_zalloc_smr(cache_zone_large_ts, M_WAITOK);
693 		ncp = &ncp_ts->nc_nc;
694 	} else {
695 		if (len <= CACHE_PATH_CUTOFF)
696 			ncp = uma_zalloc_smr(cache_zone_small, M_WAITOK);
697 		else
698 			ncp = uma_zalloc_smr(cache_zone_large, M_WAITOK);
699 	}
700 	return (ncp);
701 }
702 
703 static void
704 cache_free_uma(struct namecache *ncp)
705 {
706 	struct namecache_ts *ncp_ts;
707 
708 	if (__predict_false(ncp->nc_flag & NCF_TS)) {
709 		ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
710 		if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
711 			uma_zfree_smr(cache_zone_small_ts, ncp_ts);
712 		else
713 			uma_zfree_smr(cache_zone_large_ts, ncp_ts);
714 	} else {
715 		if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
716 			uma_zfree_smr(cache_zone_small, ncp);
717 		else
718 			uma_zfree_smr(cache_zone_large, ncp);
719 	}
720 }
721 
722 static struct namecache *
723 cache_alloc(int len, bool ts)
724 {
725 	u_long lnumcache;
726 
727 	/*
728 	 * Avoid blowout in namecache entries.
729 	 *
730 	 * Bugs:
731 	 * 1. filesystems may end up trying to add an already existing entry
732 	 * (for example this can happen after a cache miss during concurrent
733 	 * lookup), in which case we will call cache_neg_evict despite not
734 	 * adding anything.
735 	 * 2. the routine may fail to free anything and no provisions are made
736 	 * to make it try harder (see the inside for failure modes)
737 	 * 3. it only ever looks at negative entries.
738 	 */
739 	lnumcache = atomic_fetchadd_long(&numcache, 1) + 1;
740 	if (cache_neg_evict_cond(lnumcache)) {
741 		lnumcache = atomic_load_long(&numcache);
742 	}
743 	if (__predict_false(lnumcache >= ncsize)) {
744 		atomic_subtract_long(&numcache, 1);
745 		counter_u64_add(numdrops, 1);
746 		return (NULL);
747 	}
748 	return (cache_alloc_uma(len, ts));
749 }
750 
751 static void
752 cache_free(struct namecache *ncp)
753 {
754 
755 	MPASS(ncp != NULL);
756 	if ((ncp->nc_flag & NCF_DVDROP) != 0) {
757 		cache_drop_vnode(ncp->nc_dvp);
758 	}
759 	cache_free_uma(ncp);
760 	atomic_subtract_long(&numcache, 1);
761 }
762 
763 static void
764 cache_free_batch(struct cache_freebatch *batch)
765 {
766 	struct namecache *ncp, *nnp;
767 	int i;
768 
769 	i = 0;
770 	if (TAILQ_EMPTY(batch))
771 		goto out;
772 	TAILQ_FOREACH_SAFE(ncp, batch, nc_dst, nnp) {
773 		if ((ncp->nc_flag & NCF_DVDROP) != 0) {
774 			cache_drop_vnode(ncp->nc_dvp);
775 		}
776 		cache_free_uma(ncp);
777 		i++;
778 	}
779 	atomic_subtract_long(&numcache, i);
780 out:
781 	SDT_PROBE1(vfs, namecache, purge, batch, i);
782 }
783 
784 /*
785  * Hashing.
786  *
787  * The code was made to use FNV in 2001 and this choice needs to be revisited.
788  *
789  * Short summary of the difficulty:
790  * The longest name which can be inserted is NAME_MAX characters in length (or
791  * 255 at the time of writing this comment), while majority of names used in
792  * practice are significantly shorter (mostly below 10). More importantly
793  * majority of lookups performed find names are even shorter than that.
794  *
795  * This poses a problem where hashes which do better than FNV past word size
796  * (or so) tend to come with additional overhead when finalizing the result,
797  * making them noticeably slower for the most commonly used range.
798  *
799  * Consider a path like: /usr/obj/usr/src/sys/amd64/GENERIC/vnode_if.c
800  *
801  * When looking it up the most time consuming part by a large margin (at least
802  * on amd64) is hashing.  Replacing FNV with something which pessimizes short
803  * input would make the slowest part stand out even more.
804  */
805 
806 /*
807  * TODO: With the value stored we can do better than computing the hash based
808  * on the address.
809  */
810 static void
811 cache_prehash(struct vnode *vp)
812 {
813 
814 	vp->v_nchash = fnv_32_buf(&vp, sizeof(vp), FNV1_32_INIT);
815 }
816 
817 static uint32_t
818 cache_get_hash(char *name, u_char len, struct vnode *dvp)
819 {
820 
821 	return (fnv_32_buf(name, len, dvp->v_nchash));
822 }
823 
824 static uint32_t
825 cache_get_hash_iter_start(struct vnode *dvp)
826 {
827 
828 	return (dvp->v_nchash);
829 }
830 
831 static uint32_t
832 cache_get_hash_iter(char c, uint32_t hash)
833 {
834 
835 	return (fnv_32_buf(&c, 1, hash));
836 }
837 
838 static uint32_t
839 cache_get_hash_iter_finish(uint32_t hash)
840 {
841 
842 	return (hash);
843 }
844 
845 static inline struct nchashhead *
846 NCP2BUCKET(struct namecache *ncp)
847 {
848 	uint32_t hash;
849 
850 	hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
851 	return (NCHHASH(hash));
852 }
853 
854 static inline struct mtx *
855 NCP2BUCKETLOCK(struct namecache *ncp)
856 {
857 	uint32_t hash;
858 
859 	hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
860 	return (HASH2BUCKETLOCK(hash));
861 }
862 
863 #ifdef INVARIANTS
864 static void
865 cache_assert_bucket_locked(struct namecache *ncp)
866 {
867 	struct mtx *blp;
868 
869 	blp = NCP2BUCKETLOCK(ncp);
870 	mtx_assert(blp, MA_OWNED);
871 }
872 
873 static void
874 cache_assert_bucket_unlocked(struct namecache *ncp)
875 {
876 	struct mtx *blp;
877 
878 	blp = NCP2BUCKETLOCK(ncp);
879 	mtx_assert(blp, MA_NOTOWNED);
880 }
881 #else
882 #define cache_assert_bucket_locked(x) do { } while (0)
883 #define cache_assert_bucket_unlocked(x) do { } while (0)
884 #endif
885 
886 #define cache_sort_vnodes(x, y)	_cache_sort_vnodes((void **)(x), (void **)(y))
887 static void
888 _cache_sort_vnodes(void **p1, void **p2)
889 {
890 	void *tmp;
891 
892 	MPASS(*p1 != NULL || *p2 != NULL);
893 
894 	if (*p1 > *p2) {
895 		tmp = *p2;
896 		*p2 = *p1;
897 		*p1 = tmp;
898 	}
899 }
900 
901 static void
902 cache_lock_all_buckets(void)
903 {
904 	u_int i;
905 
906 	for (i = 0; i < numbucketlocks; i++)
907 		mtx_lock(&bucketlocks[i]);
908 }
909 
910 static void
911 cache_unlock_all_buckets(void)
912 {
913 	u_int i;
914 
915 	for (i = 0; i < numbucketlocks; i++)
916 		mtx_unlock(&bucketlocks[i]);
917 }
918 
919 static void
920 cache_lock_all_vnodes(void)
921 {
922 	u_int i;
923 
924 	for (i = 0; i < numvnodelocks; i++)
925 		mtx_lock(&vnodelocks[i]);
926 }
927 
928 static void
929 cache_unlock_all_vnodes(void)
930 {
931 	u_int i;
932 
933 	for (i = 0; i < numvnodelocks; i++)
934 		mtx_unlock(&vnodelocks[i]);
935 }
936 
937 static int
938 cache_trylock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
939 {
940 
941 	cache_sort_vnodes(&vlp1, &vlp2);
942 
943 	if (vlp1 != NULL) {
944 		if (!mtx_trylock(vlp1))
945 			return (EAGAIN);
946 	}
947 	if (!mtx_trylock(vlp2)) {
948 		if (vlp1 != NULL)
949 			mtx_unlock(vlp1);
950 		return (EAGAIN);
951 	}
952 
953 	return (0);
954 }
955 
956 static void
957 cache_lock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
958 {
959 
960 	MPASS(vlp1 != NULL || vlp2 != NULL);
961 	MPASS(vlp1 <= vlp2);
962 
963 	if (vlp1 != NULL)
964 		mtx_lock(vlp1);
965 	if (vlp2 != NULL)
966 		mtx_lock(vlp2);
967 }
968 
969 static void
970 cache_unlock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
971 {
972 
973 	MPASS(vlp1 != NULL || vlp2 != NULL);
974 
975 	if (vlp1 != NULL)
976 		mtx_unlock(vlp1);
977 	if (vlp2 != NULL)
978 		mtx_unlock(vlp2);
979 }
980 
981 static int
982 sysctl_nchstats(SYSCTL_HANDLER_ARGS)
983 {
984 	struct nchstats snap;
985 
986 	if (req->oldptr == NULL)
987 		return (SYSCTL_OUT(req, 0, sizeof(snap)));
988 
989 	snap = nchstats;
990 	snap.ncs_goodhits = counter_u64_fetch(numposhits);
991 	snap.ncs_neghits = counter_u64_fetch(numneghits);
992 	snap.ncs_badhits = counter_u64_fetch(numposzaps) +
993 	    counter_u64_fetch(numnegzaps);
994 	snap.ncs_miss = counter_u64_fetch(nummisszap) +
995 	    counter_u64_fetch(nummiss);
996 
997 	return (SYSCTL_OUT(req, &snap, sizeof(snap)));
998 }
999 SYSCTL_PROC(_vfs_cache, OID_AUTO, nchstats, CTLTYPE_OPAQUE | CTLFLAG_RD |
1000     CTLFLAG_MPSAFE, 0, 0, sysctl_nchstats, "LU",
1001     "VFS cache effectiveness statistics");
1002 
1003 static void
1004 cache_recalc_neg_min(u_int val)
1005 {
1006 
1007 	neg_min = (ncsize * val) / 100;
1008 }
1009 
1010 static int
1011 sysctl_negminpct(SYSCTL_HANDLER_ARGS)
1012 {
1013 	u_int val;
1014 	int error;
1015 
1016 	val = ncnegminpct;
1017 	error = sysctl_handle_int(oidp, &val, 0, req);
1018 	if (error != 0 || req->newptr == NULL)
1019 		return (error);
1020 
1021 	if (val == ncnegminpct)
1022 		return (0);
1023 	if (val < 0 || val > 99)
1024 		return (EINVAL);
1025 	ncnegminpct = val;
1026 	cache_recalc_neg_min(val);
1027 	return (0);
1028 }
1029 
1030 SYSCTL_PROC(_vfs_cache_param, OID_AUTO, negminpct,
1031     CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_negminpct,
1032     "I", "Negative entry \% of namecache capacity above which automatic eviction is allowed");
1033 
1034 #ifdef DIAGNOSTIC
1035 /*
1036  * Grab an atomic snapshot of the name cache hash chain lengths
1037  */
1038 static SYSCTL_NODE(_debug, OID_AUTO, hashstat,
1039     CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
1040     "hash table stats");
1041 
1042 static int
1043 sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS)
1044 {
1045 	struct nchashhead *ncpp;
1046 	struct namecache *ncp;
1047 	int i, error, n_nchash, *cntbuf;
1048 
1049 retry:
1050 	n_nchash = nchash + 1;	/* nchash is max index, not count */
1051 	if (req->oldptr == NULL)
1052 		return SYSCTL_OUT(req, 0, n_nchash * sizeof(int));
1053 	cntbuf = malloc(n_nchash * sizeof(int), M_TEMP, M_ZERO | M_WAITOK);
1054 	cache_lock_all_buckets();
1055 	if (n_nchash != nchash + 1) {
1056 		cache_unlock_all_buckets();
1057 		free(cntbuf, M_TEMP);
1058 		goto retry;
1059 	}
1060 	/* Scan hash tables counting entries */
1061 	for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++)
1062 		CK_SLIST_FOREACH(ncp, ncpp, nc_hash)
1063 			cntbuf[i]++;
1064 	cache_unlock_all_buckets();
1065 	for (error = 0, i = 0; i < n_nchash; i++)
1066 		if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0)
1067 			break;
1068 	free(cntbuf, M_TEMP);
1069 	return (error);
1070 }
1071 SYSCTL_PROC(_debug_hashstat, OID_AUTO, rawnchash, CTLTYPE_INT|CTLFLAG_RD|
1072     CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_rawnchash, "S,int",
1073     "nchash chain lengths");
1074 
1075 static int
1076 sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS)
1077 {
1078 	int error;
1079 	struct nchashhead *ncpp;
1080 	struct namecache *ncp;
1081 	int n_nchash;
1082 	int count, maxlength, used, pct;
1083 
1084 	if (!req->oldptr)
1085 		return SYSCTL_OUT(req, 0, 4 * sizeof(int));
1086 
1087 	cache_lock_all_buckets();
1088 	n_nchash = nchash + 1;	/* nchash is max index, not count */
1089 	used = 0;
1090 	maxlength = 0;
1091 
1092 	/* Scan hash tables for applicable entries */
1093 	for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) {
1094 		count = 0;
1095 		CK_SLIST_FOREACH(ncp, ncpp, nc_hash) {
1096 			count++;
1097 		}
1098 		if (count)
1099 			used++;
1100 		if (maxlength < count)
1101 			maxlength = count;
1102 	}
1103 	n_nchash = nchash + 1;
1104 	cache_unlock_all_buckets();
1105 	pct = (used * 100) / (n_nchash / 100);
1106 	error = SYSCTL_OUT(req, &n_nchash, sizeof(n_nchash));
1107 	if (error)
1108 		return (error);
1109 	error = SYSCTL_OUT(req, &used, sizeof(used));
1110 	if (error)
1111 		return (error);
1112 	error = SYSCTL_OUT(req, &maxlength, sizeof(maxlength));
1113 	if (error)
1114 		return (error);
1115 	error = SYSCTL_OUT(req, &pct, sizeof(pct));
1116 	if (error)
1117 		return (error);
1118 	return (0);
1119 }
1120 SYSCTL_PROC(_debug_hashstat, OID_AUTO, nchash, CTLTYPE_INT|CTLFLAG_RD|
1121     CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_nchash, "I",
1122     "nchash statistics (number of total/used buckets, maximum chain length, usage percentage)");
1123 #endif
1124 
1125 /*
1126  * Negative entries management
1127  *
1128  * Various workloads create plenty of negative entries and barely use them
1129  * afterwards. Moreover malicious users can keep performing bogus lookups
1130  * adding even more entries. For example "make tinderbox" as of writing this
1131  * comment ends up with 2.6M namecache entries in total, 1.2M of which are
1132  * negative.
1133  *
1134  * As such, a rather aggressive eviction method is needed. The currently
1135  * employed method is a placeholder.
1136  *
1137  * Entries are split over numneglists separate lists, each of which is further
1138  * split into hot and cold entries. Entries get promoted after getting a hit.
1139  * Eviction happens on addition of new entry.
1140  */
1141 static SYSCTL_NODE(_vfs_cache, OID_AUTO, neg, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1142     "Name cache negative entry statistics");
1143 
1144 SYSCTL_ULONG(_vfs_cache_neg, OID_AUTO, count, CTLFLAG_RD, &numneg, 0,
1145     "Number of negative cache entries");
1146 
1147 static COUNTER_U64_DEFINE_EARLY(neg_created);
1148 SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, created, CTLFLAG_RD, &neg_created,
1149     "Number of created negative entries");
1150 
1151 static COUNTER_U64_DEFINE_EARLY(neg_evicted);
1152 SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evicted, CTLFLAG_RD, &neg_evicted,
1153     "Number of evicted negative entries");
1154 
1155 static COUNTER_U64_DEFINE_EARLY(neg_evict_skipped_empty);
1156 SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evict_skipped_empty, CTLFLAG_RD,
1157     &neg_evict_skipped_empty,
1158     "Number of times evicting failed due to lack of entries");
1159 
1160 static COUNTER_U64_DEFINE_EARLY(neg_evict_skipped_missed);
1161 SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evict_skipped_missed, CTLFLAG_RD,
1162     &neg_evict_skipped_missed,
1163     "Number of times evicting failed due to target entry disappearing");
1164 
1165 static COUNTER_U64_DEFINE_EARLY(neg_evict_skipped_contended);
1166 SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evict_skipped_contended, CTLFLAG_RD,
1167     &neg_evict_skipped_contended,
1168     "Number of times evicting failed due to contention");
1169 
1170 SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, hits, CTLFLAG_RD, &numneghits,
1171     "Number of cache hits (negative)");
1172 
1173 static int
1174 sysctl_neg_hot(SYSCTL_HANDLER_ARGS)
1175 {
1176 	int i, out;
1177 
1178 	out = 0;
1179 	for (i = 0; i < numneglists; i++)
1180 		out += neglists[i].nl_hotnum;
1181 
1182 	return (SYSCTL_OUT(req, &out, sizeof(out)));
1183 }
1184 SYSCTL_PROC(_vfs_cache_neg, OID_AUTO, hot, CTLTYPE_INT | CTLFLAG_RD |
1185     CTLFLAG_MPSAFE, 0, 0, sysctl_neg_hot, "I",
1186     "Number of hot negative entries");
1187 
1188 static void
1189 cache_neg_init(struct namecache *ncp)
1190 {
1191 	struct negstate *ns;
1192 
1193 	ncp->nc_flag |= NCF_NEGATIVE;
1194 	ns = NCP2NEGSTATE(ncp);
1195 	ns->neg_flag = 0;
1196 	ns->neg_hit = 0;
1197 	counter_u64_add(neg_created, 1);
1198 }
1199 
1200 #define CACHE_NEG_PROMOTION_THRESH 2
1201 
1202 static bool
1203 cache_neg_hit_prep(struct namecache *ncp)
1204 {
1205 	struct negstate *ns;
1206 	u_char n;
1207 
1208 	ns = NCP2NEGSTATE(ncp);
1209 	n = atomic_load_char(&ns->neg_hit);
1210 	for (;;) {
1211 		if (n >= CACHE_NEG_PROMOTION_THRESH)
1212 			return (false);
1213 		if (atomic_fcmpset_8(&ns->neg_hit, &n, n + 1))
1214 			break;
1215 	}
1216 	return (n + 1 == CACHE_NEG_PROMOTION_THRESH);
1217 }
1218 
1219 /*
1220  * Nothing to do here but it is provided for completeness as some
1221  * cache_neg_hit_prep callers may end up returning without even
1222  * trying to promote.
1223  */
1224 #define cache_neg_hit_abort(ncp)	do { } while (0)
1225 
1226 static void
1227 cache_neg_hit_finish(struct namecache *ncp)
1228 {
1229 
1230 	SDT_PROBE2(vfs, namecache, lookup, hit__negative, ncp->nc_dvp, ncp->nc_name);
1231 	counter_u64_add(numneghits, 1);
1232 }
1233 
1234 /*
1235  * Move a negative entry to the hot list.
1236  */
1237 static void
1238 cache_neg_promote_locked(struct namecache *ncp)
1239 {
1240 	struct neglist *nl;
1241 	struct negstate *ns;
1242 
1243 	ns = NCP2NEGSTATE(ncp);
1244 	nl = NCP2NEGLIST(ncp);
1245 	mtx_assert(&nl->nl_lock, MA_OWNED);
1246 	if ((ns->neg_flag & NEG_HOT) == 0) {
1247 		TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst);
1248 		TAILQ_INSERT_TAIL(&nl->nl_hotlist, ncp, nc_dst);
1249 		nl->nl_hotnum++;
1250 		ns->neg_flag |= NEG_HOT;
1251 	}
1252 }
1253 
1254 /*
1255  * Move a hot negative entry to the cold list.
1256  */
1257 static void
1258 cache_neg_demote_locked(struct namecache *ncp)
1259 {
1260 	struct neglist *nl;
1261 	struct negstate *ns;
1262 
1263 	ns = NCP2NEGSTATE(ncp);
1264 	nl = NCP2NEGLIST(ncp);
1265 	mtx_assert(&nl->nl_lock, MA_OWNED);
1266 	MPASS(ns->neg_flag & NEG_HOT);
1267 	TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst);
1268 	TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst);
1269 	nl->nl_hotnum--;
1270 	ns->neg_flag &= ~NEG_HOT;
1271 	atomic_store_char(&ns->neg_hit, 0);
1272 }
1273 
1274 /*
1275  * Move a negative entry to the hot list if it matches the lookup.
1276  *
1277  * We have to take locks, but they may be contended and in the worst
1278  * case we may need to go off CPU. We don't want to spin within the
1279  * smr section and we can't block with it. Exiting the section means
1280  * the found entry could have been evicted. We are going to look it
1281  * up again.
1282  */
1283 static bool
1284 cache_neg_promote_cond(struct vnode *dvp, struct componentname *cnp,
1285     struct namecache *oncp, uint32_t hash)
1286 {
1287 	struct namecache *ncp;
1288 	struct neglist *nl;
1289 	u_char nc_flag;
1290 
1291 	nl = NCP2NEGLIST(oncp);
1292 
1293 	mtx_lock(&nl->nl_lock);
1294 	/*
1295 	 * For hash iteration.
1296 	 */
1297 	vfs_smr_enter();
1298 
1299 	/*
1300 	 * Avoid all surprises by only succeeding if we got the same entry and
1301 	 * bailing completely otherwise.
1302 	 * XXX There are no provisions to keep the vnode around, meaning we may
1303 	 * end up promoting a negative entry for a *new* vnode and returning
1304 	 * ENOENT on its account. This is the error we want to return anyway
1305 	 * and promotion is harmless.
1306 	 *
1307 	 * In particular at this point there can be a new ncp which matches the
1308 	 * search but hashes to a different neglist.
1309 	 */
1310 	CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
1311 		if (ncp == oncp)
1312 			break;
1313 	}
1314 
1315 	/*
1316 	 * No match to begin with.
1317 	 */
1318 	if (__predict_false(ncp == NULL)) {
1319 		goto out_abort;
1320 	}
1321 
1322 	/*
1323 	 * The newly found entry may be something different...
1324 	 */
1325 	if (!(ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
1326 	    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))) {
1327 		goto out_abort;
1328 	}
1329 
1330 	/*
1331 	 * ... and not even negative.
1332 	 */
1333 	nc_flag = atomic_load_char(&ncp->nc_flag);
1334 	if ((nc_flag & NCF_NEGATIVE) == 0) {
1335 		goto out_abort;
1336 	}
1337 
1338 	if (!cache_ncp_canuse(ncp)) {
1339 		goto out_abort;
1340 	}
1341 
1342 	cache_neg_promote_locked(ncp);
1343 	cache_neg_hit_finish(ncp);
1344 	vfs_smr_exit();
1345 	mtx_unlock(&nl->nl_lock);
1346 	return (true);
1347 out_abort:
1348 	vfs_smr_exit();
1349 	mtx_unlock(&nl->nl_lock);
1350 	return (false);
1351 }
1352 
1353 static void
1354 cache_neg_promote(struct namecache *ncp)
1355 {
1356 	struct neglist *nl;
1357 
1358 	nl = NCP2NEGLIST(ncp);
1359 	mtx_lock(&nl->nl_lock);
1360 	cache_neg_promote_locked(ncp);
1361 	mtx_unlock(&nl->nl_lock);
1362 }
1363 
1364 static void
1365 cache_neg_insert(struct namecache *ncp)
1366 {
1367 	struct neglist *nl;
1368 
1369 	MPASS(ncp->nc_flag & NCF_NEGATIVE);
1370 	cache_assert_bucket_locked(ncp);
1371 	nl = NCP2NEGLIST(ncp);
1372 	mtx_lock(&nl->nl_lock);
1373 	TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst);
1374 	mtx_unlock(&nl->nl_lock);
1375 	atomic_add_long(&numneg, 1);
1376 }
1377 
1378 static void
1379 cache_neg_remove(struct namecache *ncp)
1380 {
1381 	struct neglist *nl;
1382 	struct negstate *ns;
1383 
1384 	cache_assert_bucket_locked(ncp);
1385 	nl = NCP2NEGLIST(ncp);
1386 	ns = NCP2NEGSTATE(ncp);
1387 	mtx_lock(&nl->nl_lock);
1388 	if ((ns->neg_flag & NEG_HOT) != 0) {
1389 		TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst);
1390 		nl->nl_hotnum--;
1391 	} else {
1392 		TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst);
1393 	}
1394 	mtx_unlock(&nl->nl_lock);
1395 	atomic_subtract_long(&numneg, 1);
1396 }
1397 
1398 static struct neglist *
1399 cache_neg_evict_select_list(void)
1400 {
1401 	struct neglist *nl;
1402 	u_int c;
1403 
1404 	c = atomic_fetchadd_int(&neg_cycle, 1) + 1;
1405 	nl = &neglists[c % numneglists];
1406 	if (!mtx_trylock(&nl->nl_evict_lock)) {
1407 		counter_u64_add(neg_evict_skipped_contended, 1);
1408 		return (NULL);
1409 	}
1410 	return (nl);
1411 }
1412 
1413 static struct namecache *
1414 cache_neg_evict_select_entry(struct neglist *nl)
1415 {
1416 	struct namecache *ncp, *lncp;
1417 	struct negstate *ns, *lns;
1418 	int i;
1419 
1420 	mtx_assert(&nl->nl_evict_lock, MA_OWNED);
1421 	mtx_assert(&nl->nl_lock, MA_OWNED);
1422 	ncp = TAILQ_FIRST(&nl->nl_list);
1423 	if (ncp == NULL)
1424 		return (NULL);
1425 	lncp = ncp;
1426 	lns = NCP2NEGSTATE(lncp);
1427 	for (i = 1; i < 4; i++) {
1428 		ncp = TAILQ_NEXT(ncp, nc_dst);
1429 		if (ncp == NULL)
1430 			break;
1431 		ns = NCP2NEGSTATE(ncp);
1432 		if (ns->neg_hit < lns->neg_hit) {
1433 			lncp = ncp;
1434 			lns = ns;
1435 		}
1436 	}
1437 	return (lncp);
1438 }
1439 
1440 static bool
1441 cache_neg_evict(void)
1442 {
1443 	struct namecache *ncp, *ncp2;
1444 	struct neglist *nl;
1445 	struct vnode *dvp;
1446 	struct mtx *dvlp;
1447 	struct mtx *blp;
1448 	uint32_t hash;
1449 	u_char nlen;
1450 	bool evicted;
1451 
1452 	nl = cache_neg_evict_select_list();
1453 	if (nl == NULL) {
1454 		return (false);
1455 	}
1456 
1457 	mtx_lock(&nl->nl_lock);
1458 	ncp = TAILQ_FIRST(&nl->nl_hotlist);
1459 	if (ncp != NULL) {
1460 		cache_neg_demote_locked(ncp);
1461 	}
1462 	ncp = cache_neg_evict_select_entry(nl);
1463 	if (ncp == NULL) {
1464 		counter_u64_add(neg_evict_skipped_empty, 1);
1465 		mtx_unlock(&nl->nl_lock);
1466 		mtx_unlock(&nl->nl_evict_lock);
1467 		return (false);
1468 	}
1469 	nlen = ncp->nc_nlen;
1470 	dvp = ncp->nc_dvp;
1471 	hash = cache_get_hash(ncp->nc_name, nlen, dvp);
1472 	dvlp = VP2VNODELOCK(dvp);
1473 	blp = HASH2BUCKETLOCK(hash);
1474 	mtx_unlock(&nl->nl_lock);
1475 	mtx_unlock(&nl->nl_evict_lock);
1476 	mtx_lock(dvlp);
1477 	mtx_lock(blp);
1478 	/*
1479 	 * Note that since all locks were dropped above, the entry may be
1480 	 * gone or reallocated to be something else.
1481 	 */
1482 	CK_SLIST_FOREACH(ncp2, (NCHHASH(hash)), nc_hash) {
1483 		if (ncp2 == ncp && ncp2->nc_dvp == dvp &&
1484 		    ncp2->nc_nlen == nlen && (ncp2->nc_flag & NCF_NEGATIVE) != 0)
1485 			break;
1486 	}
1487 	if (ncp2 == NULL) {
1488 		counter_u64_add(neg_evict_skipped_missed, 1);
1489 		ncp = NULL;
1490 		evicted = false;
1491 	} else {
1492 		MPASS(dvlp == VP2VNODELOCK(ncp->nc_dvp));
1493 		MPASS(blp == NCP2BUCKETLOCK(ncp));
1494 		SDT_PROBE2(vfs, namecache, evict_negative, done, ncp->nc_dvp,
1495 		    ncp->nc_name);
1496 		cache_zap_locked(ncp);
1497 		counter_u64_add(neg_evicted, 1);
1498 		evicted = true;
1499 	}
1500 	mtx_unlock(blp);
1501 	mtx_unlock(dvlp);
1502 	if (ncp != NULL)
1503 		cache_free(ncp);
1504 	return (evicted);
1505 }
1506 
1507 /*
1508  * Maybe evict a negative entry to create more room.
1509  *
1510  * The ncnegfactor parameter limits what fraction of the total count
1511  * can comprise of negative entries. However, if the cache is just
1512  * warming up this leads to excessive evictions.  As such, ncnegminpct
1513  * (recomputed to neg_min) dictates whether the above should be
1514  * applied.
1515  *
1516  * Try evicting if the cache is close to full capacity regardless of
1517  * other considerations.
1518  */
1519 static bool
1520 cache_neg_evict_cond(u_long lnumcache)
1521 {
1522 	u_long lnumneg;
1523 
1524 	if (ncsize - 1000 < lnumcache)
1525 		goto out_evict;
1526 	lnumneg = atomic_load_long(&numneg);
1527 	if (lnumneg < neg_min)
1528 		return (false);
1529 	if (lnumneg * ncnegfactor < lnumcache)
1530 		return (false);
1531 out_evict:
1532 	return (cache_neg_evict());
1533 }
1534 
1535 /*
1536  * cache_zap_locked():
1537  *
1538  *   Removes a namecache entry from cache, whether it contains an actual
1539  *   pointer to a vnode or if it is just a negative cache entry.
1540  */
1541 static void
1542 cache_zap_locked(struct namecache *ncp)
1543 {
1544 	struct nchashhead *ncpp;
1545 	struct vnode *dvp, *vp;
1546 
1547 	dvp = ncp->nc_dvp;
1548 	vp = ncp->nc_vp;
1549 
1550 	if (!(ncp->nc_flag & NCF_NEGATIVE))
1551 		cache_assert_vnode_locked(vp);
1552 	cache_assert_vnode_locked(dvp);
1553 	cache_assert_bucket_locked(ncp);
1554 
1555 	cache_ncp_invalidate(ncp);
1556 
1557 	ncpp = NCP2BUCKET(ncp);
1558 	CK_SLIST_REMOVE(ncpp, ncp, namecache, nc_hash);
1559 	if (!(ncp->nc_flag & NCF_NEGATIVE)) {
1560 		SDT_PROBE3(vfs, namecache, zap, done, dvp, ncp->nc_name, vp);
1561 		TAILQ_REMOVE(&vp->v_cache_dst, ncp, nc_dst);
1562 		if (ncp == vp->v_cache_dd) {
1563 			atomic_store_ptr(&vp->v_cache_dd, NULL);
1564 		}
1565 	} else {
1566 		SDT_PROBE2(vfs, namecache, zap_negative, done, dvp, ncp->nc_name);
1567 		cache_neg_remove(ncp);
1568 	}
1569 	if (ncp->nc_flag & NCF_ISDOTDOT) {
1570 		if (ncp == dvp->v_cache_dd) {
1571 			atomic_store_ptr(&dvp->v_cache_dd, NULL);
1572 		}
1573 	} else {
1574 		LIST_REMOVE(ncp, nc_src);
1575 		if (LIST_EMPTY(&dvp->v_cache_src)) {
1576 			ncp->nc_flag |= NCF_DVDROP;
1577 		}
1578 	}
1579 }
1580 
1581 static void
1582 cache_zap_negative_locked_vnode_kl(struct namecache *ncp, struct vnode *vp)
1583 {
1584 	struct mtx *blp;
1585 
1586 	MPASS(ncp->nc_dvp == vp);
1587 	MPASS(ncp->nc_flag & NCF_NEGATIVE);
1588 	cache_assert_vnode_locked(vp);
1589 
1590 	blp = NCP2BUCKETLOCK(ncp);
1591 	mtx_lock(blp);
1592 	cache_zap_locked(ncp);
1593 	mtx_unlock(blp);
1594 }
1595 
1596 static bool
1597 cache_zap_locked_vnode_kl2(struct namecache *ncp, struct vnode *vp,
1598     struct mtx **vlpp)
1599 {
1600 	struct mtx *pvlp, *vlp1, *vlp2, *to_unlock;
1601 	struct mtx *blp;
1602 
1603 	MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
1604 	cache_assert_vnode_locked(vp);
1605 
1606 	if (ncp->nc_flag & NCF_NEGATIVE) {
1607 		if (*vlpp != NULL) {
1608 			mtx_unlock(*vlpp);
1609 			*vlpp = NULL;
1610 		}
1611 		cache_zap_negative_locked_vnode_kl(ncp, vp);
1612 		return (true);
1613 	}
1614 
1615 	pvlp = VP2VNODELOCK(vp);
1616 	blp = NCP2BUCKETLOCK(ncp);
1617 	vlp1 = VP2VNODELOCK(ncp->nc_dvp);
1618 	vlp2 = VP2VNODELOCK(ncp->nc_vp);
1619 
1620 	if (*vlpp == vlp1 || *vlpp == vlp2) {
1621 		to_unlock = *vlpp;
1622 		*vlpp = NULL;
1623 	} else {
1624 		if (*vlpp != NULL) {
1625 			mtx_unlock(*vlpp);
1626 			*vlpp = NULL;
1627 		}
1628 		cache_sort_vnodes(&vlp1, &vlp2);
1629 		if (vlp1 == pvlp) {
1630 			mtx_lock(vlp2);
1631 			to_unlock = vlp2;
1632 		} else {
1633 			if (!mtx_trylock(vlp1))
1634 				goto out_relock;
1635 			to_unlock = vlp1;
1636 		}
1637 	}
1638 	mtx_lock(blp);
1639 	cache_zap_locked(ncp);
1640 	mtx_unlock(blp);
1641 	if (to_unlock != NULL)
1642 		mtx_unlock(to_unlock);
1643 	return (true);
1644 
1645 out_relock:
1646 	mtx_unlock(vlp2);
1647 	mtx_lock(vlp1);
1648 	mtx_lock(vlp2);
1649 	MPASS(*vlpp == NULL);
1650 	*vlpp = vlp1;
1651 	return (false);
1652 }
1653 
1654 /*
1655  * If trylocking failed we can get here. We know enough to take all needed locks
1656  * in the right order and re-lookup the entry.
1657  */
1658 static int
1659 cache_zap_unlocked_bucket(struct namecache *ncp, struct componentname *cnp,
1660     struct vnode *dvp, struct mtx *dvlp, struct mtx *vlp, uint32_t hash,
1661     struct mtx *blp)
1662 {
1663 	struct namecache *rncp;
1664 
1665 	cache_assert_bucket_unlocked(ncp);
1666 
1667 	cache_sort_vnodes(&dvlp, &vlp);
1668 	cache_lock_vnodes(dvlp, vlp);
1669 	mtx_lock(blp);
1670 	CK_SLIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) {
1671 		if (rncp == ncp && rncp->nc_dvp == dvp &&
1672 		    rncp->nc_nlen == cnp->cn_namelen &&
1673 		    !bcmp(rncp->nc_name, cnp->cn_nameptr, rncp->nc_nlen))
1674 			break;
1675 	}
1676 	if (rncp != NULL) {
1677 		cache_zap_locked(rncp);
1678 		mtx_unlock(blp);
1679 		cache_unlock_vnodes(dvlp, vlp);
1680 		counter_u64_add(zap_bucket_relock_success, 1);
1681 		return (0);
1682 	}
1683 
1684 	mtx_unlock(blp);
1685 	cache_unlock_vnodes(dvlp, vlp);
1686 	return (EAGAIN);
1687 }
1688 
1689 static int __noinline
1690 cache_zap_locked_bucket(struct namecache *ncp, struct componentname *cnp,
1691     uint32_t hash, struct mtx *blp)
1692 {
1693 	struct mtx *dvlp, *vlp;
1694 	struct vnode *dvp;
1695 
1696 	cache_assert_bucket_locked(ncp);
1697 
1698 	dvlp = VP2VNODELOCK(ncp->nc_dvp);
1699 	vlp = NULL;
1700 	if (!(ncp->nc_flag & NCF_NEGATIVE))
1701 		vlp = VP2VNODELOCK(ncp->nc_vp);
1702 	if (cache_trylock_vnodes(dvlp, vlp) == 0) {
1703 		cache_zap_locked(ncp);
1704 		mtx_unlock(blp);
1705 		cache_unlock_vnodes(dvlp, vlp);
1706 		return (0);
1707 	}
1708 
1709 	dvp = ncp->nc_dvp;
1710 	mtx_unlock(blp);
1711 	return (cache_zap_unlocked_bucket(ncp, cnp, dvp, dvlp, vlp, hash, blp));
1712 }
1713 
1714 static __noinline int
1715 cache_remove_cnp(struct vnode *dvp, struct componentname *cnp)
1716 {
1717 	struct namecache *ncp;
1718 	struct mtx *blp;
1719 	struct mtx *dvlp, *dvlp2;
1720 	uint32_t hash;
1721 	int error;
1722 
1723 	if (cnp->cn_namelen == 2 &&
1724 	    cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
1725 		dvlp = VP2VNODELOCK(dvp);
1726 		dvlp2 = NULL;
1727 		mtx_lock(dvlp);
1728 retry_dotdot:
1729 		ncp = dvp->v_cache_dd;
1730 		if (ncp == NULL) {
1731 			mtx_unlock(dvlp);
1732 			if (dvlp2 != NULL)
1733 				mtx_unlock(dvlp2);
1734 			SDT_PROBE2(vfs, namecache, removecnp, miss, dvp, cnp);
1735 			return (0);
1736 		}
1737 		if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
1738 			if (!cache_zap_locked_vnode_kl2(ncp, dvp, &dvlp2))
1739 				goto retry_dotdot;
1740 			MPASS(dvp->v_cache_dd == NULL);
1741 			mtx_unlock(dvlp);
1742 			if (dvlp2 != NULL)
1743 				mtx_unlock(dvlp2);
1744 			cache_free(ncp);
1745 		} else {
1746 			atomic_store_ptr(&dvp->v_cache_dd, NULL);
1747 			mtx_unlock(dvlp);
1748 			if (dvlp2 != NULL)
1749 				mtx_unlock(dvlp2);
1750 		}
1751 		SDT_PROBE2(vfs, namecache, removecnp, hit, dvp, cnp);
1752 		return (1);
1753 	}
1754 
1755 	hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
1756 	blp = HASH2BUCKETLOCK(hash);
1757 retry:
1758 	if (CK_SLIST_EMPTY(NCHHASH(hash)))
1759 		goto out_no_entry;
1760 
1761 	mtx_lock(blp);
1762 
1763 	CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
1764 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
1765 		    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
1766 			break;
1767 	}
1768 
1769 	if (ncp == NULL) {
1770 		mtx_unlock(blp);
1771 		goto out_no_entry;
1772 	}
1773 
1774 	error = cache_zap_locked_bucket(ncp, cnp, hash, blp);
1775 	if (__predict_false(error != 0)) {
1776 		zap_bucket_fail++;
1777 		goto retry;
1778 	}
1779 	counter_u64_add(numposzaps, 1);
1780 	SDT_PROBE2(vfs, namecache, removecnp, hit, dvp, cnp);
1781 	cache_free(ncp);
1782 	return (1);
1783 out_no_entry:
1784 	counter_u64_add(nummisszap, 1);
1785 	SDT_PROBE2(vfs, namecache, removecnp, miss, dvp, cnp);
1786 	return (0);
1787 }
1788 
1789 static int __noinline
1790 cache_lookup_dot(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
1791     struct timespec *tsp, int *ticksp)
1792 {
1793 	int ltype;
1794 
1795 	*vpp = dvp;
1796 	counter_u64_add(dothits, 1);
1797 	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp);
1798 	if (tsp != NULL)
1799 		timespecclear(tsp);
1800 	if (ticksp != NULL)
1801 		*ticksp = ticks;
1802 	vrefact(*vpp);
1803 	/*
1804 	 * When we lookup "." we still can be asked to lock it
1805 	 * differently...
1806 	 */
1807 	ltype = cnp->cn_lkflags & LK_TYPE_MASK;
1808 	if (ltype != VOP_ISLOCKED(*vpp)) {
1809 		if (ltype == LK_EXCLUSIVE) {
1810 			vn_lock(*vpp, LK_UPGRADE | LK_RETRY);
1811 			if (VN_IS_DOOMED((*vpp))) {
1812 				/* forced unmount */
1813 				vrele(*vpp);
1814 				*vpp = NULL;
1815 				return (ENOENT);
1816 			}
1817 		} else
1818 			vn_lock(*vpp, LK_DOWNGRADE | LK_RETRY);
1819 	}
1820 	return (-1);
1821 }
1822 
1823 static int __noinline
1824 cache_lookup_dotdot(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
1825     struct timespec *tsp, int *ticksp)
1826 {
1827 	struct namecache_ts *ncp_ts;
1828 	struct namecache *ncp;
1829 	struct mtx *dvlp;
1830 	enum vgetstate vs;
1831 	int error, ltype;
1832 	bool whiteout;
1833 
1834 	MPASS((cnp->cn_flags & ISDOTDOT) != 0);
1835 
1836 	if ((cnp->cn_flags & MAKEENTRY) == 0) {
1837 		cache_remove_cnp(dvp, cnp);
1838 		return (0);
1839 	}
1840 
1841 	counter_u64_add(dotdothits, 1);
1842 retry:
1843 	dvlp = VP2VNODELOCK(dvp);
1844 	mtx_lock(dvlp);
1845 	ncp = dvp->v_cache_dd;
1846 	if (ncp == NULL) {
1847 		SDT_PROBE2(vfs, namecache, lookup, miss, dvp, "..");
1848 		mtx_unlock(dvlp);
1849 		return (0);
1850 	}
1851 	if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
1852 		if (ncp->nc_flag & NCF_NEGATIVE)
1853 			*vpp = NULL;
1854 		else
1855 			*vpp = ncp->nc_vp;
1856 	} else
1857 		*vpp = ncp->nc_dvp;
1858 	if (*vpp == NULL)
1859 		goto negative_success;
1860 	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..", *vpp);
1861 	cache_out_ts(ncp, tsp, ticksp);
1862 	if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
1863 	    NCF_DTS && tsp != NULL) {
1864 		ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
1865 		*tsp = ncp_ts->nc_dotdottime;
1866 	}
1867 
1868 	MPASS(dvp != *vpp);
1869 	ltype = VOP_ISLOCKED(dvp);
1870 	VOP_UNLOCK(dvp);
1871 	vs = vget_prep(*vpp);
1872 	mtx_unlock(dvlp);
1873 	error = vget_finish(*vpp, cnp->cn_lkflags, vs);
1874 	vn_lock(dvp, ltype | LK_RETRY);
1875 	if (VN_IS_DOOMED(dvp)) {
1876 		if (error == 0)
1877 			vput(*vpp);
1878 		*vpp = NULL;
1879 		return (ENOENT);
1880 	}
1881 	if (error) {
1882 		*vpp = NULL;
1883 		goto retry;
1884 	}
1885 	return (-1);
1886 negative_success:
1887 	if (__predict_false(cnp->cn_nameiop == CREATE)) {
1888 		if (cnp->cn_flags & ISLASTCN) {
1889 			counter_u64_add(numnegzaps, 1);
1890 			cache_zap_negative_locked_vnode_kl(ncp, dvp);
1891 			mtx_unlock(dvlp);
1892 			cache_free(ncp);
1893 			return (0);
1894 		}
1895 	}
1896 
1897 	whiteout = (ncp->nc_flag & NCF_WHITE);
1898 	cache_out_ts(ncp, tsp, ticksp);
1899 	if (cache_neg_hit_prep(ncp))
1900 		cache_neg_promote(ncp);
1901 	else
1902 		cache_neg_hit_finish(ncp);
1903 	mtx_unlock(dvlp);
1904 	if (whiteout)
1905 		cnp->cn_flags |= ISWHITEOUT;
1906 	return (ENOENT);
1907 }
1908 
1909 /**
1910  * Lookup a name in the name cache
1911  *
1912  * # Arguments
1913  *
1914  * - dvp:	Parent directory in which to search.
1915  * - vpp:	Return argument.  Will contain desired vnode on cache hit.
1916  * - cnp:	Parameters of the name search.  The most interesting bits of
1917  *   		the cn_flags field have the following meanings:
1918  *   	- MAKEENTRY:	If clear, free an entry from the cache rather than look
1919  *   			it up.
1920  *   	- ISDOTDOT:	Must be set if and only if cn_nameptr == ".."
1921  * - tsp:	Return storage for cache timestamp.  On a successful (positive
1922  *   		or negative) lookup, tsp will be filled with any timespec that
1923  *   		was stored when this cache entry was created.  However, it will
1924  *   		be clear for "." entries.
1925  * - ticks:	Return storage for alternate cache timestamp.  On a successful
1926  *   		(positive or negative) lookup, it will contain the ticks value
1927  *   		that was current when the cache entry was created, unless cnp
1928  *   		was ".".
1929  *
1930  * Either both tsp and ticks have to be provided or neither of them.
1931  *
1932  * # Returns
1933  *
1934  * - -1:	A positive cache hit.  vpp will contain the desired vnode.
1935  * - ENOENT:	A negative cache hit, or dvp was recycled out from under us due
1936  *		to a forced unmount.  vpp will not be modified.  If the entry
1937  *		is a whiteout, then the ISWHITEOUT flag will be set in
1938  *		cnp->cn_flags.
1939  * - 0:		A cache miss.  vpp will not be modified.
1940  *
1941  * # Locking
1942  *
1943  * On a cache hit, vpp will be returned locked and ref'd.  If we're looking up
1944  * .., dvp is unlocked.  If we're looking up . an extra ref is taken, but the
1945  * lock is not recursively acquired.
1946  */
1947 static int __noinline
1948 cache_lookup_fallback(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
1949     struct timespec *tsp, int *ticksp)
1950 {
1951 	struct namecache *ncp;
1952 	struct mtx *blp;
1953 	uint32_t hash;
1954 	enum vgetstate vs;
1955 	int error;
1956 	bool whiteout;
1957 
1958 	MPASS((cnp->cn_flags & ISDOTDOT) == 0);
1959 	MPASS((cnp->cn_flags & (MAKEENTRY | NC_KEEPPOSENTRY)) != 0);
1960 
1961 retry:
1962 	hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
1963 	blp = HASH2BUCKETLOCK(hash);
1964 	mtx_lock(blp);
1965 
1966 	CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
1967 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
1968 		    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
1969 			break;
1970 	}
1971 
1972 	if (__predict_false(ncp == NULL)) {
1973 		mtx_unlock(blp);
1974 		SDT_PROBE2(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr);
1975 		counter_u64_add(nummiss, 1);
1976 		return (0);
1977 	}
1978 
1979 	if (ncp->nc_flag & NCF_NEGATIVE)
1980 		goto negative_success;
1981 
1982 	counter_u64_add(numposhits, 1);
1983 	*vpp = ncp->nc_vp;
1984 	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp);
1985 	cache_out_ts(ncp, tsp, ticksp);
1986 	MPASS(dvp != *vpp);
1987 	vs = vget_prep(*vpp);
1988 	mtx_unlock(blp);
1989 	error = vget_finish(*vpp, cnp->cn_lkflags, vs);
1990 	if (error) {
1991 		*vpp = NULL;
1992 		goto retry;
1993 	}
1994 	return (-1);
1995 negative_success:
1996 	/*
1997 	 * We don't get here with regular lookup apart from corner cases.
1998 	 */
1999 	if (__predict_true(cnp->cn_nameiop == CREATE)) {
2000 		if (cnp->cn_flags & ISLASTCN) {
2001 			counter_u64_add(numnegzaps, 1);
2002 			error = cache_zap_locked_bucket(ncp, cnp, hash, blp);
2003 			if (__predict_false(error != 0)) {
2004 				zap_bucket_fail2++;
2005 				goto retry;
2006 			}
2007 			cache_free(ncp);
2008 			return (0);
2009 		}
2010 	}
2011 
2012 	whiteout = (ncp->nc_flag & NCF_WHITE);
2013 	cache_out_ts(ncp, tsp, ticksp);
2014 	if (cache_neg_hit_prep(ncp))
2015 		cache_neg_promote(ncp);
2016 	else
2017 		cache_neg_hit_finish(ncp);
2018 	mtx_unlock(blp);
2019 	if (whiteout)
2020 		cnp->cn_flags |= ISWHITEOUT;
2021 	return (ENOENT);
2022 }
2023 
2024 int
2025 cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
2026     struct timespec *tsp, int *ticksp)
2027 {
2028 	struct namecache *ncp;
2029 	uint32_t hash;
2030 	enum vgetstate vs;
2031 	int error;
2032 	bool whiteout, neg_promote;
2033 	u_short nc_flag;
2034 
2035 	MPASS((tsp == NULL && ticksp == NULL) || (tsp != NULL && ticksp != NULL));
2036 
2037 #ifdef DEBUG_CACHE
2038 	if (__predict_false(!doingcache)) {
2039 		cnp->cn_flags &= ~MAKEENTRY;
2040 		return (0);
2041 	}
2042 #endif
2043 
2044 	if (__predict_false(cnp->cn_nameptr[0] == '.')) {
2045 		if (cnp->cn_namelen == 1)
2046 			return (cache_lookup_dot(dvp, vpp, cnp, tsp, ticksp));
2047 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.')
2048 			return (cache_lookup_dotdot(dvp, vpp, cnp, tsp, ticksp));
2049 	}
2050 
2051 	MPASS((cnp->cn_flags & ISDOTDOT) == 0);
2052 
2053 	if ((cnp->cn_flags & (MAKEENTRY | NC_KEEPPOSENTRY)) == 0) {
2054 		cache_remove_cnp(dvp, cnp);
2055 		return (0);
2056 	}
2057 
2058 	hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
2059 	vfs_smr_enter();
2060 
2061 	CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
2062 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
2063 		    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
2064 			break;
2065 	}
2066 
2067 	if (__predict_false(ncp == NULL)) {
2068 		vfs_smr_exit();
2069 		SDT_PROBE2(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr);
2070 		counter_u64_add(nummiss, 1);
2071 		return (0);
2072 	}
2073 
2074 	nc_flag = atomic_load_char(&ncp->nc_flag);
2075 	if (nc_flag & NCF_NEGATIVE)
2076 		goto negative_success;
2077 
2078 	counter_u64_add(numposhits, 1);
2079 	*vpp = ncp->nc_vp;
2080 	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp);
2081 	cache_out_ts(ncp, tsp, ticksp);
2082 	MPASS(dvp != *vpp);
2083 	if (!cache_ncp_canuse(ncp)) {
2084 		vfs_smr_exit();
2085 		*vpp = NULL;
2086 		goto out_fallback;
2087 	}
2088 	vs = vget_prep_smr(*vpp);
2089 	vfs_smr_exit();
2090 	if (__predict_false(vs == VGET_NONE)) {
2091 		*vpp = NULL;
2092 		goto out_fallback;
2093 	}
2094 	error = vget_finish(*vpp, cnp->cn_lkflags, vs);
2095 	if (error) {
2096 		*vpp = NULL;
2097 		goto out_fallback;
2098 	}
2099 	return (-1);
2100 negative_success:
2101 	if (cnp->cn_nameiop == CREATE) {
2102 		if (cnp->cn_flags & ISLASTCN) {
2103 			vfs_smr_exit();
2104 			goto out_fallback;
2105 		}
2106 	}
2107 
2108 	cache_out_ts(ncp, tsp, ticksp);
2109 	whiteout = (atomic_load_char(&ncp->nc_flag) & NCF_WHITE);
2110 	neg_promote = cache_neg_hit_prep(ncp);
2111 	if (!cache_ncp_canuse(ncp)) {
2112 		cache_neg_hit_abort(ncp);
2113 		vfs_smr_exit();
2114 		goto out_fallback;
2115 	}
2116 	if (neg_promote) {
2117 		vfs_smr_exit();
2118 		if (!cache_neg_promote_cond(dvp, cnp, ncp, hash))
2119 			goto out_fallback;
2120 	} else {
2121 		cache_neg_hit_finish(ncp);
2122 		vfs_smr_exit();
2123 	}
2124 	if (whiteout)
2125 		cnp->cn_flags |= ISWHITEOUT;
2126 	return (ENOENT);
2127 out_fallback:
2128 	return (cache_lookup_fallback(dvp, vpp, cnp, tsp, ticksp));
2129 }
2130 
2131 struct celockstate {
2132 	struct mtx *vlp[3];
2133 	struct mtx *blp[2];
2134 };
2135 CTASSERT((nitems(((struct celockstate *)0)->vlp) == 3));
2136 CTASSERT((nitems(((struct celockstate *)0)->blp) == 2));
2137 
2138 static inline void
2139 cache_celockstate_init(struct celockstate *cel)
2140 {
2141 
2142 	bzero(cel, sizeof(*cel));
2143 }
2144 
2145 static void
2146 cache_lock_vnodes_cel(struct celockstate *cel, struct vnode *vp,
2147     struct vnode *dvp)
2148 {
2149 	struct mtx *vlp1, *vlp2;
2150 
2151 	MPASS(cel->vlp[0] == NULL);
2152 	MPASS(cel->vlp[1] == NULL);
2153 	MPASS(cel->vlp[2] == NULL);
2154 
2155 	MPASS(vp != NULL || dvp != NULL);
2156 
2157 	vlp1 = VP2VNODELOCK(vp);
2158 	vlp2 = VP2VNODELOCK(dvp);
2159 	cache_sort_vnodes(&vlp1, &vlp2);
2160 
2161 	if (vlp1 != NULL) {
2162 		mtx_lock(vlp1);
2163 		cel->vlp[0] = vlp1;
2164 	}
2165 	mtx_lock(vlp2);
2166 	cel->vlp[1] = vlp2;
2167 }
2168 
2169 static void
2170 cache_unlock_vnodes_cel(struct celockstate *cel)
2171 {
2172 
2173 	MPASS(cel->vlp[0] != NULL || cel->vlp[1] != NULL);
2174 
2175 	if (cel->vlp[0] != NULL)
2176 		mtx_unlock(cel->vlp[0]);
2177 	if (cel->vlp[1] != NULL)
2178 		mtx_unlock(cel->vlp[1]);
2179 	if (cel->vlp[2] != NULL)
2180 		mtx_unlock(cel->vlp[2]);
2181 }
2182 
2183 static bool
2184 cache_lock_vnodes_cel_3(struct celockstate *cel, struct vnode *vp)
2185 {
2186 	struct mtx *vlp;
2187 	bool ret;
2188 
2189 	cache_assert_vlp_locked(cel->vlp[0]);
2190 	cache_assert_vlp_locked(cel->vlp[1]);
2191 	MPASS(cel->vlp[2] == NULL);
2192 
2193 	MPASS(vp != NULL);
2194 	vlp = VP2VNODELOCK(vp);
2195 
2196 	ret = true;
2197 	if (vlp >= cel->vlp[1]) {
2198 		mtx_lock(vlp);
2199 	} else {
2200 		if (mtx_trylock(vlp))
2201 			goto out;
2202 		cache_lock_vnodes_cel_3_failures++;
2203 		cache_unlock_vnodes_cel(cel);
2204 		if (vlp < cel->vlp[0]) {
2205 			mtx_lock(vlp);
2206 			mtx_lock(cel->vlp[0]);
2207 			mtx_lock(cel->vlp[1]);
2208 		} else {
2209 			if (cel->vlp[0] != NULL)
2210 				mtx_lock(cel->vlp[0]);
2211 			mtx_lock(vlp);
2212 			mtx_lock(cel->vlp[1]);
2213 		}
2214 		ret = false;
2215 	}
2216 out:
2217 	cel->vlp[2] = vlp;
2218 	return (ret);
2219 }
2220 
2221 static void
2222 cache_lock_buckets_cel(struct celockstate *cel, struct mtx *blp1,
2223     struct mtx *blp2)
2224 {
2225 
2226 	MPASS(cel->blp[0] == NULL);
2227 	MPASS(cel->blp[1] == NULL);
2228 
2229 	cache_sort_vnodes(&blp1, &blp2);
2230 
2231 	if (blp1 != NULL) {
2232 		mtx_lock(blp1);
2233 		cel->blp[0] = blp1;
2234 	}
2235 	mtx_lock(blp2);
2236 	cel->blp[1] = blp2;
2237 }
2238 
2239 static void
2240 cache_unlock_buckets_cel(struct celockstate *cel)
2241 {
2242 
2243 	if (cel->blp[0] != NULL)
2244 		mtx_unlock(cel->blp[0]);
2245 	mtx_unlock(cel->blp[1]);
2246 }
2247 
2248 /*
2249  * Lock part of the cache affected by the insertion.
2250  *
2251  * This means vnodelocks for dvp, vp and the relevant bucketlock.
2252  * However, insertion can result in removal of an old entry. In this
2253  * case we have an additional vnode and bucketlock pair to lock.
2254  *
2255  * That is, in the worst case we have to lock 3 vnodes and 2 bucketlocks, while
2256  * preserving the locking order (smaller address first).
2257  */
2258 static void
2259 cache_enter_lock(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
2260     uint32_t hash)
2261 {
2262 	struct namecache *ncp;
2263 	struct mtx *blps[2];
2264 	u_char nc_flag;
2265 
2266 	blps[0] = HASH2BUCKETLOCK(hash);
2267 	for (;;) {
2268 		blps[1] = NULL;
2269 		cache_lock_vnodes_cel(cel, dvp, vp);
2270 		if (vp == NULL || vp->v_type != VDIR)
2271 			break;
2272 		ncp = atomic_load_consume_ptr(&vp->v_cache_dd);
2273 		if (ncp == NULL)
2274 			break;
2275 		nc_flag = atomic_load_char(&ncp->nc_flag);
2276 		if ((nc_flag & NCF_ISDOTDOT) == 0)
2277 			break;
2278 		MPASS(ncp->nc_dvp == vp);
2279 		blps[1] = NCP2BUCKETLOCK(ncp);
2280 		if ((nc_flag & NCF_NEGATIVE) != 0)
2281 			break;
2282 		if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
2283 			break;
2284 		/*
2285 		 * All vnodes got re-locked. Re-validate the state and if
2286 		 * nothing changed we are done. Otherwise restart.
2287 		 */
2288 		if (ncp == vp->v_cache_dd &&
2289 		    (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
2290 		    blps[1] == NCP2BUCKETLOCK(ncp) &&
2291 		    VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
2292 			break;
2293 		cache_unlock_vnodes_cel(cel);
2294 		cel->vlp[0] = NULL;
2295 		cel->vlp[1] = NULL;
2296 		cel->vlp[2] = NULL;
2297 	}
2298 	cache_lock_buckets_cel(cel, blps[0], blps[1]);
2299 }
2300 
2301 static void
2302 cache_enter_lock_dd(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
2303     uint32_t hash)
2304 {
2305 	struct namecache *ncp;
2306 	struct mtx *blps[2];
2307 	u_char nc_flag;
2308 
2309 	blps[0] = HASH2BUCKETLOCK(hash);
2310 	for (;;) {
2311 		blps[1] = NULL;
2312 		cache_lock_vnodes_cel(cel, dvp, vp);
2313 		ncp = atomic_load_consume_ptr(&dvp->v_cache_dd);
2314 		if (ncp == NULL)
2315 			break;
2316 		nc_flag = atomic_load_char(&ncp->nc_flag);
2317 		if ((nc_flag & NCF_ISDOTDOT) == 0)
2318 			break;
2319 		MPASS(ncp->nc_dvp == dvp);
2320 		blps[1] = NCP2BUCKETLOCK(ncp);
2321 		if ((nc_flag & NCF_NEGATIVE) != 0)
2322 			break;
2323 		if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
2324 			break;
2325 		if (ncp == dvp->v_cache_dd &&
2326 		    (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
2327 		    blps[1] == NCP2BUCKETLOCK(ncp) &&
2328 		    VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
2329 			break;
2330 		cache_unlock_vnodes_cel(cel);
2331 		cel->vlp[0] = NULL;
2332 		cel->vlp[1] = NULL;
2333 		cel->vlp[2] = NULL;
2334 	}
2335 	cache_lock_buckets_cel(cel, blps[0], blps[1]);
2336 }
2337 
2338 static void
2339 cache_enter_unlock(struct celockstate *cel)
2340 {
2341 
2342 	cache_unlock_buckets_cel(cel);
2343 	cache_unlock_vnodes_cel(cel);
2344 }
2345 
2346 static void __noinline
2347 cache_enter_dotdot_prep(struct vnode *dvp, struct vnode *vp,
2348     struct componentname *cnp)
2349 {
2350 	struct celockstate cel;
2351 	struct namecache *ncp;
2352 	uint32_t hash;
2353 	int len;
2354 
2355 	if (atomic_load_ptr(&dvp->v_cache_dd) == NULL)
2356 		return;
2357 	len = cnp->cn_namelen;
2358 	cache_celockstate_init(&cel);
2359 	hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
2360 	cache_enter_lock_dd(&cel, dvp, vp, hash);
2361 	ncp = dvp->v_cache_dd;
2362 	if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT)) {
2363 		KASSERT(ncp->nc_dvp == dvp, ("wrong isdotdot parent"));
2364 		cache_zap_locked(ncp);
2365 	} else {
2366 		ncp = NULL;
2367 	}
2368 	atomic_store_ptr(&dvp->v_cache_dd, NULL);
2369 	cache_enter_unlock(&cel);
2370 	if (ncp != NULL)
2371 		cache_free(ncp);
2372 }
2373 
2374 /*
2375  * Add an entry to the cache.
2376  */
2377 void
2378 cache_enter_time(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
2379     struct timespec *tsp, struct timespec *dtsp)
2380 {
2381 	struct celockstate cel;
2382 	struct namecache *ncp, *n2, *ndd;
2383 	struct namecache_ts *ncp_ts;
2384 	struct nchashhead *ncpp;
2385 	uint32_t hash;
2386 	int flag;
2387 	int len;
2388 
2389 	KASSERT(cnp->cn_namelen <= NAME_MAX,
2390 	    ("%s: passed len %ld exceeds NAME_MAX (%d)", __func__, cnp->cn_namelen,
2391 	    NAME_MAX));
2392 #ifdef notyet
2393 	/*
2394 	 * Not everything doing this is weeded out yet.
2395 	 */
2396 	VNPASS(dvp != vp, dvp);
2397 #endif
2398 	VNPASS(!VN_IS_DOOMED(dvp), dvp);
2399 	VNPASS(dvp->v_type != VNON, dvp);
2400 	if (vp != NULL) {
2401 		VNPASS(!VN_IS_DOOMED(vp), vp);
2402 		VNPASS(vp->v_type != VNON, vp);
2403 	}
2404 
2405 #ifdef DEBUG_CACHE
2406 	if (__predict_false(!doingcache))
2407 		return;
2408 #endif
2409 
2410 	flag = 0;
2411 	if (__predict_false(cnp->cn_nameptr[0] == '.')) {
2412 		if (cnp->cn_namelen == 1)
2413 			return;
2414 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
2415 			cache_enter_dotdot_prep(dvp, vp, cnp);
2416 			flag = NCF_ISDOTDOT;
2417 		}
2418 	}
2419 
2420 	ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
2421 	if (ncp == NULL)
2422 		return;
2423 
2424 	cache_celockstate_init(&cel);
2425 	ndd = NULL;
2426 	ncp_ts = NULL;
2427 
2428 	/*
2429 	 * Calculate the hash key and setup as much of the new
2430 	 * namecache entry as possible before acquiring the lock.
2431 	 */
2432 	ncp->nc_flag = flag | NCF_WIP;
2433 	ncp->nc_vp = vp;
2434 	if (vp == NULL)
2435 		cache_neg_init(ncp);
2436 	ncp->nc_dvp = dvp;
2437 	if (tsp != NULL) {
2438 		ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
2439 		ncp_ts->nc_time = *tsp;
2440 		ncp_ts->nc_ticks = ticks;
2441 		ncp_ts->nc_nc.nc_flag |= NCF_TS;
2442 		if (dtsp != NULL) {
2443 			ncp_ts->nc_dotdottime = *dtsp;
2444 			ncp_ts->nc_nc.nc_flag |= NCF_DTS;
2445 		}
2446 	}
2447 	len = ncp->nc_nlen = cnp->cn_namelen;
2448 	hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
2449 	memcpy(ncp->nc_name, cnp->cn_nameptr, len);
2450 	ncp->nc_name[len] = '\0';
2451 	cache_enter_lock(&cel, dvp, vp, hash);
2452 
2453 	/*
2454 	 * See if this vnode or negative entry is already in the cache
2455 	 * with this name.  This can happen with concurrent lookups of
2456 	 * the same path name.
2457 	 */
2458 	ncpp = NCHHASH(hash);
2459 	CK_SLIST_FOREACH(n2, ncpp, nc_hash) {
2460 		if (n2->nc_dvp == dvp &&
2461 		    n2->nc_nlen == cnp->cn_namelen &&
2462 		    !bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) {
2463 			MPASS(cache_ncp_canuse(n2));
2464 			if ((n2->nc_flag & NCF_NEGATIVE) != 0)
2465 				KASSERT(vp == NULL,
2466 				    ("%s: found entry pointing to a different vnode (%p != %p) ; name [%s]",
2467 				    __func__, NULL, vp, cnp->cn_nameptr));
2468 			else
2469 				KASSERT(n2->nc_vp == vp,
2470 				    ("%s: found entry pointing to a different vnode (%p != %p) ; name [%s]",
2471 				    __func__, n2->nc_vp, vp, cnp->cn_nameptr));
2472 			/*
2473 			 * Entries are supposed to be immutable unless in the
2474 			 * process of getting destroyed. Accommodating for
2475 			 * changing timestamps is possible but not worth it.
2476 			 * This should be harmless in terms of correctness, in
2477 			 * the worst case resulting in an earlier expiration.
2478 			 * Alternatively, the found entry can be replaced
2479 			 * altogether.
2480 			 */
2481 			MPASS((n2->nc_flag & (NCF_TS | NCF_DTS)) == (ncp->nc_flag & (NCF_TS | NCF_DTS)));
2482 #if 0
2483 			if (tsp != NULL) {
2484 				KASSERT((n2->nc_flag & NCF_TS) != 0,
2485 				    ("no NCF_TS"));
2486 				n2_ts = __containerof(n2, struct namecache_ts, nc_nc);
2487 				n2_ts->nc_time = ncp_ts->nc_time;
2488 				n2_ts->nc_ticks = ncp_ts->nc_ticks;
2489 				if (dtsp != NULL) {
2490 					n2_ts->nc_dotdottime = ncp_ts->nc_dotdottime;
2491 					n2_ts->nc_nc.nc_flag |= NCF_DTS;
2492 				}
2493 			}
2494 #endif
2495 			SDT_PROBE3(vfs, namecache, enter, duplicate, dvp, ncp->nc_name,
2496 			    vp);
2497 			goto out_unlock_free;
2498 		}
2499 	}
2500 
2501 	if (flag == NCF_ISDOTDOT) {
2502 		/*
2503 		 * See if we are trying to add .. entry, but some other lookup
2504 		 * has populated v_cache_dd pointer already.
2505 		 */
2506 		if (dvp->v_cache_dd != NULL)
2507 			goto out_unlock_free;
2508 		KASSERT(vp == NULL || vp->v_type == VDIR,
2509 		    ("wrong vnode type %p", vp));
2510 		atomic_thread_fence_rel();
2511 		atomic_store_ptr(&dvp->v_cache_dd, ncp);
2512 	}
2513 
2514 	if (vp != NULL) {
2515 		if (flag != NCF_ISDOTDOT) {
2516 			/*
2517 			 * For this case, the cache entry maps both the
2518 			 * directory name in it and the name ".." for the
2519 			 * directory's parent.
2520 			 */
2521 			if ((ndd = vp->v_cache_dd) != NULL) {
2522 				if ((ndd->nc_flag & NCF_ISDOTDOT) != 0)
2523 					cache_zap_locked(ndd);
2524 				else
2525 					ndd = NULL;
2526 			}
2527 			atomic_thread_fence_rel();
2528 			atomic_store_ptr(&vp->v_cache_dd, ncp);
2529 		} else if (vp->v_type != VDIR) {
2530 			if (vp->v_cache_dd != NULL) {
2531 				atomic_store_ptr(&vp->v_cache_dd, NULL);
2532 			}
2533 		}
2534 	}
2535 
2536 	if (flag != NCF_ISDOTDOT) {
2537 		if (LIST_EMPTY(&dvp->v_cache_src)) {
2538 			cache_hold_vnode(dvp);
2539 		}
2540 		LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
2541 	}
2542 
2543 	/*
2544 	 * If the entry is "negative", we place it into the
2545 	 * "negative" cache queue, otherwise, we place it into the
2546 	 * destination vnode's cache entries queue.
2547 	 */
2548 	if (vp != NULL) {
2549 		TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
2550 		SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name,
2551 		    vp);
2552 	} else {
2553 		if (cnp->cn_flags & ISWHITEOUT)
2554 			atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_WHITE);
2555 		cache_neg_insert(ncp);
2556 		SDT_PROBE2(vfs, namecache, enter_negative, done, dvp,
2557 		    ncp->nc_name);
2558 	}
2559 
2560 	/*
2561 	 * Insert the new namecache entry into the appropriate chain
2562 	 * within the cache entries table.
2563 	 */
2564 	CK_SLIST_INSERT_HEAD(ncpp, ncp, nc_hash);
2565 
2566 	atomic_thread_fence_rel();
2567 	/*
2568 	 * Mark the entry as fully constructed.
2569 	 * It is immutable past this point until its removal.
2570 	 */
2571 	atomic_store_char(&ncp->nc_flag, ncp->nc_flag & ~NCF_WIP);
2572 
2573 	cache_enter_unlock(&cel);
2574 	if (ndd != NULL)
2575 		cache_free(ndd);
2576 	return;
2577 out_unlock_free:
2578 	cache_enter_unlock(&cel);
2579 	cache_free(ncp);
2580 	return;
2581 }
2582 
2583 /*
2584  * A variant of the above accepting flags.
2585  *
2586  * - VFS_CACHE_DROPOLD -- if a conflicting entry is found, drop it.
2587  *
2588  * TODO: this routine is a hack. It blindly removes the old entry, even if it
2589  * happens to match and it is doing it in an inefficient manner. It was added
2590  * to accomodate NFS which runs into a case where the target for a given name
2591  * may change from under it. Note this does nothing to solve the following
2592  * race: 2 callers of cache_enter_time_flags pass a different target vnode for
2593  * the same [dvp, cnp]. It may be argued that code doing this is broken.
2594  */
2595 void
2596 cache_enter_time_flags(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
2597     struct timespec *tsp, struct timespec *dtsp, int flags)
2598 {
2599 
2600 	MPASS((flags & ~(VFS_CACHE_DROPOLD)) == 0);
2601 
2602 	if (flags & VFS_CACHE_DROPOLD)
2603 		cache_remove_cnp(dvp, cnp);
2604 	cache_enter_time(dvp, vp, cnp, tsp, dtsp);
2605 }
2606 
2607 static u_int
2608 cache_roundup_2(u_int val)
2609 {
2610 	u_int res;
2611 
2612 	for (res = 1; res <= val; res <<= 1)
2613 		continue;
2614 
2615 	return (res);
2616 }
2617 
2618 static struct nchashhead *
2619 nchinittbl(u_long elements, u_long *hashmask)
2620 {
2621 	struct nchashhead *hashtbl;
2622 	u_long hashsize, i;
2623 
2624 	hashsize = cache_roundup_2(elements) / 2;
2625 
2626 	hashtbl = malloc((u_long)hashsize * sizeof(*hashtbl), M_VFSCACHE, M_WAITOK);
2627 	for (i = 0; i < hashsize; i++)
2628 		CK_SLIST_INIT(&hashtbl[i]);
2629 	*hashmask = hashsize - 1;
2630 	return (hashtbl);
2631 }
2632 
2633 static void
2634 ncfreetbl(struct nchashhead *hashtbl)
2635 {
2636 
2637 	free(hashtbl, M_VFSCACHE);
2638 }
2639 
2640 /*
2641  * Name cache initialization, from vfs_init() when we are booting
2642  */
2643 static void
2644 nchinit(void *dummy __unused)
2645 {
2646 	u_int i;
2647 
2648 	cache_zone_small = uma_zcreate("S VFS Cache", CACHE_ZONE_SMALL_SIZE,
2649 	    NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
2650 	cache_zone_small_ts = uma_zcreate("STS VFS Cache", CACHE_ZONE_SMALL_TS_SIZE,
2651 	    NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
2652 	cache_zone_large = uma_zcreate("L VFS Cache", CACHE_ZONE_LARGE_SIZE,
2653 	    NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
2654 	cache_zone_large_ts = uma_zcreate("LTS VFS Cache", CACHE_ZONE_LARGE_TS_SIZE,
2655 	    NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
2656 
2657 	VFS_SMR_ZONE_SET(cache_zone_small);
2658 	VFS_SMR_ZONE_SET(cache_zone_small_ts);
2659 	VFS_SMR_ZONE_SET(cache_zone_large);
2660 	VFS_SMR_ZONE_SET(cache_zone_large_ts);
2661 
2662 	ncsize = desiredvnodes * ncsizefactor;
2663 	cache_recalc_neg_min(ncnegminpct);
2664 	nchashtbl = nchinittbl(desiredvnodes * 2, &nchash);
2665 	ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1;
2666 	if (ncbuckethash < 7) /* arbitrarily chosen to avoid having one lock */
2667 		ncbuckethash = 7;
2668 	if (ncbuckethash > nchash)
2669 		ncbuckethash = nchash;
2670 	bucketlocks = malloc(sizeof(*bucketlocks) * numbucketlocks, M_VFSCACHE,
2671 	    M_WAITOK | M_ZERO);
2672 	for (i = 0; i < numbucketlocks; i++)
2673 		mtx_init(&bucketlocks[i], "ncbuc", NULL, MTX_DUPOK | MTX_RECURSE);
2674 	ncvnodehash = ncbuckethash;
2675 	vnodelocks = malloc(sizeof(*vnodelocks) * numvnodelocks, M_VFSCACHE,
2676 	    M_WAITOK | M_ZERO);
2677 	for (i = 0; i < numvnodelocks; i++)
2678 		mtx_init(&vnodelocks[i], "ncvn", NULL, MTX_DUPOK | MTX_RECURSE);
2679 
2680 	for (i = 0; i < numneglists; i++) {
2681 		mtx_init(&neglists[i].nl_evict_lock, "ncnege", NULL, MTX_DEF);
2682 		mtx_init(&neglists[i].nl_lock, "ncnegl", NULL, MTX_DEF);
2683 		TAILQ_INIT(&neglists[i].nl_list);
2684 		TAILQ_INIT(&neglists[i].nl_hotlist);
2685 	}
2686 }
2687 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nchinit, NULL);
2688 
2689 void
2690 cache_vnode_init(struct vnode *vp)
2691 {
2692 
2693 	LIST_INIT(&vp->v_cache_src);
2694 	TAILQ_INIT(&vp->v_cache_dst);
2695 	vp->v_cache_dd = NULL;
2696 	cache_prehash(vp);
2697 }
2698 
2699 /*
2700  * Induce transient cache misses for lockless operation in cache_lookup() by
2701  * using a temporary hash table.
2702  *
2703  * This will force a fs lookup.
2704  *
2705  * Synchronisation is done in 2 steps, calling vfs_smr_synchronize each time
2706  * to observe all CPUs not performing the lookup.
2707  */
2708 static void
2709 cache_changesize_set_temp(struct nchashhead *temptbl, u_long temphash)
2710 {
2711 
2712 	MPASS(temphash < nchash);
2713 	/*
2714 	 * Change the size. The new size is smaller and can safely be used
2715 	 * against the existing table. All lookups which now hash wrong will
2716 	 * result in a cache miss, which all callers are supposed to know how
2717 	 * to handle.
2718 	 */
2719 	atomic_store_long(&nchash, temphash);
2720 	atomic_thread_fence_rel();
2721 	vfs_smr_synchronize();
2722 	/*
2723 	 * At this point everyone sees the updated hash value, but they still
2724 	 * see the old table.
2725 	 */
2726 	atomic_store_ptr(&nchashtbl, temptbl);
2727 	atomic_thread_fence_rel();
2728 	vfs_smr_synchronize();
2729 	/*
2730 	 * At this point everyone sees the updated table pointer and size pair.
2731 	 */
2732 }
2733 
2734 /*
2735  * Set the new hash table.
2736  *
2737  * Similarly to cache_changesize_set_temp(), this has to synchronize against
2738  * lockless operation in cache_lookup().
2739  */
2740 static void
2741 cache_changesize_set_new(struct nchashhead *new_tbl, u_long new_hash)
2742 {
2743 
2744 	MPASS(nchash < new_hash);
2745 	/*
2746 	 * Change the pointer first. This wont result in out of bounds access
2747 	 * since the temporary table is guaranteed to be smaller.
2748 	 */
2749 	atomic_store_ptr(&nchashtbl, new_tbl);
2750 	atomic_thread_fence_rel();
2751 	vfs_smr_synchronize();
2752 	/*
2753 	 * At this point everyone sees the updated pointer value, but they
2754 	 * still see the old size.
2755 	 */
2756 	atomic_store_long(&nchash, new_hash);
2757 	atomic_thread_fence_rel();
2758 	vfs_smr_synchronize();
2759 	/*
2760 	 * At this point everyone sees the updated table pointer and size pair.
2761 	 */
2762 }
2763 
2764 void
2765 cache_changesize(u_long newmaxvnodes)
2766 {
2767 	struct nchashhead *new_nchashtbl, *old_nchashtbl, *temptbl;
2768 	u_long new_nchash, old_nchash, temphash;
2769 	struct namecache *ncp;
2770 	uint32_t hash;
2771 	u_long newncsize;
2772 	int i;
2773 
2774 	newncsize = newmaxvnodes * ncsizefactor;
2775 	newmaxvnodes = cache_roundup_2(newmaxvnodes * 2);
2776 	if (newmaxvnodes < numbucketlocks)
2777 		newmaxvnodes = numbucketlocks;
2778 
2779 	new_nchashtbl = nchinittbl(newmaxvnodes, &new_nchash);
2780 	/* If same hash table size, nothing to do */
2781 	if (nchash == new_nchash) {
2782 		ncfreetbl(new_nchashtbl);
2783 		return;
2784 	}
2785 
2786 	temptbl = nchinittbl(1, &temphash);
2787 
2788 	/*
2789 	 * Move everything from the old hash table to the new table.
2790 	 * None of the namecache entries in the table can be removed
2791 	 * because to do so, they have to be removed from the hash table.
2792 	 */
2793 	cache_lock_all_vnodes();
2794 	cache_lock_all_buckets();
2795 	old_nchashtbl = nchashtbl;
2796 	old_nchash = nchash;
2797 	cache_changesize_set_temp(temptbl, temphash);
2798 	for (i = 0; i <= old_nchash; i++) {
2799 		while ((ncp = CK_SLIST_FIRST(&old_nchashtbl[i])) != NULL) {
2800 			hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen,
2801 			    ncp->nc_dvp);
2802 			CK_SLIST_REMOVE(&old_nchashtbl[i], ncp, namecache, nc_hash);
2803 			CK_SLIST_INSERT_HEAD(&new_nchashtbl[hash & new_nchash], ncp, nc_hash);
2804 		}
2805 	}
2806 	ncsize = newncsize;
2807 	cache_recalc_neg_min(ncnegminpct);
2808 	cache_changesize_set_new(new_nchashtbl, new_nchash);
2809 	cache_unlock_all_buckets();
2810 	cache_unlock_all_vnodes();
2811 	ncfreetbl(old_nchashtbl);
2812 	ncfreetbl(temptbl);
2813 }
2814 
2815 /*
2816  * Remove all entries from and to a particular vnode.
2817  */
2818 static void
2819 cache_purge_impl(struct vnode *vp)
2820 {
2821 	struct cache_freebatch batch;
2822 	struct namecache *ncp;
2823 	struct mtx *vlp, *vlp2;
2824 
2825 	TAILQ_INIT(&batch);
2826 	vlp = VP2VNODELOCK(vp);
2827 	vlp2 = NULL;
2828 	mtx_lock(vlp);
2829 retry:
2830 	while (!LIST_EMPTY(&vp->v_cache_src)) {
2831 		ncp = LIST_FIRST(&vp->v_cache_src);
2832 		if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
2833 			goto retry;
2834 		TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
2835 	}
2836 	while (!TAILQ_EMPTY(&vp->v_cache_dst)) {
2837 		ncp = TAILQ_FIRST(&vp->v_cache_dst);
2838 		if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
2839 			goto retry;
2840 		TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
2841 	}
2842 	ncp = vp->v_cache_dd;
2843 	if (ncp != NULL) {
2844 		KASSERT(ncp->nc_flag & NCF_ISDOTDOT,
2845 		   ("lost dotdot link"));
2846 		if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
2847 			goto retry;
2848 		TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
2849 	}
2850 	KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
2851 	mtx_unlock(vlp);
2852 	if (vlp2 != NULL)
2853 		mtx_unlock(vlp2);
2854 	cache_free_batch(&batch);
2855 }
2856 
2857 /*
2858  * Opportunistic check to see if there is anything to do.
2859  */
2860 static bool
2861 cache_has_entries(struct vnode *vp)
2862 {
2863 
2864 	if (LIST_EMPTY(&vp->v_cache_src) && TAILQ_EMPTY(&vp->v_cache_dst) &&
2865 	    atomic_load_ptr(&vp->v_cache_dd) == NULL)
2866 		return (false);
2867 	return (true);
2868 }
2869 
2870 void
2871 cache_purge(struct vnode *vp)
2872 {
2873 
2874 	SDT_PROBE1(vfs, namecache, purge, done, vp);
2875 	if (!cache_has_entries(vp))
2876 		return;
2877 	cache_purge_impl(vp);
2878 }
2879 
2880 /*
2881  * Only to be used by vgone.
2882  */
2883 void
2884 cache_purge_vgone(struct vnode *vp)
2885 {
2886 	struct mtx *vlp;
2887 
2888 	VNPASS(VN_IS_DOOMED(vp), vp);
2889 	if (cache_has_entries(vp)) {
2890 		cache_purge_impl(vp);
2891 		return;
2892 	}
2893 
2894 	/*
2895 	 * Serialize against a potential thread doing cache_purge.
2896 	 */
2897 	vlp = VP2VNODELOCK(vp);
2898 	mtx_wait_unlocked(vlp);
2899 	if (cache_has_entries(vp)) {
2900 		cache_purge_impl(vp);
2901 		return;
2902 	}
2903 	return;
2904 }
2905 
2906 /*
2907  * Remove all negative entries for a particular directory vnode.
2908  */
2909 void
2910 cache_purge_negative(struct vnode *vp)
2911 {
2912 	struct cache_freebatch batch;
2913 	struct namecache *ncp, *nnp;
2914 	struct mtx *vlp;
2915 
2916 	SDT_PROBE1(vfs, namecache, purge_negative, done, vp);
2917 	if (LIST_EMPTY(&vp->v_cache_src))
2918 		return;
2919 	TAILQ_INIT(&batch);
2920 	vlp = VP2VNODELOCK(vp);
2921 	mtx_lock(vlp);
2922 	LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) {
2923 		if (!(ncp->nc_flag & NCF_NEGATIVE))
2924 			continue;
2925 		cache_zap_negative_locked_vnode_kl(ncp, vp);
2926 		TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
2927 	}
2928 	mtx_unlock(vlp);
2929 	cache_free_batch(&batch);
2930 }
2931 
2932 /*
2933  * Entry points for modifying VOP operations.
2934  */
2935 void
2936 cache_vop_rename(struct vnode *fdvp, struct vnode *fvp, struct vnode *tdvp,
2937     struct vnode *tvp, struct componentname *fcnp, struct componentname *tcnp)
2938 {
2939 
2940 	ASSERT_VOP_IN_SEQC(fdvp);
2941 	ASSERT_VOP_IN_SEQC(fvp);
2942 	ASSERT_VOP_IN_SEQC(tdvp);
2943 	if (tvp != NULL)
2944 		ASSERT_VOP_IN_SEQC(tvp);
2945 
2946 	cache_purge(fvp);
2947 	if (tvp != NULL) {
2948 		cache_purge(tvp);
2949 		KASSERT(!cache_remove_cnp(tdvp, tcnp),
2950 		    ("%s: lingering negative entry", __func__));
2951 	} else {
2952 		cache_remove_cnp(tdvp, tcnp);
2953 	}
2954 
2955 	/*
2956 	 * TODO
2957 	 *
2958 	 * Historically renaming was always purging all revelang entries,
2959 	 * but that's quite wasteful. In particular turns out that in many cases
2960 	 * the target file is immediately accessed after rename, inducing a cache
2961 	 * miss.
2962 	 *
2963 	 * Recode this to reduce relocking and reuse the existing entry (if any)
2964 	 * instead of just removing it above and allocating a new one here.
2965 	 */
2966 	if (cache_rename_add) {
2967 		cache_enter(tdvp, fvp, tcnp);
2968 	}
2969 }
2970 
2971 void
2972 cache_vop_rmdir(struct vnode *dvp, struct vnode *vp)
2973 {
2974 
2975 	ASSERT_VOP_IN_SEQC(dvp);
2976 	ASSERT_VOP_IN_SEQC(vp);
2977 	cache_purge(vp);
2978 }
2979 
2980 #ifdef INVARIANTS
2981 /*
2982  * Validate that if an entry exists it matches.
2983  */
2984 void
2985 cache_validate(struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
2986 {
2987 	struct namecache *ncp;
2988 	struct mtx *blp;
2989 	uint32_t hash;
2990 
2991 	hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
2992 	if (CK_SLIST_EMPTY(NCHHASH(hash)))
2993 		return;
2994 	blp = HASH2BUCKETLOCK(hash);
2995 	mtx_lock(blp);
2996 	CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
2997 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
2998 		    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) {
2999 			if (ncp->nc_vp != vp)
3000 				panic("%s: mismatch (%p != %p); ncp %p [%s] dvp %p\n",
3001 				    __func__, vp, ncp->nc_vp, ncp, ncp->nc_name, ncp->nc_dvp);
3002 		}
3003 	}
3004 	mtx_unlock(blp);
3005 }
3006 #endif
3007 
3008 /*
3009  * Flush all entries referencing a particular filesystem.
3010  */
3011 void
3012 cache_purgevfs(struct mount *mp)
3013 {
3014 	struct vnode *vp, *mvp;
3015 
3016 	SDT_PROBE1(vfs, namecache, purgevfs, done, mp);
3017 	/*
3018 	 * Somewhat wasteful iteration over all vnodes. Would be better to
3019 	 * support filtering and avoid the interlock to begin with.
3020 	 */
3021 	MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
3022 		if (!cache_has_entries(vp)) {
3023 			VI_UNLOCK(vp);
3024 			continue;
3025 		}
3026 		vholdl(vp);
3027 		VI_UNLOCK(vp);
3028 		cache_purge(vp);
3029 		vdrop(vp);
3030 	}
3031 }
3032 
3033 /*
3034  * Perform canonical checks and cache lookup and pass on to filesystem
3035  * through the vop_cachedlookup only if needed.
3036  */
3037 
3038 int
3039 vfs_cache_lookup(struct vop_lookup_args *ap)
3040 {
3041 	struct vnode *dvp;
3042 	int error;
3043 	struct vnode **vpp = ap->a_vpp;
3044 	struct componentname *cnp = ap->a_cnp;
3045 	int flags = cnp->cn_flags;
3046 
3047 	*vpp = NULL;
3048 	dvp = ap->a_dvp;
3049 
3050 	if (dvp->v_type != VDIR)
3051 		return (ENOTDIR);
3052 
3053 	if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
3054 	    (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME))
3055 		return (EROFS);
3056 
3057 	error = vn_dir_check_exec(dvp, cnp);
3058 	if (error != 0)
3059 		return (error);
3060 
3061 	error = cache_lookup(dvp, vpp, cnp, NULL, NULL);
3062 	if (error == 0)
3063 		return (VOP_CACHEDLOOKUP(dvp, vpp, cnp));
3064 	if (error == -1)
3065 		return (0);
3066 	return (error);
3067 }
3068 
3069 /* Implementation of the getcwd syscall. */
3070 int
3071 sys___getcwd(struct thread *td, struct __getcwd_args *uap)
3072 {
3073 	char *buf, *retbuf;
3074 	size_t buflen;
3075 	int error;
3076 
3077 	buflen = uap->buflen;
3078 	if (__predict_false(buflen < 2))
3079 		return (EINVAL);
3080 	if (buflen > MAXPATHLEN)
3081 		buflen = MAXPATHLEN;
3082 
3083 	buf = uma_zalloc(namei_zone, M_WAITOK);
3084 	error = vn_getcwd(buf, &retbuf, &buflen);
3085 	if (error == 0)
3086 		error = copyout(retbuf, uap->buf, buflen);
3087 	uma_zfree(namei_zone, buf);
3088 	return (error);
3089 }
3090 
3091 int
3092 vn_getcwd(char *buf, char **retbuf, size_t *buflen)
3093 {
3094 	struct pwd *pwd;
3095 	int error;
3096 
3097 	vfs_smr_enter();
3098 	pwd = pwd_get_smr();
3099 	error = vn_fullpath_any_smr(pwd->pwd_cdir, pwd->pwd_rdir, buf, retbuf,
3100 	    buflen, 0);
3101 	VFS_SMR_ASSERT_NOT_ENTERED();
3102 	if (error < 0) {
3103 		pwd = pwd_hold(curthread);
3104 		error = vn_fullpath_any(pwd->pwd_cdir, pwd->pwd_rdir, buf,
3105 		    retbuf, buflen);
3106 		pwd_drop(pwd);
3107 	}
3108 
3109 #ifdef KTRACE
3110 	if (KTRPOINT(curthread, KTR_NAMEI) && error == 0)
3111 		ktrnamei(*retbuf);
3112 #endif
3113 	return (error);
3114 }
3115 
3116 static int
3117 kern___realpathat(struct thread *td, int fd, const char *path, char *buf,
3118     size_t size, int flags, enum uio_seg pathseg)
3119 {
3120 	struct nameidata nd;
3121 	char *retbuf, *freebuf;
3122 	int error;
3123 
3124 	if (flags != 0)
3125 		return (EINVAL);
3126 	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | SAVENAME | WANTPARENT | AUDITVNODE1,
3127 	    pathseg, path, fd, &cap_fstat_rights, td);
3128 	if ((error = namei(&nd)) != 0)
3129 		return (error);
3130 	error = vn_fullpath_hardlink(&nd, &retbuf, &freebuf, &size);
3131 	if (error == 0) {
3132 		error = copyout(retbuf, buf, size);
3133 		free(freebuf, M_TEMP);
3134 	}
3135 	NDFREE(&nd, 0);
3136 	return (error);
3137 }
3138 
3139 int
3140 sys___realpathat(struct thread *td, struct __realpathat_args *uap)
3141 {
3142 
3143 	return (kern___realpathat(td, uap->fd, uap->path, uap->buf, uap->size,
3144 	    uap->flags, UIO_USERSPACE));
3145 }
3146 
3147 /*
3148  * Retrieve the full filesystem path that correspond to a vnode from the name
3149  * cache (if available)
3150  */
3151 int
3152 vn_fullpath(struct vnode *vp, char **retbuf, char **freebuf)
3153 {
3154 	struct pwd *pwd;
3155 	char *buf;
3156 	size_t buflen;
3157 	int error;
3158 
3159 	if (__predict_false(vp == NULL))
3160 		return (EINVAL);
3161 
3162 	buflen = MAXPATHLEN;
3163 	buf = malloc(buflen, M_TEMP, M_WAITOK);
3164 	vfs_smr_enter();
3165 	pwd = pwd_get_smr();
3166 	error = vn_fullpath_any_smr(vp, pwd->pwd_rdir, buf, retbuf, &buflen, 0);
3167 	VFS_SMR_ASSERT_NOT_ENTERED();
3168 	if (error < 0) {
3169 		pwd = pwd_hold(curthread);
3170 		error = vn_fullpath_any(vp, pwd->pwd_rdir, buf, retbuf, &buflen);
3171 		pwd_drop(pwd);
3172 	}
3173 	if (error == 0)
3174 		*freebuf = buf;
3175 	else
3176 		free(buf, M_TEMP);
3177 	return (error);
3178 }
3179 
3180 /*
3181  * This function is similar to vn_fullpath, but it attempts to lookup the
3182  * pathname relative to the global root mount point.  This is required for the
3183  * auditing sub-system, as audited pathnames must be absolute, relative to the
3184  * global root mount point.
3185  */
3186 int
3187 vn_fullpath_global(struct vnode *vp, char **retbuf, char **freebuf)
3188 {
3189 	char *buf;
3190 	size_t buflen;
3191 	int error;
3192 
3193 	if (__predict_false(vp == NULL))
3194 		return (EINVAL);
3195 	buflen = MAXPATHLEN;
3196 	buf = malloc(buflen, M_TEMP, M_WAITOK);
3197 	vfs_smr_enter();
3198 	error = vn_fullpath_any_smr(vp, rootvnode, buf, retbuf, &buflen, 0);
3199 	VFS_SMR_ASSERT_NOT_ENTERED();
3200 	if (error < 0) {
3201 		error = vn_fullpath_any(vp, rootvnode, buf, retbuf, &buflen);
3202 	}
3203 	if (error == 0)
3204 		*freebuf = buf;
3205 	else
3206 		free(buf, M_TEMP);
3207 	return (error);
3208 }
3209 
3210 static struct namecache *
3211 vn_dd_from_dst(struct vnode *vp)
3212 {
3213 	struct namecache *ncp;
3214 
3215 	cache_assert_vnode_locked(vp);
3216 	TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) {
3217 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
3218 			return (ncp);
3219 	}
3220 	return (NULL);
3221 }
3222 
3223 int
3224 vn_vptocnp(struct vnode **vp, char *buf, size_t *buflen)
3225 {
3226 	struct vnode *dvp;
3227 	struct namecache *ncp;
3228 	struct mtx *vlp;
3229 	int error;
3230 
3231 	vlp = VP2VNODELOCK(*vp);
3232 	mtx_lock(vlp);
3233 	ncp = (*vp)->v_cache_dd;
3234 	if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT) == 0) {
3235 		KASSERT(ncp == vn_dd_from_dst(*vp),
3236 		    ("%s: mismatch for dd entry (%p != %p)", __func__,
3237 		    ncp, vn_dd_from_dst(*vp)));
3238 	} else {
3239 		ncp = vn_dd_from_dst(*vp);
3240 	}
3241 	if (ncp != NULL) {
3242 		if (*buflen < ncp->nc_nlen) {
3243 			mtx_unlock(vlp);
3244 			vrele(*vp);
3245 			counter_u64_add(numfullpathfail4, 1);
3246 			error = ENOMEM;
3247 			SDT_PROBE3(vfs, namecache, fullpath, return, error,
3248 			    vp, NULL);
3249 			return (error);
3250 		}
3251 		*buflen -= ncp->nc_nlen;
3252 		memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
3253 		SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
3254 		    ncp->nc_name, vp);
3255 		dvp = *vp;
3256 		*vp = ncp->nc_dvp;
3257 		vref(*vp);
3258 		mtx_unlock(vlp);
3259 		vrele(dvp);
3260 		return (0);
3261 	}
3262 	SDT_PROBE1(vfs, namecache, fullpath, miss, vp);
3263 
3264 	mtx_unlock(vlp);
3265 	vn_lock(*vp, LK_SHARED | LK_RETRY);
3266 	error = VOP_VPTOCNP(*vp, &dvp, buf, buflen);
3267 	vput(*vp);
3268 	if (error) {
3269 		counter_u64_add(numfullpathfail2, 1);
3270 		SDT_PROBE3(vfs, namecache, fullpath, return,  error, vp, NULL);
3271 		return (error);
3272 	}
3273 
3274 	*vp = dvp;
3275 	if (VN_IS_DOOMED(dvp)) {
3276 		/* forced unmount */
3277 		vrele(dvp);
3278 		error = ENOENT;
3279 		SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
3280 		return (error);
3281 	}
3282 	/*
3283 	 * *vp has its use count incremented still.
3284 	 */
3285 
3286 	return (0);
3287 }
3288 
3289 /*
3290  * Resolve a directory to a pathname.
3291  *
3292  * The name of the directory can always be found in the namecache or fetched
3293  * from the filesystem. There is also guaranteed to be only one parent, meaning
3294  * we can just follow vnodes up until we find the root.
3295  *
3296  * The vnode must be referenced.
3297  */
3298 static int
3299 vn_fullpath_dir(struct vnode *vp, struct vnode *rdir, char *buf, char **retbuf,
3300     size_t *len, size_t addend)
3301 {
3302 #ifdef KDTRACE_HOOKS
3303 	struct vnode *startvp = vp;
3304 #endif
3305 	struct vnode *vp1;
3306 	size_t buflen;
3307 	int error;
3308 	bool slash_prefixed;
3309 
3310 	VNPASS(vp->v_type == VDIR || VN_IS_DOOMED(vp), vp);
3311 	VNPASS(vp->v_usecount > 0, vp);
3312 
3313 	buflen = *len;
3314 
3315 	slash_prefixed = true;
3316 	if (addend == 0) {
3317 		MPASS(*len >= 2);
3318 		buflen--;
3319 		buf[buflen] = '\0';
3320 		slash_prefixed = false;
3321 	}
3322 
3323 	error = 0;
3324 
3325 	SDT_PROBE1(vfs, namecache, fullpath, entry, vp);
3326 	counter_u64_add(numfullpathcalls, 1);
3327 	while (vp != rdir && vp != rootvnode) {
3328 		/*
3329 		 * The vp vnode must be already fully constructed,
3330 		 * since it is either found in namecache or obtained
3331 		 * from VOP_VPTOCNP().  We may test for VV_ROOT safely
3332 		 * without obtaining the vnode lock.
3333 		 */
3334 		if ((vp->v_vflag & VV_ROOT) != 0) {
3335 			vn_lock(vp, LK_RETRY | LK_SHARED);
3336 
3337 			/*
3338 			 * With the vnode locked, check for races with
3339 			 * unmount, forced or not.  Note that we
3340 			 * already verified that vp is not equal to
3341 			 * the root vnode, which means that
3342 			 * mnt_vnodecovered can be NULL only for the
3343 			 * case of unmount.
3344 			 */
3345 			if (VN_IS_DOOMED(vp) ||
3346 			    (vp1 = vp->v_mount->mnt_vnodecovered) == NULL ||
3347 			    vp1->v_mountedhere != vp->v_mount) {
3348 				vput(vp);
3349 				error = ENOENT;
3350 				SDT_PROBE3(vfs, namecache, fullpath, return,
3351 				    error, vp, NULL);
3352 				break;
3353 			}
3354 
3355 			vref(vp1);
3356 			vput(vp);
3357 			vp = vp1;
3358 			continue;
3359 		}
3360 		if (vp->v_type != VDIR) {
3361 			vrele(vp);
3362 			counter_u64_add(numfullpathfail1, 1);
3363 			error = ENOTDIR;
3364 			SDT_PROBE3(vfs, namecache, fullpath, return,
3365 			    error, vp, NULL);
3366 			break;
3367 		}
3368 		error = vn_vptocnp(&vp, buf, &buflen);
3369 		if (error)
3370 			break;
3371 		if (buflen == 0) {
3372 			vrele(vp);
3373 			error = ENOMEM;
3374 			SDT_PROBE3(vfs, namecache, fullpath, return, error,
3375 			    startvp, NULL);
3376 			break;
3377 		}
3378 		buf[--buflen] = '/';
3379 		slash_prefixed = true;
3380 	}
3381 	if (error)
3382 		return (error);
3383 	if (!slash_prefixed) {
3384 		if (buflen == 0) {
3385 			vrele(vp);
3386 			counter_u64_add(numfullpathfail4, 1);
3387 			SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM,
3388 			    startvp, NULL);
3389 			return (ENOMEM);
3390 		}
3391 		buf[--buflen] = '/';
3392 	}
3393 	counter_u64_add(numfullpathfound, 1);
3394 	vrele(vp);
3395 
3396 	*retbuf = buf + buflen;
3397 	SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, *retbuf);
3398 	*len -= buflen;
3399 	*len += addend;
3400 	return (0);
3401 }
3402 
3403 /*
3404  * Resolve an arbitrary vnode to a pathname.
3405  *
3406  * Note 2 caveats:
3407  * - hardlinks are not tracked, thus if the vnode is not a directory this can
3408  *   resolve to a different path than the one used to find it
3409  * - namecache is not mandatory, meaning names are not guaranteed to be added
3410  *   (in which case resolving fails)
3411  */
3412 static void __inline
3413 cache_rev_failed_impl(int *reason, int line)
3414 {
3415 
3416 	*reason = line;
3417 }
3418 #define cache_rev_failed(var)	cache_rev_failed_impl((var), __LINE__)
3419 
3420 static int
3421 vn_fullpath_any_smr(struct vnode *vp, struct vnode *rdir, char *buf,
3422     char **retbuf, size_t *buflen, size_t addend)
3423 {
3424 #ifdef KDTRACE_HOOKS
3425 	struct vnode *startvp = vp;
3426 #endif
3427 	struct vnode *tvp;
3428 	struct mount *mp;
3429 	struct namecache *ncp;
3430 	size_t orig_buflen;
3431 	int reason;
3432 	int error;
3433 #ifdef KDTRACE_HOOKS
3434 	int i;
3435 #endif
3436 	seqc_t vp_seqc, tvp_seqc;
3437 	u_char nc_flag;
3438 
3439 	VFS_SMR_ASSERT_ENTERED();
3440 
3441 	if (!atomic_load_char(&cache_fast_lookup_enabled)) {
3442 		vfs_smr_exit();
3443 		return (-1);
3444 	}
3445 
3446 	orig_buflen = *buflen;
3447 
3448 	if (addend == 0) {
3449 		MPASS(*buflen >= 2);
3450 		*buflen -= 1;
3451 		buf[*buflen] = '\0';
3452 	}
3453 
3454 	if (vp == rdir || vp == rootvnode) {
3455 		if (addend == 0) {
3456 			*buflen -= 1;
3457 			buf[*buflen] = '/';
3458 		}
3459 		goto out_ok;
3460 	}
3461 
3462 #ifdef KDTRACE_HOOKS
3463 	i = 0;
3464 #endif
3465 	error = -1;
3466 	ncp = NULL; /* for sdt probe down below */
3467 	vp_seqc = vn_seqc_read_any(vp);
3468 	if (seqc_in_modify(vp_seqc)) {
3469 		cache_rev_failed(&reason);
3470 		goto out_abort;
3471 	}
3472 
3473 	for (;;) {
3474 #ifdef KDTRACE_HOOKS
3475 		i++;
3476 #endif
3477 		if ((vp->v_vflag & VV_ROOT) != 0) {
3478 			mp = atomic_load_ptr(&vp->v_mount);
3479 			if (mp == NULL) {
3480 				cache_rev_failed(&reason);
3481 				goto out_abort;
3482 			}
3483 			tvp = atomic_load_ptr(&mp->mnt_vnodecovered);
3484 			tvp_seqc = vn_seqc_read_any(tvp);
3485 			if (seqc_in_modify(tvp_seqc)) {
3486 				cache_rev_failed(&reason);
3487 				goto out_abort;
3488 			}
3489 			if (!vn_seqc_consistent(vp, vp_seqc)) {
3490 				cache_rev_failed(&reason);
3491 				goto out_abort;
3492 			}
3493 			vp = tvp;
3494 			vp_seqc = tvp_seqc;
3495 			continue;
3496 		}
3497 		ncp = atomic_load_consume_ptr(&vp->v_cache_dd);
3498 		if (ncp == NULL) {
3499 			cache_rev_failed(&reason);
3500 			goto out_abort;
3501 		}
3502 		nc_flag = atomic_load_char(&ncp->nc_flag);
3503 		if ((nc_flag & NCF_ISDOTDOT) != 0) {
3504 			cache_rev_failed(&reason);
3505 			goto out_abort;
3506 		}
3507 		if (ncp->nc_nlen >= *buflen) {
3508 			cache_rev_failed(&reason);
3509 			error = ENOMEM;
3510 			goto out_abort;
3511 		}
3512 		*buflen -= ncp->nc_nlen;
3513 		memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
3514 		*buflen -= 1;
3515 		buf[*buflen] = '/';
3516 		tvp = ncp->nc_dvp;
3517 		tvp_seqc = vn_seqc_read_any(tvp);
3518 		if (seqc_in_modify(tvp_seqc)) {
3519 			cache_rev_failed(&reason);
3520 			goto out_abort;
3521 		}
3522 		if (!vn_seqc_consistent(vp, vp_seqc)) {
3523 			cache_rev_failed(&reason);
3524 			goto out_abort;
3525 		}
3526 		/*
3527 		 * Acquire fence provided by vn_seqc_read_any above.
3528 		 */
3529 		if (__predict_false(atomic_load_ptr(&vp->v_cache_dd) != ncp)) {
3530 			cache_rev_failed(&reason);
3531 			goto out_abort;
3532 		}
3533 		if (!cache_ncp_canuse(ncp)) {
3534 			cache_rev_failed(&reason);
3535 			goto out_abort;
3536 		}
3537 		vp = tvp;
3538 		vp_seqc = tvp_seqc;
3539 		if (vp == rdir || vp == rootvnode)
3540 			break;
3541 	}
3542 out_ok:
3543 	vfs_smr_exit();
3544 	*retbuf = buf + *buflen;
3545 	*buflen = orig_buflen - *buflen + addend;
3546 	SDT_PROBE2(vfs, namecache, fullpath_smr, hit, startvp, *retbuf);
3547 	return (0);
3548 
3549 out_abort:
3550 	*buflen = orig_buflen;
3551 	SDT_PROBE4(vfs, namecache, fullpath_smr, miss, startvp, ncp, reason, i);
3552 	vfs_smr_exit();
3553 	return (error);
3554 }
3555 
3556 static int
3557 vn_fullpath_any(struct vnode *vp, struct vnode *rdir, char *buf, char **retbuf,
3558     size_t *buflen)
3559 {
3560 	size_t orig_buflen, addend;
3561 	int error;
3562 
3563 	if (*buflen < 2)
3564 		return (EINVAL);
3565 
3566 	orig_buflen = *buflen;
3567 
3568 	vref(vp);
3569 	addend = 0;
3570 	if (vp->v_type != VDIR) {
3571 		*buflen -= 1;
3572 		buf[*buflen] = '\0';
3573 		error = vn_vptocnp(&vp, buf, buflen);
3574 		if (error)
3575 			return (error);
3576 		if (*buflen == 0) {
3577 			vrele(vp);
3578 			return (ENOMEM);
3579 		}
3580 		*buflen -= 1;
3581 		buf[*buflen] = '/';
3582 		addend = orig_buflen - *buflen;
3583 	}
3584 
3585 	return (vn_fullpath_dir(vp, rdir, buf, retbuf, buflen, addend));
3586 }
3587 
3588 /*
3589  * Resolve an arbitrary vnode to a pathname (taking care of hardlinks).
3590  *
3591  * Since the namecache does not track hardlinks, the caller is expected to first
3592  * look up the target vnode with SAVENAME | WANTPARENT flags passed to namei.
3593  *
3594  * Then we have 2 cases:
3595  * - if the found vnode is a directory, the path can be constructed just by
3596  *   following names up the chain
3597  * - otherwise we populate the buffer with the saved name and start resolving
3598  *   from the parent
3599  */
3600 static int
3601 vn_fullpath_hardlink(struct nameidata *ndp, char **retbuf, char **freebuf,
3602     size_t *buflen)
3603 {
3604 	char *buf, *tmpbuf;
3605 	struct pwd *pwd;
3606 	struct componentname *cnp;
3607 	struct vnode *vp;
3608 	size_t addend;
3609 	int error;
3610 	enum vtype type;
3611 
3612 	if (*buflen < 2)
3613 		return (EINVAL);
3614 	if (*buflen > MAXPATHLEN)
3615 		*buflen = MAXPATHLEN;
3616 
3617 	buf = malloc(*buflen, M_TEMP, M_WAITOK);
3618 
3619 	addend = 0;
3620 	vp = ndp->ni_vp;
3621 	/*
3622 	 * Check for VBAD to work around the vp_crossmp bug in lookup().
3623 	 *
3624 	 * For example consider tmpfs on /tmp and realpath /tmp. ni_vp will be
3625 	 * set to mount point's root vnode while ni_dvp will be vp_crossmp.
3626 	 * If the type is VDIR (like in this very case) we can skip looking
3627 	 * at ni_dvp in the first place. However, since vnodes get passed here
3628 	 * unlocked the target may transition to doomed state (type == VBAD)
3629 	 * before we get to evaluate the condition. If this happens, we will
3630 	 * populate part of the buffer and descend to vn_fullpath_dir with
3631 	 * vp == vp_crossmp. Prevent the problem by checking for VBAD.
3632 	 *
3633 	 * This should be atomic_load(&vp->v_type) but it is illegal to take
3634 	 * an address of a bit field, even if said field is sized to char.
3635 	 * Work around the problem by reading the value into a full-sized enum
3636 	 * and then re-reading it with atomic_load which will still prevent
3637 	 * the compiler from re-reading down the road.
3638 	 */
3639 	type = vp->v_type;
3640 	type = atomic_load_int(&type);
3641 	if (type == VBAD) {
3642 		error = ENOENT;
3643 		goto out_bad;
3644 	}
3645 	if (type != VDIR) {
3646 		cnp = &ndp->ni_cnd;
3647 		addend = cnp->cn_namelen + 2;
3648 		if (*buflen < addend) {
3649 			error = ENOMEM;
3650 			goto out_bad;
3651 		}
3652 		*buflen -= addend;
3653 		tmpbuf = buf + *buflen;
3654 		tmpbuf[0] = '/';
3655 		memcpy(&tmpbuf[1], cnp->cn_nameptr, cnp->cn_namelen);
3656 		tmpbuf[addend - 1] = '\0';
3657 		vp = ndp->ni_dvp;
3658 	}
3659 
3660 	vfs_smr_enter();
3661 	pwd = pwd_get_smr();
3662 	error = vn_fullpath_any_smr(vp, pwd->pwd_rdir, buf, retbuf, buflen,
3663 	    addend);
3664 	VFS_SMR_ASSERT_NOT_ENTERED();
3665 	if (error < 0) {
3666 		pwd = pwd_hold(curthread);
3667 		vref(vp);
3668 		error = vn_fullpath_dir(vp, pwd->pwd_rdir, buf, retbuf, buflen,
3669 		    addend);
3670 		pwd_drop(pwd);
3671 	}
3672 	if (error != 0)
3673 		goto out_bad;
3674 
3675 	*freebuf = buf;
3676 
3677 	return (0);
3678 out_bad:
3679 	free(buf, M_TEMP);
3680 	return (error);
3681 }
3682 
3683 struct vnode *
3684 vn_dir_dd_ino(struct vnode *vp)
3685 {
3686 	struct namecache *ncp;
3687 	struct vnode *ddvp;
3688 	struct mtx *vlp;
3689 	enum vgetstate vs;
3690 
3691 	ASSERT_VOP_LOCKED(vp, "vn_dir_dd_ino");
3692 	vlp = VP2VNODELOCK(vp);
3693 	mtx_lock(vlp);
3694 	TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
3695 		if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
3696 			continue;
3697 		ddvp = ncp->nc_dvp;
3698 		vs = vget_prep(ddvp);
3699 		mtx_unlock(vlp);
3700 		if (vget_finish(ddvp, LK_SHARED | LK_NOWAIT, vs))
3701 			return (NULL);
3702 		return (ddvp);
3703 	}
3704 	mtx_unlock(vlp);
3705 	return (NULL);
3706 }
3707 
3708 int
3709 vn_commname(struct vnode *vp, char *buf, u_int buflen)
3710 {
3711 	struct namecache *ncp;
3712 	struct mtx *vlp;
3713 	int l;
3714 
3715 	vlp = VP2VNODELOCK(vp);
3716 	mtx_lock(vlp);
3717 	TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
3718 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
3719 			break;
3720 	if (ncp == NULL) {
3721 		mtx_unlock(vlp);
3722 		return (ENOENT);
3723 	}
3724 	l = min(ncp->nc_nlen, buflen - 1);
3725 	memcpy(buf, ncp->nc_name, l);
3726 	mtx_unlock(vlp);
3727 	buf[l] = '\0';
3728 	return (0);
3729 }
3730 
3731 /*
3732  * This function updates path string to vnode's full global path
3733  * and checks the size of the new path string against the pathlen argument.
3734  *
3735  * Requires a locked, referenced vnode.
3736  * Vnode is re-locked on success or ENODEV, otherwise unlocked.
3737  *
3738  * If vp is a directory, the call to vn_fullpath_global() always succeeds
3739  * because it falls back to the ".." lookup if the namecache lookup fails.
3740  */
3741 int
3742 vn_path_to_global_path(struct thread *td, struct vnode *vp, char *path,
3743     u_int pathlen)
3744 {
3745 	struct nameidata nd;
3746 	struct vnode *vp1;
3747 	char *rpath, *fbuf;
3748 	int error;
3749 
3750 	ASSERT_VOP_ELOCKED(vp, __func__);
3751 
3752 	/* Construct global filesystem path from vp. */
3753 	VOP_UNLOCK(vp);
3754 	error = vn_fullpath_global(vp, &rpath, &fbuf);
3755 
3756 	if (error != 0) {
3757 		vrele(vp);
3758 		return (error);
3759 	}
3760 
3761 	if (strlen(rpath) >= pathlen) {
3762 		vrele(vp);
3763 		error = ENAMETOOLONG;
3764 		goto out;
3765 	}
3766 
3767 	/*
3768 	 * Re-lookup the vnode by path to detect a possible rename.
3769 	 * As a side effect, the vnode is relocked.
3770 	 * If vnode was renamed, return ENOENT.
3771 	 */
3772 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
3773 	    UIO_SYSSPACE, path, td);
3774 	error = namei(&nd);
3775 	if (error != 0) {
3776 		vrele(vp);
3777 		goto out;
3778 	}
3779 	NDFREE(&nd, NDF_ONLY_PNBUF);
3780 	vp1 = nd.ni_vp;
3781 	vrele(vp);
3782 	if (vp1 == vp)
3783 		strcpy(path, rpath);
3784 	else {
3785 		vput(vp1);
3786 		error = ENOENT;
3787 	}
3788 
3789 out:
3790 	free(fbuf, M_TEMP);
3791 	return (error);
3792 }
3793 
3794 #ifdef DDB
3795 static void
3796 db_print_vpath(struct vnode *vp)
3797 {
3798 
3799 	while (vp != NULL) {
3800 		db_printf("%p: ", vp);
3801 		if (vp == rootvnode) {
3802 			db_printf("/");
3803 			vp = NULL;
3804 		} else {
3805 			if (vp->v_vflag & VV_ROOT) {
3806 				db_printf("<mount point>");
3807 				vp = vp->v_mount->mnt_vnodecovered;
3808 			} else {
3809 				struct namecache *ncp;
3810 				char *ncn;
3811 				int i;
3812 
3813 				ncp = TAILQ_FIRST(&vp->v_cache_dst);
3814 				if (ncp != NULL) {
3815 					ncn = ncp->nc_name;
3816 					for (i = 0; i < ncp->nc_nlen; i++)
3817 						db_printf("%c", *ncn++);
3818 					vp = ncp->nc_dvp;
3819 				} else {
3820 					vp = NULL;
3821 				}
3822 			}
3823 		}
3824 		db_printf("\n");
3825 	}
3826 
3827 	return;
3828 }
3829 
3830 DB_SHOW_COMMAND(vpath, db_show_vpath)
3831 {
3832 	struct vnode *vp;
3833 
3834 	if (!have_addr) {
3835 		db_printf("usage: show vpath <struct vnode *>\n");
3836 		return;
3837 	}
3838 
3839 	vp = (struct vnode *)addr;
3840 	db_print_vpath(vp);
3841 }
3842 
3843 #endif
3844 
3845 static int cache_fast_lookup = 1;
3846 
3847 #define CACHE_FPL_FAILED	-2020
3848 
3849 void
3850 cache_fast_lookup_enabled_recalc(void)
3851 {
3852 	int lookup_flag;
3853 	int mac_on;
3854 
3855 #ifdef MAC
3856 	mac_on = mac_vnode_check_lookup_enabled();
3857 	mac_on |= mac_vnode_check_readlink_enabled();
3858 #else
3859 	mac_on = 0;
3860 #endif
3861 
3862 	lookup_flag = atomic_load_int(&cache_fast_lookup);
3863 	if (lookup_flag && !mac_on) {
3864 		atomic_store_char(&cache_fast_lookup_enabled, true);
3865 	} else {
3866 		atomic_store_char(&cache_fast_lookup_enabled, false);
3867 	}
3868 }
3869 
3870 static int
3871 syscal_vfs_cache_fast_lookup(SYSCTL_HANDLER_ARGS)
3872 {
3873 	int error, old;
3874 
3875 	old = atomic_load_int(&cache_fast_lookup);
3876 	error = sysctl_handle_int(oidp, arg1, arg2, req);
3877 	if (error == 0 && req->newptr && old != atomic_load_int(&cache_fast_lookup))
3878 		cache_fast_lookup_enabled_recalc();
3879 	return (error);
3880 }
3881 SYSCTL_PROC(_vfs, OID_AUTO, cache_fast_lookup, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE,
3882     &cache_fast_lookup, 0, syscal_vfs_cache_fast_lookup, "IU", "");
3883 
3884 /*
3885  * Components of nameidata (or objects it can point to) which may
3886  * need restoring in case fast path lookup fails.
3887  */
3888 struct nameidata_outer {
3889 	size_t ni_pathlen;
3890 	int cn_flags;
3891 };
3892 
3893 struct nameidata_saved {
3894 #ifdef INVARIANTS
3895 	char *cn_nameptr;
3896 	size_t ni_pathlen;
3897 #endif
3898 };
3899 
3900 #ifdef INVARIANTS
3901 struct cache_fpl_debug {
3902 	size_t ni_pathlen;
3903 };
3904 #endif
3905 
3906 struct cache_fpl {
3907 	struct nameidata *ndp;
3908 	struct componentname *cnp;
3909 	char *nulchar;
3910 	struct vnode *dvp;
3911 	struct vnode *tvp;
3912 	seqc_t dvp_seqc;
3913 	seqc_t tvp_seqc;
3914 	uint32_t hash;
3915 	struct nameidata_saved snd;
3916 	struct nameidata_outer snd_outer;
3917 	int line;
3918 	enum cache_fpl_status status:8;
3919 	bool in_smr;
3920 	bool fsearch;
3921 	bool savename;
3922 	struct pwd **pwd;
3923 #ifdef INVARIANTS
3924 	struct cache_fpl_debug debug;
3925 #endif
3926 };
3927 
3928 static bool cache_fplookup_mp_supported(struct mount *mp);
3929 static bool cache_fplookup_is_mp(struct cache_fpl *fpl);
3930 static int cache_fplookup_cross_mount(struct cache_fpl *fpl);
3931 static int cache_fplookup_partial_setup(struct cache_fpl *fpl);
3932 static int cache_fplookup_skip_slashes(struct cache_fpl *fpl);
3933 static int cache_fplookup_trailingslash(struct cache_fpl *fpl);
3934 static void cache_fpl_pathlen_dec(struct cache_fpl *fpl);
3935 static void cache_fpl_pathlen_inc(struct cache_fpl *fpl);
3936 static void cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n);
3937 static void cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n);
3938 
3939 static void
3940 cache_fpl_cleanup_cnp(struct componentname *cnp)
3941 {
3942 
3943 	uma_zfree(namei_zone, cnp->cn_pnbuf);
3944 #ifdef DIAGNOSTIC
3945 	cnp->cn_pnbuf = NULL;
3946 	cnp->cn_nameptr = NULL;
3947 #endif
3948 }
3949 
3950 static struct vnode *
3951 cache_fpl_handle_root(struct cache_fpl *fpl)
3952 {
3953 	struct nameidata *ndp;
3954 	struct componentname *cnp;
3955 
3956 	ndp = fpl->ndp;
3957 	cnp = fpl->cnp;
3958 
3959 	MPASS(*(cnp->cn_nameptr) == '/');
3960 	cnp->cn_nameptr++;
3961 	cache_fpl_pathlen_dec(fpl);
3962 
3963 	if (__predict_false(*(cnp->cn_nameptr) == '/')) {
3964 		do {
3965 			cnp->cn_nameptr++;
3966 			cache_fpl_pathlen_dec(fpl);
3967 		} while (*(cnp->cn_nameptr) == '/');
3968 	}
3969 
3970 	return (ndp->ni_rootdir);
3971 }
3972 
3973 static void
3974 cache_fpl_checkpoint_outer(struct cache_fpl *fpl)
3975 {
3976 
3977 	fpl->snd_outer.ni_pathlen = fpl->ndp->ni_pathlen;
3978 	fpl->snd_outer.cn_flags = fpl->ndp->ni_cnd.cn_flags;
3979 }
3980 
3981 static void
3982 cache_fpl_checkpoint(struct cache_fpl *fpl)
3983 {
3984 
3985 #ifdef INVARIANTS
3986 	fpl->snd.cn_nameptr = fpl->ndp->ni_cnd.cn_nameptr;
3987 	fpl->snd.ni_pathlen = fpl->debug.ni_pathlen;
3988 #endif
3989 }
3990 
3991 static void
3992 cache_fpl_restore_partial(struct cache_fpl *fpl)
3993 {
3994 
3995 	fpl->ndp->ni_cnd.cn_flags = fpl->snd_outer.cn_flags;
3996 #ifdef INVARIANTS
3997 	fpl->debug.ni_pathlen = fpl->snd.ni_pathlen;
3998 #endif
3999 }
4000 
4001 static void
4002 cache_fpl_restore_abort(struct cache_fpl *fpl)
4003 {
4004 
4005 	cache_fpl_restore_partial(fpl);
4006 	/*
4007 	 * It is 0 on entry by API contract.
4008 	 */
4009 	fpl->ndp->ni_resflags = 0;
4010 	fpl->ndp->ni_cnd.cn_nameptr = fpl->ndp->ni_cnd.cn_pnbuf;
4011 	fpl->ndp->ni_pathlen = fpl->snd_outer.ni_pathlen;
4012 }
4013 
4014 #ifdef INVARIANTS
4015 #define cache_fpl_smr_assert_entered(fpl) ({			\
4016 	struct cache_fpl *_fpl = (fpl);				\
4017 	MPASS(_fpl->in_smr == true);				\
4018 	VFS_SMR_ASSERT_ENTERED();				\
4019 })
4020 #define cache_fpl_smr_assert_not_entered(fpl) ({		\
4021 	struct cache_fpl *_fpl = (fpl);				\
4022 	MPASS(_fpl->in_smr == false);				\
4023 	VFS_SMR_ASSERT_NOT_ENTERED();				\
4024 })
4025 static void
4026 cache_fpl_assert_status(struct cache_fpl *fpl)
4027 {
4028 
4029 	switch (fpl->status) {
4030 	case CACHE_FPL_STATUS_UNSET:
4031 		__assert_unreachable();
4032 		break;
4033 	case CACHE_FPL_STATUS_DESTROYED:
4034 	case CACHE_FPL_STATUS_ABORTED:
4035 	case CACHE_FPL_STATUS_PARTIAL:
4036 	case CACHE_FPL_STATUS_HANDLED:
4037 		break;
4038 	}
4039 }
4040 #else
4041 #define cache_fpl_smr_assert_entered(fpl) do { } while (0)
4042 #define cache_fpl_smr_assert_not_entered(fpl) do { } while (0)
4043 #define cache_fpl_assert_status(fpl) do { } while (0)
4044 #endif
4045 
4046 #define cache_fpl_smr_enter_initial(fpl) ({			\
4047 	struct cache_fpl *_fpl = (fpl);				\
4048 	vfs_smr_enter();					\
4049 	_fpl->in_smr = true;					\
4050 })
4051 
4052 #define cache_fpl_smr_enter(fpl) ({				\
4053 	struct cache_fpl *_fpl = (fpl);				\
4054 	MPASS(_fpl->in_smr == false);				\
4055 	vfs_smr_enter();					\
4056 	_fpl->in_smr = true;					\
4057 })
4058 
4059 #define cache_fpl_smr_exit(fpl) ({				\
4060 	struct cache_fpl *_fpl = (fpl);				\
4061 	MPASS(_fpl->in_smr == true);				\
4062 	vfs_smr_exit();						\
4063 	_fpl->in_smr = false;					\
4064 })
4065 
4066 static int
4067 cache_fpl_aborted_early_impl(struct cache_fpl *fpl, int line)
4068 {
4069 
4070 	if (fpl->status != CACHE_FPL_STATUS_UNSET) {
4071 		KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
4072 		    ("%s: converting to abort from %d at %d, set at %d\n",
4073 		    __func__, fpl->status, line, fpl->line));
4074 	}
4075 	cache_fpl_smr_assert_not_entered(fpl);
4076 	fpl->status = CACHE_FPL_STATUS_ABORTED;
4077 	fpl->line = line;
4078 	return (CACHE_FPL_FAILED);
4079 }
4080 
4081 #define cache_fpl_aborted_early(x)	cache_fpl_aborted_early_impl((x), __LINE__)
4082 
4083 static int __noinline
4084 cache_fpl_aborted_impl(struct cache_fpl *fpl, int line)
4085 {
4086 	struct nameidata *ndp;
4087 	struct componentname *cnp;
4088 
4089 	ndp = fpl->ndp;
4090 	cnp = fpl->cnp;
4091 
4092 	if (fpl->status != CACHE_FPL_STATUS_UNSET) {
4093 		KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
4094 		    ("%s: converting to abort from %d at %d, set at %d\n",
4095 		    __func__, fpl->status, line, fpl->line));
4096 	}
4097 	fpl->status = CACHE_FPL_STATUS_ABORTED;
4098 	fpl->line = line;
4099 	if (fpl->in_smr)
4100 		cache_fpl_smr_exit(fpl);
4101 	cache_fpl_restore_abort(fpl);
4102 	/*
4103 	 * Resolving symlinks overwrites data passed by the caller.
4104 	 * Let namei know.
4105 	 */
4106 	if (ndp->ni_loopcnt > 0) {
4107 		fpl->status = CACHE_FPL_STATUS_DESTROYED;
4108 		cache_fpl_cleanup_cnp(cnp);
4109 	}
4110 	return (CACHE_FPL_FAILED);
4111 }
4112 
4113 #define cache_fpl_aborted(x)	cache_fpl_aborted_impl((x), __LINE__)
4114 
4115 static int __noinline
4116 cache_fpl_partial_impl(struct cache_fpl *fpl, int line)
4117 {
4118 
4119 	KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
4120 	    ("%s: setting to partial at %d, but already set to %d at %d\n",
4121 	    __func__, line, fpl->status, fpl->line));
4122 	cache_fpl_smr_assert_entered(fpl);
4123 	fpl->status = CACHE_FPL_STATUS_PARTIAL;
4124 	fpl->line = line;
4125 	return (cache_fplookup_partial_setup(fpl));
4126 }
4127 
4128 #define cache_fpl_partial(x)	cache_fpl_partial_impl((x), __LINE__)
4129 
4130 static int
4131 cache_fpl_handled_impl(struct cache_fpl *fpl, int line)
4132 {
4133 
4134 	KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
4135 	    ("%s: setting to handled at %d, but already set to %d at %d\n",
4136 	    __func__, line, fpl->status, fpl->line));
4137 	cache_fpl_smr_assert_not_entered(fpl);
4138 	fpl->status = CACHE_FPL_STATUS_HANDLED;
4139 	fpl->line = line;
4140 	return (0);
4141 }
4142 
4143 #define cache_fpl_handled(x)	cache_fpl_handled_impl((x), __LINE__)
4144 
4145 static int
4146 cache_fpl_handled_error_impl(struct cache_fpl *fpl, int error, int line)
4147 {
4148 
4149 	KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
4150 	    ("%s: setting to handled at %d, but already set to %d at %d\n",
4151 	    __func__, line, fpl->status, fpl->line));
4152 	MPASS(error != 0);
4153 	MPASS(error != CACHE_FPL_FAILED);
4154 	cache_fpl_smr_assert_not_entered(fpl);
4155 	fpl->status = CACHE_FPL_STATUS_HANDLED;
4156 	fpl->line = line;
4157 	fpl->dvp = NULL;
4158 	fpl->tvp = NULL;
4159 	fpl->savename = false;
4160 	return (error);
4161 }
4162 
4163 #define cache_fpl_handled_error(x, e)	cache_fpl_handled_error_impl((x), (e), __LINE__)
4164 
4165 static bool
4166 cache_fpl_terminated(struct cache_fpl *fpl)
4167 {
4168 
4169 	return (fpl->status != CACHE_FPL_STATUS_UNSET);
4170 }
4171 
4172 #define CACHE_FPL_SUPPORTED_CN_FLAGS \
4173 	(NC_NOMAKEENTRY | NC_KEEPPOSENTRY | LOCKLEAF | LOCKPARENT | WANTPARENT | \
4174 	 FAILIFEXISTS | FOLLOW | LOCKSHARED | SAVENAME | SAVESTART | WILLBEDIR | \
4175 	 ISOPEN | NOMACCHECK | AUDITVNODE1 | AUDITVNODE2 | NOCAPCHECK | OPENREAD | \
4176 	 OPENWRITE)
4177 
4178 #define CACHE_FPL_INTERNAL_CN_FLAGS \
4179 	(ISDOTDOT | MAKEENTRY | ISLASTCN)
4180 
4181 _Static_assert((CACHE_FPL_SUPPORTED_CN_FLAGS & CACHE_FPL_INTERNAL_CN_FLAGS) == 0,
4182     "supported and internal flags overlap");
4183 
4184 static bool
4185 cache_fpl_islastcn(struct nameidata *ndp)
4186 {
4187 
4188 	return (*ndp->ni_next == 0);
4189 }
4190 
4191 static bool
4192 cache_fpl_istrailingslash(struct cache_fpl *fpl)
4193 {
4194 
4195 	return (*(fpl->nulchar - 1) == '/');
4196 }
4197 
4198 static bool
4199 cache_fpl_isdotdot(struct componentname *cnp)
4200 {
4201 
4202 	if (cnp->cn_namelen == 2 &&
4203 	    cnp->cn_nameptr[1] == '.' && cnp->cn_nameptr[0] == '.')
4204 		return (true);
4205 	return (false);
4206 }
4207 
4208 static bool
4209 cache_can_fplookup(struct cache_fpl *fpl)
4210 {
4211 	struct nameidata *ndp;
4212 	struct componentname *cnp;
4213 	struct thread *td;
4214 
4215 	ndp = fpl->ndp;
4216 	cnp = fpl->cnp;
4217 	td = cnp->cn_thread;
4218 
4219 	if (!atomic_load_char(&cache_fast_lookup_enabled)) {
4220 		cache_fpl_aborted_early(fpl);
4221 		return (false);
4222 	}
4223 	if ((cnp->cn_flags & ~CACHE_FPL_SUPPORTED_CN_FLAGS) != 0) {
4224 		cache_fpl_aborted_early(fpl);
4225 		return (false);
4226 	}
4227 	if (IN_CAPABILITY_MODE(td)) {
4228 		cache_fpl_aborted_early(fpl);
4229 		return (false);
4230 	}
4231 	if (AUDITING_TD(td)) {
4232 		cache_fpl_aborted_early(fpl);
4233 		return (false);
4234 	}
4235 	if (ndp->ni_startdir != NULL) {
4236 		cache_fpl_aborted_early(fpl);
4237 		return (false);
4238 	}
4239 	return (true);
4240 }
4241 
4242 static int
4243 cache_fplookup_dirfd(struct cache_fpl *fpl, struct vnode **vpp)
4244 {
4245 	struct nameidata *ndp;
4246 	int error;
4247 	bool fsearch;
4248 
4249 	ndp = fpl->ndp;
4250 	error = fgetvp_lookup_smr(ndp->ni_dirfd, ndp, vpp, &fsearch);
4251 	if (__predict_false(error != 0)) {
4252 		return (cache_fpl_aborted(fpl));
4253 	}
4254 	fpl->fsearch = fsearch;
4255 	return (0);
4256 }
4257 
4258 static int __noinline
4259 cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp,
4260     uint32_t hash)
4261 {
4262 	struct componentname *cnp;
4263 	struct vnode *dvp;
4264 
4265 	cnp = fpl->cnp;
4266 	dvp = fpl->dvp;
4267 
4268 	cache_fpl_smr_exit(fpl);
4269 	if (cache_neg_promote_cond(dvp, cnp, oncp, hash))
4270 		return (cache_fpl_handled_error(fpl, ENOENT));
4271 	else
4272 		return (cache_fpl_aborted(fpl));
4273 }
4274 
4275 /*
4276  * The target vnode is not supported, prepare for the slow path to take over.
4277  */
4278 static int __noinline
4279 cache_fplookup_partial_setup(struct cache_fpl *fpl)
4280 {
4281 	struct nameidata *ndp;
4282 	struct componentname *cnp;
4283 	enum vgetstate dvs;
4284 	struct vnode *dvp;
4285 	struct pwd *pwd;
4286 	seqc_t dvp_seqc;
4287 
4288 	ndp = fpl->ndp;
4289 	cnp = fpl->cnp;
4290 	pwd = *(fpl->pwd);
4291 	dvp = fpl->dvp;
4292 	dvp_seqc = fpl->dvp_seqc;
4293 
4294 	if (!pwd_hold_smr(pwd)) {
4295 		return (cache_fpl_aborted(fpl));
4296 	}
4297 
4298 	/*
4299 	 * Note that seqc is checked before the vnode is locked, so by
4300 	 * the time regular lookup gets to it it may have moved.
4301 	 *
4302 	 * Ultimately this does not affect correctness, any lookup errors
4303 	 * are userspace racing with itself. It is guaranteed that any
4304 	 * path which ultimately gets found could also have been found
4305 	 * by regular lookup going all the way in absence of concurrent
4306 	 * modifications.
4307 	 */
4308 	dvs = vget_prep_smr(dvp);
4309 	cache_fpl_smr_exit(fpl);
4310 	if (__predict_false(dvs == VGET_NONE)) {
4311 		pwd_drop(pwd);
4312 		return (cache_fpl_aborted(fpl));
4313 	}
4314 
4315 	vget_finish_ref(dvp, dvs);
4316 	if (!vn_seqc_consistent(dvp, dvp_seqc)) {
4317 		vrele(dvp);
4318 		pwd_drop(pwd);
4319 		return (cache_fpl_aborted(fpl));
4320 	}
4321 
4322 	cache_fpl_restore_partial(fpl);
4323 #ifdef INVARIANTS
4324 	if (cnp->cn_nameptr != fpl->snd.cn_nameptr) {
4325 		panic("%s: cn_nameptr mismatch (%p != %p) full [%s]\n", __func__,
4326 		    cnp->cn_nameptr, fpl->snd.cn_nameptr, cnp->cn_pnbuf);
4327 	}
4328 #endif
4329 
4330 	ndp->ni_startdir = dvp;
4331 	cnp->cn_flags |= MAKEENTRY;
4332 	if (cache_fpl_islastcn(ndp))
4333 		cnp->cn_flags |= ISLASTCN;
4334 	if (cache_fpl_isdotdot(cnp))
4335 		cnp->cn_flags |= ISDOTDOT;
4336 
4337 	/*
4338 	 * Skip potential extra slashes parsing did not take care of.
4339 	 * cache_fplookup_skip_slashes explains the mechanism.
4340 	 */
4341 	if (__predict_false(*(cnp->cn_nameptr) == '/')) {
4342 		do {
4343 			cnp->cn_nameptr++;
4344 			cache_fpl_pathlen_dec(fpl);
4345 		} while (*(cnp->cn_nameptr) == '/');
4346 	}
4347 
4348 	ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1;
4349 #ifdef INVARIANTS
4350 	if (ndp->ni_pathlen != fpl->debug.ni_pathlen) {
4351 		panic("%s: mismatch (%zu != %zu) nulchar %p nameptr %p [%s] ; full string [%s]\n",
4352 		    __func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
4353 		    cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
4354 	}
4355 #endif
4356 	return (0);
4357 }
4358 
4359 static int
4360 cache_fplookup_final_child(struct cache_fpl *fpl, enum vgetstate tvs)
4361 {
4362 	struct componentname *cnp;
4363 	struct vnode *tvp;
4364 	seqc_t tvp_seqc;
4365 	int error, lkflags;
4366 
4367 	cnp = fpl->cnp;
4368 	tvp = fpl->tvp;
4369 	tvp_seqc = fpl->tvp_seqc;
4370 
4371 	if ((cnp->cn_flags & LOCKLEAF) != 0) {
4372 		lkflags = LK_SHARED;
4373 		if ((cnp->cn_flags & LOCKSHARED) == 0)
4374 			lkflags = LK_EXCLUSIVE;
4375 		error = vget_finish(tvp, lkflags, tvs);
4376 		if (__predict_false(error != 0)) {
4377 			return (cache_fpl_aborted(fpl));
4378 		}
4379 	} else {
4380 		vget_finish_ref(tvp, tvs);
4381 	}
4382 
4383 	if (!vn_seqc_consistent(tvp, tvp_seqc)) {
4384 		if ((cnp->cn_flags & LOCKLEAF) != 0)
4385 			vput(tvp);
4386 		else
4387 			vrele(tvp);
4388 		return (cache_fpl_aborted(fpl));
4389 	}
4390 
4391 	return (cache_fpl_handled(fpl));
4392 }
4393 
4394 /*
4395  * They want to possibly modify the state of the namecache.
4396  */
4397 static int __noinline
4398 cache_fplookup_final_modifying(struct cache_fpl *fpl)
4399 {
4400 	struct nameidata *ndp;
4401 	struct componentname *cnp;
4402 	enum vgetstate dvs;
4403 	struct vnode *dvp, *tvp;
4404 	struct mount *mp;
4405 	seqc_t dvp_seqc;
4406 	int error;
4407 	bool docache;
4408 
4409 	ndp = fpl->ndp;
4410 	cnp = fpl->cnp;
4411 	dvp = fpl->dvp;
4412 	dvp_seqc = fpl->dvp_seqc;
4413 
4414 	MPASS(*(cnp->cn_nameptr) != '/');
4415 	MPASS(cache_fpl_islastcn(ndp));
4416 	if ((cnp->cn_flags & LOCKPARENT) == 0)
4417 		MPASS((cnp->cn_flags & WANTPARENT) != 0);
4418 	MPASS((cnp->cn_flags & TRAILINGSLASH) == 0);
4419 	MPASS(cnp->cn_nameiop == CREATE || cnp->cn_nameiop == DELETE ||
4420 	    cnp->cn_nameiop == RENAME);
4421 	MPASS((cnp->cn_flags & MAKEENTRY) == 0);
4422 	MPASS((cnp->cn_flags & ISDOTDOT) == 0);
4423 
4424 	docache = (cnp->cn_flags & NOCACHE) ^ NOCACHE;
4425 	if (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)
4426 		docache = false;
4427 
4428 	/*
4429 	 * Regular lookup nulifies the slash, which we don't do here.
4430 	 * Don't take chances with filesystem routines seeing it for
4431 	 * the last entry.
4432 	 */
4433 	if (cache_fpl_istrailingslash(fpl)) {
4434 		return (cache_fpl_partial(fpl));
4435 	}
4436 
4437 	mp = atomic_load_ptr(&dvp->v_mount);
4438 	if (__predict_false(mp == NULL)) {
4439 		return (cache_fpl_aborted(fpl));
4440 	}
4441 
4442 	if (__predict_false(mp->mnt_flag & MNT_RDONLY)) {
4443 		cache_fpl_smr_exit(fpl);
4444 		/*
4445 		 * Original code keeps not checking for CREATE which
4446 		 * might be a bug. For now let the old lookup decide.
4447 		 */
4448 		if (cnp->cn_nameiop == CREATE) {
4449 			return (cache_fpl_aborted(fpl));
4450 		}
4451 		return (cache_fpl_handled_error(fpl, EROFS));
4452 	}
4453 
4454 	if (fpl->tvp != NULL && (cnp->cn_flags & FAILIFEXISTS) != 0) {
4455 		cache_fpl_smr_exit(fpl);
4456 		return (cache_fpl_handled_error(fpl, EEXIST));
4457 	}
4458 
4459 	/*
4460 	 * Secure access to dvp; check cache_fplookup_partial_setup for
4461 	 * reasoning.
4462 	 *
4463 	 * XXX At least UFS requires its lookup routine to be called for
4464 	 * the last path component, which leads to some level of complication
4465 	 * and inefficiency:
4466 	 * - the target routine always locks the target vnode, but our caller
4467 	 *   may not need it locked
4468 	 * - some of the VOP machinery asserts that the parent is locked, which
4469 	 *   once more may be not required
4470 	 *
4471 	 * TODO: add a flag for filesystems which don't need this.
4472 	 */
4473 	dvs = vget_prep_smr(dvp);
4474 	cache_fpl_smr_exit(fpl);
4475 	if (__predict_false(dvs == VGET_NONE)) {
4476 		return (cache_fpl_aborted(fpl));
4477 	}
4478 
4479 	vget_finish_ref(dvp, dvs);
4480 	if (!vn_seqc_consistent(dvp, dvp_seqc)) {
4481 		vrele(dvp);
4482 		return (cache_fpl_aborted(fpl));
4483 	}
4484 
4485 	error = vn_lock(dvp, LK_EXCLUSIVE);
4486 	if (__predict_false(error != 0)) {
4487 		vrele(dvp);
4488 		return (cache_fpl_aborted(fpl));
4489 	}
4490 
4491 	tvp = NULL;
4492 	cnp->cn_flags |= ISLASTCN;
4493 	if (docache)
4494 		cnp->cn_flags |= MAKEENTRY;
4495 	if (cache_fpl_isdotdot(cnp))
4496 		cnp->cn_flags |= ISDOTDOT;
4497 	cnp->cn_lkflags = LK_EXCLUSIVE;
4498 	error = VOP_LOOKUP(dvp, &tvp, cnp);
4499 	switch (error) {
4500 	case EJUSTRETURN:
4501 	case 0:
4502 		break;
4503 	case ENOTDIR:
4504 	case ENOENT:
4505 		vput(dvp);
4506 		return (cache_fpl_handled_error(fpl, error));
4507 	default:
4508 		vput(dvp);
4509 		return (cache_fpl_aborted(fpl));
4510 	}
4511 
4512 	fpl->tvp = tvp;
4513 	fpl->savename = (cnp->cn_flags & SAVENAME) != 0;
4514 
4515 	if (tvp == NULL) {
4516 		if ((cnp->cn_flags & SAVESTART) != 0) {
4517 			ndp->ni_startdir = dvp;
4518 			vrefact(ndp->ni_startdir);
4519 			cnp->cn_flags |= SAVENAME;
4520 			fpl->savename = true;
4521 		}
4522 		MPASS(error == EJUSTRETURN);
4523 		if ((cnp->cn_flags & LOCKPARENT) == 0) {
4524 			VOP_UNLOCK(dvp);
4525 		}
4526 		return (cache_fpl_handled(fpl));
4527 	}
4528 
4529 	/*
4530 	 * There are very hairy corner cases concerning various flag combinations
4531 	 * and locking state. In particular here we only hold one lock instead of
4532 	 * two.
4533 	 *
4534 	 * Skip the complexity as it is of no significance for normal workloads.
4535 	 */
4536 	if (__predict_false(tvp == dvp)) {
4537 		vput(dvp);
4538 		vrele(tvp);
4539 		return (cache_fpl_aborted(fpl));
4540 	}
4541 
4542 	/*
4543 	 * If they want the symlink itself we are fine, but if they want to
4544 	 * follow it regular lookup has to be engaged.
4545 	 */
4546 	if (tvp->v_type == VLNK) {
4547 		if ((cnp->cn_flags & FOLLOW) != 0) {
4548 			vput(dvp);
4549 			vput(tvp);
4550 			return (cache_fpl_aborted(fpl));
4551 		}
4552 	}
4553 
4554 	/*
4555 	 * Since we expect this to be the terminal vnode it should almost never
4556 	 * be a mount point.
4557 	 */
4558 	if (__predict_false(cache_fplookup_is_mp(fpl))) {
4559 		vput(dvp);
4560 		vput(tvp);
4561 		return (cache_fpl_aborted(fpl));
4562 	}
4563 
4564 	if ((cnp->cn_flags & FAILIFEXISTS) != 0) {
4565 		vput(dvp);
4566 		vput(tvp);
4567 		return (cache_fpl_handled_error(fpl, EEXIST));
4568 	}
4569 
4570 	if ((cnp->cn_flags & LOCKLEAF) == 0) {
4571 		VOP_UNLOCK(tvp);
4572 	}
4573 
4574 	if ((cnp->cn_flags & LOCKPARENT) == 0) {
4575 		VOP_UNLOCK(dvp);
4576 	}
4577 
4578 	if ((cnp->cn_flags & SAVESTART) != 0) {
4579 		ndp->ni_startdir = dvp;
4580 		vrefact(ndp->ni_startdir);
4581 		cnp->cn_flags |= SAVENAME;
4582 		fpl->savename = true;
4583 	}
4584 
4585 	return (cache_fpl_handled(fpl));
4586 }
4587 
4588 static int __noinline
4589 cache_fplookup_modifying(struct cache_fpl *fpl)
4590 {
4591 	struct nameidata *ndp;
4592 
4593 	ndp = fpl->ndp;
4594 
4595 	if (!cache_fpl_islastcn(ndp)) {
4596 		return (cache_fpl_partial(fpl));
4597 	}
4598 	return (cache_fplookup_final_modifying(fpl));
4599 }
4600 
4601 static int __noinline
4602 cache_fplookup_final_withparent(struct cache_fpl *fpl)
4603 {
4604 	struct componentname *cnp;
4605 	enum vgetstate dvs, tvs;
4606 	struct vnode *dvp, *tvp;
4607 	seqc_t dvp_seqc;
4608 	int error;
4609 
4610 	cnp = fpl->cnp;
4611 	dvp = fpl->dvp;
4612 	dvp_seqc = fpl->dvp_seqc;
4613 	tvp = fpl->tvp;
4614 
4615 	MPASS((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0);
4616 
4617 	/*
4618 	 * This is less efficient than it can be for simplicity.
4619 	 */
4620 	dvs = vget_prep_smr(dvp);
4621 	if (__predict_false(dvs == VGET_NONE)) {
4622 		return (cache_fpl_aborted(fpl));
4623 	}
4624 	tvs = vget_prep_smr(tvp);
4625 	if (__predict_false(tvs == VGET_NONE)) {
4626 		cache_fpl_smr_exit(fpl);
4627 		vget_abort(dvp, dvs);
4628 		return (cache_fpl_aborted(fpl));
4629 	}
4630 
4631 	cache_fpl_smr_exit(fpl);
4632 
4633 	if ((cnp->cn_flags & LOCKPARENT) != 0) {
4634 		error = vget_finish(dvp, LK_EXCLUSIVE, dvs);
4635 		if (__predict_false(error != 0)) {
4636 			vget_abort(tvp, tvs);
4637 			return (cache_fpl_aborted(fpl));
4638 		}
4639 	} else {
4640 		vget_finish_ref(dvp, dvs);
4641 	}
4642 
4643 	if (!vn_seqc_consistent(dvp, dvp_seqc)) {
4644 		vget_abort(tvp, tvs);
4645 		if ((cnp->cn_flags & LOCKPARENT) != 0)
4646 			vput(dvp);
4647 		else
4648 			vrele(dvp);
4649 		return (cache_fpl_aborted(fpl));
4650 	}
4651 
4652 	error = cache_fplookup_final_child(fpl, tvs);
4653 	if (__predict_false(error != 0)) {
4654 		MPASS(fpl->status == CACHE_FPL_STATUS_ABORTED ||
4655 		    fpl->status == CACHE_FPL_STATUS_DESTROYED);
4656 		if ((cnp->cn_flags & LOCKPARENT) != 0)
4657 			vput(dvp);
4658 		else
4659 			vrele(dvp);
4660 		return (error);
4661 	}
4662 
4663 	MPASS(fpl->status == CACHE_FPL_STATUS_HANDLED);
4664 	return (0);
4665 }
4666 
4667 static int
4668 cache_fplookup_final(struct cache_fpl *fpl)
4669 {
4670 	struct componentname *cnp;
4671 	enum vgetstate tvs;
4672 	struct vnode *dvp, *tvp;
4673 	seqc_t dvp_seqc;
4674 
4675 	cnp = fpl->cnp;
4676 	dvp = fpl->dvp;
4677 	dvp_seqc = fpl->dvp_seqc;
4678 	tvp = fpl->tvp;
4679 
4680 	MPASS(*(cnp->cn_nameptr) != '/');
4681 
4682 	if (cnp->cn_nameiop != LOOKUP) {
4683 		return (cache_fplookup_final_modifying(fpl));
4684 	}
4685 
4686 	if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0)
4687 		return (cache_fplookup_final_withparent(fpl));
4688 
4689 	tvs = vget_prep_smr(tvp);
4690 	if (__predict_false(tvs == VGET_NONE)) {
4691 		return (cache_fpl_partial(fpl));
4692 	}
4693 
4694 	if (!vn_seqc_consistent(dvp, dvp_seqc)) {
4695 		cache_fpl_smr_exit(fpl);
4696 		vget_abort(tvp, tvs);
4697 		return (cache_fpl_aborted(fpl));
4698 	}
4699 
4700 	cache_fpl_smr_exit(fpl);
4701 	return (cache_fplookup_final_child(fpl, tvs));
4702 }
4703 
4704 /*
4705  * Comment from locked lookup:
4706  * Check for degenerate name (e.g. / or "") which is a way of talking about a
4707  * directory, e.g. like "/." or ".".
4708  */
4709 static int __noinline
4710 cache_fplookup_degenerate(struct cache_fpl *fpl)
4711 {
4712 	struct componentname *cnp;
4713 	struct vnode *dvp;
4714 	enum vgetstate dvs;
4715 	int error, lkflags;
4716 #ifdef INVARIANTS
4717 	char *cp;
4718 #endif
4719 
4720 	fpl->tvp = fpl->dvp;
4721 	fpl->tvp_seqc = fpl->dvp_seqc;
4722 
4723 	cnp = fpl->cnp;
4724 	dvp = fpl->dvp;
4725 
4726 #ifdef INVARIANTS
4727 	for (cp = cnp->cn_pnbuf; *cp != '\0'; cp++) {
4728 		KASSERT(*cp == '/',
4729 		    ("%s: encountered non-slash; string [%s]\n", __func__,
4730 		    cnp->cn_pnbuf));
4731 	}
4732 #endif
4733 
4734 	if (__predict_false(cnp->cn_nameiop != LOOKUP)) {
4735 		cache_fpl_smr_exit(fpl);
4736 		return (cache_fpl_handled_error(fpl, EISDIR));
4737 	}
4738 
4739 	MPASS((cnp->cn_flags & SAVESTART) == 0);
4740 
4741 	if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0) {
4742 		return (cache_fplookup_final_withparent(fpl));
4743 	}
4744 
4745 	dvs = vget_prep_smr(dvp);
4746 	cache_fpl_smr_exit(fpl);
4747 	if (__predict_false(dvs == VGET_NONE)) {
4748 		return (cache_fpl_aborted(fpl));
4749 	}
4750 
4751 	if ((cnp->cn_flags & LOCKLEAF) != 0) {
4752 		lkflags = LK_SHARED;
4753 		if ((cnp->cn_flags & LOCKSHARED) == 0)
4754 			lkflags = LK_EXCLUSIVE;
4755 		error = vget_finish(dvp, lkflags, dvs);
4756 		if (__predict_false(error != 0)) {
4757 			return (cache_fpl_aborted(fpl));
4758 		}
4759 	} else {
4760 		vget_finish_ref(dvp, dvs);
4761 	}
4762 	return (cache_fpl_handled(fpl));
4763 }
4764 
4765 static int __noinline
4766 cache_fplookup_noentry(struct cache_fpl *fpl)
4767 {
4768 	struct nameidata *ndp;
4769 	struct componentname *cnp;
4770 	enum vgetstate dvs;
4771 	struct vnode *dvp, *tvp;
4772 	seqc_t dvp_seqc;
4773 	int error;
4774 	bool docache;
4775 
4776 	ndp = fpl->ndp;
4777 	cnp = fpl->cnp;
4778 	dvp = fpl->dvp;
4779 	dvp_seqc = fpl->dvp_seqc;
4780 
4781 	MPASS((cnp->cn_flags & MAKEENTRY) == 0);
4782 	MPASS((cnp->cn_flags & ISDOTDOT) == 0);
4783 	MPASS(!cache_fpl_isdotdot(cnp));
4784 
4785 	/*
4786 	 * Hack: delayed name len checking.
4787 	 */
4788 	if (__predict_false(cnp->cn_namelen > NAME_MAX)) {
4789 		cache_fpl_smr_exit(fpl);
4790 		return (cache_fpl_handled_error(fpl, ENAMETOOLONG));
4791 	}
4792 
4793 	if (cnp->cn_nameptr[0] == '/') {
4794 		return (cache_fplookup_skip_slashes(fpl));
4795 	}
4796 
4797 	if (cnp->cn_nameptr[0] == '\0') {
4798 		if (fpl->tvp == NULL) {
4799 			return (cache_fplookup_degenerate(fpl));
4800 		}
4801 		return (cache_fplookup_trailingslash(fpl));
4802 	}
4803 
4804 	if (cnp->cn_nameiop != LOOKUP) {
4805 		fpl->tvp = NULL;
4806 		return (cache_fplookup_modifying(fpl));
4807 	}
4808 
4809 	MPASS((cnp->cn_flags & SAVESTART) == 0);
4810 
4811 	/*
4812 	 * Only try to fill in the component if it is the last one,
4813 	 * otherwise not only there may be several to handle but the
4814 	 * walk may be complicated.
4815 	 */
4816 	if (!cache_fpl_islastcn(ndp)) {
4817 		return (cache_fpl_partial(fpl));
4818 	}
4819 
4820 	/*
4821 	 * Regular lookup nulifies the slash, which we don't do here.
4822 	 * Don't take chances with filesystem routines seeing it for
4823 	 * the last entry.
4824 	 */
4825 	if (cache_fpl_istrailingslash(fpl)) {
4826 		return (cache_fpl_partial(fpl));
4827 	}
4828 
4829 	/*
4830 	 * Secure access to dvp; check cache_fplookup_partial_setup for
4831 	 * reasoning.
4832 	 */
4833 	dvs = vget_prep_smr(dvp);
4834 	cache_fpl_smr_exit(fpl);
4835 	if (__predict_false(dvs == VGET_NONE)) {
4836 		return (cache_fpl_aborted(fpl));
4837 	}
4838 
4839 	vget_finish_ref(dvp, dvs);
4840 	if (!vn_seqc_consistent(dvp, dvp_seqc)) {
4841 		vrele(dvp);
4842 		return (cache_fpl_aborted(fpl));
4843 	}
4844 
4845 	error = vn_lock(dvp, LK_SHARED);
4846 	if (__predict_false(error != 0)) {
4847 		vrele(dvp);
4848 		return (cache_fpl_aborted(fpl));
4849 	}
4850 
4851 	tvp = NULL;
4852 	/*
4853 	 * TODO: provide variants which don't require locking either vnode.
4854 	 */
4855 	cnp->cn_flags |= ISLASTCN;
4856 	docache = (cnp->cn_flags & NOCACHE) ^ NOCACHE;
4857 	if (docache)
4858 		cnp->cn_flags |= MAKEENTRY;
4859 	cnp->cn_lkflags = LK_SHARED;
4860 	if ((cnp->cn_flags & LOCKSHARED) == 0) {
4861 		cnp->cn_lkflags = LK_EXCLUSIVE;
4862 	}
4863 	error = VOP_LOOKUP(dvp, &tvp, cnp);
4864 	switch (error) {
4865 	case EJUSTRETURN:
4866 	case 0:
4867 		break;
4868 	case ENOTDIR:
4869 	case ENOENT:
4870 		vput(dvp);
4871 		return (cache_fpl_handled_error(fpl, error));
4872 	default:
4873 		vput(dvp);
4874 		return (cache_fpl_aborted(fpl));
4875 	}
4876 
4877 	fpl->tvp = tvp;
4878 	if (!fpl->savename) {
4879 		MPASS((cnp->cn_flags & SAVENAME) == 0);
4880 	}
4881 
4882 	if (tvp == NULL) {
4883 		MPASS(error == EJUSTRETURN);
4884 		if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) {
4885 			vput(dvp);
4886 		} else if ((cnp->cn_flags & LOCKPARENT) == 0) {
4887 			VOP_UNLOCK(dvp);
4888 		}
4889 		return (cache_fpl_handled(fpl));
4890 	}
4891 
4892 	if (tvp->v_type == VLNK) {
4893 		if ((cnp->cn_flags & FOLLOW) != 0) {
4894 			vput(dvp);
4895 			vput(tvp);
4896 			return (cache_fpl_aborted(fpl));
4897 		}
4898 	}
4899 
4900 	if (__predict_false(cache_fplookup_is_mp(fpl))) {
4901 		vput(dvp);
4902 		vput(tvp);
4903 		return (cache_fpl_aborted(fpl));
4904 	}
4905 
4906 	if ((cnp->cn_flags & LOCKLEAF) == 0) {
4907 		VOP_UNLOCK(tvp);
4908 	}
4909 
4910 	if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) {
4911 		vput(dvp);
4912 	} else if ((cnp->cn_flags & LOCKPARENT) == 0) {
4913 		VOP_UNLOCK(dvp);
4914 	}
4915 	return (cache_fpl_handled(fpl));
4916 }
4917 
4918 static int __noinline
4919 cache_fplookup_dot(struct cache_fpl *fpl)
4920 {
4921 	int error;
4922 
4923 	MPASS(!seqc_in_modify(fpl->dvp_seqc));
4924 	/*
4925 	 * Just re-assign the value. seqc will be checked later for the first
4926 	 * non-dot path component in line and/or before deciding to return the
4927 	 * vnode.
4928 	 */
4929 	fpl->tvp = fpl->dvp;
4930 	fpl->tvp_seqc = fpl->dvp_seqc;
4931 
4932 	counter_u64_add(dothits, 1);
4933 	SDT_PROBE3(vfs, namecache, lookup, hit, fpl->dvp, ".", fpl->dvp);
4934 
4935 	error = 0;
4936 	if (cache_fplookup_is_mp(fpl)) {
4937 		error = cache_fplookup_cross_mount(fpl);
4938 	}
4939 	return (error);
4940 }
4941 
4942 static int __noinline
4943 cache_fplookup_dotdot(struct cache_fpl *fpl)
4944 {
4945 	struct nameidata *ndp;
4946 	struct componentname *cnp;
4947 	struct namecache *ncp;
4948 	struct vnode *dvp;
4949 	struct prison *pr;
4950 	u_char nc_flag;
4951 
4952 	ndp = fpl->ndp;
4953 	cnp = fpl->cnp;
4954 	dvp = fpl->dvp;
4955 
4956 	MPASS(cache_fpl_isdotdot(cnp));
4957 
4958 	/*
4959 	 * XXX this is racy the same way regular lookup is
4960 	 */
4961 	for (pr = cnp->cn_cred->cr_prison; pr != NULL;
4962 	    pr = pr->pr_parent)
4963 		if (dvp == pr->pr_root)
4964 			break;
4965 
4966 	if (dvp == ndp->ni_rootdir ||
4967 	    dvp == ndp->ni_topdir ||
4968 	    dvp == rootvnode ||
4969 	    pr != NULL) {
4970 		fpl->tvp = dvp;
4971 		fpl->tvp_seqc = vn_seqc_read_any(dvp);
4972 		if (seqc_in_modify(fpl->tvp_seqc)) {
4973 			return (cache_fpl_aborted(fpl));
4974 		}
4975 		return (0);
4976 	}
4977 
4978 	if ((dvp->v_vflag & VV_ROOT) != 0) {
4979 		/*
4980 		 * TODO
4981 		 * The opposite of climb mount is needed here.
4982 		 */
4983 		return (cache_fpl_partial(fpl));
4984 	}
4985 
4986 	ncp = atomic_load_consume_ptr(&dvp->v_cache_dd);
4987 	if (ncp == NULL) {
4988 		return (cache_fpl_aborted(fpl));
4989 	}
4990 
4991 	nc_flag = atomic_load_char(&ncp->nc_flag);
4992 	if ((nc_flag & NCF_ISDOTDOT) != 0) {
4993 		if ((nc_flag & NCF_NEGATIVE) != 0)
4994 			return (cache_fpl_aborted(fpl));
4995 		fpl->tvp = ncp->nc_vp;
4996 	} else {
4997 		fpl->tvp = ncp->nc_dvp;
4998 	}
4999 
5000 	fpl->tvp_seqc = vn_seqc_read_any(fpl->tvp);
5001 	if (seqc_in_modify(fpl->tvp_seqc)) {
5002 		return (cache_fpl_partial(fpl));
5003 	}
5004 
5005 	/*
5006 	 * Acquire fence provided by vn_seqc_read_any above.
5007 	 */
5008 	if (__predict_false(atomic_load_ptr(&dvp->v_cache_dd) != ncp)) {
5009 		return (cache_fpl_aborted(fpl));
5010 	}
5011 
5012 	if (!cache_ncp_canuse(ncp)) {
5013 		return (cache_fpl_aborted(fpl));
5014 	}
5015 
5016 	counter_u64_add(dotdothits, 1);
5017 	return (0);
5018 }
5019 
5020 static int __noinline
5021 cache_fplookup_neg(struct cache_fpl *fpl, struct namecache *ncp, uint32_t hash)
5022 {
5023 	u_char nc_flag;
5024 	bool neg_promote;
5025 
5026 	nc_flag = atomic_load_char(&ncp->nc_flag);
5027 	MPASS((nc_flag & NCF_NEGATIVE) != 0);
5028 	/*
5029 	 * If they want to create an entry we need to replace this one.
5030 	 */
5031 	if (__predict_false(fpl->cnp->cn_nameiop != LOOKUP)) {
5032 		fpl->tvp = NULL;
5033 		return (cache_fplookup_modifying(fpl));
5034 	}
5035 	neg_promote = cache_neg_hit_prep(ncp);
5036 	if (!cache_fpl_neg_ncp_canuse(ncp)) {
5037 		cache_neg_hit_abort(ncp);
5038 		return (cache_fpl_partial(fpl));
5039 	}
5040 	if (neg_promote) {
5041 		return (cache_fplookup_negative_promote(fpl, ncp, hash));
5042 	}
5043 	cache_neg_hit_finish(ncp);
5044 	cache_fpl_smr_exit(fpl);
5045 	return (cache_fpl_handled_error(fpl, ENOENT));
5046 }
5047 
5048 /*
5049  * Resolve a symlink. Called by filesystem-specific routines.
5050  *
5051  * Code flow is:
5052  * ... -> cache_fplookup_symlink -> VOP_FPLOOKUP_SYMLINK -> cache_symlink_resolve
5053  */
5054 int
5055 cache_symlink_resolve(struct cache_fpl *fpl, const char *string, size_t len)
5056 {
5057 	struct nameidata *ndp;
5058 	struct componentname *cnp;
5059 	size_t adjust;
5060 
5061 	ndp = fpl->ndp;
5062 	cnp = fpl->cnp;
5063 
5064 	if (__predict_false(len == 0)) {
5065 		return (ENOENT);
5066 	}
5067 
5068 	if (__predict_false(len > MAXPATHLEN - 2)) {
5069 		if (cache_fpl_istrailingslash(fpl)) {
5070 			return (EAGAIN);
5071 		}
5072 	}
5073 
5074 	ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr - cnp->cn_namelen + 1;
5075 #ifdef INVARIANTS
5076 	if (ndp->ni_pathlen != fpl->debug.ni_pathlen) {
5077 		panic("%s: mismatch (%zu != %zu) nulchar %p nameptr %p [%s] ; full string [%s]\n",
5078 		    __func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
5079 		    cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
5080 	}
5081 #endif
5082 
5083 	if (__predict_false(len + ndp->ni_pathlen > MAXPATHLEN)) {
5084 		return (ENAMETOOLONG);
5085 	}
5086 
5087 	if (__predict_false(ndp->ni_loopcnt++ >= MAXSYMLINKS)) {
5088 		return (ELOOP);
5089 	}
5090 
5091 	adjust = len;
5092 	if (ndp->ni_pathlen > 1) {
5093 		bcopy(ndp->ni_next, cnp->cn_pnbuf + len, ndp->ni_pathlen);
5094 	} else {
5095 		if (cache_fpl_istrailingslash(fpl)) {
5096 			adjust = len + 1;
5097 			cnp->cn_pnbuf[len] = '/';
5098 			cnp->cn_pnbuf[len + 1] = '\0';
5099 		} else {
5100 			cnp->cn_pnbuf[len] = '\0';
5101 		}
5102 	}
5103 	bcopy(string, cnp->cn_pnbuf, len);
5104 
5105 	ndp->ni_pathlen += adjust;
5106 	cache_fpl_pathlen_add(fpl, adjust);
5107 	cnp->cn_nameptr = cnp->cn_pnbuf;
5108 	fpl->nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1];
5109 	fpl->tvp = NULL;
5110 	return (0);
5111 }
5112 
5113 static int __noinline
5114 cache_fplookup_symlink(struct cache_fpl *fpl)
5115 {
5116 	struct mount *mp;
5117 	struct nameidata *ndp;
5118 	struct componentname *cnp;
5119 	struct vnode *dvp, *tvp;
5120 	int error;
5121 
5122 	ndp = fpl->ndp;
5123 	cnp = fpl->cnp;
5124 	dvp = fpl->dvp;
5125 	tvp = fpl->tvp;
5126 
5127 	if (cache_fpl_islastcn(ndp)) {
5128 		if ((cnp->cn_flags & FOLLOW) == 0) {
5129 			return (cache_fplookup_final(fpl));
5130 		}
5131 	}
5132 
5133 	mp = atomic_load_ptr(&dvp->v_mount);
5134 	if (__predict_false(mp == NULL)) {
5135 		return (cache_fpl_aborted(fpl));
5136 	}
5137 
5138 	/*
5139 	 * Note this check races against setting the flag just like regular
5140 	 * lookup.
5141 	 */
5142 	if (__predict_false((mp->mnt_flag & MNT_NOSYMFOLLOW) != 0)) {
5143 		cache_fpl_smr_exit(fpl);
5144 		return (cache_fpl_handled_error(fpl, EACCES));
5145 	}
5146 
5147 	error = VOP_FPLOOKUP_SYMLINK(tvp, fpl);
5148 	if (__predict_false(error != 0)) {
5149 		switch (error) {
5150 		case EAGAIN:
5151 			return (cache_fpl_partial(fpl));
5152 		case ENOENT:
5153 		case ENAMETOOLONG:
5154 		case ELOOP:
5155 			cache_fpl_smr_exit(fpl);
5156 			return (cache_fpl_handled_error(fpl, error));
5157 		default:
5158 			return (cache_fpl_aborted(fpl));
5159 		}
5160 	}
5161 
5162 	if (*(cnp->cn_nameptr) == '/') {
5163 		fpl->dvp = cache_fpl_handle_root(fpl);
5164 		fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
5165 		if (seqc_in_modify(fpl->dvp_seqc)) {
5166 			return (cache_fpl_aborted(fpl));
5167 		}
5168 		/*
5169 		 * The main loop assumes that ->dvp points to a vnode belonging
5170 		 * to a filesystem which can do lockless lookup, but the absolute
5171 		 * symlink can be wandering off to one which does not.
5172 		 */
5173 		mp = atomic_load_ptr(&fpl->dvp->v_mount);
5174 		if (__predict_false(mp == NULL)) {
5175 			return (cache_fpl_aborted(fpl));
5176 		}
5177 		if (!cache_fplookup_mp_supported(mp)) {
5178 			cache_fpl_checkpoint(fpl);
5179 			return (cache_fpl_partial(fpl));
5180 		}
5181 	}
5182 	return (0);
5183 }
5184 
5185 static int
5186 cache_fplookup_next(struct cache_fpl *fpl)
5187 {
5188 	struct componentname *cnp;
5189 	struct namecache *ncp;
5190 	struct vnode *dvp, *tvp;
5191 	u_char nc_flag;
5192 	uint32_t hash;
5193 	int error;
5194 
5195 	cnp = fpl->cnp;
5196 	dvp = fpl->dvp;
5197 	hash = fpl->hash;
5198 
5199 	if (__predict_false(cnp->cn_nameptr[0] == '.')) {
5200 		if (cnp->cn_namelen == 1) {
5201 			return (cache_fplookup_dot(fpl));
5202 		}
5203 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
5204 			return (cache_fplookup_dotdot(fpl));
5205 		}
5206 	}
5207 
5208 	MPASS(!cache_fpl_isdotdot(cnp));
5209 
5210 	CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
5211 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
5212 		    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
5213 			break;
5214 	}
5215 
5216 	if (__predict_false(ncp == NULL)) {
5217 		return (cache_fplookup_noentry(fpl));
5218 	}
5219 
5220 	tvp = atomic_load_ptr(&ncp->nc_vp);
5221 	nc_flag = atomic_load_char(&ncp->nc_flag);
5222 	if ((nc_flag & NCF_NEGATIVE) != 0) {
5223 		return (cache_fplookup_neg(fpl, ncp, hash));
5224 	}
5225 
5226 	if (!cache_ncp_canuse(ncp)) {
5227 		return (cache_fpl_partial(fpl));
5228 	}
5229 
5230 	fpl->tvp = tvp;
5231 	fpl->tvp_seqc = vn_seqc_read_any(tvp);
5232 	if (seqc_in_modify(fpl->tvp_seqc)) {
5233 		return (cache_fpl_partial(fpl));
5234 	}
5235 
5236 	counter_u64_add(numposhits, 1);
5237 	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp);
5238 
5239 	error = 0;
5240 	if (cache_fplookup_is_mp(fpl)) {
5241 		error = cache_fplookup_cross_mount(fpl);
5242 	}
5243 	return (error);
5244 }
5245 
5246 static bool
5247 cache_fplookup_mp_supported(struct mount *mp)
5248 {
5249 
5250 	MPASS(mp != NULL);
5251 	if ((mp->mnt_kern_flag & MNTK_FPLOOKUP) == 0)
5252 		return (false);
5253 	return (true);
5254 }
5255 
5256 /*
5257  * Walk up the mount stack (if any).
5258  *
5259  * Correctness is provided in the following ways:
5260  * - all vnodes are protected from freeing with SMR
5261  * - struct mount objects are type stable making them always safe to access
5262  * - stability of the particular mount is provided by busying it
5263  * - relationship between the vnode which is mounted on and the mount is
5264  *   verified with the vnode sequence counter after busying
5265  * - association between root vnode of the mount and the mount is protected
5266  *   by busy
5267  *
5268  * From that point on we can read the sequence counter of the root vnode
5269  * and get the next mount on the stack (if any) using the same protection.
5270  *
5271  * By the end of successful walk we are guaranteed the reached state was
5272  * indeed present at least at some point which matches the regular lookup.
5273  */
5274 static int __noinline
5275 cache_fplookup_climb_mount(struct cache_fpl *fpl)
5276 {
5277 	struct mount *mp, *prev_mp;
5278 	struct mount_pcpu *mpcpu, *prev_mpcpu;
5279 	struct vnode *vp;
5280 	seqc_t vp_seqc;
5281 
5282 	vp = fpl->tvp;
5283 	vp_seqc = fpl->tvp_seqc;
5284 
5285 	VNPASS(vp->v_type == VDIR || vp->v_type == VBAD, vp);
5286 	mp = atomic_load_ptr(&vp->v_mountedhere);
5287 	if (__predict_false(mp == NULL)) {
5288 		return (0);
5289 	}
5290 
5291 	prev_mp = NULL;
5292 	for (;;) {
5293 		if (!vfs_op_thread_enter_crit(mp, mpcpu)) {
5294 			if (prev_mp != NULL)
5295 				vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
5296 			return (cache_fpl_partial(fpl));
5297 		}
5298 		if (prev_mp != NULL)
5299 			vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
5300 		if (!vn_seqc_consistent(vp, vp_seqc)) {
5301 			vfs_op_thread_exit_crit(mp, mpcpu);
5302 			return (cache_fpl_partial(fpl));
5303 		}
5304 		if (!cache_fplookup_mp_supported(mp)) {
5305 			vfs_op_thread_exit_crit(mp, mpcpu);
5306 			return (cache_fpl_partial(fpl));
5307 		}
5308 		vp = atomic_load_ptr(&mp->mnt_rootvnode);
5309 		if (vp == NULL) {
5310 			vfs_op_thread_exit_crit(mp, mpcpu);
5311 			return (cache_fpl_partial(fpl));
5312 		}
5313 		vp_seqc = vn_seqc_read_any(vp);
5314 		if (seqc_in_modify(vp_seqc)) {
5315 			vfs_op_thread_exit_crit(mp, mpcpu);
5316 			return (cache_fpl_partial(fpl));
5317 		}
5318 		prev_mp = mp;
5319 		prev_mpcpu = mpcpu;
5320 		mp = atomic_load_ptr(&vp->v_mountedhere);
5321 		if (mp == NULL)
5322 			break;
5323 	}
5324 
5325 	vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
5326 	fpl->tvp = vp;
5327 	fpl->tvp_seqc = vp_seqc;
5328 	return (0);
5329 }
5330 
5331 static int __noinline
5332 cache_fplookup_cross_mount(struct cache_fpl *fpl)
5333 {
5334 	struct mount *mp;
5335 	struct mount_pcpu *mpcpu;
5336 	struct vnode *vp;
5337 	seqc_t vp_seqc;
5338 
5339 	vp = fpl->tvp;
5340 	vp_seqc = fpl->tvp_seqc;
5341 
5342 	VNPASS(vp->v_type == VDIR || vp->v_type == VBAD, vp);
5343 	mp = atomic_load_ptr(&vp->v_mountedhere);
5344 	if (__predict_false(mp == NULL)) {
5345 		return (0);
5346 	}
5347 
5348 	if (!vfs_op_thread_enter_crit(mp, mpcpu)) {
5349 		return (cache_fpl_partial(fpl));
5350 	}
5351 	if (!vn_seqc_consistent(vp, vp_seqc)) {
5352 		vfs_op_thread_exit_crit(mp, mpcpu);
5353 		return (cache_fpl_partial(fpl));
5354 	}
5355 	if (!cache_fplookup_mp_supported(mp)) {
5356 		vfs_op_thread_exit_crit(mp, mpcpu);
5357 		return (cache_fpl_partial(fpl));
5358 	}
5359 	vp = atomic_load_ptr(&mp->mnt_rootvnode);
5360 	if (__predict_false(vp == NULL)) {
5361 		vfs_op_thread_exit_crit(mp, mpcpu);
5362 		return (cache_fpl_partial(fpl));
5363 	}
5364 	vp_seqc = vn_seqc_read_any(vp);
5365 	vfs_op_thread_exit_crit(mp, mpcpu);
5366 	if (seqc_in_modify(vp_seqc)) {
5367 		return (cache_fpl_partial(fpl));
5368 	}
5369 	mp = atomic_load_ptr(&vp->v_mountedhere);
5370 	if (__predict_false(mp != NULL)) {
5371 		/*
5372 		 * There are possibly more mount points on top.
5373 		 * Normally this does not happen so for simplicity just start
5374 		 * over.
5375 		 */
5376 		return (cache_fplookup_climb_mount(fpl));
5377 	}
5378 
5379 	fpl->tvp = vp;
5380 	fpl->tvp_seqc = vp_seqc;
5381 	return (0);
5382 }
5383 
5384 /*
5385  * Check if a vnode is mounted on.
5386  */
5387 static bool
5388 cache_fplookup_is_mp(struct cache_fpl *fpl)
5389 {
5390 	struct vnode *vp;
5391 
5392 	vp = fpl->tvp;
5393 	return ((vn_irflag_read(vp) & VIRF_MOUNTPOINT) != 0);
5394 }
5395 
5396 /*
5397  * Parse the path.
5398  *
5399  * The code was originally copy-pasted from regular lookup and despite
5400  * clean ups leaves performance on the table. Any modifications here
5401  * must take into account that in case off fallback the resulting
5402  * nameidata state has to be compatible with the original.
5403  */
5404 
5405 /*
5406  * Debug ni_pathlen tracking.
5407  */
5408 #ifdef INVARIANTS
5409 static void
5410 cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n)
5411 {
5412 
5413 	fpl->debug.ni_pathlen += n;
5414 	KASSERT(fpl->debug.ni_pathlen <= PATH_MAX,
5415 	    ("%s: pathlen overflow to %zd\n", __func__, fpl->debug.ni_pathlen));
5416 }
5417 
5418 static void
5419 cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n)
5420 {
5421 
5422 	fpl->debug.ni_pathlen -= n;
5423 	KASSERT(fpl->debug.ni_pathlen <= PATH_MAX,
5424 	    ("%s: pathlen underflow to %zd\n", __func__, fpl->debug.ni_pathlen));
5425 }
5426 
5427 static void
5428 cache_fpl_pathlen_inc(struct cache_fpl *fpl)
5429 {
5430 
5431 	cache_fpl_pathlen_add(fpl, 1);
5432 }
5433 
5434 static void
5435 cache_fpl_pathlen_dec(struct cache_fpl *fpl)
5436 {
5437 
5438 	cache_fpl_pathlen_sub(fpl, 1);
5439 }
5440 #else
5441 static void
5442 cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n)
5443 {
5444 }
5445 
5446 static void
5447 cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n)
5448 {
5449 }
5450 
5451 static void
5452 cache_fpl_pathlen_inc(struct cache_fpl *fpl)
5453 {
5454 }
5455 
5456 static void
5457 cache_fpl_pathlen_dec(struct cache_fpl *fpl)
5458 {
5459 }
5460 #endif
5461 
5462 static void
5463 cache_fplookup_parse(struct cache_fpl *fpl)
5464 {
5465 	struct nameidata *ndp;
5466 	struct componentname *cnp;
5467 	struct vnode *dvp;
5468 	char *cp;
5469 	uint32_t hash;
5470 
5471 	ndp = fpl->ndp;
5472 	cnp = fpl->cnp;
5473 	dvp = fpl->dvp;
5474 
5475 	/*
5476 	 * Find the end of this path component, it is either / or nul.
5477 	 *
5478 	 * Store / as a temporary sentinel so that we only have one character
5479 	 * to test for. Pathnames tend to be short so this should not be
5480 	 * resulting in cache misses.
5481 	 *
5482 	 * TODO: fix this to be word-sized.
5483 	 */
5484 	KASSERT(&cnp->cn_nameptr[fpl->debug.ni_pathlen - 1] == fpl->nulchar,
5485 	    ("%s: mismatch between pathlen (%zu) and nulchar (%p != %p), string [%s]\n",
5486 	    __func__, fpl->debug.ni_pathlen, &cnp->cn_nameptr[fpl->debug.ni_pathlen - 1],
5487 	    fpl->nulchar, cnp->cn_pnbuf));
5488 	KASSERT(*fpl->nulchar == '\0',
5489 	    ("%s: expected nul at %p; string [%s]\n", __func__, fpl->nulchar,
5490 	    cnp->cn_pnbuf));
5491 	hash = cache_get_hash_iter_start(dvp);
5492 	*fpl->nulchar = '/';
5493 	for (cp = cnp->cn_nameptr; *cp != '/'; cp++) {
5494 		KASSERT(*cp != '\0',
5495 		    ("%s: encountered unexpected nul; string [%s]\n", __func__,
5496 		    cnp->cn_nameptr));
5497 		hash = cache_get_hash_iter(*cp, hash);
5498 		continue;
5499 	}
5500 	*fpl->nulchar = '\0';
5501 	fpl->hash = cache_get_hash_iter_finish(hash);
5502 
5503 	cnp->cn_namelen = cp - cnp->cn_nameptr;
5504 	cache_fpl_pathlen_sub(fpl, cnp->cn_namelen);
5505 
5506 #ifdef INVARIANTS
5507 	/*
5508 	 * cache_get_hash only accepts lengths up to NAME_MAX. This is fine since
5509 	 * we are going to fail this lookup with ENAMETOOLONG (see below).
5510 	 */
5511 	if (cnp->cn_namelen <= NAME_MAX) {
5512 		if (fpl->hash != cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp)) {
5513 			panic("%s: mismatched hash for [%s] len %ld", __func__,
5514 			    cnp->cn_nameptr, cnp->cn_namelen);
5515 		}
5516 	}
5517 #endif
5518 
5519 	/*
5520 	 * Hack: we have to check if the found path component's length exceeds
5521 	 * NAME_MAX. However, the condition is very rarely true and check can
5522 	 * be elided in the common case -- if an entry was found in the cache,
5523 	 * then it could not have been too long to begin with.
5524 	 */
5525 	ndp->ni_next = cp;
5526 }
5527 
5528 static void
5529 cache_fplookup_parse_advance(struct cache_fpl *fpl)
5530 {
5531 	struct nameidata *ndp;
5532 	struct componentname *cnp;
5533 
5534 	ndp = fpl->ndp;
5535 	cnp = fpl->cnp;
5536 
5537 	cnp->cn_nameptr = ndp->ni_next;
5538 	KASSERT(*(cnp->cn_nameptr) == '/',
5539 	    ("%s: should have seen slash at %p ; buf %p [%s]\n", __func__,
5540 	    cnp->cn_nameptr, cnp->cn_pnbuf, cnp->cn_pnbuf));
5541 	cnp->cn_nameptr++;
5542 	cache_fpl_pathlen_dec(fpl);
5543 }
5544 
5545 /*
5546  * Skip spurious slashes in a pathname (e.g., "foo///bar") and retry.
5547  *
5548  * Lockless lookup tries to elide checking for spurious slashes and should they
5549  * be present is guaranteed to fail to find an entry. In this case the caller
5550  * must check if the name starts with a slash and call this routine.  It is
5551  * going to fast forward across the spurious slashes and set the state up for
5552  * retry.
5553  */
5554 static int __noinline
5555 cache_fplookup_skip_slashes(struct cache_fpl *fpl)
5556 {
5557 	struct nameidata *ndp;
5558 	struct componentname *cnp;
5559 
5560 	ndp = fpl->ndp;
5561 	cnp = fpl->cnp;
5562 
5563 	MPASS(*(cnp->cn_nameptr) == '/');
5564 	do {
5565 		cnp->cn_nameptr++;
5566 		cache_fpl_pathlen_dec(fpl);
5567 	} while (*(cnp->cn_nameptr) == '/');
5568 
5569 	/*
5570 	 * Go back to one slash so that cache_fplookup_parse_advance has
5571 	 * something to skip.
5572 	 */
5573 	cnp->cn_nameptr--;
5574 	cache_fpl_pathlen_inc(fpl);
5575 
5576 	/*
5577 	 * cache_fplookup_parse_advance starts from ndp->ni_next
5578 	 */
5579 	ndp->ni_next = cnp->cn_nameptr;
5580 
5581 	/*
5582 	 * See cache_fplookup_dot.
5583 	 */
5584 	fpl->tvp = fpl->dvp;
5585 	fpl->tvp_seqc = fpl->dvp_seqc;
5586 
5587 	return (0);
5588 }
5589 
5590 /*
5591  * Handle trailing slashes (e.g., "foo/").
5592  *
5593  * If a trailing slash is found the terminal vnode must be a directory.
5594  * Regular lookup shortens the path by nulifying the first trailing slash and
5595  * sets the TRAILINGSLASH flag to denote this took place. There are several
5596  * checks on it performed later.
5597  *
5598  * Similarly to spurious slashes, lockless lookup handles this in a speculative
5599  * manner relying on an invariant that a non-directory vnode will get a miss.
5600  * In this case cn_nameptr[0] == '\0' and cn_namelen == 0.
5601  *
5602  * Thus for a path like "foo/bar/" the code unwinds the state back to "bar/"
5603  * and denotes this is the last path component, which avoids looping back.
5604  *
5605  * Only plain lookups are supported for now to restrict corner cases to handle.
5606  */
5607 static int __noinline
5608 cache_fplookup_trailingslash(struct cache_fpl *fpl)
5609 {
5610 #ifdef INVARIANTS
5611 	size_t ni_pathlen;
5612 #endif
5613 	struct nameidata *ndp;
5614 	struct componentname *cnp;
5615 	struct namecache *ncp;
5616 	struct vnode *tvp;
5617 	char *cn_nameptr_orig, *cn_nameptr_slash;
5618 	seqc_t tvp_seqc;
5619 	u_char nc_flag;
5620 
5621 	ndp = fpl->ndp;
5622 	cnp = fpl->cnp;
5623 	tvp = fpl->tvp;
5624 	tvp_seqc = fpl->tvp_seqc;
5625 
5626 	MPASS(fpl->dvp == fpl->tvp);
5627 	KASSERT(cache_fpl_istrailingslash(fpl),
5628 	    ("%s: expected trailing slash at %p; string [%s]\n", __func__, fpl->nulchar - 1,
5629 	    cnp->cn_pnbuf));
5630 	KASSERT(cnp->cn_nameptr[0] == '\0',
5631 	    ("%s: expected nul char at %p; string [%s]\n", __func__, &cnp->cn_nameptr[0],
5632 	    cnp->cn_pnbuf));
5633 	KASSERT(cnp->cn_namelen == 0,
5634 	    ("%s: namelen 0 but got %ld; string [%s]\n", __func__, cnp->cn_namelen,
5635 	    cnp->cn_pnbuf));
5636 	MPASS(cnp->cn_nameptr > cnp->cn_pnbuf);
5637 
5638 	if (cnp->cn_nameiop != LOOKUP) {
5639 		return (cache_fpl_aborted(fpl));
5640 	}
5641 
5642 	if (__predict_false(tvp->v_type != VDIR)) {
5643 		if (!vn_seqc_consistent(tvp, tvp_seqc)) {
5644 			return (cache_fpl_aborted(fpl));
5645 		}
5646 		cache_fpl_smr_exit(fpl);
5647 		return (cache_fpl_handled_error(fpl, ENOTDIR));
5648 	}
5649 
5650 	/*
5651 	 * Denote the last component.
5652 	 */
5653 	ndp->ni_next = &cnp->cn_nameptr[0];
5654 	MPASS(cache_fpl_islastcn(ndp));
5655 
5656 	/*
5657 	 * Unwind trailing slashes.
5658 	 */
5659 	cn_nameptr_orig = cnp->cn_nameptr;
5660 	while (cnp->cn_nameptr >= cnp->cn_pnbuf) {
5661 		cnp->cn_nameptr--;
5662 		if (cnp->cn_nameptr[0] != '/') {
5663 			break;
5664 		}
5665 	}
5666 
5667 	/*
5668 	 * Unwind to the beginning of the path component.
5669 	 *
5670 	 * Note the path may or may not have started with a slash.
5671 	 */
5672 	cn_nameptr_slash = cnp->cn_nameptr;
5673 	while (cnp->cn_nameptr > cnp->cn_pnbuf) {
5674 		cnp->cn_nameptr--;
5675 		if (cnp->cn_nameptr[0] == '/') {
5676 			break;
5677 		}
5678 	}
5679 	if (cnp->cn_nameptr[0] == '/') {
5680 		cnp->cn_nameptr++;
5681 	}
5682 
5683 	cnp->cn_namelen = cn_nameptr_slash - cnp->cn_nameptr + 1;
5684 	cache_fpl_pathlen_add(fpl, cn_nameptr_orig - cnp->cn_nameptr);
5685 	cache_fpl_checkpoint(fpl);
5686 
5687 #ifdef INVARIANTS
5688 	ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1;
5689 	if (ni_pathlen != fpl->debug.ni_pathlen) {
5690 		panic("%s: mismatch (%zu != %zu) nulchar %p nameptr %p [%s] ; full string [%s]\n",
5691 		    __func__, ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
5692 		    cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
5693 	}
5694 #endif
5695 
5696 	/*
5697 	 * If this was a "./" lookup the parent directory is already correct.
5698 	 */
5699 	if (cnp->cn_nameptr[0] == '.' && cnp->cn_namelen == 1) {
5700 		return (0);
5701 	}
5702 
5703 	/*
5704 	 * Otherwise we need to look it up.
5705 	 */
5706 	tvp = fpl->tvp;
5707 	ncp = atomic_load_consume_ptr(&tvp->v_cache_dd);
5708 	if (__predict_false(ncp == NULL)) {
5709 		return (cache_fpl_aborted(fpl));
5710 	}
5711 	nc_flag = atomic_load_char(&ncp->nc_flag);
5712 	if ((nc_flag & NCF_ISDOTDOT) != 0) {
5713 		return (cache_fpl_aborted(fpl));
5714 	}
5715 	fpl->dvp = ncp->nc_dvp;
5716 	fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
5717 	if (seqc_in_modify(fpl->dvp_seqc)) {
5718 		return (cache_fpl_aborted(fpl));
5719 	}
5720 	return (0);
5721 }
5722 
5723 /*
5724  * See the API contract for VOP_FPLOOKUP_VEXEC.
5725  */
5726 static int __noinline
5727 cache_fplookup_failed_vexec(struct cache_fpl *fpl, int error)
5728 {
5729 	struct componentname *cnp;
5730 	struct vnode *dvp;
5731 	seqc_t dvp_seqc;
5732 
5733 	cnp = fpl->cnp;
5734 	dvp = fpl->dvp;
5735 	dvp_seqc = fpl->dvp_seqc;
5736 
5737 	/*
5738 	 * TODO: Due to ignoring trailing slashes lookup will perform a
5739 	 * permission check on the last dir when it should not be doing it.  It
5740 	 * may fail, but said failure should be ignored. It is possible to fix
5741 	 * it up fully without resorting to regular lookup, but for now just
5742 	 * abort.
5743 	 */
5744 	if (cache_fpl_istrailingslash(fpl)) {
5745 		return (cache_fpl_aborted(fpl));
5746 	}
5747 
5748 	/*
5749 	 * Hack: delayed degenerate path checking.
5750 	 */
5751 	if (cnp->cn_nameptr[0] == '\0' && fpl->tvp == NULL) {
5752 		return (cache_fplookup_degenerate(fpl));
5753 	}
5754 
5755 	/*
5756 	 * Hack: delayed name len checking.
5757 	 */
5758 	if (__predict_false(cnp->cn_namelen > NAME_MAX)) {
5759 		cache_fpl_smr_exit(fpl);
5760 		return (cache_fpl_handled_error(fpl, ENAMETOOLONG));
5761 	}
5762 
5763 	/*
5764 	 * Hack: they may be looking up foo/bar, where foo is not a directory.
5765 	 * In such a case we need to return ENOTDIR, but we may happen to get
5766 	 * here with a different error.
5767 	 */
5768 	if (dvp->v_type != VDIR) {
5769 		error = ENOTDIR;
5770 	}
5771 
5772 	/*
5773 	 * Hack: handle O_SEARCH.
5774 	 *
5775 	 * Open Group Base Specifications Issue 7, 2018 edition states:
5776 	 * <quote>
5777 	 * If the access mode of the open file description associated with the
5778 	 * file descriptor is not O_SEARCH, the function shall check whether
5779 	 * directory searches are permitted using the current permissions of
5780 	 * the directory underlying the file descriptor. If the access mode is
5781 	 * O_SEARCH, the function shall not perform the check.
5782 	 * </quote>
5783 	 *
5784 	 * Regular lookup tests for the NOEXECCHECK flag for every path
5785 	 * component to decide whether to do the permission check. However,
5786 	 * since most lookups never have the flag (and when they do it is only
5787 	 * present for the first path component), lockless lookup only acts on
5788 	 * it if there is a permission problem. Here the flag is represented
5789 	 * with a boolean so that we don't have to clear it on the way out.
5790 	 *
5791 	 * For simplicity this always aborts.
5792 	 * TODO: check if this is the first lookup and ignore the permission
5793 	 * problem. Note the flag has to survive fallback (if it happens to be
5794 	 * performed).
5795 	 */
5796 	if (fpl->fsearch) {
5797 		return (cache_fpl_aborted(fpl));
5798 	}
5799 
5800 	switch (error) {
5801 	case EAGAIN:
5802 		if (!vn_seqc_consistent(dvp, dvp_seqc)) {
5803 			error = cache_fpl_aborted(fpl);
5804 		} else {
5805 			cache_fpl_partial(fpl);
5806 		}
5807 		break;
5808 	default:
5809 		if (!vn_seqc_consistent(dvp, dvp_seqc)) {
5810 			error = cache_fpl_aborted(fpl);
5811 		} else {
5812 			cache_fpl_smr_exit(fpl);
5813 			cache_fpl_handled_error(fpl, error);
5814 		}
5815 		break;
5816 	}
5817 	return (error);
5818 }
5819 
5820 static int
5821 cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl)
5822 {
5823 	struct nameidata *ndp;
5824 	struct componentname *cnp;
5825 	struct mount *mp;
5826 	int error;
5827 
5828 	ndp = fpl->ndp;
5829 	cnp = fpl->cnp;
5830 
5831 	cache_fpl_checkpoint(fpl);
5832 
5833 	/*
5834 	 * The vnode at hand is almost always stable, skip checking for it.
5835 	 * Worst case this postpones the check towards the end of the iteration
5836 	 * of the main loop.
5837 	 */
5838 	fpl->dvp = dvp;
5839 	fpl->dvp_seqc = vn_seqc_read_notmodify(fpl->dvp);
5840 
5841 	mp = atomic_load_ptr(&dvp->v_mount);
5842 	if (__predict_false(mp == NULL || !cache_fplookup_mp_supported(mp))) {
5843 		return (cache_fpl_aborted(fpl));
5844 	}
5845 
5846 	MPASS(fpl->tvp == NULL);
5847 
5848 	for (;;) {
5849 		cache_fplookup_parse(fpl);
5850 
5851 		error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred);
5852 		if (__predict_false(error != 0)) {
5853 			error = cache_fplookup_failed_vexec(fpl, error);
5854 			break;
5855 		}
5856 
5857 		error = cache_fplookup_next(fpl);
5858 		if (__predict_false(cache_fpl_terminated(fpl))) {
5859 			break;
5860 		}
5861 
5862 		VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp);
5863 
5864 		if (fpl->tvp->v_type == VLNK) {
5865 			error = cache_fplookup_symlink(fpl);
5866 			if (cache_fpl_terminated(fpl)) {
5867 				break;
5868 			}
5869 		} else {
5870 			if (cache_fpl_islastcn(ndp)) {
5871 				error = cache_fplookup_final(fpl);
5872 				break;
5873 			}
5874 
5875 			if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) {
5876 				error = cache_fpl_aborted(fpl);
5877 				break;
5878 			}
5879 
5880 			fpl->dvp = fpl->tvp;
5881 			fpl->dvp_seqc = fpl->tvp_seqc;
5882 			cache_fplookup_parse_advance(fpl);
5883 		}
5884 
5885 		cache_fpl_checkpoint(fpl);
5886 	}
5887 
5888 	return (error);
5889 }
5890 
5891 /*
5892  * Fast path lookup protected with SMR and sequence counters.
5893  *
5894  * Note: all VOP_FPLOOKUP_VEXEC routines have a comment referencing this one.
5895  *
5896  * Filesystems can opt in by setting the MNTK_FPLOOKUP flag and meeting criteria
5897  * outlined below.
5898  *
5899  * Traditional vnode lookup conceptually looks like this:
5900  *
5901  * vn_lock(current);
5902  * for (;;) {
5903  *	next = find();
5904  *	vn_lock(next);
5905  *	vn_unlock(current);
5906  *	current = next;
5907  *	if (last)
5908  *	    break;
5909  * }
5910  * return (current);
5911  *
5912  * Each jump to the next vnode is safe memory-wise and atomic with respect to
5913  * any modifications thanks to holding respective locks.
5914  *
5915  * The same guarantee can be provided with a combination of safe memory
5916  * reclamation and sequence counters instead. If all operations which affect
5917  * the relationship between the current vnode and the one we are looking for
5918  * also modify the counter, we can verify whether all the conditions held as
5919  * we made the jump. This includes things like permissions, mount points etc.
5920  * Counter modification is provided by enclosing relevant places in
5921  * vn_seqc_write_begin()/end() calls.
5922  *
5923  * Thus this translates to:
5924  *
5925  * vfs_smr_enter();
5926  * dvp_seqc = seqc_read_any(dvp);
5927  * if (seqc_in_modify(dvp_seqc)) // someone is altering the vnode
5928  *     abort();
5929  * for (;;) {
5930  * 	tvp = find();
5931  * 	tvp_seqc = seqc_read_any(tvp);
5932  * 	if (seqc_in_modify(tvp_seqc)) // someone is altering the target vnode
5933  * 	    abort();
5934  * 	if (!seqc_consistent(dvp, dvp_seqc) // someone is altering the vnode
5935  * 	    abort();
5936  * 	dvp = tvp; // we know nothing of importance has changed
5937  * 	dvp_seqc = tvp_seqc; // store the counter for the tvp iteration
5938  * 	if (last)
5939  * 	    break;
5940  * }
5941  * vget(); // secure the vnode
5942  * if (!seqc_consistent(tvp, tvp_seqc) // final check
5943  * 	    abort();
5944  * // at this point we know nothing has changed for any parent<->child pair
5945  * // as they were crossed during the lookup, meaning we matched the guarantee
5946  * // of the locked variant
5947  * return (tvp);
5948  *
5949  * The API contract for VOP_FPLOOKUP_VEXEC routines is as follows:
5950  * - they are called while within vfs_smr protection which they must never exit
5951  * - EAGAIN can be returned to denote checking could not be performed, it is
5952  *   always valid to return it
5953  * - if the sequence counter has not changed the result must be valid
5954  * - if the sequence counter has changed both false positives and false negatives
5955  *   are permitted (since the result will be rejected later)
5956  * - for simple cases of unix permission checks vaccess_vexec_smr can be used
5957  *
5958  * Caveats to watch out for:
5959  * - vnodes are passed unlocked and unreferenced with nothing stopping
5960  *   VOP_RECLAIM, in turn meaning that ->v_data can become NULL. It is advised
5961  *   to use atomic_load_ptr to fetch it.
5962  * - the aforementioned object can also get freed, meaning absent other means it
5963  *   should be protected with vfs_smr
5964  * - either safely checking permissions as they are modified or guaranteeing
5965  *   their stability is left to the routine
5966  */
5967 int
5968 cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status,
5969     struct pwd **pwdp)
5970 {
5971 	struct cache_fpl fpl;
5972 	struct pwd *pwd;
5973 	struct vnode *dvp;
5974 	struct componentname *cnp;
5975 	int error;
5976 
5977 	fpl.status = CACHE_FPL_STATUS_UNSET;
5978 	fpl.in_smr = false;
5979 	fpl.ndp = ndp;
5980 	fpl.cnp = cnp = &ndp->ni_cnd;
5981 	MPASS(ndp->ni_lcf == 0);
5982 	MPASS(curthread == cnp->cn_thread);
5983 	KASSERT ((cnp->cn_flags & CACHE_FPL_INTERNAL_CN_FLAGS) == 0,
5984 	    ("%s: internal flags found in cn_flags %" PRIx64, __func__,
5985 	    cnp->cn_flags));
5986 	if ((cnp->cn_flags & SAVESTART) != 0) {
5987 		MPASS(cnp->cn_nameiop != LOOKUP);
5988 	}
5989 	MPASS(cnp->cn_nameptr == cnp->cn_pnbuf);
5990 
5991 	if (__predict_false(!cache_can_fplookup(&fpl))) {
5992 		*status = fpl.status;
5993 		SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);
5994 		return (EOPNOTSUPP);
5995 	}
5996 
5997 	cache_fpl_checkpoint_outer(&fpl);
5998 
5999 	cache_fpl_smr_enter_initial(&fpl);
6000 #ifdef INVARIANTS
6001 	fpl.debug.ni_pathlen = ndp->ni_pathlen;
6002 #endif
6003 	fpl.nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1];
6004 	fpl.fsearch = false;
6005 	fpl.savename = (cnp->cn_flags & SAVENAME) != 0;
6006 	fpl.tvp = NULL; /* for degenerate path handling */
6007 	fpl.pwd = pwdp;
6008 	pwd = pwd_get_smr();
6009 	*(fpl.pwd) = pwd;
6010 	ndp->ni_rootdir = pwd->pwd_rdir;
6011 	ndp->ni_topdir = pwd->pwd_jdir;
6012 
6013 	if (cnp->cn_pnbuf[0] == '/') {
6014 		dvp = cache_fpl_handle_root(&fpl);
6015 		MPASS(ndp->ni_resflags == 0);
6016 		ndp->ni_resflags = NIRES_ABS;
6017 	} else {
6018 		if (ndp->ni_dirfd == AT_FDCWD) {
6019 			dvp = pwd->pwd_cdir;
6020 		} else {
6021 			error = cache_fplookup_dirfd(&fpl, &dvp);
6022 			if (__predict_false(error != 0)) {
6023 				goto out;
6024 			}
6025 		}
6026 	}
6027 
6028 	SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true);
6029 	error = cache_fplookup_impl(dvp, &fpl);
6030 out:
6031 	cache_fpl_smr_assert_not_entered(&fpl);
6032 	cache_fpl_assert_status(&fpl);
6033 	*status = fpl.status;
6034 	if (SDT_PROBES_ENABLED()) {
6035 		SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);
6036 		if (fpl.status == CACHE_FPL_STATUS_HANDLED)
6037 			SDT_PROBE4(vfs, namei, lookup, return, error, ndp->ni_vp, true,
6038 			    ndp);
6039 	}
6040 
6041 	if (__predict_true(fpl.status == CACHE_FPL_STATUS_HANDLED)) {
6042 		MPASS(error != CACHE_FPL_FAILED);
6043 		if (error != 0) {
6044 			MPASS(fpl.dvp == NULL);
6045 			MPASS(fpl.tvp == NULL);
6046 			MPASS(fpl.savename == false);
6047 		}
6048 		ndp->ni_dvp = fpl.dvp;
6049 		ndp->ni_vp = fpl.tvp;
6050 		if (fpl.savename) {
6051 			cnp->cn_flags |= HASBUF;
6052 		} else {
6053 			cache_fpl_cleanup_cnp(cnp);
6054 		}
6055 	}
6056 	return (error);
6057 }
6058