xref: /freebsd/sys/kern/vfs_cache.c (revision 656f49f8e2b0656824a5f10aeb760a00fdd3753f)
1 /*-
2  * Copyright (c) 1989, 1993, 1995
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * Poul-Henning Kamp of the FreeBSD Project.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 4. Neither the name of the University nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  *
32  *	@(#)vfs_cache.c	8.5 (Berkeley) 3/22/95
33  */
34 
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
37 
38 #include "opt_ktrace.h"
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/filedesc.h>
43 #include <sys/fnv_hash.h>
44 #include <sys/kernel.h>
45 #include <sys/lock.h>
46 #include <sys/malloc.h>
47 #include <sys/fcntl.h>
48 #include <sys/mount.h>
49 #include <sys/namei.h>
50 #include <sys/proc.h>
51 #include <sys/rwlock.h>
52 #include <sys/sdt.h>
53 #include <sys/syscallsubr.h>
54 #include <sys/sysctl.h>
55 #include <sys/sysproto.h>
56 #include <sys/vnode.h>
57 #ifdef KTRACE
58 #include <sys/ktrace.h>
59 #endif
60 
61 #include <vm/uma.h>
62 
63 SDT_PROVIDER_DECLARE(vfs);
64 SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode *", "char *",
65     "struct vnode *");
66 SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *",
67     "char *");
68 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *");
69 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *",
70     "char *", "struct vnode *");
71 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *");
72 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int",
73     "struct vnode *", "char *");
74 SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode *", "char *",
75     "struct vnode *");
76 SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit__negative,
77     "struct vnode *", "char *");
78 SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *",
79     "char *");
80 SDT_PROBE_DEFINE1(vfs, namecache, purge, done, "struct vnode *");
81 SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *");
82 SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *");
83 SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode *", "char *",
84     "struct vnode *");
85 SDT_PROBE_DEFINE2(vfs, namecache, zap_negative, done, "struct vnode *",
86     "char *");
87 
88 /*
89  * This structure describes the elements in the cache of recent
90  * names looked up by namei.
91  */
92 
93 struct	namecache {
94 	LIST_ENTRY(namecache) nc_hash;	/* hash chain */
95 	LIST_ENTRY(namecache) nc_src;	/* source vnode list */
96 	TAILQ_ENTRY(namecache) nc_dst;	/* destination vnode list */
97 	struct	vnode *nc_dvp;		/* vnode of parent of name */
98 	struct	vnode *nc_vp;		/* vnode the name refers to */
99 	u_char	nc_flag;		/* flag bits */
100 	u_char	nc_nlen;		/* length of name */
101 	char	nc_name[0];		/* segment name + nul */
102 };
103 
104 /*
105  * struct namecache_ts repeats struct namecache layout up to the
106  * nc_nlen member.
107  * struct namecache_ts is used in place of struct namecache when time(s) need
108  * to be stored.  The nc_dotdottime field is used when a cache entry is mapping
109  * both a non-dotdot directory name plus dotdot for the directory's
110  * parent.
111  */
112 struct	namecache_ts {
113 	LIST_ENTRY(namecache) nc_hash;	/* hash chain */
114 	LIST_ENTRY(namecache) nc_src;	/* source vnode list */
115 	TAILQ_ENTRY(namecache) nc_dst;	/* destination vnode list */
116 	struct	vnode *nc_dvp;		/* vnode of parent of name */
117 	struct	vnode *nc_vp;		/* vnode the name refers to */
118 	u_char	nc_flag;		/* flag bits */
119 	u_char	nc_nlen;		/* length of name */
120 	struct	timespec nc_time;	/* timespec provided by fs */
121 	struct	timespec nc_dotdottime;	/* dotdot timespec provided by fs */
122 	int	nc_ticks;		/* ticks value when entry was added */
123 	char	nc_name[0];		/* segment name + nul */
124 };
125 
126 /*
127  * Flags in namecache.nc_flag
128  */
129 #define NCF_WHITE	0x01
130 #define NCF_ISDOTDOT	0x02
131 #define	NCF_TS		0x04
132 #define	NCF_DTS		0x08
133 
134 /*
135  * Name caching works as follows:
136  *
137  * Names found by directory scans are retained in a cache
138  * for future reference.  It is managed LRU, so frequently
139  * used names will hang around.  Cache is indexed by hash value
140  * obtained from (vp, name) where vp refers to the directory
141  * containing name.
142  *
143  * If it is a "negative" entry, (i.e. for a name that is known NOT to
144  * exist) the vnode pointer will be NULL.
145  *
146  * Upon reaching the last segment of a path, if the reference
147  * is for DELETE, or NOCACHE is set (rewrite), and the
148  * name is located in the cache, it will be dropped.
149  */
150 
151 /*
152  * Structures associated with name cacheing.
153  */
154 #define NCHHASH(hash) \
155 	(&nchashtbl[(hash) & nchash])
156 static LIST_HEAD(nchashhead, namecache) *nchashtbl;	/* Hash Table */
157 static TAILQ_HEAD(, namecache) ncneg;	/* Hash Table */
158 static u_long	nchash;			/* size of hash table */
159 SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0,
160     "Size of namecache hash table");
161 static u_long	ncnegfactor = 16;	/* ratio of negative entries */
162 SYSCTL_ULONG(_vfs, OID_AUTO, ncnegfactor, CTLFLAG_RW, &ncnegfactor, 0,
163     "Ratio of negative namecache entries");
164 static u_long	numneg;			/* number of negative entries allocated */
165 SYSCTL_ULONG(_debug, OID_AUTO, numneg, CTLFLAG_RD, &numneg, 0,
166     "Number of negative entries in namecache");
167 static u_long	numcache;		/* number of cache entries allocated */
168 SYSCTL_ULONG(_debug, OID_AUTO, numcache, CTLFLAG_RD, &numcache, 0,
169     "Number of namecache entries");
170 static u_long	numcachehv;		/* number of cache entries with vnodes held */
171 SYSCTL_ULONG(_debug, OID_AUTO, numcachehv, CTLFLAG_RD, &numcachehv, 0,
172     "Number of namecache entries with vnodes held");
173 static u_int	ncsizefactor = 2;
174 SYSCTL_UINT(_vfs, OID_AUTO, ncsizefactor, CTLFLAG_RW, &ncsizefactor, 0,
175     "Size factor for namecache");
176 
177 struct nchstats	nchstats;		/* cache effectiveness statistics */
178 
179 static struct rwlock cache_lock;
180 RW_SYSINIT(vfscache, &cache_lock, "Name Cache");
181 
182 #define	CACHE_UPGRADE_LOCK()	rw_try_upgrade(&cache_lock)
183 #define	CACHE_RLOCK()		rw_rlock(&cache_lock)
184 #define	CACHE_RUNLOCK()		rw_runlock(&cache_lock)
185 #define	CACHE_WLOCK()		rw_wlock(&cache_lock)
186 #define	CACHE_WUNLOCK()		rw_wunlock(&cache_lock)
187 
188 /*
189  * UMA zones for the VFS cache.
190  *
191  * The small cache is used for entries with short names, which are the
192  * most common.  The large cache is used for entries which are too big to
193  * fit in the small cache.
194  */
195 static uma_zone_t cache_zone_small;
196 static uma_zone_t cache_zone_small_ts;
197 static uma_zone_t cache_zone_large;
198 static uma_zone_t cache_zone_large_ts;
199 
200 #define	CACHE_PATH_CUTOFF	35
201 
202 static struct namecache *
203 cache_alloc(int len, int ts)
204 {
205 
206 	if (len > CACHE_PATH_CUTOFF) {
207 		if (ts)
208 			return (uma_zalloc(cache_zone_large_ts, M_WAITOK));
209 		else
210 			return (uma_zalloc(cache_zone_large, M_WAITOK));
211 	}
212 	if (ts)
213 		return (uma_zalloc(cache_zone_small_ts, M_WAITOK));
214 	else
215 		return (uma_zalloc(cache_zone_small, M_WAITOK));
216 }
217 
218 static void
219 cache_free(struct namecache *ncp)
220 {
221 	int ts;
222 
223 	if (ncp == NULL)
224 		return;
225 	ts = ncp->nc_flag & NCF_TS;
226 	if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) {
227 		if (ts)
228 			uma_zfree(cache_zone_small_ts, ncp);
229 		else
230 			uma_zfree(cache_zone_small, ncp);
231 	} else if (ts)
232 		uma_zfree(cache_zone_large_ts, ncp);
233 	else
234 		uma_zfree(cache_zone_large, ncp);
235 }
236 
237 static char *
238 nc_get_name(struct namecache *ncp)
239 {
240 	struct namecache_ts *ncp_ts;
241 
242 	if ((ncp->nc_flag & NCF_TS) == 0)
243 		return (ncp->nc_name);
244 	ncp_ts = (struct namecache_ts *)ncp;
245 	return (ncp_ts->nc_name);
246 }
247 
248 static void
249 cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp)
250 {
251 
252 	KASSERT((ncp->nc_flag & NCF_TS) != 0 ||
253 	    (tsp == NULL && ticksp == NULL),
254 	    ("No NCF_TS"));
255 
256 	if (tsp != NULL)
257 		*tsp = ((struct namecache_ts *)ncp)->nc_time;
258 	if (ticksp != NULL)
259 		*ticksp = ((struct namecache_ts *)ncp)->nc_ticks;
260 }
261 
262 static int	doingcache = 1;		/* 1 => enable the cache */
263 SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0,
264     "VFS namecache enabled");
265 
266 /* Export size information to userland */
267 SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR,
268     sizeof(struct namecache), "sizeof(struct namecache)");
269 
270 /*
271  * The new name cache statistics
272  */
273 static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW, 0,
274     "Name cache statistics");
275 #define STATNODE(mode, name, var, descr) \
276 	SYSCTL_ULONG(_vfs_cache, OID_AUTO, name, mode, var, 0, descr);
277 STATNODE(CTLFLAG_RD, numneg, &numneg, "Number of negative cache entries");
278 STATNODE(CTLFLAG_RD, numcache, &numcache, "Number of cache entries");
279 static u_long numcalls; STATNODE(CTLFLAG_RD, numcalls, &numcalls,
280     "Number of cache lookups");
281 static u_long dothits; STATNODE(CTLFLAG_RD, dothits, &dothits,
282     "Number of '.' hits");
283 static u_long dotdothits; STATNODE(CTLFLAG_RD, dotdothits, &dotdothits,
284     "Number of '..' hits");
285 static u_long numchecks; STATNODE(CTLFLAG_RD, numchecks, &numchecks,
286     "Number of checks in lookup");
287 static u_long nummiss; STATNODE(CTLFLAG_RD, nummiss, &nummiss,
288     "Number of cache misses");
289 static u_long nummisszap; STATNODE(CTLFLAG_RD, nummisszap, &nummisszap,
290     "Number of cache misses we do not want to cache");
291 static u_long numposzaps; STATNODE(CTLFLAG_RD, numposzaps, &numposzaps,
292     "Number of cache hits (positive) we do not want to cache");
293 static u_long numposhits; STATNODE(CTLFLAG_RD, numposhits, &numposhits,
294     "Number of cache hits (positive)");
295 static u_long numnegzaps; STATNODE(CTLFLAG_RD, numnegzaps, &numnegzaps,
296     "Number of cache hits (negative) we do not want to cache");
297 static u_long numneghits; STATNODE(CTLFLAG_RD, numneghits, &numneghits,
298     "Number of cache hits (negative)");
299 static u_long numupgrades; STATNODE(CTLFLAG_RD, numupgrades, &numupgrades,
300     "Number of updates of the cache after lookup (write lock + retry)");
301 
302 SYSCTL_OPAQUE(_vfs_cache, OID_AUTO, nchstats, CTLFLAG_RD | CTLFLAG_MPSAFE,
303     &nchstats, sizeof(nchstats), "LU",
304     "VFS cache effectiveness statistics");
305 
306 
307 
308 static void cache_zap(struct namecache *ncp);
309 static int vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
310     u_int *buflen);
311 static int vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
312     char *buf, char **retbuf, u_int buflen);
313 
314 static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");
315 
316 #ifdef DIAGNOSTIC
317 /*
318  * Grab an atomic snapshot of the name cache hash chain lengths
319  */
320 static SYSCTL_NODE(_debug, OID_AUTO, hashstat, CTLFLAG_RW, NULL,
321     "hash table stats");
322 
323 static int
324 sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS)
325 {
326 	struct nchashhead *ncpp;
327 	struct namecache *ncp;
328 	int i, error, n_nchash, *cntbuf;
329 
330 retry:
331 	n_nchash = nchash + 1;	/* nchash is max index, not count */
332 	if (req->oldptr == NULL)
333 		return SYSCTL_OUT(req, 0, n_nchash * sizeof(int));
334 	cntbuf = malloc(n_nchash * sizeof(int), M_TEMP, M_ZERO | M_WAITOK);
335 	CACHE_RLOCK();
336 	if (n_nchash != nchash + 1) {
337 		CACHE_RUNLOCK();
338 		free(cntbuf, M_TEMP);
339 		goto retry;
340 	}
341 	/* Scan hash tables counting entries */
342 	for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++)
343 		LIST_FOREACH(ncp, ncpp, nc_hash)
344 			cntbuf[i]++;
345 	CACHE_RUNLOCK();
346 	for (error = 0, i = 0; i < n_nchash; i++)
347 		if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0)
348 			break;
349 	free(cntbuf, M_TEMP);
350 	return (error);
351 }
352 SYSCTL_PROC(_debug_hashstat, OID_AUTO, rawnchash, CTLTYPE_INT|CTLFLAG_RD|
353     CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_rawnchash, "S,int",
354     "nchash chain lengths");
355 
356 static int
357 sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS)
358 {
359 	int error;
360 	struct nchashhead *ncpp;
361 	struct namecache *ncp;
362 	int n_nchash;
363 	int count, maxlength, used, pct;
364 
365 	if (!req->oldptr)
366 		return SYSCTL_OUT(req, 0, 4 * sizeof(int));
367 
368 	CACHE_RLOCK();
369 	n_nchash = nchash + 1;	/* nchash is max index, not count */
370 	used = 0;
371 	maxlength = 0;
372 
373 	/* Scan hash tables for applicable entries */
374 	for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) {
375 		count = 0;
376 		LIST_FOREACH(ncp, ncpp, nc_hash) {
377 			count++;
378 		}
379 		if (count)
380 			used++;
381 		if (maxlength < count)
382 			maxlength = count;
383 	}
384 	n_nchash = nchash + 1;
385 	CACHE_RUNLOCK();
386 	pct = (used * 100) / (n_nchash / 100);
387 	error = SYSCTL_OUT(req, &n_nchash, sizeof(n_nchash));
388 	if (error)
389 		return (error);
390 	error = SYSCTL_OUT(req, &used, sizeof(used));
391 	if (error)
392 		return (error);
393 	error = SYSCTL_OUT(req, &maxlength, sizeof(maxlength));
394 	if (error)
395 		return (error);
396 	error = SYSCTL_OUT(req, &pct, sizeof(pct));
397 	if (error)
398 		return (error);
399 	return (0);
400 }
401 SYSCTL_PROC(_debug_hashstat, OID_AUTO, nchash, CTLTYPE_INT|CTLFLAG_RD|
402     CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_nchash, "I",
403     "nchash statistics (number of total/used buckets, maximum chain length, usage percentage)");
404 #endif
405 
406 /*
407  * cache_zap():
408  *
409  *   Removes a namecache entry from cache, whether it contains an actual
410  *   pointer to a vnode or if it is just a negative cache entry.
411  */
412 static void
413 cache_zap(ncp)
414 	struct namecache *ncp;
415 {
416 	struct vnode *vp;
417 
418 	rw_assert(&cache_lock, RA_WLOCKED);
419 	CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp, ncp->nc_vp);
420 #ifdef KDTRACE_HOOKS
421 	if (ncp->nc_vp != NULL) {
422 		SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp,
423 		    nc_get_name(ncp), ncp->nc_vp);
424 	} else {
425 		SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp,
426 		    nc_get_name(ncp));
427 	}
428 #endif
429 	vp = NULL;
430 	LIST_REMOVE(ncp, nc_hash);
431 	if (ncp->nc_flag & NCF_ISDOTDOT) {
432 		if (ncp == ncp->nc_dvp->v_cache_dd)
433 			ncp->nc_dvp->v_cache_dd = NULL;
434 	} else {
435 		LIST_REMOVE(ncp, nc_src);
436 		if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) {
437 			vp = ncp->nc_dvp;
438 			numcachehv--;
439 		}
440 	}
441 	if (ncp->nc_vp) {
442 		TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst);
443 		if (ncp == ncp->nc_vp->v_cache_dd)
444 			ncp->nc_vp->v_cache_dd = NULL;
445 	} else {
446 		TAILQ_REMOVE(&ncneg, ncp, nc_dst);
447 		numneg--;
448 	}
449 	numcache--;
450 	cache_free(ncp);
451 	if (vp)
452 		vdrop(vp);
453 }
454 
455 /*
456  * Lookup an entry in the cache
457  *
458  * Lookup is called with dvp pointing to the directory to search,
459  * cnp pointing to the name of the entry being sought. If the lookup
460  * succeeds, the vnode is returned in *vpp, and a status of -1 is
461  * returned. If the lookup determines that the name does not exist
462  * (negative cacheing), a status of ENOENT is returned. If the lookup
463  * fails, a status of zero is returned.  If the directory vnode is
464  * recycled out from under us due to a forced unmount, a status of
465  * ENOENT is returned.
466  *
467  * vpp is locked and ref'd on return.  If we're looking up DOTDOT, dvp is
468  * unlocked.  If we're looking up . an extra ref is taken, but the lock is
469  * not recursively acquired.
470  */
471 
472 int
473 cache_lookup(dvp, vpp, cnp, tsp, ticksp)
474 	struct vnode *dvp;
475 	struct vnode **vpp;
476 	struct componentname *cnp;
477 	struct timespec *tsp;
478 	int *ticksp;
479 {
480 	struct namecache *ncp;
481 	uint32_t hash;
482 	int error, ltype, wlocked;
483 
484 	if (!doingcache) {
485 		cnp->cn_flags &= ~MAKEENTRY;
486 		return (0);
487 	}
488 retry:
489 	CACHE_RLOCK();
490 	wlocked = 0;
491 	numcalls++;
492 	error = 0;
493 
494 retry_wlocked:
495 	if (cnp->cn_nameptr[0] == '.') {
496 		if (cnp->cn_namelen == 1) {
497 			*vpp = dvp;
498 			CTR2(KTR_VFS, "cache_lookup(%p, %s) found via .",
499 			    dvp, cnp->cn_nameptr);
500 			dothits++;
501 			SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp);
502 			if (tsp != NULL)
503 				timespecclear(tsp);
504 			if (ticksp != NULL)
505 				*ticksp = ticks;
506 			goto success;
507 		}
508 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
509 			dotdothits++;
510 			if (dvp->v_cache_dd == NULL) {
511 				SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
512 				    "..", NULL);
513 				goto unlock;
514 			}
515 			if ((cnp->cn_flags & MAKEENTRY) == 0) {
516 				if (!wlocked && !CACHE_UPGRADE_LOCK())
517 					goto wlock;
518 				if (dvp->v_cache_dd->nc_flag & NCF_ISDOTDOT)
519 					cache_zap(dvp->v_cache_dd);
520 				dvp->v_cache_dd = NULL;
521 				CACHE_WUNLOCK();
522 				return (0);
523 			}
524 			ncp = dvp->v_cache_dd;
525 			if (ncp->nc_flag & NCF_ISDOTDOT)
526 				*vpp = ncp->nc_vp;
527 			else
528 				*vpp = ncp->nc_dvp;
529 			/* Return failure if negative entry was found. */
530 			if (*vpp == NULL)
531 				goto negative_success;
532 			CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..",
533 			    dvp, cnp->cn_nameptr, *vpp);
534 			SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..",
535 			    *vpp);
536 			cache_out_ts(ncp, tsp, ticksp);
537 			if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
538 			    NCF_DTS && tsp != NULL)
539 				*tsp = ((struct namecache_ts *)ncp)->
540 				    nc_dotdottime;
541 			goto success;
542 		}
543 	}
544 
545 	hash = fnv_32_buf(cnp->cn_nameptr, cnp->cn_namelen, FNV1_32_INIT);
546 	hash = fnv_32_buf(&dvp, sizeof(dvp), hash);
547 	LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
548 		numchecks++;
549 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
550 		    !bcmp(nc_get_name(ncp), cnp->cn_nameptr, ncp->nc_nlen))
551 			break;
552 	}
553 
554 	/* We failed to find an entry */
555 	if (ncp == NULL) {
556 		SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
557 		    NULL);
558 		if ((cnp->cn_flags & MAKEENTRY) == 0) {
559 			nummisszap++;
560 		} else {
561 			nummiss++;
562 		}
563 		nchstats.ncs_miss++;
564 		goto unlock;
565 	}
566 
567 	/* We don't want to have an entry, so dump it */
568 	if ((cnp->cn_flags & MAKEENTRY) == 0) {
569 		numposzaps++;
570 		nchstats.ncs_badhits++;
571 		if (!wlocked && !CACHE_UPGRADE_LOCK())
572 			goto wlock;
573 		cache_zap(ncp);
574 		CACHE_WUNLOCK();
575 		return (0);
576 	}
577 
578 	/* We found a "positive" match, return the vnode */
579 	if (ncp->nc_vp) {
580 		numposhits++;
581 		nchstats.ncs_goodhits++;
582 		*vpp = ncp->nc_vp;
583 		CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
584 		    dvp, cnp->cn_nameptr, *vpp, ncp);
585 		SDT_PROBE3(vfs, namecache, lookup, hit, dvp, nc_get_name(ncp),
586 		    *vpp);
587 		cache_out_ts(ncp, tsp, ticksp);
588 		goto success;
589 	}
590 
591 negative_success:
592 	/* We found a negative match, and want to create it, so purge */
593 	if (cnp->cn_nameiop == CREATE) {
594 		numnegzaps++;
595 		nchstats.ncs_badhits++;
596 		if (!wlocked && !CACHE_UPGRADE_LOCK())
597 			goto wlock;
598 		cache_zap(ncp);
599 		CACHE_WUNLOCK();
600 		return (0);
601 	}
602 
603 	if (!wlocked && !CACHE_UPGRADE_LOCK())
604 		goto wlock;
605 	numneghits++;
606 	/*
607 	 * We found a "negative" match, so we shift it to the end of
608 	 * the "negative" cache entries queue to satisfy LRU.  Also,
609 	 * check to see if the entry is a whiteout; indicate this to
610 	 * the componentname, if so.
611 	 */
612 	TAILQ_REMOVE(&ncneg, ncp, nc_dst);
613 	TAILQ_INSERT_TAIL(&ncneg, ncp, nc_dst);
614 	nchstats.ncs_neghits++;
615 	if (ncp->nc_flag & NCF_WHITE)
616 		cnp->cn_flags |= ISWHITEOUT;
617 	SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp,
618 	    nc_get_name(ncp));
619 	cache_out_ts(ncp, tsp, ticksp);
620 	CACHE_WUNLOCK();
621 	return (ENOENT);
622 
623 wlock:
624 	/*
625 	 * We need to update the cache after our lookup, so upgrade to
626 	 * a write lock and retry the operation.
627 	 */
628 	CACHE_RUNLOCK();
629 	CACHE_WLOCK();
630 	numupgrades++;
631 	wlocked = 1;
632 	goto retry_wlocked;
633 
634 success:
635 	/*
636 	 * On success we return a locked and ref'd vnode as per the lookup
637 	 * protocol.
638 	 */
639 	if (dvp == *vpp) {   /* lookup on "." */
640 		VREF(*vpp);
641 		if (wlocked)
642 			CACHE_WUNLOCK();
643 		else
644 			CACHE_RUNLOCK();
645 		/*
646 		 * When we lookup "." we still can be asked to lock it
647 		 * differently...
648 		 */
649 		ltype = cnp->cn_lkflags & LK_TYPE_MASK;
650 		if (ltype != VOP_ISLOCKED(*vpp)) {
651 			if (ltype == LK_EXCLUSIVE) {
652 				vn_lock(*vpp, LK_UPGRADE | LK_RETRY);
653 				if ((*vpp)->v_iflag & VI_DOOMED) {
654 					/* forced unmount */
655 					vrele(*vpp);
656 					*vpp = NULL;
657 					return (ENOENT);
658 				}
659 			} else
660 				vn_lock(*vpp, LK_DOWNGRADE | LK_RETRY);
661 		}
662 		return (-1);
663 	}
664 	ltype = 0;	/* silence gcc warning */
665 	if (cnp->cn_flags & ISDOTDOT) {
666 		ltype = VOP_ISLOCKED(dvp);
667 		VOP_UNLOCK(dvp, 0);
668 	}
669 	vhold(*vpp);
670 	if (wlocked)
671 		CACHE_WUNLOCK();
672 	else
673 		CACHE_RUNLOCK();
674 	error = vget(*vpp, cnp->cn_lkflags | LK_VNHELD, cnp->cn_thread);
675 	if (cnp->cn_flags & ISDOTDOT) {
676 		vn_lock(dvp, ltype | LK_RETRY);
677 		if (dvp->v_iflag & VI_DOOMED) {
678 			if (error == 0)
679 				vput(*vpp);
680 			*vpp = NULL;
681 			return (ENOENT);
682 		}
683 	}
684 	if (error) {
685 		*vpp = NULL;
686 		goto retry;
687 	}
688 	if ((cnp->cn_flags & ISLASTCN) &&
689 	    (cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) {
690 		ASSERT_VOP_ELOCKED(*vpp, "cache_lookup");
691 	}
692 	return (-1);
693 
694 unlock:
695 	if (wlocked)
696 		CACHE_WUNLOCK();
697 	else
698 		CACHE_RUNLOCK();
699 	return (0);
700 }
701 
702 /*
703  * Add an entry to the cache.
704  */
705 void
706 cache_enter_time(dvp, vp, cnp, tsp, dtsp)
707 	struct vnode *dvp;
708 	struct vnode *vp;
709 	struct componentname *cnp;
710 	struct timespec *tsp;
711 	struct timespec *dtsp;
712 {
713 	struct namecache *ncp, *n2;
714 	struct namecache_ts *n3;
715 	struct nchashhead *ncpp;
716 	uint32_t hash;
717 	int flag;
718 	int hold;
719 	int zap;
720 	int len;
721 
722 	CTR3(KTR_VFS, "cache_enter(%p, %p, %s)", dvp, vp, cnp->cn_nameptr);
723 	VNASSERT(vp == NULL || (vp->v_iflag & VI_DOOMED) == 0, vp,
724 	    ("cache_enter: Adding a doomed vnode"));
725 	VNASSERT(dvp == NULL || (dvp->v_iflag & VI_DOOMED) == 0, dvp,
726 	    ("cache_enter: Doomed vnode used as src"));
727 
728 	if (!doingcache)
729 		return;
730 
731 	/*
732 	 * Avoid blowout in namecache entries.
733 	 */
734 	if (numcache >= desiredvnodes * ncsizefactor)
735 		return;
736 
737 	flag = 0;
738 	if (cnp->cn_nameptr[0] == '.') {
739 		if (cnp->cn_namelen == 1)
740 			return;
741 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
742 			CACHE_WLOCK();
743 			/*
744 			 * If dotdot entry already exists, just retarget it
745 			 * to new parent vnode, otherwise continue with new
746 			 * namecache entry allocation.
747 			 */
748 			if ((ncp = dvp->v_cache_dd) != NULL &&
749 			    ncp->nc_flag & NCF_ISDOTDOT) {
750 				KASSERT(ncp->nc_dvp == dvp,
751 				    ("wrong isdotdot parent"));
752 				if (ncp->nc_vp != NULL) {
753 					TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst,
754 					    ncp, nc_dst);
755 				} else {
756 					TAILQ_REMOVE(&ncneg, ncp, nc_dst);
757 					numneg--;
758 				}
759 				if (vp != NULL) {
760 					TAILQ_INSERT_HEAD(&vp->v_cache_dst,
761 					    ncp, nc_dst);
762 				} else {
763 					TAILQ_INSERT_TAIL(&ncneg, ncp, nc_dst);
764 					numneg++;
765 				}
766 				ncp->nc_vp = vp;
767 				CACHE_WUNLOCK();
768 				return;
769 			}
770 			dvp->v_cache_dd = NULL;
771 			SDT_PROBE3(vfs, namecache, enter, done, dvp, "..", vp);
772 			CACHE_WUNLOCK();
773 			flag = NCF_ISDOTDOT;
774 		}
775 	}
776 
777 	hold = 0;
778 	zap = 0;
779 
780 	/*
781 	 * Calculate the hash key and setup as much of the new
782 	 * namecache entry as possible before acquiring the lock.
783 	 */
784 	ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
785 	ncp->nc_vp = vp;
786 	ncp->nc_dvp = dvp;
787 	ncp->nc_flag = flag;
788 	if (tsp != NULL) {
789 		n3 = (struct namecache_ts *)ncp;
790 		n3->nc_time = *tsp;
791 		n3->nc_ticks = ticks;
792 		n3->nc_flag |= NCF_TS;
793 		if (dtsp != NULL) {
794 			n3->nc_dotdottime = *dtsp;
795 			n3->nc_flag |= NCF_DTS;
796 		}
797 	}
798 	len = ncp->nc_nlen = cnp->cn_namelen;
799 	hash = fnv_32_buf(cnp->cn_nameptr, len, FNV1_32_INIT);
800 	strlcpy(nc_get_name(ncp), cnp->cn_nameptr, len + 1);
801 	hash = fnv_32_buf(&dvp, sizeof(dvp), hash);
802 	CACHE_WLOCK();
803 
804 	/*
805 	 * See if this vnode or negative entry is already in the cache
806 	 * with this name.  This can happen with concurrent lookups of
807 	 * the same path name.
808 	 */
809 	ncpp = NCHHASH(hash);
810 	LIST_FOREACH(n2, ncpp, nc_hash) {
811 		if (n2->nc_dvp == dvp &&
812 		    n2->nc_nlen == cnp->cn_namelen &&
813 		    !bcmp(nc_get_name(n2), cnp->cn_nameptr, n2->nc_nlen)) {
814 			if (tsp != NULL) {
815 				KASSERT((n2->nc_flag & NCF_TS) != 0,
816 				    ("no NCF_TS"));
817 				n3 = (struct namecache_ts *)n2;
818 				n3->nc_time =
819 				    ((struct namecache_ts *)ncp)->nc_time;
820 				n3->nc_ticks =
821 				    ((struct namecache_ts *)ncp)->nc_ticks;
822 				if (dtsp != NULL) {
823 					n3->nc_dotdottime =
824 					    ((struct namecache_ts *)ncp)->
825 					    nc_dotdottime;
826 					n3->nc_flag |= NCF_DTS;
827 				}
828 			}
829 			CACHE_WUNLOCK();
830 			cache_free(ncp);
831 			return;
832 		}
833 	}
834 
835 	if (flag == NCF_ISDOTDOT) {
836 		/*
837 		 * See if we are trying to add .. entry, but some other lookup
838 		 * has populated v_cache_dd pointer already.
839 		 */
840 		if (dvp->v_cache_dd != NULL) {
841 		    CACHE_WUNLOCK();
842 		    cache_free(ncp);
843 		    return;
844 		}
845 		KASSERT(vp == NULL || vp->v_type == VDIR,
846 		    ("wrong vnode type %p", vp));
847 		dvp->v_cache_dd = ncp;
848 	}
849 
850 	numcache++;
851 	if (!vp) {
852 		numneg++;
853 		if (cnp->cn_flags & ISWHITEOUT)
854 			ncp->nc_flag |= NCF_WHITE;
855 	} else if (vp->v_type == VDIR) {
856 		if (flag != NCF_ISDOTDOT) {
857 			/*
858 			 * For this case, the cache entry maps both the
859 			 * directory name in it and the name ".." for the
860 			 * directory's parent.
861 			 */
862 			if ((n2 = vp->v_cache_dd) != NULL &&
863 			    (n2->nc_flag & NCF_ISDOTDOT) != 0)
864 				cache_zap(n2);
865 			vp->v_cache_dd = ncp;
866 		}
867 	} else {
868 		vp->v_cache_dd = NULL;
869 	}
870 
871 	/*
872 	 * Insert the new namecache entry into the appropriate chain
873 	 * within the cache entries table.
874 	 */
875 	LIST_INSERT_HEAD(ncpp, ncp, nc_hash);
876 	if (flag != NCF_ISDOTDOT) {
877 		if (LIST_EMPTY(&dvp->v_cache_src)) {
878 			hold = 1;
879 			numcachehv++;
880 		}
881 		LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
882 	}
883 
884 	/*
885 	 * If the entry is "negative", we place it into the
886 	 * "negative" cache queue, otherwise, we place it into the
887 	 * destination vnode's cache entries queue.
888 	 */
889 	if (vp) {
890 		TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
891 		SDT_PROBE3(vfs, namecache, enter, done, dvp, nc_get_name(ncp),
892 		    vp);
893 	} else {
894 		TAILQ_INSERT_TAIL(&ncneg, ncp, nc_dst);
895 		SDT_PROBE2(vfs, namecache, enter_negative, done, dvp,
896 		    nc_get_name(ncp));
897 	}
898 	if (numneg * ncnegfactor > numcache) {
899 		ncp = TAILQ_FIRST(&ncneg);
900 		KASSERT(ncp->nc_vp == NULL, ("ncp %p vp %p on ncneg",
901 		    ncp, ncp->nc_vp));
902 		zap = 1;
903 	}
904 	if (hold)
905 		vhold(dvp);
906 	if (zap)
907 		cache_zap(ncp);
908 	CACHE_WUNLOCK();
909 }
910 
911 /*
912  * Name cache initialization, from vfs_init() when we are booting
913  */
914 static void
915 nchinit(void *dummy __unused)
916 {
917 
918 	TAILQ_INIT(&ncneg);
919 
920 	cache_zone_small = uma_zcreate("S VFS Cache",
921 	    sizeof(struct namecache) + CACHE_PATH_CUTOFF + 1,
922 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
923 	cache_zone_small_ts = uma_zcreate("STS VFS Cache",
924 	    sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1,
925 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
926 	cache_zone_large = uma_zcreate("L VFS Cache",
927 	    sizeof(struct namecache) + NAME_MAX + 1,
928 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
929 	cache_zone_large_ts = uma_zcreate("LTS VFS Cache",
930 	    sizeof(struct namecache_ts) + NAME_MAX + 1,
931 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
932 
933 	nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash);
934 }
935 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nchinit, NULL);
936 
937 void
938 cache_changesize(int newmaxvnodes)
939 {
940 	struct nchashhead *new_nchashtbl, *old_nchashtbl;
941 	u_long new_nchash, old_nchash;
942 	struct namecache *ncp;
943 	uint32_t hash;
944 	int i;
945 
946 	new_nchashtbl = hashinit(newmaxvnodes * 2, M_VFSCACHE, &new_nchash);
947 	/* If same hash table size, nothing to do */
948 	if (nchash == new_nchash) {
949 		free(new_nchashtbl, M_VFSCACHE);
950 		return;
951 	}
952 	/*
953 	 * Move everything from the old hash table to the new table.
954 	 * None of the namecache entries in the table can be removed
955 	 * because to do so, they have to be removed from the hash table.
956 	 */
957 	CACHE_WLOCK();
958 	old_nchashtbl = nchashtbl;
959 	old_nchash = nchash;
960 	nchashtbl = new_nchashtbl;
961 	nchash = new_nchash;
962 	for (i = 0; i <= old_nchash; i++) {
963 		while ((ncp = LIST_FIRST(&old_nchashtbl[i])) != NULL) {
964 			hash = fnv_32_buf(nc_get_name(ncp), ncp->nc_nlen,
965 			    FNV1_32_INIT);
966 			hash = fnv_32_buf(&ncp->nc_dvp, sizeof(ncp->nc_dvp),
967 			    hash);
968 			LIST_REMOVE(ncp, nc_hash);
969 			LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash);
970 		}
971 	}
972 	CACHE_WUNLOCK();
973 	free(old_nchashtbl, M_VFSCACHE);
974 }
975 
976 /*
977  * Invalidate all entries to a particular vnode.
978  */
979 void
980 cache_purge(vp)
981 	struct vnode *vp;
982 {
983 
984 	CTR1(KTR_VFS, "cache_purge(%p)", vp);
985 	SDT_PROBE1(vfs, namecache, purge, done, vp);
986 	CACHE_WLOCK();
987 	while (!LIST_EMPTY(&vp->v_cache_src))
988 		cache_zap(LIST_FIRST(&vp->v_cache_src));
989 	while (!TAILQ_EMPTY(&vp->v_cache_dst))
990 		cache_zap(TAILQ_FIRST(&vp->v_cache_dst));
991 	if (vp->v_cache_dd != NULL) {
992 		KASSERT(vp->v_cache_dd->nc_flag & NCF_ISDOTDOT,
993 		   ("lost dotdot link"));
994 		cache_zap(vp->v_cache_dd);
995 	}
996 	KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
997 	CACHE_WUNLOCK();
998 }
999 
1000 /*
1001  * Invalidate all negative entries for a particular directory vnode.
1002  */
1003 void
1004 cache_purge_negative(vp)
1005 	struct vnode *vp;
1006 {
1007 	struct namecache *cp, *ncp;
1008 
1009 	CTR1(KTR_VFS, "cache_purge_negative(%p)", vp);
1010 	SDT_PROBE1(vfs, namecache, purge_negative, done, vp);
1011 	CACHE_WLOCK();
1012 	LIST_FOREACH_SAFE(cp, &vp->v_cache_src, nc_src, ncp) {
1013 		if (cp->nc_vp == NULL)
1014 			cache_zap(cp);
1015 	}
1016 	CACHE_WUNLOCK();
1017 }
1018 
1019 /*
1020  * Flush all entries referencing a particular filesystem.
1021  */
1022 void
1023 cache_purgevfs(mp)
1024 	struct mount *mp;
1025 {
1026 	struct nchashhead *ncpp;
1027 	struct namecache *ncp, *nnp;
1028 
1029 	/* Scan hash tables for applicable entries */
1030 	SDT_PROBE1(vfs, namecache, purgevfs, done, mp);
1031 	CACHE_WLOCK();
1032 	for (ncpp = &nchashtbl[nchash]; ncpp >= nchashtbl; ncpp--) {
1033 		LIST_FOREACH_SAFE(ncp, ncpp, nc_hash, nnp) {
1034 			if (ncp->nc_dvp->v_mount == mp)
1035 				cache_zap(ncp);
1036 		}
1037 	}
1038 	CACHE_WUNLOCK();
1039 }
1040 
1041 /*
1042  * Perform canonical checks and cache lookup and pass on to filesystem
1043  * through the vop_cachedlookup only if needed.
1044  */
1045 
1046 int
1047 vfs_cache_lookup(ap)
1048 	struct vop_lookup_args /* {
1049 		struct vnode *a_dvp;
1050 		struct vnode **a_vpp;
1051 		struct componentname *a_cnp;
1052 	} */ *ap;
1053 {
1054 	struct vnode *dvp;
1055 	int error;
1056 	struct vnode **vpp = ap->a_vpp;
1057 	struct componentname *cnp = ap->a_cnp;
1058 	struct ucred *cred = cnp->cn_cred;
1059 	int flags = cnp->cn_flags;
1060 	struct thread *td = cnp->cn_thread;
1061 
1062 	*vpp = NULL;
1063 	dvp = ap->a_dvp;
1064 
1065 	if (dvp->v_type != VDIR)
1066 		return (ENOTDIR);
1067 
1068 	if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
1069 	    (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME))
1070 		return (EROFS);
1071 
1072 	error = VOP_ACCESS(dvp, VEXEC, cred, td);
1073 	if (error)
1074 		return (error);
1075 
1076 	error = cache_lookup(dvp, vpp, cnp, NULL, NULL);
1077 	if (error == 0)
1078 		return (VOP_CACHEDLOOKUP(dvp, vpp, cnp));
1079 	if (error == -1)
1080 		return (0);
1081 	return (error);
1082 }
1083 
1084 /*
1085  * XXX All of these sysctls would probably be more productive dead.
1086  */
1087 static int disablecwd;
1088 SYSCTL_INT(_debug, OID_AUTO, disablecwd, CTLFLAG_RW, &disablecwd, 0,
1089    "Disable the getcwd syscall");
1090 
1091 /* Implementation of the getcwd syscall. */
1092 int
1093 sys___getcwd(td, uap)
1094 	struct thread *td;
1095 	struct __getcwd_args *uap;
1096 {
1097 
1098 	return (kern___getcwd(td, uap->buf, UIO_USERSPACE, uap->buflen,
1099 	    MAXPATHLEN));
1100 }
1101 
1102 int
1103 kern___getcwd(struct thread *td, char *buf, enum uio_seg bufseg, u_int buflen,
1104     u_int path_max)
1105 {
1106 	char *bp, *tmpbuf;
1107 	struct filedesc *fdp;
1108 	struct vnode *cdir, *rdir;
1109 	int error;
1110 
1111 	if (disablecwd)
1112 		return (ENODEV);
1113 	if (buflen < 2)
1114 		return (EINVAL);
1115 	if (buflen > path_max)
1116 		buflen = path_max;
1117 
1118 	tmpbuf = malloc(buflen, M_TEMP, M_WAITOK);
1119 	fdp = td->td_proc->p_fd;
1120 	FILEDESC_SLOCK(fdp);
1121 	cdir = fdp->fd_cdir;
1122 	VREF(cdir);
1123 	rdir = fdp->fd_rdir;
1124 	VREF(rdir);
1125 	FILEDESC_SUNLOCK(fdp);
1126 	error = vn_fullpath1(td, cdir, rdir, tmpbuf, &bp, buflen);
1127 	vrele(rdir);
1128 	vrele(cdir);
1129 
1130 	if (!error) {
1131 		if (bufseg == UIO_SYSSPACE)
1132 			bcopy(bp, buf, strlen(bp) + 1);
1133 		else
1134 			error = copyout(bp, buf, strlen(bp) + 1);
1135 #ifdef KTRACE
1136 	if (KTRPOINT(curthread, KTR_NAMEI))
1137 		ktrnamei(bp);
1138 #endif
1139 	}
1140 	free(tmpbuf, M_TEMP);
1141 	return (error);
1142 }
1143 
1144 /*
1145  * Thus begins the fullpath magic.
1146  */
1147 
1148 #undef STATNODE
1149 #define STATNODE(name, descr)						\
1150 	static u_int name;						\
1151 	SYSCTL_UINT(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, 0, descr)
1152 
1153 static int disablefullpath;
1154 SYSCTL_INT(_debug, OID_AUTO, disablefullpath, CTLFLAG_RW, &disablefullpath, 0,
1155     "Disable the vn_fullpath function");
1156 
1157 /* These count for kern___getcwd(), too. */
1158 STATNODE(numfullpathcalls, "Number of fullpath search calls");
1159 STATNODE(numfullpathfail1, "Number of fullpath search errors (ENOTDIR)");
1160 STATNODE(numfullpathfail2,
1161     "Number of fullpath search errors (VOP_VPTOCNP failures)");
1162 STATNODE(numfullpathfail4, "Number of fullpath search errors (ENOMEM)");
1163 STATNODE(numfullpathfound, "Number of successful fullpath calls");
1164 
1165 /*
1166  * Retrieve the full filesystem path that correspond to a vnode from the name
1167  * cache (if available)
1168  */
1169 int
1170 vn_fullpath(struct thread *td, struct vnode *vn, char **retbuf, char **freebuf)
1171 {
1172 	char *buf;
1173 	struct filedesc *fdp;
1174 	struct vnode *rdir;
1175 	int error;
1176 
1177 	if (disablefullpath)
1178 		return (ENODEV);
1179 	if (vn == NULL)
1180 		return (EINVAL);
1181 
1182 	buf = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
1183 	fdp = td->td_proc->p_fd;
1184 	FILEDESC_SLOCK(fdp);
1185 	rdir = fdp->fd_rdir;
1186 	VREF(rdir);
1187 	FILEDESC_SUNLOCK(fdp);
1188 	error = vn_fullpath1(td, vn, rdir, buf, retbuf, MAXPATHLEN);
1189 	vrele(rdir);
1190 
1191 	if (!error)
1192 		*freebuf = buf;
1193 	else
1194 		free(buf, M_TEMP);
1195 	return (error);
1196 }
1197 
1198 /*
1199  * This function is similar to vn_fullpath, but it attempts to lookup the
1200  * pathname relative to the global root mount point.  This is required for the
1201  * auditing sub-system, as audited pathnames must be absolute, relative to the
1202  * global root mount point.
1203  */
1204 int
1205 vn_fullpath_global(struct thread *td, struct vnode *vn,
1206     char **retbuf, char **freebuf)
1207 {
1208 	char *buf;
1209 	int error;
1210 
1211 	if (disablefullpath)
1212 		return (ENODEV);
1213 	if (vn == NULL)
1214 		return (EINVAL);
1215 	buf = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
1216 	error = vn_fullpath1(td, vn, rootvnode, buf, retbuf, MAXPATHLEN);
1217 	if (!error)
1218 		*freebuf = buf;
1219 	else
1220 		free(buf, M_TEMP);
1221 	return (error);
1222 }
1223 
1224 int
1225 vn_vptocnp(struct vnode **vp, struct ucred *cred, char *buf, u_int *buflen)
1226 {
1227 	int error;
1228 
1229 	CACHE_RLOCK();
1230 	error = vn_vptocnp_locked(vp, cred, buf, buflen);
1231 	if (error == 0)
1232 		CACHE_RUNLOCK();
1233 	return (error);
1234 }
1235 
1236 static int
1237 vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
1238     u_int *buflen)
1239 {
1240 	struct vnode *dvp;
1241 	struct namecache *ncp;
1242 	int error;
1243 
1244 	TAILQ_FOREACH(ncp, &((*vp)->v_cache_dst), nc_dst) {
1245 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
1246 			break;
1247 	}
1248 	if (ncp != NULL) {
1249 		if (*buflen < ncp->nc_nlen) {
1250 			CACHE_RUNLOCK();
1251 			vrele(*vp);
1252 			numfullpathfail4++;
1253 			error = ENOMEM;
1254 			SDT_PROBE3(vfs, namecache, fullpath, return, error,
1255 			    vp, NULL);
1256 			return (error);
1257 		}
1258 		*buflen -= ncp->nc_nlen;
1259 		memcpy(buf + *buflen, nc_get_name(ncp), ncp->nc_nlen);
1260 		SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
1261 		    nc_get_name(ncp), vp);
1262 		dvp = *vp;
1263 		*vp = ncp->nc_dvp;
1264 		vref(*vp);
1265 		CACHE_RUNLOCK();
1266 		vrele(dvp);
1267 		CACHE_RLOCK();
1268 		return (0);
1269 	}
1270 	SDT_PROBE1(vfs, namecache, fullpath, miss, vp);
1271 
1272 	CACHE_RUNLOCK();
1273 	vn_lock(*vp, LK_SHARED | LK_RETRY);
1274 	error = VOP_VPTOCNP(*vp, &dvp, cred, buf, buflen);
1275 	vput(*vp);
1276 	if (error) {
1277 		numfullpathfail2++;
1278 		SDT_PROBE3(vfs, namecache, fullpath, return,  error, vp, NULL);
1279 		return (error);
1280 	}
1281 
1282 	*vp = dvp;
1283 	CACHE_RLOCK();
1284 	if (dvp->v_iflag & VI_DOOMED) {
1285 		/* forced unmount */
1286 		CACHE_RUNLOCK();
1287 		vrele(dvp);
1288 		error = ENOENT;
1289 		SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
1290 		return (error);
1291 	}
1292 	/*
1293 	 * *vp has its use count incremented still.
1294 	 */
1295 
1296 	return (0);
1297 }
1298 
1299 /*
1300  * The magic behind kern___getcwd() and vn_fullpath().
1301  */
1302 static int
1303 vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
1304     char *buf, char **retbuf, u_int buflen)
1305 {
1306 	int error, slash_prefixed;
1307 #ifdef KDTRACE_HOOKS
1308 	struct vnode *startvp = vp;
1309 #endif
1310 	struct vnode *vp1;
1311 
1312 	buflen--;
1313 	buf[buflen] = '\0';
1314 	error = 0;
1315 	slash_prefixed = 0;
1316 
1317 	SDT_PROBE1(vfs, namecache, fullpath, entry, vp);
1318 	numfullpathcalls++;
1319 	vref(vp);
1320 	CACHE_RLOCK();
1321 	if (vp->v_type != VDIR) {
1322 		error = vn_vptocnp_locked(&vp, td->td_ucred, buf, &buflen);
1323 		if (error)
1324 			return (error);
1325 		if (buflen == 0) {
1326 			CACHE_RUNLOCK();
1327 			vrele(vp);
1328 			return (ENOMEM);
1329 		}
1330 		buf[--buflen] = '/';
1331 		slash_prefixed = 1;
1332 	}
1333 	while (vp != rdir && vp != rootvnode) {
1334 		if (vp->v_vflag & VV_ROOT) {
1335 			if (vp->v_iflag & VI_DOOMED) {	/* forced unmount */
1336 				CACHE_RUNLOCK();
1337 				vrele(vp);
1338 				error = ENOENT;
1339 				SDT_PROBE3(vfs, namecache, fullpath, return,
1340 				    error, vp, NULL);
1341 				break;
1342 			}
1343 			vp1 = vp->v_mount->mnt_vnodecovered;
1344 			vref(vp1);
1345 			CACHE_RUNLOCK();
1346 			vrele(vp);
1347 			vp = vp1;
1348 			CACHE_RLOCK();
1349 			continue;
1350 		}
1351 		if (vp->v_type != VDIR) {
1352 			CACHE_RUNLOCK();
1353 			vrele(vp);
1354 			numfullpathfail1++;
1355 			error = ENOTDIR;
1356 			SDT_PROBE3(vfs, namecache, fullpath, return,
1357 			    error, vp, NULL);
1358 			break;
1359 		}
1360 		error = vn_vptocnp_locked(&vp, td->td_ucred, buf, &buflen);
1361 		if (error)
1362 			break;
1363 		if (buflen == 0) {
1364 			CACHE_RUNLOCK();
1365 			vrele(vp);
1366 			error = ENOMEM;
1367 			SDT_PROBE3(vfs, namecache, fullpath, return, error,
1368 			    startvp, NULL);
1369 			break;
1370 		}
1371 		buf[--buflen] = '/';
1372 		slash_prefixed = 1;
1373 	}
1374 	if (error)
1375 		return (error);
1376 	if (!slash_prefixed) {
1377 		if (buflen == 0) {
1378 			CACHE_RUNLOCK();
1379 			vrele(vp);
1380 			numfullpathfail4++;
1381 			SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM,
1382 			    startvp, NULL);
1383 			return (ENOMEM);
1384 		}
1385 		buf[--buflen] = '/';
1386 	}
1387 	numfullpathfound++;
1388 	CACHE_RUNLOCK();
1389 	vrele(vp);
1390 
1391 	SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, buf + buflen);
1392 	*retbuf = buf + buflen;
1393 	return (0);
1394 }
1395 
1396 struct vnode *
1397 vn_dir_dd_ino(struct vnode *vp)
1398 {
1399 	struct namecache *ncp;
1400 	struct vnode *ddvp;
1401 
1402 	ASSERT_VOP_LOCKED(vp, "vn_dir_dd_ino");
1403 	CACHE_RLOCK();
1404 	TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
1405 		if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
1406 			continue;
1407 		ddvp = ncp->nc_dvp;
1408 		vhold(ddvp);
1409 		CACHE_RUNLOCK();
1410 		if (vget(ddvp, LK_SHARED | LK_NOWAIT | LK_VNHELD, curthread))
1411 			return (NULL);
1412 		return (ddvp);
1413 	}
1414 	CACHE_RUNLOCK();
1415 	return (NULL);
1416 }
1417 
1418 int
1419 vn_commname(struct vnode *vp, char *buf, u_int buflen)
1420 {
1421 	struct namecache *ncp;
1422 	int l;
1423 
1424 	CACHE_RLOCK();
1425 	TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
1426 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
1427 			break;
1428 	if (ncp == NULL) {
1429 		CACHE_RUNLOCK();
1430 		return (ENOENT);
1431 	}
1432 	l = min(ncp->nc_nlen, buflen - 1);
1433 	memcpy(buf, nc_get_name(ncp), l);
1434 	CACHE_RUNLOCK();
1435 	buf[l] = '\0';
1436 	return (0);
1437 }
1438 
1439 /* ABI compat shims for old kernel modules. */
1440 #undef cache_enter
1441 
1442 void	cache_enter(struct vnode *dvp, struct vnode *vp,
1443 	    struct componentname *cnp);
1444 
1445 void
1446 cache_enter(struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
1447 {
1448 
1449 	cache_enter_time(dvp, vp, cnp, NULL, NULL);
1450 }
1451 
1452 /*
1453  * This function updates path string to vnode's full global path
1454  * and checks the size of the new path string against the pathlen argument.
1455  *
1456  * Requires a locked, referenced vnode.
1457  * Vnode is re-locked on success or ENODEV, otherwise unlocked.
1458  *
1459  * If sysctl debug.disablefullpath is set, ENODEV is returned,
1460  * vnode is left locked and path remain untouched.
1461  *
1462  * If vp is a directory, the call to vn_fullpath_global() always succeeds
1463  * because it falls back to the ".." lookup if the namecache lookup fails.
1464  */
1465 int
1466 vn_path_to_global_path(struct thread *td, struct vnode *vp, char *path,
1467     u_int pathlen)
1468 {
1469 	struct nameidata nd;
1470 	struct vnode *vp1;
1471 	char *rpath, *fbuf;
1472 	int error;
1473 
1474 	ASSERT_VOP_ELOCKED(vp, __func__);
1475 
1476 	/* Return ENODEV if sysctl debug.disablefullpath==1 */
1477 	if (disablefullpath)
1478 		return (ENODEV);
1479 
1480 	/* Construct global filesystem path from vp. */
1481 	VOP_UNLOCK(vp, 0);
1482 	error = vn_fullpath_global(td, vp, &rpath, &fbuf);
1483 
1484 	if (error != 0) {
1485 		vrele(vp);
1486 		return (error);
1487 	}
1488 
1489 	if (strlen(rpath) >= pathlen) {
1490 		vrele(vp);
1491 		error = ENAMETOOLONG;
1492 		goto out;
1493 	}
1494 
1495 	/*
1496 	 * Re-lookup the vnode by path to detect a possible rename.
1497 	 * As a side effect, the vnode is relocked.
1498 	 * If vnode was renamed, return ENOENT.
1499 	 */
1500 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
1501 	    UIO_SYSSPACE, path, td);
1502 	error = namei(&nd);
1503 	if (error != 0) {
1504 		vrele(vp);
1505 		goto out;
1506 	}
1507 	NDFREE(&nd, NDF_ONLY_PNBUF);
1508 	vp1 = nd.ni_vp;
1509 	vrele(vp);
1510 	if (vp1 == vp)
1511 		strcpy(path, rpath);
1512 	else {
1513 		vput(vp1);
1514 		error = ENOENT;
1515 	}
1516 
1517 out:
1518 	free(fbuf, M_TEMP);
1519 	return (error);
1520 }
1521