xref: /freebsd/sys/fs/unionfs/union_subr.c (revision 44682688f038edbf34591b25ce36412a7f2d6d07)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1994 Jan-Simon Pendry
5  * Copyright (c) 1994
6  *	The Regents of the University of California.  All rights reserved.
7  * Copyright (c) 2005, 2006, 2012 Masanori Ozawa <ozawa@ongs.co.jp>, ONGS Inc.
8  * Copyright (c) 2006, 2012 Daichi Goto <daichi@freebsd.org>
9  *
10  * This code is derived from software contributed to Berkeley by
11  * Jan-Simon Pendry.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. Neither the name of the University nor the names of its contributors
22  *    may be used to endorse or promote products derived from this software
23  *    without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  *
37  *	@(#)union_subr.c	8.20 (Berkeley) 5/20/95
38  * $FreeBSD$
39  */
40 
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/kernel.h>
44 #include <sys/ktr.h>
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47 #include <sys/malloc.h>
48 #include <sys/mount.h>
49 #include <sys/namei.h>
50 #include <sys/proc.h>
51 #include <sys/vnode.h>
52 #include <sys/dirent.h>
53 #include <sys/fcntl.h>
54 #include <sys/filedesc.h>
55 #include <sys/stat.h>
56 #include <sys/sysctl.h>
57 #include <sys/taskqueue.h>
58 #include <sys/resourcevar.h>
59 
60 #include <machine/atomic.h>
61 
62 #include <security/mac/mac_framework.h>
63 
64 #include <vm/uma.h>
65 
66 #include <fs/unionfs/union.h>
67 
68 #define NUNIONFSNODECACHE 16
69 #define UNIONFSHASHMASK (NUNIONFSNODECACHE - 1)
70 
71 static MALLOC_DEFINE(M_UNIONFSHASH, "UNIONFS hash", "UNIONFS hash table");
72 MALLOC_DEFINE(M_UNIONFSNODE, "UNIONFS node", "UNIONFS vnode private part");
73 MALLOC_DEFINE(M_UNIONFSPATH, "UNIONFS path", "UNIONFS path private part");
74 
75 static struct task unionfs_deferred_rele_task;
76 static struct mtx unionfs_deferred_rele_lock;
77 static STAILQ_HEAD(, unionfs_node) unionfs_deferred_rele_list =
78     STAILQ_HEAD_INITIALIZER(unionfs_deferred_rele_list);
79 static TASKQUEUE_DEFINE_THREAD(unionfs_rele);
80 
81 unsigned int unionfs_ndeferred = 0;
82 SYSCTL_UINT(_vfs, OID_AUTO, unionfs_ndeferred, CTLFLAG_RD,
83     &unionfs_ndeferred, 0, "unionfs deferred vnode release");
84 
85 static void unionfs_deferred_rele(void *, int);
86 
87 /*
88  * Initialize
89  */
90 int
91 unionfs_init(struct vfsconf *vfsp)
92 {
93 	UNIONFSDEBUG("unionfs_init\n");	/* printed during system boot */
94 	TASK_INIT(&unionfs_deferred_rele_task, 0, unionfs_deferred_rele, NULL);
95 	mtx_init(&unionfs_deferred_rele_lock, "uniondefr", NULL, MTX_DEF);
96 	return (0);
97 }
98 
99 /*
100  * Uninitialize
101  */
102 int
103 unionfs_uninit(struct vfsconf *vfsp)
104 {
105 	taskqueue_quiesce(taskqueue_unionfs_rele);
106 	taskqueue_free(taskqueue_unionfs_rele);
107 	mtx_destroy(&unionfs_deferred_rele_lock);
108 	return (0);
109 }
110 
111 static void
112 unionfs_deferred_rele(void *arg __unused, int pending __unused)
113 {
114 	STAILQ_HEAD(, unionfs_node) local_rele_list;
115 	struct unionfs_node *unp, *tunp;
116 	unsigned int ndeferred;
117 
118 	ndeferred = 0;
119 	STAILQ_INIT(&local_rele_list);
120 	mtx_lock(&unionfs_deferred_rele_lock);
121 	STAILQ_CONCAT(&local_rele_list, &unionfs_deferred_rele_list);
122 	mtx_unlock(&unionfs_deferred_rele_lock);
123 	STAILQ_FOREACH_SAFE(unp, &local_rele_list, un_rele, tunp) {
124 		++ndeferred;
125 		MPASS(unp->un_dvp != NULL);
126 		vrele(unp->un_dvp);
127 		free(unp, M_UNIONFSNODE);
128 	}
129 
130 	/* We expect this function to be single-threaded, thus no atomic */
131 	unionfs_ndeferred += ndeferred;
132 }
133 
134 static struct unionfs_node_hashhead *
135 unionfs_get_hashhead(struct vnode *dvp, struct vnode *lookup)
136 {
137 	struct unionfs_node *unp;
138 
139 	unp = VTOUNIONFS(dvp);
140 
141 	return (&(unp->un_hashtbl[vfs_hash_index(lookup) & UNIONFSHASHMASK]));
142 }
143 
144 /*
145  * Attempt to lookup a cached unionfs vnode by upper/lower vp
146  * from dvp, with dvp's interlock held.
147  */
148 static struct vnode *
149 unionfs_get_cached_vnode_locked(struct vnode *lookup, struct vnode *dvp)
150 {
151 	struct unionfs_node *unp;
152 	struct unionfs_node_hashhead *hd;
153 	struct vnode *vp;
154 
155 	hd = unionfs_get_hashhead(dvp, lookup);
156 
157 	LIST_FOREACH(unp, hd, un_hash) {
158 		if (unp->un_uppervp == lookup ||
159 		    unp->un_lowervp == lookup) {
160 			vp = UNIONFSTOV(unp);
161 			VI_LOCK_FLAGS(vp, MTX_DUPOK);
162 			vp->v_iflag &= ~VI_OWEINACT;
163 			if (VN_IS_DOOMED(vp) ||
164 			    ((vp->v_iflag & VI_DOINGINACT) != 0)) {
165 				VI_UNLOCK(vp);
166 				vp = NULLVP;
167 			} else {
168 				vrefl(vp);
169 				VI_UNLOCK(vp);
170 			}
171 			return (vp);
172 		}
173 	}
174 
175 	return (NULLVP);
176 }
177 
178 
179 /*
180  * Get the cached vnode.
181  */
182 static struct vnode *
183 unionfs_get_cached_vnode(struct vnode *uvp, struct vnode *lvp,
184     struct vnode *dvp)
185 {
186 	struct vnode *vp;
187 
188 	vp = NULLVP;
189 	VI_LOCK(dvp);
190 	if (uvp != NULLVP)
191 		vp = unionfs_get_cached_vnode_locked(uvp, dvp);
192 	else if (lvp != NULLVP)
193 		vp = unionfs_get_cached_vnode_locked(lvp, dvp);
194 	VI_UNLOCK(dvp);
195 
196 	return (vp);
197 }
198 
199 /*
200  * Add the new vnode into cache.
201  */
202 static struct vnode *
203 unionfs_ins_cached_vnode(struct unionfs_node *uncp,
204     struct vnode *dvp)
205 {
206 	struct unionfs_node_hashhead *hd;
207 	struct vnode *vp;
208 
209 	ASSERT_VOP_ELOCKED(uncp->un_uppervp, __func__);
210 	ASSERT_VOP_ELOCKED(uncp->un_lowervp, __func__);
211 	KASSERT(uncp->un_uppervp == NULLVP || uncp->un_uppervp->v_type == VDIR,
212 	    ("%s: v_type != VDIR", __func__));
213 	KASSERT(uncp->un_lowervp == NULLVP || uncp->un_lowervp->v_type == VDIR,
214 	    ("%s: v_type != VDIR", __func__));
215 
216 	vp = NULLVP;
217 	VI_LOCK(dvp);
218 	if (uncp->un_uppervp != NULL)
219 		vp = unionfs_get_cached_vnode_locked(uncp->un_uppervp, dvp);
220 	else if (uncp->un_lowervp != NULL)
221 		vp = unionfs_get_cached_vnode_locked(uncp->un_lowervp, dvp);
222 	if (vp == NULLVP) {
223 		hd = unionfs_get_hashhead(dvp, (uncp->un_uppervp != NULLVP ?
224 		    uncp->un_uppervp : uncp->un_lowervp));
225 		LIST_INSERT_HEAD(hd, uncp, un_hash);
226 	}
227 	VI_UNLOCK(dvp);
228 
229 	return (vp);
230 }
231 
232 /*
233  * Remove the vnode.
234  */
235 static void
236 unionfs_rem_cached_vnode(struct unionfs_node *unp, struct vnode *dvp)
237 {
238 	KASSERT(unp != NULL, ("%s: null node", __func__));
239 	KASSERT(dvp != NULLVP,
240 	    ("%s: null parent vnode", __func__));
241 
242 	VI_LOCK(dvp);
243 	if (unp->un_hash.le_prev != NULL) {
244 		LIST_REMOVE(unp, un_hash);
245 		unp->un_hash.le_next = NULL;
246 		unp->un_hash.le_prev = NULL;
247 	}
248 	VI_UNLOCK(dvp);
249 }
250 
251 /*
252  * Common cleanup handling for unionfs_nodeget
253  * Upper, lower, and parent directory vnodes are expected to be referenced by
254  * the caller.  Upper and lower vnodes, if non-NULL, are also expected to be
255  * exclusively locked by the caller.
256  * This function will return with the caller's locks and references undone.
257  */
258 static void
259 unionfs_nodeget_cleanup(struct vnode *vp, void *arg)
260 {
261 	struct unionfs_node *unp;
262 
263 	/*
264 	 * Lock and reset the default vnode lock; vgone() expects a locked
265 	 * vnode, and we're going to reset the vnode ops.
266 	 */
267 	lockmgr(&vp->v_lock, LK_EXCLUSIVE, NULL);
268 
269 	/*
270 	 * Clear out private data and reset the vnode ops to avoid use of
271 	 * unionfs vnode ops on a partially constructed vnode.
272 	 */
273 	VI_LOCK(vp);
274 	vp->v_data = NULL;
275 	vp->v_vnlock = &vp->v_lock;
276 	vp->v_op = &dead_vnodeops;
277 	VI_UNLOCK(vp);
278 	vgone(vp);
279 	vput(vp);
280 
281 	unp = arg;
282 	if (unp->un_dvp != NULLVP)
283 		vrele(unp->un_dvp);
284 	if (unp->un_uppervp != NULLVP)
285 		vput(unp->un_uppervp);
286 	if (unp->un_lowervp != NULLVP)
287 		vput(unp->un_lowervp);
288 	if (unp->un_hashtbl != NULL)
289 		hashdestroy(unp->un_hashtbl, M_UNIONFSHASH, UNIONFSHASHMASK);
290 	free(unp->un_path, M_UNIONFSPATH);
291 	free(unp, M_UNIONFSNODE);
292 }
293 
294 /*
295  * Make a new or get existing unionfs node.
296  *
297  * uppervp and lowervp should be unlocked. Because if new unionfs vnode is
298  * locked, uppervp or lowervp is locked too. In order to prevent dead lock,
299  * you should not lock plurality simultaneously.
300  */
301 int
302 unionfs_nodeget(struct mount *mp, struct vnode *uppervp,
303     struct vnode *lowervp, struct vnode *dvp, struct vnode **vpp,
304     struct componentname *cnp)
305 {
306 	char	       *path;
307 	struct unionfs_mount *ump;
308 	struct unionfs_node *unp;
309 	struct vnode   *vp;
310 	u_long		hashmask;
311 	int		error;
312 	int		lkflags;
313 	enum vtype	vt;
314 
315 	error = 0;
316 	ump = MOUNTTOUNIONFSMOUNT(mp);
317 	lkflags = (cnp ? cnp->cn_lkflags : 0);
318 	path = (cnp ? cnp->cn_nameptr : NULL);
319 	*vpp = NULLVP;
320 
321 	if (uppervp == NULLVP && lowervp == NULLVP)
322 		panic("%s: upper and lower is null", __func__);
323 
324 	vt = (uppervp != NULLVP ? uppervp->v_type : lowervp->v_type);
325 
326 	/* If it has no ISLASTCN flag, path check is skipped. */
327 	if (cnp && !(cnp->cn_flags & ISLASTCN))
328 		path = NULL;
329 
330 	/* check the cache */
331 	if (dvp != NULLVP && vt == VDIR) {
332 		vp = unionfs_get_cached_vnode(uppervp, lowervp, dvp);
333 		if (vp != NULLVP) {
334 			*vpp = vp;
335 			goto unionfs_nodeget_out;
336 		}
337 	}
338 
339 	if ((uppervp == NULLVP || ump->um_uppervp != uppervp) ||
340 	    (lowervp == NULLVP || ump->um_lowervp != lowervp)) {
341 		/* dvp will be NULLVP only in case of root vnode. */
342 		if (dvp == NULLVP)
343 			return (EINVAL);
344 	}
345 	unp = malloc(sizeof(struct unionfs_node),
346 	    M_UNIONFSNODE, M_WAITOK | M_ZERO);
347 
348 	error = getnewvnode("unionfs", mp, &unionfs_vnodeops, &vp);
349 	if (error != 0) {
350 		free(unp, M_UNIONFSNODE);
351 		return (error);
352 	}
353 	if (dvp != NULLVP)
354 		vref(dvp);
355 	if (uppervp != NULLVP)
356 		vref(uppervp);
357 	if (lowervp != NULLVP)
358 		vref(lowervp);
359 
360 	if (vt == VDIR) {
361 		unp->un_hashtbl = hashinit(NUNIONFSNODECACHE, M_UNIONFSHASH,
362 		    &hashmask);
363 		KASSERT(hashmask == UNIONFSHASHMASK,
364 		    ("unexpected unionfs hash mask 0x%lx", hashmask));
365 	}
366 
367 	unp->un_vnode = vp;
368 	unp->un_uppervp = uppervp;
369 	unp->un_lowervp = lowervp;
370 	unp->un_dvp = dvp;
371 	if (uppervp != NULLVP)
372 		vp->v_vnlock = uppervp->v_vnlock;
373 	else
374 		vp->v_vnlock = lowervp->v_vnlock;
375 
376 	if (path != NULL) {
377 		unp->un_path = malloc(cnp->cn_namelen + 1,
378 		    M_UNIONFSPATH, M_WAITOK | M_ZERO);
379 		bcopy(cnp->cn_nameptr, unp->un_path, cnp->cn_namelen);
380 		unp->un_path[cnp->cn_namelen] = '\0';
381 		unp->un_pathlen = cnp->cn_namelen;
382 	}
383 	vp->v_type = vt;
384 	vp->v_data = unp;
385 
386 	if ((uppervp != NULLVP && ump->um_uppervp == uppervp) &&
387 	    (lowervp != NULLVP && ump->um_lowervp == lowervp))
388 		vp->v_vflag |= VV_ROOT;
389 
390 	vn_lock_pair(lowervp, false, uppervp, false);
391 	error = insmntque1(vp, mp, unionfs_nodeget_cleanup, unp);
392 	if (error != 0)
393 		return (error);
394 	if (lowervp != NULL && VN_IS_DOOMED(lowervp)) {
395 		vput(lowervp);
396 		unp->un_lowervp = NULL;
397 	}
398 	if (uppervp != NULL && VN_IS_DOOMED(uppervp)) {
399 		vput(uppervp);
400 		unp->un_uppervp = NULL;
401 	}
402 	if (unp->un_lowervp == NULL && unp->un_uppervp == NULL) {
403 		unionfs_nodeget_cleanup(vp, unp);
404 		return (ENOENT);
405 	}
406 
407 	if (dvp != NULLVP && vt == VDIR)
408 		*vpp = unionfs_ins_cached_vnode(unp, dvp);
409 	if (*vpp != NULLVP) {
410 		unionfs_nodeget_cleanup(vp, unp);
411 		vp = *vpp;
412 	} else {
413 		if (uppervp != NULL)
414 			VOP_UNLOCK(uppervp);
415 		if (lowervp != NULL)
416 			VOP_UNLOCK(lowervp);
417 		*vpp = vp;
418 	}
419 
420 unionfs_nodeget_out:
421 	if (lkflags & LK_TYPE_MASK)
422 		vn_lock(vp, lkflags | LK_RETRY);
423 
424 	return (0);
425 }
426 
427 /*
428  * Clean up the unionfs node.
429  */
430 void
431 unionfs_noderem(struct vnode *vp)
432 {
433 	struct unionfs_node *unp, *unp_t1, *unp_t2;
434 	struct unionfs_node_hashhead *hd;
435 	struct unionfs_node_status *unsp, *unsp_tmp;
436 	struct vnode   *lvp;
437 	struct vnode   *uvp;
438 	struct vnode   *dvp;
439 	int		count;
440 	int		writerefs;
441 
442 	KASSERT(vp->v_vnlock->lk_recurse == 0,
443 	    ("%s: vnode %p locked recursively", __func__, vp));
444 	if (lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
445 		panic("%s: failed to acquire lock for vnode lock", __func__);
446 
447 	/*
448 	 * Use the interlock to protect the clearing of v_data to
449 	 * prevent faults in unionfs_lock().
450 	 */
451 	VI_LOCK(vp);
452 	unp = VTOUNIONFS(vp);
453 	lvp = unp->un_lowervp;
454 	uvp = unp->un_uppervp;
455 	dvp = unp->un_dvp;
456 	unp->un_lowervp = unp->un_uppervp = NULLVP;
457 	vp->v_vnlock = &(vp->v_lock);
458 	vp->v_data = NULL;
459 	vp->v_object = NULL;
460 	if (unp->un_hashtbl != NULL) {
461 		/*
462 		 * Clear out any cached child vnodes.  This should only
463 		 * be necessary during forced unmount, when the vnode may
464 		 * be reclaimed with a non-zero use count.  Otherwise the
465 		 * reference held by each child should prevent reclamation.
466 		 */
467 		for (count = 0; count <= UNIONFSHASHMASK; count++) {
468 			hd = unp->un_hashtbl + count;
469 			LIST_FOREACH_SAFE(unp_t1, hd, un_hash, unp_t2) {
470 				LIST_REMOVE(unp_t1, un_hash);
471 				unp_t1->un_hash.le_next = NULL;
472 				unp_t1->un_hash.le_prev = NULL;
473 			}
474 		}
475 	}
476 	VI_UNLOCK(vp);
477 
478 	writerefs = atomic_load_int(&vp->v_writecount);
479 	VNASSERT(writerefs >= 0, vp,
480 	    ("%s: write count %d, unexpected text ref", __func__, writerefs));
481 	/*
482 	 * If we were opened for write, we leased the write reference
483 	 * to the lower vnode.  If this is a reclamation due to the
484 	 * forced unmount, undo the reference now.
485 	 */
486 	if (writerefs > 0) {
487 		VNASSERT(uvp != NULL, vp,
488 		    ("%s: write reference without upper vnode", __func__));
489 		VOP_ADD_WRITECOUNT(uvp, -writerefs);
490 	}
491 	if (lvp != NULLVP)
492 		VOP_UNLOCK(lvp);
493 	if (uvp != NULLVP)
494 		VOP_UNLOCK(uvp);
495 
496 	if (dvp != NULLVP)
497 		unionfs_rem_cached_vnode(unp, dvp);
498 
499 	if (lvp != NULLVP)
500 		vrele(lvp);
501 	if (uvp != NULLVP)
502 		vrele(uvp);
503 	if (unp->un_path != NULL) {
504 		free(unp->un_path, M_UNIONFSPATH);
505 		unp->un_path = NULL;
506 		unp->un_pathlen = 0;
507 	}
508 
509 	if (unp->un_hashtbl != NULL) {
510 		hashdestroy(unp->un_hashtbl, M_UNIONFSHASH, UNIONFSHASHMASK);
511 	}
512 
513 	LIST_FOREACH_SAFE(unsp, &(unp->un_unshead), uns_list, unsp_tmp) {
514 		LIST_REMOVE(unsp, uns_list);
515 		free(unsp, M_TEMP);
516 	}
517 	if (dvp != NULLVP) {
518 		mtx_lock(&unionfs_deferred_rele_lock);
519 		STAILQ_INSERT_TAIL(&unionfs_deferred_rele_list, unp, un_rele);
520 		mtx_unlock(&unionfs_deferred_rele_lock);
521 		taskqueue_enqueue(taskqueue_unionfs_rele,
522 		    &unionfs_deferred_rele_task);
523 	} else
524 		free(unp, M_UNIONFSNODE);
525 }
526 
527 /*
528  * Get the unionfs node status object for the vnode corresponding to unp,
529  * for the process that owns td.  Allocate a new status object if one
530  * does not already exist.
531  */
532 void
533 unionfs_get_node_status(struct unionfs_node *unp, struct thread *td,
534     struct unionfs_node_status **unspp)
535 {
536 	struct unionfs_node_status *unsp;
537 	pid_t pid;
538 
539 	pid = td->td_proc->p_pid;
540 
541 	KASSERT(NULL != unspp, ("%s: NULL status", __func__));
542 	ASSERT_VOP_ELOCKED(UNIONFSTOV(unp), __func__);
543 
544 	LIST_FOREACH(unsp, &(unp->un_unshead), uns_list) {
545 		if (unsp->uns_pid == pid) {
546 			*unspp = unsp;
547 			return;
548 		}
549 	}
550 
551 	/* create a new unionfs node status */
552 	unsp = malloc(sizeof(struct unionfs_node_status),
553 	    M_TEMP, M_WAITOK | M_ZERO);
554 
555 	unsp->uns_pid = pid;
556 	LIST_INSERT_HEAD(&(unp->un_unshead), unsp, uns_list);
557 
558 	*unspp = unsp;
559 }
560 
561 /*
562  * Remove the unionfs node status, if you can.
563  * You need exclusive lock this vnode.
564  */
565 void
566 unionfs_tryrem_node_status(struct unionfs_node *unp,
567     struct unionfs_node_status *unsp)
568 {
569 	KASSERT(NULL != unsp, ("%s: NULL status", __func__));
570 	ASSERT_VOP_ELOCKED(UNIONFSTOV(unp), __func__);
571 
572 	if (0 < unsp->uns_lower_opencnt || 0 < unsp->uns_upper_opencnt)
573 		return;
574 
575 	LIST_REMOVE(unsp, uns_list);
576 	free(unsp, M_TEMP);
577 }
578 
579 /*
580  * Create upper node attr.
581  */
582 void
583 unionfs_create_uppervattr_core(struct unionfs_mount *ump, struct vattr *lva,
584     struct vattr *uva, struct thread *td)
585 {
586 	VATTR_NULL(uva);
587 	uva->va_type = lva->va_type;
588 	uva->va_atime = lva->va_atime;
589 	uva->va_mtime = lva->va_mtime;
590 	uva->va_ctime = lva->va_ctime;
591 
592 	switch (ump->um_copymode) {
593 	case UNIONFS_TRANSPARENT:
594 		uva->va_mode = lva->va_mode;
595 		uva->va_uid = lva->va_uid;
596 		uva->va_gid = lva->va_gid;
597 		break;
598 	case UNIONFS_MASQUERADE:
599 		if (ump->um_uid == lva->va_uid) {
600 			uva->va_mode = lva->va_mode & 077077;
601 			uva->va_mode |= (lva->va_type == VDIR ?
602 			    ump->um_udir : ump->um_ufile) & 0700;
603 			uva->va_uid = lva->va_uid;
604 			uva->va_gid = lva->va_gid;
605 		} else {
606 			uva->va_mode = (lva->va_type == VDIR ?
607 			    ump->um_udir : ump->um_ufile);
608 			uva->va_uid = ump->um_uid;
609 			uva->va_gid = ump->um_gid;
610 		}
611 		break;
612 	default:		/* UNIONFS_TRADITIONAL */
613 		uva->va_mode = 0777 & ~td->td_proc->p_pd->pd_cmask;
614 		uva->va_uid = ump->um_uid;
615 		uva->va_gid = ump->um_gid;
616 		break;
617 	}
618 }
619 
620 /*
621  * Create upper node attr.
622  */
623 int
624 unionfs_create_uppervattr(struct unionfs_mount *ump, struct vnode *lvp,
625     struct vattr *uva, struct ucred *cred, struct thread *td)
626 {
627 	struct vattr	lva;
628 	int		error;
629 
630 	if ((error = VOP_GETATTR(lvp, &lva, cred)))
631 		return (error);
632 
633 	unionfs_create_uppervattr_core(ump, &lva, uva, td);
634 
635 	return (error);
636 }
637 
638 /*
639  * relookup
640  *
641  * dvp should be locked on entry and will be locked on return.
642  *
643  * If an error is returned, *vpp will be invalid, otherwise it will hold a
644  * locked, referenced vnode. If *vpp == dvp then remember that only one
645  * LK_EXCLUSIVE lock is held.
646  */
647 int
648 unionfs_relookup(struct vnode *dvp, struct vnode **vpp,
649     struct componentname *cnp, struct componentname *cn, struct thread *td,
650     char *path, int pathlen, u_long nameiop)
651 {
652 	int error;
653 
654 	cn->cn_namelen = pathlen;
655 	cn->cn_pnbuf = path;
656 	cn->cn_nameiop = nameiop;
657 	cn->cn_flags = (LOCKPARENT | LOCKLEAF | HASBUF | SAVENAME | ISLASTCN);
658 	cn->cn_lkflags = LK_EXCLUSIVE;
659 	cn->cn_cred = cnp->cn_cred;
660 	cn->cn_nameptr = cn->cn_pnbuf;
661 
662 	if (nameiop == DELETE)
663 		cn->cn_flags |= (cnp->cn_flags & (DOWHITEOUT | SAVESTART));
664 	else if (RENAME == nameiop)
665 		cn->cn_flags |= (cnp->cn_flags & SAVESTART);
666 	else if (nameiop == CREATE)
667 		cn->cn_flags |= NOCACHE;
668 
669 	vref(dvp);
670 	VOP_UNLOCK(dvp);
671 
672 	if ((error = relookup(dvp, vpp, cn))) {
673 		vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
674 	} else
675 		vrele(dvp);
676 
677 	KASSERT((cn->cn_flags & HASBUF) != 0,
678 	    ("%s: HASBUF cleared", __func__));
679 	KASSERT((cn->cn_flags & SAVENAME) != 0,
680 	    ("%s: SAVENAME cleared", __func__));
681 	KASSERT(cn->cn_pnbuf == path, ("%s: cn_pnbuf changed", __func__));
682 
683 	return (error);
684 }
685 
686 /*
687  * relookup for CREATE namei operation.
688  *
689  * dvp is unionfs vnode. dvp should be locked.
690  *
691  * If it called 'unionfs_copyfile' function by unionfs_link etc,
692  * VOP_LOOKUP information is broken.
693  * So it need relookup in order to create link etc.
694  */
695 int
696 unionfs_relookup_for_create(struct vnode *dvp, struct componentname *cnp,
697     struct thread *td)
698 {
699 	struct vnode *udvp;
700 	struct vnode *vp;
701 	struct componentname cn;
702 	int error;
703 
704 	udvp = UNIONFSVPTOUPPERVP(dvp);
705 	vp = NULLVP;
706 
707 	KASSERT((cnp->cn_flags & HASBUF) != 0,
708 	    ("%s called without HASBUF", __func__));
709 	error = unionfs_relookup(udvp, &vp, cnp, &cn, td, cnp->cn_nameptr,
710 	    cnp->cn_namelen, CREATE);
711 	if (error)
712 		return (error);
713 
714 	if (vp != NULLVP) {
715 		if (udvp == vp)
716 			vrele(vp);
717 		else
718 			vput(vp);
719 
720 		error = EEXIST;
721 	}
722 
723 	return (error);
724 }
725 
726 /*
727  * relookup for DELETE namei operation.
728  *
729  * dvp is unionfs vnode. dvp should be locked.
730  */
731 int
732 unionfs_relookup_for_delete(struct vnode *dvp, struct componentname *cnp,
733     struct thread *td)
734 {
735 	struct vnode *udvp;
736 	struct vnode *vp;
737 	struct componentname cn;
738 	int error;
739 
740 	udvp = UNIONFSVPTOUPPERVP(dvp);
741 	vp = NULLVP;
742 
743 	KASSERT((cnp->cn_flags & HASBUF) != 0,
744 	    ("%s called without HASBUF", __func__));
745 	error = unionfs_relookup(udvp, &vp, cnp, &cn, td, cnp->cn_nameptr,
746 	    cnp->cn_namelen, DELETE);
747 	if (error)
748 		return (error);
749 
750 	if (vp == NULLVP)
751 		error = ENOENT;
752 	else {
753 		if (udvp == vp)
754 			vrele(vp);
755 		else
756 			vput(vp);
757 	}
758 
759 	return (error);
760 }
761 
762 /*
763  * relookup for RENAME namei operation.
764  *
765  * dvp is unionfs vnode. dvp should be locked.
766  */
767 int
768 unionfs_relookup_for_rename(struct vnode *dvp, struct componentname *cnp,
769     struct thread *td)
770 {
771 	struct vnode *udvp;
772 	struct vnode *vp;
773 	struct componentname cn;
774 	int error;
775 
776 	udvp = UNIONFSVPTOUPPERVP(dvp);
777 	vp = NULLVP;
778 
779 	KASSERT((cnp->cn_flags & HASBUF) != 0,
780 	    ("%s called without HASBUF", __func__));
781 	error = unionfs_relookup(udvp, &vp, cnp, &cn, td, cnp->cn_nameptr,
782 	    cnp->cn_namelen, RENAME);
783 	if (error)
784 		return (error);
785 
786 	if (vp != NULLVP) {
787 		if (udvp == vp)
788 			vrele(vp);
789 		else
790 			vput(vp);
791 	}
792 
793 	return (error);
794 }
795 
796 /*
797  * Update the unionfs_node.
798  *
799  * uvp is new locked upper vnode. unionfs vnode's lock will be exchanged to the
800  * uvp's lock and lower's lock will be unlocked.
801  */
802 static void
803 unionfs_node_update(struct unionfs_node *unp, struct vnode *uvp,
804     struct thread *td)
805 {
806 	struct unionfs_node_hashhead *hd;
807 	struct vnode   *vp;
808 	struct vnode   *lvp;
809 	struct vnode   *dvp;
810 	unsigned	count, lockrec;
811 
812 	vp = UNIONFSTOV(unp);
813 	lvp = unp->un_lowervp;
814 	ASSERT_VOP_ELOCKED(lvp, __func__);
815 	ASSERT_VOP_ELOCKED(uvp, __func__);
816 	dvp = unp->un_dvp;
817 
818 	VNASSERT(vp->v_writecount == 0, vp,
819 	    ("%s: non-zero writecount", __func__));
820 	/*
821 	 * Uppdate the upper vnode's lock state to match the lower vnode,
822 	 * and then switch the unionfs vnode's lock to the upper vnode.
823 	 */
824 	lockrec = lvp->v_vnlock->lk_recurse;
825 	for (count = 0; count < lockrec; count++)
826 		vn_lock(uvp, LK_EXCLUSIVE | LK_CANRECURSE | LK_RETRY);
827 	VI_LOCK(vp);
828 	unp->un_uppervp = uvp;
829 	vp->v_vnlock = uvp->v_vnlock;
830 	VI_UNLOCK(vp);
831 
832 	/*
833 	 * Re-cache the unionfs vnode against the upper vnode
834 	 */
835 	if (dvp != NULLVP && vp->v_type == VDIR) {
836 		VI_LOCK(dvp);
837 		if (unp->un_hash.le_prev != NULL) {
838 			LIST_REMOVE(unp, un_hash);
839 			hd = unionfs_get_hashhead(dvp, uvp);
840 			LIST_INSERT_HEAD(hd, unp, un_hash);
841 		}
842 		VI_UNLOCK(unp->un_dvp);
843 	}
844 }
845 
846 /*
847  * Create a new shadow dir.
848  *
849  * udvp should be locked on entry and will be locked on return.
850  *
851  * If no error returned, unp will be updated.
852  */
853 int
854 unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp,
855     struct unionfs_node *unp, struct componentname *cnp, struct thread *td)
856 {
857 	struct vnode   *lvp;
858 	struct vnode   *uvp;
859 	struct vattr	va;
860 	struct vattr	lva;
861 	struct nameidata nd;
862 	struct mount   *mp;
863 	struct ucred   *cred;
864 	struct ucred   *credbk;
865 	struct uidinfo *rootinfo;
866 	int		error;
867 
868 	if (unp->un_uppervp != NULLVP)
869 		return (EEXIST);
870 
871 	lvp = unp->un_lowervp;
872 	uvp = NULLVP;
873 	credbk = cnp->cn_cred;
874 
875 	/* Authority change to root */
876 	rootinfo = uifind((uid_t)0);
877 	cred = crdup(cnp->cn_cred);
878 	/*
879 	 * The calls to chgproccnt() are needed to compensate for change_ruid()
880 	 * calling chgproccnt().
881 	 */
882 	chgproccnt(cred->cr_ruidinfo, 1, 0);
883 	change_euid(cred, rootinfo);
884 	change_ruid(cred, rootinfo);
885 	change_svuid(cred, (uid_t)0);
886 	uifree(rootinfo);
887 	cnp->cn_cred = cred;
888 
889 	memset(&nd.ni_cnd, 0, sizeof(struct componentname));
890 	NDPREINIT(&nd);
891 
892 	if ((error = VOP_GETATTR(lvp, &lva, cnp->cn_cred)))
893 		goto unionfs_mkshadowdir_abort;
894 
895 	if ((error = unionfs_relookup(udvp, &uvp, cnp, &nd.ni_cnd, td,
896 	    cnp->cn_nameptr, cnp->cn_namelen, CREATE)))
897 		goto unionfs_mkshadowdir_abort;
898 	if (uvp != NULLVP) {
899 		if (udvp == uvp)
900 			vrele(uvp);
901 		else
902 			vput(uvp);
903 
904 		error = EEXIST;
905 		goto unionfs_mkshadowdir_abort;
906 	}
907 
908 	if ((error = vn_start_write(udvp, &mp, V_WAIT | PCATCH)))
909 		goto unionfs_mkshadowdir_abort;
910 	unionfs_create_uppervattr_core(ump, &lva, &va, td);
911 
912 	error = VOP_MKDIR(udvp, &uvp, &nd.ni_cnd, &va);
913 
914 	if (!error) {
915 		unionfs_node_update(unp, uvp, td);
916 
917 		/*
918 		 * XXX The bug which cannot set uid/gid was corrected.
919 		 * Ignore errors.
920 		 */
921 		va.va_type = VNON;
922 		VOP_SETATTR(uvp, &va, nd.ni_cnd.cn_cred);
923 	}
924 	vn_finished_write(mp);
925 
926 unionfs_mkshadowdir_abort:
927 	cnp->cn_cred = credbk;
928 	chgproccnt(cred->cr_ruidinfo, -1, 0);
929 	crfree(cred);
930 
931 	return (error);
932 }
933 
934 /*
935  * Create a new whiteout.
936  *
937  * dvp should be locked on entry and will be locked on return.
938  */
939 int
940 unionfs_mkwhiteout(struct vnode *dvp, struct componentname *cnp,
941     struct thread *td, char *path, int pathlen)
942 {
943 	struct vnode   *wvp;
944 	struct nameidata nd;
945 	struct mount   *mp;
946 	int		error;
947 
948 	wvp = NULLVP;
949 	NDPREINIT(&nd);
950 	if ((error = unionfs_relookup(dvp, &wvp, cnp, &nd.ni_cnd, td, path,
951 	    pathlen, CREATE))) {
952 		return (error);
953 	}
954 	if (wvp != NULLVP) {
955 		if (dvp == wvp)
956 			vrele(wvp);
957 		else
958 			vput(wvp);
959 
960 		return (EEXIST);
961 	}
962 
963 	if ((error = vn_start_write(dvp, &mp, V_WAIT | PCATCH)))
964 		goto unionfs_mkwhiteout_free_out;
965 	error = VOP_WHITEOUT(dvp, &nd.ni_cnd, CREATE);
966 
967 	vn_finished_write(mp);
968 
969 unionfs_mkwhiteout_free_out:
970 	return (error);
971 }
972 
973 /*
974  * Create a new vnode for create a new shadow file.
975  *
976  * If an error is returned, *vpp will be invalid, otherwise it will hold a
977  * locked, referenced and opened vnode.
978  *
979  * unp is never updated.
980  */
981 static int
982 unionfs_vn_create_on_upper(struct vnode **vpp, struct vnode *udvp,
983     struct unionfs_node *unp, struct vattr *uvap, struct thread *td)
984 {
985 	struct unionfs_mount *ump;
986 	struct vnode   *vp;
987 	struct vnode   *lvp;
988 	struct ucred   *cred;
989 	struct vattr	lva;
990 	struct nameidata nd;
991 	int		fmode;
992 	int		error;
993 
994 	ump = MOUNTTOUNIONFSMOUNT(UNIONFSTOV(unp)->v_mount);
995 	vp = NULLVP;
996 	lvp = unp->un_lowervp;
997 	cred = td->td_ucred;
998 	fmode = FFLAGS(O_WRONLY | O_CREAT | O_TRUNC | O_EXCL);
999 	error = 0;
1000 
1001 	if ((error = VOP_GETATTR(lvp, &lva, cred)) != 0)
1002 		return (error);
1003 	unionfs_create_uppervattr_core(ump, &lva, uvap, td);
1004 
1005 	if (unp->un_path == NULL)
1006 		panic("%s: NULL un_path", __func__);
1007 
1008 	nd.ni_cnd.cn_namelen = unp->un_pathlen;
1009 	nd.ni_cnd.cn_pnbuf = unp->un_path;
1010 	nd.ni_cnd.cn_nameiop = CREATE;
1011 	nd.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF | HASBUF | SAVENAME |
1012 	    ISLASTCN;
1013 	nd.ni_cnd.cn_lkflags = LK_EXCLUSIVE;
1014 	nd.ni_cnd.cn_cred = cred;
1015 	nd.ni_cnd.cn_nameptr = nd.ni_cnd.cn_pnbuf;
1016 	NDPREINIT(&nd);
1017 
1018 	vref(udvp);
1019 	if ((error = relookup(udvp, &vp, &nd.ni_cnd)) != 0)
1020 		goto unionfs_vn_create_on_upper_free_out2;
1021 	vrele(udvp);
1022 
1023 	if (vp != NULLVP) {
1024 		if (vp == udvp)
1025 			vrele(vp);
1026 		else
1027 			vput(vp);
1028 		error = EEXIST;
1029 		goto unionfs_vn_create_on_upper_free_out1;
1030 	}
1031 
1032 	if ((error = VOP_CREATE(udvp, &vp, &nd.ni_cnd, uvap)) != 0)
1033 		goto unionfs_vn_create_on_upper_free_out1;
1034 
1035 	if ((error = VOP_OPEN(vp, fmode, cred, td, NULL)) != 0) {
1036 		vput(vp);
1037 		goto unionfs_vn_create_on_upper_free_out1;
1038 	}
1039 	error = VOP_ADD_WRITECOUNT(vp, 1);
1040 	CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d",
1041 	    __func__, vp, vp->v_writecount);
1042 	if (error == 0) {
1043 		*vpp = vp;
1044 	} else {
1045 		VOP_CLOSE(vp, fmode, cred, td);
1046 	}
1047 
1048 unionfs_vn_create_on_upper_free_out1:
1049 	VOP_UNLOCK(udvp);
1050 
1051 unionfs_vn_create_on_upper_free_out2:
1052 	KASSERT((nd.ni_cnd.cn_flags & HASBUF) != 0,
1053 	    ("%s: HASBUF cleared", __func__));
1054 	KASSERT((nd.ni_cnd.cn_flags & SAVENAME) != 0,
1055 	    ("%s: SAVENAME cleared", __func__));
1056 	KASSERT(nd.ni_cnd.cn_pnbuf == unp->un_path,
1057 	    ("%s: cn_pnbuf changed", __func__));
1058 
1059 	return (error);
1060 }
1061 
1062 /*
1063  * Copy from lvp to uvp.
1064  *
1065  * lvp and uvp should be locked and opened on entry and will be locked and
1066  * opened on return.
1067  */
1068 static int
1069 unionfs_copyfile_core(struct vnode *lvp, struct vnode *uvp,
1070     struct ucred *cred, struct thread *td)
1071 {
1072 	char           *buf;
1073 	struct uio	uio;
1074 	struct iovec	iov;
1075 	off_t		offset;
1076 	int		count;
1077 	int		error;
1078 	int		bufoffset;
1079 
1080 	error = 0;
1081 	memset(&uio, 0, sizeof(uio));
1082 
1083 	uio.uio_td = td;
1084 	uio.uio_segflg = UIO_SYSSPACE;
1085 	uio.uio_offset = 0;
1086 
1087 	buf = malloc(MAXBSIZE, M_TEMP, M_WAITOK);
1088 
1089 	while (error == 0) {
1090 		offset = uio.uio_offset;
1091 
1092 		uio.uio_iov = &iov;
1093 		uio.uio_iovcnt = 1;
1094 		iov.iov_base = buf;
1095 		iov.iov_len = MAXBSIZE;
1096 		uio.uio_resid = iov.iov_len;
1097 		uio.uio_rw = UIO_READ;
1098 
1099 		if ((error = VOP_READ(lvp, &uio, 0, cred)) != 0)
1100 			break;
1101 		if ((count = MAXBSIZE - uio.uio_resid) == 0)
1102 			break;
1103 
1104 		bufoffset = 0;
1105 		while (bufoffset < count) {
1106 			uio.uio_iov = &iov;
1107 			uio.uio_iovcnt = 1;
1108 			iov.iov_base = buf + bufoffset;
1109 			iov.iov_len = count - bufoffset;
1110 			uio.uio_offset = offset + bufoffset;
1111 			uio.uio_resid = iov.iov_len;
1112 			uio.uio_rw = UIO_WRITE;
1113 
1114 			if ((error = VOP_WRITE(uvp, &uio, 0, cred)) != 0)
1115 				break;
1116 
1117 			bufoffset += (count - bufoffset) - uio.uio_resid;
1118 		}
1119 
1120 		uio.uio_offset = offset + bufoffset;
1121 	}
1122 
1123 	free(buf, M_TEMP);
1124 
1125 	return (error);
1126 }
1127 
1128 /*
1129  * Copy file from lower to upper.
1130  *
1131  * If you need copy of the contents, set 1 to docopy. Otherwise, set 0 to
1132  * docopy.
1133  *
1134  * If no error returned, unp will be updated.
1135  */
1136 int
1137 unionfs_copyfile(struct unionfs_node *unp, int docopy, struct ucred *cred,
1138     struct thread *td)
1139 {
1140 	struct mount   *mp;
1141 	struct vnode   *udvp;
1142 	struct vnode   *lvp;
1143 	struct vnode   *uvp;
1144 	struct vattr	uva;
1145 	int		error;
1146 
1147 	lvp = unp->un_lowervp;
1148 	uvp = NULLVP;
1149 
1150 	if ((UNIONFSTOV(unp)->v_mount->mnt_flag & MNT_RDONLY))
1151 		return (EROFS);
1152 	if (unp->un_dvp == NULLVP)
1153 		return (EINVAL);
1154 	if (unp->un_uppervp != NULLVP)
1155 		return (EEXIST);
1156 	udvp = VTOUNIONFS(unp->un_dvp)->un_uppervp;
1157 	if (udvp == NULLVP)
1158 		return (EROFS);
1159 	if ((udvp->v_mount->mnt_flag & MNT_RDONLY))
1160 		return (EROFS);
1161 
1162 	error = VOP_ACCESS(lvp, VREAD, cred, td);
1163 	if (error != 0)
1164 		return (error);
1165 
1166 	if ((error = vn_start_write(udvp, &mp, V_WAIT | PCATCH)) != 0)
1167 		return (error);
1168 	error = unionfs_vn_create_on_upper(&uvp, udvp, unp, &uva, td);
1169 	if (error != 0) {
1170 		vn_finished_write(mp);
1171 		return (error);
1172 	}
1173 
1174 	if (docopy != 0) {
1175 		error = VOP_OPEN(lvp, FREAD, cred, td, NULL);
1176 		if (error == 0) {
1177 			error = unionfs_copyfile_core(lvp, uvp, cred, td);
1178 			VOP_CLOSE(lvp, FREAD, cred, td);
1179 		}
1180 	}
1181 	VOP_CLOSE(uvp, FWRITE, cred, td);
1182 	VOP_ADD_WRITECOUNT_CHECKED(uvp, -1);
1183 	CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d",
1184 	    __func__, uvp, uvp->v_writecount);
1185 
1186 	vn_finished_write(mp);
1187 
1188 	if (error == 0) {
1189 		/* Reset the attributes. Ignore errors. */
1190 		uva.va_type = VNON;
1191 		VOP_SETATTR(uvp, &uva, cred);
1192 	}
1193 
1194 	unionfs_node_update(unp, uvp, td);
1195 
1196 	return (error);
1197 }
1198 
1199 /*
1200  * It checks whether vp can rmdir. (check empty)
1201  *
1202  * vp is unionfs vnode.
1203  * vp should be locked.
1204  */
1205 int
1206 unionfs_check_rmdir(struct vnode *vp, struct ucred *cred, struct thread *td)
1207 {
1208 	struct vnode   *uvp;
1209 	struct vnode   *lvp;
1210 	struct vnode   *tvp;
1211 	struct dirent  *dp;
1212 	struct dirent  *edp;
1213 	struct componentname cn;
1214 	struct iovec	iov;
1215 	struct uio	uio;
1216 	struct vattr	va;
1217 	int		error;
1218 	int		eofflag;
1219 	int		lookuperr;
1220 
1221 	/*
1222 	 * The size of buf needs to be larger than DIRBLKSIZ.
1223 	 */
1224 	char		buf[256 * 6];
1225 
1226 	ASSERT_VOP_ELOCKED(vp, __func__);
1227 
1228 	eofflag = 0;
1229 	uvp = UNIONFSVPTOUPPERVP(vp);
1230 	lvp = UNIONFSVPTOLOWERVP(vp);
1231 
1232 	/* check opaque */
1233 	if ((error = VOP_GETATTR(uvp, &va, cred)) != 0)
1234 		return (error);
1235 	if (va.va_flags & OPAQUE)
1236 		return (0);
1237 
1238 	/* open vnode */
1239 #ifdef MAC
1240 	if ((error = mac_vnode_check_open(cred, vp, VEXEC|VREAD)) != 0)
1241 		return (error);
1242 #endif
1243 	if ((error = VOP_ACCESS(vp, VEXEC|VREAD, cred, td)) != 0)
1244 		return (error);
1245 	if ((error = VOP_OPEN(vp, FREAD, cred, td, NULL)) != 0)
1246 		return (error);
1247 
1248 	uio.uio_rw = UIO_READ;
1249 	uio.uio_segflg = UIO_SYSSPACE;
1250 	uio.uio_td = td;
1251 	uio.uio_offset = 0;
1252 
1253 #ifdef MAC
1254 	error = mac_vnode_check_readdir(td->td_ucred, lvp);
1255 #endif
1256 	while (!error && !eofflag) {
1257 		iov.iov_base = buf;
1258 		iov.iov_len = sizeof(buf);
1259 		uio.uio_iov = &iov;
1260 		uio.uio_iovcnt = 1;
1261 		uio.uio_resid = iov.iov_len;
1262 
1263 		error = VOP_READDIR(lvp, &uio, cred, &eofflag, NULL, NULL);
1264 		if (error != 0)
1265 			break;
1266 		KASSERT(eofflag != 0 || uio.uio_resid < sizeof(buf),
1267 		    ("%s: empty read from lower FS", __func__));
1268 
1269 		edp = (struct dirent*)&buf[sizeof(buf) - uio.uio_resid];
1270 		for (dp = (struct dirent*)buf; !error && dp < edp;
1271 		     dp = (struct dirent*)((caddr_t)dp + dp->d_reclen)) {
1272 			if (dp->d_type == DT_WHT || dp->d_fileno == 0 ||
1273 			    (dp->d_namlen == 1 && dp->d_name[0] == '.') ||
1274 			    (dp->d_namlen == 2 && !bcmp(dp->d_name, "..", 2)))
1275 				continue;
1276 
1277 			cn.cn_namelen = dp->d_namlen;
1278 			cn.cn_pnbuf = NULL;
1279 			cn.cn_nameptr = dp->d_name;
1280 			cn.cn_nameiop = LOOKUP;
1281 			cn.cn_flags = LOCKPARENT | LOCKLEAF | SAVENAME |
1282 			    RDONLY | ISLASTCN;
1283 			cn.cn_lkflags = LK_EXCLUSIVE;
1284 			cn.cn_cred = cred;
1285 
1286 			/*
1287 			 * check entry in lower.
1288 			 * Sometimes, readdir function returns
1289 			 * wrong entry.
1290 			 */
1291 			lookuperr = VOP_LOOKUP(lvp, &tvp, &cn);
1292 
1293 			if (!lookuperr)
1294 				vput(tvp);
1295 			else
1296 				continue; /* skip entry */
1297 
1298 			/*
1299 			 * check entry
1300 			 * If it has no exist/whiteout entry in upper,
1301 			 * directory is not empty.
1302 			 */
1303 			cn.cn_flags = LOCKPARENT | LOCKLEAF | SAVENAME |
1304 			    RDONLY | ISLASTCN;
1305 			lookuperr = VOP_LOOKUP(uvp, &tvp, &cn);
1306 
1307 			if (!lookuperr)
1308 				vput(tvp);
1309 
1310 			/* ignore exist or whiteout entry */
1311 			if (!lookuperr ||
1312 			    (lookuperr == ENOENT && (cn.cn_flags & ISWHITEOUT)))
1313 				continue;
1314 
1315 			error = ENOTEMPTY;
1316 		}
1317 	}
1318 
1319 	/* close vnode */
1320 	VOP_CLOSE(vp, FREAD, cred, td);
1321 
1322 	return (error);
1323 }
1324 
1325