xref: /freebsd/sys/fs/unionfs/union_vnops.c (revision edf8578117e8844e02c0121147f45e4609b30680)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1992, 1993, 1994, 1995 Jan-Simon Pendry.
5  * Copyright (c) 1992, 1993, 1994, 1995
6  *      The Regents of the University of California.
7  * Copyright (c) 2005, 2006, 2012 Masanori Ozawa <ozawa@ongs.co.jp>, ONGS Inc.
8  * Copyright (c) 2006, 2012 Daichi Goto <daichi@freebsd.org>
9  * All rights reserved.
10  *
11  * This code is derived from software contributed to Berkeley by
12  * Jan-Simon Pendry.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions
16  * are met:
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  * 3. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  *
38  *	@(#)union_vnops.c	8.32 (Berkeley) 6/23/95
39  *
40  */
41 
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/conf.h>
45 #include <sys/kernel.h>
46 #include <sys/lock.h>
47 #include <sys/malloc.h>
48 #include <sys/mount.h>
49 #include <sys/mutex.h>
50 #include <sys/namei.h>
51 #include <sys/sysctl.h>
52 #include <sys/vnode.h>
53 #include <sys/kdb.h>
54 #include <sys/fcntl.h>
55 #include <sys/stat.h>
56 #include <sys/dirent.h>
57 #include <sys/proc.h>
58 #include <sys/bio.h>
59 #include <sys/buf.h>
60 
61 #include <fs/unionfs/union.h>
62 
63 #include <machine/atomic.h>
64 
65 #include <vm/vm.h>
66 #include <vm/vm_extern.h>
67 #include <vm/vm_object.h>
68 #include <vm/vnode_pager.h>
69 
70 #if 0
71 #define UNIONFS_INTERNAL_DEBUG(msg, args...)    printf(msg, ## args)
72 #define UNIONFS_IDBG_RENAME
73 #else
74 #define UNIONFS_INTERNAL_DEBUG(msg, args...)
75 #endif
76 
77 #define KASSERT_UNIONFS_VNODE(vp) \
78 	VNASSERT(((vp)->v_op == &unionfs_vnodeops), vp, \
79 	    ("%s: non-unionfs vnode", __func__))
80 
81 static int
82 unionfs_lookup(struct vop_cachedlookup_args *ap)
83 {
84 	struct unionfs_node *dunp;
85 	struct vnode   *dvp, *udvp, *ldvp, *vp, *uvp, *lvp, *dtmpvp;
86 	struct vattr	va;
87 	struct componentname *cnp;
88 	struct thread  *td;
89 	u_long		nameiop;
90 	u_long		cnflags, cnflagsbk;
91 	int		iswhiteout;
92 	int		lockflag;
93 	int		error , uerror, lerror;
94 
95 	iswhiteout = 0;
96 	lockflag = 0;
97 	error = uerror = lerror = ENOENT;
98 	cnp = ap->a_cnp;
99 	nameiop = cnp->cn_nameiop;
100 	cnflags = cnp->cn_flags;
101 	dvp = ap->a_dvp;
102 	dunp = VTOUNIONFS(dvp);
103 	udvp = dunp->un_uppervp;
104 	ldvp = dunp->un_lowervp;
105 	vp = uvp = lvp = NULLVP;
106 	td = curthread;
107 	*(ap->a_vpp) = NULLVP;
108 
109 	UNIONFS_INTERNAL_DEBUG(
110 	    "unionfs_lookup: enter: nameiop=%ld, flags=%lx, path=%s\n",
111 	    nameiop, cnflags, cnp->cn_nameptr);
112 
113 	if (dvp->v_type != VDIR)
114 		return (ENOTDIR);
115 
116 	/*
117 	 * If read-only and op is not LOOKUP, will return EROFS.
118 	 */
119 	if ((cnflags & ISLASTCN) &&
120 	    (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
121 	    LOOKUP != nameiop)
122 		return (EROFS);
123 
124 	/*
125 	 * lookup dotdot
126 	 */
127 	if (cnflags & ISDOTDOT) {
128 		if (LOOKUP != nameiop && udvp == NULLVP)
129 			return (EROFS);
130 
131 		if (udvp != NULLVP) {
132 			dtmpvp = udvp;
133 			if (ldvp != NULLVP)
134 				VOP_UNLOCK(ldvp);
135 		}
136 		else
137 			dtmpvp = ldvp;
138 
139 		error = VOP_LOOKUP(dtmpvp, &vp, cnp);
140 
141 		if (dtmpvp == udvp && ldvp != NULLVP) {
142 			VOP_UNLOCK(udvp);
143 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
144 		}
145 
146 		if (error == 0) {
147 			/*
148 			 * Exchange lock and reference from vp to
149 			 * dunp->un_dvp. vp is upper/lower vnode, but it
150 			 * will need to return the unionfs vnode.
151 			 */
152 			if (nameiop == DELETE  || nameiop == RENAME ||
153 			    (cnp->cn_lkflags & LK_TYPE_MASK))
154 				VOP_UNLOCK(vp);
155 			vrele(vp);
156 
157 			VOP_UNLOCK(dvp);
158 			*(ap->a_vpp) = dunp->un_dvp;
159 			vref(dunp->un_dvp);
160 
161 			if (nameiop == DELETE || nameiop == RENAME)
162 				vn_lock(dunp->un_dvp, LK_EXCLUSIVE | LK_RETRY);
163 			else if (cnp->cn_lkflags & LK_TYPE_MASK)
164 				vn_lock(dunp->un_dvp, cnp->cn_lkflags |
165 				    LK_RETRY);
166 
167 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
168 		} else if (error == ENOENT && (cnflags & MAKEENTRY) != 0)
169 			cache_enter(dvp, NULLVP, cnp);
170 
171 		goto unionfs_lookup_return;
172 	}
173 
174 	/*
175 	 * lookup upper layer
176 	 */
177 	if (udvp != NULLVP) {
178 		uerror = VOP_LOOKUP(udvp, &uvp, cnp);
179 
180 		if (uerror == 0) {
181 			if (udvp == uvp) {	/* is dot */
182 				vrele(uvp);
183 				*(ap->a_vpp) = dvp;
184 				vref(dvp);
185 
186 				error = uerror;
187 				goto unionfs_lookup_return;
188 			}
189 			if (nameiop == DELETE || nameiop == RENAME ||
190 			    (cnp->cn_lkflags & LK_TYPE_MASK))
191 				VOP_UNLOCK(uvp);
192 		}
193 
194 		/* check whiteout */
195 		if (uerror == ENOENT || uerror == EJUSTRETURN)
196 			if (cnp->cn_flags & ISWHITEOUT)
197 				iswhiteout = 1;	/* don't lookup lower */
198 		if (iswhiteout == 0 && ldvp != NULLVP)
199 			if (!VOP_GETATTR(udvp, &va, cnp->cn_cred) &&
200 			    (va.va_flags & OPAQUE))
201 				iswhiteout = 1;	/* don't lookup lower */
202 #if 0
203 		UNIONFS_INTERNAL_DEBUG(
204 		    "unionfs_lookup: debug: whiteout=%d, path=%s\n",
205 		    iswhiteout, cnp->cn_nameptr);
206 #endif
207 	}
208 
209 	/*
210 	 * lookup lower layer
211 	 */
212 	if (ldvp != NULLVP && !(cnflags & DOWHITEOUT) && iswhiteout == 0) {
213 		/* always op is LOOKUP */
214 		cnp->cn_nameiop = LOOKUP;
215 		cnflagsbk = cnp->cn_flags;
216 		cnp->cn_flags = cnflags;
217 
218 		lerror = VOP_LOOKUP(ldvp, &lvp, cnp);
219 
220 		cnp->cn_nameiop = nameiop;
221 		if (udvp != NULLVP && (uerror == 0 || uerror == EJUSTRETURN))
222 			cnp->cn_flags = cnflagsbk;
223 
224 		if (lerror == 0) {
225 			if (ldvp == lvp) {	/* is dot */
226 				if (uvp != NULLVP)
227 					vrele(uvp);	/* no need? */
228 				vrele(lvp);
229 				*(ap->a_vpp) = dvp;
230 				vref(dvp);
231 
232 				UNIONFS_INTERNAL_DEBUG(
233 				    "unionfs_lookup: leave (%d)\n", lerror);
234 
235 				return (lerror);
236 			}
237 			if (cnp->cn_lkflags & LK_TYPE_MASK)
238 				VOP_UNLOCK(lvp);
239 		}
240 	}
241 
242 	/*
243 	 * check lookup result
244 	 */
245 	if (uvp == NULLVP && lvp == NULLVP) {
246 		error = (udvp != NULLVP ? uerror : lerror);
247 		goto unionfs_lookup_return;
248 	}
249 
250 	/*
251 	 * check vnode type
252 	 */
253 	if (uvp != NULLVP && lvp != NULLVP && uvp->v_type != lvp->v_type) {
254 		vrele(lvp);
255 		lvp = NULLVP;
256 	}
257 
258 	/*
259 	 * check shadow dir
260 	 */
261 	if (uerror != 0 && uerror != EJUSTRETURN && udvp != NULLVP &&
262 	    lerror == 0 && lvp != NULLVP && lvp->v_type == VDIR &&
263 	    !(dvp->v_mount->mnt_flag & MNT_RDONLY) &&
264 	    (1 < cnp->cn_namelen || '.' != *(cnp->cn_nameptr))) {
265 		/* get unionfs vnode in order to create a new shadow dir. */
266 		error = unionfs_nodeget(dvp->v_mount, NULLVP, lvp, dvp, &vp,
267 		    cnp);
268 		if (error != 0)
269 			goto unionfs_lookup_cleanup;
270 
271 		if (LK_SHARED == (cnp->cn_lkflags & LK_TYPE_MASK))
272 			VOP_UNLOCK(vp);
273 		if (LK_EXCLUSIVE != VOP_ISLOCKED(vp)) {
274 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
275 			lockflag = 1;
276 		}
277 		error = unionfs_mkshadowdir(MOUNTTOUNIONFSMOUNT(dvp->v_mount),
278 		    udvp, VTOUNIONFS(vp), cnp, td);
279 		if (lockflag != 0)
280 			VOP_UNLOCK(vp);
281 		if (error != 0) {
282 			UNIONFSDEBUG(
283 			    "unionfs_lookup: Unable to create shadow dir.");
284 			if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE)
285 				vput(vp);
286 			else
287 				vrele(vp);
288 			goto unionfs_lookup_cleanup;
289 		}
290 		if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_SHARED)
291 			vn_lock(vp, LK_SHARED | LK_RETRY);
292 	}
293 	/*
294 	 * get unionfs vnode.
295 	 */
296 	else {
297 		if (uvp != NULLVP)
298 			error = uerror;
299 		else
300 			error = lerror;
301 		if (error != 0)
302 			goto unionfs_lookup_cleanup;
303 		/*
304 		 * get socket vnode.
305 		 */
306 		if (uvp != NULLVP && uvp->v_type == VSOCK) {
307 			vp = uvp;
308 			vref(vp);
309 			if (cnp->cn_lkflags & LK_TYPE_MASK)
310 				vn_lock(vp, cnp->cn_lkflags | LK_RETRY);
311 		}
312 		else if (lvp != NULLVP && lvp->v_type == VSOCK) {
313 			vp = lvp;
314 			vref(vp);
315 			if (cnp->cn_lkflags & LK_TYPE_MASK)
316 				vn_lock(vp, cnp->cn_lkflags | LK_RETRY);
317 		}
318 		/*
319 		 * get unionfs vnode.
320 		 */
321 		else
322 			error = unionfs_nodeget(dvp->v_mount, uvp, lvp,
323 			    dvp, &vp, cnp);
324 		if (error != 0) {
325 			UNIONFSDEBUG(
326 			    "unionfs_lookup: Unable to create unionfs vnode.");
327 			goto unionfs_lookup_cleanup;
328 		}
329 		if ((nameiop == DELETE || nameiop == RENAME) &&
330 		    (cnp->cn_lkflags & LK_TYPE_MASK) == 0)
331 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
332 	}
333 
334 	*(ap->a_vpp) = vp;
335 
336 	if ((cnflags & MAKEENTRY) && vp->v_type != VSOCK)
337 		cache_enter(dvp, vp, cnp);
338 
339 unionfs_lookup_cleanup:
340 	if (uvp != NULLVP)
341 		vrele(uvp);
342 	if (lvp != NULLVP)
343 		vrele(lvp);
344 
345 	if (error == ENOENT && (cnflags & MAKEENTRY) != 0)
346 		cache_enter(dvp, NULLVP, cnp);
347 
348 unionfs_lookup_return:
349 
350 	UNIONFS_INTERNAL_DEBUG("unionfs_lookup: leave (%d)\n", error);
351 
352 	return (error);
353 }
354 
355 static int
356 unionfs_create(struct vop_create_args *ap)
357 {
358 	struct unionfs_node *dunp;
359 	struct componentname *cnp;
360 	struct vnode   *udvp;
361 	struct vnode   *vp;
362 	int		error;
363 
364 	UNIONFS_INTERNAL_DEBUG("unionfs_create: enter\n");
365 
366 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
367 
368 	dunp = VTOUNIONFS(ap->a_dvp);
369 	cnp = ap->a_cnp;
370 	udvp = dunp->un_uppervp;
371 	error = EROFS;
372 
373 	if (udvp != NULLVP) {
374 		error = VOP_CREATE(udvp, &vp, cnp, ap->a_vap);
375 		if (error != 0)
376 			goto unionfs_create_abort;
377 
378 		if (vp->v_type == VSOCK)
379 			*(ap->a_vpp) = vp;
380 		else {
381 			VOP_UNLOCK(vp);
382 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
383 			    ap->a_dvp, ap->a_vpp, cnp);
384 			vrele(vp);
385 		}
386 	}
387 
388 unionfs_create_abort:
389 	UNIONFS_INTERNAL_DEBUG("unionfs_create: leave (%d)\n", error);
390 
391 	return (error);
392 }
393 
394 static int
395 unionfs_whiteout(struct vop_whiteout_args *ap)
396 {
397 	struct unionfs_node *dunp;
398 	struct componentname *cnp;
399 	struct vnode   *udvp;
400 	int		error;
401 
402 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: enter\n");
403 
404 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
405 
406 	dunp = VTOUNIONFS(ap->a_dvp);
407 	cnp = ap->a_cnp;
408 	udvp = dunp->un_uppervp;
409 	error = EOPNOTSUPP;
410 
411 	if (udvp != NULLVP) {
412 		switch (ap->a_flags) {
413 		case CREATE:
414 		case DELETE:
415 		case LOOKUP:
416 			error = VOP_WHITEOUT(udvp, cnp, ap->a_flags);
417 			break;
418 		default:
419 			error = EINVAL;
420 			break;
421 		}
422 	}
423 
424 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: leave (%d)\n", error);
425 
426 	return (error);
427 }
428 
429 static int
430 unionfs_mknod(struct vop_mknod_args *ap)
431 {
432 	struct unionfs_node *dunp;
433 	struct componentname *cnp;
434 	struct vnode   *udvp;
435 	struct vnode   *vp;
436 	int		error;
437 
438 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: enter\n");
439 
440 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
441 
442 	dunp = VTOUNIONFS(ap->a_dvp);
443 	cnp = ap->a_cnp;
444 	udvp = dunp->un_uppervp;
445 	error = EROFS;
446 
447 	if (udvp != NULLVP) {
448 		error = VOP_MKNOD(udvp, &vp, cnp, ap->a_vap);
449 		if (error != 0)
450 			goto unionfs_mknod_abort;
451 
452 		if (vp->v_type == VSOCK)
453 			*(ap->a_vpp) = vp;
454 		else {
455 			VOP_UNLOCK(vp);
456 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
457 			    ap->a_dvp, ap->a_vpp, cnp);
458 			vrele(vp);
459 		}
460 	}
461 
462 unionfs_mknod_abort:
463 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: leave (%d)\n", error);
464 
465 	return (error);
466 }
467 
468 enum unionfs_lkupgrade {
469 	UNIONFS_LKUPGRADE_SUCCESS, /* lock successfully upgraded */
470 	UNIONFS_LKUPGRADE_ALREADY, /* lock already held exclusive */
471 	UNIONFS_LKUPGRADE_DOOMED   /* lock was upgraded, but vnode reclaimed */
472 };
473 
474 static inline enum unionfs_lkupgrade
475 unionfs_upgrade_lock(struct vnode *vp)
476 {
477 	ASSERT_VOP_LOCKED(vp, __func__);
478 
479 	if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
480 		return (UNIONFS_LKUPGRADE_ALREADY);
481 
482 	if (vn_lock(vp, LK_UPGRADE) != 0) {
483 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
484 		if (VN_IS_DOOMED(vp))
485 			return (UNIONFS_LKUPGRADE_DOOMED);
486 	}
487 	return (UNIONFS_LKUPGRADE_SUCCESS);
488 }
489 
490 static inline void
491 unionfs_downgrade_lock(struct vnode *vp, enum unionfs_lkupgrade status)
492 {
493 	if (status != UNIONFS_LKUPGRADE_ALREADY)
494 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
495 }
496 
497 static int
498 unionfs_open(struct vop_open_args *ap)
499 {
500 	struct unionfs_node *unp;
501 	struct unionfs_node_status *unsp;
502 	struct vnode   *vp;
503 	struct vnode   *uvp;
504 	struct vnode   *lvp;
505 	struct vnode   *targetvp;
506 	struct ucred   *cred;
507 	struct thread  *td;
508 	int		error;
509 	enum unionfs_lkupgrade lkstatus;
510 
511 	UNIONFS_INTERNAL_DEBUG("unionfs_open: enter\n");
512 
513 	KASSERT_UNIONFS_VNODE(ap->a_vp);
514 
515 	error = 0;
516 	vp = ap->a_vp;
517 	targetvp = NULLVP;
518 	cred = ap->a_cred;
519 	td = ap->a_td;
520 
521 	/*
522 	 * The executable loader path may call this function with vp locked
523 	 * shared.  If the vnode is reclaimed while upgrading, we can't safely
524 	 * use unp or do anything else unionfs- specific.
525 	 */
526 	lkstatus = unionfs_upgrade_lock(vp);
527 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED) {
528 		error = ENOENT;
529 		goto unionfs_open_cleanup;
530 	}
531 
532 	unp = VTOUNIONFS(vp);
533 	uvp = unp->un_uppervp;
534 	lvp = unp->un_lowervp;
535 	unionfs_get_node_status(unp, td, &unsp);
536 
537 	if (unsp->uns_lower_opencnt > 0 || unsp->uns_upper_opencnt > 0) {
538 		/* vnode is already opend. */
539 		if (unsp->uns_upper_opencnt > 0)
540 			targetvp = uvp;
541 		else
542 			targetvp = lvp;
543 
544 		if (targetvp == lvp &&
545 		    (ap->a_mode & FWRITE) && lvp->v_type == VREG)
546 			targetvp = NULLVP;
547 	}
548 	if (targetvp == NULLVP) {
549 		if (uvp == NULLVP) {
550 			if ((ap->a_mode & FWRITE) && lvp->v_type == VREG) {
551 				error = unionfs_copyfile(unp,
552 				    !(ap->a_mode & O_TRUNC), cred, td);
553 				if (error != 0)
554 					goto unionfs_open_abort;
555 				targetvp = uvp = unp->un_uppervp;
556 			} else
557 				targetvp = lvp;
558 		} else
559 			targetvp = uvp;
560 	}
561 
562 	error = VOP_OPEN(targetvp, ap->a_mode, cred, td, ap->a_fp);
563 	if (error == 0) {
564 		if (targetvp == uvp) {
565 			if (uvp->v_type == VDIR && lvp != NULLVP &&
566 			    unsp->uns_lower_opencnt <= 0) {
567 				/* open lower for readdir */
568 				error = VOP_OPEN(lvp, FREAD, cred, td, NULL);
569 				if (error != 0) {
570 					VOP_CLOSE(uvp, ap->a_mode, cred, td);
571 					goto unionfs_open_abort;
572 				}
573 				unsp->uns_node_flag |= UNS_OPENL_4_READDIR;
574 				unsp->uns_lower_opencnt++;
575 			}
576 			unsp->uns_upper_opencnt++;
577 		} else {
578 			unsp->uns_lower_opencnt++;
579 			unsp->uns_lower_openmode = ap->a_mode;
580 		}
581 		vp->v_object = targetvp->v_object;
582 	}
583 
584 unionfs_open_abort:
585 	if (error != 0)
586 		unionfs_tryrem_node_status(unp, unsp);
587 
588 unionfs_open_cleanup:
589 	unionfs_downgrade_lock(vp, lkstatus);
590 
591 	UNIONFS_INTERNAL_DEBUG("unionfs_open: leave (%d)\n", error);
592 
593 	return (error);
594 }
595 
596 static int
597 unionfs_close(struct vop_close_args *ap)
598 {
599 	struct unionfs_node *unp;
600 	struct unionfs_node_status *unsp;
601 	struct ucred   *cred;
602 	struct thread  *td;
603 	struct vnode   *vp;
604 	struct vnode   *ovp;
605 	int		error;
606 	enum unionfs_lkupgrade lkstatus;;
607 
608 	UNIONFS_INTERNAL_DEBUG("unionfs_close: enter\n");
609 
610 	KASSERT_UNIONFS_VNODE(ap->a_vp);
611 
612 	vp = ap->a_vp;
613 	cred = ap->a_cred;
614 	td = ap->a_td;
615 	error = 0;
616 
617 	/*
618 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
619 	 * or do anything else unionfs- specific.
620 	 */
621 	lkstatus = unionfs_upgrade_lock(vp);
622 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
623 		goto unionfs_close_cleanup;
624 
625 	unp = VTOUNIONFS(vp);
626 	unionfs_get_node_status(unp, td, &unsp);
627 
628 	if (unsp->uns_lower_opencnt <= 0 && unsp->uns_upper_opencnt <= 0) {
629 #ifdef DIAGNOSTIC
630 		printf("unionfs_close: warning: open count is 0\n");
631 #endif
632 		if (unp->un_uppervp != NULLVP)
633 			ovp = unp->un_uppervp;
634 		else
635 			ovp = unp->un_lowervp;
636 	} else if (unsp->uns_upper_opencnt > 0)
637 		ovp = unp->un_uppervp;
638 	else
639 		ovp = unp->un_lowervp;
640 
641 	error = VOP_CLOSE(ovp, ap->a_fflag, cred, td);
642 
643 	if (error != 0)
644 		goto unionfs_close_abort;
645 
646 	vp->v_object = ovp->v_object;
647 
648 	if (ovp == unp->un_uppervp) {
649 		unsp->uns_upper_opencnt--;
650 		if (unsp->uns_upper_opencnt == 0) {
651 			if (unsp->uns_node_flag & UNS_OPENL_4_READDIR) {
652 				VOP_CLOSE(unp->un_lowervp, FREAD, cred, td);
653 				unsp->uns_node_flag &= ~UNS_OPENL_4_READDIR;
654 				unsp->uns_lower_opencnt--;
655 			}
656 			if (unsp->uns_lower_opencnt > 0)
657 				vp->v_object = unp->un_lowervp->v_object;
658 		}
659 	} else
660 		unsp->uns_lower_opencnt--;
661 
662 unionfs_close_abort:
663 	unionfs_tryrem_node_status(unp, unsp);
664 
665 unionfs_close_cleanup:
666 	unionfs_downgrade_lock(vp, lkstatus);
667 
668 	UNIONFS_INTERNAL_DEBUG("unionfs_close: leave (%d)\n", error);
669 
670 	return (error);
671 }
672 
673 /*
674  * Check the access mode toward shadow file/dir.
675  */
676 static int
677 unionfs_check_corrected_access(accmode_t accmode, struct vattr *va,
678     struct ucred *cred)
679 {
680 	uid_t		uid;	/* upper side vnode's uid */
681 	gid_t		gid;	/* upper side vnode's gid */
682 	u_short		vmode;	/* upper side vnode's mode */
683 	u_short		mask;
684 
685 	mask = 0;
686 	uid = va->va_uid;
687 	gid = va->va_gid;
688 	vmode = va->va_mode;
689 
690 	/* check owner */
691 	if (cred->cr_uid == uid) {
692 		if (accmode & VEXEC)
693 			mask |= S_IXUSR;
694 		if (accmode & VREAD)
695 			mask |= S_IRUSR;
696 		if (accmode & VWRITE)
697 			mask |= S_IWUSR;
698 		return ((vmode & mask) == mask ? 0 : EACCES);
699 	}
700 
701 	/* check group */
702 	if (groupmember(gid, cred)) {
703 		if (accmode & VEXEC)
704 			mask |= S_IXGRP;
705 		if (accmode & VREAD)
706 			mask |= S_IRGRP;
707 		if (accmode & VWRITE)
708 			mask |= S_IWGRP;
709 		return ((vmode & mask) == mask ? 0 : EACCES);
710 	}
711 
712 	/* check other */
713 	if (accmode & VEXEC)
714 		mask |= S_IXOTH;
715 	if (accmode & VREAD)
716 		mask |= S_IROTH;
717 	if (accmode & VWRITE)
718 		mask |= S_IWOTH;
719 
720 	return ((vmode & mask) == mask ? 0 : EACCES);
721 }
722 
723 static int
724 unionfs_access(struct vop_access_args *ap)
725 {
726 	struct unionfs_mount *ump;
727 	struct unionfs_node *unp;
728 	struct vnode   *uvp;
729 	struct vnode   *lvp;
730 	struct thread  *td;
731 	struct vattr	va;
732 	accmode_t	accmode;
733 	int		error;
734 
735 	UNIONFS_INTERNAL_DEBUG("unionfs_access: enter\n");
736 
737 	KASSERT_UNIONFS_VNODE(ap->a_vp);
738 
739 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
740 	unp = VTOUNIONFS(ap->a_vp);
741 	uvp = unp->un_uppervp;
742 	lvp = unp->un_lowervp;
743 	td = ap->a_td;
744 	accmode = ap->a_accmode;
745 	error = EACCES;
746 
747 	if ((accmode & VWRITE) &&
748 	    (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)) {
749 		switch (ap->a_vp->v_type) {
750 		case VREG:
751 		case VDIR:
752 		case VLNK:
753 			return (EROFS);
754 		default:
755 			break;
756 		}
757 	}
758 
759 	if (uvp != NULLVP) {
760 		error = VOP_ACCESS(uvp, accmode, ap->a_cred, td);
761 
762 		UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
763 
764 		return (error);
765 	}
766 
767 	if (lvp != NULLVP) {
768 		if (accmode & VWRITE) {
769 			if (ump->um_uppervp->v_mount->mnt_flag & MNT_RDONLY) {
770 				switch (ap->a_vp->v_type) {
771 				case VREG:
772 				case VDIR:
773 				case VLNK:
774 					return (EROFS);
775 				default:
776 					break;
777 				}
778 			} else if (ap->a_vp->v_type == VREG ||
779 			    ap->a_vp->v_type == VDIR) {
780 				/* check shadow file/dir */
781 				if (ump->um_copymode != UNIONFS_TRANSPARENT) {
782 					error = unionfs_create_uppervattr(ump,
783 					    lvp, &va, ap->a_cred, td);
784 					if (error != 0)
785 						return (error);
786 
787 					error = unionfs_check_corrected_access(
788 					    accmode, &va, ap->a_cred);
789 					if (error != 0)
790 						return (error);
791 				}
792 			}
793 			accmode &= ~(VWRITE | VAPPEND);
794 			accmode |= VREAD; /* will copy to upper */
795 		}
796 		error = VOP_ACCESS(lvp, accmode, ap->a_cred, td);
797 	}
798 
799 	UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
800 
801 	return (error);
802 }
803 
804 static int
805 unionfs_getattr(struct vop_getattr_args *ap)
806 {
807 	struct unionfs_node *unp;
808 	struct unionfs_mount *ump;
809 	struct vnode   *uvp;
810 	struct vnode   *lvp;
811 	struct thread  *td;
812 	struct vattr	va;
813 	int		error;
814 
815 	UNIONFS_INTERNAL_DEBUG("unionfs_getattr: enter\n");
816 
817 	KASSERT_UNIONFS_VNODE(ap->a_vp);
818 
819 	unp = VTOUNIONFS(ap->a_vp);
820 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
821 	uvp = unp->un_uppervp;
822 	lvp = unp->un_lowervp;
823 	td = curthread;
824 
825 	if (uvp != NULLVP) {
826 		if ((error = VOP_GETATTR(uvp, ap->a_vap, ap->a_cred)) == 0)
827 			ap->a_vap->va_fsid =
828 			    ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
829 
830 		UNIONFS_INTERNAL_DEBUG(
831 		    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
832 		    ap->a_vap->va_mode, ap->a_vap->va_uid,
833 		    ap->a_vap->va_gid, error);
834 
835 		return (error);
836 	}
837 
838 	error = VOP_GETATTR(lvp, ap->a_vap, ap->a_cred);
839 
840 	if (error == 0 && !(ump->um_uppervp->v_mount->mnt_flag & MNT_RDONLY)) {
841 		/* correct the attr toward shadow file/dir. */
842 		if (ap->a_vp->v_type == VREG || ap->a_vp->v_type == VDIR) {
843 			unionfs_create_uppervattr_core(ump, ap->a_vap, &va, td);
844 			ap->a_vap->va_mode = va.va_mode;
845 			ap->a_vap->va_uid = va.va_uid;
846 			ap->a_vap->va_gid = va.va_gid;
847 		}
848 	}
849 
850 	if (error == 0)
851 		ap->a_vap->va_fsid = ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
852 
853 	UNIONFS_INTERNAL_DEBUG(
854 	    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
855 	    ap->a_vap->va_mode, ap->a_vap->va_uid, ap->a_vap->va_gid, error);
856 
857 	return (error);
858 }
859 
860 static int
861 unionfs_setattr(struct vop_setattr_args *ap)
862 {
863 	struct unionfs_node *unp;
864 	struct vnode   *uvp;
865 	struct vnode   *lvp;
866 	struct thread  *td;
867 	struct vattr   *vap;
868 	int		error;
869 
870 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: enter\n");
871 
872 	KASSERT_UNIONFS_VNODE(ap->a_vp);
873 
874 	error = EROFS;
875 	unp = VTOUNIONFS(ap->a_vp);
876 	uvp = unp->un_uppervp;
877 	lvp = unp->un_lowervp;
878 	td = curthread;
879 	vap = ap->a_vap;
880 
881 	if ((ap->a_vp->v_mount->mnt_flag & MNT_RDONLY) &&
882 	    (vap->va_flags != VNOVAL || vap->va_uid != (uid_t)VNOVAL ||
883 	     vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL ||
884 	     vap->va_mtime.tv_sec != VNOVAL || vap->va_mode != (mode_t)VNOVAL))
885 		return (EROFS);
886 
887 	if (uvp == NULLVP && lvp->v_type == VREG) {
888 		error = unionfs_copyfile(unp, (vap->va_size != 0),
889 		    ap->a_cred, td);
890 		if (error != 0)
891 			return (error);
892 		uvp = unp->un_uppervp;
893 	}
894 
895 	if (uvp != NULLVP)
896 		error = VOP_SETATTR(uvp, vap, ap->a_cred);
897 
898 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: leave (%d)\n", error);
899 
900 	return (error);
901 }
902 
903 static int
904 unionfs_read(struct vop_read_args *ap)
905 {
906 	struct unionfs_node *unp;
907 	struct vnode   *tvp;
908 	int		error;
909 
910 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: enter\n"); */
911 
912 	KASSERT_UNIONFS_VNODE(ap->a_vp);
913 
914 	unp = VTOUNIONFS(ap->a_vp);
915 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
916 
917 	error = VOP_READ(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
918 
919 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: leave (%d)\n", error); */
920 
921 	return (error);
922 }
923 
924 static int
925 unionfs_write(struct vop_write_args *ap)
926 {
927 	struct unionfs_node *unp;
928 	struct vnode   *tvp;
929 	int		error;
930 
931 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: enter\n"); */
932 
933 	KASSERT_UNIONFS_VNODE(ap->a_vp);
934 
935 	unp = VTOUNIONFS(ap->a_vp);
936 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
937 
938 	error = VOP_WRITE(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
939 
940 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: leave (%d)\n", error); */
941 
942 	return (error);
943 }
944 
945 static int
946 unionfs_ioctl(struct vop_ioctl_args *ap)
947 {
948 	struct unionfs_node *unp;
949 	struct unionfs_node_status *unsp;
950 	struct vnode   *ovp;
951 	int error;
952 
953 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: enter\n");
954 
955 	KASSERT_UNIONFS_VNODE(ap->a_vp);
956 
957  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
958 	unp = VTOUNIONFS(ap->a_vp);
959 	unionfs_get_node_status(unp, ap->a_td, &unsp);
960 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
961 	unionfs_tryrem_node_status(unp, unsp);
962 	VOP_UNLOCK(ap->a_vp);
963 
964 	if (ovp == NULLVP)
965 		return (EBADF);
966 
967 	error = VOP_IOCTL(ovp, ap->a_command, ap->a_data, ap->a_fflag,
968 	    ap->a_cred, ap->a_td);
969 
970 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: leave (%d)\n", error);
971 
972 	return (error);
973 }
974 
975 static int
976 unionfs_poll(struct vop_poll_args *ap)
977 {
978 	struct unionfs_node *unp;
979 	struct unionfs_node_status *unsp;
980 	struct vnode *ovp;
981 
982 	KASSERT_UNIONFS_VNODE(ap->a_vp);
983 
984  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
985 	unp = VTOUNIONFS(ap->a_vp);
986 	unionfs_get_node_status(unp, ap->a_td, &unsp);
987 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
988 	unionfs_tryrem_node_status(unp, unsp);
989 	VOP_UNLOCK(ap->a_vp);
990 
991 	if (ovp == NULLVP)
992 		return (EBADF);
993 
994 	return (VOP_POLL(ovp, ap->a_events, ap->a_cred, ap->a_td));
995 }
996 
997 static int
998 unionfs_fsync(struct vop_fsync_args *ap)
999 {
1000 	struct unionfs_node *unp;
1001 	struct unionfs_node_status *unsp;
1002 	struct vnode *ovp;
1003 
1004 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1005 
1006 	unp = VTOUNIONFS(ap->a_vp);
1007 	unionfs_get_node_status(unp, ap->a_td, &unsp);
1008 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
1009 	unionfs_tryrem_node_status(unp, unsp);
1010 
1011 	if (ovp == NULLVP)
1012 		return (EBADF);
1013 
1014 	return (VOP_FSYNC(ovp, ap->a_waitfor, ap->a_td));
1015 }
1016 
1017 static int
1018 unionfs_remove(struct vop_remove_args *ap)
1019 {
1020 	char	       *path;
1021 	struct unionfs_node *dunp;
1022 	struct unionfs_node *unp;
1023 	struct unionfs_mount *ump;
1024 	struct vnode   *udvp;
1025 	struct vnode   *uvp;
1026 	struct vnode   *lvp;
1027 	struct vnode   *vp;
1028 	struct componentname *cnp;
1029 	struct componentname cn;
1030 	struct thread  *td;
1031 	int		error;
1032 	int		pathlen;
1033 
1034 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: enter\n");
1035 
1036 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1037 
1038 	error = 0;
1039 	dunp = VTOUNIONFS(ap->a_dvp);
1040 	udvp = dunp->un_uppervp;
1041 	cnp = ap->a_cnp;
1042 	td = curthread;
1043 
1044 	if (ap->a_vp->v_op != &unionfs_vnodeops) {
1045 		if (ap->a_vp->v_type != VSOCK)
1046 			return (EINVAL);
1047 		ump = NULL;
1048 		vp = uvp = lvp = NULLVP;
1049 		/* search vnode */
1050 		VOP_UNLOCK(ap->a_vp);
1051 		error = unionfs_relookup(udvp, &vp, cnp, &cn, td,
1052 		    cnp->cn_nameptr, cnp->cn_namelen, DELETE);
1053 		if (error != 0 && error != ENOENT) {
1054 			vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1055 			return (error);
1056 		}
1057 
1058 		if (error == 0 && vp == ap->a_vp) {
1059 			/* target vnode in upper */
1060 			uvp = vp;
1061 			vrele(vp);
1062 		} else {
1063 			/* target vnode in lower */
1064 			if (vp != NULLVP) {
1065 				if (udvp == vp)
1066 					vrele(vp);
1067 				else
1068 					vput(vp);
1069 			}
1070 			vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1071 			lvp = ap->a_vp;
1072 		}
1073 		path = cnp->cn_nameptr;
1074 		pathlen = cnp->cn_namelen;
1075 	} else {
1076 		ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1077 		unp = VTOUNIONFS(ap->a_vp);
1078 		uvp = unp->un_uppervp;
1079 		lvp = unp->un_lowervp;
1080 		path = unp->un_path;
1081 		pathlen = unp->un_pathlen;
1082 	}
1083 
1084 	if (udvp == NULLVP)
1085 		return (EROFS);
1086 
1087 	if (uvp != NULLVP) {
1088 		/*
1089 		 * XXX: if the vnode type is VSOCK, it will create whiteout
1090 		 *      after remove.
1091 		 */
1092 		if (ump == NULL || ump->um_whitemode == UNIONFS_WHITE_ALWAYS ||
1093 		    lvp != NULLVP)
1094 			cnp->cn_flags |= DOWHITEOUT;
1095 		error = VOP_REMOVE(udvp, uvp, cnp);
1096 	} else if (lvp != NULLVP)
1097 		error = unionfs_mkwhiteout(udvp, cnp, td, path, pathlen);
1098 
1099 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: leave (%d)\n", error);
1100 
1101 	return (error);
1102 }
1103 
1104 static int
1105 unionfs_link(struct vop_link_args *ap)
1106 {
1107 	struct unionfs_node *dunp;
1108 	struct unionfs_node *unp;
1109 	struct vnode   *udvp;
1110 	struct vnode   *uvp;
1111 	struct componentname *cnp;
1112 	struct thread  *td;
1113 	int		error;
1114 	int		needrelookup;
1115 
1116 	UNIONFS_INTERNAL_DEBUG("unionfs_link: enter\n");
1117 
1118 	KASSERT_UNIONFS_VNODE(ap->a_tdvp);
1119 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1120 
1121 	error = 0;
1122 	needrelookup = 0;
1123 	dunp = VTOUNIONFS(ap->a_tdvp);
1124 	unp = NULL;
1125 	udvp = dunp->un_uppervp;
1126 	uvp = NULLVP;
1127 	cnp = ap->a_cnp;
1128 	td = curthread;
1129 
1130 	if (udvp == NULLVP)
1131 		return (EROFS);
1132 
1133 	if (ap->a_vp->v_op != &unionfs_vnodeops)
1134 		uvp = ap->a_vp;
1135 	else {
1136 		unp = VTOUNIONFS(ap->a_vp);
1137 
1138 		if (unp->un_uppervp == NULLVP) {
1139 			if (ap->a_vp->v_type != VREG)
1140 				return (EOPNOTSUPP);
1141 
1142 			error = unionfs_copyfile(unp, 1, cnp->cn_cred, td);
1143 			if (error != 0)
1144 				return (error);
1145 			needrelookup = 1;
1146 		}
1147 		uvp = unp->un_uppervp;
1148 	}
1149 
1150 	if (needrelookup != 0)
1151 		error = unionfs_relookup_for_create(ap->a_tdvp, cnp, td);
1152 
1153 	if (error == 0)
1154 		error = VOP_LINK(udvp, uvp, cnp);
1155 
1156 	UNIONFS_INTERNAL_DEBUG("unionfs_link: leave (%d)\n", error);
1157 
1158 	return (error);
1159 }
1160 
1161 static int
1162 unionfs_rename(struct vop_rename_args *ap)
1163 {
1164 	struct vnode   *fdvp;
1165 	struct vnode   *fvp;
1166 	struct componentname *fcnp;
1167 	struct vnode   *tdvp;
1168 	struct vnode   *tvp;
1169 	struct componentname *tcnp;
1170 	struct vnode   *ltdvp;
1171 	struct vnode   *ltvp;
1172 	struct thread  *td;
1173 
1174 	/* rename target vnodes */
1175 	struct vnode   *rfdvp;
1176 	struct vnode   *rfvp;
1177 	struct vnode   *rtdvp;
1178 	struct vnode   *rtvp;
1179 
1180 	struct unionfs_mount *ump;
1181 	struct unionfs_node *unp;
1182 	int		error;
1183 	int		needrelookup;
1184 
1185 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: enter\n");
1186 
1187 	error = 0;
1188 	fdvp = ap->a_fdvp;
1189 	fvp = ap->a_fvp;
1190 	fcnp = ap->a_fcnp;
1191 	tdvp = ap->a_tdvp;
1192 	tvp = ap->a_tvp;
1193 	tcnp = ap->a_tcnp;
1194 	ltdvp = NULLVP;
1195 	ltvp = NULLVP;
1196 	td = curthread;
1197 	rfdvp = fdvp;
1198 	rfvp = fvp;
1199 	rtdvp = tdvp;
1200 	rtvp = tvp;
1201 	needrelookup = 0;
1202 
1203 	/* check for cross device rename */
1204 	if (fvp->v_mount != tdvp->v_mount ||
1205 	    (tvp != NULLVP && fvp->v_mount != tvp->v_mount)) {
1206 		if (fvp->v_op != &unionfs_vnodeops)
1207 			error = ENODEV;
1208 		else
1209 			error = EXDEV;
1210 		goto unionfs_rename_abort;
1211 	}
1212 
1213 	/* Renaming a file to itself has no effect. */
1214 	if (fvp == tvp)
1215 		goto unionfs_rename_abort;
1216 
1217 	/*
1218 	 * from/to vnode is unionfs node.
1219 	 */
1220 
1221 	KASSERT_UNIONFS_VNODE(fdvp);
1222 	KASSERT_UNIONFS_VNODE(fvp);
1223 	KASSERT_UNIONFS_VNODE(tdvp);
1224 	if (tvp != NULLVP)
1225 		KASSERT_UNIONFS_VNODE(tvp);
1226 
1227 	unp = VTOUNIONFS(fdvp);
1228 #ifdef UNIONFS_IDBG_RENAME
1229 	UNIONFS_INTERNAL_DEBUG("fdvp=%p, ufdvp=%p, lfdvp=%p\n",
1230 	    fdvp, unp->un_uppervp, unp->un_lowervp);
1231 #endif
1232 	if (unp->un_uppervp == NULLVP) {
1233 		error = ENODEV;
1234 		goto unionfs_rename_abort;
1235 	}
1236 	rfdvp = unp->un_uppervp;
1237 	vref(rfdvp);
1238 
1239 	unp = VTOUNIONFS(fvp);
1240 #ifdef UNIONFS_IDBG_RENAME
1241 	UNIONFS_INTERNAL_DEBUG("fvp=%p, ufvp=%p, lfvp=%p\n",
1242 	    fvp, unp->un_uppervp, unp->un_lowervp);
1243 #endif
1244 	ump = MOUNTTOUNIONFSMOUNT(fvp->v_mount);
1245 	if (unp->un_uppervp == NULLVP) {
1246 		switch (fvp->v_type) {
1247 		case VREG:
1248 			if ((error = vn_lock(fvp, LK_EXCLUSIVE)) != 0)
1249 				goto unionfs_rename_abort;
1250 			error = unionfs_copyfile(unp, 1, fcnp->cn_cred, td);
1251 			VOP_UNLOCK(fvp);
1252 			if (error != 0)
1253 				goto unionfs_rename_abort;
1254 			break;
1255 		case VDIR:
1256 			if ((error = vn_lock(fvp, LK_EXCLUSIVE)) != 0)
1257 				goto unionfs_rename_abort;
1258 			error = unionfs_mkshadowdir(ump, rfdvp, unp, fcnp, td);
1259 			VOP_UNLOCK(fvp);
1260 			if (error != 0)
1261 				goto unionfs_rename_abort;
1262 			break;
1263 		default:
1264 			error = ENODEV;
1265 			goto unionfs_rename_abort;
1266 		}
1267 
1268 		needrelookup = 1;
1269 	}
1270 
1271 	if (unp->un_lowervp != NULLVP)
1272 		fcnp->cn_flags |= DOWHITEOUT;
1273 	rfvp = unp->un_uppervp;
1274 	vref(rfvp);
1275 
1276 	unp = VTOUNIONFS(tdvp);
1277 #ifdef UNIONFS_IDBG_RENAME
1278 	UNIONFS_INTERNAL_DEBUG("tdvp=%p, utdvp=%p, ltdvp=%p\n",
1279 	    tdvp, unp->un_uppervp, unp->un_lowervp);
1280 #endif
1281 	if (unp->un_uppervp == NULLVP) {
1282 		error = ENODEV;
1283 		goto unionfs_rename_abort;
1284 	}
1285 	rtdvp = unp->un_uppervp;
1286 	ltdvp = unp->un_lowervp;
1287 	vref(rtdvp);
1288 
1289 	if (tdvp == tvp) {
1290 		rtvp = rtdvp;
1291 		vref(rtvp);
1292 	} else if (tvp != NULLVP) {
1293 		unp = VTOUNIONFS(tvp);
1294 #ifdef UNIONFS_IDBG_RENAME
1295 		UNIONFS_INTERNAL_DEBUG("tvp=%p, utvp=%p, ltvp=%p\n",
1296 		    tvp, unp->un_uppervp, unp->un_lowervp);
1297 #endif
1298 		if (unp->un_uppervp == NULLVP)
1299 			rtvp = NULLVP;
1300 		else {
1301 			if (tvp->v_type == VDIR) {
1302 				error = EINVAL;
1303 				goto unionfs_rename_abort;
1304 			}
1305 			rtvp = unp->un_uppervp;
1306 			ltvp = unp->un_lowervp;
1307 			vref(rtvp);
1308 		}
1309 	}
1310 
1311 	if (rfvp == rtvp)
1312 		goto unionfs_rename_abort;
1313 
1314 	if (needrelookup != 0) {
1315 		if ((error = vn_lock(fdvp, LK_EXCLUSIVE)) != 0)
1316 			goto unionfs_rename_abort;
1317 		error = unionfs_relookup_for_delete(fdvp, fcnp, td);
1318 		VOP_UNLOCK(fdvp);
1319 		if (error != 0)
1320 			goto unionfs_rename_abort;
1321 
1322 		/* Lock of tvp is canceled in order to avoid recursive lock. */
1323 		if (tvp != NULLVP && tvp != tdvp)
1324 			VOP_UNLOCK(tvp);
1325 		error = unionfs_relookup_for_rename(tdvp, tcnp, td);
1326 		if (tvp != NULLVP && tvp != tdvp)
1327 			vn_lock(tvp, LK_EXCLUSIVE | LK_RETRY);
1328 		if (error != 0)
1329 			goto unionfs_rename_abort;
1330 	}
1331 
1332 	error = VOP_RENAME(rfdvp, rfvp, fcnp, rtdvp, rtvp, tcnp);
1333 
1334 	if (error == 0) {
1335 		if (rtvp != NULLVP && rtvp->v_type == VDIR)
1336 			cache_purge(tdvp);
1337 		if (fvp->v_type == VDIR && fdvp != tdvp)
1338 			cache_purge(fdvp);
1339 	}
1340 
1341 	if (ltdvp != NULLVP)
1342 		VOP_UNLOCK(ltdvp);
1343 	if (tdvp != rtdvp)
1344 		vrele(tdvp);
1345 	if (ltvp != NULLVP)
1346 		VOP_UNLOCK(ltvp);
1347 	if (tvp != rtvp && tvp != NULLVP) {
1348 		if (rtvp == NULLVP)
1349 			vput(tvp);
1350 		else
1351 			vrele(tvp);
1352 	}
1353 	if (fdvp != rfdvp)
1354 		vrele(fdvp);
1355 	if (fvp != rfvp)
1356 		vrele(fvp);
1357 
1358 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1359 
1360 	return (error);
1361 
1362 unionfs_rename_abort:
1363 	vput(tdvp);
1364 	if (tdvp != rtdvp)
1365 		vrele(rtdvp);
1366 	if (tvp != NULLVP) {
1367 		if (tdvp != tvp)
1368 			vput(tvp);
1369 		else
1370 			vrele(tvp);
1371 	}
1372 	if (tvp != rtvp && rtvp != NULLVP)
1373 		vrele(rtvp);
1374 	if (fdvp != rfdvp)
1375 		vrele(rfdvp);
1376 	if (fvp != rfvp)
1377 		vrele(rfvp);
1378 	vrele(fdvp);
1379 	vrele(fvp);
1380 
1381 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1382 
1383 	return (error);
1384 }
1385 
1386 static int
1387 unionfs_mkdir(struct vop_mkdir_args *ap)
1388 {
1389 	struct unionfs_node *dunp;
1390 	struct componentname *cnp;
1391 	struct vnode   *dvp;
1392 	struct vnode   *udvp;
1393 	struct vnode   *uvp;
1394 	struct vattr	va;
1395 	int		error;
1396 	int		lkflags;
1397 
1398 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: enter\n");
1399 
1400 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1401 
1402 	error = EROFS;
1403 	dvp = ap->a_dvp;
1404 	dunp = VTOUNIONFS(dvp);
1405 	cnp = ap->a_cnp;
1406 	lkflags = cnp->cn_lkflags;
1407 	udvp = dunp->un_uppervp;
1408 
1409 	if (udvp != NULLVP) {
1410 		vref(udvp);
1411 		/* check opaque */
1412 		if (!(cnp->cn_flags & ISWHITEOUT)) {
1413 			error = VOP_GETATTR(udvp, &va, cnp->cn_cred);
1414 			if (error != 0)
1415 				goto unionfs_mkdir_cleanup;
1416 			if ((va.va_flags & OPAQUE) != 0)
1417 				cnp->cn_flags |= ISWHITEOUT;
1418 		}
1419 
1420 		if ((error = VOP_MKDIR(udvp, &uvp, cnp, ap->a_vap)) == 0) {
1421 			VOP_UNLOCK(uvp);
1422 			cnp->cn_lkflags = LK_EXCLUSIVE;
1423 			/*
1424 			 * The underlying VOP_MKDIR() implementation may have
1425 			 * temporarily dropped the parent directory vnode lock.
1426 			 * Because the unionfs vnode ordinarily shares that
1427 			 * lock, this may allow the unionfs vnode to be reclaimed
1428 			 * and its lock field reset.  In that case, the unionfs
1429 			 * vnode is effectively no longer locked, and we must
1430 			 * explicitly lock it before returning in order to meet
1431 			 * the locking requirements of VOP_MKDIR().
1432 			 */
1433 			if (__predict_false(VTOUNIONFS(dvp) == NULL)) {
1434 				error = ENOENT;
1435 				goto unionfs_mkdir_cleanup;
1436 			}
1437 			error = unionfs_nodeget(dvp->v_mount, uvp, NULLVP,
1438 			    dvp, ap->a_vpp, cnp);
1439 			cnp->cn_lkflags = lkflags;
1440 			vrele(uvp);
1441 		}
1442 	}
1443 
1444 unionfs_mkdir_cleanup:
1445 
1446 	if (__predict_false(VTOUNIONFS(dvp) == NULL)) {
1447 		vput(udvp);
1448 		vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
1449 	} else if (udvp != NULLVP)
1450 		vrele(udvp);
1451 
1452 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: leave (%d)\n", error);
1453 
1454 	return (error);
1455 }
1456 
1457 static int
1458 unionfs_rmdir(struct vop_rmdir_args *ap)
1459 {
1460 	struct unionfs_node *dunp;
1461 	struct unionfs_node *unp;
1462 	struct unionfs_mount *ump;
1463 	struct componentname *cnp;
1464 	struct thread  *td;
1465 	struct vnode   *udvp;
1466 	struct vnode   *uvp;
1467 	struct vnode   *lvp;
1468 	int		error;
1469 
1470 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: enter\n");
1471 
1472 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1473 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1474 
1475 	error = 0;
1476 	dunp = VTOUNIONFS(ap->a_dvp);
1477 	unp = VTOUNIONFS(ap->a_vp);
1478 	cnp = ap->a_cnp;
1479 	td = curthread;
1480 	udvp = dunp->un_uppervp;
1481 	uvp = unp->un_uppervp;
1482 	lvp = unp->un_lowervp;
1483 
1484 	if (udvp == NULLVP)
1485 		return (EROFS);
1486 
1487 	if (udvp == uvp)
1488 		return (EOPNOTSUPP);
1489 
1490 	if (uvp != NULLVP) {
1491 		if (lvp != NULLVP) {
1492 			error = unionfs_check_rmdir(ap->a_vp, cnp->cn_cred, td);
1493 			if (error != 0)
1494 				return (error);
1495 		}
1496 		ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1497 		if (ump->um_whitemode == UNIONFS_WHITE_ALWAYS || lvp != NULLVP)
1498 			cnp->cn_flags |= DOWHITEOUT;
1499 		/*
1500 		 * The relookup path will need to relock the parent dvp and
1501 		 * possibly the vp as well.  Locking is expected to be done
1502 		 * in parent->child order; drop the lock on vp to avoid LOR
1503 		 * and potential recursion on vp's lock.
1504 		 * vp is expected to remain referenced during VOP_RMDIR(),
1505 		 * so vref/vrele should not be necessary here.
1506 		 */
1507 		VOP_UNLOCK(ap->a_vp);
1508 		VNPASS(vrefcnt(ap->a_vp) > 0, ap->a_vp);
1509 		error = unionfs_relookup_for_delete(ap->a_dvp, cnp, td);
1510 		vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1511 		/*
1512 		 * VOP_RMDIR is dispatched against udvp, so if uvp became
1513 		 * doomed while the lock was dropped above the target
1514 		 * filesystem may not be able to cope.
1515 		 */
1516 		if (error == 0 && VN_IS_DOOMED(uvp))
1517 			error = ENOENT;
1518 		if (error == 0)
1519 			error = VOP_RMDIR(udvp, uvp, cnp);
1520 	} else if (lvp != NULLVP)
1521 		error = unionfs_mkwhiteout(udvp, cnp, td,
1522 		    unp->un_path, unp->un_pathlen);
1523 
1524 	if (error == 0) {
1525 		cache_purge(ap->a_dvp);
1526 		cache_purge(ap->a_vp);
1527 	}
1528 
1529 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: leave (%d)\n", error);
1530 
1531 	return (error);
1532 }
1533 
1534 static int
1535 unionfs_symlink(struct vop_symlink_args *ap)
1536 {
1537 	struct unionfs_node *dunp;
1538 	struct componentname *cnp;
1539 	struct vnode   *udvp;
1540 	struct vnode   *uvp;
1541 	int		error;
1542 	int		lkflags;
1543 
1544 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: enter\n");
1545 
1546 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1547 
1548 	error = EROFS;
1549 	dunp = VTOUNIONFS(ap->a_dvp);
1550 	cnp = ap->a_cnp;
1551 	lkflags = cnp->cn_lkflags;
1552 	udvp = dunp->un_uppervp;
1553 
1554 	if (udvp != NULLVP) {
1555 		error = VOP_SYMLINK(udvp, &uvp, cnp, ap->a_vap, ap->a_target);
1556 		if (error == 0) {
1557 			VOP_UNLOCK(uvp);
1558 			cnp->cn_lkflags = LK_EXCLUSIVE;
1559 			error = unionfs_nodeget(ap->a_dvp->v_mount, uvp, NULLVP,
1560 			    ap->a_dvp, ap->a_vpp, cnp);
1561 			cnp->cn_lkflags = lkflags;
1562 			vrele(uvp);
1563 		}
1564 	}
1565 
1566 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: leave (%d)\n", error);
1567 
1568 	return (error);
1569 }
1570 
1571 static int
1572 unionfs_readdir(struct vop_readdir_args *ap)
1573 {
1574 	struct unionfs_node *unp;
1575 	struct unionfs_node_status *unsp;
1576 	struct uio     *uio;
1577 	struct vnode   *vp;
1578 	struct vnode   *uvp;
1579 	struct vnode   *lvp;
1580 	struct thread  *td;
1581 	struct vattr    va;
1582 
1583 	uint64_t	*cookies_bk;
1584 	int		error;
1585 	int		eofflag;
1586 	int		ncookies_bk;
1587 	int		uio_offset_bk;
1588 	enum unionfs_lkupgrade lkstatus;
1589 
1590 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: enter\n");
1591 
1592 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1593 
1594 	error = 0;
1595 	eofflag = 0;
1596 	uio_offset_bk = 0;
1597 	uio = ap->a_uio;
1598 	uvp = NULLVP;
1599 	lvp = NULLVP;
1600 	td = uio->uio_td;
1601 	ncookies_bk = 0;
1602 	cookies_bk = NULL;
1603 
1604 	vp = ap->a_vp;
1605 	if (vp->v_type != VDIR)
1606 		return (ENOTDIR);
1607 
1608 	/*
1609 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
1610 	 * or do anything else unionfs- specific.
1611 	 */
1612 	lkstatus = unionfs_upgrade_lock(vp);
1613 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
1614 		error = EBADF;
1615 	if (error == 0) {
1616 		unp = VTOUNIONFS(vp);
1617 		uvp = unp->un_uppervp;
1618 		lvp = unp->un_lowervp;
1619 		/* check the open count. unionfs needs open before readdir. */
1620 		unionfs_get_node_status(unp, td, &unsp);
1621 		if ((uvp != NULLVP && unsp->uns_upper_opencnt <= 0) ||
1622 			(lvp != NULLVP && unsp->uns_lower_opencnt <= 0)) {
1623 			unionfs_tryrem_node_status(unp, unsp);
1624 			error = EBADF;
1625 		}
1626 	}
1627 	unionfs_downgrade_lock(vp, lkstatus);
1628 	if (error != 0)
1629 		goto unionfs_readdir_exit;
1630 
1631 	/* check opaque */
1632 	if (uvp != NULLVP && lvp != NULLVP) {
1633 		if ((error = VOP_GETATTR(uvp, &va, ap->a_cred)) != 0)
1634 			goto unionfs_readdir_exit;
1635 		if (va.va_flags & OPAQUE)
1636 			lvp = NULLVP;
1637 	}
1638 
1639 	/* upper only */
1640 	if (uvp != NULLVP && lvp == NULLVP) {
1641 		error = VOP_READDIR(uvp, uio, ap->a_cred, ap->a_eofflag,
1642 		    ap->a_ncookies, ap->a_cookies);
1643 		unsp->uns_readdir_status = 0;
1644 
1645 		goto unionfs_readdir_exit;
1646 	}
1647 
1648 	/* lower only */
1649 	if (uvp == NULLVP && lvp != NULLVP) {
1650 		error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1651 		    ap->a_ncookies, ap->a_cookies);
1652 		unsp->uns_readdir_status = 2;
1653 
1654 		goto unionfs_readdir_exit;
1655 	}
1656 
1657 	/*
1658 	 * readdir upper and lower
1659 	 */
1660 	KASSERT(uvp != NULLVP, ("unionfs_readdir: null upper vp"));
1661 	KASSERT(lvp != NULLVP, ("unionfs_readdir: null lower vp"));
1662 	if (uio->uio_offset == 0)
1663 		unsp->uns_readdir_status = 0;
1664 
1665 	if (unsp->uns_readdir_status == 0) {
1666 		/* read upper */
1667 		error = VOP_READDIR(uvp, uio, ap->a_cred, &eofflag,
1668 				    ap->a_ncookies, ap->a_cookies);
1669 
1670 		if (error != 0 || eofflag == 0)
1671 			goto unionfs_readdir_exit;
1672 		unsp->uns_readdir_status = 1;
1673 
1674 		/*
1675 		 * UFS(and other FS) needs size of uio_resid larger than
1676 		 * DIRBLKSIZ.
1677 		 * size of DIRBLKSIZ equals DEV_BSIZE.
1678 		 * (see: ufs/ufs/ufs_vnops.c ufs_readdir func , ufs/ufs/dir.h)
1679 		 */
1680 		if (uio->uio_resid <= (uio->uio_resid & (DEV_BSIZE -1)))
1681 			goto unionfs_readdir_exit;
1682 
1683 		/*
1684 		 * Backup cookies.
1685 		 * It prepares to readdir in lower.
1686 		 */
1687 		if (ap->a_ncookies != NULL) {
1688 			ncookies_bk = *(ap->a_ncookies);
1689 			*(ap->a_ncookies) = 0;
1690 		}
1691 		if (ap->a_cookies != NULL) {
1692 			cookies_bk = *(ap->a_cookies);
1693 			*(ap->a_cookies) = NULL;
1694 		}
1695 	}
1696 
1697 	/* initialize for readdir in lower */
1698 	if (unsp->uns_readdir_status == 1) {
1699 		unsp->uns_readdir_status = 2;
1700 		/*
1701 		 * Backup uio_offset. See the comment after the
1702 		 * VOP_READDIR call on the lower layer.
1703 		 */
1704 		uio_offset_bk = uio->uio_offset;
1705 		uio->uio_offset = 0;
1706 	}
1707 
1708 	if (lvp == NULLVP) {
1709 		error = EBADF;
1710 		goto unionfs_readdir_exit;
1711 	}
1712 	/* read lower */
1713 	error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1714 			    ap->a_ncookies, ap->a_cookies);
1715 
1716 	/*
1717 	 * We can't return an uio_offset of 0: this would trigger an
1718 	 * infinite loop, because the next call to unionfs_readdir would
1719 	 * always restart with the upper layer (uio_offset == 0) and
1720 	 * always return some data.
1721 	 *
1722 	 * This happens when the lower layer root directory is removed.
1723 	 * (A root directory deleting of unionfs should not be permitted.
1724 	 *  But current VFS can not do it.)
1725 	 */
1726 	if (uio->uio_offset == 0)
1727 		uio->uio_offset = uio_offset_bk;
1728 
1729 	if (cookies_bk != NULL) {
1730 		/* merge cookies */
1731 		int		size;
1732 		uint64_t         *newcookies, *pos;
1733 
1734 		size = *(ap->a_ncookies) + ncookies_bk;
1735 		newcookies = (uint64_t *) malloc(size * sizeof(*newcookies),
1736 		    M_TEMP, M_WAITOK);
1737 		pos = newcookies;
1738 
1739 		memcpy(pos, cookies_bk, ncookies_bk * sizeof(*newcookies));
1740 		pos += ncookies_bk;
1741 		memcpy(pos, *(ap->a_cookies),
1742 		    *(ap->a_ncookies) * sizeof(*newcookies));
1743 		free(cookies_bk, M_TEMP);
1744 		free(*(ap->a_cookies), M_TEMP);
1745 		*(ap->a_ncookies) = size;
1746 		*(ap->a_cookies) = newcookies;
1747 	}
1748 
1749 unionfs_readdir_exit:
1750 	if (error != 0 && ap->a_eofflag != NULL)
1751 		*(ap->a_eofflag) = 1;
1752 
1753 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: leave (%d)\n", error);
1754 
1755 	return (error);
1756 }
1757 
1758 static int
1759 unionfs_readlink(struct vop_readlink_args *ap)
1760 {
1761 	struct unionfs_node *unp;
1762 	struct vnode   *vp;
1763 	int error;
1764 
1765 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: enter\n");
1766 
1767 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1768 
1769 	unp = VTOUNIONFS(ap->a_vp);
1770 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
1771 
1772 	error = VOP_READLINK(vp, ap->a_uio, ap->a_cred);
1773 
1774 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: leave (%d)\n", error);
1775 
1776 	return (error);
1777 }
1778 
1779 static int
1780 unionfs_getwritemount(struct vop_getwritemount_args *ap)
1781 {
1782 	struct unionfs_node *unp;
1783 	struct vnode   *uvp;
1784 	struct vnode   *vp, *ovp;
1785 	int		error;
1786 
1787 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: enter\n");
1788 
1789 	error = 0;
1790 	vp = ap->a_vp;
1791 	uvp = NULLVP;
1792 
1793 	VI_LOCK(vp);
1794 	unp = VTOUNIONFS(vp);
1795 	if (unp != NULL)
1796 		uvp = unp->un_uppervp;
1797 
1798 	/*
1799 	 * If our node has no upper vnode, check the parent directory.
1800 	 * We may be initiating a write operation that will produce a
1801 	 * new upper vnode through CoW.
1802 	 */
1803 	if (uvp == NULLVP && unp != NULL) {
1804 		ovp = vp;
1805 		vp = unp->un_dvp;
1806 		/*
1807 		 * Only the root vnode should have an empty parent, but it
1808 		 * should not have an empty uppervp, so we shouldn't get here.
1809 		 */
1810 		VNASSERT(vp != NULL, ovp, ("%s: NULL parent vnode", __func__));
1811 		VI_UNLOCK(ovp);
1812 		VI_LOCK(vp);
1813 		unp = VTOUNIONFS(vp);
1814 		if (unp != NULL)
1815 			uvp = unp->un_uppervp;
1816 		if (uvp == NULLVP)
1817 			error = EACCES;
1818 	}
1819 
1820 	if (uvp != NULLVP) {
1821 		vholdnz(uvp);
1822 		VI_UNLOCK(vp);
1823 		error = VOP_GETWRITEMOUNT(uvp, ap->a_mpp);
1824 		vdrop(uvp);
1825 	} else {
1826 		VI_UNLOCK(vp);
1827 		*(ap->a_mpp) = NULL;
1828 	}
1829 
1830 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: leave (%d)\n", error);
1831 
1832 	return (error);
1833 }
1834 
1835 static int
1836 unionfs_inactive(struct vop_inactive_args *ap)
1837 {
1838 	ap->a_vp->v_object = NULL;
1839 	vrecycle(ap->a_vp);
1840 	return (0);
1841 }
1842 
1843 static int
1844 unionfs_reclaim(struct vop_reclaim_args *ap)
1845 {
1846 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: enter\n"); */
1847 
1848 	unionfs_noderem(ap->a_vp);
1849 
1850 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: leave\n"); */
1851 
1852 	return (0);
1853 }
1854 
1855 static int
1856 unionfs_print(struct vop_print_args *ap)
1857 {
1858 	struct unionfs_node *unp;
1859 	/* struct unionfs_node_status *unsp; */
1860 
1861 	unp = VTOUNIONFS(ap->a_vp);
1862 	/* unionfs_get_node_status(unp, curthread, &unsp); */
1863 
1864 	printf("unionfs_vp=%p, uppervp=%p, lowervp=%p\n",
1865 	    ap->a_vp, unp->un_uppervp, unp->un_lowervp);
1866 	/*
1867 	printf("unionfs opencnt: uppervp=%d, lowervp=%d\n",
1868 	    unsp->uns_upper_opencnt, unsp->uns_lower_opencnt);
1869 	*/
1870 
1871 	if (unp->un_uppervp != NULLVP)
1872 		vn_printf(unp->un_uppervp, "unionfs: upper ");
1873 	if (unp->un_lowervp != NULLVP)
1874 		vn_printf(unp->un_lowervp, "unionfs: lower ");
1875 
1876 	return (0);
1877 }
1878 
1879 static int
1880 unionfs_get_llt_revlock(struct vnode *vp, int flags)
1881 {
1882 	int revlock;
1883 
1884 	revlock = 0;
1885 
1886 	switch (flags & LK_TYPE_MASK) {
1887 	case LK_SHARED:
1888 		if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
1889 			revlock = LK_UPGRADE;
1890 		else
1891 			revlock = LK_RELEASE;
1892 		break;
1893 	case LK_EXCLUSIVE:
1894 	case LK_UPGRADE:
1895 		revlock = LK_RELEASE;
1896 		break;
1897 	case LK_DOWNGRADE:
1898 		revlock = LK_UPGRADE;
1899 		break;
1900 	default:
1901 		break;
1902 	}
1903 
1904 	return (revlock);
1905 }
1906 
1907 /*
1908  * The state of an acquired lock is adjusted similarly to
1909  * the time of error generating.
1910  * flags: LK_RELEASE or LK_UPGRADE
1911  */
1912 static void
1913 unionfs_revlock(struct vnode *vp, int flags)
1914 {
1915 	if (flags & LK_RELEASE)
1916 		VOP_UNLOCK_FLAGS(vp, flags);
1917 	else {
1918 		/* UPGRADE */
1919 		if (vn_lock(vp, flags) != 0)
1920 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1921 	}
1922 }
1923 
1924 static int
1925 unionfs_lock(struct vop_lock1_args *ap)
1926 {
1927 	struct unionfs_node *unp;
1928 	struct vnode   *vp;
1929 	struct vnode   *uvp;
1930 	struct vnode   *lvp;
1931 	int		error;
1932 	int		flags;
1933 	int		revlock;
1934 	int		interlock;
1935 	int		uhold;
1936 
1937 	/*
1938 	 * TODO: rework the unionfs locking scheme.
1939 	 * It's not guaranteed to be safe to blindly lock two vnodes on
1940 	 * different mounts as is done here.  Further, the entanglement
1941 	 * of locking both vnodes with the various options that can be
1942 	 * passed to VOP_LOCK() makes this code hard to reason about.
1943 	 * Instead, consider locking only the upper vnode, or the lower
1944 	 * vnode is the upper is not present, and taking separate measures
1945 	 * to lock both vnodes in the few cases when that is needed.
1946 	 */
1947 	error = 0;
1948 	interlock = 1;
1949 	uhold = 0;
1950 	flags = ap->a_flags;
1951 	vp = ap->a_vp;
1952 
1953 	if (LK_RELEASE == (flags & LK_TYPE_MASK) || !(flags & LK_TYPE_MASK))
1954 		return (VOP_UNLOCK_FLAGS(vp, flags | LK_RELEASE));
1955 
1956 	if ((flags & LK_INTERLOCK) == 0)
1957 		VI_LOCK(vp);
1958 
1959 	unp = VTOUNIONFS(vp);
1960 	if (unp == NULL)
1961 		goto unionfs_lock_null_vnode;
1962 
1963 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1964 
1965 	lvp = unp->un_lowervp;
1966 	uvp = unp->un_uppervp;
1967 
1968 	if ((revlock = unionfs_get_llt_revlock(vp, flags)) == 0)
1969 		panic("unknown lock type: 0x%x", flags & LK_TYPE_MASK);
1970 
1971 	/*
1972 	 * During unmount, the root vnode lock may be taken recursively,
1973 	 * because it may share the same v_vnlock field as the vnode covered by
1974 	 * the unionfs mount.  The covered vnode is locked across VFS_UNMOUNT(),
1975 	 * and the same lock may be taken recursively here during vflush()
1976 	 * issued by unionfs_unmount().
1977 	 */
1978 	if ((flags & LK_TYPE_MASK) == LK_EXCLUSIVE &&
1979 	    (vp->v_vflag & VV_ROOT) != 0)
1980 		flags |= LK_CANRECURSE;
1981 
1982 	if (lvp != NULLVP) {
1983 		if (uvp != NULLVP && flags & LK_UPGRADE) {
1984 			/*
1985 			 * Share Lock is once released and a deadlock is
1986 			 * avoided.
1987 			 */
1988 			vholdnz(uvp);
1989 			uhold = 1;
1990 			VOP_UNLOCK(uvp);
1991 		}
1992 		VI_LOCK_FLAGS(lvp, MTX_DUPOK);
1993 		flags |= LK_INTERLOCK;
1994 		vholdl(lvp);
1995 
1996 		VI_UNLOCK(vp);
1997 		ap->a_flags &= ~LK_INTERLOCK;
1998 
1999 		error = VOP_LOCK(lvp, flags);
2000 
2001 		VI_LOCK(vp);
2002 		unp = VTOUNIONFS(vp);
2003 		if (unp == NULL) {
2004 			/* vnode is released. */
2005 			VI_UNLOCK(vp);
2006 			if (error == 0)
2007 				VOP_UNLOCK(lvp);
2008 			vdrop(lvp);
2009 			if (uhold != 0)
2010 				vdrop(uvp);
2011 			goto unionfs_lock_fallback;
2012 		}
2013 	}
2014 
2015 	if (error == 0 && uvp != NULLVP) {
2016 		if (uhold && flags & LK_UPGRADE) {
2017 			flags &= ~LK_TYPE_MASK;
2018 			flags |= LK_EXCLUSIVE;
2019 		}
2020 		VI_LOCK_FLAGS(uvp, MTX_DUPOK);
2021 		flags |= LK_INTERLOCK;
2022 		if (uhold == 0) {
2023 			vholdl(uvp);
2024 			uhold = 1;
2025 		}
2026 
2027 		VI_UNLOCK(vp);
2028 		ap->a_flags &= ~LK_INTERLOCK;
2029 
2030 		error = VOP_LOCK(uvp, flags);
2031 
2032 		VI_LOCK(vp);
2033 		unp = VTOUNIONFS(vp);
2034 		if (unp == NULL) {
2035 			/* vnode is released. */
2036 			VI_UNLOCK(vp);
2037 			if (error == 0)
2038 				VOP_UNLOCK(uvp);
2039 			vdrop(uvp);
2040 			if (lvp != NULLVP) {
2041 				VOP_UNLOCK(lvp);
2042 				vdrop(lvp);
2043 			}
2044 			goto unionfs_lock_fallback;
2045 		}
2046 		if (error != 0 && lvp != NULLVP) {
2047 			/* rollback */
2048 			VI_UNLOCK(vp);
2049 			unionfs_revlock(lvp, revlock);
2050 			interlock = 0;
2051 		}
2052 	}
2053 
2054 	if (interlock)
2055 		VI_UNLOCK(vp);
2056 	if (lvp != NULLVP)
2057 		vdrop(lvp);
2058 	if (uhold != 0)
2059 		vdrop(uvp);
2060 
2061 	return (error);
2062 
2063 unionfs_lock_null_vnode:
2064 	ap->a_flags |= LK_INTERLOCK;
2065 	return (vop_stdlock(ap));
2066 
2067 unionfs_lock_fallback:
2068 	/*
2069 	 * If we reach this point, we've discovered the unionfs vnode
2070 	 * has been reclaimed while the upper/lower vnode locks were
2071 	 * temporarily dropped.  Such temporary droppage may happen
2072 	 * during the course of an LK_UPGRADE operation itself, and in
2073 	 * that case LK_UPGRADE must be cleared as the unionfs vnode's
2074 	 * lock has been reset to point to the standard v_lock field,
2075 	 * which has not previously been held.
2076 	 */
2077 	if (flags & LK_UPGRADE) {
2078 		ap->a_flags &= ~LK_TYPE_MASK;
2079 		ap->a_flags |= LK_EXCLUSIVE;
2080 	}
2081 	return (vop_stdlock(ap));
2082 }
2083 
2084 static int
2085 unionfs_unlock(struct vop_unlock_args *ap)
2086 {
2087 	struct vnode   *vp;
2088 	struct vnode   *lvp;
2089 	struct vnode   *uvp;
2090 	struct unionfs_node *unp;
2091 	int		error;
2092 	int		uhold;
2093 
2094 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2095 
2096 	error = 0;
2097 	uhold = 0;
2098 	vp = ap->a_vp;
2099 
2100 	unp = VTOUNIONFS(vp);
2101 	if (unp == NULL)
2102 		goto unionfs_unlock_null_vnode;
2103 	lvp = unp->un_lowervp;
2104 	uvp = unp->un_uppervp;
2105 
2106 	if (lvp != NULLVP) {
2107 		vholdnz(lvp);
2108 		error = VOP_UNLOCK(lvp);
2109 	}
2110 
2111 	if (error == 0 && uvp != NULLVP) {
2112 		vholdnz(uvp);
2113 		uhold = 1;
2114 		error = VOP_UNLOCK(uvp);
2115 	}
2116 
2117 	if (lvp != NULLVP)
2118 		vdrop(lvp);
2119 	if (uhold != 0)
2120 		vdrop(uvp);
2121 
2122 	return error;
2123 
2124 unionfs_unlock_null_vnode:
2125 	return (vop_stdunlock(ap));
2126 }
2127 
2128 static int
2129 unionfs_pathconf(struct vop_pathconf_args *ap)
2130 {
2131 	struct unionfs_node *unp;
2132 	struct vnode   *vp;
2133 
2134 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2135 
2136 	unp = VTOUNIONFS(ap->a_vp);
2137 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2138 
2139 	return (VOP_PATHCONF(vp, ap->a_name, ap->a_retval));
2140 }
2141 
2142 static int
2143 unionfs_advlock(struct vop_advlock_args *ap)
2144 {
2145 	struct unionfs_node *unp;
2146 	struct unionfs_node_status *unsp;
2147 	struct vnode   *vp;
2148 	struct vnode   *uvp;
2149 	struct thread  *td;
2150 	int error;
2151 
2152 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: enter\n");
2153 
2154 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2155 
2156 	vp = ap->a_vp;
2157 	td = curthread;
2158 
2159 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2160 
2161 	unp = VTOUNIONFS(ap->a_vp);
2162 	uvp = unp->un_uppervp;
2163 
2164 	if (uvp == NULLVP) {
2165 		error = unionfs_copyfile(unp, 1, td->td_ucred, td);
2166 		if (error != 0)
2167 			goto unionfs_advlock_abort;
2168 		uvp = unp->un_uppervp;
2169 
2170 		unionfs_get_node_status(unp, td, &unsp);
2171 		if (unsp->uns_lower_opencnt > 0) {
2172 			/* try reopen the vnode */
2173 			error = VOP_OPEN(uvp, unsp->uns_lower_openmode,
2174 				td->td_ucred, td, NULL);
2175 			if (error)
2176 				goto unionfs_advlock_abort;
2177 			unsp->uns_upper_opencnt++;
2178 			VOP_CLOSE(unp->un_lowervp, unsp->uns_lower_openmode,
2179 			    td->td_ucred, td);
2180 			unsp->uns_lower_opencnt--;
2181 		} else
2182 			unionfs_tryrem_node_status(unp, unsp);
2183 	}
2184 
2185 	VOP_UNLOCK(vp);
2186 
2187 	error = VOP_ADVLOCK(uvp, ap->a_id, ap->a_op, ap->a_fl, ap->a_flags);
2188 
2189 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2190 
2191 	return error;
2192 
2193 unionfs_advlock_abort:
2194 	VOP_UNLOCK(vp);
2195 
2196 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2197 
2198 	return error;
2199 }
2200 
2201 static int
2202 unionfs_strategy(struct vop_strategy_args *ap)
2203 {
2204 	struct unionfs_node *unp;
2205 	struct vnode   *vp;
2206 
2207 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2208 
2209 	unp = VTOUNIONFS(ap->a_vp);
2210 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2211 
2212 #ifdef DIAGNOSTIC
2213 	if (vp == NULLVP)
2214 		panic("unionfs_strategy: nullvp");
2215 
2216 	if (ap->a_bp->b_iocmd == BIO_WRITE && vp == unp->un_lowervp)
2217 		panic("unionfs_strategy: writing to lowervp");
2218 #endif
2219 
2220 	return (VOP_STRATEGY(vp, ap->a_bp));
2221 }
2222 
2223 static int
2224 unionfs_getacl(struct vop_getacl_args *ap)
2225 {
2226 	struct unionfs_node *unp;
2227 	struct vnode   *vp;
2228 	int		error;
2229 
2230 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2231 
2232 	unp = VTOUNIONFS(ap->a_vp);
2233 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2234 
2235 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: enter\n");
2236 
2237 	error = VOP_GETACL(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2238 
2239 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: leave (%d)\n", error);
2240 
2241 	return (error);
2242 }
2243 
2244 static int
2245 unionfs_setacl(struct vop_setacl_args *ap)
2246 {
2247 	struct unionfs_node *unp;
2248 	struct vnode   *uvp;
2249 	struct vnode   *lvp;
2250 	struct thread  *td;
2251 	int		error;
2252 
2253 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: enter\n");
2254 
2255 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2256 
2257 	error = EROFS;
2258 	unp = VTOUNIONFS(ap->a_vp);
2259 	uvp = unp->un_uppervp;
2260 	lvp = unp->un_lowervp;
2261 	td = ap->a_td;
2262 
2263 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2264 		return (EROFS);
2265 
2266 	if (uvp == NULLVP && lvp->v_type == VREG) {
2267 		if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0)
2268 			return (error);
2269 		uvp = unp->un_uppervp;
2270 	}
2271 
2272 	if (uvp != NULLVP)
2273 		error = VOP_SETACL(uvp, ap->a_type, ap->a_aclp, ap->a_cred, td);
2274 
2275 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: leave (%d)\n", error);
2276 
2277 	return (error);
2278 }
2279 
2280 static int
2281 unionfs_aclcheck(struct vop_aclcheck_args *ap)
2282 {
2283 	struct unionfs_node *unp;
2284 	struct vnode   *vp;
2285 	int		error;
2286 
2287 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: enter\n");
2288 
2289 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2290 
2291 	unp = VTOUNIONFS(ap->a_vp);
2292 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2293 
2294 	error = VOP_ACLCHECK(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2295 
2296 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: leave (%d)\n", error);
2297 
2298 	return (error);
2299 }
2300 
2301 static int
2302 unionfs_openextattr(struct vop_openextattr_args *ap)
2303 {
2304 	struct unionfs_node *unp;
2305 	struct vnode   *vp;
2306 	struct vnode   *tvp;
2307 	int		error;
2308 
2309 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2310 
2311 	vp = ap->a_vp;
2312 	unp = VTOUNIONFS(vp);
2313 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2314 
2315 	if ((tvp == unp->un_uppervp && (unp->un_flag & UNIONFS_OPENEXTU)) ||
2316 	    (tvp == unp->un_lowervp && (unp->un_flag & UNIONFS_OPENEXTL)))
2317 		return (EBUSY);
2318 
2319 	error = VOP_OPENEXTATTR(tvp, ap->a_cred, ap->a_td);
2320 
2321 	if (error == 0) {
2322 		if (vn_lock(vp, LK_UPGRADE) != 0)
2323 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2324 		if (!VN_IS_DOOMED(vp)) {
2325 			if (tvp == unp->un_uppervp)
2326 				unp->un_flag |= UNIONFS_OPENEXTU;
2327 			else
2328 				unp->un_flag |= UNIONFS_OPENEXTL;
2329 		}
2330 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2331 	}
2332 
2333 	return (error);
2334 }
2335 
2336 static int
2337 unionfs_closeextattr(struct vop_closeextattr_args *ap)
2338 {
2339 	struct unionfs_node *unp;
2340 	struct vnode   *vp;
2341 	struct vnode   *tvp;
2342 	int		error;
2343 
2344 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2345 
2346 	vp = ap->a_vp;
2347 	unp = VTOUNIONFS(vp);
2348 	tvp = NULLVP;
2349 
2350 	if (unp->un_flag & UNIONFS_OPENEXTU)
2351 		tvp = unp->un_uppervp;
2352 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2353 		tvp = unp->un_lowervp;
2354 
2355 	if (tvp == NULLVP)
2356 		return (EOPNOTSUPP);
2357 
2358 	error = VOP_CLOSEEXTATTR(tvp, ap->a_commit, ap->a_cred, ap->a_td);
2359 
2360 	if (error == 0) {
2361 		if (vn_lock(vp, LK_UPGRADE) != 0)
2362 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2363 		if (!VN_IS_DOOMED(vp)) {
2364 			if (tvp == unp->un_uppervp)
2365 				unp->un_flag &= ~UNIONFS_OPENEXTU;
2366 			else
2367 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2368 		}
2369 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2370 	}
2371 
2372 	return (error);
2373 }
2374 
2375 static int
2376 unionfs_getextattr(struct vop_getextattr_args *ap)
2377 {
2378 	struct unionfs_node *unp;
2379 	struct vnode   *vp;
2380 
2381 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2382 
2383 	unp = VTOUNIONFS(ap->a_vp);
2384 	vp = NULLVP;
2385 
2386 	if (unp->un_flag & UNIONFS_OPENEXTU)
2387 		vp = unp->un_uppervp;
2388 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2389 		vp = unp->un_lowervp;
2390 
2391 	if (vp == NULLVP)
2392 		return (EOPNOTSUPP);
2393 
2394 	return (VOP_GETEXTATTR(vp, ap->a_attrnamespace, ap->a_name,
2395 	    ap->a_uio, ap->a_size, ap->a_cred, ap->a_td));
2396 }
2397 
2398 static int
2399 unionfs_setextattr(struct vop_setextattr_args *ap)
2400 {
2401 	struct unionfs_node *unp;
2402 	struct vnode   *uvp;
2403 	struct vnode   *lvp;
2404 	struct vnode   *ovp;
2405 	struct ucred   *cred;
2406 	struct thread  *td;
2407 	int		error;
2408 
2409 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2410 
2411 	error = EROFS;
2412 	unp = VTOUNIONFS(ap->a_vp);
2413 	uvp = unp->un_uppervp;
2414 	lvp = unp->un_lowervp;
2415 	ovp = NULLVP;
2416 	cred = ap->a_cred;
2417 	td = ap->a_td;
2418 
2419 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: enter (un_flag=%x)\n",
2420 	    unp->un_flag);
2421 
2422 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2423 		return (EROFS);
2424 
2425 	if (unp->un_flag & UNIONFS_OPENEXTU)
2426 		ovp = unp->un_uppervp;
2427 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2428 		ovp = unp->un_lowervp;
2429 
2430 	if (ovp == NULLVP)
2431 		return (EOPNOTSUPP);
2432 
2433 	if (ovp == lvp && lvp->v_type == VREG) {
2434 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2435 		if (uvp == NULLVP &&
2436 		    (error = unionfs_copyfile(unp, 1, cred, td)) != 0) {
2437 unionfs_setextattr_reopen:
2438 			if ((unp->un_flag & UNIONFS_OPENEXTL) &&
2439 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2440 #ifdef DIAGNOSTIC
2441 				panic("unionfs: VOP_OPENEXTATTR failed");
2442 #endif
2443 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2444 			}
2445 			goto unionfs_setextattr_abort;
2446 		}
2447 		uvp = unp->un_uppervp;
2448 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2449 			goto unionfs_setextattr_reopen;
2450 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2451 		unp->un_flag |= UNIONFS_OPENEXTU;
2452 		ovp = uvp;
2453 	}
2454 
2455 	if (ovp == uvp)
2456 		error = VOP_SETEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2457 		    ap->a_uio, cred, td);
2458 
2459 unionfs_setextattr_abort:
2460 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: leave (%d)\n", error);
2461 
2462 	return (error);
2463 }
2464 
2465 static int
2466 unionfs_listextattr(struct vop_listextattr_args *ap)
2467 {
2468 	struct unionfs_node *unp;
2469 	struct vnode *vp;
2470 
2471 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2472 
2473 	unp = VTOUNIONFS(ap->a_vp);
2474 	vp = NULLVP;
2475 
2476 	if (unp->un_flag & UNIONFS_OPENEXTU)
2477 		vp = unp->un_uppervp;
2478 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2479 		vp = unp->un_lowervp;
2480 
2481 	if (vp == NULLVP)
2482 		return (EOPNOTSUPP);
2483 
2484 	return (VOP_LISTEXTATTR(vp, ap->a_attrnamespace, ap->a_uio,
2485 	    ap->a_size, ap->a_cred, ap->a_td));
2486 }
2487 
2488 static int
2489 unionfs_deleteextattr(struct vop_deleteextattr_args *ap)
2490 {
2491 	struct unionfs_node *unp;
2492 	struct vnode   *uvp;
2493 	struct vnode   *lvp;
2494 	struct vnode   *ovp;
2495 	struct ucred   *cred;
2496 	struct thread  *td;
2497 	int		error;
2498 
2499 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2500 
2501 	error = EROFS;
2502 	unp = VTOUNIONFS(ap->a_vp);
2503 	uvp = unp->un_uppervp;
2504 	lvp = unp->un_lowervp;
2505 	ovp = NULLVP;
2506 	cred = ap->a_cred;
2507 	td = ap->a_td;
2508 
2509 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: enter (un_flag=%x)\n",
2510 	    unp->un_flag);
2511 
2512 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2513 		return (EROFS);
2514 
2515 	if (unp->un_flag & UNIONFS_OPENEXTU)
2516 		ovp = unp->un_uppervp;
2517 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2518 		ovp = unp->un_lowervp;
2519 
2520 	if (ovp == NULLVP)
2521 		return (EOPNOTSUPP);
2522 
2523 	if (ovp == lvp && lvp->v_type == VREG) {
2524 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2525 		if (uvp == NULLVP &&
2526 		    (error = unionfs_copyfile(unp, 1, cred, td)) != 0) {
2527 unionfs_deleteextattr_reopen:
2528 			if ((unp->un_flag & UNIONFS_OPENEXTL) &&
2529 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2530 #ifdef DIAGNOSTIC
2531 				panic("unionfs: VOP_OPENEXTATTR failed");
2532 #endif
2533 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2534 			}
2535 			goto unionfs_deleteextattr_abort;
2536 		}
2537 		uvp = unp->un_uppervp;
2538 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2539 			goto unionfs_deleteextattr_reopen;
2540 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2541 		unp->un_flag |= UNIONFS_OPENEXTU;
2542 		ovp = uvp;
2543 	}
2544 
2545 	if (ovp == uvp)
2546 		error = VOP_DELETEEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2547 		    ap->a_cred, ap->a_td);
2548 
2549 unionfs_deleteextattr_abort:
2550 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: leave (%d)\n", error);
2551 
2552 	return (error);
2553 }
2554 
2555 static int
2556 unionfs_setlabel(struct vop_setlabel_args *ap)
2557 {
2558 	struct unionfs_node *unp;
2559 	struct vnode   *uvp;
2560 	struct vnode   *lvp;
2561 	struct thread  *td;
2562 	int		error;
2563 
2564 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: enter\n");
2565 
2566 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2567 
2568 	error = EROFS;
2569 	unp = VTOUNIONFS(ap->a_vp);
2570 	uvp = unp->un_uppervp;
2571 	lvp = unp->un_lowervp;
2572 	td = ap->a_td;
2573 
2574 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2575 		return (EROFS);
2576 
2577 	if (uvp == NULLVP && lvp->v_type == VREG) {
2578 		if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0)
2579 			return (error);
2580 		uvp = unp->un_uppervp;
2581 	}
2582 
2583 	if (uvp != NULLVP)
2584 		error = VOP_SETLABEL(uvp, ap->a_label, ap->a_cred, td);
2585 
2586 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: leave (%d)\n", error);
2587 
2588 	return (error);
2589 }
2590 
2591 static int
2592 unionfs_vptofh(struct vop_vptofh_args *ap)
2593 {
2594 	return (EOPNOTSUPP);
2595 }
2596 
2597 static int
2598 unionfs_add_writecount(struct vop_add_writecount_args *ap)
2599 {
2600 	struct vnode *tvp, *vp;
2601 	struct unionfs_node *unp;
2602 	int error, writerefs __diagused;
2603 
2604 	vp = ap->a_vp;
2605 	unp = VTOUNIONFS(vp);
2606 	tvp = unp->un_uppervp;
2607 	KASSERT(tvp != NULL,
2608 	    ("%s: adding write ref without upper vnode", __func__));
2609 	error = VOP_ADD_WRITECOUNT(tvp, ap->a_inc);
2610 	if (error != 0)
2611 		return (error);
2612 	/*
2613 	 * We need to track the write refs we've passed to the underlying
2614 	 * vnodes so that we can undo them in case we are forcibly unmounted.
2615 	 */
2616 	writerefs = atomic_fetchadd_int(&vp->v_writecount, ap->a_inc);
2617 	/* text refs are bypassed to lowervp */
2618 	VNASSERT(writerefs >= 0, vp,
2619 	    ("%s: invalid write count %d", __func__, writerefs));
2620 	VNASSERT(writerefs + ap->a_inc >= 0, vp,
2621 	    ("%s: invalid write count inc %d + %d", __func__,
2622 	    writerefs, ap->a_inc));
2623 	return (0);
2624 }
2625 
2626 static int
2627 unionfs_vput_pair(struct vop_vput_pair_args *ap)
2628 {
2629 	struct mount *mp;
2630 	struct vnode *dvp, *vp, **vpp, *lvp, *ldvp, *uvp, *udvp, *tempvp;
2631 	struct unionfs_node *dunp, *unp;
2632 	int error, res;
2633 
2634 	dvp = ap->a_dvp;
2635 	vpp = ap->a_vpp;
2636 	vp = NULLVP;
2637 	lvp = NULLVP;
2638 	uvp = NULLVP;
2639 	unp = NULL;
2640 
2641 	dunp = VTOUNIONFS(dvp);
2642 	udvp = dunp->un_uppervp;
2643 	ldvp = dunp->un_lowervp;
2644 
2645 	/*
2646 	 * Underlying vnodes should be locked because the encompassing unionfs
2647 	 * node is locked, but will not be referenced, as the reference will
2648 	 * only be on the unionfs node.  Reference them now so that the vput()s
2649 	 * performed by VOP_VPUT_PAIR() will have a reference to drop.
2650 	 */
2651 	if (udvp != NULLVP)
2652 		vref(udvp);
2653 	if (ldvp != NULLVP)
2654 		vref(ldvp);
2655 
2656 	if (vpp != NULL)
2657 		vp = *vpp;
2658 
2659 	if (vp != NULLVP) {
2660 		unp = VTOUNIONFS(vp);
2661 		uvp = unp->un_uppervp;
2662 		lvp = unp->un_lowervp;
2663 		if (uvp != NULLVP)
2664 			vref(uvp);
2665 		if (lvp != NULLVP)
2666 			vref(lvp);
2667 
2668 		/*
2669 		 * If we're being asked to return a locked child vnode, then
2670 		 * we may need to create a replacement vnode in case the
2671 		 * original is reclaimed while the lock is dropped.  In that
2672 		 * case we'll need to ensure the mount and the underlying
2673 		 * vnodes aren't also recycled during that window.
2674 		 */
2675 		if (!ap->a_unlock_vp) {
2676 			vhold(vp);
2677 			if (uvp != NULLVP)
2678 				vhold(uvp);
2679 			if (lvp != NULLVP)
2680 				vhold(lvp);
2681 			mp = vp->v_mount;
2682 			vfs_ref(mp);
2683 		}
2684 	}
2685 
2686 	/*
2687 	 * TODO: Because unionfs_lock() locks both the lower and upper vnodes
2688 	 * (if available), we must also call VOP_VPUT_PAIR() on both the lower
2689 	 * and upper parent/child pairs.  If unionfs_lock() is reworked to lock
2690 	 * only a single vnode, this code will need to change to also only
2691 	 * operate on one vnode pair.
2692 	 */
2693 	ASSERT_VOP_LOCKED(ldvp, __func__);
2694 	ASSERT_VOP_LOCKED(udvp, __func__);
2695 	ASSERT_VOP_LOCKED(lvp, __func__);
2696 	ASSERT_VOP_LOCKED(uvp, __func__);
2697 
2698 	KASSERT(lvp == NULLVP || ldvp != NULLVP,
2699 	    ("%s: NULL ldvp with non-NULL lvp", __func__));
2700 	if (ldvp != NULLVP)
2701 		res = VOP_VPUT_PAIR(ldvp, lvp != NULLVP ? &lvp : NULL, true);
2702 	KASSERT(uvp == NULLVP || udvp != NULLVP,
2703 	    ("%s: NULL udvp with non-NULL uvp", __func__));
2704 	if (udvp != NULLVP)
2705 		res = VOP_VPUT_PAIR(udvp, uvp != NULLVP ? &uvp : NULL, true);
2706 
2707 	ASSERT_VOP_UNLOCKED(ldvp, __func__);
2708 	ASSERT_VOP_UNLOCKED(udvp, __func__);
2709 	ASSERT_VOP_UNLOCKED(lvp, __func__);
2710 	ASSERT_VOP_UNLOCKED(uvp, __func__);
2711 
2712 	/*
2713 	 * VOP_VPUT_PAIR() dropped the references we added to the underlying
2714 	 * vnodes, now drop the caller's reference to the unionfs vnodes.
2715 	 */
2716 	if (vp != NULLVP && ap->a_unlock_vp)
2717 		vrele(vp);
2718 	vrele(dvp);
2719 
2720 	if (vp == NULLVP || ap->a_unlock_vp)
2721 		return (res);
2722 
2723 	/*
2724 	 * We're being asked to return a locked vnode.  At this point, the
2725 	 * underlying vnodes have been unlocked, so vp may have been reclaimed.
2726 	 */
2727 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2728 	if (vp->v_data == NULL && vfs_busy(mp, MBF_NOWAIT) == 0) {
2729 		vput(vp);
2730 		error = unionfs_nodeget(mp, uvp, lvp, dvp, &tempvp, NULL);
2731 		if (error == 0) {
2732 			vn_lock(tempvp, LK_EXCLUSIVE | LK_RETRY);
2733 			*vpp = tempvp;
2734 		} else
2735 			vget(vp, LK_EXCLUSIVE | LK_RETRY);
2736 		vfs_unbusy(mp);
2737 	}
2738 	if (lvp != NULLVP)
2739 		vdrop(lvp);
2740 	if (uvp != NULLVP)
2741 		vdrop(uvp);
2742 	vdrop(vp);
2743 	vfs_rel(mp);
2744 
2745 	return (res);
2746 }
2747 
2748 static int
2749 unionfs_set_text(struct vop_set_text_args *ap)
2750 {
2751 	struct vnode *tvp;
2752 	struct unionfs_node *unp;
2753 	int error;
2754 
2755 	/*
2756 	 * We assume text refs are managed against lvp/uvp through the
2757 	 * executable mapping backed by its VM object.  We therefore don't
2758 	 * need to track leased text refs in the case of a forcible unmount.
2759 	 */
2760 	unp = VTOUNIONFS(ap->a_vp);
2761 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2762 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2763 	error = VOP_SET_TEXT(tvp);
2764 	return (error);
2765 }
2766 
2767 static int
2768 unionfs_unset_text(struct vop_unset_text_args *ap)
2769 {
2770 	struct vnode *tvp;
2771 	struct unionfs_node *unp;
2772 
2773 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2774 	unp = VTOUNIONFS(ap->a_vp);
2775 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2776 	VOP_UNSET_TEXT_CHECKED(tvp);
2777 	return (0);
2778 }
2779 
2780 struct vop_vector unionfs_vnodeops = {
2781 	.vop_default =		&default_vnodeops,
2782 
2783 	.vop_access =		unionfs_access,
2784 	.vop_aclcheck =		unionfs_aclcheck,
2785 	.vop_advlock =		unionfs_advlock,
2786 	.vop_bmap =		VOP_EOPNOTSUPP,
2787 	.vop_cachedlookup =	unionfs_lookup,
2788 	.vop_close =		unionfs_close,
2789 	.vop_closeextattr =	unionfs_closeextattr,
2790 	.vop_create =		unionfs_create,
2791 	.vop_deleteextattr =	unionfs_deleteextattr,
2792 	.vop_fsync =		unionfs_fsync,
2793 	.vop_getacl =		unionfs_getacl,
2794 	.vop_getattr =		unionfs_getattr,
2795 	.vop_getextattr =	unionfs_getextattr,
2796 	.vop_getwritemount =	unionfs_getwritemount,
2797 	.vop_inactive =		unionfs_inactive,
2798 	.vop_need_inactive =	vop_stdneed_inactive,
2799 	.vop_islocked =		vop_stdislocked,
2800 	.vop_ioctl =		unionfs_ioctl,
2801 	.vop_link =		unionfs_link,
2802 	.vop_listextattr =	unionfs_listextattr,
2803 	.vop_lock1 =		unionfs_lock,
2804 	.vop_lookup =		vfs_cache_lookup,
2805 	.vop_mkdir =		unionfs_mkdir,
2806 	.vop_mknod =		unionfs_mknod,
2807 	.vop_open =		unionfs_open,
2808 	.vop_openextattr =	unionfs_openextattr,
2809 	.vop_pathconf =		unionfs_pathconf,
2810 	.vop_poll =		unionfs_poll,
2811 	.vop_print =		unionfs_print,
2812 	.vop_read =		unionfs_read,
2813 	.vop_readdir =		unionfs_readdir,
2814 	.vop_readlink =		unionfs_readlink,
2815 	.vop_reclaim =		unionfs_reclaim,
2816 	.vop_remove =		unionfs_remove,
2817 	.vop_rename =		unionfs_rename,
2818 	.vop_rmdir =		unionfs_rmdir,
2819 	.vop_setacl =		unionfs_setacl,
2820 	.vop_setattr =		unionfs_setattr,
2821 	.vop_setextattr =	unionfs_setextattr,
2822 	.vop_setlabel =		unionfs_setlabel,
2823 	.vop_strategy =		unionfs_strategy,
2824 	.vop_symlink =		unionfs_symlink,
2825 	.vop_unlock =		unionfs_unlock,
2826 	.vop_whiteout =		unionfs_whiteout,
2827 	.vop_write =		unionfs_write,
2828 	.vop_vptofh =		unionfs_vptofh,
2829 	.vop_add_writecount =	unionfs_add_writecount,
2830 	.vop_vput_pair =	unionfs_vput_pair,
2831 	.vop_set_text =		unionfs_set_text,
2832 	.vop_unset_text = 	unionfs_unset_text,
2833 };
2834 VFS_VOP_VECTOR_REGISTER(unionfs_vnodeops);
2835