xref: /freebsd/sys/fs/unionfs/union_vnops.c (revision b9fa1500cb2265b95927e19b9d2119ca26d65be3)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1992, 1993, 1994, 1995 Jan-Simon Pendry.
5  * Copyright (c) 1992, 1993, 1994, 1995
6  *      The Regents of the University of California.
7  * Copyright (c) 2005, 2006, 2012 Masanori Ozawa <ozawa@ongs.co.jp>, ONGS Inc.
8  * Copyright (c) 2006, 2012 Daichi Goto <daichi@freebsd.org>
9  * All rights reserved.
10  *
11  * This code is derived from software contributed to Berkeley by
12  * Jan-Simon Pendry.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions
16  * are met:
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  * 3. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  *
38  */
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/conf.h>
43 #include <sys/kernel.h>
44 #include <sys/lock.h>
45 #include <sys/malloc.h>
46 #include <sys/mount.h>
47 #include <sys/mutex.h>
48 #include <sys/namei.h>
49 #include <sys/sysctl.h>
50 #include <sys/vnode.h>
51 #include <sys/kdb.h>
52 #include <sys/fcntl.h>
53 #include <sys/stat.h>
54 #include <sys/dirent.h>
55 #include <sys/proc.h>
56 #include <sys/bio.h>
57 #include <sys/buf.h>
58 
59 #include <fs/unionfs/union.h>
60 
61 #include <machine/atomic.h>
62 
63 #include <vm/vm.h>
64 #include <vm/vm_extern.h>
65 #include <vm/vm_object.h>
66 #include <vm/vnode_pager.h>
67 
68 #if 0
69 #define UNIONFS_INTERNAL_DEBUG(msg, args...)    printf(msg, ## args)
70 #define UNIONFS_IDBG_RENAME
71 #else
72 #define UNIONFS_INTERNAL_DEBUG(msg, args...)
73 #endif
74 
75 #define KASSERT_UNIONFS_VNODE(vp) \
76 	VNASSERT(((vp)->v_op == &unionfs_vnodeops), vp, \
77 	    ("%s: non-unionfs vnode", __func__))
78 
79 static int
80 unionfs_lookup(struct vop_cachedlookup_args *ap)
81 {
82 	struct unionfs_node *dunp;
83 	struct vnode   *dvp, *udvp, *ldvp, *vp, *uvp, *lvp, *dtmpvp;
84 	struct vattr	va;
85 	struct componentname *cnp;
86 	struct thread  *td;
87 	u_long		nameiop;
88 	u_long		cnflags, cnflagsbk;
89 	int		iswhiteout;
90 	int		lockflag;
91 	int		error , uerror, lerror;
92 
93 	iswhiteout = 0;
94 	lockflag = 0;
95 	error = uerror = lerror = ENOENT;
96 	cnp = ap->a_cnp;
97 	nameiop = cnp->cn_nameiop;
98 	cnflags = cnp->cn_flags;
99 	dvp = ap->a_dvp;
100 	dunp = VTOUNIONFS(dvp);
101 	udvp = dunp->un_uppervp;
102 	ldvp = dunp->un_lowervp;
103 	vp = uvp = lvp = NULLVP;
104 	td = curthread;
105 	*(ap->a_vpp) = NULLVP;
106 
107 	UNIONFS_INTERNAL_DEBUG(
108 	    "unionfs_lookup: enter: nameiop=%ld, flags=%lx, path=%s\n",
109 	    nameiop, cnflags, cnp->cn_nameptr);
110 
111 	if (dvp->v_type != VDIR)
112 		return (ENOTDIR);
113 
114 	/*
115 	 * If read-only and op is not LOOKUP, will return EROFS.
116 	 */
117 	if ((cnflags & ISLASTCN) &&
118 	    (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
119 	    LOOKUP != nameiop)
120 		return (EROFS);
121 
122 	/*
123 	 * lookup dotdot
124 	 */
125 	if (cnflags & ISDOTDOT) {
126 		if (LOOKUP != nameiop && udvp == NULLVP)
127 			return (EROFS);
128 
129 		if (udvp != NULLVP) {
130 			dtmpvp = udvp;
131 			if (ldvp != NULLVP)
132 				VOP_UNLOCK(ldvp);
133 		}
134 		else
135 			dtmpvp = ldvp;
136 
137 		error = VOP_LOOKUP(dtmpvp, &vp, cnp);
138 
139 		if (dtmpvp == udvp && ldvp != NULLVP) {
140 			VOP_UNLOCK(udvp);
141 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
142 		}
143 
144 		if (error == 0) {
145 			/*
146 			 * Exchange lock and reference from vp to
147 			 * dunp->un_dvp. vp is upper/lower vnode, but it
148 			 * will need to return the unionfs vnode.
149 			 */
150 			if (nameiop == DELETE  || nameiop == RENAME ||
151 			    (cnp->cn_lkflags & LK_TYPE_MASK))
152 				VOP_UNLOCK(vp);
153 			vrele(vp);
154 
155 			VOP_UNLOCK(dvp);
156 			*(ap->a_vpp) = dunp->un_dvp;
157 			vref(dunp->un_dvp);
158 
159 			if (nameiop == DELETE || nameiop == RENAME)
160 				vn_lock(dunp->un_dvp, LK_EXCLUSIVE | LK_RETRY);
161 			else if (cnp->cn_lkflags & LK_TYPE_MASK)
162 				vn_lock(dunp->un_dvp, cnp->cn_lkflags |
163 				    LK_RETRY);
164 
165 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
166 		} else if (error == ENOENT && (cnflags & MAKEENTRY) != 0)
167 			cache_enter(dvp, NULLVP, cnp);
168 
169 		goto unionfs_lookup_return;
170 	}
171 
172 	/*
173 	 * lookup upper layer
174 	 */
175 	if (udvp != NULLVP) {
176 		uerror = VOP_LOOKUP(udvp, &uvp, cnp);
177 
178 		if (uerror == 0) {
179 			if (udvp == uvp) {	/* is dot */
180 				vrele(uvp);
181 				*(ap->a_vpp) = dvp;
182 				vref(dvp);
183 
184 				error = uerror;
185 				goto unionfs_lookup_return;
186 			}
187 			if (nameiop == DELETE || nameiop == RENAME ||
188 			    (cnp->cn_lkflags & LK_TYPE_MASK))
189 				VOP_UNLOCK(uvp);
190 		}
191 
192 		/* check whiteout */
193 		if (uerror == ENOENT || uerror == EJUSTRETURN)
194 			if (cnp->cn_flags & ISWHITEOUT)
195 				iswhiteout = 1;	/* don't lookup lower */
196 		if (iswhiteout == 0 && ldvp != NULLVP)
197 			if (!VOP_GETATTR(udvp, &va, cnp->cn_cred) &&
198 			    (va.va_flags & OPAQUE))
199 				iswhiteout = 1;	/* don't lookup lower */
200 #if 0
201 		UNIONFS_INTERNAL_DEBUG(
202 		    "unionfs_lookup: debug: whiteout=%d, path=%s\n",
203 		    iswhiteout, cnp->cn_nameptr);
204 #endif
205 	}
206 
207 	/*
208 	 * lookup lower layer
209 	 */
210 	if (ldvp != NULLVP && !(cnflags & DOWHITEOUT) && iswhiteout == 0) {
211 		/* always op is LOOKUP */
212 		cnp->cn_nameiop = LOOKUP;
213 		cnflagsbk = cnp->cn_flags;
214 		cnp->cn_flags = cnflags;
215 
216 		lerror = VOP_LOOKUP(ldvp, &lvp, cnp);
217 
218 		cnp->cn_nameiop = nameiop;
219 		if (udvp != NULLVP && (uerror == 0 || uerror == EJUSTRETURN))
220 			cnp->cn_flags = cnflagsbk;
221 
222 		if (lerror == 0) {
223 			if (ldvp == lvp) {	/* is dot */
224 				if (uvp != NULLVP)
225 					vrele(uvp);	/* no need? */
226 				vrele(lvp);
227 				*(ap->a_vpp) = dvp;
228 				vref(dvp);
229 
230 				UNIONFS_INTERNAL_DEBUG(
231 				    "unionfs_lookup: leave (%d)\n", lerror);
232 
233 				return (lerror);
234 			}
235 			if (cnp->cn_lkflags & LK_TYPE_MASK)
236 				VOP_UNLOCK(lvp);
237 		}
238 	}
239 
240 	/*
241 	 * check lookup result
242 	 */
243 	if (uvp == NULLVP && lvp == NULLVP) {
244 		error = (udvp != NULLVP ? uerror : lerror);
245 		goto unionfs_lookup_return;
246 	}
247 
248 	/*
249 	 * check vnode type
250 	 */
251 	if (uvp != NULLVP && lvp != NULLVP && uvp->v_type != lvp->v_type) {
252 		vrele(lvp);
253 		lvp = NULLVP;
254 	}
255 
256 	/*
257 	 * check shadow dir
258 	 */
259 	if (uerror != 0 && uerror != EJUSTRETURN && udvp != NULLVP &&
260 	    lerror == 0 && lvp != NULLVP && lvp->v_type == VDIR &&
261 	    !(dvp->v_mount->mnt_flag & MNT_RDONLY) &&
262 	    (1 < cnp->cn_namelen || '.' != *(cnp->cn_nameptr))) {
263 		/* get unionfs vnode in order to create a new shadow dir. */
264 		error = unionfs_nodeget(dvp->v_mount, NULLVP, lvp, dvp, &vp,
265 		    cnp);
266 		if (error != 0)
267 			goto unionfs_lookup_cleanup;
268 
269 		if (LK_SHARED == (cnp->cn_lkflags & LK_TYPE_MASK))
270 			VOP_UNLOCK(vp);
271 		if (LK_EXCLUSIVE != VOP_ISLOCKED(vp)) {
272 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
273 			lockflag = 1;
274 		}
275 		error = unionfs_mkshadowdir(MOUNTTOUNIONFSMOUNT(dvp->v_mount),
276 		    udvp, VTOUNIONFS(vp), cnp, td);
277 		if (lockflag != 0)
278 			VOP_UNLOCK(vp);
279 		if (error != 0) {
280 			UNIONFSDEBUG(
281 			    "unionfs_lookup: Unable to create shadow dir.");
282 			if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE)
283 				vput(vp);
284 			else
285 				vrele(vp);
286 			goto unionfs_lookup_cleanup;
287 		}
288 		if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_SHARED)
289 			vn_lock(vp, LK_SHARED | LK_RETRY);
290 	}
291 	/*
292 	 * get unionfs vnode.
293 	 */
294 	else {
295 		if (uvp != NULLVP)
296 			error = uerror;
297 		else
298 			error = lerror;
299 		if (error != 0)
300 			goto unionfs_lookup_cleanup;
301 		error = unionfs_nodeget(dvp->v_mount, uvp, lvp,
302 		    dvp, &vp, cnp);
303 		if (error != 0) {
304 			UNIONFSDEBUG(
305 			    "unionfs_lookup: Unable to create unionfs vnode.");
306 			goto unionfs_lookup_cleanup;
307 		}
308 		if ((nameiop == DELETE || nameiop == RENAME) &&
309 		    (cnp->cn_lkflags & LK_TYPE_MASK) == 0)
310 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
311 	}
312 
313 	*(ap->a_vpp) = vp;
314 
315 	if (cnflags & MAKEENTRY)
316 		cache_enter(dvp, vp, cnp);
317 
318 unionfs_lookup_cleanup:
319 	if (uvp != NULLVP)
320 		vrele(uvp);
321 	if (lvp != NULLVP)
322 		vrele(lvp);
323 
324 	if (error == ENOENT && (cnflags & MAKEENTRY) != 0)
325 		cache_enter(dvp, NULLVP, cnp);
326 
327 unionfs_lookup_return:
328 
329 	UNIONFS_INTERNAL_DEBUG("unionfs_lookup: leave (%d)\n", error);
330 
331 	return (error);
332 }
333 
334 static int
335 unionfs_create(struct vop_create_args *ap)
336 {
337 	struct unionfs_node *dunp;
338 	struct componentname *cnp;
339 	struct vnode   *udvp;
340 	struct vnode   *vp;
341 	int		error;
342 
343 	UNIONFS_INTERNAL_DEBUG("unionfs_create: enter\n");
344 
345 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
346 
347 	dunp = VTOUNIONFS(ap->a_dvp);
348 	cnp = ap->a_cnp;
349 	udvp = dunp->un_uppervp;
350 	error = EROFS;
351 
352 	if (udvp != NULLVP) {
353 		int lkflags;
354 		bool vp_created = false;
355 		unionfs_forward_vop_start(udvp, &lkflags);
356 		error = VOP_CREATE(udvp, &vp, cnp, ap->a_vap);
357 		if (error == 0)
358 			vp_created = true;
359 		if (__predict_false(unionfs_forward_vop_finish(ap->a_dvp, udvp,
360 		    lkflags)) && error == 0) {
361 			error = ENOENT;
362 		}
363 		if (error == 0) {
364 			VOP_UNLOCK(vp);
365 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
366 			    ap->a_dvp, ap->a_vpp, cnp);
367 			vrele(vp);
368 		} else if (vp_created)
369 			vput(vp);
370 	}
371 
372 	UNIONFS_INTERNAL_DEBUG("unionfs_create: leave (%d)\n", error);
373 
374 	return (error);
375 }
376 
377 static int
378 unionfs_whiteout(struct vop_whiteout_args *ap)
379 {
380 	struct unionfs_node *dunp;
381 	struct componentname *cnp;
382 	struct vnode   *udvp;
383 	int		error;
384 
385 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: enter\n");
386 
387 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
388 
389 	dunp = VTOUNIONFS(ap->a_dvp);
390 	cnp = ap->a_cnp;
391 	udvp = dunp->un_uppervp;
392 	error = EOPNOTSUPP;
393 
394 	if (udvp != NULLVP) {
395 		int lkflags;
396 		switch (ap->a_flags) {
397 		case CREATE:
398 		case DELETE:
399 		case LOOKUP:
400 			unionfs_forward_vop_start(udvp, &lkflags);
401 			error = VOP_WHITEOUT(udvp, cnp, ap->a_flags);
402 			unionfs_forward_vop_finish(ap->a_dvp, udvp, lkflags);
403 			break;
404 		default:
405 			error = EINVAL;
406 			break;
407 		}
408 	}
409 
410 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: leave (%d)\n", error);
411 
412 	return (error);
413 }
414 
415 static int
416 unionfs_mknod(struct vop_mknod_args *ap)
417 {
418 	struct unionfs_node *dunp;
419 	struct componentname *cnp;
420 	struct vnode   *udvp;
421 	struct vnode   *vp;
422 	int		error;
423 
424 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: enter\n");
425 
426 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
427 
428 	dunp = VTOUNIONFS(ap->a_dvp);
429 	cnp = ap->a_cnp;
430 	udvp = dunp->un_uppervp;
431 	error = EROFS;
432 
433 	if (udvp != NULLVP) {
434 		int lkflags;
435 		bool vp_created = false;
436 		unionfs_forward_vop_start(udvp, &lkflags);
437 		error = VOP_MKNOD(udvp, &vp, cnp, ap->a_vap);
438 		if (error == 0)
439 			vp_created = true;
440 		if (__predict_false(unionfs_forward_vop_finish(ap->a_dvp, udvp,
441 		    lkflags)) && error == 0) {
442 			error = ENOENT;
443 		}
444 		if (error == 0) {
445 			VOP_UNLOCK(vp);
446 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
447 			    ap->a_dvp, ap->a_vpp, cnp);
448 			vrele(vp);
449 		} else if (vp_created)
450 			vput(vp);
451 	}
452 
453 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: leave (%d)\n", error);
454 
455 	return (error);
456 }
457 
458 enum unionfs_lkupgrade {
459 	UNIONFS_LKUPGRADE_SUCCESS, /* lock successfully upgraded */
460 	UNIONFS_LKUPGRADE_ALREADY, /* lock already held exclusive */
461 	UNIONFS_LKUPGRADE_DOOMED   /* lock was upgraded, but vnode reclaimed */
462 };
463 
464 static inline enum unionfs_lkupgrade
465 unionfs_upgrade_lock(struct vnode *vp)
466 {
467 	ASSERT_VOP_LOCKED(vp, __func__);
468 
469 	if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
470 		return (UNIONFS_LKUPGRADE_ALREADY);
471 
472 	if (vn_lock(vp, LK_UPGRADE) != 0) {
473 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
474 		if (VN_IS_DOOMED(vp))
475 			return (UNIONFS_LKUPGRADE_DOOMED);
476 	}
477 	return (UNIONFS_LKUPGRADE_SUCCESS);
478 }
479 
480 static inline void
481 unionfs_downgrade_lock(struct vnode *vp, enum unionfs_lkupgrade status)
482 {
483 	if (status != UNIONFS_LKUPGRADE_ALREADY)
484 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
485 }
486 
487 static int
488 unionfs_open(struct vop_open_args *ap)
489 {
490 	struct unionfs_node *unp;
491 	struct unionfs_node_status *unsp;
492 	struct vnode   *vp;
493 	struct vnode   *uvp;
494 	struct vnode   *lvp;
495 	struct vnode   *targetvp;
496 	struct ucred   *cred;
497 	struct thread  *td;
498 	int		error;
499 	enum unionfs_lkupgrade lkstatus;
500 
501 	UNIONFS_INTERNAL_DEBUG("unionfs_open: enter\n");
502 
503 	KASSERT_UNIONFS_VNODE(ap->a_vp);
504 
505 	error = 0;
506 	vp = ap->a_vp;
507 	targetvp = NULLVP;
508 	cred = ap->a_cred;
509 	td = ap->a_td;
510 
511 	/*
512 	 * The executable loader path may call this function with vp locked
513 	 * shared.  If the vnode is reclaimed while upgrading, we can't safely
514 	 * use unp or do anything else unionfs- specific.
515 	 */
516 	lkstatus = unionfs_upgrade_lock(vp);
517 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED) {
518 		error = ENOENT;
519 		goto unionfs_open_cleanup;
520 	}
521 
522 	unp = VTOUNIONFS(vp);
523 	uvp = unp->un_uppervp;
524 	lvp = unp->un_lowervp;
525 	unionfs_get_node_status(unp, td, &unsp);
526 
527 	if (unsp->uns_lower_opencnt > 0 || unsp->uns_upper_opencnt > 0) {
528 		/* vnode is already opend. */
529 		if (unsp->uns_upper_opencnt > 0)
530 			targetvp = uvp;
531 		else
532 			targetvp = lvp;
533 
534 		if (targetvp == lvp &&
535 		    (ap->a_mode & FWRITE) && lvp->v_type == VREG)
536 			targetvp = NULLVP;
537 	}
538 	if (targetvp == NULLVP) {
539 		if (uvp == NULLVP) {
540 			if ((ap->a_mode & FWRITE) && lvp->v_type == VREG) {
541 				error = unionfs_copyfile(unp,
542 				    !(ap->a_mode & O_TRUNC), cred, td);
543 				if (error != 0)
544 					goto unionfs_open_abort;
545 				targetvp = uvp = unp->un_uppervp;
546 			} else
547 				targetvp = lvp;
548 		} else
549 			targetvp = uvp;
550 	}
551 
552 	error = VOP_OPEN(targetvp, ap->a_mode, cred, td, ap->a_fp);
553 	if (error == 0) {
554 		if (targetvp == uvp) {
555 			if (uvp->v_type == VDIR && lvp != NULLVP &&
556 			    unsp->uns_lower_opencnt <= 0) {
557 				/* open lower for readdir */
558 				error = VOP_OPEN(lvp, FREAD, cred, td, NULL);
559 				if (error != 0) {
560 					VOP_CLOSE(uvp, ap->a_mode, cred, td);
561 					goto unionfs_open_abort;
562 				}
563 				unsp->uns_node_flag |= UNS_OPENL_4_READDIR;
564 				unsp->uns_lower_opencnt++;
565 			}
566 			unsp->uns_upper_opencnt++;
567 		} else {
568 			unsp->uns_lower_opencnt++;
569 			unsp->uns_lower_openmode = ap->a_mode;
570 		}
571 		vp->v_object = targetvp->v_object;
572 	}
573 
574 unionfs_open_abort:
575 	if (error != 0)
576 		unionfs_tryrem_node_status(unp, unsp);
577 
578 unionfs_open_cleanup:
579 	unionfs_downgrade_lock(vp, lkstatus);
580 
581 	UNIONFS_INTERNAL_DEBUG("unionfs_open: leave (%d)\n", error);
582 
583 	return (error);
584 }
585 
586 static int
587 unionfs_close(struct vop_close_args *ap)
588 {
589 	struct unionfs_node *unp;
590 	struct unionfs_node_status *unsp;
591 	struct ucred   *cred;
592 	struct thread  *td;
593 	struct vnode   *vp;
594 	struct vnode   *ovp;
595 	int		error;
596 	enum unionfs_lkupgrade lkstatus;
597 
598 	UNIONFS_INTERNAL_DEBUG("unionfs_close: enter\n");
599 
600 	KASSERT_UNIONFS_VNODE(ap->a_vp);
601 
602 	vp = ap->a_vp;
603 	cred = ap->a_cred;
604 	td = ap->a_td;
605 	error = 0;
606 
607 	/*
608 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
609 	 * or do anything else unionfs- specific.
610 	 */
611 	lkstatus = unionfs_upgrade_lock(vp);
612 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
613 		goto unionfs_close_cleanup;
614 
615 	unp = VTOUNIONFS(vp);
616 	unionfs_get_node_status(unp, td, &unsp);
617 
618 	if (unsp->uns_lower_opencnt <= 0 && unsp->uns_upper_opencnt <= 0) {
619 #ifdef DIAGNOSTIC
620 		printf("unionfs_close: warning: open count is 0\n");
621 #endif
622 		if (unp->un_uppervp != NULLVP)
623 			ovp = unp->un_uppervp;
624 		else
625 			ovp = unp->un_lowervp;
626 	} else if (unsp->uns_upper_opencnt > 0)
627 		ovp = unp->un_uppervp;
628 	else
629 		ovp = unp->un_lowervp;
630 
631 	error = VOP_CLOSE(ovp, ap->a_fflag, cred, td);
632 
633 	if (error != 0)
634 		goto unionfs_close_abort;
635 
636 	vp->v_object = ovp->v_object;
637 
638 	if (ovp == unp->un_uppervp) {
639 		unsp->uns_upper_opencnt--;
640 		if (unsp->uns_upper_opencnt == 0) {
641 			if (unsp->uns_node_flag & UNS_OPENL_4_READDIR) {
642 				VOP_CLOSE(unp->un_lowervp, FREAD, cred, td);
643 				unsp->uns_node_flag &= ~UNS_OPENL_4_READDIR;
644 				unsp->uns_lower_opencnt--;
645 			}
646 			if (unsp->uns_lower_opencnt > 0)
647 				vp->v_object = unp->un_lowervp->v_object;
648 		}
649 	} else
650 		unsp->uns_lower_opencnt--;
651 
652 unionfs_close_abort:
653 	unionfs_tryrem_node_status(unp, unsp);
654 
655 unionfs_close_cleanup:
656 	unionfs_downgrade_lock(vp, lkstatus);
657 
658 	UNIONFS_INTERNAL_DEBUG("unionfs_close: leave (%d)\n", error);
659 
660 	return (error);
661 }
662 
663 /*
664  * Check the access mode toward shadow file/dir.
665  */
666 static int
667 unionfs_check_corrected_access(accmode_t accmode, struct vattr *va,
668     struct ucred *cred)
669 {
670 	uid_t		uid;	/* upper side vnode's uid */
671 	gid_t		gid;	/* upper side vnode's gid */
672 	u_short		vmode;	/* upper side vnode's mode */
673 	u_short		mask;
674 
675 	mask = 0;
676 	uid = va->va_uid;
677 	gid = va->va_gid;
678 	vmode = va->va_mode;
679 
680 	/* check owner */
681 	if (cred->cr_uid == uid) {
682 		if (accmode & VEXEC)
683 			mask |= S_IXUSR;
684 		if (accmode & VREAD)
685 			mask |= S_IRUSR;
686 		if (accmode & VWRITE)
687 			mask |= S_IWUSR;
688 		return ((vmode & mask) == mask ? 0 : EACCES);
689 	}
690 
691 	/* check group */
692 	if (groupmember(gid, cred)) {
693 		if (accmode & VEXEC)
694 			mask |= S_IXGRP;
695 		if (accmode & VREAD)
696 			mask |= S_IRGRP;
697 		if (accmode & VWRITE)
698 			mask |= S_IWGRP;
699 		return ((vmode & mask) == mask ? 0 : EACCES);
700 	}
701 
702 	/* check other */
703 	if (accmode & VEXEC)
704 		mask |= S_IXOTH;
705 	if (accmode & VREAD)
706 		mask |= S_IROTH;
707 	if (accmode & VWRITE)
708 		mask |= S_IWOTH;
709 
710 	return ((vmode & mask) == mask ? 0 : EACCES);
711 }
712 
713 static int
714 unionfs_access(struct vop_access_args *ap)
715 {
716 	struct unionfs_mount *ump;
717 	struct unionfs_node *unp;
718 	struct vnode   *uvp;
719 	struct vnode   *lvp;
720 	struct thread  *td;
721 	struct vattr	va;
722 	accmode_t	accmode;
723 	int		error;
724 
725 	UNIONFS_INTERNAL_DEBUG("unionfs_access: enter\n");
726 
727 	KASSERT_UNIONFS_VNODE(ap->a_vp);
728 
729 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
730 	unp = VTOUNIONFS(ap->a_vp);
731 	uvp = unp->un_uppervp;
732 	lvp = unp->un_lowervp;
733 	td = ap->a_td;
734 	accmode = ap->a_accmode;
735 	error = EACCES;
736 
737 	if ((accmode & VWRITE) &&
738 	    (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)) {
739 		switch (ap->a_vp->v_type) {
740 		case VREG:
741 		case VDIR:
742 		case VLNK:
743 			return (EROFS);
744 		default:
745 			break;
746 		}
747 	}
748 
749 	if (uvp != NULLVP) {
750 		error = VOP_ACCESS(uvp, accmode, ap->a_cred, td);
751 
752 		UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
753 
754 		return (error);
755 	}
756 
757 	if (lvp != NULLVP) {
758 		if (accmode & VWRITE) {
759 			if ((ump->um_uppermp->mnt_flag & MNT_RDONLY) != 0) {
760 				switch (ap->a_vp->v_type) {
761 				case VREG:
762 				case VDIR:
763 				case VLNK:
764 					return (EROFS);
765 				default:
766 					break;
767 				}
768 			} else if (ap->a_vp->v_type == VREG ||
769 			    ap->a_vp->v_type == VDIR) {
770 				/* check shadow file/dir */
771 				if (ump->um_copymode != UNIONFS_TRANSPARENT) {
772 					error = unionfs_create_uppervattr(ump,
773 					    lvp, &va, ap->a_cred, td);
774 					if (error != 0)
775 						return (error);
776 
777 					error = unionfs_check_corrected_access(
778 					    accmode, &va, ap->a_cred);
779 					if (error != 0)
780 						return (error);
781 				}
782 			}
783 			accmode &= ~(VWRITE | VAPPEND);
784 			accmode |= VREAD; /* will copy to upper */
785 		}
786 		error = VOP_ACCESS(lvp, accmode, ap->a_cred, td);
787 	}
788 
789 	UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
790 
791 	return (error);
792 }
793 
794 static int
795 unionfs_getattr(struct vop_getattr_args *ap)
796 {
797 	struct unionfs_node *unp;
798 	struct unionfs_mount *ump;
799 	struct vnode   *uvp;
800 	struct vnode   *lvp;
801 	struct thread  *td;
802 	struct vattr	va;
803 	int		error;
804 
805 	UNIONFS_INTERNAL_DEBUG("unionfs_getattr: enter\n");
806 
807 	KASSERT_UNIONFS_VNODE(ap->a_vp);
808 
809 	unp = VTOUNIONFS(ap->a_vp);
810 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
811 	uvp = unp->un_uppervp;
812 	lvp = unp->un_lowervp;
813 	td = curthread;
814 
815 	if (uvp != NULLVP) {
816 		if ((error = VOP_GETATTR(uvp, ap->a_vap, ap->a_cred)) == 0)
817 			ap->a_vap->va_fsid =
818 			    ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
819 
820 		UNIONFS_INTERNAL_DEBUG(
821 		    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
822 		    ap->a_vap->va_mode, ap->a_vap->va_uid,
823 		    ap->a_vap->va_gid, error);
824 
825 		return (error);
826 	}
827 
828 	error = VOP_GETATTR(lvp, ap->a_vap, ap->a_cred);
829 
830 	if (error == 0 && (ump->um_uppermp->mnt_flag & MNT_RDONLY) == 0) {
831 		/* correct the attr toward shadow file/dir. */
832 		if (ap->a_vp->v_type == VREG || ap->a_vp->v_type == VDIR) {
833 			unionfs_create_uppervattr_core(ump, ap->a_vap, &va, td);
834 			ap->a_vap->va_mode = va.va_mode;
835 			ap->a_vap->va_uid = va.va_uid;
836 			ap->a_vap->va_gid = va.va_gid;
837 		}
838 	}
839 
840 	if (error == 0)
841 		ap->a_vap->va_fsid = ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
842 
843 	UNIONFS_INTERNAL_DEBUG(
844 	    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
845 	    ap->a_vap->va_mode, ap->a_vap->va_uid, ap->a_vap->va_gid, error);
846 
847 	return (error);
848 }
849 
850 static int
851 unionfs_setattr(struct vop_setattr_args *ap)
852 {
853 	struct unionfs_node *unp;
854 	struct vnode   *uvp;
855 	struct vnode   *lvp;
856 	struct thread  *td;
857 	struct vattr   *vap;
858 	int		error;
859 
860 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: enter\n");
861 
862 	KASSERT_UNIONFS_VNODE(ap->a_vp);
863 
864 	error = EROFS;
865 	unp = VTOUNIONFS(ap->a_vp);
866 	uvp = unp->un_uppervp;
867 	lvp = unp->un_lowervp;
868 	td = curthread;
869 	vap = ap->a_vap;
870 
871 	if ((ap->a_vp->v_mount->mnt_flag & MNT_RDONLY) &&
872 	    (vap->va_flags != VNOVAL || vap->va_uid != (uid_t)VNOVAL ||
873 	     vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL ||
874 	     vap->va_mtime.tv_sec != VNOVAL || vap->va_mode != (mode_t)VNOVAL))
875 		return (EROFS);
876 
877 	if (uvp == NULLVP && lvp->v_type == VREG) {
878 		error = unionfs_copyfile(unp, (vap->va_size != 0),
879 		    ap->a_cred, td);
880 		if (error != 0)
881 			return (error);
882 		uvp = unp->un_uppervp;
883 	}
884 
885 	if (uvp != NULLVP) {
886 		int lkflags;
887 		unionfs_forward_vop_start(uvp, &lkflags);
888 		error = VOP_SETATTR(uvp, vap, ap->a_cred);
889 		unionfs_forward_vop_finish(ap->a_vp, uvp, lkflags);
890 	}
891 
892 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: leave (%d)\n", error);
893 
894 	return (error);
895 }
896 
897 static int
898 unionfs_read(struct vop_read_args *ap)
899 {
900 	struct unionfs_node *unp;
901 	struct vnode   *tvp;
902 	int		error;
903 
904 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: enter\n"); */
905 
906 	KASSERT_UNIONFS_VNODE(ap->a_vp);
907 
908 	unp = VTOUNIONFS(ap->a_vp);
909 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
910 
911 	error = VOP_READ(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
912 
913 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: leave (%d)\n", error); */
914 
915 	return (error);
916 }
917 
918 static int
919 unionfs_write(struct vop_write_args *ap)
920 {
921 	struct unionfs_node *unp;
922 	struct vnode   *tvp;
923 	int		error;
924 	int		lkflags;
925 
926 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: enter\n"); */
927 
928 	KASSERT_UNIONFS_VNODE(ap->a_vp);
929 
930 	unp = VTOUNIONFS(ap->a_vp);
931 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
932 
933 	unionfs_forward_vop_start(tvp, &lkflags);
934 	error = VOP_WRITE(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
935 	unionfs_forward_vop_finish(ap->a_vp, tvp, lkflags);
936 
937 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: leave (%d)\n", error); */
938 
939 	return (error);
940 }
941 
942 static int
943 unionfs_ioctl(struct vop_ioctl_args *ap)
944 {
945 	struct unionfs_node *unp;
946 	struct unionfs_node_status *unsp;
947 	struct vnode   *ovp;
948 	int error;
949 
950 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: enter\n");
951 
952 	KASSERT_UNIONFS_VNODE(ap->a_vp);
953 
954  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
955 	unp = VTOUNIONFS(ap->a_vp);
956 	unionfs_get_node_status(unp, ap->a_td, &unsp);
957 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
958 	unionfs_tryrem_node_status(unp, unsp);
959 	VOP_UNLOCK(ap->a_vp);
960 
961 	if (ovp == NULLVP)
962 		return (EBADF);
963 
964 	error = VOP_IOCTL(ovp, ap->a_command, ap->a_data, ap->a_fflag,
965 	    ap->a_cred, ap->a_td);
966 
967 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: leave (%d)\n", error);
968 
969 	return (error);
970 }
971 
972 static int
973 unionfs_poll(struct vop_poll_args *ap)
974 {
975 	struct unionfs_node *unp;
976 	struct unionfs_node_status *unsp;
977 	struct vnode *ovp;
978 
979 	KASSERT_UNIONFS_VNODE(ap->a_vp);
980 
981  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
982 	unp = VTOUNIONFS(ap->a_vp);
983 	unionfs_get_node_status(unp, ap->a_td, &unsp);
984 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
985 	unionfs_tryrem_node_status(unp, unsp);
986 	VOP_UNLOCK(ap->a_vp);
987 
988 	if (ovp == NULLVP)
989 		return (EBADF);
990 
991 	return (VOP_POLL(ovp, ap->a_events, ap->a_cred, ap->a_td));
992 }
993 
994 static int
995 unionfs_fsync(struct vop_fsync_args *ap)
996 {
997 	struct unionfs_node *unp;
998 	struct unionfs_node_status *unsp;
999 	struct vnode *ovp;
1000 	enum unionfs_lkupgrade lkstatus;
1001 	int error, lkflags;
1002 
1003 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1004 
1005 	unp = VTOUNIONFS(ap->a_vp);
1006 	lkstatus = unionfs_upgrade_lock(ap->a_vp);
1007 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED) {
1008 		unionfs_downgrade_lock(ap->a_vp, lkstatus);
1009 		return (ENOENT);
1010 	}
1011 	unionfs_get_node_status(unp, ap->a_td, &unsp);
1012 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
1013 	unionfs_tryrem_node_status(unp, unsp);
1014 
1015 	unionfs_downgrade_lock(ap->a_vp, lkstatus);
1016 
1017 	if (ovp == NULLVP)
1018 		return (EBADF);
1019 
1020 	unionfs_forward_vop_start(ovp, &lkflags);
1021 	error = VOP_FSYNC(ovp, ap->a_waitfor, ap->a_td);
1022 	unionfs_forward_vop_finish(ap->a_vp, ovp, lkflags);
1023 
1024 	return (error);
1025 }
1026 
1027 static int
1028 unionfs_remove(struct vop_remove_args *ap)
1029 {
1030 	char	       *path;
1031 	struct unionfs_node *dunp;
1032 	struct unionfs_node *unp;
1033 	struct unionfs_mount *ump;
1034 	struct vnode   *udvp;
1035 	struct vnode   *uvp;
1036 	struct vnode   *lvp;
1037 	struct componentname *cnp;
1038 	struct thread  *td;
1039 	int		error;
1040 	int		pathlen;
1041 
1042 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: enter\n");
1043 
1044 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1045 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1046 
1047 	error = 0;
1048 	dunp = VTOUNIONFS(ap->a_dvp);
1049 	udvp = dunp->un_uppervp;
1050 	cnp = ap->a_cnp;
1051 	td = curthread;
1052 
1053 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1054 	unp = VTOUNIONFS(ap->a_vp);
1055 	uvp = unp->un_uppervp;
1056 	lvp = unp->un_lowervp;
1057 	path = unp->un_path;
1058 	pathlen = unp->un_pathlen;
1059 
1060 	if (udvp == NULLVP)
1061 		return (EROFS);
1062 
1063 	if (uvp != NULLVP) {
1064 		int udvp_lkflags, uvp_lkflags;
1065 		if (ump == NULL || ump->um_whitemode == UNIONFS_WHITE_ALWAYS ||
1066 		    lvp != NULLVP)
1067 			cnp->cn_flags |= DOWHITEOUT;
1068 		unionfs_forward_vop_start_pair(udvp, &udvp_lkflags,
1069 		    uvp, &uvp_lkflags);
1070 		error = VOP_REMOVE(udvp, uvp, cnp);
1071 		unionfs_forward_vop_finish_pair(ap->a_dvp, udvp, udvp_lkflags,
1072 		    ap->a_vp, uvp, uvp_lkflags);
1073 	} else if (lvp != NULLVP)
1074 		error = unionfs_mkwhiteout(ap->a_dvp, udvp, cnp, td, path, pathlen);
1075 
1076 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: leave (%d)\n", error);
1077 
1078 	return (error);
1079 }
1080 
1081 static int
1082 unionfs_link(struct vop_link_args *ap)
1083 {
1084 	struct unionfs_node *dunp;
1085 	struct unionfs_node *unp;
1086 	struct vnode   *udvp;
1087 	struct vnode   *uvp;
1088 	struct componentname *cnp;
1089 	struct thread  *td;
1090 	int		error;
1091 	int		needrelookup;
1092 
1093 	UNIONFS_INTERNAL_DEBUG("unionfs_link: enter\n");
1094 
1095 	KASSERT_UNIONFS_VNODE(ap->a_tdvp);
1096 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1097 
1098 	error = 0;
1099 	needrelookup = 0;
1100 	dunp = VTOUNIONFS(ap->a_tdvp);
1101 	unp = NULL;
1102 	udvp = dunp->un_uppervp;
1103 	uvp = NULLVP;
1104 	cnp = ap->a_cnp;
1105 	td = curthread;
1106 
1107 	if (udvp == NULLVP)
1108 		return (EROFS);
1109 
1110 	unp = VTOUNIONFS(ap->a_vp);
1111 
1112 	if (unp->un_uppervp == NULLVP) {
1113 		if (ap->a_vp->v_type != VREG)
1114 			return (EOPNOTSUPP);
1115 
1116 		error = unionfs_copyfile(unp, 1, cnp->cn_cred, td);
1117 		if (error != 0)
1118 			return (error);
1119 		needrelookup = 1;
1120 	}
1121 	uvp = unp->un_uppervp;
1122 
1123 	if (needrelookup != 0)
1124 		error = unionfs_relookup_for_create(ap->a_tdvp, cnp, td);
1125 
1126 	if (error == 0) {
1127 		int udvp_lkflags, uvp_lkflags;
1128 		unionfs_forward_vop_start_pair(udvp, &udvp_lkflags,
1129 		    uvp, &uvp_lkflags);
1130 		error = VOP_LINK(udvp, uvp, cnp);
1131 		unionfs_forward_vop_finish_pair(ap->a_tdvp, udvp, udvp_lkflags,
1132 		    ap->a_vp, uvp, uvp_lkflags);
1133 	}
1134 
1135 	UNIONFS_INTERNAL_DEBUG("unionfs_link: leave (%d)\n", error);
1136 
1137 	return (error);
1138 }
1139 
1140 static int
1141 unionfs_rename(struct vop_rename_args *ap)
1142 {
1143 	struct vnode   *fdvp;
1144 	struct vnode   *fvp;
1145 	struct componentname *fcnp;
1146 	struct vnode   *tdvp;
1147 	struct vnode   *tvp;
1148 	struct componentname *tcnp;
1149 	struct vnode   *ltdvp;
1150 	struct vnode   *ltvp;
1151 	struct thread  *td;
1152 
1153 	/* rename target vnodes */
1154 	struct vnode   *rfdvp;
1155 	struct vnode   *rfvp;
1156 	struct vnode   *rtdvp;
1157 	struct vnode   *rtvp;
1158 
1159 	struct unionfs_mount *ump;
1160 	struct unionfs_node *unp;
1161 	int		error;
1162 	int		needrelookup;
1163 
1164 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: enter\n");
1165 
1166 	error = 0;
1167 	fdvp = ap->a_fdvp;
1168 	fvp = ap->a_fvp;
1169 	fcnp = ap->a_fcnp;
1170 	tdvp = ap->a_tdvp;
1171 	tvp = ap->a_tvp;
1172 	tcnp = ap->a_tcnp;
1173 	ltdvp = NULLVP;
1174 	ltvp = NULLVP;
1175 	td = curthread;
1176 	rfdvp = fdvp;
1177 	rfvp = fvp;
1178 	rtdvp = tdvp;
1179 	rtvp = tvp;
1180 	needrelookup = 0;
1181 
1182 	/* check for cross device rename */
1183 	if (fvp->v_mount != tdvp->v_mount ||
1184 	    (tvp != NULLVP && fvp->v_mount != tvp->v_mount)) {
1185 		if (fvp->v_op != &unionfs_vnodeops)
1186 			error = ENODEV;
1187 		else
1188 			error = EXDEV;
1189 		goto unionfs_rename_abort;
1190 	}
1191 
1192 	/* Renaming a file to itself has no effect. */
1193 	if (fvp == tvp)
1194 		goto unionfs_rename_abort;
1195 
1196 	/*
1197 	 * from/to vnode is unionfs node.
1198 	 */
1199 
1200 	KASSERT_UNIONFS_VNODE(fdvp);
1201 	KASSERT_UNIONFS_VNODE(fvp);
1202 	KASSERT_UNIONFS_VNODE(tdvp);
1203 	if (tvp != NULLVP)
1204 		KASSERT_UNIONFS_VNODE(tvp);
1205 
1206 	unp = VTOUNIONFS(fdvp);
1207 #ifdef UNIONFS_IDBG_RENAME
1208 	UNIONFS_INTERNAL_DEBUG("fdvp=%p, ufdvp=%p, lfdvp=%p\n",
1209 	    fdvp, unp->un_uppervp, unp->un_lowervp);
1210 #endif
1211 	if (unp->un_uppervp == NULLVP) {
1212 		error = ENODEV;
1213 		goto unionfs_rename_abort;
1214 	}
1215 	rfdvp = unp->un_uppervp;
1216 	vref(rfdvp);
1217 
1218 	unp = VTOUNIONFS(fvp);
1219 #ifdef UNIONFS_IDBG_RENAME
1220 	UNIONFS_INTERNAL_DEBUG("fvp=%p, ufvp=%p, lfvp=%p\n",
1221 	    fvp, unp->un_uppervp, unp->un_lowervp);
1222 #endif
1223 	ump = MOUNTTOUNIONFSMOUNT(fvp->v_mount);
1224 	if (unp->un_uppervp == NULLVP) {
1225 		switch (fvp->v_type) {
1226 		case VREG:
1227 			if ((error = vn_lock(fvp, LK_EXCLUSIVE)) != 0)
1228 				goto unionfs_rename_abort;
1229 			error = unionfs_copyfile(unp, 1, fcnp->cn_cred, td);
1230 			VOP_UNLOCK(fvp);
1231 			if (error != 0)
1232 				goto unionfs_rename_abort;
1233 			break;
1234 		case VDIR:
1235 			if ((error = vn_lock(fvp, LK_EXCLUSIVE)) != 0)
1236 				goto unionfs_rename_abort;
1237 			error = unionfs_mkshadowdir(ump, rfdvp, unp, fcnp, td);
1238 			VOP_UNLOCK(fvp);
1239 			if (error != 0)
1240 				goto unionfs_rename_abort;
1241 			break;
1242 		default:
1243 			error = ENODEV;
1244 			goto unionfs_rename_abort;
1245 		}
1246 
1247 		needrelookup = 1;
1248 	}
1249 
1250 	if (unp->un_lowervp != NULLVP)
1251 		fcnp->cn_flags |= DOWHITEOUT;
1252 	rfvp = unp->un_uppervp;
1253 	vref(rfvp);
1254 
1255 	unp = VTOUNIONFS(tdvp);
1256 #ifdef UNIONFS_IDBG_RENAME
1257 	UNIONFS_INTERNAL_DEBUG("tdvp=%p, utdvp=%p, ltdvp=%p\n",
1258 	    tdvp, unp->un_uppervp, unp->un_lowervp);
1259 #endif
1260 	if (unp->un_uppervp == NULLVP) {
1261 		error = ENODEV;
1262 		goto unionfs_rename_abort;
1263 	}
1264 	rtdvp = unp->un_uppervp;
1265 	ltdvp = unp->un_lowervp;
1266 	vref(rtdvp);
1267 
1268 	if (tdvp == tvp) {
1269 		rtvp = rtdvp;
1270 		vref(rtvp);
1271 	} else if (tvp != NULLVP) {
1272 		unp = VTOUNIONFS(tvp);
1273 #ifdef UNIONFS_IDBG_RENAME
1274 		UNIONFS_INTERNAL_DEBUG("tvp=%p, utvp=%p, ltvp=%p\n",
1275 		    tvp, unp->un_uppervp, unp->un_lowervp);
1276 #endif
1277 		if (unp->un_uppervp == NULLVP)
1278 			rtvp = NULLVP;
1279 		else {
1280 			if (tvp->v_type == VDIR) {
1281 				error = EINVAL;
1282 				goto unionfs_rename_abort;
1283 			}
1284 			rtvp = unp->un_uppervp;
1285 			ltvp = unp->un_lowervp;
1286 			vref(rtvp);
1287 		}
1288 	}
1289 
1290 	if (rfvp == rtvp)
1291 		goto unionfs_rename_abort;
1292 
1293 	if (needrelookup != 0) {
1294 		if ((error = vn_lock(fdvp, LK_EXCLUSIVE)) != 0)
1295 			goto unionfs_rename_abort;
1296 		error = unionfs_relookup_for_delete(fdvp, fcnp, td);
1297 		VOP_UNLOCK(fdvp);
1298 		if (error != 0)
1299 			goto unionfs_rename_abort;
1300 
1301 		/* Lock of tvp is canceled in order to avoid recursive lock. */
1302 		if (tvp != NULLVP && tvp != tdvp)
1303 			VOP_UNLOCK(tvp);
1304 		error = unionfs_relookup_for_rename(tdvp, tcnp, td);
1305 		if (tvp != NULLVP && tvp != tdvp)
1306 			vn_lock(tvp, LK_EXCLUSIVE | LK_RETRY);
1307 		if (error != 0)
1308 			goto unionfs_rename_abort;
1309 	}
1310 
1311 	error = VOP_RENAME(rfdvp, rfvp, fcnp, rtdvp, rtvp, tcnp);
1312 
1313 	if (error == 0) {
1314 		if (rtvp != NULLVP && rtvp->v_type == VDIR)
1315 			cache_purge(tdvp);
1316 		if (fvp->v_type == VDIR && fdvp != tdvp)
1317 			cache_purge(fdvp);
1318 	}
1319 
1320 	if (ltdvp != NULLVP)
1321 		VOP_UNLOCK(ltdvp);
1322 	if (tdvp != rtdvp)
1323 		vrele(tdvp);
1324 	if (ltvp != NULLVP)
1325 		VOP_UNLOCK(ltvp);
1326 	if (tvp != rtvp && tvp != NULLVP) {
1327 		if (rtvp == NULLVP)
1328 			vput(tvp);
1329 		else
1330 			vrele(tvp);
1331 	}
1332 	if (fdvp != rfdvp)
1333 		vrele(fdvp);
1334 	if (fvp != rfvp)
1335 		vrele(fvp);
1336 
1337 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1338 
1339 	return (error);
1340 
1341 unionfs_rename_abort:
1342 	vput(tdvp);
1343 	if (tdvp != rtdvp)
1344 		vrele(rtdvp);
1345 	if (tvp != NULLVP) {
1346 		if (tdvp != tvp)
1347 			vput(tvp);
1348 		else
1349 			vrele(tvp);
1350 	}
1351 	if (tvp != rtvp && rtvp != NULLVP)
1352 		vrele(rtvp);
1353 	if (fdvp != rfdvp)
1354 		vrele(rfdvp);
1355 	if (fvp != rfvp)
1356 		vrele(rfvp);
1357 	vrele(fdvp);
1358 	vrele(fvp);
1359 
1360 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1361 
1362 	return (error);
1363 }
1364 
1365 static int
1366 unionfs_mkdir(struct vop_mkdir_args *ap)
1367 {
1368 	struct unionfs_node *dunp;
1369 	struct componentname *cnp;
1370 	struct vnode   *dvp;
1371 	struct vnode   *udvp;
1372 	struct vnode   *uvp;
1373 	struct vattr	va;
1374 	int		error;
1375 	int		lkflags;
1376 
1377 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: enter\n");
1378 
1379 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1380 
1381 	error = EROFS;
1382 	dvp = ap->a_dvp;
1383 	dunp = VTOUNIONFS(dvp);
1384 	cnp = ap->a_cnp;
1385 	lkflags = cnp->cn_lkflags;
1386 	udvp = dunp->un_uppervp;
1387 
1388 	if (udvp != NULLVP) {
1389 		/* check opaque */
1390 		if (!(cnp->cn_flags & ISWHITEOUT)) {
1391 			error = VOP_GETATTR(udvp, &va, cnp->cn_cred);
1392 			if (error != 0)
1393 				goto unionfs_mkdir_cleanup;
1394 			if ((va.va_flags & OPAQUE) != 0)
1395 				cnp->cn_flags |= ISWHITEOUT;
1396 		}
1397 
1398 		int udvp_lkflags;
1399 		bool uvp_created = false;
1400 		unionfs_forward_vop_start(udvp, &udvp_lkflags);
1401 		error = VOP_MKDIR(udvp, &uvp, cnp, ap->a_vap);
1402 		if (error == 0)
1403 			uvp_created = true;
1404 		if (__predict_false(unionfs_forward_vop_finish(dvp, udvp,
1405 		    udvp_lkflags)) && error == 0)
1406 			error = ENOENT;
1407 		if (error == 0) {
1408 			VOP_UNLOCK(uvp);
1409 			cnp->cn_lkflags = LK_EXCLUSIVE;
1410 			error = unionfs_nodeget(dvp->v_mount, uvp, NULLVP,
1411 			    dvp, ap->a_vpp, cnp);
1412 			vrele(uvp);
1413 			cnp->cn_lkflags = lkflags;
1414 		} else if (uvp_created)
1415 			vput(uvp);
1416 	}
1417 
1418 unionfs_mkdir_cleanup:
1419 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: leave (%d)\n", error);
1420 
1421 	return (error);
1422 }
1423 
1424 static int
1425 unionfs_rmdir(struct vop_rmdir_args *ap)
1426 {
1427 	struct unionfs_node *dunp;
1428 	struct unionfs_node *unp;
1429 	struct unionfs_mount *ump;
1430 	struct componentname *cnp;
1431 	struct thread  *td;
1432 	struct vnode   *udvp;
1433 	struct vnode   *uvp;
1434 	struct vnode   *lvp;
1435 	int		error;
1436 
1437 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: enter\n");
1438 
1439 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1440 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1441 
1442 	error = 0;
1443 	dunp = VTOUNIONFS(ap->a_dvp);
1444 	unp = VTOUNIONFS(ap->a_vp);
1445 	cnp = ap->a_cnp;
1446 	td = curthread;
1447 	udvp = dunp->un_uppervp;
1448 	uvp = unp->un_uppervp;
1449 	lvp = unp->un_lowervp;
1450 
1451 	if (udvp == NULLVP)
1452 		return (EROFS);
1453 
1454 	if (udvp == uvp)
1455 		return (EOPNOTSUPP);
1456 
1457 	if (uvp != NULLVP) {
1458 		if (lvp != NULLVP) {
1459 			error = unionfs_check_rmdir(ap->a_vp, cnp->cn_cred, td);
1460 			if (error != 0)
1461 				return (error);
1462 		}
1463 		ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1464 		if (ump->um_whitemode == UNIONFS_WHITE_ALWAYS || lvp != NULLVP)
1465 			cnp->cn_flags |= DOWHITEOUT;
1466 		/*
1467 		 * The relookup path will need to relock the parent dvp and
1468 		 * possibly the vp as well.  Locking is expected to be done
1469 		 * in parent->child order; drop the lock on vp to avoid LOR
1470 		 * and potential recursion on vp's lock.
1471 		 * vp is expected to remain referenced during VOP_RMDIR(),
1472 		 * so vref/vrele should not be necessary here.
1473 		 */
1474 		VOP_UNLOCK(ap->a_vp);
1475 		VNPASS(vrefcnt(ap->a_vp) > 0, ap->a_vp);
1476 		error = unionfs_relookup_for_delete(ap->a_dvp, cnp, td);
1477 		vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1478 		/*
1479 		 * VOP_RMDIR is dispatched against udvp, so if uvp became
1480 		 * doomed while the lock was dropped above the target
1481 		 * filesystem may not be able to cope.
1482 		 */
1483 		if (error == 0 && VN_IS_DOOMED(uvp))
1484 			error = ENOENT;
1485 		if (error == 0) {
1486 			int udvp_lkflags, uvp_lkflags;
1487 			unionfs_forward_vop_start_pair(udvp, &udvp_lkflags,
1488 			    uvp, &uvp_lkflags);
1489 			error = VOP_RMDIR(udvp, uvp, cnp);
1490 			unionfs_forward_vop_finish_pair(ap->a_dvp, udvp, udvp_lkflags,
1491 			    ap->a_vp, uvp, uvp_lkflags);
1492 		}
1493 	} else if (lvp != NULLVP)
1494 		error = unionfs_mkwhiteout(ap->a_dvp, udvp, cnp, td,
1495 		    unp->un_path, unp->un_pathlen);
1496 
1497 	if (error == 0) {
1498 		cache_purge(ap->a_dvp);
1499 		cache_purge(ap->a_vp);
1500 	}
1501 
1502 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: leave (%d)\n", error);
1503 
1504 	return (error);
1505 }
1506 
1507 static int
1508 unionfs_symlink(struct vop_symlink_args *ap)
1509 {
1510 	struct unionfs_node *dunp;
1511 	struct componentname *cnp;
1512 	struct vnode   *udvp;
1513 	struct vnode   *uvp;
1514 	int		error;
1515 	int		lkflags;
1516 
1517 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: enter\n");
1518 
1519 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1520 
1521 	error = EROFS;
1522 	dunp = VTOUNIONFS(ap->a_dvp);
1523 	cnp = ap->a_cnp;
1524 	lkflags = cnp->cn_lkflags;
1525 	udvp = dunp->un_uppervp;
1526 
1527 	if (udvp != NULLVP) {
1528 		int udvp_lkflags;
1529 		bool uvp_created = false;
1530 		unionfs_forward_vop_start(udvp, &udvp_lkflags);
1531 		error = VOP_SYMLINK(udvp, &uvp, cnp, ap->a_vap, ap->a_target);
1532 		if (error == 0)
1533 			uvp_created = true;
1534 		if (__predict_false(unionfs_forward_vop_finish(ap->a_dvp, udvp,
1535 		    udvp_lkflags)) && error == 0)
1536 			error = ENOENT;
1537 		if (error == 0) {
1538 			VOP_UNLOCK(uvp);
1539 			cnp->cn_lkflags = LK_EXCLUSIVE;
1540 			error = unionfs_nodeget(ap->a_dvp->v_mount, uvp, NULLVP,
1541 			    ap->a_dvp, ap->a_vpp, cnp);
1542 			vrele(uvp);
1543 			cnp->cn_lkflags = lkflags;
1544 		} else if (uvp_created)
1545 			vput(uvp);
1546 	}
1547 
1548 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: leave (%d)\n", error);
1549 
1550 	return (error);
1551 }
1552 
1553 static int
1554 unionfs_readdir(struct vop_readdir_args *ap)
1555 {
1556 	struct unionfs_node *unp;
1557 	struct unionfs_node_status *unsp;
1558 	struct uio     *uio;
1559 	struct vnode   *vp;
1560 	struct vnode   *uvp;
1561 	struct vnode   *lvp;
1562 	struct thread  *td;
1563 	struct vattr    va;
1564 
1565 	uint64_t	*cookies_bk;
1566 	int		error;
1567 	int		eofflag;
1568 	int		ncookies_bk;
1569 	int		uio_offset_bk;
1570 	enum unionfs_lkupgrade lkstatus;
1571 
1572 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: enter\n");
1573 
1574 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1575 
1576 	error = 0;
1577 	eofflag = 0;
1578 	uio_offset_bk = 0;
1579 	uio = ap->a_uio;
1580 	uvp = NULLVP;
1581 	lvp = NULLVP;
1582 	td = uio->uio_td;
1583 	ncookies_bk = 0;
1584 	cookies_bk = NULL;
1585 
1586 	vp = ap->a_vp;
1587 	if (vp->v_type != VDIR)
1588 		return (ENOTDIR);
1589 
1590 	/*
1591 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
1592 	 * or do anything else unionfs- specific.
1593 	 */
1594 	lkstatus = unionfs_upgrade_lock(vp);
1595 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
1596 		error = EBADF;
1597 	if (error == 0) {
1598 		unp = VTOUNIONFS(vp);
1599 		uvp = unp->un_uppervp;
1600 		lvp = unp->un_lowervp;
1601 		/* check the open count. unionfs needs open before readdir. */
1602 		unionfs_get_node_status(unp, td, &unsp);
1603 		if ((uvp != NULLVP && unsp->uns_upper_opencnt <= 0) ||
1604 			(lvp != NULLVP && unsp->uns_lower_opencnt <= 0)) {
1605 			unionfs_tryrem_node_status(unp, unsp);
1606 			error = EBADF;
1607 		}
1608 	}
1609 	unionfs_downgrade_lock(vp, lkstatus);
1610 	if (error != 0)
1611 		goto unionfs_readdir_exit;
1612 
1613 	/* check opaque */
1614 	if (uvp != NULLVP && lvp != NULLVP) {
1615 		if ((error = VOP_GETATTR(uvp, &va, ap->a_cred)) != 0)
1616 			goto unionfs_readdir_exit;
1617 		if (va.va_flags & OPAQUE)
1618 			lvp = NULLVP;
1619 	}
1620 
1621 	/* upper only */
1622 	if (uvp != NULLVP && lvp == NULLVP) {
1623 		error = VOP_READDIR(uvp, uio, ap->a_cred, ap->a_eofflag,
1624 		    ap->a_ncookies, ap->a_cookies);
1625 		unsp->uns_readdir_status = 0;
1626 
1627 		goto unionfs_readdir_exit;
1628 	}
1629 
1630 	/* lower only */
1631 	if (uvp == NULLVP && lvp != NULLVP) {
1632 		error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1633 		    ap->a_ncookies, ap->a_cookies);
1634 		unsp->uns_readdir_status = 2;
1635 
1636 		goto unionfs_readdir_exit;
1637 	}
1638 
1639 	/*
1640 	 * readdir upper and lower
1641 	 */
1642 	KASSERT(uvp != NULLVP, ("unionfs_readdir: null upper vp"));
1643 	KASSERT(lvp != NULLVP, ("unionfs_readdir: null lower vp"));
1644 	if (uio->uio_offset == 0)
1645 		unsp->uns_readdir_status = 0;
1646 
1647 	if (unsp->uns_readdir_status == 0) {
1648 		/* read upper */
1649 		error = VOP_READDIR(uvp, uio, ap->a_cred, &eofflag,
1650 				    ap->a_ncookies, ap->a_cookies);
1651 
1652 		if (error != 0 || eofflag == 0)
1653 			goto unionfs_readdir_exit;
1654 		unsp->uns_readdir_status = 1;
1655 
1656 		/*
1657 		 * UFS(and other FS) needs size of uio_resid larger than
1658 		 * DIRBLKSIZ.
1659 		 * size of DIRBLKSIZ equals DEV_BSIZE.
1660 		 * (see: ufs/ufs/ufs_vnops.c ufs_readdir func , ufs/ufs/dir.h)
1661 		 */
1662 		if (uio->uio_resid <= (uio->uio_resid & (DEV_BSIZE -1)))
1663 			goto unionfs_readdir_exit;
1664 
1665 		/*
1666 		 * Backup cookies.
1667 		 * It prepares to readdir in lower.
1668 		 */
1669 		if (ap->a_ncookies != NULL) {
1670 			ncookies_bk = *(ap->a_ncookies);
1671 			*(ap->a_ncookies) = 0;
1672 		}
1673 		if (ap->a_cookies != NULL) {
1674 			cookies_bk = *(ap->a_cookies);
1675 			*(ap->a_cookies) = NULL;
1676 		}
1677 	}
1678 
1679 	/* initialize for readdir in lower */
1680 	if (unsp->uns_readdir_status == 1) {
1681 		unsp->uns_readdir_status = 2;
1682 		/*
1683 		 * Backup uio_offset. See the comment after the
1684 		 * VOP_READDIR call on the lower layer.
1685 		 */
1686 		uio_offset_bk = uio->uio_offset;
1687 		uio->uio_offset = 0;
1688 	}
1689 
1690 	if (lvp == NULLVP) {
1691 		error = EBADF;
1692 		goto unionfs_readdir_exit;
1693 	}
1694 	/* read lower */
1695 	error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1696 			    ap->a_ncookies, ap->a_cookies);
1697 
1698 	/*
1699 	 * We can't return an uio_offset of 0: this would trigger an
1700 	 * infinite loop, because the next call to unionfs_readdir would
1701 	 * always restart with the upper layer (uio_offset == 0) and
1702 	 * always return some data.
1703 	 *
1704 	 * This happens when the lower layer root directory is removed.
1705 	 * (A root directory deleting of unionfs should not be permitted.
1706 	 *  But current VFS can not do it.)
1707 	 */
1708 	if (uio->uio_offset == 0)
1709 		uio->uio_offset = uio_offset_bk;
1710 
1711 	if (cookies_bk != NULL) {
1712 		/* merge cookies */
1713 		int		size;
1714 		uint64_t         *newcookies, *pos;
1715 
1716 		size = *(ap->a_ncookies) + ncookies_bk;
1717 		newcookies = (uint64_t *) malloc(size * sizeof(*newcookies),
1718 		    M_TEMP, M_WAITOK);
1719 		pos = newcookies;
1720 
1721 		memcpy(pos, cookies_bk, ncookies_bk * sizeof(*newcookies));
1722 		pos += ncookies_bk;
1723 		memcpy(pos, *(ap->a_cookies),
1724 		    *(ap->a_ncookies) * sizeof(*newcookies));
1725 		free(cookies_bk, M_TEMP);
1726 		free(*(ap->a_cookies), M_TEMP);
1727 		*(ap->a_ncookies) = size;
1728 		*(ap->a_cookies) = newcookies;
1729 	}
1730 
1731 unionfs_readdir_exit:
1732 	if (error != 0 && ap->a_eofflag != NULL)
1733 		*(ap->a_eofflag) = 1;
1734 
1735 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: leave (%d)\n", error);
1736 
1737 	return (error);
1738 }
1739 
1740 static int
1741 unionfs_readlink(struct vop_readlink_args *ap)
1742 {
1743 	struct unionfs_node *unp;
1744 	struct vnode   *vp;
1745 	int error;
1746 
1747 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: enter\n");
1748 
1749 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1750 
1751 	unp = VTOUNIONFS(ap->a_vp);
1752 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
1753 
1754 	error = VOP_READLINK(vp, ap->a_uio, ap->a_cred);
1755 
1756 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: leave (%d)\n", error);
1757 
1758 	return (error);
1759 }
1760 
1761 static int
1762 unionfs_getwritemount(struct vop_getwritemount_args *ap)
1763 {
1764 	struct unionfs_node *unp;
1765 	struct vnode   *uvp;
1766 	struct vnode   *vp, *ovp;
1767 	int		error;
1768 
1769 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: enter\n");
1770 
1771 	error = 0;
1772 	vp = ap->a_vp;
1773 	uvp = NULLVP;
1774 
1775 	VI_LOCK(vp);
1776 	unp = VTOUNIONFS(vp);
1777 	if (unp != NULL)
1778 		uvp = unp->un_uppervp;
1779 
1780 	/*
1781 	 * If our node has no upper vnode, check the parent directory.
1782 	 * We may be initiating a write operation that will produce a
1783 	 * new upper vnode through CoW.
1784 	 */
1785 	if (uvp == NULLVP && unp != NULL) {
1786 		ovp = vp;
1787 		vp = unp->un_dvp;
1788 		/*
1789 		 * Only the root vnode should have an empty parent, but it
1790 		 * should not have an empty uppervp, so we shouldn't get here.
1791 		 */
1792 		VNASSERT(vp != NULL, ovp, ("%s: NULL parent vnode", __func__));
1793 		VI_UNLOCK(ovp);
1794 		VI_LOCK(vp);
1795 		unp = VTOUNIONFS(vp);
1796 		if (unp != NULL)
1797 			uvp = unp->un_uppervp;
1798 		if (uvp == NULLVP)
1799 			error = EACCES;
1800 	}
1801 
1802 	if (uvp != NULLVP) {
1803 		vholdnz(uvp);
1804 		VI_UNLOCK(vp);
1805 		error = VOP_GETWRITEMOUNT(uvp, ap->a_mpp);
1806 		vdrop(uvp);
1807 	} else {
1808 		VI_UNLOCK(vp);
1809 		*(ap->a_mpp) = NULL;
1810 	}
1811 
1812 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: leave (%d)\n", error);
1813 
1814 	return (error);
1815 }
1816 
1817 static int
1818 unionfs_inactive(struct vop_inactive_args *ap)
1819 {
1820 	ap->a_vp->v_object = NULL;
1821 	vrecycle(ap->a_vp);
1822 	return (0);
1823 }
1824 
1825 static int
1826 unionfs_reclaim(struct vop_reclaim_args *ap)
1827 {
1828 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: enter\n"); */
1829 
1830 	unionfs_noderem(ap->a_vp);
1831 
1832 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: leave\n"); */
1833 
1834 	return (0);
1835 }
1836 
1837 static int
1838 unionfs_print(struct vop_print_args *ap)
1839 {
1840 	struct unionfs_node *unp;
1841 	/* struct unionfs_node_status *unsp; */
1842 
1843 	unp = VTOUNIONFS(ap->a_vp);
1844 	/* unionfs_get_node_status(unp, curthread, &unsp); */
1845 
1846 	printf("unionfs_vp=%p, uppervp=%p, lowervp=%p\n",
1847 	    ap->a_vp, unp->un_uppervp, unp->un_lowervp);
1848 	/*
1849 	printf("unionfs opencnt: uppervp=%d, lowervp=%d\n",
1850 	    unsp->uns_upper_opencnt, unsp->uns_lower_opencnt);
1851 	*/
1852 
1853 	if (unp->un_uppervp != NULLVP)
1854 		vn_printf(unp->un_uppervp, "unionfs: upper ");
1855 	if (unp->un_lowervp != NULLVP)
1856 		vn_printf(unp->un_lowervp, "unionfs: lower ");
1857 
1858 	return (0);
1859 }
1860 
1861 static int
1862 unionfs_get_llt_revlock(struct vnode *vp, int flags)
1863 {
1864 	int revlock;
1865 
1866 	revlock = 0;
1867 
1868 	switch (flags & LK_TYPE_MASK) {
1869 	case LK_SHARED:
1870 		if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
1871 			revlock = LK_UPGRADE;
1872 		else
1873 			revlock = LK_RELEASE;
1874 		break;
1875 	case LK_EXCLUSIVE:
1876 	case LK_UPGRADE:
1877 		revlock = LK_RELEASE;
1878 		break;
1879 	case LK_DOWNGRADE:
1880 		revlock = LK_UPGRADE;
1881 		break;
1882 	default:
1883 		break;
1884 	}
1885 
1886 	return (revlock);
1887 }
1888 
1889 /*
1890  * The state of an acquired lock is adjusted similarly to
1891  * the time of error generating.
1892  * flags: LK_RELEASE or LK_UPGRADE
1893  */
1894 static void
1895 unionfs_revlock(struct vnode *vp, int flags)
1896 {
1897 	if (flags & LK_RELEASE)
1898 		VOP_UNLOCK_FLAGS(vp, flags);
1899 	else {
1900 		/* UPGRADE */
1901 		if (vn_lock(vp, flags) != 0)
1902 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1903 	}
1904 }
1905 
1906 static int
1907 unionfs_lock(struct vop_lock1_args *ap)
1908 {
1909 	struct unionfs_node *unp;
1910 	struct vnode   *vp;
1911 	struct vnode   *uvp;
1912 	struct vnode   *lvp;
1913 	int		error;
1914 	int		flags;
1915 	int		revlock;
1916 	int		interlock;
1917 	int		uhold;
1918 
1919 	/*
1920 	 * TODO: rework the unionfs locking scheme.
1921 	 * It's not guaranteed to be safe to blindly lock two vnodes on
1922 	 * different mounts as is done here.  Further, the entanglement
1923 	 * of locking both vnodes with the various options that can be
1924 	 * passed to VOP_LOCK() makes this code hard to reason about.
1925 	 * Instead, consider locking only the upper vnode, or the lower
1926 	 * vnode is the upper is not present, and taking separate measures
1927 	 * to lock both vnodes in the few cases when that is needed.
1928 	 */
1929 	error = 0;
1930 	interlock = 1;
1931 	uhold = 0;
1932 	flags = ap->a_flags;
1933 	vp = ap->a_vp;
1934 
1935 	if (LK_RELEASE == (flags & LK_TYPE_MASK) || !(flags & LK_TYPE_MASK))
1936 		return (VOP_UNLOCK_FLAGS(vp, flags | LK_RELEASE));
1937 
1938 	if ((flags & LK_INTERLOCK) == 0)
1939 		VI_LOCK(vp);
1940 
1941 	unp = VTOUNIONFS(vp);
1942 	if (unp == NULL)
1943 		goto unionfs_lock_null_vnode;
1944 
1945 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1946 
1947 	lvp = unp->un_lowervp;
1948 	uvp = unp->un_uppervp;
1949 
1950 	if ((revlock = unionfs_get_llt_revlock(vp, flags)) == 0)
1951 		panic("unknown lock type: 0x%x", flags & LK_TYPE_MASK);
1952 
1953 	/*
1954 	 * During unmount, the root vnode lock may be taken recursively,
1955 	 * because it may share the same v_vnlock field as the vnode covered by
1956 	 * the unionfs mount.  The covered vnode is locked across VFS_UNMOUNT(),
1957 	 * and the same lock may be taken recursively here during vflush()
1958 	 * issued by unionfs_unmount().
1959 	 */
1960 	if ((flags & LK_TYPE_MASK) == LK_EXCLUSIVE &&
1961 	    (vp->v_vflag & VV_ROOT) != 0)
1962 		flags |= LK_CANRECURSE;
1963 
1964 	if (lvp != NULLVP) {
1965 		if (uvp != NULLVP && flags & LK_UPGRADE) {
1966 			/*
1967 			 * Share Lock is once released and a deadlock is
1968 			 * avoided.
1969 			 */
1970 			vholdnz(uvp);
1971 			uhold = 1;
1972 			VOP_UNLOCK(uvp);
1973 		}
1974 		VI_LOCK_FLAGS(lvp, MTX_DUPOK);
1975 		flags |= LK_INTERLOCK;
1976 		vholdl(lvp);
1977 
1978 		VI_UNLOCK(vp);
1979 		ap->a_flags &= ~LK_INTERLOCK;
1980 
1981 		error = VOP_LOCK(lvp, flags);
1982 
1983 		VI_LOCK(vp);
1984 		unp = VTOUNIONFS(vp);
1985 		if (unp == NULL) {
1986 			/* vnode is released. */
1987 			VI_UNLOCK(vp);
1988 			if (error == 0)
1989 				VOP_UNLOCK(lvp);
1990 			vdrop(lvp);
1991 			if (uhold != 0)
1992 				vdrop(uvp);
1993 			goto unionfs_lock_fallback;
1994 		}
1995 	}
1996 
1997 	if (error == 0 && uvp != NULLVP) {
1998 		if (uhold && flags & LK_UPGRADE) {
1999 			flags &= ~LK_TYPE_MASK;
2000 			flags |= LK_EXCLUSIVE;
2001 		}
2002 		VI_LOCK_FLAGS(uvp, MTX_DUPOK);
2003 		flags |= LK_INTERLOCK;
2004 		if (uhold == 0) {
2005 			vholdl(uvp);
2006 			uhold = 1;
2007 		}
2008 
2009 		VI_UNLOCK(vp);
2010 		ap->a_flags &= ~LK_INTERLOCK;
2011 
2012 		error = VOP_LOCK(uvp, flags);
2013 
2014 		VI_LOCK(vp);
2015 		unp = VTOUNIONFS(vp);
2016 		if (unp == NULL) {
2017 			/* vnode is released. */
2018 			VI_UNLOCK(vp);
2019 			if (error == 0)
2020 				VOP_UNLOCK(uvp);
2021 			vdrop(uvp);
2022 			if (lvp != NULLVP) {
2023 				VOP_UNLOCK(lvp);
2024 				vdrop(lvp);
2025 			}
2026 			goto unionfs_lock_fallback;
2027 		}
2028 		if (error != 0 && lvp != NULLVP) {
2029 			/* rollback */
2030 			VI_UNLOCK(vp);
2031 			unionfs_revlock(lvp, revlock);
2032 			interlock = 0;
2033 		}
2034 	}
2035 
2036 	if (interlock)
2037 		VI_UNLOCK(vp);
2038 	if (lvp != NULLVP)
2039 		vdrop(lvp);
2040 	if (uhold != 0)
2041 		vdrop(uvp);
2042 
2043 	return (error);
2044 
2045 unionfs_lock_null_vnode:
2046 	ap->a_flags |= LK_INTERLOCK;
2047 	return (vop_stdlock(ap));
2048 
2049 unionfs_lock_fallback:
2050 	/*
2051 	 * If we reach this point, we've discovered the unionfs vnode
2052 	 * has been reclaimed while the upper/lower vnode locks were
2053 	 * temporarily dropped.  Such temporary droppage may happen
2054 	 * during the course of an LK_UPGRADE operation itself, and in
2055 	 * that case LK_UPGRADE must be cleared as the unionfs vnode's
2056 	 * lock has been reset to point to the standard v_lock field,
2057 	 * which has not previously been held.
2058 	 */
2059 	if (flags & LK_UPGRADE) {
2060 		ap->a_flags &= ~LK_TYPE_MASK;
2061 		ap->a_flags |= LK_EXCLUSIVE;
2062 	}
2063 	return (vop_stdlock(ap));
2064 }
2065 
2066 static int
2067 unionfs_unlock(struct vop_unlock_args *ap)
2068 {
2069 	struct vnode   *vp;
2070 	struct vnode   *lvp;
2071 	struct vnode   *uvp;
2072 	struct unionfs_node *unp;
2073 	int		error;
2074 	int		uhold;
2075 
2076 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2077 
2078 	error = 0;
2079 	uhold = 0;
2080 	vp = ap->a_vp;
2081 
2082 	unp = VTOUNIONFS(vp);
2083 	if (unp == NULL)
2084 		goto unionfs_unlock_null_vnode;
2085 	lvp = unp->un_lowervp;
2086 	uvp = unp->un_uppervp;
2087 
2088 	if (lvp != NULLVP) {
2089 		vholdnz(lvp);
2090 		error = VOP_UNLOCK(lvp);
2091 	}
2092 
2093 	if (error == 0 && uvp != NULLVP) {
2094 		vholdnz(uvp);
2095 		uhold = 1;
2096 		error = VOP_UNLOCK(uvp);
2097 	}
2098 
2099 	if (lvp != NULLVP)
2100 		vdrop(lvp);
2101 	if (uhold != 0)
2102 		vdrop(uvp);
2103 
2104 	return error;
2105 
2106 unionfs_unlock_null_vnode:
2107 	return (vop_stdunlock(ap));
2108 }
2109 
2110 static int
2111 unionfs_pathconf(struct vop_pathconf_args *ap)
2112 {
2113 	struct unionfs_node *unp;
2114 	struct vnode   *vp;
2115 
2116 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2117 
2118 	unp = VTOUNIONFS(ap->a_vp);
2119 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2120 
2121 	return (VOP_PATHCONF(vp, ap->a_name, ap->a_retval));
2122 }
2123 
2124 static int
2125 unionfs_advlock(struct vop_advlock_args *ap)
2126 {
2127 	struct unionfs_node *unp;
2128 	struct unionfs_node_status *unsp;
2129 	struct vnode   *vp;
2130 	struct vnode   *uvp;
2131 	struct thread  *td;
2132 	int error;
2133 
2134 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: enter\n");
2135 
2136 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2137 
2138 	vp = ap->a_vp;
2139 	td = curthread;
2140 
2141 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2142 
2143 	unp = VTOUNIONFS(ap->a_vp);
2144 	uvp = unp->un_uppervp;
2145 
2146 	if (uvp == NULLVP) {
2147 		error = unionfs_copyfile(unp, 1, td->td_ucred, td);
2148 		if (error != 0)
2149 			goto unionfs_advlock_abort;
2150 		uvp = unp->un_uppervp;
2151 
2152 		unionfs_get_node_status(unp, td, &unsp);
2153 		if (unsp->uns_lower_opencnt > 0) {
2154 			/* try reopen the vnode */
2155 			error = VOP_OPEN(uvp, unsp->uns_lower_openmode,
2156 				td->td_ucred, td, NULL);
2157 			if (error)
2158 				goto unionfs_advlock_abort;
2159 			unsp->uns_upper_opencnt++;
2160 			VOP_CLOSE(unp->un_lowervp, unsp->uns_lower_openmode,
2161 			    td->td_ucred, td);
2162 			unsp->uns_lower_opencnt--;
2163 		} else
2164 			unionfs_tryrem_node_status(unp, unsp);
2165 	}
2166 
2167 	VOP_UNLOCK(vp);
2168 
2169 	error = VOP_ADVLOCK(uvp, ap->a_id, ap->a_op, ap->a_fl, ap->a_flags);
2170 
2171 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2172 
2173 	return error;
2174 
2175 unionfs_advlock_abort:
2176 	VOP_UNLOCK(vp);
2177 
2178 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2179 
2180 	return error;
2181 }
2182 
2183 static int
2184 unionfs_strategy(struct vop_strategy_args *ap)
2185 {
2186 	struct unionfs_node *unp;
2187 	struct vnode   *vp;
2188 
2189 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2190 
2191 	unp = VTOUNIONFS(ap->a_vp);
2192 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2193 
2194 #ifdef DIAGNOSTIC
2195 	if (vp == NULLVP)
2196 		panic("unionfs_strategy: nullvp");
2197 
2198 	if (ap->a_bp->b_iocmd == BIO_WRITE && vp == unp->un_lowervp)
2199 		panic("unionfs_strategy: writing to lowervp");
2200 #endif
2201 
2202 	return (VOP_STRATEGY(vp, ap->a_bp));
2203 }
2204 
2205 static int
2206 unionfs_getacl(struct vop_getacl_args *ap)
2207 {
2208 	struct unionfs_node *unp;
2209 	struct vnode   *vp;
2210 	int		error;
2211 
2212 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2213 
2214 	unp = VTOUNIONFS(ap->a_vp);
2215 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2216 
2217 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: enter\n");
2218 
2219 	error = VOP_GETACL(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2220 
2221 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: leave (%d)\n", error);
2222 
2223 	return (error);
2224 }
2225 
2226 static int
2227 unionfs_setacl(struct vop_setacl_args *ap)
2228 {
2229 	struct unionfs_node *unp;
2230 	struct vnode   *uvp;
2231 	struct vnode   *lvp;
2232 	struct thread  *td;
2233 	int		error;
2234 
2235 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: enter\n");
2236 
2237 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2238 
2239 	error = EROFS;
2240 	unp = VTOUNIONFS(ap->a_vp);
2241 	uvp = unp->un_uppervp;
2242 	lvp = unp->un_lowervp;
2243 	td = ap->a_td;
2244 
2245 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2246 		return (EROFS);
2247 
2248 	if (uvp == NULLVP && lvp->v_type == VREG) {
2249 		if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0)
2250 			return (error);
2251 		uvp = unp->un_uppervp;
2252 	}
2253 
2254 	if (uvp != NULLVP) {
2255 		int lkflags;
2256 		unionfs_forward_vop_start(uvp, &lkflags);
2257 		error = VOP_SETACL(uvp, ap->a_type, ap->a_aclp, ap->a_cred, td);
2258 		unionfs_forward_vop_finish(ap->a_vp, uvp, lkflags);
2259 	}
2260 
2261 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: leave (%d)\n", error);
2262 
2263 	return (error);
2264 }
2265 
2266 static int
2267 unionfs_aclcheck(struct vop_aclcheck_args *ap)
2268 {
2269 	struct unionfs_node *unp;
2270 	struct vnode   *vp;
2271 	int		error;
2272 
2273 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: enter\n");
2274 
2275 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2276 
2277 	unp = VTOUNIONFS(ap->a_vp);
2278 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2279 
2280 	error = VOP_ACLCHECK(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2281 
2282 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: leave (%d)\n", error);
2283 
2284 	return (error);
2285 }
2286 
2287 static int
2288 unionfs_openextattr(struct vop_openextattr_args *ap)
2289 {
2290 	struct unionfs_node *unp;
2291 	struct vnode   *vp;
2292 	struct vnode   *tvp;
2293 	int		error;
2294 
2295 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2296 
2297 	vp = ap->a_vp;
2298 	unp = VTOUNIONFS(vp);
2299 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2300 
2301 	if ((tvp == unp->un_uppervp && (unp->un_flag & UNIONFS_OPENEXTU)) ||
2302 	    (tvp == unp->un_lowervp && (unp->un_flag & UNIONFS_OPENEXTL)))
2303 		return (EBUSY);
2304 
2305 	error = VOP_OPENEXTATTR(tvp, ap->a_cred, ap->a_td);
2306 
2307 	if (error == 0) {
2308 		if (vn_lock(vp, LK_UPGRADE) != 0)
2309 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2310 		if (!VN_IS_DOOMED(vp)) {
2311 			if (tvp == unp->un_uppervp)
2312 				unp->un_flag |= UNIONFS_OPENEXTU;
2313 			else
2314 				unp->un_flag |= UNIONFS_OPENEXTL;
2315 		}
2316 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2317 	}
2318 
2319 	return (error);
2320 }
2321 
2322 static int
2323 unionfs_closeextattr(struct vop_closeextattr_args *ap)
2324 {
2325 	struct unionfs_node *unp;
2326 	struct vnode   *vp;
2327 	struct vnode   *tvp;
2328 	int		error;
2329 
2330 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2331 
2332 	vp = ap->a_vp;
2333 	unp = VTOUNIONFS(vp);
2334 	tvp = NULLVP;
2335 
2336 	if (unp->un_flag & UNIONFS_OPENEXTU)
2337 		tvp = unp->un_uppervp;
2338 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2339 		tvp = unp->un_lowervp;
2340 
2341 	if (tvp == NULLVP)
2342 		return (EOPNOTSUPP);
2343 
2344 	error = VOP_CLOSEEXTATTR(tvp, ap->a_commit, ap->a_cred, ap->a_td);
2345 
2346 	if (error == 0) {
2347 		if (vn_lock(vp, LK_UPGRADE) != 0)
2348 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2349 		if (!VN_IS_DOOMED(vp)) {
2350 			if (tvp == unp->un_uppervp)
2351 				unp->un_flag &= ~UNIONFS_OPENEXTU;
2352 			else
2353 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2354 		}
2355 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2356 	}
2357 
2358 	return (error);
2359 }
2360 
2361 static int
2362 unionfs_getextattr(struct vop_getextattr_args *ap)
2363 {
2364 	struct unionfs_node *unp;
2365 	struct vnode   *vp;
2366 
2367 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2368 
2369 	unp = VTOUNIONFS(ap->a_vp);
2370 	vp = NULLVP;
2371 
2372 	if (unp->un_flag & UNIONFS_OPENEXTU)
2373 		vp = unp->un_uppervp;
2374 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2375 		vp = unp->un_lowervp;
2376 
2377 	if (vp == NULLVP)
2378 		return (EOPNOTSUPP);
2379 
2380 	return (VOP_GETEXTATTR(vp, ap->a_attrnamespace, ap->a_name,
2381 	    ap->a_uio, ap->a_size, ap->a_cred, ap->a_td));
2382 }
2383 
2384 static int
2385 unionfs_setextattr(struct vop_setextattr_args *ap)
2386 {
2387 	struct unionfs_node *unp;
2388 	struct vnode   *uvp;
2389 	struct vnode   *lvp;
2390 	struct vnode   *ovp;
2391 	struct ucred   *cred;
2392 	struct thread  *td;
2393 	int		error;
2394 
2395 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2396 
2397 	error = EROFS;
2398 	unp = VTOUNIONFS(ap->a_vp);
2399 	uvp = unp->un_uppervp;
2400 	lvp = unp->un_lowervp;
2401 	ovp = NULLVP;
2402 	cred = ap->a_cred;
2403 	td = ap->a_td;
2404 
2405 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: enter (un_flag=%x)\n",
2406 	    unp->un_flag);
2407 
2408 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2409 		return (EROFS);
2410 
2411 	if (unp->un_flag & UNIONFS_OPENEXTU)
2412 		ovp = unp->un_uppervp;
2413 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2414 		ovp = unp->un_lowervp;
2415 
2416 	if (ovp == NULLVP)
2417 		return (EOPNOTSUPP);
2418 
2419 	if (ovp == lvp && lvp->v_type == VREG) {
2420 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2421 		if (uvp == NULLVP &&
2422 		    (error = unionfs_copyfile(unp, 1, cred, td)) != 0) {
2423 unionfs_setextattr_reopen:
2424 			if ((unp->un_flag & UNIONFS_OPENEXTL) &&
2425 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2426 #ifdef DIAGNOSTIC
2427 				panic("unionfs: VOP_OPENEXTATTR failed");
2428 #endif
2429 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2430 			}
2431 			goto unionfs_setextattr_abort;
2432 		}
2433 		uvp = unp->un_uppervp;
2434 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2435 			goto unionfs_setextattr_reopen;
2436 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2437 		unp->un_flag |= UNIONFS_OPENEXTU;
2438 		ovp = uvp;
2439 	}
2440 
2441 	if (ovp == uvp) {
2442 		int lkflags;
2443 		unionfs_forward_vop_start(ovp, &lkflags);
2444 		error = VOP_SETEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2445 		    ap->a_uio, cred, td);
2446 		unionfs_forward_vop_finish(ap->a_vp, ovp, lkflags);
2447 	}
2448 
2449 unionfs_setextattr_abort:
2450 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: leave (%d)\n", error);
2451 
2452 	return (error);
2453 }
2454 
2455 static int
2456 unionfs_listextattr(struct vop_listextattr_args *ap)
2457 {
2458 	struct unionfs_node *unp;
2459 	struct vnode *vp;
2460 
2461 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2462 
2463 	unp = VTOUNIONFS(ap->a_vp);
2464 	vp = NULLVP;
2465 
2466 	if (unp->un_flag & UNIONFS_OPENEXTU)
2467 		vp = unp->un_uppervp;
2468 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2469 		vp = unp->un_lowervp;
2470 
2471 	if (vp == NULLVP)
2472 		return (EOPNOTSUPP);
2473 
2474 	return (VOP_LISTEXTATTR(vp, ap->a_attrnamespace, ap->a_uio,
2475 	    ap->a_size, ap->a_cred, ap->a_td));
2476 }
2477 
2478 static int
2479 unionfs_deleteextattr(struct vop_deleteextattr_args *ap)
2480 {
2481 	struct unionfs_node *unp;
2482 	struct vnode   *uvp;
2483 	struct vnode   *lvp;
2484 	struct vnode   *ovp;
2485 	struct ucred   *cred;
2486 	struct thread  *td;
2487 	int		error;
2488 
2489 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2490 
2491 	error = EROFS;
2492 	unp = VTOUNIONFS(ap->a_vp);
2493 	uvp = unp->un_uppervp;
2494 	lvp = unp->un_lowervp;
2495 	ovp = NULLVP;
2496 	cred = ap->a_cred;
2497 	td = ap->a_td;
2498 
2499 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: enter (un_flag=%x)\n",
2500 	    unp->un_flag);
2501 
2502 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2503 		return (EROFS);
2504 
2505 	if (unp->un_flag & UNIONFS_OPENEXTU)
2506 		ovp = unp->un_uppervp;
2507 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2508 		ovp = unp->un_lowervp;
2509 
2510 	if (ovp == NULLVP)
2511 		return (EOPNOTSUPP);
2512 
2513 	if (ovp == lvp && lvp->v_type == VREG) {
2514 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2515 		if (uvp == NULLVP &&
2516 		    (error = unionfs_copyfile(unp, 1, cred, td)) != 0) {
2517 unionfs_deleteextattr_reopen:
2518 			if ((unp->un_flag & UNIONFS_OPENEXTL) &&
2519 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2520 #ifdef DIAGNOSTIC
2521 				panic("unionfs: VOP_OPENEXTATTR failed");
2522 #endif
2523 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2524 			}
2525 			goto unionfs_deleteextattr_abort;
2526 		}
2527 		uvp = unp->un_uppervp;
2528 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2529 			goto unionfs_deleteextattr_reopen;
2530 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2531 		unp->un_flag |= UNIONFS_OPENEXTU;
2532 		ovp = uvp;
2533 	}
2534 
2535 	if (ovp == uvp)
2536 		error = VOP_DELETEEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2537 		    ap->a_cred, ap->a_td);
2538 
2539 unionfs_deleteextattr_abort:
2540 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: leave (%d)\n", error);
2541 
2542 	return (error);
2543 }
2544 
2545 static int
2546 unionfs_setlabel(struct vop_setlabel_args *ap)
2547 {
2548 	struct unionfs_node *unp;
2549 	struct vnode   *uvp;
2550 	struct vnode   *lvp;
2551 	struct thread  *td;
2552 	int		error;
2553 
2554 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: enter\n");
2555 
2556 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2557 
2558 	error = EROFS;
2559 	unp = VTOUNIONFS(ap->a_vp);
2560 	uvp = unp->un_uppervp;
2561 	lvp = unp->un_lowervp;
2562 	td = ap->a_td;
2563 
2564 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2565 		return (EROFS);
2566 
2567 	if (uvp == NULLVP && lvp->v_type == VREG) {
2568 		if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0)
2569 			return (error);
2570 		uvp = unp->un_uppervp;
2571 	}
2572 
2573 	if (uvp != NULLVP)
2574 		error = VOP_SETLABEL(uvp, ap->a_label, ap->a_cred, td);
2575 
2576 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: leave (%d)\n", error);
2577 
2578 	return (error);
2579 }
2580 
2581 static int
2582 unionfs_vptofh(struct vop_vptofh_args *ap)
2583 {
2584 	return (EOPNOTSUPP);
2585 }
2586 
2587 static int
2588 unionfs_add_writecount(struct vop_add_writecount_args *ap)
2589 {
2590 	struct vnode *tvp, *vp;
2591 	struct unionfs_node *unp;
2592 	int error, writerefs __diagused;
2593 
2594 	vp = ap->a_vp;
2595 	unp = VTOUNIONFS(vp);
2596 	tvp = unp->un_uppervp;
2597 	KASSERT(tvp != NULL,
2598 	    ("%s: adding write ref without upper vnode", __func__));
2599 	error = VOP_ADD_WRITECOUNT(tvp, ap->a_inc);
2600 	if (error != 0)
2601 		return (error);
2602 	/*
2603 	 * We need to track the write refs we've passed to the underlying
2604 	 * vnodes so that we can undo them in case we are forcibly unmounted.
2605 	 */
2606 	writerefs = atomic_fetchadd_int(&vp->v_writecount, ap->a_inc);
2607 	/* text refs are bypassed to lowervp */
2608 	VNASSERT(writerefs >= 0, vp,
2609 	    ("%s: invalid write count %d", __func__, writerefs));
2610 	VNASSERT(writerefs + ap->a_inc >= 0, vp,
2611 	    ("%s: invalid write count inc %d + %d", __func__,
2612 	    writerefs, ap->a_inc));
2613 	return (0);
2614 }
2615 
2616 static int
2617 unionfs_vput_pair(struct vop_vput_pair_args *ap)
2618 {
2619 	struct mount *mp;
2620 	struct vnode *dvp, *vp, **vpp, *lvp, *ldvp, *uvp, *udvp, *tempvp;
2621 	struct unionfs_node *dunp, *unp;
2622 	int error, res;
2623 
2624 	dvp = ap->a_dvp;
2625 	vpp = ap->a_vpp;
2626 	vp = NULLVP;
2627 	lvp = NULLVP;
2628 	uvp = NULLVP;
2629 	unp = NULL;
2630 
2631 	dunp = VTOUNIONFS(dvp);
2632 	udvp = dunp->un_uppervp;
2633 	ldvp = dunp->un_lowervp;
2634 
2635 	/*
2636 	 * Underlying vnodes should be locked because the encompassing unionfs
2637 	 * node is locked, but will not be referenced, as the reference will
2638 	 * only be on the unionfs node.  Reference them now so that the vput()s
2639 	 * performed by VOP_VPUT_PAIR() will have a reference to drop.
2640 	 */
2641 	if (udvp != NULLVP)
2642 		vref(udvp);
2643 	if (ldvp != NULLVP)
2644 		vref(ldvp);
2645 
2646 	if (vpp != NULL)
2647 		vp = *vpp;
2648 
2649 	if (vp != NULLVP) {
2650 		unp = VTOUNIONFS(vp);
2651 		uvp = unp->un_uppervp;
2652 		lvp = unp->un_lowervp;
2653 		if (uvp != NULLVP)
2654 			vref(uvp);
2655 		if (lvp != NULLVP)
2656 			vref(lvp);
2657 
2658 		/*
2659 		 * If we're being asked to return a locked child vnode, then
2660 		 * we may need to create a replacement vnode in case the
2661 		 * original is reclaimed while the lock is dropped.  In that
2662 		 * case we'll need to ensure the mount and the underlying
2663 		 * vnodes aren't also recycled during that window.
2664 		 */
2665 		if (!ap->a_unlock_vp) {
2666 			vhold(vp);
2667 			if (uvp != NULLVP)
2668 				vhold(uvp);
2669 			if (lvp != NULLVP)
2670 				vhold(lvp);
2671 			mp = vp->v_mount;
2672 			vfs_ref(mp);
2673 		}
2674 	}
2675 
2676 	/*
2677 	 * TODO: Because unionfs_lock() locks both the lower and upper vnodes
2678 	 * (if available), we must also call VOP_VPUT_PAIR() on both the lower
2679 	 * and upper parent/child pairs.  If unionfs_lock() is reworked to lock
2680 	 * only a single vnode, this code will need to change to also only
2681 	 * operate on one vnode pair.
2682 	 */
2683 	ASSERT_VOP_LOCKED(ldvp, __func__);
2684 	ASSERT_VOP_LOCKED(udvp, __func__);
2685 	ASSERT_VOP_LOCKED(lvp, __func__);
2686 	ASSERT_VOP_LOCKED(uvp, __func__);
2687 
2688 	KASSERT(lvp == NULLVP || ldvp != NULLVP,
2689 	    ("%s: NULL ldvp with non-NULL lvp", __func__));
2690 	if (ldvp != NULLVP)
2691 		res = VOP_VPUT_PAIR(ldvp, lvp != NULLVP ? &lvp : NULL, true);
2692 	KASSERT(uvp == NULLVP || udvp != NULLVP,
2693 	    ("%s: NULL udvp with non-NULL uvp", __func__));
2694 	if (udvp != NULLVP)
2695 		res = VOP_VPUT_PAIR(udvp, uvp != NULLVP ? &uvp : NULL, true);
2696 
2697 	ASSERT_VOP_UNLOCKED(ldvp, __func__);
2698 	ASSERT_VOP_UNLOCKED(udvp, __func__);
2699 	ASSERT_VOP_UNLOCKED(lvp, __func__);
2700 	ASSERT_VOP_UNLOCKED(uvp, __func__);
2701 
2702 	/*
2703 	 * VOP_VPUT_PAIR() dropped the references we added to the underlying
2704 	 * vnodes, now drop the caller's reference to the unionfs vnodes.
2705 	 */
2706 	if (vp != NULLVP && ap->a_unlock_vp)
2707 		vrele(vp);
2708 	vrele(dvp);
2709 
2710 	if (vp == NULLVP || ap->a_unlock_vp)
2711 		return (res);
2712 
2713 	/*
2714 	 * We're being asked to return a locked vnode.  At this point, the
2715 	 * underlying vnodes have been unlocked, so vp may have been reclaimed.
2716 	 */
2717 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2718 	if (vp->v_data == NULL && vfs_busy(mp, MBF_NOWAIT) == 0) {
2719 		vput(vp);
2720 		error = unionfs_nodeget(mp, uvp, lvp, dvp, &tempvp, NULL);
2721 		if (error == 0) {
2722 			vn_lock(tempvp, LK_EXCLUSIVE | LK_RETRY);
2723 			*vpp = tempvp;
2724 		} else
2725 			vget(vp, LK_EXCLUSIVE | LK_RETRY);
2726 		vfs_unbusy(mp);
2727 	}
2728 	if (lvp != NULLVP)
2729 		vdrop(lvp);
2730 	if (uvp != NULLVP)
2731 		vdrop(uvp);
2732 	vdrop(vp);
2733 	vfs_rel(mp);
2734 
2735 	return (res);
2736 }
2737 
2738 static int
2739 unionfs_set_text(struct vop_set_text_args *ap)
2740 {
2741 	struct vnode *tvp;
2742 	struct unionfs_node *unp;
2743 	int error;
2744 
2745 	/*
2746 	 * We assume text refs are managed against lvp/uvp through the
2747 	 * executable mapping backed by its VM object.  We therefore don't
2748 	 * need to track leased text refs in the case of a forcible unmount.
2749 	 */
2750 	unp = VTOUNIONFS(ap->a_vp);
2751 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2752 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2753 	error = VOP_SET_TEXT(tvp);
2754 	return (error);
2755 }
2756 
2757 static int
2758 unionfs_unset_text(struct vop_unset_text_args *ap)
2759 {
2760 	struct vnode *tvp;
2761 	struct unionfs_node *unp;
2762 
2763 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2764 	unp = VTOUNIONFS(ap->a_vp);
2765 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2766 	VOP_UNSET_TEXT_CHECKED(tvp);
2767 	return (0);
2768 }
2769 
2770 static int
2771 unionfs_unp_bind(struct vop_unp_bind_args *ap)
2772 {
2773 	struct vnode *tvp;
2774 	struct unionfs_node *unp;
2775 
2776 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2777 	unp = VTOUNIONFS(ap->a_vp);
2778 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2779 	VOP_UNP_BIND(tvp, ap->a_unpcb);
2780 	return (0);
2781 }
2782 
2783 static int
2784 unionfs_unp_connect(struct vop_unp_connect_args *ap)
2785 {
2786 	struct vnode *tvp;
2787 	struct unionfs_node *unp;
2788 
2789 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2790 	unp = VTOUNIONFS(ap->a_vp);
2791 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2792 	VOP_UNP_CONNECT(tvp, ap->a_unpcb);
2793 	return (0);
2794 }
2795 
2796 static int
2797 unionfs_unp_detach(struct vop_unp_detach_args *ap)
2798 {
2799 	struct vnode *tvp;
2800 	struct unionfs_node *unp;
2801 
2802 	tvp = NULL;
2803 	/*
2804 	 * VOP_UNP_DETACH() is not guaranteed to be called with the unionfs
2805 	 * vnode locked, so we take the interlock to prevent a concurrent
2806 	 * unmount from freeing the unionfs private data.
2807 	 */
2808 	VI_LOCK(ap->a_vp);
2809 	unp = VTOUNIONFS(ap->a_vp);
2810 	if (unp != NULL) {
2811 		tvp = unp->un_uppervp != NULL ?
2812 		    unp->un_uppervp : unp->un_lowervp;
2813 		/*
2814 		 * Hold the target vnode to prevent a concurrent unionfs
2815 		 * unmount from causing it to be recycled once the interlock
2816 		 * is dropped.
2817 		 */
2818 		vholdnz(tvp);
2819 	}
2820 	VI_UNLOCK(ap->a_vp);
2821 	if (tvp != NULL) {
2822 		VOP_UNP_DETACH(tvp);
2823 		vdrop(tvp);
2824 	}
2825 	return (0);
2826 }
2827 
2828 struct vop_vector unionfs_vnodeops = {
2829 	.vop_default =		&default_vnodeops,
2830 
2831 	.vop_access =		unionfs_access,
2832 	.vop_aclcheck =		unionfs_aclcheck,
2833 	.vop_advlock =		unionfs_advlock,
2834 	.vop_bmap =		VOP_EOPNOTSUPP,
2835 	.vop_cachedlookup =	unionfs_lookup,
2836 	.vop_close =		unionfs_close,
2837 	.vop_closeextattr =	unionfs_closeextattr,
2838 	.vop_create =		unionfs_create,
2839 	.vop_deleteextattr =	unionfs_deleteextattr,
2840 	.vop_fsync =		unionfs_fsync,
2841 	.vop_getacl =		unionfs_getacl,
2842 	.vop_getattr =		unionfs_getattr,
2843 	.vop_getextattr =	unionfs_getextattr,
2844 	.vop_getwritemount =	unionfs_getwritemount,
2845 	.vop_inactive =		unionfs_inactive,
2846 	.vop_need_inactive =	vop_stdneed_inactive,
2847 	.vop_islocked =		vop_stdislocked,
2848 	.vop_ioctl =		unionfs_ioctl,
2849 	.vop_link =		unionfs_link,
2850 	.vop_listextattr =	unionfs_listextattr,
2851 	.vop_lock1 =		unionfs_lock,
2852 	.vop_lookup =		vfs_cache_lookup,
2853 	.vop_mkdir =		unionfs_mkdir,
2854 	.vop_mknod =		unionfs_mknod,
2855 	.vop_open =		unionfs_open,
2856 	.vop_openextattr =	unionfs_openextattr,
2857 	.vop_pathconf =		unionfs_pathconf,
2858 	.vop_poll =		unionfs_poll,
2859 	.vop_print =		unionfs_print,
2860 	.vop_read =		unionfs_read,
2861 	.vop_readdir =		unionfs_readdir,
2862 	.vop_readlink =		unionfs_readlink,
2863 	.vop_reclaim =		unionfs_reclaim,
2864 	.vop_remove =		unionfs_remove,
2865 	.vop_rename =		unionfs_rename,
2866 	.vop_rmdir =		unionfs_rmdir,
2867 	.vop_setacl =		unionfs_setacl,
2868 	.vop_setattr =		unionfs_setattr,
2869 	.vop_setextattr =	unionfs_setextattr,
2870 	.vop_setlabel =		unionfs_setlabel,
2871 	.vop_strategy =		unionfs_strategy,
2872 	.vop_symlink =		unionfs_symlink,
2873 	.vop_unlock =		unionfs_unlock,
2874 	.vop_whiteout =		unionfs_whiteout,
2875 	.vop_write =		unionfs_write,
2876 	.vop_vptofh =		unionfs_vptofh,
2877 	.vop_add_writecount =	unionfs_add_writecount,
2878 	.vop_vput_pair =	unionfs_vput_pair,
2879 	.vop_set_text =		unionfs_set_text,
2880 	.vop_unset_text = 	unionfs_unset_text,
2881 	.vop_unp_bind =		unionfs_unp_bind,
2882 	.vop_unp_connect =	unionfs_unp_connect,
2883 	.vop_unp_detach =	unionfs_unp_detach,
2884 };
2885 VFS_VOP_VECTOR_REGISTER(unionfs_vnodeops);
2886