xref: /freebsd/sys/fs/unionfs/union_vnops.c (revision d9a42747950146bf03cda7f6e25d219253f8a57a)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1992, 1993, 1994, 1995 Jan-Simon Pendry.
5  * Copyright (c) 1992, 1993, 1994, 1995
6  *      The Regents of the University of California.
7  * Copyright (c) 2005, 2006, 2012 Masanori Ozawa <ozawa@ongs.co.jp>, ONGS Inc.
8  * Copyright (c) 2006, 2012 Daichi Goto <daichi@freebsd.org>
9  * All rights reserved.
10  *
11  * This code is derived from software contributed to Berkeley by
12  * Jan-Simon Pendry.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions
16  * are met:
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  * 3. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  *
38  *	@(#)union_vnops.c	8.32 (Berkeley) 6/23/95
39  * $FreeBSD$
40  *
41  */
42 
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/conf.h>
46 #include <sys/kernel.h>
47 #include <sys/lock.h>
48 #include <sys/malloc.h>
49 #include <sys/mount.h>
50 #include <sys/mutex.h>
51 #include <sys/namei.h>
52 #include <sys/sysctl.h>
53 #include <sys/vnode.h>
54 #include <sys/kdb.h>
55 #include <sys/fcntl.h>
56 #include <sys/stat.h>
57 #include <sys/dirent.h>
58 #include <sys/proc.h>
59 #include <sys/bio.h>
60 #include <sys/buf.h>
61 
62 #include <fs/unionfs/union.h>
63 
64 #include <machine/atomic.h>
65 
66 #include <vm/vm.h>
67 #include <vm/vm_extern.h>
68 #include <vm/vm_object.h>
69 #include <vm/vnode_pager.h>
70 
71 #if 0
72 #define UNIONFS_INTERNAL_DEBUG(msg, args...)    printf(msg, ## args)
73 #define UNIONFS_IDBG_RENAME
74 #else
75 #define UNIONFS_INTERNAL_DEBUG(msg, args...)
76 #endif
77 
78 #define KASSERT_UNIONFS_VNODE(vp) \
79 	VNASSERT(((vp)->v_op == &unionfs_vnodeops), vp, \
80 	    ("%s: non-unionfs vnode", __func__))
81 
82 static int
83 unionfs_lookup(struct vop_cachedlookup_args *ap)
84 {
85 	struct unionfs_node *dunp;
86 	struct vnode   *dvp, *udvp, *ldvp, *vp, *uvp, *lvp, *dtmpvp;
87 	struct vattr	va;
88 	struct componentname *cnp;
89 	struct thread  *td;
90 	u_long		nameiop;
91 	u_long		cnflags, cnflagsbk;
92 	int		iswhiteout;
93 	int		lockflag;
94 	int		error , uerror, lerror;
95 
96 	iswhiteout = 0;
97 	lockflag = 0;
98 	error = uerror = lerror = ENOENT;
99 	cnp = ap->a_cnp;
100 	nameiop = cnp->cn_nameiop;
101 	cnflags = cnp->cn_flags;
102 	dvp = ap->a_dvp;
103 	dunp = VTOUNIONFS(dvp);
104 	udvp = dunp->un_uppervp;
105 	ldvp = dunp->un_lowervp;
106 	vp = uvp = lvp = NULLVP;
107 	td = curthread;
108 	*(ap->a_vpp) = NULLVP;
109 
110 	UNIONFS_INTERNAL_DEBUG(
111 	    "unionfs_lookup: enter: nameiop=%ld, flags=%lx, path=%s\n",
112 	    nameiop, cnflags, cnp->cn_nameptr);
113 
114 	if (dvp->v_type != VDIR)
115 		return (ENOTDIR);
116 
117 	/*
118 	 * If read-only and op is not LOOKUP, will return EROFS.
119 	 */
120 	if ((cnflags & ISLASTCN) &&
121 	    (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
122 	    LOOKUP != nameiop)
123 		return (EROFS);
124 
125 	/*
126 	 * lookup dotdot
127 	 */
128 	if (cnflags & ISDOTDOT) {
129 		if (LOOKUP != nameiop && udvp == NULLVP)
130 			return (EROFS);
131 
132 		if (udvp != NULLVP) {
133 			dtmpvp = udvp;
134 			if (ldvp != NULLVP)
135 				VOP_UNLOCK(ldvp);
136 		}
137 		else
138 			dtmpvp = ldvp;
139 
140 		error = VOP_LOOKUP(dtmpvp, &vp, cnp);
141 
142 		if (dtmpvp == udvp && ldvp != NULLVP) {
143 			VOP_UNLOCK(udvp);
144 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
145 		}
146 
147 		if (error == 0) {
148 			/*
149 			 * Exchange lock and reference from vp to
150 			 * dunp->un_dvp. vp is upper/lower vnode, but it
151 			 * will need to return the unionfs vnode.
152 			 */
153 			if (nameiop == DELETE  || nameiop == RENAME ||
154 			    (cnp->cn_lkflags & LK_TYPE_MASK))
155 				VOP_UNLOCK(vp);
156 			vrele(vp);
157 
158 			VOP_UNLOCK(dvp);
159 			*(ap->a_vpp) = dunp->un_dvp;
160 			vref(dunp->un_dvp);
161 
162 			if (nameiop == DELETE || nameiop == RENAME)
163 				vn_lock(dunp->un_dvp, LK_EXCLUSIVE | LK_RETRY);
164 			else if (cnp->cn_lkflags & LK_TYPE_MASK)
165 				vn_lock(dunp->un_dvp, cnp->cn_lkflags |
166 				    LK_RETRY);
167 
168 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
169 		} else if (error == ENOENT && (cnflags & MAKEENTRY) != 0)
170 			cache_enter(dvp, NULLVP, cnp);
171 
172 		goto unionfs_lookup_return;
173 	}
174 
175 	/*
176 	 * lookup upper layer
177 	 */
178 	if (udvp != NULLVP) {
179 		uerror = VOP_LOOKUP(udvp, &uvp, cnp);
180 
181 		if (uerror == 0) {
182 			if (udvp == uvp) {	/* is dot */
183 				vrele(uvp);
184 				*(ap->a_vpp) = dvp;
185 				vref(dvp);
186 
187 				error = uerror;
188 				goto unionfs_lookup_return;
189 			}
190 			if (nameiop == DELETE || nameiop == RENAME ||
191 			    (cnp->cn_lkflags & LK_TYPE_MASK))
192 				VOP_UNLOCK(uvp);
193 		}
194 
195 		/* check whiteout */
196 		if (uerror == ENOENT || uerror == EJUSTRETURN)
197 			if (cnp->cn_flags & ISWHITEOUT)
198 				iswhiteout = 1;	/* don't lookup lower */
199 		if (iswhiteout == 0 && ldvp != NULLVP)
200 			if (!VOP_GETATTR(udvp, &va, cnp->cn_cred) &&
201 			    (va.va_flags & OPAQUE))
202 				iswhiteout = 1;	/* don't lookup lower */
203 #if 0
204 		UNIONFS_INTERNAL_DEBUG(
205 		    "unionfs_lookup: debug: whiteout=%d, path=%s\n",
206 		    iswhiteout, cnp->cn_nameptr);
207 #endif
208 	}
209 
210 	/*
211 	 * lookup lower layer
212 	 */
213 	if (ldvp != NULLVP && !(cnflags & DOWHITEOUT) && iswhiteout == 0) {
214 		/* always op is LOOKUP */
215 		cnp->cn_nameiop = LOOKUP;
216 		cnflagsbk = cnp->cn_flags;
217 		cnp->cn_flags = cnflags;
218 
219 		lerror = VOP_LOOKUP(ldvp, &lvp, cnp);
220 
221 		cnp->cn_nameiop = nameiop;
222 		if (udvp != NULLVP && (uerror == 0 || uerror == EJUSTRETURN))
223 			cnp->cn_flags = cnflagsbk;
224 
225 		if (lerror == 0) {
226 			if (ldvp == lvp) {	/* is dot */
227 				if (uvp != NULLVP)
228 					vrele(uvp);	/* no need? */
229 				vrele(lvp);
230 				*(ap->a_vpp) = dvp;
231 				vref(dvp);
232 
233 				UNIONFS_INTERNAL_DEBUG(
234 				    "unionfs_lookup: leave (%d)\n", lerror);
235 
236 				return (lerror);
237 			}
238 			if (cnp->cn_lkflags & LK_TYPE_MASK)
239 				VOP_UNLOCK(lvp);
240 		}
241 	}
242 
243 	/*
244 	 * check lookup result
245 	 */
246 	if (uvp == NULLVP && lvp == NULLVP) {
247 		error = (udvp != NULLVP ? uerror : lerror);
248 		goto unionfs_lookup_return;
249 	}
250 
251 	/*
252 	 * check vnode type
253 	 */
254 	if (uvp != NULLVP && lvp != NULLVP && uvp->v_type != lvp->v_type) {
255 		vrele(lvp);
256 		lvp = NULLVP;
257 	}
258 
259 	/*
260 	 * check shadow dir
261 	 */
262 	if (uerror != 0 && uerror != EJUSTRETURN && udvp != NULLVP &&
263 	    lerror == 0 && lvp != NULLVP && lvp->v_type == VDIR &&
264 	    !(dvp->v_mount->mnt_flag & MNT_RDONLY) &&
265 	    (1 < cnp->cn_namelen || '.' != *(cnp->cn_nameptr))) {
266 		/* get unionfs vnode in order to create a new shadow dir. */
267 		error = unionfs_nodeget(dvp->v_mount, NULLVP, lvp, dvp, &vp,
268 		    cnp);
269 		if (error != 0)
270 			goto unionfs_lookup_cleanup;
271 
272 		if (LK_SHARED == (cnp->cn_lkflags & LK_TYPE_MASK))
273 			VOP_UNLOCK(vp);
274 		if (LK_EXCLUSIVE != VOP_ISLOCKED(vp)) {
275 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
276 			lockflag = 1;
277 		}
278 		error = unionfs_mkshadowdir(MOUNTTOUNIONFSMOUNT(dvp->v_mount),
279 		    udvp, VTOUNIONFS(vp), cnp, td);
280 		if (lockflag != 0)
281 			VOP_UNLOCK(vp);
282 		if (error != 0) {
283 			UNIONFSDEBUG(
284 			    "unionfs_lookup: Unable to create shadow dir.");
285 			if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE)
286 				vput(vp);
287 			else
288 				vrele(vp);
289 			goto unionfs_lookup_cleanup;
290 		}
291 		if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_SHARED)
292 			vn_lock(vp, LK_SHARED | LK_RETRY);
293 	}
294 	/*
295 	 * get unionfs vnode.
296 	 */
297 	else {
298 		if (uvp != NULLVP)
299 			error = uerror;
300 		else
301 			error = lerror;
302 		if (error != 0)
303 			goto unionfs_lookup_cleanup;
304 		/*
305 		 * get socket vnode.
306 		 */
307 		if (uvp != NULLVP && uvp->v_type == VSOCK) {
308 			vp = uvp;
309 			vref(vp);
310 			if (cnp->cn_lkflags & LK_TYPE_MASK)
311 				vn_lock(vp, cnp->cn_lkflags | LK_RETRY);
312 		}
313 		else if (lvp != NULLVP && lvp->v_type == VSOCK) {
314 			vp = lvp;
315 			vref(vp);
316 			if (cnp->cn_lkflags & LK_TYPE_MASK)
317 				vn_lock(vp, cnp->cn_lkflags | LK_RETRY);
318 		}
319 		/*
320 		 * get unionfs vnode.
321 		 */
322 		else
323 			error = unionfs_nodeget(dvp->v_mount, uvp, lvp,
324 			    dvp, &vp, cnp);
325 		if (error != 0) {
326 			UNIONFSDEBUG(
327 			    "unionfs_lookup: Unable to create unionfs vnode.");
328 			goto unionfs_lookup_cleanup;
329 		}
330 		if ((nameiop == DELETE || nameiop == RENAME) &&
331 		    (cnp->cn_lkflags & LK_TYPE_MASK) == 0)
332 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
333 	}
334 
335 	*(ap->a_vpp) = vp;
336 
337 	if ((cnflags & MAKEENTRY) && vp->v_type != VSOCK)
338 		cache_enter(dvp, vp, cnp);
339 
340 unionfs_lookup_cleanup:
341 	if (uvp != NULLVP)
342 		vrele(uvp);
343 	if (lvp != NULLVP)
344 		vrele(lvp);
345 
346 	if (error == ENOENT && (cnflags & MAKEENTRY) != 0)
347 		cache_enter(dvp, NULLVP, cnp);
348 
349 unionfs_lookup_return:
350 
351 	UNIONFS_INTERNAL_DEBUG("unionfs_lookup: leave (%d)\n", error);
352 
353 	return (error);
354 }
355 
356 static int
357 unionfs_create(struct vop_create_args *ap)
358 {
359 	struct unionfs_node *dunp;
360 	struct componentname *cnp;
361 	struct vnode   *udvp;
362 	struct vnode   *vp;
363 	int		error;
364 
365 	UNIONFS_INTERNAL_DEBUG("unionfs_create: enter\n");
366 
367 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
368 
369 	dunp = VTOUNIONFS(ap->a_dvp);
370 	cnp = ap->a_cnp;
371 	udvp = dunp->un_uppervp;
372 	error = EROFS;
373 
374 	if (udvp != NULLVP) {
375 		error = VOP_CREATE(udvp, &vp, cnp, ap->a_vap);
376 		if (error != 0)
377 			goto unionfs_create_abort;
378 
379 		if (vp->v_type == VSOCK)
380 			*(ap->a_vpp) = vp;
381 		else {
382 			VOP_UNLOCK(vp);
383 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
384 			    ap->a_dvp, ap->a_vpp, cnp);
385 			vrele(vp);
386 		}
387 	}
388 
389 unionfs_create_abort:
390 	UNIONFS_INTERNAL_DEBUG("unionfs_create: leave (%d)\n", error);
391 
392 	return (error);
393 }
394 
395 static int
396 unionfs_whiteout(struct vop_whiteout_args *ap)
397 {
398 	struct unionfs_node *dunp;
399 	struct componentname *cnp;
400 	struct vnode   *udvp;
401 	int		error;
402 
403 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: enter\n");
404 
405 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
406 
407 	dunp = VTOUNIONFS(ap->a_dvp);
408 	cnp = ap->a_cnp;
409 	udvp = dunp->un_uppervp;
410 	error = EOPNOTSUPP;
411 
412 	if (udvp != NULLVP) {
413 		switch (ap->a_flags) {
414 		case CREATE:
415 		case DELETE:
416 		case LOOKUP:
417 			error = VOP_WHITEOUT(udvp, cnp, ap->a_flags);
418 			break;
419 		default:
420 			error = EINVAL;
421 			break;
422 		}
423 	}
424 
425 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: leave (%d)\n", error);
426 
427 	return (error);
428 }
429 
430 static int
431 unionfs_mknod(struct vop_mknod_args *ap)
432 {
433 	struct unionfs_node *dunp;
434 	struct componentname *cnp;
435 	struct vnode   *udvp;
436 	struct vnode   *vp;
437 	int		error;
438 
439 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: enter\n");
440 
441 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
442 
443 	dunp = VTOUNIONFS(ap->a_dvp);
444 	cnp = ap->a_cnp;
445 	udvp = dunp->un_uppervp;
446 	error = EROFS;
447 
448 	if (udvp != NULLVP) {
449 		error = VOP_MKNOD(udvp, &vp, cnp, ap->a_vap);
450 		if (error != 0)
451 			goto unionfs_mknod_abort;
452 
453 		if (vp->v_type == VSOCK)
454 			*(ap->a_vpp) = vp;
455 		else {
456 			VOP_UNLOCK(vp);
457 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
458 			    ap->a_dvp, ap->a_vpp, cnp);
459 			vrele(vp);
460 		}
461 	}
462 
463 unionfs_mknod_abort:
464 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: leave (%d)\n", error);
465 
466 	return (error);
467 }
468 
469 enum unionfs_lkupgrade {
470 	UNIONFS_LKUPGRADE_SUCCESS, /* lock successfully upgraded */
471 	UNIONFS_LKUPGRADE_ALREADY, /* lock already held exclusive */
472 	UNIONFS_LKUPGRADE_DOOMED   /* lock was upgraded, but vnode reclaimed */
473 };
474 
475 static inline enum unionfs_lkupgrade
476 unionfs_upgrade_lock(struct vnode *vp)
477 {
478 	ASSERT_VOP_LOCKED(vp, __func__);
479 
480 	if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
481 		return (UNIONFS_LKUPGRADE_ALREADY);
482 
483 	if (vn_lock(vp, LK_UPGRADE) != 0) {
484 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
485 		if (VN_IS_DOOMED(vp))
486 			return (UNIONFS_LKUPGRADE_DOOMED);
487 	}
488 	return (UNIONFS_LKUPGRADE_SUCCESS);
489 }
490 
491 static inline void
492 unionfs_downgrade_lock(struct vnode *vp, enum unionfs_lkupgrade status)
493 {
494 	if (status != UNIONFS_LKUPGRADE_ALREADY)
495 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
496 }
497 
498 static int
499 unionfs_open(struct vop_open_args *ap)
500 {
501 	struct unionfs_node *unp;
502 	struct unionfs_node_status *unsp;
503 	struct vnode   *vp;
504 	struct vnode   *uvp;
505 	struct vnode   *lvp;
506 	struct vnode   *targetvp;
507 	struct ucred   *cred;
508 	struct thread  *td;
509 	int		error;
510 	enum unionfs_lkupgrade lkstatus;
511 
512 	UNIONFS_INTERNAL_DEBUG("unionfs_open: enter\n");
513 
514 	KASSERT_UNIONFS_VNODE(ap->a_vp);
515 
516 	error = 0;
517 	vp = ap->a_vp;
518 	targetvp = NULLVP;
519 	cred = ap->a_cred;
520 	td = ap->a_td;
521 
522 	/*
523 	 * The executable loader path may call this function with vp locked
524 	 * shared.  If the vnode is reclaimed while upgrading, we can't safely
525 	 * use unp or do anything else unionfs- specific.
526 	 */
527 	lkstatus = unionfs_upgrade_lock(vp);
528 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED) {
529 		error = ENOENT;
530 		goto unionfs_open_cleanup;
531 	}
532 
533 	unp = VTOUNIONFS(vp);
534 	uvp = unp->un_uppervp;
535 	lvp = unp->un_lowervp;
536 	unionfs_get_node_status(unp, td, &unsp);
537 
538 	if (unsp->uns_lower_opencnt > 0 || unsp->uns_upper_opencnt > 0) {
539 		/* vnode is already opend. */
540 		if (unsp->uns_upper_opencnt > 0)
541 			targetvp = uvp;
542 		else
543 			targetvp = lvp;
544 
545 		if (targetvp == lvp &&
546 		    (ap->a_mode & FWRITE) && lvp->v_type == VREG)
547 			targetvp = NULLVP;
548 	}
549 	if (targetvp == NULLVP) {
550 		if (uvp == NULLVP) {
551 			if ((ap->a_mode & FWRITE) && lvp->v_type == VREG) {
552 				error = unionfs_copyfile(unp,
553 				    !(ap->a_mode & O_TRUNC), cred, td);
554 				if (error != 0)
555 					goto unionfs_open_abort;
556 				targetvp = uvp = unp->un_uppervp;
557 			} else
558 				targetvp = lvp;
559 		} else
560 			targetvp = uvp;
561 	}
562 
563 	error = VOP_OPEN(targetvp, ap->a_mode, cred, td, ap->a_fp);
564 	if (error == 0) {
565 		if (targetvp == uvp) {
566 			if (uvp->v_type == VDIR && lvp != NULLVP &&
567 			    unsp->uns_lower_opencnt <= 0) {
568 				/* open lower for readdir */
569 				error = VOP_OPEN(lvp, FREAD, cred, td, NULL);
570 				if (error != 0) {
571 					VOP_CLOSE(uvp, ap->a_mode, cred, td);
572 					goto unionfs_open_abort;
573 				}
574 				unsp->uns_node_flag |= UNS_OPENL_4_READDIR;
575 				unsp->uns_lower_opencnt++;
576 			}
577 			unsp->uns_upper_opencnt++;
578 		} else {
579 			unsp->uns_lower_opencnt++;
580 			unsp->uns_lower_openmode = ap->a_mode;
581 		}
582 		vp->v_object = targetvp->v_object;
583 	}
584 
585 unionfs_open_abort:
586 	if (error != 0)
587 		unionfs_tryrem_node_status(unp, unsp);
588 
589 unionfs_open_cleanup:
590 	unionfs_downgrade_lock(vp, lkstatus);
591 
592 	UNIONFS_INTERNAL_DEBUG("unionfs_open: leave (%d)\n", error);
593 
594 	return (error);
595 }
596 
597 static int
598 unionfs_close(struct vop_close_args *ap)
599 {
600 	struct unionfs_node *unp;
601 	struct unionfs_node_status *unsp;
602 	struct ucred   *cred;
603 	struct thread  *td;
604 	struct vnode   *vp;
605 	struct vnode   *ovp;
606 	int		error;
607 	enum unionfs_lkupgrade lkstatus;;
608 
609 	UNIONFS_INTERNAL_DEBUG("unionfs_close: enter\n");
610 
611 	KASSERT_UNIONFS_VNODE(ap->a_vp);
612 
613 	vp = ap->a_vp;
614 	cred = ap->a_cred;
615 	td = ap->a_td;
616 	error = 0;
617 
618 	/*
619 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
620 	 * or do anything else unionfs- specific.
621 	 */
622 	lkstatus = unionfs_upgrade_lock(vp);
623 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
624 		goto unionfs_close_cleanup;
625 
626 	unp = VTOUNIONFS(vp);
627 	unionfs_get_node_status(unp, td, &unsp);
628 
629 	if (unsp->uns_lower_opencnt <= 0 && unsp->uns_upper_opencnt <= 0) {
630 #ifdef DIAGNOSTIC
631 		printf("unionfs_close: warning: open count is 0\n");
632 #endif
633 		if (unp->un_uppervp != NULLVP)
634 			ovp = unp->un_uppervp;
635 		else
636 			ovp = unp->un_lowervp;
637 	} else if (unsp->uns_upper_opencnt > 0)
638 		ovp = unp->un_uppervp;
639 	else
640 		ovp = unp->un_lowervp;
641 
642 	error = VOP_CLOSE(ovp, ap->a_fflag, cred, td);
643 
644 	if (error != 0)
645 		goto unionfs_close_abort;
646 
647 	vp->v_object = ovp->v_object;
648 
649 	if (ovp == unp->un_uppervp) {
650 		unsp->uns_upper_opencnt--;
651 		if (unsp->uns_upper_opencnt == 0) {
652 			if (unsp->uns_node_flag & UNS_OPENL_4_READDIR) {
653 				VOP_CLOSE(unp->un_lowervp, FREAD, cred, td);
654 				unsp->uns_node_flag &= ~UNS_OPENL_4_READDIR;
655 				unsp->uns_lower_opencnt--;
656 			}
657 			if (unsp->uns_lower_opencnt > 0)
658 				vp->v_object = unp->un_lowervp->v_object;
659 		}
660 	} else
661 		unsp->uns_lower_opencnt--;
662 
663 unionfs_close_abort:
664 	unionfs_tryrem_node_status(unp, unsp);
665 
666 unionfs_close_cleanup:
667 	unionfs_downgrade_lock(vp, lkstatus);
668 
669 	UNIONFS_INTERNAL_DEBUG("unionfs_close: leave (%d)\n", error);
670 
671 	return (error);
672 }
673 
674 /*
675  * Check the access mode toward shadow file/dir.
676  */
677 static int
678 unionfs_check_corrected_access(accmode_t accmode, struct vattr *va,
679     struct ucred *cred)
680 {
681 	uid_t		uid;	/* upper side vnode's uid */
682 	gid_t		gid;	/* upper side vnode's gid */
683 	u_short		vmode;	/* upper side vnode's mode */
684 	u_short		mask;
685 
686 	mask = 0;
687 	uid = va->va_uid;
688 	gid = va->va_gid;
689 	vmode = va->va_mode;
690 
691 	/* check owner */
692 	if (cred->cr_uid == uid) {
693 		if (accmode & VEXEC)
694 			mask |= S_IXUSR;
695 		if (accmode & VREAD)
696 			mask |= S_IRUSR;
697 		if (accmode & VWRITE)
698 			mask |= S_IWUSR;
699 		return ((vmode & mask) == mask ? 0 : EACCES);
700 	}
701 
702 	/* check group */
703 	if (groupmember(gid, cred)) {
704 		if (accmode & VEXEC)
705 			mask |= S_IXGRP;
706 		if (accmode & VREAD)
707 			mask |= S_IRGRP;
708 		if (accmode & VWRITE)
709 			mask |= S_IWGRP;
710 		return ((vmode & mask) == mask ? 0 : EACCES);
711 	}
712 
713 	/* check other */
714 	if (accmode & VEXEC)
715 		mask |= S_IXOTH;
716 	if (accmode & VREAD)
717 		mask |= S_IROTH;
718 	if (accmode & VWRITE)
719 		mask |= S_IWOTH;
720 
721 	return ((vmode & mask) == mask ? 0 : EACCES);
722 }
723 
724 static int
725 unionfs_access(struct vop_access_args *ap)
726 {
727 	struct unionfs_mount *ump;
728 	struct unionfs_node *unp;
729 	struct vnode   *uvp;
730 	struct vnode   *lvp;
731 	struct thread  *td;
732 	struct vattr	va;
733 	accmode_t	accmode;
734 	int		error;
735 
736 	UNIONFS_INTERNAL_DEBUG("unionfs_access: enter\n");
737 
738 	KASSERT_UNIONFS_VNODE(ap->a_vp);
739 
740 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
741 	unp = VTOUNIONFS(ap->a_vp);
742 	uvp = unp->un_uppervp;
743 	lvp = unp->un_lowervp;
744 	td = ap->a_td;
745 	accmode = ap->a_accmode;
746 	error = EACCES;
747 
748 	if ((accmode & VWRITE) &&
749 	    (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)) {
750 		switch (ap->a_vp->v_type) {
751 		case VREG:
752 		case VDIR:
753 		case VLNK:
754 			return (EROFS);
755 		default:
756 			break;
757 		}
758 	}
759 
760 	if (uvp != NULLVP) {
761 		error = VOP_ACCESS(uvp, accmode, ap->a_cred, td);
762 
763 		UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
764 
765 		return (error);
766 	}
767 
768 	if (lvp != NULLVP) {
769 		if (accmode & VWRITE) {
770 			if (ump->um_uppervp->v_mount->mnt_flag & MNT_RDONLY) {
771 				switch (ap->a_vp->v_type) {
772 				case VREG:
773 				case VDIR:
774 				case VLNK:
775 					return (EROFS);
776 				default:
777 					break;
778 				}
779 			} else if (ap->a_vp->v_type == VREG ||
780 			    ap->a_vp->v_type == VDIR) {
781 				/* check shadow file/dir */
782 				if (ump->um_copymode != UNIONFS_TRANSPARENT) {
783 					error = unionfs_create_uppervattr(ump,
784 					    lvp, &va, ap->a_cred, td);
785 					if (error != 0)
786 						return (error);
787 
788 					error = unionfs_check_corrected_access(
789 					    accmode, &va, ap->a_cred);
790 					if (error != 0)
791 						return (error);
792 				}
793 			}
794 			accmode &= ~(VWRITE | VAPPEND);
795 			accmode |= VREAD; /* will copy to upper */
796 		}
797 		error = VOP_ACCESS(lvp, accmode, ap->a_cred, td);
798 	}
799 
800 	UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
801 
802 	return (error);
803 }
804 
805 static int
806 unionfs_getattr(struct vop_getattr_args *ap)
807 {
808 	struct unionfs_node *unp;
809 	struct unionfs_mount *ump;
810 	struct vnode   *uvp;
811 	struct vnode   *lvp;
812 	struct thread  *td;
813 	struct vattr	va;
814 	int		error;
815 
816 	UNIONFS_INTERNAL_DEBUG("unionfs_getattr: enter\n");
817 
818 	KASSERT_UNIONFS_VNODE(ap->a_vp);
819 
820 	unp = VTOUNIONFS(ap->a_vp);
821 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
822 	uvp = unp->un_uppervp;
823 	lvp = unp->un_lowervp;
824 	td = curthread;
825 
826 	if (uvp != NULLVP) {
827 		if ((error = VOP_GETATTR(uvp, ap->a_vap, ap->a_cred)) == 0)
828 			ap->a_vap->va_fsid =
829 			    ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
830 
831 		UNIONFS_INTERNAL_DEBUG(
832 		    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
833 		    ap->a_vap->va_mode, ap->a_vap->va_uid,
834 		    ap->a_vap->va_gid, error);
835 
836 		return (error);
837 	}
838 
839 	error = VOP_GETATTR(lvp, ap->a_vap, ap->a_cred);
840 
841 	if (error == 0 && !(ump->um_uppervp->v_mount->mnt_flag & MNT_RDONLY)) {
842 		/* correct the attr toward shadow file/dir. */
843 		if (ap->a_vp->v_type == VREG || ap->a_vp->v_type == VDIR) {
844 			unionfs_create_uppervattr_core(ump, ap->a_vap, &va, td);
845 			ap->a_vap->va_mode = va.va_mode;
846 			ap->a_vap->va_uid = va.va_uid;
847 			ap->a_vap->va_gid = va.va_gid;
848 		}
849 	}
850 
851 	if (error == 0)
852 		ap->a_vap->va_fsid = ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
853 
854 	UNIONFS_INTERNAL_DEBUG(
855 	    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
856 	    ap->a_vap->va_mode, ap->a_vap->va_uid, ap->a_vap->va_gid, error);
857 
858 	return (error);
859 }
860 
861 static int
862 unionfs_setattr(struct vop_setattr_args *ap)
863 {
864 	struct unionfs_node *unp;
865 	struct vnode   *uvp;
866 	struct vnode   *lvp;
867 	struct thread  *td;
868 	struct vattr   *vap;
869 	int		error;
870 
871 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: enter\n");
872 
873 	KASSERT_UNIONFS_VNODE(ap->a_vp);
874 
875 	error = EROFS;
876 	unp = VTOUNIONFS(ap->a_vp);
877 	uvp = unp->un_uppervp;
878 	lvp = unp->un_lowervp;
879 	td = curthread;
880 	vap = ap->a_vap;
881 
882 	if ((ap->a_vp->v_mount->mnt_flag & MNT_RDONLY) &&
883 	    (vap->va_flags != VNOVAL || vap->va_uid != (uid_t)VNOVAL ||
884 	     vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL ||
885 	     vap->va_mtime.tv_sec != VNOVAL || vap->va_mode != (mode_t)VNOVAL))
886 		return (EROFS);
887 
888 	if (uvp == NULLVP && lvp->v_type == VREG) {
889 		error = unionfs_copyfile(unp, (vap->va_size != 0),
890 		    ap->a_cred, td);
891 		if (error != 0)
892 			return (error);
893 		uvp = unp->un_uppervp;
894 	}
895 
896 	if (uvp != NULLVP)
897 		error = VOP_SETATTR(uvp, vap, ap->a_cred);
898 
899 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: leave (%d)\n", error);
900 
901 	return (error);
902 }
903 
904 static int
905 unionfs_read(struct vop_read_args *ap)
906 {
907 	struct unionfs_node *unp;
908 	struct vnode   *tvp;
909 	int		error;
910 
911 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: enter\n"); */
912 
913 	KASSERT_UNIONFS_VNODE(ap->a_vp);
914 
915 	unp = VTOUNIONFS(ap->a_vp);
916 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
917 
918 	error = VOP_READ(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
919 
920 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: leave (%d)\n", error); */
921 
922 	return (error);
923 }
924 
925 static int
926 unionfs_write(struct vop_write_args *ap)
927 {
928 	struct unionfs_node *unp;
929 	struct vnode   *tvp;
930 	int		error;
931 
932 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: enter\n"); */
933 
934 	KASSERT_UNIONFS_VNODE(ap->a_vp);
935 
936 	unp = VTOUNIONFS(ap->a_vp);
937 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
938 
939 	error = VOP_WRITE(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
940 
941 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: leave (%d)\n", error); */
942 
943 	return (error);
944 }
945 
946 static int
947 unionfs_ioctl(struct vop_ioctl_args *ap)
948 {
949 	struct unionfs_node *unp;
950 	struct unionfs_node_status *unsp;
951 	struct vnode   *ovp;
952 	int error;
953 
954 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: enter\n");
955 
956 	KASSERT_UNIONFS_VNODE(ap->a_vp);
957 
958  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
959 	unp = VTOUNIONFS(ap->a_vp);
960 	unionfs_get_node_status(unp, ap->a_td, &unsp);
961 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
962 	unionfs_tryrem_node_status(unp, unsp);
963 	VOP_UNLOCK(ap->a_vp);
964 
965 	if (ovp == NULLVP)
966 		return (EBADF);
967 
968 	error = VOP_IOCTL(ovp, ap->a_command, ap->a_data, ap->a_fflag,
969 	    ap->a_cred, ap->a_td);
970 
971 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: leave (%d)\n", error);
972 
973 	return (error);
974 }
975 
976 static int
977 unionfs_poll(struct vop_poll_args *ap)
978 {
979 	struct unionfs_node *unp;
980 	struct unionfs_node_status *unsp;
981 	struct vnode *ovp;
982 
983 	KASSERT_UNIONFS_VNODE(ap->a_vp);
984 
985  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
986 	unp = VTOUNIONFS(ap->a_vp);
987 	unionfs_get_node_status(unp, ap->a_td, &unsp);
988 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
989 	unionfs_tryrem_node_status(unp, unsp);
990 	VOP_UNLOCK(ap->a_vp);
991 
992 	if (ovp == NULLVP)
993 		return (EBADF);
994 
995 	return (VOP_POLL(ovp, ap->a_events, ap->a_cred, ap->a_td));
996 }
997 
998 static int
999 unionfs_fsync(struct vop_fsync_args *ap)
1000 {
1001 	struct unionfs_node *unp;
1002 	struct unionfs_node_status *unsp;
1003 	struct vnode *ovp;
1004 
1005 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1006 
1007 	unp = VTOUNIONFS(ap->a_vp);
1008 	unionfs_get_node_status(unp, ap->a_td, &unsp);
1009 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
1010 	unionfs_tryrem_node_status(unp, unsp);
1011 
1012 	if (ovp == NULLVP)
1013 		return (EBADF);
1014 
1015 	return (VOP_FSYNC(ovp, ap->a_waitfor, ap->a_td));
1016 }
1017 
1018 static int
1019 unionfs_remove(struct vop_remove_args *ap)
1020 {
1021 	char	       *path;
1022 	struct unionfs_node *dunp;
1023 	struct unionfs_node *unp;
1024 	struct unionfs_mount *ump;
1025 	struct vnode   *udvp;
1026 	struct vnode   *uvp;
1027 	struct vnode   *lvp;
1028 	struct vnode   *vp;
1029 	struct componentname *cnp;
1030 	struct componentname cn;
1031 	struct thread  *td;
1032 	int		error;
1033 	int		pathlen;
1034 
1035 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: enter\n");
1036 
1037 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1038 
1039 	error = 0;
1040 	dunp = VTOUNIONFS(ap->a_dvp);
1041 	udvp = dunp->un_uppervp;
1042 	cnp = ap->a_cnp;
1043 	td = curthread;
1044 
1045 	if (ap->a_vp->v_op != &unionfs_vnodeops) {
1046 		if (ap->a_vp->v_type != VSOCK)
1047 			return (EINVAL);
1048 		ump = NULL;
1049 		vp = uvp = lvp = NULLVP;
1050 		/* search vnode */
1051 		VOP_UNLOCK(ap->a_vp);
1052 		error = unionfs_relookup(udvp, &vp, cnp, &cn, td,
1053 		    cnp->cn_nameptr, cnp->cn_namelen, DELETE);
1054 		if (error != 0 && error != ENOENT) {
1055 			vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1056 			return (error);
1057 		}
1058 
1059 		if (error == 0 && vp == ap->a_vp) {
1060 			/* target vnode in upper */
1061 			uvp = vp;
1062 			vrele(vp);
1063 		} else {
1064 			/* target vnode in lower */
1065 			if (vp != NULLVP) {
1066 				if (udvp == vp)
1067 					vrele(vp);
1068 				else
1069 					vput(vp);
1070 			}
1071 			vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1072 			lvp = ap->a_vp;
1073 		}
1074 		path = cnp->cn_nameptr;
1075 		pathlen = cnp->cn_namelen;
1076 	} else {
1077 		ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1078 		unp = VTOUNIONFS(ap->a_vp);
1079 		uvp = unp->un_uppervp;
1080 		lvp = unp->un_lowervp;
1081 		path = unp->un_path;
1082 		pathlen = unp->un_pathlen;
1083 	}
1084 
1085 	if (udvp == NULLVP)
1086 		return (EROFS);
1087 
1088 	if (uvp != NULLVP) {
1089 		/*
1090 		 * XXX: if the vnode type is VSOCK, it will create whiteout
1091 		 *      after remove.
1092 		 */
1093 		if (ump == NULL || ump->um_whitemode == UNIONFS_WHITE_ALWAYS ||
1094 		    lvp != NULLVP)
1095 			cnp->cn_flags |= DOWHITEOUT;
1096 		error = VOP_REMOVE(udvp, uvp, cnp);
1097 	} else if (lvp != NULLVP)
1098 		error = unionfs_mkwhiteout(udvp, cnp, td, path, pathlen);
1099 
1100 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: leave (%d)\n", error);
1101 
1102 	return (error);
1103 }
1104 
1105 static int
1106 unionfs_link(struct vop_link_args *ap)
1107 {
1108 	struct unionfs_node *dunp;
1109 	struct unionfs_node *unp;
1110 	struct vnode   *udvp;
1111 	struct vnode   *uvp;
1112 	struct componentname *cnp;
1113 	struct thread  *td;
1114 	int		error;
1115 	int		needrelookup;
1116 
1117 	UNIONFS_INTERNAL_DEBUG("unionfs_link: enter\n");
1118 
1119 	KASSERT_UNIONFS_VNODE(ap->a_tdvp);
1120 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1121 
1122 	error = 0;
1123 	needrelookup = 0;
1124 	dunp = VTOUNIONFS(ap->a_tdvp);
1125 	unp = NULL;
1126 	udvp = dunp->un_uppervp;
1127 	uvp = NULLVP;
1128 	cnp = ap->a_cnp;
1129 	td = curthread;
1130 
1131 	if (udvp == NULLVP)
1132 		return (EROFS);
1133 
1134 	if (ap->a_vp->v_op != &unionfs_vnodeops)
1135 		uvp = ap->a_vp;
1136 	else {
1137 		unp = VTOUNIONFS(ap->a_vp);
1138 
1139 		if (unp->un_uppervp == NULLVP) {
1140 			if (ap->a_vp->v_type != VREG)
1141 				return (EOPNOTSUPP);
1142 
1143 			error = unionfs_copyfile(unp, 1, cnp->cn_cred, td);
1144 			if (error != 0)
1145 				return (error);
1146 			needrelookup = 1;
1147 		}
1148 		uvp = unp->un_uppervp;
1149 	}
1150 
1151 	if (needrelookup != 0)
1152 		error = unionfs_relookup_for_create(ap->a_tdvp, cnp, td);
1153 
1154 	if (error == 0)
1155 		error = VOP_LINK(udvp, uvp, cnp);
1156 
1157 	UNIONFS_INTERNAL_DEBUG("unionfs_link: leave (%d)\n", error);
1158 
1159 	return (error);
1160 }
1161 
1162 static int
1163 unionfs_rename(struct vop_rename_args *ap)
1164 {
1165 	struct vnode   *fdvp;
1166 	struct vnode   *fvp;
1167 	struct componentname *fcnp;
1168 	struct vnode   *tdvp;
1169 	struct vnode   *tvp;
1170 	struct componentname *tcnp;
1171 	struct vnode   *ltdvp;
1172 	struct vnode   *ltvp;
1173 	struct thread  *td;
1174 
1175 	/* rename target vnodes */
1176 	struct vnode   *rfdvp;
1177 	struct vnode   *rfvp;
1178 	struct vnode   *rtdvp;
1179 	struct vnode   *rtvp;
1180 
1181 	struct unionfs_mount *ump;
1182 	struct unionfs_node *unp;
1183 	int		error;
1184 	int		needrelookup;
1185 
1186 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: enter\n");
1187 
1188 	error = 0;
1189 	fdvp = ap->a_fdvp;
1190 	fvp = ap->a_fvp;
1191 	fcnp = ap->a_fcnp;
1192 	tdvp = ap->a_tdvp;
1193 	tvp = ap->a_tvp;
1194 	tcnp = ap->a_tcnp;
1195 	ltdvp = NULLVP;
1196 	ltvp = NULLVP;
1197 	td = curthread;
1198 	rfdvp = fdvp;
1199 	rfvp = fvp;
1200 	rtdvp = tdvp;
1201 	rtvp = tvp;
1202 	needrelookup = 0;
1203 
1204 	/* check for cross device rename */
1205 	if (fvp->v_mount != tdvp->v_mount ||
1206 	    (tvp != NULLVP && fvp->v_mount != tvp->v_mount)) {
1207 		if (fvp->v_op != &unionfs_vnodeops)
1208 			error = ENODEV;
1209 		else
1210 			error = EXDEV;
1211 		goto unionfs_rename_abort;
1212 	}
1213 
1214 	/* Renaming a file to itself has no effect. */
1215 	if (fvp == tvp)
1216 		goto unionfs_rename_abort;
1217 
1218 	/*
1219 	 * from/to vnode is unionfs node.
1220 	 */
1221 
1222 	KASSERT_UNIONFS_VNODE(fdvp);
1223 	KASSERT_UNIONFS_VNODE(fvp);
1224 	KASSERT_UNIONFS_VNODE(tdvp);
1225 	if (tvp != NULLVP)
1226 		KASSERT_UNIONFS_VNODE(tvp);
1227 
1228 	unp = VTOUNIONFS(fdvp);
1229 #ifdef UNIONFS_IDBG_RENAME
1230 	UNIONFS_INTERNAL_DEBUG("fdvp=%p, ufdvp=%p, lfdvp=%p\n",
1231 	    fdvp, unp->un_uppervp, unp->un_lowervp);
1232 #endif
1233 	if (unp->un_uppervp == NULLVP) {
1234 		error = ENODEV;
1235 		goto unionfs_rename_abort;
1236 	}
1237 	rfdvp = unp->un_uppervp;
1238 	vref(rfdvp);
1239 
1240 	unp = VTOUNIONFS(fvp);
1241 #ifdef UNIONFS_IDBG_RENAME
1242 	UNIONFS_INTERNAL_DEBUG("fvp=%p, ufvp=%p, lfvp=%p\n",
1243 	    fvp, unp->un_uppervp, unp->un_lowervp);
1244 #endif
1245 	ump = MOUNTTOUNIONFSMOUNT(fvp->v_mount);
1246 	if (unp->un_uppervp == NULLVP) {
1247 		switch (fvp->v_type) {
1248 		case VREG:
1249 			if ((error = vn_lock(fvp, LK_EXCLUSIVE)) != 0)
1250 				goto unionfs_rename_abort;
1251 			error = unionfs_copyfile(unp, 1, fcnp->cn_cred, td);
1252 			VOP_UNLOCK(fvp);
1253 			if (error != 0)
1254 				goto unionfs_rename_abort;
1255 			break;
1256 		case VDIR:
1257 			if ((error = vn_lock(fvp, LK_EXCLUSIVE)) != 0)
1258 				goto unionfs_rename_abort;
1259 			error = unionfs_mkshadowdir(ump, rfdvp, unp, fcnp, td);
1260 			VOP_UNLOCK(fvp);
1261 			if (error != 0)
1262 				goto unionfs_rename_abort;
1263 			break;
1264 		default:
1265 			error = ENODEV;
1266 			goto unionfs_rename_abort;
1267 		}
1268 
1269 		needrelookup = 1;
1270 	}
1271 
1272 	if (unp->un_lowervp != NULLVP)
1273 		fcnp->cn_flags |= DOWHITEOUT;
1274 	rfvp = unp->un_uppervp;
1275 	vref(rfvp);
1276 
1277 	unp = VTOUNIONFS(tdvp);
1278 #ifdef UNIONFS_IDBG_RENAME
1279 	UNIONFS_INTERNAL_DEBUG("tdvp=%p, utdvp=%p, ltdvp=%p\n",
1280 	    tdvp, unp->un_uppervp, unp->un_lowervp);
1281 #endif
1282 	if (unp->un_uppervp == NULLVP) {
1283 		error = ENODEV;
1284 		goto unionfs_rename_abort;
1285 	}
1286 	rtdvp = unp->un_uppervp;
1287 	ltdvp = unp->un_lowervp;
1288 	vref(rtdvp);
1289 
1290 	if (tdvp == tvp) {
1291 		rtvp = rtdvp;
1292 		vref(rtvp);
1293 	} else if (tvp != NULLVP) {
1294 		unp = VTOUNIONFS(tvp);
1295 #ifdef UNIONFS_IDBG_RENAME
1296 		UNIONFS_INTERNAL_DEBUG("tvp=%p, utvp=%p, ltvp=%p\n",
1297 		    tvp, unp->un_uppervp, unp->un_lowervp);
1298 #endif
1299 		if (unp->un_uppervp == NULLVP)
1300 			rtvp = NULLVP;
1301 		else {
1302 			if (tvp->v_type == VDIR) {
1303 				error = EINVAL;
1304 				goto unionfs_rename_abort;
1305 			}
1306 			rtvp = unp->un_uppervp;
1307 			ltvp = unp->un_lowervp;
1308 			vref(rtvp);
1309 		}
1310 	}
1311 
1312 	if (rfvp == rtvp)
1313 		goto unionfs_rename_abort;
1314 
1315 	if (needrelookup != 0) {
1316 		if ((error = vn_lock(fdvp, LK_EXCLUSIVE)) != 0)
1317 			goto unionfs_rename_abort;
1318 		error = unionfs_relookup_for_delete(fdvp, fcnp, td);
1319 		VOP_UNLOCK(fdvp);
1320 		if (error != 0)
1321 			goto unionfs_rename_abort;
1322 
1323 		/* Lock of tvp is canceled in order to avoid recursive lock. */
1324 		if (tvp != NULLVP && tvp != tdvp)
1325 			VOP_UNLOCK(tvp);
1326 		error = unionfs_relookup_for_rename(tdvp, tcnp, td);
1327 		if (tvp != NULLVP && tvp != tdvp)
1328 			vn_lock(tvp, LK_EXCLUSIVE | LK_RETRY);
1329 		if (error != 0)
1330 			goto unionfs_rename_abort;
1331 	}
1332 
1333 	error = VOP_RENAME(rfdvp, rfvp, fcnp, rtdvp, rtvp, tcnp);
1334 
1335 	if (error == 0) {
1336 		if (rtvp != NULLVP && rtvp->v_type == VDIR)
1337 			cache_purge(tdvp);
1338 		if (fvp->v_type == VDIR && fdvp != tdvp)
1339 			cache_purge(fdvp);
1340 	}
1341 
1342 	if (ltdvp != NULLVP)
1343 		VOP_UNLOCK(ltdvp);
1344 	if (tdvp != rtdvp)
1345 		vrele(tdvp);
1346 	if (ltvp != NULLVP)
1347 		VOP_UNLOCK(ltvp);
1348 	if (tvp != rtvp && tvp != NULLVP) {
1349 		if (rtvp == NULLVP)
1350 			vput(tvp);
1351 		else
1352 			vrele(tvp);
1353 	}
1354 	if (fdvp != rfdvp)
1355 		vrele(fdvp);
1356 	if (fvp != rfvp)
1357 		vrele(fvp);
1358 
1359 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1360 
1361 	return (error);
1362 
1363 unionfs_rename_abort:
1364 	vput(tdvp);
1365 	if (tdvp != rtdvp)
1366 		vrele(rtdvp);
1367 	if (tvp != NULLVP) {
1368 		if (tdvp != tvp)
1369 			vput(tvp);
1370 		else
1371 			vrele(tvp);
1372 	}
1373 	if (tvp != rtvp && rtvp != NULLVP)
1374 		vrele(rtvp);
1375 	if (fdvp != rfdvp)
1376 		vrele(rfdvp);
1377 	if (fvp != rfvp)
1378 		vrele(rfvp);
1379 	vrele(fdvp);
1380 	vrele(fvp);
1381 
1382 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1383 
1384 	return (error);
1385 }
1386 
1387 static int
1388 unionfs_mkdir(struct vop_mkdir_args *ap)
1389 {
1390 	struct unionfs_node *dunp;
1391 	struct componentname *cnp;
1392 	struct vnode   *dvp;
1393 	struct vnode   *udvp;
1394 	struct vnode   *uvp;
1395 	struct vattr	va;
1396 	int		error;
1397 	int		lkflags;
1398 
1399 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: enter\n");
1400 
1401 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1402 
1403 	error = EROFS;
1404 	dvp = ap->a_dvp;
1405 	dunp = VTOUNIONFS(dvp);
1406 	cnp = ap->a_cnp;
1407 	lkflags = cnp->cn_lkflags;
1408 	udvp = dunp->un_uppervp;
1409 
1410 	if (udvp != NULLVP) {
1411 		vref(udvp);
1412 		/* check opaque */
1413 		if (!(cnp->cn_flags & ISWHITEOUT)) {
1414 			error = VOP_GETATTR(udvp, &va, cnp->cn_cred);
1415 			if (error != 0)
1416 				goto unionfs_mkdir_cleanup;
1417 			if ((va.va_flags & OPAQUE) != 0)
1418 				cnp->cn_flags |= ISWHITEOUT;
1419 		}
1420 
1421 		if ((error = VOP_MKDIR(udvp, &uvp, cnp, ap->a_vap)) == 0) {
1422 			VOP_UNLOCK(uvp);
1423 			cnp->cn_lkflags = LK_EXCLUSIVE;
1424 			/*
1425 			 * The underlying VOP_MKDIR() implementation may have
1426 			 * temporarily dropped the parent directory vnode lock.
1427 			 * Because the unionfs vnode ordinarily shares that
1428 			 * lock, this may allow the unionfs vnode to be reclaimed
1429 			 * and its lock field reset.  In that case, the unionfs
1430 			 * vnode is effectively no longer locked, and we must
1431 			 * explicitly lock it before returning in order to meet
1432 			 * the locking requirements of VOP_MKDIR().
1433 			 */
1434 			if (__predict_false(VTOUNIONFS(dvp) == NULL)) {
1435 				error = ENOENT;
1436 				goto unionfs_mkdir_cleanup;
1437 			}
1438 			error = unionfs_nodeget(dvp->v_mount, uvp, NULLVP,
1439 			    dvp, ap->a_vpp, cnp);
1440 			cnp->cn_lkflags = lkflags;
1441 			vrele(uvp);
1442 		}
1443 	}
1444 
1445 unionfs_mkdir_cleanup:
1446 
1447 	if (__predict_false(VTOUNIONFS(dvp) == NULL)) {
1448 		vput(udvp);
1449 		vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
1450 	} else if (udvp != NULLVP)
1451 		vrele(udvp);
1452 
1453 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: leave (%d)\n", error);
1454 
1455 	return (error);
1456 }
1457 
1458 static int
1459 unionfs_rmdir(struct vop_rmdir_args *ap)
1460 {
1461 	struct unionfs_node *dunp;
1462 	struct unionfs_node *unp;
1463 	struct unionfs_mount *ump;
1464 	struct componentname *cnp;
1465 	struct thread  *td;
1466 	struct vnode   *udvp;
1467 	struct vnode   *uvp;
1468 	struct vnode   *lvp;
1469 	int		error;
1470 
1471 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: enter\n");
1472 
1473 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1474 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1475 
1476 	error = 0;
1477 	dunp = VTOUNIONFS(ap->a_dvp);
1478 	unp = VTOUNIONFS(ap->a_vp);
1479 	cnp = ap->a_cnp;
1480 	td = curthread;
1481 	udvp = dunp->un_uppervp;
1482 	uvp = unp->un_uppervp;
1483 	lvp = unp->un_lowervp;
1484 
1485 	if (udvp == NULLVP)
1486 		return (EROFS);
1487 
1488 	if (udvp == uvp)
1489 		return (EOPNOTSUPP);
1490 
1491 	if (uvp != NULLVP) {
1492 		if (lvp != NULLVP) {
1493 			error = unionfs_check_rmdir(ap->a_vp, cnp->cn_cred, td);
1494 			if (error != 0)
1495 				return (error);
1496 		}
1497 		ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1498 		if (ump->um_whitemode == UNIONFS_WHITE_ALWAYS || lvp != NULLVP)
1499 			cnp->cn_flags |= DOWHITEOUT;
1500 		/*
1501 		 * The relookup path will need to relock the parent dvp and
1502 		 * possibly the vp as well.  Locking is expected to be done
1503 		 * in parent->child order; drop the lock on vp to avoid LOR
1504 		 * and potential recursion on vp's lock.
1505 		 * vp is expected to remain referenced during VOP_RMDIR(),
1506 		 * so vref/vrele should not be necessary here.
1507 		 */
1508 		VOP_UNLOCK(ap->a_vp);
1509 		VNPASS(vrefcnt(ap->a_vp) > 0, ap->a_vp);
1510 		error = unionfs_relookup_for_delete(ap->a_dvp, cnp, td);
1511 		vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1512 		/*
1513 		 * VOP_RMDIR is dispatched against udvp, so if uvp became
1514 		 * doomed while the lock was dropped above the target
1515 		 * filesystem may not be able to cope.
1516 		 */
1517 		if (error == 0 && VN_IS_DOOMED(uvp))
1518 			error = ENOENT;
1519 		if (error == 0)
1520 			error = VOP_RMDIR(udvp, uvp, cnp);
1521 	} else if (lvp != NULLVP)
1522 		error = unionfs_mkwhiteout(udvp, cnp, td,
1523 		    unp->un_path, unp->un_pathlen);
1524 
1525 	if (error == 0) {
1526 		cache_purge(ap->a_dvp);
1527 		cache_purge(ap->a_vp);
1528 	}
1529 
1530 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: leave (%d)\n", error);
1531 
1532 	return (error);
1533 }
1534 
1535 static int
1536 unionfs_symlink(struct vop_symlink_args *ap)
1537 {
1538 	struct unionfs_node *dunp;
1539 	struct componentname *cnp;
1540 	struct vnode   *udvp;
1541 	struct vnode   *uvp;
1542 	int		error;
1543 	int		lkflags;
1544 
1545 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: enter\n");
1546 
1547 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1548 
1549 	error = EROFS;
1550 	dunp = VTOUNIONFS(ap->a_dvp);
1551 	cnp = ap->a_cnp;
1552 	lkflags = cnp->cn_lkflags;
1553 	udvp = dunp->un_uppervp;
1554 
1555 	if (udvp != NULLVP) {
1556 		error = VOP_SYMLINK(udvp, &uvp, cnp, ap->a_vap, ap->a_target);
1557 		if (error == 0) {
1558 			VOP_UNLOCK(uvp);
1559 			cnp->cn_lkflags = LK_EXCLUSIVE;
1560 			error = unionfs_nodeget(ap->a_dvp->v_mount, uvp, NULLVP,
1561 			    ap->a_dvp, ap->a_vpp, cnp);
1562 			cnp->cn_lkflags = lkflags;
1563 			vrele(uvp);
1564 		}
1565 	}
1566 
1567 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: leave (%d)\n", error);
1568 
1569 	return (error);
1570 }
1571 
1572 static int
1573 unionfs_readdir(struct vop_readdir_args *ap)
1574 {
1575 	struct unionfs_node *unp;
1576 	struct unionfs_node_status *unsp;
1577 	struct uio     *uio;
1578 	struct vnode   *vp;
1579 	struct vnode   *uvp;
1580 	struct vnode   *lvp;
1581 	struct thread  *td;
1582 	struct vattr    va;
1583 
1584 	uint64_t	*cookies_bk;
1585 	int		error;
1586 	int		eofflag;
1587 	int		ncookies_bk;
1588 	int		uio_offset_bk;
1589 	enum unionfs_lkupgrade lkstatus;
1590 
1591 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: enter\n");
1592 
1593 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1594 
1595 	error = 0;
1596 	eofflag = 0;
1597 	uio_offset_bk = 0;
1598 	uio = ap->a_uio;
1599 	uvp = NULLVP;
1600 	lvp = NULLVP;
1601 	td = uio->uio_td;
1602 	ncookies_bk = 0;
1603 	cookies_bk = NULL;
1604 
1605 	vp = ap->a_vp;
1606 	if (vp->v_type != VDIR)
1607 		return (ENOTDIR);
1608 
1609 	/*
1610 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
1611 	 * or do anything else unionfs- specific.
1612 	 */
1613 	lkstatus = unionfs_upgrade_lock(vp);
1614 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
1615 		error = EBADF;
1616 	if (error == 0) {
1617 		unp = VTOUNIONFS(vp);
1618 		uvp = unp->un_uppervp;
1619 		lvp = unp->un_lowervp;
1620 		/* check the open count. unionfs needs open before readdir. */
1621 		unionfs_get_node_status(unp, td, &unsp);
1622 		if ((uvp != NULLVP && unsp->uns_upper_opencnt <= 0) ||
1623 			(lvp != NULLVP && unsp->uns_lower_opencnt <= 0)) {
1624 			unionfs_tryrem_node_status(unp, unsp);
1625 			error = EBADF;
1626 		}
1627 	}
1628 	unionfs_downgrade_lock(vp, lkstatus);
1629 	if (error != 0)
1630 		goto unionfs_readdir_exit;
1631 
1632 	/* check opaque */
1633 	if (uvp != NULLVP && lvp != NULLVP) {
1634 		if ((error = VOP_GETATTR(uvp, &va, ap->a_cred)) != 0)
1635 			goto unionfs_readdir_exit;
1636 		if (va.va_flags & OPAQUE)
1637 			lvp = NULLVP;
1638 	}
1639 
1640 	/* upper only */
1641 	if (uvp != NULLVP && lvp == NULLVP) {
1642 		error = VOP_READDIR(uvp, uio, ap->a_cred, ap->a_eofflag,
1643 		    ap->a_ncookies, ap->a_cookies);
1644 		unsp->uns_readdir_status = 0;
1645 
1646 		goto unionfs_readdir_exit;
1647 	}
1648 
1649 	/* lower only */
1650 	if (uvp == NULLVP && lvp != NULLVP) {
1651 		error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1652 		    ap->a_ncookies, ap->a_cookies);
1653 		unsp->uns_readdir_status = 2;
1654 
1655 		goto unionfs_readdir_exit;
1656 	}
1657 
1658 	/*
1659 	 * readdir upper and lower
1660 	 */
1661 	KASSERT(uvp != NULLVP, ("unionfs_readdir: null upper vp"));
1662 	KASSERT(lvp != NULLVP, ("unionfs_readdir: null lower vp"));
1663 	if (uio->uio_offset == 0)
1664 		unsp->uns_readdir_status = 0;
1665 
1666 	if (unsp->uns_readdir_status == 0) {
1667 		/* read upper */
1668 		error = VOP_READDIR(uvp, uio, ap->a_cred, &eofflag,
1669 				    ap->a_ncookies, ap->a_cookies);
1670 
1671 		if (error != 0 || eofflag == 0)
1672 			goto unionfs_readdir_exit;
1673 		unsp->uns_readdir_status = 1;
1674 
1675 		/*
1676 		 * UFS(and other FS) needs size of uio_resid larger than
1677 		 * DIRBLKSIZ.
1678 		 * size of DIRBLKSIZ equals DEV_BSIZE.
1679 		 * (see: ufs/ufs/ufs_vnops.c ufs_readdir func , ufs/ufs/dir.h)
1680 		 */
1681 		if (uio->uio_resid <= (uio->uio_resid & (DEV_BSIZE -1)))
1682 			goto unionfs_readdir_exit;
1683 
1684 		/*
1685 		 * Backup cookies.
1686 		 * It prepares to readdir in lower.
1687 		 */
1688 		if (ap->a_ncookies != NULL) {
1689 			ncookies_bk = *(ap->a_ncookies);
1690 			*(ap->a_ncookies) = 0;
1691 		}
1692 		if (ap->a_cookies != NULL) {
1693 			cookies_bk = *(ap->a_cookies);
1694 			*(ap->a_cookies) = NULL;
1695 		}
1696 	}
1697 
1698 	/* initialize for readdir in lower */
1699 	if (unsp->uns_readdir_status == 1) {
1700 		unsp->uns_readdir_status = 2;
1701 		/*
1702 		 * Backup uio_offset. See the comment after the
1703 		 * VOP_READDIR call on the lower layer.
1704 		 */
1705 		uio_offset_bk = uio->uio_offset;
1706 		uio->uio_offset = 0;
1707 	}
1708 
1709 	if (lvp == NULLVP) {
1710 		error = EBADF;
1711 		goto unionfs_readdir_exit;
1712 	}
1713 	/* read lower */
1714 	error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1715 			    ap->a_ncookies, ap->a_cookies);
1716 
1717 	/*
1718 	 * We can't return an uio_offset of 0: this would trigger an
1719 	 * infinite loop, because the next call to unionfs_readdir would
1720 	 * always restart with the upper layer (uio_offset == 0) and
1721 	 * always return some data.
1722 	 *
1723 	 * This happens when the lower layer root directory is removed.
1724 	 * (A root directory deleting of unionfs should not be permitted.
1725 	 *  But current VFS can not do it.)
1726 	 */
1727 	if (uio->uio_offset == 0)
1728 		uio->uio_offset = uio_offset_bk;
1729 
1730 	if (cookies_bk != NULL) {
1731 		/* merge cookies */
1732 		int		size;
1733 		uint64_t         *newcookies, *pos;
1734 
1735 		size = *(ap->a_ncookies) + ncookies_bk;
1736 		newcookies = (uint64_t *) malloc(size * sizeof(*newcookies),
1737 		    M_TEMP, M_WAITOK);
1738 		pos = newcookies;
1739 
1740 		memcpy(pos, cookies_bk, ncookies_bk * sizeof(*newcookies));
1741 		pos += ncookies_bk;
1742 		memcpy(pos, *(ap->a_cookies),
1743 		    *(ap->a_ncookies) * sizeof(*newcookies));
1744 		free(cookies_bk, M_TEMP);
1745 		free(*(ap->a_cookies), M_TEMP);
1746 		*(ap->a_ncookies) = size;
1747 		*(ap->a_cookies) = newcookies;
1748 	}
1749 
1750 unionfs_readdir_exit:
1751 	if (error != 0 && ap->a_eofflag != NULL)
1752 		*(ap->a_eofflag) = 1;
1753 
1754 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: leave (%d)\n", error);
1755 
1756 	return (error);
1757 }
1758 
1759 static int
1760 unionfs_readlink(struct vop_readlink_args *ap)
1761 {
1762 	struct unionfs_node *unp;
1763 	struct vnode   *vp;
1764 	int error;
1765 
1766 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: enter\n");
1767 
1768 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1769 
1770 	unp = VTOUNIONFS(ap->a_vp);
1771 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
1772 
1773 	error = VOP_READLINK(vp, ap->a_uio, ap->a_cred);
1774 
1775 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: leave (%d)\n", error);
1776 
1777 	return (error);
1778 }
1779 
1780 static int
1781 unionfs_getwritemount(struct vop_getwritemount_args *ap)
1782 {
1783 	struct unionfs_node *unp;
1784 	struct vnode   *uvp;
1785 	struct vnode   *vp, *ovp;
1786 	int		error;
1787 
1788 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: enter\n");
1789 
1790 	error = 0;
1791 	vp = ap->a_vp;
1792 	uvp = NULLVP;
1793 
1794 	VI_LOCK(vp);
1795 	unp = VTOUNIONFS(vp);
1796 	if (unp != NULL)
1797 		uvp = unp->un_uppervp;
1798 
1799 	/*
1800 	 * If our node has no upper vnode, check the parent directory.
1801 	 * We may be initiating a write operation that will produce a
1802 	 * new upper vnode through CoW.
1803 	 */
1804 	if (uvp == NULLVP && unp != NULL) {
1805 		ovp = vp;
1806 		vp = unp->un_dvp;
1807 		/*
1808 		 * Only the root vnode should have an empty parent, but it
1809 		 * should not have an empty uppervp, so we shouldn't get here.
1810 		 */
1811 		VNASSERT(vp != NULL, ovp, ("%s: NULL parent vnode", __func__));
1812 		VI_UNLOCK(ovp);
1813 		VI_LOCK(vp);
1814 		unp = VTOUNIONFS(vp);
1815 		if (unp != NULL)
1816 			uvp = unp->un_uppervp;
1817 		if (uvp == NULLVP)
1818 			error = EACCES;
1819 	}
1820 
1821 	if (uvp != NULLVP) {
1822 		vholdnz(uvp);
1823 		VI_UNLOCK(vp);
1824 		error = VOP_GETWRITEMOUNT(uvp, ap->a_mpp);
1825 		vdrop(uvp);
1826 	} else {
1827 		VI_UNLOCK(vp);
1828 		*(ap->a_mpp) = NULL;
1829 	}
1830 
1831 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: leave (%d)\n", error);
1832 
1833 	return (error);
1834 }
1835 
1836 static int
1837 unionfs_inactive(struct vop_inactive_args *ap)
1838 {
1839 	ap->a_vp->v_object = NULL;
1840 	vrecycle(ap->a_vp);
1841 	return (0);
1842 }
1843 
1844 static int
1845 unionfs_reclaim(struct vop_reclaim_args *ap)
1846 {
1847 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: enter\n"); */
1848 
1849 	unionfs_noderem(ap->a_vp);
1850 
1851 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: leave\n"); */
1852 
1853 	return (0);
1854 }
1855 
1856 static int
1857 unionfs_print(struct vop_print_args *ap)
1858 {
1859 	struct unionfs_node *unp;
1860 	/* struct unionfs_node_status *unsp; */
1861 
1862 	unp = VTOUNIONFS(ap->a_vp);
1863 	/* unionfs_get_node_status(unp, curthread, &unsp); */
1864 
1865 	printf("unionfs_vp=%p, uppervp=%p, lowervp=%p\n",
1866 	    ap->a_vp, unp->un_uppervp, unp->un_lowervp);
1867 	/*
1868 	printf("unionfs opencnt: uppervp=%d, lowervp=%d\n",
1869 	    unsp->uns_upper_opencnt, unsp->uns_lower_opencnt);
1870 	*/
1871 
1872 	if (unp->un_uppervp != NULLVP)
1873 		vn_printf(unp->un_uppervp, "unionfs: upper ");
1874 	if (unp->un_lowervp != NULLVP)
1875 		vn_printf(unp->un_lowervp, "unionfs: lower ");
1876 
1877 	return (0);
1878 }
1879 
1880 static int
1881 unionfs_get_llt_revlock(struct vnode *vp, int flags)
1882 {
1883 	int revlock;
1884 
1885 	revlock = 0;
1886 
1887 	switch (flags & LK_TYPE_MASK) {
1888 	case LK_SHARED:
1889 		if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
1890 			revlock = LK_UPGRADE;
1891 		else
1892 			revlock = LK_RELEASE;
1893 		break;
1894 	case LK_EXCLUSIVE:
1895 	case LK_UPGRADE:
1896 		revlock = LK_RELEASE;
1897 		break;
1898 	case LK_DOWNGRADE:
1899 		revlock = LK_UPGRADE;
1900 		break;
1901 	default:
1902 		break;
1903 	}
1904 
1905 	return (revlock);
1906 }
1907 
1908 /*
1909  * The state of an acquired lock is adjusted similarly to
1910  * the time of error generating.
1911  * flags: LK_RELEASE or LK_UPGRADE
1912  */
1913 static void
1914 unionfs_revlock(struct vnode *vp, int flags)
1915 {
1916 	if (flags & LK_RELEASE)
1917 		VOP_UNLOCK_FLAGS(vp, flags);
1918 	else {
1919 		/* UPGRADE */
1920 		if (vn_lock(vp, flags) != 0)
1921 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1922 	}
1923 }
1924 
1925 static int
1926 unionfs_lock(struct vop_lock1_args *ap)
1927 {
1928 	struct unionfs_node *unp;
1929 	struct vnode   *vp;
1930 	struct vnode   *uvp;
1931 	struct vnode   *lvp;
1932 	int		error;
1933 	int		flags;
1934 	int		revlock;
1935 	int		interlock;
1936 	int		uhold;
1937 
1938 	/*
1939 	 * TODO: rework the unionfs locking scheme.
1940 	 * It's not guaranteed to be safe to blindly lock two vnodes on
1941 	 * different mounts as is done here.  Further, the entanglement
1942 	 * of locking both vnodes with the various options that can be
1943 	 * passed to VOP_LOCK() makes this code hard to reason about.
1944 	 * Instead, consider locking only the upper vnode, or the lower
1945 	 * vnode is the upper is not present, and taking separate measures
1946 	 * to lock both vnodes in the few cases when that is needed.
1947 	 */
1948 	error = 0;
1949 	interlock = 1;
1950 	uhold = 0;
1951 	flags = ap->a_flags;
1952 	vp = ap->a_vp;
1953 
1954 	if (LK_RELEASE == (flags & LK_TYPE_MASK) || !(flags & LK_TYPE_MASK))
1955 		return (VOP_UNLOCK_FLAGS(vp, flags | LK_RELEASE));
1956 
1957 	if ((flags & LK_INTERLOCK) == 0)
1958 		VI_LOCK(vp);
1959 
1960 	unp = VTOUNIONFS(vp);
1961 	if (unp == NULL)
1962 		goto unionfs_lock_null_vnode;
1963 
1964 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1965 
1966 	lvp = unp->un_lowervp;
1967 	uvp = unp->un_uppervp;
1968 
1969 	if ((revlock = unionfs_get_llt_revlock(vp, flags)) == 0)
1970 		panic("unknown lock type: 0x%x", flags & LK_TYPE_MASK);
1971 
1972 	/*
1973 	 * During unmount, the root vnode lock may be taken recursively,
1974 	 * because it may share the same v_vnlock field as the vnode covered by
1975 	 * the unionfs mount.  The covered vnode is locked across VFS_UNMOUNT(),
1976 	 * and the same lock may be taken recursively here during vflush()
1977 	 * issued by unionfs_unmount().
1978 	 */
1979 	if ((flags & LK_TYPE_MASK) == LK_EXCLUSIVE &&
1980 	    (vp->v_vflag & VV_ROOT) != 0)
1981 		flags |= LK_CANRECURSE;
1982 
1983 	if (lvp != NULLVP) {
1984 		if (uvp != NULLVP && flags & LK_UPGRADE) {
1985 			/*
1986 			 * Share Lock is once released and a deadlock is
1987 			 * avoided.
1988 			 */
1989 			vholdnz(uvp);
1990 			uhold = 1;
1991 			VOP_UNLOCK(uvp);
1992 		}
1993 		VI_LOCK_FLAGS(lvp, MTX_DUPOK);
1994 		flags |= LK_INTERLOCK;
1995 		vholdl(lvp);
1996 
1997 		VI_UNLOCK(vp);
1998 		ap->a_flags &= ~LK_INTERLOCK;
1999 
2000 		error = VOP_LOCK(lvp, flags);
2001 
2002 		VI_LOCK(vp);
2003 		unp = VTOUNIONFS(vp);
2004 		if (unp == NULL) {
2005 			/* vnode is released. */
2006 			VI_UNLOCK(vp);
2007 			if (error == 0)
2008 				VOP_UNLOCK(lvp);
2009 			vdrop(lvp);
2010 			if (uhold != 0)
2011 				vdrop(uvp);
2012 			goto unionfs_lock_fallback;
2013 		}
2014 	}
2015 
2016 	if (error == 0 && uvp != NULLVP) {
2017 		if (uhold && flags & LK_UPGRADE) {
2018 			flags &= ~LK_TYPE_MASK;
2019 			flags |= LK_EXCLUSIVE;
2020 		}
2021 		VI_LOCK_FLAGS(uvp, MTX_DUPOK);
2022 		flags |= LK_INTERLOCK;
2023 		if (uhold == 0) {
2024 			vholdl(uvp);
2025 			uhold = 1;
2026 		}
2027 
2028 		VI_UNLOCK(vp);
2029 		ap->a_flags &= ~LK_INTERLOCK;
2030 
2031 		error = VOP_LOCK(uvp, flags);
2032 
2033 		VI_LOCK(vp);
2034 		unp = VTOUNIONFS(vp);
2035 		if (unp == NULL) {
2036 			/* vnode is released. */
2037 			VI_UNLOCK(vp);
2038 			if (error == 0)
2039 				VOP_UNLOCK(uvp);
2040 			vdrop(uvp);
2041 			if (lvp != NULLVP) {
2042 				VOP_UNLOCK(lvp);
2043 				vdrop(lvp);
2044 			}
2045 			goto unionfs_lock_fallback;
2046 		}
2047 		if (error != 0 && lvp != NULLVP) {
2048 			/* rollback */
2049 			VI_UNLOCK(vp);
2050 			unionfs_revlock(lvp, revlock);
2051 			interlock = 0;
2052 		}
2053 	}
2054 
2055 	if (interlock)
2056 		VI_UNLOCK(vp);
2057 	if (lvp != NULLVP)
2058 		vdrop(lvp);
2059 	if (uhold != 0)
2060 		vdrop(uvp);
2061 
2062 	return (error);
2063 
2064 unionfs_lock_null_vnode:
2065 	ap->a_flags |= LK_INTERLOCK;
2066 	return (vop_stdlock(ap));
2067 
2068 unionfs_lock_fallback:
2069 	/*
2070 	 * If we reach this point, we've discovered the unionfs vnode
2071 	 * has been reclaimed while the upper/lower vnode locks were
2072 	 * temporarily dropped.  Such temporary droppage may happen
2073 	 * during the course of an LK_UPGRADE operation itself, and in
2074 	 * that case LK_UPGRADE must be cleared as the unionfs vnode's
2075 	 * lock has been reset to point to the standard v_lock field,
2076 	 * which has not previously been held.
2077 	 */
2078 	if (flags & LK_UPGRADE) {
2079 		ap->a_flags &= ~LK_TYPE_MASK;
2080 		ap->a_flags |= LK_EXCLUSIVE;
2081 	}
2082 	return (vop_stdlock(ap));
2083 }
2084 
2085 static int
2086 unionfs_unlock(struct vop_unlock_args *ap)
2087 {
2088 	struct vnode   *vp;
2089 	struct vnode   *lvp;
2090 	struct vnode   *uvp;
2091 	struct unionfs_node *unp;
2092 	int		error;
2093 	int		uhold;
2094 
2095 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2096 
2097 	error = 0;
2098 	uhold = 0;
2099 	vp = ap->a_vp;
2100 
2101 	unp = VTOUNIONFS(vp);
2102 	if (unp == NULL)
2103 		goto unionfs_unlock_null_vnode;
2104 	lvp = unp->un_lowervp;
2105 	uvp = unp->un_uppervp;
2106 
2107 	if (lvp != NULLVP) {
2108 		vholdnz(lvp);
2109 		error = VOP_UNLOCK(lvp);
2110 	}
2111 
2112 	if (error == 0 && uvp != NULLVP) {
2113 		vholdnz(uvp);
2114 		uhold = 1;
2115 		error = VOP_UNLOCK(uvp);
2116 	}
2117 
2118 	if (lvp != NULLVP)
2119 		vdrop(lvp);
2120 	if (uhold != 0)
2121 		vdrop(uvp);
2122 
2123 	return error;
2124 
2125 unionfs_unlock_null_vnode:
2126 	return (vop_stdunlock(ap));
2127 }
2128 
2129 static int
2130 unionfs_pathconf(struct vop_pathconf_args *ap)
2131 {
2132 	struct unionfs_node *unp;
2133 	struct vnode   *vp;
2134 
2135 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2136 
2137 	unp = VTOUNIONFS(ap->a_vp);
2138 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2139 
2140 	return (VOP_PATHCONF(vp, ap->a_name, ap->a_retval));
2141 }
2142 
2143 static int
2144 unionfs_advlock(struct vop_advlock_args *ap)
2145 {
2146 	struct unionfs_node *unp;
2147 	struct unionfs_node_status *unsp;
2148 	struct vnode   *vp;
2149 	struct vnode   *uvp;
2150 	struct thread  *td;
2151 	int error;
2152 
2153 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: enter\n");
2154 
2155 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2156 
2157 	vp = ap->a_vp;
2158 	td = curthread;
2159 
2160 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2161 
2162 	unp = VTOUNIONFS(ap->a_vp);
2163 	uvp = unp->un_uppervp;
2164 
2165 	if (uvp == NULLVP) {
2166 		error = unionfs_copyfile(unp, 1, td->td_ucred, td);
2167 		if (error != 0)
2168 			goto unionfs_advlock_abort;
2169 		uvp = unp->un_uppervp;
2170 
2171 		unionfs_get_node_status(unp, td, &unsp);
2172 		if (unsp->uns_lower_opencnt > 0) {
2173 			/* try reopen the vnode */
2174 			error = VOP_OPEN(uvp, unsp->uns_lower_openmode,
2175 				td->td_ucred, td, NULL);
2176 			if (error)
2177 				goto unionfs_advlock_abort;
2178 			unsp->uns_upper_opencnt++;
2179 			VOP_CLOSE(unp->un_lowervp, unsp->uns_lower_openmode,
2180 			    td->td_ucred, td);
2181 			unsp->uns_lower_opencnt--;
2182 		} else
2183 			unionfs_tryrem_node_status(unp, unsp);
2184 	}
2185 
2186 	VOP_UNLOCK(vp);
2187 
2188 	error = VOP_ADVLOCK(uvp, ap->a_id, ap->a_op, ap->a_fl, ap->a_flags);
2189 
2190 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2191 
2192 	return error;
2193 
2194 unionfs_advlock_abort:
2195 	VOP_UNLOCK(vp);
2196 
2197 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2198 
2199 	return error;
2200 }
2201 
2202 static int
2203 unionfs_strategy(struct vop_strategy_args *ap)
2204 {
2205 	struct unionfs_node *unp;
2206 	struct vnode   *vp;
2207 
2208 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2209 
2210 	unp = VTOUNIONFS(ap->a_vp);
2211 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2212 
2213 #ifdef DIAGNOSTIC
2214 	if (vp == NULLVP)
2215 		panic("unionfs_strategy: nullvp");
2216 
2217 	if (ap->a_bp->b_iocmd == BIO_WRITE && vp == unp->un_lowervp)
2218 		panic("unionfs_strategy: writing to lowervp");
2219 #endif
2220 
2221 	return (VOP_STRATEGY(vp, ap->a_bp));
2222 }
2223 
2224 static int
2225 unionfs_getacl(struct vop_getacl_args *ap)
2226 {
2227 	struct unionfs_node *unp;
2228 	struct vnode   *vp;
2229 	int		error;
2230 
2231 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2232 
2233 	unp = VTOUNIONFS(ap->a_vp);
2234 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2235 
2236 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: enter\n");
2237 
2238 	error = VOP_GETACL(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2239 
2240 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: leave (%d)\n", error);
2241 
2242 	return (error);
2243 }
2244 
2245 static int
2246 unionfs_setacl(struct vop_setacl_args *ap)
2247 {
2248 	struct unionfs_node *unp;
2249 	struct vnode   *uvp;
2250 	struct vnode   *lvp;
2251 	struct thread  *td;
2252 	int		error;
2253 
2254 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: enter\n");
2255 
2256 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2257 
2258 	error = EROFS;
2259 	unp = VTOUNIONFS(ap->a_vp);
2260 	uvp = unp->un_uppervp;
2261 	lvp = unp->un_lowervp;
2262 	td = ap->a_td;
2263 
2264 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2265 		return (EROFS);
2266 
2267 	if (uvp == NULLVP && lvp->v_type == VREG) {
2268 		if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0)
2269 			return (error);
2270 		uvp = unp->un_uppervp;
2271 	}
2272 
2273 	if (uvp != NULLVP)
2274 		error = VOP_SETACL(uvp, ap->a_type, ap->a_aclp, ap->a_cred, td);
2275 
2276 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: leave (%d)\n", error);
2277 
2278 	return (error);
2279 }
2280 
2281 static int
2282 unionfs_aclcheck(struct vop_aclcheck_args *ap)
2283 {
2284 	struct unionfs_node *unp;
2285 	struct vnode   *vp;
2286 	int		error;
2287 
2288 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: enter\n");
2289 
2290 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2291 
2292 	unp = VTOUNIONFS(ap->a_vp);
2293 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2294 
2295 	error = VOP_ACLCHECK(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2296 
2297 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: leave (%d)\n", error);
2298 
2299 	return (error);
2300 }
2301 
2302 static int
2303 unionfs_openextattr(struct vop_openextattr_args *ap)
2304 {
2305 	struct unionfs_node *unp;
2306 	struct vnode   *vp;
2307 	struct vnode   *tvp;
2308 	int		error;
2309 
2310 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2311 
2312 	vp = ap->a_vp;
2313 	unp = VTOUNIONFS(vp);
2314 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2315 
2316 	if ((tvp == unp->un_uppervp && (unp->un_flag & UNIONFS_OPENEXTU)) ||
2317 	    (tvp == unp->un_lowervp && (unp->un_flag & UNIONFS_OPENEXTL)))
2318 		return (EBUSY);
2319 
2320 	error = VOP_OPENEXTATTR(tvp, ap->a_cred, ap->a_td);
2321 
2322 	if (error == 0) {
2323 		if (vn_lock(vp, LK_UPGRADE) != 0)
2324 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2325 		if (!VN_IS_DOOMED(vp)) {
2326 			if (tvp == unp->un_uppervp)
2327 				unp->un_flag |= UNIONFS_OPENEXTU;
2328 			else
2329 				unp->un_flag |= UNIONFS_OPENEXTL;
2330 		}
2331 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2332 	}
2333 
2334 	return (error);
2335 }
2336 
2337 static int
2338 unionfs_closeextattr(struct vop_closeextattr_args *ap)
2339 {
2340 	struct unionfs_node *unp;
2341 	struct vnode   *vp;
2342 	struct vnode   *tvp;
2343 	int		error;
2344 
2345 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2346 
2347 	vp = ap->a_vp;
2348 	unp = VTOUNIONFS(vp);
2349 	tvp = NULLVP;
2350 
2351 	if (unp->un_flag & UNIONFS_OPENEXTU)
2352 		tvp = unp->un_uppervp;
2353 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2354 		tvp = unp->un_lowervp;
2355 
2356 	if (tvp == NULLVP)
2357 		return (EOPNOTSUPP);
2358 
2359 	error = VOP_CLOSEEXTATTR(tvp, ap->a_commit, ap->a_cred, ap->a_td);
2360 
2361 	if (error == 0) {
2362 		if (vn_lock(vp, LK_UPGRADE) != 0)
2363 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2364 		if (!VN_IS_DOOMED(vp)) {
2365 			if (tvp == unp->un_uppervp)
2366 				unp->un_flag &= ~UNIONFS_OPENEXTU;
2367 			else
2368 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2369 		}
2370 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2371 	}
2372 
2373 	return (error);
2374 }
2375 
2376 static int
2377 unionfs_getextattr(struct vop_getextattr_args *ap)
2378 {
2379 	struct unionfs_node *unp;
2380 	struct vnode   *vp;
2381 
2382 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2383 
2384 	unp = VTOUNIONFS(ap->a_vp);
2385 	vp = NULLVP;
2386 
2387 	if (unp->un_flag & UNIONFS_OPENEXTU)
2388 		vp = unp->un_uppervp;
2389 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2390 		vp = unp->un_lowervp;
2391 
2392 	if (vp == NULLVP)
2393 		return (EOPNOTSUPP);
2394 
2395 	return (VOP_GETEXTATTR(vp, ap->a_attrnamespace, ap->a_name,
2396 	    ap->a_uio, ap->a_size, ap->a_cred, ap->a_td));
2397 }
2398 
2399 static int
2400 unionfs_setextattr(struct vop_setextattr_args *ap)
2401 {
2402 	struct unionfs_node *unp;
2403 	struct vnode   *uvp;
2404 	struct vnode   *lvp;
2405 	struct vnode   *ovp;
2406 	struct ucred   *cred;
2407 	struct thread  *td;
2408 	int		error;
2409 
2410 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2411 
2412 	error = EROFS;
2413 	unp = VTOUNIONFS(ap->a_vp);
2414 	uvp = unp->un_uppervp;
2415 	lvp = unp->un_lowervp;
2416 	ovp = NULLVP;
2417 	cred = ap->a_cred;
2418 	td = ap->a_td;
2419 
2420 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: enter (un_flag=%x)\n",
2421 	    unp->un_flag);
2422 
2423 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2424 		return (EROFS);
2425 
2426 	if (unp->un_flag & UNIONFS_OPENEXTU)
2427 		ovp = unp->un_uppervp;
2428 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2429 		ovp = unp->un_lowervp;
2430 
2431 	if (ovp == NULLVP)
2432 		return (EOPNOTSUPP);
2433 
2434 	if (ovp == lvp && lvp->v_type == VREG) {
2435 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2436 		if (uvp == NULLVP &&
2437 		    (error = unionfs_copyfile(unp, 1, cred, td)) != 0) {
2438 unionfs_setextattr_reopen:
2439 			if ((unp->un_flag & UNIONFS_OPENEXTL) &&
2440 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2441 #ifdef DIAGNOSTIC
2442 				panic("unionfs: VOP_OPENEXTATTR failed");
2443 #endif
2444 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2445 			}
2446 			goto unionfs_setextattr_abort;
2447 		}
2448 		uvp = unp->un_uppervp;
2449 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2450 			goto unionfs_setextattr_reopen;
2451 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2452 		unp->un_flag |= UNIONFS_OPENEXTU;
2453 		ovp = uvp;
2454 	}
2455 
2456 	if (ovp == uvp)
2457 		error = VOP_SETEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2458 		    ap->a_uio, cred, td);
2459 
2460 unionfs_setextattr_abort:
2461 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: leave (%d)\n", error);
2462 
2463 	return (error);
2464 }
2465 
2466 static int
2467 unionfs_listextattr(struct vop_listextattr_args *ap)
2468 {
2469 	struct unionfs_node *unp;
2470 	struct vnode *vp;
2471 
2472 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2473 
2474 	unp = VTOUNIONFS(ap->a_vp);
2475 	vp = NULLVP;
2476 
2477 	if (unp->un_flag & UNIONFS_OPENEXTU)
2478 		vp = unp->un_uppervp;
2479 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2480 		vp = unp->un_lowervp;
2481 
2482 	if (vp == NULLVP)
2483 		return (EOPNOTSUPP);
2484 
2485 	return (VOP_LISTEXTATTR(vp, ap->a_attrnamespace, ap->a_uio,
2486 	    ap->a_size, ap->a_cred, ap->a_td));
2487 }
2488 
2489 static int
2490 unionfs_deleteextattr(struct vop_deleteextattr_args *ap)
2491 {
2492 	struct unionfs_node *unp;
2493 	struct vnode   *uvp;
2494 	struct vnode   *lvp;
2495 	struct vnode   *ovp;
2496 	struct ucred   *cred;
2497 	struct thread  *td;
2498 	int		error;
2499 
2500 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2501 
2502 	error = EROFS;
2503 	unp = VTOUNIONFS(ap->a_vp);
2504 	uvp = unp->un_uppervp;
2505 	lvp = unp->un_lowervp;
2506 	ovp = NULLVP;
2507 	cred = ap->a_cred;
2508 	td = ap->a_td;
2509 
2510 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: enter (un_flag=%x)\n",
2511 	    unp->un_flag);
2512 
2513 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2514 		return (EROFS);
2515 
2516 	if (unp->un_flag & UNIONFS_OPENEXTU)
2517 		ovp = unp->un_uppervp;
2518 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2519 		ovp = unp->un_lowervp;
2520 
2521 	if (ovp == NULLVP)
2522 		return (EOPNOTSUPP);
2523 
2524 	if (ovp == lvp && lvp->v_type == VREG) {
2525 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2526 		if (uvp == NULLVP &&
2527 		    (error = unionfs_copyfile(unp, 1, cred, td)) != 0) {
2528 unionfs_deleteextattr_reopen:
2529 			if ((unp->un_flag & UNIONFS_OPENEXTL) &&
2530 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2531 #ifdef DIAGNOSTIC
2532 				panic("unionfs: VOP_OPENEXTATTR failed");
2533 #endif
2534 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2535 			}
2536 			goto unionfs_deleteextattr_abort;
2537 		}
2538 		uvp = unp->un_uppervp;
2539 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2540 			goto unionfs_deleteextattr_reopen;
2541 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2542 		unp->un_flag |= UNIONFS_OPENEXTU;
2543 		ovp = uvp;
2544 	}
2545 
2546 	if (ovp == uvp)
2547 		error = VOP_DELETEEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2548 		    ap->a_cred, ap->a_td);
2549 
2550 unionfs_deleteextattr_abort:
2551 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: leave (%d)\n", error);
2552 
2553 	return (error);
2554 }
2555 
2556 static int
2557 unionfs_setlabel(struct vop_setlabel_args *ap)
2558 {
2559 	struct unionfs_node *unp;
2560 	struct vnode   *uvp;
2561 	struct vnode   *lvp;
2562 	struct thread  *td;
2563 	int		error;
2564 
2565 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: enter\n");
2566 
2567 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2568 
2569 	error = EROFS;
2570 	unp = VTOUNIONFS(ap->a_vp);
2571 	uvp = unp->un_uppervp;
2572 	lvp = unp->un_lowervp;
2573 	td = ap->a_td;
2574 
2575 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2576 		return (EROFS);
2577 
2578 	if (uvp == NULLVP && lvp->v_type == VREG) {
2579 		if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0)
2580 			return (error);
2581 		uvp = unp->un_uppervp;
2582 	}
2583 
2584 	if (uvp != NULLVP)
2585 		error = VOP_SETLABEL(uvp, ap->a_label, ap->a_cred, td);
2586 
2587 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: leave (%d)\n", error);
2588 
2589 	return (error);
2590 }
2591 
2592 static int
2593 unionfs_vptofh(struct vop_vptofh_args *ap)
2594 {
2595 	return (EOPNOTSUPP);
2596 }
2597 
2598 static int
2599 unionfs_add_writecount(struct vop_add_writecount_args *ap)
2600 {
2601 	struct vnode *tvp, *vp;
2602 	struct unionfs_node *unp;
2603 	int error, writerefs __diagused;
2604 
2605 	vp = ap->a_vp;
2606 	unp = VTOUNIONFS(vp);
2607 	tvp = unp->un_uppervp;
2608 	KASSERT(tvp != NULL,
2609 	    ("%s: adding write ref without upper vnode", __func__));
2610 	error = VOP_ADD_WRITECOUNT(tvp, ap->a_inc);
2611 	if (error != 0)
2612 		return (error);
2613 	/*
2614 	 * We need to track the write refs we've passed to the underlying
2615 	 * vnodes so that we can undo them in case we are forcibly unmounted.
2616 	 */
2617 	writerefs = atomic_fetchadd_int(&vp->v_writecount, ap->a_inc);
2618 	/* text refs are bypassed to lowervp */
2619 	VNASSERT(writerefs >= 0, vp,
2620 	    ("%s: invalid write count %d", __func__, writerefs));
2621 	VNASSERT(writerefs + ap->a_inc >= 0, vp,
2622 	    ("%s: invalid write count inc %d + %d", __func__,
2623 	    writerefs, ap->a_inc));
2624 	return (0);
2625 }
2626 
2627 static int
2628 unionfs_vput_pair(struct vop_vput_pair_args *ap)
2629 {
2630 	struct mount *mp;
2631 	struct vnode *dvp, *vp, **vpp, *lvp, *ldvp, *uvp, *udvp, *tempvp;
2632 	struct unionfs_node *dunp, *unp;
2633 	int error, res;
2634 
2635 	dvp = ap->a_dvp;
2636 	vpp = ap->a_vpp;
2637 	vp = NULLVP;
2638 	lvp = NULLVP;
2639 	uvp = NULLVP;
2640 	unp = NULL;
2641 
2642 	dunp = VTOUNIONFS(dvp);
2643 	udvp = dunp->un_uppervp;
2644 	ldvp = dunp->un_lowervp;
2645 
2646 	/*
2647 	 * Underlying vnodes should be locked because the encompassing unionfs
2648 	 * node is locked, but will not be referenced, as the reference will
2649 	 * only be on the unionfs node.  Reference them now so that the vput()s
2650 	 * performed by VOP_VPUT_PAIR() will have a reference to drop.
2651 	 */
2652 	if (udvp != NULLVP)
2653 		vref(udvp);
2654 	if (ldvp != NULLVP)
2655 		vref(ldvp);
2656 
2657 	if (vpp != NULL)
2658 		vp = *vpp;
2659 
2660 	if (vp != NULLVP) {
2661 		unp = VTOUNIONFS(vp);
2662 		uvp = unp->un_uppervp;
2663 		lvp = unp->un_lowervp;
2664 		if (uvp != NULLVP)
2665 			vref(uvp);
2666 		if (lvp != NULLVP)
2667 			vref(lvp);
2668 
2669 		/*
2670 		 * If we're being asked to return a locked child vnode, then
2671 		 * we may need to create a replacement vnode in case the
2672 		 * original is reclaimed while the lock is dropped.  In that
2673 		 * case we'll need to ensure the mount and the underlying
2674 		 * vnodes aren't also recycled during that window.
2675 		 */
2676 		if (!ap->a_unlock_vp) {
2677 			vhold(vp);
2678 			if (uvp != NULLVP)
2679 				vhold(uvp);
2680 			if (lvp != NULLVP)
2681 				vhold(lvp);
2682 			mp = vp->v_mount;
2683 			vfs_ref(mp);
2684 		}
2685 	}
2686 
2687 	/*
2688 	 * TODO: Because unionfs_lock() locks both the lower and upper vnodes
2689 	 * (if available), we must also call VOP_VPUT_PAIR() on both the lower
2690 	 * and upper parent/child pairs.  If unionfs_lock() is reworked to lock
2691 	 * only a single vnode, this code will need to change to also only
2692 	 * operate on one vnode pair.
2693 	 */
2694 	ASSERT_VOP_LOCKED(ldvp, __func__);
2695 	ASSERT_VOP_LOCKED(udvp, __func__);
2696 	ASSERT_VOP_LOCKED(lvp, __func__);
2697 	ASSERT_VOP_LOCKED(uvp, __func__);
2698 
2699 	KASSERT(lvp == NULLVP || ldvp != NULLVP,
2700 	    ("%s: NULL ldvp with non-NULL lvp", __func__));
2701 	if (ldvp != NULLVP)
2702 		res = VOP_VPUT_PAIR(ldvp, lvp != NULLVP ? &lvp : NULL, true);
2703 	KASSERT(uvp == NULLVP || udvp != NULLVP,
2704 	    ("%s: NULL udvp with non-NULL uvp", __func__));
2705 	if (udvp != NULLVP)
2706 		res = VOP_VPUT_PAIR(udvp, uvp != NULLVP ? &uvp : NULL, true);
2707 
2708 	ASSERT_VOP_UNLOCKED(ldvp, __func__);
2709 	ASSERT_VOP_UNLOCKED(udvp, __func__);
2710 	ASSERT_VOP_UNLOCKED(lvp, __func__);
2711 	ASSERT_VOP_UNLOCKED(uvp, __func__);
2712 
2713 	/*
2714 	 * VOP_VPUT_PAIR() dropped the references we added to the underlying
2715 	 * vnodes, now drop the caller's reference to the unionfs vnodes.
2716 	 */
2717 	if (vp != NULLVP && ap->a_unlock_vp)
2718 		vrele(vp);
2719 	vrele(dvp);
2720 
2721 	if (vp == NULLVP || ap->a_unlock_vp)
2722 		return (res);
2723 
2724 	/*
2725 	 * We're being asked to return a locked vnode.  At this point, the
2726 	 * underlying vnodes have been unlocked, so vp may have been reclaimed.
2727 	 */
2728 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2729 	if (vp->v_data == NULL && vfs_busy(mp, MBF_NOWAIT) == 0) {
2730 		vput(vp);
2731 		error = unionfs_nodeget(mp, uvp, lvp, dvp, &tempvp, NULL);
2732 		if (error == 0) {
2733 			vn_lock(tempvp, LK_EXCLUSIVE | LK_RETRY);
2734 			*vpp = tempvp;
2735 		} else
2736 			vget(vp, LK_EXCLUSIVE | LK_RETRY);
2737 		vfs_unbusy(mp);
2738 	}
2739 	if (lvp != NULLVP)
2740 		vdrop(lvp);
2741 	if (uvp != NULLVP)
2742 		vdrop(uvp);
2743 	vdrop(vp);
2744 	vfs_rel(mp);
2745 
2746 	return (res);
2747 }
2748 
2749 static int
2750 unionfs_set_text(struct vop_set_text_args *ap)
2751 {
2752 	struct vnode *tvp;
2753 	struct unionfs_node *unp;
2754 	int error;
2755 
2756 	/*
2757 	 * We assume text refs are managed against lvp/uvp through the
2758 	 * executable mapping backed by its VM object.  We therefore don't
2759 	 * need to track leased text refs in the case of a forcible unmount.
2760 	 */
2761 	unp = VTOUNIONFS(ap->a_vp);
2762 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2763 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2764 	error = VOP_SET_TEXT(tvp);
2765 	return (error);
2766 }
2767 
2768 static int
2769 unionfs_unset_text(struct vop_unset_text_args *ap)
2770 {
2771 	struct vnode *tvp;
2772 	struct unionfs_node *unp;
2773 
2774 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2775 	unp = VTOUNIONFS(ap->a_vp);
2776 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2777 	VOP_UNSET_TEXT_CHECKED(tvp);
2778 	return (0);
2779 }
2780 
2781 struct vop_vector unionfs_vnodeops = {
2782 	.vop_default =		&default_vnodeops,
2783 
2784 	.vop_access =		unionfs_access,
2785 	.vop_aclcheck =		unionfs_aclcheck,
2786 	.vop_advlock =		unionfs_advlock,
2787 	.vop_bmap =		VOP_EOPNOTSUPP,
2788 	.vop_cachedlookup =	unionfs_lookup,
2789 	.vop_close =		unionfs_close,
2790 	.vop_closeextattr =	unionfs_closeextattr,
2791 	.vop_create =		unionfs_create,
2792 	.vop_deleteextattr =	unionfs_deleteextattr,
2793 	.vop_fsync =		unionfs_fsync,
2794 	.vop_getacl =		unionfs_getacl,
2795 	.vop_getattr =		unionfs_getattr,
2796 	.vop_getextattr =	unionfs_getextattr,
2797 	.vop_getwritemount =	unionfs_getwritemount,
2798 	.vop_inactive =		unionfs_inactive,
2799 	.vop_need_inactive =	vop_stdneed_inactive,
2800 	.vop_islocked =		vop_stdislocked,
2801 	.vop_ioctl =		unionfs_ioctl,
2802 	.vop_link =		unionfs_link,
2803 	.vop_listextattr =	unionfs_listextattr,
2804 	.vop_lock1 =		unionfs_lock,
2805 	.vop_lookup =		vfs_cache_lookup,
2806 	.vop_mkdir =		unionfs_mkdir,
2807 	.vop_mknod =		unionfs_mknod,
2808 	.vop_open =		unionfs_open,
2809 	.vop_openextattr =	unionfs_openextattr,
2810 	.vop_pathconf =		unionfs_pathconf,
2811 	.vop_poll =		unionfs_poll,
2812 	.vop_print =		unionfs_print,
2813 	.vop_read =		unionfs_read,
2814 	.vop_readdir =		unionfs_readdir,
2815 	.vop_readlink =		unionfs_readlink,
2816 	.vop_reclaim =		unionfs_reclaim,
2817 	.vop_remove =		unionfs_remove,
2818 	.vop_rename =		unionfs_rename,
2819 	.vop_rmdir =		unionfs_rmdir,
2820 	.vop_setacl =		unionfs_setacl,
2821 	.vop_setattr =		unionfs_setattr,
2822 	.vop_setextattr =	unionfs_setextattr,
2823 	.vop_setlabel =		unionfs_setlabel,
2824 	.vop_strategy =		unionfs_strategy,
2825 	.vop_symlink =		unionfs_symlink,
2826 	.vop_unlock =		unionfs_unlock,
2827 	.vop_whiteout =		unionfs_whiteout,
2828 	.vop_write =		unionfs_write,
2829 	.vop_vptofh =		unionfs_vptofh,
2830 	.vop_add_writecount =	unionfs_add_writecount,
2831 	.vop_vput_pair =	unionfs_vput_pair,
2832 	.vop_set_text =		unionfs_set_text,
2833 	.vop_unset_text = 	unionfs_unset_text,
2834 };
2835 VFS_VOP_VECTOR_REGISTER(unionfs_vnodeops);
2836