xref: /freebsd/sys/fs/unionfs/union_vnops.c (revision a64729f5077d77e13b9497cb33ecb3c82e606ee8)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1992, 1993, 1994, 1995 Jan-Simon Pendry.
5  * Copyright (c) 1992, 1993, 1994, 1995
6  *      The Regents of the University of California.
7  * Copyright (c) 2005, 2006, 2012 Masanori Ozawa <ozawa@ongs.co.jp>, ONGS Inc.
8  * Copyright (c) 2006, 2012 Daichi Goto <daichi@freebsd.org>
9  * All rights reserved.
10  *
11  * This code is derived from software contributed to Berkeley by
12  * Jan-Simon Pendry.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions
16  * are met:
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  * 3. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  *
38  */
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/conf.h>
43 #include <sys/kernel.h>
44 #include <sys/lock.h>
45 #include <sys/malloc.h>
46 #include <sys/mount.h>
47 #include <sys/mutex.h>
48 #include <sys/namei.h>
49 #include <sys/sysctl.h>
50 #include <sys/vnode.h>
51 #include <sys/kdb.h>
52 #include <sys/fcntl.h>
53 #include <sys/stat.h>
54 #include <sys/dirent.h>
55 #include <sys/proc.h>
56 #include <sys/bio.h>
57 #include <sys/buf.h>
58 
59 #include <fs/unionfs/union.h>
60 
61 #include <machine/atomic.h>
62 
63 #include <vm/vm.h>
64 #include <vm/vm_extern.h>
65 #include <vm/vm_object.h>
66 #include <vm/vnode_pager.h>
67 
68 #if 0
69 #define UNIONFS_INTERNAL_DEBUG(msg, args...)    printf(msg, ## args)
70 #define UNIONFS_IDBG_RENAME
71 #else
72 #define UNIONFS_INTERNAL_DEBUG(msg, args...)
73 #endif
74 
75 #define KASSERT_UNIONFS_VNODE(vp) \
76 	VNASSERT(((vp)->v_op == &unionfs_vnodeops), vp, \
77 	    ("%s: non-unionfs vnode", __func__))
78 
79 static int
80 unionfs_lookup(struct vop_cachedlookup_args *ap)
81 {
82 	struct unionfs_node *dunp, *unp;
83 	struct vnode   *dvp, *udvp, *ldvp, *vp, *uvp, *lvp, *dtmpvp;
84 	struct vattr	va;
85 	struct componentname *cnp;
86 	struct thread  *td;
87 	u_long		nameiop;
88 	u_long		cnflags;
89 	int		lockflag;
90 	int		lkflags;
91 	int		error, uerror, lerror;
92 
93 	lockflag = 0;
94 	error = uerror = lerror = ENOENT;
95 	cnp = ap->a_cnp;
96 	nameiop = cnp->cn_nameiop;
97 	cnflags = cnp->cn_flags;
98 	dvp = ap->a_dvp;
99 	dunp = VTOUNIONFS(dvp);
100 	udvp = dunp->un_uppervp;
101 	ldvp = dunp->un_lowervp;
102 	vp = uvp = lvp = NULLVP;
103 	td = curthread;
104 	*(ap->a_vpp) = NULLVP;
105 
106 	UNIONFS_INTERNAL_DEBUG(
107 	    "unionfs_lookup: enter: nameiop=%ld, flags=%lx, path=%s\n",
108 	    nameiop, cnflags, cnp->cn_nameptr);
109 
110 	if (dvp->v_type != VDIR)
111 		return (ENOTDIR);
112 
113 	/*
114 	 * If read-only and op is not LOOKUP, will return EROFS.
115 	 */
116 	if ((cnflags & ISLASTCN) &&
117 	    (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
118 	    LOOKUP != nameiop)
119 		return (EROFS);
120 
121 	/*
122 	 * Note that a lookup is in-flight, and block if another lookup
123 	 * is already in-flight against dvp.  This is done because we may
124 	 * end up dropping dvp's lock to look up a lower vnode or to create
125 	 * a shadow directory, opening up the possibility of parallel lookups
126 	 * against the same directory creating duplicate unionfs vnodes for
127 	 * the same file(s).  Note that if this function encounters an
128 	 * in-progress lookup for the directory, it will block until the
129 	 * lookup is complete and then return ERELOOKUP to allow any
130 	 * existing unionfs vnode to be loaded from the VFS cache.
131 	 * This is really a hack; filesystems that support MNTK_LOOKUP_SHARED
132 	 * (which unionfs currently doesn't) seem to deal with this by using
133 	 * the vfs_hash_* functions to manage a per-mount vnode cache keyed
134 	 * by the inode number (or some roughly equivalent unique ID
135 	 * usually assocated with the storage medium).  It may make sense
136 	 * for unionfs to adopt something similar as a replacement for its
137 	 * current half-baked directory-only cache implementation, particularly
138 	 * if we want to support MNTK_LOOKUP_SHARED here.
139 	 */
140 	error = unionfs_set_in_progress_flag(dvp, UNIONFS_LOOKUP_IN_PROGRESS);
141 	if (error != 0)
142 		return (error);
143 	/*
144 	 * lookup dotdot
145 	 */
146 	if (cnflags & ISDOTDOT) {
147 		if (LOOKUP != nameiop && udvp == NULLVP) {
148 			error = EROFS;
149 			goto unionfs_lookup_return;
150 		}
151 
152 		if (udvp != NULLVP)
153 			dtmpvp = udvp;
154 		else
155 			dtmpvp = ldvp;
156 
157 		unionfs_forward_vop_start(dtmpvp, &lkflags);
158 		error = VOP_LOOKUP(dtmpvp, &vp, cnp);
159 		unionfs_forward_vop_finish(dvp, dtmpvp, lkflags);
160 
161 		/*
162 		 * Drop the lock and reference on vp.  If the lookup was
163 		 * successful, we'll either need to exchange vp's lock and
164 		 * reference for the unionfs parent vnode's lock and
165 		 * reference, or (if dvp was reclaimed) we'll need to drop
166 		 * vp's lock and reference to return early.
167 		 */
168 		if (vp != NULLVP)
169 			vput(vp);
170 		dunp = VTOUNIONFS(dvp);
171 		if (error == 0 && dunp == NULL)
172 			error = ENOENT;
173 
174 		if (error == 0) {
175 			dtmpvp = dunp->un_dvp;
176 			vref(dtmpvp);
177 			VOP_UNLOCK(dvp);
178 			*(ap->a_vpp) = dtmpvp;
179 
180 			vn_lock(dtmpvp, cnp->cn_lkflags | LK_RETRY);
181 
182 			if (VN_IS_DOOMED(dtmpvp)) {
183 				vput(dtmpvp);
184 				*(ap->a_vpp) = NULLVP;
185 				error = ENOENT;
186 			}
187 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
188 		}
189 
190 		goto unionfs_lookup_cleanup;
191 	}
192 
193 	/*
194 	 * Lookup lower layer.  We do this before looking up the the upper
195 	 * layer, as we may drop the upper parent directory's lock, and we
196 	 * want to ensure the upper parent remains locked from the point of
197 	 * lookup through any ensuing VOP that may require it to be locked.
198 	 * The cost of this is that we may end up performing an unnecessary
199 	 * lower layer lookup if a whiteout is present in the upper layer.
200 	 */
201 	if (ldvp != NULLVP && !(cnflags & DOWHITEOUT)) {
202 		struct componentname lcn;
203 		bool is_dot;
204 
205 		if (udvp != NULLVP) {
206 			vref(ldvp);
207 			VOP_UNLOCK(dvp);
208 			vn_lock(ldvp, LK_EXCLUSIVE | LK_RETRY);
209 		}
210 
211 		lcn = *cnp;
212 		/* always op is LOOKUP */
213 		lcn.cn_nameiop = LOOKUP;
214 		lcn.cn_flags = cnflags;
215 		is_dot = false;
216 
217 		if (udvp == NULLVP)
218 			unionfs_forward_vop_start(ldvp, &lkflags);
219 		lerror = VOP_LOOKUP(ldvp, &lvp, &lcn);
220 		if (udvp == NULLVP &&
221 		    unionfs_forward_vop_finish(dvp, ldvp, lkflags)) {
222 			if (lvp != NULLVP)
223 				VOP_UNLOCK(lvp);
224 			error =  ENOENT;
225 			goto unionfs_lookup_cleanup;
226 		}
227 
228 		if (udvp == NULLVP)
229 			cnp->cn_flags = lcn.cn_flags;
230 
231 		if (lerror == 0) {
232 			if (ldvp == lvp) {	/* is dot */
233 				vrele(lvp);
234 				*(ap->a_vpp) = dvp;
235 				vref(dvp);
236 				is_dot = true;
237 				error = lerror;
238 			} else if (lvp != NULLVP)
239 				VOP_UNLOCK(lvp);
240 		}
241 
242 		if (udvp != NULLVP) {
243 			vput(ldvp);
244 			vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY);
245 			if (VN_IS_DOOMED(dvp))
246 				error = ENOENT;
247 		}
248 		if (is_dot)
249 			goto unionfs_lookup_return;
250 		else if (error != 0)
251 			goto unionfs_lookup_cleanup;
252 	}
253 	/*
254 	 * lookup upper layer
255 	 */
256 	if (udvp != NULLVP) {
257 		bool iswhiteout = false;
258 
259 		unionfs_forward_vop_start(udvp, &lkflags);
260 		uerror = VOP_LOOKUP(udvp, &uvp, cnp);
261 		if (unionfs_forward_vop_finish(dvp, udvp, lkflags)) {
262 			if (uvp != NULLVP)
263 				VOP_UNLOCK(uvp);
264 			error = ENOENT;
265 			goto unionfs_lookup_cleanup;
266 		}
267 
268 		if (uerror == 0) {
269 			if (udvp == uvp) {	/* is dot */
270 				if (lvp != NULLVP)
271 					vrele(lvp);
272 				vrele(uvp);
273 				*(ap->a_vpp) = dvp;
274 				vref(dvp);
275 
276 				error = uerror;
277 				goto unionfs_lookup_return;
278 			} else if (uvp != NULLVP)
279 				VOP_UNLOCK(uvp);
280 		}
281 
282 		/* check whiteout */
283 		if ((uerror == ENOENT || uerror == EJUSTRETURN) &&
284 		    (cnp->cn_flags & ISWHITEOUT))
285 			iswhiteout = true;
286 		else if (VOP_GETATTR(udvp, &va, cnp->cn_cred) == 0 &&
287 		    (va.va_flags & OPAQUE))
288 			iswhiteout = true;
289 
290 		if (iswhiteout && lvp != NULLVP) {
291 			vrele(lvp);
292 			lvp = NULLVP;
293 		}
294 
295 #if 0
296 		UNIONFS_INTERNAL_DEBUG(
297 		    "unionfs_lookup: debug: whiteout=%d, path=%s\n",
298 		    iswhiteout, cnp->cn_nameptr);
299 #endif
300 	}
301 
302 	/*
303 	 * check lookup result
304 	 */
305 	if (uvp == NULLVP && lvp == NULLVP) {
306 		error = (udvp != NULLVP ? uerror : lerror);
307 		goto unionfs_lookup_return;
308 	}
309 
310 	/*
311 	 * check vnode type
312 	 */
313 	if (uvp != NULLVP && lvp != NULLVP && uvp->v_type != lvp->v_type) {
314 		vrele(lvp);
315 		lvp = NULLVP;
316 	}
317 
318 	/*
319 	 * check shadow dir
320 	 */
321 	if (uerror != 0 && uerror != EJUSTRETURN && udvp != NULLVP &&
322 	    lerror == 0 && lvp != NULLVP && lvp->v_type == VDIR &&
323 	    !(dvp->v_mount->mnt_flag & MNT_RDONLY) &&
324 	    (1 < cnp->cn_namelen || '.' != *(cnp->cn_nameptr))) {
325 		/* get unionfs vnode in order to create a new shadow dir. */
326 		error = unionfs_nodeget(dvp->v_mount, NULLVP, lvp, dvp, &vp,
327 		    cnp);
328 		if (error != 0)
329 			goto unionfs_lookup_cleanup;
330 
331 		if (LK_SHARED == (cnp->cn_lkflags & LK_TYPE_MASK))
332 			VOP_UNLOCK(vp);
333 		if (LK_EXCLUSIVE != VOP_ISLOCKED(vp)) {
334 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
335 			lockflag = 1;
336 		}
337 		unp = VTOUNIONFS(vp);
338 		if (unp == NULL)
339 			error = ENOENT;
340 		else
341 			error = unionfs_mkshadowdir(dvp, vp, cnp, td);
342 		if (lockflag != 0)
343 			VOP_UNLOCK(vp);
344 		if (error != 0) {
345 			UNIONFSDEBUG(
346 			    "unionfs_lookup: Unable to create shadow dir.");
347 			if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE)
348 				vput(vp);
349 			else
350 				vrele(vp);
351 			goto unionfs_lookup_cleanup;
352 		}
353 		/*
354 		 * TODO: Since unionfs_mkshadowdir() relocks udvp after
355 		 * creating the new directory, return ERELOOKUP here?
356 		 */
357 		if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_SHARED)
358 			vn_lock(vp, LK_SHARED | LK_RETRY);
359 	}
360 	/*
361 	 * get unionfs vnode.
362 	 */
363 	else {
364 		if (uvp != NULLVP)
365 			error = uerror;
366 		else
367 			error = lerror;
368 		if (error != 0)
369 			goto unionfs_lookup_cleanup;
370 		error = unionfs_nodeget(dvp->v_mount, uvp, lvp,
371 		    dvp, &vp, cnp);
372 		if (error != 0) {
373 			UNIONFSDEBUG(
374 			    "unionfs_lookup: Unable to create unionfs vnode.");
375 			goto unionfs_lookup_cleanup;
376 		}
377 	}
378 
379 	if (VN_IS_DOOMED(dvp) || VN_IS_DOOMED(vp)) {
380 		error = ENOENT;
381 		vput(vp);
382 		goto unionfs_lookup_cleanup;
383 	}
384 
385 	*(ap->a_vpp) = vp;
386 
387 	if (cnflags & MAKEENTRY)
388 		cache_enter(dvp, vp, cnp);
389 
390 unionfs_lookup_cleanup:
391 	if (uvp != NULLVP)
392 		vrele(uvp);
393 	if (lvp != NULLVP)
394 		vrele(lvp);
395 
396 	if (error == ENOENT && (cnflags & MAKEENTRY) != 0 &&
397 	    !VN_IS_DOOMED(dvp))
398 		cache_enter(dvp, NULLVP, cnp);
399 
400 unionfs_lookup_return:
401 	unionfs_clear_in_progress_flag(dvp, UNIONFS_LOOKUP_IN_PROGRESS);
402 
403 	UNIONFS_INTERNAL_DEBUG("unionfs_lookup: leave (%d)\n", error);
404 
405 	return (error);
406 }
407 
408 static int
409 unionfs_create(struct vop_create_args *ap)
410 {
411 	struct unionfs_node *dunp;
412 	struct componentname *cnp;
413 	struct vnode   *udvp;
414 	struct vnode   *vp;
415 	int		error;
416 
417 	UNIONFS_INTERNAL_DEBUG("unionfs_create: enter\n");
418 
419 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
420 
421 	dunp = VTOUNIONFS(ap->a_dvp);
422 	cnp = ap->a_cnp;
423 	udvp = dunp->un_uppervp;
424 	error = EROFS;
425 
426 	if (udvp != NULLVP) {
427 		int lkflags;
428 		bool vp_created = false;
429 		unionfs_forward_vop_start(udvp, &lkflags);
430 		error = VOP_CREATE(udvp, &vp, cnp, ap->a_vap);
431 		if (error == 0)
432 			vp_created = true;
433 		if (__predict_false(unionfs_forward_vop_finish(ap->a_dvp, udvp,
434 		    lkflags)) && error == 0) {
435 			error = ENOENT;
436 		}
437 		if (error == 0) {
438 			VOP_UNLOCK(vp);
439 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
440 			    ap->a_dvp, ap->a_vpp, cnp);
441 			vrele(vp);
442 		} else if (vp_created)
443 			vput(vp);
444 	}
445 
446 	UNIONFS_INTERNAL_DEBUG("unionfs_create: leave (%d)\n", error);
447 
448 	return (error);
449 }
450 
451 static int
452 unionfs_whiteout(struct vop_whiteout_args *ap)
453 {
454 	struct unionfs_node *dunp;
455 	struct componentname *cnp;
456 	struct vnode   *udvp;
457 	int		error;
458 
459 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: enter\n");
460 
461 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
462 
463 	dunp = VTOUNIONFS(ap->a_dvp);
464 	cnp = ap->a_cnp;
465 	udvp = dunp->un_uppervp;
466 	error = EOPNOTSUPP;
467 
468 	if (udvp != NULLVP) {
469 		int lkflags;
470 		switch (ap->a_flags) {
471 		case CREATE:
472 		case DELETE:
473 		case LOOKUP:
474 			unionfs_forward_vop_start(udvp, &lkflags);
475 			error = VOP_WHITEOUT(udvp, cnp, ap->a_flags);
476 			unionfs_forward_vop_finish(ap->a_dvp, udvp, lkflags);
477 			break;
478 		default:
479 			error = EINVAL;
480 			break;
481 		}
482 	}
483 
484 	UNIONFS_INTERNAL_DEBUG("unionfs_whiteout: leave (%d)\n", error);
485 
486 	return (error);
487 }
488 
489 static int
490 unionfs_mknod(struct vop_mknod_args *ap)
491 {
492 	struct unionfs_node *dunp;
493 	struct componentname *cnp;
494 	struct vnode   *udvp;
495 	struct vnode   *vp;
496 	int		error;
497 
498 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: enter\n");
499 
500 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
501 
502 	dunp = VTOUNIONFS(ap->a_dvp);
503 	cnp = ap->a_cnp;
504 	udvp = dunp->un_uppervp;
505 	error = EROFS;
506 
507 	if (udvp != NULLVP) {
508 		int lkflags;
509 		bool vp_created = false;
510 		unionfs_forward_vop_start(udvp, &lkflags);
511 		error = VOP_MKNOD(udvp, &vp, cnp, ap->a_vap);
512 		if (error == 0)
513 			vp_created = true;
514 		if (__predict_false(unionfs_forward_vop_finish(ap->a_dvp, udvp,
515 		    lkflags)) && error == 0) {
516 			error = ENOENT;
517 		}
518 		if (error == 0) {
519 			VOP_UNLOCK(vp);
520 			error = unionfs_nodeget(ap->a_dvp->v_mount, vp, NULLVP,
521 			    ap->a_dvp, ap->a_vpp, cnp);
522 			vrele(vp);
523 		} else if (vp_created)
524 			vput(vp);
525 	}
526 
527 	UNIONFS_INTERNAL_DEBUG("unionfs_mknod: leave (%d)\n", error);
528 
529 	return (error);
530 }
531 
532 enum unionfs_lkupgrade {
533 	UNIONFS_LKUPGRADE_SUCCESS, /* lock successfully upgraded */
534 	UNIONFS_LKUPGRADE_ALREADY, /* lock already held exclusive */
535 	UNIONFS_LKUPGRADE_DOOMED   /* lock was upgraded, but vnode reclaimed */
536 };
537 
538 static inline enum unionfs_lkupgrade
539 unionfs_upgrade_lock(struct vnode *vp)
540 {
541 	ASSERT_VOP_LOCKED(vp, __func__);
542 
543 	if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
544 		return (UNIONFS_LKUPGRADE_ALREADY);
545 
546 	if (vn_lock(vp, LK_UPGRADE) != 0) {
547 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
548 		if (VN_IS_DOOMED(vp))
549 			return (UNIONFS_LKUPGRADE_DOOMED);
550 	}
551 	return (UNIONFS_LKUPGRADE_SUCCESS);
552 }
553 
554 static inline void
555 unionfs_downgrade_lock(struct vnode *vp, enum unionfs_lkupgrade status)
556 {
557 	if (status != UNIONFS_LKUPGRADE_ALREADY)
558 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
559 }
560 
561 /*
562  * Exchange the default (upper vnode) lock on a unionfs vnode for the lower
563  * vnode lock, in support of operations that require access to the lower vnode
564  * even when an upper vnode is present.  We don't use vn_lock_pair() to hold
565  * both vnodes at the same time, primarily because the caller may proceed
566  * to issue VOPs to the lower layer which re-lock or perform other operations
567  * which may not be safe in the presence of a locked vnode from another FS.
568  * Moreover, vn_lock_pair()'s deadlock resolution approach can introduce
569  * additional overhead that isn't necessary on these paths.
570  *
571  * vp must be a locked unionfs vnode; the lock state of this vnode is
572  * returned through *lkflags for later use in unionfs_unlock_lvp().
573  *
574  * Returns the locked lower vnode, or NULL if the lower vnode (and therefore
575  * also the unionfs vnode above it) has been doomed.
576  */
577 static struct vnode *
578 unionfs_lock_lvp(struct vnode *vp, int *lkflags)
579 {
580 	struct unionfs_node *unp;
581 	struct vnode *lvp;
582 
583 	unp = VTOUNIONFS(vp);
584 	lvp = unp->un_lowervp;
585 	ASSERT_VOP_LOCKED(vp, __func__);
586 	ASSERT_VOP_UNLOCKED(lvp, __func__);
587 	*lkflags = VOP_ISLOCKED(vp);
588 	vref(lvp);
589 	VOP_UNLOCK(vp);
590 	vn_lock(lvp, *lkflags | LK_RETRY);
591 	if (VN_IS_DOOMED(lvp)) {
592 		vput(lvp);
593 		lvp = NULLVP;
594 		vn_lock(vp, *lkflags | LK_RETRY);
595 	}
596 	return (lvp);
597 }
598 
599 /*
600  * Undo a previous call to unionfs_lock_lvp(), restoring the default lock
601  * on the unionfs vnode.  This function reloads and returns the vnode
602  * private data for the unionfs vnode, which will be NULL if the unionfs
603  * vnode became doomed while its lock was dropped.  The caller must check
604  * for this case.
605  */
606 static struct unionfs_node *
607 unionfs_unlock_lvp(struct vnode *vp, struct vnode *lvp, int lkflags)
608 {
609 	ASSERT_VOP_LOCKED(lvp, __func__);
610 	ASSERT_VOP_UNLOCKED(vp, __func__);
611 	vput(lvp);
612 	vn_lock(vp, lkflags | LK_RETRY);
613 	return (VTOUNIONFS(vp));
614 }
615 
616 static int
617 unionfs_open(struct vop_open_args *ap)
618 {
619 	struct unionfs_node *unp;
620 	struct unionfs_node_status *unsp;
621 	struct vnode   *vp;
622 	struct vnode   *uvp;
623 	struct vnode   *lvp;
624 	struct vnode   *targetvp;
625 	struct ucred   *cred;
626 	struct thread  *td;
627 	int		error;
628 	int		lkflags;
629 	enum unionfs_lkupgrade lkstatus;
630 	bool		lock_lvp, open_lvp;
631 
632 	UNIONFS_INTERNAL_DEBUG("unionfs_open: enter\n");
633 
634 	KASSERT_UNIONFS_VNODE(ap->a_vp);
635 
636 	error = 0;
637 	vp = ap->a_vp;
638 	targetvp = NULLVP;
639 	cred = ap->a_cred;
640 	td = ap->a_td;
641 	open_lvp = lock_lvp = false;
642 
643 	/*
644 	 * The executable loader path may call this function with vp locked
645 	 * shared.  If the vnode is reclaimed while upgrading, we can't safely
646 	 * use unp or do anything else unionfs- specific.
647 	 */
648 	lkstatus = unionfs_upgrade_lock(vp);
649 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED) {
650 		error = ENOENT;
651 		goto unionfs_open_cleanup;
652 	}
653 
654 	unp = VTOUNIONFS(vp);
655 	uvp = unp->un_uppervp;
656 	lvp = unp->un_lowervp;
657 	unionfs_get_node_status(unp, td, &unsp);
658 
659 	if (unsp->uns_lower_opencnt > 0 || unsp->uns_upper_opencnt > 0) {
660 		/* vnode is already opend. */
661 		if (unsp->uns_upper_opencnt > 0)
662 			targetvp = uvp;
663 		else
664 			targetvp = lvp;
665 
666 		if (targetvp == lvp &&
667 		    (ap->a_mode & FWRITE) && lvp->v_type == VREG)
668 			targetvp = NULLVP;
669 	}
670 	if (targetvp == NULLVP) {
671 		if (uvp == NULLVP) {
672 			if ((ap->a_mode & FWRITE) && lvp->v_type == VREG) {
673 				error = unionfs_copyfile(vp,
674 				    !(ap->a_mode & O_TRUNC), cred, td);
675 				if (error != 0) {
676 					unp = VTOUNIONFS(vp);
677 					goto unionfs_open_abort;
678 				}
679 				targetvp = uvp = unp->un_uppervp;
680 			} else
681 				targetvp = lvp;
682 		} else
683 			targetvp = uvp;
684 	}
685 
686 	if (targetvp == uvp && uvp->v_type == VDIR && lvp != NULLVP &&
687 	    unsp->uns_lower_opencnt <= 0)
688 		open_lvp = true;
689 	else if (targetvp == lvp && uvp != NULLVP)
690 		lock_lvp = true;
691 
692 	if (lock_lvp) {
693 		unp = NULL;
694 		lvp = unionfs_lock_lvp(vp, &lkflags);
695 		if (lvp == NULLVP) {
696 			error = ENOENT;
697 			goto unionfs_open_abort;
698 		}
699 	} else
700 		unionfs_forward_vop_start(targetvp, &lkflags);
701 
702 	error = VOP_OPEN(targetvp, ap->a_mode, cred, td, ap->a_fp);
703 
704 	if (lock_lvp) {
705 		unp = unionfs_unlock_lvp(vp, lvp, lkflags);
706 		if (unp == NULL && error == 0)
707 			error = ENOENT;
708 	} else if (unionfs_forward_vop_finish(vp, targetvp, lkflags))
709 		error = error ? error : ENOENT;
710 
711 	if (error != 0)
712 		goto unionfs_open_abort;
713 
714 	if (targetvp == uvp) {
715 		if (open_lvp) {
716 			unp = NULL;
717 			lvp = unionfs_lock_lvp(vp, &lkflags);
718 			if (lvp == NULLVP) {
719 				error = ENOENT;
720 				goto unionfs_open_abort;
721 			}
722 			/* open lower for readdir */
723 			error = VOP_OPEN(lvp, FREAD, cred, td, NULL);
724 			unp = unionfs_unlock_lvp(vp, lvp, lkflags);
725 			if (unp == NULL) {
726 				error = error ? error : ENOENT;
727 				goto unionfs_open_abort;
728 			}
729 			if (error != 0) {
730 				unionfs_forward_vop_start(uvp, &lkflags);
731 				VOP_CLOSE(uvp, ap->a_mode, cred, td);
732 				if (unionfs_forward_vop_finish(vp, uvp, lkflags))
733 					unp = NULL;
734 				goto unionfs_open_abort;
735 			}
736 			unsp->uns_node_flag |= UNS_OPENL_4_READDIR;
737 			unsp->uns_lower_opencnt++;
738 		}
739 		unsp->uns_upper_opencnt++;
740 	} else {
741 		unsp->uns_lower_opencnt++;
742 		unsp->uns_lower_openmode = ap->a_mode;
743 	}
744 	vp->v_object = targetvp->v_object;
745 
746 unionfs_open_abort:
747 
748 	if (error != 0 && unp != NULL)
749 		unionfs_tryrem_node_status(unp, unsp);
750 
751 unionfs_open_cleanup:
752 	unionfs_downgrade_lock(vp, lkstatus);
753 
754 	UNIONFS_INTERNAL_DEBUG("unionfs_open: leave (%d)\n", error);
755 
756 	return (error);
757 }
758 
759 static int
760 unionfs_close(struct vop_close_args *ap)
761 {
762 	struct unionfs_node *unp;
763 	struct unionfs_node_status *unsp;
764 	struct ucred   *cred;
765 	struct thread  *td;
766 	struct vnode   *vp;
767 	struct vnode   *uvp;
768 	struct vnode   *lvp;
769 	struct vnode   *ovp;
770 	int		error;
771 	int		lkflags;
772 	enum unionfs_lkupgrade lkstatus;
773 	bool		lock_lvp;
774 
775 	UNIONFS_INTERNAL_DEBUG("unionfs_close: enter\n");
776 
777 	KASSERT_UNIONFS_VNODE(ap->a_vp);
778 
779 	vp = ap->a_vp;
780 	cred = ap->a_cred;
781 	td = ap->a_td;
782 	error = 0;
783 	lock_lvp = false;
784 
785 	/*
786 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
787 	 * or do anything else unionfs- specific.
788 	 */
789 	lkstatus = unionfs_upgrade_lock(vp);
790 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
791 		goto unionfs_close_cleanup;
792 
793 	unp = VTOUNIONFS(vp);
794 	lvp = unp->un_lowervp;
795 	uvp = unp->un_uppervp;
796 	unsp = unionfs_find_node_status(unp, td);
797 
798 	if (unsp == NULL ||
799 	    (unsp->uns_lower_opencnt <= 0 && unsp->uns_upper_opencnt <= 0)) {
800 #ifdef DIAGNOSTIC
801 		if (unsp != NULL)
802 			printf("unionfs_close: warning: open count is 0\n");
803 #endif
804 		if (uvp != NULLVP)
805 			ovp = uvp;
806 		else
807 			ovp = lvp;
808 	} else if (unsp->uns_upper_opencnt > 0)
809 		ovp = uvp;
810 	else
811 		ovp = lvp;
812 
813 	if (ovp == lvp && uvp != NULLVP) {
814 		lock_lvp = true;
815 		unp = NULL;
816 		lvp = unionfs_lock_lvp(vp, &lkflags);
817 		if (lvp == NULLVP) {
818 			error = ENOENT;
819 			goto unionfs_close_abort;
820 		}
821 	} else
822 		unionfs_forward_vop_start(ovp, &lkflags);
823 
824 	error = VOP_CLOSE(ovp, ap->a_fflag, cred, td);
825 
826 	if (lock_lvp) {
827 		unp = unionfs_unlock_lvp(vp, lvp, lkflags);
828 		if (unp == NULL && error == 0)
829 			error = ENOENT;
830 	} else if (unionfs_forward_vop_finish(vp, ovp, lkflags))
831 		error = error ? error : ENOENT;
832 
833 	if (error != 0)
834 		goto unionfs_close_abort;
835 
836 	vp->v_object = ovp->v_object;
837 
838 	if (ovp == uvp) {
839 		if (unsp != NULL && ((--unsp->uns_upper_opencnt) == 0)) {
840 			if (unsp->uns_node_flag & UNS_OPENL_4_READDIR) {
841 				unp = NULL;
842 				lvp = unionfs_lock_lvp(vp, &lkflags);
843 				if (lvp == NULLVP) {
844 					error = ENOENT;
845 					goto unionfs_close_abort;
846 				}
847 				VOP_CLOSE(lvp, FREAD, cred, td);
848 				unp = unionfs_unlock_lvp(vp, lvp, lkflags);
849 				if (unp == NULL) {
850 					error = ENOENT;
851 					goto unionfs_close_abort;
852 				}
853 				unsp->uns_node_flag &= ~UNS_OPENL_4_READDIR;
854 				unsp->uns_lower_opencnt--;
855 			}
856 			if (unsp->uns_lower_opencnt > 0)
857 				vp->v_object = lvp->v_object;
858 		}
859 	} else if (unsp != NULL)
860 		unsp->uns_lower_opencnt--;
861 
862 unionfs_close_abort:
863 	if (unp != NULL && unsp != NULL)
864 		unionfs_tryrem_node_status(unp, unsp);
865 
866 unionfs_close_cleanup:
867 	unionfs_downgrade_lock(vp, lkstatus);
868 
869 	UNIONFS_INTERNAL_DEBUG("unionfs_close: leave (%d)\n", error);
870 
871 	return (error);
872 }
873 
874 /*
875  * Check the access mode toward shadow file/dir.
876  */
877 static int
878 unionfs_check_corrected_access(accmode_t accmode, struct vattr *va,
879     struct ucred *cred)
880 {
881 	uid_t		uid;	/* upper side vnode's uid */
882 	gid_t		gid;	/* upper side vnode's gid */
883 	u_short		vmode;	/* upper side vnode's mode */
884 	u_short		mask;
885 
886 	mask = 0;
887 	uid = va->va_uid;
888 	gid = va->va_gid;
889 	vmode = va->va_mode;
890 
891 	/* check owner */
892 	if (cred->cr_uid == uid) {
893 		if (accmode & VEXEC)
894 			mask |= S_IXUSR;
895 		if (accmode & VREAD)
896 			mask |= S_IRUSR;
897 		if (accmode & VWRITE)
898 			mask |= S_IWUSR;
899 		return ((vmode & mask) == mask ? 0 : EACCES);
900 	}
901 
902 	/* check group */
903 	if (groupmember(gid, cred)) {
904 		if (accmode & VEXEC)
905 			mask |= S_IXGRP;
906 		if (accmode & VREAD)
907 			mask |= S_IRGRP;
908 		if (accmode & VWRITE)
909 			mask |= S_IWGRP;
910 		return ((vmode & mask) == mask ? 0 : EACCES);
911 	}
912 
913 	/* check other */
914 	if (accmode & VEXEC)
915 		mask |= S_IXOTH;
916 	if (accmode & VREAD)
917 		mask |= S_IROTH;
918 	if (accmode & VWRITE)
919 		mask |= S_IWOTH;
920 
921 	return ((vmode & mask) == mask ? 0 : EACCES);
922 }
923 
924 static int
925 unionfs_access(struct vop_access_args *ap)
926 {
927 	struct unionfs_mount *ump;
928 	struct unionfs_node *unp;
929 	struct vnode   *uvp;
930 	struct vnode   *lvp;
931 	struct thread  *td;
932 	struct vattr	va;
933 	accmode_t	accmode;
934 	int		error;
935 
936 	UNIONFS_INTERNAL_DEBUG("unionfs_access: enter\n");
937 
938 	KASSERT_UNIONFS_VNODE(ap->a_vp);
939 
940 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
941 	unp = VTOUNIONFS(ap->a_vp);
942 	uvp = unp->un_uppervp;
943 	lvp = unp->un_lowervp;
944 	td = ap->a_td;
945 	accmode = ap->a_accmode;
946 	error = EACCES;
947 
948 	if ((accmode & VWRITE) &&
949 	    (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)) {
950 		switch (ap->a_vp->v_type) {
951 		case VREG:
952 		case VDIR:
953 		case VLNK:
954 			return (EROFS);
955 		default:
956 			break;
957 		}
958 	}
959 
960 	if (uvp != NULLVP) {
961 		error = VOP_ACCESS(uvp, accmode, ap->a_cred, td);
962 
963 		UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
964 
965 		return (error);
966 	}
967 
968 	if (lvp != NULLVP) {
969 		if (accmode & VWRITE) {
970 			if ((ump->um_uppermp->mnt_flag & MNT_RDONLY) != 0) {
971 				switch (ap->a_vp->v_type) {
972 				case VREG:
973 				case VDIR:
974 				case VLNK:
975 					return (EROFS);
976 				default:
977 					break;
978 				}
979 			} else if (ap->a_vp->v_type == VREG ||
980 			    ap->a_vp->v_type == VDIR) {
981 				/* check shadow file/dir */
982 				if (ump->um_copymode != UNIONFS_TRANSPARENT) {
983 					error = unionfs_create_uppervattr(ump,
984 					    lvp, &va, ap->a_cred, td);
985 					if (error != 0)
986 						return (error);
987 
988 					error = unionfs_check_corrected_access(
989 					    accmode, &va, ap->a_cred);
990 					if (error != 0)
991 						return (error);
992 				}
993 			}
994 			accmode &= ~(VWRITE | VAPPEND);
995 			accmode |= VREAD; /* will copy to upper */
996 		}
997 		error = VOP_ACCESS(lvp, accmode, ap->a_cred, td);
998 	}
999 
1000 	UNIONFS_INTERNAL_DEBUG("unionfs_access: leave (%d)\n", error);
1001 
1002 	return (error);
1003 }
1004 
1005 static int
1006 unionfs_getattr(struct vop_getattr_args *ap)
1007 {
1008 	struct unionfs_node *unp;
1009 	struct unionfs_mount *ump;
1010 	struct vnode   *uvp;
1011 	struct vnode   *lvp;
1012 	struct thread  *td;
1013 	struct vattr	va;
1014 	int		error;
1015 
1016 	UNIONFS_INTERNAL_DEBUG("unionfs_getattr: enter\n");
1017 
1018 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1019 
1020 	unp = VTOUNIONFS(ap->a_vp);
1021 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1022 	uvp = unp->un_uppervp;
1023 	lvp = unp->un_lowervp;
1024 	td = curthread;
1025 
1026 	if (uvp != NULLVP) {
1027 		if ((error = VOP_GETATTR(uvp, ap->a_vap, ap->a_cred)) == 0)
1028 			ap->a_vap->va_fsid =
1029 			    ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
1030 
1031 		UNIONFS_INTERNAL_DEBUG(
1032 		    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
1033 		    ap->a_vap->va_mode, ap->a_vap->va_uid,
1034 		    ap->a_vap->va_gid, error);
1035 
1036 		return (error);
1037 	}
1038 
1039 	error = VOP_GETATTR(lvp, ap->a_vap, ap->a_cred);
1040 
1041 	if (error == 0 && (ump->um_uppermp->mnt_flag & MNT_RDONLY) == 0) {
1042 		/* correct the attr toward shadow file/dir. */
1043 		if (ap->a_vp->v_type == VREG || ap->a_vp->v_type == VDIR) {
1044 			unionfs_create_uppervattr_core(ump, ap->a_vap, &va, td);
1045 			ap->a_vap->va_mode = va.va_mode;
1046 			ap->a_vap->va_uid = va.va_uid;
1047 			ap->a_vap->va_gid = va.va_gid;
1048 		}
1049 	}
1050 
1051 	if (error == 0)
1052 		ap->a_vap->va_fsid = ap->a_vp->v_mount->mnt_stat.f_fsid.val[0];
1053 
1054 	UNIONFS_INTERNAL_DEBUG(
1055 	    "unionfs_getattr: leave mode=%o, uid=%d, gid=%d (%d)\n",
1056 	    ap->a_vap->va_mode, ap->a_vap->va_uid, ap->a_vap->va_gid, error);
1057 
1058 	return (error);
1059 }
1060 
1061 static int
1062 unionfs_setattr(struct vop_setattr_args *ap)
1063 {
1064 	struct unionfs_node *unp;
1065 	struct vnode   *uvp;
1066 	struct vnode   *lvp;
1067 	struct thread  *td;
1068 	struct vattr   *vap;
1069 	int		error;
1070 
1071 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: enter\n");
1072 
1073 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1074 
1075 	error = EROFS;
1076 	unp = VTOUNIONFS(ap->a_vp);
1077 	uvp = unp->un_uppervp;
1078 	lvp = unp->un_lowervp;
1079 	td = curthread;
1080 	vap = ap->a_vap;
1081 
1082 	if ((ap->a_vp->v_mount->mnt_flag & MNT_RDONLY) &&
1083 	    (vap->va_flags != VNOVAL || vap->va_uid != (uid_t)VNOVAL ||
1084 	     vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL ||
1085 	     vap->va_mtime.tv_sec != VNOVAL || vap->va_mode != (mode_t)VNOVAL))
1086 		return (EROFS);
1087 
1088 	if (uvp == NULLVP && lvp->v_type == VREG) {
1089 		error = unionfs_copyfile(ap->a_vp, (vap->va_size != 0),
1090 		    ap->a_cred, td);
1091 		if (error != 0)
1092 			return (error);
1093 		uvp = unp->un_uppervp;
1094 	}
1095 
1096 	if (uvp != NULLVP) {
1097 		int lkflags;
1098 		unionfs_forward_vop_start(uvp, &lkflags);
1099 		error = VOP_SETATTR(uvp, vap, ap->a_cred);
1100 		unionfs_forward_vop_finish(ap->a_vp, uvp, lkflags);
1101 	}
1102 
1103 	UNIONFS_INTERNAL_DEBUG("unionfs_setattr: leave (%d)\n", error);
1104 
1105 	return (error);
1106 }
1107 
1108 static int
1109 unionfs_read(struct vop_read_args *ap)
1110 {
1111 	struct unionfs_node *unp;
1112 	struct vnode   *tvp;
1113 	int		error;
1114 
1115 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: enter\n"); */
1116 
1117 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1118 
1119 	unp = VTOUNIONFS(ap->a_vp);
1120 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
1121 
1122 	error = VOP_READ(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
1123 
1124 	/* UNIONFS_INTERNAL_DEBUG("unionfs_read: leave (%d)\n", error); */
1125 
1126 	return (error);
1127 }
1128 
1129 static int
1130 unionfs_write(struct vop_write_args *ap)
1131 {
1132 	struct unionfs_node *unp;
1133 	struct vnode   *tvp;
1134 	int		error;
1135 	int		lkflags;
1136 
1137 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: enter\n"); */
1138 
1139 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1140 
1141 	unp = VTOUNIONFS(ap->a_vp);
1142 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
1143 
1144 	unionfs_forward_vop_start(tvp, &lkflags);
1145 	error = VOP_WRITE(tvp, ap->a_uio, ap->a_ioflag, ap->a_cred);
1146 	unionfs_forward_vop_finish(ap->a_vp, tvp, lkflags);
1147 
1148 	/* UNIONFS_INTERNAL_DEBUG("unionfs_write: leave (%d)\n", error); */
1149 
1150 	return (error);
1151 }
1152 
1153 static int
1154 unionfs_ioctl(struct vop_ioctl_args *ap)
1155 {
1156 	struct unionfs_node *unp;
1157 	struct unionfs_node_status *unsp;
1158 	struct vnode   *ovp;
1159 	int error;
1160 
1161 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: enter\n");
1162 
1163 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1164 
1165  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1166 	unp = VTOUNIONFS(ap->a_vp);
1167 	unionfs_get_node_status(unp, ap->a_td, &unsp);
1168 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
1169 	unionfs_tryrem_node_status(unp, unsp);
1170 	VOP_UNLOCK(ap->a_vp);
1171 
1172 	if (ovp == NULLVP)
1173 		return (EBADF);
1174 
1175 	error = VOP_IOCTL(ovp, ap->a_command, ap->a_data, ap->a_fflag,
1176 	    ap->a_cred, ap->a_td);
1177 
1178 	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: leave (%d)\n", error);
1179 
1180 	return (error);
1181 }
1182 
1183 static int
1184 unionfs_poll(struct vop_poll_args *ap)
1185 {
1186 	struct unionfs_node *unp;
1187 	struct unionfs_node_status *unsp;
1188 	struct vnode *ovp;
1189 
1190 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1191 
1192  	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1193 	unp = VTOUNIONFS(ap->a_vp);
1194 	unionfs_get_node_status(unp, ap->a_td, &unsp);
1195 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
1196 	unionfs_tryrem_node_status(unp, unsp);
1197 	VOP_UNLOCK(ap->a_vp);
1198 
1199 	if (ovp == NULLVP)
1200 		return (EBADF);
1201 
1202 	return (VOP_POLL(ovp, ap->a_events, ap->a_cred, ap->a_td));
1203 }
1204 
1205 static int
1206 unionfs_fsync(struct vop_fsync_args *ap)
1207 {
1208 	struct unionfs_node *unp;
1209 	struct unionfs_node_status *unsp;
1210 	struct vnode *ovp;
1211 	enum unionfs_lkupgrade lkstatus;
1212 	int error, lkflags;
1213 
1214 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1215 
1216 	unp = VTOUNIONFS(ap->a_vp);
1217 	lkstatus = unionfs_upgrade_lock(ap->a_vp);
1218 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED) {
1219 		unionfs_downgrade_lock(ap->a_vp, lkstatus);
1220 		return (ENOENT);
1221 	}
1222 	unionfs_get_node_status(unp, ap->a_td, &unsp);
1223 	ovp = (unsp->uns_upper_opencnt ? unp->un_uppervp : unp->un_lowervp);
1224 	unionfs_tryrem_node_status(unp, unsp);
1225 
1226 	unionfs_downgrade_lock(ap->a_vp, lkstatus);
1227 
1228 	if (ovp == NULLVP)
1229 		return (EBADF);
1230 
1231 	unionfs_forward_vop_start(ovp, &lkflags);
1232 	error = VOP_FSYNC(ovp, ap->a_waitfor, ap->a_td);
1233 	unionfs_forward_vop_finish(ap->a_vp, ovp, lkflags);
1234 
1235 	return (error);
1236 }
1237 
1238 static int
1239 unionfs_remove(struct vop_remove_args *ap)
1240 {
1241 	char	       *path;
1242 	struct unionfs_node *dunp;
1243 	struct unionfs_node *unp;
1244 	struct unionfs_mount *ump;
1245 	struct vnode   *udvp;
1246 	struct vnode   *uvp;
1247 	struct vnode   *lvp;
1248 	struct componentname *cnp;
1249 	struct thread  *td;
1250 	int		error;
1251 	int		pathlen;
1252 
1253 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: enter\n");
1254 
1255 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1256 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1257 
1258 	error = 0;
1259 	dunp = VTOUNIONFS(ap->a_dvp);
1260 	udvp = dunp->un_uppervp;
1261 	cnp = ap->a_cnp;
1262 	td = curthread;
1263 
1264 	ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1265 	unp = VTOUNIONFS(ap->a_vp);
1266 	uvp = unp->un_uppervp;
1267 	lvp = unp->un_lowervp;
1268 	path = unp->un_path;
1269 	pathlen = unp->un_pathlen;
1270 
1271 	if (udvp == NULLVP)
1272 		return (EROFS);
1273 
1274 	if (uvp != NULLVP) {
1275 		int udvp_lkflags, uvp_lkflags;
1276 		if (ump == NULL || ump->um_whitemode == UNIONFS_WHITE_ALWAYS ||
1277 		    lvp != NULLVP)
1278 			cnp->cn_flags |= DOWHITEOUT;
1279 		unionfs_forward_vop_start_pair(udvp, &udvp_lkflags,
1280 		    uvp, &uvp_lkflags);
1281 		error = VOP_REMOVE(udvp, uvp, cnp);
1282 		unionfs_forward_vop_finish_pair(ap->a_dvp, udvp, udvp_lkflags,
1283 		    ap->a_vp, uvp, uvp_lkflags);
1284 	} else if (lvp != NULLVP) {
1285 		error = unionfs_mkwhiteout(ap->a_dvp, ap->a_vp, cnp, td,
1286 		    path, pathlen);
1287 	}
1288 
1289 	UNIONFS_INTERNAL_DEBUG("unionfs_remove: leave (%d)\n", error);
1290 
1291 	return (error);
1292 }
1293 
1294 static int
1295 unionfs_link(struct vop_link_args *ap)
1296 {
1297 	struct unionfs_node *dunp;
1298 	struct unionfs_node *unp;
1299 	struct vnode   *udvp;
1300 	struct vnode   *uvp;
1301 	struct componentname *cnp;
1302 	struct thread  *td;
1303 	int		error;
1304 
1305 	UNIONFS_INTERNAL_DEBUG("unionfs_link: enter\n");
1306 
1307 	KASSERT_UNIONFS_VNODE(ap->a_tdvp);
1308 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1309 
1310 	error = 0;
1311 	dunp = VTOUNIONFS(ap->a_tdvp);
1312 	unp = NULL;
1313 	udvp = dunp->un_uppervp;
1314 	uvp = NULLVP;
1315 	cnp = ap->a_cnp;
1316 	td = curthread;
1317 
1318 	if (udvp == NULLVP)
1319 		return (EROFS);
1320 
1321 	unp = VTOUNIONFS(ap->a_vp);
1322 
1323 	if (unp->un_uppervp == NULLVP) {
1324 		if (ap->a_vp->v_type != VREG)
1325 			return (EOPNOTSUPP);
1326 
1327 		VOP_UNLOCK(ap->a_tdvp);
1328 		error = unionfs_copyfile(ap->a_vp, 1, cnp->cn_cred, td);
1329 		vn_lock(ap->a_tdvp, LK_EXCLUSIVE | LK_RETRY);
1330 		if (error == 0)
1331 			error = ERELOOKUP;
1332 		return (error);
1333 	}
1334 	uvp = unp->un_uppervp;
1335 
1336 	if (error == 0) {
1337 		int udvp_lkflags, uvp_lkflags;
1338 		unionfs_forward_vop_start_pair(udvp, &udvp_lkflags,
1339 		    uvp, &uvp_lkflags);
1340 		error = VOP_LINK(udvp, uvp, cnp);
1341 		unionfs_forward_vop_finish_pair(ap->a_tdvp, udvp, udvp_lkflags,
1342 		    ap->a_vp, uvp, uvp_lkflags);
1343 	}
1344 
1345 	UNIONFS_INTERNAL_DEBUG("unionfs_link: leave (%d)\n", error);
1346 
1347 	return (error);
1348 }
1349 
1350 static int
1351 unionfs_rename(struct vop_rename_args *ap)
1352 {
1353 	struct vnode   *fdvp;
1354 	struct vnode   *fvp;
1355 	struct componentname *fcnp;
1356 	struct vnode   *tdvp;
1357 	struct vnode   *tvp;
1358 	struct componentname *tcnp;
1359 	struct thread  *td;
1360 
1361 	/* rename target vnodes */
1362 	struct vnode   *rfdvp;
1363 	struct vnode   *rfvp;
1364 	struct vnode   *rtdvp;
1365 	struct vnode   *rtvp;
1366 
1367 	struct unionfs_node *unp;
1368 	int		error;
1369 
1370 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: enter\n");
1371 
1372 	error = 0;
1373 	fdvp = ap->a_fdvp;
1374 	fvp = ap->a_fvp;
1375 	fcnp = ap->a_fcnp;
1376 	tdvp = ap->a_tdvp;
1377 	tvp = ap->a_tvp;
1378 	tcnp = ap->a_tcnp;
1379 	td = curthread;
1380 	rfdvp = fdvp;
1381 	rfvp = fvp;
1382 	rtdvp = tdvp;
1383 	rtvp = tvp;
1384 
1385 	/* check for cross device rename */
1386 	if (fvp->v_mount != tdvp->v_mount ||
1387 	    (tvp != NULLVP && fvp->v_mount != tvp->v_mount)) {
1388 		if (fvp->v_op != &unionfs_vnodeops)
1389 			error = ENODEV;
1390 		else
1391 			error = EXDEV;
1392 		goto unionfs_rename_abort;
1393 	}
1394 
1395 	/* Renaming a file to itself has no effect. */
1396 	if (fvp == tvp)
1397 		goto unionfs_rename_abort;
1398 
1399 	KASSERT_UNIONFS_VNODE(tdvp);
1400 	if (tvp != NULLVP)
1401 		KASSERT_UNIONFS_VNODE(tvp);
1402 	if (fdvp != tdvp)
1403 		VI_LOCK(fdvp);
1404 	unp = VTOUNIONFS(fdvp);
1405 	if (unp == NULL) {
1406 		if (fdvp != tdvp)
1407 			VI_UNLOCK(fdvp);
1408 		error = ENOENT;
1409 		goto unionfs_rename_abort;
1410 	}
1411 #ifdef UNIONFS_IDBG_RENAME
1412 	UNIONFS_INTERNAL_DEBUG("fdvp=%p, ufdvp=%p, lfdvp=%p\n",
1413 	    fdvp, unp->un_uppervp, unp->un_lowervp);
1414 #endif
1415 	if (unp->un_uppervp == NULLVP) {
1416 		error = ENODEV;
1417 	} else {
1418 		rfdvp = unp->un_uppervp;
1419 		vref(rfdvp);
1420 	}
1421 	if (fdvp != tdvp)
1422 		VI_UNLOCK(fdvp);
1423 	if (error != 0)
1424 		goto unionfs_rename_abort;
1425 
1426 	VI_LOCK(fvp);
1427 	unp = VTOUNIONFS(fvp);
1428 	if (unp == NULL) {
1429 		VI_UNLOCK(fvp);
1430 		error = ENOENT;
1431 		goto unionfs_rename_abort;
1432 	}
1433 
1434 #ifdef UNIONFS_IDBG_RENAME
1435 	UNIONFS_INTERNAL_DEBUG("fvp=%p, ufvp=%p, lfvp=%p\n",
1436 	    fvp, unp->un_uppervp, unp->un_lowervp);
1437 #endif
1438 	/*
1439 	 * If we only have a lower vnode, copy the source file to the upper
1440 	 * FS so that the rename operation can be issued against the upper FS.
1441 	 */
1442 	if (unp->un_uppervp == NULLVP) {
1443 		bool unlock_fdvp = false, relock_tdvp = false;
1444 		VI_UNLOCK(fvp);
1445 		if (tvp != NULLVP)
1446 			VOP_UNLOCK(tvp);
1447 		if (fvp->v_type == VREG) {
1448 			/*
1449 			 * For regular files, unionfs_copyfile() will expect
1450 			 * fdvp's upper parent directory vnode to be unlocked
1451 			 * and will temporarily lock it.  If fdvp == tdvp, we
1452 			 * should unlock tdvp to avoid recursion on tdvp's
1453 			 * lock.  If fdvp != tdvp, we should also unlock tdvp
1454 			 * to avoid potential deadlock due to holding tdvp's
1455 			 * lock while locking unrelated vnodes associated with
1456 			 * fdvp/fvp.
1457 			 */
1458 			VOP_UNLOCK(tdvp);
1459 			relock_tdvp = true;
1460 		} else if (fvp->v_type == VDIR && tdvp != fdvp) {
1461 			/*
1462 			 * For directories, unionfs_mkshadowdir() will expect
1463 			 * fdvp's upper parent directory vnode to be locked
1464 			 * and will temporarily unlock it.  If fdvp == tdvp,
1465 			 * we can therefore leave tdvp locked.  If fdvp !=
1466 			 * tdvp, we should exchange the lock on tdvp for a
1467 			 * lock on fdvp.
1468 			 */
1469 			VOP_UNLOCK(tdvp);
1470 			unlock_fdvp = true;
1471 			relock_tdvp = true;
1472 			vn_lock(fdvp, LK_EXCLUSIVE | LK_RETRY);
1473 		}
1474 		vn_lock(fvp, LK_EXCLUSIVE | LK_RETRY);
1475 		unp = VTOUNIONFS(fvp);
1476 		if (unp == NULL)
1477 			error = ENOENT;
1478 		else if (unp->un_uppervp == NULLVP) {
1479 			switch (fvp->v_type) {
1480 			case VREG:
1481 				error = unionfs_copyfile(fvp, 1, fcnp->cn_cred, td);
1482 				break;
1483 			case VDIR:
1484 				error = unionfs_mkshadowdir(fdvp, fvp, fcnp, td);
1485 				break;
1486 			default:
1487 				error = ENODEV;
1488 				break;
1489 			}
1490 		}
1491 		VOP_UNLOCK(fvp);
1492 		if (unlock_fdvp)
1493 			VOP_UNLOCK(fdvp);
1494 		if (relock_tdvp)
1495 			vn_lock(tdvp, LK_EXCLUSIVE | LK_RETRY);
1496 		if (tvp != NULLVP)
1497 			vn_lock(tvp, LK_EXCLUSIVE | LK_RETRY);
1498 		/*
1499 		 * Since we've dropped tdvp's lock at some point in the copy
1500 		 * sequence above, force the caller to re-drive the lookup
1501 		 * in case the relationship between tdvp and tvp has changed.
1502 		 */
1503 		if (error == 0)
1504 			error = ERELOOKUP;
1505 		goto unionfs_rename_abort;
1506 	}
1507 
1508 	if (unp->un_lowervp != NULLVP)
1509 		fcnp->cn_flags |= DOWHITEOUT;
1510 	rfvp = unp->un_uppervp;
1511 	vref(rfvp);
1512 
1513 	VI_UNLOCK(fvp);
1514 
1515 	unp = VTOUNIONFS(tdvp);
1516 
1517 #ifdef UNIONFS_IDBG_RENAME
1518 	UNIONFS_INTERNAL_DEBUG("tdvp=%p, utdvp=%p, ltdvp=%p\n",
1519 	    tdvp, unp->un_uppervp, unp->un_lowervp);
1520 #endif
1521 	if (unp->un_uppervp == NULLVP) {
1522 		error = ENODEV;
1523 		goto unionfs_rename_abort;
1524 	}
1525 	rtdvp = unp->un_uppervp;
1526 	vref(rtdvp);
1527 
1528 	if (tvp != NULLVP) {
1529 		unp = VTOUNIONFS(tvp);
1530 		if (unp == NULL) {
1531 			error = ENOENT;
1532 			goto unionfs_rename_abort;
1533 		}
1534 #ifdef UNIONFS_IDBG_RENAME
1535 		UNIONFS_INTERNAL_DEBUG("tvp=%p, utvp=%p, ltvp=%p\n",
1536 		    tvp, unp->un_uppervp, unp->un_lowervp);
1537 #endif
1538 		if (unp->un_uppervp == NULLVP)
1539 			rtvp = NULLVP;
1540 		else {
1541 			if (tvp->v_type == VDIR) {
1542 				error = EINVAL;
1543 				goto unionfs_rename_abort;
1544 			}
1545 			rtvp = unp->un_uppervp;
1546 			vref(rtvp);
1547 		}
1548 	}
1549 
1550 	if (rfvp == rtvp)
1551 		goto unionfs_rename_abort;
1552 
1553 	error = VOP_RENAME(rfdvp, rfvp, fcnp, rtdvp, rtvp, tcnp);
1554 
1555 	if (error == 0) {
1556 		if (rtvp != NULLVP && rtvp->v_type == VDIR)
1557 			cache_purge(tdvp);
1558 		if (fvp->v_type == VDIR && fdvp != tdvp)
1559 			cache_purge(fdvp);
1560 	}
1561 
1562 	if (tdvp != rtdvp)
1563 		vrele(tdvp);
1564 	if (tvp != rtvp && tvp != NULLVP) {
1565 		if (rtvp == NULLVP)
1566 			vput(tvp);
1567 		else
1568 			vrele(tvp);
1569 	}
1570 	if (fdvp != rfdvp)
1571 		vrele(fdvp);
1572 	if (fvp != rfvp)
1573 		vrele(fvp);
1574 
1575 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1576 
1577 	return (error);
1578 
1579 unionfs_rename_abort:
1580 	vput(tdvp);
1581 	if (tdvp != rtdvp)
1582 		vrele(rtdvp);
1583 	if (tvp != NULLVP) {
1584 		if (tdvp != tvp)
1585 			vput(tvp);
1586 		else
1587 			vrele(tvp);
1588 	}
1589 	if (tvp != rtvp && rtvp != NULLVP)
1590 		vrele(rtvp);
1591 	if (fdvp != rfdvp)
1592 		vrele(rfdvp);
1593 	if (fvp != rfvp)
1594 		vrele(rfvp);
1595 	vrele(fdvp);
1596 	vrele(fvp);
1597 
1598 	UNIONFS_INTERNAL_DEBUG("unionfs_rename: leave (%d)\n", error);
1599 
1600 	return (error);
1601 }
1602 
1603 static int
1604 unionfs_mkdir(struct vop_mkdir_args *ap)
1605 {
1606 	struct unionfs_node *dunp;
1607 	struct componentname *cnp;
1608 	struct vnode   *dvp;
1609 	struct vnode   *udvp;
1610 	struct vnode   *uvp;
1611 	struct vattr	va;
1612 	int		error;
1613 	int		lkflags;
1614 
1615 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: enter\n");
1616 
1617 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1618 
1619 	error = EROFS;
1620 	dvp = ap->a_dvp;
1621 	dunp = VTOUNIONFS(dvp);
1622 	cnp = ap->a_cnp;
1623 	lkflags = cnp->cn_lkflags;
1624 	udvp = dunp->un_uppervp;
1625 
1626 	if (udvp != NULLVP) {
1627 		/* check opaque */
1628 		if (!(cnp->cn_flags & ISWHITEOUT)) {
1629 			error = VOP_GETATTR(udvp, &va, cnp->cn_cred);
1630 			if (error != 0)
1631 				goto unionfs_mkdir_cleanup;
1632 			if ((va.va_flags & OPAQUE) != 0)
1633 				cnp->cn_flags |= ISWHITEOUT;
1634 		}
1635 
1636 		int udvp_lkflags;
1637 		bool uvp_created = false;
1638 		unionfs_forward_vop_start(udvp, &udvp_lkflags);
1639 		error = VOP_MKDIR(udvp, &uvp, cnp, ap->a_vap);
1640 		if (error == 0)
1641 			uvp_created = true;
1642 		if (__predict_false(unionfs_forward_vop_finish(dvp, udvp,
1643 		    udvp_lkflags)) && error == 0)
1644 			error = ENOENT;
1645 		if (error == 0) {
1646 			VOP_UNLOCK(uvp);
1647 			cnp->cn_lkflags = LK_EXCLUSIVE;
1648 			error = unionfs_nodeget(dvp->v_mount, uvp, NULLVP,
1649 			    dvp, ap->a_vpp, cnp);
1650 			vrele(uvp);
1651 			cnp->cn_lkflags = lkflags;
1652 		} else if (uvp_created)
1653 			vput(uvp);
1654 	}
1655 
1656 unionfs_mkdir_cleanup:
1657 	UNIONFS_INTERNAL_DEBUG("unionfs_mkdir: leave (%d)\n", error);
1658 
1659 	return (error);
1660 }
1661 
1662 static int
1663 unionfs_rmdir(struct vop_rmdir_args *ap)
1664 {
1665 	struct unionfs_node *dunp;
1666 	struct unionfs_node *unp;
1667 	struct unionfs_mount *ump;
1668 	struct componentname *cnp;
1669 	struct thread  *td;
1670 	struct vnode   *udvp;
1671 	struct vnode   *uvp;
1672 	struct vnode   *lvp;
1673 	int		error;
1674 
1675 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: enter\n");
1676 
1677 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1678 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1679 
1680 	error = 0;
1681 	dunp = VTOUNIONFS(ap->a_dvp);
1682 	unp = VTOUNIONFS(ap->a_vp);
1683 	cnp = ap->a_cnp;
1684 	td = curthread;
1685 	udvp = dunp->un_uppervp;
1686 	uvp = unp->un_uppervp;
1687 	lvp = unp->un_lowervp;
1688 
1689 	if (udvp == NULLVP)
1690 		return (EROFS);
1691 
1692 	if (udvp == uvp)
1693 		return (EOPNOTSUPP);
1694 
1695 	if (uvp != NULLVP) {
1696 		if (lvp != NULLVP) {
1697 			/*
1698 			 * We need to keep dvp and vp's upper vnodes locked
1699 			 * going into the VOP_RMDIR() call, but the empty
1700 			 * directory check also requires the lower vnode lock.
1701 			 * For this third, cross-filesystem lock we use a
1702 			 * similar approach taken by various FS' VOP_RENAME
1703 			 * implementations (which require 2-4 vnode locks).
1704 			 * First we attempt a NOWAIT acquisition, then if
1705 			 * that fails we drops the other two vnode locks,
1706 			 * acquire lvp's lock in the normal fashion to reduce
1707 			 * the likelihood of spinning on it in the future,
1708 			 * then drop, reacquire the other locks, and return
1709 			 * ERELOOKUP to re-drive the lookup in case the dvp->
1710 			 * vp relationship has changed.
1711 			 */
1712 			if (vn_lock(lvp, LK_SHARED | LK_NOWAIT) != 0) {
1713 				VOP_UNLOCK(ap->a_vp);
1714 				VOP_UNLOCK(ap->a_dvp);
1715 				vn_lock(lvp, LK_SHARED | LK_RETRY);
1716 				VOP_UNLOCK(lvp);
1717 				vn_lock(ap->a_dvp, LK_EXCLUSIVE | LK_RETRY);
1718 				vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY);
1719 				return (ERELOOKUP);
1720 			}
1721 			error = unionfs_check_rmdir(ap->a_vp, cnp->cn_cred, td);
1722 			/*
1723 			 * It's possible for a direct operation on the lower FS
1724 			 * to make the lower directory non-empty after we drop
1725 			 * the lock, but it's also possible for the upper-layer
1726 			 * VOP_RMDIR to relock udvp/uvp which would lead to
1727 			 * LOR if we kept lvp locked across that call.
1728 			 */
1729 			VOP_UNLOCK(lvp);
1730 			if (error != 0)
1731 				return (error);
1732 		}
1733 		ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount);
1734 		if (ump->um_whitemode == UNIONFS_WHITE_ALWAYS || lvp != NULLVP)
1735 			cnp->cn_flags |= (DOWHITEOUT | IGNOREWHITEOUT);
1736 		int udvp_lkflags, uvp_lkflags;
1737 		unionfs_forward_vop_start_pair(udvp, &udvp_lkflags,
1738 		    uvp, &uvp_lkflags);
1739 		error = VOP_RMDIR(udvp, uvp, cnp);
1740 		unionfs_forward_vop_finish_pair(ap->a_dvp, udvp, udvp_lkflags,
1741 		    ap->a_vp, uvp, uvp_lkflags);
1742 	} else if (lvp != NULLVP) {
1743 		error = unionfs_mkwhiteout(ap->a_dvp, ap->a_vp, cnp, td,
1744 		    unp->un_path, unp->un_pathlen);
1745 	}
1746 
1747 	if (error == 0) {
1748 		cache_purge(ap->a_dvp);
1749 		cache_purge(ap->a_vp);
1750 	}
1751 
1752 	UNIONFS_INTERNAL_DEBUG("unionfs_rmdir: leave (%d)\n", error);
1753 
1754 	return (error);
1755 }
1756 
1757 static int
1758 unionfs_symlink(struct vop_symlink_args *ap)
1759 {
1760 	struct unionfs_node *dunp;
1761 	struct componentname *cnp;
1762 	struct vnode   *udvp;
1763 	struct vnode   *uvp;
1764 	int		error;
1765 	int		lkflags;
1766 
1767 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: enter\n");
1768 
1769 	KASSERT_UNIONFS_VNODE(ap->a_dvp);
1770 
1771 	error = EROFS;
1772 	dunp = VTOUNIONFS(ap->a_dvp);
1773 	cnp = ap->a_cnp;
1774 	lkflags = cnp->cn_lkflags;
1775 	udvp = dunp->un_uppervp;
1776 
1777 	if (udvp != NULLVP) {
1778 		int udvp_lkflags;
1779 		bool uvp_created = false;
1780 		unionfs_forward_vop_start(udvp, &udvp_lkflags);
1781 		error = VOP_SYMLINK(udvp, &uvp, cnp, ap->a_vap, ap->a_target);
1782 		if (error == 0)
1783 			uvp_created = true;
1784 		if (__predict_false(unionfs_forward_vop_finish(ap->a_dvp, udvp,
1785 		    udvp_lkflags)) && error == 0)
1786 			error = ENOENT;
1787 		if (error == 0) {
1788 			VOP_UNLOCK(uvp);
1789 			cnp->cn_lkflags = LK_EXCLUSIVE;
1790 			error = unionfs_nodeget(ap->a_dvp->v_mount, uvp, NULLVP,
1791 			    ap->a_dvp, ap->a_vpp, cnp);
1792 			vrele(uvp);
1793 			cnp->cn_lkflags = lkflags;
1794 		} else if (uvp_created)
1795 			vput(uvp);
1796 	}
1797 
1798 	UNIONFS_INTERNAL_DEBUG("unionfs_symlink: leave (%d)\n", error);
1799 
1800 	return (error);
1801 }
1802 
1803 static int
1804 unionfs_readdir(struct vop_readdir_args *ap)
1805 {
1806 	struct unionfs_node *unp;
1807 	struct unionfs_node_status *unsp;
1808 	struct uio     *uio;
1809 	struct vnode   *vp;
1810 	struct vnode   *uvp;
1811 	struct vnode   *lvp;
1812 	struct thread  *td;
1813 	struct vattr    va;
1814 
1815 	uint64_t	*cookies_bk;
1816 	int		error;
1817 	int		eofflag;
1818 	int		lkflags;
1819 	int		ncookies_bk;
1820 	int		uio_offset_bk;
1821 	enum unionfs_lkupgrade lkstatus;
1822 
1823 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: enter\n");
1824 
1825 	KASSERT_UNIONFS_VNODE(ap->a_vp);
1826 
1827 	error = 0;
1828 	eofflag = 0;
1829 	uio_offset_bk = 0;
1830 	uio = ap->a_uio;
1831 	uvp = NULLVP;
1832 	lvp = NULLVP;
1833 	td = uio->uio_td;
1834 	ncookies_bk = 0;
1835 	cookies_bk = NULL;
1836 
1837 	vp = ap->a_vp;
1838 	if (vp->v_type != VDIR)
1839 		return (ENOTDIR);
1840 
1841 	/*
1842 	 * If the vnode is reclaimed while upgrading, we can't safely use unp
1843 	 * or do anything else unionfs- specific.
1844 	 */
1845 	lkstatus = unionfs_upgrade_lock(vp);
1846 	if (lkstatus == UNIONFS_LKUPGRADE_DOOMED)
1847 		error = EBADF;
1848 	if (error == 0) {
1849 		unp = VTOUNIONFS(vp);
1850 		uvp = unp->un_uppervp;
1851 		lvp = unp->un_lowervp;
1852 		/* check the open count. unionfs needs open before readdir. */
1853 		unionfs_get_node_status(unp, td, &unsp);
1854 		if ((uvp != NULLVP && unsp->uns_upper_opencnt <= 0) ||
1855 			(lvp != NULLVP && unsp->uns_lower_opencnt <= 0)) {
1856 			unionfs_tryrem_node_status(unp, unsp);
1857 			error = EBADF;
1858 		}
1859 	}
1860 	unionfs_downgrade_lock(vp, lkstatus);
1861 	if (error != 0)
1862 		goto unionfs_readdir_exit;
1863 
1864 	/* check opaque */
1865 	if (uvp != NULLVP && lvp != NULLVP) {
1866 		if ((error = VOP_GETATTR(uvp, &va, ap->a_cred)) != 0)
1867 			goto unionfs_readdir_exit;
1868 		if (va.va_flags & OPAQUE)
1869 			lvp = NULLVP;
1870 	}
1871 
1872 	/* upper only */
1873 	if (uvp != NULLVP && lvp == NULLVP) {
1874 		unionfs_forward_vop_start(uvp, &lkflags);
1875 		error = VOP_READDIR(uvp, uio, ap->a_cred, ap->a_eofflag,
1876 		    ap->a_ncookies, ap->a_cookies);
1877 		if (unionfs_forward_vop_finish(vp, uvp, lkflags))
1878 			error = error ? error : ENOENT;
1879 		else
1880 			unsp->uns_readdir_status = 0;
1881 
1882 		goto unionfs_readdir_exit;
1883 	}
1884 
1885 	/* lower only */
1886 	if (uvp == NULLVP && lvp != NULLVP) {
1887 		unionfs_forward_vop_start(lvp, &lkflags);
1888 		error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1889 		    ap->a_ncookies, ap->a_cookies);
1890 		if (unionfs_forward_vop_finish(vp, lvp, lkflags))
1891 			error = error ? error : ENOENT;
1892 		else
1893 			unsp->uns_readdir_status = 2;
1894 
1895 		goto unionfs_readdir_exit;
1896 	}
1897 
1898 	/*
1899 	 * readdir upper and lower
1900 	 */
1901 	KASSERT(uvp != NULLVP, ("unionfs_readdir: null upper vp"));
1902 	KASSERT(lvp != NULLVP, ("unionfs_readdir: null lower vp"));
1903 
1904 	if (uio->uio_offset == 0)
1905 		unsp->uns_readdir_status = 0;
1906 
1907 	if (unsp->uns_readdir_status == 0) {
1908 		/* read upper */
1909 		unionfs_forward_vop_start(uvp, &lkflags);
1910 		error = VOP_READDIR(uvp, uio, ap->a_cred, &eofflag,
1911 				    ap->a_ncookies, ap->a_cookies);
1912 		if (unionfs_forward_vop_finish(vp, uvp, lkflags) && error == 0)
1913 			error = ENOENT;
1914 		if (error != 0 || eofflag == 0)
1915 			goto unionfs_readdir_exit;
1916 		unsp->uns_readdir_status = 1;
1917 
1918 		/*
1919 		 * UFS(and other FS) needs size of uio_resid larger than
1920 		 * DIRBLKSIZ.
1921 		 * size of DIRBLKSIZ equals DEV_BSIZE.
1922 		 * (see: ufs/ufs/ufs_vnops.c ufs_readdir func , ufs/ufs/dir.h)
1923 		 */
1924 		if (uio->uio_resid <= (uio->uio_resid & (DEV_BSIZE -1)))
1925 			goto unionfs_readdir_exit;
1926 
1927 		/*
1928 		 * Backup cookies.
1929 		 * It prepares to readdir in lower.
1930 		 */
1931 		if (ap->a_ncookies != NULL) {
1932 			ncookies_bk = *(ap->a_ncookies);
1933 			*(ap->a_ncookies) = 0;
1934 		}
1935 		if (ap->a_cookies != NULL) {
1936 			cookies_bk = *(ap->a_cookies);
1937 			*(ap->a_cookies) = NULL;
1938 		}
1939 	}
1940 
1941 	/* initialize for readdir in lower */
1942 	if (unsp->uns_readdir_status == 1) {
1943 		unsp->uns_readdir_status = 2;
1944 		/*
1945 		 * Backup uio_offset. See the comment after the
1946 		 * VOP_READDIR call on the lower layer.
1947 		 */
1948 		uio_offset_bk = uio->uio_offset;
1949 		uio->uio_offset = 0;
1950 	}
1951 
1952 	lvp = unionfs_lock_lvp(vp, &lkflags);
1953 	if (lvp == NULL) {
1954 		error = ENOENT;
1955 		goto unionfs_readdir_exit;
1956 	}
1957 
1958 	/* read lower */
1959 	error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag,
1960 			    ap->a_ncookies, ap->a_cookies);
1961 
1962 
1963 	unp = unionfs_unlock_lvp(vp, lvp, lkflags);
1964 	if (unp == NULL && error == 0)
1965 		error = ENOENT;
1966 
1967 
1968 	/*
1969 	 * We can't return an uio_offset of 0: this would trigger an
1970 	 * infinite loop, because the next call to unionfs_readdir would
1971 	 * always restart with the upper layer (uio_offset == 0) and
1972 	 * always return some data.
1973 	 *
1974 	 * This happens when the lower layer root directory is removed.
1975 	 * (A root directory deleting of unionfs should not be permitted.
1976 	 *  But current VFS can not do it.)
1977 	 */
1978 	if (uio->uio_offset == 0)
1979 		uio->uio_offset = uio_offset_bk;
1980 
1981 	if (cookies_bk != NULL) {
1982 		/* merge cookies */
1983 		int		size;
1984 		uint64_t         *newcookies, *pos;
1985 
1986 		size = *(ap->a_ncookies) + ncookies_bk;
1987 		newcookies = (uint64_t *) malloc(size * sizeof(*newcookies),
1988 		    M_TEMP, M_WAITOK);
1989 		pos = newcookies;
1990 
1991 		memcpy(pos, cookies_bk, ncookies_bk * sizeof(*newcookies));
1992 		pos += ncookies_bk;
1993 		memcpy(pos, *(ap->a_cookies),
1994 		    *(ap->a_ncookies) * sizeof(*newcookies));
1995 		free(cookies_bk, M_TEMP);
1996 		free(*(ap->a_cookies), M_TEMP);
1997 		*(ap->a_ncookies) = size;
1998 		*(ap->a_cookies) = newcookies;
1999 	}
2000 
2001 unionfs_readdir_exit:
2002 	if (error != 0 && ap->a_eofflag != NULL)
2003 		*(ap->a_eofflag) = 1;
2004 
2005 	UNIONFS_INTERNAL_DEBUG("unionfs_readdir: leave (%d)\n", error);
2006 
2007 	return (error);
2008 }
2009 
2010 static int
2011 unionfs_readlink(struct vop_readlink_args *ap)
2012 {
2013 	struct unionfs_node *unp;
2014 	struct vnode   *vp;
2015 	int error;
2016 
2017 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: enter\n");
2018 
2019 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2020 
2021 	unp = VTOUNIONFS(ap->a_vp);
2022 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2023 
2024 	error = VOP_READLINK(vp, ap->a_uio, ap->a_cred);
2025 
2026 	UNIONFS_INTERNAL_DEBUG("unionfs_readlink: leave (%d)\n", error);
2027 
2028 	return (error);
2029 }
2030 
2031 static int
2032 unionfs_getwritemount(struct vop_getwritemount_args *ap)
2033 {
2034 	struct unionfs_node *unp;
2035 	struct vnode   *uvp;
2036 	struct vnode   *vp, *ovp;
2037 	int		error;
2038 
2039 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: enter\n");
2040 
2041 	error = 0;
2042 	vp = ap->a_vp;
2043 	uvp = NULLVP;
2044 
2045 	VI_LOCK(vp);
2046 	unp = VTOUNIONFS(vp);
2047 	if (unp != NULL)
2048 		uvp = unp->un_uppervp;
2049 
2050 	/*
2051 	 * If our node has no upper vnode, check the parent directory.
2052 	 * We may be initiating a write operation that will produce a
2053 	 * new upper vnode through CoW.
2054 	 */
2055 	if (uvp == NULLVP && unp != NULL) {
2056 		ovp = vp;
2057 		vp = unp->un_dvp;
2058 		/*
2059 		 * Only the root vnode should have an empty parent, but it
2060 		 * should not have an empty uppervp, so we shouldn't get here.
2061 		 */
2062 		VNASSERT(vp != NULL, ovp, ("%s: NULL parent vnode", __func__));
2063 		VI_UNLOCK(ovp);
2064 		VI_LOCK(vp);
2065 		unp = VTOUNIONFS(vp);
2066 		if (unp != NULL)
2067 			uvp = unp->un_uppervp;
2068 		if (uvp == NULLVP)
2069 			error = EACCES;
2070 	}
2071 
2072 	if (uvp != NULLVP) {
2073 		vholdnz(uvp);
2074 		VI_UNLOCK(vp);
2075 		error = VOP_GETWRITEMOUNT(uvp, ap->a_mpp);
2076 		vdrop(uvp);
2077 	} else {
2078 		VI_UNLOCK(vp);
2079 		*(ap->a_mpp) = NULL;
2080 	}
2081 
2082 	UNIONFS_INTERNAL_DEBUG("unionfs_getwritemount: leave (%d)\n", error);
2083 
2084 	return (error);
2085 }
2086 
2087 static int
2088 unionfs_inactive(struct vop_inactive_args *ap)
2089 {
2090 	ap->a_vp->v_object = NULL;
2091 	vrecycle(ap->a_vp);
2092 	return (0);
2093 }
2094 
2095 static int
2096 unionfs_reclaim(struct vop_reclaim_args *ap)
2097 {
2098 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: enter\n"); */
2099 
2100 	unionfs_noderem(ap->a_vp);
2101 
2102 	/* UNIONFS_INTERNAL_DEBUG("unionfs_reclaim: leave\n"); */
2103 
2104 	return (0);
2105 }
2106 
2107 static int
2108 unionfs_print(struct vop_print_args *ap)
2109 {
2110 	struct unionfs_node *unp;
2111 	/* struct unionfs_node_status *unsp; */
2112 
2113 	unp = VTOUNIONFS(ap->a_vp);
2114 	/* unionfs_get_node_status(unp, curthread, &unsp); */
2115 
2116 	printf("unionfs_vp=%p, uppervp=%p, lowervp=%p\n",
2117 	    ap->a_vp, unp->un_uppervp, unp->un_lowervp);
2118 	/*
2119 	printf("unionfs opencnt: uppervp=%d, lowervp=%d\n",
2120 	    unsp->uns_upper_opencnt, unsp->uns_lower_opencnt);
2121 	*/
2122 
2123 	if (unp->un_uppervp != NULLVP)
2124 		vn_printf(unp->un_uppervp, "unionfs: upper ");
2125 	if (unp->un_lowervp != NULLVP)
2126 		vn_printf(unp->un_lowervp, "unionfs: lower ");
2127 
2128 	return (0);
2129 }
2130 
2131 static int
2132 unionfs_lock(struct vop_lock1_args *ap)
2133 {
2134 	struct unionfs_node *unp;
2135 	struct vnode   *vp;
2136 	struct vnode   *tvp;
2137 	int		error;
2138 	int		flags;
2139 	bool		lvp_locked;
2140 
2141 	error = 0;
2142 	flags = ap->a_flags;
2143 	vp = ap->a_vp;
2144 
2145 	if (LK_RELEASE == (flags & LK_TYPE_MASK) || !(flags & LK_TYPE_MASK))
2146 		return (VOP_UNLOCK_FLAGS(vp, flags | LK_RELEASE));
2147 
2148 unionfs_lock_restart:
2149 	/*
2150 	 * We currently need the interlock here to ensure we can safely
2151 	 * access the unionfs vnode's private data.  We may be able to
2152 	 * eliminate this extra locking by instead using vfs_smr_enter()
2153 	 * and vn_load_v_data_smr() here in conjunction with an SMR UMA
2154 	 * zone for unionfs nodes.
2155 	 */
2156 	if ((flags & LK_INTERLOCK) == 0)
2157 		VI_LOCK(vp);
2158 	else
2159 		flags &= ~LK_INTERLOCK;
2160 
2161 	unp = VTOUNIONFS(vp);
2162 	if (unp == NULL) {
2163 		VI_UNLOCK(vp);
2164 		ap->a_flags = flags;
2165 		return (vop_stdlock(ap));
2166 	}
2167 
2168 	if (unp->un_uppervp != NULL) {
2169 		tvp = unp->un_uppervp;
2170 		lvp_locked = false;
2171 	} else {
2172 		tvp = unp->un_lowervp;
2173 		lvp_locked = true;
2174 	}
2175 
2176 	/*
2177 	 * During unmount, the root vnode lock may be taken recursively,
2178 	 * because it may share the same v_vnlock field as the vnode covered by
2179 	 * the unionfs mount.  The covered vnode is locked across VFS_UNMOUNT(),
2180 	 * and the same lock may be taken recursively here during vflush()
2181 	 * issued by unionfs_unmount().
2182 	 */
2183 	if ((flags & LK_TYPE_MASK) == LK_EXCLUSIVE &&
2184 	    (vp->v_vflag & VV_ROOT) != 0)
2185 		flags |= LK_CANRECURSE;
2186 
2187 	vholdnz(tvp);
2188 	VI_UNLOCK(vp);
2189 	error = VOP_LOCK(tvp, flags);
2190 	vdrop(tvp);
2191 	if (error == 0 && (lvp_locked || VTOUNIONFS(vp) == NULL)) {
2192 		/*
2193 		 * After dropping the interlock above, there exists a window
2194 		 * in which another thread may acquire the lower vnode lock
2195 		 * and then either doom the unionfs vnode or create an upper
2196 		 * vnode.  In either case, we will effectively be holding the
2197 		 * wrong lock, so we must drop the lower vnode lock and
2198 		 * restart the lock operation.
2199 		 *
2200 		 * If unp is not already NULL, we assume that we can safely
2201 		 * access it because we currently hold lvp's lock.
2202 		 * unionfs_noderem() acquires lvp's lock before freeing
2203 		 * the vnode private data, ensuring it can't be concurrently
2204 		 * freed while we are using it here.  Likewise,
2205 		 * unionfs_node_update() acquires lvp's lock before installing
2206 		 * an upper vnode.  Without those guarantees, we would need to
2207 		 * reacquire the vnode interlock here.
2208 		 * Note that unionfs_noderem() doesn't acquire lvp's lock if
2209 		 * this is the root vnode, but the root vnode should always
2210 		 * have an upper vnode and therefore we should never use its
2211 		 * lower vnode lock here.
2212 		 */
2213 		unp = VTOUNIONFS(vp);
2214 		if (unp == NULL || unp->un_uppervp != NULLVP) {
2215 			VOP_UNLOCK(tvp);
2216 			/*
2217 			 * If we previously held the lock, the upgrade may
2218 			 * have temporarily dropped the lock, in which case
2219 			 * concurrent dooming or copy-up will necessitate
2220 			 * acquiring a different lock.  Since we never held
2221 			 * the new lock, LK_UPGRADE must be cleared here to
2222 			 * avoid triggering a lockmgr panic.
2223 			 */
2224 			if (flags & LK_UPGRADE)
2225 				flags = (flags & ~LK_TYPE_MASK) | LK_EXCLUSIVE;
2226 			VNASSERT((flags & LK_DOWNGRADE) == 0, vp,
2227 			    ("%s: vnode doomed during downgrade", __func__));
2228 			goto unionfs_lock_restart;
2229 		}
2230 	}
2231 
2232 	return (error);
2233 }
2234 
2235 static int
2236 unionfs_unlock(struct vop_unlock_args *ap)
2237 {
2238 	struct vnode   *vp;
2239 	struct vnode   *tvp;
2240 	struct unionfs_node *unp;
2241 	int		error;
2242 
2243 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2244 
2245 	vp = ap->a_vp;
2246 
2247 	unp = VTOUNIONFS(vp);
2248 	if (unp == NULL)
2249 		return (vop_stdunlock(ap));
2250 
2251 	tvp = (unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp);
2252 
2253 	vholdnz(tvp);
2254 	error = VOP_UNLOCK(tvp);
2255 	vdrop(tvp);
2256 
2257 	return (error);
2258 }
2259 
2260 static int
2261 unionfs_pathconf(struct vop_pathconf_args *ap)
2262 {
2263 	struct unionfs_node *unp;
2264 	struct vnode   *vp;
2265 
2266 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2267 
2268 	unp = VTOUNIONFS(ap->a_vp);
2269 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2270 
2271 	return (VOP_PATHCONF(vp, ap->a_name, ap->a_retval));
2272 }
2273 
2274 static int
2275 unionfs_advlock(struct vop_advlock_args *ap)
2276 {
2277 	struct unionfs_node *unp;
2278 	struct unionfs_node_status *unsp;
2279 	struct vnode   *vp;
2280 	struct vnode   *uvp;
2281 	struct thread  *td;
2282 	int error;
2283 
2284 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: enter\n");
2285 
2286 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2287 
2288 	vp = ap->a_vp;
2289 	td = curthread;
2290 
2291 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2292 
2293 	unp = VTOUNIONFS(ap->a_vp);
2294 	uvp = unp->un_uppervp;
2295 
2296 	if (uvp == NULLVP) {
2297 		error = unionfs_copyfile(ap->a_vp, 1, td->td_ucred, td);
2298 		if (error != 0)
2299 			goto unionfs_advlock_abort;
2300 		uvp = unp->un_uppervp;
2301 
2302 		unionfs_get_node_status(unp, td, &unsp);
2303 		if (unsp->uns_lower_opencnt > 0) {
2304 			/* try reopen the vnode */
2305 			error = VOP_OPEN(uvp, unsp->uns_lower_openmode,
2306 				td->td_ucred, td, NULL);
2307 			if (error)
2308 				goto unionfs_advlock_abort;
2309 			unsp->uns_upper_opencnt++;
2310 			VOP_CLOSE(unp->un_lowervp, unsp->uns_lower_openmode,
2311 			    td->td_ucred, td);
2312 			unsp->uns_lower_opencnt--;
2313 		} else
2314 			unionfs_tryrem_node_status(unp, unsp);
2315 	}
2316 
2317 	VOP_UNLOCK(vp);
2318 
2319 	error = VOP_ADVLOCK(uvp, ap->a_id, ap->a_op, ap->a_fl, ap->a_flags);
2320 
2321 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2322 
2323 	return error;
2324 
2325 unionfs_advlock_abort:
2326 	VOP_UNLOCK(vp);
2327 
2328 	UNIONFS_INTERNAL_DEBUG("unionfs_advlock: leave (%d)\n", error);
2329 
2330 	return error;
2331 }
2332 
2333 static int
2334 unionfs_strategy(struct vop_strategy_args *ap)
2335 {
2336 	struct unionfs_node *unp;
2337 	struct vnode   *vp;
2338 
2339 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2340 
2341 	unp = VTOUNIONFS(ap->a_vp);
2342 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2343 
2344 #ifdef DIAGNOSTIC
2345 	if (vp == NULLVP)
2346 		panic("unionfs_strategy: nullvp");
2347 
2348 	if (ap->a_bp->b_iocmd == BIO_WRITE && vp == unp->un_lowervp)
2349 		panic("unionfs_strategy: writing to lowervp");
2350 #endif
2351 
2352 	return (VOP_STRATEGY(vp, ap->a_bp));
2353 }
2354 
2355 static int
2356 unionfs_getacl(struct vop_getacl_args *ap)
2357 {
2358 	struct unionfs_node *unp;
2359 	struct vnode   *vp;
2360 	int		error;
2361 
2362 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2363 
2364 	unp = VTOUNIONFS(ap->a_vp);
2365 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2366 
2367 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: enter\n");
2368 
2369 	error = VOP_GETACL(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2370 
2371 	UNIONFS_INTERNAL_DEBUG("unionfs_getacl: leave (%d)\n", error);
2372 
2373 	return (error);
2374 }
2375 
2376 static int
2377 unionfs_setacl(struct vop_setacl_args *ap)
2378 {
2379 	struct unionfs_node *unp;
2380 	struct vnode   *uvp;
2381 	struct vnode   *lvp;
2382 	struct thread  *td;
2383 	int		error;
2384 
2385 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: enter\n");
2386 
2387 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2388 
2389 	error = EROFS;
2390 	unp = VTOUNIONFS(ap->a_vp);
2391 	uvp = unp->un_uppervp;
2392 	lvp = unp->un_lowervp;
2393 	td = ap->a_td;
2394 
2395 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2396 		return (EROFS);
2397 
2398 	if (uvp == NULLVP && lvp->v_type == VREG) {
2399 		if ((error = unionfs_copyfile(ap->a_vp, 1, ap->a_cred, td)) != 0)
2400 			return (error);
2401 		uvp = unp->un_uppervp;
2402 	}
2403 
2404 	if (uvp != NULLVP) {
2405 		int lkflags;
2406 		unionfs_forward_vop_start(uvp, &lkflags);
2407 		error = VOP_SETACL(uvp, ap->a_type, ap->a_aclp, ap->a_cred, td);
2408 		unionfs_forward_vop_finish(ap->a_vp, uvp, lkflags);
2409 	}
2410 
2411 	UNIONFS_INTERNAL_DEBUG("unionfs_setacl: leave (%d)\n", error);
2412 
2413 	return (error);
2414 }
2415 
2416 static int
2417 unionfs_aclcheck(struct vop_aclcheck_args *ap)
2418 {
2419 	struct unionfs_node *unp;
2420 	struct vnode   *vp;
2421 	int		error;
2422 
2423 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: enter\n");
2424 
2425 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2426 
2427 	unp = VTOUNIONFS(ap->a_vp);
2428 	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2429 
2430 	error = VOP_ACLCHECK(vp, ap->a_type, ap->a_aclp, ap->a_cred, ap->a_td);
2431 
2432 	UNIONFS_INTERNAL_DEBUG("unionfs_aclcheck: leave (%d)\n", error);
2433 
2434 	return (error);
2435 }
2436 
2437 static int
2438 unionfs_openextattr(struct vop_openextattr_args *ap)
2439 {
2440 	struct unionfs_node *unp;
2441 	struct vnode   *vp;
2442 	struct vnode   *tvp;
2443 	int		error;
2444 
2445 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2446 
2447 	vp = ap->a_vp;
2448 	unp = VTOUNIONFS(vp);
2449 	tvp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
2450 
2451 	if ((tvp == unp->un_uppervp && (unp->un_flag & UNIONFS_OPENEXTU)) ||
2452 	    (tvp == unp->un_lowervp && (unp->un_flag & UNIONFS_OPENEXTL)))
2453 		return (EBUSY);
2454 
2455 	error = VOP_OPENEXTATTR(tvp, ap->a_cred, ap->a_td);
2456 
2457 	if (error == 0) {
2458 		if (vn_lock(vp, LK_UPGRADE) != 0)
2459 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2460 		if (!VN_IS_DOOMED(vp)) {
2461 			if (tvp == unp->un_uppervp)
2462 				unp->un_flag |= UNIONFS_OPENEXTU;
2463 			else
2464 				unp->un_flag |= UNIONFS_OPENEXTL;
2465 		}
2466 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2467 	}
2468 
2469 	return (error);
2470 }
2471 
2472 static int
2473 unionfs_closeextattr(struct vop_closeextattr_args *ap)
2474 {
2475 	struct unionfs_node *unp;
2476 	struct vnode   *vp;
2477 	struct vnode   *tvp;
2478 	int		error;
2479 
2480 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2481 
2482 	vp = ap->a_vp;
2483 	unp = VTOUNIONFS(vp);
2484 	tvp = NULLVP;
2485 
2486 	if (unp->un_flag & UNIONFS_OPENEXTU)
2487 		tvp = unp->un_uppervp;
2488 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2489 		tvp = unp->un_lowervp;
2490 
2491 	if (tvp == NULLVP)
2492 		return (EOPNOTSUPP);
2493 
2494 	error = VOP_CLOSEEXTATTR(tvp, ap->a_commit, ap->a_cred, ap->a_td);
2495 
2496 	if (error == 0) {
2497 		if (vn_lock(vp, LK_UPGRADE) != 0)
2498 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2499 		if (!VN_IS_DOOMED(vp)) {
2500 			if (tvp == unp->un_uppervp)
2501 				unp->un_flag &= ~UNIONFS_OPENEXTU;
2502 			else
2503 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2504 		}
2505 		vn_lock(vp, LK_DOWNGRADE | LK_RETRY);
2506 	}
2507 
2508 	return (error);
2509 }
2510 
2511 static int
2512 unionfs_getextattr(struct vop_getextattr_args *ap)
2513 {
2514 	struct unionfs_node *unp;
2515 	struct vnode   *vp;
2516 
2517 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2518 
2519 	unp = VTOUNIONFS(ap->a_vp);
2520 	vp = NULLVP;
2521 
2522 	if (unp->un_flag & UNIONFS_OPENEXTU)
2523 		vp = unp->un_uppervp;
2524 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2525 		vp = unp->un_lowervp;
2526 
2527 	if (vp == NULLVP)
2528 		return (EOPNOTSUPP);
2529 
2530 	return (VOP_GETEXTATTR(vp, ap->a_attrnamespace, ap->a_name,
2531 	    ap->a_uio, ap->a_size, ap->a_cred, ap->a_td));
2532 }
2533 
2534 static int
2535 unionfs_setextattr(struct vop_setextattr_args *ap)
2536 {
2537 	struct unionfs_node *unp;
2538 	struct vnode   *uvp;
2539 	struct vnode   *lvp;
2540 	struct vnode   *ovp;
2541 	struct ucred   *cred;
2542 	struct thread  *td;
2543 	int		error;
2544 
2545 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2546 
2547 	error = EROFS;
2548 	unp = VTOUNIONFS(ap->a_vp);
2549 	uvp = unp->un_uppervp;
2550 	lvp = unp->un_lowervp;
2551 	ovp = NULLVP;
2552 	cred = ap->a_cred;
2553 	td = ap->a_td;
2554 
2555 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: enter (un_flag=%x)\n",
2556 	    unp->un_flag);
2557 
2558 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2559 		return (EROFS);
2560 
2561 	if (unp->un_flag & UNIONFS_OPENEXTU)
2562 		ovp = unp->un_uppervp;
2563 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2564 		ovp = unp->un_lowervp;
2565 
2566 	if (ovp == NULLVP)
2567 		return (EOPNOTSUPP);
2568 
2569 	if (ovp == lvp && lvp->v_type == VREG) {
2570 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2571 		if (uvp == NULLVP &&
2572 		    (error = unionfs_copyfile(ap->a_vp, 1, cred, td)) != 0) {
2573 unionfs_setextattr_reopen:
2574 			unp = VTOUNIONFS(ap->a_vp);
2575 			if (unp != NULL && (unp->un_flag & UNIONFS_OPENEXTL) &&
2576 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2577 #ifdef DIAGNOSTIC
2578 				panic("unionfs: VOP_OPENEXTATTR failed");
2579 #endif
2580 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2581 			}
2582 			goto unionfs_setextattr_abort;
2583 		}
2584 		uvp = unp->un_uppervp;
2585 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2586 			goto unionfs_setextattr_reopen;
2587 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2588 		unp->un_flag |= UNIONFS_OPENEXTU;
2589 		ovp = uvp;
2590 	}
2591 
2592 	if (ovp == uvp) {
2593 		int lkflags;
2594 		unionfs_forward_vop_start(ovp, &lkflags);
2595 		error = VOP_SETEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2596 		    ap->a_uio, cred, td);
2597 		unionfs_forward_vop_finish(ap->a_vp, ovp, lkflags);
2598 	}
2599 
2600 unionfs_setextattr_abort:
2601 	UNIONFS_INTERNAL_DEBUG("unionfs_setextattr: leave (%d)\n", error);
2602 
2603 	return (error);
2604 }
2605 
2606 static int
2607 unionfs_listextattr(struct vop_listextattr_args *ap)
2608 {
2609 	struct unionfs_node *unp;
2610 	struct vnode *vp;
2611 
2612 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2613 
2614 	unp = VTOUNIONFS(ap->a_vp);
2615 	vp = NULLVP;
2616 
2617 	if (unp->un_flag & UNIONFS_OPENEXTU)
2618 		vp = unp->un_uppervp;
2619 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2620 		vp = unp->un_lowervp;
2621 
2622 	if (vp == NULLVP)
2623 		return (EOPNOTSUPP);
2624 
2625 	return (VOP_LISTEXTATTR(vp, ap->a_attrnamespace, ap->a_uio,
2626 	    ap->a_size, ap->a_cred, ap->a_td));
2627 }
2628 
2629 static int
2630 unionfs_deleteextattr(struct vop_deleteextattr_args *ap)
2631 {
2632 	struct unionfs_node *unp;
2633 	struct vnode   *uvp;
2634 	struct vnode   *lvp;
2635 	struct vnode   *ovp;
2636 	struct ucred   *cred;
2637 	struct thread  *td;
2638 	int		error;
2639 
2640 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2641 
2642 	error = EROFS;
2643 	unp = VTOUNIONFS(ap->a_vp);
2644 	uvp = unp->un_uppervp;
2645 	lvp = unp->un_lowervp;
2646 	ovp = NULLVP;
2647 	cred = ap->a_cred;
2648 	td = ap->a_td;
2649 
2650 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: enter (un_flag=%x)\n",
2651 	    unp->un_flag);
2652 
2653 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2654 		return (EROFS);
2655 
2656 	if (unp->un_flag & UNIONFS_OPENEXTU)
2657 		ovp = unp->un_uppervp;
2658 	else if (unp->un_flag & UNIONFS_OPENEXTL)
2659 		ovp = unp->un_lowervp;
2660 
2661 	if (ovp == NULLVP)
2662 		return (EOPNOTSUPP);
2663 
2664 	if (ovp == lvp && lvp->v_type == VREG) {
2665 		VOP_CLOSEEXTATTR(lvp, 0, cred, td);
2666 		if (uvp == NULLVP &&
2667 		    (error = unionfs_copyfile(ap->a_vp, 1, cred, td)) != 0) {
2668 unionfs_deleteextattr_reopen:
2669 			unp = VTOUNIONFS(ap->a_vp);
2670 			if (unp != NULL && (unp->un_flag & UNIONFS_OPENEXTL) &&
2671 			    VOP_OPENEXTATTR(lvp, cred, td)) {
2672 #ifdef DIAGNOSTIC
2673 				panic("unionfs: VOP_OPENEXTATTR failed");
2674 #endif
2675 				unp->un_flag &= ~UNIONFS_OPENEXTL;
2676 			}
2677 			goto unionfs_deleteextattr_abort;
2678 		}
2679 		uvp = unp->un_uppervp;
2680 		if ((error = VOP_OPENEXTATTR(uvp, cred, td)) != 0)
2681 			goto unionfs_deleteextattr_reopen;
2682 		unp->un_flag &= ~UNIONFS_OPENEXTL;
2683 		unp->un_flag |= UNIONFS_OPENEXTU;
2684 		ovp = uvp;
2685 	}
2686 
2687 	if (ovp == uvp)
2688 		error = VOP_DELETEEXTATTR(ovp, ap->a_attrnamespace, ap->a_name,
2689 		    ap->a_cred, ap->a_td);
2690 
2691 unionfs_deleteextattr_abort:
2692 	UNIONFS_INTERNAL_DEBUG("unionfs_deleteextattr: leave (%d)\n", error);
2693 
2694 	return (error);
2695 }
2696 
2697 static int
2698 unionfs_setlabel(struct vop_setlabel_args *ap)
2699 {
2700 	struct unionfs_node *unp;
2701 	struct vnode   *uvp;
2702 	struct vnode   *lvp;
2703 	struct thread  *td;
2704 	int		error;
2705 
2706 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: enter\n");
2707 
2708 	KASSERT_UNIONFS_VNODE(ap->a_vp);
2709 
2710 	error = EROFS;
2711 	unp = VTOUNIONFS(ap->a_vp);
2712 	uvp = unp->un_uppervp;
2713 	lvp = unp->un_lowervp;
2714 	td = ap->a_td;
2715 
2716 	if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
2717 		return (EROFS);
2718 
2719 	if (uvp == NULLVP && lvp->v_type == VREG) {
2720 		if ((error = unionfs_copyfile(ap->a_vp, 1, ap->a_cred, td)) != 0)
2721 			return (error);
2722 		uvp = unp->un_uppervp;
2723 	}
2724 
2725 	if (uvp != NULLVP)
2726 		error = VOP_SETLABEL(uvp, ap->a_label, ap->a_cred, td);
2727 
2728 	UNIONFS_INTERNAL_DEBUG("unionfs_setlabel: leave (%d)\n", error);
2729 
2730 	return (error);
2731 }
2732 
2733 static int
2734 unionfs_vptofh(struct vop_vptofh_args *ap)
2735 {
2736 	return (EOPNOTSUPP);
2737 }
2738 
2739 static int
2740 unionfs_add_writecount(struct vop_add_writecount_args *ap)
2741 {
2742 	struct vnode *tvp, *vp;
2743 	struct unionfs_node *unp;
2744 	int error, writerefs __diagused;
2745 
2746 	vp = ap->a_vp;
2747 	unp = VTOUNIONFS(vp);
2748 	tvp = unp->un_uppervp;
2749 	KASSERT(tvp != NULL,
2750 	    ("%s: adding write ref without upper vnode", __func__));
2751 	error = VOP_ADD_WRITECOUNT(tvp, ap->a_inc);
2752 	if (error != 0)
2753 		return (error);
2754 	/*
2755 	 * We need to track the write refs we've passed to the underlying
2756 	 * vnodes so that we can undo them in case we are forcibly unmounted.
2757 	 */
2758 	writerefs = atomic_fetchadd_int(&vp->v_writecount, ap->a_inc);
2759 	/* text refs are bypassed to lowervp */
2760 	VNASSERT(writerefs >= 0, vp,
2761 	    ("%s: invalid write count %d", __func__, writerefs));
2762 	VNASSERT(writerefs + ap->a_inc >= 0, vp,
2763 	    ("%s: invalid write count inc %d + %d", __func__,
2764 	    writerefs, ap->a_inc));
2765 	return (0);
2766 }
2767 
2768 static int
2769 unionfs_vput_pair(struct vop_vput_pair_args *ap)
2770 {
2771 	struct mount *mp;
2772 	struct vnode *dvp, *vp, **vpp, *lvp, *uvp, *tvp, *tdvp, *tempvp;
2773 	struct unionfs_node *dunp, *unp;
2774 	int error, res;
2775 
2776 	dvp = ap->a_dvp;
2777 	vpp = ap->a_vpp;
2778 	vp = NULLVP;
2779 	lvp = NULLVP;
2780 	uvp = NULLVP;
2781 	tvp = NULLVP;
2782 	unp = NULL;
2783 
2784 	dunp = VTOUNIONFS(dvp);
2785 	if (dunp->un_uppervp != NULL)
2786 		tdvp = dunp->un_uppervp;
2787 	else
2788 		tdvp = dunp->un_lowervp;
2789 
2790 	/*
2791 	 * Underlying vnodes should be locked because the encompassing unionfs
2792 	 * node is locked, but will not be referenced, as the reference will
2793 	 * only be on the unionfs node.  Reference them now so that the vput()s
2794 	 * performed by VOP_VPUT_PAIR() will have a reference to drop.
2795 	 */
2796 	vref(tdvp);
2797 
2798 	if (vpp != NULL)
2799 		vp = *vpp;
2800 
2801 	if (vp != NULLVP) {
2802 		unp = VTOUNIONFS(vp);
2803 		uvp = unp->un_uppervp;
2804 		lvp = unp->un_lowervp;
2805 		if (uvp != NULLVP)
2806 			tvp = uvp;
2807 		else
2808 			tvp = lvp;
2809 		vref(tvp);
2810 
2811 		/*
2812 		 * If we're being asked to return a locked child vnode, then
2813 		 * we may need to create a replacement vnode in case the
2814 		 * original is reclaimed while the lock is dropped.  In that
2815 		 * case we'll need to ensure the mount and the underlying
2816 		 * vnodes aren't also recycled during that window.
2817 		 */
2818 		if (!ap->a_unlock_vp) {
2819 			vhold(vp);
2820 			if (uvp != NULLVP)
2821 				vhold(uvp);
2822 			if (lvp != NULLVP)
2823 				vhold(lvp);
2824 			mp = vp->v_mount;
2825 			vfs_ref(mp);
2826 		}
2827 	}
2828 
2829 	ASSERT_VOP_LOCKED(tdvp, __func__);
2830 	ASSERT_VOP_LOCKED(tvp, __func__);
2831 
2832 	if (tdvp == dunp->un_uppervp && tvp != NULLVP && tvp == lvp) {
2833 		vput(tvp);
2834 		vput(tdvp);
2835 		res = 0;
2836 	} else {
2837 		res = VOP_VPUT_PAIR(tdvp, tvp != NULLVP ? &tvp : NULL, true);
2838 	}
2839 
2840 	ASSERT_VOP_UNLOCKED(tdvp, __func__);
2841 	ASSERT_VOP_UNLOCKED(tvp, __func__);
2842 
2843 	/*
2844 	 * VOP_VPUT_PAIR() dropped the references we added to the underlying
2845 	 * vnodes, now drop the caller's reference to the unionfs vnodes.
2846 	 */
2847 	if (vp != NULLVP && ap->a_unlock_vp)
2848 		vrele(vp);
2849 	vrele(dvp);
2850 
2851 	if (vp == NULLVP || ap->a_unlock_vp)
2852 		return (res);
2853 
2854 	/*
2855 	 * We're being asked to return a locked vnode.  At this point, the
2856 	 * underlying vnodes have been unlocked, so vp may have been reclaimed.
2857 	 */
2858 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2859 	if (vp->v_data == NULL && vfs_busy(mp, MBF_NOWAIT) == 0) {
2860 		vput(vp);
2861 		error = unionfs_nodeget(mp, uvp, lvp, dvp, &tempvp, NULL);
2862 		if (error == 0) {
2863 			vn_lock(tempvp, LK_EXCLUSIVE | LK_RETRY);
2864 			*vpp = tempvp;
2865 		} else
2866 			vget(vp, LK_EXCLUSIVE | LK_RETRY);
2867 		vfs_unbusy(mp);
2868 	}
2869 	if (lvp != NULLVP)
2870 		vdrop(lvp);
2871 	if (uvp != NULLVP)
2872 		vdrop(uvp);
2873 	vdrop(vp);
2874 	vfs_rel(mp);
2875 
2876 	return (res);
2877 }
2878 
2879 static int
2880 unionfs_set_text(struct vop_set_text_args *ap)
2881 {
2882 	struct vnode *tvp;
2883 	struct unionfs_node *unp;
2884 	int error;
2885 
2886 	/*
2887 	 * We assume text refs are managed against lvp/uvp through the
2888 	 * executable mapping backed by its VM object.  We therefore don't
2889 	 * need to track leased text refs in the case of a forcible unmount.
2890 	 */
2891 	unp = VTOUNIONFS(ap->a_vp);
2892 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2893 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2894 	error = VOP_SET_TEXT(tvp);
2895 	return (error);
2896 }
2897 
2898 static int
2899 unionfs_unset_text(struct vop_unset_text_args *ap)
2900 {
2901 	struct vnode *tvp;
2902 	struct unionfs_node *unp;
2903 
2904 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2905 	unp = VTOUNIONFS(ap->a_vp);
2906 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2907 	VOP_UNSET_TEXT_CHECKED(tvp);
2908 	return (0);
2909 }
2910 
2911 static int
2912 unionfs_unp_bind(struct vop_unp_bind_args *ap)
2913 {
2914 	struct vnode *tvp;
2915 	struct unionfs_node *unp;
2916 
2917 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2918 	unp = VTOUNIONFS(ap->a_vp);
2919 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2920 	VOP_UNP_BIND(tvp, ap->a_unpcb);
2921 	return (0);
2922 }
2923 
2924 static int
2925 unionfs_unp_connect(struct vop_unp_connect_args *ap)
2926 {
2927 	struct vnode *tvp;
2928 	struct unionfs_node *unp;
2929 
2930 	ASSERT_VOP_LOCKED(ap->a_vp, __func__);
2931 	unp = VTOUNIONFS(ap->a_vp);
2932 	tvp = unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp;
2933 	VOP_UNP_CONNECT(tvp, ap->a_unpcb);
2934 	return (0);
2935 }
2936 
2937 static int
2938 unionfs_unp_detach(struct vop_unp_detach_args *ap)
2939 {
2940 	struct vnode *tvp;
2941 	struct unionfs_node *unp;
2942 
2943 	tvp = NULL;
2944 	/*
2945 	 * VOP_UNP_DETACH() is not guaranteed to be called with the unionfs
2946 	 * vnode locked, so we take the interlock to prevent a concurrent
2947 	 * unmount from freeing the unionfs private data.
2948 	 */
2949 	VI_LOCK(ap->a_vp);
2950 	unp = VTOUNIONFS(ap->a_vp);
2951 	if (unp != NULL) {
2952 		tvp = unp->un_uppervp != NULL ?
2953 		    unp->un_uppervp : unp->un_lowervp;
2954 		/*
2955 		 * Hold the target vnode to prevent a concurrent unionfs
2956 		 * unmount from causing it to be recycled once the interlock
2957 		 * is dropped.
2958 		 */
2959 		vholdnz(tvp);
2960 	}
2961 	VI_UNLOCK(ap->a_vp);
2962 	if (tvp != NULL) {
2963 		VOP_UNP_DETACH(tvp);
2964 		vdrop(tvp);
2965 	}
2966 	return (0);
2967 }
2968 
2969 struct vop_vector unionfs_vnodeops = {
2970 	.vop_default =		&default_vnodeops,
2971 
2972 	.vop_access =		unionfs_access,
2973 	.vop_aclcheck =		unionfs_aclcheck,
2974 	.vop_advlock =		unionfs_advlock,
2975 	.vop_bmap =		VOP_EOPNOTSUPP,
2976 	.vop_cachedlookup =	unionfs_lookup,
2977 	.vop_close =		unionfs_close,
2978 	.vop_closeextattr =	unionfs_closeextattr,
2979 	.vop_create =		unionfs_create,
2980 	.vop_deleteextattr =	unionfs_deleteextattr,
2981 	.vop_fsync =		unionfs_fsync,
2982 	.vop_getacl =		unionfs_getacl,
2983 	.vop_getattr =		unionfs_getattr,
2984 	.vop_getextattr =	unionfs_getextattr,
2985 	.vop_getwritemount =	unionfs_getwritemount,
2986 	.vop_inactive =		unionfs_inactive,
2987 	.vop_need_inactive =	vop_stdneed_inactive,
2988 	.vop_islocked =		vop_stdislocked,
2989 	.vop_ioctl =		unionfs_ioctl,
2990 	.vop_link =		unionfs_link,
2991 	.vop_listextattr =	unionfs_listextattr,
2992 	.vop_lock1 =		unionfs_lock,
2993 	.vop_lookup =		vfs_cache_lookup,
2994 	.vop_mkdir =		unionfs_mkdir,
2995 	.vop_mknod =		unionfs_mknod,
2996 	.vop_open =		unionfs_open,
2997 	.vop_openextattr =	unionfs_openextattr,
2998 	.vop_pathconf =		unionfs_pathconf,
2999 	.vop_poll =		unionfs_poll,
3000 	.vop_print =		unionfs_print,
3001 	.vop_read =		unionfs_read,
3002 	.vop_readdir =		unionfs_readdir,
3003 	.vop_readlink =		unionfs_readlink,
3004 	.vop_reclaim =		unionfs_reclaim,
3005 	.vop_remove =		unionfs_remove,
3006 	.vop_rename =		unionfs_rename,
3007 	.vop_rmdir =		unionfs_rmdir,
3008 	.vop_setacl =		unionfs_setacl,
3009 	.vop_setattr =		unionfs_setattr,
3010 	.vop_setextattr =	unionfs_setextattr,
3011 	.vop_setlabel =		unionfs_setlabel,
3012 	.vop_strategy =		unionfs_strategy,
3013 	.vop_symlink =		unionfs_symlink,
3014 	.vop_unlock =		unionfs_unlock,
3015 	.vop_whiteout =		unionfs_whiteout,
3016 	.vop_write =		unionfs_write,
3017 	.vop_vptofh =		unionfs_vptofh,
3018 	.vop_add_writecount =	unionfs_add_writecount,
3019 	.vop_vput_pair =	unionfs_vput_pair,
3020 	.vop_set_text =		unionfs_set_text,
3021 	.vop_unset_text = 	unionfs_unset_text,
3022 	.vop_unp_bind =		unionfs_unp_bind,
3023 	.vop_unp_connect =	unionfs_unp_connect,
3024 	.vop_unp_detach =	unionfs_unp_detach,
3025 };
3026 VFS_VOP_VECTOR_REGISTER(unionfs_vnodeops);
3027