xref: /illumos-gate/usr/src/uts/common/fs/nfs/nfs4_vfsops.c (revision d8a7fe16f62711cdc5c4267da8b34ff24a6b668c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  *	Copyright (c) 1983,1984,1985,1986,1987,1988,1989  AT&T.
28  *	All Rights Reserved
29  */
30 
31 #include <sys/param.h>
32 #include <sys/types.h>
33 #include <sys/systm.h>
34 #include <sys/cred.h>
35 #include <sys/vfs.h>
36 #include <sys/vfs_opreg.h>
37 #include <sys/vnode.h>
38 #include <sys/pathname.h>
39 #include <sys/sysmacros.h>
40 #include <sys/kmem.h>
41 #include <sys/mkdev.h>
42 #include <sys/mount.h>
43 #include <sys/statvfs.h>
44 #include <sys/errno.h>
45 #include <sys/debug.h>
46 #include <sys/cmn_err.h>
47 #include <sys/utsname.h>
48 #include <sys/bootconf.h>
49 #include <sys/modctl.h>
50 #include <sys/acl.h>
51 #include <sys/flock.h>
52 #include <sys/time.h>
53 #include <sys/disp.h>
54 #include <sys/policy.h>
55 #include <sys/socket.h>
56 #include <sys/netconfig.h>
57 #include <sys/dnlc.h>
58 #include <sys/list.h>
59 #include <sys/mntent.h>
60 #include <sys/tsol/label.h>
61 
62 #include <rpc/types.h>
63 #include <rpc/auth.h>
64 #include <rpc/rpcsec_gss.h>
65 #include <rpc/clnt.h>
66 
67 #include <nfs/nfs.h>
68 #include <nfs/nfs_clnt.h>
69 #include <nfs/mount.h>
70 #include <nfs/nfs_acl.h>
71 
72 #include <fs/fs_subr.h>
73 
74 #include <nfs/nfs4.h>
75 #include <nfs/rnode4.h>
76 #include <nfs/nfs4_clnt.h>
77 #include <sys/fs/autofs.h>
78 
79 #include <sys/sdt.h>
80 
81 
82 /*
83  * Arguments passed to thread to free data structures from forced unmount.
84  */
85 
86 typedef struct {
87 	vfs_t	*fm_vfsp;
88 	int	fm_flag;
89 	cred_t	*fm_cr;
90 } freemountargs_t;
91 
92 static void	async_free_mount(vfs_t *, int, cred_t *);
93 static void	nfs4_free_mount(vfs_t *, int, cred_t *);
94 static void	nfs4_free_mount_thread(freemountargs_t *);
95 static int nfs4_chkdup_servinfo4(servinfo4_t *, servinfo4_t *);
96 
97 /*
98  * From rpcsec module (common/rpcsec).
99  */
100 extern int sec_clnt_loadinfo(struct sec_data *, struct sec_data **, model_t);
101 extern void sec_clnt_freeinfo(struct sec_data *);
102 
103 /*
104  * The order and contents of this structure must be kept in sync with that of
105  * rfsreqcnt_v4_tmpl in nfs_stats.c
106  */
107 static char *rfsnames_v4[] = {
108 	"null", "compound", "reserved",	"access", "close", "commit", "create",
109 	"delegpurge", "delegreturn", "getattr",	"getfh", "link", "lock",
110 	"lockt", "locku", "lookup", "lookupp", "nverify", "open", "openattr",
111 	"open_confirm",	"open_downgrade", "putfh", "putpubfh", "putrootfh",
112 	"read", "readdir", "readlink", "remove", "rename", "renew",
113 	"restorefh", "savefh", "secinfo", "setattr", "setclientid",
114 	"setclientid_confirm", "verify", "write"
115 };
116 
117 /*
118  * nfs4_max_mount_retry is the number of times the client will redrive
119  * a mount compound before giving up and returning failure.  The intent
120  * is to redrive mount compounds which fail NFS4ERR_STALE so that
121  * if a component of the server path being mounted goes stale, it can
122  * "recover" by redriving the mount compund (LOOKUP ops).  This recovery
123  * code is needed outside of the recovery framework because mount is a
124  * special case.  The client doesn't create vnodes/rnodes for components
125  * of the server path being mounted.  The recovery code recovers real
126  * client objects, not STALE FHs which map to components of the server
127  * path being mounted.
128  *
129  * We could just fail the mount on the first time, but that would
130  * instantly trigger failover (from nfs4_mount), and the client should
131  * try to re-lookup the STALE FH before doing failover.  The easiest
132  * way to "re-lookup" is to simply redrive the mount compound.
133  */
134 static int nfs4_max_mount_retry = 2;
135 
136 /*
137  * nfs4 vfs operations.
138  */
139 int		nfs4_mount(vfs_t *, vnode_t *, struct mounta *, cred_t *);
140 static int	nfs4_unmount(vfs_t *, int, cred_t *);
141 static int	nfs4_root(vfs_t *, vnode_t **);
142 static int	nfs4_statvfs(vfs_t *, struct statvfs64 *);
143 static int	nfs4_sync(vfs_t *, short, cred_t *);
144 static int	nfs4_vget(vfs_t *, vnode_t **, fid_t *);
145 static int	nfs4_mountroot(vfs_t *, whymountroot_t);
146 static void	nfs4_freevfs(vfs_t *);
147 
148 static int	nfs4rootvp(vnode_t **, vfs_t *, struct servinfo4 *,
149 		    int, cred_t *, zone_t *);
150 
151 vfsops_t	*nfs4_vfsops;
152 
153 int nfs4_vfsinit(void);
154 void nfs4_vfsfini(void);
155 static void nfs4setclientid_init(void);
156 static void nfs4setclientid_fini(void);
157 static void nfs4setclientid_otw(mntinfo4_t *, servinfo4_t *,  cred_t *,
158 		struct nfs4_server *, nfs4_error_t *, int *);
159 static void	destroy_nfs4_server(nfs4_server_t *);
160 static void	remove_mi(nfs4_server_t *, mntinfo4_t *);
161 
162 extern void nfs4_ephemeral_init(void);
163 extern void nfs4_ephemeral_fini(void);
164 
165 /* referral related routines */
166 static servinfo4_t *copy_svp(servinfo4_t *);
167 static void free_knconf_contents(struct knetconfig *k);
168 static char *extract_referral_point(const char *, int);
169 static void setup_newsvpath(servinfo4_t *, int);
170 static void update_servinfo4(servinfo4_t *, fs_location4 *,
171 		struct nfs_fsl_info *, char *, int);
172 
173 /*
174  * Initialize the vfs structure
175  */
176 
177 static int nfs4fstyp;
178 
179 
180 /*
181  * Debug variable to check for rdma based
182  * transport startup and cleanup. Controlled
183  * through /etc/system. Off by default.
184  */
185 extern int rdma_debug;
186 
187 int
188 nfs4init(int fstyp, char *name)
189 {
190 	static const fs_operation_def_t nfs4_vfsops_template[] = {
191 		VFSNAME_MOUNT,		{ .vfs_mount = nfs4_mount },
192 		VFSNAME_UNMOUNT,	{ .vfs_unmount = nfs4_unmount },
193 		VFSNAME_ROOT,		{ .vfs_root = nfs4_root },
194 		VFSNAME_STATVFS,	{ .vfs_statvfs = nfs4_statvfs },
195 		VFSNAME_SYNC,		{ .vfs_sync = nfs4_sync },
196 		VFSNAME_VGET,		{ .vfs_vget = nfs4_vget },
197 		VFSNAME_MOUNTROOT,	{ .vfs_mountroot = nfs4_mountroot },
198 		VFSNAME_FREEVFS,	{ .vfs_freevfs = nfs4_freevfs },
199 		NULL,			NULL
200 	};
201 	int error;
202 
203 	nfs4_vfsops = NULL;
204 	nfs4_vnodeops = NULL;
205 	nfs4_trigger_vnodeops = NULL;
206 
207 	error = vfs_setfsops(fstyp, nfs4_vfsops_template, &nfs4_vfsops);
208 	if (error != 0) {
209 		zcmn_err(GLOBAL_ZONEID, CE_WARN,
210 		    "nfs4init: bad vfs ops template");
211 		goto out;
212 	}
213 
214 	error = vn_make_ops(name, nfs4_vnodeops_template, &nfs4_vnodeops);
215 	if (error != 0) {
216 		zcmn_err(GLOBAL_ZONEID, CE_WARN,
217 		    "nfs4init: bad vnode ops template");
218 		goto out;
219 	}
220 
221 	error = vn_make_ops("nfs4_trigger", nfs4_trigger_vnodeops_template,
222 	    &nfs4_trigger_vnodeops);
223 	if (error != 0) {
224 		zcmn_err(GLOBAL_ZONEID, CE_WARN,
225 		    "nfs4init: bad trigger vnode ops template");
226 		goto out;
227 	}
228 
229 	nfs4fstyp = fstyp;
230 	(void) nfs4_vfsinit();
231 	(void) nfs4_init_dot_entries();
232 
233 out:
234 	if (error) {
235 		if (nfs4_trigger_vnodeops != NULL)
236 			vn_freevnodeops(nfs4_trigger_vnodeops);
237 
238 		if (nfs4_vnodeops != NULL)
239 			vn_freevnodeops(nfs4_vnodeops);
240 
241 		(void) vfs_freevfsops_by_type(fstyp);
242 	}
243 
244 	return (error);
245 }
246 
247 void
248 nfs4fini(void)
249 {
250 	(void) nfs4_destroy_dot_entries();
251 	nfs4_vfsfini();
252 }
253 
254 /*
255  * Create a new sec_data structure to store AUTH_DH related data:
256  * netname, syncaddr, knetconfig. There is no AUTH_F_RPCTIMESYNC
257  * flag set for NFS V4 since we are avoiding to contact the rpcbind
258  * daemon and is using the IP time service (IPPORT_TIMESERVER).
259  *
260  * sec_data can be freed by sec_clnt_freeinfo().
261  */
262 static struct sec_data *
263 create_authdh_data(char *netname, int nlen, struct netbuf *syncaddr,
264 		struct knetconfig *knconf) {
265 	struct sec_data *secdata;
266 	dh_k4_clntdata_t *data;
267 	char *pf, *p;
268 
269 	if (syncaddr == NULL || syncaddr->buf == NULL || nlen == 0)
270 		return (NULL);
271 
272 	secdata = kmem_alloc(sizeof (*secdata), KM_SLEEP);
273 	secdata->flags = 0;
274 
275 	data = kmem_alloc(sizeof (*data), KM_SLEEP);
276 
277 	data->syncaddr.maxlen = syncaddr->maxlen;
278 	data->syncaddr.len = syncaddr->len;
279 	data->syncaddr.buf = (char *)kmem_alloc(syncaddr->len, KM_SLEEP);
280 	bcopy(syncaddr->buf, data->syncaddr.buf, syncaddr->len);
281 
282 	/*
283 	 * duplicate the knconf information for the
284 	 * new opaque data.
285 	 */
286 	data->knconf = kmem_alloc(sizeof (*knconf), KM_SLEEP);
287 	*data->knconf = *knconf;
288 	pf = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
289 	p = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
290 	bcopy(knconf->knc_protofmly, pf, KNC_STRSIZE);
291 	bcopy(knconf->knc_proto, p, KNC_STRSIZE);
292 	data->knconf->knc_protofmly = pf;
293 	data->knconf->knc_proto = p;
294 
295 	/* move server netname to the sec_data structure */
296 	data->netname = kmem_alloc(nlen, KM_SLEEP);
297 	bcopy(netname, data->netname, nlen);
298 	data->netnamelen = (int)nlen;
299 
300 	secdata->secmod = AUTH_DH;
301 	secdata->rpcflavor = AUTH_DH;
302 	secdata->data = (caddr_t)data;
303 
304 	return (secdata);
305 }
306 
307 /*
308  * Returns (deep) copy of sec_data_t. Allocates all memory required; caller
309  * is responsible for freeing.
310  */
311 sec_data_t *
312 copy_sec_data(sec_data_t *fsecdata) {
313 	sec_data_t *tsecdata;
314 
315 	if (fsecdata == NULL)
316 		return (NULL);
317 
318 	if (fsecdata->rpcflavor == AUTH_DH) {
319 		dh_k4_clntdata_t *fdata = (dh_k4_clntdata_t *)fsecdata->data;
320 
321 		if (fdata == NULL)
322 			return (NULL);
323 
324 		tsecdata = (sec_data_t *)create_authdh_data(fdata->netname,
325 		    fdata->netnamelen, &fdata->syncaddr, fdata->knconf);
326 
327 		return (tsecdata);
328 	}
329 
330 	tsecdata = kmem_zalloc(sizeof (sec_data_t), KM_SLEEP);
331 
332 	tsecdata->secmod = fsecdata->secmod;
333 	tsecdata->rpcflavor = fsecdata->rpcflavor;
334 	tsecdata->flags = fsecdata->flags;
335 	tsecdata->uid = fsecdata->uid;
336 
337 	if (fsecdata->rpcflavor == RPCSEC_GSS) {
338 		gss_clntdata_t *gcd = (gss_clntdata_t *)fsecdata->data;
339 
340 		tsecdata->data = (caddr_t)copy_sec_data_gss(gcd);
341 	} else {
342 		tsecdata->data = NULL;
343 	}
344 
345 	return (tsecdata);
346 }
347 
348 gss_clntdata_t *
349 copy_sec_data_gss(gss_clntdata_t *fdata)
350 {
351 	gss_clntdata_t *tdata;
352 
353 	if (fdata == NULL)
354 		return (NULL);
355 
356 	tdata = kmem_zalloc(sizeof (gss_clntdata_t), KM_SLEEP);
357 
358 	tdata->mechanism.length = fdata->mechanism.length;
359 	tdata->mechanism.elements = kmem_zalloc(fdata->mechanism.length,
360 	    KM_SLEEP);
361 	bcopy(fdata->mechanism.elements, tdata->mechanism.elements,
362 	    fdata->mechanism.length);
363 
364 	tdata->service = fdata->service;
365 
366 	(void) strcpy(tdata->uname, fdata->uname);
367 	(void) strcpy(tdata->inst, fdata->inst);
368 	(void) strcpy(tdata->realm, fdata->realm);
369 
370 	tdata->qop = fdata->qop;
371 
372 	return (tdata);
373 }
374 
375 static int
376 nfs4_chkdup_servinfo4(servinfo4_t *svp_head, servinfo4_t *svp)
377 {
378 	servinfo4_t *si;
379 
380 	/*
381 	 * Iterate over the servinfo4 list to make sure
382 	 * we do not have a duplicate. Skip any servinfo4
383 	 * that has been marked "NOT IN USE"
384 	 */
385 	for (si = svp_head; si; si = si->sv_next) {
386 		(void) nfs_rw_enter_sig(&si->sv_lock, RW_READER, 0);
387 		if (si->sv_flags & SV4_NOTINUSE) {
388 			nfs_rw_exit(&si->sv_lock);
389 			continue;
390 		}
391 		nfs_rw_exit(&si->sv_lock);
392 		if (si == svp)
393 			continue;
394 		if (si->sv_addr.len == svp->sv_addr.len &&
395 		    strcmp(si->sv_knconf->knc_protofmly,
396 		    svp->sv_knconf->knc_protofmly) == 0 &&
397 		    bcmp(si->sv_addr.buf, svp->sv_addr.buf,
398 		    si->sv_addr.len) == 0) {
399 			/* it's a duplicate */
400 			return (1);
401 		}
402 	}
403 	/* it's not a duplicate */
404 	return (0);
405 }
406 
407 void
408 nfs4_free_args(struct nfs_args *nargs)
409 {
410 	if (nargs->knconf) {
411 		if (nargs->knconf->knc_protofmly)
412 			kmem_free(nargs->knconf->knc_protofmly,
413 			    KNC_STRSIZE);
414 		if (nargs->knconf->knc_proto)
415 			kmem_free(nargs->knconf->knc_proto, KNC_STRSIZE);
416 		kmem_free(nargs->knconf, sizeof (*nargs->knconf));
417 		nargs->knconf = NULL;
418 	}
419 
420 	if (nargs->fh) {
421 		kmem_free(nargs->fh, strlen(nargs->fh) + 1);
422 		nargs->fh = NULL;
423 	}
424 
425 	if (nargs->hostname) {
426 		kmem_free(nargs->hostname, strlen(nargs->hostname) + 1);
427 		nargs->hostname = NULL;
428 	}
429 
430 	if (nargs->addr) {
431 		if (nargs->addr->buf) {
432 			ASSERT(nargs->addr->len);
433 			kmem_free(nargs->addr->buf, nargs->addr->len);
434 		}
435 		kmem_free(nargs->addr, sizeof (struct netbuf));
436 		nargs->addr = NULL;
437 	}
438 
439 	if (nargs->syncaddr) {
440 		ASSERT(nargs->syncaddr->len);
441 		if (nargs->syncaddr->buf) {
442 			ASSERT(nargs->syncaddr->len);
443 			kmem_free(nargs->syncaddr->buf, nargs->syncaddr->len);
444 		}
445 		kmem_free(nargs->syncaddr, sizeof (struct netbuf));
446 		nargs->syncaddr = NULL;
447 	}
448 
449 	if (nargs->netname) {
450 		kmem_free(nargs->netname, strlen(nargs->netname) + 1);
451 		nargs->netname = NULL;
452 	}
453 
454 	if (nargs->nfs_ext_u.nfs_extA.secdata) {
455 		sec_clnt_freeinfo(
456 		    nargs->nfs_ext_u.nfs_extA.secdata);
457 		nargs->nfs_ext_u.nfs_extA.secdata = NULL;
458 	}
459 }
460 
461 
462 int
463 nfs4_copyin(char *data, int datalen, struct nfs_args *nargs)
464 {
465 
466 	int error;
467 	size_t hlen;			/* length of hostname */
468 	size_t nlen;			/* length of netname */
469 	char netname[MAXNETNAMELEN+1];	/* server's netname */
470 	struct netbuf addr;		/* server's address */
471 	struct netbuf syncaddr;		/* AUTH_DES time sync addr */
472 	struct knetconfig *knconf;		/* transport structure */
473 	struct sec_data *secdata = NULL;	/* security data */
474 	STRUCT_DECL(nfs_args, args);		/* nfs mount arguments */
475 	STRUCT_DECL(knetconfig, knconf_tmp);
476 	STRUCT_DECL(netbuf, addr_tmp);
477 	int flags;
478 	char *p, *pf;
479 	struct pathname pn;
480 	char *userbufptr;
481 
482 
483 	bzero(nargs, sizeof (*nargs));
484 
485 	STRUCT_INIT(args, get_udatamodel());
486 	bzero(STRUCT_BUF(args), SIZEOF_STRUCT(nfs_args, DATAMODEL_NATIVE));
487 	if (copyin(data, STRUCT_BUF(args), MIN(datalen,
488 	    STRUCT_SIZE(args))))
489 		return (EFAULT);
490 
491 	nargs->wsize = STRUCT_FGET(args, wsize);
492 	nargs->rsize = STRUCT_FGET(args, rsize);
493 	nargs->timeo = STRUCT_FGET(args, timeo);
494 	nargs->retrans = STRUCT_FGET(args, retrans);
495 	nargs->acregmin = STRUCT_FGET(args, acregmin);
496 	nargs->acregmax = STRUCT_FGET(args, acregmax);
497 	nargs->acdirmin = STRUCT_FGET(args, acdirmin);
498 	nargs->acdirmax = STRUCT_FGET(args, acdirmax);
499 
500 	flags = STRUCT_FGET(args, flags);
501 	nargs->flags = flags;
502 
503 	addr.buf = NULL;
504 	syncaddr.buf = NULL;
505 
506 
507 	/*
508 	 * Allocate space for a knetconfig structure and
509 	 * its strings and copy in from user-land.
510 	 */
511 	knconf = kmem_zalloc(sizeof (*knconf), KM_SLEEP);
512 	STRUCT_INIT(knconf_tmp, get_udatamodel());
513 	if (copyin(STRUCT_FGETP(args, knconf), STRUCT_BUF(knconf_tmp),
514 	    STRUCT_SIZE(knconf_tmp))) {
515 		kmem_free(knconf, sizeof (*knconf));
516 		return (EFAULT);
517 	}
518 
519 	knconf->knc_semantics = STRUCT_FGET(knconf_tmp, knc_semantics);
520 	knconf->knc_protofmly = STRUCT_FGETP(knconf_tmp, knc_protofmly);
521 	knconf->knc_proto = STRUCT_FGETP(knconf_tmp, knc_proto);
522 	if (get_udatamodel() != DATAMODEL_LP64) {
523 		knconf->knc_rdev = expldev(STRUCT_FGET(knconf_tmp, knc_rdev));
524 	} else {
525 		knconf->knc_rdev = STRUCT_FGET(knconf_tmp, knc_rdev);
526 	}
527 
528 	pf = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
529 	p = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
530 	error = copyinstr(knconf->knc_protofmly, pf, KNC_STRSIZE, NULL);
531 	if (error) {
532 		kmem_free(pf, KNC_STRSIZE);
533 		kmem_free(p, KNC_STRSIZE);
534 		kmem_free(knconf, sizeof (*knconf));
535 		return (error);
536 	}
537 
538 	error = copyinstr(knconf->knc_proto, p, KNC_STRSIZE, NULL);
539 	if (error) {
540 		kmem_free(pf, KNC_STRSIZE);
541 		kmem_free(p, KNC_STRSIZE);
542 		kmem_free(knconf, sizeof (*knconf));
543 		return (error);
544 	}
545 
546 
547 	knconf->knc_protofmly = pf;
548 	knconf->knc_proto = p;
549 
550 	nargs->knconf = knconf;
551 
552 	/*
553 	 * Get server address
554 	 */
555 	STRUCT_INIT(addr_tmp, get_udatamodel());
556 	if (copyin(STRUCT_FGETP(args, addr), STRUCT_BUF(addr_tmp),
557 	    STRUCT_SIZE(addr_tmp))) {
558 		error = EFAULT;
559 		goto errout;
560 	}
561 
562 	nargs->addr = kmem_zalloc(sizeof (struct netbuf), KM_SLEEP);
563 	userbufptr = STRUCT_FGETP(addr_tmp, buf);
564 	addr.len = STRUCT_FGET(addr_tmp, len);
565 	addr.buf = kmem_alloc(addr.len, KM_SLEEP);
566 	addr.maxlen = addr.len;
567 	if (copyin(userbufptr, addr.buf, addr.len)) {
568 		kmem_free(addr.buf, addr.len);
569 		error = EFAULT;
570 		goto errout;
571 	}
572 	bcopy(&addr, nargs->addr, sizeof (struct netbuf));
573 
574 	/*
575 	 * Get the root fhandle
576 	 */
577 	error = pn_get(STRUCT_FGETP(args, fh), UIO_USERSPACE, &pn);
578 	if (error)
579 		goto errout;
580 
581 	/* Volatile fh: keep server paths, so use actual-size strings */
582 	nargs->fh = kmem_alloc(pn.pn_pathlen + 1, KM_SLEEP);
583 	bcopy(pn.pn_path, nargs->fh, pn.pn_pathlen);
584 	nargs->fh[pn.pn_pathlen] = '\0';
585 	pn_free(&pn);
586 
587 
588 	/*
589 	 * Get server's hostname
590 	 */
591 	if (flags & NFSMNT_HOSTNAME) {
592 		error = copyinstr(STRUCT_FGETP(args, hostname),
593 		    netname, sizeof (netname), &hlen);
594 		if (error)
595 			goto errout;
596 		nargs->hostname = kmem_zalloc(hlen, KM_SLEEP);
597 		(void) strcpy(nargs->hostname, netname);
598 
599 	} else {
600 		nargs->hostname = NULL;
601 	}
602 
603 
604 	/*
605 	 * If there are syncaddr and netname data, load them in. This is
606 	 * to support data needed for NFSV4 when AUTH_DH is the negotiated
607 	 * flavor via SECINFO. (instead of using MOUNT protocol in V3).
608 	 */
609 	netname[0] = '\0';
610 	if (flags & NFSMNT_SECURE) {
611 
612 		/* get syncaddr */
613 		STRUCT_INIT(addr_tmp, get_udatamodel());
614 		if (copyin(STRUCT_FGETP(args, syncaddr), STRUCT_BUF(addr_tmp),
615 		    STRUCT_SIZE(addr_tmp))) {
616 			error = EINVAL;
617 			goto errout;
618 		}
619 		userbufptr = STRUCT_FGETP(addr_tmp, buf);
620 		syncaddr.len = STRUCT_FGET(addr_tmp, len);
621 		syncaddr.buf = kmem_alloc(syncaddr.len, KM_SLEEP);
622 		syncaddr.maxlen = syncaddr.len;
623 		if (copyin(userbufptr, syncaddr.buf, syncaddr.len)) {
624 			kmem_free(syncaddr.buf, syncaddr.len);
625 			error = EFAULT;
626 			goto errout;
627 		}
628 
629 		nargs->syncaddr = kmem_alloc(sizeof (struct netbuf), KM_SLEEP);
630 		bcopy(&syncaddr, nargs->syncaddr, sizeof (struct netbuf));
631 
632 		/* get server's netname */
633 		if (copyinstr(STRUCT_FGETP(args, netname), netname,
634 		    sizeof (netname), &nlen)) {
635 			error = EFAULT;
636 			goto errout;
637 		}
638 
639 		netname[nlen] = '\0';
640 		nargs->netname = kmem_zalloc(nlen, KM_SLEEP);
641 		(void) strcpy(nargs->netname, netname);
642 	}
643 
644 	/*
645 	 * Get the extention data which has the security data structure.
646 	 * This includes data for AUTH_SYS as well.
647 	 */
648 	if (flags & NFSMNT_NEWARGS) {
649 		nargs->nfs_args_ext = STRUCT_FGET(args, nfs_args_ext);
650 		if (nargs->nfs_args_ext == NFS_ARGS_EXTA ||
651 		    nargs->nfs_args_ext == NFS_ARGS_EXTB) {
652 			/*
653 			 * Indicating the application is using the new
654 			 * sec_data structure to pass in the security
655 			 * data.
656 			 */
657 			if (STRUCT_FGETP(args,
658 			    nfs_ext_u.nfs_extA.secdata) != NULL) {
659 				error = sec_clnt_loadinfo(
660 				    (struct sec_data *)STRUCT_FGETP(args,
661 				    nfs_ext_u.nfs_extA.secdata),
662 				    &secdata, get_udatamodel());
663 			}
664 			nargs->nfs_ext_u.nfs_extA.secdata = secdata;
665 		}
666 	}
667 
668 	if (error)
669 		goto errout;
670 
671 	/*
672 	 * Failover support:
673 	 *
674 	 * We may have a linked list of nfs_args structures,
675 	 * which means the user is looking for failover.  If
676 	 * the mount is either not "read-only" or "soft",
677 	 * we want to bail out with EINVAL.
678 	 */
679 	if (nargs->nfs_args_ext == NFS_ARGS_EXTB)
680 		nargs->nfs_ext_u.nfs_extB.next =
681 		    STRUCT_FGETP(args, nfs_ext_u.nfs_extB.next);
682 
683 errout:
684 	if (error)
685 		nfs4_free_args(nargs);
686 
687 	return (error);
688 }
689 
690 
691 /*
692  * nfs mount vfsop
693  * Set up mount info record and attach it to vfs struct.
694  */
695 int
696 nfs4_mount(vfs_t *vfsp, vnode_t *mvp, struct mounta *uap, cred_t *cr)
697 {
698 	char *data = uap->dataptr;
699 	int error;
700 	vnode_t *rtvp;			/* the server's root */
701 	mntinfo4_t *mi;			/* mount info, pointed at by vfs */
702 	struct knetconfig *rdma_knconf;	/* rdma transport structure */
703 	rnode4_t *rp;
704 	struct servinfo4 *svp;		/* nfs server info */
705 	struct servinfo4 *svp_tail = NULL; /* previous nfs server info */
706 	struct servinfo4 *svp_head;	/* first nfs server info */
707 	struct servinfo4 *svp_2ndlast;	/* 2nd last in server info list */
708 	struct sec_data *secdata;	/* security data */
709 	struct nfs_args *args = NULL;
710 	int flags, addr_type, removed;
711 	zone_t *zone = nfs_zone();
712 	nfs4_error_t n4e;
713 	zone_t *mntzone = NULL;
714 
715 	if (secpolicy_fs_mount(cr, mvp, vfsp) != 0)
716 		return (EPERM);
717 	if (mvp->v_type != VDIR)
718 		return (ENOTDIR);
719 
720 	/*
721 	 * get arguments
722 	 *
723 	 * nfs_args is now versioned and is extensible, so
724 	 * uap->datalen might be different from sizeof (args)
725 	 * in a compatible situation.
726 	 */
727 more:
728 	if (!(uap->flags & MS_SYSSPACE)) {
729 		if (args == NULL)
730 			args = kmem_zalloc(sizeof (struct nfs_args), KM_SLEEP);
731 		else
732 			nfs4_free_args(args);
733 		error = nfs4_copyin(data, uap->datalen, args);
734 		if (error) {
735 			if (args) {
736 				kmem_free(args, sizeof (*args));
737 			}
738 			return (error);
739 		}
740 	} else {
741 		args = (struct nfs_args *)data;
742 	}
743 
744 	flags = args->flags;
745 
746 	/*
747 	 * If the request changes the locking type, disallow the remount,
748 	 * because it's questionable whether we can transfer the
749 	 * locking state correctly.
750 	 */
751 	if (uap->flags & MS_REMOUNT) {
752 		if (!(uap->flags & MS_SYSSPACE)) {
753 			nfs4_free_args(args);
754 			kmem_free(args, sizeof (*args));
755 		}
756 		if ((mi = VFTOMI4(vfsp)) != NULL) {
757 			uint_t new_mi_llock;
758 			uint_t old_mi_llock;
759 			new_mi_llock = (flags & NFSMNT_LLOCK) ? 1 : 0;
760 			old_mi_llock = (mi->mi_flags & MI4_LLOCK) ? 1 : 0;
761 			if (old_mi_llock != new_mi_llock)
762 				return (EBUSY);
763 		}
764 		return (0);
765 	}
766 
767 	/*
768 	 * For ephemeral mount trigger stub vnodes, we have two problems
769 	 * to solve: racing threads will likely fail the v_count check, and
770 	 * we want only one to proceed with the mount.
771 	 *
772 	 * For stubs, if the mount has already occurred (via a racing thread),
773 	 * just return success. If not, skip the v_count check and proceed.
774 	 * Note that we are already serialised at this point.
775 	 */
776 	mutex_enter(&mvp->v_lock);
777 	if (vn_matchops(mvp, nfs4_trigger_vnodeops)) {
778 		/* mntpt is a v4 stub vnode */
779 		ASSERT(RP_ISSTUB(VTOR4(mvp)));
780 		ASSERT(!(uap->flags & MS_OVERLAY));
781 		ASSERT(!(mvp->v_flag & VROOT));
782 		if (vn_mountedvfs(mvp) != NULL) {
783 			/* ephemeral mount has already occurred */
784 			ASSERT(uap->flags & MS_SYSSPACE);
785 			mutex_exit(&mvp->v_lock);
786 			return (0);
787 		}
788 	} else {
789 		/* mntpt is a non-v4 or v4 non-stub vnode */
790 		if (!(uap->flags & MS_OVERLAY) &&
791 		    (mvp->v_count != 1 || (mvp->v_flag & VROOT))) {
792 			mutex_exit(&mvp->v_lock);
793 			if (!(uap->flags & MS_SYSSPACE)) {
794 				nfs4_free_args(args);
795 				kmem_free(args, sizeof (*args));
796 			}
797 			return (EBUSY);
798 		}
799 	}
800 	mutex_exit(&mvp->v_lock);
801 
802 	/* make sure things are zeroed for errout: */
803 	rtvp = NULL;
804 	mi = NULL;
805 	secdata = NULL;
806 
807 	/*
808 	 * A valid knetconfig structure is required.
809 	 */
810 	if (!(flags & NFSMNT_KNCONF) ||
811 	    args->knconf == NULL || args->knconf->knc_protofmly == NULL ||
812 	    args->knconf->knc_proto == NULL ||
813 	    (strcmp(args->knconf->knc_proto, NC_UDP) == 0)) {
814 		if (!(uap->flags & MS_SYSSPACE)) {
815 			nfs4_free_args(args);
816 			kmem_free(args, sizeof (*args));
817 		}
818 		return (EINVAL);
819 	}
820 
821 	if ((strlen(args->knconf->knc_protofmly) >= KNC_STRSIZE) ||
822 	    (strlen(args->knconf->knc_proto) >= KNC_STRSIZE)) {
823 		if (!(uap->flags & MS_SYSSPACE)) {
824 			nfs4_free_args(args);
825 			kmem_free(args, sizeof (*args));
826 		}
827 		return (EINVAL);
828 	}
829 
830 	/*
831 	 * Allocate a servinfo4 struct.
832 	 */
833 	svp = kmem_zalloc(sizeof (*svp), KM_SLEEP);
834 	nfs_rw_init(&svp->sv_lock, NULL, RW_DEFAULT, NULL);
835 	if (svp_tail) {
836 		svp_2ndlast = svp_tail;
837 		svp_tail->sv_next = svp;
838 	} else {
839 		svp_head = svp;
840 		svp_2ndlast = svp;
841 	}
842 
843 	svp_tail = svp;
844 	svp->sv_knconf = args->knconf;
845 	args->knconf = NULL;
846 
847 	/*
848 	 * Get server address
849 	 */
850 	if (args->addr == NULL || args->addr->buf == NULL) {
851 		error = EINVAL;
852 		goto errout;
853 	}
854 
855 	svp->sv_addr.maxlen = args->addr->maxlen;
856 	svp->sv_addr.len = args->addr->len;
857 	svp->sv_addr.buf = args->addr->buf;
858 	args->addr->buf = NULL;
859 
860 	/*
861 	 * Get the root fhandle
862 	 */
863 	if (args->fh == NULL || (strlen(args->fh) >= MAXPATHLEN)) {
864 		error = EINVAL;
865 		goto errout;
866 	}
867 
868 	svp->sv_path = args->fh;
869 	svp->sv_pathlen = strlen(args->fh) + 1;
870 	args->fh = NULL;
871 
872 	/*
873 	 * Get server's hostname
874 	 */
875 	if (flags & NFSMNT_HOSTNAME) {
876 		if (args->hostname == NULL || (strlen(args->hostname) >
877 		    MAXNETNAMELEN)) {
878 			error = EINVAL;
879 			goto errout;
880 		}
881 		svp->sv_hostnamelen = strlen(args->hostname) + 1;
882 		svp->sv_hostname = args->hostname;
883 		args->hostname = NULL;
884 	} else {
885 		char *p = "unknown-host";
886 		svp->sv_hostnamelen = strlen(p) + 1;
887 		svp->sv_hostname = kmem_zalloc(svp->sv_hostnamelen, KM_SLEEP);
888 		(void) strcpy(svp->sv_hostname, p);
889 	}
890 
891 	/*
892 	 * RDMA MOUNT SUPPORT FOR NFS v4.
893 	 * Establish, is it possible to use RDMA, if so overload the
894 	 * knconf with rdma specific knconf and free the orignal knconf.
895 	 */
896 	if ((flags & NFSMNT_TRYRDMA) || (flags & NFSMNT_DORDMA)) {
897 		/*
898 		 * Determine the addr type for RDMA, IPv4 or v6.
899 		 */
900 		if (strcmp(svp->sv_knconf->knc_protofmly, NC_INET) == 0)
901 			addr_type = AF_INET;
902 		else if (strcmp(svp->sv_knconf->knc_protofmly, NC_INET6) == 0)
903 			addr_type = AF_INET6;
904 
905 		if (rdma_reachable(addr_type, &svp->sv_addr,
906 		    &rdma_knconf) == 0) {
907 			/*
908 			 * If successful, hijack the orignal knconf and
909 			 * replace with the new one, depending on the flags.
910 			 */
911 			svp->sv_origknconf = svp->sv_knconf;
912 			svp->sv_knconf = rdma_knconf;
913 		} else {
914 			if (flags & NFSMNT_TRYRDMA) {
915 #ifdef	DEBUG
916 				if (rdma_debug)
917 					zcmn_err(getzoneid(), CE_WARN,
918 					    "no RDMA onboard, revert\n");
919 #endif
920 			}
921 
922 			if (flags & NFSMNT_DORDMA) {
923 				/*
924 				 * If proto=rdma is specified and no RDMA
925 				 * path to this server is avialable then
926 				 * ditch this server.
927 				 * This is not included in the mountable
928 				 * server list or the replica list.
929 				 * Check if more servers are specified;
930 				 * Failover case, otherwise bail out of mount.
931 				 */
932 				if (args->nfs_args_ext == NFS_ARGS_EXTB &&
933 				    args->nfs_ext_u.nfs_extB.next != NULL) {
934 					data = (char *)
935 					    args->nfs_ext_u.nfs_extB.next;
936 					if (uap->flags & MS_RDONLY &&
937 					    !(flags & NFSMNT_SOFT)) {
938 						if (svp_head->sv_next == NULL) {
939 							svp_tail = NULL;
940 							svp_2ndlast = NULL;
941 							sv4_free(svp_head);
942 							goto more;
943 						} else {
944 							svp_tail = svp_2ndlast;
945 							svp_2ndlast->sv_next =
946 							    NULL;
947 							sv4_free(svp);
948 							goto more;
949 						}
950 					}
951 				} else {
952 					/*
953 					 * This is the last server specified
954 					 * in the nfs_args list passed down
955 					 * and its not rdma capable.
956 					 */
957 					if (svp_head->sv_next == NULL) {
958 						/*
959 						 * Is this the only one
960 						 */
961 						error = EINVAL;
962 #ifdef	DEBUG
963 						if (rdma_debug)
964 							zcmn_err(getzoneid(),
965 							    CE_WARN,
966 							    "No RDMA srv");
967 #endif
968 						goto errout;
969 					} else {
970 						/*
971 						 * There is list, since some
972 						 * servers specified before
973 						 * this passed all requirements
974 						 */
975 						svp_tail = svp_2ndlast;
976 						svp_2ndlast->sv_next = NULL;
977 						sv4_free(svp);
978 						goto proceed;
979 					}
980 				}
981 			}
982 		}
983 	}
984 
985 	/*
986 	 * If there are syncaddr and netname data, load them in. This is
987 	 * to support data needed for NFSV4 when AUTH_DH is the negotiated
988 	 * flavor via SECINFO. (instead of using MOUNT protocol in V3).
989 	 */
990 	if (args->flags & NFSMNT_SECURE) {
991 		svp->sv_dhsec = create_authdh_data(args->netname,
992 		    strlen(args->netname),
993 		    args->syncaddr, svp->sv_knconf);
994 	}
995 
996 	/*
997 	 * Get the extention data which has the security data structure.
998 	 * This includes data for AUTH_SYS as well.
999 	 */
1000 	if (flags & NFSMNT_NEWARGS) {
1001 		switch (args->nfs_args_ext) {
1002 		case NFS_ARGS_EXTA:
1003 		case NFS_ARGS_EXTB:
1004 			/*
1005 			 * Indicating the application is using the new
1006 			 * sec_data structure to pass in the security
1007 			 * data.
1008 			 */
1009 			secdata = args->nfs_ext_u.nfs_extA.secdata;
1010 			if (secdata == NULL) {
1011 				error = EINVAL;
1012 			} else if (uap->flags & MS_SYSSPACE) {
1013 				/*
1014 				 * Need to validate the flavor here if
1015 				 * sysspace, userspace was already
1016 				 * validate from the nfs_copyin function.
1017 				 */
1018 				switch (secdata->rpcflavor) {
1019 				case AUTH_NONE:
1020 				case AUTH_UNIX:
1021 				case AUTH_LOOPBACK:
1022 				case AUTH_DES:
1023 				case RPCSEC_GSS:
1024 					break;
1025 				default:
1026 					error = EINVAL;
1027 					goto errout;
1028 				}
1029 			}
1030 			args->nfs_ext_u.nfs_extA.secdata = NULL;
1031 			break;
1032 
1033 		default:
1034 			error = EINVAL;
1035 			break;
1036 		}
1037 
1038 	} else if (flags & NFSMNT_SECURE) {
1039 		/*
1040 		 * NFSMNT_SECURE is deprecated but we keep it
1041 		 * to support the rogue user-generated application
1042 		 * that may use this undocumented interface to do
1043 		 * AUTH_DH security, e.g. our own rexd.
1044 		 *
1045 		 * Also note that NFSMNT_SECURE is used for passing
1046 		 * AUTH_DH info to be used in negotiation.
1047 		 */
1048 		secdata = create_authdh_data(args->netname,
1049 		    strlen(args->netname), args->syncaddr, svp->sv_knconf);
1050 
1051 	} else {
1052 		secdata = kmem_alloc(sizeof (*secdata), KM_SLEEP);
1053 		secdata->secmod = secdata->rpcflavor = AUTH_SYS;
1054 		secdata->data = NULL;
1055 	}
1056 
1057 	svp->sv_secdata = secdata;
1058 
1059 	/*
1060 	 * User does not explictly specify a flavor, and a user
1061 	 * defined default flavor is passed down.
1062 	 */
1063 	if (flags & NFSMNT_SECDEFAULT) {
1064 		(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
1065 		svp->sv_flags |= SV4_TRYSECDEFAULT;
1066 		nfs_rw_exit(&svp->sv_lock);
1067 	}
1068 
1069 	/*
1070 	 * Failover support:
1071 	 *
1072 	 * We may have a linked list of nfs_args structures,
1073 	 * which means the user is looking for failover.  If
1074 	 * the mount is either not "read-only" or "soft",
1075 	 * we want to bail out with EINVAL.
1076 	 */
1077 	if (args->nfs_args_ext == NFS_ARGS_EXTB &&
1078 	    args->nfs_ext_u.nfs_extB.next != NULL) {
1079 		if (uap->flags & MS_RDONLY && !(flags & NFSMNT_SOFT)) {
1080 			data = (char *)args->nfs_ext_u.nfs_extB.next;
1081 			goto more;
1082 		}
1083 		error = EINVAL;
1084 		goto errout;
1085 	}
1086 
1087 	/*
1088 	 * Determine the zone we're being mounted into.
1089 	 */
1090 	zone_hold(mntzone = zone);		/* start with this assumption */
1091 	if (getzoneid() == GLOBAL_ZONEID) {
1092 		zone_rele(mntzone);
1093 		mntzone = zone_find_by_path(refstr_value(vfsp->vfs_mntpt));
1094 		ASSERT(mntzone != NULL);
1095 		if (mntzone != zone) {
1096 			error = EBUSY;
1097 			goto errout;
1098 		}
1099 	}
1100 
1101 	if (is_system_labeled()) {
1102 		error = nfs_mount_label_policy(vfsp, &svp->sv_addr,
1103 		    svp->sv_knconf, cr);
1104 
1105 		if (error > 0)
1106 			goto errout;
1107 
1108 		if (error == -1) {
1109 			/* change mount to read-only to prevent write-down */
1110 			vfs_setmntopt(vfsp, MNTOPT_RO, NULL, 0);
1111 		}
1112 	}
1113 
1114 	/*
1115 	 * Stop the mount from going any further if the zone is going away.
1116 	 */
1117 	if (zone_status_get(mntzone) >= ZONE_IS_SHUTTING_DOWN) {
1118 		error = EBUSY;
1119 		goto errout;
1120 	}
1121 
1122 	/*
1123 	 * Get root vnode.
1124 	 */
1125 proceed:
1126 	error = nfs4rootvp(&rtvp, vfsp, svp_head, flags, cr, mntzone);
1127 	if (error) {
1128 		/* if nfs4rootvp failed, it will free svp_head */
1129 		svp_head = NULL;
1130 		goto errout;
1131 	}
1132 
1133 	mi = VTOMI4(rtvp);
1134 
1135 	/*
1136 	 * Send client id to the server, if necessary
1137 	 */
1138 	nfs4_error_zinit(&n4e);
1139 	nfs4setclientid(mi, cr, FALSE, &n4e);
1140 
1141 	error = n4e.error;
1142 
1143 	if (error)
1144 		goto errout;
1145 
1146 	/*
1147 	 * Set option fields in the mount info record
1148 	 */
1149 
1150 	if (svp_head->sv_next) {
1151 		mutex_enter(&mi->mi_lock);
1152 		mi->mi_flags |= MI4_LLOCK;
1153 		mutex_exit(&mi->mi_lock);
1154 	}
1155 	error = nfs4_setopts(rtvp, DATAMODEL_NATIVE, args);
1156 	if (error)
1157 		goto errout;
1158 
1159 	/*
1160 	 * Time to tie in the mirror mount info at last!
1161 	 */
1162 	if (flags & NFSMNT_EPHEMERAL)
1163 		error = nfs4_record_ephemeral_mount(mi, mvp);
1164 
1165 errout:
1166 	if (error) {
1167 		if (rtvp != NULL) {
1168 			rp = VTOR4(rtvp);
1169 			if (rp->r_flags & R4HASHED)
1170 				rp4_rmhash(rp);
1171 		}
1172 		if (mi != NULL) {
1173 			nfs4_async_stop(vfsp);
1174 			nfs4_async_manager_stop(vfsp);
1175 			nfs4_remove_mi_from_server(mi, NULL);
1176 			if (rtvp != NULL)
1177 				VN_RELE(rtvp);
1178 			if (mntzone != NULL)
1179 				zone_rele(mntzone);
1180 			/* need to remove it from the zone */
1181 			removed = nfs4_mi_zonelist_remove(mi);
1182 			if (removed)
1183 				zone_rele(mi->mi_zone);
1184 			MI4_RELE(mi);
1185 			if (!(uap->flags & MS_SYSSPACE) && args) {
1186 				nfs4_free_args(args);
1187 				kmem_free(args, sizeof (*args));
1188 			}
1189 			return (error);
1190 		}
1191 		if (svp_head)
1192 			sv4_free(svp_head);
1193 	}
1194 
1195 	if (!(uap->flags & MS_SYSSPACE) && args) {
1196 		nfs4_free_args(args);
1197 		kmem_free(args, sizeof (*args));
1198 	}
1199 	if (rtvp != NULL)
1200 		VN_RELE(rtvp);
1201 
1202 	if (mntzone != NULL)
1203 		zone_rele(mntzone);
1204 
1205 	return (error);
1206 }
1207 
1208 #ifdef  DEBUG
1209 #define	VERS_MSG	"NFS4 server "
1210 #else
1211 #define	VERS_MSG	"NFS server "
1212 #endif
1213 
1214 #define	READ_MSG        \
1215 	VERS_MSG "%s returned 0 for read transfer size"
1216 #define	WRITE_MSG       \
1217 	VERS_MSG "%s returned 0 for write transfer size"
1218 #define	SIZE_MSG        \
1219 	VERS_MSG "%s returned 0 for maximum file size"
1220 
1221 /*
1222  * Get the symbolic link text from the server for a given filehandle
1223  * of that symlink.
1224  *
1225  *      (get symlink text) PUTFH READLINK
1226  */
1227 static int
1228 getlinktext_otw(mntinfo4_t *mi, nfs_fh4 *fh, char **linktextp, cred_t *cr,
1229     int flags)
1230 {
1231 	COMPOUND4args_clnt args;
1232 	COMPOUND4res_clnt res;
1233 	int doqueue;
1234 	nfs_argop4 argop[2];
1235 	nfs_resop4 *resop;
1236 	READLINK4res *lr_res;
1237 	uint_t len;
1238 	bool_t needrecov = FALSE;
1239 	nfs4_recov_state_t recov_state;
1240 	nfs4_sharedfh_t *sfh;
1241 	nfs4_error_t e;
1242 	int num_retry = nfs4_max_mount_retry;
1243 	int recovery = !(flags & NFS4_GETFH_NEEDSOP);
1244 
1245 	sfh = sfh4_get(fh, mi);
1246 	recov_state.rs_flags = 0;
1247 	recov_state.rs_num_retry_despite_err = 0;
1248 
1249 recov_retry:
1250 	nfs4_error_zinit(&e);
1251 
1252 	args.array_len = 2;
1253 	args.array = argop;
1254 	args.ctag = TAG_GET_SYMLINK;
1255 
1256 	if (! recovery) {
1257 		e.error = nfs4_start_op(mi, NULL, NULL, &recov_state);
1258 		if (e.error) {
1259 			sfh4_rele(&sfh);
1260 			return (e.error);
1261 		}
1262 	}
1263 
1264 	/* 0. putfh symlink fh */
1265 	argop[0].argop = OP_CPUTFH;
1266 	argop[0].nfs_argop4_u.opcputfh.sfh = sfh;
1267 
1268 	/* 1. readlink */
1269 	argop[1].argop = OP_READLINK;
1270 
1271 	doqueue = 1;
1272 
1273 	rfs4call(mi, &args, &res, cr, &doqueue, 0, &e);
1274 
1275 	needrecov = nfs4_needs_recovery(&e, FALSE, mi->mi_vfsp);
1276 
1277 	if (needrecov && !recovery && num_retry-- > 0) {
1278 
1279 		NFS4_DEBUG(nfs4_client_recov_debug, (CE_NOTE,
1280 		    "getlinktext_otw: initiating recovery\n"));
1281 
1282 		if (nfs4_start_recovery(&e, mi, NULL, NULL, NULL, NULL,
1283 		    OP_READLINK, NULL, NULL, NULL) == FALSE) {
1284 			nfs4_end_op(mi, NULL, NULL, &recov_state, needrecov);
1285 			if (!e.error)
1286 				(void) xdr_free(xdr_COMPOUND4res_clnt,
1287 				    (caddr_t)&res);
1288 			goto recov_retry;
1289 		}
1290 	}
1291 
1292 	/*
1293 	 * If non-NFS4 pcol error and/or we weren't able to recover.
1294 	 */
1295 	if (e.error != 0) {
1296 		if (! recovery)
1297 			nfs4_end_op(mi, NULL, NULL, &recov_state, needrecov);
1298 		sfh4_rele(&sfh);
1299 		return (e.error);
1300 	}
1301 
1302 	if (res.status) {
1303 		e.error = geterrno4(res.status);
1304 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
1305 		if (! recovery)
1306 			nfs4_end_op(mi, NULL, NULL, &recov_state, needrecov);
1307 		sfh4_rele(&sfh);
1308 		return (e.error);
1309 	}
1310 
1311 	/* res.status == NFS4_OK */
1312 	ASSERT(res.status == NFS4_OK);
1313 
1314 	resop = &res.array[1];  /* readlink res */
1315 	lr_res = &resop->nfs_resop4_u.opreadlink;
1316 
1317 	/* treat symlink name as data */
1318 	*linktextp = utf8_to_str(&lr_res->link, &len, NULL);
1319 
1320 	if (! recovery)
1321 		nfs4_end_op(mi, NULL, NULL, &recov_state, needrecov);
1322 	sfh4_rele(&sfh);
1323 	(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
1324 	return (0);
1325 }
1326 
1327 /*
1328  * Skip over consecutive slashes and "/./" in a pathname.
1329  */
1330 void
1331 pathname_skipslashdot(struct pathname *pnp)
1332 {
1333 	char *c1, *c2;
1334 
1335 	while (pnp->pn_pathlen > 0 && *pnp->pn_path == '/') {
1336 
1337 		c1 = pnp->pn_path + 1;
1338 		c2 = pnp->pn_path + 2;
1339 
1340 		if (*c1 == '.' && (*c2 == '/' || *c2 == '\0')) {
1341 			pnp->pn_path = pnp->pn_path + 2; /* skip "/." */
1342 			pnp->pn_pathlen = pnp->pn_pathlen - 2;
1343 		} else {
1344 			pnp->pn_path++;
1345 			pnp->pn_pathlen--;
1346 		}
1347 	}
1348 }
1349 
1350 /*
1351  * Resolve a symbolic link path. The symlink is in the nth component of
1352  * svp->sv_path and has an nfs4 file handle "fh".
1353  * Upon return, the sv_path will point to the new path that has the nth
1354  * component resolved to its symlink text.
1355  */
1356 int
1357 resolve_sympath(mntinfo4_t *mi, servinfo4_t *svp, int nth, nfs_fh4 *fh,
1358     cred_t *cr, int flags)
1359 {
1360 	char *oldpath;
1361 	char *symlink, *newpath;
1362 	struct pathname oldpn, newpn;
1363 	char component[MAXNAMELEN];
1364 	int i, addlen, error = 0;
1365 	int oldpathlen;
1366 
1367 	/* Get the symbolic link text over the wire. */
1368 	error = getlinktext_otw(mi, fh, &symlink, cr, flags);
1369 
1370 	if (error || symlink == NULL || strlen(symlink) == 0)
1371 		return (error);
1372 
1373 	/*
1374 	 * Compose the new pathname.
1375 	 * Note:
1376 	 *    - only the nth component is resolved for the pathname.
1377 	 *    - pathname.pn_pathlen does not count the ending null byte.
1378 	 */
1379 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
1380 	oldpath = svp->sv_path;
1381 	oldpathlen = svp->sv_pathlen;
1382 	if (error = pn_get(oldpath, UIO_SYSSPACE, &oldpn)) {
1383 		nfs_rw_exit(&svp->sv_lock);
1384 		kmem_free(symlink, strlen(symlink) + 1);
1385 		return (error);
1386 	}
1387 	nfs_rw_exit(&svp->sv_lock);
1388 	pn_alloc(&newpn);
1389 
1390 	/*
1391 	 * Skip over previous components from the oldpath so that the
1392 	 * oldpn.pn_path will point to the symlink component. Skip
1393 	 * leading slashes and "/./" (no OP_LOOKUP on ".") so that
1394 	 * pn_getcompnent can get the component.
1395 	 */
1396 	for (i = 1; i < nth; i++) {
1397 		pathname_skipslashdot(&oldpn);
1398 		error = pn_getcomponent(&oldpn, component);
1399 		if (error)
1400 			goto out;
1401 	}
1402 
1403 	/*
1404 	 * Copy the old path upto the component right before the symlink
1405 	 * if the symlink is not an absolute path.
1406 	 */
1407 	if (symlink[0] != '/') {
1408 		addlen = oldpn.pn_path - oldpn.pn_buf;
1409 		bcopy(oldpn.pn_buf, newpn.pn_path, addlen);
1410 		newpn.pn_pathlen += addlen;
1411 		newpn.pn_path += addlen;
1412 		newpn.pn_buf[newpn.pn_pathlen] = '/';
1413 		newpn.pn_pathlen++;
1414 		newpn.pn_path++;
1415 	}
1416 
1417 	/* copy the resolved symbolic link text */
1418 	addlen = strlen(symlink);
1419 	if (newpn.pn_pathlen + addlen >= newpn.pn_bufsize) {
1420 		error = ENAMETOOLONG;
1421 		goto out;
1422 	}
1423 	bcopy(symlink, newpn.pn_path, addlen);
1424 	newpn.pn_pathlen += addlen;
1425 	newpn.pn_path += addlen;
1426 
1427 	/*
1428 	 * Check if there is any remaining path after the symlink component.
1429 	 * First, skip the symlink component.
1430 	 */
1431 	pathname_skipslashdot(&oldpn);
1432 	if (error = pn_getcomponent(&oldpn, component))
1433 		goto out;
1434 
1435 	addlen = pn_pathleft(&oldpn); /* includes counting the slash */
1436 
1437 	/*
1438 	 * Copy the remaining path to the new pathname if there is any.
1439 	 */
1440 	if (addlen > 0) {
1441 		if (newpn.pn_pathlen + addlen >= newpn.pn_bufsize) {
1442 			error = ENAMETOOLONG;
1443 			goto out;
1444 		}
1445 		bcopy(oldpn.pn_path, newpn.pn_path, addlen);
1446 		newpn.pn_pathlen += addlen;
1447 	}
1448 	newpn.pn_buf[newpn.pn_pathlen] = '\0';
1449 
1450 	/* get the newpath and store it in the servinfo4_t */
1451 	newpath = kmem_alloc(newpn.pn_pathlen + 1, KM_SLEEP);
1452 	bcopy(newpn.pn_buf, newpath, newpn.pn_pathlen);
1453 	newpath[newpn.pn_pathlen] = '\0';
1454 
1455 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
1456 	svp->sv_path = newpath;
1457 	svp->sv_pathlen = strlen(newpath) + 1;
1458 	nfs_rw_exit(&svp->sv_lock);
1459 
1460 	kmem_free(oldpath, oldpathlen);
1461 out:
1462 	kmem_free(symlink, strlen(symlink) + 1);
1463 	pn_free(&newpn);
1464 	pn_free(&oldpn);
1465 
1466 	return (error);
1467 }
1468 
1469 /*
1470  * This routine updates servinfo4 structure with the new referred server
1471  * info.
1472  * nfsfsloc has the location related information
1473  * fsp has the hostname and pathname info.
1474  * new path = pathname from referral + part of orig pathname(based on nth).
1475  */
1476 static void
1477 update_servinfo4(servinfo4_t *svp, fs_location4 *fsp,
1478     struct nfs_fsl_info *nfsfsloc, char *orig_path, int nth)
1479 {
1480 	struct knetconfig *knconf, *svknconf;
1481 	struct netbuf *saddr;
1482 	sec_data_t	*secdata;
1483 	utf8string *host;
1484 	int i = 0, num_slashes = 0;
1485 	char *p, *spath, *op, *new_path;
1486 
1487 	/* Update knconf */
1488 	knconf = svp->sv_knconf;
1489 	free_knconf_contents(knconf);
1490 	bzero(knconf, sizeof (struct knetconfig));
1491 	svknconf = nfsfsloc->knconf;
1492 	knconf->knc_semantics = svknconf->knc_semantics;
1493 	knconf->knc_protofmly = kmem_zalloc(KNC_STRSIZE, KM_SLEEP);
1494 	knconf->knc_proto = kmem_zalloc(KNC_STRSIZE, KM_SLEEP);
1495 	knconf->knc_rdev = svknconf->knc_rdev;
1496 	bcopy(svknconf->knc_protofmly, knconf->knc_protofmly, KNC_STRSIZE);
1497 	bcopy(svknconf->knc_proto, knconf->knc_proto, KNC_STRSIZE);
1498 
1499 	/* Update server address */
1500 	saddr = &svp->sv_addr;
1501 	if (saddr->buf != NULL)
1502 		kmem_free(saddr->buf, saddr->maxlen);
1503 	saddr->buf  = kmem_alloc(nfsfsloc->addr->maxlen, KM_SLEEP);
1504 	saddr->len = nfsfsloc->addr->len;
1505 	saddr->maxlen = nfsfsloc->addr->maxlen;
1506 	bcopy(nfsfsloc->addr->buf, saddr->buf, nfsfsloc->addr->len);
1507 
1508 	/* Update server name */
1509 	host = fsp->server_val;
1510 	kmem_free(svp->sv_hostname, svp->sv_hostnamelen);
1511 	svp->sv_hostname = kmem_zalloc(host->utf8string_len + 1, KM_SLEEP);
1512 	bcopy(host->utf8string_val, svp->sv_hostname, host->utf8string_len);
1513 	svp->sv_hostname[host->utf8string_len] = '\0';
1514 	svp->sv_hostnamelen = host->utf8string_len + 1;
1515 
1516 	/*
1517 	 * Update server path.
1518 	 * We need to setup proper path here.
1519 	 * For ex., If we got a path name serv1:/rp/aaa/bbb
1520 	 * where aaa is a referral and points to serv2:/rpool/aa
1521 	 * we need to set the path to serv2:/rpool/aa/bbb
1522 	 * The first part of this below code generates /rpool/aa
1523 	 * and the second part appends /bbb to the server path.
1524 	 */
1525 	spath = p = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
1526 	*p++ = '/';
1527 	for (i = 0; i < fsp->rootpath.pathname4_len; i++) {
1528 		component4 *comp;
1529 
1530 		comp = &fsp->rootpath.pathname4_val[i];
1531 		/* If no space, null the string and bail */
1532 		if ((p - spath) + comp->utf8string_len + 1 > MAXPATHLEN) {
1533 			p = spath + MAXPATHLEN - 1;
1534 			spath[0] = '\0';
1535 			break;
1536 		}
1537 		bcopy(comp->utf8string_val, p, comp->utf8string_len);
1538 		p += comp->utf8string_len;
1539 		*p++ = '/';
1540 	}
1541 	if (fsp->rootpath.pathname4_len != 0)
1542 		*(p - 1) = '\0';
1543 	else
1544 		*p = '\0';
1545 	p = spath;
1546 
1547 	new_path = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
1548 	(void) strlcpy(new_path, p, MAXPATHLEN);
1549 	kmem_free(p, MAXPATHLEN);
1550 	i = strlen(new_path);
1551 
1552 	for (op = orig_path; *op; op++) {
1553 		if (*op == '/')
1554 			num_slashes++;
1555 		if (num_slashes == nth + 2) {
1556 			while (*op != '\0') {
1557 				new_path[i] = *op;
1558 				i++;
1559 				op++;
1560 			}
1561 			break;
1562 		}
1563 	}
1564 	new_path[i] = '\0';
1565 
1566 	kmem_free(svp->sv_path, svp->sv_pathlen);
1567 	svp->sv_pathlen = strlen(new_path) + 1;
1568 	svp->sv_path = kmem_alloc(svp->sv_pathlen, KM_SLEEP);
1569 	bcopy(new_path, svp->sv_path, svp->sv_pathlen);
1570 	kmem_free(new_path, MAXPATHLEN);
1571 
1572 	/*
1573 	 * All the security data is specific to old server.
1574 	 * Clean it up except secdata which deals with mount options.
1575 	 * We need to inherit that data. Copy secdata into our new servinfo4.
1576 	 */
1577 	if (svp->sv_dhsec) {
1578 		sec_clnt_freeinfo(svp->sv_dhsec);
1579 		svp->sv_dhsec = NULL;
1580 	}
1581 	if (svp->sv_save_secinfo &&
1582 	    svp->sv_save_secinfo != svp->sv_secinfo) {
1583 		secinfo_free(svp->sv_save_secinfo);
1584 		svp->sv_save_secinfo = NULL;
1585 	}
1586 	if (svp->sv_secinfo) {
1587 		secinfo_free(svp->sv_secinfo);
1588 		svp->sv_secinfo = NULL;
1589 	}
1590 	svp->sv_currsec = NULL;
1591 
1592 	secdata = kmem_alloc(sizeof (*secdata), KM_SLEEP);
1593 	*secdata = *svp->sv_secdata;
1594 	secdata->data = NULL;
1595 	if (svp->sv_secdata) {
1596 		sec_clnt_freeinfo(svp->sv_secdata);
1597 		svp->sv_secdata = NULL;
1598 	}
1599 	svp->sv_secdata = secdata;
1600 }
1601 
1602 /*
1603  * Resolve a referral. The referral is in the n+1th component of
1604  * svp->sv_path and has a parent nfs4 file handle "fh".
1605  * Upon return, the sv_path will point to the new path that has referral
1606  * component resolved to its referred path and part of original path.
1607  * Hostname and other address information is also updated.
1608  */
1609 int
1610 resolve_referral(mntinfo4_t *mi, servinfo4_t *svp, cred_t *cr, int nth,
1611     nfs_fh4 *fh)
1612 {
1613 	nfs4_sharedfh_t	*sfh;
1614 	struct nfs_fsl_info nfsfsloc;
1615 	nfs4_ga_res_t garp;
1616 	COMPOUND4res_clnt callres;
1617 	fs_location4	*fsp;
1618 	char *nm, *orig_path;
1619 	int orig_pathlen = 0, ret = -1, index;
1620 
1621 	if (svp->sv_pathlen <= 0)
1622 		return (ret);
1623 
1624 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
1625 	orig_pathlen = svp->sv_pathlen;
1626 	orig_path = kmem_alloc(orig_pathlen, KM_SLEEP);
1627 	bcopy(svp->sv_path, orig_path, orig_pathlen);
1628 	nm = extract_referral_point(svp->sv_path, nth);
1629 	setup_newsvpath(svp, nth);
1630 	nfs_rw_exit(&svp->sv_lock);
1631 
1632 	sfh = sfh4_get(fh, mi);
1633 	index = nfs4_process_referral(mi, sfh, nm, cr,
1634 	    &garp, &callres, &nfsfsloc);
1635 	sfh4_rele(&sfh);
1636 	kmem_free(nm, MAXPATHLEN);
1637 	if (index < 0) {
1638 		kmem_free(orig_path, orig_pathlen);
1639 		return (index);
1640 	}
1641 
1642 	fsp =  &garp.n4g_ext_res->n4g_fslocations.locations_val[index];
1643 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
1644 	update_servinfo4(svp, fsp, &nfsfsloc, orig_path, nth);
1645 	nfs_rw_exit(&svp->sv_lock);
1646 
1647 	mutex_enter(&mi->mi_lock);
1648 	mi->mi_vfs_referral_loop_cnt++;
1649 	mutex_exit(&mi->mi_lock);
1650 
1651 	ret = 0;
1652 bad:
1653 	/* Free up XDR memory allocated in nfs4_process_referral() */
1654 	xdr_free(xdr_nfs_fsl_info, (char *)&nfsfsloc);
1655 	xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&callres);
1656 	kmem_free(orig_path, orig_pathlen);
1657 
1658 	return (ret);
1659 }
1660 
1661 /*
1662  * Get the root filehandle for the given filesystem and server, and update
1663  * svp.
1664  *
1665  * If NFS4_GETFH_NEEDSOP is set, then use nfs4_start_fop and nfs4_end_fop
1666  * to coordinate with recovery.  Otherwise, the caller is assumed to be
1667  * the recovery thread or have already done a start_fop.
1668  *
1669  * Errors are returned by the nfs4_error_t parameter.
1670  */
1671 static void
1672 nfs4getfh_otw(struct mntinfo4 *mi, servinfo4_t *svp, vtype_t *vtp,
1673     int flags, cred_t *cr, nfs4_error_t *ep)
1674 {
1675 	COMPOUND4args_clnt args;
1676 	COMPOUND4res_clnt res;
1677 	int doqueue = 1;
1678 	nfs_argop4 *argop;
1679 	nfs_resop4 *resop;
1680 	nfs4_ga_res_t *garp;
1681 	int num_argops;
1682 	lookup4_param_t lookuparg;
1683 	nfs_fh4 *tmpfhp;
1684 	nfs_fh4 *resfhp;
1685 	bool_t needrecov = FALSE;
1686 	nfs4_recov_state_t recov_state;
1687 	int llndx;
1688 	int nthcomp;
1689 	int recovery = !(flags & NFS4_GETFH_NEEDSOP);
1690 
1691 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
1692 	ASSERT(svp->sv_path != NULL);
1693 	if (svp->sv_path[0] == '\0') {
1694 		nfs_rw_exit(&svp->sv_lock);
1695 		nfs4_error_init(ep, EINVAL);
1696 		return;
1697 	}
1698 	nfs_rw_exit(&svp->sv_lock);
1699 
1700 	recov_state.rs_flags = 0;
1701 	recov_state.rs_num_retry_despite_err = 0;
1702 
1703 recov_retry:
1704 	if (mi->mi_vfs_referral_loop_cnt >= NFS4_REFERRAL_LOOP_MAX) {
1705 		DTRACE_PROBE3(nfs4clnt__debug__referral__loop, mntinfo4 *,
1706 		    mi, servinfo4_t *, svp, char *, "nfs4getfh_otw");
1707 		nfs4_error_init(ep, EINVAL);
1708 		return;
1709 	}
1710 	nfs4_error_zinit(ep);
1711 
1712 	if (!recovery) {
1713 		ep->error = nfs4_start_fop(mi, NULL, NULL, OH_MOUNT,
1714 		    &recov_state, NULL);
1715 
1716 		/*
1717 		 * If recovery has been started and this request as
1718 		 * initiated by a mount, then we must wait for recovery
1719 		 * to finish before proceeding, otherwise, the error
1720 		 * cleanup would remove data structures needed by the
1721 		 * recovery thread.
1722 		 */
1723 		if (ep->error) {
1724 			mutex_enter(&mi->mi_lock);
1725 			if (mi->mi_flags & MI4_MOUNTING) {
1726 				mi->mi_flags |= MI4_RECOV_FAIL;
1727 				mi->mi_error = EIO;
1728 
1729 				NFS4_DEBUG(nfs4_client_recov_debug, (CE_NOTE,
1730 				    "nfs4getfh_otw: waiting 4 recovery\n"));
1731 
1732 				while (mi->mi_flags & MI4_RECOV_ACTIV)
1733 					cv_wait(&mi->mi_failover_cv,
1734 					    &mi->mi_lock);
1735 			}
1736 			mutex_exit(&mi->mi_lock);
1737 			return;
1738 		}
1739 
1740 		/*
1741 		 * If the client does not specify a specific flavor to use
1742 		 * and has not gotten a secinfo list from the server yet,
1743 		 * retrieve the secinfo list from the server and use a
1744 		 * flavor from the list to mount.
1745 		 *
1746 		 * If fail to get the secinfo list from the server, then
1747 		 * try the default flavor.
1748 		 */
1749 		if ((svp->sv_flags & SV4_TRYSECDEFAULT) &&
1750 		    svp->sv_secinfo == NULL) {
1751 			(void) nfs4_secinfo_path(mi, cr, FALSE);
1752 		}
1753 	}
1754 
1755 	if (recovery)
1756 		args.ctag = TAG_REMAP_MOUNT;
1757 	else
1758 		args.ctag = TAG_MOUNT;
1759 
1760 	lookuparg.l4_getattrs = LKP4_ALL_ATTRIBUTES;
1761 	lookuparg.argsp = &args;
1762 	lookuparg.resp = &res;
1763 	lookuparg.header_len = 2;	/* Putrootfh, getfh */
1764 	lookuparg.trailer_len = 0;
1765 	lookuparg.ga_bits = FATTR4_FSINFO_MASK;
1766 	lookuparg.mi = mi;
1767 
1768 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
1769 	ASSERT(svp->sv_path != NULL);
1770 	llndx = nfs4lookup_setup(svp->sv_path, &lookuparg, 0);
1771 	nfs_rw_exit(&svp->sv_lock);
1772 
1773 	argop = args.array;
1774 	num_argops = args.array_len;
1775 
1776 	/* choose public or root filehandle */
1777 	if (flags & NFS4_GETFH_PUBLIC)
1778 		argop[0].argop = OP_PUTPUBFH;
1779 	else
1780 		argop[0].argop = OP_PUTROOTFH;
1781 
1782 	/* get fh */
1783 	argop[1].argop = OP_GETFH;
1784 
1785 	NFS4_DEBUG(nfs4_client_call_debug, (CE_NOTE,
1786 	    "nfs4getfh_otw: %s call, mi 0x%p",
1787 	    needrecov ? "recov" : "first", (void *)mi));
1788 
1789 	rfs4call(mi, &args, &res, cr, &doqueue, RFSCALL_SOFT, ep);
1790 
1791 	needrecov = nfs4_needs_recovery(ep, FALSE, mi->mi_vfsp);
1792 
1793 	if (needrecov) {
1794 		bool_t abort;
1795 
1796 		if (recovery) {
1797 			nfs4args_lookup_free(argop, num_argops);
1798 			kmem_free(argop,
1799 			    lookuparg.arglen * sizeof (nfs_argop4));
1800 			if (!ep->error)
1801 				(void) xdr_free(xdr_COMPOUND4res_clnt,
1802 				    (caddr_t)&res);
1803 			return;
1804 		}
1805 
1806 		NFS4_DEBUG(nfs4_client_recov_debug,
1807 		    (CE_NOTE, "nfs4getfh_otw: initiating recovery\n"));
1808 
1809 		abort = nfs4_start_recovery(ep, mi, NULL,
1810 		    NULL, NULL, NULL, OP_GETFH, NULL, NULL, NULL);
1811 		if (!ep->error) {
1812 			ep->error = geterrno4(res.status);
1813 			(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
1814 		}
1815 		nfs4args_lookup_free(argop, num_argops);
1816 		kmem_free(argop, lookuparg.arglen * sizeof (nfs_argop4));
1817 		nfs4_end_fop(mi, NULL, NULL, OH_MOUNT, &recov_state, needrecov);
1818 		/* have another go? */
1819 		if (abort == FALSE)
1820 			goto recov_retry;
1821 		return;
1822 	}
1823 
1824 	/*
1825 	 * No recovery, but check if error is set.
1826 	 */
1827 	if (ep->error)  {
1828 		nfs4args_lookup_free(argop, num_argops);
1829 		kmem_free(argop, lookuparg.arglen * sizeof (nfs_argop4));
1830 		if (!recovery)
1831 			nfs4_end_fop(mi, NULL, NULL, OH_MOUNT, &recov_state,
1832 			    needrecov);
1833 		return;
1834 	}
1835 
1836 is_link_err:
1837 
1838 	/* for non-recovery errors */
1839 	if (res.status && res.status != NFS4ERR_SYMLINK &&
1840 	    res.status != NFS4ERR_MOVED) {
1841 		if (!recovery) {
1842 			nfs4_end_fop(mi, NULL, NULL, OH_MOUNT, &recov_state,
1843 			    needrecov);
1844 		}
1845 		nfs4args_lookup_free(argop, num_argops);
1846 		kmem_free(argop, lookuparg.arglen * sizeof (nfs_argop4));
1847 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
1848 		return;
1849 	}
1850 
1851 	/*
1852 	 * If any intermediate component in the path is a symbolic link,
1853 	 * resolve the symlink, then try mount again using the new path.
1854 	 */
1855 	if (res.status == NFS4ERR_SYMLINK || res.status == NFS4ERR_MOVED) {
1856 		int where;
1857 
1858 		/*
1859 		 * Need to call nfs4_end_op before resolve_sympath to avoid
1860 		 * potential nfs4_start_op deadlock.
1861 		 */
1862 		if (!recovery)
1863 			nfs4_end_fop(mi, NULL, NULL, OH_MOUNT, &recov_state,
1864 			    needrecov);
1865 
1866 		/*
1867 		 * This must be from OP_LOOKUP failure. The (cfh) for this
1868 		 * OP_LOOKUP is a symlink node. Found out where the
1869 		 * OP_GETFH is for the (cfh) that is a symlink node.
1870 		 *
1871 		 * Example:
1872 		 * (mount) PUTROOTFH, GETFH, LOOKUP comp1, GETFH, GETATTR,
1873 		 * LOOKUP comp2, GETFH, GETATTR, LOOKUP comp3, GETFH, GETATTR
1874 		 *
1875 		 * LOOKUP comp3 fails with SYMLINK because comp2 is a symlink.
1876 		 * In this case, where = 7, nthcomp = 2.
1877 		 */
1878 		where = res.array_len - 2;
1879 		ASSERT(where > 0);
1880 
1881 		if (res.status == NFS4ERR_SYMLINK) {
1882 
1883 			resop = &res.array[where - 1];
1884 			ASSERT(resop->resop == OP_GETFH);
1885 			tmpfhp = &resop->nfs_resop4_u.opgetfh.object;
1886 			nthcomp = res.array_len/3 - 1;
1887 			ep->error = resolve_sympath(mi, svp, nthcomp,
1888 			    tmpfhp, cr, flags);
1889 
1890 		} else if (res.status == NFS4ERR_MOVED) {
1891 
1892 			resop = &res.array[where - 2];
1893 			ASSERT(resop->resop == OP_GETFH);
1894 			tmpfhp = &resop->nfs_resop4_u.opgetfh.object;
1895 			nthcomp = res.array_len/3 - 1;
1896 			ep->error = resolve_referral(mi, svp, cr, nthcomp,
1897 			    tmpfhp);
1898 		}
1899 
1900 		nfs4args_lookup_free(argop, num_argops);
1901 		kmem_free(argop, lookuparg.arglen * sizeof (nfs_argop4));
1902 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
1903 
1904 		if (ep->error)
1905 			return;
1906 
1907 		goto recov_retry;
1908 	}
1909 
1910 	/* getfh */
1911 	resop = &res.array[res.array_len - 2];
1912 	ASSERT(resop->resop == OP_GETFH);
1913 	resfhp = &resop->nfs_resop4_u.opgetfh.object;
1914 
1915 	/* getattr fsinfo res */
1916 	resop++;
1917 	garp = &resop->nfs_resop4_u.opgetattr.ga_res;
1918 
1919 	*vtp = garp->n4g_va.va_type;
1920 
1921 	mi->mi_fh_expire_type = garp->n4g_ext_res->n4g_fet;
1922 
1923 	mutex_enter(&mi->mi_lock);
1924 	if (garp->n4g_ext_res->n4g_pc4.pc4_link_support)
1925 		mi->mi_flags |= MI4_LINK;
1926 	if (garp->n4g_ext_res->n4g_pc4.pc4_symlink_support)
1927 		mi->mi_flags |= MI4_SYMLINK;
1928 	if (garp->n4g_ext_res->n4g_suppattrs & FATTR4_ACL_MASK)
1929 		mi->mi_flags |= MI4_ACL;
1930 	mutex_exit(&mi->mi_lock);
1931 
1932 	if (garp->n4g_ext_res->n4g_maxread == 0)
1933 		mi->mi_tsize =
1934 		    MIN(MAXBSIZE, mi->mi_tsize);
1935 	else
1936 		mi->mi_tsize =
1937 		    MIN(garp->n4g_ext_res->n4g_maxread,
1938 		    mi->mi_tsize);
1939 
1940 	if (garp->n4g_ext_res->n4g_maxwrite == 0)
1941 		mi->mi_stsize =
1942 		    MIN(MAXBSIZE, mi->mi_stsize);
1943 	else
1944 		mi->mi_stsize =
1945 		    MIN(garp->n4g_ext_res->n4g_maxwrite,
1946 		    mi->mi_stsize);
1947 
1948 	if (garp->n4g_ext_res->n4g_maxfilesize != 0)
1949 		mi->mi_maxfilesize =
1950 		    MIN(garp->n4g_ext_res->n4g_maxfilesize,
1951 		    mi->mi_maxfilesize);
1952 
1953 	/*
1954 	 * If the final component is a a symbolic link, resolve the symlink,
1955 	 * then try mount again using the new path.
1956 	 *
1957 	 * Assume no symbolic link for root filesysm "/".
1958 	 */
1959 	if (*vtp == VLNK) {
1960 		/*
1961 		 * nthcomp is the total result length minus
1962 		 * the 1st 2 OPs (PUTROOTFH, GETFH),
1963 		 * then divided by 3 (LOOKUP,GETFH,GETATTR)
1964 		 *
1965 		 * e.g. PUTROOTFH GETFH LOOKUP 1st-comp GETFH GETATTR
1966 		 *	LOOKUP 2nd-comp GETFH GETATTR
1967 		 *
1968 		 *	(8 - 2)/3 = 2
1969 		 */
1970 		nthcomp = (res.array_len - 2)/3;
1971 
1972 		/*
1973 		 * Need to call nfs4_end_op before resolve_sympath to avoid
1974 		 * potential nfs4_start_op deadlock. See RFE 4777612.
1975 		 */
1976 		if (!recovery)
1977 			nfs4_end_fop(mi, NULL, NULL, OH_MOUNT, &recov_state,
1978 			    needrecov);
1979 
1980 		ep->error = resolve_sympath(mi, svp, nthcomp, resfhp, cr,
1981 		    flags);
1982 
1983 		nfs4args_lookup_free(argop, num_argops);
1984 		kmem_free(argop, lookuparg.arglen * sizeof (nfs_argop4));
1985 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
1986 
1987 		if (ep->error)
1988 			return;
1989 
1990 		goto recov_retry;
1991 	}
1992 
1993 	/*
1994 	 * We need to figure out where in the compound the getfh
1995 	 * for the parent directory is. If the object to be mounted is
1996 	 * the root, then there is no lookup at all:
1997 	 * PUTROOTFH, GETFH.
1998 	 * If the object to be mounted is in the root, then the compound is:
1999 	 * PUTROOTFH, GETFH, LOOKUP, GETFH, GETATTR.
2000 	 * In either of these cases, the index of the GETFH is 1.
2001 	 * If it is not at the root, then it's something like:
2002 	 * PUTROOTFH, GETFH, LOOKUP, GETFH, GETATTR,
2003 	 * LOOKUP, GETFH, GETATTR
2004 	 * In this case, the index is llndx (last lookup index) - 2.
2005 	 */
2006 	if (llndx == -1 || llndx == 2)
2007 		resop = &res.array[1];
2008 	else {
2009 		ASSERT(llndx > 2);
2010 		resop = &res.array[llndx-2];
2011 	}
2012 
2013 	ASSERT(resop->resop == OP_GETFH);
2014 	tmpfhp = &resop->nfs_resop4_u.opgetfh.object;
2015 
2016 	/* save the filehandles for the replica */
2017 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
2018 	ASSERT(tmpfhp->nfs_fh4_len <= NFS4_FHSIZE);
2019 	svp->sv_pfhandle.fh_len = tmpfhp->nfs_fh4_len;
2020 	bcopy(tmpfhp->nfs_fh4_val, svp->sv_pfhandle.fh_buf,
2021 	    tmpfhp->nfs_fh4_len);
2022 	ASSERT(resfhp->nfs_fh4_len <= NFS4_FHSIZE);
2023 	svp->sv_fhandle.fh_len = resfhp->nfs_fh4_len;
2024 	bcopy(resfhp->nfs_fh4_val, svp->sv_fhandle.fh_buf, resfhp->nfs_fh4_len);
2025 
2026 	/* initialize fsid and supp_attrs for server fs */
2027 	svp->sv_fsid = garp->n4g_fsid;
2028 	svp->sv_supp_attrs =
2029 	    garp->n4g_ext_res->n4g_suppattrs | FATTR4_MANDATTR_MASK;
2030 
2031 	nfs_rw_exit(&svp->sv_lock);
2032 	nfs4args_lookup_free(argop, num_argops);
2033 	kmem_free(argop, lookuparg.arglen * sizeof (nfs_argop4));
2034 	(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
2035 	if (!recovery)
2036 		nfs4_end_fop(mi, NULL, NULL, OH_MOUNT, &recov_state, needrecov);
2037 }
2038 
2039 /*
2040  * Save a copy of Servinfo4_t structure.
2041  * We might need when there is a failure in getting file handle
2042  * in case of a referral to replace servinfo4 struct and try again.
2043  */
2044 static struct servinfo4 *
2045 copy_svp(servinfo4_t *nsvp)
2046 {
2047 	servinfo4_t *svp = NULL;
2048 	struct knetconfig *sknconf, *tknconf;
2049 	struct netbuf *saddr, *taddr;
2050 
2051 	svp = kmem_zalloc(sizeof (*svp), KM_SLEEP);
2052 	nfs_rw_init(&svp->sv_lock, NULL, RW_DEFAULT, NULL);
2053 	svp->sv_flags = nsvp->sv_flags;
2054 	svp->sv_fsid = nsvp->sv_fsid;
2055 	svp->sv_hostnamelen = nsvp->sv_hostnamelen;
2056 	svp->sv_pathlen = nsvp->sv_pathlen;
2057 	svp->sv_supp_attrs = nsvp->sv_supp_attrs;
2058 
2059 	svp->sv_path = kmem_alloc(svp->sv_pathlen, KM_SLEEP);
2060 	svp->sv_hostname = kmem_alloc(svp->sv_hostnamelen, KM_SLEEP);
2061 	bcopy(nsvp->sv_hostname, svp->sv_hostname, svp->sv_hostnamelen);
2062 	bcopy(nsvp->sv_path, svp->sv_path, svp->sv_pathlen);
2063 
2064 	saddr = &nsvp->sv_addr;
2065 	taddr = &svp->sv_addr;
2066 	taddr->maxlen = saddr->maxlen;
2067 	taddr->len = saddr->len;
2068 	if (saddr->len > 0) {
2069 		taddr->buf = kmem_zalloc(saddr->maxlen, KM_SLEEP);
2070 		bcopy(saddr->buf, taddr->buf, saddr->len);
2071 	}
2072 
2073 	svp->sv_knconf = kmem_zalloc(sizeof (struct knetconfig), KM_SLEEP);
2074 	sknconf = nsvp->sv_knconf;
2075 	tknconf = svp->sv_knconf;
2076 	tknconf->knc_semantics = sknconf->knc_semantics;
2077 	tknconf->knc_rdev = sknconf->knc_rdev;
2078 	if (sknconf->knc_proto != NULL) {
2079 		tknconf->knc_proto = kmem_zalloc(KNC_STRSIZE, KM_SLEEP);
2080 		bcopy(sknconf->knc_proto, (char *)tknconf->knc_proto,
2081 		    KNC_STRSIZE);
2082 	}
2083 	if (sknconf->knc_protofmly != NULL) {
2084 		tknconf->knc_protofmly = kmem_zalloc(KNC_STRSIZE, KM_SLEEP);
2085 		bcopy(sknconf->knc_protofmly, (char *)tknconf->knc_protofmly,
2086 		    KNC_STRSIZE);
2087 	}
2088 
2089 	if (nsvp->sv_origknconf != NULL) {
2090 		svp->sv_origknconf = kmem_zalloc(sizeof (struct knetconfig),
2091 		    KM_SLEEP);
2092 		sknconf = nsvp->sv_origknconf;
2093 		tknconf = svp->sv_origknconf;
2094 		tknconf->knc_semantics = sknconf->knc_semantics;
2095 		tknconf->knc_rdev = sknconf->knc_rdev;
2096 		if (sknconf->knc_proto != NULL) {
2097 			tknconf->knc_proto = kmem_zalloc(KNC_STRSIZE, KM_SLEEP);
2098 			bcopy(sknconf->knc_proto, (char *)tknconf->knc_proto,
2099 			    KNC_STRSIZE);
2100 		}
2101 		if (sknconf->knc_protofmly != NULL) {
2102 			tknconf->knc_protofmly = kmem_zalloc(KNC_STRSIZE,
2103 			    KM_SLEEP);
2104 			bcopy(sknconf->knc_protofmly,
2105 			    (char *)tknconf->knc_protofmly, KNC_STRSIZE);
2106 		}
2107 	}
2108 
2109 	svp->sv_secdata = copy_sec_data(nsvp->sv_secdata);
2110 	svp->sv_dhsec = copy_sec_data(svp->sv_dhsec);
2111 	/*
2112 	 * Rest of the security information is not copied as they are built
2113 	 * with the information available from secdata and dhsec.
2114 	 */
2115 	svp->sv_next = NULL;
2116 
2117 	return (svp);
2118 }
2119 
2120 servinfo4_t *
2121 restore_svp(mntinfo4_t *mi, servinfo4_t *svp, servinfo4_t *origsvp)
2122 {
2123 	servinfo4_t *srvnext, *tmpsrv;
2124 
2125 	if (strcmp(svp->sv_hostname, origsvp->sv_hostname) != 0) {
2126 		/*
2127 		 * Since the hostname changed, we must be dealing
2128 		 * with a referral, and the lookup failed.  We will
2129 		 * restore the whole servinfo4_t to what it was before.
2130 		 */
2131 		srvnext = svp->sv_next;
2132 		svp->sv_next = NULL;
2133 		tmpsrv = copy_svp(origsvp);
2134 		sv4_free(svp);
2135 		svp = tmpsrv;
2136 		svp->sv_next = srvnext;
2137 		mutex_enter(&mi->mi_lock);
2138 		mi->mi_servers = svp;
2139 		mi->mi_curr_serv = svp;
2140 		mutex_exit(&mi->mi_lock);
2141 
2142 	} else if (origsvp->sv_pathlen != svp->sv_pathlen) {
2143 
2144 		/*
2145 		 * For symlink case: restore original path because
2146 		 * it might have contained symlinks that were
2147 		 * expanded by nfsgetfh_otw before the failure occurred.
2148 		 */
2149 		(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
2150 		kmem_free(svp->sv_path, svp->sv_pathlen);
2151 		svp->sv_path =
2152 		    kmem_alloc(origsvp->sv_pathlen, KM_SLEEP);
2153 		svp->sv_pathlen = origsvp->sv_pathlen;
2154 		bcopy(origsvp->sv_path, svp->sv_path,
2155 		    origsvp->sv_pathlen);
2156 		nfs_rw_exit(&svp->sv_lock);
2157 	}
2158 	return (svp);
2159 }
2160 
2161 static ushort_t nfs4_max_threads = 8;	/* max number of active async threads */
2162 static uint_t nfs4_bsize = 32 * 1024;	/* client `block' size */
2163 static uint_t nfs4_async_clusters = 1;	/* # of reqs from each async queue */
2164 static uint_t nfs4_cots_timeo = NFS_COTS_TIMEO;
2165 
2166 /*
2167  * Remap the root filehandle for the given filesystem.
2168  *
2169  * results returned via the nfs4_error_t parameter.
2170  */
2171 void
2172 nfs4_remap_root(mntinfo4_t *mi, nfs4_error_t *ep, int flags)
2173 {
2174 	struct servinfo4 *svp, *origsvp;
2175 	vtype_t vtype;
2176 	nfs_fh4 rootfh;
2177 	int getfh_flags;
2178 	int num_retry;
2179 
2180 	mutex_enter(&mi->mi_lock);
2181 
2182 remap_retry:
2183 	svp = mi->mi_curr_serv;
2184 	getfh_flags =
2185 	    (flags & NFS4_REMAP_NEEDSOP) ? NFS4_GETFH_NEEDSOP : 0;
2186 	getfh_flags |=
2187 	    (mi->mi_flags & MI4_PUBLIC) ? NFS4_GETFH_PUBLIC : 0;
2188 	mutex_exit(&mi->mi_lock);
2189 
2190 	/*
2191 	 * Just in case server path being mounted contains
2192 	 * symlinks and fails w/STALE, save the initial sv_path
2193 	 * so we can redrive the initial mount compound with the
2194 	 * initial sv_path -- not a symlink-expanded version.
2195 	 *
2196 	 * This could only happen if a symlink was expanded
2197 	 * and the expanded mount compound failed stale.  Because
2198 	 * it could be the case that the symlink was removed at
2199 	 * the server (and replaced with another symlink/dir,
2200 	 * we need to use the initial sv_path when attempting
2201 	 * to re-lookup everything and recover.
2202 	 */
2203 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
2204 	origsvp = copy_svp(svp);
2205 	nfs_rw_exit(&svp->sv_lock);
2206 
2207 	num_retry = nfs4_max_mount_retry;
2208 
2209 	do {
2210 		/*
2211 		 * Get the root fh from the server.  Retry nfs4_max_mount_retry
2212 		 * (2) times if it fails with STALE since the recovery
2213 		 * infrastructure doesn't do STALE recovery for components
2214 		 * of the server path to the object being mounted.
2215 		 */
2216 		nfs4getfh_otw(mi, svp, &vtype, getfh_flags, CRED(), ep);
2217 
2218 		if (ep->error == 0 && ep->stat == NFS4_OK)
2219 			break;
2220 
2221 		/*
2222 		 * For some reason, the mount compound failed.  Before
2223 		 * retrying, we need to restore original conditions.
2224 		 */
2225 		svp = restore_svp(mi, svp, origsvp);
2226 
2227 	} while (num_retry-- > 0);
2228 
2229 	sv4_free(origsvp);
2230 
2231 	if (ep->error != 0 || ep->stat != 0) {
2232 		return;
2233 	}
2234 
2235 	if (vtype != VNON && vtype != mi->mi_type) {
2236 		/* shouldn't happen */
2237 		zcmn_err(mi->mi_zone->zone_id, CE_WARN,
2238 		    "nfs4_remap_root: server root vnode type (%d) doesn't "
2239 		    "match mount info (%d)", vtype, mi->mi_type);
2240 	}
2241 
2242 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
2243 	rootfh.nfs_fh4_val = svp->sv_fhandle.fh_buf;
2244 	rootfh.nfs_fh4_len = svp->sv_fhandle.fh_len;
2245 	nfs_rw_exit(&svp->sv_lock);
2246 	sfh4_update(mi->mi_rootfh, &rootfh);
2247 
2248 	/*
2249 	 * It's possible that recovery took place on the filesystem
2250 	 * and the server has been updated between the time we did
2251 	 * the nfs4getfh_otw and now. Re-drive the otw operation
2252 	 * to make sure we have a good fh.
2253 	 */
2254 	mutex_enter(&mi->mi_lock);
2255 	if (mi->mi_curr_serv != svp)
2256 		goto remap_retry;
2257 
2258 	mutex_exit(&mi->mi_lock);
2259 }
2260 
2261 static int
2262 nfs4rootvp(vnode_t **rtvpp, vfs_t *vfsp, struct servinfo4 *svp_head,
2263     int flags, cred_t *cr, zone_t *zone)
2264 {
2265 	vnode_t *rtvp = NULL;
2266 	mntinfo4_t *mi;
2267 	dev_t nfs_dev;
2268 	int error = 0;
2269 	rnode4_t *rp;
2270 	int i, len;
2271 	struct vattr va;
2272 	vtype_t vtype = VNON;
2273 	vtype_t tmp_vtype = VNON;
2274 	struct servinfo4 *firstsvp = NULL, *svp = svp_head;
2275 	nfs4_oo_hash_bucket_t *bucketp;
2276 	nfs_fh4 fh;
2277 	char *droptext = "";
2278 	struct nfs_stats *nfsstatsp;
2279 	nfs4_fname_t *mfname;
2280 	nfs4_error_t e;
2281 	int num_retry, removed;
2282 	cred_t *lcr = NULL, *tcr = cr;
2283 	struct servinfo4 *origsvp;
2284 	char *resource;
2285 
2286 	nfsstatsp = zone_getspecific(nfsstat_zone_key, nfs_zone());
2287 	ASSERT(nfsstatsp != NULL);
2288 
2289 	ASSERT(nfs_zone() == zone);
2290 	ASSERT(crgetref(cr));
2291 
2292 	/*
2293 	 * Create a mount record and link it to the vfs struct.
2294 	 */
2295 	mi = kmem_zalloc(sizeof (*mi), KM_SLEEP);
2296 	mutex_init(&mi->mi_lock, NULL, MUTEX_DEFAULT, NULL);
2297 	nfs_rw_init(&mi->mi_recovlock, NULL, RW_DEFAULT, NULL);
2298 	nfs_rw_init(&mi->mi_rename_lock, NULL, RW_DEFAULT, NULL);
2299 	nfs_rw_init(&mi->mi_fh_lock, NULL, RW_DEFAULT, NULL);
2300 
2301 	if (!(flags & NFSMNT_SOFT))
2302 		mi->mi_flags |= MI4_HARD;
2303 	if ((flags & NFSMNT_NOPRINT))
2304 		mi->mi_flags |= MI4_NOPRINT;
2305 	if (flags & NFSMNT_INT)
2306 		mi->mi_flags |= MI4_INT;
2307 	if (flags & NFSMNT_PUBLIC)
2308 		mi->mi_flags |= MI4_PUBLIC;
2309 	if (flags & NFSMNT_MIRRORMOUNT)
2310 		mi->mi_flags |= MI4_MIRRORMOUNT;
2311 	if (flags & NFSMNT_REFERRAL)
2312 		mi->mi_flags |= MI4_REFERRAL;
2313 	mi->mi_retrans = NFS_RETRIES;
2314 	if (svp->sv_knconf->knc_semantics == NC_TPI_COTS_ORD ||
2315 	    svp->sv_knconf->knc_semantics == NC_TPI_COTS)
2316 		mi->mi_timeo = nfs4_cots_timeo;
2317 	else
2318 		mi->mi_timeo = NFS_TIMEO;
2319 	mi->mi_prog = NFS_PROGRAM;
2320 	mi->mi_vers = NFS_V4;
2321 	mi->mi_rfsnames = rfsnames_v4;
2322 	mi->mi_reqs = nfsstatsp->nfs_stats_v4.rfsreqcnt_ptr;
2323 	cv_init(&mi->mi_failover_cv, NULL, CV_DEFAULT, NULL);
2324 	mi->mi_servers = svp;
2325 	mi->mi_curr_serv = svp;
2326 	mi->mi_acregmin = SEC2HR(ACREGMIN);
2327 	mi->mi_acregmax = SEC2HR(ACREGMAX);
2328 	mi->mi_acdirmin = SEC2HR(ACDIRMIN);
2329 	mi->mi_acdirmax = SEC2HR(ACDIRMAX);
2330 	mi->mi_fh_expire_type = FH4_PERSISTENT;
2331 	mi->mi_clientid_next = NULL;
2332 	mi->mi_clientid_prev = NULL;
2333 	mi->mi_srv = NULL;
2334 	mi->mi_grace_wait = 0;
2335 	mi->mi_error = 0;
2336 	mi->mi_srvsettime = 0;
2337 	mi->mi_srvset_cnt = 0;
2338 
2339 	mi->mi_count = 1;
2340 
2341 	mi->mi_tsize = nfs4_tsize(svp->sv_knconf);
2342 	mi->mi_stsize = mi->mi_tsize;
2343 
2344 	if (flags & NFSMNT_DIRECTIO)
2345 		mi->mi_flags |= MI4_DIRECTIO;
2346 
2347 	mi->mi_flags |= MI4_MOUNTING;
2348 
2349 	/*
2350 	 * Make a vfs struct for nfs.  We do this here instead of below
2351 	 * because rtvp needs a vfs before we can do a getattr on it.
2352 	 *
2353 	 * Assign a unique device id to the mount
2354 	 */
2355 	mutex_enter(&nfs_minor_lock);
2356 	do {
2357 		nfs_minor = (nfs_minor + 1) & MAXMIN32;
2358 		nfs_dev = makedevice(nfs_major, nfs_minor);
2359 	} while (vfs_devismounted(nfs_dev));
2360 	mutex_exit(&nfs_minor_lock);
2361 
2362 	vfsp->vfs_dev = nfs_dev;
2363 	vfs_make_fsid(&vfsp->vfs_fsid, nfs_dev, nfs4fstyp);
2364 	vfsp->vfs_data = (caddr_t)mi;
2365 	vfsp->vfs_fstype = nfsfstyp;
2366 	vfsp->vfs_bsize = nfs4_bsize;
2367 
2368 	/*
2369 	 * Initialize fields used to support async putpage operations.
2370 	 */
2371 	for (i = 0; i < NFS4_ASYNC_TYPES; i++)
2372 		mi->mi_async_clusters[i] = nfs4_async_clusters;
2373 	mi->mi_async_init_clusters = nfs4_async_clusters;
2374 	mi->mi_async_curr = &mi->mi_async_reqs[0];
2375 	mi->mi_max_threads = nfs4_max_threads;
2376 	mutex_init(&mi->mi_async_lock, NULL, MUTEX_DEFAULT, NULL);
2377 	cv_init(&mi->mi_async_reqs_cv, NULL, CV_DEFAULT, NULL);
2378 	cv_init(&mi->mi_async_work_cv, NULL, CV_DEFAULT, NULL);
2379 	cv_init(&mi->mi_async_cv, NULL, CV_DEFAULT, NULL);
2380 	cv_init(&mi->mi_inact_req_cv, NULL, CV_DEFAULT, NULL);
2381 
2382 	mi->mi_vfsp = vfsp;
2383 	zone_hold(mi->mi_zone = zone);
2384 	nfs4_mi_zonelist_add(mi);
2385 
2386 	/*
2387 	 * Initialize the <open owner/cred> hash table.
2388 	 */
2389 	for (i = 0; i < NFS4_NUM_OO_BUCKETS; i++) {
2390 		bucketp = &(mi->mi_oo_list[i]);
2391 		mutex_init(&bucketp->b_lock, NULL, MUTEX_DEFAULT, NULL);
2392 		list_create(&bucketp->b_oo_hash_list,
2393 		    sizeof (nfs4_open_owner_t),
2394 		    offsetof(nfs4_open_owner_t, oo_hash_node));
2395 	}
2396 
2397 	/*
2398 	 * Initialize the freed open owner list.
2399 	 */
2400 	mi->mi_foo_num = 0;
2401 	mi->mi_foo_max = NFS4_NUM_FREED_OPEN_OWNERS;
2402 	list_create(&mi->mi_foo_list, sizeof (nfs4_open_owner_t),
2403 	    offsetof(nfs4_open_owner_t, oo_foo_node));
2404 
2405 	list_create(&mi->mi_lost_state, sizeof (nfs4_lost_rqst_t),
2406 	    offsetof(nfs4_lost_rqst_t, lr_node));
2407 
2408 	list_create(&mi->mi_bseqid_list, sizeof (nfs4_bseqid_entry_t),
2409 	    offsetof(nfs4_bseqid_entry_t, bs_node));
2410 
2411 	/*
2412 	 * Initialize the msg buffer.
2413 	 */
2414 	list_create(&mi->mi_msg_list, sizeof (nfs4_debug_msg_t),
2415 	    offsetof(nfs4_debug_msg_t, msg_node));
2416 	mi->mi_msg_count = 0;
2417 	mutex_init(&mi->mi_msg_list_lock, NULL, MUTEX_DEFAULT, NULL);
2418 
2419 	/*
2420 	 * Initialize kstats
2421 	 */
2422 	nfs4_mnt_kstat_init(vfsp);
2423 
2424 	/*
2425 	 * Initialize the shared filehandle pool.
2426 	 */
2427 	sfh4_createtab(&mi->mi_filehandles);
2428 
2429 	/*
2430 	 * Save server path we're attempting to mount.
2431 	 */
2432 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
2433 	origsvp = copy_svp(svp);
2434 	nfs_rw_exit(&svp->sv_lock);
2435 
2436 	/*
2437 	 * Make the GETFH call to get root fh for each replica.
2438 	 */
2439 	if (svp_head->sv_next)
2440 		droptext = ", dropping replica";
2441 
2442 	/*
2443 	 * If the uid is set then set the creds for secure mounts
2444 	 * by proxy processes such as automountd.
2445 	 */
2446 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
2447 	if (svp->sv_secdata->uid != 0 &&
2448 	    svp->sv_secdata->rpcflavor == RPCSEC_GSS) {
2449 		lcr = crdup(cr);
2450 		(void) crsetugid(lcr, svp->sv_secdata->uid, crgetgid(cr));
2451 		tcr = lcr;
2452 	}
2453 	nfs_rw_exit(&svp->sv_lock);
2454 	for (svp = svp_head; svp; svp = svp->sv_next) {
2455 		if (nfs4_chkdup_servinfo4(svp_head, svp)) {
2456 			nfs_cmn_err(error, CE_WARN,
2457 			    VERS_MSG "Host %s is a duplicate%s",
2458 			    svp->sv_hostname, droptext);
2459 			(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
2460 			svp->sv_flags |= SV4_NOTINUSE;
2461 			nfs_rw_exit(&svp->sv_lock);
2462 			continue;
2463 		}
2464 		mi->mi_curr_serv = svp;
2465 
2466 		/*
2467 		 * Just in case server path being mounted contains
2468 		 * symlinks and fails w/STALE, save the initial sv_path
2469 		 * so we can redrive the initial mount compound with the
2470 		 * initial sv_path -- not a symlink-expanded version.
2471 		 *
2472 		 * This could only happen if a symlink was expanded
2473 		 * and the expanded mount compound failed stale.  Because
2474 		 * it could be the case that the symlink was removed at
2475 		 * the server (and replaced with another symlink/dir,
2476 		 * we need to use the initial sv_path when attempting
2477 		 * to re-lookup everything and recover.
2478 		 *
2479 		 * Other mount errors should evenutally be handled here also
2480 		 * (NFS4ERR_DELAY, NFS4ERR_RESOURCE).  For now, all mount
2481 		 * failures will result in mount being redriven a few times.
2482 		 */
2483 		num_retry = nfs4_max_mount_retry;
2484 		do {
2485 			nfs4getfh_otw(mi, svp, &tmp_vtype,
2486 			    ((flags & NFSMNT_PUBLIC) ? NFS4_GETFH_PUBLIC : 0) |
2487 			    NFS4_GETFH_NEEDSOP, tcr, &e);
2488 
2489 			if (e.error == 0 && e.stat == NFS4_OK)
2490 				break;
2491 
2492 			/*
2493 			 * For some reason, the mount compound failed.  Before
2494 			 * retrying, we need to restore original conditions.
2495 			 */
2496 			svp = restore_svp(mi, svp, origsvp);
2497 			svp_head = svp;
2498 
2499 		} while (num_retry-- > 0);
2500 		error = e.error ? e.error : geterrno4(e.stat);
2501 		if (error) {
2502 			nfs_cmn_err(error, CE_WARN,
2503 			    VERS_MSG "initial call to %s failed%s: %m",
2504 			    svp->sv_hostname, droptext);
2505 			(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
2506 			svp->sv_flags |= SV4_NOTINUSE;
2507 			nfs_rw_exit(&svp->sv_lock);
2508 			mi->mi_flags &= ~MI4_RECOV_FAIL;
2509 			mi->mi_error = 0;
2510 			continue;
2511 		}
2512 
2513 		if (tmp_vtype == VBAD) {
2514 			zcmn_err(mi->mi_zone->zone_id, CE_WARN,
2515 			    VERS_MSG "%s returned a bad file type for "
2516 			    "root%s", svp->sv_hostname, droptext);
2517 			(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
2518 			svp->sv_flags |= SV4_NOTINUSE;
2519 			nfs_rw_exit(&svp->sv_lock);
2520 			continue;
2521 		}
2522 
2523 		if (vtype == VNON) {
2524 			vtype = tmp_vtype;
2525 		} else if (vtype != tmp_vtype) {
2526 			zcmn_err(mi->mi_zone->zone_id, CE_WARN,
2527 			    VERS_MSG "%s returned a different file type "
2528 			    "for root%s", svp->sv_hostname, droptext);
2529 			(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
2530 			svp->sv_flags |= SV4_NOTINUSE;
2531 			nfs_rw_exit(&svp->sv_lock);
2532 			continue;
2533 		}
2534 		if (firstsvp == NULL)
2535 			firstsvp = svp;
2536 	}
2537 
2538 	if (firstsvp == NULL) {
2539 		if (error == 0)
2540 			error = ENOENT;
2541 		goto bad;
2542 	}
2543 
2544 	mi->mi_curr_serv = svp = firstsvp;
2545 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
2546 	ASSERT((mi->mi_curr_serv->sv_flags & SV4_NOTINUSE) == 0);
2547 	fh.nfs_fh4_len = svp->sv_fhandle.fh_len;
2548 	fh.nfs_fh4_val = svp->sv_fhandle.fh_buf;
2549 	mi->mi_rootfh = sfh4_get(&fh, mi);
2550 	fh.nfs_fh4_len = svp->sv_pfhandle.fh_len;
2551 	fh.nfs_fh4_val = svp->sv_pfhandle.fh_buf;
2552 	mi->mi_srvparentfh = sfh4_get(&fh, mi);
2553 	nfs_rw_exit(&svp->sv_lock);
2554 
2555 	/*
2556 	 * Get the fname for filesystem root.
2557 	 */
2558 	mi->mi_fname = fn_get(NULL, ".", mi->mi_rootfh);
2559 	mfname = mi->mi_fname;
2560 	fn_hold(mfname);
2561 
2562 	/*
2563 	 * Make the root vnode without attributes.
2564 	 */
2565 	rtvp = makenfs4node_by_fh(mi->mi_rootfh, NULL,
2566 	    &mfname, NULL, mi, cr, gethrtime());
2567 	rtvp->v_type = vtype;
2568 
2569 	mi->mi_curread = mi->mi_tsize;
2570 	mi->mi_curwrite = mi->mi_stsize;
2571 
2572 	/*
2573 	 * Start the manager thread responsible for handling async worker
2574 	 * threads.
2575 	 */
2576 	MI4_HOLD(mi);
2577 	VFS_HOLD(vfsp);	/* add reference for thread */
2578 	mi->mi_manager_thread = zthread_create(NULL, 0, nfs4_async_manager,
2579 	    vfsp, 0, minclsyspri);
2580 	ASSERT(mi->mi_manager_thread != NULL);
2581 
2582 	/*
2583 	 * Create the thread that handles over-the-wire calls for
2584 	 * VOP_INACTIVE.
2585 	 * This needs to happen after the manager thread is created.
2586 	 */
2587 	MI4_HOLD(mi);
2588 	mi->mi_inactive_thread = zthread_create(NULL, 0, nfs4_inactive_thread,
2589 	    mi, 0, minclsyspri);
2590 	ASSERT(mi->mi_inactive_thread != NULL);
2591 
2592 	/* If we didn't get a type, get one now */
2593 	if (rtvp->v_type == VNON) {
2594 		va.va_mask = AT_TYPE;
2595 		error = nfs4getattr(rtvp, &va, tcr);
2596 		if (error)
2597 			goto bad;
2598 		rtvp->v_type = va.va_type;
2599 	}
2600 
2601 	mi->mi_type = rtvp->v_type;
2602 
2603 	mutex_enter(&mi->mi_lock);
2604 	mi->mi_flags &= ~MI4_MOUNTING;
2605 	mutex_exit(&mi->mi_lock);
2606 
2607 	/* Update VFS with new server and path info */
2608 	if ((strcmp(svp->sv_hostname, origsvp->sv_hostname) != 0) ||
2609 	    (strcmp(svp->sv_path, origsvp->sv_path) != 0)) {
2610 		len = svp->sv_hostnamelen + svp->sv_pathlen;
2611 		resource = kmem_zalloc(len, KM_SLEEP);
2612 		(void) strcat(resource, svp->sv_hostname);
2613 		(void) strcat(resource, ":");
2614 		(void) strcat(resource, svp->sv_path);
2615 		vfs_setresource(vfsp, resource);
2616 		kmem_free(resource, len);
2617 	}
2618 
2619 	sv4_free(origsvp);
2620 	*rtvpp = rtvp;
2621 	if (lcr != NULL)
2622 		crfree(lcr);
2623 
2624 	return (0);
2625 bad:
2626 	/*
2627 	 * An error occurred somewhere, need to clean up...
2628 	 */
2629 	if (lcr != NULL)
2630 		crfree(lcr);
2631 
2632 	if (rtvp != NULL) {
2633 		/*
2634 		 * We need to release our reference to the root vnode and
2635 		 * destroy the mntinfo4 struct that we just created.
2636 		 */
2637 		rp = VTOR4(rtvp);
2638 		if (rp->r_flags & R4HASHED)
2639 			rp4_rmhash(rp);
2640 		VN_RELE(rtvp);
2641 	}
2642 	nfs4_async_stop(vfsp);
2643 	nfs4_async_manager_stop(vfsp);
2644 	removed = nfs4_mi_zonelist_remove(mi);
2645 	if (removed)
2646 		zone_rele(mi->mi_zone);
2647 
2648 	/*
2649 	 * This releases the initial "hold" of the mi since it will never
2650 	 * be referenced by the vfsp.  Also, when mount returns to vfs.c
2651 	 * with an error, the vfsp will be destroyed, not rele'd.
2652 	 */
2653 	MI4_RELE(mi);
2654 
2655 	if (origsvp != NULL)
2656 		sv4_free(origsvp);
2657 
2658 	*rtvpp = NULL;
2659 	return (error);
2660 }
2661 
2662 /*
2663  * vfs operations
2664  */
2665 static int
2666 nfs4_unmount(vfs_t *vfsp, int flag, cred_t *cr)
2667 {
2668 	mntinfo4_t		*mi;
2669 	ushort_t		omax;
2670 	int			removed;
2671 
2672 	bool_t			must_unlock;
2673 
2674 	nfs4_ephemeral_tree_t	*eph_tree;
2675 
2676 	if (secpolicy_fs_unmount(cr, vfsp) != 0)
2677 		return (EPERM);
2678 
2679 	mi = VFTOMI4(vfsp);
2680 
2681 	if (flag & MS_FORCE) {
2682 		vfsp->vfs_flag |= VFS_UNMOUNTED;
2683 		if (nfs_zone() != mi->mi_zone) {
2684 			/*
2685 			 * If the request is coming from the wrong zone,
2686 			 * we don't want to create any new threads, and
2687 			 * performance is not a concern.  Do everything
2688 			 * inline.
2689 			 */
2690 			NFS4_DEBUG(nfs4_client_zone_debug, (CE_NOTE,
2691 			    "nfs4_unmount x-zone forced unmount of vfs %p\n",
2692 			    (void *)vfsp));
2693 			nfs4_free_mount(vfsp, flag, cr);
2694 		} else {
2695 			/*
2696 			 * Free data structures asynchronously, to avoid
2697 			 * blocking the current thread (for performance
2698 			 * reasons only).
2699 			 */
2700 			async_free_mount(vfsp, flag, cr);
2701 		}
2702 
2703 		return (0);
2704 	}
2705 
2706 	/*
2707 	 * Wait until all asynchronous putpage operations on
2708 	 * this file system are complete before flushing rnodes
2709 	 * from the cache.
2710 	 */
2711 	omax = mi->mi_max_threads;
2712 	if (nfs4_async_stop_sig(vfsp))
2713 		return (EINTR);
2714 
2715 	r4flush(vfsp, cr);
2716 
2717 	/*
2718 	 * About the only reason that this would fail would be
2719 	 * that the harvester is already busy tearing down this
2720 	 * node. So we fail back to the caller and let them try
2721 	 * again when needed.
2722 	 */
2723 	if (nfs4_ephemeral_umount(mi, flag, cr,
2724 	    &must_unlock, &eph_tree)) {
2725 		ASSERT(must_unlock == FALSE);
2726 		mutex_enter(&mi->mi_async_lock);
2727 		mi->mi_max_threads = omax;
2728 		mutex_exit(&mi->mi_async_lock);
2729 
2730 		return (EBUSY);
2731 	}
2732 
2733 	/*
2734 	 * If there are any active vnodes on this file system,
2735 	 * then the file system is busy and can't be unmounted.
2736 	 */
2737 	if (check_rtable4(vfsp)) {
2738 		nfs4_ephemeral_umount_unlock(&must_unlock, &eph_tree);
2739 
2740 		mutex_enter(&mi->mi_async_lock);
2741 		mi->mi_max_threads = omax;
2742 		mutex_exit(&mi->mi_async_lock);
2743 
2744 		return (EBUSY);
2745 	}
2746 
2747 	/*
2748 	 * The unmount can't fail from now on, so record any
2749 	 * ephemeral changes.
2750 	 */
2751 	nfs4_ephemeral_umount_activate(mi, &must_unlock, &eph_tree);
2752 
2753 	/*
2754 	 * There are no active files that could require over-the-wire
2755 	 * calls to the server, so stop the async manager and the
2756 	 * inactive thread.
2757 	 */
2758 	nfs4_async_manager_stop(vfsp);
2759 
2760 	/*
2761 	 * Destroy all rnodes belonging to this file system from the
2762 	 * rnode hash queues and purge any resources allocated to
2763 	 * them.
2764 	 */
2765 	destroy_rtable4(vfsp, cr);
2766 	vfsp->vfs_flag |= VFS_UNMOUNTED;
2767 
2768 	nfs4_remove_mi_from_server(mi, NULL);
2769 	removed = nfs4_mi_zonelist_remove(mi);
2770 	if (removed)
2771 		zone_rele(mi->mi_zone);
2772 
2773 	return (0);
2774 }
2775 
2776 /*
2777  * find root of nfs
2778  */
2779 static int
2780 nfs4_root(vfs_t *vfsp, vnode_t **vpp)
2781 {
2782 	mntinfo4_t *mi;
2783 	vnode_t *vp;
2784 	nfs4_fname_t *mfname;
2785 	servinfo4_t *svp;
2786 
2787 	mi = VFTOMI4(vfsp);
2788 
2789 	if (nfs_zone() != mi->mi_zone)
2790 		return (EPERM);
2791 
2792 	svp = mi->mi_curr_serv;
2793 	if (svp) {
2794 		(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
2795 		if (svp->sv_flags & SV4_ROOT_STALE) {
2796 			nfs_rw_exit(&svp->sv_lock);
2797 
2798 			(void) nfs_rw_enter_sig(&svp->sv_lock, RW_WRITER, 0);
2799 			if (svp->sv_flags & SV4_ROOT_STALE) {
2800 				svp->sv_flags &= ~SV4_ROOT_STALE;
2801 				nfs_rw_exit(&svp->sv_lock);
2802 				return (ENOENT);
2803 			}
2804 			nfs_rw_exit(&svp->sv_lock);
2805 		} else
2806 			nfs_rw_exit(&svp->sv_lock);
2807 	}
2808 
2809 	mfname = mi->mi_fname;
2810 	fn_hold(mfname);
2811 	vp = makenfs4node_by_fh(mi->mi_rootfh, NULL, &mfname, NULL,
2812 	    VFTOMI4(vfsp), CRED(), gethrtime());
2813 
2814 	if (VTOR4(vp)->r_flags & R4STALE) {
2815 		VN_RELE(vp);
2816 		return (ENOENT);
2817 	}
2818 
2819 	ASSERT(vp->v_type == VNON || vp->v_type == mi->mi_type);
2820 
2821 	vp->v_type = mi->mi_type;
2822 
2823 	*vpp = vp;
2824 
2825 	return (0);
2826 }
2827 
2828 static int
2829 nfs4_statfs_otw(vnode_t *vp, struct statvfs64 *sbp, cred_t *cr)
2830 {
2831 	int error;
2832 	nfs4_ga_res_t gar;
2833 	nfs4_ga_ext_res_t ger;
2834 
2835 	gar.n4g_ext_res = &ger;
2836 
2837 	if (error = nfs4_attr_otw(vp, TAG_FSINFO, &gar,
2838 	    NFS4_STATFS_ATTR_MASK, cr))
2839 		return (error);
2840 
2841 	*sbp = gar.n4g_ext_res->n4g_sb;
2842 
2843 	return (0);
2844 }
2845 
2846 /*
2847  * Get file system statistics.
2848  */
2849 static int
2850 nfs4_statvfs(vfs_t *vfsp, struct statvfs64 *sbp)
2851 {
2852 	int error;
2853 	vnode_t *vp;
2854 	cred_t *cr;
2855 
2856 	error = nfs4_root(vfsp, &vp);
2857 	if (error)
2858 		return (error);
2859 
2860 	cr = CRED();
2861 
2862 	error = nfs4_statfs_otw(vp, sbp, cr);
2863 	if (!error) {
2864 		(void) strncpy(sbp->f_basetype,
2865 		    vfssw[vfsp->vfs_fstype].vsw_name, FSTYPSZ);
2866 		sbp->f_flag = vf_to_stf(vfsp->vfs_flag);
2867 	} else {
2868 		nfs4_purge_stale_fh(error, vp, cr);
2869 	}
2870 
2871 	VN_RELE(vp);
2872 
2873 	return (error);
2874 }
2875 
2876 static kmutex_t nfs4_syncbusy;
2877 
2878 /*
2879  * Flush dirty nfs files for file system vfsp.
2880  * If vfsp == NULL, all nfs files are flushed.
2881  *
2882  * SYNC_CLOSE in flag is passed to us to
2883  * indicate that we are shutting down and or
2884  * rebooting.
2885  */
2886 static int
2887 nfs4_sync(vfs_t *vfsp, short flag, cred_t *cr)
2888 {
2889 	/*
2890 	 * Cross-zone calls are OK here, since this translates to a
2891 	 * VOP_PUTPAGE(B_ASYNC), which gets picked up by the right zone.
2892 	 */
2893 	if (!(flag & SYNC_ATTR) && mutex_tryenter(&nfs4_syncbusy) != 0) {
2894 		r4flush(vfsp, cr);
2895 		mutex_exit(&nfs4_syncbusy);
2896 	}
2897 
2898 	/*
2899 	 * if SYNC_CLOSE is set then we know that
2900 	 * the system is rebooting, mark the mntinfo
2901 	 * for later examination.
2902 	 */
2903 	if (vfsp && (flag & SYNC_CLOSE)) {
2904 		mntinfo4_t *mi;
2905 
2906 		mi = VFTOMI4(vfsp);
2907 		if (!(mi->mi_flags & MI4_SHUTDOWN)) {
2908 			mutex_enter(&mi->mi_lock);
2909 			mi->mi_flags |= MI4_SHUTDOWN;
2910 			mutex_exit(&mi->mi_lock);
2911 		}
2912 	}
2913 	return (0);
2914 }
2915 
2916 /*
2917  * vget is difficult, if not impossible, to support in v4 because we don't
2918  * know the parent directory or name, which makes it impossible to create a
2919  * useful shadow vnode.  And we need the shadow vnode for things like
2920  * OPEN.
2921  */
2922 
2923 /* ARGSUSED */
2924 /*
2925  * XXX Check nfs4_vget_pseudo() for dependency.
2926  */
2927 static int
2928 nfs4_vget(vfs_t *vfsp, vnode_t **vpp, fid_t *fidp)
2929 {
2930 	return (EREMOTE);
2931 }
2932 
2933 /*
2934  * nfs4_mountroot get called in the case where we are diskless booting.  All
2935  * we need from here is the ability to get the server info and from there we
2936  * can simply call nfs4_rootvp.
2937  */
2938 /* ARGSUSED */
2939 static int
2940 nfs4_mountroot(vfs_t *vfsp, whymountroot_t why)
2941 {
2942 	vnode_t *rtvp;
2943 	char root_hostname[SYS_NMLN+1];
2944 	struct servinfo4 *svp;
2945 	int error;
2946 	int vfsflags;
2947 	size_t size;
2948 	char *root_path;
2949 	struct pathname pn;
2950 	char *name;
2951 	cred_t *cr;
2952 	mntinfo4_t *mi;
2953 	struct nfs_args args;		/* nfs mount arguments */
2954 	static char token[10];
2955 	nfs4_error_t n4e;
2956 
2957 	bzero(&args, sizeof (args));
2958 
2959 	/* do this BEFORE getfile which causes xid stamps to be initialized */
2960 	clkset(-1L);		/* hack for now - until we get time svc? */
2961 
2962 	if (why == ROOT_REMOUNT) {
2963 		/*
2964 		 * Shouldn't happen.
2965 		 */
2966 		panic("nfs4_mountroot: why == ROOT_REMOUNT");
2967 	}
2968 
2969 	if (why == ROOT_UNMOUNT) {
2970 		/*
2971 		 * Nothing to do for NFS.
2972 		 */
2973 		return (0);
2974 	}
2975 
2976 	/*
2977 	 * why == ROOT_INIT
2978 	 */
2979 
2980 	name = token;
2981 	*name = 0;
2982 	(void) getfsname("root", name, sizeof (token));
2983 
2984 	pn_alloc(&pn);
2985 	root_path = pn.pn_path;
2986 
2987 	svp = kmem_zalloc(sizeof (*svp), KM_SLEEP);
2988 	nfs_rw_init(&svp->sv_lock, NULL, RW_DEFAULT, NULL);
2989 	svp->sv_knconf = kmem_zalloc(sizeof (*svp->sv_knconf), KM_SLEEP);
2990 	svp->sv_knconf->knc_protofmly = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
2991 	svp->sv_knconf->knc_proto = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
2992 
2993 	/*
2994 	 * Get server address
2995 	 * Get the root path
2996 	 * Get server's transport
2997 	 * Get server's hostname
2998 	 * Get options
2999 	 */
3000 	args.addr = &svp->sv_addr;
3001 	(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
3002 	args.fh = (char *)&svp->sv_fhandle;
3003 	args.knconf = svp->sv_knconf;
3004 	args.hostname = root_hostname;
3005 	vfsflags = 0;
3006 	if (error = mount_root(*name ? name : "root", root_path, NFS_V4,
3007 	    &args, &vfsflags)) {
3008 		if (error == EPROTONOSUPPORT)
3009 			nfs_cmn_err(error, CE_WARN, "nfs4_mountroot: "
3010 			    "mount_root failed: server doesn't support NFS V4");
3011 		else
3012 			nfs_cmn_err(error, CE_WARN,
3013 			    "nfs4_mountroot: mount_root failed: %m");
3014 		nfs_rw_exit(&svp->sv_lock);
3015 		sv4_free(svp);
3016 		pn_free(&pn);
3017 		return (error);
3018 	}
3019 	nfs_rw_exit(&svp->sv_lock);
3020 	svp->sv_hostnamelen = (int)(strlen(root_hostname) + 1);
3021 	svp->sv_hostname = kmem_alloc(svp->sv_hostnamelen, KM_SLEEP);
3022 	(void) strcpy(svp->sv_hostname, root_hostname);
3023 
3024 	svp->sv_pathlen = (int)(strlen(root_path) + 1);
3025 	svp->sv_path = kmem_alloc(svp->sv_pathlen, KM_SLEEP);
3026 	(void) strcpy(svp->sv_path, root_path);
3027 
3028 	/*
3029 	 * Force root partition to always be mounted with AUTH_UNIX for now
3030 	 */
3031 	svp->sv_secdata = kmem_alloc(sizeof (*svp->sv_secdata), KM_SLEEP);
3032 	svp->sv_secdata->secmod = AUTH_UNIX;
3033 	svp->sv_secdata->rpcflavor = AUTH_UNIX;
3034 	svp->sv_secdata->data = NULL;
3035 
3036 	cr = crgetcred();
3037 	rtvp = NULL;
3038 
3039 	error = nfs4rootvp(&rtvp, vfsp, svp, args.flags, cr, global_zone);
3040 
3041 	if (error) {
3042 		crfree(cr);
3043 		pn_free(&pn);
3044 		sv4_free(svp);
3045 		return (error);
3046 	}
3047 
3048 	mi = VTOMI4(rtvp);
3049 
3050 	/*
3051 	 * Send client id to the server, if necessary
3052 	 */
3053 	nfs4_error_zinit(&n4e);
3054 	nfs4setclientid(mi, cr, FALSE, &n4e);
3055 	error = n4e.error;
3056 
3057 	crfree(cr);
3058 
3059 	if (error) {
3060 		pn_free(&pn);
3061 		goto errout;
3062 	}
3063 
3064 	error = nfs4_setopts(rtvp, DATAMODEL_NATIVE, &args);
3065 	if (error) {
3066 		nfs_cmn_err(error, CE_WARN,
3067 		    "nfs4_mountroot: invalid root mount options");
3068 		pn_free(&pn);
3069 		goto errout;
3070 	}
3071 
3072 	(void) vfs_lock_wait(vfsp);
3073 	vfs_add(NULL, vfsp, vfsflags);
3074 	vfs_unlock(vfsp);
3075 
3076 	size = strlen(svp->sv_hostname);
3077 	(void) strcpy(rootfs.bo_name, svp->sv_hostname);
3078 	rootfs.bo_name[size] = ':';
3079 	(void) strcpy(&rootfs.bo_name[size + 1], root_path);
3080 
3081 	pn_free(&pn);
3082 
3083 errout:
3084 	if (error) {
3085 		sv4_free(svp);
3086 		nfs4_async_stop(vfsp);
3087 		nfs4_async_manager_stop(vfsp);
3088 	}
3089 
3090 	if (rtvp != NULL)
3091 		VN_RELE(rtvp);
3092 
3093 	return (error);
3094 }
3095 
3096 /*
3097  * Initialization routine for VFS routines.  Should only be called once
3098  */
3099 int
3100 nfs4_vfsinit(void)
3101 {
3102 	mutex_init(&nfs4_syncbusy, NULL, MUTEX_DEFAULT, NULL);
3103 	nfs4setclientid_init();
3104 	nfs4_ephemeral_init();
3105 	return (0);
3106 }
3107 
3108 void
3109 nfs4_vfsfini(void)
3110 {
3111 	nfs4_ephemeral_fini();
3112 	nfs4setclientid_fini();
3113 	mutex_destroy(&nfs4_syncbusy);
3114 }
3115 
3116 void
3117 nfs4_freevfs(vfs_t *vfsp)
3118 {
3119 	mntinfo4_t *mi;
3120 
3121 	/* need to release the initial hold */
3122 	mi = VFTOMI4(vfsp);
3123 
3124 	/*
3125 	 * At this point, we can no longer reference the vfs
3126 	 * and need to inform other holders of the reference
3127 	 * to the mntinfo4_t.
3128 	 */
3129 	mi->mi_vfsp = NULL;
3130 
3131 	MI4_RELE(mi);
3132 }
3133 
3134 /*
3135  * Client side SETCLIENTID and SETCLIENTID_CONFIRM
3136  */
3137 struct nfs4_server nfs4_server_lst =
3138 	{ &nfs4_server_lst, &nfs4_server_lst };
3139 
3140 kmutex_t nfs4_server_lst_lock;
3141 
3142 static void
3143 nfs4setclientid_init(void)
3144 {
3145 	mutex_init(&nfs4_server_lst_lock, NULL, MUTEX_DEFAULT, NULL);
3146 }
3147 
3148 static void
3149 nfs4setclientid_fini(void)
3150 {
3151 	mutex_destroy(&nfs4_server_lst_lock);
3152 }
3153 
3154 int nfs4_retry_sclid_delay = NFS4_RETRY_SCLID_DELAY;
3155 int nfs4_num_sclid_retries = NFS4_NUM_SCLID_RETRIES;
3156 
3157 /*
3158  * Set the clientid for the server for "mi".  No-op if the clientid is
3159  * already set.
3160  *
3161  * The recovery boolean should be set to TRUE if this function was called
3162  * by the recovery code, and FALSE otherwise.  This is used to determine
3163  * if we need to call nfs4_start/end_op as well as grab the mi_recovlock
3164  * for adding a mntinfo4_t to a nfs4_server_t.
3165  *
3166  * Error is returned via 'n4ep'.  If there was a 'n4ep->stat' error, then
3167  * 'n4ep->error' is set to geterrno4(n4ep->stat).
3168  */
3169 void
3170 nfs4setclientid(mntinfo4_t *mi, cred_t *cr, bool_t recovery, nfs4_error_t *n4ep)
3171 {
3172 	struct nfs4_server *np;
3173 	struct servinfo4 *svp = mi->mi_curr_serv;
3174 	nfs4_recov_state_t recov_state;
3175 	int num_retries = 0;
3176 	bool_t retry;
3177 	cred_t *lcr = NULL;
3178 	int retry_inuse = 1; /* only retry once on NFS4ERR_CLID_INUSE */
3179 	time_t lease_time = 0;
3180 
3181 	recov_state.rs_flags = 0;
3182 	recov_state.rs_num_retry_despite_err = 0;
3183 	ASSERT(n4ep != NULL);
3184 
3185 recov_retry:
3186 	retry = FALSE;
3187 	nfs4_error_zinit(n4ep);
3188 	if (!recovery)
3189 		(void) nfs_rw_enter_sig(&mi->mi_recovlock, RW_READER, 0);
3190 
3191 	mutex_enter(&nfs4_server_lst_lock);
3192 	np = servinfo4_to_nfs4_server(svp); /* This locks np if it is found */
3193 	mutex_exit(&nfs4_server_lst_lock);
3194 	if (!np) {
3195 		struct nfs4_server *tnp;
3196 		np = new_nfs4_server(svp, cr);
3197 		mutex_enter(&np->s_lock);
3198 
3199 		mutex_enter(&nfs4_server_lst_lock);
3200 		tnp = servinfo4_to_nfs4_server(svp);
3201 		if (tnp) {
3202 			/*
3203 			 * another thread snuck in and put server on list.
3204 			 * since we aren't adding it to the nfs4_server_list
3205 			 * we need to set the ref count to 0 and destroy it.
3206 			 */
3207 			np->s_refcnt = 0;
3208 			destroy_nfs4_server(np);
3209 			np = tnp;
3210 		} else {
3211 			/*
3212 			 * do not give list a reference until everything
3213 			 * succeeds
3214 			 */
3215 			insque(np, &nfs4_server_lst);
3216 		}
3217 		mutex_exit(&nfs4_server_lst_lock);
3218 	}
3219 	ASSERT(MUTEX_HELD(&np->s_lock));
3220 	/*
3221 	 * If we find the server already has N4S_CLIENTID_SET, then
3222 	 * just return, we've already done SETCLIENTID to that server
3223 	 */
3224 	if (np->s_flags & N4S_CLIENTID_SET) {
3225 		/* add mi to np's mntinfo4_list */
3226 		nfs4_add_mi_to_server(np, mi);
3227 		if (!recovery)
3228 			nfs_rw_exit(&mi->mi_recovlock);
3229 		mutex_exit(&np->s_lock);
3230 		nfs4_server_rele(np);
3231 		return;
3232 	}
3233 	mutex_exit(&np->s_lock);
3234 
3235 
3236 	/*
3237 	 * Drop the mi_recovlock since nfs4_start_op will
3238 	 * acquire it again for us.
3239 	 */
3240 	if (!recovery) {
3241 		nfs_rw_exit(&mi->mi_recovlock);
3242 
3243 		n4ep->error = nfs4_start_op(mi, NULL, NULL, &recov_state);
3244 		if (n4ep->error) {
3245 			nfs4_server_rele(np);
3246 			return;
3247 		}
3248 	}
3249 
3250 	mutex_enter(&np->s_lock);
3251 	while (np->s_flags & N4S_CLIENTID_PEND) {
3252 		if (!cv_wait_sig(&np->s_clientid_pend, &np->s_lock)) {
3253 			mutex_exit(&np->s_lock);
3254 			nfs4_server_rele(np);
3255 			if (!recovery)
3256 				nfs4_end_op(mi, NULL, NULL, &recov_state,
3257 				    recovery);
3258 			n4ep->error = EINTR;
3259 			return;
3260 		}
3261 	}
3262 
3263 	if (np->s_flags & N4S_CLIENTID_SET) {
3264 		/* XXX copied/pasted from above */
3265 		/* add mi to np's mntinfo4_list */
3266 		nfs4_add_mi_to_server(np, mi);
3267 		mutex_exit(&np->s_lock);
3268 		nfs4_server_rele(np);
3269 		if (!recovery)
3270 			nfs4_end_op(mi, NULL, NULL, &recov_state, recovery);
3271 		return;
3272 	}
3273 
3274 	/*
3275 	 * Reset the N4S_CB_PINGED flag. This is used to
3276 	 * indicate if we have received a CB_NULL from the
3277 	 * server. Also we reset the waiter flag.
3278 	 */
3279 	np->s_flags &= ~(N4S_CB_PINGED | N4S_CB_WAITER);
3280 	/* any failure must now clear this flag */
3281 	np->s_flags |= N4S_CLIENTID_PEND;
3282 	mutex_exit(&np->s_lock);
3283 	nfs4setclientid_otw(mi, svp, cr, np, n4ep, &retry_inuse);
3284 
3285 	if (n4ep->error == EACCES) {
3286 		/*
3287 		 * If the uid is set then set the creds for secure mounts
3288 		 * by proxy processes such as automountd.
3289 		 */
3290 		(void) nfs_rw_enter_sig(&svp->sv_lock, RW_READER, 0);
3291 		if (svp->sv_secdata->uid != 0) {
3292 			lcr = crdup(cr);
3293 			(void) crsetugid(lcr, svp->sv_secdata->uid,
3294 			    crgetgid(cr));
3295 		}
3296 		nfs_rw_exit(&svp->sv_lock);
3297 
3298 		if (lcr != NULL) {
3299 			mutex_enter(&np->s_lock);
3300 			crfree(np->s_cred);
3301 			np->s_cred = lcr;
3302 			mutex_exit(&np->s_lock);
3303 			nfs4setclientid_otw(mi, svp, lcr, np, n4ep,
3304 			    &retry_inuse);
3305 		}
3306 	}
3307 	mutex_enter(&np->s_lock);
3308 	lease_time = np->s_lease_time;
3309 	np->s_flags &= ~N4S_CLIENTID_PEND;
3310 	mutex_exit(&np->s_lock);
3311 
3312 	if (n4ep->error != 0 || n4ep->stat != NFS4_OK) {
3313 		/*
3314 		 * Start recovery if failover is a possibility.  If
3315 		 * invoked by the recovery thread itself, then just
3316 		 * return and let it handle the failover first.  NB:
3317 		 * recovery is not allowed if the mount is in progress
3318 		 * since the infrastructure is not sufficiently setup
3319 		 * to allow it.  Just return the error (after suitable
3320 		 * retries).
3321 		 */
3322 		if (FAILOVER_MOUNT4(mi) && nfs4_try_failover(n4ep)) {
3323 			(void) nfs4_start_recovery(n4ep, mi, NULL,
3324 			    NULL, NULL, NULL, OP_SETCLIENTID, NULL, NULL, NULL);
3325 			/*
3326 			 * Don't retry here, just return and let
3327 			 * recovery take over.
3328 			 */
3329 			if (recovery)
3330 				retry = FALSE;
3331 		} else if (nfs4_rpc_retry_error(n4ep->error) ||
3332 		    n4ep->stat == NFS4ERR_RESOURCE ||
3333 		    n4ep->stat == NFS4ERR_STALE_CLIENTID) {
3334 
3335 			retry = TRUE;
3336 			/*
3337 			 * Always retry if in recovery or once had
3338 			 * contact with the server (but now it's
3339 			 * overloaded).
3340 			 */
3341 			if (recovery == TRUE ||
3342 			    n4ep->error == ETIMEDOUT ||
3343 			    n4ep->error == ECONNRESET)
3344 				num_retries = 0;
3345 		} else if (retry_inuse && n4ep->error == 0 &&
3346 		    n4ep->stat == NFS4ERR_CLID_INUSE) {
3347 			retry = TRUE;
3348 			num_retries = 0;
3349 		}
3350 	} else {
3351 		/*
3352 		 * Since everything succeeded give the list a reference count if
3353 		 * it hasn't been given one by add_new_nfs4_server() or if this
3354 		 * is not a recovery situation in which case it is already on
3355 		 * the list.
3356 		 */
3357 		mutex_enter(&np->s_lock);
3358 		if ((np->s_flags & N4S_INSERTED) == 0) {
3359 			np->s_refcnt++;
3360 			np->s_flags |= N4S_INSERTED;
3361 		}
3362 		mutex_exit(&np->s_lock);
3363 	}
3364 
3365 	if (!recovery)
3366 		nfs4_end_op(mi, NULL, NULL, &recov_state, recovery);
3367 
3368 
3369 	if (retry && num_retries++ < nfs4_num_sclid_retries) {
3370 		if (retry_inuse) {
3371 			delay(SEC_TO_TICK(lease_time + nfs4_retry_sclid_delay));
3372 			retry_inuse = 0;
3373 		} else
3374 			delay(SEC_TO_TICK(nfs4_retry_sclid_delay));
3375 
3376 		nfs4_server_rele(np);
3377 		goto recov_retry;
3378 	}
3379 
3380 
3381 	if (n4ep->error == 0)
3382 		n4ep->error = geterrno4(n4ep->stat);
3383 
3384 	/* broadcast before release in case no other threads are waiting */
3385 	cv_broadcast(&np->s_clientid_pend);
3386 	nfs4_server_rele(np);
3387 }
3388 
3389 int nfs4setclientid_otw_debug = 0;
3390 
3391 /*
3392  * This function handles the recovery of STALE_CLIENTID for SETCLIENTID_CONFRIM,
3393  * but nothing else; the calling function must be designed to handle those
3394  * other errors.
3395  */
3396 static void
3397 nfs4setclientid_otw(mntinfo4_t *mi, struct servinfo4 *svp,  cred_t *cr,
3398     struct nfs4_server *np, nfs4_error_t *ep, int *retry_inusep)
3399 {
3400 	COMPOUND4args_clnt args;
3401 	COMPOUND4res_clnt res;
3402 	nfs_argop4 argop[3];
3403 	SETCLIENTID4args *s_args;
3404 	SETCLIENTID4resok *s_resok;
3405 	int doqueue = 1;
3406 	nfs4_ga_res_t *garp = NULL;
3407 	timespec_t prop_time, after_time;
3408 	verifier4 verf;
3409 	clientid4 tmp_clientid;
3410 
3411 	ASSERT(!MUTEX_HELD(&np->s_lock));
3412 
3413 	args.ctag = TAG_SETCLIENTID;
3414 
3415 	args.array = argop;
3416 	args.array_len = 3;
3417 
3418 	/* PUTROOTFH */
3419 	argop[0].argop = OP_PUTROOTFH;
3420 
3421 	/* GETATTR */
3422 	argop[1].argop = OP_GETATTR;
3423 	argop[1].nfs_argop4_u.opgetattr.attr_request = FATTR4_LEASE_TIME_MASK;
3424 	argop[1].nfs_argop4_u.opgetattr.mi = mi;
3425 
3426 	/* SETCLIENTID */
3427 	argop[2].argop = OP_SETCLIENTID;
3428 
3429 	s_args = &argop[2].nfs_argop4_u.opsetclientid;
3430 
3431 	mutex_enter(&np->s_lock);
3432 
3433 	s_args->client.verifier = np->clidtosend.verifier;
3434 	s_args->client.id_len = np->clidtosend.id_len;
3435 	ASSERT(s_args->client.id_len <= NFS4_OPAQUE_LIMIT);
3436 	s_args->client.id_val = np->clidtosend.id_val;
3437 
3438 	/*
3439 	 * Callback needs to happen on non-RDMA transport
3440 	 * Check if we have saved the original knetconfig
3441 	 * if so, use that instead.
3442 	 */
3443 	if (svp->sv_origknconf != NULL)
3444 		nfs4_cb_args(np, svp->sv_origknconf, s_args);
3445 	else
3446 		nfs4_cb_args(np, svp->sv_knconf, s_args);
3447 
3448 	mutex_exit(&np->s_lock);
3449 
3450 	rfs4call(mi, &args, &res, cr, &doqueue, 0, ep);
3451 
3452 	if (ep->error)
3453 		return;
3454 
3455 	/* getattr lease_time res */
3456 	if ((res.array_len >= 2) &&
3457 	    (res.array[1].nfs_resop4_u.opgetattr.status == NFS4_OK)) {
3458 		garp = &res.array[1].nfs_resop4_u.opgetattr.ga_res;
3459 
3460 #ifndef _LP64
3461 		/*
3462 		 * The 32 bit client cannot handle a lease time greater than
3463 		 * (INT32_MAX/1000000).  This is due to the use of the
3464 		 * lease_time in calls to drv_usectohz() in
3465 		 * nfs4_renew_lease_thread().  The problem is that
3466 		 * drv_usectohz() takes a time_t (which is just a long = 4
3467 		 * bytes) as its parameter.  The lease_time is multiplied by
3468 		 * 1000000 to convert seconds to usecs for the parameter.  If
3469 		 * a number bigger than (INT32_MAX/1000000) is used then we
3470 		 * overflow on the 32bit client.
3471 		 */
3472 		if (garp->n4g_ext_res->n4g_leasetime > (INT32_MAX/1000000)) {
3473 			garp->n4g_ext_res->n4g_leasetime = INT32_MAX/1000000;
3474 		}
3475 #endif
3476 
3477 		mutex_enter(&np->s_lock);
3478 		np->s_lease_time = garp->n4g_ext_res->n4g_leasetime;
3479 
3480 		/*
3481 		 * Keep track of the lease period for the mi's
3482 		 * mi_msg_list.  We need an appropiate time
3483 		 * bound to associate past facts with a current
3484 		 * event.  The lease period is perfect for this.
3485 		 */
3486 		mutex_enter(&mi->mi_msg_list_lock);
3487 		mi->mi_lease_period = np->s_lease_time;
3488 		mutex_exit(&mi->mi_msg_list_lock);
3489 		mutex_exit(&np->s_lock);
3490 	}
3491 
3492 
3493 	if (res.status == NFS4ERR_CLID_INUSE) {
3494 		clientaddr4 *clid_inuse;
3495 
3496 		if (!(*retry_inusep)) {
3497 			clid_inuse = &res.array->nfs_resop4_u.
3498 			    opsetclientid.SETCLIENTID4res_u.client_using;
3499 
3500 			zcmn_err(mi->mi_zone->zone_id, CE_NOTE,
3501 			    "NFS4 mount (SETCLIENTID failed)."
3502 			    "  nfs4_client_id.id is in"
3503 			    "use already by: r_netid<%s> r_addr<%s>",
3504 			    clid_inuse->r_netid, clid_inuse->r_addr);
3505 		}
3506 
3507 		/*
3508 		 * XXX - The client should be more robust in its
3509 		 * handling of clientid in use errors (regen another
3510 		 * clientid and try again?)
3511 		 */
3512 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
3513 		return;
3514 	}
3515 
3516 	if (res.status) {
3517 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
3518 		return;
3519 	}
3520 
3521 	s_resok = &res.array[2].nfs_resop4_u.
3522 	    opsetclientid.SETCLIENTID4res_u.resok4;
3523 
3524 	tmp_clientid = s_resok->clientid;
3525 
3526 	verf = s_resok->setclientid_confirm;
3527 
3528 #ifdef	DEBUG
3529 	if (nfs4setclientid_otw_debug) {
3530 		union {
3531 			clientid4	clientid;
3532 			int		foo[2];
3533 		} cid;
3534 
3535 		cid.clientid = s_resok->clientid;
3536 
3537 		zcmn_err(mi->mi_zone->zone_id, CE_NOTE,
3538 		"nfs4setclientid_otw: OK, clientid = %x,%x, "
3539 		"verifier = %" PRIx64 "\n", cid.foo[0], cid.foo[1], verf);
3540 	}
3541 #endif
3542 
3543 	(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
3544 
3545 	/* Confirm the client id and get the lease_time attribute */
3546 
3547 	args.ctag = TAG_SETCLIENTID_CF;
3548 
3549 	args.array = argop;
3550 	args.array_len = 1;
3551 
3552 	argop[0].argop = OP_SETCLIENTID_CONFIRM;
3553 
3554 	argop[0].nfs_argop4_u.opsetclientid_confirm.clientid = tmp_clientid;
3555 	argop[0].nfs_argop4_u.opsetclientid_confirm.setclientid_confirm = verf;
3556 
3557 	/* used to figure out RTT for np */
3558 	gethrestime(&prop_time);
3559 
3560 	NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE, "nfs4setlientid_otw: "
3561 	    "start time: %ld sec %ld nsec", prop_time.tv_sec,
3562 	    prop_time.tv_nsec));
3563 
3564 	rfs4call(mi, &args, &res, cr, &doqueue, 0, ep);
3565 
3566 	gethrestime(&after_time);
3567 	mutex_enter(&np->s_lock);
3568 	np->propagation_delay.tv_sec =
3569 	    MAX(1, after_time.tv_sec - prop_time.tv_sec);
3570 	mutex_exit(&np->s_lock);
3571 
3572 	NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE, "nfs4setlcientid_otw: "
3573 	    "finish time: %ld sec ", after_time.tv_sec));
3574 
3575 	NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE, "nfs4setclientid_otw: "
3576 	    "propagation delay set to %ld sec",
3577 	    np->propagation_delay.tv_sec));
3578 
3579 	if (ep->error)
3580 		return;
3581 
3582 	if (res.status == NFS4ERR_CLID_INUSE) {
3583 		clientaddr4 *clid_inuse;
3584 
3585 		if (!(*retry_inusep)) {
3586 			clid_inuse = &res.array->nfs_resop4_u.
3587 			    opsetclientid.SETCLIENTID4res_u.client_using;
3588 
3589 			zcmn_err(mi->mi_zone->zone_id, CE_NOTE,
3590 			    "SETCLIENTID_CONFIRM failed.  "
3591 			    "nfs4_client_id.id is in use already by: "
3592 			    "r_netid<%s> r_addr<%s>",
3593 			    clid_inuse->r_netid, clid_inuse->r_addr);
3594 		}
3595 
3596 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
3597 		return;
3598 	}
3599 
3600 	if (res.status) {
3601 		(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
3602 		return;
3603 	}
3604 
3605 	mutex_enter(&np->s_lock);
3606 	np->clientid = tmp_clientid;
3607 	np->s_flags |= N4S_CLIENTID_SET;
3608 
3609 	/* Add mi to np's mntinfo4 list */
3610 	nfs4_add_mi_to_server(np, mi);
3611 
3612 	if (np->lease_valid == NFS4_LEASE_NOT_STARTED) {
3613 		/*
3614 		 * Start lease management thread.
3615 		 * Keep trying until we succeed.
3616 		 */
3617 
3618 		np->s_refcnt++;		/* pass reference to thread */
3619 		(void) zthread_create(NULL, 0, nfs4_renew_lease_thread, np, 0,
3620 		    minclsyspri);
3621 	}
3622 	mutex_exit(&np->s_lock);
3623 
3624 	(void) xdr_free(xdr_COMPOUND4res_clnt, (caddr_t)&res);
3625 }
3626 
3627 /*
3628  * Add mi to sp's mntinfo4_list if it isn't already in the list.  Makes
3629  * mi's clientid the same as sp's.
3630  * Assumes sp is locked down.
3631  */
3632 void
3633 nfs4_add_mi_to_server(nfs4_server_t *sp, mntinfo4_t *mi)
3634 {
3635 	mntinfo4_t *tmi;
3636 	int in_list = 0;
3637 
3638 	ASSERT(nfs_rw_lock_held(&mi->mi_recovlock, RW_READER) ||
3639 	    nfs_rw_lock_held(&mi->mi_recovlock, RW_WRITER));
3640 	ASSERT(sp != &nfs4_server_lst);
3641 	ASSERT(MUTEX_HELD(&sp->s_lock));
3642 
3643 	NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE,
3644 	    "nfs4_add_mi_to_server: add mi %p to sp %p",
3645 	    (void*)mi, (void*)sp));
3646 
3647 	for (tmi = sp->mntinfo4_list;
3648 	    tmi != NULL;
3649 	    tmi = tmi->mi_clientid_next) {
3650 		if (tmi == mi) {
3651 			NFS4_DEBUG(nfs4_client_lease_debug,
3652 			    (CE_NOTE,
3653 			    "nfs4_add_mi_to_server: mi in list"));
3654 			in_list = 1;
3655 		}
3656 	}
3657 
3658 	/*
3659 	 * First put a hold on the mntinfo4's vfsp so that references via
3660 	 * mntinfo4_list will be valid.
3661 	 */
3662 	if (!in_list)
3663 		VFS_HOLD(mi->mi_vfsp);
3664 
3665 	NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE, "nfs4_add_mi_to_server: "
3666 	    "hold vfs %p for mi: %p", (void*)mi->mi_vfsp, (void*)mi));
3667 
3668 	if (!in_list) {
3669 		if (sp->mntinfo4_list)
3670 			sp->mntinfo4_list->mi_clientid_prev = mi;
3671 		mi->mi_clientid_next = sp->mntinfo4_list;
3672 		mi->mi_srv = sp;
3673 		sp->mntinfo4_list = mi;
3674 		mi->mi_srvsettime = gethrestime_sec();
3675 		mi->mi_srvset_cnt++;
3676 	}
3677 
3678 	/* set mi's clientid to that of sp's for later matching */
3679 	mi->mi_clientid = sp->clientid;
3680 
3681 	/*
3682 	 * Update the clientid for any other mi's belonging to sp.  This
3683 	 * must be done here while we hold sp->s_lock, so that
3684 	 * find_nfs4_server() continues to work.
3685 	 */
3686 
3687 	for (tmi = sp->mntinfo4_list;
3688 	    tmi != NULL;
3689 	    tmi = tmi->mi_clientid_next) {
3690 		if (tmi != mi) {
3691 			tmi->mi_clientid = sp->clientid;
3692 		}
3693 	}
3694 }
3695 
3696 /*
3697  * Remove the mi from sp's mntinfo4_list and release its reference.
3698  * Exception: if mi still has open files, flag it for later removal (when
3699  * all the files are closed).
3700  *
3701  * If this is the last mntinfo4 in sp's list then tell the lease renewal
3702  * thread to exit.
3703  */
3704 static void
3705 nfs4_remove_mi_from_server_nolock(mntinfo4_t *mi, nfs4_server_t *sp)
3706 {
3707 	NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE,
3708 	    "nfs4_remove_mi_from_server_nolock: remove mi %p from sp %p",
3709 	    (void*)mi, (void*)sp));
3710 
3711 	ASSERT(sp != NULL);
3712 	ASSERT(MUTEX_HELD(&sp->s_lock));
3713 	ASSERT(mi->mi_open_files >= 0);
3714 
3715 	/*
3716 	 * First make sure this mntinfo4 can be taken off of the list,
3717 	 * ie: it doesn't have any open files remaining.
3718 	 */
3719 	if (mi->mi_open_files > 0) {
3720 		NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE,
3721 		    "nfs4_remove_mi_from_server_nolock: don't "
3722 		    "remove mi since it still has files open"));
3723 
3724 		mutex_enter(&mi->mi_lock);
3725 		mi->mi_flags |= MI4_REMOVE_ON_LAST_CLOSE;
3726 		mutex_exit(&mi->mi_lock);
3727 		return;
3728 	}
3729 
3730 	VFS_HOLD(mi->mi_vfsp);
3731 	remove_mi(sp, mi);
3732 	VFS_RELE(mi->mi_vfsp);
3733 
3734 	if (sp->mntinfo4_list == NULL) {
3735 		/* last fs unmounted, kill the thread */
3736 		NFS4_DEBUG(nfs4_client_lease_debug, (CE_NOTE,
3737 		    "remove_mi_from_nfs4_server_nolock: kill the thread"));
3738 		nfs4_mark_srv_dead(sp);
3739 	}
3740 }
3741 
3742 /*
3743  * Remove mi from sp's mntinfo4_list and release the vfs reference.
3744  */
3745 static void
3746 remove_mi(nfs4_server_t *sp, mntinfo4_t *mi)
3747 {
3748 	ASSERT(MUTEX_HELD(&sp->s_lock));
3749 
3750 	/*
3751 	 * We release a reference, and the caller must still have a
3752 	 * reference.
3753 	 */
3754 	ASSERT(mi->mi_vfsp->vfs_count >= 2);
3755 
3756 	if (mi->mi_clientid_prev) {
3757 		mi->mi_clientid_prev->mi_clientid_next = mi->mi_clientid_next;
3758 	} else {
3759 		/* This is the first mi in sp's mntinfo4_list */
3760 		/*
3761 		 * Make sure the first mntinfo4 in the list is the actual
3762 		 * mntinfo4 passed in.
3763 		 */
3764 		ASSERT(sp->mntinfo4_list == mi);
3765 
3766 		sp->mntinfo4_list = mi->mi_clientid_next;
3767 	}
3768 	if (mi->mi_clientid_next)
3769 		mi->mi_clientid_next->mi_clientid_prev = mi->mi_clientid_prev;
3770 
3771 	/* Now mark the mntinfo4's links as being removed */
3772 	mi->mi_clientid_prev = mi->mi_clientid_next = NULL;
3773 	mi->mi_srv = NULL;
3774 	mi->mi_srvset_cnt++;
3775 
3776 	VFS_RELE(mi->mi_vfsp);
3777 }
3778 
3779 /*
3780  * Free all the entries in sp's mntinfo4_list.
3781  */
3782 static void
3783 remove_all_mi(nfs4_server_t *sp)
3784 {
3785 	mntinfo4_t *mi;
3786 
3787 	ASSERT(MUTEX_HELD(&sp->s_lock));
3788 
3789 	while (sp->mntinfo4_list != NULL) {
3790 		mi = sp->mntinfo4_list;
3791 		/*
3792 		 * Grab a reference in case there is only one left (which
3793 		 * remove_mi() frees).
3794 		 */
3795 		VFS_HOLD(mi->mi_vfsp);
3796 		remove_mi(sp, mi);
3797 		VFS_RELE(mi->mi_vfsp);
3798 	}
3799 }
3800 
3801 /*
3802  * Remove the mi from sp's mntinfo4_list as above, and rele the vfs.
3803  *
3804  * This version can be called with a null nfs4_server_t arg,
3805  * and will either find the right one and handle locking, or
3806  * do nothing because the mi wasn't added to an sp's mntinfo4_list.
3807  */
3808 void
3809 nfs4_remove_mi_from_server(mntinfo4_t *mi, nfs4_server_t *esp)
3810 {
3811 	nfs4_server_t	*sp;
3812 
3813 	if (esp) {
3814 		nfs4_remove_mi_from_server_nolock(mi, esp);
3815 		return;
3816 	}
3817 
3818 	(void) nfs_rw_enter_sig(&mi->mi_recovlock, RW_READER, 0);
3819 	if (sp = find_nfs4_server_all(mi, 1)) {
3820 		nfs4_remove_mi_from_server_nolock(mi, sp);
3821 		mutex_exit(&sp->s_lock);
3822 		nfs4_server_rele(sp);
3823 	}
3824 	nfs_rw_exit(&mi->mi_recovlock);
3825 }
3826 
3827 /*
3828  * Return TRUE if the given server has any non-unmounted filesystems.
3829  */
3830 
3831 bool_t
3832 nfs4_fs_active(nfs4_server_t *sp)
3833 {
3834 	mntinfo4_t *mi;
3835 
3836 	ASSERT(MUTEX_HELD(&sp->s_lock));
3837 
3838 	for (mi = sp->mntinfo4_list; mi != NULL; mi = mi->mi_clientid_next) {
3839 		if (!(mi->mi_vfsp->vfs_flag & VFS_UNMOUNTED))
3840 			return (TRUE);
3841 	}
3842 
3843 	return (FALSE);
3844 }
3845 
3846 /*
3847  * Mark sp as finished and notify any waiters.
3848  */
3849 
3850 void
3851 nfs4_mark_srv_dead(nfs4_server_t *sp)
3852 {
3853 	ASSERT(MUTEX_HELD(&sp->s_lock));
3854 
3855 	sp->s_thread_exit = NFS4_THREAD_EXIT;
3856 	cv_broadcast(&sp->cv_thread_exit);
3857 }
3858 
3859 /*
3860  * Create a new nfs4_server_t structure.
3861  * Returns new node unlocked and not in list, but with a reference count of
3862  * 1.
3863  */
3864 struct nfs4_server *
3865 new_nfs4_server(struct servinfo4 *svp, cred_t *cr)
3866 {
3867 	struct nfs4_server *np;
3868 	timespec_t tt;
3869 	union {
3870 		struct {
3871 			uint32_t sec;
3872 			uint32_t subsec;
3873 		} un_curtime;
3874 		verifier4	un_verifier;
3875 	} nfs4clientid_verifier;
3876 	/*
3877 	 * We change this ID string carefully and with the Solaris
3878 	 * NFS server behaviour in mind.  "+referrals" indicates
3879 	 * a client that can handle an NFSv4 referral.
3880 	 */
3881 	char id_val[] = "Solaris: %s, NFSv4 kernel client +referrals";
3882 	int len;
3883 
3884 	np = kmem_zalloc(sizeof (struct nfs4_server), KM_SLEEP);
3885 	np->saddr.len = svp->sv_addr.len;
3886 	np->saddr.maxlen = svp->sv_addr.maxlen;
3887 	np->saddr.buf = kmem_alloc(svp->sv_addr.maxlen, KM_SLEEP);
3888 	bcopy(svp->sv_addr.buf, np->saddr.buf, svp->sv_addr.len);
3889 	np->s_refcnt = 1;
3890 
3891 	/*
3892 	 * Build the nfs_client_id4 for this server mount.  Ensure
3893 	 * the verifier is useful and that the identification is
3894 	 * somehow based on the server's address for the case of
3895 	 * multi-homed servers.
3896 	 */
3897 	nfs4clientid_verifier.un_verifier = 0;
3898 	gethrestime(&tt);
3899 	nfs4clientid_verifier.un_curtime.sec = (uint32_t)tt.tv_sec;
3900 	nfs4clientid_verifier.un_curtime.subsec = (uint32_t)tt.tv_nsec;
3901 	np->clidtosend.verifier = nfs4clientid_verifier.un_verifier;
3902 
3903 	/*
3904 	 * calculate the length of the opaque identifier.  Subtract 2
3905 	 * for the "%s" and add the traditional +1 for null
3906 	 * termination.
3907 	 */
3908 	len = strlen(id_val) - 2 + strlen(uts_nodename()) + 1;
3909 	np->clidtosend.id_len = len + np->saddr.maxlen;
3910 
3911 	np->clidtosend.id_val = kmem_alloc(np->clidtosend.id_len, KM_SLEEP);
3912 	(void) sprintf(np->clidtosend.id_val, id_val, uts_nodename());
3913 	bcopy(np->saddr.buf, &np->clidtosend.id_val[len], np->saddr.len);
3914 
3915 	np->s_flags = 0;
3916 	np->mntinfo4_list = NULL;
3917 	/* save cred for issuing rfs4calls inside the renew thread */
3918 	crhold(cr);
3919 	np->s_cred = cr;
3920 	cv_init(&np->cv_thread_exit, NULL, CV_DEFAULT, NULL);
3921 	mutex_init(&np->s_lock, NULL, MUTEX_DEFAULT, NULL);
3922 	nfs_rw_init(&np->s_recovlock, NULL, RW_DEFAULT, NULL);
3923 	list_create(&np->s_deleg_list, sizeof (rnode4_t),
3924 	    offsetof(rnode4_t, r_deleg_link));
3925 	np->s_thread_exit = 0;
3926 	np->state_ref_count = 0;
3927 	np->lease_valid = NFS4_LEASE_NOT_STARTED;
3928 	cv_init(&np->s_cv_otw_count, NULL, CV_DEFAULT, NULL);
3929 	cv_init(&np->s_clientid_pend, NULL, CV_DEFAULT, NULL);
3930 	np->s_otw_call_count = 0;
3931 	cv_init(&np->wait_cb_null, NULL, CV_DEFAULT, NULL);
3932 	np->zoneid = getzoneid();
3933 	np->zone_globals = nfs4_get_callback_globals();
3934 	ASSERT(np->zone_globals != NULL);
3935 	return (np);
3936 }
3937 
3938 /*
3939  * Create a new nfs4_server_t structure and add it to the list.
3940  * Returns new node locked; reference must eventually be freed.
3941  */
3942 static struct nfs4_server *
3943 add_new_nfs4_server(struct servinfo4 *svp, cred_t *cr)
3944 {
3945 	nfs4_server_t *sp;
3946 
3947 	ASSERT(MUTEX_HELD(&nfs4_server_lst_lock));
3948 	sp = new_nfs4_server(svp, cr);
3949 	mutex_enter(&sp->s_lock);
3950 	insque(sp, &nfs4_server_lst);
3951 	sp->s_refcnt++;			/* list gets a reference */
3952 	sp->s_flags |= N4S_INSERTED;
3953 	sp->clientid = 0;
3954 	return (sp);
3955 }
3956 
3957 int nfs4_server_t_debug = 0;
3958 
3959 #ifdef lint
3960 extern void
3961 dumpnfs4slist(char *, mntinfo4_t *, clientid4, servinfo4_t *);
3962 #endif
3963 
3964 #ifndef lint
3965 #ifdef DEBUG
3966 void
3967 dumpnfs4slist(char *txt, mntinfo4_t *mi, clientid4 clientid, servinfo4_t *srv_p)
3968 {
3969 	int hash16(void *p, int len);
3970 	nfs4_server_t *np;
3971 
3972 	NFS4_DEBUG(nfs4_server_t_debug, (CE_NOTE,
3973 	    "dumping nfs4_server_t list in %s", txt));
3974 	NFS4_DEBUG(nfs4_server_t_debug, (CE_CONT,
3975 	    "mi 0x%p, want clientid %llx, addr %d/%04X",
3976 	    mi, (longlong_t)clientid, srv_p->sv_addr.len,
3977 	    hash16((void *)srv_p->sv_addr.buf, srv_p->sv_addr.len)));
3978 	for (np = nfs4_server_lst.forw; np != &nfs4_server_lst;
3979 	    np = np->forw) {
3980 		NFS4_DEBUG(nfs4_server_t_debug, (CE_CONT,
3981 		    "node 0x%p,    clientid %llx, addr %d/%04X, cnt %d",
3982 		    np, (longlong_t)np->clientid, np->saddr.len,
3983 		    hash16((void *)np->saddr.buf, np->saddr.len),
3984 		    np->state_ref_count));
3985 		if (np->saddr.len == srv_p->sv_addr.len &&
3986 		    bcmp(np->saddr.buf, srv_p->sv_addr.buf,
3987 		    np->saddr.len) == 0)
3988 			NFS4_DEBUG(nfs4_server_t_debug, (CE_CONT,
3989 			    " - address matches"));
3990 		if (np->clientid == clientid || np->clientid == 0)
3991 			NFS4_DEBUG(nfs4_server_t_debug, (CE_CONT,
3992 			    " - clientid matches"));
3993 		if (np->s_thread_exit != NFS4_THREAD_EXIT)
3994 			NFS4_DEBUG(nfs4_server_t_debug, (CE_CONT,
3995 			    " - thread not exiting"));
3996 	}
3997 	delay(hz);
3998 }
3999 #endif
4000 #endif
4001 
4002 
4003 /*
4004  * Move a mntinfo4_t from one server list to another.
4005  * Locking of the two nfs4_server_t nodes will be done in list order.
4006  *
4007  * Returns NULL if the current nfs4_server_t for the filesystem could not
4008  * be found (e.g., due to forced unmount).  Otherwise returns a reference
4009  * to the new nfs4_server_t, which must eventually be freed.
4010  */
4011 nfs4_server_t *
4012 nfs4_move_mi(mntinfo4_t *mi, servinfo4_t *old, servinfo4_t *new)
4013 {
4014 	nfs4_server_t *p, *op = NULL, *np = NULL;
4015 	int num_open;
4016 	zoneid_t zoneid = nfs_zoneid();
4017 
4018 	ASSERT(nfs_zone() == mi->mi_zone);
4019 
4020 	mutex_enter(&nfs4_server_lst_lock);
4021 #ifdef DEBUG
4022 	if (nfs4_server_t_debug)
4023 		dumpnfs4slist("nfs4_move_mi", mi, (clientid4)0, new);
4024 #endif
4025 	for (p = nfs4_server_lst.forw; p != &nfs4_server_lst; p = p->forw) {
4026 		if (p->zoneid != zoneid)
4027 			continue;
4028 		if (p->saddr.len == old->sv_addr.len &&
4029 		    bcmp(p->saddr.buf, old->sv_addr.buf, p->saddr.len) == 0 &&
4030 		    p->s_thread_exit != NFS4_THREAD_EXIT) {
4031 			op = p;
4032 			mutex_enter(&op->s_lock);
4033 			op->s_refcnt++;
4034 		}
4035 		if (p->saddr.len == new->sv_addr.len &&
4036 		    bcmp(p->saddr.buf, new->sv_addr.buf, p->saddr.len) == 0 &&
4037 		    p->s_thread_exit != NFS4_THREAD_EXIT) {
4038 			np = p;
4039 			mutex_enter(&np->s_lock);
4040 		}
4041 		if (op != NULL && np != NULL)
4042 			break;
4043 	}
4044 	if (op == NULL) {
4045 		/*
4046 		 * Filesystem has been forcibly unmounted.  Bail out.
4047 		 */
4048 		if (np != NULL)
4049 			mutex_exit(&np->s_lock);
4050 		mutex_exit(&nfs4_server_lst_lock);
4051 		return (NULL);
4052 	}
4053 	if (np != NULL) {
4054 		np->s_refcnt++;
4055 	} else {
4056 #ifdef DEBUG
4057 		NFS4_DEBUG(nfs4_client_failover_debug, (CE_NOTE,
4058 		    "nfs4_move_mi: no target nfs4_server, will create."));
4059 #endif
4060 		np = add_new_nfs4_server(new, kcred);
4061 	}
4062 	mutex_exit(&nfs4_server_lst_lock);
4063 
4064 	NFS4_DEBUG(nfs4_client_failover_debug, (CE_NOTE,
4065 	    "nfs4_move_mi: for mi 0x%p, "
4066 	    "old servinfo4 0x%p, new servinfo4 0x%p, "
4067 	    "old nfs4_server 0x%p, new nfs4_server 0x%p, ",
4068 	    (void*)mi, (void*)old, (void*)new,
4069 	    (void*)op, (void*)np));
4070 	ASSERT(op != NULL && np != NULL);
4071 
4072 	/* discard any delegations */
4073 	nfs4_deleg_discard(mi, op);
4074 
4075 	num_open = mi->mi_open_files;
4076 	mi->mi_open_files = 0;
4077 	op->state_ref_count -= num_open;
4078 	ASSERT(op->state_ref_count >= 0);
4079 	np->state_ref_count += num_open;
4080 	nfs4_remove_mi_from_server_nolock(mi, op);
4081 	mi->mi_open_files = num_open;
4082 	NFS4_DEBUG(nfs4_client_failover_debug, (CE_NOTE,
4083 	    "nfs4_move_mi: mi_open_files %d, op->cnt %d, np->cnt %d",
4084 	    mi->mi_open_files, op->state_ref_count, np->state_ref_count));
4085 
4086 	nfs4_add_mi_to_server(np, mi);
4087 
4088 	mutex_exit(&op->s_lock);
4089 	mutex_exit(&np->s_lock);
4090 	nfs4_server_rele(op);
4091 
4092 	return (np);
4093 }
4094 
4095 /*
4096  * Need to have the nfs4_server_lst_lock.
4097  * Search the nfs4_server list to find a match on this servinfo4
4098  * based on its address.
4099  *
4100  * Returns NULL if no match is found.  Otherwise returns a reference (which
4101  * must eventually be freed) to a locked nfs4_server.
4102  */
4103 nfs4_server_t *
4104 servinfo4_to_nfs4_server(servinfo4_t *srv_p)
4105 {
4106 	nfs4_server_t *np;
4107 	zoneid_t zoneid = nfs_zoneid();
4108 
4109 	ASSERT(MUTEX_HELD(&nfs4_server_lst_lock));
4110 	for (np = nfs4_server_lst.forw; np != &nfs4_server_lst; np = np->forw) {
4111 		if (np->zoneid == zoneid &&
4112 		    np->saddr.len == srv_p->sv_addr.len &&
4113 		    bcmp(np->saddr.buf, srv_p->sv_addr.buf,
4114 		    np->saddr.len) == 0 &&
4115 		    np->s_thread_exit != NFS4_THREAD_EXIT) {
4116 			mutex_enter(&np->s_lock);
4117 			np->s_refcnt++;
4118 			return (np);
4119 		}
4120 	}
4121 	return (NULL);
4122 }
4123 
4124 /*
4125  * Locks the nfs4_server down if it is found and returns a reference that
4126  * must eventually be freed.
4127  */
4128 static nfs4_server_t *
4129 lookup_nfs4_server(nfs4_server_t *sp, int any_state)
4130 {
4131 	nfs4_server_t *np;
4132 
4133 	mutex_enter(&nfs4_server_lst_lock);
4134 	for (np = nfs4_server_lst.forw; np != &nfs4_server_lst; np = np->forw) {
4135 		mutex_enter(&np->s_lock);
4136 		if (np == sp && np->s_refcnt > 0 &&
4137 		    (np->s_thread_exit != NFS4_THREAD_EXIT || any_state)) {
4138 			mutex_exit(&nfs4_server_lst_lock);
4139 			np->s_refcnt++;
4140 			return (np);
4141 		}
4142 		mutex_exit(&np->s_lock);
4143 	}
4144 	mutex_exit(&nfs4_server_lst_lock);
4145 
4146 	return (NULL);
4147 }
4148 
4149 /*
4150  * The caller should be holding mi->mi_recovlock, and it should continue to
4151  * hold the lock until done with the returned nfs4_server_t.  Once
4152  * mi->mi_recovlock is released, there is no guarantee that the returned
4153  * mi->nfs4_server_t will continue to correspond to mi.
4154  */
4155 nfs4_server_t *
4156 find_nfs4_server(mntinfo4_t *mi)
4157 {
4158 	ASSERT(nfs_rw_lock_held(&mi->mi_recovlock, RW_READER) ||
4159 	    nfs_rw_lock_held(&mi->mi_recovlock, RW_WRITER));
4160 
4161 	return (lookup_nfs4_server(mi->mi_srv, 0));
4162 }
4163 
4164 /*
4165  * Same as above, but takes an "any_state" parameter which can be
4166  * set to 1 if the caller wishes to find nfs4_server_t's which
4167  * have been marked for termination by the exit of the renew
4168  * thread.  This should only be used by operations which are
4169  * cleaning up and will not cause an OTW op.
4170  */
4171 nfs4_server_t *
4172 find_nfs4_server_all(mntinfo4_t *mi, int any_state)
4173 {
4174 	ASSERT(nfs_rw_lock_held(&mi->mi_recovlock, RW_READER) ||
4175 	    nfs_rw_lock_held(&mi->mi_recovlock, RW_WRITER));
4176 
4177 	return (lookup_nfs4_server(mi->mi_srv, any_state));
4178 }
4179 
4180 /*
4181  * Lock sp, but only if it's still active (in the list and hasn't been
4182  * flagged as exiting) or 'any_state' is non-zero.
4183  * Returns TRUE if sp got locked and adds a reference to sp.
4184  */
4185 bool_t
4186 nfs4_server_vlock(nfs4_server_t *sp, int any_state)
4187 {
4188 	return (lookup_nfs4_server(sp, any_state) != NULL);
4189 }
4190 
4191 /*
4192  * Release the reference to sp and destroy it if that's the last one.
4193  */
4194 
4195 void
4196 nfs4_server_rele(nfs4_server_t *sp)
4197 {
4198 	mutex_enter(&sp->s_lock);
4199 	ASSERT(sp->s_refcnt > 0);
4200 	sp->s_refcnt--;
4201 	if (sp->s_refcnt > 0) {
4202 		mutex_exit(&sp->s_lock);
4203 		return;
4204 	}
4205 	mutex_exit(&sp->s_lock);
4206 
4207 	mutex_enter(&nfs4_server_lst_lock);
4208 	mutex_enter(&sp->s_lock);
4209 	if (sp->s_refcnt > 0) {
4210 		mutex_exit(&sp->s_lock);
4211 		mutex_exit(&nfs4_server_lst_lock);
4212 		return;
4213 	}
4214 	remque(sp);
4215 	sp->forw = sp->back = NULL;
4216 	mutex_exit(&nfs4_server_lst_lock);
4217 	destroy_nfs4_server(sp);
4218 }
4219 
4220 static void
4221 destroy_nfs4_server(nfs4_server_t *sp)
4222 {
4223 	ASSERT(MUTEX_HELD(&sp->s_lock));
4224 	ASSERT(sp->s_refcnt == 0);
4225 	ASSERT(sp->s_otw_call_count == 0);
4226 
4227 	remove_all_mi(sp);
4228 
4229 	crfree(sp->s_cred);
4230 	kmem_free(sp->saddr.buf, sp->saddr.maxlen);
4231 	kmem_free(sp->clidtosend.id_val, sp->clidtosend.id_len);
4232 	mutex_exit(&sp->s_lock);
4233 
4234 	/* destroy the nfs4_server */
4235 	nfs4callback_destroy(sp);
4236 	list_destroy(&sp->s_deleg_list);
4237 	mutex_destroy(&sp->s_lock);
4238 	cv_destroy(&sp->cv_thread_exit);
4239 	cv_destroy(&sp->s_cv_otw_count);
4240 	cv_destroy(&sp->s_clientid_pend);
4241 	cv_destroy(&sp->wait_cb_null);
4242 	nfs_rw_destroy(&sp->s_recovlock);
4243 	kmem_free(sp, sizeof (*sp));
4244 }
4245 
4246 /*
4247  * Fork off a thread to free the data structures for a mount.
4248  */
4249 
4250 static void
4251 async_free_mount(vfs_t *vfsp, int flag, cred_t *cr)
4252 {
4253 	freemountargs_t *args;
4254 	args = kmem_alloc(sizeof (freemountargs_t), KM_SLEEP);
4255 	args->fm_vfsp = vfsp;
4256 	VFS_HOLD(vfsp);
4257 	MI4_HOLD(VFTOMI4(vfsp));
4258 	args->fm_flag = flag;
4259 	args->fm_cr = cr;
4260 	crhold(cr);
4261 	(void) zthread_create(NULL, 0, nfs4_free_mount_thread, args, 0,
4262 	    minclsyspri);
4263 }
4264 
4265 static void
4266 nfs4_free_mount_thread(freemountargs_t *args)
4267 {
4268 	mntinfo4_t *mi;
4269 	nfs4_free_mount(args->fm_vfsp, args->fm_flag, args->fm_cr);
4270 	mi = VFTOMI4(args->fm_vfsp);
4271 	crfree(args->fm_cr);
4272 	VFS_RELE(args->fm_vfsp);
4273 	MI4_RELE(mi);
4274 	kmem_free(args, sizeof (freemountargs_t));
4275 	zthread_exit();
4276 	/* NOTREACHED */
4277 }
4278 
4279 /*
4280  * Thread to free the data structures for a given filesystem.
4281  */
4282 static void
4283 nfs4_free_mount(vfs_t *vfsp, int flag, cred_t *cr)
4284 {
4285 	mntinfo4_t		*mi = VFTOMI4(vfsp);
4286 	nfs4_server_t		*sp;
4287 	callb_cpr_t		cpr_info;
4288 	kmutex_t		cpr_lock;
4289 	boolean_t		async_thread;
4290 	int			removed;
4291 
4292 	bool_t			must_unlock;
4293 	nfs4_ephemeral_tree_t	*eph_tree;
4294 
4295 	/*
4296 	 * We need to participate in the CPR framework if this is a kernel
4297 	 * thread.
4298 	 */
4299 	async_thread = (curproc == nfs_zone()->zone_zsched);
4300 	if (async_thread) {
4301 		mutex_init(&cpr_lock, NULL, MUTEX_DEFAULT, NULL);
4302 		CALLB_CPR_INIT(&cpr_info, &cpr_lock, callb_generic_cpr,
4303 		    "nfsv4AsyncUnmount");
4304 	}
4305 
4306 	/*
4307 	 * We need to wait for all outstanding OTW calls
4308 	 * and recovery to finish before we remove the mi
4309 	 * from the nfs4_server_t, as current pending
4310 	 * calls might still need this linkage (in order
4311 	 * to find a nfs4_server_t from a mntinfo4_t).
4312 	 */
4313 	(void) nfs_rw_enter_sig(&mi->mi_recovlock, RW_READER, FALSE);
4314 	sp = find_nfs4_server(mi);
4315 	nfs_rw_exit(&mi->mi_recovlock);
4316 
4317 	if (sp) {
4318 		while (sp->s_otw_call_count != 0) {
4319 			if (async_thread) {
4320 				mutex_enter(&cpr_lock);
4321 				CALLB_CPR_SAFE_BEGIN(&cpr_info);
4322 				mutex_exit(&cpr_lock);
4323 			}
4324 			cv_wait(&sp->s_cv_otw_count, &sp->s_lock);
4325 			if (async_thread) {
4326 				mutex_enter(&cpr_lock);
4327 				CALLB_CPR_SAFE_END(&cpr_info, &cpr_lock);
4328 				mutex_exit(&cpr_lock);
4329 			}
4330 		}
4331 		mutex_exit(&sp->s_lock);
4332 		nfs4_server_rele(sp);
4333 		sp = NULL;
4334 	}
4335 
4336 	mutex_enter(&mi->mi_lock);
4337 	while (mi->mi_in_recovery != 0) {
4338 		if (async_thread) {
4339 			mutex_enter(&cpr_lock);
4340 			CALLB_CPR_SAFE_BEGIN(&cpr_info);
4341 			mutex_exit(&cpr_lock);
4342 		}
4343 		cv_wait(&mi->mi_cv_in_recov, &mi->mi_lock);
4344 		if (async_thread) {
4345 			mutex_enter(&cpr_lock);
4346 			CALLB_CPR_SAFE_END(&cpr_info, &cpr_lock);
4347 			mutex_exit(&cpr_lock);
4348 		}
4349 	}
4350 	mutex_exit(&mi->mi_lock);
4351 
4352 	/*
4353 	 * If we got an error, then do not nuke the
4354 	 * tree. Either the harvester is busy reclaiming
4355 	 * this node or we ran into some busy condition.
4356 	 *
4357 	 * The harvester will eventually come along and cleanup.
4358 	 * The only problem would be the root mount point.
4359 	 *
4360 	 * Since the busy node can occur for a variety
4361 	 * of reasons and can result in an entry staying
4362 	 * in df output but no longer accessible from the
4363 	 * directory tree, we are okay.
4364 	 */
4365 	if (!nfs4_ephemeral_umount(mi, flag, cr,
4366 	    &must_unlock, &eph_tree))
4367 		nfs4_ephemeral_umount_activate(mi, &must_unlock,
4368 		    &eph_tree);
4369 
4370 	/*
4371 	 * The original purge of the dnlc via 'dounmount'
4372 	 * doesn't guarantee that another dnlc entry was not
4373 	 * added while we waitied for all outstanding OTW
4374 	 * and recovery calls to finish.  So re-purge the
4375 	 * dnlc now.
4376 	 */
4377 	(void) dnlc_purge_vfsp(vfsp, 0);
4378 
4379 	/*
4380 	 * We need to explicitly stop the manager thread; the asyc worker
4381 	 * threads can timeout and exit on their own.
4382 	 */
4383 	mutex_enter(&mi->mi_async_lock);
4384 	mi->mi_max_threads = 0;
4385 	cv_broadcast(&mi->mi_async_work_cv);
4386 	mutex_exit(&mi->mi_async_lock);
4387 	if (mi->mi_manager_thread)
4388 		nfs4_async_manager_stop(vfsp);
4389 
4390 	destroy_rtable4(vfsp, cr);
4391 
4392 	nfs4_remove_mi_from_server(mi, NULL);
4393 
4394 	if (async_thread) {
4395 		mutex_enter(&cpr_lock);
4396 		CALLB_CPR_EXIT(&cpr_info);	/* drops cpr_lock */
4397 		mutex_destroy(&cpr_lock);
4398 	}
4399 
4400 	removed = nfs4_mi_zonelist_remove(mi);
4401 	if (removed)
4402 		zone_rele(mi->mi_zone);
4403 }
4404 
4405 /* Referral related sub-routines */
4406 
4407 /* Freeup knetconfig */
4408 static void
4409 free_knconf_contents(struct knetconfig *k)
4410 {
4411 	if (k == NULL)
4412 		return;
4413 	if (k->knc_protofmly)
4414 		kmem_free(k->knc_protofmly, KNC_STRSIZE);
4415 	if (k->knc_proto)
4416 		kmem_free(k->knc_proto, KNC_STRSIZE);
4417 }
4418 
4419 /*
4420  * This updates newpath variable with exact name component from the
4421  * path which gave us a NFS4ERR_MOVED error.
4422  * If the path is /rp/aaa/bbb and nth value is 1, aaa is returned.
4423  */
4424 static char *
4425 extract_referral_point(const char *svp, int nth)
4426 {
4427 	int num_slashes = 0;
4428 	const char *p;
4429 	char *newpath = NULL;
4430 	int i = 0;
4431 
4432 	newpath = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
4433 	for (p = svp; *p; p++) {
4434 		if (*p == '/')
4435 			num_slashes++;
4436 		if (num_slashes == nth + 1) {
4437 			p++;
4438 			while (*p != '/') {
4439 				if (*p == '\0')
4440 					break;
4441 				newpath[i] = *p;
4442 				i++;
4443 				p++;
4444 			}
4445 			newpath[i++] = '\0';
4446 			break;
4447 		}
4448 	}
4449 	return (newpath);
4450 }
4451 
4452 /*
4453  * This sets up a new path in sv_path to do a lookup of the referral point.
4454  * If the path is /rp/aaa/bbb and the referral point is aaa,
4455  * this updates /rp/aaa. This path will be used to get referral
4456  * location.
4457  */
4458 static void
4459 setup_newsvpath(servinfo4_t *svp, int nth)
4460 {
4461 	int num_slashes = 0, pathlen, i = 0;
4462 	char *newpath, *p;
4463 
4464 	newpath = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
4465 	for (p = svp->sv_path; *p; p++) {
4466 		newpath[i] =  *p;
4467 		if (*p == '/')
4468 			num_slashes++;
4469 		if (num_slashes == nth + 1) {
4470 			newpath[i] = '\0';
4471 			pathlen = strlen(newpath) + 1;
4472 			kmem_free(svp->sv_path, svp->sv_pathlen);
4473 			svp->sv_path = kmem_alloc(pathlen, KM_SLEEP);
4474 			svp->sv_pathlen = pathlen;
4475 			bcopy(newpath, svp->sv_path, pathlen);
4476 			break;
4477 		}
4478 		i++;
4479 	}
4480 	kmem_free(newpath, MAXPATHLEN);
4481 }
4482