xref: /freebsd/sys/rpc/clnt_vc.c (revision dd41de95a84d979615a2ef11df6850622bf6184e)
1 /*	$NetBSD: clnt_vc.c,v 1.4 2000/07/14 08:40:42 fvdl Exp $	*/
2 
3 /*-
4  * SPDX-License-Identifier: BSD-3-Clause
5  *
6  * Copyright (c) 2009, Sun Microsystems, Inc.
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions are met:
11  * - Redistributions of source code must retain the above copyright notice,
12  *   this list of conditions and the following disclaimer.
13  * - Redistributions in binary form must reproduce the above copyright notice,
14  *   this list of conditions and the following disclaimer in the documentation
15  *   and/or other materials provided with the distribution.
16  * - Neither the name of Sun Microsystems, Inc. nor the names of its
17  *   contributors may be used to endorse or promote products derived
18  *   from this software without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
24  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #if defined(LIBC_SCCS) && !defined(lint)
34 static char *sccsid2 = "@(#)clnt_tcp.c 1.37 87/10/05 Copyr 1984 Sun Micro";
35 static char *sccsid = "@(#)clnt_tcp.c	2.2 88/08/01 4.0 RPCSRC";
36 static char sccsid3[] = "@(#)clnt_vc.c 1.19 89/03/16 Copyr 1988 Sun Micro";
37 #endif
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 /*
42  * clnt_tcp.c, Implements a TCP/IP based, client side RPC.
43  *
44  * Copyright (C) 1984, Sun Microsystems, Inc.
45  *
46  * TCP based RPC supports 'batched calls'.
47  * A sequence of calls may be batched-up in a send buffer.  The rpc call
48  * return immediately to the client even though the call was not necessarily
49  * sent.  The batching occurs if the results' xdr routine is NULL (0) AND
50  * the rpc timeout value is zero (see clnt.h, rpc).
51  *
52  * Clients should NOT casually batch calls that in fact return results; that is,
53  * the server side should be aware that a call is batched and not produce any
54  * return message.  Batched calls that produce many result messages can
55  * deadlock (netlock) the client and the server....
56  *
57  * Now go hang yourself.
58  */
59 
60 #include "opt_kern_tls.h"
61 
62 #include <sys/param.h>
63 #include <sys/systm.h>
64 #include <sys/kernel.h>
65 #include <sys/kthread.h>
66 #include <sys/ktls.h>
67 #include <sys/lock.h>
68 #include <sys/malloc.h>
69 #include <sys/mbuf.h>
70 #include <sys/mutex.h>
71 #include <sys/pcpu.h>
72 #include <sys/proc.h>
73 #include <sys/protosw.h>
74 #include <sys/socket.h>
75 #include <sys/socketvar.h>
76 #include <sys/sx.h>
77 #include <sys/syslog.h>
78 #include <sys/time.h>
79 #include <sys/uio.h>
80 
81 #include <net/vnet.h>
82 
83 #include <netinet/tcp.h>
84 
85 #include <rpc/rpc.h>
86 #include <rpc/rpc_com.h>
87 #include <rpc/krpc.h>
88 #include <rpc/rpcsec_tls.h>
89 
90 struct cmessage {
91         struct cmsghdr cmsg;
92         struct cmsgcred cmcred;
93 };
94 
95 static enum clnt_stat clnt_vc_call(CLIENT *, struct rpc_callextra *,
96     rpcproc_t, struct mbuf *, struct mbuf **, struct timeval);
97 static void clnt_vc_geterr(CLIENT *, struct rpc_err *);
98 static bool_t clnt_vc_freeres(CLIENT *, xdrproc_t, void *);
99 static void clnt_vc_abort(CLIENT *);
100 static bool_t clnt_vc_control(CLIENT *, u_int, void *);
101 static void clnt_vc_close(CLIENT *);
102 static void clnt_vc_destroy(CLIENT *);
103 static bool_t time_not_ok(struct timeval *);
104 static int clnt_vc_soupcall(struct socket *so, void *arg, int waitflag);
105 static void clnt_vc_dotlsupcall(void *data);
106 
107 static struct clnt_ops clnt_vc_ops = {
108 	.cl_call =	clnt_vc_call,
109 	.cl_abort =	clnt_vc_abort,
110 	.cl_geterr =	clnt_vc_geterr,
111 	.cl_freeres =	clnt_vc_freeres,
112 	.cl_close =	clnt_vc_close,
113 	.cl_destroy =	clnt_vc_destroy,
114 	.cl_control =	clnt_vc_control
115 };
116 
117 static void clnt_vc_upcallsdone(struct ct_data *);
118 
119 static int	fake_wchan;
120 
121 /*
122  * Create a client handle for a connection.
123  * Default options are set, which the user can change using clnt_control()'s.
124  * The rpc/vc package does buffering similar to stdio, so the client
125  * must pick send and receive buffer sizes, 0 => use the default.
126  * NB: fd is copied into a private area.
127  * NB: The rpch->cl_auth is set null authentication. Caller may wish to
128  * set this something more useful.
129  *
130  * fd should be an open socket
131  */
132 CLIENT *
133 clnt_vc_create(
134 	struct socket *so,		/* open file descriptor */
135 	struct sockaddr *raddr,		/* servers address */
136 	const rpcprog_t prog,		/* program number */
137 	const rpcvers_t vers,		/* version number */
138 	size_t sendsz,			/* buffer recv size */
139 	size_t recvsz,			/* buffer send size */
140 	int intrflag)			/* interruptible */
141 {
142 	CLIENT *cl;			/* client handle */
143 	struct ct_data *ct = NULL;	/* client handle */
144 	struct timeval now;
145 	struct rpc_msg call_msg;
146 	static uint32_t disrupt;
147 	struct __rpc_sockinfo si;
148 	XDR xdrs;
149 	int error, interrupted, one = 1, sleep_flag;
150 	struct sockopt sopt;
151 
152 	if (disrupt == 0)
153 		disrupt = (uint32_t)(long)raddr;
154 
155 	cl = (CLIENT *)mem_alloc(sizeof (*cl));
156 	ct = (struct ct_data *)mem_alloc(sizeof (*ct));
157 
158 	mtx_init(&ct->ct_lock, "ct->ct_lock", NULL, MTX_DEF);
159 	ct->ct_threads = 0;
160 	ct->ct_closing = FALSE;
161 	ct->ct_closed = FALSE;
162 	ct->ct_upcallrefs = 0;
163 	ct->ct_rcvstate = RPCRCVSTATE_NORMAL;
164 
165 	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
166 		error = soconnect(so, raddr, curthread);
167 		SOCK_LOCK(so);
168 		interrupted = 0;
169 		sleep_flag = PSOCK;
170 		if (intrflag != 0)
171 			sleep_flag |= PCATCH;
172 		while ((so->so_state & SS_ISCONNECTING)
173 		    && so->so_error == 0) {
174 			error = msleep(&so->so_timeo, SOCK_MTX(so),
175 			    sleep_flag, "connec", 0);
176 			if (error) {
177 				if (error == EINTR || error == ERESTART)
178 					interrupted = 1;
179 				break;
180 			}
181 		}
182 		if (error == 0) {
183 			error = so->so_error;
184 			so->so_error = 0;
185 		}
186 		SOCK_UNLOCK(so);
187 		if (error) {
188 			if (!interrupted)
189 				so->so_state &= ~SS_ISCONNECTING;
190 			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
191 			rpc_createerr.cf_error.re_errno = error;
192 			goto err;
193 		}
194 	}
195 
196 	if (!__rpc_socket2sockinfo(so, &si)) {
197 		goto err;
198 	}
199 
200 	if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
201 		bzero(&sopt, sizeof(sopt));
202 		sopt.sopt_dir = SOPT_SET;
203 		sopt.sopt_level = SOL_SOCKET;
204 		sopt.sopt_name = SO_KEEPALIVE;
205 		sopt.sopt_val = &one;
206 		sopt.sopt_valsize = sizeof(one);
207 		sosetopt(so, &sopt);
208 	}
209 
210 	if (so->so_proto->pr_protocol == IPPROTO_TCP) {
211 		bzero(&sopt, sizeof(sopt));
212 		sopt.sopt_dir = SOPT_SET;
213 		sopt.sopt_level = IPPROTO_TCP;
214 		sopt.sopt_name = TCP_NODELAY;
215 		sopt.sopt_val = &one;
216 		sopt.sopt_valsize = sizeof(one);
217 		sosetopt(so, &sopt);
218 	}
219 
220 	ct->ct_closeit = FALSE;
221 
222 	/*
223 	 * Set up private data struct
224 	 */
225 	ct->ct_socket = so;
226 	ct->ct_wait.tv_sec = -1;
227 	ct->ct_wait.tv_usec = -1;
228 	memcpy(&ct->ct_addr, raddr, raddr->sa_len);
229 
230 	/*
231 	 * Initialize call message
232 	 */
233 	getmicrotime(&now);
234 	ct->ct_xid = ((uint32_t)++disrupt) ^ __RPC_GETXID(&now);
235 	call_msg.rm_xid = ct->ct_xid;
236 	call_msg.rm_direction = CALL;
237 	call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
238 	call_msg.rm_call.cb_prog = (uint32_t)prog;
239 	call_msg.rm_call.cb_vers = (uint32_t)vers;
240 
241 	/*
242 	 * pre-serialize the static part of the call msg and stash it away
243 	 */
244 	xdrmem_create(&xdrs, ct->ct_mcallc, MCALL_MSG_SIZE,
245 	    XDR_ENCODE);
246 	if (! xdr_callhdr(&xdrs, &call_msg)) {
247 		if (ct->ct_closeit) {
248 			soclose(ct->ct_socket);
249 		}
250 		goto err;
251 	}
252 	ct->ct_mpos = XDR_GETPOS(&xdrs);
253 	XDR_DESTROY(&xdrs);
254 	ct->ct_waitchan = "rpcrecv";
255 	ct->ct_waitflag = 0;
256 
257 	/*
258 	 * Create a client handle which uses xdrrec for serialization
259 	 * and authnone for authentication.
260 	 */
261 	sendsz = __rpc_get_t_size(si.si_af, si.si_proto, (int)sendsz);
262 	recvsz = __rpc_get_t_size(si.si_af, si.si_proto, (int)recvsz);
263 	error = soreserve(ct->ct_socket, sendsz, recvsz);
264 	if (error != 0) {
265 		if (ct->ct_closeit) {
266 			soclose(ct->ct_socket);
267 		}
268 		goto err;
269 	}
270 	cl->cl_refs = 1;
271 	cl->cl_ops = &clnt_vc_ops;
272 	cl->cl_private = ct;
273 	cl->cl_auth = authnone_create();
274 
275 	SOCKBUF_LOCK(&ct->ct_socket->so_rcv);
276 	soupcall_set(ct->ct_socket, SO_RCV, clnt_vc_soupcall, ct);
277 	SOCKBUF_UNLOCK(&ct->ct_socket->so_rcv);
278 
279 	ct->ct_raw = NULL;
280 	ct->ct_record = NULL;
281 	ct->ct_record_resid = 0;
282 	ct->ct_sslrefno = 0;
283 	TAILQ_INIT(&ct->ct_pending);
284 	return (cl);
285 
286 err:
287 	mtx_destroy(&ct->ct_lock);
288 	mem_free(ct, sizeof (struct ct_data));
289 	mem_free(cl, sizeof (CLIENT));
290 
291 	return ((CLIENT *)NULL);
292 }
293 
294 static enum clnt_stat
295 clnt_vc_call(
296 	CLIENT		*cl,		/* client handle */
297 	struct rpc_callextra *ext,	/* call metadata */
298 	rpcproc_t	proc,		/* procedure number */
299 	struct mbuf	*args,		/* pointer to args */
300 	struct mbuf	**resultsp,	/* pointer to results */
301 	struct timeval	utimeout)
302 {
303 	struct ct_data *ct = (struct ct_data *) cl->cl_private;
304 	AUTH *auth;
305 	struct rpc_err *errp;
306 	enum clnt_stat stat;
307 	XDR xdrs;
308 	struct rpc_msg reply_msg;
309 	bool_t ok;
310 	int nrefreshes = 2;		/* number of times to refresh cred */
311 	struct timeval timeout;
312 	uint32_t xid;
313 	struct mbuf *mreq = NULL, *results;
314 	struct ct_request *cr;
315 	int error, maxextsiz, trycnt;
316 #ifdef KERN_TLS
317 	u_int maxlen;
318 #endif
319 
320 	cr = malloc(sizeof(struct ct_request), M_RPC, M_WAITOK);
321 
322 	mtx_lock(&ct->ct_lock);
323 
324 	if (ct->ct_closing || ct->ct_closed) {
325 		mtx_unlock(&ct->ct_lock);
326 		free(cr, M_RPC);
327 		return (RPC_CANTSEND);
328 	}
329 	ct->ct_threads++;
330 
331 	if (ext) {
332 		auth = ext->rc_auth;
333 		errp = &ext->rc_err;
334 	} else {
335 		auth = cl->cl_auth;
336 		errp = &ct->ct_error;
337 	}
338 
339 	cr->cr_mrep = NULL;
340 	cr->cr_error = 0;
341 
342 	if (ct->ct_wait.tv_usec == -1) {
343 		timeout = utimeout;	/* use supplied timeout */
344 	} else {
345 		timeout = ct->ct_wait;	/* use default timeout */
346 	}
347 
348 	/*
349 	 * After 15sec of looping, allow it to return RPC_CANTSEND, which will
350 	 * cause the clnt_reconnect layer to create a new TCP connection.
351 	 */
352 	trycnt = 15 * hz;
353 call_again:
354 	mtx_assert(&ct->ct_lock, MA_OWNED);
355 	if (ct->ct_closing || ct->ct_closed) {
356 		ct->ct_threads--;
357 		wakeup(ct);
358 		mtx_unlock(&ct->ct_lock);
359 		free(cr, M_RPC);
360 		return (RPC_CANTSEND);
361 	}
362 
363 	ct->ct_xid++;
364 	xid = ct->ct_xid;
365 
366 	mtx_unlock(&ct->ct_lock);
367 
368 	/*
369 	 * Leave space to pre-pend the record mark.
370 	 */
371 	mreq = m_gethdr(M_WAITOK, MT_DATA);
372 	mreq->m_data += sizeof(uint32_t);
373 	KASSERT(ct->ct_mpos + sizeof(uint32_t) <= MHLEN,
374 	    ("RPC header too big"));
375 	bcopy(ct->ct_mcallc, mreq->m_data, ct->ct_mpos);
376 	mreq->m_len = ct->ct_mpos;
377 
378 	/*
379 	 * The XID is the first thing in the request.
380 	 */
381 	*mtod(mreq, uint32_t *) = htonl(xid);
382 
383 	xdrmbuf_create(&xdrs, mreq, XDR_ENCODE);
384 
385 	errp->re_status = stat = RPC_SUCCESS;
386 
387 	if ((! XDR_PUTINT32(&xdrs, &proc)) ||
388 	    (! AUTH_MARSHALL(auth, xid, &xdrs,
389 		m_copym(args, 0, M_COPYALL, M_WAITOK)))) {
390 		errp->re_status = stat = RPC_CANTENCODEARGS;
391 		mtx_lock(&ct->ct_lock);
392 		goto out;
393 	}
394 	mreq->m_pkthdr.len = m_length(mreq, NULL);
395 
396 	/*
397 	 * Prepend a record marker containing the packet length.
398 	 */
399 	M_PREPEND(mreq, sizeof(uint32_t), M_WAITOK);
400 	*mtod(mreq, uint32_t *) =
401 		htonl(0x80000000 | (mreq->m_pkthdr.len - sizeof(uint32_t)));
402 
403 	cr->cr_xid = xid;
404 	mtx_lock(&ct->ct_lock);
405 	/*
406 	 * Check to see if the other end has already started to close down
407 	 * the connection. The upcall will have set ct_error.re_status
408 	 * to RPC_CANTRECV if this is the case.
409 	 * If the other end starts to close down the connection after this
410 	 * point, it will be detected later when cr_error is checked,
411 	 * since the request is in the ct_pending queue.
412 	 */
413 	if (ct->ct_error.re_status == RPC_CANTRECV) {
414 		if (errp != &ct->ct_error) {
415 			errp->re_errno = ct->ct_error.re_errno;
416 			errp->re_status = RPC_CANTRECV;
417 		}
418 		stat = RPC_CANTRECV;
419 		goto out;
420 	}
421 
422 	/* For TLS, wait for an upcall to be done, as required. */
423 	while ((ct->ct_rcvstate & (RPCRCVSTATE_NORMAL |
424 	    RPCRCVSTATE_NONAPPDATA)) == 0)
425 		msleep(&ct->ct_rcvstate, &ct->ct_lock, 0, "rpcrcvst", hz);
426 
427 	TAILQ_INSERT_TAIL(&ct->ct_pending, cr, cr_link);
428 	mtx_unlock(&ct->ct_lock);
429 
430 	if (ct->ct_sslrefno != 0) {
431 		/*
432 		 * Copy the mbuf chain to a chain of ext_pgs mbuf(s)
433 		 * as required by KERN_TLS.
434 		 */
435 		maxextsiz = TLS_MAX_MSG_SIZE_V10_2;
436 #ifdef KERN_TLS
437 		if (rpctls_getinfo(&maxlen, false, false))
438 			maxextsiz = min(maxextsiz, maxlen);
439 #endif
440 		mreq = _rpc_copym_into_ext_pgs(mreq, maxextsiz);
441 	}
442 	/*
443 	 * sosend consumes mreq.
444 	 */
445 	error = sosend(ct->ct_socket, NULL, NULL, mreq, NULL, 0, curthread);
446 	mreq = NULL;
447 	if (error == EMSGSIZE || (error == ERESTART &&
448 	    (ct->ct_waitflag & PCATCH) == 0 && trycnt-- > 0)) {
449 		SOCKBUF_LOCK(&ct->ct_socket->so_snd);
450 		sbwait(&ct->ct_socket->so_snd);
451 		SOCKBUF_UNLOCK(&ct->ct_socket->so_snd);
452 		AUTH_VALIDATE(auth, xid, NULL, NULL);
453 		mtx_lock(&ct->ct_lock);
454 		TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
455 		/* Sleep for 1 clock tick before trying the sosend() again. */
456 		msleep(&fake_wchan, &ct->ct_lock, 0, "rpclpsnd", 1);
457 		goto call_again;
458 	}
459 
460 	reply_msg.acpted_rply.ar_verf.oa_flavor = AUTH_NULL;
461 	reply_msg.acpted_rply.ar_verf.oa_base = cr->cr_verf;
462 	reply_msg.acpted_rply.ar_verf.oa_length = 0;
463 	reply_msg.acpted_rply.ar_results.where = NULL;
464 	reply_msg.acpted_rply.ar_results.proc = (xdrproc_t)xdr_void;
465 
466 	mtx_lock(&ct->ct_lock);
467 	if (error) {
468 		TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
469 		errp->re_errno = error;
470 		errp->re_status = stat = RPC_CANTSEND;
471 		goto out;
472 	}
473 
474 	/*
475 	 * Check to see if we got an upcall while waiting for the
476 	 * lock. In both these cases, the request has been removed
477 	 * from ct->ct_pending.
478 	 */
479 	if (cr->cr_error) {
480 		TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
481 		errp->re_errno = cr->cr_error;
482 		errp->re_status = stat = RPC_CANTRECV;
483 		goto out;
484 	}
485 	if (cr->cr_mrep) {
486 		TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
487 		goto got_reply;
488 	}
489 
490 	/*
491 	 * Hack to provide rpc-based message passing
492 	 */
493 	if (timeout.tv_sec == 0 && timeout.tv_usec == 0) {
494 		TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
495 		errp->re_status = stat = RPC_TIMEDOUT;
496 		goto out;
497 	}
498 
499 	error = msleep(cr, &ct->ct_lock, ct->ct_waitflag, ct->ct_waitchan,
500 	    tvtohz(&timeout));
501 
502 	TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
503 
504 	if (error) {
505 		/*
506 		 * The sleep returned an error so our request is still
507 		 * on the list. Turn the error code into an
508 		 * appropriate client status.
509 		 */
510 		errp->re_errno = error;
511 		switch (error) {
512 		case EINTR:
513 			stat = RPC_INTR;
514 			break;
515 		case EWOULDBLOCK:
516 			stat = RPC_TIMEDOUT;
517 			break;
518 		default:
519 			stat = RPC_CANTRECV;
520 		}
521 		errp->re_status = stat;
522 		goto out;
523 	} else {
524 		/*
525 		 * We were woken up by the upcall.  If the
526 		 * upcall had a receive error, report that,
527 		 * otherwise we have a reply.
528 		 */
529 		if (cr->cr_error) {
530 			errp->re_errno = cr->cr_error;
531 			errp->re_status = stat = RPC_CANTRECV;
532 			goto out;
533 		}
534 	}
535 
536 got_reply:
537 	/*
538 	 * Now decode and validate the response. We need to drop the
539 	 * lock since xdr_replymsg may end up sleeping in malloc.
540 	 */
541 	mtx_unlock(&ct->ct_lock);
542 
543 	if (ext && ext->rc_feedback)
544 		ext->rc_feedback(FEEDBACK_OK, proc, ext->rc_feedback_arg);
545 
546 	xdrmbuf_create(&xdrs, cr->cr_mrep, XDR_DECODE);
547 	ok = xdr_replymsg(&xdrs, &reply_msg);
548 	cr->cr_mrep = NULL;
549 
550 	if (ok) {
551 		if ((reply_msg.rm_reply.rp_stat == MSG_ACCEPTED) &&
552 		    (reply_msg.acpted_rply.ar_stat == SUCCESS))
553 			errp->re_status = stat = RPC_SUCCESS;
554 		else
555 			stat = _seterr_reply(&reply_msg, errp);
556 
557 		if (stat == RPC_SUCCESS) {
558 			results = xdrmbuf_getall(&xdrs);
559 			if (!AUTH_VALIDATE(auth, xid,
560 				&reply_msg.acpted_rply.ar_verf,
561 				&results)) {
562 				errp->re_status = stat = RPC_AUTHERROR;
563 				errp->re_why = AUTH_INVALIDRESP;
564 			} else {
565 				KASSERT(results,
566 				    ("auth validated but no result"));
567 				*resultsp = results;
568 			}
569 		}		/* end successful completion */
570 		/*
571 		 * If unsuccessful AND error is an authentication error
572 		 * then refresh credentials and try again, else break
573 		 */
574 		else if (stat == RPC_AUTHERROR)
575 			/* maybe our credentials need to be refreshed ... */
576 			if (nrefreshes > 0 &&
577 			    AUTH_REFRESH(auth, &reply_msg)) {
578 				nrefreshes--;
579 				XDR_DESTROY(&xdrs);
580 				mtx_lock(&ct->ct_lock);
581 				goto call_again;
582 			}
583 		/* end of unsuccessful completion */
584 	}	/* end of valid reply message */
585 	else {
586 		errp->re_status = stat = RPC_CANTDECODERES;
587 	}
588 	XDR_DESTROY(&xdrs);
589 	mtx_lock(&ct->ct_lock);
590 out:
591 	mtx_assert(&ct->ct_lock, MA_OWNED);
592 
593 	KASSERT(stat != RPC_SUCCESS || *resultsp,
594 	    ("RPC_SUCCESS without reply"));
595 
596 	if (mreq)
597 		m_freem(mreq);
598 	if (cr->cr_mrep)
599 		m_freem(cr->cr_mrep);
600 
601 	ct->ct_threads--;
602 	if (ct->ct_closing)
603 		wakeup(ct);
604 
605 	mtx_unlock(&ct->ct_lock);
606 
607 	if (auth && stat != RPC_SUCCESS)
608 		AUTH_VALIDATE(auth, xid, NULL, NULL);
609 
610 	free(cr, M_RPC);
611 
612 	return (stat);
613 }
614 
615 static void
616 clnt_vc_geterr(CLIENT *cl, struct rpc_err *errp)
617 {
618 	struct ct_data *ct = (struct ct_data *) cl->cl_private;
619 
620 	*errp = ct->ct_error;
621 }
622 
623 static bool_t
624 clnt_vc_freeres(CLIENT *cl, xdrproc_t xdr_res, void *res_ptr)
625 {
626 	XDR xdrs;
627 	bool_t dummy;
628 
629 	xdrs.x_op = XDR_FREE;
630 	dummy = (*xdr_res)(&xdrs, res_ptr);
631 
632 	return (dummy);
633 }
634 
635 /*ARGSUSED*/
636 static void
637 clnt_vc_abort(CLIENT *cl)
638 {
639 }
640 
641 static bool_t
642 clnt_vc_control(CLIENT *cl, u_int request, void *info)
643 {
644 	struct ct_data *ct = (struct ct_data *)cl->cl_private;
645 	void *infop = info;
646 	SVCXPRT *xprt;
647 	uint64_t *p;
648 	int error;
649 	static u_int thrdnum = 0;
650 
651 	mtx_lock(&ct->ct_lock);
652 
653 	switch (request) {
654 	case CLSET_FD_CLOSE:
655 		ct->ct_closeit = TRUE;
656 		mtx_unlock(&ct->ct_lock);
657 		return (TRUE);
658 	case CLSET_FD_NCLOSE:
659 		ct->ct_closeit = FALSE;
660 		mtx_unlock(&ct->ct_lock);
661 		return (TRUE);
662 	default:
663 		break;
664 	}
665 
666 	/* for other requests which use info */
667 	if (info == NULL) {
668 		mtx_unlock(&ct->ct_lock);
669 		return (FALSE);
670 	}
671 	switch (request) {
672 	case CLSET_TIMEOUT:
673 		if (time_not_ok((struct timeval *)info)) {
674 			mtx_unlock(&ct->ct_lock);
675 			return (FALSE);
676 		}
677 		ct->ct_wait = *(struct timeval *)infop;
678 		break;
679 	case CLGET_TIMEOUT:
680 		*(struct timeval *)infop = ct->ct_wait;
681 		break;
682 	case CLGET_SERVER_ADDR:
683 		(void) memcpy(info, &ct->ct_addr, (size_t)ct->ct_addr.ss_len);
684 		break;
685 	case CLGET_SVC_ADDR:
686 		/*
687 		 * Slightly different semantics to userland - we use
688 		 * sockaddr instead of netbuf.
689 		 */
690 		memcpy(info, &ct->ct_addr, ct->ct_addr.ss_len);
691 		break;
692 	case CLSET_SVC_ADDR:		/* set to new address */
693 		mtx_unlock(&ct->ct_lock);
694 		return (FALSE);
695 	case CLGET_XID:
696 		*(uint32_t *)info = ct->ct_xid;
697 		break;
698 	case CLSET_XID:
699 		/* This will set the xid of the NEXT call */
700 		/* decrement by 1 as clnt_vc_call() increments once */
701 		ct->ct_xid = *(uint32_t *)info - 1;
702 		break;
703 	case CLGET_VERS:
704 		/*
705 		 * This RELIES on the information that, in the call body,
706 		 * the version number field is the fifth field from the
707 		 * beginning of the RPC header. MUST be changed if the
708 		 * call_struct is changed
709 		 */
710 		*(uint32_t *)info =
711 		    ntohl(*(uint32_t *)(void *)(ct->ct_mcallc +
712 		    4 * BYTES_PER_XDR_UNIT));
713 		break;
714 
715 	case CLSET_VERS:
716 		*(uint32_t *)(void *)(ct->ct_mcallc +
717 		    4 * BYTES_PER_XDR_UNIT) =
718 		    htonl(*(uint32_t *)info);
719 		break;
720 
721 	case CLGET_PROG:
722 		/*
723 		 * This RELIES on the information that, in the call body,
724 		 * the program number field is the fourth field from the
725 		 * beginning of the RPC header. MUST be changed if the
726 		 * call_struct is changed
727 		 */
728 		*(uint32_t *)info =
729 		    ntohl(*(uint32_t *)(void *)(ct->ct_mcallc +
730 		    3 * BYTES_PER_XDR_UNIT));
731 		break;
732 
733 	case CLSET_PROG:
734 		*(uint32_t *)(void *)(ct->ct_mcallc +
735 		    3 * BYTES_PER_XDR_UNIT) =
736 		    htonl(*(uint32_t *)info);
737 		break;
738 
739 	case CLSET_WAITCHAN:
740 		ct->ct_waitchan = (const char *)info;
741 		break;
742 
743 	case CLGET_WAITCHAN:
744 		*(const char **) info = ct->ct_waitchan;
745 		break;
746 
747 	case CLSET_INTERRUPTIBLE:
748 		if (*(int *) info)
749 			ct->ct_waitflag = PCATCH;
750 		else
751 			ct->ct_waitflag = 0;
752 		break;
753 
754 	case CLGET_INTERRUPTIBLE:
755 		if (ct->ct_waitflag)
756 			*(int *) info = TRUE;
757 		else
758 			*(int *) info = FALSE;
759 		break;
760 
761 	case CLSET_BACKCHANNEL:
762 		xprt = (SVCXPRT *)info;
763 		if (ct->ct_backchannelxprt == NULL) {
764 			xprt->xp_p2 = ct;
765 			if (ct->ct_sslrefno != 0)
766 				xprt->xp_tls = RPCTLS_FLAGS_HANDSHAKE;
767 			ct->ct_backchannelxprt = xprt;
768 		}
769 		break;
770 
771 	case CLSET_TLS:
772 		p = (uint64_t *)info;
773 		ct->ct_sslsec = *p++;
774 		ct->ct_sslusec = *p++;
775 		ct->ct_sslrefno = *p;
776 		if (ct->ct_sslrefno != RPCTLS_REFNO_HANDSHAKE) {
777 			mtx_unlock(&ct->ct_lock);
778 			/* Start the kthread that handles upcalls. */
779 			error = kthread_add(clnt_vc_dotlsupcall, ct,
780 			    NULL, NULL, 0, 0, "krpctls%u", thrdnum++);
781 			if (error != 0)
782 				panic("Can't add KRPC thread error %d", error);
783 		} else
784 			mtx_unlock(&ct->ct_lock);
785 		return (TRUE);
786 
787 	case CLSET_BLOCKRCV:
788 		if (*(int *) info) {
789 			ct->ct_rcvstate &= ~RPCRCVSTATE_NORMAL;
790 			ct->ct_rcvstate |= RPCRCVSTATE_TLSHANDSHAKE;
791 		} else {
792 			ct->ct_rcvstate &= ~RPCRCVSTATE_TLSHANDSHAKE;
793 			ct->ct_rcvstate |= RPCRCVSTATE_NORMAL;
794 		}
795 		break;
796 
797 	default:
798 		mtx_unlock(&ct->ct_lock);
799 		return (FALSE);
800 	}
801 
802 	mtx_unlock(&ct->ct_lock);
803 	return (TRUE);
804 }
805 
806 static void
807 clnt_vc_close(CLIENT *cl)
808 {
809 	struct ct_data *ct = (struct ct_data *) cl->cl_private;
810 	struct ct_request *cr;
811 
812 	mtx_lock(&ct->ct_lock);
813 
814 	if (ct->ct_closed) {
815 		mtx_unlock(&ct->ct_lock);
816 		return;
817 	}
818 
819 	if (ct->ct_closing) {
820 		while (ct->ct_closing)
821 			msleep(ct, &ct->ct_lock, 0, "rpcclose", 0);
822 		KASSERT(ct->ct_closed, ("client should be closed"));
823 		mtx_unlock(&ct->ct_lock);
824 		return;
825 	}
826 
827 	if (ct->ct_socket) {
828 		ct->ct_closing = TRUE;
829 		mtx_unlock(&ct->ct_lock);
830 
831 		SOCKBUF_LOCK(&ct->ct_socket->so_rcv);
832 		if (ct->ct_socket->so_rcv.sb_upcall != NULL) {
833 			soupcall_clear(ct->ct_socket, SO_RCV);
834 			clnt_vc_upcallsdone(ct);
835 		}
836 		SOCKBUF_UNLOCK(&ct->ct_socket->so_rcv);
837 
838 		/*
839 		 * Abort any pending requests and wait until everyone
840 		 * has finished with clnt_vc_call.
841 		 */
842 		mtx_lock(&ct->ct_lock);
843 		TAILQ_FOREACH(cr, &ct->ct_pending, cr_link) {
844 			cr->cr_xid = 0;
845 			cr->cr_error = ESHUTDOWN;
846 			wakeup(cr);
847 		}
848 
849 		while (ct->ct_threads)
850 			msleep(ct, &ct->ct_lock, 0, "rpcclose", 0);
851 	}
852 
853 	ct->ct_closing = FALSE;
854 	ct->ct_closed = TRUE;
855 	wakeup(&ct->ct_sslrefno);
856 	mtx_unlock(&ct->ct_lock);
857 	wakeup(ct);
858 }
859 
860 static void
861 clnt_vc_destroy(CLIENT *cl)
862 {
863 	struct ct_data *ct = (struct ct_data *) cl->cl_private;
864 	struct socket *so = NULL;
865 	SVCXPRT *xprt;
866 	enum clnt_stat stat;
867 	uint32_t reterr;
868 
869 	clnt_vc_close(cl);
870 
871 	mtx_lock(&ct->ct_lock);
872 	xprt = ct->ct_backchannelxprt;
873 	ct->ct_backchannelxprt = NULL;
874 	if (xprt != NULL) {
875 		mtx_unlock(&ct->ct_lock);	/* To avoid a LOR. */
876 		sx_xlock(&xprt->xp_lock);
877 		mtx_lock(&ct->ct_lock);
878 		xprt->xp_p2 = NULL;
879 		sx_xunlock(&xprt->xp_lock);
880 	}
881 
882 	if (ct->ct_socket) {
883 		if (ct->ct_closeit) {
884 			so = ct->ct_socket;
885 		}
886 	}
887 
888 	/* Wait for the upcall kthread to terminate. */
889 	while ((ct->ct_rcvstate & RPCRCVSTATE_UPCALLTHREAD) != 0)
890 		msleep(&ct->ct_sslrefno, &ct->ct_lock, 0,
891 		    "clntvccl", hz);
892 	mtx_unlock(&ct->ct_lock);
893 
894 	mtx_destroy(&ct->ct_lock);
895 	if (so) {
896 		if (ct->ct_sslrefno != 0) {
897 			/*
898 			 * If the TLS handshake is in progress, the upcall
899 			 * will fail, but the socket should be closed by the
900 			 * daemon, since the connect upcall has just failed.
901 			 */
902 			if (ct->ct_sslrefno != RPCTLS_REFNO_HANDSHAKE) {
903 				/*
904 				 * If the upcall fails, the socket has
905 				 * probably been closed via the rpctlscd
906 				 * daemon having crashed or been
907 				 * restarted, so ignore return stat.
908 				 */
909 				stat = rpctls_cl_disconnect(ct->ct_sslsec,
910 				    ct->ct_sslusec, ct->ct_sslrefno,
911 				    &reterr);
912 			}
913 			/* Must sorele() to get rid of reference. */
914 			CURVNET_SET(so->so_vnet);
915 			SOCK_LOCK(so);
916 			sorele(so);
917 			CURVNET_RESTORE();
918 		} else {
919 			soshutdown(so, SHUT_WR);
920 			soclose(so);
921 		}
922 	}
923 	m_freem(ct->ct_record);
924 	m_freem(ct->ct_raw);
925 	mem_free(ct, sizeof(struct ct_data));
926 	if (cl->cl_netid && cl->cl_netid[0])
927 		mem_free(cl->cl_netid, strlen(cl->cl_netid) +1);
928 	if (cl->cl_tp && cl->cl_tp[0])
929 		mem_free(cl->cl_tp, strlen(cl->cl_tp) +1);
930 	mem_free(cl, sizeof(CLIENT));
931 }
932 
933 /*
934  * Make sure that the time is not garbage.   -1 value is disallowed.
935  * Note this is different from time_not_ok in clnt_dg.c
936  */
937 static bool_t
938 time_not_ok(struct timeval *t)
939 {
940 	return (t->tv_sec <= -1 || t->tv_sec > 100000000 ||
941 		t->tv_usec <= -1 || t->tv_usec > 1000000);
942 }
943 
944 int
945 clnt_vc_soupcall(struct socket *so, void *arg, int waitflag)
946 {
947 	struct ct_data *ct = (struct ct_data *) arg;
948 	struct uio uio;
949 	struct mbuf *m, *m2, **ctrlp;
950 	struct ct_request *cr;
951 	int error, rcvflag, foundreq;
952 	uint32_t xid_plus_direction[2], header;
953 	SVCXPRT *xprt;
954 	struct cf_conn *cd;
955 	u_int rawlen;
956 	struct cmsghdr *cmsg;
957 	struct tls_get_record tgr;
958 
959 	/*
960 	 * RPC-over-TLS needs to block reception during
961 	 * upcalls since the upcall will be doing I/O on
962 	 * the socket via openssl library calls.
963 	 */
964 	mtx_lock(&ct->ct_lock);
965 	if ((ct->ct_rcvstate & (RPCRCVSTATE_NORMAL |
966 	    RPCRCVSTATE_NONAPPDATA)) == 0) {
967 		/* Mark that a socket upcall needs to be done. */
968 		if ((ct->ct_rcvstate & (RPCRCVSTATE_UPCALLNEEDED |
969 		    RPCRCVSTATE_UPCALLINPROG)) != 0)
970 			ct->ct_rcvstate |= RPCRCVSTATE_SOUPCALLNEEDED;
971 		mtx_unlock(&ct->ct_lock);
972 		return (SU_OK);
973 	}
974 	mtx_unlock(&ct->ct_lock);
975 
976 	/*
977 	 * If another thread is already here, it must be in
978 	 * soreceive(), so just return to avoid races with it.
979 	 * ct_upcallrefs is protected by the SOCKBUF_LOCK(),
980 	 * which is held in this function, except when
981 	 * soreceive() is called.
982 	 */
983 	if (ct->ct_upcallrefs > 0)
984 		return (SU_OK);
985 	ct->ct_upcallrefs++;
986 
987 	/*
988 	 * Read as much as possible off the socket and link it
989 	 * onto ct_raw.
990 	 */
991 	for (;;) {
992 		uio.uio_resid = 1000000000;
993 		uio.uio_td = curthread;
994 		m2 = m = NULL;
995 		rcvflag = MSG_DONTWAIT | MSG_SOCALLBCK;
996 		if (ct->ct_sslrefno != 0 && (ct->ct_rcvstate &
997 		    RPCRCVSTATE_NORMAL) != 0) {
998 			rcvflag |= MSG_TLSAPPDATA;
999 			ctrlp = NULL;
1000 		} else
1001 			ctrlp = &m2;
1002 		SOCKBUF_UNLOCK(&so->so_rcv);
1003 		error = soreceive(so, NULL, &uio, &m, ctrlp, &rcvflag);
1004 		SOCKBUF_LOCK(&so->so_rcv);
1005 
1006 		if (error == EWOULDBLOCK) {
1007 			/*
1008 			 * We must re-test for readability after
1009 			 * taking the lock to protect us in the case
1010 			 * where a new packet arrives on the socket
1011 			 * after our call to soreceive fails with
1012 			 * EWOULDBLOCK.
1013 			 */
1014 			error = 0;
1015 			if (!soreadable(so))
1016 				break;
1017 			continue;
1018 		}
1019 		if (error == 0 && m == NULL) {
1020 			/*
1021 			 * We must have got EOF trying
1022 			 * to read from the stream.
1023 			 */
1024 			error = ECONNRESET;
1025 		}
1026 
1027 		/*
1028 		 * A return of ENXIO indicates that there is a
1029 		 * non-application data record at the head of the
1030 		 * socket's receive queue, for TLS connections.
1031 		 * This record needs to be handled in userland
1032 		 * via an SSL_read() call, so do an upcall to the daemon.
1033 		 */
1034 		if (ct->ct_sslrefno != 0 && error == ENXIO) {
1035 			/* Disable reception, marking an upcall needed. */
1036 			mtx_lock(&ct->ct_lock);
1037 			ct->ct_rcvstate |= RPCRCVSTATE_UPCALLNEEDED;
1038 			/*
1039 			 * If an upcall in needed, wake up the kthread
1040 			 * that runs clnt_vc_dotlsupcall().
1041 			 */
1042 			wakeup(&ct->ct_sslrefno);
1043 			mtx_unlock(&ct->ct_lock);
1044 			break;
1045 		}
1046 		if (error != 0)
1047 			break;
1048 
1049 		/* Process any record header(s). */
1050 		if (m2 != NULL) {
1051 			cmsg = mtod(m2, struct cmsghdr *);
1052 			if (cmsg->cmsg_type == TLS_GET_RECORD &&
1053 			    cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) {
1054 				memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr));
1055 				/*
1056 				 * This should have been handled by
1057 				 * setting RPCRCVSTATE_UPCALLNEEDED in
1058 				 * ct_rcvstate but if not, all we can do
1059 				 * is toss it away.
1060 				 */
1061 				if (tgr.tls_type != TLS_RLTYPE_APP) {
1062 					m_freem(m);
1063 					m_free(m2);
1064 					mtx_lock(&ct->ct_lock);
1065 					ct->ct_rcvstate &=
1066 					    ~RPCRCVSTATE_NONAPPDATA;
1067 					ct->ct_rcvstate |= RPCRCVSTATE_NORMAL;
1068 					mtx_unlock(&ct->ct_lock);
1069 					continue;
1070 				}
1071 			}
1072 			m_free(m2);
1073 		}
1074 
1075 		if (ct->ct_raw != NULL)
1076 			m_last(ct->ct_raw)->m_next = m;
1077 		else
1078 			ct->ct_raw = m;
1079 	}
1080 	rawlen = m_length(ct->ct_raw, NULL);
1081 
1082 	/* Now, process as much of ct_raw as possible. */
1083 	for (;;) {
1084 		/*
1085 		 * If ct_record_resid is zero, we are waiting for a
1086 		 * record mark.
1087 		 */
1088 		if (ct->ct_record_resid == 0) {
1089 			if (rawlen < sizeof(uint32_t))
1090 				break;
1091 			m_copydata(ct->ct_raw, 0, sizeof(uint32_t),
1092 			    (char *)&header);
1093 			header = ntohl(header);
1094 			ct->ct_record_resid = header & 0x7fffffff;
1095 			ct->ct_record_eor = ((header & 0x80000000) != 0);
1096 			m_adj(ct->ct_raw, sizeof(uint32_t));
1097 			rawlen -= sizeof(uint32_t);
1098 		} else {
1099 			/*
1100 			 * Move as much of the record as possible to
1101 			 * ct_record.
1102 			 */
1103 			if (rawlen == 0)
1104 				break;
1105 			if (rawlen <= ct->ct_record_resid) {
1106 				if (ct->ct_record != NULL)
1107 					m_last(ct->ct_record)->m_next =
1108 					    ct->ct_raw;
1109 				else
1110 					ct->ct_record = ct->ct_raw;
1111 				ct->ct_raw = NULL;
1112 				ct->ct_record_resid -= rawlen;
1113 				rawlen = 0;
1114 			} else {
1115 				m = m_split(ct->ct_raw, ct->ct_record_resid,
1116 				    M_NOWAIT);
1117 				if (m == NULL)
1118 					break;
1119 				if (ct->ct_record != NULL)
1120 					m_last(ct->ct_record)->m_next =
1121 					    ct->ct_raw;
1122 				else
1123 					ct->ct_record = ct->ct_raw;
1124 				rawlen -= ct->ct_record_resid;
1125 				ct->ct_record_resid = 0;
1126 				ct->ct_raw = m;
1127 			}
1128 			if (ct->ct_record_resid > 0)
1129 				break;
1130 
1131 			/*
1132 			 * If we have the entire record, see if we can
1133 			 * match it to a request.
1134 			 */
1135 			if (ct->ct_record_eor) {
1136 				/*
1137 				 * The XID is in the first uint32_t of
1138 				 * the reply and the message direction
1139 				 * is the second one.
1140 				 */
1141 				if (ct->ct_record->m_len <
1142 				    sizeof(xid_plus_direction) &&
1143 				    m_length(ct->ct_record, NULL) <
1144 				    sizeof(xid_plus_direction)) {
1145 					/*
1146 					 * What to do now?
1147 					 * The data in the TCP stream is
1148 					 * corrupted such that there is no
1149 					 * valid RPC message to parse.
1150 					 * I think it best to close this
1151 					 * connection and allow
1152 					 * clnt_reconnect_call() to try
1153 					 * and establish a new one.
1154 					 */
1155 					printf("clnt_vc_soupcall: "
1156 					    "connection data corrupted\n");
1157 					error = ECONNRESET;
1158 					goto wakeup_all;
1159 				}
1160 				m_copydata(ct->ct_record, 0,
1161 				    sizeof(xid_plus_direction),
1162 				    (char *)xid_plus_direction);
1163 				xid_plus_direction[0] =
1164 				    ntohl(xid_plus_direction[0]);
1165 				xid_plus_direction[1] =
1166 				    ntohl(xid_plus_direction[1]);
1167 				/* Check message direction. */
1168 				if (xid_plus_direction[1] == CALL) {
1169 					/* This is a backchannel request. */
1170 					mtx_lock(&ct->ct_lock);
1171 					xprt = ct->ct_backchannelxprt;
1172 					if (xprt == NULL) {
1173 						mtx_unlock(&ct->ct_lock);
1174 						/* Just throw it away. */
1175 						m_freem(ct->ct_record);
1176 						ct->ct_record = NULL;
1177 					} else {
1178 						cd = (struct cf_conn *)
1179 						    xprt->xp_p1;
1180 						m2 = cd->mreq;
1181 						/*
1182 						 * The requests are chained
1183 						 * in the m_nextpkt list.
1184 						 */
1185 						while (m2 != NULL &&
1186 						    m2->m_nextpkt != NULL)
1187 							/* Find end of list. */
1188 							m2 = m2->m_nextpkt;
1189 						if (m2 != NULL)
1190 							m2->m_nextpkt =
1191 							    ct->ct_record;
1192 						else
1193 							cd->mreq =
1194 							    ct->ct_record;
1195 						ct->ct_record->m_nextpkt =
1196 						    NULL;
1197 						ct->ct_record = NULL;
1198 						xprt_active(xprt);
1199 						mtx_unlock(&ct->ct_lock);
1200 					}
1201 				} else {
1202 					mtx_lock(&ct->ct_lock);
1203 					foundreq = 0;
1204 					TAILQ_FOREACH(cr, &ct->ct_pending,
1205 					    cr_link) {
1206 						if (cr->cr_xid ==
1207 						    xid_plus_direction[0]) {
1208 							/*
1209 							 * This one
1210 							 * matches. We leave
1211 							 * the reply mbuf in
1212 							 * cr->cr_mrep. Set
1213 							 * the XID to zero so
1214 							 * that we will ignore
1215 							 * any duplicated
1216 							 * replies.
1217 							 */
1218 							cr->cr_xid = 0;
1219 							cr->cr_mrep =
1220 							    ct->ct_record;
1221 							cr->cr_error = 0;
1222 							foundreq = 1;
1223 							wakeup(cr);
1224 							break;
1225 						}
1226 					}
1227 					mtx_unlock(&ct->ct_lock);
1228 
1229 					if (!foundreq)
1230 						m_freem(ct->ct_record);
1231 					ct->ct_record = NULL;
1232 				}
1233 			}
1234 		}
1235 	}
1236 
1237 	if (error != 0) {
1238 	wakeup_all:
1239 		/*
1240 		 * This socket is broken, so mark that it cannot
1241 		 * receive and fail all RPCs waiting for a reply
1242 		 * on it, so that they will be retried on a new
1243 		 * TCP connection created by clnt_reconnect_X().
1244 		 */
1245 		mtx_lock(&ct->ct_lock);
1246 		ct->ct_error.re_status = RPC_CANTRECV;
1247 		ct->ct_error.re_errno = error;
1248 		TAILQ_FOREACH(cr, &ct->ct_pending, cr_link) {
1249 			cr->cr_error = error;
1250 			wakeup(cr);
1251 		}
1252 		mtx_unlock(&ct->ct_lock);
1253 	}
1254 
1255 	ct->ct_upcallrefs--;
1256 	if (ct->ct_upcallrefs < 0)
1257 		panic("rpcvc upcall refcnt");
1258 	if (ct->ct_upcallrefs == 0)
1259 		wakeup(&ct->ct_upcallrefs);
1260 	return (SU_OK);
1261 }
1262 
1263 /*
1264  * Wait for all upcalls in progress to complete.
1265  */
1266 static void
1267 clnt_vc_upcallsdone(struct ct_data *ct)
1268 {
1269 
1270 	SOCKBUF_LOCK_ASSERT(&ct->ct_socket->so_rcv);
1271 
1272 	while (ct->ct_upcallrefs > 0)
1273 		(void) msleep(&ct->ct_upcallrefs,
1274 		    SOCKBUF_MTX(&ct->ct_socket->so_rcv), 0, "rpcvcup", 0);
1275 }
1276 
1277 /*
1278  * Do a TLS upcall to the rpctlscd daemon, as required.
1279  * This function runs as a kthread.
1280  */
1281 static void
1282 clnt_vc_dotlsupcall(void *data)
1283 {
1284 	struct ct_data *ct = (struct ct_data *)data;
1285 	enum clnt_stat ret;
1286 	uint32_t reterr;
1287 
1288 	mtx_lock(&ct->ct_lock);
1289 	ct->ct_rcvstate |= RPCRCVSTATE_UPCALLTHREAD;
1290 	while (!ct->ct_closed) {
1291 		if ((ct->ct_rcvstate & RPCRCVSTATE_UPCALLNEEDED) != 0) {
1292 			ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLNEEDED;
1293 			ct->ct_rcvstate |= RPCRCVSTATE_UPCALLINPROG;
1294 			if (ct->ct_sslrefno != 0 && ct->ct_sslrefno !=
1295 			    RPCTLS_REFNO_HANDSHAKE) {
1296 				mtx_unlock(&ct->ct_lock);
1297 				ret = rpctls_cl_handlerecord(ct->ct_sslsec,
1298 				    ct->ct_sslusec, ct->ct_sslrefno, &reterr);
1299 				mtx_lock(&ct->ct_lock);
1300 			}
1301 			ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLINPROG;
1302 			if (ret == RPC_SUCCESS && reterr == RPCTLSERR_OK)
1303 				ct->ct_rcvstate |= RPCRCVSTATE_NORMAL;
1304 			else
1305 				ct->ct_rcvstate |= RPCRCVSTATE_NONAPPDATA;
1306 			wakeup(&ct->ct_rcvstate);
1307 		}
1308 		if ((ct->ct_rcvstate & RPCRCVSTATE_SOUPCALLNEEDED) != 0) {
1309 			ct->ct_rcvstate &= ~RPCRCVSTATE_SOUPCALLNEEDED;
1310 			mtx_unlock(&ct->ct_lock);
1311 			SOCKBUF_LOCK(&ct->ct_socket->so_rcv);
1312 			clnt_vc_soupcall(ct->ct_socket, ct, M_NOWAIT);
1313 			SOCKBUF_UNLOCK(&ct->ct_socket->so_rcv);
1314 			mtx_lock(&ct->ct_lock);
1315 		}
1316 		msleep(&ct->ct_sslrefno, &ct->ct_lock, 0, "clntvcdu", hz);
1317 	}
1318 	ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLTHREAD;
1319 	wakeup(&ct->ct_sslrefno);
1320 	mtx_unlock(&ct->ct_lock);
1321 	kthread_exit();
1322 }
1323