xref: /freebsd/sys/cam/ctl/ctl_ha.c (revision 43faedc1339a9624c7acedb7f3e5624e64da5b99)
1 /*-
2  * Copyright (c) 2015 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer,
10  *    without modification, immediately at the beginning of the file.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
33 #include <sys/kthread.h>
34 #include <sys/types.h>
35 #include <sys/limits.h>
36 #include <sys/lock.h>
37 #include <sys/module.h>
38 #include <sys/mutex.h>
39 #include <sys/condvar.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/proc.h>
43 #include <sys/conf.h>
44 #include <sys/queue.h>
45 #include <sys/sysctl.h>
46 #include <sys/socket.h>
47 #include <sys/socketvar.h>
48 #include <sys/uio.h>
49 #include <netinet/in.h>
50 #include <netinet/tcp.h>
51 #include <vm/uma.h>
52 
53 #include <cam/cam.h>
54 #include <cam/scsi/scsi_all.h>
55 #include <cam/scsi/scsi_da.h>
56 #include <cam/ctl/ctl_io.h>
57 #include <cam/ctl/ctl.h>
58 #include <cam/ctl/ctl_frontend.h>
59 #include <cam/ctl/ctl_util.h>
60 #include <cam/ctl/ctl_backend.h>
61 #include <cam/ctl/ctl_ioctl.h>
62 #include <cam/ctl/ctl_ha.h>
63 #include <cam/ctl/ctl_private.h>
64 #include <cam/ctl/ctl_debug.h>
65 #include <cam/ctl/ctl_error.h>
66 
67 #if (__FreeBSD_version < 1100000)
68 struct mbufq {
69 	struct mbuf *head;
70 	struct mbuf *tail;
71 };
72 
73 static void
74 mbufq_init(struct mbufq *q, int limit)
75 {
76 
77 	q->head = q->tail = NULL;
78 }
79 
80 static void
81 mbufq_drain(struct mbufq *q)
82 {
83 	struct mbuf *m;
84 
85 	while ((m = q->head) != NULL) {
86 		q->head = m->m_nextpkt;
87 		m_freem(m);
88 	}
89 	q->tail = NULL;
90 }
91 
92 static struct mbuf *
93 mbufq_dequeue(struct mbufq *q)
94 {
95 	struct mbuf *m;
96 
97 	m = q->head;
98 	if (m) {
99 		if (q->tail == m)
100 			q->tail = NULL;
101 		q->head = m->m_nextpkt;
102 		m->m_nextpkt = NULL;
103 	}
104 	return (m);
105 }
106 
107 static void
108 mbufq_enqueue(struct mbufq *q, struct mbuf *m)
109 {
110 
111 	m->m_nextpkt = NULL;
112 	if (q->tail)
113 		q->tail->m_nextpkt = m;
114 	else
115 		q->head = m;
116 	q->tail = m;
117 }
118 
119 static u_int
120 sbavail(struct sockbuf *sb)
121 {
122 	return (sb->sb_cc);
123 }
124 
125 #if (__FreeBSD_version < 1000000)
126 #define	mtodo(m, o)	((void *)(((m)->m_data) + (o)))
127 #endif
128 #endif
129 
130 struct ha_msg_wire {
131 	uint32_t	 channel;
132 	uint32_t	 length;
133 };
134 
135 struct ha_dt_msg_wire {
136 	ctl_ha_dt_cmd	command;
137 	uint32_t	size;
138 	uint8_t		*local;
139 	uint8_t		*remote;
140 };
141 
142 struct ha_softc {
143 	struct ctl_softc *ha_ctl_softc;
144 	ctl_evt_handler	 ha_handler[CTL_HA_CHAN_MAX];
145 	char		 ha_peer[128];
146 	struct sockaddr_in  ha_peer_in;
147 	struct socket	*ha_lso;
148 	struct socket	*ha_so;
149 	struct mbufq	 ha_sendq;
150 	struct mbuf	*ha_sending;
151 	struct mtx	 ha_lock;
152 	int		 ha_connect;
153 	int		 ha_listen;
154 	int		 ha_connected;
155 	int		 ha_receiving;
156 	int		 ha_wakeup;
157 	int		 ha_disconnect;
158 	int		 ha_shutdown;
159 	eventhandler_tag ha_shutdown_eh;
160 	TAILQ_HEAD(, ctl_ha_dt_req) ha_dts;
161 } ha_softc;
162 
163 static void
164 ctl_ha_conn_wake(struct ha_softc *softc)
165 {
166 
167 	mtx_lock(&softc->ha_lock);
168 	softc->ha_wakeup = 1;
169 	mtx_unlock(&softc->ha_lock);
170 	wakeup(&softc->ha_wakeup);
171 }
172 
173 static int
174 ctl_ha_lupcall(struct socket *so, void *arg, int waitflag)
175 {
176 	struct ha_softc *softc = arg;
177 
178 	ctl_ha_conn_wake(softc);
179 	return (SU_OK);
180 }
181 
182 static int
183 ctl_ha_rupcall(struct socket *so, void *arg, int waitflag)
184 {
185 	struct ha_softc *softc = arg;
186 
187 	wakeup(&softc->ha_receiving);
188 	return (SU_OK);
189 }
190 
191 static int
192 ctl_ha_supcall(struct socket *so, void *arg, int waitflag)
193 {
194 	struct ha_softc *softc = arg;
195 
196 	ctl_ha_conn_wake(softc);
197 	return (SU_OK);
198 }
199 
200 static void
201 ctl_ha_evt(struct ha_softc *softc, ctl_ha_channel ch, ctl_ha_event evt,
202     int param)
203 {
204 	int i;
205 
206 	if (ch < CTL_HA_CHAN_MAX) {
207 		if (softc->ha_handler[ch])
208 			softc->ha_handler[ch](ch, evt, param);
209 		return;
210 	}
211 	for (i = 0; i < CTL_HA_CHAN_MAX; i++) {
212 		if (softc->ha_handler[i])
213 			softc->ha_handler[i](i, evt, param);
214 	}
215 }
216 
217 static void
218 ctl_ha_close(struct ha_softc *softc)
219 {
220 	struct socket *so = softc->ha_so;
221 	int report = 0;
222 
223 	if (softc->ha_connected || softc->ha_disconnect) {
224 		softc->ha_connected = 0;
225 		mbufq_drain(&softc->ha_sendq);
226 		m_freem(softc->ha_sending);
227 		softc->ha_sending = NULL;
228 		report = 1;
229 	}
230 	if (so) {
231 		SOCKBUF_LOCK(&so->so_rcv);
232 		soupcall_clear(so, SO_RCV);
233 		while (softc->ha_receiving) {
234 			wakeup(&softc->ha_receiving);
235 			msleep(&softc->ha_receiving, SOCKBUF_MTX(&so->so_rcv),
236 			    0, "ha_rx exit", 0);
237 		}
238 		SOCKBUF_UNLOCK(&so->so_rcv);
239 		SOCKBUF_LOCK(&so->so_snd);
240 		soupcall_clear(so, SO_SND);
241 		SOCKBUF_UNLOCK(&so->so_snd);
242 		softc->ha_so = NULL;
243 		if (softc->ha_connect)
244 			pause("reconnect", hz / 2);
245 		soclose(so);
246 	}
247 	if (report) {
248 		ctl_ha_evt(softc, CTL_HA_CHAN_MAX, CTL_HA_EVT_LINK_CHANGE,
249 		    (softc->ha_connect || softc->ha_listen) ?
250 		    CTL_HA_LINK_UNKNOWN : CTL_HA_LINK_OFFLINE);
251 	}
252 }
253 
254 static void
255 ctl_ha_lclose(struct ha_softc *softc)
256 {
257 
258 	if (softc->ha_lso) {
259 		SOCKBUF_LOCK(&softc->ha_lso->so_rcv);
260 		soupcall_clear(softc->ha_lso, SO_RCV);
261 		SOCKBUF_UNLOCK(&softc->ha_lso->so_rcv);
262 		soclose(softc->ha_lso);
263 		softc->ha_lso = NULL;
264 	}
265 }
266 
267 static void
268 ctl_ha_rx_thread(void *arg)
269 {
270 	struct ha_softc *softc = arg;
271 	struct socket *so = softc->ha_so;
272 	struct ha_msg_wire wire_hdr;
273 	struct uio uio;
274 	struct iovec iov;
275 	int error, flags, next;
276 
277 	bzero(&wire_hdr, sizeof(wire_hdr));
278 	while (1) {
279 		if (wire_hdr.length > 0)
280 			next = wire_hdr.length;
281 		else
282 			next = sizeof(wire_hdr);
283 		SOCKBUF_LOCK(&so->so_rcv);
284 		while (sbavail(&so->so_rcv) < next || softc->ha_disconnect) {
285 			if (softc->ha_connected == 0 || softc->ha_disconnect ||
286 			    so->so_error ||
287 			    (so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
288 				goto errout;
289 			}
290 			so->so_rcv.sb_lowat = next;
291 			msleep(&softc->ha_receiving, SOCKBUF_MTX(&so->so_rcv),
292 			    0, "-", 0);
293 		}
294 		SOCKBUF_UNLOCK(&so->so_rcv);
295 
296 		if (wire_hdr.length == 0) {
297 			iov.iov_base = &wire_hdr;
298 			iov.iov_len = sizeof(wire_hdr);
299 			uio.uio_iov = &iov;
300 			uio.uio_iovcnt = 1;
301 			uio.uio_rw = UIO_READ;
302 			uio.uio_segflg = UIO_SYSSPACE;
303 			uio.uio_td = curthread;
304 			uio.uio_resid = sizeof(wire_hdr);
305 			flags = MSG_DONTWAIT;
306 			error = soreceive(softc->ha_so, NULL, &uio, NULL,
307 			    NULL, &flags);
308 			if (error != 0) {
309 				printf("%s: header receive error %d\n",
310 				    __func__, error);
311 				SOCKBUF_LOCK(&so->so_rcv);
312 				goto errout;
313 			}
314 		} else {
315 			ctl_ha_evt(softc, wire_hdr.channel,
316 			    CTL_HA_EVT_MSG_RECV, wire_hdr.length);
317 			wire_hdr.length = 0;
318 		}
319 	}
320 
321 errout:
322 	softc->ha_receiving = 0;
323 	wakeup(&softc->ha_receiving);
324 	SOCKBUF_UNLOCK(&so->so_rcv);
325 	ctl_ha_conn_wake(softc);
326 	kthread_exit();
327 }
328 
329 static void
330 ctl_ha_send(struct ha_softc *softc)
331 {
332 	struct socket *so = softc->ha_so;
333 	int error;
334 
335 	while (1) {
336 		if (softc->ha_sending == NULL) {
337 			mtx_lock(&softc->ha_lock);
338 			softc->ha_sending = mbufq_dequeue(&softc->ha_sendq);
339 			mtx_unlock(&softc->ha_lock);
340 			if (softc->ha_sending == NULL) {
341 				so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1;
342 				break;
343 			}
344 		}
345 		SOCKBUF_LOCK(&so->so_snd);
346 		if (sbspace(&so->so_snd) < softc->ha_sending->m_pkthdr.len) {
347 			so->so_snd.sb_lowat = softc->ha_sending->m_pkthdr.len;
348 			SOCKBUF_UNLOCK(&so->so_snd);
349 			break;
350 		}
351 		SOCKBUF_UNLOCK(&so->so_snd);
352 		error = sosend(softc->ha_so, NULL, NULL, softc->ha_sending,
353 		    NULL, MSG_DONTWAIT, curthread);
354 		softc->ha_sending = NULL;
355 		if (error != 0) {
356 			printf("%s: sosend() error %d\n", __func__, error);
357 			return;
358 		}
359 	};
360 }
361 
362 static void
363 ctl_ha_sock_setup(struct ha_softc *softc)
364 {
365 	struct sockopt opt;
366 	struct socket *so = softc->ha_so;
367 	int error, val;
368 
369 	val = 1024 * 1024;
370 	error = soreserve(so, val, val);
371 	if (error)
372 		printf("%s: soreserve failed %d\n", __func__, error);
373 
374 	SOCKBUF_LOCK(&so->so_rcv);
375 	so->so_rcv.sb_lowat = sizeof(struct ha_msg_wire);
376 	soupcall_set(so, SO_RCV, ctl_ha_rupcall, softc);
377 	SOCKBUF_UNLOCK(&so->so_rcv);
378 	SOCKBUF_LOCK(&so->so_snd);
379 	so->so_snd.sb_lowat = sizeof(struct ha_msg_wire);
380 	soupcall_set(so, SO_SND, ctl_ha_supcall, softc);
381 	SOCKBUF_UNLOCK(&so->so_snd);
382 
383 	bzero(&opt, sizeof(struct sockopt));
384 	opt.sopt_dir = SOPT_SET;
385 	opt.sopt_level = SOL_SOCKET;
386 	opt.sopt_name = SO_KEEPALIVE;
387 	opt.sopt_val = &val;
388 	opt.sopt_valsize = sizeof(val);
389 	val = 1;
390 	error = sosetopt(so, &opt);
391 	if (error)
392 		printf("%s: KEEPALIVE setting failed %d\n", __func__, error);
393 
394 	opt.sopt_level = IPPROTO_TCP;
395 	opt.sopt_name = TCP_NODELAY;
396 	val = 1;
397 	error = sosetopt(so, &opt);
398 	if (error)
399 		printf("%s: NODELAY setting failed %d\n", __func__, error);
400 
401 	opt.sopt_name = TCP_KEEPINIT;
402 	val = 3;
403 	error = sosetopt(so, &opt);
404 	if (error)
405 		printf("%s: KEEPINIT setting failed %d\n", __func__, error);
406 
407 	opt.sopt_name = TCP_KEEPIDLE;
408 	val = 1;
409 	error = sosetopt(so, &opt);
410 	if (error)
411 		printf("%s: KEEPIDLE setting failed %d\n", __func__, error);
412 
413 	opt.sopt_name = TCP_KEEPINTVL;
414 	val = 1;
415 	error = sosetopt(so, &opt);
416 	if (error)
417 		printf("%s: KEEPINTVL setting failed %d\n", __func__, error);
418 
419 	opt.sopt_name = TCP_KEEPCNT;
420 	val = 5;
421 	error = sosetopt(so, &opt);
422 	if (error)
423 		printf("%s: KEEPCNT setting failed %d\n", __func__, error);
424 }
425 
426 static int
427 ctl_ha_connect(struct ha_softc *softc)
428 {
429 	struct thread *td = curthread;
430 	struct sockaddr_in sa;
431 	struct socket *so;
432 	int error;
433 
434 	/* Create the socket */
435 	error = socreate(PF_INET, &so, SOCK_STREAM,
436 	    IPPROTO_TCP, td->td_ucred, td);
437 	if (error != 0) {
438 		printf("%s: socreate() error %d\n", __func__, error);
439 		return (error);
440 	}
441 	softc->ha_so = so;
442 	ctl_ha_sock_setup(softc);
443 
444 	memcpy(&sa, &softc->ha_peer_in, sizeof(sa));
445 	error = soconnect(so, (struct sockaddr *)&sa, td);
446 	if (error != 0 && bootverbose) {
447 		printf("%s: soconnect() error %d\n", __func__, error);
448 		goto out;
449 	}
450 	return (0);
451 
452 out:
453 	ctl_ha_close(softc);
454 	return (error);
455 }
456 
457 static int
458 ctl_ha_accept(struct ha_softc *softc)
459 {
460 	struct socket *so;
461 	struct sockaddr *sap;
462 	int error;
463 
464 	ACCEPT_LOCK();
465 	if (softc->ha_lso->so_rcv.sb_state & SBS_CANTRCVMORE)
466 		softc->ha_lso->so_error = ECONNABORTED;
467 	if (softc->ha_lso->so_error) {
468 		error = softc->ha_lso->so_error;
469 		softc->ha_lso->so_error = 0;
470 		ACCEPT_UNLOCK();
471 		printf("%s: socket error %d\n", __func__, error);
472 		goto out;
473 	}
474 	so = TAILQ_FIRST(&softc->ha_lso->so_comp);
475 	if (so == NULL) {
476 		ACCEPT_UNLOCK();
477 		return (EWOULDBLOCK);
478 	}
479 	KASSERT(!(so->so_qstate & SQ_INCOMP), ("accept1: so SQ_INCOMP"));
480 	KASSERT(so->so_qstate & SQ_COMP, ("accept1: so not SQ_COMP"));
481 
482 	/*
483 	 * Before changing the flags on the socket, we have to bump the
484 	 * reference count.  Otherwise, if the protocol calls sofree(),
485 	 * the socket will be released due to a zero refcount.
486 	 */
487 	SOCK_LOCK(so);			/* soref() and so_state update */
488 	soref(so);			/* file descriptor reference */
489 
490 	TAILQ_REMOVE(&softc->ha_lso->so_comp, so, so_list);
491 	softc->ha_lso->so_qlen--;
492 	so->so_state |= SS_NBIO;
493 	so->so_qstate &= ~SQ_COMP;
494 	so->so_head = NULL;
495 
496 	SOCK_UNLOCK(so);
497 	ACCEPT_UNLOCK();
498 
499 	sap = NULL;
500 	error = soaccept(so, &sap);
501 	if (error != 0) {
502 		printf("%s: soaccept() error %d\n", __func__, error);
503 		if (sap != NULL)
504 			free(sap, M_SONAME);
505 		goto out;
506 	}
507 	if (sap != NULL)
508 		free(sap, M_SONAME);
509 	softc->ha_so = so;
510 	ctl_ha_sock_setup(softc);
511 	return (0);
512 
513 out:
514 	ctl_ha_lclose(softc);
515 	return (error);
516 }
517 
518 static int
519 ctl_ha_listen(struct ha_softc *softc)
520 {
521 	struct thread *td = curthread;
522 	struct sockaddr_in sa;
523 	struct sockopt opt;
524 	int error, val;
525 
526 	/* Create the socket */
527 	if (softc->ha_lso == NULL) {
528 		error = socreate(PF_INET, &softc->ha_lso, SOCK_STREAM,
529 		    IPPROTO_TCP, td->td_ucred, td);
530 		if (error != 0) {
531 			printf("%s: socreate() error %d\n", __func__, error);
532 			return (error);
533 		}
534 		bzero(&opt, sizeof(struct sockopt));
535 		opt.sopt_dir = SOPT_SET;
536 		opt.sopt_level = SOL_SOCKET;
537 		opt.sopt_name = SO_REUSEADDR;
538 		opt.sopt_val = &val;
539 		opt.sopt_valsize = sizeof(val);
540 		val = 1;
541 		error = sosetopt(softc->ha_lso, &opt);
542 		if (error) {
543 			printf("%s: REUSEADDR setting failed %d\n",
544 			    __func__, error);
545 		}
546 		bzero(&opt, sizeof(struct sockopt));
547 		opt.sopt_dir = SOPT_SET;
548 		opt.sopt_level = SOL_SOCKET;
549 		opt.sopt_name = SO_REUSEPORT;
550 		opt.sopt_val = &val;
551 		opt.sopt_valsize = sizeof(val);
552 		val = 1;
553 		error = sosetopt(softc->ha_lso, &opt);
554 		if (error) {
555 			printf("%s: REUSEPORT setting failed %d\n",
556 			    __func__, error);
557 		}
558 		SOCKBUF_LOCK(&softc->ha_lso->so_rcv);
559 		soupcall_set(softc->ha_lso, SO_RCV, ctl_ha_lupcall, softc);
560 		SOCKBUF_UNLOCK(&softc->ha_lso->so_rcv);
561 	}
562 
563 	memcpy(&sa, &softc->ha_peer_in, sizeof(sa));
564 	error = sobind(softc->ha_lso, (struct sockaddr *)&sa, td);
565 	if (error != 0) {
566 		printf("%s: sobind() error %d\n", __func__, error);
567 		goto out;
568 	}
569 	error = solisten(softc->ha_lso, 1, td);
570 	if (error != 0) {
571 		printf("%s: solisten() error %d\n", __func__, error);
572 		goto out;
573 	}
574 	return (0);
575 
576 out:
577 	ctl_ha_lclose(softc);
578 	return (error);
579 }
580 
581 static void
582 ctl_ha_conn_thread(void *arg)
583 {
584 	struct ha_softc *softc = arg;
585 	int error;
586 
587 	while (1) {
588 		if (softc->ha_disconnect || softc->ha_shutdown) {
589 			ctl_ha_close(softc);
590 			if (softc->ha_disconnect == 2 || softc->ha_shutdown)
591 				ctl_ha_lclose(softc);
592 			softc->ha_disconnect = 0;
593 			if (softc->ha_shutdown)
594 				break;
595 		} else if (softc->ha_so != NULL &&
596 		    (softc->ha_so->so_error ||
597 		     softc->ha_so->so_rcv.sb_state & SBS_CANTRCVMORE))
598 			ctl_ha_close(softc);
599 		if (softc->ha_so == NULL) {
600 			if (softc->ha_lso != NULL)
601 				ctl_ha_accept(softc);
602 			else if (softc->ha_listen)
603 				ctl_ha_listen(softc);
604 			else if (softc->ha_connect)
605 				ctl_ha_connect(softc);
606 		}
607 		if (softc->ha_so != NULL) {
608 			if (softc->ha_connected == 0 &&
609 			    softc->ha_so->so_error == 0 &&
610 			    (softc->ha_so->so_state & SS_ISCONNECTING) == 0) {
611 				softc->ha_connected = 1;
612 				ctl_ha_evt(softc, CTL_HA_CHAN_MAX,
613 				    CTL_HA_EVT_LINK_CHANGE,
614 				    CTL_HA_LINK_ONLINE);
615 				softc->ha_receiving = 1;
616 				error = kproc_kthread_add(ctl_ha_rx_thread,
617 				    softc, &softc->ha_ctl_softc->ctl_proc,
618 				    NULL, 0, 0, "ctl", "ha_rx");
619 				if (error != 0) {
620 					printf("Error creating CTL HA rx thread!\n");
621 					softc->ha_receiving = 0;
622 					softc->ha_disconnect = 1;
623 				}
624 			}
625 			ctl_ha_send(softc);
626 		}
627 		mtx_lock(&softc->ha_lock);
628 		if (softc->ha_so != NULL &&
629 		    (softc->ha_so->so_error ||
630 		     softc->ha_so->so_rcv.sb_state & SBS_CANTRCVMORE))
631 			;
632 		else if (!softc->ha_wakeup)
633 			msleep(&softc->ha_wakeup, &softc->ha_lock, 0, "-", hz);
634 		softc->ha_wakeup = 0;
635 		mtx_unlock(&softc->ha_lock);
636 	}
637 	mtx_lock(&softc->ha_lock);
638 	softc->ha_shutdown = 2;
639 	wakeup(&softc->ha_wakeup);
640 	mtx_unlock(&softc->ha_lock);
641 	kthread_exit();
642 }
643 
644 static int
645 ctl_ha_peer_sysctl(SYSCTL_HANDLER_ARGS)
646 {
647 	struct ha_softc *softc = (struct ha_softc *)arg1;
648 	struct sockaddr_in *sa;
649 	int error, b1, b2, b3, b4, p, num;
650 	char buf[128];
651 
652 	strlcpy(buf, softc->ha_peer, sizeof(buf));
653 	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
654 	if ((error != 0) || (req->newptr == NULL) ||
655 	    strncmp(buf, softc->ha_peer, sizeof(buf)) == 0)
656 		return (error);
657 
658 	sa = &softc->ha_peer_in;
659 	mtx_lock(&softc->ha_lock);
660 	if ((num = sscanf(buf, "connect %d.%d.%d.%d:%d",
661 	    &b1, &b2, &b3, &b4, &p)) >= 4) {
662 		softc->ha_connect = 1;
663 		softc->ha_listen = 0;
664 	} else if ((num = sscanf(buf, "listen %d.%d.%d.%d:%d",
665 	    &b1, &b2, &b3, &b4, &p)) >= 4) {
666 		softc->ha_connect = 0;
667 		softc->ha_listen = 1;
668 	} else {
669 		softc->ha_connect = 0;
670 		softc->ha_listen = 0;
671 		if (buf[0] != 0) {
672 			buf[0] = 0;
673 			error = EINVAL;
674 		}
675 	}
676 	strlcpy(softc->ha_peer, buf, sizeof(softc->ha_peer));
677 	if (softc->ha_connect || softc->ha_listen) {
678 		memset(sa, 0, sizeof(*sa));
679 		sa->sin_len = sizeof(struct sockaddr_in);
680 		sa->sin_family = AF_INET;
681 		sa->sin_port = htons((num >= 5) ? p : 999);
682 		sa->sin_addr.s_addr =
683 		    htonl((b1 << 24) + (b2 << 16) + (b3 << 8) + b4);
684 	}
685 	softc->ha_disconnect = 2;
686 	softc->ha_wakeup = 1;
687 	mtx_unlock(&softc->ha_lock);
688 	wakeup(&softc->ha_wakeup);
689 	return (error);
690 }
691 
692 ctl_ha_status
693 ctl_ha_msg_register(ctl_ha_channel channel, ctl_evt_handler handler)
694 {
695 	struct ha_softc *softc = &ha_softc;
696 
697 	KASSERT(channel < CTL_HA_CHAN_MAX,
698 	    ("Wrong CTL HA channel %d", channel));
699 	softc->ha_handler[channel] = handler;
700 	return (CTL_HA_STATUS_SUCCESS);
701 }
702 
703 ctl_ha_status
704 ctl_ha_msg_deregister(ctl_ha_channel channel)
705 {
706 	struct ha_softc *softc = &ha_softc;
707 
708 	KASSERT(channel < CTL_HA_CHAN_MAX,
709 	    ("Wrong CTL HA channel %d", channel));
710 	softc->ha_handler[channel] = NULL;
711 	return (CTL_HA_STATUS_SUCCESS);
712 }
713 
714 /*
715  * Receive a message of the specified size.
716  */
717 ctl_ha_status
718 ctl_ha_msg_recv(ctl_ha_channel channel, void *addr, size_t len,
719 		int wait)
720 {
721 	struct ha_softc *softc = &ha_softc;
722 	struct uio uio;
723 	struct iovec iov;
724 	int error, flags;
725 
726 	if (!softc->ha_connected)
727 		return (CTL_HA_STATUS_DISCONNECT);
728 
729 	iov.iov_base = addr;
730 	iov.iov_len = len;
731 	uio.uio_iov = &iov;
732 	uio.uio_iovcnt = 1;
733 	uio.uio_rw = UIO_READ;
734 	uio.uio_segflg = UIO_SYSSPACE;
735 	uio.uio_td = curthread;
736 	uio.uio_resid = len;
737 	flags = wait ? 0 : MSG_DONTWAIT;
738 	error = soreceive(softc->ha_so, NULL, &uio, NULL, NULL, &flags);
739 	if (error == 0)
740 		return (CTL_HA_STATUS_SUCCESS);
741 
742 	/* Consider all errors fatal for HA sanity. */
743 	mtx_lock(&softc->ha_lock);
744 	if (softc->ha_connected) {
745 		softc->ha_disconnect = 1;
746 		softc->ha_wakeup = 1;
747 		wakeup(&softc->ha_wakeup);
748 	}
749 	mtx_unlock(&softc->ha_lock);
750 	return (CTL_HA_STATUS_ERROR);
751 }
752 
753 /*
754  * Send a message of the specified size.
755  */
756 ctl_ha_status
757 ctl_ha_msg_send2(ctl_ha_channel channel, const void *addr, size_t len,
758     const void *addr2, size_t len2, int wait)
759 {
760 	struct ha_softc *softc = &ha_softc;
761 	struct mbuf *mb, *newmb;
762 	struct ha_msg_wire hdr;
763 	size_t copylen, off;
764 
765 	if (!softc->ha_connected)
766 		return (CTL_HA_STATUS_DISCONNECT);
767 
768 	newmb = m_getm2(NULL, sizeof(hdr) + len + len2, wait, MT_DATA,
769 	    M_PKTHDR);
770 	if (newmb == NULL) {
771 		/* Consider all errors fatal for HA sanity. */
772 		mtx_lock(&softc->ha_lock);
773 		if (softc->ha_connected) {
774 			softc->ha_disconnect = 1;
775 			softc->ha_wakeup = 1;
776 			wakeup(&softc->ha_wakeup);
777 		}
778 		mtx_unlock(&softc->ha_lock);
779 		printf("%s: Can't allocate mbuf chain\n", __func__);
780 		return (CTL_HA_STATUS_ERROR);
781 	}
782 	hdr.channel = channel;
783 	hdr.length = len + len2;
784 	mb = newmb;
785 	memcpy(mtodo(mb, 0), &hdr, sizeof(hdr));
786 	mb->m_len += sizeof(hdr);
787 	off = 0;
788 	for (; mb != NULL && off < len; mb = mb->m_next) {
789 		copylen = min(M_TRAILINGSPACE(mb), len - off);
790 		memcpy(mtodo(mb, mb->m_len), (const char *)addr + off, copylen);
791 		mb->m_len += copylen;
792 		off += copylen;
793 		if (off == len)
794 			break;
795 	}
796 	KASSERT(off == len, ("%s: off (%zu) != len (%zu)", __func__,
797 	    off, len));
798 	off = 0;
799 	for (; mb != NULL && off < len2; mb = mb->m_next) {
800 		copylen = min(M_TRAILINGSPACE(mb), len2 - off);
801 		memcpy(mtodo(mb, mb->m_len), (const char *)addr2 + off, copylen);
802 		mb->m_len += copylen;
803 		off += copylen;
804 	}
805 	KASSERT(off == len2, ("%s: off (%zu) != len2 (%zu)", __func__,
806 	    off, len2));
807 	newmb->m_pkthdr.len = sizeof(hdr) + len + len2;
808 
809 	mtx_lock(&softc->ha_lock);
810 	if (!softc->ha_connected) {
811 		mtx_unlock(&softc->ha_lock);
812 		m_freem(newmb);
813 		return (CTL_HA_STATUS_DISCONNECT);
814 	}
815 	mbufq_enqueue(&softc->ha_sendq, newmb);
816 	softc->ha_wakeup = 1;
817 	mtx_unlock(&softc->ha_lock);
818 	wakeup(&softc->ha_wakeup);
819 	return (CTL_HA_STATUS_SUCCESS);
820 }
821 
822 ctl_ha_status
823 ctl_ha_msg_send(ctl_ha_channel channel, const void *addr, size_t len,
824     int wait)
825 {
826 
827 	return (ctl_ha_msg_send2(channel, addr, len, NULL, 0, wait));
828 }
829 
830 ctl_ha_status
831 ctl_ha_msg_abort(ctl_ha_channel channel)
832 {
833 	struct ha_softc *softc = &ha_softc;
834 
835 	mtx_lock(&softc->ha_lock);
836 	softc->ha_disconnect = 1;
837 	softc->ha_wakeup = 1;
838 	mtx_unlock(&softc->ha_lock);
839 	wakeup(&softc->ha_wakeup);
840 	return (CTL_HA_STATUS_SUCCESS);
841 }
842 
843 /*
844  * Allocate a data transfer request structure.
845  */
846 struct ctl_ha_dt_req *
847 ctl_dt_req_alloc(void)
848 {
849 
850 	return (malloc(sizeof(struct ctl_ha_dt_req), M_CTL, M_WAITOK | M_ZERO));
851 }
852 
853 /*
854  * Free a data transfer request structure.
855  */
856 void
857 ctl_dt_req_free(struct ctl_ha_dt_req *req)
858 {
859 
860 	free(req, M_CTL);
861 }
862 
863 /*
864  * Issue a DMA request for a single buffer.
865  */
866 ctl_ha_status
867 ctl_dt_single(struct ctl_ha_dt_req *req)
868 {
869 	struct ha_softc *softc = &ha_softc;
870 	struct ha_dt_msg_wire wire_dt;
871 	ctl_ha_status status;
872 
873 	wire_dt.command = req->command;
874 	wire_dt.size = req->size;
875 	wire_dt.local = req->local;
876 	wire_dt.remote = req->remote;
877 	if (req->command == CTL_HA_DT_CMD_READ && req->callback != NULL) {
878 		mtx_lock(&softc->ha_lock);
879 		TAILQ_INSERT_TAIL(&softc->ha_dts, req, links);
880 		mtx_unlock(&softc->ha_lock);
881 		ctl_ha_msg_send(CTL_HA_CHAN_DATA, &wire_dt, sizeof(wire_dt),
882 		    M_WAITOK);
883 		return (CTL_HA_STATUS_WAIT);
884 	}
885 	if (req->command == CTL_HA_DT_CMD_READ) {
886 		status = ctl_ha_msg_send(CTL_HA_CHAN_DATA, &wire_dt,
887 		    sizeof(wire_dt), M_WAITOK);
888 	} else {
889 		status = ctl_ha_msg_send2(CTL_HA_CHAN_DATA, &wire_dt,
890 		    sizeof(wire_dt), req->local, req->size, M_WAITOK);
891 	}
892 	return (status);
893 }
894 
895 static void
896 ctl_dt_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
897 {
898 	struct ha_softc *softc = &ha_softc;
899 	struct ctl_ha_dt_req *req;
900 	ctl_ha_status isc_status;
901 
902 	if (event == CTL_HA_EVT_MSG_RECV) {
903 		struct ha_dt_msg_wire wire_dt;
904 		uint8_t *tmp;
905 		int size;
906 
907 		size = min(sizeof(wire_dt), param);
908 		isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_DATA, &wire_dt,
909 					     size, M_WAITOK);
910 		if (isc_status != CTL_HA_STATUS_SUCCESS) {
911 			printf("%s: Error receiving message: %d\n",
912 			    __func__, isc_status);
913 			return;
914 		}
915 
916 		if (wire_dt.command == CTL_HA_DT_CMD_READ) {
917 			wire_dt.command = CTL_HA_DT_CMD_WRITE;
918 			tmp = wire_dt.local;
919 			wire_dt.local = wire_dt.remote;
920 			wire_dt.remote = tmp;
921 			ctl_ha_msg_send2(CTL_HA_CHAN_DATA, &wire_dt,
922 			    sizeof(wire_dt), wire_dt.local, wire_dt.size,
923 			    M_WAITOK);
924 		} else if (wire_dt.command == CTL_HA_DT_CMD_WRITE) {
925 			isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_DATA,
926 			    wire_dt.remote, wire_dt.size, M_WAITOK);
927 			mtx_lock(&softc->ha_lock);
928 			TAILQ_FOREACH(req, &softc->ha_dts, links) {
929 				if (req->local == wire_dt.remote) {
930 					TAILQ_REMOVE(&softc->ha_dts, req, links);
931 					break;
932 				}
933 			}
934 			mtx_unlock(&softc->ha_lock);
935 			if (req) {
936 				req->ret = isc_status;
937 				req->callback(req);
938 			}
939 		}
940 	} else if (event == CTL_HA_EVT_LINK_CHANGE) {
941 		CTL_DEBUG_PRINT(("%s: Link state change to %d\n", __func__,
942 		    param));
943 		if (param != CTL_HA_LINK_ONLINE) {
944 			mtx_lock(&softc->ha_lock);
945 			while ((req = TAILQ_FIRST(&softc->ha_dts)) != NULL) {
946 				TAILQ_REMOVE(&softc->ha_dts, req, links);
947 				mtx_unlock(&softc->ha_lock);
948 				req->ret = CTL_HA_STATUS_DISCONNECT;
949 				req->callback(req);
950 				mtx_lock(&softc->ha_lock);
951 			}
952 			mtx_unlock(&softc->ha_lock);
953 		}
954 	} else {
955 		printf("%s: Unknown event %d\n", __func__, event);
956 	}
957 }
958 
959 
960 ctl_ha_status
961 ctl_ha_msg_init(struct ctl_softc *ctl_softc)
962 {
963 	struct ha_softc *softc = &ha_softc;
964 	int error;
965 
966 	softc->ha_ctl_softc = ctl_softc;
967 	mtx_init(&softc->ha_lock, "CTL HA mutex", NULL, MTX_DEF);
968 	mbufq_init(&softc->ha_sendq, INT_MAX);
969 	TAILQ_INIT(&softc->ha_dts);
970 	error = kproc_kthread_add(ctl_ha_conn_thread, softc,
971 	    &ctl_softc->ctl_proc, NULL, 0, 0, "ctl", "ha_tx");
972 	if (error != 0) {
973 		printf("error creating CTL HA connection thread!\n");
974 		mtx_destroy(&softc->ha_lock);
975 		return (CTL_HA_STATUS_ERROR);
976 	}
977 	softc->ha_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
978 	    ctl_ha_msg_shutdown, ctl_softc, SHUTDOWN_PRI_FIRST);
979 	SYSCTL_ADD_PROC(&ctl_softc->sysctl_ctx,
980 	    SYSCTL_CHILDREN(ctl_softc->sysctl_tree),
981 	    OID_AUTO, "ha_peer", CTLTYPE_STRING | CTLFLAG_RWTUN,
982 	    softc, 0, ctl_ha_peer_sysctl, "A", "HA peer connection method");
983 
984 	if (ctl_ha_msg_register(CTL_HA_CHAN_DATA, ctl_dt_event_handler)
985 	    != CTL_HA_STATUS_SUCCESS) {
986 		printf("%s: ctl_ha_msg_register failed.\n", __func__);
987 	}
988 
989 	return (CTL_HA_STATUS_SUCCESS);
990 };
991 
992 void
993 ctl_ha_msg_shutdown(struct ctl_softc *ctl_softc)
994 {
995 	struct ha_softc *softc = &ha_softc;
996 
997 	/* Disconnect and shutdown threads. */
998 	mtx_lock(&softc->ha_lock);
999 	if (softc->ha_shutdown < 2) {
1000 		softc->ha_shutdown = 1;
1001 		softc->ha_wakeup = 1;
1002 		wakeup(&softc->ha_wakeup);
1003 		while (softc->ha_shutdown < 2) {
1004 			msleep(&softc->ha_wakeup, &softc->ha_lock, 0,
1005 			    "shutdown", hz);
1006 		}
1007 	}
1008 	mtx_unlock(&softc->ha_lock);
1009 };
1010 
1011 ctl_ha_status
1012 ctl_ha_msg_destroy(struct ctl_softc *ctl_softc)
1013 {
1014 	struct ha_softc *softc = &ha_softc;
1015 
1016 	if (softc->ha_shutdown_eh != NULL) {
1017 		EVENTHANDLER_DEREGISTER(shutdown_pre_sync,
1018 		    softc->ha_shutdown_eh);
1019 		softc->ha_shutdown_eh = NULL;
1020 	}
1021 
1022 	ctl_ha_msg_shutdown(ctl_softc);	/* Just in case. */
1023 
1024 	if (ctl_ha_msg_deregister(CTL_HA_CHAN_DATA) != CTL_HA_STATUS_SUCCESS)
1025 		printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
1026 
1027 	mtx_destroy(&softc->ha_lock);
1028 	return (CTL_HA_STATUS_SUCCESS);
1029 };
1030