xref: /freebsd/sbin/hastd/secondary.c (revision 6b66c350a7fe8a2e29a86e8898fc1f27cb84f3e6)
1 /*-
2  * Copyright (c) 2009-2010 The FreeBSD Foundation
3  * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
4  * All rights reserved.
5  *
6  * This software was developed by Pawel Jakub Dawidek under sponsorship from
7  * the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/time.h>
36 #include <sys/bio.h>
37 #include <sys/disk.h>
38 #include <sys/stat.h>
39 
40 #include <err.h>
41 #include <errno.h>
42 #include <fcntl.h>
43 #include <libgeom.h>
44 #include <pthread.h>
45 #include <signal.h>
46 #include <stdint.h>
47 #include <stdio.h>
48 #include <string.h>
49 #include <sysexits.h>
50 #include <unistd.h>
51 
52 #include <activemap.h>
53 #include <nv.h>
54 #include <pjdlog.h>
55 
56 #include "control.h"
57 #include "event.h"
58 #include "hast.h"
59 #include "hast_proto.h"
60 #include "hastd.h"
61 #include "hooks.h"
62 #include "metadata.h"
63 #include "proto.h"
64 #include "subr.h"
65 #include "synch.h"
66 
67 struct hio {
68 	uint64_t	 hio_seq;
69 	int		 hio_error;
70 	void		*hio_data;
71 	uint8_t		 hio_cmd;
72 	uint64_t	 hio_offset;
73 	uint64_t	 hio_length;
74 	bool		 hio_memsync;
75 	TAILQ_ENTRY(hio) hio_next;
76 };
77 
78 static struct hast_resource *gres;
79 
80 /*
81  * Free list holds unused structures. When free list is empty, we have to wait
82  * until some in-progress requests are freed.
83  */
84 static TAILQ_HEAD(, hio) hio_free_list;
85 static size_t hio_free_list_size;
86 static pthread_mutex_t hio_free_list_lock;
87 static pthread_cond_t hio_free_list_cond;
88 /*
89  * Disk thread (the one that does I/O requests) takes requests from this list.
90  */
91 static TAILQ_HEAD(, hio) hio_disk_list;
92 static size_t hio_disk_list_size;
93 static pthread_mutex_t hio_disk_list_lock;
94 static pthread_cond_t hio_disk_list_cond;
95 /*
96  * Thread that sends requests back to primary takes requests from this list.
97  */
98 static TAILQ_HEAD(, hio) hio_send_list;
99 static size_t hio_send_list_size;
100 static pthread_mutex_t hio_send_list_lock;
101 static pthread_cond_t hio_send_list_cond;
102 
103 /*
104  * Maximum number of outstanding I/O requests.
105  */
106 #define	HAST_HIO_MAX	256
107 
108 static void *recv_thread(void *arg);
109 static void *disk_thread(void *arg);
110 static void *send_thread(void *arg);
111 
112 #define	QUEUE_INSERT(name, hio)	do {					\
113 	bool _wakeup;							\
114 									\
115 	mtx_lock(&hio_##name##_list_lock);				\
116 	_wakeup = TAILQ_EMPTY(&hio_##name##_list);			\
117 	TAILQ_INSERT_TAIL(&hio_##name##_list, (hio), hio_next);		\
118 	hio_##name##_list_size++;					\
119 	mtx_unlock(&hio_##name##_list_lock);				\
120 	if (_wakeup)							\
121 		cv_broadcast(&hio_##name##_list_cond);			\
122 } while (0)
123 #define	QUEUE_TAKE(name, hio)	do {					\
124 	mtx_lock(&hio_##name##_list_lock);				\
125 	while (((hio) = TAILQ_FIRST(&hio_##name##_list)) == NULL) {	\
126 		cv_wait(&hio_##name##_list_cond,			\
127 		    &hio_##name##_list_lock);				\
128 	}								\
129 	PJDLOG_ASSERT(hio_##name##_list_size != 0);			\
130 	hio_##name##_list_size--;					\
131 	TAILQ_REMOVE(&hio_##name##_list, (hio), hio_next);		\
132 	mtx_unlock(&hio_##name##_list_lock);				\
133 } while (0)
134 
135 static void
136 output_status_aux(struct nv *nvout)
137 {
138 
139 	nv_add_uint64(nvout, (uint64_t)hio_free_list_size, "idle_queue_size");
140 	nv_add_uint64(nvout, (uint64_t)hio_disk_list_size, "local_queue_size");
141 	nv_add_uint64(nvout, (uint64_t)hio_send_list_size, "send_queue_size");
142 }
143 
144 static void
145 hio_clear(struct hio *hio)
146 {
147 
148 	hio->hio_seq = 0;
149 	hio->hio_error = 0;
150 	hio->hio_cmd = HIO_UNDEF;
151 	hio->hio_offset = 0;
152 	hio->hio_length = 0;
153 	hio->hio_memsync = false;
154 }
155 
156 static void
157 hio_copy(const struct hio *srchio, struct hio *dsthio)
158 {
159 
160 	/*
161 	 * We don't copy hio_error, hio_data and hio_next fields.
162 	 */
163 
164 	dsthio->hio_seq = srchio->hio_seq;
165 	dsthio->hio_cmd = srchio->hio_cmd;
166 	dsthio->hio_offset = srchio->hio_offset;
167 	dsthio->hio_length = srchio->hio_length;
168 	dsthio->hio_memsync = srchio->hio_memsync;
169 }
170 
171 static void
172 init_environment(void)
173 {
174 	struct hio *hio;
175 	unsigned int ii;
176 
177 	/*
178 	 * Initialize lists, their locks and theirs condition variables.
179 	 */
180 	TAILQ_INIT(&hio_free_list);
181 	mtx_init(&hio_free_list_lock);
182 	cv_init(&hio_free_list_cond);
183 	TAILQ_INIT(&hio_disk_list);
184 	mtx_init(&hio_disk_list_lock);
185 	cv_init(&hio_disk_list_cond);
186 	TAILQ_INIT(&hio_send_list);
187 	mtx_init(&hio_send_list_lock);
188 	cv_init(&hio_send_list_cond);
189 
190 	/*
191 	 * Allocate requests pool and initialize requests.
192 	 */
193 	for (ii = 0; ii < HAST_HIO_MAX; ii++) {
194 		hio = malloc(sizeof(*hio));
195 		if (hio == NULL) {
196 			pjdlog_exitx(EX_TEMPFAIL,
197 			    "Unable to allocate memory (%zu bytes) for hio request.",
198 			    sizeof(*hio));
199 		}
200 		hio->hio_data = malloc(MAXPHYS);
201 		if (hio->hio_data == NULL) {
202 			pjdlog_exitx(EX_TEMPFAIL,
203 			    "Unable to allocate memory (%zu bytes) for gctl_data.",
204 			    (size_t)MAXPHYS);
205 		}
206 		hio_clear(hio);
207 		TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_next);
208 		hio_free_list_size++;
209 	}
210 }
211 
212 static void
213 init_local(struct hast_resource *res)
214 {
215 
216 	if (metadata_read(res, true) == -1)
217 		exit(EX_NOINPUT);
218 }
219 
220 static void
221 init_remote(struct hast_resource *res, struct nv *nvin)
222 {
223 	uint64_t resuid;
224 	struct nv *nvout;
225 	unsigned char *map;
226 	size_t mapsize;
227 
228 #ifdef notyet
229 	/* Setup direction. */
230 	if (proto_send(res->hr_remoteout, NULL, 0) == -1)
231 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
232 #endif
233 
234 	nvout = nv_alloc();
235 	nv_add_int64(nvout, (int64_t)res->hr_datasize, "datasize");
236 	nv_add_int32(nvout, (int32_t)res->hr_extentsize, "extentsize");
237 	resuid = nv_get_uint64(nvin, "resuid");
238 	res->hr_primary_localcnt = nv_get_uint64(nvin, "localcnt");
239 	res->hr_primary_remotecnt = nv_get_uint64(nvin, "remotecnt");
240 	nv_add_uint64(nvout, res->hr_secondary_localcnt, "localcnt");
241 	nv_add_uint64(nvout, res->hr_secondary_remotecnt, "remotecnt");
242 	mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
243 	    METADATA_SIZE, res->hr_extentsize, res->hr_local_sectorsize);
244 	map = malloc(mapsize);
245 	if (map == NULL) {
246 		pjdlog_exitx(EX_TEMPFAIL,
247 		    "Unable to allocate memory (%zu bytes) for activemap.",
248 		    mapsize);
249 	}
250 	/*
251 	 * When we work as primary and secondary is missing we will increase
252 	 * localcnt in our metadata. When secondary is connected and synced
253 	 * we make localcnt be equal to remotecnt, which means nodes are more
254 	 * or less in sync.
255 	 * Split-brain condition is when both nodes are not able to communicate
256 	 * and are both configured as primary nodes. In turn, they can both
257 	 * make incompatible changes to the data and we have to detect that.
258 	 * Under split-brain condition we will increase our localcnt on first
259 	 * write and remote node will increase its localcnt on first write.
260 	 * When we connect we can see that primary's localcnt is greater than
261 	 * our remotecnt (primary was modified while we weren't watching) and
262 	 * our localcnt is greater than primary's remotecnt (we were modified
263 	 * while primary wasn't watching).
264 	 * There are many possible combinations which are all gathered below.
265 	 * Don't pay too much attention to exact numbers, the more important
266 	 * is to compare them. We compare secondary's local with primary's
267 	 * remote and secondary's remote with primary's local.
268 	 * Note that every case where primary's localcnt is smaller than
269 	 * secondary's remotecnt and where secondary's localcnt is smaller than
270 	 * primary's remotecnt should be impossible in practise. We will perform
271 	 * full synchronization then. Those cases are marked with an asterisk.
272 	 * Regular synchronization means that only extents marked as dirty are
273 	 * synchronized (regular synchronization).
274 	 *
275 	 * SECONDARY METADATA PRIMARY METADATA
276 	 * local=3 remote=3   local=2 remote=2*  ?! Full sync from secondary.
277 	 * local=3 remote=3   local=2 remote=3*  ?! Full sync from primary.
278 	 * local=3 remote=3   local=2 remote=4*  ?! Full sync from primary.
279 	 * local=3 remote=3   local=3 remote=2   Primary is out-of-date,
280 	 *                                       regular sync from secondary.
281 	 * local=3 remote=3   local=3 remote=3   Regular sync just in case.
282 	 * local=3 remote=3   local=3 remote=4*  ?! Full sync from primary.
283 	 * local=3 remote=3   local=4 remote=2   Split-brain condition.
284 	 * local=3 remote=3   local=4 remote=3   Secondary out-of-date,
285 	 *                                       regular sync from primary.
286 	 * local=3 remote=3   local=4 remote=4*  ?! Full sync from primary.
287 	 */
288 	if (res->hr_resuid == 0) {
289 		/*
290 		 * Provider is used for the first time. If primary node done no
291 		 * writes yet as well (we will find "virgin" argument) then
292 		 * there is no need to synchronize anything. If primary node
293 		 * done any writes already we have to synchronize everything.
294 		 */
295 		PJDLOG_ASSERT(res->hr_secondary_localcnt == 0);
296 		res->hr_resuid = resuid;
297 		if (metadata_write(res) == -1)
298 			exit(EX_NOINPUT);
299 		if (nv_exists(nvin, "virgin")) {
300 			free(map);
301 			map = NULL;
302 			mapsize = 0;
303 		} else {
304 			memset(map, 0xff, mapsize);
305 		}
306 		nv_add_int8(nvout, 1, "virgin");
307 		nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
308 	} else if (res->hr_resuid != resuid) {
309 		char errmsg[256];
310 
311 		free(map);
312 		(void)snprintf(errmsg, sizeof(errmsg),
313 		    "Resource unique ID mismatch (primary=%ju, secondary=%ju).",
314 		    (uintmax_t)resuid, (uintmax_t)res->hr_resuid);
315 		pjdlog_error("%s", errmsg);
316 		nv_add_string(nvout, errmsg, "errmsg");
317 		if (hast_proto_send(res, res->hr_remotein, nvout,
318 		    NULL, 0) == -1) {
319 			pjdlog_exit(EX_TEMPFAIL,
320 			    "Unable to send response to %s",
321 			    res->hr_remoteaddr);
322 		}
323 		nv_free(nvout);
324 		exit(EX_CONFIG);
325 	} else if (
326 	    /* Is primary out-of-date? */
327 	    (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
328 	     res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
329 	    /* Are the nodes more or less in sync? */
330 	    (res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
331 	     res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
332 	    /* Is secondary out-of-date? */
333 	    (res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
334 	     res->hr_secondary_remotecnt < res->hr_primary_localcnt)) {
335 		/*
336 		 * Nodes are more or less in sync or one of the nodes is
337 		 * out-of-date.
338 		 * It doesn't matter at this point which one, we just have to
339 		 * send out local bitmap to the remote node.
340 		 */
341 		if (pread(res->hr_localfd, map, mapsize, METADATA_SIZE) !=
342 		    (ssize_t)mapsize) {
343 			pjdlog_exit(LOG_ERR, "Unable to read activemap");
344 		}
345 		if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
346 		     res->hr_secondary_remotecnt == res->hr_primary_localcnt) {
347 			/* Primary is out-of-date, sync from secondary. */
348 			nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
349 		} else {
350 			/*
351 			 * Secondary is out-of-date or counts match.
352 			 * Sync from primary.
353 			 */
354 			nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
355 		}
356 	} else if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
357 	     res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
358 		/*
359 		 * Not good, we have split-brain condition.
360 		 */
361 		free(map);
362 		pjdlog_error("Split-brain detected, exiting.");
363 		nv_add_string(nvout, "Split-brain condition!", "errmsg");
364 		if (hast_proto_send(res, res->hr_remotein, nvout,
365 		    NULL, 0) == -1) {
366 			pjdlog_exit(EX_TEMPFAIL,
367 			    "Unable to send response to %s",
368 			    res->hr_remoteaddr);
369 		}
370 		nv_free(nvout);
371 		/* Exit on split-brain. */
372 		event_send(res, EVENT_SPLITBRAIN);
373 		exit(EX_CONFIG);
374 	} else /* if (res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
375 	    res->hr_primary_localcnt < res->hr_secondary_remotecnt) */ {
376 		/*
377 		 * This should never happen in practise, but we will perform
378 		 * full synchronization.
379 		 */
380 		PJDLOG_ASSERT(res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
381 		    res->hr_primary_localcnt < res->hr_secondary_remotecnt);
382 		mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
383 		    METADATA_SIZE, res->hr_extentsize,
384 		    res->hr_local_sectorsize);
385 		memset(map, 0xff, mapsize);
386 		if (res->hr_secondary_localcnt > res->hr_primary_remotecnt) {
387 			/* In this one of five cases sync from secondary. */
388 			nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
389 		} else {
390 			/* For the rest four cases sync from primary. */
391 			nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
392 		}
393 		pjdlog_warning("This should never happen, asking for full synchronization (primary(local=%ju, remote=%ju), secondary(local=%ju, remote=%ju)).",
394 		    (uintmax_t)res->hr_primary_localcnt,
395 		    (uintmax_t)res->hr_primary_remotecnt,
396 		    (uintmax_t)res->hr_secondary_localcnt,
397 		    (uintmax_t)res->hr_secondary_remotecnt);
398 	}
399 	nv_add_uint32(nvout, (uint32_t)mapsize, "mapsize");
400 	if (hast_proto_send(res, res->hr_remotein, nvout, map, mapsize) == -1) {
401 		pjdlog_exit(EX_TEMPFAIL, "Unable to send activemap to %s",
402 		    res->hr_remoteaddr);
403 	}
404 	if (map != NULL)
405 		free(map);
406 	nv_free(nvout);
407 #ifdef notyet
408 	/* Setup direction. */
409 	if (proto_recv(res->hr_remotein, NULL, 0) == -1)
410 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
411 #endif
412 }
413 
414 void
415 hastd_secondary(struct hast_resource *res, struct nv *nvin)
416 {
417 	sigset_t mask;
418 	pthread_t td;
419 	pid_t pid;
420 	int error, mode, debuglevel;
421 
422 	/*
423 	 * Create communication channel between parent and child.
424 	 */
425 	if (proto_client(NULL, "socketpair://", &res->hr_ctrl) == -1) {
426 		KEEP_ERRNO((void)pidfile_remove(pfh));
427 		pjdlog_exit(EX_OSERR,
428 		    "Unable to create control sockets between parent and child");
429 	}
430 	/*
431 	 * Create communication channel between child and parent.
432 	 */
433 	if (proto_client(NULL, "socketpair://", &res->hr_event) == -1) {
434 		KEEP_ERRNO((void)pidfile_remove(pfh));
435 		pjdlog_exit(EX_OSERR,
436 		    "Unable to create event sockets between child and parent");
437 	}
438 
439 	pid = fork();
440 	if (pid == -1) {
441 		KEEP_ERRNO((void)pidfile_remove(pfh));
442 		pjdlog_exit(EX_OSERR, "Unable to fork");
443 	}
444 
445 	if (pid > 0) {
446 		/* This is parent. */
447 		proto_close(res->hr_remotein);
448 		res->hr_remotein = NULL;
449 		proto_close(res->hr_remoteout);
450 		res->hr_remoteout = NULL;
451 		/* Declare that we are receiver. */
452 		proto_recv(res->hr_event, NULL, 0);
453 		/* Declare that we are sender. */
454 		proto_send(res->hr_ctrl, NULL, 0);
455 		res->hr_workerpid = pid;
456 		return;
457 	}
458 
459 	gres = res;
460 	res->output_status_aux = output_status_aux;
461 	mode = pjdlog_mode_get();
462 	debuglevel = pjdlog_debug_get();
463 
464 	/* Declare that we are sender. */
465 	proto_send(res->hr_event, NULL, 0);
466 	/* Declare that we are receiver. */
467 	proto_recv(res->hr_ctrl, NULL, 0);
468 	descriptors_cleanup(res);
469 
470 	descriptors_assert(res, mode);
471 
472 	pjdlog_init(mode);
473 	pjdlog_debug_set(debuglevel);
474 	pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
475 	setproctitle("%s (%s)", res->hr_name, role2str(res->hr_role));
476 
477 	PJDLOG_VERIFY(sigemptyset(&mask) == 0);
478 	PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);
479 
480 	/* Error in setting timeout is not critical, but why should it fail? */
481 	if (proto_timeout(res->hr_remotein, 2 * HAST_KEEPALIVE) == -1)
482 		pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
483 	if (proto_timeout(res->hr_remoteout, res->hr_timeout) == -1)
484 		pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
485 
486 	init_local(res);
487 	init_environment();
488 
489 	if (drop_privs(res) != 0)
490 		exit(EX_CONFIG);
491 	pjdlog_info("Privileges successfully dropped.");
492 
493 	/*
494 	 * Create the control thread before sending any event to the parent,
495 	 * as we can deadlock when parent sends control request to worker,
496 	 * but worker has no control thread started yet, so parent waits.
497 	 * In the meantime worker sends an event to the parent, but parent
498 	 * is unable to handle the event, because it waits for control
499 	 * request response.
500 	 */
501 	error = pthread_create(&td, NULL, ctrl_thread, res);
502 	PJDLOG_ASSERT(error == 0);
503 
504 	init_remote(res, nvin);
505 	event_send(res, EVENT_CONNECT);
506 
507 	error = pthread_create(&td, NULL, recv_thread, res);
508 	PJDLOG_ASSERT(error == 0);
509 	error = pthread_create(&td, NULL, disk_thread, res);
510 	PJDLOG_ASSERT(error == 0);
511 	(void)send_thread(res);
512 }
513 
514 static void
515 reqlog(int loglevel, int debuglevel, int error, struct hio *hio,
516     const char *fmt, ...)
517 {
518 	char msg[1024];
519 	va_list ap;
520 	int len;
521 
522 	va_start(ap, fmt);
523 	len = vsnprintf(msg, sizeof(msg), fmt, ap);
524 	va_end(ap);
525 	if ((size_t)len < sizeof(msg)) {
526 		switch (hio->hio_cmd) {
527 		case HIO_READ:
528 			(void)snprintf(msg + len, sizeof(msg) - len,
529 			    "READ(%ju, %ju).", (uintmax_t)hio->hio_offset,
530 			    (uintmax_t)hio->hio_length);
531 			break;
532 		case HIO_DELETE:
533 			(void)snprintf(msg + len, sizeof(msg) - len,
534 			    "DELETE(%ju, %ju).", (uintmax_t)hio->hio_offset,
535 			    (uintmax_t)hio->hio_length);
536 			break;
537 		case HIO_FLUSH:
538 			(void)snprintf(msg + len, sizeof(msg) - len, "FLUSH.");
539 			break;
540 		case HIO_WRITE:
541 			(void)snprintf(msg + len, sizeof(msg) - len,
542 			    "WRITE(%ju, %ju).", (uintmax_t)hio->hio_offset,
543 			    (uintmax_t)hio->hio_length);
544 			break;
545 		case HIO_KEEPALIVE:
546 			(void)snprintf(msg + len, sizeof(msg) - len, "KEEPALIVE.");
547 			break;
548 		default:
549 			(void)snprintf(msg + len, sizeof(msg) - len,
550 			    "UNKNOWN(%u).", (unsigned int)hio->hio_cmd);
551 			break;
552 		}
553 	}
554 	pjdlog_common(loglevel, debuglevel, error, "%s", msg);
555 }
556 
557 static int
558 requnpack(struct hast_resource *res, struct hio *hio, struct nv *nv)
559 {
560 
561 	hio->hio_cmd = nv_get_uint8(nv, "cmd");
562 	if (hio->hio_cmd == 0) {
563 		pjdlog_error("Header contains no 'cmd' field.");
564 		hio->hio_error = EINVAL;
565 		goto end;
566 	}
567 	if (hio->hio_cmd != HIO_KEEPALIVE) {
568 		hio->hio_seq = nv_get_uint64(nv, "seq");
569 		if (hio->hio_seq == 0) {
570 			pjdlog_error("Header contains no 'seq' field.");
571 			hio->hio_error = EINVAL;
572 			goto end;
573 		}
574 	}
575 	switch (hio->hio_cmd) {
576 	case HIO_FLUSH:
577 	case HIO_KEEPALIVE:
578 		break;
579 	case HIO_WRITE:
580 		hio->hio_memsync = nv_exists(nv, "memsync");
581 		/* FALLTHROUGH */
582 	case HIO_READ:
583 	case HIO_DELETE:
584 		hio->hio_offset = nv_get_uint64(nv, "offset");
585 		if (nv_error(nv) != 0) {
586 			pjdlog_error("Header is missing 'offset' field.");
587 			hio->hio_error = EINVAL;
588 			goto end;
589 		}
590 		hio->hio_length = nv_get_uint64(nv, "length");
591 		if (nv_error(nv) != 0) {
592 			pjdlog_error("Header is missing 'length' field.");
593 			hio->hio_error = EINVAL;
594 			goto end;
595 		}
596 		if (hio->hio_length == 0) {
597 			pjdlog_error("Data length is zero.");
598 			hio->hio_error = EINVAL;
599 			goto end;
600 		}
601 		if (hio->hio_cmd != HIO_DELETE && hio->hio_length > MAXPHYS) {
602 			pjdlog_error("Data length is too large (%ju > %ju).",
603 			    (uintmax_t)hio->hio_length, (uintmax_t)MAXPHYS);
604 			hio->hio_error = EINVAL;
605 			goto end;
606 		}
607 		if ((hio->hio_offset % res->hr_local_sectorsize) != 0) {
608 			pjdlog_error("Offset %ju is not multiple of sector size.",
609 			    (uintmax_t)hio->hio_offset);
610 			hio->hio_error = EINVAL;
611 			goto end;
612 		}
613 		if ((hio->hio_length % res->hr_local_sectorsize) != 0) {
614 			pjdlog_error("Length %ju is not multiple of sector size.",
615 			    (uintmax_t)hio->hio_length);
616 			hio->hio_error = EINVAL;
617 			goto end;
618 		}
619 		if (hio->hio_offset + hio->hio_length >
620 		    (uint64_t)res->hr_datasize) {
621 			pjdlog_error("Data offset is too large (%ju > %ju).",
622 			    (uintmax_t)(hio->hio_offset + hio->hio_length),
623 			    (uintmax_t)res->hr_datasize);
624 			hio->hio_error = EINVAL;
625 			goto end;
626 		}
627 		break;
628 	default:
629 		pjdlog_error("Header contains invalid 'cmd' (%hhu).",
630 		    hio->hio_cmd);
631 		hio->hio_error = EINVAL;
632 		goto end;
633 	}
634 	hio->hio_error = 0;
635 end:
636 	return (hio->hio_error);
637 }
638 
639 static __dead2 void
640 secondary_exit(int exitcode, const char *fmt, ...)
641 {
642 	va_list ap;
643 
644 	PJDLOG_ASSERT(exitcode != EX_OK);
645 	va_start(ap, fmt);
646 	pjdlogv_errno(LOG_ERR, fmt, ap);
647 	va_end(ap);
648 	event_send(gres, EVENT_DISCONNECT);
649 	exit(exitcode);
650 }
651 
652 /*
653  * Thread receives requests from the primary node.
654  */
655 static void *
656 recv_thread(void *arg)
657 {
658 	struct hast_resource *res = arg;
659 	struct hio *hio, *mshio;
660 	struct nv *nv;
661 
662 	for (;;) {
663 		pjdlog_debug(2, "recv: Taking free request.");
664 		QUEUE_TAKE(free, hio);
665 		pjdlog_debug(2, "recv: (%p) Got request.", hio);
666 		if (hast_proto_recv_hdr(res->hr_remotein, &nv) == -1) {
667 			secondary_exit(EX_TEMPFAIL,
668 			    "Unable to receive request header");
669 		}
670 		if (requnpack(res, hio, nv) != 0) {
671 			nv_free(nv);
672 			pjdlog_debug(2,
673 			    "recv: (%p) Moving request to the send queue.",
674 			    hio);
675 			QUEUE_INSERT(send, hio);
676 			continue;
677 		}
678 		switch (hio->hio_cmd) {
679 		case HIO_READ:
680 			res->hr_stat_read++;
681 			break;
682 		case HIO_WRITE:
683 			res->hr_stat_write++;
684 			break;
685 		case HIO_DELETE:
686 			res->hr_stat_delete++;
687 			break;
688 		case HIO_FLUSH:
689 			res->hr_stat_flush++;
690 			break;
691 		case HIO_KEEPALIVE:
692 			break;
693 		default:
694 			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
695 			    hio->hio_cmd);
696 		}
697 		reqlog(LOG_DEBUG, 2, -1, hio,
698 		    "recv: (%p) Got request header: ", hio);
699 		if (hio->hio_cmd == HIO_KEEPALIVE) {
700 			nv_free(nv);
701 			pjdlog_debug(2,
702 			    "recv: (%p) Moving request to the free queue.",
703 			    hio);
704 			hio_clear(hio);
705 			QUEUE_INSERT(free, hio);
706 			continue;
707 		} else if (hio->hio_cmd == HIO_WRITE) {
708 			if (hast_proto_recv_data(res, res->hr_remotein, nv,
709 			    hio->hio_data, MAXPHYS) == -1) {
710 				secondary_exit(EX_TEMPFAIL,
711 				    "Unable to receive request data");
712 			}
713 			if (hio->hio_memsync) {
714 				/*
715 				 * For memsync requests we expect two replies.
716 				 * Clone the hio so we can handle both of them.
717 				 */
718 				pjdlog_debug(2, "recv: Taking free request.");
719 				QUEUE_TAKE(free, mshio);
720 				pjdlog_debug(2, "recv: (%p) Got request.",
721 				    mshio);
722 				hio_copy(hio, mshio);
723 				mshio->hio_error = 0;
724 				/*
725 				 * We want to keep 'memsync' tag only on the
726 				 * request going onto send queue (mshio).
727 				 */
728 				hio->hio_memsync = false;
729 				pjdlog_debug(2,
730 				    "recv: (%p) Moving memsync request to the send queue.",
731 				    mshio);
732 				QUEUE_INSERT(send, mshio);
733 			}
734 		}
735 		nv_free(nv);
736 		pjdlog_debug(2, "recv: (%p) Moving request to the disk queue.",
737 		    hio);
738 		QUEUE_INSERT(disk, hio);
739 	}
740 	/* NOTREACHED */
741 	return (NULL);
742 }
743 
744 /*
745  * Thread reads from or writes to local component and also handles DELETE and
746  * FLUSH requests.
747  */
748 static void *
749 disk_thread(void *arg)
750 {
751 	struct hast_resource *res = arg;
752 	struct hio *hio;
753 	ssize_t ret;
754 	bool clear_activemap, logerror;
755 
756 	clear_activemap = true;
757 
758 	for (;;) {
759 		pjdlog_debug(2, "disk: Taking request.");
760 		QUEUE_TAKE(disk, hio);
761 		while (clear_activemap) {
762 			unsigned char *map;
763 			size_t mapsize;
764 
765 			/*
766 			 * When first request is received, it means that primary
767 			 * already received our activemap, merged it and stored
768 			 * locally. We can now safely clear our activemap.
769 			 */
770 			mapsize =
771 			    activemap_calc_ondisk_size(res->hr_local_mediasize -
772 			    METADATA_SIZE, res->hr_extentsize,
773 			    res->hr_local_sectorsize);
774 			map = calloc(1, mapsize);
775 			if (map == NULL) {
776 				pjdlog_warning("Unable to allocate memory to clear local activemap.");
777 				break;
778 			}
779 			if (pwrite(res->hr_localfd, map, mapsize,
780 			    METADATA_SIZE) != (ssize_t)mapsize) {
781 				pjdlog_errno(LOG_WARNING,
782 				    "Unable to store cleared activemap");
783 				free(map);
784 				res->hr_stat_activemap_write_error++;
785 				break;
786 			}
787 			free(map);
788 			clear_activemap = false;
789 			pjdlog_debug(1, "Local activemap cleared.");
790 			break;
791 		}
792 		reqlog(LOG_DEBUG, 2, -1, hio, "disk: (%p) Got request: ", hio);
793 		logerror = true;
794 		/* Handle the actual request. */
795 		switch (hio->hio_cmd) {
796 		case HIO_READ:
797 			ret = pread(res->hr_localfd, hio->hio_data,
798 			    hio->hio_length,
799 			    hio->hio_offset + res->hr_localoff);
800 			if (ret == -1)
801 				hio->hio_error = errno;
802 			else if (ret != (int64_t)hio->hio_length)
803 				hio->hio_error = EIO;
804 			else
805 				hio->hio_error = 0;
806 			break;
807 		case HIO_WRITE:
808 			ret = pwrite(res->hr_localfd, hio->hio_data,
809 			    hio->hio_length,
810 			    hio->hio_offset + res->hr_localoff);
811 			if (ret == -1)
812 				hio->hio_error = errno;
813 			else if (ret != (int64_t)hio->hio_length)
814 				hio->hio_error = EIO;
815 			else
816 				hio->hio_error = 0;
817 			break;
818 		case HIO_DELETE:
819 			ret = g_delete(res->hr_localfd,
820 			    hio->hio_offset + res->hr_localoff,
821 			    hio->hio_length);
822 			if (ret == -1)
823 				hio->hio_error = errno;
824 			else
825 				hio->hio_error = 0;
826 			break;
827 		case HIO_FLUSH:
828 			if (!res->hr_localflush) {
829 				ret = -1;
830 				hio->hio_error = EOPNOTSUPP;
831 				logerror = false;
832 				break;
833 			}
834 			ret = g_flush(res->hr_localfd);
835 			if (ret == -1) {
836 				if (errno == EOPNOTSUPP)
837 					res->hr_localflush = false;
838 				hio->hio_error = errno;
839 			} else {
840 				hio->hio_error = 0;
841 			}
842 			break;
843 		default:
844 			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
845 			    hio->hio_cmd);
846 		}
847 		if (logerror && hio->hio_error != 0) {
848 			reqlog(LOG_ERR, 0, hio->hio_error, hio,
849 			    "Request failed: ");
850 		}
851 		pjdlog_debug(2, "disk: (%p) Moving request to the send queue.",
852 		    hio);
853 		QUEUE_INSERT(send, hio);
854 	}
855 	/* NOTREACHED */
856 	return (NULL);
857 }
858 
859 /*
860  * Thread sends requests back to primary node.
861  */
862 static void *
863 send_thread(void *arg)
864 {
865 	struct hast_resource *res = arg;
866 	struct nv *nvout;
867 	struct hio *hio;
868 	void *data;
869 	size_t length;
870 
871 	for (;;) {
872 		pjdlog_debug(2, "send: Taking request.");
873 		QUEUE_TAKE(send, hio);
874 		reqlog(LOG_DEBUG, 2, -1, hio, "send: (%p) Got request: ", hio);
875 		nvout = nv_alloc();
876 		/* Copy sequence number. */
877 		nv_add_uint64(nvout, hio->hio_seq, "seq");
878 		if (hio->hio_memsync) {
879 			PJDLOG_ASSERT(hio->hio_cmd == HIO_WRITE);
880 			nv_add_int8(nvout, 1, "received");
881 		}
882 		switch (hio->hio_cmd) {
883 		case HIO_READ:
884 			if (hio->hio_error == 0) {
885 				data = hio->hio_data;
886 				length = hio->hio_length;
887 				break;
888 			}
889 			/*
890 			 * We send no data in case of an error.
891 			 */
892 			/* FALLTHROUGH */
893 		case HIO_DELETE:
894 		case HIO_FLUSH:
895 		case HIO_WRITE:
896 			data = NULL;
897 			length = 0;
898 			break;
899 		default:
900 			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
901 			    hio->hio_cmd);
902 		}
903 		if (hio->hio_error != 0) {
904 			switch (hio->hio_cmd) {
905 			case HIO_READ:
906 				res->hr_stat_read_error++;
907 				break;
908 			case HIO_WRITE:
909 				res->hr_stat_write_error++;
910 				break;
911 			case HIO_DELETE:
912 				res->hr_stat_delete_error++;
913 				break;
914 			case HIO_FLUSH:
915 				res->hr_stat_flush_error++;
916 				break;
917 			}
918 			nv_add_int16(nvout, hio->hio_error, "error");
919 		}
920 		if (hast_proto_send(res, res->hr_remoteout, nvout, data,
921 		    length) == -1) {
922 			secondary_exit(EX_TEMPFAIL, "Unable to send reply");
923 		}
924 		nv_free(nvout);
925 		pjdlog_debug(2, "send: (%p) Moving request to the free queue.",
926 		    hio);
927 		hio_clear(hio);
928 		QUEUE_INSERT(free, hio);
929 	}
930 	/* NOTREACHED */
931 	return (NULL);
932 }
933