xref: /freebsd/sbin/hastd/hastd.c (revision 4f1f4356f3012928b463f9ef1710fb908e48b1e2)
1 /*-
2  * Copyright (c) 2009-2010 The FreeBSD Foundation
3  * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
4  * All rights reserved.
5  *
6  * This software was developed by Pawel Jakub Dawidek under sponsorship from
7  * the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/linker.h>
36 #include <sys/module.h>
37 #include <sys/wait.h>
38 
39 #include <assert.h>
40 #include <err.h>
41 #include <errno.h>
42 #include <libutil.h>
43 #include <signal.h>
44 #include <stdbool.h>
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <string.h>
48 #include <sysexits.h>
49 #include <unistd.h>
50 
51 #include <activemap.h>
52 #include <pjdlog.h>
53 
54 #include "control.h"
55 #include "hast.h"
56 #include "hast_proto.h"
57 #include "hastd.h"
58 #include "subr.h"
59 
60 /* Path to configuration file. */
61 const char *cfgpath = HAST_CONFIG;
62 /* Hastd configuration. */
63 static struct hastd_config *cfg;
64 /* Was SIGCHLD signal received? */
65 bool sigchld_received = false;
66 /* Was SIGHUP signal received? */
67 bool sighup_received = false;
68 /* Was SIGINT or SIGTERM signal received? */
69 bool sigexit_received = false;
70 /* PID file handle. */
71 struct pidfh *pfh;
72 
73 static void
74 usage(void)
75 {
76 
77 	errx(EX_USAGE, "[-dFh] [-c config] [-P pidfile]");
78 }
79 
80 static void
81 sighandler(int sig)
82 {
83 
84 	switch (sig) {
85 	case SIGINT:
86 	case SIGTERM:
87 		sigexit_received = true;
88 		break;
89 	case SIGCHLD:
90 		sigchld_received = true;
91 		break;
92 	case SIGHUP:
93 		sighup_received = true;
94 		break;
95 	default:
96 		assert(!"invalid condition");
97 	}
98 }
99 
100 static void
101 g_gate_load(void)
102 {
103 
104 	if (modfind("g_gate") == -1) {
105 		/* Not present in kernel, try loading it. */
106 		if (kldload("geom_gate") == -1 || modfind("g_gate") == -1) {
107 			if (errno != EEXIST) {
108 				pjdlog_exit(EX_OSERR,
109 				    "Unable to load geom_gate module");
110 			}
111 		}
112 	}
113 }
114 
115 static void
116 child_exit_log(unsigned int pid, int status)
117 {
118 
119 	if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
120 		pjdlog_debug(1, "Worker process exited gracefully (pid=%u).",
121 		    pid);
122 	} else if (WIFSIGNALED(status)) {
123 		pjdlog_error("Worker process killed (pid=%u, signal=%d).",
124 		    pid, WTERMSIG(status));
125 	} else {
126 		pjdlog_error("Worker process exited ungracefully (pid=%u, exitcode=%d).",
127 		    pid, WIFEXITED(status) ? WEXITSTATUS(status) : -1);
128 	}
129 }
130 
131 static void
132 child_exit(void)
133 {
134 	struct hast_resource *res;
135 	int status;
136 	pid_t pid;
137 
138 	while ((pid = wait3(&status, WNOHANG, NULL)) > 0) {
139 		/* Find resource related to the process that just exited. */
140 		TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
141 			if (pid == res->hr_workerpid)
142 				break;
143 		}
144 		if (res == NULL) {
145 			/*
146 			 * This can happen when new connection arrives and we
147 			 * cancel child responsible for the old one.
148 			 */
149 			continue;
150 		}
151 		pjdlog_prefix_set("[%s] (%s) ", res->hr_name,
152 		    role2str(res->hr_role));
153 		child_exit_log(pid, status);
154 		proto_close(res->hr_ctrl);
155 		res->hr_workerpid = 0;
156 		if (res->hr_role == HAST_ROLE_PRIMARY) {
157 			/*
158 			 * Restart child process if it was killed by signal
159 			 * or exited because of temporary problem.
160 			 */
161 			if (WIFSIGNALED(status) ||
162 			    (WIFEXITED(status) &&
163 			     WEXITSTATUS(status) == EX_TEMPFAIL)) {
164 				sleep(1);
165 				pjdlog_info("Restarting worker process.");
166 				hastd_primary(res);
167 			} else {
168 				res->hr_role = HAST_ROLE_INIT;
169 				pjdlog_info("Changing resource role back to %s.",
170 				    role2str(res->hr_role));
171 			}
172 		}
173 		pjdlog_prefix_set("%s", "");
174 	}
175 }
176 
177 static bool
178 resource_needs_restart(const struct hast_resource *res0,
179     const struct hast_resource *res1)
180 {
181 
182 	assert(strcmp(res0->hr_name, res1->hr_name) == 0);
183 
184 	if (strcmp(res0->hr_provname, res1->hr_provname) != 0)
185 		return (true);
186 	if (strcmp(res0->hr_localpath, res1->hr_localpath) != 0)
187 		return (true);
188 	if (res0->hr_role == HAST_ROLE_INIT ||
189 	    res0->hr_role == HAST_ROLE_SECONDARY) {
190 		if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0)
191 			return (true);
192 		if (res0->hr_replication != res1->hr_replication)
193 			return (true);
194 		if (res0->hr_timeout != res1->hr_timeout)
195 			return (true);
196 		if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
197 			return (true);
198 	}
199 	return (false);
200 }
201 
202 static bool
203 resource_needs_reload(const struct hast_resource *res0,
204     const struct hast_resource *res1)
205 {
206 
207 	assert(strcmp(res0->hr_name, res1->hr_name) == 0);
208 	assert(strcmp(res0->hr_provname, res1->hr_provname) == 0);
209 	assert(strcmp(res0->hr_localpath, res1->hr_localpath) == 0);
210 
211 	if (res0->hr_role != HAST_ROLE_PRIMARY)
212 		return (false);
213 
214 	if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0)
215 		return (true);
216 	if (res0->hr_replication != res1->hr_replication)
217 		return (true);
218 	if (res0->hr_timeout != res1->hr_timeout)
219 		return (true);
220 	if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
221 		return (true);
222 	return (false);
223 }
224 
225 static void
226 hastd_reload(void)
227 {
228 	struct hastd_config *newcfg;
229 	struct hast_resource *nres, *cres, *tres;
230 	uint8_t role;
231 
232 	pjdlog_info("Reloading configuration...");
233 
234 	newcfg = yy_config_parse(cfgpath, false);
235 	if (newcfg == NULL)
236 		goto failed;
237 
238 	/*
239 	 * Check if control address has changed.
240 	 */
241 	if (strcmp(cfg->hc_controladdr, newcfg->hc_controladdr) != 0) {
242 		if (proto_server(newcfg->hc_controladdr,
243 		    &newcfg->hc_controlconn) < 0) {
244 			pjdlog_errno(LOG_ERR,
245 			    "Unable to listen on control address %s",
246 			    newcfg->hc_controladdr);
247 			goto failed;
248 		}
249 	}
250 	/*
251 	 * Check if listen address has changed.
252 	 */
253 	if (strcmp(cfg->hc_listenaddr, newcfg->hc_listenaddr) != 0) {
254 		if (proto_server(newcfg->hc_listenaddr,
255 		    &newcfg->hc_listenconn) < 0) {
256 			pjdlog_errno(LOG_ERR, "Unable to listen on address %s",
257 			    newcfg->hc_listenaddr);
258 			goto failed;
259 		}
260 	}
261 	/*
262 	 * Only when both control and listen sockets are successfully
263 	 * initialized switch them to new configuration.
264 	 */
265 	if (newcfg->hc_controlconn != NULL) {
266 		pjdlog_info("Control socket changed from %s to %s.",
267 		    cfg->hc_controladdr, newcfg->hc_controladdr);
268 		proto_close(cfg->hc_controlconn);
269 		cfg->hc_controlconn = newcfg->hc_controlconn;
270 		newcfg->hc_controlconn = NULL;
271 		strlcpy(cfg->hc_controladdr, newcfg->hc_controladdr,
272 		    sizeof(cfg->hc_controladdr));
273 	}
274 	if (newcfg->hc_listenconn != NULL) {
275 		pjdlog_info("Listen socket changed from %s to %s.",
276 		    cfg->hc_listenaddr, newcfg->hc_listenaddr);
277 		proto_close(cfg->hc_listenconn);
278 		cfg->hc_listenconn = newcfg->hc_listenconn;
279 		newcfg->hc_listenconn = NULL;
280 		strlcpy(cfg->hc_listenaddr, newcfg->hc_listenaddr,
281 		    sizeof(cfg->hc_listenaddr));
282 	}
283 
284 	/*
285 	 * Stop and remove resources that were removed from the configuration.
286 	 */
287 	TAILQ_FOREACH_SAFE(cres, &cfg->hc_resources, hr_next, tres) {
288 		TAILQ_FOREACH(nres, &newcfg->hc_resources, hr_next) {
289 			if (strcmp(cres->hr_name, nres->hr_name) == 0)
290 				break;
291 		}
292 		if (nres == NULL) {
293 			control_set_role(cres, HAST_ROLE_INIT);
294 			TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next);
295 			pjdlog_info("Resource %s removed.", cres->hr_name);
296 			free(cres);
297 		}
298 	}
299 	/*
300 	 * Move new resources to the current configuration.
301 	 */
302 	TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) {
303 		TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) {
304 			if (strcmp(cres->hr_name, nres->hr_name) == 0)
305 				break;
306 		}
307 		if (cres == NULL) {
308 			TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next);
309 			TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next);
310 			pjdlog_info("Resource %s added.", nres->hr_name);
311 		}
312 	}
313 	/*
314 	 * Deal with modified resources.
315 	 * Depending on what has changed exactly we might want to perform
316 	 * different actions.
317 	 *
318 	 * We do full resource restart in the following situations:
319 	 * Resource role is INIT or SECONDARY.
320 	 * Resource role is PRIMARY and path to local component or provider
321 	 * name has changed.
322 	 * In case of PRIMARY, the worker process will be killed and restarted,
323 	 * which also means removing /dev/hast/<name> provider and
324 	 * recreating it.
325 	 *
326 	 * We do just reload (send SIGHUP to worker process) if we act as
327 	 * PRIMARY, but only remote address, replication mode and timeout
328 	 * has changed. For those, there is no need to restart worker process.
329 	 * If PRIMARY receives SIGHUP, it will reconnect if remote address or
330 	 * replication mode has changed or simply set new timeout if only
331 	 * timeout has changed.
332 	 */
333 	TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) {
334 		TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) {
335 			if (strcmp(cres->hr_name, nres->hr_name) == 0)
336 				break;
337 		}
338 		assert(cres != NULL);
339 		if (resource_needs_restart(cres, nres)) {
340 			pjdlog_info("Resource %s configuration was modified, restarting it.",
341 			    cres->hr_name);
342 			role = cres->hr_role;
343 			control_set_role(cres, HAST_ROLE_INIT);
344 			TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next);
345 			free(cres);
346 			TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next);
347 			TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next);
348 			control_set_role(nres, role);
349 		} else if (resource_needs_reload(cres, nres)) {
350 			pjdlog_info("Resource %s configuration was modified, reloading it.",
351 			    cres->hr_name);
352 			strlcpy(cres->hr_remoteaddr, nres->hr_remoteaddr,
353 			    sizeof(cres->hr_remoteaddr));
354 			cres->hr_replication = nres->hr_replication;
355 			cres->hr_timeout = nres->hr_timeout;
356 			if (cres->hr_workerpid != 0) {
357 				if (kill(cres->hr_workerpid, SIGHUP) < 0) {
358 					pjdlog_errno(LOG_WARNING,
359 					    "Unable to send SIGHUP to worker process %u",
360 					    (unsigned int)cres->hr_workerpid);
361 				}
362 			}
363 		}
364 	}
365 
366 	yy_config_free(newcfg);
367 	pjdlog_info("Configuration reloaded successfully.");
368 	return;
369 failed:
370 	if (newcfg != NULL) {
371 		if (newcfg->hc_controlconn != NULL)
372 			proto_close(newcfg->hc_controlconn);
373 		if (newcfg->hc_listenconn != NULL)
374 			proto_close(newcfg->hc_listenconn);
375 		yy_config_free(newcfg);
376 	}
377 	pjdlog_warning("Configuration not reloaded.");
378 }
379 
380 static void
381 terminate_workers(void)
382 {
383 	struct hast_resource *res;
384 
385 	pjdlog_info("Termination signal received, exiting.");
386 	TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
387 		if (res->hr_workerpid == 0)
388 			continue;
389 		pjdlog_info("Terminating worker process (resource=%s, role=%s, pid=%u).",
390 		    res->hr_name, role2str(res->hr_role), res->hr_workerpid);
391 		if (kill(res->hr_workerpid, SIGTERM) == 0)
392 			continue;
393 		pjdlog_errno(LOG_WARNING,
394 		    "Unable to send signal to worker process (resource=%s, role=%s, pid=%u).",
395 		    res->hr_name, role2str(res->hr_role), res->hr_workerpid);
396 	}
397 }
398 
399 static void
400 listen_accept(void)
401 {
402 	struct hast_resource *res;
403 	struct proto_conn *conn;
404 	struct nv *nvin, *nvout, *nverr;
405 	const char *resname;
406 	const unsigned char *token;
407 	char laddr[256], raddr[256];
408 	size_t size;
409 	pid_t pid;
410 	int status;
411 
412 	proto_local_address(cfg->hc_listenconn, laddr, sizeof(laddr));
413 	pjdlog_debug(1, "Accepting connection to %s.", laddr);
414 
415 	if (proto_accept(cfg->hc_listenconn, &conn) < 0) {
416 		pjdlog_errno(LOG_ERR, "Unable to accept connection %s", laddr);
417 		return;
418 	}
419 
420 	proto_local_address(conn, laddr, sizeof(laddr));
421 	proto_remote_address(conn, raddr, sizeof(raddr));
422 	pjdlog_info("Connection from %s to %s.", raddr, laddr);
423 
424 	/* Error in setting timeout is not critical, but why should it fail? */
425 	if (proto_timeout(conn, HAST_TIMEOUT) < 0)
426 		pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
427 
428 	nvin = nvout = nverr = NULL;
429 
430 	/*
431 	 * Before receiving any data see if remote host have access to any
432 	 * resource.
433 	 */
434 	TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
435 		if (proto_address_match(conn, res->hr_remoteaddr))
436 			break;
437 	}
438 	if (res == NULL) {
439 		pjdlog_error("Client %s isn't known.", raddr);
440 		goto close;
441 	}
442 	/* Ok, remote host can access at least one resource. */
443 
444 	if (hast_proto_recv_hdr(conn, &nvin) < 0) {
445 		pjdlog_errno(LOG_ERR, "Unable to receive header from %s",
446 		    raddr);
447 		goto close;
448 	}
449 
450 	resname = nv_get_string(nvin, "resource");
451 	if (resname == NULL) {
452 		pjdlog_error("No 'resource' field in the header received from %s.",
453 		    raddr);
454 		goto close;
455 	}
456 	pjdlog_debug(2, "%s: resource=%s", raddr, resname);
457 	token = nv_get_uint8_array(nvin, &size, "token");
458 	/*
459 	 * NULL token means that this is first conection.
460 	 */
461 	if (token != NULL && size != sizeof(res->hr_token)) {
462 		pjdlog_error("Received token of invalid size from %s (expected %zu, got %zu).",
463 		    raddr, sizeof(res->hr_token), size);
464 		goto close;
465 	}
466 
467 	/*
468 	 * From now on we want to send errors to the remote node.
469 	 */
470 	nverr = nv_alloc();
471 
472 	/* Find resource related to this connection. */
473 	TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
474 		if (strcmp(resname, res->hr_name) == 0)
475 			break;
476 	}
477 	/* Have we found the resource? */
478 	if (res == NULL) {
479 		pjdlog_error("No resource '%s' as requested by %s.",
480 		    resname, raddr);
481 		nv_add_stringf(nverr, "errmsg", "Resource not configured.");
482 		goto fail;
483 	}
484 
485 	/* Now that we know resource name setup log prefix. */
486 	pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
487 
488 	/* Does the remote host have access to this resource? */
489 	if (!proto_address_match(conn, res->hr_remoteaddr)) {
490 		pjdlog_error("Client %s has no access to the resource.", raddr);
491 		nv_add_stringf(nverr, "errmsg", "No access to the resource.");
492 		goto fail;
493 	}
494 	/* Is the resource marked as secondary? */
495 	if (res->hr_role != HAST_ROLE_SECONDARY) {
496 		pjdlog_error("We act as %s for the resource and not as %s as requested by %s.",
497 		    role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY),
498 		    raddr);
499 		nv_add_stringf(nverr, "errmsg",
500 		    "Remote node acts as %s for the resource and not as %s.",
501 		    role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY));
502 		goto fail;
503 	}
504 	/* Does token (if exists) match? */
505 	if (token != NULL && memcmp(token, res->hr_token,
506 	    sizeof(res->hr_token)) != 0) {
507 		pjdlog_error("Token received from %s doesn't match.", raddr);
508 		nv_add_stringf(nverr, "errmsg", "Token doesn't match.");
509 		goto fail;
510 	}
511 	/*
512 	 * If there is no token, but we have half-open connection
513 	 * (only remotein) or full connection (worker process is running)
514 	 * we have to cancel those and accept the new connection.
515 	 */
516 	if (token == NULL) {
517 		assert(res->hr_remoteout == NULL);
518 		pjdlog_debug(1, "Initial connection from %s.", raddr);
519 		if (res->hr_workerpid != 0) {
520 			assert(res->hr_remotein == NULL);
521 			pjdlog_debug(1,
522 			    "Worker process exists (pid=%u), stopping it.",
523 			    (unsigned int)res->hr_workerpid);
524 			/* Stop child process. */
525 			if (kill(res->hr_workerpid, SIGINT) < 0) {
526 				pjdlog_errno(LOG_ERR,
527 				    "Unable to stop worker process (pid=%u)",
528 				    (unsigned int)res->hr_workerpid);
529 				/*
530 				 * Other than logging the problem we
531 				 * ignore it - nothing smart to do.
532 				 */
533 			}
534 			/* Wait for it to exit. */
535 			else if ((pid = waitpid(res->hr_workerpid,
536 			    &status, 0)) != res->hr_workerpid) {
537 				/* We can only log the problem. */
538 				pjdlog_errno(LOG_ERR,
539 				    "Waiting for worker process (pid=%u) failed",
540 				    (unsigned int)res->hr_workerpid);
541 			} else {
542 				child_exit_log(res->hr_workerpid, status);
543 			}
544 			res->hr_workerpid = 0;
545 		} else if (res->hr_remotein != NULL) {
546 			char oaddr[256];
547 
548 			proto_remote_address(conn, oaddr, sizeof(oaddr));
549 			pjdlog_debug(1,
550 			    "Canceling half-open connection from %s on connection from %s.",
551 			    oaddr, raddr);
552 			proto_close(res->hr_remotein);
553 			res->hr_remotein = NULL;
554 		}
555 	}
556 
557 	/*
558 	 * Checks and cleanups are done.
559 	 */
560 
561 	if (token == NULL) {
562 		arc4random_buf(res->hr_token, sizeof(res->hr_token));
563 		nvout = nv_alloc();
564 		nv_add_uint8_array(nvout, res->hr_token,
565 		    sizeof(res->hr_token), "token");
566 		if (nv_error(nvout) != 0) {
567 			pjdlog_common(LOG_ERR, 0, nv_error(nvout),
568 			    "Unable to prepare return header for %s", raddr);
569 			nv_add_stringf(nverr, "errmsg",
570 			    "Remote node was unable to prepare return header: %s.",
571 			    strerror(nv_error(nvout)));
572 			goto fail;
573 		}
574 		if (hast_proto_send(NULL, conn, nvout, NULL, 0) < 0) {
575 			int error = errno;
576 
577 			pjdlog_errno(LOG_ERR, "Unable to send response to %s",
578 			    raddr);
579 			nv_add_stringf(nverr, "errmsg",
580 			    "Remote node was unable to send response: %s.",
581 			    strerror(error));
582 			goto fail;
583 		}
584 		res->hr_remotein = conn;
585 		pjdlog_debug(1, "Incoming connection from %s configured.",
586 		    raddr);
587 	} else {
588 		res->hr_remoteout = conn;
589 		pjdlog_debug(1, "Outgoing connection to %s configured.", raddr);
590 		hastd_secondary(res, nvin);
591 	}
592 	nv_free(nvin);
593 	nv_free(nvout);
594 	nv_free(nverr);
595 	pjdlog_prefix_set("%s", "");
596 	return;
597 fail:
598 	if (nv_error(nverr) != 0) {
599 		pjdlog_common(LOG_ERR, 0, nv_error(nverr),
600 		    "Unable to prepare error header for %s", raddr);
601 		goto close;
602 	}
603 	if (hast_proto_send(NULL, conn, nverr, NULL, 0) < 0) {
604 		pjdlog_errno(LOG_ERR, "Unable to send error to %s", raddr);
605 		goto close;
606 	}
607 close:
608 	if (nvin != NULL)
609 		nv_free(nvin);
610 	if (nvout != NULL)
611 		nv_free(nvout);
612 	if (nverr != NULL)
613 		nv_free(nverr);
614 	proto_close(conn);
615 	pjdlog_prefix_set("%s", "");
616 }
617 
618 static void
619 main_loop(void)
620 {
621 	fd_set rfds, wfds;
622 	int cfd, lfd, maxfd, ret;
623 
624 	for (;;) {
625 		if (sigexit_received) {
626 			sigexit_received = false;
627 			terminate_workers();
628 			exit(EX_OK);
629 		}
630 		if (sigchld_received) {
631 			sigchld_received = false;
632 			child_exit();
633 		}
634 		if (sighup_received) {
635 			sighup_received = false;
636 			hastd_reload();
637 		}
638 
639 		cfd = proto_descriptor(cfg->hc_controlconn);
640 		lfd = proto_descriptor(cfg->hc_listenconn);
641 		maxfd = cfd > lfd ? cfd : lfd;
642 
643 		/* Setup descriptors for select(2). */
644 		FD_ZERO(&rfds);
645 		FD_SET(cfd, &rfds);
646 		FD_SET(lfd, &rfds);
647 		FD_ZERO(&wfds);
648 		FD_SET(cfd, &wfds);
649 		FD_SET(lfd, &wfds);
650 
651 		ret = select(maxfd + 1, &rfds, &wfds, NULL, NULL);
652 		if (ret == -1) {
653 			if (errno == EINTR)
654 				continue;
655 			KEEP_ERRNO((void)pidfile_remove(pfh));
656 			pjdlog_exit(EX_OSERR, "select() failed");
657 		}
658 
659 		if (FD_ISSET(cfd, &rfds) || FD_ISSET(cfd, &wfds))
660 			control_handle(cfg);
661 		if (FD_ISSET(lfd, &rfds) || FD_ISSET(lfd, &wfds))
662 			listen_accept();
663 	}
664 }
665 
666 int
667 main(int argc, char *argv[])
668 {
669 	const char *pidfile;
670 	pid_t otherpid;
671 	bool foreground;
672 	int debuglevel;
673 
674 	g_gate_load();
675 
676 	foreground = false;
677 	debuglevel = 0;
678 	pidfile = HASTD_PIDFILE;
679 
680 	for (;;) {
681 		int ch;
682 
683 		ch = getopt(argc, argv, "c:dFhP:");
684 		if (ch == -1)
685 			break;
686 		switch (ch) {
687 		case 'c':
688 			cfgpath = optarg;
689 			break;
690 		case 'd':
691 			debuglevel++;
692 			break;
693 		case 'F':
694 			foreground = true;
695 			break;
696 		case 'P':
697 			pidfile = optarg;
698 			break;
699 		case 'h':
700 		default:
701 			usage();
702 		}
703 	}
704 	argc -= optind;
705 	argv += optind;
706 
707 	pjdlog_debug_set(debuglevel);
708 
709 	pfh = pidfile_open(pidfile, 0600, &otherpid);
710 	if (pfh == NULL) {
711 		if (errno == EEXIST) {
712 			pjdlog_exitx(EX_TEMPFAIL,
713 			    "Another hastd is already running, pid: %jd.",
714 			    (intmax_t)otherpid);
715 		}
716 		/* If we cannot create pidfile from other reasons, only warn. */
717 		pjdlog_errno(LOG_WARNING, "Unable to open or create pidfile");
718 	}
719 
720 	cfg = yy_config_parse(cfgpath, true);
721 	assert(cfg != NULL);
722 
723 	signal(SIGINT, sighandler);
724 	signal(SIGTERM, sighandler);
725 	signal(SIGHUP, sighandler);
726 	signal(SIGCHLD, sighandler);
727 
728 	/* Listen on control address. */
729 	if (proto_server(cfg->hc_controladdr, &cfg->hc_controlconn) < 0) {
730 		KEEP_ERRNO((void)pidfile_remove(pfh));
731 		pjdlog_exit(EX_OSERR, "Unable to listen on control address %s",
732 		    cfg->hc_controladdr);
733 	}
734 	/* Listen for remote connections. */
735 	if (proto_server(cfg->hc_listenaddr, &cfg->hc_listenconn) < 0) {
736 		KEEP_ERRNO((void)pidfile_remove(pfh));
737 		pjdlog_exit(EX_OSERR, "Unable to listen on address %s",
738 		    cfg->hc_listenaddr);
739 	}
740 
741 	if (!foreground) {
742 		if (daemon(0, 0) < 0) {
743 			KEEP_ERRNO((void)pidfile_remove(pfh));
744 			pjdlog_exit(EX_OSERR, "Unable to daemonize");
745 		}
746 
747 		/* Start logging to syslog. */
748 		pjdlog_mode_set(PJDLOG_MODE_SYSLOG);
749 
750 		/* Write PID to a file. */
751 		if (pidfile_write(pfh) < 0) {
752 			pjdlog_errno(LOG_WARNING,
753 			    "Unable to write PID to a file");
754 		}
755 	}
756 
757 	main_loop();
758 
759 	exit(0);
760 }
761