1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2009-2010 The FreeBSD Foundation
5 *
6 * This software was developed by Pawel Jakub Dawidek under sponsorship from
7 * the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/types.h>
32 #include <sys/wait.h>
33
34 #include <errno.h>
35 #include <pthread.h>
36 #include <signal.h>
37 #include <stdio.h>
38 #include <string.h>
39 #include <unistd.h>
40
41 #include "hast.h"
42 #include "hastd.h"
43 #include "hast_checksum.h"
44 #include "hast_compression.h"
45 #include "hast_proto.h"
46 #include "hooks.h"
47 #include "nv.h"
48 #include "pjdlog.h"
49 #include "proto.h"
50 #include "subr.h"
51
52 #include "control.h"
53
54 void
child_cleanup(struct hast_resource * res)55 child_cleanup(struct hast_resource *res)
56 {
57
58 proto_close(res->hr_ctrl);
59 res->hr_ctrl = NULL;
60 if (res->hr_event != NULL) {
61 proto_close(res->hr_event);
62 res->hr_event = NULL;
63 }
64 if (res->hr_conn != NULL) {
65 proto_close(res->hr_conn);
66 res->hr_conn = NULL;
67 }
68 res->hr_workerpid = 0;
69 }
70
71 static void
control_set_role_common(struct hastd_config * cfg,struct nv * nvout,uint8_t role,struct hast_resource * res,const char * name,unsigned int no)72 control_set_role_common(struct hastd_config *cfg, struct nv *nvout,
73 uint8_t role, struct hast_resource *res, const char *name, unsigned int no)
74 {
75 int oldrole;
76
77 /* Name is always needed. */
78 if (name != NULL)
79 nv_add_string(nvout, name, "resource%u", no);
80
81 if (res == NULL) {
82 PJDLOG_ASSERT(cfg != NULL);
83 PJDLOG_ASSERT(name != NULL);
84
85 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
86 if (strcmp(res->hr_name, name) == 0)
87 break;
88 }
89 if (res == NULL) {
90 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no);
91 return;
92 }
93 }
94 PJDLOG_ASSERT(res != NULL);
95
96 /* Send previous role back. */
97 nv_add_string(nvout, role2str(res->hr_role), "role%u", no);
98
99 /* Nothing changed, return here. */
100 if (role == res->hr_role)
101 return;
102
103 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
104 pjdlog_info("Role changed to %s.", role2str(role));
105
106 /* Change role to the new one. */
107 oldrole = res->hr_role;
108 res->hr_role = role;
109 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
110
111 /*
112 * If previous role was primary or secondary we have to kill process
113 * doing that work.
114 */
115 if (res->hr_workerpid != 0) {
116 if (kill(res->hr_workerpid, SIGTERM) == -1) {
117 pjdlog_errno(LOG_WARNING,
118 "Unable to kill worker process %u",
119 (unsigned int)res->hr_workerpid);
120 } else if (waitpid(res->hr_workerpid, NULL, 0) !=
121 res->hr_workerpid) {
122 pjdlog_errno(LOG_WARNING,
123 "Error while waiting for worker process %u",
124 (unsigned int)res->hr_workerpid);
125 } else {
126 pjdlog_debug(1, "Worker process %u stopped.",
127 (unsigned int)res->hr_workerpid);
128 }
129 child_cleanup(res);
130 }
131
132 /* Start worker process if we are changing to primary. */
133 if (role == HAST_ROLE_PRIMARY)
134 hastd_primary(res);
135 pjdlog_prefix_set("%s", "");
136 hook_exec(res->hr_exec, "role", res->hr_name, role2str(oldrole),
137 role2str(res->hr_role), NULL);
138 }
139
140 void
control_set_role(struct hast_resource * res,uint8_t role)141 control_set_role(struct hast_resource *res, uint8_t role)
142 {
143
144 control_set_role_common(NULL, NULL, role, res, NULL, 0);
145 }
146
147 static void
control_status_worker(struct hast_resource * res,struct nv * nvout,unsigned int no)148 control_status_worker(struct hast_resource *res, struct nv *nvout,
149 unsigned int no)
150 {
151 struct nv *cnvin, *cnvout;
152 const char *str;
153 int error;
154
155 cnvin = NULL;
156
157 /*
158 * Prepare and send command to worker process.
159 */
160 cnvout = nv_alloc();
161 nv_add_uint8(cnvout, CONTROL_STATUS, "cmd");
162 error = nv_error(cnvout);
163 if (error != 0) {
164 pjdlog_common(LOG_ERR, 0, error,
165 "Unable to prepare control header");
166 goto end;
167 }
168 if (hast_proto_send(res, res->hr_ctrl, cnvout, NULL, 0) == -1) {
169 error = errno;
170 pjdlog_errno(LOG_ERR, "Unable to send control header");
171 goto end;
172 }
173
174 /*
175 * Receive response.
176 */
177 if (hast_proto_recv_hdr(res->hr_ctrl, &cnvin) == -1) {
178 error = errno;
179 pjdlog_errno(LOG_ERR, "Unable to receive control header");
180 goto end;
181 }
182
183 error = nv_get_int16(cnvin, "error");
184 if (error != 0)
185 goto end;
186
187 if ((str = nv_get_string(cnvin, "status")) == NULL) {
188 error = ENOENT;
189 pjdlog_errno(LOG_ERR, "Field 'status' is missing.");
190 goto end;
191 }
192 nv_add_string(nvout, str, "status%u", no);
193 nv_add_uint64(nvout, nv_get_uint64(cnvin, "dirty"), "dirty%u", no);
194 nv_add_uint32(nvout, nv_get_uint32(cnvin, "extentsize"),
195 "extentsize%u", no);
196 nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"),
197 "keepdirty%u", no);
198 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"),
199 "stat_read%u", no);
200 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"),
201 "stat_write%u", no);
202 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"),
203 "stat_delete%u", no);
204 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"),
205 "stat_flush%u", no);
206 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"),
207 "stat_activemap_update%u", no);
208 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read_error"),
209 "stat_read_error%u", no);
210 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write_error"),
211 "stat_write_error%u", no);
212 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete_error"),
213 "stat_delete_error%u", no);
214 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush_error"),
215 "stat_flush_error%u", no);
216 nv_add_uint64(nvout, nv_get_uint64(cnvin, "idle_queue_size"),
217 "idle_queue_size%u", no);
218 nv_add_uint64(nvout, nv_get_uint64(cnvin, "local_queue_size"),
219 "local_queue_size%u", no);
220 nv_add_uint64(nvout, nv_get_uint64(cnvin, "send_queue_size"),
221 "send_queue_size%u", no);
222 nv_add_uint64(nvout, nv_get_uint64(cnvin, "recv_queue_size"),
223 "recv_queue_size%u", no);
224 nv_add_uint64(nvout, nv_get_uint64(cnvin, "done_queue_size"),
225 "done_queue_size%u", no);
226 end:
227 if (cnvin != NULL)
228 nv_free(cnvin);
229 if (cnvout != NULL)
230 nv_free(cnvout);
231 if (error != 0)
232 nv_add_int16(nvout, error, "error");
233 }
234
235 static void
control_status(struct hastd_config * cfg,struct nv * nvout,struct hast_resource * res,const char * name,unsigned int no)236 control_status(struct hastd_config *cfg, struct nv *nvout,
237 struct hast_resource *res, const char *name, unsigned int no)
238 {
239
240 PJDLOG_ASSERT(cfg != NULL);
241 PJDLOG_ASSERT(nvout != NULL);
242 PJDLOG_ASSERT(name != NULL);
243
244 /* Name is always needed. */
245 nv_add_string(nvout, name, "resource%u", no);
246
247 if (res == NULL) {
248 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
249 if (strcmp(res->hr_name, name) == 0)
250 break;
251 }
252 if (res == NULL) {
253 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no);
254 return;
255 }
256 }
257 PJDLOG_ASSERT(res != NULL);
258 nv_add_string(nvout, res->hr_provname, "provname%u", no);
259 nv_add_string(nvout, res->hr_localpath, "localpath%u", no);
260 nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no);
261 if (res->hr_sourceaddr[0] != '\0')
262 nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr%u", no);
263 switch (res->hr_replication) {
264 case HAST_REPLICATION_FULLSYNC:
265 nv_add_string(nvout, "fullsync", "replication%u", no);
266 break;
267 case HAST_REPLICATION_MEMSYNC:
268 nv_add_string(nvout, "memsync", "replication%u", no);
269 break;
270 case HAST_REPLICATION_ASYNC:
271 nv_add_string(nvout, "async", "replication%u", no);
272 break;
273 default:
274 nv_add_string(nvout, "unknown", "replication%u", no);
275 break;
276 }
277 nv_add_string(nvout, checksum_name(res->hr_checksum),
278 "checksum%u", no);
279 nv_add_string(nvout, compression_name(res->hr_compression),
280 "compression%u", no);
281 nv_add_string(nvout, role2str(res->hr_role), "role%u", no);
282 nv_add_int32(nvout, res->hr_workerpid, "workerpid%u", no);
283
284 switch (res->hr_role) {
285 case HAST_ROLE_PRIMARY:
286 PJDLOG_ASSERT(res->hr_workerpid != 0);
287 /* FALLTHROUGH */
288 case HAST_ROLE_SECONDARY:
289 if (res->hr_workerpid != 0)
290 break;
291 /* FALLTHROUGH */
292 default:
293 return;
294 }
295
296 /*
297 * If we are here, it means that we have a worker process, which we
298 * want to ask some questions.
299 */
300 control_status_worker(res, nvout, no);
301 }
302
303 static void
control_stop(struct hastd_config * cfg,struct nv * nvout)304 control_stop(struct hastd_config *cfg, struct nv *nvout)
305 {
306 pjdlog_info("Stop message received.");
307 (void)cfg;
308 (void)nvout;
309 sigexit_received = true;
310 }
311
312 void
control_handle(struct hastd_config * cfg)313 control_handle(struct hastd_config *cfg)
314 {
315 struct proto_conn *conn;
316 struct nv *nvin, *nvout;
317 unsigned int ii;
318 const char *str;
319 uint8_t cmd, role;
320 int error;
321
322 if (proto_accept(cfg->hc_controlconn, &conn) == -1) {
323 pjdlog_errno(LOG_ERR, "Unable to accept control connection");
324 return;
325 }
326
327 cfg->hc_controlin = conn;
328 nvin = nvout = NULL;
329 role = HAST_ROLE_UNDEF;
330
331 if (hast_proto_recv_hdr(conn, &nvin) == -1) {
332 pjdlog_errno(LOG_ERR, "Unable to receive control header");
333 nvin = NULL;
334 goto close;
335 }
336
337 /* Obtain command code. 0 means that nv_get_uint8() failed. */
338 cmd = nv_get_uint8(nvin, "cmd");
339 if (cmd == 0) {
340 pjdlog_error("Control header is missing 'cmd' field.");
341 goto close;
342 }
343
344 /* Allocate outgoing nv structure. */
345 nvout = nv_alloc();
346 if (nvout == NULL) {
347 pjdlog_error("Unable to allocate header for control response.");
348 goto close;
349 }
350
351 error = 0;
352
353 /* The stop command does not expect any arguments. */
354 if (cmd == HASTCTL_CMD_STOP) {
355 control_stop(cfg, nvout);
356 if (nv_error(nvout) != 0)
357 goto close;
358 goto fail;
359 }
360
361 str = nv_get_string(nvin, "resource0");
362 if (str == NULL) {
363 pjdlog_error("Control header is missing 'resource0' field.");
364 error = EHAST_INVALID;
365 goto fail;
366 }
367 if (cmd == HASTCTL_CMD_SETROLE) {
368 role = nv_get_uint8(nvin, "role");
369 switch (role) {
370 case HAST_ROLE_INIT:
371 case HAST_ROLE_PRIMARY:
372 case HAST_ROLE_SECONDARY:
373 break;
374 default:
375 pjdlog_error("Invalid role received (%hhu).", role);
376 error = EHAST_INVALID;
377 goto fail;
378 }
379 }
380 if (strcmp(str, "all") == 0) {
381 struct hast_resource *res;
382
383 /* All configured resources. */
384
385 ii = 0;
386 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
387 switch (cmd) {
388 case HASTCTL_CMD_SETROLE:
389 control_set_role_common(cfg, nvout, role, res,
390 res->hr_name, ii++);
391 break;
392 case HASTCTL_CMD_STATUS:
393 control_status(cfg, nvout, res, res->hr_name,
394 ii++);
395 break;
396 default:
397 pjdlog_error("Invalid command received (%hhu).",
398 cmd);
399 error = EHAST_UNIMPLEMENTED;
400 goto fail;
401 }
402 }
403 } else {
404 /* Only selected resources. */
405
406 for (ii = 0; ; ii++) {
407 str = nv_get_string(nvin, "resource%u", ii);
408 if (str == NULL)
409 break;
410 switch (cmd) {
411 case HASTCTL_CMD_SETROLE:
412 control_set_role_common(cfg, nvout, role, NULL,
413 str, ii);
414 break;
415 case HASTCTL_CMD_STATUS:
416 control_status(cfg, nvout, NULL, str, ii);
417 break;
418 default:
419 pjdlog_error("Invalid command received (%hhu).",
420 cmd);
421 error = EHAST_UNIMPLEMENTED;
422 goto fail;
423 }
424 }
425 }
426 if (nv_error(nvout) != 0)
427 goto close;
428 fail:
429 if (error != 0)
430 nv_add_int16(nvout, error, "error");
431
432 if (hast_proto_send(NULL, conn, nvout, NULL, 0) == -1)
433 pjdlog_errno(LOG_ERR, "Unable to send control response");
434 close:
435 if (nvin != NULL)
436 nv_free(nvin);
437 if (nvout != NULL)
438 nv_free(nvout);
439 proto_close(conn);
440 cfg->hc_controlin = NULL;
441 }
442
443 /*
444 * Thread handles control requests from the parent.
445 */
446 void *
ctrl_thread(void * arg)447 ctrl_thread(void *arg)
448 {
449 struct hast_resource *res = arg;
450 struct nv *nvin, *nvout;
451 uint8_t cmd;
452
453 for (;;) {
454 if (hast_proto_recv_hdr(res->hr_ctrl, &nvin) == -1) {
455 if (sigexit_received)
456 pthread_exit(NULL);
457 pjdlog_errno(LOG_ERR,
458 "Unable to receive control message");
459 kill(getpid(), SIGTERM);
460 pthread_exit(NULL);
461 }
462 cmd = nv_get_uint8(nvin, "cmd");
463 if (cmd == 0) {
464 pjdlog_error("Control message is missing 'cmd' field.");
465 nv_free(nvin);
466 continue;
467 }
468 nvout = nv_alloc();
469 switch (cmd) {
470 case CONTROL_STATUS:
471 if (res->hr_remotein != NULL &&
472 res->hr_remoteout != NULL) {
473 nv_add_string(nvout, "complete", "status");
474 } else {
475 nv_add_string(nvout, "degraded", "status");
476 }
477 nv_add_uint32(nvout, (uint32_t)res->hr_extentsize,
478 "extentsize");
479 if (res->hr_role == HAST_ROLE_PRIMARY) {
480 nv_add_uint32(nvout,
481 (uint32_t)res->hr_keepdirty, "keepdirty");
482 nv_add_uint64(nvout,
483 (uint64_t)(activemap_ndirty(res->hr_amp) *
484 res->hr_extentsize), "dirty");
485 } else {
486 nv_add_uint32(nvout, (uint32_t)0, "keepdirty");
487 nv_add_uint64(nvout, (uint64_t)0, "dirty");
488 }
489 nv_add_uint64(nvout, res->hr_stat_read, "stat_read");
490 nv_add_uint64(nvout, res->hr_stat_write, "stat_write");
491 nv_add_uint64(nvout, res->hr_stat_delete,
492 "stat_delete");
493 nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush");
494 nv_add_uint64(nvout, res->hr_stat_activemap_update,
495 "stat_activemap_update");
496 nv_add_uint64(nvout, res->hr_stat_read_error,
497 "stat_read_error");
498 nv_add_uint64(nvout, res->hr_stat_write_error +
499 res->hr_stat_activemap_write_error,
500 "stat_write_error");
501 nv_add_uint64(nvout, res->hr_stat_delete_error,
502 "stat_delete_error");
503 nv_add_uint64(nvout, res->hr_stat_flush_error +
504 res->hr_stat_activemap_flush_error,
505 "stat_flush_error");
506 res->output_status_aux(nvout);
507 nv_add_int16(nvout, 0, "error");
508 break;
509 case CONTROL_RELOAD:
510 /*
511 * When parent receives SIGHUP and discovers that
512 * something related to us has changes, it sends reload
513 * message to us.
514 */
515 PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY);
516 primary_config_reload(res, nvin);
517 nv_add_int16(nvout, 0, "error");
518 break;
519 default:
520 nv_add_int16(nvout, EINVAL, "error");
521 break;
522 }
523 nv_free(nvin);
524 if (nv_error(nvout) != 0) {
525 pjdlog_error("Unable to create answer on control message.");
526 nv_free(nvout);
527 continue;
528 }
529 if (hast_proto_send(NULL, res->hr_ctrl, nvout, NULL, 0) == -1) {
530 pjdlog_errno(LOG_ERR,
531 "Unable to send reply to control message");
532 }
533 nv_free(nvout);
534 }
535 /* NOTREACHED */
536 return (NULL);
537 }
538