1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2009-2010 The FreeBSD Foundation 5 * 6 * This software was developed by Pawel Jakub Dawidek under sponsorship from 7 * the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/wait.h> 33 34 #include <errno.h> 35 #include <pthread.h> 36 #include <signal.h> 37 #include <stdio.h> 38 #include <string.h> 39 #include <unistd.h> 40 41 #include "hast.h" 42 #include "hastd.h" 43 #include "hast_checksum.h" 44 #include "hast_compression.h" 45 #include "hast_proto.h" 46 #include "hooks.h" 47 #include "nv.h" 48 #include "pjdlog.h" 49 #include "proto.h" 50 #include "subr.h" 51 52 #include "control.h" 53 54 void 55 child_cleanup(struct hast_resource *res) 56 { 57 58 proto_close(res->hr_ctrl); 59 res->hr_ctrl = NULL; 60 if (res->hr_event != NULL) { 61 proto_close(res->hr_event); 62 res->hr_event = NULL; 63 } 64 if (res->hr_conn != NULL) { 65 proto_close(res->hr_conn); 66 res->hr_conn = NULL; 67 } 68 res->hr_workerpid = 0; 69 } 70 71 static void 72 control_set_role_common(struct hastd_config *cfg, struct nv *nvout, 73 uint8_t role, struct hast_resource *res, const char *name, unsigned int no) 74 { 75 int oldrole; 76 77 /* Name is always needed. */ 78 if (name != NULL) 79 nv_add_string(nvout, name, "resource%u", no); 80 81 if (res == NULL) { 82 PJDLOG_ASSERT(cfg != NULL); 83 PJDLOG_ASSERT(name != NULL); 84 85 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 86 if (strcmp(res->hr_name, name) == 0) 87 break; 88 } 89 if (res == NULL) { 90 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no); 91 return; 92 } 93 } 94 PJDLOG_ASSERT(res != NULL); 95 96 /* Send previous role back. */ 97 nv_add_string(nvout, role2str(res->hr_role), "role%u", no); 98 99 /* Nothing changed, return here. */ 100 if (role == res->hr_role) 101 return; 102 103 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 104 pjdlog_info("Role changed to %s.", role2str(role)); 105 106 /* Change role to the new one. */ 107 oldrole = res->hr_role; 108 res->hr_role = role; 109 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 110 111 /* 112 * If previous role was primary or secondary we have to kill process 113 * doing that work. 114 */ 115 if (res->hr_workerpid != 0) { 116 if (kill(res->hr_workerpid, SIGTERM) == -1) { 117 pjdlog_errno(LOG_WARNING, 118 "Unable to kill worker process %u", 119 (unsigned int)res->hr_workerpid); 120 } else if (waitpid(res->hr_workerpid, NULL, 0) != 121 res->hr_workerpid) { 122 pjdlog_errno(LOG_WARNING, 123 "Error while waiting for worker process %u", 124 (unsigned int)res->hr_workerpid); 125 } else { 126 pjdlog_debug(1, "Worker process %u stopped.", 127 (unsigned int)res->hr_workerpid); 128 } 129 child_cleanup(res); 130 } 131 132 /* Start worker process if we are changing to primary. */ 133 if (role == HAST_ROLE_PRIMARY) 134 hastd_primary(res); 135 pjdlog_prefix_set("%s", ""); 136 hook_exec(res->hr_exec, "role", res->hr_name, role2str(oldrole), 137 role2str(res->hr_role), NULL); 138 } 139 140 void 141 control_set_role(struct hast_resource *res, uint8_t role) 142 { 143 144 control_set_role_common(NULL, NULL, role, res, NULL, 0); 145 } 146 147 static void 148 control_status_worker(struct hast_resource *res, struct nv *nvout, 149 unsigned int no) 150 { 151 struct nv *cnvin, *cnvout; 152 const char *str; 153 int error; 154 155 cnvin = NULL; 156 157 /* 158 * Prepare and send command to worker process. 159 */ 160 cnvout = nv_alloc(); 161 nv_add_uint8(cnvout, CONTROL_STATUS, "cmd"); 162 error = nv_error(cnvout); 163 if (error != 0) { 164 pjdlog_common(LOG_ERR, 0, error, 165 "Unable to prepare control header"); 166 goto end; 167 } 168 if (hast_proto_send(res, res->hr_ctrl, cnvout, NULL, 0) == -1) { 169 error = errno; 170 pjdlog_errno(LOG_ERR, "Unable to send control header"); 171 goto end; 172 } 173 174 /* 175 * Receive response. 176 */ 177 if (hast_proto_recv_hdr(res->hr_ctrl, &cnvin) == -1) { 178 error = errno; 179 pjdlog_errno(LOG_ERR, "Unable to receive control header"); 180 goto end; 181 } 182 183 error = nv_get_int16(cnvin, "error"); 184 if (error != 0) 185 goto end; 186 187 if ((str = nv_get_string(cnvin, "status")) == NULL) { 188 error = ENOENT; 189 pjdlog_errno(LOG_ERR, "Field 'status' is missing."); 190 goto end; 191 } 192 nv_add_string(nvout, str, "status%u", no); 193 nv_add_uint64(nvout, nv_get_uint64(cnvin, "dirty"), "dirty%u", no); 194 nv_add_uint32(nvout, nv_get_uint32(cnvin, "extentsize"), 195 "extentsize%u", no); 196 nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"), 197 "keepdirty%u", no); 198 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"), 199 "stat_read%u", no); 200 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"), 201 "stat_write%u", no); 202 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"), 203 "stat_delete%u", no); 204 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"), 205 "stat_flush%u", no); 206 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"), 207 "stat_activemap_update%u", no); 208 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read_error"), 209 "stat_read_error%u", no); 210 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write_error"), 211 "stat_write_error%u", no); 212 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete_error"), 213 "stat_delete_error%u", no); 214 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush_error"), 215 "stat_flush_error%u", no); 216 nv_add_uint64(nvout, nv_get_uint64(cnvin, "idle_queue_size"), 217 "idle_queue_size%u", no); 218 nv_add_uint64(nvout, nv_get_uint64(cnvin, "local_queue_size"), 219 "local_queue_size%u", no); 220 nv_add_uint64(nvout, nv_get_uint64(cnvin, "send_queue_size"), 221 "send_queue_size%u", no); 222 nv_add_uint64(nvout, nv_get_uint64(cnvin, "recv_queue_size"), 223 "recv_queue_size%u", no); 224 nv_add_uint64(nvout, nv_get_uint64(cnvin, "done_queue_size"), 225 "done_queue_size%u", no); 226 end: 227 if (cnvin != NULL) 228 nv_free(cnvin); 229 if (cnvout != NULL) 230 nv_free(cnvout); 231 if (error != 0) 232 nv_add_int16(nvout, error, "error"); 233 } 234 235 static void 236 control_status(struct hastd_config *cfg, struct nv *nvout, 237 struct hast_resource *res, const char *name, unsigned int no) 238 { 239 240 PJDLOG_ASSERT(cfg != NULL); 241 PJDLOG_ASSERT(nvout != NULL); 242 PJDLOG_ASSERT(name != NULL); 243 244 /* Name is always needed. */ 245 nv_add_string(nvout, name, "resource%u", no); 246 247 if (res == NULL) { 248 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 249 if (strcmp(res->hr_name, name) == 0) 250 break; 251 } 252 if (res == NULL) { 253 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no); 254 return; 255 } 256 } 257 PJDLOG_ASSERT(res != NULL); 258 nv_add_string(nvout, res->hr_provname, "provname%u", no); 259 nv_add_string(nvout, res->hr_localpath, "localpath%u", no); 260 nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no); 261 if (res->hr_sourceaddr[0] != '\0') 262 nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr%u", no); 263 switch (res->hr_replication) { 264 case HAST_REPLICATION_FULLSYNC: 265 nv_add_string(nvout, "fullsync", "replication%u", no); 266 break; 267 case HAST_REPLICATION_MEMSYNC: 268 nv_add_string(nvout, "memsync", "replication%u", no); 269 break; 270 case HAST_REPLICATION_ASYNC: 271 nv_add_string(nvout, "async", "replication%u", no); 272 break; 273 default: 274 nv_add_string(nvout, "unknown", "replication%u", no); 275 break; 276 } 277 nv_add_string(nvout, checksum_name(res->hr_checksum), 278 "checksum%u", no); 279 nv_add_string(nvout, compression_name(res->hr_compression), 280 "compression%u", no); 281 nv_add_string(nvout, role2str(res->hr_role), "role%u", no); 282 nv_add_int32(nvout, res->hr_workerpid, "workerpid%u", no); 283 284 switch (res->hr_role) { 285 case HAST_ROLE_PRIMARY: 286 PJDLOG_ASSERT(res->hr_workerpid != 0); 287 /* FALLTHROUGH */ 288 case HAST_ROLE_SECONDARY: 289 if (res->hr_workerpid != 0) 290 break; 291 /* FALLTHROUGH */ 292 default: 293 return; 294 } 295 296 /* 297 * If we are here, it means that we have a worker process, which we 298 * want to ask some questions. 299 */ 300 control_status_worker(res, nvout, no); 301 } 302 303 static void 304 control_stop(struct hastd_config *cfg, struct nv *nvout) 305 { 306 pjdlog_info("Stop message received."); 307 (void)cfg; 308 (void)nvout; 309 sigexit_received = true; 310 } 311 312 void 313 control_handle(struct hastd_config *cfg) 314 { 315 struct proto_conn *conn; 316 struct nv *nvin, *nvout; 317 unsigned int ii; 318 const char *str; 319 uint8_t cmd, role; 320 int error; 321 322 if (proto_accept(cfg->hc_controlconn, &conn) == -1) { 323 pjdlog_errno(LOG_ERR, "Unable to accept control connection"); 324 return; 325 } 326 327 cfg->hc_controlin = conn; 328 nvin = nvout = NULL; 329 role = HAST_ROLE_UNDEF; 330 331 if (hast_proto_recv_hdr(conn, &nvin) == -1) { 332 pjdlog_errno(LOG_ERR, "Unable to receive control header"); 333 nvin = NULL; 334 goto close; 335 } 336 337 /* Obtain command code. 0 means that nv_get_uint8() failed. */ 338 cmd = nv_get_uint8(nvin, "cmd"); 339 if (cmd == 0) { 340 pjdlog_error("Control header is missing 'cmd' field."); 341 goto close; 342 } 343 344 /* Allocate outgoing nv structure. */ 345 nvout = nv_alloc(); 346 if (nvout == NULL) { 347 pjdlog_error("Unable to allocate header for control response."); 348 goto close; 349 } 350 351 error = 0; 352 353 /* The stop command does not expect any arguments. */ 354 if (cmd == HASTCTL_CMD_STOP) { 355 control_stop(cfg, nvout); 356 if (nv_error(nvout) != 0) 357 goto close; 358 goto fail; 359 } 360 361 str = nv_get_string(nvin, "resource0"); 362 if (str == NULL) { 363 pjdlog_error("Control header is missing 'resource0' field."); 364 error = EHAST_INVALID; 365 goto fail; 366 } 367 if (cmd == HASTCTL_CMD_SETROLE) { 368 role = nv_get_uint8(nvin, "role"); 369 switch (role) { 370 case HAST_ROLE_INIT: 371 case HAST_ROLE_PRIMARY: 372 case HAST_ROLE_SECONDARY: 373 break; 374 default: 375 pjdlog_error("Invalid role received (%hhu).", role); 376 error = EHAST_INVALID; 377 goto fail; 378 } 379 } 380 if (strcmp(str, "all") == 0) { 381 struct hast_resource *res; 382 383 /* All configured resources. */ 384 385 ii = 0; 386 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 387 switch (cmd) { 388 case HASTCTL_CMD_SETROLE: 389 control_set_role_common(cfg, nvout, role, res, 390 res->hr_name, ii++); 391 break; 392 case HASTCTL_CMD_STATUS: 393 control_status(cfg, nvout, res, res->hr_name, 394 ii++); 395 break; 396 default: 397 pjdlog_error("Invalid command received (%hhu).", 398 cmd); 399 error = EHAST_UNIMPLEMENTED; 400 goto fail; 401 } 402 } 403 } else { 404 /* Only selected resources. */ 405 406 for (ii = 0; ; ii++) { 407 str = nv_get_string(nvin, "resource%u", ii); 408 if (str == NULL) 409 break; 410 switch (cmd) { 411 case HASTCTL_CMD_SETROLE: 412 control_set_role_common(cfg, nvout, role, NULL, 413 str, ii); 414 break; 415 case HASTCTL_CMD_STATUS: 416 control_status(cfg, nvout, NULL, str, ii); 417 break; 418 default: 419 pjdlog_error("Invalid command received (%hhu).", 420 cmd); 421 error = EHAST_UNIMPLEMENTED; 422 goto fail; 423 } 424 } 425 } 426 if (nv_error(nvout) != 0) 427 goto close; 428 fail: 429 if (error != 0) 430 nv_add_int16(nvout, error, "error"); 431 432 if (hast_proto_send(NULL, conn, nvout, NULL, 0) == -1) 433 pjdlog_errno(LOG_ERR, "Unable to send control response"); 434 close: 435 if (nvin != NULL) 436 nv_free(nvin); 437 if (nvout != NULL) 438 nv_free(nvout); 439 proto_close(conn); 440 cfg->hc_controlin = NULL; 441 } 442 443 /* 444 * Thread handles control requests from the parent. 445 */ 446 void * 447 ctrl_thread(void *arg) 448 { 449 struct hast_resource *res = arg; 450 struct nv *nvin, *nvout; 451 uint8_t cmd; 452 453 for (;;) { 454 if (hast_proto_recv_hdr(res->hr_ctrl, &nvin) == -1) { 455 if (sigexit_received) 456 pthread_exit(NULL); 457 pjdlog_errno(LOG_ERR, 458 "Unable to receive control message"); 459 kill(getpid(), SIGTERM); 460 pthread_exit(NULL); 461 } 462 cmd = nv_get_uint8(nvin, "cmd"); 463 if (cmd == 0) { 464 pjdlog_error("Control message is missing 'cmd' field."); 465 nv_free(nvin); 466 continue; 467 } 468 nvout = nv_alloc(); 469 switch (cmd) { 470 case CONTROL_STATUS: 471 if (res->hr_remotein != NULL && 472 res->hr_remoteout != NULL) { 473 nv_add_string(nvout, "complete", "status"); 474 } else { 475 nv_add_string(nvout, "degraded", "status"); 476 } 477 nv_add_uint32(nvout, (uint32_t)res->hr_extentsize, 478 "extentsize"); 479 if (res->hr_role == HAST_ROLE_PRIMARY) { 480 nv_add_uint32(nvout, 481 (uint32_t)res->hr_keepdirty, "keepdirty"); 482 nv_add_uint64(nvout, 483 (uint64_t)(activemap_ndirty(res->hr_amp) * 484 res->hr_extentsize), "dirty"); 485 } else { 486 nv_add_uint32(nvout, (uint32_t)0, "keepdirty"); 487 nv_add_uint64(nvout, (uint64_t)0, "dirty"); 488 } 489 nv_add_uint64(nvout, res->hr_stat_read, "stat_read"); 490 nv_add_uint64(nvout, res->hr_stat_write, "stat_write"); 491 nv_add_uint64(nvout, res->hr_stat_delete, 492 "stat_delete"); 493 nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush"); 494 nv_add_uint64(nvout, res->hr_stat_activemap_update, 495 "stat_activemap_update"); 496 nv_add_uint64(nvout, res->hr_stat_read_error, 497 "stat_read_error"); 498 nv_add_uint64(nvout, res->hr_stat_write_error + 499 res->hr_stat_activemap_write_error, 500 "stat_write_error"); 501 nv_add_uint64(nvout, res->hr_stat_delete_error, 502 "stat_delete_error"); 503 nv_add_uint64(nvout, res->hr_stat_flush_error + 504 res->hr_stat_activemap_flush_error, 505 "stat_flush_error"); 506 res->output_status_aux(nvout); 507 nv_add_int16(nvout, 0, "error"); 508 break; 509 case CONTROL_RELOAD: 510 /* 511 * When parent receives SIGHUP and discovers that 512 * something related to us has changes, it sends reload 513 * message to us. 514 */ 515 PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY); 516 primary_config_reload(res, nvin); 517 nv_add_int16(nvout, 0, "error"); 518 break; 519 default: 520 nv_add_int16(nvout, EINVAL, "error"); 521 break; 522 } 523 nv_free(nvin); 524 if (nv_error(nvout) != 0) { 525 pjdlog_error("Unable to create answer on control message."); 526 nv_free(nvout); 527 continue; 528 } 529 if (hast_proto_send(NULL, res->hr_ctrl, nvout, NULL, 0) == -1) { 530 pjdlog_errno(LOG_ERR, 531 "Unable to send reply to control message"); 532 } 533 nv_free(nvout); 534 } 535 /* NOTREACHED */ 536 return (NULL); 537 } 538