1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2009-2010 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Pawel Jakub Dawidek under sponsorship from 8 * the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/types.h> 36 #include <sys/wait.h> 37 38 #include <errno.h> 39 #include <pthread.h> 40 #include <signal.h> 41 #include <stdio.h> 42 #include <string.h> 43 #include <unistd.h> 44 45 #include "hast.h" 46 #include "hastd.h" 47 #include "hast_checksum.h" 48 #include "hast_compression.h" 49 #include "hast_proto.h" 50 #include "hooks.h" 51 #include "nv.h" 52 #include "pjdlog.h" 53 #include "proto.h" 54 #include "subr.h" 55 56 #include "control.h" 57 58 void 59 child_cleanup(struct hast_resource *res) 60 { 61 62 proto_close(res->hr_ctrl); 63 res->hr_ctrl = NULL; 64 if (res->hr_event != NULL) { 65 proto_close(res->hr_event); 66 res->hr_event = NULL; 67 } 68 if (res->hr_conn != NULL) { 69 proto_close(res->hr_conn); 70 res->hr_conn = NULL; 71 } 72 res->hr_workerpid = 0; 73 } 74 75 static void 76 control_set_role_common(struct hastd_config *cfg, struct nv *nvout, 77 uint8_t role, struct hast_resource *res, const char *name, unsigned int no) 78 { 79 int oldrole; 80 81 /* Name is always needed. */ 82 if (name != NULL) 83 nv_add_string(nvout, name, "resource%u", no); 84 85 if (res == NULL) { 86 PJDLOG_ASSERT(cfg != NULL); 87 PJDLOG_ASSERT(name != NULL); 88 89 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 90 if (strcmp(res->hr_name, name) == 0) 91 break; 92 } 93 if (res == NULL) { 94 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no); 95 return; 96 } 97 } 98 PJDLOG_ASSERT(res != NULL); 99 100 /* Send previous role back. */ 101 nv_add_string(nvout, role2str(res->hr_role), "role%u", no); 102 103 /* Nothing changed, return here. */ 104 if (role == res->hr_role) 105 return; 106 107 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 108 pjdlog_info("Role changed to %s.", role2str(role)); 109 110 /* Change role to the new one. */ 111 oldrole = res->hr_role; 112 res->hr_role = role; 113 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 114 115 /* 116 * If previous role was primary or secondary we have to kill process 117 * doing that work. 118 */ 119 if (res->hr_workerpid != 0) { 120 if (kill(res->hr_workerpid, SIGTERM) == -1) { 121 pjdlog_errno(LOG_WARNING, 122 "Unable to kill worker process %u", 123 (unsigned int)res->hr_workerpid); 124 } else if (waitpid(res->hr_workerpid, NULL, 0) != 125 res->hr_workerpid) { 126 pjdlog_errno(LOG_WARNING, 127 "Error while waiting for worker process %u", 128 (unsigned int)res->hr_workerpid); 129 } else { 130 pjdlog_debug(1, "Worker process %u stopped.", 131 (unsigned int)res->hr_workerpid); 132 } 133 child_cleanup(res); 134 } 135 136 /* Start worker process if we are changing to primary. */ 137 if (role == HAST_ROLE_PRIMARY) 138 hastd_primary(res); 139 pjdlog_prefix_set("%s", ""); 140 hook_exec(res->hr_exec, "role", res->hr_name, role2str(oldrole), 141 role2str(res->hr_role), NULL); 142 } 143 144 void 145 control_set_role(struct hast_resource *res, uint8_t role) 146 { 147 148 control_set_role_common(NULL, NULL, role, res, NULL, 0); 149 } 150 151 static void 152 control_status_worker(struct hast_resource *res, struct nv *nvout, 153 unsigned int no) 154 { 155 struct nv *cnvin, *cnvout; 156 const char *str; 157 int error; 158 159 cnvin = NULL; 160 161 /* 162 * Prepare and send command to worker process. 163 */ 164 cnvout = nv_alloc(); 165 nv_add_uint8(cnvout, CONTROL_STATUS, "cmd"); 166 error = nv_error(cnvout); 167 if (error != 0) { 168 pjdlog_common(LOG_ERR, 0, error, 169 "Unable to prepare control header"); 170 goto end; 171 } 172 if (hast_proto_send(res, res->hr_ctrl, cnvout, NULL, 0) == -1) { 173 error = errno; 174 pjdlog_errno(LOG_ERR, "Unable to send control header"); 175 goto end; 176 } 177 178 /* 179 * Receive response. 180 */ 181 if (hast_proto_recv_hdr(res->hr_ctrl, &cnvin) == -1) { 182 error = errno; 183 pjdlog_errno(LOG_ERR, "Unable to receive control header"); 184 goto end; 185 } 186 187 error = nv_get_int16(cnvin, "error"); 188 if (error != 0) 189 goto end; 190 191 if ((str = nv_get_string(cnvin, "status")) == NULL) { 192 error = ENOENT; 193 pjdlog_errno(LOG_ERR, "Field 'status' is missing."); 194 goto end; 195 } 196 nv_add_string(nvout, str, "status%u", no); 197 nv_add_uint64(nvout, nv_get_uint64(cnvin, "dirty"), "dirty%u", no); 198 nv_add_uint32(nvout, nv_get_uint32(cnvin, "extentsize"), 199 "extentsize%u", no); 200 nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"), 201 "keepdirty%u", no); 202 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"), 203 "stat_read%u", no); 204 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"), 205 "stat_write%u", no); 206 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"), 207 "stat_delete%u", no); 208 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"), 209 "stat_flush%u", no); 210 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"), 211 "stat_activemap_update%u", no); 212 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read_error"), 213 "stat_read_error%u", no); 214 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write_error"), 215 "stat_write_error%u", no); 216 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete_error"), 217 "stat_delete_error%u", no); 218 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush_error"), 219 "stat_flush_error%u", no); 220 nv_add_uint64(nvout, nv_get_uint64(cnvin, "idle_queue_size"), 221 "idle_queue_size%u", no); 222 nv_add_uint64(nvout, nv_get_uint64(cnvin, "local_queue_size"), 223 "local_queue_size%u", no); 224 nv_add_uint64(nvout, nv_get_uint64(cnvin, "send_queue_size"), 225 "send_queue_size%u", no); 226 nv_add_uint64(nvout, nv_get_uint64(cnvin, "recv_queue_size"), 227 "recv_queue_size%u", no); 228 nv_add_uint64(nvout, nv_get_uint64(cnvin, "done_queue_size"), 229 "done_queue_size%u", no); 230 end: 231 if (cnvin != NULL) 232 nv_free(cnvin); 233 if (cnvout != NULL) 234 nv_free(cnvout); 235 if (error != 0) 236 nv_add_int16(nvout, error, "error"); 237 } 238 239 static void 240 control_status(struct hastd_config *cfg, struct nv *nvout, 241 struct hast_resource *res, const char *name, unsigned int no) 242 { 243 244 PJDLOG_ASSERT(cfg != NULL); 245 PJDLOG_ASSERT(nvout != NULL); 246 PJDLOG_ASSERT(name != NULL); 247 248 /* Name is always needed. */ 249 nv_add_string(nvout, name, "resource%u", no); 250 251 if (res == NULL) { 252 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 253 if (strcmp(res->hr_name, name) == 0) 254 break; 255 } 256 if (res == NULL) { 257 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no); 258 return; 259 } 260 } 261 PJDLOG_ASSERT(res != NULL); 262 nv_add_string(nvout, res->hr_provname, "provname%u", no); 263 nv_add_string(nvout, res->hr_localpath, "localpath%u", no); 264 nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no); 265 if (res->hr_sourceaddr[0] != '\0') 266 nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr%u", no); 267 switch (res->hr_replication) { 268 case HAST_REPLICATION_FULLSYNC: 269 nv_add_string(nvout, "fullsync", "replication%u", no); 270 break; 271 case HAST_REPLICATION_MEMSYNC: 272 nv_add_string(nvout, "memsync", "replication%u", no); 273 break; 274 case HAST_REPLICATION_ASYNC: 275 nv_add_string(nvout, "async", "replication%u", no); 276 break; 277 default: 278 nv_add_string(nvout, "unknown", "replication%u", no); 279 break; 280 } 281 nv_add_string(nvout, checksum_name(res->hr_checksum), 282 "checksum%u", no); 283 nv_add_string(nvout, compression_name(res->hr_compression), 284 "compression%u", no); 285 nv_add_string(nvout, role2str(res->hr_role), "role%u", no); 286 nv_add_int32(nvout, res->hr_workerpid, "workerpid%u", no); 287 288 switch (res->hr_role) { 289 case HAST_ROLE_PRIMARY: 290 PJDLOG_ASSERT(res->hr_workerpid != 0); 291 /* FALLTHROUGH */ 292 case HAST_ROLE_SECONDARY: 293 if (res->hr_workerpid != 0) 294 break; 295 /* FALLTHROUGH */ 296 default: 297 return; 298 } 299 300 /* 301 * If we are here, it means that we have a worker process, which we 302 * want to ask some questions. 303 */ 304 control_status_worker(res, nvout, no); 305 } 306 307 void 308 control_handle(struct hastd_config *cfg) 309 { 310 struct proto_conn *conn; 311 struct nv *nvin, *nvout; 312 unsigned int ii; 313 const char *str; 314 uint8_t cmd, role; 315 int error; 316 317 if (proto_accept(cfg->hc_controlconn, &conn) == -1) { 318 pjdlog_errno(LOG_ERR, "Unable to accept control connection"); 319 return; 320 } 321 322 cfg->hc_controlin = conn; 323 nvin = nvout = NULL; 324 role = HAST_ROLE_UNDEF; 325 326 if (hast_proto_recv_hdr(conn, &nvin) == -1) { 327 pjdlog_errno(LOG_ERR, "Unable to receive control header"); 328 nvin = NULL; 329 goto close; 330 } 331 332 /* Obtain command code. 0 means that nv_get_uint8() failed. */ 333 cmd = nv_get_uint8(nvin, "cmd"); 334 if (cmd == 0) { 335 pjdlog_error("Control header is missing 'cmd' field."); 336 goto close; 337 } 338 339 /* Allocate outgoing nv structure. */ 340 nvout = nv_alloc(); 341 if (nvout == NULL) { 342 pjdlog_error("Unable to allocate header for control response."); 343 goto close; 344 } 345 346 error = 0; 347 348 str = nv_get_string(nvin, "resource0"); 349 if (str == NULL) { 350 pjdlog_error("Control header is missing 'resource0' field."); 351 error = EHAST_INVALID; 352 goto fail; 353 } 354 if (cmd == HASTCTL_CMD_SETROLE) { 355 role = nv_get_uint8(nvin, "role"); 356 switch (role) { 357 case HAST_ROLE_INIT: 358 case HAST_ROLE_PRIMARY: 359 case HAST_ROLE_SECONDARY: 360 break; 361 default: 362 pjdlog_error("Invalid role received (%hhu).", role); 363 error = EHAST_INVALID; 364 goto fail; 365 } 366 } 367 if (strcmp(str, "all") == 0) { 368 struct hast_resource *res; 369 370 /* All configured resources. */ 371 372 ii = 0; 373 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 374 switch (cmd) { 375 case HASTCTL_CMD_SETROLE: 376 control_set_role_common(cfg, nvout, role, res, 377 res->hr_name, ii++); 378 break; 379 case HASTCTL_CMD_STATUS: 380 control_status(cfg, nvout, res, res->hr_name, 381 ii++); 382 break; 383 default: 384 pjdlog_error("Invalid command received (%hhu).", 385 cmd); 386 error = EHAST_UNIMPLEMENTED; 387 goto fail; 388 } 389 } 390 } else { 391 /* Only selected resources. */ 392 393 for (ii = 0; ; ii++) { 394 str = nv_get_string(nvin, "resource%u", ii); 395 if (str == NULL) 396 break; 397 switch (cmd) { 398 case HASTCTL_CMD_SETROLE: 399 control_set_role_common(cfg, nvout, role, NULL, 400 str, ii); 401 break; 402 case HASTCTL_CMD_STATUS: 403 control_status(cfg, nvout, NULL, str, ii); 404 break; 405 default: 406 pjdlog_error("Invalid command received (%hhu).", 407 cmd); 408 error = EHAST_UNIMPLEMENTED; 409 goto fail; 410 } 411 } 412 } 413 if (nv_error(nvout) != 0) 414 goto close; 415 fail: 416 if (error != 0) 417 nv_add_int16(nvout, error, "error"); 418 419 if (hast_proto_send(NULL, conn, nvout, NULL, 0) == -1) 420 pjdlog_errno(LOG_ERR, "Unable to send control response"); 421 close: 422 if (nvin != NULL) 423 nv_free(nvin); 424 if (nvout != NULL) 425 nv_free(nvout); 426 proto_close(conn); 427 cfg->hc_controlin = NULL; 428 } 429 430 /* 431 * Thread handles control requests from the parent. 432 */ 433 void * 434 ctrl_thread(void *arg) 435 { 436 struct hast_resource *res = arg; 437 struct nv *nvin, *nvout; 438 uint8_t cmd; 439 440 for (;;) { 441 if (hast_proto_recv_hdr(res->hr_ctrl, &nvin) == -1) { 442 if (sigexit_received) 443 pthread_exit(NULL); 444 pjdlog_errno(LOG_ERR, 445 "Unable to receive control message"); 446 kill(getpid(), SIGTERM); 447 pthread_exit(NULL); 448 } 449 cmd = nv_get_uint8(nvin, "cmd"); 450 if (cmd == 0) { 451 pjdlog_error("Control message is missing 'cmd' field."); 452 nv_free(nvin); 453 continue; 454 } 455 nvout = nv_alloc(); 456 switch (cmd) { 457 case CONTROL_STATUS: 458 if (res->hr_remotein != NULL && 459 res->hr_remoteout != NULL) { 460 nv_add_string(nvout, "complete", "status"); 461 } else { 462 nv_add_string(nvout, "degraded", "status"); 463 } 464 nv_add_uint32(nvout, (uint32_t)res->hr_extentsize, 465 "extentsize"); 466 if (res->hr_role == HAST_ROLE_PRIMARY) { 467 nv_add_uint32(nvout, 468 (uint32_t)res->hr_keepdirty, "keepdirty"); 469 nv_add_uint64(nvout, 470 (uint64_t)(activemap_ndirty(res->hr_amp) * 471 res->hr_extentsize), "dirty"); 472 } else { 473 nv_add_uint32(nvout, (uint32_t)0, "keepdirty"); 474 nv_add_uint64(nvout, (uint64_t)0, "dirty"); 475 } 476 nv_add_uint64(nvout, res->hr_stat_read, "stat_read"); 477 nv_add_uint64(nvout, res->hr_stat_write, "stat_write"); 478 nv_add_uint64(nvout, res->hr_stat_delete, 479 "stat_delete"); 480 nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush"); 481 nv_add_uint64(nvout, res->hr_stat_activemap_update, 482 "stat_activemap_update"); 483 nv_add_uint64(nvout, res->hr_stat_read_error, 484 "stat_read_error"); 485 nv_add_uint64(nvout, res->hr_stat_write_error + 486 res->hr_stat_activemap_write_error, 487 "stat_write_error"); 488 nv_add_uint64(nvout, res->hr_stat_delete_error, 489 "stat_delete_error"); 490 nv_add_uint64(nvout, res->hr_stat_flush_error + 491 res->hr_stat_activemap_flush_error, 492 "stat_flush_error"); 493 res->output_status_aux(nvout); 494 nv_add_int16(nvout, 0, "error"); 495 break; 496 case CONTROL_RELOAD: 497 /* 498 * When parent receives SIGHUP and discovers that 499 * something related to us has changes, it sends reload 500 * message to us. 501 */ 502 PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY); 503 primary_config_reload(res, nvin); 504 nv_add_int16(nvout, 0, "error"); 505 break; 506 default: 507 nv_add_int16(nvout, EINVAL, "error"); 508 break; 509 } 510 nv_free(nvin); 511 if (nv_error(nvout) != 0) { 512 pjdlog_error("Unable to create answer on control message."); 513 nv_free(nvout); 514 continue; 515 } 516 if (hast_proto_send(NULL, res->hr_ctrl, nvout, NULL, 0) == -1) { 517 pjdlog_errno(LOG_ERR, 518 "Unable to send reply to control message"); 519 } 520 nv_free(nvout); 521 } 522 /* NOTREACHED */ 523 return (NULL); 524 } 525