1 /*- 2 * Copyright (c) 2009-2010 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Pawel Jakub Dawidek under sponsorship from 6 * the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/types.h> 34 #include <sys/wait.h> 35 36 #include <errno.h> 37 #include <pthread.h> 38 #include <signal.h> 39 #include <stdio.h> 40 #include <string.h> 41 #include <unistd.h> 42 43 #include "hast.h" 44 #include "hastd.h" 45 #include "hast_checksum.h" 46 #include "hast_compression.h" 47 #include "hast_proto.h" 48 #include "hooks.h" 49 #include "nv.h" 50 #include "pjdlog.h" 51 #include "proto.h" 52 #include "subr.h" 53 54 #include "control.h" 55 56 void 57 child_cleanup(struct hast_resource *res) 58 { 59 60 proto_close(res->hr_ctrl); 61 res->hr_ctrl = NULL; 62 if (res->hr_event != NULL) { 63 proto_close(res->hr_event); 64 res->hr_event = NULL; 65 } 66 if (res->hr_conn != NULL) { 67 proto_close(res->hr_conn); 68 res->hr_conn = NULL; 69 } 70 res->hr_workerpid = 0; 71 } 72 73 static void 74 control_set_role_common(struct hastd_config *cfg, struct nv *nvout, 75 uint8_t role, struct hast_resource *res, const char *name, unsigned int no) 76 { 77 int oldrole; 78 79 /* Name is always needed. */ 80 if (name != NULL) 81 nv_add_string(nvout, name, "resource%u", no); 82 83 if (res == NULL) { 84 PJDLOG_ASSERT(cfg != NULL); 85 PJDLOG_ASSERT(name != NULL); 86 87 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 88 if (strcmp(res->hr_name, name) == 0) 89 break; 90 } 91 if (res == NULL) { 92 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no); 93 return; 94 } 95 } 96 PJDLOG_ASSERT(res != NULL); 97 98 /* Send previous role back. */ 99 nv_add_string(nvout, role2str(res->hr_role), "role%u", no); 100 101 /* Nothing changed, return here. */ 102 if (role == res->hr_role) 103 return; 104 105 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 106 pjdlog_info("Role changed to %s.", role2str(role)); 107 108 /* Change role to the new one. */ 109 oldrole = res->hr_role; 110 res->hr_role = role; 111 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 112 113 /* 114 * If previous role was primary or secondary we have to kill process 115 * doing that work. 116 */ 117 if (res->hr_workerpid != 0) { 118 if (kill(res->hr_workerpid, SIGTERM) == -1) { 119 pjdlog_errno(LOG_WARNING, 120 "Unable to kill worker process %u", 121 (unsigned int)res->hr_workerpid); 122 } else if (waitpid(res->hr_workerpid, NULL, 0) != 123 res->hr_workerpid) { 124 pjdlog_errno(LOG_WARNING, 125 "Error while waiting for worker process %u", 126 (unsigned int)res->hr_workerpid); 127 } else { 128 pjdlog_debug(1, "Worker process %u stopped.", 129 (unsigned int)res->hr_workerpid); 130 } 131 child_cleanup(res); 132 } 133 134 /* Start worker process if we are changing to primary. */ 135 if (role == HAST_ROLE_PRIMARY) 136 hastd_primary(res); 137 pjdlog_prefix_set("%s", ""); 138 hook_exec(res->hr_exec, "role", res->hr_name, role2str(oldrole), 139 role2str(res->hr_role), NULL); 140 } 141 142 void 143 control_set_role(struct hast_resource *res, uint8_t role) 144 { 145 146 control_set_role_common(NULL, NULL, role, res, NULL, 0); 147 } 148 149 static void 150 control_status_worker(struct hast_resource *res, struct nv *nvout, 151 unsigned int no) 152 { 153 struct nv *cnvin, *cnvout; 154 const char *str; 155 int error; 156 157 cnvin = NULL; 158 159 /* 160 * Prepare and send command to worker process. 161 */ 162 cnvout = nv_alloc(); 163 nv_add_uint8(cnvout, CONTROL_STATUS, "cmd"); 164 error = nv_error(cnvout); 165 if (error != 0) { 166 pjdlog_common(LOG_ERR, 0, error, 167 "Unable to prepare control header"); 168 goto end; 169 } 170 if (hast_proto_send(res, res->hr_ctrl, cnvout, NULL, 0) == -1) { 171 error = errno; 172 pjdlog_errno(LOG_ERR, "Unable to send control header"); 173 goto end; 174 } 175 176 /* 177 * Receive response. 178 */ 179 if (hast_proto_recv_hdr(res->hr_ctrl, &cnvin) == -1) { 180 error = errno; 181 pjdlog_errno(LOG_ERR, "Unable to receive control header"); 182 goto end; 183 } 184 185 error = nv_get_int16(cnvin, "error"); 186 if (error != 0) 187 goto end; 188 189 if ((str = nv_get_string(cnvin, "status")) == NULL) { 190 error = ENOENT; 191 pjdlog_errno(LOG_ERR, "Field 'status' is missing."); 192 goto end; 193 } 194 nv_add_string(nvout, str, "status%u", no); 195 nv_add_uint64(nvout, nv_get_uint64(cnvin, "dirty"), "dirty%u", no); 196 nv_add_uint32(nvout, nv_get_uint32(cnvin, "extentsize"), 197 "extentsize%u", no); 198 nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"), 199 "keepdirty%u", no); 200 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"), 201 "stat_read%u", no); 202 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"), 203 "stat_write%u", no); 204 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"), 205 "stat_delete%u", no); 206 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"), 207 "stat_flush%u", no); 208 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"), 209 "stat_activemap_update%u", no); 210 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read_error"), 211 "stat_read_error%u", no); 212 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write_error"), 213 "stat_write_error%u", no); 214 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete_error"), 215 "stat_delete_error%u", no); 216 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush_error"), 217 "stat_flush_error%u", no); 218 nv_add_uint64(nvout, nv_get_uint64(cnvin, "idle_queue_size"), 219 "idle_queue_size%u", no); 220 nv_add_uint64(nvout, nv_get_uint64(cnvin, "local_queue_size"), 221 "local_queue_size%u", no); 222 nv_add_uint64(nvout, nv_get_uint64(cnvin, "send_queue_size"), 223 "send_queue_size%u", no); 224 nv_add_uint64(nvout, nv_get_uint64(cnvin, "recv_queue_size"), 225 "recv_queue_size%u", no); 226 nv_add_uint64(nvout, nv_get_uint64(cnvin, "done_queue_size"), 227 "done_queue_size%u", no); 228 end: 229 if (cnvin != NULL) 230 nv_free(cnvin); 231 if (cnvout != NULL) 232 nv_free(cnvout); 233 if (error != 0) 234 nv_add_int16(nvout, error, "error"); 235 } 236 237 static void 238 control_status(struct hastd_config *cfg, struct nv *nvout, 239 struct hast_resource *res, const char *name, unsigned int no) 240 { 241 242 PJDLOG_ASSERT(cfg != NULL); 243 PJDLOG_ASSERT(nvout != NULL); 244 PJDLOG_ASSERT(name != NULL); 245 246 /* Name is always needed. */ 247 nv_add_string(nvout, name, "resource%u", no); 248 249 if (res == NULL) { 250 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 251 if (strcmp(res->hr_name, name) == 0) 252 break; 253 } 254 if (res == NULL) { 255 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no); 256 return; 257 } 258 } 259 PJDLOG_ASSERT(res != NULL); 260 nv_add_string(nvout, res->hr_provname, "provname%u", no); 261 nv_add_string(nvout, res->hr_localpath, "localpath%u", no); 262 nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no); 263 if (res->hr_sourceaddr[0] != '\0') 264 nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr%u", no); 265 switch (res->hr_replication) { 266 case HAST_REPLICATION_FULLSYNC: 267 nv_add_string(nvout, "fullsync", "replication%u", no); 268 break; 269 case HAST_REPLICATION_MEMSYNC: 270 nv_add_string(nvout, "memsync", "replication%u", no); 271 break; 272 case HAST_REPLICATION_ASYNC: 273 nv_add_string(nvout, "async", "replication%u", no); 274 break; 275 default: 276 nv_add_string(nvout, "unknown", "replication%u", no); 277 break; 278 } 279 nv_add_string(nvout, checksum_name(res->hr_checksum), 280 "checksum%u", no); 281 nv_add_string(nvout, compression_name(res->hr_compression), 282 "compression%u", no); 283 nv_add_string(nvout, role2str(res->hr_role), "role%u", no); 284 nv_add_int32(nvout, res->hr_workerpid, "workerpid%u", no); 285 286 switch (res->hr_role) { 287 case HAST_ROLE_PRIMARY: 288 PJDLOG_ASSERT(res->hr_workerpid != 0); 289 /* FALLTHROUGH */ 290 case HAST_ROLE_SECONDARY: 291 if (res->hr_workerpid != 0) 292 break; 293 /* FALLTHROUGH */ 294 default: 295 return; 296 } 297 298 /* 299 * If we are here, it means that we have a worker process, which we 300 * want to ask some questions. 301 */ 302 control_status_worker(res, nvout, no); 303 } 304 305 void 306 control_handle(struct hastd_config *cfg) 307 { 308 struct proto_conn *conn; 309 struct nv *nvin, *nvout; 310 unsigned int ii; 311 const char *str; 312 uint8_t cmd, role; 313 int error; 314 315 if (proto_accept(cfg->hc_controlconn, &conn) == -1) { 316 pjdlog_errno(LOG_ERR, "Unable to accept control connection"); 317 return; 318 } 319 320 cfg->hc_controlin = conn; 321 nvin = nvout = NULL; 322 role = HAST_ROLE_UNDEF; 323 324 if (hast_proto_recv_hdr(conn, &nvin) == -1) { 325 pjdlog_errno(LOG_ERR, "Unable to receive control header"); 326 nvin = NULL; 327 goto close; 328 } 329 330 /* Obtain command code. 0 means that nv_get_uint8() failed. */ 331 cmd = nv_get_uint8(nvin, "cmd"); 332 if (cmd == 0) { 333 pjdlog_error("Control header is missing 'cmd' field."); 334 goto close; 335 } 336 337 /* Allocate outgoing nv structure. */ 338 nvout = nv_alloc(); 339 if (nvout == NULL) { 340 pjdlog_error("Unable to allocate header for control response."); 341 goto close; 342 } 343 344 error = 0; 345 346 str = nv_get_string(nvin, "resource0"); 347 if (str == NULL) { 348 pjdlog_error("Control header is missing 'resource0' field."); 349 error = EHAST_INVALID; 350 goto fail; 351 } 352 if (cmd == HASTCTL_CMD_SETROLE) { 353 role = nv_get_uint8(nvin, "role"); 354 switch (role) { 355 case HAST_ROLE_INIT: 356 case HAST_ROLE_PRIMARY: 357 case HAST_ROLE_SECONDARY: 358 break; 359 default: 360 pjdlog_error("Invalid role received (%hhu).", role); 361 error = EHAST_INVALID; 362 goto fail; 363 } 364 } 365 if (strcmp(str, "all") == 0) { 366 struct hast_resource *res; 367 368 /* All configured resources. */ 369 370 ii = 0; 371 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 372 switch (cmd) { 373 case HASTCTL_CMD_SETROLE: 374 control_set_role_common(cfg, nvout, role, res, 375 res->hr_name, ii++); 376 break; 377 case HASTCTL_CMD_STATUS: 378 control_status(cfg, nvout, res, res->hr_name, 379 ii++); 380 break; 381 default: 382 pjdlog_error("Invalid command received (%hhu).", 383 cmd); 384 error = EHAST_UNIMPLEMENTED; 385 goto fail; 386 } 387 } 388 } else { 389 /* Only selected resources. */ 390 391 for (ii = 0; ; ii++) { 392 str = nv_get_string(nvin, "resource%u", ii); 393 if (str == NULL) 394 break; 395 switch (cmd) { 396 case HASTCTL_CMD_SETROLE: 397 control_set_role_common(cfg, nvout, role, NULL, 398 str, ii); 399 break; 400 case HASTCTL_CMD_STATUS: 401 control_status(cfg, nvout, NULL, str, ii); 402 break; 403 default: 404 pjdlog_error("Invalid command received (%hhu).", 405 cmd); 406 error = EHAST_UNIMPLEMENTED; 407 goto fail; 408 } 409 } 410 } 411 if (nv_error(nvout) != 0) 412 goto close; 413 fail: 414 if (error != 0) 415 nv_add_int16(nvout, error, "error"); 416 417 if (hast_proto_send(NULL, conn, nvout, NULL, 0) == -1) 418 pjdlog_errno(LOG_ERR, "Unable to send control response"); 419 close: 420 if (nvin != NULL) 421 nv_free(nvin); 422 if (nvout != NULL) 423 nv_free(nvout); 424 proto_close(conn); 425 cfg->hc_controlin = NULL; 426 } 427 428 /* 429 * Thread handles control requests from the parent. 430 */ 431 void * 432 ctrl_thread(void *arg) 433 { 434 struct hast_resource *res = arg; 435 struct nv *nvin, *nvout; 436 uint8_t cmd; 437 438 for (;;) { 439 if (hast_proto_recv_hdr(res->hr_ctrl, &nvin) == -1) { 440 if (sigexit_received) 441 pthread_exit(NULL); 442 pjdlog_errno(LOG_ERR, 443 "Unable to receive control message"); 444 kill(getpid(), SIGTERM); 445 pthread_exit(NULL); 446 } 447 cmd = nv_get_uint8(nvin, "cmd"); 448 if (cmd == 0) { 449 pjdlog_error("Control message is missing 'cmd' field."); 450 nv_free(nvin); 451 continue; 452 } 453 nvout = nv_alloc(); 454 switch (cmd) { 455 case CONTROL_STATUS: 456 if (res->hr_remotein != NULL && 457 res->hr_remoteout != NULL) { 458 nv_add_string(nvout, "complete", "status"); 459 } else { 460 nv_add_string(nvout, "degraded", "status"); 461 } 462 nv_add_uint32(nvout, (uint32_t)res->hr_extentsize, 463 "extentsize"); 464 if (res->hr_role == HAST_ROLE_PRIMARY) { 465 nv_add_uint32(nvout, 466 (uint32_t)res->hr_keepdirty, "keepdirty"); 467 nv_add_uint64(nvout, 468 (uint64_t)(activemap_ndirty(res->hr_amp) * 469 res->hr_extentsize), "dirty"); 470 } else { 471 nv_add_uint32(nvout, (uint32_t)0, "keepdirty"); 472 nv_add_uint64(nvout, (uint64_t)0, "dirty"); 473 } 474 nv_add_uint64(nvout, res->hr_stat_read, "stat_read"); 475 nv_add_uint64(nvout, res->hr_stat_write, "stat_write"); 476 nv_add_uint64(nvout, res->hr_stat_delete, 477 "stat_delete"); 478 nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush"); 479 nv_add_uint64(nvout, res->hr_stat_activemap_update, 480 "stat_activemap_update"); 481 nv_add_uint64(nvout, res->hr_stat_read_error, 482 "stat_read_error"); 483 nv_add_uint64(nvout, res->hr_stat_write_error + 484 res->hr_stat_activemap_write_error, 485 "stat_write_error"); 486 nv_add_uint64(nvout, res->hr_stat_delete_error, 487 "stat_delete_error"); 488 nv_add_uint64(nvout, res->hr_stat_flush_error + 489 res->hr_stat_activemap_flush_error, 490 "stat_flush_error"); 491 res->output_status_aux(nvout); 492 nv_add_int16(nvout, 0, "error"); 493 break; 494 case CONTROL_RELOAD: 495 /* 496 * When parent receives SIGHUP and discovers that 497 * something related to us has changes, it sends reload 498 * message to us. 499 */ 500 PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY); 501 primary_config_reload(res, nvin); 502 nv_add_int16(nvout, 0, "error"); 503 break; 504 default: 505 nv_add_int16(nvout, EINVAL, "error"); 506 break; 507 } 508 nv_free(nvin); 509 if (nv_error(nvout) != 0) { 510 pjdlog_error("Unable to create answer on control message."); 511 nv_free(nvout); 512 continue; 513 } 514 if (hast_proto_send(NULL, res->hr_ctrl, nvout, NULL, 0) == -1) { 515 pjdlog_errno(LOG_ERR, 516 "Unable to send reply to control message"); 517 } 518 nv_free(nvout); 519 } 520 /* NOTREACHED */ 521 return (NULL); 522 } 523