1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 25 */ 26 27 /* 28 * restarter.c - service manipulation 29 * 30 * This component manages services whose restarter is svc.startd, the standard 31 * restarter. It translates restarter protocol events from the graph engine 32 * into actions on processes, as a delegated restarter would do. 33 * 34 * The master restarter manages a number of always-running threads: 35 * - restarter event thread: events from the graph engine 36 * - timeout thread: thread to fire queued timeouts 37 * - contract thread: thread to handle contract events 38 * - wait thread: thread to handle wait-based services 39 * 40 * The other threads are created as-needed: 41 * - per-instance method threads 42 * - per-instance event processing threads 43 * 44 * The interaction of all threads must result in the following conditions 45 * being satisfied (on a per-instance basis): 46 * - restarter events must be processed in order 47 * - method execution must be serialized 48 * - instance delete must be held until outstanding methods are complete 49 * - contract events shouldn't be processed while a method is running 50 * - timeouts should fire even when a method is running 51 * 52 * Service instances are represented by restarter_inst_t's and are kept in the 53 * instance_list list. 54 * 55 * Service States 56 * The current state of a service instance is kept in 57 * restarter_inst_t->ri_i.i_state. If transition to a new state could take 58 * some time, then before we effect the transition we set 59 * restarter_inst_t->ri_i.i_next_state to the target state, and afterwards we 60 * rotate i_next_state to i_state and set i_next_state to 61 * RESTARTER_STATE_NONE. So usually i_next_state is _NONE when ri_lock is not 62 * held. The exception is when we launch methods, which are done with 63 * a separate thread. To keep any other threads from grabbing ri_lock before 64 * method_thread() does, we set ri_method_thread to the thread id of the 65 * method thread, and when it is nonzero any thread with a different thread id 66 * waits on ri_method_cv. 67 * 68 * Method execution is serialized by blocking on ri_method_cv in 69 * inst_lookup_by_id() and waiting for a 0 value of ri_method_thread. This 70 * also prevents the instance structure from being deleted until all 71 * outstanding operations such as method_thread() have finished. 72 * 73 * Lock ordering: 74 * 75 * dgraph_lock [can be held when taking:] 76 * utmpx_lock 77 * dictionary->dict_lock 78 * st->st_load_lock 79 * wait_info_lock 80 * ru->restarter_update_lock 81 * restarter_queue->rpeq_lock 82 * instance_list.ril_lock 83 * inst->ri_lock 84 * st->st_configd_live_lock 85 * 86 * instance_list.ril_lock 87 * graph_queue->gpeq_lock 88 * gu->gu_lock 89 * st->st_configd_live_lock 90 * dictionary->dict_lock 91 * inst->ri_lock 92 * graph_queue->gpeq_lock 93 * gu->gu_lock 94 * tu->tu_lock 95 * tq->tq_lock 96 * inst->ri_queue_lock 97 * wait_info_lock 98 * bp->cb_lock 99 * utmpx_lock 100 * 101 * single_user_thread_lock 102 * wait_info_lock 103 * utmpx_lock 104 * 105 * gu_freeze_lock 106 * 107 * logbuf_mutex nests inside pretty much everything. 108 */ 109 110 #include <sys/contract/process.h> 111 #include <sys/ctfs.h> 112 #include <sys/stat.h> 113 #include <sys/time.h> 114 #include <sys/types.h> 115 #include <sys/uio.h> 116 #include <sys/wait.h> 117 #include <assert.h> 118 #include <errno.h> 119 #include <fcntl.h> 120 #include <libcontract.h> 121 #include <libcontract_priv.h> 122 #include <libintl.h> 123 #include <librestart.h> 124 #include <librestart_priv.h> 125 #include <libuutil.h> 126 #include <limits.h> 127 #include <poll.h> 128 #include <port.h> 129 #include <pthread.h> 130 #include <stdarg.h> 131 #include <stdio.h> 132 #include <strings.h> 133 #include <unistd.h> 134 135 #include "startd.h" 136 #include "protocol.h" 137 138 static uu_list_pool_t *restarter_instance_pool; 139 static restarter_instance_list_t instance_list; 140 141 static uu_list_pool_t *restarter_queue_pool; 142 143 #define WT_SVC_ERR_THROTTLE 1 /* 1 sec delay for erroring wait svc */ 144 145 /* 146 * Function used to reset the restart times for an instance, when 147 * an administrative task comes along and essentially makes the times 148 * in this array ineffective. 149 */ 150 static void 151 reset_start_times(restarter_inst_t *inst) 152 { 153 inst->ri_start_index = 0; 154 bzero(inst->ri_start_time, sizeof (inst->ri_start_time)); 155 } 156 157 /*ARGSUSED*/ 158 static int 159 restarter_instance_compare(const void *lc_arg, const void *rc_arg, 160 void *private) 161 { 162 int lc_id = ((const restarter_inst_t *)lc_arg)->ri_id; 163 int rc_id = *(int *)rc_arg; 164 165 if (lc_id > rc_id) 166 return (1); 167 if (lc_id < rc_id) 168 return (-1); 169 return (0); 170 } 171 172 static restarter_inst_t * 173 inst_lookup_by_name(const char *name) 174 { 175 int id; 176 177 id = dict_lookup_byname(name); 178 if (id == -1) 179 return (NULL); 180 181 return (inst_lookup_by_id(id)); 182 } 183 184 restarter_inst_t * 185 inst_lookup_by_id(int id) 186 { 187 restarter_inst_t *inst; 188 189 MUTEX_LOCK(&instance_list.ril_lock); 190 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL); 191 if (inst != NULL) 192 MUTEX_LOCK(&inst->ri_lock); 193 MUTEX_UNLOCK(&instance_list.ril_lock); 194 195 if (inst != NULL) { 196 while (inst->ri_method_thread != 0 && 197 !pthread_equal(inst->ri_method_thread, pthread_self())) { 198 ++inst->ri_method_waiters; 199 (void) pthread_cond_wait(&inst->ri_method_cv, 200 &inst->ri_lock); 201 assert(inst->ri_method_waiters > 0); 202 --inst->ri_method_waiters; 203 } 204 } 205 206 return (inst); 207 } 208 209 static restarter_inst_t * 210 inst_lookup_queue(const char *name) 211 { 212 int id; 213 restarter_inst_t *inst; 214 215 id = dict_lookup_byname(name); 216 if (id == -1) 217 return (NULL); 218 219 MUTEX_LOCK(&instance_list.ril_lock); 220 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL); 221 if (inst != NULL) 222 MUTEX_LOCK(&inst->ri_queue_lock); 223 MUTEX_UNLOCK(&instance_list.ril_lock); 224 225 return (inst); 226 } 227 228 const char * 229 service_style(int flags) 230 { 231 switch (flags & RINST_STYLE_MASK) { 232 case RINST_CONTRACT: return ("contract"); 233 case RINST_TRANSIENT: return ("transient"); 234 case RINST_WAIT: return ("wait"); 235 236 default: 237 #ifndef NDEBUG 238 uu_warn("%s:%d: Bad flags 0x%x.\n", __FILE__, __LINE__, flags); 239 #endif 240 abort(); 241 /* NOTREACHED */ 242 } 243 } 244 245 /* 246 * Fails with ECONNABORTED or ECANCELED. 247 */ 248 static int 249 check_contract(restarter_inst_t *inst, boolean_t primary, 250 scf_instance_t *scf_inst) 251 { 252 ctid_t *ctidp; 253 int fd, r; 254 255 ctidp = primary ? &inst->ri_i.i_primary_ctid : 256 &inst->ri_i.i_transient_ctid; 257 258 assert(*ctidp >= 1); 259 260 fd = contract_open(*ctidp, NULL, "status", O_RDONLY); 261 if (fd >= 0) { 262 r = close(fd); 263 assert(r == 0); 264 return (0); 265 } 266 267 r = restarter_remove_contract(scf_inst, *ctidp, primary ? 268 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT); 269 switch (r) { 270 case 0: 271 case ECONNABORTED: 272 case ECANCELED: 273 *ctidp = 0; 274 return (r); 275 276 case ENOMEM: 277 uu_die("Out of memory\n"); 278 /* NOTREACHED */ 279 280 case EPERM: 281 uu_die("Insufficient privilege.\n"); 282 /* NOTREACHED */ 283 284 case EACCES: 285 uu_die("Repository backend access denied.\n"); 286 /* NOTREACHED */ 287 288 case EROFS: 289 log_error(LOG_INFO, "Could not remove unusable contract id %ld " 290 "for %s from repository.\n", *ctidp, inst->ri_i.i_fmri); 291 return (0); 292 293 case EINVAL: 294 case EBADF: 295 default: 296 assert(0); 297 abort(); 298 /* NOTREACHED */ 299 } 300 } 301 302 static int stop_instance(scf_handle_t *, restarter_inst_t *, stop_cause_t); 303 304 /* 305 * int restarter_insert_inst(scf_handle_t *, char *) 306 * If the inst is already in the restarter list, return its id. If the inst 307 * is not in the restarter list, initialize a restarter_inst_t, initialize its 308 * states, insert it into the list, and return 0. 309 * 310 * Fails with 311 * ENOENT - name is not in the repository 312 */ 313 static int 314 restarter_insert_inst(scf_handle_t *h, const char *name) 315 { 316 int id, r; 317 restarter_inst_t *inst; 318 uu_list_index_t idx; 319 scf_service_t *scf_svc; 320 scf_instance_t *scf_inst; 321 scf_snapshot_t *snap = NULL; 322 scf_propertygroup_t *pg; 323 char *svc_name, *inst_name; 324 char logfilebuf[PATH_MAX]; 325 char *c; 326 boolean_t do_commit_states; 327 restarter_instance_state_t state, next_state; 328 protocol_states_t *ps; 329 pid_t start_pid; 330 restarter_str_t reason = restarter_str_insert_in_graph; 331 332 MUTEX_LOCK(&instance_list.ril_lock); 333 334 /* 335 * We don't use inst_lookup_by_name() here because we want the lookup 336 * & insert to be atomic. 337 */ 338 id = dict_lookup_byname(name); 339 if (id != -1) { 340 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, 341 &idx); 342 if (inst != NULL) { 343 MUTEX_UNLOCK(&instance_list.ril_lock); 344 return (0); 345 } 346 } 347 348 /* Allocate an instance */ 349 inst = startd_zalloc(sizeof (restarter_inst_t)); 350 inst->ri_utmpx_prefix = startd_alloc(max_scf_value_size); 351 inst->ri_utmpx_prefix[0] = '\0'; 352 353 inst->ri_i.i_fmri = startd_alloc(strlen(name) + 1); 354 (void) strcpy((char *)inst->ri_i.i_fmri, name); 355 356 inst->ri_queue = startd_list_create(restarter_queue_pool, inst, 0); 357 358 /* 359 * id shouldn't be -1 since we use the same dictionary as graph.c, but 360 * just in case. 361 */ 362 inst->ri_id = (id != -1 ? id : dict_insert(name)); 363 364 special_online_hooks_get(name, &inst->ri_pre_online_hook, 365 &inst->ri_post_online_hook, &inst->ri_post_offline_hook); 366 367 scf_svc = safe_scf_service_create(h); 368 scf_inst = safe_scf_instance_create(h); 369 pg = safe_scf_pg_create(h); 370 svc_name = startd_alloc(max_scf_name_size); 371 inst_name = startd_alloc(max_scf_name_size); 372 373 rep_retry: 374 if (snap != NULL) 375 scf_snapshot_destroy(snap); 376 if (inst->ri_logstem != NULL) 377 startd_free(inst->ri_logstem, PATH_MAX); 378 if (inst->ri_common_name != NULL) 379 free(inst->ri_common_name); 380 if (inst->ri_C_common_name != NULL) 381 free(inst->ri_C_common_name); 382 snap = NULL; 383 inst->ri_logstem = NULL; 384 inst->ri_common_name = NULL; 385 inst->ri_C_common_name = NULL; 386 387 if (scf_handle_decode_fmri(h, name, NULL, scf_svc, scf_inst, NULL, 388 NULL, SCF_DECODE_FMRI_EXACT) != 0) { 389 switch (scf_error()) { 390 case SCF_ERROR_CONNECTION_BROKEN: 391 libscf_handle_rebind(h); 392 goto rep_retry; 393 394 case SCF_ERROR_NOT_FOUND: 395 goto deleted; 396 } 397 398 uu_die("Can't decode FMRI %s: %s\n", name, 399 scf_strerror(scf_error())); 400 } 401 402 /* 403 * If there's no running snapshot, then we execute using the editing 404 * snapshot. Pending snapshots will be taken later. 405 */ 406 snap = libscf_get_running_snapshot(scf_inst); 407 408 if ((scf_service_get_name(scf_svc, svc_name, max_scf_name_size) < 0) || 409 (scf_instance_get_name(scf_inst, inst_name, max_scf_name_size) < 410 0)) { 411 switch (scf_error()) { 412 case SCF_ERROR_NOT_SET: 413 break; 414 415 case SCF_ERROR_CONNECTION_BROKEN: 416 libscf_handle_rebind(h); 417 goto rep_retry; 418 419 default: 420 assert(0); 421 abort(); 422 } 423 424 goto deleted; 425 } 426 427 (void) snprintf(logfilebuf, PATH_MAX, "%s:%s", svc_name, inst_name); 428 for (c = logfilebuf; *c != '\0'; c++) 429 if (*c == '/') 430 *c = '-'; 431 432 inst->ri_logstem = startd_alloc(PATH_MAX); 433 (void) snprintf(inst->ri_logstem, PATH_MAX, "%s%s", logfilebuf, 434 LOG_SUFFIX); 435 436 /* 437 * If the restarter group is missing, use uninit/none. Otherwise, 438 * we're probably being restarted & don't want to mess up the states 439 * that are there. 440 */ 441 state = RESTARTER_STATE_UNINIT; 442 next_state = RESTARTER_STATE_NONE; 443 444 r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg); 445 if (r != 0) { 446 switch (scf_error()) { 447 case SCF_ERROR_CONNECTION_BROKEN: 448 libscf_handle_rebind(h); 449 goto rep_retry; 450 451 case SCF_ERROR_NOT_SET: 452 goto deleted; 453 454 case SCF_ERROR_NOT_FOUND: 455 /* 456 * This shouldn't happen since the graph engine should 457 * have initialized the state to uninitialized/none if 458 * there was no restarter pg. In case somebody 459 * deleted it, though.... 460 */ 461 do_commit_states = B_TRUE; 462 break; 463 464 default: 465 assert(0); 466 abort(); 467 } 468 } else { 469 r = libscf_read_states(pg, &state, &next_state); 470 if (r != 0) { 471 do_commit_states = B_TRUE; 472 } else { 473 if (next_state != RESTARTER_STATE_NONE) { 474 /* 475 * Force next_state to _NONE since we 476 * don't look for method processes. 477 */ 478 next_state = RESTARTER_STATE_NONE; 479 do_commit_states = B_TRUE; 480 } else { 481 /* 482 * The reason for transition will depend on 483 * state. 484 */ 485 if (st->st_initial == 0) 486 reason = restarter_str_startd_restart; 487 else if (state == RESTARTER_STATE_MAINT) 488 reason = restarter_str_bad_repo_state; 489 /* 490 * Inform the restarter of our state without 491 * changing the STIME in the repository. 492 */ 493 ps = startd_alloc(sizeof (*ps)); 494 inst->ri_i.i_state = ps->ps_state = state; 495 inst->ri_i.i_next_state = ps->ps_state_next = 496 next_state; 497 ps->ps_reason = reason; 498 499 graph_protocol_send_event(inst->ri_i.i_fmri, 500 GRAPH_UPDATE_STATE_CHANGE, ps); 501 502 do_commit_states = B_FALSE; 503 } 504 } 505 } 506 507 switch (libscf_get_startd_properties(scf_inst, snap, &inst->ri_flags, 508 &inst->ri_utmpx_prefix)) { 509 case 0: 510 break; 511 512 case ECONNABORTED: 513 libscf_handle_rebind(h); 514 goto rep_retry; 515 516 case ECANCELED: 517 goto deleted; 518 519 case ENOENT: 520 /* 521 * This is odd, because the graph engine should have required 522 * the general property group. So we'll just use default 523 * flags in anticipation of the graph engine sending us 524 * REMOVE_INSTANCE when it finds out that the general property 525 * group has been deleted. 526 */ 527 inst->ri_flags = RINST_CONTRACT; 528 break; 529 530 default: 531 assert(0); 532 abort(); 533 } 534 535 r = libscf_get_template_values(scf_inst, snap, 536 &inst->ri_common_name, &inst->ri_C_common_name); 537 538 /* 539 * Copy our names to smaller buffers to reduce our memory footprint. 540 */ 541 if (inst->ri_common_name != NULL) { 542 char *tmp = safe_strdup(inst->ri_common_name); 543 startd_free(inst->ri_common_name, max_scf_value_size); 544 inst->ri_common_name = tmp; 545 } 546 547 if (inst->ri_C_common_name != NULL) { 548 char *tmp = safe_strdup(inst->ri_C_common_name); 549 startd_free(inst->ri_C_common_name, max_scf_value_size); 550 inst->ri_C_common_name = tmp; 551 } 552 553 switch (r) { 554 case 0: 555 break; 556 557 case ECONNABORTED: 558 libscf_handle_rebind(h); 559 goto rep_retry; 560 561 case ECANCELED: 562 goto deleted; 563 564 case ECHILD: 565 case ENOENT: 566 break; 567 568 default: 569 assert(0); 570 abort(); 571 } 572 573 switch (libscf_read_method_ids(h, scf_inst, inst->ri_i.i_fmri, 574 &inst->ri_i.i_primary_ctid, &inst->ri_i.i_transient_ctid, 575 &start_pid)) { 576 case 0: 577 break; 578 579 case ECONNABORTED: 580 libscf_handle_rebind(h); 581 goto rep_retry; 582 583 case ECANCELED: 584 goto deleted; 585 586 default: 587 assert(0); 588 abort(); 589 } 590 591 if (inst->ri_i.i_primary_ctid >= 1) { 592 contract_hash_store(inst->ri_i.i_primary_ctid, inst->ri_id); 593 594 switch (check_contract(inst, B_TRUE, scf_inst)) { 595 case 0: 596 break; 597 598 case ECONNABORTED: 599 libscf_handle_rebind(h); 600 goto rep_retry; 601 602 case ECANCELED: 603 goto deleted; 604 605 default: 606 assert(0); 607 abort(); 608 } 609 } 610 611 if (inst->ri_i.i_transient_ctid >= 1) { 612 switch (check_contract(inst, B_FALSE, scf_inst)) { 613 case 0: 614 break; 615 616 case ECONNABORTED: 617 libscf_handle_rebind(h); 618 goto rep_retry; 619 620 case ECANCELED: 621 goto deleted; 622 623 default: 624 assert(0); 625 abort(); 626 } 627 } 628 629 /* No more failures we live through, so add it to the list. */ 630 (void) pthread_mutex_init(&inst->ri_lock, &mutex_attrs); 631 (void) pthread_mutex_init(&inst->ri_queue_lock, &mutex_attrs); 632 MUTEX_LOCK(&inst->ri_lock); 633 MUTEX_LOCK(&inst->ri_queue_lock); 634 635 (void) pthread_cond_init(&inst->ri_method_cv, NULL); 636 637 uu_list_node_init(inst, &inst->ri_link, restarter_instance_pool); 638 uu_list_insert(instance_list.ril_instance_list, inst, idx); 639 MUTEX_UNLOCK(&instance_list.ril_lock); 640 641 if (start_pid != -1 && 642 (inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT) { 643 int ret; 644 ret = wait_register(start_pid, inst->ri_i.i_fmri, 0, 1); 645 if (ret == -1) { 646 /* 647 * Implication: if we can't reregister the 648 * instance, we will start another one. Two 649 * instances may or may not result in a resource 650 * conflict. 651 */ 652 log_error(LOG_WARNING, 653 "%s: couldn't reregister %ld for wait\n", 654 inst->ri_i.i_fmri, start_pid); 655 } else if (ret == 1) { 656 /* 657 * Leading PID has exited. 658 */ 659 (void) stop_instance(h, inst, RSTOP_EXIT); 660 } 661 } 662 663 664 scf_pg_destroy(pg); 665 666 if (do_commit_states) 667 (void) restarter_instance_update_states(h, inst, state, 668 next_state, RERR_NONE, reason); 669 670 log_framework(LOG_DEBUG, "%s is a %s-style service\n", name, 671 service_style(inst->ri_flags)); 672 673 MUTEX_UNLOCK(&inst->ri_queue_lock); 674 MUTEX_UNLOCK(&inst->ri_lock); 675 676 startd_free(svc_name, max_scf_name_size); 677 startd_free(inst_name, max_scf_name_size); 678 scf_snapshot_destroy(snap); 679 scf_instance_destroy(scf_inst); 680 scf_service_destroy(scf_svc); 681 682 log_framework(LOG_DEBUG, "%s: inserted instance into restarter list\n", 683 name); 684 685 return (0); 686 687 deleted: 688 MUTEX_UNLOCK(&instance_list.ril_lock); 689 startd_free(inst_name, max_scf_name_size); 690 startd_free(svc_name, max_scf_name_size); 691 if (snap != NULL) 692 scf_snapshot_destroy(snap); 693 scf_pg_destroy(pg); 694 scf_instance_destroy(scf_inst); 695 scf_service_destroy(scf_svc); 696 startd_free((void *)inst->ri_i.i_fmri, strlen(inst->ri_i.i_fmri) + 1); 697 uu_list_destroy(inst->ri_queue); 698 if (inst->ri_logstem != NULL) 699 startd_free(inst->ri_logstem, PATH_MAX); 700 if (inst->ri_common_name != NULL) 701 free(inst->ri_common_name); 702 if (inst->ri_C_common_name != NULL) 703 free(inst->ri_C_common_name); 704 startd_free(inst->ri_utmpx_prefix, max_scf_value_size); 705 startd_free(inst, sizeof (restarter_inst_t)); 706 return (ENOENT); 707 } 708 709 static void 710 restarter_delete_inst(restarter_inst_t *ri) 711 { 712 int id; 713 restarter_inst_t *rip; 714 void *cookie = NULL; 715 restarter_instance_qentry_t *e; 716 717 assert(MUTEX_HELD(&ri->ri_lock)); 718 719 /* 720 * Must drop the instance lock so we can pick up the instance_list 721 * lock & remove the instance. 722 */ 723 id = ri->ri_id; 724 MUTEX_UNLOCK(&ri->ri_lock); 725 726 MUTEX_LOCK(&instance_list.ril_lock); 727 728 rip = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL); 729 if (rip == NULL) { 730 MUTEX_UNLOCK(&instance_list.ril_lock); 731 return; 732 } 733 734 assert(ri == rip); 735 736 uu_list_remove(instance_list.ril_instance_list, ri); 737 738 log_framework(LOG_DEBUG, "%s: deleted instance from restarter list\n", 739 ri->ri_i.i_fmri); 740 741 MUTEX_UNLOCK(&instance_list.ril_lock); 742 743 /* 744 * We can lock the instance without holding the instance_list lock 745 * since we removed the instance from the list. 746 */ 747 MUTEX_LOCK(&ri->ri_lock); 748 MUTEX_LOCK(&ri->ri_queue_lock); 749 750 if (ri->ri_i.i_primary_ctid >= 1) 751 contract_hash_remove(ri->ri_i.i_primary_ctid); 752 753 while (ri->ri_method_thread != 0 || ri->ri_method_waiters > 0) 754 (void) pthread_cond_wait(&ri->ri_method_cv, &ri->ri_lock); 755 756 while ((e = uu_list_teardown(ri->ri_queue, &cookie)) != NULL) 757 startd_free(e, sizeof (*e)); 758 uu_list_destroy(ri->ri_queue); 759 760 startd_free((void *)ri->ri_i.i_fmri, strlen(ri->ri_i.i_fmri) + 1); 761 startd_free(ri->ri_logstem, PATH_MAX); 762 if (ri->ri_common_name != NULL) 763 free(ri->ri_common_name); 764 if (ri->ri_C_common_name != NULL) 765 free(ri->ri_C_common_name); 766 startd_free(ri->ri_utmpx_prefix, max_scf_value_size); 767 (void) pthread_mutex_destroy(&ri->ri_lock); 768 (void) pthread_mutex_destroy(&ri->ri_queue_lock); 769 startd_free(ri, sizeof (restarter_inst_t)); 770 } 771 772 /* 773 * instance_is_wait_style() 774 * 775 * Returns 1 if the given instance is a "wait-style" service instance. 776 */ 777 int 778 instance_is_wait_style(restarter_inst_t *inst) 779 { 780 assert(MUTEX_HELD(&inst->ri_lock)); 781 return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT); 782 } 783 784 /* 785 * instance_is_transient_style() 786 * 787 * Returns 1 if the given instance is a transient service instance. 788 */ 789 int 790 instance_is_transient_style(restarter_inst_t *inst) 791 { 792 assert(MUTEX_HELD(&inst->ri_lock)); 793 return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_TRANSIENT); 794 } 795 796 /* 797 * instance_in_transition() 798 * Returns 1 if instance is in transition, 0 if not 799 */ 800 int 801 instance_in_transition(restarter_inst_t *inst) 802 { 803 assert(MUTEX_HELD(&inst->ri_lock)); 804 if (inst->ri_i.i_next_state == RESTARTER_STATE_NONE) 805 return (0); 806 return (1); 807 } 808 809 /* 810 * returns 1 if instance is already started, 0 if not 811 */ 812 static int 813 instance_started(restarter_inst_t *inst) 814 { 815 int ret; 816 817 assert(MUTEX_HELD(&inst->ri_lock)); 818 819 if (inst->ri_i.i_state == RESTARTER_STATE_ONLINE || 820 inst->ri_i.i_state == RESTARTER_STATE_DEGRADED) 821 ret = 1; 822 else 823 ret = 0; 824 825 return (ret); 826 } 827 828 /* 829 * Returns 830 * 0 - success 831 * ECONNRESET - success, but h was rebound 832 */ 833 int 834 restarter_instance_update_states(scf_handle_t *h, restarter_inst_t *ri, 835 restarter_instance_state_t new_state, 836 restarter_instance_state_t new_state_next, restarter_error_t err, 837 restarter_str_t reason) 838 { 839 protocol_states_t *states; 840 int e; 841 uint_t retry_count = 0, msecs = ALLOC_DELAY; 842 boolean_t rebound = B_FALSE; 843 int prev_state_online; 844 int state_online; 845 846 assert(MUTEX_HELD(&ri->ri_lock)); 847 848 prev_state_online = instance_started(ri); 849 850 retry: 851 e = _restarter_commit_states(h, &ri->ri_i, new_state, new_state_next, 852 restarter_get_str_short(reason)); 853 switch (e) { 854 case 0: 855 break; 856 857 case ENOMEM: 858 ++retry_count; 859 if (retry_count < ALLOC_RETRY) { 860 (void) poll(NULL, 0, msecs); 861 msecs *= ALLOC_DELAY_MULT; 862 goto retry; 863 } 864 865 /* Like startd_alloc(). */ 866 uu_die("Insufficient memory.\n"); 867 /* NOTREACHED */ 868 869 case ECONNABORTED: 870 libscf_handle_rebind(h); 871 rebound = B_TRUE; 872 goto retry; 873 874 case EPERM: 875 case EACCES: 876 case EROFS: 877 log_error(LOG_NOTICE, "Could not commit state change for %s " 878 "to repository: %s.\n", ri->ri_i.i_fmri, strerror(e)); 879 /* FALLTHROUGH */ 880 881 case ENOENT: 882 ri->ri_i.i_state = new_state; 883 ri->ri_i.i_next_state = new_state_next; 884 break; 885 886 case EINVAL: 887 default: 888 bad_error("_restarter_commit_states", e); 889 } 890 891 states = startd_alloc(sizeof (protocol_states_t)); 892 states->ps_state = new_state; 893 states->ps_state_next = new_state_next; 894 states->ps_err = err; 895 states->ps_reason = reason; 896 graph_protocol_send_event(ri->ri_i.i_fmri, GRAPH_UPDATE_STATE_CHANGE, 897 (void *)states); 898 899 state_online = instance_started(ri); 900 901 if (prev_state_online && !state_online) 902 ri->ri_post_offline_hook(); 903 else if (!prev_state_online && state_online) 904 ri->ri_post_online_hook(); 905 906 return (rebound ? ECONNRESET : 0); 907 } 908 909 void 910 restarter_mark_pending_snapshot(const char *fmri, uint_t flag) 911 { 912 restarter_inst_t *inst; 913 914 assert(flag == RINST_RETAKE_RUNNING || flag == RINST_RETAKE_START); 915 916 inst = inst_lookup_by_name(fmri); 917 if (inst == NULL) 918 return; 919 920 inst->ri_flags |= flag; 921 922 MUTEX_UNLOCK(&inst->ri_lock); 923 } 924 925 static void 926 restarter_take_pending_snapshots(scf_handle_t *h) 927 { 928 restarter_inst_t *inst; 929 int r; 930 931 MUTEX_LOCK(&instance_list.ril_lock); 932 933 for (inst = uu_list_first(instance_list.ril_instance_list); 934 inst != NULL; 935 inst = uu_list_next(instance_list.ril_instance_list, inst)) { 936 const char *fmri; 937 scf_instance_t *sinst = NULL; 938 939 MUTEX_LOCK(&inst->ri_lock); 940 941 /* 942 * This is where we'd check inst->ri_method_thread and if it 943 * were nonzero we'd wait in anticipation of another thread 944 * executing a method for inst. Doing so with the instance_list 945 * locked, though, leads to deadlock. Since taking a snapshot 946 * during that window won't hurt anything, we'll just continue. 947 */ 948 949 fmri = inst->ri_i.i_fmri; 950 951 if (inst->ri_flags & RINST_RETAKE_RUNNING) { 952 scf_snapshot_t *rsnap; 953 954 (void) libscf_fmri_get_instance(h, fmri, &sinst); 955 956 rsnap = libscf_get_or_make_running_snapshot(sinst, 957 fmri, B_FALSE); 958 959 scf_instance_destroy(sinst); 960 961 if (rsnap != NULL) 962 inst->ri_flags &= ~RINST_RETAKE_RUNNING; 963 964 scf_snapshot_destroy(rsnap); 965 } 966 967 if (inst->ri_flags & RINST_RETAKE_START) { 968 switch (r = libscf_snapshots_poststart(h, fmri, 969 B_FALSE)) { 970 case 0: 971 case ENOENT: 972 inst->ri_flags &= ~RINST_RETAKE_START; 973 break; 974 975 case ECONNABORTED: 976 break; 977 978 case EACCES: 979 default: 980 bad_error("libscf_snapshots_poststart", r); 981 } 982 } 983 984 MUTEX_UNLOCK(&inst->ri_lock); 985 } 986 987 MUTEX_UNLOCK(&instance_list.ril_lock); 988 } 989 990 /* ARGSUSED */ 991 void * 992 restarter_post_fsminimal_thread(void *unused) 993 { 994 scf_handle_t *h; 995 int r; 996 997 h = libscf_handle_create_bound_loop(); 998 999 for (;;) { 1000 r = libscf_create_self(h); 1001 if (r == 0) 1002 break; 1003 1004 assert(r == ECONNABORTED); 1005 libscf_handle_rebind(h); 1006 } 1007 1008 restarter_take_pending_snapshots(h); 1009 1010 (void) scf_handle_unbind(h); 1011 scf_handle_destroy(h); 1012 1013 return (NULL); 1014 } 1015 1016 /* 1017 * int stop_instance() 1018 * 1019 * Stop the instance identified by the instance given as the second argument, 1020 * for the cause stated. 1021 * 1022 * Returns 1023 * 0 - success 1024 * -1 - inst is in transition 1025 */ 1026 static int 1027 stop_instance(scf_handle_t *local_handle, restarter_inst_t *inst, 1028 stop_cause_t cause) 1029 { 1030 fork_info_t *info; 1031 const char *cp; 1032 int err; 1033 restarter_error_t re; 1034 restarter_str_t reason; 1035 restarter_instance_state_t new_state; 1036 1037 assert(MUTEX_HELD(&inst->ri_lock)); 1038 assert(inst->ri_method_thread == 0); 1039 1040 switch (cause) { 1041 case RSTOP_EXIT: 1042 re = RERR_RESTART; 1043 reason = restarter_str_ct_ev_exit; 1044 cp = "all processes in service exited"; 1045 break; 1046 case RSTOP_ERR_CFG: 1047 re = RERR_FAULT; 1048 reason = restarter_str_method_failed; 1049 cp = "service exited with a configuration error"; 1050 break; 1051 case RSTOP_ERR_EXIT: 1052 re = RERR_RESTART; 1053 reason = restarter_str_ct_ev_exit; 1054 cp = "service exited with an error"; 1055 break; 1056 case RSTOP_CORE: 1057 re = RERR_FAULT; 1058 reason = restarter_str_ct_ev_core; 1059 cp = "process dumped core"; 1060 break; 1061 case RSTOP_SIGNAL: 1062 re = RERR_FAULT; 1063 reason = restarter_str_ct_ev_signal; 1064 cp = "process received fatal signal from outside the service"; 1065 break; 1066 case RSTOP_HWERR: 1067 re = RERR_FAULT; 1068 reason = restarter_str_ct_ev_hwerr; 1069 cp = "process killed due to uncorrectable hardware error"; 1070 break; 1071 case RSTOP_DEPENDENCY: 1072 re = RERR_RESTART; 1073 reason = restarter_str_dependency_activity; 1074 cp = "dependency activity requires stop"; 1075 break; 1076 case RSTOP_DISABLE: 1077 re = RERR_RESTART; 1078 reason = restarter_str_disable_request; 1079 cp = "service disabled"; 1080 break; 1081 case RSTOP_RESTART: 1082 re = RERR_RESTART; 1083 reason = restarter_str_restart_request; 1084 cp = "service restarting"; 1085 break; 1086 default: 1087 #ifndef NDEBUG 1088 (void) fprintf(stderr, "Unknown cause %d at %s:%d.\n", 1089 cause, __FILE__, __LINE__); 1090 #endif 1091 abort(); 1092 } 1093 1094 /* Services in the disabled and maintenance state are ignored */ 1095 if (inst->ri_i.i_state == RESTARTER_STATE_MAINT || 1096 inst->ri_i.i_state == RESTARTER_STATE_DISABLED) { 1097 log_framework(LOG_DEBUG, 1098 "%s: stop_instance -> is maint/disabled\n", 1099 inst->ri_i.i_fmri); 1100 return (0); 1101 } 1102 1103 /* Already stopped instances are left alone */ 1104 if (instance_started(inst) == 0) { 1105 log_framework(LOG_DEBUG, "Restarter: %s is already stopped.\n", 1106 inst->ri_i.i_fmri); 1107 return (0); 1108 } 1109 1110 if (instance_in_transition(inst)) { 1111 /* requeue event by returning -1 */ 1112 log_framework(LOG_DEBUG, 1113 "Restarter: Not stopping %s, in transition.\n", 1114 inst->ri_i.i_fmri); 1115 return (-1); 1116 } 1117 1118 log_instance(inst, B_TRUE, "Stopping because %s.", cp); 1119 1120 log_framework(re == RERR_FAULT ? LOG_INFO : LOG_DEBUG, 1121 "%s: Instance stopping because %s.\n", inst->ri_i.i_fmri, cp); 1122 1123 if (instance_is_wait_style(inst) && 1124 (cause == RSTOP_EXIT || 1125 cause == RSTOP_ERR_CFG || 1126 cause == RSTOP_ERR_EXIT)) { 1127 /* 1128 * No need to stop instance, as child has exited; remove 1129 * contract and move the instance to the offline state. 1130 */ 1131 switch (err = restarter_instance_update_states(local_handle, 1132 inst, inst->ri_i.i_state, RESTARTER_STATE_OFFLINE, re, 1133 reason)) { 1134 case 0: 1135 case ECONNRESET: 1136 break; 1137 1138 default: 1139 bad_error("restarter_instance_update_states", err); 1140 } 1141 1142 if (cause == RSTOP_ERR_EXIT) { 1143 /* 1144 * The RSTOP_ERR_EXIT cause is set via the 1145 * wait_thread -> wait_remove code path when we have 1146 * a "wait" style svc that exited with an error. If 1147 * the svc is failing too quickly, we throttle it so 1148 * that we don't restart it more than once/second. 1149 * Since we know we're running in the wait thread its 1150 * ok to throttle it right here. 1151 */ 1152 (void) update_fault_count(inst, FAULT_COUNT_INCR); 1153 if (method_rate_critical(inst)) { 1154 log_instance(inst, B_TRUE, "Failing too " 1155 "quickly, throttling."); 1156 (void) sleep(WT_SVC_ERR_THROTTLE); 1157 } 1158 } else { 1159 (void) update_fault_count(inst, FAULT_COUNT_RESET); 1160 reset_start_times(inst); 1161 } 1162 1163 if (inst->ri_i.i_primary_ctid != 0) { 1164 inst->ri_m_inst = 1165 safe_scf_instance_create(local_handle); 1166 inst->ri_mi_deleted = B_FALSE; 1167 1168 libscf_reget_instance(inst); 1169 method_remove_contract(inst, B_TRUE, B_TRUE); 1170 1171 scf_instance_destroy(inst->ri_m_inst); 1172 inst->ri_m_inst = NULL; 1173 } 1174 1175 switch (err = restarter_instance_update_states(local_handle, 1176 inst, inst->ri_i.i_next_state, RESTARTER_STATE_NONE, re, 1177 reason)) { 1178 case 0: 1179 case ECONNRESET: 1180 break; 1181 1182 default: 1183 bad_error("restarter_instance_update_states", err); 1184 } 1185 1186 if (cause != RSTOP_ERR_CFG) 1187 return (0); 1188 } else if (instance_is_wait_style(inst) && re == RERR_RESTART) { 1189 /* 1190 * Stopping a wait service through means other than the pid 1191 * exiting should keep wait_thread() from restarting the 1192 * service, by removing it from the wait list. 1193 * We cannot remove it right now otherwise the process will 1194 * end up <defunct> so mark it to be ignored. 1195 */ 1196 wait_ignore_by_fmri(inst->ri_i.i_fmri); 1197 } 1198 1199 /* 1200 * There are some configuration errors which we cannot detect until we 1201 * try to run the method. For example, see exec_method() where the 1202 * restarter_set_method_context() call can return SMF_EXIT_ERR_CONFIG 1203 * in several cases. If this happens for a "wait-style" svc, 1204 * wait_remove() sets the cause as RSTOP_ERR_CFG so that we can detect 1205 * the configuration error and go into maintenance, even though it is 1206 * a "wait-style" svc. 1207 */ 1208 if (cause == RSTOP_ERR_CFG) 1209 new_state = RESTARTER_STATE_MAINT; 1210 else 1211 new_state = inst->ri_i.i_enabled ? 1212 RESTARTER_STATE_OFFLINE : RESTARTER_STATE_DISABLED; 1213 1214 switch (err = restarter_instance_update_states(local_handle, inst, 1215 inst->ri_i.i_state, new_state, RERR_NONE, reason)) { 1216 case 0: 1217 case ECONNRESET: 1218 break; 1219 1220 default: 1221 bad_error("restarter_instance_update_states", err); 1222 } 1223 1224 info = startd_zalloc(sizeof (fork_info_t)); 1225 1226 info->sf_id = inst->ri_id; 1227 info->sf_method_type = METHOD_STOP; 1228 info->sf_event_type = re; 1229 info->sf_reason = reason; 1230 inst->ri_method_thread = startd_thread_create(method_thread, info); 1231 1232 return (0); 1233 } 1234 1235 /* 1236 * Returns 1237 * ENOENT - fmri is not in instance_list 1238 * 0 - success 1239 * ECONNRESET - success, though handle was rebound 1240 * -1 - instance is in transition 1241 */ 1242 int 1243 stop_instance_fmri(scf_handle_t *h, const char *fmri, uint_t flags) 1244 { 1245 restarter_inst_t *rip; 1246 int r; 1247 1248 rip = inst_lookup_by_name(fmri); 1249 if (rip == NULL) 1250 return (ENOENT); 1251 1252 r = stop_instance(h, rip, flags); 1253 1254 MUTEX_UNLOCK(&rip->ri_lock); 1255 1256 return (r); 1257 } 1258 1259 static void 1260 unmaintain_instance(scf_handle_t *h, restarter_inst_t *rip, 1261 unmaint_cause_t cause) 1262 { 1263 ctid_t ctid; 1264 scf_instance_t *inst; 1265 int r; 1266 uint_t tries = 0, msecs = ALLOC_DELAY; 1267 const char *cp; 1268 restarter_str_t reason; 1269 1270 assert(MUTEX_HELD(&rip->ri_lock)); 1271 1272 if (rip->ri_i.i_state != RESTARTER_STATE_MAINT) { 1273 log_error(LOG_DEBUG, "Restarter: " 1274 "Ignoring maintenance off command because %s is not in the " 1275 "maintenance state.\n", rip->ri_i.i_fmri); 1276 return; 1277 } 1278 1279 switch (cause) { 1280 case RUNMAINT_CLEAR: 1281 cp = "clear requested"; 1282 reason = restarter_str_clear_request; 1283 break; 1284 case RUNMAINT_DISABLE: 1285 cp = "disable requested"; 1286 reason = restarter_str_disable_request; 1287 break; 1288 default: 1289 #ifndef NDEBUG 1290 (void) fprintf(stderr, "Uncaught case for %d at %s:%d.\n", 1291 cause, __FILE__, __LINE__); 1292 #endif 1293 abort(); 1294 } 1295 1296 log_instance(rip, B_TRUE, "Leaving maintenance because %s.", 1297 cp); 1298 log_framework(LOG_DEBUG, "%s: Instance leaving maintenance because " 1299 "%s.\n", rip->ri_i.i_fmri, cp); 1300 1301 (void) restarter_instance_update_states(h, rip, RESTARTER_STATE_UNINIT, 1302 RESTARTER_STATE_NONE, RERR_RESTART, reason); 1303 1304 /* 1305 * If we did ADMIN_MAINT_ON_IMMEDIATE, then there might still be 1306 * a primary contract. 1307 */ 1308 if (rip->ri_i.i_primary_ctid == 0) 1309 return; 1310 1311 ctid = rip->ri_i.i_primary_ctid; 1312 contract_abandon(ctid); 1313 rip->ri_i.i_primary_ctid = 0; 1314 1315 rep_retry: 1316 switch (r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst)) { 1317 case 0: 1318 break; 1319 1320 case ECONNABORTED: 1321 libscf_handle_rebind(h); 1322 goto rep_retry; 1323 1324 case ENOENT: 1325 /* Must have been deleted. */ 1326 return; 1327 1328 case EINVAL: 1329 case ENOTSUP: 1330 default: 1331 bad_error("libscf_handle_rebind", r); 1332 } 1333 1334 again: 1335 r = restarter_remove_contract(inst, ctid, RESTARTER_CONTRACT_PRIMARY); 1336 switch (r) { 1337 case 0: 1338 break; 1339 1340 case ENOMEM: 1341 ++tries; 1342 if (tries < ALLOC_RETRY) { 1343 (void) poll(NULL, 0, msecs); 1344 msecs *= ALLOC_DELAY_MULT; 1345 goto again; 1346 } 1347 1348 uu_die("Insufficient memory.\n"); 1349 /* NOTREACHED */ 1350 1351 case ECONNABORTED: 1352 scf_instance_destroy(inst); 1353 libscf_handle_rebind(h); 1354 goto rep_retry; 1355 1356 case ECANCELED: 1357 break; 1358 1359 case EPERM: 1360 case EACCES: 1361 case EROFS: 1362 log_error(LOG_INFO, 1363 "Could not remove contract id %lu for %s (%s).\n", ctid, 1364 rip->ri_i.i_fmri, strerror(r)); 1365 break; 1366 1367 case EINVAL: 1368 case EBADF: 1369 default: 1370 bad_error("restarter_remove_contract", r); 1371 } 1372 1373 scf_instance_destroy(inst); 1374 } 1375 1376 /* 1377 * enable_inst() 1378 * Set inst->ri_i.i_enabled. Expects 'e' to be _ENABLE, _DISABLE, or 1379 * _ADMIN_DISABLE. If the event is _ENABLE and inst is uninitialized or 1380 * disabled, move it to offline. If the event is _DISABLE or 1381 * _ADMIN_DISABLE, make sure inst will move to disabled. 1382 * 1383 * Returns 1384 * 0 - success 1385 * ECONNRESET - h was rebound 1386 */ 1387 static int 1388 enable_inst(scf_handle_t *h, restarter_inst_t *inst, 1389 restarter_instance_qentry_t *riq) 1390 { 1391 restarter_instance_state_t state; 1392 restarter_event_type_t e = riq->riq_type; 1393 restarter_str_t reason = restarter_str_per_configuration; 1394 int r; 1395 1396 assert(MUTEX_HELD(&inst->ri_lock)); 1397 assert(e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE || 1398 e == RESTARTER_EVENT_TYPE_DISABLE || 1399 e == RESTARTER_EVENT_TYPE_ENABLE); 1400 assert(instance_in_transition(inst) == 0); 1401 1402 state = inst->ri_i.i_state; 1403 1404 if (e == RESTARTER_EVENT_TYPE_ENABLE) { 1405 inst->ri_i.i_enabled = 1; 1406 1407 if (state == RESTARTER_STATE_UNINIT || 1408 state == RESTARTER_STATE_DISABLED) { 1409 /* 1410 * B_FALSE: Don't log an error if the log_instance() 1411 * fails because it will fail on the miniroot before 1412 * install-discovery runs. 1413 */ 1414 log_instance(inst, B_FALSE, "Enabled."); 1415 log_framework(LOG_DEBUG, "%s: Instance enabled.\n", 1416 inst->ri_i.i_fmri); 1417 1418 /* 1419 * If we are coming from DISABLED, it was obviously an 1420 * enable request. If we are coming from UNINIT, it may 1421 * have been a sevice in MAINT that was cleared. 1422 */ 1423 if (riq->riq_reason == restarter_str_clear_request) 1424 reason = restarter_str_clear_request; 1425 else if (state == RESTARTER_STATE_DISABLED) 1426 reason = restarter_str_enable_request; 1427 (void) restarter_instance_update_states(h, inst, 1428 RESTARTER_STATE_OFFLINE, RESTARTER_STATE_NONE, 1429 RERR_NONE, reason); 1430 } else { 1431 log_framework(LOG_DEBUG, "Restarter: " 1432 "Not changing state of %s for enable command.\n", 1433 inst->ri_i.i_fmri); 1434 } 1435 } else { 1436 inst->ri_i.i_enabled = 0; 1437 1438 switch (state) { 1439 case RESTARTER_STATE_ONLINE: 1440 case RESTARTER_STATE_DEGRADED: 1441 r = stop_instance(h, inst, RSTOP_DISABLE); 1442 return (r == ECONNRESET ? 0 : r); 1443 1444 case RESTARTER_STATE_OFFLINE: 1445 case RESTARTER_STATE_UNINIT: 1446 if (inst->ri_i.i_primary_ctid != 0) { 1447 inst->ri_m_inst = safe_scf_instance_create(h); 1448 inst->ri_mi_deleted = B_FALSE; 1449 1450 libscf_reget_instance(inst); 1451 method_remove_contract(inst, B_TRUE, B_TRUE); 1452 1453 scf_instance_destroy(inst->ri_m_inst); 1454 } 1455 /* B_FALSE: See log_instance(..., "Enabled."); above */ 1456 log_instance(inst, B_FALSE, "Disabled."); 1457 log_framework(LOG_DEBUG, "%s: Instance disabled.\n", 1458 inst->ri_i.i_fmri); 1459 1460 /* 1461 * If we are coming from OFFLINE, it was obviously a 1462 * disable request. But if we are coming from 1463 * UNINIT, it may have been a disable request for a 1464 * service in MAINT. 1465 */ 1466 if (riq->riq_reason == restarter_str_disable_request || 1467 state == RESTARTER_STATE_OFFLINE) 1468 reason = restarter_str_disable_request; 1469 (void) restarter_instance_update_states(h, inst, 1470 RESTARTER_STATE_DISABLED, RESTARTER_STATE_NONE, 1471 RERR_RESTART, reason); 1472 return (0); 1473 1474 case RESTARTER_STATE_DISABLED: 1475 break; 1476 1477 case RESTARTER_STATE_MAINT: 1478 /* 1479 * We only want to pull the instance out of maintenance 1480 * if the disable is on adminstrative request. The 1481 * graph engine sends _DISABLE events whenever a 1482 * service isn't in the disabled state, and we don't 1483 * want to pull the service out of maintenance if, 1484 * for example, it is there due to a dependency cycle. 1485 */ 1486 if (e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE) 1487 unmaintain_instance(h, inst, RUNMAINT_DISABLE); 1488 break; 1489 1490 default: 1491 #ifndef NDEBUG 1492 (void) fprintf(stderr, "Restarter instance %s has " 1493 "unknown state %d.\n", inst->ri_i.i_fmri, state); 1494 #endif 1495 abort(); 1496 } 1497 } 1498 1499 return (0); 1500 } 1501 1502 static void 1503 start_instance(scf_handle_t *local_handle, restarter_inst_t *inst, 1504 int32_t reason) 1505 { 1506 fork_info_t *info; 1507 restarter_str_t new_reason; 1508 1509 assert(MUTEX_HELD(&inst->ri_lock)); 1510 assert(instance_in_transition(inst) == 0); 1511 assert(inst->ri_method_thread == 0); 1512 1513 log_framework(LOG_DEBUG, "%s: trying to start instance\n", 1514 inst->ri_i.i_fmri); 1515 1516 /* 1517 * We want to keep the original reason for restarts and clear actions 1518 */ 1519 switch (reason) { 1520 case restarter_str_restart_request: 1521 case restarter_str_clear_request: 1522 new_reason = reason; 1523 break; 1524 default: 1525 new_reason = restarter_str_dependencies_satisfied; 1526 } 1527 1528 /* Services in the disabled and maintenance state are ignored */ 1529 if (inst->ri_i.i_state == RESTARTER_STATE_MAINT || 1530 inst->ri_i.i_state == RESTARTER_STATE_DISABLED || 1531 inst->ri_i.i_enabled == 0) { 1532 log_framework(LOG_DEBUG, 1533 "%s: start_instance -> is maint/disabled\n", 1534 inst->ri_i.i_fmri); 1535 return; 1536 } 1537 1538 /* Already started instances are left alone */ 1539 if (instance_started(inst) == 1) { 1540 log_framework(LOG_DEBUG, 1541 "%s: start_instance -> is already started\n", 1542 inst->ri_i.i_fmri); 1543 return; 1544 } 1545 1546 log_framework(LOG_DEBUG, "%s: starting instance.\n", inst->ri_i.i_fmri); 1547 1548 (void) restarter_instance_update_states(local_handle, inst, 1549 inst->ri_i.i_state, RESTARTER_STATE_ONLINE, RERR_NONE, new_reason); 1550 1551 info = startd_zalloc(sizeof (fork_info_t)); 1552 1553 info->sf_id = inst->ri_id; 1554 info->sf_method_type = METHOD_START; 1555 info->sf_event_type = RERR_NONE; 1556 info->sf_reason = new_reason; 1557 inst->ri_method_thread = startd_thread_create(method_thread, info); 1558 } 1559 1560 static int 1561 event_from_tty(scf_handle_t *h, restarter_inst_t *rip) 1562 { 1563 scf_instance_t *inst; 1564 int ret = 0; 1565 1566 if (libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst)) 1567 return (-1); 1568 1569 ret = restarter_inst_ractions_from_tty(inst); 1570 1571 scf_instance_destroy(inst); 1572 return (ret); 1573 } 1574 1575 static boolean_t 1576 restart_dump(scf_handle_t *h, restarter_inst_t *rip) 1577 { 1578 scf_instance_t *inst; 1579 boolean_t ret = B_FALSE; 1580 1581 if (libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst)) 1582 return (-1); 1583 1584 if (restarter_inst_dump(inst) == 1) 1585 ret = B_TRUE; 1586 1587 scf_instance_destroy(inst); 1588 return (ret); 1589 } 1590 1591 static void 1592 maintain_instance(scf_handle_t *h, restarter_inst_t *rip, int immediate, 1593 restarter_str_t reason) 1594 { 1595 fork_info_t *info; 1596 scf_instance_t *scf_inst = NULL; 1597 1598 assert(MUTEX_HELD(&rip->ri_lock)); 1599 assert(reason != restarter_str_none); 1600 assert(rip->ri_method_thread == 0); 1601 1602 log_instance(rip, B_TRUE, "Stopping for maintenance due to %s.", 1603 restarter_get_str_short(reason)); 1604 log_framework(LOG_DEBUG, "%s: stopping for maintenance due to %s.\n", 1605 rip->ri_i.i_fmri, restarter_get_str_short(reason)); 1606 1607 /* Services in the maintenance state are ignored */ 1608 if (rip->ri_i.i_state == RESTARTER_STATE_MAINT) { 1609 log_framework(LOG_DEBUG, 1610 "%s: maintain_instance -> is already in maintenance\n", 1611 rip->ri_i.i_fmri); 1612 return; 1613 } 1614 1615 /* 1616 * If reason state is restarter_str_service_request and 1617 * restarter_actions/auxiliary_fmri property is set with a valid fmri, 1618 * copy the fmri to restarter/auxiliary_fmri so svcs -x can use. 1619 */ 1620 if (reason == restarter_str_service_request && 1621 libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &scf_inst) == 0) { 1622 if (restarter_inst_validate_ractions_aux_fmri(scf_inst) == 0) { 1623 if (restarter_inst_set_aux_fmri(scf_inst)) 1624 log_framework(LOG_DEBUG, "%s: " 1625 "restarter_inst_set_aux_fmri failed: ", 1626 rip->ri_i.i_fmri); 1627 } else { 1628 log_framework(LOG_DEBUG, "%s: " 1629 "restarter_inst_validate_ractions_aux_fmri " 1630 "failed: ", rip->ri_i.i_fmri); 1631 1632 if (restarter_inst_reset_aux_fmri(scf_inst)) 1633 log_framework(LOG_DEBUG, "%s: " 1634 "restarter_inst_reset_aux_fmri failed: ", 1635 rip->ri_i.i_fmri); 1636 } 1637 scf_instance_destroy(scf_inst); 1638 } 1639 1640 if (immediate || !instance_started(rip)) { 1641 if (rip->ri_i.i_primary_ctid != 0) { 1642 rip->ri_m_inst = safe_scf_instance_create(h); 1643 rip->ri_mi_deleted = B_FALSE; 1644 1645 libscf_reget_instance(rip); 1646 method_remove_contract(rip, B_TRUE, B_TRUE); 1647 1648 scf_instance_destroy(rip->ri_m_inst); 1649 } 1650 1651 (void) restarter_instance_update_states(h, rip, 1652 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_RESTART, 1653 reason); 1654 return; 1655 } 1656 1657 (void) restarter_instance_update_states(h, rip, rip->ri_i.i_state, 1658 RESTARTER_STATE_MAINT, RERR_NONE, reason); 1659 1660 log_transition(rip, MAINT_REQUESTED); 1661 1662 info = startd_zalloc(sizeof (*info)); 1663 info->sf_id = rip->ri_id; 1664 info->sf_method_type = METHOD_STOP; 1665 info->sf_event_type = RERR_RESTART; 1666 info->sf_reason = reason; 1667 rip->ri_method_thread = startd_thread_create(method_thread, info); 1668 } 1669 1670 static void 1671 refresh_instance(scf_handle_t *h, restarter_inst_t *rip) 1672 { 1673 scf_instance_t *inst; 1674 scf_snapshot_t *snap; 1675 fork_info_t *info; 1676 int r; 1677 1678 assert(MUTEX_HELD(&rip->ri_lock)); 1679 1680 log_instance(rip, B_TRUE, "Rereading configuration."); 1681 log_framework(LOG_DEBUG, "%s: rereading configuration.\n", 1682 rip->ri_i.i_fmri); 1683 1684 rep_retry: 1685 r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst); 1686 switch (r) { 1687 case 0: 1688 break; 1689 1690 case ECONNABORTED: 1691 libscf_handle_rebind(h); 1692 goto rep_retry; 1693 1694 case ENOENT: 1695 /* Must have been deleted. */ 1696 return; 1697 1698 case EINVAL: 1699 case ENOTSUP: 1700 default: 1701 bad_error("libscf_fmri_get_instance", r); 1702 } 1703 1704 snap = libscf_get_running_snapshot(inst); 1705 1706 r = libscf_get_startd_properties(inst, snap, &rip->ri_flags, 1707 &rip->ri_utmpx_prefix); 1708 switch (r) { 1709 case 0: 1710 log_framework(LOG_DEBUG, "%s is a %s-style service\n", 1711 rip->ri_i.i_fmri, service_style(rip->ri_flags)); 1712 break; 1713 1714 case ECONNABORTED: 1715 scf_instance_destroy(inst); 1716 scf_snapshot_destroy(snap); 1717 libscf_handle_rebind(h); 1718 goto rep_retry; 1719 1720 case ECANCELED: 1721 case ENOENT: 1722 /* Succeed in anticipation of REMOVE_INSTANCE. */ 1723 break; 1724 1725 default: 1726 bad_error("libscf_get_startd_properties", r); 1727 } 1728 1729 if (instance_started(rip)) { 1730 /* Refresh does not change the state. */ 1731 (void) restarter_instance_update_states(h, rip, 1732 rip->ri_i.i_state, rip->ri_i.i_state, RERR_NONE, 1733 restarter_str_refresh); 1734 1735 info = startd_zalloc(sizeof (*info)); 1736 info->sf_id = rip->ri_id; 1737 info->sf_method_type = METHOD_REFRESH; 1738 info->sf_event_type = RERR_REFRESH; 1739 info->sf_reason = NULL; 1740 1741 assert(rip->ri_method_thread == 0); 1742 rip->ri_method_thread = 1743 startd_thread_create(method_thread, info); 1744 } 1745 1746 scf_snapshot_destroy(snap); 1747 scf_instance_destroy(inst); 1748 } 1749 1750 const char *event_names[] = { "INVALID", "ADD_INSTANCE", "REMOVE_INSTANCE", 1751 "ENABLE", "DISABLE", "ADMIN_DEGRADED", "ADMIN_REFRESH", 1752 "ADMIN_RESTART", "ADMIN_MAINT_OFF", "ADMIN_MAINT_ON", 1753 "ADMIN_MAINT_ON_IMMEDIATE", "STOP", "START", "DEPENDENCY_CYCLE", 1754 "INVALID_DEPENDENCY", "ADMIN_DISABLE", "STOP_RESET" 1755 }; 1756 1757 /* 1758 * void *restarter_process_events() 1759 * 1760 * Called in a separate thread to process the events on an instance's 1761 * queue. Empties the queue completely, and tries to keep the thread 1762 * around for a little while after the queue is empty to save on 1763 * startup costs. 1764 */ 1765 static void * 1766 restarter_process_events(void *arg) 1767 { 1768 scf_handle_t *h; 1769 restarter_instance_qentry_t *event; 1770 restarter_inst_t *rip; 1771 char *fmri = (char *)arg; 1772 struct timespec to; 1773 1774 assert(fmri != NULL); 1775 1776 h = libscf_handle_create_bound_loop(); 1777 1778 /* grab the queue lock */ 1779 rip = inst_lookup_queue(fmri); 1780 if (rip == NULL) 1781 goto out; 1782 1783 again: 1784 1785 while ((event = uu_list_first(rip->ri_queue)) != NULL) { 1786 restarter_inst_t *inst; 1787 1788 /* drop the queue lock */ 1789 MUTEX_UNLOCK(&rip->ri_queue_lock); 1790 1791 /* 1792 * Grab the inst lock -- this waits until any outstanding 1793 * method finishes running. 1794 */ 1795 inst = inst_lookup_by_name(fmri); 1796 if (inst == NULL) { 1797 /* Getting deleted in the middle isn't an error. */ 1798 goto cont; 1799 } 1800 1801 assert(instance_in_transition(inst) == 0); 1802 1803 /* process the event */ 1804 switch (event->riq_type) { 1805 case RESTARTER_EVENT_TYPE_ENABLE: 1806 case RESTARTER_EVENT_TYPE_DISABLE: 1807 (void) enable_inst(h, inst, event); 1808 break; 1809 1810 case RESTARTER_EVENT_TYPE_ADMIN_DISABLE: 1811 if (enable_inst(h, inst, event) == 0) 1812 reset_start_times(inst); 1813 break; 1814 1815 case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE: 1816 restarter_delete_inst(inst); 1817 inst = NULL; 1818 goto cont; 1819 1820 case RESTARTER_EVENT_TYPE_STOP_RESET: 1821 reset_start_times(inst); 1822 /* FALLTHROUGH */ 1823 case RESTARTER_EVENT_TYPE_STOP: 1824 (void) stop_instance(h, inst, RSTOP_DEPENDENCY); 1825 break; 1826 1827 case RESTARTER_EVENT_TYPE_START: 1828 start_instance(h, inst, event->riq_reason); 1829 break; 1830 1831 case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE: 1832 maintain_instance(h, inst, 0, 1833 restarter_str_dependency_cycle); 1834 break; 1835 1836 case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY: 1837 maintain_instance(h, inst, 0, 1838 restarter_str_invalid_dependency); 1839 break; 1840 1841 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON: 1842 if (event_from_tty(h, inst) == 0) 1843 maintain_instance(h, inst, 0, 1844 restarter_str_service_request); 1845 else 1846 maintain_instance(h, inst, 0, 1847 restarter_str_administrative_request); 1848 break; 1849 1850 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE: 1851 if (event_from_tty(h, inst) == 0) 1852 maintain_instance(h, inst, 1, 1853 restarter_str_service_request); 1854 else 1855 maintain_instance(h, inst, 1, 1856 restarter_str_administrative_request); 1857 break; 1858 1859 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF: 1860 unmaintain_instance(h, inst, RUNMAINT_CLEAR); 1861 reset_start_times(inst); 1862 break; 1863 1864 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH: 1865 refresh_instance(h, inst); 1866 break; 1867 1868 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED: 1869 log_framework(LOG_WARNING, "Restarter: " 1870 "%s command (for %s) unimplemented.\n", 1871 event_names[event->riq_type], inst->ri_i.i_fmri); 1872 break; 1873 1874 case RESTARTER_EVENT_TYPE_ADMIN_RESTART: 1875 if (!instance_started(inst)) { 1876 log_framework(LOG_DEBUG, "Restarter: " 1877 "Not restarting %s; not running.\n", 1878 inst->ri_i.i_fmri); 1879 } else { 1880 /* 1881 * Stop the instance. If it can be restarted, 1882 * the graph engine will send a new event. 1883 */ 1884 if (restart_dump(h, inst)) { 1885 (void) contract_kill( 1886 inst->ri_i.i_primary_ctid, SIGABRT, 1887 inst->ri_i.i_fmri); 1888 } else if (stop_instance(h, inst, 1889 RSTOP_RESTART) == 0) { 1890 reset_start_times(inst); 1891 } 1892 } 1893 break; 1894 1895 case RESTARTER_EVENT_TYPE_ADD_INSTANCE: 1896 default: 1897 #ifndef NDEBUG 1898 uu_warn("%s:%d: Bad restarter event %d. " 1899 "Aborting.\n", __FILE__, __LINE__, event->riq_type); 1900 #endif 1901 abort(); 1902 } 1903 1904 assert(inst != NULL); 1905 MUTEX_UNLOCK(&inst->ri_lock); 1906 1907 cont: 1908 /* grab the queue lock */ 1909 rip = inst_lookup_queue(fmri); 1910 if (rip == NULL) 1911 goto out; 1912 1913 /* delete the event */ 1914 uu_list_remove(rip->ri_queue, event); 1915 startd_free(event, sizeof (restarter_instance_qentry_t)); 1916 } 1917 1918 assert(rip != NULL); 1919 1920 /* 1921 * Try to preserve the thread for a little while for future use. 1922 */ 1923 to.tv_sec = 3; 1924 to.tv_nsec = 0; 1925 (void) pthread_cond_reltimedwait_np(&rip->ri_queue_cv, 1926 &rip->ri_queue_lock, &to); 1927 1928 if (uu_list_first(rip->ri_queue) != NULL) 1929 goto again; 1930 1931 rip->ri_queue_thread = 0; 1932 MUTEX_UNLOCK(&rip->ri_queue_lock); 1933 1934 out: 1935 (void) scf_handle_unbind(h); 1936 scf_handle_destroy(h); 1937 free(fmri); 1938 return (NULL); 1939 } 1940 1941 static int 1942 is_admin_event(restarter_event_type_t t) { 1943 1944 switch (t) { 1945 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON: 1946 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE: 1947 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF: 1948 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH: 1949 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED: 1950 case RESTARTER_EVENT_TYPE_ADMIN_RESTART: 1951 return (1); 1952 default: 1953 return (0); 1954 } 1955 } 1956 1957 static void 1958 restarter_queue_event(restarter_inst_t *ri, restarter_protocol_event_t *e) 1959 { 1960 restarter_instance_qentry_t *qe; 1961 int r; 1962 1963 assert(MUTEX_HELD(&ri->ri_queue_lock)); 1964 assert(!MUTEX_HELD(&ri->ri_lock)); 1965 1966 qe = startd_zalloc(sizeof (restarter_instance_qentry_t)); 1967 qe->riq_type = e->rpe_type; 1968 qe->riq_reason = e->rpe_reason; 1969 1970 uu_list_node_init(qe, &qe->riq_link, restarter_queue_pool); 1971 r = uu_list_insert_before(ri->ri_queue, NULL, qe); 1972 assert(r == 0); 1973 } 1974 1975 /* 1976 * void *restarter_event_thread() 1977 * 1978 * Handle incoming graph events by placing them on a per-instance 1979 * queue. We can't lock the main part of the instance structure, so 1980 * just modify the seprarately locked event queue portion. 1981 */ 1982 /*ARGSUSED*/ 1983 static void * 1984 restarter_event_thread(void *unused) 1985 { 1986 scf_handle_t *h; 1987 1988 /* 1989 * This is a new thread, and thus, gets its own handle 1990 * to the repository. 1991 */ 1992 h = libscf_handle_create_bound_loop(); 1993 1994 MUTEX_LOCK(&ru->restarter_update_lock); 1995 1996 /*CONSTCOND*/ 1997 while (1) { 1998 restarter_protocol_event_t *e; 1999 2000 while (ru->restarter_update_wakeup == 0) 2001 (void) pthread_cond_wait(&ru->restarter_update_cv, 2002 &ru->restarter_update_lock); 2003 2004 ru->restarter_update_wakeup = 0; 2005 2006 while ((e = restarter_event_dequeue()) != NULL) { 2007 restarter_inst_t *rip; 2008 char *fmri; 2009 2010 MUTEX_UNLOCK(&ru->restarter_update_lock); 2011 2012 /* 2013 * ADD_INSTANCE is special: there's likely no 2014 * instance structure yet, so we need to handle the 2015 * addition synchronously. 2016 */ 2017 switch (e->rpe_type) { 2018 case RESTARTER_EVENT_TYPE_ADD_INSTANCE: 2019 if (restarter_insert_inst(h, e->rpe_inst) != 0) 2020 log_error(LOG_INFO, "Restarter: " 2021 "Could not add %s.\n", e->rpe_inst); 2022 2023 MUTEX_LOCK(&st->st_load_lock); 2024 if (--st->st_load_instances == 0) 2025 (void) pthread_cond_broadcast( 2026 &st->st_load_cv); 2027 MUTEX_UNLOCK(&st->st_load_lock); 2028 2029 goto nolookup; 2030 } 2031 2032 /* 2033 * Lookup the instance, locking only the event queue. 2034 * Can't grab ri_lock here because it might be held 2035 * by a long-running method. 2036 */ 2037 rip = inst_lookup_queue(e->rpe_inst); 2038 if (rip == NULL) { 2039 log_error(LOG_INFO, "Restarter: " 2040 "Ignoring %s command for unknown service " 2041 "%s.\n", event_names[e->rpe_type], 2042 e->rpe_inst); 2043 goto nolookup; 2044 } 2045 2046 /* Keep ADMIN events from filling up the queue. */ 2047 if (is_admin_event(e->rpe_type) && 2048 uu_list_numnodes(rip->ri_queue) > 2049 RINST_QUEUE_THRESHOLD) { 2050 MUTEX_UNLOCK(&rip->ri_queue_lock); 2051 log_instance(rip, B_TRUE, "Instance event " 2052 "queue overflow. Dropping administrative " 2053 "request."); 2054 log_framework(LOG_DEBUG, "%s: Instance event " 2055 "queue overflow. Dropping administrative " 2056 "request.\n", rip->ri_i.i_fmri); 2057 goto nolookup; 2058 } 2059 2060 /* Now add the event to the instance queue. */ 2061 restarter_queue_event(rip, e); 2062 2063 if (rip->ri_queue_thread == 0) { 2064 /* 2065 * Start a thread if one isn't already 2066 * running. 2067 */ 2068 fmri = safe_strdup(e->rpe_inst); 2069 rip->ri_queue_thread = startd_thread_create( 2070 restarter_process_events, (void *)fmri); 2071 } else { 2072 /* 2073 * Signal the existing thread that there's 2074 * a new event. 2075 */ 2076 (void) pthread_cond_broadcast( 2077 &rip->ri_queue_cv); 2078 } 2079 2080 MUTEX_UNLOCK(&rip->ri_queue_lock); 2081 nolookup: 2082 restarter_event_release(e); 2083 2084 MUTEX_LOCK(&ru->restarter_update_lock); 2085 } 2086 } 2087 2088 /* 2089 * Unreachable for now -- there's currently no graceful cleanup 2090 * called on exit(). 2091 */ 2092 (void) scf_handle_unbind(h); 2093 scf_handle_destroy(h); 2094 return (NULL); 2095 } 2096 2097 static restarter_inst_t * 2098 contract_to_inst(ctid_t ctid) 2099 { 2100 restarter_inst_t *inst; 2101 int id; 2102 2103 id = lookup_inst_by_contract(ctid); 2104 if (id == -1) 2105 return (NULL); 2106 2107 inst = inst_lookup_by_id(id); 2108 if (inst != NULL) { 2109 /* 2110 * Since ri_lock isn't held by the contract id lookup, this 2111 * instance may have been restarted and now be in a new 2112 * contract, making the old contract no longer valid for this 2113 * instance. 2114 */ 2115 if (ctid != inst->ri_i.i_primary_ctid) { 2116 MUTEX_UNLOCK(&inst->ri_lock); 2117 inst = NULL; 2118 } 2119 } 2120 return (inst); 2121 } 2122 2123 /* 2124 * void contract_action() 2125 * Take action on contract events. 2126 */ 2127 static void 2128 contract_action(scf_handle_t *h, restarter_inst_t *inst, ctid_t id, 2129 uint32_t type) 2130 { 2131 const char *fmri = inst->ri_i.i_fmri; 2132 2133 assert(MUTEX_HELD(&inst->ri_lock)); 2134 2135 /* 2136 * If startd has stopped this contract, there is no need to 2137 * stop it again. 2138 */ 2139 if (inst->ri_i.i_primary_ctid > 0 && 2140 inst->ri_i.i_primary_ctid_stopped) 2141 return; 2142 2143 if ((type & (CT_PR_EV_EMPTY | CT_PR_EV_CORE | CT_PR_EV_SIGNAL 2144 | CT_PR_EV_HWERR)) == 0) { 2145 /* 2146 * There shouldn't be other events, since that's not how we set 2147 * the terms. Thus, just log an error and drive on. 2148 */ 2149 log_framework(LOG_NOTICE, 2150 "%s: contract %ld received unexpected critical event " 2151 "(%d)\n", fmri, id, type); 2152 return; 2153 } 2154 2155 assert(instance_in_transition(inst) == 0); 2156 2157 if (instance_is_wait_style(inst)) { 2158 /* 2159 * We ignore all events; if they impact the 2160 * process we're monitoring, then the 2161 * wait_thread will stop the instance. 2162 */ 2163 log_framework(LOG_DEBUG, 2164 "%s: ignoring contract event on wait-style service\n", 2165 fmri); 2166 } else { 2167 /* 2168 * A CT_PR_EV_EMPTY event is an RSTOP_EXIT request. 2169 */ 2170 switch (type) { 2171 case CT_PR_EV_EMPTY: 2172 (void) stop_instance(h, inst, RSTOP_EXIT); 2173 break; 2174 case CT_PR_EV_CORE: 2175 (void) stop_instance(h, inst, RSTOP_CORE); 2176 break; 2177 case CT_PR_EV_SIGNAL: 2178 (void) stop_instance(h, inst, RSTOP_SIGNAL); 2179 break; 2180 case CT_PR_EV_HWERR: 2181 (void) stop_instance(h, inst, RSTOP_HWERR); 2182 break; 2183 } 2184 } 2185 } 2186 2187 /* 2188 * void *restarter_contract_event_thread(void *) 2189 * Listens to the process contract bundle for critical events, taking action 2190 * on events from contracts we know we are responsible for. 2191 */ 2192 /*ARGSUSED*/ 2193 static void * 2194 restarter_contracts_event_thread(void *unused) 2195 { 2196 int fd, err; 2197 scf_handle_t *local_handle; 2198 2199 /* 2200 * Await graph load completion. That is, stop here, until we've scanned 2201 * the repository for contract - instance associations. 2202 */ 2203 MUTEX_LOCK(&st->st_load_lock); 2204 while (!(st->st_load_complete && st->st_load_instances == 0)) 2205 (void) pthread_cond_wait(&st->st_load_cv, &st->st_load_lock); 2206 MUTEX_UNLOCK(&st->st_load_lock); 2207 2208 /* 2209 * This is a new thread, and thus, gets its own handle 2210 * to the repository. 2211 */ 2212 if ((local_handle = libscf_handle_create_bound(SCF_VERSION)) == NULL) 2213 uu_die("Unable to bind a new repository handle: %s\n", 2214 scf_strerror(scf_error())); 2215 2216 fd = open64(CTFS_ROOT "/process/pbundle", O_RDONLY); 2217 if (fd == -1) 2218 uu_die("process bundle open failed"); 2219 2220 /* 2221 * Make sure we get all events (including those generated by configd 2222 * before this thread was started). 2223 */ 2224 err = ct_event_reset(fd); 2225 assert(err == 0); 2226 2227 for (;;) { 2228 int efd, sfd; 2229 ct_evthdl_t ev; 2230 uint32_t type; 2231 ctevid_t evid; 2232 ct_stathdl_t status; 2233 ctid_t ctid; 2234 restarter_inst_t *inst; 2235 uint64_t cookie; 2236 2237 if (err = ct_event_read_critical(fd, &ev)) { 2238 log_error(LOG_WARNING, 2239 "Error reading next contract event: %s", 2240 strerror(err)); 2241 continue; 2242 } 2243 2244 evid = ct_event_get_evid(ev); 2245 ctid = ct_event_get_ctid(ev); 2246 type = ct_event_get_type(ev); 2247 2248 /* Fetch cookie. */ 2249 if ((sfd = contract_open(ctid, "process", "status", O_RDONLY)) 2250 < 0) { 2251 ct_event_free(ev); 2252 continue; 2253 } 2254 2255 if (err = ct_status_read(sfd, CTD_COMMON, &status)) { 2256 log_framework(LOG_WARNING, "Could not get status for " 2257 "contract %ld: %s\n", ctid, strerror(err)); 2258 2259 startd_close(sfd); 2260 ct_event_free(ev); 2261 continue; 2262 } 2263 2264 cookie = ct_status_get_cookie(status); 2265 2266 log_framework(LOG_DEBUG, "Received event %d for ctid %ld " 2267 "cookie %lld\n", type, ctid, cookie); 2268 2269 ct_status_free(status); 2270 2271 startd_close(sfd); 2272 2273 /* 2274 * svc.configd(1M) restart handling performed by the 2275 * fork_configd_thread. We don't acknowledge, as that thread 2276 * will do so. 2277 */ 2278 if (cookie == CONFIGD_COOKIE) { 2279 ct_event_free(ev); 2280 continue; 2281 } 2282 2283 inst = NULL; 2284 if (storing_contract != 0 && 2285 (inst = contract_to_inst(ctid)) == NULL) { 2286 /* 2287 * This can happen for two reasons: 2288 * - method_run() has not yet stored the 2289 * the contract into the internal hash table. 2290 * - we receive an EMPTY event for an abandoned 2291 * contract. 2292 * If there is any contract in the process of 2293 * being stored into the hash table then re-read 2294 * the event later. 2295 */ 2296 log_framework(LOG_DEBUG, 2297 "Reset event %d for unknown " 2298 "contract id %ld\n", type, ctid); 2299 2300 /* don't go too fast */ 2301 (void) poll(NULL, 0, 100); 2302 2303 (void) ct_event_reset(fd); 2304 ct_event_free(ev); 2305 continue; 2306 } 2307 2308 /* 2309 * Do not call contract_to_inst() again if first 2310 * call succeeded. 2311 */ 2312 if (inst == NULL) 2313 inst = contract_to_inst(ctid); 2314 if (inst == NULL) { 2315 /* 2316 * This can happen if we receive an EMPTY 2317 * event for an abandoned contract. 2318 */ 2319 log_framework(LOG_DEBUG, 2320 "Received event %d for unknown contract id " 2321 "%ld\n", type, ctid); 2322 } else { 2323 log_framework(LOG_DEBUG, 2324 "Received event %d for contract id " 2325 "%ld (%s)\n", type, ctid, 2326 inst->ri_i.i_fmri); 2327 2328 contract_action(local_handle, inst, ctid, type); 2329 2330 MUTEX_UNLOCK(&inst->ri_lock); 2331 } 2332 2333 efd = contract_open(ct_event_get_ctid(ev), "process", "ctl", 2334 O_WRONLY); 2335 if (efd != -1) { 2336 (void) ct_ctl_ack(efd, evid); 2337 startd_close(efd); 2338 } 2339 2340 ct_event_free(ev); 2341 2342 } 2343 2344 /*NOTREACHED*/ 2345 return (NULL); 2346 } 2347 2348 /* 2349 * Timeout queue, processed by restarter_timeouts_event_thread(). 2350 */ 2351 timeout_queue_t *timeouts; 2352 static uu_list_pool_t *timeout_pool; 2353 2354 typedef struct timeout_update { 2355 pthread_mutex_t tu_lock; 2356 pthread_cond_t tu_cv; 2357 int tu_wakeup; 2358 } timeout_update_t; 2359 2360 timeout_update_t *tu; 2361 2362 static const char *timeout_ovr_svcs[] = { 2363 "svc:/system/manifest-import:default", 2364 "svc:/network/initial:default", 2365 "svc:/network/service:default", 2366 "svc:/system/rmtmpfiles:default", 2367 "svc:/network/loopback:default", 2368 "svc:/network/physical:default", 2369 "svc:/system/device/local:default", 2370 "svc:/system/filesystem/usr:default", 2371 "svc:/system/filesystem/minimal:default", 2372 "svc:/system/filesystem/local:default", 2373 NULL 2374 }; 2375 2376 int 2377 is_timeout_ovr(restarter_inst_t *inst) 2378 { 2379 int i; 2380 2381 for (i = 0; timeout_ovr_svcs[i] != NULL; ++i) { 2382 if (strcmp(inst->ri_i.i_fmri, timeout_ovr_svcs[i]) == 0) { 2383 log_instance(inst, B_TRUE, "Timeout override by " 2384 "svc.startd. Using infinite timeout."); 2385 return (1); 2386 } 2387 } 2388 2389 return (0); 2390 } 2391 2392 /*ARGSUSED*/ 2393 static int 2394 timeout_compare(const void *lc_arg, const void *rc_arg, void *private) 2395 { 2396 hrtime_t t1 = ((const timeout_entry_t *)lc_arg)->te_timeout; 2397 hrtime_t t2 = ((const timeout_entry_t *)rc_arg)->te_timeout; 2398 2399 if (t1 > t2) 2400 return (1); 2401 else if (t1 < t2) 2402 return (-1); 2403 return (0); 2404 } 2405 2406 void 2407 timeout_init() 2408 { 2409 timeouts = startd_zalloc(sizeof (timeout_queue_t)); 2410 2411 (void) pthread_mutex_init(&timeouts->tq_lock, &mutex_attrs); 2412 2413 timeout_pool = startd_list_pool_create("timeouts", 2414 sizeof (timeout_entry_t), offsetof(timeout_entry_t, te_link), 2415 timeout_compare, UU_LIST_POOL_DEBUG); 2416 assert(timeout_pool != NULL); 2417 2418 timeouts->tq_list = startd_list_create(timeout_pool, 2419 timeouts, UU_LIST_SORTED); 2420 assert(timeouts->tq_list != NULL); 2421 2422 tu = startd_zalloc(sizeof (timeout_update_t)); 2423 (void) pthread_cond_init(&tu->tu_cv, NULL); 2424 (void) pthread_mutex_init(&tu->tu_lock, &mutex_attrs); 2425 } 2426 2427 void 2428 timeout_insert(restarter_inst_t *inst, ctid_t cid, uint64_t timeout_sec) 2429 { 2430 hrtime_t now, timeout; 2431 timeout_entry_t *entry; 2432 uu_list_index_t idx; 2433 2434 assert(MUTEX_HELD(&inst->ri_lock)); 2435 2436 now = gethrtime(); 2437 2438 /* 2439 * If we overflow LLONG_MAX, we're never timing out anyways, so 2440 * just return. 2441 */ 2442 if (timeout_sec >= (LLONG_MAX - now) / 1000000000LL) { 2443 log_instance(inst, B_TRUE, "timeout_seconds too large, " 2444 "treating as infinite."); 2445 return; 2446 } 2447 2448 /* hrtime is in nanoseconds. Convert timeout_sec. */ 2449 timeout = now + (timeout_sec * 1000000000LL); 2450 2451 entry = startd_alloc(sizeof (timeout_entry_t)); 2452 entry->te_timeout = timeout; 2453 entry->te_ctid = cid; 2454 entry->te_fmri = safe_strdup(inst->ri_i.i_fmri); 2455 entry->te_logstem = safe_strdup(inst->ri_logstem); 2456 entry->te_fired = 0; 2457 /* Insert the calculated timeout time onto the queue. */ 2458 MUTEX_LOCK(&timeouts->tq_lock); 2459 (void) uu_list_find(timeouts->tq_list, entry, NULL, &idx); 2460 uu_list_node_init(entry, &entry->te_link, timeout_pool); 2461 uu_list_insert(timeouts->tq_list, entry, idx); 2462 MUTEX_UNLOCK(&timeouts->tq_lock); 2463 2464 assert(inst->ri_timeout == NULL); 2465 inst->ri_timeout = entry; 2466 2467 MUTEX_LOCK(&tu->tu_lock); 2468 tu->tu_wakeup = 1; 2469 (void) pthread_cond_broadcast(&tu->tu_cv); 2470 MUTEX_UNLOCK(&tu->tu_lock); 2471 } 2472 2473 2474 void 2475 timeout_remove(restarter_inst_t *inst, ctid_t cid) 2476 { 2477 assert(MUTEX_HELD(&inst->ri_lock)); 2478 2479 if (inst->ri_timeout == NULL) 2480 return; 2481 2482 assert(inst->ri_timeout->te_ctid == cid); 2483 2484 MUTEX_LOCK(&timeouts->tq_lock); 2485 uu_list_remove(timeouts->tq_list, inst->ri_timeout); 2486 MUTEX_UNLOCK(&timeouts->tq_lock); 2487 2488 free(inst->ri_timeout->te_fmri); 2489 free(inst->ri_timeout->te_logstem); 2490 startd_free(inst->ri_timeout, sizeof (timeout_entry_t)); 2491 inst->ri_timeout = NULL; 2492 } 2493 2494 static int 2495 timeout_now() 2496 { 2497 timeout_entry_t *e; 2498 hrtime_t now; 2499 int ret; 2500 2501 now = gethrtime(); 2502 2503 /* 2504 * Walk through the (sorted) timeouts list. While the timeout 2505 * at the head of the list is <= the current time, kill the 2506 * method. 2507 */ 2508 MUTEX_LOCK(&timeouts->tq_lock); 2509 2510 for (e = uu_list_first(timeouts->tq_list); 2511 e != NULL && e->te_timeout <= now; 2512 e = uu_list_next(timeouts->tq_list, e)) { 2513 log_framework(LOG_WARNING, "%s: Method or service exit timed " 2514 "out. Killing contract %ld.\n", e->te_fmri, e->te_ctid); 2515 log_instance_fmri(e->te_fmri, e->te_logstem, B_TRUE, 2516 "Method or service exit timed out. Killing contract %ld.", 2517 e->te_ctid); 2518 e->te_fired = 1; 2519 (void) contract_kill(e->te_ctid, SIGKILL, e->te_fmri); 2520 } 2521 2522 if (uu_list_numnodes(timeouts->tq_list) > 0) 2523 ret = 0; 2524 else 2525 ret = -1; 2526 2527 MUTEX_UNLOCK(&timeouts->tq_lock); 2528 2529 return (ret); 2530 } 2531 2532 /* 2533 * void *restarter_timeouts_event_thread(void *) 2534 * Responsible for monitoring the method timeouts. This thread must 2535 * be started before any methods are called. 2536 */ 2537 /*ARGSUSED*/ 2538 static void * 2539 restarter_timeouts_event_thread(void *unused) 2540 { 2541 /* 2542 * Timeouts are entered on a priority queue, which is processed by 2543 * this thread. As timeouts are specified in seconds, we'll do 2544 * the necessary processing every second, as long as the queue 2545 * is not empty. 2546 */ 2547 2548 /*CONSTCOND*/ 2549 while (1) { 2550 /* 2551 * As long as the timeout list isn't empty, process it 2552 * every second. 2553 */ 2554 if (timeout_now() == 0) { 2555 (void) sleep(1); 2556 continue; 2557 } 2558 2559 /* The list is empty, wait until we have more timeouts. */ 2560 MUTEX_LOCK(&tu->tu_lock); 2561 2562 while (tu->tu_wakeup == 0) 2563 (void) pthread_cond_wait(&tu->tu_cv, &tu->tu_lock); 2564 2565 tu->tu_wakeup = 0; 2566 MUTEX_UNLOCK(&tu->tu_lock); 2567 } 2568 2569 return (NULL); 2570 } 2571 2572 void 2573 restarter_start() 2574 { 2575 (void) startd_thread_create(restarter_timeouts_event_thread, NULL); 2576 (void) startd_thread_create(restarter_event_thread, NULL); 2577 (void) startd_thread_create(restarter_contracts_event_thread, NULL); 2578 (void) startd_thread_create(wait_thread, NULL); 2579 } 2580 2581 2582 void 2583 restarter_init() 2584 { 2585 restarter_instance_pool = startd_list_pool_create("restarter_instances", 2586 sizeof (restarter_inst_t), offsetof(restarter_inst_t, 2587 ri_link), restarter_instance_compare, UU_LIST_POOL_DEBUG); 2588 (void) memset(&instance_list, 0, sizeof (instance_list)); 2589 2590 (void) pthread_mutex_init(&instance_list.ril_lock, &mutex_attrs); 2591 instance_list.ril_instance_list = startd_list_create( 2592 restarter_instance_pool, &instance_list, UU_LIST_SORTED); 2593 2594 restarter_queue_pool = startd_list_pool_create( 2595 "restarter_instance_queue", sizeof (restarter_instance_qentry_t), 2596 offsetof(restarter_instance_qentry_t, riq_link), NULL, 2597 UU_LIST_POOL_DEBUG); 2598 2599 contract_list_pool = startd_list_pool_create( 2600 "contract_list", sizeof (contract_entry_t), 2601 offsetof(contract_entry_t, ce_link), NULL, 2602 UU_LIST_POOL_DEBUG); 2603 contract_hash_init(); 2604 2605 log_framework(LOG_DEBUG, "Initialized restarter\n"); 2606 } 2607