1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2011 Joyent Inc.
25 */
26
27 /*
28 * method.c - method execution functions
29 *
30 * This file contains the routines needed to run a method: a fork(2)-exec(2)
31 * invocation monitored using either the contract filesystem or waitpid(2).
32 * (Plain fork1(2) support is provided in fork.c.)
33 *
34 * Contract Transfer
35 * When we restart a service, we want to transfer any contracts that the old
36 * service's contract inherited. This means that (a) we must not abandon the
37 * old contract when the service dies and (b) we must write the id of the old
38 * contract into the terms of the new contract. There should be limits to
39 * (a), though, since we don't want to keep the contract around forever. To
40 * this end we'll say that services in the offline state may have a contract
41 * to be transfered and services in the disabled or maintenance states cannot.
42 * This means that when a service transitions from online (or degraded) to
43 * offline, the contract should be preserved, and when the service transitions
44 * from offline to online (i.e., the start method), we'll transfer inherited
45 * contracts.
46 */
47
48 #include <sys/contract/process.h>
49 #include <sys/ctfs.h>
50 #include <sys/stat.h>
51 #include <sys/time.h>
52 #include <sys/types.h>
53 #include <sys/uio.h>
54 #include <sys/wait.h>
55 #include <alloca.h>
56 #include <assert.h>
57 #include <errno.h>
58 #include <fcntl.h>
59 #include <libcontract.h>
60 #include <libcontract_priv.h>
61 #include <libgen.h>
62 #include <librestart.h>
63 #include <libscf.h>
64 #include <limits.h>
65 #include <port.h>
66 #include <sac.h>
67 #include <signal.h>
68 #include <stdlib.h>
69 #include <string.h>
70 #include <strings.h>
71 #include <unistd.h>
72 #include <atomic.h>
73 #include <poll.h>
74 #include <libscf_priv.h>
75
76 #include "startd.h"
77
78 #define SBIN_SH "/sbin/sh"
79
80 /*
81 * Used to tell if contracts are in the process of being
82 * stored into the svc.startd internal hash table.
83 */
84 volatile uint16_t storing_contract = 0;
85
86 /*
87 * Mapping from restart_on method-type to contract events. Must correspond to
88 * enum method_restart_t.
89 */
90 static uint_t method_events[] = {
91 /* METHOD_RESTART_ALL */
92 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
93 /* METHOD_RESTART_EXTERNAL_FAULT */
94 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
95 /* METHOD_RESTART_ANY_FAULT */
96 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
97 };
98
99 /*
100 * method_record_start(restarter_inst_t *)
101 * Record a service start for rate limiting. Place the current time
102 * in the circular array of instance starts.
103 */
104 static void
method_record_start(restarter_inst_t * inst)105 method_record_start(restarter_inst_t *inst)
106 {
107 int index = inst->ri_start_index++ % RINST_START_TIMES;
108
109 inst->ri_start_time[index] = gethrtime();
110 }
111
112 /*
113 * method_rate_critical(restarter_inst_t *)
114 * Return true if the average start interval is less than the permitted
115 * interval. The implicit interval defaults to RINST_FAILURE_RATE_NS and
116 * RINST_START_TIMES but may be overridden with the svc properties
117 * startd/critical_failure_count and startd/critical_failure_period
118 * which represent the number of failures to consider and the amount of
119 * time in seconds in which that number may occur, respectively. Note that
120 * this time is measured as of the transition to 'enabled' rather than wall
121 * clock time.
122 * Implicit success if insufficient measurements for an average exist.
123 */
124 int
method_rate_critical(restarter_inst_t * inst)125 method_rate_critical(restarter_inst_t *inst)
126 {
127 hrtime_t critical_failure_period;
128 uint_t critical_failure_count = RINST_START_TIMES;
129 uint_t n = inst->ri_start_index;
130 hrtime_t avg_ns = 0;
131 uint64_t scf_fr, scf_st;
132 scf_propvec_t *prop = NULL;
133 scf_propvec_t restart_critical[] = {
134 { "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
135 { "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
136 { NULL }
137 };
138
139 if (instance_is_wait_style(inst))
140 critical_failure_period = RINST_WT_SVC_FAILURE_RATE_NS;
141 else
142 critical_failure_period = RINST_FAILURE_RATE_NS;
143
144 restart_critical[0].pv_ptr = &scf_fr;
145 restart_critical[1].pv_ptr = &scf_st;
146
147 if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
148 B_TRUE, restart_critical, &prop) != SCF_FAILED) {
149 /*
150 * critical_failure_period is expressed
151 * in seconds but tracked in ns
152 */
153 critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
154 critical_failure_count = (uint_t)scf_st;
155 }
156 if (inst->ri_start_index < critical_failure_count)
157 return (0);
158
159 avg_ns =
160 (inst->ri_start_time[(n - 1) % critical_failure_count] -
161 inst->ri_start_time[n % critical_failure_count]) /
162 (critical_failure_count - 1);
163
164 return (avg_ns < critical_failure_period);
165 }
166
167 /*
168 * int method_is_transient()
169 * Determine if the method for the given instance is transient,
170 * from a contract perspective. Return 1 if it is, and 0 if it isn't.
171 */
172 static int
method_is_transient(restarter_inst_t * inst,int type)173 method_is_transient(restarter_inst_t *inst, int type)
174 {
175 if (instance_is_transient_style(inst) || type != METHOD_START)
176 return (1);
177 else
178 return (0);
179 }
180
181 /*
182 * void method_store_contract()
183 * Store the newly created contract id into local structures and
184 * the repository. If the repository connection is broken it is rebound.
185 */
186 static void
method_store_contract(restarter_inst_t * inst,int type,ctid_t * cid)187 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
188 {
189 int r;
190 boolean_t primary;
191
192 if (errno = contract_latest(cid))
193 uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
194
195 primary = !method_is_transient(inst, type);
196
197 if (!primary) {
198 if (inst->ri_i.i_transient_ctid != 0) {
199 log_framework(LOG_INFO,
200 "%s: transient ctid expected to be 0 but "
201 "was set to %ld\n", inst->ri_i.i_fmri,
202 inst->ri_i.i_transient_ctid);
203 }
204
205 inst->ri_i.i_transient_ctid = *cid;
206 } else {
207 if (inst->ri_i.i_primary_ctid != 0) {
208 /*
209 * There was an old contract that we transferred.
210 * Remove it.
211 */
212 method_remove_contract(inst, B_TRUE, B_FALSE);
213 }
214
215 if (inst->ri_i.i_primary_ctid != 0) {
216 log_framework(LOG_INFO,
217 "%s: primary ctid expected to be 0 but "
218 "was set to %ld\n", inst->ri_i.i_fmri,
219 inst->ri_i.i_primary_ctid);
220 }
221
222 inst->ri_i.i_primary_ctid = *cid;
223 inst->ri_i.i_primary_ctid_stopped = 0;
224
225 log_framework(LOG_DEBUG, "Storing primary contract %ld for "
226 "%s.\n", *cid, inst->ri_i.i_fmri);
227
228 contract_hash_store(*cid, inst->ri_id);
229 }
230
231 again:
232 if (inst->ri_mi_deleted)
233 return;
234
235 r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
236 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
237 switch (r) {
238 case 0:
239 break;
240
241 case ECANCELED:
242 inst->ri_mi_deleted = B_TRUE;
243 break;
244
245 case ECONNABORTED:
246 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
247 /* FALLTHROUGH */
248
249 case EBADF:
250 libscf_reget_instance(inst);
251 goto again;
252
253 case ENOMEM:
254 case EPERM:
255 case EACCES:
256 case EROFS:
257 uu_die("%s: Couldn't store contract id %ld",
258 inst->ri_i.i_fmri, *cid);
259 /* NOTREACHED */
260
261 case EINVAL:
262 default:
263 bad_error("restarter_store_contract", r);
264 }
265 }
266
267 /*
268 * void method_remove_contract()
269 * Remove any non-permanent contracts from internal structures and
270 * the repository, then abandon them.
271 * Returns
272 * 0 - success
273 * ECANCELED - inst was deleted from the repository
274 *
275 * If the repository connection was broken, it is rebound.
276 */
277 void
method_remove_contract(restarter_inst_t * inst,boolean_t primary,boolean_t abandon)278 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
279 boolean_t abandon)
280 {
281 ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
282 &inst->ri_i.i_transient_ctid;
283
284 int r;
285
286 assert(*ctidp != 0);
287
288 log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
289 primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
290
291 if (abandon)
292 contract_abandon(*ctidp);
293
294 again:
295 if (inst->ri_mi_deleted) {
296 r = ECANCELED;
297 goto out;
298 }
299
300 r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
301 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
302 switch (r) {
303 case 0:
304 break;
305
306 case ECANCELED:
307 inst->ri_mi_deleted = B_TRUE;
308 break;
309
310 case ECONNABORTED:
311 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
312 /* FALLTHROUGH */
313
314 case EBADF:
315 libscf_reget_instance(inst);
316 goto again;
317
318 case ENOMEM:
319 case EPERM:
320 case EACCES:
321 case EROFS:
322 log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
323 "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
324 break;
325
326 case EINVAL:
327 default:
328 bad_error("restarter_remove_contract", r);
329 }
330
331 out:
332 if (primary)
333 contract_hash_remove(*ctidp);
334
335 *ctidp = 0;
336 }
337
338 static const char *method_names[] = { "start", "stop", "refresh" };
339
340 /*
341 * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
342 *
343 * Activate a contract template for the type method of inst. type,
344 * restart_on, and cte_mask dictate the critical events term of the contract.
345 * Returns
346 * 0 - success
347 * ECANCELED - inst has been deleted from the repository
348 */
349 static int
method_ready_contract(restarter_inst_t * inst,int type,method_restart_t restart_on,uint_t cte_mask)350 method_ready_contract(restarter_inst_t *inst, int type,
351 method_restart_t restart_on, uint_t cte_mask)
352 {
353 int tmpl, err, istrans, iswait, ret;
354 uint_t cevents, fevents;
355
356 /*
357 * Correctly supporting wait-style services is tricky without
358 * rearchitecting startd to cope with multiple event sources
359 * simultaneously trying to stop an instance. Until a better
360 * solution is implemented, we avoid this problem for
361 * wait-style services by making contract events fatal and
362 * letting the wait code alone handle stopping the service.
363 */
364 iswait = instance_is_wait_style(inst);
365 istrans = method_is_transient(inst, type);
366
367 tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
368 if (tmpl == -1)
369 uu_die("Could not create contract template");
370
371 /*
372 * We assume non-login processes are unlikely to create
373 * multiple process groups, and set CT_PR_PGRPONLY for all
374 * wait-style services' contracts.
375 */
376 err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
377 (iswait ? CT_PR_PGRPONLY : 0));
378 assert(err == 0);
379
380 if (istrans) {
381 cevents = 0;
382 fevents = 0;
383 } else {
384 assert(restart_on >= 0);
385 assert(restart_on <= METHOD_RESTART_ANY_FAULT);
386 cevents = method_events[restart_on] & ~cte_mask;
387 fevents = iswait ?
388 (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
389 0;
390 }
391
392 err = ct_tmpl_set_critical(tmpl, cevents);
393 assert(err == 0);
394
395 err = ct_tmpl_set_informative(tmpl, 0);
396 assert(err == 0);
397 err = ct_pr_tmpl_set_fatal(tmpl, fevents);
398 assert(err == 0);
399
400 err = ct_tmpl_set_cookie(tmpl, istrans ? METHOD_OTHER_COOKIE :
401 METHOD_START_COOKIE);
402 assert(err == 0);
403
404 if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
405 ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
406 switch (ret) {
407 case 0:
408 break;
409
410 case ENOTEMPTY:
411 /* No contracts for you! */
412 method_remove_contract(inst, B_TRUE, B_TRUE);
413 if (inst->ri_mi_deleted) {
414 ret = ECANCELED;
415 goto out;
416 }
417 break;
418
419 case EINVAL:
420 case ESRCH:
421 case EACCES:
422 default:
423 bad_error("ct_pr_tmpl_set_transfer", ret);
424 }
425 }
426
427 err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
428 assert(err == 0);
429 err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
430 assert(err == 0);
431
432 err = ct_tmpl_activate(tmpl);
433 assert(err == 0);
434
435 ret = 0;
436
437 out:
438 err = close(tmpl);
439 assert(err == 0);
440
441 return (ret);
442 }
443
444 static void
exec_method(const restarter_inst_t * inst,int type,const char * method,struct method_context * mcp,uint8_t need_session)445 exec_method(const restarter_inst_t *inst, int type, const char *method,
446 struct method_context *mcp, uint8_t need_session)
447 {
448 char *cmd;
449 const char *errf;
450 char **nenv;
451 int rsmc_errno = 0;
452
453 cmd = uu_msprintf("exec %s", method);
454
455 if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
456 (void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
457
458 setlog(inst->ri_logstem);
459 log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
460 method_names[type], method);
461
462 if (need_session)
463 (void) setpgrp();
464
465 /* Set credentials. */
466 rsmc_errno = restarter_set_method_context(mcp, &errf);
467 if (rsmc_errno != 0) {
468 log_instance(inst, B_FALSE,
469 "svc.startd could not set context for method: ");
470
471 if (rsmc_errno == -1) {
472 if (strcmp(errf, "core_set_process_path") == 0) {
473 log_instance(inst, B_FALSE,
474 "Could not set corefile path.");
475 } else if (strcmp(errf, "setproject") == 0) {
476 log_instance(inst, B_FALSE, "%s: a resource "
477 "control assignment failed", errf);
478 } else if (strcmp(errf, "pool_set_binding") == 0) {
479 log_instance(inst, B_FALSE, "%s: a system "
480 "error occurred", errf);
481 } else {
482 #ifndef NDEBUG
483 uu_warn("%s:%d: Bad function name \"%s\" for "
484 "error %d from "
485 "restarter_set_method_context().\n",
486 __FILE__, __LINE__, errf, rsmc_errno);
487 #endif
488 abort();
489 }
490
491 exit(1);
492 }
493
494 if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
495 switch (rsmc_errno) {
496 case ENOENT:
497 log_instance(inst, B_FALSE, "%s: the pool "
498 "could not be found", errf);
499 break;
500
501 case EBADF:
502 log_instance(inst, B_FALSE, "%s: the "
503 "configuration is invalid", errf);
504 break;
505
506 case EINVAL:
507 log_instance(inst, B_FALSE, "%s: pool name "
508 "\"%s\" is invalid", errf,
509 mcp->resource_pool);
510 break;
511
512 default:
513 #ifndef NDEBUG
514 uu_warn("%s:%d: Bad error %d for function %s "
515 "in restarter_set_method_context().\n",
516 __FILE__, __LINE__, rsmc_errno, errf);
517 #endif
518 abort();
519 }
520
521 exit(SMF_EXIT_ERR_CONFIG);
522 }
523
524 if (errf != NULL && strcmp(errf, "chdir") == 0) {
525 switch (rsmc_errno) {
526 case EACCES:
527 case EFAULT:
528 case EIO:
529 case ELOOP:
530 case ENAMETOOLONG:
531 case ENOENT:
532 case ENOLINK:
533 case ENOTDIR:
534 log_instance(inst, B_FALSE, "%s: %s (\"%s\")",
535 errf,
536 strerror(rsmc_errno), mcp->working_dir);
537 break;
538
539 default:
540 #ifndef NDEBUG
541 uu_warn("%s:%d: Bad error %d for function %s "
542 "in restarter_set_method_context().\n",
543 __FILE__, __LINE__, rsmc_errno, errf);
544 #endif
545 abort();
546 }
547
548 exit(SMF_EXIT_ERR_CONFIG);
549 }
550
551 if (errf != NULL) {
552 errno = rsmc_errno;
553 perror(errf);
554
555 switch (rsmc_errno) {
556 case EINVAL:
557 case EPERM:
558 case ENOENT:
559 case ENAMETOOLONG:
560 case ERANGE:
561 case ESRCH:
562 exit(SMF_EXIT_ERR_CONFIG);
563 /* NOTREACHED */
564
565 default:
566 exit(1);
567 }
568 }
569
570 switch (rsmc_errno) {
571 case ENOMEM:
572 log_instance(inst, B_FALSE, "Out of memory.");
573 exit(1);
574 /* NOTREACHED */
575
576 case ENOENT:
577 log_instance(inst, B_FALSE, "Missing passwd entry for "
578 "user.");
579 exit(SMF_EXIT_ERR_CONFIG);
580 /* NOTREACHED */
581
582 default:
583 #ifndef NDEBUG
584 uu_warn("%s:%d: Bad miscellaneous error %d from "
585 "restarter_set_method_context().\n", __FILE__,
586 __LINE__, rsmc_errno);
587 #endif
588 abort();
589 }
590 }
591
592 nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
593 method_names[type]);
594
595 log_preexec();
596
597 (void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
598
599 exit(10);
600 }
601
602 static void
write_status(restarter_inst_t * inst,const char * mname,int stat)603 write_status(restarter_inst_t *inst, const char *mname, int stat)
604 {
605 int r;
606
607 again:
608 if (inst->ri_mi_deleted)
609 return;
610
611 r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
612 switch (r) {
613 case 0:
614 break;
615
616 case ECONNABORTED:
617 libscf_reget_instance(inst);
618 goto again;
619
620 case ECANCELED:
621 inst->ri_mi_deleted = 1;
622 break;
623
624 case EPERM:
625 case EACCES:
626 case EROFS:
627 log_framework(LOG_INFO, "Could not write exit status "
628 "for %s method of %s: %s.\n", mname,
629 inst->ri_i.i_fmri, strerror(r));
630 break;
631
632 case ENAMETOOLONG:
633 default:
634 bad_error("libscf_write_method_status", r);
635 }
636 }
637
638 /*
639 * int method_run()
640 * Execute the type method of instp. If it requires a fork(), wait for it
641 * to return and return its exit code in *exit_code. Otherwise set
642 * *exit_code to 0 if the method succeeds & -1 if it fails. If the
643 * repository connection is broken, it is rebound, but inst may not be
644 * reset.
645 * Returns
646 * 0 - success
647 * EINVAL - A correct method or method context couldn't be retrieved.
648 * EIO - Contract kill failed.
649 * EFAULT - Method couldn't be executed successfully.
650 * ELOOP - Retry threshold exceeded.
651 * ECANCELED - inst was deleted from the repository before method was run
652 * ERANGE - Timeout retry threshold exceeded.
653 * EAGAIN - Failed due to external cause, retry.
654 */
655 int
method_run(restarter_inst_t ** instp,int type,int * exit_code)656 method_run(restarter_inst_t **instp, int type, int *exit_code)
657 {
658 char *method;
659 int ret_status;
660 pid_t pid;
661 method_restart_t restart_on;
662 uint_t cte_mask;
663 uint8_t need_session;
664 scf_handle_t *h;
665 scf_snapshot_t *snap;
666 const char *mname;
667 mc_error_t *m_error;
668 struct method_context *mcp;
669 int result = 0, timeout_fired = 0;
670 int sig, r;
671 boolean_t transient;
672 uint64_t timeout;
673 uint8_t timeout_retry;
674 ctid_t ctid;
675 int ctfd = -1;
676 restarter_inst_t *inst = *instp;
677 int id = inst->ri_id;
678 int forkerr;
679
680 assert(MUTEX_HELD(&inst->ri_lock));
681 assert(instance_in_transition(inst));
682
683 if (inst->ri_mi_deleted)
684 return (ECANCELED);
685
686 *exit_code = 0;
687
688 assert(0 <= type && type <= 2);
689 mname = method_names[type];
690
691 if (type == METHOD_START)
692 inst->ri_pre_online_hook();
693
694 h = scf_instance_handle(inst->ri_m_inst);
695
696 snap = scf_snapshot_create(h);
697 if (snap == NULL ||
698 scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
699 log_framework(LOG_DEBUG,
700 "Could not get running snapshot for %s. "
701 "Using editing version to run method %s.\n",
702 inst->ri_i.i_fmri, mname);
703 scf_snapshot_destroy(snap);
704 snap = NULL;
705 }
706
707 /*
708 * After this point, we may be logging to the instance log.
709 * Make sure we've noted where that log is as a property of
710 * the instance.
711 */
712 r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
713 inst->ri_logstem);
714 if (r != 0) {
715 log_framework(LOG_WARNING,
716 "%s: couldn't note log location: %s\n",
717 inst->ri_i.i_fmri, strerror(r));
718 }
719
720 if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
721 &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
722 if (errno == LIBSCF_PGROUP_ABSENT) {
723 log_framework(LOG_DEBUG,
724 "%s: instance has no method property group '%s'.\n",
725 inst->ri_i.i_fmri, mname);
726 if (type == METHOD_REFRESH)
727 log_instance(inst, B_TRUE, "No '%s' method "
728 "defined. Treating as :true.", mname);
729 else
730 log_instance(inst, B_TRUE, "Method property "
731 "group '%s' is not present.", mname);
732 scf_snapshot_destroy(snap);
733 return (0);
734 } else if (errno == LIBSCF_PROPERTY_ABSENT) {
735 log_framework(LOG_DEBUG,
736 "%s: instance has no '%s/exec' method property.\n",
737 inst->ri_i.i_fmri, mname);
738 log_instance(inst, B_TRUE, "Method property '%s/exec "
739 "is not present.", mname);
740 scf_snapshot_destroy(snap);
741 return (0);
742 } else {
743 log_error(LOG_WARNING,
744 "%s: instance libscf_get_method failed\n",
745 inst->ri_i.i_fmri);
746 scf_snapshot_destroy(snap);
747 return (EINVAL);
748 }
749 }
750
751 /* open service contract if stopping a non-transient service */
752 if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
753 if (inst->ri_i.i_primary_ctid == 0) {
754 /* service is not running, nothing to stop */
755 log_framework(LOG_DEBUG, "%s: instance has no primary "
756 "contract, no service to stop.\n",
757 inst->ri_i.i_fmri);
758 scf_snapshot_destroy(snap);
759 return (0);
760 }
761 if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
762 "events", O_RDONLY)) < 0) {
763 result = EFAULT;
764 log_instance(inst, B_TRUE, "Could not open service "
765 "contract %ld. Stop method not run.",
766 inst->ri_i.i_primary_ctid);
767 goto out;
768 }
769 }
770
771 if (restarter_is_null_method(method)) {
772 log_framework(LOG_DEBUG, "%s: null method succeeds\n",
773 inst->ri_i.i_fmri);
774
775 log_instance(inst, B_TRUE, "Executing %s method (null).",
776 mname);
777
778 if (type == METHOD_START)
779 write_status(inst, mname, 0);
780 goto out;
781 }
782
783 sig = restarter_is_kill_method(method);
784 if (sig >= 0) {
785
786 if (inst->ri_i.i_primary_ctid == 0) {
787 log_error(LOG_ERR, "%s: :kill with no contract\n",
788 inst->ri_i.i_fmri);
789 log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
790 "as stop method for transient service.");
791 result = EINVAL;
792 goto out;
793 }
794
795 log_framework(LOG_DEBUG,
796 "%s: :killing contract with signal %d\n",
797 inst->ri_i.i_fmri, sig);
798
799 log_instance(inst, B_TRUE, "Executing %s method (:kill).",
800 mname);
801
802 if (contract_kill(inst->ri_i.i_primary_ctid, sig,
803 inst->ri_i.i_fmri) != 0) {
804 result = EIO;
805 goto out;
806 } else
807 goto assured_kill;
808 }
809
810 log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
811 inst->ri_i.i_fmri, method);
812
813 m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
814 inst->ri_m_inst, snap, mname, method, &mcp);
815
816 if (m_error != NULL) {
817 log_instance(inst, B_TRUE, "%s", m_error->msg);
818 restarter_mc_error_destroy(m_error);
819 result = EINVAL;
820 goto out;
821 }
822
823 r = method_ready_contract(inst, type, restart_on, cte_mask);
824 if (r != 0) {
825 assert(r == ECANCELED);
826 assert(inst->ri_mi_deleted);
827 restarter_free_method_context(mcp);
828 result = ECANCELED;
829 goto out;
830 }
831
832 /*
833 * Validate safety of method contexts, to save children work.
834 */
835 if (!restarter_rm_libs_loadable())
836 log_framework(LOG_DEBUG, "%s: method contexts limited "
837 "to root-accessible libraries\n", inst->ri_i.i_fmri);
838
839 /*
840 * For wait-style svc, sanity check that method exists to prevent an
841 * infinite loop.
842 */
843 if (instance_is_wait_style(inst) && type == METHOD_START) {
844 char *pend;
845 struct stat64 sbuf;
846
847 /*
848 * We need to handle start method strings that have arguments,
849 * such as '/lib/svc/method/console-login %i'.
850 */
851 if ((pend = strchr(method, ' ')) != NULL)
852 *pend = '\0';
853
854 if (*method == '/' && stat64(method, &sbuf) == -1 &&
855 errno == ENOENT) {
856 log_instance(inst, B_TRUE, "Missing start method (%s), "
857 "changing state to maintenance.", method);
858 restarter_free_method_context(mcp);
859 result = ENOENT;
860 goto out;
861 }
862 if (pend != NULL)
863 *pend = ' ';
864 }
865
866 /*
867 * If the service is restarting too quickly, send it to
868 * maintenance.
869 */
870 if (type == METHOD_START) {
871 method_record_start(inst);
872 if (method_rate_critical(inst) &&
873 !instance_is_wait_style(inst)) {
874 log_instance(inst, B_TRUE, "Restarting too quickly, "
875 "changing state to maintenance.");
876 result = ELOOP;
877 restarter_free_method_context(mcp);
878 goto out;
879 }
880 }
881
882 atomic_add_16(&storing_contract, 1);
883 pid = startd_fork1(&forkerr);
884 if (pid == 0)
885 exec_method(inst, type, method, mcp, need_session);
886
887 if (pid == -1) {
888 atomic_add_16(&storing_contract, -1);
889 if (forkerr == EAGAIN)
890 result = EAGAIN;
891 else
892 result = EFAULT;
893
894 log_error(LOG_WARNING,
895 "%s: Couldn't fork to execute method %s: %s\n",
896 inst->ri_i.i_fmri, method, strerror(forkerr));
897
898 restarter_free_method_context(mcp);
899 goto out;
900 }
901
902
903 /*
904 * Get the contract id, decide whether it is primary or transient, and
905 * stash it in inst & the repository.
906 */
907 method_store_contract(inst, type, &ctid);
908 atomic_add_16(&storing_contract, -1);
909
910 restarter_free_method_context(mcp);
911
912 /*
913 * Similarly for the start method PID.
914 */
915 if (type == METHOD_START && !inst->ri_mi_deleted)
916 (void) libscf_write_start_pid(inst->ri_m_inst, pid);
917
918 if (instance_is_wait_style(inst) && type == METHOD_START) {
919 /* Wait style instances don't get timeouts on start methods. */
920 if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
921 log_error(LOG_WARNING,
922 "%s: couldn't register %ld for wait\n",
923 inst->ri_i.i_fmri, pid);
924 result = EFAULT;
925 goto contract_out;
926 }
927 write_status(inst, mname, 0);
928
929 } else {
930 int r, err;
931 time_t start_time;
932 time_t end_time;
933
934 /*
935 * Because on upgrade/live-upgrade we may have no chance
936 * to override faulty timeout values on the way to
937 * manifest import, all services on the path to manifest
938 * import are treated the same as INFINITE timeout services.
939 */
940
941 start_time = time(NULL);
942 if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
943 timeout_insert(inst, ctid, timeout);
944 else
945 timeout = METHOD_TIMEOUT_INFINITE;
946
947 /* Unlock the instance while waiting for the method. */
948 MUTEX_UNLOCK(&inst->ri_lock);
949
950 do {
951 r = waitpid(pid, &ret_status, NULL);
952 } while (r == -1 && errno == EINTR);
953 if (r == -1)
954 err = errno;
955
956 /* Re-grab the lock. */
957 inst = inst_lookup_by_id(id);
958
959 /*
960 * inst can't be removed, as the removal thread waits
961 * for completion of this one.
962 */
963 assert(inst != NULL);
964 *instp = inst;
965
966 if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
967 timeout_fired = 1;
968
969 timeout_remove(inst, ctid);
970
971 log_framework(LOG_DEBUG,
972 "%s method for %s exited with status %d.\n", mname,
973 inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
974
975 if (r == -1) {
976 log_error(LOG_WARNING,
977 "Couldn't waitpid() for %s method of %s (%s).\n",
978 mname, inst->ri_i.i_fmri, strerror(err));
979 result = EFAULT;
980 goto contract_out;
981 }
982
983 if (type == METHOD_START)
984 write_status(inst, mname, ret_status);
985
986 /* return ERANGE if this service doesn't retry on timeout */
987 if (timeout_fired == 1 && timeout_retry == 0) {
988 result = ERANGE;
989 goto contract_out;
990 }
991
992 if (!WIFEXITED(ret_status)) {
993 /*
994 * If method didn't exit itself (it was killed by an
995 * external entity, etc.), consider the entire
996 * method_run as failed.
997 */
998 if (WIFSIGNALED(ret_status)) {
999 char buf[SIG2STR_MAX];
1000 (void) sig2str(WTERMSIG(ret_status), buf);
1001
1002 log_error(LOG_WARNING, "%s: Method \"%s\" "
1003 "failed due to signal %s.\n",
1004 inst->ri_i.i_fmri, method, buf);
1005 log_instance(inst, B_TRUE, "Method \"%s\" "
1006 "failed due to signal %s.", mname, buf);
1007 } else {
1008 log_error(LOG_WARNING, "%s: Method \"%s\" "
1009 "failed with exit status %d.\n",
1010 inst->ri_i.i_fmri, method,
1011 WEXITSTATUS(ret_status));
1012 log_instance(inst, B_TRUE, "Method \"%s\" "
1013 "failed with exit status %d.", mname,
1014 WEXITSTATUS(ret_status));
1015 }
1016 result = EAGAIN;
1017 goto contract_out;
1018 }
1019
1020 *exit_code = WEXITSTATUS(ret_status);
1021 if (*exit_code != 0) {
1022 log_error(LOG_WARNING,
1023 "%s: Method \"%s\" failed with exit status %d.\n",
1024 inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
1025 }
1026
1027 log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
1028 "%d.", mname, *exit_code);
1029
1030 if (*exit_code != 0)
1031 goto contract_out;
1032
1033 end_time = time(NULL);
1034
1035 /* Give service contract remaining seconds to empty */
1036 if (timeout != METHOD_TIMEOUT_INFINITE)
1037 timeout -= (end_time - start_time);
1038 }
1039
1040 assured_kill:
1041 /*
1042 * For stop methods, assure that the service contract has emptied
1043 * before returning.
1044 */
1045 if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
1046 !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
1047 int times = 0;
1048
1049 if (timeout != METHOD_TIMEOUT_INFINITE)
1050 timeout_insert(inst, inst->ri_i.i_primary_ctid,
1051 timeout);
1052
1053 for (;;) {
1054 /*
1055 * Check frequently at first, then back off. This
1056 * keeps startd from idling while shutting down.
1057 */
1058 if (times < 20) {
1059 (void) poll(NULL, 0, 5);
1060 times++;
1061 } else {
1062 (void) poll(NULL, 0, 100);
1063 }
1064 if (contract_is_empty(inst->ri_i.i_primary_ctid))
1065 break;
1066 }
1067
1068 if (timeout != METHOD_TIMEOUT_INFINITE)
1069 if (inst->ri_timeout->te_fired)
1070 result = EFAULT;
1071
1072 timeout_remove(inst, inst->ri_i.i_primary_ctid);
1073 }
1074
1075 contract_out:
1076 /* Abandon contracts for transient methods & methods that fail. */
1077 transient = method_is_transient(inst, type);
1078 if ((transient || *exit_code != 0 || result != 0) &&
1079 (restarter_is_kill_method(method) < 0))
1080 method_remove_contract(inst, !transient, B_TRUE);
1081
1082 out:
1083 if (ctfd >= 0)
1084 (void) close(ctfd);
1085 scf_snapshot_destroy(snap);
1086 free(method);
1087 return (result);
1088 }
1089
1090 /*
1091 * The method thread executes a service method to effect a state transition.
1092 * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1093 * be _NONE on exit (state will either be what next_state was (on success), or
1094 * it will be _MAINT (on error)).
1095 *
1096 * There are six classes of methods to consider: start & other (stop, refresh)
1097 * for each of "normal" services, wait services, and transient services. For
1098 * each, the method must be fetched from the repository & executed. fork()ed
1099 * methods must be waited on, except for the start method of wait services
1100 * (which must be registered with the wait subsystem via wait_register()). If
1101 * the method succeeded (returned 0), then for start methods its contract
1102 * should be recorded as the primary contract for the service. For other
1103 * methods, it should be abandoned. If the method fails, then depending on
1104 * the failure, either the method should be reexecuted or the service should
1105 * be put into maintenance. Either way the contract should be abandoned.
1106 */
1107 void *
method_thread(void * arg)1108 method_thread(void *arg)
1109 {
1110 fork_info_t *info = arg;
1111 restarter_inst_t *inst;
1112 scf_handle_t *local_handle;
1113 scf_instance_t *s_inst = NULL;
1114 int r, exit_code;
1115 boolean_t retryable;
1116 restarter_str_t reason;
1117
1118 assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1119
1120 /* Get (and lock) the restarter_inst_t. */
1121 inst = inst_lookup_by_id(info->sf_id);
1122
1123 assert(inst->ri_method_thread != 0);
1124 assert(instance_in_transition(inst) == 1);
1125
1126 /*
1127 * We cannot leave this function with inst in transition, because
1128 * protocol.c withholds messages for inst otherwise.
1129 */
1130
1131 log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1132 method_names[info->sf_method_type], inst->ri_i.i_fmri);
1133
1134 local_handle = libscf_handle_create_bound_loop();
1135
1136 rebind_retry:
1137 /* get scf_instance_t */
1138 switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1139 &s_inst)) {
1140 case 0:
1141 break;
1142
1143 case ECONNABORTED:
1144 libscf_handle_rebind(local_handle);
1145 goto rebind_retry;
1146
1147 case ENOENT:
1148 /*
1149 * It's not there, but we need to call this so protocol.c
1150 * doesn't think it's in transition anymore.
1151 */
1152 (void) restarter_instance_update_states(local_handle, inst,
1153 inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1154 restarter_str_none);
1155 goto out;
1156
1157 case EINVAL:
1158 case ENOTSUP:
1159 default:
1160 bad_error("libscf_fmri_get_instance", r);
1161 }
1162
1163 inst->ri_m_inst = s_inst;
1164 inst->ri_mi_deleted = B_FALSE;
1165
1166 retry:
1167 if (info->sf_method_type == METHOD_START)
1168 log_transition(inst, START_REQUESTED);
1169
1170 r = method_run(&inst, info->sf_method_type, &exit_code);
1171
1172 if (r == 0 && exit_code == 0) {
1173 /* Success! */
1174 assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1175
1176 /*
1177 * When a stop method succeeds, remove the primary contract of
1178 * the service, unless we're going to offline, in which case
1179 * retain the contract so we can transfer inherited contracts to
1180 * the replacement service.
1181 */
1182
1183 if (info->sf_method_type == METHOD_STOP &&
1184 inst->ri_i.i_primary_ctid != 0) {
1185 if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1186 inst->ri_i.i_primary_ctid_stopped = 1;
1187 else
1188 method_remove_contract(inst, B_TRUE, B_TRUE);
1189 }
1190 /*
1191 * We don't care whether the handle was rebound because this is
1192 * the last thing we do with it.
1193 */
1194 (void) restarter_instance_update_states(local_handle, inst,
1195 inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1196 info->sf_event_type, info->sf_reason);
1197
1198 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1199
1200 goto out;
1201 }
1202
1203 /* Failure. Retry or go to maintenance. */
1204
1205 if (r != 0 && r != EAGAIN) {
1206 retryable = B_FALSE;
1207 } else {
1208 switch (exit_code) {
1209 case SMF_EXIT_ERR_CONFIG:
1210 case SMF_EXIT_ERR_NOSMF:
1211 case SMF_EXIT_ERR_PERM:
1212 case SMF_EXIT_ERR_FATAL:
1213 retryable = B_FALSE;
1214 break;
1215
1216 default:
1217 retryable = B_TRUE;
1218 }
1219 }
1220
1221 if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1222 goto retry;
1223
1224 /* maintenance */
1225 if (r == ELOOP)
1226 log_transition(inst, START_FAILED_REPEATEDLY);
1227 else if (r == ERANGE)
1228 log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1229 else if (exit_code == SMF_EXIT_ERR_CONFIG)
1230 log_transition(inst, START_FAILED_CONFIGURATION);
1231 else if (exit_code == SMF_EXIT_ERR_FATAL)
1232 log_transition(inst, START_FAILED_FATAL);
1233 else
1234 log_transition(inst, START_FAILED_OTHER);
1235
1236 if (r == ELOOP) {
1237 reason = restarter_str_restarting_too_quickly;
1238 } else if (retryable) {
1239 reason = restarter_str_fault_threshold_reached;
1240 } else {
1241 reason = restarter_str_method_failed;
1242 }
1243
1244 (void) restarter_instance_update_states(local_handle, inst,
1245 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1246 reason);
1247
1248 if (!method_is_transient(inst, info->sf_method_type) &&
1249 inst->ri_i.i_primary_ctid != 0)
1250 method_remove_contract(inst, B_TRUE, B_TRUE);
1251
1252 out:
1253 inst->ri_method_thread = 0;
1254
1255 /*
1256 * Unlock the mutex after broadcasting to avoid a race condition
1257 * with restarter_delete_inst() when the 'inst' structure is freed.
1258 */
1259 (void) pthread_cond_broadcast(&inst->ri_method_cv);
1260 MUTEX_UNLOCK(&inst->ri_lock);
1261
1262 scf_instance_destroy(s_inst);
1263 scf_handle_destroy(local_handle);
1264 startd_free(info, sizeof (fork_info_t));
1265 return (NULL);
1266 }
1267