1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Joyent, Inc.
25 * Copyright 2025 OmniOS Community Edition (OmniOSce) Association.
26 * Copyright 2026 Oxide Computer Company
27 */
28
29 /*
30 * method.c - method execution functions
31 *
32 * This file contains the routines needed to run a method: a fork(2)-exec(2)
33 * invocation monitored using either the contract filesystem or waitpid(3C).
34 * (Plain fork1(2) support is provided in fork.c.)
35 *
36 * Contract Transfer
37 * When we restart a service, we want to transfer any contracts that the old
38 * service's contract inherited. This means that (a) we must not abandon the
39 * old contract when the service dies and (b) we must write the id of the old
40 * contract into the terms of the new contract. There should be limits to
41 * (a), though, since we don't want to keep the contract around forever. To
42 * this end we'll say that services in the offline state may have a contract
43 * to be transfered and services in the disabled or maintenance states cannot.
44 * This means that when a service transitions from online (or degraded) to
45 * offline, the contract should be preserved, and when the service transitions
46 * from offline to online (i.e., the start method), we'll transfer inherited
47 * contracts.
48 */
49
50 #include <sys/contract/process.h>
51 #include <sys/ctfs.h>
52 #include <sys/stat.h>
53 #include <sys/time.h>
54 #include <sys/types.h>
55 #include <sys/uio.h>
56 #include <sys/wait.h>
57 #include <alloca.h>
58 #include <assert.h>
59 #include <errno.h>
60 #include <fcntl.h>
61 #include <libcontract.h>
62 #include <libcontract_priv.h>
63 #include <libgen.h>
64 #include <librestart.h>
65 #include <libscf.h>
66 #include <limits.h>
67 #include <port.h>
68 #include <sac.h>
69 #include <signal.h>
70 #include <stdlib.h>
71 #include <string.h>
72 #include <strings.h>
73 #include <unistd.h>
74 #include <atomic.h>
75 #include <poll.h>
76 #include <libscf_priv.h>
77
78 #include "startd.h"
79
80 #define SBIN_SH "/sbin/sh"
81
82 /*
83 * Used to tell if contracts are in the process of being
84 * stored into the svc.startd internal hash table.
85 */
86 volatile uint16_t storing_contract = 0;
87
88 /*
89 * Mapping from restart_on method-type to contract events. Must correspond to
90 * enum method_restart_t.
91 */
92 static uint_t method_events[] = {
93 /* METHOD_RESTART_ALL */
94 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
95 /* METHOD_RESTART_EXTERNAL_FAULT */
96 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
97 /* METHOD_RESTART_ANY_FAULT */
98 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
99 };
100
101 /*
102 * method_record_start(restarter_inst_t *)
103 * Record a service start for rate limiting. Place the current time
104 * in the circular array of instance starts.
105 */
106 static void
method_record_start(restarter_inst_t * inst)107 method_record_start(restarter_inst_t *inst)
108 {
109 int index = inst->ri_start_index++ % RINST_START_TIMES;
110
111 inst->ri_start_time[index] = gethrtime();
112 }
113
114 /*
115 * method_rate_critical(restarter_inst_t *)
116 * Return true if the average start interval is less than the permitted
117 * interval. The implicit interval defaults to RINST_FAILURE_RATE_NS and
118 * RINST_START_TIMES but may be overridden with the svc properties
119 * startd/critical_failure_count and startd/critical_failure_period
120 * which represent the number of failures to consider and the amount of
121 * time in seconds in which that number may occur, respectively. Note that
122 * this time is measured as of the transition to 'enabled' rather than wall
123 * clock time.
124 * Implicit success if insufficient measurements for an average exist.
125 */
126 int
method_rate_critical(restarter_inst_t * inst)127 method_rate_critical(restarter_inst_t *inst)
128 {
129 hrtime_t critical_failure_period;
130 uint_t critical_failure_count = RINST_START_TIMES;
131 uint_t n = inst->ri_start_index;
132 hrtime_t avg_ns = 0;
133 uint64_t scf_fr, scf_st;
134 scf_propvec_t *prop = NULL;
135 scf_propvec_t restart_critical[] = {
136 { "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
137 { "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
138 { NULL }
139 };
140
141 if (instance_is_wait_style(inst))
142 critical_failure_period = RINST_WT_SVC_FAILURE_RATE_NS;
143 else
144 critical_failure_period = RINST_FAILURE_RATE_NS;
145
146 restart_critical[0].pv_ptr = &scf_fr;
147 restart_critical[1].pv_ptr = &scf_st;
148
149 if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
150 B_TRUE, restart_critical, &prop) != SCF_FAILED) {
151 /*
152 * critical_failure_period is expressed
153 * in seconds but tracked in ns
154 */
155 critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
156 critical_failure_count = (uint_t)scf_st;
157 }
158 if (inst->ri_start_index < critical_failure_count)
159 return (0);
160
161 avg_ns =
162 (inst->ri_start_time[(n - 1) % critical_failure_count] -
163 inst->ri_start_time[n % critical_failure_count]) /
164 (critical_failure_count - 1);
165
166 return (avg_ns < critical_failure_period);
167 }
168
169 /*
170 * int method_is_transient()
171 * Determine if the method for the given instance is transient,
172 * from a contract perspective. Return 1 if it is, and 0 if it isn't.
173 */
174 static int
method_is_transient(restarter_inst_t * inst,int type)175 method_is_transient(restarter_inst_t *inst, int type)
176 {
177 if (instance_is_transient_style(inst) || type != METHOD_START)
178 return (1);
179 else
180 return (0);
181 }
182
183 /*
184 * void method_store_contract()
185 * Store the newly created contract id into local structures and
186 * the repository. If the repository connection is broken it is rebound.
187 */
188 static void
method_store_contract(restarter_inst_t * inst,int type,ctid_t * cid)189 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
190 {
191 int r;
192 boolean_t primary;
193
194 if (errno = contract_latest(cid))
195 uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
196
197 primary = !method_is_transient(inst, type);
198
199 if (!primary) {
200 if (inst->ri_i.i_transient_ctid != 0) {
201 log_framework(LOG_INFO,
202 "%s: transient ctid expected to be 0 but "
203 "was set to %ld\n", inst->ri_i.i_fmri,
204 inst->ri_i.i_transient_ctid);
205 }
206
207 inst->ri_i.i_transient_ctid = *cid;
208 } else {
209 if (inst->ri_i.i_primary_ctid != 0) {
210 /*
211 * There was an old contract that we transferred.
212 * Remove it.
213 */
214 method_remove_contract(inst, B_TRUE, B_FALSE);
215 }
216
217 if (inst->ri_i.i_primary_ctid != 0) {
218 log_framework(LOG_INFO,
219 "%s: primary ctid expected to be 0 but "
220 "was set to %ld\n", inst->ri_i.i_fmri,
221 inst->ri_i.i_primary_ctid);
222 }
223
224 inst->ri_i.i_primary_ctid = *cid;
225 inst->ri_i.i_primary_ctid_stopped = 0;
226
227 log_framework(LOG_DEBUG, "Storing primary contract %ld for "
228 "%s.\n", *cid, inst->ri_i.i_fmri);
229
230 contract_hash_store(*cid, inst->ri_id);
231 }
232
233 again:
234 if (inst->ri_mi_deleted)
235 return;
236
237 r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
238 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
239 switch (r) {
240 case 0:
241 break;
242
243 case ECANCELED:
244 inst->ri_mi_deleted = B_TRUE;
245 break;
246
247 case ECONNABORTED:
248 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
249 /* FALLTHROUGH */
250
251 case EBADF:
252 libscf_reget_instance(inst);
253 goto again;
254
255 case ENOMEM:
256 case EPERM:
257 case EACCES:
258 case EROFS:
259 uu_die("%s: Couldn't store contract id %ld",
260 inst->ri_i.i_fmri, *cid);
261 /* NOTREACHED */
262
263 case EINVAL:
264 default:
265 bad_error("restarter_store_contract", r);
266 }
267 }
268
269 /*
270 * void method_remove_contract()
271 * Remove any non-permanent contracts from internal structures and
272 * the repository, then abandon them.
273 * Returns
274 * 0 - success
275 * ECANCELED - inst was deleted from the repository
276 *
277 * If the repository connection was broken, it is rebound.
278 */
279 void
method_remove_contract(restarter_inst_t * inst,boolean_t primary,boolean_t abandon)280 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
281 boolean_t abandon)
282 {
283 ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
284 &inst->ri_i.i_transient_ctid;
285
286 int r;
287
288 assert(*ctidp != 0);
289
290 log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
291 primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
292
293 if (abandon)
294 contract_abandon(*ctidp);
295
296 again:
297 if (inst->ri_mi_deleted) {
298 r = ECANCELED;
299 goto out;
300 }
301
302 r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
303 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
304 switch (r) {
305 case 0:
306 break;
307
308 case ECANCELED:
309 inst->ri_mi_deleted = B_TRUE;
310 break;
311
312 case ECONNABORTED:
313 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
314 /* FALLTHROUGH */
315
316 case EBADF:
317 libscf_reget_instance(inst);
318 goto again;
319
320 case ENOMEM:
321 case EPERM:
322 case EACCES:
323 case EROFS:
324 log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
325 "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
326 break;
327
328 case EINVAL:
329 default:
330 bad_error("restarter_remove_contract", r);
331 }
332
333 out:
334 if (primary)
335 contract_hash_remove(*ctidp);
336
337 *ctidp = 0;
338 }
339
340 static const char *method_names[] = { "start", "stop", "refresh" };
341
342 /*
343 * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
344 *
345 * Activate a contract template for the type method of inst. type,
346 * restart_on, and cte_mask dictate the critical events term of the contract.
347 * Returns
348 * 0 - success
349 * ECANCELED - inst has been deleted from the repository
350 */
351 static int
method_ready_contract(restarter_inst_t * inst,int type,method_restart_t restart_on,uint_t cte_mask)352 method_ready_contract(restarter_inst_t *inst, int type,
353 method_restart_t restart_on, uint_t cte_mask)
354 {
355 int tmpl, err, istrans, iswait, ret;
356 uint_t cevents, fevents;
357
358 /*
359 * Correctly supporting wait-style services is tricky without
360 * rearchitecting startd to cope with multiple event sources
361 * simultaneously trying to stop an instance. Until a better
362 * solution is implemented, we avoid this problem for
363 * wait-style services by making contract events fatal and
364 * letting the wait code alone handle stopping the service.
365 */
366 iswait = instance_is_wait_style(inst);
367 istrans = method_is_transient(inst, type);
368
369 tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
370 if (tmpl == -1)
371 uu_die("Could not create contract template");
372
373 /*
374 * We assume non-login processes are unlikely to create
375 * multiple process groups, and set CT_PR_PGRPONLY for all
376 * wait-style services' contracts.
377 */
378 err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
379 (iswait ? CT_PR_PGRPONLY : 0));
380 assert(err == 0);
381
382 if (istrans) {
383 cevents = 0;
384 fevents = 0;
385 } else {
386 assert(restart_on >= 0);
387 assert(restart_on <= METHOD_RESTART_ANY_FAULT);
388 cevents = method_events[restart_on] & ~cte_mask;
389 fevents = iswait ?
390 (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
391 0;
392 }
393
394 err = ct_tmpl_set_critical(tmpl, cevents);
395 assert(err == 0);
396
397 err = ct_tmpl_set_informative(tmpl, 0);
398 assert(err == 0);
399 err = ct_pr_tmpl_set_fatal(tmpl, fevents);
400 assert(err == 0);
401
402 err = ct_tmpl_set_cookie(tmpl, istrans ? METHOD_OTHER_COOKIE :
403 METHOD_START_COOKIE);
404 assert(err == 0);
405
406 if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
407 ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
408 switch (ret) {
409 case 0:
410 break;
411
412 case ENOTEMPTY:
413 /* No contracts for you! */
414 method_remove_contract(inst, B_TRUE, B_TRUE);
415 if (inst->ri_mi_deleted) {
416 ret = ECANCELED;
417 goto out;
418 }
419 break;
420
421 case EINVAL:
422 case ESRCH:
423 case EACCES:
424 default:
425 bad_error("ct_pr_tmpl_set_transfer", ret);
426 }
427 }
428
429 err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
430 assert(err == 0);
431 err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
432 assert(err == 0);
433
434 err = ct_tmpl_activate(tmpl);
435 assert(err == 0);
436
437 ret = 0;
438
439 out:
440 err = close(tmpl);
441 assert(err == 0);
442
443 return (ret);
444 }
445
446 static void
exec_method(const restarter_inst_t * inst,int type,const char * method,struct method_context * mcp,uint8_t need_session)447 exec_method(const restarter_inst_t *inst, int type, const char *method,
448 struct method_context *mcp, uint8_t need_session)
449 {
450 char *cmd;
451 const char *errf;
452 char **nenv;
453 int rsmc_errno = 0;
454
455 cmd = uu_msprintf("exec %s", method);
456
457 if (inst->ri_utmpx_prefix != NULL && inst->ri_utmpx_prefix[0] != '\0')
458 (void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
459
460 setlog(inst->ri_logstem);
461 log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
462 method_names[type], method);
463
464 if (need_session)
465 (void) setpgrp();
466
467 /* Set credentials. */
468 rsmc_errno = restarter_set_method_context(mcp, &errf);
469 if (rsmc_errno != 0) {
470 log_instance(inst, B_FALSE,
471 "svc.startd could not set context for method: ");
472
473 if (rsmc_errno == -1) {
474 if (strcmp(errf, "core_set_process_path") == 0) {
475 log_instance(inst, B_FALSE,
476 "Could not set corefile path.");
477 } else if (strcmp(errf, "setproject") == 0) {
478 log_instance(inst, B_FALSE, "%s: a resource "
479 "control assignment failed", errf);
480 } else if (strcmp(errf, "pool_set_binding") == 0) {
481 log_instance(inst, B_FALSE, "%s: a system "
482 "error occurred", errf);
483 } else {
484 #ifndef NDEBUG
485 uu_warn("%s:%d: Bad function name \"%s\" for "
486 "error %d from "
487 "restarter_set_method_context().\n",
488 __FILE__, __LINE__, errf, rsmc_errno);
489 #endif
490 abort();
491 }
492
493 exit(1);
494 }
495
496 if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
497 switch (rsmc_errno) {
498 case ENOENT:
499 log_instance(inst, B_FALSE, "%s: the pool "
500 "could not be found", errf);
501 break;
502
503 case EBADF:
504 log_instance(inst, B_FALSE, "%s: the "
505 "configuration is invalid", errf);
506 break;
507
508 case EINVAL:
509 log_instance(inst, B_FALSE, "%s: pool name "
510 "\"%s\" is invalid", errf,
511 mcp->resource_pool);
512 break;
513
514 default:
515 #ifndef NDEBUG
516 uu_warn("%s:%d: Bad error %d for function %s "
517 "in restarter_set_method_context().\n",
518 __FILE__, __LINE__, rsmc_errno, errf);
519 #endif
520 abort();
521 }
522
523 exit(SMF_EXIT_ERR_CONFIG);
524 }
525
526 if (errf != NULL && strcmp(errf, "chdir") == 0) {
527 switch (rsmc_errno) {
528 case EACCES:
529 case EFAULT:
530 case EIO:
531 case ELOOP:
532 case ENAMETOOLONG:
533 case ENOENT:
534 case ENOLINK:
535 case ENOTDIR:
536 log_instance(inst, B_FALSE, "%s: %s (\"%s\")",
537 errf,
538 strerror(rsmc_errno), mcp->working_dir);
539 break;
540
541 default:
542 #ifndef NDEBUG
543 uu_warn("%s:%d: Bad error %d for function %s "
544 "in restarter_set_method_context().\n",
545 __FILE__, __LINE__, rsmc_errno, errf);
546 #endif
547 abort();
548 }
549
550 exit(SMF_EXIT_ERR_CONFIG);
551 }
552
553 if (errf != NULL) {
554 errno = rsmc_errno;
555 perror(errf);
556
557 switch (rsmc_errno) {
558 case EINVAL:
559 case EPERM:
560 case ENOENT:
561 case ENAMETOOLONG:
562 case ERANGE:
563 case ESRCH:
564 exit(SMF_EXIT_ERR_CONFIG);
565 /* NOTREACHED */
566
567 default:
568 exit(1);
569 }
570 }
571
572 switch (rsmc_errno) {
573 case ENOMEM:
574 log_instance(inst, B_FALSE, "Out of memory.");
575 exit(1);
576 /* NOTREACHED */
577
578 case ENOENT:
579 log_instance(inst, B_FALSE, "Missing passwd entry for "
580 "user.");
581 exit(SMF_EXIT_ERR_CONFIG);
582 /* NOTREACHED */
583
584 default:
585 #ifndef NDEBUG
586 uu_warn("%s:%d: Bad miscellaneous error %d from "
587 "restarter_set_method_context().\n", __FILE__,
588 __LINE__, rsmc_errno);
589 #endif
590 abort();
591 }
592 }
593
594 nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
595 method_names[type]);
596
597 log_preexec();
598
599 (void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
600
601 (void) fprintf(stderr, "Failed to exec %s -c '%s': %s\n",
602 SBIN_SH, cmd, strerror(errno));
603
604 exit(10);
605 }
606
607 static void
write_status(restarter_inst_t * inst,const char * mname,int stat)608 write_status(restarter_inst_t *inst, const char *mname, int stat)
609 {
610 int r;
611
612 again:
613 if (inst->ri_mi_deleted)
614 return;
615
616 r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
617 switch (r) {
618 case 0:
619 break;
620
621 case ECONNABORTED:
622 libscf_reget_instance(inst);
623 goto again;
624
625 case ECANCELED:
626 inst->ri_mi_deleted = 1;
627 break;
628
629 case EPERM:
630 case EACCES:
631 case EROFS:
632 log_framework(LOG_INFO, "Could not write exit status "
633 "for %s method of %s: %s.\n", mname,
634 inst->ri_i.i_fmri, strerror(r));
635 break;
636
637 case ENAMETOOLONG:
638 default:
639 bad_error("libscf_write_method_status", r);
640 }
641 }
642
643 /*
644 * int method_run()
645 * Execute the type method of instp. If it requires a fork(), wait for it
646 * to return and return its exit code in *exit_code. Otherwise set
647 * *exit_code to 0 if the method succeeds & -1 if it fails. If the
648 * repository connection is broken, it is rebound, but inst may not be
649 * reset.
650 * Returns
651 * 0 - success
652 * EINVAL - A correct method or method context couldn't be retrieved.
653 * EIO - Contract kill failed.
654 * EFAULT - Method couldn't be executed successfully.
655 * ELOOP - Retry threshold exceeded.
656 * ECANCELED - inst was deleted from the repository before method was run
657 * ERANGE - Timeout retry threshold exceeded.
658 * EAGAIN - Failed due to external cause, retry.
659 */
660 int
method_run(restarter_inst_t ** instp,int type,int * exit_code)661 method_run(restarter_inst_t **instp, int type, int *exit_code)
662 {
663 char *method;
664 int ret_status;
665 pid_t pid;
666 method_restart_t restart_on;
667 uint_t cte_mask;
668 uint8_t need_session;
669 scf_handle_t *h;
670 scf_snapshot_t *snap;
671 const char *mname;
672 mc_error_t *m_error;
673 struct method_context *mcp;
674 int result = 0, timeout_fired = 0;
675 int sig, r;
676 boolean_t transient;
677 uint64_t timeout;
678 uint8_t timeout_retry;
679 ctid_t ctid;
680 int ctfd = -1;
681 restarter_inst_t *inst = *instp;
682 int id = inst->ri_id;
683 int forkerr;
684
685 assert(MUTEX_HELD(&inst->ri_lock));
686 assert(instance_in_transition(inst));
687
688 if (inst->ri_mi_deleted)
689 return (ECANCELED);
690
691 *exit_code = SMF_EXIT_OK;
692
693 assert(0 <= type && type <= 2);
694 mname = method_names[type];
695
696 if (type == METHOD_START)
697 inst->ri_pre_online_hook();
698
699 h = scf_instance_handle(inst->ri_m_inst);
700
701 snap = scf_snapshot_create(h);
702 if (snap == NULL ||
703 scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
704 log_framework(LOG_DEBUG,
705 "Could not get running snapshot for %s. "
706 "Using editing version to run method %s.\n",
707 inst->ri_i.i_fmri, mname);
708 scf_snapshot_destroy(snap);
709 snap = NULL;
710 }
711
712 /*
713 * After this point, we may be logging to the instance log.
714 * Make sure we've noted where that log is as a property of
715 * the instance.
716 */
717 r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
718 inst->ri_logstem);
719 if (r != 0) {
720 log_framework(LOG_WARNING,
721 "%s: couldn't note log location: %s\n",
722 inst->ri_i.i_fmri, strerror(r));
723 }
724
725 if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
726 &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
727 if (errno == LIBSCF_PGROUP_ABSENT) {
728 log_framework(LOG_DEBUG,
729 "%s: instance has no method property group '%s'.\n",
730 inst->ri_i.i_fmri, mname);
731 if (type == METHOD_REFRESH)
732 log_instance(inst, B_TRUE, "No '%s' method "
733 "defined. Treating as :true.", mname);
734 else
735 log_instance(inst, B_TRUE, "Method property "
736 "group '%s' is not present.", mname);
737 scf_snapshot_destroy(snap);
738 return (0);
739 } else if (errno == LIBSCF_PROPERTY_ABSENT) {
740 log_framework(LOG_DEBUG,
741 "%s: instance has no '%s/exec' method property.\n",
742 inst->ri_i.i_fmri, mname);
743 log_instance(inst, B_TRUE, "Method property '%s/exec "
744 "is not present.", mname);
745 scf_snapshot_destroy(snap);
746 return (0);
747 } else {
748 log_error(LOG_WARNING,
749 "%s: instance libscf_get_method failed\n",
750 inst->ri_i.i_fmri);
751 scf_snapshot_destroy(snap);
752 return (EINVAL);
753 }
754 }
755
756 /* open service contract if stopping a non-transient service */
757 if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
758 if (inst->ri_i.i_primary_ctid == 0) {
759 /* service is not running, nothing to stop */
760 log_framework(LOG_DEBUG, "%s: instance has no primary "
761 "contract, no service to stop.\n",
762 inst->ri_i.i_fmri);
763 scf_snapshot_destroy(snap);
764 return (0);
765 }
766 if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
767 "events", O_RDONLY)) < 0) {
768 result = EFAULT;
769 log_instance(inst, B_TRUE, "Could not open service "
770 "contract %ld. Stop method not run.",
771 inst->ri_i.i_primary_ctid);
772 goto out;
773 }
774 }
775
776 if (restarter_is_null_method(method)) {
777 log_framework(LOG_DEBUG, "%s: null method succeeds\n",
778 inst->ri_i.i_fmri);
779
780 log_instance(inst, B_TRUE, "Executing %s method (null).",
781 mname);
782
783 if (type == METHOD_START)
784 write_status(inst, mname, 0);
785 goto out;
786 }
787
788 sig = restarter_is_kill_method(method);
789 if (sig >= 0) {
790
791 if (inst->ri_i.i_primary_ctid == 0) {
792 log_error(LOG_ERR, "%s: :kill with no contract\n",
793 inst->ri_i.i_fmri);
794 log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
795 "as stop method for transient service.");
796 result = EINVAL;
797 goto out;
798 }
799
800 log_framework(LOG_DEBUG,
801 "%s: :killing contract with signal %d\n",
802 inst->ri_i.i_fmri, sig);
803
804 log_instance(inst, B_TRUE, "Executing %s method (:kill).",
805 mname);
806
807 if (contract_kill(inst->ri_i.i_primary_ctid, sig,
808 inst->ri_i.i_fmri) != 0) {
809 result = EIO;
810 goto out;
811 } else
812 goto assured_kill;
813 }
814
815 log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
816 inst->ri_i.i_fmri, method);
817
818 m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
819 inst->ri_m_inst, snap, mname, method, &mcp);
820
821 if (m_error != NULL) {
822 log_instance(inst, B_TRUE, "%s", m_error->msg);
823 restarter_mc_error_destroy(m_error);
824 result = EINVAL;
825 goto out;
826 }
827
828 r = method_ready_contract(inst, type, restart_on, cte_mask);
829 if (r != 0) {
830 assert(r == ECANCELED);
831 assert(inst->ri_mi_deleted);
832 restarter_free_method_context(mcp);
833 result = ECANCELED;
834 goto out;
835 }
836
837 /*
838 * Validate safety of method contexts, to save children work.
839 */
840 if (!restarter_rm_libs_loadable())
841 log_framework(LOG_DEBUG, "%s: method contexts limited "
842 "to root-accessible libraries\n", inst->ri_i.i_fmri);
843
844 /*
845 * For wait-style svc, sanity check that method exists to prevent an
846 * infinite loop.
847 */
848 if (instance_is_wait_style(inst) && type == METHOD_START) {
849 char *pend;
850 struct stat64 sbuf;
851
852 /*
853 * We need to handle start method strings that have arguments,
854 * such as '/lib/svc/method/console-login %i'.
855 */
856 if ((pend = strchr(method, ' ')) != NULL)
857 *pend = '\0';
858
859 if (*method == '/' && stat64(method, &sbuf) == -1 &&
860 errno == ENOENT) {
861 log_instance(inst, B_TRUE, "Missing start method (%s), "
862 "changing state to maintenance.", method);
863 restarter_free_method_context(mcp);
864 result = ENOENT;
865 goto out;
866 }
867 if (pend != NULL)
868 *pend = ' ';
869 }
870
871 /*
872 * If the service is restarting too quickly, send it to
873 * maintenance.
874 */
875 if (type == METHOD_START) {
876 method_record_start(inst);
877 if (method_rate_critical(inst) &&
878 !instance_is_wait_style(inst)) {
879 log_instance(inst, B_TRUE, "Restarting too quickly, "
880 "changing state to maintenance.");
881 result = ELOOP;
882 restarter_free_method_context(mcp);
883 goto out;
884 }
885 }
886
887 atomic_add_16(&storing_contract, 1);
888 pid = startd_fork1(&forkerr);
889 if (pid == 0)
890 exec_method(inst, type, method, mcp, need_session);
891
892 if (pid == -1) {
893 atomic_add_16(&storing_contract, -1);
894 if (forkerr == EAGAIN)
895 result = EAGAIN;
896 else
897 result = EFAULT;
898
899 log_error(LOG_WARNING,
900 "%s: Couldn't fork to execute method %s: %s\n",
901 inst->ri_i.i_fmri, method, strerror(forkerr));
902
903 restarter_free_method_context(mcp);
904 goto out;
905 }
906
907
908 /*
909 * Get the contract id, decide whether it is primary or transient, and
910 * stash it in inst & the repository.
911 */
912 method_store_contract(inst, type, &ctid);
913 atomic_add_16(&storing_contract, -1);
914
915 restarter_free_method_context(mcp);
916
917 /*
918 * Similarly for the start method PID.
919 */
920 if (type == METHOD_START && !inst->ri_mi_deleted)
921 (void) libscf_write_start_pid(inst->ri_m_inst, pid);
922
923 if (instance_is_wait_style(inst) && type == METHOD_START) {
924 /* Wait style instances don't get timeouts on start methods. */
925 if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
926 log_error(LOG_WARNING,
927 "%s: couldn't register %ld for wait\n",
928 inst->ri_i.i_fmri, pid);
929 result = EFAULT;
930 goto contract_out;
931 }
932 write_status(inst, mname, 0);
933
934 } else {
935 int r, err;
936 time_t start_time;
937 time_t end_time;
938
939 /*
940 * Because on upgrade/live-upgrade we may have no chance
941 * to override faulty timeout values on the way to
942 * manifest import, all services on the path to manifest
943 * import are treated the same as INFINITE timeout services.
944 */
945
946 start_time = time(NULL);
947 if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
948 timeout_insert(inst, ctid, timeout);
949 else
950 timeout = METHOD_TIMEOUT_INFINITE;
951
952 /* Unlock the instance while waiting for the method. */
953 MUTEX_UNLOCK(&inst->ri_lock);
954
955 do {
956 r = waitpid(pid, &ret_status, 0);
957 } while (r == -1 && errno == EINTR);
958 if (r == -1)
959 err = errno;
960
961 /* Re-grab the lock. */
962 inst = inst_lookup_by_id(id);
963
964 /*
965 * inst can't be removed, as the removal thread waits
966 * for completion of this one.
967 */
968 assert(inst != NULL);
969 *instp = inst;
970
971 if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
972 timeout_fired = 1;
973
974 timeout_remove(inst, ctid);
975
976 log_framework(LOG_DEBUG,
977 "%s method for %s exited with status %d.\n", mname,
978 inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
979
980 if (r == -1) {
981 log_error(LOG_WARNING,
982 "Couldn't waitpid() for %s method of %s (%s).\n",
983 mname, inst->ri_i.i_fmri, strerror(err));
984 result = EFAULT;
985 goto contract_out;
986 }
987
988 if (type == METHOD_START)
989 write_status(inst, mname, ret_status);
990
991 /* return ERANGE if this service doesn't retry on timeout */
992 if (timeout_fired == 1 && timeout_retry == 0) {
993 result = ERANGE;
994 goto contract_out;
995 }
996
997 if (!WIFEXITED(ret_status)) {
998 /*
999 * If method didn't exit itself (it was killed by an
1000 * external entity, etc.), consider the entire
1001 * method_run as failed.
1002 */
1003 if (WIFSIGNALED(ret_status)) {
1004 char buf[SIG2STR_MAX];
1005 (void) sig2str(WTERMSIG(ret_status), buf);
1006
1007 log_error(LOG_WARNING, "%s: Method \"%s\" "
1008 "failed due to signal %s.\n",
1009 inst->ri_i.i_fmri, method, buf);
1010 log_instance(inst, B_TRUE, "Method \"%s\" "
1011 "failed due to signal %s.", mname, buf);
1012 } else {
1013 log_error(LOG_WARNING, "%s: Method \"%s\" "
1014 "failed with exit status %d.\n",
1015 inst->ri_i.i_fmri, method,
1016 WEXITSTATUS(ret_status));
1017 log_instance(inst, B_TRUE, "Method \"%s\" "
1018 "failed with exit status %d.", mname,
1019 WEXITSTATUS(ret_status));
1020 }
1021 result = EAGAIN;
1022 goto contract_out;
1023 }
1024
1025 *exit_code = WEXITSTATUS(ret_status);
1026 if (*exit_code != SMF_EXIT_OK &&
1027 *exit_code != SMF_EXIT_NODAEMON &&
1028 *exit_code != SMF_EXIT_MON_DEGRADE &&
1029 *exit_code != SMF_EXIT_TEMP_DISABLE) {
1030 log_error(LOG_WARNING,
1031 "%s: Method \"%s\" failed with exit status %d.\n",
1032 inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
1033 }
1034
1035 log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
1036 "%d.", mname, *exit_code);
1037
1038 /*
1039 * Note: we will take this path for SMF_EXIT_NODAEMON and
1040 * SMF_EXIT_TEMP_DISABLE.
1041 */
1042 if (*exit_code != SMF_EXIT_OK &&
1043 *exit_code != SMF_EXIT_MON_DEGRADE) {
1044 goto contract_out;
1045 }
1046
1047 end_time = time(NULL);
1048
1049 /* Give service contract remaining seconds to empty */
1050 if (timeout != METHOD_TIMEOUT_INFINITE)
1051 timeout -= (end_time - start_time);
1052 }
1053
1054 assured_kill:
1055 /*
1056 * For stop methods, assure that the service contract has emptied
1057 * before returning.
1058 */
1059 if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
1060 !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
1061 int times = 0;
1062
1063 if (timeout != METHOD_TIMEOUT_INFINITE)
1064 timeout_insert(inst, inst->ri_i.i_primary_ctid,
1065 timeout);
1066
1067 for (;;) {
1068 /*
1069 * Check frequently at first, then back off. This
1070 * keeps startd from idling while shutting down.
1071 */
1072 if (times < 20) {
1073 (void) poll(NULL, 0, 5);
1074 times++;
1075 } else {
1076 (void) poll(NULL, 0, 100);
1077 }
1078 if (contract_is_empty(inst->ri_i.i_primary_ctid))
1079 break;
1080 }
1081
1082 if (timeout != METHOD_TIMEOUT_INFINITE)
1083 if (inst->ri_timeout->te_fired)
1084 result = EFAULT;
1085
1086 timeout_remove(inst, inst->ri_i.i_primary_ctid);
1087 }
1088
1089 contract_out:
1090 /*
1091 * Abandon contracts for transient methods, methods that exit with
1092 * SMF_EXIT_NODAEMON/SMF_EXIT_TEMP_DISABLE and methods that fail.
1093 * Non-transient degraded services are left alone here. If their
1094 * contract is or later becomes empty, then that will be handled in the
1095 * same way as for any other non-transient service.
1096 */
1097 transient = method_is_transient(inst, type);
1098 if ((transient ||
1099 (*exit_code != SMF_EXIT_OK && *exit_code != SMF_EXIT_MON_DEGRADE) ||
1100 result != 0) &&
1101 restarter_is_kill_method(method) < 0) {
1102 method_remove_contract(inst, !transient, B_TRUE);
1103 }
1104
1105 out:
1106 if (ctfd >= 0)
1107 (void) close(ctfd);
1108 scf_snapshot_destroy(snap);
1109 free(method);
1110 return (result);
1111 }
1112
1113 /*
1114 * A start method has exited with SMF_EXIT_TEMP_DISABLE, requesting that the
1115 * service be temporarily disabled. We do this in the same way as
1116 * 'svcadm disable -t', by setting the enabled override in the non-persistent
1117 * general_ovr property group. The graph engine notices this change and drives
1118 * the instance to the disabled state through the normal disable path, so we do
1119 * not change the instance state ourselves here. Because the override is
1120 * non-persistent, it does not survive a reboot, and the method will run again
1121 * on next boot in case the condition that prompted the service to disable
1122 * itself has since cleared.
1123 */
1124 static void
method_temp_disable(restarter_inst_t * inst)1125 method_temp_disable(restarter_inst_t *inst)
1126 {
1127 int r;
1128
1129 assert(MUTEX_HELD(&inst->ri_lock));
1130
1131 for (;;) {
1132 switch (r = libscf_set_enable_ovr(inst->ri_m_inst, 0)) {
1133 case 0:
1134 case ECANCELED: /* the instance has been deleted */
1135 return;
1136
1137 case ECONNABORTED:
1138 libscf_reget_instance(inst);
1139 continue;
1140
1141 case EPERM:
1142 case EROFS:
1143 log_error(LOG_WARNING, "%s: could not set the enabled "
1144 "override to temporarily disable the service: "
1145 "%s.\n", inst->ri_i.i_fmri, strerror(r));
1146 return;
1147
1148 default:
1149 bad_error("libscf_set_enable_ovr", r);
1150 }
1151 }
1152 }
1153
1154 /*
1155 * The method thread executes a service method to effect a state transition.
1156 * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1157 * be _NONE on exit (state will either be what next_state was (on success), or
1158 * it will be _MAINT (on error)).
1159 *
1160 * There are six classes of methods to consider: start & other (stop, refresh)
1161 * for each of "normal" services, wait services, and transient services. For
1162 * each, the method must be fetched from the repository & executed. fork()ed
1163 * methods must be waited on, except for the start method of wait services
1164 * (which must be registered with the wait subsystem via wait_register()). If
1165 * the method succeeded (returned 0), then for start methods its contract
1166 * should be recorded as the primary contract for the service. For other
1167 * methods, it should be abandoned. If the method fails, then depending on
1168 * the failure, either the method should be reexecuted or the service should
1169 * be put into maintenance. Either way the contract should be abandoned.
1170 */
1171 void *
method_thread(void * arg)1172 method_thread(void *arg)
1173 {
1174 fork_info_t *info = arg;
1175 restarter_inst_t *inst;
1176 scf_handle_t *local_handle;
1177 scf_instance_t *s_inst = NULL;
1178 int r, exit_code;
1179 boolean_t retryable;
1180 restarter_str_t reason;
1181
1182 (void) pthread_setname_np(pthread_self(), "method");
1183
1184 assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1185
1186 /* Get (and lock) the restarter_inst_t. */
1187 inst = inst_lookup_by_id(info->sf_id);
1188
1189 assert(inst->ri_method_thread != 0);
1190 assert(instance_in_transition(inst) == 1);
1191
1192 /*
1193 * We cannot leave this function with inst in transition, because
1194 * protocol.c withholds messages for inst otherwise.
1195 */
1196
1197 log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1198 method_names[info->sf_method_type], inst->ri_i.i_fmri);
1199
1200 local_handle = libscf_handle_create_bound_loop();
1201
1202 rebind_retry:
1203 /* get scf_instance_t */
1204 switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1205 &s_inst)) {
1206 case 0:
1207 break;
1208
1209 case ECONNABORTED:
1210 libscf_handle_rebind(local_handle);
1211 goto rebind_retry;
1212
1213 case ENOENT:
1214 /*
1215 * It's not there, but we need to call this so protocol.c
1216 * doesn't think it's in transition anymore.
1217 */
1218 (void) restarter_instance_update_states(local_handle, inst,
1219 inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1220 restarter_str_none);
1221 goto out;
1222
1223 case EINVAL:
1224 case ENOTSUP:
1225 default:
1226 bad_error("libscf_fmri_get_instance", r);
1227 }
1228
1229 inst->ri_m_inst = s_inst;
1230 inst->ri_mi_deleted = B_FALSE;
1231
1232 retry:
1233 if (info->sf_method_type == METHOD_START)
1234 log_transition(inst, START_REQUESTED);
1235
1236 r = method_run(&inst, info->sf_method_type, &exit_code);
1237
1238 if (r == 0 &&
1239 (exit_code == SMF_EXIT_OK || exit_code == SMF_EXIT_NODAEMON ||
1240 exit_code == SMF_EXIT_MON_DEGRADE ||
1241 exit_code == SMF_EXIT_TEMP_DISABLE)) {
1242 /* Success! */
1243 assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1244
1245 /*
1246 * When a stop method succeeds, remove the primary contract of
1247 * the service, unless we're going to offline, in which case
1248 * retain the contract so we can transfer inherited contracts to
1249 * the replacement service.
1250 */
1251
1252 if (info->sf_method_type == METHOD_STOP &&
1253 inst->ri_i.i_primary_ctid != 0) {
1254 if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1255 inst->ri_i.i_primary_ctid_stopped = 1;
1256 else
1257 method_remove_contract(inst, B_TRUE, B_TRUE);
1258 }
1259
1260 /*
1261 * For methods that exit with SMF_EXIT_NODAEMON, we already
1262 * called method_remove_contract in method_run.
1263 */
1264
1265 /*
1266 * When a start method returns with SMF_EXIT_MON_DEGRADE we
1267 * transition the service into degraded.
1268 */
1269 if (info->sf_method_type == METHOD_START &&
1270 exit_code == SMF_EXIT_MON_DEGRADE) {
1271 inst->ri_i.i_next_state = RESTARTER_STATE_DEGRADED;
1272 info->sf_reason = restarter_str_method_failed;
1273 log_transition(inst, START_FAILED_DEGRADED);
1274 }
1275
1276 /*
1277 * When a start method exits with SMF_EXIT_TEMP_DISABLE the
1278 * service has asked to be temporarily disabled. Request the
1279 * disable here; the instance is left to transition normally.
1280 * See the comment above method_temp_disable() for details.
1281 * The status is honoured only for start methods, so log a
1282 * warning for any other method type to give the service
1283 * author a chance to notice the error.
1284 */
1285 if (exit_code == SMF_EXIT_TEMP_DISABLE) {
1286 if (info->sf_method_type == METHOD_START) {
1287 method_temp_disable(inst);
1288 } else {
1289 log_instance(inst, B_TRUE, "Method \"%s\" "
1290 "exited with SMF_EXIT_TEMP_DISABLE, "
1291 "which is only supported for start "
1292 "methods; ignoring.",
1293 method_names[info->sf_method_type]);
1294 }
1295 }
1296
1297 /*
1298 * We don't care whether the handle was rebound because this is
1299 * the last thing we do with it.
1300 */
1301 (void) restarter_instance_update_states(local_handle, inst,
1302 inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1303 info->sf_event_type, info->sf_reason);
1304
1305 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1306
1307 goto out;
1308 }
1309
1310 /* Failure. Retry or go to maintenance. */
1311
1312 if (r != 0 && r != EAGAIN) {
1313 retryable = B_FALSE;
1314 } else {
1315 switch (exit_code) {
1316 case SMF_EXIT_ERR_CONFIG:
1317 case SMF_EXIT_ERR_NOSMF:
1318 case SMF_EXIT_ERR_PERM:
1319 case SMF_EXIT_ERR_FATAL:
1320 retryable = B_FALSE;
1321 break;
1322
1323 default:
1324 retryable = B_TRUE;
1325 }
1326 }
1327
1328 if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1329 goto retry;
1330
1331 /* maintenance */
1332 if (r == ELOOP)
1333 log_transition(inst, START_FAILED_REPEATEDLY);
1334 else if (r == ERANGE)
1335 log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1336 else if (exit_code == SMF_EXIT_ERR_CONFIG)
1337 log_transition(inst, START_FAILED_CONFIGURATION);
1338 else if (exit_code == SMF_EXIT_ERR_FATAL)
1339 log_transition(inst, START_FAILED_FATAL);
1340 else
1341 log_transition(inst, START_FAILED_OTHER);
1342
1343 if (r == ELOOP) {
1344 reason = restarter_str_restarting_too_quickly;
1345 } else if (retryable) {
1346 reason = restarter_str_fault_threshold_reached;
1347 } else {
1348 reason = restarter_str_method_failed;
1349 }
1350
1351 (void) restarter_instance_update_states(local_handle, inst,
1352 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1353 reason);
1354
1355 if (!method_is_transient(inst, info->sf_method_type) &&
1356 inst->ri_i.i_primary_ctid != 0)
1357 method_remove_contract(inst, B_TRUE, B_TRUE);
1358
1359 out:
1360 inst->ri_method_thread = 0;
1361
1362 /*
1363 * Unlock the mutex after broadcasting to avoid a race condition
1364 * with restarter_delete_inst() when the 'inst' structure is freed.
1365 */
1366 (void) pthread_cond_broadcast(&inst->ri_method_cv);
1367 MUTEX_UNLOCK(&inst->ri_lock);
1368
1369 scf_instance_destroy(s_inst);
1370 scf_handle_destroy(local_handle);
1371 startd_free(info, sizeof (fork_info_t));
1372 return (NULL);
1373 }
1374