1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2026 by Garth Snyder. All rights reserved.
15 */
16
17 /*
18 * This is a watchdog timer and multithread backtrace dumper that's used by
19 * zstream selftest. libspl_backtrace() does all the real work, but it can
20 * only dump the current thread's stack. We need to get every thread to call
21 * libspl_backtrace() in an organized sequence. However, there's no "give me
22 * a list of all pthreads" function in the POSIX API.
23 *
24 * Rather than constructing an ad-hoc thread registry, we can approach the
25 * problem by unblocking THREAD_BACKTRACE_SIGNAL (an arbitrary choice) when
26 * zstream first starts. All created threads then inherit this signal mask.
27 *
28 * In selftest mode, we add the backtrace dumper as a handler for
29 * THREAD_BACKTRACE_SIGNAL. The handler calls libspl_backtrace(), which is
30 * signal-handler safe. It then signals a semaphore to indicate that the
31 * current thread has finished its dump and suspends itself.
32 *
33 * The watchdog supervisor is a separate thread that sigwait()s for SIGALRM.
34 * It blocks THREAD_BACKTRACE_SIGNAL and runs its own libspl_backtrace(). It
35 * then enters a loop in which it sends THREAD_BACKTRACE_SIGNAL to the
36 * process as a whole and runs sem_timedwait() to see if any thread woke up
37 * and dumped its backtrace. If that call times out, either the receiving
38 * thread wedged while trying to backtrace or there were no more threads to
39 * backtrace.
40 *
41 * These two scenarios can be distinguished by calling sigpending(). If
42 * THREAD_BACKTRACE_SIGNAL still shows as being pending on the process, then
43 * there was no thread to receive it and we are done. If there's no pending
44 * signal, then some thread did receive the signal but failed to post to the
45 * semaphore; we print a "thread wedged while backtracing" message and
46 * continue the loop.
47 */
48
49 #include <err.h>
50 #include <errno.h>
51 #include <semaphore.h>
52 #include <signal.h>
53 #include <stdio.h>
54 #include <stdlib.h>
55 #include <sys/backtrace.h>
56 #include <sys/debug.h>
57 #include <sys/stdtypes.h>
58 #include <sys/types.h>
59 #include <time.h>
60 #include <unistd.h>
61
62 #include "zstream.h"
63 #include "zstream_backtrace.h"
64 #include "zstream_util.h"
65
66 /*
67 * Watchdog timeout in seconds. A test that hits this limit is almost
68 * certainly deadlocked, and the watchdog converts the hang into a test
69 * failure instead of a stuck test run.
70 */
71 #define WATCHDOG_TIMEOUT_SECS 120
72 #define MAX_SECS_FOR_BACKTRACE 2
73
74 static sem_t sem_thread_bt_complete; /* thread -> watchdog */
75
76 /*
77 * Signal handler for THREAD_BACKTRACE_SIGNAL, run by all threads except the
78 * watchdog thread
79 */
80 static void
backtrace_self(int signal)81 backtrace_self(int signal)
82 {
83 (void) signal;
84 ssize_t dummy __maybe_unused = write(STDERR_FILENO, "\n", 1);
85 libspl_backtrace(STDERR_FILENO);
86 sem_post(&sem_thread_bt_complete);
87
88 sigset_t mask;
89 sigfillset(&mask);
90 sigsuspend(&mask);
91 }
92
93 static void
backtrace_all_threads(void)94 backtrace_all_threads(void)
95 {
96 while (B_TRUE) {
97 if (kill(getpid(), THREAD_BACKTRACE_SIGNAL) != 0)
98 err(1, "failed to send thread backtrace signal");
99 struct timespec deadline;
100 clock_gettime(CLOCK_REALTIME, &deadline);
101 deadline.tv_sec += MAX_SECS_FOR_BACKTRACE;
102 if (sem_timedwait(&sem_thread_bt_complete, &deadline) != 0) {
103 sigset_t pending;
104 if (sigpending(&pending) != 0)
105 err(1, "sigpending failed");
106 if (sigismember(&pending, THREAD_BACKTRACE_SIGNAL)) {
107 return;
108 } else {
109 warnx("a thread failed to generate a backtrace,"
110 " continuing...");
111 }
112 }
113 }
114 }
115
116 /*
117 * Body of the watchdog thread
118 */
119 static void *
watchdog(void * nope)120 watchdog(void *nope)
121 {
122 (void) nope;
123 int signal;
124 sigset_t bt_mask, dog_mask;
125
126 /*
127 * This thread does its own backtrace, so we block the
128 * backtrace signal.
129 */
130 sigemptyset(&bt_mask);
131 sigaddset(&bt_mask, THREAD_BACKTRACE_SIGNAL);
132 if (pthread_sigmask(SIG_BLOCK, &bt_mask, NULL) != 0)
133 err(1, "pthread_sigmask failed");
134
135 sigemptyset(&dog_mask);
136 sigaddset(&dog_mask, WATCHDOG_SIGNAL);
137
138 int rc = sigwait(&dog_mask, &signal);
139 if (rc != 0) {
140 errno = rc;
141 err(1, "watchdog sigwait failed");
142 } else if (signal != WATCHDOG_SIGNAL) {
143 errx(1, "unexpected signal %d received by watchdog", signal);
144 }
145
146 fprintf(stderr, "\n\nWATCHDOG TIMER EXPIRED\n"
147 "Dumping backtrace for all threads...\n\n");
148 fflush(stderr);
149 libspl_backtrace(STDERR_FILENO);
150 backtrace_all_threads();
151 fprintf(stderr, "\nAll threads backtraced, exiting.\n");
152 exit(1);
153 }
154
155 void
watchdog_init(void)156 watchdog_init(void)
157 {
158 if (sem_init(&sem_thread_bt_complete, 0, 0) != 0)
159 err(1, "watchdog sem_init failed");
160
161 safe_create_thread(watchdog, NULL, "watchdog", B_TRUE);
162
163 struct sigaction sa = {
164 .sa_handler = backtrace_self,
165 .sa_flags = SA_RESTART
166 };
167 if (sigaction(THREAD_BACKTRACE_SIGNAL, &sa, NULL) != 0)
168 err(1, "backtrace sigaction failed");
169 }
170
171 void
watchdog_arm(void)172 watchdog_arm(void)
173 {
174 (void) alarm(WATCHDOG_TIMEOUT_SECS);
175 }
176
177 void
watchdog_disarm(void)178 watchdog_disarm(void)
179 {
180 (void) alarm(0);
181 }
182