1 /*
2 * CDDL HEADER START
3 *
4 * This file and its contents are supplied under the terms of the
5 * Common Development and Distribution License ("CDDL"), version 1.0.
6 * You may only use this file in accordance with the terms of version
7 * 1.0 of the CDDL.
8 *
9 * A full copy of the text of the CDDL should have accompanied this
10 * source. A copy of the CDDL is also available via the Internet at
11 * http://www.illumos.org/license/CDDL.
12 *
13 * CDDL HEADER END
14 */
15
16 /*
17 * Copyright (c) 2017, 2019 by Delphix. All rights reserved.
18 */
19
20 /*
21 * ZTHR Infrastructure
22 * ===================
23 *
24 * ZTHR threads are used for isolated operations that span multiple txgs
25 * within a SPA. They generally exist from SPA creation/loading and until
26 * the SPA is exported/destroyed. The ideal requirements for an operation
27 * to be modeled with a zthr are the following:
28 *
29 * 1] The operation needs to run over multiple txgs.
30 * 2] There is be a single point of reference in memory or on disk that
31 * indicates whether the operation should run/is running or has
32 * stopped.
33 *
34 * If the operation satisfies the above then the following rules guarantee
35 * a certain level of correctness:
36 *
37 * 1] Any thread EXCEPT the zthr changes the work indicator from stopped
38 * to running but not the opposite.
39 * 2] Only the zthr can change the work indicator from running to stopped
40 * (e.g. when it is done) but not the opposite.
41 *
42 * This way a normal zthr cycle should go like this:
43 *
44 * 1] An external thread changes the work indicator from stopped to
45 * running and wakes up the zthr.
46 * 2] The zthr wakes up, checks the indicator and starts working.
47 * 3] When the zthr is done, it changes the indicator to stopped, allowing
48 * a new cycle to start.
49 *
50 * Besides being awakened by other threads, a zthr can be configured
51 * during creation to wakeup on it's own after a specified interval
52 * [see zthr_create_timer()].
53 *
54 * Note: ZTHR threads are NOT a replacement for generic threads! Please
55 * ensure that they fit your use-case well before using them.
56 *
57 * == ZTHR creation
58 *
59 * Every zthr needs three inputs to start running:
60 *
61 * 1] A user-defined checker function (checkfunc) that decides whether
62 * the zthr should start working or go to sleep. The function should
63 * return TRUE when the zthr needs to work or FALSE to let it sleep,
64 * and should adhere to the following signature:
65 * boolean_t checkfunc_name(void *args, zthr_t *t);
66 *
67 * 2] A user-defined ZTHR function (func) which the zthr executes when
68 * it is not sleeping. The function should adhere to the following
69 * signature type:
70 * void func_name(void *args, zthr_t *t);
71 *
72 * 3] A void args pointer that will be passed to checkfunc and func
73 * implicitly by the infrastructure.
74 *
75 * The reason why the above API needs two different functions,
76 * instead of one that both checks and does the work, has to do with
77 * the zthr's internal state lock (zthr_state_lock) and the allowed
78 * cancellation windows. We want to hold the zthr_state_lock while
79 * running checkfunc but not while running func. This way the zthr
80 * can be cancelled while doing work and not while checking for work.
81 *
82 * To start a zthr:
83 * zthr_t *zthr_pointer = zthr_create(checkfunc, func, args);
84 * or
85 * zthr_t *zthr_pointer = zthr_create_timer(checkfunc, func,
86 * args, max_sleep);
87 *
88 * After that you should be able to wakeup, cancel, and resume the
89 * zthr from another thread using the zthr_pointer.
90 *
91 * NOTE: ZTHR threads could potentially wake up spuriously and the
92 * user should take this into account when writing a checkfunc.
93 * [see ZTHR state transitions]
94 *
95 * == ZTHR cancellation
96 *
97 * ZTHR threads must be cancelled when their SPA is being exported
98 * or when they need to be paused so they don't interfere with other
99 * operations.
100 *
101 * To cancel a zthr:
102 * zthr_cancel(zthr_pointer);
103 *
104 * To resume it:
105 * zthr_resume(zthr_pointer);
106 *
107 * A zthr will implicitly check if it has received a cancellation
108 * signal every time func returns and every time it wakes up [see
109 * ZTHR state transitions below].
110 *
111 * At times, waiting for the zthr's func to finish its job may take
112 * time. This may be very time-consuming for some operations that
113 * need to cancel the SPA's zthrs (e.g spa_export). For this scenario
114 * the user can explicitly make their ZTHR function aware of incoming
115 * cancellation signals using zthr_iscancelled(). A common pattern for
116 * that looks like this:
117 *
118 * int
119 * func_name(void *args, zthr_t *t)
120 * {
121 * ... <unpack args> ...
122 * while (!work_done && !zthr_iscancelled(t)) {
123 * ... <do more work> ...
124 * }
125 * }
126 *
127 * == ZTHR cleanup
128 *
129 * Cancelling a zthr doesn't clean up its metadata (internal locks,
130 * function pointers to func and checkfunc, etc..). This is because
131 * we want to keep them around in case we want to resume the execution
132 * of the zthr later. Similarly for zthrs that exit themselves.
133 *
134 * To completely cleanup a zthr, cancel it first to ensure that it
135 * is not running and then use zthr_destroy().
136 *
137 * == ZTHR state transitions
138 *
139 * zthr creation
140 * +
141 * |
142 * | woke up
143 * | +--------------+ sleep
144 * | | ^
145 * | | |
146 * | | | FALSE
147 * | | |
148 * v v FALSE +
149 * cancelled? +---------> checkfunc?
150 * + ^ +
151 * | | |
152 * | | | TRUE
153 * | | |
154 * | | func returned v
155 * | +---------------+ func
156 * |
157 * | TRUE
158 * |
159 * v
160 * zthr stopped running
161 *
162 * == Implementation of ZTHR requests
163 *
164 * ZTHR wakeup, cancel, and resume are requests on a zthr to
165 * change its internal state. Requests on a zthr are serialized
166 * using the zthr_request_lock, while changes in its internal
167 * state are protected by the zthr_state_lock. A request will
168 * first acquire the zthr_request_lock and then immediately
169 * acquire the zthr_state_lock. We do this so that incoming
170 * requests are serialized using the request lock, while still
171 * allowing us to use the state lock for thread communication
172 * via zthr_cv.
173 */
174
175 #include <sys/zfs_context.h>
176 #include <sys/zthr.h>
177
178 struct zthr {
179 /* running thread doing the work */
180 kthread_t *zthr_thread;
181
182 /* lock protecting internal data & invariants */
183 kmutex_t zthr_state_lock;
184
185 /* mutex that serializes external requests */
186 kmutex_t zthr_request_lock;
187
188 /* notification mechanism for requests */
189 kcondvar_t zthr_cv;
190
191 /* flag set to true if we are canceling the zthr */
192 boolean_t zthr_cancel;
193
194 /*
195 * maximum amount of time that the zthr is spent sleeping;
196 * if this is 0, the thread doesn't wake up until it gets
197 * signaled.
198 */
199 hrtime_t zthr_wait_time;
200
201 /* consumer-provided callbacks & data */
202 zthr_checkfunc_t *zthr_checkfunc;
203 zthr_func_t *zthr_func;
204 void *zthr_arg;
205 };
206
207 static void
zthr_procedure(void * arg)208 zthr_procedure(void *arg)
209 {
210 zthr_t *t = arg;
211
212 mutex_enter(&t->zthr_state_lock);
213 ASSERT3P(t->zthr_thread, ==, curthread);
214
215 while (!t->zthr_cancel) {
216 if (t->zthr_checkfunc(t->zthr_arg, t)) {
217 mutex_exit(&t->zthr_state_lock);
218 t->zthr_func(t->zthr_arg, t);
219 mutex_enter(&t->zthr_state_lock);
220 } else {
221 /* go to sleep */
222 if (t->zthr_wait_time == 0) {
223 cv_wait(&t->zthr_cv, &t->zthr_state_lock);
224 } else {
225 (void) cv_timedwait_hires(&t->zthr_cv,
226 &t->zthr_state_lock, t->zthr_wait_time,
227 MSEC2NSEC(1), 0);
228 }
229 }
230 }
231
232 /*
233 * Clear out the kernel thread metadata and notify the
234 * zthr_cancel() thread that we've stopped running.
235 */
236 t->zthr_thread = NULL;
237 t->zthr_cancel = B_FALSE;
238 cv_broadcast(&t->zthr_cv);
239
240 mutex_exit(&t->zthr_state_lock);
241 thread_exit();
242 }
243
244 zthr_t *
zthr_create(zthr_checkfunc_t * checkfunc,zthr_func_t * func,void * arg)245 zthr_create(zthr_checkfunc_t *checkfunc, zthr_func_t *func, void *arg)
246 {
247 return (zthr_create_timer(checkfunc, func, arg, (hrtime_t)0));
248 }
249
250 /*
251 * Create a zthr with specified maximum sleep time. If the time
252 * in sleeping state exceeds max_sleep, a wakeup(do the check and
253 * start working if required) will be triggered.
254 */
255 zthr_t *
zthr_create_timer(zthr_checkfunc_t * checkfunc,zthr_func_t * func,void * arg,hrtime_t max_sleep)256 zthr_create_timer(zthr_checkfunc_t *checkfunc, zthr_func_t *func,
257 void *arg, hrtime_t max_sleep)
258 {
259 zthr_t *t = kmem_zalloc(sizeof (*t), KM_SLEEP);
260 mutex_init(&t->zthr_state_lock, NULL, MUTEX_DEFAULT, NULL);
261 mutex_init(&t->zthr_request_lock, NULL, MUTEX_DEFAULT, NULL);
262 cv_init(&t->zthr_cv, NULL, CV_DEFAULT, NULL);
263
264 mutex_enter(&t->zthr_state_lock);
265 t->zthr_checkfunc = checkfunc;
266 t->zthr_func = func;
267 t->zthr_arg = arg;
268 t->zthr_wait_time = max_sleep;
269
270 t->zthr_thread = thread_create(NULL, 0, zthr_procedure, t,
271 0, &p0, TS_RUN, minclsyspri);
272 mutex_exit(&t->zthr_state_lock);
273
274 return (t);
275 }
276
277 void
zthr_destroy(zthr_t * t)278 zthr_destroy(zthr_t *t)
279 {
280 ASSERT(!MUTEX_HELD(&t->zthr_state_lock));
281 ASSERT(!MUTEX_HELD(&t->zthr_request_lock));
282 VERIFY3P(t->zthr_thread, ==, NULL);
283 mutex_destroy(&t->zthr_request_lock);
284 mutex_destroy(&t->zthr_state_lock);
285 cv_destroy(&t->zthr_cv);
286 kmem_free(t, sizeof (*t));
287 }
288
289 /*
290 * Wake up the zthr if it is sleeping. If the thread has been
291 * cancelled that does nothing.
292 */
293 void
zthr_wakeup(zthr_t * t)294 zthr_wakeup(zthr_t *t)
295 {
296 mutex_enter(&t->zthr_request_lock);
297 mutex_enter(&t->zthr_state_lock);
298
299 /*
300 * There are 4 states that we can find the zthr when issuing
301 * this broadcast:
302 *
303 * [1] The common case of the thread being asleep, at which
304 * point the broadcast will wake it up.
305 * [2] The thread has been cancelled. Waking up a cancelled
306 * thread is a no-op. Any work that is still left to be
307 * done should be handled the next time the thread is
308 * resumed.
309 * [3] The thread is doing work and is already up, so this
310 * is basically a no-op.
311 * [4] The thread was just created/resumed, in which case the
312 * behavior is similar to [3].
313 */
314 cv_broadcast(&t->zthr_cv);
315
316 mutex_exit(&t->zthr_state_lock);
317 mutex_exit(&t->zthr_request_lock);
318 }
319
320 /*
321 * Sends a cancel request to the zthr and blocks until the zthr is
322 * cancelled. If the zthr is not running (e.g. has been cancelled
323 * already), this is a no-op.
324 */
325 void
zthr_cancel(zthr_t * t)326 zthr_cancel(zthr_t *t)
327 {
328 mutex_enter(&t->zthr_request_lock);
329 mutex_enter(&t->zthr_state_lock);
330
331 /*
332 * Since we are holding the zthr_state_lock at this point
333 * we can find the state in one of the following 4 states:
334 *
335 * [1] The thread has already been cancelled, therefore
336 * there is nothing for us to do.
337 * [2] The thread is sleeping, so we broadcast the CV first
338 * to wake it up and then we set the flag and we are
339 * waiting for it to exit.
340 * [3] The thread is doing work, in which case we just set
341 * the flag and wait for it to finish.
342 * [4] The thread was just created/resumed, in which case
343 * the behavior is similar to [3].
344 *
345 * Since requests are serialized, by the time that we get
346 * control back we expect that the zthr is cancelled and
347 * not running anymore.
348 */
349 if (t->zthr_thread != NULL) {
350 t->zthr_cancel = B_TRUE;
351
352 /* broadcast in case the zthr is sleeping */
353 cv_broadcast(&t->zthr_cv);
354
355 while (t->zthr_thread != NULL)
356 cv_wait(&t->zthr_cv, &t->zthr_state_lock);
357
358 ASSERT(!t->zthr_cancel);
359 }
360
361 mutex_exit(&t->zthr_state_lock);
362 mutex_exit(&t->zthr_request_lock);
363 }
364
365 /*
366 * Sends a resume request to the supplied zthr. If the zthr is
367 * already running this is a no-op.
368 */
369 void
zthr_resume(zthr_t * t)370 zthr_resume(zthr_t *t)
371 {
372 mutex_enter(&t->zthr_request_lock);
373 mutex_enter(&t->zthr_state_lock);
374
375 ASSERT3P(&t->zthr_checkfunc, !=, NULL);
376 ASSERT3P(&t->zthr_func, !=, NULL);
377 ASSERT(!t->zthr_cancel);
378
379 /*
380 * There are 4 states that we find the zthr in at this point
381 * given the locks that we hold:
382 *
383 * [1] The zthr was cancelled, so we spawn a new thread for
384 * the zthr (common case).
385 * [2] The zthr is running at which point this is a no-op.
386 * [3] The zthr is sleeping at which point this is a no-op.
387 * [4] The zthr was just spawned at which point this is a
388 * no-op.
389 */
390 if (t->zthr_thread == NULL) {
391 t->zthr_thread = thread_create(NULL, 0, zthr_procedure, t,
392 0, &p0, TS_RUN, minclsyspri);
393 }
394
395 mutex_exit(&t->zthr_state_lock);
396 mutex_exit(&t->zthr_request_lock);
397 }
398
399 /*
400 * This function is intended to be used by the zthr itself
401 * (specifically the zthr_func callback provided) to check
402 * if another thread has signaled it to stop running before
403 * doing some expensive operation.
404 *
405 * returns TRUE if we are in the middle of trying to cancel
406 * this thread.
407 *
408 * returns FALSE otherwise.
409 */
410 boolean_t
zthr_iscancelled(zthr_t * t)411 zthr_iscancelled(zthr_t *t)
412 {
413 ASSERT3P(t->zthr_thread, ==, curthread);
414
415 /*
416 * The majority of the functions here grab zthr_request_lock
417 * first and then zthr_state_lock. This function only grabs
418 * the zthr_state_lock. That is because this function should
419 * only be called from the zthr_func to check if someone has
420 * issued a zthr_cancel() on the thread. If there is a zthr_cancel()
421 * happening concurrently, attempting to grab the request lock
422 * here would result in a deadlock.
423 *
424 * By grabbing only the zthr_state_lock this function is allowed
425 * to run concurrently with a zthr_cancel() request.
426 */
427 mutex_enter(&t->zthr_state_lock);
428 boolean_t cancelled = t->zthr_cancel;
429 mutex_exit(&t->zthr_state_lock);
430 return (cancelled);
431 }
432