1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2017, 2020 by Delphix. All rights reserved.
15 */
16
17 /*
18 * ZTHR Infrastructure
19 * ===================
20 *
21 * ZTHR threads are used for isolated operations that span multiple txgs
22 * within a SPA. They generally exist from SPA creation/loading and until
23 * the SPA is exported/destroyed. The ideal requirements for an operation
24 * to be modeled with a zthr are the following:
25 *
26 * 1] The operation needs to run over multiple txgs.
27 * 2] There is be a single point of reference in memory or on disk that
28 * indicates whether the operation should run/is running or has
29 * stopped.
30 *
31 * If the operation satisfies the above then the following rules guarantee
32 * a certain level of correctness:
33 *
34 * 1] Any thread EXCEPT the zthr changes the work indicator from stopped
35 * to running but not the opposite.
36 * 2] Only the zthr can change the work indicator from running to stopped
37 * (e.g. when it is done) but not the opposite.
38 *
39 * This way a normal zthr cycle should go like this:
40 *
41 * 1] An external thread changes the work indicator from stopped to
42 * running and wakes up the zthr.
43 * 2] The zthr wakes up, checks the indicator and starts working.
44 * 3] When the zthr is done, it changes the indicator to stopped, allowing
45 * a new cycle to start.
46 *
47 * Besides being awakened by other threads, a zthr can be configured
48 * during creation to wakeup on its own after a specified interval
49 * [see zthr_create_timer()].
50 *
51 * Note: ZTHR threads are NOT a replacement for generic threads! Please
52 * ensure that they fit your use-case well before using them.
53 *
54 * == ZTHR creation
55 *
56 * Every zthr needs four inputs to start running:
57 *
58 * 1] A user-defined checker function (checkfunc) that decides whether
59 * the zthr should start working or go to sleep. The function should
60 * return TRUE when the zthr needs to work or FALSE to let it sleep,
61 * and should adhere to the following signature:
62 * boolean_t checkfunc_name(void *args, zthr_t *t);
63 *
64 * 2] A user-defined ZTHR function (func) which the zthr executes when
65 * it is not sleeping. The function should adhere to the following
66 * signature type:
67 * void func_name(void *args, zthr_t *t);
68 *
69 * 3] A void args pointer that will be passed to checkfunc and func
70 * implicitly by the infrastructure.
71 *
72 * 4] A name for the thread. This string must be valid for the lifetime
73 * of the zthr.
74 *
75 * The reason why the above API needs two different functions,
76 * instead of one that both checks and does the work, has to do with
77 * the zthr's internal state lock (zthr_state_lock) and the allowed
78 * cancellation windows. We want to hold the zthr_state_lock while
79 * running checkfunc but not while running func. This way the zthr
80 * can be cancelled while doing work and not while checking for work.
81 *
82 * To start a zthr:
83 * zthr_t *zthr_pointer = zthr_create(checkfunc, func, args,
84 * pri);
85 * or
86 * zthr_t *zthr_pointer = zthr_create_timer(checkfunc, func,
87 * args, max_sleep, pri);
88 *
89 * After that you should be able to wakeup, cancel, and resume the
90 * zthr from another thread using the zthr_pointer.
91 *
92 * NOTE: ZTHR threads could potentially wake up spuriously and the
93 * user should take this into account when writing a checkfunc.
94 * [see ZTHR state transitions]
95 *
96 * == ZTHR wakeup
97 *
98 * ZTHR wakeup should be used when new work is added for the zthr. The
99 * sleeping zthr will wakeup, see that it has more work to complete
100 * and proceed. This can be invoked from open or syncing context.
101 *
102 * To wakeup a zthr:
103 * zthr_wakeup(zthr_t *t)
104 *
105 * == ZTHR cancellation and resumption
106 *
107 * ZTHR threads must be cancelled when their SPA is being exported
108 * or when they need to be paused so they don't interfere with other
109 * operations.
110 *
111 * To cancel a zthr:
112 * zthr_cancel(zthr_pointer);
113 *
114 * To resume it:
115 * zthr_resume(zthr_pointer);
116 *
117 * ZTHR cancel and resume should be invoked in open context during the
118 * lifecycle of the pool as it is imported, exported or destroyed.
119 *
120 * A zthr will implicitly check if it has received a cancellation
121 * signal every time func returns and every time it wakes up [see
122 * ZTHR state transitions below].
123 *
124 * At times, waiting for the zthr's func to finish its job may take
125 * time. This may be very time-consuming for some operations that
126 * need to cancel the SPA's zthrs (e.g spa_export). For this scenario
127 * the user can explicitly make their ZTHR function aware of incoming
128 * cancellation signals using zthr_iscancelled(). A common pattern for
129 * that looks like this:
130 *
131 * int
132 * func_name(void *args, zthr_t *t)
133 * {
134 * ... <unpack args> ...
135 * while (!work_done && !zthr_iscancelled(t)) {
136 * ... <do more work> ...
137 * }
138 * }
139 *
140 * == ZTHR cleanup
141 *
142 * Cancelling a zthr doesn't clean up its metadata (internal locks,
143 * function pointers to func and checkfunc, etc..). This is because
144 * we want to keep them around in case we want to resume the execution
145 * of the zthr later. Similarly for zthrs that exit themselves.
146 *
147 * To completely cleanup a zthr, cancel it first to ensure that it
148 * is not running and then use zthr_destroy().
149 *
150 * == ZTHR state transitions
151 *
152 * zthr creation
153 * +
154 * |
155 * | woke up
156 * | +--------------+ sleep
157 * | | ^
158 * | | |
159 * | | | FALSE
160 * | | |
161 * v v FALSE +
162 * cancelled? +---------> checkfunc?
163 * + ^ +
164 * | | |
165 * | | | TRUE
166 * | | |
167 * | | func returned v
168 * | +---------------+ func
169 * |
170 * | TRUE
171 * |
172 * v
173 * zthr stopped running
174 *
175 * == Implementation of ZTHR requests
176 *
177 * ZTHR cancel and resume are requests on a zthr to change its
178 * internal state. These requests are serialized using the
179 * zthr_request_lock, while changes in its internal state are
180 * protected by the zthr_state_lock. A request will first acquire
181 * the zthr_request_lock and then immediately acquire the
182 * zthr_state_lock. We do this so that incoming requests are
183 * serialized using the request lock, while still allowing us
184 * to use the state lock for thread communication via zthr_cv.
185 *
186 * ZTHR wakeup broadcasts to zthr_cv, causing sleeping threads
187 * to wakeup. It acquires the zthr_state_lock but not the
188 * zthr_request_lock, so that a wakeup on a zthr in the middle
189 * of being cancelled will not block.
190 */
191
192 #include <sys/zfs_context.h>
193 #include <sys/zthr.h>
194
195 struct zthr {
196 /* running thread doing the work */
197 kthread_t *zthr_thread;
198
199 /* lock protecting internal data & invariants */
200 kmutex_t zthr_state_lock;
201
202 /* mutex that serializes external requests */
203 kmutex_t zthr_request_lock;
204
205 /* notification mechanism for requests */
206 kcondvar_t zthr_cv;
207
208 /* flag set to true if we are canceling the zthr */
209 boolean_t zthr_cancel;
210
211 /* flag set to true if we are waiting for the zthr to finish */
212 boolean_t zthr_haswaiters;
213 kcondvar_t zthr_wait_cv;
214 /*
215 * maximum amount of time that the zthr is spent sleeping;
216 * if this is 0, the thread doesn't wake up until it gets
217 * signaled.
218 */
219 hrtime_t zthr_sleep_timeout;
220
221 /* Thread priority */
222 pri_t zthr_pri;
223
224 /* consumer-provided callbacks & data */
225 zthr_checkfunc_t *zthr_checkfunc;
226 zthr_func_t *zthr_func;
227 void *zthr_arg;
228 const char *zthr_name;
229 };
230
231 static __attribute__((noreturn)) void
zthr_procedure(void * arg)232 zthr_procedure(void *arg)
233 {
234 zthr_t *t = arg;
235
236 mutex_enter(&t->zthr_state_lock);
237 ASSERT3P(t->zthr_thread, ==, curthread);
238
239 while (!t->zthr_cancel) {
240 if (t->zthr_checkfunc(t->zthr_arg, t)) {
241 mutex_exit(&t->zthr_state_lock);
242 t->zthr_func(t->zthr_arg, t);
243 mutex_enter(&t->zthr_state_lock);
244 } else {
245 if (t->zthr_sleep_timeout == 0) {
246 cv_wait_idle(&t->zthr_cv, &t->zthr_state_lock);
247 } else {
248 (void) cv_timedwait_idle_hires(&t->zthr_cv,
249 &t->zthr_state_lock, t->zthr_sleep_timeout,
250 MSEC2NSEC(1), 0);
251 }
252 }
253 if (t->zthr_haswaiters) {
254 t->zthr_haswaiters = B_FALSE;
255 cv_broadcast(&t->zthr_wait_cv);
256 }
257 }
258
259 /*
260 * Clear out the kernel thread metadata and notify the
261 * zthr_cancel() thread that we've stopped running.
262 */
263 t->zthr_thread = NULL;
264 t->zthr_cancel = B_FALSE;
265 cv_broadcast(&t->zthr_cv);
266
267 mutex_exit(&t->zthr_state_lock);
268 thread_exit();
269 }
270
271 zthr_t *
zthr_create(const char * zthr_name,zthr_checkfunc_t * checkfunc,zthr_func_t * func,void * arg,pri_t pri)272 zthr_create(const char *zthr_name, zthr_checkfunc_t *checkfunc,
273 zthr_func_t *func, void *arg, pri_t pri)
274 {
275 return (zthr_create_timer(zthr_name, checkfunc,
276 func, arg, (hrtime_t)0, pri));
277 }
278
279 /*
280 * Create a zthr with specified maximum sleep time. If the time
281 * in sleeping state exceeds max_sleep, a wakeup(do the check and
282 * start working if required) will be triggered.
283 */
284 zthr_t *
zthr_create_timer(const char * zthr_name,zthr_checkfunc_t * checkfunc,zthr_func_t * func,void * arg,hrtime_t max_sleep,pri_t pri)285 zthr_create_timer(const char *zthr_name, zthr_checkfunc_t *checkfunc,
286 zthr_func_t *func, void *arg, hrtime_t max_sleep, pri_t pri)
287 {
288 zthr_t *t = kmem_zalloc(sizeof (*t), KM_SLEEP);
289 mutex_init(&t->zthr_state_lock, NULL, MUTEX_DEFAULT, NULL);
290 mutex_init(&t->zthr_request_lock, NULL, MUTEX_DEFAULT, NULL);
291 cv_init(&t->zthr_cv, NULL, CV_DEFAULT, NULL);
292 cv_init(&t->zthr_wait_cv, NULL, CV_DEFAULT, NULL);
293
294 mutex_enter(&t->zthr_state_lock);
295 t->zthr_checkfunc = checkfunc;
296 t->zthr_func = func;
297 t->zthr_arg = arg;
298 t->zthr_sleep_timeout = max_sleep;
299 t->zthr_name = zthr_name;
300 t->zthr_pri = pri;
301
302 t->zthr_thread = thread_create_named(zthr_name, NULL, 0,
303 zthr_procedure, t, 0, &p0, TS_RUN, pri);
304
305 mutex_exit(&t->zthr_state_lock);
306
307 return (t);
308 }
309
310 void
zthr_destroy(zthr_t * t)311 zthr_destroy(zthr_t *t)
312 {
313 ASSERT(!MUTEX_HELD(&t->zthr_state_lock));
314 ASSERT(!MUTEX_HELD(&t->zthr_request_lock));
315 VERIFY0P(t->zthr_thread);
316 mutex_destroy(&t->zthr_request_lock);
317 mutex_destroy(&t->zthr_state_lock);
318 cv_destroy(&t->zthr_cv);
319 cv_destroy(&t->zthr_wait_cv);
320 kmem_free(t, sizeof (*t));
321 }
322
323 /*
324 * Wake up the zthr if it is sleeping. If the thread has been cancelled
325 * or is in the process of being cancelled, this is a no-op.
326 */
327 void
zthr_wakeup(zthr_t * t)328 zthr_wakeup(zthr_t *t)
329 {
330 mutex_enter(&t->zthr_state_lock);
331
332 /*
333 * There are 5 states that we can find the zthr when issuing
334 * this broadcast:
335 *
336 * [1] The common case of the thread being asleep, at which
337 * point the broadcast will wake it up.
338 * [2] The thread has been cancelled. Waking up a cancelled
339 * thread is a no-op. Any work that is still left to be
340 * done should be handled the next time the thread is
341 * resumed.
342 * [3] The thread is doing work and is already up, so this
343 * is basically a no-op.
344 * [4] The thread was just created/resumed, in which case the
345 * behavior is similar to [3].
346 * [5] The thread is in the middle of being cancelled, which
347 * will be a no-op.
348 */
349 cv_broadcast(&t->zthr_cv);
350
351 mutex_exit(&t->zthr_state_lock);
352 }
353
354 /*
355 * Sends a cancel request to the zthr and blocks until the zthr is
356 * cancelled. If the zthr is not running (e.g. has been cancelled
357 * already), this is a no-op. Note that this function should not be
358 * called from syncing context as it could deadlock with the zthr_func.
359 */
360 void
zthr_cancel(zthr_t * t)361 zthr_cancel(zthr_t *t)
362 {
363 mutex_enter(&t->zthr_request_lock);
364 mutex_enter(&t->zthr_state_lock);
365
366 /*
367 * Since we are holding the zthr_state_lock at this point
368 * we can find the state in one of the following 4 states:
369 *
370 * [1] The thread has already been cancelled, therefore
371 * there is nothing for us to do.
372 * [2] The thread is sleeping so we set the flag, broadcast
373 * the CV and wait for it to exit.
374 * [3] The thread is doing work, in which case we just set
375 * the flag and wait for it to finish.
376 * [4] The thread was just created/resumed, in which case
377 * the behavior is similar to [3].
378 *
379 * Since requests are serialized, by the time that we get
380 * control back we expect that the zthr is cancelled and
381 * not running anymore.
382 */
383 if (t->zthr_thread != NULL) {
384 t->zthr_cancel = B_TRUE;
385
386 /* broadcast in case the zthr is sleeping */
387 cv_broadcast(&t->zthr_cv);
388
389 while (t->zthr_thread != NULL)
390 cv_wait(&t->zthr_cv, &t->zthr_state_lock);
391
392 ASSERT(!t->zthr_cancel);
393 }
394
395 mutex_exit(&t->zthr_state_lock);
396 mutex_exit(&t->zthr_request_lock);
397 }
398
399 /*
400 * Sends a resume request to the supplied zthr. If the zthr is already
401 * running this is a no-op. Note that this function should not be
402 * called from syncing context as it could deadlock with the zthr_func.
403 */
404 void
zthr_resume(zthr_t * t)405 zthr_resume(zthr_t *t)
406 {
407 mutex_enter(&t->zthr_request_lock);
408 mutex_enter(&t->zthr_state_lock);
409
410 ASSERT3P(&t->zthr_checkfunc, !=, NULL);
411 ASSERT3P(&t->zthr_func, !=, NULL);
412 ASSERT(!t->zthr_cancel);
413 ASSERT(!t->zthr_haswaiters);
414
415 /*
416 * There are 4 states that we find the zthr in at this point
417 * given the locks that we hold:
418 *
419 * [1] The zthr was cancelled, so we spawn a new thread for
420 * the zthr (common case).
421 * [2] The zthr is running at which point this is a no-op.
422 * [3] The zthr is sleeping at which point this is a no-op.
423 * [4] The zthr was just spawned at which point this is a
424 * no-op.
425 */
426 if (t->zthr_thread == NULL) {
427 t->zthr_thread = thread_create_named(t->zthr_name, NULL, 0,
428 zthr_procedure, t, 0, &p0, TS_RUN, t->zthr_pri);
429 }
430
431 mutex_exit(&t->zthr_state_lock);
432 mutex_exit(&t->zthr_request_lock);
433 }
434
435 /*
436 * This function is intended to be used by the zthr itself
437 * (specifically the zthr_func callback provided) to check
438 * if another thread has signaled it to stop running before
439 * doing some expensive operation.
440 *
441 * returns TRUE if we are in the middle of trying to cancel
442 * this thread.
443 *
444 * returns FALSE otherwise.
445 */
446 boolean_t
zthr_iscancelled(zthr_t * t)447 zthr_iscancelled(zthr_t *t)
448 {
449 ASSERT3P(t->zthr_thread, ==, curthread);
450
451 /*
452 * The majority of the functions here grab zthr_request_lock
453 * first and then zthr_state_lock. This function only grabs
454 * the zthr_state_lock. That is because this function should
455 * only be called from the zthr_func to check if someone has
456 * issued a zthr_cancel() on the thread. If there is a zthr_cancel()
457 * happening concurrently, attempting to grab the request lock
458 * here would result in a deadlock.
459 *
460 * By grabbing only the zthr_state_lock this function is allowed
461 * to run concurrently with a zthr_cancel() request.
462 */
463 mutex_enter(&t->zthr_state_lock);
464 boolean_t cancelled = t->zthr_cancel;
465 mutex_exit(&t->zthr_state_lock);
466 return (cancelled);
467 }
468
469 boolean_t
zthr_iscurthread(zthr_t * t)470 zthr_iscurthread(zthr_t *t)
471 {
472 return (t->zthr_thread == curthread);
473 }
474
475 /*
476 * Wait for the zthr to finish its current function. Similar to
477 * zthr_iscancelled, you can use zthr_has_waiters to have the zthr_func end
478 * early. Unlike zthr_cancel, the thread is not destroyed. If the zthr was
479 * sleeping or cancelled, return immediately.
480 */
481 void
zthr_wait_cycle_done(zthr_t * t)482 zthr_wait_cycle_done(zthr_t *t)
483 {
484 mutex_enter(&t->zthr_state_lock);
485
486 /*
487 * Since we are holding the zthr_state_lock at this point
488 * we can find the state in one of the following 5 states:
489 *
490 * [1] The thread has already cancelled, therefore
491 * there is nothing for us to do.
492 * [2] The thread is sleeping so we set the flag, broadcast
493 * the CV and wait for it to exit.
494 * [3] The thread is doing work, in which case we just set
495 * the flag and wait for it to finish.
496 * [4] The thread was just created/resumed, in which case
497 * the behavior is similar to [3].
498 * [5] The thread is the middle of being cancelled, which is
499 * similar to [3]. We'll wait for the cancel, which is
500 * waiting for the zthr func.
501 *
502 * Since requests are serialized, by the time that we get
503 * control back we expect that the zthr has completed it's
504 * zthr_func.
505 */
506 if (t->zthr_thread != NULL) {
507 t->zthr_haswaiters = B_TRUE;
508
509 /* broadcast in case the zthr is sleeping */
510 cv_broadcast(&t->zthr_cv);
511
512 while ((t->zthr_haswaiters) && (t->zthr_thread != NULL))
513 cv_wait(&t->zthr_wait_cv, &t->zthr_state_lock);
514
515 ASSERT(!t->zthr_haswaiters);
516 }
517
518 mutex_exit(&t->zthr_state_lock);
519 }
520
521 /*
522 * This function is intended to be used by the zthr itself
523 * to check if another thread is waiting on it to finish
524 *
525 * returns TRUE if we have been asked to finish.
526 *
527 * returns FALSE otherwise.
528 */
529 boolean_t
zthr_has_waiters(zthr_t * t)530 zthr_has_waiters(zthr_t *t)
531 {
532 ASSERT3P(t->zthr_thread, ==, curthread);
533
534 mutex_enter(&t->zthr_state_lock);
535
536 /*
537 * Similarly to zthr_iscancelled(), we only grab the
538 * zthr_state_lock so that the zthr itself can use this
539 * to check for the request.
540 */
541 boolean_t has_waiters = t->zthr_haswaiters;
542 mutex_exit(&t->zthr_state_lock);
543 return (has_waiters);
544 }
545