1 /*
2 * CDDL HEADER START
3 *
4 * This file and its contents are supplied under the terms of the
5 * Common Development and Distribution License ("CDDL"), version 1.0.
6 * You may only use this file in accordance with the terms of version
7 * 1.0 of the CDDL.
8 *
9 * A full copy of the text of the CDDL should have accompanied this
10 * source. A copy of the CDDL is also available via the Internet at
11 * http://www.illumos.org/license/CDDL.
12 *
13 * CDDL HEADER END
14 */
15
16 /*
17 * Copyright (c) 2017, 2020 by Delphix. All rights reserved.
18 */
19
20 /*
21 * ZTHR Infrastructure
22 * ===================
23 *
24 * ZTHR threads are used for isolated operations that span multiple txgs
25 * within a SPA. They generally exist from SPA creation/loading and until
26 * the SPA is exported/destroyed. The ideal requirements for an operation
27 * to be modeled with a zthr are the following:
28 *
29 * 1] The operation needs to run over multiple txgs.
30 * 2] There is be a single point of reference in memory or on disk that
31 * indicates whether the operation should run/is running or has
32 * stopped.
33 *
34 * If the operation satisfies the above then the following rules guarantee
35 * a certain level of correctness:
36 *
37 * 1] Any thread EXCEPT the zthr changes the work indicator from stopped
38 * to running but not the opposite.
39 * 2] Only the zthr can change the work indicator from running to stopped
40 * (e.g. when it is done) but not the opposite.
41 *
42 * This way a normal zthr cycle should go like this:
43 *
44 * 1] An external thread changes the work indicator from stopped to
45 * running and wakes up the zthr.
46 * 2] The zthr wakes up, checks the indicator and starts working.
47 * 3] When the zthr is done, it changes the indicator to stopped, allowing
48 * a new cycle to start.
49 *
50 * Besides being awakened by other threads, a zthr can be configured
51 * during creation to wakeup on its own after a specified interval
52 * [see zthr_create_timer()].
53 *
54 * Note: ZTHR threads are NOT a replacement for generic threads! Please
55 * ensure that they fit your use-case well before using them.
56 *
57 * == ZTHR creation
58 *
59 * Every zthr needs four inputs to start running:
60 *
61 * 1] A user-defined checker function (checkfunc) that decides whether
62 * the zthr should start working or go to sleep. The function should
63 * return TRUE when the zthr needs to work or FALSE to let it sleep,
64 * and should adhere to the following signature:
65 * boolean_t checkfunc_name(void *args, zthr_t *t);
66 *
67 * 2] A user-defined ZTHR function (func) which the zthr executes when
68 * it is not sleeping. The function should adhere to the following
69 * signature type:
70 * void func_name(void *args, zthr_t *t);
71 *
72 * 3] A void args pointer that will be passed to checkfunc and func
73 * implicitly by the infrastructure.
74 *
75 * 4] A name for the thread. This string must be valid for the lifetime
76 * of the zthr.
77 *
78 * The reason why the above API needs two different functions,
79 * instead of one that both checks and does the work, has to do with
80 * the zthr's internal state lock (zthr_state_lock) and the allowed
81 * cancellation windows. We want to hold the zthr_state_lock while
82 * running checkfunc but not while running func. This way the zthr
83 * can be cancelled while doing work and not while checking for work.
84 *
85 * To start a zthr:
86 * zthr_t *zthr_pointer = zthr_create(checkfunc, func, args,
87 * pri);
88 * or
89 * zthr_t *zthr_pointer = zthr_create_timer(checkfunc, func,
90 * args, max_sleep, pri);
91 *
92 * After that you should be able to wakeup, cancel, and resume the
93 * zthr from another thread using the zthr_pointer.
94 *
95 * NOTE: ZTHR threads could potentially wake up spuriously and the
96 * user should take this into account when writing a checkfunc.
97 * [see ZTHR state transitions]
98 *
99 * == ZTHR wakeup
100 *
101 * ZTHR wakeup should be used when new work is added for the zthr. The
102 * sleeping zthr will wakeup, see that it has more work to complete
103 * and proceed. This can be invoked from open or syncing context.
104 *
105 * To wakeup a zthr:
106 * zthr_wakeup(zthr_t *t)
107 *
108 * == ZTHR cancellation and resumption
109 *
110 * ZTHR threads must be cancelled when their SPA is being exported
111 * or when they need to be paused so they don't interfere with other
112 * operations.
113 *
114 * To cancel a zthr:
115 * zthr_cancel(zthr_pointer);
116 *
117 * To resume it:
118 * zthr_resume(zthr_pointer);
119 *
120 * ZTHR cancel and resume should be invoked in open context during the
121 * lifecycle of the pool as it is imported, exported or destroyed.
122 *
123 * A zthr will implicitly check if it has received a cancellation
124 * signal every time func returns and every time it wakes up [see
125 * ZTHR state transitions below].
126 *
127 * At times, waiting for the zthr's func to finish its job may take
128 * time. This may be very time-consuming for some operations that
129 * need to cancel the SPA's zthrs (e.g spa_export). For this scenario
130 * the user can explicitly make their ZTHR function aware of incoming
131 * cancellation signals using zthr_iscancelled(). A common pattern for
132 * that looks like this:
133 *
134 * int
135 * func_name(void *args, zthr_t *t)
136 * {
137 * ... <unpack args> ...
138 * while (!work_done && !zthr_iscancelled(t)) {
139 * ... <do more work> ...
140 * }
141 * }
142 *
143 * == ZTHR cleanup
144 *
145 * Cancelling a zthr doesn't clean up its metadata (internal locks,
146 * function pointers to func and checkfunc, etc..). This is because
147 * we want to keep them around in case we want to resume the execution
148 * of the zthr later. Similarly for zthrs that exit themselves.
149 *
150 * To completely cleanup a zthr, cancel it first to ensure that it
151 * is not running and then use zthr_destroy().
152 *
153 * == ZTHR state transitions
154 *
155 * zthr creation
156 * +
157 * |
158 * | woke up
159 * | +--------------+ sleep
160 * | | ^
161 * | | |
162 * | | | FALSE
163 * | | |
164 * v v FALSE +
165 * cancelled? +---------> checkfunc?
166 * + ^ +
167 * | | |
168 * | | | TRUE
169 * | | |
170 * | | func returned v
171 * | +---------------+ func
172 * |
173 * | TRUE
174 * |
175 * v
176 * zthr stopped running
177 *
178 * == Implementation of ZTHR requests
179 *
180 * ZTHR cancel and resume are requests on a zthr to change its
181 * internal state. These requests are serialized using the
182 * zthr_request_lock, while changes in its internal state are
183 * protected by the zthr_state_lock. A request will first acquire
184 * the zthr_request_lock and then immediately acquire the
185 * zthr_state_lock. We do this so that incoming requests are
186 * serialized using the request lock, while still allowing us
187 * to use the state lock for thread communication via zthr_cv.
188 *
189 * ZTHR wakeup broadcasts to zthr_cv, causing sleeping threads
190 * to wakeup. It acquires the zthr_state_lock but not the
191 * zthr_request_lock, so that a wakeup on a zthr in the middle
192 * of being cancelled will not block.
193 */
194
195 #include <sys/zfs_context.h>
196 #include <sys/zthr.h>
197
198 struct zthr {
199 /* running thread doing the work */
200 kthread_t *zthr_thread;
201
202 /* lock protecting internal data & invariants */
203 kmutex_t zthr_state_lock;
204
205 /* mutex that serializes external requests */
206 kmutex_t zthr_request_lock;
207
208 /* notification mechanism for requests */
209 kcondvar_t zthr_cv;
210
211 /* flag set to true if we are canceling the zthr */
212 boolean_t zthr_cancel;
213
214 /* flag set to true if we are waiting for the zthr to finish */
215 boolean_t zthr_haswaiters;
216 kcondvar_t zthr_wait_cv;
217 /*
218 * maximum amount of time that the zthr is spent sleeping;
219 * if this is 0, the thread doesn't wake up until it gets
220 * signaled.
221 */
222 hrtime_t zthr_sleep_timeout;
223
224 /* Thread priority */
225 pri_t zthr_pri;
226
227 /* consumer-provided callbacks & data */
228 zthr_checkfunc_t *zthr_checkfunc;
229 zthr_func_t *zthr_func;
230 void *zthr_arg;
231 const char *zthr_name;
232 };
233
234 static __attribute__((noreturn)) void
zthr_procedure(void * arg)235 zthr_procedure(void *arg)
236 {
237 zthr_t *t = arg;
238
239 mutex_enter(&t->zthr_state_lock);
240 ASSERT3P(t->zthr_thread, ==, curthread);
241
242 while (!t->zthr_cancel) {
243 if (t->zthr_checkfunc(t->zthr_arg, t)) {
244 mutex_exit(&t->zthr_state_lock);
245 t->zthr_func(t->zthr_arg, t);
246 mutex_enter(&t->zthr_state_lock);
247 } else {
248 if (t->zthr_sleep_timeout == 0) {
249 cv_wait_idle(&t->zthr_cv, &t->zthr_state_lock);
250 } else {
251 (void) cv_timedwait_idle_hires(&t->zthr_cv,
252 &t->zthr_state_lock, t->zthr_sleep_timeout,
253 MSEC2NSEC(1), 0);
254 }
255 }
256 if (t->zthr_haswaiters) {
257 t->zthr_haswaiters = B_FALSE;
258 cv_broadcast(&t->zthr_wait_cv);
259 }
260 }
261
262 /*
263 * Clear out the kernel thread metadata and notify the
264 * zthr_cancel() thread that we've stopped running.
265 */
266 t->zthr_thread = NULL;
267 t->zthr_cancel = B_FALSE;
268 cv_broadcast(&t->zthr_cv);
269
270 mutex_exit(&t->zthr_state_lock);
271 thread_exit();
272 }
273
274 zthr_t *
zthr_create(const char * zthr_name,zthr_checkfunc_t * checkfunc,zthr_func_t * func,void * arg,pri_t pri)275 zthr_create(const char *zthr_name, zthr_checkfunc_t *checkfunc,
276 zthr_func_t *func, void *arg, pri_t pri)
277 {
278 return (zthr_create_timer(zthr_name, checkfunc,
279 func, arg, (hrtime_t)0, pri));
280 }
281
282 /*
283 * Create a zthr with specified maximum sleep time. If the time
284 * in sleeping state exceeds max_sleep, a wakeup(do the check and
285 * start working if required) will be triggered.
286 */
287 zthr_t *
zthr_create_timer(const char * zthr_name,zthr_checkfunc_t * checkfunc,zthr_func_t * func,void * arg,hrtime_t max_sleep,pri_t pri)288 zthr_create_timer(const char *zthr_name, zthr_checkfunc_t *checkfunc,
289 zthr_func_t *func, void *arg, hrtime_t max_sleep, pri_t pri)
290 {
291 zthr_t *t = kmem_zalloc(sizeof (*t), KM_SLEEP);
292 mutex_init(&t->zthr_state_lock, NULL, MUTEX_DEFAULT, NULL);
293 mutex_init(&t->zthr_request_lock, NULL, MUTEX_DEFAULT, NULL);
294 cv_init(&t->zthr_cv, NULL, CV_DEFAULT, NULL);
295 cv_init(&t->zthr_wait_cv, NULL, CV_DEFAULT, NULL);
296
297 mutex_enter(&t->zthr_state_lock);
298 t->zthr_checkfunc = checkfunc;
299 t->zthr_func = func;
300 t->zthr_arg = arg;
301 t->zthr_sleep_timeout = max_sleep;
302 t->zthr_name = zthr_name;
303 t->zthr_pri = pri;
304
305 t->zthr_thread = thread_create_named(zthr_name, NULL, 0,
306 zthr_procedure, t, 0, &p0, TS_RUN, pri);
307
308 mutex_exit(&t->zthr_state_lock);
309
310 return (t);
311 }
312
313 void
zthr_destroy(zthr_t * t)314 zthr_destroy(zthr_t *t)
315 {
316 ASSERT(!MUTEX_HELD(&t->zthr_state_lock));
317 ASSERT(!MUTEX_HELD(&t->zthr_request_lock));
318 VERIFY3P(t->zthr_thread, ==, NULL);
319 mutex_destroy(&t->zthr_request_lock);
320 mutex_destroy(&t->zthr_state_lock);
321 cv_destroy(&t->zthr_cv);
322 cv_destroy(&t->zthr_wait_cv);
323 kmem_free(t, sizeof (*t));
324 }
325
326 /*
327 * Wake up the zthr if it is sleeping. If the thread has been cancelled
328 * or is in the process of being cancelled, this is a no-op.
329 */
330 void
zthr_wakeup(zthr_t * t)331 zthr_wakeup(zthr_t *t)
332 {
333 mutex_enter(&t->zthr_state_lock);
334
335 /*
336 * There are 5 states that we can find the zthr when issuing
337 * this broadcast:
338 *
339 * [1] The common case of the thread being asleep, at which
340 * point the broadcast will wake it up.
341 * [2] The thread has been cancelled. Waking up a cancelled
342 * thread is a no-op. Any work that is still left to be
343 * done should be handled the next time the thread is
344 * resumed.
345 * [3] The thread is doing work and is already up, so this
346 * is basically a no-op.
347 * [4] The thread was just created/resumed, in which case the
348 * behavior is similar to [3].
349 * [5] The thread is in the middle of being cancelled, which
350 * will be a no-op.
351 */
352 cv_broadcast(&t->zthr_cv);
353
354 mutex_exit(&t->zthr_state_lock);
355 }
356
357 /*
358 * Sends a cancel request to the zthr and blocks until the zthr is
359 * cancelled. If the zthr is not running (e.g. has been cancelled
360 * already), this is a no-op. Note that this function should not be
361 * called from syncing context as it could deadlock with the zthr_func.
362 */
363 void
zthr_cancel(zthr_t * t)364 zthr_cancel(zthr_t *t)
365 {
366 mutex_enter(&t->zthr_request_lock);
367 mutex_enter(&t->zthr_state_lock);
368
369 /*
370 * Since we are holding the zthr_state_lock at this point
371 * we can find the state in one of the following 4 states:
372 *
373 * [1] The thread has already been cancelled, therefore
374 * there is nothing for us to do.
375 * [2] The thread is sleeping so we set the flag, broadcast
376 * the CV and wait for it to exit.
377 * [3] The thread is doing work, in which case we just set
378 * the flag and wait for it to finish.
379 * [4] The thread was just created/resumed, in which case
380 * the behavior is similar to [3].
381 *
382 * Since requests are serialized, by the time that we get
383 * control back we expect that the zthr is cancelled and
384 * not running anymore.
385 */
386 if (t->zthr_thread != NULL) {
387 t->zthr_cancel = B_TRUE;
388
389 /* broadcast in case the zthr is sleeping */
390 cv_broadcast(&t->zthr_cv);
391
392 while (t->zthr_thread != NULL)
393 cv_wait(&t->zthr_cv, &t->zthr_state_lock);
394
395 ASSERT(!t->zthr_cancel);
396 }
397
398 mutex_exit(&t->zthr_state_lock);
399 mutex_exit(&t->zthr_request_lock);
400 }
401
402 /*
403 * Sends a resume request to the supplied zthr. If the zthr is already
404 * running this is a no-op. Note that this function should not be
405 * called from syncing context as it could deadlock with the zthr_func.
406 */
407 void
zthr_resume(zthr_t * t)408 zthr_resume(zthr_t *t)
409 {
410 mutex_enter(&t->zthr_request_lock);
411 mutex_enter(&t->zthr_state_lock);
412
413 ASSERT3P(&t->zthr_checkfunc, !=, NULL);
414 ASSERT3P(&t->zthr_func, !=, NULL);
415 ASSERT(!t->zthr_cancel);
416 ASSERT(!t->zthr_haswaiters);
417
418 /*
419 * There are 4 states that we find the zthr in at this point
420 * given the locks that we hold:
421 *
422 * [1] The zthr was cancelled, so we spawn a new thread for
423 * the zthr (common case).
424 * [2] The zthr is running at which point this is a no-op.
425 * [3] The zthr is sleeping at which point this is a no-op.
426 * [4] The zthr was just spawned at which point this is a
427 * no-op.
428 */
429 if (t->zthr_thread == NULL) {
430 t->zthr_thread = thread_create_named(t->zthr_name, NULL, 0,
431 zthr_procedure, t, 0, &p0, TS_RUN, t->zthr_pri);
432 }
433
434 mutex_exit(&t->zthr_state_lock);
435 mutex_exit(&t->zthr_request_lock);
436 }
437
438 /*
439 * This function is intended to be used by the zthr itself
440 * (specifically the zthr_func callback provided) to check
441 * if another thread has signaled it to stop running before
442 * doing some expensive operation.
443 *
444 * returns TRUE if we are in the middle of trying to cancel
445 * this thread.
446 *
447 * returns FALSE otherwise.
448 */
449 boolean_t
zthr_iscancelled(zthr_t * t)450 zthr_iscancelled(zthr_t *t)
451 {
452 ASSERT3P(t->zthr_thread, ==, curthread);
453
454 /*
455 * The majority of the functions here grab zthr_request_lock
456 * first and then zthr_state_lock. This function only grabs
457 * the zthr_state_lock. That is because this function should
458 * only be called from the zthr_func to check if someone has
459 * issued a zthr_cancel() on the thread. If there is a zthr_cancel()
460 * happening concurrently, attempting to grab the request lock
461 * here would result in a deadlock.
462 *
463 * By grabbing only the zthr_state_lock this function is allowed
464 * to run concurrently with a zthr_cancel() request.
465 */
466 mutex_enter(&t->zthr_state_lock);
467 boolean_t cancelled = t->zthr_cancel;
468 mutex_exit(&t->zthr_state_lock);
469 return (cancelled);
470 }
471
472 boolean_t
zthr_iscurthread(zthr_t * t)473 zthr_iscurthread(zthr_t *t)
474 {
475 return (t->zthr_thread == curthread);
476 }
477
478 /*
479 * Wait for the zthr to finish its current function. Similar to
480 * zthr_iscancelled, you can use zthr_has_waiters to have the zthr_func end
481 * early. Unlike zthr_cancel, the thread is not destroyed. If the zthr was
482 * sleeping or cancelled, return immediately.
483 */
484 void
zthr_wait_cycle_done(zthr_t * t)485 zthr_wait_cycle_done(zthr_t *t)
486 {
487 mutex_enter(&t->zthr_state_lock);
488
489 /*
490 * Since we are holding the zthr_state_lock at this point
491 * we can find the state in one of the following 5 states:
492 *
493 * [1] The thread has already cancelled, therefore
494 * there is nothing for us to do.
495 * [2] The thread is sleeping so we set the flag, broadcast
496 * the CV and wait for it to exit.
497 * [3] The thread is doing work, in which case we just set
498 * the flag and wait for it to finish.
499 * [4] The thread was just created/resumed, in which case
500 * the behavior is similar to [3].
501 * [5] The thread is the middle of being cancelled, which is
502 * similar to [3]. We'll wait for the cancel, which is
503 * waiting for the zthr func.
504 *
505 * Since requests are serialized, by the time that we get
506 * control back we expect that the zthr has completed it's
507 * zthr_func.
508 */
509 if (t->zthr_thread != NULL) {
510 t->zthr_haswaiters = B_TRUE;
511
512 /* broadcast in case the zthr is sleeping */
513 cv_broadcast(&t->zthr_cv);
514
515 while ((t->zthr_haswaiters) && (t->zthr_thread != NULL))
516 cv_wait(&t->zthr_wait_cv, &t->zthr_state_lock);
517
518 ASSERT(!t->zthr_haswaiters);
519 }
520
521 mutex_exit(&t->zthr_state_lock);
522 }
523
524 /*
525 * This function is intended to be used by the zthr itself
526 * to check if another thread is waiting on it to finish
527 *
528 * returns TRUE if we have been asked to finish.
529 *
530 * returns FALSE otherwise.
531 */
532 boolean_t
zthr_has_waiters(zthr_t * t)533 zthr_has_waiters(zthr_t *t)
534 {
535 ASSERT3P(t->zthr_thread, ==, curthread);
536
537 mutex_enter(&t->zthr_state_lock);
538
539 /*
540 * Similarly to zthr_iscancelled(), we only grab the
541 * zthr_state_lock so that the zthr itself can use this
542 * to check for the request.
543 */
544 boolean_t has_waiters = t->zthr_haswaiters;
545 mutex_exit(&t->zthr_state_lock);
546 return (has_waiters);
547 }
548