1 /* 2 * CDDL HEADER START 3 * 4 * This file and its contents are supplied under the terms of the 5 * Common Development and Distribution License ("CDDL"), version 1.0. 6 * You may only use this file in accordance with the terms of version 7 * 1.0 of the CDDL. 8 * 9 * A full copy of the text of the CDDL should have accompanied this 10 * source. A copy of the CDDL is also available via the Internet at 11 * http://www.illumos.org/license/CDDL. 12 * 13 * CDDL HEADER END 14 */ 15 16 /* 17 * Copyright (c) 2017, 2019 by Delphix. All rights reserved. 18 */ 19 20 /* 21 * ZTHR Infrastructure 22 * =================== 23 * 24 * ZTHR threads are used for isolated operations that span multiple txgs 25 * within a SPA. They generally exist from SPA creation/loading and until 26 * the SPA is exported/destroyed. The ideal requirements for an operation 27 * to be modeled with a zthr are the following: 28 * 29 * 1] The operation needs to run over multiple txgs. 30 * 2] There is be a single point of reference in memory or on disk that 31 * indicates whether the operation should run/is running or has 32 * stopped. 33 * 34 * If the operation satisfies the above then the following rules guarantee 35 * a certain level of correctness: 36 * 37 * 1] Any thread EXCEPT the zthr changes the work indicator from stopped 38 * to running but not the opposite. 39 * 2] Only the zthr can change the work indicator from running to stopped 40 * (e.g. when it is done) but not the opposite. 41 * 42 * This way a normal zthr cycle should go like this: 43 * 44 * 1] An external thread changes the work indicator from stopped to 45 * running and wakes up the zthr. 46 * 2] The zthr wakes up, checks the indicator and starts working. 47 * 3] When the zthr is done, it changes the indicator to stopped, allowing 48 * a new cycle to start. 49 * 50 * Besides being awakened by other threads, a zthr can be configured 51 * during creation to wakeup on it's own after a specified interval 52 * [see zthr_create_timer()]. 53 * 54 * Note: ZTHR threads are NOT a replacement for generic threads! Please 55 * ensure that they fit your use-case well before using them. 56 * 57 * == ZTHR creation 58 * 59 * Every zthr needs three inputs to start running: 60 * 61 * 1] A user-defined checker function (checkfunc) that decides whether 62 * the zthr should start working or go to sleep. The function should 63 * return TRUE when the zthr needs to work or FALSE to let it sleep, 64 * and should adhere to the following signature: 65 * boolean_t checkfunc_name(void *args, zthr_t *t); 66 * 67 * 2] A user-defined ZTHR function (func) which the zthr executes when 68 * it is not sleeping. The function should adhere to the following 69 * signature type: 70 * void func_name(void *args, zthr_t *t); 71 * 72 * 3] A void args pointer that will be passed to checkfunc and func 73 * implicitly by the infrastructure. 74 * 75 * The reason why the above API needs two different functions, 76 * instead of one that both checks and does the work, has to do with 77 * the zthr's internal state lock (zthr_state_lock) and the allowed 78 * cancellation windows. We want to hold the zthr_state_lock while 79 * running checkfunc but not while running func. This way the zthr 80 * can be cancelled while doing work and not while checking for work. 81 * 82 * To start a zthr: 83 * zthr_t *zthr_pointer = zthr_create(checkfunc, func, args); 84 * or 85 * zthr_t *zthr_pointer = zthr_create_timer(checkfunc, func, 86 * args, max_sleep); 87 * 88 * After that you should be able to wakeup, cancel, and resume the 89 * zthr from another thread using the zthr_pointer. 90 * 91 * NOTE: ZTHR threads could potentially wake up spuriously and the 92 * user should take this into account when writing a checkfunc. 93 * [see ZTHR state transitions] 94 * 95 * == ZTHR cancellation 96 * 97 * ZTHR threads must be cancelled when their SPA is being exported 98 * or when they need to be paused so they don't interfere with other 99 * operations. 100 * 101 * To cancel a zthr: 102 * zthr_cancel(zthr_pointer); 103 * 104 * To resume it: 105 * zthr_resume(zthr_pointer); 106 * 107 * A zthr will implicitly check if it has received a cancellation 108 * signal every time func returns and every time it wakes up [see 109 * ZTHR state transitions below]. 110 * 111 * At times, waiting for the zthr's func to finish its job may take 112 * time. This may be very time-consuming for some operations that 113 * need to cancel the SPA's zthrs (e.g spa_export). For this scenario 114 * the user can explicitly make their ZTHR function aware of incoming 115 * cancellation signals using zthr_iscancelled(). A common pattern for 116 * that looks like this: 117 * 118 * int 119 * func_name(void *args, zthr_t *t) 120 * { 121 * ... <unpack args> ... 122 * while (!work_done && !zthr_iscancelled(t)) { 123 * ... <do more work> ... 124 * } 125 * } 126 * 127 * == ZTHR cleanup 128 * 129 * Cancelling a zthr doesn't clean up its metadata (internal locks, 130 * function pointers to func and checkfunc, etc..). This is because 131 * we want to keep them around in case we want to resume the execution 132 * of the zthr later. Similarly for zthrs that exit themselves. 133 * 134 * To completely cleanup a zthr, cancel it first to ensure that it 135 * is not running and then use zthr_destroy(). 136 * 137 * == ZTHR state transitions 138 * 139 * zthr creation 140 * + 141 * | 142 * | woke up 143 * | +--------------+ sleep 144 * | | ^ 145 * | | | 146 * | | | FALSE 147 * | | | 148 * v v FALSE + 149 * cancelled? +---------> checkfunc? 150 * + ^ + 151 * | | | 152 * | | | TRUE 153 * | | | 154 * | | func returned v 155 * | +---------------+ func 156 * | 157 * | TRUE 158 * | 159 * v 160 * zthr stopped running 161 * 162 * == Implementation of ZTHR requests 163 * 164 * ZTHR wakeup, cancel, and resume are requests on a zthr to 165 * change its internal state. Requests on a zthr are serialized 166 * using the zthr_request_lock, while changes in its internal 167 * state are protected by the zthr_state_lock. A request will 168 * first acquire the zthr_request_lock and then immediately 169 * acquire the zthr_state_lock. We do this so that incoming 170 * requests are serialized using the request lock, while still 171 * allowing us to use the state lock for thread communication 172 * via zthr_cv. 173 */ 174 175 #include <sys/zfs_context.h> 176 #include <sys/zthr.h> 177 178 struct zthr { 179 /* running thread doing the work */ 180 kthread_t *zthr_thread; 181 182 /* lock protecting internal data & invariants */ 183 kmutex_t zthr_state_lock; 184 185 /* mutex that serializes external requests */ 186 kmutex_t zthr_request_lock; 187 188 /* notification mechanism for requests */ 189 kcondvar_t zthr_cv; 190 191 /* flag set to true if we are canceling the zthr */ 192 boolean_t zthr_cancel; 193 194 /* 195 * maximum amount of time that the zthr is spent sleeping; 196 * if this is 0, the thread doesn't wake up until it gets 197 * signaled. 198 */ 199 hrtime_t zthr_wait_time; 200 201 /* consumer-provided callbacks & data */ 202 zthr_checkfunc_t *zthr_checkfunc; 203 zthr_func_t *zthr_func; 204 void *zthr_arg; 205 }; 206 207 static void 208 zthr_procedure(void *arg) 209 { 210 zthr_t *t = arg; 211 212 mutex_enter(&t->zthr_state_lock); 213 ASSERT3P(t->zthr_thread, ==, curthread); 214 215 while (!t->zthr_cancel) { 216 if (t->zthr_checkfunc(t->zthr_arg, t)) { 217 mutex_exit(&t->zthr_state_lock); 218 t->zthr_func(t->zthr_arg, t); 219 mutex_enter(&t->zthr_state_lock); 220 } else { 221 /* go to sleep */ 222 if (t->zthr_wait_time == 0) { 223 cv_wait(&t->zthr_cv, &t->zthr_state_lock); 224 } else { 225 (void) cv_timedwait_hires(&t->zthr_cv, 226 &t->zthr_state_lock, t->zthr_wait_time, 227 MSEC2NSEC(1), 0); 228 } 229 } 230 } 231 232 /* 233 * Clear out the kernel thread metadata and notify the 234 * zthr_cancel() thread that we've stopped running. 235 */ 236 t->zthr_thread = NULL; 237 t->zthr_cancel = B_FALSE; 238 cv_broadcast(&t->zthr_cv); 239 240 mutex_exit(&t->zthr_state_lock); 241 thread_exit(); 242 } 243 244 zthr_t * 245 zthr_create(zthr_checkfunc_t *checkfunc, zthr_func_t *func, void *arg) 246 { 247 return (zthr_create_timer(checkfunc, func, arg, (hrtime_t)0)); 248 } 249 250 /* 251 * Create a zthr with specified maximum sleep time. If the time 252 * in sleeping state exceeds max_sleep, a wakeup(do the check and 253 * start working if required) will be triggered. 254 */ 255 zthr_t * 256 zthr_create_timer(zthr_checkfunc_t *checkfunc, zthr_func_t *func, 257 void *arg, hrtime_t max_sleep) 258 { 259 zthr_t *t = kmem_zalloc(sizeof (*t), KM_SLEEP); 260 mutex_init(&t->zthr_state_lock, NULL, MUTEX_DEFAULT, NULL); 261 mutex_init(&t->zthr_request_lock, NULL, MUTEX_DEFAULT, NULL); 262 cv_init(&t->zthr_cv, NULL, CV_DEFAULT, NULL); 263 264 mutex_enter(&t->zthr_state_lock); 265 t->zthr_checkfunc = checkfunc; 266 t->zthr_func = func; 267 t->zthr_arg = arg; 268 t->zthr_wait_time = max_sleep; 269 270 t->zthr_thread = thread_create(NULL, 0, zthr_procedure, t, 271 0, &p0, TS_RUN, minclsyspri); 272 mutex_exit(&t->zthr_state_lock); 273 274 return (t); 275 } 276 277 void 278 zthr_destroy(zthr_t *t) 279 { 280 ASSERT(!MUTEX_HELD(&t->zthr_state_lock)); 281 ASSERT(!MUTEX_HELD(&t->zthr_request_lock)); 282 VERIFY3P(t->zthr_thread, ==, NULL); 283 mutex_destroy(&t->zthr_request_lock); 284 mutex_destroy(&t->zthr_state_lock); 285 cv_destroy(&t->zthr_cv); 286 kmem_free(t, sizeof (*t)); 287 } 288 289 /* 290 * Wake up the zthr if it is sleeping. If the thread has been 291 * cancelled that does nothing. 292 */ 293 void 294 zthr_wakeup(zthr_t *t) 295 { 296 mutex_enter(&t->zthr_request_lock); 297 mutex_enter(&t->zthr_state_lock); 298 299 /* 300 * There are 4 states that we can find the zthr when issuing 301 * this broadcast: 302 * 303 * [1] The common case of the thread being asleep, at which 304 * point the broadcast will wake it up. 305 * [2] The thread has been cancelled. Waking up a cancelled 306 * thread is a no-op. Any work that is still left to be 307 * done should be handled the next time the thread is 308 * resumed. 309 * [3] The thread is doing work and is already up, so this 310 * is basically a no-op. 311 * [4] The thread was just created/resumed, in which case the 312 * behavior is similar to [3]. 313 */ 314 cv_broadcast(&t->zthr_cv); 315 316 mutex_exit(&t->zthr_state_lock); 317 mutex_exit(&t->zthr_request_lock); 318 } 319 320 /* 321 * Sends a cancel request to the zthr and blocks until the zthr is 322 * cancelled. If the zthr is not running (e.g. has been cancelled 323 * already), this is a no-op. 324 */ 325 void 326 zthr_cancel(zthr_t *t) 327 { 328 mutex_enter(&t->zthr_request_lock); 329 mutex_enter(&t->zthr_state_lock); 330 331 /* 332 * Since we are holding the zthr_state_lock at this point 333 * we can find the state in one of the following 4 states: 334 * 335 * [1] The thread has already been cancelled, therefore 336 * there is nothing for us to do. 337 * [2] The thread is sleeping, so we broadcast the CV first 338 * to wake it up and then we set the flag and we are 339 * waiting for it to exit. 340 * [3] The thread is doing work, in which case we just set 341 * the flag and wait for it to finish. 342 * [4] The thread was just created/resumed, in which case 343 * the behavior is similar to [3]. 344 * 345 * Since requests are serialized, by the time that we get 346 * control back we expect that the zthr is cancelled and 347 * not running anymore. 348 */ 349 if (t->zthr_thread != NULL) { 350 t->zthr_cancel = B_TRUE; 351 352 /* broadcast in case the zthr is sleeping */ 353 cv_broadcast(&t->zthr_cv); 354 355 while (t->zthr_thread != NULL) 356 cv_wait(&t->zthr_cv, &t->zthr_state_lock); 357 358 ASSERT(!t->zthr_cancel); 359 } 360 361 mutex_exit(&t->zthr_state_lock); 362 mutex_exit(&t->zthr_request_lock); 363 } 364 365 /* 366 * Sends a resume request to the supplied zthr. If the zthr is 367 * already running this is a no-op. 368 */ 369 void 370 zthr_resume(zthr_t *t) 371 { 372 mutex_enter(&t->zthr_request_lock); 373 mutex_enter(&t->zthr_state_lock); 374 375 ASSERT3P(&t->zthr_checkfunc, !=, NULL); 376 ASSERT3P(&t->zthr_func, !=, NULL); 377 ASSERT(!t->zthr_cancel); 378 379 /* 380 * There are 4 states that we find the zthr in at this point 381 * given the locks that we hold: 382 * 383 * [1] The zthr was cancelled, so we spawn a new thread for 384 * the zthr (common case). 385 * [2] The zthr is running at which point this is a no-op. 386 * [3] The zthr is sleeping at which point this is a no-op. 387 * [4] The zthr was just spawned at which point this is a 388 * no-op. 389 */ 390 if (t->zthr_thread == NULL) { 391 t->zthr_thread = thread_create(NULL, 0, zthr_procedure, t, 392 0, &p0, TS_RUN, minclsyspri); 393 } 394 395 mutex_exit(&t->zthr_state_lock); 396 mutex_exit(&t->zthr_request_lock); 397 } 398 399 /* 400 * This function is intended to be used by the zthr itself 401 * (specifically the zthr_func callback provided) to check 402 * if another thread has signaled it to stop running before 403 * doing some expensive operation. 404 * 405 * returns TRUE if we are in the middle of trying to cancel 406 * this thread. 407 * 408 * returns FALSE otherwise. 409 */ 410 boolean_t 411 zthr_iscancelled(zthr_t *t) 412 { 413 ASSERT3P(t->zthr_thread, ==, curthread); 414 415 /* 416 * The majority of the functions here grab zthr_request_lock 417 * first and then zthr_state_lock. This function only grabs 418 * the zthr_state_lock. That is because this function should 419 * only be called from the zthr_func to check if someone has 420 * issued a zthr_cancel() on the thread. If there is a zthr_cancel() 421 * happening concurrently, attempting to grab the request lock 422 * here would result in a deadlock. 423 * 424 * By grabbing only the zthr_state_lock this function is allowed 425 * to run concurrently with a zthr_cancel() request. 426 */ 427 mutex_enter(&t->zthr_state_lock); 428 boolean_t cancelled = t->zthr_cancel; 429 mutex_exit(&t->zthr_state_lock); 430 return (cancelled); 431 } 432