1 /* 2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. 3 * Copyright (C) 2007 The Regents of the University of California. 4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 6 * UCRL-CODE-235197 7 * 8 * This file is part of the SPL, Solaris Porting Layer. 9 * 10 * The SPL is free software; you can redistribute it and/or modify it 11 * under the terms of the GNU General Public License as published by the 12 * Free Software Foundation; either version 2 of the License, or (at your 13 * option) any later version. 14 * 15 * The SPL is distributed in the hope that it will be useful, but WITHOUT 16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 18 * for more details. 19 * 20 * You should have received a copy of the GNU General Public License along 21 * with the SPL. If not, see <http://www.gnu.org/licenses/>. 22 * 23 * Solaris Porting Layer (SPL) Task Queue Implementation. 24 */ 25 26 #include <sys/timer.h> 27 #include <sys/taskq.h> 28 #include <sys/kmem.h> 29 #include <sys/tsd.h> 30 #include <sys/trace_spl.h> 31 #ifdef HAVE_CPU_HOTPLUG 32 #include <linux/cpuhotplug.h> 33 #endif 34 35 int spl_taskq_thread_bind = 0; 36 module_param(spl_taskq_thread_bind, int, 0644); 37 MODULE_PARM_DESC(spl_taskq_thread_bind, "Bind taskq thread to CPU by default"); 38 39 40 int spl_taskq_thread_dynamic = 1; 41 module_param(spl_taskq_thread_dynamic, int, 0444); 42 MODULE_PARM_DESC(spl_taskq_thread_dynamic, "Allow dynamic taskq threads"); 43 44 int spl_taskq_thread_priority = 1; 45 module_param(spl_taskq_thread_priority, int, 0644); 46 MODULE_PARM_DESC(spl_taskq_thread_priority, 47 "Allow non-default priority for taskq threads"); 48 49 int spl_taskq_thread_sequential = 4; 50 module_param(spl_taskq_thread_sequential, int, 0644); 51 MODULE_PARM_DESC(spl_taskq_thread_sequential, 52 "Create new taskq threads after N sequential tasks"); 53 54 /* Global system-wide dynamic task queue available for all consumers */ 55 taskq_t *system_taskq; 56 EXPORT_SYMBOL(system_taskq); 57 /* Global dynamic task queue for long delay */ 58 taskq_t *system_delay_taskq; 59 EXPORT_SYMBOL(system_delay_taskq); 60 61 /* Private dedicated taskq for creating new taskq threads on demand. */ 62 static taskq_t *dynamic_taskq; 63 static taskq_thread_t *taskq_thread_create(taskq_t *); 64 65 #ifdef HAVE_CPU_HOTPLUG 66 /* Multi-callback id for cpu hotplugging. */ 67 static int spl_taskq_cpuhp_state; 68 #endif 69 70 /* List of all taskqs */ 71 LIST_HEAD(tq_list); 72 struct rw_semaphore tq_list_sem; 73 static uint_t taskq_tsd; 74 75 static int 76 task_km_flags(uint_t flags) 77 { 78 if (flags & TQ_NOSLEEP) 79 return (KM_NOSLEEP); 80 81 if (flags & TQ_PUSHPAGE) 82 return (KM_PUSHPAGE); 83 84 return (KM_SLEEP); 85 } 86 87 /* 88 * taskq_find_by_name - Find the largest instance number of a named taskq. 89 */ 90 static int 91 taskq_find_by_name(const char *name) 92 { 93 struct list_head *tql = NULL; 94 taskq_t *tq; 95 96 list_for_each_prev(tql, &tq_list) { 97 tq = list_entry(tql, taskq_t, tq_taskqs); 98 if (strcmp(name, tq->tq_name) == 0) 99 return (tq->tq_instance); 100 } 101 return (-1); 102 } 103 104 /* 105 * NOTE: Must be called with tq->tq_lock held, returns a list_t which 106 * is not attached to the free, work, or pending taskq lists. 107 */ 108 static taskq_ent_t * 109 task_alloc(taskq_t *tq, uint_t flags, unsigned long *irqflags) 110 { 111 taskq_ent_t *t; 112 int count = 0; 113 114 ASSERT(tq); 115 retry: 116 /* Acquire taskq_ent_t's from free list if available */ 117 if (!list_empty(&tq->tq_free_list) && !(flags & TQ_NEW)) { 118 t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list); 119 120 ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); 121 ASSERT(!(t->tqent_flags & TQENT_FLAG_CANCEL)); 122 ASSERT(!timer_pending(&t->tqent_timer)); 123 124 list_del_init(&t->tqent_list); 125 return (t); 126 } 127 128 /* Free list is empty and memory allocations are prohibited */ 129 if (flags & TQ_NOALLOC) 130 return (NULL); 131 132 /* Hit maximum taskq_ent_t pool size */ 133 if (tq->tq_nalloc >= tq->tq_maxalloc) { 134 if (flags & TQ_NOSLEEP) 135 return (NULL); 136 137 /* 138 * Sleep periodically polling the free list for an available 139 * taskq_ent_t. Dispatching with TQ_SLEEP should always succeed 140 * but we cannot block forever waiting for an taskq_ent_t to 141 * show up in the free list, otherwise a deadlock can happen. 142 * 143 * Therefore, we need to allocate a new task even if the number 144 * of allocated tasks is above tq->tq_maxalloc, but we still 145 * end up delaying the task allocation by one second, thereby 146 * throttling the task dispatch rate. 147 */ 148 spin_unlock_irqrestore(&tq->tq_lock, *irqflags); 149 schedule_timeout(HZ / 100); 150 spin_lock_irqsave_nested(&tq->tq_lock, *irqflags, 151 tq->tq_lock_class); 152 if (count < 100) { 153 count++; 154 goto retry; 155 } 156 } 157 158 spin_unlock_irqrestore(&tq->tq_lock, *irqflags); 159 t = kmem_alloc(sizeof (taskq_ent_t), task_km_flags(flags)); 160 spin_lock_irqsave_nested(&tq->tq_lock, *irqflags, tq->tq_lock_class); 161 162 if (t) { 163 taskq_init_ent(t); 164 tq->tq_nalloc++; 165 } 166 167 return (t); 168 } 169 170 /* 171 * NOTE: Must be called with tq->tq_lock held, expects the taskq_ent_t 172 * to already be removed from the free, work, or pending taskq lists. 173 */ 174 static void 175 task_free(taskq_t *tq, taskq_ent_t *t) 176 { 177 ASSERT(tq); 178 ASSERT(t); 179 ASSERT(list_empty(&t->tqent_list)); 180 ASSERT(!timer_pending(&t->tqent_timer)); 181 182 kmem_free(t, sizeof (taskq_ent_t)); 183 tq->tq_nalloc--; 184 } 185 186 /* 187 * NOTE: Must be called with tq->tq_lock held, either destroys the 188 * taskq_ent_t if too many exist or moves it to the free list for later use. 189 */ 190 static void 191 task_done(taskq_t *tq, taskq_ent_t *t) 192 { 193 ASSERT(tq); 194 ASSERT(t); 195 196 /* Wake tasks blocked in taskq_wait_id() */ 197 wake_up_all(&t->tqent_waitq); 198 199 list_del_init(&t->tqent_list); 200 201 if (tq->tq_nalloc <= tq->tq_minalloc) { 202 t->tqent_id = TASKQID_INVALID; 203 t->tqent_func = NULL; 204 t->tqent_arg = NULL; 205 t->tqent_flags = 0; 206 207 list_add_tail(&t->tqent_list, &tq->tq_free_list); 208 } else { 209 task_free(tq, t); 210 } 211 } 212 213 /* 214 * When a delayed task timer expires remove it from the delay list and 215 * add it to the priority list in order for immediate processing. 216 */ 217 static void 218 task_expire_impl(taskq_ent_t *t) 219 { 220 taskq_ent_t *w; 221 taskq_t *tq = t->tqent_taskq; 222 struct list_head *l = NULL; 223 unsigned long flags; 224 225 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 226 227 if (t->tqent_flags & TQENT_FLAG_CANCEL) { 228 ASSERT(list_empty(&t->tqent_list)); 229 spin_unlock_irqrestore(&tq->tq_lock, flags); 230 return; 231 } 232 233 t->tqent_birth = jiffies; 234 DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t); 235 236 /* 237 * The priority list must be maintained in strict task id order 238 * from lowest to highest for lowest_id to be easily calculable. 239 */ 240 list_del(&t->tqent_list); 241 list_for_each_prev(l, &tq->tq_prio_list) { 242 w = list_entry(l, taskq_ent_t, tqent_list); 243 if (w->tqent_id < t->tqent_id) { 244 list_add(&t->tqent_list, l); 245 break; 246 } 247 } 248 if (l == &tq->tq_prio_list) 249 list_add(&t->tqent_list, &tq->tq_prio_list); 250 251 spin_unlock_irqrestore(&tq->tq_lock, flags); 252 253 wake_up(&tq->tq_work_waitq); 254 } 255 256 static void 257 task_expire(spl_timer_list_t tl) 258 { 259 struct timer_list *tmr = (struct timer_list *)tl; 260 taskq_ent_t *t = from_timer(t, tmr, tqent_timer); 261 task_expire_impl(t); 262 } 263 264 /* 265 * Returns the lowest incomplete taskqid_t. The taskqid_t may 266 * be queued on the pending list, on the priority list, on the 267 * delay list, or on the work list currently being handled, but 268 * it is not 100% complete yet. 269 */ 270 static taskqid_t 271 taskq_lowest_id(taskq_t *tq) 272 { 273 taskqid_t lowest_id = tq->tq_next_id; 274 taskq_ent_t *t; 275 taskq_thread_t *tqt; 276 277 if (!list_empty(&tq->tq_pend_list)) { 278 t = list_entry(tq->tq_pend_list.next, taskq_ent_t, tqent_list); 279 lowest_id = MIN(lowest_id, t->tqent_id); 280 } 281 282 if (!list_empty(&tq->tq_prio_list)) { 283 t = list_entry(tq->tq_prio_list.next, taskq_ent_t, tqent_list); 284 lowest_id = MIN(lowest_id, t->tqent_id); 285 } 286 287 if (!list_empty(&tq->tq_delay_list)) { 288 t = list_entry(tq->tq_delay_list.next, taskq_ent_t, tqent_list); 289 lowest_id = MIN(lowest_id, t->tqent_id); 290 } 291 292 if (!list_empty(&tq->tq_active_list)) { 293 tqt = list_entry(tq->tq_active_list.next, taskq_thread_t, 294 tqt_active_list); 295 ASSERT(tqt->tqt_id != TASKQID_INVALID); 296 lowest_id = MIN(lowest_id, tqt->tqt_id); 297 } 298 299 return (lowest_id); 300 } 301 302 /* 303 * Insert a task into a list keeping the list sorted by increasing taskqid. 304 */ 305 static void 306 taskq_insert_in_order(taskq_t *tq, taskq_thread_t *tqt) 307 { 308 taskq_thread_t *w; 309 struct list_head *l = NULL; 310 311 ASSERT(tq); 312 ASSERT(tqt); 313 314 list_for_each_prev(l, &tq->tq_active_list) { 315 w = list_entry(l, taskq_thread_t, tqt_active_list); 316 if (w->tqt_id < tqt->tqt_id) { 317 list_add(&tqt->tqt_active_list, l); 318 break; 319 } 320 } 321 if (l == &tq->tq_active_list) 322 list_add(&tqt->tqt_active_list, &tq->tq_active_list); 323 } 324 325 /* 326 * Find and return a task from the given list if it exists. The list 327 * must be in lowest to highest task id order. 328 */ 329 static taskq_ent_t * 330 taskq_find_list(taskq_t *tq, struct list_head *lh, taskqid_t id) 331 { 332 struct list_head *l = NULL; 333 taskq_ent_t *t; 334 335 list_for_each(l, lh) { 336 t = list_entry(l, taskq_ent_t, tqent_list); 337 338 if (t->tqent_id == id) 339 return (t); 340 341 if (t->tqent_id > id) 342 break; 343 } 344 345 return (NULL); 346 } 347 348 /* 349 * Find an already dispatched task given the task id regardless of what 350 * state it is in. If a task is still pending it will be returned. 351 * If a task is executing, then -EBUSY will be returned instead. 352 * If the task has already been run then NULL is returned. 353 */ 354 static taskq_ent_t * 355 taskq_find(taskq_t *tq, taskqid_t id) 356 { 357 taskq_thread_t *tqt; 358 struct list_head *l = NULL; 359 taskq_ent_t *t; 360 361 t = taskq_find_list(tq, &tq->tq_delay_list, id); 362 if (t) 363 return (t); 364 365 t = taskq_find_list(tq, &tq->tq_prio_list, id); 366 if (t) 367 return (t); 368 369 t = taskq_find_list(tq, &tq->tq_pend_list, id); 370 if (t) 371 return (t); 372 373 list_for_each(l, &tq->tq_active_list) { 374 tqt = list_entry(l, taskq_thread_t, tqt_active_list); 375 if (tqt->tqt_id == id) { 376 /* 377 * Instead of returning tqt_task, we just return a non 378 * NULL value to prevent misuse, since tqt_task only 379 * has two valid fields. 380 */ 381 return (ERR_PTR(-EBUSY)); 382 } 383 } 384 385 return (NULL); 386 } 387 388 /* 389 * Theory for the taskq_wait_id(), taskq_wait_outstanding(), and 390 * taskq_wait() functions below. 391 * 392 * Taskq waiting is accomplished by tracking the lowest outstanding task 393 * id and the next available task id. As tasks are dispatched they are 394 * added to the tail of the pending, priority, or delay lists. As worker 395 * threads become available the tasks are removed from the heads of these 396 * lists and linked to the worker threads. This ensures the lists are 397 * kept sorted by lowest to highest task id. 398 * 399 * Therefore the lowest outstanding task id can be quickly determined by 400 * checking the head item from all of these lists. This value is stored 401 * with the taskq as the lowest id. It only needs to be recalculated when 402 * either the task with the current lowest id completes or is canceled. 403 * 404 * By blocking until the lowest task id exceeds the passed task id the 405 * taskq_wait_outstanding() function can be easily implemented. Similarly, 406 * by blocking until the lowest task id matches the next task id taskq_wait() 407 * can be implemented. 408 * 409 * Callers should be aware that when there are multiple worked threads it 410 * is possible for larger task ids to complete before smaller ones. Also 411 * when the taskq contains delay tasks with small task ids callers may 412 * block for a considerable length of time waiting for them to expire and 413 * execute. 414 */ 415 static int 416 taskq_wait_id_check(taskq_t *tq, taskqid_t id) 417 { 418 int rc; 419 unsigned long flags; 420 421 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 422 rc = (taskq_find(tq, id) == NULL); 423 spin_unlock_irqrestore(&tq->tq_lock, flags); 424 425 return (rc); 426 } 427 428 /* 429 * The taskq_wait_id() function blocks until the passed task id completes. 430 * This does not guarantee that all lower task ids have completed. 431 */ 432 void 433 taskq_wait_id(taskq_t *tq, taskqid_t id) 434 { 435 wait_event(tq->tq_wait_waitq, taskq_wait_id_check(tq, id)); 436 } 437 EXPORT_SYMBOL(taskq_wait_id); 438 439 static int 440 taskq_wait_outstanding_check(taskq_t *tq, taskqid_t id) 441 { 442 int rc; 443 unsigned long flags; 444 445 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 446 rc = (id < tq->tq_lowest_id); 447 spin_unlock_irqrestore(&tq->tq_lock, flags); 448 449 return (rc); 450 } 451 452 /* 453 * The taskq_wait_outstanding() function will block until all tasks with a 454 * lower taskqid than the passed 'id' have been completed. Note that all 455 * task id's are assigned monotonically at dispatch time. Zero may be 456 * passed for the id to indicate all tasks dispatch up to this point, 457 * but not after, should be waited for. 458 */ 459 void 460 taskq_wait_outstanding(taskq_t *tq, taskqid_t id) 461 { 462 id = id ? id : tq->tq_next_id - 1; 463 wait_event(tq->tq_wait_waitq, taskq_wait_outstanding_check(tq, id)); 464 } 465 EXPORT_SYMBOL(taskq_wait_outstanding); 466 467 static int 468 taskq_wait_check(taskq_t *tq) 469 { 470 int rc; 471 unsigned long flags; 472 473 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 474 rc = (tq->tq_lowest_id == tq->tq_next_id); 475 spin_unlock_irqrestore(&tq->tq_lock, flags); 476 477 return (rc); 478 } 479 480 /* 481 * The taskq_wait() function will block until the taskq is empty. 482 * This means that if a taskq re-dispatches work to itself taskq_wait() 483 * callers will block indefinitely. 484 */ 485 void 486 taskq_wait(taskq_t *tq) 487 { 488 wait_event(tq->tq_wait_waitq, taskq_wait_check(tq)); 489 } 490 EXPORT_SYMBOL(taskq_wait); 491 492 int 493 taskq_member(taskq_t *tq, kthread_t *t) 494 { 495 return (tq == (taskq_t *)tsd_get_by_thread(taskq_tsd, t)); 496 } 497 EXPORT_SYMBOL(taskq_member); 498 499 taskq_t * 500 taskq_of_curthread(void) 501 { 502 return (tsd_get(taskq_tsd)); 503 } 504 EXPORT_SYMBOL(taskq_of_curthread); 505 506 /* 507 * Cancel an already dispatched task given the task id. Still pending tasks 508 * will be immediately canceled, and if the task is active the function will 509 * block until it completes. Preallocated tasks which are canceled must be 510 * freed by the caller. 511 */ 512 int 513 taskq_cancel_id(taskq_t *tq, taskqid_t id) 514 { 515 taskq_ent_t *t; 516 int rc = ENOENT; 517 unsigned long flags; 518 519 ASSERT(tq); 520 521 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 522 t = taskq_find(tq, id); 523 if (t && t != ERR_PTR(-EBUSY)) { 524 list_del_init(&t->tqent_list); 525 t->tqent_flags |= TQENT_FLAG_CANCEL; 526 527 /* 528 * When canceling the lowest outstanding task id we 529 * must recalculate the new lowest outstanding id. 530 */ 531 if (tq->tq_lowest_id == t->tqent_id) { 532 tq->tq_lowest_id = taskq_lowest_id(tq); 533 ASSERT3S(tq->tq_lowest_id, >, t->tqent_id); 534 } 535 536 /* 537 * The task_expire() function takes the tq->tq_lock so drop 538 * drop the lock before synchronously cancelling the timer. 539 */ 540 if (timer_pending(&t->tqent_timer)) { 541 spin_unlock_irqrestore(&tq->tq_lock, flags); 542 del_timer_sync(&t->tqent_timer); 543 spin_lock_irqsave_nested(&tq->tq_lock, flags, 544 tq->tq_lock_class); 545 } 546 547 if (!(t->tqent_flags & TQENT_FLAG_PREALLOC)) 548 task_done(tq, t); 549 550 rc = 0; 551 } 552 spin_unlock_irqrestore(&tq->tq_lock, flags); 553 554 if (t == ERR_PTR(-EBUSY)) { 555 taskq_wait_id(tq, id); 556 rc = EBUSY; 557 } 558 559 return (rc); 560 } 561 EXPORT_SYMBOL(taskq_cancel_id); 562 563 static int taskq_thread_spawn(taskq_t *tq); 564 565 taskqid_t 566 taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) 567 { 568 taskq_ent_t *t; 569 taskqid_t rc = TASKQID_INVALID; 570 unsigned long irqflags; 571 572 ASSERT(tq); 573 ASSERT(func); 574 575 spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class); 576 577 /* Taskq being destroyed and all tasks drained */ 578 if (!(tq->tq_flags & TASKQ_ACTIVE)) 579 goto out; 580 581 /* Do not queue the task unless there is idle thread for it */ 582 ASSERT(tq->tq_nactive <= tq->tq_nthreads); 583 if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) { 584 /* Dynamic taskq may be able to spawn another thread */ 585 if (!(tq->tq_flags & TASKQ_DYNAMIC) || 586 taskq_thread_spawn(tq) == 0) 587 goto out; 588 } 589 590 if ((t = task_alloc(tq, flags, &irqflags)) == NULL) 591 goto out; 592 593 spin_lock(&t->tqent_lock); 594 595 /* Queue to the front of the list to enforce TQ_NOQUEUE semantics */ 596 if (flags & TQ_NOQUEUE) 597 list_add(&t->tqent_list, &tq->tq_prio_list); 598 /* Queue to the priority list instead of the pending list */ 599 else if (flags & TQ_FRONT) 600 list_add_tail(&t->tqent_list, &tq->tq_prio_list); 601 else 602 list_add_tail(&t->tqent_list, &tq->tq_pend_list); 603 604 t->tqent_id = rc = tq->tq_next_id; 605 tq->tq_next_id++; 606 t->tqent_func = func; 607 t->tqent_arg = arg; 608 t->tqent_taskq = tq; 609 t->tqent_timer.function = NULL; 610 t->tqent_timer.expires = 0; 611 612 t->tqent_birth = jiffies; 613 DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t); 614 615 ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); 616 617 spin_unlock(&t->tqent_lock); 618 619 wake_up(&tq->tq_work_waitq); 620 out: 621 /* Spawn additional taskq threads if required. */ 622 if (!(flags & TQ_NOQUEUE) && tq->tq_nactive == tq->tq_nthreads) 623 (void) taskq_thread_spawn(tq); 624 625 spin_unlock_irqrestore(&tq->tq_lock, irqflags); 626 return (rc); 627 } 628 EXPORT_SYMBOL(taskq_dispatch); 629 630 taskqid_t 631 taskq_dispatch_delay(taskq_t *tq, task_func_t func, void *arg, 632 uint_t flags, clock_t expire_time) 633 { 634 taskqid_t rc = TASKQID_INVALID; 635 taskq_ent_t *t; 636 unsigned long irqflags; 637 638 ASSERT(tq); 639 ASSERT(func); 640 641 spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class); 642 643 /* Taskq being destroyed and all tasks drained */ 644 if (!(tq->tq_flags & TASKQ_ACTIVE)) 645 goto out; 646 647 if ((t = task_alloc(tq, flags, &irqflags)) == NULL) 648 goto out; 649 650 spin_lock(&t->tqent_lock); 651 652 /* Queue to the delay list for subsequent execution */ 653 list_add_tail(&t->tqent_list, &tq->tq_delay_list); 654 655 t->tqent_id = rc = tq->tq_next_id; 656 tq->tq_next_id++; 657 t->tqent_func = func; 658 t->tqent_arg = arg; 659 t->tqent_taskq = tq; 660 t->tqent_timer.function = task_expire; 661 t->tqent_timer.expires = (unsigned long)expire_time; 662 add_timer(&t->tqent_timer); 663 664 ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); 665 666 spin_unlock(&t->tqent_lock); 667 out: 668 /* Spawn additional taskq threads if required. */ 669 if (tq->tq_nactive == tq->tq_nthreads) 670 (void) taskq_thread_spawn(tq); 671 spin_unlock_irqrestore(&tq->tq_lock, irqflags); 672 return (rc); 673 } 674 EXPORT_SYMBOL(taskq_dispatch_delay); 675 676 void 677 taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags, 678 taskq_ent_t *t) 679 { 680 unsigned long irqflags; 681 ASSERT(tq); 682 ASSERT(func); 683 684 spin_lock_irqsave_nested(&tq->tq_lock, irqflags, 685 tq->tq_lock_class); 686 687 /* Taskq being destroyed and all tasks drained */ 688 if (!(tq->tq_flags & TASKQ_ACTIVE)) { 689 t->tqent_id = TASKQID_INVALID; 690 goto out; 691 } 692 693 if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) { 694 /* Dynamic taskq may be able to spawn another thread */ 695 if (!(tq->tq_flags & TASKQ_DYNAMIC) || 696 taskq_thread_spawn(tq) == 0) 697 goto out2; 698 flags |= TQ_FRONT; 699 } 700 701 spin_lock(&t->tqent_lock); 702 703 /* 704 * Make sure the entry is not on some other taskq; it is important to 705 * ASSERT() under lock 706 */ 707 ASSERT(taskq_empty_ent(t)); 708 709 /* 710 * Mark it as a prealloc'd task. This is important 711 * to ensure that we don't free it later. 712 */ 713 t->tqent_flags |= TQENT_FLAG_PREALLOC; 714 715 /* Queue to the priority list instead of the pending list */ 716 if (flags & TQ_FRONT) 717 list_add_tail(&t->tqent_list, &tq->tq_prio_list); 718 else 719 list_add_tail(&t->tqent_list, &tq->tq_pend_list); 720 721 t->tqent_id = tq->tq_next_id; 722 tq->tq_next_id++; 723 t->tqent_func = func; 724 t->tqent_arg = arg; 725 t->tqent_taskq = tq; 726 727 t->tqent_birth = jiffies; 728 DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t); 729 730 spin_unlock(&t->tqent_lock); 731 732 wake_up(&tq->tq_work_waitq); 733 out: 734 /* Spawn additional taskq threads if required. */ 735 if (tq->tq_nactive == tq->tq_nthreads) 736 (void) taskq_thread_spawn(tq); 737 out2: 738 spin_unlock_irqrestore(&tq->tq_lock, irqflags); 739 } 740 EXPORT_SYMBOL(taskq_dispatch_ent); 741 742 int 743 taskq_empty_ent(taskq_ent_t *t) 744 { 745 return (list_empty(&t->tqent_list)); 746 } 747 EXPORT_SYMBOL(taskq_empty_ent); 748 749 void 750 taskq_init_ent(taskq_ent_t *t) 751 { 752 spin_lock_init(&t->tqent_lock); 753 init_waitqueue_head(&t->tqent_waitq); 754 timer_setup(&t->tqent_timer, NULL, 0); 755 INIT_LIST_HEAD(&t->tqent_list); 756 t->tqent_id = 0; 757 t->tqent_func = NULL; 758 t->tqent_arg = NULL; 759 t->tqent_flags = 0; 760 t->tqent_taskq = NULL; 761 } 762 EXPORT_SYMBOL(taskq_init_ent); 763 764 /* 765 * Return the next pending task, preference is given to tasks on the 766 * priority list which were dispatched with TQ_FRONT. 767 */ 768 static taskq_ent_t * 769 taskq_next_ent(taskq_t *tq) 770 { 771 struct list_head *list; 772 773 if (!list_empty(&tq->tq_prio_list)) 774 list = &tq->tq_prio_list; 775 else if (!list_empty(&tq->tq_pend_list)) 776 list = &tq->tq_pend_list; 777 else 778 return (NULL); 779 780 return (list_entry(list->next, taskq_ent_t, tqent_list)); 781 } 782 783 /* 784 * Spawns a new thread for the specified taskq. 785 */ 786 static void 787 taskq_thread_spawn_task(void *arg) 788 { 789 taskq_t *tq = (taskq_t *)arg; 790 unsigned long flags; 791 792 if (taskq_thread_create(tq) == NULL) { 793 /* restore spawning count if failed */ 794 spin_lock_irqsave_nested(&tq->tq_lock, flags, 795 tq->tq_lock_class); 796 tq->tq_nspawn--; 797 spin_unlock_irqrestore(&tq->tq_lock, flags); 798 } 799 } 800 801 /* 802 * Spawn addition threads for dynamic taskqs (TASKQ_DYNAMIC) the current 803 * number of threads is insufficient to handle the pending tasks. These 804 * new threads must be created by the dedicated dynamic_taskq to avoid 805 * deadlocks between thread creation and memory reclaim. The system_taskq 806 * which is also a dynamic taskq cannot be safely used for this. 807 */ 808 static int 809 taskq_thread_spawn(taskq_t *tq) 810 { 811 int spawning = 0; 812 813 if (!(tq->tq_flags & TASKQ_DYNAMIC)) 814 return (0); 815 816 if ((tq->tq_nthreads + tq->tq_nspawn < tq->tq_maxthreads) && 817 (tq->tq_flags & TASKQ_ACTIVE)) { 818 spawning = (++tq->tq_nspawn); 819 taskq_dispatch(dynamic_taskq, taskq_thread_spawn_task, 820 tq, TQ_NOSLEEP); 821 } 822 823 return (spawning); 824 } 825 826 /* 827 * Threads in a dynamic taskq should only exit once it has been completely 828 * drained and no other threads are actively servicing tasks. This prevents 829 * threads from being created and destroyed more than is required. 830 * 831 * The first thread is the thread list is treated as the primary thread. 832 * There is nothing special about the primary thread but in order to avoid 833 * all the taskq pids from changing we opt to make it long running. 834 */ 835 static int 836 taskq_thread_should_stop(taskq_t *tq, taskq_thread_t *tqt) 837 { 838 if (!(tq->tq_flags & TASKQ_DYNAMIC)) 839 return (0); 840 841 if (list_first_entry(&(tq->tq_thread_list), taskq_thread_t, 842 tqt_thread_list) == tqt) 843 return (0); 844 845 return 846 ((tq->tq_nspawn == 0) && /* No threads are being spawned */ 847 (tq->tq_nactive == 0) && /* No threads are handling tasks */ 848 (tq->tq_nthreads > 1) && /* More than 1 thread is running */ 849 (!taskq_next_ent(tq)) && /* There are no pending tasks */ 850 (spl_taskq_thread_dynamic)); /* Dynamic taskqs are allowed */ 851 } 852 853 static int 854 taskq_thread(void *args) 855 { 856 DECLARE_WAITQUEUE(wait, current); 857 sigset_t blocked; 858 taskq_thread_t *tqt = args; 859 taskq_t *tq; 860 taskq_ent_t *t; 861 int seq_tasks = 0; 862 unsigned long flags; 863 taskq_ent_t dup_task = {}; 864 865 ASSERT(tqt); 866 ASSERT(tqt->tqt_tq); 867 tq = tqt->tqt_tq; 868 current->flags |= PF_NOFREEZE; 869 870 (void) spl_fstrans_mark(); 871 872 sigfillset(&blocked); 873 sigprocmask(SIG_BLOCK, &blocked, NULL); 874 flush_signals(current); 875 876 tsd_set(taskq_tsd, tq); 877 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 878 /* 879 * If we are dynamically spawned, decrease spawning count. Note that 880 * we could be created during taskq_create, in which case we shouldn't 881 * do the decrement. But it's fine because taskq_create will reset 882 * tq_nspawn later. 883 */ 884 if (tq->tq_flags & TASKQ_DYNAMIC) 885 tq->tq_nspawn--; 886 887 /* Immediately exit if more threads than allowed were created. */ 888 if (tq->tq_nthreads >= tq->tq_maxthreads) 889 goto error; 890 891 tq->tq_nthreads++; 892 list_add_tail(&tqt->tqt_thread_list, &tq->tq_thread_list); 893 wake_up(&tq->tq_wait_waitq); 894 set_current_state(TASK_INTERRUPTIBLE); 895 896 while (!kthread_should_stop()) { 897 898 if (list_empty(&tq->tq_pend_list) && 899 list_empty(&tq->tq_prio_list)) { 900 901 if (taskq_thread_should_stop(tq, tqt)) { 902 wake_up_all(&tq->tq_wait_waitq); 903 break; 904 } 905 906 add_wait_queue_exclusive(&tq->tq_work_waitq, &wait); 907 spin_unlock_irqrestore(&tq->tq_lock, flags); 908 909 schedule(); 910 seq_tasks = 0; 911 912 spin_lock_irqsave_nested(&tq->tq_lock, flags, 913 tq->tq_lock_class); 914 remove_wait_queue(&tq->tq_work_waitq, &wait); 915 } else { 916 __set_current_state(TASK_RUNNING); 917 } 918 919 if ((t = taskq_next_ent(tq)) != NULL) { 920 list_del_init(&t->tqent_list); 921 922 /* 923 * A TQENT_FLAG_PREALLOC task may be reused or freed 924 * during the task function call. Store tqent_id and 925 * tqent_flags here. 926 * 927 * Also use an on stack taskq_ent_t for tqt_task 928 * assignment in this case; we want to make sure 929 * to duplicate all fields, so the values are 930 * correct when it's accessed via DTRACE_PROBE*. 931 */ 932 tqt->tqt_id = t->tqent_id; 933 tqt->tqt_flags = t->tqent_flags; 934 935 if (t->tqent_flags & TQENT_FLAG_PREALLOC) { 936 dup_task = *t; 937 t = &dup_task; 938 } 939 tqt->tqt_task = t; 940 941 taskq_insert_in_order(tq, tqt); 942 tq->tq_nactive++; 943 spin_unlock_irqrestore(&tq->tq_lock, flags); 944 945 DTRACE_PROBE1(taskq_ent__start, taskq_ent_t *, t); 946 947 /* Perform the requested task */ 948 t->tqent_func(t->tqent_arg); 949 950 DTRACE_PROBE1(taskq_ent__finish, taskq_ent_t *, t); 951 952 spin_lock_irqsave_nested(&tq->tq_lock, flags, 953 tq->tq_lock_class); 954 tq->tq_nactive--; 955 list_del_init(&tqt->tqt_active_list); 956 tqt->tqt_task = NULL; 957 958 /* For prealloc'd tasks, we don't free anything. */ 959 if (!(tqt->tqt_flags & TQENT_FLAG_PREALLOC)) 960 task_done(tq, t); 961 962 /* 963 * When the current lowest outstanding taskqid is 964 * done calculate the new lowest outstanding id 965 */ 966 if (tq->tq_lowest_id == tqt->tqt_id) { 967 tq->tq_lowest_id = taskq_lowest_id(tq); 968 ASSERT3S(tq->tq_lowest_id, >, tqt->tqt_id); 969 } 970 971 /* Spawn additional taskq threads if required. */ 972 if ((++seq_tasks) > spl_taskq_thread_sequential && 973 taskq_thread_spawn(tq)) 974 seq_tasks = 0; 975 976 tqt->tqt_id = TASKQID_INVALID; 977 tqt->tqt_flags = 0; 978 wake_up_all(&tq->tq_wait_waitq); 979 } else { 980 if (taskq_thread_should_stop(tq, tqt)) 981 break; 982 } 983 984 set_current_state(TASK_INTERRUPTIBLE); 985 986 } 987 988 __set_current_state(TASK_RUNNING); 989 tq->tq_nthreads--; 990 list_del_init(&tqt->tqt_thread_list); 991 error: 992 kmem_free(tqt, sizeof (taskq_thread_t)); 993 spin_unlock_irqrestore(&tq->tq_lock, flags); 994 995 tsd_set(taskq_tsd, NULL); 996 thread_exit(); 997 998 return (0); 999 } 1000 1001 static taskq_thread_t * 1002 taskq_thread_create(taskq_t *tq) 1003 { 1004 static int last_used_cpu = 0; 1005 taskq_thread_t *tqt; 1006 1007 tqt = kmem_alloc(sizeof (*tqt), KM_PUSHPAGE); 1008 INIT_LIST_HEAD(&tqt->tqt_thread_list); 1009 INIT_LIST_HEAD(&tqt->tqt_active_list); 1010 tqt->tqt_tq = tq; 1011 tqt->tqt_id = TASKQID_INVALID; 1012 1013 tqt->tqt_thread = spl_kthread_create(taskq_thread, tqt, 1014 "%s", tq->tq_name); 1015 if (tqt->tqt_thread == NULL) { 1016 kmem_free(tqt, sizeof (taskq_thread_t)); 1017 return (NULL); 1018 } 1019 1020 if (spl_taskq_thread_bind) { 1021 last_used_cpu = (last_used_cpu + 1) % num_online_cpus(); 1022 kthread_bind(tqt->tqt_thread, last_used_cpu); 1023 } 1024 1025 if (spl_taskq_thread_priority) 1026 set_user_nice(tqt->tqt_thread, PRIO_TO_NICE(tq->tq_pri)); 1027 1028 wake_up_process(tqt->tqt_thread); 1029 1030 return (tqt); 1031 } 1032 1033 taskq_t * 1034 taskq_create(const char *name, int threads_arg, pri_t pri, 1035 int minalloc, int maxalloc, uint_t flags) 1036 { 1037 taskq_t *tq; 1038 taskq_thread_t *tqt; 1039 int count = 0, rc = 0, i; 1040 unsigned long irqflags; 1041 int nthreads = threads_arg; 1042 1043 ASSERT(name != NULL); 1044 ASSERT(minalloc >= 0); 1045 ASSERT(maxalloc <= INT_MAX); 1046 ASSERT(!(flags & (TASKQ_CPR_SAFE))); /* Unsupported */ 1047 1048 /* Scale the number of threads using nthreads as a percentage */ 1049 if (flags & TASKQ_THREADS_CPU_PCT) { 1050 ASSERT(nthreads <= 100); 1051 ASSERT(nthreads >= 0); 1052 nthreads = MIN(threads_arg, 100); 1053 nthreads = MAX(nthreads, 0); 1054 nthreads = MAX((num_online_cpus() * nthreads) /100, 1); 1055 } 1056 1057 tq = kmem_alloc(sizeof (*tq), KM_PUSHPAGE); 1058 if (tq == NULL) 1059 return (NULL); 1060 1061 tq->tq_hp_support = B_FALSE; 1062 #ifdef HAVE_CPU_HOTPLUG 1063 if (flags & TASKQ_THREADS_CPU_PCT) { 1064 tq->tq_hp_support = B_TRUE; 1065 if (cpuhp_state_add_instance_nocalls(spl_taskq_cpuhp_state, 1066 &tq->tq_hp_cb_node) != 0) { 1067 kmem_free(tq, sizeof (*tq)); 1068 return (NULL); 1069 } 1070 } 1071 #endif 1072 1073 spin_lock_init(&tq->tq_lock); 1074 INIT_LIST_HEAD(&tq->tq_thread_list); 1075 INIT_LIST_HEAD(&tq->tq_active_list); 1076 tq->tq_name = kmem_strdup(name); 1077 tq->tq_nactive = 0; 1078 tq->tq_nthreads = 0; 1079 tq->tq_nspawn = 0; 1080 tq->tq_maxthreads = nthreads; 1081 tq->tq_cpu_pct = threads_arg; 1082 tq->tq_pri = pri; 1083 tq->tq_minalloc = minalloc; 1084 tq->tq_maxalloc = maxalloc; 1085 tq->tq_nalloc = 0; 1086 tq->tq_flags = (flags | TASKQ_ACTIVE); 1087 tq->tq_next_id = TASKQID_INITIAL; 1088 tq->tq_lowest_id = TASKQID_INITIAL; 1089 INIT_LIST_HEAD(&tq->tq_free_list); 1090 INIT_LIST_HEAD(&tq->tq_pend_list); 1091 INIT_LIST_HEAD(&tq->tq_prio_list); 1092 INIT_LIST_HEAD(&tq->tq_delay_list); 1093 init_waitqueue_head(&tq->tq_work_waitq); 1094 init_waitqueue_head(&tq->tq_wait_waitq); 1095 tq->tq_lock_class = TQ_LOCK_GENERAL; 1096 INIT_LIST_HEAD(&tq->tq_taskqs); 1097 1098 if (flags & TASKQ_PREPOPULATE) { 1099 spin_lock_irqsave_nested(&tq->tq_lock, irqflags, 1100 tq->tq_lock_class); 1101 1102 for (i = 0; i < minalloc; i++) 1103 task_done(tq, task_alloc(tq, TQ_PUSHPAGE | TQ_NEW, 1104 &irqflags)); 1105 1106 spin_unlock_irqrestore(&tq->tq_lock, irqflags); 1107 } 1108 1109 if ((flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) 1110 nthreads = 1; 1111 1112 for (i = 0; i < nthreads; i++) { 1113 tqt = taskq_thread_create(tq); 1114 if (tqt == NULL) 1115 rc = 1; 1116 else 1117 count++; 1118 } 1119 1120 /* Wait for all threads to be started before potential destroy */ 1121 wait_event(tq->tq_wait_waitq, tq->tq_nthreads == count); 1122 /* 1123 * taskq_thread might have touched nspawn, but we don't want them to 1124 * because they're not dynamically spawned. So we reset it to 0 1125 */ 1126 tq->tq_nspawn = 0; 1127 1128 if (rc) { 1129 taskq_destroy(tq); 1130 tq = NULL; 1131 } else { 1132 down_write(&tq_list_sem); 1133 tq->tq_instance = taskq_find_by_name(name) + 1; 1134 list_add_tail(&tq->tq_taskqs, &tq_list); 1135 up_write(&tq_list_sem); 1136 } 1137 1138 return (tq); 1139 } 1140 EXPORT_SYMBOL(taskq_create); 1141 1142 void 1143 taskq_destroy(taskq_t *tq) 1144 { 1145 struct task_struct *thread; 1146 taskq_thread_t *tqt; 1147 taskq_ent_t *t; 1148 unsigned long flags; 1149 1150 ASSERT(tq); 1151 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 1152 tq->tq_flags &= ~TASKQ_ACTIVE; 1153 spin_unlock_irqrestore(&tq->tq_lock, flags); 1154 1155 #ifdef HAVE_CPU_HOTPLUG 1156 if (tq->tq_hp_support) { 1157 VERIFY0(cpuhp_state_remove_instance_nocalls( 1158 spl_taskq_cpuhp_state, &tq->tq_hp_cb_node)); 1159 } 1160 #endif 1161 /* 1162 * When TASKQ_ACTIVE is clear new tasks may not be added nor may 1163 * new worker threads be spawned for dynamic taskq. 1164 */ 1165 if (dynamic_taskq != NULL) 1166 taskq_wait_outstanding(dynamic_taskq, 0); 1167 1168 taskq_wait(tq); 1169 1170 /* remove taskq from global list used by the kstats */ 1171 down_write(&tq_list_sem); 1172 list_del(&tq->tq_taskqs); 1173 up_write(&tq_list_sem); 1174 1175 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 1176 /* wait for spawning threads to insert themselves to the list */ 1177 while (tq->tq_nspawn) { 1178 spin_unlock_irqrestore(&tq->tq_lock, flags); 1179 schedule_timeout_interruptible(1); 1180 spin_lock_irqsave_nested(&tq->tq_lock, flags, 1181 tq->tq_lock_class); 1182 } 1183 1184 /* 1185 * Signal each thread to exit and block until it does. Each thread 1186 * is responsible for removing itself from the list and freeing its 1187 * taskq_thread_t. This allows for idle threads to opt to remove 1188 * themselves from the taskq. They can be recreated as needed. 1189 */ 1190 while (!list_empty(&tq->tq_thread_list)) { 1191 tqt = list_entry(tq->tq_thread_list.next, 1192 taskq_thread_t, tqt_thread_list); 1193 thread = tqt->tqt_thread; 1194 spin_unlock_irqrestore(&tq->tq_lock, flags); 1195 1196 kthread_stop(thread); 1197 1198 spin_lock_irqsave_nested(&tq->tq_lock, flags, 1199 tq->tq_lock_class); 1200 } 1201 1202 while (!list_empty(&tq->tq_free_list)) { 1203 t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list); 1204 1205 ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); 1206 1207 list_del_init(&t->tqent_list); 1208 task_free(tq, t); 1209 } 1210 1211 ASSERT0(tq->tq_nthreads); 1212 ASSERT0(tq->tq_nalloc); 1213 ASSERT0(tq->tq_nspawn); 1214 ASSERT(list_empty(&tq->tq_thread_list)); 1215 ASSERT(list_empty(&tq->tq_active_list)); 1216 ASSERT(list_empty(&tq->tq_free_list)); 1217 ASSERT(list_empty(&tq->tq_pend_list)); 1218 ASSERT(list_empty(&tq->tq_prio_list)); 1219 ASSERT(list_empty(&tq->tq_delay_list)); 1220 1221 spin_unlock_irqrestore(&tq->tq_lock, flags); 1222 1223 kmem_strfree(tq->tq_name); 1224 kmem_free(tq, sizeof (taskq_t)); 1225 } 1226 EXPORT_SYMBOL(taskq_destroy); 1227 1228 static unsigned int spl_taskq_kick = 0; 1229 1230 /* 1231 * 2.6.36 API Change 1232 * module_param_cb is introduced to take kernel_param_ops and 1233 * module_param_call is marked as obsolete. Also set and get operations 1234 * were changed to take a 'const struct kernel_param *'. 1235 */ 1236 static int 1237 #ifdef module_param_cb 1238 param_set_taskq_kick(const char *val, const struct kernel_param *kp) 1239 #else 1240 param_set_taskq_kick(const char *val, struct kernel_param *kp) 1241 #endif 1242 { 1243 int ret; 1244 taskq_t *tq = NULL; 1245 taskq_ent_t *t; 1246 unsigned long flags; 1247 1248 ret = param_set_uint(val, kp); 1249 if (ret < 0 || !spl_taskq_kick) 1250 return (ret); 1251 /* reset value */ 1252 spl_taskq_kick = 0; 1253 1254 down_read(&tq_list_sem); 1255 list_for_each_entry(tq, &tq_list, tq_taskqs) { 1256 spin_lock_irqsave_nested(&tq->tq_lock, flags, 1257 tq->tq_lock_class); 1258 /* Check if the first pending is older than 5 seconds */ 1259 t = taskq_next_ent(tq); 1260 if (t && time_after(jiffies, t->tqent_birth + 5*HZ)) { 1261 (void) taskq_thread_spawn(tq); 1262 printk(KERN_INFO "spl: Kicked taskq %s/%d\n", 1263 tq->tq_name, tq->tq_instance); 1264 } 1265 spin_unlock_irqrestore(&tq->tq_lock, flags); 1266 } 1267 up_read(&tq_list_sem); 1268 return (ret); 1269 } 1270 1271 #ifdef module_param_cb 1272 static const struct kernel_param_ops param_ops_taskq_kick = { 1273 .set = param_set_taskq_kick, 1274 .get = param_get_uint, 1275 }; 1276 module_param_cb(spl_taskq_kick, ¶m_ops_taskq_kick, &spl_taskq_kick, 0644); 1277 #else 1278 module_param_call(spl_taskq_kick, param_set_taskq_kick, param_get_uint, 1279 &spl_taskq_kick, 0644); 1280 #endif 1281 MODULE_PARM_DESC(spl_taskq_kick, 1282 "Write nonzero to kick stuck taskqs to spawn more threads"); 1283 1284 #ifdef HAVE_CPU_HOTPLUG 1285 /* 1286 * This callback will be called exactly once for each core that comes online, 1287 * for each dynamic taskq. We attempt to expand taskqs that have 1288 * TASKQ_THREADS_CPU_PCT set. We need to redo the percentage calculation every 1289 * time, to correctly determine whether or not to add a thread. 1290 */ 1291 static int 1292 spl_taskq_expand(unsigned int cpu, struct hlist_node *node) 1293 { 1294 taskq_t *tq = list_entry(node, taskq_t, tq_hp_cb_node); 1295 unsigned long flags; 1296 int err = 0; 1297 1298 ASSERT(tq); 1299 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 1300 1301 if (!(tq->tq_flags & TASKQ_ACTIVE)) { 1302 spin_unlock_irqrestore(&tq->tq_lock, flags); 1303 return (err); 1304 } 1305 1306 ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT); 1307 int nthreads = MIN(tq->tq_cpu_pct, 100); 1308 nthreads = MAX(((num_online_cpus() + 1) * nthreads) / 100, 1); 1309 tq->tq_maxthreads = nthreads; 1310 1311 if (!((tq->tq_flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) && 1312 tq->tq_maxthreads > tq->tq_nthreads) { 1313 spin_unlock_irqrestore(&tq->tq_lock, flags); 1314 taskq_thread_t *tqt = taskq_thread_create(tq); 1315 if (tqt == NULL) 1316 err = -1; 1317 return (err); 1318 } 1319 spin_unlock_irqrestore(&tq->tq_lock, flags); 1320 return (err); 1321 } 1322 1323 /* 1324 * While we don't support offlining CPUs, it is possible that CPUs will fail 1325 * to online successfully. We do need to be able to handle this case 1326 * gracefully. 1327 */ 1328 static int 1329 spl_taskq_prepare_down(unsigned int cpu, struct hlist_node *node) 1330 { 1331 taskq_t *tq = list_entry(node, taskq_t, tq_hp_cb_node); 1332 unsigned long flags; 1333 1334 ASSERT(tq); 1335 spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); 1336 1337 if (!(tq->tq_flags & TASKQ_ACTIVE)) 1338 goto out; 1339 1340 ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT); 1341 int nthreads = MIN(tq->tq_cpu_pct, 100); 1342 nthreads = MAX(((num_online_cpus()) * nthreads) / 100, 1); 1343 tq->tq_maxthreads = nthreads; 1344 1345 if (!((tq->tq_flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) && 1346 tq->tq_maxthreads < tq->tq_nthreads) { 1347 ASSERT3U(tq->tq_maxthreads, ==, tq->tq_nthreads - 1); 1348 taskq_thread_t *tqt = list_entry(tq->tq_thread_list.next, 1349 taskq_thread_t, tqt_thread_list); 1350 struct task_struct *thread = tqt->tqt_thread; 1351 spin_unlock_irqrestore(&tq->tq_lock, flags); 1352 1353 kthread_stop(thread); 1354 1355 return (0); 1356 } 1357 1358 out: 1359 spin_unlock_irqrestore(&tq->tq_lock, flags); 1360 return (0); 1361 } 1362 #endif 1363 1364 int 1365 spl_taskq_init(void) 1366 { 1367 init_rwsem(&tq_list_sem); 1368 tsd_create(&taskq_tsd, NULL); 1369 1370 #ifdef HAVE_CPU_HOTPLUG 1371 spl_taskq_cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 1372 "fs/spl_taskq:online", spl_taskq_expand, spl_taskq_prepare_down); 1373 #endif 1374 1375 system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64), 1376 maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC); 1377 if (system_taskq == NULL) 1378 return (1); 1379 1380 system_delay_taskq = taskq_create("spl_delay_taskq", MAX(boot_ncpus, 4), 1381 maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC); 1382 if (system_delay_taskq == NULL) { 1383 #ifdef HAVE_CPU_HOTPLUG 1384 cpuhp_remove_multi_state(spl_taskq_cpuhp_state); 1385 #endif 1386 taskq_destroy(system_taskq); 1387 return (1); 1388 } 1389 1390 dynamic_taskq = taskq_create("spl_dynamic_taskq", 1, 1391 maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE); 1392 if (dynamic_taskq == NULL) { 1393 #ifdef HAVE_CPU_HOTPLUG 1394 cpuhp_remove_multi_state(spl_taskq_cpuhp_state); 1395 #endif 1396 taskq_destroy(system_taskq); 1397 taskq_destroy(system_delay_taskq); 1398 return (1); 1399 } 1400 1401 /* 1402 * This is used to annotate tq_lock, so 1403 * taskq_dispatch -> taskq_thread_spawn -> taskq_dispatch 1404 * does not trigger a lockdep warning re: possible recursive locking 1405 */ 1406 dynamic_taskq->tq_lock_class = TQ_LOCK_DYNAMIC; 1407 1408 return (0); 1409 } 1410 1411 void 1412 spl_taskq_fini(void) 1413 { 1414 taskq_destroy(dynamic_taskq); 1415 dynamic_taskq = NULL; 1416 1417 taskq_destroy(system_delay_taskq); 1418 system_delay_taskq = NULL; 1419 1420 taskq_destroy(system_taskq); 1421 system_taskq = NULL; 1422 1423 tsd_destroy(&taskq_tsd); 1424 1425 #ifdef HAVE_CPU_HOTPLUG 1426 cpuhp_remove_multi_state(spl_taskq_cpuhp_state); 1427 spl_taskq_cpuhp_state = 0; 1428 #endif 1429 } 1430