1 /*- 2 * Copyright (c) 2017-2019 Hans Petter Selasky 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 #include <linux/workqueue.h> 29 #include <linux/wait.h> 30 #include <linux/compat.h> 31 #include <linux/spinlock.h> 32 #include <linux/rcupdate.h> 33 #include <linux/irq_work.h> 34 35 #include <sys/kernel.h> 36 37 /* 38 * Define all work struct states 39 */ 40 enum { 41 WORK_ST_IDLE, /* idle - not started */ 42 WORK_ST_TIMER, /* timer is being started */ 43 WORK_ST_TASK, /* taskqueue is being queued */ 44 WORK_ST_EXEC, /* callback is being called */ 45 WORK_ST_CANCEL, /* cancel is being requested */ 46 WORK_ST_MAX, 47 }; 48 49 /* 50 * Define global workqueues 51 */ 52 static struct workqueue_struct *linux_system_short_wq; 53 static struct workqueue_struct *linux_system_long_wq; 54 55 struct workqueue_struct *system_wq; 56 struct workqueue_struct *system_long_wq; 57 struct workqueue_struct *system_unbound_wq; 58 struct workqueue_struct *system_highpri_wq; 59 struct workqueue_struct *system_power_efficient_wq; 60 61 struct taskqueue *linux_irq_work_tq; 62 63 static int linux_default_wq_cpus = 4; 64 65 static void linux_delayed_work_timer_fn(void *); 66 67 /* 68 * This function atomically updates the work state and returns the 69 * previous state at the time of update. 70 */ 71 static uint8_t 72 linux_update_state(atomic_t *v, const uint8_t *pstate) 73 { 74 int c, old; 75 76 c = v->counter; 77 78 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c) 79 c = old; 80 81 return (c); 82 } 83 84 /* 85 * A LinuxKPI task is allowed to free itself inside the callback function 86 * and cannot safely be referred after the callback function has 87 * completed. This function gives the linux_work_fn() function a hint, 88 * that the task is not going away and can have its state checked 89 * again. Without this extra hint LinuxKPI tasks cannot be serialized 90 * across multiple worker threads. 91 */ 92 static bool 93 linux_work_exec_unblock(struct work_struct *work) 94 { 95 struct workqueue_struct *wq; 96 struct work_exec *exec; 97 bool retval = false; 98 99 wq = work->work_queue; 100 if (unlikely(wq == NULL)) 101 goto done; 102 103 WQ_EXEC_LOCK(wq); 104 TAILQ_FOREACH(exec, &wq->exec_head, entry) { 105 if (exec->target == work) { 106 exec->target = NULL; 107 retval = true; 108 break; 109 } 110 } 111 WQ_EXEC_UNLOCK(wq); 112 done: 113 return (retval); 114 } 115 116 static void 117 linux_delayed_work_enqueue(struct delayed_work *dwork) 118 { 119 struct taskqueue *tq; 120 121 tq = dwork->work.work_queue->taskqueue; 122 taskqueue_enqueue(tq, &dwork->work.work_task); 123 } 124 125 /* 126 * This function queues the given work structure on the given 127 * workqueue. It returns non-zero if the work was successfully 128 * [re-]queued. Else the work is already pending for completion. 129 */ 130 bool 131 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq, 132 struct work_struct *work) 133 { 134 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 135 [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */ 136 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */ 137 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ 138 [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */ 139 [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */ 140 }; 141 142 if (atomic_read(&wq->draining) != 0) 143 return (!work_pending(work)); 144 145 switch (linux_update_state(&work->state, states)) { 146 case WORK_ST_EXEC: 147 case WORK_ST_CANCEL: 148 if (linux_work_exec_unblock(work) != 0) 149 return (true); 150 /* FALLTHROUGH */ 151 case WORK_ST_IDLE: 152 work->work_queue = wq; 153 taskqueue_enqueue(wq->taskqueue, &work->work_task); 154 return (true); 155 default: 156 return (false); /* already on a queue */ 157 } 158 } 159 160 /* 161 * Callback func for linux_queue_rcu_work 162 */ 163 static void 164 rcu_work_func(struct rcu_head *rcu) 165 { 166 struct rcu_work *rwork; 167 168 rwork = container_of(rcu, struct rcu_work, rcu); 169 linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); 170 } 171 172 /* 173 * This function queue a work after a grace period 174 * If the work was already pending it returns false, 175 * if not it calls call_rcu and returns true. 176 */ 177 bool 178 linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) 179 { 180 181 if (!linux_work_pending(&rwork->work)) { 182 rwork->wq = wq; 183 linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func); 184 return (true); 185 } 186 return (false); 187 } 188 189 /* 190 * This function waits for the last execution of a work and then 191 * flush the work. 192 * It returns true if the work was pending and we waited, it returns 193 * false otherwise. 194 */ 195 bool 196 linux_flush_rcu_work(struct rcu_work *rwork) 197 { 198 199 if (linux_work_pending(&rwork->work)) { 200 linux_rcu_barrier(RCU_TYPE_REGULAR); 201 linux_flush_work(&rwork->work); 202 return (true); 203 } 204 return (linux_flush_work(&rwork->work)); 205 } 206 207 /* 208 * This function queues the given work structure on the given 209 * workqueue after a given delay in ticks. It returns true if the 210 * work was successfully [re-]queued. Else the work is already pending 211 * for completion. 212 */ 213 bool 214 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq, 215 struct delayed_work *dwork, unsigned delay) 216 { 217 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 218 [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */ 219 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */ 220 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ 221 [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */ 222 [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */ 223 }; 224 bool res; 225 226 if (atomic_read(&wq->draining) != 0) 227 return (!work_pending(&dwork->work)); 228 229 mtx_lock(&dwork->timer.mtx); 230 switch (linux_update_state(&dwork->work.state, states)) { 231 case WORK_ST_EXEC: 232 case WORK_ST_CANCEL: 233 if (delay == 0 && linux_work_exec_unblock(&dwork->work)) { 234 dwork->timer.expires = jiffies; 235 res = true; 236 goto out; 237 } 238 /* FALLTHROUGH */ 239 case WORK_ST_IDLE: 240 dwork->work.work_queue = wq; 241 dwork->timer.expires = jiffies + delay; 242 243 if (delay == 0) { 244 linux_delayed_work_enqueue(dwork); 245 } else if (unlikely(cpu != WORK_CPU_UNBOUND)) { 246 callout_reset_on(&dwork->timer.callout, delay, 247 &linux_delayed_work_timer_fn, dwork, cpu); 248 } else { 249 callout_reset(&dwork->timer.callout, delay, 250 &linux_delayed_work_timer_fn, dwork); 251 } 252 res = true; 253 break; 254 default: 255 res = false; 256 break; 257 } 258 out: 259 mtx_unlock(&dwork->timer.mtx); 260 return (res); 261 } 262 263 void 264 linux_work_fn(void *context, int pending) 265 { 266 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 267 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 268 [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */ 269 [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */ 270 [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */ 271 [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */ 272 }; 273 struct work_struct *work; 274 struct workqueue_struct *wq; 275 struct work_exec exec; 276 struct task_struct *task; 277 278 task = current; 279 280 /* setup local variables */ 281 work = context; 282 wq = work->work_queue; 283 284 /* store target pointer */ 285 exec.target = work; 286 287 /* insert executor into list */ 288 WQ_EXEC_LOCK(wq); 289 TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry); 290 while (1) { 291 switch (linux_update_state(&work->state, states)) { 292 case WORK_ST_TIMER: 293 case WORK_ST_TASK: 294 case WORK_ST_CANCEL: 295 WQ_EXEC_UNLOCK(wq); 296 297 /* set current work structure */ 298 task->work = work; 299 300 /* call work function */ 301 work->func(work); 302 303 /* set current work structure */ 304 task->work = NULL; 305 306 WQ_EXEC_LOCK(wq); 307 /* check if unblocked */ 308 if (exec.target != work) { 309 /* reapply block */ 310 exec.target = work; 311 break; 312 } 313 /* FALLTHROUGH */ 314 default: 315 goto done; 316 } 317 } 318 done: 319 /* remove executor from list */ 320 TAILQ_REMOVE(&wq->exec_head, &exec, entry); 321 WQ_EXEC_UNLOCK(wq); 322 } 323 324 void 325 linux_delayed_work_fn(void *context, int pending) 326 { 327 struct delayed_work *dwork = context; 328 329 /* 330 * Make sure the timer belonging to the delayed work gets 331 * drained before invoking the work function. Else the timer 332 * mutex may still be in use which can lead to use-after-free 333 * situations, because the work function might free the work 334 * structure before returning. 335 */ 336 callout_drain(&dwork->timer.callout); 337 338 linux_work_fn(&dwork->work, pending); 339 } 340 341 static void 342 linux_delayed_work_timer_fn(void *arg) 343 { 344 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 345 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 346 [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */ 347 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ 348 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */ 349 [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */ 350 }; 351 struct delayed_work *dwork = arg; 352 353 switch (linux_update_state(&dwork->work.state, states)) { 354 case WORK_ST_TIMER: 355 case WORK_ST_CANCEL: 356 linux_delayed_work_enqueue(dwork); 357 break; 358 default: 359 break; 360 } 361 } 362 363 /* 364 * This function cancels the given work structure in a 365 * non-blocking fashion. It returns non-zero if the work was 366 * successfully cancelled. Else the work may still be busy or already 367 * cancelled. 368 */ 369 bool 370 linux_cancel_work(struct work_struct *work) 371 { 372 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 373 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 374 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */ 375 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel */ 376 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */ 377 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* can't happen */ 378 }; 379 struct taskqueue *tq; 380 381 MPASS(atomic_read(&work->state) != WORK_ST_TIMER); 382 MPASS(atomic_read(&work->state) != WORK_ST_CANCEL); 383 384 switch (linux_update_state(&work->state, states)) { 385 case WORK_ST_TASK: 386 tq = work->work_queue->taskqueue; 387 if (taskqueue_cancel(tq, &work->work_task, NULL) == 0) 388 return (true); 389 /* FALLTHROUGH */ 390 default: 391 return (false); 392 } 393 } 394 395 /* 396 * This function cancels the given work structure in a synchronous 397 * fashion. It returns non-zero if the work was successfully 398 * cancelled. Else the work was already cancelled. 399 */ 400 bool 401 linux_cancel_work_sync(struct work_struct *work) 402 { 403 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 404 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 405 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */ 406 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */ 407 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */ 408 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */ 409 }; 410 struct taskqueue *tq; 411 bool retval = false; 412 413 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 414 "linux_cancel_work_sync() might sleep"); 415 retry: 416 switch (linux_update_state(&work->state, states)) { 417 case WORK_ST_IDLE: 418 case WORK_ST_TIMER: 419 return (retval); 420 case WORK_ST_EXEC: 421 tq = work->work_queue->taskqueue; 422 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0) 423 taskqueue_drain(tq, &work->work_task); 424 goto retry; /* work may have restarted itself */ 425 default: 426 tq = work->work_queue->taskqueue; 427 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0) 428 taskqueue_drain(tq, &work->work_task); 429 retval = true; 430 goto retry; 431 } 432 } 433 434 /* 435 * This function atomically stops the timer and callback. The timer 436 * callback will not be called after this function returns. This 437 * functions returns true when the timeout was cancelled. Else the 438 * timeout was not started or has already been called. 439 */ 440 static inline bool 441 linux_cancel_timer(struct delayed_work *dwork, bool drain) 442 { 443 bool cancelled; 444 445 mtx_lock(&dwork->timer.mtx); 446 cancelled = (callout_stop(&dwork->timer.callout) == 1); 447 mtx_unlock(&dwork->timer.mtx); 448 449 /* check if we should drain */ 450 if (drain) 451 callout_drain(&dwork->timer.callout); 452 return (cancelled); 453 } 454 455 /* 456 * This function cancels the given delayed work structure in a 457 * non-blocking fashion. It returns non-zero if the work was 458 * successfully cancelled. Else the work may still be busy or already 459 * cancelled. 460 */ 461 bool 462 linux_cancel_delayed_work(struct delayed_work *dwork) 463 { 464 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 465 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 466 [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */ 467 [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */ 468 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */ 469 [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */ 470 }; 471 struct taskqueue *tq; 472 bool cancelled; 473 474 mtx_lock(&dwork->timer.mtx); 475 switch (linux_update_state(&dwork->work.state, states)) { 476 case WORK_ST_TIMER: 477 case WORK_ST_CANCEL: 478 cancelled = (callout_stop(&dwork->timer.callout) == 1); 479 if (cancelled) { 480 atomic_cmpxchg(&dwork->work.state, 481 WORK_ST_CANCEL, WORK_ST_IDLE); 482 mtx_unlock(&dwork->timer.mtx); 483 return (true); 484 } 485 /* FALLTHROUGH */ 486 case WORK_ST_TASK: 487 tq = dwork->work.work_queue->taskqueue; 488 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) { 489 atomic_cmpxchg(&dwork->work.state, 490 WORK_ST_CANCEL, WORK_ST_IDLE); 491 mtx_unlock(&dwork->timer.mtx); 492 return (true); 493 } 494 /* FALLTHROUGH */ 495 default: 496 mtx_unlock(&dwork->timer.mtx); 497 return (false); 498 } 499 } 500 501 /* 502 * This function cancels the given work structure in a synchronous 503 * fashion. It returns true if the work was successfully 504 * cancelled. Else the work was already cancelled. 505 */ 506 static bool 507 linux_cancel_delayed_work_sync_int(struct delayed_work *dwork) 508 { 509 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 510 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 511 [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */ 512 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */ 513 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */ 514 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */ 515 }; 516 struct taskqueue *tq; 517 int ret, state; 518 bool cancelled; 519 520 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 521 "linux_cancel_delayed_work_sync() might sleep"); 522 mtx_lock(&dwork->timer.mtx); 523 524 state = linux_update_state(&dwork->work.state, states); 525 switch (state) { 526 case WORK_ST_IDLE: 527 mtx_unlock(&dwork->timer.mtx); 528 return (false); 529 case WORK_ST_TIMER: 530 case WORK_ST_CANCEL: 531 cancelled = (callout_stop(&dwork->timer.callout) == 1); 532 533 tq = dwork->work.work_queue->taskqueue; 534 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL); 535 mtx_unlock(&dwork->timer.mtx); 536 537 callout_drain(&dwork->timer.callout); 538 taskqueue_drain(tq, &dwork->work.work_task); 539 return (cancelled || (ret != 0)); 540 default: 541 tq = dwork->work.work_queue->taskqueue; 542 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL); 543 mtx_unlock(&dwork->timer.mtx); 544 if (ret != 0) 545 taskqueue_drain(tq, &dwork->work.work_task); 546 return (ret != 0); 547 } 548 } 549 550 bool 551 linux_cancel_delayed_work_sync(struct delayed_work *dwork) 552 { 553 bool res; 554 555 res = false; 556 while (linux_cancel_delayed_work_sync_int(dwork)) 557 res = true; 558 return (res); 559 } 560 561 /* 562 * This function waits until the given work structure is completed. 563 * It returns non-zero if the work was successfully 564 * waited for. Else the work was not waited for. 565 */ 566 bool 567 linux_flush_work(struct work_struct *work) 568 { 569 struct taskqueue *tq; 570 bool retval; 571 572 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 573 "linux_flush_work() might sleep"); 574 575 switch (atomic_read(&work->state)) { 576 case WORK_ST_IDLE: 577 return (false); 578 default: 579 tq = work->work_queue->taskqueue; 580 retval = taskqueue_poll_is_busy(tq, &work->work_task); 581 taskqueue_drain(tq, &work->work_task); 582 return (retval); 583 } 584 } 585 586 /* 587 * This function waits until the given delayed work structure is 588 * completed. It returns non-zero if the work was successfully waited 589 * for. Else the work was not waited for. 590 */ 591 bool 592 linux_flush_delayed_work(struct delayed_work *dwork) 593 { 594 struct taskqueue *tq; 595 bool retval; 596 597 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 598 "linux_flush_delayed_work() might sleep"); 599 600 switch (atomic_read(&dwork->work.state)) { 601 case WORK_ST_IDLE: 602 return (false); 603 case WORK_ST_TIMER: 604 if (linux_cancel_timer(dwork, 1)) 605 linux_delayed_work_enqueue(dwork); 606 /* FALLTHROUGH */ 607 default: 608 tq = dwork->work.work_queue->taskqueue; 609 retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task); 610 taskqueue_drain(tq, &dwork->work.work_task); 611 return (retval); 612 } 613 } 614 615 /* 616 * This function returns true if the given work is pending, and not 617 * yet executing: 618 */ 619 bool 620 linux_work_pending(struct work_struct *work) 621 { 622 switch (atomic_read(&work->state)) { 623 case WORK_ST_TIMER: 624 case WORK_ST_TASK: 625 case WORK_ST_CANCEL: 626 return (true); 627 default: 628 return (false); 629 } 630 } 631 632 /* 633 * This function returns true if the given work is busy. 634 */ 635 bool 636 linux_work_busy(struct work_struct *work) 637 { 638 struct taskqueue *tq; 639 640 switch (atomic_read(&work->state)) { 641 case WORK_ST_IDLE: 642 return (false); 643 case WORK_ST_EXEC: 644 tq = work->work_queue->taskqueue; 645 return (taskqueue_poll_is_busy(tq, &work->work_task)); 646 default: 647 return (true); 648 } 649 } 650 651 struct workqueue_struct * 652 linux_create_workqueue_common(const char *name, int cpus) 653 { 654 struct workqueue_struct *wq; 655 656 /* 657 * If zero CPUs are specified use the default number of CPUs: 658 */ 659 if (cpus == 0) 660 cpus = linux_default_wq_cpus; 661 662 wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO); 663 wq->taskqueue = taskqueue_create(name, M_WAITOK, 664 taskqueue_thread_enqueue, &wq->taskqueue); 665 atomic_set(&wq->draining, 0); 666 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); 667 TAILQ_INIT(&wq->exec_head); 668 mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF); 669 670 return (wq); 671 } 672 673 void 674 linux_destroy_workqueue(struct workqueue_struct *wq) 675 { 676 atomic_inc(&wq->draining); 677 drain_workqueue(wq); 678 taskqueue_free(wq->taskqueue); 679 mtx_destroy(&wq->exec_mtx); 680 kfree(wq); 681 } 682 683 void 684 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func) 685 { 686 memset(dwork, 0, sizeof(*dwork)); 687 dwork->work.func = func; 688 TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork); 689 mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL, 690 MTX_DEF | MTX_NOWITNESS); 691 callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0); 692 } 693 694 struct work_struct * 695 linux_current_work(void) 696 { 697 return (current->work); 698 } 699 700 static void 701 linux_work_init(void *arg) 702 { 703 int max_wq_cpus = mp_ncpus + 1; 704 705 /* avoid deadlock when there are too few threads */ 706 if (max_wq_cpus < 4) 707 max_wq_cpus = 4; 708 709 /* set default number of CPUs */ 710 linux_default_wq_cpus = max_wq_cpus; 711 712 linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus); 713 linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus); 714 715 /* populate the workqueue pointers */ 716 system_long_wq = linux_system_long_wq; 717 system_wq = linux_system_short_wq; 718 system_power_efficient_wq = linux_system_short_wq; 719 system_unbound_wq = linux_system_short_wq; 720 system_highpri_wq = linux_system_short_wq; 721 } 722 SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL); 723 724 static void 725 linux_work_uninit(void *arg) 726 { 727 destroy_workqueue(linux_system_short_wq); 728 destroy_workqueue(linux_system_long_wq); 729 730 /* clear workqueue pointers */ 731 system_long_wq = NULL; 732 system_wq = NULL; 733 system_power_efficient_wq = NULL; 734 system_unbound_wq = NULL; 735 system_highpri_wq = NULL; 736 } 737 SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL); 738 739 void 740 linux_irq_work_fn(void *context, int pending) 741 { 742 struct irq_work *irqw = context; 743 744 irqw->func(irqw); 745 } 746 747 static void 748 linux_irq_work_init_fn(void *context, int pending) 749 { 750 /* 751 * LinuxKPI performs lazy allocation of memory structures required by 752 * current on the first access to it. As some irq_work clients read 753 * it with spinlock taken, we have to preallocate td_lkpi_task before 754 * first call to irq_work_queue(). As irq_work uses a single thread, 755 * it is enough to read current once at SYSINIT stage. 756 */ 757 if (current == NULL) 758 panic("irq_work taskqueue is not initialized"); 759 } 760 static struct task linux_irq_work_init_task = 761 TASK_INITIALIZER(0, linux_irq_work_init_fn, &linux_irq_work_init_task); 762 763 static void 764 linux_irq_work_init(void *arg) 765 { 766 linux_irq_work_tq = taskqueue_create_fast("linuxkpi_irq_wq", 767 M_WAITOK, taskqueue_thread_enqueue, &linux_irq_work_tq); 768 taskqueue_start_threads(&linux_irq_work_tq, 1, PWAIT, 769 "linuxkpi_irq_wq"); 770 taskqueue_enqueue(linux_irq_work_tq, &linux_irq_work_init_task); 771 } 772 SYSINIT(linux_irq_work_init, SI_SUB_TASKQ, SI_ORDER_SECOND, 773 linux_irq_work_init, NULL); 774 775 static void 776 linux_irq_work_uninit(void *arg) 777 { 778 taskqueue_drain_all(linux_irq_work_tq); 779 taskqueue_free(linux_irq_work_tq); 780 } 781 SYSUNINIT(linux_irq_work_uninit, SI_SUB_TASKQ, SI_ORDER_SECOND, 782 linux_irq_work_uninit, NULL); 783