1 /*-
2 * Copyright (c) 2017-2019 Hans Petter Selasky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #include <linux/workqueue.h>
29 #include <linux/wait.h>
30 #include <linux/compat.h>
31 #include <linux/spinlock.h>
32 #include <linux/rcupdate.h>
33 #include <linux/irq_work.h>
34
35 #include <sys/kernel.h>
36
37 /*
38 * Define all work struct states
39 */
40 enum {
41 WORK_ST_IDLE, /* idle - not started */
42 WORK_ST_TIMER, /* timer is being started */
43 WORK_ST_TASK, /* taskqueue is being queued */
44 WORK_ST_EXEC, /* callback is being called */
45 WORK_ST_CANCEL, /* cancel is being requested */
46 WORK_ST_MAX,
47 };
48
49 /*
50 * Define global workqueues
51 */
52 static struct workqueue_struct *linux_system_short_wq;
53 static struct workqueue_struct *linux_system_long_wq;
54
55 struct workqueue_struct *system_wq;
56 struct workqueue_struct *system_long_wq;
57 struct workqueue_struct *system_unbound_wq;
58 struct workqueue_struct *system_highpri_wq;
59 struct workqueue_struct *system_power_efficient_wq;
60
61 struct taskqueue *linux_irq_work_tq;
62
63 static int linux_default_wq_cpus = 4;
64
65 static void linux_delayed_work_timer_fn(void *);
66
67 /*
68 * This function atomically updates the work state and returns the
69 * previous state at the time of update.
70 */
71 static uint8_t
linux_update_state(atomic_t * v,const uint8_t * pstate)72 linux_update_state(atomic_t *v, const uint8_t *pstate)
73 {
74 int c, old;
75
76 c = v->counter;
77
78 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c)
79 c = old;
80
81 return (c);
82 }
83
84 /*
85 * A LinuxKPI task is allowed to free itself inside the callback function
86 * and cannot safely be referred after the callback function has
87 * completed. This function gives the linux_work_fn() function a hint,
88 * that the task is not going away and can have its state checked
89 * again. Without this extra hint LinuxKPI tasks cannot be serialized
90 * across multiple worker threads.
91 */
92 static bool
linux_work_exec_unblock(struct work_struct * work)93 linux_work_exec_unblock(struct work_struct *work)
94 {
95 struct workqueue_struct *wq;
96 struct work_exec *exec;
97 bool retval = false;
98
99 wq = work->work_queue;
100 if (unlikely(wq == NULL))
101 goto done;
102
103 WQ_EXEC_LOCK(wq);
104 TAILQ_FOREACH(exec, &wq->exec_head, entry) {
105 if (exec->target == work) {
106 exec->target = NULL;
107 retval = true;
108 break;
109 }
110 }
111 WQ_EXEC_UNLOCK(wq);
112 done:
113 return (retval);
114 }
115
116 static void
linux_delayed_work_enqueue(struct delayed_work * dwork)117 linux_delayed_work_enqueue(struct delayed_work *dwork)
118 {
119 struct taskqueue *tq;
120
121 tq = dwork->work.work_queue->taskqueue;
122 taskqueue_enqueue(tq, &dwork->work.work_task);
123 }
124
125 /*
126 * This function queues the given work structure on the given
127 * workqueue. It returns non-zero if the work was successfully
128 * [re-]queued. Else the work is already pending for completion.
129 */
130 bool
linux_queue_work_on(int cpu __unused,struct workqueue_struct * wq,struct work_struct * work)131 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq,
132 struct work_struct *work)
133 {
134 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
135 [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */
136 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
137 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
138 [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */
139 [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */
140 };
141
142 if (atomic_read(&wq->draining) != 0)
143 return (!work_pending(work));
144
145 switch (linux_update_state(&work->state, states)) {
146 case WORK_ST_EXEC:
147 case WORK_ST_CANCEL:
148 if (linux_work_exec_unblock(work) != 0)
149 return (true);
150 /* FALLTHROUGH */
151 case WORK_ST_IDLE:
152 work->work_queue = wq;
153 taskqueue_enqueue(wq->taskqueue, &work->work_task);
154 return (true);
155 default:
156 return (false); /* already on a queue */
157 }
158 }
159
160 /*
161 * Callback func for linux_queue_rcu_work
162 */
163 static void
rcu_work_func(struct rcu_head * rcu)164 rcu_work_func(struct rcu_head *rcu)
165 {
166 struct rcu_work *rwork;
167
168 rwork = container_of(rcu, struct rcu_work, rcu);
169 linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
170 }
171
172 /*
173 * This function queue a work after a grace period
174 * If the work was already pending it returns false,
175 * if not it calls call_rcu and returns true.
176 */
177 bool
linux_queue_rcu_work(struct workqueue_struct * wq,struct rcu_work * rwork)178 linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
179 {
180
181 if (!linux_work_pending(&rwork->work)) {
182 rwork->wq = wq;
183 linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func);
184 return (true);
185 }
186 return (false);
187 }
188
189 /*
190 * This function waits for the last execution of a work and then
191 * flush the work.
192 * It returns true if the work was pending and we waited, it returns
193 * false otherwise.
194 */
195 bool
linux_flush_rcu_work(struct rcu_work * rwork)196 linux_flush_rcu_work(struct rcu_work *rwork)
197 {
198
199 if (linux_work_pending(&rwork->work)) {
200 linux_rcu_barrier(RCU_TYPE_REGULAR);
201 linux_flush_work(&rwork->work);
202 return (true);
203 }
204 return (linux_flush_work(&rwork->work));
205 }
206
207 /*
208 * This function queues the given work structure on the given
209 * workqueue after a given delay in ticks. It returns true if the
210 * work was successfully [re-]queued. Else the work is already pending
211 * for completion.
212 */
213 bool
linux_queue_delayed_work_on(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned long delay)214 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
215 struct delayed_work *dwork, unsigned long delay)
216 {
217 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
218 [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */
219 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
220 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
221 [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */
222 [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */
223 };
224 bool res;
225
226 if (atomic_read(&wq->draining) != 0)
227 return (!work_pending(&dwork->work));
228
229 /*
230 * Clamp the delay to a valid ticks value, some consumers pass
231 * MAX_SCHEDULE_TIMEOUT.
232 */
233 if (delay > INT_MAX)
234 delay = INT_MAX;
235
236 mtx_lock(&dwork->timer.mtx);
237 switch (linux_update_state(&dwork->work.state, states)) {
238 case WORK_ST_EXEC:
239 case WORK_ST_CANCEL:
240 if (delay == 0 && linux_work_exec_unblock(&dwork->work)) {
241 dwork->timer.expires = jiffies;
242 res = true;
243 goto out;
244 }
245 /* FALLTHROUGH */
246 case WORK_ST_IDLE:
247 dwork->work.work_queue = wq;
248 dwork->timer.expires = jiffies + delay;
249
250 if (delay == 0) {
251 linux_delayed_work_enqueue(dwork);
252 } else if (unlikely(cpu != WORK_CPU_UNBOUND)) {
253 callout_reset_on(&dwork->timer.callout, delay,
254 &linux_delayed_work_timer_fn, dwork, cpu);
255 } else {
256 callout_reset(&dwork->timer.callout, delay,
257 &linux_delayed_work_timer_fn, dwork);
258 }
259 res = true;
260 break;
261 default:
262 res = false;
263 break;
264 }
265 out:
266 mtx_unlock(&dwork->timer.mtx);
267 return (res);
268 }
269
270 void
linux_work_fn(void * context,int pending)271 linux_work_fn(void *context, int pending)
272 {
273 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
274 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
275 [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */
276 [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */
277 [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */
278 [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */
279 };
280 struct work_struct *work;
281 struct workqueue_struct *wq;
282 struct work_exec exec;
283 struct task_struct *task;
284
285 task = current;
286
287 /* setup local variables */
288 work = context;
289 wq = work->work_queue;
290
291 /* store target pointer */
292 exec.target = work;
293
294 /* insert executor into list */
295 WQ_EXEC_LOCK(wq);
296 TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry);
297 while (1) {
298 switch (linux_update_state(&work->state, states)) {
299 case WORK_ST_TIMER:
300 case WORK_ST_TASK:
301 case WORK_ST_CANCEL:
302 WQ_EXEC_UNLOCK(wq);
303
304 /* set current work structure */
305 task->work = work;
306
307 /* call work function */
308 work->func(work);
309
310 /* set current work structure */
311 task->work = NULL;
312
313 WQ_EXEC_LOCK(wq);
314 /* check if unblocked */
315 if (exec.target != work) {
316 /* reapply block */
317 exec.target = work;
318 break;
319 }
320 /* FALLTHROUGH */
321 default:
322 goto done;
323 }
324 }
325 done:
326 /* remove executor from list */
327 TAILQ_REMOVE(&wq->exec_head, &exec, entry);
328 WQ_EXEC_UNLOCK(wq);
329 }
330
331 void
linux_delayed_work_fn(void * context,int pending)332 linux_delayed_work_fn(void *context, int pending)
333 {
334 struct delayed_work *dwork = context;
335
336 /*
337 * Make sure the timer belonging to the delayed work gets
338 * drained before invoking the work function. Else the timer
339 * mutex may still be in use which can lead to use-after-free
340 * situations, because the work function might free the work
341 * structure before returning.
342 */
343 callout_drain(&dwork->timer.callout);
344
345 linux_work_fn(&dwork->work, pending);
346 }
347
348 static void
linux_delayed_work_timer_fn(void * arg)349 linux_delayed_work_timer_fn(void *arg)
350 {
351 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
352 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
353 [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */
354 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
355 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
356 [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */
357 };
358 struct delayed_work *dwork = arg;
359
360 switch (linux_update_state(&dwork->work.state, states)) {
361 case WORK_ST_TIMER:
362 case WORK_ST_CANCEL:
363 linux_delayed_work_enqueue(dwork);
364 break;
365 default:
366 break;
367 }
368 }
369
370 /*
371 * This function cancels the given work structure in a
372 * non-blocking fashion. It returns non-zero if the work was
373 * successfully cancelled. Else the work may still be busy or already
374 * cancelled.
375 */
376 bool
linux_cancel_work(struct work_struct * work)377 linux_cancel_work(struct work_struct *work)
378 {
379 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
380 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
381 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
382 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel */
383 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
384 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* can't happen */
385 };
386 struct taskqueue *tq;
387
388 MPASS(atomic_read(&work->state) != WORK_ST_TIMER);
389 MPASS(atomic_read(&work->state) != WORK_ST_CANCEL);
390
391 switch (linux_update_state(&work->state, states)) {
392 case WORK_ST_TASK:
393 tq = work->work_queue->taskqueue;
394 if (taskqueue_cancel(tq, &work->work_task, NULL) == 0)
395 return (true);
396 /* FALLTHROUGH */
397 default:
398 return (false);
399 }
400 }
401
402 /*
403 * This function cancels the given work structure in a synchronous
404 * fashion. It returns non-zero if the work was successfully
405 * cancelled. Else the work was already cancelled.
406 */
407 bool
linux_cancel_work_sync(struct work_struct * work)408 linux_cancel_work_sync(struct work_struct *work)
409 {
410 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
411 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
412 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
413 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
414 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
415 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
416 };
417 struct taskqueue *tq;
418 bool retval = false;
419
420 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
421 "linux_cancel_work_sync() might sleep");
422 retry:
423 switch (linux_update_state(&work->state, states)) {
424 case WORK_ST_IDLE:
425 case WORK_ST_TIMER:
426 return (retval);
427 case WORK_ST_EXEC:
428 tq = work->work_queue->taskqueue;
429 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
430 taskqueue_drain(tq, &work->work_task);
431 goto retry; /* work may have restarted itself */
432 default:
433 tq = work->work_queue->taskqueue;
434 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
435 taskqueue_drain(tq, &work->work_task);
436 retval = true;
437 goto retry;
438 }
439 }
440
441 /*
442 * This function atomically stops the timer and callback. The timer
443 * callback will not be called after this function returns. This
444 * functions returns true when the timeout was cancelled. Else the
445 * timeout was not started or has already been called.
446 */
447 static inline bool
linux_cancel_timer(struct delayed_work * dwork,bool drain)448 linux_cancel_timer(struct delayed_work *dwork, bool drain)
449 {
450 bool cancelled;
451
452 mtx_lock(&dwork->timer.mtx);
453 cancelled = (callout_stop(&dwork->timer.callout) == 1);
454 mtx_unlock(&dwork->timer.mtx);
455
456 /* check if we should drain */
457 if (drain)
458 callout_drain(&dwork->timer.callout);
459 return (cancelled);
460 }
461
462 /*
463 * This function cancels the given delayed work structure in a
464 * non-blocking fashion. It returns non-zero if the work was
465 * successfully cancelled. Else the work may still be busy or already
466 * cancelled.
467 */
468 bool
linux_cancel_delayed_work(struct delayed_work * dwork)469 linux_cancel_delayed_work(struct delayed_work *dwork)
470 {
471 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
472 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
473 [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */
474 [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */
475 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
476 [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */
477 };
478 struct taskqueue *tq;
479 bool cancelled;
480
481 mtx_lock(&dwork->timer.mtx);
482 switch (linux_update_state(&dwork->work.state, states)) {
483 case WORK_ST_TIMER:
484 case WORK_ST_CANCEL:
485 cancelled = (callout_stop(&dwork->timer.callout) == 1);
486 if (cancelled) {
487 atomic_cmpxchg(&dwork->work.state,
488 WORK_ST_CANCEL, WORK_ST_IDLE);
489 mtx_unlock(&dwork->timer.mtx);
490 return (true);
491 }
492 /* FALLTHROUGH */
493 case WORK_ST_TASK:
494 tq = dwork->work.work_queue->taskqueue;
495 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) {
496 atomic_cmpxchg(&dwork->work.state,
497 WORK_ST_CANCEL, WORK_ST_IDLE);
498 mtx_unlock(&dwork->timer.mtx);
499 return (true);
500 }
501 /* FALLTHROUGH */
502 default:
503 mtx_unlock(&dwork->timer.mtx);
504 return (false);
505 }
506 }
507
508 /*
509 * This function cancels the given work structure in a synchronous
510 * fashion. It returns true if the work was successfully
511 * cancelled. Else the work was already cancelled.
512 */
513 static bool
linux_cancel_delayed_work_sync_int(struct delayed_work * dwork)514 linux_cancel_delayed_work_sync_int(struct delayed_work *dwork)
515 {
516 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
517 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
518 [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */
519 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
520 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
521 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
522 };
523 struct taskqueue *tq;
524 int ret, state;
525 bool cancelled;
526
527 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
528 "linux_cancel_delayed_work_sync() might sleep");
529 mtx_lock(&dwork->timer.mtx);
530
531 state = linux_update_state(&dwork->work.state, states);
532 switch (state) {
533 case WORK_ST_IDLE:
534 mtx_unlock(&dwork->timer.mtx);
535 return (false);
536 case WORK_ST_TIMER:
537 case WORK_ST_CANCEL:
538 cancelled = (callout_stop(&dwork->timer.callout) == 1);
539
540 tq = dwork->work.work_queue->taskqueue;
541 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL);
542 mtx_unlock(&dwork->timer.mtx);
543
544 callout_drain(&dwork->timer.callout);
545 taskqueue_drain(tq, &dwork->work.work_task);
546 return (cancelled || (ret != 0));
547 default:
548 tq = dwork->work.work_queue->taskqueue;
549 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL);
550 mtx_unlock(&dwork->timer.mtx);
551 if (ret != 0)
552 taskqueue_drain(tq, &dwork->work.work_task);
553 return (ret != 0);
554 }
555 }
556
557 bool
linux_cancel_delayed_work_sync(struct delayed_work * dwork)558 linux_cancel_delayed_work_sync(struct delayed_work *dwork)
559 {
560 bool res;
561
562 res = false;
563 while (linux_cancel_delayed_work_sync_int(dwork))
564 res = true;
565 return (res);
566 }
567
568 /*
569 * This function waits until the given work structure is completed.
570 * It returns non-zero if the work was successfully
571 * waited for. Else the work was not waited for.
572 */
573 bool
linux_flush_work(struct work_struct * work)574 linux_flush_work(struct work_struct *work)
575 {
576 struct taskqueue *tq;
577 bool retval;
578
579 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
580 "linux_flush_work() might sleep");
581
582 switch (atomic_read(&work->state)) {
583 case WORK_ST_IDLE:
584 return (false);
585 default:
586 tq = work->work_queue->taskqueue;
587 retval = taskqueue_poll_is_busy(tq, &work->work_task);
588 taskqueue_drain(tq, &work->work_task);
589 return (retval);
590 }
591 }
592
593 /*
594 * This function waits until the given delayed work structure is
595 * completed. It returns non-zero if the work was successfully waited
596 * for. Else the work was not waited for.
597 */
598 bool
linux_flush_delayed_work(struct delayed_work * dwork)599 linux_flush_delayed_work(struct delayed_work *dwork)
600 {
601 struct taskqueue *tq;
602 bool retval;
603
604 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
605 "linux_flush_delayed_work() might sleep");
606
607 switch (atomic_read(&dwork->work.state)) {
608 case WORK_ST_IDLE:
609 return (false);
610 case WORK_ST_TIMER:
611 if (linux_cancel_timer(dwork, 1))
612 linux_delayed_work_enqueue(dwork);
613 /* FALLTHROUGH */
614 default:
615 tq = dwork->work.work_queue->taskqueue;
616 retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task);
617 taskqueue_drain(tq, &dwork->work.work_task);
618 return (retval);
619 }
620 }
621
622 /*
623 * This function returns true if the given work is pending, and not
624 * yet executing:
625 */
626 bool
linux_work_pending(struct work_struct * work)627 linux_work_pending(struct work_struct *work)
628 {
629 switch (atomic_read(&work->state)) {
630 case WORK_ST_TIMER:
631 case WORK_ST_TASK:
632 case WORK_ST_CANCEL:
633 return (true);
634 default:
635 return (false);
636 }
637 }
638
639 /*
640 * This function returns true if the given work is busy.
641 */
642 bool
linux_work_busy(struct work_struct * work)643 linux_work_busy(struct work_struct *work)
644 {
645 struct taskqueue *tq;
646
647 switch (atomic_read(&work->state)) {
648 case WORK_ST_IDLE:
649 return (false);
650 case WORK_ST_EXEC:
651 tq = work->work_queue->taskqueue;
652 return (taskqueue_poll_is_busy(tq, &work->work_task));
653 default:
654 return (true);
655 }
656 }
657
658 struct workqueue_struct *
linux_create_workqueue_common(const char * name,int cpus)659 linux_create_workqueue_common(const char *name, int cpus)
660 {
661 struct workqueue_struct *wq;
662
663 /*
664 * If zero CPUs are specified use the default number of CPUs:
665 */
666 if (cpus == 0)
667 cpus = linux_default_wq_cpus;
668
669 wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO);
670 wq->taskqueue = taskqueue_create(name, M_WAITOK,
671 taskqueue_thread_enqueue, &wq->taskqueue);
672 atomic_set(&wq->draining, 0);
673 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
674 TAILQ_INIT(&wq->exec_head);
675 mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF);
676
677 return (wq);
678 }
679
680 void
linux_destroy_workqueue(struct workqueue_struct * wq)681 linux_destroy_workqueue(struct workqueue_struct *wq)
682 {
683 atomic_inc(&wq->draining);
684 drain_workqueue(wq);
685 taskqueue_free(wq->taskqueue);
686 mtx_destroy(&wq->exec_mtx);
687 kfree(wq);
688 }
689
690 void
linux_init_delayed_work(struct delayed_work * dwork,work_func_t func)691 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func)
692 {
693 memset(dwork, 0, sizeof(*dwork));
694 dwork->work.func = func;
695 TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork);
696 mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL,
697 MTX_DEF | MTX_NOWITNESS);
698 callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0);
699 }
700
701 struct work_struct *
linux_current_work(void)702 linux_current_work(void)
703 {
704 return (current->work);
705 }
706
707 static void
linux_work_init(void * arg)708 linux_work_init(void *arg)
709 {
710 int max_wq_cpus = mp_ncpus + 1;
711
712 /* avoid deadlock when there are too few threads */
713 if (max_wq_cpus < 4)
714 max_wq_cpus = 4;
715
716 /* set default number of CPUs */
717 linux_default_wq_cpus = max_wq_cpus;
718
719 linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus);
720 linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus);
721
722 /* populate the workqueue pointers */
723 system_long_wq = linux_system_long_wq;
724 system_wq = linux_system_short_wq;
725 system_power_efficient_wq = linux_system_short_wq;
726 system_unbound_wq = linux_system_short_wq;
727 system_highpri_wq = linux_system_short_wq;
728 }
729 SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL);
730
731 static void
linux_work_uninit(void * arg)732 linux_work_uninit(void *arg)
733 {
734 destroy_workqueue(linux_system_short_wq);
735 destroy_workqueue(linux_system_long_wq);
736
737 /* clear workqueue pointers */
738 system_long_wq = NULL;
739 system_wq = NULL;
740 system_power_efficient_wq = NULL;
741 system_unbound_wq = NULL;
742 system_highpri_wq = NULL;
743 }
744 SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL);
745
746 void
linux_irq_work_fn(void * context,int pending)747 linux_irq_work_fn(void *context, int pending)
748 {
749 struct irq_work *irqw = context;
750
751 irqw->func(irqw);
752 }
753
754 static void
linux_irq_work_init_fn(void * context,int pending)755 linux_irq_work_init_fn(void *context, int pending)
756 {
757 /*
758 * LinuxKPI performs lazy allocation of memory structures required by
759 * current on the first access to it. As some irq_work clients read
760 * it with spinlock taken, we have to preallocate td_lkpi_task before
761 * first call to irq_work_queue(). As irq_work uses a single thread,
762 * it is enough to read current once at SYSINIT stage.
763 */
764 if (current == NULL)
765 panic("irq_work taskqueue is not initialized");
766 }
767 static struct task linux_irq_work_init_task =
768 TASK_INITIALIZER(0, linux_irq_work_init_fn, &linux_irq_work_init_task);
769
770 static void
linux_irq_work_init(void * arg)771 linux_irq_work_init(void *arg)
772 {
773 linux_irq_work_tq = taskqueue_create_fast("linuxkpi_irq_wq",
774 M_WAITOK, taskqueue_thread_enqueue, &linux_irq_work_tq);
775 taskqueue_start_threads(&linux_irq_work_tq, 1, PWAIT,
776 "linuxkpi_irq_wq");
777 taskqueue_enqueue(linux_irq_work_tq, &linux_irq_work_init_task);
778 }
779 SYSINIT(linux_irq_work_init, SI_SUB_TASKQ, SI_ORDER_SECOND,
780 linux_irq_work_init, NULL);
781
782 static void
linux_irq_work_uninit(void * arg)783 linux_irq_work_uninit(void *arg)
784 {
785 taskqueue_drain_all(linux_irq_work_tq);
786 taskqueue_free(linux_irq_work_tq);
787 }
788 SYSUNINIT(linux_irq_work_uninit, SI_SUB_TASKQ, SI_ORDER_SECOND,
789 linux_irq_work_uninit, NULL);
790