1 /*-
2 * Copyright (c) 2017-2019 Hans Petter Selasky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #include <linux/workqueue.h>
29 #include <linux/wait.h>
30 #include <linux/compat.h>
31 #include <linux/spinlock.h>
32 #include <linux/rcupdate.h>
33 #include <linux/irq_work.h>
34
35 #include <sys/kernel.h>
36
37 /*
38 * Define all work struct states
39 */
40 enum {
41 WORK_ST_IDLE, /* idle - not started */
42 WORK_ST_TIMER, /* timer is being started */
43 WORK_ST_TASK, /* taskqueue is being queued */
44 WORK_ST_EXEC, /* callback is being called */
45 WORK_ST_CANCEL, /* cancel is being requested */
46 WORK_ST_MAX,
47 };
48
49 /*
50 * Define global workqueues
51 */
52 static struct workqueue_struct *linux_system_short_wq;
53 static struct workqueue_struct *linux_system_long_wq;
54
55 struct workqueue_struct *system_wq;
56 struct workqueue_struct *system_long_wq;
57 struct workqueue_struct *system_unbound_wq;
58 struct workqueue_struct *system_highpri_wq;
59 struct workqueue_struct *system_power_efficient_wq;
60 struct workqueue_struct *system_percpu_wq;
61
62 struct taskqueue *linux_irq_work_tq;
63
64 static int linux_default_wq_cpus = 4;
65
66 static void linux_delayed_work_timer_fn(void *);
67
68 /*
69 * This function atomically updates the work state and returns the
70 * previous state at the time of update.
71 */
72 static uint8_t
linux_update_state(atomic_t * v,const uint8_t * pstate)73 linux_update_state(atomic_t *v, const uint8_t *pstate)
74 {
75 int c, old;
76
77 c = v->counter;
78
79 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c)
80 c = old;
81
82 return (c);
83 }
84
85 /*
86 * A LinuxKPI task is allowed to free itself inside the callback function
87 * and cannot safely be referred after the callback function has
88 * completed. This function gives the linux_work_fn() function a hint,
89 * that the task is not going away and can have its state checked
90 * again. Without this extra hint LinuxKPI tasks cannot be serialized
91 * across multiple worker threads.
92 */
93 static bool
linux_work_exec_unblock(struct work_struct * work)94 linux_work_exec_unblock(struct work_struct *work)
95 {
96 struct workqueue_struct *wq;
97 struct work_exec *exec;
98 bool retval = false;
99
100 wq = work->work_queue;
101 if (unlikely(wq == NULL))
102 goto done;
103
104 WQ_EXEC_LOCK(wq);
105 TAILQ_FOREACH(exec, &wq->exec_head, entry) {
106 if (exec->target == work) {
107 exec->target = NULL;
108 retval = true;
109 break;
110 }
111 }
112 WQ_EXEC_UNLOCK(wq);
113 done:
114 return (retval);
115 }
116
117 static void
linux_delayed_work_enqueue(struct delayed_work * dwork)118 linux_delayed_work_enqueue(struct delayed_work *dwork)
119 {
120 struct taskqueue *tq;
121
122 tq = dwork->work.work_queue->taskqueue;
123 taskqueue_enqueue(tq, &dwork->work.work_task);
124 }
125
126 /*
127 * This function queues the given work structure on the given
128 * workqueue. It returns non-zero if the work was successfully
129 * [re-]queued. Else the work is already pending for completion.
130 */
131 bool
linux_queue_work_on(int cpu __unused,struct workqueue_struct * wq,struct work_struct * work)132 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq,
133 struct work_struct *work)
134 {
135 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
136 [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */
137 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
138 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
139 [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */
140 [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */
141 };
142
143 if (atomic_read(&wq->draining) != 0)
144 return (!work_pending(work));
145
146 switch (linux_update_state(&work->state, states)) {
147 case WORK_ST_EXEC:
148 case WORK_ST_CANCEL:
149 if (linux_work_exec_unblock(work) != 0)
150 return (true);
151 /* FALLTHROUGH */
152 case WORK_ST_IDLE:
153 work->work_queue = wq;
154 taskqueue_enqueue(wq->taskqueue, &work->work_task);
155 return (true);
156 default:
157 return (false); /* already on a queue */
158 }
159 }
160
161 /*
162 * Callback func for linux_queue_rcu_work
163 */
164 static void
rcu_work_func(struct rcu_head * rcu)165 rcu_work_func(struct rcu_head *rcu)
166 {
167 struct rcu_work *rwork;
168
169 rwork = container_of(rcu, struct rcu_work, rcu);
170 linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
171 }
172
173 /*
174 * This function queue a work after a grace period
175 * If the work was already pending it returns false,
176 * if not it calls call_rcu and returns true.
177 */
178 bool
linux_queue_rcu_work(struct workqueue_struct * wq,struct rcu_work * rwork)179 linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
180 {
181
182 if (!linux_work_pending(&rwork->work)) {
183 rwork->wq = wq;
184 linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func);
185 return (true);
186 }
187 return (false);
188 }
189
190 /*
191 * This function waits for the last execution of a work and then
192 * flush the work.
193 * It returns true if the work was pending and we waited, it returns
194 * false otherwise.
195 */
196 bool
linux_flush_rcu_work(struct rcu_work * rwork)197 linux_flush_rcu_work(struct rcu_work *rwork)
198 {
199
200 if (linux_work_pending(&rwork->work)) {
201 linux_rcu_barrier(RCU_TYPE_REGULAR);
202 linux_flush_work(&rwork->work);
203 return (true);
204 }
205 return (linux_flush_work(&rwork->work));
206 }
207
208 /*
209 * This function queues the given work structure on the given
210 * workqueue after a given delay in ticks. It returns true if the
211 * work was successfully [re-]queued. Else the work is already pending
212 * for completion.
213 */
214 bool
linux_queue_delayed_work_on(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned long delay)215 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
216 struct delayed_work *dwork, unsigned long delay)
217 {
218 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
219 [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */
220 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
221 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
222 [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */
223 [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */
224 };
225 bool res;
226
227 if (atomic_read(&wq->draining) != 0)
228 return (!work_pending(&dwork->work));
229
230 /*
231 * Clamp the delay to a valid ticks value, some consumers pass
232 * MAX_SCHEDULE_TIMEOUT.
233 */
234 if (delay > INT_MAX)
235 delay = INT_MAX;
236
237 mtx_lock(&dwork->timer.mtx);
238 switch (linux_update_state(&dwork->work.state, states)) {
239 case WORK_ST_EXEC:
240 case WORK_ST_CANCEL:
241 if (delay == 0 && linux_work_exec_unblock(&dwork->work)) {
242 dwork->timer.expires = jiffies;
243 res = true;
244 goto out;
245 }
246 /* FALLTHROUGH */
247 case WORK_ST_IDLE:
248 dwork->work.work_queue = wq;
249 dwork->timer.expires = jiffies + delay;
250
251 if (delay == 0) {
252 linux_delayed_work_enqueue(dwork);
253 } else if (unlikely(cpu != WORK_CPU_UNBOUND)) {
254 callout_reset_on(&dwork->timer.callout, delay,
255 &linux_delayed_work_timer_fn, dwork, cpu);
256 } else {
257 callout_reset(&dwork->timer.callout, delay,
258 &linux_delayed_work_timer_fn, dwork);
259 }
260 res = true;
261 break;
262 default:
263 res = false;
264 break;
265 }
266 out:
267 mtx_unlock(&dwork->timer.mtx);
268 return (res);
269 }
270
271 void
linux_work_fn(void * context,int pending)272 linux_work_fn(void *context, int pending)
273 {
274 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
275 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
276 [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */
277 [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */
278 [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */
279 [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */
280 };
281 struct work_struct *work;
282 struct workqueue_struct *wq;
283 struct work_exec exec;
284 struct task_struct *task;
285
286 task = current;
287
288 /* setup local variables */
289 work = context;
290 wq = work->work_queue;
291
292 /* store target pointer */
293 exec.target = work;
294
295 /* insert executor into list */
296 WQ_EXEC_LOCK(wq);
297 TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry);
298 while (1) {
299 switch (linux_update_state(&work->state, states)) {
300 case WORK_ST_TIMER:
301 case WORK_ST_TASK:
302 case WORK_ST_CANCEL:
303 WQ_EXEC_UNLOCK(wq);
304
305 /* set current work structure */
306 task->work = work;
307
308 /* call work function */
309 work->func(work);
310
311 /* set current work structure */
312 task->work = NULL;
313
314 WQ_EXEC_LOCK(wq);
315 /* check if unblocked */
316 if (exec.target != work) {
317 /* reapply block */
318 exec.target = work;
319 break;
320 }
321 /* FALLTHROUGH */
322 default:
323 goto done;
324 }
325 }
326 done:
327 /* remove executor from list */
328 TAILQ_REMOVE(&wq->exec_head, &exec, entry);
329 WQ_EXEC_UNLOCK(wq);
330 }
331
332 void
linux_delayed_work_fn(void * context,int pending)333 linux_delayed_work_fn(void *context, int pending)
334 {
335 struct delayed_work *dwork = context;
336
337 /*
338 * Make sure the timer belonging to the delayed work gets
339 * drained before invoking the work function. Else the timer
340 * mutex may still be in use which can lead to use-after-free
341 * situations, because the work function might free the work
342 * structure before returning.
343 */
344 callout_drain(&dwork->timer.callout);
345
346 linux_work_fn(&dwork->work, pending);
347 }
348
349 static void
linux_delayed_work_timer_fn(void * arg)350 linux_delayed_work_timer_fn(void *arg)
351 {
352 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
353 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
354 [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */
355 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
356 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
357 [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */
358 };
359 struct delayed_work *dwork = arg;
360
361 switch (linux_update_state(&dwork->work.state, states)) {
362 case WORK_ST_TIMER:
363 case WORK_ST_CANCEL:
364 linux_delayed_work_enqueue(dwork);
365 break;
366 default:
367 break;
368 }
369 }
370
371 /*
372 * This function cancels the given work structure in a
373 * non-blocking fashion. It returns non-zero if the work was
374 * successfully cancelled. Else the work may still be busy or already
375 * cancelled.
376 */
377 bool
linux_cancel_work(struct work_struct * work)378 linux_cancel_work(struct work_struct *work)
379 {
380 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
381 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
382 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
383 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel */
384 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
385 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* can't happen */
386 };
387 struct taskqueue *tq;
388
389 MPASS(atomic_read(&work->state) != WORK_ST_TIMER);
390 MPASS(atomic_read(&work->state) != WORK_ST_CANCEL);
391
392 switch (linux_update_state(&work->state, states)) {
393 case WORK_ST_TASK:
394 tq = work->work_queue->taskqueue;
395 if (taskqueue_cancel(tq, &work->work_task, NULL) == 0)
396 return (true);
397 /* FALLTHROUGH */
398 default:
399 return (false);
400 }
401 }
402
403 /*
404 * This function cancels the given work structure in a synchronous
405 * fashion. It returns non-zero if the work was successfully
406 * cancelled. Else the work was already cancelled.
407 */
408 bool
linux_cancel_work_sync(struct work_struct * work)409 linux_cancel_work_sync(struct work_struct *work)
410 {
411 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
412 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
413 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
414 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
415 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
416 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
417 };
418 struct taskqueue *tq;
419 bool retval = false;
420
421 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
422 "linux_cancel_work_sync() might sleep");
423 retry:
424 switch (linux_update_state(&work->state, states)) {
425 case WORK_ST_IDLE:
426 case WORK_ST_TIMER:
427 return (retval);
428 case WORK_ST_EXEC:
429 tq = work->work_queue->taskqueue;
430 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
431 taskqueue_drain(tq, &work->work_task);
432 goto retry; /* work may have restarted itself */
433 default:
434 tq = work->work_queue->taskqueue;
435 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
436 taskqueue_drain(tq, &work->work_task);
437 retval = true;
438 goto retry;
439 }
440 }
441
442 /*
443 * This function atomically stops the timer and callback. The timer
444 * callback will not be called after this function returns. This
445 * functions returns true when the timeout was cancelled. Else the
446 * timeout was not started or has already been called.
447 */
448 static inline bool
linux_cancel_timer(struct delayed_work * dwork,bool drain)449 linux_cancel_timer(struct delayed_work *dwork, bool drain)
450 {
451 bool cancelled;
452
453 mtx_lock(&dwork->timer.mtx);
454 cancelled = (callout_stop(&dwork->timer.callout) == 1);
455 mtx_unlock(&dwork->timer.mtx);
456
457 /* check if we should drain */
458 if (drain)
459 callout_drain(&dwork->timer.callout);
460 return (cancelled);
461 }
462
463 /*
464 * This function cancels the given delayed work structure in a
465 * non-blocking fashion. It returns non-zero if the work was
466 * successfully cancelled. Else the work may still be busy or already
467 * cancelled.
468 */
469 bool
linux_cancel_delayed_work(struct delayed_work * dwork)470 linux_cancel_delayed_work(struct delayed_work *dwork)
471 {
472 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
473 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
474 [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */
475 [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */
476 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
477 [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */
478 };
479 struct taskqueue *tq;
480 bool cancelled;
481
482 mtx_lock(&dwork->timer.mtx);
483 switch (linux_update_state(&dwork->work.state, states)) {
484 case WORK_ST_TIMER:
485 case WORK_ST_CANCEL:
486 cancelled = (callout_stop(&dwork->timer.callout) == 1);
487 if (cancelled) {
488 atomic_cmpxchg(&dwork->work.state,
489 WORK_ST_CANCEL, WORK_ST_IDLE);
490 mtx_unlock(&dwork->timer.mtx);
491 return (true);
492 }
493 /* FALLTHROUGH */
494 case WORK_ST_TASK:
495 tq = dwork->work.work_queue->taskqueue;
496 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) {
497 atomic_cmpxchg(&dwork->work.state,
498 WORK_ST_CANCEL, WORK_ST_IDLE);
499 mtx_unlock(&dwork->timer.mtx);
500 return (true);
501 }
502 /* FALLTHROUGH */
503 default:
504 mtx_unlock(&dwork->timer.mtx);
505 return (false);
506 }
507 }
508
509 /*
510 * This function cancels the given work structure in a synchronous
511 * fashion. It returns true if the work was successfully
512 * cancelled. Else the work was already cancelled.
513 */
514 static bool
linux_cancel_delayed_work_sync_int(struct delayed_work * dwork)515 linux_cancel_delayed_work_sync_int(struct delayed_work *dwork)
516 {
517 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
518 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
519 [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */
520 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
521 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
522 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
523 };
524 struct taskqueue *tq;
525 int ret, state;
526 bool cancelled;
527
528 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
529 "linux_cancel_delayed_work_sync() might sleep");
530 mtx_lock(&dwork->timer.mtx);
531
532 state = linux_update_state(&dwork->work.state, states);
533 switch (state) {
534 case WORK_ST_IDLE:
535 mtx_unlock(&dwork->timer.mtx);
536 return (false);
537 case WORK_ST_TIMER:
538 case WORK_ST_CANCEL:
539 cancelled = (callout_stop(&dwork->timer.callout) == 1);
540
541 tq = dwork->work.work_queue->taskqueue;
542 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL);
543 mtx_unlock(&dwork->timer.mtx);
544
545 callout_drain(&dwork->timer.callout);
546 taskqueue_drain(tq, &dwork->work.work_task);
547 return (cancelled || (ret != 0));
548 default:
549 tq = dwork->work.work_queue->taskqueue;
550 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL);
551 mtx_unlock(&dwork->timer.mtx);
552 if (ret != 0)
553 taskqueue_drain(tq, &dwork->work.work_task);
554 return (ret != 0);
555 }
556 }
557
558 bool
linux_cancel_delayed_work_sync(struct delayed_work * dwork)559 linux_cancel_delayed_work_sync(struct delayed_work *dwork)
560 {
561 bool res;
562
563 res = false;
564 while (linux_cancel_delayed_work_sync_int(dwork))
565 res = true;
566 return (res);
567 }
568
569 /*
570 * This function waits until the given work structure is completed.
571 * It returns non-zero if the work was successfully
572 * waited for. Else the work was not waited for.
573 */
574 bool
linux_flush_work(struct work_struct * work)575 linux_flush_work(struct work_struct *work)
576 {
577 struct taskqueue *tq;
578 bool retval;
579
580 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
581 "linux_flush_work() might sleep");
582
583 switch (atomic_read(&work->state)) {
584 case WORK_ST_IDLE:
585 return (false);
586 default:
587 tq = work->work_queue->taskqueue;
588 retval = taskqueue_poll_is_busy(tq, &work->work_task);
589 taskqueue_drain(tq, &work->work_task);
590 return (retval);
591 }
592 }
593
594 /*
595 * This function waits until the given delayed work structure is
596 * completed. It returns non-zero if the work was successfully waited
597 * for. Else the work was not waited for.
598 */
599 bool
linux_flush_delayed_work(struct delayed_work * dwork)600 linux_flush_delayed_work(struct delayed_work *dwork)
601 {
602 struct taskqueue *tq;
603 bool retval;
604
605 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
606 "linux_flush_delayed_work() might sleep");
607
608 switch (atomic_read(&dwork->work.state)) {
609 case WORK_ST_IDLE:
610 return (false);
611 case WORK_ST_TIMER:
612 if (linux_cancel_timer(dwork, 1))
613 linux_delayed_work_enqueue(dwork);
614 /* FALLTHROUGH */
615 default:
616 tq = dwork->work.work_queue->taskqueue;
617 retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task);
618 taskqueue_drain(tq, &dwork->work.work_task);
619 return (retval);
620 }
621 }
622
623 /*
624 * This function returns true if the given work is pending, and not
625 * yet executing:
626 */
627 bool
linux_work_pending(struct work_struct * work)628 linux_work_pending(struct work_struct *work)
629 {
630 switch (atomic_read(&work->state)) {
631 case WORK_ST_TIMER:
632 case WORK_ST_TASK:
633 case WORK_ST_CANCEL:
634 return (true);
635 default:
636 return (false);
637 }
638 }
639
640 /*
641 * This function returns true if the given work is busy.
642 */
643 bool
linux_work_busy(struct work_struct * work)644 linux_work_busy(struct work_struct *work)
645 {
646 struct taskqueue *tq;
647
648 switch (atomic_read(&work->state)) {
649 case WORK_ST_IDLE:
650 return (false);
651 case WORK_ST_EXEC:
652 tq = work->work_queue->taskqueue;
653 return (taskqueue_poll_is_busy(tq, &work->work_task));
654 default:
655 return (true);
656 }
657 }
658
659 struct workqueue_struct *
linux_create_workqueue_common(const char * name,int cpus)660 linux_create_workqueue_common(const char *name, int cpus)
661 {
662 struct workqueue_struct *wq;
663
664 /*
665 * If zero CPUs are specified use the default number of CPUs:
666 */
667 if (cpus == 0)
668 cpus = linux_default_wq_cpus;
669
670 wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO);
671 wq->taskqueue = taskqueue_create(name, M_WAITOK,
672 taskqueue_thread_enqueue, &wq->taskqueue);
673 atomic_set(&wq->draining, 0);
674 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
675 TAILQ_INIT(&wq->exec_head);
676 mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF);
677
678 return (wq);
679 }
680
681 void
linux_destroy_workqueue(struct workqueue_struct * wq)682 linux_destroy_workqueue(struct workqueue_struct *wq)
683 {
684 atomic_inc(&wq->draining);
685 drain_workqueue(wq);
686 taskqueue_free(wq->taskqueue);
687 mtx_destroy(&wq->exec_mtx);
688 kfree(wq);
689 }
690
691 void
linux_init_delayed_work(struct delayed_work * dwork,work_func_t func)692 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func)
693 {
694 memset(dwork, 0, sizeof(*dwork));
695 dwork->work.func = func;
696 TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork);
697 mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL,
698 MTX_DEF | MTX_NOWITNESS);
699 callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0);
700 }
701
702 struct work_struct *
linux_current_work(void)703 linux_current_work(void)
704 {
705 return (current->work);
706 }
707
708 static void
linux_work_init(void * arg)709 linux_work_init(void *arg)
710 {
711 int max_wq_cpus = mp_ncpus + 1;
712
713 /* avoid deadlock when there are too few threads */
714 if (max_wq_cpus < 4)
715 max_wq_cpus = 4;
716
717 /* set default number of CPUs */
718 linux_default_wq_cpus = max_wq_cpus;
719
720 linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus);
721 linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus);
722
723 /* populate the workqueue pointers */
724 system_long_wq = linux_system_long_wq;
725 /*
726 * With Linux v6.17 system_wq was "renamed" to system_percpu_wq with the
727 * old name staying around.
728 * Note: neither implementation here does fully implement the per-cpu
729 * characteristics upstream expects.
730 */
731 system_wq = linux_system_short_wq;
732 system_percpu_wq = linux_system_short_wq;
733 system_power_efficient_wq = linux_system_short_wq;
734 system_unbound_wq = linux_system_short_wq;
735 system_highpri_wq = linux_system_short_wq;
736 }
737 SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL);
738
739 static void
linux_work_uninit(void * arg)740 linux_work_uninit(void *arg)
741 {
742 destroy_workqueue(linux_system_short_wq);
743 destroy_workqueue(linux_system_long_wq);
744
745 /* clear workqueue pointers */
746 system_long_wq = NULL;
747 system_wq = NULL;
748 system_percpu_wq = NULL;
749 system_power_efficient_wq = NULL;
750 system_unbound_wq = NULL;
751 system_highpri_wq = NULL;
752 }
753 SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL);
754
755 void
linux_irq_work_fn(void * context,int pending)756 linux_irq_work_fn(void *context, int pending)
757 {
758 struct irq_work *irqw = context;
759
760 irqw->func(irqw);
761 }
762
763 static void
linux_irq_work_init_fn(void * context,int pending)764 linux_irq_work_init_fn(void *context, int pending)
765 {
766 /*
767 * LinuxKPI performs lazy allocation of memory structures required by
768 * current on the first access to it. As some irq_work clients read
769 * it with spinlock taken, we have to preallocate td_lkpi_task before
770 * first call to irq_work_queue(). As irq_work uses a single thread,
771 * it is enough to read current once at SYSINIT stage.
772 */
773 if (current == NULL)
774 panic("irq_work taskqueue is not initialized");
775 }
776 static struct task linux_irq_work_init_task =
777 TASK_INITIALIZER(0, linux_irq_work_init_fn, &linux_irq_work_init_task);
778
779 static void
linux_irq_work_init(void * arg)780 linux_irq_work_init(void *arg)
781 {
782 linux_irq_work_tq = taskqueue_create_fast("linuxkpi_irq_wq",
783 M_WAITOK, taskqueue_thread_enqueue, &linux_irq_work_tq);
784 taskqueue_start_threads(&linux_irq_work_tq, 1, PWAIT,
785 "linuxkpi_irq_wq");
786 taskqueue_enqueue(linux_irq_work_tq, &linux_irq_work_init_task);
787 }
788 SYSINIT(linux_irq_work_init, SI_SUB_TASKQ, SI_ORDER_SECOND,
789 linux_irq_work_init, NULL);
790
791 static void
linux_irq_work_uninit(void * arg)792 linux_irq_work_uninit(void *arg)
793 {
794 taskqueue_drain_all(linux_irq_work_tq);
795 taskqueue_free(linux_irq_work_tq);
796 }
797 SYSUNINIT(linux_irq_work_uninit, SI_SUB_TASKQ, SI_ORDER_SECOND,
798 linux_irq_work_uninit, NULL);
799