1 /*-
2 * Copyright (c) 2017-2019 Hans Petter Selasky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #include <linux/workqueue.h>
29 #include <linux/wait.h>
30 #include <linux/compat.h>
31 #include <linux/spinlock.h>
32 #include <linux/rcupdate.h>
33 #include <linux/irq_work.h>
34
35 #include <sys/kernel.h>
36
37 /*
38 * Define all work struct states
39 */
40 enum {
41 WORK_ST_IDLE, /* idle - not started */
42 WORK_ST_TIMER, /* timer is being started */
43 WORK_ST_TASK, /* taskqueue is being queued */
44 WORK_ST_EXEC, /* callback is being called */
45 WORK_ST_CANCEL, /* cancel is being requested */
46 WORK_ST_MAX,
47 };
48
49 /*
50 * Define global workqueues
51 */
52 static struct workqueue_struct *linux_system_short_wq;
53 static struct workqueue_struct *linux_system_long_wq;
54
55 struct workqueue_struct *system_wq;
56 struct workqueue_struct *system_long_wq;
57 struct workqueue_struct *system_unbound_wq;
58 struct workqueue_struct *system_highpri_wq;
59 struct workqueue_struct *system_power_efficient_wq;
60 struct workqueue_struct *system_percpu_wq;
61
62 struct taskqueue *linux_irq_work_tq;
63
64 static int linux_default_wq_cpus = 4;
65
66 static void linux_delayed_work_timer_fn(void *);
67
68 /*
69 * This function atomically updates the work state and returns the
70 * previous state at the time of update.
71 */
72 static uint8_t
linux_update_state(atomic_t * v,const uint8_t * pstate)73 linux_update_state(atomic_t *v, const uint8_t *pstate)
74 {
75 int c, old;
76
77 c = v->counter;
78
79 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c)
80 c = old;
81
82 return (c);
83 }
84
85 /*
86 * A LinuxKPI task is allowed to free itself inside the callback function
87 * and cannot safely be referred after the callback function has
88 * completed. This function gives the linux_work_fn() function a hint,
89 * that the task is not going away and can have its state checked
90 * again. Without this extra hint LinuxKPI tasks cannot be serialized
91 * across multiple worker threads.
92 */
93 static bool
linux_work_exec_unblock(struct work_struct * work)94 linux_work_exec_unblock(struct work_struct *work)
95 {
96 struct workqueue_struct *wq;
97 struct work_exec *exec;
98 bool retval = false;
99
100 wq = work->work_queue;
101 if (unlikely(wq == NULL))
102 goto done;
103
104 WQ_EXEC_LOCK(wq);
105 TAILQ_FOREACH(exec, &wq->exec_head, entry) {
106 if (exec->target == work) {
107 exec->target = NULL;
108 retval = true;
109 break;
110 }
111 }
112 WQ_EXEC_UNLOCK(wq);
113 done:
114 return (retval);
115 }
116
117 static void
linux_delayed_work_enqueue(struct delayed_work * dwork)118 linux_delayed_work_enqueue(struct delayed_work *dwork)
119 {
120 struct taskqueue *tq;
121
122 tq = dwork->work.work_queue->taskqueue;
123 taskqueue_enqueue(tq, &dwork->work.work_task);
124 }
125
126 /*
127 * This function queues the given work structure on the given
128 * workqueue. It returns non-zero if the work was successfully
129 * [re-]queued. Else the work is already pending for completion.
130 */
131 bool
linux_queue_work_on(int cpu __unused,struct workqueue_struct * wq,struct work_struct * work)132 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq,
133 struct work_struct *work)
134 {
135 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
136 [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */
137 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
138 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
139 [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */
140 [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */
141 };
142
143 if (atomic_read(&wq->draining) != 0)
144 return (!work_pending(work));
145
146 switch (linux_update_state(&work->state, states)) {
147 case WORK_ST_EXEC:
148 case WORK_ST_CANCEL:
149 if (linux_work_exec_unblock(work) != 0)
150 return (true);
151 /* FALLTHROUGH */
152 case WORK_ST_IDLE:
153 work->work_queue = wq;
154 taskqueue_enqueue(wq->taskqueue, &work->work_task);
155 return (true);
156 default:
157 return (false); /* already on a queue */
158 }
159 }
160
161 /*
162 * Callback func for linux_queue_rcu_work
163 */
164 static void
rcu_work_func(struct rcu_head * rcu)165 rcu_work_func(struct rcu_head *rcu)
166 {
167 struct rcu_work *rwork;
168
169 rwork = container_of(rcu, struct rcu_work, rcu);
170 linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
171 }
172
173 /*
174 * This function queue a work after a grace period
175 * If the work was already pending it returns false,
176 * if not it calls call_rcu and returns true.
177 */
178 bool
linux_queue_rcu_work(struct workqueue_struct * wq,struct rcu_work * rwork)179 linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
180 {
181
182 if (!linux_work_pending(&rwork->work)) {
183 rwork->wq = wq;
184 linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func);
185 return (true);
186 }
187 return (false);
188 }
189
190 /*
191 * This function waits for the last execution of a work and then
192 * flush the work.
193 * It returns true if the work was pending and we waited, it returns
194 * false otherwise.
195 */
196 bool
linux_flush_rcu_work(struct rcu_work * rwork)197 linux_flush_rcu_work(struct rcu_work *rwork)
198 {
199
200 if (linux_work_pending(&rwork->work)) {
201 linux_rcu_barrier(RCU_TYPE_REGULAR);
202 linux_flush_work(&rwork->work);
203 return (true);
204 }
205 return (linux_flush_work(&rwork->work));
206 }
207
208 /*
209 * This function queues the given work structure on the given
210 * workqueue after a given delay in ticks. It returns true if the
211 * work was successfully [re-]queued. Else the work is already pending
212 * for completion.
213 */
214 bool
linux_queue_delayed_work_on(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned long delay)215 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
216 struct delayed_work *dwork, unsigned long delay)
217 {
218 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
219 [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */
220 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
221 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
222 [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */
223 [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */
224 };
225 bool res;
226
227 if (atomic_read(&wq->draining) != 0)
228 return (!work_pending(&dwork->work));
229
230 /*
231 * Clamp the delay to a valid ticks value, some consumers pass
232 * MAX_SCHEDULE_TIMEOUT.
233 */
234 if (delay > INT_MAX)
235 delay = INT_MAX;
236
237 mtx_lock(&dwork->timer.mtx);
238 switch (linux_update_state(&dwork->work.state, states)) {
239 case WORK_ST_EXEC:
240 case WORK_ST_CANCEL:
241 if (delay == 0 && linux_work_exec_unblock(&dwork->work)) {
242 dwork->timer.expires = jiffies;
243 res = true;
244 goto out;
245 }
246 /* FALLTHROUGH */
247 case WORK_ST_IDLE:
248 dwork->work.work_queue = wq;
249 dwork->timer.expires = jiffies + delay;
250
251 if (delay == 0) {
252 linux_delayed_work_enqueue(dwork);
253 } else if (unlikely(cpu != WORK_CPU_UNBOUND)) {
254 callout_reset_on(&dwork->timer.callout, delay,
255 &linux_delayed_work_timer_fn, dwork, cpu);
256 } else {
257 callout_reset(&dwork->timer.callout, delay,
258 &linux_delayed_work_timer_fn, dwork);
259 }
260 res = true;
261 break;
262 default:
263 res = false;
264 break;
265 }
266 out:
267 mtx_unlock(&dwork->timer.mtx);
268 return (res);
269 }
270
271 void
linux_work_fn(void * context,int pending)272 linux_work_fn(void *context, int pending)
273 {
274 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
275 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
276 [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */
277 [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */
278 [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */
279 [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */
280 };
281 struct work_struct *work;
282 struct workqueue_struct *wq;
283 struct work_exec exec;
284 struct task_struct *task;
285
286 task = current;
287
288 /* setup local variables */
289 work = context;
290 wq = work->work_queue;
291
292 /* store target pointer */
293 exec.target = work;
294
295 /* insert executor into list */
296 WQ_EXEC_LOCK(wq);
297 TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry);
298 while (1) {
299 switch (linux_update_state(&work->state, states)) {
300 case WORK_ST_TIMER:
301 case WORK_ST_TASK:
302 case WORK_ST_CANCEL:
303 WQ_EXEC_UNLOCK(wq);
304
305 /* set current work structure */
306 task->work = work;
307
308 /* call work function */
309 work->func(work);
310
311 /* set current work structure */
312 task->work = NULL;
313
314 WQ_EXEC_LOCK(wq);
315 /* check if unblocked */
316 if (exec.target != work) {
317 /* reapply block */
318 exec.target = work;
319 break;
320 }
321 /* FALLTHROUGH */
322 default:
323 goto done;
324 }
325 }
326 done:
327 /* remove executor from list */
328 TAILQ_REMOVE(&wq->exec_head, &exec, entry);
329 WQ_EXEC_UNLOCK(wq);
330 }
331
332 void
linux_delayed_work_fn(void * context,int pending)333 linux_delayed_work_fn(void *context, int pending)
334 {
335 struct delayed_work *dwork = context;
336
337 /*
338 * Make sure the timer belonging to the delayed work gets
339 * drained before invoking the work function. Else the timer
340 * mutex may still be in use which can lead to use-after-free
341 * situations, because the work function might free the work
342 * structure before returning.
343 */
344 callout_drain(&dwork->timer.callout);
345
346 linux_work_fn(&dwork->work, pending);
347 }
348
349 static void
linux_delayed_work_timer_fn(void * arg)350 linux_delayed_work_timer_fn(void *arg)
351 {
352 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
353 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
354 [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */
355 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
356 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
357 [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */
358 };
359 struct delayed_work *dwork = arg;
360
361 switch (linux_update_state(&dwork->work.state, states)) {
362 case WORK_ST_TIMER:
363 case WORK_ST_CANCEL:
364 linux_delayed_work_enqueue(dwork);
365 break;
366 default:
367 break;
368 }
369 }
370
371 /*
372 * This function cancels the given work structure in a
373 * non-blocking fashion. It returns non-zero if the work was
374 * successfully cancelled. Else the work may still be busy or already
375 * cancelled.
376 */
377 bool
linux_cancel_work(struct work_struct * work)378 linux_cancel_work(struct work_struct *work)
379 {
380 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
381 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
382 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
383 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel */
384 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
385 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* can't happen */
386 };
387 struct taskqueue *tq;
388
389 MPASS(atomic_read(&work->state) != WORK_ST_TIMER);
390 MPASS(atomic_read(&work->state) != WORK_ST_CANCEL);
391
392 switch (linux_update_state(&work->state, states)) {
393 case WORK_ST_TASK:
394 tq = work->work_queue->taskqueue;
395 if (taskqueue_cancel(tq, &work->work_task, NULL) == 0)
396 return (true);
397 /* FALLTHROUGH */
398 default:
399 return (false);
400 }
401 }
402
403 /*
404 * This function cancels the given work structure in a synchronous
405 * fashion. It returns non-zero if the work was successfully
406 * cancelled. Else the work was already cancelled.
407 */
408 bool
linux_cancel_work_sync(struct work_struct * work)409 linux_cancel_work_sync(struct work_struct *work)
410 {
411 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
412 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
413 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
414 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
415 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
416 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
417 };
418 struct taskqueue *tq;
419 bool retval = false;
420
421 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
422 "linux_cancel_work_sync() might sleep");
423 retry:
424 switch (linux_update_state(&work->state, states)) {
425 case WORK_ST_IDLE:
426 case WORK_ST_TIMER:
427 return (retval);
428 case WORK_ST_EXEC:
429 tq = work->work_queue->taskqueue;
430 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
431 taskqueue_drain(tq, &work->work_task);
432 goto retry; /* work may have restarted itself */
433 default:
434 tq = work->work_queue->taskqueue;
435 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
436 taskqueue_drain(tq, &work->work_task);
437 retval = true;
438 goto retry;
439 }
440 }
441
442 /*
443 * This function atomically stops the timer and callback. The timer
444 * callback will not be called after this function returns. This
445 * functions returns true when the timeout was cancelled. Else the
446 * timeout was not started or has already been called.
447 */
448 static inline bool
linux_cancel_timer(struct delayed_work * dwork,bool drain)449 linux_cancel_timer(struct delayed_work *dwork, bool drain)
450 {
451 bool cancelled;
452
453 mtx_lock(&dwork->timer.mtx);
454 cancelled = (callout_stop(&dwork->timer.callout) == 1);
455 mtx_unlock(&dwork->timer.mtx);
456
457 /* check if we should drain */
458 if (drain)
459 callout_drain(&dwork->timer.callout);
460 return (cancelled);
461 }
462
463 /*
464 * This function cancels the given delayed work structure in a
465 * non-blocking fashion. It returns non-zero if the work was
466 * successfully cancelled. Else the work may still be busy or already
467 * cancelled.
468 */
469 bool
linux_cancel_delayed_work(struct delayed_work * dwork)470 linux_cancel_delayed_work(struct delayed_work *dwork)
471 {
472 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
473 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
474 [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */
475 [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */
476 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
477 [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */
478 };
479 struct taskqueue *tq;
480 bool cancelled;
481
482 mtx_lock(&dwork->timer.mtx);
483 switch (linux_update_state(&dwork->work.state, states)) {
484 case WORK_ST_TIMER:
485 case WORK_ST_CANCEL:
486 cancelled = (callout_stop(&dwork->timer.callout) == 1);
487 if (cancelled) {
488 atomic_cmpxchg(&dwork->work.state,
489 WORK_ST_CANCEL, WORK_ST_IDLE);
490 mtx_unlock(&dwork->timer.mtx);
491 return (true);
492 }
493 /* FALLTHROUGH */
494 case WORK_ST_TASK:
495 tq = dwork->work.work_queue->taskqueue;
496 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) {
497 atomic_cmpxchg(&dwork->work.state,
498 WORK_ST_CANCEL, WORK_ST_IDLE);
499 mtx_unlock(&dwork->timer.mtx);
500 return (true);
501 }
502 /* FALLTHROUGH */
503 default:
504 mtx_unlock(&dwork->timer.mtx);
505 return (false);
506 }
507 }
508
509 /*
510 * Cancel and drain delayed work, reporting cancelled pending work through
511 * cancelled and the taskqueue pending count through pending.
512 *
513 * Return true if the caller needs to re-check the state after draining.
514 */
515 static bool
linux_cancel_delayed_work_sync_int(struct delayed_work * dwork,u_int * pending,bool * cancelled)516 linux_cancel_delayed_work_sync_int(struct delayed_work *dwork, u_int *pending,
517 bool *cancelled)
518 {
519 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
520 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
521 [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */
522 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
523 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
524 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
525 };
526 struct taskqueue *tq;
527 int ret, state;
528 bool stopped;
529
530 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
531 "linux_cancel_delayed_work_sync() might sleep");
532 mtx_lock(&dwork->timer.mtx);
533
534 state = linux_update_state(&dwork->work.state, states);
535 /*
536 * A taskqueue worker can have dequeued the task without linux_work_fn()
537 * having claimed the callback yet. In that case taskqueue_cancel()
538 * reports EBUSY with no pending tasks, but our transition to IDLE still
539 * prevents the callback. Account for this using the state consumed by
540 * the atomic transition, not just the native queue's pending count.
541 * WORK_ST_EXEC, in contrast, means the callback was already claimed.
542 */
543 *cancelled = (state == WORK_ST_TIMER || state == WORK_ST_TASK ||
544 state == WORK_ST_CANCEL);
545 switch (state) {
546 case WORK_ST_IDLE:
547 mtx_unlock(&dwork->timer.mtx);
548 return (false);
549 case WORK_ST_TIMER:
550 case WORK_ST_CANCEL:
551 stopped = (callout_stop(&dwork->timer.callout) == 1);
552
553 tq = dwork->work.work_queue->taskqueue;
554 ret = taskqueue_cancel(tq, &dwork->work.work_task, pending);
555 mtx_unlock(&dwork->timer.mtx);
556
557 callout_drain(&dwork->timer.callout);
558 taskqueue_drain(tq, &dwork->work.work_task);
559 return (stopped || (ret != 0));
560 default:
561 tq = dwork->work.work_queue->taskqueue;
562 ret = taskqueue_cancel(tq, &dwork->work.work_task, pending);
563 mtx_unlock(&dwork->timer.mtx);
564 if (ret != 0)
565 taskqueue_drain(tq, &dwork->work.work_task);
566 return (ret != 0);
567 }
568 }
569
570 bool
linux_cancel_delayed_work_sync(struct delayed_work * dwork)571 linux_cancel_delayed_work_sync(struct delayed_work *dwork)
572 {
573 u_int pending;
574 bool cancelled;
575 bool res = false;
576 bool ret;
577
578 do {
579 pending = 0;
580 cancelled = false;
581 ret = linux_cancel_delayed_work_sync_int(dwork, &pending,
582 &cancelled);
583 res = res || cancelled || pending != 0;
584 } while (ret);
585
586 return (res);
587 }
588
589 /*
590 * This function waits until the given work structure is completed.
591 * It returns non-zero if the work was successfully
592 * waited for. Else the work was not waited for.
593 */
594 bool
linux_flush_work(struct work_struct * work)595 linux_flush_work(struct work_struct *work)
596 {
597 struct taskqueue *tq;
598 bool retval;
599
600 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
601 "linux_flush_work() might sleep");
602
603 switch (atomic_read(&work->state)) {
604 case WORK_ST_IDLE:
605 return (false);
606 default:
607 tq = work->work_queue->taskqueue;
608 retval = taskqueue_poll_is_busy(tq, &work->work_task);
609 taskqueue_drain(tq, &work->work_task);
610 return (retval);
611 }
612 }
613
614 /*
615 * This function waits until the given delayed work structure is
616 * completed. It returns non-zero if the work was successfully waited
617 * for. Else the work was not waited for.
618 */
619 bool
linux_flush_delayed_work(struct delayed_work * dwork)620 linux_flush_delayed_work(struct delayed_work *dwork)
621 {
622 struct taskqueue *tq;
623 bool retval;
624
625 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
626 "linux_flush_delayed_work() might sleep");
627
628 switch (atomic_read(&dwork->work.state)) {
629 case WORK_ST_IDLE:
630 return (false);
631 case WORK_ST_TIMER:
632 if (linux_cancel_timer(dwork, 1))
633 linux_delayed_work_enqueue(dwork);
634 /* FALLTHROUGH */
635 default:
636 tq = dwork->work.work_queue->taskqueue;
637 retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task);
638 taskqueue_drain(tq, &dwork->work.work_task);
639 return (retval);
640 }
641 }
642
643 /*
644 * This function returns true if the given work is pending, and not
645 * yet executing:
646 */
647 bool
linux_work_pending(struct work_struct * work)648 linux_work_pending(struct work_struct *work)
649 {
650 switch (atomic_read(&work->state)) {
651 case WORK_ST_TIMER:
652 case WORK_ST_TASK:
653 case WORK_ST_CANCEL:
654 return (true);
655 default:
656 return (false);
657 }
658 }
659
660 /*
661 * This function returns true if the given work is busy.
662 */
663 bool
linux_work_busy(struct work_struct * work)664 linux_work_busy(struct work_struct *work)
665 {
666 struct taskqueue *tq;
667
668 switch (atomic_read(&work->state)) {
669 case WORK_ST_IDLE:
670 return (false);
671 case WORK_ST_EXEC:
672 tq = work->work_queue->taskqueue;
673 return (taskqueue_poll_is_busy(tq, &work->work_task));
674 default:
675 return (true);
676 }
677 }
678
679 struct workqueue_struct *
linux_create_workqueue_common(const char * name,int cpus)680 linux_create_workqueue_common(const char *name, int cpus)
681 {
682 struct workqueue_struct *wq;
683
684 /*
685 * If zero CPUs are specified use the default number of CPUs:
686 */
687 if (cpus == 0)
688 cpus = linux_default_wq_cpus;
689
690 wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO);
691 wq->taskqueue = taskqueue_create(name, M_WAITOK,
692 taskqueue_thread_enqueue, &wq->taskqueue);
693 atomic_set(&wq->draining, 0);
694 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
695 TAILQ_INIT(&wq->exec_head);
696 mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF);
697
698 return (wq);
699 }
700
701 void
linux_destroy_workqueue(struct workqueue_struct * wq)702 linux_destroy_workqueue(struct workqueue_struct *wq)
703 {
704 atomic_inc(&wq->draining);
705 drain_workqueue(wq);
706 taskqueue_free(wq->taskqueue);
707 mtx_destroy(&wq->exec_mtx);
708 kfree(wq);
709 }
710
711 void
linux_init_delayed_work(struct delayed_work * dwork,work_func_t func)712 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func)
713 {
714 memset(dwork, 0, sizeof(*dwork));
715 dwork->work.func = func;
716 TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork);
717 mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL,
718 MTX_DEF | MTX_NOWITNESS);
719 callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0);
720 }
721
722 struct work_struct *
linux_current_work(void)723 linux_current_work(void)
724 {
725 return (current->work);
726 }
727
728 static void
linux_work_init(void * arg)729 linux_work_init(void *arg)
730 {
731 int max_wq_cpus = mp_ncpus + 1;
732
733 /* avoid deadlock when there are too few threads */
734 if (max_wq_cpus < 4)
735 max_wq_cpus = 4;
736
737 /* set default number of CPUs */
738 linux_default_wq_cpus = max_wq_cpus;
739
740 linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus);
741 linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus);
742
743 /* populate the workqueue pointers */
744 system_long_wq = linux_system_long_wq;
745 /*
746 * With Linux v6.17 system_wq was "renamed" to system_percpu_wq with the
747 * old name staying around.
748 * Note: neither implementation here does fully implement the per-cpu
749 * characteristics upstream expects.
750 */
751 system_wq = linux_system_short_wq;
752 system_percpu_wq = linux_system_short_wq;
753 system_power_efficient_wq = linux_system_short_wq;
754 system_unbound_wq = linux_system_short_wq;
755 system_highpri_wq = linux_system_short_wq;
756 }
757 SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL);
758
759 static void
linux_work_uninit(void * arg)760 linux_work_uninit(void *arg)
761 {
762 destroy_workqueue(linux_system_short_wq);
763 destroy_workqueue(linux_system_long_wq);
764
765 /* clear workqueue pointers */
766 system_long_wq = NULL;
767 system_wq = NULL;
768 system_percpu_wq = NULL;
769 system_power_efficient_wq = NULL;
770 system_unbound_wq = NULL;
771 system_highpri_wq = NULL;
772 }
773 SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL);
774
775 void
linux_irq_work_fn(void * context,int pending)776 linux_irq_work_fn(void *context, int pending)
777 {
778 struct irq_work *irqw = context;
779
780 rcu_read_lock();
781 irqw->func(irqw);
782 rcu_read_unlock();
783 }
784
785 static void
linux_irq_work_init_fn(void * context,int pending)786 linux_irq_work_init_fn(void *context, int pending)
787 {
788 /*
789 * LinuxKPI performs lazy allocation of memory structures required by
790 * current on the first access to it. As some irq_work clients read
791 * it with spinlock taken, we have to preallocate td_lkpi_task before
792 * first call to irq_work_queue(). As irq_work uses a single thread,
793 * it is enough to read current once at SYSINIT stage.
794 */
795 if (current == NULL)
796 panic("irq_work taskqueue is not initialized");
797 }
798 static struct task linux_irq_work_init_task =
799 TASK_INITIALIZER(0, linux_irq_work_init_fn, &linux_irq_work_init_task);
800
801 static void
linux_irq_work_init(void * arg)802 linux_irq_work_init(void *arg)
803 {
804 linux_irq_work_tq = taskqueue_create_fast("linuxkpi_irq_wq",
805 M_WAITOK, taskqueue_thread_enqueue, &linux_irq_work_tq);
806 taskqueue_start_threads(&linux_irq_work_tq, 1, PWAIT,
807 "linuxkpi_irq_wq");
808 taskqueue_enqueue(linux_irq_work_tq, &linux_irq_work_init_task);
809 }
810 SYSINIT(linux_irq_work_init, SI_SUB_TASKQ, SI_ORDER_SECOND,
811 linux_irq_work_init, NULL);
812
813 static void
linux_irq_work_uninit(void * arg)814 linux_irq_work_uninit(void *arg)
815 {
816 /* taskqueue_drain_all() executes synchronize_rcu() implicitly */
817 taskqueue_drain_all(linux_irq_work_tq);
818 taskqueue_free(linux_irq_work_tq);
819 }
820 SYSUNINIT(linux_irq_work_uninit, SI_SUB_TASKQ, SI_ORDER_SECOND,
821 linux_irq_work_uninit, NULL);
822