1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. 4 * Copyright (C) 2007 The Regents of the University of California. 5 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 6 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 7 * UCRL-CODE-235197 8 * 9 * This file is part of the SPL, Solaris Porting Layer. 10 * 11 * The SPL is free software; you can redistribute it and/or modify it 12 * under the terms of the GNU General Public License as published by the 13 * Free Software Foundation; either version 2 of the License, or (at your 14 * option) any later version. 15 * 16 * The SPL is distributed in the hope that it will be useful, but WITHOUT 17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 19 * for more details. 20 * 21 * You should have received a copy of the GNU General Public License along 22 * with the SPL. If not, see <http://www.gnu.org/licenses/>. 23 */ 24 /* 25 * Copyright (c) 2024, Klara Inc. 26 * Copyright (c) 2024, Syneto 27 */ 28 29 #ifndef _SPL_TASKQ_H 30 #define _SPL_TASKQ_H 31 32 #include <linux/module.h> 33 #include <linux/gfp.h> 34 #include <linux/slab.h> 35 #include <linux/interrupt.h> 36 #include <linux/kthread.h> 37 #include <sys/types.h> 38 #include <sys/thread.h> 39 #include <sys/rwlock.h> 40 #include <sys/wait.h> 41 #include <sys/wmsum.h> 42 #include <sys/kstat.h> 43 44 #define TASKQ_NAMELEN 31 45 46 #define TASKQ_PREPOPULATE 0x00000001 47 #define TASKQ_CPR_SAFE 0x00000002 48 #define TASKQ_DYNAMIC 0x00000004 49 #define TASKQ_THREADS_CPU_PCT 0x00000008 50 #define TASKQ_DC_BATCH 0x00000010 51 #define TASKQ_ACTIVE 0x80000000 52 53 /* 54 * Flags for taskq_dispatch. TQ_SLEEP/TQ_NOSLEEP should be same as 55 * KM_SLEEP/KM_NOSLEEP. TQ_NOQUEUE/TQ_NOALLOC are set particularly 56 * large so as not to conflict with already used GFP_* defines. 57 */ 58 #define TQ_SLEEP 0x00000000 59 #define TQ_NOSLEEP 0x00000001 60 #define TQ_PUSHPAGE 0x00000002 61 #define TQ_NOQUEUE 0x01000000 62 #define TQ_NOALLOC 0x02000000 63 #define TQ_NEW 0x04000000 64 #define TQ_FRONT 0x08000000 65 66 /* 67 * Reserved taskqid values. 68 */ 69 #define TASKQID_INVALID ((taskqid_t)0) 70 #define TASKQID_INITIAL ((taskqid_t)1) 71 72 /* 73 * spin_lock(lock) and spin_lock_nested(lock,0) are equivalent, 74 * so TQ_LOCK_DYNAMIC must not evaluate to 0 75 */ 76 typedef enum tq_lock_role { 77 TQ_LOCK_GENERAL = 0, 78 TQ_LOCK_DYNAMIC = 1, 79 } tq_lock_role_t; 80 81 typedef unsigned long taskqid_t; 82 typedef void (task_func_t)(void *); 83 84 typedef struct taskq_sums { 85 /* gauges (inc/dec counters, current value) */ 86 wmsum_t tqs_threads_active; /* threads running a task */ 87 wmsum_t tqs_threads_idle; /* threads waiting for work */ 88 wmsum_t tqs_threads_total; /* total threads */ 89 wmsum_t tqs_tasks_pending; /* tasks waiting to execute */ 90 wmsum_t tqs_tasks_priority; /* hi-pri tasks waiting */ 91 wmsum_t tqs_tasks_total; /* total waiting tasks */ 92 wmsum_t tqs_tasks_delayed; /* tasks deferred to future */ 93 wmsum_t tqs_entries_free; /* task entries on free list */ 94 95 /* counters (inc only, since taskq creation) */ 96 wmsum_t tqs_threads_created; /* threads created */ 97 wmsum_t tqs_threads_destroyed; /* threads destroyed */ 98 wmsum_t tqs_tasks_dispatched; /* tasks dispatched */ 99 wmsum_t tqs_tasks_dispatched_delayed; /* tasks delayed to future */ 100 wmsum_t tqs_tasks_executed_normal; /* normal pri tasks executed */ 101 wmsum_t tqs_tasks_executed_priority; /* high pri tasks executed */ 102 wmsum_t tqs_tasks_executed; /* total tasks executed */ 103 wmsum_t tqs_tasks_delayed_requeued; /* delayed tasks requeued */ 104 wmsum_t tqs_tasks_cancelled; /* tasks cancelled before run */ 105 wmsum_t tqs_thread_wakeups; /* total thread wakeups */ 106 wmsum_t tqs_thread_wakeups_nowork; /* thread woken but no tasks */ 107 wmsum_t tqs_thread_sleeps; /* total thread sleeps */ 108 } taskq_sums_t; 109 110 typedef struct taskq { 111 spinlock_t tq_lock; /* protects taskq_t */ 112 char *tq_name; /* taskq name */ 113 int tq_instance; /* instance of tq_name */ 114 struct list_head tq_thread_list; /* list of all threads */ 115 struct list_head tq_active_list; /* list of active threads */ 116 int tq_nactive; /* # of active threads */ 117 int tq_nthreads; /* # of existing threads */ 118 int tq_nspawn; /* # of threads being spawned */ 119 int tq_maxthreads; /* # of threads maximum */ 120 /* If PERCPU flag is set, percent of NCPUs to have as threads */ 121 int tq_cpu_pct; 122 int tq_pri; /* priority */ 123 int tq_minalloc; /* min taskq_ent_t pool size */ 124 int tq_maxalloc; /* max taskq_ent_t pool size */ 125 int tq_nalloc; /* cur taskq_ent_t pool size */ 126 uint_t tq_flags; /* flags */ 127 taskqid_t tq_next_id; /* next pend/work id */ 128 taskqid_t tq_lowest_id; /* lowest pend/work id */ 129 struct list_head tq_free_list; /* free taskq_ent_t's */ 130 struct list_head tq_pend_list; /* pending taskq_ent_t's */ 131 struct list_head tq_prio_list; /* priority taskq_ent_t's */ 132 struct list_head tq_delay_list; /* delayed taskq_ent_t's */ 133 struct list_head tq_taskqs; /* all taskq_t's */ 134 wait_queue_head_t tq_work_waitq; /* new work waitq */ 135 wait_queue_head_t tq_wait_waitq; /* wait waitq */ 136 tq_lock_role_t tq_lock_class; /* class when taking tq_lock */ 137 /* list node for the cpu hotplug callback */ 138 struct hlist_node tq_hp_cb_node; 139 boolean_t tq_hp_support; 140 unsigned long lastspawnstop; /* when to purge dynamic */ 141 taskq_sums_t tq_sums; 142 kstat_t *tq_ksp; 143 } taskq_t; 144 145 typedef struct taskq_ent { 146 spinlock_t tqent_lock; 147 wait_queue_head_t tqent_waitq; 148 struct timer_list tqent_timer; 149 struct list_head tqent_list; 150 taskqid_t tqent_id; 151 task_func_t *tqent_func; 152 void *tqent_arg; 153 taskq_t *tqent_taskq; 154 uintptr_t tqent_flags; 155 unsigned long tqent_birth; 156 } taskq_ent_t; 157 158 #define TQENT_FLAG_PREALLOC 0x1 159 #define TQENT_FLAG_CANCEL 0x2 160 161 /* bits 2-3 are which list tqent is on */ 162 #define TQENT_LIST_NONE 0x0 163 #define TQENT_LIST_PENDING 0x4 164 #define TQENT_LIST_PRIORITY 0x8 165 #define TQENT_LIST_DELAY 0xc 166 #define TQENT_LIST_MASK 0xc 167 168 typedef struct taskq_thread { 169 struct list_head tqt_thread_list; 170 struct list_head tqt_active_list; 171 struct task_struct *tqt_thread; 172 taskq_t *tqt_tq; 173 taskqid_t tqt_id; 174 taskq_ent_t *tqt_task; 175 uintptr_t tqt_flags; 176 } taskq_thread_t; 177 178 /* Global system-wide dynamic task queue available for all consumers */ 179 extern taskq_t *system_taskq; 180 /* Global dynamic task queue for long delay */ 181 extern taskq_t *system_delay_taskq; 182 183 /* List of all taskqs */ 184 extern struct list_head tq_list; 185 extern struct rw_semaphore tq_list_sem; 186 187 extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t); 188 extern taskqid_t taskq_dispatch_delay(taskq_t *, task_func_t, void *, 189 uint_t, clock_t); 190 extern void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, 191 taskq_ent_t *); 192 extern int taskq_empty_ent(taskq_ent_t *); 193 extern void taskq_init_ent(taskq_ent_t *); 194 extern taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t); 195 extern taskq_t *taskq_create_synced(const char *, int, pri_t, int, int, uint_t, 196 kthread_t ***); 197 extern void taskq_destroy(taskq_t *); 198 extern void taskq_wait_id(taskq_t *, taskqid_t); 199 extern void taskq_wait_outstanding(taskq_t *, taskqid_t); 200 extern void taskq_wait(taskq_t *); 201 extern int taskq_cancel_id(taskq_t *, taskqid_t); 202 extern int taskq_member(taskq_t *, kthread_t *); 203 extern taskq_t *taskq_of_curthread(void); 204 205 #define taskq_create_proc(name, nthreads, pri, min, max, proc, flags) \ 206 taskq_create(name, nthreads, pri, min, max, flags) 207 #define taskq_create_sysdc(name, nthreads, min, max, proc, dc, flags) \ 208 ((void) sizeof (dc), \ 209 taskq_create(name, nthreads, maxclsyspri, min, max, flags)) 210 211 int spl_taskq_init(void); 212 void spl_taskq_fini(void); 213 214 #endif /* _SPL_TASKQ_H */ 215