1 /* 2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. 3 * Copyright (C) 2007 The Regents of the University of California. 4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 6 * UCRL-CODE-235197 7 * 8 * This file is part of the SPL, Solaris Porting Layer. 9 * 10 * The SPL is free software; you can redistribute it and/or modify it 11 * under the terms of the GNU General Public License as published by the 12 * Free Software Foundation; either version 2 of the License, or (at your 13 * option) any later version. 14 * 15 * The SPL is distributed in the hope that it will be useful, but WITHOUT 16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 18 * for more details. 19 * 20 * You should have received a copy of the GNU General Public License along 21 * with the SPL. If not, see <http://www.gnu.org/licenses/>. 22 */ 23 /* 24 * Copyright (c) 2024, Klara Inc. 25 * Copyright (c) 2024, Syneto 26 */ 27 28 #ifndef _SPL_TASKQ_H 29 #define _SPL_TASKQ_H 30 31 #include <linux/module.h> 32 #include <linux/gfp.h> 33 #include <linux/slab.h> 34 #include <linux/interrupt.h> 35 #include <linux/kthread.h> 36 #include <sys/types.h> 37 #include <sys/thread.h> 38 #include <sys/rwlock.h> 39 #include <sys/wait.h> 40 #include <sys/wmsum.h> 41 #include <sys/kstat.h> 42 43 #define TASKQ_NAMELEN 31 44 45 #define TASKQ_PREPOPULATE 0x00000001 46 #define TASKQ_CPR_SAFE 0x00000002 47 #define TASKQ_DYNAMIC 0x00000004 48 #define TASKQ_THREADS_CPU_PCT 0x00000008 49 #define TASKQ_DC_BATCH 0x00000010 50 #define TASKQ_ACTIVE 0x80000000 51 52 /* 53 * Flags for taskq_dispatch. TQ_SLEEP/TQ_NOSLEEP should be same as 54 * KM_SLEEP/KM_NOSLEEP. TQ_NOQUEUE/TQ_NOALLOC are set particularly 55 * large so as not to conflict with already used GFP_* defines. 56 */ 57 #define TQ_SLEEP 0x00000000 58 #define TQ_NOSLEEP 0x00000001 59 #define TQ_PUSHPAGE 0x00000002 60 #define TQ_NOQUEUE 0x01000000 61 #define TQ_NOALLOC 0x02000000 62 #define TQ_NEW 0x04000000 63 #define TQ_FRONT 0x08000000 64 65 /* 66 * Reserved taskqid values. 67 */ 68 #define TASKQID_INVALID ((taskqid_t)0) 69 #define TASKQID_INITIAL ((taskqid_t)1) 70 71 /* 72 * spin_lock(lock) and spin_lock_nested(lock,0) are equivalent, 73 * so TQ_LOCK_DYNAMIC must not evaluate to 0 74 */ 75 typedef enum tq_lock_role { 76 TQ_LOCK_GENERAL = 0, 77 TQ_LOCK_DYNAMIC = 1, 78 } tq_lock_role_t; 79 80 typedef unsigned long taskqid_t; 81 typedef void (task_func_t)(void *); 82 83 typedef struct taskq_sums { 84 /* gauges (inc/dec counters, current value) */ 85 wmsum_t tqs_threads_active; /* threads running a task */ 86 wmsum_t tqs_threads_idle; /* threads waiting for work */ 87 wmsum_t tqs_threads_total; /* total threads */ 88 wmsum_t tqs_tasks_pending; /* tasks waiting to execute */ 89 wmsum_t tqs_tasks_priority; /* hi-pri tasks waiting */ 90 wmsum_t tqs_tasks_total; /* total waiting tasks */ 91 wmsum_t tqs_tasks_delayed; /* tasks deferred to future */ 92 wmsum_t tqs_entries_free; /* task entries on free list */ 93 94 /* counters (inc only, since taskq creation) */ 95 wmsum_t tqs_threads_created; /* threads created */ 96 wmsum_t tqs_threads_destroyed; /* threads destroyed */ 97 wmsum_t tqs_tasks_dispatched; /* tasks dispatched */ 98 wmsum_t tqs_tasks_dispatched_delayed; /* tasks delayed to future */ 99 wmsum_t tqs_tasks_executed_normal; /* normal pri tasks executed */ 100 wmsum_t tqs_tasks_executed_priority; /* high pri tasks executed */ 101 wmsum_t tqs_tasks_executed; /* total tasks executed */ 102 wmsum_t tqs_tasks_delayed_requeued; /* delayed tasks requeued */ 103 wmsum_t tqs_tasks_cancelled; /* tasks cancelled before run */ 104 wmsum_t tqs_thread_wakeups; /* total thread wakeups */ 105 wmsum_t tqs_thread_wakeups_nowork; /* thread woken but no tasks */ 106 wmsum_t tqs_thread_sleeps; /* total thread sleeps */ 107 } taskq_sums_t; 108 109 typedef struct taskq { 110 spinlock_t tq_lock; /* protects taskq_t */ 111 char *tq_name; /* taskq name */ 112 int tq_instance; /* instance of tq_name */ 113 struct list_head tq_thread_list; /* list of all threads */ 114 struct list_head tq_active_list; /* list of active threads */ 115 int tq_nactive; /* # of active threads */ 116 int tq_nthreads; /* # of existing threads */ 117 int tq_nspawn; /* # of threads being spawned */ 118 int tq_maxthreads; /* # of threads maximum */ 119 /* If PERCPU flag is set, percent of NCPUs to have as threads */ 120 int tq_cpu_pct; 121 int tq_pri; /* priority */ 122 int tq_minalloc; /* min taskq_ent_t pool size */ 123 int tq_maxalloc; /* max taskq_ent_t pool size */ 124 int tq_nalloc; /* cur taskq_ent_t pool size */ 125 uint_t tq_flags; /* flags */ 126 taskqid_t tq_next_id; /* next pend/work id */ 127 taskqid_t tq_lowest_id; /* lowest pend/work id */ 128 struct list_head tq_free_list; /* free taskq_ent_t's */ 129 struct list_head tq_pend_list; /* pending taskq_ent_t's */ 130 struct list_head tq_prio_list; /* priority taskq_ent_t's */ 131 struct list_head tq_delay_list; /* delayed taskq_ent_t's */ 132 struct list_head tq_taskqs; /* all taskq_t's */ 133 wait_queue_head_t tq_work_waitq; /* new work waitq */ 134 wait_queue_head_t tq_wait_waitq; /* wait waitq */ 135 tq_lock_role_t tq_lock_class; /* class when taking tq_lock */ 136 /* list node for the cpu hotplug callback */ 137 struct hlist_node tq_hp_cb_node; 138 boolean_t tq_hp_support; 139 unsigned long lastspawnstop; /* when to purge dynamic */ 140 taskq_sums_t tq_sums; 141 kstat_t *tq_ksp; 142 } taskq_t; 143 144 typedef struct taskq_ent { 145 spinlock_t tqent_lock; 146 wait_queue_head_t tqent_waitq; 147 struct timer_list tqent_timer; 148 struct list_head tqent_list; 149 taskqid_t tqent_id; 150 task_func_t *tqent_func; 151 void *tqent_arg; 152 taskq_t *tqent_taskq; 153 uintptr_t tqent_flags; 154 unsigned long tqent_birth; 155 } taskq_ent_t; 156 157 #define TQENT_FLAG_PREALLOC 0x1 158 #define TQENT_FLAG_CANCEL 0x2 159 160 /* bits 2-3 are which list tqent is on */ 161 #define TQENT_LIST_NONE 0x0 162 #define TQENT_LIST_PENDING 0x4 163 #define TQENT_LIST_PRIORITY 0x8 164 #define TQENT_LIST_DELAY 0xc 165 #define TQENT_LIST_MASK 0xc 166 167 typedef struct taskq_thread { 168 struct list_head tqt_thread_list; 169 struct list_head tqt_active_list; 170 struct task_struct *tqt_thread; 171 taskq_t *tqt_tq; 172 taskqid_t tqt_id; 173 taskq_ent_t *tqt_task; 174 uintptr_t tqt_flags; 175 } taskq_thread_t; 176 177 /* Global system-wide dynamic task queue available for all consumers */ 178 extern taskq_t *system_taskq; 179 /* Global dynamic task queue for long delay */ 180 extern taskq_t *system_delay_taskq; 181 182 /* List of all taskqs */ 183 extern struct list_head tq_list; 184 extern struct rw_semaphore tq_list_sem; 185 186 extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t); 187 extern taskqid_t taskq_dispatch_delay(taskq_t *, task_func_t, void *, 188 uint_t, clock_t); 189 extern void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, 190 taskq_ent_t *); 191 extern int taskq_empty_ent(taskq_ent_t *); 192 extern void taskq_init_ent(taskq_ent_t *); 193 extern taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t); 194 extern taskq_t *taskq_create_synced(const char *, int, pri_t, int, int, uint_t, 195 kthread_t ***); 196 extern void taskq_destroy(taskq_t *); 197 extern void taskq_wait_id(taskq_t *, taskqid_t); 198 extern void taskq_wait_outstanding(taskq_t *, taskqid_t); 199 extern void taskq_wait(taskq_t *); 200 extern int taskq_cancel_id(taskq_t *, taskqid_t); 201 extern int taskq_member(taskq_t *, kthread_t *); 202 extern taskq_t *taskq_of_curthread(void); 203 204 #define taskq_create_proc(name, nthreads, pri, min, max, proc, flags) \ 205 taskq_create(name, nthreads, pri, min, max, flags) 206 #define taskq_create_sysdc(name, nthreads, min, max, proc, dc, flags) \ 207 ((void) sizeof (dc), \ 208 taskq_create(name, nthreads, maxclsyspri, min, max, flags)) 209 210 int spl_taskq_init(void); 211 void spl_taskq_fini(void); 212 213 #endif /* _SPL_TASKQ_H */ 214