1 /*- 2 * Copyright (c) 2017 Hans Petter Selasky 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/types.h> 31 #include <sys/malloc.h> 32 #include <sys/gtaskqueue.h> 33 #include <sys/proc.h> 34 #include <sys/sched.h> 35 36 #include <linux/compiler.h> 37 #include <linux/interrupt.h> 38 #include <linux/compat.h> 39 40 #define TASKLET_ST_IDLE 0 41 #define TASKLET_ST_BUSY 1 42 #define TASKLET_ST_EXEC 2 43 #define TASKLET_ST_LOOP 3 44 45 #define TASKLET_ST_CMPSET(ts, old, new) \ 46 atomic_cmpset_int((volatile u_int *)&(ts)->tasklet_state, old, new) 47 48 #define TASKLET_ST_SET(ts, new) \ 49 WRITE_ONCE(*(volatile u_int *)&(ts)->tasklet_state, new) 50 51 #define TASKLET_ST_GET(ts) \ 52 READ_ONCE(*(volatile u_int *)&(ts)->tasklet_state) 53 54 #define TASKLET_ST_TESTANDSET(ts, new) \ 55 atomic_testandset_int((volatile u_int *)&(ts)->tasklet_state, new) 56 57 struct tasklet_worker { 58 struct mtx mtx; 59 TAILQ_HEAD(tasklet_list, tasklet_struct) head; 60 struct grouptask gtask; 61 } __aligned(CACHE_LINE_SIZE); 62 63 #define TASKLET_WORKER_LOCK(tw) mtx_lock(&(tw)->mtx) 64 #define TASKLET_WORKER_UNLOCK(tw) mtx_unlock(&(tw)->mtx) 65 66 DPCPU_DEFINE_STATIC(struct tasklet_worker, tasklet_worker); 67 68 static void 69 tasklet_handler(void *arg) 70 { 71 struct tasklet_worker *tw = (struct tasklet_worker *)arg; 72 struct tasklet_struct *ts; 73 struct tasklet_struct *last; 74 75 linux_set_current(curthread); 76 77 TASKLET_WORKER_LOCK(tw); 78 last = TAILQ_LAST(&tw->head, tasklet_list); 79 while (1) { 80 ts = TAILQ_FIRST(&tw->head); 81 if (ts == NULL) 82 break; 83 TAILQ_REMOVE(&tw->head, ts, entry); 84 85 if (!atomic_read(&ts->count)) { 86 TASKLET_WORKER_UNLOCK(tw); 87 do { 88 /* reset executing state */ 89 TASKLET_ST_SET(ts, TASKLET_ST_EXEC); 90 91 ts->func(ts->data); 92 93 } while (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, 94 TASKLET_ST_IDLE) == 0); 95 TASKLET_WORKER_LOCK(tw); 96 } else { 97 TAILQ_INSERT_TAIL(&tw->head, ts, entry); 98 } 99 if (ts == last) 100 break; 101 } 102 TASKLET_WORKER_UNLOCK(tw); 103 } 104 105 static void 106 tasklet_subsystem_init(void *arg __unused) 107 { 108 struct tasklet_worker *tw; 109 char buf[32]; 110 int i; 111 112 CPU_FOREACH(i) { 113 if (CPU_ABSENT(i)) 114 continue; 115 116 tw = DPCPU_ID_PTR(i, tasklet_worker); 117 118 mtx_init(&tw->mtx, "linux_tasklet", NULL, MTX_DEF); 119 TAILQ_INIT(&tw->head); 120 GROUPTASK_INIT(&tw->gtask, 0, tasklet_handler, tw); 121 snprintf(buf, sizeof(buf), "softirq%d", i); 122 taskqgroup_attach_cpu(qgroup_softirq, &tw->gtask, 123 "tasklet", i, NULL, NULL, buf); 124 } 125 } 126 SYSINIT(linux_tasklet, SI_SUB_TASKQ, SI_ORDER_THIRD, tasklet_subsystem_init, NULL); 127 128 static void 129 tasklet_subsystem_uninit(void *arg __unused) 130 { 131 struct tasklet_worker *tw; 132 int i; 133 134 CPU_FOREACH(i) { 135 if (CPU_ABSENT(i)) 136 continue; 137 138 tw = DPCPU_ID_PTR(i, tasklet_worker); 139 140 taskqgroup_detach(qgroup_softirq, &tw->gtask); 141 mtx_destroy(&tw->mtx); 142 } 143 } 144 SYSUNINIT(linux_tasklet, SI_SUB_TASKQ, SI_ORDER_THIRD, tasklet_subsystem_uninit, NULL); 145 146 void 147 tasklet_init(struct tasklet_struct *ts, 148 tasklet_func_t *func, unsigned long data) 149 { 150 ts->entry.tqe_prev = NULL; 151 ts->entry.tqe_next = NULL; 152 ts->func = func; 153 ts->data = data; 154 atomic_set_int(&ts->tasklet_state, TASKLET_ST_IDLE); 155 atomic_set(&ts->count, 0); 156 } 157 158 void 159 local_bh_enable(void) 160 { 161 sched_unpin(); 162 } 163 164 void 165 local_bh_disable(void) 166 { 167 sched_pin(); 168 } 169 170 void 171 tasklet_schedule(struct tasklet_struct *ts) 172 { 173 174 /* tasklet is paused */ 175 if (atomic_read(&ts->count)) 176 return; 177 178 if (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, TASKLET_ST_LOOP)) { 179 /* tasklet_handler() will loop */ 180 } else if (TASKLET_ST_CMPSET(ts, TASKLET_ST_IDLE, TASKLET_ST_BUSY)) { 181 struct tasklet_worker *tw; 182 183 tw = &DPCPU_GET(tasklet_worker); 184 185 /* tasklet_handler() was not queued */ 186 TASKLET_WORKER_LOCK(tw); 187 /* enqueue tasklet */ 188 TAILQ_INSERT_TAIL(&tw->head, ts, entry); 189 /* schedule worker */ 190 GROUPTASK_ENQUEUE(&tw->gtask); 191 TASKLET_WORKER_UNLOCK(tw); 192 } else { 193 /* 194 * tasklet_handler() is already executing 195 * 196 * If the state is neither EXEC nor IDLE, it is either 197 * LOOP or BUSY. If the state changed between the two 198 * CMPSET's above the only possible transitions by 199 * elimination are LOOP->EXEC and BUSY->EXEC. If a 200 * EXEC->LOOP transition was missed that is not a 201 * problem because the callback function is then 202 * already about to be called again. 203 */ 204 } 205 } 206 207 void 208 tasklet_kill(struct tasklet_struct *ts) 209 { 210 211 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "tasklet_kill() can sleep"); 212 213 /* wait until tasklet is no longer busy */ 214 while (TASKLET_ST_GET(ts) != TASKLET_ST_IDLE) 215 pause("W", 1); 216 } 217 218 void 219 tasklet_enable(struct tasklet_struct *ts) 220 { 221 222 atomic_dec(&ts->count); 223 } 224 225 void 226 tasklet_disable(struct tasklet_struct *ts) 227 { 228 229 atomic_inc(&ts->count); 230 tasklet_unlock_wait(ts); 231 } 232 233 int 234 tasklet_trylock(struct tasklet_struct *ts) 235 { 236 237 return (!TASKLET_ST_TESTANDSET(ts, TASKLET_ST_BUSY)); 238 } 239 240 void 241 tasklet_unlock(struct tasklet_struct *ts) 242 { 243 244 TASKLET_ST_SET(ts, TASKLET_ST_IDLE); 245 } 246 247 void 248 tasklet_unlock_wait(struct tasklet_struct *ts) 249 { 250 251 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "tasklet_kill() can sleep"); 252 253 /* wait until tasklet is no longer busy */ 254 while (TASKLET_ST_GET(ts) != TASKLET_ST_IDLE) 255 pause("W", 1); 256 } 257