1 /*- 2 * Copyright (c) 2017 Hans Petter Selasky 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <linux/compat.h> 31 #include <linux/completion.h> 32 #include <linux/mm.h> 33 #include <linux/kthread.h> 34 #include <linux/moduleparam.h> 35 36 #include <sys/kernel.h> 37 #include <sys/eventhandler.h> 38 #include <sys/malloc.h> 39 #include <sys/sysctl.h> 40 #include <vm/uma.h> 41 42 #if defined(__i386__) || defined(__amd64__) 43 extern u_int first_msi_irq, num_msi_irqs; 44 #endif 45 46 static eventhandler_tag linuxkpi_thread_dtor_tag; 47 48 static atomic_t linux_current_allocs; 49 static uma_zone_t linux_current_zone; 50 static uma_zone_t linux_mm_zone; 51 52 /* check if another thread already has a mm_struct */ 53 static struct mm_struct * 54 find_other_mm(struct proc *p) 55 { 56 struct thread *td; 57 struct task_struct *ts; 58 struct mm_struct *mm; 59 60 PROC_LOCK_ASSERT(p, MA_OWNED); 61 FOREACH_THREAD_IN_PROC(p, td) { 62 ts = td->td_lkpi_task; 63 if (ts == NULL) 64 continue; 65 mm = ts->mm; 66 if (mm == NULL) 67 continue; 68 /* try to share other mm_struct */ 69 if (atomic_inc_not_zero(&mm->mm_users)) 70 return (mm); 71 } 72 return (NULL); 73 } 74 75 int 76 linux_alloc_current(struct thread *td, int flags) 77 { 78 struct proc *proc; 79 struct task_struct *ts; 80 struct mm_struct *mm, *mm_other; 81 82 MPASS(td->td_lkpi_task == NULL); 83 84 if ((td->td_pflags & TDP_ITHREAD) != 0 || !THREAD_CAN_SLEEP()) { 85 flags &= ~M_WAITOK; 86 flags |= M_NOWAIT | M_USE_RESERVE; 87 } 88 89 ts = uma_zalloc(linux_current_zone, flags | M_ZERO); 90 if (ts == NULL) { 91 if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 92 panic("linux_alloc_current: failed to allocate task"); 93 return (ENOMEM); 94 } 95 mm = NULL; 96 97 /* setup new task structure */ 98 atomic_set(&ts->kthread_flags, 0); 99 ts->task_thread = td; 100 ts->comm = td->td_name; 101 ts->pid = td->td_tid; 102 ts->group_leader = ts; 103 atomic_set(&ts->usage, 1); 104 atomic_set(&ts->state, TASK_RUNNING); 105 init_completion(&ts->parked); 106 init_completion(&ts->exited); 107 108 proc = td->td_proc; 109 110 PROC_LOCK(proc); 111 mm_other = find_other_mm(proc); 112 113 /* use allocated mm_struct as a fallback */ 114 if (mm_other == NULL) { 115 PROC_UNLOCK(proc); 116 mm = uma_zalloc(linux_mm_zone, flags | M_ZERO); 117 if (mm == NULL) { 118 if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 119 panic( 120 "linux_alloc_current: failed to allocate mm"); 121 uma_zfree(linux_current_zone, mm); 122 return (ENOMEM); 123 } 124 125 PROC_LOCK(proc); 126 mm_other = find_other_mm(proc); 127 if (mm_other == NULL) { 128 /* setup new mm_struct */ 129 init_rwsem(&mm->mmap_sem); 130 atomic_set(&mm->mm_count, 1); 131 atomic_set(&mm->mm_users, 1); 132 /* set mm_struct pointer */ 133 ts->mm = mm; 134 /* clear pointer to not free memory */ 135 mm = NULL; 136 } else { 137 ts->mm = mm_other; 138 } 139 } else { 140 ts->mm = mm_other; 141 } 142 143 /* store pointer to task struct */ 144 td->td_lkpi_task = ts; 145 PROC_UNLOCK(proc); 146 147 /* free mm_struct pointer, if any */ 148 uma_zfree(linux_mm_zone, mm); 149 150 /* keep track of number of allocations */ 151 if (atomic_add_return(1, &linux_current_allocs) == INT_MAX) 152 panic("linux_alloc_current: Refcount too high!"); 153 154 return (0); 155 } 156 157 struct mm_struct * 158 linux_get_task_mm(struct task_struct *task) 159 { 160 struct mm_struct *mm; 161 162 mm = task->mm; 163 if (mm != NULL) { 164 atomic_inc(&mm->mm_users); 165 return (mm); 166 } 167 return (NULL); 168 } 169 170 void 171 linux_mm_dtor(struct mm_struct *mm) 172 { 173 uma_zfree(linux_mm_zone, mm); 174 } 175 176 void 177 linux_free_current(struct task_struct *ts) 178 { 179 mmput(ts->mm); 180 uma_zfree(linux_current_zone, ts); 181 182 /* keep track of number of allocations */ 183 if (atomic_sub_return(1, &linux_current_allocs) < 0) 184 panic("linux_free_current: Negative refcount!"); 185 } 186 187 static void 188 linuxkpi_thread_dtor(void *arg __unused, struct thread *td) 189 { 190 struct task_struct *ts; 191 192 ts = td->td_lkpi_task; 193 if (ts == NULL) 194 return; 195 196 td->td_lkpi_task = NULL; 197 put_task_struct(ts); 198 } 199 200 static struct task_struct * 201 linux_get_pid_task_int(pid_t pid, const bool do_get) 202 { 203 struct thread *td; 204 struct proc *p; 205 struct task_struct *ts; 206 207 if (pid > PID_MAX) { 208 /* try to find corresponding thread */ 209 td = tdfind(pid, -1); 210 if (td != NULL) { 211 ts = td->td_lkpi_task; 212 if (do_get && ts != NULL) 213 get_task_struct(ts); 214 PROC_UNLOCK(td->td_proc); 215 return (ts); 216 } 217 } else { 218 /* try to find corresponding procedure */ 219 p = pfind(pid); 220 if (p != NULL) { 221 FOREACH_THREAD_IN_PROC(p, td) { 222 ts = td->td_lkpi_task; 223 if (ts != NULL) { 224 if (do_get) 225 get_task_struct(ts); 226 PROC_UNLOCK(p); 227 return (ts); 228 } 229 } 230 PROC_UNLOCK(p); 231 } 232 } 233 return (NULL); 234 } 235 236 struct task_struct * 237 linux_pid_task(pid_t pid) 238 { 239 return (linux_get_pid_task_int(pid, false)); 240 } 241 242 struct task_struct * 243 linux_get_pid_task(pid_t pid) 244 { 245 return (linux_get_pid_task_int(pid, true)); 246 } 247 248 bool 249 linux_task_exiting(struct task_struct *task) 250 { 251 struct thread *td; 252 struct proc *p; 253 bool ret; 254 255 ret = false; 256 257 /* try to find corresponding thread */ 258 td = tdfind(task->pid, -1); 259 if (td != NULL) { 260 p = td->td_proc; 261 } else { 262 /* try to find corresponding procedure */ 263 p = pfind(task->pid); 264 } 265 266 if (p != NULL) { 267 if ((p->p_flag & P_WEXIT) != 0) 268 ret = true; 269 PROC_UNLOCK(p); 270 } 271 return (ret); 272 } 273 274 static int lkpi_task_resrv; 275 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, task_struct_reserve, 276 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lkpi_task_resrv, 0, 277 "Number of struct task and struct mm to reserve for non-sleepable " 278 "allocations"); 279 280 static void 281 linux_current_init(void *arg __unused) 282 { 283 TUNABLE_INT_FETCH("compat.linuxkpi.task_struct_reserve", 284 &lkpi_task_resrv); 285 if (lkpi_task_resrv == 0) { 286 #if defined(__i386__) || defined(__amd64__) 287 /* 288 * Number of interrupt threads plus per-cpu callout 289 * SWI threads. 290 */ 291 lkpi_task_resrv = first_msi_irq + num_msi_irqs + MAXCPU; 292 #else 293 lkpi_task_resrv = 1024; /* XXXKIB arbitrary */ 294 #endif 295 } 296 linux_current_zone = uma_zcreate("lkpicurr", 297 sizeof(struct task_struct), NULL, NULL, NULL, NULL, 298 UMA_ALIGN_PTR, 0); 299 uma_zone_reserve(linux_current_zone, lkpi_task_resrv); 300 uma_prealloc(linux_current_zone, lkpi_task_resrv); 301 linux_mm_zone = uma_zcreate("lkpimm", 302 sizeof(struct task_struct), NULL, NULL, NULL, NULL, 303 UMA_ALIGN_PTR, 0); 304 uma_zone_reserve(linux_mm_zone, lkpi_task_resrv); 305 uma_prealloc(linux_mm_zone, lkpi_task_resrv); 306 307 atomic_thread_fence_seq_cst(); 308 309 lkpi_alloc_current = linux_alloc_current; 310 linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 311 linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 312 } 313 SYSINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, 314 linux_current_init, NULL); 315 316 static void 317 linux_current_uninit(void *arg __unused) 318 { 319 struct proc *p; 320 struct task_struct *ts; 321 struct thread *td; 322 323 lkpi_alloc_current = linux_alloc_current_noop; 324 325 atomic_thread_fence_seq_cst(); 326 327 sx_slock(&allproc_lock); 328 FOREACH_PROC_IN_SYSTEM(p) { 329 PROC_LOCK(p); 330 FOREACH_THREAD_IN_PROC(p, td) { 331 if ((ts = td->td_lkpi_task) != NULL) { 332 td->td_lkpi_task = NULL; 333 put_task_struct(ts); 334 } 335 } 336 PROC_UNLOCK(p); 337 } 338 sx_sunlock(&allproc_lock); 339 340 /* 341 * There is a window where threads are removed from the 342 * process list and where the thread destructor is invoked. 343 * Catch that window by waiting for all task_struct 344 * allocations to be returned before freeing the UMA zone. 345 */ 346 while (atomic_read(&linux_current_allocs) != 0) 347 pause("W", 1); 348 349 EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); 350 351 uma_zdestroy(linux_current_zone); 352 uma_zdestroy(linux_mm_zone); 353 } 354 SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, 355 linux_current_uninit, NULL); 356