1 /*- 2 * Copyright (c) 2017 Hans Petter Selasky 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 #ifdef __amd64__ 29 #define DEV_APIC 30 #elif defined(__i386__) 31 #include "opt_apic.h" 32 #endif 33 34 #include <linux/compat.h> 35 #include <linux/completion.h> 36 #include <linux/mm.h> 37 #include <linux/kthread.h> 38 #include <linux/moduleparam.h> 39 40 #include <sys/kernel.h> 41 #include <sys/eventhandler.h> 42 #include <sys/malloc.h> 43 #include <sys/sysctl.h> 44 #include <vm/uma.h> 45 46 #if defined(__aarch64__) || defined(__arm__) || defined(__amd64__) || \ 47 defined(__i386__) 48 #include <machine/fpu.h> 49 #endif 50 51 #ifdef DEV_APIC 52 extern u_int first_msi_irq, num_msi_irqs; 53 #endif 54 55 static eventhandler_tag linuxkpi_thread_dtor_tag; 56 57 static uma_zone_t linux_current_zone; 58 static uma_zone_t linux_mm_zone; 59 60 /* check if another thread already has a mm_struct */ 61 static struct mm_struct * 62 find_other_mm(struct proc *p) 63 { 64 struct thread *td; 65 struct task_struct *ts; 66 struct mm_struct *mm; 67 68 PROC_LOCK_ASSERT(p, MA_OWNED); 69 FOREACH_THREAD_IN_PROC(p, td) { 70 ts = td->td_lkpi_task; 71 if (ts == NULL) 72 continue; 73 mm = ts->mm; 74 if (mm == NULL) 75 continue; 76 /* try to share other mm_struct */ 77 if (atomic_inc_not_zero(&mm->mm_users)) 78 return (mm); 79 } 80 return (NULL); 81 } 82 83 int 84 linux_alloc_current(struct thread *td, int flags) 85 { 86 struct proc *proc; 87 struct task_struct *ts; 88 struct mm_struct *mm, *mm_other; 89 90 MPASS(td->td_lkpi_task == NULL); 91 92 if ((td->td_pflags & TDP_ITHREAD) != 0 || !THREAD_CAN_SLEEP()) { 93 flags &= ~M_WAITOK; 94 flags |= M_NOWAIT | M_USE_RESERVE; 95 } 96 97 ts = uma_zalloc(linux_current_zone, flags | M_ZERO); 98 if (ts == NULL) { 99 if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 100 panic("linux_alloc_current: failed to allocate task"); 101 return (ENOMEM); 102 } 103 mm = NULL; 104 105 /* setup new task structure */ 106 atomic_set(&ts->kthread_flags, 0); 107 ts->task_thread = td; 108 ts->comm = td->td_name; 109 ts->pid = td->td_tid; 110 ts->group_leader = ts; 111 atomic_set(&ts->usage, 1); 112 atomic_set(&ts->state, TASK_RUNNING); 113 init_completion(&ts->parked); 114 init_completion(&ts->exited); 115 116 proc = td->td_proc; 117 118 PROC_LOCK(proc); 119 mm_other = find_other_mm(proc); 120 121 /* use allocated mm_struct as a fallback */ 122 if (mm_other == NULL) { 123 PROC_UNLOCK(proc); 124 mm = uma_zalloc(linux_mm_zone, flags | M_ZERO); 125 if (mm == NULL) { 126 if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 127 panic( 128 "linux_alloc_current: failed to allocate mm"); 129 uma_zfree(linux_current_zone, mm); 130 return (ENOMEM); 131 } 132 133 PROC_LOCK(proc); 134 mm_other = find_other_mm(proc); 135 if (mm_other == NULL) { 136 /* setup new mm_struct */ 137 init_rwsem(&mm->mmap_sem); 138 atomic_set(&mm->mm_count, 1); 139 atomic_set(&mm->mm_users, 1); 140 /* set mm_struct pointer */ 141 ts->mm = mm; 142 /* clear pointer to not free memory */ 143 mm = NULL; 144 } else { 145 ts->mm = mm_other; 146 } 147 } else { 148 ts->mm = mm_other; 149 } 150 151 /* store pointer to task struct */ 152 td->td_lkpi_task = ts; 153 PROC_UNLOCK(proc); 154 155 /* free mm_struct pointer, if any */ 156 uma_zfree(linux_mm_zone, mm); 157 158 return (0); 159 } 160 161 int 162 linux_set_fpu_ctx(struct task_struct *task) 163 { 164 #if defined(__aarch64__) || defined(__arm__) || defined(__amd64__) || \ 165 defined(__i386__) 166 if (task->fpu_ctx == NULL && curthread->td_critnest == 0) 167 task->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NOWAIT); 168 #endif 169 return (task->fpu_ctx != NULL ? 0 : ENOMEM); 170 } 171 172 struct mm_struct * 173 linux_get_task_mm(struct task_struct *task) 174 { 175 struct mm_struct *mm; 176 177 mm = task->mm; 178 if (mm != NULL) { 179 atomic_inc(&mm->mm_users); 180 return (mm); 181 } 182 return (NULL); 183 } 184 185 void 186 linux_mm_dtor(struct mm_struct *mm) 187 { 188 uma_zfree(linux_mm_zone, mm); 189 } 190 191 void 192 linux_free_current(struct task_struct *ts) 193 { 194 mmput(ts->mm); 195 #if defined(__aarch64__) || defined(__arm__) || defined(__amd64__) || \ 196 defined(__i386__) 197 if (ts->fpu_ctx != NULL) 198 fpu_kern_free_ctx(ts->fpu_ctx); 199 #endif 200 uma_zfree(linux_current_zone, ts); 201 } 202 203 static void 204 linuxkpi_thread_dtor(void *arg __unused, struct thread *td) 205 { 206 struct task_struct *ts; 207 208 ts = td->td_lkpi_task; 209 if (ts == NULL) 210 return; 211 212 td->td_lkpi_task = NULL; 213 put_task_struct(ts); 214 } 215 216 static struct task_struct * 217 linux_get_pid_task_int(pid_t pid, const bool do_get) 218 { 219 struct thread *td; 220 struct proc *p; 221 struct task_struct *ts; 222 223 if (pid > PID_MAX) { 224 /* try to find corresponding thread */ 225 td = tdfind(pid, -1); 226 if (td != NULL) { 227 ts = td->td_lkpi_task; 228 if (do_get && ts != NULL) 229 get_task_struct(ts); 230 PROC_UNLOCK(td->td_proc); 231 return (ts); 232 } 233 } else { 234 /* try to find corresponding procedure */ 235 p = pfind(pid); 236 if (p != NULL) { 237 FOREACH_THREAD_IN_PROC(p, td) { 238 ts = td->td_lkpi_task; 239 if (ts != NULL) { 240 if (do_get) 241 get_task_struct(ts); 242 PROC_UNLOCK(p); 243 return (ts); 244 } 245 } 246 PROC_UNLOCK(p); 247 } 248 } 249 return (NULL); 250 } 251 252 struct task_struct * 253 linux_pid_task(pid_t pid) 254 { 255 return (linux_get_pid_task_int(pid, false)); 256 } 257 258 struct task_struct * 259 linux_get_pid_task(pid_t pid) 260 { 261 return (linux_get_pid_task_int(pid, true)); 262 } 263 264 bool 265 linux_task_exiting(struct task_struct *task) 266 { 267 struct thread *td; 268 struct proc *p; 269 bool ret; 270 271 ret = false; 272 273 /* try to find corresponding thread */ 274 td = tdfind(task->pid, -1); 275 if (td != NULL) { 276 p = td->td_proc; 277 } else { 278 /* try to find corresponding procedure */ 279 p = pfind(task->pid); 280 } 281 282 if (p != NULL) { 283 if ((p->p_flag & P_WEXIT) != 0) 284 ret = true; 285 PROC_UNLOCK(p); 286 } 287 return (ret); 288 } 289 290 static int lkpi_task_resrv; 291 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, task_struct_reserve, 292 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lkpi_task_resrv, 0, 293 "Number of struct task and struct mm to reserve for non-sleepable " 294 "allocations"); 295 296 static void 297 linux_current_init(void *arg __unused) 298 { 299 TUNABLE_INT_FETCH("compat.linuxkpi.task_struct_reserve", 300 &lkpi_task_resrv); 301 if (lkpi_task_resrv == 0) { 302 #ifdef DEV_APIC 303 /* 304 * Number of interrupt threads plus per-cpu callout 305 * SWI threads. 306 */ 307 lkpi_task_resrv = first_msi_irq + num_msi_irqs + MAXCPU; 308 #else 309 lkpi_task_resrv = 1024; /* XXXKIB arbitrary */ 310 #endif 311 } 312 linux_current_zone = uma_zcreate("lkpicurr", 313 sizeof(struct task_struct), NULL, NULL, NULL, NULL, 314 UMA_ALIGN_PTR, 0); 315 uma_zone_reserve(linux_current_zone, lkpi_task_resrv); 316 uma_prealloc(linux_current_zone, lkpi_task_resrv); 317 linux_mm_zone = uma_zcreate("lkpimm", 318 sizeof(struct task_struct), NULL, NULL, NULL, NULL, 319 UMA_ALIGN_PTR, 0); 320 uma_zone_reserve(linux_mm_zone, lkpi_task_resrv); 321 uma_prealloc(linux_mm_zone, lkpi_task_resrv); 322 323 atomic_thread_fence_seq_cst(); 324 325 linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 326 linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 327 lkpi_alloc_current = linux_alloc_current; 328 } 329 SYSINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND, 330 linux_current_init, NULL); 331 332 static void 333 linux_current_uninit(void *arg __unused) 334 { 335 struct proc *p; 336 struct task_struct *ts; 337 struct thread *td; 338 339 lkpi_alloc_current = linux_alloc_current_noop; 340 341 atomic_thread_fence_seq_cst(); 342 343 sx_slock(&allproc_lock); 344 FOREACH_PROC_IN_SYSTEM(p) { 345 PROC_LOCK(p); 346 FOREACH_THREAD_IN_PROC(p, td) { 347 if ((ts = td->td_lkpi_task) != NULL) { 348 td->td_lkpi_task = NULL; 349 put_task_struct(ts); 350 } 351 } 352 PROC_UNLOCK(p); 353 } 354 sx_sunlock(&allproc_lock); 355 356 thread_reap_barrier(); 357 358 EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); 359 360 uma_zdestroy(linux_current_zone); 361 uma_zdestroy(linux_mm_zone); 362 } 363 SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND, 364 linux_current_uninit, NULL); 365