1 /*- 2 * Copyright (c) 2017 Hans Petter Selasky 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #ifdef __amd64__ 31 #define DEV_APIC 32 #elif defined(__i386__) 33 #include "opt_apic.h" 34 #endif 35 36 #include <linux/compat.h> 37 #include <linux/completion.h> 38 #include <linux/mm.h> 39 #include <linux/kthread.h> 40 #include <linux/moduleparam.h> 41 42 #include <sys/kernel.h> 43 #include <sys/eventhandler.h> 44 #include <sys/malloc.h> 45 #include <sys/sysctl.h> 46 #include <vm/uma.h> 47 48 #ifdef DEV_APIC 49 extern u_int first_msi_irq, num_msi_irqs; 50 #endif 51 52 static eventhandler_tag linuxkpi_thread_dtor_tag; 53 54 static uma_zone_t linux_current_zone; 55 static uma_zone_t linux_mm_zone; 56 57 /* check if another thread already has a mm_struct */ 58 static struct mm_struct * 59 find_other_mm(struct proc *p) 60 { 61 struct thread *td; 62 struct task_struct *ts; 63 struct mm_struct *mm; 64 65 PROC_LOCK_ASSERT(p, MA_OWNED); 66 FOREACH_THREAD_IN_PROC(p, td) { 67 ts = td->td_lkpi_task; 68 if (ts == NULL) 69 continue; 70 mm = ts->mm; 71 if (mm == NULL) 72 continue; 73 /* try to share other mm_struct */ 74 if (atomic_inc_not_zero(&mm->mm_users)) 75 return (mm); 76 } 77 return (NULL); 78 } 79 80 int 81 linux_alloc_current(struct thread *td, int flags) 82 { 83 struct proc *proc; 84 struct task_struct *ts; 85 struct mm_struct *mm, *mm_other; 86 87 MPASS(td->td_lkpi_task == NULL); 88 89 if ((td->td_pflags & TDP_ITHREAD) != 0 || !THREAD_CAN_SLEEP()) { 90 flags &= ~M_WAITOK; 91 flags |= M_NOWAIT | M_USE_RESERVE; 92 } 93 94 ts = uma_zalloc(linux_current_zone, flags | M_ZERO); 95 if (ts == NULL) { 96 if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 97 panic("linux_alloc_current: failed to allocate task"); 98 return (ENOMEM); 99 } 100 mm = NULL; 101 102 /* setup new task structure */ 103 atomic_set(&ts->kthread_flags, 0); 104 ts->task_thread = td; 105 ts->comm = td->td_name; 106 ts->pid = td->td_tid; 107 ts->group_leader = ts; 108 atomic_set(&ts->usage, 1); 109 atomic_set(&ts->state, TASK_RUNNING); 110 init_completion(&ts->parked); 111 init_completion(&ts->exited); 112 113 proc = td->td_proc; 114 115 PROC_LOCK(proc); 116 mm_other = find_other_mm(proc); 117 118 /* use allocated mm_struct as a fallback */ 119 if (mm_other == NULL) { 120 PROC_UNLOCK(proc); 121 mm = uma_zalloc(linux_mm_zone, flags | M_ZERO); 122 if (mm == NULL) { 123 if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 124 panic( 125 "linux_alloc_current: failed to allocate mm"); 126 uma_zfree(linux_current_zone, mm); 127 return (ENOMEM); 128 } 129 130 PROC_LOCK(proc); 131 mm_other = find_other_mm(proc); 132 if (mm_other == NULL) { 133 /* setup new mm_struct */ 134 init_rwsem(&mm->mmap_sem); 135 atomic_set(&mm->mm_count, 1); 136 atomic_set(&mm->mm_users, 1); 137 /* set mm_struct pointer */ 138 ts->mm = mm; 139 /* clear pointer to not free memory */ 140 mm = NULL; 141 } else { 142 ts->mm = mm_other; 143 } 144 } else { 145 ts->mm = mm_other; 146 } 147 148 /* store pointer to task struct */ 149 td->td_lkpi_task = ts; 150 PROC_UNLOCK(proc); 151 152 /* free mm_struct pointer, if any */ 153 uma_zfree(linux_mm_zone, mm); 154 155 return (0); 156 } 157 158 struct mm_struct * 159 linux_get_task_mm(struct task_struct *task) 160 { 161 struct mm_struct *mm; 162 163 mm = task->mm; 164 if (mm != NULL) { 165 atomic_inc(&mm->mm_users); 166 return (mm); 167 } 168 return (NULL); 169 } 170 171 void 172 linux_mm_dtor(struct mm_struct *mm) 173 { 174 uma_zfree(linux_mm_zone, mm); 175 } 176 177 void 178 linux_free_current(struct task_struct *ts) 179 { 180 mmput(ts->mm); 181 uma_zfree(linux_current_zone, ts); 182 } 183 184 static void 185 linuxkpi_thread_dtor(void *arg __unused, struct thread *td) 186 { 187 struct task_struct *ts; 188 189 ts = td->td_lkpi_task; 190 if (ts == NULL) 191 return; 192 193 td->td_lkpi_task = NULL; 194 put_task_struct(ts); 195 } 196 197 static struct task_struct * 198 linux_get_pid_task_int(pid_t pid, const bool do_get) 199 { 200 struct thread *td; 201 struct proc *p; 202 struct task_struct *ts; 203 204 if (pid > PID_MAX) { 205 /* try to find corresponding thread */ 206 td = tdfind(pid, -1); 207 if (td != NULL) { 208 ts = td->td_lkpi_task; 209 if (do_get && ts != NULL) 210 get_task_struct(ts); 211 PROC_UNLOCK(td->td_proc); 212 return (ts); 213 } 214 } else { 215 /* try to find corresponding procedure */ 216 p = pfind(pid); 217 if (p != NULL) { 218 FOREACH_THREAD_IN_PROC(p, td) { 219 ts = td->td_lkpi_task; 220 if (ts != NULL) { 221 if (do_get) 222 get_task_struct(ts); 223 PROC_UNLOCK(p); 224 return (ts); 225 } 226 } 227 PROC_UNLOCK(p); 228 } 229 } 230 return (NULL); 231 } 232 233 struct task_struct * 234 linux_pid_task(pid_t pid) 235 { 236 return (linux_get_pid_task_int(pid, false)); 237 } 238 239 struct task_struct * 240 linux_get_pid_task(pid_t pid) 241 { 242 return (linux_get_pid_task_int(pid, true)); 243 } 244 245 bool 246 linux_task_exiting(struct task_struct *task) 247 { 248 struct thread *td; 249 struct proc *p; 250 bool ret; 251 252 ret = false; 253 254 /* try to find corresponding thread */ 255 td = tdfind(task->pid, -1); 256 if (td != NULL) { 257 p = td->td_proc; 258 } else { 259 /* try to find corresponding procedure */ 260 p = pfind(task->pid); 261 } 262 263 if (p != NULL) { 264 if ((p->p_flag & P_WEXIT) != 0) 265 ret = true; 266 PROC_UNLOCK(p); 267 } 268 return (ret); 269 } 270 271 static int lkpi_task_resrv; 272 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, task_struct_reserve, 273 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lkpi_task_resrv, 0, 274 "Number of struct task and struct mm to reserve for non-sleepable " 275 "allocations"); 276 277 static void 278 linux_current_init(void *arg __unused) 279 { 280 TUNABLE_INT_FETCH("compat.linuxkpi.task_struct_reserve", 281 &lkpi_task_resrv); 282 if (lkpi_task_resrv == 0) { 283 #ifdef DEV_APIC 284 /* 285 * Number of interrupt threads plus per-cpu callout 286 * SWI threads. 287 */ 288 lkpi_task_resrv = first_msi_irq + num_msi_irqs + MAXCPU; 289 #else 290 lkpi_task_resrv = 1024; /* XXXKIB arbitrary */ 291 #endif 292 } 293 linux_current_zone = uma_zcreate("lkpicurr", 294 sizeof(struct task_struct), NULL, NULL, NULL, NULL, 295 UMA_ALIGN_PTR, 0); 296 uma_zone_reserve(linux_current_zone, lkpi_task_resrv); 297 uma_prealloc(linux_current_zone, lkpi_task_resrv); 298 linux_mm_zone = uma_zcreate("lkpimm", 299 sizeof(struct task_struct), NULL, NULL, NULL, NULL, 300 UMA_ALIGN_PTR, 0); 301 uma_zone_reserve(linux_mm_zone, lkpi_task_resrv); 302 uma_prealloc(linux_mm_zone, lkpi_task_resrv); 303 304 atomic_thread_fence_seq_cst(); 305 306 linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 307 linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 308 lkpi_alloc_current = linux_alloc_current; 309 } 310 SYSINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND, 311 linux_current_init, NULL); 312 313 static void 314 linux_current_uninit(void *arg __unused) 315 { 316 struct proc *p; 317 struct task_struct *ts; 318 struct thread *td; 319 320 lkpi_alloc_current = linux_alloc_current_noop; 321 322 atomic_thread_fence_seq_cst(); 323 324 sx_slock(&allproc_lock); 325 FOREACH_PROC_IN_SYSTEM(p) { 326 PROC_LOCK(p); 327 FOREACH_THREAD_IN_PROC(p, td) { 328 if ((ts = td->td_lkpi_task) != NULL) { 329 td->td_lkpi_task = NULL; 330 put_task_struct(ts); 331 } 332 } 333 PROC_UNLOCK(p); 334 } 335 sx_sunlock(&allproc_lock); 336 337 thread_reap_barrier(); 338 339 EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); 340 341 uma_zdestroy(linux_current_zone); 342 uma_zdestroy(linux_mm_zone); 343 } 344 SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND, 345 linux_current_uninit, NULL); 346