1 /*- 2 * Copyright (c) 2017 Hans Petter Selasky 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 #ifdef __amd64__ 29 #define DEV_APIC 30 #elif defined(__i386__) 31 #include "opt_apic.h" 32 #endif 33 34 #include <linux/compat.h> 35 #include <linux/completion.h> 36 #include <linux/mm.h> 37 #include <linux/kthread.h> 38 #include <linux/moduleparam.h> 39 40 #include <sys/kernel.h> 41 #include <sys/eventhandler.h> 42 #include <sys/malloc.h> 43 #include <sys/sysctl.h> 44 #include <vm/uma.h> 45 46 #ifdef DEV_APIC 47 extern u_int first_msi_irq, num_msi_irqs; 48 #endif 49 50 static eventhandler_tag linuxkpi_thread_dtor_tag; 51 52 static uma_zone_t linux_current_zone; 53 static uma_zone_t linux_mm_zone; 54 55 /* check if another thread already has a mm_struct */ 56 static struct mm_struct * 57 find_other_mm(struct proc *p) 58 { 59 struct thread *td; 60 struct task_struct *ts; 61 struct mm_struct *mm; 62 63 PROC_LOCK_ASSERT(p, MA_OWNED); 64 FOREACH_THREAD_IN_PROC(p, td) { 65 ts = td->td_lkpi_task; 66 if (ts == NULL) 67 continue; 68 mm = ts->mm; 69 if (mm == NULL) 70 continue; 71 /* try to share other mm_struct */ 72 if (atomic_inc_not_zero(&mm->mm_users)) 73 return (mm); 74 } 75 return (NULL); 76 } 77 78 int 79 linux_alloc_current(struct thread *td, int flags) 80 { 81 struct proc *proc; 82 struct task_struct *ts; 83 struct mm_struct *mm, *mm_other; 84 85 MPASS(td->td_lkpi_task == NULL); 86 87 if ((td->td_pflags & TDP_ITHREAD) != 0 || !THREAD_CAN_SLEEP()) { 88 flags &= ~M_WAITOK; 89 flags |= M_NOWAIT | M_USE_RESERVE; 90 } 91 92 ts = uma_zalloc(linux_current_zone, flags | M_ZERO); 93 if (ts == NULL) 94 return (ENOMEM); 95 mm = NULL; 96 97 /* setup new task structure */ 98 atomic_set(&ts->kthread_flags, 0); 99 ts->task_thread = td; 100 ts->comm = td->td_name; 101 ts->pid = td->td_tid; 102 ts->group_leader = ts; 103 atomic_set(&ts->usage, 1); 104 atomic_set(&ts->state, TASK_RUNNING); 105 init_completion(&ts->parked); 106 init_completion(&ts->exited); 107 108 proc = td->td_proc; 109 110 PROC_LOCK(proc); 111 mm_other = find_other_mm(proc); 112 113 /* use allocated mm_struct as a fallback */ 114 if (mm_other == NULL) { 115 PROC_UNLOCK(proc); 116 mm = uma_zalloc(linux_mm_zone, flags | M_ZERO); 117 if (mm == NULL) { 118 uma_zfree(linux_current_zone, ts); 119 return (ENOMEM); 120 } 121 122 PROC_LOCK(proc); 123 mm_other = find_other_mm(proc); 124 if (mm_other == NULL) { 125 /* setup new mm_struct */ 126 init_rwsem(&mm->mmap_sem); 127 atomic_set(&mm->mm_count, 1); 128 atomic_set(&mm->mm_users, 1); 129 /* set mm_struct pointer */ 130 ts->mm = mm; 131 /* clear pointer to not free memory */ 132 mm = NULL; 133 } else { 134 ts->mm = mm_other; 135 } 136 } else { 137 ts->mm = mm_other; 138 } 139 140 /* store pointer to task struct */ 141 td->td_lkpi_task = ts; 142 PROC_UNLOCK(proc); 143 144 /* free mm_struct pointer, if any */ 145 uma_zfree(linux_mm_zone, mm); 146 147 return (0); 148 } 149 150 struct mm_struct * 151 linux_get_task_mm(struct task_struct *task) 152 { 153 struct mm_struct *mm; 154 155 mm = task->mm; 156 if (mm != NULL) { 157 atomic_inc(&mm->mm_users); 158 return (mm); 159 } 160 return (NULL); 161 } 162 163 void 164 linux_mm_dtor(struct mm_struct *mm) 165 { 166 uma_zfree(linux_mm_zone, mm); 167 } 168 169 void 170 linux_free_current(struct task_struct *ts) 171 { 172 mmput(ts->mm); 173 uma_zfree(linux_current_zone, ts); 174 } 175 176 static void 177 linuxkpi_thread_dtor(void *arg __unused, struct thread *td) 178 { 179 struct task_struct *ts; 180 181 ts = td->td_lkpi_task; 182 if (ts == NULL) 183 return; 184 185 td->td_lkpi_task = NULL; 186 put_task_struct(ts); 187 } 188 189 static struct task_struct * 190 linux_get_pid_task_int(pid_t pid, const bool do_get) 191 { 192 struct thread *td; 193 struct proc *p; 194 struct task_struct *ts; 195 196 if (pid > PID_MAX) { 197 /* try to find corresponding thread */ 198 td = tdfind(pid, -1); 199 if (td != NULL) { 200 ts = td->td_lkpi_task; 201 if (do_get && ts != NULL) 202 get_task_struct(ts); 203 PROC_UNLOCK(td->td_proc); 204 return (ts); 205 } 206 } else { 207 /* try to find corresponding procedure */ 208 p = pfind(pid); 209 if (p != NULL) { 210 FOREACH_THREAD_IN_PROC(p, td) { 211 ts = td->td_lkpi_task; 212 if (ts != NULL) { 213 if (do_get) 214 get_task_struct(ts); 215 PROC_UNLOCK(p); 216 return (ts); 217 } 218 } 219 PROC_UNLOCK(p); 220 } 221 } 222 return (NULL); 223 } 224 225 struct task_struct * 226 linux_pid_task(pid_t pid) 227 { 228 return (linux_get_pid_task_int(pid, false)); 229 } 230 231 struct task_struct * 232 linux_get_pid_task(pid_t pid) 233 { 234 return (linux_get_pid_task_int(pid, true)); 235 } 236 237 bool 238 linux_task_exiting(struct task_struct *task) 239 { 240 struct thread *td; 241 struct proc *p; 242 bool ret; 243 244 ret = false; 245 246 /* try to find corresponding thread */ 247 td = tdfind(task->pid, -1); 248 if (td != NULL) { 249 p = td->td_proc; 250 } else { 251 /* try to find corresponding procedure */ 252 p = pfind(task->pid); 253 } 254 255 if (p != NULL) { 256 if ((p->p_flag & P_WEXIT) != 0) 257 ret = true; 258 PROC_UNLOCK(p); 259 } 260 return (ret); 261 } 262 263 static int lkpi_task_resrv; 264 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, task_struct_reserve, 265 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lkpi_task_resrv, 0, 266 "Number of struct task and struct mm to reserve for non-sleepable " 267 "allocations"); 268 269 static void 270 linux_current_init(void *arg __unused) 271 { 272 TUNABLE_INT_FETCH("compat.linuxkpi.task_struct_reserve", 273 &lkpi_task_resrv); 274 if (lkpi_task_resrv == 0) { 275 #ifdef DEV_APIC 276 /* 277 * Number of interrupt threads plus per-cpu callout 278 * SWI threads. 279 */ 280 lkpi_task_resrv = first_msi_irq + num_msi_irqs + MAXCPU; 281 #else 282 lkpi_task_resrv = 1024; /* XXXKIB arbitrary */ 283 #endif 284 } 285 linux_current_zone = uma_zcreate("lkpicurr", 286 sizeof(struct task_struct), NULL, NULL, NULL, NULL, 287 UMA_ALIGN_PTR, 0); 288 uma_zone_reserve(linux_current_zone, lkpi_task_resrv); 289 uma_prealloc(linux_current_zone, lkpi_task_resrv); 290 linux_mm_zone = uma_zcreate("lkpimm", 291 sizeof(struct mm_struct), NULL, NULL, NULL, NULL, 292 UMA_ALIGN_PTR, 0); 293 uma_zone_reserve(linux_mm_zone, lkpi_task_resrv); 294 uma_prealloc(linux_mm_zone, lkpi_task_resrv); 295 296 atomic_thread_fence_seq_cst(); 297 298 linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 299 linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 300 lkpi_alloc_current = linux_alloc_current; 301 } 302 SYSINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND, 303 linux_current_init, NULL); 304 305 static void 306 linux_current_uninit(void *arg __unused) 307 { 308 struct proc *p; 309 struct task_struct *ts; 310 struct thread *td; 311 312 lkpi_alloc_current = linux_alloc_current_noop; 313 314 atomic_thread_fence_seq_cst(); 315 316 sx_slock(&allproc_lock); 317 FOREACH_PROC_IN_SYSTEM(p) { 318 PROC_LOCK(p); 319 FOREACH_THREAD_IN_PROC(p, td) { 320 if ((ts = td->td_lkpi_task) != NULL) { 321 td->td_lkpi_task = NULL; 322 put_task_struct(ts); 323 } 324 } 325 PROC_UNLOCK(p); 326 } 327 sx_sunlock(&allproc_lock); 328 329 thread_reap_barrier(); 330 331 EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); 332 333 uma_zdestroy(linux_current_zone); 334 uma_zdestroy(linux_mm_zone); 335 } 336 SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND, 337 linux_current_uninit, NULL); 338