1 /*-
2 * Copyright (c) 2017 Hans Petter Selasky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #ifdef __amd64__
29 #define DEV_APIC
30 #elif defined(__i386__)
31 #include "opt_apic.h"
32 #endif
33
34 #include <linux/compat.h>
35 #include <linux/completion.h>
36 #include <linux/mm.h>
37 #include <linux/kthread.h>
38 #include <linux/moduleparam.h>
39
40 #include <sys/kernel.h>
41 #include <sys/eventhandler.h>
42 #include <sys/malloc.h>
43 #include <sys/sysctl.h>
44 #include <vm/uma.h>
45
46 #ifdef DEV_APIC
47 extern u_int first_msi_irq, num_msi_irqs;
48 #endif
49
50 static eventhandler_tag linuxkpi_thread_dtor_tag;
51
52 static uma_zone_t linux_current_zone;
53 static uma_zone_t linux_mm_zone;
54
55 /* check if another thread already has a mm_struct */
56 static struct mm_struct *
find_other_mm(struct proc * p)57 find_other_mm(struct proc *p)
58 {
59 struct thread *td;
60 struct task_struct *ts;
61 struct mm_struct *mm;
62
63 PROC_LOCK_ASSERT(p, MA_OWNED);
64 FOREACH_THREAD_IN_PROC(p, td) {
65 ts = td->td_lkpi_task;
66 if (ts == NULL)
67 continue;
68 mm = ts->mm;
69 if (mm == NULL)
70 continue;
71 /* try to share other mm_struct */
72 if (atomic_inc_not_zero(&mm->mm_users))
73 return (mm);
74 }
75 return (NULL);
76 }
77
78 int
linux_alloc_current(struct thread * td,int flags)79 linux_alloc_current(struct thread *td, int flags)
80 {
81 struct proc *proc;
82 struct task_struct *ts;
83 struct mm_struct *mm, *mm_other;
84
85 MPASS(td->td_lkpi_task == NULL);
86
87 if ((td->td_pflags & TDP_ITHREAD) != 0 || !THREAD_CAN_SLEEP()) {
88 flags &= ~M_WAITOK;
89 flags |= M_NOWAIT | M_USE_RESERVE;
90 }
91
92 ts = uma_zalloc(linux_current_zone, flags | M_ZERO);
93 if (ts == NULL)
94 return (ENOMEM);
95 mm = NULL;
96
97 /* setup new task structure */
98 atomic_set(&ts->kthread_flags, 0);
99 ts->task_thread = td;
100 ts->comm = td->td_name;
101 ts->pid = td->td_tid;
102 ts->group_leader = ts;
103 atomic_set(&ts->usage, 1);
104 atomic_set(&ts->state, TASK_RUNNING);
105 init_completion(&ts->parked);
106 init_completion(&ts->exited);
107
108 proc = td->td_proc;
109
110 PROC_LOCK(proc);
111 mm_other = find_other_mm(proc);
112
113 /* use allocated mm_struct as a fallback */
114 if (mm_other == NULL) {
115 PROC_UNLOCK(proc);
116 mm = uma_zalloc(linux_mm_zone, flags | M_ZERO);
117 if (mm == NULL) {
118 uma_zfree(linux_current_zone, ts);
119 return (ENOMEM);
120 }
121
122 PROC_LOCK(proc);
123 mm_other = find_other_mm(proc);
124 if (mm_other == NULL) {
125 /* setup new mm_struct */
126 init_rwsem(&mm->mmap_sem);
127 atomic_set(&mm->mm_count, 1);
128 atomic_set(&mm->mm_users, 1);
129 /* set mm_struct pointer */
130 ts->mm = mm;
131 /* clear pointer to not free memory */
132 mm = NULL;
133 } else {
134 ts->mm = mm_other;
135 }
136 } else {
137 ts->mm = mm_other;
138 }
139
140 /* store pointer to task struct */
141 td->td_lkpi_task = ts;
142 PROC_UNLOCK(proc);
143
144 /* free mm_struct pointer, if any */
145 uma_zfree(linux_mm_zone, mm);
146
147 return (0);
148 }
149
150 struct mm_struct *
linux_get_task_mm(struct task_struct * task)151 linux_get_task_mm(struct task_struct *task)
152 {
153 struct mm_struct *mm;
154
155 mm = task->mm;
156 if (mm != NULL) {
157 atomic_inc(&mm->mm_users);
158 return (mm);
159 }
160 return (NULL);
161 }
162
163 void
linux_mm_dtor(struct mm_struct * mm)164 linux_mm_dtor(struct mm_struct *mm)
165 {
166 uma_zfree(linux_mm_zone, mm);
167 }
168
169 void
linux_free_current(struct task_struct * ts)170 linux_free_current(struct task_struct *ts)
171 {
172 mmput(ts->mm);
173 uma_zfree(linux_current_zone, ts);
174 }
175
176 static void
linuxkpi_thread_dtor(void * arg __unused,struct thread * td)177 linuxkpi_thread_dtor(void *arg __unused, struct thread *td)
178 {
179 struct task_struct *ts;
180
181 ts = td->td_lkpi_task;
182 if (ts == NULL)
183 return;
184
185 td->td_lkpi_task = NULL;
186 put_task_struct(ts);
187 }
188
189 static struct task_struct *
linux_get_pid_task_int(pid_t pid,const bool do_get)190 linux_get_pid_task_int(pid_t pid, const bool do_get)
191 {
192 struct thread *td;
193 struct proc *p;
194 struct task_struct *ts;
195
196 if (pid > PID_MAX) {
197 /* try to find corresponding thread */
198 td = tdfind(pid, -1);
199 if (td != NULL) {
200 ts = td->td_lkpi_task;
201 if (do_get && ts != NULL)
202 get_task_struct(ts);
203 PROC_UNLOCK(td->td_proc);
204 return (ts);
205 }
206 } else {
207 /* try to find corresponding procedure */
208 p = pfind(pid);
209 if (p != NULL) {
210 FOREACH_THREAD_IN_PROC(p, td) {
211 ts = td->td_lkpi_task;
212 if (ts != NULL) {
213 if (do_get)
214 get_task_struct(ts);
215 PROC_UNLOCK(p);
216 return (ts);
217 }
218 }
219 PROC_UNLOCK(p);
220 }
221 }
222 return (NULL);
223 }
224
225 struct task_struct *
linux_pid_task(pid_t pid)226 linux_pid_task(pid_t pid)
227 {
228 return (linux_get_pid_task_int(pid, false));
229 }
230
231 struct task_struct *
linux_get_pid_task(pid_t pid)232 linux_get_pid_task(pid_t pid)
233 {
234 return (linux_get_pid_task_int(pid, true));
235 }
236
237 bool
linux_task_exiting(struct task_struct * task)238 linux_task_exiting(struct task_struct *task)
239 {
240 struct thread *td;
241 struct proc *p;
242 bool ret;
243
244 ret = false;
245
246 /* try to find corresponding thread */
247 td = tdfind(task->pid, -1);
248 if (td != NULL) {
249 p = td->td_proc;
250 } else {
251 /* try to find corresponding procedure */
252 p = pfind(task->pid);
253 }
254
255 if (p != NULL) {
256 if ((p->p_flag & P_WEXIT) != 0)
257 ret = true;
258 PROC_UNLOCK(p);
259 }
260 return (ret);
261 }
262
263 static int lkpi_task_resrv;
264 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, task_struct_reserve,
265 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lkpi_task_resrv, 0,
266 "Number of struct task and struct mm to reserve for non-sleepable "
267 "allocations");
268
269 static void
linux_current_init(void * arg __unused)270 linux_current_init(void *arg __unused)
271 {
272 TUNABLE_INT_FETCH("compat.linuxkpi.task_struct_reserve",
273 &lkpi_task_resrv);
274 if (lkpi_task_resrv == 0) {
275 #ifdef DEV_APIC
276 /*
277 * Number of interrupt threads plus per-cpu callout
278 * SWI threads.
279 */
280 lkpi_task_resrv = first_msi_irq + num_msi_irqs + MAXCPU;
281 #else
282 lkpi_task_resrv = 1024; /* XXXKIB arbitrary */
283 #endif
284 }
285 linux_current_zone = uma_zcreate("lkpicurr",
286 sizeof(struct task_struct), NULL, NULL, NULL, NULL,
287 UMA_ALIGN_PTR, 0);
288 uma_zone_reserve(linux_current_zone, lkpi_task_resrv);
289 uma_prealloc(linux_current_zone, lkpi_task_resrv);
290 linux_mm_zone = uma_zcreate("lkpimm",
291 sizeof(struct mm_struct), NULL, NULL, NULL, NULL,
292 UMA_ALIGN_PTR, 0);
293 uma_zone_reserve(linux_mm_zone, lkpi_task_resrv);
294 uma_prealloc(linux_mm_zone, lkpi_task_resrv);
295
296 atomic_thread_fence_seq_cst();
297
298 linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
299 linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY);
300 lkpi_alloc_current = linux_alloc_current;
301 }
302 SYSINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND,
303 linux_current_init, NULL);
304
305 static void
linux_current_uninit(void * arg __unused)306 linux_current_uninit(void *arg __unused)
307 {
308 struct proc *p;
309 struct task_struct *ts;
310 struct thread *td;
311
312 lkpi_alloc_current = linux_alloc_current_noop;
313
314 atomic_thread_fence_seq_cst();
315
316 sx_slock(&allproc_lock);
317 FOREACH_PROC_IN_SYSTEM(p) {
318 PROC_LOCK(p);
319 FOREACH_THREAD_IN_PROC(p, td) {
320 if ((ts = td->td_lkpi_task) != NULL) {
321 td->td_lkpi_task = NULL;
322 put_task_struct(ts);
323 }
324 }
325 PROC_UNLOCK(p);
326 }
327 sx_sunlock(&allproc_lock);
328
329 thread_reap_barrier();
330
331 EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag);
332
333 uma_zdestroy(linux_current_zone);
334 uma_zdestroy(linux_mm_zone);
335 }
336 SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND,
337 linux_current_uninit, NULL);
338