1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
5 *
6 * This work was supported by Innovate UK project 105694, "Digital Security
7 * by Design (DSbD) Technology Platform Prototype".
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /* Hardware Trace (HWT) framework. */
32
33 #include <sys/param.h>
34 #include <sys/proc.h>
35 #include <sys/ioccom.h>
36 #include <sys/kernel.h>
37 #include <sys/malloc.h>
38 #include <sys/mman.h>
39 #include <sys/mutex.h>
40 #include <sys/refcount.h>
41 #include <sys/rwlock.h>
42 #include <sys/smp.h>
43 #include <sys/hwt.h>
44
45 #include <dev/hwt/hwt_hook.h>
46 #include <dev/hwt/hwt_context.h>
47 #include <dev/hwt/hwt_contexthash.h>
48 #include <dev/hwt/hwt_config.h>
49 #include <dev/hwt/hwt_cpu.h>
50 #include <dev/hwt/hwt_thread.h>
51 #include <dev/hwt/hwt_owner.h>
52 #include <dev/hwt/hwt_ownerhash.h>
53 #include <dev/hwt/hwt_backend.h>
54 #include <dev/hwt/hwt_record.h>
55 #include <dev/hwt/hwt_ioctl.h>
56 #include <dev/hwt/hwt_vm.h>
57
58 #define HWT_IOCTL_DEBUG
59 #undef HWT_IOCTL_DEBUG
60
61 #ifdef HWT_IOCTL_DEBUG
62 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
63 #else
64 #define dprintf(fmt, ...)
65 #endif
66
67 /* No real reason for these limitations just sanity checks. */
68 #define HWT_MAXBUFSIZE (32UL * 1024 * 1024 * 1024) /* 32 GB */
69
70 static MALLOC_DEFINE(M_HWT_IOCTL, "hwt_ioctl", "Hardware Trace");
71
72 /*
73 * Check if owner process *o can trace target process *t.
74 */
75
76 static int
hwt_priv_check(struct proc * o,struct proc * t)77 hwt_priv_check(struct proc *o, struct proc *t)
78 {
79 struct ucred *oc, *tc;
80 int error;
81 int i;
82
83 PROC_LOCK(o);
84 oc = o->p_ucred;
85 crhold(oc);
86 PROC_UNLOCK(o);
87
88 PROC_LOCK_ASSERT(t, MA_OWNED);
89 tc = t->p_ucred;
90 crhold(tc);
91
92 error = 0;
93
94 /*
95 * The effective uid of the HWT owner should match at least one
96 * of the effective / real / saved uids of the target process.
97 */
98
99 if (oc->cr_uid != tc->cr_uid &&
100 oc->cr_uid != tc->cr_svuid &&
101 oc->cr_uid != tc->cr_ruid) {
102 error = EPERM;
103 goto done;
104 }
105
106 /*
107 * Everyone of the target's group ids must be in the owner's
108 * group list.
109 */
110 for (i = 0; i < tc->cr_ngroups; i++)
111 if (!groupmember(tc->cr_groups[i], oc)) {
112 error = EPERM;
113 goto done;
114 }
115
116 /* Check the read and saved GIDs too. */
117 if (!groupmember(tc->cr_rgid, oc) ||
118 !groupmember(tc->cr_svgid, oc)) {
119 error = EPERM;
120 goto done;
121 }
122
123 done:
124 crfree(tc);
125 crfree(oc);
126
127 return (error);
128 }
129
130 static int
hwt_ioctl_alloc_mode_thread(struct thread * td,struct hwt_owner * ho,struct hwt_backend * backend,struct hwt_alloc * halloc)131 hwt_ioctl_alloc_mode_thread(struct thread *td, struct hwt_owner *ho,
132 struct hwt_backend *backend, struct hwt_alloc *halloc)
133 {
134 struct thread **threads, *td1;
135 struct hwt_record_entry *entry;
136 struct hwt_context *ctx, *ctx1;
137 struct hwt_thread *thr;
138 char path[MAXPATHLEN];
139 struct proc *p;
140 int thread_id;
141 int error;
142 int cnt;
143 int i;
144
145 /* Check if the owner have this pid configured already. */
146 ctx = hwt_owner_lookup_ctx(ho, halloc->pid);
147 if (ctx)
148 return (EEXIST);
149
150 /* Allocate a new HWT context. */
151 error = hwt_ctx_alloc(&ctx);
152 if (error)
153 return (error);
154 ctx->bufsize = halloc->bufsize;
155 ctx->pid = halloc->pid;
156 ctx->hwt_backend = backend;
157 ctx->hwt_owner = ho;
158 ctx->mode = HWT_MODE_THREAD;
159 ctx->hwt_td = td;
160 ctx->kqueue_fd = halloc->kqueue_fd;
161
162 error = copyout(&ctx->ident, halloc->ident, sizeof(int));
163 if (error) {
164 hwt_ctx_free(ctx);
165 return (error);
166 }
167
168 /* Now get the victim proc. */
169 p = pfind(halloc->pid);
170 if (p == NULL) {
171 hwt_ctx_free(ctx);
172 return (ENXIO);
173 }
174
175 /* Ensure we can trace it. */
176 error = hwt_priv_check(td->td_proc, p);
177 if (error) {
178 PROC_UNLOCK(p);
179 hwt_ctx_free(ctx);
180 return (error);
181 }
182
183 /* Ensure it is not being traced already. */
184 ctx1 = hwt_contexthash_lookup(p);
185 if (ctx1) {
186 refcount_release(&ctx1->refcnt);
187 PROC_UNLOCK(p);
188 hwt_ctx_free(ctx);
189 return (EEXIST);
190 }
191
192 /* Allocate hwt threads and buffers. */
193
194 cnt = 0;
195
196 FOREACH_THREAD_IN_PROC(p, td1) {
197 cnt += 1;
198 }
199
200 KASSERT(cnt > 0, ("no threads"));
201
202 threads = malloc(sizeof(struct thread *) * cnt, M_HWT_IOCTL,
203 M_NOWAIT | M_ZERO);
204 if (threads == NULL) {
205 PROC_UNLOCK(p);
206 hwt_ctx_free(ctx);
207 return (ENOMEM);
208 }
209
210 i = 0;
211
212 FOREACH_THREAD_IN_PROC(p, td1) {
213 threads[i++] = td1;
214 }
215
216 ctx->proc = p;
217 PROC_UNLOCK(p);
218
219 for (i = 0; i < cnt; i++) {
220 thread_id = atomic_fetchadd_int(&ctx->thread_counter, 1);
221 sprintf(path, "hwt_%d_%d", ctx->ident, thread_id);
222
223 error = hwt_thread_alloc(&thr, path, ctx->bufsize,
224 ctx->hwt_backend->kva_req);
225 if (error) {
226 free(threads, M_HWT_IOCTL);
227 hwt_ctx_free(ctx);
228 return (error);
229 }
230 /* Allocate backend-specific thread data. */
231 error = hwt_backend_thread_alloc(ctx, thr);
232 if (error != 0) {
233 dprintf("%s: failed to allocate thread backend data\n",
234 __func__);
235 free(threads, M_HWT_IOCTL);
236 hwt_ctx_free(ctx);
237 return (error);
238 }
239
240 /*
241 * Insert a THREAD_CREATE record so userspace picks up
242 * the thread's tracing buffers.
243 */
244 entry = hwt_record_entry_alloc();
245 entry->record_type = HWT_RECORD_THREAD_CREATE;
246 entry->thread_id = thread_id;
247
248 thr->vm->ctx = ctx;
249 thr->td = threads[i];
250 thr->ctx = ctx;
251 thr->backend = ctx->hwt_backend;
252 thr->thread_id = thread_id;
253
254 HWT_CTX_LOCK(ctx);
255 hwt_thread_insert(ctx, thr, entry);
256 HWT_CTX_UNLOCK(ctx);
257 }
258
259 free(threads, M_HWT_IOCTL);
260
261 error = hwt_backend_init(ctx);
262 if (error) {
263 hwt_ctx_free(ctx);
264 return (error);
265 }
266
267 /* hwt_owner_insert_ctx? */
268 mtx_lock(&ho->mtx);
269 LIST_INSERT_HEAD(&ho->hwts, ctx, next_hwts);
270 mtx_unlock(&ho->mtx);
271
272 /*
273 * Hooks are now in action after this, but the ctx is not in RUNNING
274 * state.
275 */
276 hwt_contexthash_insert(ctx);
277
278 p = pfind(halloc->pid);
279 if (p) {
280 p->p_flag2 |= P2_HWT;
281 PROC_UNLOCK(p);
282 }
283
284 return (0);
285 }
286
287 static int
hwt_ioctl_alloc_mode_cpu(struct thread * td,struct hwt_owner * ho,struct hwt_backend * backend,struct hwt_alloc * halloc)288 hwt_ioctl_alloc_mode_cpu(struct thread *td, struct hwt_owner *ho,
289 struct hwt_backend *backend, struct hwt_alloc *halloc)
290 {
291 struct hwt_context *ctx;
292 struct hwt_cpu *cpu;
293 struct hwt_vm *vm;
294 char path[MAXPATHLEN];
295 size_t cpusetsize;
296 cpuset_t cpu_map;
297 int cpu_count = 0;
298 int cpu_id;
299 int error;
300
301 CPU_ZERO(&cpu_map);
302 cpusetsize = min(halloc->cpusetsize, sizeof(cpuset_t));
303 error = copyin(halloc->cpu_map, &cpu_map, cpusetsize);
304 if (error)
305 return (error);
306
307 CPU_FOREACH_ISSET(cpu_id, &cpu_map) {
308 #ifdef SMP
309 /* Ensure CPU is not halted. */
310 if (CPU_ISSET(cpu_id, &hlt_cpus_mask))
311 return (ENXIO);
312 #endif
313 #if 0
314 /* TODO: Check if the owner have this cpu configured already. */
315 ctx = hwt_owner_lookup_ctx_by_cpu(ho, halloc->cpu);
316 if (ctx)
317 return (EEXIST);
318 #endif
319
320 cpu_count++;
321 }
322
323 if (cpu_count == 0)
324 return (ENODEV);
325
326 /* Allocate a new HWT context. */
327 error = hwt_ctx_alloc(&ctx);
328 if (error)
329 return (error);
330 ctx->bufsize = halloc->bufsize;
331 ctx->hwt_backend = backend;
332 ctx->hwt_owner = ho;
333 ctx->mode = HWT_MODE_CPU;
334 ctx->cpu_map = cpu_map;
335 ctx->hwt_td = td;
336 ctx->kqueue_fd = halloc->kqueue_fd;
337
338 error = copyout(&ctx->ident, halloc->ident, sizeof(int));
339 if (error) {
340 hwt_ctx_free(ctx);
341 return (error);
342 }
343
344 CPU_FOREACH_ISSET(cpu_id, &cpu_map) {
345 sprintf(path, "hwt_%d_%d", ctx->ident, cpu_id);
346 error = hwt_vm_alloc(ctx->bufsize, ctx->hwt_backend->kva_req,
347 path, &vm);
348 if (error) {
349 /* TODO: remove all allocated cpus. */
350 hwt_ctx_free(ctx);
351 return (error);
352 }
353
354 cpu = hwt_cpu_alloc();
355 cpu->cpu_id = cpu_id;
356 cpu->vm = vm;
357
358 vm->cpu = cpu;
359 vm->ctx = ctx;
360
361 HWT_CTX_LOCK(ctx);
362 hwt_cpu_insert(ctx, cpu);
363 HWT_CTX_UNLOCK(ctx);
364 }
365
366 error = hwt_backend_init(ctx);
367 if (error) {
368 /* TODO: remove all allocated cpus. */
369 hwt_ctx_free(ctx);
370 return (error);
371 }
372
373 /* hwt_owner_insert_ctx? */
374 mtx_lock(&ho->mtx);
375 LIST_INSERT_HEAD(&ho->hwts, ctx, next_hwts);
376 mtx_unlock(&ho->mtx);
377
378 hwt_record_kernel_objects(ctx);
379
380 return (0);
381 }
382
383 static int
hwt_ioctl_alloc(struct thread * td,struct hwt_alloc * halloc)384 hwt_ioctl_alloc(struct thread *td, struct hwt_alloc *halloc)
385 {
386 char backend_name[HWT_BACKEND_MAXNAMELEN];
387 struct hwt_backend *backend;
388 struct hwt_owner *ho;
389 int error;
390
391 if (halloc->bufsize > HWT_MAXBUFSIZE)
392 return (EINVAL);
393 if (halloc->bufsize % PAGE_SIZE)
394 return (EINVAL);
395 if (halloc->backend_name == NULL)
396 return (EINVAL);
397
398 error = copyinstr(halloc->backend_name, (void *)backend_name,
399 HWT_BACKEND_MAXNAMELEN, NULL);
400 if (error)
401 return (error);
402
403 backend = hwt_backend_lookup(backend_name);
404 if (backend == NULL)
405 return (ENODEV);
406
407 /* First get the owner. */
408 ho = hwt_ownerhash_lookup(td->td_proc);
409 if (ho == NULL) {
410 /* Create a new owner. */
411 ho = hwt_owner_alloc(td->td_proc);
412 if (ho == NULL)
413 return (ENOMEM);
414 hwt_ownerhash_insert(ho);
415 }
416
417 switch (halloc->mode) {
418 case HWT_MODE_THREAD:
419 error = hwt_ioctl_alloc_mode_thread(td, ho, backend, halloc);
420 break;
421 case HWT_MODE_CPU:
422 error = hwt_ioctl_alloc_mode_cpu(td, ho, backend, halloc);
423 break;
424 default:
425 error = ENXIO;
426 };
427
428 return (error);
429 }
430
431 int
hwt_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flags,struct thread * td)432 hwt_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
433 struct thread *td)
434 {
435 int error;
436
437 switch (cmd) {
438 case HWT_IOC_ALLOC:
439 /* Allocate HWT context. */
440 error = hwt_ioctl_alloc(td, (struct hwt_alloc *)addr);
441 return (error);
442 default:
443 return (ENXIO);
444 };
445 }
446