1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
5 *
6 * This work was supported by Innovate UK project 105694, "Digital Security
7 * by Design (DSbD) Technology Platform Prototype".
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /* Hardware Trace (HWT) framework. */
32
33 #include <sys/param.h>
34 #include <sys/proc.h>
35 #include <sys/ioccom.h>
36 #include <sys/kernel.h>
37 #include <sys/malloc.h>
38 #include <sys/mman.h>
39 #include <sys/mutex.h>
40 #include <sys/refcount.h>
41 #include <sys/rwlock.h>
42 #include <sys/smp.h>
43 #include <sys/hwt.h>
44
45 #include <dev/hwt/hwt_hook.h>
46 #include <dev/hwt/hwt_context.h>
47 #include <dev/hwt/hwt_contexthash.h>
48 #include <dev/hwt/hwt_config.h>
49 #include <dev/hwt/hwt_cpu.h>
50 #include <dev/hwt/hwt_thread.h>
51 #include <dev/hwt/hwt_owner.h>
52 #include <dev/hwt/hwt_ownerhash.h>
53 #include <dev/hwt/hwt_backend.h>
54 #include <dev/hwt/hwt_record.h>
55 #include <dev/hwt/hwt_ioctl.h>
56 #include <dev/hwt/hwt_vm.h>
57
58 #define HWT_IOCTL_DEBUG
59 #undef HWT_IOCTL_DEBUG
60
61 #ifdef HWT_IOCTL_DEBUG
62 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
63 #else
64 #define dprintf(fmt, ...)
65 #endif
66
67 /* No real reason for these limitations just sanity checks. */
68 #define HWT_MAXBUFSIZE (32UL * 1024 * 1024 * 1024) /* 32 GB */
69
70 static MALLOC_DEFINE(M_HWT_IOCTL, "hwt_ioctl", "Hardware Trace");
71
72 /*
73 * Check if owner process *o can trace target process *t.
74 */
75
76 static int
hwt_priv_check(struct proc * o,struct proc * t)77 hwt_priv_check(struct proc *o, struct proc *t)
78 {
79 struct ucred *oc, *tc;
80 int error;
81 int i;
82
83 PROC_LOCK(o);
84 oc = o->p_ucred;
85 crhold(oc);
86 PROC_UNLOCK(o);
87
88 PROC_LOCK_ASSERT(t, MA_OWNED);
89 tc = t->p_ucred;
90 crhold(tc);
91
92 error = 0;
93
94 /*
95 * The effective uid of the HWT owner should match at least one
96 * of the effective / real / saved uids of the target process.
97 */
98
99 if (oc->cr_uid != tc->cr_uid &&
100 oc->cr_uid != tc->cr_svuid &&
101 oc->cr_uid != tc->cr_ruid) {
102 error = EPERM;
103 goto done;
104 }
105
106 /*
107 * Everyone of the target's group ids must be in the owner's
108 * group list.
109 */
110 for (i = 0; i < tc->cr_ngroups; i++)
111 if (!groupmember(tc->cr_groups[i], oc)) {
112 error = EPERM;
113 goto done;
114 }
115 if (!groupmember(tc->cr_gid, oc) ||
116 !groupmember(tc->cr_rgid, oc) ||
117 !groupmember(tc->cr_svgid, oc)) {
118 error = EPERM;
119 goto done;
120 }
121
122 done:
123 crfree(tc);
124 crfree(oc);
125
126 return (error);
127 }
128
129 static int
hwt_ioctl_alloc_mode_thread(struct thread * td,struct hwt_owner * ho,struct hwt_backend * backend,struct hwt_alloc * halloc)130 hwt_ioctl_alloc_mode_thread(struct thread *td, struct hwt_owner *ho,
131 struct hwt_backend *backend, struct hwt_alloc *halloc)
132 {
133 struct thread **threads, *td1;
134 struct hwt_record_entry *entry;
135 struct hwt_context *ctx, *ctx1;
136 struct hwt_thread *thr;
137 char path[MAXPATHLEN];
138 struct proc *p;
139 int thread_id;
140 int error;
141 int cnt;
142 int i;
143
144 /* Check if the owner have this pid configured already. */
145 ctx = hwt_owner_lookup_ctx(ho, halloc->pid);
146 if (ctx)
147 return (EEXIST);
148
149 /* Allocate a new HWT context. */
150 error = hwt_ctx_alloc(&ctx);
151 if (error)
152 return (error);
153 ctx->bufsize = halloc->bufsize;
154 ctx->pid = halloc->pid;
155 ctx->hwt_backend = backend;
156 ctx->hwt_owner = ho;
157 ctx->mode = HWT_MODE_THREAD;
158 ctx->hwt_td = td;
159 ctx->kqueue_fd = halloc->kqueue_fd;
160
161 error = copyout(&ctx->ident, halloc->ident, sizeof(int));
162 if (error) {
163 hwt_ctx_free(ctx);
164 return (error);
165 }
166
167 /* Now get the victim proc. */
168 p = pfind(halloc->pid);
169 if (p == NULL) {
170 hwt_ctx_free(ctx);
171 return (ENXIO);
172 }
173
174 /* Ensure we can trace it. */
175 error = hwt_priv_check(td->td_proc, p);
176 if (error) {
177 PROC_UNLOCK(p);
178 hwt_ctx_free(ctx);
179 return (error);
180 }
181
182 /* Ensure it is not being traced already. */
183 ctx1 = hwt_contexthash_lookup(p);
184 if (ctx1) {
185 refcount_release(&ctx1->refcnt);
186 PROC_UNLOCK(p);
187 hwt_ctx_free(ctx);
188 return (EEXIST);
189 }
190
191 /* Allocate hwt threads and buffers. */
192
193 cnt = 0;
194
195 FOREACH_THREAD_IN_PROC(p, td1) {
196 cnt += 1;
197 }
198
199 KASSERT(cnt > 0, ("no threads"));
200
201 threads = malloc(sizeof(struct thread *) * cnt, M_HWT_IOCTL,
202 M_NOWAIT | M_ZERO);
203 if (threads == NULL) {
204 PROC_UNLOCK(p);
205 hwt_ctx_free(ctx);
206 return (ENOMEM);
207 }
208
209 i = 0;
210
211 FOREACH_THREAD_IN_PROC(p, td1) {
212 threads[i++] = td1;
213 }
214
215 ctx->proc = p;
216 PROC_UNLOCK(p);
217
218 for (i = 0; i < cnt; i++) {
219 thread_id = atomic_fetchadd_int(&ctx->thread_counter, 1);
220 sprintf(path, "hwt_%d_%d", ctx->ident, thread_id);
221
222 error = hwt_thread_alloc(&thr, path, ctx->bufsize,
223 ctx->hwt_backend->kva_req);
224 if (error) {
225 free(threads, M_HWT_IOCTL);
226 hwt_ctx_free(ctx);
227 return (error);
228 }
229 /* Allocate backend-specific thread data. */
230 error = hwt_backend_thread_alloc(ctx, thr);
231 if (error != 0) {
232 dprintf("%s: failed to allocate thread backend data\n",
233 __func__);
234 free(threads, M_HWT_IOCTL);
235 hwt_ctx_free(ctx);
236 return (error);
237 }
238
239 /*
240 * Insert a THREAD_CREATE record so userspace picks up
241 * the thread's tracing buffers.
242 */
243 entry = hwt_record_entry_alloc();
244 entry->record_type = HWT_RECORD_THREAD_CREATE;
245 entry->thread_id = thread_id;
246
247 thr->vm->ctx = ctx;
248 thr->td = threads[i];
249 thr->ctx = ctx;
250 thr->backend = ctx->hwt_backend;
251 thr->thread_id = thread_id;
252
253 HWT_CTX_LOCK(ctx);
254 hwt_thread_insert(ctx, thr, entry);
255 HWT_CTX_UNLOCK(ctx);
256 }
257
258 free(threads, M_HWT_IOCTL);
259
260 error = hwt_backend_init(ctx);
261 if (error) {
262 hwt_ctx_free(ctx);
263 return (error);
264 }
265
266 /* hwt_owner_insert_ctx? */
267 mtx_lock(&ho->mtx);
268 LIST_INSERT_HEAD(&ho->hwts, ctx, next_hwts);
269 mtx_unlock(&ho->mtx);
270
271 /*
272 * Hooks are now in action after this, but the ctx is not in RUNNING
273 * state.
274 */
275 hwt_contexthash_insert(ctx);
276
277 p = pfind(halloc->pid);
278 if (p) {
279 p->p_flag2 |= P2_HWT;
280 PROC_UNLOCK(p);
281 }
282
283 return (0);
284 }
285
286 static int
hwt_ioctl_alloc_mode_cpu(struct thread * td,struct hwt_owner * ho,struct hwt_backend * backend,struct hwt_alloc * halloc)287 hwt_ioctl_alloc_mode_cpu(struct thread *td, struct hwt_owner *ho,
288 struct hwt_backend *backend, struct hwt_alloc *halloc)
289 {
290 struct hwt_context *ctx;
291 struct hwt_cpu *cpu;
292 struct hwt_vm *vm;
293 char path[MAXPATHLEN];
294 size_t cpusetsize;
295 cpuset_t cpu_map;
296 int cpu_count = 0;
297 int cpu_id;
298 int error;
299
300 CPU_ZERO(&cpu_map);
301 cpusetsize = min(halloc->cpusetsize, sizeof(cpuset_t));
302 error = copyin(halloc->cpu_map, &cpu_map, cpusetsize);
303 if (error)
304 return (error);
305
306 CPU_FOREACH_ISSET(cpu_id, &cpu_map) {
307 #ifdef SMP
308 /* Ensure CPU is not halted. */
309 if (CPU_ISSET(cpu_id, &hlt_cpus_mask))
310 return (ENXIO);
311 #endif
312 #if 0
313 /* TODO: Check if the owner have this cpu configured already. */
314 ctx = hwt_owner_lookup_ctx_by_cpu(ho, halloc->cpu);
315 if (ctx)
316 return (EEXIST);
317 #endif
318
319 cpu_count++;
320 }
321
322 if (cpu_count == 0)
323 return (ENODEV);
324
325 /* Allocate a new HWT context. */
326 error = hwt_ctx_alloc(&ctx);
327 if (error)
328 return (error);
329 ctx->bufsize = halloc->bufsize;
330 ctx->hwt_backend = backend;
331 ctx->hwt_owner = ho;
332 ctx->mode = HWT_MODE_CPU;
333 ctx->cpu_map = cpu_map;
334 ctx->hwt_td = td;
335 ctx->kqueue_fd = halloc->kqueue_fd;
336
337 error = copyout(&ctx->ident, halloc->ident, sizeof(int));
338 if (error) {
339 hwt_ctx_free(ctx);
340 return (error);
341 }
342
343 CPU_FOREACH_ISSET(cpu_id, &cpu_map) {
344 sprintf(path, "hwt_%d_%d", ctx->ident, cpu_id);
345 error = hwt_vm_alloc(ctx->bufsize, ctx->hwt_backend->kva_req,
346 path, &vm);
347 if (error) {
348 /* TODO: remove all allocated cpus. */
349 hwt_ctx_free(ctx);
350 return (error);
351 }
352
353 cpu = hwt_cpu_alloc();
354 cpu->cpu_id = cpu_id;
355 cpu->vm = vm;
356
357 vm->cpu = cpu;
358 vm->ctx = ctx;
359
360 HWT_CTX_LOCK(ctx);
361 hwt_cpu_insert(ctx, cpu);
362 HWT_CTX_UNLOCK(ctx);
363 }
364
365 error = hwt_backend_init(ctx);
366 if (error) {
367 /* TODO: remove all allocated cpus. */
368 hwt_ctx_free(ctx);
369 return (error);
370 }
371
372 /* hwt_owner_insert_ctx? */
373 mtx_lock(&ho->mtx);
374 LIST_INSERT_HEAD(&ho->hwts, ctx, next_hwts);
375 mtx_unlock(&ho->mtx);
376
377 hwt_record_kernel_objects(ctx);
378
379 return (0);
380 }
381
382 static int
hwt_ioctl_alloc(struct thread * td,struct hwt_alloc * halloc)383 hwt_ioctl_alloc(struct thread *td, struct hwt_alloc *halloc)
384 {
385 char backend_name[HWT_BACKEND_MAXNAMELEN];
386 struct hwt_backend *backend;
387 struct hwt_owner *ho;
388 int error;
389
390 if (halloc->bufsize > HWT_MAXBUFSIZE)
391 return (EINVAL);
392 if (halloc->bufsize % PAGE_SIZE)
393 return (EINVAL);
394 if (halloc->backend_name == NULL)
395 return (EINVAL);
396
397 error = copyinstr(halloc->backend_name, (void *)backend_name,
398 HWT_BACKEND_MAXNAMELEN, NULL);
399 if (error)
400 return (error);
401
402 backend = hwt_backend_lookup(backend_name);
403 if (backend == NULL)
404 return (ENODEV);
405
406 /* First get the owner. */
407 ho = hwt_ownerhash_lookup(td->td_proc);
408 if (ho == NULL) {
409 /* Create a new owner. */
410 ho = hwt_owner_alloc(td->td_proc);
411 if (ho == NULL)
412 return (ENOMEM);
413 hwt_ownerhash_insert(ho);
414 }
415
416 switch (halloc->mode) {
417 case HWT_MODE_THREAD:
418 error = hwt_ioctl_alloc_mode_thread(td, ho, backend, halloc);
419 break;
420 case HWT_MODE_CPU:
421 error = hwt_ioctl_alloc_mode_cpu(td, ho, backend, halloc);
422 break;
423 default:
424 error = ENXIO;
425 };
426
427 return (error);
428 }
429
430 int
hwt_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flags,struct thread * td)431 hwt_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
432 struct thread *td)
433 {
434 int error;
435
436 switch (cmd) {
437 case HWT_IOC_ALLOC:
438 /* Allocate HWT context. */
439 error = hwt_ioctl_alloc(td, (struct hwt_alloc *)addr);
440 return (error);
441 default:
442 return (ENXIO);
443 };
444 }
445