1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
5 *
6 * This work was supported by Innovate UK project 105694, "Digital Security
7 * by Design (DSbD) Technology Platform Prototype".
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/ioccom.h>
33 #include <sys/conf.h>
34 #include <sys/proc.h>
35 #include <sys/kernel.h>
36 #include <sys/malloc.h>
37 #include <sys/mman.h>
38 #include <sys/refcount.h>
39 #include <sys/rwlock.h>
40 #include <sys/hwt.h>
41 #include <sys/smp.h>
42
43 #include <vm/vm.h>
44 #include <vm/pmap.h>
45 #include <vm/vm_extern.h>
46 #include <vm/vm_param.h>
47 #include <vm/vm_kern.h>
48 #include <vm/vm_page.h>
49 #include <vm/vm_object.h>
50 #include <vm/vm_pager.h>
51 #include <vm/vm_pageout.h>
52 #include <vm/vm_phys.h>
53
54 #include <dev/hwt/hwt_hook.h>
55 #include <dev/hwt/hwt_context.h>
56 #include <dev/hwt/hwt_contexthash.h>
57 #include <dev/hwt/hwt_config.h>
58 #include <dev/hwt/hwt_cpu.h>
59 #include <dev/hwt/hwt_owner.h>
60 #include <dev/hwt/hwt_ownerhash.h>
61 #include <dev/hwt/hwt_thread.h>
62 #include <dev/hwt/hwt_backend.h>
63 #include <dev/hwt/hwt_vm.h>
64 #include <dev/hwt/hwt_record.h>
65
66 #define HWT_THREAD_DEBUG
67 #undef HWT_THREAD_DEBUG
68
69 #ifdef HWT_THREAD_DEBUG
70 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
71 #else
72 #define dprintf(fmt, ...)
73 #endif
74
75 static MALLOC_DEFINE(M_HWT_VM, "hwt_vm", "Hardware Trace");
76
77 static int
hwt_vm_fault(vm_object_t vm_obj,vm_ooffset_t offset,int prot,vm_page_t * mres)78 hwt_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
79 int prot, vm_page_t *mres)
80 {
81
82 return (0);
83 }
84
85 static int
hwt_vm_ctor(void * handle,vm_ooffset_t size,vm_prot_t prot,vm_ooffset_t foff,struct ucred * cred,u_short * color)86 hwt_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
87 vm_ooffset_t foff, struct ucred *cred, u_short *color)
88 {
89
90 *color = 0;
91
92 return (0);
93 }
94
95 static void
hwt_vm_dtor(void * handle)96 hwt_vm_dtor(void *handle)
97 {
98
99 }
100
101 static struct cdev_pager_ops hwt_vm_pager_ops = {
102 .cdev_pg_fault = hwt_vm_fault,
103 .cdev_pg_ctor = hwt_vm_ctor,
104 .cdev_pg_dtor = hwt_vm_dtor
105 };
106
107 static int
hwt_vm_alloc_pages(struct hwt_vm * vm,int kva_req)108 hwt_vm_alloc_pages(struct hwt_vm *vm, int kva_req)
109 {
110 vm_paddr_t low, high, boundary;
111 vm_memattr_t memattr;
112 int alignment;
113 vm_page_t m;
114 int pflags;
115 int tries;
116 int i;
117
118 alignment = PAGE_SIZE;
119 low = 0;
120 high = -1UL;
121 boundary = 0;
122 pflags = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO;
123 memattr = VM_MEMATTR_DEVICE;
124
125 if (kva_req) {
126 vm->kvaddr = kva_alloc(vm->npages * PAGE_SIZE);
127 if (vm->kvaddr == NULL)
128 return (ENOMEM);
129 }
130
131 vm->obj = cdev_pager_allocate(vm, OBJT_MGTDEVICE,
132 &hwt_vm_pager_ops, vm->npages * PAGE_SIZE, PROT_READ, 0,
133 curthread->td_ucred);
134
135 for (i = 0; i < vm->npages; i++) {
136 tries = 0;
137 retry:
138 m = vm_page_alloc_noobj_contig(pflags, 1, low, high,
139 alignment, boundary, memattr);
140 if (m == NULL) {
141 if (tries < 3) {
142 if (!vm_page_reclaim_contig(pflags, 1, low,
143 high, alignment, boundary))
144 vm_wait(NULL);
145 tries++;
146 goto retry;
147 }
148
149 return (ENOMEM);
150 }
151
152 #if 0
153 /* TODO: could not clean device memory on arm64. */
154 if ((m->flags & PG_ZERO) == 0)
155 pmap_zero_page(m);
156 #endif
157
158 #ifdef __aarch64__
159 cpu_dcache_wb_range(VM_PAGE_TO_DMAP(m), PAGE_SIZE);
160 #endif
161
162 m->valid = VM_PAGE_BITS_ALL;
163 m->oflags &= ~VPO_UNMANAGED;
164 m->flags |= PG_FICTITIOUS;
165 vm->pages[i] = m;
166
167 VM_OBJECT_WLOCK(vm->obj);
168 vm_page_insert(m, vm->obj, i);
169 if (kva_req)
170 pmap_qenter((char *)vm->kvaddr + i * PAGE_SIZE, &m, 1);
171 VM_OBJECT_WUNLOCK(vm->obj);
172 }
173
174 return (0);
175 }
176
177 static int
hwt_vm_open(struct cdev * cdev,int oflags,int devtype,struct thread * td)178 hwt_vm_open(struct cdev *cdev, int oflags, int devtype, struct thread *td)
179 {
180
181 dprintf("%s\n", __func__);
182
183 return (0);
184 }
185
186 static int
hwt_vm_mmap_single(struct cdev * cdev,vm_ooffset_t * offset,vm_size_t mapsize,struct vm_object ** objp,int nprot)187 hwt_vm_mmap_single(struct cdev *cdev, vm_ooffset_t *offset,
188 vm_size_t mapsize, struct vm_object **objp, int nprot)
189 {
190 struct hwt_vm *vm;
191
192 vm = cdev->si_drv1;
193
194 if (nprot != PROT_READ || *offset != 0)
195 return (ENXIO);
196
197 vm_object_reference(vm->obj);
198 *objp = vm->obj;
199
200 return (0);
201 }
202
203 static void
hwt_vm_start_cpu_mode(struct hwt_context * ctx)204 hwt_vm_start_cpu_mode(struct hwt_context *ctx)
205 {
206 cpuset_t enable_cpus;
207 int cpu_id;
208
209 CPU_ZERO(&enable_cpus);
210
211 CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
212 #ifdef SMP
213 /* Ensure CPU is not halted. */
214 if (CPU_ISSET(cpu_id, &hlt_cpus_mask))
215 continue;
216 #endif
217
218 hwt_backend_configure(ctx, cpu_id, cpu_id);
219
220 CPU_SET(cpu_id, &enable_cpus);
221 }
222
223 if (ctx->hwt_backend->ops->hwt_backend_enable_smp == NULL) {
224 CPU_FOREACH_ISSET(cpu_id, &enable_cpus)
225 hwt_backend_enable(ctx, cpu_id);
226 } else {
227 /* Some backends require enabling all CPUs at once. */
228 hwt_backend_enable_smp(ctx);
229 }
230 }
231
232 static int
hwt_vm_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flags,struct thread * td)233 hwt_vm_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
234 struct thread *td)
235 {
236 struct hwt_record_get *rget;
237 struct hwt_set_config *sconf;
238 struct hwt_bufptr_get *ptr_get;
239 struct hwt_svc_buf *sbuf;
240
241 struct hwt_context *ctx;
242 struct hwt_vm *vm;
243 struct hwt_owner *ho;
244
245 vm_offset_t offset;
246 int ident;
247 int error;
248 uint64_t data = 0;
249 void *data2;
250 size_t data_size;
251 int data_version;
252
253 vm = dev->si_drv1;
254 KASSERT(vm != NULL, ("si_drv1 is NULL"));
255
256 ctx = vm->ctx;
257
258 /* Ensure process is registered owner of this HWT. */
259 ho = hwt_ownerhash_lookup(td->td_proc);
260 if (ho == NULL)
261 return (ENXIO);
262
263 if (ctx->hwt_owner != ho)
264 return (EPERM);
265
266 switch (cmd) {
267 case HWT_IOC_START:
268 dprintf("%s: start tracing\n", __func__);
269
270 HWT_CTX_LOCK(ctx);
271 if (ctx->state == CTX_STATE_RUNNING) {
272 /* Already running ? */
273 HWT_CTX_UNLOCK(ctx);
274 return (ENXIO);
275 }
276 ctx->state = CTX_STATE_RUNNING;
277 HWT_CTX_UNLOCK(ctx);
278
279 if (ctx->mode == HWT_MODE_CPU)
280 hwt_vm_start_cpu_mode(ctx);
281 else {
282 /*
283 * Tracing backend will be configured and enabled
284 * during hook invocation. See hwt_hook.c.
285 */
286 }
287
288 break;
289
290 case HWT_IOC_STOP:
291 if (ctx->state == CTX_STATE_STOPPED)
292 return (ENXIO);
293 hwt_backend_stop(ctx);
294 ctx->state = CTX_STATE_STOPPED;
295 break;
296
297 case HWT_IOC_RECORD_GET:
298 rget = (struct hwt_record_get *)addr;
299 error = hwt_record_send(ctx, rget);
300 if (error)
301 return (error);
302 break;
303
304 case HWT_IOC_SET_CONFIG:
305 if (ctx->state == CTX_STATE_RUNNING) {
306 return (ENXIO);
307 }
308 sconf = (struct hwt_set_config *)addr;
309 error = hwt_config_set(td, ctx, sconf);
310 if (error)
311 return (error);
312 ctx->pause_on_mmap = sconf->pause_on_mmap ? 1 : 0;
313 break;
314
315 case HWT_IOC_WAKEUP:
316
317 if (ctx->mode == HWT_MODE_CPU)
318 return (ENXIO);
319
320 KASSERT(vm->thr != NULL, ("thr is NULL"));
321
322 wakeup(vm->thr);
323
324 break;
325
326 case HWT_IOC_BUFPTR_GET:
327 ptr_get = (struct hwt_bufptr_get *)addr;
328
329 error = hwt_backend_read(ctx, vm, &ident, &offset, &data);
330 if (error)
331 return (error);
332
333 if (ptr_get->ident)
334 error = copyout(&ident, ptr_get->ident, sizeof(int));
335 if (error)
336 return (error);
337
338 if (ptr_get->offset)
339 error = copyout(&offset, ptr_get->offset,
340 sizeof(vm_offset_t));
341 if (error)
342 return (error);
343
344 if (ptr_get->data)
345 error = copyout(&data, ptr_get->data, sizeof(uint64_t));
346 if (error)
347 return (error);
348
349 break;
350
351 case HWT_IOC_SVC_BUF:
352 if (ctx->state == CTX_STATE_STOPPED) {
353 return (ENXIO);
354 }
355
356 sbuf = (struct hwt_svc_buf *)addr;
357 data_size = sbuf->data_size;
358 data_version = sbuf->data_version;
359
360 if (data_size == 0 || data_size > PAGE_SIZE)
361 return (EINVAL);
362
363 data2 = malloc(data_size, M_HWT_VM, M_WAITOK | M_ZERO);
364 error = copyin(sbuf->data, data2, data_size);
365 if (error) {
366 free(data2, M_HWT_VM);
367 return (error);
368 }
369
370 error = hwt_backend_svc_buf(ctx, data2, data_size, data_version);
371 if (error) {
372 free(data2, M_HWT_VM);
373 return (error);
374 }
375
376 free(data2, M_HWT_VM);
377 break;
378
379 default:
380 break;
381 }
382
383 return (0);
384 }
385
386 static struct cdevsw hwt_vm_cdevsw = {
387 .d_version = D_VERSION,
388 .d_name = "hwt",
389 .d_open = hwt_vm_open,
390 .d_mmap_single = hwt_vm_mmap_single,
391 .d_ioctl = hwt_vm_ioctl,
392 };
393
394 static int
hwt_vm_create_cdev(struct hwt_vm * vm,char * path)395 hwt_vm_create_cdev(struct hwt_vm *vm, char *path)
396 {
397 struct make_dev_args args;
398 int error;
399
400 dprintf("%s: path %s\n", __func__, path);
401
402 make_dev_args_init(&args);
403 args.mda_devsw = &hwt_vm_cdevsw;
404 args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK;
405 args.mda_uid = UID_ROOT;
406 args.mda_gid = GID_WHEEL;
407 args.mda_mode = 0660;
408 args.mda_si_drv1 = vm;
409
410 error = make_dev_s(&args, &vm->cdev, "%s", path);
411 if (error != 0)
412 return (error);
413
414 return (0);
415 }
416
417 static int
hwt_vm_alloc_buffers(struct hwt_vm * vm,int kva_req)418 hwt_vm_alloc_buffers(struct hwt_vm *vm, int kva_req)
419 {
420 int error;
421
422 vm->pages = malloc(sizeof(struct vm_page *) * vm->npages,
423 M_HWT_VM, M_WAITOK | M_ZERO);
424
425 error = hwt_vm_alloc_pages(vm, kva_req);
426 if (error) {
427 printf("%s: could not alloc pages\n", __func__);
428 return (error);
429 }
430
431 return (0);
432 }
433
434 static void
hwt_vm_destroy_buffers(struct hwt_vm * vm)435 hwt_vm_destroy_buffers(struct hwt_vm *vm)
436 {
437 vm_page_t m;
438 int i;
439
440 if (vm->ctx->hwt_backend->kva_req && vm->kvaddr != NULL) {
441 pmap_qremove(vm->kvaddr, vm->npages);
442 kva_free(vm->kvaddr, vm->npages * PAGE_SIZE);
443 }
444 VM_OBJECT_WLOCK(vm->obj);
445 for (i = 0; i < vm->npages; i++) {
446 m = vm->pages[i];
447 if (m == NULL)
448 break;
449
450 vm_page_busy_acquire(m, 0);
451 cdev_pager_free_page(vm->obj, m);
452 m->flags &= ~PG_FICTITIOUS;
453 vm_page_unwire_noq(m);
454 vm_page_free(m);
455
456 }
457 vm_pager_deallocate(vm->obj);
458 VM_OBJECT_WUNLOCK(vm->obj);
459
460 free(vm->pages, M_HWT_VM);
461 }
462
463 void
hwt_vm_free(struct hwt_vm * vm)464 hwt_vm_free(struct hwt_vm *vm)
465 {
466
467 dprintf("%s\n", __func__);
468
469 if (vm->cdev)
470 destroy_dev_sched(vm->cdev);
471 hwt_vm_destroy_buffers(vm);
472 free(vm, M_HWT_VM);
473 }
474
475 int
hwt_vm_alloc(size_t bufsize,int kva_req,char * path,struct hwt_vm ** vm0)476 hwt_vm_alloc(size_t bufsize, int kva_req, char *path, struct hwt_vm **vm0)
477 {
478 struct hwt_vm *vm;
479 int error;
480
481 vm = malloc(sizeof(struct hwt_vm), M_HWT_VM, M_WAITOK | M_ZERO);
482 vm->npages = bufsize / PAGE_SIZE;
483
484 error = hwt_vm_alloc_buffers(vm, kva_req);
485 if (error) {
486 free(vm, M_HWT_VM);
487 return (error);
488 }
489
490 error = hwt_vm_create_cdev(vm, path);
491 if (error) {
492 hwt_vm_free(vm);
493 return (error);
494 }
495
496 *vm0 = vm;
497
498 return (0);
499 }
500