1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This work was supported by Innovate UK project 105694, "Digital Security 7 * by Design (DSbD) Technology Platform Prototype". 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/ioccom.h> 33 #include <sys/conf.h> 34 #include <sys/proc.h> 35 #include <sys/kernel.h> 36 #include <sys/malloc.h> 37 #include <sys/mman.h> 38 #include <sys/refcount.h> 39 #include <sys/rwlock.h> 40 #include <sys/hwt.h> 41 #include <sys/smp.h> 42 43 #include <vm/vm.h> 44 #include <vm/pmap.h> 45 #include <vm/vm_extern.h> 46 #include <vm/vm_param.h> 47 #include <vm/vm_kern.h> 48 #include <vm/vm_page.h> 49 #include <vm/vm_object.h> 50 #include <vm/vm_pager.h> 51 #include <vm/vm_pageout.h> 52 #include <vm/vm_phys.h> 53 54 #include <dev/hwt/hwt_hook.h> 55 #include <dev/hwt/hwt_context.h> 56 #include <dev/hwt/hwt_contexthash.h> 57 #include <dev/hwt/hwt_config.h> 58 #include <dev/hwt/hwt_cpu.h> 59 #include <dev/hwt/hwt_owner.h> 60 #include <dev/hwt/hwt_ownerhash.h> 61 #include <dev/hwt/hwt_thread.h> 62 #include <dev/hwt/hwt_backend.h> 63 #include <dev/hwt/hwt_vm.h> 64 #include <dev/hwt/hwt_record.h> 65 66 #define HWT_THREAD_DEBUG 67 #undef HWT_THREAD_DEBUG 68 69 #ifdef HWT_THREAD_DEBUG 70 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) 71 #else 72 #define dprintf(fmt, ...) 73 #endif 74 75 static MALLOC_DEFINE(M_HWT_VM, "hwt_vm", "Hardware Trace"); 76 77 static int 78 hwt_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset, 79 int prot, vm_page_t *mres) 80 { 81 82 return (0); 83 } 84 85 static int 86 hwt_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 87 vm_ooffset_t foff, struct ucred *cred, u_short *color) 88 { 89 90 *color = 0; 91 92 return (0); 93 } 94 95 static void 96 hwt_vm_dtor(void *handle) 97 { 98 99 } 100 101 static struct cdev_pager_ops hwt_vm_pager_ops = { 102 .cdev_pg_fault = hwt_vm_fault, 103 .cdev_pg_ctor = hwt_vm_ctor, 104 .cdev_pg_dtor = hwt_vm_dtor 105 }; 106 107 static int 108 hwt_vm_alloc_pages(struct hwt_vm *vm, int kva_req) 109 { 110 vm_paddr_t low, high, boundary; 111 vm_memattr_t memattr; 112 #ifdef __aarch64__ 113 uintptr_t va; 114 #endif 115 int alignment; 116 vm_page_t m; 117 int pflags; 118 int tries; 119 int i; 120 121 alignment = PAGE_SIZE; 122 low = 0; 123 high = -1UL; 124 boundary = 0; 125 pflags = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO; 126 memattr = VM_MEMATTR_DEVICE; 127 128 if (kva_req) { 129 vm->kvaddr = kva_alloc(vm->npages * PAGE_SIZE); 130 if (!vm->kvaddr) 131 return (ENOMEM); 132 } 133 134 vm->obj = cdev_pager_allocate(vm, OBJT_MGTDEVICE, 135 &hwt_vm_pager_ops, vm->npages * PAGE_SIZE, PROT_READ, 0, 136 curthread->td_ucred); 137 138 for (i = 0; i < vm->npages; i++) { 139 tries = 0; 140 retry: 141 m = vm_page_alloc_noobj_contig(pflags, 1, low, high, 142 alignment, boundary, memattr); 143 if (m == NULL) { 144 if (tries < 3) { 145 if (!vm_page_reclaim_contig(pflags, 1, low, 146 high, alignment, boundary)) 147 vm_wait(NULL); 148 tries++; 149 goto retry; 150 } 151 152 return (ENOMEM); 153 } 154 155 #if 0 156 /* TODO: could not clean device memory on arm64. */ 157 if ((m->flags & PG_ZERO) == 0) 158 pmap_zero_page(m); 159 #endif 160 161 #ifdef __aarch64__ 162 va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 163 cpu_dcache_wb_range((void *)va, PAGE_SIZE); 164 #endif 165 166 m->valid = VM_PAGE_BITS_ALL; 167 m->oflags &= ~VPO_UNMANAGED; 168 m->flags |= PG_FICTITIOUS; 169 vm->pages[i] = m; 170 171 VM_OBJECT_WLOCK(vm->obj); 172 vm_page_insert(m, vm->obj, i); 173 if (kva_req) 174 pmap_qenter(vm->kvaddr + i * PAGE_SIZE, &m, 1); 175 VM_OBJECT_WUNLOCK(vm->obj); 176 } 177 178 return (0); 179 } 180 181 static int 182 hwt_vm_open(struct cdev *cdev, int oflags, int devtype, struct thread *td) 183 { 184 185 dprintf("%s\n", __func__); 186 187 return (0); 188 } 189 190 static int 191 hwt_vm_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, 192 vm_size_t mapsize, struct vm_object **objp, int nprot) 193 { 194 struct hwt_vm *vm; 195 196 vm = cdev->si_drv1; 197 198 if (nprot != PROT_READ || *offset != 0) 199 return (ENXIO); 200 201 vm_object_reference(vm->obj); 202 *objp = vm->obj; 203 204 return (0); 205 } 206 207 static void 208 hwt_vm_start_cpu_mode(struct hwt_context *ctx) 209 { 210 cpuset_t enable_cpus; 211 int cpu_id; 212 213 CPU_ZERO(&enable_cpus); 214 215 CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) { 216 #ifdef SMP 217 /* Ensure CPU is not halted. */ 218 if (CPU_ISSET(cpu_id, &hlt_cpus_mask)) 219 continue; 220 #endif 221 222 hwt_backend_configure(ctx, cpu_id, cpu_id); 223 224 CPU_SET(cpu_id, &enable_cpus); 225 } 226 227 if (ctx->hwt_backend->ops->hwt_backend_enable_smp == NULL) { 228 CPU_FOREACH_ISSET(cpu_id, &enable_cpus) 229 hwt_backend_enable(ctx, cpu_id); 230 } else { 231 /* Some backends require enabling all CPUs at once. */ 232 hwt_backend_enable_smp(ctx); 233 } 234 } 235 236 static int 237 hwt_vm_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, 238 struct thread *td) 239 { 240 struct hwt_record_get *rget; 241 struct hwt_set_config *sconf; 242 struct hwt_bufptr_get *ptr_get; 243 struct hwt_svc_buf *sbuf; 244 245 struct hwt_context *ctx; 246 struct hwt_vm *vm; 247 struct hwt_owner *ho; 248 249 vm_offset_t offset; 250 int ident; 251 int error; 252 uint64_t data = 0; 253 void *data2; 254 size_t data_size; 255 int data_version; 256 257 vm = dev->si_drv1; 258 KASSERT(vm != NULL, ("si_drv1 is NULL")); 259 260 ctx = vm->ctx; 261 262 /* Ensure process is registered owner of this HWT. */ 263 ho = hwt_ownerhash_lookup(td->td_proc); 264 if (ho == NULL) 265 return (ENXIO); 266 267 if (ctx->hwt_owner != ho) 268 return (EPERM); 269 270 switch (cmd) { 271 case HWT_IOC_START: 272 dprintf("%s: start tracing\n", __func__); 273 274 HWT_CTX_LOCK(ctx); 275 if (ctx->state == CTX_STATE_RUNNING) { 276 /* Already running ? */ 277 HWT_CTX_UNLOCK(ctx); 278 return (ENXIO); 279 } 280 ctx->state = CTX_STATE_RUNNING; 281 HWT_CTX_UNLOCK(ctx); 282 283 if (ctx->mode == HWT_MODE_CPU) 284 hwt_vm_start_cpu_mode(ctx); 285 else { 286 /* 287 * Tracing backend will be configured and enabled 288 * during hook invocation. See hwt_hook.c. 289 */ 290 } 291 292 break; 293 294 case HWT_IOC_STOP: 295 if (ctx->state == CTX_STATE_STOPPED) 296 return (ENXIO); 297 hwt_backend_stop(ctx); 298 ctx->state = CTX_STATE_STOPPED; 299 break; 300 301 case HWT_IOC_RECORD_GET: 302 rget = (struct hwt_record_get *)addr; 303 error = hwt_record_send(ctx, rget); 304 if (error) 305 return (error); 306 break; 307 308 case HWT_IOC_SET_CONFIG: 309 if (ctx->state == CTX_STATE_RUNNING) { 310 return (ENXIO); 311 } 312 sconf = (struct hwt_set_config *)addr; 313 error = hwt_config_set(td, ctx, sconf); 314 if (error) 315 return (error); 316 ctx->pause_on_mmap = sconf->pause_on_mmap ? 1 : 0; 317 break; 318 319 case HWT_IOC_WAKEUP: 320 321 if (ctx->mode == HWT_MODE_CPU) 322 return (ENXIO); 323 324 KASSERT(vm->thr != NULL, ("thr is NULL")); 325 326 wakeup(vm->thr); 327 328 break; 329 330 case HWT_IOC_BUFPTR_GET: 331 ptr_get = (struct hwt_bufptr_get *)addr; 332 333 error = hwt_backend_read(ctx, vm, &ident, &offset, &data); 334 if (error) 335 return (error); 336 337 if (ptr_get->ident) 338 error = copyout(&ident, ptr_get->ident, sizeof(int)); 339 if (error) 340 return (error); 341 342 if (ptr_get->offset) 343 error = copyout(&offset, ptr_get->offset, 344 sizeof(vm_offset_t)); 345 if (error) 346 return (error); 347 348 if (ptr_get->data) 349 error = copyout(&data, ptr_get->data, sizeof(uint64_t)); 350 if (error) 351 return (error); 352 353 break; 354 355 case HWT_IOC_SVC_BUF: 356 if (ctx->state == CTX_STATE_STOPPED) { 357 return (ENXIO); 358 } 359 360 sbuf = (struct hwt_svc_buf *)addr; 361 data_size = sbuf->data_size; 362 data_version = sbuf->data_version; 363 364 if (data_size == 0 || data_size > PAGE_SIZE) 365 return (EINVAL); 366 367 data2 = malloc(data_size, M_HWT_VM, M_WAITOK | M_ZERO); 368 error = copyin(sbuf->data, data2, data_size); 369 if (error) { 370 free(data2, M_HWT_VM); 371 return (error); 372 } 373 374 error = hwt_backend_svc_buf(ctx, data2, data_size, data_version); 375 if (error) { 376 free(data2, M_HWT_VM); 377 return (error); 378 } 379 380 free(data2, M_HWT_VM); 381 break; 382 383 default: 384 break; 385 } 386 387 return (0); 388 } 389 390 static struct cdevsw hwt_vm_cdevsw = { 391 .d_version = D_VERSION, 392 .d_name = "hwt", 393 .d_open = hwt_vm_open, 394 .d_mmap_single = hwt_vm_mmap_single, 395 .d_ioctl = hwt_vm_ioctl, 396 }; 397 398 static int 399 hwt_vm_create_cdev(struct hwt_vm *vm, char *path) 400 { 401 struct make_dev_args args; 402 int error; 403 404 dprintf("%s: path %s\n", __func__, path); 405 406 make_dev_args_init(&args); 407 args.mda_devsw = &hwt_vm_cdevsw; 408 args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK; 409 args.mda_uid = UID_ROOT; 410 args.mda_gid = GID_WHEEL; 411 args.mda_mode = 0660; 412 args.mda_si_drv1 = vm; 413 414 error = make_dev_s(&args, &vm->cdev, "%s", path); 415 if (error != 0) 416 return (error); 417 418 return (0); 419 } 420 421 static int 422 hwt_vm_alloc_buffers(struct hwt_vm *vm, int kva_req) 423 { 424 int error; 425 426 vm->pages = malloc(sizeof(struct vm_page *) * vm->npages, 427 M_HWT_VM, M_WAITOK | M_ZERO); 428 429 error = hwt_vm_alloc_pages(vm, kva_req); 430 if (error) { 431 printf("%s: could not alloc pages\n", __func__); 432 return (error); 433 } 434 435 return (0); 436 } 437 438 static void 439 hwt_vm_destroy_buffers(struct hwt_vm *vm) 440 { 441 vm_page_t m; 442 int i; 443 444 if (vm->ctx->hwt_backend->kva_req && vm->kvaddr != 0) { 445 pmap_qremove(vm->kvaddr, vm->npages); 446 kva_free(vm->kvaddr, vm->npages * PAGE_SIZE); 447 } 448 VM_OBJECT_WLOCK(vm->obj); 449 for (i = 0; i < vm->npages; i++) { 450 m = vm->pages[i]; 451 if (m == NULL) 452 break; 453 454 vm_page_busy_acquire(m, 0); 455 cdev_pager_free_page(vm->obj, m); 456 m->flags &= ~PG_FICTITIOUS; 457 vm_page_unwire_noq(m); 458 vm_page_free(m); 459 460 } 461 vm_pager_deallocate(vm->obj); 462 VM_OBJECT_WUNLOCK(vm->obj); 463 464 free(vm->pages, M_HWT_VM); 465 } 466 467 void 468 hwt_vm_free(struct hwt_vm *vm) 469 { 470 471 dprintf("%s\n", __func__); 472 473 if (vm->cdev) 474 destroy_dev_sched(vm->cdev); 475 hwt_vm_destroy_buffers(vm); 476 free(vm, M_HWT_VM); 477 } 478 479 int 480 hwt_vm_alloc(size_t bufsize, int kva_req, char *path, struct hwt_vm **vm0) 481 { 482 struct hwt_vm *vm; 483 int error; 484 485 vm = malloc(sizeof(struct hwt_vm), M_HWT_VM, M_WAITOK | M_ZERO); 486 vm->npages = bufsize / PAGE_SIZE; 487 488 error = hwt_vm_alloc_buffers(vm, kva_req); 489 if (error) { 490 free(vm, M_HWT_VM); 491 return (error); 492 } 493 494 error = hwt_vm_create_cdev(vm, path); 495 if (error) { 496 hwt_vm_free(vm); 497 return (error); 498 } 499 500 *vm0 = vm; 501 502 return (0); 503 } 504