1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. 5 * Copyright (C) 2018, 2019 Andrew Turner 6 * 7 * This software was developed by Mitchell Horne under sponsorship of 8 * the FreeBSD Foundation. 9 * 10 * This software was developed by SRI International and the University of 11 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 12 * ("CTSRD"), as part of the DARPA CRASH research programme. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * $FreeBSD$ 36 */ 37 38 #include <sys/cdefs.h> 39 __FBSDID("$FreeBSD$"); 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/conf.h> 44 #include <sys/eventhandler.h> 45 #include <sys/kcov.h> 46 #include <sys/kernel.h> 47 #include <sys/limits.h> 48 #include <sys/lock.h> 49 #include <sys/malloc.h> 50 #include <sys/mman.h> 51 #include <sys/mutex.h> 52 #include <sys/proc.h> 53 #include <sys/rwlock.h> 54 #include <sys/sysctl.h> 55 56 #include <vm/vm.h> 57 #include <vm/pmap.h> 58 #include <vm/vm_extern.h> 59 #include <vm/vm_object.h> 60 #include <vm/vm_page.h> 61 #include <vm/vm_pager.h> 62 #include <vm/vm_param.h> 63 64 MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); 65 66 #define KCOV_ELEMENT_SIZE sizeof(uint64_t) 67 68 /* 69 * To know what the code can safely perform at any point in time we use a 70 * state machine. In the normal case the state transitions are: 71 * 72 * OPEN -> READY -> RUNNING -> DYING 73 * | | ^ | ^ ^ 74 * | | +--------+ | | 75 * | +-------------------+ | 76 * +-----------------------------+ 77 * 78 * The states are: 79 * OPEN: The kcov fd has been opened, but no buffer is available to store 80 * coverage data. 81 * READY: The buffer to store coverage data has been allocated. Userspace 82 * can set this by using ioctl(fd, KIOSETBUFSIZE, entries);. When 83 * this has been set the buffer can be written to by the kernel, 84 * and mmaped by userspace. 85 * RUNNING: The coverage probes are able to store coverage data in the buffer. 86 * This is entered with ioctl(fd, KIOENABLE, mode);. The READY state 87 * can be exited by ioctl(fd, KIODISABLE); or exiting the thread to 88 * return to the READY state to allow tracing to be reused, or by 89 * closing the kcov fd to enter the DYING state. 90 * DYING: The fd has been closed. All states can enter into this state when 91 * userspace closes the kcov fd. 92 * 93 * We need to be careful when moving into and out of the RUNNING state. As 94 * an interrupt may happen while this is happening the ordering of memory 95 * operations is important so struct kcov_info is valid for the tracing 96 * functions. 97 * 98 * When moving into the RUNNING state prior stores to struct kcov_info need 99 * to be observed before the state is set. This allows for interrupts that 100 * may call into one of the coverage functions to fire at any point while 101 * being enabled and see a consistent struct kcov_info. 102 * 103 * When moving out of the RUNNING state any later stores to struct kcov_info 104 * need to be observed after the state is set. As with entering this is to 105 * present a consistent struct kcov_info to interrupts. 106 */ 107 typedef enum { 108 KCOV_STATE_INVALID, 109 KCOV_STATE_OPEN, /* The device is open, but with no buffer */ 110 KCOV_STATE_READY, /* The buffer has been allocated */ 111 KCOV_STATE_RUNNING, /* Recording trace data */ 112 KCOV_STATE_DYING, /* The fd was closed */ 113 } kcov_state_t; 114 115 /* 116 * (l) Set while holding the kcov_lock mutex and not in the RUNNING state. 117 * (o) Only set once while in the OPEN state. Cleaned up while in the DYING 118 * state, and with no thread associated with the struct kcov_info. 119 * (s) Set atomically to enter or exit the RUNNING state, non-atomically 120 * otherwise. See above for a description of the other constraints while 121 * moving into or out of the RUNNING state. 122 */ 123 struct kcov_info { 124 struct thread *thread; /* (l) */ 125 vm_object_t bufobj; /* (o) */ 126 vm_offset_t kvaddr; /* (o) */ 127 size_t entries; /* (o) */ 128 size_t bufsize; /* (o) */ 129 kcov_state_t state; /* (s) */ 130 int mode; /* (l) */ 131 }; 132 133 /* Prototypes */ 134 static d_open_t kcov_open; 135 static d_close_t kcov_close; 136 static d_mmap_single_t kcov_mmap_single; 137 static d_ioctl_t kcov_ioctl; 138 139 static int kcov_alloc(struct kcov_info *info, size_t entries); 140 static void kcov_free(struct kcov_info *info); 141 static void kcov_init(const void *unused); 142 143 static struct cdevsw kcov_cdevsw = { 144 .d_version = D_VERSION, 145 .d_open = kcov_open, 146 .d_close = kcov_close, 147 .d_mmap_single = kcov_mmap_single, 148 .d_ioctl = kcov_ioctl, 149 .d_name = "kcov", 150 }; 151 152 SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW, 0, "Kernel coverage"); 153 154 static u_int kcov_max_entries = KCOV_MAXENTRIES; 155 SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW, 156 &kcov_max_entries, 0, 157 "Maximum number of entries in the kcov buffer"); 158 159 static struct mtx kcov_lock; 160 static int active_count; 161 162 static struct kcov_info * 163 get_kinfo(struct thread *td) 164 { 165 struct kcov_info *info; 166 167 /* We might have a NULL thread when releasing the secondary CPUs */ 168 if (td == NULL) 169 return (NULL); 170 171 /* 172 * We are in an interrupt, stop tracing as it is not explicitly 173 * part of a syscall. 174 */ 175 if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL) 176 return (NULL); 177 178 /* 179 * If info is NULL or the state is not running we are not tracing. 180 */ 181 info = td->td_kcov_info; 182 if (info == NULL || 183 atomic_load_acq_int(&info->state) != KCOV_STATE_RUNNING) 184 return (NULL); 185 186 return (info); 187 } 188 189 static void 190 trace_pc(uintptr_t ret) 191 { 192 struct thread *td; 193 struct kcov_info *info; 194 uint64_t *buf, index; 195 196 td = curthread; 197 info = get_kinfo(td); 198 if (info == NULL) 199 return; 200 201 /* 202 * Check we are in the PC-trace mode. 203 */ 204 if (info->mode != KCOV_MODE_TRACE_PC) 205 return; 206 207 KASSERT(info->kvaddr != 0, 208 ("__sanitizer_cov_trace_pc: NULL buf while running")); 209 210 buf = (uint64_t *)info->kvaddr; 211 212 /* The first entry of the buffer holds the index */ 213 index = buf[0]; 214 if (index + 2 > info->entries) 215 return; 216 217 buf[index + 1] = ret; 218 buf[0] = index + 1; 219 } 220 221 static bool 222 trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret) 223 { 224 struct thread *td; 225 struct kcov_info *info; 226 uint64_t *buf, index; 227 228 td = curthread; 229 info = get_kinfo(td); 230 if (info == NULL) 231 return (false); 232 233 /* 234 * Check we are in the comparison-trace mode. 235 */ 236 if (info->mode != KCOV_MODE_TRACE_CMP) 237 return (false); 238 239 KASSERT(info->kvaddr != 0, 240 ("__sanitizer_cov_trace_pc: NULL buf while running")); 241 242 buf = (uint64_t *)info->kvaddr; 243 244 /* The first entry of the buffer holds the index */ 245 index = buf[0]; 246 247 /* Check we have space to store all elements */ 248 if (index * 4 + 4 + 1 > info->entries) 249 return (false); 250 251 while (1) { 252 buf[index * 4 + 1] = type; 253 buf[index * 4 + 2] = arg1; 254 buf[index * 4 + 3] = arg2; 255 buf[index * 4 + 4] = ret; 256 257 if (atomic_cmpset_64(&buf[0], index, index + 1)) 258 break; 259 buf[0] = index; 260 } 261 262 return (true); 263 } 264 265 /* 266 * The fd is being closed, cleanup everything we can. 267 */ 268 static void 269 kcov_mmap_cleanup(void *arg) 270 { 271 struct kcov_info *info = arg; 272 struct thread *thread; 273 274 mtx_lock_spin(&kcov_lock); 275 /* 276 * Move to KCOV_STATE_DYING to stop adding new entries. 277 * 278 * If the thread is running we need to wait until thread exit to 279 * clean up as it may currently be adding a new entry. If this is 280 * the case being in KCOV_STATE_DYING will signal that the buffer 281 * needs to be cleaned up. 282 */ 283 atomic_store_int(&info->state, KCOV_STATE_DYING); 284 atomic_thread_fence_seq_cst(); 285 thread = info->thread; 286 mtx_unlock_spin(&kcov_lock); 287 288 if (thread != NULL) 289 return; 290 291 /* 292 * We can safely clean up the info struct as it is in the 293 * KCOV_STATE_DYING state with no thread associated. 294 * 295 * The KCOV_STATE_DYING stops new threads from using it. 296 * The lack of a thread means nothing is currently using the buffers. 297 */ 298 kcov_free(info); 299 } 300 301 static int 302 kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 303 { 304 struct kcov_info *info; 305 int error; 306 307 info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK); 308 info->state = KCOV_STATE_OPEN; 309 info->thread = NULL; 310 info->mode = -1; 311 312 if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0) 313 kcov_mmap_cleanup(info); 314 315 return (error); 316 } 317 318 static int 319 kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 320 { 321 struct kcov_info *info; 322 int error; 323 324 325 if ((error = devfs_get_cdevpriv((void **)&info)) != 0) 326 return (error); 327 328 KASSERT(info != NULL, ("kcov_close with no kcov_info structure")); 329 330 /* Trying to close, but haven't disabled */ 331 if (info->state == KCOV_STATE_RUNNING) 332 return (EBUSY); 333 334 return (0); 335 } 336 337 static int 338 kcov_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size, 339 struct vm_object **object, int nprot) 340 { 341 struct kcov_info *info; 342 int error; 343 344 if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) != 345 (PROT_READ | PROT_WRITE)) 346 return (EINVAL); 347 348 if ((error = devfs_get_cdevpriv((void **)&info)) != 0) 349 return (error); 350 351 if (info->kvaddr == 0 || size / KCOV_ELEMENT_SIZE != info->entries) 352 return (EINVAL); 353 354 vm_object_reference(info->bufobj); 355 *offset = 0; 356 *object = info->bufobj; 357 return (0); 358 } 359 360 static int 361 kcov_alloc(struct kcov_info *info, size_t entries) 362 { 363 size_t n, pages; 364 vm_page_t m; 365 366 KASSERT(info->kvaddr == 0, ("kcov_alloc: Already have a buffer")); 367 KASSERT(info->state == KCOV_STATE_OPEN, 368 ("kcov_alloc: Not in open state (%x)", info->state)); 369 370 if (entries < 2 || entries > kcov_max_entries) 371 return (EINVAL); 372 373 /* Align to page size so mmap can't access other kernel memory */ 374 info->bufsize = roundup2(entries * KCOV_ELEMENT_SIZE, PAGE_SIZE); 375 pages = info->bufsize / PAGE_SIZE; 376 377 if ((info->kvaddr = kva_alloc(info->bufsize)) == 0) 378 return (ENOMEM); 379 380 info->bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->bufsize, 381 PROT_READ | PROT_WRITE, 0, curthread->td_ucred); 382 383 VM_OBJECT_WLOCK(info->bufobj); 384 for (n = 0; n < pages; n++) { 385 m = vm_page_grab(info->bufobj, n, 386 VM_ALLOC_ZERO | VM_ALLOC_WIRED); 387 vm_page_valid(m); 388 vm_page_xunbusy(m); 389 pmap_qenter(info->kvaddr + n * PAGE_SIZE, &m, 1); 390 } 391 VM_OBJECT_WUNLOCK(info->bufobj); 392 393 info->entries = entries; 394 395 return (0); 396 } 397 398 static void 399 kcov_free(struct kcov_info *info) 400 { 401 vm_page_t m; 402 size_t i; 403 404 if (info->kvaddr != 0) { 405 pmap_qremove(info->kvaddr, info->bufsize / PAGE_SIZE); 406 kva_free(info->kvaddr, info->bufsize); 407 } 408 if (info->bufobj != NULL) { 409 VM_OBJECT_WLOCK(info->bufobj); 410 m = vm_page_lookup(info->bufobj, 0); 411 for (i = 0; i < info->bufsize / PAGE_SIZE; i++) { 412 vm_page_unwire_noq(m); 413 m = vm_page_next(m); 414 } 415 VM_OBJECT_WUNLOCK(info->bufobj); 416 vm_object_deallocate(info->bufobj); 417 } 418 free(info, M_KCOV_INFO); 419 } 420 421 static int 422 kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused, 423 struct thread *td) 424 { 425 struct kcov_info *info; 426 int mode, error; 427 428 if ((error = devfs_get_cdevpriv((void **)&info)) != 0) 429 return (error); 430 431 if (cmd == KIOSETBUFSIZE) { 432 /* 433 * Set the size of the coverage buffer. Should be called 434 * before enabling coverage collection for that thread. 435 */ 436 if (info->state != KCOV_STATE_OPEN) { 437 return (EBUSY); 438 } 439 error = kcov_alloc(info, *(u_int *)data); 440 if (error == 0) 441 info->state = KCOV_STATE_READY; 442 return (error); 443 } 444 445 mtx_lock_spin(&kcov_lock); 446 switch (cmd) { 447 case KIOENABLE: 448 if (info->state != KCOV_STATE_READY) { 449 error = EBUSY; 450 break; 451 } 452 if (td->td_kcov_info != NULL) { 453 error = EINVAL; 454 break; 455 } 456 mode = *(int *)data; 457 if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) { 458 error = EINVAL; 459 break; 460 } 461 462 /* Lets hope nobody opens this 2 billion times */ 463 KASSERT(active_count < INT_MAX, 464 ("%s: Open too many times", __func__)); 465 active_count++; 466 if (active_count == 1) { 467 cov_register_pc(&trace_pc); 468 cov_register_cmp(&trace_cmp); 469 } 470 471 KASSERT(info->thread == NULL, 472 ("Enabling kcov when already enabled")); 473 info->thread = td; 474 info->mode = mode; 475 /* 476 * Ensure the mode has been set before starting coverage 477 * tracing. 478 */ 479 atomic_store_rel_int(&info->state, KCOV_STATE_RUNNING); 480 td->td_kcov_info = info; 481 break; 482 case KIODISABLE: 483 /* Only the currently enabled thread may disable itself */ 484 if (info->state != KCOV_STATE_RUNNING || 485 info != td->td_kcov_info) { 486 error = EINVAL; 487 break; 488 } 489 KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); 490 active_count--; 491 if (active_count == 0) { 492 cov_unregister_pc(); 493 cov_unregister_cmp(); 494 } 495 496 td->td_kcov_info = NULL; 497 atomic_store_int(&info->state, KCOV_STATE_READY); 498 /* 499 * Ensure we have exited the READY state before clearing the 500 * rest of the info struct. 501 */ 502 atomic_thread_fence_rel(); 503 info->mode = -1; 504 info->thread = NULL; 505 break; 506 default: 507 error = EINVAL; 508 break; 509 } 510 mtx_unlock_spin(&kcov_lock); 511 512 return (error); 513 } 514 515 static void 516 kcov_thread_dtor(void *arg __unused, struct thread *td) 517 { 518 struct kcov_info *info; 519 520 info = td->td_kcov_info; 521 if (info == NULL) 522 return; 523 524 mtx_lock_spin(&kcov_lock); 525 KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); 526 active_count--; 527 if (active_count == 0) { 528 cov_unregister_pc(); 529 cov_unregister_cmp(); 530 } 531 td->td_kcov_info = NULL; 532 if (info->state != KCOV_STATE_DYING) { 533 /* 534 * The kcov file is still open. Mark it as unused and 535 * wait for it to be closed before cleaning up. 536 */ 537 atomic_store_int(&info->state, KCOV_STATE_READY); 538 atomic_thread_fence_seq_cst(); 539 /* This info struct is unused */ 540 info->thread = NULL; 541 mtx_unlock_spin(&kcov_lock); 542 return; 543 } 544 mtx_unlock_spin(&kcov_lock); 545 546 /* 547 * We can safely clean up the info struct as it is in the 548 * KCOV_STATE_DYING state where the info struct is associated with 549 * the current thread that's about to exit. 550 * 551 * The KCOV_STATE_DYING stops new threads from using it. 552 * It also stops the current thread from trying to use the info struct. 553 */ 554 kcov_free(info); 555 } 556 557 static void 558 kcov_init(const void *unused) 559 { 560 struct make_dev_args args; 561 struct cdev *dev; 562 563 mtx_init(&kcov_lock, "kcov lock", NULL, MTX_SPIN); 564 565 make_dev_args_init(&args); 566 args.mda_devsw = &kcov_cdevsw; 567 args.mda_uid = UID_ROOT; 568 args.mda_gid = GID_WHEEL; 569 args.mda_mode = 0600; 570 if (make_dev_s(&args, &dev, "kcov") != 0) { 571 printf("%s", "Failed to create kcov device"); 572 return; 573 } 574 575 EVENTHANDLER_REGISTER(thread_dtor, kcov_thread_dtor, NULL, 576 EVENTHANDLER_PRI_ANY); 577 } 578 579 SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL); 580