1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved.
5 * Copyright (C) 2018, 2019 Andrew Turner
6 *
7 * This software was developed by Mitchell Horne under sponsorship of
8 * the FreeBSD Foundation.
9 *
10 * This software was developed by SRI International and the University of
11 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
12 * ("CTSRD"), as part of the DARPA CRASH research programme.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
16 * are met:
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 */
35
36 /* Interceptors are required for KMSAN. */
37 #if defined(KASAN) || defined(KCSAN)
38 #define SAN_RUNTIME
39 #endif
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/conf.h>
44 #include <sys/eventhandler.h>
45 #include <sys/kcov.h>
46 #include <sys/kernel.h>
47 #include <sys/limits.h>
48 #include <sys/lock.h>
49 #include <sys/malloc.h>
50 #include <sys/mman.h>
51 #include <sys/mutex.h>
52 #include <sys/proc.h>
53 #include <sys/rwlock.h>
54 #include <sys/sysctl.h>
55
56 #include <vm/vm.h>
57 #include <vm/pmap.h>
58 #include <vm/vm_extern.h>
59 #include <vm/vm_object.h>
60 #include <vm/vm_page.h>
61 #include <vm/vm_pager.h>
62 #include <vm/vm_param.h>
63 #include <vm/vm_radix.h>
64
65 MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type");
66
67 #define KCOV_ELEMENT_SIZE sizeof(uint64_t)
68
69 /*
70 * To know what the code can safely perform at any point in time we use a
71 * state machine. In the normal case the state transitions are:
72 *
73 * OPEN -> READY -> RUNNING -> DYING
74 * | | ^ | ^ ^
75 * | | +--------+ | |
76 * | +-------------------+ |
77 * +-----------------------------+
78 *
79 * The states are:
80 * OPEN: The kcov fd has been opened, but no buffer is available to store
81 * coverage data.
82 * READY: The buffer to store coverage data has been allocated. Userspace
83 * can set this by using ioctl(fd, KIOSETBUFSIZE, entries);. When
84 * this has been set the buffer can be written to by the kernel,
85 * and mmaped by userspace.
86 * RUNNING: The coverage probes are able to store coverage data in the buffer.
87 * This is entered with ioctl(fd, KIOENABLE, mode);. The READY state
88 * can be exited by ioctl(fd, KIODISABLE); or exiting the thread to
89 * return to the READY state to allow tracing to be reused, or by
90 * closing the kcov fd to enter the DYING state.
91 * DYING: The fd has been closed. All states can enter into this state when
92 * userspace closes the kcov fd.
93 *
94 * We need to be careful when moving into and out of the RUNNING state. As
95 * an interrupt may happen while this is happening the ordering of memory
96 * operations is important so struct kcov_info is valid for the tracing
97 * functions.
98 *
99 * When moving into the RUNNING state prior stores to struct kcov_info need
100 * to be observed before the state is set. This allows for interrupts that
101 * may call into one of the coverage functions to fire at any point while
102 * being enabled and see a consistent struct kcov_info.
103 *
104 * When moving out of the RUNNING state any later stores to struct kcov_info
105 * need to be observed after the state is set. As with entering this is to
106 * present a consistent struct kcov_info to interrupts.
107 */
108 typedef enum {
109 KCOV_STATE_INVALID,
110 KCOV_STATE_OPEN, /* The device is open, but with no buffer */
111 KCOV_STATE_READY, /* The buffer has been allocated */
112 KCOV_STATE_RUNNING, /* Recording trace data */
113 KCOV_STATE_DYING, /* The fd was closed */
114 } kcov_state_t;
115
116 /*
117 * (l) Set while holding the kcov_lock mutex and not in the RUNNING state.
118 * (o) Only set once while in the OPEN state. Cleaned up while in the DYING
119 * state, and with no thread associated with the struct kcov_info.
120 * (s) Set atomically to enter or exit the RUNNING state, non-atomically
121 * otherwise. See above for a description of the other constraints while
122 * moving into or out of the RUNNING state.
123 */
124 struct kcov_info {
125 struct thread *thread; /* (l) */
126 vm_object_t bufobj; /* (o) */
127 vm_offset_t kvaddr; /* (o) */
128 size_t entries; /* (o) */
129 size_t bufsize; /* (o) */
130 kcov_state_t state; /* (s) */
131 int mode; /* (l) */
132 };
133
134 /* Prototypes */
135 static d_open_t kcov_open;
136 static d_close_t kcov_close;
137 static d_mmap_single_t kcov_mmap_single;
138 static d_ioctl_t kcov_ioctl;
139
140 static int kcov_alloc(struct kcov_info *info, size_t entries);
141 static void kcov_free(struct kcov_info *info);
142 static void kcov_init(const void *unused);
143
144 static struct cdevsw kcov_cdevsw = {
145 .d_version = D_VERSION,
146 .d_open = kcov_open,
147 .d_close = kcov_close,
148 .d_mmap_single = kcov_mmap_single,
149 .d_ioctl = kcov_ioctl,
150 .d_name = "kcov",
151 };
152
153 SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
154 "Kernel coverage");
155
156 static u_int kcov_max_entries = KCOV_MAXENTRIES;
157 SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW,
158 &kcov_max_entries, 0,
159 "Maximum number of entries in the kcov buffer");
160
161 static struct mtx kcov_lock;
162 static int active_count;
163
164 static struct kcov_info * __nosanitizeaddress __nosanitizememory
get_kinfo(struct thread * td)165 get_kinfo(struct thread *td)
166 {
167 struct kcov_info *info;
168
169 /* We might have a NULL thread when releasing the secondary CPUs */
170 if (td == NULL)
171 return (NULL);
172
173 /*
174 * We are in an interrupt, stop tracing as it is not explicitly
175 * part of a syscall.
176 */
177 if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL)
178 return (NULL);
179
180 /*
181 * If info is NULL or the state is not running we are not tracing.
182 */
183 info = td->td_kcov_info;
184 if (info == NULL ||
185 atomic_load_acq_int(&info->state) != KCOV_STATE_RUNNING)
186 return (NULL);
187
188 return (info);
189 }
190
191 static void __nosanitizeaddress __nosanitizememory
trace_pc(uintptr_t ret)192 trace_pc(uintptr_t ret)
193 {
194 struct thread *td;
195 struct kcov_info *info;
196 uint64_t *buf, index;
197
198 td = curthread;
199 info = get_kinfo(td);
200 if (info == NULL)
201 return;
202
203 /*
204 * Check we are in the PC-trace mode.
205 */
206 if (info->mode != KCOV_MODE_TRACE_PC)
207 return;
208
209 KASSERT(info->kvaddr != 0, ("%s: NULL buf while running", __func__));
210
211 buf = (uint64_t *)info->kvaddr;
212
213 /* The first entry of the buffer holds the index */
214 index = buf[0];
215 if (index + 2 > info->entries)
216 return;
217
218 buf[index + 1] = ret;
219 buf[0] = index + 1;
220 }
221
222 static bool __nosanitizeaddress __nosanitizememory
trace_cmp(uint64_t type,uint64_t arg1,uint64_t arg2,uint64_t ret)223 trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret)
224 {
225 struct thread *td;
226 struct kcov_info *info;
227 uint64_t *buf, index;
228
229 td = curthread;
230 info = get_kinfo(td);
231 if (info == NULL)
232 return (false);
233
234 /*
235 * Check we are in the comparison-trace mode.
236 */
237 if (info->mode != KCOV_MODE_TRACE_CMP)
238 return (false);
239
240 KASSERT(info->kvaddr != 0, ("%s: NULL buf while running", __func__));
241
242 buf = (uint64_t *)info->kvaddr;
243
244 /* The first entry of the buffer holds the index */
245 index = buf[0];
246
247 /* Check we have space to store all elements */
248 if (index * 4 + 4 + 1 > info->entries)
249 return (false);
250
251 while (1) {
252 buf[index * 4 + 1] = type;
253 buf[index * 4 + 2] = arg1;
254 buf[index * 4 + 3] = arg2;
255 buf[index * 4 + 4] = ret;
256
257 if (atomic_cmpset_64(&buf[0], index, index + 1))
258 break;
259 buf[0] = index;
260 }
261
262 return (true);
263 }
264
265 /*
266 * The fd is being closed, cleanup everything we can.
267 */
268 static void
kcov_mmap_cleanup(void * arg)269 kcov_mmap_cleanup(void *arg)
270 {
271 struct kcov_info *info = arg;
272 struct thread *thread;
273
274 mtx_lock_spin(&kcov_lock);
275 /*
276 * Move to KCOV_STATE_DYING to stop adding new entries.
277 *
278 * If the thread is running we need to wait until thread exit to
279 * clean up as it may currently be adding a new entry. If this is
280 * the case being in KCOV_STATE_DYING will signal that the buffer
281 * needs to be cleaned up.
282 */
283 atomic_store_int(&info->state, KCOV_STATE_DYING);
284 atomic_thread_fence_seq_cst();
285 thread = info->thread;
286 mtx_unlock_spin(&kcov_lock);
287
288 if (thread != NULL)
289 return;
290
291 /*
292 * We can safely clean up the info struct as it is in the
293 * KCOV_STATE_DYING state with no thread associated.
294 *
295 * The KCOV_STATE_DYING stops new threads from using it.
296 * The lack of a thread means nothing is currently using the buffers.
297 */
298 kcov_free(info);
299 }
300
301 static int
kcov_open(struct cdev * dev,int oflags,int devtype,struct thread * td)302 kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
303 {
304 struct kcov_info *info;
305 int error;
306
307 info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK);
308 info->state = KCOV_STATE_OPEN;
309 info->thread = NULL;
310 info->mode = -1;
311
312 if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0)
313 kcov_mmap_cleanup(info);
314
315 return (error);
316 }
317
318 static int
kcov_close(struct cdev * dev,int fflag,int devtype,struct thread * td)319 kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
320 {
321 struct kcov_info *info;
322 int error;
323
324 if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
325 return (error);
326
327 KASSERT(info != NULL, ("kcov_close with no kcov_info structure"));
328
329 /* Trying to close, but haven't disabled */
330 if (info->state == KCOV_STATE_RUNNING)
331 return (EBUSY);
332
333 return (0);
334 }
335
336 static int
kcov_mmap_single(struct cdev * dev,vm_ooffset_t * offset,vm_size_t size,struct vm_object ** object,int nprot)337 kcov_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size,
338 struct vm_object **object, int nprot)
339 {
340 struct kcov_info *info;
341 int error;
342
343 if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) !=
344 (PROT_READ | PROT_WRITE))
345 return (EINVAL);
346
347 if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
348 return (error);
349
350 if (info->kvaddr == 0 || size / KCOV_ELEMENT_SIZE != info->entries)
351 return (EINVAL);
352
353 vm_object_reference(info->bufobj);
354 *offset = 0;
355 *object = info->bufobj;
356 return (0);
357 }
358
359 static int
kcov_alloc(struct kcov_info * info,size_t entries)360 kcov_alloc(struct kcov_info *info, size_t entries)
361 {
362 size_t n, pages;
363 vm_page_t m;
364
365 KASSERT(info->kvaddr == 0, ("kcov_alloc: Already have a buffer"));
366 KASSERT(info->state == KCOV_STATE_OPEN,
367 ("kcov_alloc: Not in open state (%x)", info->state));
368
369 if (entries < 2 || entries > kcov_max_entries)
370 return (EINVAL);
371
372 /* Align to page size so mmap can't access other kernel memory */
373 info->bufsize = roundup2(entries * KCOV_ELEMENT_SIZE, PAGE_SIZE);
374 pages = info->bufsize / PAGE_SIZE;
375
376 if ((info->kvaddr = kva_alloc(info->bufsize)) == 0)
377 return (ENOMEM);
378
379 info->bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->bufsize,
380 PROT_READ | PROT_WRITE, 0, curthread->td_ucred);
381
382 VM_OBJECT_WLOCK(info->bufobj);
383 for (n = 0; n < pages; n++) {
384 m = vm_page_grab(info->bufobj, n,
385 VM_ALLOC_ZERO | VM_ALLOC_WIRED);
386 vm_page_valid(m);
387 vm_page_xunbusy(m);
388 pmap_qenter(info->kvaddr + n * PAGE_SIZE, &m, 1);
389 }
390 VM_OBJECT_WUNLOCK(info->bufobj);
391
392 info->entries = entries;
393
394 return (0);
395 }
396
397 static void
kcov_free(struct kcov_info * info)398 kcov_free(struct kcov_info *info)
399 {
400 struct pctrie_iter pages;
401 vm_page_t m;
402
403 if (info->kvaddr != 0) {
404 pmap_qremove(info->kvaddr, info->bufsize / PAGE_SIZE);
405 kva_free(info->kvaddr, info->bufsize);
406 }
407 if (info->bufobj != NULL) {
408 vm_page_iter_limit_init(&pages, info->bufobj,
409 info->bufsize / PAGE_SIZE);
410 VM_OBJECT_WLOCK(info->bufobj);
411 VM_RADIX_FORALL(m, &pages)
412 vm_page_unwire_noq(m);
413 VM_OBJECT_WUNLOCK(info->bufobj);
414 vm_object_deallocate(info->bufobj);
415 }
416 free(info, M_KCOV_INFO);
417 }
418
419 static int
kcov_ioctl(struct cdev * dev,u_long cmd,caddr_t data,int fflag __unused,struct thread * td)420 kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused,
421 struct thread *td)
422 {
423 struct kcov_info *info;
424 int mode, error;
425
426 if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
427 return (error);
428
429 if (cmd == KIOSETBUFSIZE) {
430 /*
431 * Set the size of the coverage buffer. Should be called
432 * before enabling coverage collection for that thread.
433 */
434 if (info->state != KCOV_STATE_OPEN) {
435 return (EBUSY);
436 }
437 error = kcov_alloc(info, *(u_int *)data);
438 if (error == 0)
439 info->state = KCOV_STATE_READY;
440 return (error);
441 }
442
443 mtx_lock_spin(&kcov_lock);
444 switch (cmd) {
445 case KIOENABLE:
446 if (info->state != KCOV_STATE_READY) {
447 error = EBUSY;
448 break;
449 }
450 if (td->td_kcov_info != NULL) {
451 error = EINVAL;
452 break;
453 }
454 mode = *(int *)data;
455 if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) {
456 error = EINVAL;
457 break;
458 }
459
460 /* Lets hope nobody opens this 2 billion times */
461 KASSERT(active_count < INT_MAX,
462 ("%s: Open too many times", __func__));
463 active_count++;
464 if (active_count == 1) {
465 cov_register_pc(&trace_pc);
466 cov_register_cmp(&trace_cmp);
467 }
468
469 KASSERT(info->thread == NULL,
470 ("Enabling kcov when already enabled"));
471 info->thread = td;
472 info->mode = mode;
473 /*
474 * Ensure the mode has been set before starting coverage
475 * tracing.
476 */
477 atomic_store_rel_int(&info->state, KCOV_STATE_RUNNING);
478 td->td_kcov_info = info;
479 break;
480 case KIODISABLE:
481 /* Only the currently enabled thread may disable itself */
482 if (info->state != KCOV_STATE_RUNNING ||
483 info != td->td_kcov_info) {
484 error = EINVAL;
485 break;
486 }
487 KASSERT(active_count > 0, ("%s: Open count is zero", __func__));
488 active_count--;
489 if (active_count == 0) {
490 cov_unregister_pc();
491 cov_unregister_cmp();
492 }
493
494 td->td_kcov_info = NULL;
495 atomic_store_int(&info->state, KCOV_STATE_READY);
496 /*
497 * Ensure we have exited the READY state before clearing the
498 * rest of the info struct.
499 */
500 atomic_thread_fence_rel();
501 info->mode = -1;
502 info->thread = NULL;
503 break;
504 default:
505 error = EINVAL;
506 break;
507 }
508 mtx_unlock_spin(&kcov_lock);
509
510 return (error);
511 }
512
513 static void
kcov_thread_dtor(void * arg __unused,struct thread * td)514 kcov_thread_dtor(void *arg __unused, struct thread *td)
515 {
516 struct kcov_info *info;
517
518 info = td->td_kcov_info;
519 if (info == NULL)
520 return;
521
522 mtx_lock_spin(&kcov_lock);
523 KASSERT(active_count > 0, ("%s: Open count is zero", __func__));
524 active_count--;
525 if (active_count == 0) {
526 cov_unregister_pc();
527 cov_unregister_cmp();
528 }
529 td->td_kcov_info = NULL;
530 if (info->state != KCOV_STATE_DYING) {
531 /*
532 * The kcov file is still open. Mark it as unused and
533 * wait for it to be closed before cleaning up.
534 */
535 atomic_store_int(&info->state, KCOV_STATE_READY);
536 atomic_thread_fence_seq_cst();
537 /* This info struct is unused */
538 info->thread = NULL;
539 mtx_unlock_spin(&kcov_lock);
540 return;
541 }
542 mtx_unlock_spin(&kcov_lock);
543
544 /*
545 * We can safely clean up the info struct as it is in the
546 * KCOV_STATE_DYING state where the info struct is associated with
547 * the current thread that's about to exit.
548 *
549 * The KCOV_STATE_DYING stops new threads from using it.
550 * It also stops the current thread from trying to use the info struct.
551 */
552 kcov_free(info);
553 }
554
555 static void
kcov_init(const void * unused)556 kcov_init(const void *unused)
557 {
558 struct make_dev_args args;
559 struct cdev *dev;
560
561 mtx_init(&kcov_lock, "kcov lock", NULL, MTX_SPIN);
562
563 make_dev_args_init(&args);
564 args.mda_devsw = &kcov_cdevsw;
565 args.mda_uid = UID_ROOT;
566 args.mda_gid = GID_WHEEL;
567 args.mda_mode = 0600;
568 if (make_dev_s(&args, &dev, "kcov") != 0) {
569 printf("%s", "Failed to create kcov device");
570 return;
571 }
572
573 EVENTHANDLER_REGISTER(thread_dtor, kcov_thread_dtor, NULL,
574 EVENTHANDLER_PRI_ANY);
575 }
576
577 SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL);
578