1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright 2022 Christos Margiolis <christos@FreeBSD.org> 5 * Copyright 2022 Mark Johnston <markj@FreeBSD.org> 6 */ 7 8 #include <sys/param.h> 9 #include <sys/bitset.h> 10 #include <sys/cred.h> 11 #include <sys/eventhandler.h> 12 #include <sys/kernel.h> 13 #include <sys/lock.h> 14 #include <sys/malloc.h> 15 #include <sys/proc.h> 16 #include <sys/queue.h> 17 #include <sys/sx.h> 18 19 #include <vm/vm.h> 20 #include <vm/vm_param.h> 21 #include <vm/pmap.h> 22 #include <vm/vm_map.h> 23 #include <vm/vm_kern.h> 24 #include <vm/vm_object.h> 25 26 #include <cddl/dev/dtrace/dtrace_cddl.h> 27 28 #include "kinst.h" 29 #include "kinst_isa.h" 30 31 #define KINST_TRAMPS_PER_CHUNK (KINST_TRAMPCHUNK_SIZE / KINST_TRAMP_SIZE) 32 33 struct trampchunk { 34 TAILQ_ENTRY(trampchunk) next; 35 uint8_t *addr; 36 /* 0 -> allocated, 1 -> free */ 37 BITSET_DEFINE(, KINST_TRAMPS_PER_CHUNK) free; 38 }; 39 40 static TAILQ_HEAD(, trampchunk) kinst_trampchunks = 41 TAILQ_HEAD_INITIALIZER(kinst_trampchunks); 42 static struct sx kinst_tramp_sx; 43 SX_SYSINIT(kinst_tramp_sx, &kinst_tramp_sx, "kinst tramp"); 44 static eventhandler_tag kinst_thread_ctor_handler; 45 static eventhandler_tag kinst_thread_dtor_handler; 46 47 /* 48 * Fill the trampolines with KINST_TRAMP_FILL_PATTERN so that the kernel will 49 * crash cleanly if things somehow go wrong. 50 */ 51 static void 52 kinst_trampoline_fill(uint8_t *addr, int size) 53 { 54 int i; 55 56 for (i = 0; i < size; i += KINST_TRAMP_FILL_SIZE) { 57 memcpy(&addr[i], KINST_TRAMP_FILL_PATTERN, 58 KINST_TRAMP_FILL_SIZE); 59 } 60 } 61 62 static struct trampchunk * 63 kinst_trampchunk_alloc(void) 64 { 65 struct trampchunk *chunk; 66 vm_offset_t trampaddr; 67 int error __diagused; 68 69 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 70 71 /* 72 * Allocate virtual memory for the trampoline chunk. The returned 73 * address is saved in "trampaddr". To simplify population of 74 * trampolines, we follow the amd64 kernel's code model and allocate 75 * them above KERNBASE, i.e., in the top 2GB of the kernel's virtual 76 * address space. Trampolines must be executable so max_prot must 77 * include VM_PROT_EXECUTE. 78 */ 79 trampaddr = KERNBASE; 80 error = vm_map_find(kernel_map, NULL, 0, &trampaddr, 81 KINST_TRAMPCHUNK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, 82 0); 83 if (error != KERN_SUCCESS) { 84 KINST_LOG("trampoline chunk allocation failed: %d", error); 85 return (NULL); 86 } 87 88 error = kmem_back(kernel_object, trampaddr, KINST_TRAMPCHUNK_SIZE, 89 M_WAITOK | M_EXEC); 90 KASSERT(error == KERN_SUCCESS, ("kmem_back failed: %d", error)); 91 92 kinst_trampoline_fill((uint8_t *)trampaddr, KINST_TRAMPCHUNK_SIZE); 93 94 /* Allocate a tracker for this chunk. */ 95 chunk = malloc(sizeof(*chunk), M_KINST, M_WAITOK); 96 chunk->addr = (void *)trampaddr; 97 BIT_FILL(KINST_TRAMPS_PER_CHUNK, &chunk->free); 98 99 TAILQ_INSERT_HEAD(&kinst_trampchunks, chunk, next); 100 101 return (chunk); 102 } 103 104 static void 105 kinst_trampchunk_free(struct trampchunk *chunk) 106 { 107 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 108 109 TAILQ_REMOVE(&kinst_trampchunks, chunk, next); 110 kmem_unback(kernel_object, (vm_offset_t)chunk->addr, 111 KINST_TRAMPCHUNK_SIZE); 112 (void)vm_map_remove(kernel_map, (vm_offset_t)chunk->addr, 113 (vm_offset_t)(chunk->addr + KINST_TRAMPCHUNK_SIZE)); 114 free(chunk, M_KINST); 115 } 116 117 static uint8_t * 118 kinst_trampoline_alloc_locked(int how) 119 { 120 struct trampchunk *chunk; 121 uint8_t *tramp; 122 int off; 123 124 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 125 126 TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { 127 /* All trampolines from this chunk are already allocated. */ 128 if ((off = BIT_FFS(KINST_TRAMPS_PER_CHUNK, &chunk->free)) == 0) 129 continue; 130 /* BIT_FFS() returns indices starting at 1 instead of 0. */ 131 off--; 132 break; 133 } 134 if (chunk == NULL) { 135 if ((how & M_NOWAIT) != 0) 136 return (NULL); 137 138 /* 139 * We didn't find any free trampoline in the current list, 140 * allocate a new one. If that fails the provider will no 141 * longer be reliable, so try to warn the user. 142 */ 143 if ((chunk = kinst_trampchunk_alloc()) == NULL) { 144 static bool once = true; 145 146 if (once) { 147 once = false; 148 KINST_LOG( 149 "kinst: failed to allocate trampoline, " 150 "probes may not fire"); 151 } 152 return (NULL); 153 } 154 off = 0; 155 } 156 BIT_CLR(KINST_TRAMPS_PER_CHUNK, off, &chunk->free); 157 tramp = chunk->addr + off * KINST_TRAMP_SIZE; 158 return (tramp); 159 } 160 161 uint8_t * 162 kinst_trampoline_alloc(int how) 163 { 164 uint8_t *tramp; 165 166 sx_xlock(&kinst_tramp_sx); 167 tramp = kinst_trampoline_alloc_locked(how); 168 sx_xunlock(&kinst_tramp_sx); 169 return (tramp); 170 } 171 172 static void 173 kinst_trampoline_dealloc_locked(uint8_t *tramp, bool freechunks) 174 { 175 struct trampchunk *chunk; 176 int off; 177 178 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 179 180 if (tramp == NULL) 181 return; 182 183 TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { 184 for (off = 0; off < KINST_TRAMPS_PER_CHUNK; off++) { 185 if (chunk->addr + off * KINST_TRAMP_SIZE == tramp) { 186 kinst_trampoline_fill(tramp, KINST_TRAMP_SIZE); 187 BIT_SET(KINST_TRAMPS_PER_CHUNK, off, 188 &chunk->free); 189 if (freechunks && 190 BIT_ISFULLSET(KINST_TRAMPS_PER_CHUNK, 191 &chunk->free)) 192 kinst_trampchunk_free(chunk); 193 return; 194 } 195 } 196 } 197 panic("%s: did not find trampoline chunk for %p", __func__, tramp); 198 } 199 200 void 201 kinst_trampoline_dealloc(uint8_t *tramp) 202 { 203 sx_xlock(&kinst_tramp_sx); 204 kinst_trampoline_dealloc_locked(tramp, true); 205 sx_xunlock(&kinst_tramp_sx); 206 } 207 208 static void 209 kinst_thread_ctor(void *arg __unused, struct thread *td) 210 { 211 td->t_kinst = kinst_trampoline_alloc(M_WAITOK); 212 } 213 214 static void 215 kinst_thread_dtor(void *arg __unused, struct thread *td) 216 { 217 void *tramp; 218 219 tramp = td->t_kinst; 220 td->t_kinst = NULL; 221 222 /* 223 * This assumes that the thread_dtor event permits sleeping, which 224 * appears to be true for the time being. 225 */ 226 kinst_trampoline_dealloc(tramp); 227 } 228 229 int 230 kinst_trampoline_init(void) 231 { 232 struct proc *p; 233 struct thread *td; 234 void *tramp; 235 int error; 236 237 kinst_thread_ctor_handler = EVENTHANDLER_REGISTER(thread_ctor, 238 kinst_thread_ctor, NULL, EVENTHANDLER_PRI_ANY); 239 kinst_thread_dtor_handler = EVENTHANDLER_REGISTER(thread_dtor, 240 kinst_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 241 242 error = 0; 243 tramp = NULL; 244 245 sx_slock(&allproc_lock); 246 sx_xlock(&kinst_tramp_sx); 247 FOREACH_PROC_IN_SYSTEM(p) { 248 retry: 249 PROC_LOCK(p); 250 FOREACH_THREAD_IN_PROC(p, td) { 251 if (td->t_kinst != NULL) 252 continue; 253 if (tramp == NULL) { 254 /* 255 * Try to allocate a trampoline without dropping 256 * the process lock. If all chunks are fully 257 * utilized, we must release the lock and try 258 * again. 259 */ 260 tramp = kinst_trampoline_alloc_locked(M_NOWAIT); 261 if (tramp == NULL) { 262 PROC_UNLOCK(p); 263 tramp = kinst_trampoline_alloc_locked( 264 M_WAITOK); 265 if (tramp == NULL) { 266 /* 267 * Let the unload handler clean 268 * up. 269 */ 270 error = ENOMEM; 271 goto out; 272 } else 273 goto retry; 274 } 275 } 276 td->t_kinst = tramp; 277 tramp = NULL; 278 } 279 PROC_UNLOCK(p); 280 } 281 out: 282 sx_xunlock(&kinst_tramp_sx); 283 sx_sunlock(&allproc_lock); 284 return (error); 285 } 286 287 int 288 kinst_trampoline_deinit(void) 289 { 290 struct trampchunk *chunk, *tmp; 291 struct proc *p; 292 struct thread *td; 293 294 EVENTHANDLER_DEREGISTER(thread_ctor, kinst_thread_ctor_handler); 295 EVENTHANDLER_DEREGISTER(thread_dtor, kinst_thread_dtor_handler); 296 297 sx_slock(&allproc_lock); 298 sx_xlock(&kinst_tramp_sx); 299 FOREACH_PROC_IN_SYSTEM(p) { 300 PROC_LOCK(p); 301 FOREACH_THREAD_IN_PROC(p, td) { 302 kinst_trampoline_dealloc_locked(td->t_kinst, false); 303 td->t_kinst = NULL; 304 } 305 PROC_UNLOCK(p); 306 } 307 sx_sunlock(&allproc_lock); 308 TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp) 309 kinst_trampchunk_free(chunk); 310 sx_xunlock(&kinst_tramp_sx); 311 312 return (0); 313 } 314