1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright 2022 Christos Margiolis <christos@FreeBSD.org> 5 * Copyright 2022 Mark Johnston <markj@FreeBSD.org> 6 */ 7 8 #include <sys/param.h> 9 #include <sys/bitset.h> 10 #include <sys/cred.h> 11 #include <sys/eventhandler.h> 12 #include <sys/kernel.h> 13 #include <sys/lock.h> 14 #include <sys/malloc.h> 15 #include <sys/proc.h> 16 #include <sys/queue.h> 17 #include <sys/sx.h> 18 19 #include <vm/vm.h> 20 #include <vm/vm_param.h> 21 #include <vm/pmap.h> 22 #include <vm/vm_map.h> 23 #include <vm/vm_kern.h> 24 #include <vm/vm_object.h> 25 26 #include <cddl/dev/dtrace/dtrace_cddl.h> 27 28 #include "kinst.h" 29 #include "kinst_isa.h" 30 31 /* 32 * We can have 4KB/32B = 128 trampolines per chunk. 33 */ 34 #define KINST_TRAMPS_PER_CHUNK (KINST_TRAMPCHUNK_SIZE / KINST_TRAMP_SIZE) 35 36 struct trampchunk { 37 TAILQ_ENTRY(trampchunk) next; 38 uint8_t *addr; 39 /* 0 -> allocated, 1 -> free */ 40 BITSET_DEFINE(, KINST_TRAMPS_PER_CHUNK) free; 41 }; 42 43 static TAILQ_HEAD(, trampchunk) kinst_trampchunks = 44 TAILQ_HEAD_INITIALIZER(kinst_trampchunks); 45 static struct sx kinst_tramp_sx; 46 SX_SYSINIT(kinst_tramp_sx, &kinst_tramp_sx, "kinst tramp"); 47 static eventhandler_tag kinst_thread_ctor_handler; 48 static eventhandler_tag kinst_thread_dtor_handler; 49 50 static struct trampchunk * 51 kinst_trampchunk_alloc(void) 52 { 53 struct trampchunk *chunk; 54 vm_offset_t trampaddr; 55 int error __diagused; 56 57 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 58 59 /* 60 * Allocate virtual memory for the trampoline chunk. The returned 61 * address is saved in "trampaddr". To simplify population of 62 * trampolines, we follow the amd64 kernel's code model and allocate 63 * them above KERNBASE, i.e., in the top 2GB of the kernel's virtual 64 * address space. Trampolines must be executable so max_prot must 65 * include VM_PROT_EXECUTE. 66 */ 67 trampaddr = KERNBASE; 68 error = vm_map_find(kernel_map, NULL, 0, &trampaddr, 69 KINST_TRAMPCHUNK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, 70 0); 71 if (error != KERN_SUCCESS) { 72 KINST_LOG("trampoline chunk allocation failed: %d", error); 73 return (NULL); 74 } 75 76 error = kmem_back(kernel_object, trampaddr, KINST_TRAMPCHUNK_SIZE, 77 M_WAITOK | M_EXEC); 78 KASSERT(error == KERN_SUCCESS, ("kmem_back failed: %d", error)); 79 80 KINST_TRAMP_INIT((void *)trampaddr, KINST_TRAMPCHUNK_SIZE); 81 82 /* Allocate a tracker for this chunk. */ 83 chunk = malloc(sizeof(*chunk), M_KINST, M_WAITOK); 84 chunk->addr = (void *)trampaddr; 85 BIT_FILL(KINST_TRAMPS_PER_CHUNK, &chunk->free); 86 87 TAILQ_INSERT_HEAD(&kinst_trampchunks, chunk, next); 88 89 return (chunk); 90 } 91 92 static void 93 kinst_trampchunk_free(struct trampchunk *chunk) 94 { 95 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 96 97 TAILQ_REMOVE(&kinst_trampchunks, chunk, next); 98 kmem_unback(kernel_object, (vm_offset_t)chunk->addr, 99 KINST_TRAMPCHUNK_SIZE); 100 (void)vm_map_remove(kernel_map, (vm_offset_t)chunk->addr, 101 (vm_offset_t)(chunk->addr + KINST_TRAMPCHUNK_SIZE)); 102 free(chunk, M_KINST); 103 } 104 105 static uint8_t * 106 kinst_trampoline_alloc_locked(int how) 107 { 108 struct trampchunk *chunk; 109 uint8_t *tramp; 110 int off; 111 112 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 113 114 TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { 115 /* All trampolines from this chunk are already allocated. */ 116 if ((off = BIT_FFS(KINST_TRAMPS_PER_CHUNK, &chunk->free)) == 0) 117 continue; 118 /* BIT_FFS() returns indices starting at 1 instead of 0. */ 119 off--; 120 break; 121 } 122 if (chunk == NULL) { 123 if ((how & M_NOWAIT) != 0) 124 return (NULL); 125 126 /* 127 * We didn't find any free trampoline in the current list, 128 * allocate a new one. If that fails the provider will no 129 * longer be reliable, so try to warn the user. 130 */ 131 if ((chunk = kinst_trampchunk_alloc()) == NULL) { 132 static bool once = true; 133 134 if (once) { 135 once = false; 136 KINST_LOG( 137 "kinst: failed to allocate trampoline, " 138 "probes may not fire"); 139 } 140 return (NULL); 141 } 142 off = 0; 143 } 144 BIT_CLR(KINST_TRAMPS_PER_CHUNK, off, &chunk->free); 145 tramp = chunk->addr + off * KINST_TRAMP_SIZE; 146 return (tramp); 147 } 148 149 uint8_t * 150 kinst_trampoline_alloc(int how) 151 { 152 uint8_t *tramp; 153 154 sx_xlock(&kinst_tramp_sx); 155 tramp = kinst_trampoline_alloc_locked(how); 156 sx_xunlock(&kinst_tramp_sx); 157 return (tramp); 158 } 159 160 static void 161 kinst_trampoline_dealloc_locked(uint8_t *tramp, bool freechunks) 162 { 163 struct trampchunk *chunk; 164 int off; 165 166 sx_assert(&kinst_tramp_sx, SX_XLOCKED); 167 168 if (tramp == NULL) 169 return; 170 171 TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { 172 for (off = 0; off < KINST_TRAMPS_PER_CHUNK; off++) { 173 if (chunk->addr + off * KINST_TRAMP_SIZE == tramp) { 174 KINST_TRAMP_INIT(tramp, KINST_TRAMP_SIZE); 175 BIT_SET(KINST_TRAMPS_PER_CHUNK, off, 176 &chunk->free); 177 if (freechunks && 178 BIT_ISFULLSET(KINST_TRAMPS_PER_CHUNK, 179 &chunk->free)) 180 kinst_trampchunk_free(chunk); 181 return; 182 } 183 } 184 } 185 panic("%s: did not find trampoline chunk for %p", __func__, tramp); 186 } 187 188 void 189 kinst_trampoline_dealloc(uint8_t *tramp) 190 { 191 sx_xlock(&kinst_tramp_sx); 192 kinst_trampoline_dealloc_locked(tramp, true); 193 sx_xunlock(&kinst_tramp_sx); 194 } 195 196 static void 197 kinst_thread_ctor(void *arg __unused, struct thread *td) 198 { 199 td->t_kinst = kinst_trampoline_alloc(M_WAITOK); 200 } 201 202 static void 203 kinst_thread_dtor(void *arg __unused, struct thread *td) 204 { 205 void *tramp; 206 207 tramp = td->t_kinst; 208 td->t_kinst = NULL; 209 210 /* 211 * This assumes that the thread_dtor event permits sleeping, which 212 * appears to be true for the time being. 213 */ 214 kinst_trampoline_dealloc(tramp); 215 } 216 217 int 218 kinst_trampoline_init(void) 219 { 220 struct proc *p; 221 struct thread *td; 222 void *tramp; 223 int error; 224 225 kinst_thread_ctor_handler = EVENTHANDLER_REGISTER(thread_ctor, 226 kinst_thread_ctor, NULL, EVENTHANDLER_PRI_ANY); 227 kinst_thread_dtor_handler = EVENTHANDLER_REGISTER(thread_dtor, 228 kinst_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 229 230 error = 0; 231 tramp = NULL; 232 233 sx_slock(&allproc_lock); 234 sx_xlock(&kinst_tramp_sx); 235 FOREACH_PROC_IN_SYSTEM(p) { 236 retry: 237 PROC_LOCK(p); 238 FOREACH_THREAD_IN_PROC(p, td) { 239 if (td->t_kinst != NULL) 240 continue; 241 if (tramp == NULL) { 242 /* 243 * Try to allocate a trampoline without dropping 244 * the process lock. If all chunks are fully 245 * utilized, we must release the lock and try 246 * again. 247 */ 248 tramp = kinst_trampoline_alloc_locked(M_NOWAIT); 249 if (tramp == NULL) { 250 PROC_UNLOCK(p); 251 tramp = kinst_trampoline_alloc_locked( 252 M_WAITOK); 253 if (tramp == NULL) { 254 /* 255 * Let the unload handler clean 256 * up. 257 */ 258 error = ENOMEM; 259 goto out; 260 } else 261 goto retry; 262 } 263 } 264 td->t_kinst = tramp; 265 tramp = NULL; 266 } 267 PROC_UNLOCK(p); 268 } 269 out: 270 sx_xunlock(&kinst_tramp_sx); 271 sx_sunlock(&allproc_lock); 272 return (error); 273 } 274 275 int 276 kinst_trampoline_deinit(void) 277 { 278 struct trampchunk *chunk, *tmp; 279 struct proc *p; 280 struct thread *td; 281 282 EVENTHANDLER_DEREGISTER(thread_ctor, kinst_thread_ctor_handler); 283 EVENTHANDLER_DEREGISTER(thread_dtor, kinst_thread_dtor_handler); 284 285 sx_slock(&allproc_lock); 286 sx_xlock(&kinst_tramp_sx); 287 FOREACH_PROC_IN_SYSTEM(p) { 288 PROC_LOCK(p); 289 FOREACH_THREAD_IN_PROC(p, td) { 290 kinst_trampoline_dealloc_locked(td->t_kinst, false); 291 td->t_kinst = NULL; 292 } 293 PROC_UNLOCK(p); 294 } 295 sx_sunlock(&allproc_lock); 296 TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp) 297 kinst_trampchunk_free(chunk); 298 sx_xunlock(&kinst_tramp_sx); 299 300 return (0); 301 } 302