1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> 5 * Copyright (c) 2023 The FreeBSD Foundation 6 * 7 * Portions of this software were developed by Christos Margiolis 8 * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 9 */ 10 11 #include <sys/param.h> 12 #include <sys/systm.h> 13 #include <sys/conf.h> 14 #include <sys/kernel.h> 15 #include <sys/linker.h> 16 #include <sys/module.h> 17 18 #include <sys/dtrace.h> 19 20 #include "kinst.h" 21 22 MALLOC_DEFINE(M_KINST, "kinst", "Kernel Instruction Tracing"); 23 24 static d_open_t kinst_open; 25 static d_close_t kinst_close; 26 static d_ioctl_t kinst_ioctl; 27 28 static void kinst_provide_module(void *, modctl_t *); 29 static void kinst_getargdesc(void *, dtrace_id_t, void *, 30 dtrace_argdesc_t *); 31 static void kinst_destroy(void *, dtrace_id_t, void *); 32 static void kinst_enable(void *, dtrace_id_t, void *); 33 static void kinst_disable(void *, dtrace_id_t, void *); 34 static int kinst_load(void *); 35 static int kinst_unload(void *); 36 static int kinst_modevent(module_t, int, void *); 37 38 static dtrace_pattr_t kinst_attr = { 39 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 40 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 41 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 42 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 43 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 44 }; 45 46 static const dtrace_pops_t kinst_pops = { 47 .dtps_provide = NULL, 48 .dtps_provide_module = kinst_provide_module, 49 .dtps_enable = kinst_enable, 50 .dtps_disable = kinst_disable, 51 .dtps_suspend = NULL, 52 .dtps_resume = NULL, 53 .dtps_getargdesc = kinst_getargdesc, 54 .dtps_getargval = NULL, 55 .dtps_usermode = NULL, 56 .dtps_destroy = kinst_destroy 57 }; 58 59 static struct cdevsw kinst_cdevsw = { 60 .d_name = "kinst", 61 .d_version = D_VERSION, 62 .d_flags = D_TRACKCLOSE, 63 .d_open = kinst_open, 64 .d_close = kinst_close, 65 .d_ioctl = kinst_ioctl, 66 }; 67 68 static dtrace_provider_id_t kinst_id; 69 struct kinst_probe_list *kinst_probetab; 70 static struct cdev *kinst_cdev; 71 72 /* 73 * Tracing memcpy() will crash the kernel when kinst tries to trace an instance 74 * of the memcpy() calls in kinst_invop(). To fix this, we can use 75 * kinst_memcpy() in those cases, with its arguments marked as 'volatile' to 76 * "outsmart" the compiler and avoid having it replaced by a regular memcpy(). 77 */ 78 volatile void * 79 kinst_memcpy(volatile void *dst, volatile const void *src, size_t len) 80 { 81 volatile const unsigned char *src0; 82 volatile unsigned char *dst0; 83 84 src0 = src; 85 dst0 = dst; 86 87 while (len--) 88 *dst0++ = *src0++; 89 90 return (dst); 91 } 92 93 bool 94 kinst_excluded(const char *name) 95 { 96 if (kinst_md_excluded(name)) 97 return (true); 98 99 /* 100 * cpu_switch() can cause a crash if it modifies the value of curthread 101 * while in probe context. 102 */ 103 if (strcmp(name, "cpu_switch") == 0) 104 return (true); 105 106 /* 107 * Anything beginning with "dtrace_" may be called from probe context 108 * unless it explicitly indicates that it won't be called from probe 109 * context by using the prefix "dtrace_safe_". 110 */ 111 if (strncmp(name, "dtrace_", strlen("dtrace_")) == 0 && 112 strncmp(name, "dtrace_safe_", strlen("dtrace_safe_")) != 0) 113 return (true); 114 115 /* 116 * Omit instrumentation of functions that are probably in DDB. It 117 * makes it too hard to debug broken kinst. 118 * 119 * NB: kdb_enter() can be excluded, but its call to printf() can't be. 120 * This is generally OK since we're not yet in debugging context. 121 */ 122 if (strncmp(name, "db_", strlen("db_")) == 0 || 123 strncmp(name, "kdb_", strlen("kdb_")) == 0) 124 return (true); 125 126 /* 127 * Lock owner methods may be called from probe context. 128 */ 129 if (strcmp(name, "owner_mtx") == 0 || 130 strcmp(name, "owner_rm") == 0 || 131 strcmp(name, "owner_rw") == 0 || 132 strcmp(name, "owner_sx") == 0) 133 return (true); 134 135 /* 136 * The KMSAN runtime can't be instrumented safely. 137 */ 138 if (strncmp(name, "__msan", 6) == 0 || 139 strncmp(name, "kmsan_", 6) == 0) 140 return (1); 141 142 /* 143 * When DTrace is built into the kernel we need to exclude the kinst 144 * functions from instrumentation. 145 */ 146 #ifndef _KLD_MODULE 147 if (strncmp(name, "kinst_", strlen("kinst_")) == 0) 148 return (true); 149 #endif 150 151 if (strcmp(name, "trap_check") == 0) 152 return (true); 153 154 return (false); 155 } 156 157 void 158 kinst_probe_create(struct kinst_probe *kp, linker_file_t lf) 159 { 160 kp->kp_id = dtrace_probe_create(kinst_id, lf->filename, 161 kp->kp_func, kp->kp_name, 3, kp); 162 163 LIST_INSERT_HEAD(KINST_GETPROBE(kp->kp_patchpoint), kp, kp_hashnext); 164 } 165 166 static int 167 kinst_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, 168 struct thread *td __unused) 169 { 170 return (0); 171 } 172 173 static int 174 kinst_close(struct cdev *dev __unused, int fflag __unused, int devtype __unused, 175 struct thread *td __unused) 176 { 177 dtrace_condense(kinst_id); 178 return (0); 179 } 180 181 static int 182 kinst_linker_file_cb(linker_file_t lf, void *arg) 183 { 184 dtrace_kinst_probedesc_t *pd; 185 186 pd = arg; 187 if (pd->kpd_mod[0] != '\0' && strcmp(pd->kpd_mod, lf->filename) != 0) 188 return (0); 189 190 /* 191 * Invoke kinst_make_probe_function() once for each function symbol in 192 * the module "lf". 193 */ 194 return (linker_file_function_listall(lf, kinst_make_probe, arg)); 195 } 196 197 static int 198 kinst_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t addr, 199 int flags __unused, struct thread *td __unused) 200 { 201 dtrace_kinst_probedesc_t *pd; 202 int error = 0; 203 204 switch (cmd) { 205 case KINSTIOC_MAKEPROBE: 206 pd = (dtrace_kinst_probedesc_t *)addr; 207 pd->kpd_func[sizeof(pd->kpd_func) - 1] = '\0'; 208 pd->kpd_mod[sizeof(pd->kpd_mod) - 1] = '\0'; 209 210 /* Loop over all functions in the kernel and loaded modules. */ 211 error = linker_file_foreach(kinst_linker_file_cb, pd); 212 break; 213 default: 214 error = ENOTTY; 215 break; 216 } 217 218 return (error); 219 } 220 221 static void 222 kinst_provide_module(void *arg, modctl_t *lf) 223 { 224 } 225 226 static void 227 kinst_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) 228 { 229 desc->dtargd_ndx = DTRACE_ARGNONE; 230 } 231 232 static void 233 kinst_destroy(void *arg, dtrace_id_t id, void *parg) 234 { 235 struct kinst_probe *kp = parg; 236 237 LIST_REMOVE(kp, kp_hashnext); 238 #ifndef __amd64__ 239 kinst_trampoline_dealloc(kp->kp_tramp); 240 #endif 241 free(kp, M_KINST); 242 } 243 244 static void 245 kinst_enable(void *arg, dtrace_id_t id, void *parg) 246 { 247 struct kinst_probe *kp = parg; 248 static bool warned = false; 249 250 if (!warned) { 251 KINST_LOG( 252 "kinst: This provider is experimental, exercise caution"); 253 warned = true; 254 } 255 256 kinst_patch_tracepoint(kp, kp->kp_patchval); 257 } 258 259 static void 260 kinst_disable(void *arg, dtrace_id_t id, void *parg) 261 { 262 struct kinst_probe *kp = parg; 263 264 kinst_patch_tracepoint(kp, kp->kp_savedval); 265 } 266 267 static int 268 kinst_load(void *dummy) 269 { 270 int error; 271 272 error = kinst_trampoline_init(); 273 if (error != 0) 274 return (error); 275 error = kinst_md_init(); 276 if (error != 0) { 277 kinst_trampoline_deinit(); 278 return (error); 279 } 280 281 error = dtrace_register("kinst", &kinst_attr, DTRACE_PRIV_USER, NULL, 282 &kinst_pops, NULL, &kinst_id); 283 if (error != 0) { 284 kinst_md_deinit(); 285 kinst_trampoline_deinit(); 286 return (error); 287 } 288 kinst_probetab = malloc(KINST_PROBETAB_MAX * 289 sizeof(struct kinst_probe_list), M_KINST, M_WAITOK | M_ZERO); 290 for (int i = 0; i < KINST_PROBETAB_MAX; i++) 291 LIST_INIT(&kinst_probetab[i]); 292 kinst_cdev = make_dev(&kinst_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 293 "dtrace/kinst"); 294 dtrace_invop_add(kinst_invop); 295 return (0); 296 } 297 298 static int 299 kinst_unload(void *dummy) 300 { 301 free(kinst_probetab, M_KINST); 302 kinst_md_deinit(); 303 kinst_trampoline_deinit(); 304 dtrace_invop_remove(kinst_invop); 305 destroy_dev(kinst_cdev); 306 307 return (dtrace_unregister(kinst_id)); 308 } 309 310 static int 311 kinst_modevent(module_t mod __unused, int type, void *data __unused) 312 { 313 int error = 0; 314 315 switch (type) { 316 case MOD_LOAD: 317 break; 318 case MOD_UNLOAD: 319 break; 320 case MOD_SHUTDOWN: 321 break; 322 default: 323 error = EOPNOTSUPP; 324 break; 325 } 326 327 return (error); 328 } 329 330 SYSINIT(kinst_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_load, NULL); 331 SYSUNINIT(kinst_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_unload, 332 NULL); 333 334 DEV_MODULE(kinst, kinst_modevent, NULL); 335 MODULE_VERSION(kinst, 1); 336 MODULE_DEPEND(kinst, dtrace, 1, 1, 1); 337 MODULE_DEPEND(kinst, opensolaris, 1, 1, 1); 338