1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> 5 * Copyright (c) 2023 The FreeBSD Foundation 6 * 7 * Portions of this software were developed by Christos Margiolis 8 * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 9 */ 10 11 #include <sys/param.h> 12 #include <sys/systm.h> 13 #include <sys/conf.h> 14 #include <sys/kernel.h> 15 #include <sys/linker.h> 16 #include <sys/module.h> 17 18 #include <sys/dtrace.h> 19 20 #include "kinst.h" 21 22 MALLOC_DEFINE(M_KINST, "kinst", "Kernel Instruction Tracing"); 23 24 static d_open_t kinst_open; 25 static d_close_t kinst_close; 26 static d_ioctl_t kinst_ioctl; 27 28 static void kinst_provide_module(void *, modctl_t *); 29 static void kinst_getargdesc(void *, dtrace_id_t, void *, 30 dtrace_argdesc_t *); 31 static void kinst_destroy(void *, dtrace_id_t, void *); 32 static void kinst_enable(void *, dtrace_id_t, void *); 33 static void kinst_disable(void *, dtrace_id_t, void *); 34 static int kinst_load(void *); 35 static int kinst_unload(void *); 36 static int kinst_modevent(module_t, int, void *); 37 38 static dtrace_pattr_t kinst_attr = { 39 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 40 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 41 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 42 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 43 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 44 }; 45 46 static const dtrace_pops_t kinst_pops = { 47 .dtps_provide = NULL, 48 .dtps_provide_module = kinst_provide_module, 49 .dtps_enable = kinst_enable, 50 .dtps_disable = kinst_disable, 51 .dtps_suspend = NULL, 52 .dtps_resume = NULL, 53 .dtps_getargdesc = kinst_getargdesc, 54 .dtps_getargval = NULL, 55 .dtps_usermode = NULL, 56 .dtps_destroy = kinst_destroy 57 }; 58 59 static struct cdevsw kinst_cdevsw = { 60 .d_name = "kinst", 61 .d_version = D_VERSION, 62 .d_flags = D_TRACKCLOSE, 63 .d_open = kinst_open, 64 .d_close = kinst_close, 65 .d_ioctl = kinst_ioctl, 66 }; 67 68 static dtrace_provider_id_t kinst_id; 69 struct kinst_probe_list *kinst_probetab; 70 static struct cdev *kinst_cdev; 71 72 /* 73 * Tracing memcpy() will crash the kernel when kinst tries to trace an instance 74 * of the memcpy() calls in kinst_invop(). To fix this, we can use 75 * kinst_memcpy() in those cases, with its arguments marked as 'volatile' to 76 * "outsmart" the compiler and avoid having it replaced by a regular memcpy(). 77 */ 78 volatile void * 79 kinst_memcpy(volatile void *dst, volatile const void *src, size_t len) 80 { 81 volatile const unsigned char *src0; 82 volatile unsigned char *dst0; 83 84 src0 = src; 85 dst0 = dst; 86 87 while (len--) 88 *dst0++ = *src0++; 89 90 return (dst); 91 } 92 93 bool 94 kinst_excluded(const char *name) 95 { 96 if (kinst_md_excluded(name)) 97 return (true); 98 99 /* 100 * cpu_switch() can cause a crash if it modifies the value of curthread 101 * while in probe context. 102 */ 103 if (strcmp(name, "cpu_switch") == 0) 104 return (true); 105 106 /* 107 * Anything beginning with "dtrace_" may be called from probe context 108 * unless it explicitly indicates that it won't be called from probe 109 * context by using the prefix "dtrace_safe_". 110 */ 111 if (strncmp(name, "dtrace_", strlen("dtrace_")) == 0 && 112 strncmp(name, "dtrace_safe_", strlen("dtrace_safe_")) != 0) 113 return (true); 114 115 /* 116 * Omit instrumentation of functions that are probably in DDB. It 117 * makes it too hard to debug broken kinst. 118 * 119 * NB: kdb_enter() can be excluded, but its call to printf() can't be. 120 * This is generally OK since we're not yet in debugging context. 121 */ 122 if (strncmp(name, "db_", strlen("db_")) == 0 || 123 strncmp(name, "kdb_", strlen("kdb_")) == 0) 124 return (true); 125 126 /* 127 * Lock owner methods may be called from probe context. 128 */ 129 if (strcmp(name, "owner_mtx") == 0 || 130 strcmp(name, "owner_rm") == 0 || 131 strcmp(name, "owner_rw") == 0 || 132 strcmp(name, "owner_sx") == 0) 133 return (true); 134 135 /* 136 * When DTrace is built into the kernel we need to exclude the kinst 137 * functions from instrumentation. 138 */ 139 #ifndef _KLD_MODULE 140 if (strncmp(name, "kinst_", strlen("kinst_")) == 0) 141 return (true); 142 #endif 143 144 if (strcmp(name, "trap_check") == 0) 145 return (true); 146 147 return (false); 148 } 149 150 void 151 kinst_probe_create(struct kinst_probe *kp, linker_file_t lf) 152 { 153 kp->kp_id = dtrace_probe_create(kinst_id, lf->filename, 154 kp->kp_func, kp->kp_name, 3, kp); 155 156 LIST_INSERT_HEAD(KINST_GETPROBE(kp->kp_patchpoint), kp, kp_hashnext); 157 } 158 159 static int 160 kinst_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, 161 struct thread *td __unused) 162 { 163 return (0); 164 } 165 166 static int 167 kinst_close(struct cdev *dev __unused, int fflag __unused, int devtype __unused, 168 struct thread *td __unused) 169 { 170 dtrace_condense(kinst_id); 171 return (0); 172 } 173 174 static int 175 kinst_linker_file_cb(linker_file_t lf, void *arg) 176 { 177 dtrace_kinst_probedesc_t *pd; 178 179 pd = arg; 180 if (pd->kpd_mod[0] != '\0' && strcmp(pd->kpd_mod, lf->filename) != 0) 181 return (0); 182 183 /* 184 * Invoke kinst_make_probe_function() once for each function symbol in 185 * the module "lf". 186 */ 187 return (linker_file_function_listall(lf, kinst_make_probe, arg)); 188 } 189 190 static int 191 kinst_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t addr, 192 int flags __unused, struct thread *td __unused) 193 { 194 dtrace_kinst_probedesc_t *pd; 195 int error = 0; 196 197 switch (cmd) { 198 case KINSTIOC_MAKEPROBE: 199 pd = (dtrace_kinst_probedesc_t *)addr; 200 pd->kpd_func[sizeof(pd->kpd_func) - 1] = '\0'; 201 pd->kpd_mod[sizeof(pd->kpd_mod) - 1] = '\0'; 202 203 /* Loop over all functions in the kernel and loaded modules. */ 204 error = linker_file_foreach(kinst_linker_file_cb, pd); 205 break; 206 default: 207 error = ENOTTY; 208 break; 209 } 210 211 return (error); 212 } 213 214 static void 215 kinst_provide_module(void *arg, modctl_t *lf) 216 { 217 } 218 219 static void 220 kinst_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) 221 { 222 desc->dtargd_ndx = DTRACE_ARGNONE; 223 } 224 225 static void 226 kinst_destroy(void *arg, dtrace_id_t id, void *parg) 227 { 228 struct kinst_probe *kp = parg; 229 230 LIST_REMOVE(kp, kp_hashnext); 231 #ifndef __amd64__ 232 kinst_trampoline_dealloc(kp->kp_tramp); 233 #endif 234 free(kp, M_KINST); 235 } 236 237 static void 238 kinst_enable(void *arg, dtrace_id_t id, void *parg) 239 { 240 struct kinst_probe *kp = parg; 241 static bool warned = false; 242 243 if (!warned) { 244 KINST_LOG( 245 "kinst: This provider is experimental, exercise caution"); 246 warned = true; 247 } 248 249 kinst_patch_tracepoint(kp, kp->kp_patchval); 250 } 251 252 static void 253 kinst_disable(void *arg, dtrace_id_t id, void *parg) 254 { 255 struct kinst_probe *kp = parg; 256 257 kinst_patch_tracepoint(kp, kp->kp_savedval); 258 } 259 260 static int 261 kinst_load(void *dummy) 262 { 263 int error; 264 265 error = kinst_trampoline_init(); 266 if (error != 0) 267 return (error); 268 error = kinst_md_init(); 269 if (error != 0) { 270 kinst_trampoline_deinit(); 271 return (error); 272 } 273 274 error = dtrace_register("kinst", &kinst_attr, DTRACE_PRIV_USER, NULL, 275 &kinst_pops, NULL, &kinst_id); 276 if (error != 0) { 277 kinst_md_deinit(); 278 kinst_trampoline_deinit(); 279 return (error); 280 } 281 kinst_probetab = malloc(KINST_PROBETAB_MAX * 282 sizeof(struct kinst_probe_list), M_KINST, M_WAITOK | M_ZERO); 283 for (int i = 0; i < KINST_PROBETAB_MAX; i++) 284 LIST_INIT(&kinst_probetab[i]); 285 kinst_cdev = make_dev(&kinst_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 286 "dtrace/kinst"); 287 dtrace_invop_add(kinst_invop); 288 return (0); 289 } 290 291 static int 292 kinst_unload(void *dummy) 293 { 294 free(kinst_probetab, M_KINST); 295 kinst_md_deinit(); 296 kinst_trampoline_deinit(); 297 dtrace_invop_remove(kinst_invop); 298 destroy_dev(kinst_cdev); 299 300 return (dtrace_unregister(kinst_id)); 301 } 302 303 static int 304 kinst_modevent(module_t mod __unused, int type, void *data __unused) 305 { 306 int error = 0; 307 308 switch (type) { 309 case MOD_LOAD: 310 break; 311 case MOD_UNLOAD: 312 break; 313 case MOD_SHUTDOWN: 314 break; 315 default: 316 error = EOPNOTSUPP; 317 break; 318 } 319 320 return (error); 321 } 322 323 SYSINIT(kinst_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_load, NULL); 324 SYSUNINIT(kinst_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_unload, 325 NULL); 326 327 DEV_MODULE(kinst, kinst_modevent, NULL); 328 MODULE_VERSION(kinst, 1); 329 MODULE_DEPEND(kinst, dtrace, 1, 1, 1); 330 MODULE_DEPEND(kinst, opensolaris, 1, 1, 1); 331