1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org 22 * Copyright 2024 Mark Johnston <markj@FreeBSD.org> 23 */ 24 25 /* 26 * This file contains a reimplementation of the statically-defined tracing (SDT) 27 * framework for DTrace. Probes and SDT providers are defined using the macros 28 * in sys/sdt.h, which append all the needed structures to linker sets. When 29 * this module is loaded, it iterates over all of the loaded modules and 30 * registers probes and providers with the DTrace framework based on the 31 * contents of these linker sets. 32 * 33 * A list of SDT providers is maintained here since a provider may span multiple 34 * modules. When a kernel module is unloaded, a provider defined in that module 35 * is unregistered only if no other modules refer to it. The DTrace framework is 36 * responsible for destroying individual probes when a kernel module is 37 * unloaded; in particular, probes may not span multiple kernel modules. 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 43 #include <sys/conf.h> 44 #include <sys/endian.h> 45 #include <sys/eventhandler.h> 46 #include <sys/kernel.h> 47 #include <sys/limits.h> 48 #include <sys/linker.h> 49 #include <sys/linker_set.h> 50 #include <sys/lock.h> 51 #include <sys/lockstat.h> 52 #include <sys/malloc.h> 53 #include <sys/module.h> 54 #include <sys/mutex.h> 55 #include <sys/queue.h> 56 #include <sys/sdt.h> 57 58 #include <sys/dtrace.h> 59 #include <sys/dtrace_bsd.h> 60 61 /* DTrace methods. */ 62 static void sdt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); 63 static void sdt_provide_probes(void *, dtrace_probedesc_t *); 64 static void sdt_destroy(void *, dtrace_id_t, void *); 65 static void sdt_enable(void *, dtrace_id_t, void *); 66 static void sdt_disable(void *, dtrace_id_t, void *); 67 68 static void sdt_load(void); 69 static int sdt_unload(void); 70 static void sdt_create_provider(struct sdt_provider *); 71 static void sdt_create_probe(struct sdt_probe *); 72 static void sdt_kld_load(void *, struct linker_file *); 73 static void sdt_kld_unload_try(void *, struct linker_file *, int *); 74 75 static MALLOC_DEFINE(M_SDT, "SDT", "DTrace SDT providers"); 76 77 static int sdt_probes_enabled_count; 78 static int lockstat_enabled_count; 79 80 static dtrace_pattr_t sdt_attr = { 81 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 82 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 83 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 84 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 85 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 86 }; 87 88 static dtrace_pops_t sdt_pops = { 89 .dtps_provide = sdt_provide_probes, 90 .dtps_provide_module = NULL, 91 .dtps_enable = sdt_enable, 92 .dtps_disable = sdt_disable, 93 .dtps_suspend = NULL, 94 .dtps_resume = NULL, 95 .dtps_getargdesc = sdt_getargdesc, 96 .dtps_getargval = NULL, 97 .dtps_usermode = NULL, 98 .dtps_destroy = sdt_destroy, 99 }; 100 101 static TAILQ_HEAD(, sdt_provider) sdt_prov_list; 102 103 static eventhandler_tag sdt_kld_load_tag; 104 static eventhandler_tag sdt_kld_unload_try_tag; 105 106 static void 107 sdt_create_provider(struct sdt_provider *prov) 108 { 109 struct sdt_provider *curr, *newprov; 110 111 TAILQ_FOREACH(curr, &sdt_prov_list, prov_entry) 112 if (strcmp(prov->name, curr->name) == 0) { 113 /* The provider has already been defined. */ 114 curr->sdt_refs++; 115 return; 116 } 117 118 /* 119 * Make a copy of prov so that we don't lose fields if its module is 120 * unloaded but the provider isn't destroyed. This could happen with 121 * a provider that spans multiple modules. 122 */ 123 newprov = malloc(sizeof(*newprov), M_SDT, M_WAITOK | M_ZERO); 124 newprov->name = strdup(prov->name, M_SDT); 125 prov->sdt_refs = newprov->sdt_refs = 1; 126 127 TAILQ_INSERT_TAIL(&sdt_prov_list, newprov, prov_entry); 128 129 (void)dtrace_register(newprov->name, &sdt_attr, DTRACE_PRIV_USER, NULL, 130 &sdt_pops, NULL, (dtrace_provider_id_t *)&newprov->id); 131 prov->id = newprov->id; 132 } 133 134 static void 135 sdt_create_probe(struct sdt_probe *probe) 136 { 137 struct sdt_provider *prov; 138 char mod[DTRACE_MODNAMELEN]; 139 char func[DTRACE_FUNCNAMELEN]; 140 char name[DTRACE_NAMELEN]; 141 const char *from; 142 char *to; 143 size_t len; 144 145 if (probe->version != (int)sizeof(*probe)) { 146 printf("ignoring probe %p, version %u expected %u\n", 147 probe, probe->version, (int)sizeof(*probe)); 148 return; 149 } 150 151 TAILQ_FOREACH(prov, &sdt_prov_list, prov_entry) 152 if (strcmp(prov->name, probe->prov->name) == 0) 153 break; 154 155 KASSERT(prov != NULL, ("probe defined without a provider")); 156 157 /* If no module name was specified, use the module filename. */ 158 if (*probe->mod == 0) { 159 len = strlcpy(mod, probe->sdtp_lf->filename, sizeof(mod)); 160 if (len > 3 && strcmp(mod + len - 3, ".ko") == 0) 161 mod[len - 3] = '\0'; 162 } else 163 strlcpy(mod, probe->mod, sizeof(mod)); 164 165 /* 166 * Unfortunately this is necessary because the Solaris DTrace 167 * code mixes consts and non-consts with casts to override 168 * the incompatibilies. On FreeBSD, we use strict warnings 169 * in the C compiler, so we have to respect const vs non-const. 170 */ 171 strlcpy(func, probe->func, sizeof(func)); 172 if (func[0] == '\0') 173 strcpy(func, "none"); 174 175 from = probe->name; 176 to = name; 177 for (len = 0; len < (sizeof(name) - 1) && *from != '\0'; 178 len++, from++, to++) { 179 if (from[0] == '_' && from[1] == '_') { 180 *to = '-'; 181 from++; 182 } else 183 *to = *from; 184 } 185 *to = '\0'; 186 187 if (dtrace_probe_lookup(prov->id, mod, func, name) != DTRACE_IDNONE) 188 return; 189 190 (void)dtrace_probe_create(prov->id, mod, func, name, 0, probe); 191 } 192 193 /* 194 * Probes are created through the SDT module load/unload hook, so this function 195 * has nothing to do. It only exists because the DTrace provider framework 196 * requires one of provide_probes and provide_module to be defined. 197 */ 198 static void 199 sdt_provide_probes(void *arg, dtrace_probedesc_t *desc) 200 { 201 } 202 203 struct sdt_enable_cb_arg { 204 struct sdt_probe *probe; 205 int cpu; 206 int arrived; 207 int done; 208 bool enable; 209 }; 210 211 static void 212 sdt_probe_update_cb(void *_arg) 213 { 214 struct sdt_enable_cb_arg *arg; 215 struct sdt_tracepoint *tp; 216 217 arg = _arg; 218 if (arg->cpu != curcpu) { 219 atomic_add_rel_int(&arg->arrived, 1); 220 while (atomic_load_acq_int(&arg->done) == 0) 221 cpu_spinwait(); 222 return; 223 } else { 224 while (atomic_load_acq_int(&arg->arrived) != mp_ncpus - 1) 225 cpu_spinwait(); 226 } 227 228 STAILQ_FOREACH(tp, &arg->probe->tracepoint_list, tracepoint_entry) { 229 if (arg->enable) 230 sdt_tracepoint_patch(tp->patchpoint, tp->target); 231 else 232 sdt_tracepoint_restore(tp->patchpoint); 233 } 234 235 atomic_store_rel_int(&arg->done, 1); 236 } 237 238 static void 239 sdt_probe_update(struct sdt_probe *probe, bool enable) 240 { 241 struct sdt_enable_cb_arg cbarg; 242 243 sched_pin(); 244 cbarg.probe = probe; 245 cbarg.cpu = curcpu; 246 atomic_store_rel_int(&cbarg.arrived, 0); 247 atomic_store_rel_int(&cbarg.done, 0); 248 cbarg.enable = enable; 249 smp_rendezvous(NULL, sdt_probe_update_cb, NULL, &cbarg); 250 sched_unpin(); 251 } 252 253 static void 254 sdt_enable(void *arg __unused, dtrace_id_t id, void *parg) 255 { 256 struct sdt_probe *probe; 257 258 probe = parg; 259 260 probe->id = id; 261 probe->sdtp_lf->nenabled++; 262 if (strcmp(probe->prov->name, "lockstat") == 0) { 263 lockstat_enabled_count++; 264 if (lockstat_enabled_count == 1) 265 lockstat_enabled = true; 266 } 267 sdt_probes_enabled_count++; 268 if (sdt_probes_enabled_count == 1) 269 sdt_probes_enabled = true; 270 271 sdt_probe_update(probe, true); 272 } 273 274 static void 275 sdt_disable(void *arg __unused, dtrace_id_t id, void *parg) 276 { 277 struct sdt_probe *probe; 278 279 probe = parg; 280 KASSERT(probe->sdtp_lf->nenabled > 0, ("no probes enabled")); 281 282 sdt_probe_update(probe, false); 283 284 sdt_probes_enabled_count--; 285 if (sdt_probes_enabled_count == 0) 286 sdt_probes_enabled = false; 287 if (strcmp(probe->prov->name, "lockstat") == 0) { 288 lockstat_enabled_count--; 289 if (lockstat_enabled_count == 0) 290 lockstat_enabled = false; 291 } 292 probe->id = 0; 293 probe->sdtp_lf->nenabled--; 294 } 295 296 static void 297 sdt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) 298 { 299 struct sdt_argtype *argtype; 300 struct sdt_probe *probe = parg; 301 302 if (desc->dtargd_ndx >= probe->n_args) { 303 desc->dtargd_ndx = DTRACE_ARGNONE; 304 return; 305 } 306 307 TAILQ_FOREACH(argtype, &probe->argtype_list, argtype_entry) { 308 if (desc->dtargd_ndx == argtype->ndx) { 309 desc->dtargd_mapping = desc->dtargd_ndx; 310 if (argtype->type == NULL) { 311 desc->dtargd_native[0] = '\0'; 312 desc->dtargd_xlate[0] = '\0'; 313 continue; 314 } 315 strlcpy(desc->dtargd_native, argtype->type, 316 sizeof(desc->dtargd_native)); 317 if (argtype->xtype != NULL) 318 strlcpy(desc->dtargd_xlate, argtype->xtype, 319 sizeof(desc->dtargd_xlate)); 320 } 321 } 322 } 323 324 static void 325 sdt_destroy(void *arg, dtrace_id_t id, void *parg) 326 { 327 } 328 329 static void 330 sdt_kld_load_providers(struct linker_file *lf) 331 { 332 struct sdt_provider **prov, **begin, **end; 333 334 if (linker_file_lookup_set(lf, "sdt_providers_set", &begin, &end, 335 NULL) == 0) { 336 for (prov = begin; prov < end; prov++) 337 sdt_create_provider(*prov); 338 } 339 } 340 341 static void 342 sdt_kld_load_probes(struct linker_file *lf) 343 { 344 struct sdt_probe **p_begin, **p_end; 345 struct sdt_argtype **a_begin, **a_end; 346 struct sdt_tracepoint *tp_begin, *tp_end; 347 348 if (linker_file_lookup_set(lf, "sdt_probes_set", &p_begin, &p_end, 349 NULL) == 0) { 350 for (struct sdt_probe **probe = p_begin; probe < p_end; 351 probe++) { 352 (*probe)->sdtp_lf = lf; 353 sdt_create_probe(*probe); 354 TAILQ_INIT(&(*probe)->argtype_list); 355 STAILQ_INIT(&(*probe)->tracepoint_list); 356 } 357 } 358 359 if (linker_file_lookup_set(lf, "sdt_argtypes_set", &a_begin, &a_end, 360 NULL) == 0) { 361 for (struct sdt_argtype **argtype = a_begin; argtype < a_end; 362 argtype++) { 363 (*argtype)->probe->n_args++; 364 TAILQ_INSERT_TAIL(&(*argtype)->probe->argtype_list, 365 *argtype, argtype_entry); 366 } 367 } 368 369 if (linker_file_lookup_set(lf, __XSTRING(_SDT_TRACEPOINT_SET), 370 &tp_begin, &tp_end, NULL) == 0) { 371 for (struct sdt_tracepoint *tp = tp_begin; tp < tp_end; tp++) { 372 if (!sdt_tracepoint_valid(tp->patchpoint, tp->target)) { 373 printf( 374 "invalid tracepoint %#jx->%#jx for %s:%s:%s:%s\n", 375 (uintmax_t)tp->patchpoint, 376 (uintmax_t)tp->target, 377 tp->probe->prov->name, tp->probe->mod, 378 tp->probe->func, tp->probe->name); 379 continue; 380 } 381 STAILQ_INSERT_TAIL(&tp->probe->tracepoint_list, tp, 382 tracepoint_entry); 383 } 384 } 385 } 386 387 /* 388 * Called from the kernel linker when a module is loaded, before 389 * dtrace_module_loaded() is called. This is done so that it's possible to 390 * register new providers when modules are loaded. The DTrace framework 391 * explicitly disallows calling into the framework from the provide_module 392 * provider method, so we cannot do this there. 393 */ 394 static void 395 sdt_kld_load(void *arg __unused, struct linker_file *lf) 396 { 397 sdt_kld_load_providers(lf); 398 sdt_kld_load_probes(lf); 399 } 400 401 static void 402 sdt_kld_unload_try(void *arg __unused, struct linker_file *lf, int *error) 403 { 404 struct sdt_provider *prov, **curr, **begin, **end, *tmp; 405 406 if (*error != 0) 407 /* We already have an error, so don't do anything. */ 408 return; 409 else if (linker_file_lookup_set(lf, "sdt_providers_set", &begin, &end, 410 NULL)) 411 /* No DTrace providers are declared in this file. */ 412 return; 413 414 /* 415 * Go through all the providers declared in this linker file and 416 * unregister any that aren't declared in another loaded file. 417 */ 418 for (curr = begin; curr < end; curr++) { 419 TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) { 420 if (strcmp(prov->name, (*curr)->name) != 0) 421 continue; 422 423 if (prov->sdt_refs == 1) { 424 if (dtrace_unregister(prov->id) != 0) { 425 *error = 1; 426 return; 427 } 428 TAILQ_REMOVE(&sdt_prov_list, prov, prov_entry); 429 free(prov->name, M_SDT); 430 free(prov, M_SDT); 431 } else 432 prov->sdt_refs--; 433 break; 434 } 435 } 436 } 437 438 static int 439 sdt_load_providers_cb(linker_file_t lf, void *arg __unused) 440 { 441 sdt_kld_load_providers(lf); 442 return (0); 443 } 444 445 static int 446 sdt_load_probes_cb(linker_file_t lf, void *arg __unused) 447 { 448 sdt_kld_load_probes(lf); 449 return (0); 450 } 451 452 static void 453 sdt_load(void) 454 { 455 456 TAILQ_INIT(&sdt_prov_list); 457 458 sdt_probe_func = dtrace_probe; 459 sdt_probe6_func = (sdt_probe6_func_t)dtrace_probe; 460 461 sdt_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, sdt_kld_load, NULL, 462 EVENTHANDLER_PRI_ANY); 463 sdt_kld_unload_try_tag = EVENTHANDLER_REGISTER(kld_unload_try, 464 sdt_kld_unload_try, NULL, EVENTHANDLER_PRI_ANY); 465 466 /* 467 * Pick up probes from the kernel and already-loaded linker files. 468 * Define providers in a separate pass since a linker file may be using 469 * providers defined in a file that appears later in the list. 470 */ 471 linker_file_foreach(sdt_load_providers_cb, NULL); 472 linker_file_foreach(sdt_load_probes_cb, NULL); 473 } 474 475 static int 476 sdt_unload(void) 477 { 478 struct sdt_provider *prov, *tmp; 479 int ret; 480 481 EVENTHANDLER_DEREGISTER(kld_load, sdt_kld_load_tag); 482 EVENTHANDLER_DEREGISTER(kld_unload_try, sdt_kld_unload_try_tag); 483 484 sdt_probe_func = sdt_probe_stub; 485 sdt_probe6_func = (sdt_probe6_func_t)sdt_probe_stub; 486 487 TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) { 488 ret = dtrace_unregister(prov->id); 489 if (ret != 0) 490 return (ret); 491 TAILQ_REMOVE(&sdt_prov_list, prov, prov_entry); 492 free(prov->name, M_SDT); 493 free(prov, M_SDT); 494 } 495 496 return (0); 497 } 498 499 static int 500 sdt_modevent(module_t mod __unused, int type, void *data __unused) 501 { 502 switch (type) { 503 case MOD_LOAD: 504 case MOD_UNLOAD: 505 case MOD_SHUTDOWN: 506 return (0); 507 default: 508 return (EOPNOTSUPP); 509 } 510 } 511 512 SYSINIT(sdt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, sdt_load, NULL); 513 SYSUNINIT(sdt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, sdt_unload, NULL); 514 515 DEV_MODULE(sdt, sdt_modevent, NULL); 516 MODULE_VERSION(sdt, 1); 517 MODULE_DEPEND(sdt, dtrace, 1, 1, 1); 518