1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org 22 * 23 */ 24 25 /* 26 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 */ 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/conf.h> 33 #include <sys/cpuvar.h> 34 #include <sys/endian.h> 35 #include <sys/fcntl.h> 36 #include <sys/filio.h> 37 #include <sys/kdb.h> 38 #include <sys/kernel.h> 39 #include <sys/kmem.h> 40 #include <sys/kthread.h> 41 #include <sys/limits.h> 42 #include <sys/linker.h> 43 #include <sys/lock.h> 44 #include <sys/malloc.h> 45 #include <sys/module.h> 46 #include <sys/mutex.h> 47 #include <sys/pcpu.h> 48 #include <sys/poll.h> 49 #include <sys/proc.h> 50 #include <sys/selinfo.h> 51 #include <sys/smp.h> 52 #include <sys/syscall.h> 53 #include <sys/sysent.h> 54 #include <sys/sysproto.h> 55 #include <sys/uio.h> 56 #include <sys/unistd.h> 57 #include <machine/stdarg.h> 58 59 #include <sys/dtrace.h> 60 #include <sys/dtrace_bsd.h> 61 62 #include "fbt.h" 63 64 MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing"); 65 66 dtrace_provider_id_t fbt_id; 67 fbt_probe_t **fbt_probetab; 68 int fbt_probetab_mask; 69 70 static int fbt_unload(void); 71 static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); 72 static void fbt_provide_module(void *, modctl_t *); 73 static void fbt_destroy(void *, dtrace_id_t, void *); 74 static void fbt_enable(void *, dtrace_id_t, void *); 75 static void fbt_disable(void *, dtrace_id_t, void *); 76 static void fbt_load(void *); 77 static void fbt_suspend(void *, dtrace_id_t, void *); 78 static void fbt_resume(void *, dtrace_id_t, void *); 79 80 static dtrace_pattr_t fbt_attr = { 81 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 82 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 83 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 84 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 85 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 86 }; 87 88 static dtrace_pops_t fbt_pops = { 89 .dtps_provide = NULL, 90 .dtps_provide_module = fbt_provide_module, 91 .dtps_enable = fbt_enable, 92 .dtps_disable = fbt_disable, 93 .dtps_suspend = fbt_suspend, 94 .dtps_resume = fbt_resume, 95 .dtps_getargdesc = fbt_getargdesc, 96 .dtps_getargval = NULL, 97 .dtps_usermode = NULL, 98 .dtps_destroy = fbt_destroy 99 }; 100 101 static int fbt_probetab_size; 102 static int fbt_verbose = 0; 103 104 int 105 fbt_excluded(const char *name) 106 { 107 108 if (strncmp(name, "dtrace_", 7) == 0 && 109 strncmp(name, "dtrace_safe_", 12) != 0) { 110 /* 111 * Anything beginning with "dtrace_" may be called 112 * from probe context unless it explicitly indicates 113 * that it won't be called from probe context by 114 * using the prefix "dtrace_safe_". 115 */ 116 return (1); 117 } 118 119 /* 120 * Omit instrumentation of functions that are probably in DDB. It 121 * makes it too hard to debug broken FBT. 122 * 123 * NB: kdb_enter() can be excluded, but its call to printf() can't be. 124 * This is generally OK since we're not yet in debugging context. 125 */ 126 if (strncmp(name, "db_", 3) == 0 || 127 strncmp(name, "kdb_", 4) == 0) 128 return (1); 129 130 /* 131 * Lock owner methods may be called from probe context. 132 */ 133 if (strcmp(name, "owner_mtx") == 0 || 134 strcmp(name, "owner_rm") == 0 || 135 strcmp(name, "owner_rw") == 0 || 136 strcmp(name, "owner_sx") == 0) 137 return (1); 138 139 /* 140 * The KMSAN runtime can't be instrumented safely. 141 */ 142 if (strncmp(name, "__msan", 6) == 0 || 143 strncmp(name, "kmsan_", 6) == 0) 144 return (1); 145 146 /* 147 * Stack unwinders may be called from probe context on some 148 * platforms. 149 */ 150 #if defined(__aarch64__) || defined(__riscv) 151 if (strcmp(name, "unwind_frame") == 0) 152 return (1); 153 #endif 154 155 /* 156 * When DTrace is built into the kernel we need to exclude 157 * the FBT functions from instrumentation. 158 */ 159 #ifndef _KLD_MODULE 160 if (strncmp(name, "fbt_", 4) == 0) 161 return (1); 162 #endif 163 164 return (0); 165 } 166 167 static void 168 fbt_doubletrap(void) 169 { 170 fbt_probe_t *fbt; 171 int i; 172 173 for (i = 0; i < fbt_probetab_size; i++) { 174 fbt = fbt_probetab[i]; 175 176 for (; fbt != NULL; fbt = fbt->fbtp_probenext) 177 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval); 178 } 179 } 180 181 static void 182 fbt_provide_module(void *arg, modctl_t *lf) 183 { 184 char modname[MAXPATHLEN]; 185 int i; 186 size_t len; 187 188 strlcpy(modname, lf->filename, sizeof(modname)); 189 len = strlen(modname); 190 if (len > 3 && strcmp(modname + len - 3, ".ko") == 0) 191 modname[len - 3] = '\0'; 192 193 /* 194 * Employees of dtrace and their families are ineligible. Void 195 * where prohibited. 196 */ 197 if (strcmp(modname, "dtrace") == 0) 198 return; 199 200 /* 201 * To register with DTrace, a module must list 'dtrace' as a 202 * dependency in order for the kernel linker to resolve 203 * symbols like dtrace_register(). All modules with such a 204 * dependency are ineligible for FBT tracing. 205 */ 206 for (i = 0; i < lf->ndeps; i++) 207 if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0) 208 return; 209 210 if (lf->fbt_nentries) { 211 /* 212 * This module has some FBT entries allocated; we're afraid 213 * to screw with it. 214 */ 215 return; 216 } 217 218 /* 219 * List the functions in the module and the symbol values. 220 */ 221 (void) linker_file_function_listall(lf, fbt_provide_module_function, modname); 222 } 223 224 static void 225 fbt_destroy_one(fbt_probe_t *fbt) 226 { 227 fbt_probe_t *hash, *hashprev, *next; 228 int ndx; 229 230 ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint); 231 for (hash = fbt_probetab[ndx], hashprev = NULL; hash != NULL; 232 hashprev = hash, hash = hash->fbtp_hashnext) { 233 if (hash == fbt) { 234 if ((next = fbt->fbtp_tracenext) != NULL) 235 next->fbtp_hashnext = hash->fbtp_hashnext; 236 else 237 next = hash->fbtp_hashnext; 238 if (hashprev != NULL) 239 hashprev->fbtp_hashnext = next; 240 else 241 fbt_probetab[ndx] = next; 242 goto free; 243 } else if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) { 244 for (next = hash; next->fbtp_tracenext != NULL; 245 next = next->fbtp_tracenext) { 246 if (fbt == next->fbtp_tracenext) { 247 next->fbtp_tracenext = 248 fbt->fbtp_tracenext; 249 goto free; 250 } 251 } 252 } 253 } 254 panic("probe %p not found in hash table", fbt); 255 free: 256 free(fbt, M_FBT); 257 } 258 259 static void 260 fbt_destroy(void *arg, dtrace_id_t id, void *parg) 261 { 262 fbt_probe_t *fbt = parg, *next; 263 modctl_t *ctl; 264 265 do { 266 ctl = fbt->fbtp_ctl; 267 ctl->fbt_nentries--; 268 269 next = fbt->fbtp_probenext; 270 fbt_destroy_one(fbt); 271 fbt = next; 272 } while (fbt != NULL); 273 } 274 275 static void 276 fbt_enable(void *arg, dtrace_id_t id, void *parg) 277 { 278 fbt_probe_t *fbt = parg; 279 modctl_t *ctl = fbt->fbtp_ctl; 280 281 ctl->nenabled++; 282 283 /* 284 * Now check that our modctl has the expected load count. If it 285 * doesn't, this module must have been unloaded and reloaded -- and 286 * we're not going to touch it. 287 */ 288 if (ctl->loadcnt != fbt->fbtp_loadcnt) { 289 if (fbt_verbose) { 290 printf("fbt is failing for probe %s " 291 "(module %s reloaded)", 292 fbt->fbtp_name, ctl->filename); 293 } 294 295 return; 296 } 297 298 for (; fbt != NULL; fbt = fbt->fbtp_probenext) { 299 fbt_patch_tracepoint(fbt, fbt->fbtp_patchval); 300 fbt->fbtp_enabled++; 301 } 302 } 303 304 static void 305 fbt_disable(void *arg, dtrace_id_t id, void *parg) 306 { 307 fbt_probe_t *fbt = parg, *hash; 308 modctl_t *ctl = fbt->fbtp_ctl; 309 310 ASSERT(ctl->nenabled > 0); 311 ctl->nenabled--; 312 313 if ((ctl->loadcnt != fbt->fbtp_loadcnt)) 314 return; 315 316 for (; fbt != NULL; fbt = fbt->fbtp_probenext) { 317 fbt->fbtp_enabled--; 318 319 for (hash = fbt_probetab[FBT_ADDR2NDX(fbt->fbtp_patchpoint)]; 320 hash != NULL; hash = hash->fbtp_hashnext) { 321 if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) { 322 for (; hash != NULL; hash = hash->fbtp_tracenext) 323 if (hash->fbtp_enabled > 0) 324 break; 325 break; 326 } 327 } 328 if (hash == NULL) 329 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval); 330 } 331 } 332 333 static void 334 fbt_suspend(void *arg, dtrace_id_t id, void *parg) 335 { 336 fbt_probe_t *fbt = parg; 337 modctl_t *ctl = fbt->fbtp_ctl; 338 339 ASSERT(ctl->nenabled > 0); 340 341 if ((ctl->loadcnt != fbt->fbtp_loadcnt)) 342 return; 343 344 for (; fbt != NULL; fbt = fbt->fbtp_probenext) 345 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval); 346 } 347 348 static void 349 fbt_resume(void *arg, dtrace_id_t id, void *parg) 350 { 351 fbt_probe_t *fbt = parg; 352 modctl_t *ctl = fbt->fbtp_ctl; 353 354 ASSERT(ctl->nenabled > 0); 355 356 if ((ctl->loadcnt != fbt->fbtp_loadcnt)) 357 return; 358 359 for (; fbt != NULL; fbt = fbt->fbtp_probenext) 360 fbt_patch_tracepoint(fbt, fbt->fbtp_patchval); 361 } 362 363 static int 364 fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc) 365 { 366 const Elf_Sym *symp = lc->symtab; 367 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 368 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); 369 size_t idwidth; 370 int i; 371 uint32_t *ctfoff; 372 uint32_t objtoff = hp->cth_objtoff; 373 uint32_t funcoff = hp->cth_funcoff; 374 uint_t kind, info, vlen; 375 376 /* Sanity check. */ 377 if (hp->cth_magic != CTF_MAGIC) { 378 printf("Bad magic value in CTF data of '%s'\n",lf->pathname); 379 return (EINVAL); 380 } 381 382 if (lc->symtab == NULL) { 383 printf("No symbol table in '%s'\n",lf->pathname); 384 return (EINVAL); 385 } 386 387 ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK); 388 *lc->ctfoffp = ctfoff; 389 390 idwidth = hp->cth_version == CTF_VERSION_2 ? 2 : 4; 391 392 for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) { 393 if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) { 394 *ctfoff = 0xffffffff; 395 continue; 396 } 397 398 switch (ELF_ST_TYPE(symp->st_info)) { 399 case STT_OBJECT: 400 if (objtoff >= hp->cth_funcoff || 401 (symp->st_shndx == SHN_ABS && symp->st_value == 0)) { 402 *ctfoff = 0xffffffff; 403 break; 404 } 405 406 *ctfoff = objtoff; 407 objtoff += idwidth; 408 break; 409 410 case STT_FUNC: 411 if (funcoff >= hp->cth_typeoff) { 412 *ctfoff = 0xffffffff; 413 break; 414 } 415 416 *ctfoff = funcoff; 417 418 info = 0; 419 memcpy(&info, ctfdata + funcoff, idwidth); 420 if (hp->cth_version == CTF_VERSION_2) { 421 kind = CTF_V2_INFO_KIND(info); 422 vlen = CTF_V2_INFO_VLEN(info); 423 } else { 424 kind = CTF_V3_INFO_KIND(info); 425 vlen = CTF_V3_INFO_VLEN(info); 426 } 427 428 /* 429 * If we encounter a zero pad at the end, just skip it. 430 * Otherwise skip over the function and its return type 431 * (+2) and the argument list (vlen). 432 */ 433 if (kind == CTF_K_UNKNOWN && vlen == 0) 434 funcoff += idwidth; 435 else 436 funcoff += idwidth * (vlen + 2); 437 break; 438 439 default: 440 *ctfoff = 0xffffffff; 441 break; 442 } 443 } 444 445 return (0); 446 } 447 448 static void 449 fbt_get_ctt_index(uint8_t version, const void *v, uint_t *indexp, 450 uint_t *typep, int *ischildp) 451 { 452 uint_t index, type; 453 int ischild; 454 455 if (version == CTF_VERSION_2) { 456 const struct ctf_type_v2 *ctt = v; 457 458 type = ctt->ctt_type; 459 index = CTF_V2_TYPE_TO_INDEX(ctt->ctt_type); 460 ischild = CTF_V2_TYPE_ISCHILD(ctt->ctt_type); 461 } else { 462 const struct ctf_type_v3 *ctt = v; 463 464 type = ctt->ctt_type; 465 index = CTF_V3_TYPE_TO_INDEX(ctt->ctt_type); 466 ischild = CTF_V3_TYPE_ISCHILD(ctt->ctt_type); 467 } 468 469 if (indexp != NULL) 470 *indexp = index; 471 if (typep != NULL) 472 *typep = type; 473 if (ischildp != NULL) 474 *ischildp = ischild; 475 } 476 477 static ssize_t 478 fbt_get_ctt_size(uint8_t version, const void *tp, ssize_t *sizep, 479 ssize_t *incrementp) 480 { 481 ssize_t size, increment; 482 483 if (version == CTF_VERSION_2) { 484 const struct ctf_type_v2 *ctt = tp; 485 486 if (ctt->ctt_size == CTF_V2_LSIZE_SENT) { 487 size = CTF_TYPE_LSIZE(ctt); 488 increment = sizeof (struct ctf_type_v2); 489 } else { 490 size = ctt->ctt_size; 491 increment = sizeof (struct ctf_stype_v2); 492 } 493 } else { 494 const struct ctf_type_v3 *ctt = tp; 495 496 if (ctt->ctt_size == CTF_V3_LSIZE_SENT) { 497 size = CTF_TYPE_LSIZE(ctt); 498 increment = sizeof (struct ctf_type_v3); 499 } else { 500 size = ctt->ctt_size; 501 increment = sizeof (struct ctf_stype_v3); 502 } 503 } 504 505 if (sizep) 506 *sizep = size; 507 if (incrementp) 508 *incrementp = increment; 509 510 return (size); 511 } 512 513 static void 514 fbt_get_ctt_info(uint8_t version, const void *tp, uint_t *kindp, uint_t *vlenp, 515 int *isrootp) 516 { 517 uint_t kind, vlen; 518 int isroot; 519 520 if (version == CTF_VERSION_2) { 521 const struct ctf_type_v2 *ctt = tp; 522 523 kind = CTF_V2_INFO_KIND(ctt->ctt_info); 524 vlen = CTF_V2_INFO_VLEN(ctt->ctt_info); 525 isroot = CTF_V2_INFO_ISROOT(ctt->ctt_info); 526 } else { 527 const struct ctf_type_v3 *ctt = tp; 528 529 kind = CTF_V3_INFO_KIND(ctt->ctt_info); 530 vlen = CTF_V3_INFO_VLEN(ctt->ctt_info); 531 isroot = CTF_V3_INFO_ISROOT(ctt->ctt_info); 532 } 533 534 if (kindp != NULL) 535 *kindp = kind; 536 if (vlenp != NULL) 537 *vlenp = vlen; 538 if (isrootp != NULL) 539 *isrootp = isroot; 540 } 541 542 static int 543 fbt_typoff_init(linker_ctf_t *lc) 544 { 545 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 546 const void *tbuf, *tend, *tp; 547 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); 548 size_t idwidth; 549 int ctf_typemax = 0; 550 uint32_t *xp; 551 ulong_t pop[CTF_K_MAX + 1] = { 0 }; 552 uint8_t version; 553 554 /* Sanity check. */ 555 if (hp->cth_magic != CTF_MAGIC) 556 return (EINVAL); 557 558 version = hp->cth_version; 559 idwidth = version == CTF_VERSION_2 ? 2 : 4; 560 561 tbuf = (const void *) (ctfdata + hp->cth_typeoff); 562 tend = (const void *) (ctfdata + hp->cth_stroff); 563 564 /* 565 * We make two passes through the entire type section. In this first 566 * pass, we count the number of each type and the total number of types. 567 */ 568 for (tp = tbuf; tp < tend; ctf_typemax++) { 569 uint_t kind, type, vlen; 570 ssize_t size, increment; 571 size_t vbytes; 572 573 (void) fbt_get_ctt_size(version, tp, &size, &increment); 574 fbt_get_ctt_info(version, tp, &kind, &vlen, NULL); 575 fbt_get_ctt_index(version, tp, NULL, &type, NULL); 576 577 switch (kind) { 578 case CTF_K_INTEGER: 579 case CTF_K_FLOAT: 580 vbytes = sizeof (uint_t); 581 break; 582 case CTF_K_ARRAY: 583 if (version == CTF_VERSION_2) 584 vbytes = sizeof (struct ctf_array_v2); 585 else 586 vbytes = sizeof (struct ctf_array_v3); 587 break; 588 case CTF_K_FUNCTION: 589 vbytes = roundup2(idwidth * vlen, sizeof(uint32_t)); 590 break; 591 case CTF_K_STRUCT: 592 case CTF_K_UNION: 593 if (version == CTF_VERSION_2) { 594 if (size < CTF_V2_LSTRUCT_THRESH) 595 vbytes = 596 sizeof (struct ctf_member_v2) * vlen; 597 else 598 vbytes = 599 sizeof (struct ctf_lmember_v2) * vlen; 600 } else { 601 if (size < CTF_V3_LSTRUCT_THRESH) 602 vbytes = 603 sizeof (struct ctf_member_v3) * vlen; 604 else 605 vbytes = 606 sizeof (struct ctf_lmember_v3) * vlen; 607 } 608 break; 609 case CTF_K_ENUM: 610 vbytes = sizeof (ctf_enum_t) * vlen; 611 break; 612 case CTF_K_FORWARD: 613 /* 614 * For forward declarations, ctt_type is the CTF_K_* 615 * kind for the tag, so bump that population count too. 616 * If ctt_type is unknown, treat the tag as a struct. 617 */ 618 if (type == CTF_K_UNKNOWN || type >= CTF_K_MAX) 619 pop[CTF_K_STRUCT]++; 620 else 621 pop[type]++; 622 /*FALLTHRU*/ 623 case CTF_K_UNKNOWN: 624 vbytes = 0; 625 break; 626 case CTF_K_POINTER: 627 case CTF_K_TYPEDEF: 628 case CTF_K_VOLATILE: 629 case CTF_K_CONST: 630 case CTF_K_RESTRICT: 631 vbytes = 0; 632 break; 633 default: 634 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); 635 return (EIO); 636 } 637 tp = (const void *)((uintptr_t)tp + increment + vbytes); 638 pop[kind]++; 639 } 640 641 /* account for a sentinel value below */ 642 ctf_typemax++; 643 *lc->typlenp = ctf_typemax; 644 645 xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER, 646 M_ZERO | M_WAITOK); 647 648 *lc->typoffp = xp; 649 650 /* type id 0 is used as a sentinel value */ 651 *xp++ = 0; 652 653 /* 654 * In the second pass, fill in the type offset. 655 */ 656 for (tp = tbuf; tp < tend; xp++) { 657 ssize_t size, increment; 658 uint_t kind, vlen; 659 660 size_t vbytes; 661 662 (void) fbt_get_ctt_size(version, tp, &size, &increment); 663 fbt_get_ctt_info(version, tp, &kind, &vlen, NULL); 664 665 switch (kind) { 666 case CTF_K_INTEGER: 667 case CTF_K_FLOAT: 668 vbytes = sizeof (uint_t); 669 break; 670 case CTF_K_ARRAY: 671 if (version == CTF_VERSION_2) 672 vbytes = sizeof (struct ctf_array_v2); 673 else 674 vbytes = sizeof (struct ctf_array_v3); 675 break; 676 case CTF_K_FUNCTION: 677 vbytes = roundup2(idwidth * vlen, sizeof(uint32_t)); 678 break; 679 case CTF_K_STRUCT: 680 case CTF_K_UNION: 681 if (version == CTF_VERSION_2) { 682 if (size < CTF_V2_LSTRUCT_THRESH) 683 vbytes = 684 sizeof (struct ctf_member_v2) * vlen; 685 else 686 vbytes = 687 sizeof (struct ctf_lmember_v2) * vlen; 688 } else { 689 if (size < CTF_V3_LSTRUCT_THRESH) 690 vbytes = 691 sizeof (struct ctf_member_v3) * vlen; 692 else 693 vbytes = 694 sizeof (struct ctf_lmember_v3) * vlen; 695 } 696 break; 697 case CTF_K_ENUM: 698 vbytes = sizeof (ctf_enum_t) * vlen; 699 break; 700 case CTF_K_FORWARD: 701 case CTF_K_UNKNOWN: 702 vbytes = 0; 703 break; 704 case CTF_K_POINTER: 705 case CTF_K_TYPEDEF: 706 case CTF_K_VOLATILE: 707 case CTF_K_CONST: 708 case CTF_K_RESTRICT: 709 vbytes = 0; 710 break; 711 default: 712 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); 713 return (EIO); 714 } 715 *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata); 716 tp = (const void *)((uintptr_t)tp + increment + vbytes); 717 } 718 719 return (0); 720 } 721 722 /* 723 * CTF Declaration Stack 724 * 725 * In order to implement ctf_type_name(), we must convert a type graph back 726 * into a C type declaration. Unfortunately, a type graph represents a storage 727 * class ordering of the type whereas a type declaration must obey the C rules 728 * for operator precedence, and the two orderings are frequently in conflict. 729 * For example, consider these CTF type graphs and their C declarations: 730 * 731 * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)() 732 * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[] 733 * 734 * In each case, parentheses are used to raise operator * to higher lexical 735 * precedence, so the string form of the C declaration cannot be constructed by 736 * walking the type graph links and forming the string from left to right. 737 * 738 * The functions in this file build a set of stacks from the type graph nodes 739 * corresponding to the C operator precedence levels in the appropriate order. 740 * The code in ctf_type_name() can then iterate over the levels and nodes in 741 * lexical precedence order and construct the final C declaration string. 742 */ 743 typedef struct ctf_list { 744 struct ctf_list *l_prev; /* previous pointer or tail pointer */ 745 struct ctf_list *l_next; /* next pointer or head pointer */ 746 } ctf_list_t; 747 748 #define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev)) 749 #define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next)) 750 751 typedef enum { 752 CTF_PREC_BASE, 753 CTF_PREC_POINTER, 754 CTF_PREC_ARRAY, 755 CTF_PREC_FUNCTION, 756 CTF_PREC_MAX 757 } ctf_decl_prec_t; 758 759 typedef struct ctf_decl_node { 760 ctf_list_t cd_list; /* linked list pointers */ 761 ctf_id_t cd_type; /* type identifier */ 762 uint_t cd_kind; /* type kind */ 763 uint_t cd_n; /* type dimension if array */ 764 } ctf_decl_node_t; 765 766 typedef struct ctf_decl { 767 ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */ 768 int cd_order[CTF_PREC_MAX]; /* storage order of decls */ 769 ctf_decl_prec_t cd_qualp; /* qualifier precision */ 770 ctf_decl_prec_t cd_ordp; /* ordered precision */ 771 char *cd_buf; /* buffer for output */ 772 char *cd_ptr; /* buffer location */ 773 char *cd_end; /* buffer limit */ 774 size_t cd_len; /* buffer space required */ 775 int cd_err; /* saved error value */ 776 } ctf_decl_t; 777 778 /* 779 * Simple doubly-linked list append routine. This implementation assumes that 780 * each list element contains an embedded ctf_list_t as the first member. 781 * An additional ctf_list_t is used to store the head (l_next) and tail 782 * (l_prev) pointers. The current head and tail list elements have their 783 * previous and next pointers set to NULL, respectively. 784 */ 785 static void 786 ctf_list_append(ctf_list_t *lp, void *new) 787 { 788 ctf_list_t *p = lp->l_prev; /* p = tail list element */ 789 ctf_list_t *q = new; /* q = new list element */ 790 791 lp->l_prev = q; 792 q->l_prev = p; 793 q->l_next = NULL; 794 795 if (p != NULL) 796 p->l_next = q; 797 else 798 lp->l_next = q; 799 } 800 801 /* 802 * Prepend the specified existing element to the given ctf_list_t. The 803 * existing pointer should be pointing at a struct with embedded ctf_list_t. 804 */ 805 static void 806 ctf_list_prepend(ctf_list_t *lp, void *new) 807 { 808 ctf_list_t *p = new; /* p = new list element */ 809 ctf_list_t *q = lp->l_next; /* q = head list element */ 810 811 lp->l_next = p; 812 p->l_prev = NULL; 813 p->l_next = q; 814 815 if (q != NULL) 816 q->l_prev = p; 817 else 818 lp->l_prev = p; 819 } 820 821 static void 822 ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len) 823 { 824 int i; 825 826 bzero(cd, sizeof (ctf_decl_t)); 827 828 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) 829 cd->cd_order[i] = CTF_PREC_BASE - 1; 830 831 cd->cd_qualp = CTF_PREC_BASE; 832 cd->cd_ordp = CTF_PREC_BASE; 833 834 cd->cd_buf = buf; 835 cd->cd_ptr = buf; 836 cd->cd_end = buf + len; 837 } 838 839 static void 840 ctf_decl_fini(ctf_decl_t *cd) 841 { 842 ctf_decl_node_t *cdp, *ndp; 843 int i; 844 845 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) { 846 for (cdp = ctf_list_next(&cd->cd_nodes[i]); 847 cdp != NULL; cdp = ndp) { 848 ndp = ctf_list_next(cdp); 849 free(cdp, M_FBT); 850 } 851 } 852 } 853 854 static const void * 855 ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type) 856 { 857 const void *tp; 858 uint32_t offset; 859 uint32_t *typoff = *lc->typoffp; 860 861 if (type >= *lc->typlenp) { 862 printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp); 863 return(NULL); 864 } 865 866 /* Check if the type isn't cross-referenced. */ 867 if ((offset = typoff[type]) == 0) { 868 printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type); 869 return(NULL); 870 } 871 872 tp = (const void *) (lc->ctftab + offset + sizeof(ctf_header_t)); 873 874 return (tp); 875 } 876 877 static void 878 fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp) 879 { 880 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 881 const void *tp; 882 ssize_t increment; 883 uint_t kind; 884 885 bzero(arp, sizeof(*arp)); 886 887 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) 888 return; 889 890 fbt_get_ctt_info(hp->cth_version, tp, &kind, NULL, NULL); 891 if (kind != CTF_K_ARRAY) 892 return; 893 894 (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment); 895 896 if (hp->cth_version == CTF_VERSION_2) { 897 const struct ctf_array_v2 *ap; 898 899 ap = (const struct ctf_array_v2 *)((uintptr_t)tp + increment); 900 arp->ctr_contents = ap->cta_contents; 901 arp->ctr_index = ap->cta_index; 902 arp->ctr_nelems = ap->cta_nelems; 903 } else { 904 const struct ctf_array_v3 *ap; 905 906 ap = (const struct ctf_array_v3 *)((uintptr_t)tp + increment); 907 arp->ctr_contents = ap->cta_contents; 908 arp->ctr_index = ap->cta_index; 909 arp->ctr_nelems = ap->cta_nelems; 910 } 911 } 912 913 static const char * 914 ctf_strptr(linker_ctf_t *lc, int name) 915 { 916 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 917 const char *strp = ""; 918 919 if (name < 0 || name >= hp->cth_strlen) 920 return(strp); 921 922 strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t)); 923 924 return (strp); 925 } 926 927 static const char * 928 ctf_type_rname(linker_ctf_t *lc, const void *v) 929 { 930 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 931 uint_t name; 932 933 if (hp->cth_version == CTF_VERSION_2) { 934 const struct ctf_type_v2 *ctt = v; 935 936 name = ctt->ctt_name; 937 } else { 938 const struct ctf_type_v3 *ctt = v; 939 940 name = ctt->ctt_name; 941 } 942 943 return (ctf_strptr(lc, name)); 944 } 945 946 static void 947 ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type) 948 { 949 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 950 ctf_decl_node_t *cdp; 951 ctf_decl_prec_t prec; 952 uint_t kind, n = 1, t; 953 int is_qual = 0; 954 955 const void *tp; 956 ctf_arinfo_t ar; 957 958 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) { 959 cd->cd_err = ENOENT; 960 return; 961 } 962 963 fbt_get_ctt_info(hp->cth_version, tp, &kind, NULL, NULL); 964 fbt_get_ctt_index(hp->cth_version, tp, NULL, &t, NULL); 965 966 switch (kind) { 967 case CTF_K_ARRAY: 968 fbt_array_info(lc, type, &ar); 969 ctf_decl_push(cd, lc, ar.ctr_contents); 970 n = ar.ctr_nelems; 971 prec = CTF_PREC_ARRAY; 972 break; 973 974 case CTF_K_TYPEDEF: 975 if (ctf_type_rname(lc, tp)[0] == '\0') { 976 ctf_decl_push(cd, lc, t); 977 return; 978 } 979 prec = CTF_PREC_BASE; 980 break; 981 982 case CTF_K_FUNCTION: 983 ctf_decl_push(cd, lc, t); 984 prec = CTF_PREC_FUNCTION; 985 break; 986 987 case CTF_K_POINTER: 988 ctf_decl_push(cd, lc, t); 989 prec = CTF_PREC_POINTER; 990 break; 991 992 case CTF_K_VOLATILE: 993 case CTF_K_CONST: 994 case CTF_K_RESTRICT: 995 ctf_decl_push(cd, lc, t); 996 prec = cd->cd_qualp; 997 is_qual++; 998 break; 999 1000 default: 1001 prec = CTF_PREC_BASE; 1002 } 1003 1004 cdp = malloc(sizeof(*cdp), M_FBT, M_WAITOK); 1005 cdp->cd_type = type; 1006 cdp->cd_kind = kind; 1007 cdp->cd_n = n; 1008 1009 if (ctf_list_next(&cd->cd_nodes[prec]) == NULL) 1010 cd->cd_order[prec] = cd->cd_ordp++; 1011 1012 /* 1013 * Reset cd_qualp to the highest precedence level that we've seen so 1014 * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER). 1015 */ 1016 if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY) 1017 cd->cd_qualp = prec; 1018 1019 /* 1020 * C array declarators are ordered inside out so prepend them. Also by 1021 * convention qualifiers of base types precede the type specifier (e.g. 1022 * const int vs. int const) even though the two forms are equivalent. 1023 */ 1024 if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE)) 1025 ctf_list_prepend(&cd->cd_nodes[prec], cdp); 1026 else 1027 ctf_list_append(&cd->cd_nodes[prec], cdp); 1028 } 1029 1030 static void 1031 ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...) 1032 { 1033 size_t len = (size_t)(cd->cd_end - cd->cd_ptr); 1034 va_list ap; 1035 size_t n; 1036 1037 va_start(ap, format); 1038 n = vsnprintf(cd->cd_ptr, len, format, ap); 1039 va_end(ap); 1040 1041 cd->cd_ptr += MIN(n, len); 1042 cd->cd_len += n; 1043 } 1044 1045 static ssize_t 1046 fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len) 1047 { 1048 ctf_decl_t cd; 1049 ctf_decl_node_t *cdp; 1050 ctf_decl_prec_t prec, lp, rp; 1051 int ptr, arr; 1052 uint_t k; 1053 1054 if (lc == NULL && type == CTF_ERR) 1055 return (-1); /* simplify caller code by permitting CTF_ERR */ 1056 1057 ctf_decl_init(&cd, buf, len); 1058 ctf_decl_push(&cd, lc, type); 1059 1060 if (cd.cd_err != 0) { 1061 ctf_decl_fini(&cd); 1062 return (-1); 1063 } 1064 1065 /* 1066 * If the type graph's order conflicts with lexical precedence order 1067 * for pointers or arrays, then we need to surround the declarations at 1068 * the corresponding lexical precedence with parentheses. This can 1069 * result in either a parenthesized pointer (*) as in int (*)() or 1070 * int (*)[], or in a parenthesized pointer and array as in int (*[])(). 1071 */ 1072 ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER; 1073 arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY; 1074 1075 rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1; 1076 lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1; 1077 1078 k = CTF_K_POINTER; /* avoid leading whitespace (see below) */ 1079 1080 for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) { 1081 for (cdp = ctf_list_next(&cd.cd_nodes[prec]); 1082 cdp != NULL; cdp = ctf_list_next(cdp)) { 1083 1084 const void *tp = ctf_lookup_by_id(lc, cdp->cd_type); 1085 const char *name = ctf_type_rname(lc, tp); 1086 1087 if (k != CTF_K_POINTER && k != CTF_K_ARRAY) 1088 ctf_decl_sprintf(&cd, " "); 1089 1090 if (lp == prec) { 1091 ctf_decl_sprintf(&cd, "("); 1092 lp = -1; 1093 } 1094 1095 switch (cdp->cd_kind) { 1096 case CTF_K_INTEGER: 1097 case CTF_K_FLOAT: 1098 case CTF_K_TYPEDEF: 1099 ctf_decl_sprintf(&cd, "%s", name); 1100 break; 1101 case CTF_K_POINTER: 1102 ctf_decl_sprintf(&cd, "*"); 1103 break; 1104 case CTF_K_ARRAY: 1105 ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n); 1106 break; 1107 case CTF_K_FUNCTION: 1108 ctf_decl_sprintf(&cd, "()"); 1109 break; 1110 case CTF_K_STRUCT: 1111 case CTF_K_FORWARD: 1112 ctf_decl_sprintf(&cd, "struct %s", name); 1113 break; 1114 case CTF_K_UNION: 1115 ctf_decl_sprintf(&cd, "union %s", name); 1116 break; 1117 case CTF_K_ENUM: 1118 ctf_decl_sprintf(&cd, "enum %s", name); 1119 break; 1120 case CTF_K_VOLATILE: 1121 ctf_decl_sprintf(&cd, "volatile"); 1122 break; 1123 case CTF_K_CONST: 1124 ctf_decl_sprintf(&cd, "const"); 1125 break; 1126 case CTF_K_RESTRICT: 1127 ctf_decl_sprintf(&cd, "restrict"); 1128 break; 1129 } 1130 1131 k = cdp->cd_kind; 1132 } 1133 1134 if (rp == prec) 1135 ctf_decl_sprintf(&cd, ")"); 1136 } 1137 1138 ctf_decl_fini(&cd); 1139 return (cd.cd_len); 1140 } 1141 1142 static void 1143 fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc) 1144 { 1145 const ctf_header_t *hp; 1146 const char *dp; 1147 fbt_probe_t *fbt = parg; 1148 linker_ctf_t lc; 1149 modctl_t *ctl = fbt->fbtp_ctl; 1150 size_t idwidth; 1151 int ndx = desc->dtargd_ndx; 1152 int symindx = fbt->fbtp_symindx; 1153 uint32_t *ctfoff; 1154 uint32_t offset, type; 1155 uint_t info, n; 1156 ushort_t kind; 1157 1158 if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) { 1159 (void) strcpy(desc->dtargd_native, "int"); 1160 return; 1161 } 1162 1163 desc->dtargd_ndx = DTRACE_ARGNONE; 1164 1165 /* Get a pointer to the CTF data and it's length. */ 1166 if (linker_ctf_get(ctl, &lc) != 0) 1167 /* No CTF data? Something wrong? *shrug* */ 1168 return; 1169 1170 /* Check if this module hasn't been initialised yet. */ 1171 if (*lc.ctfoffp == NULL) { 1172 /* 1173 * Initialise the CTF object and function symindx to 1174 * byte offset array. 1175 */ 1176 if (fbt_ctfoff_init(ctl, &lc) != 0) 1177 return; 1178 1179 /* Initialise the CTF type to byte offset array. */ 1180 if (fbt_typoff_init(&lc) != 0) 1181 return; 1182 } 1183 1184 ctfoff = *lc.ctfoffp; 1185 1186 if (ctfoff == NULL || *lc.typoffp == NULL) 1187 return; 1188 1189 /* Check if the symbol index is out of range. */ 1190 if (symindx >= lc.nsym) 1191 return; 1192 1193 /* Check if the symbol isn't cross-referenced. */ 1194 if ((offset = ctfoff[symindx]) == 0xffffffff) 1195 return; 1196 1197 hp = (const ctf_header_t *) lc.ctftab; 1198 idwidth = hp->cth_version == CTF_VERSION_2 ? 2 : 4; 1199 dp = (const char *)(lc.ctftab + offset + sizeof(ctf_header_t)); 1200 1201 info = 0; 1202 memcpy(&info, dp, idwidth); 1203 dp += idwidth; 1204 if (hp->cth_version == CTF_VERSION_2) { 1205 kind = CTF_V2_INFO_KIND(info); 1206 n = CTF_V2_INFO_VLEN(info); 1207 } else { 1208 kind = CTF_V3_INFO_KIND(info); 1209 n = CTF_V3_INFO_VLEN(info); 1210 } 1211 1212 if (kind == CTF_K_UNKNOWN && n == 0) { 1213 printf("%s(%d): Unknown function!\n",__func__,__LINE__); 1214 return; 1215 } 1216 1217 if (kind != CTF_K_FUNCTION) { 1218 printf("%s(%d): Expected a function!\n",__func__,__LINE__); 1219 return; 1220 } 1221 1222 if (fbt->fbtp_roffset != 0) { 1223 /* Only return type is available for args[1] in return probe. */ 1224 if (ndx > 1) 1225 return; 1226 ASSERT(ndx == 1); 1227 } else { 1228 /* Check if the requested argument doesn't exist. */ 1229 if (ndx >= n) 1230 return; 1231 1232 /* Skip the return type and arguments up to the one requested. */ 1233 dp += idwidth * (ndx + 1); 1234 } 1235 1236 type = 0; 1237 memcpy(&type, dp, idwidth); 1238 if (fbt_type_name(&lc, type, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0) 1239 desc->dtargd_ndx = ndx; 1240 } 1241 1242 static int 1243 fbt_linker_file_cb(linker_file_t lf, void *arg) 1244 { 1245 1246 fbt_provide_module(arg, lf); 1247 1248 return (0); 1249 } 1250 1251 static void 1252 fbt_load(void *dummy) 1253 { 1254 /* Default the probe table size if not specified. */ 1255 if (fbt_probetab_size == 0) 1256 fbt_probetab_size = FBT_PROBETAB_SIZE; 1257 1258 /* Choose the hash mask for the probe table. */ 1259 fbt_probetab_mask = fbt_probetab_size - 1; 1260 1261 /* Allocate memory for the probe table. */ 1262 fbt_probetab = 1263 malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO); 1264 1265 dtrace_doubletrap_func = fbt_doubletrap; 1266 dtrace_invop_add(fbt_invop); 1267 1268 if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER, 1269 NULL, &fbt_pops, NULL, &fbt_id) != 0) 1270 return; 1271 1272 /* Create probes for the kernel and already-loaded modules. */ 1273 linker_file_foreach(fbt_linker_file_cb, NULL); 1274 } 1275 1276 static int 1277 fbt_unload(void) 1278 { 1279 int error = 0; 1280 1281 /* De-register the invalid opcode handler. */ 1282 dtrace_invop_remove(fbt_invop); 1283 1284 dtrace_doubletrap_func = NULL; 1285 1286 /* De-register this DTrace provider. */ 1287 if ((error = dtrace_unregister(fbt_id)) != 0) 1288 return (error); 1289 1290 /* Free the probe table. */ 1291 free(fbt_probetab, M_FBT); 1292 fbt_probetab = NULL; 1293 fbt_probetab_mask = 0; 1294 1295 return (error); 1296 } 1297 1298 static int 1299 fbt_modevent(module_t mod __unused, int type, void *data __unused) 1300 { 1301 int error = 0; 1302 1303 switch (type) { 1304 case MOD_LOAD: 1305 break; 1306 1307 case MOD_UNLOAD: 1308 break; 1309 1310 case MOD_SHUTDOWN: 1311 break; 1312 1313 default: 1314 error = EOPNOTSUPP; 1315 break; 1316 1317 } 1318 1319 return (error); 1320 } 1321 1322 SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL); 1323 SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL); 1324 1325 DEV_MODULE(fbt, fbt_modevent, NULL); 1326 MODULE_VERSION(fbt, 1); 1327 MODULE_DEPEND(fbt, dtrace, 1, 1, 1); 1328 MODULE_DEPEND(fbt, opensolaris, 1, 1, 1); 1329