1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org 22 * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org 23 * Portions Copyright 2013 Howard Su howardsu@freebsd.org 24 * 25 * $FreeBSD$ 26 * 27 */ 28 29 /* 30 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 31 * Use is subject to license terms. 32 */ 33 34 #include <sys/cdefs.h> 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/conf.h> 38 #include <sys/cpuvar.h> 39 #include <sys/fcntl.h> 40 #include <sys/filio.h> 41 #include <sys/kdb.h> 42 #include <sys/kernel.h> 43 #include <sys/kmem.h> 44 #include <sys/kthread.h> 45 #include <sys/limits.h> 46 #include <sys/linker.h> 47 #include <sys/lock.h> 48 #include <sys/malloc.h> 49 #include <sys/module.h> 50 #include <sys/mutex.h> 51 #include <sys/pcpu.h> 52 #include <sys/poll.h> 53 #include <sys/proc.h> 54 #include <sys/selinfo.h> 55 #include <sys/smp.h> 56 #include <sys/syscall.h> 57 #include <sys/sysent.h> 58 #include <sys/sysproto.h> 59 #include <sys/uio.h> 60 #include <sys/unistd.h> 61 #include <machine/frame.h> 62 #include <machine/md_var.h> 63 #include <machine/stdarg.h> 64 65 #include <sys/dtrace.h> 66 #include <sys/dtrace_bsd.h> 67 68 static MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing"); 69 70 #define FBT_PATCHVAL 0xe06a0cfe // illegal instruction 71 72 #define FBT_PUSHM 0xe92d0000 73 #define FBT_POPM 0xe8bd0000 74 #define FBT_JUMP 0xea000000 75 76 static d_open_t fbt_open; 77 static int fbt_unload(void); 78 static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); 79 static void fbt_provide_module(void *, modctl_t *); 80 static void fbt_destroy(void *, dtrace_id_t, void *); 81 static void fbt_enable(void *, dtrace_id_t, void *); 82 static void fbt_disable(void *, dtrace_id_t, void *); 83 static void fbt_load(void *); 84 static void fbt_suspend(void *, dtrace_id_t, void *); 85 static void fbt_resume(void *, dtrace_id_t, void *); 86 87 #define FBT_ENTRY "entry" 88 #define FBT_RETURN "return" 89 #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) 90 #define FBT_PROBETAB_SIZE 0x8000 /* 32k entries -- 128K total */ 91 92 static struct cdevsw fbt_cdevsw = { 93 .d_version = D_VERSION, 94 .d_open = fbt_open, 95 .d_name = "fbt", 96 }; 97 98 static dtrace_pattr_t fbt_attr = { 99 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 100 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 101 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 102 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 103 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 104 }; 105 106 static dtrace_pops_t fbt_pops = { 107 NULL, 108 fbt_provide_module, 109 fbt_enable, 110 fbt_disable, 111 fbt_suspend, 112 fbt_resume, 113 fbt_getargdesc, 114 NULL, 115 NULL, 116 fbt_destroy 117 }; 118 119 typedef struct fbt_probe { 120 struct fbt_probe *fbtp_hashnext; 121 uint32_t *fbtp_patchpoint; 122 int8_t fbtp_rval; 123 uint32_t fbtp_patchval; 124 uint32_t fbtp_savedval; 125 uintptr_t fbtp_roffset; 126 dtrace_id_t fbtp_id; 127 const char *fbtp_name; 128 modctl_t *fbtp_ctl; 129 int fbtp_loadcnt; 130 int fbtp_primary; 131 int fbtp_invop_cnt; 132 int fbtp_symindx; 133 struct fbt_probe *fbtp_next; 134 } fbt_probe_t; 135 136 static struct cdev *fbt_cdev; 137 static dtrace_provider_id_t fbt_id; 138 static fbt_probe_t **fbt_probetab; 139 static int fbt_probetab_size; 140 static int fbt_probetab_mask; 141 static int fbt_verbose = 0; 142 143 static int 144 fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval) 145 { 146 struct trapframe *frame = (struct trapframe *)stack; 147 solaris_cpu_t *cpu = &solaris_cpu[curcpu]; 148 fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; 149 150 for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { 151 if ((uintptr_t)fbt->fbtp_patchpoint == addr) { 152 fbt->fbtp_invop_cnt++; 153 cpu->cpu_dtrace_caller = addr; 154 155 dtrace_probe(fbt->fbtp_id, frame->tf_r0, 156 frame->tf_r1, frame->tf_r2, 157 frame->tf_r3, 0); // TODO: Need 5th parameter from stack 158 159 cpu->cpu_dtrace_caller = 0; 160 161 return (fbt->fbtp_rval); 162 } 163 } 164 165 return (0); 166 } 167 168 static int 169 fbt_provide_module_function(linker_file_t lf, int symindx, 170 linker_symval_t *symval, void *opaque) 171 { 172 char *modname = opaque; 173 const char *name = symval->name; 174 fbt_probe_t *fbt, *retfbt; 175 int popm; 176 u_int32_t *instr, *limit; 177 178 if (strncmp(name, "dtrace_", 7) == 0 && 179 strncmp(name, "dtrace_safe_", 12) != 0) { 180 /* 181 * Anything beginning with "dtrace_" may be called 182 * from probe context unless it explicitly indicates 183 * that it won't be called from probe context by 184 * using the prefix "dtrace_safe_". 185 */ 186 return (0); 187 } 188 189 if (name[0] == '_' && name[1] == '_') 190 return (0); 191 192 instr = (u_int32_t *) symval->value; 193 limit = (u_int32_t *)(symval->value + symval->size); 194 195 for (; instr < limit; instr++) 196 if ((*instr & 0xffff0000) == FBT_PUSHM && (*instr & 0x4000) != 0) 197 break; 198 199 if (instr >= limit) 200 return (0); 201 202 fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); 203 fbt->fbtp_name = name; 204 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, 205 name, FBT_ENTRY, 3, fbt); 206 fbt->fbtp_patchpoint = instr; 207 fbt->fbtp_ctl = lf; 208 fbt->fbtp_loadcnt = lf->loadcnt; 209 fbt->fbtp_savedval = *instr; 210 fbt->fbtp_patchval = FBT_PATCHVAL; 211 fbt->fbtp_rval = DTRACE_INVOP_PUSHM; 212 fbt->fbtp_symindx = symindx; 213 214 fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; 215 fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; 216 217 lf->fbt_nentries++; 218 219 popm = FBT_POPM | ((*instr) & 0x3FFF) | 0x8000; 220 221 222 retfbt = NULL; 223 again: 224 for(; instr < limit; instr++) 225 { 226 if (*instr == popm) 227 break; 228 else if ((*instr & 0xff000000) == FBT_JUMP) 229 { 230 int offset; 231 u_int32_t *target, *start; 232 offset = (*instr & 0xffffff); 233 offset <<= 8; 234 offset /= 64; 235 target = instr + (2 + offset); 236 start = (u_int32_t *) symval->value; 237 if (target >= limit || target < start) 238 break; 239 instr++; //skip delay slot 240 } 241 } 242 243 if (instr >= limit) 244 return (0); 245 246 /* 247 * We have a winner! 248 */ 249 fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); 250 fbt->fbtp_name = name; 251 if (retfbt == NULL) { 252 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, 253 name, FBT_RETURN, 5, fbt); 254 } else { 255 retfbt->fbtp_next = fbt; 256 fbt->fbtp_id = retfbt->fbtp_id; 257 } 258 retfbt = fbt; 259 260 fbt->fbtp_patchpoint = instr; 261 fbt->fbtp_ctl = lf; 262 fbt->fbtp_loadcnt = lf->loadcnt; 263 fbt->fbtp_symindx = symindx; 264 if ((*instr & 0xff000000) == FBT_JUMP) 265 fbt->fbtp_rval = DTRACE_INVOP_B; 266 else 267 fbt->fbtp_rval = DTRACE_INVOP_POPM; 268 fbt->fbtp_savedval = *instr; 269 fbt->fbtp_patchval = FBT_PATCHVAL; 270 fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; 271 fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; 272 273 lf->fbt_nentries++; 274 275 instr++; 276 goto again; 277 } 278 279 static void 280 fbt_provide_module(void *arg, modctl_t *lf) 281 { 282 char modname[MAXPATHLEN]; 283 int i; 284 size_t len; 285 286 strlcpy(modname, lf->filename, sizeof(modname)); 287 len = strlen(modname); 288 if (len > 3 && strcmp(modname + len - 3, ".ko") == 0) 289 modname[len - 3] = '\0'; 290 291 /* 292 * Employees of dtrace and their families are ineligible. Void 293 * where prohibited. 294 */ 295 if (strcmp(modname, "dtrace") == 0) 296 return; 297 298 /* 299 * The cyclic timer subsystem can be built as a module and DTrace 300 * depends on that, so it is ineligible too. 301 */ 302 if (strcmp(modname, "cyclic") == 0) 303 return; 304 305 /* 306 * To register with DTrace, a module must list 'dtrace' as a 307 * dependency in order for the kernel linker to resolve 308 * symbols like dtrace_register(). All modules with such a 309 * dependency are ineligible for FBT tracing. 310 */ 311 for (i = 0; i < lf->ndeps; i++) 312 if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0) 313 return; 314 315 if (lf->fbt_nentries) { 316 /* 317 * This module has some FBT entries allocated; we're afraid 318 * to screw with it. 319 */ 320 return; 321 } 322 323 /* 324 * List the functions in the module and the symbol values. 325 */ 326 (void) linker_file_function_listall(lf, fbt_provide_module_function, modname); 327 } 328 329 static void 330 fbt_destroy(void *arg, dtrace_id_t id, void *parg) 331 { 332 fbt_probe_t *fbt = parg, *next, *hash, *last; 333 modctl_t *ctl; 334 int ndx; 335 336 do { 337 ctl = fbt->fbtp_ctl; 338 339 ctl->fbt_nentries--; 340 341 /* 342 * Now we need to remove this probe from the fbt_probetab. 343 */ 344 ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint); 345 last = NULL; 346 hash = fbt_probetab[ndx]; 347 348 while (hash != fbt) { 349 ASSERT(hash != NULL); 350 last = hash; 351 hash = hash->fbtp_hashnext; 352 } 353 354 if (last != NULL) { 355 last->fbtp_hashnext = fbt->fbtp_hashnext; 356 } else { 357 fbt_probetab[ndx] = fbt->fbtp_hashnext; 358 } 359 360 next = fbt->fbtp_next; 361 free(fbt, M_FBT); 362 363 fbt = next; 364 } while (fbt != NULL); 365 } 366 367 static void 368 fbt_enable(void *arg, dtrace_id_t id, void *parg) 369 { 370 fbt_probe_t *fbt = parg; 371 modctl_t *ctl = fbt->fbtp_ctl; 372 373 ctl->nenabled++; 374 375 /* 376 * Now check that our modctl has the expected load count. If it 377 * doesn't, this module must have been unloaded and reloaded -- and 378 * we're not going to touch it. 379 */ 380 if (ctl->loadcnt != fbt->fbtp_loadcnt) { 381 if (fbt_verbose) { 382 printf("fbt is failing for probe %s " 383 "(module %s reloaded)", 384 fbt->fbtp_name, ctl->filename); 385 } 386 387 return; 388 } 389 390 for (; fbt != NULL; fbt = fbt->fbtp_next) { 391 *fbt->fbtp_patchpoint = fbt->fbtp_patchval; 392 cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); 393 } 394 } 395 396 static void 397 fbt_disable(void *arg, dtrace_id_t id, void *parg) 398 { 399 fbt_probe_t *fbt = parg; 400 modctl_t *ctl = fbt->fbtp_ctl; 401 402 ASSERT(ctl->nenabled > 0); 403 ctl->nenabled--; 404 405 if ((ctl->loadcnt != fbt->fbtp_loadcnt)) 406 return; 407 408 for (; fbt != NULL; fbt = fbt->fbtp_next) { 409 *fbt->fbtp_patchpoint = fbt->fbtp_savedval; 410 cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); 411 } 412 } 413 414 static void 415 fbt_suspend(void *arg, dtrace_id_t id, void *parg) 416 { 417 fbt_probe_t *fbt = parg; 418 modctl_t *ctl = fbt->fbtp_ctl; 419 420 ASSERT(ctl->nenabled > 0); 421 422 if ((ctl->loadcnt != fbt->fbtp_loadcnt)) 423 return; 424 425 for (; fbt != NULL; fbt = fbt->fbtp_next) { 426 *fbt->fbtp_patchpoint = fbt->fbtp_savedval; 427 cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); 428 } 429 } 430 431 static void 432 fbt_resume(void *arg, dtrace_id_t id, void *parg) 433 { 434 fbt_probe_t *fbt = parg; 435 modctl_t *ctl = fbt->fbtp_ctl; 436 437 ASSERT(ctl->nenabled > 0); 438 439 if ((ctl->loadcnt != fbt->fbtp_loadcnt)) 440 return; 441 442 for (; fbt != NULL; fbt = fbt->fbtp_next) { 443 *fbt->fbtp_patchpoint = fbt->fbtp_patchval; 444 cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); 445 } 446 } 447 448 static int 449 fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc) 450 { 451 const Elf_Sym *symp = lc->symtab;; 452 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 453 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); 454 int i; 455 uint32_t *ctfoff; 456 uint32_t objtoff = hp->cth_objtoff; 457 uint32_t funcoff = hp->cth_funcoff; 458 ushort_t info; 459 ushort_t vlen; 460 461 /* Sanity check. */ 462 if (hp->cth_magic != CTF_MAGIC) { 463 printf("Bad magic value in CTF data of '%s'\n",lf->pathname); 464 return (EINVAL); 465 } 466 467 if (lc->symtab == NULL) { 468 printf("No symbol table in '%s'\n",lf->pathname); 469 return (EINVAL); 470 } 471 472 if ((ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK)) == NULL) 473 return (ENOMEM); 474 475 *lc->ctfoffp = ctfoff; 476 477 for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) { 478 if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) { 479 *ctfoff = 0xffffffff; 480 continue; 481 } 482 483 switch (ELF_ST_TYPE(symp->st_info)) { 484 case STT_OBJECT: 485 if (objtoff >= hp->cth_funcoff || 486 (symp->st_shndx == SHN_ABS && symp->st_value == 0)) { 487 *ctfoff = 0xffffffff; 488 break; 489 } 490 491 *ctfoff = objtoff; 492 objtoff += sizeof (ushort_t); 493 break; 494 495 case STT_FUNC: 496 if (funcoff >= hp->cth_typeoff) { 497 *ctfoff = 0xffffffff; 498 break; 499 } 500 501 *ctfoff = funcoff; 502 503 info = *((const ushort_t *)(ctfdata + funcoff)); 504 vlen = CTF_INFO_VLEN(info); 505 506 /* 507 * If we encounter a zero pad at the end, just skip it. 508 * Otherwise skip over the function and its return type 509 * (+2) and the argument list (vlen). 510 */ 511 if (CTF_INFO_KIND(info) == CTF_K_UNKNOWN && vlen == 0) 512 funcoff += sizeof (ushort_t); /* skip pad */ 513 else 514 funcoff += sizeof (ushort_t) * (vlen + 2); 515 break; 516 517 default: 518 *ctfoff = 0xffffffff; 519 break; 520 } 521 } 522 523 return (0); 524 } 525 526 static ssize_t 527 fbt_get_ctt_size(uint8_t version, const ctf_type_t *tp, ssize_t *sizep, 528 ssize_t *incrementp) 529 { 530 ssize_t size, increment; 531 532 if (version > CTF_VERSION_1 && 533 tp->ctt_size == CTF_LSIZE_SENT) { 534 size = CTF_TYPE_LSIZE(tp); 535 increment = sizeof (ctf_type_t); 536 } else { 537 size = tp->ctt_size; 538 increment = sizeof (ctf_stype_t); 539 } 540 541 if (sizep) 542 *sizep = size; 543 if (incrementp) 544 *incrementp = increment; 545 546 return (size); 547 } 548 549 static int 550 fbt_typoff_init(linker_ctf_t *lc) 551 { 552 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 553 const ctf_type_t *tbuf; 554 const ctf_type_t *tend; 555 const ctf_type_t *tp; 556 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); 557 int ctf_typemax = 0; 558 uint32_t *xp; 559 ulong_t pop[CTF_K_MAX + 1] = { 0 }; 560 561 562 /* Sanity check. */ 563 if (hp->cth_magic != CTF_MAGIC) 564 return (EINVAL); 565 566 tbuf = (const ctf_type_t *) (ctfdata + hp->cth_typeoff); 567 tend = (const ctf_type_t *) (ctfdata + hp->cth_stroff); 568 569 int child = hp->cth_parname != 0; 570 571 /* 572 * We make two passes through the entire type section. In this first 573 * pass, we count the number of each type and the total number of types. 574 */ 575 for (tp = tbuf; tp < tend; ctf_typemax++) { 576 ushort_t kind = CTF_INFO_KIND(tp->ctt_info); 577 ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info); 578 ssize_t size, increment; 579 580 size_t vbytes; 581 uint_t n; 582 583 (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment); 584 585 switch (kind) { 586 case CTF_K_INTEGER: 587 case CTF_K_FLOAT: 588 vbytes = sizeof (uint_t); 589 break; 590 case CTF_K_ARRAY: 591 vbytes = sizeof (ctf_array_t); 592 break; 593 case CTF_K_FUNCTION: 594 vbytes = sizeof (ushort_t) * (vlen + (vlen & 1)); 595 break; 596 case CTF_K_STRUCT: 597 case CTF_K_UNION: 598 if (size < CTF_LSTRUCT_THRESH) { 599 ctf_member_t *mp = (ctf_member_t *) 600 ((uintptr_t)tp + increment); 601 602 vbytes = sizeof (ctf_member_t) * vlen; 603 for (n = vlen; n != 0; n--, mp++) 604 child |= CTF_TYPE_ISCHILD(mp->ctm_type); 605 } else { 606 ctf_lmember_t *lmp = (ctf_lmember_t *) 607 ((uintptr_t)tp + increment); 608 609 vbytes = sizeof (ctf_lmember_t) * vlen; 610 for (n = vlen; n != 0; n--, lmp++) 611 child |= 612 CTF_TYPE_ISCHILD(lmp->ctlm_type); 613 } 614 break; 615 case CTF_K_ENUM: 616 vbytes = sizeof (ctf_enum_t) * vlen; 617 break; 618 case CTF_K_FORWARD: 619 /* 620 * For forward declarations, ctt_type is the CTF_K_* 621 * kind for the tag, so bump that population count too. 622 * If ctt_type is unknown, treat the tag as a struct. 623 */ 624 if (tp->ctt_type == CTF_K_UNKNOWN || 625 tp->ctt_type >= CTF_K_MAX) 626 pop[CTF_K_STRUCT]++; 627 else 628 pop[tp->ctt_type]++; 629 /*FALLTHRU*/ 630 case CTF_K_UNKNOWN: 631 vbytes = 0; 632 break; 633 case CTF_K_POINTER: 634 case CTF_K_TYPEDEF: 635 case CTF_K_VOLATILE: 636 case CTF_K_CONST: 637 case CTF_K_RESTRICT: 638 child |= CTF_TYPE_ISCHILD(tp->ctt_type); 639 vbytes = 0; 640 break; 641 default: 642 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); 643 return (EIO); 644 } 645 tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes); 646 pop[kind]++; 647 } 648 649 /* account for a sentinel value below */ 650 ctf_typemax++; 651 *lc->typlenp = ctf_typemax; 652 653 if ((xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER, M_ZERO | M_WAITOK)) == NULL) 654 return (ENOMEM); 655 656 *lc->typoffp = xp; 657 658 /* type id 0 is used as a sentinel value */ 659 *xp++ = 0; 660 661 /* 662 * In the second pass, fill in the type offset. 663 */ 664 for (tp = tbuf; tp < tend; xp++) { 665 ushort_t kind = CTF_INFO_KIND(tp->ctt_info); 666 ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info); 667 ssize_t size, increment; 668 669 size_t vbytes; 670 uint_t n; 671 672 (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment); 673 674 switch (kind) { 675 case CTF_K_INTEGER: 676 case CTF_K_FLOAT: 677 vbytes = sizeof (uint_t); 678 break; 679 case CTF_K_ARRAY: 680 vbytes = sizeof (ctf_array_t); 681 break; 682 case CTF_K_FUNCTION: 683 vbytes = sizeof (ushort_t) * (vlen + (vlen & 1)); 684 break; 685 case CTF_K_STRUCT: 686 case CTF_K_UNION: 687 if (size < CTF_LSTRUCT_THRESH) { 688 ctf_member_t *mp = (ctf_member_t *) 689 ((uintptr_t)tp + increment); 690 691 vbytes = sizeof (ctf_member_t) * vlen; 692 for (n = vlen; n != 0; n--, mp++) 693 child |= CTF_TYPE_ISCHILD(mp->ctm_type); 694 } else { 695 ctf_lmember_t *lmp = (ctf_lmember_t *) 696 ((uintptr_t)tp + increment); 697 698 vbytes = sizeof (ctf_lmember_t) * vlen; 699 for (n = vlen; n != 0; n--, lmp++) 700 child |= 701 CTF_TYPE_ISCHILD(lmp->ctlm_type); 702 } 703 break; 704 case CTF_K_ENUM: 705 vbytes = sizeof (ctf_enum_t) * vlen; 706 break; 707 case CTF_K_FORWARD: 708 case CTF_K_UNKNOWN: 709 vbytes = 0; 710 break; 711 case CTF_K_POINTER: 712 case CTF_K_TYPEDEF: 713 case CTF_K_VOLATILE: 714 case CTF_K_CONST: 715 case CTF_K_RESTRICT: 716 vbytes = 0; 717 break; 718 default: 719 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); 720 return (EIO); 721 } 722 *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata); 723 tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes); 724 } 725 726 return (0); 727 } 728 729 /* 730 * CTF Declaration Stack 731 * 732 * In order to implement ctf_type_name(), we must convert a type graph back 733 * into a C type declaration. Unfortunately, a type graph represents a storage 734 * class ordering of the type whereas a type declaration must obey the C rules 735 * for operator precedence, and the two orderings are frequently in conflict. 736 * For example, consider these CTF type graphs and their C declarations: 737 * 738 * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)() 739 * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[] 740 * 741 * In each case, parentheses are used to raise operator * to higher lexical 742 * precedence, so the string form of the C declaration cannot be constructed by 743 * walking the type graph links and forming the string from left to right. 744 * 745 * The functions in this file build a set of stacks from the type graph nodes 746 * corresponding to the C operator precedence levels in the appropriate order. 747 * The code in ctf_type_name() can then iterate over the levels and nodes in 748 * lexical precedence order and construct the final C declaration string. 749 */ 750 typedef struct ctf_list { 751 struct ctf_list *l_prev; /* previous pointer or tail pointer */ 752 struct ctf_list *l_next; /* next pointer or head pointer */ 753 } ctf_list_t; 754 755 #define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev)) 756 #define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next)) 757 758 typedef enum { 759 CTF_PREC_BASE, 760 CTF_PREC_POINTER, 761 CTF_PREC_ARRAY, 762 CTF_PREC_FUNCTION, 763 CTF_PREC_MAX 764 } ctf_decl_prec_t; 765 766 typedef struct ctf_decl_node { 767 ctf_list_t cd_list; /* linked list pointers */ 768 ctf_id_t cd_type; /* type identifier */ 769 uint_t cd_kind; /* type kind */ 770 uint_t cd_n; /* type dimension if array */ 771 } ctf_decl_node_t; 772 773 typedef struct ctf_decl { 774 ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */ 775 int cd_order[CTF_PREC_MAX]; /* storage order of decls */ 776 ctf_decl_prec_t cd_qualp; /* qualifier precision */ 777 ctf_decl_prec_t cd_ordp; /* ordered precision */ 778 char *cd_buf; /* buffer for output */ 779 char *cd_ptr; /* buffer location */ 780 char *cd_end; /* buffer limit */ 781 size_t cd_len; /* buffer space required */ 782 int cd_err; /* saved error value */ 783 } ctf_decl_t; 784 785 /* 786 * Simple doubly-linked list append routine. This implementation assumes that 787 * each list element contains an embedded ctf_list_t as the first member. 788 * An additional ctf_list_t is used to store the head (l_next) and tail 789 * (l_prev) pointers. The current head and tail list elements have their 790 * previous and next pointers set to NULL, respectively. 791 */ 792 static void 793 ctf_list_append(ctf_list_t *lp, void *new) 794 { 795 ctf_list_t *p = lp->l_prev; /* p = tail list element */ 796 ctf_list_t *q = new; /* q = new list element */ 797 798 lp->l_prev = q; 799 q->l_prev = p; 800 q->l_next = NULL; 801 802 if (p != NULL) 803 p->l_next = q; 804 else 805 lp->l_next = q; 806 } 807 808 /* 809 * Prepend the specified existing element to the given ctf_list_t. The 810 * existing pointer should be pointing at a struct with embedded ctf_list_t. 811 */ 812 static void 813 ctf_list_prepend(ctf_list_t *lp, void *new) 814 { 815 ctf_list_t *p = new; /* p = new list element */ 816 ctf_list_t *q = lp->l_next; /* q = head list element */ 817 818 lp->l_next = p; 819 p->l_prev = NULL; 820 p->l_next = q; 821 822 if (q != NULL) 823 q->l_prev = p; 824 else 825 lp->l_prev = p; 826 } 827 828 static void 829 ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len) 830 { 831 int i; 832 833 bzero(cd, sizeof (ctf_decl_t)); 834 835 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) 836 cd->cd_order[i] = CTF_PREC_BASE - 1; 837 838 cd->cd_qualp = CTF_PREC_BASE; 839 cd->cd_ordp = CTF_PREC_BASE; 840 841 cd->cd_buf = buf; 842 cd->cd_ptr = buf; 843 cd->cd_end = buf + len; 844 } 845 846 static void 847 ctf_decl_fini(ctf_decl_t *cd) 848 { 849 ctf_decl_node_t *cdp, *ndp; 850 int i; 851 852 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) { 853 for (cdp = ctf_list_next(&cd->cd_nodes[i]); 854 cdp != NULL; cdp = ndp) { 855 ndp = ctf_list_next(cdp); 856 free(cdp, M_FBT); 857 } 858 } 859 } 860 861 static const ctf_type_t * 862 ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type) 863 { 864 const ctf_type_t *tp; 865 uint32_t offset; 866 uint32_t *typoff = *lc->typoffp; 867 868 if (type >= *lc->typlenp) { 869 printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp); 870 return(NULL); 871 } 872 873 /* Check if the type isn't cross-referenced. */ 874 if ((offset = typoff[type]) == 0) { 875 printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type); 876 return(NULL); 877 } 878 879 tp = (const ctf_type_t *)(lc->ctftab + offset + sizeof(ctf_header_t)); 880 881 return (tp); 882 } 883 884 static void 885 fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp) 886 { 887 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; 888 const ctf_type_t *tp; 889 const ctf_array_t *ap; 890 ssize_t increment; 891 892 bzero(arp, sizeof(*arp)); 893 894 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) 895 return; 896 897 if (CTF_INFO_KIND(tp->ctt_info) != CTF_K_ARRAY) 898 return; 899 900 (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment); 901 902 ap = (const ctf_array_t *)((uintptr_t)tp + increment); 903 arp->ctr_contents = ap->cta_contents; 904 arp->ctr_index = ap->cta_index; 905 arp->ctr_nelems = ap->cta_nelems; 906 } 907 908 static const char * 909 ctf_strptr(linker_ctf_t *lc, int name) 910 { 911 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;; 912 const char *strp = ""; 913 914 if (name < 0 || name >= hp->cth_strlen) 915 return (strp); 916 917 strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t)); 918 919 return (strp); 920 } 921 922 static void 923 ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type) 924 { 925 ctf_decl_node_t *cdp; 926 ctf_decl_prec_t prec; 927 uint_t kind, n = 1; 928 int is_qual = 0; 929 930 const ctf_type_t *tp; 931 ctf_arinfo_t ar; 932 933 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) { 934 cd->cd_err = ENOENT; 935 return; 936 } 937 938 switch (kind = CTF_INFO_KIND(tp->ctt_info)) { 939 case CTF_K_ARRAY: 940 fbt_array_info(lc, type, &ar); 941 ctf_decl_push(cd, lc, ar.ctr_contents); 942 n = ar.ctr_nelems; 943 prec = CTF_PREC_ARRAY; 944 break; 945 946 case CTF_K_TYPEDEF: 947 if (ctf_strptr(lc, tp->ctt_name)[0] == '\0') { 948 ctf_decl_push(cd, lc, tp->ctt_type); 949 return; 950 } 951 prec = CTF_PREC_BASE; 952 break; 953 954 case CTF_K_FUNCTION: 955 ctf_decl_push(cd, lc, tp->ctt_type); 956 prec = CTF_PREC_FUNCTION; 957 break; 958 959 case CTF_K_POINTER: 960 ctf_decl_push(cd, lc, tp->ctt_type); 961 prec = CTF_PREC_POINTER; 962 break; 963 964 case CTF_K_VOLATILE: 965 case CTF_K_CONST: 966 case CTF_K_RESTRICT: 967 ctf_decl_push(cd, lc, tp->ctt_type); 968 prec = cd->cd_qualp; 969 is_qual++; 970 break; 971 972 default: 973 prec = CTF_PREC_BASE; 974 } 975 976 if ((cdp = malloc(sizeof (ctf_decl_node_t), M_FBT, M_WAITOK)) == NULL) { 977 cd->cd_err = EAGAIN; 978 return; 979 } 980 981 cdp->cd_type = type; 982 cdp->cd_kind = kind; 983 cdp->cd_n = n; 984 985 if (ctf_list_next(&cd->cd_nodes[prec]) == NULL) 986 cd->cd_order[prec] = cd->cd_ordp++; 987 988 /* 989 * Reset cd_qualp to the highest precedence level that we've seen so 990 * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER). 991 */ 992 if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY) 993 cd->cd_qualp = prec; 994 995 /* 996 * C array declarators are ordered inside out so prepend them. Also by 997 * convention qualifiers of base types precede the type specifier (e.g. 998 * const int vs. int const) even though the two forms are equivalent. 999 */ 1000 if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE)) 1001 ctf_list_prepend(&cd->cd_nodes[prec], cdp); 1002 else 1003 ctf_list_append(&cd->cd_nodes[prec], cdp); 1004 } 1005 1006 static void 1007 ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...) 1008 { 1009 size_t len = (size_t)(cd->cd_end - cd->cd_ptr); 1010 va_list ap; 1011 size_t n; 1012 1013 va_start(ap, format); 1014 n = vsnprintf(cd->cd_ptr, len, format, ap); 1015 va_end(ap); 1016 1017 cd->cd_ptr += MIN(n, len); 1018 cd->cd_len += n; 1019 } 1020 1021 static ssize_t 1022 fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len) 1023 { 1024 ctf_decl_t cd; 1025 ctf_decl_node_t *cdp; 1026 ctf_decl_prec_t prec, lp, rp; 1027 int ptr, arr; 1028 uint_t k; 1029 1030 if (lc == NULL && type == CTF_ERR) 1031 return (-1); /* simplify caller code by permitting CTF_ERR */ 1032 1033 ctf_decl_init(&cd, buf, len); 1034 ctf_decl_push(&cd, lc, type); 1035 1036 if (cd.cd_err != 0) { 1037 ctf_decl_fini(&cd); 1038 return (-1); 1039 } 1040 1041 /* 1042 * If the type graph's order conflicts with lexical precedence order 1043 * for pointers or arrays, then we need to surround the declarations at 1044 * the corresponding lexical precedence with parentheses. This can 1045 * result in either a parenthesized pointer (*) as in int (*)() or 1046 * int (*)[], or in a parenthesized pointer and array as in int (*[])(). 1047 */ 1048 ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER; 1049 arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY; 1050 1051 rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1; 1052 lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1; 1053 1054 k = CTF_K_POINTER; /* avoid leading whitespace (see below) */ 1055 1056 for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) { 1057 for (cdp = ctf_list_next(&cd.cd_nodes[prec]); 1058 cdp != NULL; cdp = ctf_list_next(cdp)) { 1059 1060 const ctf_type_t *tp = 1061 ctf_lookup_by_id(lc, cdp->cd_type); 1062 const char *name = ctf_strptr(lc, tp->ctt_name); 1063 1064 if (k != CTF_K_POINTER && k != CTF_K_ARRAY) 1065 ctf_decl_sprintf(&cd, " "); 1066 1067 if (lp == prec) { 1068 ctf_decl_sprintf(&cd, "("); 1069 lp = -1; 1070 } 1071 1072 switch (cdp->cd_kind) { 1073 case CTF_K_INTEGER: 1074 case CTF_K_FLOAT: 1075 case CTF_K_TYPEDEF: 1076 ctf_decl_sprintf(&cd, "%s", name); 1077 break; 1078 case CTF_K_POINTER: 1079 ctf_decl_sprintf(&cd, "*"); 1080 break; 1081 case CTF_K_ARRAY: 1082 ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n); 1083 break; 1084 case CTF_K_FUNCTION: 1085 ctf_decl_sprintf(&cd, "()"); 1086 break; 1087 case CTF_K_STRUCT: 1088 case CTF_K_FORWARD: 1089 ctf_decl_sprintf(&cd, "struct %s", name); 1090 break; 1091 case CTF_K_UNION: 1092 ctf_decl_sprintf(&cd, "union %s", name); 1093 break; 1094 case CTF_K_ENUM: 1095 ctf_decl_sprintf(&cd, "enum %s", name); 1096 break; 1097 case CTF_K_VOLATILE: 1098 ctf_decl_sprintf(&cd, "volatile"); 1099 break; 1100 case CTF_K_CONST: 1101 ctf_decl_sprintf(&cd, "const"); 1102 break; 1103 case CTF_K_RESTRICT: 1104 ctf_decl_sprintf(&cd, "restrict"); 1105 break; 1106 } 1107 1108 k = cdp->cd_kind; 1109 } 1110 1111 if (rp == prec) 1112 ctf_decl_sprintf(&cd, ")"); 1113 } 1114 1115 ctf_decl_fini(&cd); 1116 return (cd.cd_len); 1117 } 1118 1119 static void 1120 fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc) 1121 { 1122 const ushort_t *dp; 1123 fbt_probe_t *fbt = parg; 1124 linker_ctf_t lc; 1125 modctl_t *ctl = fbt->fbtp_ctl; 1126 int ndx = desc->dtargd_ndx; 1127 int symindx = fbt->fbtp_symindx; 1128 uint32_t *ctfoff; 1129 uint32_t offset; 1130 ushort_t info, kind, n; 1131 1132 if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) { 1133 (void) strcpy(desc->dtargd_native, "int"); 1134 return; 1135 } 1136 1137 desc->dtargd_ndx = DTRACE_ARGNONE; 1138 1139 /* Get a pointer to the CTF data and it's length. */ 1140 if (linker_ctf_get(ctl, &lc) != 0) 1141 /* No CTF data? Something wrong? *shrug* */ 1142 return; 1143 1144 /* Check if this module hasn't been initialised yet. */ 1145 if (*lc.ctfoffp == NULL) { 1146 /* 1147 * Initialise the CTF object and function symindx to 1148 * byte offset array. 1149 */ 1150 if (fbt_ctfoff_init(ctl, &lc) != 0) 1151 return; 1152 1153 /* Initialise the CTF type to byte offset array. */ 1154 if (fbt_typoff_init(&lc) != 0) 1155 return; 1156 } 1157 1158 ctfoff = *lc.ctfoffp; 1159 1160 if (ctfoff == NULL || *lc.typoffp == NULL) 1161 return; 1162 1163 /* Check if the symbol index is out of range. */ 1164 if (symindx >= lc.nsym) 1165 return; 1166 1167 /* Check if the symbol isn't cross-referenced. */ 1168 if ((offset = ctfoff[symindx]) == 0xffffffff) 1169 return; 1170 1171 dp = (const ushort_t *)(lc.ctftab + offset + sizeof(ctf_header_t)); 1172 1173 info = *dp++; 1174 kind = CTF_INFO_KIND(info); 1175 n = CTF_INFO_VLEN(info); 1176 1177 if (kind == CTF_K_UNKNOWN && n == 0) { 1178 printf("%s(%d): Unknown function!\n",__func__,__LINE__); 1179 return; 1180 } 1181 1182 if (kind != CTF_K_FUNCTION) { 1183 printf("%s(%d): Expected a function!\n",__func__,__LINE__); 1184 return; 1185 } 1186 1187 if (fbt->fbtp_roffset != 0) { 1188 /* Only return type is available for args[1] in return probe. */ 1189 if (ndx > 1) 1190 return; 1191 ASSERT(ndx == 1); 1192 } else { 1193 /* Check if the requested argument doesn't exist. */ 1194 if (ndx >= n) 1195 return; 1196 1197 /* Skip the return type and arguments up to the one requested. */ 1198 dp += ndx + 1; 1199 } 1200 1201 if (fbt_type_name(&lc, *dp, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0) 1202 desc->dtargd_ndx = ndx; 1203 1204 return; 1205 } 1206 1207 static int 1208 fbt_linker_file_cb(linker_file_t lf, void *arg) 1209 { 1210 1211 fbt_provide_module(arg, lf); 1212 1213 return (0); 1214 } 1215 1216 static void 1217 fbt_load(void *dummy) 1218 { 1219 /* Create the /dev/dtrace/fbt entry. */ 1220 fbt_cdev = make_dev(&fbt_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 1221 "dtrace/fbt"); 1222 1223 /* Default the probe table size if not specified. */ 1224 if (fbt_probetab_size == 0) 1225 fbt_probetab_size = FBT_PROBETAB_SIZE; 1226 1227 /* Choose the hash mask for the probe table. */ 1228 fbt_probetab_mask = fbt_probetab_size - 1; 1229 1230 /* Allocate memory for the probe table. */ 1231 fbt_probetab = 1232 malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO); 1233 1234 dtrace_invop_add(fbt_invop); 1235 1236 if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER, 1237 NULL, &fbt_pops, NULL, &fbt_id) != 0) 1238 return; 1239 1240 /* Create probes for the kernel and already-loaded modules. */ 1241 linker_file_foreach(fbt_linker_file_cb, NULL); 1242 } 1243 1244 1245 static int 1246 fbt_unload() 1247 { 1248 int error = 0; 1249 1250 /* De-register the invalid opcode handler. */ 1251 dtrace_invop_remove(fbt_invop); 1252 1253 /* De-register this DTrace provider. */ 1254 if ((error = dtrace_unregister(fbt_id)) != 0) 1255 return (error); 1256 1257 /* Free the probe table. */ 1258 free(fbt_probetab, M_FBT); 1259 fbt_probetab = NULL; 1260 fbt_probetab_mask = 0; 1261 1262 destroy_dev(fbt_cdev); 1263 1264 return (error); 1265 } 1266 1267 static int 1268 fbt_modevent(module_t mod __unused, int type, void *data __unused) 1269 { 1270 int error = 0; 1271 1272 switch (type) { 1273 case MOD_LOAD: 1274 break; 1275 1276 case MOD_UNLOAD: 1277 break; 1278 1279 case MOD_SHUTDOWN: 1280 break; 1281 1282 default: 1283 error = EOPNOTSUPP; 1284 break; 1285 1286 } 1287 1288 return (error); 1289 } 1290 1291 static int 1292 fbt_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) 1293 { 1294 return (0); 1295 } 1296 1297 SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL); 1298 SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL); 1299 1300 DEV_MODULE(fbt, fbt_modevent, NULL); 1301 MODULE_VERSION(fbt, 1); 1302 MODULE_DEPEND(fbt, dtrace, 1, 1, 1); 1303 MODULE_DEPEND(fbt, opensolaris, 1, 1, 1); 1304