1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 27 #include <sys/errno.h> 28 #include <sys/stat.h> 29 #include <sys/modctl.h> 30 #include <sys/conf.h> 31 #include <sys/systm.h> 32 #include <sys/ddi.h> 33 #include <sys/sunddi.h> 34 #include <sys/cpuvar.h> 35 #include <sys/kmem.h> 36 #include <sys/strsubr.h> 37 #include <sys/dtrace.h> 38 #include <sys/kobj.h> 39 #include <sys/modctl.h> 40 #include <sys/atomic.h> 41 #include <vm/seg_kmem.h> 42 #include <sys/stack.h> 43 #include <sys/ctf_api.h> 44 #include <sys/sysmacros.h> 45 46 static dev_info_t *fbt_devi; 47 static dtrace_provider_id_t fbt_id; 48 static uintptr_t fbt_trampoline; 49 static caddr_t fbt_trampoline_window; 50 static size_t fbt_trampoline_size; 51 static int fbt_verbose = 0; 52 53 /* 54 * Various interesting bean counters. 55 */ 56 static int fbt_entry; 57 static int fbt_ret; 58 static int fbt_retl; 59 static int fbt_retl_jmptab; 60 static int fbt_retl_twoinstr; 61 static int fbt_retl_tailcall; 62 static int fbt_retl_tailjmpl; 63 static int fbt_leaf_functions; 64 65 extern char stubs_base[]; 66 extern char stubs_end[]; 67 68 #define FBT_REG_G0 0 69 #define FBT_REG_G1 1 70 #define FBT_REG_O0 8 71 #define FBT_REG_O1 9 72 #define FBT_REG_O2 10 73 #define FBT_REG_O3 11 74 #define FBT_REG_O4 12 75 #define FBT_REG_O5 13 76 #define FBT_REG_O6 14 77 #define FBT_REG_O7 15 78 #define FBT_REG_I0 24 79 #define FBT_REG_I1 25 80 #define FBT_REG_I2 26 81 #define FBT_REG_I3 27 82 #define FBT_REG_I4 28 83 #define FBT_REG_I7 31 84 #define FBT_REG_L0 16 85 #define FBT_REG_L1 17 86 #define FBT_REG_L2 18 87 #define FBT_REG_L3 19 88 #define FBT_REG_PC 5 89 90 #define FBT_REG_ISGLOBAL(r) ((r) < 8) 91 #define FBT_REG_ISOUTPUT(r) ((r) >= 8 && (r) < 16) 92 #define FBT_REG_ISLOCAL(r) ((r) >= 16 && (r) < 24) 93 #define FBT_REG_ISVOLATILE(r) \ 94 ((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0) 95 #define FBT_REG_NLOCALS 8 96 97 #define FBT_REG_MARKLOCAL(locals, r) \ 98 if (FBT_REG_ISLOCAL(r)) \ 99 (locals)[(r) - FBT_REG_L0] = 1; 100 101 #define FBT_REG_INITLOCALS(local, locals) \ 102 for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++) \ 103 (locals)[(local)] = 0; \ 104 (local) = FBT_REG_L0 105 106 #define FBT_REG_ALLOCLOCAL(local, locals) \ 107 while ((locals)[(local) - FBT_REG_L0]) \ 108 (local)++; \ 109 (locals)[(local) - FBT_REG_L0] = 1; 110 111 #define FBT_OP_MASK 0xc0000000 112 #define FBT_OP_SHIFT 30 113 #define FBT_OP(val) ((val) & FBT_FMT1_MASK) 114 115 #define FBT_SIMM13_MASK 0x1fff 116 #define FBT_SIMM13_MAX ((int32_t)0xfff) 117 #define FBT_IMM22_MASK 0x3fffff 118 #define FBT_IMM22_SHIFT 10 119 #define FBT_IMM10_MASK 0x3ff 120 121 #define FBT_DISP30_MASK 0x3fffffff 122 #define FBT_DISP30(from, to) \ 123 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK) 124 125 #define FBT_DISP22_MASK 0x3fffff 126 #define FBT_DISP22(from, to) \ 127 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK) 128 129 #define FBT_DISP19_MASK 0x7ffff 130 #define FBT_DISP19(from, to) \ 131 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK) 132 133 #define FBT_DISP16_HISHIFT 20 134 #define FBT_DISP16_HIMASK (0x3 << FBT_DISP16_HISHIFT) 135 #define FBT_DISP16_LOMASK (0x3fff) 136 #define FBT_DISP16_MASK (FBT_DISP16_HIMASK | FBT_DISP16_LOMASK) 137 #define FBT_DISP16(val) \ 138 ((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK)) 139 140 #define FBT_DISP14_MASK 0x3fff 141 #define FBT_DISP14(from, to) \ 142 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK) 143 144 #define FBT_OP0 (((uint32_t)0) << FBT_OP_SHIFT) 145 #define FBT_OP1 (((uint32_t)1) << FBT_OP_SHIFT) 146 #define FBT_OP2 (((uint32_t)2) << FBT_OP_SHIFT) 147 #define FBT_ILLTRAP 0 148 149 #define FBT_ANNUL_SHIFT 29 150 #define FBT_ANNUL (1 << FBT_ANNUL_SHIFT) 151 152 #define FBT_FMT3_OP3_SHIFT 19 153 #define FBT_FMT3_OP_MASK 0xc1f80000 154 #define FBT_FMT3_OP(val) ((val) & FBT_FMT3_OP_MASK) 155 156 #define FBT_FMT3_RD_SHIFT 25 157 #define FBT_FMT3_RD_MASK (0x1f << FBT_FMT3_RD_SHIFT) 158 #define FBT_FMT3_RD(val) \ 159 (((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT) 160 161 #define FBT_FMT3_RS1_SHIFT 14 162 #define FBT_FMT3_RS1_MASK (0x1f << FBT_FMT3_RS1_SHIFT) 163 #define FBT_FMT3_RS1(val) \ 164 (((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT) 165 #define FBT_FMT3_RS1_SET(val, rs1) \ 166 (val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT) 167 168 #define FBT_FMT3_RS2_SHIFT 0 169 #define FBT_FMT3_RS2_MASK (0x1f << FBT_FMT3_RS2_SHIFT) 170 #define FBT_FMT3_RS2(val) \ 171 (((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT) 172 #define FBT_FMT3_RS2_SET(val, rs2) \ 173 (val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT) 174 175 #define FBT_FMT3_IMM_SHIFT 13 176 #define FBT_FMT3_IMM (1 << FBT_FMT3_IMM_SHIFT) 177 #define FBT_FMT3_SIMM13_MASK FBT_SIMM13_MASK 178 179 #define FBT_FMT3_ISIMM(val) ((val) & FBT_FMT3_IMM) 180 #define FBT_FMT3_SIMM13(val) ((val) & FBT_FMT3_SIMM13_MASK) 181 182 #define FBT_FMT2_OP2_SHIFT 22 183 #define FBT_FMT2_OP2_MASK (0x7 << FBT_FMT2_OP2_SHIFT) 184 #define FBT_FMT2_RD_SHIFT 25 185 186 #define FBT_FMT1_OP(val) ((val) & FBT_OP_MASK) 187 #define FBT_FMT1_DISP30(val) ((val) & FBT_DISP30_MASK) 188 189 #define FBT_FMT2_OP2_BPCC (0x01 << FBT_FMT2_OP2_SHIFT) 190 #define FBT_FMT2_OP2_BCC (0x02 << FBT_FMT2_OP2_SHIFT) 191 #define FBT_FMT2_OP2_BPR (0x03 << FBT_FMT2_OP2_SHIFT) 192 #define FBT_FMT2_OP2_SETHI (0x04 << FBT_FMT2_OP2_SHIFT) 193 194 #define FBT_FMT2_COND_SHIFT 25 195 #define FBT_FMT2_COND_BA (0x8 << FBT_FMT2_COND_SHIFT) 196 #define FBT_FMT2_COND_BL (0x3 << FBT_FMT2_COND_SHIFT) 197 #define FBT_FMT2_COND_BGE (0xb << FBT_FMT2_COND_SHIFT) 198 199 #define FBT_OP_RESTORE (FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT)) 200 #define FBT_OP_SAVE (FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT)) 201 #define FBT_OP_JMPL (FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT)) 202 #define FBT_OP_RETURN (FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT)) 203 #define FBT_OP_CALL FBT_OP1 204 #define FBT_OP_SETHI (FBT_OP0 | FBT_FMT2_OP2_SETHI) 205 #define FBT_OP_ADD (FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT)) 206 #define FBT_OP_OR (FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT)) 207 #define FBT_OP_SUB (FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT)) 208 #define FBT_OP_CC (FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT)) 209 #define FBT_OP_BA (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA) 210 #define FBT_OP_BL (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL) 211 #define FBT_OP_BGE (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE) 212 #define FBT_OP_BAPCC (FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA) 213 #define FBT_OP_RD (FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT)) 214 215 #define FBT_ORLO(rs, val, rd) \ 216 (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \ 217 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK)) 218 219 #define FBT_ORSIMM13(rs, val, rd) \ 220 (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \ 221 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) 222 223 #define FBT_ADDSIMM13(rs, val, rd) \ 224 (FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \ 225 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) 226 227 #define FBT_ADD(rs1, rs2, rd) \ 228 (FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ 229 ((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT)) 230 231 #define FBT_CMP(rs1, rs2) \ 232 (FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ 233 ((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT)) 234 235 #define FBT_MOV(rs, rd) \ 236 (FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \ 237 ((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT)) 238 239 #define FBT_SETHI(val, reg) \ 240 (FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \ 241 ((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK)) 242 243 #define FBT_CALL(orig, dest) (FBT_OP_CALL | FBT_DISP30(orig, dest)) 244 245 #define FBT_RET \ 246 (FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \ 247 (FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1)) 248 249 #define FBT_SAVEIMM(rd, val, rs1) \ 250 (FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ 251 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) 252 253 #define FBT_RESTORE(rd, rs1, rs2) \ 254 (FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ 255 ((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT)) 256 257 #define FBT_RETURN(rs1, val) \ 258 (FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ 259 FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) 260 261 #define FBT_BA(orig, dest) (FBT_OP_BA | FBT_DISP22(orig, dest)) 262 #define FBT_BAA(orig, dest) (FBT_BA(orig, dest) | FBT_ANNUL) 263 #define FBT_BL(orig, dest) (FBT_OP_BL | FBT_DISP22(orig, dest)) 264 #define FBT_BGE(orig, dest) (FBT_OP_BGE | FBT_DISP22(orig, dest)) 265 #define FBT_BDEST(va, instr) ((uintptr_t)(va) + \ 266 (((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8)) 267 #define FBT_BPCCDEST(va, instr) ((uintptr_t)(va) + \ 268 (((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11)) 269 #define FBT_BPRDEST(va, instr) ((uintptr_t)(va) + \ 270 (((int32_t)((FBT_DISP16(instr)) << 16)) >> 14)) 271 272 /* 273 * We're only going to treat a save as safe if (a) both rs1 and rd are 274 * %sp and (b) if the instruction has a simm, the value isn't 0. 275 */ 276 #define FBT_IS_SAVE(instr) \ 277 (FBT_FMT3_OP(instr) == FBT_OP_SAVE && \ 278 FBT_FMT3_RD(instr) == FBT_REG_O6 && \ 279 FBT_FMT3_RS1(instr) == FBT_REG_O6 && \ 280 !(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0)) 281 282 #define FBT_IS_BA(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA) 283 #define FBT_IS_BAPCC(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC) 284 285 #define FBT_IS_RDPC(instr) ((FBT_FMT3_OP(instr) == FBT_OP_RD) && \ 286 (FBT_FMT3_RD(instr) == FBT_REG_PC)) 287 288 #define FBT_IS_PCRELATIVE(instr) \ 289 ((((instr) & FBT_OP_MASK) == FBT_OP0 && \ 290 ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \ 291 ((instr) & FBT_OP_MASK) == FBT_OP1 || \ 292 FBT_IS_RDPC(instr)) 293 294 #define FBT_IS_CTI(instr) \ 295 ((((instr) & FBT_OP_MASK) == FBT_OP0 && \ 296 ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \ 297 ((instr) & FBT_OP_MASK) == FBT_OP1 || \ 298 (FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \ 299 (FBT_FMT3_OP(instr) == FBT_OP_RETURN)) 300 301 #define FBT_PROBENAME_ENTRY "entry" 302 #define FBT_PROBENAME_RETURN "return" 303 #define FBT_ESTIMATE_ID (UINT32_MAX) 304 #define FBT_COUNTER(id, count) if ((id) != FBT_ESTIMATE_ID) (count)++ 305 306 #define FBT_ENTENT_MAXSIZE (16 * sizeof (uint32_t)) 307 #define FBT_RETENT_MAXSIZE (11 * sizeof (uint32_t)) 308 #define FBT_RETLENT_MAXSIZE (23 * sizeof (uint32_t)) 309 #define FBT_ENT_MAXSIZE \ 310 MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE) 311 312 typedef struct fbt_probe { 313 char *fbtp_name; 314 dtrace_id_t fbtp_id; 315 uintptr_t fbtp_addr; 316 struct modctl *fbtp_ctl; 317 int fbtp_loadcnt; 318 int fbtp_symndx; 319 int fbtp_primary; 320 int fbtp_return; 321 uint32_t *fbtp_patchpoint; 322 uint32_t fbtp_patchval; 323 uint32_t fbtp_savedval; 324 struct fbt_probe *fbtp_next; 325 } fbt_probe_t; 326 327 typedef struct fbt_trampoline { 328 uintptr_t fbtt_va; 329 uintptr_t fbtt_limit; 330 uintptr_t fbtt_next; 331 } fbt_trampoline_t; 332 333 static caddr_t 334 fbt_trampoline_map(uintptr_t tramp, size_t size) 335 { 336 uintptr_t offs; 337 page_t **ppl; 338 339 ASSERT(fbt_trampoline_window == NULL); 340 ASSERT(fbt_trampoline_size == 0); 341 ASSERT(fbt_trampoline == NULL); 342 343 size += tramp & PAGEOFFSET; 344 fbt_trampoline = tramp & PAGEMASK; 345 fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK; 346 fbt_trampoline_window = 347 vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP); 348 349 (void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline, 350 fbt_trampoline_size, S_WRITE); 351 352 for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) { 353 hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE, 354 hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs), 355 PROT_READ | PROT_WRITE, 356 HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); 357 } 358 359 as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size, 360 S_WRITE); 361 362 return (fbt_trampoline_window + (tramp & PAGEOFFSET)); 363 } 364 365 static void 366 fbt_trampoline_unmap() 367 { 368 ASSERT(fbt_trampoline_window != NULL); 369 ASSERT(fbt_trampoline_size != 0); 370 ASSERT(fbt_trampoline != NULL); 371 372 membar_enter(); 373 sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size); 374 sync_icache(fbt_trampoline_window, fbt_trampoline_size); 375 376 hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size, 377 HAT_UNLOAD_UNLOCK); 378 379 vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size); 380 381 fbt_trampoline_window = NULL; 382 fbt_trampoline = NULL; 383 fbt_trampoline_size = 0; 384 } 385 386 static uintptr_t 387 fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp, 388 int nargs) 389 { 390 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next; 391 uint32_t first = *instr; 392 uintptr_t va = tramp->fbtt_va; 393 uintptr_t base = tramp->fbtt_next; 394 395 if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) { 396 /* 397 * There isn't sufficient room for this entry; return failure. 398 */ 399 return (0); 400 } 401 402 FBT_COUNTER(id, fbt_entry); 403 404 if (FBT_IS_SAVE(first)) { 405 *tinstr++ = first; 406 } else { 407 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6); 408 } 409 410 if (id > (uint32_t)FBT_SIMM13_MAX) { 411 *tinstr++ = FBT_SETHI(id, FBT_REG_O0); 412 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0); 413 } else { 414 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0); 415 } 416 417 if (nargs >= 1) 418 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1); 419 420 if (nargs >= 2) 421 *tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2); 422 423 if (nargs >= 3) 424 *tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3); 425 426 if (nargs >= 4) 427 *tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4); 428 429 if (nargs >= 5) 430 *tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5); 431 432 if (FBT_IS_SAVE(first)) { 433 uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t); 434 435 *tinstr++ = FBT_SETHI(ret, FBT_REG_G1); 436 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); 437 tinstr++; 438 *tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7); 439 } else { 440 uintptr_t slot = *--tinstr; 441 uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t); 442 uint32_t delay = first; 443 444 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); 445 tinstr++; 446 *tinstr++ = slot; 447 *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0); 448 449 if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) { 450 /* 451 * This is a special case: we are instrumenting a 452 * a non-annulled branch-always (or variant). We'll 453 * return directly to the destination of the branch, 454 * copying the instruction in the delay slot here, 455 * and then executing it in the slot of a ba. 456 */ 457 if (FBT_IS_BA(first)) { 458 ret = FBT_BDEST(instr, *instr); 459 } else { 460 ret = FBT_BPCCDEST(instr, *instr); 461 } 462 463 delay = *(instr + 1); 464 } 465 466 if ((first & FBT_OP_MASK) != FBT_OP0 || 467 (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) { 468 *tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret); 469 tinstr++; 470 *tinstr++ = delay; 471 } else { 472 /* 473 * If this is a branch-on-register, we have a little 474 * more work to do: because the displacement is only 475 * sixteen bits, we're going to thunk the branch into 476 * the trampoline, and then ba,a to the appropriate 477 * destination in the branch targets. That is, we're 478 * constructing this sequence in the trampoline: 479 * 480 * br[cc] %[rs], 1f 481 * <delay-instruction> 482 * ba,a <not-taken-destination> 483 * 1: ba,a <taken-destination> 484 * 485 */ 486 uintptr_t targ = FBT_BPRDEST(instr, first); 487 488 *tinstr = first & ~(FBT_DISP16_MASK); 489 *tinstr |= FBT_DISP14(tinstr, &tinstr[3]); 490 tinstr++; 491 *tinstr++ = *(instr + 1); 492 *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, 493 ret + sizeof (uint32_t)); 494 tinstr++; 495 *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ); 496 tinstr++; 497 } 498 } 499 500 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next; 501 tramp->fbtt_next = (uintptr_t)tinstr; 502 503 return (1); 504 } 505 506 /* 507 * We are patching control-transfer/restore couplets. There are three 508 * variants of couplet: 509 * 510 * (a) return rs1 + imm 511 * delay 512 * 513 * (b) jmpl rs1 + (rs2 | offset), rd 514 * restore rs1, rs2 | imm, rd 515 * 516 * (c) call displacement 517 * restore rs1, rs2 | imm, rd 518 * 519 * If rs1 in (a) is anything other than %i7, or imm is anything other than 8, 520 * or delay is a DCTI, we fail. If rd from the jmpl in (b) is something other 521 * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call 522 * through a register), we fail. 523 * 524 * Note that rs1 and rs2 in the restore instructions in (b) and (c) are 525 * potentially outputs and/or globals. Because these registers cannot be 526 * relied upon across the call to dtrace_probe(), we move rs1 into an unused 527 * local, ls0, and rs2 into an unused local, ls1, and restructure the restore 528 * to be: 529 * 530 * restore ls0, ls1, rd 531 * 532 * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals. 533 * If the jmpl uses outputs or globals, we restructure it to be: 534 * 535 * jmpl ls2 + (ls3 | offset), (%g0 | %o7) 536 * 537 */ 538 /*ARGSUSED*/ 539 static int 540 fbt_canpatch_return(uint32_t *instr, int offset, const char *name) 541 { 542 int rd; 543 544 if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) { 545 uint32_t delay = *(instr + 1); 546 547 if (*instr != FBT_RETURN(FBT_REG_I7, 8)) { 548 /* 549 * It's unclear if we should warn about this or not. 550 * We really wouldn't expect the compiler to generate 551 * return instructions with something other than %i7 552 * as rs1 and 8 as the simm13 -- it would just be 553 * mean-spirited. That said, such a construct isn't 554 * necessarily incorrect. Sill, we err on the side of 555 * caution and warn about it... 556 */ 557 cmn_err(CE_NOTE, "cannot instrument return of %s at " 558 "%p: non-canonical return instruction", name, 559 (void *)instr); 560 return (0); 561 } 562 563 if (FBT_IS_CTI(delay)) { 564 /* 565 * This is even weirder -- a DCTI coupled with a 566 * return instruction. Similar constructs are used to 567 * return from utraps, but these typically have the 568 * return in the slot -- and we wouldn't expect to see 569 * it in the kernel regardless. At any rate, we don't 570 * want to try to instrument this construct, whatever 571 * it may be. 572 */ 573 cmn_err(CE_NOTE, "cannot instrument return of %s at " 574 "%p: CTI in delay slot of return instruction", 575 name, (void *)instr); 576 return (0); 577 } 578 579 if (FBT_IS_PCRELATIVE(delay)) { 580 /* 581 * This is also very weird, but might be correct code 582 * if the function is (for example) returning the 583 * address of the delay instruction of the return as 584 * its return value (e.g. "rd %pc, %o0" in the slot). 585 * Perhaps correct, but still too weird to not warn 586 * about it... 587 */ 588 cmn_err(CE_NOTE, "cannot instrument return of %s at " 589 "%p: PC-relative instruction in delay slot of " 590 "return instruction", name, (void *)instr); 591 return (0); 592 } 593 594 return (1); 595 } 596 597 if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE) 598 return (0); 599 600 if (FBT_FMT1_OP(*instr) == FBT_OP_CALL) 601 return (1); 602 603 if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL) 604 return (0); 605 606 rd = FBT_FMT3_RD(*instr); 607 608 if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0) 609 return (1); 610 611 /* 612 * We have encountered a jmpl that is storing the calling %pc in 613 * some register besides %i7, %o7 or %g0. This is strange; emit 614 * a warning and fail. 615 */ 616 cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected " 617 "jmpl destination register", name, (void *)instr); 618 return (0); 619 } 620 621 static int 622 fbt_canpatch_retl(uint32_t *instr, int offset, const char *name) 623 { 624 if (FBT_FMT1_OP(*instr) == FBT_OP_CALL || 625 (FBT_FMT3_OP(*instr) == FBT_OP_JMPL && 626 FBT_FMT3_RD(*instr) == FBT_REG_O7)) { 627 /* 628 * If this is a call (or a jmpl that links into %o7), we can 629 * patch it iff the next instruction uses %o7 as a destination 630 * register. Because there is an ABI responsibility to 631 * restore %o7 to the value before the call/jmpl, we don't 632 * particularly care how this routine is managing to restore 633 * it (mov, add, ld or divx for all we care). If it doesn't 634 * seem to be restoring it at all, however, we'll refuse 635 * to patch it. 636 */ 637 uint32_t delay = *(instr + 1); 638 uint32_t op, rd; 639 640 op = FBT_FMT1_OP(delay); 641 rd = FBT_FMT3_RD(delay); 642 643 if (op != FBT_OP2 || rd != FBT_REG_O7) { 644 /* 645 * This is odd. Before we assume that we're looking 646 * at something bizarre (and warn accordingly), we'll 647 * check to see if it's obviously a jump table entry. 648 */ 649 if (*instr < (uintptr_t)instr && 650 *instr >= (uintptr_t)instr - offset) 651 return (0); 652 653 cmn_err(CE_NOTE, "cannot instrument return of %s at " 654 "%p: leaf jmpl/call delay isn't restoring %%o7", 655 name, (void *)instr); 656 return (0); 657 } 658 659 return (1); 660 } 661 662 if (offset == sizeof (uint32_t)) { 663 /* 664 * If this is the second instruction in the function, we're 665 * going to allow it to be patched if the first instruction 666 * is a patchable return-from-leaf instruction. 667 */ 668 if (fbt_canpatch_retl(instr - 1, 0, name)) 669 return (1); 670 } 671 672 if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL) 673 return (0); 674 675 if (FBT_FMT3_RD(*instr) != FBT_REG_G0) 676 return (0); 677 678 return (1); 679 } 680 681 /*ARGSUSED*/ 682 static uint32_t 683 fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim, 684 int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name) 685 { 686 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next; 687 uint32_t cti = *instr, restore = *(instr + 1), rs1, dest; 688 uintptr_t va = tramp->fbtt_va; 689 uintptr_t base = tramp->fbtt_next; 690 uint32_t locals[FBT_REG_NLOCALS], local; 691 692 if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) { 693 /* 694 * There isn't sufficient room for this entry; return failure. 695 */ 696 return (FBT_ILLTRAP); 697 } 698 699 FBT_COUNTER(id, fbt_ret); 700 701 if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) { 702 /* 703 * To handle the case of the return instruction, we'll emit a 704 * restore, followed by the instruction in the slot (which 705 * we'll transplant here), and then another save. While it 706 * may seem intellectually unsatisfying to emit the additional 707 * restore/save couplet, one can take solace in the fact that 708 * we don't do this if the instruction in the return delay 709 * slot is a nop -- which it is nearly 90% of the time with 710 * gcc. (And besides, this couplet can't induce unnecessary 711 * spill/fill traps; rewriting the delay instruction to be 712 * in terms of the current window hardly seems worth the 713 * trouble -- let alone the risk.) 714 */ 715 uint32_t delay = *(instr + 1); 716 ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8)); 717 718 cti = FBT_RET; 719 restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0); 720 721 if (delay != FBT_SETHI(0, FBT_REG_G0)) { 722 *tinstr++ = restore; 723 *tinstr++ = delay; 724 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, 725 -SA(MINFRAME), FBT_REG_O6); 726 } 727 } 728 729 FBT_REG_INITLOCALS(local, locals); 730 731 /* 732 * Mark the locals used in the jmpl. 733 */ 734 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { 735 uint32_t rs1 = FBT_FMT3_RS1(cti); 736 FBT_REG_MARKLOCAL(locals, rs1); 737 738 if (!FBT_FMT3_ISIMM(cti)) { 739 uint32_t rs2 = FBT_FMT3_RS2(cti); 740 FBT_REG_MARKLOCAL(locals, rs2); 741 } 742 } 743 744 /* 745 * And mark the locals used in the restore. 746 */ 747 rs1 = FBT_FMT3_RS1(restore); 748 FBT_REG_MARKLOCAL(locals, rs1); 749 750 if (!FBT_FMT3_ISIMM(restore)) { 751 uint32_t rs2 = FBT_FMT3_RS2(restore); 752 FBT_REG_MARKLOCAL(locals, rs2); 753 } 754 755 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { 756 uint32_t rs1 = FBT_FMT3_RS1(cti); 757 758 if (FBT_REG_ISVOLATILE(rs1)) { 759 FBT_REG_ALLOCLOCAL(local, locals); 760 FBT_FMT3_RS1_SET(cti, local); 761 *tinstr++ = FBT_MOV(rs1, local); 762 } 763 764 if (!FBT_FMT3_ISIMM(cti)) { 765 uint32_t rs2 = FBT_FMT3_RS2(cti); 766 767 if (FBT_REG_ISVOLATILE(rs2)) { 768 FBT_REG_ALLOCLOCAL(local, locals); 769 FBT_FMT3_RS2_SET(cti, local); 770 *tinstr++ = FBT_MOV(rs2, local); 771 } 772 } 773 } 774 775 rs1 = FBT_FMT3_RS1(restore); 776 777 if (FBT_REG_ISVOLATILE(rs1)) { 778 FBT_REG_ALLOCLOCAL(local, locals); 779 FBT_FMT3_RS1_SET(restore, local); 780 *tinstr++ = FBT_MOV(rs1, local); 781 } 782 783 if (!FBT_FMT3_ISIMM(restore)) { 784 uint32_t rs2 = FBT_FMT3_RS2(restore); 785 786 if (FBT_REG_ISVOLATILE(rs2)) { 787 FBT_REG_ALLOCLOCAL(local, locals); 788 FBT_FMT3_RS2_SET(restore, local); 789 *tinstr++ = FBT_MOV(rs2, local); 790 } 791 } 792 793 if (id > (uint32_t)FBT_SIMM13_MAX) { 794 *tinstr++ = FBT_SETHI(id, FBT_REG_O0); 795 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0); 796 } else { 797 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0); 798 } 799 800 if (offset > (uint32_t)FBT_SIMM13_MAX) { 801 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1); 802 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1); 803 } else { 804 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1); 805 } 806 807 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); 808 tinstr++; 809 810 if (FBT_FMT3_RD(restore) == FBT_REG_O0) { 811 /* 812 * If the destination register of the restore is %o0, we 813 * need to perform the implied calculation to derive the 814 * return value. 815 */ 816 uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD; 817 add &= ~FBT_FMT3_RD_MASK; 818 *tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT); 819 } else { 820 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2); 821 } 822 823 /* 824 * If the control transfer instruction is %pc-relative (i.e. a 825 * call), we need to reset it appropriately. 826 */ 827 if (FBT_FMT1_OP(cti) == FBT_OP_CALL) { 828 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2); 829 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest); 830 tinstr++; 831 } else { 832 *tinstr++ = cti; 833 } 834 835 *tinstr++ = restore; 836 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next; 837 tramp->fbtt_next = (uintptr_t)tinstr; 838 839 return (FBT_BAA(instr, va)); 840 } 841 842 static uint32_t 843 fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim, 844 int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name) 845 { 846 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next; 847 uintptr_t va = tramp->fbtt_va; 848 uintptr_t base = tramp->fbtt_next; 849 uint32_t cti = *instr, dest; 850 int annul = 0; 851 852 FBT_COUNTER(id, fbt_retl); 853 854 if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) { 855 /* 856 * There isn't sufficient room for this entry; return failure. 857 */ 858 return (FBT_ILLTRAP); 859 } 860 861 if (offset == sizeof (uint32_t) && 862 fbt_canpatch_retl(instr - 1, 0, name)) { 863 *tinstr++ = *instr; 864 annul = 1; 865 FBT_COUNTER(id, fbt_retl_twoinstr); 866 } else { 867 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL && 868 FBT_FMT3_RD(cti) != FBT_REG_O7 && 869 FBT_FMT3_RS1(cti) != FBT_REG_O7) { 870 annul = 1; 871 *tinstr++ = *(instr + 1); 872 } 873 } 874 875 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6); 876 877 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { 878 uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0; 879 880 /* 881 * If we have a jmpl and it's in terms of output registers, we 882 * need to rewrite it to be in terms of the corresponding input 883 * registers. If it's in terms of the globals, we'll rewrite 884 * it to be in terms of locals. 885 */ 886 rs1 = FBT_FMT3_RS1(cti); 887 888 if (FBT_REG_ISOUTPUT(rs1)) 889 rs1 += o2i; 890 891 if (FBT_REG_ISGLOBAL(rs1)) { 892 *tinstr++ = FBT_MOV(rs1, FBT_REG_L0); 893 rs1 = FBT_REG_L0; 894 } 895 896 FBT_FMT3_RS1_SET(cti, rs1); 897 898 if (!FBT_FMT3_ISIMM(cti)) { 899 rs2 = FBT_FMT3_RS2(cti); 900 901 if (FBT_REG_ISOUTPUT(rs2)) 902 rs2 += o2i; 903 904 if (FBT_REG_ISGLOBAL(rs2)) { 905 *tinstr++ = FBT_MOV(rs2, FBT_REG_L1); 906 rs2 = FBT_REG_L1; 907 } 908 909 FBT_FMT3_RS2_SET(cti, rs2); 910 } 911 912 /* 913 * Now we need to check the rd and source register for the jmpl; 914 * If neither rd nor the source register is %o7, then we might 915 * have a jmp that is actually part of a jump table. We need 916 * to generate the code to compare it to the base and limit of 917 * the function. 918 */ 919 if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) { 920 uintptr_t base = (uintptr_t)funcbase; 921 uintptr_t limit = (uintptr_t)funclim; 922 923 FBT_COUNTER(id, fbt_retl_jmptab); 924 925 if (FBT_FMT3_ISIMM(cti)) { 926 *tinstr++ = FBT_ADDSIMM13(rs1, 927 FBT_FMT3_SIMM13(cti), FBT_REG_L2); 928 } else { 929 *tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2); 930 } 931 932 *tinstr++ = FBT_SETHI(base, FBT_REG_L3); 933 *tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3); 934 *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3); 935 *tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t)); 936 *tinstr++ = FBT_SETHI(limit, FBT_REG_L3); 937 *tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3); 938 *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3); 939 *tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t)); 940 *tinstr++ = FBT_SETHI(0, FBT_REG_G0); 941 *tinstr++ = cti; 942 *tinstr++ = FBT_RESTORE(FBT_REG_G0, 943 FBT_REG_G0, FBT_REG_G0); 944 } 945 } 946 947 if (id > (uint32_t)FBT_SIMM13_MAX) { 948 *tinstr++ = FBT_SETHI(id, FBT_REG_O0); 949 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0); 950 } else { 951 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0); 952 } 953 954 if (offset > (uint32_t)FBT_SIMM13_MAX) { 955 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1); 956 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1); 957 } else { 958 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1); 959 } 960 961 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); 962 tinstr++; 963 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2); 964 965 /* 966 * If the control transfer instruction is %pc-relative (i.e. a 967 * call), we need to reset it appropriately. 968 */ 969 if (FBT_FMT1_OP(cti) == FBT_OP_CALL) { 970 FBT_COUNTER(id, fbt_retl_tailcall); 971 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2); 972 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest); 973 tinstr++; 974 annul = 1; 975 } else { 976 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { 977 *tinstr++ = cti; 978 979 if (FBT_FMT3_RD(cti) == FBT_REG_O7) { 980 FBT_COUNTER(id, fbt_retl_tailjmpl); 981 annul = 1; 982 } 983 } else { 984 *tinstr++ = FBT_RET; 985 } 986 } 987 988 *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0); 989 990 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next; 991 tramp->fbtt_next = (uintptr_t)tinstr; 992 993 return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va)); 994 } 995 996 /*ARGSUSED*/ 997 static void 998 fbt_provide_module(void *arg, struct modctl *ctl) 999 { 1000 struct module *mp = ctl->mod_mp; 1001 char *modname = ctl->mod_modname; 1002 char *str = mp->strings; 1003 int nsyms = mp->nsyms; 1004 Shdr *symhdr = mp->symhdr; 1005 size_t symsize; 1006 char *name; 1007 int i; 1008 fbt_probe_t *fbt, *retfbt; 1009 fbt_trampoline_t tramp; 1010 uintptr_t offset; 1011 int primary = 0; 1012 ctf_file_t *fp = NULL; 1013 int error; 1014 int estimate = 1; 1015 uint32_t faketramp[50]; 1016 size_t fbt_size = 0; 1017 1018 /* 1019 * Employees of dtrace and their families are ineligible. Void 1020 * where prohibited. 1021 */ 1022 if (strcmp(modname, "dtrace") == 0) 1023 return; 1024 1025 if (ctl->mod_requisites != NULL) { 1026 struct modctl_list *list; 1027 1028 list = (struct modctl_list *)ctl->mod_requisites; 1029 1030 for (; list != NULL; list = list->modl_next) { 1031 if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0) 1032 return; 1033 } 1034 } 1035 1036 /* 1037 * KMDB is ineligible for instrumentation -- it may execute in 1038 * any context, including probe context. 1039 */ 1040 if (strcmp(modname, "kmdbmod") == 0) 1041 return; 1042 1043 if (str == NULL || symhdr == NULL || symhdr->sh_addr == NULL) { 1044 /* 1045 * If this module doesn't (yet) have its string or symbol 1046 * table allocated, clear out. 1047 */ 1048 return; 1049 } 1050 1051 symsize = symhdr->sh_entsize; 1052 1053 if (mp->fbt_nentries) { 1054 /* 1055 * This module has some FBT entries allocated; we're afraid 1056 * to screw with it. 1057 */ 1058 return; 1059 } 1060 1061 if (mp->fbt_tab != NULL) 1062 estimate = 0; 1063 1064 /* 1065 * This is a hack for unix/genunix/krtld. 1066 */ 1067 primary = vmem_contains(heap_arena, (void *)ctl, 1068 sizeof (struct modctl)) == 0; 1069 kobj_textwin_alloc(mp); 1070 1071 /* 1072 * Open the CTF data for the module. We'll use this to determine the 1073 * functions that can be instrumented. Note that this call can fail, 1074 * in which case we'll use heuristics to determine the functions that 1075 * can be instrumented. (But in particular, leaf functions will not be 1076 * instrumented.) 1077 */ 1078 fp = ctf_modopen(mp, &error); 1079 1080 forreal: 1081 if (!estimate) { 1082 tramp.fbtt_next = 1083 (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab, 1084 mp->fbt_size); 1085 tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size; 1086 tramp.fbtt_va = (uintptr_t)mp->fbt_tab; 1087 } 1088 1089 for (i = 1; i < nsyms; i++) { 1090 ctf_funcinfo_t f; 1091 uint32_t *instr, *base, *limit; 1092 Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize); 1093 int have_ctf = 0, is_leaf = 0, nargs, cti = 0; 1094 int (*canpatch)(uint32_t *, int, const char *); 1095 uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int, 1096 uint32_t, fbt_trampoline_t *, const char *); 1097 1098 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC) 1099 continue; 1100 1101 /* 1102 * Weak symbols are not candidates. This could be made to 1103 * work (where weak functions and their underlying function 1104 * appear as two disjoint probes), but it's not simple. 1105 */ 1106 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 1107 continue; 1108 1109 name = str + sym->st_name; 1110 1111 if (strstr(name, "dtrace_") == name && 1112 strstr(name, "dtrace_safe_") != name) { 1113 /* 1114 * Anything beginning with "dtrace_" may be called 1115 * from probe context unless it explitly indicates 1116 * that it won't be called from probe context by 1117 * using the prefix "dtrace_safe_". 1118 */ 1119 continue; 1120 } 1121 1122 if (strstr(name, "kdi_") == name || 1123 strstr(name, "_kdi_") != NULL) { 1124 /* 1125 * Any function name beginning with "kdi_" or 1126 * containing the string "_kdi_" is a part of the 1127 * kernel debugger interface and may be called in 1128 * arbitrary context -- including probe context. 1129 */ 1130 continue; 1131 } 1132 1133 if (strstr(name, "__relocatable") != NULL) { 1134 /* 1135 * Anything with the string "__relocatable" anywhere 1136 * in the function name is considered to be a function 1137 * that may be manually relocated before execution. 1138 * Because FBT uses a PC-relative technique for 1139 * instrumentation, these functions cannot safely 1140 * be instrumented by us. 1141 */ 1142 continue; 1143 } 1144 1145 if (strstr(name, "ip_ocsum") == name) { 1146 /* 1147 * The ip_ocsum_* family of routines are all ABI 1148 * violators. (They expect incoming arguments in the 1149 * globals!) Break the ABI? No soup for you! 1150 */ 1151 continue; 1152 } 1153 1154 /* 1155 * We want to scan the function for one (and only one) save. 1156 * Any more indicates that something fancy is going on. 1157 */ 1158 base = (uint32_t *)sym->st_value; 1159 limit = (uint32_t *)(sym->st_value + sym->st_size); 1160 1161 /* 1162 * We don't want to interpose on the module stubs. 1163 */ 1164 if (base >= (uint32_t *)stubs_base && 1165 base <= (uint32_t *)stubs_end) 1166 continue; 1167 1168 /* 1169 * We can't safely trace a zero-length function... 1170 */ 1171 if (base == limit) 1172 continue; 1173 1174 /* 1175 * Due to 4524008, _init and _fini may have a bloated st_size. 1176 * While this bug was fixed quite some time ago, old drivers 1177 * may be lurking. We need to develop a better solution to 1178 * this problem, such that correct _init and _fini functions 1179 * (the vast majority) may be correctly traced. One solution 1180 * may be to scan through the entire symbol table to see if 1181 * any symbol overlaps with _init. If none does, set a bit in 1182 * the module structure that this module has correct _init and 1183 * _fini sizes. This will cause some pain the first time a 1184 * module is scanned, but at least it would be O(N) instead of 1185 * O(N log N)... 1186 */ 1187 if (strcmp(name, "_init") == 0) 1188 continue; 1189 1190 if (strcmp(name, "_fini") == 0) 1191 continue; 1192 1193 instr = base; 1194 1195 /* 1196 * While we try hard to only trace safe functions (that is, 1197 * functions at TL=0), one unsafe function manages to otherwise 1198 * appear safe: prom_trap(). We could discover prom_trap() 1199 * if we added an additional rule: in order to trace a 1200 * function, we must either (a) discover a restore or (b) 1201 * determine that the function does not have any unlinked 1202 * control transfers to another function (i.e., the function 1203 * never returns). Unfortunately, as of this writing, one 1204 * legitimate function (resume_from_zombie()) transfers 1205 * control to a different function (_resume_from_idle()) 1206 * without executing a restore. Barring a rule to figure out 1207 * that resume_from_zombie() is safe while prom_trap() is not, 1208 * we resort to hard-coding prom_trap() here. 1209 */ 1210 if (strcmp(name, "prom_trap") == 0) 1211 continue; 1212 1213 if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) { 1214 nargs = f.ctc_argc; 1215 have_ctf = 1; 1216 } else { 1217 nargs = 32; 1218 } 1219 1220 /* 1221 * If the first instruction of the function is a branch and 1222 * it's not a branch-always-not-annulled, we're going to refuse 1223 * to patch it. 1224 */ 1225 if ((*instr & FBT_OP_MASK) == FBT_OP0 && 1226 (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI && 1227 (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) { 1228 if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) { 1229 if (have_ctf) { 1230 cmn_err(CE_NOTE, "cannot instrument %s:" 1231 " begins with non-ba, " 1232 "non-br CTI", name); 1233 } 1234 continue; 1235 } 1236 } 1237 1238 while (!FBT_IS_SAVE(*instr)) { 1239 /* 1240 * Before we assume that this is a leaf routine, check 1241 * forward in the basic block for a save. 1242 */ 1243 int op = *instr & FBT_OP_MASK; 1244 int op2 = *instr & FBT_FMT2_OP2_MASK; 1245 1246 if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) { 1247 /* 1248 * This is a CTI. If we see a subsequent 1249 * save, we will refuse to process this 1250 * routine unless both of the following are 1251 * true: 1252 * 1253 * (a) The branch is not annulled 1254 * 1255 * (b) The subsequent save is in the delay 1256 * slot of the branch 1257 */ 1258 if ((*instr & FBT_ANNUL) || 1259 !FBT_IS_SAVE(*(instr + 1))) { 1260 cti = 1; 1261 } else { 1262 instr++; 1263 break; 1264 } 1265 } 1266 1267 if (op == FBT_OP1) 1268 cti = 1; 1269 1270 if (++instr == limit) 1271 break; 1272 } 1273 1274 if (instr < limit && cti) { 1275 /* 1276 * If we found a CTI before the save, we need to not 1277 * do anything. But if we have CTF information, this 1278 * is weird enough that it merits a message. 1279 */ 1280 if (!have_ctf) 1281 continue; 1282 1283 cmn_err(CE_NOTE, "cannot instrument %s: " 1284 "save not in first basic block", name); 1285 continue; 1286 } 1287 1288 if (instr == limit) { 1289 if (!have_ctf) 1290 continue; 1291 is_leaf = 1; 1292 1293 if (!estimate) 1294 fbt_leaf_functions++; 1295 1296 canpatch = fbt_canpatch_retl; 1297 patch = fbt_patch_retl; 1298 } else { 1299 canpatch = fbt_canpatch_return; 1300 patch = fbt_patch_return; 1301 } 1302 1303 if (!have_ctf && !is_leaf) { 1304 /* 1305 * Before we assume that this isn't something tricky, 1306 * look for other saves. If we find them, there are 1307 * multiple entry points here (or something), and we'll 1308 * leave it alone. 1309 */ 1310 while (++instr < limit) { 1311 if (FBT_IS_SAVE(*instr)) 1312 break; 1313 } 1314 1315 if (instr != limit) 1316 continue; 1317 } 1318 1319 instr = base; 1320 1321 if (FBT_IS_CTI(*instr)) { 1322 /* 1323 * If we have a CTI, we want to be sure that we don't 1324 * have a CTI or a PC-relative instruction in the 1325 * delay slot -- we want to be able to thunk the 1326 * instruction into the trampoline without worrying 1327 * about either DCTIs or relocations. It would be 1328 * very odd for the compiler to generate this kind of 1329 * code, so we warn about it if we have CTF 1330 * information. 1331 */ 1332 if (FBT_IS_CTI(*(instr + 1))) { 1333 if (!have_ctf) 1334 continue; 1335 1336 cmn_err(CE_NOTE, "cannot instrument %s: " 1337 "CTI in delay slot of first instruction", 1338 name); 1339 continue; 1340 } 1341 1342 if (FBT_IS_PCRELATIVE(*(instr + 1))) { 1343 if (!have_ctf) 1344 continue; 1345 1346 cmn_err(CE_NOTE, "cannot instrument %s: " 1347 "PC-relative instruction in delay slot of" 1348 " first instruction", name); 1349 continue; 1350 } 1351 } 1352 1353 if (estimate) { 1354 tramp.fbtt_next = (uintptr_t)faketramp; 1355 tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp); 1356 (void) fbt_patch_entry(instr, FBT_ESTIMATE_ID, 1357 &tramp, nargs); 1358 fbt_size += tramp.fbtt_next - (uintptr_t)faketramp; 1359 } else { 1360 fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP); 1361 fbt->fbtp_name = name; 1362 fbt->fbtp_ctl = ctl; 1363 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, 1364 name, FBT_PROBENAME_ENTRY, 1, fbt); 1365 fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va); 1366 1367 if (!fbt_patch_entry(instr, fbt->fbtp_id, 1368 &tramp, nargs)) { 1369 cmn_err(CE_WARN, "unexpectedly short FBT table " 1370 "in module %s (sym %d of %d)", modname, 1371 i, nsyms); 1372 break; 1373 } 1374 1375 fbt->fbtp_patchpoint = 1376 (uint32_t *)((uintptr_t)mp->textwin + 1377 ((uintptr_t)instr - (uintptr_t)mp->text)); 1378 fbt->fbtp_savedval = *instr; 1379 1380 fbt->fbtp_loadcnt = ctl->mod_loadcnt; 1381 fbt->fbtp_primary = primary; 1382 fbt->fbtp_symndx = i; 1383 mp->fbt_nentries++; 1384 } 1385 1386 retfbt = NULL; 1387 again: 1388 if (++instr == limit) 1389 continue; 1390 1391 offset = (uintptr_t)instr - (uintptr_t)base; 1392 1393 if (!(*canpatch)(instr, offset, name)) 1394 goto again; 1395 1396 if (estimate) { 1397 tramp.fbtt_next = (uintptr_t)faketramp; 1398 tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp); 1399 (void) (*patch)(instr, base, limit, 1400 offset, FBT_ESTIMATE_ID, &tramp, name); 1401 fbt_size += tramp.fbtt_next - (uintptr_t)faketramp; 1402 1403 goto again; 1404 } 1405 1406 fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP); 1407 fbt->fbtp_name = name; 1408 fbt->fbtp_ctl = ctl; 1409 1410 if (retfbt == NULL) { 1411 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, 1412 name, FBT_PROBENAME_RETURN, 1, fbt); 1413 } else { 1414 retfbt->fbtp_next = fbt; 1415 fbt->fbtp_id = retfbt->fbtp_id; 1416 } 1417 1418 fbt->fbtp_return = 1; 1419 retfbt = fbt; 1420 1421 if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset, 1422 fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) { 1423 cmn_err(CE_WARN, "unexpectedly short FBT table " 1424 "in module %s (sym %d of %d)", modname, i, nsyms); 1425 break; 1426 } 1427 1428 fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin + 1429 ((uintptr_t)instr - (uintptr_t)mp->text)); 1430 fbt->fbtp_savedval = *instr; 1431 fbt->fbtp_loadcnt = ctl->mod_loadcnt; 1432 fbt->fbtp_primary = primary; 1433 fbt->fbtp_symndx = i; 1434 mp->fbt_nentries++; 1435 1436 goto again; 1437 } 1438 1439 if (estimate) { 1440 /* 1441 * Slosh on another entry's worth... 1442 */ 1443 fbt_size += FBT_ENT_MAXSIZE; 1444 mp->fbt_size = fbt_size; 1445 mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size); 1446 1447 if (mp->fbt_tab == NULL) { 1448 cmn_err(CE_WARN, "couldn't allocate FBT table " 1449 "for module %s", modname); 1450 } else { 1451 estimate = 0; 1452 goto forreal; 1453 } 1454 } else { 1455 fbt_trampoline_unmap(); 1456 } 1457 1458 error: 1459 if (fp != NULL) 1460 ctf_close(fp); 1461 } 1462 1463 /*ARGSUSED*/ 1464 static void 1465 fbt_destroy(void *arg, dtrace_id_t id, void *parg) 1466 { 1467 fbt_probe_t *fbt = parg, *next; 1468 struct modctl *ctl = fbt->fbtp_ctl; 1469 1470 do { 1471 if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) { 1472 if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt && 1473 ctl->mod_loaded) || fbt->fbtp_primary) { 1474 ((struct module *) 1475 (ctl->mod_mp))->fbt_nentries--; 1476 } 1477 } 1478 1479 next = fbt->fbtp_next; 1480 kmem_free(fbt, sizeof (fbt_probe_t)); 1481 fbt = next; 1482 } while (fbt != NULL); 1483 } 1484 1485 /*ARGSUSED*/ 1486 static int 1487 fbt_enable(void *arg, dtrace_id_t id, void *parg) 1488 { 1489 fbt_probe_t *fbt = parg, *f; 1490 struct modctl *ctl = fbt->fbtp_ctl; 1491 1492 ctl->mod_nenabled++; 1493 1494 for (f = fbt; f != NULL; f = f->fbtp_next) { 1495 if (f->fbtp_patchpoint == NULL) { 1496 /* 1497 * Due to a shortened FBT table, this entry was never 1498 * completed; refuse to enable it. 1499 */ 1500 if (fbt_verbose) { 1501 cmn_err(CE_NOTE, "fbt is failing for probe %s " 1502 "(short FBT table in %s)", 1503 fbt->fbtp_name, ctl->mod_modname); 1504 } 1505 1506 return (0); 1507 } 1508 } 1509 1510 /* 1511 * If this module has disappeared since we discovered its probes, 1512 * refuse to enable it. 1513 */ 1514 if (!fbt->fbtp_primary && !ctl->mod_loaded) { 1515 if (fbt_verbose) { 1516 cmn_err(CE_NOTE, "fbt is failing for probe %s " 1517 "(module %s unloaded)", 1518 fbt->fbtp_name, ctl->mod_modname); 1519 } 1520 1521 return (0); 1522 } 1523 1524 /* 1525 * Now check that our modctl has the expected load count. If it 1526 * doesn't, this module must have been unloaded and reloaded -- and 1527 * we're not going to touch it. 1528 */ 1529 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) { 1530 if (fbt_verbose) { 1531 cmn_err(CE_NOTE, "fbt is failing for probe %s " 1532 "(module %s reloaded)", 1533 fbt->fbtp_name, ctl->mod_modname); 1534 } 1535 1536 return (0); 1537 } 1538 1539 for (; fbt != NULL; fbt = fbt->fbtp_next) 1540 *fbt->fbtp_patchpoint = fbt->fbtp_patchval; 1541 1542 return (0); 1543 } 1544 1545 /*ARGSUSED*/ 1546 static void 1547 fbt_disable(void *arg, dtrace_id_t id, void *parg) 1548 { 1549 fbt_probe_t *fbt = parg, *f; 1550 struct modctl *ctl = fbt->fbtp_ctl; 1551 1552 ASSERT(ctl->mod_nenabled > 0); 1553 ctl->mod_nenabled--; 1554 1555 for (f = fbt; f != NULL; f = f->fbtp_next) { 1556 if (f->fbtp_patchpoint == NULL) 1557 return; 1558 } 1559 1560 if ((!fbt->fbtp_primary && !ctl->mod_loaded) || 1561 (ctl->mod_loadcnt != fbt->fbtp_loadcnt)) 1562 return; 1563 1564 for (; fbt != NULL; fbt = fbt->fbtp_next) 1565 *fbt->fbtp_patchpoint = fbt->fbtp_savedval; 1566 } 1567 1568 /*ARGSUSED*/ 1569 static void 1570 fbt_suspend(void *arg, dtrace_id_t id, void *parg) 1571 { 1572 fbt_probe_t *fbt = parg; 1573 struct modctl *ctl = fbt->fbtp_ctl; 1574 1575 if (!fbt->fbtp_primary && !ctl->mod_loaded) 1576 return; 1577 1578 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) 1579 return; 1580 1581 ASSERT(ctl->mod_nenabled > 0); 1582 1583 for (; fbt != NULL; fbt = fbt->fbtp_next) 1584 *fbt->fbtp_patchpoint = fbt->fbtp_savedval; 1585 } 1586 1587 /*ARGSUSED*/ 1588 static void 1589 fbt_resume(void *arg, dtrace_id_t id, void *parg) 1590 { 1591 fbt_probe_t *fbt = parg; 1592 struct modctl *ctl = fbt->fbtp_ctl; 1593 1594 if (!fbt->fbtp_primary && !ctl->mod_loaded) 1595 return; 1596 1597 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) 1598 return; 1599 1600 ASSERT(ctl->mod_nenabled > 0); 1601 1602 for (; fbt != NULL; fbt = fbt->fbtp_next) 1603 *fbt->fbtp_patchpoint = fbt->fbtp_patchval; 1604 } 1605 1606 /*ARGSUSED*/ 1607 static void 1608 fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) 1609 { 1610 fbt_probe_t *fbt = parg; 1611 struct modctl *ctl = fbt->fbtp_ctl; 1612 struct module *mp = ctl->mod_mp; 1613 ctf_file_t *fp = NULL, *pfp; 1614 ctf_funcinfo_t f; 1615 int error; 1616 ctf_id_t argv[32], type; 1617 int argc = sizeof (argv) / sizeof (ctf_id_t); 1618 const char *parent; 1619 1620 if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt)) 1621 goto err; 1622 1623 if (fbt->fbtp_return && desc->dtargd_ndx == 0) { 1624 (void) strcpy(desc->dtargd_native, "int"); 1625 return; 1626 } 1627 1628 if ((fp = ctf_modopen(mp, &error)) == NULL) { 1629 /* 1630 * We have no CTF information for this module -- and therefore 1631 * no args[] information. 1632 */ 1633 goto err; 1634 } 1635 1636 /* 1637 * If we have a parent container, we must manually import it. 1638 */ 1639 if ((parent = ctf_parent_name(fp)) != NULL) { 1640 struct modctl *mp = &modules; 1641 struct modctl *mod = NULL; 1642 1643 /* 1644 * We must iterate over all modules to find the module that 1645 * is our parent. 1646 */ 1647 do { 1648 if (strcmp(mp->mod_modname, parent) == 0) { 1649 mod = mp; 1650 break; 1651 } 1652 } while ((mp = mp->mod_next) != &modules); 1653 1654 if (mod == NULL) 1655 goto err; 1656 1657 if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL) 1658 goto err; 1659 1660 if (ctf_import(fp, pfp) != 0) { 1661 ctf_close(pfp); 1662 goto err; 1663 } 1664 1665 ctf_close(pfp); 1666 } 1667 1668 if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR) 1669 goto err; 1670 1671 if (fbt->fbtp_return) { 1672 if (desc->dtargd_ndx > 1) 1673 goto err; 1674 1675 ASSERT(desc->dtargd_ndx == 1); 1676 type = f.ctc_return; 1677 } else { 1678 if (desc->dtargd_ndx + 1 > f.ctc_argc) 1679 goto err; 1680 1681 if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR) 1682 goto err; 1683 1684 type = argv[desc->dtargd_ndx]; 1685 } 1686 1687 if (ctf_type_name(fp, type, desc->dtargd_native, 1688 DTRACE_ARGTYPELEN) != NULL) { 1689 ctf_close(fp); 1690 return; 1691 } 1692 err: 1693 if (fp != NULL) 1694 ctf_close(fp); 1695 1696 desc->dtargd_ndx = DTRACE_ARGNONE; 1697 } 1698 1699 static dtrace_pattr_t fbt_attr = { 1700 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1701 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1702 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1703 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1704 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 1705 }; 1706 1707 static dtrace_pops_t fbt_pops = { 1708 NULL, 1709 fbt_provide_module, 1710 fbt_enable, 1711 fbt_disable, 1712 fbt_suspend, 1713 fbt_resume, 1714 fbt_getargdesc, 1715 NULL, 1716 NULL, 1717 fbt_destroy 1718 }; 1719 1720 static int 1721 fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 1722 { 1723 switch (cmd) { 1724 case DDI_ATTACH: 1725 break; 1726 case DDI_RESUME: 1727 return (DDI_SUCCESS); 1728 default: 1729 return (DDI_FAILURE); 1730 } 1731 1732 if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0, 1733 DDI_PSEUDO, NULL) == DDI_FAILURE || 1734 dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL, 1735 &fbt_pops, NULL, &fbt_id) != 0) { 1736 ddi_remove_minor_node(devi, NULL); 1737 return (DDI_FAILURE); 1738 } 1739 1740 ddi_report_dev(devi); 1741 fbt_devi = devi; 1742 return (DDI_SUCCESS); 1743 } 1744 1745 static int 1746 fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 1747 { 1748 switch (cmd) { 1749 case DDI_DETACH: 1750 break; 1751 case DDI_SUSPEND: 1752 return (DDI_SUCCESS); 1753 default: 1754 return (DDI_FAILURE); 1755 } 1756 1757 if (dtrace_unregister(fbt_id) != 0) 1758 return (DDI_FAILURE); 1759 1760 ddi_remove_minor_node(devi, NULL); 1761 return (DDI_SUCCESS); 1762 } 1763 1764 /*ARGSUSED*/ 1765 static int 1766 fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 1767 { 1768 int error; 1769 1770 switch (infocmd) { 1771 case DDI_INFO_DEVT2DEVINFO: 1772 *result = (void *)fbt_devi; 1773 error = DDI_SUCCESS; 1774 break; 1775 case DDI_INFO_DEVT2INSTANCE: 1776 *result = (void *)0; 1777 error = DDI_SUCCESS; 1778 break; 1779 default: 1780 error = DDI_FAILURE; 1781 } 1782 return (error); 1783 } 1784 1785 /*ARGSUSED*/ 1786 static int 1787 fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 1788 { 1789 return (0); 1790 } 1791 1792 static struct cb_ops fbt_cb_ops = { 1793 fbt_open, /* open */ 1794 nodev, /* close */ 1795 nulldev, /* strategy */ 1796 nulldev, /* print */ 1797 nodev, /* dump */ 1798 nodev, /* read */ 1799 nodev, /* write */ 1800 nodev, /* ioctl */ 1801 nodev, /* devmap */ 1802 nodev, /* mmap */ 1803 nodev, /* segmap */ 1804 nochpoll, /* poll */ 1805 ddi_prop_op, /* cb_prop_op */ 1806 0, /* streamtab */ 1807 D_NEW | D_MP /* Driver compatibility flag */ 1808 }; 1809 1810 static struct dev_ops fbt_ops = { 1811 DEVO_REV, /* devo_rev */ 1812 0, /* refcnt */ 1813 fbt_info, /* get_dev_info */ 1814 nulldev, /* identify */ 1815 nulldev, /* probe */ 1816 fbt_attach, /* attach */ 1817 fbt_detach, /* detach */ 1818 nodev, /* reset */ 1819 &fbt_cb_ops, /* driver operations */ 1820 NULL, /* bus operations */ 1821 nodev, /* dev power */ 1822 ddi_quiesce_not_needed, /* quiesce */ 1823 }; 1824 1825 /* 1826 * Module linkage information for the kernel. 1827 */ 1828 static struct modldrv modldrv = { 1829 &mod_driverops, /* module type (this is a pseudo driver) */ 1830 "Function Boundary Tracing", /* name of module */ 1831 &fbt_ops, /* driver ops */ 1832 }; 1833 1834 static struct modlinkage modlinkage = { 1835 MODREV_1, 1836 (void *)&modldrv, 1837 NULL 1838 }; 1839 1840 int 1841 _init(void) 1842 { 1843 return (mod_install(&modlinkage)); 1844 } 1845 1846 int 1847 _info(struct modinfo *modinfop) 1848 { 1849 return (mod_info(&modlinkage, modinfop)); 1850 } 1851 1852 int 1853 _fini(void) 1854 { 1855 return (mod_remove(&modlinkage)); 1856 } 1857