/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include <sys/errno.h> #include <sys/stat.h> #include <sys/modctl.h> #include <sys/conf.h> #include <sys/systm.h> #include <sys/ddi.h> #include <sys/sunddi.h> #include <sys/cpuvar.h> #include <sys/kmem.h> #include <sys/strsubr.h> #include <sys/dtrace.h> #include <sys/kobj.h> #include <sys/modctl.h> #include <sys/atomic.h> #include <vm/seg_kmem.h> #include <sys/stack.h> #include <sys/ctf_api.h> #include <sys/sysmacros.h> static dev_info_t *fbt_devi; static dtrace_provider_id_t fbt_id; static uintptr_t fbt_trampoline; static caddr_t fbt_trampoline_window; static size_t fbt_trampoline_size; static int fbt_verbose = 0; /* * Various interesting bean counters. */ static int fbt_entry; static int fbt_ret; static int fbt_retl; static int fbt_retl_jmptab; static int fbt_retl_twoinstr; static int fbt_retl_tailcall; static int fbt_retl_tailjmpl; static int fbt_leaf_functions; extern char stubs_base[]; extern char stubs_end[]; #define FBT_REG_G0 0 #define FBT_REG_G1 1 #define FBT_REG_O0 8 #define FBT_REG_O1 9 #define FBT_REG_O2 10 #define FBT_REG_O3 11 #define FBT_REG_O4 12 #define FBT_REG_O5 13 #define FBT_REG_O6 14 #define FBT_REG_O7 15 #define FBT_REG_I0 24 #define FBT_REG_I1 25 #define FBT_REG_I2 26 #define FBT_REG_I3 27 #define FBT_REG_I4 28 #define FBT_REG_I7 31 #define FBT_REG_L0 16 #define FBT_REG_L1 17 #define FBT_REG_L2 18 #define FBT_REG_L3 19 #define FBT_REG_PC 5 #define FBT_REG_ISGLOBAL(r) ((r) < 8) #define FBT_REG_ISOUTPUT(r) ((r) >= 8 && (r) < 16) #define FBT_REG_ISLOCAL(r) ((r) >= 16 && (r) < 24) #define FBT_REG_ISVOLATILE(r) \ ((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0) #define FBT_REG_NLOCALS 8 #define FBT_REG_MARKLOCAL(locals, r) \ if (FBT_REG_ISLOCAL(r)) \ (locals)[(r) - FBT_REG_L0] = 1; #define FBT_REG_INITLOCALS(local, locals) \ for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++) \ (locals)[(local)] = 0; \ (local) = FBT_REG_L0 #define FBT_REG_ALLOCLOCAL(local, locals) \ while ((locals)[(local) - FBT_REG_L0]) \ (local)++; \ (locals)[(local) - FBT_REG_L0] = 1; #define FBT_OP_MASK 0xc0000000 #define FBT_OP_SHIFT 30 #define FBT_OP(val) ((val) & FBT_FMT1_MASK) #define FBT_SIMM13_MASK 0x1fff #define FBT_SIMM13_MAX ((int32_t)0xfff) #define FBT_IMM22_MASK 0x3fffff #define FBT_IMM22_SHIFT 10 #define FBT_IMM10_MASK 0x3ff #define FBT_DISP30_MASK 0x3fffffff #define FBT_DISP30(from, to) \ (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK) #define FBT_DISP22_MASK 0x3fffff #define FBT_DISP22(from, to) \ (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK) #define FBT_DISP19_MASK 0x7ffff #define FBT_DISP19(from, to) \ (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK) #define FBT_DISP16_HISHIFT 20 #define FBT_DISP16_HIMASK (0x3 << FBT_DISP16_HISHIFT) #define FBT_DISP16_LOMASK (0x3fff) #define FBT_DISP16_MASK (FBT_DISP16_HIMASK | FBT_DISP16_LOMASK) #define FBT_DISP16(val) \ ((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK)) #define FBT_DISP14_MASK 0x3fff #define FBT_DISP14(from, to) \ (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK) #define FBT_OP0 (((uint32_t)0) << FBT_OP_SHIFT) #define FBT_OP1 (((uint32_t)1) << FBT_OP_SHIFT) #define FBT_OP2 (((uint32_t)2) << FBT_OP_SHIFT) #define FBT_ILLTRAP 0 #define FBT_ANNUL_SHIFT 29 #define FBT_ANNUL (1 << FBT_ANNUL_SHIFT) #define FBT_FMT3_OP3_SHIFT 19 #define FBT_FMT3_OP_MASK 0xc1f80000 #define FBT_FMT3_OP(val) ((val) & FBT_FMT3_OP_MASK) #define FBT_FMT3_RD_SHIFT 25 #define FBT_FMT3_RD_MASK (0x1f << FBT_FMT3_RD_SHIFT) #define FBT_FMT3_RD(val) \ (((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT) #define FBT_FMT3_RS1_SHIFT 14 #define FBT_FMT3_RS1_MASK (0x1f << FBT_FMT3_RS1_SHIFT) #define FBT_FMT3_RS1(val) \ (((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT) #define FBT_FMT3_RS1_SET(val, rs1) \ (val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT) #define FBT_FMT3_RS2_SHIFT 0 #define FBT_FMT3_RS2_MASK (0x1f << FBT_FMT3_RS2_SHIFT) #define FBT_FMT3_RS2(val) \ (((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT) #define FBT_FMT3_RS2_SET(val, rs2) \ (val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT) #define FBT_FMT3_IMM_SHIFT 13 #define FBT_FMT3_IMM (1 << FBT_FMT3_IMM_SHIFT) #define FBT_FMT3_SIMM13_MASK FBT_SIMM13_MASK #define FBT_FMT3_ISIMM(val) ((val) & FBT_FMT3_IMM) #define FBT_FMT3_SIMM13(val) ((val) & FBT_FMT3_SIMM13_MASK) #define FBT_FMT2_OP2_SHIFT 22 #define FBT_FMT2_OP2_MASK (0x7 << FBT_FMT2_OP2_SHIFT) #define FBT_FMT2_RD_SHIFT 25 #define FBT_FMT1_OP(val) ((val) & FBT_OP_MASK) #define FBT_FMT1_DISP30(val) ((val) & FBT_DISP30_MASK) #define FBT_FMT2_OP2_BPCC (0x01 << FBT_FMT2_OP2_SHIFT) #define FBT_FMT2_OP2_BCC (0x02 << FBT_FMT2_OP2_SHIFT) #define FBT_FMT2_OP2_BPR (0x03 << FBT_FMT2_OP2_SHIFT) #define FBT_FMT2_OP2_SETHI (0x04 << FBT_FMT2_OP2_SHIFT) #define FBT_FMT2_COND_SHIFT 25 #define FBT_FMT2_COND_BA (0x8 << FBT_FMT2_COND_SHIFT) #define FBT_FMT2_COND_BL (0x3 << FBT_FMT2_COND_SHIFT) #define FBT_FMT2_COND_BGE (0xb << FBT_FMT2_COND_SHIFT) #define FBT_OP_RESTORE (FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_SAVE (FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_JMPL (FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_RETURN (FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_CALL FBT_OP1 #define FBT_OP_SETHI (FBT_OP0 | FBT_FMT2_OP2_SETHI) #define FBT_OP_ADD (FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_OR (FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_SUB (FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_CC (FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT)) #define FBT_OP_BA (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA) #define FBT_OP_BL (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL) #define FBT_OP_BGE (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE) #define FBT_OP_BAPCC (FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA) #define FBT_OP_RD (FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT)) #define FBT_ORLO(rs, val, rd) \ (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \ ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK)) #define FBT_ORSIMM13(rs, val, rd) \ (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \ ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) #define FBT_ADDSIMM13(rs, val, rd) \ (FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \ ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) #define FBT_ADD(rs1, rs2, rd) \ (FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ ((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT)) #define FBT_CMP(rs1, rs2) \ (FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ ((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT)) #define FBT_MOV(rs, rd) \ (FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \ ((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT)) #define FBT_SETHI(val, reg) \ (FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \ ((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK)) #define FBT_CALL(orig, dest) (FBT_OP_CALL | FBT_DISP30(orig, dest)) #define FBT_RET \ (FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \ (FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1)) #define FBT_SAVEIMM(rd, val, rs1) \ (FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) #define FBT_RESTORE(rd, rs1, rs2) \ (FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ ((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT)) #define FBT_RETURN(rs1, val) \ (FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \ FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK)) #define FBT_BA(orig, dest) (FBT_OP_BA | FBT_DISP22(orig, dest)) #define FBT_BAA(orig, dest) (FBT_BA(orig, dest) | FBT_ANNUL) #define FBT_BL(orig, dest) (FBT_OP_BL | FBT_DISP22(orig, dest)) #define FBT_BGE(orig, dest) (FBT_OP_BGE | FBT_DISP22(orig, dest)) #define FBT_BDEST(va, instr) ((uintptr_t)(va) + \ (((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8)) #define FBT_BPCCDEST(va, instr) ((uintptr_t)(va) + \ (((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11)) #define FBT_BPRDEST(va, instr) ((uintptr_t)(va) + \ (((int32_t)((FBT_DISP16(instr)) << 16)) >> 14)) /* * We're only going to treat a save as safe if (a) both rs1 and rd are * %sp and (b) if the instruction has a simm, the value isn't 0. */ #define FBT_IS_SAVE(instr) \ (FBT_FMT3_OP(instr) == FBT_OP_SAVE && \ FBT_FMT3_RD(instr) == FBT_REG_O6 && \ FBT_FMT3_RS1(instr) == FBT_REG_O6 && \ !(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0)) #define FBT_IS_BA(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA) #define FBT_IS_BAPCC(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC) #define FBT_IS_RDPC(instr) ((FBT_FMT3_OP(instr) == FBT_OP_RD) && \ (FBT_FMT3_RD(instr) == FBT_REG_PC)) #define FBT_IS_PCRELATIVE(instr) \ ((((instr) & FBT_OP_MASK) == FBT_OP0 && \ ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \ ((instr) & FBT_OP_MASK) == FBT_OP1 || \ FBT_IS_RDPC(instr)) #define FBT_IS_CTI(instr) \ ((((instr) & FBT_OP_MASK) == FBT_OP0 && \ ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \ ((instr) & FBT_OP_MASK) == FBT_OP1 || \ (FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \ (FBT_FMT3_OP(instr) == FBT_OP_RETURN)) #define FBT_PROBENAME_ENTRY "entry" #define FBT_PROBENAME_RETURN "return" #define FBT_ESTIMATE_ID (UINT32_MAX) #define FBT_COUNTER(id, count) if ((id) != FBT_ESTIMATE_ID) (count)++ #define FBT_ENTENT_MAXSIZE (16 * sizeof (uint32_t)) #define FBT_RETENT_MAXSIZE (11 * sizeof (uint32_t)) #define FBT_RETLENT_MAXSIZE (23 * sizeof (uint32_t)) #define FBT_ENT_MAXSIZE \ MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE) typedef struct fbt_probe { char *fbtp_name; dtrace_id_t fbtp_id; uintptr_t fbtp_addr; struct modctl *fbtp_ctl; int fbtp_loadcnt; int fbtp_symndx; int fbtp_primary; int fbtp_return; uint32_t *fbtp_patchpoint; uint32_t fbtp_patchval; uint32_t fbtp_savedval; struct fbt_probe *fbtp_next; } fbt_probe_t; typedef struct fbt_trampoline { uintptr_t fbtt_va; uintptr_t fbtt_limit; uintptr_t fbtt_next; } fbt_trampoline_t; static caddr_t fbt_trampoline_map(uintptr_t tramp, size_t size) { uintptr_t offs; page_t **ppl; ASSERT(fbt_trampoline_window == NULL); ASSERT(fbt_trampoline_size == 0); ASSERT(fbt_trampoline == NULL); size += tramp & PAGEOFFSET; fbt_trampoline = tramp & PAGEMASK; fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK; fbt_trampoline_window = vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP); (void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size, S_WRITE); for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) { hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE, hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs), PROT_READ | PROT_WRITE, HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); } as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size, S_WRITE); return (fbt_trampoline_window + (tramp & PAGEOFFSET)); } static void fbt_trampoline_unmap() { ASSERT(fbt_trampoline_window != NULL); ASSERT(fbt_trampoline_size != 0); ASSERT(fbt_trampoline != NULL); membar_enter(); sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size); sync_icache(fbt_trampoline_window, fbt_trampoline_size); hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size, HAT_UNLOAD_UNLOCK); vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size); fbt_trampoline_window = NULL; fbt_trampoline = NULL; fbt_trampoline_size = 0; } static uintptr_t fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp, int nargs) { uint32_t *tinstr = (uint32_t *)tramp->fbtt_next; uint32_t first = *instr; uintptr_t va = tramp->fbtt_va; uintptr_t base = tramp->fbtt_next; if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) { /* * There isn't sufficient room for this entry; return failure. */ return (0); } FBT_COUNTER(id, fbt_entry); if (FBT_IS_SAVE(first)) { *tinstr++ = first; } else { *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6); } if (id > (uint32_t)FBT_SIMM13_MAX) { *tinstr++ = FBT_SETHI(id, FBT_REG_O0); *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0); } else { *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0); } if (nargs >= 1) *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1); if (nargs >= 2) *tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2); if (nargs >= 3) *tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3); if (nargs >= 4) *tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4); if (nargs >= 5) *tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5); if (FBT_IS_SAVE(first)) { uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t); *tinstr++ = FBT_SETHI(ret, FBT_REG_G1); *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); tinstr++; *tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7); } else { uintptr_t slot = *--tinstr; uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t); uint32_t delay = first; *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); tinstr++; *tinstr++ = slot; *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0); if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) { /* * This is a special case: we are instrumenting a * a non-annulled branch-always (or variant). We'll * return directly to the destination of the branch, * copying the instruction in the delay slot here, * and then executing it in the slot of a ba. */ if (FBT_IS_BA(first)) { ret = FBT_BDEST(instr, *instr); } else { ret = FBT_BPCCDEST(instr, *instr); } delay = *(instr + 1); } if ((first & FBT_OP_MASK) != FBT_OP0 || (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) { *tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret); tinstr++; *tinstr++ = delay; } else { /* * If this is a branch-on-register, we have a little * more work to do: because the displacement is only * sixteen bits, we're going to thunk the branch into * the trampoline, and then ba,a to the appropriate * destination in the branch targets. That is, we're * constructing this sequence in the trampoline: * * br[cc] %[rs], 1f * <delay-instruction> * ba,a <not-taken-destination> * 1: ba,a <taken-destination> * */ uintptr_t targ = FBT_BPRDEST(instr, first); *tinstr = first & ~(FBT_DISP16_MASK); *tinstr |= FBT_DISP14(tinstr, &tinstr[3]); tinstr++; *tinstr++ = *(instr + 1); *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, ret + sizeof (uint32_t)); tinstr++; *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ); tinstr++; } } tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next; tramp->fbtt_next = (uintptr_t)tinstr; return (1); } /* * We are patching control-transfer/restore couplets. There are three * variants of couplet: * * (a) return rs1 + imm * delay * * (b) jmpl rs1 + (rs2 | offset), rd * restore rs1, rs2 | imm, rd * * (c) call displacement * restore rs1, rs2 | imm, rd * * If rs1 in (a) is anything other than %i7, or imm is anything other than 8, * or delay is a DCTI, we fail. If rd from the jmpl in (b) is something other * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call * through a register), we fail. * * Note that rs1 and rs2 in the restore instructions in (b) and (c) are * potentially outputs and/or globals. Because these registers cannot be * relied upon across the call to dtrace_probe(), we move rs1 into an unused * local, ls0, and rs2 into an unused local, ls1, and restructure the restore * to be: * * restore ls0, ls1, rd * * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals. * If the jmpl uses outputs or globals, we restructure it to be: * * jmpl ls2 + (ls3 | offset), (%g0 | %o7) * */ /*ARGSUSED*/ static int fbt_canpatch_return(uint32_t *instr, int offset, const char *name) { int rd; if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) { uint32_t delay = *(instr + 1); if (*instr != FBT_RETURN(FBT_REG_I7, 8)) { /* * It's unclear if we should warn about this or not. * We really wouldn't expect the compiler to generate * return instructions with something other than %i7 * as rs1 and 8 as the simm13 -- it would just be * mean-spirited. That said, such a construct isn't * necessarily incorrect. Sill, we err on the side of * caution and warn about it... */ cmn_err(CE_NOTE, "cannot instrument return of %s at " "%p: non-canonical return instruction", name, (void *)instr); return (0); } if (FBT_IS_CTI(delay)) { /* * This is even weirder -- a DCTI coupled with a * return instruction. Similar constructs are used to * return from utraps, but these typically have the * return in the slot -- and we wouldn't expect to see * it in the kernel regardless. At any rate, we don't * want to try to instrument this construct, whatever * it may be. */ cmn_err(CE_NOTE, "cannot instrument return of %s at " "%p: CTI in delay slot of return instruction", name, (void *)instr); return (0); } if (FBT_IS_PCRELATIVE(delay)) { /* * This is also very weird, but might be correct code * if the function is (for example) returning the * address of the delay instruction of the return as * its return value (e.g. "rd %pc, %o0" in the slot). * Perhaps correct, but still too weird to not warn * about it... */ cmn_err(CE_NOTE, "cannot instrument return of %s at " "%p: PC-relative instruction in delay slot of " "return instruction", name, (void *)instr); return (0); } return (1); } if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE) return (0); if (FBT_FMT1_OP(*instr) == FBT_OP_CALL) return (1); if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL) return (0); rd = FBT_FMT3_RD(*instr); if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0) return (1); /* * We have encountered a jmpl that is storing the calling %pc in * some register besides %i7, %o7 or %g0. This is strange; emit * a warning and fail. */ cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected " "jmpl destination register", name, (void *)instr); return (0); } static int fbt_canpatch_retl(uint32_t *instr, int offset, const char *name) { if (FBT_FMT1_OP(*instr) == FBT_OP_CALL || (FBT_FMT3_OP(*instr) == FBT_OP_JMPL && FBT_FMT3_RD(*instr) == FBT_REG_O7)) { /* * If this is a call (or a jmpl that links into %o7), we can * patch it iff the next instruction uses %o7 as a destination * register. Because there is an ABI responsibility to * restore %o7 to the value before the call/jmpl, we don't * particularly care how this routine is managing to restore * it (mov, add, ld or divx for all we care). If it doesn't * seem to be restoring it at all, however, we'll refuse * to patch it. */ uint32_t delay = *(instr + 1); uint32_t op, rd; op = FBT_FMT1_OP(delay); rd = FBT_FMT3_RD(delay); if (op != FBT_OP2 || rd != FBT_REG_O7) { /* * This is odd. Before we assume that we're looking * at something bizarre (and warn accordingly), we'll * check to see if it's obviously a jump table entry. */ if (*instr < (uintptr_t)instr && *instr >= (uintptr_t)instr - offset) return (0); cmn_err(CE_NOTE, "cannot instrument return of %s at " "%p: leaf jmpl/call delay isn't restoring %%o7", name, (void *)instr); return (0); } return (1); } if (offset == sizeof (uint32_t)) { /* * If this is the second instruction in the function, we're * going to allow it to be patched if the first instruction * is a patchable return-from-leaf instruction. */ if (fbt_canpatch_retl(instr - 1, 0, name)) return (1); } if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL) return (0); if (FBT_FMT3_RD(*instr) != FBT_REG_G0) return (0); return (1); } /*ARGSUSED*/ static uint32_t fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim, int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name) { uint32_t *tinstr = (uint32_t *)tramp->fbtt_next; uint32_t cti = *instr, restore = *(instr + 1), rs1, dest; uintptr_t va = tramp->fbtt_va; uintptr_t base = tramp->fbtt_next; uint32_t locals[FBT_REG_NLOCALS], local; if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) { /* * There isn't sufficient room for this entry; return failure. */ return (FBT_ILLTRAP); } FBT_COUNTER(id, fbt_ret); if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) { /* * To handle the case of the return instruction, we'll emit a * restore, followed by the instruction in the slot (which * we'll transplant here), and then another save. While it * may seem intellectually unsatisfying to emit the additional * restore/save couplet, one can take solace in the fact that * we don't do this if the instruction in the return delay * slot is a nop -- which it is nearly 90% of the time with * gcc. (And besides, this couplet can't induce unnecessary * spill/fill traps; rewriting the delay instruction to be * in terms of the current window hardly seems worth the * trouble -- let alone the risk.) */ uint32_t delay = *(instr + 1); ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8)); cti = FBT_RET; restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0); if (delay != FBT_SETHI(0, FBT_REG_G0)) { *tinstr++ = restore; *tinstr++ = delay; *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6); } } FBT_REG_INITLOCALS(local, locals); /* * Mark the locals used in the jmpl. */ if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { uint32_t rs1 = FBT_FMT3_RS1(cti); FBT_REG_MARKLOCAL(locals, rs1); if (!FBT_FMT3_ISIMM(cti)) { uint32_t rs2 = FBT_FMT3_RS2(cti); FBT_REG_MARKLOCAL(locals, rs2); } } /* * And mark the locals used in the restore. */ rs1 = FBT_FMT3_RS1(restore); FBT_REG_MARKLOCAL(locals, rs1); if (!FBT_FMT3_ISIMM(restore)) { uint32_t rs2 = FBT_FMT3_RS2(restore); FBT_REG_MARKLOCAL(locals, rs2); } if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { uint32_t rs1 = FBT_FMT3_RS1(cti); if (FBT_REG_ISVOLATILE(rs1)) { FBT_REG_ALLOCLOCAL(local, locals); FBT_FMT3_RS1_SET(cti, local); *tinstr++ = FBT_MOV(rs1, local); } if (!FBT_FMT3_ISIMM(cti)) { uint32_t rs2 = FBT_FMT3_RS2(cti); if (FBT_REG_ISVOLATILE(rs2)) { FBT_REG_ALLOCLOCAL(local, locals); FBT_FMT3_RS2_SET(cti, local); *tinstr++ = FBT_MOV(rs2, local); } } } rs1 = FBT_FMT3_RS1(restore); if (FBT_REG_ISVOLATILE(rs1)) { FBT_REG_ALLOCLOCAL(local, locals); FBT_FMT3_RS1_SET(restore, local); *tinstr++ = FBT_MOV(rs1, local); } if (!FBT_FMT3_ISIMM(restore)) { uint32_t rs2 = FBT_FMT3_RS2(restore); if (FBT_REG_ISVOLATILE(rs2)) { FBT_REG_ALLOCLOCAL(local, locals); FBT_FMT3_RS2_SET(restore, local); *tinstr++ = FBT_MOV(rs2, local); } } if (id > (uint32_t)FBT_SIMM13_MAX) { *tinstr++ = FBT_SETHI(id, FBT_REG_O0); *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0); } else { *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0); } if (offset > (uint32_t)FBT_SIMM13_MAX) { *tinstr++ = FBT_SETHI(offset, FBT_REG_O1); *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1); } else { *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1); } *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); tinstr++; if (FBT_FMT3_RD(restore) == FBT_REG_O0) { /* * If the destination register of the restore is %o0, we * need to perform the implied calculation to derive the * return value. */ uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD; add &= ~FBT_FMT3_RD_MASK; *tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT); } else { *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2); } /* * If the control transfer instruction is %pc-relative (i.e. a * call), we need to reset it appropriately. */ if (FBT_FMT1_OP(cti) == FBT_OP_CALL) { dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2); *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest); tinstr++; } else { *tinstr++ = cti; } *tinstr++ = restore; tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next; tramp->fbtt_next = (uintptr_t)tinstr; return (FBT_BAA(instr, va)); } static uint32_t fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim, int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name) { uint32_t *tinstr = (uint32_t *)tramp->fbtt_next; uintptr_t va = tramp->fbtt_va; uintptr_t base = tramp->fbtt_next; uint32_t cti = *instr, dest; int annul = 0; FBT_COUNTER(id, fbt_retl); if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) { /* * There isn't sufficient room for this entry; return failure. */ return (FBT_ILLTRAP); } if (offset == sizeof (uint32_t) && fbt_canpatch_retl(instr - 1, 0, name)) { *tinstr++ = *instr; annul = 1; FBT_COUNTER(id, fbt_retl_twoinstr); } else { if (FBT_FMT3_OP(cti) == FBT_OP_JMPL && FBT_FMT3_RD(cti) != FBT_REG_O7 && FBT_FMT3_RS1(cti) != FBT_REG_O7) { annul = 1; *tinstr++ = *(instr + 1); } } *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6); if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0; /* * If we have a jmpl and it's in terms of output registers, we * need to rewrite it to be in terms of the corresponding input * registers. If it's in terms of the globals, we'll rewrite * it to be in terms of locals. */ rs1 = FBT_FMT3_RS1(cti); if (FBT_REG_ISOUTPUT(rs1)) rs1 += o2i; if (FBT_REG_ISGLOBAL(rs1)) { *tinstr++ = FBT_MOV(rs1, FBT_REG_L0); rs1 = FBT_REG_L0; } FBT_FMT3_RS1_SET(cti, rs1); if (!FBT_FMT3_ISIMM(cti)) { rs2 = FBT_FMT3_RS2(cti); if (FBT_REG_ISOUTPUT(rs2)) rs2 += o2i; if (FBT_REG_ISGLOBAL(rs2)) { *tinstr++ = FBT_MOV(rs2, FBT_REG_L1); rs2 = FBT_REG_L1; } FBT_FMT3_RS2_SET(cti, rs2); } /* * Now we need to check the rd and source register for the jmpl; * If neither rd nor the source register is %o7, then we might * have a jmp that is actually part of a jump table. We need * to generate the code to compare it to the base and limit of * the function. */ if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) { uintptr_t base = (uintptr_t)funcbase; uintptr_t limit = (uintptr_t)funclim; FBT_COUNTER(id, fbt_retl_jmptab); if (FBT_FMT3_ISIMM(cti)) { *tinstr++ = FBT_ADDSIMM13(rs1, FBT_FMT3_SIMM13(cti), FBT_REG_L2); } else { *tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2); } *tinstr++ = FBT_SETHI(base, FBT_REG_L3); *tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3); *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3); *tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t)); *tinstr++ = FBT_SETHI(limit, FBT_REG_L3); *tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3); *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3); *tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t)); *tinstr++ = FBT_SETHI(0, FBT_REG_G0); *tinstr++ = cti; *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0); } } if (id > (uint32_t)FBT_SIMM13_MAX) { *tinstr++ = FBT_SETHI(id, FBT_REG_O0); *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0); } else { *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0); } if (offset > (uint32_t)FBT_SIMM13_MAX) { *tinstr++ = FBT_SETHI(offset, FBT_REG_O1); *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1); } else { *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1); } *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe); tinstr++; *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2); /* * If the control transfer instruction is %pc-relative (i.e. a * call), we need to reset it appropriately. */ if (FBT_FMT1_OP(cti) == FBT_OP_CALL) { FBT_COUNTER(id, fbt_retl_tailcall); dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2); *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest); tinstr++; annul = 1; } else { if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) { *tinstr++ = cti; if (FBT_FMT3_RD(cti) == FBT_REG_O7) { FBT_COUNTER(id, fbt_retl_tailjmpl); annul = 1; } } else { *tinstr++ = FBT_RET; } } *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0); tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next; tramp->fbtt_next = (uintptr_t)tinstr; return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va)); } /*ARGSUSED*/ static void fbt_provide_module(void *arg, struct modctl *ctl) { struct module *mp = ctl->mod_mp; char *modname = ctl->mod_modname; char *str = mp->strings; int nsyms = mp->nsyms; Shdr *symhdr = mp->symhdr; size_t symsize; char *name; int i; fbt_probe_t *fbt, *retfbt; fbt_trampoline_t tramp; uintptr_t offset; int primary = 0; ctf_file_t *fp = NULL; int error; int estimate = 1; uint32_t faketramp[50]; size_t fbt_size = 0; /* * Employees of dtrace and their families are ineligible. Void * where prohibited. */ if (strcmp(modname, "dtrace") == 0) return; if (ctl->mod_requisites != NULL) { struct modctl_list *list; list = (struct modctl_list *)ctl->mod_requisites; for (; list != NULL; list = list->modl_next) { if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0) return; } } /* * KMDB is ineligible for instrumentation -- it may execute in * any context, including probe context. */ if (strcmp(modname, "kmdbmod") == 0) return; if (str == NULL || symhdr == NULL || symhdr->sh_addr == NULL) { /* * If this module doesn't (yet) have its string or symbol * table allocated, clear out. */ return; } symsize = symhdr->sh_entsize; if (mp->fbt_nentries) { /* * This module has some FBT entries allocated; we're afraid * to screw with it. */ return; } if (mp->fbt_tab != NULL) estimate = 0; /* * This is a hack for unix/genunix/krtld. */ primary = vmem_contains(heap_arena, (void *)ctl, sizeof (struct modctl)) == 0; kobj_textwin_alloc(mp); /* * Open the CTF data for the module. We'll use this to determine the * functions that can be instrumented. Note that this call can fail, * in which case we'll use heuristics to determine the functions that * can be instrumented. (But in particular, leaf functions will not be * instrumented.) */ fp = ctf_modopen(mp, &error); forreal: if (!estimate) { tramp.fbtt_next = (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab, mp->fbt_size); tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size; tramp.fbtt_va = (uintptr_t)mp->fbt_tab; } for (i = 1; i < nsyms; i++) { ctf_funcinfo_t f; uint32_t *instr, *base, *limit; Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize); int have_ctf = 0, is_leaf = 0, nargs, cti = 0; int (*canpatch)(uint32_t *, int, const char *); uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int, uint32_t, fbt_trampoline_t *, const char *); if (ELF_ST_TYPE(sym->st_info) != STT_FUNC) continue; /* * Weak symbols are not candidates. This could be made to * work (where weak functions and their underlying function * appear as two disjoint probes), but it's not simple. */ if (ELF_ST_BIND(sym->st_info) == STB_WEAK) continue; name = str + sym->st_name; if (strstr(name, "dtrace_") == name && strstr(name, "dtrace_safe_") != name) { /* * Anything beginning with "dtrace_" may be called * from probe context unless it explitly indicates * that it won't be called from probe context by * using the prefix "dtrace_safe_". */ continue; } if (strstr(name, "kdi_") == name || strstr(name, "_kdi_") != NULL) { /* * Any function name beginning with "kdi_" or * containing the string "_kdi_" is a part of the * kernel debugger interface and may be called in * arbitrary context -- including probe context. */ continue; } if (strstr(name, "__relocatable") != NULL) { /* * Anything with the string "__relocatable" anywhere * in the function name is considered to be a function * that may be manually relocated before execution. * Because FBT uses a PC-relative technique for * instrumentation, these functions cannot safely * be instrumented by us. */ continue; } if (strstr(name, "ip_ocsum") == name) { /* * The ip_ocsum_* family of routines are all ABI * violators. (They expect incoming arguments in the * globals!) Break the ABI? No soup for you! */ continue; } /* * We want to scan the function for one (and only one) save. * Any more indicates that something fancy is going on. */ base = (uint32_t *)sym->st_value; limit = (uint32_t *)(sym->st_value + sym->st_size); /* * We don't want to interpose on the module stubs. */ if (base >= (uint32_t *)stubs_base && base <= (uint32_t *)stubs_end) continue; /* * We can't safely trace a zero-length function... */ if (base == limit) continue; /* * Due to 4524008, _init and _fini may have a bloated st_size. * While this bug was fixed quite some time ago, old drivers * may be lurking. We need to develop a better solution to * this problem, such that correct _init and _fini functions * (the vast majority) may be correctly traced. One solution * may be to scan through the entire symbol table to see if * any symbol overlaps with _init. If none does, set a bit in * the module structure that this module has correct _init and * _fini sizes. This will cause some pain the first time a * module is scanned, but at least it would be O(N) instead of * O(N log N)... */ if (strcmp(name, "_init") == 0) continue; if (strcmp(name, "_fini") == 0) continue; instr = base; /* * While we try hard to only trace safe functions (that is, * functions at TL=0), one unsafe function manages to otherwise * appear safe: prom_trap(). We could discover prom_trap() * if we added an additional rule: in order to trace a * function, we must either (a) discover a restore or (b) * determine that the function does not have any unlinked * control transfers to another function (i.e., the function * never returns). Unfortunately, as of this writing, one * legitimate function (resume_from_zombie()) transfers * control to a different function (_resume_from_idle()) * without executing a restore. Barring a rule to figure out * that resume_from_zombie() is safe while prom_trap() is not, * we resort to hard-coding prom_trap() here. */ if (strcmp(name, "prom_trap") == 0) continue; if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) { nargs = f.ctc_argc; have_ctf = 1; } else { nargs = 32; } /* * If the first instruction of the function is a branch and * it's not a branch-always-not-annulled, we're going to refuse * to patch it. */ if ((*instr & FBT_OP_MASK) == FBT_OP0 && (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI && (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) { if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) { if (have_ctf) { cmn_err(CE_NOTE, "cannot instrument %s:" " begins with non-ba, " "non-br CTI", name); } continue; } } while (!FBT_IS_SAVE(*instr)) { /* * Before we assume that this is a leaf routine, check * forward in the basic block for a save. */ int op = *instr & FBT_OP_MASK; int op2 = *instr & FBT_FMT2_OP2_MASK; if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) { /* * This is a CTI. If we see a subsequent * save, we will refuse to process this * routine unless both of the following are * true: * * (a) The branch is not annulled * * (b) The subsequent save is in the delay * slot of the branch */ if ((*instr & FBT_ANNUL) || !FBT_IS_SAVE(*(instr + 1))) { cti = 1; } else { instr++; break; } } if (op == FBT_OP1) cti = 1; if (++instr == limit) break; } if (instr < limit && cti) { /* * If we found a CTI before the save, we need to not * do anything. But if we have CTF information, this * is weird enough that it merits a message. */ if (!have_ctf) continue; cmn_err(CE_NOTE, "cannot instrument %s: " "save not in first basic block", name); continue; } if (instr == limit) { if (!have_ctf) continue; is_leaf = 1; if (!estimate) fbt_leaf_functions++; canpatch = fbt_canpatch_retl; patch = fbt_patch_retl; } else { canpatch = fbt_canpatch_return; patch = fbt_patch_return; } if (!have_ctf && !is_leaf) { /* * Before we assume that this isn't something tricky, * look for other saves. If we find them, there are * multiple entry points here (or something), and we'll * leave it alone. */ while (++instr < limit) { if (FBT_IS_SAVE(*instr)) break; } if (instr != limit) continue; } instr = base; if (FBT_IS_CTI(*instr)) { /* * If we have a CTI, we want to be sure that we don't * have a CTI or a PC-relative instruction in the * delay slot -- we want to be able to thunk the * instruction into the trampoline without worrying * about either DCTIs or relocations. It would be * very odd for the compiler to generate this kind of * code, so we warn about it if we have CTF * information. */ if (FBT_IS_CTI(*(instr + 1))) { if (!have_ctf) continue; cmn_err(CE_NOTE, "cannot instrument %s: " "CTI in delay slot of first instruction", name); continue; } if (FBT_IS_PCRELATIVE(*(instr + 1))) { if (!have_ctf) continue; cmn_err(CE_NOTE, "cannot instrument %s: " "PC-relative instruction in delay slot of" " first instruction", name); continue; } } if (estimate) { tramp.fbtt_next = (uintptr_t)faketramp; tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp); (void) fbt_patch_entry(instr, FBT_ESTIMATE_ID, &tramp, nargs); fbt_size += tramp.fbtt_next - (uintptr_t)faketramp; } else { fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP); fbt->fbtp_name = name; fbt->fbtp_ctl = ctl; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_PROBENAME_ENTRY, 1, fbt); fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va); if (!fbt_patch_entry(instr, fbt->fbtp_id, &tramp, nargs)) { cmn_err(CE_WARN, "unexpectedly short FBT table " "in module %s (sym %d of %d)", modname, i, nsyms); break; } fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin + ((uintptr_t)instr - (uintptr_t)mp->text)); fbt->fbtp_savedval = *instr; fbt->fbtp_loadcnt = ctl->mod_loadcnt; fbt->fbtp_primary = primary; fbt->fbtp_symndx = i; mp->fbt_nentries++; } retfbt = NULL; again: if (++instr == limit) continue; offset = (uintptr_t)instr - (uintptr_t)base; if (!(*canpatch)(instr, offset, name)) goto again; if (estimate) { tramp.fbtt_next = (uintptr_t)faketramp; tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp); (void) (*patch)(instr, base, limit, offset, FBT_ESTIMATE_ID, &tramp, name); fbt_size += tramp.fbtt_next - (uintptr_t)faketramp; goto again; } fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP); fbt->fbtp_name = name; fbt->fbtp_ctl = ctl; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_PROBENAME_RETURN, 1, fbt); } else { retfbt->fbtp_next = fbt; fbt->fbtp_id = retfbt->fbtp_id; } fbt->fbtp_return = 1; retfbt = fbt; if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset, fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) { cmn_err(CE_WARN, "unexpectedly short FBT table " "in module %s (sym %d of %d)", modname, i, nsyms); break; } fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin + ((uintptr_t)instr - (uintptr_t)mp->text)); fbt->fbtp_savedval = *instr; fbt->fbtp_loadcnt = ctl->mod_loadcnt; fbt->fbtp_primary = primary; fbt->fbtp_symndx = i; mp->fbt_nentries++; goto again; } if (estimate) { /* * Slosh on another entry's worth... */ fbt_size += FBT_ENT_MAXSIZE; mp->fbt_size = fbt_size; mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size); if (mp->fbt_tab == NULL) { cmn_err(CE_WARN, "couldn't allocate FBT table " "for module %s", modname); } else { estimate = 0; goto forreal; } } else { fbt_trampoline_unmap(); } error: if (fp != NULL) ctf_close(fp); } /*ARGSUSED*/ static void fbt_destroy(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg, *next; struct modctl *ctl = fbt->fbtp_ctl; do { if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) { if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt && ctl->mod_loaded) || fbt->fbtp_primary) { ((struct module *) (ctl->mod_mp))->fbt_nentries--; } } next = fbt->fbtp_next; kmem_free(fbt, sizeof (fbt_probe_t)); fbt = next; } while (fbt != NULL); } /*ARGSUSED*/ static int fbt_enable(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg, *f; struct modctl *ctl = fbt->fbtp_ctl; ctl->mod_nenabled++; for (f = fbt; f != NULL; f = f->fbtp_next) { if (f->fbtp_patchpoint == NULL) { /* * Due to a shortened FBT table, this entry was never * completed; refuse to enable it. */ if (fbt_verbose) { cmn_err(CE_NOTE, "fbt is failing for probe %s " "(short FBT table in %s)", fbt->fbtp_name, ctl->mod_modname); } return (0); } } /* * If this module has disappeared since we discovered its probes, * refuse to enable it. */ if (!fbt->fbtp_primary && !ctl->mod_loaded) { if (fbt_verbose) { cmn_err(CE_NOTE, "fbt is failing for probe %s " "(module %s unloaded)", fbt->fbtp_name, ctl->mod_modname); } return (0); } /* * Now check that our modctl has the expected load count. If it * doesn't, this module must have been unloaded and reloaded -- and * we're not going to touch it. */ if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) { if (fbt_verbose) { cmn_err(CE_NOTE, "fbt is failing for probe %s " "(module %s reloaded)", fbt->fbtp_name, ctl->mod_modname); } return (0); } for (; fbt != NULL; fbt = fbt->fbtp_next) *fbt->fbtp_patchpoint = fbt->fbtp_patchval; return (0); } /*ARGSUSED*/ static void fbt_disable(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg, *f; struct modctl *ctl = fbt->fbtp_ctl; ASSERT(ctl->mod_nenabled > 0); ctl->mod_nenabled--; for (f = fbt; f != NULL; f = f->fbtp_next) { if (f->fbtp_patchpoint == NULL) return; } if ((!fbt->fbtp_primary && !ctl->mod_loaded) || (ctl->mod_loadcnt != fbt->fbtp_loadcnt)) return; for (; fbt != NULL; fbt = fbt->fbtp_next) *fbt->fbtp_patchpoint = fbt->fbtp_savedval; } /*ARGSUSED*/ static void fbt_suspend(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg; struct modctl *ctl = fbt->fbtp_ctl; if (!fbt->fbtp_primary && !ctl->mod_loaded) return; if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) return; ASSERT(ctl->mod_nenabled > 0); for (; fbt != NULL; fbt = fbt->fbtp_next) *fbt->fbtp_patchpoint = fbt->fbtp_savedval; } /*ARGSUSED*/ static void fbt_resume(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg; struct modctl *ctl = fbt->fbtp_ctl; if (!fbt->fbtp_primary && !ctl->mod_loaded) return; if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) return; ASSERT(ctl->mod_nenabled > 0); for (; fbt != NULL; fbt = fbt->fbtp_next) *fbt->fbtp_patchpoint = fbt->fbtp_patchval; } /*ARGSUSED*/ static void fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) { fbt_probe_t *fbt = parg; struct modctl *ctl = fbt->fbtp_ctl; struct module *mp = ctl->mod_mp; ctf_file_t *fp = NULL, *pfp; ctf_funcinfo_t f; int error; ctf_id_t argv[32], type; int argc = sizeof (argv) / sizeof (ctf_id_t); const char *parent; if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt)) goto err; if (fbt->fbtp_return && desc->dtargd_ndx == 0) { (void) strcpy(desc->dtargd_native, "int"); return; } if ((fp = ctf_modopen(mp, &error)) == NULL) { /* * We have no CTF information for this module -- and therefore * no args[] information. */ goto err; } /* * If we have a parent container, we must manually import it. */ if ((parent = ctf_parent_name(fp)) != NULL) { struct modctl *mp = &modules; struct modctl *mod = NULL; /* * We must iterate over all modules to find the module that * is our parent. */ do { if (strcmp(mp->mod_modname, parent) == 0) { mod = mp; break; } } while ((mp = mp->mod_next) != &modules); if (mod == NULL) goto err; if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL) goto err; if (ctf_import(fp, pfp) != 0) { ctf_close(pfp); goto err; } ctf_close(pfp); } if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR) goto err; if (fbt->fbtp_return) { if (desc->dtargd_ndx > 1) goto err; ASSERT(desc->dtargd_ndx == 1); type = f.ctc_return; } else { if (desc->dtargd_ndx + 1 > f.ctc_argc) goto err; if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR) goto err; type = argv[desc->dtargd_ndx]; } if (ctf_type_name(fp, type, desc->dtargd_native, DTRACE_ARGTYPELEN) != NULL) { ctf_close(fp); return; } err: if (fp != NULL) ctf_close(fp); desc->dtargd_ndx = DTRACE_ARGNONE; } static dtrace_pattr_t fbt_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, }; static dtrace_pops_t fbt_pops = { NULL, fbt_provide_module, fbt_enable, fbt_disable, fbt_suspend, fbt_resume, fbt_getargdesc, NULL, NULL, fbt_destroy }; static int fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) { switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: return (DDI_SUCCESS); default: return (DDI_FAILURE); } if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0, DDI_PSEUDO, NULL) == DDI_FAILURE || dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL, &fbt_pops, NULL, &fbt_id) != 0) { ddi_remove_minor_node(devi, NULL); return (DDI_FAILURE); } ddi_report_dev(devi); fbt_devi = devi; return (DDI_SUCCESS); } static int fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) { switch (cmd) { case DDI_DETACH: break; case DDI_SUSPEND: return (DDI_SUCCESS); default: return (DDI_FAILURE); } if (dtrace_unregister(fbt_id) != 0) return (DDI_FAILURE); ddi_remove_minor_node(devi, NULL); return (DDI_SUCCESS); } /*ARGSUSED*/ static int fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) { int error; switch (infocmd) { case DDI_INFO_DEVT2DEVINFO: *result = (void *)fbt_devi; error = DDI_SUCCESS; break; case DDI_INFO_DEVT2INSTANCE: *result = (void *)0; error = DDI_SUCCESS; break; default: error = DDI_FAILURE; } return (error); } /*ARGSUSED*/ static int fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) { return (0); } static struct cb_ops fbt_cb_ops = { fbt_open, /* open */ nodev, /* close */ nulldev, /* strategy */ nulldev, /* print */ nodev, /* dump */ nodev, /* read */ nodev, /* write */ nodev, /* ioctl */ nodev, /* devmap */ nodev, /* mmap */ nodev, /* segmap */ nochpoll, /* poll */ ddi_prop_op, /* cb_prop_op */ 0, /* streamtab */ D_NEW | D_MP /* Driver compatibility flag */ }; static struct dev_ops fbt_ops = { DEVO_REV, /* devo_rev */ 0, /* refcnt */ fbt_info, /* get_dev_info */ nulldev, /* identify */ nulldev, /* probe */ fbt_attach, /* attach */ fbt_detach, /* detach */ nodev, /* reset */ &fbt_cb_ops, /* driver operations */ NULL, /* bus operations */ nodev, /* dev power */ ddi_quiesce_not_needed, /* quiesce */ }; /* * Module linkage information for the kernel. */ static struct modldrv modldrv = { &mod_driverops, /* module type (this is a pseudo driver) */ "Function Boundary Tracing", /* name of module */ &fbt_ops, /* driver ops */ }; static struct modlinkage modlinkage = { MODREV_1, (void *)&modldrv, NULL }; int _init(void) { return (mod_install(&modlinkage)); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } int _fini(void) { return (mod_remove(&modlinkage)); }