xref: /titanic_41/usr/src/uts/sparc/dtrace/fbt.c (revision b9e93c10c0a2a4bb069d38bb311021a9478c4711)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 
27 #include <sys/errno.h>
28 #include <sys/stat.h>
29 #include <sys/modctl.h>
30 #include <sys/conf.h>
31 #include <sys/systm.h>
32 #include <sys/ddi.h>
33 #include <sys/sunddi.h>
34 #include <sys/cpuvar.h>
35 #include <sys/kmem.h>
36 #include <sys/strsubr.h>
37 #include <sys/dtrace.h>
38 #include <sys/kobj.h>
39 #include <sys/modctl.h>
40 #include <sys/atomic.h>
41 #include <vm/seg_kmem.h>
42 #include <sys/stack.h>
43 #include <sys/ctf_api.h>
44 #include <sys/sysmacros.h>
45 
46 static dev_info_t		*fbt_devi;
47 static dtrace_provider_id_t	fbt_id;
48 static uintptr_t		fbt_trampoline;
49 static caddr_t			fbt_trampoline_window;
50 static size_t			fbt_trampoline_size;
51 static int			fbt_verbose = 0;
52 
53 /*
54  * Various interesting bean counters.
55  */
56 static int			fbt_entry;
57 static int			fbt_ret;
58 static int			fbt_retl;
59 static int			fbt_retl_jmptab;
60 static int			fbt_retl_twoinstr;
61 static int			fbt_retl_tailcall;
62 static int			fbt_retl_tailjmpl;
63 static int			fbt_leaf_functions;
64 
65 extern char			stubs_base[];
66 extern char			stubs_end[];
67 
68 #define	FBT_REG_G0		0
69 #define	FBT_REG_G1		1
70 #define	FBT_REG_O0		8
71 #define	FBT_REG_O1		9
72 #define	FBT_REG_O2		10
73 #define	FBT_REG_O3		11
74 #define	FBT_REG_O4		12
75 #define	FBT_REG_O5		13
76 #define	FBT_REG_O6		14
77 #define	FBT_REG_O7		15
78 #define	FBT_REG_I0		24
79 #define	FBT_REG_I1		25
80 #define	FBT_REG_I2		26
81 #define	FBT_REG_I3		27
82 #define	FBT_REG_I4		28
83 #define	FBT_REG_I7		31
84 #define	FBT_REG_L0		16
85 #define	FBT_REG_L1		17
86 #define	FBT_REG_L2		18
87 #define	FBT_REG_L3		19
88 #define	FBT_REG_PC		5
89 
90 #define	FBT_REG_ISGLOBAL(r)	((r) < 8)
91 #define	FBT_REG_ISOUTPUT(r)	((r) >= 8 && (r) < 16)
92 #define	FBT_REG_ISLOCAL(r)	((r) >= 16 && (r) < 24)
93 #define	FBT_REG_ISVOLATILE(r)	\
94 	((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0)
95 #define	FBT_REG_NLOCALS		8
96 
97 #define	FBT_REG_MARKLOCAL(locals, r)	\
98 	if (FBT_REG_ISLOCAL(r)) \
99 		(locals)[(r) - FBT_REG_L0] = 1;
100 
101 #define	FBT_REG_INITLOCALS(local, locals)	\
102 	for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++)  \
103 		(locals)[(local)] = 0; \
104 	(local) = FBT_REG_L0
105 
106 #define	FBT_REG_ALLOCLOCAL(local, locals)	\
107 	while ((locals)[(local) - FBT_REG_L0]) \
108 		(local)++; \
109 	(locals)[(local) - FBT_REG_L0] = 1;
110 
111 #define	FBT_OP_MASK		0xc0000000
112 #define	FBT_OP_SHIFT		30
113 #define	FBT_OP(val)		((val) & FBT_FMT1_MASK)
114 
115 #define	FBT_SIMM13_MASK		0x1fff
116 #define	FBT_SIMM13_MAX		((int32_t)0xfff)
117 #define	FBT_IMM22_MASK		0x3fffff
118 #define	FBT_IMM22_SHIFT		10
119 #define	FBT_IMM10_MASK		0x3ff
120 
121 #define	FBT_DISP30_MASK		0x3fffffff
122 #define	FBT_DISP30(from, to)	\
123 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK)
124 
125 #define	FBT_DISP22_MASK		0x3fffff
126 #define	FBT_DISP22(from, to)	\
127 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK)
128 
129 #define	FBT_DISP19_MASK		0x7ffff
130 #define	FBT_DISP19(from, to)	\
131 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK)
132 
133 #define	FBT_DISP16_HISHIFT	20
134 #define	FBT_DISP16_HIMASK	(0x3 << FBT_DISP16_HISHIFT)
135 #define	FBT_DISP16_LOMASK	(0x3fff)
136 #define	FBT_DISP16_MASK		(FBT_DISP16_HIMASK | FBT_DISP16_LOMASK)
137 #define	FBT_DISP16(val)	\
138 	((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK))
139 
140 #define	FBT_DISP14_MASK		0x3fff
141 #define	FBT_DISP14(from, to)	\
142 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK)
143 
144 #define	FBT_OP0			(((uint32_t)0) << FBT_OP_SHIFT)
145 #define	FBT_OP1			(((uint32_t)1) << FBT_OP_SHIFT)
146 #define	FBT_OP2			(((uint32_t)2) << FBT_OP_SHIFT)
147 #define	FBT_ILLTRAP		0
148 
149 #define	FBT_ANNUL_SHIFT		29
150 #define	FBT_ANNUL		(1 << FBT_ANNUL_SHIFT)
151 
152 #define	FBT_FMT3_OP3_SHIFT	19
153 #define	FBT_FMT3_OP_MASK	0xc1f80000
154 #define	FBT_FMT3_OP(val)	((val) & FBT_FMT3_OP_MASK)
155 
156 #define	FBT_FMT3_RD_SHIFT	25
157 #define	FBT_FMT3_RD_MASK	(0x1f << FBT_FMT3_RD_SHIFT)
158 #define	FBT_FMT3_RD(val)	\
159 	(((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT)
160 
161 #define	FBT_FMT3_RS1_SHIFT	14
162 #define	FBT_FMT3_RS1_MASK	(0x1f << FBT_FMT3_RS1_SHIFT)
163 #define	FBT_FMT3_RS1(val)	\
164 	(((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT)
165 #define	FBT_FMT3_RS1_SET(val, rs1) \
166 	(val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT)
167 
168 #define	FBT_FMT3_RS2_SHIFT	0
169 #define	FBT_FMT3_RS2_MASK	(0x1f << FBT_FMT3_RS2_SHIFT)
170 #define	FBT_FMT3_RS2(val)	\
171 	(((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT)
172 #define	FBT_FMT3_RS2_SET(val, rs2) \
173 	(val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT)
174 
175 #define	FBT_FMT3_IMM_SHIFT	13
176 #define	FBT_FMT3_IMM		(1 << FBT_FMT3_IMM_SHIFT)
177 #define	FBT_FMT3_SIMM13_MASK	FBT_SIMM13_MASK
178 
179 #define	FBT_FMT3_ISIMM(val)	((val) & FBT_FMT3_IMM)
180 #define	FBT_FMT3_SIMM13(val)	((val) & FBT_FMT3_SIMM13_MASK)
181 
182 #define	FBT_FMT2_OP2_SHIFT	22
183 #define	FBT_FMT2_OP2_MASK	(0x7 << FBT_FMT2_OP2_SHIFT)
184 #define	FBT_FMT2_RD_SHIFT	25
185 
186 #define	FBT_FMT1_OP(val)	((val) & FBT_OP_MASK)
187 #define	FBT_FMT1_DISP30(val)	((val) & FBT_DISP30_MASK)
188 
189 #define	FBT_FMT2_OP2_BPCC	(0x01 << FBT_FMT2_OP2_SHIFT)
190 #define	FBT_FMT2_OP2_BCC	(0x02 << FBT_FMT2_OP2_SHIFT)
191 #define	FBT_FMT2_OP2_BPR	(0x03 << FBT_FMT2_OP2_SHIFT)
192 #define	FBT_FMT2_OP2_SETHI	(0x04 << FBT_FMT2_OP2_SHIFT)
193 
194 #define	FBT_FMT2_COND_SHIFT	25
195 #define	FBT_FMT2_COND_BA	(0x8 << FBT_FMT2_COND_SHIFT)
196 #define	FBT_FMT2_COND_BL	(0x3 << FBT_FMT2_COND_SHIFT)
197 #define	FBT_FMT2_COND_BGE	(0xb << FBT_FMT2_COND_SHIFT)
198 
199 #define	FBT_OP_RESTORE		(FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT))
200 #define	FBT_OP_SAVE		(FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT))
201 #define	FBT_OP_JMPL		(FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT))
202 #define	FBT_OP_RETURN		(FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT))
203 #define	FBT_OP_CALL		FBT_OP1
204 #define	FBT_OP_SETHI		(FBT_OP0 | FBT_FMT2_OP2_SETHI)
205 #define	FBT_OP_ADD		(FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT))
206 #define	FBT_OP_OR		(FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT))
207 #define	FBT_OP_SUB		(FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT))
208 #define	FBT_OP_CC		(FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT))
209 #define	FBT_OP_BA		(FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA)
210 #define	FBT_OP_BL		(FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL)
211 #define	FBT_OP_BGE		(FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE)
212 #define	FBT_OP_BAPCC		(FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA)
213 #define	FBT_OP_RD		(FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT))
214 
215 #define	FBT_ORLO(rs, val, rd) \
216 	(FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
217 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK))
218 
219 #define	FBT_ORSIMM13(rs, val, rd) \
220 	(FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
221 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
222 
223 #define	FBT_ADDSIMM13(rs, val, rd) \
224 	(FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \
225 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
226 
227 #define	FBT_ADD(rs1, rs2, rd) \
228 	(FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
229 	((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
230 
231 #define	FBT_CMP(rs1, rs2) \
232 	(FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
233 	((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT))
234 
235 #define	FBT_MOV(rs, rd) \
236 	(FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \
237 	((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
238 
239 #define	FBT_SETHI(val, reg)	\
240 	(FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \
241 	((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK))
242 
243 #define	FBT_CALL(orig, dest)	(FBT_OP_CALL | FBT_DISP30(orig, dest))
244 
245 #define	FBT_RET \
246 	(FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \
247 	(FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1))
248 
249 #define	FBT_SAVEIMM(rd, val, rs1)	\
250 	(FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
251 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
252 
253 #define	FBT_RESTORE(rd, rs1, rs2)	\
254 	(FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
255 	((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT))
256 
257 #define	FBT_RETURN(rs1, val)		\
258 	(FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
259 	FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
260 
261 #define	FBT_BA(orig, dest)	(FBT_OP_BA | FBT_DISP22(orig, dest))
262 #define	FBT_BAA(orig, dest)	(FBT_BA(orig, dest) | FBT_ANNUL)
263 #define	FBT_BL(orig, dest)	(FBT_OP_BL | FBT_DISP22(orig, dest))
264 #define	FBT_BGE(orig, dest)	(FBT_OP_BGE | FBT_DISP22(orig, dest))
265 #define	FBT_BDEST(va, instr)	((uintptr_t)(va) + \
266 	(((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8))
267 #define	FBT_BPCCDEST(va, instr)	((uintptr_t)(va) + \
268 	(((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11))
269 #define	FBT_BPRDEST(va, instr)	((uintptr_t)(va) + \
270 	(((int32_t)((FBT_DISP16(instr)) << 16)) >> 14))
271 
272 /*
273  * We're only going to treat a save as safe if (a) both rs1 and rd are
274  * %sp and (b) if the instruction has a simm, the value isn't 0.
275  */
276 #define	FBT_IS_SAVE(instr)	\
277 	(FBT_FMT3_OP(instr) == FBT_OP_SAVE && \
278 	FBT_FMT3_RD(instr) == FBT_REG_O6 && \
279 	FBT_FMT3_RS1(instr) == FBT_REG_O6 && \
280 	!(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0))
281 
282 #define	FBT_IS_BA(instr)	(((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA)
283 #define	FBT_IS_BAPCC(instr)	(((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC)
284 
285 #define	FBT_IS_RDPC(instr)	((FBT_FMT3_OP(instr) == FBT_OP_RD) && \
286 	(FBT_FMT3_RD(instr) == FBT_REG_PC))
287 
288 #define	FBT_IS_PCRELATIVE(instr)	\
289 	((((instr) & FBT_OP_MASK) == FBT_OP0 && \
290 	((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
291 	((instr) & FBT_OP_MASK) == FBT_OP1 || \
292 	FBT_IS_RDPC(instr))
293 
294 #define	FBT_IS_CTI(instr)	\
295 	((((instr) & FBT_OP_MASK) == FBT_OP0 && \
296 	((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
297 	((instr) & FBT_OP_MASK) == FBT_OP1 || \
298 	(FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \
299 	(FBT_FMT3_OP(instr) == FBT_OP_RETURN))
300 
301 #define	FBT_PROBENAME_ENTRY	"entry"
302 #define	FBT_PROBENAME_RETURN	"return"
303 #define	FBT_ESTIMATE_ID		(UINT32_MAX)
304 #define	FBT_COUNTER(id, count)	if ((id) != FBT_ESTIMATE_ID) (count)++
305 
306 #define	FBT_ENTENT_MAXSIZE	(16 * sizeof (uint32_t))
307 #define	FBT_RETENT_MAXSIZE	(11 * sizeof (uint32_t))
308 #define	FBT_RETLENT_MAXSIZE	(23 * sizeof (uint32_t))
309 #define	FBT_ENT_MAXSIZE		\
310 	MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE)
311 
312 typedef struct fbt_probe {
313 	char		*fbtp_name;
314 	dtrace_id_t	fbtp_id;
315 	uintptr_t	fbtp_addr;
316 	struct modctl	*fbtp_ctl;
317 	int		fbtp_loadcnt;
318 	int		fbtp_symndx;
319 	int		fbtp_primary;
320 	int		fbtp_return;
321 	uint32_t	*fbtp_patchpoint;
322 	uint32_t	fbtp_patchval;
323 	uint32_t	fbtp_savedval;
324 	struct fbt_probe *fbtp_next;
325 } fbt_probe_t;
326 
327 typedef struct fbt_trampoline {
328 	uintptr_t	fbtt_va;
329 	uintptr_t	fbtt_limit;
330 	uintptr_t	fbtt_next;
331 } fbt_trampoline_t;
332 
333 static caddr_t
fbt_trampoline_map(uintptr_t tramp,size_t size)334 fbt_trampoline_map(uintptr_t tramp, size_t size)
335 {
336 	uintptr_t offs;
337 	page_t **ppl;
338 
339 	ASSERT(fbt_trampoline_window == NULL);
340 	ASSERT(fbt_trampoline_size == 0);
341 	ASSERT(fbt_trampoline == NULL);
342 
343 	size += tramp & PAGEOFFSET;
344 	fbt_trampoline = tramp & PAGEMASK;
345 	fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK;
346 	fbt_trampoline_window =
347 	    vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP);
348 
349 	(void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline,
350 	    fbt_trampoline_size, S_WRITE);
351 
352 	for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) {
353 		hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE,
354 		    hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs),
355 		    PROT_READ | PROT_WRITE,
356 		    HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
357 	}
358 
359 	as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size,
360 	    S_WRITE);
361 
362 	return (fbt_trampoline_window + (tramp & PAGEOFFSET));
363 }
364 
365 static void
fbt_trampoline_unmap()366 fbt_trampoline_unmap()
367 {
368 	ASSERT(fbt_trampoline_window != NULL);
369 	ASSERT(fbt_trampoline_size != 0);
370 	ASSERT(fbt_trampoline != NULL);
371 
372 	membar_enter();
373 	sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size);
374 	sync_icache(fbt_trampoline_window, fbt_trampoline_size);
375 
376 	hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size,
377 	    HAT_UNLOAD_UNLOCK);
378 
379 	vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size);
380 
381 	fbt_trampoline_window = NULL;
382 	fbt_trampoline = NULL;
383 	fbt_trampoline_size = 0;
384 }
385 
386 static uintptr_t
fbt_patch_entry(uint32_t * instr,uint32_t id,fbt_trampoline_t * tramp,int nargs)387 fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp,
388     int nargs)
389 {
390 	uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
391 	uint32_t first = *instr;
392 	uintptr_t va = tramp->fbtt_va;
393 	uintptr_t base = tramp->fbtt_next;
394 
395 	if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) {
396 		/*
397 		 * There isn't sufficient room for this entry; return failure.
398 		 */
399 		return (0);
400 	}
401 
402 	FBT_COUNTER(id, fbt_entry);
403 
404 	if (FBT_IS_SAVE(first)) {
405 		*tinstr++ = first;
406 	} else {
407 		*tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
408 	}
409 
410 	if (id > (uint32_t)FBT_SIMM13_MAX) {
411 		*tinstr++ = FBT_SETHI(id, FBT_REG_O0);
412 		*tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
413 	} else {
414 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
415 	}
416 
417 	if (nargs >= 1)
418 		*tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1);
419 
420 	if (nargs >= 2)
421 		*tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2);
422 
423 	if (nargs >= 3)
424 		*tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3);
425 
426 	if (nargs >= 4)
427 		*tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4);
428 
429 	if (nargs >= 5)
430 		*tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5);
431 
432 	if (FBT_IS_SAVE(first)) {
433 		uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t);
434 
435 		*tinstr++ = FBT_SETHI(ret, FBT_REG_G1);
436 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
437 		tinstr++;
438 		*tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7);
439 	} else {
440 		uintptr_t slot = *--tinstr;
441 		uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t);
442 		uint32_t delay = first;
443 
444 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
445 		tinstr++;
446 		*tinstr++ = slot;
447 		*tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
448 
449 		if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) {
450 			/*
451 			 * This is a special case:  we are instrumenting a
452 			 * a non-annulled branch-always (or variant).  We'll
453 			 * return directly to the destination of the branch,
454 			 * copying the instruction in the delay slot here,
455 			 * and then executing it in the slot of a ba.
456 			 */
457 			if (FBT_IS_BA(first)) {
458 				ret = FBT_BDEST(instr, *instr);
459 			} else {
460 				ret = FBT_BPCCDEST(instr, *instr);
461 			}
462 
463 			delay = *(instr + 1);
464 		}
465 
466 		if ((first & FBT_OP_MASK) != FBT_OP0 ||
467 		    (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
468 			*tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret);
469 			tinstr++;
470 			*tinstr++ = delay;
471 		} else {
472 			/*
473 			 * If this is a branch-on-register, we have a little
474 			 * more work to do:  because the displacement is only
475 			 * sixteen bits, we're going to thunk the branch into
476 			 * the trampoline, and then ba,a to the appropriate
477 			 * destination in the branch targets.  That is, we're
478 			 * constructing this sequence in the trampoline:
479 			 *
480 			 *		br[cc]	%[rs], 1f
481 			 *		<delay-instruction>
482 			 *		ba,a	<not-taken-destination>
483 			 *	1:	ba,a	<taken-destination>
484 			 *
485 			 */
486 			uintptr_t targ = FBT_BPRDEST(instr, first);
487 
488 			*tinstr = first & ~(FBT_DISP16_MASK);
489 			*tinstr |= FBT_DISP14(tinstr, &tinstr[3]);
490 			tinstr++;
491 			*tinstr++ = *(instr + 1);
492 			*tinstr = FBT_BAA((uintptr_t)tinstr - base + va,
493 			    ret + sizeof (uint32_t));
494 			tinstr++;
495 			*tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ);
496 			tinstr++;
497 		}
498 	}
499 
500 	tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
501 	tramp->fbtt_next = (uintptr_t)tinstr;
502 
503 	return (1);
504 }
505 
506 /*
507  * We are patching control-transfer/restore couplets.  There are three
508  * variants of couplet:
509  *
510  * (a)	return		rs1 + imm
511  *	delay
512  *
513  * (b)	jmpl		rs1 + (rs2 | offset), rd
514  *	restore		rs1, rs2 | imm, rd
515  *
516  * (c)	call		displacement
517  *	restore		rs1, rs2 | imm, rd
518  *
519  * If rs1 in (a) is anything other than %i7, or imm is anything other than 8,
520  * or delay is a DCTI, we fail.  If rd from the jmpl in (b) is something other
521  * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call
522  * through a register), we fail.
523  *
524  * Note that rs1 and rs2 in the restore instructions in (b) and (c) are
525  * potentially outputs and/or globals.  Because these registers cannot be
526  * relied upon across the call to dtrace_probe(), we move rs1 into an unused
527  * local, ls0, and rs2 into an unused local, ls1, and restructure the restore
528  * to be:
529  *
530  *	restore		ls0, ls1, rd
531  *
532  * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals.
533  * If the jmpl uses outputs or globals, we restructure it to be:
534  *
535  * 	jmpl		ls2 + (ls3 | offset), (%g0 | %o7)
536  *
537  */
538 /*ARGSUSED*/
539 static int
fbt_canpatch_return(uint32_t * instr,int offset,const char * name)540 fbt_canpatch_return(uint32_t *instr, int offset, const char *name)
541 {
542 	int rd;
543 
544 	if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
545 		uint32_t delay = *(instr + 1);
546 
547 		if (*instr != FBT_RETURN(FBT_REG_I7, 8)) {
548 			/*
549 			 * It's unclear if we should warn about this or not.
550 			 * We really wouldn't expect the compiler to generate
551 			 * return instructions with something other than %i7
552 			 * as rs1 and 8 as the simm13 -- it would just be
553 			 * mean-spirited.  That said, such a construct isn't
554 			 * necessarily incorrect.  Sill, we err on the side of
555 			 * caution and warn about it...
556 			 */
557 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
558 			    "%p: non-canonical return instruction", name,
559 			    (void *)instr);
560 			return (0);
561 		}
562 
563 		if (FBT_IS_CTI(delay)) {
564 			/*
565 			 * This is even weirder -- a DCTI coupled with a
566 			 * return instruction.  Similar constructs are used to
567 			 * return from utraps, but these typically have the
568 			 * return in the slot -- and we wouldn't expect to see
569 			 * it in the kernel regardless.  At any rate, we don't
570 			 * want to try to instrument this construct, whatever
571 			 * it may be.
572 			 */
573 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
574 			    "%p: CTI in delay slot of return instruction",
575 			    name, (void *)instr);
576 			return (0);
577 		}
578 
579 		if (FBT_IS_PCRELATIVE(delay)) {
580 			/*
581 			 * This is also very weird, but might be correct code
582 			 * if the function is (for example) returning the
583 			 * address of the delay instruction of the return as
584 			 * its return value (e.g. "rd %pc, %o0" in the slot).
585 			 * Perhaps correct, but still too weird to not warn
586 			 * about it...
587 			 */
588 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
589 			    "%p: PC-relative instruction in delay slot of "
590 			    "return instruction", name, (void *)instr);
591 			return (0);
592 		}
593 
594 		return (1);
595 	}
596 
597 	if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE)
598 		return (0);
599 
600 	if (FBT_FMT1_OP(*instr) == FBT_OP_CALL)
601 		return (1);
602 
603 	if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
604 		return (0);
605 
606 	rd = FBT_FMT3_RD(*instr);
607 
608 	if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0)
609 		return (1);
610 
611 	/*
612 	 * We have encountered a jmpl that is storing the calling %pc in
613 	 * some register besides %i7, %o7 or %g0.  This is strange; emit
614 	 * a warning and fail.
615 	 */
616 	cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected "
617 	    "jmpl destination register", name, (void *)instr);
618 	return (0);
619 }
620 
621 static int
fbt_canpatch_retl(uint32_t * instr,int offset,const char * name)622 fbt_canpatch_retl(uint32_t *instr, int offset, const char *name)
623 {
624 	if (FBT_FMT1_OP(*instr) == FBT_OP_CALL ||
625 	    (FBT_FMT3_OP(*instr) == FBT_OP_JMPL &&
626 	    FBT_FMT3_RD(*instr) == FBT_REG_O7)) {
627 		/*
628 		 * If this is a call (or a jmpl that links into %o7), we can
629 		 * patch it iff the next instruction uses %o7 as a destination
630 		 * register.  Because there is an ABI responsibility to
631 		 * restore %o7 to the value before the call/jmpl, we don't
632 		 * particularly care how this routine is managing to restore
633 		 * it (mov, add, ld or divx for all we care).  If it doesn't
634 		 * seem to be restoring it at all, however, we'll refuse
635 		 * to patch it.
636 		 */
637 		uint32_t delay = *(instr + 1);
638 		uint32_t op, rd;
639 
640 		op = FBT_FMT1_OP(delay);
641 		rd = FBT_FMT3_RD(delay);
642 
643 		if (op != FBT_OP2 || rd != FBT_REG_O7) {
644 			/*
645 			 * This is odd.  Before we assume that we're looking
646 			 * at something bizarre (and warn accordingly), we'll
647 			 * check to see if it's obviously a jump table entry.
648 			 */
649 			if (*instr < (uintptr_t)instr &&
650 			    *instr >= (uintptr_t)instr - offset)
651 				return (0);
652 
653 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
654 			    "%p: leaf jmpl/call delay isn't restoring %%o7",
655 			    name, (void *)instr);
656 			return (0);
657 		}
658 
659 		return (1);
660 	}
661 
662 	if (offset == sizeof (uint32_t)) {
663 		/*
664 		 * If this is the second instruction in the function, we're
665 		 * going to allow it to be patched if the first instruction
666 		 * is a patchable return-from-leaf instruction.
667 		 */
668 		if (fbt_canpatch_retl(instr - 1, 0, name))
669 			return (1);
670 	}
671 
672 	if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
673 		return (0);
674 
675 	if (FBT_FMT3_RD(*instr) != FBT_REG_G0)
676 		return (0);
677 
678 	return (1);
679 }
680 
681 /*ARGSUSED*/
682 static uint32_t
fbt_patch_return(uint32_t * instr,uint32_t * funcbase,uint32_t * funclim,int offset,uint32_t id,fbt_trampoline_t * tramp,const char * name)683 fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
684     int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
685 {
686 	uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
687 	uint32_t cti = *instr, restore = *(instr + 1), rs1, dest;
688 	uintptr_t va = tramp->fbtt_va;
689 	uintptr_t base = tramp->fbtt_next;
690 	uint32_t locals[FBT_REG_NLOCALS], local;
691 
692 	if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) {
693 		/*
694 		 * There isn't sufficient room for this entry; return failure.
695 		 */
696 		return (FBT_ILLTRAP);
697 	}
698 
699 	FBT_COUNTER(id, fbt_ret);
700 
701 	if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
702 		/*
703 		 * To handle the case of the return instruction, we'll emit a
704 		 * restore, followed by the instruction in the slot (which
705 		 * we'll transplant here), and then another save.  While it
706 		 * may seem intellectually unsatisfying to emit the additional
707 		 * restore/save couplet, one can take solace in the fact that
708 		 * we don't do this if the instruction in the return delay
709 		 * slot is a nop -- which it is nearly 90% of the time with
710 		 * gcc.  (And besides, this couplet can't induce unnecessary
711 		 * spill/fill traps; rewriting the delay instruction to be
712 		 * in terms of the current window hardly seems worth the
713 		 * trouble -- let alone the risk.)
714 		 */
715 		uint32_t delay = *(instr + 1);
716 		ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8));
717 
718 		cti = FBT_RET;
719 		restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
720 
721 		if (delay != FBT_SETHI(0, FBT_REG_G0)) {
722 			*tinstr++ = restore;
723 			*tinstr++ = delay;
724 			*tinstr++ = FBT_SAVEIMM(FBT_REG_O6,
725 			    -SA(MINFRAME), FBT_REG_O6);
726 		}
727 	}
728 
729 	FBT_REG_INITLOCALS(local, locals);
730 
731 	/*
732 	 * Mark the locals used in the jmpl.
733 	 */
734 	if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
735 		uint32_t rs1 = FBT_FMT3_RS1(cti);
736 		FBT_REG_MARKLOCAL(locals, rs1);
737 
738 		if (!FBT_FMT3_ISIMM(cti)) {
739 			uint32_t rs2 = FBT_FMT3_RS2(cti);
740 			FBT_REG_MARKLOCAL(locals, rs2);
741 		}
742 	}
743 
744 	/*
745 	 * And mark the locals used in the restore.
746 	 */
747 	rs1 = FBT_FMT3_RS1(restore);
748 	FBT_REG_MARKLOCAL(locals, rs1);
749 
750 	if (!FBT_FMT3_ISIMM(restore)) {
751 		uint32_t rs2 = FBT_FMT3_RS2(restore);
752 		FBT_REG_MARKLOCAL(locals, rs2);
753 	}
754 
755 	if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
756 		uint32_t rs1 = FBT_FMT3_RS1(cti);
757 
758 		if (FBT_REG_ISVOLATILE(rs1)) {
759 			FBT_REG_ALLOCLOCAL(local, locals);
760 			FBT_FMT3_RS1_SET(cti, local);
761 			*tinstr++ = FBT_MOV(rs1, local);
762 		}
763 
764 		if (!FBT_FMT3_ISIMM(cti)) {
765 			uint32_t rs2 = FBT_FMT3_RS2(cti);
766 
767 			if (FBT_REG_ISVOLATILE(rs2)) {
768 				FBT_REG_ALLOCLOCAL(local, locals);
769 				FBT_FMT3_RS2_SET(cti, local);
770 				*tinstr++ = FBT_MOV(rs2, local);
771 			}
772 		}
773 	}
774 
775 	rs1 = FBT_FMT3_RS1(restore);
776 
777 	if (FBT_REG_ISVOLATILE(rs1)) {
778 		FBT_REG_ALLOCLOCAL(local, locals);
779 		FBT_FMT3_RS1_SET(restore, local);
780 		*tinstr++ = FBT_MOV(rs1, local);
781 	}
782 
783 	if (!FBT_FMT3_ISIMM(restore)) {
784 		uint32_t rs2 = FBT_FMT3_RS2(restore);
785 
786 		if (FBT_REG_ISVOLATILE(rs2)) {
787 			FBT_REG_ALLOCLOCAL(local, locals);
788 			FBT_FMT3_RS2_SET(restore, local);
789 			*tinstr++ = FBT_MOV(rs2, local);
790 		}
791 	}
792 
793 	if (id > (uint32_t)FBT_SIMM13_MAX) {
794 		*tinstr++ = FBT_SETHI(id, FBT_REG_O0);
795 		*tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
796 	} else {
797 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
798 	}
799 
800 	if (offset > (uint32_t)FBT_SIMM13_MAX) {
801 		*tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
802 		*tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
803 	} else {
804 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
805 	}
806 
807 	*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
808 	tinstr++;
809 
810 	if (FBT_FMT3_RD(restore) == FBT_REG_O0) {
811 		/*
812 		 * If the destination register of the restore is %o0, we
813 		 * need to perform the implied calculation to derive the
814 		 * return value.
815 		 */
816 		uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD;
817 		add &= ~FBT_FMT3_RD_MASK;
818 		*tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT);
819 	} else {
820 		*tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
821 	}
822 
823 	/*
824 	 * If the control transfer instruction is %pc-relative (i.e. a
825 	 * call), we need to reset it appropriately.
826 	 */
827 	if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
828 		dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
829 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
830 		tinstr++;
831 	} else {
832 		*tinstr++ = cti;
833 	}
834 
835 	*tinstr++ = restore;
836 	tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
837 	tramp->fbtt_next = (uintptr_t)tinstr;
838 
839 	return (FBT_BAA(instr, va));
840 }
841 
842 static uint32_t
fbt_patch_retl(uint32_t * instr,uint32_t * funcbase,uint32_t * funclim,int offset,uint32_t id,fbt_trampoline_t * tramp,const char * name)843 fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
844     int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
845 {
846 	uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
847 	uintptr_t va = tramp->fbtt_va;
848 	uintptr_t base = tramp->fbtt_next;
849 	uint32_t cti = *instr, dest;
850 	int annul = 0;
851 
852 	FBT_COUNTER(id, fbt_retl);
853 
854 	if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) {
855 		/*
856 		 * There isn't sufficient room for this entry; return failure.
857 		 */
858 		return (FBT_ILLTRAP);
859 	}
860 
861 	if (offset == sizeof (uint32_t) &&
862 	    fbt_canpatch_retl(instr - 1, 0, name)) {
863 		*tinstr++ = *instr;
864 		annul = 1;
865 		FBT_COUNTER(id, fbt_retl_twoinstr);
866 	} else {
867 		if (FBT_FMT3_OP(cti) == FBT_OP_JMPL &&
868 		    FBT_FMT3_RD(cti) != FBT_REG_O7 &&
869 		    FBT_FMT3_RS1(cti) != FBT_REG_O7) {
870 			annul = 1;
871 			*tinstr++ = *(instr + 1);
872 		}
873 	}
874 
875 	*tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
876 
877 	if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
878 		uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0;
879 
880 		/*
881 		 * If we have a jmpl and it's in terms of output registers, we
882 		 * need to rewrite it to be in terms of the corresponding input
883 		 * registers.  If it's in terms of the globals, we'll rewrite
884 		 * it to be in terms of locals.
885 		 */
886 		rs1 = FBT_FMT3_RS1(cti);
887 
888 		if (FBT_REG_ISOUTPUT(rs1))
889 			rs1 += o2i;
890 
891 		if (FBT_REG_ISGLOBAL(rs1)) {
892 			*tinstr++ = FBT_MOV(rs1, FBT_REG_L0);
893 			rs1 = FBT_REG_L0;
894 		}
895 
896 		FBT_FMT3_RS1_SET(cti, rs1);
897 
898 		if (!FBT_FMT3_ISIMM(cti)) {
899 			rs2 = FBT_FMT3_RS2(cti);
900 
901 			if (FBT_REG_ISOUTPUT(rs2))
902 				rs2 += o2i;
903 
904 			if (FBT_REG_ISGLOBAL(rs2)) {
905 				*tinstr++ = FBT_MOV(rs2, FBT_REG_L1);
906 				rs2 = FBT_REG_L1;
907 			}
908 
909 			FBT_FMT3_RS2_SET(cti, rs2);
910 		}
911 
912 		/*
913 		 * Now we need to check the rd and source register for the jmpl;
914 		 * If neither rd nor the source register is %o7, then we might
915 		 * have a jmp that is actually part of a jump table.  We need
916 		 * to generate the code to compare it to the base and limit of
917 		 * the function.
918 		 */
919 		if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) {
920 			uintptr_t base = (uintptr_t)funcbase;
921 			uintptr_t limit = (uintptr_t)funclim;
922 
923 			FBT_COUNTER(id, fbt_retl_jmptab);
924 
925 			if (FBT_FMT3_ISIMM(cti)) {
926 				*tinstr++ = FBT_ADDSIMM13(rs1,
927 				    FBT_FMT3_SIMM13(cti), FBT_REG_L2);
928 			} else {
929 				*tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2);
930 			}
931 
932 			*tinstr++ = FBT_SETHI(base, FBT_REG_L3);
933 			*tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3);
934 			*tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
935 			*tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t));
936 			*tinstr++ = FBT_SETHI(limit, FBT_REG_L3);
937 			*tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3);
938 			*tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
939 			*tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t));
940 			*tinstr++ = FBT_SETHI(0, FBT_REG_G0);
941 			*tinstr++ = cti;
942 			*tinstr++ = FBT_RESTORE(FBT_REG_G0,
943 			    FBT_REG_G0, FBT_REG_G0);
944 		}
945 	}
946 
947 	if (id > (uint32_t)FBT_SIMM13_MAX) {
948 		*tinstr++ = FBT_SETHI(id, FBT_REG_O0);
949 		*tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
950 	} else {
951 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
952 	}
953 
954 	if (offset > (uint32_t)FBT_SIMM13_MAX) {
955 		*tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
956 		*tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
957 	} else {
958 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
959 	}
960 
961 	*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
962 	tinstr++;
963 	*tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
964 
965 	/*
966 	 * If the control transfer instruction is %pc-relative (i.e. a
967 	 * call), we need to reset it appropriately.
968 	 */
969 	if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
970 		FBT_COUNTER(id, fbt_retl_tailcall);
971 		dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
972 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
973 		tinstr++;
974 		annul = 1;
975 	} else {
976 		if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
977 			*tinstr++ = cti;
978 
979 			if (FBT_FMT3_RD(cti) == FBT_REG_O7) {
980 				FBT_COUNTER(id, fbt_retl_tailjmpl);
981 				annul = 1;
982 			}
983 		} else {
984 			*tinstr++ = FBT_RET;
985 		}
986 	}
987 
988 	*tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
989 
990 	tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
991 	tramp->fbtt_next = (uintptr_t)tinstr;
992 
993 	return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va));
994 }
995 
996 /*ARGSUSED*/
997 static void
fbt_provide_module(void * arg,struct modctl * ctl)998 fbt_provide_module(void *arg, struct modctl *ctl)
999 {
1000 	struct module *mp = ctl->mod_mp;
1001 	char *modname = ctl->mod_modname;
1002 	char *str = mp->strings;
1003 	int nsyms = mp->nsyms;
1004 	Shdr *symhdr = mp->symhdr;
1005 	size_t symsize;
1006 	char *name;
1007 	int i;
1008 	fbt_probe_t *fbt, *retfbt;
1009 	fbt_trampoline_t tramp;
1010 	uintptr_t offset;
1011 	int primary = 0;
1012 	ctf_file_t *fp = NULL;
1013 	int error;
1014 	int estimate = 1;
1015 	uint32_t faketramp[50];
1016 	size_t fbt_size = 0;
1017 
1018 	/*
1019 	 * Employees of dtrace and their families are ineligible.  Void
1020 	 * where prohibited.
1021 	 */
1022 	if (strcmp(modname, "dtrace") == 0)
1023 		return;
1024 
1025 	if (ctl->mod_requisites != NULL) {
1026 		struct modctl_list *list;
1027 
1028 		list = (struct modctl_list *)ctl->mod_requisites;
1029 
1030 		for (; list != NULL; list = list->modl_next) {
1031 			if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
1032 				return;
1033 		}
1034 	}
1035 
1036 	/*
1037 	 * KMDB is ineligible for instrumentation -- it may execute in
1038 	 * any context, including probe context.
1039 	 */
1040 	if (strcmp(modname, "kmdbmod") == 0)
1041 		return;
1042 
1043 	if (str == NULL || symhdr == NULL || symhdr->sh_addr == NULL) {
1044 		/*
1045 		 * If this module doesn't (yet) have its string or symbol
1046 		 * table allocated, clear out.
1047 		 */
1048 		return;
1049 	}
1050 
1051 	symsize = symhdr->sh_entsize;
1052 
1053 	if (mp->fbt_nentries) {
1054 		/*
1055 		 * This module has some FBT entries allocated; we're afraid
1056 		 * to screw with it.
1057 		 */
1058 		return;
1059 	}
1060 
1061 	if (mp->fbt_tab != NULL)
1062 		estimate = 0;
1063 
1064 	/*
1065 	 * This is a hack for unix/genunix/krtld.
1066 	 */
1067 	primary = vmem_contains(heap_arena, (void *)ctl,
1068 	    sizeof (struct modctl)) == 0;
1069 	kobj_textwin_alloc(mp);
1070 
1071 	/*
1072 	 * Open the CTF data for the module.  We'll use this to determine the
1073 	 * functions that can be instrumented.  Note that this call can fail,
1074 	 * in which case we'll use heuristics to determine the functions that
1075 	 * can be instrumented.  (But in particular, leaf functions will not be
1076 	 * instrumented.)
1077 	 */
1078 	fp = ctf_modopen(mp, &error);
1079 
1080 forreal:
1081 	if (!estimate) {
1082 		tramp.fbtt_next =
1083 		    (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab,
1084 		    mp->fbt_size);
1085 		tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size;
1086 		tramp.fbtt_va = (uintptr_t)mp->fbt_tab;
1087 	}
1088 
1089 	for (i = 1; i < nsyms; i++) {
1090 		ctf_funcinfo_t f;
1091 		uint32_t *instr, *base, *limit;
1092 		Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
1093 		int have_ctf = 0, is_leaf = 0, nargs, cti = 0;
1094 		int (*canpatch)(uint32_t *, int, const char *);
1095 		uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int,
1096 		    uint32_t, fbt_trampoline_t *, const char *);
1097 
1098 		if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
1099 			continue;
1100 
1101 		/*
1102 		 * Weak symbols are not candidates.  This could be made to
1103 		 * work (where weak functions and their underlying function
1104 		 * appear as two disjoint probes), but it's not simple.
1105 		 */
1106 		if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
1107 			continue;
1108 
1109 		name = str + sym->st_name;
1110 
1111 		if (strstr(name, "dtrace_") == name &&
1112 		    strstr(name, "dtrace_safe_") != name) {
1113 			/*
1114 			 * Anything beginning with "dtrace_" may be called
1115 			 * from probe context unless it explitly indicates
1116 			 * that it won't be called from probe context by
1117 			 * using the prefix "dtrace_safe_".
1118 			 */
1119 			continue;
1120 		}
1121 
1122 		if (strstr(name, "kdi_") == name ||
1123 		    strstr(name, "_kdi_") != NULL) {
1124 			/*
1125 			 * Any function name beginning with "kdi_" or
1126 			 * containing the string "_kdi_" is a part of the
1127 			 * kernel debugger interface and may be called in
1128 			 * arbitrary context -- including probe context.
1129 			 */
1130 			continue;
1131 		}
1132 
1133 		if (strstr(name, "__relocatable") != NULL) {
1134 			/*
1135 			 * Anything with the string "__relocatable" anywhere
1136 			 * in the function name is considered to be a function
1137 			 * that may be manually relocated before execution.
1138 			 * Because FBT uses a PC-relative technique for
1139 			 * instrumentation, these functions cannot safely
1140 			 * be instrumented by us.
1141 			 */
1142 			continue;
1143 		}
1144 
1145 		if (strstr(name, "ip_ocsum") == name) {
1146 			/*
1147 			 * The ip_ocsum_* family of routines are all ABI
1148 			 * violators.  (They expect incoming arguments in the
1149 			 * globals!)  Break the ABI?  No soup for you!
1150 			 */
1151 			continue;
1152 		}
1153 
1154 		/*
1155 		 * We want to scan the function for one (and only one) save.
1156 		 * Any more indicates that something fancy is going on.
1157 		 */
1158 		base = (uint32_t *)sym->st_value;
1159 		limit = (uint32_t *)(sym->st_value + sym->st_size);
1160 
1161 		/*
1162 		 * We don't want to interpose on the module stubs.
1163 		 */
1164 		if (base >= (uint32_t *)stubs_base &&
1165 		    base <= (uint32_t *)stubs_end)
1166 			continue;
1167 
1168 		/*
1169 		 * We can't safely trace a zero-length function...
1170 		 */
1171 		if (base == limit)
1172 			continue;
1173 
1174 		/*
1175 		 * Due to 4524008, _init and _fini may have a bloated st_size.
1176 		 * While this bug was fixed quite some time ago, old drivers
1177 		 * may be lurking.  We need to develop a better solution to
1178 		 * this problem, such that correct _init and _fini functions
1179 		 * (the vast majority) may be correctly traced.  One solution
1180 		 * may be to scan through the entire symbol table to see if
1181 		 * any symbol overlaps with _init.  If none does, set a bit in
1182 		 * the module structure that this module has correct _init and
1183 		 * _fini sizes.  This will cause some pain the first time a
1184 		 * module is scanned, but at least it would be O(N) instead of
1185 		 * O(N log N)...
1186 		 */
1187 		if (strcmp(name, "_init") == 0)
1188 			continue;
1189 
1190 		if (strcmp(name, "_fini") == 0)
1191 			continue;
1192 
1193 		instr = base;
1194 
1195 		/*
1196 		 * While we try hard to only trace safe functions (that is,
1197 		 * functions at TL=0), one unsafe function manages to otherwise
1198 		 * appear safe:  prom_trap().  We could discover prom_trap()
1199 		 * if we added an additional rule:  in order to trace a
1200 		 * function, we must either (a) discover a restore or (b)
1201 		 * determine that the function does not have any unlinked
1202 		 * control transfers to another function (i.e., the function
1203 		 * never returns).  Unfortunately, as of this writing, one
1204 		 * legitimate function (resume_from_zombie()) transfers
1205 		 * control to a different function (_resume_from_idle())
1206 		 * without executing a restore.  Barring a rule to figure out
1207 		 * that resume_from_zombie() is safe while prom_trap() is not,
1208 		 * we resort to hard-coding prom_trap() here.
1209 		 */
1210 		if (strcmp(name, "prom_trap") == 0)
1211 			continue;
1212 
1213 		if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) {
1214 			nargs = f.ctc_argc;
1215 			have_ctf = 1;
1216 		} else {
1217 			nargs = 32;
1218 		}
1219 
1220 		/*
1221 		 * If the first instruction of the function is a branch and
1222 		 * it's not a branch-always-not-annulled, we're going to refuse
1223 		 * to patch it.
1224 		 */
1225 		if ((*instr & FBT_OP_MASK) == FBT_OP0 &&
1226 		    (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI &&
1227 		    (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
1228 			if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) {
1229 				if (have_ctf) {
1230 					cmn_err(CE_NOTE, "cannot instrument %s:"
1231 					    " begins with non-ba, "
1232 					    "non-br CTI", name);
1233 				}
1234 				continue;
1235 			}
1236 		}
1237 
1238 		while (!FBT_IS_SAVE(*instr)) {
1239 			/*
1240 			 * Before we assume that this is a leaf routine, check
1241 			 * forward in the basic block for a save.
1242 			 */
1243 			int op = *instr & FBT_OP_MASK;
1244 			int op2 = *instr & FBT_FMT2_OP2_MASK;
1245 
1246 			if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) {
1247 				/*
1248 				 * This is a CTI.  If we see a subsequent
1249 				 * save, we will refuse to process this
1250 				 * routine unless both of the following are
1251 				 * true:
1252 				 *
1253 				 *  (a)	The branch is not annulled
1254 				 *
1255 				 *  (b)	The subsequent save is in the delay
1256 				 *	slot of the branch
1257 				 */
1258 				if ((*instr & FBT_ANNUL) ||
1259 				    !FBT_IS_SAVE(*(instr + 1))) {
1260 					cti = 1;
1261 				} else {
1262 					instr++;
1263 					break;
1264 				}
1265 			}
1266 
1267 			if (op == FBT_OP1)
1268 				cti = 1;
1269 
1270 			if (++instr == limit)
1271 				break;
1272 		}
1273 
1274 		if (instr < limit && cti) {
1275 			/*
1276 			 * If we found a CTI before the save, we need to not
1277 			 * do anything.  But if we have CTF information, this
1278 			 * is weird enough that it merits a message.
1279 			 */
1280 			if (!have_ctf)
1281 				continue;
1282 
1283 			cmn_err(CE_NOTE, "cannot instrument %s: "
1284 			    "save not in first basic block", name);
1285 			continue;
1286 		}
1287 
1288 		if (instr == limit) {
1289 			if (!have_ctf)
1290 				continue;
1291 			is_leaf = 1;
1292 
1293 			if (!estimate)
1294 				fbt_leaf_functions++;
1295 
1296 			canpatch = fbt_canpatch_retl;
1297 			patch = fbt_patch_retl;
1298 		} else {
1299 			canpatch = fbt_canpatch_return;
1300 			patch = fbt_patch_return;
1301 		}
1302 
1303 		if (!have_ctf && !is_leaf) {
1304 			/*
1305 			 * Before we assume that this isn't something tricky,
1306 			 * look for other saves.  If we find them, there are
1307 			 * multiple entry points here (or something), and we'll
1308 			 * leave it alone.
1309 			 */
1310 			while (++instr < limit) {
1311 				if (FBT_IS_SAVE(*instr))
1312 					break;
1313 			}
1314 
1315 			if (instr != limit)
1316 				continue;
1317 		}
1318 
1319 		instr = base;
1320 
1321 		if (FBT_IS_CTI(*instr)) {
1322 			/*
1323 			 * If we have a CTI, we want to be sure that we don't
1324 			 * have a CTI or a PC-relative instruction in the
1325 			 * delay slot -- we want to be able to thunk the
1326 			 * instruction into the trampoline without worrying
1327 			 * about either DCTIs or relocations.  It would be
1328 			 * very odd for the compiler to generate this kind of
1329 			 * code, so we warn about it if we have CTF
1330 			 * information.
1331 			 */
1332 			if (FBT_IS_CTI(*(instr + 1))) {
1333 				if (!have_ctf)
1334 					continue;
1335 
1336 				cmn_err(CE_NOTE, "cannot instrument %s: "
1337 				    "CTI in delay slot of first instruction",
1338 				    name);
1339 				continue;
1340 			}
1341 
1342 			if (FBT_IS_PCRELATIVE(*(instr + 1))) {
1343 				if (!have_ctf)
1344 					continue;
1345 
1346 				cmn_err(CE_NOTE, "cannot instrument %s: "
1347 				    "PC-relative instruction in delay slot of"
1348 				    " first instruction", name);
1349 				continue;
1350 			}
1351 		}
1352 
1353 		if (estimate) {
1354 			tramp.fbtt_next = (uintptr_t)faketramp;
1355 			tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1356 			(void) fbt_patch_entry(instr, FBT_ESTIMATE_ID,
1357 			    &tramp, nargs);
1358 			fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1359 		} else {
1360 			fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1361 			fbt->fbtp_name = name;
1362 			fbt->fbtp_ctl = ctl;
1363 			fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1364 			    name, FBT_PROBENAME_ENTRY, 1, fbt);
1365 			fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va);
1366 
1367 			if (!fbt_patch_entry(instr, fbt->fbtp_id,
1368 			    &tramp, nargs)) {
1369 				cmn_err(CE_WARN, "unexpectedly short FBT table "
1370 				    "in module %s (sym %d of %d)", modname,
1371 				    i, nsyms);
1372 				break;
1373 			}
1374 
1375 			fbt->fbtp_patchpoint =
1376 			    (uint32_t *)((uintptr_t)mp->textwin +
1377 			    ((uintptr_t)instr - (uintptr_t)mp->text));
1378 			fbt->fbtp_savedval = *instr;
1379 
1380 			fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1381 			fbt->fbtp_primary = primary;
1382 			fbt->fbtp_symndx = i;
1383 			mp->fbt_nentries++;
1384 		}
1385 
1386 		retfbt = NULL;
1387 again:
1388 		if (++instr == limit)
1389 			continue;
1390 
1391 		offset = (uintptr_t)instr - (uintptr_t)base;
1392 
1393 		if (!(*canpatch)(instr, offset, name))
1394 			goto again;
1395 
1396 		if (estimate) {
1397 			tramp.fbtt_next = (uintptr_t)faketramp;
1398 			tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1399 			(void) (*patch)(instr, base, limit,
1400 			    offset, FBT_ESTIMATE_ID, &tramp, name);
1401 			fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1402 
1403 			goto again;
1404 		}
1405 
1406 		fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1407 		fbt->fbtp_name = name;
1408 		fbt->fbtp_ctl = ctl;
1409 
1410 		if (retfbt == NULL) {
1411 			fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1412 			    name, FBT_PROBENAME_RETURN, 1, fbt);
1413 		} else {
1414 			retfbt->fbtp_next = fbt;
1415 			fbt->fbtp_id = retfbt->fbtp_id;
1416 		}
1417 
1418 		fbt->fbtp_return = 1;
1419 		retfbt = fbt;
1420 
1421 		if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset,
1422 		    fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) {
1423 			cmn_err(CE_WARN, "unexpectedly short FBT table "
1424 			    "in module %s (sym %d of %d)", modname, i, nsyms);
1425 			break;
1426 		}
1427 
1428 		fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin +
1429 		    ((uintptr_t)instr - (uintptr_t)mp->text));
1430 		fbt->fbtp_savedval = *instr;
1431 		fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1432 		fbt->fbtp_primary = primary;
1433 		fbt->fbtp_symndx = i;
1434 		mp->fbt_nentries++;
1435 
1436 		goto again;
1437 	}
1438 
1439 	if (estimate) {
1440 		/*
1441 		 * Slosh on another entry's worth...
1442 		 */
1443 		fbt_size += FBT_ENT_MAXSIZE;
1444 		mp->fbt_size = fbt_size;
1445 		mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size);
1446 
1447 		if (mp->fbt_tab == NULL) {
1448 			cmn_err(CE_WARN, "couldn't allocate FBT table "
1449 			    "for module %s", modname);
1450 		} else {
1451 			estimate = 0;
1452 			goto forreal;
1453 		}
1454 	} else {
1455 		fbt_trampoline_unmap();
1456 	}
1457 
1458 error:
1459 	if (fp != NULL)
1460 		ctf_close(fp);
1461 }
1462 
1463 /*ARGSUSED*/
1464 static void
fbt_destroy(void * arg,dtrace_id_t id,void * parg)1465 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
1466 {
1467 	fbt_probe_t *fbt = parg, *next;
1468 	struct modctl *ctl = fbt->fbtp_ctl;
1469 
1470 	do {
1471 		if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) {
1472 			if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt &&
1473 			    ctl->mod_loaded) || fbt->fbtp_primary) {
1474 				((struct module *)
1475 				    (ctl->mod_mp))->fbt_nentries--;
1476 			}
1477 		}
1478 
1479 		next = fbt->fbtp_next;
1480 		kmem_free(fbt, sizeof (fbt_probe_t));
1481 		fbt = next;
1482 	} while (fbt != NULL);
1483 }
1484 
1485 /*ARGSUSED*/
1486 static int
fbt_enable(void * arg,dtrace_id_t id,void * parg)1487 fbt_enable(void *arg, dtrace_id_t id, void *parg)
1488 {
1489 	fbt_probe_t *fbt = parg, *f;
1490 	struct modctl *ctl = fbt->fbtp_ctl;
1491 
1492 	ctl->mod_nenabled++;
1493 
1494 	for (f = fbt; f != NULL; f = f->fbtp_next) {
1495 		if (f->fbtp_patchpoint == NULL) {
1496 			/*
1497 			 * Due to a shortened FBT table, this entry was never
1498 			 * completed; refuse to enable it.
1499 			 */
1500 			if (fbt_verbose) {
1501 				cmn_err(CE_NOTE, "fbt is failing for probe %s "
1502 				    "(short FBT table in %s)",
1503 				    fbt->fbtp_name, ctl->mod_modname);
1504 			}
1505 
1506 			return (0);
1507 		}
1508 	}
1509 
1510 	/*
1511 	 * If this module has disappeared since we discovered its probes,
1512 	 * refuse to enable it.
1513 	 */
1514 	if (!fbt->fbtp_primary && !ctl->mod_loaded) {
1515 		if (fbt_verbose) {
1516 			cmn_err(CE_NOTE, "fbt is failing for probe %s "
1517 			    "(module %s unloaded)",
1518 			    fbt->fbtp_name, ctl->mod_modname);
1519 		}
1520 
1521 		return (0);
1522 	}
1523 
1524 	/*
1525 	 * Now check that our modctl has the expected load count.  If it
1526 	 * doesn't, this module must have been unloaded and reloaded -- and
1527 	 * we're not going to touch it.
1528 	 */
1529 	if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) {
1530 		if (fbt_verbose) {
1531 			cmn_err(CE_NOTE, "fbt is failing for probe %s "
1532 			    "(module %s reloaded)",
1533 			    fbt->fbtp_name, ctl->mod_modname);
1534 		}
1535 
1536 		return (0);
1537 	}
1538 
1539 	for (; fbt != NULL; fbt = fbt->fbtp_next)
1540 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1541 
1542 	return (0);
1543 }
1544 
1545 /*ARGSUSED*/
1546 static void
fbt_disable(void * arg,dtrace_id_t id,void * parg)1547 fbt_disable(void *arg, dtrace_id_t id, void *parg)
1548 {
1549 	fbt_probe_t *fbt = parg, *f;
1550 	struct modctl *ctl = fbt->fbtp_ctl;
1551 
1552 	ASSERT(ctl->mod_nenabled > 0);
1553 	ctl->mod_nenabled--;
1554 
1555 	for (f = fbt; f != NULL; f = f->fbtp_next) {
1556 		if (f->fbtp_patchpoint == NULL)
1557 			return;
1558 	}
1559 
1560 	if ((!fbt->fbtp_primary && !ctl->mod_loaded) ||
1561 	    (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1562 		return;
1563 
1564 	for (; fbt != NULL; fbt = fbt->fbtp_next)
1565 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1566 }
1567 
1568 /*ARGSUSED*/
1569 static void
fbt_suspend(void * arg,dtrace_id_t id,void * parg)1570 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
1571 {
1572 	fbt_probe_t *fbt = parg;
1573 	struct modctl *ctl = fbt->fbtp_ctl;
1574 
1575 	if (!fbt->fbtp_primary && !ctl->mod_loaded)
1576 		return;
1577 
1578 	if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1579 		return;
1580 
1581 	ASSERT(ctl->mod_nenabled > 0);
1582 
1583 	for (; fbt != NULL; fbt = fbt->fbtp_next)
1584 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1585 }
1586 
1587 /*ARGSUSED*/
1588 static void
fbt_resume(void * arg,dtrace_id_t id,void * parg)1589 fbt_resume(void *arg, dtrace_id_t id, void *parg)
1590 {
1591 	fbt_probe_t *fbt = parg;
1592 	struct modctl *ctl = fbt->fbtp_ctl;
1593 
1594 	if (!fbt->fbtp_primary && !ctl->mod_loaded)
1595 		return;
1596 
1597 	if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1598 		return;
1599 
1600 	ASSERT(ctl->mod_nenabled > 0);
1601 
1602 	for (; fbt != NULL; fbt = fbt->fbtp_next)
1603 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1604 }
1605 
1606 /*ARGSUSED*/
1607 static void
fbt_getargdesc(void * arg,dtrace_id_t id,void * parg,dtrace_argdesc_t * desc)1608 fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
1609 {
1610 	fbt_probe_t *fbt = parg;
1611 	struct modctl *ctl = fbt->fbtp_ctl;
1612 	struct module *mp = ctl->mod_mp;
1613 	ctf_file_t *fp = NULL, *pfp;
1614 	ctf_funcinfo_t f;
1615 	int error;
1616 	ctf_id_t argv[32], type;
1617 	int argc = sizeof (argv) / sizeof (ctf_id_t);
1618 	const char *parent;
1619 
1620 	if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1621 		goto err;
1622 
1623 	if (fbt->fbtp_return && desc->dtargd_ndx == 0) {
1624 		(void) strcpy(desc->dtargd_native, "int");
1625 		return;
1626 	}
1627 
1628 	if ((fp = ctf_modopen(mp, &error)) == NULL) {
1629 		/*
1630 		 * We have no CTF information for this module -- and therefore
1631 		 * no args[] information.
1632 		 */
1633 		goto err;
1634 	}
1635 
1636 	/*
1637 	 * If we have a parent container, we must manually import it.
1638 	 */
1639 	if ((parent = ctf_parent_name(fp)) != NULL) {
1640 		struct modctl *mp = &modules;
1641 		struct modctl *mod = NULL;
1642 
1643 		/*
1644 		 * We must iterate over all modules to find the module that
1645 		 * is our parent.
1646 		 */
1647 		do {
1648 			if (strcmp(mp->mod_modname, parent) == 0) {
1649 				mod = mp;
1650 				break;
1651 			}
1652 		} while ((mp = mp->mod_next) != &modules);
1653 
1654 		if (mod == NULL)
1655 			goto err;
1656 
1657 		if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL)
1658 			goto err;
1659 
1660 		if (ctf_import(fp, pfp) != 0) {
1661 			ctf_close(pfp);
1662 			goto err;
1663 		}
1664 
1665 		ctf_close(pfp);
1666 	}
1667 
1668 	if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR)
1669 		goto err;
1670 
1671 	if (fbt->fbtp_return) {
1672 		if (desc->dtargd_ndx > 1)
1673 			goto err;
1674 
1675 		ASSERT(desc->dtargd_ndx == 1);
1676 		type = f.ctc_return;
1677 	} else {
1678 		if (desc->dtargd_ndx + 1 > f.ctc_argc)
1679 			goto err;
1680 
1681 		if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR)
1682 			goto err;
1683 
1684 		type = argv[desc->dtargd_ndx];
1685 	}
1686 
1687 	if (ctf_type_name(fp, type, desc->dtargd_native,
1688 	    DTRACE_ARGTYPELEN) != NULL) {
1689 		ctf_close(fp);
1690 		return;
1691 	}
1692 err:
1693 	if (fp != NULL)
1694 		ctf_close(fp);
1695 
1696 	desc->dtargd_ndx = DTRACE_ARGNONE;
1697 }
1698 
1699 static dtrace_pattr_t fbt_attr = {
1700 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1701 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1702 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1703 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1704 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
1705 };
1706 
1707 static dtrace_pops_t fbt_pops = {
1708 	NULL,
1709 	fbt_provide_module,
1710 	fbt_enable,
1711 	fbt_disable,
1712 	fbt_suspend,
1713 	fbt_resume,
1714 	fbt_getargdesc,
1715 	NULL,
1716 	NULL,
1717 	fbt_destroy
1718 };
1719 
1720 static int
fbt_attach(dev_info_t * devi,ddi_attach_cmd_t cmd)1721 fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
1722 {
1723 	switch (cmd) {
1724 	case DDI_ATTACH:
1725 		break;
1726 	case DDI_RESUME:
1727 		return (DDI_SUCCESS);
1728 	default:
1729 		return (DDI_FAILURE);
1730 	}
1731 
1732 	if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
1733 	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
1734 	    dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
1735 	    &fbt_pops, NULL, &fbt_id) != 0) {
1736 		ddi_remove_minor_node(devi, NULL);
1737 		return (DDI_FAILURE);
1738 	}
1739 
1740 	ddi_report_dev(devi);
1741 	fbt_devi = devi;
1742 	return (DDI_SUCCESS);
1743 }
1744 
1745 static int
fbt_detach(dev_info_t * devi,ddi_detach_cmd_t cmd)1746 fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
1747 {
1748 	switch (cmd) {
1749 	case DDI_DETACH:
1750 		break;
1751 	case DDI_SUSPEND:
1752 		return (DDI_SUCCESS);
1753 	default:
1754 		return (DDI_FAILURE);
1755 	}
1756 
1757 	if (dtrace_unregister(fbt_id) != 0)
1758 		return (DDI_FAILURE);
1759 
1760 	ddi_remove_minor_node(devi, NULL);
1761 	return (DDI_SUCCESS);
1762 }
1763 
1764 /*ARGSUSED*/
1765 static int
fbt_info(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** result)1766 fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1767 {
1768 	int error;
1769 
1770 	switch (infocmd) {
1771 	case DDI_INFO_DEVT2DEVINFO:
1772 		*result = (void *)fbt_devi;
1773 		error = DDI_SUCCESS;
1774 		break;
1775 	case DDI_INFO_DEVT2INSTANCE:
1776 		*result = (void *)0;
1777 		error = DDI_SUCCESS;
1778 		break;
1779 	default:
1780 		error = DDI_FAILURE;
1781 	}
1782 	return (error);
1783 }
1784 
1785 /*ARGSUSED*/
1786 static int
fbt_open(dev_t * devp,int flag,int otyp,cred_t * cred_p)1787 fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1788 {
1789 	return (0);
1790 }
1791 
1792 static struct cb_ops fbt_cb_ops = {
1793 	fbt_open,		/* open */
1794 	nodev,			/* close */
1795 	nulldev,		/* strategy */
1796 	nulldev,		/* print */
1797 	nodev,			/* dump */
1798 	nodev,			/* read */
1799 	nodev,			/* write */
1800 	nodev,			/* ioctl */
1801 	nodev,			/* devmap */
1802 	nodev,			/* mmap */
1803 	nodev,			/* segmap */
1804 	nochpoll,		/* poll */
1805 	ddi_prop_op,		/* cb_prop_op */
1806 	0,			/* streamtab  */
1807 	D_NEW | D_MP		/* Driver compatibility flag */
1808 };
1809 
1810 static struct dev_ops fbt_ops = {
1811 	DEVO_REV,		/* devo_rev */
1812 	0,			/* refcnt */
1813 	fbt_info,		/* get_dev_info */
1814 	nulldev,		/* identify */
1815 	nulldev,		/* probe */
1816 	fbt_attach,		/* attach */
1817 	fbt_detach,		/* detach */
1818 	nodev,			/* reset */
1819 	&fbt_cb_ops,		/* driver operations */
1820 	NULL,			/* bus operations */
1821 	nodev,			/* dev power */
1822 	ddi_quiesce_not_needed,		/* quiesce */
1823 };
1824 
1825 /*
1826  * Module linkage information for the kernel.
1827  */
1828 static struct modldrv modldrv = {
1829 	&mod_driverops,		/* module type (this is a pseudo driver) */
1830 	"Function Boundary Tracing",	/* name of module */
1831 	&fbt_ops,		/* driver ops */
1832 };
1833 
1834 static struct modlinkage modlinkage = {
1835 	MODREV_1,
1836 	(void *)&modldrv,
1837 	NULL
1838 };
1839 
1840 int
_init(void)1841 _init(void)
1842 {
1843 	return (mod_install(&modlinkage));
1844 }
1845 
1846 int
_info(struct modinfo * modinfop)1847 _info(struct modinfo *modinfop)
1848 {
1849 	return (mod_info(&modlinkage, modinfop));
1850 }
1851 
1852 int
_fini(void)1853 _fini(void)
1854 {
1855 	return (mod_remove(&modlinkage));
1856 }
1857