xref: /titanic_52/usr/src/uts/sparc/dtrace/dtrace_isa.c (revision 40e5e17b3361b3eea56a9723071c406894a20b78)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/dtrace_impl.h>
30 #include <sys/atomic.h>
31 #include <sys/model.h>
32 #include <sys/frame.h>
33 #include <sys/stack.h>
34 #include <sys/machpcb.h>
35 #include <sys/procfs_isa.h>
36 #include <sys/cmn_err.h>
37 #include <sys/sysmacros.h>
38 
39 #define	DTRACE_FMT3OP3_MASK	0x81000000
40 #define	DTRACE_FMT3OP3		0x80000000
41 #define	DTRACE_FMT3RS1_SHIFT	14
42 #define	DTRACE_FMT3RD_SHIFT	25
43 #define	DTRACE_DISP22_SHIFT	10
44 #define	DTRACE_RMASK		0x1f
45 #define	DTRACE_REG_L0		16
46 #define	DTRACE_REG_O7		15
47 #define	DTRACE_REG_I0		24
48 #define	DTRACE_REG_I6		30
49 #define	DTRACE_RET		0x81c7e008
50 #define	DTRACE_RETL		0x81c3e008
51 #define	DTRACE_SAVE_MASK	0xc1f80000
52 #define	DTRACE_SAVE		0x81e00000
53 #define	DTRACE_RESTORE		0x81e80000
54 #define	DTRACE_CALL_MASK	0xc0000000
55 #define	DTRACE_CALL		0x40000000
56 #define	DTRACE_JMPL_MASK	0x81f10000
57 #define	DTRACE_JMPL		0x81c00000
58 #define	DTRACE_BA_MASK		0xdfc00000
59 #define	DTRACE_BA		0x10800000
60 #define	DTRACE_BA_MAX		10
61 
62 extern int dtrace_getupcstack_top(uint64_t *, int, uintptr_t *);
63 extern int dtrace_getustackdepth_top(uintptr_t *);
64 extern ulong_t dtrace_getreg_win(uint_t, uint_t);
65 extern void dtrace_putreg_win(uint_t, ulong_t);
66 extern int dtrace_fish(int, int, uintptr_t *);
67 
68 int	dtrace_ustackdepth_max = 2048;
69 
70 /*
71  * This is similar in principle to getpcstack(), but there are several marked
72  * differences in implementation:
73  *
74  * (a)	dtrace_getpcstack() is called from probe context.  Thus, the call
75  *	to flush_windows() from getpcstack() is a call to the probe-safe
76  *	equivalent here.
77  *
78  * (b)  dtrace_getpcstack() is willing to sacrifice some performance to get
79  *	a correct stack.  While consumers of getpcstack() are largely
80  *	subsystem-specific in-kernel debugging facilities, DTrace consumers
81  *	are arbitrary user-level analysis tools; dtrace_getpcstack() must
82  *	deliver as correct a stack as possible.  Details on the issues
83  *	surrounding stack correctness are found below.
84  *
85  * (c)	dtrace_getpcstack() _always_ fills in pcstack_limit pc_t's -- filling
86  *	in the difference between the stack depth and pcstack_limit with NULLs.
87  *	Due to this behavior dtrace_getpcstack() returns void.
88  *
89  * (d)	dtrace_getpcstack() takes a third parameter, aframes, that
90  *	denotes the number of _artificial frames_ on the bottom of the
91  *	stack.  An artificial frame is one induced by the provider; all
92  *	artificial frames are stripped off before frames are stored to
93  *	pcstack.
94  *
95  * (e)	dtrace_getpcstack() takes a fourth parameter, pc, that indicates
96  *	an interrupted program counter (if any).  This should be a non-NULL
97  *	value if and only if the hit probe is unanchored.  (Anchored probes
98  *	don't fire through an interrupt source.)  This parameter is used to
99  *	assure (b), above.
100  */
101 void
102 dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, uint32_t *pc)
103 {
104 	struct frame *fp, *nextfp, *minfp, *stacktop;
105 	int depth = 0;
106 	int on_intr, j = 0;
107 	uint32_t i, r;
108 
109 	fp = (struct frame *)((caddr_t)dtrace_getfp() + STACK_BIAS);
110 	dtrace_flush_windows();
111 
112 	if (pc != NULL) {
113 		/*
114 		 * If we've been passed a non-NULL pc, we need to determine
115 		 * whether or not the specified program counter falls in a leaf
116 		 * function.  If it falls within a leaf function, we know that
117 		 * %o7 is valid in its frame (and we can just drive on).  If
118 		 * it's a non-leaf, however, we know that %o7 is garbage in the
119 		 * bottom frame.  To trim this frame, we simply increment
120 		 * aframes and drop into the stack-walking loop.
121 		 *
122 		 * To quickly determine if the specified program counter is in
123 		 * a leaf function, we exploit the fact that leaf functions
124 		 * tend to be short and non-leaf functions tend to frequently
125 		 * perform operations that are only permitted in a non-leaf
126 		 * function (e.g., using the %i's or %l's; calling a function;
127 		 * performing a restore).  We exploit these tendencies by
128 		 * simply scanning forward from the specified %pc -- if we see
129 		 * an operation only permitted in a non-leaf, we know we're in
130 		 * a non-leaf; if we see a retl, we know we're in a leaf.
131 		 * Fortunately, one need not perform anywhere near full
132 		 * disassembly to effectively determine the former: determining
133 		 * that an instruction is a format-3 instruction and decoding
134 		 * its rd and rs1 fields, for example, requires very little
135 		 * manipulation.  Overall, this method of leaf determination
136 		 * performs quite well:  on average, we only examine between
137 		 * 1.5 and 2.5 instructions before making the determination.
138 		 * (Outliers do exist, however; of note is the non-leaf
139 		 * function ip_sioctl_not_ours() which -- as of this writing --
140 		 * has a whopping 455 straight instructions that manipulate
141 		 * only %g's and %o's.)
142 		 */
143 		int delay = 0, branches = 0, taken = 0;
144 
145 		if (depth < pcstack_limit)
146 			pcstack[depth++] = (pc_t)(uintptr_t)pc;
147 
148 		/*
149 		 * Our heuristic is exactly that -- a heuristic -- and there
150 		 * exists a possibility that we could be either be vectored
151 		 * off into the weeds (by following a bogus branch) or could
152 		 * wander off the end of the function and off the end of a
153 		 * text mapping (by not following a conditional branch at the
154 		 * end of the function that is effectively always taken).  So
155 		 * as a precautionary measure, we set the NOFAULT flag.
156 		 */
157 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
158 
159 		for (;;) {
160 			i = pc[j++];
161 
162 			if ((i & DTRACE_FMT3OP3_MASK) == DTRACE_FMT3OP3) {
163 				/*
164 				 * This is a format-3 instruction.  We can
165 				 * look at rd and rs1.
166 				 */
167 				r = (i >> DTRACE_FMT3RS1_SHIFT) & DTRACE_RMASK;
168 
169 				if (r >= DTRACE_REG_L0)
170 					goto nonleaf;
171 
172 				r = (i >> DTRACE_FMT3RD_SHIFT) & DTRACE_RMASK;
173 
174 				if (r >= DTRACE_REG_L0)
175 					goto nonleaf;
176 
177 				if ((i & DTRACE_JMPL_MASK) == DTRACE_JMPL) {
178 					delay = 1;
179 					continue;
180 				}
181 
182 				/*
183 				 * If we see explicit manipulation with %o7
184 				 * as a destination register, we know that
185 				 * %o7 is likely bogus -- and we treat this
186 				 * function as a non-leaf.
187 				 */
188 				if (r == DTRACE_REG_O7) {
189 					if (delay)
190 						goto leaf;
191 
192 					i &= DTRACE_JMPL_MASK;
193 
194 					if (i == DTRACE_JMPL) {
195 						delay = 1;
196 						continue;
197 					}
198 
199 					goto nonleaf;
200 				}
201 			} else {
202 				/*
203 				 * If this is a call, it may or may not be
204 				 * a leaf; we need to check the delay slot.
205 				 */
206 				if ((i & DTRACE_CALL_MASK) == DTRACE_CALL) {
207 					delay = 1;
208 					continue;
209 				}
210 
211 				/*
212 				 * If we see a ret it's not a leaf; if we
213 				 * see a retl, it is a leaf.
214 				 */
215 				if (i == DTRACE_RET)
216 					goto nonleaf;
217 
218 				if (i == DTRACE_RETL)
219 					goto leaf;
220 
221 				/*
222 				 * If this is a ba (annulled or not), then we
223 				 * need to actually follow the branch.  No, we
224 				 * don't look at the delay slot -- hopefully
225 				 * anything that can be gleaned from the delay
226 				 * slot can also be gleaned from the branch
227 				 * target.  To prevent ourselves from iterating
228 				 * infinitely, we clamp the number of branches
229 				 * that we'll follow, and we refuse to follow
230 				 * the same branch twice consecutively.  In
231 				 * both cases, we abort by deciding that we're
232 				 * looking at a leaf.  While in theory this
233 				 * could be wrong (we could be in the middle of
234 				 * a loop in a non-leaf that ends with a ba and
235 				 * only manipulates outputs and globals in the
236 				 * body of the loop -- therefore leading us to
237 				 * the wrong conclusion), this doesn't seem to
238 				 * crop up in practice.  (Or rather, this
239 				 * condition could not be deliberately induced,
240 				 * despite concerted effort.)
241 				 */
242 				if ((i & DTRACE_BA_MASK) == DTRACE_BA) {
243 					if (++branches == DTRACE_BA_MAX ||
244 					    taken == j)
245 						goto nonleaf;
246 
247 					taken = j;
248 					j += ((int)(i << DTRACE_DISP22_SHIFT) >>
249 					    DTRACE_DISP22_SHIFT) - 1;
250 					continue;
251 				}
252 
253 				/*
254 				 * Finally, if it's a save, it should be
255 				 * treated as a leaf; if it's a restore it
256 				 * should not be treated as a leaf.
257 				 */
258 				if ((i & DTRACE_SAVE_MASK) == DTRACE_SAVE)
259 					goto leaf;
260 
261 				if ((i & DTRACE_SAVE_MASK) == DTRACE_RESTORE)
262 					goto nonleaf;
263 			}
264 
265 			if (delay) {
266 				/*
267 				 * If this was a delay slot instruction and
268 				 * we didn't pick it up elsewhere, this is a
269 				 * non-leaf.
270 				 */
271 				goto nonleaf;
272 			}
273 		}
274 nonleaf:
275 		aframes++;
276 leaf:
277 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
278 	}
279 
280 	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
281 		stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
282 	else
283 		stacktop = (struct frame *)curthread->t_stk;
284 	minfp = fp;
285 
286 	while (depth < pcstack_limit) {
287 		nextfp = (struct frame *)((caddr_t)fp->fr_savfp + STACK_BIAS);
288 		if (nextfp <= minfp || nextfp >= stacktop) {
289 			if (!on_intr && nextfp == stacktop && aframes != 0) {
290 				/*
291 				 * If we are exactly at the top of the stack
292 				 * with a non-zero number of artificial frames,
293 				 * it must be that the stack is filled with
294 				 * nothing _but_ artificial frames.  In this
295 				 * case, we assert that this is so, zero
296 				 * pcstack, and return.
297 				 */
298 				ASSERT(aframes == 1);
299 				ASSERT(depth == 0);
300 
301 				while (depth < pcstack_limit)
302 					pcstack[depth++] = NULL;
303 				return;
304 			}
305 
306 			if (on_intr) {
307 				/*
308 				 * Hop from interrupt stack to thread stack.
309 				 */
310 				stacktop = (struct frame *)curthread->t_stk;
311 				minfp = (struct frame *)curthread->t_stkbase;
312 
313 				on_intr = 0;
314 
315 				if (nextfp > minfp && nextfp < stacktop)
316 					continue;
317 			} else {
318 				/*
319 				 * High-level interrupts may occur when %sp is
320 				 * not necessarily contained in the stack
321 				 * bounds implied by %g7 -- interrupt thread
322 				 * management runs with %pil at DISP_LEVEL,
323 				 * and high-level interrupts may thus occur
324 				 * in windows when %sp and %g7 are not self-
325 				 * consistent.  If we call dtrace_getpcstack()
326 				 * from a high-level interrupt that has occurred
327 				 * in such a window, we will fail the above test
328 				 * of nextfp against minfp/stacktop.  If the
329 				 * high-level interrupt has in turn interrupted
330 				 * a non-passivated interrupt thread, we
331 				 * will execute the below code with non-zero
332 				 * aframes.  We therefore want to assert that
333 				 * aframes is zero _or_ we are in a high-level
334 				 * interrupt -- but because cpu_intr_actv is
335 				 * updated with high-level interrupts enabled,
336 				 * we must reduce this to only asserting that
337 				 * %pil is greater than DISP_LEVEL.
338 				 */
339 				ASSERT(aframes == 0 ||
340 				    dtrace_getipl() > DISP_LEVEL);
341 				pcstack[depth++] = (pc_t)fp->fr_savpc;
342 			}
343 
344 			while (depth < pcstack_limit)
345 				pcstack[depth++] = NULL;
346 			return;
347 		}
348 
349 		if (aframes > 0) {
350 			aframes--;
351 		} else {
352 			pcstack[depth++] = (pc_t)fp->fr_savpc;
353 		}
354 
355 		fp = nextfp;
356 		minfp = fp;
357 	}
358 }
359 
360 static int
361 dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t sp)
362 {
363 	proc_t *p = curproc;
364 	int ret = 0;
365 	uintptr_t oldsp;
366 	volatile uint16_t *flags =
367 	    (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
368 
369 	ASSERT(pcstack == NULL || pcstack_limit > 0);
370 	ASSERT(dtrace_ustackdepth_max > 0);
371 
372 	if (p->p_model == DATAMODEL_NATIVE) {
373 		for (;;) {
374 			struct frame *fr = (struct frame *)(sp + STACK_BIAS);
375 			uintptr_t pc;
376 
377 			if (sp == 0 || fr == NULL ||
378 			    !IS_P2ALIGNED((uintptr_t)fr, STACK_ALIGN))
379 				break;
380 
381 			oldsp = sp;
382 
383 			pc = dtrace_fulword(&fr->fr_savpc);
384 			sp = dtrace_fulword(&fr->fr_savfp);
385 
386 			if (pc == 0)
387 				break;
388 
389 			/*
390 			 * We limit the number of times we can go around this
391 			 * loop to account for a circular stack.
392 			 */
393 			if (sp == oldsp || ret++ >= dtrace_ustackdepth_max) {
394 				*flags |= CPU_DTRACE_BADSTACK;
395 				cpu_core[CPU->cpu_id].cpuc_dtrace_illval = sp;
396 				break;
397 			}
398 
399 			if (pcstack != NULL) {
400 				*pcstack++ = pc;
401 				pcstack_limit--;
402 				if (pcstack_limit == 0)
403 					break;
404 			}
405 		}
406 	} else {
407 		/*
408 		 * Truncate the stack pointer to 32-bits as there may be
409 		 * garbage in the upper bits which would normally be ignored
410 		 * by the processor in 32-bit mode.
411 		 */
412 		sp = (uint32_t)sp;
413 
414 		for (;;) {
415 			struct frame32 *fr = (struct frame32 *)sp;
416 			uint32_t pc;
417 
418 			if (sp == 0 ||
419 			    !IS_P2ALIGNED((uintptr_t)fr, STACK_ALIGN32))
420 				break;
421 
422 			oldsp = sp;
423 
424 			pc = dtrace_fuword32(&fr->fr_savpc);
425 			sp = dtrace_fuword32(&fr->fr_savfp);
426 
427 			if (pc == 0)
428 				break;
429 
430 			if (sp == oldsp || ret++ >= dtrace_ustackdepth_max) {
431 				*flags |= CPU_DTRACE_BADSTACK;
432 				cpu_core[CPU->cpu_id].cpuc_dtrace_illval = sp;
433 				break;
434 			}
435 
436 			if (pcstack != NULL) {
437 				*pcstack++ = pc;
438 				pcstack_limit--;
439 				if (pcstack_limit == 0)
440 					break;
441 			}
442 		}
443 	}
444 
445 	return (ret);
446 }
447 
448 void
449 dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit)
450 {
451 	klwp_t *lwp = ttolwp(curthread);
452 	proc_t *p = curproc;
453 	struct regs *rp;
454 	uintptr_t sp;
455 	int n;
456 
457 	if (pcstack_limit <= 0)
458 		return;
459 
460 	/*
461 	 * If there's no user context we still need to zero the stack.
462 	 */
463 	if (lwp == NULL || p == NULL || (rp = lwp->lwp_regs) == NULL)
464 		goto zero;
465 
466 	*pcstack++ = (uint64_t)p->p_pid;
467 	pcstack_limit--;
468 
469 	if (pcstack_limit <= 0)
470 		return;
471 
472 	*pcstack++ = (uint64_t)rp->r_pc;
473 	pcstack_limit--;
474 
475 	if (pcstack_limit <= 0)
476 		return;
477 
478 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
479 		*pcstack++ = (uint64_t)rp->r_o7;
480 		pcstack_limit--;
481 		if (pcstack_limit <= 0)
482 			return;
483 	}
484 
485 	sp = rp->r_sp;
486 
487 	n = dtrace_getupcstack_top(pcstack, pcstack_limit, &sp);
488 	ASSERT(n >= 0);
489 	ASSERT(n <= pcstack_limit);
490 
491 	pcstack += n;
492 	pcstack_limit -= n;
493 	if (pcstack_limit <= 0)
494 		return;
495 
496 	n = dtrace_getustack_common(pcstack, pcstack_limit, sp);
497 	ASSERT(n >= 0);
498 	ASSERT(n <= pcstack_limit);
499 
500 	pcstack += n;
501 	pcstack_limit -= n;
502 
503 zero:
504 	while (pcstack_limit-- > 0)
505 		*pcstack++ = NULL;
506 }
507 
508 int
509 dtrace_getustackdepth(void)
510 {
511 	klwp_t *lwp = ttolwp(curthread);
512 	proc_t *p = curproc;
513 	struct regs *rp;
514 	uintptr_t sp;
515 	int n = 1;
516 
517 	if (lwp == NULL || p == NULL || (rp = lwp->lwp_regs) == NULL)
518 		return (0);
519 
520 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
521 		return (-1);
522 
523 	sp = rp->r_sp;
524 
525 	n += dtrace_getustackdepth_top(&sp);
526 	n += dtrace_getustack_common(NULL, 0, sp);
527 
528 	/*
529 	 * Add one more to the stack depth if we're in an entry probe as long
530 	 * as the return address is non-NULL or there are additional frames
531 	 * beyond that NULL return address.
532 	 */
533 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY) &&
534 	    (rp->r_o7 != NULL || n != 1))
535 		n++;
536 
537 	return (n);
538 }
539 
540 void
541 dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit)
542 {
543 	klwp_t *lwp = ttolwp(curthread);
544 	proc_t *p = ttoproc(curthread);
545 	struct regs *rp;
546 	uintptr_t sp;
547 
548 	if (pcstack_limit <= 0)
549 		return;
550 
551 	/*
552 	 * If there's no user context we still need to zero the stack.
553 	 */
554 	if (lwp == NULL || p == NULL || (rp = lwp->lwp_regs) == NULL)
555 		goto zero;
556 
557 	*pcstack++ = (uint64_t)p->p_pid;
558 	pcstack_limit--;
559 
560 	if (pcstack_limit <= 0)
561 		return;
562 
563 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
564 		*fpstack++ = 0;
565 		*pcstack++ = (uint64_t)rp->r_pc;
566 		pcstack_limit--;
567 		if (pcstack_limit <= 0)
568 			return;
569 
570 		*fpstack++ = (uint64_t)rp->r_sp;
571 		*pcstack++ = (uint64_t)rp->r_o7;
572 		pcstack_limit--;
573 	} else {
574 		*fpstack++ = (uint64_t)rp->r_sp;
575 		*pcstack++ = (uint64_t)rp->r_pc;
576 		pcstack_limit--;
577 	}
578 
579 	if (pcstack_limit <= 0)
580 		return;
581 
582 	sp = rp->r_sp;
583 
584 	dtrace_flush_user_windows();
585 
586 	if (p->p_model == DATAMODEL_NATIVE) {
587 		while (pcstack_limit > 0) {
588 			struct frame *fr = (struct frame *)(sp + STACK_BIAS);
589 			uintptr_t pc;
590 
591 			if (sp == 0 || fr == NULL ||
592 			    ((uintptr_t)&fr->fr_savpc & 3) != 0 ||
593 			    ((uintptr_t)&fr->fr_savfp & 3) != 0)
594 				break;
595 
596 			pc = dtrace_fulword(&fr->fr_savpc);
597 			sp = dtrace_fulword(&fr->fr_savfp);
598 
599 			if (pc == 0)
600 				break;
601 
602 			*fpstack++ = sp;
603 			*pcstack++ = pc;
604 			pcstack_limit--;
605 		}
606 	} else {
607 		/*
608 		 * Truncate the stack pointer to 32-bits as there may be
609 		 * garbage in the upper bits which would normally be ignored
610 		 * by the processor in 32-bit mode.
611 		 */
612 		sp = (uint32_t)sp;
613 
614 		while (pcstack_limit > 0) {
615 			struct frame32 *fr = (struct frame32 *)sp;
616 			uint32_t pc;
617 
618 			if (sp == 0 ||
619 			    ((uintptr_t)&fr->fr_savpc & 3) != 0 ||
620 			    ((uintptr_t)&fr->fr_savfp & 3) != 0)
621 				break;
622 
623 			pc = dtrace_fuword32(&fr->fr_savpc);
624 			sp = dtrace_fuword32(&fr->fr_savfp);
625 
626 			if (pc == 0)
627 				break;
628 
629 			*fpstack++ = sp;
630 			*pcstack++ = pc;
631 			pcstack_limit--;
632 		}
633 	}
634 
635 zero:
636 	while (pcstack_limit-- > 0)
637 		*pcstack++ = NULL;
638 }
639 
640 uint64_t
641 dtrace_getarg(int arg, int aframes)
642 {
643 	uintptr_t val;
644 	struct frame *fp;
645 	uint64_t rval;
646 
647 	/*
648 	 * Account for the fact that dtrace_getarg() consumes an additional
649 	 * stack frame.
650 	 */
651 	aframes++;
652 
653 	if (arg < 6) {
654 		if (dtrace_fish(aframes, DTRACE_REG_I0 + arg, &val) == 0)
655 			return (val);
656 	} else {
657 		if (dtrace_fish(aframes, DTRACE_REG_I6, &val) == 0) {
658 			/*
659 			 * We have a stack pointer; grab the argument.
660 			 */
661 			fp = (struct frame *)(val + STACK_BIAS);
662 
663 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
664 			rval = fp->fr_argx[arg - 6];
665 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
666 
667 			return (rval);
668 		}
669 	}
670 
671 	/*
672 	 * There are other ways to do this.  But the slow, painful way works
673 	 * just fine.  Because this requires some loads, we need to set
674 	 * CPU_DTRACE_NOFAULT to protect against looking for an argument that
675 	 * isn't there.
676 	 */
677 	fp = (struct frame *)((caddr_t)dtrace_getfp() + STACK_BIAS);
678 	dtrace_flush_windows();
679 
680 	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
681 
682 	for (aframes -= 1; aframes; aframes--)
683 		fp = (struct frame *)((caddr_t)fp->fr_savfp + STACK_BIAS);
684 
685 	if (arg < 6) {
686 		rval = fp->fr_arg[arg];
687 	} else {
688 		fp = (struct frame *)((caddr_t)fp->fr_savfp + STACK_BIAS);
689 		rval = fp->fr_argx[arg - 6];
690 	}
691 
692 	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
693 
694 	return (rval);
695 }
696 
697 int
698 dtrace_getstackdepth(int aframes)
699 {
700 	struct frame *fp, *nextfp, *minfp, *stacktop;
701 	int depth = 0;
702 	int on_intr;
703 
704 	fp = (struct frame *)((caddr_t)dtrace_getfp() + STACK_BIAS);
705 	dtrace_flush_windows();
706 
707 	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
708 		stacktop = (struct frame *)CPU->cpu_intr_stack + SA(MINFRAME);
709 	else
710 		stacktop = (struct frame *)curthread->t_stk;
711 	minfp = fp;
712 
713 	for (;;) {
714 		nextfp = (struct frame *)((caddr_t)fp->fr_savfp + STACK_BIAS);
715 		if (nextfp <= minfp || nextfp >= stacktop) {
716 			if (on_intr) {
717 				/*
718 				 * Hop from interrupt stack to thread stack.
719 				 */
720 				stacktop = (struct frame *)curthread->t_stk;
721 				minfp = (struct frame *)curthread->t_stkbase;
722 				on_intr = 0;
723 				continue;
724 			}
725 
726 			return (++depth);
727 		}
728 
729 		if (aframes > 0) {
730 			aframes--;
731 		} else {
732 			depth++;
733 		}
734 
735 		fp = nextfp;
736 		minfp = fp;
737 	}
738 }
739 
740 /*
741  * This uses the same register numbering scheme as in sys/procfs_isa.h.
742  */
743 ulong_t
744 dtrace_getreg(struct regs *rp, uint_t reg)
745 {
746 	ulong_t value;
747 	uintptr_t fp;
748 	struct machpcb *mpcb;
749 
750 	if (reg == R_G0)
751 		return (0);
752 
753 	if (reg <= R_G7)
754 		return ((&rp->r_g1)[reg - 1]);
755 
756 	if (reg > R_I7) {
757 		switch (reg) {
758 		case R_CCR:
759 			return ((rp->r_tstate >> TSTATE_CCR_SHIFT) &
760 			    TSTATE_CCR_MASK);
761 		case R_PC:
762 			return (rp->r_pc);
763 		case R_nPC:
764 			return (rp->r_npc);
765 		case R_Y:
766 			return (rp->r_y);
767 		case R_ASI:
768 			return ((rp->r_tstate >> TSTATE_ASI_SHIFT) &
769 			    TSTATE_ASI_MASK);
770 		case R_FPRS:
771 			return (dtrace_getfprs());
772 		default:
773 			DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
774 			return (0);
775 		}
776 	}
777 
778 	/*
779 	 * We reach go to the fake restore case if the probe we hit was a pid
780 	 * return probe on a restore instruction. We partially emulate the
781 	 * restore in the kernel and then execute a simple restore
782 	 * instruction that we've secreted away to do the actual register
783 	 * window manipulation. We need to go one register window further
784 	 * down to get at the %ls, and %is and we need to treat %os like %is
785 	 * to pull them out of the topmost user frame.
786 	 */
787 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAKERESTORE)) {
788 		if (reg > R_O7)
789 			goto fake_restore;
790 		else
791 			reg += R_I0 - R_O0;
792 
793 	} else if (reg <= R_O7) {
794 		return ((&rp->r_g1)[reg - 1]);
795 	}
796 
797 	if (dtrace_getotherwin() > 0)
798 		return (dtrace_getreg_win(reg, 1));
799 
800 	mpcb = (struct machpcb *)((caddr_t)rp - REGOFF);
801 
802 	if (curproc->p_model == DATAMODEL_NATIVE) {
803 		struct frame *fr = (void *)(rp->r_sp + STACK_BIAS);
804 
805 		if (mpcb->mpcb_wbcnt > 0) {
806 			struct rwindow *rwin = (void *)mpcb->mpcb_wbuf;
807 			int i = mpcb->mpcb_wbcnt;
808 			do {
809 				i--;
810 				if ((long)mpcb->mpcb_spbuf[i] == rp->r_sp)
811 					return (rwin[i].rw_local[reg - 16]);
812 			} while (i > 0);
813 		}
814 
815 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
816 		value = dtrace_fulword(&fr->fr_local[reg - 16]);
817 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
818 	} else {
819 		struct frame32 *fr = (void *)(uintptr_t)(caddr32_t)rp->r_sp;
820 
821 		if (mpcb->mpcb_wbcnt > 0) {
822 			struct rwindow32 *rwin = (void *)mpcb->mpcb_wbuf;
823 			int i = mpcb->mpcb_wbcnt;
824 			do {
825 				i--;
826 				if ((long)mpcb->mpcb_spbuf[i] == rp->r_sp)
827 					return (rwin[i].rw_local[reg - 16]);
828 			} while (i > 0);
829 		}
830 
831 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
832 		value = dtrace_fuword32(&fr->fr_local[reg - 16]);
833 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
834 	}
835 
836 	return (value);
837 
838 fake_restore:
839 	ASSERT(R_L0 <= reg && reg <= R_I7);
840 
841 	/*
842 	 * We first look two user windows down to see if we can dig out
843 	 * the register we're looking for.
844 	 */
845 	if (dtrace_getotherwin() > 1)
846 		return (dtrace_getreg_win(reg, 2));
847 
848 	/*
849 	 * First we need to get the frame pointer and then we perform
850 	 * the same computation as in the non-fake-o-restore case.
851 	 */
852 
853 	mpcb = (struct machpcb *)((caddr_t)rp - REGOFF);
854 
855 	if (dtrace_getotherwin() > 0) {
856 		fp = dtrace_getreg_win(R_FP, 1);
857 		goto got_fp;
858 	}
859 
860 	if (curproc->p_model == DATAMODEL_NATIVE) {
861 		struct frame *fr = (void *)(rp->r_sp + STACK_BIAS);
862 
863 		if (mpcb->mpcb_wbcnt > 0) {
864 			struct rwindow *rwin = (void *)mpcb->mpcb_wbuf;
865 			int i = mpcb->mpcb_wbcnt;
866 			do {
867 				i--;
868 				if ((long)mpcb->mpcb_spbuf[i] == rp->r_sp) {
869 					fp = rwin[i].rw_fp;
870 					goto got_fp;
871 				}
872 			} while (i > 0);
873 		}
874 
875 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
876 		fp = dtrace_fulword(&fr->fr_savfp);
877 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
878 		if (cpu_core[CPU->cpu_id].cpuc_dtrace_flags & CPU_DTRACE_FAULT)
879 			return (0);
880 	} else {
881 		struct frame32 *fr = (void *)(uintptr_t)(caddr32_t)rp->r_sp;
882 
883 		if (mpcb->mpcb_wbcnt > 0) {
884 			struct rwindow32 *rwin = (void *)mpcb->mpcb_wbuf;
885 			int i = mpcb->mpcb_wbcnt;
886 			do {
887 				i--;
888 				if ((long)mpcb->mpcb_spbuf[i] == rp->r_sp) {
889 					fp = rwin[i].rw_fp;
890 					goto got_fp;
891 				}
892 			} while (i > 0);
893 		}
894 
895 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
896 		fp = dtrace_fuword32(&fr->fr_savfp);
897 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
898 		if (cpu_core[CPU->cpu_id].cpuc_dtrace_flags & CPU_DTRACE_FAULT)
899 			return (0);
900 	}
901 got_fp:
902 
903 	if (curproc->p_model == DATAMODEL_NATIVE) {
904 		struct frame *fr = (void *)(fp + STACK_BIAS);
905 
906 		if (mpcb->mpcb_wbcnt > 0) {
907 			struct rwindow *rwin = (void *)mpcb->mpcb_wbuf;
908 			int i = mpcb->mpcb_wbcnt;
909 			do {
910 				i--;
911 				if ((long)mpcb->mpcb_spbuf[i] == fp)
912 					return (rwin[i].rw_local[reg - 16]);
913 			} while (i > 0);
914 		}
915 
916 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
917 		value = dtrace_fulword(&fr->fr_local[reg - 16]);
918 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
919 	} else {
920 		struct frame32 *fr = (void *)(uintptr_t)(caddr32_t)fp;
921 
922 		if (mpcb->mpcb_wbcnt > 0) {
923 			struct rwindow32 *rwin = (void *)mpcb->mpcb_wbuf;
924 			int i = mpcb->mpcb_wbcnt;
925 			do {
926 				i--;
927 				if ((long)mpcb->mpcb_spbuf[i] == fp)
928 					return (rwin[i].rw_local[reg - 16]);
929 			} while (i > 0);
930 		}
931 
932 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
933 		value = dtrace_fuword32(&fr->fr_local[reg - 16]);
934 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
935 	}
936 
937 	return (value);
938 }
939