xref: /freebsd/sys/cddl/dev/dtrace/amd64/dtrace_isa.c (revision 9aabab09c4e163fa8187b265d6836d822516b1c9)
191eaf3e1SJohn Birrell /*
291eaf3e1SJohn Birrell  * CDDL HEADER START
391eaf3e1SJohn Birrell  *
491eaf3e1SJohn Birrell  * The contents of this file are subject to the terms of the
591eaf3e1SJohn Birrell  * Common Development and Distribution License, Version 1.0 only
691eaf3e1SJohn Birrell  * (the "License").  You may not use this file except in compliance
791eaf3e1SJohn Birrell  * with the License.
891eaf3e1SJohn Birrell  *
991eaf3e1SJohn Birrell  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
1091eaf3e1SJohn Birrell  * or http://www.opensolaris.org/os/licensing.
1191eaf3e1SJohn Birrell  * See the License for the specific language governing permissions
1291eaf3e1SJohn Birrell  * and limitations under the License.
1391eaf3e1SJohn Birrell  *
1491eaf3e1SJohn Birrell  * When distributing Covered Code, include this CDDL HEADER in each
1591eaf3e1SJohn Birrell  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1691eaf3e1SJohn Birrell  * If applicable, add the following below this CDDL HEADER, with the
1791eaf3e1SJohn Birrell  * fields enclosed by brackets "[]" replaced with your own identifying
1891eaf3e1SJohn Birrell  * information: Portions Copyright [yyyy] [name of copyright owner]
1991eaf3e1SJohn Birrell  *
2091eaf3e1SJohn Birrell  * CDDL HEADER END
2191eaf3e1SJohn Birrell  */
2291eaf3e1SJohn Birrell /*
2391eaf3e1SJohn Birrell  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
2491eaf3e1SJohn Birrell  * Use is subject to license terms.
2591eaf3e1SJohn Birrell  */
2691eaf3e1SJohn Birrell #include <sys/cdefs.h>
2791eaf3e1SJohn Birrell 
2891eaf3e1SJohn Birrell #include <sys/param.h>
2991eaf3e1SJohn Birrell #include <sys/systm.h>
3082283cadSMark Johnston #include <sys/dtrace_impl.h>
3191eaf3e1SJohn Birrell #include <sys/kernel.h>
3291eaf3e1SJohn Birrell #include <sys/stack.h>
3391eaf3e1SJohn Birrell #include <sys/pcpu.h>
3491eaf3e1SJohn Birrell 
35*9aabab09SMark Johnston #include <cddl/dev/dtrace/dtrace_cddl.h>
36*9aabab09SMark Johnston 
3791eaf3e1SJohn Birrell #include <machine/frame.h>
3891eaf3e1SJohn Birrell #include <machine/md_var.h>
3991eaf3e1SJohn Birrell #include <machine/stack.h>
408ca79fbdSMateusz Guzik #include <x86/ifunc.h>
4191eaf3e1SJohn Birrell 
4291eaf3e1SJohn Birrell #include <vm/vm.h>
4391eaf3e1SJohn Birrell #include <vm/vm_param.h>
4491eaf3e1SJohn Birrell #include <vm/pmap.h>
4591eaf3e1SJohn Birrell 
46c6f5742fSRui Paulo #include "regset.h"
4791eaf3e1SJohn Birrell 
4891eaf3e1SJohn Birrell uint8_t dtrace_fuword8_nocheck(void *);
4991eaf3e1SJohn Birrell uint16_t dtrace_fuword16_nocheck(void *);
5091eaf3e1SJohn Birrell uint32_t dtrace_fuword32_nocheck(void *);
5191eaf3e1SJohn Birrell uint64_t dtrace_fuword64_nocheck(void *);
5291eaf3e1SJohn Birrell 
5309a15aa3SMark Johnston int	dtrace_ustackdepth_max = 2048;
5409a15aa3SMark Johnston 
5591eaf3e1SJohn Birrell void
5691eaf3e1SJohn Birrell dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes,
5791eaf3e1SJohn Birrell     uint32_t *intrpc)
5891eaf3e1SJohn Birrell {
59fcc0db17SMark Johnston 	struct thread *td;
6091eaf3e1SJohn Birrell 	int depth = 0;
6191eaf3e1SJohn Birrell 	register_t rbp;
6291eaf3e1SJohn Birrell 	struct amd64_frame *frame;
6391eaf3e1SJohn Birrell 	vm_offset_t callpc;
6491eaf3e1SJohn Birrell 	pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller;
6591eaf3e1SJohn Birrell 
6691eaf3e1SJohn Birrell 	if (intrpc != 0)
6791eaf3e1SJohn Birrell 		pcstack[depth++] = (pc_t) intrpc;
6891eaf3e1SJohn Birrell 
6991eaf3e1SJohn Birrell 	aframes++;
7091eaf3e1SJohn Birrell 
7191eaf3e1SJohn Birrell 	__asm __volatile("movq %%rbp,%0" : "=r" (rbp));
7291eaf3e1SJohn Birrell 
7391eaf3e1SJohn Birrell 	frame = (struct amd64_frame *)rbp;
74fcc0db17SMark Johnston 	td = curthread;
7591eaf3e1SJohn Birrell 	while (depth < pcstack_limit) {
765941edfcSJohn Baldwin 		if (!kstack_contains(curthread, (vm_offset_t)frame,
7724f10b03SKonstantin Belousov 		    sizeof(*frame)))
78fcc0db17SMark Johnston 			break;
79fcc0db17SMark Johnston 
8091eaf3e1SJohn Birrell 		callpc = frame->f_retaddr;
8191eaf3e1SJohn Birrell 
8291eaf3e1SJohn Birrell 		if (!INKERNEL(callpc))
8391eaf3e1SJohn Birrell 			break;
8491eaf3e1SJohn Birrell 
8591eaf3e1SJohn Birrell 		if (aframes > 0) {
8691eaf3e1SJohn Birrell 			aframes--;
8791eaf3e1SJohn Birrell 			if ((aframes == 0) && (caller != 0)) {
8891eaf3e1SJohn Birrell 				pcstack[depth++] = caller;
8991eaf3e1SJohn Birrell 			}
90fcc0db17SMark Johnston 		} else {
9191eaf3e1SJohn Birrell 			pcstack[depth++] = callpc;
9291eaf3e1SJohn Birrell 		}
9391eaf3e1SJohn Birrell 
94fcc0db17SMark Johnston 		if ((vm_offset_t)frame->f_frame <= (vm_offset_t)frame)
9591eaf3e1SJohn Birrell 			break;
9691eaf3e1SJohn Birrell 		frame = frame->f_frame;
9791eaf3e1SJohn Birrell 	}
9891eaf3e1SJohn Birrell 
9991eaf3e1SJohn Birrell 	for (; depth < pcstack_limit; depth++) {
10091eaf3e1SJohn Birrell 		pcstack[depth] = 0;
10191eaf3e1SJohn Birrell 	}
10291eaf3e1SJohn Birrell }
10391eaf3e1SJohn Birrell 
10491eaf3e1SJohn Birrell static int
10591eaf3e1SJohn Birrell dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc,
10691eaf3e1SJohn Birrell     uintptr_t sp)
10791eaf3e1SJohn Birrell {
10809a15aa3SMark Johnston 	uintptr_t oldsp;
10991eaf3e1SJohn Birrell 	volatile uint16_t *flags =
11091eaf3e1SJohn Birrell 	    (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
11191eaf3e1SJohn Birrell 	int ret = 0;
11291eaf3e1SJohn Birrell 
11391eaf3e1SJohn Birrell 	ASSERT(pcstack == NULL || pcstack_limit > 0);
11409a15aa3SMark Johnston 	ASSERT(dtrace_ustackdepth_max > 0);
11591eaf3e1SJohn Birrell 
116c6f5742fSRui Paulo 	while (pc != 0) {
11709a15aa3SMark Johnston 		/*
11809a15aa3SMark Johnston 		 * We limit the number of times we can go around this
11909a15aa3SMark Johnston 		 * loop to account for a circular stack.
12009a15aa3SMark Johnston 		 */
12109a15aa3SMark Johnston 		if (ret++ >= dtrace_ustackdepth_max) {
12209a15aa3SMark Johnston 			*flags |= CPU_DTRACE_BADSTACK;
12309a15aa3SMark Johnston 			cpu_core[curcpu].cpuc_dtrace_illval = sp;
12409a15aa3SMark Johnston 			break;
12509a15aa3SMark Johnston 		}
12609a15aa3SMark Johnston 
12791eaf3e1SJohn Birrell 		if (pcstack != NULL) {
12891eaf3e1SJohn Birrell 			*pcstack++ = (uint64_t)pc;
12991eaf3e1SJohn Birrell 			pcstack_limit--;
13091eaf3e1SJohn Birrell 			if (pcstack_limit <= 0)
13191eaf3e1SJohn Birrell 				break;
13291eaf3e1SJohn Birrell 		}
13391eaf3e1SJohn Birrell 
134c6f5742fSRui Paulo 		if (sp == 0)
135c6f5742fSRui Paulo 			break;
13691eaf3e1SJohn Birrell 
13709a15aa3SMark Johnston 		oldsp = sp;
13809a15aa3SMark Johnston 
139c6f5742fSRui Paulo 		pc = dtrace_fuword64((void *)(sp +
140c6f5742fSRui Paulo 			offsetof(struct amd64_frame, f_retaddr)));
141c6f5742fSRui Paulo 		sp = dtrace_fuword64((void *)sp);
14291eaf3e1SJohn Birrell 
14309a15aa3SMark Johnston 		if (sp == oldsp) {
14409a15aa3SMark Johnston 			*flags |= CPU_DTRACE_BADSTACK;
14509a15aa3SMark Johnston 			cpu_core[curcpu].cpuc_dtrace_illval = sp;
14609a15aa3SMark Johnston 			break;
14709a15aa3SMark Johnston 		}
14809a15aa3SMark Johnston 
14991eaf3e1SJohn Birrell 		/*
15091eaf3e1SJohn Birrell 		 * This is totally bogus:  if we faulted, we're going to clear
15191eaf3e1SJohn Birrell 		 * the fault and break.  This is to deal with the apparently
15291eaf3e1SJohn Birrell 		 * broken Java stacks on x86.
15391eaf3e1SJohn Birrell 		 */
15491eaf3e1SJohn Birrell 		if (*flags & CPU_DTRACE_FAULT) {
15591eaf3e1SJohn Birrell 			*flags &= ~CPU_DTRACE_FAULT;
15691eaf3e1SJohn Birrell 			break;
15791eaf3e1SJohn Birrell 		}
15891eaf3e1SJohn Birrell 	}
15991eaf3e1SJohn Birrell 
16091eaf3e1SJohn Birrell 	return (ret);
16191eaf3e1SJohn Birrell }
16291eaf3e1SJohn Birrell 
16391eaf3e1SJohn Birrell void
16491eaf3e1SJohn Birrell dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit)
16591eaf3e1SJohn Birrell {
16691eaf3e1SJohn Birrell 	proc_t *p = curproc;
16791eaf3e1SJohn Birrell 	struct trapframe *tf;
168c6f5742fSRui Paulo 	uintptr_t pc, sp, fp;
16991eaf3e1SJohn Birrell 	volatile uint16_t *flags =
17091eaf3e1SJohn Birrell 	    (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
17191eaf3e1SJohn Birrell 	int n;
17291eaf3e1SJohn Birrell 
17391eaf3e1SJohn Birrell 	if (*flags & CPU_DTRACE_FAULT)
17491eaf3e1SJohn Birrell 		return;
17591eaf3e1SJohn Birrell 
17691eaf3e1SJohn Birrell 	if (pcstack_limit <= 0)
17791eaf3e1SJohn Birrell 		return;
17891eaf3e1SJohn Birrell 
17991eaf3e1SJohn Birrell 	/*
18091eaf3e1SJohn Birrell 	 * If there's no user context we still need to zero the stack.
18191eaf3e1SJohn Birrell 	 */
18291eaf3e1SJohn Birrell 	if (p == NULL || (tf = curthread->td_frame) == NULL)
18391eaf3e1SJohn Birrell 		goto zero;
18491eaf3e1SJohn Birrell 
18591eaf3e1SJohn Birrell 	*pcstack++ = (uint64_t)p->p_pid;
18691eaf3e1SJohn Birrell 	pcstack_limit--;
18791eaf3e1SJohn Birrell 
18891eaf3e1SJohn Birrell 	if (pcstack_limit <= 0)
18991eaf3e1SJohn Birrell 		return;
19091eaf3e1SJohn Birrell 
19191eaf3e1SJohn Birrell 	pc = tf->tf_rip;
192c6f5742fSRui Paulo 	fp = tf->tf_rbp;
19391eaf3e1SJohn Birrell 	sp = tf->tf_rsp;
19491eaf3e1SJohn Birrell 
19591eaf3e1SJohn Birrell 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
196c6f5742fSRui Paulo 		/*
197c6f5742fSRui Paulo 		 * In an entry probe.  The frame pointer has not yet been
198c6f5742fSRui Paulo 		 * pushed (that happens in the function prologue).  The
199c6f5742fSRui Paulo 		 * best approach is to add the current pc as a missing top
200c6f5742fSRui Paulo 		 * of stack and back the pc up to the caller, which is stored
201c6f5742fSRui Paulo 		 * at the current stack pointer address since the call
202c6f5742fSRui Paulo 		 * instruction puts it there right before the branch.
203c6f5742fSRui Paulo 		 */
204c6f5742fSRui Paulo 
20591eaf3e1SJohn Birrell 		*pcstack++ = (uint64_t)pc;
20691eaf3e1SJohn Birrell 		pcstack_limit--;
20791eaf3e1SJohn Birrell 		if (pcstack_limit <= 0)
20891eaf3e1SJohn Birrell 			return;
20991eaf3e1SJohn Birrell 
210c6f5742fSRui Paulo 		pc = dtrace_fuword64((void *) sp);
21191eaf3e1SJohn Birrell 	}
21291eaf3e1SJohn Birrell 
213c6f5742fSRui Paulo 	n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
21491eaf3e1SJohn Birrell 	ASSERT(n >= 0);
21591eaf3e1SJohn Birrell 	ASSERT(n <= pcstack_limit);
21691eaf3e1SJohn Birrell 
21791eaf3e1SJohn Birrell 	pcstack += n;
21891eaf3e1SJohn Birrell 	pcstack_limit -= n;
21991eaf3e1SJohn Birrell 
22091eaf3e1SJohn Birrell zero:
22191eaf3e1SJohn Birrell 	while (pcstack_limit-- > 0)
22291eaf3e1SJohn Birrell 		*pcstack++ = 0;
22391eaf3e1SJohn Birrell }
22491eaf3e1SJohn Birrell 
22591eaf3e1SJohn Birrell int
22691eaf3e1SJohn Birrell dtrace_getustackdepth(void)
22791eaf3e1SJohn Birrell {
22891eaf3e1SJohn Birrell 	proc_t *p = curproc;
22991eaf3e1SJohn Birrell 	struct trapframe *tf;
230c6f5742fSRui Paulo 	uintptr_t pc, fp, sp;
23191eaf3e1SJohn Birrell 	int n = 0;
23291eaf3e1SJohn Birrell 
23391eaf3e1SJohn Birrell 	if (p == NULL || (tf = curthread->td_frame) == NULL)
23491eaf3e1SJohn Birrell 		return (0);
23591eaf3e1SJohn Birrell 
23691eaf3e1SJohn Birrell 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
23791eaf3e1SJohn Birrell 		return (-1);
23891eaf3e1SJohn Birrell 
23991eaf3e1SJohn Birrell 	pc = tf->tf_rip;
240c6f5742fSRui Paulo 	fp = tf->tf_rbp;
24191eaf3e1SJohn Birrell 	sp = tf->tf_rsp;
24291eaf3e1SJohn Birrell 
24391eaf3e1SJohn Birrell 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
244c6f5742fSRui Paulo 		/*
245c6f5742fSRui Paulo 		 * In an entry probe.  The frame pointer has not yet been
246c6f5742fSRui Paulo 		 * pushed (that happens in the function prologue).  The
247c6f5742fSRui Paulo 		 * best approach is to add the current pc as a missing top
248c6f5742fSRui Paulo 		 * of stack and back the pc up to the caller, which is stored
249c6f5742fSRui Paulo 		 * at the current stack pointer address since the call
250c6f5742fSRui Paulo 		 * instruction puts it there right before the branch.
251c6f5742fSRui Paulo 		 */
25291eaf3e1SJohn Birrell 
253c6f5742fSRui Paulo 		pc = dtrace_fuword64((void *) sp);
254c6f5742fSRui Paulo 		n++;
25591eaf3e1SJohn Birrell 	}
25691eaf3e1SJohn Birrell 
257c6f5742fSRui Paulo 	n += dtrace_getustack_common(NULL, 0, pc, fp);
25891eaf3e1SJohn Birrell 
25991eaf3e1SJohn Birrell 	return (n);
26091eaf3e1SJohn Birrell }
26191eaf3e1SJohn Birrell 
26291eaf3e1SJohn Birrell void
26391eaf3e1SJohn Birrell dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit)
26491eaf3e1SJohn Birrell {
26591eaf3e1SJohn Birrell 	proc_t *p = curproc;
266c6f5742fSRui Paulo 	struct trapframe *tf;
267c6f5742fSRui Paulo 	uintptr_t pc, sp, fp;
26891eaf3e1SJohn Birrell 	volatile uint16_t *flags =
26991eaf3e1SJohn Birrell 	    (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
270c6f5742fSRui Paulo #ifdef notyet	/* XXX signal stack */
271c6f5742fSRui Paulo 	uintptr_t oldcontext;
27291eaf3e1SJohn Birrell 	size_t s1, s2;
273c6f5742fSRui Paulo #endif
27491eaf3e1SJohn Birrell 
27591eaf3e1SJohn Birrell 	if (*flags & CPU_DTRACE_FAULT)
27691eaf3e1SJohn Birrell 		return;
27791eaf3e1SJohn Birrell 
27891eaf3e1SJohn Birrell 	if (pcstack_limit <= 0)
27991eaf3e1SJohn Birrell 		return;
28091eaf3e1SJohn Birrell 
28191eaf3e1SJohn Birrell 	/*
28291eaf3e1SJohn Birrell 	 * If there's no user context we still need to zero the stack.
28391eaf3e1SJohn Birrell 	 */
284c6f5742fSRui Paulo 	if (p == NULL || (tf = curthread->td_frame) == NULL)
28591eaf3e1SJohn Birrell 		goto zero;
28691eaf3e1SJohn Birrell 
28791eaf3e1SJohn Birrell 	*pcstack++ = (uint64_t)p->p_pid;
28891eaf3e1SJohn Birrell 	pcstack_limit--;
28991eaf3e1SJohn Birrell 
29091eaf3e1SJohn Birrell 	if (pcstack_limit <= 0)
29191eaf3e1SJohn Birrell 		return;
29291eaf3e1SJohn Birrell 
293c6f5742fSRui Paulo 	pc = tf->tf_rip;
294c6f5742fSRui Paulo 	sp = tf->tf_rsp;
295c6f5742fSRui Paulo 	fp = tf->tf_rbp;
29691eaf3e1SJohn Birrell 
297c6f5742fSRui Paulo #ifdef notyet /* XXX signal stack */
298c6f5742fSRui Paulo 	oldcontext = lwp->lwp_oldcontext;
29991eaf3e1SJohn Birrell 	s1 = sizeof (struct xframe) + 2 * sizeof (long);
30091eaf3e1SJohn Birrell 	s2 = s1 + sizeof (siginfo_t);
301c6f5742fSRui Paulo #endif
30291eaf3e1SJohn Birrell 
30391eaf3e1SJohn Birrell 	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
30491eaf3e1SJohn Birrell 		*pcstack++ = (uint64_t)pc;
30591eaf3e1SJohn Birrell 		*fpstack++ = 0;
30691eaf3e1SJohn Birrell 		pcstack_limit--;
30791eaf3e1SJohn Birrell 		if (pcstack_limit <= 0)
30891eaf3e1SJohn Birrell 			return;
30991eaf3e1SJohn Birrell 
310c6f5742fSRui Paulo 		pc = dtrace_fuword64((void *)sp);
31191eaf3e1SJohn Birrell 	}
31291eaf3e1SJohn Birrell 
313c6f5742fSRui Paulo 	while (pc != 0) {
31491eaf3e1SJohn Birrell 		*pcstack++ = (uint64_t)pc;
315c6f5742fSRui Paulo 		*fpstack++ = fp;
31691eaf3e1SJohn Birrell 		pcstack_limit--;
31791eaf3e1SJohn Birrell 		if (pcstack_limit <= 0)
31891eaf3e1SJohn Birrell 			break;
31991eaf3e1SJohn Birrell 
320c6f5742fSRui Paulo 		if (fp == 0)
321c6f5742fSRui Paulo 			break;
322c6f5742fSRui Paulo 
323c6f5742fSRui Paulo #ifdef notyet /* XXX signal stack */
32491eaf3e1SJohn Birrell 		if (oldcontext == sp + s1 || oldcontext == sp + s2) {
32591eaf3e1SJohn Birrell 			ucontext_t *ucp = (ucontext_t *)oldcontext;
32691eaf3e1SJohn Birrell 			greg_t *gregs = ucp->uc_mcontext.gregs;
32791eaf3e1SJohn Birrell 
32891eaf3e1SJohn Birrell 			sp = dtrace_fulword(&gregs[REG_FP]);
32991eaf3e1SJohn Birrell 			pc = dtrace_fulword(&gregs[REG_PC]);
33091eaf3e1SJohn Birrell 
33191eaf3e1SJohn Birrell 			oldcontext = dtrace_fulword(&ucp->uc_link);
332c6f5742fSRui Paulo 		} else
333c6f5742fSRui Paulo #endif /* XXX */
334c6f5742fSRui Paulo 		{
335c6f5742fSRui Paulo 			pc = dtrace_fuword64((void *)(fp +
336c6f5742fSRui Paulo 				offsetof(struct amd64_frame, f_retaddr)));
337c6f5742fSRui Paulo 			fp = dtrace_fuword64((void *)fp);
33891eaf3e1SJohn Birrell 		}
33991eaf3e1SJohn Birrell 
34091eaf3e1SJohn Birrell 		/*
34191eaf3e1SJohn Birrell 		 * This is totally bogus:  if we faulted, we're going to clear
34291eaf3e1SJohn Birrell 		 * the fault and break.  This is to deal with the apparently
34391eaf3e1SJohn Birrell 		 * broken Java stacks on x86.
34491eaf3e1SJohn Birrell 		 */
34591eaf3e1SJohn Birrell 		if (*flags & CPU_DTRACE_FAULT) {
34691eaf3e1SJohn Birrell 			*flags &= ~CPU_DTRACE_FAULT;
34791eaf3e1SJohn Birrell 			break;
34891eaf3e1SJohn Birrell 		}
34991eaf3e1SJohn Birrell 	}
35091eaf3e1SJohn Birrell 
35191eaf3e1SJohn Birrell zero:
35291eaf3e1SJohn Birrell 	while (pcstack_limit-- > 0)
353c6f5742fSRui Paulo 		*pcstack++ = 0;
35491eaf3e1SJohn Birrell }
35591eaf3e1SJohn Birrell 
35691eaf3e1SJohn Birrell /*ARGSUSED*/
35791eaf3e1SJohn Birrell uint64_t
35891eaf3e1SJohn Birrell dtrace_getarg(int arg, int aframes)
35991eaf3e1SJohn Birrell {
360*9aabab09SMark Johnston 	struct thread *td;
36191eaf3e1SJohn Birrell 	uintptr_t val;
36291eaf3e1SJohn Birrell 	struct amd64_frame *fp = (struct amd64_frame *)dtrace_getfp();
36391eaf3e1SJohn Birrell 	uintptr_t *stack;
36491eaf3e1SJohn Birrell 	int i;
36591eaf3e1SJohn Birrell 
36691eaf3e1SJohn Birrell 	/*
36791eaf3e1SJohn Birrell 	 * A total of 6 arguments are passed via registers; any argument with
36891eaf3e1SJohn Birrell 	 * index of 5 or lower is therefore in a register.
36991eaf3e1SJohn Birrell 	 */
37091eaf3e1SJohn Birrell 	int inreg = 5;
37191eaf3e1SJohn Birrell 
37291eaf3e1SJohn Birrell 	/*
373*9aabab09SMark Johnston 	 * Did we arrive here via dtrace_invop()?  We can simply fetch arguments
374*9aabab09SMark Johnston 	 * from the trap frame if so.
37591eaf3e1SJohn Birrell 	 */
376*9aabab09SMark Johnston 	td = curthread;
377*9aabab09SMark Johnston 	if (td->t_dtrace_trapframe != NULL) {
378*9aabab09SMark Johnston 		struct trapframe *tf = td->t_dtrace_trapframe;
37991eaf3e1SJohn Birrell 
38091eaf3e1SJohn Birrell 		if (arg <= inreg) {
3817e75d586SMark Johnston 			switch (arg) {
3827e75d586SMark Johnston 			case 0:
383*9aabab09SMark Johnston 				return (tf->tf_rdi);
3847e75d586SMark Johnston 			case 1:
385*9aabab09SMark Johnston 				return (tf->tf_rsi);
3867e75d586SMark Johnston 			case 2:
387*9aabab09SMark Johnston 				return (tf->tf_rdx);
3887e75d586SMark Johnston 			case 3:
389*9aabab09SMark Johnston 				return (tf->tf_rcx);
3907e75d586SMark Johnston 			case 4:
391*9aabab09SMark Johnston 				return (tf->tf_r8);
3927e75d586SMark Johnston 			case 5:
393*9aabab09SMark Johnston 				return (tf->tf_r9);
3947e75d586SMark Johnston 			}
395*9aabab09SMark Johnston 		}
396*9aabab09SMark Johnston 
39791eaf3e1SJohn Birrell 		arg -= inreg;
398*9aabab09SMark Johnston 		stack = (uintptr_t *)tf->tf_rsp;
39991eaf3e1SJohn Birrell 		goto load;
40091eaf3e1SJohn Birrell 	}
40191eaf3e1SJohn Birrell 
402*9aabab09SMark Johnston 	for (i = 1; i <= aframes; i++)
403*9aabab09SMark Johnston 		fp = fp->f_frame;
40491eaf3e1SJohn Birrell 
40591eaf3e1SJohn Birrell 	/*
40691eaf3e1SJohn Birrell 	 * We know that we did not come through a trap to get into
40791eaf3e1SJohn Birrell 	 * dtrace_probe() -- the provider simply called dtrace_probe()
40891eaf3e1SJohn Birrell 	 * directly.  As this is the case, we need to shift the argument
40991eaf3e1SJohn Birrell 	 * that we're looking for:  the probe ID is the first argument to
41091eaf3e1SJohn Birrell 	 * dtrace_probe(), so the argument n will actually be found where
41191eaf3e1SJohn Birrell 	 * one would expect to find argument (n + 1).
41291eaf3e1SJohn Birrell 	 */
41391eaf3e1SJohn Birrell 	arg++;
41491eaf3e1SJohn Birrell 
41591eaf3e1SJohn Birrell 	if (arg <= inreg) {
41691eaf3e1SJohn Birrell 		/*
41791eaf3e1SJohn Birrell 		 * This shouldn't happen.  If the argument is passed in a
41891eaf3e1SJohn Birrell 		 * register then it should have been, well, passed in a
41991eaf3e1SJohn Birrell 		 * register...
42091eaf3e1SJohn Birrell 		 */
42191eaf3e1SJohn Birrell 		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
42291eaf3e1SJohn Birrell 		return (0);
42391eaf3e1SJohn Birrell 	}
42491eaf3e1SJohn Birrell 
42591eaf3e1SJohn Birrell 	arg -= (inreg + 1);
4261e954a7cSMark Johnston 	stack = (uintptr_t *)&fp[1];
42791eaf3e1SJohn Birrell 
42891eaf3e1SJohn Birrell load:
42991eaf3e1SJohn Birrell 	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
43091eaf3e1SJohn Birrell 	val = stack[arg];
43191eaf3e1SJohn Birrell 	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
43291eaf3e1SJohn Birrell 
43391eaf3e1SJohn Birrell 	return (val);
43491eaf3e1SJohn Birrell }
43591eaf3e1SJohn Birrell 
43691eaf3e1SJohn Birrell int
43791eaf3e1SJohn Birrell dtrace_getstackdepth(int aframes)
43891eaf3e1SJohn Birrell {
43991eaf3e1SJohn Birrell 	int depth = 0;
44091eaf3e1SJohn Birrell 	struct amd64_frame *frame;
44191eaf3e1SJohn Birrell 	vm_offset_t rbp;
44291eaf3e1SJohn Birrell 
44391eaf3e1SJohn Birrell 	aframes++;
44491eaf3e1SJohn Birrell 	rbp = dtrace_getfp();
44591eaf3e1SJohn Birrell 	frame = (struct amd64_frame *)rbp;
44691eaf3e1SJohn Birrell 	depth++;
44791eaf3e1SJohn Birrell 	for(;;) {
4485941edfcSJohn Baldwin 		if (!kstack_contains(curthread, (vm_offset_t)frame,
44924f10b03SKonstantin Belousov 		    sizeof(*frame)))
45091eaf3e1SJohn Birrell 			break;
45191eaf3e1SJohn Birrell 		depth++;
4525941edfcSJohn Baldwin 		if (frame->f_frame <= frame)
45391eaf3e1SJohn Birrell 			break;
45491eaf3e1SJohn Birrell 		frame = frame->f_frame;
45591eaf3e1SJohn Birrell 	}
45691eaf3e1SJohn Birrell 	if (depth < aframes)
45791eaf3e1SJohn Birrell 		return 0;
45891eaf3e1SJohn Birrell 	else
45991eaf3e1SJohn Birrell 		return depth - aframes;
46091eaf3e1SJohn Birrell }
46191eaf3e1SJohn Birrell 
46291eaf3e1SJohn Birrell ulong_t
46398ab9802SChristos Margiolis dtrace_getreg(struct trapframe *frame, uint_t reg)
46491eaf3e1SJohn Birrell {
465c6f5742fSRui Paulo 	/* This table is dependent on reg.d. */
46691eaf3e1SJohn Birrell 	int regmap[] = {
467c6f5742fSRui Paulo 		REG_GS,		/* 0  GS */
468c6f5742fSRui Paulo 		REG_FS,		/* 1  FS */
469c6f5742fSRui Paulo 		REG_ES,		/* 2  ES */
470c6f5742fSRui Paulo 		REG_DS,		/* 3  DS */
471c6f5742fSRui Paulo 		REG_RDI,	/* 4  EDI */
472c6f5742fSRui Paulo 		REG_RSI,	/* 5  ESI */
473c6f5742fSRui Paulo 		REG_RBP,	/* 6  EBP, REG_FP */
474c6f5742fSRui Paulo 		REG_RSP,	/* 7  ESP */
475c6f5742fSRui Paulo 		REG_RBX,	/* 8  EBX, REG_R1 */
476c6f5742fSRui Paulo 		REG_RDX,	/* 9  EDX */
477c6f5742fSRui Paulo 		REG_RCX,	/* 10 ECX */
478c6f5742fSRui Paulo 		REG_RAX,	/* 11 EAX, REG_R0 */
479c6f5742fSRui Paulo 		REG_TRAPNO,	/* 12 TRAPNO */
480c6f5742fSRui Paulo 		REG_ERR,	/* 13 ERR */
481c6f5742fSRui Paulo 		REG_RIP,	/* 14 EIP, REG_PC */
482c6f5742fSRui Paulo 		REG_CS,		/* 15 CS */
483c6f5742fSRui Paulo 		REG_RFL,	/* 16 EFL, REG_PS */
484c6f5742fSRui Paulo 		REG_RSP,	/* 17 UESP, REG_SP */
485c6f5742fSRui Paulo 		REG_SS		/* 18 SS */
48691eaf3e1SJohn Birrell 	};
48791eaf3e1SJohn Birrell 
4885eb65c4cSMariusz Zaborski 	if (reg <= GS) {
48991eaf3e1SJohn Birrell 		if (reg >= sizeof (regmap) / sizeof (int)) {
49091eaf3e1SJohn Birrell 			DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
49191eaf3e1SJohn Birrell 			return (0);
49291eaf3e1SJohn Birrell 		}
49391eaf3e1SJohn Birrell 
49491eaf3e1SJohn Birrell 		reg = regmap[reg];
49591eaf3e1SJohn Birrell 	} else {
496c6f5742fSRui Paulo 		/* This is dependent on reg.d. */
4978da024d9SMariusz Zaborski 		reg -= GS + 1;
49891eaf3e1SJohn Birrell 	}
49991eaf3e1SJohn Birrell 
50091eaf3e1SJohn Birrell 	switch (reg) {
50191eaf3e1SJohn Birrell 	case REG_RDI:
50298ab9802SChristos Margiolis 		return (frame->tf_rdi);
50391eaf3e1SJohn Birrell 	case REG_RSI:
50498ab9802SChristos Margiolis 		return (frame->tf_rsi);
50591eaf3e1SJohn Birrell 	case REG_RDX:
50698ab9802SChristos Margiolis 		return (frame->tf_rdx);
50791eaf3e1SJohn Birrell 	case REG_RCX:
50898ab9802SChristos Margiolis 		return (frame->tf_rcx);
50991eaf3e1SJohn Birrell 	case REG_R8:
51098ab9802SChristos Margiolis 		return (frame->tf_r8);
51191eaf3e1SJohn Birrell 	case REG_R9:
51298ab9802SChristos Margiolis 		return (frame->tf_r9);
51391eaf3e1SJohn Birrell 	case REG_RAX:
51498ab9802SChristos Margiolis 		return (frame->tf_rax);
51591eaf3e1SJohn Birrell 	case REG_RBX:
51698ab9802SChristos Margiolis 		return (frame->tf_rbx);
51791eaf3e1SJohn Birrell 	case REG_RBP:
51898ab9802SChristos Margiolis 		return (frame->tf_rbp);
51991eaf3e1SJohn Birrell 	case REG_R10:
52098ab9802SChristos Margiolis 		return (frame->tf_r10);
52191eaf3e1SJohn Birrell 	case REG_R11:
52298ab9802SChristos Margiolis 		return (frame->tf_r11);
52391eaf3e1SJohn Birrell 	case REG_R12:
52498ab9802SChristos Margiolis 		return (frame->tf_r12);
52591eaf3e1SJohn Birrell 	case REG_R13:
52698ab9802SChristos Margiolis 		return (frame->tf_r13);
52791eaf3e1SJohn Birrell 	case REG_R14:
52898ab9802SChristos Margiolis 		return (frame->tf_r14);
52991eaf3e1SJohn Birrell 	case REG_R15:
53098ab9802SChristos Margiolis 		return (frame->tf_r15);
53191eaf3e1SJohn Birrell 	case REG_DS:
53298ab9802SChristos Margiolis 		return (frame->tf_ds);
53391eaf3e1SJohn Birrell 	case REG_ES:
53498ab9802SChristos Margiolis 		return (frame->tf_es);
53591eaf3e1SJohn Birrell 	case REG_FS:
53698ab9802SChristos Margiolis 		return (frame->tf_fs);
53791eaf3e1SJohn Birrell 	case REG_GS:
53898ab9802SChristos Margiolis 		return (frame->tf_gs);
53991eaf3e1SJohn Birrell 	case REG_TRAPNO:
54098ab9802SChristos Margiolis 		return (frame->tf_trapno);
54191eaf3e1SJohn Birrell 	case REG_ERR:
54298ab9802SChristos Margiolis 		return (frame->tf_err);
54391eaf3e1SJohn Birrell 	case REG_RIP:
54498ab9802SChristos Margiolis 		return (frame->tf_rip);
54591eaf3e1SJohn Birrell 	case REG_CS:
54698ab9802SChristos Margiolis 		return (frame->tf_cs);
54791eaf3e1SJohn Birrell 	case REG_SS:
54898ab9802SChristos Margiolis 		return (frame->tf_ss);
54991eaf3e1SJohn Birrell 	case REG_RFL:
55098ab9802SChristos Margiolis 		return (frame->tf_rflags);
55191eaf3e1SJohn Birrell 	case REG_RSP:
55298ab9802SChristos Margiolis 		return (frame->tf_rsp);
55391eaf3e1SJohn Birrell 	default:
55491eaf3e1SJohn Birrell 		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
55591eaf3e1SJohn Birrell 		return (0);
55691eaf3e1SJohn Birrell 	}
55791eaf3e1SJohn Birrell }
55891eaf3e1SJohn Birrell 
55991eaf3e1SJohn Birrell static int
56091eaf3e1SJohn Birrell dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size)
56191eaf3e1SJohn Birrell {
562f340e9feSAndriy Gapon 	ASSERT(INKERNEL(kaddr) && kaddr + size >= kaddr);
56391eaf3e1SJohn Birrell 
564f340e9feSAndriy Gapon 	if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) {
56591eaf3e1SJohn Birrell 		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
56691eaf3e1SJohn Birrell 		cpu_core[curcpu].cpuc_dtrace_illval = uaddr;
56791eaf3e1SJohn Birrell 		return (0);
56891eaf3e1SJohn Birrell 	}
56991eaf3e1SJohn Birrell 
57091eaf3e1SJohn Birrell 	return (1);
57191eaf3e1SJohn Birrell }
57291eaf3e1SJohn Birrell 
57391eaf3e1SJohn Birrell void
57491eaf3e1SJohn Birrell dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size,
57591eaf3e1SJohn Birrell     volatile uint16_t *flags)
57691eaf3e1SJohn Birrell {
57791eaf3e1SJohn Birrell 	if (dtrace_copycheck(uaddr, kaddr, size))
57891eaf3e1SJohn Birrell 		dtrace_copy(uaddr, kaddr, size);
57991eaf3e1SJohn Birrell }
58091eaf3e1SJohn Birrell 
58191eaf3e1SJohn Birrell void
58291eaf3e1SJohn Birrell dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size,
58391eaf3e1SJohn Birrell     volatile uint16_t *flags)
58491eaf3e1SJohn Birrell {
58591eaf3e1SJohn Birrell 	if (dtrace_copycheck(uaddr, kaddr, size))
58691eaf3e1SJohn Birrell 		dtrace_copy(kaddr, uaddr, size);
58791eaf3e1SJohn Birrell }
58891eaf3e1SJohn Birrell 
58991eaf3e1SJohn Birrell void
59091eaf3e1SJohn Birrell dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
59191eaf3e1SJohn Birrell     volatile uint16_t *flags)
59291eaf3e1SJohn Birrell {
59391eaf3e1SJohn Birrell 	if (dtrace_copycheck(uaddr, kaddr, size))
59491eaf3e1SJohn Birrell 		dtrace_copystr(uaddr, kaddr, size, flags);
59591eaf3e1SJohn Birrell }
59691eaf3e1SJohn Birrell 
59791eaf3e1SJohn Birrell void
59891eaf3e1SJohn Birrell dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size,
59991eaf3e1SJohn Birrell     volatile uint16_t *flags)
60091eaf3e1SJohn Birrell {
60191eaf3e1SJohn Birrell 	if (dtrace_copycheck(uaddr, kaddr, size))
60291eaf3e1SJohn Birrell 		dtrace_copystr(kaddr, uaddr, size, flags);
60391eaf3e1SJohn Birrell }
60491eaf3e1SJohn Birrell 
60591eaf3e1SJohn Birrell uint8_t
60691eaf3e1SJohn Birrell dtrace_fuword8(void *uaddr)
60791eaf3e1SJohn Birrell {
608f340e9feSAndriy Gapon 	if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
60991eaf3e1SJohn Birrell 		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
61091eaf3e1SJohn Birrell 		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
61191eaf3e1SJohn Birrell 		return (0);
61291eaf3e1SJohn Birrell 	}
61391eaf3e1SJohn Birrell 	return (dtrace_fuword8_nocheck(uaddr));
61491eaf3e1SJohn Birrell }
61591eaf3e1SJohn Birrell 
61691eaf3e1SJohn Birrell uint16_t
61791eaf3e1SJohn Birrell dtrace_fuword16(void *uaddr)
61891eaf3e1SJohn Birrell {
619f340e9feSAndriy Gapon 	if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
62091eaf3e1SJohn Birrell 		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
62191eaf3e1SJohn Birrell 		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
62291eaf3e1SJohn Birrell 		return (0);
62391eaf3e1SJohn Birrell 	}
62491eaf3e1SJohn Birrell 	return (dtrace_fuword16_nocheck(uaddr));
62591eaf3e1SJohn Birrell }
62691eaf3e1SJohn Birrell 
62791eaf3e1SJohn Birrell uint32_t
62891eaf3e1SJohn Birrell dtrace_fuword32(void *uaddr)
62991eaf3e1SJohn Birrell {
630f340e9feSAndriy Gapon 	if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
63191eaf3e1SJohn Birrell 		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
63291eaf3e1SJohn Birrell 		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
63391eaf3e1SJohn Birrell 		return (0);
63491eaf3e1SJohn Birrell 	}
63591eaf3e1SJohn Birrell 	return (dtrace_fuword32_nocheck(uaddr));
63691eaf3e1SJohn Birrell }
63791eaf3e1SJohn Birrell 
63891eaf3e1SJohn Birrell uint64_t
63991eaf3e1SJohn Birrell dtrace_fuword64(void *uaddr)
64091eaf3e1SJohn Birrell {
641f340e9feSAndriy Gapon 	if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
64291eaf3e1SJohn Birrell 		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
64391eaf3e1SJohn Birrell 		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
64491eaf3e1SJohn Birrell 		return (0);
64591eaf3e1SJohn Birrell 	}
64691eaf3e1SJohn Birrell 	return (dtrace_fuword64_nocheck(uaddr));
64791eaf3e1SJohn Birrell }
6488ca79fbdSMateusz Guzik 
6498ca79fbdSMateusz Guzik /*
6508ca79fbdSMateusz Guzik  * ifunc resolvers for SMAP support
6518ca79fbdSMateusz Guzik  */
6528ca79fbdSMateusz Guzik void dtrace_copy_nosmap(uintptr_t, uintptr_t, size_t);
6538ca79fbdSMateusz Guzik void dtrace_copy_smap(uintptr_t, uintptr_t, size_t);
6547c5a46a1SKonstantin Belousov DEFINE_IFUNC(, void, dtrace_copy, (uintptr_t, uintptr_t, size_t))
6558ca79fbdSMateusz Guzik {
6568ca79fbdSMateusz Guzik 
6578ca79fbdSMateusz Guzik 	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
6588ca79fbdSMateusz Guzik 	    dtrace_copy_smap : dtrace_copy_nosmap);
6598ca79fbdSMateusz Guzik }
6608ca79fbdSMateusz Guzik 
6618ca79fbdSMateusz Guzik void dtrace_copystr_nosmap(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
6628ca79fbdSMateusz Guzik void dtrace_copystr_smap(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
6638ca79fbdSMateusz Guzik DEFINE_IFUNC(, void, dtrace_copystr, (uintptr_t, uintptr_t, size_t,
6647c5a46a1SKonstantin Belousov     volatile uint16_t *))
6658ca79fbdSMateusz Guzik {
6668ca79fbdSMateusz Guzik 
6678ca79fbdSMateusz Guzik 	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
6688ca79fbdSMateusz Guzik 	    dtrace_copystr_smap : dtrace_copystr_nosmap);
6698ca79fbdSMateusz Guzik }
6708ca79fbdSMateusz Guzik 
6718ca79fbdSMateusz Guzik uintptr_t dtrace_fulword_nosmap(void *);
6728ca79fbdSMateusz Guzik uintptr_t dtrace_fulword_smap(void *);
6737c5a46a1SKonstantin Belousov DEFINE_IFUNC(, uintptr_t, dtrace_fulword, (void *))
6748ca79fbdSMateusz Guzik {
6758ca79fbdSMateusz Guzik 
6768ca79fbdSMateusz Guzik 	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
6778ca79fbdSMateusz Guzik 	    dtrace_fulword_smap : dtrace_fulword_nosmap);
6788ca79fbdSMateusz Guzik }
6798ca79fbdSMateusz Guzik 
6808ca79fbdSMateusz Guzik uint8_t dtrace_fuword8_nocheck_nosmap(void *);
6818ca79fbdSMateusz Guzik uint8_t dtrace_fuword8_nocheck_smap(void *);
6827c5a46a1SKonstantin Belousov DEFINE_IFUNC(, uint8_t, dtrace_fuword8_nocheck, (void *))
6838ca79fbdSMateusz Guzik {
6848ca79fbdSMateusz Guzik 
6858ca79fbdSMateusz Guzik 	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
6868ca79fbdSMateusz Guzik 	    dtrace_fuword8_nocheck_smap : dtrace_fuword8_nocheck_nosmap);
6878ca79fbdSMateusz Guzik }
6888ca79fbdSMateusz Guzik 
6898ca79fbdSMateusz Guzik uint16_t dtrace_fuword16_nocheck_nosmap(void *);
6908ca79fbdSMateusz Guzik uint16_t dtrace_fuword16_nocheck_smap(void *);
6917c5a46a1SKonstantin Belousov DEFINE_IFUNC(, uint16_t, dtrace_fuword16_nocheck, (void *))
6928ca79fbdSMateusz Guzik {
6938ca79fbdSMateusz Guzik 
6948ca79fbdSMateusz Guzik 	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
6958ca79fbdSMateusz Guzik 	    dtrace_fuword16_nocheck_smap : dtrace_fuword16_nocheck_nosmap);
6968ca79fbdSMateusz Guzik }
6978ca79fbdSMateusz Guzik 
6988ca79fbdSMateusz Guzik uint32_t dtrace_fuword32_nocheck_nosmap(void *);
6998ca79fbdSMateusz Guzik uint32_t dtrace_fuword32_nocheck_smap(void *);
7007c5a46a1SKonstantin Belousov DEFINE_IFUNC(, uint32_t, dtrace_fuword32_nocheck, (void *))
7018ca79fbdSMateusz Guzik {
7028ca79fbdSMateusz Guzik 
7038ca79fbdSMateusz Guzik 	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
7048ca79fbdSMateusz Guzik 	    dtrace_fuword32_nocheck_smap : dtrace_fuword32_nocheck_nosmap);
7058ca79fbdSMateusz Guzik }
7068ca79fbdSMateusz Guzik 
7078ca79fbdSMateusz Guzik uint64_t dtrace_fuword64_nocheck_nosmap(void *);
7088ca79fbdSMateusz Guzik uint64_t dtrace_fuword64_nocheck_smap(void *);
7097c5a46a1SKonstantin Belousov DEFINE_IFUNC(, uint64_t, dtrace_fuword64_nocheck, (void *))
7108ca79fbdSMateusz Guzik {
7118ca79fbdSMateusz Guzik 
7128ca79fbdSMateusz Guzik 	return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
7138ca79fbdSMateusz Guzik 	    dtrace_fuword64_nocheck_smap : dtrace_fuword64_nocheck_nosmap);
7148ca79fbdSMateusz Guzik }
715