xref: /freebsd/sys/cddl/dev/dtrace/i386/dtrace_subr.c (revision 291624fdf61dc70790e695364293821ee89063b9)
191eaf3e1SJohn Birrell /*
291eaf3e1SJohn Birrell  * CDDL HEADER START
391eaf3e1SJohn Birrell  *
491eaf3e1SJohn Birrell  * The contents of this file are subject to the terms of the
591eaf3e1SJohn Birrell  * Common Development and Distribution License, Version 1.0 only
691eaf3e1SJohn Birrell  * (the "License").  You may not use this file except in compliance
791eaf3e1SJohn Birrell  * with the License.
891eaf3e1SJohn Birrell  *
991eaf3e1SJohn Birrell  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
1091eaf3e1SJohn Birrell  * or http://www.opensolaris.org/os/licensing.
1191eaf3e1SJohn Birrell  * See the License for the specific language governing permissions
1291eaf3e1SJohn Birrell  * and limitations under the License.
1391eaf3e1SJohn Birrell  *
1491eaf3e1SJohn Birrell  * When distributing Covered Code, include this CDDL HEADER in each
1591eaf3e1SJohn Birrell  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1691eaf3e1SJohn Birrell  * If applicable, add the following below this CDDL HEADER, with the
1791eaf3e1SJohn Birrell  * fields enclosed by brackets "[]" replaced with your own identifying
1891eaf3e1SJohn Birrell  * information: Portions Copyright [yyyy] [name of copyright owner]
1991eaf3e1SJohn Birrell  *
2091eaf3e1SJohn Birrell  * CDDL HEADER END
2191eaf3e1SJohn Birrell  *
2291eaf3e1SJohn Birrell  * $FreeBSD$
2391eaf3e1SJohn Birrell  *
2491eaf3e1SJohn Birrell  */
2591eaf3e1SJohn Birrell /*
2691eaf3e1SJohn Birrell  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
2791eaf3e1SJohn Birrell  * Use is subject to license terms.
2891eaf3e1SJohn Birrell  */
2991eaf3e1SJohn Birrell 
304737d389SGeorge V. Neville-Neil /*
314737d389SGeorge V. Neville-Neil  * Copyright (c) 2011, Joyent, Inc. All rights reserved.
324737d389SGeorge V. Neville-Neil  */
334737d389SGeorge V. Neville-Neil 
3491eaf3e1SJohn Birrell #include <sys/param.h>
3591eaf3e1SJohn Birrell #include <sys/systm.h>
3691eaf3e1SJohn Birrell #include <sys/types.h>
3771a19bdcSAttilio Rao #include <sys/cpuset.h>
3891eaf3e1SJohn Birrell #include <sys/kernel.h>
3991eaf3e1SJohn Birrell #include <sys/malloc.h>
4091eaf3e1SJohn Birrell #include <sys/kmem.h>
4191eaf3e1SJohn Birrell #include <sys/smp.h>
4291eaf3e1SJohn Birrell #include <sys/dtrace_impl.h>
4391eaf3e1SJohn Birrell #include <sys/dtrace_bsd.h>
4491eaf3e1SJohn Birrell #include <machine/clock.h>
4591eaf3e1SJohn Birrell #include <machine/frame.h>
4691eaf3e1SJohn Birrell #include <vm/pmap.h>
4791eaf3e1SJohn Birrell 
4891eaf3e1SJohn Birrell extern uintptr_t 	kernelbase;
4991eaf3e1SJohn Birrell extern uintptr_t 	dtrace_in_probe_addr;
5091eaf3e1SJohn Birrell extern int		dtrace_in_probe;
5191eaf3e1SJohn Birrell 
5257d025c3SGeorge V. Neville-Neil extern void dtrace_getnanotime(struct timespec *tsp);
5357d025c3SGeorge V. Neville-Neil 
5491eaf3e1SJohn Birrell int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t);
5591eaf3e1SJohn Birrell 
5691eaf3e1SJohn Birrell typedef struct dtrace_invop_hdlr {
5791eaf3e1SJohn Birrell 	int (*dtih_func)(uintptr_t, uintptr_t *, uintptr_t);
5891eaf3e1SJohn Birrell 	struct dtrace_invop_hdlr *dtih_next;
5991eaf3e1SJohn Birrell } dtrace_invop_hdlr_t;
6091eaf3e1SJohn Birrell 
6191eaf3e1SJohn Birrell dtrace_invop_hdlr_t *dtrace_invop_hdlr;
6291eaf3e1SJohn Birrell 
6391eaf3e1SJohn Birrell int
6491eaf3e1SJohn Birrell dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax)
6591eaf3e1SJohn Birrell {
6691eaf3e1SJohn Birrell 	dtrace_invop_hdlr_t *hdlr;
6791eaf3e1SJohn Birrell 	int rval;
6891eaf3e1SJohn Birrell 
6991eaf3e1SJohn Birrell 	for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next)
7091eaf3e1SJohn Birrell 		if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0)
7191eaf3e1SJohn Birrell 			return (rval);
7291eaf3e1SJohn Birrell 
7391eaf3e1SJohn Birrell 	return (0);
7491eaf3e1SJohn Birrell }
7591eaf3e1SJohn Birrell 
7691eaf3e1SJohn Birrell void
7791eaf3e1SJohn Birrell dtrace_invop_add(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
7891eaf3e1SJohn Birrell {
7991eaf3e1SJohn Birrell 	dtrace_invop_hdlr_t *hdlr;
8091eaf3e1SJohn Birrell 
8191eaf3e1SJohn Birrell 	hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP);
8291eaf3e1SJohn Birrell 	hdlr->dtih_func = func;
8391eaf3e1SJohn Birrell 	hdlr->dtih_next = dtrace_invop_hdlr;
8491eaf3e1SJohn Birrell 	dtrace_invop_hdlr = hdlr;
8591eaf3e1SJohn Birrell }
8691eaf3e1SJohn Birrell 
8791eaf3e1SJohn Birrell void
8891eaf3e1SJohn Birrell dtrace_invop_remove(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
8991eaf3e1SJohn Birrell {
9091eaf3e1SJohn Birrell 	dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL;
9191eaf3e1SJohn Birrell 
9291eaf3e1SJohn Birrell 	for (;;) {
9391eaf3e1SJohn Birrell 		if (hdlr == NULL)
9491eaf3e1SJohn Birrell 			panic("attempt to remove non-existent invop handler");
9591eaf3e1SJohn Birrell 
9691eaf3e1SJohn Birrell 		if (hdlr->dtih_func == func)
9791eaf3e1SJohn Birrell 			break;
9891eaf3e1SJohn Birrell 
9991eaf3e1SJohn Birrell 		prev = hdlr;
10091eaf3e1SJohn Birrell 		hdlr = hdlr->dtih_next;
10191eaf3e1SJohn Birrell 	}
10291eaf3e1SJohn Birrell 
10391eaf3e1SJohn Birrell 	if (prev == NULL) {
10491eaf3e1SJohn Birrell 		ASSERT(dtrace_invop_hdlr == hdlr);
10591eaf3e1SJohn Birrell 		dtrace_invop_hdlr = hdlr->dtih_next;
10691eaf3e1SJohn Birrell 	} else {
10791eaf3e1SJohn Birrell 		ASSERT(dtrace_invop_hdlr != hdlr);
10891eaf3e1SJohn Birrell 		prev->dtih_next = hdlr->dtih_next;
10991eaf3e1SJohn Birrell 	}
11091eaf3e1SJohn Birrell 
11191eaf3e1SJohn Birrell 	kmem_free(hdlr, 0);
11291eaf3e1SJohn Birrell }
11391eaf3e1SJohn Birrell 
11491eaf3e1SJohn Birrell void
11591eaf3e1SJohn Birrell dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
11691eaf3e1SJohn Birrell {
11791eaf3e1SJohn Birrell 	(*func)(0, kernelbase);
11891eaf3e1SJohn Birrell }
11991eaf3e1SJohn Birrell 
12091eaf3e1SJohn Birrell void
12191eaf3e1SJohn Birrell dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
12291eaf3e1SJohn Birrell {
12371a19bdcSAttilio Rao 	cpuset_t cpus;
12491eaf3e1SJohn Birrell 
12591eaf3e1SJohn Birrell 	if (cpu == DTRACE_CPUALL)
12691eaf3e1SJohn Birrell 		cpus = all_cpus;
12791eaf3e1SJohn Birrell 	else
12871a19bdcSAttilio Rao 		CPU_SETOF(cpu, &cpus);
12991eaf3e1SJohn Birrell 
130fe8c7b3dSAndriy Gapon 	smp_rendezvous_cpus(cpus, smp_no_rendevous_barrier, func,
131fe8c7b3dSAndriy Gapon 	    smp_no_rendevous_barrier, arg);
13291eaf3e1SJohn Birrell }
13391eaf3e1SJohn Birrell 
13491eaf3e1SJohn Birrell static void
13591eaf3e1SJohn Birrell dtrace_sync_func(void)
13691eaf3e1SJohn Birrell {
13791eaf3e1SJohn Birrell }
13891eaf3e1SJohn Birrell 
13991eaf3e1SJohn Birrell void
14091eaf3e1SJohn Birrell dtrace_sync(void)
14191eaf3e1SJohn Birrell {
14291eaf3e1SJohn Birrell         dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
14391eaf3e1SJohn Birrell }
14491eaf3e1SJohn Birrell 
14591eaf3e1SJohn Birrell #ifdef notyet
14691eaf3e1SJohn Birrell int (*dtrace_fasttrap_probe_ptr)(struct regs *);
14791eaf3e1SJohn Birrell int (*dtrace_pid_probe_ptr)(struct regs *);
14891eaf3e1SJohn Birrell int (*dtrace_return_probe_ptr)(struct regs *);
14991eaf3e1SJohn Birrell 
15091eaf3e1SJohn Birrell void
15191eaf3e1SJohn Birrell dtrace_user_probe(struct regs *rp, caddr_t addr, processorid_t cpuid)
15291eaf3e1SJohn Birrell {
15391eaf3e1SJohn Birrell 	krwlock_t *rwp;
15491eaf3e1SJohn Birrell 	proc_t *p = curproc;
15591eaf3e1SJohn Birrell 	extern void trap(struct regs *, caddr_t, processorid_t);
15691eaf3e1SJohn Birrell 
15791eaf3e1SJohn Birrell 	if (USERMODE(rp->r_cs) || (rp->r_ps & PS_VM)) {
15891eaf3e1SJohn Birrell 		if (curthread->t_cred != p->p_cred) {
15991eaf3e1SJohn Birrell 			cred_t *oldcred = curthread->t_cred;
16091eaf3e1SJohn Birrell 			/*
16191eaf3e1SJohn Birrell 			 * DTrace accesses t_cred in probe context.  t_cred
16291eaf3e1SJohn Birrell 			 * must always be either NULL, or point to a valid,
16391eaf3e1SJohn Birrell 			 * allocated cred structure.
16491eaf3e1SJohn Birrell 			 */
16591eaf3e1SJohn Birrell 			curthread->t_cred = crgetcred();
16691eaf3e1SJohn Birrell 			crfree(oldcred);
16791eaf3e1SJohn Birrell 		}
16891eaf3e1SJohn Birrell 	}
16991eaf3e1SJohn Birrell 
17091eaf3e1SJohn Birrell 	if (rp->r_trapno == T_DTRACE_RET) {
17191eaf3e1SJohn Birrell 		uint8_t step = curthread->t_dtrace_step;
17291eaf3e1SJohn Birrell 		uint8_t ret = curthread->t_dtrace_ret;
17391eaf3e1SJohn Birrell 		uintptr_t npc = curthread->t_dtrace_npc;
17491eaf3e1SJohn Birrell 
17591eaf3e1SJohn Birrell 		if (curthread->t_dtrace_ast) {
17691eaf3e1SJohn Birrell 			aston(curthread);
17791eaf3e1SJohn Birrell 			curthread->t_sig_check = 1;
17891eaf3e1SJohn Birrell 		}
17991eaf3e1SJohn Birrell 
18091eaf3e1SJohn Birrell 		/*
18191eaf3e1SJohn Birrell 		 * Clear all user tracing flags.
18291eaf3e1SJohn Birrell 		 */
18391eaf3e1SJohn Birrell 		curthread->t_dtrace_ft = 0;
18491eaf3e1SJohn Birrell 
18591eaf3e1SJohn Birrell 		/*
18691eaf3e1SJohn Birrell 		 * If we weren't expecting to take a return probe trap, kill
18791eaf3e1SJohn Birrell 		 * the process as though it had just executed an unassigned
18891eaf3e1SJohn Birrell 		 * trap instruction.
18991eaf3e1SJohn Birrell 		 */
19091eaf3e1SJohn Birrell 		if (step == 0) {
19191eaf3e1SJohn Birrell 			tsignal(curthread, SIGILL);
19291eaf3e1SJohn Birrell 			return;
19391eaf3e1SJohn Birrell 		}
19491eaf3e1SJohn Birrell 
19591eaf3e1SJohn Birrell 		/*
19691eaf3e1SJohn Birrell 		 * If we hit this trap unrelated to a return probe, we're
19791eaf3e1SJohn Birrell 		 * just here to reset the AST flag since we deferred a signal
19891eaf3e1SJohn Birrell 		 * until after we logically single-stepped the instruction we
19991eaf3e1SJohn Birrell 		 * copied out.
20091eaf3e1SJohn Birrell 		 */
20191eaf3e1SJohn Birrell 		if (ret == 0) {
20291eaf3e1SJohn Birrell 			rp->r_pc = npc;
20391eaf3e1SJohn Birrell 			return;
20491eaf3e1SJohn Birrell 		}
20591eaf3e1SJohn Birrell 
20691eaf3e1SJohn Birrell 		/*
20791eaf3e1SJohn Birrell 		 * We need to wait until after we've called the
20891eaf3e1SJohn Birrell 		 * dtrace_return_probe_ptr function pointer to set %pc.
20991eaf3e1SJohn Birrell 		 */
21091eaf3e1SJohn Birrell 		rwp = &CPU->cpu_ft_lock;
21191eaf3e1SJohn Birrell 		rw_enter(rwp, RW_READER);
21291eaf3e1SJohn Birrell 		if (dtrace_return_probe_ptr != NULL)
21391eaf3e1SJohn Birrell 			(void) (*dtrace_return_probe_ptr)(rp);
21491eaf3e1SJohn Birrell 		rw_exit(rwp);
21591eaf3e1SJohn Birrell 		rp->r_pc = npc;
21691eaf3e1SJohn Birrell 
21791eaf3e1SJohn Birrell 	} else if (rp->r_trapno == T_DTRACE_PROBE) {
21891eaf3e1SJohn Birrell 		rwp = &CPU->cpu_ft_lock;
21991eaf3e1SJohn Birrell 		rw_enter(rwp, RW_READER);
22091eaf3e1SJohn Birrell 		if (dtrace_fasttrap_probe_ptr != NULL)
22191eaf3e1SJohn Birrell 			(void) (*dtrace_fasttrap_probe_ptr)(rp);
22291eaf3e1SJohn Birrell 		rw_exit(rwp);
22391eaf3e1SJohn Birrell 
22491eaf3e1SJohn Birrell 	} else if (rp->r_trapno == T_BPTFLT) {
22591eaf3e1SJohn Birrell 		uint8_t instr;
22691eaf3e1SJohn Birrell 		rwp = &CPU->cpu_ft_lock;
22791eaf3e1SJohn Birrell 
22891eaf3e1SJohn Birrell 		/*
22991eaf3e1SJohn Birrell 		 * The DTrace fasttrap provider uses the breakpoint trap
23091eaf3e1SJohn Birrell 		 * (int 3). We let DTrace take the first crack at handling
23191eaf3e1SJohn Birrell 		 * this trap; if it's not a probe that DTrace knowns about,
23291eaf3e1SJohn Birrell 		 * we call into the trap() routine to handle it like a
23391eaf3e1SJohn Birrell 		 * breakpoint placed by a conventional debugger.
23491eaf3e1SJohn Birrell 		 */
23591eaf3e1SJohn Birrell 		rw_enter(rwp, RW_READER);
23691eaf3e1SJohn Birrell 		if (dtrace_pid_probe_ptr != NULL &&
23791eaf3e1SJohn Birrell 		    (*dtrace_pid_probe_ptr)(rp) == 0) {
23891eaf3e1SJohn Birrell 			rw_exit(rwp);
23991eaf3e1SJohn Birrell 			return;
24091eaf3e1SJohn Birrell 		}
24191eaf3e1SJohn Birrell 		rw_exit(rwp);
24291eaf3e1SJohn Birrell 
24391eaf3e1SJohn Birrell 		/*
24491eaf3e1SJohn Birrell 		 * If the instruction that caused the breakpoint trap doesn't
24591eaf3e1SJohn Birrell 		 * look like an int 3 anymore, it may be that this tracepoint
24691eaf3e1SJohn Birrell 		 * was removed just after the user thread executed it. In
24791eaf3e1SJohn Birrell 		 * that case, return to user land to retry the instuction.
24891eaf3e1SJohn Birrell 		 */
24991eaf3e1SJohn Birrell 		if (fuword8((void *)(rp->r_pc - 1), &instr) == 0 &&
25091eaf3e1SJohn Birrell 		    instr != FASTTRAP_INSTR) {
25191eaf3e1SJohn Birrell 			rp->r_pc--;
25291eaf3e1SJohn Birrell 			return;
25391eaf3e1SJohn Birrell 		}
25491eaf3e1SJohn Birrell 
25591eaf3e1SJohn Birrell 		trap(rp, addr, cpuid);
25691eaf3e1SJohn Birrell 
25791eaf3e1SJohn Birrell 	} else {
25891eaf3e1SJohn Birrell 		trap(rp, addr, cpuid);
25991eaf3e1SJohn Birrell 	}
26091eaf3e1SJohn Birrell }
26191eaf3e1SJohn Birrell 
26291eaf3e1SJohn Birrell void
26391eaf3e1SJohn Birrell dtrace_safe_synchronous_signal(void)
26491eaf3e1SJohn Birrell {
26591eaf3e1SJohn Birrell 	kthread_t *t = curthread;
26691eaf3e1SJohn Birrell 	struct regs *rp = lwptoregs(ttolwp(t));
26791eaf3e1SJohn Birrell 	size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
26891eaf3e1SJohn Birrell 
26991eaf3e1SJohn Birrell 	ASSERT(t->t_dtrace_on);
27091eaf3e1SJohn Birrell 
27191eaf3e1SJohn Birrell 	/*
27291eaf3e1SJohn Birrell 	 * If we're not in the range of scratch addresses, we're not actually
27391eaf3e1SJohn Birrell 	 * tracing user instructions so turn off the flags. If the instruction
27491eaf3e1SJohn Birrell 	 * we copied out caused a synchonous trap, reset the pc back to its
27591eaf3e1SJohn Birrell 	 * original value and turn off the flags.
27691eaf3e1SJohn Birrell 	 */
27791eaf3e1SJohn Birrell 	if (rp->r_pc < t->t_dtrace_scrpc ||
27891eaf3e1SJohn Birrell 	    rp->r_pc > t->t_dtrace_astpc + isz) {
27991eaf3e1SJohn Birrell 		t->t_dtrace_ft = 0;
28091eaf3e1SJohn Birrell 	} else if (rp->r_pc == t->t_dtrace_scrpc ||
28191eaf3e1SJohn Birrell 	    rp->r_pc == t->t_dtrace_astpc) {
28291eaf3e1SJohn Birrell 		rp->r_pc = t->t_dtrace_pc;
28391eaf3e1SJohn Birrell 		t->t_dtrace_ft = 0;
28491eaf3e1SJohn Birrell 	}
28591eaf3e1SJohn Birrell }
28691eaf3e1SJohn Birrell 
28791eaf3e1SJohn Birrell int
28891eaf3e1SJohn Birrell dtrace_safe_defer_signal(void)
28991eaf3e1SJohn Birrell {
29091eaf3e1SJohn Birrell 	kthread_t *t = curthread;
29191eaf3e1SJohn Birrell 	struct regs *rp = lwptoregs(ttolwp(t));
29291eaf3e1SJohn Birrell 	size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
29391eaf3e1SJohn Birrell 
29491eaf3e1SJohn Birrell 	ASSERT(t->t_dtrace_on);
29591eaf3e1SJohn Birrell 
29691eaf3e1SJohn Birrell 	/*
29791eaf3e1SJohn Birrell 	 * If we're not in the range of scratch addresses, we're not actually
29891eaf3e1SJohn Birrell 	 * tracing user instructions so turn off the flags.
29991eaf3e1SJohn Birrell 	 */
30091eaf3e1SJohn Birrell 	if (rp->r_pc < t->t_dtrace_scrpc ||
30191eaf3e1SJohn Birrell 	    rp->r_pc > t->t_dtrace_astpc + isz) {
30291eaf3e1SJohn Birrell 		t->t_dtrace_ft = 0;
30391eaf3e1SJohn Birrell 		return (0);
30491eaf3e1SJohn Birrell 	}
30591eaf3e1SJohn Birrell 
30691eaf3e1SJohn Birrell 	/*
3074737d389SGeorge V. Neville-Neil 	 * If we have executed the original instruction, but we have performed
3084737d389SGeorge V. Neville-Neil 	 * neither the jmp back to t->t_dtrace_npc nor the clean up of any
3094737d389SGeorge V. Neville-Neil 	 * registers used to emulate %rip-relative instructions in 64-bit mode,
3104737d389SGeorge V. Neville-Neil 	 * we'll save ourselves some effort by doing that here and taking the
3114737d389SGeorge V. Neville-Neil 	 * signal right away.  We detect this condition by seeing if the program
3124737d389SGeorge V. Neville-Neil 	 * counter is the range [scrpc + isz, astpc).
31391eaf3e1SJohn Birrell 	 */
3144737d389SGeorge V. Neville-Neil 	if (rp->r_pc >= t->t_dtrace_scrpc + isz &&
3154737d389SGeorge V. Neville-Neil 	    rp->r_pc < t->t_dtrace_astpc) {
31691eaf3e1SJohn Birrell #ifdef __amd64
31791eaf3e1SJohn Birrell 		/*
31891eaf3e1SJohn Birrell 		 * If there is a scratch register and we're on the
31991eaf3e1SJohn Birrell 		 * instruction immediately after the modified instruction,
32091eaf3e1SJohn Birrell 		 * restore the value of that scratch register.
32191eaf3e1SJohn Birrell 		 */
32291eaf3e1SJohn Birrell 		if (t->t_dtrace_reg != 0 &&
32391eaf3e1SJohn Birrell 		    rp->r_pc == t->t_dtrace_scrpc + isz) {
32491eaf3e1SJohn Birrell 			switch (t->t_dtrace_reg) {
32591eaf3e1SJohn Birrell 			case REG_RAX:
32691eaf3e1SJohn Birrell 				rp->r_rax = t->t_dtrace_regv;
32791eaf3e1SJohn Birrell 				break;
32891eaf3e1SJohn Birrell 			case REG_RCX:
32991eaf3e1SJohn Birrell 				rp->r_rcx = t->t_dtrace_regv;
33091eaf3e1SJohn Birrell 				break;
33191eaf3e1SJohn Birrell 			case REG_R8:
33291eaf3e1SJohn Birrell 				rp->r_r8 = t->t_dtrace_regv;
33391eaf3e1SJohn Birrell 				break;
33491eaf3e1SJohn Birrell 			case REG_R9:
33591eaf3e1SJohn Birrell 				rp->r_r9 = t->t_dtrace_regv;
33691eaf3e1SJohn Birrell 				break;
33791eaf3e1SJohn Birrell 			}
33891eaf3e1SJohn Birrell 		}
33991eaf3e1SJohn Birrell #endif
34091eaf3e1SJohn Birrell 		rp->r_pc = t->t_dtrace_npc;
34191eaf3e1SJohn Birrell 		t->t_dtrace_ft = 0;
34291eaf3e1SJohn Birrell 		return (0);
34391eaf3e1SJohn Birrell 	}
34491eaf3e1SJohn Birrell 
34591eaf3e1SJohn Birrell 	/*
34691eaf3e1SJohn Birrell 	 * Otherwise, make sure we'll return to the kernel after executing
34791eaf3e1SJohn Birrell 	 * the copied out instruction and defer the signal.
34891eaf3e1SJohn Birrell 	 */
34991eaf3e1SJohn Birrell 	if (!t->t_dtrace_step) {
35091eaf3e1SJohn Birrell 		ASSERT(rp->r_pc < t->t_dtrace_astpc);
35191eaf3e1SJohn Birrell 		rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc;
35291eaf3e1SJohn Birrell 		t->t_dtrace_step = 1;
35391eaf3e1SJohn Birrell 	}
35491eaf3e1SJohn Birrell 
35591eaf3e1SJohn Birrell 	t->t_dtrace_ast = 1;
35691eaf3e1SJohn Birrell 
35791eaf3e1SJohn Birrell 	return (1);
35891eaf3e1SJohn Birrell }
35991eaf3e1SJohn Birrell #endif
36091eaf3e1SJohn Birrell 
36191eaf3e1SJohn Birrell static int64_t	tgt_cpu_tsc;
36291eaf3e1SJohn Birrell static int64_t	hst_cpu_tsc;
36391eaf3e1SJohn Birrell static int64_t	tsc_skew[MAXCPU];
364b064b6d1SAndriy Gapon static uint64_t	nsec_scale;
365b064b6d1SAndriy Gapon 
366b064b6d1SAndriy Gapon /* See below for the explanation of this macro. */
367b064b6d1SAndriy Gapon #define SCALE_SHIFT	28
36891eaf3e1SJohn Birrell 
36991eaf3e1SJohn Birrell static void
37091eaf3e1SJohn Birrell dtrace_gethrtime_init_cpu(void *arg)
37191eaf3e1SJohn Birrell {
37291eaf3e1SJohn Birrell 	uintptr_t cpu = (uintptr_t) arg;
37391eaf3e1SJohn Birrell 
37491eaf3e1SJohn Birrell 	if (cpu == curcpu)
37591eaf3e1SJohn Birrell 		tgt_cpu_tsc = rdtsc();
37691eaf3e1SJohn Birrell 	else
37791eaf3e1SJohn Birrell 		hst_cpu_tsc = rdtsc();
37891eaf3e1SJohn Birrell }
37991eaf3e1SJohn Birrell 
38091eaf3e1SJohn Birrell static void
38191eaf3e1SJohn Birrell dtrace_gethrtime_init(void *arg)
38291eaf3e1SJohn Birrell {
38371a19bdcSAttilio Rao 	cpuset_t map;
3847becfa95SAndriy Gapon 	struct pcpu *pc;
385b064b6d1SAndriy Gapon 	uint64_t tsc_f;
38691eaf3e1SJohn Birrell 	int i;
38791eaf3e1SJohn Birrell 
388b064b6d1SAndriy Gapon 	/*
389b064b6d1SAndriy Gapon 	 * Get TSC frequency known at this moment.
390b064b6d1SAndriy Gapon 	 * This should be constant if TSC is invariant.
391b064b6d1SAndriy Gapon 	 * Otherwise tick->time conversion will be inaccurate, but
392b064b6d1SAndriy Gapon 	 * will preserve monotonic property of TSC.
393b064b6d1SAndriy Gapon 	 */
3943453537fSJung-uk Kim 	tsc_f = atomic_load_acq_64(&tsc_freq);
395b064b6d1SAndriy Gapon 
396b064b6d1SAndriy Gapon 	/*
397b064b6d1SAndriy Gapon 	 * The following line checks that nsec_scale calculated below
398b064b6d1SAndriy Gapon 	 * doesn't overflow 32-bit unsigned integer, so that it can multiply
399b064b6d1SAndriy Gapon 	 * another 32-bit integer without overflowing 64-bit.
400b064b6d1SAndriy Gapon 	 * Thus minimum supported TSC frequency is 62.5MHz.
401b064b6d1SAndriy Gapon 	 */
402b064b6d1SAndriy Gapon 	KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("TSC frequency is too low"));
403b064b6d1SAndriy Gapon 
404b064b6d1SAndriy Gapon 	/*
405b064b6d1SAndriy Gapon 	 * We scale up NANOSEC/tsc_f ratio to preserve as much precision
406b064b6d1SAndriy Gapon 	 * as possible.
407b064b6d1SAndriy Gapon 	 * 2^28 factor was chosen quite arbitrarily from practical
408b064b6d1SAndriy Gapon 	 * considerations:
409b064b6d1SAndriy Gapon 	 * - it supports TSC frequencies as low as 62.5MHz (see above);
410b064b6d1SAndriy Gapon 	 * - it provides quite good precision (e < 0.01%) up to THz
411b064b6d1SAndriy Gapon 	 *   (terahertz) values;
412b064b6d1SAndriy Gapon 	 */
413b064b6d1SAndriy Gapon 	nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f;
414b064b6d1SAndriy Gapon 
41591eaf3e1SJohn Birrell 	/* The current CPU is the reference one. */
4167becfa95SAndriy Gapon 	sched_pin();
41791eaf3e1SJohn Birrell 	tsc_skew[curcpu] = 0;
4183aa6d94eSJohn Baldwin 	CPU_FOREACH(i) {
41991eaf3e1SJohn Birrell 		if (i == curcpu)
42091eaf3e1SJohn Birrell 			continue;
42191eaf3e1SJohn Birrell 
4227becfa95SAndriy Gapon 		pc = pcpu_find(i);
423ada5b739SAttilio Rao 		CPU_SETOF(PCPU_GET(cpuid), &map);
424ada5b739SAttilio Rao 		CPU_SET(pc->pc_cpuid, &map);
42591eaf3e1SJohn Birrell 
426d9b8935fSAndriy Gapon 		smp_rendezvous_cpus(map, NULL,
42791eaf3e1SJohn Birrell 		    dtrace_gethrtime_init_cpu,
42891eaf3e1SJohn Birrell 		    smp_no_rendevous_barrier, (void *)(uintptr_t) i);
42991eaf3e1SJohn Birrell 
43091eaf3e1SJohn Birrell 		tsc_skew[i] = tgt_cpu_tsc - hst_cpu_tsc;
43191eaf3e1SJohn Birrell 	}
4327becfa95SAndriy Gapon 	sched_unpin();
43391eaf3e1SJohn Birrell }
43491eaf3e1SJohn Birrell 
43591eaf3e1SJohn Birrell SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL);
43691eaf3e1SJohn Birrell 
43791eaf3e1SJohn Birrell /*
43891eaf3e1SJohn Birrell  * DTrace needs a high resolution time function which can
43991eaf3e1SJohn Birrell  * be called from a probe context and guaranteed not to have
44091eaf3e1SJohn Birrell  * instrumented with probes itself.
44191eaf3e1SJohn Birrell  *
44291eaf3e1SJohn Birrell  * Returns nanoseconds since boot.
44391eaf3e1SJohn Birrell  */
44491eaf3e1SJohn Birrell uint64_t
44591eaf3e1SJohn Birrell dtrace_gethrtime()
44691eaf3e1SJohn Birrell {
447b064b6d1SAndriy Gapon 	uint64_t tsc;
448b064b6d1SAndriy Gapon 	uint32_t lo;
449b064b6d1SAndriy Gapon 	uint32_t hi;
450b064b6d1SAndriy Gapon 
451b064b6d1SAndriy Gapon 	/*
452b064b6d1SAndriy Gapon 	 * We split TSC value into lower and higher 32-bit halves and separately
453b064b6d1SAndriy Gapon 	 * scale them with nsec_scale, then we scale them down by 2^28
454b064b6d1SAndriy Gapon 	 * (see nsec_scale calculations) taking into account 32-bit shift of
455b064b6d1SAndriy Gapon 	 * the higher half and finally add.
456b064b6d1SAndriy Gapon 	 */
457db5c7d36SZachary Loafman 	tsc = rdtsc() - tsc_skew[curcpu];
458b064b6d1SAndriy Gapon 	lo = tsc;
459b064b6d1SAndriy Gapon 	hi = tsc >> 32;
460b064b6d1SAndriy Gapon 	return (((lo * nsec_scale) >> SCALE_SHIFT) +
461b064b6d1SAndriy Gapon 	    ((hi * nsec_scale) << (32 - SCALE_SHIFT)));
46291eaf3e1SJohn Birrell }
46391eaf3e1SJohn Birrell 
46491eaf3e1SJohn Birrell uint64_t
46591eaf3e1SJohn Birrell dtrace_gethrestime(void)
46691eaf3e1SJohn Birrell {
46757d025c3SGeorge V. Neville-Neil 	struct timespec current_time;
46857d025c3SGeorge V. Neville-Neil 
46957d025c3SGeorge V. Neville-Neil 	dtrace_getnanotime(&current_time);
47057d025c3SGeorge V. Neville-Neil 
471d638e8dcSGeorge V. Neville-Neil 	return (current_time.tv_sec * 1000000000ULL + current_time.tv_nsec);
47291eaf3e1SJohn Birrell }
47391eaf3e1SJohn Birrell 
47491eaf3e1SJohn Birrell /* Function to handle DTrace traps during probes. See i386/i386/trap.c */
47591eaf3e1SJohn Birrell int
476*291624fdSMark Johnston dtrace_trap(struct trapframe *frame)
47791eaf3e1SJohn Birrell {
47891eaf3e1SJohn Birrell 	/*
47991eaf3e1SJohn Birrell 	 * A trap can occur while DTrace executes a probe. Before
48091eaf3e1SJohn Birrell 	 * executing the probe, DTrace blocks re-scheduling and sets
481*291624fdSMark Johnston 	 * a flag in its per-cpu flags to indicate that it doesn't
4826bccea7cSRebecca Cran 	 * want to fault. On returning from the probe, the no-fault
48391eaf3e1SJohn Birrell 	 * flag is cleared and finally re-scheduling is enabled.
48491eaf3e1SJohn Birrell 	 *
48591eaf3e1SJohn Birrell 	 * Check if DTrace has enabled 'no-fault' mode:
48691eaf3e1SJohn Birrell 	 */
48791eaf3e1SJohn Birrell 	if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) {
48891eaf3e1SJohn Birrell 		/*
48991eaf3e1SJohn Birrell 		 * There are only a couple of trap types that are expected.
49091eaf3e1SJohn Birrell 		 * All the rest will be handled in the usual way.
49191eaf3e1SJohn Birrell 		 */
492*291624fdSMark Johnston 		switch (frame->tf_trapno) {
49391eaf3e1SJohn Birrell 		/* General protection fault. */
49491eaf3e1SJohn Birrell 		case T_PROTFLT:
49591eaf3e1SJohn Birrell 			/* Flag an illegal operation. */
49691eaf3e1SJohn Birrell 			cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
49791eaf3e1SJohn Birrell 
49891eaf3e1SJohn Birrell 			/*
49991eaf3e1SJohn Birrell 			 * Offset the instruction pointer to the instruction
50091eaf3e1SJohn Birrell 			 * following the one causing the fault.
50191eaf3e1SJohn Birrell 			 */
50291eaf3e1SJohn Birrell 			frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip);
50391eaf3e1SJohn Birrell 			return (1);
50491eaf3e1SJohn Birrell 		/* Page fault. */
50591eaf3e1SJohn Birrell 		case T_PAGEFLT:
50691eaf3e1SJohn Birrell 			/* Flag a bad address. */
50791eaf3e1SJohn Birrell 			cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
50891eaf3e1SJohn Birrell 			cpu_core[curcpu].cpuc_dtrace_illval = rcr2();
50991eaf3e1SJohn Birrell 
51091eaf3e1SJohn Birrell 			/*
51191eaf3e1SJohn Birrell 			 * Offset the instruction pointer to the instruction
51291eaf3e1SJohn Birrell 			 * following the one causing the fault.
51391eaf3e1SJohn Birrell 			 */
51491eaf3e1SJohn Birrell 			frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip);
51591eaf3e1SJohn Birrell 			return (1);
51691eaf3e1SJohn Birrell 		default:
51791eaf3e1SJohn Birrell 			/* Handle all other traps in the usual way. */
51891eaf3e1SJohn Birrell 			break;
51991eaf3e1SJohn Birrell 		}
52091eaf3e1SJohn Birrell 	}
52191eaf3e1SJohn Birrell 
52291eaf3e1SJohn Birrell 	/* Handle the trap in the usual way. */
52391eaf3e1SJohn Birrell 	return (0);
52491eaf3e1SJohn Birrell }
525