1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2008 John Birrell <jb@freebsd.org> 22 * 23 * $FreeBSD$ 24 * 25 */ 26/* 27 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#define _ASM 32 33#include <machine/asmacros.h> 34#include <sys/cpuvar_defs.h> 35#include <sys/dtrace.h> 36 37#include "assym.s" 38 39#define INTR_POP \ 40 MEXITCOUNT; \ 41 movq TF_RDI(%rsp),%rdi; \ 42 movq TF_RSI(%rsp),%rsi; \ 43 movq TF_RDX(%rsp),%rdx; \ 44 movq TF_RCX(%rsp),%rcx; \ 45 movq TF_R8(%rsp),%r8; \ 46 movq TF_R9(%rsp),%r9; \ 47 movq TF_RAX(%rsp),%rax; \ 48 movq TF_RBX(%rsp),%rbx; \ 49 movq TF_RBP(%rsp),%rbp; \ 50 movq TF_R10(%rsp),%r10; \ 51 movq TF_R11(%rsp),%r11; \ 52 movq TF_R12(%rsp),%r12; \ 53 movq TF_R13(%rsp),%r13; \ 54 movq TF_R14(%rsp),%r14; \ 55 movq TF_R15(%rsp),%r15; \ 56 testb $SEL_RPL_MASK,TF_CS(%rsp); \ 57 jz 1f; \ 58 cli; \ 59 swapgs; \ 601: addq $TF_RIP,%rsp; 61 62 63 ENTRY(dtrace_invop_start) 64 65 /* 66 * #BP traps with %rip set to the next address. We need to decrement 67 * the value to indicate the address of the int3 (0xcc) instruction 68 * that we substituted. 69 */ 70 movq TF_RIP(%rsp), %rdi 71 decq %rdi 72 movq TF_RSP(%rsp), %rsi 73 movq TF_RAX(%rsp), %rdx 74 pushq (%rsi) 75 movq %rsp, %rsi 76 call dtrace_invop 77 ALTENTRY(dtrace_invop_callsite) 78 addq $8, %rsp 79 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 80 je bp_push 81 cmpl $DTRACE_INVOP_LEAVE, %eax 82 je bp_leave 83 cmpl $DTRACE_INVOP_NOP, %eax 84 je bp_nop 85 cmpl $DTRACE_INVOP_RET, %eax 86 je bp_ret 87 88 /* When all else fails handle the trap in the usual way. */ 89 jmpq *dtrace_invop_calltrap_addr 90 91bp_push: 92 /* 93 * We must emulate a "pushq %rbp". To do this, we pull the stack 94 * down 8 bytes, and then store the base pointer. 95 */ 96 INTR_POP 97 subq $16, %rsp /* make room for %rbp */ 98 pushq %rax /* push temp */ 99 movq 24(%rsp), %rax /* load calling RIP */ 100 movq %rax, 8(%rsp) /* store calling RIP */ 101 movq 32(%rsp), %rax /* load calling CS */ 102 movq %rax, 16(%rsp) /* store calling CS */ 103 movq 40(%rsp), %rax /* load calling RFLAGS */ 104 movq %rax, 24(%rsp) /* store calling RFLAGS */ 105 movq 48(%rsp), %rax /* load calling RSP */ 106 subq $8, %rax /* make room for %rbp */ 107 movq %rax, 32(%rsp) /* store calling RSP */ 108 movq 56(%rsp), %rax /* load calling SS */ 109 movq %rax, 40(%rsp) /* store calling SS */ 110 movq 32(%rsp), %rax /* reload calling RSP */ 111 movq %rbp, (%rax) /* store %rbp there */ 112 popq %rax /* pop off temp */ 113 iretq /* return from interrupt */ 114 /*NOTREACHED*/ 115 116bp_leave: 117 /* 118 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 119 * followed by a "popq %rbp". This is quite a bit simpler on amd64 120 * than it is on i386 -- we can exploit the fact that the %rsp is 121 * explicitly saved to effect the pop without having to reshuffle 122 * the other data pushed for the trap. 123 */ 124 INTR_POP 125 pushq %rax /* push temp */ 126 movq 8(%rsp), %rax /* load calling RIP */ 127 movq %rax, 8(%rsp) /* store calling RIP */ 128 movq (%rbp), %rax /* get new %rbp */ 129 addq $8, %rbp /* adjust new %rsp */ 130 movq %rbp, 32(%rsp) /* store new %rsp */ 131 movq %rax, %rbp /* set new %rbp */ 132 popq %rax /* pop off temp */ 133 iretq /* return from interrupt */ 134 /*NOTREACHED*/ 135 136bp_nop: 137 /* We must emulate a "nop". */ 138 INTR_POP 139 iretq 140 /*NOTREACHED*/ 141 142bp_ret: 143 INTR_POP 144 pushq %rax /* push temp */ 145 movq 32(%rsp), %rax /* load %rsp */ 146 movq (%rax), %rax /* load calling RIP */ 147 movq %rax, 8(%rsp) /* store calling RIP */ 148 addq $8, 32(%rsp) /* adjust new %rsp */ 149 popq %rax /* pop off temp */ 150 iretq /* return from interrupt */ 151 /*NOTREACHED*/ 152 153 END(dtrace_invop_start) 154 155/* 156void dtrace_invop_init(void) 157*/ 158 ENTRY(dtrace_invop_init) 159 movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) 160 ret 161 END(dtrace_invop_init) 162 163/* 164void dtrace_invop_uninit(void) 165*/ 166 ENTRY(dtrace_invop_uninit) 167 movq $0, dtrace_invop_jump_addr(%rip) 168 ret 169 END(dtrace_invop_uninit) 170 171/* 172greg_t dtrace_getfp(void) 173*/ 174 ENTRY(dtrace_getfp) 175 movq %rbp, %rax 176 ret 177 END(dtrace_getfp) 178 179/* 180uint32_t 181dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) 182*/ 183 ENTRY(dtrace_cas32) 184 movl %esi, %eax 185 lock 186 cmpxchgl %edx, (%rdi) 187 ret 188 END(dtrace_cas32) 189 190/* 191void * 192dtrace_casptr(void *target, void *cmp, void *new) 193*/ 194 ENTRY(dtrace_casptr) 195 movq %rsi, %rax 196 lock 197 cmpxchgq %rdx, (%rdi) 198 ret 199 END(dtrace_casptr) 200 201/* 202uintptr_t 203dtrace_caller(int aframes) 204*/ 205 ENTRY(dtrace_caller) 206 movq $-1, %rax 207 ret 208 END(dtrace_caller) 209 210/* 211void 212dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) 213*/ 214 ENTRY(dtrace_copy) 215 pushq %rbp 216 movq %rsp, %rbp 217 218 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 219 movq %rdx, %rcx /* load count */ 220 repz /* repeat for count ... */ 221 smovb /* move from %ds:rsi to %ed:rdi */ 222 leave 223 ret 224 END(dtrace_copy) 225 226/* 227void 228dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, 229 volatile uint16_t *flags) 230*/ 231 ENTRY(dtrace_copystr) 232 pushq %rbp 233 movq %rsp, %rbp 234 2350: 236 movb (%rdi), %al /* load from source */ 237 movb %al, (%rsi) /* store to destination */ 238 addq $1, %rdi /* increment source pointer */ 239 addq $1, %rsi /* increment destination pointer */ 240 subq $1, %rdx /* decrement remaining count */ 241 cmpb $0, %al 242 je 2f 243 testq $0xfff, %rdx /* test if count is 4k-aligned */ 244 jnz 1f /* if not, continue with copying */ 245 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 246 jnz 2f 2471: 248 cmpq $0, %rdx 249 jne 0b 2502: 251 leave 252 ret 253 254 END(dtrace_copystr) 255 256/* 257uintptr_t 258dtrace_fulword(void *addr) 259*/ 260 ENTRY(dtrace_fulword) 261 movq (%rdi), %rax 262 ret 263 END(dtrace_fulword) 264 265/* 266uint8_t 267dtrace_fuword8_nocheck(void *addr) 268*/ 269 ENTRY(dtrace_fuword8_nocheck) 270 xorq %rax, %rax 271 movb (%rdi), %al 272 ret 273 END(dtrace_fuword8_nocheck) 274 275/* 276uint16_t 277dtrace_fuword16_nocheck(void *addr) 278*/ 279 ENTRY(dtrace_fuword16_nocheck) 280 xorq %rax, %rax 281 movw (%rdi), %ax 282 ret 283 END(dtrace_fuword16_nocheck) 284 285/* 286uint32_t 287dtrace_fuword32_nocheck(void *addr) 288*/ 289 ENTRY(dtrace_fuword32_nocheck) 290 xorq %rax, %rax 291 movl (%rdi), %eax 292 ret 293 END(dtrace_fuword32_nocheck) 294 295/* 296uint64_t 297dtrace_fuword64_nocheck(void *addr) 298*/ 299 ENTRY(dtrace_fuword64_nocheck) 300 movq (%rdi), %rax 301 ret 302 END(dtrace_fuword64_nocheck) 303 304/* 305void 306dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, 307 int fault, int fltoffs, uintptr_t illval) 308*/ 309 ENTRY(dtrace_probe_error) 310 pushq %rbp 311 movq %rsp, %rbp 312 subq $0x8, %rsp 313 movq %r9, (%rsp) 314 movq %r8, %r9 315 movq %rcx, %r8 316 movq %rdx, %rcx 317 movq %rsi, %rdx 318 movq %rdi, %rsi 319 movl dtrace_probeid_error(%rip), %edi 320 call dtrace_probe 321 addq $0x8, %rsp 322 leave 323 ret 324 END(dtrace_probe_error) 325 326/* 327void 328dtrace_membar_producer(void) 329*/ 330 ENTRY(dtrace_membar_producer) 331 rep; ret /* use 2 byte return instruction when branch target */ 332 /* AMD Software Optimization Guide - Section 6.2 */ 333 END(dtrace_membar_producer) 334 335/* 336void 337dtrace_membar_consumer(void) 338*/ 339 ENTRY(dtrace_membar_consumer) 340 rep; ret /* use 2 byte return instruction when branch target */ 341 /* AMD Software Optimization Guide - Section 6.2 */ 342 END(dtrace_membar_consumer) 343 344/* 345dtrace_icookie_t 346dtrace_interrupt_disable(void) 347*/ 348 ENTRY(dtrace_interrupt_disable) 349 pushfq 350 popq %rax 351 cli 352 ret 353 END(dtrace_interrupt_disable) 354 355/* 356void 357dtrace_interrupt_enable(dtrace_icookie_t cookie) 358*/ 359 ENTRY(dtrace_interrupt_enable) 360 pushq %rdi 361 popfq 362 ret 363 END(dtrace_interrupt_enable) 364