1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2008 John Birrell <jb@freebsd.org> 22 * 23 * $FreeBSD$ 24 * 25 */ 26/* 27 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#define _ASM 32 33#include <machine/asmacros.h> 34#include <sys/cpuvar_defs.h> 35#include <sys/dtrace.h> 36 37#include "assym.inc" 38 39#define INTR_POP \ 40 movq TF_RDI(%rsp),%rdi; \ 41 movq TF_RSI(%rsp),%rsi; \ 42 movq TF_RDX(%rsp),%rdx; \ 43 movq TF_RCX(%rsp),%rcx; \ 44 movq TF_R8(%rsp),%r8; \ 45 movq TF_R9(%rsp),%r9; \ 46 movq TF_RAX(%rsp),%rax; \ 47 movq TF_RBX(%rsp),%rbx; \ 48 movq TF_RBP(%rsp),%rbp; \ 49 movq TF_R10(%rsp),%r10; \ 50 movq TF_R11(%rsp),%r11; \ 51 movq TF_R12(%rsp),%r12; \ 52 movq TF_R13(%rsp),%r13; \ 53 movq TF_R14(%rsp),%r14; \ 54 movq TF_R15(%rsp),%r15; \ 55 testb $SEL_RPL_MASK,TF_CS(%rsp); \ 56 jz 1f; \ 57 cli; \ 58 swapgs; \ 591: addq $TF_RIP,%rsp; 60 61 62 ENTRY(dtrace_invop_start) 63 64 /* 65 * #BP traps with %rip set to the next address. We need to decrement 66 * the value to indicate the address of the int3 (0xcc) instruction 67 * that we substituted. 68 */ 69 movq TF_RIP(%rsp), %rdi 70 decq %rdi 71 movq %rsp, %rsi 72 movq TF_RAX(%rsp), %rdx 73 call dtrace_invop 74 ENTRY(dtrace_invop_callsite) 75 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 76 je bp_push 77 cmpl $DTRACE_INVOP_LEAVE, %eax 78 je bp_leave 79 cmpl $DTRACE_INVOP_NOP, %eax 80 je bp_nop 81 cmpl $DTRACE_INVOP_RET, %eax 82 je bp_ret 83 84 /* When all else fails handle the trap in the usual way. */ 85 jmpq *dtrace_invop_calltrap_addr 86 87bp_push: 88 /* 89 * We must emulate a "pushq %rbp". To do this, we pull the stack 90 * down 8 bytes, and then store the base pointer. 91 */ 92 INTR_POP 93 subq $16, %rsp /* make room for %rbp */ 94 pushq %rax /* push temp */ 95 movq 24(%rsp), %rax /* load calling RIP */ 96 movq %rax, 8(%rsp) /* store calling RIP */ 97 movq 32(%rsp), %rax /* load calling CS */ 98 movq %rax, 16(%rsp) /* store calling CS */ 99 movq 40(%rsp), %rax /* load calling RFLAGS */ 100 movq %rax, 24(%rsp) /* store calling RFLAGS */ 101 movq 48(%rsp), %rax /* load calling RSP */ 102 subq $8, %rax /* make room for %rbp */ 103 movq %rax, 32(%rsp) /* store calling RSP */ 104 movq 56(%rsp), %rax /* load calling SS */ 105 movq %rax, 40(%rsp) /* store calling SS */ 106 movq 32(%rsp), %rax /* reload calling RSP */ 107 movq %rbp, (%rax) /* store %rbp there */ 108 popq %rax /* pop off temp */ 109 iretq /* return from interrupt */ 110 /*NOTREACHED*/ 111 112bp_leave: 113 /* 114 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 115 * followed by a "popq %rbp". This is quite a bit simpler on amd64 116 * than it is on i386 -- we can exploit the fact that the %rsp is 117 * explicitly saved to effect the pop without having to reshuffle 118 * the other data pushed for the trap. 119 */ 120 INTR_POP 121 pushq %rax /* push temp */ 122 movq 8(%rsp), %rax /* load calling RIP */ 123 movq %rax, 8(%rsp) /* store calling RIP */ 124 movq (%rbp), %rax /* get new %rbp */ 125 addq $8, %rbp /* adjust new %rsp */ 126 movq %rbp, 32(%rsp) /* store new %rsp */ 127 movq %rax, %rbp /* set new %rbp */ 128 popq %rax /* pop off temp */ 129 iretq /* return from interrupt */ 130 /*NOTREACHED*/ 131 132bp_nop: 133 /* We must emulate a "nop". */ 134 INTR_POP 135 iretq 136 /*NOTREACHED*/ 137 138bp_ret: 139 INTR_POP 140 pushq %rax /* push temp */ 141 movq 32(%rsp), %rax /* load %rsp */ 142 movq (%rax), %rax /* load calling RIP */ 143 movq %rax, 8(%rsp) /* store calling RIP */ 144 addq $8, 32(%rsp) /* adjust new %rsp */ 145 popq %rax /* pop off temp */ 146 iretq /* return from interrupt */ 147 /*NOTREACHED*/ 148 149 END(dtrace_invop_start) 150 151/* 152greg_t dtrace_getfp(void) 153*/ 154 ENTRY(dtrace_getfp) 155 movq %rbp, %rax 156 ret 157 END(dtrace_getfp) 158 159/* 160uint32_t 161dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) 162*/ 163 ENTRY(dtrace_cas32) 164 movl %esi, %eax 165 lock 166 cmpxchgl %edx, (%rdi) 167 ret 168 END(dtrace_cas32) 169 170/* 171void * 172dtrace_casptr(void *target, void *cmp, void *new) 173*/ 174 ENTRY(dtrace_casptr) 175 movq %rsi, %rax 176 lock 177 cmpxchgq %rdx, (%rdi) 178 ret 179 END(dtrace_casptr) 180 181/* 182uintptr_t 183dtrace_caller(int aframes) 184*/ 185 ENTRY(dtrace_caller) 186 movq $-1, %rax 187 ret 188 END(dtrace_caller) 189 190/* 191void 192dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) 193*/ 194 ENTRY(dtrace_copy_nosmap) 195 pushq %rbp 196 movq %rsp, %rbp 197 198 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 199 movq %rdx, %rcx /* load count */ 200 repz /* repeat for count ... */ 201 smovb /* move from %ds:rsi to %ed:rdi */ 202 leave 203 ret 204 END(dtrace_copy_nosmap) 205 206 ENTRY(dtrace_copy_smap) 207 pushq %rbp 208 movq %rsp, %rbp 209 210 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 211 movq %rdx, %rcx /* load count */ 212 stac 213 repz /* repeat for count ... */ 214 smovb /* move from %ds:rsi to %ed:rdi */ 215 clac 216 leave 217 ret 218 END(dtrace_copy_smap) 219 220/* 221void 222dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, 223 volatile uint16_t *flags) 224*/ 225 ENTRY(dtrace_copystr_nosmap) 226 pushq %rbp 227 movq %rsp, %rbp 228 2290: 230 movb (%rdi), %al /* load from source */ 231 movb %al, (%rsi) /* store to destination */ 232 addq $1, %rdi /* increment source pointer */ 233 addq $1, %rsi /* increment destination pointer */ 234 subq $1, %rdx /* decrement remaining count */ 235 cmpb $0, %al 236 je 2f 237 testq $0xfff, %rdx /* test if count is 4k-aligned */ 238 jnz 1f /* if not, continue with copying */ 239 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 240 jnz 2f 2411: 242 cmpq $0, %rdx 243 jne 0b 2442: 245 leave 246 ret 247 248 END(dtrace_copystr_nosmap) 249 250 ENTRY(dtrace_copystr_smap) 251 pushq %rbp 252 movq %rsp, %rbp 253 254 stac 2550: 256 movb (%rdi), %al /* load from source */ 257 movb %al, (%rsi) /* store to destination */ 258 addq $1, %rdi /* increment source pointer */ 259 addq $1, %rsi /* increment destination pointer */ 260 subq $1, %rdx /* decrement remaining count */ 261 cmpb $0, %al 262 je 2f 263 testq $0xfff, %rdx /* test if count is 4k-aligned */ 264 jnz 1f /* if not, continue with copying */ 265 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 266 jnz 2f 2671: 268 cmpq $0, %rdx 269 jne 0b 2702: 271 clac 272 leave 273 ret 274 275 END(dtrace_copystr_smap) 276 277/* 278uintptr_t 279dtrace_fulword(void *addr) 280*/ 281 ENTRY(dtrace_fulword_nosmap) 282 movq (%rdi), %rax 283 ret 284 END(dtrace_fulword_nosmap) 285 286 ENTRY(dtrace_fulword_smap) 287 stac 288 movq (%rdi), %rax 289 clac 290 ret 291 END(dtrace_fulword_smap) 292 293/* 294uint8_t 295dtrace_fuword8_nocheck(void *addr) 296*/ 297 ENTRY(dtrace_fuword8_nocheck_nosmap) 298 xorq %rax, %rax 299 movb (%rdi), %al 300 ret 301 END(dtrace_fuword8_nocheck_nosmap) 302 303 ENTRY(dtrace_fuword8_nocheck_smap) 304 stac 305 xorq %rax, %rax 306 movb (%rdi), %al 307 clac 308 ret 309 END(dtrace_fuword8_nocheck_smap) 310 311/* 312uint16_t 313dtrace_fuword16_nocheck(void *addr) 314*/ 315 ENTRY(dtrace_fuword16_nocheck_nosmap) 316 xorq %rax, %rax 317 movw (%rdi), %ax 318 ret 319 END(dtrace_fuword16_nocheck_nosmap) 320 321 ENTRY(dtrace_fuword16_nocheck_smap) 322 stac 323 xorq %rax, %rax 324 movw (%rdi), %ax 325 clac 326 ret 327 END(dtrace_fuword16_nocheck_smap) 328 329/* 330uint32_t 331dtrace_fuword32_nocheck(void *addr) 332*/ 333 ENTRY(dtrace_fuword32_nocheck_nosmap) 334 xorq %rax, %rax 335 movl (%rdi), %eax 336 ret 337 END(dtrace_fuword32_nocheck_nosmap) 338 339 ENTRY(dtrace_fuword32_nocheck_smap) 340 stac 341 xorq %rax, %rax 342 movl (%rdi), %eax 343 clac 344 ret 345 END(dtrace_fuword32_nocheck_smap) 346 347/* 348uint64_t 349dtrace_fuword64_nocheck(void *addr) 350*/ 351 ENTRY(dtrace_fuword64_nocheck_nosmap) 352 movq (%rdi), %rax 353 ret 354 END(dtrace_fuword64_nocheck_nosmap) 355 356 ENTRY(dtrace_fuword64_nocheck_smap) 357 stac 358 movq (%rdi), %rax 359 clac 360 ret 361 END(dtrace_fuword64_nocheck_smap) 362 363/* 364void 365dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, 366 int fault, int fltoffs, uintptr_t illval) 367*/ 368 ENTRY(dtrace_probe_error) 369 pushq %rbp 370 movq %rsp, %rbp 371 subq $0x8, %rsp 372 movq %r9, (%rsp) 373 movq %r8, %r9 374 movq %rcx, %r8 375 movq %rdx, %rcx 376 movq %rsi, %rdx 377 movq %rdi, %rsi 378 movl dtrace_probeid_error(%rip), %edi 379 call dtrace_probe 380 addq $0x8, %rsp 381 leave 382 ret 383 END(dtrace_probe_error) 384 385/* 386void 387dtrace_membar_producer(void) 388*/ 389 ENTRY(dtrace_membar_producer) 390 rep; ret /* use 2 byte return instruction when branch target */ 391 /* AMD Software Optimization Guide - Section 6.2 */ 392 END(dtrace_membar_producer) 393 394/* 395void 396dtrace_membar_consumer(void) 397*/ 398 ENTRY(dtrace_membar_consumer) 399 rep; ret /* use 2 byte return instruction when branch target */ 400 /* AMD Software Optimization Guide - Section 6.2 */ 401 END(dtrace_membar_consumer) 402 403/* 404dtrace_icookie_t 405dtrace_interrupt_disable(void) 406*/ 407 ENTRY(dtrace_interrupt_disable) 408 pushfq 409 popq %rax 410 cli 411 ret 412 END(dtrace_interrupt_disable) 413 414/* 415void 416dtrace_interrupt_enable(dtrace_icookie_t cookie) 417*/ 418 ENTRY(dtrace_interrupt_enable) 419 pushq %rdi 420 popfq 421 ret 422 END(dtrace_interrupt_enable) 423