1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2008 John Birrell <jb@freebsd.org> 22 * 23 * $FreeBSD$ 24 * 25 */ 26/* 27 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#define _ASM 32 33#include <machine/asmacros.h> 34#include <sys/cpuvar_defs.h> 35#include <sys/dtrace.h> 36 37#include "assym.inc" 38 39#define INTR_POP \ 40 MEXITCOUNT; \ 41 movq TF_RDI(%rsp),%rdi; \ 42 movq TF_RSI(%rsp),%rsi; \ 43 movq TF_RDX(%rsp),%rdx; \ 44 movq TF_RCX(%rsp),%rcx; \ 45 movq TF_R8(%rsp),%r8; \ 46 movq TF_R9(%rsp),%r9; \ 47 movq TF_RAX(%rsp),%rax; \ 48 movq TF_RBX(%rsp),%rbx; \ 49 movq TF_RBP(%rsp),%rbp; \ 50 movq TF_R10(%rsp),%r10; \ 51 movq TF_R11(%rsp),%r11; \ 52 movq TF_R12(%rsp),%r12; \ 53 movq TF_R13(%rsp),%r13; \ 54 movq TF_R14(%rsp),%r14; \ 55 movq TF_R15(%rsp),%r15; \ 56 testb $SEL_RPL_MASK,TF_CS(%rsp); \ 57 jz 1f; \ 58 cli; \ 59 swapgs; \ 601: addq $TF_RIP,%rsp; 61 62 63 ENTRY(dtrace_invop_start) 64 65 /* 66 * #BP traps with %rip set to the next address. We need to decrement 67 * the value to indicate the address of the int3 (0xcc) instruction 68 * that we substituted. 69 */ 70 movq TF_RIP(%rsp), %rdi 71 decq %rdi 72 movq %rsp, %rsi 73 movq TF_RAX(%rsp), %rdx 74 call dtrace_invop 75 ALTENTRY(dtrace_invop_callsite) 76 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 77 je bp_push 78 cmpl $DTRACE_INVOP_LEAVE, %eax 79 je bp_leave 80 cmpl $DTRACE_INVOP_NOP, %eax 81 je bp_nop 82 cmpl $DTRACE_INVOP_RET, %eax 83 je bp_ret 84 85 /* When all else fails handle the trap in the usual way. */ 86 jmpq *dtrace_invop_calltrap_addr 87 88bp_push: 89 /* 90 * We must emulate a "pushq %rbp". To do this, we pull the stack 91 * down 8 bytes, and then store the base pointer. 92 */ 93 INTR_POP 94 subq $16, %rsp /* make room for %rbp */ 95 pushq %rax /* push temp */ 96 movq 24(%rsp), %rax /* load calling RIP */ 97 movq %rax, 8(%rsp) /* store calling RIP */ 98 movq 32(%rsp), %rax /* load calling CS */ 99 movq %rax, 16(%rsp) /* store calling CS */ 100 movq 40(%rsp), %rax /* load calling RFLAGS */ 101 movq %rax, 24(%rsp) /* store calling RFLAGS */ 102 movq 48(%rsp), %rax /* load calling RSP */ 103 subq $8, %rax /* make room for %rbp */ 104 movq %rax, 32(%rsp) /* store calling RSP */ 105 movq 56(%rsp), %rax /* load calling SS */ 106 movq %rax, 40(%rsp) /* store calling SS */ 107 movq 32(%rsp), %rax /* reload calling RSP */ 108 movq %rbp, (%rax) /* store %rbp there */ 109 popq %rax /* pop off temp */ 110 iretq /* return from interrupt */ 111 /*NOTREACHED*/ 112 113bp_leave: 114 /* 115 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 116 * followed by a "popq %rbp". This is quite a bit simpler on amd64 117 * than it is on i386 -- we can exploit the fact that the %rsp is 118 * explicitly saved to effect the pop without having to reshuffle 119 * the other data pushed for the trap. 120 */ 121 INTR_POP 122 pushq %rax /* push temp */ 123 movq 8(%rsp), %rax /* load calling RIP */ 124 movq %rax, 8(%rsp) /* store calling RIP */ 125 movq (%rbp), %rax /* get new %rbp */ 126 addq $8, %rbp /* adjust new %rsp */ 127 movq %rbp, 32(%rsp) /* store new %rsp */ 128 movq %rax, %rbp /* set new %rbp */ 129 popq %rax /* pop off temp */ 130 iretq /* return from interrupt */ 131 /*NOTREACHED*/ 132 133bp_nop: 134 /* We must emulate a "nop". */ 135 INTR_POP 136 iretq 137 /*NOTREACHED*/ 138 139bp_ret: 140 INTR_POP 141 pushq %rax /* push temp */ 142 movq 32(%rsp), %rax /* load %rsp */ 143 movq (%rax), %rax /* load calling RIP */ 144 movq %rax, 8(%rsp) /* store calling RIP */ 145 addq $8, 32(%rsp) /* adjust new %rsp */ 146 popq %rax /* pop off temp */ 147 iretq /* return from interrupt */ 148 /*NOTREACHED*/ 149 150 END(dtrace_invop_start) 151 152/* 153void dtrace_invop_init(void) 154*/ 155 ENTRY(dtrace_invop_init) 156 movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) 157 ret 158 END(dtrace_invop_init) 159 160/* 161void dtrace_invop_uninit(void) 162*/ 163 ENTRY(dtrace_invop_uninit) 164 movq $0, dtrace_invop_jump_addr(%rip) 165 ret 166 END(dtrace_invop_uninit) 167 168/* 169greg_t dtrace_getfp(void) 170*/ 171 ENTRY(dtrace_getfp) 172 movq %rbp, %rax 173 ret 174 END(dtrace_getfp) 175 176/* 177uint32_t 178dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) 179*/ 180 ENTRY(dtrace_cas32) 181 movl %esi, %eax 182 lock 183 cmpxchgl %edx, (%rdi) 184 ret 185 END(dtrace_cas32) 186 187/* 188void * 189dtrace_casptr(void *target, void *cmp, void *new) 190*/ 191 ENTRY(dtrace_casptr) 192 movq %rsi, %rax 193 lock 194 cmpxchgq %rdx, (%rdi) 195 ret 196 END(dtrace_casptr) 197 198/* 199uintptr_t 200dtrace_caller(int aframes) 201*/ 202 ENTRY(dtrace_caller) 203 movq $-1, %rax 204 ret 205 END(dtrace_caller) 206 207/* 208void 209dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) 210*/ 211 ENTRY(dtrace_copy_nosmap) 212 pushq %rbp 213 movq %rsp, %rbp 214 215 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 216 movq %rdx, %rcx /* load count */ 217 repz /* repeat for count ... */ 218 smovb /* move from %ds:rsi to %ed:rdi */ 219 leave 220 ret 221 END(dtrace_copy_nosmap) 222 223 ENTRY(dtrace_copy_smap) 224 pushq %rbp 225 movq %rsp, %rbp 226 227 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 228 movq %rdx, %rcx /* load count */ 229 stac 230 repz /* repeat for count ... */ 231 smovb /* move from %ds:rsi to %ed:rdi */ 232 clac 233 leave 234 ret 235 END(dtrace_copy_smap) 236 237/* 238void 239dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, 240 volatile uint16_t *flags) 241*/ 242 ENTRY(dtrace_copystr_nosmap) 243 pushq %rbp 244 movq %rsp, %rbp 245 2460: 247 movb (%rdi), %al /* load from source */ 248 movb %al, (%rsi) /* store to destination */ 249 addq $1, %rdi /* increment source pointer */ 250 addq $1, %rsi /* increment destination pointer */ 251 subq $1, %rdx /* decrement remaining count */ 252 cmpb $0, %al 253 je 2f 254 testq $0xfff, %rdx /* test if count is 4k-aligned */ 255 jnz 1f /* if not, continue with copying */ 256 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 257 jnz 2f 2581: 259 cmpq $0, %rdx 260 jne 0b 2612: 262 leave 263 ret 264 265 END(dtrace_copystr_nosmap) 266 267 ENTRY(dtrace_copystr_smap) 268 pushq %rbp 269 movq %rsp, %rbp 270 271 stac 2720: 273 movb (%rdi), %al /* load from source */ 274 movb %al, (%rsi) /* store to destination */ 275 addq $1, %rdi /* increment source pointer */ 276 addq $1, %rsi /* increment destination pointer */ 277 subq $1, %rdx /* decrement remaining count */ 278 cmpb $0, %al 279 je 2f 280 testq $0xfff, %rdx /* test if count is 4k-aligned */ 281 jnz 1f /* if not, continue with copying */ 282 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 283 jnz 2f 2841: 285 cmpq $0, %rdx 286 jne 0b 2872: 288 clac 289 leave 290 ret 291 292 END(dtrace_copystr_smap) 293 294/* 295uintptr_t 296dtrace_fulword(void *addr) 297*/ 298 ENTRY(dtrace_fulword_nosmap) 299 movq (%rdi), %rax 300 ret 301 END(dtrace_fulword_nosmap) 302 303 ENTRY(dtrace_fulword_smap) 304 stac 305 movq (%rdi), %rax 306 clac 307 ret 308 END(dtrace_fulword_smap) 309 310/* 311uint8_t 312dtrace_fuword8_nocheck(void *addr) 313*/ 314 ENTRY(dtrace_fuword8_nocheck_nosmap) 315 xorq %rax, %rax 316 movb (%rdi), %al 317 ret 318 END(dtrace_fuword8_nocheck_nosmap) 319 320 ENTRY(dtrace_fuword8_nocheck_smap) 321 stac 322 xorq %rax, %rax 323 movb (%rdi), %al 324 clac 325 ret 326 END(dtrace_fuword8_nocheck_smap) 327 328/* 329uint16_t 330dtrace_fuword16_nocheck(void *addr) 331*/ 332 ENTRY(dtrace_fuword16_nocheck_nosmap) 333 xorq %rax, %rax 334 movw (%rdi), %ax 335 ret 336 END(dtrace_fuword16_nocheck_nosmap) 337 338 ENTRY(dtrace_fuword16_nocheck_smap) 339 stac 340 xorq %rax, %rax 341 movw (%rdi), %ax 342 clac 343 ret 344 END(dtrace_fuword16_nocheck_smap) 345 346/* 347uint32_t 348dtrace_fuword32_nocheck(void *addr) 349*/ 350 ENTRY(dtrace_fuword32_nocheck_nosmap) 351 xorq %rax, %rax 352 movl (%rdi), %eax 353 ret 354 END(dtrace_fuword32_nocheck_nosmap) 355 356 ENTRY(dtrace_fuword32_nocheck_smap) 357 stac 358 xorq %rax, %rax 359 movl (%rdi), %eax 360 clac 361 ret 362 END(dtrace_fuword32_nocheck_smap) 363 364/* 365uint64_t 366dtrace_fuword64_nocheck(void *addr) 367*/ 368 ENTRY(dtrace_fuword64_nocheck_nosmap) 369 movq (%rdi), %rax 370 ret 371 END(dtrace_fuword64_nocheck_nosmap) 372 373 ENTRY(dtrace_fuword64_nocheck_smap) 374 stac 375 movq (%rdi), %rax 376 clac 377 ret 378 END(dtrace_fuword64_nocheck_smap) 379 380/* 381void 382dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, 383 int fault, int fltoffs, uintptr_t illval) 384*/ 385 ENTRY(dtrace_probe_error) 386 pushq %rbp 387 movq %rsp, %rbp 388 subq $0x8, %rsp 389 movq %r9, (%rsp) 390 movq %r8, %r9 391 movq %rcx, %r8 392 movq %rdx, %rcx 393 movq %rsi, %rdx 394 movq %rdi, %rsi 395 movl dtrace_probeid_error(%rip), %edi 396 call dtrace_probe 397 addq $0x8, %rsp 398 leave 399 ret 400 END(dtrace_probe_error) 401 402/* 403void 404dtrace_membar_producer(void) 405*/ 406 ENTRY(dtrace_membar_producer) 407 rep; ret /* use 2 byte return instruction when branch target */ 408 /* AMD Software Optimization Guide - Section 6.2 */ 409 END(dtrace_membar_producer) 410 411/* 412void 413dtrace_membar_consumer(void) 414*/ 415 ENTRY(dtrace_membar_consumer) 416 rep; ret /* use 2 byte return instruction when branch target */ 417 /* AMD Software Optimization Guide - Section 6.2 */ 418 END(dtrace_membar_consumer) 419 420/* 421dtrace_icookie_t 422dtrace_interrupt_disable(void) 423*/ 424 ENTRY(dtrace_interrupt_disable) 425 pushfq 426 popq %rax 427 cli 428 ret 429 END(dtrace_interrupt_disable) 430 431/* 432void 433dtrace_interrupt_enable(dtrace_icookie_t cookie) 434*/ 435 ENTRY(dtrace_interrupt_enable) 436 pushq %rdi 437 popfq 438 ret 439 END(dtrace_interrupt_enable) 440