1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2008 John Birrell <jb@freebsd.org> 22 * 23 * $FreeBSD$ 24 * 25 */ 26/* 27 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#define _ASM 32 33#include <machine/asmacros.h> 34#include <sys/cpuvar_defs.h> 35#include <sys/dtrace.h> 36 37#include "assym.s" 38 39#define INTR_POP \ 40 MEXITCOUNT; \ 41 movq TF_RDI(%rsp),%rdi; \ 42 movq TF_RSI(%rsp),%rsi; \ 43 movq TF_RDX(%rsp),%rdx; \ 44 movq TF_RCX(%rsp),%rcx; \ 45 movq TF_R8(%rsp),%r8; \ 46 movq TF_R9(%rsp),%r9; \ 47 movq TF_RAX(%rsp),%rax; \ 48 movq TF_RBX(%rsp),%rbx; \ 49 movq TF_RBP(%rsp),%rbp; \ 50 movq TF_R10(%rsp),%r10; \ 51 movq TF_R11(%rsp),%r11; \ 52 movq TF_R12(%rsp),%r12; \ 53 movq TF_R13(%rsp),%r13; \ 54 movq TF_R14(%rsp),%r14; \ 55 movq TF_R15(%rsp),%r15; \ 56 testb $SEL_RPL_MASK,TF_CS(%rsp); \ 57 jz 1f; \ 58 cli; \ 59 swapgs; \ 601: addq $TF_RIP,%rsp; 61 62 63 .globl calltrap 64 .type calltrap,@function 65 ENTRY(dtrace_invop_start) 66 67 /* 68 * #BP traps with %rip set to the next address. We need to decrement 69 * the value to indicate the address of the int3 (0xcc) instruction 70 * that we substituted. 71 */ 72 movq TF_RIP(%rsp), %rdi 73 decq %rdi 74 movq TF_RSP(%rsp), %rsi 75 movq TF_RAX(%rsp), %rdx 76 pushq (%rsi) 77 movq %rsp, %rsi 78 call dtrace_invop 79 ALTENTRY(dtrace_invop_callsite) 80 addq $8, %rsp 81 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 82 je bp_push 83 cmpl $DTRACE_INVOP_LEAVE, %eax 84 je bp_leave 85 cmpl $DTRACE_INVOP_NOP, %eax 86 je bp_nop 87 cmpl $DTRACE_INVOP_RET, %eax 88 je bp_ret 89 90 /* When all else fails handle the trap in the usual way. */ 91 jmpq *dtrace_invop_calltrap_addr 92 93bp_push: 94 /* 95 * We must emulate a "pushq %rbp". To do this, we pull the stack 96 * down 8 bytes, and then store the base pointer. 97 */ 98 INTR_POP 99 subq $16, %rsp /* make room for %rbp */ 100 pushq %rax /* push temp */ 101 movq 24(%rsp), %rax /* load calling RIP */ 102 movq %rax, 8(%rsp) /* store calling RIP */ 103 movq 32(%rsp), %rax /* load calling CS */ 104 movq %rax, 16(%rsp) /* store calling CS */ 105 movq 40(%rsp), %rax /* load calling RFLAGS */ 106 movq %rax, 24(%rsp) /* store calling RFLAGS */ 107 movq 48(%rsp), %rax /* load calling RSP */ 108 subq $8, %rax /* make room for %rbp */ 109 movq %rax, 32(%rsp) /* store calling RSP */ 110 movq 56(%rsp), %rax /* load calling SS */ 111 movq %rax, 40(%rsp) /* store calling SS */ 112 movq 32(%rsp), %rax /* reload calling RSP */ 113 movq %rbp, (%rax) /* store %rbp there */ 114 popq %rax /* pop off temp */ 115 iretq /* return from interrupt */ 116 /*NOTREACHED*/ 117 118bp_leave: 119 /* 120 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 121 * followed by a "popq %rbp". This is quite a bit simpler on amd64 122 * than it is on i386 -- we can exploit the fact that the %rsp is 123 * explicitly saved to effect the pop without having to reshuffle 124 * the other data pushed for the trap. 125 */ 126 INTR_POP 127 pushq %rax /* push temp */ 128 movq 8(%rsp), %rax /* load calling RIP */ 129 movq %rax, 8(%rsp) /* store calling RIP */ 130 movq (%rbp), %rax /* get new %rbp */ 131 addq $8, %rbp /* adjust new %rsp */ 132 movq %rbp, 32(%rsp) /* store new %rsp */ 133 movq %rax, %rbp /* set new %rbp */ 134 popq %rax /* pop off temp */ 135 iretq /* return from interrupt */ 136 /*NOTREACHED*/ 137 138bp_nop: 139 /* We must emulate a "nop". */ 140 INTR_POP 141 iretq 142 /*NOTREACHED*/ 143 144bp_ret: 145 INTR_POP 146 pushq %rax /* push temp */ 147 movq 32(%rsp), %rax /* load %rsp */ 148 movq (%rax), %rax /* load calling RIP */ 149 movq %rax, 8(%rsp) /* store calling RIP */ 150 addq $8, 32(%rsp) /* adjust new %rsp */ 151 popq %rax /* pop off temp */ 152 iretq /* return from interrupt */ 153 /*NOTREACHED*/ 154 155 END(dtrace_invop_start) 156 157/* 158void dtrace_invop_init(void) 159*/ 160 ENTRY(dtrace_invop_init) 161 movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) 162 ret 163 END(dtrace_invop_init) 164 165/* 166void dtrace_invop_uninit(void) 167*/ 168 ENTRY(dtrace_invop_uninit) 169 movq $0, dtrace_invop_jump_addr(%rip) 170 ret 171 END(dtrace_invop_uninit) 172 173/* 174greg_t dtrace_getfp(void) 175*/ 176 ENTRY(dtrace_getfp) 177 movq %rbp, %rax 178 ret 179 END(dtrace_getfp) 180 181/* 182uint32_t 183dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) 184*/ 185 ENTRY(dtrace_cas32) 186 movl %esi, %eax 187 lock 188 cmpxchgl %edx, (%rdi) 189 ret 190 END(dtrace_cas32) 191 192/* 193void * 194dtrace_casptr(void *target, void *cmp, void *new) 195*/ 196 ENTRY(dtrace_casptr) 197 movq %rsi, %rax 198 lock 199 cmpxchgq %rdx, (%rdi) 200 ret 201 END(dtrace_casptr) 202 203/* 204uintptr_t 205dtrace_caller(int aframes) 206*/ 207 ENTRY(dtrace_caller) 208 movq $-1, %rax 209 ret 210 END(dtrace_caller) 211 212/* 213void 214dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) 215*/ 216 ENTRY(dtrace_copy) 217 pushq %rbp 218 movq %rsp, %rbp 219 220 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 221 movq %rdx, %rcx /* load count */ 222 repz /* repeat for count ... */ 223 smovb /* move from %ds:rsi to %ed:rdi */ 224 leave 225 ret 226 END(dtrace_copy) 227 228/* 229void 230dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, 231 volatile uint16_t *flags) 232*/ 233 ENTRY(dtrace_copystr) 234 pushq %rbp 235 movq %rsp, %rbp 236 2370: 238 movb (%rdi), %al /* load from source */ 239 movb %al, (%rsi) /* store to destination */ 240 addq $1, %rdi /* increment source pointer */ 241 addq $1, %rsi /* increment destination pointer */ 242 subq $1, %rdx /* decrement remaining count */ 243 cmpb $0, %al 244 je 2f 245 testq $0xfff, %rdx /* test if count is 4k-aligned */ 246 jnz 1f /* if not, continue with copying */ 247 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 248 jnz 2f 2491: 250 cmpq $0, %rdx 251 jne 0b 2522: 253 leave 254 ret 255 256 END(dtrace_copystr) 257 258/* 259uintptr_t 260dtrace_fulword(void *addr) 261*/ 262 ENTRY(dtrace_fulword) 263 movq (%rdi), %rax 264 ret 265 END(dtrace_fulword) 266 267/* 268uint8_t 269dtrace_fuword8_nocheck(void *addr) 270*/ 271 ENTRY(dtrace_fuword8_nocheck) 272 xorq %rax, %rax 273 movb (%rdi), %al 274 ret 275 END(dtrace_fuword8_nocheck) 276 277/* 278uint16_t 279dtrace_fuword16_nocheck(void *addr) 280*/ 281 ENTRY(dtrace_fuword16_nocheck) 282 xorq %rax, %rax 283 movw (%rdi), %ax 284 ret 285 END(dtrace_fuword16_nocheck) 286 287/* 288uint32_t 289dtrace_fuword32_nocheck(void *addr) 290*/ 291 ENTRY(dtrace_fuword32_nocheck) 292 xorq %rax, %rax 293 movl (%rdi), %eax 294 ret 295 END(dtrace_fuword32_nocheck) 296 297/* 298uint64_t 299dtrace_fuword64_nocheck(void *addr) 300*/ 301 ENTRY(dtrace_fuword64_nocheck) 302 movq (%rdi), %rax 303 ret 304 END(dtrace_fuword64_nocheck) 305 306/* 307void 308dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, 309 int fault, int fltoffs, uintptr_t illval) 310*/ 311 ENTRY(dtrace_probe_error) 312 pushq %rbp 313 movq %rsp, %rbp 314 subq $0x8, %rsp 315 movq %r9, (%rsp) 316 movq %r8, %r9 317 movq %rcx, %r8 318 movq %rdx, %rcx 319 movq %rsi, %rdx 320 movq %rdi, %rsi 321 movl dtrace_probeid_error(%rip), %edi 322 call dtrace_probe 323 addq $0x8, %rsp 324 leave 325 ret 326 END(dtrace_probe_error) 327 328/* 329void 330dtrace_membar_producer(void) 331*/ 332 ENTRY(dtrace_membar_producer) 333 rep; ret /* use 2 byte return instruction when branch target */ 334 /* AMD Software Optimization Guide - Section 6.2 */ 335 END(dtrace_membar_producer) 336 337/* 338void 339dtrace_membar_consumer(void) 340*/ 341 ENTRY(dtrace_membar_consumer) 342 rep; ret /* use 2 byte return instruction when branch target */ 343 /* AMD Software Optimization Guide - Section 6.2 */ 344 END(dtrace_membar_consumer) 345 346/* 347dtrace_icookie_t 348dtrace_interrupt_disable(void) 349*/ 350 ENTRY(dtrace_interrupt_disable) 351 pushfq 352 popq %rax 353 cli 354 ret 355 END(dtrace_interrupt_disable) 356 357/* 358void 359dtrace_interrupt_enable(dtrace_icookie_t cookie) 360*/ 361 ENTRY(dtrace_interrupt_enable) 362 pushq %rdi 363 popfq 364 ret 365 END(dtrace_interrupt_enable) 366 367/* 368 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 369 * into the panic code implemented in panicsys(). vpanic() is responsible 370 * for passing through the format string and arguments, and constructing a 371 * regs structure on the stack into which it saves the current register 372 * values. If we are not dying due to a fatal trap, these registers will 373 * then be preserved in panicbuf as the current processor state. Before 374 * invoking panicsys(), vpanic() activates the first panic trigger (see 375 * common/os/panic.c) and switches to the panic_stack if successful. Note that 376 * DTrace takes a slightly different panic path if it must panic from probe 377 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 378 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 379 * branches back into vpanic(). 380 */ 381 382/* 383void 384vpanic(const char *format, va_list alist) 385*/ 386 ENTRY(vpanic) /* Initial stack layout: */ 387 388 pushq %rbp /* | %rip | 0x60 */ 389 movq %rsp, %rbp /* | %rbp | 0x58 */ 390 pushfq /* | rfl | 0x50 */ 391 pushq %r11 /* | %r11 | 0x48 */ 392 pushq %r10 /* | %r10 | 0x40 */ 393 pushq %rbx /* | %rbx | 0x38 */ 394 pushq %rax /* | %rax | 0x30 */ 395 pushq %r9 /* | %r9 | 0x28 */ 396 pushq %r8 /* | %r8 | 0x20 */ 397 pushq %rcx /* | %rcx | 0x18 */ 398 pushq %rdx /* | %rdx | 0x10 */ 399 pushq %rsi /* | %rsi | 0x8 alist */ 400 pushq %rdi /* | %rdi | 0x0 format */ 401 402 movq %rsp, %rbx /* %rbx = current %rsp */ 403 404 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 405 call panic_trigger /* %eax = panic_trigger() */ 406 407vpanic_common: 408 /* 409 * The panic_trigger result is in %eax from the call above, and 410 * dtrace_panic places it in %eax before branching here. 411 * The rdmsr instructions that follow below will clobber %eax so 412 * we stash the panic_trigger result in %r11d. 413 */ 414 movl %eax, %r11d 415 cmpl $0, %r11d 416 je 0f 417 418 /* 419 * If panic_trigger() was successful, we are the first to initiate a 420 * panic: we now switch to the reserved panic_stack before continuing. 421 */ 422 leaq panic_stack(%rip), %rsp 423 addq $PANICSTKSIZE, %rsp 4240: subq $REGSIZE, %rsp 425 /* 426 * Now that we've got everything set up, store the register values as 427 * they were when we entered vpanic() to the designated location in 428 * the regs structure we allocated on the stack. 429 */ 430#ifdef notyet 431 movq 0x0(%rbx), %rcx 432 movq %rcx, REGOFF_RDI(%rsp) 433 movq 0x8(%rbx), %rcx 434 movq %rcx, REGOFF_RSI(%rsp) 435 movq 0x10(%rbx), %rcx 436 movq %rcx, REGOFF_RDX(%rsp) 437 movq 0x18(%rbx), %rcx 438 movq %rcx, REGOFF_RCX(%rsp) 439 movq 0x20(%rbx), %rcx 440 441 movq %rcx, REGOFF_R8(%rsp) 442 movq 0x28(%rbx), %rcx 443 movq %rcx, REGOFF_R9(%rsp) 444 movq 0x30(%rbx), %rcx 445 movq %rcx, REGOFF_RAX(%rsp) 446 movq 0x38(%rbx), %rcx 447 movq %rcx, REGOFF_RBX(%rsp) 448 movq 0x58(%rbx), %rcx 449 450 movq %rcx, REGOFF_RBP(%rsp) 451 movq 0x40(%rbx), %rcx 452 movq %rcx, REGOFF_R10(%rsp) 453 movq 0x48(%rbx), %rcx 454 movq %rcx, REGOFF_R11(%rsp) 455 movq %r12, REGOFF_R12(%rsp) 456 457 movq %r13, REGOFF_R13(%rsp) 458 movq %r14, REGOFF_R14(%rsp) 459 movq %r15, REGOFF_R15(%rsp) 460 461 xorl %ecx, %ecx 462 movw %ds, %cx 463 movq %rcx, REGOFF_DS(%rsp) 464 movw %es, %cx 465 movq %rcx, REGOFF_ES(%rsp) 466 movw %fs, %cx 467 movq %rcx, REGOFF_FS(%rsp) 468 movw %gs, %cx 469 movq %rcx, REGOFF_GS(%rsp) 470 471 movq $0, REGOFF_TRAPNO(%rsp) 472 473 movq $0, REGOFF_ERR(%rsp) 474 leaq vpanic(%rip), %rcx 475 movq %rcx, REGOFF_RIP(%rsp) 476 movw %cs, %cx 477 movzwq %cx, %rcx 478 movq %rcx, REGOFF_CS(%rsp) 479 movq 0x50(%rbx), %rcx 480 movq %rcx, REGOFF_RFL(%rsp) 481 movq %rbx, %rcx 482 addq $0x60, %rcx 483 movq %rcx, REGOFF_RSP(%rsp) 484 movw %ss, %cx 485 movzwq %cx, %rcx 486 movq %rcx, REGOFF_SS(%rsp) 487 488 /* 489 * panicsys(format, alist, rp, on_panic_stack) 490 */ 491 movq REGOFF_RDI(%rsp), %rdi /* format */ 492 movq REGOFF_RSI(%rsp), %rsi /* alist */ 493 movq %rsp, %rdx /* struct regs */ 494 movl %r11d, %ecx /* on_panic_stack */ 495 call panicsys 496 addq $REGSIZE, %rsp 497#endif 498 popq %rdi 499 popq %rsi 500 popq %rdx 501 popq %rcx 502 popq %r8 503 popq %r9 504 popq %rax 505 popq %rbx 506 popq %r10 507 popq %r11 508 popfq 509 leave 510 ret 511 END(vpanic) 512 513/* 514void 515dtrace_vpanic(const char *format, va_list alist) 516*/ 517 ENTRY(dtrace_vpanic) /* Initial stack layout: */ 518 519 pushq %rbp /* | %rip | 0x60 */ 520 movq %rsp, %rbp /* | %rbp | 0x58 */ 521 pushfq /* | rfl | 0x50 */ 522 pushq %r11 /* | %r11 | 0x48 */ 523 pushq %r10 /* | %r10 | 0x40 */ 524 pushq %rbx /* | %rbx | 0x38 */ 525 pushq %rax /* | %rax | 0x30 */ 526 pushq %r9 /* | %r9 | 0x28 */ 527 pushq %r8 /* | %r8 | 0x20 */ 528 pushq %rcx /* | %rcx | 0x18 */ 529 pushq %rdx /* | %rdx | 0x10 */ 530 pushq %rsi /* | %rsi | 0x8 alist */ 531 pushq %rdi /* | %rdi | 0x0 format */ 532 533 movq %rsp, %rbx /* %rbx = current %rsp */ 534 535 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 536 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 537 jmp vpanic_common 538 539 END(dtrace_vpanic) 540 541/* 542int 543panic_trigger(int *tp) 544*/ 545 ENTRY(panic_trigger) 546 xorl %eax, %eax 547 movl $0xdefacedd, %edx 548 lock 549 xchgl %edx, (%rdi) 550 cmpl $0, %edx 551 je 0f 552 movl $0, %eax 553 ret 5540: movl $1, %eax 555 ret 556 END(panic_trigger) 557 558/* 559int 560dtrace_panic_trigger(int *tp) 561*/ 562 ENTRY(dtrace_panic_trigger) 563 xorl %eax, %eax 564 movl $0xdefacedd, %edx 565 lock 566 xchgl %edx, (%rdi) 567 cmpl $0, %edx 568 je 0f 569 movl $0, %eax 570 ret 5710: movl $1, %eax 572 ret 573 END(dtrace_panic_trigger) 574