1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2014 by Delphix. All rights reserved. 26 * Copyright 2019 Joyent, Inc. 27 */ 28 29/* 30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. 31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T 32 * All Rights Reserved 33 */ 34 35/* 36 * Copyright (c) 2009, Intel Corporation. 37 * All rights reserved. 38 */ 39 40/* 41 * General assembly language routines. 42 * It is the intent of this file to contain routines that are 43 * independent of the specific kernel architecture, and those that are 44 * common across kernel architectures. 45 * As architectures diverge, and implementations of specific 46 * architecture-dependent routines change, the routines should be moved 47 * from this file into the respective ../`arch -k`/subr.s file. 48 */ 49 50#include <sys/asm_linkage.h> 51#include <sys/asm_misc.h> 52#include <sys/panic.h> 53#include <sys/ontrap.h> 54#include <sys/regset.h> 55#include <sys/privregs.h> 56#include <sys/reboot.h> 57#include <sys/psw.h> 58#include <sys/x86_archext.h> 59 60#include "assym.h" 61#include <sys/dditypes.h> 62 63/* 64 * on_fault() 65 * 66 * Catch lofault faults. Like setjmp except it returns one 67 * if code following causes uncorrectable fault. Turned off 68 * by calling no_fault(). Note that while under on_fault(), 69 * SMAP is disabled. For more information see 70 * uts/intel/ml/copy.s. 71 */ 72 73 ENTRY(on_fault) 74 movq %gs:CPU_THREAD, %rsi 75 leaq catch_fault(%rip), %rdx 76 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */ 77 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */ 78 call smap_disable /* allow user accesses */ 79 jmp setjmp /* let setjmp do the rest */ 80 81catch_fault: 82 movq %gs:CPU_THREAD, %rsi 83 movq T_ONFAULT(%rsi), %rdi /* address of save area */ 84 xorl %eax, %eax 85 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 86 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 87 call smap_enable /* disallow user accesses */ 88 jmp longjmp /* let longjmp do the rest */ 89 SET_SIZE(on_fault) 90 91 ENTRY(no_fault) 92 movq %gs:CPU_THREAD, %rsi 93 xorl %eax, %eax 94 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 95 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 96 call smap_enable /* disallow user accesses */ 97 ret 98 SET_SIZE(no_fault) 99 100/* 101 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just 102 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called. 103 */ 104 105 ENTRY(on_trap_trampoline) 106 movq %gs:CPU_THREAD, %rsi 107 movq T_ONTRAP(%rsi), %rdi 108 addq $OT_JMPBUF, %rdi 109 jmp longjmp 110 SET_SIZE(on_trap_trampoline) 111 112/* 113 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 114 * more information about the on_trap() mechanism. If the on_trap_data is the 115 * same as the topmost stack element, we just modify that element. 116 */ 117 118 ENTRY(on_trap) 119 movw %si, OT_PROT(%rdi) /* ot_prot = prot */ 120 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */ 121 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */ 122 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */ 123 xorl %ecx, %ecx 124 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */ 125 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */ 126 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */ 127 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */ 128 cmpq %rdi, %rcx /* if (otp == %rcx) */ 129 je 0f /* don't modify t_ontrap */ 130 131 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */ 132 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */ 133 1340: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */ 135 jmp setjmp 136 SET_SIZE(on_trap) 137 138/* 139 * Setjmp and longjmp implement non-local gotos using state vectors 140 * type label_t. 141 */ 142 143#if LABEL_PC != 0 144#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded 145#endif /* LABEL_PC != 0 */ 146 147 ENTRY(setjmp) 148 movq %rsp, LABEL_SP(%rdi) 149 movq %rbp, LABEL_RBP(%rdi) 150 movq %rbx, LABEL_RBX(%rdi) 151 movq %r12, LABEL_R12(%rdi) 152 movq %r13, LABEL_R13(%rdi) 153 movq %r14, LABEL_R14(%rdi) 154 movq %r15, LABEL_R15(%rdi) 155 movq (%rsp), %rdx /* return address */ 156 movq %rdx, (%rdi) /* LABEL_PC is 0 */ 157 xorl %eax, %eax /* return 0 */ 158 ret 159 SET_SIZE(setjmp) 160 161 ENTRY(longjmp) 162 movq LABEL_SP(%rdi), %rsp 163 movq LABEL_RBP(%rdi), %rbp 164 movq LABEL_RBX(%rdi), %rbx 165 movq LABEL_R12(%rdi), %r12 166 movq LABEL_R13(%rdi), %r13 167 movq LABEL_R14(%rdi), %r14 168 movq LABEL_R15(%rdi), %r15 169 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */ 170 movq %rdx, (%rsp) 171 xorl %eax, %eax 172 incl %eax /* return 1 */ 173 ret 174 SET_SIZE(longjmp) 175 176/* 177 * if a() calls b() calls caller(), 178 * caller() returns return address in a(). 179 * (Note: We assume a() and b() are C routines which do the normal entry/exit 180 * sequence.) 181 */ 182 183 ENTRY(caller) 184 movq 8(%rbp), %rax /* b()'s return pc, in a() */ 185 ret 186 SET_SIZE(caller) 187 188/* 189 * if a() calls callee(), callee() returns the 190 * return address in a(); 191 */ 192 193 ENTRY(callee) 194 movq (%rsp), %rax /* callee()'s return pc, in a() */ 195 ret 196 SET_SIZE(callee) 197 198/* 199 * return the current frame pointer 200 */ 201 202 ENTRY(getfp) 203 movq %rbp, %rax 204 ret 205 SET_SIZE(getfp) 206 207/* 208 * Invalidate a single page table entry in the TLB 209 */ 210 211 ENTRY(mmu_invlpg) 212 invlpg (%rdi) 213 ret 214 SET_SIZE(mmu_invlpg) 215 216 217/* 218 * Get/Set the value of various control registers 219 */ 220 221 ENTRY(getcr0) 222 movq %cr0, %rax 223 ret 224 SET_SIZE(getcr0) 225 226 ENTRY(setcr0) 227 movq %rdi, %cr0 228 ret 229 SET_SIZE(setcr0) 230 231 ENTRY(getcr2) 232#if defined(__xpv) 233 movq %gs:CPU_VCPU_INFO, %rax 234 movq VCPU_INFO_ARCH_CR2(%rax), %rax 235#else 236 movq %cr2, %rax 237#endif 238 ret 239 SET_SIZE(getcr2) 240 241 ENTRY(getcr3) 242 movq %cr3, %rax 243 ret 244 SET_SIZE(getcr3) 245 246#if !defined(__xpv) 247 248 ENTRY(setcr3) 249 movq %rdi, %cr3 250 ret 251 SET_SIZE(setcr3) 252 253 ENTRY(reload_cr3) 254 movq %cr3, %rdi 255 movq %rdi, %cr3 256 ret 257 SET_SIZE(reload_cr3) 258 259#endif /* __xpv */ 260 261 ENTRY(getcr4) 262 movq %cr4, %rax 263 ret 264 SET_SIZE(getcr4) 265 266 ENTRY(setcr4) 267 movq %rdi, %cr4 268 ret 269 SET_SIZE(setcr4) 270 271 ENTRY(getcr8) 272 movq %cr8, %rax 273 ret 274 SET_SIZE(getcr8) 275 276 ENTRY(setcr8) 277 movq %rdi, %cr8 278 ret 279 SET_SIZE(setcr8) 280 281 ENTRY(__cpuid_insn) 282 movq %rbx, %r8 283 movq %rcx, %r9 284 movq %rdx, %r11 285 movl (%rdi), %eax /* %eax = regs->cp_eax */ 286 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */ 287 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */ 288 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */ 289 cpuid 290 movl %eax, (%rdi) /* regs->cp_eax = %eax */ 291 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */ 292 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */ 293 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */ 294 movq %r8, %rbx 295 movq %r9, %rcx 296 movq %r11, %rdx 297 ret 298 SET_SIZE(__cpuid_insn) 299 300 ENTRY_NP(i86_monitor) 301 pushq %rbp 302 movq %rsp, %rbp 303 movq %rdi, %rax /* addr */ 304 movq %rsi, %rcx /* extensions */ 305 /* rdx contains input arg3: hints */ 306 clflush (%rax) 307 .byte 0x0f, 0x01, 0xc8 /* monitor */ 308 leave 309 ret 310 SET_SIZE(i86_monitor) 311 312 ENTRY_NP(i86_mwait) 313 pushq %rbp 314 call x86_md_clear 315 movq %rsp, %rbp 316 movq %rdi, %rax /* data */ 317 movq %rsi, %rcx /* extensions */ 318 .byte 0x0f, 0x01, 0xc9 /* mwait */ 319 leave 320 ret 321 SET_SIZE(i86_mwait) 322 323#if defined(__xpv) 324 /* 325 * Defined in C 326 */ 327#else 328 329 ENTRY_NP(tsc_read) 330 movq %rbx, %r11 331 movl $0, %eax 332 cpuid 333 rdtsc 334 movq %r11, %rbx 335 shlq $32, %rdx 336 orq %rdx, %rax 337 ret 338 .globl _tsc_mfence_start 339_tsc_mfence_start: 340 mfence 341 rdtsc 342 shlq $32, %rdx 343 orq %rdx, %rax 344 ret 345 .globl _tsc_mfence_end 346_tsc_mfence_end: 347 .globl _tscp_start 348_tscp_start: 349 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 350 shlq $32, %rdx 351 orq %rdx, %rax 352 ret 353 .globl _tscp_end 354_tscp_end: 355 .globl _no_rdtsc_start 356_no_rdtsc_start: 357 xorl %edx, %edx 358 xorl %eax, %eax 359 ret 360 .globl _no_rdtsc_end 361_no_rdtsc_end: 362 .globl _tsc_lfence_start 363_tsc_lfence_start: 364 lfence 365 rdtsc 366 shlq $32, %rdx 367 orq %rdx, %rax 368 ret 369 .globl _tsc_lfence_end 370_tsc_lfence_end: 371 SET_SIZE(tsc_read) 372 373 374#endif /* __xpv */ 375 376 ENTRY_NP(randtick) 377 rdtsc 378 shlq $32, %rdx 379 orq %rdx, %rax 380 ret 381 SET_SIZE(randtick) 382/* 383 * Insert entryp after predp in a doubly linked list. 384 */ 385 386 ENTRY(_insque) 387 movq (%rsi), %rax /* predp->forw */ 388 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */ 389 movq %rax, (%rdi) /* entryp->forw = predp->forw */ 390 movq %rdi, (%rsi) /* predp->forw = entryp */ 391 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */ 392 ret 393 SET_SIZE(_insque) 394 395/* 396 * Remove entryp from a doubly linked list 397 */ 398 399 ENTRY(_remque) 400 movq (%rdi), %rax /* entry->forw */ 401 movq CPTRSIZE(%rdi), %rdx /* entry->back */ 402 movq %rax, (%rdx) /* entry->back->forw = entry->forw */ 403 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */ 404 ret 405 SET_SIZE(_remque) 406 407/* 408 * Returns the number of 409 * non-NULL bytes in string argument. 410 */ 411 412/* 413 * This is close to a simple transliteration of a C version of this 414 * routine. We should either just -make- this be a C version, or 415 * justify having it in assembler by making it significantly faster. 416 * 417 * size_t 418 * strlen(const char *s) 419 * { 420 * const char *s0; 421 * #if defined(DEBUG) 422 * if ((uintptr_t)s < KERNELBASE) 423 * panic(.str_panic_msg); 424 * #endif 425 * for (s0 = s; *s; s++) 426 * ; 427 * return (s - s0); 428 * } 429 */ 430 431 ENTRY(strlen) 432#ifdef DEBUG 433 movq postbootkernelbase(%rip), %rax 434 cmpq %rax, %rdi 435 jae str_valid 436 pushq %rbp 437 movq %rsp, %rbp 438 leaq .str_panic_msg(%rip), %rdi 439 xorl %eax, %eax 440 call panic 441#endif /* DEBUG */ 442str_valid: 443 cmpb $0, (%rdi) 444 movq %rdi, %rax 445 je .null_found 446 .align 4 447.strlen_loop: 448 incq %rdi 449 cmpb $0, (%rdi) 450 jne .strlen_loop 451.null_found: 452 subq %rax, %rdi 453 movq %rdi, %rax 454 ret 455 SET_SIZE(strlen) 456 457#ifdef DEBUG 458 .text 459.str_panic_msg: 460 .string "strlen: argument below kernelbase" 461#endif /* DEBUG */ 462 463 /* 464 * Berkeley 4.3 introduced symbolically named interrupt levels 465 * as a way deal with priority in a machine independent fashion. 466 * Numbered priorities are machine specific, and should be 467 * discouraged where possible. 468 * 469 * Note, for the machine specific priorities there are 470 * examples listed for devices that use a particular priority. 471 * It should not be construed that all devices of that 472 * type should be at that priority. It is currently were 473 * the current devices fit into the priority scheme based 474 * upon time criticalness. 475 * 476 * The underlying assumption of these assignments is that 477 * IPL 10 is the highest level from which a device 478 * routine can call wakeup. Devices that interrupt from higher 479 * levels are restricted in what they can do. If they need 480 * kernels services they should schedule a routine at a lower 481 * level (via software interrupt) to do the required 482 * processing. 483 * 484 * Examples of this higher usage: 485 * Level Usage 486 * 14 Profiling clock (and PROM uart polling clock) 487 * 12 Serial ports 488 * 489 * The serial ports request lower level processing on level 6. 490 * 491 * Also, almost all splN routines (where N is a number or a 492 * mnemonic) will do a RAISE(), on the assumption that they are 493 * never used to lower our priority. 494 * The exceptions are: 495 * spl8() Because you can't be above 15 to begin with! 496 * splzs() Because this is used at boot time to lower our 497 * priority, to allow the PROM to poll the uart. 498 * spl0() Used to lower priority to 0. 499 */ 500 501#define SETPRI(level) \ 502 movl $##level, %edi; /* new priority */ \ 503 jmp do_splx /* redirect to do_splx */ 504 505#define RAISE(level) \ 506 movl $##level, %edi; /* new priority */ \ 507 jmp splr /* redirect to splr */ 508 509 /* locks out all interrupts, including memory errors */ 510 ENTRY(spl8) 511 SETPRI(15) 512 SET_SIZE(spl8) 513 514 /* just below the level that profiling runs */ 515 ENTRY(spl7) 516 RAISE(13) 517 SET_SIZE(spl7) 518 519 /* sun specific - highest priority onboard serial i/o asy ports */ 520 ENTRY(splzs) 521 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */ 522 SET_SIZE(splzs) 523 524 ENTRY(splhi) 525 ALTENTRY(splhigh) 526 ALTENTRY(spl6) 527 ALTENTRY(i_ddi_splhigh) 528 529 RAISE(DISP_LEVEL) 530 531 SET_SIZE(i_ddi_splhigh) 532 SET_SIZE(spl6) 533 SET_SIZE(splhigh) 534 SET_SIZE(splhi) 535 536 /* allow all interrupts */ 537 ENTRY(spl0) 538 SETPRI(0) 539 SET_SIZE(spl0) 540 541 542 /* splx implementation */ 543 ENTRY(splx) 544 jmp do_splx /* redirect to common splx code */ 545 SET_SIZE(splx) 546 547 ENTRY(wait_500ms) 548 pushq %rbx 549 movl $50000, %ebx 5501: 551 call tenmicrosec 552 decl %ebx 553 jnz 1b 554 popq %rbx 555 ret 556 SET_SIZE(wait_500ms) 557 558#define RESET_METHOD_KBC 1 559#define RESET_METHOD_PORT92 2 560#define RESET_METHOD_PCI 4 561 562 DGDEF3(pc_reset_methods, 4, 8) 563 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI; 564 565 ENTRY(pc_reset) 566 567 testl $RESET_METHOD_KBC, pc_reset_methods(%rip) 568 jz 1f 569 570 / 571 / Try the classic keyboard controller-triggered reset. 572 / 573 movw $0x64, %dx 574 movb $0xfe, %al 575 outb (%dx) 576 577 / Wait up to 500 milliseconds here for the keyboard controller 578 / to pull the reset line. On some systems where the keyboard 579 / controller is slow to pull the reset line, the next reset method 580 / may be executed (which may be bad if those systems hang when the 581 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92), 582 / and Ferrari 4000 (doesn't like the cf9 reset method)) 583 584 call wait_500ms 585 5861: 587 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip) 588 jz 3f 589 590 / 591 / Try port 0x92 fast reset 592 / 593 movw $0x92, %dx 594 inb (%dx) 595 cmpb $0xff, %al / If port's not there, we should get back 0xFF 596 je 1f 597 testb $1, %al / If bit 0 598 jz 2f / is clear, jump to perform the reset 599 andb $0xfe, %al / otherwise, 600 outb (%dx) / clear bit 0 first, then 6012: 602 orb $1, %al / Set bit 0 603 outb (%dx) / and reset the system 6041: 605 606 call wait_500ms 607 6083: 609 testl $RESET_METHOD_PCI, pc_reset_methods(%rip) 610 jz 4f 611 612 / Try the PCI (soft) reset vector (should work on all modern systems, 613 / but has been shown to cause problems on 450NX systems, and some newer 614 / systems (e.g. ATI IXP400-equipped systems)) 615 / When resetting via this method, 2 writes are required. The first 616 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with 617 / power cycle). 618 / The reset occurs on the second write, during bit 2's transition from 619 / 0->1. 620 movw $0xcf9, %dx 621 movb $0x2, %al / Reset mode = hard, no power cycle 622 outb (%dx) 623 movb $0x6, %al 624 outb (%dx) 625 626 call wait_500ms 627 6284: 629 / 630 / port 0xcf9 failed also. Last-ditch effort is to 631 / triple-fault the CPU. 632 / Also, use triple fault for EFI firmware 633 / 634 ENTRY(efi_reset) 635 pushq $0x0 636 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 637 lidt (%rsp) 638 int $0x0 / Trigger interrupt, generate triple-fault 639 640 cli 641 hlt / Wait forever 642 /*NOTREACHED*/ 643 SET_SIZE(efi_reset) 644 SET_SIZE(pc_reset) 645 646/* 647 * C callable in and out routines 648 */ 649 650 ENTRY(outl) 651 movw %di, %dx 652 movl %esi, %eax 653 outl (%dx) 654 ret 655 SET_SIZE(outl) 656 657 ENTRY(outw) 658 movw %di, %dx 659 movw %si, %ax 660 D16 outl (%dx) /* XX64 why not outw? */ 661 ret 662 SET_SIZE(outw) 663 664 ENTRY(outb) 665 movw %di, %dx 666 movb %sil, %al 667 outb (%dx) 668 ret 669 SET_SIZE(outb) 670 671 ENTRY(inl) 672 xorl %eax, %eax 673 movw %di, %dx 674 inl (%dx) 675 ret 676 SET_SIZE(inl) 677 678 ENTRY(inw) 679 xorl %eax, %eax 680 movw %di, %dx 681 D16 inl (%dx) 682 ret 683 SET_SIZE(inw) 684 685 686 ENTRY(inb) 687 xorl %eax, %eax 688 movw %di, %dx 689 inb (%dx) 690 ret 691 SET_SIZE(inb) 692 693/* 694 * void int3(void) 695 * void int18(void) 696 * void int20(void) 697 * void int_cmci(void) 698 */ 699 700 ENTRY(int3) 701 int $T_BPTFLT 702 ret 703 SET_SIZE(int3) 704 705 ENTRY(int18) 706 int $T_MCE 707 ret 708 SET_SIZE(int18) 709 710 ENTRY(int20) 711 movl boothowto, %eax 712 andl $RB_DEBUG, %eax 713 jz 1f 714 715 int $T_DBGENTR 7161: 717 rep; ret /* use 2 byte return instruction when branch target */ 718 /* AMD Software Optimization Guide - Section 6.2 */ 719 SET_SIZE(int20) 720 721 ENTRY(int_cmci) 722 int $T_ENOEXTFLT 723 ret 724 SET_SIZE(int_cmci) 725 726 ENTRY(scanc) 727 /* rdi == size */ 728 /* rsi == cp */ 729 /* rdx == table */ 730 /* rcx == mask */ 731 addq %rsi, %rdi /* end = &cp[size] */ 732.scanloop: 733 cmpq %rdi, %rsi /* while (cp < end */ 734 jnb .scandone 735 movzbq (%rsi), %r8 /* %r8 = *cp */ 736 incq %rsi /* cp++ */ 737 testb %cl, (%r8, %rdx) 738 jz .scanloop /* && (table[*cp] & mask) == 0) */ 739 decq %rsi /* (fix post-increment) */ 740.scandone: 741 movl %edi, %eax 742 subl %esi, %eax /* return (end - cp) */ 743 ret 744 SET_SIZE(scanc) 745 746/* 747 * Replacement functions for ones that are normally inlined. 748 * In addition to the inline copies, they are defined here just in case. 749 */ 750 751 ENTRY(intr_clear) 752 ENTRY(clear_int_flag) 753 pushfq 754 popq %rax 755#if defined(__xpv) 756 leaq xpv_panicking, %rdi 757 movl (%rdi), %edi 758 cmpl $0, %edi 759 jne 2f 760 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 761 /* 762 * Synthesize the PS_IE bit from the event mask bit 763 */ 764 andq $_BITNOT(PS_IE), %rax 765 testb $1, %dl 766 jnz 1f 767 orq $PS_IE, %rax 7681: 769 ret 7702: 771#endif 772 CLI(%rdi) 773 ret 774 SET_SIZE(clear_int_flag) 775 SET_SIZE(intr_clear) 776 777 ENTRY(curcpup) 778 movq %gs:CPU_SELF, %rax 779 ret 780 SET_SIZE(curcpup) 781 782/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs() 783 * These functions reverse the byte order of the input parameter and returns 784 * the result. This is to convert the byte order from host byte order 785 * (little endian) to network byte order (big endian), or vice versa. 786 */ 787 788 ENTRY(htonll) 789 ALTENTRY(ntohll) 790 movq %rdi, %rax 791 bswapq %rax 792 ret 793 SET_SIZE(ntohll) 794 SET_SIZE(htonll) 795 796 /* XX64 there must be shorter sequences for this */ 797 ENTRY(htonl) 798 ALTENTRY(ntohl) 799 movl %edi, %eax 800 bswap %eax 801 ret 802 SET_SIZE(ntohl) 803 SET_SIZE(htonl) 804 805 /* XX64 there must be better sequences for this */ 806 ENTRY(htons) 807 ALTENTRY(ntohs) 808 movl %edi, %eax 809 bswap %eax 810 shrl $16, %eax 811 ret 812 SET_SIZE(ntohs) 813 SET_SIZE(htons) 814 815 816 ENTRY(intr_restore) 817 ENTRY(restore_int_flag) 818 testq $PS_IE, %rdi 819 jz 1f 820#if defined(__xpv) 821 leaq xpv_panicking, %rsi 822 movl (%rsi), %esi 823 cmpl $0, %esi 824 jne 1f 825 /* 826 * Since we're -really- running unprivileged, our attempt 827 * to change the state of the IF bit will be ignored. 828 * The virtual IF bit is tweaked by CLI and STI. 829 */ 830 IE_TO_EVENT_MASK(%rsi, %rdi) 831#else 832 sti 833#endif 8341: 835 ret 836 SET_SIZE(restore_int_flag) 837 SET_SIZE(intr_restore) 838 839 ENTRY(sti) 840 STI 841 ret 842 SET_SIZE(sti) 843 844 ENTRY(cli) 845 CLI(%rax) 846 ret 847 SET_SIZE(cli) 848 849 ENTRY(dtrace_interrupt_disable) 850 pushfq 851 popq %rax 852#if defined(__xpv) 853 leaq xpv_panicking, %rdi 854 movl (%rdi), %edi 855 cmpl $0, %edi 856 jne .dtrace_interrupt_disable_done 857 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 858 /* 859 * Synthesize the PS_IE bit from the event mask bit 860 */ 861 andq $_BITNOT(PS_IE), %rax 862 testb $1, %dl 863 jnz .dtrace_interrupt_disable_done 864 orq $PS_IE, %rax 865#else 866 CLI(%rdx) 867#endif 868.dtrace_interrupt_disable_done: 869 ret 870 SET_SIZE(dtrace_interrupt_disable) 871 872 ENTRY(dtrace_interrupt_enable) 873 pushq %rdi 874 popfq 875#if defined(__xpv) 876 leaq xpv_panicking, %rdx 877 movl (%rdx), %edx 878 cmpl $0, %edx 879 jne .dtrace_interrupt_enable_done 880 /* 881 * Since we're -really- running unprivileged, our attempt 882 * to change the state of the IF bit will be ignored. The 883 * virtual IF bit is tweaked by CLI and STI. 884 */ 885 IE_TO_EVENT_MASK(%rdx, %rdi) 886#endif 887.dtrace_interrupt_enable_done: 888 ret 889 SET_SIZE(dtrace_interrupt_enable) 890 891 892 ENTRY(dtrace_membar_producer) 893 rep; ret /* use 2 byte return instruction when branch target */ 894 /* AMD Software Optimization Guide - Section 6.2 */ 895 SET_SIZE(dtrace_membar_producer) 896 897 ENTRY(dtrace_membar_consumer) 898 rep; ret /* use 2 byte return instruction when branch target */ 899 /* AMD Software Optimization Guide - Section 6.2 */ 900 SET_SIZE(dtrace_membar_consumer) 901 902 ENTRY(threadp) 903 movq %gs:CPU_THREAD, %rax 904 ret 905 SET_SIZE(threadp) 906 907/* 908 * Checksum routine for Internet Protocol Headers 909 */ 910 911 ENTRY(ip_ocsum) 912 pushq %rbp 913 movq %rsp, %rbp 914#ifdef DEBUG 915 movq postbootkernelbase(%rip), %rax 916 cmpq %rax, %rdi 917 jnb 1f 918 xorl %eax, %eax 919 movq %rdi, %rsi 920 leaq .ip_ocsum_panic_msg(%rip), %rdi 921 call panic 922 /*NOTREACHED*/ 923.ip_ocsum_panic_msg: 924 .string "ip_ocsum: address 0x%p below kernelbase\n" 9251: 926#endif 927 movl %esi, %ecx /* halfword_count */ 928 movq %rdi, %rsi /* address */ 929 /* partial sum in %edx */ 930 xorl %eax, %eax 931 testl %ecx, %ecx 932 jz .ip_ocsum_done 933 testq $3, %rsi 934 jnz .ip_csum_notaligned 935.ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */ 936.next_iter: 937 /* XX64 opportunities for prefetch? */ 938 /* XX64 compute csum with 64 bit quantities? */ 939 subl $32, %ecx 940 jl .less_than_32 941 942 addl 0(%rsi), %edx 943.only60: 944 adcl 4(%rsi), %eax 945.only56: 946 adcl 8(%rsi), %edx 947.only52: 948 adcl 12(%rsi), %eax 949.only48: 950 adcl 16(%rsi), %edx 951.only44: 952 adcl 20(%rsi), %eax 953.only40: 954 adcl 24(%rsi), %edx 955.only36: 956 adcl 28(%rsi), %eax 957.only32: 958 adcl 32(%rsi), %edx 959.only28: 960 adcl 36(%rsi), %eax 961.only24: 962 adcl 40(%rsi), %edx 963.only20: 964 adcl 44(%rsi), %eax 965.only16: 966 adcl 48(%rsi), %edx 967.only12: 968 adcl 52(%rsi), %eax 969.only8: 970 adcl 56(%rsi), %edx 971.only4: 972 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */ 973.only0: 974 adcl $0, %eax /* could be adding -1 in eax with a carry */ 975 adcl $0, %eax 976 977 addq $64, %rsi 978 testl %ecx, %ecx 979 jnz .next_iter 980 981.ip_ocsum_done: 982 addl %eax, %edx 983 adcl $0, %edx 984 movl %edx, %eax /* form a 16 bit checksum by */ 985 shrl $16, %eax /* adding two halves of 32 bit checksum */ 986 addw %dx, %ax 987 adcw $0, %ax 988 andl $0xffff, %eax 989 leave 990 ret 991 992.ip_csum_notaligned: 993 xorl %edi, %edi 994 movw (%rsi), %di 995 addl %edi, %edx 996 adcl $0, %edx 997 addq $2, %rsi 998 decl %ecx 999 jmp .ip_csum_aligned 1000 1001.less_than_32: 1002 addl $32, %ecx 1003 testl $1, %ecx 1004 jz .size_aligned 1005 andl $0xfe, %ecx 1006 movzwl (%rsi, %rcx, 2), %edi 1007 addl %edi, %edx 1008 adcl $0, %edx 1009.size_aligned: 1010 movl %ecx, %edi 1011 shrl $1, %ecx 1012 shl $1, %edi 1013 subq $64, %rdi 1014 addq %rdi, %rsi 1015 leaq .ip_ocsum_jmptbl(%rip), %rdi 1016 leaq (%rdi, %rcx, 8), %rdi 1017 xorl %ecx, %ecx 1018 clc 1019 movq (%rdi), %rdi 1020 INDIRECT_JMP_REG(rdi) 1021 1022 .align 8 1023.ip_ocsum_jmptbl: 1024 .quad .only0, .only4, .only8, .only12, .only16, .only20 1025 .quad .only24, .only28, .only32, .only36, .only40, .only44 1026 .quad .only48, .only52, .only56, .only60 1027 SET_SIZE(ip_ocsum) 1028 1029/* 1030 * multiply two long numbers and yield a u_longlong_t result, callable from C. 1031 * Provided to manipulate hrtime_t values. 1032 */ 1033 1034 ENTRY(mul32) 1035 xorl %edx, %edx /* XX64 joe, paranoia? */ 1036 movl %edi, %eax 1037 mull %esi 1038 shlq $32, %rdx 1039 orq %rdx, %rax 1040 ret 1041 SET_SIZE(mul32) 1042 1043 ENTRY(scan_memory) 1044 shrq $3, %rsi /* convert %rsi from byte to quadword count */ 1045 jz .scanm_done 1046 movq %rsi, %rcx /* move count into rep control register */ 1047 movq %rdi, %rsi /* move addr into lodsq control reg. */ 1048 rep lodsq /* scan the memory range */ 1049.scanm_done: 1050 rep; ret /* use 2 byte return instruction when branch target */ 1051 /* AMD Software Optimization Guide - Section 6.2 */ 1052 SET_SIZE(scan_memory) 1053 1054 1055 ENTRY(lowbit) 1056 movl $-1, %eax 1057 bsfq %rdi, %rdi 1058 cmovnz %edi, %eax 1059 incl %eax 1060 ret 1061 SET_SIZE(lowbit) 1062 1063 ENTRY(highbit) 1064 ALTENTRY(highbit64) 1065 movl $-1, %eax 1066 bsrq %rdi, %rdi 1067 cmovnz %edi, %eax 1068 incl %eax 1069 ret 1070 SET_SIZE(highbit64) 1071 SET_SIZE(highbit) 1072 1073#define XMSR_ACCESS_VAL $0x9c5a203a 1074 1075 ENTRY(rdmsr) 1076 movl %edi, %ecx 1077 rdmsr 1078 shlq $32, %rdx 1079 orq %rdx, %rax 1080 ret 1081 SET_SIZE(rdmsr) 1082 1083 ENTRY(wrmsr) 1084 movq %rsi, %rdx 1085 shrq $32, %rdx 1086 movl %esi, %eax 1087 movl %edi, %ecx 1088 wrmsr 1089 ret 1090 SET_SIZE(wrmsr) 1091 1092 ENTRY(xrdmsr) 1093 pushq %rbp 1094 movq %rsp, %rbp 1095 movl %edi, %ecx 1096 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 1097 rdmsr 1098 shlq $32, %rdx 1099 orq %rdx, %rax 1100 leave 1101 ret 1102 SET_SIZE(xrdmsr) 1103 1104 ENTRY(xwrmsr) 1105 pushq %rbp 1106 movq %rsp, %rbp 1107 movl %edi, %ecx 1108 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 1109 movq %rsi, %rdx 1110 shrq $32, %rdx 1111 movl %esi, %eax 1112 wrmsr 1113 leave 1114 ret 1115 SET_SIZE(xwrmsr) 1116 1117 ENTRY(get_xcr) 1118 movl %edi, %ecx 1119 #xgetbv 1120 .byte 0x0f,0x01,0xd0 1121 shlq $32, %rdx 1122 orq %rdx, %rax 1123 ret 1124 SET_SIZE(get_xcr) 1125 1126 ENTRY(set_xcr) 1127 movq %rsi, %rdx 1128 shrq $32, %rdx 1129 movl %esi, %eax 1130 movl %edi, %ecx 1131 #xsetbv 1132 .byte 0x0f,0x01,0xd1 1133 ret 1134 SET_SIZE(set_xcr) 1135 1136 ENTRY(invalidate_cache) 1137 wbinvd 1138 ret 1139 SET_SIZE(invalidate_cache) 1140 1141 ENTRY_NP(getcregs) 1142#if defined(__xpv) 1143 /* 1144 * Only a few of the hardware control registers or descriptor tables 1145 * are directly accessible to us, so just zero the structure. 1146 * 1147 * XXPV Perhaps it would be helpful for the hypervisor to return 1148 * virtualized versions of these for post-mortem use. 1149 * (Need to reevaluate - perhaps it already does!) 1150 */ 1151 pushq %rdi /* save *crp */ 1152 movq $CREGSZ, %rsi 1153 call bzero 1154 popq %rdi 1155 1156 /* 1157 * Dump what limited information we can 1158 */ 1159 movq %cr0, %rax 1160 movq %rax, CREG_CR0(%rdi) /* cr0 */ 1161 movq %cr2, %rax 1162 movq %rax, CREG_CR2(%rdi) /* cr2 */ 1163 movq %cr3, %rax 1164 movq %rax, CREG_CR3(%rdi) /* cr3 */ 1165 movq %cr4, %rax 1166 movq %rax, CREG_CR4(%rdi) /* cr4 */ 1167 1168#else /* __xpv */ 1169 1170#define GETMSR(r, off, d) \ 1171 movl $r, %ecx; \ 1172 rdmsr; \ 1173 movl %eax, off(d); \ 1174 movl %edx, off+4(d) 1175 1176 xorl %eax, %eax 1177 movq %rax, CREG_GDT+8(%rdi) 1178 sgdt CREG_GDT(%rdi) /* 10 bytes */ 1179 movq %rax, CREG_IDT+8(%rdi) 1180 sidt CREG_IDT(%rdi) /* 10 bytes */ 1181 movq %rax, CREG_LDT(%rdi) 1182 sldt CREG_LDT(%rdi) /* 2 bytes */ 1183 movq %rax, CREG_TASKR(%rdi) 1184 str CREG_TASKR(%rdi) /* 2 bytes */ 1185 movq %cr0, %rax 1186 movq %rax, CREG_CR0(%rdi) /* cr0 */ 1187 movq %cr2, %rax 1188 movq %rax, CREG_CR2(%rdi) /* cr2 */ 1189 movq %cr3, %rax 1190 movq %rax, CREG_CR3(%rdi) /* cr3 */ 1191 movq %cr4, %rax 1192 movq %rax, CREG_CR4(%rdi) /* cr4 */ 1193 movq %cr8, %rax 1194 movq %rax, CREG_CR8(%rdi) /* cr8 */ 1195 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi) 1196 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi) 1197#endif /* __xpv */ 1198 ret 1199 SET_SIZE(getcregs) 1200 1201#undef GETMSR 1202 1203 1204/* 1205 * A panic trigger is a word which is updated atomically and can only be set 1206 * once. We atomically store 0xDEFACEDD and load the old value. If the 1207 * previous value was 0, we succeed and return 1; otherwise return 0. 1208 * This allows a partially corrupt trigger to still trigger correctly. DTrace 1209 * has its own version of this function to allow it to panic correctly from 1210 * probe context. 1211 */ 1212 1213 ENTRY_NP(panic_trigger) 1214 xorl %eax, %eax 1215 movl $0xdefacedd, %edx 1216 lock 1217 xchgl %edx, (%rdi) 1218 cmpl $0, %edx 1219 je 0f 1220 movl $0, %eax 1221 ret 12220: movl $1, %eax 1223 ret 1224 SET_SIZE(panic_trigger) 1225 1226 ENTRY_NP(dtrace_panic_trigger) 1227 xorl %eax, %eax 1228 movl $0xdefacedd, %edx 1229 lock 1230 xchgl %edx, (%rdi) 1231 cmpl $0, %edx 1232 je 0f 1233 movl $0, %eax 1234 ret 12350: movl $1, %eax 1236 ret 1237 SET_SIZE(dtrace_panic_trigger) 1238 1239/* 1240 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 1241 * into the panic code implemented in panicsys(). vpanic() is responsible 1242 * for passing through the format string and arguments, and constructing a 1243 * regs structure on the stack into which it saves the current register 1244 * values. If we are not dying due to a fatal trap, these registers will 1245 * then be preserved in panicbuf as the current processor state. Before 1246 * invoking panicsys(), vpanic() activates the first panic trigger (see 1247 * common/os/panic.c) and switches to the panic_stack if successful. Note that 1248 * DTrace takes a slightly different panic path if it must panic from probe 1249 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 1250 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 1251 * branches back into vpanic(). 1252 */ 1253 1254 ENTRY_NP(vpanic) /* Initial stack layout: */ 1255 1256 pushq %rbp /* | %rip | 0x60 */ 1257 movq %rsp, %rbp /* | %rbp | 0x58 */ 1258 pushfq /* | rfl | 0x50 */ 1259 pushq %r11 /* | %r11 | 0x48 */ 1260 pushq %r10 /* | %r10 | 0x40 */ 1261 pushq %rbx /* | %rbx | 0x38 */ 1262 pushq %rax /* | %rax | 0x30 */ 1263 pushq %r9 /* | %r9 | 0x28 */ 1264 pushq %r8 /* | %r8 | 0x20 */ 1265 pushq %rcx /* | %rcx | 0x18 */ 1266 pushq %rdx /* | %rdx | 0x10 */ 1267 pushq %rsi /* | %rsi | 0x8 alist */ 1268 pushq %rdi /* | %rdi | 0x0 format */ 1269 1270 movq %rsp, %rbx /* %rbx = current %rsp */ 1271 1272 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 1273 call panic_trigger /* %eax = panic_trigger() */ 1274 1275vpanic_common: 1276 /* 1277 * The panic_trigger result is in %eax from the call above, and 1278 * dtrace_panic places it in %eax before branching here. 1279 * The rdmsr instructions that follow below will clobber %eax so 1280 * we stash the panic_trigger result in %r11d. 1281 */ 1282 movl %eax, %r11d 1283 cmpl $0, %r11d 1284 je 0f 1285 1286 /* 1287 * If panic_trigger() was successful, we are the first to initiate a 1288 * panic: we now switch to the reserved panic_stack before continuing. 1289 */ 1290 leaq panic_stack(%rip), %rsp 1291 addq $PANICSTKSIZE, %rsp 12920: subq $REGSIZE, %rsp 1293 /* 1294 * Now that we've got everything set up, store the register values as 1295 * they were when we entered vpanic() to the designated location in 1296 * the regs structure we allocated on the stack. 1297 */ 1298 movq 0x0(%rbx), %rcx 1299 movq %rcx, REGOFF_RDI(%rsp) 1300 movq 0x8(%rbx), %rcx 1301 movq %rcx, REGOFF_RSI(%rsp) 1302 movq 0x10(%rbx), %rcx 1303 movq %rcx, REGOFF_RDX(%rsp) 1304 movq 0x18(%rbx), %rcx 1305 movq %rcx, REGOFF_RCX(%rsp) 1306 movq 0x20(%rbx), %rcx 1307 1308 movq %rcx, REGOFF_R8(%rsp) 1309 movq 0x28(%rbx), %rcx 1310 movq %rcx, REGOFF_R9(%rsp) 1311 movq 0x30(%rbx), %rcx 1312 movq %rcx, REGOFF_RAX(%rsp) 1313 movq 0x38(%rbx), %rcx 1314 movq %rcx, REGOFF_RBX(%rsp) 1315 movq 0x58(%rbx), %rcx 1316 1317 movq %rcx, REGOFF_RBP(%rsp) 1318 movq 0x40(%rbx), %rcx 1319 movq %rcx, REGOFF_R10(%rsp) 1320 movq 0x48(%rbx), %rcx 1321 movq %rcx, REGOFF_R11(%rsp) 1322 movq %r12, REGOFF_R12(%rsp) 1323 1324 movq %r13, REGOFF_R13(%rsp) 1325 movq %r14, REGOFF_R14(%rsp) 1326 movq %r15, REGOFF_R15(%rsp) 1327 1328 xorl %ecx, %ecx 1329 movw %ds, %cx 1330 movq %rcx, REGOFF_DS(%rsp) 1331 movw %es, %cx 1332 movq %rcx, REGOFF_ES(%rsp) 1333 movw %fs, %cx 1334 movq %rcx, REGOFF_FS(%rsp) 1335 movw %gs, %cx 1336 movq %rcx, REGOFF_GS(%rsp) 1337 1338 movq $0, REGOFF_TRAPNO(%rsp) 1339 1340 movq $0, REGOFF_ERR(%rsp) 1341 leaq vpanic(%rip), %rcx 1342 movq %rcx, REGOFF_RIP(%rsp) 1343 movw %cs, %cx 1344 movzwq %cx, %rcx 1345 movq %rcx, REGOFF_CS(%rsp) 1346 movq 0x50(%rbx), %rcx 1347 movq %rcx, REGOFF_RFL(%rsp) 1348 movq %rbx, %rcx 1349 addq $0x60, %rcx 1350 movq %rcx, REGOFF_RSP(%rsp) 1351 movw %ss, %cx 1352 movzwq %cx, %rcx 1353 movq %rcx, REGOFF_SS(%rsp) 1354 1355 /* 1356 * panicsys(format, alist, rp, on_panic_stack) 1357 */ 1358 movq REGOFF_RDI(%rsp), %rdi /* format */ 1359 movq REGOFF_RSI(%rsp), %rsi /* alist */ 1360 movq %rsp, %rdx /* struct regs */ 1361 movl %r11d, %ecx /* on_panic_stack */ 1362 call panicsys 1363 addq $REGSIZE, %rsp 1364 popq %rdi 1365 popq %rsi 1366 popq %rdx 1367 popq %rcx 1368 popq %r8 1369 popq %r9 1370 popq %rax 1371 popq %rbx 1372 popq %r10 1373 popq %r11 1374 popfq 1375 leave 1376 ret 1377 SET_SIZE(vpanic) 1378 1379 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */ 1380 1381 pushq %rbp /* | %rip | 0x60 */ 1382 movq %rsp, %rbp /* | %rbp | 0x58 */ 1383 pushfq /* | rfl | 0x50 */ 1384 pushq %r11 /* | %r11 | 0x48 */ 1385 pushq %r10 /* | %r10 | 0x40 */ 1386 pushq %rbx /* | %rbx | 0x38 */ 1387 pushq %rax /* | %rax | 0x30 */ 1388 pushq %r9 /* | %r9 | 0x28 */ 1389 pushq %r8 /* | %r8 | 0x20 */ 1390 pushq %rcx /* | %rcx | 0x18 */ 1391 pushq %rdx /* | %rdx | 0x10 */ 1392 pushq %rsi /* | %rsi | 0x8 alist */ 1393 pushq %rdi /* | %rdi | 0x0 format */ 1394 1395 movq %rsp, %rbx /* %rbx = current %rsp */ 1396 1397 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 1398 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 1399 jmp vpanic_common 1400 1401 SET_SIZE(dtrace_vpanic) 1402 1403 DGDEF3(timedelta, 8, 8) 1404 .long 0, 0 1405 1406 /* 1407 * initialized to a non zero value to make pc_gethrtime() 1408 * work correctly even before clock is initialized 1409 */ 1410 DGDEF3(hrtime_base, 8, 8) 1411 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0 1412 1413 DGDEF3(adj_shift, 4, 4) 1414 .long ADJ_SHIFT 1415 1416 ENTRY_NP(hres_tick) 1417 pushq %rbp 1418 movq %rsp, %rbp 1419 1420 /* 1421 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 1422 * hres_last_tick can only be modified while holding CLOCK_LOCK). 1423 * At worst, performing this now instead of under CLOCK_LOCK may 1424 * introduce some jitter in pc_gethrestime(). 1425 */ 1426 movq gethrtimef(%rip), %rsi 1427 INDIRECT_CALL_REG(rsi) 1428 movq %rax, %r8 1429 1430 leaq hres_lock(%rip), %rax 1431 movb $-1, %dl 1432.CL1: 1433 xchgb %dl, (%rax) 1434 testb %dl, %dl 1435 jz .CL3 /* got it */ 1436.CL2: 1437 cmpb $0, (%rax) /* possible to get lock? */ 1438 pause 1439 jne .CL2 1440 jmp .CL1 /* yes, try again */ 1441.CL3: 1442 /* 1443 * compute the interval since last time hres_tick was called 1444 * and adjust hrtime_base and hrestime accordingly 1445 * hrtime_base is an 8 byte value (in nsec), hrestime is 1446 * a timestruc_t (sec, nsec) 1447 */ 1448 leaq hres_last_tick(%rip), %rax 1449 movq %r8, %r11 1450 subq (%rax), %r8 1451 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */ 1452 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */ 1453 /* 1454 * Now that we have CLOCK_LOCK, we can update hres_last_tick 1455 */ 1456 movq %r11, (%rax) 1457 1458 call __adj_hrestime 1459 1460 /* 1461 * release the hres_lock 1462 */ 1463 incl hres_lock(%rip) 1464 leave 1465 ret 1466 SET_SIZE(hres_tick) 1467 1468/* 1469 * void prefetch_smap_w(void *) 1470 * 1471 * Prefetch ahead within a linear list of smap structures. 1472 * Not implemented for ia32. Stub for compatibility. 1473 */ 1474 1475 ENTRY(prefetch_smap_w) 1476 rep; ret /* use 2 byte return instruction when branch target */ 1477 /* AMD Software Optimization Guide - Section 6.2 */ 1478 SET_SIZE(prefetch_smap_w) 1479 1480/* 1481 * prefetch_page_r(page_t *) 1482 * issue prefetch instructions for a page_t 1483 */ 1484 1485 ENTRY(prefetch_page_r) 1486 rep; ret /* use 2 byte return instruction when branch target */ 1487 /* AMD Software Optimization Guide - Section 6.2 */ 1488 SET_SIZE(prefetch_page_r) 1489 1490 ENTRY(bcmp) 1491 pushq %rbp 1492 movq %rsp, %rbp 1493#ifdef DEBUG 1494 testq %rdx,%rdx 1495 je 1f 1496 movq postbootkernelbase(%rip), %r11 1497 cmpq %r11, %rdi 1498 jb 0f 1499 cmpq %r11, %rsi 1500 jnb 1f 15010: leaq .bcmp_panic_msg(%rip), %rdi 1502 xorl %eax, %eax 1503 call panic 15041: 1505#endif /* DEBUG */ 1506 call memcmp 1507 testl %eax, %eax 1508 setne %dl 1509 leave 1510 movzbl %dl, %eax 1511 ret 1512 SET_SIZE(bcmp) 1513 1514#ifdef DEBUG 1515 .text 1516.bcmp_panic_msg: 1517 .string "bcmp: arguments below kernelbase" 1518#endif /* DEBUG */ 1519 1520 ENTRY_NP(bsrw_insn) 1521 xorl %eax, %eax 1522 bsrw %di, %ax 1523 ret 1524 SET_SIZE(bsrw_insn) 1525 1526 ENTRY_NP(switch_sp_and_call) 1527 pushq %rbp 1528 movq %rsp, %rbp /* set up stack frame */ 1529 movq %rdi, %rsp /* switch stack pointer */ 1530 movq %rdx, %rdi /* pass func arg 1 */ 1531 movq %rsi, %r11 /* save function to call */ 1532 movq %rcx, %rsi /* pass func arg 2 */ 1533 INDIRECT_CALL_REG(r11) /* call function */ 1534 leave /* restore stack */ 1535 ret 1536 SET_SIZE(switch_sp_and_call) 1537 1538 ENTRY_NP(kmdb_enter) 1539 pushq %rbp 1540 movq %rsp, %rbp 1541 1542 /* 1543 * Save flags, do a 'cli' then return the saved flags 1544 */ 1545 call intr_clear 1546 1547 int $T_DBGENTR 1548 1549 /* 1550 * Restore the saved flags 1551 */ 1552 movq %rax, %rdi 1553 call intr_restore 1554 1555 leave 1556 ret 1557 SET_SIZE(kmdb_enter) 1558 1559 ENTRY_NP(return_instr) 1560 rep; ret /* use 2 byte instruction when branch target */ 1561 /* AMD Software Optimization Guide - Section 6.2 */ 1562 SET_SIZE(return_instr) 1563 1564 ENTRY(getflags) 1565 pushfq 1566 popq %rax 1567#if defined(__xpv) 1568 CURTHREAD(%rdi) 1569 KPREEMPT_DISABLE(%rdi) 1570 /* 1571 * Synthesize the PS_IE bit from the event mask bit 1572 */ 1573 CURVCPU(%r11) 1574 andq $_BITNOT(PS_IE), %rax 1575 XEN_TEST_UPCALL_MASK(%r11) 1576 jnz 1f 1577 orq $PS_IE, %rax 15781: 1579 KPREEMPT_ENABLE_NOKP(%rdi) 1580#endif 1581 ret 1582 SET_SIZE(getflags) 1583 1584 ENTRY(ftrace_interrupt_disable) 1585 pushfq 1586 popq %rax 1587 CLI(%rdx) 1588 ret 1589 SET_SIZE(ftrace_interrupt_disable) 1590 1591 ENTRY(ftrace_interrupt_enable) 1592 pushq %rdi 1593 popfq 1594 ret 1595 SET_SIZE(ftrace_interrupt_enable) 1596 1597 ENTRY(clflush_insn) 1598 clflush (%rdi) 1599 ret 1600 SET_SIZE(clflush_insn) 1601 1602 ENTRY(mfence_insn) 1603 mfence 1604 ret 1605 SET_SIZE(mfence_insn) 1606 1607/* 1608 * VMware implements an I/O port that programs can query to detect if software 1609 * is running in a VMware hypervisor. This hypervisor port behaves differently 1610 * depending on magic values in certain registers and modifies some registers 1611 * as a side effect. 1612 * 1613 * References: http://kb.vmware.com/kb/1009458 1614 */ 1615 1616 ENTRY(vmware_port) 1617 pushq %rbx 1618 movl $VMWARE_HVMAGIC, %eax 1619 movl $0xffffffff, %ebx 1620 movl %edi, %ecx 1621 movl $VMWARE_HVPORT, %edx 1622 inl (%dx) 1623 movl %eax, (%rsi) 1624 movl %ebx, 4(%rsi) 1625 movl %ecx, 8(%rsi) 1626 movl %edx, 12(%rsi) 1627 popq %rbx 1628 ret 1629 SET_SIZE(vmware_port) 1630