1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 25 */ 26 27/* 28 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. 29 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T 30 * All Rights Reserved 31 */ 32 33/* 34 * Copyright (c) 2009, Intel Corporation. 35 * All rights reserved. 36 */ 37 38/* 39 * General assembly language routines. 40 * It is the intent of this file to contain routines that are 41 * independent of the specific kernel architecture, and those that are 42 * common across kernel architectures. 43 * As architectures diverge, and implementations of specific 44 * architecture-dependent routines change, the routines should be moved 45 * from this file into the respective ../`arch -k`/subr.s file. 46 */ 47 48#include <sys/asm_linkage.h> 49#include <sys/asm_misc.h> 50#include <sys/panic.h> 51#include <sys/ontrap.h> 52#include <sys/regset.h> 53#include <sys/privregs.h> 54#include <sys/reboot.h> 55#include <sys/psw.h> 56#include <sys/x86_archext.h> 57 58#if defined(__lint) 59#include <sys/types.h> 60#include <sys/systm.h> 61#include <sys/thread.h> 62#include <sys/archsystm.h> 63#include <sys/byteorder.h> 64#include <sys/dtrace.h> 65#include <sys/ftrace.h> 66#else /* __lint */ 67#include "assym.h" 68#endif /* __lint */ 69#include <sys/dditypes.h> 70 71/* 72 * on_fault() 73 * Catch lofault faults. Like setjmp except it returns one 74 * if code following causes uncorrectable fault. Turned off 75 * by calling no_fault(). 76 */ 77 78#if defined(__lint) 79 80/* ARGSUSED */ 81int 82on_fault(label_t *ljb) 83{ return (0); } 84 85void 86no_fault(void) 87{} 88 89#else /* __lint */ 90 91#if defined(__amd64) 92 93 ENTRY(on_fault) 94 movq %gs:CPU_THREAD, %rsi 95 leaq catch_fault(%rip), %rdx 96 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */ 97 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */ 98 jmp setjmp /* let setjmp do the rest */ 99 100catch_fault: 101 movq %gs:CPU_THREAD, %rsi 102 movq T_ONFAULT(%rsi), %rdi /* address of save area */ 103 xorl %eax, %eax 104 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 105 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 106 jmp longjmp /* let longjmp do the rest */ 107 SET_SIZE(on_fault) 108 109 ENTRY(no_fault) 110 movq %gs:CPU_THREAD, %rsi 111 xorl %eax, %eax 112 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 113 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 114 ret 115 SET_SIZE(no_fault) 116 117#elif defined(__i386) 118 119 ENTRY(on_fault) 120 movl %gs:CPU_THREAD, %edx 121 movl 4(%esp), %eax /* jumpbuf address */ 122 leal catch_fault, %ecx 123 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */ 124 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */ 125 jmp setjmp /* let setjmp do the rest */ 126 127catch_fault: 128 movl %gs:CPU_THREAD, %edx 129 xorl %eax, %eax 130 movl T_ONFAULT(%edx), %ecx /* address of save area */ 131 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 132 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 133 pushl %ecx 134 call longjmp /* let longjmp do the rest */ 135 SET_SIZE(on_fault) 136 137 ENTRY(no_fault) 138 movl %gs:CPU_THREAD, %edx 139 xorl %eax, %eax 140 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 141 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 142 ret 143 SET_SIZE(no_fault) 144 145#endif /* __i386 */ 146#endif /* __lint */ 147 148/* 149 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just 150 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called. 151 */ 152 153#if defined(lint) 154 155void 156on_trap_trampoline(void) 157{} 158 159#else /* __lint */ 160 161#if defined(__amd64) 162 163 ENTRY(on_trap_trampoline) 164 movq %gs:CPU_THREAD, %rsi 165 movq T_ONTRAP(%rsi), %rdi 166 addq $OT_JMPBUF, %rdi 167 jmp longjmp 168 SET_SIZE(on_trap_trampoline) 169 170#elif defined(__i386) 171 172 ENTRY(on_trap_trampoline) 173 movl %gs:CPU_THREAD, %eax 174 movl T_ONTRAP(%eax), %eax 175 addl $OT_JMPBUF, %eax 176 pushl %eax 177 call longjmp 178 SET_SIZE(on_trap_trampoline) 179 180#endif /* __i386 */ 181#endif /* __lint */ 182 183/* 184 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 185 * more information about the on_trap() mechanism. If the on_trap_data is the 186 * same as the topmost stack element, we just modify that element. 187 */ 188#if defined(lint) 189 190/*ARGSUSED*/ 191int 192on_trap(on_trap_data_t *otp, uint_t prot) 193{ return (0); } 194 195#else /* __lint */ 196 197#if defined(__amd64) 198 199 ENTRY(on_trap) 200 movw %si, OT_PROT(%rdi) /* ot_prot = prot */ 201 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */ 202 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */ 203 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */ 204 xorl %ecx, %ecx 205 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */ 206 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */ 207 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */ 208 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */ 209 cmpq %rdi, %rcx /* if (otp == %rcx) */ 210 je 0f /* don't modify t_ontrap */ 211 212 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */ 213 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */ 214 2150: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */ 216 jmp setjmp 217 SET_SIZE(on_trap) 218 219#elif defined(__i386) 220 221 ENTRY(on_trap) 222 movl 4(%esp), %eax /* %eax = otp */ 223 movl 8(%esp), %edx /* %edx = prot */ 224 225 movw %dx, OT_PROT(%eax) /* ot_prot = prot */ 226 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */ 227 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */ 228 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */ 229 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */ 230 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */ 231 movl %gs:CPU_THREAD, %edx /* %edx = curthread */ 232 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */ 233 cmpl %eax, %ecx /* if (otp == %ecx) */ 234 je 0f /* don't modify t_ontrap */ 235 236 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */ 237 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */ 238 2390: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */ 240 movl %eax, 4(%esp) /* put %eax back on the stack */ 241 jmp setjmp /* let setjmp do the rest */ 242 SET_SIZE(on_trap) 243 244#endif /* __i386 */ 245#endif /* __lint */ 246 247/* 248 * Setjmp and longjmp implement non-local gotos using state vectors 249 * type label_t. 250 */ 251 252#if defined(__lint) 253 254/* ARGSUSED */ 255int 256setjmp(label_t *lp) 257{ return (0); } 258 259/* ARGSUSED */ 260void 261longjmp(label_t *lp) 262{} 263 264#else /* __lint */ 265 266#if LABEL_PC != 0 267#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded 268#endif /* LABEL_PC != 0 */ 269 270#if defined(__amd64) 271 272 ENTRY(setjmp) 273 movq %rsp, LABEL_SP(%rdi) 274 movq %rbp, LABEL_RBP(%rdi) 275 movq %rbx, LABEL_RBX(%rdi) 276 movq %r12, LABEL_R12(%rdi) 277 movq %r13, LABEL_R13(%rdi) 278 movq %r14, LABEL_R14(%rdi) 279 movq %r15, LABEL_R15(%rdi) 280 movq (%rsp), %rdx /* return address */ 281 movq %rdx, (%rdi) /* LABEL_PC is 0 */ 282 xorl %eax, %eax /* return 0 */ 283 ret 284 SET_SIZE(setjmp) 285 286 ENTRY(longjmp) 287 movq LABEL_SP(%rdi), %rsp 288 movq LABEL_RBP(%rdi), %rbp 289 movq LABEL_RBX(%rdi), %rbx 290 movq LABEL_R12(%rdi), %r12 291 movq LABEL_R13(%rdi), %r13 292 movq LABEL_R14(%rdi), %r14 293 movq LABEL_R15(%rdi), %r15 294 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */ 295 movq %rdx, (%rsp) 296 xorl %eax, %eax 297 incl %eax /* return 1 */ 298 ret 299 SET_SIZE(longjmp) 300 301#elif defined(__i386) 302 303 ENTRY(setjmp) 304 movl 4(%esp), %edx /* address of save area */ 305 movl %ebp, LABEL_EBP(%edx) 306 movl %ebx, LABEL_EBX(%edx) 307 movl %esi, LABEL_ESI(%edx) 308 movl %edi, LABEL_EDI(%edx) 309 movl %esp, 4(%edx) 310 movl (%esp), %ecx /* %eip (return address) */ 311 movl %ecx, (%edx) /* LABEL_PC is 0 */ 312 subl %eax, %eax /* return 0 */ 313 ret 314 SET_SIZE(setjmp) 315 316 ENTRY(longjmp) 317 movl 4(%esp), %edx /* address of save area */ 318 movl LABEL_EBP(%edx), %ebp 319 movl LABEL_EBX(%edx), %ebx 320 movl LABEL_ESI(%edx), %esi 321 movl LABEL_EDI(%edx), %edi 322 movl 4(%edx), %esp 323 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */ 324 movl $1, %eax 325 addl $4, %esp /* pop ret adr */ 326 jmp *%ecx /* indirect */ 327 SET_SIZE(longjmp) 328 329#endif /* __i386 */ 330#endif /* __lint */ 331 332/* 333 * if a() calls b() calls caller(), 334 * caller() returns return address in a(). 335 * (Note: We assume a() and b() are C routines which do the normal entry/exit 336 * sequence.) 337 */ 338 339#if defined(__lint) 340 341caddr_t 342caller(void) 343{ return (0); } 344 345#else /* __lint */ 346 347#if defined(__amd64) 348 349 ENTRY(caller) 350 movq 8(%rbp), %rax /* b()'s return pc, in a() */ 351 ret 352 SET_SIZE(caller) 353 354#elif defined(__i386) 355 356 ENTRY(caller) 357 movl 4(%ebp), %eax /* b()'s return pc, in a() */ 358 ret 359 SET_SIZE(caller) 360 361#endif /* __i386 */ 362#endif /* __lint */ 363 364/* 365 * if a() calls callee(), callee() returns the 366 * return address in a(); 367 */ 368 369#if defined(__lint) 370 371caddr_t 372callee(void) 373{ return (0); } 374 375#else /* __lint */ 376 377#if defined(__amd64) 378 379 ENTRY(callee) 380 movq (%rsp), %rax /* callee()'s return pc, in a() */ 381 ret 382 SET_SIZE(callee) 383 384#elif defined(__i386) 385 386 ENTRY(callee) 387 movl (%esp), %eax /* callee()'s return pc, in a() */ 388 ret 389 SET_SIZE(callee) 390 391#endif /* __i386 */ 392#endif /* __lint */ 393 394/* 395 * return the current frame pointer 396 */ 397 398#if defined(__lint) 399 400greg_t 401getfp(void) 402{ return (0); } 403 404#else /* __lint */ 405 406#if defined(__amd64) 407 408 ENTRY(getfp) 409 movq %rbp, %rax 410 ret 411 SET_SIZE(getfp) 412 413#elif defined(__i386) 414 415 ENTRY(getfp) 416 movl %ebp, %eax 417 ret 418 SET_SIZE(getfp) 419 420#endif /* __i386 */ 421#endif /* __lint */ 422 423/* 424 * Invalidate a single page table entry in the TLB 425 */ 426 427#if defined(__lint) 428 429/* ARGSUSED */ 430void 431mmu_tlbflush_entry(caddr_t m) 432{} 433 434#else /* __lint */ 435 436#if defined(__amd64) 437 438 ENTRY(mmu_tlbflush_entry) 439 invlpg (%rdi) 440 ret 441 SET_SIZE(mmu_tlbflush_entry) 442 443#elif defined(__i386) 444 445 ENTRY(mmu_tlbflush_entry) 446 movl 4(%esp), %eax 447 invlpg (%eax) 448 ret 449 SET_SIZE(mmu_tlbflush_entry) 450 451#endif /* __i386 */ 452#endif /* __lint */ 453 454 455/* 456 * Get/Set the value of various control registers 457 */ 458 459#if defined(__lint) 460 461ulong_t 462getcr0(void) 463{ return (0); } 464 465/* ARGSUSED */ 466void 467setcr0(ulong_t value) 468{} 469 470ulong_t 471getcr2(void) 472{ return (0); } 473 474ulong_t 475getcr3(void) 476{ return (0); } 477 478#if !defined(__xpv) 479/* ARGSUSED */ 480void 481setcr3(ulong_t val) 482{} 483 484void 485reload_cr3(void) 486{} 487#endif 488 489ulong_t 490getcr4(void) 491{ return (0); } 492 493/* ARGSUSED */ 494void 495setcr4(ulong_t val) 496{} 497 498#if defined(__amd64) 499 500ulong_t 501getcr8(void) 502{ return (0); } 503 504/* ARGSUSED */ 505void 506setcr8(ulong_t val) 507{} 508 509#endif /* __amd64 */ 510 511#else /* __lint */ 512 513#if defined(__amd64) 514 515 ENTRY(getcr0) 516 movq %cr0, %rax 517 ret 518 SET_SIZE(getcr0) 519 520 ENTRY(setcr0) 521 movq %rdi, %cr0 522 ret 523 SET_SIZE(setcr0) 524 525 ENTRY(getcr2) 526#if defined(__xpv) 527 movq %gs:CPU_VCPU_INFO, %rax 528 movq VCPU_INFO_ARCH_CR2(%rax), %rax 529#else 530 movq %cr2, %rax 531#endif 532 ret 533 SET_SIZE(getcr2) 534 535 ENTRY(getcr3) 536 movq %cr3, %rax 537 ret 538 SET_SIZE(getcr3) 539 540#if !defined(__xpv) 541 542 ENTRY(setcr3) 543 movq %rdi, %cr3 544 ret 545 SET_SIZE(setcr3) 546 547 ENTRY(reload_cr3) 548 movq %cr3, %rdi 549 movq %rdi, %cr3 550 ret 551 SET_SIZE(reload_cr3) 552 553#endif /* __xpv */ 554 555 ENTRY(getcr4) 556 movq %cr4, %rax 557 ret 558 SET_SIZE(getcr4) 559 560 ENTRY(setcr4) 561 movq %rdi, %cr4 562 ret 563 SET_SIZE(setcr4) 564 565 ENTRY(getcr8) 566 movq %cr8, %rax 567 ret 568 SET_SIZE(getcr8) 569 570 ENTRY(setcr8) 571 movq %rdi, %cr8 572 ret 573 SET_SIZE(setcr8) 574 575#elif defined(__i386) 576 577 ENTRY(getcr0) 578 movl %cr0, %eax 579 ret 580 SET_SIZE(getcr0) 581 582 ENTRY(setcr0) 583 movl 4(%esp), %eax 584 movl %eax, %cr0 585 ret 586 SET_SIZE(setcr0) 587 588 /* 589 * "lock mov %cr0" is used on processors which indicate it is 590 * supported via CPUID. Normally the 32 bit TPR is accessed via 591 * the local APIC. 592 */ 593 ENTRY(getcr8) 594 lock 595 movl %cr0, %eax 596 ret 597 SET_SIZE(getcr8) 598 599 ENTRY(setcr8) 600 movl 4(%esp), %eax 601 lock 602 movl %eax, %cr0 603 ret 604 SET_SIZE(setcr8) 605 606 ENTRY(getcr2) 607#if defined(__xpv) 608 movl %gs:CPU_VCPU_INFO, %eax 609 movl VCPU_INFO_ARCH_CR2(%eax), %eax 610#else 611 movl %cr2, %eax 612#endif 613 ret 614 SET_SIZE(getcr2) 615 616 ENTRY(getcr3) 617 movl %cr3, %eax 618 ret 619 SET_SIZE(getcr3) 620 621#if !defined(__xpv) 622 623 ENTRY(setcr3) 624 movl 4(%esp), %eax 625 movl %eax, %cr3 626 ret 627 SET_SIZE(setcr3) 628 629 ENTRY(reload_cr3) 630 movl %cr3, %eax 631 movl %eax, %cr3 632 ret 633 SET_SIZE(reload_cr3) 634 635#endif /* __xpv */ 636 637 ENTRY(getcr4) 638 movl %cr4, %eax 639 ret 640 SET_SIZE(getcr4) 641 642 ENTRY(setcr4) 643 movl 4(%esp), %eax 644 movl %eax, %cr4 645 ret 646 SET_SIZE(setcr4) 647 648#endif /* __i386 */ 649#endif /* __lint */ 650 651#if defined(__lint) 652 653/*ARGSUSED*/ 654uint32_t 655__cpuid_insn(struct cpuid_regs *regs) 656{ return (0); } 657 658#else /* __lint */ 659 660#if defined(__amd64) 661 662 ENTRY(__cpuid_insn) 663 movq %rbx, %r8 664 movq %rcx, %r9 665 movq %rdx, %r11 666 movl (%rdi), %eax /* %eax = regs->cp_eax */ 667 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */ 668 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */ 669 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */ 670 cpuid 671 movl %eax, (%rdi) /* regs->cp_eax = %eax */ 672 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */ 673 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */ 674 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */ 675 movq %r8, %rbx 676 movq %r9, %rcx 677 movq %r11, %rdx 678 ret 679 SET_SIZE(__cpuid_insn) 680 681#elif defined(__i386) 682 683 ENTRY(__cpuid_insn) 684 pushl %ebp 685 movl 0x8(%esp), %ebp /* %ebp = regs */ 686 pushl %ebx 687 pushl %ecx 688 pushl %edx 689 movl (%ebp), %eax /* %eax = regs->cp_eax */ 690 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */ 691 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */ 692 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */ 693 cpuid 694 movl %eax, (%ebp) /* regs->cp_eax = %eax */ 695 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */ 696 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */ 697 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */ 698 popl %edx 699 popl %ecx 700 popl %ebx 701 popl %ebp 702 ret 703 SET_SIZE(__cpuid_insn) 704 705#endif /* __i386 */ 706#endif /* __lint */ 707 708#if defined(__lint) 709 710/*ARGSUSED*/ 711void 712i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints) 713{} 714 715#else /* __lint */ 716 717#if defined(__amd64) 718 719 ENTRY_NP(i86_monitor) 720 pushq %rbp 721 movq %rsp, %rbp 722 movq %rdi, %rax /* addr */ 723 movq %rsi, %rcx /* extensions */ 724 /* rdx contains input arg3: hints */ 725 clflush (%rax) 726 .byte 0x0f, 0x01, 0xc8 /* monitor */ 727 leave 728 ret 729 SET_SIZE(i86_monitor) 730 731#elif defined(__i386) 732 733ENTRY_NP(i86_monitor) 734 pushl %ebp 735 movl %esp, %ebp 736 movl 0x8(%ebp),%eax /* addr */ 737 movl 0xc(%ebp),%ecx /* extensions */ 738 movl 0x10(%ebp),%edx /* hints */ 739 clflush (%eax) 740 .byte 0x0f, 0x01, 0xc8 /* monitor */ 741 leave 742 ret 743 SET_SIZE(i86_monitor) 744 745#endif /* __i386 */ 746#endif /* __lint */ 747 748#if defined(__lint) 749 750/*ARGSUSED*/ 751void 752i86_mwait(uint32_t data, uint32_t extensions) 753{} 754 755#else /* __lint */ 756 757#if defined(__amd64) 758 759 ENTRY_NP(i86_mwait) 760 pushq %rbp 761 movq %rsp, %rbp 762 movq %rdi, %rax /* data */ 763 movq %rsi, %rcx /* extensions */ 764 .byte 0x0f, 0x01, 0xc9 /* mwait */ 765 leave 766 ret 767 SET_SIZE(i86_mwait) 768 769#elif defined(__i386) 770 771 ENTRY_NP(i86_mwait) 772 pushl %ebp 773 movl %esp, %ebp 774 movl 0x8(%ebp),%eax /* data */ 775 movl 0xc(%ebp),%ecx /* extensions */ 776 .byte 0x0f, 0x01, 0xc9 /* mwait */ 777 leave 778 ret 779 SET_SIZE(i86_mwait) 780 781#endif /* __i386 */ 782#endif /* __lint */ 783 784#if defined(__xpv) 785 /* 786 * Defined in C 787 */ 788#else 789 790#if defined(__lint) 791 792hrtime_t 793tsc_read(void) 794{ 795 return (0); 796} 797 798#else /* __lint */ 799 800#if defined(__amd64) 801 802 ENTRY_NP(tsc_read) 803 movq %rbx, %r11 804 movl $0, %eax 805 cpuid 806 rdtsc 807 movq %r11, %rbx 808 shlq $32, %rdx 809 orq %rdx, %rax 810 ret 811 .globl _tsc_mfence_start 812_tsc_mfence_start: 813 mfence 814 rdtsc 815 shlq $32, %rdx 816 orq %rdx, %rax 817 ret 818 .globl _tsc_mfence_end 819_tsc_mfence_end: 820 .globl _tscp_start 821_tscp_start: 822 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 823 shlq $32, %rdx 824 orq %rdx, %rax 825 ret 826 .globl _tscp_end 827_tscp_end: 828 .globl _no_rdtsc_start 829_no_rdtsc_start: 830 xorl %edx, %edx 831 xorl %eax, %eax 832 ret 833 .globl _no_rdtsc_end 834_no_rdtsc_end: 835 .globl _tsc_lfence_start 836_tsc_lfence_start: 837 lfence 838 rdtsc 839 shlq $32, %rdx 840 orq %rdx, %rax 841 ret 842 .globl _tsc_lfence_end 843_tsc_lfence_end: 844 SET_SIZE(tsc_read) 845 846#else /* __i386 */ 847 848 ENTRY_NP(tsc_read) 849 pushl %ebx 850 movl $0, %eax 851 cpuid 852 rdtsc 853 popl %ebx 854 ret 855 .globl _tsc_mfence_start 856_tsc_mfence_start: 857 mfence 858 rdtsc 859 ret 860 .globl _tsc_mfence_end 861_tsc_mfence_end: 862 .globl _tscp_start 863_tscp_start: 864 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 865 ret 866 .globl _tscp_end 867_tscp_end: 868 .globl _no_rdtsc_start 869_no_rdtsc_start: 870 xorl %edx, %edx 871 xorl %eax, %eax 872 ret 873 .globl _no_rdtsc_end 874_no_rdtsc_end: 875 .globl _tsc_lfence_start 876_tsc_lfence_start: 877 lfence 878 rdtsc 879 ret 880 .globl _tsc_lfence_end 881_tsc_lfence_end: 882 SET_SIZE(tsc_read) 883 884#endif /* __i386 */ 885 886#endif /* __lint */ 887 888 889#endif /* __xpv */ 890 891#ifdef __lint 892/* 893 * Do not use this function for obtaining clock tick. This 894 * is called by callers who do not need to have a guarenteed 895 * correct tick value. The proper routine to use is tsc_read(). 896 */ 897u_longlong_t 898randtick(void) 899{ 900 return (0); 901} 902#else 903#if defined(__amd64) 904 ENTRY_NP(randtick) 905 rdtsc 906 shlq $32, %rdx 907 orq %rdx, %rax 908 ret 909 SET_SIZE(randtick) 910#else 911 ENTRY_NP(randtick) 912 rdtsc 913 ret 914 SET_SIZE(randtick) 915#endif /* __i386 */ 916#endif /* __lint */ 917/* 918 * Insert entryp after predp in a doubly linked list. 919 */ 920 921#if defined(__lint) 922 923/*ARGSUSED*/ 924void 925_insque(caddr_t entryp, caddr_t predp) 926{} 927 928#else /* __lint */ 929 930#if defined(__amd64) 931 932 ENTRY(_insque) 933 movq (%rsi), %rax /* predp->forw */ 934 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */ 935 movq %rax, (%rdi) /* entryp->forw = predp->forw */ 936 movq %rdi, (%rsi) /* predp->forw = entryp */ 937 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */ 938 ret 939 SET_SIZE(_insque) 940 941#elif defined(__i386) 942 943 ENTRY(_insque) 944 movl 8(%esp), %edx 945 movl 4(%esp), %ecx 946 movl (%edx), %eax /* predp->forw */ 947 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */ 948 movl %eax, (%ecx) /* entryp->forw = predp->forw */ 949 movl %ecx, (%edx) /* predp->forw = entryp */ 950 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */ 951 ret 952 SET_SIZE(_insque) 953 954#endif /* __i386 */ 955#endif /* __lint */ 956 957/* 958 * Remove entryp from a doubly linked list 959 */ 960 961#if defined(__lint) 962 963/*ARGSUSED*/ 964void 965_remque(caddr_t entryp) 966{} 967 968#else /* __lint */ 969 970#if defined(__amd64) 971 972 ENTRY(_remque) 973 movq (%rdi), %rax /* entry->forw */ 974 movq CPTRSIZE(%rdi), %rdx /* entry->back */ 975 movq %rax, (%rdx) /* entry->back->forw = entry->forw */ 976 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */ 977 ret 978 SET_SIZE(_remque) 979 980#elif defined(__i386) 981 982 ENTRY(_remque) 983 movl 4(%esp), %ecx 984 movl (%ecx), %eax /* entry->forw */ 985 movl CPTRSIZE(%ecx), %edx /* entry->back */ 986 movl %eax, (%edx) /* entry->back->forw = entry->forw */ 987 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */ 988 ret 989 SET_SIZE(_remque) 990 991#endif /* __i386 */ 992#endif /* __lint */ 993 994/* 995 * Returns the number of 996 * non-NULL bytes in string argument. 997 */ 998 999#if defined(__lint) 1000 1001/* ARGSUSED */ 1002size_t 1003strlen(const char *str) 1004{ return (0); } 1005 1006#else /* __lint */ 1007 1008#if defined(__amd64) 1009 1010/* 1011 * This is close to a simple transliteration of a C version of this 1012 * routine. We should either just -make- this be a C version, or 1013 * justify having it in assembler by making it significantly faster. 1014 * 1015 * size_t 1016 * strlen(const char *s) 1017 * { 1018 * const char *s0; 1019 * #if defined(DEBUG) 1020 * if ((uintptr_t)s < KERNELBASE) 1021 * panic(.str_panic_msg); 1022 * #endif 1023 * for (s0 = s; *s; s++) 1024 * ; 1025 * return (s - s0); 1026 * } 1027 */ 1028 1029 ENTRY(strlen) 1030#ifdef DEBUG 1031 movq postbootkernelbase(%rip), %rax 1032 cmpq %rax, %rdi 1033 jae str_valid 1034 pushq %rbp 1035 movq %rsp, %rbp 1036 leaq .str_panic_msg(%rip), %rdi 1037 xorl %eax, %eax 1038 call panic 1039#endif /* DEBUG */ 1040str_valid: 1041 cmpb $0, (%rdi) 1042 movq %rdi, %rax 1043 je .null_found 1044 .align 4 1045.strlen_loop: 1046 incq %rdi 1047 cmpb $0, (%rdi) 1048 jne .strlen_loop 1049.null_found: 1050 subq %rax, %rdi 1051 movq %rdi, %rax 1052 ret 1053 SET_SIZE(strlen) 1054 1055#elif defined(__i386) 1056 1057 ENTRY(strlen) 1058#ifdef DEBUG 1059 movl postbootkernelbase, %eax 1060 cmpl %eax, 4(%esp) 1061 jae str_valid 1062 pushl %ebp 1063 movl %esp, %ebp 1064 pushl $.str_panic_msg 1065 call panic 1066#endif /* DEBUG */ 1067 1068str_valid: 1069 movl 4(%esp), %eax /* %eax = string address */ 1070 testl $3, %eax /* if %eax not word aligned */ 1071 jnz .not_word_aligned /* goto .not_word_aligned */ 1072 .align 4 1073.word_aligned: 1074 movl (%eax), %edx /* move 1 word from (%eax) to %edx */ 1075 movl $0x7f7f7f7f, %ecx 1076 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */ 1077 addl $4, %eax /* next word */ 1078 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */ 1079 orl %edx, %ecx /* %ecx |= %edx */ 1080 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */ 1081 cmpl $0x80808080, %ecx /* if no null byte in this word */ 1082 je .word_aligned /* goto .word_aligned */ 1083 subl $4, %eax /* post-incremented */ 1084.not_word_aligned: 1085 cmpb $0, (%eax) /* if a byte in (%eax) is null */ 1086 je .null_found /* goto .null_found */ 1087 incl %eax /* next byte */ 1088 testl $3, %eax /* if %eax not word aligned */ 1089 jnz .not_word_aligned /* goto .not_word_aligned */ 1090 jmp .word_aligned /* goto .word_aligned */ 1091 .align 4 1092.null_found: 1093 subl 4(%esp), %eax /* %eax -= string address */ 1094 ret 1095 SET_SIZE(strlen) 1096 1097#endif /* __i386 */ 1098 1099#ifdef DEBUG 1100 .text 1101.str_panic_msg: 1102 .string "strlen: argument below kernelbase" 1103#endif /* DEBUG */ 1104 1105#endif /* __lint */ 1106 1107 /* 1108 * Berkeley 4.3 introduced symbolically named interrupt levels 1109 * as a way deal with priority in a machine independent fashion. 1110 * Numbered priorities are machine specific, and should be 1111 * discouraged where possible. 1112 * 1113 * Note, for the machine specific priorities there are 1114 * examples listed for devices that use a particular priority. 1115 * It should not be construed that all devices of that 1116 * type should be at that priority. It is currently were 1117 * the current devices fit into the priority scheme based 1118 * upon time criticalness. 1119 * 1120 * The underlying assumption of these assignments is that 1121 * IPL 10 is the highest level from which a device 1122 * routine can call wakeup. Devices that interrupt from higher 1123 * levels are restricted in what they can do. If they need 1124 * kernels services they should schedule a routine at a lower 1125 * level (via software interrupt) to do the required 1126 * processing. 1127 * 1128 * Examples of this higher usage: 1129 * Level Usage 1130 * 14 Profiling clock (and PROM uart polling clock) 1131 * 12 Serial ports 1132 * 1133 * The serial ports request lower level processing on level 6. 1134 * 1135 * Also, almost all splN routines (where N is a number or a 1136 * mnemonic) will do a RAISE(), on the assumption that they are 1137 * never used to lower our priority. 1138 * The exceptions are: 1139 * spl8() Because you can't be above 15 to begin with! 1140 * splzs() Because this is used at boot time to lower our 1141 * priority, to allow the PROM to poll the uart. 1142 * spl0() Used to lower priority to 0. 1143 */ 1144 1145#if defined(__lint) 1146 1147int spl0(void) { return (0); } 1148int spl6(void) { return (0); } 1149int spl7(void) { return (0); } 1150int spl8(void) { return (0); } 1151int splhigh(void) { return (0); } 1152int splhi(void) { return (0); } 1153int splzs(void) { return (0); } 1154 1155/* ARGSUSED */ 1156void 1157splx(int level) 1158{} 1159 1160#else /* __lint */ 1161 1162#if defined(__amd64) 1163 1164#define SETPRI(level) \ 1165 movl $/**/level, %edi; /* new priority */ \ 1166 jmp do_splx /* redirect to do_splx */ 1167 1168#define RAISE(level) \ 1169 movl $/**/level, %edi; /* new priority */ \ 1170 jmp splr /* redirect to splr */ 1171 1172#elif defined(__i386) 1173 1174#define SETPRI(level) \ 1175 pushl $/**/level; /* new priority */ \ 1176 call do_splx; /* invoke common splx code */ \ 1177 addl $4, %esp; /* unstack arg */ \ 1178 ret 1179 1180#define RAISE(level) \ 1181 pushl $/**/level; /* new priority */ \ 1182 call splr; /* invoke common splr code */ \ 1183 addl $4, %esp; /* unstack args */ \ 1184 ret 1185 1186#endif /* __i386 */ 1187 1188 /* locks out all interrupts, including memory errors */ 1189 ENTRY(spl8) 1190 SETPRI(15) 1191 SET_SIZE(spl8) 1192 1193 /* just below the level that profiling runs */ 1194 ENTRY(spl7) 1195 RAISE(13) 1196 SET_SIZE(spl7) 1197 1198 /* sun specific - highest priority onboard serial i/o asy ports */ 1199 ENTRY(splzs) 1200 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */ 1201 SET_SIZE(splzs) 1202 1203 ENTRY(splhi) 1204 ALTENTRY(splhigh) 1205 ALTENTRY(spl6) 1206 ALTENTRY(i_ddi_splhigh) 1207 1208 RAISE(DISP_LEVEL) 1209 1210 SET_SIZE(i_ddi_splhigh) 1211 SET_SIZE(spl6) 1212 SET_SIZE(splhigh) 1213 SET_SIZE(splhi) 1214 1215 /* allow all interrupts */ 1216 ENTRY(spl0) 1217 SETPRI(0) 1218 SET_SIZE(spl0) 1219 1220 1221 /* splx implementation */ 1222 ENTRY(splx) 1223 jmp do_splx /* redirect to common splx code */ 1224 SET_SIZE(splx) 1225 1226#endif /* __lint */ 1227 1228#if defined(__i386) 1229 1230/* 1231 * Read and write the %gs register 1232 */ 1233 1234#if defined(__lint) 1235 1236/*ARGSUSED*/ 1237uint16_t 1238getgs(void) 1239{ return (0); } 1240 1241/*ARGSUSED*/ 1242void 1243setgs(uint16_t sel) 1244{} 1245 1246#else /* __lint */ 1247 1248 ENTRY(getgs) 1249 clr %eax 1250 movw %gs, %ax 1251 ret 1252 SET_SIZE(getgs) 1253 1254 ENTRY(setgs) 1255 movw 4(%esp), %gs 1256 ret 1257 SET_SIZE(setgs) 1258 1259#endif /* __lint */ 1260#endif /* __i386 */ 1261 1262#if defined(__lint) 1263 1264void 1265pc_reset(void) 1266{} 1267 1268void 1269efi_reset(void) 1270{} 1271 1272#else /* __lint */ 1273 1274 ENTRY(wait_500ms) 1275#if defined(__amd64) 1276 pushq %rbx 1277#elif defined(__i386) 1278 push %ebx 1279#endif 1280 movl $50000, %ebx 12811: 1282 call tenmicrosec 1283 decl %ebx 1284 jnz 1b 1285#if defined(__amd64) 1286 popq %rbx 1287#elif defined(__i386) 1288 pop %ebx 1289#endif 1290 ret 1291 SET_SIZE(wait_500ms) 1292 1293#define RESET_METHOD_KBC 1 1294#define RESET_METHOD_PORT92 2 1295#define RESET_METHOD_PCI 4 1296 1297 DGDEF3(pc_reset_methods, 4, 8) 1298 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI; 1299 1300 ENTRY(pc_reset) 1301 1302#if defined(__i386) 1303 testl $RESET_METHOD_KBC, pc_reset_methods 1304#elif defined(__amd64) 1305 testl $RESET_METHOD_KBC, pc_reset_methods(%rip) 1306#endif 1307 jz 1f 1308 1309 / 1310 / Try the classic keyboard controller-triggered reset. 1311 / 1312 movw $0x64, %dx 1313 movb $0xfe, %al 1314 outb (%dx) 1315 1316 / Wait up to 500 milliseconds here for the keyboard controller 1317 / to pull the reset line. On some systems where the keyboard 1318 / controller is slow to pull the reset line, the next reset method 1319 / may be executed (which may be bad if those systems hang when the 1320 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92), 1321 / and Ferrari 4000 (doesn't like the cf9 reset method)) 1322 1323 call wait_500ms 1324 13251: 1326#if defined(__i386) 1327 testl $RESET_METHOD_PORT92, pc_reset_methods 1328#elif defined(__amd64) 1329 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip) 1330#endif 1331 jz 3f 1332 1333 / 1334 / Try port 0x92 fast reset 1335 / 1336 movw $0x92, %dx 1337 inb (%dx) 1338 cmpb $0xff, %al / If port's not there, we should get back 0xFF 1339 je 1f 1340 testb $1, %al / If bit 0 1341 jz 2f / is clear, jump to perform the reset 1342 andb $0xfe, %al / otherwise, 1343 outb (%dx) / clear bit 0 first, then 13442: 1345 orb $1, %al / Set bit 0 1346 outb (%dx) / and reset the system 13471: 1348 1349 call wait_500ms 1350 13513: 1352#if defined(__i386) 1353 testl $RESET_METHOD_PCI, pc_reset_methods 1354#elif defined(__amd64) 1355 testl $RESET_METHOD_PCI, pc_reset_methods(%rip) 1356#endif 1357 jz 4f 1358 1359 / Try the PCI (soft) reset vector (should work on all modern systems, 1360 / but has been shown to cause problems on 450NX systems, and some newer 1361 / systems (e.g. ATI IXP400-equipped systems)) 1362 / When resetting via this method, 2 writes are required. The first 1363 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with 1364 / power cycle). 1365 / The reset occurs on the second write, during bit 2's transition from 1366 / 0->1. 1367 movw $0xcf9, %dx 1368 movb $0x2, %al / Reset mode = hard, no power cycle 1369 outb (%dx) 1370 movb $0x6, %al 1371 outb (%dx) 1372 1373 call wait_500ms 1374 13754: 1376 / 1377 / port 0xcf9 failed also. Last-ditch effort is to 1378 / triple-fault the CPU. 1379 / Also, use triple fault for EFI firmware 1380 / 1381 ENTRY(efi_reset) 1382#if defined(__amd64) 1383 pushq $0x0 1384 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1385 lidt (%rsp) 1386#elif defined(__i386) 1387 pushl $0x0 1388 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1389 lidt (%esp) 1390#endif 1391 int $0x0 / Trigger interrupt, generate triple-fault 1392 1393 cli 1394 hlt / Wait forever 1395 /*NOTREACHED*/ 1396 SET_SIZE(efi_reset) 1397 SET_SIZE(pc_reset) 1398 1399#endif /* __lint */ 1400 1401/* 1402 * C callable in and out routines 1403 */ 1404 1405#if defined(__lint) 1406 1407/* ARGSUSED */ 1408void 1409outl(int port_address, uint32_t val) 1410{} 1411 1412#else /* __lint */ 1413 1414#if defined(__amd64) 1415 1416 ENTRY(outl) 1417 movw %di, %dx 1418 movl %esi, %eax 1419 outl (%dx) 1420 ret 1421 SET_SIZE(outl) 1422 1423#elif defined(__i386) 1424 1425 .set PORT, 4 1426 .set VAL, 8 1427 1428 ENTRY(outl) 1429 movw PORT(%esp), %dx 1430 movl VAL(%esp), %eax 1431 outl (%dx) 1432 ret 1433 SET_SIZE(outl) 1434 1435#endif /* __i386 */ 1436#endif /* __lint */ 1437 1438#if defined(__lint) 1439 1440/* ARGSUSED */ 1441void 1442outw(int port_address, uint16_t val) 1443{} 1444 1445#else /* __lint */ 1446 1447#if defined(__amd64) 1448 1449 ENTRY(outw) 1450 movw %di, %dx 1451 movw %si, %ax 1452 D16 outl (%dx) /* XX64 why not outw? */ 1453 ret 1454 SET_SIZE(outw) 1455 1456#elif defined(__i386) 1457 1458 ENTRY(outw) 1459 movw PORT(%esp), %dx 1460 movw VAL(%esp), %ax 1461 D16 outl (%dx) 1462 ret 1463 SET_SIZE(outw) 1464 1465#endif /* __i386 */ 1466#endif /* __lint */ 1467 1468#if defined(__lint) 1469 1470/* ARGSUSED */ 1471void 1472outb(int port_address, uint8_t val) 1473{} 1474 1475#else /* __lint */ 1476 1477#if defined(__amd64) 1478 1479 ENTRY(outb) 1480 movw %di, %dx 1481 movb %sil, %al 1482 outb (%dx) 1483 ret 1484 SET_SIZE(outb) 1485 1486#elif defined(__i386) 1487 1488 ENTRY(outb) 1489 movw PORT(%esp), %dx 1490 movb VAL(%esp), %al 1491 outb (%dx) 1492 ret 1493 SET_SIZE(outb) 1494 1495#endif /* __i386 */ 1496#endif /* __lint */ 1497 1498#if defined(__lint) 1499 1500/* ARGSUSED */ 1501uint32_t 1502inl(int port_address) 1503{ return (0); } 1504 1505#else /* __lint */ 1506 1507#if defined(__amd64) 1508 1509 ENTRY(inl) 1510 xorl %eax, %eax 1511 movw %di, %dx 1512 inl (%dx) 1513 ret 1514 SET_SIZE(inl) 1515 1516#elif defined(__i386) 1517 1518 ENTRY(inl) 1519 movw PORT(%esp), %dx 1520 inl (%dx) 1521 ret 1522 SET_SIZE(inl) 1523 1524#endif /* __i386 */ 1525#endif /* __lint */ 1526 1527#if defined(__lint) 1528 1529/* ARGSUSED */ 1530uint16_t 1531inw(int port_address) 1532{ return (0); } 1533 1534#else /* __lint */ 1535 1536#if defined(__amd64) 1537 1538 ENTRY(inw) 1539 xorl %eax, %eax 1540 movw %di, %dx 1541 D16 inl (%dx) 1542 ret 1543 SET_SIZE(inw) 1544 1545#elif defined(__i386) 1546 1547 ENTRY(inw) 1548 subl %eax, %eax 1549 movw PORT(%esp), %dx 1550 D16 inl (%dx) 1551 ret 1552 SET_SIZE(inw) 1553 1554#endif /* __i386 */ 1555#endif /* __lint */ 1556 1557 1558#if defined(__lint) 1559 1560/* ARGSUSED */ 1561uint8_t 1562inb(int port_address) 1563{ return (0); } 1564 1565#else /* __lint */ 1566 1567#if defined(__amd64) 1568 1569 ENTRY(inb) 1570 xorl %eax, %eax 1571 movw %di, %dx 1572 inb (%dx) 1573 ret 1574 SET_SIZE(inb) 1575 1576#elif defined(__i386) 1577 1578 ENTRY(inb) 1579 subl %eax, %eax 1580 movw PORT(%esp), %dx 1581 inb (%dx) 1582 ret 1583 SET_SIZE(inb) 1584 1585#endif /* __i386 */ 1586#endif /* __lint */ 1587 1588 1589#if defined(__lint) 1590 1591/* ARGSUSED */ 1592void 1593repoutsw(int port, uint16_t *addr, int cnt) 1594{} 1595 1596#else /* __lint */ 1597 1598#if defined(__amd64) 1599 1600 ENTRY(repoutsw) 1601 movl %edx, %ecx 1602 movw %di, %dx 1603 rep 1604 D16 outsl 1605 ret 1606 SET_SIZE(repoutsw) 1607 1608#elif defined(__i386) 1609 1610 /* 1611 * The arguments and saved registers are on the stack in the 1612 * following order: 1613 * | cnt | +16 1614 * | *addr | +12 1615 * | port | +8 1616 * | eip | +4 1617 * | esi | <-- %esp 1618 * If additional values are pushed onto the stack, make sure 1619 * to adjust the following constants accordingly. 1620 */ 1621 .set PORT, 8 1622 .set ADDR, 12 1623 .set COUNT, 16 1624 1625 ENTRY(repoutsw) 1626 pushl %esi 1627 movl PORT(%esp), %edx 1628 movl ADDR(%esp), %esi 1629 movl COUNT(%esp), %ecx 1630 rep 1631 D16 outsl 1632 popl %esi 1633 ret 1634 SET_SIZE(repoutsw) 1635 1636#endif /* __i386 */ 1637#endif /* __lint */ 1638 1639 1640#if defined(__lint) 1641 1642/* ARGSUSED */ 1643void 1644repinsw(int port_addr, uint16_t *addr, int cnt) 1645{} 1646 1647#else /* __lint */ 1648 1649#if defined(__amd64) 1650 1651 ENTRY(repinsw) 1652 movl %edx, %ecx 1653 movw %di, %dx 1654 rep 1655 D16 insl 1656 ret 1657 SET_SIZE(repinsw) 1658 1659#elif defined(__i386) 1660 1661 ENTRY(repinsw) 1662 pushl %edi 1663 movl PORT(%esp), %edx 1664 movl ADDR(%esp), %edi 1665 movl COUNT(%esp), %ecx 1666 rep 1667 D16 insl 1668 popl %edi 1669 ret 1670 SET_SIZE(repinsw) 1671 1672#endif /* __i386 */ 1673#endif /* __lint */ 1674 1675 1676#if defined(__lint) 1677 1678/* ARGSUSED */ 1679void 1680repinsb(int port, uint8_t *addr, int count) 1681{} 1682 1683#else /* __lint */ 1684 1685#if defined(__amd64) 1686 1687 ENTRY(repinsb) 1688 movl %edx, %ecx 1689 movw %di, %dx 1690 movq %rsi, %rdi 1691 rep 1692 insb 1693 ret 1694 SET_SIZE(repinsb) 1695 1696#elif defined(__i386) 1697 1698 /* 1699 * The arguments and saved registers are on the stack in the 1700 * following order: 1701 * | cnt | +16 1702 * | *addr | +12 1703 * | port | +8 1704 * | eip | +4 1705 * | esi | <-- %esp 1706 * If additional values are pushed onto the stack, make sure 1707 * to adjust the following constants accordingly. 1708 */ 1709 .set IO_PORT, 8 1710 .set IO_ADDR, 12 1711 .set IO_COUNT, 16 1712 1713 ENTRY(repinsb) 1714 pushl %edi 1715 movl IO_ADDR(%esp), %edi 1716 movl IO_COUNT(%esp), %ecx 1717 movl IO_PORT(%esp), %edx 1718 rep 1719 insb 1720 popl %edi 1721 ret 1722 SET_SIZE(repinsb) 1723 1724#endif /* __i386 */ 1725#endif /* __lint */ 1726 1727 1728/* 1729 * Input a stream of 32-bit words. 1730 * NOTE: count is a DWORD count. 1731 */ 1732#if defined(__lint) 1733 1734/* ARGSUSED */ 1735void 1736repinsd(int port, uint32_t *addr, int count) 1737{} 1738 1739#else /* __lint */ 1740 1741#if defined(__amd64) 1742 1743 ENTRY(repinsd) 1744 movl %edx, %ecx 1745 movw %di, %dx 1746 movq %rsi, %rdi 1747 rep 1748 insl 1749 ret 1750 SET_SIZE(repinsd) 1751 1752#elif defined(__i386) 1753 1754 ENTRY(repinsd) 1755 pushl %edi 1756 movl IO_ADDR(%esp), %edi 1757 movl IO_COUNT(%esp), %ecx 1758 movl IO_PORT(%esp), %edx 1759 rep 1760 insl 1761 popl %edi 1762 ret 1763 SET_SIZE(repinsd) 1764 1765#endif /* __i386 */ 1766#endif /* __lint */ 1767 1768/* 1769 * Output a stream of bytes 1770 * NOTE: count is a byte count 1771 */ 1772#if defined(__lint) 1773 1774/* ARGSUSED */ 1775void 1776repoutsb(int port, uint8_t *addr, int count) 1777{} 1778 1779#else /* __lint */ 1780 1781#if defined(__amd64) 1782 1783 ENTRY(repoutsb) 1784 movl %edx, %ecx 1785 movw %di, %dx 1786 rep 1787 outsb 1788 ret 1789 SET_SIZE(repoutsb) 1790 1791#elif defined(__i386) 1792 1793 ENTRY(repoutsb) 1794 pushl %esi 1795 movl IO_ADDR(%esp), %esi 1796 movl IO_COUNT(%esp), %ecx 1797 movl IO_PORT(%esp), %edx 1798 rep 1799 outsb 1800 popl %esi 1801 ret 1802 SET_SIZE(repoutsb) 1803 1804#endif /* __i386 */ 1805#endif /* __lint */ 1806 1807/* 1808 * Output a stream of 32-bit words 1809 * NOTE: count is a DWORD count 1810 */ 1811#if defined(__lint) 1812 1813/* ARGSUSED */ 1814void 1815repoutsd(int port, uint32_t *addr, int count) 1816{} 1817 1818#else /* __lint */ 1819 1820#if defined(__amd64) 1821 1822 ENTRY(repoutsd) 1823 movl %edx, %ecx 1824 movw %di, %dx 1825 rep 1826 outsl 1827 ret 1828 SET_SIZE(repoutsd) 1829 1830#elif defined(__i386) 1831 1832 ENTRY(repoutsd) 1833 pushl %esi 1834 movl IO_ADDR(%esp), %esi 1835 movl IO_COUNT(%esp), %ecx 1836 movl IO_PORT(%esp), %edx 1837 rep 1838 outsl 1839 popl %esi 1840 ret 1841 SET_SIZE(repoutsd) 1842 1843#endif /* __i386 */ 1844#endif /* __lint */ 1845 1846/* 1847 * void int3(void) 1848 * void int18(void) 1849 * void int20(void) 1850 * void int_cmci(void) 1851 */ 1852 1853#if defined(__lint) 1854 1855void 1856int3(void) 1857{} 1858 1859void 1860int18(void) 1861{} 1862 1863void 1864int20(void) 1865{} 1866 1867void 1868int_cmci(void) 1869{} 1870 1871#else /* __lint */ 1872 1873 ENTRY(int3) 1874 int $T_BPTFLT 1875 ret 1876 SET_SIZE(int3) 1877 1878 ENTRY(int18) 1879 int $T_MCE 1880 ret 1881 SET_SIZE(int18) 1882 1883 ENTRY(int20) 1884 movl boothowto, %eax 1885 andl $RB_DEBUG, %eax 1886 jz 1f 1887 1888 int $T_DBGENTR 18891: 1890 rep; ret /* use 2 byte return instruction when branch target */ 1891 /* AMD Software Optimization Guide - Section 6.2 */ 1892 SET_SIZE(int20) 1893 1894 ENTRY(int_cmci) 1895 int $T_ENOEXTFLT 1896 ret 1897 SET_SIZE(int_cmci) 1898 1899#endif /* __lint */ 1900 1901#if defined(__lint) 1902 1903/* ARGSUSED */ 1904int 1905scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask) 1906{ return (0); } 1907 1908#else /* __lint */ 1909 1910#if defined(__amd64) 1911 1912 ENTRY(scanc) 1913 /* rdi == size */ 1914 /* rsi == cp */ 1915 /* rdx == table */ 1916 /* rcx == mask */ 1917 addq %rsi, %rdi /* end = &cp[size] */ 1918.scanloop: 1919 cmpq %rdi, %rsi /* while (cp < end */ 1920 jnb .scandone 1921 movzbq (%rsi), %r8 /* %r8 = *cp */ 1922 incq %rsi /* cp++ */ 1923 testb %cl, (%r8, %rdx) 1924 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1925 decq %rsi /* (fix post-increment) */ 1926.scandone: 1927 movl %edi, %eax 1928 subl %esi, %eax /* return (end - cp) */ 1929 ret 1930 SET_SIZE(scanc) 1931 1932#elif defined(__i386) 1933 1934 ENTRY(scanc) 1935 pushl %edi 1936 pushl %esi 1937 movb 24(%esp), %cl /* mask = %cl */ 1938 movl 16(%esp), %esi /* cp = %esi */ 1939 movl 20(%esp), %edx /* table = %edx */ 1940 movl %esi, %edi 1941 addl 12(%esp), %edi /* end = &cp[size]; */ 1942.scanloop: 1943 cmpl %edi, %esi /* while (cp < end */ 1944 jnb .scandone 1945 movzbl (%esi), %eax /* %al = *cp */ 1946 incl %esi /* cp++ */ 1947 movb (%edx, %eax), %al /* %al = table[*cp] */ 1948 testb %al, %cl 1949 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1950 dec %esi /* post-incremented */ 1951.scandone: 1952 movl %edi, %eax 1953 subl %esi, %eax /* return (end - cp) */ 1954 popl %esi 1955 popl %edi 1956 ret 1957 SET_SIZE(scanc) 1958 1959#endif /* __i386 */ 1960#endif /* __lint */ 1961 1962/* 1963 * Replacement functions for ones that are normally inlined. 1964 * In addition to the copy in i86.il, they are defined here just in case. 1965 */ 1966 1967#if defined(__lint) 1968 1969ulong_t 1970intr_clear(void) 1971{ return (0); } 1972 1973ulong_t 1974clear_int_flag(void) 1975{ return (0); } 1976 1977#else /* __lint */ 1978 1979#if defined(__amd64) 1980 1981 ENTRY(intr_clear) 1982 ENTRY(clear_int_flag) 1983 pushfq 1984 popq %rax 1985#if defined(__xpv) 1986 leaq xpv_panicking, %rdi 1987 movl (%rdi), %edi 1988 cmpl $0, %edi 1989 jne 2f 1990 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 1991 /* 1992 * Synthesize the PS_IE bit from the event mask bit 1993 */ 1994 andq $_BITNOT(PS_IE), %rax 1995 testb $1, %dl 1996 jnz 1f 1997 orq $PS_IE, %rax 19981: 1999 ret 20002: 2001#endif 2002 CLI(%rdi) 2003 ret 2004 SET_SIZE(clear_int_flag) 2005 SET_SIZE(intr_clear) 2006 2007#elif defined(__i386) 2008 2009 ENTRY(intr_clear) 2010 ENTRY(clear_int_flag) 2011 pushfl 2012 popl %eax 2013#if defined(__xpv) 2014 leal xpv_panicking, %edx 2015 movl (%edx), %edx 2016 cmpl $0, %edx 2017 jne 2f 2018 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2019 /* 2020 * Synthesize the PS_IE bit from the event mask bit 2021 */ 2022 andl $_BITNOT(PS_IE), %eax 2023 testb $1, %cl 2024 jnz 1f 2025 orl $PS_IE, %eax 20261: 2027 ret 20282: 2029#endif 2030 CLI(%edx) 2031 ret 2032 SET_SIZE(clear_int_flag) 2033 SET_SIZE(intr_clear) 2034 2035#endif /* __i386 */ 2036#endif /* __lint */ 2037 2038#if defined(__lint) 2039 2040struct cpu * 2041curcpup(void) 2042{ return 0; } 2043 2044#else /* __lint */ 2045 2046#if defined(__amd64) 2047 2048 ENTRY(curcpup) 2049 movq %gs:CPU_SELF, %rax 2050 ret 2051 SET_SIZE(curcpup) 2052 2053#elif defined(__i386) 2054 2055 ENTRY(curcpup) 2056 movl %gs:CPU_SELF, %eax 2057 ret 2058 SET_SIZE(curcpup) 2059 2060#endif /* __i386 */ 2061#endif /* __lint */ 2062 2063/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs() 2064 * These functions reverse the byte order of the input parameter and returns 2065 * the result. This is to convert the byte order from host byte order 2066 * (little endian) to network byte order (big endian), or vice versa. 2067 */ 2068 2069#if defined(__lint) 2070 2071uint64_t 2072htonll(uint64_t i) 2073{ return (i); } 2074 2075uint64_t 2076ntohll(uint64_t i) 2077{ return (i); } 2078 2079uint32_t 2080htonl(uint32_t i) 2081{ return (i); } 2082 2083uint32_t 2084ntohl(uint32_t i) 2085{ return (i); } 2086 2087uint16_t 2088htons(uint16_t i) 2089{ return (i); } 2090 2091uint16_t 2092ntohs(uint16_t i) 2093{ return (i); } 2094 2095#else /* __lint */ 2096 2097#if defined(__amd64) 2098 2099 ENTRY(htonll) 2100 ALTENTRY(ntohll) 2101 movq %rdi, %rax 2102 bswapq %rax 2103 ret 2104 SET_SIZE(ntohll) 2105 SET_SIZE(htonll) 2106 2107 /* XX64 there must be shorter sequences for this */ 2108 ENTRY(htonl) 2109 ALTENTRY(ntohl) 2110 movl %edi, %eax 2111 bswap %eax 2112 ret 2113 SET_SIZE(ntohl) 2114 SET_SIZE(htonl) 2115 2116 /* XX64 there must be better sequences for this */ 2117 ENTRY(htons) 2118 ALTENTRY(ntohs) 2119 movl %edi, %eax 2120 bswap %eax 2121 shrl $16, %eax 2122 ret 2123 SET_SIZE(ntohs) 2124 SET_SIZE(htons) 2125 2126#elif defined(__i386) 2127 2128 ENTRY(htonll) 2129 ALTENTRY(ntohll) 2130 movl 4(%esp), %edx 2131 movl 8(%esp), %eax 2132 bswap %edx 2133 bswap %eax 2134 ret 2135 SET_SIZE(ntohll) 2136 SET_SIZE(htonll) 2137 2138 ENTRY(htonl) 2139 ALTENTRY(ntohl) 2140 movl 4(%esp), %eax 2141 bswap %eax 2142 ret 2143 SET_SIZE(ntohl) 2144 SET_SIZE(htonl) 2145 2146 ENTRY(htons) 2147 ALTENTRY(ntohs) 2148 movl 4(%esp), %eax 2149 bswap %eax 2150 shrl $16, %eax 2151 ret 2152 SET_SIZE(ntohs) 2153 SET_SIZE(htons) 2154 2155#endif /* __i386 */ 2156#endif /* __lint */ 2157 2158 2159#if defined(__lint) 2160 2161/* ARGSUSED */ 2162void 2163intr_restore(ulong_t i) 2164{ return; } 2165 2166/* ARGSUSED */ 2167void 2168restore_int_flag(ulong_t i) 2169{ return; } 2170 2171#else /* __lint */ 2172 2173#if defined(__amd64) 2174 2175 ENTRY(intr_restore) 2176 ENTRY(restore_int_flag) 2177 testq $PS_IE, %rdi 2178 jz 1f 2179#if defined(__xpv) 2180 leaq xpv_panicking, %rsi 2181 movl (%rsi), %esi 2182 cmpl $0, %esi 2183 jne 1f 2184 /* 2185 * Since we're -really- running unprivileged, our attempt 2186 * to change the state of the IF bit will be ignored. 2187 * The virtual IF bit is tweaked by CLI and STI. 2188 */ 2189 IE_TO_EVENT_MASK(%rsi, %rdi) 2190#else 2191 sti 2192#endif 21931: 2194 ret 2195 SET_SIZE(restore_int_flag) 2196 SET_SIZE(intr_restore) 2197 2198#elif defined(__i386) 2199 2200 ENTRY(intr_restore) 2201 ENTRY(restore_int_flag) 2202 testl $PS_IE, 4(%esp) 2203 jz 1f 2204#if defined(__xpv) 2205 leal xpv_panicking, %edx 2206 movl (%edx), %edx 2207 cmpl $0, %edx 2208 jne 1f 2209 /* 2210 * Since we're -really- running unprivileged, our attempt 2211 * to change the state of the IF bit will be ignored. 2212 * The virtual IF bit is tweaked by CLI and STI. 2213 */ 2214 IE_TO_EVENT_MASK(%edx, 4(%esp)) 2215#else 2216 sti 2217#endif 22181: 2219 ret 2220 SET_SIZE(restore_int_flag) 2221 SET_SIZE(intr_restore) 2222 2223#endif /* __i386 */ 2224#endif /* __lint */ 2225 2226#if defined(__lint) 2227 2228void 2229sti(void) 2230{} 2231 2232void 2233cli(void) 2234{} 2235 2236#else /* __lint */ 2237 2238 ENTRY(sti) 2239 STI 2240 ret 2241 SET_SIZE(sti) 2242 2243 ENTRY(cli) 2244#if defined(__amd64) 2245 CLI(%rax) 2246#elif defined(__i386) 2247 CLI(%eax) 2248#endif /* __i386 */ 2249 ret 2250 SET_SIZE(cli) 2251 2252#endif /* __lint */ 2253 2254#if defined(__lint) 2255 2256dtrace_icookie_t 2257dtrace_interrupt_disable(void) 2258{ return (0); } 2259 2260#else /* __lint */ 2261 2262#if defined(__amd64) 2263 2264 ENTRY(dtrace_interrupt_disable) 2265 pushfq 2266 popq %rax 2267#if defined(__xpv) 2268 leaq xpv_panicking, %rdi 2269 movl (%rdi), %edi 2270 cmpl $0, %edi 2271 jne .dtrace_interrupt_disable_done 2272 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 2273 /* 2274 * Synthesize the PS_IE bit from the event mask bit 2275 */ 2276 andq $_BITNOT(PS_IE), %rax 2277 testb $1, %dl 2278 jnz .dtrace_interrupt_disable_done 2279 orq $PS_IE, %rax 2280#else 2281 CLI(%rdx) 2282#endif 2283.dtrace_interrupt_disable_done: 2284 ret 2285 SET_SIZE(dtrace_interrupt_disable) 2286 2287#elif defined(__i386) 2288 2289 ENTRY(dtrace_interrupt_disable) 2290 pushfl 2291 popl %eax 2292#if defined(__xpv) 2293 leal xpv_panicking, %edx 2294 movl (%edx), %edx 2295 cmpl $0, %edx 2296 jne .dtrace_interrupt_disable_done 2297 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2298 /* 2299 * Synthesize the PS_IE bit from the event mask bit 2300 */ 2301 andl $_BITNOT(PS_IE), %eax 2302 testb $1, %cl 2303 jnz .dtrace_interrupt_disable_done 2304 orl $PS_IE, %eax 2305#else 2306 CLI(%edx) 2307#endif 2308.dtrace_interrupt_disable_done: 2309 ret 2310 SET_SIZE(dtrace_interrupt_disable) 2311 2312#endif /* __i386 */ 2313#endif /* __lint */ 2314 2315#if defined(__lint) 2316 2317/*ARGSUSED*/ 2318void 2319dtrace_interrupt_enable(dtrace_icookie_t cookie) 2320{} 2321 2322#else /* __lint */ 2323 2324#if defined(__amd64) 2325 2326 ENTRY(dtrace_interrupt_enable) 2327 pushq %rdi 2328 popfq 2329#if defined(__xpv) 2330 leaq xpv_panicking, %rdx 2331 movl (%rdx), %edx 2332 cmpl $0, %edx 2333 jne .dtrace_interrupt_enable_done 2334 /* 2335 * Since we're -really- running unprivileged, our attempt 2336 * to change the state of the IF bit will be ignored. The 2337 * virtual IF bit is tweaked by CLI and STI. 2338 */ 2339 IE_TO_EVENT_MASK(%rdx, %rdi) 2340#endif 2341.dtrace_interrupt_enable_done: 2342 ret 2343 SET_SIZE(dtrace_interrupt_enable) 2344 2345#elif defined(__i386) 2346 2347 ENTRY(dtrace_interrupt_enable) 2348 movl 4(%esp), %eax 2349 pushl %eax 2350 popfl 2351#if defined(__xpv) 2352 leal xpv_panicking, %edx 2353 movl (%edx), %edx 2354 cmpl $0, %edx 2355 jne .dtrace_interrupt_enable_done 2356 /* 2357 * Since we're -really- running unprivileged, our attempt 2358 * to change the state of the IF bit will be ignored. The 2359 * virtual IF bit is tweaked by CLI and STI. 2360 */ 2361 IE_TO_EVENT_MASK(%edx, %eax) 2362#endif 2363.dtrace_interrupt_enable_done: 2364 ret 2365 SET_SIZE(dtrace_interrupt_enable) 2366 2367#endif /* __i386 */ 2368#endif /* __lint */ 2369 2370 2371#if defined(lint) 2372 2373void 2374dtrace_membar_producer(void) 2375{} 2376 2377void 2378dtrace_membar_consumer(void) 2379{} 2380 2381#else /* __lint */ 2382 2383 ENTRY(dtrace_membar_producer) 2384 rep; ret /* use 2 byte return instruction when branch target */ 2385 /* AMD Software Optimization Guide - Section 6.2 */ 2386 SET_SIZE(dtrace_membar_producer) 2387 2388 ENTRY(dtrace_membar_consumer) 2389 rep; ret /* use 2 byte return instruction when branch target */ 2390 /* AMD Software Optimization Guide - Section 6.2 */ 2391 SET_SIZE(dtrace_membar_consumer) 2392 2393#endif /* __lint */ 2394 2395#if defined(__lint) 2396 2397kthread_id_t 2398threadp(void) 2399{ return ((kthread_id_t)0); } 2400 2401#else /* __lint */ 2402 2403#if defined(__amd64) 2404 2405 ENTRY(threadp) 2406 movq %gs:CPU_THREAD, %rax 2407 ret 2408 SET_SIZE(threadp) 2409 2410#elif defined(__i386) 2411 2412 ENTRY(threadp) 2413 movl %gs:CPU_THREAD, %eax 2414 ret 2415 SET_SIZE(threadp) 2416 2417#endif /* __i386 */ 2418#endif /* __lint */ 2419 2420/* 2421 * Checksum routine for Internet Protocol Headers 2422 */ 2423 2424#if defined(__lint) 2425 2426/* ARGSUSED */ 2427unsigned int 2428ip_ocsum( 2429 ushort_t *address, /* ptr to 1st message buffer */ 2430 int halfword_count, /* length of data */ 2431 unsigned int sum) /* partial checksum */ 2432{ 2433 int i; 2434 unsigned int psum = 0; /* partial sum */ 2435 2436 for (i = 0; i < halfword_count; i++, address++) { 2437 psum += *address; 2438 } 2439 2440 while ((psum >> 16) != 0) { 2441 psum = (psum & 0xffff) + (psum >> 16); 2442 } 2443 2444 psum += sum; 2445 2446 while ((psum >> 16) != 0) { 2447 psum = (psum & 0xffff) + (psum >> 16); 2448 } 2449 2450 return (psum); 2451} 2452 2453#else /* __lint */ 2454 2455#if defined(__amd64) 2456 2457 ENTRY(ip_ocsum) 2458 pushq %rbp 2459 movq %rsp, %rbp 2460#ifdef DEBUG 2461 movq postbootkernelbase(%rip), %rax 2462 cmpq %rax, %rdi 2463 jnb 1f 2464 xorl %eax, %eax 2465 movq %rdi, %rsi 2466 leaq .ip_ocsum_panic_msg(%rip), %rdi 2467 call panic 2468 /*NOTREACHED*/ 2469.ip_ocsum_panic_msg: 2470 .string "ip_ocsum: address 0x%p below kernelbase\n" 24711: 2472#endif 2473 movl %esi, %ecx /* halfword_count */ 2474 movq %rdi, %rsi /* address */ 2475 /* partial sum in %edx */ 2476 xorl %eax, %eax 2477 testl %ecx, %ecx 2478 jz .ip_ocsum_done 2479 testq $3, %rsi 2480 jnz .ip_csum_notaligned 2481.ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */ 2482.next_iter: 2483 /* XX64 opportunities for prefetch? */ 2484 /* XX64 compute csum with 64 bit quantities? */ 2485 subl $32, %ecx 2486 jl .less_than_32 2487 2488 addl 0(%rsi), %edx 2489.only60: 2490 adcl 4(%rsi), %eax 2491.only56: 2492 adcl 8(%rsi), %edx 2493.only52: 2494 adcl 12(%rsi), %eax 2495.only48: 2496 adcl 16(%rsi), %edx 2497.only44: 2498 adcl 20(%rsi), %eax 2499.only40: 2500 adcl 24(%rsi), %edx 2501.only36: 2502 adcl 28(%rsi), %eax 2503.only32: 2504 adcl 32(%rsi), %edx 2505.only28: 2506 adcl 36(%rsi), %eax 2507.only24: 2508 adcl 40(%rsi), %edx 2509.only20: 2510 adcl 44(%rsi), %eax 2511.only16: 2512 adcl 48(%rsi), %edx 2513.only12: 2514 adcl 52(%rsi), %eax 2515.only8: 2516 adcl 56(%rsi), %edx 2517.only4: 2518 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */ 2519.only0: 2520 adcl $0, %eax /* could be adding -1 in eax with a carry */ 2521 adcl $0, %eax 2522 2523 addq $64, %rsi 2524 testl %ecx, %ecx 2525 jnz .next_iter 2526 2527.ip_ocsum_done: 2528 addl %eax, %edx 2529 adcl $0, %edx 2530 movl %edx, %eax /* form a 16 bit checksum by */ 2531 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2532 addw %dx, %ax 2533 adcw $0, %ax 2534 andl $0xffff, %eax 2535 leave 2536 ret 2537 2538.ip_csum_notaligned: 2539 xorl %edi, %edi 2540 movw (%rsi), %di 2541 addl %edi, %edx 2542 adcl $0, %edx 2543 addq $2, %rsi 2544 decl %ecx 2545 jmp .ip_csum_aligned 2546 2547.less_than_32: 2548 addl $32, %ecx 2549 testl $1, %ecx 2550 jz .size_aligned 2551 andl $0xfe, %ecx 2552 movzwl (%rsi, %rcx, 2), %edi 2553 addl %edi, %edx 2554 adcl $0, %edx 2555.size_aligned: 2556 movl %ecx, %edi 2557 shrl $1, %ecx 2558 shl $1, %edi 2559 subq $64, %rdi 2560 addq %rdi, %rsi 2561 leaq .ip_ocsum_jmptbl(%rip), %rdi 2562 leaq (%rdi, %rcx, 8), %rdi 2563 xorl %ecx, %ecx 2564 clc 2565 jmp *(%rdi) 2566 2567 .align 8 2568.ip_ocsum_jmptbl: 2569 .quad .only0, .only4, .only8, .only12, .only16, .only20 2570 .quad .only24, .only28, .only32, .only36, .only40, .only44 2571 .quad .only48, .only52, .only56, .only60 2572 SET_SIZE(ip_ocsum) 2573 2574#elif defined(__i386) 2575 2576 ENTRY(ip_ocsum) 2577 pushl %ebp 2578 movl %esp, %ebp 2579 pushl %ebx 2580 pushl %esi 2581 pushl %edi 2582 movl 12(%ebp), %ecx /* count of half words */ 2583 movl 16(%ebp), %edx /* partial checksum */ 2584 movl 8(%ebp), %esi 2585 xorl %eax, %eax 2586 testl %ecx, %ecx 2587 jz .ip_ocsum_done 2588 2589 testl $3, %esi 2590 jnz .ip_csum_notaligned 2591.ip_csum_aligned: 2592.next_iter: 2593 subl $32, %ecx 2594 jl .less_than_32 2595 2596 addl 0(%esi), %edx 2597.only60: 2598 adcl 4(%esi), %eax 2599.only56: 2600 adcl 8(%esi), %edx 2601.only52: 2602 adcl 12(%esi), %eax 2603.only48: 2604 adcl 16(%esi), %edx 2605.only44: 2606 adcl 20(%esi), %eax 2607.only40: 2608 adcl 24(%esi), %edx 2609.only36: 2610 adcl 28(%esi), %eax 2611.only32: 2612 adcl 32(%esi), %edx 2613.only28: 2614 adcl 36(%esi), %eax 2615.only24: 2616 adcl 40(%esi), %edx 2617.only20: 2618 adcl 44(%esi), %eax 2619.only16: 2620 adcl 48(%esi), %edx 2621.only12: 2622 adcl 52(%esi), %eax 2623.only8: 2624 adcl 56(%esi), %edx 2625.only4: 2626 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */ 2627.only0: 2628 adcl $0, %eax /* we could be adding -1 in eax with a carry */ 2629 adcl $0, %eax 2630 2631 addl $64, %esi 2632 andl %ecx, %ecx 2633 jnz .next_iter 2634 2635.ip_ocsum_done: 2636 addl %eax, %edx 2637 adcl $0, %edx 2638 movl %edx, %eax /* form a 16 bit checksum by */ 2639 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2640 addw %dx, %ax 2641 adcw $0, %ax 2642 andl $0xffff, %eax 2643 popl %edi /* restore registers */ 2644 popl %esi 2645 popl %ebx 2646 leave 2647 ret 2648 2649.ip_csum_notaligned: 2650 xorl %edi, %edi 2651 movw (%esi), %di 2652 addl %edi, %edx 2653 adcl $0, %edx 2654 addl $2, %esi 2655 decl %ecx 2656 jmp .ip_csum_aligned 2657 2658.less_than_32: 2659 addl $32, %ecx 2660 testl $1, %ecx 2661 jz .size_aligned 2662 andl $0xfe, %ecx 2663 movzwl (%esi, %ecx, 2), %edi 2664 addl %edi, %edx 2665 adcl $0, %edx 2666.size_aligned: 2667 movl %ecx, %edi 2668 shrl $1, %ecx 2669 shl $1, %edi 2670 subl $64, %edi 2671 addl %edi, %esi 2672 movl $.ip_ocsum_jmptbl, %edi 2673 lea (%edi, %ecx, 4), %edi 2674 xorl %ecx, %ecx 2675 clc 2676 jmp *(%edi) 2677 SET_SIZE(ip_ocsum) 2678 2679 .data 2680 .align 4 2681 2682.ip_ocsum_jmptbl: 2683 .long .only0, .only4, .only8, .only12, .only16, .only20 2684 .long .only24, .only28, .only32, .only36, .only40, .only44 2685 .long .only48, .only52, .only56, .only60 2686 2687 2688#endif /* __i386 */ 2689#endif /* __lint */ 2690 2691/* 2692 * multiply two long numbers and yield a u_longlong_t result, callable from C. 2693 * Provided to manipulate hrtime_t values. 2694 */ 2695#if defined(__lint) 2696 2697/* result = a * b; */ 2698 2699/* ARGSUSED */ 2700unsigned long long 2701mul32(uint_t a, uint_t b) 2702{ return (0); } 2703 2704#else /* __lint */ 2705 2706#if defined(__amd64) 2707 2708 ENTRY(mul32) 2709 xorl %edx, %edx /* XX64 joe, paranoia? */ 2710 movl %edi, %eax 2711 mull %esi 2712 shlq $32, %rdx 2713 orq %rdx, %rax 2714 ret 2715 SET_SIZE(mul32) 2716 2717#elif defined(__i386) 2718 2719 ENTRY(mul32) 2720 movl 8(%esp), %eax 2721 movl 4(%esp), %ecx 2722 mull %ecx 2723 ret 2724 SET_SIZE(mul32) 2725 2726#endif /* __i386 */ 2727#endif /* __lint */ 2728 2729#if defined(notused) 2730#if defined(__lint) 2731/* ARGSUSED */ 2732void 2733load_pte64(uint64_t *pte, uint64_t pte_value) 2734{} 2735#else /* __lint */ 2736 .globl load_pte64 2737load_pte64: 2738 movl 4(%esp), %eax 2739 movl 8(%esp), %ecx 2740 movl 12(%esp), %edx 2741 movl %edx, 4(%eax) 2742 movl %ecx, (%eax) 2743 ret 2744#endif /* __lint */ 2745#endif /* notused */ 2746 2747#if defined(__lint) 2748 2749/*ARGSUSED*/ 2750void 2751scan_memory(caddr_t addr, size_t size) 2752{} 2753 2754#else /* __lint */ 2755 2756#if defined(__amd64) 2757 2758 ENTRY(scan_memory) 2759 shrq $3, %rsi /* convert %rsi from byte to quadword count */ 2760 jz .scanm_done 2761 movq %rsi, %rcx /* move count into rep control register */ 2762 movq %rdi, %rsi /* move addr into lodsq control reg. */ 2763 rep lodsq /* scan the memory range */ 2764.scanm_done: 2765 rep; ret /* use 2 byte return instruction when branch target */ 2766 /* AMD Software Optimization Guide - Section 6.2 */ 2767 SET_SIZE(scan_memory) 2768 2769#elif defined(__i386) 2770 2771 ENTRY(scan_memory) 2772 pushl %ecx 2773 pushl %esi 2774 movl 16(%esp), %ecx /* move 2nd arg into rep control register */ 2775 shrl $2, %ecx /* convert from byte count to word count */ 2776 jz .scanm_done 2777 movl 12(%esp), %esi /* move 1st arg into lodsw control register */ 2778 .byte 0xf3 /* rep prefix. lame assembler. sigh. */ 2779 lodsl 2780.scanm_done: 2781 popl %esi 2782 popl %ecx 2783 ret 2784 SET_SIZE(scan_memory) 2785 2786#endif /* __i386 */ 2787#endif /* __lint */ 2788 2789 2790#if defined(__lint) 2791 2792/*ARGSUSED */ 2793int 2794lowbit(ulong_t i) 2795{ return (0); } 2796 2797#else /* __lint */ 2798 2799#if defined(__amd64) 2800 2801 ENTRY(lowbit) 2802 movl $-1, %eax 2803 bsfq %rdi, %rax 2804 incl %eax 2805 ret 2806 SET_SIZE(lowbit) 2807 2808#elif defined(__i386) 2809 2810 ENTRY(lowbit) 2811 movl $-1, %eax 2812 bsfl 4(%esp), %eax 2813 incl %eax 2814 ret 2815 SET_SIZE(lowbit) 2816 2817#endif /* __i386 */ 2818#endif /* __lint */ 2819 2820#if defined(__lint) 2821 2822/*ARGSUSED*/ 2823int 2824highbit(ulong_t i) 2825{ return (0); } 2826 2827#else /* __lint */ 2828 2829#if defined(__amd64) 2830 2831 ENTRY(highbit) 2832 movl $-1, %eax 2833 bsrq %rdi, %rax 2834 incl %eax 2835 ret 2836 SET_SIZE(highbit) 2837 2838#elif defined(__i386) 2839 2840 ENTRY(highbit) 2841 movl $-1, %eax 2842 bsrl 4(%esp), %eax 2843 incl %eax 2844 ret 2845 SET_SIZE(highbit) 2846 2847#endif /* __i386 */ 2848#endif /* __lint */ 2849 2850#if defined(__lint) 2851 2852/*ARGSUSED*/ 2853uint64_t 2854rdmsr(uint_t r) 2855{ return (0); } 2856 2857/*ARGSUSED*/ 2858void 2859wrmsr(uint_t r, const uint64_t val) 2860{} 2861 2862/*ARGSUSED*/ 2863uint64_t 2864xrdmsr(uint_t r) 2865{ return (0); } 2866 2867/*ARGSUSED*/ 2868void 2869xwrmsr(uint_t r, const uint64_t val) 2870{} 2871 2872void 2873invalidate_cache(void) 2874{} 2875 2876/*ARGSUSED*/ 2877uint64_t 2878get_xcr(uint_t r) 2879{ return (0); } 2880 2881/*ARGSUSED*/ 2882void 2883set_xcr(uint_t r, const uint64_t val) 2884{} 2885 2886#else /* __lint */ 2887 2888#define XMSR_ACCESS_VAL $0x9c5a203a 2889 2890#if defined(__amd64) 2891 2892 ENTRY(rdmsr) 2893 movl %edi, %ecx 2894 rdmsr 2895 shlq $32, %rdx 2896 orq %rdx, %rax 2897 ret 2898 SET_SIZE(rdmsr) 2899 2900 ENTRY(wrmsr) 2901 movq %rsi, %rdx 2902 shrq $32, %rdx 2903 movl %esi, %eax 2904 movl %edi, %ecx 2905 wrmsr 2906 ret 2907 SET_SIZE(wrmsr) 2908 2909 ENTRY(xrdmsr) 2910 pushq %rbp 2911 movq %rsp, %rbp 2912 movl %edi, %ecx 2913 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2914 rdmsr 2915 shlq $32, %rdx 2916 orq %rdx, %rax 2917 leave 2918 ret 2919 SET_SIZE(xrdmsr) 2920 2921 ENTRY(xwrmsr) 2922 pushq %rbp 2923 movq %rsp, %rbp 2924 movl %edi, %ecx 2925 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2926 movq %rsi, %rdx 2927 shrq $32, %rdx 2928 movl %esi, %eax 2929 wrmsr 2930 leave 2931 ret 2932 SET_SIZE(xwrmsr) 2933 2934 ENTRY(get_xcr) 2935 movl %edi, %ecx 2936 #xgetbv 2937 .byte 0x0f,0x01,0xd0 2938 shlq $32, %rdx 2939 orq %rdx, %rax 2940 ret 2941 SET_SIZE(get_xcr) 2942 2943 ENTRY(set_xcr) 2944 movq %rsi, %rdx 2945 shrq $32, %rdx 2946 movl %esi, %eax 2947 movl %edi, %ecx 2948 #xsetbv 2949 .byte 0x0f,0x01,0xd1 2950 ret 2951 SET_SIZE(set_xcr) 2952 2953#elif defined(__i386) 2954 2955 ENTRY(rdmsr) 2956 movl 4(%esp), %ecx 2957 rdmsr 2958 ret 2959 SET_SIZE(rdmsr) 2960 2961 ENTRY(wrmsr) 2962 movl 4(%esp), %ecx 2963 movl 8(%esp), %eax 2964 movl 12(%esp), %edx 2965 wrmsr 2966 ret 2967 SET_SIZE(wrmsr) 2968 2969 ENTRY(xrdmsr) 2970 pushl %ebp 2971 movl %esp, %ebp 2972 movl 8(%esp), %ecx 2973 pushl %edi 2974 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2975 rdmsr 2976 popl %edi 2977 leave 2978 ret 2979 SET_SIZE(xrdmsr) 2980 2981 ENTRY(xwrmsr) 2982 pushl %ebp 2983 movl %esp, %ebp 2984 movl 8(%esp), %ecx 2985 movl 12(%esp), %eax 2986 movl 16(%esp), %edx 2987 pushl %edi 2988 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2989 wrmsr 2990 popl %edi 2991 leave 2992 ret 2993 SET_SIZE(xwrmsr) 2994 2995 ENTRY(get_xcr) 2996 movl 4(%esp), %ecx 2997 #xgetbv 2998 .byte 0x0f,0x01,0xd0 2999 ret 3000 SET_SIZE(get_xcr) 3001 3002 ENTRY(set_xcr) 3003 movl 4(%esp), %ecx 3004 movl 8(%esp), %eax 3005 movl 12(%esp), %edx 3006 #xsetbv 3007 .byte 0x0f,0x01,0xd1 3008 ret 3009 SET_SIZE(set_xcr) 3010 3011#endif /* __i386 */ 3012 3013 ENTRY(invalidate_cache) 3014 wbinvd 3015 ret 3016 SET_SIZE(invalidate_cache) 3017 3018#endif /* __lint */ 3019 3020#if defined(__lint) 3021 3022/*ARGSUSED*/ 3023void 3024getcregs(struct cregs *crp) 3025{} 3026 3027#else /* __lint */ 3028 3029#if defined(__amd64) 3030 3031 ENTRY_NP(getcregs) 3032#if defined(__xpv) 3033 /* 3034 * Only a few of the hardware control registers or descriptor tables 3035 * are directly accessible to us, so just zero the structure. 3036 * 3037 * XXPV Perhaps it would be helpful for the hypervisor to return 3038 * virtualized versions of these for post-mortem use. 3039 * (Need to reevaluate - perhaps it already does!) 3040 */ 3041 pushq %rdi /* save *crp */ 3042 movq $CREGSZ, %rsi 3043 call bzero 3044 popq %rdi 3045 3046 /* 3047 * Dump what limited information we can 3048 */ 3049 movq %cr0, %rax 3050 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3051 movq %cr2, %rax 3052 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3053 movq %cr3, %rax 3054 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3055 movq %cr4, %rax 3056 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3057 3058#else /* __xpv */ 3059 3060#define GETMSR(r, off, d) \ 3061 movl $r, %ecx; \ 3062 rdmsr; \ 3063 movl %eax, off(d); \ 3064 movl %edx, off+4(d) 3065 3066 xorl %eax, %eax 3067 movq %rax, CREG_GDT+8(%rdi) 3068 sgdt CREG_GDT(%rdi) /* 10 bytes */ 3069 movq %rax, CREG_IDT+8(%rdi) 3070 sidt CREG_IDT(%rdi) /* 10 bytes */ 3071 movq %rax, CREG_LDT(%rdi) 3072 sldt CREG_LDT(%rdi) /* 2 bytes */ 3073 movq %rax, CREG_TASKR(%rdi) 3074 str CREG_TASKR(%rdi) /* 2 bytes */ 3075 movq %cr0, %rax 3076 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3077 movq %cr2, %rax 3078 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3079 movq %cr3, %rax 3080 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3081 movq %cr4, %rax 3082 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3083 movq %cr8, %rax 3084 movq %rax, CREG_CR8(%rdi) /* cr8 */ 3085 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi) 3086 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi) 3087#endif /* __xpv */ 3088 ret 3089 SET_SIZE(getcregs) 3090 3091#undef GETMSR 3092 3093#elif defined(__i386) 3094 3095 ENTRY_NP(getcregs) 3096#if defined(__xpv) 3097 /* 3098 * Only a few of the hardware control registers or descriptor tables 3099 * are directly accessible to us, so just zero the structure. 3100 * 3101 * XXPV Perhaps it would be helpful for the hypervisor to return 3102 * virtualized versions of these for post-mortem use. 3103 * (Need to reevaluate - perhaps it already does!) 3104 */ 3105 movl 4(%esp), %edx 3106 pushl $CREGSZ 3107 pushl %edx 3108 call bzero 3109 addl $8, %esp 3110 movl 4(%esp), %edx 3111 3112 /* 3113 * Dump what limited information we can 3114 */ 3115 movl %cr0, %eax 3116 movl %eax, CREG_CR0(%edx) /* cr0 */ 3117 movl %cr2, %eax 3118 movl %eax, CREG_CR2(%edx) /* cr2 */ 3119 movl %cr3, %eax 3120 movl %eax, CREG_CR3(%edx) /* cr3 */ 3121 movl %cr4, %eax 3122 movl %eax, CREG_CR4(%edx) /* cr4 */ 3123 3124#else /* __xpv */ 3125 3126 movl 4(%esp), %edx 3127 movw $0, CREG_GDT+6(%edx) 3128 movw $0, CREG_IDT+6(%edx) 3129 sgdt CREG_GDT(%edx) /* gdt */ 3130 sidt CREG_IDT(%edx) /* idt */ 3131 sldt CREG_LDT(%edx) /* ldt */ 3132 str CREG_TASKR(%edx) /* task */ 3133 movl %cr0, %eax 3134 movl %eax, CREG_CR0(%edx) /* cr0 */ 3135 movl %cr2, %eax 3136 movl %eax, CREG_CR2(%edx) /* cr2 */ 3137 movl %cr3, %eax 3138 movl %eax, CREG_CR3(%edx) /* cr3 */ 3139 bt $X86FSET_LARGEPAGE, x86_featureset 3140 jnc .nocr4 3141 movl %cr4, %eax 3142 movl %eax, CREG_CR4(%edx) /* cr4 */ 3143 jmp .skip 3144.nocr4: 3145 movl $0, CREG_CR4(%edx) 3146.skip: 3147#endif 3148 ret 3149 SET_SIZE(getcregs) 3150 3151#endif /* __i386 */ 3152#endif /* __lint */ 3153 3154 3155/* 3156 * A panic trigger is a word which is updated atomically and can only be set 3157 * once. We atomically store 0xDEFACEDD and load the old value. If the 3158 * previous value was 0, we succeed and return 1; otherwise return 0. 3159 * This allows a partially corrupt trigger to still trigger correctly. DTrace 3160 * has its own version of this function to allow it to panic correctly from 3161 * probe context. 3162 */ 3163#if defined(__lint) 3164 3165/*ARGSUSED*/ 3166int 3167panic_trigger(int *tp) 3168{ return (0); } 3169 3170/*ARGSUSED*/ 3171int 3172dtrace_panic_trigger(int *tp) 3173{ return (0); } 3174 3175#else /* __lint */ 3176 3177#if defined(__amd64) 3178 3179 ENTRY_NP(panic_trigger) 3180 xorl %eax, %eax 3181 movl $0xdefacedd, %edx 3182 lock 3183 xchgl %edx, (%rdi) 3184 cmpl $0, %edx 3185 je 0f 3186 movl $0, %eax 3187 ret 31880: movl $1, %eax 3189 ret 3190 SET_SIZE(panic_trigger) 3191 3192 ENTRY_NP(dtrace_panic_trigger) 3193 xorl %eax, %eax 3194 movl $0xdefacedd, %edx 3195 lock 3196 xchgl %edx, (%rdi) 3197 cmpl $0, %edx 3198 je 0f 3199 movl $0, %eax 3200 ret 32010: movl $1, %eax 3202 ret 3203 SET_SIZE(dtrace_panic_trigger) 3204 3205#elif defined(__i386) 3206 3207 ENTRY_NP(panic_trigger) 3208 movl 4(%esp), %edx / %edx = address of trigger 3209 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3210 lock / assert lock 3211 xchgl %eax, (%edx) / exchange %eax and the trigger 3212 cmpl $0, %eax / if (%eax == 0x0) 3213 je 0f / return (1); 3214 movl $0, %eax / else 3215 ret / return (0); 32160: movl $1, %eax 3217 ret 3218 SET_SIZE(panic_trigger) 3219 3220 ENTRY_NP(dtrace_panic_trigger) 3221 movl 4(%esp), %edx / %edx = address of trigger 3222 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3223 lock / assert lock 3224 xchgl %eax, (%edx) / exchange %eax and the trigger 3225 cmpl $0, %eax / if (%eax == 0x0) 3226 je 0f / return (1); 3227 movl $0, %eax / else 3228 ret / return (0); 32290: movl $1, %eax 3230 ret 3231 SET_SIZE(dtrace_panic_trigger) 3232 3233#endif /* __i386 */ 3234#endif /* __lint */ 3235 3236/* 3237 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 3238 * into the panic code implemented in panicsys(). vpanic() is responsible 3239 * for passing through the format string and arguments, and constructing a 3240 * regs structure on the stack into which it saves the current register 3241 * values. If we are not dying due to a fatal trap, these registers will 3242 * then be preserved in panicbuf as the current processor state. Before 3243 * invoking panicsys(), vpanic() activates the first panic trigger (see 3244 * common/os/panic.c) and switches to the panic_stack if successful. Note that 3245 * DTrace takes a slightly different panic path if it must panic from probe 3246 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 3247 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 3248 * branches back into vpanic(). 3249 */ 3250#if defined(__lint) 3251 3252/*ARGSUSED*/ 3253void 3254vpanic(const char *format, va_list alist) 3255{} 3256 3257/*ARGSUSED*/ 3258void 3259dtrace_vpanic(const char *format, va_list alist) 3260{} 3261 3262#else /* __lint */ 3263 3264#if defined(__amd64) 3265 3266 ENTRY_NP(vpanic) /* Initial stack layout: */ 3267 3268 pushq %rbp /* | %rip | 0x60 */ 3269 movq %rsp, %rbp /* | %rbp | 0x58 */ 3270 pushfq /* | rfl | 0x50 */ 3271 pushq %r11 /* | %r11 | 0x48 */ 3272 pushq %r10 /* | %r10 | 0x40 */ 3273 pushq %rbx /* | %rbx | 0x38 */ 3274 pushq %rax /* | %rax | 0x30 */ 3275 pushq %r9 /* | %r9 | 0x28 */ 3276 pushq %r8 /* | %r8 | 0x20 */ 3277 pushq %rcx /* | %rcx | 0x18 */ 3278 pushq %rdx /* | %rdx | 0x10 */ 3279 pushq %rsi /* | %rsi | 0x8 alist */ 3280 pushq %rdi /* | %rdi | 0x0 format */ 3281 3282 movq %rsp, %rbx /* %rbx = current %rsp */ 3283 3284 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3285 call panic_trigger /* %eax = panic_trigger() */ 3286 3287vpanic_common: 3288 /* 3289 * The panic_trigger result is in %eax from the call above, and 3290 * dtrace_panic places it in %eax before branching here. 3291 * The rdmsr instructions that follow below will clobber %eax so 3292 * we stash the panic_trigger result in %r11d. 3293 */ 3294 movl %eax, %r11d 3295 cmpl $0, %r11d 3296 je 0f 3297 3298 /* 3299 * If panic_trigger() was successful, we are the first to initiate a 3300 * panic: we now switch to the reserved panic_stack before continuing. 3301 */ 3302 leaq panic_stack(%rip), %rsp 3303 addq $PANICSTKSIZE, %rsp 33040: subq $REGSIZE, %rsp 3305 /* 3306 * Now that we've got everything set up, store the register values as 3307 * they were when we entered vpanic() to the designated location in 3308 * the regs structure we allocated on the stack. 3309 */ 3310 movq 0x0(%rbx), %rcx 3311 movq %rcx, REGOFF_RDI(%rsp) 3312 movq 0x8(%rbx), %rcx 3313 movq %rcx, REGOFF_RSI(%rsp) 3314 movq 0x10(%rbx), %rcx 3315 movq %rcx, REGOFF_RDX(%rsp) 3316 movq 0x18(%rbx), %rcx 3317 movq %rcx, REGOFF_RCX(%rsp) 3318 movq 0x20(%rbx), %rcx 3319 3320 movq %rcx, REGOFF_R8(%rsp) 3321 movq 0x28(%rbx), %rcx 3322 movq %rcx, REGOFF_R9(%rsp) 3323 movq 0x30(%rbx), %rcx 3324 movq %rcx, REGOFF_RAX(%rsp) 3325 movq 0x38(%rbx), %rcx 3326 movq %rcx, REGOFF_RBX(%rsp) 3327 movq 0x58(%rbx), %rcx 3328 3329 movq %rcx, REGOFF_RBP(%rsp) 3330 movq 0x40(%rbx), %rcx 3331 movq %rcx, REGOFF_R10(%rsp) 3332 movq 0x48(%rbx), %rcx 3333 movq %rcx, REGOFF_R11(%rsp) 3334 movq %r12, REGOFF_R12(%rsp) 3335 3336 movq %r13, REGOFF_R13(%rsp) 3337 movq %r14, REGOFF_R14(%rsp) 3338 movq %r15, REGOFF_R15(%rsp) 3339 3340 xorl %ecx, %ecx 3341 movw %ds, %cx 3342 movq %rcx, REGOFF_DS(%rsp) 3343 movw %es, %cx 3344 movq %rcx, REGOFF_ES(%rsp) 3345 movw %fs, %cx 3346 movq %rcx, REGOFF_FS(%rsp) 3347 movw %gs, %cx 3348 movq %rcx, REGOFF_GS(%rsp) 3349 3350 movq $0, REGOFF_TRAPNO(%rsp) 3351 3352 movq $0, REGOFF_ERR(%rsp) 3353 leaq vpanic(%rip), %rcx 3354 movq %rcx, REGOFF_RIP(%rsp) 3355 movw %cs, %cx 3356 movzwq %cx, %rcx 3357 movq %rcx, REGOFF_CS(%rsp) 3358 movq 0x50(%rbx), %rcx 3359 movq %rcx, REGOFF_RFL(%rsp) 3360 movq %rbx, %rcx 3361 addq $0x60, %rcx 3362 movq %rcx, REGOFF_RSP(%rsp) 3363 movw %ss, %cx 3364 movzwq %cx, %rcx 3365 movq %rcx, REGOFF_SS(%rsp) 3366 3367 /* 3368 * panicsys(format, alist, rp, on_panic_stack) 3369 */ 3370 movq REGOFF_RDI(%rsp), %rdi /* format */ 3371 movq REGOFF_RSI(%rsp), %rsi /* alist */ 3372 movq %rsp, %rdx /* struct regs */ 3373 movl %r11d, %ecx /* on_panic_stack */ 3374 call panicsys 3375 addq $REGSIZE, %rsp 3376 popq %rdi 3377 popq %rsi 3378 popq %rdx 3379 popq %rcx 3380 popq %r8 3381 popq %r9 3382 popq %rax 3383 popq %rbx 3384 popq %r10 3385 popq %r11 3386 popfq 3387 leave 3388 ret 3389 SET_SIZE(vpanic) 3390 3391 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */ 3392 3393 pushq %rbp /* | %rip | 0x60 */ 3394 movq %rsp, %rbp /* | %rbp | 0x58 */ 3395 pushfq /* | rfl | 0x50 */ 3396 pushq %r11 /* | %r11 | 0x48 */ 3397 pushq %r10 /* | %r10 | 0x40 */ 3398 pushq %rbx /* | %rbx | 0x38 */ 3399 pushq %rax /* | %rax | 0x30 */ 3400 pushq %r9 /* | %r9 | 0x28 */ 3401 pushq %r8 /* | %r8 | 0x20 */ 3402 pushq %rcx /* | %rcx | 0x18 */ 3403 pushq %rdx /* | %rdx | 0x10 */ 3404 pushq %rsi /* | %rsi | 0x8 alist */ 3405 pushq %rdi /* | %rdi | 0x0 format */ 3406 3407 movq %rsp, %rbx /* %rbx = current %rsp */ 3408 3409 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3410 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 3411 jmp vpanic_common 3412 3413 SET_SIZE(dtrace_vpanic) 3414 3415#elif defined(__i386) 3416 3417 ENTRY_NP(vpanic) / Initial stack layout: 3418 3419 pushl %ebp / | %eip | 20 3420 movl %esp, %ebp / | %ebp | 16 3421 pushl %eax / | %eax | 12 3422 pushl %ebx / | %ebx | 8 3423 pushl %ecx / | %ecx | 4 3424 pushl %edx / | %edx | 0 3425 3426 movl %esp, %ebx / %ebx = current stack pointer 3427 3428 lea panic_quiesce, %eax / %eax = &panic_quiesce 3429 pushl %eax / push &panic_quiesce 3430 call panic_trigger / %eax = panic_trigger() 3431 addl $4, %esp / reset stack pointer 3432 3433vpanic_common: 3434 cmpl $0, %eax / if (%eax == 0) 3435 je 0f / goto 0f; 3436 3437 /* 3438 * If panic_trigger() was successful, we are the first to initiate a 3439 * panic: we now switch to the reserved panic_stack before continuing. 3440 */ 3441 lea panic_stack, %esp / %esp = panic_stack 3442 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE 3443 34440: subl $REGSIZE, %esp / allocate struct regs 3445 3446 /* 3447 * Now that we've got everything set up, store the register values as 3448 * they were when we entered vpanic() to the designated location in 3449 * the regs structure we allocated on the stack. 3450 */ 3451#if !defined(__GNUC_AS__) 3452 movw %gs, %edx 3453 movl %edx, REGOFF_GS(%esp) 3454 movw %fs, %edx 3455 movl %edx, REGOFF_FS(%esp) 3456 movw %es, %edx 3457 movl %edx, REGOFF_ES(%esp) 3458 movw %ds, %edx 3459 movl %edx, REGOFF_DS(%esp) 3460#else /* __GNUC_AS__ */ 3461 mov %gs, %edx 3462 mov %edx, REGOFF_GS(%esp) 3463 mov %fs, %edx 3464 mov %edx, REGOFF_FS(%esp) 3465 mov %es, %edx 3466 mov %edx, REGOFF_ES(%esp) 3467 mov %ds, %edx 3468 mov %edx, REGOFF_DS(%esp) 3469#endif /* __GNUC_AS__ */ 3470 movl %edi, REGOFF_EDI(%esp) 3471 movl %esi, REGOFF_ESI(%esp) 3472 movl 16(%ebx), %ecx 3473 movl %ecx, REGOFF_EBP(%esp) 3474 movl %ebx, %ecx 3475 addl $20, %ecx 3476 movl %ecx, REGOFF_ESP(%esp) 3477 movl 8(%ebx), %ecx 3478 movl %ecx, REGOFF_EBX(%esp) 3479 movl 0(%ebx), %ecx 3480 movl %ecx, REGOFF_EDX(%esp) 3481 movl 4(%ebx), %ecx 3482 movl %ecx, REGOFF_ECX(%esp) 3483 movl 12(%ebx), %ecx 3484 movl %ecx, REGOFF_EAX(%esp) 3485 movl $0, REGOFF_TRAPNO(%esp) 3486 movl $0, REGOFF_ERR(%esp) 3487 lea vpanic, %ecx 3488 movl %ecx, REGOFF_EIP(%esp) 3489#if !defined(__GNUC_AS__) 3490 movw %cs, %edx 3491#else /* __GNUC_AS__ */ 3492 mov %cs, %edx 3493#endif /* __GNUC_AS__ */ 3494 movl %edx, REGOFF_CS(%esp) 3495 pushfl 3496 popl %ecx 3497#if defined(__xpv) 3498 /* 3499 * Synthesize the PS_IE bit from the event mask bit 3500 */ 3501 CURTHREAD(%edx) 3502 KPREEMPT_DISABLE(%edx) 3503 EVENT_MASK_TO_IE(%edx, %ecx) 3504 CURTHREAD(%edx) 3505 KPREEMPT_ENABLE_NOKP(%edx) 3506#endif 3507 movl %ecx, REGOFF_EFL(%esp) 3508 movl $0, REGOFF_UESP(%esp) 3509#if !defined(__GNUC_AS__) 3510 movw %ss, %edx 3511#else /* __GNUC_AS__ */ 3512 mov %ss, %edx 3513#endif /* __GNUC_AS__ */ 3514 movl %edx, REGOFF_SS(%esp) 3515 3516 movl %esp, %ecx / %ecx = ®s 3517 pushl %eax / push on_panic_stack 3518 pushl %ecx / push ®s 3519 movl 12(%ebp), %ecx / %ecx = alist 3520 pushl %ecx / push alist 3521 movl 8(%ebp), %ecx / %ecx = format 3522 pushl %ecx / push format 3523 call panicsys / panicsys(); 3524 addl $16, %esp / pop arguments 3525 3526 addl $REGSIZE, %esp 3527 popl %edx 3528 popl %ecx 3529 popl %ebx 3530 popl %eax 3531 leave 3532 ret 3533 SET_SIZE(vpanic) 3534 3535 ENTRY_NP(dtrace_vpanic) / Initial stack layout: 3536 3537 pushl %ebp / | %eip | 20 3538 movl %esp, %ebp / | %ebp | 16 3539 pushl %eax / | %eax | 12 3540 pushl %ebx / | %ebx | 8 3541 pushl %ecx / | %ecx | 4 3542 pushl %edx / | %edx | 0 3543 3544 movl %esp, %ebx / %ebx = current stack pointer 3545 3546 lea panic_quiesce, %eax / %eax = &panic_quiesce 3547 pushl %eax / push &panic_quiesce 3548 call dtrace_panic_trigger / %eax = dtrace_panic_trigger() 3549 addl $4, %esp / reset stack pointer 3550 jmp vpanic_common / jump back to common code 3551 3552 SET_SIZE(dtrace_vpanic) 3553 3554#endif /* __i386 */ 3555#endif /* __lint */ 3556 3557#if defined(__lint) 3558 3559void 3560hres_tick(void) 3561{} 3562 3563int64_t timedelta; 3564hrtime_t hres_last_tick; 3565volatile timestruc_t hrestime; 3566int64_t hrestime_adj; 3567volatile int hres_lock; 3568hrtime_t hrtime_base; 3569 3570#else /* __lint */ 3571 3572 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8) 3573 .NWORD 0, 0 3574 3575 DGDEF3(hrestime_adj, 8, 8) 3576 .long 0, 0 3577 3578 DGDEF3(hres_last_tick, 8, 8) 3579 .long 0, 0 3580 3581 DGDEF3(timedelta, 8, 8) 3582 .long 0, 0 3583 3584 DGDEF3(hres_lock, 4, 8) 3585 .long 0 3586 3587 /* 3588 * initialized to a non zero value to make pc_gethrtime() 3589 * work correctly even before clock is initialized 3590 */ 3591 DGDEF3(hrtime_base, 8, 8) 3592 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0 3593 3594 DGDEF3(adj_shift, 4, 4) 3595 .long ADJ_SHIFT 3596 3597#if defined(__amd64) 3598 3599 ENTRY_NP(hres_tick) 3600 pushq %rbp 3601 movq %rsp, %rbp 3602 3603 /* 3604 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3605 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3606 * At worst, performing this now instead of under CLOCK_LOCK may 3607 * introduce some jitter in pc_gethrestime(). 3608 */ 3609 call *gethrtimef(%rip) 3610 movq %rax, %r8 3611 3612 leaq hres_lock(%rip), %rax 3613 movb $-1, %dl 3614.CL1: 3615 xchgb %dl, (%rax) 3616 testb %dl, %dl 3617 jz .CL3 /* got it */ 3618.CL2: 3619 cmpb $0, (%rax) /* possible to get lock? */ 3620 pause 3621 jne .CL2 3622 jmp .CL1 /* yes, try again */ 3623.CL3: 3624 /* 3625 * compute the interval since last time hres_tick was called 3626 * and adjust hrtime_base and hrestime accordingly 3627 * hrtime_base is an 8 byte value (in nsec), hrestime is 3628 * a timestruc_t (sec, nsec) 3629 */ 3630 leaq hres_last_tick(%rip), %rax 3631 movq %r8, %r11 3632 subq (%rax), %r8 3633 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */ 3634 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */ 3635 /* 3636 * Now that we have CLOCK_LOCK, we can update hres_last_tick 3637 */ 3638 movq %r11, (%rax) 3639 3640 call __adj_hrestime 3641 3642 /* 3643 * release the hres_lock 3644 */ 3645 incl hres_lock(%rip) 3646 leave 3647 ret 3648 SET_SIZE(hres_tick) 3649 3650#elif defined(__i386) 3651 3652 ENTRY_NP(hres_tick) 3653 pushl %ebp 3654 movl %esp, %ebp 3655 pushl %esi 3656 pushl %ebx 3657 3658 /* 3659 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3660 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3661 * At worst, performing this now instead of under CLOCK_LOCK may 3662 * introduce some jitter in pc_gethrestime(). 3663 */ 3664 call *gethrtimef 3665 movl %eax, %ebx 3666 movl %edx, %esi 3667 3668 movl $hres_lock, %eax 3669 movl $-1, %edx 3670.CL1: 3671 xchgb %dl, (%eax) 3672 testb %dl, %dl 3673 jz .CL3 / got it 3674.CL2: 3675 cmpb $0, (%eax) / possible to get lock? 3676 pause 3677 jne .CL2 3678 jmp .CL1 / yes, try again 3679.CL3: 3680 /* 3681 * compute the interval since last time hres_tick was called 3682 * and adjust hrtime_base and hrestime accordingly 3683 * hrtime_base is an 8 byte value (in nsec), hrestime is 3684 * timestruc_t (sec, nsec) 3685 */ 3686 3687 lea hres_last_tick, %eax 3688 3689 movl %ebx, %edx 3690 movl %esi, %ecx 3691 3692 subl (%eax), %edx 3693 sbbl 4(%eax), %ecx 3694 3695 addl %edx, hrtime_base / add interval to hrtime_base 3696 adcl %ecx, hrtime_base+4 3697 3698 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec 3699 3700 / 3701 / Now that we have CLOCK_LOCK, we can update hres_last_tick. 3702 / 3703 movl %ebx, (%eax) 3704 movl %esi, 4(%eax) 3705 3706 / get hrestime at this moment. used as base for pc_gethrestime 3707 / 3708 / Apply adjustment, if any 3709 / 3710 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT) 3711 / (max_hres_adj) 3712 / 3713 / void 3714 / adj_hrestime() 3715 / { 3716 / long long adj; 3717 / 3718 / if (hrestime_adj == 0) 3719 / adj = 0; 3720 / else if (hrestime_adj > 0) { 3721 / if (hrestime_adj < HRES_ADJ) 3722 / adj = hrestime_adj; 3723 / else 3724 / adj = HRES_ADJ; 3725 / } 3726 / else { 3727 / if (hrestime_adj < -(HRES_ADJ)) 3728 / adj = -(HRES_ADJ); 3729 / else 3730 / adj = hrestime_adj; 3731 / } 3732 / 3733 / timedelta -= adj; 3734 / hrestime_adj = timedelta; 3735 / hrestime.tv_nsec += adj; 3736 / 3737 / while (hrestime.tv_nsec >= NANOSEC) { 3738 / one_sec++; 3739 / hrestime.tv_sec++; 3740 / hrestime.tv_nsec -= NANOSEC; 3741 / } 3742 / } 3743__adj_hrestime: 3744 movl hrestime_adj, %esi / if (hrestime_adj == 0) 3745 movl hrestime_adj+4, %edx 3746 andl %esi, %esi 3747 jne .CL4 / no 3748 andl %edx, %edx 3749 jne .CL4 / no 3750 subl %ecx, %ecx / yes, adj = 0; 3751 subl %edx, %edx 3752 jmp .CL5 3753.CL4: 3754 subl %ecx, %ecx 3755 subl %eax, %eax 3756 subl %esi, %ecx 3757 sbbl %edx, %eax 3758 andl %eax, %eax / if (hrestime_adj > 0) 3759 jge .CL6 3760 3761 / In the following comments, HRES_ADJ is used, while in the code 3762 / max_hres_adj is used. 3763 / 3764 / The test for "hrestime_adj < HRES_ADJ" is complicated because 3765 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3766 / on the logical equivalence of: 3767 / 3768 / !(hrestime_adj < HRES_ADJ) 3769 / 3770 / and the two step sequence: 3771 / 3772 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry 3773 / 3774 / which computes whether or not the least significant 32-bits 3775 / of hrestime_adj is greater than HRES_ADJ, followed by: 3776 / 3777 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry 3778 / 3779 / which generates a carry whenever step 1 is true or the most 3780 / significant long of the longlong hrestime_adj is non-zero. 3781 3782 movl max_hres_adj, %ecx / hrestime_adj is positive 3783 subl %esi, %ecx 3784 movl %edx, %eax 3785 adcl $-1, %eax 3786 jnc .CL7 3787 movl max_hres_adj, %ecx / adj = HRES_ADJ; 3788 subl %edx, %edx 3789 jmp .CL5 3790 3791 / The following computation is similar to the one above. 3792 / 3793 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because 3794 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3795 / on the logical equivalence of: 3796 / 3797 / (hrestime_adj > -HRES_ADJ) 3798 / 3799 / and the two step sequence: 3800 / 3801 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry 3802 / 3803 / which means the least significant 32-bits of hrestime_adj is 3804 / greater than -HRES_ADJ, followed by: 3805 / 3806 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry 3807 / 3808 / which generates a carry only when step 1 is true and the most 3809 / significant long of the longlong hrestime_adj is -1. 3810 3811.CL6: / hrestime_adj is negative 3812 movl %esi, %ecx 3813 addl max_hres_adj, %ecx 3814 movl %edx, %eax 3815 adcl $0, %eax 3816 jc .CL7 3817 xor %ecx, %ecx 3818 subl max_hres_adj, %ecx / adj = -(HRES_ADJ); 3819 movl $-1, %edx 3820 jmp .CL5 3821.CL7: 3822 movl %esi, %ecx / adj = hrestime_adj; 3823.CL5: 3824 movl timedelta, %esi 3825 subl %ecx, %esi 3826 movl timedelta+4, %eax 3827 sbbl %edx, %eax 3828 movl %esi, timedelta 3829 movl %eax, timedelta+4 / timedelta -= adj; 3830 movl %esi, hrestime_adj 3831 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta; 3832 addl hrestime+4, %ecx 3833 3834 movl %ecx, %eax / eax = tv_nsec 38351: 3836 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC) 3837 jb .CL8 / no 3838 incl one_sec / yes, one_sec++; 3839 incl hrestime / hrestime.tv_sec++; 3840 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC 3841 jmp 1b / check for more seconds 3842 3843.CL8: 3844 movl %eax, hrestime+4 / store final into hrestime.tv_nsec 3845 incl hres_lock / release the hres_lock 3846 3847 popl %ebx 3848 popl %esi 3849 leave 3850 ret 3851 SET_SIZE(hres_tick) 3852 3853#endif /* __i386 */ 3854#endif /* __lint */ 3855 3856/* 3857 * void prefetch_smap_w(void *) 3858 * 3859 * Prefetch ahead within a linear list of smap structures. 3860 * Not implemented for ia32. Stub for compatibility. 3861 */ 3862 3863#if defined(__lint) 3864 3865/*ARGSUSED*/ 3866void prefetch_smap_w(void *smp) 3867{} 3868 3869#else /* __lint */ 3870 3871 ENTRY(prefetch_smap_w) 3872 rep; ret /* use 2 byte return instruction when branch target */ 3873 /* AMD Software Optimization Guide - Section 6.2 */ 3874 SET_SIZE(prefetch_smap_w) 3875 3876#endif /* __lint */ 3877 3878/* 3879 * prefetch_page_r(page_t *) 3880 * issue prefetch instructions for a page_t 3881 */ 3882#if defined(__lint) 3883 3884/*ARGSUSED*/ 3885void 3886prefetch_page_r(void *pp) 3887{} 3888 3889#else /* __lint */ 3890 3891 ENTRY(prefetch_page_r) 3892 rep; ret /* use 2 byte return instruction when branch target */ 3893 /* AMD Software Optimization Guide - Section 6.2 */ 3894 SET_SIZE(prefetch_page_r) 3895 3896#endif /* __lint */ 3897 3898#if defined(__lint) 3899 3900/*ARGSUSED*/ 3901int 3902bcmp(const void *s1, const void *s2, size_t count) 3903{ return (0); } 3904 3905#else /* __lint */ 3906 3907#if defined(__amd64) 3908 3909 ENTRY(bcmp) 3910 pushq %rbp 3911 movq %rsp, %rbp 3912#ifdef DEBUG 3913 movq postbootkernelbase(%rip), %r11 3914 cmpq %r11, %rdi 3915 jb 0f 3916 cmpq %r11, %rsi 3917 jnb 1f 39180: leaq .bcmp_panic_msg(%rip), %rdi 3919 xorl %eax, %eax 3920 call panic 39211: 3922#endif /* DEBUG */ 3923 call memcmp 3924 testl %eax, %eax 3925 setne %dl 3926 leave 3927 movzbl %dl, %eax 3928 ret 3929 SET_SIZE(bcmp) 3930 3931#elif defined(__i386) 3932 3933#define ARG_S1 8 3934#define ARG_S2 12 3935#define ARG_LENGTH 16 3936 3937 ENTRY(bcmp) 3938 pushl %ebp 3939 movl %esp, %ebp / create new stack frame 3940#ifdef DEBUG 3941 movl postbootkernelbase, %eax 3942 cmpl %eax, ARG_S1(%ebp) 3943 jb 0f 3944 cmpl %eax, ARG_S2(%ebp) 3945 jnb 1f 39460: pushl $.bcmp_panic_msg 3947 call panic 39481: 3949#endif /* DEBUG */ 3950 3951 pushl %edi / save register variable 3952 movl ARG_S1(%ebp), %eax / %eax = address of string 1 3953 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2 3954 cmpl %eax, %ecx / if the same string 3955 je .equal / goto .equal 3956 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes 3957 cmpl $4, %edi / if %edi < 4 3958 jb .byte_check / goto .byte_check 3959 .align 4 3960.word_loop: 3961 movl (%ecx), %edx / move 1 word from (%ecx) to %edx 3962 leal -4(%edi), %edi / %edi -= 4 3963 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx 3964 jne .word_not_equal / if not equal, goto .word_not_equal 3965 leal 4(%ecx), %ecx / %ecx += 4 (next word) 3966 leal 4(%eax), %eax / %eax += 4 (next word) 3967 cmpl $4, %edi / if %edi >= 4 3968 jae .word_loop / goto .word_loop 3969.byte_check: 3970 cmpl $0, %edi / if %edi == 0 3971 je .equal / goto .equal 3972 jmp .byte_loop / goto .byte_loop (checks in bytes) 3973.word_not_equal: 3974 leal 4(%edi), %edi / %edi += 4 (post-decremented) 3975 .align 4 3976.byte_loop: 3977 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl 3978 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax) 3979 jne .not_equal / if not equal, goto .not_equal 3980 incl %ecx / %ecx++ (next byte) 3981 incl %eax / %eax++ (next byte) 3982 decl %edi / %edi-- 3983 jnz .byte_loop / if not zero, goto .byte_loop 3984.equal: 3985 xorl %eax, %eax / %eax = 0 3986 popl %edi / restore register variable 3987 leave / restore old stack frame 3988 ret / return (NULL) 3989 .align 4 3990.not_equal: 3991 movl $1, %eax / return 1 3992 popl %edi / restore register variable 3993 leave / restore old stack frame 3994 ret / return (NULL) 3995 SET_SIZE(bcmp) 3996 3997#endif /* __i386 */ 3998 3999#ifdef DEBUG 4000 .text 4001.bcmp_panic_msg: 4002 .string "bcmp: arguments below kernelbase" 4003#endif /* DEBUG */ 4004 4005#endif /* __lint */ 4006 4007#if defined(__lint) 4008 4009uint_t 4010bsrw_insn(uint16_t mask) 4011{ 4012 uint_t index = sizeof (mask) * NBBY - 1; 4013 4014 while ((mask & (1 << index)) == 0) 4015 index--; 4016 return (index); 4017} 4018 4019#else /* __lint */ 4020 4021#if defined(__amd64) 4022 4023 ENTRY_NP(bsrw_insn) 4024 xorl %eax, %eax 4025 bsrw %di, %ax 4026 ret 4027 SET_SIZE(bsrw_insn) 4028 4029#elif defined(__i386) 4030 4031 ENTRY_NP(bsrw_insn) 4032 movw 4(%esp), %cx 4033 xorl %eax, %eax 4034 bsrw %cx, %ax 4035 ret 4036 SET_SIZE(bsrw_insn) 4037 4038#endif /* __i386 */ 4039#endif /* __lint */ 4040 4041#if defined(__lint) 4042 4043uint_t 4044atomic_btr32(uint32_t *pending, uint_t pil) 4045{ 4046 return (*pending &= ~(1 << pil)); 4047} 4048 4049#else /* __lint */ 4050 4051#if defined(__i386) 4052 4053 ENTRY_NP(atomic_btr32) 4054 movl 4(%esp), %ecx 4055 movl 8(%esp), %edx 4056 xorl %eax, %eax 4057 lock 4058 btrl %edx, (%ecx) 4059 setc %al 4060 ret 4061 SET_SIZE(atomic_btr32) 4062 4063#endif /* __i386 */ 4064#endif /* __lint */ 4065 4066#if defined(__lint) 4067 4068/*ARGSUSED*/ 4069void 4070switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1, 4071 uint_t arg2) 4072{} 4073 4074#else /* __lint */ 4075 4076#if defined(__amd64) 4077 4078 ENTRY_NP(switch_sp_and_call) 4079 pushq %rbp 4080 movq %rsp, %rbp /* set up stack frame */ 4081 movq %rdi, %rsp /* switch stack pointer */ 4082 movq %rdx, %rdi /* pass func arg 1 */ 4083 movq %rsi, %r11 /* save function to call */ 4084 movq %rcx, %rsi /* pass func arg 2 */ 4085 call *%r11 /* call function */ 4086 leave /* restore stack */ 4087 ret 4088 SET_SIZE(switch_sp_and_call) 4089 4090#elif defined(__i386) 4091 4092 ENTRY_NP(switch_sp_and_call) 4093 pushl %ebp 4094 mov %esp, %ebp /* set up stack frame */ 4095 movl 8(%ebp), %esp /* switch stack pointer */ 4096 pushl 20(%ebp) /* push func arg 2 */ 4097 pushl 16(%ebp) /* push func arg 1 */ 4098 call *12(%ebp) /* call function */ 4099 addl $8, %esp /* pop arguments */ 4100 leave /* restore stack */ 4101 ret 4102 SET_SIZE(switch_sp_and_call) 4103 4104#endif /* __i386 */ 4105#endif /* __lint */ 4106 4107#if defined(__lint) 4108 4109void 4110kmdb_enter(void) 4111{} 4112 4113#else /* __lint */ 4114 4115#if defined(__amd64) 4116 4117 ENTRY_NP(kmdb_enter) 4118 pushq %rbp 4119 movq %rsp, %rbp 4120 4121 /* 4122 * Save flags, do a 'cli' then return the saved flags 4123 */ 4124 call intr_clear 4125 4126 int $T_DBGENTR 4127 4128 /* 4129 * Restore the saved flags 4130 */ 4131 movq %rax, %rdi 4132 call intr_restore 4133 4134 leave 4135 ret 4136 SET_SIZE(kmdb_enter) 4137 4138#elif defined(__i386) 4139 4140 ENTRY_NP(kmdb_enter) 4141 pushl %ebp 4142 movl %esp, %ebp 4143 4144 /* 4145 * Save flags, do a 'cli' then return the saved flags 4146 */ 4147 call intr_clear 4148 4149 int $T_DBGENTR 4150 4151 /* 4152 * Restore the saved flags 4153 */ 4154 pushl %eax 4155 call intr_restore 4156 addl $4, %esp 4157 4158 leave 4159 ret 4160 SET_SIZE(kmdb_enter) 4161 4162#endif /* __i386 */ 4163#endif /* __lint */ 4164 4165#if defined(__lint) 4166 4167void 4168return_instr(void) 4169{} 4170 4171#else /* __lint */ 4172 4173 ENTRY_NP(return_instr) 4174 rep; ret /* use 2 byte instruction when branch target */ 4175 /* AMD Software Optimization Guide - Section 6.2 */ 4176 SET_SIZE(return_instr) 4177 4178#endif /* __lint */ 4179 4180#if defined(__lint) 4181 4182ulong_t 4183getflags(void) 4184{ 4185 return (0); 4186} 4187 4188#else /* __lint */ 4189 4190#if defined(__amd64) 4191 4192 ENTRY(getflags) 4193 pushfq 4194 popq %rax 4195#if defined(__xpv) 4196 CURTHREAD(%rdi) 4197 KPREEMPT_DISABLE(%rdi) 4198 /* 4199 * Synthesize the PS_IE bit from the event mask bit 4200 */ 4201 CURVCPU(%r11) 4202 andq $_BITNOT(PS_IE), %rax 4203 XEN_TEST_UPCALL_MASK(%r11) 4204 jnz 1f 4205 orq $PS_IE, %rax 42061: 4207 KPREEMPT_ENABLE_NOKP(%rdi) 4208#endif 4209 ret 4210 SET_SIZE(getflags) 4211 4212#elif defined(__i386) 4213 4214 ENTRY(getflags) 4215 pushfl 4216 popl %eax 4217#if defined(__xpv) 4218 CURTHREAD(%ecx) 4219 KPREEMPT_DISABLE(%ecx) 4220 /* 4221 * Synthesize the PS_IE bit from the event mask bit 4222 */ 4223 CURVCPU(%edx) 4224 andl $_BITNOT(PS_IE), %eax 4225 XEN_TEST_UPCALL_MASK(%edx) 4226 jnz 1f 4227 orl $PS_IE, %eax 42281: 4229 KPREEMPT_ENABLE_NOKP(%ecx) 4230#endif 4231 ret 4232 SET_SIZE(getflags) 4233 4234#endif /* __i386 */ 4235 4236#endif /* __lint */ 4237 4238#if defined(__lint) 4239 4240ftrace_icookie_t 4241ftrace_interrupt_disable(void) 4242{ return (0); } 4243 4244#else /* __lint */ 4245 4246#if defined(__amd64) 4247 4248 ENTRY(ftrace_interrupt_disable) 4249 pushfq 4250 popq %rax 4251 CLI(%rdx) 4252 ret 4253 SET_SIZE(ftrace_interrupt_disable) 4254 4255#elif defined(__i386) 4256 4257 ENTRY(ftrace_interrupt_disable) 4258 pushfl 4259 popl %eax 4260 CLI(%edx) 4261 ret 4262 SET_SIZE(ftrace_interrupt_disable) 4263 4264#endif /* __i386 */ 4265#endif /* __lint */ 4266 4267#if defined(__lint) 4268 4269/*ARGSUSED*/ 4270void 4271ftrace_interrupt_enable(ftrace_icookie_t cookie) 4272{} 4273 4274#else /* __lint */ 4275 4276#if defined(__amd64) 4277 4278 ENTRY(ftrace_interrupt_enable) 4279 pushq %rdi 4280 popfq 4281 ret 4282 SET_SIZE(ftrace_interrupt_enable) 4283 4284#elif defined(__i386) 4285 4286 ENTRY(ftrace_interrupt_enable) 4287 movl 4(%esp), %eax 4288 pushl %eax 4289 popfl 4290 ret 4291 SET_SIZE(ftrace_interrupt_enable) 4292 4293#endif /* __i386 */ 4294#endif /* __lint */ 4295 4296#if defined (__lint) 4297 4298/*ARGSUSED*/ 4299void 4300clflush_insn(caddr_t addr) 4301{} 4302 4303#else /* __lint */ 4304 4305#if defined (__amd64) 4306 ENTRY(clflush_insn) 4307 clflush (%rdi) 4308 ret 4309 SET_SIZE(clflush_insn) 4310#elif defined (__i386) 4311 ENTRY(clflush_insn) 4312 movl 4(%esp), %eax 4313 clflush (%eax) 4314 ret 4315 SET_SIZE(clflush_insn) 4316 4317#endif /* __i386 */ 4318#endif /* __lint */ 4319 4320#if defined (__lint) 4321/*ARGSUSED*/ 4322void 4323mfence_insn(void) 4324{} 4325 4326#else /* __lint */ 4327 4328#if defined (__amd64) 4329 ENTRY(mfence_insn) 4330 mfence 4331 ret 4332 SET_SIZE(mfence_insn) 4333#elif defined (__i386) 4334 ENTRY(mfence_insn) 4335 mfence 4336 ret 4337 SET_SIZE(mfence_insn) 4338 4339#endif /* __i386 */ 4340#endif /* __lint */ 4341 4342/* 4343 * VMware implements an I/O port that programs can query to detect if software 4344 * is running in a VMware hypervisor. This hypervisor port behaves differently 4345 * depending on magic values in certain registers and modifies some registers 4346 * as a side effect. 4347 * 4348 * References: http://kb.vmware.com/kb/1009458 4349 */ 4350 4351#if defined(__lint) 4352 4353/* ARGSUSED */ 4354void 4355vmware_port(int cmd, uint32_t *regs) { return; } 4356 4357#else 4358 4359#if defined(__amd64) 4360 4361 ENTRY(vmware_port) 4362 pushq %rbx 4363 movl $VMWARE_HVMAGIC, %eax 4364 movl $0xffffffff, %ebx 4365 movl %edi, %ecx 4366 movl $VMWARE_HVPORT, %edx 4367 inl (%dx) 4368 movl %eax, (%rsi) 4369 movl %ebx, 4(%rsi) 4370 movl %ecx, 8(%rsi) 4371 movl %edx, 12(%rsi) 4372 popq %rbx 4373 ret 4374 SET_SIZE(vmware_port) 4375 4376#elif defined(__i386) 4377 4378 ENTRY(vmware_port) 4379 pushl %ebx 4380 pushl %esi 4381 movl $VMWARE_HVMAGIC, %eax 4382 movl $0xffffffff, %ebx 4383 movl 12(%esp), %ecx 4384 movl $VMWARE_HVPORT, %edx 4385 inl (%dx) 4386 movl 16(%esp), %esi 4387 movl %eax, (%esi) 4388 movl %ebx, 4(%esi) 4389 movl %ecx, 8(%esi) 4390 movl %edx, 12(%esi) 4391 popl %esi 4392 popl %ebx 4393 ret 4394 SET_SIZE(vmware_port) 4395 4396#endif /* __i386 */ 4397#endif /* __lint */ 4398