1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2014 by Delphix. All rights reserved. 26 */ 27 28/* 29 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. 30 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T 31 * All Rights Reserved 32 */ 33 34/* 35 * Copyright (c) 2009, Intel Corporation. 36 * All rights reserved. 37 */ 38 39/* 40 * General assembly language routines. 41 * It is the intent of this file to contain routines that are 42 * independent of the specific kernel architecture, and those that are 43 * common across kernel architectures. 44 * As architectures diverge, and implementations of specific 45 * architecture-dependent routines change, the routines should be moved 46 * from this file into the respective ../`arch -k`/subr.s file. 47 */ 48 49#include <sys/asm_linkage.h> 50#include <sys/asm_misc.h> 51#include <sys/panic.h> 52#include <sys/ontrap.h> 53#include <sys/regset.h> 54#include <sys/privregs.h> 55#include <sys/reboot.h> 56#include <sys/psw.h> 57#include <sys/x86_archext.h> 58 59#if defined(__lint) 60#include <sys/types.h> 61#include <sys/systm.h> 62#include <sys/thread.h> 63#include <sys/archsystm.h> 64#include <sys/byteorder.h> 65#include <sys/dtrace.h> 66#include <sys/ftrace.h> 67#else /* __lint */ 68#include "assym.h" 69#endif /* __lint */ 70#include <sys/dditypes.h> 71 72/* 73 * on_fault() 74 * Catch lofault faults. Like setjmp except it returns one 75 * if code following causes uncorrectable fault. Turned off 76 * by calling no_fault(). 77 */ 78 79#if defined(__lint) 80 81/* ARGSUSED */ 82int 83on_fault(label_t *ljb) 84{ return (0); } 85 86void 87no_fault(void) 88{} 89 90#else /* __lint */ 91 92#if defined(__amd64) 93 94 ENTRY(on_fault) 95 movq %gs:CPU_THREAD, %rsi 96 leaq catch_fault(%rip), %rdx 97 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */ 98 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */ 99 jmp setjmp /* let setjmp do the rest */ 100 101catch_fault: 102 movq %gs:CPU_THREAD, %rsi 103 movq T_ONFAULT(%rsi), %rdi /* address of save area */ 104 xorl %eax, %eax 105 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 106 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 107 jmp longjmp /* let longjmp do the rest */ 108 SET_SIZE(on_fault) 109 110 ENTRY(no_fault) 111 movq %gs:CPU_THREAD, %rsi 112 xorl %eax, %eax 113 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 114 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 115 ret 116 SET_SIZE(no_fault) 117 118#elif defined(__i386) 119 120 ENTRY(on_fault) 121 movl %gs:CPU_THREAD, %edx 122 movl 4(%esp), %eax /* jumpbuf address */ 123 leal catch_fault, %ecx 124 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */ 125 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */ 126 jmp setjmp /* let setjmp do the rest */ 127 128catch_fault: 129 movl %gs:CPU_THREAD, %edx 130 xorl %eax, %eax 131 movl T_ONFAULT(%edx), %ecx /* address of save area */ 132 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 133 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 134 pushl %ecx 135 call longjmp /* let longjmp do the rest */ 136 SET_SIZE(on_fault) 137 138 ENTRY(no_fault) 139 movl %gs:CPU_THREAD, %edx 140 xorl %eax, %eax 141 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 142 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 143 ret 144 SET_SIZE(no_fault) 145 146#endif /* __i386 */ 147#endif /* __lint */ 148 149/* 150 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just 151 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called. 152 */ 153 154#if defined(lint) 155 156void 157on_trap_trampoline(void) 158{} 159 160#else /* __lint */ 161 162#if defined(__amd64) 163 164 ENTRY(on_trap_trampoline) 165 movq %gs:CPU_THREAD, %rsi 166 movq T_ONTRAP(%rsi), %rdi 167 addq $OT_JMPBUF, %rdi 168 jmp longjmp 169 SET_SIZE(on_trap_trampoline) 170 171#elif defined(__i386) 172 173 ENTRY(on_trap_trampoline) 174 movl %gs:CPU_THREAD, %eax 175 movl T_ONTRAP(%eax), %eax 176 addl $OT_JMPBUF, %eax 177 pushl %eax 178 call longjmp 179 SET_SIZE(on_trap_trampoline) 180 181#endif /* __i386 */ 182#endif /* __lint */ 183 184/* 185 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 186 * more information about the on_trap() mechanism. If the on_trap_data is the 187 * same as the topmost stack element, we just modify that element. 188 */ 189#if defined(lint) 190 191/*ARGSUSED*/ 192int 193on_trap(on_trap_data_t *otp, uint_t prot) 194{ return (0); } 195 196#else /* __lint */ 197 198#if defined(__amd64) 199 200 ENTRY(on_trap) 201 movw %si, OT_PROT(%rdi) /* ot_prot = prot */ 202 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */ 203 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */ 204 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */ 205 xorl %ecx, %ecx 206 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */ 207 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */ 208 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */ 209 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */ 210 cmpq %rdi, %rcx /* if (otp == %rcx) */ 211 je 0f /* don't modify t_ontrap */ 212 213 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */ 214 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */ 215 2160: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */ 217 jmp setjmp 218 SET_SIZE(on_trap) 219 220#elif defined(__i386) 221 222 ENTRY(on_trap) 223 movl 4(%esp), %eax /* %eax = otp */ 224 movl 8(%esp), %edx /* %edx = prot */ 225 226 movw %dx, OT_PROT(%eax) /* ot_prot = prot */ 227 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */ 228 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */ 229 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */ 230 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */ 231 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */ 232 movl %gs:CPU_THREAD, %edx /* %edx = curthread */ 233 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */ 234 cmpl %eax, %ecx /* if (otp == %ecx) */ 235 je 0f /* don't modify t_ontrap */ 236 237 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */ 238 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */ 239 2400: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */ 241 movl %eax, 4(%esp) /* put %eax back on the stack */ 242 jmp setjmp /* let setjmp do the rest */ 243 SET_SIZE(on_trap) 244 245#endif /* __i386 */ 246#endif /* __lint */ 247 248/* 249 * Setjmp and longjmp implement non-local gotos using state vectors 250 * type label_t. 251 */ 252 253#if defined(__lint) 254 255/* ARGSUSED */ 256int 257setjmp(label_t *lp) 258{ return (0); } 259 260/* ARGSUSED */ 261void 262longjmp(label_t *lp) 263{} 264 265#else /* __lint */ 266 267#if LABEL_PC != 0 268#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded 269#endif /* LABEL_PC != 0 */ 270 271#if defined(__amd64) 272 273 ENTRY(setjmp) 274 movq %rsp, LABEL_SP(%rdi) 275 movq %rbp, LABEL_RBP(%rdi) 276 movq %rbx, LABEL_RBX(%rdi) 277 movq %r12, LABEL_R12(%rdi) 278 movq %r13, LABEL_R13(%rdi) 279 movq %r14, LABEL_R14(%rdi) 280 movq %r15, LABEL_R15(%rdi) 281 movq (%rsp), %rdx /* return address */ 282 movq %rdx, (%rdi) /* LABEL_PC is 0 */ 283 xorl %eax, %eax /* return 0 */ 284 ret 285 SET_SIZE(setjmp) 286 287 ENTRY(longjmp) 288 movq LABEL_SP(%rdi), %rsp 289 movq LABEL_RBP(%rdi), %rbp 290 movq LABEL_RBX(%rdi), %rbx 291 movq LABEL_R12(%rdi), %r12 292 movq LABEL_R13(%rdi), %r13 293 movq LABEL_R14(%rdi), %r14 294 movq LABEL_R15(%rdi), %r15 295 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */ 296 movq %rdx, (%rsp) 297 xorl %eax, %eax 298 incl %eax /* return 1 */ 299 ret 300 SET_SIZE(longjmp) 301 302#elif defined(__i386) 303 304 ENTRY(setjmp) 305 movl 4(%esp), %edx /* address of save area */ 306 movl %ebp, LABEL_EBP(%edx) 307 movl %ebx, LABEL_EBX(%edx) 308 movl %esi, LABEL_ESI(%edx) 309 movl %edi, LABEL_EDI(%edx) 310 movl %esp, 4(%edx) 311 movl (%esp), %ecx /* %eip (return address) */ 312 movl %ecx, (%edx) /* LABEL_PC is 0 */ 313 subl %eax, %eax /* return 0 */ 314 ret 315 SET_SIZE(setjmp) 316 317 ENTRY(longjmp) 318 movl 4(%esp), %edx /* address of save area */ 319 movl LABEL_EBP(%edx), %ebp 320 movl LABEL_EBX(%edx), %ebx 321 movl LABEL_ESI(%edx), %esi 322 movl LABEL_EDI(%edx), %edi 323 movl 4(%edx), %esp 324 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */ 325 movl $1, %eax 326 addl $4, %esp /* pop ret adr */ 327 jmp *%ecx /* indirect */ 328 SET_SIZE(longjmp) 329 330#endif /* __i386 */ 331#endif /* __lint */ 332 333/* 334 * if a() calls b() calls caller(), 335 * caller() returns return address in a(). 336 * (Note: We assume a() and b() are C routines which do the normal entry/exit 337 * sequence.) 338 */ 339 340#if defined(__lint) 341 342caddr_t 343caller(void) 344{ return (0); } 345 346#else /* __lint */ 347 348#if defined(__amd64) 349 350 ENTRY(caller) 351 movq 8(%rbp), %rax /* b()'s return pc, in a() */ 352 ret 353 SET_SIZE(caller) 354 355#elif defined(__i386) 356 357 ENTRY(caller) 358 movl 4(%ebp), %eax /* b()'s return pc, in a() */ 359 ret 360 SET_SIZE(caller) 361 362#endif /* __i386 */ 363#endif /* __lint */ 364 365/* 366 * if a() calls callee(), callee() returns the 367 * return address in a(); 368 */ 369 370#if defined(__lint) 371 372caddr_t 373callee(void) 374{ return (0); } 375 376#else /* __lint */ 377 378#if defined(__amd64) 379 380 ENTRY(callee) 381 movq (%rsp), %rax /* callee()'s return pc, in a() */ 382 ret 383 SET_SIZE(callee) 384 385#elif defined(__i386) 386 387 ENTRY(callee) 388 movl (%esp), %eax /* callee()'s return pc, in a() */ 389 ret 390 SET_SIZE(callee) 391 392#endif /* __i386 */ 393#endif /* __lint */ 394 395/* 396 * return the current frame pointer 397 */ 398 399#if defined(__lint) 400 401greg_t 402getfp(void) 403{ return (0); } 404 405#else /* __lint */ 406 407#if defined(__amd64) 408 409 ENTRY(getfp) 410 movq %rbp, %rax 411 ret 412 SET_SIZE(getfp) 413 414#elif defined(__i386) 415 416 ENTRY(getfp) 417 movl %ebp, %eax 418 ret 419 SET_SIZE(getfp) 420 421#endif /* __i386 */ 422#endif /* __lint */ 423 424/* 425 * Invalidate a single page table entry in the TLB 426 */ 427 428#if defined(__lint) 429 430/* ARGSUSED */ 431void 432mmu_tlbflush_entry(caddr_t m) 433{} 434 435#else /* __lint */ 436 437#if defined(__amd64) 438 439 ENTRY(mmu_tlbflush_entry) 440 invlpg (%rdi) 441 ret 442 SET_SIZE(mmu_tlbflush_entry) 443 444#elif defined(__i386) 445 446 ENTRY(mmu_tlbflush_entry) 447 movl 4(%esp), %eax 448 invlpg (%eax) 449 ret 450 SET_SIZE(mmu_tlbflush_entry) 451 452#endif /* __i386 */ 453#endif /* __lint */ 454 455 456/* 457 * Get/Set the value of various control registers 458 */ 459 460#if defined(__lint) 461 462ulong_t 463getcr0(void) 464{ return (0); } 465 466/* ARGSUSED */ 467void 468setcr0(ulong_t value) 469{} 470 471ulong_t 472getcr2(void) 473{ return (0); } 474 475ulong_t 476getcr3(void) 477{ return (0); } 478 479#if !defined(__xpv) 480/* ARGSUSED */ 481void 482setcr3(ulong_t val) 483{} 484 485void 486reload_cr3(void) 487{} 488#endif 489 490ulong_t 491getcr4(void) 492{ return (0); } 493 494/* ARGSUSED */ 495void 496setcr4(ulong_t val) 497{} 498 499#if defined(__amd64) 500 501ulong_t 502getcr8(void) 503{ return (0); } 504 505/* ARGSUSED */ 506void 507setcr8(ulong_t val) 508{} 509 510#endif /* __amd64 */ 511 512#else /* __lint */ 513 514#if defined(__amd64) 515 516 ENTRY(getcr0) 517 movq %cr0, %rax 518 ret 519 SET_SIZE(getcr0) 520 521 ENTRY(setcr0) 522 movq %rdi, %cr0 523 ret 524 SET_SIZE(setcr0) 525 526 ENTRY(getcr2) 527#if defined(__xpv) 528 movq %gs:CPU_VCPU_INFO, %rax 529 movq VCPU_INFO_ARCH_CR2(%rax), %rax 530#else 531 movq %cr2, %rax 532#endif 533 ret 534 SET_SIZE(getcr2) 535 536 ENTRY(getcr3) 537 movq %cr3, %rax 538 ret 539 SET_SIZE(getcr3) 540 541#if !defined(__xpv) 542 543 ENTRY(setcr3) 544 movq %rdi, %cr3 545 ret 546 SET_SIZE(setcr3) 547 548 ENTRY(reload_cr3) 549 movq %cr3, %rdi 550 movq %rdi, %cr3 551 ret 552 SET_SIZE(reload_cr3) 553 554#endif /* __xpv */ 555 556 ENTRY(getcr4) 557 movq %cr4, %rax 558 ret 559 SET_SIZE(getcr4) 560 561 ENTRY(setcr4) 562 movq %rdi, %cr4 563 ret 564 SET_SIZE(setcr4) 565 566 ENTRY(getcr8) 567 movq %cr8, %rax 568 ret 569 SET_SIZE(getcr8) 570 571 ENTRY(setcr8) 572 movq %rdi, %cr8 573 ret 574 SET_SIZE(setcr8) 575 576#elif defined(__i386) 577 578 ENTRY(getcr0) 579 movl %cr0, %eax 580 ret 581 SET_SIZE(getcr0) 582 583 ENTRY(setcr0) 584 movl 4(%esp), %eax 585 movl %eax, %cr0 586 ret 587 SET_SIZE(setcr0) 588 589 /* 590 * "lock mov %cr0" is used on processors which indicate it is 591 * supported via CPUID. Normally the 32 bit TPR is accessed via 592 * the local APIC. 593 */ 594 ENTRY(getcr8) 595 lock 596 movl %cr0, %eax 597 ret 598 SET_SIZE(getcr8) 599 600 ENTRY(setcr8) 601 movl 4(%esp), %eax 602 lock 603 movl %eax, %cr0 604 ret 605 SET_SIZE(setcr8) 606 607 ENTRY(getcr2) 608#if defined(__xpv) 609 movl %gs:CPU_VCPU_INFO, %eax 610 movl VCPU_INFO_ARCH_CR2(%eax), %eax 611#else 612 movl %cr2, %eax 613#endif 614 ret 615 SET_SIZE(getcr2) 616 617 ENTRY(getcr3) 618 movl %cr3, %eax 619 ret 620 SET_SIZE(getcr3) 621 622#if !defined(__xpv) 623 624 ENTRY(setcr3) 625 movl 4(%esp), %eax 626 movl %eax, %cr3 627 ret 628 SET_SIZE(setcr3) 629 630 ENTRY(reload_cr3) 631 movl %cr3, %eax 632 movl %eax, %cr3 633 ret 634 SET_SIZE(reload_cr3) 635 636#endif /* __xpv */ 637 638 ENTRY(getcr4) 639 movl %cr4, %eax 640 ret 641 SET_SIZE(getcr4) 642 643 ENTRY(setcr4) 644 movl 4(%esp), %eax 645 movl %eax, %cr4 646 ret 647 SET_SIZE(setcr4) 648 649#endif /* __i386 */ 650#endif /* __lint */ 651 652#if defined(__lint) 653 654/*ARGSUSED*/ 655uint32_t 656__cpuid_insn(struct cpuid_regs *regs) 657{ return (0); } 658 659#else /* __lint */ 660 661#if defined(__amd64) 662 663 ENTRY(__cpuid_insn) 664 movq %rbx, %r8 665 movq %rcx, %r9 666 movq %rdx, %r11 667 movl (%rdi), %eax /* %eax = regs->cp_eax */ 668 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */ 669 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */ 670 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */ 671 cpuid 672 movl %eax, (%rdi) /* regs->cp_eax = %eax */ 673 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */ 674 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */ 675 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */ 676 movq %r8, %rbx 677 movq %r9, %rcx 678 movq %r11, %rdx 679 ret 680 SET_SIZE(__cpuid_insn) 681 682#elif defined(__i386) 683 684 ENTRY(__cpuid_insn) 685 pushl %ebp 686 movl 0x8(%esp), %ebp /* %ebp = regs */ 687 pushl %ebx 688 pushl %ecx 689 pushl %edx 690 movl (%ebp), %eax /* %eax = regs->cp_eax */ 691 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */ 692 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */ 693 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */ 694 cpuid 695 movl %eax, (%ebp) /* regs->cp_eax = %eax */ 696 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */ 697 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */ 698 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */ 699 popl %edx 700 popl %ecx 701 popl %ebx 702 popl %ebp 703 ret 704 SET_SIZE(__cpuid_insn) 705 706#endif /* __i386 */ 707#endif /* __lint */ 708 709#if defined(__lint) 710 711/*ARGSUSED*/ 712void 713i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints) 714{} 715 716#else /* __lint */ 717 718#if defined(__amd64) 719 720 ENTRY_NP(i86_monitor) 721 pushq %rbp 722 movq %rsp, %rbp 723 movq %rdi, %rax /* addr */ 724 movq %rsi, %rcx /* extensions */ 725 /* rdx contains input arg3: hints */ 726 clflush (%rax) 727 .byte 0x0f, 0x01, 0xc8 /* monitor */ 728 leave 729 ret 730 SET_SIZE(i86_monitor) 731 732#elif defined(__i386) 733 734ENTRY_NP(i86_monitor) 735 pushl %ebp 736 movl %esp, %ebp 737 movl 0x8(%ebp),%eax /* addr */ 738 movl 0xc(%ebp),%ecx /* extensions */ 739 movl 0x10(%ebp),%edx /* hints */ 740 clflush (%eax) 741 .byte 0x0f, 0x01, 0xc8 /* monitor */ 742 leave 743 ret 744 SET_SIZE(i86_monitor) 745 746#endif /* __i386 */ 747#endif /* __lint */ 748 749#if defined(__lint) 750 751/*ARGSUSED*/ 752void 753i86_mwait(uint32_t data, uint32_t extensions) 754{} 755 756#else /* __lint */ 757 758#if defined(__amd64) 759 760 ENTRY_NP(i86_mwait) 761 pushq %rbp 762 movq %rsp, %rbp 763 movq %rdi, %rax /* data */ 764 movq %rsi, %rcx /* extensions */ 765 .byte 0x0f, 0x01, 0xc9 /* mwait */ 766 leave 767 ret 768 SET_SIZE(i86_mwait) 769 770#elif defined(__i386) 771 772 ENTRY_NP(i86_mwait) 773 pushl %ebp 774 movl %esp, %ebp 775 movl 0x8(%ebp),%eax /* data */ 776 movl 0xc(%ebp),%ecx /* extensions */ 777 .byte 0x0f, 0x01, 0xc9 /* mwait */ 778 leave 779 ret 780 SET_SIZE(i86_mwait) 781 782#endif /* __i386 */ 783#endif /* __lint */ 784 785#if defined(__xpv) 786 /* 787 * Defined in C 788 */ 789#else 790 791#if defined(__lint) 792 793hrtime_t 794tsc_read(void) 795{ 796 return (0); 797} 798 799#else /* __lint */ 800 801#if defined(__amd64) 802 803 ENTRY_NP(tsc_read) 804 movq %rbx, %r11 805 movl $0, %eax 806 cpuid 807 rdtsc 808 movq %r11, %rbx 809 shlq $32, %rdx 810 orq %rdx, %rax 811 ret 812 .globl _tsc_mfence_start 813_tsc_mfence_start: 814 mfence 815 rdtsc 816 shlq $32, %rdx 817 orq %rdx, %rax 818 ret 819 .globl _tsc_mfence_end 820_tsc_mfence_end: 821 .globl _tscp_start 822_tscp_start: 823 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 824 shlq $32, %rdx 825 orq %rdx, %rax 826 ret 827 .globl _tscp_end 828_tscp_end: 829 .globl _no_rdtsc_start 830_no_rdtsc_start: 831 xorl %edx, %edx 832 xorl %eax, %eax 833 ret 834 .globl _no_rdtsc_end 835_no_rdtsc_end: 836 .globl _tsc_lfence_start 837_tsc_lfence_start: 838 lfence 839 rdtsc 840 shlq $32, %rdx 841 orq %rdx, %rax 842 ret 843 .globl _tsc_lfence_end 844_tsc_lfence_end: 845 SET_SIZE(tsc_read) 846 847#else /* __i386 */ 848 849 ENTRY_NP(tsc_read) 850 pushl %ebx 851 movl $0, %eax 852 cpuid 853 rdtsc 854 popl %ebx 855 ret 856 .globl _tsc_mfence_start 857_tsc_mfence_start: 858 mfence 859 rdtsc 860 ret 861 .globl _tsc_mfence_end 862_tsc_mfence_end: 863 .globl _tscp_start 864_tscp_start: 865 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 866 ret 867 .globl _tscp_end 868_tscp_end: 869 .globl _no_rdtsc_start 870_no_rdtsc_start: 871 xorl %edx, %edx 872 xorl %eax, %eax 873 ret 874 .globl _no_rdtsc_end 875_no_rdtsc_end: 876 .globl _tsc_lfence_start 877_tsc_lfence_start: 878 lfence 879 rdtsc 880 ret 881 .globl _tsc_lfence_end 882_tsc_lfence_end: 883 SET_SIZE(tsc_read) 884 885#endif /* __i386 */ 886 887#endif /* __lint */ 888 889 890#endif /* __xpv */ 891 892#ifdef __lint 893/* 894 * Do not use this function for obtaining clock tick. This 895 * is called by callers who do not need to have a guarenteed 896 * correct tick value. The proper routine to use is tsc_read(). 897 */ 898u_longlong_t 899randtick(void) 900{ 901 return (0); 902} 903#else 904#if defined(__amd64) 905 ENTRY_NP(randtick) 906 rdtsc 907 shlq $32, %rdx 908 orq %rdx, %rax 909 ret 910 SET_SIZE(randtick) 911#else 912 ENTRY_NP(randtick) 913 rdtsc 914 ret 915 SET_SIZE(randtick) 916#endif /* __i386 */ 917#endif /* __lint */ 918/* 919 * Insert entryp after predp in a doubly linked list. 920 */ 921 922#if defined(__lint) 923 924/*ARGSUSED*/ 925void 926_insque(caddr_t entryp, caddr_t predp) 927{} 928 929#else /* __lint */ 930 931#if defined(__amd64) 932 933 ENTRY(_insque) 934 movq (%rsi), %rax /* predp->forw */ 935 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */ 936 movq %rax, (%rdi) /* entryp->forw = predp->forw */ 937 movq %rdi, (%rsi) /* predp->forw = entryp */ 938 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */ 939 ret 940 SET_SIZE(_insque) 941 942#elif defined(__i386) 943 944 ENTRY(_insque) 945 movl 8(%esp), %edx 946 movl 4(%esp), %ecx 947 movl (%edx), %eax /* predp->forw */ 948 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */ 949 movl %eax, (%ecx) /* entryp->forw = predp->forw */ 950 movl %ecx, (%edx) /* predp->forw = entryp */ 951 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */ 952 ret 953 SET_SIZE(_insque) 954 955#endif /* __i386 */ 956#endif /* __lint */ 957 958/* 959 * Remove entryp from a doubly linked list 960 */ 961 962#if defined(__lint) 963 964/*ARGSUSED*/ 965void 966_remque(caddr_t entryp) 967{} 968 969#else /* __lint */ 970 971#if defined(__amd64) 972 973 ENTRY(_remque) 974 movq (%rdi), %rax /* entry->forw */ 975 movq CPTRSIZE(%rdi), %rdx /* entry->back */ 976 movq %rax, (%rdx) /* entry->back->forw = entry->forw */ 977 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */ 978 ret 979 SET_SIZE(_remque) 980 981#elif defined(__i386) 982 983 ENTRY(_remque) 984 movl 4(%esp), %ecx 985 movl (%ecx), %eax /* entry->forw */ 986 movl CPTRSIZE(%ecx), %edx /* entry->back */ 987 movl %eax, (%edx) /* entry->back->forw = entry->forw */ 988 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */ 989 ret 990 SET_SIZE(_remque) 991 992#endif /* __i386 */ 993#endif /* __lint */ 994 995/* 996 * Returns the number of 997 * non-NULL bytes in string argument. 998 */ 999 1000#if defined(__lint) 1001 1002/* ARGSUSED */ 1003size_t 1004strlen(const char *str) 1005{ return (0); } 1006 1007#else /* __lint */ 1008 1009#if defined(__amd64) 1010 1011/* 1012 * This is close to a simple transliteration of a C version of this 1013 * routine. We should either just -make- this be a C version, or 1014 * justify having it in assembler by making it significantly faster. 1015 * 1016 * size_t 1017 * strlen(const char *s) 1018 * { 1019 * const char *s0; 1020 * #if defined(DEBUG) 1021 * if ((uintptr_t)s < KERNELBASE) 1022 * panic(.str_panic_msg); 1023 * #endif 1024 * for (s0 = s; *s; s++) 1025 * ; 1026 * return (s - s0); 1027 * } 1028 */ 1029 1030 ENTRY(strlen) 1031#ifdef DEBUG 1032 movq postbootkernelbase(%rip), %rax 1033 cmpq %rax, %rdi 1034 jae str_valid 1035 pushq %rbp 1036 movq %rsp, %rbp 1037 leaq .str_panic_msg(%rip), %rdi 1038 xorl %eax, %eax 1039 call panic 1040#endif /* DEBUG */ 1041str_valid: 1042 cmpb $0, (%rdi) 1043 movq %rdi, %rax 1044 je .null_found 1045 .align 4 1046.strlen_loop: 1047 incq %rdi 1048 cmpb $0, (%rdi) 1049 jne .strlen_loop 1050.null_found: 1051 subq %rax, %rdi 1052 movq %rdi, %rax 1053 ret 1054 SET_SIZE(strlen) 1055 1056#elif defined(__i386) 1057 1058 ENTRY(strlen) 1059#ifdef DEBUG 1060 movl postbootkernelbase, %eax 1061 cmpl %eax, 4(%esp) 1062 jae str_valid 1063 pushl %ebp 1064 movl %esp, %ebp 1065 pushl $.str_panic_msg 1066 call panic 1067#endif /* DEBUG */ 1068 1069str_valid: 1070 movl 4(%esp), %eax /* %eax = string address */ 1071 testl $3, %eax /* if %eax not word aligned */ 1072 jnz .not_word_aligned /* goto .not_word_aligned */ 1073 .align 4 1074.word_aligned: 1075 movl (%eax), %edx /* move 1 word from (%eax) to %edx */ 1076 movl $0x7f7f7f7f, %ecx 1077 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */ 1078 addl $4, %eax /* next word */ 1079 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */ 1080 orl %edx, %ecx /* %ecx |= %edx */ 1081 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */ 1082 cmpl $0x80808080, %ecx /* if no null byte in this word */ 1083 je .word_aligned /* goto .word_aligned */ 1084 subl $4, %eax /* post-incremented */ 1085.not_word_aligned: 1086 cmpb $0, (%eax) /* if a byte in (%eax) is null */ 1087 je .null_found /* goto .null_found */ 1088 incl %eax /* next byte */ 1089 testl $3, %eax /* if %eax not word aligned */ 1090 jnz .not_word_aligned /* goto .not_word_aligned */ 1091 jmp .word_aligned /* goto .word_aligned */ 1092 .align 4 1093.null_found: 1094 subl 4(%esp), %eax /* %eax -= string address */ 1095 ret 1096 SET_SIZE(strlen) 1097 1098#endif /* __i386 */ 1099 1100#ifdef DEBUG 1101 .text 1102.str_panic_msg: 1103 .string "strlen: argument below kernelbase" 1104#endif /* DEBUG */ 1105 1106#endif /* __lint */ 1107 1108 /* 1109 * Berkeley 4.3 introduced symbolically named interrupt levels 1110 * as a way deal with priority in a machine independent fashion. 1111 * Numbered priorities are machine specific, and should be 1112 * discouraged where possible. 1113 * 1114 * Note, for the machine specific priorities there are 1115 * examples listed for devices that use a particular priority. 1116 * It should not be construed that all devices of that 1117 * type should be at that priority. It is currently were 1118 * the current devices fit into the priority scheme based 1119 * upon time criticalness. 1120 * 1121 * The underlying assumption of these assignments is that 1122 * IPL 10 is the highest level from which a device 1123 * routine can call wakeup. Devices that interrupt from higher 1124 * levels are restricted in what they can do. If they need 1125 * kernels services they should schedule a routine at a lower 1126 * level (via software interrupt) to do the required 1127 * processing. 1128 * 1129 * Examples of this higher usage: 1130 * Level Usage 1131 * 14 Profiling clock (and PROM uart polling clock) 1132 * 12 Serial ports 1133 * 1134 * The serial ports request lower level processing on level 6. 1135 * 1136 * Also, almost all splN routines (where N is a number or a 1137 * mnemonic) will do a RAISE(), on the assumption that they are 1138 * never used to lower our priority. 1139 * The exceptions are: 1140 * spl8() Because you can't be above 15 to begin with! 1141 * splzs() Because this is used at boot time to lower our 1142 * priority, to allow the PROM to poll the uart. 1143 * spl0() Used to lower priority to 0. 1144 */ 1145 1146#if defined(__lint) 1147 1148int spl0(void) { return (0); } 1149int spl6(void) { return (0); } 1150int spl7(void) { return (0); } 1151int spl8(void) { return (0); } 1152int splhigh(void) { return (0); } 1153int splhi(void) { return (0); } 1154int splzs(void) { return (0); } 1155 1156/* ARGSUSED */ 1157void 1158splx(int level) 1159{} 1160 1161#else /* __lint */ 1162 1163#if defined(__amd64) 1164 1165#define SETPRI(level) \ 1166 movl $/**/level, %edi; /* new priority */ \ 1167 jmp do_splx /* redirect to do_splx */ 1168 1169#define RAISE(level) \ 1170 movl $/**/level, %edi; /* new priority */ \ 1171 jmp splr /* redirect to splr */ 1172 1173#elif defined(__i386) 1174 1175#define SETPRI(level) \ 1176 pushl $/**/level; /* new priority */ \ 1177 call do_splx; /* invoke common splx code */ \ 1178 addl $4, %esp; /* unstack arg */ \ 1179 ret 1180 1181#define RAISE(level) \ 1182 pushl $/**/level; /* new priority */ \ 1183 call splr; /* invoke common splr code */ \ 1184 addl $4, %esp; /* unstack args */ \ 1185 ret 1186 1187#endif /* __i386 */ 1188 1189 /* locks out all interrupts, including memory errors */ 1190 ENTRY(spl8) 1191 SETPRI(15) 1192 SET_SIZE(spl8) 1193 1194 /* just below the level that profiling runs */ 1195 ENTRY(spl7) 1196 RAISE(13) 1197 SET_SIZE(spl7) 1198 1199 /* sun specific - highest priority onboard serial i/o asy ports */ 1200 ENTRY(splzs) 1201 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */ 1202 SET_SIZE(splzs) 1203 1204 ENTRY(splhi) 1205 ALTENTRY(splhigh) 1206 ALTENTRY(spl6) 1207 ALTENTRY(i_ddi_splhigh) 1208 1209 RAISE(DISP_LEVEL) 1210 1211 SET_SIZE(i_ddi_splhigh) 1212 SET_SIZE(spl6) 1213 SET_SIZE(splhigh) 1214 SET_SIZE(splhi) 1215 1216 /* allow all interrupts */ 1217 ENTRY(spl0) 1218 SETPRI(0) 1219 SET_SIZE(spl0) 1220 1221 1222 /* splx implementation */ 1223 ENTRY(splx) 1224 jmp do_splx /* redirect to common splx code */ 1225 SET_SIZE(splx) 1226 1227#endif /* __lint */ 1228 1229#if defined(__i386) 1230 1231/* 1232 * Read and write the %gs register 1233 */ 1234 1235#if defined(__lint) 1236 1237/*ARGSUSED*/ 1238uint16_t 1239getgs(void) 1240{ return (0); } 1241 1242/*ARGSUSED*/ 1243void 1244setgs(uint16_t sel) 1245{} 1246 1247#else /* __lint */ 1248 1249 ENTRY(getgs) 1250 clr %eax 1251 movw %gs, %ax 1252 ret 1253 SET_SIZE(getgs) 1254 1255 ENTRY(setgs) 1256 movw 4(%esp), %gs 1257 ret 1258 SET_SIZE(setgs) 1259 1260#endif /* __lint */ 1261#endif /* __i386 */ 1262 1263#if defined(__lint) 1264 1265void 1266pc_reset(void) 1267{} 1268 1269void 1270efi_reset(void) 1271{} 1272 1273#else /* __lint */ 1274 1275 ENTRY(wait_500ms) 1276#if defined(__amd64) 1277 pushq %rbx 1278#elif defined(__i386) 1279 push %ebx 1280#endif 1281 movl $50000, %ebx 12821: 1283 call tenmicrosec 1284 decl %ebx 1285 jnz 1b 1286#if defined(__amd64) 1287 popq %rbx 1288#elif defined(__i386) 1289 pop %ebx 1290#endif 1291 ret 1292 SET_SIZE(wait_500ms) 1293 1294#define RESET_METHOD_KBC 1 1295#define RESET_METHOD_PORT92 2 1296#define RESET_METHOD_PCI 4 1297 1298 DGDEF3(pc_reset_methods, 4, 8) 1299 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI; 1300 1301 ENTRY(pc_reset) 1302 1303#if defined(__i386) 1304 testl $RESET_METHOD_KBC, pc_reset_methods 1305#elif defined(__amd64) 1306 testl $RESET_METHOD_KBC, pc_reset_methods(%rip) 1307#endif 1308 jz 1f 1309 1310 / 1311 / Try the classic keyboard controller-triggered reset. 1312 / 1313 movw $0x64, %dx 1314 movb $0xfe, %al 1315 outb (%dx) 1316 1317 / Wait up to 500 milliseconds here for the keyboard controller 1318 / to pull the reset line. On some systems where the keyboard 1319 / controller is slow to pull the reset line, the next reset method 1320 / may be executed (which may be bad if those systems hang when the 1321 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92), 1322 / and Ferrari 4000 (doesn't like the cf9 reset method)) 1323 1324 call wait_500ms 1325 13261: 1327#if defined(__i386) 1328 testl $RESET_METHOD_PORT92, pc_reset_methods 1329#elif defined(__amd64) 1330 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip) 1331#endif 1332 jz 3f 1333 1334 / 1335 / Try port 0x92 fast reset 1336 / 1337 movw $0x92, %dx 1338 inb (%dx) 1339 cmpb $0xff, %al / If port's not there, we should get back 0xFF 1340 je 1f 1341 testb $1, %al / If bit 0 1342 jz 2f / is clear, jump to perform the reset 1343 andb $0xfe, %al / otherwise, 1344 outb (%dx) / clear bit 0 first, then 13452: 1346 orb $1, %al / Set bit 0 1347 outb (%dx) / and reset the system 13481: 1349 1350 call wait_500ms 1351 13523: 1353#if defined(__i386) 1354 testl $RESET_METHOD_PCI, pc_reset_methods 1355#elif defined(__amd64) 1356 testl $RESET_METHOD_PCI, pc_reset_methods(%rip) 1357#endif 1358 jz 4f 1359 1360 / Try the PCI (soft) reset vector (should work on all modern systems, 1361 / but has been shown to cause problems on 450NX systems, and some newer 1362 / systems (e.g. ATI IXP400-equipped systems)) 1363 / When resetting via this method, 2 writes are required. The first 1364 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with 1365 / power cycle). 1366 / The reset occurs on the second write, during bit 2's transition from 1367 / 0->1. 1368 movw $0xcf9, %dx 1369 movb $0x2, %al / Reset mode = hard, no power cycle 1370 outb (%dx) 1371 movb $0x6, %al 1372 outb (%dx) 1373 1374 call wait_500ms 1375 13764: 1377 / 1378 / port 0xcf9 failed also. Last-ditch effort is to 1379 / triple-fault the CPU. 1380 / Also, use triple fault for EFI firmware 1381 / 1382 ENTRY(efi_reset) 1383#if defined(__amd64) 1384 pushq $0x0 1385 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1386 lidt (%rsp) 1387#elif defined(__i386) 1388 pushl $0x0 1389 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1390 lidt (%esp) 1391#endif 1392 int $0x0 / Trigger interrupt, generate triple-fault 1393 1394 cli 1395 hlt / Wait forever 1396 /*NOTREACHED*/ 1397 SET_SIZE(efi_reset) 1398 SET_SIZE(pc_reset) 1399 1400#endif /* __lint */ 1401 1402/* 1403 * C callable in and out routines 1404 */ 1405 1406#if defined(__lint) 1407 1408/* ARGSUSED */ 1409void 1410outl(int port_address, uint32_t val) 1411{} 1412 1413#else /* __lint */ 1414 1415#if defined(__amd64) 1416 1417 ENTRY(outl) 1418 movw %di, %dx 1419 movl %esi, %eax 1420 outl (%dx) 1421 ret 1422 SET_SIZE(outl) 1423 1424#elif defined(__i386) 1425 1426 .set PORT, 4 1427 .set VAL, 8 1428 1429 ENTRY(outl) 1430 movw PORT(%esp), %dx 1431 movl VAL(%esp), %eax 1432 outl (%dx) 1433 ret 1434 SET_SIZE(outl) 1435 1436#endif /* __i386 */ 1437#endif /* __lint */ 1438 1439#if defined(__lint) 1440 1441/* ARGSUSED */ 1442void 1443outw(int port_address, uint16_t val) 1444{} 1445 1446#else /* __lint */ 1447 1448#if defined(__amd64) 1449 1450 ENTRY(outw) 1451 movw %di, %dx 1452 movw %si, %ax 1453 D16 outl (%dx) /* XX64 why not outw? */ 1454 ret 1455 SET_SIZE(outw) 1456 1457#elif defined(__i386) 1458 1459 ENTRY(outw) 1460 movw PORT(%esp), %dx 1461 movw VAL(%esp), %ax 1462 D16 outl (%dx) 1463 ret 1464 SET_SIZE(outw) 1465 1466#endif /* __i386 */ 1467#endif /* __lint */ 1468 1469#if defined(__lint) 1470 1471/* ARGSUSED */ 1472void 1473outb(int port_address, uint8_t val) 1474{} 1475 1476#else /* __lint */ 1477 1478#if defined(__amd64) 1479 1480 ENTRY(outb) 1481 movw %di, %dx 1482 movb %sil, %al 1483 outb (%dx) 1484 ret 1485 SET_SIZE(outb) 1486 1487#elif defined(__i386) 1488 1489 ENTRY(outb) 1490 movw PORT(%esp), %dx 1491 movb VAL(%esp), %al 1492 outb (%dx) 1493 ret 1494 SET_SIZE(outb) 1495 1496#endif /* __i386 */ 1497#endif /* __lint */ 1498 1499#if defined(__lint) 1500 1501/* ARGSUSED */ 1502uint32_t 1503inl(int port_address) 1504{ return (0); } 1505 1506#else /* __lint */ 1507 1508#if defined(__amd64) 1509 1510 ENTRY(inl) 1511 xorl %eax, %eax 1512 movw %di, %dx 1513 inl (%dx) 1514 ret 1515 SET_SIZE(inl) 1516 1517#elif defined(__i386) 1518 1519 ENTRY(inl) 1520 movw PORT(%esp), %dx 1521 inl (%dx) 1522 ret 1523 SET_SIZE(inl) 1524 1525#endif /* __i386 */ 1526#endif /* __lint */ 1527 1528#if defined(__lint) 1529 1530/* ARGSUSED */ 1531uint16_t 1532inw(int port_address) 1533{ return (0); } 1534 1535#else /* __lint */ 1536 1537#if defined(__amd64) 1538 1539 ENTRY(inw) 1540 xorl %eax, %eax 1541 movw %di, %dx 1542 D16 inl (%dx) 1543 ret 1544 SET_SIZE(inw) 1545 1546#elif defined(__i386) 1547 1548 ENTRY(inw) 1549 subl %eax, %eax 1550 movw PORT(%esp), %dx 1551 D16 inl (%dx) 1552 ret 1553 SET_SIZE(inw) 1554 1555#endif /* __i386 */ 1556#endif /* __lint */ 1557 1558 1559#if defined(__lint) 1560 1561/* ARGSUSED */ 1562uint8_t 1563inb(int port_address) 1564{ return (0); } 1565 1566#else /* __lint */ 1567 1568#if defined(__amd64) 1569 1570 ENTRY(inb) 1571 xorl %eax, %eax 1572 movw %di, %dx 1573 inb (%dx) 1574 ret 1575 SET_SIZE(inb) 1576 1577#elif defined(__i386) 1578 1579 ENTRY(inb) 1580 subl %eax, %eax 1581 movw PORT(%esp), %dx 1582 inb (%dx) 1583 ret 1584 SET_SIZE(inb) 1585 1586#endif /* __i386 */ 1587#endif /* __lint */ 1588 1589 1590#if defined(__lint) 1591 1592/* ARGSUSED */ 1593void 1594repoutsw(int port, uint16_t *addr, int cnt) 1595{} 1596 1597#else /* __lint */ 1598 1599#if defined(__amd64) 1600 1601 ENTRY(repoutsw) 1602 movl %edx, %ecx 1603 movw %di, %dx 1604 rep 1605 D16 outsl 1606 ret 1607 SET_SIZE(repoutsw) 1608 1609#elif defined(__i386) 1610 1611 /* 1612 * The arguments and saved registers are on the stack in the 1613 * following order: 1614 * | cnt | +16 1615 * | *addr | +12 1616 * | port | +8 1617 * | eip | +4 1618 * | esi | <-- %esp 1619 * If additional values are pushed onto the stack, make sure 1620 * to adjust the following constants accordingly. 1621 */ 1622 .set PORT, 8 1623 .set ADDR, 12 1624 .set COUNT, 16 1625 1626 ENTRY(repoutsw) 1627 pushl %esi 1628 movl PORT(%esp), %edx 1629 movl ADDR(%esp), %esi 1630 movl COUNT(%esp), %ecx 1631 rep 1632 D16 outsl 1633 popl %esi 1634 ret 1635 SET_SIZE(repoutsw) 1636 1637#endif /* __i386 */ 1638#endif /* __lint */ 1639 1640 1641#if defined(__lint) 1642 1643/* ARGSUSED */ 1644void 1645repinsw(int port_addr, uint16_t *addr, int cnt) 1646{} 1647 1648#else /* __lint */ 1649 1650#if defined(__amd64) 1651 1652 ENTRY(repinsw) 1653 movl %edx, %ecx 1654 movw %di, %dx 1655 rep 1656 D16 insl 1657 ret 1658 SET_SIZE(repinsw) 1659 1660#elif defined(__i386) 1661 1662 ENTRY(repinsw) 1663 pushl %edi 1664 movl PORT(%esp), %edx 1665 movl ADDR(%esp), %edi 1666 movl COUNT(%esp), %ecx 1667 rep 1668 D16 insl 1669 popl %edi 1670 ret 1671 SET_SIZE(repinsw) 1672 1673#endif /* __i386 */ 1674#endif /* __lint */ 1675 1676 1677#if defined(__lint) 1678 1679/* ARGSUSED */ 1680void 1681repinsb(int port, uint8_t *addr, int count) 1682{} 1683 1684#else /* __lint */ 1685 1686#if defined(__amd64) 1687 1688 ENTRY(repinsb) 1689 movl %edx, %ecx 1690 movw %di, %dx 1691 movq %rsi, %rdi 1692 rep 1693 insb 1694 ret 1695 SET_SIZE(repinsb) 1696 1697#elif defined(__i386) 1698 1699 /* 1700 * The arguments and saved registers are on the stack in the 1701 * following order: 1702 * | cnt | +16 1703 * | *addr | +12 1704 * | port | +8 1705 * | eip | +4 1706 * | esi | <-- %esp 1707 * If additional values are pushed onto the stack, make sure 1708 * to adjust the following constants accordingly. 1709 */ 1710 .set IO_PORT, 8 1711 .set IO_ADDR, 12 1712 .set IO_COUNT, 16 1713 1714 ENTRY(repinsb) 1715 pushl %edi 1716 movl IO_ADDR(%esp), %edi 1717 movl IO_COUNT(%esp), %ecx 1718 movl IO_PORT(%esp), %edx 1719 rep 1720 insb 1721 popl %edi 1722 ret 1723 SET_SIZE(repinsb) 1724 1725#endif /* __i386 */ 1726#endif /* __lint */ 1727 1728 1729/* 1730 * Input a stream of 32-bit words. 1731 * NOTE: count is a DWORD count. 1732 */ 1733#if defined(__lint) 1734 1735/* ARGSUSED */ 1736void 1737repinsd(int port, uint32_t *addr, int count) 1738{} 1739 1740#else /* __lint */ 1741 1742#if defined(__amd64) 1743 1744 ENTRY(repinsd) 1745 movl %edx, %ecx 1746 movw %di, %dx 1747 movq %rsi, %rdi 1748 rep 1749 insl 1750 ret 1751 SET_SIZE(repinsd) 1752 1753#elif defined(__i386) 1754 1755 ENTRY(repinsd) 1756 pushl %edi 1757 movl IO_ADDR(%esp), %edi 1758 movl IO_COUNT(%esp), %ecx 1759 movl IO_PORT(%esp), %edx 1760 rep 1761 insl 1762 popl %edi 1763 ret 1764 SET_SIZE(repinsd) 1765 1766#endif /* __i386 */ 1767#endif /* __lint */ 1768 1769/* 1770 * Output a stream of bytes 1771 * NOTE: count is a byte count 1772 */ 1773#if defined(__lint) 1774 1775/* ARGSUSED */ 1776void 1777repoutsb(int port, uint8_t *addr, int count) 1778{} 1779 1780#else /* __lint */ 1781 1782#if defined(__amd64) 1783 1784 ENTRY(repoutsb) 1785 movl %edx, %ecx 1786 movw %di, %dx 1787 rep 1788 outsb 1789 ret 1790 SET_SIZE(repoutsb) 1791 1792#elif defined(__i386) 1793 1794 ENTRY(repoutsb) 1795 pushl %esi 1796 movl IO_ADDR(%esp), %esi 1797 movl IO_COUNT(%esp), %ecx 1798 movl IO_PORT(%esp), %edx 1799 rep 1800 outsb 1801 popl %esi 1802 ret 1803 SET_SIZE(repoutsb) 1804 1805#endif /* __i386 */ 1806#endif /* __lint */ 1807 1808/* 1809 * Output a stream of 32-bit words 1810 * NOTE: count is a DWORD count 1811 */ 1812#if defined(__lint) 1813 1814/* ARGSUSED */ 1815void 1816repoutsd(int port, uint32_t *addr, int count) 1817{} 1818 1819#else /* __lint */ 1820 1821#if defined(__amd64) 1822 1823 ENTRY(repoutsd) 1824 movl %edx, %ecx 1825 movw %di, %dx 1826 rep 1827 outsl 1828 ret 1829 SET_SIZE(repoutsd) 1830 1831#elif defined(__i386) 1832 1833 ENTRY(repoutsd) 1834 pushl %esi 1835 movl IO_ADDR(%esp), %esi 1836 movl IO_COUNT(%esp), %ecx 1837 movl IO_PORT(%esp), %edx 1838 rep 1839 outsl 1840 popl %esi 1841 ret 1842 SET_SIZE(repoutsd) 1843 1844#endif /* __i386 */ 1845#endif /* __lint */ 1846 1847/* 1848 * void int3(void) 1849 * void int18(void) 1850 * void int20(void) 1851 * void int_cmci(void) 1852 */ 1853 1854#if defined(__lint) 1855 1856void 1857int3(void) 1858{} 1859 1860void 1861int18(void) 1862{} 1863 1864void 1865int20(void) 1866{} 1867 1868void 1869int_cmci(void) 1870{} 1871 1872#else /* __lint */ 1873 1874 ENTRY(int3) 1875 int $T_BPTFLT 1876 ret 1877 SET_SIZE(int3) 1878 1879 ENTRY(int18) 1880 int $T_MCE 1881 ret 1882 SET_SIZE(int18) 1883 1884 ENTRY(int20) 1885 movl boothowto, %eax 1886 andl $RB_DEBUG, %eax 1887 jz 1f 1888 1889 int $T_DBGENTR 18901: 1891 rep; ret /* use 2 byte return instruction when branch target */ 1892 /* AMD Software Optimization Guide - Section 6.2 */ 1893 SET_SIZE(int20) 1894 1895 ENTRY(int_cmci) 1896 int $T_ENOEXTFLT 1897 ret 1898 SET_SIZE(int_cmci) 1899 1900#endif /* __lint */ 1901 1902#if defined(__lint) 1903 1904/* ARGSUSED */ 1905int 1906scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask) 1907{ return (0); } 1908 1909#else /* __lint */ 1910 1911#if defined(__amd64) 1912 1913 ENTRY(scanc) 1914 /* rdi == size */ 1915 /* rsi == cp */ 1916 /* rdx == table */ 1917 /* rcx == mask */ 1918 addq %rsi, %rdi /* end = &cp[size] */ 1919.scanloop: 1920 cmpq %rdi, %rsi /* while (cp < end */ 1921 jnb .scandone 1922 movzbq (%rsi), %r8 /* %r8 = *cp */ 1923 incq %rsi /* cp++ */ 1924 testb %cl, (%r8, %rdx) 1925 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1926 decq %rsi /* (fix post-increment) */ 1927.scandone: 1928 movl %edi, %eax 1929 subl %esi, %eax /* return (end - cp) */ 1930 ret 1931 SET_SIZE(scanc) 1932 1933#elif defined(__i386) 1934 1935 ENTRY(scanc) 1936 pushl %edi 1937 pushl %esi 1938 movb 24(%esp), %cl /* mask = %cl */ 1939 movl 16(%esp), %esi /* cp = %esi */ 1940 movl 20(%esp), %edx /* table = %edx */ 1941 movl %esi, %edi 1942 addl 12(%esp), %edi /* end = &cp[size]; */ 1943.scanloop: 1944 cmpl %edi, %esi /* while (cp < end */ 1945 jnb .scandone 1946 movzbl (%esi), %eax /* %al = *cp */ 1947 incl %esi /* cp++ */ 1948 movb (%edx, %eax), %al /* %al = table[*cp] */ 1949 testb %al, %cl 1950 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1951 dec %esi /* post-incremented */ 1952.scandone: 1953 movl %edi, %eax 1954 subl %esi, %eax /* return (end - cp) */ 1955 popl %esi 1956 popl %edi 1957 ret 1958 SET_SIZE(scanc) 1959 1960#endif /* __i386 */ 1961#endif /* __lint */ 1962 1963/* 1964 * Replacement functions for ones that are normally inlined. 1965 * In addition to the copy in i86.il, they are defined here just in case. 1966 */ 1967 1968#if defined(__lint) 1969 1970ulong_t 1971intr_clear(void) 1972{ return (0); } 1973 1974ulong_t 1975clear_int_flag(void) 1976{ return (0); } 1977 1978#else /* __lint */ 1979 1980#if defined(__amd64) 1981 1982 ENTRY(intr_clear) 1983 ENTRY(clear_int_flag) 1984 pushfq 1985 popq %rax 1986#if defined(__xpv) 1987 leaq xpv_panicking, %rdi 1988 movl (%rdi), %edi 1989 cmpl $0, %edi 1990 jne 2f 1991 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 1992 /* 1993 * Synthesize the PS_IE bit from the event mask bit 1994 */ 1995 andq $_BITNOT(PS_IE), %rax 1996 testb $1, %dl 1997 jnz 1f 1998 orq $PS_IE, %rax 19991: 2000 ret 20012: 2002#endif 2003 CLI(%rdi) 2004 ret 2005 SET_SIZE(clear_int_flag) 2006 SET_SIZE(intr_clear) 2007 2008#elif defined(__i386) 2009 2010 ENTRY(intr_clear) 2011 ENTRY(clear_int_flag) 2012 pushfl 2013 popl %eax 2014#if defined(__xpv) 2015 leal xpv_panicking, %edx 2016 movl (%edx), %edx 2017 cmpl $0, %edx 2018 jne 2f 2019 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2020 /* 2021 * Synthesize the PS_IE bit from the event mask bit 2022 */ 2023 andl $_BITNOT(PS_IE), %eax 2024 testb $1, %cl 2025 jnz 1f 2026 orl $PS_IE, %eax 20271: 2028 ret 20292: 2030#endif 2031 CLI(%edx) 2032 ret 2033 SET_SIZE(clear_int_flag) 2034 SET_SIZE(intr_clear) 2035 2036#endif /* __i386 */ 2037#endif /* __lint */ 2038 2039#if defined(__lint) 2040 2041struct cpu * 2042curcpup(void) 2043{ return 0; } 2044 2045#else /* __lint */ 2046 2047#if defined(__amd64) 2048 2049 ENTRY(curcpup) 2050 movq %gs:CPU_SELF, %rax 2051 ret 2052 SET_SIZE(curcpup) 2053 2054#elif defined(__i386) 2055 2056 ENTRY(curcpup) 2057 movl %gs:CPU_SELF, %eax 2058 ret 2059 SET_SIZE(curcpup) 2060 2061#endif /* __i386 */ 2062#endif /* __lint */ 2063 2064/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs() 2065 * These functions reverse the byte order of the input parameter and returns 2066 * the result. This is to convert the byte order from host byte order 2067 * (little endian) to network byte order (big endian), or vice versa. 2068 */ 2069 2070#if defined(__lint) 2071 2072uint64_t 2073htonll(uint64_t i) 2074{ return (i); } 2075 2076uint64_t 2077ntohll(uint64_t i) 2078{ return (i); } 2079 2080uint32_t 2081htonl(uint32_t i) 2082{ return (i); } 2083 2084uint32_t 2085ntohl(uint32_t i) 2086{ return (i); } 2087 2088uint16_t 2089htons(uint16_t i) 2090{ return (i); } 2091 2092uint16_t 2093ntohs(uint16_t i) 2094{ return (i); } 2095 2096#else /* __lint */ 2097 2098#if defined(__amd64) 2099 2100 ENTRY(htonll) 2101 ALTENTRY(ntohll) 2102 movq %rdi, %rax 2103 bswapq %rax 2104 ret 2105 SET_SIZE(ntohll) 2106 SET_SIZE(htonll) 2107 2108 /* XX64 there must be shorter sequences for this */ 2109 ENTRY(htonl) 2110 ALTENTRY(ntohl) 2111 movl %edi, %eax 2112 bswap %eax 2113 ret 2114 SET_SIZE(ntohl) 2115 SET_SIZE(htonl) 2116 2117 /* XX64 there must be better sequences for this */ 2118 ENTRY(htons) 2119 ALTENTRY(ntohs) 2120 movl %edi, %eax 2121 bswap %eax 2122 shrl $16, %eax 2123 ret 2124 SET_SIZE(ntohs) 2125 SET_SIZE(htons) 2126 2127#elif defined(__i386) 2128 2129 ENTRY(htonll) 2130 ALTENTRY(ntohll) 2131 movl 4(%esp), %edx 2132 movl 8(%esp), %eax 2133 bswap %edx 2134 bswap %eax 2135 ret 2136 SET_SIZE(ntohll) 2137 SET_SIZE(htonll) 2138 2139 ENTRY(htonl) 2140 ALTENTRY(ntohl) 2141 movl 4(%esp), %eax 2142 bswap %eax 2143 ret 2144 SET_SIZE(ntohl) 2145 SET_SIZE(htonl) 2146 2147 ENTRY(htons) 2148 ALTENTRY(ntohs) 2149 movl 4(%esp), %eax 2150 bswap %eax 2151 shrl $16, %eax 2152 ret 2153 SET_SIZE(ntohs) 2154 SET_SIZE(htons) 2155 2156#endif /* __i386 */ 2157#endif /* __lint */ 2158 2159 2160#if defined(__lint) 2161 2162/* ARGSUSED */ 2163void 2164intr_restore(ulong_t i) 2165{ return; } 2166 2167/* ARGSUSED */ 2168void 2169restore_int_flag(ulong_t i) 2170{ return; } 2171 2172#else /* __lint */ 2173 2174#if defined(__amd64) 2175 2176 ENTRY(intr_restore) 2177 ENTRY(restore_int_flag) 2178 testq $PS_IE, %rdi 2179 jz 1f 2180#if defined(__xpv) 2181 leaq xpv_panicking, %rsi 2182 movl (%rsi), %esi 2183 cmpl $0, %esi 2184 jne 1f 2185 /* 2186 * Since we're -really- running unprivileged, our attempt 2187 * to change the state of the IF bit will be ignored. 2188 * The virtual IF bit is tweaked by CLI and STI. 2189 */ 2190 IE_TO_EVENT_MASK(%rsi, %rdi) 2191#else 2192 sti 2193#endif 21941: 2195 ret 2196 SET_SIZE(restore_int_flag) 2197 SET_SIZE(intr_restore) 2198 2199#elif defined(__i386) 2200 2201 ENTRY(intr_restore) 2202 ENTRY(restore_int_flag) 2203 testl $PS_IE, 4(%esp) 2204 jz 1f 2205#if defined(__xpv) 2206 leal xpv_panicking, %edx 2207 movl (%edx), %edx 2208 cmpl $0, %edx 2209 jne 1f 2210 /* 2211 * Since we're -really- running unprivileged, our attempt 2212 * to change the state of the IF bit will be ignored. 2213 * The virtual IF bit is tweaked by CLI and STI. 2214 */ 2215 IE_TO_EVENT_MASK(%edx, 4(%esp)) 2216#else 2217 sti 2218#endif 22191: 2220 ret 2221 SET_SIZE(restore_int_flag) 2222 SET_SIZE(intr_restore) 2223 2224#endif /* __i386 */ 2225#endif /* __lint */ 2226 2227#if defined(__lint) 2228 2229void 2230sti(void) 2231{} 2232 2233void 2234cli(void) 2235{} 2236 2237#else /* __lint */ 2238 2239 ENTRY(sti) 2240 STI 2241 ret 2242 SET_SIZE(sti) 2243 2244 ENTRY(cli) 2245#if defined(__amd64) 2246 CLI(%rax) 2247#elif defined(__i386) 2248 CLI(%eax) 2249#endif /* __i386 */ 2250 ret 2251 SET_SIZE(cli) 2252 2253#endif /* __lint */ 2254 2255#if defined(__lint) 2256 2257dtrace_icookie_t 2258dtrace_interrupt_disable(void) 2259{ return (0); } 2260 2261#else /* __lint */ 2262 2263#if defined(__amd64) 2264 2265 ENTRY(dtrace_interrupt_disable) 2266 pushfq 2267 popq %rax 2268#if defined(__xpv) 2269 leaq xpv_panicking, %rdi 2270 movl (%rdi), %edi 2271 cmpl $0, %edi 2272 jne .dtrace_interrupt_disable_done 2273 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 2274 /* 2275 * Synthesize the PS_IE bit from the event mask bit 2276 */ 2277 andq $_BITNOT(PS_IE), %rax 2278 testb $1, %dl 2279 jnz .dtrace_interrupt_disable_done 2280 orq $PS_IE, %rax 2281#else 2282 CLI(%rdx) 2283#endif 2284.dtrace_interrupt_disable_done: 2285 ret 2286 SET_SIZE(dtrace_interrupt_disable) 2287 2288#elif defined(__i386) 2289 2290 ENTRY(dtrace_interrupt_disable) 2291 pushfl 2292 popl %eax 2293#if defined(__xpv) 2294 leal xpv_panicking, %edx 2295 movl (%edx), %edx 2296 cmpl $0, %edx 2297 jne .dtrace_interrupt_disable_done 2298 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2299 /* 2300 * Synthesize the PS_IE bit from the event mask bit 2301 */ 2302 andl $_BITNOT(PS_IE), %eax 2303 testb $1, %cl 2304 jnz .dtrace_interrupt_disable_done 2305 orl $PS_IE, %eax 2306#else 2307 CLI(%edx) 2308#endif 2309.dtrace_interrupt_disable_done: 2310 ret 2311 SET_SIZE(dtrace_interrupt_disable) 2312 2313#endif /* __i386 */ 2314#endif /* __lint */ 2315 2316#if defined(__lint) 2317 2318/*ARGSUSED*/ 2319void 2320dtrace_interrupt_enable(dtrace_icookie_t cookie) 2321{} 2322 2323#else /* __lint */ 2324 2325#if defined(__amd64) 2326 2327 ENTRY(dtrace_interrupt_enable) 2328 pushq %rdi 2329 popfq 2330#if defined(__xpv) 2331 leaq xpv_panicking, %rdx 2332 movl (%rdx), %edx 2333 cmpl $0, %edx 2334 jne .dtrace_interrupt_enable_done 2335 /* 2336 * Since we're -really- running unprivileged, our attempt 2337 * to change the state of the IF bit will be ignored. The 2338 * virtual IF bit is tweaked by CLI and STI. 2339 */ 2340 IE_TO_EVENT_MASK(%rdx, %rdi) 2341#endif 2342.dtrace_interrupt_enable_done: 2343 ret 2344 SET_SIZE(dtrace_interrupt_enable) 2345 2346#elif defined(__i386) 2347 2348 ENTRY(dtrace_interrupt_enable) 2349 movl 4(%esp), %eax 2350 pushl %eax 2351 popfl 2352#if defined(__xpv) 2353 leal xpv_panicking, %edx 2354 movl (%edx), %edx 2355 cmpl $0, %edx 2356 jne .dtrace_interrupt_enable_done 2357 /* 2358 * Since we're -really- running unprivileged, our attempt 2359 * to change the state of the IF bit will be ignored. The 2360 * virtual IF bit is tweaked by CLI and STI. 2361 */ 2362 IE_TO_EVENT_MASK(%edx, %eax) 2363#endif 2364.dtrace_interrupt_enable_done: 2365 ret 2366 SET_SIZE(dtrace_interrupt_enable) 2367 2368#endif /* __i386 */ 2369#endif /* __lint */ 2370 2371 2372#if defined(lint) 2373 2374void 2375dtrace_membar_producer(void) 2376{} 2377 2378void 2379dtrace_membar_consumer(void) 2380{} 2381 2382#else /* __lint */ 2383 2384 ENTRY(dtrace_membar_producer) 2385 rep; ret /* use 2 byte return instruction when branch target */ 2386 /* AMD Software Optimization Guide - Section 6.2 */ 2387 SET_SIZE(dtrace_membar_producer) 2388 2389 ENTRY(dtrace_membar_consumer) 2390 rep; ret /* use 2 byte return instruction when branch target */ 2391 /* AMD Software Optimization Guide - Section 6.2 */ 2392 SET_SIZE(dtrace_membar_consumer) 2393 2394#endif /* __lint */ 2395 2396#if defined(__lint) 2397 2398kthread_id_t 2399threadp(void) 2400{ return ((kthread_id_t)0); } 2401 2402#else /* __lint */ 2403 2404#if defined(__amd64) 2405 2406 ENTRY(threadp) 2407 movq %gs:CPU_THREAD, %rax 2408 ret 2409 SET_SIZE(threadp) 2410 2411#elif defined(__i386) 2412 2413 ENTRY(threadp) 2414 movl %gs:CPU_THREAD, %eax 2415 ret 2416 SET_SIZE(threadp) 2417 2418#endif /* __i386 */ 2419#endif /* __lint */ 2420 2421/* 2422 * Checksum routine for Internet Protocol Headers 2423 */ 2424 2425#if defined(__lint) 2426 2427/* ARGSUSED */ 2428unsigned int 2429ip_ocsum( 2430 ushort_t *address, /* ptr to 1st message buffer */ 2431 int halfword_count, /* length of data */ 2432 unsigned int sum) /* partial checksum */ 2433{ 2434 int i; 2435 unsigned int psum = 0; /* partial sum */ 2436 2437 for (i = 0; i < halfword_count; i++, address++) { 2438 psum += *address; 2439 } 2440 2441 while ((psum >> 16) != 0) { 2442 psum = (psum & 0xffff) + (psum >> 16); 2443 } 2444 2445 psum += sum; 2446 2447 while ((psum >> 16) != 0) { 2448 psum = (psum & 0xffff) + (psum >> 16); 2449 } 2450 2451 return (psum); 2452} 2453 2454#else /* __lint */ 2455 2456#if defined(__amd64) 2457 2458 ENTRY(ip_ocsum) 2459 pushq %rbp 2460 movq %rsp, %rbp 2461#ifdef DEBUG 2462 movq postbootkernelbase(%rip), %rax 2463 cmpq %rax, %rdi 2464 jnb 1f 2465 xorl %eax, %eax 2466 movq %rdi, %rsi 2467 leaq .ip_ocsum_panic_msg(%rip), %rdi 2468 call panic 2469 /*NOTREACHED*/ 2470.ip_ocsum_panic_msg: 2471 .string "ip_ocsum: address 0x%p below kernelbase\n" 24721: 2473#endif 2474 movl %esi, %ecx /* halfword_count */ 2475 movq %rdi, %rsi /* address */ 2476 /* partial sum in %edx */ 2477 xorl %eax, %eax 2478 testl %ecx, %ecx 2479 jz .ip_ocsum_done 2480 testq $3, %rsi 2481 jnz .ip_csum_notaligned 2482.ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */ 2483.next_iter: 2484 /* XX64 opportunities for prefetch? */ 2485 /* XX64 compute csum with 64 bit quantities? */ 2486 subl $32, %ecx 2487 jl .less_than_32 2488 2489 addl 0(%rsi), %edx 2490.only60: 2491 adcl 4(%rsi), %eax 2492.only56: 2493 adcl 8(%rsi), %edx 2494.only52: 2495 adcl 12(%rsi), %eax 2496.only48: 2497 adcl 16(%rsi), %edx 2498.only44: 2499 adcl 20(%rsi), %eax 2500.only40: 2501 adcl 24(%rsi), %edx 2502.only36: 2503 adcl 28(%rsi), %eax 2504.only32: 2505 adcl 32(%rsi), %edx 2506.only28: 2507 adcl 36(%rsi), %eax 2508.only24: 2509 adcl 40(%rsi), %edx 2510.only20: 2511 adcl 44(%rsi), %eax 2512.only16: 2513 adcl 48(%rsi), %edx 2514.only12: 2515 adcl 52(%rsi), %eax 2516.only8: 2517 adcl 56(%rsi), %edx 2518.only4: 2519 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */ 2520.only0: 2521 adcl $0, %eax /* could be adding -1 in eax with a carry */ 2522 adcl $0, %eax 2523 2524 addq $64, %rsi 2525 testl %ecx, %ecx 2526 jnz .next_iter 2527 2528.ip_ocsum_done: 2529 addl %eax, %edx 2530 adcl $0, %edx 2531 movl %edx, %eax /* form a 16 bit checksum by */ 2532 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2533 addw %dx, %ax 2534 adcw $0, %ax 2535 andl $0xffff, %eax 2536 leave 2537 ret 2538 2539.ip_csum_notaligned: 2540 xorl %edi, %edi 2541 movw (%rsi), %di 2542 addl %edi, %edx 2543 adcl $0, %edx 2544 addq $2, %rsi 2545 decl %ecx 2546 jmp .ip_csum_aligned 2547 2548.less_than_32: 2549 addl $32, %ecx 2550 testl $1, %ecx 2551 jz .size_aligned 2552 andl $0xfe, %ecx 2553 movzwl (%rsi, %rcx, 2), %edi 2554 addl %edi, %edx 2555 adcl $0, %edx 2556.size_aligned: 2557 movl %ecx, %edi 2558 shrl $1, %ecx 2559 shl $1, %edi 2560 subq $64, %rdi 2561 addq %rdi, %rsi 2562 leaq .ip_ocsum_jmptbl(%rip), %rdi 2563 leaq (%rdi, %rcx, 8), %rdi 2564 xorl %ecx, %ecx 2565 clc 2566 jmp *(%rdi) 2567 2568 .align 8 2569.ip_ocsum_jmptbl: 2570 .quad .only0, .only4, .only8, .only12, .only16, .only20 2571 .quad .only24, .only28, .only32, .only36, .only40, .only44 2572 .quad .only48, .only52, .only56, .only60 2573 SET_SIZE(ip_ocsum) 2574 2575#elif defined(__i386) 2576 2577 ENTRY(ip_ocsum) 2578 pushl %ebp 2579 movl %esp, %ebp 2580 pushl %ebx 2581 pushl %esi 2582 pushl %edi 2583 movl 12(%ebp), %ecx /* count of half words */ 2584 movl 16(%ebp), %edx /* partial checksum */ 2585 movl 8(%ebp), %esi 2586 xorl %eax, %eax 2587 testl %ecx, %ecx 2588 jz .ip_ocsum_done 2589 2590 testl $3, %esi 2591 jnz .ip_csum_notaligned 2592.ip_csum_aligned: 2593.next_iter: 2594 subl $32, %ecx 2595 jl .less_than_32 2596 2597 addl 0(%esi), %edx 2598.only60: 2599 adcl 4(%esi), %eax 2600.only56: 2601 adcl 8(%esi), %edx 2602.only52: 2603 adcl 12(%esi), %eax 2604.only48: 2605 adcl 16(%esi), %edx 2606.only44: 2607 adcl 20(%esi), %eax 2608.only40: 2609 adcl 24(%esi), %edx 2610.only36: 2611 adcl 28(%esi), %eax 2612.only32: 2613 adcl 32(%esi), %edx 2614.only28: 2615 adcl 36(%esi), %eax 2616.only24: 2617 adcl 40(%esi), %edx 2618.only20: 2619 adcl 44(%esi), %eax 2620.only16: 2621 adcl 48(%esi), %edx 2622.only12: 2623 adcl 52(%esi), %eax 2624.only8: 2625 adcl 56(%esi), %edx 2626.only4: 2627 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */ 2628.only0: 2629 adcl $0, %eax /* we could be adding -1 in eax with a carry */ 2630 adcl $0, %eax 2631 2632 addl $64, %esi 2633 andl %ecx, %ecx 2634 jnz .next_iter 2635 2636.ip_ocsum_done: 2637 addl %eax, %edx 2638 adcl $0, %edx 2639 movl %edx, %eax /* form a 16 bit checksum by */ 2640 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2641 addw %dx, %ax 2642 adcw $0, %ax 2643 andl $0xffff, %eax 2644 popl %edi /* restore registers */ 2645 popl %esi 2646 popl %ebx 2647 leave 2648 ret 2649 2650.ip_csum_notaligned: 2651 xorl %edi, %edi 2652 movw (%esi), %di 2653 addl %edi, %edx 2654 adcl $0, %edx 2655 addl $2, %esi 2656 decl %ecx 2657 jmp .ip_csum_aligned 2658 2659.less_than_32: 2660 addl $32, %ecx 2661 testl $1, %ecx 2662 jz .size_aligned 2663 andl $0xfe, %ecx 2664 movzwl (%esi, %ecx, 2), %edi 2665 addl %edi, %edx 2666 adcl $0, %edx 2667.size_aligned: 2668 movl %ecx, %edi 2669 shrl $1, %ecx 2670 shl $1, %edi 2671 subl $64, %edi 2672 addl %edi, %esi 2673 movl $.ip_ocsum_jmptbl, %edi 2674 lea (%edi, %ecx, 4), %edi 2675 xorl %ecx, %ecx 2676 clc 2677 jmp *(%edi) 2678 SET_SIZE(ip_ocsum) 2679 2680 .data 2681 .align 4 2682 2683.ip_ocsum_jmptbl: 2684 .long .only0, .only4, .only8, .only12, .only16, .only20 2685 .long .only24, .only28, .only32, .only36, .only40, .only44 2686 .long .only48, .only52, .only56, .only60 2687 2688 2689#endif /* __i386 */ 2690#endif /* __lint */ 2691 2692/* 2693 * multiply two long numbers and yield a u_longlong_t result, callable from C. 2694 * Provided to manipulate hrtime_t values. 2695 */ 2696#if defined(__lint) 2697 2698/* result = a * b; */ 2699 2700/* ARGSUSED */ 2701unsigned long long 2702mul32(uint_t a, uint_t b) 2703{ return (0); } 2704 2705#else /* __lint */ 2706 2707#if defined(__amd64) 2708 2709 ENTRY(mul32) 2710 xorl %edx, %edx /* XX64 joe, paranoia? */ 2711 movl %edi, %eax 2712 mull %esi 2713 shlq $32, %rdx 2714 orq %rdx, %rax 2715 ret 2716 SET_SIZE(mul32) 2717 2718#elif defined(__i386) 2719 2720 ENTRY(mul32) 2721 movl 8(%esp), %eax 2722 movl 4(%esp), %ecx 2723 mull %ecx 2724 ret 2725 SET_SIZE(mul32) 2726 2727#endif /* __i386 */ 2728#endif /* __lint */ 2729 2730#if defined(notused) 2731#if defined(__lint) 2732/* ARGSUSED */ 2733void 2734load_pte64(uint64_t *pte, uint64_t pte_value) 2735{} 2736#else /* __lint */ 2737 .globl load_pte64 2738load_pte64: 2739 movl 4(%esp), %eax 2740 movl 8(%esp), %ecx 2741 movl 12(%esp), %edx 2742 movl %edx, 4(%eax) 2743 movl %ecx, (%eax) 2744 ret 2745#endif /* __lint */ 2746#endif /* notused */ 2747 2748#if defined(__lint) 2749 2750/*ARGSUSED*/ 2751void 2752scan_memory(caddr_t addr, size_t size) 2753{} 2754 2755#else /* __lint */ 2756 2757#if defined(__amd64) 2758 2759 ENTRY(scan_memory) 2760 shrq $3, %rsi /* convert %rsi from byte to quadword count */ 2761 jz .scanm_done 2762 movq %rsi, %rcx /* move count into rep control register */ 2763 movq %rdi, %rsi /* move addr into lodsq control reg. */ 2764 rep lodsq /* scan the memory range */ 2765.scanm_done: 2766 rep; ret /* use 2 byte return instruction when branch target */ 2767 /* AMD Software Optimization Guide - Section 6.2 */ 2768 SET_SIZE(scan_memory) 2769 2770#elif defined(__i386) 2771 2772 ENTRY(scan_memory) 2773 pushl %ecx 2774 pushl %esi 2775 movl 16(%esp), %ecx /* move 2nd arg into rep control register */ 2776 shrl $2, %ecx /* convert from byte count to word count */ 2777 jz .scanm_done 2778 movl 12(%esp), %esi /* move 1st arg into lodsw control register */ 2779 .byte 0xf3 /* rep prefix. lame assembler. sigh. */ 2780 lodsl 2781.scanm_done: 2782 popl %esi 2783 popl %ecx 2784 ret 2785 SET_SIZE(scan_memory) 2786 2787#endif /* __i386 */ 2788#endif /* __lint */ 2789 2790 2791#if defined(__lint) 2792 2793/*ARGSUSED */ 2794int 2795lowbit(ulong_t i) 2796{ return (0); } 2797 2798#else /* __lint */ 2799 2800#if defined(__amd64) 2801 2802 ENTRY(lowbit) 2803 movl $-1, %eax 2804 bsfq %rdi, %rdi 2805 cmovnz %edi, %eax 2806 incl %eax 2807 ret 2808 SET_SIZE(lowbit) 2809 2810#elif defined(__i386) 2811 2812 ENTRY(lowbit) 2813 bsfl 4(%esp), %eax 2814 jz 0f 2815 incl %eax 2816 ret 28170: 2818 xorl %eax, %eax 2819 ret 2820 SET_SIZE(lowbit) 2821 2822#endif /* __i386 */ 2823#endif /* __lint */ 2824 2825#if defined(__lint) 2826 2827/*ARGSUSED*/ 2828int 2829highbit(ulong_t i) 2830{ return (0); } 2831 2832/*ARGSUSED*/ 2833int 2834highbit64(uint64_t i) 2835{ return (0); } 2836 2837#else /* __lint */ 2838 2839#if defined(__amd64) 2840 2841 ENTRY(highbit) 2842 ALTENTRY(highbit64) 2843 movl $-1, %eax 2844 bsrq %rdi, %rdi 2845 cmovnz %edi, %eax 2846 incl %eax 2847 ret 2848 SET_SIZE(highbit64) 2849 SET_SIZE(highbit) 2850 2851#elif defined(__i386) 2852 2853 ENTRY(highbit) 2854 bsrl 4(%esp), %eax 2855 jz 0f 2856 incl %eax 2857 ret 28580: 2859 xorl %eax, %eax 2860 ret 2861 SET_SIZE(highbit) 2862 2863 ENTRY(highbit64) 2864 bsrl 8(%esp), %eax 2865 jz highbit 2866 addl $33, %eax 2867 ret 2868 SET_SIZE(highbit64) 2869 2870#endif /* __i386 */ 2871#endif /* __lint */ 2872 2873#if defined(__lint) 2874 2875/*ARGSUSED*/ 2876uint64_t 2877rdmsr(uint_t r) 2878{ return (0); } 2879 2880/*ARGSUSED*/ 2881void 2882wrmsr(uint_t r, const uint64_t val) 2883{} 2884 2885/*ARGSUSED*/ 2886uint64_t 2887xrdmsr(uint_t r) 2888{ return (0); } 2889 2890/*ARGSUSED*/ 2891void 2892xwrmsr(uint_t r, const uint64_t val) 2893{} 2894 2895void 2896invalidate_cache(void) 2897{} 2898 2899/*ARGSUSED*/ 2900uint64_t 2901get_xcr(uint_t r) 2902{ return (0); } 2903 2904/*ARGSUSED*/ 2905void 2906set_xcr(uint_t r, const uint64_t val) 2907{} 2908 2909#else /* __lint */ 2910 2911#define XMSR_ACCESS_VAL $0x9c5a203a 2912 2913#if defined(__amd64) 2914 2915 ENTRY(rdmsr) 2916 movl %edi, %ecx 2917 rdmsr 2918 shlq $32, %rdx 2919 orq %rdx, %rax 2920 ret 2921 SET_SIZE(rdmsr) 2922 2923 ENTRY(wrmsr) 2924 movq %rsi, %rdx 2925 shrq $32, %rdx 2926 movl %esi, %eax 2927 movl %edi, %ecx 2928 wrmsr 2929 ret 2930 SET_SIZE(wrmsr) 2931 2932 ENTRY(xrdmsr) 2933 pushq %rbp 2934 movq %rsp, %rbp 2935 movl %edi, %ecx 2936 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2937 rdmsr 2938 shlq $32, %rdx 2939 orq %rdx, %rax 2940 leave 2941 ret 2942 SET_SIZE(xrdmsr) 2943 2944 ENTRY(xwrmsr) 2945 pushq %rbp 2946 movq %rsp, %rbp 2947 movl %edi, %ecx 2948 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2949 movq %rsi, %rdx 2950 shrq $32, %rdx 2951 movl %esi, %eax 2952 wrmsr 2953 leave 2954 ret 2955 SET_SIZE(xwrmsr) 2956 2957 ENTRY(get_xcr) 2958 movl %edi, %ecx 2959 #xgetbv 2960 .byte 0x0f,0x01,0xd0 2961 shlq $32, %rdx 2962 orq %rdx, %rax 2963 ret 2964 SET_SIZE(get_xcr) 2965 2966 ENTRY(set_xcr) 2967 movq %rsi, %rdx 2968 shrq $32, %rdx 2969 movl %esi, %eax 2970 movl %edi, %ecx 2971 #xsetbv 2972 .byte 0x0f,0x01,0xd1 2973 ret 2974 SET_SIZE(set_xcr) 2975 2976#elif defined(__i386) 2977 2978 ENTRY(rdmsr) 2979 movl 4(%esp), %ecx 2980 rdmsr 2981 ret 2982 SET_SIZE(rdmsr) 2983 2984 ENTRY(wrmsr) 2985 movl 4(%esp), %ecx 2986 movl 8(%esp), %eax 2987 movl 12(%esp), %edx 2988 wrmsr 2989 ret 2990 SET_SIZE(wrmsr) 2991 2992 ENTRY(xrdmsr) 2993 pushl %ebp 2994 movl %esp, %ebp 2995 movl 8(%esp), %ecx 2996 pushl %edi 2997 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2998 rdmsr 2999 popl %edi 3000 leave 3001 ret 3002 SET_SIZE(xrdmsr) 3003 3004 ENTRY(xwrmsr) 3005 pushl %ebp 3006 movl %esp, %ebp 3007 movl 8(%esp), %ecx 3008 movl 12(%esp), %eax 3009 movl 16(%esp), %edx 3010 pushl %edi 3011 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 3012 wrmsr 3013 popl %edi 3014 leave 3015 ret 3016 SET_SIZE(xwrmsr) 3017 3018 ENTRY(get_xcr) 3019 movl 4(%esp), %ecx 3020 #xgetbv 3021 .byte 0x0f,0x01,0xd0 3022 ret 3023 SET_SIZE(get_xcr) 3024 3025 ENTRY(set_xcr) 3026 movl 4(%esp), %ecx 3027 movl 8(%esp), %eax 3028 movl 12(%esp), %edx 3029 #xsetbv 3030 .byte 0x0f,0x01,0xd1 3031 ret 3032 SET_SIZE(set_xcr) 3033 3034#endif /* __i386 */ 3035 3036 ENTRY(invalidate_cache) 3037 wbinvd 3038 ret 3039 SET_SIZE(invalidate_cache) 3040 3041#endif /* __lint */ 3042 3043#if defined(__lint) 3044 3045/*ARGSUSED*/ 3046void 3047getcregs(struct cregs *crp) 3048{} 3049 3050#else /* __lint */ 3051 3052#if defined(__amd64) 3053 3054 ENTRY_NP(getcregs) 3055#if defined(__xpv) 3056 /* 3057 * Only a few of the hardware control registers or descriptor tables 3058 * are directly accessible to us, so just zero the structure. 3059 * 3060 * XXPV Perhaps it would be helpful for the hypervisor to return 3061 * virtualized versions of these for post-mortem use. 3062 * (Need to reevaluate - perhaps it already does!) 3063 */ 3064 pushq %rdi /* save *crp */ 3065 movq $CREGSZ, %rsi 3066 call bzero 3067 popq %rdi 3068 3069 /* 3070 * Dump what limited information we can 3071 */ 3072 movq %cr0, %rax 3073 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3074 movq %cr2, %rax 3075 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3076 movq %cr3, %rax 3077 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3078 movq %cr4, %rax 3079 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3080 3081#else /* __xpv */ 3082 3083#define GETMSR(r, off, d) \ 3084 movl $r, %ecx; \ 3085 rdmsr; \ 3086 movl %eax, off(d); \ 3087 movl %edx, off+4(d) 3088 3089 xorl %eax, %eax 3090 movq %rax, CREG_GDT+8(%rdi) 3091 sgdt CREG_GDT(%rdi) /* 10 bytes */ 3092 movq %rax, CREG_IDT+8(%rdi) 3093 sidt CREG_IDT(%rdi) /* 10 bytes */ 3094 movq %rax, CREG_LDT(%rdi) 3095 sldt CREG_LDT(%rdi) /* 2 bytes */ 3096 movq %rax, CREG_TASKR(%rdi) 3097 str CREG_TASKR(%rdi) /* 2 bytes */ 3098 movq %cr0, %rax 3099 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3100 movq %cr2, %rax 3101 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3102 movq %cr3, %rax 3103 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3104 movq %cr4, %rax 3105 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3106 movq %cr8, %rax 3107 movq %rax, CREG_CR8(%rdi) /* cr8 */ 3108 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi) 3109 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi) 3110#endif /* __xpv */ 3111 ret 3112 SET_SIZE(getcregs) 3113 3114#undef GETMSR 3115 3116#elif defined(__i386) 3117 3118 ENTRY_NP(getcregs) 3119#if defined(__xpv) 3120 /* 3121 * Only a few of the hardware control registers or descriptor tables 3122 * are directly accessible to us, so just zero the structure. 3123 * 3124 * XXPV Perhaps it would be helpful for the hypervisor to return 3125 * virtualized versions of these for post-mortem use. 3126 * (Need to reevaluate - perhaps it already does!) 3127 */ 3128 movl 4(%esp), %edx 3129 pushl $CREGSZ 3130 pushl %edx 3131 call bzero 3132 addl $8, %esp 3133 movl 4(%esp), %edx 3134 3135 /* 3136 * Dump what limited information we can 3137 */ 3138 movl %cr0, %eax 3139 movl %eax, CREG_CR0(%edx) /* cr0 */ 3140 movl %cr2, %eax 3141 movl %eax, CREG_CR2(%edx) /* cr2 */ 3142 movl %cr3, %eax 3143 movl %eax, CREG_CR3(%edx) /* cr3 */ 3144 movl %cr4, %eax 3145 movl %eax, CREG_CR4(%edx) /* cr4 */ 3146 3147#else /* __xpv */ 3148 3149 movl 4(%esp), %edx 3150 movw $0, CREG_GDT+6(%edx) 3151 movw $0, CREG_IDT+6(%edx) 3152 sgdt CREG_GDT(%edx) /* gdt */ 3153 sidt CREG_IDT(%edx) /* idt */ 3154 sldt CREG_LDT(%edx) /* ldt */ 3155 str CREG_TASKR(%edx) /* task */ 3156 movl %cr0, %eax 3157 movl %eax, CREG_CR0(%edx) /* cr0 */ 3158 movl %cr2, %eax 3159 movl %eax, CREG_CR2(%edx) /* cr2 */ 3160 movl %cr3, %eax 3161 movl %eax, CREG_CR3(%edx) /* cr3 */ 3162 bt $X86FSET_LARGEPAGE, x86_featureset 3163 jnc .nocr4 3164 movl %cr4, %eax 3165 movl %eax, CREG_CR4(%edx) /* cr4 */ 3166 jmp .skip 3167.nocr4: 3168 movl $0, CREG_CR4(%edx) 3169.skip: 3170#endif 3171 ret 3172 SET_SIZE(getcregs) 3173 3174#endif /* __i386 */ 3175#endif /* __lint */ 3176 3177 3178/* 3179 * A panic trigger is a word which is updated atomically and can only be set 3180 * once. We atomically store 0xDEFACEDD and load the old value. If the 3181 * previous value was 0, we succeed and return 1; otherwise return 0. 3182 * This allows a partially corrupt trigger to still trigger correctly. DTrace 3183 * has its own version of this function to allow it to panic correctly from 3184 * probe context. 3185 */ 3186#if defined(__lint) 3187 3188/*ARGSUSED*/ 3189int 3190panic_trigger(int *tp) 3191{ return (0); } 3192 3193/*ARGSUSED*/ 3194int 3195dtrace_panic_trigger(int *tp) 3196{ return (0); } 3197 3198#else /* __lint */ 3199 3200#if defined(__amd64) 3201 3202 ENTRY_NP(panic_trigger) 3203 xorl %eax, %eax 3204 movl $0xdefacedd, %edx 3205 lock 3206 xchgl %edx, (%rdi) 3207 cmpl $0, %edx 3208 je 0f 3209 movl $0, %eax 3210 ret 32110: movl $1, %eax 3212 ret 3213 SET_SIZE(panic_trigger) 3214 3215 ENTRY_NP(dtrace_panic_trigger) 3216 xorl %eax, %eax 3217 movl $0xdefacedd, %edx 3218 lock 3219 xchgl %edx, (%rdi) 3220 cmpl $0, %edx 3221 je 0f 3222 movl $0, %eax 3223 ret 32240: movl $1, %eax 3225 ret 3226 SET_SIZE(dtrace_panic_trigger) 3227 3228#elif defined(__i386) 3229 3230 ENTRY_NP(panic_trigger) 3231 movl 4(%esp), %edx / %edx = address of trigger 3232 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3233 lock / assert lock 3234 xchgl %eax, (%edx) / exchange %eax and the trigger 3235 cmpl $0, %eax / if (%eax == 0x0) 3236 je 0f / return (1); 3237 movl $0, %eax / else 3238 ret / return (0); 32390: movl $1, %eax 3240 ret 3241 SET_SIZE(panic_trigger) 3242 3243 ENTRY_NP(dtrace_panic_trigger) 3244 movl 4(%esp), %edx / %edx = address of trigger 3245 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3246 lock / assert lock 3247 xchgl %eax, (%edx) / exchange %eax and the trigger 3248 cmpl $0, %eax / if (%eax == 0x0) 3249 je 0f / return (1); 3250 movl $0, %eax / else 3251 ret / return (0); 32520: movl $1, %eax 3253 ret 3254 SET_SIZE(dtrace_panic_trigger) 3255 3256#endif /* __i386 */ 3257#endif /* __lint */ 3258 3259/* 3260 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 3261 * into the panic code implemented in panicsys(). vpanic() is responsible 3262 * for passing through the format string and arguments, and constructing a 3263 * regs structure on the stack into which it saves the current register 3264 * values. If we are not dying due to a fatal trap, these registers will 3265 * then be preserved in panicbuf as the current processor state. Before 3266 * invoking panicsys(), vpanic() activates the first panic trigger (see 3267 * common/os/panic.c) and switches to the panic_stack if successful. Note that 3268 * DTrace takes a slightly different panic path if it must panic from probe 3269 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 3270 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 3271 * branches back into vpanic(). 3272 */ 3273#if defined(__lint) 3274 3275/*ARGSUSED*/ 3276void 3277vpanic(const char *format, va_list alist) 3278{} 3279 3280/*ARGSUSED*/ 3281void 3282dtrace_vpanic(const char *format, va_list alist) 3283{} 3284 3285#else /* __lint */ 3286 3287#if defined(__amd64) 3288 3289 ENTRY_NP(vpanic) /* Initial stack layout: */ 3290 3291 pushq %rbp /* | %rip | 0x60 */ 3292 movq %rsp, %rbp /* | %rbp | 0x58 */ 3293 pushfq /* | rfl | 0x50 */ 3294 pushq %r11 /* | %r11 | 0x48 */ 3295 pushq %r10 /* | %r10 | 0x40 */ 3296 pushq %rbx /* | %rbx | 0x38 */ 3297 pushq %rax /* | %rax | 0x30 */ 3298 pushq %r9 /* | %r9 | 0x28 */ 3299 pushq %r8 /* | %r8 | 0x20 */ 3300 pushq %rcx /* | %rcx | 0x18 */ 3301 pushq %rdx /* | %rdx | 0x10 */ 3302 pushq %rsi /* | %rsi | 0x8 alist */ 3303 pushq %rdi /* | %rdi | 0x0 format */ 3304 3305 movq %rsp, %rbx /* %rbx = current %rsp */ 3306 3307 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3308 call panic_trigger /* %eax = panic_trigger() */ 3309 3310vpanic_common: 3311 /* 3312 * The panic_trigger result is in %eax from the call above, and 3313 * dtrace_panic places it in %eax before branching here. 3314 * The rdmsr instructions that follow below will clobber %eax so 3315 * we stash the panic_trigger result in %r11d. 3316 */ 3317 movl %eax, %r11d 3318 cmpl $0, %r11d 3319 je 0f 3320 3321 /* 3322 * If panic_trigger() was successful, we are the first to initiate a 3323 * panic: we now switch to the reserved panic_stack before continuing. 3324 */ 3325 leaq panic_stack(%rip), %rsp 3326 addq $PANICSTKSIZE, %rsp 33270: subq $REGSIZE, %rsp 3328 /* 3329 * Now that we've got everything set up, store the register values as 3330 * they were when we entered vpanic() to the designated location in 3331 * the regs structure we allocated on the stack. 3332 */ 3333 movq 0x0(%rbx), %rcx 3334 movq %rcx, REGOFF_RDI(%rsp) 3335 movq 0x8(%rbx), %rcx 3336 movq %rcx, REGOFF_RSI(%rsp) 3337 movq 0x10(%rbx), %rcx 3338 movq %rcx, REGOFF_RDX(%rsp) 3339 movq 0x18(%rbx), %rcx 3340 movq %rcx, REGOFF_RCX(%rsp) 3341 movq 0x20(%rbx), %rcx 3342 3343 movq %rcx, REGOFF_R8(%rsp) 3344 movq 0x28(%rbx), %rcx 3345 movq %rcx, REGOFF_R9(%rsp) 3346 movq 0x30(%rbx), %rcx 3347 movq %rcx, REGOFF_RAX(%rsp) 3348 movq 0x38(%rbx), %rcx 3349 movq %rcx, REGOFF_RBX(%rsp) 3350 movq 0x58(%rbx), %rcx 3351 3352 movq %rcx, REGOFF_RBP(%rsp) 3353 movq 0x40(%rbx), %rcx 3354 movq %rcx, REGOFF_R10(%rsp) 3355 movq 0x48(%rbx), %rcx 3356 movq %rcx, REGOFF_R11(%rsp) 3357 movq %r12, REGOFF_R12(%rsp) 3358 3359 movq %r13, REGOFF_R13(%rsp) 3360 movq %r14, REGOFF_R14(%rsp) 3361 movq %r15, REGOFF_R15(%rsp) 3362 3363 xorl %ecx, %ecx 3364 movw %ds, %cx 3365 movq %rcx, REGOFF_DS(%rsp) 3366 movw %es, %cx 3367 movq %rcx, REGOFF_ES(%rsp) 3368 movw %fs, %cx 3369 movq %rcx, REGOFF_FS(%rsp) 3370 movw %gs, %cx 3371 movq %rcx, REGOFF_GS(%rsp) 3372 3373 movq $0, REGOFF_TRAPNO(%rsp) 3374 3375 movq $0, REGOFF_ERR(%rsp) 3376 leaq vpanic(%rip), %rcx 3377 movq %rcx, REGOFF_RIP(%rsp) 3378 movw %cs, %cx 3379 movzwq %cx, %rcx 3380 movq %rcx, REGOFF_CS(%rsp) 3381 movq 0x50(%rbx), %rcx 3382 movq %rcx, REGOFF_RFL(%rsp) 3383 movq %rbx, %rcx 3384 addq $0x60, %rcx 3385 movq %rcx, REGOFF_RSP(%rsp) 3386 movw %ss, %cx 3387 movzwq %cx, %rcx 3388 movq %rcx, REGOFF_SS(%rsp) 3389 3390 /* 3391 * panicsys(format, alist, rp, on_panic_stack) 3392 */ 3393 movq REGOFF_RDI(%rsp), %rdi /* format */ 3394 movq REGOFF_RSI(%rsp), %rsi /* alist */ 3395 movq %rsp, %rdx /* struct regs */ 3396 movl %r11d, %ecx /* on_panic_stack */ 3397 call panicsys 3398 addq $REGSIZE, %rsp 3399 popq %rdi 3400 popq %rsi 3401 popq %rdx 3402 popq %rcx 3403 popq %r8 3404 popq %r9 3405 popq %rax 3406 popq %rbx 3407 popq %r10 3408 popq %r11 3409 popfq 3410 leave 3411 ret 3412 SET_SIZE(vpanic) 3413 3414 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */ 3415 3416 pushq %rbp /* | %rip | 0x60 */ 3417 movq %rsp, %rbp /* | %rbp | 0x58 */ 3418 pushfq /* | rfl | 0x50 */ 3419 pushq %r11 /* | %r11 | 0x48 */ 3420 pushq %r10 /* | %r10 | 0x40 */ 3421 pushq %rbx /* | %rbx | 0x38 */ 3422 pushq %rax /* | %rax | 0x30 */ 3423 pushq %r9 /* | %r9 | 0x28 */ 3424 pushq %r8 /* | %r8 | 0x20 */ 3425 pushq %rcx /* | %rcx | 0x18 */ 3426 pushq %rdx /* | %rdx | 0x10 */ 3427 pushq %rsi /* | %rsi | 0x8 alist */ 3428 pushq %rdi /* | %rdi | 0x0 format */ 3429 3430 movq %rsp, %rbx /* %rbx = current %rsp */ 3431 3432 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3433 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 3434 jmp vpanic_common 3435 3436 SET_SIZE(dtrace_vpanic) 3437 3438#elif defined(__i386) 3439 3440 ENTRY_NP(vpanic) / Initial stack layout: 3441 3442 pushl %ebp / | %eip | 20 3443 movl %esp, %ebp / | %ebp | 16 3444 pushl %eax / | %eax | 12 3445 pushl %ebx / | %ebx | 8 3446 pushl %ecx / | %ecx | 4 3447 pushl %edx / | %edx | 0 3448 3449 movl %esp, %ebx / %ebx = current stack pointer 3450 3451 lea panic_quiesce, %eax / %eax = &panic_quiesce 3452 pushl %eax / push &panic_quiesce 3453 call panic_trigger / %eax = panic_trigger() 3454 addl $4, %esp / reset stack pointer 3455 3456vpanic_common: 3457 cmpl $0, %eax / if (%eax == 0) 3458 je 0f / goto 0f; 3459 3460 /* 3461 * If panic_trigger() was successful, we are the first to initiate a 3462 * panic: we now switch to the reserved panic_stack before continuing. 3463 */ 3464 lea panic_stack, %esp / %esp = panic_stack 3465 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE 3466 34670: subl $REGSIZE, %esp / allocate struct regs 3468 3469 /* 3470 * Now that we've got everything set up, store the register values as 3471 * they were when we entered vpanic() to the designated location in 3472 * the regs structure we allocated on the stack. 3473 */ 3474#if !defined(__GNUC_AS__) 3475 movw %gs, %edx 3476 movl %edx, REGOFF_GS(%esp) 3477 movw %fs, %edx 3478 movl %edx, REGOFF_FS(%esp) 3479 movw %es, %edx 3480 movl %edx, REGOFF_ES(%esp) 3481 movw %ds, %edx 3482 movl %edx, REGOFF_DS(%esp) 3483#else /* __GNUC_AS__ */ 3484 mov %gs, %edx 3485 mov %edx, REGOFF_GS(%esp) 3486 mov %fs, %edx 3487 mov %edx, REGOFF_FS(%esp) 3488 mov %es, %edx 3489 mov %edx, REGOFF_ES(%esp) 3490 mov %ds, %edx 3491 mov %edx, REGOFF_DS(%esp) 3492#endif /* __GNUC_AS__ */ 3493 movl %edi, REGOFF_EDI(%esp) 3494 movl %esi, REGOFF_ESI(%esp) 3495 movl 16(%ebx), %ecx 3496 movl %ecx, REGOFF_EBP(%esp) 3497 movl %ebx, %ecx 3498 addl $20, %ecx 3499 movl %ecx, REGOFF_ESP(%esp) 3500 movl 8(%ebx), %ecx 3501 movl %ecx, REGOFF_EBX(%esp) 3502 movl 0(%ebx), %ecx 3503 movl %ecx, REGOFF_EDX(%esp) 3504 movl 4(%ebx), %ecx 3505 movl %ecx, REGOFF_ECX(%esp) 3506 movl 12(%ebx), %ecx 3507 movl %ecx, REGOFF_EAX(%esp) 3508 movl $0, REGOFF_TRAPNO(%esp) 3509 movl $0, REGOFF_ERR(%esp) 3510 lea vpanic, %ecx 3511 movl %ecx, REGOFF_EIP(%esp) 3512#if !defined(__GNUC_AS__) 3513 movw %cs, %edx 3514#else /* __GNUC_AS__ */ 3515 mov %cs, %edx 3516#endif /* __GNUC_AS__ */ 3517 movl %edx, REGOFF_CS(%esp) 3518 pushfl 3519 popl %ecx 3520#if defined(__xpv) 3521 /* 3522 * Synthesize the PS_IE bit from the event mask bit 3523 */ 3524 CURTHREAD(%edx) 3525 KPREEMPT_DISABLE(%edx) 3526 EVENT_MASK_TO_IE(%edx, %ecx) 3527 CURTHREAD(%edx) 3528 KPREEMPT_ENABLE_NOKP(%edx) 3529#endif 3530 movl %ecx, REGOFF_EFL(%esp) 3531 movl $0, REGOFF_UESP(%esp) 3532#if !defined(__GNUC_AS__) 3533 movw %ss, %edx 3534#else /* __GNUC_AS__ */ 3535 mov %ss, %edx 3536#endif /* __GNUC_AS__ */ 3537 movl %edx, REGOFF_SS(%esp) 3538 3539 movl %esp, %ecx / %ecx = ®s 3540 pushl %eax / push on_panic_stack 3541 pushl %ecx / push ®s 3542 movl 12(%ebp), %ecx / %ecx = alist 3543 pushl %ecx / push alist 3544 movl 8(%ebp), %ecx / %ecx = format 3545 pushl %ecx / push format 3546 call panicsys / panicsys(); 3547 addl $16, %esp / pop arguments 3548 3549 addl $REGSIZE, %esp 3550 popl %edx 3551 popl %ecx 3552 popl %ebx 3553 popl %eax 3554 leave 3555 ret 3556 SET_SIZE(vpanic) 3557 3558 ENTRY_NP(dtrace_vpanic) / Initial stack layout: 3559 3560 pushl %ebp / | %eip | 20 3561 movl %esp, %ebp / | %ebp | 16 3562 pushl %eax / | %eax | 12 3563 pushl %ebx / | %ebx | 8 3564 pushl %ecx / | %ecx | 4 3565 pushl %edx / | %edx | 0 3566 3567 movl %esp, %ebx / %ebx = current stack pointer 3568 3569 lea panic_quiesce, %eax / %eax = &panic_quiesce 3570 pushl %eax / push &panic_quiesce 3571 call dtrace_panic_trigger / %eax = dtrace_panic_trigger() 3572 addl $4, %esp / reset stack pointer 3573 jmp vpanic_common / jump back to common code 3574 3575 SET_SIZE(dtrace_vpanic) 3576 3577#endif /* __i386 */ 3578#endif /* __lint */ 3579 3580#if defined(__lint) 3581 3582void 3583hres_tick(void) 3584{} 3585 3586int64_t timedelta; 3587hrtime_t hres_last_tick; 3588volatile timestruc_t hrestime; 3589int64_t hrestime_adj; 3590volatile int hres_lock; 3591hrtime_t hrtime_base; 3592 3593#else /* __lint */ 3594 3595 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8) 3596 .NWORD 0, 0 3597 3598 DGDEF3(hrestime_adj, 8, 8) 3599 .long 0, 0 3600 3601 DGDEF3(hres_last_tick, 8, 8) 3602 .long 0, 0 3603 3604 DGDEF3(timedelta, 8, 8) 3605 .long 0, 0 3606 3607 DGDEF3(hres_lock, 4, 8) 3608 .long 0 3609 3610 /* 3611 * initialized to a non zero value to make pc_gethrtime() 3612 * work correctly even before clock is initialized 3613 */ 3614 DGDEF3(hrtime_base, 8, 8) 3615 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0 3616 3617 DGDEF3(adj_shift, 4, 4) 3618 .long ADJ_SHIFT 3619 3620#if defined(__amd64) 3621 3622 ENTRY_NP(hres_tick) 3623 pushq %rbp 3624 movq %rsp, %rbp 3625 3626 /* 3627 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3628 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3629 * At worst, performing this now instead of under CLOCK_LOCK may 3630 * introduce some jitter in pc_gethrestime(). 3631 */ 3632 call *gethrtimef(%rip) 3633 movq %rax, %r8 3634 3635 leaq hres_lock(%rip), %rax 3636 movb $-1, %dl 3637.CL1: 3638 xchgb %dl, (%rax) 3639 testb %dl, %dl 3640 jz .CL3 /* got it */ 3641.CL2: 3642 cmpb $0, (%rax) /* possible to get lock? */ 3643 pause 3644 jne .CL2 3645 jmp .CL1 /* yes, try again */ 3646.CL3: 3647 /* 3648 * compute the interval since last time hres_tick was called 3649 * and adjust hrtime_base and hrestime accordingly 3650 * hrtime_base is an 8 byte value (in nsec), hrestime is 3651 * a timestruc_t (sec, nsec) 3652 */ 3653 leaq hres_last_tick(%rip), %rax 3654 movq %r8, %r11 3655 subq (%rax), %r8 3656 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */ 3657 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */ 3658 /* 3659 * Now that we have CLOCK_LOCK, we can update hres_last_tick 3660 */ 3661 movq %r11, (%rax) 3662 3663 call __adj_hrestime 3664 3665 /* 3666 * release the hres_lock 3667 */ 3668 incl hres_lock(%rip) 3669 leave 3670 ret 3671 SET_SIZE(hres_tick) 3672 3673#elif defined(__i386) 3674 3675 ENTRY_NP(hres_tick) 3676 pushl %ebp 3677 movl %esp, %ebp 3678 pushl %esi 3679 pushl %ebx 3680 3681 /* 3682 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3683 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3684 * At worst, performing this now instead of under CLOCK_LOCK may 3685 * introduce some jitter in pc_gethrestime(). 3686 */ 3687 call *gethrtimef 3688 movl %eax, %ebx 3689 movl %edx, %esi 3690 3691 movl $hres_lock, %eax 3692 movl $-1, %edx 3693.CL1: 3694 xchgb %dl, (%eax) 3695 testb %dl, %dl 3696 jz .CL3 / got it 3697.CL2: 3698 cmpb $0, (%eax) / possible to get lock? 3699 pause 3700 jne .CL2 3701 jmp .CL1 / yes, try again 3702.CL3: 3703 /* 3704 * compute the interval since last time hres_tick was called 3705 * and adjust hrtime_base and hrestime accordingly 3706 * hrtime_base is an 8 byte value (in nsec), hrestime is 3707 * timestruc_t (sec, nsec) 3708 */ 3709 3710 lea hres_last_tick, %eax 3711 3712 movl %ebx, %edx 3713 movl %esi, %ecx 3714 3715 subl (%eax), %edx 3716 sbbl 4(%eax), %ecx 3717 3718 addl %edx, hrtime_base / add interval to hrtime_base 3719 adcl %ecx, hrtime_base+4 3720 3721 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec 3722 3723 / 3724 / Now that we have CLOCK_LOCK, we can update hres_last_tick. 3725 / 3726 movl %ebx, (%eax) 3727 movl %esi, 4(%eax) 3728 3729 / get hrestime at this moment. used as base for pc_gethrestime 3730 / 3731 / Apply adjustment, if any 3732 / 3733 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT) 3734 / (max_hres_adj) 3735 / 3736 / void 3737 / adj_hrestime() 3738 / { 3739 / long long adj; 3740 / 3741 / if (hrestime_adj == 0) 3742 / adj = 0; 3743 / else if (hrestime_adj > 0) { 3744 / if (hrestime_adj < HRES_ADJ) 3745 / adj = hrestime_adj; 3746 / else 3747 / adj = HRES_ADJ; 3748 / } 3749 / else { 3750 / if (hrestime_adj < -(HRES_ADJ)) 3751 / adj = -(HRES_ADJ); 3752 / else 3753 / adj = hrestime_adj; 3754 / } 3755 / 3756 / timedelta -= adj; 3757 / hrestime_adj = timedelta; 3758 / hrestime.tv_nsec += adj; 3759 / 3760 / while (hrestime.tv_nsec >= NANOSEC) { 3761 / one_sec++; 3762 / hrestime.tv_sec++; 3763 / hrestime.tv_nsec -= NANOSEC; 3764 / } 3765 / } 3766__adj_hrestime: 3767 movl hrestime_adj, %esi / if (hrestime_adj == 0) 3768 movl hrestime_adj+4, %edx 3769 andl %esi, %esi 3770 jne .CL4 / no 3771 andl %edx, %edx 3772 jne .CL4 / no 3773 subl %ecx, %ecx / yes, adj = 0; 3774 subl %edx, %edx 3775 jmp .CL5 3776.CL4: 3777 subl %ecx, %ecx 3778 subl %eax, %eax 3779 subl %esi, %ecx 3780 sbbl %edx, %eax 3781 andl %eax, %eax / if (hrestime_adj > 0) 3782 jge .CL6 3783 3784 / In the following comments, HRES_ADJ is used, while in the code 3785 / max_hres_adj is used. 3786 / 3787 / The test for "hrestime_adj < HRES_ADJ" is complicated because 3788 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3789 / on the logical equivalence of: 3790 / 3791 / !(hrestime_adj < HRES_ADJ) 3792 / 3793 / and the two step sequence: 3794 / 3795 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry 3796 / 3797 / which computes whether or not the least significant 32-bits 3798 / of hrestime_adj is greater than HRES_ADJ, followed by: 3799 / 3800 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry 3801 / 3802 / which generates a carry whenever step 1 is true or the most 3803 / significant long of the longlong hrestime_adj is non-zero. 3804 3805 movl max_hres_adj, %ecx / hrestime_adj is positive 3806 subl %esi, %ecx 3807 movl %edx, %eax 3808 adcl $-1, %eax 3809 jnc .CL7 3810 movl max_hres_adj, %ecx / adj = HRES_ADJ; 3811 subl %edx, %edx 3812 jmp .CL5 3813 3814 / The following computation is similar to the one above. 3815 / 3816 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because 3817 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3818 / on the logical equivalence of: 3819 / 3820 / (hrestime_adj > -HRES_ADJ) 3821 / 3822 / and the two step sequence: 3823 / 3824 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry 3825 / 3826 / which means the least significant 32-bits of hrestime_adj is 3827 / greater than -HRES_ADJ, followed by: 3828 / 3829 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry 3830 / 3831 / which generates a carry only when step 1 is true and the most 3832 / significant long of the longlong hrestime_adj is -1. 3833 3834.CL6: / hrestime_adj is negative 3835 movl %esi, %ecx 3836 addl max_hres_adj, %ecx 3837 movl %edx, %eax 3838 adcl $0, %eax 3839 jc .CL7 3840 xor %ecx, %ecx 3841 subl max_hres_adj, %ecx / adj = -(HRES_ADJ); 3842 movl $-1, %edx 3843 jmp .CL5 3844.CL7: 3845 movl %esi, %ecx / adj = hrestime_adj; 3846.CL5: 3847 movl timedelta, %esi 3848 subl %ecx, %esi 3849 movl timedelta+4, %eax 3850 sbbl %edx, %eax 3851 movl %esi, timedelta 3852 movl %eax, timedelta+4 / timedelta -= adj; 3853 movl %esi, hrestime_adj 3854 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta; 3855 addl hrestime+4, %ecx 3856 3857 movl %ecx, %eax / eax = tv_nsec 38581: 3859 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC) 3860 jb .CL8 / no 3861 incl one_sec / yes, one_sec++; 3862 incl hrestime / hrestime.tv_sec++; 3863 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC 3864 jmp 1b / check for more seconds 3865 3866.CL8: 3867 movl %eax, hrestime+4 / store final into hrestime.tv_nsec 3868 incl hres_lock / release the hres_lock 3869 3870 popl %ebx 3871 popl %esi 3872 leave 3873 ret 3874 SET_SIZE(hres_tick) 3875 3876#endif /* __i386 */ 3877#endif /* __lint */ 3878 3879/* 3880 * void prefetch_smap_w(void *) 3881 * 3882 * Prefetch ahead within a linear list of smap structures. 3883 * Not implemented for ia32. Stub for compatibility. 3884 */ 3885 3886#if defined(__lint) 3887 3888/*ARGSUSED*/ 3889void prefetch_smap_w(void *smp) 3890{} 3891 3892#else /* __lint */ 3893 3894 ENTRY(prefetch_smap_w) 3895 rep; ret /* use 2 byte return instruction when branch target */ 3896 /* AMD Software Optimization Guide - Section 6.2 */ 3897 SET_SIZE(prefetch_smap_w) 3898 3899#endif /* __lint */ 3900 3901/* 3902 * prefetch_page_r(page_t *) 3903 * issue prefetch instructions for a page_t 3904 */ 3905#if defined(__lint) 3906 3907/*ARGSUSED*/ 3908void 3909prefetch_page_r(void *pp) 3910{} 3911 3912#else /* __lint */ 3913 3914 ENTRY(prefetch_page_r) 3915 rep; ret /* use 2 byte return instruction when branch target */ 3916 /* AMD Software Optimization Guide - Section 6.2 */ 3917 SET_SIZE(prefetch_page_r) 3918 3919#endif /* __lint */ 3920 3921#if defined(__lint) 3922 3923/*ARGSUSED*/ 3924int 3925bcmp(const void *s1, const void *s2, size_t count) 3926{ return (0); } 3927 3928#else /* __lint */ 3929 3930#if defined(__amd64) 3931 3932 ENTRY(bcmp) 3933 pushq %rbp 3934 movq %rsp, %rbp 3935#ifdef DEBUG 3936 testq %rdx,%rdx 3937 je 1f 3938 movq postbootkernelbase(%rip), %r11 3939 cmpq %r11, %rdi 3940 jb 0f 3941 cmpq %r11, %rsi 3942 jnb 1f 39430: leaq .bcmp_panic_msg(%rip), %rdi 3944 xorl %eax, %eax 3945 call panic 39461: 3947#endif /* DEBUG */ 3948 call memcmp 3949 testl %eax, %eax 3950 setne %dl 3951 leave 3952 movzbl %dl, %eax 3953 ret 3954 SET_SIZE(bcmp) 3955 3956#elif defined(__i386) 3957 3958#define ARG_S1 8 3959#define ARG_S2 12 3960#define ARG_LENGTH 16 3961 3962 ENTRY(bcmp) 3963 pushl %ebp 3964 movl %esp, %ebp / create new stack frame 3965#ifdef DEBUG 3966 cmpl $0, ARG_LENGTH(%ebp) 3967 je 1f 3968 movl postbootkernelbase, %eax 3969 cmpl %eax, ARG_S1(%ebp) 3970 jb 0f 3971 cmpl %eax, ARG_S2(%ebp) 3972 jnb 1f 39730: pushl $.bcmp_panic_msg 3974 call panic 39751: 3976#endif /* DEBUG */ 3977 3978 pushl %edi / save register variable 3979 movl ARG_S1(%ebp), %eax / %eax = address of string 1 3980 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2 3981 cmpl %eax, %ecx / if the same string 3982 je .equal / goto .equal 3983 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes 3984 cmpl $4, %edi / if %edi < 4 3985 jb .byte_check / goto .byte_check 3986 .align 4 3987.word_loop: 3988 movl (%ecx), %edx / move 1 word from (%ecx) to %edx 3989 leal -4(%edi), %edi / %edi -= 4 3990 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx 3991 jne .word_not_equal / if not equal, goto .word_not_equal 3992 leal 4(%ecx), %ecx / %ecx += 4 (next word) 3993 leal 4(%eax), %eax / %eax += 4 (next word) 3994 cmpl $4, %edi / if %edi >= 4 3995 jae .word_loop / goto .word_loop 3996.byte_check: 3997 cmpl $0, %edi / if %edi == 0 3998 je .equal / goto .equal 3999 jmp .byte_loop / goto .byte_loop (checks in bytes) 4000.word_not_equal: 4001 leal 4(%edi), %edi / %edi += 4 (post-decremented) 4002 .align 4 4003.byte_loop: 4004 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl 4005 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax) 4006 jne .not_equal / if not equal, goto .not_equal 4007 incl %ecx / %ecx++ (next byte) 4008 incl %eax / %eax++ (next byte) 4009 decl %edi / %edi-- 4010 jnz .byte_loop / if not zero, goto .byte_loop 4011.equal: 4012 xorl %eax, %eax / %eax = 0 4013 popl %edi / restore register variable 4014 leave / restore old stack frame 4015 ret / return (NULL) 4016 .align 4 4017.not_equal: 4018 movl $1, %eax / return 1 4019 popl %edi / restore register variable 4020 leave / restore old stack frame 4021 ret / return (NULL) 4022 SET_SIZE(bcmp) 4023 4024#endif /* __i386 */ 4025 4026#ifdef DEBUG 4027 .text 4028.bcmp_panic_msg: 4029 .string "bcmp: arguments below kernelbase" 4030#endif /* DEBUG */ 4031 4032#endif /* __lint */ 4033 4034#if defined(__lint) 4035 4036uint_t 4037bsrw_insn(uint16_t mask) 4038{ 4039 uint_t index = sizeof (mask) * NBBY - 1; 4040 4041 while ((mask & (1 << index)) == 0) 4042 index--; 4043 return (index); 4044} 4045 4046#else /* __lint */ 4047 4048#if defined(__amd64) 4049 4050 ENTRY_NP(bsrw_insn) 4051 xorl %eax, %eax 4052 bsrw %di, %ax 4053 ret 4054 SET_SIZE(bsrw_insn) 4055 4056#elif defined(__i386) 4057 4058 ENTRY_NP(bsrw_insn) 4059 movw 4(%esp), %cx 4060 xorl %eax, %eax 4061 bsrw %cx, %ax 4062 ret 4063 SET_SIZE(bsrw_insn) 4064 4065#endif /* __i386 */ 4066#endif /* __lint */ 4067 4068#if defined(__lint) 4069 4070uint_t 4071atomic_btr32(uint32_t *pending, uint_t pil) 4072{ 4073 return (*pending &= ~(1 << pil)); 4074} 4075 4076#else /* __lint */ 4077 4078#if defined(__i386) 4079 4080 ENTRY_NP(atomic_btr32) 4081 movl 4(%esp), %ecx 4082 movl 8(%esp), %edx 4083 xorl %eax, %eax 4084 lock 4085 btrl %edx, (%ecx) 4086 setc %al 4087 ret 4088 SET_SIZE(atomic_btr32) 4089 4090#endif /* __i386 */ 4091#endif /* __lint */ 4092 4093#if defined(__lint) 4094 4095/*ARGSUSED*/ 4096void 4097switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1, 4098 uint_t arg2) 4099{} 4100 4101#else /* __lint */ 4102 4103#if defined(__amd64) 4104 4105 ENTRY_NP(switch_sp_and_call) 4106 pushq %rbp 4107 movq %rsp, %rbp /* set up stack frame */ 4108 movq %rdi, %rsp /* switch stack pointer */ 4109 movq %rdx, %rdi /* pass func arg 1 */ 4110 movq %rsi, %r11 /* save function to call */ 4111 movq %rcx, %rsi /* pass func arg 2 */ 4112 call *%r11 /* call function */ 4113 leave /* restore stack */ 4114 ret 4115 SET_SIZE(switch_sp_and_call) 4116 4117#elif defined(__i386) 4118 4119 ENTRY_NP(switch_sp_and_call) 4120 pushl %ebp 4121 mov %esp, %ebp /* set up stack frame */ 4122 movl 8(%ebp), %esp /* switch stack pointer */ 4123 pushl 20(%ebp) /* push func arg 2 */ 4124 pushl 16(%ebp) /* push func arg 1 */ 4125 call *12(%ebp) /* call function */ 4126 addl $8, %esp /* pop arguments */ 4127 leave /* restore stack */ 4128 ret 4129 SET_SIZE(switch_sp_and_call) 4130 4131#endif /* __i386 */ 4132#endif /* __lint */ 4133 4134#if defined(__lint) 4135 4136void 4137kmdb_enter(void) 4138{} 4139 4140#else /* __lint */ 4141 4142#if defined(__amd64) 4143 4144 ENTRY_NP(kmdb_enter) 4145 pushq %rbp 4146 movq %rsp, %rbp 4147 4148 /* 4149 * Save flags, do a 'cli' then return the saved flags 4150 */ 4151 call intr_clear 4152 4153 int $T_DBGENTR 4154 4155 /* 4156 * Restore the saved flags 4157 */ 4158 movq %rax, %rdi 4159 call intr_restore 4160 4161 leave 4162 ret 4163 SET_SIZE(kmdb_enter) 4164 4165#elif defined(__i386) 4166 4167 ENTRY_NP(kmdb_enter) 4168 pushl %ebp 4169 movl %esp, %ebp 4170 4171 /* 4172 * Save flags, do a 'cli' then return the saved flags 4173 */ 4174 call intr_clear 4175 4176 int $T_DBGENTR 4177 4178 /* 4179 * Restore the saved flags 4180 */ 4181 pushl %eax 4182 call intr_restore 4183 addl $4, %esp 4184 4185 leave 4186 ret 4187 SET_SIZE(kmdb_enter) 4188 4189#endif /* __i386 */ 4190#endif /* __lint */ 4191 4192#if defined(__lint) 4193 4194void 4195return_instr(void) 4196{} 4197 4198#else /* __lint */ 4199 4200 ENTRY_NP(return_instr) 4201 rep; ret /* use 2 byte instruction when branch target */ 4202 /* AMD Software Optimization Guide - Section 6.2 */ 4203 SET_SIZE(return_instr) 4204 4205#endif /* __lint */ 4206 4207#if defined(__lint) 4208 4209ulong_t 4210getflags(void) 4211{ 4212 return (0); 4213} 4214 4215#else /* __lint */ 4216 4217#if defined(__amd64) 4218 4219 ENTRY(getflags) 4220 pushfq 4221 popq %rax 4222#if defined(__xpv) 4223 CURTHREAD(%rdi) 4224 KPREEMPT_DISABLE(%rdi) 4225 /* 4226 * Synthesize the PS_IE bit from the event mask bit 4227 */ 4228 CURVCPU(%r11) 4229 andq $_BITNOT(PS_IE), %rax 4230 XEN_TEST_UPCALL_MASK(%r11) 4231 jnz 1f 4232 orq $PS_IE, %rax 42331: 4234 KPREEMPT_ENABLE_NOKP(%rdi) 4235#endif 4236 ret 4237 SET_SIZE(getflags) 4238 4239#elif defined(__i386) 4240 4241 ENTRY(getflags) 4242 pushfl 4243 popl %eax 4244#if defined(__xpv) 4245 CURTHREAD(%ecx) 4246 KPREEMPT_DISABLE(%ecx) 4247 /* 4248 * Synthesize the PS_IE bit from the event mask bit 4249 */ 4250 CURVCPU(%edx) 4251 andl $_BITNOT(PS_IE), %eax 4252 XEN_TEST_UPCALL_MASK(%edx) 4253 jnz 1f 4254 orl $PS_IE, %eax 42551: 4256 KPREEMPT_ENABLE_NOKP(%ecx) 4257#endif 4258 ret 4259 SET_SIZE(getflags) 4260 4261#endif /* __i386 */ 4262 4263#endif /* __lint */ 4264 4265#if defined(__lint) 4266 4267ftrace_icookie_t 4268ftrace_interrupt_disable(void) 4269{ return (0); } 4270 4271#else /* __lint */ 4272 4273#if defined(__amd64) 4274 4275 ENTRY(ftrace_interrupt_disable) 4276 pushfq 4277 popq %rax 4278 CLI(%rdx) 4279 ret 4280 SET_SIZE(ftrace_interrupt_disable) 4281 4282#elif defined(__i386) 4283 4284 ENTRY(ftrace_interrupt_disable) 4285 pushfl 4286 popl %eax 4287 CLI(%edx) 4288 ret 4289 SET_SIZE(ftrace_interrupt_disable) 4290 4291#endif /* __i386 */ 4292#endif /* __lint */ 4293 4294#if defined(__lint) 4295 4296/*ARGSUSED*/ 4297void 4298ftrace_interrupt_enable(ftrace_icookie_t cookie) 4299{} 4300 4301#else /* __lint */ 4302 4303#if defined(__amd64) 4304 4305 ENTRY(ftrace_interrupt_enable) 4306 pushq %rdi 4307 popfq 4308 ret 4309 SET_SIZE(ftrace_interrupt_enable) 4310 4311#elif defined(__i386) 4312 4313 ENTRY(ftrace_interrupt_enable) 4314 movl 4(%esp), %eax 4315 pushl %eax 4316 popfl 4317 ret 4318 SET_SIZE(ftrace_interrupt_enable) 4319 4320#endif /* __i386 */ 4321#endif /* __lint */ 4322 4323#if defined (__lint) 4324 4325/*ARGSUSED*/ 4326void 4327clflush_insn(caddr_t addr) 4328{} 4329 4330#else /* __lint */ 4331 4332#if defined (__amd64) 4333 ENTRY(clflush_insn) 4334 clflush (%rdi) 4335 ret 4336 SET_SIZE(clflush_insn) 4337#elif defined (__i386) 4338 ENTRY(clflush_insn) 4339 movl 4(%esp), %eax 4340 clflush (%eax) 4341 ret 4342 SET_SIZE(clflush_insn) 4343 4344#endif /* __i386 */ 4345#endif /* __lint */ 4346 4347#if defined (__lint) 4348/*ARGSUSED*/ 4349void 4350mfence_insn(void) 4351{} 4352 4353#else /* __lint */ 4354 4355#if defined (__amd64) 4356 ENTRY(mfence_insn) 4357 mfence 4358 ret 4359 SET_SIZE(mfence_insn) 4360#elif defined (__i386) 4361 ENTRY(mfence_insn) 4362 mfence 4363 ret 4364 SET_SIZE(mfence_insn) 4365 4366#endif /* __i386 */ 4367#endif /* __lint */ 4368 4369/* 4370 * VMware implements an I/O port that programs can query to detect if software 4371 * is running in a VMware hypervisor. This hypervisor port behaves differently 4372 * depending on magic values in certain registers and modifies some registers 4373 * as a side effect. 4374 * 4375 * References: http://kb.vmware.com/kb/1009458 4376 */ 4377 4378#if defined(__lint) 4379 4380/* ARGSUSED */ 4381void 4382vmware_port(int cmd, uint32_t *regs) { return; } 4383 4384#else 4385 4386#if defined(__amd64) 4387 4388 ENTRY(vmware_port) 4389 pushq %rbx 4390 movl $VMWARE_HVMAGIC, %eax 4391 movl $0xffffffff, %ebx 4392 movl %edi, %ecx 4393 movl $VMWARE_HVPORT, %edx 4394 inl (%dx) 4395 movl %eax, (%rsi) 4396 movl %ebx, 4(%rsi) 4397 movl %ecx, 8(%rsi) 4398 movl %edx, 12(%rsi) 4399 popq %rbx 4400 ret 4401 SET_SIZE(vmware_port) 4402 4403#elif defined(__i386) 4404 4405 ENTRY(vmware_port) 4406 pushl %ebx 4407 pushl %esi 4408 movl $VMWARE_HVMAGIC, %eax 4409 movl $0xffffffff, %ebx 4410 movl 12(%esp), %ecx 4411 movl $VMWARE_HVPORT, %edx 4412 inl (%dx) 4413 movl 16(%esp), %esi 4414 movl %eax, (%esi) 4415 movl %ebx, 4(%esi) 4416 movl %ecx, 8(%esi) 4417 movl %edx, 12(%esi) 4418 popl %esi 4419 popl %ebx 4420 ret 4421 SET_SIZE(vmware_port) 4422 4423#endif /* __i386 */ 4424#endif /* __lint */ 4425