1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27/* 28 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. 29 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T 30 * All Rights Reserved 31 */ 32 33/* 34 * General assembly language routines. 35 * It is the intent of this file to contain routines that are 36 * independent of the specific kernel architecture, and those that are 37 * common across kernel architectures. 38 * As architectures diverge, and implementations of specific 39 * architecture-dependent routines change, the routines should be moved 40 * from this file into the respective ../`arch -k`/subr.s file. 41 */ 42 43#include <sys/asm_linkage.h> 44#include <sys/asm_misc.h> 45#include <sys/panic.h> 46#include <sys/ontrap.h> 47#include <sys/regset.h> 48#include <sys/privregs.h> 49#include <sys/reboot.h> 50#include <sys/psw.h> 51#include <sys/x86_archext.h> 52 53#if defined(__lint) 54#include <sys/types.h> 55#include <sys/systm.h> 56#include <sys/thread.h> 57#include <sys/archsystm.h> 58#include <sys/byteorder.h> 59#include <sys/dtrace.h> 60#include <sys/ftrace.h> 61#else /* __lint */ 62#include "assym.h" 63#endif /* __lint */ 64#include <sys/dditypes.h> 65 66/* 67 * on_fault() 68 * Catch lofault faults. Like setjmp except it returns one 69 * if code following causes uncorrectable fault. Turned off 70 * by calling no_fault(). 71 */ 72 73#if defined(__lint) 74 75/* ARGSUSED */ 76int 77on_fault(label_t *ljb) 78{ return (0); } 79 80void 81no_fault(void) 82{} 83 84#else /* __lint */ 85 86#if defined(__amd64) 87 88 ENTRY(on_fault) 89 movq %gs:CPU_THREAD, %rsi 90 leaq catch_fault(%rip), %rdx 91 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */ 92 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */ 93 jmp setjmp /* let setjmp do the rest */ 94 95catch_fault: 96 movq %gs:CPU_THREAD, %rsi 97 movq T_ONFAULT(%rsi), %rdi /* address of save area */ 98 xorl %eax, %eax 99 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 100 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 101 jmp longjmp /* let longjmp do the rest */ 102 SET_SIZE(on_fault) 103 104 ENTRY(no_fault) 105 movq %gs:CPU_THREAD, %rsi 106 xorl %eax, %eax 107 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 108 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 109 ret 110 SET_SIZE(no_fault) 111 112#elif defined(__i386) 113 114 ENTRY(on_fault) 115 movl %gs:CPU_THREAD, %edx 116 movl 4(%esp), %eax /* jumpbuf address */ 117 leal catch_fault, %ecx 118 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */ 119 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */ 120 jmp setjmp /* let setjmp do the rest */ 121 122catch_fault: 123 movl %gs:CPU_THREAD, %edx 124 xorl %eax, %eax 125 movl T_ONFAULT(%edx), %ecx /* address of save area */ 126 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 127 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 128 pushl %ecx 129 call longjmp /* let longjmp do the rest */ 130 SET_SIZE(on_fault) 131 132 ENTRY(no_fault) 133 movl %gs:CPU_THREAD, %edx 134 xorl %eax, %eax 135 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 136 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 137 ret 138 SET_SIZE(no_fault) 139 140#endif /* __i386 */ 141#endif /* __lint */ 142 143/* 144 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just 145 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called. 146 */ 147 148#if defined(lint) 149 150void 151on_trap_trampoline(void) 152{} 153 154#else /* __lint */ 155 156#if defined(__amd64) 157 158 ENTRY(on_trap_trampoline) 159 movq %gs:CPU_THREAD, %rsi 160 movq T_ONTRAP(%rsi), %rdi 161 addq $OT_JMPBUF, %rdi 162 jmp longjmp 163 SET_SIZE(on_trap_trampoline) 164 165#elif defined(__i386) 166 167 ENTRY(on_trap_trampoline) 168 movl %gs:CPU_THREAD, %eax 169 movl T_ONTRAP(%eax), %eax 170 addl $OT_JMPBUF, %eax 171 pushl %eax 172 call longjmp 173 SET_SIZE(on_trap_trampoline) 174 175#endif /* __i386 */ 176#endif /* __lint */ 177 178/* 179 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 180 * more information about the on_trap() mechanism. If the on_trap_data is the 181 * same as the topmost stack element, we just modify that element. 182 */ 183#if defined(lint) 184 185/*ARGSUSED*/ 186int 187on_trap(on_trap_data_t *otp, uint_t prot) 188{ return (0); } 189 190#else /* __lint */ 191 192#if defined(__amd64) 193 194 ENTRY(on_trap) 195 movw %si, OT_PROT(%rdi) /* ot_prot = prot */ 196 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */ 197 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */ 198 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */ 199 xorl %ecx, %ecx 200 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */ 201 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */ 202 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */ 203 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */ 204 cmpq %rdi, %rcx /* if (otp == %rcx) */ 205 je 0f /* don't modify t_ontrap */ 206 207 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */ 208 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */ 209 2100: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */ 211 jmp setjmp 212 SET_SIZE(on_trap) 213 214#elif defined(__i386) 215 216 ENTRY(on_trap) 217 movl 4(%esp), %eax /* %eax = otp */ 218 movl 8(%esp), %edx /* %edx = prot */ 219 220 movw %dx, OT_PROT(%eax) /* ot_prot = prot */ 221 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */ 222 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */ 223 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */ 224 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */ 225 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */ 226 movl %gs:CPU_THREAD, %edx /* %edx = curthread */ 227 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */ 228 cmpl %eax, %ecx /* if (otp == %ecx) */ 229 je 0f /* don't modify t_ontrap */ 230 231 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */ 232 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */ 233 2340: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */ 235 movl %eax, 4(%esp) /* put %eax back on the stack */ 236 jmp setjmp /* let setjmp do the rest */ 237 SET_SIZE(on_trap) 238 239#endif /* __i386 */ 240#endif /* __lint */ 241 242/* 243 * Setjmp and longjmp implement non-local gotos using state vectors 244 * type label_t. 245 */ 246 247#if defined(__lint) 248 249/* ARGSUSED */ 250int 251setjmp(label_t *lp) 252{ return (0); } 253 254/* ARGSUSED */ 255void 256longjmp(label_t *lp) 257{} 258 259#else /* __lint */ 260 261#if LABEL_PC != 0 262#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded 263#endif /* LABEL_PC != 0 */ 264 265#if defined(__amd64) 266 267 ENTRY(setjmp) 268 movq %rsp, LABEL_SP(%rdi) 269 movq %rbp, LABEL_RBP(%rdi) 270 movq %rbx, LABEL_RBX(%rdi) 271 movq %r12, LABEL_R12(%rdi) 272 movq %r13, LABEL_R13(%rdi) 273 movq %r14, LABEL_R14(%rdi) 274 movq %r15, LABEL_R15(%rdi) 275 movq (%rsp), %rdx /* return address */ 276 movq %rdx, (%rdi) /* LABEL_PC is 0 */ 277 xorl %eax, %eax /* return 0 */ 278 ret 279 SET_SIZE(setjmp) 280 281 ENTRY(longjmp) 282 movq LABEL_SP(%rdi), %rsp 283 movq LABEL_RBP(%rdi), %rbp 284 movq LABEL_RBX(%rdi), %rbx 285 movq LABEL_R12(%rdi), %r12 286 movq LABEL_R13(%rdi), %r13 287 movq LABEL_R14(%rdi), %r14 288 movq LABEL_R15(%rdi), %r15 289 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */ 290 movq %rdx, (%rsp) 291 xorl %eax, %eax 292 incl %eax /* return 1 */ 293 ret 294 SET_SIZE(longjmp) 295 296#elif defined(__i386) 297 298 ENTRY(setjmp) 299 movl 4(%esp), %edx /* address of save area */ 300 movl %ebp, LABEL_EBP(%edx) 301 movl %ebx, LABEL_EBX(%edx) 302 movl %esi, LABEL_ESI(%edx) 303 movl %edi, LABEL_EDI(%edx) 304 movl %esp, 4(%edx) 305 movl (%esp), %ecx /* %eip (return address) */ 306 movl %ecx, (%edx) /* LABEL_PC is 0 */ 307 subl %eax, %eax /* return 0 */ 308 ret 309 SET_SIZE(setjmp) 310 311 ENTRY(longjmp) 312 movl 4(%esp), %edx /* address of save area */ 313 movl LABEL_EBP(%edx), %ebp 314 movl LABEL_EBX(%edx), %ebx 315 movl LABEL_ESI(%edx), %esi 316 movl LABEL_EDI(%edx), %edi 317 movl 4(%edx), %esp 318 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */ 319 movl $1, %eax 320 addl $4, %esp /* pop ret adr */ 321 jmp *%ecx /* indirect */ 322 SET_SIZE(longjmp) 323 324#endif /* __i386 */ 325#endif /* __lint */ 326 327/* 328 * if a() calls b() calls caller(), 329 * caller() returns return address in a(). 330 * (Note: We assume a() and b() are C routines which do the normal entry/exit 331 * sequence.) 332 */ 333 334#if defined(__lint) 335 336caddr_t 337caller(void) 338{ return (0); } 339 340#else /* __lint */ 341 342#if defined(__amd64) 343 344 ENTRY(caller) 345 movq 8(%rbp), %rax /* b()'s return pc, in a() */ 346 ret 347 SET_SIZE(caller) 348 349#elif defined(__i386) 350 351 ENTRY(caller) 352 movl 4(%ebp), %eax /* b()'s return pc, in a() */ 353 ret 354 SET_SIZE(caller) 355 356#endif /* __i386 */ 357#endif /* __lint */ 358 359/* 360 * if a() calls callee(), callee() returns the 361 * return address in a(); 362 */ 363 364#if defined(__lint) 365 366caddr_t 367callee(void) 368{ return (0); } 369 370#else /* __lint */ 371 372#if defined(__amd64) 373 374 ENTRY(callee) 375 movq (%rsp), %rax /* callee()'s return pc, in a() */ 376 ret 377 SET_SIZE(callee) 378 379#elif defined(__i386) 380 381 ENTRY(callee) 382 movl (%esp), %eax /* callee()'s return pc, in a() */ 383 ret 384 SET_SIZE(callee) 385 386#endif /* __i386 */ 387#endif /* __lint */ 388 389/* 390 * return the current frame pointer 391 */ 392 393#if defined(__lint) 394 395greg_t 396getfp(void) 397{ return (0); } 398 399#else /* __lint */ 400 401#if defined(__amd64) 402 403 ENTRY(getfp) 404 movq %rbp, %rax 405 ret 406 SET_SIZE(getfp) 407 408#elif defined(__i386) 409 410 ENTRY(getfp) 411 movl %ebp, %eax 412 ret 413 SET_SIZE(getfp) 414 415#endif /* __i386 */ 416#endif /* __lint */ 417 418/* 419 * Invalidate a single page table entry in the TLB 420 */ 421 422#if defined(__lint) 423 424/* ARGSUSED */ 425void 426mmu_tlbflush_entry(caddr_t m) 427{} 428 429#else /* __lint */ 430 431#if defined(__amd64) 432 433 ENTRY(mmu_tlbflush_entry) 434 invlpg (%rdi) 435 ret 436 SET_SIZE(mmu_tlbflush_entry) 437 438#elif defined(__i386) 439 440 ENTRY(mmu_tlbflush_entry) 441 movl 4(%esp), %eax 442 invlpg (%eax) 443 ret 444 SET_SIZE(mmu_tlbflush_entry) 445 446#endif /* __i386 */ 447#endif /* __lint */ 448 449 450/* 451 * Get/Set the value of various control registers 452 */ 453 454#if defined(__lint) 455 456ulong_t 457getcr0(void) 458{ return (0); } 459 460/* ARGSUSED */ 461void 462setcr0(ulong_t value) 463{} 464 465ulong_t 466getcr2(void) 467{ return (0); } 468 469ulong_t 470getcr3(void) 471{ return (0); } 472 473#if !defined(__xpv) 474/* ARGSUSED */ 475void 476setcr3(ulong_t val) 477{} 478 479void 480reload_cr3(void) 481{} 482#endif 483 484ulong_t 485getcr4(void) 486{ return (0); } 487 488/* ARGSUSED */ 489void 490setcr4(ulong_t val) 491{} 492 493#if defined(__amd64) 494 495ulong_t 496getcr8(void) 497{ return (0); } 498 499/* ARGSUSED */ 500void 501setcr8(ulong_t val) 502{} 503 504#endif /* __amd64 */ 505 506#else /* __lint */ 507 508#if defined(__amd64) 509 510 ENTRY(getcr0) 511 movq %cr0, %rax 512 ret 513 SET_SIZE(getcr0) 514 515 ENTRY(setcr0) 516 movq %rdi, %cr0 517 ret 518 SET_SIZE(setcr0) 519 520 ENTRY(getcr2) 521#if defined(__xpv) 522 movq %gs:CPU_VCPU_INFO, %rax 523 movq VCPU_INFO_ARCH_CR2(%rax), %rax 524#else 525 movq %cr2, %rax 526#endif 527 ret 528 SET_SIZE(getcr2) 529 530 ENTRY(getcr3) 531 movq %cr3, %rax 532 ret 533 SET_SIZE(getcr3) 534 535#if !defined(__xpv) 536 537 ENTRY(setcr3) 538 movq %rdi, %cr3 539 ret 540 SET_SIZE(setcr3) 541 542 ENTRY(reload_cr3) 543 movq %cr3, %rdi 544 movq %rdi, %cr3 545 ret 546 SET_SIZE(reload_cr3) 547 548#endif /* __xpv */ 549 550 ENTRY(getcr4) 551 movq %cr4, %rax 552 ret 553 SET_SIZE(getcr4) 554 555 ENTRY(setcr4) 556 movq %rdi, %cr4 557 ret 558 SET_SIZE(setcr4) 559 560 ENTRY(getcr8) 561 movq %cr8, %rax 562 ret 563 SET_SIZE(getcr8) 564 565 ENTRY(setcr8) 566 movq %rdi, %cr8 567 ret 568 SET_SIZE(setcr8) 569 570#elif defined(__i386) 571 572 ENTRY(getcr0) 573 movl %cr0, %eax 574 ret 575 SET_SIZE(getcr0) 576 577 ENTRY(setcr0) 578 movl 4(%esp), %eax 579 movl %eax, %cr0 580 ret 581 SET_SIZE(setcr0) 582 583 /* 584 * "lock mov %cr0" is used on processors which indicate it is 585 * supported via CPUID. Normally the 32 bit TPR is accessed via 586 * the local APIC. 587 */ 588 ENTRY(getcr8) 589 lock 590 movl %cr0, %eax 591 ret 592 SET_SIZE(getcr8) 593 594 ENTRY(setcr8) 595 movl 4(%esp), %eax 596 lock 597 movl %eax, %cr0 598 ret 599 SET_SIZE(setcr8) 600 601 ENTRY(getcr2) 602#if defined(__xpv) 603 movl %gs:CPU_VCPU_INFO, %eax 604 movl VCPU_INFO_ARCH_CR2(%eax), %eax 605#else 606 movl %cr2, %eax 607#endif 608 ret 609 SET_SIZE(getcr2) 610 611 ENTRY(getcr3) 612 movl %cr3, %eax 613 ret 614 SET_SIZE(getcr3) 615 616#if !defined(__xpv) 617 618 ENTRY(setcr3) 619 movl 4(%esp), %eax 620 movl %eax, %cr3 621 ret 622 SET_SIZE(setcr3) 623 624 ENTRY(reload_cr3) 625 movl %cr3, %eax 626 movl %eax, %cr3 627 ret 628 SET_SIZE(reload_cr3) 629 630#endif /* __xpv */ 631 632 ENTRY(getcr4) 633 movl %cr4, %eax 634 ret 635 SET_SIZE(getcr4) 636 637 ENTRY(setcr4) 638 movl 4(%esp), %eax 639 movl %eax, %cr4 640 ret 641 SET_SIZE(setcr4) 642 643#endif /* __i386 */ 644#endif /* __lint */ 645 646#if defined(__lint) 647 648/*ARGSUSED*/ 649uint32_t 650__cpuid_insn(struct cpuid_regs *regs) 651{ return (0); } 652 653#else /* __lint */ 654 655#if defined(__amd64) 656 657 ENTRY(__cpuid_insn) 658 movq %rbx, %r8 659 movq %rcx, %r9 660 movq %rdx, %r11 661 movl (%rdi), %eax /* %eax = regs->cp_eax */ 662 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */ 663 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */ 664 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */ 665 cpuid 666 movl %eax, (%rdi) /* regs->cp_eax = %eax */ 667 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */ 668 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */ 669 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */ 670 movq %r8, %rbx 671 movq %r9, %rcx 672 movq %r11, %rdx 673 ret 674 SET_SIZE(__cpuid_insn) 675 676#elif defined(__i386) 677 678 ENTRY(__cpuid_insn) 679 pushl %ebp 680 movl 0x8(%esp), %ebp /* %ebp = regs */ 681 pushl %ebx 682 pushl %ecx 683 pushl %edx 684 movl (%ebp), %eax /* %eax = regs->cp_eax */ 685 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */ 686 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */ 687 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */ 688 cpuid 689 movl %eax, (%ebp) /* regs->cp_eax = %eax */ 690 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */ 691 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */ 692 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */ 693 popl %edx 694 popl %ecx 695 popl %ebx 696 popl %ebp 697 ret 698 SET_SIZE(__cpuid_insn) 699 700#endif /* __i386 */ 701#endif /* __lint */ 702 703#if defined(__lint) 704 705/*ARGSUSED*/ 706void 707i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints) 708{} 709 710#else /* __lint */ 711 712#if defined(__amd64) 713 714 ENTRY_NP(i86_monitor) 715 pushq %rbp 716 movq %rsp, %rbp 717 movq %rdi, %rax /* addr */ 718 movq %rsi, %rcx /* extensions */ 719 /* rdx contains input arg3: hints */ 720 clflush (%rax) 721 .byte 0x0f, 0x01, 0xc8 /* monitor */ 722 leave 723 ret 724 SET_SIZE(i86_monitor) 725 726#elif defined(__i386) 727 728ENTRY_NP(i86_monitor) 729 pushl %ebp 730 movl %esp, %ebp 731 movl 0x8(%ebp),%eax /* addr */ 732 movl 0xc(%ebp),%ecx /* extensions */ 733 movl 0x10(%ebp),%edx /* hints */ 734 clflush (%eax) 735 .byte 0x0f, 0x01, 0xc8 /* monitor */ 736 leave 737 ret 738 SET_SIZE(i86_monitor) 739 740#endif /* __i386 */ 741#endif /* __lint */ 742 743#if defined(__lint) 744 745/*ARGSUSED*/ 746void 747i86_mwait(uint32_t data, uint32_t extensions) 748{} 749 750#else /* __lint */ 751 752#if defined(__amd64) 753 754 ENTRY_NP(i86_mwait) 755 pushq %rbp 756 movq %rsp, %rbp 757 movq %rdi, %rax /* data */ 758 movq %rsi, %rcx /* extensions */ 759 .byte 0x0f, 0x01, 0xc9 /* mwait */ 760 leave 761 ret 762 SET_SIZE(i86_mwait) 763 764#elif defined(__i386) 765 766 ENTRY_NP(i86_mwait) 767 pushl %ebp 768 movl %esp, %ebp 769 movl 0x8(%ebp),%eax /* data */ 770 movl 0xc(%ebp),%ecx /* extensions */ 771 .byte 0x0f, 0x01, 0xc9 /* mwait */ 772 leave 773 ret 774 SET_SIZE(i86_mwait) 775 776#endif /* __i386 */ 777#endif /* __lint */ 778 779#if defined(__xpv) 780 /* 781 * Defined in C 782 */ 783#else 784 785#if defined(__lint) 786 787hrtime_t 788tsc_read(void) 789{ 790 return (0); 791} 792 793#else /* __lint */ 794 795#if defined(__amd64) 796 797 ENTRY_NP(tsc_read) 798 movq %rbx, %r11 799 movl $0, %eax 800 cpuid 801 rdtsc 802 movq %r11, %rbx 803 shlq $32, %rdx 804 orq %rdx, %rax 805 ret 806 .globl _tsc_mfence_start 807_tsc_mfence_start: 808 mfence 809 rdtsc 810 shlq $32, %rdx 811 orq %rdx, %rax 812 ret 813 .globl _tsc_mfence_end 814_tsc_mfence_end: 815 .globl _tscp_start 816_tscp_start: 817 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 818 shlq $32, %rdx 819 orq %rdx, %rax 820 ret 821 .globl _tscp_end 822_tscp_end: 823 .globl _no_rdtsc_start 824_no_rdtsc_start: 825 xorl %edx, %edx 826 xorl %eax, %eax 827 ret 828 .globl _no_rdtsc_end 829_no_rdtsc_end: 830 .globl _tsc_lfence_start 831_tsc_lfence_start: 832 lfence 833 rdtsc 834 shlq $32, %rdx 835 orq %rdx, %rax 836 ret 837 .globl _tsc_lfence_end 838_tsc_lfence_end: 839 SET_SIZE(tsc_read) 840 841#else /* __i386 */ 842 843 ENTRY_NP(tsc_read) 844 pushl %ebx 845 movl $0, %eax 846 cpuid 847 rdtsc 848 popl %ebx 849 ret 850 .globl _tsc_mfence_start 851_tsc_mfence_start: 852 mfence 853 rdtsc 854 ret 855 .globl _tsc_mfence_end 856_tsc_mfence_end: 857 .globl _tscp_start 858_tscp_start: 859 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 860 ret 861 .globl _tscp_end 862_tscp_end: 863 .globl _no_rdtsc_start 864_no_rdtsc_start: 865 xorl %edx, %edx 866 xorl %eax, %eax 867 ret 868 .globl _no_rdtsc_end 869_no_rdtsc_end: 870 .globl _tsc_lfence_start 871_tsc_lfence_start: 872 lfence 873 rdtsc 874 ret 875 .globl _tsc_lfence_end 876_tsc_lfence_end: 877 SET_SIZE(tsc_read) 878 879#endif /* __i386 */ 880 881#endif /* __lint */ 882 883 884#endif /* __xpv */ 885 886#ifdef __lint 887/* 888 * Do not use this function for obtaining clock tick. This 889 * is called by callers who do not need to have a guarenteed 890 * correct tick value. The proper routine to use is tsc_read(). 891 */ 892hrtime_t 893randtick(void) 894{ 895 return (0); 896} 897#else 898#if defined(__amd64) 899 ENTRY_NP(randtick) 900 rdtsc 901 shlq $32, %rdx 902 orq %rdx, %rax 903 ret 904 SET_SIZE(randtick) 905#else 906 ENTRY_NP(randtick) 907 rdtsc 908 ret 909 SET_SIZE(randtick) 910#endif /* __i386 */ 911#endif /* __lint */ 912/* 913 * Insert entryp after predp in a doubly linked list. 914 */ 915 916#if defined(__lint) 917 918/*ARGSUSED*/ 919void 920_insque(caddr_t entryp, caddr_t predp) 921{} 922 923#else /* __lint */ 924 925#if defined(__amd64) 926 927 ENTRY(_insque) 928 movq (%rsi), %rax /* predp->forw */ 929 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */ 930 movq %rax, (%rdi) /* entryp->forw = predp->forw */ 931 movq %rdi, (%rsi) /* predp->forw = entryp */ 932 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */ 933 ret 934 SET_SIZE(_insque) 935 936#elif defined(__i386) 937 938 ENTRY(_insque) 939 movl 8(%esp), %edx 940 movl 4(%esp), %ecx 941 movl (%edx), %eax /* predp->forw */ 942 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */ 943 movl %eax, (%ecx) /* entryp->forw = predp->forw */ 944 movl %ecx, (%edx) /* predp->forw = entryp */ 945 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */ 946 ret 947 SET_SIZE(_insque) 948 949#endif /* __i386 */ 950#endif /* __lint */ 951 952/* 953 * Remove entryp from a doubly linked list 954 */ 955 956#if defined(__lint) 957 958/*ARGSUSED*/ 959void 960_remque(caddr_t entryp) 961{} 962 963#else /* __lint */ 964 965#if defined(__amd64) 966 967 ENTRY(_remque) 968 movq (%rdi), %rax /* entry->forw */ 969 movq CPTRSIZE(%rdi), %rdx /* entry->back */ 970 movq %rax, (%rdx) /* entry->back->forw = entry->forw */ 971 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */ 972 ret 973 SET_SIZE(_remque) 974 975#elif defined(__i386) 976 977 ENTRY(_remque) 978 movl 4(%esp), %ecx 979 movl (%ecx), %eax /* entry->forw */ 980 movl CPTRSIZE(%ecx), %edx /* entry->back */ 981 movl %eax, (%edx) /* entry->back->forw = entry->forw */ 982 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */ 983 ret 984 SET_SIZE(_remque) 985 986#endif /* __i386 */ 987#endif /* __lint */ 988 989/* 990 * Returns the number of 991 * non-NULL bytes in string argument. 992 */ 993 994#if defined(__lint) 995 996/* ARGSUSED */ 997size_t 998strlen(const char *str) 999{ return (0); } 1000 1001#else /* __lint */ 1002 1003#if defined(__amd64) 1004 1005/* 1006 * This is close to a simple transliteration of a C version of this 1007 * routine. We should either just -make- this be a C version, or 1008 * justify having it in assembler by making it significantly faster. 1009 * 1010 * size_t 1011 * strlen(const char *s) 1012 * { 1013 * const char *s0; 1014 * #if defined(DEBUG) 1015 * if ((uintptr_t)s < KERNELBASE) 1016 * panic(.str_panic_msg); 1017 * #endif 1018 * for (s0 = s; *s; s++) 1019 * ; 1020 * return (s - s0); 1021 * } 1022 */ 1023 1024 ENTRY(strlen) 1025#ifdef DEBUG 1026 movq postbootkernelbase(%rip), %rax 1027 cmpq %rax, %rdi 1028 jae str_valid 1029 pushq %rbp 1030 movq %rsp, %rbp 1031 leaq .str_panic_msg(%rip), %rdi 1032 xorl %eax, %eax 1033 call panic 1034#endif /* DEBUG */ 1035str_valid: 1036 cmpb $0, (%rdi) 1037 movq %rdi, %rax 1038 je .null_found 1039 .align 4 1040.strlen_loop: 1041 incq %rdi 1042 cmpb $0, (%rdi) 1043 jne .strlen_loop 1044.null_found: 1045 subq %rax, %rdi 1046 movq %rdi, %rax 1047 ret 1048 SET_SIZE(strlen) 1049 1050#elif defined(__i386) 1051 1052 ENTRY(strlen) 1053#ifdef DEBUG 1054 movl postbootkernelbase, %eax 1055 cmpl %eax, 4(%esp) 1056 jae str_valid 1057 pushl %ebp 1058 movl %esp, %ebp 1059 pushl $.str_panic_msg 1060 call panic 1061#endif /* DEBUG */ 1062 1063str_valid: 1064 movl 4(%esp), %eax /* %eax = string address */ 1065 testl $3, %eax /* if %eax not word aligned */ 1066 jnz .not_word_aligned /* goto .not_word_aligned */ 1067 .align 4 1068.word_aligned: 1069 movl (%eax), %edx /* move 1 word from (%eax) to %edx */ 1070 movl $0x7f7f7f7f, %ecx 1071 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */ 1072 addl $4, %eax /* next word */ 1073 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */ 1074 orl %edx, %ecx /* %ecx |= %edx */ 1075 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */ 1076 cmpl $0x80808080, %ecx /* if no null byte in this word */ 1077 je .word_aligned /* goto .word_aligned */ 1078 subl $4, %eax /* post-incremented */ 1079.not_word_aligned: 1080 cmpb $0, (%eax) /* if a byte in (%eax) is null */ 1081 je .null_found /* goto .null_found */ 1082 incl %eax /* next byte */ 1083 testl $3, %eax /* if %eax not word aligned */ 1084 jnz .not_word_aligned /* goto .not_word_aligned */ 1085 jmp .word_aligned /* goto .word_aligned */ 1086 .align 4 1087.null_found: 1088 subl 4(%esp), %eax /* %eax -= string address */ 1089 ret 1090 SET_SIZE(strlen) 1091 1092#endif /* __i386 */ 1093 1094#ifdef DEBUG 1095 .text 1096.str_panic_msg: 1097 .string "strlen: argument below kernelbase" 1098#endif /* DEBUG */ 1099 1100#endif /* __lint */ 1101 1102 /* 1103 * Berkeley 4.3 introduced symbolically named interrupt levels 1104 * as a way deal with priority in a machine independent fashion. 1105 * Numbered priorities are machine specific, and should be 1106 * discouraged where possible. 1107 * 1108 * Note, for the machine specific priorities there are 1109 * examples listed for devices that use a particular priority. 1110 * It should not be construed that all devices of that 1111 * type should be at that priority. It is currently were 1112 * the current devices fit into the priority scheme based 1113 * upon time criticalness. 1114 * 1115 * The underlying assumption of these assignments is that 1116 * IPL 10 is the highest level from which a device 1117 * routine can call wakeup. Devices that interrupt from higher 1118 * levels are restricted in what they can do. If they need 1119 * kernels services they should schedule a routine at a lower 1120 * level (via software interrupt) to do the required 1121 * processing. 1122 * 1123 * Examples of this higher usage: 1124 * Level Usage 1125 * 14 Profiling clock (and PROM uart polling clock) 1126 * 12 Serial ports 1127 * 1128 * The serial ports request lower level processing on level 6. 1129 * 1130 * Also, almost all splN routines (where N is a number or a 1131 * mnemonic) will do a RAISE(), on the assumption that they are 1132 * never used to lower our priority. 1133 * The exceptions are: 1134 * spl8() Because you can't be above 15 to begin with! 1135 * splzs() Because this is used at boot time to lower our 1136 * priority, to allow the PROM to poll the uart. 1137 * spl0() Used to lower priority to 0. 1138 */ 1139 1140#if defined(__lint) 1141 1142int spl0(void) { return (0); } 1143int spl6(void) { return (0); } 1144int spl7(void) { return (0); } 1145int spl8(void) { return (0); } 1146int splhigh(void) { return (0); } 1147int splhi(void) { return (0); } 1148int splzs(void) { return (0); } 1149 1150/* ARGSUSED */ 1151void 1152splx(int level) 1153{} 1154 1155#else /* __lint */ 1156 1157#if defined(__amd64) 1158 1159#define SETPRI(level) \ 1160 movl $/**/level, %edi; /* new priority */ \ 1161 jmp do_splx /* redirect to do_splx */ 1162 1163#define RAISE(level) \ 1164 movl $/**/level, %edi; /* new priority */ \ 1165 jmp splr /* redirect to splr */ 1166 1167#elif defined(__i386) 1168 1169#define SETPRI(level) \ 1170 pushl $/**/level; /* new priority */ \ 1171 call do_splx; /* invoke common splx code */ \ 1172 addl $4, %esp; /* unstack arg */ \ 1173 ret 1174 1175#define RAISE(level) \ 1176 pushl $/**/level; /* new priority */ \ 1177 call splr; /* invoke common splr code */ \ 1178 addl $4, %esp; /* unstack args */ \ 1179 ret 1180 1181#endif /* __i386 */ 1182 1183 /* locks out all interrupts, including memory errors */ 1184 ENTRY(spl8) 1185 SETPRI(15) 1186 SET_SIZE(spl8) 1187 1188 /* just below the level that profiling runs */ 1189 ENTRY(spl7) 1190 RAISE(13) 1191 SET_SIZE(spl7) 1192 1193 /* sun specific - highest priority onboard serial i/o asy ports */ 1194 ENTRY(splzs) 1195 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */ 1196 SET_SIZE(splzs) 1197 1198 ENTRY(splhi) 1199 ALTENTRY(splhigh) 1200 ALTENTRY(spl6) 1201 ALTENTRY(i_ddi_splhigh) 1202 1203 RAISE(DISP_LEVEL) 1204 1205 SET_SIZE(i_ddi_splhigh) 1206 SET_SIZE(spl6) 1207 SET_SIZE(splhigh) 1208 SET_SIZE(splhi) 1209 1210 /* allow all interrupts */ 1211 ENTRY(spl0) 1212 SETPRI(0) 1213 SET_SIZE(spl0) 1214 1215 1216 /* splx implementation */ 1217 ENTRY(splx) 1218 jmp do_splx /* redirect to common splx code */ 1219 SET_SIZE(splx) 1220 1221#endif /* __lint */ 1222 1223#if defined(__i386) 1224 1225/* 1226 * Read and write the %gs register 1227 */ 1228 1229#if defined(__lint) 1230 1231/*ARGSUSED*/ 1232uint16_t 1233getgs(void) 1234{ return (0); } 1235 1236/*ARGSUSED*/ 1237void 1238setgs(uint16_t sel) 1239{} 1240 1241#else /* __lint */ 1242 1243 ENTRY(getgs) 1244 clr %eax 1245 movw %gs, %ax 1246 ret 1247 SET_SIZE(getgs) 1248 1249 ENTRY(setgs) 1250 movw 4(%esp), %gs 1251 ret 1252 SET_SIZE(setgs) 1253 1254#endif /* __lint */ 1255#endif /* __i386 */ 1256 1257#if defined(__lint) 1258 1259void 1260pc_reset(void) 1261{} 1262 1263void 1264efi_reset(void) 1265{} 1266 1267#else /* __lint */ 1268 1269 ENTRY(wait_500ms) 1270#if defined(__amd64) 1271 pushq %rbx 1272#elif defined(__i386) 1273 push %ebx 1274#endif 1275 movl $50000, %ebx 12761: 1277 call tenmicrosec 1278 decl %ebx 1279 jnz 1b 1280#if defined(__amd64) 1281 popq %rbx 1282#elif defined(__i386) 1283 pop %ebx 1284#endif 1285 ret 1286 SET_SIZE(wait_500ms) 1287 1288#define RESET_METHOD_KBC 1 1289#define RESET_METHOD_PORT92 2 1290#define RESET_METHOD_PCI 4 1291 1292 DGDEF3(pc_reset_methods, 4, 8) 1293 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI; 1294 1295 ENTRY(pc_reset) 1296 1297#if defined(__i386) 1298 testl $RESET_METHOD_KBC, pc_reset_methods 1299#elif defined(__amd64) 1300 testl $RESET_METHOD_KBC, pc_reset_methods(%rip) 1301#endif 1302 jz 1f 1303 1304 / 1305 / Try the classic keyboard controller-triggered reset. 1306 / 1307 movw $0x64, %dx 1308 movb $0xfe, %al 1309 outb (%dx) 1310 1311 / Wait up to 500 milliseconds here for the keyboard controller 1312 / to pull the reset line. On some systems where the keyboard 1313 / controller is slow to pull the reset line, the next reset method 1314 / may be executed (which may be bad if those systems hang when the 1315 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92), 1316 / and Ferrari 4000 (doesn't like the cf9 reset method)) 1317 1318 call wait_500ms 1319 13201: 1321#if defined(__i386) 1322 testl $RESET_METHOD_PORT92, pc_reset_methods 1323#elif defined(__amd64) 1324 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip) 1325#endif 1326 jz 3f 1327 1328 / 1329 / Try port 0x92 fast reset 1330 / 1331 movw $0x92, %dx 1332 inb (%dx) 1333 cmpb $0xff, %al / If port's not there, we should get back 0xFF 1334 je 1f 1335 testb $1, %al / If bit 0 1336 jz 2f / is clear, jump to perform the reset 1337 andb $0xfe, %al / otherwise, 1338 outb (%dx) / clear bit 0 first, then 13392: 1340 orb $1, %al / Set bit 0 1341 outb (%dx) / and reset the system 13421: 1343 1344 call wait_500ms 1345 13463: 1347#if defined(__i386) 1348 testl $RESET_METHOD_PCI, pc_reset_methods 1349#elif defined(__amd64) 1350 testl $RESET_METHOD_PCI, pc_reset_methods(%rip) 1351#endif 1352 jz 4f 1353 1354 / Try the PCI (soft) reset vector (should work on all modern systems, 1355 / but has been shown to cause problems on 450NX systems, and some newer 1356 / systems (e.g. ATI IXP400-equipped systems)) 1357 / When resetting via this method, 2 writes are required. The first 1358 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with 1359 / power cycle). 1360 / The reset occurs on the second write, during bit 2's transition from 1361 / 0->1. 1362 movw $0xcf9, %dx 1363 movb $0x2, %al / Reset mode = hard, no power cycle 1364 outb (%dx) 1365 movb $0x6, %al 1366 outb (%dx) 1367 1368 call wait_500ms 1369 13704: 1371 / 1372 / port 0xcf9 failed also. Last-ditch effort is to 1373 / triple-fault the CPU. 1374 / Also, use triple fault for EFI firmware 1375 / 1376 ENTRY(efi_reset) 1377#if defined(__amd64) 1378 pushq $0x0 1379 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1380 lidt (%rsp) 1381#elif defined(__i386) 1382 pushl $0x0 1383 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1384 lidt (%esp) 1385#endif 1386 int $0x0 / Trigger interrupt, generate triple-fault 1387 1388 cli 1389 hlt / Wait forever 1390 /*NOTREACHED*/ 1391 SET_SIZE(efi_reset) 1392 SET_SIZE(pc_reset) 1393 1394#endif /* __lint */ 1395 1396/* 1397 * C callable in and out routines 1398 */ 1399 1400#if defined(__lint) 1401 1402/* ARGSUSED */ 1403void 1404outl(int port_address, uint32_t val) 1405{} 1406 1407#else /* __lint */ 1408 1409#if defined(__amd64) 1410 1411 ENTRY(outl) 1412 movw %di, %dx 1413 movl %esi, %eax 1414 outl (%dx) 1415 ret 1416 SET_SIZE(outl) 1417 1418#elif defined(__i386) 1419 1420 .set PORT, 4 1421 .set VAL, 8 1422 1423 ENTRY(outl) 1424 movw PORT(%esp), %dx 1425 movl VAL(%esp), %eax 1426 outl (%dx) 1427 ret 1428 SET_SIZE(outl) 1429 1430#endif /* __i386 */ 1431#endif /* __lint */ 1432 1433#if defined(__lint) 1434 1435/* ARGSUSED */ 1436void 1437outw(int port_address, uint16_t val) 1438{} 1439 1440#else /* __lint */ 1441 1442#if defined(__amd64) 1443 1444 ENTRY(outw) 1445 movw %di, %dx 1446 movw %si, %ax 1447 D16 outl (%dx) /* XX64 why not outw? */ 1448 ret 1449 SET_SIZE(outw) 1450 1451#elif defined(__i386) 1452 1453 ENTRY(outw) 1454 movw PORT(%esp), %dx 1455 movw VAL(%esp), %ax 1456 D16 outl (%dx) 1457 ret 1458 SET_SIZE(outw) 1459 1460#endif /* __i386 */ 1461#endif /* __lint */ 1462 1463#if defined(__lint) 1464 1465/* ARGSUSED */ 1466void 1467outb(int port_address, uint8_t val) 1468{} 1469 1470#else /* __lint */ 1471 1472#if defined(__amd64) 1473 1474 ENTRY(outb) 1475 movw %di, %dx 1476 movb %sil, %al 1477 outb (%dx) 1478 ret 1479 SET_SIZE(outb) 1480 1481#elif defined(__i386) 1482 1483 ENTRY(outb) 1484 movw PORT(%esp), %dx 1485 movb VAL(%esp), %al 1486 outb (%dx) 1487 ret 1488 SET_SIZE(outb) 1489 1490#endif /* __i386 */ 1491#endif /* __lint */ 1492 1493#if defined(__lint) 1494 1495/* ARGSUSED */ 1496uint32_t 1497inl(int port_address) 1498{ return (0); } 1499 1500#else /* __lint */ 1501 1502#if defined(__amd64) 1503 1504 ENTRY(inl) 1505 xorl %eax, %eax 1506 movw %di, %dx 1507 inl (%dx) 1508 ret 1509 SET_SIZE(inl) 1510 1511#elif defined(__i386) 1512 1513 ENTRY(inl) 1514 movw PORT(%esp), %dx 1515 inl (%dx) 1516 ret 1517 SET_SIZE(inl) 1518 1519#endif /* __i386 */ 1520#endif /* __lint */ 1521 1522#if defined(__lint) 1523 1524/* ARGSUSED */ 1525uint16_t 1526inw(int port_address) 1527{ return (0); } 1528 1529#else /* __lint */ 1530 1531#if defined(__amd64) 1532 1533 ENTRY(inw) 1534 xorl %eax, %eax 1535 movw %di, %dx 1536 D16 inl (%dx) 1537 ret 1538 SET_SIZE(inw) 1539 1540#elif defined(__i386) 1541 1542 ENTRY(inw) 1543 subl %eax, %eax 1544 movw PORT(%esp), %dx 1545 D16 inl (%dx) 1546 ret 1547 SET_SIZE(inw) 1548 1549#endif /* __i386 */ 1550#endif /* __lint */ 1551 1552 1553#if defined(__lint) 1554 1555/* ARGSUSED */ 1556uint8_t 1557inb(int port_address) 1558{ return (0); } 1559 1560#else /* __lint */ 1561 1562#if defined(__amd64) 1563 1564 ENTRY(inb) 1565 xorl %eax, %eax 1566 movw %di, %dx 1567 inb (%dx) 1568 ret 1569 SET_SIZE(inb) 1570 1571#elif defined(__i386) 1572 1573 ENTRY(inb) 1574 subl %eax, %eax 1575 movw PORT(%esp), %dx 1576 inb (%dx) 1577 ret 1578 SET_SIZE(inb) 1579 1580#endif /* __i386 */ 1581#endif /* __lint */ 1582 1583 1584#if defined(__lint) 1585 1586/* ARGSUSED */ 1587void 1588repoutsw(int port, uint16_t *addr, int cnt) 1589{} 1590 1591#else /* __lint */ 1592 1593#if defined(__amd64) 1594 1595 ENTRY(repoutsw) 1596 movl %edx, %ecx 1597 movw %di, %dx 1598 rep 1599 D16 outsl 1600 ret 1601 SET_SIZE(repoutsw) 1602 1603#elif defined(__i386) 1604 1605 /* 1606 * The arguments and saved registers are on the stack in the 1607 * following order: 1608 * | cnt | +16 1609 * | *addr | +12 1610 * | port | +8 1611 * | eip | +4 1612 * | esi | <-- %esp 1613 * If additional values are pushed onto the stack, make sure 1614 * to adjust the following constants accordingly. 1615 */ 1616 .set PORT, 8 1617 .set ADDR, 12 1618 .set COUNT, 16 1619 1620 ENTRY(repoutsw) 1621 pushl %esi 1622 movl PORT(%esp), %edx 1623 movl ADDR(%esp), %esi 1624 movl COUNT(%esp), %ecx 1625 rep 1626 D16 outsl 1627 popl %esi 1628 ret 1629 SET_SIZE(repoutsw) 1630 1631#endif /* __i386 */ 1632#endif /* __lint */ 1633 1634 1635#if defined(__lint) 1636 1637/* ARGSUSED */ 1638void 1639repinsw(int port_addr, uint16_t *addr, int cnt) 1640{} 1641 1642#else /* __lint */ 1643 1644#if defined(__amd64) 1645 1646 ENTRY(repinsw) 1647 movl %edx, %ecx 1648 movw %di, %dx 1649 rep 1650 D16 insl 1651 ret 1652 SET_SIZE(repinsw) 1653 1654#elif defined(__i386) 1655 1656 ENTRY(repinsw) 1657 pushl %edi 1658 movl PORT(%esp), %edx 1659 movl ADDR(%esp), %edi 1660 movl COUNT(%esp), %ecx 1661 rep 1662 D16 insl 1663 popl %edi 1664 ret 1665 SET_SIZE(repinsw) 1666 1667#endif /* __i386 */ 1668#endif /* __lint */ 1669 1670 1671#if defined(__lint) 1672 1673/* ARGSUSED */ 1674void 1675repinsb(int port, uint8_t *addr, int count) 1676{} 1677 1678#else /* __lint */ 1679 1680#if defined(__amd64) 1681 1682 ENTRY(repinsb) 1683 movl %edx, %ecx 1684 movw %di, %dx 1685 movq %rsi, %rdi 1686 rep 1687 insb 1688 ret 1689 SET_SIZE(repinsb) 1690 1691#elif defined(__i386) 1692 1693 /* 1694 * The arguments and saved registers are on the stack in the 1695 * following order: 1696 * | cnt | +16 1697 * | *addr | +12 1698 * | port | +8 1699 * | eip | +4 1700 * | esi | <-- %esp 1701 * If additional values are pushed onto the stack, make sure 1702 * to adjust the following constants accordingly. 1703 */ 1704 .set IO_PORT, 8 1705 .set IO_ADDR, 12 1706 .set IO_COUNT, 16 1707 1708 ENTRY(repinsb) 1709 pushl %edi 1710 movl IO_ADDR(%esp), %edi 1711 movl IO_COUNT(%esp), %ecx 1712 movl IO_PORT(%esp), %edx 1713 rep 1714 insb 1715 popl %edi 1716 ret 1717 SET_SIZE(repinsb) 1718 1719#endif /* __i386 */ 1720#endif /* __lint */ 1721 1722 1723/* 1724 * Input a stream of 32-bit words. 1725 * NOTE: count is a DWORD count. 1726 */ 1727#if defined(__lint) 1728 1729/* ARGSUSED */ 1730void 1731repinsd(int port, uint32_t *addr, int count) 1732{} 1733 1734#else /* __lint */ 1735 1736#if defined(__amd64) 1737 1738 ENTRY(repinsd) 1739 movl %edx, %ecx 1740 movw %di, %dx 1741 movq %rsi, %rdi 1742 rep 1743 insl 1744 ret 1745 SET_SIZE(repinsd) 1746 1747#elif defined(__i386) 1748 1749 ENTRY(repinsd) 1750 pushl %edi 1751 movl IO_ADDR(%esp), %edi 1752 movl IO_COUNT(%esp), %ecx 1753 movl IO_PORT(%esp), %edx 1754 rep 1755 insl 1756 popl %edi 1757 ret 1758 SET_SIZE(repinsd) 1759 1760#endif /* __i386 */ 1761#endif /* __lint */ 1762 1763/* 1764 * Output a stream of bytes 1765 * NOTE: count is a byte count 1766 */ 1767#if defined(__lint) 1768 1769/* ARGSUSED */ 1770void 1771repoutsb(int port, uint8_t *addr, int count) 1772{} 1773 1774#else /* __lint */ 1775 1776#if defined(__amd64) 1777 1778 ENTRY(repoutsb) 1779 movl %edx, %ecx 1780 movw %di, %dx 1781 rep 1782 outsb 1783 ret 1784 SET_SIZE(repoutsb) 1785 1786#elif defined(__i386) 1787 1788 ENTRY(repoutsb) 1789 pushl %esi 1790 movl IO_ADDR(%esp), %esi 1791 movl IO_COUNT(%esp), %ecx 1792 movl IO_PORT(%esp), %edx 1793 rep 1794 outsb 1795 popl %esi 1796 ret 1797 SET_SIZE(repoutsb) 1798 1799#endif /* __i386 */ 1800#endif /* __lint */ 1801 1802/* 1803 * Output a stream of 32-bit words 1804 * NOTE: count is a DWORD count 1805 */ 1806#if defined(__lint) 1807 1808/* ARGSUSED */ 1809void 1810repoutsd(int port, uint32_t *addr, int count) 1811{} 1812 1813#else /* __lint */ 1814 1815#if defined(__amd64) 1816 1817 ENTRY(repoutsd) 1818 movl %edx, %ecx 1819 movw %di, %dx 1820 rep 1821 outsl 1822 ret 1823 SET_SIZE(repoutsd) 1824 1825#elif defined(__i386) 1826 1827 ENTRY(repoutsd) 1828 pushl %esi 1829 movl IO_ADDR(%esp), %esi 1830 movl IO_COUNT(%esp), %ecx 1831 movl IO_PORT(%esp), %edx 1832 rep 1833 outsl 1834 popl %esi 1835 ret 1836 SET_SIZE(repoutsd) 1837 1838#endif /* __i386 */ 1839#endif /* __lint */ 1840 1841/* 1842 * void int3(void) 1843 * void int18(void) 1844 * void int20(void) 1845 * void int_cmci(void) 1846 */ 1847 1848#if defined(__lint) 1849 1850void 1851int3(void) 1852{} 1853 1854void 1855int18(void) 1856{} 1857 1858void 1859int20(void) 1860{} 1861 1862void 1863int_cmci(void) 1864{} 1865 1866#else /* __lint */ 1867 1868 ENTRY(int3) 1869 int $T_BPTFLT 1870 ret 1871 SET_SIZE(int3) 1872 1873 ENTRY(int18) 1874 int $T_MCE 1875 ret 1876 SET_SIZE(int18) 1877 1878 ENTRY(int20) 1879 movl boothowto, %eax 1880 andl $RB_DEBUG, %eax 1881 jz 1f 1882 1883 int $T_DBGENTR 18841: 1885 rep; ret /* use 2 byte return instruction when branch target */ 1886 /* AMD Software Optimization Guide - Section 6.2 */ 1887 SET_SIZE(int20) 1888 1889 ENTRY(int_cmci) 1890 int $T_ENOEXTFLT 1891 ret 1892 SET_SIZE(int_cmci) 1893 1894#endif /* __lint */ 1895 1896#if defined(__lint) 1897 1898/* ARGSUSED */ 1899int 1900scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask) 1901{ return (0); } 1902 1903#else /* __lint */ 1904 1905#if defined(__amd64) 1906 1907 ENTRY(scanc) 1908 /* rdi == size */ 1909 /* rsi == cp */ 1910 /* rdx == table */ 1911 /* rcx == mask */ 1912 addq %rsi, %rdi /* end = &cp[size] */ 1913.scanloop: 1914 cmpq %rdi, %rsi /* while (cp < end */ 1915 jnb .scandone 1916 movzbq (%rsi), %r8 /* %r8 = *cp */ 1917 incq %rsi /* cp++ */ 1918 testb %cl, (%r8, %rdx) 1919 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1920 decq %rsi /* (fix post-increment) */ 1921.scandone: 1922 movl %edi, %eax 1923 subl %esi, %eax /* return (end - cp) */ 1924 ret 1925 SET_SIZE(scanc) 1926 1927#elif defined(__i386) 1928 1929 ENTRY(scanc) 1930 pushl %edi 1931 pushl %esi 1932 movb 24(%esp), %cl /* mask = %cl */ 1933 movl 16(%esp), %esi /* cp = %esi */ 1934 movl 20(%esp), %edx /* table = %edx */ 1935 movl %esi, %edi 1936 addl 12(%esp), %edi /* end = &cp[size]; */ 1937.scanloop: 1938 cmpl %edi, %esi /* while (cp < end */ 1939 jnb .scandone 1940 movzbl (%esi), %eax /* %al = *cp */ 1941 incl %esi /* cp++ */ 1942 movb (%edx, %eax), %al /* %al = table[*cp] */ 1943 testb %al, %cl 1944 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1945 dec %esi /* post-incremented */ 1946.scandone: 1947 movl %edi, %eax 1948 subl %esi, %eax /* return (end - cp) */ 1949 popl %esi 1950 popl %edi 1951 ret 1952 SET_SIZE(scanc) 1953 1954#endif /* __i386 */ 1955#endif /* __lint */ 1956 1957/* 1958 * Replacement functions for ones that are normally inlined. 1959 * In addition to the copy in i86.il, they are defined here just in case. 1960 */ 1961 1962#if defined(__lint) 1963 1964ulong_t 1965intr_clear(void) 1966{ return (0); } 1967 1968ulong_t 1969clear_int_flag(void) 1970{ return (0); } 1971 1972#else /* __lint */ 1973 1974#if defined(__amd64) 1975 1976 ENTRY(intr_clear) 1977 ENTRY(clear_int_flag) 1978 pushfq 1979 popq %rax 1980#if defined(__xpv) 1981 leaq xpv_panicking, %rdi 1982 movl (%rdi), %edi 1983 cmpl $0, %edi 1984 jne 2f 1985 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 1986 /* 1987 * Synthesize the PS_IE bit from the event mask bit 1988 */ 1989 andq $_BITNOT(PS_IE), %rax 1990 testb $1, %dl 1991 jnz 1f 1992 orq $PS_IE, %rax 19931: 1994 ret 19952: 1996#endif 1997 CLI(%rdi) 1998 ret 1999 SET_SIZE(clear_int_flag) 2000 SET_SIZE(intr_clear) 2001 2002#elif defined(__i386) 2003 2004 ENTRY(intr_clear) 2005 ENTRY(clear_int_flag) 2006 pushfl 2007 popl %eax 2008#if defined(__xpv) 2009 leal xpv_panicking, %edx 2010 movl (%edx), %edx 2011 cmpl $0, %edx 2012 jne 2f 2013 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2014 /* 2015 * Synthesize the PS_IE bit from the event mask bit 2016 */ 2017 andl $_BITNOT(PS_IE), %eax 2018 testb $1, %cl 2019 jnz 1f 2020 orl $PS_IE, %eax 20211: 2022 ret 20232: 2024#endif 2025 CLI(%edx) 2026 ret 2027 SET_SIZE(clear_int_flag) 2028 SET_SIZE(intr_clear) 2029 2030#endif /* __i386 */ 2031#endif /* __lint */ 2032 2033#if defined(__lint) 2034 2035struct cpu * 2036curcpup(void) 2037{ return 0; } 2038 2039#else /* __lint */ 2040 2041#if defined(__amd64) 2042 2043 ENTRY(curcpup) 2044 movq %gs:CPU_SELF, %rax 2045 ret 2046 SET_SIZE(curcpup) 2047 2048#elif defined(__i386) 2049 2050 ENTRY(curcpup) 2051 movl %gs:CPU_SELF, %eax 2052 ret 2053 SET_SIZE(curcpup) 2054 2055#endif /* __i386 */ 2056#endif /* __lint */ 2057 2058/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs() 2059 * These functions reverse the byte order of the input parameter and returns 2060 * the result. This is to convert the byte order from host byte order 2061 * (little endian) to network byte order (big endian), or vice versa. 2062 */ 2063 2064#if defined(__lint) 2065 2066uint64_t 2067htonll(uint64_t i) 2068{ return (i); } 2069 2070uint64_t 2071ntohll(uint64_t i) 2072{ return (i); } 2073 2074uint32_t 2075htonl(uint32_t i) 2076{ return (i); } 2077 2078uint32_t 2079ntohl(uint32_t i) 2080{ return (i); } 2081 2082uint16_t 2083htons(uint16_t i) 2084{ return (i); } 2085 2086uint16_t 2087ntohs(uint16_t i) 2088{ return (i); } 2089 2090#else /* __lint */ 2091 2092#if defined(__amd64) 2093 2094 ENTRY(htonll) 2095 ALTENTRY(ntohll) 2096 movq %rdi, %rax 2097 bswapq %rax 2098 ret 2099 SET_SIZE(ntohll) 2100 SET_SIZE(htonll) 2101 2102 /* XX64 there must be shorter sequences for this */ 2103 ENTRY(htonl) 2104 ALTENTRY(ntohl) 2105 movl %edi, %eax 2106 bswap %eax 2107 ret 2108 SET_SIZE(ntohl) 2109 SET_SIZE(htonl) 2110 2111 /* XX64 there must be better sequences for this */ 2112 ENTRY(htons) 2113 ALTENTRY(ntohs) 2114 movl %edi, %eax 2115 bswap %eax 2116 shrl $16, %eax 2117 ret 2118 SET_SIZE(ntohs) 2119 SET_SIZE(htons) 2120 2121#elif defined(__i386) 2122 2123 ENTRY(htonll) 2124 ALTENTRY(ntohll) 2125 movl 4(%esp), %edx 2126 movl 8(%esp), %eax 2127 bswap %edx 2128 bswap %eax 2129 ret 2130 SET_SIZE(ntohll) 2131 SET_SIZE(htonll) 2132 2133 ENTRY(htonl) 2134 ALTENTRY(ntohl) 2135 movl 4(%esp), %eax 2136 bswap %eax 2137 ret 2138 SET_SIZE(ntohl) 2139 SET_SIZE(htonl) 2140 2141 ENTRY(htons) 2142 ALTENTRY(ntohs) 2143 movl 4(%esp), %eax 2144 bswap %eax 2145 shrl $16, %eax 2146 ret 2147 SET_SIZE(ntohs) 2148 SET_SIZE(htons) 2149 2150#endif /* __i386 */ 2151#endif /* __lint */ 2152 2153 2154#if defined(__lint) 2155 2156/* ARGSUSED */ 2157void 2158intr_restore(ulong_t i) 2159{ return; } 2160 2161/* ARGSUSED */ 2162void 2163restore_int_flag(ulong_t i) 2164{ return; } 2165 2166#else /* __lint */ 2167 2168#if defined(__amd64) 2169 2170 ENTRY(intr_restore) 2171 ENTRY(restore_int_flag) 2172 testq $PS_IE, %rdi 2173 jz 1f 2174#if defined(__xpv) 2175 leaq xpv_panicking, %rsi 2176 movl (%rsi), %esi 2177 cmpl $0, %esi 2178 jne 1f 2179 /* 2180 * Since we're -really- running unprivileged, our attempt 2181 * to change the state of the IF bit will be ignored. 2182 * The virtual IF bit is tweaked by CLI and STI. 2183 */ 2184 IE_TO_EVENT_MASK(%rsi, %rdi) 2185#else 2186 sti 2187#endif 21881: 2189 ret 2190 SET_SIZE(restore_int_flag) 2191 SET_SIZE(intr_restore) 2192 2193#elif defined(__i386) 2194 2195 ENTRY(intr_restore) 2196 ENTRY(restore_int_flag) 2197 testl $PS_IE, 4(%esp) 2198 jz 1f 2199#if defined(__xpv) 2200 leal xpv_panicking, %edx 2201 movl (%edx), %edx 2202 cmpl $0, %edx 2203 jne 1f 2204 /* 2205 * Since we're -really- running unprivileged, our attempt 2206 * to change the state of the IF bit will be ignored. 2207 * The virtual IF bit is tweaked by CLI and STI. 2208 */ 2209 IE_TO_EVENT_MASK(%edx, 4(%esp)) 2210#else 2211 sti 2212#endif 22131: 2214 ret 2215 SET_SIZE(restore_int_flag) 2216 SET_SIZE(intr_restore) 2217 2218#endif /* __i386 */ 2219#endif /* __lint */ 2220 2221#if defined(__lint) 2222 2223void 2224sti(void) 2225{} 2226 2227void 2228cli(void) 2229{} 2230 2231#else /* __lint */ 2232 2233 ENTRY(sti) 2234 STI 2235 ret 2236 SET_SIZE(sti) 2237 2238 ENTRY(cli) 2239#if defined(__amd64) 2240 CLI(%rax) 2241#elif defined(__i386) 2242 CLI(%eax) 2243#endif /* __i386 */ 2244 ret 2245 SET_SIZE(cli) 2246 2247#endif /* __lint */ 2248 2249#if defined(__lint) 2250 2251dtrace_icookie_t 2252dtrace_interrupt_disable(void) 2253{ return (0); } 2254 2255#else /* __lint */ 2256 2257#if defined(__amd64) 2258 2259 ENTRY(dtrace_interrupt_disable) 2260 pushfq 2261 popq %rax 2262#if defined(__xpv) 2263 leaq xpv_panicking, %rdi 2264 movl (%rdi), %edi 2265 cmpl $0, %edi 2266 jne .dtrace_interrupt_disable_done 2267 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 2268 /* 2269 * Synthesize the PS_IE bit from the event mask bit 2270 */ 2271 andq $_BITNOT(PS_IE), %rax 2272 testb $1, %dl 2273 jnz .dtrace_interrupt_disable_done 2274 orq $PS_IE, %rax 2275#else 2276 CLI(%rdx) 2277#endif 2278.dtrace_interrupt_disable_done: 2279 ret 2280 SET_SIZE(dtrace_interrupt_disable) 2281 2282#elif defined(__i386) 2283 2284 ENTRY(dtrace_interrupt_disable) 2285 pushfl 2286 popl %eax 2287#if defined(__xpv) 2288 leal xpv_panicking, %edx 2289 movl (%edx), %edx 2290 cmpl $0, %edx 2291 jne .dtrace_interrupt_disable_done 2292 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2293 /* 2294 * Synthesize the PS_IE bit from the event mask bit 2295 */ 2296 andl $_BITNOT(PS_IE), %eax 2297 testb $1, %cl 2298 jnz .dtrace_interrupt_disable_done 2299 orl $PS_IE, %eax 2300#else 2301 CLI(%edx) 2302#endif 2303.dtrace_interrupt_disable_done: 2304 ret 2305 SET_SIZE(dtrace_interrupt_disable) 2306 2307#endif /* __i386 */ 2308#endif /* __lint */ 2309 2310#if defined(__lint) 2311 2312/*ARGSUSED*/ 2313void 2314dtrace_interrupt_enable(dtrace_icookie_t cookie) 2315{} 2316 2317#else /* __lint */ 2318 2319#if defined(__amd64) 2320 2321 ENTRY(dtrace_interrupt_enable) 2322 pushq %rdi 2323 popfq 2324#if defined(__xpv) 2325 leaq xpv_panicking, %rdx 2326 movl (%rdx), %edx 2327 cmpl $0, %edx 2328 jne .dtrace_interrupt_enable_done 2329 /* 2330 * Since we're -really- running unprivileged, our attempt 2331 * to change the state of the IF bit will be ignored. The 2332 * virtual IF bit is tweaked by CLI and STI. 2333 */ 2334 IE_TO_EVENT_MASK(%rdx, %rdi) 2335#endif 2336.dtrace_interrupt_enable_done: 2337 ret 2338 SET_SIZE(dtrace_interrupt_enable) 2339 2340#elif defined(__i386) 2341 2342 ENTRY(dtrace_interrupt_enable) 2343 movl 4(%esp), %eax 2344 pushl %eax 2345 popfl 2346#if defined(__xpv) 2347 leal xpv_panicking, %edx 2348 movl (%edx), %edx 2349 cmpl $0, %edx 2350 jne .dtrace_interrupt_enable_done 2351 /* 2352 * Since we're -really- running unprivileged, our attempt 2353 * to change the state of the IF bit will be ignored. The 2354 * virtual IF bit is tweaked by CLI and STI. 2355 */ 2356 IE_TO_EVENT_MASK(%edx, %eax) 2357#endif 2358.dtrace_interrupt_enable_done: 2359 ret 2360 SET_SIZE(dtrace_interrupt_enable) 2361 2362#endif /* __i386 */ 2363#endif /* __lint */ 2364 2365 2366#if defined(lint) 2367 2368void 2369dtrace_membar_producer(void) 2370{} 2371 2372void 2373dtrace_membar_consumer(void) 2374{} 2375 2376#else /* __lint */ 2377 2378 ENTRY(dtrace_membar_producer) 2379 rep; ret /* use 2 byte return instruction when branch target */ 2380 /* AMD Software Optimization Guide - Section 6.2 */ 2381 SET_SIZE(dtrace_membar_producer) 2382 2383 ENTRY(dtrace_membar_consumer) 2384 rep; ret /* use 2 byte return instruction when branch target */ 2385 /* AMD Software Optimization Guide - Section 6.2 */ 2386 SET_SIZE(dtrace_membar_consumer) 2387 2388#endif /* __lint */ 2389 2390#if defined(__lint) 2391 2392kthread_id_t 2393threadp(void) 2394{ return ((kthread_id_t)0); } 2395 2396#else /* __lint */ 2397 2398#if defined(__amd64) 2399 2400 ENTRY(threadp) 2401 movq %gs:CPU_THREAD, %rax 2402 ret 2403 SET_SIZE(threadp) 2404 2405#elif defined(__i386) 2406 2407 ENTRY(threadp) 2408 movl %gs:CPU_THREAD, %eax 2409 ret 2410 SET_SIZE(threadp) 2411 2412#endif /* __i386 */ 2413#endif /* __lint */ 2414 2415/* 2416 * Checksum routine for Internet Protocol Headers 2417 */ 2418 2419#if defined(__lint) 2420 2421/* ARGSUSED */ 2422unsigned int 2423ip_ocsum( 2424 ushort_t *address, /* ptr to 1st message buffer */ 2425 int halfword_count, /* length of data */ 2426 unsigned int sum) /* partial checksum */ 2427{ 2428 int i; 2429 unsigned int psum = 0; /* partial sum */ 2430 2431 for (i = 0; i < halfword_count; i++, address++) { 2432 psum += *address; 2433 } 2434 2435 while ((psum >> 16) != 0) { 2436 psum = (psum & 0xffff) + (psum >> 16); 2437 } 2438 2439 psum += sum; 2440 2441 while ((psum >> 16) != 0) { 2442 psum = (psum & 0xffff) + (psum >> 16); 2443 } 2444 2445 return (psum); 2446} 2447 2448#else /* __lint */ 2449 2450#if defined(__amd64) 2451 2452 ENTRY(ip_ocsum) 2453 pushq %rbp 2454 movq %rsp, %rbp 2455#ifdef DEBUG 2456 movq postbootkernelbase(%rip), %rax 2457 cmpq %rax, %rdi 2458 jnb 1f 2459 xorl %eax, %eax 2460 movq %rdi, %rsi 2461 leaq .ip_ocsum_panic_msg(%rip), %rdi 2462 call panic 2463 /*NOTREACHED*/ 2464.ip_ocsum_panic_msg: 2465 .string "ip_ocsum: address 0x%p below kernelbase\n" 24661: 2467#endif 2468 movl %esi, %ecx /* halfword_count */ 2469 movq %rdi, %rsi /* address */ 2470 /* partial sum in %edx */ 2471 xorl %eax, %eax 2472 testl %ecx, %ecx 2473 jz .ip_ocsum_done 2474 testq $3, %rsi 2475 jnz .ip_csum_notaligned 2476.ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */ 2477.next_iter: 2478 /* XX64 opportunities for prefetch? */ 2479 /* XX64 compute csum with 64 bit quantities? */ 2480 subl $32, %ecx 2481 jl .less_than_32 2482 2483 addl 0(%rsi), %edx 2484.only60: 2485 adcl 4(%rsi), %eax 2486.only56: 2487 adcl 8(%rsi), %edx 2488.only52: 2489 adcl 12(%rsi), %eax 2490.only48: 2491 adcl 16(%rsi), %edx 2492.only44: 2493 adcl 20(%rsi), %eax 2494.only40: 2495 adcl 24(%rsi), %edx 2496.only36: 2497 adcl 28(%rsi), %eax 2498.only32: 2499 adcl 32(%rsi), %edx 2500.only28: 2501 adcl 36(%rsi), %eax 2502.only24: 2503 adcl 40(%rsi), %edx 2504.only20: 2505 adcl 44(%rsi), %eax 2506.only16: 2507 adcl 48(%rsi), %edx 2508.only12: 2509 adcl 52(%rsi), %eax 2510.only8: 2511 adcl 56(%rsi), %edx 2512.only4: 2513 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */ 2514.only0: 2515 adcl $0, %eax /* could be adding -1 in eax with a carry */ 2516 adcl $0, %eax 2517 2518 addq $64, %rsi 2519 testl %ecx, %ecx 2520 jnz .next_iter 2521 2522.ip_ocsum_done: 2523 addl %eax, %edx 2524 adcl $0, %edx 2525 movl %edx, %eax /* form a 16 bit checksum by */ 2526 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2527 addw %dx, %ax 2528 adcw $0, %ax 2529 andl $0xffff, %eax 2530 leave 2531 ret 2532 2533.ip_csum_notaligned: 2534 xorl %edi, %edi 2535 movw (%rsi), %di 2536 addl %edi, %edx 2537 adcl $0, %edx 2538 addq $2, %rsi 2539 decl %ecx 2540 jmp .ip_csum_aligned 2541 2542.less_than_32: 2543 addl $32, %ecx 2544 testl $1, %ecx 2545 jz .size_aligned 2546 andl $0xfe, %ecx 2547 movzwl (%rsi, %rcx, 2), %edi 2548 addl %edi, %edx 2549 adcl $0, %edx 2550.size_aligned: 2551 movl %ecx, %edi 2552 shrl $1, %ecx 2553 shl $1, %edi 2554 subq $64, %rdi 2555 addq %rdi, %rsi 2556 leaq .ip_ocsum_jmptbl(%rip), %rdi 2557 leaq (%rdi, %rcx, 8), %rdi 2558 xorl %ecx, %ecx 2559 clc 2560 jmp *(%rdi) 2561 2562 .align 8 2563.ip_ocsum_jmptbl: 2564 .quad .only0, .only4, .only8, .only12, .only16, .only20 2565 .quad .only24, .only28, .only32, .only36, .only40, .only44 2566 .quad .only48, .only52, .only56, .only60 2567 SET_SIZE(ip_ocsum) 2568 2569#elif defined(__i386) 2570 2571 ENTRY(ip_ocsum) 2572 pushl %ebp 2573 movl %esp, %ebp 2574 pushl %ebx 2575 pushl %esi 2576 pushl %edi 2577 movl 12(%ebp), %ecx /* count of half words */ 2578 movl 16(%ebp), %edx /* partial checksum */ 2579 movl 8(%ebp), %esi 2580 xorl %eax, %eax 2581 testl %ecx, %ecx 2582 jz .ip_ocsum_done 2583 2584 testl $3, %esi 2585 jnz .ip_csum_notaligned 2586.ip_csum_aligned: 2587.next_iter: 2588 subl $32, %ecx 2589 jl .less_than_32 2590 2591 addl 0(%esi), %edx 2592.only60: 2593 adcl 4(%esi), %eax 2594.only56: 2595 adcl 8(%esi), %edx 2596.only52: 2597 adcl 12(%esi), %eax 2598.only48: 2599 adcl 16(%esi), %edx 2600.only44: 2601 adcl 20(%esi), %eax 2602.only40: 2603 adcl 24(%esi), %edx 2604.only36: 2605 adcl 28(%esi), %eax 2606.only32: 2607 adcl 32(%esi), %edx 2608.only28: 2609 adcl 36(%esi), %eax 2610.only24: 2611 adcl 40(%esi), %edx 2612.only20: 2613 adcl 44(%esi), %eax 2614.only16: 2615 adcl 48(%esi), %edx 2616.only12: 2617 adcl 52(%esi), %eax 2618.only8: 2619 adcl 56(%esi), %edx 2620.only4: 2621 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */ 2622.only0: 2623 adcl $0, %eax /* we could be adding -1 in eax with a carry */ 2624 adcl $0, %eax 2625 2626 addl $64, %esi 2627 andl %ecx, %ecx 2628 jnz .next_iter 2629 2630.ip_ocsum_done: 2631 addl %eax, %edx 2632 adcl $0, %edx 2633 movl %edx, %eax /* form a 16 bit checksum by */ 2634 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2635 addw %dx, %ax 2636 adcw $0, %ax 2637 andl $0xffff, %eax 2638 popl %edi /* restore registers */ 2639 popl %esi 2640 popl %ebx 2641 leave 2642 ret 2643 2644.ip_csum_notaligned: 2645 xorl %edi, %edi 2646 movw (%esi), %di 2647 addl %edi, %edx 2648 adcl $0, %edx 2649 addl $2, %esi 2650 decl %ecx 2651 jmp .ip_csum_aligned 2652 2653.less_than_32: 2654 addl $32, %ecx 2655 testl $1, %ecx 2656 jz .size_aligned 2657 andl $0xfe, %ecx 2658 movzwl (%esi, %ecx, 2), %edi 2659 addl %edi, %edx 2660 adcl $0, %edx 2661.size_aligned: 2662 movl %ecx, %edi 2663 shrl $1, %ecx 2664 shl $1, %edi 2665 subl $64, %edi 2666 addl %edi, %esi 2667 movl $.ip_ocsum_jmptbl, %edi 2668 lea (%edi, %ecx, 4), %edi 2669 xorl %ecx, %ecx 2670 clc 2671 jmp *(%edi) 2672 SET_SIZE(ip_ocsum) 2673 2674 .data 2675 .align 4 2676 2677.ip_ocsum_jmptbl: 2678 .long .only0, .only4, .only8, .only12, .only16, .only20 2679 .long .only24, .only28, .only32, .only36, .only40, .only44 2680 .long .only48, .only52, .only56, .only60 2681 2682 2683#endif /* __i386 */ 2684#endif /* __lint */ 2685 2686/* 2687 * multiply two long numbers and yield a u_longlong_t result, callable from C. 2688 * Provided to manipulate hrtime_t values. 2689 */ 2690#if defined(__lint) 2691 2692/* result = a * b; */ 2693 2694/* ARGSUSED */ 2695unsigned long long 2696mul32(uint_t a, uint_t b) 2697{ return (0); } 2698 2699#else /* __lint */ 2700 2701#if defined(__amd64) 2702 2703 ENTRY(mul32) 2704 xorl %edx, %edx /* XX64 joe, paranoia? */ 2705 movl %edi, %eax 2706 mull %esi 2707 shlq $32, %rdx 2708 orq %rdx, %rax 2709 ret 2710 SET_SIZE(mul32) 2711 2712#elif defined(__i386) 2713 2714 ENTRY(mul32) 2715 movl 8(%esp), %eax 2716 movl 4(%esp), %ecx 2717 mull %ecx 2718 ret 2719 SET_SIZE(mul32) 2720 2721#endif /* __i386 */ 2722#endif /* __lint */ 2723 2724#if defined(notused) 2725#if defined(__lint) 2726/* ARGSUSED */ 2727void 2728load_pte64(uint64_t *pte, uint64_t pte_value) 2729{} 2730#else /* __lint */ 2731 .globl load_pte64 2732load_pte64: 2733 movl 4(%esp), %eax 2734 movl 8(%esp), %ecx 2735 movl 12(%esp), %edx 2736 movl %edx, 4(%eax) 2737 movl %ecx, (%eax) 2738 ret 2739#endif /* __lint */ 2740#endif /* notused */ 2741 2742#if defined(__lint) 2743 2744/*ARGSUSED*/ 2745void 2746scan_memory(caddr_t addr, size_t size) 2747{} 2748 2749#else /* __lint */ 2750 2751#if defined(__amd64) 2752 2753 ENTRY(scan_memory) 2754 shrq $3, %rsi /* convert %rsi from byte to quadword count */ 2755 jz .scanm_done 2756 movq %rsi, %rcx /* move count into rep control register */ 2757 movq %rdi, %rsi /* move addr into lodsq control reg. */ 2758 rep lodsq /* scan the memory range */ 2759.scanm_done: 2760 rep; ret /* use 2 byte return instruction when branch target */ 2761 /* AMD Software Optimization Guide - Section 6.2 */ 2762 SET_SIZE(scan_memory) 2763 2764#elif defined(__i386) 2765 2766 ENTRY(scan_memory) 2767 pushl %ecx 2768 pushl %esi 2769 movl 16(%esp), %ecx /* move 2nd arg into rep control register */ 2770 shrl $2, %ecx /* convert from byte count to word count */ 2771 jz .scanm_done 2772 movl 12(%esp), %esi /* move 1st arg into lodsw control register */ 2773 .byte 0xf3 /* rep prefix. lame assembler. sigh. */ 2774 lodsl 2775.scanm_done: 2776 popl %esi 2777 popl %ecx 2778 ret 2779 SET_SIZE(scan_memory) 2780 2781#endif /* __i386 */ 2782#endif /* __lint */ 2783 2784 2785#if defined(__lint) 2786 2787/*ARGSUSED */ 2788int 2789lowbit(ulong_t i) 2790{ return (0); } 2791 2792#else /* __lint */ 2793 2794#if defined(__amd64) 2795 2796 ENTRY(lowbit) 2797 movl $-1, %eax 2798 bsfq %rdi, %rax 2799 incl %eax 2800 ret 2801 SET_SIZE(lowbit) 2802 2803#elif defined(__i386) 2804 2805 ENTRY(lowbit) 2806 movl $-1, %eax 2807 bsfl 4(%esp), %eax 2808 incl %eax 2809 ret 2810 SET_SIZE(lowbit) 2811 2812#endif /* __i386 */ 2813#endif /* __lint */ 2814 2815#if defined(__lint) 2816 2817/*ARGSUSED*/ 2818int 2819highbit(ulong_t i) 2820{ return (0); } 2821 2822#else /* __lint */ 2823 2824#if defined(__amd64) 2825 2826 ENTRY(highbit) 2827 movl $-1, %eax 2828 bsrq %rdi, %rax 2829 incl %eax 2830 ret 2831 SET_SIZE(highbit) 2832 2833#elif defined(__i386) 2834 2835 ENTRY(highbit) 2836 movl $-1, %eax 2837 bsrl 4(%esp), %eax 2838 incl %eax 2839 ret 2840 SET_SIZE(highbit) 2841 2842#endif /* __i386 */ 2843#endif /* __lint */ 2844 2845#if defined(__lint) 2846 2847/*ARGSUSED*/ 2848uint64_t 2849rdmsr(uint_t r) 2850{ return (0); } 2851 2852/*ARGSUSED*/ 2853void 2854wrmsr(uint_t r, const uint64_t val) 2855{} 2856 2857/*ARGSUSED*/ 2858uint64_t 2859xrdmsr(uint_t r) 2860{ return (0); } 2861 2862/*ARGSUSED*/ 2863void 2864xwrmsr(uint_t r, const uint64_t val) 2865{} 2866 2867void 2868invalidate_cache(void) 2869{} 2870 2871#else /* __lint */ 2872 2873#define XMSR_ACCESS_VAL $0x9c5a203a 2874 2875#if defined(__amd64) 2876 2877 ENTRY(rdmsr) 2878 movl %edi, %ecx 2879 rdmsr 2880 shlq $32, %rdx 2881 orq %rdx, %rax 2882 ret 2883 SET_SIZE(rdmsr) 2884 2885 ENTRY(wrmsr) 2886 movq %rsi, %rdx 2887 shrq $32, %rdx 2888 movl %esi, %eax 2889 movl %edi, %ecx 2890 wrmsr 2891 ret 2892 SET_SIZE(wrmsr) 2893 2894 ENTRY(xrdmsr) 2895 pushq %rbp 2896 movq %rsp, %rbp 2897 movl %edi, %ecx 2898 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2899 rdmsr 2900 shlq $32, %rdx 2901 orq %rdx, %rax 2902 leave 2903 ret 2904 SET_SIZE(xrdmsr) 2905 2906 ENTRY(xwrmsr) 2907 pushq %rbp 2908 movq %rsp, %rbp 2909 movl %edi, %ecx 2910 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2911 movq %rsi, %rdx 2912 shrq $32, %rdx 2913 movl %esi, %eax 2914 wrmsr 2915 leave 2916 ret 2917 SET_SIZE(xwrmsr) 2918 2919#elif defined(__i386) 2920 2921 ENTRY(rdmsr) 2922 movl 4(%esp), %ecx 2923 rdmsr 2924 ret 2925 SET_SIZE(rdmsr) 2926 2927 ENTRY(wrmsr) 2928 movl 4(%esp), %ecx 2929 movl 8(%esp), %eax 2930 movl 12(%esp), %edx 2931 wrmsr 2932 ret 2933 SET_SIZE(wrmsr) 2934 2935 ENTRY(xrdmsr) 2936 pushl %ebp 2937 movl %esp, %ebp 2938 movl 8(%esp), %ecx 2939 pushl %edi 2940 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2941 rdmsr 2942 popl %edi 2943 leave 2944 ret 2945 SET_SIZE(xrdmsr) 2946 2947 ENTRY(xwrmsr) 2948 pushl %ebp 2949 movl %esp, %ebp 2950 movl 8(%esp), %ecx 2951 movl 12(%esp), %eax 2952 movl 16(%esp), %edx 2953 pushl %edi 2954 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2955 wrmsr 2956 popl %edi 2957 leave 2958 ret 2959 SET_SIZE(xwrmsr) 2960 2961#endif /* __i386 */ 2962 2963 ENTRY(invalidate_cache) 2964 wbinvd 2965 ret 2966 SET_SIZE(invalidate_cache) 2967 2968#endif /* __lint */ 2969 2970#if defined(__lint) 2971 2972/*ARGSUSED*/ 2973void 2974getcregs(struct cregs *crp) 2975{} 2976 2977#else /* __lint */ 2978 2979#if defined(__amd64) 2980 2981 ENTRY_NP(getcregs) 2982#if defined(__xpv) 2983 /* 2984 * Only a few of the hardware control registers or descriptor tables 2985 * are directly accessible to us, so just zero the structure. 2986 * 2987 * XXPV Perhaps it would be helpful for the hypervisor to return 2988 * virtualized versions of these for post-mortem use. 2989 * (Need to reevaluate - perhaps it already does!) 2990 */ 2991 pushq %rdi /* save *crp */ 2992 movq $CREGSZ, %rsi 2993 call bzero 2994 popq %rdi 2995 2996 /* 2997 * Dump what limited information we can 2998 */ 2999 movq %cr0, %rax 3000 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3001 movq %cr2, %rax 3002 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3003 movq %cr3, %rax 3004 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3005 movq %cr4, %rax 3006 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3007 3008#else /* __xpv */ 3009 3010#define GETMSR(r, off, d) \ 3011 movl $r, %ecx; \ 3012 rdmsr; \ 3013 movl %eax, off(d); \ 3014 movl %edx, off+4(d) 3015 3016 xorl %eax, %eax 3017 movq %rax, CREG_GDT+8(%rdi) 3018 sgdt CREG_GDT(%rdi) /* 10 bytes */ 3019 movq %rax, CREG_IDT+8(%rdi) 3020 sidt CREG_IDT(%rdi) /* 10 bytes */ 3021 movq %rax, CREG_LDT(%rdi) 3022 sldt CREG_LDT(%rdi) /* 2 bytes */ 3023 movq %rax, CREG_TASKR(%rdi) 3024 str CREG_TASKR(%rdi) /* 2 bytes */ 3025 movq %cr0, %rax 3026 movq %rax, CREG_CR0(%rdi) /* cr0 */ 3027 movq %cr2, %rax 3028 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3029 movq %cr3, %rax 3030 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3031 movq %cr4, %rax 3032 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3033 movq %cr8, %rax 3034 movq %rax, CREG_CR8(%rdi) /* cr8 */ 3035 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi) 3036 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi) 3037#endif /* __xpv */ 3038 ret 3039 SET_SIZE(getcregs) 3040 3041#undef GETMSR 3042 3043#elif defined(__i386) 3044 3045 ENTRY_NP(getcregs) 3046#if defined(__xpv) 3047 /* 3048 * Only a few of the hardware control registers or descriptor tables 3049 * are directly accessible to us, so just zero the structure. 3050 * 3051 * XXPV Perhaps it would be helpful for the hypervisor to return 3052 * virtualized versions of these for post-mortem use. 3053 * (Need to reevaluate - perhaps it already does!) 3054 */ 3055 movl 4(%esp), %edx 3056 pushl $CREGSZ 3057 pushl %edx 3058 call bzero 3059 addl $8, %esp 3060 movl 4(%esp), %edx 3061 3062 /* 3063 * Dump what limited information we can 3064 */ 3065 movl %cr0, %eax 3066 movl %eax, CREG_CR0(%edx) /* cr0 */ 3067 movl %cr2, %eax 3068 movl %eax, CREG_CR2(%edx) /* cr2 */ 3069 movl %cr3, %eax 3070 movl %eax, CREG_CR3(%edx) /* cr3 */ 3071 movl %cr4, %eax 3072 movl %eax, CREG_CR4(%edx) /* cr4 */ 3073 3074#else /* __xpv */ 3075 3076 movl 4(%esp), %edx 3077 movw $0, CREG_GDT+6(%edx) 3078 movw $0, CREG_IDT+6(%edx) 3079 sgdt CREG_GDT(%edx) /* gdt */ 3080 sidt CREG_IDT(%edx) /* idt */ 3081 sldt CREG_LDT(%edx) /* ldt */ 3082 str CREG_TASKR(%edx) /* task */ 3083 movl %cr0, %eax 3084 movl %eax, CREG_CR0(%edx) /* cr0 */ 3085 movl %cr2, %eax 3086 movl %eax, CREG_CR2(%edx) /* cr2 */ 3087 movl %cr3, %eax 3088 movl %eax, CREG_CR3(%edx) /* cr3 */ 3089 testl $X86_LARGEPAGE, x86_feature 3090 jz .nocr4 3091 movl %cr4, %eax 3092 movl %eax, CREG_CR4(%edx) /* cr4 */ 3093 jmp .skip 3094.nocr4: 3095 movl $0, CREG_CR4(%edx) 3096.skip: 3097#endif 3098 ret 3099 SET_SIZE(getcregs) 3100 3101#endif /* __i386 */ 3102#endif /* __lint */ 3103 3104 3105/* 3106 * A panic trigger is a word which is updated atomically and can only be set 3107 * once. We atomically store 0xDEFACEDD and load the old value. If the 3108 * previous value was 0, we succeed and return 1; otherwise return 0. 3109 * This allows a partially corrupt trigger to still trigger correctly. DTrace 3110 * has its own version of this function to allow it to panic correctly from 3111 * probe context. 3112 */ 3113#if defined(__lint) 3114 3115/*ARGSUSED*/ 3116int 3117panic_trigger(int *tp) 3118{ return (0); } 3119 3120/*ARGSUSED*/ 3121int 3122dtrace_panic_trigger(int *tp) 3123{ return (0); } 3124 3125#else /* __lint */ 3126 3127#if defined(__amd64) 3128 3129 ENTRY_NP(panic_trigger) 3130 xorl %eax, %eax 3131 movl $0xdefacedd, %edx 3132 lock 3133 xchgl %edx, (%rdi) 3134 cmpl $0, %edx 3135 je 0f 3136 movl $0, %eax 3137 ret 31380: movl $1, %eax 3139 ret 3140 SET_SIZE(panic_trigger) 3141 3142 ENTRY_NP(dtrace_panic_trigger) 3143 xorl %eax, %eax 3144 movl $0xdefacedd, %edx 3145 lock 3146 xchgl %edx, (%rdi) 3147 cmpl $0, %edx 3148 je 0f 3149 movl $0, %eax 3150 ret 31510: movl $1, %eax 3152 ret 3153 SET_SIZE(dtrace_panic_trigger) 3154 3155#elif defined(__i386) 3156 3157 ENTRY_NP(panic_trigger) 3158 movl 4(%esp), %edx / %edx = address of trigger 3159 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3160 lock / assert lock 3161 xchgl %eax, (%edx) / exchange %eax and the trigger 3162 cmpl $0, %eax / if (%eax == 0x0) 3163 je 0f / return (1); 3164 movl $0, %eax / else 3165 ret / return (0); 31660: movl $1, %eax 3167 ret 3168 SET_SIZE(panic_trigger) 3169 3170 ENTRY_NP(dtrace_panic_trigger) 3171 movl 4(%esp), %edx / %edx = address of trigger 3172 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3173 lock / assert lock 3174 xchgl %eax, (%edx) / exchange %eax and the trigger 3175 cmpl $0, %eax / if (%eax == 0x0) 3176 je 0f / return (1); 3177 movl $0, %eax / else 3178 ret / return (0); 31790: movl $1, %eax 3180 ret 3181 SET_SIZE(dtrace_panic_trigger) 3182 3183#endif /* __i386 */ 3184#endif /* __lint */ 3185 3186/* 3187 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 3188 * into the panic code implemented in panicsys(). vpanic() is responsible 3189 * for passing through the format string and arguments, and constructing a 3190 * regs structure on the stack into which it saves the current register 3191 * values. If we are not dying due to a fatal trap, these registers will 3192 * then be preserved in panicbuf as the current processor state. Before 3193 * invoking panicsys(), vpanic() activates the first panic trigger (see 3194 * common/os/panic.c) and switches to the panic_stack if successful. Note that 3195 * DTrace takes a slightly different panic path if it must panic from probe 3196 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 3197 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 3198 * branches back into vpanic(). 3199 */ 3200#if defined(__lint) 3201 3202/*ARGSUSED*/ 3203void 3204vpanic(const char *format, va_list alist) 3205{} 3206 3207/*ARGSUSED*/ 3208void 3209dtrace_vpanic(const char *format, va_list alist) 3210{} 3211 3212#else /* __lint */ 3213 3214#if defined(__amd64) 3215 3216 ENTRY_NP(vpanic) /* Initial stack layout: */ 3217 3218 pushq %rbp /* | %rip | 0x60 */ 3219 movq %rsp, %rbp /* | %rbp | 0x58 */ 3220 pushfq /* | rfl | 0x50 */ 3221 pushq %r11 /* | %r11 | 0x48 */ 3222 pushq %r10 /* | %r10 | 0x40 */ 3223 pushq %rbx /* | %rbx | 0x38 */ 3224 pushq %rax /* | %rax | 0x30 */ 3225 pushq %r9 /* | %r9 | 0x28 */ 3226 pushq %r8 /* | %r8 | 0x20 */ 3227 pushq %rcx /* | %rcx | 0x18 */ 3228 pushq %rdx /* | %rdx | 0x10 */ 3229 pushq %rsi /* | %rsi | 0x8 alist */ 3230 pushq %rdi /* | %rdi | 0x0 format */ 3231 3232 movq %rsp, %rbx /* %rbx = current %rsp */ 3233 3234 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3235 call panic_trigger /* %eax = panic_trigger() */ 3236 3237vpanic_common: 3238 /* 3239 * The panic_trigger result is in %eax from the call above, and 3240 * dtrace_panic places it in %eax before branching here. 3241 * The rdmsr instructions that follow below will clobber %eax so 3242 * we stash the panic_trigger result in %r11d. 3243 */ 3244 movl %eax, %r11d 3245 cmpl $0, %r11d 3246 je 0f 3247 3248 /* 3249 * If panic_trigger() was successful, we are the first to initiate a 3250 * panic: we now switch to the reserved panic_stack before continuing. 3251 */ 3252 leaq panic_stack(%rip), %rsp 3253 addq $PANICSTKSIZE, %rsp 32540: subq $REGSIZE, %rsp 3255 /* 3256 * Now that we've got everything set up, store the register values as 3257 * they were when we entered vpanic() to the designated location in 3258 * the regs structure we allocated on the stack. 3259 */ 3260 movq 0x0(%rbx), %rcx 3261 movq %rcx, REGOFF_RDI(%rsp) 3262 movq 0x8(%rbx), %rcx 3263 movq %rcx, REGOFF_RSI(%rsp) 3264 movq 0x10(%rbx), %rcx 3265 movq %rcx, REGOFF_RDX(%rsp) 3266 movq 0x18(%rbx), %rcx 3267 movq %rcx, REGOFF_RCX(%rsp) 3268 movq 0x20(%rbx), %rcx 3269 3270 movq %rcx, REGOFF_R8(%rsp) 3271 movq 0x28(%rbx), %rcx 3272 movq %rcx, REGOFF_R9(%rsp) 3273 movq 0x30(%rbx), %rcx 3274 movq %rcx, REGOFF_RAX(%rsp) 3275 movq 0x38(%rbx), %rcx 3276 movq %rcx, REGOFF_RBX(%rsp) 3277 movq 0x58(%rbx), %rcx 3278 3279 movq %rcx, REGOFF_RBP(%rsp) 3280 movq 0x40(%rbx), %rcx 3281 movq %rcx, REGOFF_R10(%rsp) 3282 movq 0x48(%rbx), %rcx 3283 movq %rcx, REGOFF_R11(%rsp) 3284 movq %r12, REGOFF_R12(%rsp) 3285 3286 movq %r13, REGOFF_R13(%rsp) 3287 movq %r14, REGOFF_R14(%rsp) 3288 movq %r15, REGOFF_R15(%rsp) 3289 3290 xorl %ecx, %ecx 3291 movw %ds, %cx 3292 movq %rcx, REGOFF_DS(%rsp) 3293 movw %es, %cx 3294 movq %rcx, REGOFF_ES(%rsp) 3295 movw %fs, %cx 3296 movq %rcx, REGOFF_FS(%rsp) 3297 movw %gs, %cx 3298 movq %rcx, REGOFF_GS(%rsp) 3299 3300 movq $0, REGOFF_TRAPNO(%rsp) 3301 3302 movq $0, REGOFF_ERR(%rsp) 3303 leaq vpanic(%rip), %rcx 3304 movq %rcx, REGOFF_RIP(%rsp) 3305 movw %cs, %cx 3306 movzwq %cx, %rcx 3307 movq %rcx, REGOFF_CS(%rsp) 3308 movq 0x50(%rbx), %rcx 3309 movq %rcx, REGOFF_RFL(%rsp) 3310 movq %rbx, %rcx 3311 addq $0x60, %rcx 3312 movq %rcx, REGOFF_RSP(%rsp) 3313 movw %ss, %cx 3314 movzwq %cx, %rcx 3315 movq %rcx, REGOFF_SS(%rsp) 3316 3317 /* 3318 * panicsys(format, alist, rp, on_panic_stack) 3319 */ 3320 movq REGOFF_RDI(%rsp), %rdi /* format */ 3321 movq REGOFF_RSI(%rsp), %rsi /* alist */ 3322 movq %rsp, %rdx /* struct regs */ 3323 movl %r11d, %ecx /* on_panic_stack */ 3324 call panicsys 3325 addq $REGSIZE, %rsp 3326 popq %rdi 3327 popq %rsi 3328 popq %rdx 3329 popq %rcx 3330 popq %r8 3331 popq %r9 3332 popq %rax 3333 popq %rbx 3334 popq %r10 3335 popq %r11 3336 popfq 3337 leave 3338 ret 3339 SET_SIZE(vpanic) 3340 3341 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */ 3342 3343 pushq %rbp /* | %rip | 0x60 */ 3344 movq %rsp, %rbp /* | %rbp | 0x58 */ 3345 pushfq /* | rfl | 0x50 */ 3346 pushq %r11 /* | %r11 | 0x48 */ 3347 pushq %r10 /* | %r10 | 0x40 */ 3348 pushq %rbx /* | %rbx | 0x38 */ 3349 pushq %rax /* | %rax | 0x30 */ 3350 pushq %r9 /* | %r9 | 0x28 */ 3351 pushq %r8 /* | %r8 | 0x20 */ 3352 pushq %rcx /* | %rcx | 0x18 */ 3353 pushq %rdx /* | %rdx | 0x10 */ 3354 pushq %rsi /* | %rsi | 0x8 alist */ 3355 pushq %rdi /* | %rdi | 0x0 format */ 3356 3357 movq %rsp, %rbx /* %rbx = current %rsp */ 3358 3359 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3360 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 3361 jmp vpanic_common 3362 3363 SET_SIZE(dtrace_vpanic) 3364 3365#elif defined(__i386) 3366 3367 ENTRY_NP(vpanic) / Initial stack layout: 3368 3369 pushl %ebp / | %eip | 20 3370 movl %esp, %ebp / | %ebp | 16 3371 pushl %eax / | %eax | 12 3372 pushl %ebx / | %ebx | 8 3373 pushl %ecx / | %ecx | 4 3374 pushl %edx / | %edx | 0 3375 3376 movl %esp, %ebx / %ebx = current stack pointer 3377 3378 lea panic_quiesce, %eax / %eax = &panic_quiesce 3379 pushl %eax / push &panic_quiesce 3380 call panic_trigger / %eax = panic_trigger() 3381 addl $4, %esp / reset stack pointer 3382 3383vpanic_common: 3384 cmpl $0, %eax / if (%eax == 0) 3385 je 0f / goto 0f; 3386 3387 /* 3388 * If panic_trigger() was successful, we are the first to initiate a 3389 * panic: we now switch to the reserved panic_stack before continuing. 3390 */ 3391 lea panic_stack, %esp / %esp = panic_stack 3392 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE 3393 33940: subl $REGSIZE, %esp / allocate struct regs 3395 3396 /* 3397 * Now that we've got everything set up, store the register values as 3398 * they were when we entered vpanic() to the designated location in 3399 * the regs structure we allocated on the stack. 3400 */ 3401#if !defined(__GNUC_AS__) 3402 movw %gs, %edx 3403 movl %edx, REGOFF_GS(%esp) 3404 movw %fs, %edx 3405 movl %edx, REGOFF_FS(%esp) 3406 movw %es, %edx 3407 movl %edx, REGOFF_ES(%esp) 3408 movw %ds, %edx 3409 movl %edx, REGOFF_DS(%esp) 3410#else /* __GNUC_AS__ */ 3411 mov %gs, %edx 3412 mov %edx, REGOFF_GS(%esp) 3413 mov %fs, %edx 3414 mov %edx, REGOFF_FS(%esp) 3415 mov %es, %edx 3416 mov %edx, REGOFF_ES(%esp) 3417 mov %ds, %edx 3418 mov %edx, REGOFF_DS(%esp) 3419#endif /* __GNUC_AS__ */ 3420 movl %edi, REGOFF_EDI(%esp) 3421 movl %esi, REGOFF_ESI(%esp) 3422 movl 16(%ebx), %ecx 3423 movl %ecx, REGOFF_EBP(%esp) 3424 movl %ebx, %ecx 3425 addl $20, %ecx 3426 movl %ecx, REGOFF_ESP(%esp) 3427 movl 8(%ebx), %ecx 3428 movl %ecx, REGOFF_EBX(%esp) 3429 movl 0(%ebx), %ecx 3430 movl %ecx, REGOFF_EDX(%esp) 3431 movl 4(%ebx), %ecx 3432 movl %ecx, REGOFF_ECX(%esp) 3433 movl 12(%ebx), %ecx 3434 movl %ecx, REGOFF_EAX(%esp) 3435 movl $0, REGOFF_TRAPNO(%esp) 3436 movl $0, REGOFF_ERR(%esp) 3437 lea vpanic, %ecx 3438 movl %ecx, REGOFF_EIP(%esp) 3439#if !defined(__GNUC_AS__) 3440 movw %cs, %edx 3441#else /* __GNUC_AS__ */ 3442 mov %cs, %edx 3443#endif /* __GNUC_AS__ */ 3444 movl %edx, REGOFF_CS(%esp) 3445 pushfl 3446 popl %ecx 3447#if defined(__xpv) 3448 /* 3449 * Synthesize the PS_IE bit from the event mask bit 3450 */ 3451 CURTHREAD(%edx) 3452 KPREEMPT_DISABLE(%edx) 3453 EVENT_MASK_TO_IE(%edx, %ecx) 3454 CURTHREAD(%edx) 3455 KPREEMPT_ENABLE_NOKP(%edx) 3456#endif 3457 movl %ecx, REGOFF_EFL(%esp) 3458 movl $0, REGOFF_UESP(%esp) 3459#if !defined(__GNUC_AS__) 3460 movw %ss, %edx 3461#else /* __GNUC_AS__ */ 3462 mov %ss, %edx 3463#endif /* __GNUC_AS__ */ 3464 movl %edx, REGOFF_SS(%esp) 3465 3466 movl %esp, %ecx / %ecx = ®s 3467 pushl %eax / push on_panic_stack 3468 pushl %ecx / push ®s 3469 movl 12(%ebp), %ecx / %ecx = alist 3470 pushl %ecx / push alist 3471 movl 8(%ebp), %ecx / %ecx = format 3472 pushl %ecx / push format 3473 call panicsys / panicsys(); 3474 addl $16, %esp / pop arguments 3475 3476 addl $REGSIZE, %esp 3477 popl %edx 3478 popl %ecx 3479 popl %ebx 3480 popl %eax 3481 leave 3482 ret 3483 SET_SIZE(vpanic) 3484 3485 ENTRY_NP(dtrace_vpanic) / Initial stack layout: 3486 3487 pushl %ebp / | %eip | 20 3488 movl %esp, %ebp / | %ebp | 16 3489 pushl %eax / | %eax | 12 3490 pushl %ebx / | %ebx | 8 3491 pushl %ecx / | %ecx | 4 3492 pushl %edx / | %edx | 0 3493 3494 movl %esp, %ebx / %ebx = current stack pointer 3495 3496 lea panic_quiesce, %eax / %eax = &panic_quiesce 3497 pushl %eax / push &panic_quiesce 3498 call dtrace_panic_trigger / %eax = dtrace_panic_trigger() 3499 addl $4, %esp / reset stack pointer 3500 jmp vpanic_common / jump back to common code 3501 3502 SET_SIZE(dtrace_vpanic) 3503 3504#endif /* __i386 */ 3505#endif /* __lint */ 3506 3507#if defined(__lint) 3508 3509void 3510hres_tick(void) 3511{} 3512 3513int64_t timedelta; 3514hrtime_t hres_last_tick; 3515volatile timestruc_t hrestime; 3516int64_t hrestime_adj; 3517volatile int hres_lock; 3518hrtime_t hrtime_base; 3519 3520#else /* __lint */ 3521 3522 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8) 3523 .NWORD 0, 0 3524 3525 DGDEF3(hrestime_adj, 8, 8) 3526 .long 0, 0 3527 3528 DGDEF3(hres_last_tick, 8, 8) 3529 .long 0, 0 3530 3531 DGDEF3(timedelta, 8, 8) 3532 .long 0, 0 3533 3534 DGDEF3(hres_lock, 4, 8) 3535 .long 0 3536 3537 /* 3538 * initialized to a non zero value to make pc_gethrtime() 3539 * work correctly even before clock is initialized 3540 */ 3541 DGDEF3(hrtime_base, 8, 8) 3542 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0 3543 3544 DGDEF3(adj_shift, 4, 4) 3545 .long ADJ_SHIFT 3546 3547#if defined(__amd64) 3548 3549 ENTRY_NP(hres_tick) 3550 pushq %rbp 3551 movq %rsp, %rbp 3552 3553 /* 3554 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3555 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3556 * At worst, performing this now instead of under CLOCK_LOCK may 3557 * introduce some jitter in pc_gethrestime(). 3558 */ 3559 call *gethrtimef(%rip) 3560 movq %rax, %r8 3561 3562 leaq hres_lock(%rip), %rax 3563 movb $-1, %dl 3564.CL1: 3565 xchgb %dl, (%rax) 3566 testb %dl, %dl 3567 jz .CL3 /* got it */ 3568.CL2: 3569 cmpb $0, (%rax) /* possible to get lock? */ 3570 pause 3571 jne .CL2 3572 jmp .CL1 /* yes, try again */ 3573.CL3: 3574 /* 3575 * compute the interval since last time hres_tick was called 3576 * and adjust hrtime_base and hrestime accordingly 3577 * hrtime_base is an 8 byte value (in nsec), hrestime is 3578 * a timestruc_t (sec, nsec) 3579 */ 3580 leaq hres_last_tick(%rip), %rax 3581 movq %r8, %r11 3582 subq (%rax), %r8 3583 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */ 3584 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */ 3585 /* 3586 * Now that we have CLOCK_LOCK, we can update hres_last_tick 3587 */ 3588 movq %r11, (%rax) 3589 3590 call __adj_hrestime 3591 3592 /* 3593 * release the hres_lock 3594 */ 3595 incl hres_lock(%rip) 3596 leave 3597 ret 3598 SET_SIZE(hres_tick) 3599 3600#elif defined(__i386) 3601 3602 ENTRY_NP(hres_tick) 3603 pushl %ebp 3604 movl %esp, %ebp 3605 pushl %esi 3606 pushl %ebx 3607 3608 /* 3609 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3610 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3611 * At worst, performing this now instead of under CLOCK_LOCK may 3612 * introduce some jitter in pc_gethrestime(). 3613 */ 3614 call *gethrtimef 3615 movl %eax, %ebx 3616 movl %edx, %esi 3617 3618 movl $hres_lock, %eax 3619 movl $-1, %edx 3620.CL1: 3621 xchgb %dl, (%eax) 3622 testb %dl, %dl 3623 jz .CL3 / got it 3624.CL2: 3625 cmpb $0, (%eax) / possible to get lock? 3626 pause 3627 jne .CL2 3628 jmp .CL1 / yes, try again 3629.CL3: 3630 /* 3631 * compute the interval since last time hres_tick was called 3632 * and adjust hrtime_base and hrestime accordingly 3633 * hrtime_base is an 8 byte value (in nsec), hrestime is 3634 * timestruc_t (sec, nsec) 3635 */ 3636 3637 lea hres_last_tick, %eax 3638 3639 movl %ebx, %edx 3640 movl %esi, %ecx 3641 3642 subl (%eax), %edx 3643 sbbl 4(%eax), %ecx 3644 3645 addl %edx, hrtime_base / add interval to hrtime_base 3646 adcl %ecx, hrtime_base+4 3647 3648 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec 3649 3650 / 3651 / Now that we have CLOCK_LOCK, we can update hres_last_tick. 3652 / 3653 movl %ebx, (%eax) 3654 movl %esi, 4(%eax) 3655 3656 / get hrestime at this moment. used as base for pc_gethrestime 3657 / 3658 / Apply adjustment, if any 3659 / 3660 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT) 3661 / (max_hres_adj) 3662 / 3663 / void 3664 / adj_hrestime() 3665 / { 3666 / long long adj; 3667 / 3668 / if (hrestime_adj == 0) 3669 / adj = 0; 3670 / else if (hrestime_adj > 0) { 3671 / if (hrestime_adj < HRES_ADJ) 3672 / adj = hrestime_adj; 3673 / else 3674 / adj = HRES_ADJ; 3675 / } 3676 / else { 3677 / if (hrestime_adj < -(HRES_ADJ)) 3678 / adj = -(HRES_ADJ); 3679 / else 3680 / adj = hrestime_adj; 3681 / } 3682 / 3683 / timedelta -= adj; 3684 / hrestime_adj = timedelta; 3685 / hrestime.tv_nsec += adj; 3686 / 3687 / while (hrestime.tv_nsec >= NANOSEC) { 3688 / one_sec++; 3689 / hrestime.tv_sec++; 3690 / hrestime.tv_nsec -= NANOSEC; 3691 / } 3692 / } 3693__adj_hrestime: 3694 movl hrestime_adj, %esi / if (hrestime_adj == 0) 3695 movl hrestime_adj+4, %edx 3696 andl %esi, %esi 3697 jne .CL4 / no 3698 andl %edx, %edx 3699 jne .CL4 / no 3700 subl %ecx, %ecx / yes, adj = 0; 3701 subl %edx, %edx 3702 jmp .CL5 3703.CL4: 3704 subl %ecx, %ecx 3705 subl %eax, %eax 3706 subl %esi, %ecx 3707 sbbl %edx, %eax 3708 andl %eax, %eax / if (hrestime_adj > 0) 3709 jge .CL6 3710 3711 / In the following comments, HRES_ADJ is used, while in the code 3712 / max_hres_adj is used. 3713 / 3714 / The test for "hrestime_adj < HRES_ADJ" is complicated because 3715 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3716 / on the logical equivalence of: 3717 / 3718 / !(hrestime_adj < HRES_ADJ) 3719 / 3720 / and the two step sequence: 3721 / 3722 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry 3723 / 3724 / which computes whether or not the least significant 32-bits 3725 / of hrestime_adj is greater than HRES_ADJ, followed by: 3726 / 3727 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry 3728 / 3729 / which generates a carry whenever step 1 is true or the most 3730 / significant long of the longlong hrestime_adj is non-zero. 3731 3732 movl max_hres_adj, %ecx / hrestime_adj is positive 3733 subl %esi, %ecx 3734 movl %edx, %eax 3735 adcl $-1, %eax 3736 jnc .CL7 3737 movl max_hres_adj, %ecx / adj = HRES_ADJ; 3738 subl %edx, %edx 3739 jmp .CL5 3740 3741 / The following computation is similar to the one above. 3742 / 3743 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because 3744 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3745 / on the logical equivalence of: 3746 / 3747 / (hrestime_adj > -HRES_ADJ) 3748 / 3749 / and the two step sequence: 3750 / 3751 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry 3752 / 3753 / which means the least significant 32-bits of hrestime_adj is 3754 / greater than -HRES_ADJ, followed by: 3755 / 3756 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry 3757 / 3758 / which generates a carry only when step 1 is true and the most 3759 / significant long of the longlong hrestime_adj is -1. 3760 3761.CL6: / hrestime_adj is negative 3762 movl %esi, %ecx 3763 addl max_hres_adj, %ecx 3764 movl %edx, %eax 3765 adcl $0, %eax 3766 jc .CL7 3767 xor %ecx, %ecx 3768 subl max_hres_adj, %ecx / adj = -(HRES_ADJ); 3769 movl $-1, %edx 3770 jmp .CL5 3771.CL7: 3772 movl %esi, %ecx / adj = hrestime_adj; 3773.CL5: 3774 movl timedelta, %esi 3775 subl %ecx, %esi 3776 movl timedelta+4, %eax 3777 sbbl %edx, %eax 3778 movl %esi, timedelta 3779 movl %eax, timedelta+4 / timedelta -= adj; 3780 movl %esi, hrestime_adj 3781 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta; 3782 addl hrestime+4, %ecx 3783 3784 movl %ecx, %eax / eax = tv_nsec 37851: 3786 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC) 3787 jb .CL8 / no 3788 incl one_sec / yes, one_sec++; 3789 incl hrestime / hrestime.tv_sec++; 3790 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC 3791 jmp 1b / check for more seconds 3792 3793.CL8: 3794 movl %eax, hrestime+4 / store final into hrestime.tv_nsec 3795 incl hres_lock / release the hres_lock 3796 3797 popl %ebx 3798 popl %esi 3799 leave 3800 ret 3801 SET_SIZE(hres_tick) 3802 3803#endif /* __i386 */ 3804#endif /* __lint */ 3805 3806/* 3807 * void prefetch_smap_w(void *) 3808 * 3809 * Prefetch ahead within a linear list of smap structures. 3810 * Not implemented for ia32. Stub for compatibility. 3811 */ 3812 3813#if defined(__lint) 3814 3815/*ARGSUSED*/ 3816void prefetch_smap_w(void *smp) 3817{} 3818 3819#else /* __lint */ 3820 3821 ENTRY(prefetch_smap_w) 3822 rep; ret /* use 2 byte return instruction when branch target */ 3823 /* AMD Software Optimization Guide - Section 6.2 */ 3824 SET_SIZE(prefetch_smap_w) 3825 3826#endif /* __lint */ 3827 3828/* 3829 * prefetch_page_r(page_t *) 3830 * issue prefetch instructions for a page_t 3831 */ 3832#if defined(__lint) 3833 3834/*ARGSUSED*/ 3835void 3836prefetch_page_r(void *pp) 3837{} 3838 3839#else /* __lint */ 3840 3841 ENTRY(prefetch_page_r) 3842 rep; ret /* use 2 byte return instruction when branch target */ 3843 /* AMD Software Optimization Guide - Section 6.2 */ 3844 SET_SIZE(prefetch_page_r) 3845 3846#endif /* __lint */ 3847 3848#if defined(__lint) 3849 3850/*ARGSUSED*/ 3851int 3852bcmp(const void *s1, const void *s2, size_t count) 3853{ return (0); } 3854 3855#else /* __lint */ 3856 3857#if defined(__amd64) 3858 3859 ENTRY(bcmp) 3860 pushq %rbp 3861 movq %rsp, %rbp 3862#ifdef DEBUG 3863 movq postbootkernelbase(%rip), %r11 3864 cmpq %r11, %rdi 3865 jb 0f 3866 cmpq %r11, %rsi 3867 jnb 1f 38680: leaq .bcmp_panic_msg(%rip), %rdi 3869 xorl %eax, %eax 3870 call panic 38711: 3872#endif /* DEBUG */ 3873 call memcmp 3874 testl %eax, %eax 3875 setne %dl 3876 leave 3877 movzbl %dl, %eax 3878 ret 3879 SET_SIZE(bcmp) 3880 3881#elif defined(__i386) 3882 3883#define ARG_S1 8 3884#define ARG_S2 12 3885#define ARG_LENGTH 16 3886 3887 ENTRY(bcmp) 3888 pushl %ebp 3889 movl %esp, %ebp / create new stack frame 3890#ifdef DEBUG 3891 movl postbootkernelbase, %eax 3892 cmpl %eax, ARG_S1(%ebp) 3893 jb 0f 3894 cmpl %eax, ARG_S2(%ebp) 3895 jnb 1f 38960: pushl $.bcmp_panic_msg 3897 call panic 38981: 3899#endif /* DEBUG */ 3900 3901 pushl %edi / save register variable 3902 movl ARG_S1(%ebp), %eax / %eax = address of string 1 3903 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2 3904 cmpl %eax, %ecx / if the same string 3905 je .equal / goto .equal 3906 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes 3907 cmpl $4, %edi / if %edi < 4 3908 jb .byte_check / goto .byte_check 3909 .align 4 3910.word_loop: 3911 movl (%ecx), %edx / move 1 word from (%ecx) to %edx 3912 leal -4(%edi), %edi / %edi -= 4 3913 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx 3914 jne .word_not_equal / if not equal, goto .word_not_equal 3915 leal 4(%ecx), %ecx / %ecx += 4 (next word) 3916 leal 4(%eax), %eax / %eax += 4 (next word) 3917 cmpl $4, %edi / if %edi >= 4 3918 jae .word_loop / goto .word_loop 3919.byte_check: 3920 cmpl $0, %edi / if %edi == 0 3921 je .equal / goto .equal 3922 jmp .byte_loop / goto .byte_loop (checks in bytes) 3923.word_not_equal: 3924 leal 4(%edi), %edi / %edi += 4 (post-decremented) 3925 .align 4 3926.byte_loop: 3927 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl 3928 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax) 3929 jne .not_equal / if not equal, goto .not_equal 3930 incl %ecx / %ecx++ (next byte) 3931 incl %eax / %eax++ (next byte) 3932 decl %edi / %edi-- 3933 jnz .byte_loop / if not zero, goto .byte_loop 3934.equal: 3935 xorl %eax, %eax / %eax = 0 3936 popl %edi / restore register variable 3937 leave / restore old stack frame 3938 ret / return (NULL) 3939 .align 4 3940.not_equal: 3941 movl $1, %eax / return 1 3942 popl %edi / restore register variable 3943 leave / restore old stack frame 3944 ret / return (NULL) 3945 SET_SIZE(bcmp) 3946 3947#endif /* __i386 */ 3948 3949#ifdef DEBUG 3950 .text 3951.bcmp_panic_msg: 3952 .string "bcmp: arguments below kernelbase" 3953#endif /* DEBUG */ 3954 3955#endif /* __lint */ 3956 3957#if defined(__lint) 3958 3959uint_t 3960bsrw_insn(uint16_t mask) 3961{ 3962 uint_t index = sizeof (mask) * NBBY - 1; 3963 3964 while ((mask & (1 << index)) == 0) 3965 index--; 3966 return (index); 3967} 3968 3969#else /* __lint */ 3970 3971#if defined(__amd64) 3972 3973 ENTRY_NP(bsrw_insn) 3974 xorl %eax, %eax 3975 bsrw %di, %ax 3976 ret 3977 SET_SIZE(bsrw_insn) 3978 3979#elif defined(__i386) 3980 3981 ENTRY_NP(bsrw_insn) 3982 movw 4(%esp), %cx 3983 xorl %eax, %eax 3984 bsrw %cx, %ax 3985 ret 3986 SET_SIZE(bsrw_insn) 3987 3988#endif /* __i386 */ 3989#endif /* __lint */ 3990 3991#if defined(__lint) 3992 3993uint_t 3994atomic_btr32(uint32_t *pending, uint_t pil) 3995{ 3996 return (*pending &= ~(1 << pil)); 3997} 3998 3999#else /* __lint */ 4000 4001#if defined(__i386) 4002 4003 ENTRY_NP(atomic_btr32) 4004 movl 4(%esp), %ecx 4005 movl 8(%esp), %edx 4006 xorl %eax, %eax 4007 lock 4008 btrl %edx, (%ecx) 4009 setc %al 4010 ret 4011 SET_SIZE(atomic_btr32) 4012 4013#endif /* __i386 */ 4014#endif /* __lint */ 4015 4016#if defined(__lint) 4017 4018/*ARGSUSED*/ 4019void 4020switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1, 4021 uint_t arg2) 4022{} 4023 4024#else /* __lint */ 4025 4026#if defined(__amd64) 4027 4028 ENTRY_NP(switch_sp_and_call) 4029 pushq %rbp 4030 movq %rsp, %rbp /* set up stack frame */ 4031 movq %rdi, %rsp /* switch stack pointer */ 4032 movq %rdx, %rdi /* pass func arg 1 */ 4033 movq %rsi, %r11 /* save function to call */ 4034 movq %rcx, %rsi /* pass func arg 2 */ 4035 call *%r11 /* call function */ 4036 leave /* restore stack */ 4037 ret 4038 SET_SIZE(switch_sp_and_call) 4039 4040#elif defined(__i386) 4041 4042 ENTRY_NP(switch_sp_and_call) 4043 pushl %ebp 4044 mov %esp, %ebp /* set up stack frame */ 4045 movl 8(%ebp), %esp /* switch stack pointer */ 4046 pushl 20(%ebp) /* push func arg 2 */ 4047 pushl 16(%ebp) /* push func arg 1 */ 4048 call *12(%ebp) /* call function */ 4049 addl $8, %esp /* pop arguments */ 4050 leave /* restore stack */ 4051 ret 4052 SET_SIZE(switch_sp_and_call) 4053 4054#endif /* __i386 */ 4055#endif /* __lint */ 4056 4057#if defined(__lint) 4058 4059void 4060kmdb_enter(void) 4061{} 4062 4063#else /* __lint */ 4064 4065#if defined(__amd64) 4066 4067 ENTRY_NP(kmdb_enter) 4068 pushq %rbp 4069 movq %rsp, %rbp 4070 4071 /* 4072 * Save flags, do a 'cli' then return the saved flags 4073 */ 4074 call intr_clear 4075 4076 int $T_DBGENTR 4077 4078 /* 4079 * Restore the saved flags 4080 */ 4081 movq %rax, %rdi 4082 call intr_restore 4083 4084 leave 4085 ret 4086 SET_SIZE(kmdb_enter) 4087 4088#elif defined(__i386) 4089 4090 ENTRY_NP(kmdb_enter) 4091 pushl %ebp 4092 movl %esp, %ebp 4093 4094 /* 4095 * Save flags, do a 'cli' then return the saved flags 4096 */ 4097 call intr_clear 4098 4099 int $T_DBGENTR 4100 4101 /* 4102 * Restore the saved flags 4103 */ 4104 pushl %eax 4105 call intr_restore 4106 addl $4, %esp 4107 4108 leave 4109 ret 4110 SET_SIZE(kmdb_enter) 4111 4112#endif /* __i386 */ 4113#endif /* __lint */ 4114 4115#if defined(__lint) 4116 4117void 4118return_instr(void) 4119{} 4120 4121#else /* __lint */ 4122 4123 ENTRY_NP(return_instr) 4124 rep; ret /* use 2 byte instruction when branch target */ 4125 /* AMD Software Optimization Guide - Section 6.2 */ 4126 SET_SIZE(return_instr) 4127 4128#endif /* __lint */ 4129 4130#if defined(__lint) 4131 4132ulong_t 4133getflags(void) 4134{ 4135 return (0); 4136} 4137 4138#else /* __lint */ 4139 4140#if defined(__amd64) 4141 4142 ENTRY(getflags) 4143 pushfq 4144 popq %rax 4145#if defined(__xpv) 4146 CURTHREAD(%rdi) 4147 KPREEMPT_DISABLE(%rdi) 4148 /* 4149 * Synthesize the PS_IE bit from the event mask bit 4150 */ 4151 CURVCPU(%r11) 4152 andq $_BITNOT(PS_IE), %rax 4153 XEN_TEST_UPCALL_MASK(%r11) 4154 jnz 1f 4155 orq $PS_IE, %rax 41561: 4157 KPREEMPT_ENABLE_NOKP(%rdi) 4158#endif 4159 ret 4160 SET_SIZE(getflags) 4161 4162#elif defined(__i386) 4163 4164 ENTRY(getflags) 4165 pushfl 4166 popl %eax 4167#if defined(__xpv) 4168 CURTHREAD(%ecx) 4169 KPREEMPT_DISABLE(%ecx) 4170 /* 4171 * Synthesize the PS_IE bit from the event mask bit 4172 */ 4173 CURVCPU(%edx) 4174 andl $_BITNOT(PS_IE), %eax 4175 XEN_TEST_UPCALL_MASK(%edx) 4176 jnz 1f 4177 orl $PS_IE, %eax 41781: 4179 KPREEMPT_ENABLE_NOKP(%ecx) 4180#endif 4181 ret 4182 SET_SIZE(getflags) 4183 4184#endif /* __i386 */ 4185 4186#endif /* __lint */ 4187 4188#if defined(__lint) 4189 4190ftrace_icookie_t 4191ftrace_interrupt_disable(void) 4192{ return (0); } 4193 4194#else /* __lint */ 4195 4196#if defined(__amd64) 4197 4198 ENTRY(ftrace_interrupt_disable) 4199 pushfq 4200 popq %rax 4201 CLI(%rdx) 4202 ret 4203 SET_SIZE(ftrace_interrupt_disable) 4204 4205#elif defined(__i386) 4206 4207 ENTRY(ftrace_interrupt_disable) 4208 pushfl 4209 popl %eax 4210 CLI(%edx) 4211 ret 4212 SET_SIZE(ftrace_interrupt_disable) 4213 4214#endif /* __i386 */ 4215#endif /* __lint */ 4216 4217#if defined(__lint) 4218 4219/*ARGSUSED*/ 4220void 4221ftrace_interrupt_enable(ftrace_icookie_t cookie) 4222{} 4223 4224#else /* __lint */ 4225 4226#if defined(__amd64) 4227 4228 ENTRY(ftrace_interrupt_enable) 4229 pushq %rdi 4230 popfq 4231 ret 4232 SET_SIZE(ftrace_interrupt_enable) 4233 4234#elif defined(__i386) 4235 4236 ENTRY(ftrace_interrupt_enable) 4237 movl 4(%esp), %eax 4238 pushl %eax 4239 popfl 4240 ret 4241 SET_SIZE(ftrace_interrupt_enable) 4242 4243#endif /* __i386 */ 4244#endif /* __lint */ 4245 4246#if defined (__lint) 4247 4248/*ARGSUSED*/ 4249void 4250iommu_cpu_nop(void) 4251{} 4252 4253#else /* __lint */ 4254 4255 ENTRY(iommu_cpu_nop) 4256 rep; nop 4257 ret 4258 SET_SIZE(iommu_cpu_nop) 4259 4260#endif /* __lint */ 4261 4262#if defined (__lint) 4263 4264/*ARGSUSED*/ 4265void 4266clflush_insn(caddr_t addr) 4267{} 4268 4269#else /* __lint */ 4270 4271#if defined (__amd64) 4272 ENTRY(clflush_insn) 4273 clflush (%rdi) 4274 ret 4275 SET_SIZE(clflush_insn) 4276#elif defined (__i386) 4277 ENTRY(clflush_insn) 4278 movl 4(%esp), %eax 4279 clflush (%eax) 4280 ret 4281 SET_SIZE(clflush_insn) 4282 4283#endif /* __i386 */ 4284#endif /* __lint */ 4285 4286#if defined (__lint) 4287/*ARGSUSED*/ 4288void 4289mfence_insn(void) 4290{} 4291 4292#else /* __lint */ 4293 4294#if defined (__amd64) 4295 ENTRY(mfence_insn) 4296 mfence 4297 ret 4298 SET_SIZE(mfence_insn) 4299#elif defined (__i386) 4300 ENTRY(mfence_insn) 4301 mfence 4302 ret 4303 SET_SIZE(mfence_insn) 4304 4305#endif /* __i386 */ 4306#endif /* __lint */ 4307 4308/* 4309 * This is how VMware lets the guests figure that they are running 4310 * on top of VMWare platform : 4311 * Write 0xA in the ECX register and put the I/O port address value of 4312 * 0x564D5868 in the EAX register. Then read a word from port 0x5658. 4313 * If VMWare is installed than this code will be executed correctly and 4314 * the EBX register will contain the same I/O port address value of 0x564D5868. 4315 * If VMWare is not installed then OS will return an exception on port access. 4316 */ 4317#if defined(__lint) 4318 4319int 4320vmware_platform(void) { return (1); } 4321 4322#else 4323 4324#if defined(__amd64) 4325 4326 ENTRY(vmware_platform) 4327 pushq %rbx 4328 xorl %ebx, %ebx 4329 movl $0x564d5868, %eax 4330 movl $0xa, %ecx 4331 movl $0x5658, %edx 4332 inl (%dx) 4333 movl $0x564d5868, %ecx 4334 xorl %eax, %eax 4335 cmpl %ecx, %ebx 4336 jne 1f 4337 incl %eax 43381: 4339 popq %rbx 4340 ret 4341 SET_SIZE(vmware_platform) 4342 4343#elif defined(__i386) 4344 4345 ENTRY(vmware_platform) 4346 pushl %ebx 4347 pushl %ecx 4348 pushl %edx 4349 xorl %ebx, %ebx 4350 movl $0x564d5868, %eax 4351 movl $0xa, %ecx 4352 movl $0x5658, %edx 4353 inl (%dx) 4354 movl $0x564d5868, %ecx 4355 xorl %eax, %eax 4356 cmpl %ecx, %ebx 4357 jne 1f 4358 incl %eax 43591: 4360 popl %edx 4361 popl %ecx 4362 popl %ebx 4363 ret 4364 SET_SIZE(vmware_platform) 4365 4366#endif /* __i386 */ 4367#endif /* __lint */ 4368