1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27/* 28 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. 29 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T 30 * All Rights Reserved 31 */ 32 33/* 34 * General assembly language routines. 35 * It is the intent of this file to contain routines that are 36 * independent of the specific kernel architecture, and those that are 37 * common across kernel architectures. 38 * As architectures diverge, and implementations of specific 39 * architecture-dependent routines change, the routines should be moved 40 * from this file into the respective ../`arch -k`/subr.s file. 41 */ 42 43#include <sys/asm_linkage.h> 44#include <sys/asm_misc.h> 45#include <sys/panic.h> 46#include <sys/ontrap.h> 47#include <sys/regset.h> 48#include <sys/privregs.h> 49#include <sys/reboot.h> 50#include <sys/psw.h> 51#include <sys/x86_archext.h> 52 53#if defined(__lint) 54#include <sys/types.h> 55#include <sys/systm.h> 56#include <sys/thread.h> 57#include <sys/archsystm.h> 58#include <sys/byteorder.h> 59#include <sys/dtrace.h> 60#include <sys/ftrace.h> 61#else /* __lint */ 62#include "assym.h" 63#endif /* __lint */ 64#include <sys/dditypes.h> 65 66/* 67 * on_fault() 68 * Catch lofault faults. Like setjmp except it returns one 69 * if code following causes uncorrectable fault. Turned off 70 * by calling no_fault(). 71 */ 72 73#if defined(__lint) 74 75/* ARGSUSED */ 76int 77on_fault(label_t *ljb) 78{ return (0); } 79 80void 81no_fault(void) 82{} 83 84#else /* __lint */ 85 86#if defined(__amd64) 87 88 ENTRY(on_fault) 89 movq %gs:CPU_THREAD, %rsi 90 leaq catch_fault(%rip), %rdx 91 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */ 92 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */ 93 jmp setjmp /* let setjmp do the rest */ 94 95catch_fault: 96 movq %gs:CPU_THREAD, %rsi 97 movq T_ONFAULT(%rsi), %rdi /* address of save area */ 98 xorl %eax, %eax 99 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 100 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 101 jmp longjmp /* let longjmp do the rest */ 102 SET_SIZE(on_fault) 103 104 ENTRY(no_fault) 105 movq %gs:CPU_THREAD, %rsi 106 xorl %eax, %eax 107 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 108 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 109 ret 110 SET_SIZE(no_fault) 111 112#elif defined(__i386) 113 114 ENTRY(on_fault) 115 movl %gs:CPU_THREAD, %edx 116 movl 4(%esp), %eax /* jumpbuf address */ 117 leal catch_fault, %ecx 118 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */ 119 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */ 120 jmp setjmp /* let setjmp do the rest */ 121 122catch_fault: 123 movl %gs:CPU_THREAD, %edx 124 xorl %eax, %eax 125 movl T_ONFAULT(%edx), %ecx /* address of save area */ 126 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 127 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 128 pushl %ecx 129 call longjmp /* let longjmp do the rest */ 130 SET_SIZE(on_fault) 131 132 ENTRY(no_fault) 133 movl %gs:CPU_THREAD, %edx 134 xorl %eax, %eax 135 movl %eax, T_ONFAULT(%edx) /* turn off onfault */ 136 movl %eax, T_LOFAULT(%edx) /* turn off lofault */ 137 ret 138 SET_SIZE(no_fault) 139 140#endif /* __i386 */ 141#endif /* __lint */ 142 143/* 144 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just 145 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called. 146 */ 147 148#if defined(lint) 149 150void 151on_trap_trampoline(void) 152{} 153 154#else /* __lint */ 155 156#if defined(__amd64) 157 158 ENTRY(on_trap_trampoline) 159 movq %gs:CPU_THREAD, %rsi 160 movq T_ONTRAP(%rsi), %rdi 161 addq $OT_JMPBUF, %rdi 162 jmp longjmp 163 SET_SIZE(on_trap_trampoline) 164 165#elif defined(__i386) 166 167 ENTRY(on_trap_trampoline) 168 movl %gs:CPU_THREAD, %eax 169 movl T_ONTRAP(%eax), %eax 170 addl $OT_JMPBUF, %eax 171 pushl %eax 172 call longjmp 173 SET_SIZE(on_trap_trampoline) 174 175#endif /* __i386 */ 176#endif /* __lint */ 177 178/* 179 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 180 * more information about the on_trap() mechanism. If the on_trap_data is the 181 * same as the topmost stack element, we just modify that element. 182 */ 183#if defined(lint) 184 185/*ARGSUSED*/ 186int 187on_trap(on_trap_data_t *otp, uint_t prot) 188{ return (0); } 189 190#else /* __lint */ 191 192#if defined(__amd64) 193 194 ENTRY(on_trap) 195 movw %si, OT_PROT(%rdi) /* ot_prot = prot */ 196 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */ 197 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */ 198 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */ 199 xorl %ecx, %ecx 200 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */ 201 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */ 202 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */ 203 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */ 204 cmpq %rdi, %rcx /* if (otp == %rcx) */ 205 je 0f /* don't modify t_ontrap */ 206 207 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */ 208 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */ 209 2100: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */ 211 jmp setjmp 212 SET_SIZE(on_trap) 213 214#elif defined(__i386) 215 216 ENTRY(on_trap) 217 movl 4(%esp), %eax /* %eax = otp */ 218 movl 8(%esp), %edx /* %edx = prot */ 219 220 movw %dx, OT_PROT(%eax) /* ot_prot = prot */ 221 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */ 222 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */ 223 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */ 224 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */ 225 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */ 226 movl %gs:CPU_THREAD, %edx /* %edx = curthread */ 227 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */ 228 cmpl %eax, %ecx /* if (otp == %ecx) */ 229 je 0f /* don't modify t_ontrap */ 230 231 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */ 232 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */ 233 2340: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */ 235 movl %eax, 4(%esp) /* put %eax back on the stack */ 236 jmp setjmp /* let setjmp do the rest */ 237 SET_SIZE(on_trap) 238 239#endif /* __i386 */ 240#endif /* __lint */ 241 242/* 243 * Setjmp and longjmp implement non-local gotos using state vectors 244 * type label_t. 245 */ 246 247#if defined(__lint) 248 249/* ARGSUSED */ 250int 251setjmp(label_t *lp) 252{ return (0); } 253 254/* ARGSUSED */ 255void 256longjmp(label_t *lp) 257{} 258 259#else /* __lint */ 260 261#if LABEL_PC != 0 262#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded 263#endif /* LABEL_PC != 0 */ 264 265#if defined(__amd64) 266 267 ENTRY(setjmp) 268 movq %rsp, LABEL_SP(%rdi) 269 movq %rbp, LABEL_RBP(%rdi) 270 movq %rbx, LABEL_RBX(%rdi) 271 movq %r12, LABEL_R12(%rdi) 272 movq %r13, LABEL_R13(%rdi) 273 movq %r14, LABEL_R14(%rdi) 274 movq %r15, LABEL_R15(%rdi) 275 movq (%rsp), %rdx /* return address */ 276 movq %rdx, (%rdi) /* LABEL_PC is 0 */ 277 xorl %eax, %eax /* return 0 */ 278 ret 279 SET_SIZE(setjmp) 280 281 ENTRY(longjmp) 282 movq LABEL_SP(%rdi), %rsp 283 movq LABEL_RBP(%rdi), %rbp 284 movq LABEL_RBX(%rdi), %rbx 285 movq LABEL_R12(%rdi), %r12 286 movq LABEL_R13(%rdi), %r13 287 movq LABEL_R14(%rdi), %r14 288 movq LABEL_R15(%rdi), %r15 289 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */ 290 movq %rdx, (%rsp) 291 xorl %eax, %eax 292 incl %eax /* return 1 */ 293 ret 294 SET_SIZE(longjmp) 295 296#elif defined(__i386) 297 298 ENTRY(setjmp) 299 movl 4(%esp), %edx /* address of save area */ 300 movl %ebp, LABEL_EBP(%edx) 301 movl %ebx, LABEL_EBX(%edx) 302 movl %esi, LABEL_ESI(%edx) 303 movl %edi, LABEL_EDI(%edx) 304 movl %esp, 4(%edx) 305 movl (%esp), %ecx /* %eip (return address) */ 306 movl %ecx, (%edx) /* LABEL_PC is 0 */ 307 subl %eax, %eax /* return 0 */ 308 ret 309 SET_SIZE(setjmp) 310 311 ENTRY(longjmp) 312 movl 4(%esp), %edx /* address of save area */ 313 movl LABEL_EBP(%edx), %ebp 314 movl LABEL_EBX(%edx), %ebx 315 movl LABEL_ESI(%edx), %esi 316 movl LABEL_EDI(%edx), %edi 317 movl 4(%edx), %esp 318 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */ 319 movl $1, %eax 320 addl $4, %esp /* pop ret adr */ 321 jmp *%ecx /* indirect */ 322 SET_SIZE(longjmp) 323 324#endif /* __i386 */ 325#endif /* __lint */ 326 327/* 328 * if a() calls b() calls caller(), 329 * caller() returns return address in a(). 330 * (Note: We assume a() and b() are C routines which do the normal entry/exit 331 * sequence.) 332 */ 333 334#if defined(__lint) 335 336caddr_t 337caller(void) 338{ return (0); } 339 340#else /* __lint */ 341 342#if defined(__amd64) 343 344 ENTRY(caller) 345 movq 8(%rbp), %rax /* b()'s return pc, in a() */ 346 ret 347 SET_SIZE(caller) 348 349#elif defined(__i386) 350 351 ENTRY(caller) 352 movl 4(%ebp), %eax /* b()'s return pc, in a() */ 353 ret 354 SET_SIZE(caller) 355 356#endif /* __i386 */ 357#endif /* __lint */ 358 359/* 360 * if a() calls callee(), callee() returns the 361 * return address in a(); 362 */ 363 364#if defined(__lint) 365 366caddr_t 367callee(void) 368{ return (0); } 369 370#else /* __lint */ 371 372#if defined(__amd64) 373 374 ENTRY(callee) 375 movq (%rsp), %rax /* callee()'s return pc, in a() */ 376 ret 377 SET_SIZE(callee) 378 379#elif defined(__i386) 380 381 ENTRY(callee) 382 movl (%esp), %eax /* callee()'s return pc, in a() */ 383 ret 384 SET_SIZE(callee) 385 386#endif /* __i386 */ 387#endif /* __lint */ 388 389/* 390 * return the current frame pointer 391 */ 392 393#if defined(__lint) 394 395greg_t 396getfp(void) 397{ return (0); } 398 399#else /* __lint */ 400 401#if defined(__amd64) 402 403 ENTRY(getfp) 404 movq %rbp, %rax 405 ret 406 SET_SIZE(getfp) 407 408#elif defined(__i386) 409 410 ENTRY(getfp) 411 movl %ebp, %eax 412 ret 413 SET_SIZE(getfp) 414 415#endif /* __i386 */ 416#endif /* __lint */ 417 418/* 419 * Invalidate a single page table entry in the TLB 420 */ 421 422#if defined(__lint) 423 424/* ARGSUSED */ 425void 426mmu_tlbflush_entry(caddr_t m) 427{} 428 429#else /* __lint */ 430 431#if defined(__amd64) 432 433 ENTRY(mmu_tlbflush_entry) 434 invlpg (%rdi) 435 ret 436 SET_SIZE(mmu_tlbflush_entry) 437 438#elif defined(__i386) 439 440 ENTRY(mmu_tlbflush_entry) 441 movl 4(%esp), %eax 442 invlpg (%eax) 443 ret 444 SET_SIZE(mmu_tlbflush_entry) 445 446#endif /* __i386 */ 447#endif /* __lint */ 448 449 450/* 451 * Get/Set the value of various control registers 452 */ 453 454#if defined(__lint) 455 456ulong_t 457getcr0(void) 458{ return (0); } 459 460/* ARGSUSED */ 461void 462setcr0(ulong_t value) 463{} 464 465ulong_t 466getcr2(void) 467{ return (0); } 468 469ulong_t 470getcr3(void) 471{ return (0); } 472 473#if !defined(__xpv) 474/* ARGSUSED */ 475void 476setcr3(ulong_t val) 477{} 478 479void 480reload_cr3(void) 481{} 482#endif 483 484ulong_t 485getcr4(void) 486{ return (0); } 487 488/* ARGSUSED */ 489void 490setcr4(ulong_t val) 491{} 492 493#if defined(__amd64) 494 495ulong_t 496getcr8(void) 497{ return (0); } 498 499/* ARGSUSED */ 500void 501setcr8(ulong_t val) 502{} 503 504#endif /* __amd64 */ 505 506#else /* __lint */ 507 508#if defined(__amd64) 509 510 ENTRY(getcr0) 511 movq %cr0, %rax 512 ret 513 SET_SIZE(getcr0) 514 515 ENTRY(setcr0) 516 movq %rdi, %cr0 517 ret 518 SET_SIZE(setcr0) 519 520 ENTRY(getcr2) 521#if defined(__xpv) 522 movq %gs:CPU_VCPU_INFO, %rax 523 movq VCPU_INFO_ARCH_CR2(%rax), %rax 524#else 525 movq %cr2, %rax 526#endif 527 ret 528 SET_SIZE(getcr2) 529 530 ENTRY(getcr3) 531 movq %cr3, %rax 532 ret 533 SET_SIZE(getcr3) 534 535#if !defined(__xpv) 536 537 ENTRY(setcr3) 538 movq %rdi, %cr3 539 ret 540 SET_SIZE(setcr3) 541 542 ENTRY(reload_cr3) 543 movq %cr3, %rdi 544 movq %rdi, %cr3 545 ret 546 SET_SIZE(reload_cr3) 547 548#endif /* __xpv */ 549 550 ENTRY(getcr4) 551 movq %cr4, %rax 552 ret 553 SET_SIZE(getcr4) 554 555 ENTRY(setcr4) 556 movq %rdi, %cr4 557 ret 558 SET_SIZE(setcr4) 559 560 ENTRY(getcr8) 561 movq %cr8, %rax 562 ret 563 SET_SIZE(getcr8) 564 565 ENTRY(setcr8) 566 movq %rdi, %cr8 567 ret 568 SET_SIZE(setcr8) 569 570#elif defined(__i386) 571 572 ENTRY(getcr0) 573 movl %cr0, %eax 574 ret 575 SET_SIZE(getcr0) 576 577 ENTRY(setcr0) 578 movl 4(%esp), %eax 579 movl %eax, %cr0 580 ret 581 SET_SIZE(setcr0) 582 583 ENTRY(getcr2) 584#if defined(__xpv) 585 movl %gs:CPU_VCPU_INFO, %eax 586 movl VCPU_INFO_ARCH_CR2(%eax), %eax 587#else 588 movl %cr2, %eax 589#endif 590 ret 591 SET_SIZE(getcr2) 592 593 ENTRY(getcr3) 594 movl %cr3, %eax 595 ret 596 SET_SIZE(getcr3) 597 598#if !defined(__xpv) 599 600 ENTRY(setcr3) 601 movl 4(%esp), %eax 602 movl %eax, %cr3 603 ret 604 SET_SIZE(setcr3) 605 606 ENTRY(reload_cr3) 607 movl %cr3, %eax 608 movl %eax, %cr3 609 ret 610 SET_SIZE(reload_cr3) 611 612#endif /* __xpv */ 613 614 ENTRY(getcr4) 615 movl %cr4, %eax 616 ret 617 SET_SIZE(getcr4) 618 619 ENTRY(setcr4) 620 movl 4(%esp), %eax 621 movl %eax, %cr4 622 ret 623 SET_SIZE(setcr4) 624 625#endif /* __i386 */ 626#endif /* __lint */ 627 628#if defined(__lint) 629 630/*ARGSUSED*/ 631uint32_t 632__cpuid_insn(struct cpuid_regs *regs) 633{ return (0); } 634 635#else /* __lint */ 636 637#if defined(__amd64) 638 639 ENTRY(__cpuid_insn) 640 movq %rbx, %r8 641 movq %rcx, %r9 642 movq %rdx, %r11 643 movl (%rdi), %eax /* %eax = regs->cp_eax */ 644 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */ 645 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */ 646 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */ 647 cpuid 648 movl %eax, (%rdi) /* regs->cp_eax = %eax */ 649 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */ 650 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */ 651 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */ 652 movq %r8, %rbx 653 movq %r9, %rcx 654 movq %r11, %rdx 655 ret 656 SET_SIZE(__cpuid_insn) 657 658#elif defined(__i386) 659 660 ENTRY(__cpuid_insn) 661 pushl %ebp 662 movl 0x8(%esp), %ebp /* %ebp = regs */ 663 pushl %ebx 664 pushl %ecx 665 pushl %edx 666 movl (%ebp), %eax /* %eax = regs->cp_eax */ 667 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */ 668 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */ 669 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */ 670 cpuid 671 movl %eax, (%ebp) /* regs->cp_eax = %eax */ 672 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */ 673 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */ 674 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */ 675 popl %edx 676 popl %ecx 677 popl %ebx 678 popl %ebp 679 ret 680 SET_SIZE(__cpuid_insn) 681 682#endif /* __i386 */ 683#endif /* __lint */ 684 685#if defined(__xpv) 686 /* 687 * Defined in C 688 */ 689#else 690 691#if defined(__lint) 692 693/*ARGSUSED*/ 694void 695i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints) 696{ return; } 697 698#else /* __lint */ 699 700#if defined(__amd64) 701 702 ENTRY_NP(i86_monitor) 703 pushq %rbp 704 movq %rsp, %rbp 705 movq %rdi, %rax /* addr */ 706 movq %rsi, %rcx /* extensions */ 707 /* rdx contains input arg3: hints */ 708 .byte 0x0f, 0x01, 0xc8 /* monitor */ 709 leave 710 ret 711 SET_SIZE(i86_monitor) 712 713#elif defined(__i386) 714 715ENTRY_NP(i86_monitor) 716 pushl %ebp 717 movl %esp, %ebp 718 movl 0x8(%ebp),%eax /* addr */ 719 movl 0xc(%ebp),%ecx /* extensions */ 720 movl 0x10(%ebp),%edx /* hints */ 721 .byte 0x0f, 0x01, 0xc8 /* monitor */ 722 leave 723 ret 724 SET_SIZE(i86_monitor) 725 726#endif /* __i386 */ 727#endif /* __lint */ 728 729#if defined(__lint) 730 731/*ARGSUSED*/ 732void 733i86_mwait(uint32_t data, uint32_t extensions) 734{ return; } 735 736#else /* __lint */ 737 738#if defined(__amd64) 739 740 ENTRY_NP(i86_mwait) 741 pushq %rbp 742 movq %rsp, %rbp 743 movq %rdi, %rax /* data */ 744 movq %rsi, %rcx /* extensions */ 745 .byte 0x0f, 0x01, 0xc9 /* mwait */ 746 leave 747 ret 748 SET_SIZE(i86_mwait) 749 750#elif defined(__i386) 751 752 ENTRY_NP(i86_mwait) 753 pushl %ebp 754 movl %esp, %ebp 755 movl 0x8(%ebp),%eax /* data */ 756 movl 0xc(%ebp),%ecx /* extensions */ 757 .byte 0x0f, 0x01, 0xc9 /* mwait */ 758 leave 759 ret 760 SET_SIZE(i86_mwait) 761 762#endif /* __i386 */ 763#endif /* __lint */ 764 765#if defined(__lint) 766 767hrtime_t 768tsc_read(void) 769{ 770 return (0); 771} 772 773#else /* __lint */ 774 775#if defined(__amd64) 776 777 ENTRY_NP(tsc_read) 778 movq %rbx, %r11 779 movl $0, %eax 780 cpuid 781 rdtsc 782 movq %r11, %rbx 783 shlq $32, %rdx 784 orq %rdx, %rax 785 ret 786 .globl _tsc_mfence_start 787_tsc_mfence_start: 788 mfence 789 rdtsc 790 shlq $32, %rdx 791 orq %rdx, %rax 792 ret 793 .globl _tsc_mfence_end 794_tsc_mfence_end: 795 .globl _tscp_start 796_tscp_start: 797 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 798 shlq $32, %rdx 799 orq %rdx, %rax 800 ret 801 .globl _tscp_end 802_tscp_end: 803 .globl _no_rdtsc_start 804_no_rdtsc_start: 805 xorl %edx, %edx 806 xorl %eax, %eax 807 ret 808 .globl _no_rdtsc_end 809_no_rdtsc_end: 810 .globl _tsc_lfence_start 811_tsc_lfence_start: 812 lfence 813 rdtsc 814 shlq $32, %rdx 815 orq %rdx, %rax 816 ret 817 .globl _tsc_lfence_end 818_tsc_lfence_end: 819 SET_SIZE(tsc_read) 820 821#else /* __i386 */ 822 823 ENTRY_NP(tsc_read) 824 pushl %ebx 825 movl $0, %eax 826 cpuid 827 rdtsc 828 popl %ebx 829 ret 830 .globl _tsc_mfence_start 831_tsc_mfence_start: 832 mfence 833 rdtsc 834 ret 835 .globl _tsc_mfence_end 836_tsc_mfence_end: 837 .globl _tscp_start 838_tscp_start: 839 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 840 ret 841 .globl _tscp_end 842_tscp_end: 843 .globl _no_rdtsc_start 844_no_rdtsc_start: 845 xorl %edx, %edx 846 xorl %eax, %eax 847 ret 848 .globl _no_rdtsc_end 849_no_rdtsc_end: 850 .globl _tsc_lfence_start 851_tsc_lfence_start: 852 lfence 853 rdtsc 854 ret 855 .globl _tsc_lfence_end 856_tsc_lfence_end: 857 SET_SIZE(tsc_read) 858 859#endif /* __i386 */ 860 861#endif /* __lint */ 862 863 864#endif /* __xpv */ 865 866#ifdef __lint 867/* 868 * Do not use this function for obtaining clock tick. This 869 * is called by callers who do not need to have a guarenteed 870 * correct tick value. The proper routine to use is tsc_read(). 871 */ 872hrtime_t 873randtick(void) 874{ 875 return (0); 876} 877#else 878#if defined(__amd64) 879 ENTRY_NP(randtick) 880 rdtsc 881 shlq $32, %rdx 882 orq %rdx, %rax 883 ret 884 SET_SIZE(randtick) 885#else 886 ENTRY_NP(randtick) 887 rdtsc 888 ret 889 SET_SIZE(randtick) 890#endif /* __i386 */ 891#endif /* __lint */ 892/* 893 * Insert entryp after predp in a doubly linked list. 894 */ 895 896#if defined(__lint) 897 898/*ARGSUSED*/ 899void 900_insque(caddr_t entryp, caddr_t predp) 901{} 902 903#else /* __lint */ 904 905#if defined(__amd64) 906 907 ENTRY(_insque) 908 movq (%rsi), %rax /* predp->forw */ 909 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */ 910 movq %rax, (%rdi) /* entryp->forw = predp->forw */ 911 movq %rdi, (%rsi) /* predp->forw = entryp */ 912 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */ 913 ret 914 SET_SIZE(_insque) 915 916#elif defined(__i386) 917 918 ENTRY(_insque) 919 movl 8(%esp), %edx 920 movl 4(%esp), %ecx 921 movl (%edx), %eax /* predp->forw */ 922 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */ 923 movl %eax, (%ecx) /* entryp->forw = predp->forw */ 924 movl %ecx, (%edx) /* predp->forw = entryp */ 925 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */ 926 ret 927 SET_SIZE(_insque) 928 929#endif /* __i386 */ 930#endif /* __lint */ 931 932/* 933 * Remove entryp from a doubly linked list 934 */ 935 936#if defined(__lint) 937 938/*ARGSUSED*/ 939void 940_remque(caddr_t entryp) 941{} 942 943#else /* __lint */ 944 945#if defined(__amd64) 946 947 ENTRY(_remque) 948 movq (%rdi), %rax /* entry->forw */ 949 movq CPTRSIZE(%rdi), %rdx /* entry->back */ 950 movq %rax, (%rdx) /* entry->back->forw = entry->forw */ 951 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */ 952 ret 953 SET_SIZE(_remque) 954 955#elif defined(__i386) 956 957 ENTRY(_remque) 958 movl 4(%esp), %ecx 959 movl (%ecx), %eax /* entry->forw */ 960 movl CPTRSIZE(%ecx), %edx /* entry->back */ 961 movl %eax, (%edx) /* entry->back->forw = entry->forw */ 962 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */ 963 ret 964 SET_SIZE(_remque) 965 966#endif /* __i386 */ 967#endif /* __lint */ 968 969/* 970 * Returns the number of 971 * non-NULL bytes in string argument. 972 */ 973 974#if defined(__lint) 975 976/* ARGSUSED */ 977size_t 978strlen(const char *str) 979{ return (0); } 980 981#else /* __lint */ 982 983#if defined(__amd64) 984 985/* 986 * This is close to a simple transliteration of a C version of this 987 * routine. We should either just -make- this be a C version, or 988 * justify having it in assembler by making it significantly faster. 989 * 990 * size_t 991 * strlen(const char *s) 992 * { 993 * const char *s0; 994 * #if defined(DEBUG) 995 * if ((uintptr_t)s < KERNELBASE) 996 * panic(.str_panic_msg); 997 * #endif 998 * for (s0 = s; *s; s++) 999 * ; 1000 * return (s - s0); 1001 * } 1002 */ 1003 1004 ENTRY(strlen) 1005#ifdef DEBUG 1006 movq postbootkernelbase(%rip), %rax 1007 cmpq %rax, %rdi 1008 jae str_valid 1009 pushq %rbp 1010 movq %rsp, %rbp 1011 leaq .str_panic_msg(%rip), %rdi 1012 xorl %eax, %eax 1013 call panic 1014#endif /* DEBUG */ 1015str_valid: 1016 cmpb $0, (%rdi) 1017 movq %rdi, %rax 1018 je .null_found 1019 .align 4 1020.strlen_loop: 1021 incq %rdi 1022 cmpb $0, (%rdi) 1023 jne .strlen_loop 1024.null_found: 1025 subq %rax, %rdi 1026 movq %rdi, %rax 1027 ret 1028 SET_SIZE(strlen) 1029 1030#elif defined(__i386) 1031 1032 ENTRY(strlen) 1033#ifdef DEBUG 1034 movl postbootkernelbase, %eax 1035 cmpl %eax, 4(%esp) 1036 jae str_valid 1037 pushl %ebp 1038 movl %esp, %ebp 1039 pushl $.str_panic_msg 1040 call panic 1041#endif /* DEBUG */ 1042 1043str_valid: 1044 movl 4(%esp), %eax /* %eax = string address */ 1045 testl $3, %eax /* if %eax not word aligned */ 1046 jnz .not_word_aligned /* goto .not_word_aligned */ 1047 .align 4 1048.word_aligned: 1049 movl (%eax), %edx /* move 1 word from (%eax) to %edx */ 1050 movl $0x7f7f7f7f, %ecx 1051 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */ 1052 addl $4, %eax /* next word */ 1053 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */ 1054 orl %edx, %ecx /* %ecx |= %edx */ 1055 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */ 1056 cmpl $0x80808080, %ecx /* if no null byte in this word */ 1057 je .word_aligned /* goto .word_aligned */ 1058 subl $4, %eax /* post-incremented */ 1059.not_word_aligned: 1060 cmpb $0, (%eax) /* if a byte in (%eax) is null */ 1061 je .null_found /* goto .null_found */ 1062 incl %eax /* next byte */ 1063 testl $3, %eax /* if %eax not word aligned */ 1064 jnz .not_word_aligned /* goto .not_word_aligned */ 1065 jmp .word_aligned /* goto .word_aligned */ 1066 .align 4 1067.null_found: 1068 subl 4(%esp), %eax /* %eax -= string address */ 1069 ret 1070 SET_SIZE(strlen) 1071 1072#endif /* __i386 */ 1073 1074#ifdef DEBUG 1075 .text 1076.str_panic_msg: 1077 .string "strlen: argument below kernelbase" 1078#endif /* DEBUG */ 1079 1080#endif /* __lint */ 1081 1082 /* 1083 * Berkeley 4.3 introduced symbolically named interrupt levels 1084 * as a way deal with priority in a machine independent fashion. 1085 * Numbered priorities are machine specific, and should be 1086 * discouraged where possible. 1087 * 1088 * Note, for the machine specific priorities there are 1089 * examples listed for devices that use a particular priority. 1090 * It should not be construed that all devices of that 1091 * type should be at that priority. It is currently were 1092 * the current devices fit into the priority scheme based 1093 * upon time criticalness. 1094 * 1095 * The underlying assumption of these assignments is that 1096 * IPL 10 is the highest level from which a device 1097 * routine can call wakeup. Devices that interrupt from higher 1098 * levels are restricted in what they can do. If they need 1099 * kernels services they should schedule a routine at a lower 1100 * level (via software interrupt) to do the required 1101 * processing. 1102 * 1103 * Examples of this higher usage: 1104 * Level Usage 1105 * 14 Profiling clock (and PROM uart polling clock) 1106 * 12 Serial ports 1107 * 1108 * The serial ports request lower level processing on level 6. 1109 * 1110 * Also, almost all splN routines (where N is a number or a 1111 * mnemonic) will do a RAISE(), on the assumption that they are 1112 * never used to lower our priority. 1113 * The exceptions are: 1114 * spl8() Because you can't be above 15 to begin with! 1115 * splzs() Because this is used at boot time to lower our 1116 * priority, to allow the PROM to poll the uart. 1117 * spl0() Used to lower priority to 0. 1118 */ 1119 1120#if defined(__lint) 1121 1122int spl0(void) { return (0); } 1123int spl6(void) { return (0); } 1124int spl7(void) { return (0); } 1125int spl8(void) { return (0); } 1126int splhigh(void) { return (0); } 1127int splhi(void) { return (0); } 1128int splzs(void) { return (0); } 1129 1130/* ARGSUSED */ 1131void 1132splx(int level) 1133{} 1134 1135#else /* __lint */ 1136 1137#if defined(__amd64) 1138 1139#define SETPRI(level) \ 1140 movl $/**/level, %edi; /* new priority */ \ 1141 jmp do_splx /* redirect to do_splx */ 1142 1143#define RAISE(level) \ 1144 movl $/**/level, %edi; /* new priority */ \ 1145 jmp splr /* redirect to splr */ 1146 1147#elif defined(__i386) 1148 1149#define SETPRI(level) \ 1150 pushl $/**/level; /* new priority */ \ 1151 call do_splx; /* invoke common splx code */ \ 1152 addl $4, %esp; /* unstack arg */ \ 1153 ret 1154 1155#define RAISE(level) \ 1156 pushl $/**/level; /* new priority */ \ 1157 call splr; /* invoke common splr code */ \ 1158 addl $4, %esp; /* unstack args */ \ 1159 ret 1160 1161#endif /* __i386 */ 1162 1163 /* locks out all interrupts, including memory errors */ 1164 ENTRY(spl8) 1165 SETPRI(15) 1166 SET_SIZE(spl8) 1167 1168 /* just below the level that profiling runs */ 1169 ENTRY(spl7) 1170 RAISE(13) 1171 SET_SIZE(spl7) 1172 1173 /* sun specific - highest priority onboard serial i/o asy ports */ 1174 ENTRY(splzs) 1175 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */ 1176 SET_SIZE(splzs) 1177 1178 ENTRY(splhi) 1179 ALTENTRY(splhigh) 1180 ALTENTRY(spl6) 1181 ALTENTRY(i_ddi_splhigh) 1182 1183 RAISE(DISP_LEVEL) 1184 1185 SET_SIZE(i_ddi_splhigh) 1186 SET_SIZE(spl6) 1187 SET_SIZE(splhigh) 1188 SET_SIZE(splhi) 1189 1190 /* allow all interrupts */ 1191 ENTRY(spl0) 1192 SETPRI(0) 1193 SET_SIZE(spl0) 1194 1195 1196 /* splx implementation */ 1197 ENTRY(splx) 1198 jmp do_splx /* redirect to common splx code */ 1199 SET_SIZE(splx) 1200 1201#endif /* __lint */ 1202 1203#if defined(__i386) 1204 1205/* 1206 * Read and write the %gs register 1207 */ 1208 1209#if defined(__lint) 1210 1211/*ARGSUSED*/ 1212uint16_t 1213getgs(void) 1214{ return (0); } 1215 1216/*ARGSUSED*/ 1217void 1218setgs(uint16_t sel) 1219{} 1220 1221#else /* __lint */ 1222 1223 ENTRY(getgs) 1224 clr %eax 1225 movw %gs, %ax 1226 ret 1227 SET_SIZE(getgs) 1228 1229 ENTRY(setgs) 1230 movw 4(%esp), %gs 1231 ret 1232 SET_SIZE(setgs) 1233 1234#endif /* __lint */ 1235#endif /* __i386 */ 1236 1237#if defined(__lint) 1238 1239void 1240pc_reset(void) 1241{} 1242 1243void 1244efi_reset(void) 1245{} 1246 1247#else /* __lint */ 1248 1249 ENTRY(wait_500ms) 1250 push %ebx 1251 movl $50000, %ebx 12521: 1253 call tenmicrosec 1254 decl %ebx 1255 jnz 1b 1256 pop %ebx 1257 ret 1258 SET_SIZE(wait_500ms) 1259 1260#define RESET_METHOD_KBC 1 1261#define RESET_METHOD_PORT92 2 1262#define RESET_METHOD_PCI 4 1263 1264 DGDEF3(pc_reset_methods, 4, 8) 1265 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI; 1266 1267 ENTRY(pc_reset) 1268 1269#if defined(__i386) 1270 testl $RESET_METHOD_KBC, pc_reset_methods 1271#elif defined(__amd64) 1272 testl $RESET_METHOD_KBC, pc_reset_methods(%rip) 1273#endif 1274 jz 1f 1275 1276 / 1277 / Try the classic keyboard controller-triggered reset. 1278 / 1279 movw $0x64, %dx 1280 movb $0xfe, %al 1281 outb (%dx) 1282 1283 / Wait up to 500 milliseconds here for the keyboard controller 1284 / to pull the reset line. On some systems where the keyboard 1285 / controller is slow to pull the reset line, the next reset method 1286 / may be executed (which may be bad if those systems hang when the 1287 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92), 1288 / and Ferrari 4000 (doesn't like the cf9 reset method)) 1289 1290 call wait_500ms 1291 12921: 1293#if defined(__i386) 1294 testl $RESET_METHOD_PORT92, pc_reset_methods 1295#elif defined(__amd64) 1296 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip) 1297#endif 1298 jz 3f 1299 1300 / 1301 / Try port 0x92 fast reset 1302 / 1303 movw $0x92, %dx 1304 inb (%dx) 1305 cmpb $0xff, %al / If port's not there, we should get back 0xFF 1306 je 1f 1307 testb $1, %al / If bit 0 1308 jz 2f / is clear, jump to perform the reset 1309 andb $0xfe, %al / otherwise, 1310 outb (%dx) / clear bit 0 first, then 13112: 1312 orb $1, %al / Set bit 0 1313 outb (%dx) / and reset the system 13141: 1315 1316 call wait_500ms 1317 13183: 1319#if defined(__i386) 1320 testl $RESET_METHOD_PCI, pc_reset_methods 1321#elif defined(__amd64) 1322 testl $RESET_METHOD_PCI, pc_reset_methods(%rip) 1323#endif 1324 jz 4f 1325 1326 / Try the PCI (soft) reset vector (should work on all modern systems, 1327 / but has been shown to cause problems on 450NX systems, and some newer 1328 / systems (e.g. ATI IXP400-equipped systems)) 1329 / When resetting via this method, 2 writes are required. The first 1330 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with 1331 / power cycle). 1332 / The reset occurs on the second write, during bit 2's transition from 1333 / 0->1. 1334 movw $0xcf9, %dx 1335 movb $0x2, %al / Reset mode = hard, no power cycle 1336 outb (%dx) 1337 movb $0x6, %al 1338 outb (%dx) 1339 1340 call wait_500ms 1341 13424: 1343 / 1344 / port 0xcf9 failed also. Last-ditch effort is to 1345 / triple-fault the CPU. 1346 / Also, use triple fault for EFI firmware 1347 / 1348 ENTRY(efi_reset) 1349#if defined(__amd64) 1350 pushq $0x0 1351 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1352 lidt (%rsp) 1353#elif defined(__i386) 1354 pushl $0x0 1355 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 1356 lidt (%esp) 1357#endif 1358 int $0x0 / Trigger interrupt, generate triple-fault 1359 1360 cli 1361 hlt / Wait forever 1362 /*NOTREACHED*/ 1363 SET_SIZE(efi_reset) 1364 SET_SIZE(pc_reset) 1365 1366#endif /* __lint */ 1367 1368/* 1369 * C callable in and out routines 1370 */ 1371 1372#if defined(__lint) 1373 1374/* ARGSUSED */ 1375void 1376outl(int port_address, uint32_t val) 1377{} 1378 1379#else /* __lint */ 1380 1381#if defined(__amd64) 1382 1383 ENTRY(outl) 1384 movw %di, %dx 1385 movl %esi, %eax 1386 outl (%dx) 1387 ret 1388 SET_SIZE(outl) 1389 1390#elif defined(__i386) 1391 1392 .set PORT, 4 1393 .set VAL, 8 1394 1395 ENTRY(outl) 1396 movw PORT(%esp), %dx 1397 movl VAL(%esp), %eax 1398 outl (%dx) 1399 ret 1400 SET_SIZE(outl) 1401 1402#endif /* __i386 */ 1403#endif /* __lint */ 1404 1405#if defined(__lint) 1406 1407/* ARGSUSED */ 1408void 1409outw(int port_address, uint16_t val) 1410{} 1411 1412#else /* __lint */ 1413 1414#if defined(__amd64) 1415 1416 ENTRY(outw) 1417 movw %di, %dx 1418 movw %si, %ax 1419 D16 outl (%dx) /* XX64 why not outw? */ 1420 ret 1421 SET_SIZE(outw) 1422 1423#elif defined(__i386) 1424 1425 ENTRY(outw) 1426 movw PORT(%esp), %dx 1427 movw VAL(%esp), %ax 1428 D16 outl (%dx) 1429 ret 1430 SET_SIZE(outw) 1431 1432#endif /* __i386 */ 1433#endif /* __lint */ 1434 1435#if defined(__lint) 1436 1437/* ARGSUSED */ 1438void 1439outb(int port_address, uint8_t val) 1440{} 1441 1442#else /* __lint */ 1443 1444#if defined(__amd64) 1445 1446 ENTRY(outb) 1447 movw %di, %dx 1448 movb %sil, %al 1449 outb (%dx) 1450 ret 1451 SET_SIZE(outb) 1452 1453#elif defined(__i386) 1454 1455 ENTRY(outb) 1456 movw PORT(%esp), %dx 1457 movb VAL(%esp), %al 1458 outb (%dx) 1459 ret 1460 SET_SIZE(outb) 1461 1462#endif /* __i386 */ 1463#endif /* __lint */ 1464 1465#if defined(__lint) 1466 1467/* ARGSUSED */ 1468uint32_t 1469inl(int port_address) 1470{ return (0); } 1471 1472#else /* __lint */ 1473 1474#if defined(__amd64) 1475 1476 ENTRY(inl) 1477 xorl %eax, %eax 1478 movw %di, %dx 1479 inl (%dx) 1480 ret 1481 SET_SIZE(inl) 1482 1483#elif defined(__i386) 1484 1485 ENTRY(inl) 1486 movw PORT(%esp), %dx 1487 inl (%dx) 1488 ret 1489 SET_SIZE(inl) 1490 1491#endif /* __i386 */ 1492#endif /* __lint */ 1493 1494#if defined(__lint) 1495 1496/* ARGSUSED */ 1497uint16_t 1498inw(int port_address) 1499{ return (0); } 1500 1501#else /* __lint */ 1502 1503#if defined(__amd64) 1504 1505 ENTRY(inw) 1506 xorl %eax, %eax 1507 movw %di, %dx 1508 D16 inl (%dx) 1509 ret 1510 SET_SIZE(inw) 1511 1512#elif defined(__i386) 1513 1514 ENTRY(inw) 1515 subl %eax, %eax 1516 movw PORT(%esp), %dx 1517 D16 inl (%dx) 1518 ret 1519 SET_SIZE(inw) 1520 1521#endif /* __i386 */ 1522#endif /* __lint */ 1523 1524 1525#if defined(__lint) 1526 1527/* ARGSUSED */ 1528uint8_t 1529inb(int port_address) 1530{ return (0); } 1531 1532#else /* __lint */ 1533 1534#if defined(__amd64) 1535 1536 ENTRY(inb) 1537 xorl %eax, %eax 1538 movw %di, %dx 1539 inb (%dx) 1540 ret 1541 SET_SIZE(inb) 1542 1543#elif defined(__i386) 1544 1545 ENTRY(inb) 1546 subl %eax, %eax 1547 movw PORT(%esp), %dx 1548 inb (%dx) 1549 ret 1550 SET_SIZE(inb) 1551 1552#endif /* __i386 */ 1553#endif /* __lint */ 1554 1555 1556#if defined(__lint) 1557 1558/* ARGSUSED */ 1559void 1560repoutsw(int port, uint16_t *addr, int cnt) 1561{} 1562 1563#else /* __lint */ 1564 1565#if defined(__amd64) 1566 1567 ENTRY(repoutsw) 1568 movl %edx, %ecx 1569 movw %di, %dx 1570 rep 1571 D16 outsl 1572 ret 1573 SET_SIZE(repoutsw) 1574 1575#elif defined(__i386) 1576 1577 /* 1578 * The arguments and saved registers are on the stack in the 1579 * following order: 1580 * | cnt | +16 1581 * | *addr | +12 1582 * | port | +8 1583 * | eip | +4 1584 * | esi | <-- %esp 1585 * If additional values are pushed onto the stack, make sure 1586 * to adjust the following constants accordingly. 1587 */ 1588 .set PORT, 8 1589 .set ADDR, 12 1590 .set COUNT, 16 1591 1592 ENTRY(repoutsw) 1593 pushl %esi 1594 movl PORT(%esp), %edx 1595 movl ADDR(%esp), %esi 1596 movl COUNT(%esp), %ecx 1597 rep 1598 D16 outsl 1599 popl %esi 1600 ret 1601 SET_SIZE(repoutsw) 1602 1603#endif /* __i386 */ 1604#endif /* __lint */ 1605 1606 1607#if defined(__lint) 1608 1609/* ARGSUSED */ 1610void 1611repinsw(int port_addr, uint16_t *addr, int cnt) 1612{} 1613 1614#else /* __lint */ 1615 1616#if defined(__amd64) 1617 1618 ENTRY(repinsw) 1619 movl %edx, %ecx 1620 movw %di, %dx 1621 rep 1622 D16 insl 1623 ret 1624 SET_SIZE(repinsw) 1625 1626#elif defined(__i386) 1627 1628 ENTRY(repinsw) 1629 pushl %edi 1630 movl PORT(%esp), %edx 1631 movl ADDR(%esp), %edi 1632 movl COUNT(%esp), %ecx 1633 rep 1634 D16 insl 1635 popl %edi 1636 ret 1637 SET_SIZE(repinsw) 1638 1639#endif /* __i386 */ 1640#endif /* __lint */ 1641 1642 1643#if defined(__lint) 1644 1645/* ARGSUSED */ 1646void 1647repinsb(int port, uint8_t *addr, int count) 1648{} 1649 1650#else /* __lint */ 1651 1652#if defined(__amd64) 1653 1654 ENTRY(repinsb) 1655 movl %edx, %ecx 1656 movw %di, %dx 1657 movq %rsi, %rdi 1658 rep 1659 insb 1660 ret 1661 SET_SIZE(repinsb) 1662 1663#elif defined(__i386) 1664 1665 /* 1666 * The arguments and saved registers are on the stack in the 1667 * following order: 1668 * | cnt | +16 1669 * | *addr | +12 1670 * | port | +8 1671 * | eip | +4 1672 * | esi | <-- %esp 1673 * If additional values are pushed onto the stack, make sure 1674 * to adjust the following constants accordingly. 1675 */ 1676 .set IO_PORT, 8 1677 .set IO_ADDR, 12 1678 .set IO_COUNT, 16 1679 1680 ENTRY(repinsb) 1681 pushl %edi 1682 movl IO_ADDR(%esp), %edi 1683 movl IO_COUNT(%esp), %ecx 1684 movl IO_PORT(%esp), %edx 1685 rep 1686 insb 1687 popl %edi 1688 ret 1689 SET_SIZE(repinsb) 1690 1691#endif /* __i386 */ 1692#endif /* __lint */ 1693 1694 1695/* 1696 * Input a stream of 32-bit words. 1697 * NOTE: count is a DWORD count. 1698 */ 1699#if defined(__lint) 1700 1701/* ARGSUSED */ 1702void 1703repinsd(int port, uint32_t *addr, int count) 1704{} 1705 1706#else /* __lint */ 1707 1708#if defined(__amd64) 1709 1710 ENTRY(repinsd) 1711 movl %edx, %ecx 1712 movw %di, %dx 1713 movq %rsi, %rdi 1714 rep 1715 insl 1716 ret 1717 SET_SIZE(repinsd) 1718 1719#elif defined(__i386) 1720 1721 ENTRY(repinsd) 1722 pushl %edi 1723 movl IO_ADDR(%esp), %edi 1724 movl IO_COUNT(%esp), %ecx 1725 movl IO_PORT(%esp), %edx 1726 rep 1727 insl 1728 popl %edi 1729 ret 1730 SET_SIZE(repinsd) 1731 1732#endif /* __i386 */ 1733#endif /* __lint */ 1734 1735/* 1736 * Output a stream of bytes 1737 * NOTE: count is a byte count 1738 */ 1739#if defined(__lint) 1740 1741/* ARGSUSED */ 1742void 1743repoutsb(int port, uint8_t *addr, int count) 1744{} 1745 1746#else /* __lint */ 1747 1748#if defined(__amd64) 1749 1750 ENTRY(repoutsb) 1751 movl %edx, %ecx 1752 movw %di, %dx 1753 rep 1754 outsb 1755 ret 1756 SET_SIZE(repoutsb) 1757 1758#elif defined(__i386) 1759 1760 ENTRY(repoutsb) 1761 pushl %esi 1762 movl IO_ADDR(%esp), %esi 1763 movl IO_COUNT(%esp), %ecx 1764 movl IO_PORT(%esp), %edx 1765 rep 1766 outsb 1767 popl %esi 1768 ret 1769 SET_SIZE(repoutsb) 1770 1771#endif /* __i386 */ 1772#endif /* __lint */ 1773 1774/* 1775 * Output a stream of 32-bit words 1776 * NOTE: count is a DWORD count 1777 */ 1778#if defined(__lint) 1779 1780/* ARGSUSED */ 1781void 1782repoutsd(int port, uint32_t *addr, int count) 1783{} 1784 1785#else /* __lint */ 1786 1787#if defined(__amd64) 1788 1789 ENTRY(repoutsd) 1790 movl %edx, %ecx 1791 movw %di, %dx 1792 rep 1793 outsl 1794 ret 1795 SET_SIZE(repoutsd) 1796 1797#elif defined(__i386) 1798 1799 ENTRY(repoutsd) 1800 pushl %esi 1801 movl IO_ADDR(%esp), %esi 1802 movl IO_COUNT(%esp), %ecx 1803 movl IO_PORT(%esp), %edx 1804 rep 1805 outsl 1806 popl %esi 1807 ret 1808 SET_SIZE(repoutsd) 1809 1810#endif /* __i386 */ 1811#endif /* __lint */ 1812 1813/* 1814 * void int3(void) 1815 * void int18(void) 1816 * void int20(void) 1817 * void int_cmci(void) 1818 */ 1819 1820#if defined(__lint) 1821 1822void 1823int3(void) 1824{} 1825 1826void 1827int18(void) 1828{} 1829 1830void 1831int20(void) 1832{} 1833 1834void 1835int_cmci(void) 1836{} 1837 1838#else /* __lint */ 1839 1840 ENTRY(int3) 1841 int $T_BPTFLT 1842 ret 1843 SET_SIZE(int3) 1844 1845 ENTRY(int18) 1846 int $T_MCE 1847 ret 1848 SET_SIZE(int18) 1849 1850 ENTRY(int20) 1851 movl boothowto, %eax 1852 andl $RB_DEBUG, %eax 1853 jz 1f 1854 1855 int $T_DBGENTR 18561: 1857 rep; ret /* use 2 byte return instruction when branch target */ 1858 /* AMD Software Optimization Guide - Section 6.2 */ 1859 SET_SIZE(int20) 1860 1861 ENTRY(int_cmci) 1862 int $T_ENOEXTFLT 1863 ret 1864 SET_SIZE(int_cmci) 1865 1866#endif /* __lint */ 1867 1868#if defined(__lint) 1869 1870/* ARGSUSED */ 1871int 1872scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask) 1873{ return (0); } 1874 1875#else /* __lint */ 1876 1877#if defined(__amd64) 1878 1879 ENTRY(scanc) 1880 /* rdi == size */ 1881 /* rsi == cp */ 1882 /* rdx == table */ 1883 /* rcx == mask */ 1884 addq %rsi, %rdi /* end = &cp[size] */ 1885.scanloop: 1886 cmpq %rdi, %rsi /* while (cp < end */ 1887 jnb .scandone 1888 movzbq (%rsi), %r8 /* %r8 = *cp */ 1889 incq %rsi /* cp++ */ 1890 testb %cl, (%r8, %rdx) 1891 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1892 decq %rsi /* (fix post-increment) */ 1893.scandone: 1894 movl %edi, %eax 1895 subl %esi, %eax /* return (end - cp) */ 1896 ret 1897 SET_SIZE(scanc) 1898 1899#elif defined(__i386) 1900 1901 ENTRY(scanc) 1902 pushl %edi 1903 pushl %esi 1904 movb 24(%esp), %cl /* mask = %cl */ 1905 movl 16(%esp), %esi /* cp = %esi */ 1906 movl 20(%esp), %edx /* table = %edx */ 1907 movl %esi, %edi 1908 addl 12(%esp), %edi /* end = &cp[size]; */ 1909.scanloop: 1910 cmpl %edi, %esi /* while (cp < end */ 1911 jnb .scandone 1912 movzbl (%esi), %eax /* %al = *cp */ 1913 incl %esi /* cp++ */ 1914 movb (%edx, %eax), %al /* %al = table[*cp] */ 1915 testb %al, %cl 1916 jz .scanloop /* && (table[*cp] & mask) == 0) */ 1917 dec %esi /* post-incremented */ 1918.scandone: 1919 movl %edi, %eax 1920 subl %esi, %eax /* return (end - cp) */ 1921 popl %esi 1922 popl %edi 1923 ret 1924 SET_SIZE(scanc) 1925 1926#endif /* __i386 */ 1927#endif /* __lint */ 1928 1929/* 1930 * Replacement functions for ones that are normally inlined. 1931 * In addition to the copy in i86.il, they are defined here just in case. 1932 */ 1933 1934#if defined(__lint) 1935 1936ulong_t 1937intr_clear(void) 1938{ return (0); } 1939 1940ulong_t 1941clear_int_flag(void) 1942{ return (0); } 1943 1944#else /* __lint */ 1945 1946#if defined(__amd64) 1947 1948 ENTRY(intr_clear) 1949 ENTRY(clear_int_flag) 1950 pushfq 1951 popq %rax 1952#if defined(__xpv) 1953 leaq xpv_panicking, %rdi 1954 movl (%rdi), %edi 1955 cmpl $0, %edi 1956 jne 2f 1957 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 1958 /* 1959 * Synthesize the PS_IE bit from the event mask bit 1960 */ 1961 andq $_BITNOT(PS_IE), %rax 1962 testb $1, %dl 1963 jnz 1f 1964 orq $PS_IE, %rax 19651: 1966 ret 19672: 1968#endif 1969 CLI(%rdi) 1970 ret 1971 SET_SIZE(clear_int_flag) 1972 SET_SIZE(intr_clear) 1973 1974#elif defined(__i386) 1975 1976 ENTRY(intr_clear) 1977 ENTRY(clear_int_flag) 1978 pushfl 1979 popl %eax 1980#if defined(__xpv) 1981 leal xpv_panicking, %edx 1982 movl (%edx), %edx 1983 cmpl $0, %edx 1984 jne 2f 1985 CLIRET(%edx, %cl) /* returns event mask in %cl */ 1986 /* 1987 * Synthesize the PS_IE bit from the event mask bit 1988 */ 1989 andl $_BITNOT(PS_IE), %eax 1990 testb $1, %cl 1991 jnz 1f 1992 orl $PS_IE, %eax 19931: 1994 ret 19952: 1996#endif 1997 CLI(%edx) 1998 ret 1999 SET_SIZE(clear_int_flag) 2000 SET_SIZE(intr_clear) 2001 2002#endif /* __i386 */ 2003#endif /* __lint */ 2004 2005#if defined(__lint) 2006 2007struct cpu * 2008curcpup(void) 2009{ return 0; } 2010 2011#else /* __lint */ 2012 2013#if defined(__amd64) 2014 2015 ENTRY(curcpup) 2016 movq %gs:CPU_SELF, %rax 2017 ret 2018 SET_SIZE(curcpup) 2019 2020#elif defined(__i386) 2021 2022 ENTRY(curcpup) 2023 movl %gs:CPU_SELF, %eax 2024 ret 2025 SET_SIZE(curcpup) 2026 2027#endif /* __i386 */ 2028#endif /* __lint */ 2029 2030/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs() 2031 * These functions reverse the byte order of the input parameter and returns 2032 * the result. This is to convert the byte order from host byte order 2033 * (little endian) to network byte order (big endian), or vice versa. 2034 */ 2035 2036#if defined(__lint) 2037 2038uint64_t 2039htonll(uint64_t i) 2040{ return (i); } 2041 2042uint64_t 2043ntohll(uint64_t i) 2044{ return (i); } 2045 2046uint32_t 2047htonl(uint32_t i) 2048{ return (i); } 2049 2050uint32_t 2051ntohl(uint32_t i) 2052{ return (i); } 2053 2054uint16_t 2055htons(uint16_t i) 2056{ return (i); } 2057 2058uint16_t 2059ntohs(uint16_t i) 2060{ return (i); } 2061 2062#else /* __lint */ 2063 2064#if defined(__amd64) 2065 2066 ENTRY(htonll) 2067 ALTENTRY(ntohll) 2068 movq %rdi, %rax 2069 bswapq %rax 2070 ret 2071 SET_SIZE(ntohll) 2072 SET_SIZE(htonll) 2073 2074 /* XX64 there must be shorter sequences for this */ 2075 ENTRY(htonl) 2076 ALTENTRY(ntohl) 2077 movl %edi, %eax 2078 bswap %eax 2079 ret 2080 SET_SIZE(ntohl) 2081 SET_SIZE(htonl) 2082 2083 /* XX64 there must be better sequences for this */ 2084 ENTRY(htons) 2085 ALTENTRY(ntohs) 2086 movl %edi, %eax 2087 bswap %eax 2088 shrl $16, %eax 2089 ret 2090 SET_SIZE(ntohs) 2091 SET_SIZE(htons) 2092 2093#elif defined(__i386) 2094 2095 ENTRY(htonll) 2096 ALTENTRY(ntohll) 2097 movl 4(%esp), %edx 2098 movl 8(%esp), %eax 2099 bswap %edx 2100 bswap %eax 2101 ret 2102 SET_SIZE(ntohll) 2103 SET_SIZE(htonll) 2104 2105 ENTRY(htonl) 2106 ALTENTRY(ntohl) 2107 movl 4(%esp), %eax 2108 bswap %eax 2109 ret 2110 SET_SIZE(ntohl) 2111 SET_SIZE(htonl) 2112 2113 ENTRY(htons) 2114 ALTENTRY(ntohs) 2115 movl 4(%esp), %eax 2116 bswap %eax 2117 shrl $16, %eax 2118 ret 2119 SET_SIZE(ntohs) 2120 SET_SIZE(htons) 2121 2122#endif /* __i386 */ 2123#endif /* __lint */ 2124 2125 2126#if defined(__lint) 2127 2128/* ARGSUSED */ 2129void 2130intr_restore(ulong_t i) 2131{ return; } 2132 2133/* ARGSUSED */ 2134void 2135restore_int_flag(ulong_t i) 2136{ return; } 2137 2138#else /* __lint */ 2139 2140#if defined(__amd64) 2141 2142 ENTRY(intr_restore) 2143 ENTRY(restore_int_flag) 2144 testq $PS_IE, %rdi 2145 jz 1f 2146#if defined(__xpv) 2147 leaq xpv_panicking, %rsi 2148 movl (%rsi), %esi 2149 cmpl $0, %esi 2150 jne 1f 2151 /* 2152 * Since we're -really- running unprivileged, our attempt 2153 * to change the state of the IF bit will be ignored. 2154 * The virtual IF bit is tweaked by CLI and STI. 2155 */ 2156 IE_TO_EVENT_MASK(%rsi, %rdi) 2157#else 2158 sti 2159#endif 21601: 2161 ret 2162 SET_SIZE(restore_int_flag) 2163 SET_SIZE(intr_restore) 2164 2165#elif defined(__i386) 2166 2167 ENTRY(intr_restore) 2168 ENTRY(restore_int_flag) 2169 testl $PS_IE, 4(%esp) 2170 jz 1f 2171#if defined(__xpv) 2172 leal xpv_panicking, %edx 2173 movl (%edx), %edx 2174 cmpl $0, %edx 2175 jne 1f 2176 /* 2177 * Since we're -really- running unprivileged, our attempt 2178 * to change the state of the IF bit will be ignored. 2179 * The virtual IF bit is tweaked by CLI and STI. 2180 */ 2181 IE_TO_EVENT_MASK(%edx, 4(%esp)) 2182#else 2183 sti 2184#endif 21851: 2186 ret 2187 SET_SIZE(restore_int_flag) 2188 SET_SIZE(intr_restore) 2189 2190#endif /* __i386 */ 2191#endif /* __lint */ 2192 2193#if defined(__lint) 2194 2195void 2196sti(void) 2197{} 2198 2199void 2200cli(void) 2201{} 2202 2203#else /* __lint */ 2204 2205 ENTRY(sti) 2206 STI 2207 ret 2208 SET_SIZE(sti) 2209 2210 ENTRY(cli) 2211#if defined(__amd64) 2212 CLI(%rax) 2213#elif defined(__i386) 2214 CLI(%eax) 2215#endif /* __i386 */ 2216 ret 2217 SET_SIZE(cli) 2218 2219#endif /* __lint */ 2220 2221#if defined(__lint) 2222 2223dtrace_icookie_t 2224dtrace_interrupt_disable(void) 2225{ return (0); } 2226 2227#else /* __lint */ 2228 2229#if defined(__amd64) 2230 2231 ENTRY(dtrace_interrupt_disable) 2232 pushfq 2233 popq %rax 2234#if defined(__xpv) 2235 leaq xpv_panicking, %rdi 2236 movl (%rdi), %edi 2237 cmpl $0, %edi 2238 jne .dtrace_interrupt_disable_done 2239 CLIRET(%rdi, %dl) /* returns event mask in %dl */ 2240 /* 2241 * Synthesize the PS_IE bit from the event mask bit 2242 */ 2243 andq $_BITNOT(PS_IE), %rax 2244 testb $1, %dl 2245 jnz .dtrace_interrupt_disable_done 2246 orq $PS_IE, %rax 2247#else 2248 CLI(%rdx) 2249#endif 2250.dtrace_interrupt_disable_done: 2251 ret 2252 SET_SIZE(dtrace_interrupt_disable) 2253 2254#elif defined(__i386) 2255 2256 ENTRY(dtrace_interrupt_disable) 2257 pushfl 2258 popl %eax 2259#if defined(__xpv) 2260 leal xpv_panicking, %edx 2261 movl (%edx), %edx 2262 cmpl $0, %edx 2263 jne .dtrace_interrupt_disable_done 2264 CLIRET(%edx, %cl) /* returns event mask in %cl */ 2265 /* 2266 * Synthesize the PS_IE bit from the event mask bit 2267 */ 2268 andl $_BITNOT(PS_IE), %eax 2269 testb $1, %cl 2270 jnz .dtrace_interrupt_disable_done 2271 orl $PS_IE, %eax 2272#else 2273 CLI(%edx) 2274#endif 2275.dtrace_interrupt_disable_done: 2276 ret 2277 SET_SIZE(dtrace_interrupt_disable) 2278 2279#endif /* __i386 */ 2280#endif /* __lint */ 2281 2282#if defined(__lint) 2283 2284/*ARGSUSED*/ 2285void 2286dtrace_interrupt_enable(dtrace_icookie_t cookie) 2287{} 2288 2289#else /* __lint */ 2290 2291#if defined(__amd64) 2292 2293 ENTRY(dtrace_interrupt_enable) 2294 pushq %rdi 2295 popfq 2296#if defined(__xpv) 2297 leaq xpv_panicking, %rdx 2298 movl (%rdx), %edx 2299 cmpl $0, %edx 2300 jne .dtrace_interrupt_enable_done 2301 /* 2302 * Since we're -really- running unprivileged, our attempt 2303 * to change the state of the IF bit will be ignored. The 2304 * virtual IF bit is tweaked by CLI and STI. 2305 */ 2306 IE_TO_EVENT_MASK(%rdx, %rdi) 2307#endif 2308.dtrace_interrupt_enable_done: 2309 ret 2310 SET_SIZE(dtrace_interrupt_enable) 2311 2312#elif defined(__i386) 2313 2314 ENTRY(dtrace_interrupt_enable) 2315 movl 4(%esp), %eax 2316 pushl %eax 2317 popfl 2318#if defined(__xpv) 2319 leal xpv_panicking, %edx 2320 movl (%edx), %edx 2321 cmpl $0, %edx 2322 jne .dtrace_interrupt_enable_done 2323 /* 2324 * Since we're -really- running unprivileged, our attempt 2325 * to change the state of the IF bit will be ignored. The 2326 * virtual IF bit is tweaked by CLI and STI. 2327 */ 2328 IE_TO_EVENT_MASK(%edx, %eax) 2329#endif 2330.dtrace_interrupt_enable_done: 2331 ret 2332 SET_SIZE(dtrace_interrupt_enable) 2333 2334#endif /* __i386 */ 2335#endif /* __lint */ 2336 2337 2338#if defined(lint) 2339 2340void 2341dtrace_membar_producer(void) 2342{} 2343 2344void 2345dtrace_membar_consumer(void) 2346{} 2347 2348#else /* __lint */ 2349 2350 ENTRY(dtrace_membar_producer) 2351 rep; ret /* use 2 byte return instruction when branch target */ 2352 /* AMD Software Optimization Guide - Section 6.2 */ 2353 SET_SIZE(dtrace_membar_producer) 2354 2355 ENTRY(dtrace_membar_consumer) 2356 rep; ret /* use 2 byte return instruction when branch target */ 2357 /* AMD Software Optimization Guide - Section 6.2 */ 2358 SET_SIZE(dtrace_membar_consumer) 2359 2360#endif /* __lint */ 2361 2362#if defined(__lint) 2363 2364kthread_id_t 2365threadp(void) 2366{ return ((kthread_id_t)0); } 2367 2368#else /* __lint */ 2369 2370#if defined(__amd64) 2371 2372 ENTRY(threadp) 2373 movq %gs:CPU_THREAD, %rax 2374 ret 2375 SET_SIZE(threadp) 2376 2377#elif defined(__i386) 2378 2379 ENTRY(threadp) 2380 movl %gs:CPU_THREAD, %eax 2381 ret 2382 SET_SIZE(threadp) 2383 2384#endif /* __i386 */ 2385#endif /* __lint */ 2386 2387/* 2388 * Checksum routine for Internet Protocol Headers 2389 */ 2390 2391#if defined(__lint) 2392 2393/* ARGSUSED */ 2394unsigned int 2395ip_ocsum( 2396 ushort_t *address, /* ptr to 1st message buffer */ 2397 int halfword_count, /* length of data */ 2398 unsigned int sum) /* partial checksum */ 2399{ 2400 int i; 2401 unsigned int psum = 0; /* partial sum */ 2402 2403 for (i = 0; i < halfword_count; i++, address++) { 2404 psum += *address; 2405 } 2406 2407 while ((psum >> 16) != 0) { 2408 psum = (psum & 0xffff) + (psum >> 16); 2409 } 2410 2411 psum += sum; 2412 2413 while ((psum >> 16) != 0) { 2414 psum = (psum & 0xffff) + (psum >> 16); 2415 } 2416 2417 return (psum); 2418} 2419 2420#else /* __lint */ 2421 2422#if defined(__amd64) 2423 2424 ENTRY(ip_ocsum) 2425 pushq %rbp 2426 movq %rsp, %rbp 2427#ifdef DEBUG 2428 movq postbootkernelbase(%rip), %rax 2429 cmpq %rax, %rdi 2430 jnb 1f 2431 xorl %eax, %eax 2432 movq %rdi, %rsi 2433 leaq .ip_ocsum_panic_msg(%rip), %rdi 2434 call panic 2435 /*NOTREACHED*/ 2436.ip_ocsum_panic_msg: 2437 .string "ip_ocsum: address 0x%p below kernelbase\n" 24381: 2439#endif 2440 movl %esi, %ecx /* halfword_count */ 2441 movq %rdi, %rsi /* address */ 2442 /* partial sum in %edx */ 2443 xorl %eax, %eax 2444 testl %ecx, %ecx 2445 jz .ip_ocsum_done 2446 testq $3, %rsi 2447 jnz .ip_csum_notaligned 2448.ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */ 2449.next_iter: 2450 /* XX64 opportunities for prefetch? */ 2451 /* XX64 compute csum with 64 bit quantities? */ 2452 subl $32, %ecx 2453 jl .less_than_32 2454 2455 addl 0(%rsi), %edx 2456.only60: 2457 adcl 4(%rsi), %eax 2458.only56: 2459 adcl 8(%rsi), %edx 2460.only52: 2461 adcl 12(%rsi), %eax 2462.only48: 2463 adcl 16(%rsi), %edx 2464.only44: 2465 adcl 20(%rsi), %eax 2466.only40: 2467 adcl 24(%rsi), %edx 2468.only36: 2469 adcl 28(%rsi), %eax 2470.only32: 2471 adcl 32(%rsi), %edx 2472.only28: 2473 adcl 36(%rsi), %eax 2474.only24: 2475 adcl 40(%rsi), %edx 2476.only20: 2477 adcl 44(%rsi), %eax 2478.only16: 2479 adcl 48(%rsi), %edx 2480.only12: 2481 adcl 52(%rsi), %eax 2482.only8: 2483 adcl 56(%rsi), %edx 2484.only4: 2485 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */ 2486.only0: 2487 adcl $0, %eax /* could be adding -1 in eax with a carry */ 2488 adcl $0, %eax 2489 2490 addq $64, %rsi 2491 testl %ecx, %ecx 2492 jnz .next_iter 2493 2494.ip_ocsum_done: 2495 addl %eax, %edx 2496 adcl $0, %edx 2497 movl %edx, %eax /* form a 16 bit checksum by */ 2498 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2499 addw %dx, %ax 2500 adcw $0, %ax 2501 andl $0xffff, %eax 2502 leave 2503 ret 2504 2505.ip_csum_notaligned: 2506 xorl %edi, %edi 2507 movw (%rsi), %di 2508 addl %edi, %edx 2509 adcl $0, %edx 2510 addq $2, %rsi 2511 decl %ecx 2512 jmp .ip_csum_aligned 2513 2514.less_than_32: 2515 addl $32, %ecx 2516 testl $1, %ecx 2517 jz .size_aligned 2518 andl $0xfe, %ecx 2519 movzwl (%rsi, %rcx, 2), %edi 2520 addl %edi, %edx 2521 adcl $0, %edx 2522.size_aligned: 2523 movl %ecx, %edi 2524 shrl $1, %ecx 2525 shl $1, %edi 2526 subq $64, %rdi 2527 addq %rdi, %rsi 2528 leaq .ip_ocsum_jmptbl(%rip), %rdi 2529 leaq (%rdi, %rcx, 8), %rdi 2530 xorl %ecx, %ecx 2531 clc 2532 jmp *(%rdi) 2533 2534 .align 8 2535.ip_ocsum_jmptbl: 2536 .quad .only0, .only4, .only8, .only12, .only16, .only20 2537 .quad .only24, .only28, .only32, .only36, .only40, .only44 2538 .quad .only48, .only52, .only56, .only60 2539 SET_SIZE(ip_ocsum) 2540 2541#elif defined(__i386) 2542 2543 ENTRY(ip_ocsum) 2544 pushl %ebp 2545 movl %esp, %ebp 2546 pushl %ebx 2547 pushl %esi 2548 pushl %edi 2549 movl 12(%ebp), %ecx /* count of half words */ 2550 movl 16(%ebp), %edx /* partial checksum */ 2551 movl 8(%ebp), %esi 2552 xorl %eax, %eax 2553 testl %ecx, %ecx 2554 jz .ip_ocsum_done 2555 2556 testl $3, %esi 2557 jnz .ip_csum_notaligned 2558.ip_csum_aligned: 2559.next_iter: 2560 subl $32, %ecx 2561 jl .less_than_32 2562 2563 addl 0(%esi), %edx 2564.only60: 2565 adcl 4(%esi), %eax 2566.only56: 2567 adcl 8(%esi), %edx 2568.only52: 2569 adcl 12(%esi), %eax 2570.only48: 2571 adcl 16(%esi), %edx 2572.only44: 2573 adcl 20(%esi), %eax 2574.only40: 2575 adcl 24(%esi), %edx 2576.only36: 2577 adcl 28(%esi), %eax 2578.only32: 2579 adcl 32(%esi), %edx 2580.only28: 2581 adcl 36(%esi), %eax 2582.only24: 2583 adcl 40(%esi), %edx 2584.only20: 2585 adcl 44(%esi), %eax 2586.only16: 2587 adcl 48(%esi), %edx 2588.only12: 2589 adcl 52(%esi), %eax 2590.only8: 2591 adcl 56(%esi), %edx 2592.only4: 2593 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */ 2594.only0: 2595 adcl $0, %eax /* we could be adding -1 in eax with a carry */ 2596 adcl $0, %eax 2597 2598 addl $64, %esi 2599 andl %ecx, %ecx 2600 jnz .next_iter 2601 2602.ip_ocsum_done: 2603 addl %eax, %edx 2604 adcl $0, %edx 2605 movl %edx, %eax /* form a 16 bit checksum by */ 2606 shrl $16, %eax /* adding two halves of 32 bit checksum */ 2607 addw %dx, %ax 2608 adcw $0, %ax 2609 andl $0xffff, %eax 2610 popl %edi /* restore registers */ 2611 popl %esi 2612 popl %ebx 2613 leave 2614 ret 2615 2616.ip_csum_notaligned: 2617 xorl %edi, %edi 2618 movw (%esi), %di 2619 addl %edi, %edx 2620 adcl $0, %edx 2621 addl $2, %esi 2622 decl %ecx 2623 jmp .ip_csum_aligned 2624 2625.less_than_32: 2626 addl $32, %ecx 2627 testl $1, %ecx 2628 jz .size_aligned 2629 andl $0xfe, %ecx 2630 movzwl (%esi, %ecx, 2), %edi 2631 addl %edi, %edx 2632 adcl $0, %edx 2633.size_aligned: 2634 movl %ecx, %edi 2635 shrl $1, %ecx 2636 shl $1, %edi 2637 subl $64, %edi 2638 addl %edi, %esi 2639 movl $.ip_ocsum_jmptbl, %edi 2640 lea (%edi, %ecx, 4), %edi 2641 xorl %ecx, %ecx 2642 clc 2643 jmp *(%edi) 2644 SET_SIZE(ip_ocsum) 2645 2646 .data 2647 .align 4 2648 2649.ip_ocsum_jmptbl: 2650 .long .only0, .only4, .only8, .only12, .only16, .only20 2651 .long .only24, .only28, .only32, .only36, .only40, .only44 2652 .long .only48, .only52, .only56, .only60 2653 2654 2655#endif /* __i386 */ 2656#endif /* __lint */ 2657 2658/* 2659 * multiply two long numbers and yield a u_longlong_t result, callable from C. 2660 * Provided to manipulate hrtime_t values. 2661 */ 2662#if defined(__lint) 2663 2664/* result = a * b; */ 2665 2666/* ARGSUSED */ 2667unsigned long long 2668mul32(uint_t a, uint_t b) 2669{ return (0); } 2670 2671#else /* __lint */ 2672 2673#if defined(__amd64) 2674 2675 ENTRY(mul32) 2676 xorl %edx, %edx /* XX64 joe, paranoia? */ 2677 movl %edi, %eax 2678 mull %esi 2679 shlq $32, %rdx 2680 orq %rdx, %rax 2681 ret 2682 SET_SIZE(mul32) 2683 2684#elif defined(__i386) 2685 2686 ENTRY(mul32) 2687 movl 8(%esp), %eax 2688 movl 4(%esp), %ecx 2689 mull %ecx 2690 ret 2691 SET_SIZE(mul32) 2692 2693#endif /* __i386 */ 2694#endif /* __lint */ 2695 2696#if defined(notused) 2697#if defined(__lint) 2698/* ARGSUSED */ 2699void 2700load_pte64(uint64_t *pte, uint64_t pte_value) 2701{} 2702#else /* __lint */ 2703 .globl load_pte64 2704load_pte64: 2705 movl 4(%esp), %eax 2706 movl 8(%esp), %ecx 2707 movl 12(%esp), %edx 2708 movl %edx, 4(%eax) 2709 movl %ecx, (%eax) 2710 ret 2711#endif /* __lint */ 2712#endif /* notused */ 2713 2714#if defined(__lint) 2715 2716/*ARGSUSED*/ 2717void 2718scan_memory(caddr_t addr, size_t size) 2719{} 2720 2721#else /* __lint */ 2722 2723#if defined(__amd64) 2724 2725 ENTRY(scan_memory) 2726 shrq $3, %rsi /* convert %rsi from byte to quadword count */ 2727 jz .scanm_done 2728 movq %rsi, %rcx /* move count into rep control register */ 2729 movq %rdi, %rsi /* move addr into lodsq control reg. */ 2730 rep lodsq /* scan the memory range */ 2731.scanm_done: 2732 rep; ret /* use 2 byte return instruction when branch target */ 2733 /* AMD Software Optimization Guide - Section 6.2 */ 2734 SET_SIZE(scan_memory) 2735 2736#elif defined(__i386) 2737 2738 ENTRY(scan_memory) 2739 pushl %ecx 2740 pushl %esi 2741 movl 16(%esp), %ecx /* move 2nd arg into rep control register */ 2742 shrl $2, %ecx /* convert from byte count to word count */ 2743 jz .scanm_done 2744 movl 12(%esp), %esi /* move 1st arg into lodsw control register */ 2745 .byte 0xf3 /* rep prefix. lame assembler. sigh. */ 2746 lodsl 2747.scanm_done: 2748 popl %esi 2749 popl %ecx 2750 ret 2751 SET_SIZE(scan_memory) 2752 2753#endif /* __i386 */ 2754#endif /* __lint */ 2755 2756 2757#if defined(__lint) 2758 2759/*ARGSUSED */ 2760int 2761lowbit(ulong_t i) 2762{ return (0); } 2763 2764#else /* __lint */ 2765 2766#if defined(__amd64) 2767 2768 ENTRY(lowbit) 2769 movl $-1, %eax 2770 bsfq %rdi, %rax 2771 incl %eax 2772 ret 2773 SET_SIZE(lowbit) 2774 2775#elif defined(__i386) 2776 2777 ENTRY(lowbit) 2778 movl $-1, %eax 2779 bsfl 4(%esp), %eax 2780 incl %eax 2781 ret 2782 SET_SIZE(lowbit) 2783 2784#endif /* __i386 */ 2785#endif /* __lint */ 2786 2787#if defined(__lint) 2788 2789/*ARGSUSED*/ 2790int 2791highbit(ulong_t i) 2792{ return (0); } 2793 2794#else /* __lint */ 2795 2796#if defined(__amd64) 2797 2798 ENTRY(highbit) 2799 movl $-1, %eax 2800 bsrq %rdi, %rax 2801 incl %eax 2802 ret 2803 SET_SIZE(highbit) 2804 2805#elif defined(__i386) 2806 2807 ENTRY(highbit) 2808 movl $-1, %eax 2809 bsrl 4(%esp), %eax 2810 incl %eax 2811 ret 2812 SET_SIZE(highbit) 2813 2814#endif /* __i386 */ 2815#endif /* __lint */ 2816 2817#if defined(__lint) 2818 2819/*ARGSUSED*/ 2820uint64_t 2821rdmsr(uint_t r) 2822{ return (0); } 2823 2824/*ARGSUSED*/ 2825void 2826wrmsr(uint_t r, const uint64_t val) 2827{} 2828 2829/*ARGSUSED*/ 2830uint64_t 2831xrdmsr(uint_t r) 2832{ return (0); } 2833 2834/*ARGSUSED*/ 2835void 2836xwrmsr(uint_t r, const uint64_t val) 2837{} 2838 2839void 2840invalidate_cache(void) 2841{} 2842 2843#else /* __lint */ 2844 2845#define XMSR_ACCESS_VAL $0x9c5a203a 2846 2847#if defined(__amd64) 2848 2849 ENTRY(rdmsr) 2850 movl %edi, %ecx 2851 rdmsr 2852 shlq $32, %rdx 2853 orq %rdx, %rax 2854 ret 2855 SET_SIZE(rdmsr) 2856 2857 ENTRY(wrmsr) 2858 movq %rsi, %rdx 2859 shrq $32, %rdx 2860 movl %esi, %eax 2861 movl %edi, %ecx 2862 wrmsr 2863 ret 2864 SET_SIZE(wrmsr) 2865 2866 ENTRY(xrdmsr) 2867 pushq %rbp 2868 movq %rsp, %rbp 2869 movl %edi, %ecx 2870 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2871 rdmsr 2872 shlq $32, %rdx 2873 orq %rdx, %rax 2874 leave 2875 ret 2876 SET_SIZE(xrdmsr) 2877 2878 ENTRY(xwrmsr) 2879 pushq %rbp 2880 movq %rsp, %rbp 2881 movl %edi, %ecx 2882 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2883 movq %rsi, %rdx 2884 shrq $32, %rdx 2885 movl %esi, %eax 2886 wrmsr 2887 leave 2888 ret 2889 SET_SIZE(xwrmsr) 2890 2891#elif defined(__i386) 2892 2893 ENTRY(rdmsr) 2894 movl 4(%esp), %ecx 2895 rdmsr 2896 ret 2897 SET_SIZE(rdmsr) 2898 2899 ENTRY(wrmsr) 2900 movl 4(%esp), %ecx 2901 movl 8(%esp), %eax 2902 movl 12(%esp), %edx 2903 wrmsr 2904 ret 2905 SET_SIZE(wrmsr) 2906 2907 ENTRY(xrdmsr) 2908 pushl %ebp 2909 movl %esp, %ebp 2910 movl 8(%esp), %ecx 2911 pushl %edi 2912 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2913 rdmsr 2914 popl %edi 2915 leave 2916 ret 2917 SET_SIZE(xrdmsr) 2918 2919 ENTRY(xwrmsr) 2920 pushl %ebp 2921 movl %esp, %ebp 2922 movl 8(%esp), %ecx 2923 movl 12(%esp), %eax 2924 movl 16(%esp), %edx 2925 pushl %edi 2926 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 2927 wrmsr 2928 popl %edi 2929 leave 2930 ret 2931 SET_SIZE(xwrmsr) 2932 2933#endif /* __i386 */ 2934 2935 ENTRY(invalidate_cache) 2936 wbinvd 2937 ret 2938 SET_SIZE(invalidate_cache) 2939 2940#endif /* __lint */ 2941 2942#if defined(__lint) 2943 2944/*ARGSUSED*/ 2945void 2946getcregs(struct cregs *crp) 2947{} 2948 2949#else /* __lint */ 2950 2951#if defined(__amd64) 2952 2953 ENTRY_NP(getcregs) 2954#if defined(__xpv) 2955 /* 2956 * Only a few of the hardware control registers or descriptor tables 2957 * are directly accessible to us, so just zero the structure. 2958 * 2959 * XXPV Perhaps it would be helpful for the hypervisor to return 2960 * virtualized versions of these for post-mortem use. 2961 * (Need to reevaluate - perhaps it already does!) 2962 */ 2963 pushq %rdi /* save *crp */ 2964 movq $CREGSZ, %rsi 2965 call bzero 2966 popq %rdi 2967 2968 /* 2969 * Dump what limited information we can 2970 */ 2971 movq %cr0, %rax 2972 movq %rax, CREG_CR0(%rdi) /* cr0 */ 2973 movq %cr2, %rax 2974 movq %rax, CREG_CR2(%rdi) /* cr2 */ 2975 movq %cr3, %rax 2976 movq %rax, CREG_CR3(%rdi) /* cr3 */ 2977 movq %cr4, %rax 2978 movq %rax, CREG_CR4(%rdi) /* cr4 */ 2979 2980#else /* __xpv */ 2981 2982#define GETMSR(r, off, d) \ 2983 movl $r, %ecx; \ 2984 rdmsr; \ 2985 movl %eax, off(d); \ 2986 movl %edx, off+4(d) 2987 2988 xorl %eax, %eax 2989 movq %rax, CREG_GDT+8(%rdi) 2990 sgdt CREG_GDT(%rdi) /* 10 bytes */ 2991 movq %rax, CREG_IDT+8(%rdi) 2992 sidt CREG_IDT(%rdi) /* 10 bytes */ 2993 movq %rax, CREG_LDT(%rdi) 2994 sldt CREG_LDT(%rdi) /* 2 bytes */ 2995 movq %rax, CREG_TASKR(%rdi) 2996 str CREG_TASKR(%rdi) /* 2 bytes */ 2997 movq %cr0, %rax 2998 movq %rax, CREG_CR0(%rdi) /* cr0 */ 2999 movq %cr2, %rax 3000 movq %rax, CREG_CR2(%rdi) /* cr2 */ 3001 movq %cr3, %rax 3002 movq %rax, CREG_CR3(%rdi) /* cr3 */ 3003 movq %cr4, %rax 3004 movq %rax, CREG_CR4(%rdi) /* cr4 */ 3005 movq %cr8, %rax 3006 movq %rax, CREG_CR8(%rdi) /* cr8 */ 3007 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi) 3008 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi) 3009#endif /* __xpv */ 3010 ret 3011 SET_SIZE(getcregs) 3012 3013#undef GETMSR 3014 3015#elif defined(__i386) 3016 3017 ENTRY_NP(getcregs) 3018#if defined(__xpv) 3019 /* 3020 * Only a few of the hardware control registers or descriptor tables 3021 * are directly accessible to us, so just zero the structure. 3022 * 3023 * XXPV Perhaps it would be helpful for the hypervisor to return 3024 * virtualized versions of these for post-mortem use. 3025 * (Need to reevaluate - perhaps it already does!) 3026 */ 3027 movl 4(%esp), %edx 3028 pushl $CREGSZ 3029 pushl %edx 3030 call bzero 3031 addl $8, %esp 3032 movl 4(%esp), %edx 3033 3034 /* 3035 * Dump what limited information we can 3036 */ 3037 movl %cr0, %eax 3038 movl %eax, CREG_CR0(%edx) /* cr0 */ 3039 movl %cr2, %eax 3040 movl %eax, CREG_CR2(%edx) /* cr2 */ 3041 movl %cr3, %eax 3042 movl %eax, CREG_CR3(%edx) /* cr3 */ 3043 movl %cr4, %eax 3044 movl %eax, CREG_CR4(%edx) /* cr4 */ 3045 3046#else /* __xpv */ 3047 3048 movl 4(%esp), %edx 3049 movw $0, CREG_GDT+6(%edx) 3050 movw $0, CREG_IDT+6(%edx) 3051 sgdt CREG_GDT(%edx) /* gdt */ 3052 sidt CREG_IDT(%edx) /* idt */ 3053 sldt CREG_LDT(%edx) /* ldt */ 3054 str CREG_TASKR(%edx) /* task */ 3055 movl %cr0, %eax 3056 movl %eax, CREG_CR0(%edx) /* cr0 */ 3057 movl %cr2, %eax 3058 movl %eax, CREG_CR2(%edx) /* cr2 */ 3059 movl %cr3, %eax 3060 movl %eax, CREG_CR3(%edx) /* cr3 */ 3061 testl $X86_LARGEPAGE, x86_feature 3062 jz .nocr4 3063 movl %cr4, %eax 3064 movl %eax, CREG_CR4(%edx) /* cr4 */ 3065 jmp .skip 3066.nocr4: 3067 movl $0, CREG_CR4(%edx) 3068.skip: 3069#endif 3070 ret 3071 SET_SIZE(getcregs) 3072 3073#endif /* __i386 */ 3074#endif /* __lint */ 3075 3076 3077/* 3078 * A panic trigger is a word which is updated atomically and can only be set 3079 * once. We atomically store 0xDEFACEDD and load the old value. If the 3080 * previous value was 0, we succeed and return 1; otherwise return 0. 3081 * This allows a partially corrupt trigger to still trigger correctly. DTrace 3082 * has its own version of this function to allow it to panic correctly from 3083 * probe context. 3084 */ 3085#if defined(__lint) 3086 3087/*ARGSUSED*/ 3088int 3089panic_trigger(int *tp) 3090{ return (0); } 3091 3092/*ARGSUSED*/ 3093int 3094dtrace_panic_trigger(int *tp) 3095{ return (0); } 3096 3097#else /* __lint */ 3098 3099#if defined(__amd64) 3100 3101 ENTRY_NP(panic_trigger) 3102 xorl %eax, %eax 3103 movl $0xdefacedd, %edx 3104 lock 3105 xchgl %edx, (%rdi) 3106 cmpl $0, %edx 3107 je 0f 3108 movl $0, %eax 3109 ret 31100: movl $1, %eax 3111 ret 3112 SET_SIZE(panic_trigger) 3113 3114 ENTRY_NP(dtrace_panic_trigger) 3115 xorl %eax, %eax 3116 movl $0xdefacedd, %edx 3117 lock 3118 xchgl %edx, (%rdi) 3119 cmpl $0, %edx 3120 je 0f 3121 movl $0, %eax 3122 ret 31230: movl $1, %eax 3124 ret 3125 SET_SIZE(dtrace_panic_trigger) 3126 3127#elif defined(__i386) 3128 3129 ENTRY_NP(panic_trigger) 3130 movl 4(%esp), %edx / %edx = address of trigger 3131 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3132 lock / assert lock 3133 xchgl %eax, (%edx) / exchange %eax and the trigger 3134 cmpl $0, %eax / if (%eax == 0x0) 3135 je 0f / return (1); 3136 movl $0, %eax / else 3137 ret / return (0); 31380: movl $1, %eax 3139 ret 3140 SET_SIZE(panic_trigger) 3141 3142 ENTRY_NP(dtrace_panic_trigger) 3143 movl 4(%esp), %edx / %edx = address of trigger 3144 movl $0xdefacedd, %eax / %eax = 0xdefacedd 3145 lock / assert lock 3146 xchgl %eax, (%edx) / exchange %eax and the trigger 3147 cmpl $0, %eax / if (%eax == 0x0) 3148 je 0f / return (1); 3149 movl $0, %eax / else 3150 ret / return (0); 31510: movl $1, %eax 3152 ret 3153 SET_SIZE(dtrace_panic_trigger) 3154 3155#endif /* __i386 */ 3156#endif /* __lint */ 3157 3158/* 3159 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 3160 * into the panic code implemented in panicsys(). vpanic() is responsible 3161 * for passing through the format string and arguments, and constructing a 3162 * regs structure on the stack into which it saves the current register 3163 * values. If we are not dying due to a fatal trap, these registers will 3164 * then be preserved in panicbuf as the current processor state. Before 3165 * invoking panicsys(), vpanic() activates the first panic trigger (see 3166 * common/os/panic.c) and switches to the panic_stack if successful. Note that 3167 * DTrace takes a slightly different panic path if it must panic from probe 3168 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 3169 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 3170 * branches back into vpanic(). 3171 */ 3172#if defined(__lint) 3173 3174/*ARGSUSED*/ 3175void 3176vpanic(const char *format, va_list alist) 3177{} 3178 3179/*ARGSUSED*/ 3180void 3181dtrace_vpanic(const char *format, va_list alist) 3182{} 3183 3184#else /* __lint */ 3185 3186#if defined(__amd64) 3187 3188 ENTRY_NP(vpanic) /* Initial stack layout: */ 3189 3190 pushq %rbp /* | %rip | 0x60 */ 3191 movq %rsp, %rbp /* | %rbp | 0x58 */ 3192 pushfq /* | rfl | 0x50 */ 3193 pushq %r11 /* | %r11 | 0x48 */ 3194 pushq %r10 /* | %r10 | 0x40 */ 3195 pushq %rbx /* | %rbx | 0x38 */ 3196 pushq %rax /* | %rax | 0x30 */ 3197 pushq %r9 /* | %r9 | 0x28 */ 3198 pushq %r8 /* | %r8 | 0x20 */ 3199 pushq %rcx /* | %rcx | 0x18 */ 3200 pushq %rdx /* | %rdx | 0x10 */ 3201 pushq %rsi /* | %rsi | 0x8 alist */ 3202 pushq %rdi /* | %rdi | 0x0 format */ 3203 3204 movq %rsp, %rbx /* %rbx = current %rsp */ 3205 3206 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3207 call panic_trigger /* %eax = panic_trigger() */ 3208 3209vpanic_common: 3210 /* 3211 * The panic_trigger result is in %eax from the call above, and 3212 * dtrace_panic places it in %eax before branching here. 3213 * The rdmsr instructions that follow below will clobber %eax so 3214 * we stash the panic_trigger result in %r11d. 3215 */ 3216 movl %eax, %r11d 3217 cmpl $0, %r11d 3218 je 0f 3219 3220 /* 3221 * If panic_trigger() was successful, we are the first to initiate a 3222 * panic: we now switch to the reserved panic_stack before continuing. 3223 */ 3224 leaq panic_stack(%rip), %rsp 3225 addq $PANICSTKSIZE, %rsp 32260: subq $REGSIZE, %rsp 3227 /* 3228 * Now that we've got everything set up, store the register values as 3229 * they were when we entered vpanic() to the designated location in 3230 * the regs structure we allocated on the stack. 3231 */ 3232 movq 0x0(%rbx), %rcx 3233 movq %rcx, REGOFF_RDI(%rsp) 3234 movq 0x8(%rbx), %rcx 3235 movq %rcx, REGOFF_RSI(%rsp) 3236 movq 0x10(%rbx), %rcx 3237 movq %rcx, REGOFF_RDX(%rsp) 3238 movq 0x18(%rbx), %rcx 3239 movq %rcx, REGOFF_RCX(%rsp) 3240 movq 0x20(%rbx), %rcx 3241 3242 movq %rcx, REGOFF_R8(%rsp) 3243 movq 0x28(%rbx), %rcx 3244 movq %rcx, REGOFF_R9(%rsp) 3245 movq 0x30(%rbx), %rcx 3246 movq %rcx, REGOFF_RAX(%rsp) 3247 movq 0x38(%rbx), %rcx 3248 movq %rcx, REGOFF_RBX(%rsp) 3249 movq 0x58(%rbx), %rcx 3250 3251 movq %rcx, REGOFF_RBP(%rsp) 3252 movq 0x40(%rbx), %rcx 3253 movq %rcx, REGOFF_R10(%rsp) 3254 movq 0x48(%rbx), %rcx 3255 movq %rcx, REGOFF_R11(%rsp) 3256 movq %r12, REGOFF_R12(%rsp) 3257 3258 movq %r13, REGOFF_R13(%rsp) 3259 movq %r14, REGOFF_R14(%rsp) 3260 movq %r15, REGOFF_R15(%rsp) 3261 3262 xorl %ecx, %ecx 3263 movw %ds, %cx 3264 movq %rcx, REGOFF_DS(%rsp) 3265 movw %es, %cx 3266 movq %rcx, REGOFF_ES(%rsp) 3267 movw %fs, %cx 3268 movq %rcx, REGOFF_FS(%rsp) 3269 movw %gs, %cx 3270 movq %rcx, REGOFF_GS(%rsp) 3271 3272 movq $0, REGOFF_TRAPNO(%rsp) 3273 3274 movq $0, REGOFF_ERR(%rsp) 3275 leaq vpanic(%rip), %rcx 3276 movq %rcx, REGOFF_RIP(%rsp) 3277 movw %cs, %cx 3278 movzwq %cx, %rcx 3279 movq %rcx, REGOFF_CS(%rsp) 3280 movq 0x50(%rbx), %rcx 3281 movq %rcx, REGOFF_RFL(%rsp) 3282 movq %rbx, %rcx 3283 addq $0x60, %rcx 3284 movq %rcx, REGOFF_RSP(%rsp) 3285 movw %ss, %cx 3286 movzwq %cx, %rcx 3287 movq %rcx, REGOFF_SS(%rsp) 3288 3289 /* 3290 * panicsys(format, alist, rp, on_panic_stack) 3291 */ 3292 movq REGOFF_RDI(%rsp), %rdi /* format */ 3293 movq REGOFF_RSI(%rsp), %rsi /* alist */ 3294 movq %rsp, %rdx /* struct regs */ 3295 movl %r11d, %ecx /* on_panic_stack */ 3296 call panicsys 3297 addq $REGSIZE, %rsp 3298 popq %rdi 3299 popq %rsi 3300 popq %rdx 3301 popq %rcx 3302 popq %r8 3303 popq %r9 3304 popq %rax 3305 popq %rbx 3306 popq %r10 3307 popq %r11 3308 popfq 3309 leave 3310 ret 3311 SET_SIZE(vpanic) 3312 3313 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */ 3314 3315 pushq %rbp /* | %rip | 0x60 */ 3316 movq %rsp, %rbp /* | %rbp | 0x58 */ 3317 pushfq /* | rfl | 0x50 */ 3318 pushq %r11 /* | %r11 | 0x48 */ 3319 pushq %r10 /* | %r10 | 0x40 */ 3320 pushq %rbx /* | %rbx | 0x38 */ 3321 pushq %rax /* | %rax | 0x30 */ 3322 pushq %r9 /* | %r9 | 0x28 */ 3323 pushq %r8 /* | %r8 | 0x20 */ 3324 pushq %rcx /* | %rcx | 0x18 */ 3325 pushq %rdx /* | %rdx | 0x10 */ 3326 pushq %rsi /* | %rsi | 0x8 alist */ 3327 pushq %rdi /* | %rdi | 0x0 format */ 3328 3329 movq %rsp, %rbx /* %rbx = current %rsp */ 3330 3331 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 3332 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 3333 jmp vpanic_common 3334 3335 SET_SIZE(dtrace_vpanic) 3336 3337#elif defined(__i386) 3338 3339 ENTRY_NP(vpanic) / Initial stack layout: 3340 3341 pushl %ebp / | %eip | 20 3342 movl %esp, %ebp / | %ebp | 16 3343 pushl %eax / | %eax | 12 3344 pushl %ebx / | %ebx | 8 3345 pushl %ecx / | %ecx | 4 3346 pushl %edx / | %edx | 0 3347 3348 movl %esp, %ebx / %ebx = current stack pointer 3349 3350 lea panic_quiesce, %eax / %eax = &panic_quiesce 3351 pushl %eax / push &panic_quiesce 3352 call panic_trigger / %eax = panic_trigger() 3353 addl $4, %esp / reset stack pointer 3354 3355vpanic_common: 3356 cmpl $0, %eax / if (%eax == 0) 3357 je 0f / goto 0f; 3358 3359 /* 3360 * If panic_trigger() was successful, we are the first to initiate a 3361 * panic: we now switch to the reserved panic_stack before continuing. 3362 */ 3363 lea panic_stack, %esp / %esp = panic_stack 3364 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE 3365 33660: subl $REGSIZE, %esp / allocate struct regs 3367 3368 /* 3369 * Now that we've got everything set up, store the register values as 3370 * they were when we entered vpanic() to the designated location in 3371 * the regs structure we allocated on the stack. 3372 */ 3373#if !defined(__GNUC_AS__) 3374 movw %gs, %edx 3375 movl %edx, REGOFF_GS(%esp) 3376 movw %fs, %edx 3377 movl %edx, REGOFF_FS(%esp) 3378 movw %es, %edx 3379 movl %edx, REGOFF_ES(%esp) 3380 movw %ds, %edx 3381 movl %edx, REGOFF_DS(%esp) 3382#else /* __GNUC_AS__ */ 3383 mov %gs, %edx 3384 mov %edx, REGOFF_GS(%esp) 3385 mov %fs, %edx 3386 mov %edx, REGOFF_FS(%esp) 3387 mov %es, %edx 3388 mov %edx, REGOFF_ES(%esp) 3389 mov %ds, %edx 3390 mov %edx, REGOFF_DS(%esp) 3391#endif /* __GNUC_AS__ */ 3392 movl %edi, REGOFF_EDI(%esp) 3393 movl %esi, REGOFF_ESI(%esp) 3394 movl 16(%ebx), %ecx 3395 movl %ecx, REGOFF_EBP(%esp) 3396 movl %ebx, %ecx 3397 addl $20, %ecx 3398 movl %ecx, REGOFF_ESP(%esp) 3399 movl 8(%ebx), %ecx 3400 movl %ecx, REGOFF_EBX(%esp) 3401 movl 0(%ebx), %ecx 3402 movl %ecx, REGOFF_EDX(%esp) 3403 movl 4(%ebx), %ecx 3404 movl %ecx, REGOFF_ECX(%esp) 3405 movl 12(%ebx), %ecx 3406 movl %ecx, REGOFF_EAX(%esp) 3407 movl $0, REGOFF_TRAPNO(%esp) 3408 movl $0, REGOFF_ERR(%esp) 3409 lea vpanic, %ecx 3410 movl %ecx, REGOFF_EIP(%esp) 3411#if !defined(__GNUC_AS__) 3412 movw %cs, %edx 3413#else /* __GNUC_AS__ */ 3414 mov %cs, %edx 3415#endif /* __GNUC_AS__ */ 3416 movl %edx, REGOFF_CS(%esp) 3417 pushfl 3418 popl %ecx 3419#if defined(__xpv) 3420 /* 3421 * Synthesize the PS_IE bit from the event mask bit 3422 */ 3423 CURTHREAD(%edx) 3424 KPREEMPT_DISABLE(%edx) 3425 EVENT_MASK_TO_IE(%edx, %ecx) 3426 CURTHREAD(%edx) 3427 KPREEMPT_ENABLE_NOKP(%edx) 3428#endif 3429 movl %ecx, REGOFF_EFL(%esp) 3430 movl $0, REGOFF_UESP(%esp) 3431#if !defined(__GNUC_AS__) 3432 movw %ss, %edx 3433#else /* __GNUC_AS__ */ 3434 mov %ss, %edx 3435#endif /* __GNUC_AS__ */ 3436 movl %edx, REGOFF_SS(%esp) 3437 3438 movl %esp, %ecx / %ecx = ®s 3439 pushl %eax / push on_panic_stack 3440 pushl %ecx / push ®s 3441 movl 12(%ebp), %ecx / %ecx = alist 3442 pushl %ecx / push alist 3443 movl 8(%ebp), %ecx / %ecx = format 3444 pushl %ecx / push format 3445 call panicsys / panicsys(); 3446 addl $16, %esp / pop arguments 3447 3448 addl $REGSIZE, %esp 3449 popl %edx 3450 popl %ecx 3451 popl %ebx 3452 popl %eax 3453 leave 3454 ret 3455 SET_SIZE(vpanic) 3456 3457 ENTRY_NP(dtrace_vpanic) / Initial stack layout: 3458 3459 pushl %ebp / | %eip | 20 3460 movl %esp, %ebp / | %ebp | 16 3461 pushl %eax / | %eax | 12 3462 pushl %ebx / | %ebx | 8 3463 pushl %ecx / | %ecx | 4 3464 pushl %edx / | %edx | 0 3465 3466 movl %esp, %ebx / %ebx = current stack pointer 3467 3468 lea panic_quiesce, %eax / %eax = &panic_quiesce 3469 pushl %eax / push &panic_quiesce 3470 call dtrace_panic_trigger / %eax = dtrace_panic_trigger() 3471 addl $4, %esp / reset stack pointer 3472 jmp vpanic_common / jump back to common code 3473 3474 SET_SIZE(dtrace_vpanic) 3475 3476#endif /* __i386 */ 3477#endif /* __lint */ 3478 3479#if defined(__lint) 3480 3481void 3482hres_tick(void) 3483{} 3484 3485int64_t timedelta; 3486hrtime_t hres_last_tick; 3487volatile timestruc_t hrestime; 3488int64_t hrestime_adj; 3489volatile int hres_lock; 3490hrtime_t hrtime_base; 3491 3492#else /* __lint */ 3493 3494 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8) 3495 .NWORD 0, 0 3496 3497 DGDEF3(hrestime_adj, 8, 8) 3498 .long 0, 0 3499 3500 DGDEF3(hres_last_tick, 8, 8) 3501 .long 0, 0 3502 3503 DGDEF3(timedelta, 8, 8) 3504 .long 0, 0 3505 3506 DGDEF3(hres_lock, 4, 8) 3507 .long 0 3508 3509 /* 3510 * initialized to a non zero value to make pc_gethrtime() 3511 * work correctly even before clock is initialized 3512 */ 3513 DGDEF3(hrtime_base, 8, 8) 3514 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0 3515 3516 DGDEF3(adj_shift, 4, 4) 3517 .long ADJ_SHIFT 3518 3519#if defined(__amd64) 3520 3521 ENTRY_NP(hres_tick) 3522 pushq %rbp 3523 movq %rsp, %rbp 3524 3525 /* 3526 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3527 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3528 * At worst, performing this now instead of under CLOCK_LOCK may 3529 * introduce some jitter in pc_gethrestime(). 3530 */ 3531 call *gethrtimef(%rip) 3532 movq %rax, %r8 3533 3534 leaq hres_lock(%rip), %rax 3535 movb $-1, %dl 3536.CL1: 3537 xchgb %dl, (%rax) 3538 testb %dl, %dl 3539 jz .CL3 /* got it */ 3540.CL2: 3541 cmpb $0, (%rax) /* possible to get lock? */ 3542 pause 3543 jne .CL2 3544 jmp .CL1 /* yes, try again */ 3545.CL3: 3546 /* 3547 * compute the interval since last time hres_tick was called 3548 * and adjust hrtime_base and hrestime accordingly 3549 * hrtime_base is an 8 byte value (in nsec), hrestime is 3550 * a timestruc_t (sec, nsec) 3551 */ 3552 leaq hres_last_tick(%rip), %rax 3553 movq %r8, %r11 3554 subq (%rax), %r8 3555 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */ 3556 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */ 3557 /* 3558 * Now that we have CLOCK_LOCK, we can update hres_last_tick 3559 */ 3560 movq %r11, (%rax) 3561 3562 call __adj_hrestime 3563 3564 /* 3565 * release the hres_lock 3566 */ 3567 incl hres_lock(%rip) 3568 leave 3569 ret 3570 SET_SIZE(hres_tick) 3571 3572#elif defined(__i386) 3573 3574 ENTRY_NP(hres_tick) 3575 pushl %ebp 3576 movl %esp, %ebp 3577 pushl %esi 3578 pushl %ebx 3579 3580 /* 3581 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 3582 * hres_last_tick can only be modified while holding CLOCK_LOCK). 3583 * At worst, performing this now instead of under CLOCK_LOCK may 3584 * introduce some jitter in pc_gethrestime(). 3585 */ 3586 call *gethrtimef 3587 movl %eax, %ebx 3588 movl %edx, %esi 3589 3590 movl $hres_lock, %eax 3591 movl $-1, %edx 3592.CL1: 3593 xchgb %dl, (%eax) 3594 testb %dl, %dl 3595 jz .CL3 / got it 3596.CL2: 3597 cmpb $0, (%eax) / possible to get lock? 3598 pause 3599 jne .CL2 3600 jmp .CL1 / yes, try again 3601.CL3: 3602 /* 3603 * compute the interval since last time hres_tick was called 3604 * and adjust hrtime_base and hrestime accordingly 3605 * hrtime_base is an 8 byte value (in nsec), hrestime is 3606 * timestruc_t (sec, nsec) 3607 */ 3608 3609 lea hres_last_tick, %eax 3610 3611 movl %ebx, %edx 3612 movl %esi, %ecx 3613 3614 subl (%eax), %edx 3615 sbbl 4(%eax), %ecx 3616 3617 addl %edx, hrtime_base / add interval to hrtime_base 3618 adcl %ecx, hrtime_base+4 3619 3620 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec 3621 3622 / 3623 / Now that we have CLOCK_LOCK, we can update hres_last_tick. 3624 / 3625 movl %ebx, (%eax) 3626 movl %esi, 4(%eax) 3627 3628 / get hrestime at this moment. used as base for pc_gethrestime 3629 / 3630 / Apply adjustment, if any 3631 / 3632 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT) 3633 / (max_hres_adj) 3634 / 3635 / void 3636 / adj_hrestime() 3637 / { 3638 / long long adj; 3639 / 3640 / if (hrestime_adj == 0) 3641 / adj = 0; 3642 / else if (hrestime_adj > 0) { 3643 / if (hrestime_adj < HRES_ADJ) 3644 / adj = hrestime_adj; 3645 / else 3646 / adj = HRES_ADJ; 3647 / } 3648 / else { 3649 / if (hrestime_adj < -(HRES_ADJ)) 3650 / adj = -(HRES_ADJ); 3651 / else 3652 / adj = hrestime_adj; 3653 / } 3654 / 3655 / timedelta -= adj; 3656 / hrestime_adj = timedelta; 3657 / hrestime.tv_nsec += adj; 3658 / 3659 / while (hrestime.tv_nsec >= NANOSEC) { 3660 / one_sec++; 3661 / hrestime.tv_sec++; 3662 / hrestime.tv_nsec -= NANOSEC; 3663 / } 3664 / } 3665__adj_hrestime: 3666 movl hrestime_adj, %esi / if (hrestime_adj == 0) 3667 movl hrestime_adj+4, %edx 3668 andl %esi, %esi 3669 jne .CL4 / no 3670 andl %edx, %edx 3671 jne .CL4 / no 3672 subl %ecx, %ecx / yes, adj = 0; 3673 subl %edx, %edx 3674 jmp .CL5 3675.CL4: 3676 subl %ecx, %ecx 3677 subl %eax, %eax 3678 subl %esi, %ecx 3679 sbbl %edx, %eax 3680 andl %eax, %eax / if (hrestime_adj > 0) 3681 jge .CL6 3682 3683 / In the following comments, HRES_ADJ is used, while in the code 3684 / max_hres_adj is used. 3685 / 3686 / The test for "hrestime_adj < HRES_ADJ" is complicated because 3687 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3688 / on the logical equivalence of: 3689 / 3690 / !(hrestime_adj < HRES_ADJ) 3691 / 3692 / and the two step sequence: 3693 / 3694 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry 3695 / 3696 / which computes whether or not the least significant 32-bits 3697 / of hrestime_adj is greater than HRES_ADJ, followed by: 3698 / 3699 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry 3700 / 3701 / which generates a carry whenever step 1 is true or the most 3702 / significant long of the longlong hrestime_adj is non-zero. 3703 3704 movl max_hres_adj, %ecx / hrestime_adj is positive 3705 subl %esi, %ecx 3706 movl %edx, %eax 3707 adcl $-1, %eax 3708 jnc .CL7 3709 movl max_hres_adj, %ecx / adj = HRES_ADJ; 3710 subl %edx, %edx 3711 jmp .CL5 3712 3713 / The following computation is similar to the one above. 3714 / 3715 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because 3716 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely 3717 / on the logical equivalence of: 3718 / 3719 / (hrestime_adj > -HRES_ADJ) 3720 / 3721 / and the two step sequence: 3722 / 3723 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry 3724 / 3725 / which means the least significant 32-bits of hrestime_adj is 3726 / greater than -HRES_ADJ, followed by: 3727 / 3728 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry 3729 / 3730 / which generates a carry only when step 1 is true and the most 3731 / significant long of the longlong hrestime_adj is -1. 3732 3733.CL6: / hrestime_adj is negative 3734 movl %esi, %ecx 3735 addl max_hres_adj, %ecx 3736 movl %edx, %eax 3737 adcl $0, %eax 3738 jc .CL7 3739 xor %ecx, %ecx 3740 subl max_hres_adj, %ecx / adj = -(HRES_ADJ); 3741 movl $-1, %edx 3742 jmp .CL5 3743.CL7: 3744 movl %esi, %ecx / adj = hrestime_adj; 3745.CL5: 3746 movl timedelta, %esi 3747 subl %ecx, %esi 3748 movl timedelta+4, %eax 3749 sbbl %edx, %eax 3750 movl %esi, timedelta 3751 movl %eax, timedelta+4 / timedelta -= adj; 3752 movl %esi, hrestime_adj 3753 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta; 3754 addl hrestime+4, %ecx 3755 3756 movl %ecx, %eax / eax = tv_nsec 37571: 3758 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC) 3759 jb .CL8 / no 3760 incl one_sec / yes, one_sec++; 3761 incl hrestime / hrestime.tv_sec++; 3762 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC 3763 jmp 1b / check for more seconds 3764 3765.CL8: 3766 movl %eax, hrestime+4 / store final into hrestime.tv_nsec 3767 incl hres_lock / release the hres_lock 3768 3769 popl %ebx 3770 popl %esi 3771 leave 3772 ret 3773 SET_SIZE(hres_tick) 3774 3775#endif /* __i386 */ 3776#endif /* __lint */ 3777 3778/* 3779 * void prefetch_smap_w(void *) 3780 * 3781 * Prefetch ahead within a linear list of smap structures. 3782 * Not implemented for ia32. Stub for compatibility. 3783 */ 3784 3785#if defined(__lint) 3786 3787/*ARGSUSED*/ 3788void prefetch_smap_w(void *smp) 3789{} 3790 3791#else /* __lint */ 3792 3793 ENTRY(prefetch_smap_w) 3794 rep; ret /* use 2 byte return instruction when branch target */ 3795 /* AMD Software Optimization Guide - Section 6.2 */ 3796 SET_SIZE(prefetch_smap_w) 3797 3798#endif /* __lint */ 3799 3800/* 3801 * prefetch_page_r(page_t *) 3802 * issue prefetch instructions for a page_t 3803 */ 3804#if defined(__lint) 3805 3806/*ARGSUSED*/ 3807void 3808prefetch_page_r(void *pp) 3809{} 3810 3811#else /* __lint */ 3812 3813 ENTRY(prefetch_page_r) 3814 rep; ret /* use 2 byte return instruction when branch target */ 3815 /* AMD Software Optimization Guide - Section 6.2 */ 3816 SET_SIZE(prefetch_page_r) 3817 3818#endif /* __lint */ 3819 3820#if defined(__lint) 3821 3822/*ARGSUSED*/ 3823int 3824bcmp(const void *s1, const void *s2, size_t count) 3825{ return (0); } 3826 3827#else /* __lint */ 3828 3829#if defined(__amd64) 3830 3831 ENTRY(bcmp) 3832 pushq %rbp 3833 movq %rsp, %rbp 3834#ifdef DEBUG 3835 movq postbootkernelbase(%rip), %r11 3836 cmpq %r11, %rdi 3837 jb 0f 3838 cmpq %r11, %rsi 3839 jnb 1f 38400: leaq .bcmp_panic_msg(%rip), %rdi 3841 xorl %eax, %eax 3842 call panic 38431: 3844#endif /* DEBUG */ 3845 call memcmp 3846 testl %eax, %eax 3847 setne %dl 3848 leave 3849 movzbl %dl, %eax 3850 ret 3851 SET_SIZE(bcmp) 3852 3853#elif defined(__i386) 3854 3855#define ARG_S1 8 3856#define ARG_S2 12 3857#define ARG_LENGTH 16 3858 3859 ENTRY(bcmp) 3860 pushl %ebp 3861 movl %esp, %ebp / create new stack frame 3862#ifdef DEBUG 3863 movl postbootkernelbase, %eax 3864 cmpl %eax, ARG_S1(%ebp) 3865 jb 0f 3866 cmpl %eax, ARG_S2(%ebp) 3867 jnb 1f 38680: pushl $.bcmp_panic_msg 3869 call panic 38701: 3871#endif /* DEBUG */ 3872 3873 pushl %edi / save register variable 3874 movl ARG_S1(%ebp), %eax / %eax = address of string 1 3875 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2 3876 cmpl %eax, %ecx / if the same string 3877 je .equal / goto .equal 3878 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes 3879 cmpl $4, %edi / if %edi < 4 3880 jb .byte_check / goto .byte_check 3881 .align 4 3882.word_loop: 3883 movl (%ecx), %edx / move 1 word from (%ecx) to %edx 3884 leal -4(%edi), %edi / %edi -= 4 3885 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx 3886 jne .word_not_equal / if not equal, goto .word_not_equal 3887 leal 4(%ecx), %ecx / %ecx += 4 (next word) 3888 leal 4(%eax), %eax / %eax += 4 (next word) 3889 cmpl $4, %edi / if %edi >= 4 3890 jae .word_loop / goto .word_loop 3891.byte_check: 3892 cmpl $0, %edi / if %edi == 0 3893 je .equal / goto .equal 3894 jmp .byte_loop / goto .byte_loop (checks in bytes) 3895.word_not_equal: 3896 leal 4(%edi), %edi / %edi += 4 (post-decremented) 3897 .align 4 3898.byte_loop: 3899 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl 3900 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax) 3901 jne .not_equal / if not equal, goto .not_equal 3902 incl %ecx / %ecx++ (next byte) 3903 incl %eax / %eax++ (next byte) 3904 decl %edi / %edi-- 3905 jnz .byte_loop / if not zero, goto .byte_loop 3906.equal: 3907 xorl %eax, %eax / %eax = 0 3908 popl %edi / restore register variable 3909 leave / restore old stack frame 3910 ret / return (NULL) 3911 .align 4 3912.not_equal: 3913 movl $1, %eax / return 1 3914 popl %edi / restore register variable 3915 leave / restore old stack frame 3916 ret / return (NULL) 3917 SET_SIZE(bcmp) 3918 3919#endif /* __i386 */ 3920 3921#ifdef DEBUG 3922 .text 3923.bcmp_panic_msg: 3924 .string "bcmp: arguments below kernelbase" 3925#endif /* DEBUG */ 3926 3927#endif /* __lint */ 3928 3929#if defined(__lint) 3930 3931uint_t 3932bsrw_insn(uint16_t mask) 3933{ 3934 uint_t index = sizeof (mask) * NBBY - 1; 3935 3936 while ((mask & (1 << index)) == 0) 3937 index--; 3938 return (index); 3939} 3940 3941#else /* __lint */ 3942 3943#if defined(__amd64) 3944 3945 ENTRY_NP(bsrw_insn) 3946 xorl %eax, %eax 3947 bsrw %di, %ax 3948 ret 3949 SET_SIZE(bsrw_insn) 3950 3951#elif defined(__i386) 3952 3953 ENTRY_NP(bsrw_insn) 3954 movw 4(%esp), %cx 3955 xorl %eax, %eax 3956 bsrw %cx, %ax 3957 ret 3958 SET_SIZE(bsrw_insn) 3959 3960#endif /* __i386 */ 3961#endif /* __lint */ 3962 3963#if defined(__lint) 3964 3965uint_t 3966atomic_btr32(uint32_t *pending, uint_t pil) 3967{ 3968 return (*pending &= ~(1 << pil)); 3969} 3970 3971#else /* __lint */ 3972 3973#if defined(__i386) 3974 3975 ENTRY_NP(atomic_btr32) 3976 movl 4(%esp), %ecx 3977 movl 8(%esp), %edx 3978 xorl %eax, %eax 3979 lock 3980 btrl %edx, (%ecx) 3981 setc %al 3982 ret 3983 SET_SIZE(atomic_btr32) 3984 3985#endif /* __i386 */ 3986#endif /* __lint */ 3987 3988#if defined(__lint) 3989 3990/*ARGSUSED*/ 3991void 3992switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1, 3993 uint_t arg2) 3994{} 3995 3996#else /* __lint */ 3997 3998#if defined(__amd64) 3999 4000 ENTRY_NP(switch_sp_and_call) 4001 pushq %rbp 4002 movq %rsp, %rbp /* set up stack frame */ 4003 movq %rdi, %rsp /* switch stack pointer */ 4004 movq %rdx, %rdi /* pass func arg 1 */ 4005 movq %rsi, %r11 /* save function to call */ 4006 movq %rcx, %rsi /* pass func arg 2 */ 4007 call *%r11 /* call function */ 4008 leave /* restore stack */ 4009 ret 4010 SET_SIZE(switch_sp_and_call) 4011 4012#elif defined(__i386) 4013 4014 ENTRY_NP(switch_sp_and_call) 4015 pushl %ebp 4016 mov %esp, %ebp /* set up stack frame */ 4017 movl 8(%ebp), %esp /* switch stack pointer */ 4018 pushl 20(%ebp) /* push func arg 2 */ 4019 pushl 16(%ebp) /* push func arg 1 */ 4020 call *12(%ebp) /* call function */ 4021 addl $8, %esp /* pop arguments */ 4022 leave /* restore stack */ 4023 ret 4024 SET_SIZE(switch_sp_and_call) 4025 4026#endif /* __i386 */ 4027#endif /* __lint */ 4028 4029#if defined(__lint) 4030 4031void 4032kmdb_enter(void) 4033{} 4034 4035#else /* __lint */ 4036 4037#if defined(__amd64) 4038 4039 ENTRY_NP(kmdb_enter) 4040 pushq %rbp 4041 movq %rsp, %rbp 4042 4043 /* 4044 * Save flags, do a 'cli' then return the saved flags 4045 */ 4046 call intr_clear 4047 4048 int $T_DBGENTR 4049 4050 /* 4051 * Restore the saved flags 4052 */ 4053 movq %rax, %rdi 4054 call intr_restore 4055 4056 leave 4057 ret 4058 SET_SIZE(kmdb_enter) 4059 4060#elif defined(__i386) 4061 4062 ENTRY_NP(kmdb_enter) 4063 pushl %ebp 4064 movl %esp, %ebp 4065 4066 /* 4067 * Save flags, do a 'cli' then return the saved flags 4068 */ 4069 call intr_clear 4070 4071 int $T_DBGENTR 4072 4073 /* 4074 * Restore the saved flags 4075 */ 4076 pushl %eax 4077 call intr_restore 4078 addl $4, %esp 4079 4080 leave 4081 ret 4082 SET_SIZE(kmdb_enter) 4083 4084#endif /* __i386 */ 4085#endif /* __lint */ 4086 4087#if defined(__lint) 4088 4089void 4090return_instr(void) 4091{} 4092 4093#else /* __lint */ 4094 4095 ENTRY_NP(return_instr) 4096 rep; ret /* use 2 byte instruction when branch target */ 4097 /* AMD Software Optimization Guide - Section 6.2 */ 4098 SET_SIZE(return_instr) 4099 4100#endif /* __lint */ 4101 4102#if defined(__lint) 4103 4104ulong_t 4105getflags(void) 4106{ 4107 return (0); 4108} 4109 4110#else /* __lint */ 4111 4112#if defined(__amd64) 4113 4114 ENTRY(getflags) 4115 pushfq 4116 popq %rax 4117#if defined(__xpv) 4118 CURTHREAD(%rdi) 4119 KPREEMPT_DISABLE(%rdi) 4120 /* 4121 * Synthesize the PS_IE bit from the event mask bit 4122 */ 4123 CURVCPU(%r11) 4124 andq $_BITNOT(PS_IE), %rax 4125 XEN_TEST_UPCALL_MASK(%r11) 4126 jnz 1f 4127 orq $PS_IE, %rax 41281: 4129 KPREEMPT_ENABLE_NOKP(%rdi) 4130#endif 4131 ret 4132 SET_SIZE(getflags) 4133 4134#elif defined(__i386) 4135 4136 ENTRY(getflags) 4137 pushfl 4138 popl %eax 4139#if defined(__xpv) 4140 CURTHREAD(%ecx) 4141 KPREEMPT_DISABLE(%ecx) 4142 /* 4143 * Synthesize the PS_IE bit from the event mask bit 4144 */ 4145 CURVCPU(%edx) 4146 andl $_BITNOT(PS_IE), %eax 4147 XEN_TEST_UPCALL_MASK(%edx) 4148 jnz 1f 4149 orl $PS_IE, %eax 41501: 4151 KPREEMPT_ENABLE_NOKP(%ecx) 4152#endif 4153 ret 4154 SET_SIZE(getflags) 4155 4156#endif /* __i386 */ 4157 4158#endif /* __lint */ 4159 4160#if defined(__lint) 4161 4162ftrace_icookie_t 4163ftrace_interrupt_disable(void) 4164{ return (0); } 4165 4166#else /* __lint */ 4167 4168#if defined(__amd64) 4169 4170 ENTRY(ftrace_interrupt_disable) 4171 pushfq 4172 popq %rax 4173 CLI(%rdx) 4174 ret 4175 SET_SIZE(ftrace_interrupt_disable) 4176 4177#elif defined(__i386) 4178 4179 ENTRY(ftrace_interrupt_disable) 4180 pushfl 4181 popl %eax 4182 CLI(%edx) 4183 ret 4184 SET_SIZE(ftrace_interrupt_disable) 4185 4186#endif /* __i386 */ 4187#endif /* __lint */ 4188 4189#if defined(__lint) 4190 4191/*ARGSUSED*/ 4192void 4193ftrace_interrupt_enable(ftrace_icookie_t cookie) 4194{} 4195 4196#else /* __lint */ 4197 4198#if defined(__amd64) 4199 4200 ENTRY(ftrace_interrupt_enable) 4201 pushq %rdi 4202 popfq 4203 ret 4204 SET_SIZE(ftrace_interrupt_enable) 4205 4206#elif defined(__i386) 4207 4208 ENTRY(ftrace_interrupt_enable) 4209 movl 4(%esp), %eax 4210 pushl %eax 4211 popfl 4212 ret 4213 SET_SIZE(ftrace_interrupt_enable) 4214 4215#endif /* __i386 */ 4216#endif /* __lint */ 4217 4218#if defined (__lint) 4219 4220/*ARGSUSED*/ 4221void 4222iommu_cpu_nop(void) 4223{} 4224 4225#else /* __lint */ 4226 4227 ENTRY(iommu_cpu_nop) 4228 rep; nop 4229 ret 4230 SET_SIZE(iommu_cpu_nop) 4231 4232#endif /* __lint */ 4233 4234#if defined (__lint) 4235 4236/*ARGSUSED*/ 4237void 4238clflush_insn(caddr_t addr) 4239{} 4240 4241#else /* __lint */ 4242 4243#if defined (__amd64) 4244 ENTRY(clflush_insn) 4245 clflush (%rdi) 4246 ret 4247 SET_SIZE(clflush_insn) 4248#elif defined (__i386) 4249 ENTRY(clflush_insn) 4250 movl 4(%esp), %eax 4251 clflush (%eax) 4252 ret 4253 SET_SIZE(clflush_insn) 4254 4255#endif /* __i386 */ 4256#endif /* __lint */ 4257 4258#if defined (__lint) 4259/*ARGSUSED*/ 4260void 4261mfence_insn(void) 4262{} 4263 4264#else /* __lint */ 4265 4266#if defined (__amd64) 4267 ENTRY(mfence_insn) 4268 mfence 4269 ret 4270 SET_SIZE(mfence_insn) 4271#elif defined (__i386) 4272 ENTRY(mfence_insn) 4273 mfence 4274 ret 4275 SET_SIZE(mfence_insn) 4276 4277#endif /* __i386 */ 4278#endif /* __lint */ 4279