1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28#if defined(lint) || defined(__lint) 29#include <sys/types.h> 30#include <sys/thread.h> 31#include <sys/cpuvar.h> 32#include <vm/page.h> 33#else /* __lint */ 34#include "assym.h" 35#endif /* __lint */ 36 37#include <sys/mutex_impl.h> 38#include <sys/asm_linkage.h> 39#include <sys/asm_misc.h> 40#include <sys/regset.h> 41#include <sys/rwlock_impl.h> 42#include <sys/lockstat.h> 43 44/* 45 * lock_try(lp), ulock_try(lp) 46 * - returns non-zero on success. 47 * - doesn't block interrupts so don't use this to spin on a lock. 48 * 49 * ulock_try() is for a lock in the user address space. 50 */ 51 52#if defined(lint) || defined(__lint) 53 54/* ARGSUSED */ 55int 56lock_try(lock_t *lp) 57{ return (0); } 58 59/* ARGSUSED */ 60int 61lock_spin_try(lock_t *lp) 62{ return (0); } 63 64/* ARGSUSED */ 65int 66ulock_try(lock_t *lp) 67{ return (0); } 68 69#else /* __lint */ 70 .globl kernelbase 71 72#if defined(__amd64) 73 74 ENTRY(lock_try) 75 movb $-1, %dl 76 movzbq %dl, %rax 77 xchgb %dl, (%rdi) 78 xorb %dl, %al 79.lock_try_lockstat_patch_point: 80 ret 81 testb %al, %al 82 jnz 0f 83 ret 840: 85 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 86 movq %rdi, %rsi /* rsi = lock addr */ 87 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */ 88 jmp lockstat_wrapper 89 SET_SIZE(lock_try) 90 91 ENTRY(lock_spin_try) 92 movb $-1, %dl 93 movzbq %dl, %rax 94 xchgb %dl, (%rdi) 95 xorb %dl, %al 96 ret 97 SET_SIZE(lock_spin_try) 98 99 ENTRY(ulock_try) 100#ifdef DEBUG 101 movq kernelbase(%rip), %rax 102 cmpq %rax, %rdi /* test uaddr < kernelbase */ 103 jb ulock_pass /* uaddr < kernelbase, proceed */ 104 105 movq %rdi, %r12 /* preserve lock ptr for debugging */ 106 leaq .ulock_panic_msg(%rip), %rdi 107 pushq %rbp /* align stack properly */ 108 movq %rsp, %rbp 109 xorl %eax, %eax /* clear for varargs */ 110 call panic 111 112#endif /* DEBUG */ 113 114ulock_pass: 115 movl $1, %eax 116 xchgb %al, (%rdi) 117 xorb $1, %al 118 ret 119 SET_SIZE(ulock_try) 120 121#else 122 123 ENTRY(lock_try) 124 movl $1,%edx 125 movl 4(%esp),%ecx /* ecx = lock addr */ 126 xorl %eax,%eax 127 xchgb %dl, (%ecx) /* using dl will avoid partial */ 128 testb %dl,%dl /* stalls on P6 ? */ 129 setz %al 130.lock_try_lockstat_patch_point: 131 ret 132 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 133 testl %eax, %eax 134 jz 0f 135 movl $LS_LOCK_TRY_ACQUIRE, %eax 136 jmp lockstat_wrapper 1370: 138 ret 139 SET_SIZE(lock_try) 140 141 ENTRY(lock_spin_try) 142 movl $-1,%edx 143 movl 4(%esp),%ecx /* ecx = lock addr */ 144 xorl %eax,%eax 145 xchgb %dl, (%ecx) /* using dl will avoid partial */ 146 testb %dl,%dl /* stalls on P6 ? */ 147 setz %al 148 ret 149 SET_SIZE(lock_spin_try) 150 151 ENTRY(ulock_try) 152#ifdef DEBUG 153 movl kernelbase, %eax 154 cmpl %eax, 4(%esp) /* test uaddr < kernelbase */ 155 jb ulock_pass /* uaddr < kernelbase, proceed */ 156 157 pushl $.ulock_panic_msg 158 call panic 159 160#endif /* DEBUG */ 161 162ulock_pass: 163 movl $1,%eax 164 movl 4(%esp),%ecx 165 xchgb %al, (%ecx) 166 xorb $1, %al 167 ret 168 SET_SIZE(ulock_try) 169 170#endif /* !__amd64 */ 171 172#ifdef DEBUG 173 .data 174.ulock_panic_msg: 175 .string "ulock_try: Argument is above kernelbase" 176 .text 177#endif /* DEBUG */ 178 179#endif /* __lint */ 180 181/* 182 * lock_clear(lp) 183 * - unlock lock without changing interrupt priority level. 184 */ 185 186#if defined(lint) || defined(__lint) 187 188/* ARGSUSED */ 189void 190lock_clear(lock_t *lp) 191{} 192 193/* ARGSUSED */ 194void 195ulock_clear(lock_t *lp) 196{} 197 198#else /* __lint */ 199 200#if defined(__amd64) 201 202 ENTRY(lock_clear) 203 movb $0, (%rdi) 204.lock_clear_lockstat_patch_point: 205 ret 206 movq %rdi, %rsi /* rsi = lock addr */ 207 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 208 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */ 209 jmp lockstat_wrapper 210 SET_SIZE(lock_clear) 211 212 ENTRY(ulock_clear) 213#ifdef DEBUG 214 movq kernelbase(%rip), %rcx 215 cmpq %rcx, %rdi /* test uaddr < kernelbase */ 216 jb ulock_clr /* uaddr < kernelbase, proceed */ 217 218 leaq .ulock_clear_msg(%rip), %rdi 219 pushq %rbp /* align stack properly */ 220 movq %rsp, %rbp 221 xorl %eax, %eax /* clear for varargs */ 222 call panic 223#endif 224 225ulock_clr: 226 movb $0, (%rdi) 227 ret 228 SET_SIZE(ulock_clear) 229 230#else 231 232 ENTRY(lock_clear) 233 movl 4(%esp), %eax 234 movb $0, (%eax) 235.lock_clear_lockstat_patch_point: 236 ret 237 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 238 movl %eax, %ecx /* ecx = lock pointer */ 239 movl $LS_LOCK_CLEAR_RELEASE, %eax 240 jmp lockstat_wrapper 241 SET_SIZE(lock_clear) 242 243 ENTRY(ulock_clear) 244#ifdef DEBUG 245 movl kernelbase, %ecx 246 cmpl %ecx, 4(%esp) /* test uaddr < kernelbase */ 247 jb ulock_clr /* uaddr < kernelbase, proceed */ 248 249 pushl $.ulock_clear_msg 250 call panic 251#endif 252 253ulock_clr: 254 movl 4(%esp),%eax 255 xorl %ecx,%ecx 256 movb %cl, (%eax) 257 ret 258 SET_SIZE(ulock_clear) 259 260#endif /* !__amd64 */ 261 262#ifdef DEBUG 263 .data 264.ulock_clear_msg: 265 .string "ulock_clear: Argument is above kernelbase" 266 .text 267#endif /* DEBUG */ 268 269 270#endif /* __lint */ 271 272/* 273 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil) 274 * Drops lp, sets pil to new_pil, stores old pil in *old_pil. 275 */ 276 277#if defined(lint) || defined(__lint) 278 279/* ARGSUSED */ 280void 281lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil) 282{} 283 284#else /* __lint */ 285 286#if defined(__amd64) 287 288 ENTRY(lock_set_spl) 289 pushq %rbp 290 movq %rsp, %rbp 291 subq $32, %rsp 292 movl %esi, 8(%rsp) /* save priority level */ 293 movq %rdx, 16(%rsp) /* save old pil ptr */ 294 movq %rdi, 24(%rsp) /* save lock pointer */ 295 movl %esi, %edi /* pass priority level */ 296 call splr /* raise priority level */ 297 movq 24(%rsp), %rdi /* rdi = lock addr */ 298 movb $-1, %dl 299 xchgb %dl, (%rdi) /* try to set lock */ 300 testb %dl, %dl /* did we get the lock? ... */ 301 jnz .lss_miss /* ... no, go to C for the hard case */ 302 movq 16(%rsp), %rdx /* rdx = old pil addr */ 303 movw %ax, (%rdx) /* store old pil */ 304 leave 305.lock_set_spl_lockstat_patch_point: 306 ret 307 movq %rdi, %rsi /* rsi = lock addr */ 308 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 309 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi 310 jmp lockstat_wrapper 311.lss_miss: 312 movl 8(%rsp), %esi /* new_pil */ 313 movq 16(%rsp), %rdx /* old_pil_addr */ 314 movl %eax, %ecx /* original pil */ 315 leave /* unwind stack */ 316 jmp lock_set_spl_spin 317 SET_SIZE(lock_set_spl) 318 319#else 320 321 ENTRY(lock_set_spl) 322 movl 8(%esp), %eax /* get priority level */ 323 pushl %eax 324 call splr /* raise priority level */ 325 movl 8(%esp), %ecx /* ecx = lock addr */ 326 movl $-1, %edx 327 addl $4, %esp 328 xchgb %dl, (%ecx) /* try to set lock */ 329 testb %dl, %dl /* did we get the lock? ... */ 330 movl 12(%esp), %edx /* edx = olp pil addr (ZF unaffected) */ 331 jnz .lss_miss /* ... no, go to C for the hard case */ 332 movw %ax, (%edx) /* store old pil */ 333.lock_set_spl_lockstat_patch_point: 334 ret 335 movl %gs:CPU_THREAD, %edx /* edx = thread addr*/ 336 movl $LS_LOCK_SET_SPL_ACQUIRE, %eax 337 jmp lockstat_wrapper 338.lss_miss: 339 pushl %eax /* original pil */ 340 pushl %edx /* old_pil addr */ 341 pushl 16(%esp) /* new_pil */ 342 pushl %ecx /* lock addr */ 343 call lock_set_spl_spin 344 addl $16, %esp 345 ret 346 SET_SIZE(lock_set_spl) 347 348#endif /* !__amd64 */ 349 350#endif /* __lint */ 351 352/* 353 * void 354 * lock_init(lp) 355 */ 356 357#if defined(__lint) 358 359/* ARGSUSED */ 360void 361lock_init(lock_t *lp) 362{} 363 364#else /* __lint */ 365 366#if defined(__amd64) 367 368 ENTRY(lock_init) 369 movb $0, (%rdi) 370 ret 371 SET_SIZE(lock_init) 372 373#else 374 375 ENTRY(lock_init) 376 movl 4(%esp), %eax 377 movb $0, (%eax) 378 ret 379 SET_SIZE(lock_init) 380 381#endif /* !__amd64 */ 382 383#endif /* __lint */ 384 385/* 386 * void 387 * lock_set(lp) 388 */ 389 390#if defined(lint) || defined(__lint) 391 392/* ARGSUSED */ 393void 394lock_set(lock_t *lp) 395{} 396 397#else /* __lint */ 398 399#if defined(__amd64) 400 401 ENTRY(lock_set) 402 movb $-1, %dl 403 xchgb %dl, (%rdi) /* try to set lock */ 404 testb %dl, %dl /* did we get it? */ 405 jnz lock_set_spin /* no, go to C for the hard case */ 406.lock_set_lockstat_patch_point: 407 ret 408 movq %rdi, %rsi /* rsi = lock addr */ 409 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 410 movl $LS_LOCK_SET_ACQUIRE, %edi 411 jmp lockstat_wrapper 412 SET_SIZE(lock_set) 413 414#else 415 416 ENTRY(lock_set) 417 movl 4(%esp), %ecx /* ecx = lock addr */ 418 movl $-1, %edx 419 xchgb %dl, (%ecx) /* try to set lock */ 420 testb %dl, %dl /* did we get it? */ 421 jnz lock_set_spin /* no, go to C for the hard case */ 422.lock_set_lockstat_patch_point: 423 ret 424 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 425 movl $LS_LOCK_SET_ACQUIRE, %eax 426 jmp lockstat_wrapper 427 SET_SIZE(lock_set) 428 429#endif /* !__amd64 */ 430 431#endif /* __lint */ 432 433/* 434 * lock_clear_splx(lp, s) 435 */ 436 437#if defined(lint) || defined(__lint) 438 439/* ARGSUSED */ 440void 441lock_clear_splx(lock_t *lp, int s) 442{} 443 444#else /* __lint */ 445 446#if defined(__amd64) 447 448 ENTRY(lock_clear_splx) 449 movb $0, (%rdi) /* clear lock */ 450.lock_clear_splx_lockstat_patch_point: 451 jmp 0f 4520: 453 movl %esi, %edi /* arg for splx */ 454 jmp splx /* let splx do its thing */ 455.lock_clear_splx_lockstat: 456 pushq %rbp /* align stack properly */ 457 movq %rsp, %rbp 458 subq $16, %rsp /* space to save args across splx */ 459 movq %rdi, 8(%rsp) /* save lock ptr across splx call */ 460 movl %esi, %edi /* arg for splx */ 461 call splx /* lower the priority */ 462 movq 8(%rsp), %rsi /* rsi = lock ptr */ 463 leave /* unwind stack */ 464 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 465 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi 466 jmp lockstat_wrapper 467 SET_SIZE(lock_clear_splx) 468 469#else 470 471 ENTRY(lock_clear_splx) 472 movl 4(%esp), %eax /* eax = lock addr */ 473 movb $0, (%eax) /* clear lock */ 474.lock_clear_splx_lockstat_patch_point: 475 jmp 0f 4760: 477 movl 8(%esp), %edx /* edx = desired pil */ 478 movl %edx, 4(%esp) /* set spl arg up for splx */ 479 jmp splx /* let splx do it's thing */ 480.lock_clear_splx_lockstat: 481 movl 8(%esp), %edx /* edx = desired pil */ 482 pushl %ebp /* set up stack frame */ 483 movl %esp, %ebp 484 pushl %edx 485 call splx 486 leave /* unwind stack */ 487 movl 4(%esp), %ecx /* ecx = lock pointer */ 488 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 489 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %eax 490 jmp lockstat_wrapper 491 SET_SIZE(lock_clear_splx) 492 493#endif /* !__amd64 */ 494 495#if defined(__GNUC_AS__) 496#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \ 497 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2) 498 499#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \ 500 (.lock_clear_splx_lockstat_patch_point + 1) 501#else 502#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \ 503 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2] 504 505#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \ 506 [.lock_clear_splx_lockstat_patch_point + 1] 507#endif 508 509#endif /* __lint */ 510 511/* 512 * mutex_enter() and mutex_exit(). 513 * 514 * These routines handle the simple cases of mutex_enter() (adaptive 515 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). 516 * If anything complicated is going on we punt to mutex_vector_enter(). 517 * 518 * mutex_tryenter() is similar to mutex_enter() but returns zero if 519 * the lock cannot be acquired, nonzero on success. 520 * 521 * If mutex_exit() gets preempted in the window between checking waiters 522 * and clearing the lock, we can miss wakeups. Disabling preemption 523 * in the mutex code is prohibitively expensive, so instead we detect 524 * mutex preemption by examining the trapped PC in the interrupt path. 525 * If we interrupt a thread in mutex_exit() that has not yet cleared 526 * the lock, cmnint() resets its PC back to the beginning of 527 * mutex_exit() so it will check again for waiters when it resumes. 528 * 529 * The lockstat code below is activated when the lockstat driver 530 * calls lockstat_hot_patch() to hot-patch the kernel mutex code. 531 * Note that we don't need to test lockstat_event_mask here -- we won't 532 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats. 533 */ 534#if defined(lint) || defined(__lint) 535 536/* ARGSUSED */ 537void 538mutex_enter(kmutex_t *lp) 539{} 540 541/* ARGSUSED */ 542int 543mutex_tryenter(kmutex_t *lp) 544{ return (0); } 545 546/* ARGSUSED */ 547int 548mutex_adaptive_tryenter(mutex_impl_t *lp) 549{ return (0); } 550 551/* ARGSUSED */ 552void 553mutex_exit(kmutex_t *lp) 554{} 555 556#else 557 558#if defined(__amd64) 559 560 ENTRY_NP(mutex_enter) 561 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 562 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 563 lock 564 cmpxchgq %rdx, (%rdi) 565 jnz mutex_vector_enter 566.mutex_enter_lockstat_patch_point: 567#if defined(OPTERON_WORKAROUND_6323525) 568.mutex_enter_6323525_patch_point: 569 ret /* nop space for lfence */ 570 nop 571 nop 572.mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */ 573 nop 574#else /* OPTERON_WORKAROUND_6323525 */ 575 ret 576#endif /* OPTERON_WORKAROUND_6323525 */ 577 movq %rdi, %rsi 578 movl $LS_MUTEX_ENTER_ACQUIRE, %edi 579/* 580 * expects %rdx=thread, %rsi=lock, %edi=lockstat event 581 */ 582 ALTENTRY(lockstat_wrapper) 583 incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */ 584 leaq lockstat_probemap(%rip), %rax 585 movl (%rax, %rdi, DTRACE_IDSIZE), %eax 586 testl %eax, %eax /* check for non-zero probe */ 587 jz 1f 588 pushq %rbp /* align stack properly */ 589 movq %rsp, %rbp 590 movl %eax, %edi 591 call *lockstat_probe 592 leave /* unwind stack */ 5931: 594 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 595 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 596 movl $1, %eax /* return success if tryenter */ 597 ret 598 SET_SIZE(lockstat_wrapper) 599 SET_SIZE(mutex_enter) 600 601/* 602 * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event 603 */ 604 ENTRY(lockstat_wrapper_arg) 605 incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */ 606 leaq lockstat_probemap(%rip), %rax 607 movl (%rax, %rdi, DTRACE_IDSIZE), %eax 608 testl %eax, %eax /* check for non-zero probe */ 609 jz 1f 610 pushq %rbp /* align stack properly */ 611 movq %rsp, %rbp 612 movl %eax, %edi 613 call *lockstat_probe 614 leave /* unwind stack */ 6151: 616 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 617 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 618 movl $1, %eax /* return success if tryenter */ 619 ret 620 SET_SIZE(lockstat_wrapper_arg) 621 622 623 ENTRY(mutex_tryenter) 624 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 625 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 626 lock 627 cmpxchgq %rdx, (%rdi) 628 jnz mutex_vector_tryenter 629 not %eax /* return success (nonzero) */ 630#if defined(OPTERON_WORKAROUND_6323525) 631.mutex_tryenter_lockstat_patch_point: 632.mutex_tryenter_6323525_patch_point: 633 ret /* nop space for lfence */ 634 nop 635 nop 636.mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */ 637 nop 638#else /* OPTERON_WORKAROUND_6323525 */ 639.mutex_tryenter_lockstat_patch_point: 640 ret 641#endif /* OPTERON_WORKAROUND_6323525 */ 642 movq %rdi, %rsi 643 movl $LS_MUTEX_ENTER_ACQUIRE, %edi 644 jmp lockstat_wrapper 645 SET_SIZE(mutex_tryenter) 646 647 ENTRY(mutex_adaptive_tryenter) 648 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 649 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 650 lock 651 cmpxchgq %rdx, (%rdi) 652 jnz 0f 653 not %eax /* return success (nonzero) */ 654#if defined(OPTERON_WORKAROUND_6323525) 655.mutex_atryenter_6323525_patch_point: 656 ret /* nop space for lfence */ 657 nop 658 nop 659 nop 660#else /* OPTERON_WORKAROUND_6323525 */ 661 ret 662#endif /* OPTERON_WORKAROUND_6323525 */ 6630: 664 xorl %eax, %eax /* return failure */ 665 ret 666 SET_SIZE(mutex_adaptive_tryenter) 667 668 .globl mutex_owner_running_critical_start 669 670 ENTRY(mutex_owner_running) 671mutex_owner_running_critical_start: 672 movq (%rdi), %r11 /* get owner field */ 673 andq $MUTEX_THREAD, %r11 /* remove waiters bit */ 674 cmpq $0, %r11 /* if free, skip */ 675 je 1f /* go return 0 */ 676 movq T_CPU(%r11), %r8 /* get owner->t_cpu */ 677 movq CPU_THREAD(%r8), %r9 /* get t_cpu->cpu_thread */ 678.mutex_owner_running_critical_end: 679 cmpq %r11, %r9 /* owner == running thread? */ 680 je 2f /* yes, go return cpu */ 6811: 682 xorq %rax, %rax /* return 0 */ 683 ret 6842: 685 movq %r8, %rax /* return cpu */ 686 ret 687 SET_SIZE(mutex_owner_running) 688 689 .globl mutex_owner_running_critical_size 690 .type mutex_owner_running_critical_size, @object 691 .align CPTRSIZE 692mutex_owner_running_critical_size: 693 .quad .mutex_owner_running_critical_end - mutex_owner_running_critical_start 694 SET_SIZE(mutex_owner_running_critical_size) 695 696 .globl mutex_exit_critical_start 697 698 ENTRY(mutex_exit) 699mutex_exit_critical_start: /* If interrupted, restart here */ 700 movq %gs:CPU_THREAD, %rdx 701 cmpq %rdx, (%rdi) 702 jne mutex_vector_exit /* wrong type or wrong owner */ 703 movq $0, (%rdi) /* clear owner AND lock */ 704.mutex_exit_critical_end: 705.mutex_exit_lockstat_patch_point: 706 ret 707 movq %rdi, %rsi 708 movl $LS_MUTEX_EXIT_RELEASE, %edi 709 jmp lockstat_wrapper 710 SET_SIZE(mutex_exit) 711 712 .globl mutex_exit_critical_size 713 .type mutex_exit_critical_size, @object 714 .align CPTRSIZE 715mutex_exit_critical_size: 716 .quad .mutex_exit_critical_end - mutex_exit_critical_start 717 SET_SIZE(mutex_exit_critical_size) 718 719#else 720 721 ENTRY_NP(mutex_enter) 722 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 723 movl 4(%esp), %ecx /* ecx = lock ptr */ 724 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 725 lock 726 cmpxchgl %edx, (%ecx) 727 jnz mutex_vector_enter 728#if defined(OPTERON_WORKAROUND_6323525) 729.mutex_enter_lockstat_patch_point: 730.mutex_enter_6323525_patch_point: 731 ret /* nop space for lfence */ 732 nop 733 nop 734.mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */ 735 nop 736#else /* OPTERON_WORKAROUND_6323525 */ 737.mutex_enter_lockstat_patch_point: 738 ret 739#endif /* OPTERON_WORKAROUND_6323525 */ 740 movl $LS_MUTEX_ENTER_ACQUIRE, %eax 741 ALTENTRY(lockstat_wrapper) /* expects edx=thread, ecx=lock, */ 742 /* eax=lockstat event */ 743 pushl %ebp /* buy a frame */ 744 movl %esp, %ebp 745 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */ 746 pushl %edx /* save thread pointer */ 747 movl $lockstat_probemap, %edx 748 movl (%edx, %eax, DTRACE_IDSIZE), %eax 749 testl %eax, %eax /* check for non-zero probe */ 750 jz 1f 751 pushl %ecx /* push lock */ 752 pushl %eax /* push probe ID */ 753 call *lockstat_probe 754 addl $8, %esp 7551: 756 popl %edx /* restore thread pointer */ 757 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */ 758 movl $1, %eax /* return success if tryenter */ 759 popl %ebp /* pop off frame */ 760 ret 761 SET_SIZE(lockstat_wrapper) 762 SET_SIZE(mutex_enter) 763 764 ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */ 765 /* eax=lockstat event, pushed arg */ 766 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */ 767 pushl %edx /* save thread pointer */ 768 movl $lockstat_probemap, %edx 769 movl (%edx, %eax, DTRACE_IDSIZE), %eax 770 testl %eax, %eax /* check for non-zero probe */ 771 jz 1f 772 pushl %ebp /* save %ebp */ 773 pushl 8(%esp) /* push arg1 */ 774 movl %ebp, 12(%esp) /* fake up the stack frame */ 775 movl %esp, %ebp /* fake up base pointer */ 776 addl $12, %ebp /* adjust faked base pointer */ 777 pushl %ecx /* push lock */ 778 pushl %eax /* push probe ID */ 779 call *lockstat_probe 780 addl $12, %esp /* adjust for arguments */ 781 popl %ebp /* pop frame */ 7821: 783 popl %edx /* restore thread pointer */ 784 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */ 785 movl $1, %eax /* return success if tryenter */ 786 addl $4, %esp /* pop argument */ 787 ret 788 SET_SIZE(lockstat_wrapper_arg) 789 790 791 ENTRY(mutex_tryenter) 792 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 793 movl 4(%esp), %ecx /* ecx = lock ptr */ 794 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 795 lock 796 cmpxchgl %edx, (%ecx) 797 jnz mutex_vector_tryenter 798 movl %ecx, %eax 799#if defined(OPTERON_WORKAROUND_6323525) 800.mutex_tryenter_lockstat_patch_point: 801.mutex_tryenter_6323525_patch_point: 802 ret /* nop space for lfence */ 803 nop 804 nop 805.mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */ 806 nop 807#else /* OPTERON_WORKAROUND_6323525 */ 808.mutex_tryenter_lockstat_patch_point: 809 ret 810#endif /* OPTERON_WORKAROUND_6323525 */ 811 movl $LS_MUTEX_ENTER_ACQUIRE, %eax 812 jmp lockstat_wrapper 813 SET_SIZE(mutex_tryenter) 814 815 ENTRY(mutex_adaptive_tryenter) 816 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 817 movl 4(%esp), %ecx /* ecx = lock ptr */ 818 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 819 lock 820 cmpxchgl %edx, (%ecx) 821 jnz 0f 822 movl %ecx, %eax 823#if defined(OPTERON_WORKAROUND_6323525) 824.mutex_atryenter_6323525_patch_point: 825 ret /* nop space for lfence */ 826 nop 827 nop 828 nop 829#else /* OPTERON_WORKAROUND_6323525 */ 830 ret 831#endif /* OPTERON_WORKAROUND_6323525 */ 8320: 833 xorl %eax, %eax 834 ret 835 SET_SIZE(mutex_adaptive_tryenter) 836 837 .globl mutex_owner_running_critical_start 838 839 ENTRY(mutex_owner_running) 840mutex_owner_running_critical_start: 841 movl 4(%esp), %eax /* get owner field */ 842 movl (%eax), %eax 843 andl $MUTEX_THREAD, %eax /* remove waiters bit */ 844 cmpl $0, %eax /* if free, skip */ 845 je 1f /* go return 0 */ 846 movl T_CPU(%eax), %ecx /* get owner->t_cpu */ 847 movl CPU_THREAD(%ecx), %edx /* get t_cpu->cpu_thread */ 848.mutex_owner_running_critical_end: 849 cmpl %eax, %edx /* owner == running thread? */ 850 je 2f /* yes, go return cpu */ 8511: 852 xorl %eax, %eax /* return 0 */ 853 ret 8542: 855 movl %ecx, %eax /* return cpu */ 856 ret 857 858 SET_SIZE(mutex_owner_running) 859 860 .globl mutex_owner_running_critical_size 861 .type mutex_owner_running_critical_size, @object 862 .align CPTRSIZE 863mutex_owner_running_critical_size: 864 .long .mutex_owner_running_critical_end - mutex_owner_running_critical_start 865 SET_SIZE(mutex_owner_running_critical_size) 866 867 .globl mutex_exit_critical_start 868 869 ENTRY(mutex_exit) 870mutex_exit_critical_start: /* If interrupted, restart here */ 871 movl %gs:CPU_THREAD, %edx 872 movl 4(%esp), %ecx 873 cmpl %edx, (%ecx) 874 jne mutex_vector_exit /* wrong type or wrong owner */ 875 movl $0, (%ecx) /* clear owner AND lock */ 876.mutex_exit_critical_end: 877.mutex_exit_lockstat_patch_point: 878 ret 879 movl $LS_MUTEX_EXIT_RELEASE, %eax 880 jmp lockstat_wrapper 881 SET_SIZE(mutex_exit) 882 883 .globl mutex_exit_critical_size 884 .type mutex_exit_critical_size, @object 885 .align CPTRSIZE 886mutex_exit_critical_size: 887 .long .mutex_exit_critical_end - mutex_exit_critical_start 888 SET_SIZE(mutex_exit_critical_size) 889 890#endif /* !__amd64 */ 891 892#endif /* __lint */ 893 894/* 895 * rw_enter() and rw_exit(). 896 * 897 * These routines handle the simple cases of rw_enter (write-locking an unheld 898 * lock or read-locking a lock that's neither write-locked nor write-wanted) 899 * and rw_exit (no waiters or not the last reader). If anything complicated 900 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. 901 */ 902#if defined(lint) || defined(__lint) 903 904/* ARGSUSED */ 905void 906rw_enter(krwlock_t *lp, krw_t rw) 907{} 908 909/* ARGSUSED */ 910void 911rw_exit(krwlock_t *lp) 912{} 913 914#else /* __lint */ 915 916#if defined(__amd64) 917 918 ENTRY(rw_enter) 919 cmpl $RW_WRITER, %esi 920 je .rw_write_enter 921 movq (%rdi), %rax /* rax = old rw_wwwh value */ 922 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax 923 jnz rw_enter_sleep 924 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */ 925 lock 926 cmpxchgq %rdx, (%rdi) /* try to grab read lock */ 927 jnz rw_enter_sleep 928.rw_read_enter_lockstat_patch_point: 929 ret 930 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 931 movq %rdi, %rsi /* rsi = lock ptr */ 932 movl $LS_RW_ENTER_ACQUIRE, %edi 933 movl $RW_READER, %edx 934 jmp lockstat_wrapper_arg 935.rw_write_enter: 936 movq %gs:CPU_THREAD, %rdx 937 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */ 938 xorl %eax, %eax /* rax = unheld value */ 939 lock 940 cmpxchgq %rdx, (%rdi) /* try to grab write lock */ 941 jnz rw_enter_sleep 942 943#if defined(OPTERON_WORKAROUND_6323525) 944.rw_write_enter_lockstat_patch_point: 945.rw_write_enter_6323525_patch_point: 946 ret 947 nop 948 nop 949.rw_write_enter_lockstat_6323525_patch_point: 950 nop 951#else /* OPTERON_WORKAROUND_6323525 */ 952.rw_write_enter_lockstat_patch_point: 953 ret 954#endif /* OPTERON_WORKAROUND_6323525 */ 955 956 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 957 movq %rdi, %rsi /* rsi = lock ptr */ 958 movl $LS_RW_ENTER_ACQUIRE, %edi 959 movl $RW_WRITER, %edx 960 jmp lockstat_wrapper_arg 961 SET_SIZE(rw_enter) 962 963 ENTRY(rw_exit) 964 movq (%rdi), %rax /* rax = old rw_wwwh value */ 965 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */ 966 jne .rw_not_single_reader 967 xorl %edx, %edx /* rdx = new value (unheld) */ 968.rw_read_exit: 969 lock 970 cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 971 jnz rw_exit_wakeup 972.rw_read_exit_lockstat_patch_point: 973 ret 974 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 975 movq %rdi, %rsi /* rsi = lock ptr */ 976 movl $LS_RW_EXIT_RELEASE, %edi 977 movl $RW_READER, %edx 978 jmp lockstat_wrapper_arg 979.rw_not_single_reader: 980 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */ 981 jnz .rw_write_exit 982 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */ 983 cmpl $RW_READ_LOCK, %edx 984 jge .rw_read_exit /* not last reader, safe to drop */ 985 jmp rw_exit_wakeup /* last reader with waiters */ 986.rw_write_exit: 987 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */ 988 xorl %edx, %edx /* rdx = new value (unheld) */ 989 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */ 990 lock 991 cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 992 jnz rw_exit_wakeup 993.rw_write_exit_lockstat_patch_point: 994 ret 995 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 996 movq %rdi, %rsi /* rsi - lock ptr */ 997 movl $LS_RW_EXIT_RELEASE, %edi 998 movl $RW_WRITER, %edx 999 jmp lockstat_wrapper_arg 1000 SET_SIZE(rw_exit) 1001 1002#else 1003 1004 ENTRY(rw_enter) 1005 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1006 movl 4(%esp), %ecx /* ecx = lock ptr */ 1007 cmpl $RW_WRITER, 8(%esp) 1008 je .rw_write_enter 1009 incl T_KPRI_REQ(%edx) /* THREAD_KPRI_REQUEST() */ 1010 movl (%ecx), %eax /* eax = old rw_wwwh value */ 1011 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax 1012 jnz rw_enter_sleep 1013 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */ 1014 lock 1015 cmpxchgl %edx, (%ecx) /* try to grab read lock */ 1016 jnz rw_enter_sleep 1017.rw_read_enter_lockstat_patch_point: 1018 ret 1019 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1020 movl $LS_RW_ENTER_ACQUIRE, %eax 1021 pushl $RW_READER 1022 jmp lockstat_wrapper_arg 1023.rw_write_enter: 1024 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */ 1025 xorl %eax, %eax /* eax = unheld value */ 1026 lock 1027 cmpxchgl %edx, (%ecx) /* try to grab write lock */ 1028 jnz rw_enter_sleep 1029 1030#if defined(OPTERON_WORKAROUND_6323525) 1031.rw_write_enter_lockstat_patch_point: 1032.rw_write_enter_6323525_patch_point: 1033 ret 1034 nop 1035 nop 1036.rw_write_enter_lockstat_6323525_patch_point: 1037 nop 1038#else /* OPTERON_WORKAROUND_6323525 */ 1039.rw_write_enter_lockstat_patch_point: 1040 ret 1041#endif /* OPTERON_WORKAROUND_6323525 */ 1042 1043 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1044 movl $LS_RW_ENTER_ACQUIRE, %eax 1045 pushl $RW_WRITER 1046 jmp lockstat_wrapper_arg 1047 SET_SIZE(rw_enter) 1048 1049 ENTRY(rw_exit) 1050 movl 4(%esp), %ecx /* ecx = lock ptr */ 1051 movl (%ecx), %eax /* eax = old rw_wwwh value */ 1052 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */ 1053 jne .rw_not_single_reader 1054 xorl %edx, %edx /* edx = new value (unheld) */ 1055.rw_read_exit: 1056 lock 1057 cmpxchgl %edx, (%ecx) /* try to drop read lock */ 1058 jnz rw_exit_wakeup 1059 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1060 decl T_KPRI_REQ(%edx) /* THREAD_KPRI_RELEASE() */ 1061.rw_read_exit_lockstat_patch_point: 1062 ret 1063 movl $LS_RW_EXIT_RELEASE, %eax 1064 pushl $RW_READER 1065 jmp lockstat_wrapper_arg 1066.rw_not_single_reader: 1067 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */ 1068 jnz .rw_write_exit 1069 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */ 1070 cmpl $RW_READ_LOCK, %edx 1071 jge .rw_read_exit /* not last reader, safe to drop */ 1072 jmp rw_exit_wakeup /* last reader with waiters */ 1073.rw_write_exit: 1074 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */ 1075 xorl %edx, %edx /* edx = new value (unheld) */ 1076 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */ 1077 lock 1078 cmpxchgl %edx, (%ecx) /* try to drop read lock */ 1079 jnz rw_exit_wakeup 1080.rw_write_exit_lockstat_patch_point: 1081 ret 1082 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1083 movl $LS_RW_EXIT_RELEASE, %eax 1084 pushl $RW_WRITER 1085 jmp lockstat_wrapper_arg 1086 SET_SIZE(rw_exit) 1087 1088#endif /* !__amd64 */ 1089 1090#endif /* __lint */ 1091 1092#if defined(OPTERON_WORKAROUND_6323525) 1093#if defined(lint) || defined(__lint) 1094 1095int workaround_6323525_patched; 1096 1097void 1098patch_workaround_6323525(void) 1099{} 1100 1101#else /* lint */ 1102 1103/* 1104 * If it is necessary to patch the lock enter routines with the lfence 1105 * workaround, workaround_6323525_patched is set to a non-zero value so that 1106 * the lockstat_hat_patch routine can patch to the new location of the 'ret' 1107 * instruction. 1108 */ 1109 DGDEF3(workaround_6323525_patched, 4, 4) 1110 .long 0 1111 1112#if defined(__amd64) 1113 1114#define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \ 1115 movq $size, %rbx; \ 1116 movq $dstaddr, %r13; \ 1117 addq %rbx, %r13; \ 1118 movq $srcaddr, %r12; \ 1119 addq %rbx, %r12; \ 11200: \ 1121 decq %r13; \ 1122 decq %r12; \ 1123 movzbl (%r12), %esi; \ 1124 movq $1, %rdx; \ 1125 movq %r13, %rdi; \ 1126 call hot_patch_kernel_text; \ 1127 decq %rbx; \ 1128 testq %rbx, %rbx; \ 1129 jg 0b; 1130 1131/* 1132 * patch_workaround_6323525: provide workaround for 6323525 1133 * 1134 * The workaround is to place a fencing instruction (lfence) between the 1135 * mutex operation and the subsequent read-modify-write instruction. 1136 * 1137 * This routine hot patches the lfence instruction on top of the space 1138 * reserved by nops in the lock enter routines. 1139 */ 1140 ENTRY_NP(patch_workaround_6323525) 1141 pushq %rbp 1142 movq %rsp, %rbp 1143 pushq %r12 1144 pushq %r13 1145 pushq %rbx 1146 1147 /* 1148 * lockstat_hot_patch() to use the alternate lockstat workaround 1149 * 6323525 patch points (points past the lfence instruction to the 1150 * new ret) when workaround_6323525_patched is set. 1151 */ 1152 movl $1, workaround_6323525_patched 1153 1154 /* 1155 * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter 1156 * routines. The 4 bytes are patched in reverse order so that the 1157 * the existing ret is overwritten last. This provides lock enter 1158 * sanity during the intermediate patching stages. 1159 */ 1160 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4) 1161 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4) 1162 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4) 1163 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4) 1164 1165 popq %rbx 1166 popq %r13 1167 popq %r12 1168 movq %rbp, %rsp 1169 popq %rbp 1170 ret 1171_lfence_insn: 1172 lfence 1173 ret 1174 SET_SIZE(patch_workaround_6323525) 1175 1176 1177#else /* __amd64 */ 1178 1179#define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \ 1180 movl $size, %ebx; \ 1181 movl $srcaddr, %esi; \ 1182 addl %ebx, %esi; \ 1183 movl $dstaddr, %edi; \ 1184 addl %ebx, %edi; \ 11850: \ 1186 decl %esi; \ 1187 decl %edi; \ 1188 pushl $1; \ 1189 movzbl (%esi), %eax; \ 1190 pushl %eax; \ 1191 pushl %edi; \ 1192 call hot_patch_kernel_text; \ 1193 addl $12, %esp; \ 1194 decl %ebx; \ 1195 testl %ebx, %ebx; \ 1196 jg 0b; 1197 1198 1199 /* see comments above */ 1200 ENTRY_NP(patch_workaround_6323525) 1201 pushl %ebp 1202 movl %esp, %ebp 1203 pushl %ebx 1204 pushl %esi 1205 pushl %edi 1206 1207 movl $1, workaround_6323525_patched 1208 1209 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4) 1210 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4) 1211 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4) 1212 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4) 1213 1214 popl %edi 1215 popl %esi 1216 popl %ebx 1217 movl %ebp, %esp 1218 popl %ebp 1219 ret 1220_lfence_insn: 1221 .byte 0xf, 0xae, 0xe8 / [lfence instruction] 1222 ret 1223 SET_SIZE(patch_workaround_6323525) 1224 1225#endif /* !__amd64 */ 1226#endif /* !lint */ 1227#endif /* OPTERON_WORKAROUND_6323525 */ 1228 1229 1230#if defined(lint) || defined(__lint) 1231 1232void 1233lockstat_hot_patch(void) 1234{} 1235 1236#else 1237 1238#if defined(__amd64) 1239 1240#define HOT_PATCH(addr, event, active_instr, normal_instr, len) \ 1241 movq $normal_instr, %rsi; \ 1242 movq $active_instr, %rdi; \ 1243 leaq lockstat_probemap(%rip), %rax; \ 1244 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \ 1245 testl %eax, %eax; \ 1246 jz 9f; \ 1247 movq %rdi, %rsi; \ 12489: \ 1249 movq $len, %rdx; \ 1250 movq $addr, %rdi; \ 1251 call hot_patch_kernel_text 1252 1253#else 1254 1255#define HOT_PATCH(addr, event, active_instr, normal_instr, len) \ 1256 movl $normal_instr, %ecx; \ 1257 movl $active_instr, %edx; \ 1258 movl $lockstat_probemap, %eax; \ 1259 movl _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \ 1260 testl %eax, %eax; \ 1261 jz . + 4; \ 1262 movl %edx, %ecx; \ 1263 pushl $len; \ 1264 pushl %ecx; \ 1265 pushl $addr; \ 1266 call hot_patch_kernel_text; \ 1267 addl $12, %esp; 1268 1269#endif /* !__amd64 */ 1270 1271 ENTRY(lockstat_hot_patch) 1272#if defined(__amd64) 1273 pushq %rbp /* align stack properly */ 1274 movq %rsp, %rbp 1275#endif /* __amd64 */ 1276 1277#if defined(OPTERON_WORKAROUND_6323525) 1278 cmpl $0, workaround_6323525_patched 1279 je 1f 1280 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point, 1281 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1282 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point, 1283 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1284 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point, 1285 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1286 jmp 2f 12871: 1288 HOT_PATCH(.mutex_enter_lockstat_patch_point, 1289 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1290 HOT_PATCH(.mutex_tryenter_lockstat_patch_point, 1291 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1292 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 1293 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 12942: 1295#else /* OPTERON_WORKAROUND_6323525 */ 1296 HOT_PATCH(.mutex_enter_lockstat_patch_point, 1297 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1298 HOT_PATCH(.mutex_tryenter_lockstat_patch_point, 1299 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1300 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 1301 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1302#endif /* !OPTERON_WORKAROUND_6323525 */ 1303 HOT_PATCH(.mutex_exit_lockstat_patch_point, 1304 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1305 HOT_PATCH(.rw_read_enter_lockstat_patch_point, 1306 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1307 HOT_PATCH(.rw_write_exit_lockstat_patch_point, 1308 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1309 HOT_PATCH(.rw_read_exit_lockstat_patch_point, 1310 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1311 HOT_PATCH(.lock_set_lockstat_patch_point, 1312 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1313 HOT_PATCH(.lock_try_lockstat_patch_point, 1314 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1315 HOT_PATCH(.lock_clear_lockstat_patch_point, 1316 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1) 1317 HOT_PATCH(.lock_set_spl_lockstat_patch_point, 1318 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1319 1320 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT, 1321 LS_LOCK_CLEAR_SPLX_RELEASE, 1322 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1); 1323#if defined(__amd64) 1324 leave /* unwind stack */ 1325#endif /* __amd64 */ 1326 ret 1327 SET_SIZE(lockstat_hot_patch) 1328 1329#endif /* __lint */ 1330 1331#if defined(lint) || defined(__lint) 1332 1333/* XX64 membar_*() should be inlines */ 1334 1335void 1336membar_sync(void) 1337{} 1338 1339void 1340membar_enter(void) 1341{} 1342 1343void 1344membar_exit(void) 1345{} 1346 1347void 1348membar_producer(void) 1349{} 1350 1351void 1352membar_consumer(void) 1353{} 1354 1355#else /* __lint */ 1356 1357#if defined(__amd64) 1358 1359 ENTRY(membar_enter) 1360 ALTENTRY(membar_exit) 1361 ALTENTRY(membar_sync) 1362 mfence /* lighter weight than lock; xorq $0,(%rsp) */ 1363 ret 1364 SET_SIZE(membar_sync) 1365 SET_SIZE(membar_exit) 1366 SET_SIZE(membar_enter) 1367 1368 ENTRY(membar_producer) 1369 sfence 1370 ret 1371 SET_SIZE(membar_producer) 1372 1373 ENTRY(membar_consumer) 1374 lfence 1375 ret 1376 SET_SIZE(membar_consumer) 1377 1378#else 1379 1380 ENTRY(membar_enter) 1381 ALTENTRY(membar_exit) 1382 ALTENTRY(membar_sync) 1383 lock 1384 xorl $0, (%esp) 1385 ret 1386 SET_SIZE(membar_sync) 1387 SET_SIZE(membar_exit) 1388 SET_SIZE(membar_enter) 1389 1390/* 1391 * On machines that support sfence and lfence, these 1392 * memory barriers can be more precisely implemented 1393 * without causing the whole world to stop 1394 */ 1395 ENTRY(membar_producer) 1396 .globl _patch_sfence_ret 1397_patch_sfence_ret: /* c.f. membar #StoreStore */ 1398 lock 1399 xorl $0, (%esp) 1400 ret 1401 SET_SIZE(membar_producer) 1402 1403 ENTRY(membar_consumer) 1404 .globl _patch_lfence_ret 1405_patch_lfence_ret: /* c.f. membar #LoadLoad */ 1406 lock 1407 xorl $0, (%esp) 1408 ret 1409 SET_SIZE(membar_consumer) 1410 1411#endif /* !__amd64 */ 1412 1413#endif /* __lint */ 1414 1415/* 1416 * thread_onproc() 1417 * Set thread in onproc state for the specified CPU. 1418 * Also set the thread lock pointer to the CPU's onproc lock. 1419 * Since the new lock isn't held, the store ordering is important. 1420 * If not done in assembler, the compiler could reorder the stores. 1421 */ 1422#if defined(lint) || defined(__lint) 1423 1424void 1425thread_onproc(kthread_id_t t, cpu_t *cp) 1426{ 1427 t->t_state = TS_ONPROC; 1428 t->t_lockp = &cp->cpu_thread_lock; 1429} 1430 1431#else /* __lint */ 1432 1433#if defined(__amd64) 1434 1435 ENTRY(thread_onproc) 1436 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */ 1437 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */ 1438 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */ 1439 ret 1440 SET_SIZE(thread_onproc) 1441 1442#else 1443 1444 ENTRY(thread_onproc) 1445 movl 4(%esp), %eax 1446 movl 8(%esp), %ecx 1447 addl $CPU_THREAD_LOCK, %ecx /* pointer to disp_lock while running */ 1448 movl $ONPROC_THREAD, T_STATE(%eax) /* set state to TS_ONPROC */ 1449 movl %ecx, T_LOCKP(%eax) /* store new lock pointer */ 1450 ret 1451 SET_SIZE(thread_onproc) 1452 1453#endif /* !__amd64 */ 1454 1455#endif /* __lint */ 1456 1457/* 1458 * mutex_delay_default(void) 1459 * Spins for approx a few hundred processor cycles and returns to caller. 1460 */ 1461 1462#if defined(lint) || defined(__lint) 1463 1464void 1465mutex_delay_default(void) 1466{} 1467 1468#else /* __lint */ 1469 1470#if defined(__amd64) 1471 1472 ENTRY(mutex_delay_default) 1473 movq $92,%r11 14740: decq %r11 1475 jg 0b 1476 ret 1477 SET_SIZE(mutex_delay_default) 1478 1479#else 1480 1481 ENTRY(mutex_delay_default) 1482 push %ebp 1483 movl %esp,%ebp 1484 andl $-16,%esp 1485 push %ebx 1486 movl $93,%ebx 14870: decl %ebx 1488 jg 0b 1489 pop %ebx 1490 leave 1491 ret 1492 SET_SIZE(mutex_delay_default) 1493 1494#endif /* !__amd64 */ 1495#endif /* __lint */ 1496