1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28#if defined(lint) || defined(__lint) 29#include <sys/types.h> 30#include <sys/thread.h> 31#include <sys/cpuvar.h> 32#include <vm/page.h> 33#include <sys/mutex_impl.h> 34#else /* __lint */ 35#include "assym.h" 36#endif /* __lint */ 37 38#include <sys/asm_linkage.h> 39#include <sys/asm_misc.h> 40#include <sys/regset.h> 41#include <sys/rwlock_impl.h> 42#include <sys/lockstat.h> 43 44/* 45 * lock_try(lp), ulock_try(lp) 46 * - returns non-zero on success. 47 * - doesn't block interrupts so don't use this to spin on a lock. 48 * 49 * ulock_try() is for a lock in the user address space. 50 */ 51 52#if defined(lint) || defined(__lint) 53 54/* ARGSUSED */ 55int 56lock_try(lock_t *lp) 57{ return (0); } 58 59/* ARGSUSED */ 60int 61lock_spin_try(lock_t *lp) 62{ return (0); } 63 64/* ARGSUSED */ 65int 66ulock_try(lock_t *lp) 67{ return (0); } 68 69#else /* __lint */ 70 .globl kernelbase 71 72#if defined(__amd64) 73 74 ENTRY(lock_try) 75 movb $-1, %dl 76 movzbq %dl, %rax 77 xchgb %dl, (%rdi) 78 xorb %dl, %al 79.lock_try_lockstat_patch_point: 80 ret 81 testb %al, %al 82 jnz 0f 83 ret 840: 85 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 86 movq %rdi, %rsi /* rsi = lock addr */ 87 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */ 88 jmp lockstat_wrapper 89 SET_SIZE(lock_try) 90 91 ENTRY(lock_spin_try) 92 movb $-1, %dl 93 movzbq %dl, %rax 94 xchgb %dl, (%rdi) 95 xorb %dl, %al 96 ret 97 SET_SIZE(lock_spin_try) 98 99 ENTRY(ulock_try) 100#ifdef DEBUG 101 movq kernelbase(%rip), %rax 102 cmpq %rax, %rdi /* test uaddr < kernelbase */ 103 jb ulock_pass /* uaddr < kernelbase, proceed */ 104 105 movq %rdi, %r12 /* preserve lock ptr for debugging */ 106 leaq .ulock_panic_msg(%rip), %rdi 107 pushq %rbp /* align stack properly */ 108 movq %rsp, %rbp 109 xorl %eax, %eax /* clear for varargs */ 110 call panic 111 112#endif /* DEBUG */ 113 114ulock_pass: 115 movl $1, %eax 116 xchgb %al, (%rdi) 117 xorb $1, %al 118 ret 119 SET_SIZE(ulock_try) 120 121#else 122 123 ENTRY(lock_try) 124 movl $1,%edx 125 movl 4(%esp),%ecx /* ecx = lock addr */ 126 xorl %eax,%eax 127 xchgb %dl, (%ecx) /* using dl will avoid partial */ 128 testb %dl,%dl /* stalls on P6 ? */ 129 setz %al 130.lock_try_lockstat_patch_point: 131 ret 132 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 133 testl %eax, %eax 134 jz 0f 135 movl $LS_LOCK_TRY_ACQUIRE, %eax 136 jmp lockstat_wrapper 1370: 138 ret 139 SET_SIZE(lock_try) 140 141 ENTRY(lock_spin_try) 142 movl $-1,%edx 143 movl 4(%esp),%ecx /* ecx = lock addr */ 144 xorl %eax,%eax 145 xchgb %dl, (%ecx) /* using dl will avoid partial */ 146 testb %dl,%dl /* stalls on P6 ? */ 147 setz %al 148 ret 149 SET_SIZE(lock_spin_try) 150 151 ENTRY(ulock_try) 152#ifdef DEBUG 153 movl kernelbase, %eax 154 cmpl %eax, 4(%esp) /* test uaddr < kernelbase */ 155 jb ulock_pass /* uaddr < kernelbase, proceed */ 156 157 pushl $.ulock_panic_msg 158 call panic 159 160#endif /* DEBUG */ 161 162ulock_pass: 163 movl $1,%eax 164 movl 4(%esp),%ecx 165 xchgb %al, (%ecx) 166 xorb $1, %al 167 ret 168 SET_SIZE(ulock_try) 169 170#endif /* !__amd64 */ 171 172#ifdef DEBUG 173 .data 174.ulock_panic_msg: 175 .string "ulock_try: Argument is above kernelbase" 176 .text 177#endif /* DEBUG */ 178 179#endif /* __lint */ 180 181/* 182 * lock_clear(lp) 183 * - unlock lock without changing interrupt priority level. 184 */ 185 186#if defined(lint) || defined(__lint) 187 188/* ARGSUSED */ 189void 190lock_clear(lock_t *lp) 191{} 192 193/* ARGSUSED */ 194void 195ulock_clear(lock_t *lp) 196{} 197 198#else /* __lint */ 199 200#if defined(__amd64) 201 202 ENTRY(lock_clear) 203 movb $0, (%rdi) 204.lock_clear_lockstat_patch_point: 205 ret 206 movq %rdi, %rsi /* rsi = lock addr */ 207 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 208 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */ 209 jmp lockstat_wrapper 210 SET_SIZE(lock_clear) 211 212 ENTRY(ulock_clear) 213#ifdef DEBUG 214 movq kernelbase(%rip), %rcx 215 cmpq %rcx, %rdi /* test uaddr < kernelbase */ 216 jb ulock_clr /* uaddr < kernelbase, proceed */ 217 218 leaq .ulock_clear_msg(%rip), %rdi 219 pushq %rbp /* align stack properly */ 220 movq %rsp, %rbp 221 xorl %eax, %eax /* clear for varargs */ 222 call panic 223#endif 224 225ulock_clr: 226 movb $0, (%rdi) 227 ret 228 SET_SIZE(ulock_clear) 229 230#else 231 232 ENTRY(lock_clear) 233 movl 4(%esp), %eax 234 movb $0, (%eax) 235.lock_clear_lockstat_patch_point: 236 ret 237 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 238 movl %eax, %ecx /* ecx = lock pointer */ 239 movl $LS_LOCK_CLEAR_RELEASE, %eax 240 jmp lockstat_wrapper 241 SET_SIZE(lock_clear) 242 243 ENTRY(ulock_clear) 244#ifdef DEBUG 245 movl kernelbase, %ecx 246 cmpl %ecx, 4(%esp) /* test uaddr < kernelbase */ 247 jb ulock_clr /* uaddr < kernelbase, proceed */ 248 249 pushl $.ulock_clear_msg 250 call panic 251#endif 252 253ulock_clr: 254 movl 4(%esp),%eax 255 xorl %ecx,%ecx 256 movb %cl, (%eax) 257 ret 258 SET_SIZE(ulock_clear) 259 260#endif /* !__amd64 */ 261 262#ifdef DEBUG 263 .data 264.ulock_clear_msg: 265 .string "ulock_clear: Argument is above kernelbase" 266 .text 267#endif /* DEBUG */ 268 269 270#endif /* __lint */ 271 272/* 273 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil) 274 * Drops lp, sets pil to new_pil, stores old pil in *old_pil. 275 */ 276 277#if defined(lint) || defined(__lint) 278 279/* ARGSUSED */ 280void 281lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil) 282{} 283 284#else /* __lint */ 285 286#if defined(__amd64) 287 288 ENTRY(lock_set_spl) 289 pushq %rbp 290 movq %rsp, %rbp 291 subq $32, %rsp 292 movl %esi, 8(%rsp) /* save priority level */ 293 movq %rdx, 16(%rsp) /* save old pil ptr */ 294 movq %rdi, 24(%rsp) /* save lock pointer */ 295 movl %esi, %edi /* pass priority level */ 296 call splr /* raise priority level */ 297 movq 24(%rsp), %rdi /* rdi = lock addr */ 298 movb $-1, %dl 299 xchgb %dl, (%rdi) /* try to set lock */ 300 testb %dl, %dl /* did we get the lock? ... */ 301 jnz .lss_miss /* ... no, go to C for the hard case */ 302 movq 16(%rsp), %rdx /* rdx = old pil addr */ 303 movw %ax, (%rdx) /* store old pil */ 304 leave 305.lock_set_spl_lockstat_patch_point: 306 ret 307 movq %rdi, %rsi /* rsi = lock addr */ 308 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 309 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi 310 jmp lockstat_wrapper 311.lss_miss: 312 movl 8(%rsp), %esi /* new_pil */ 313 movq 16(%rsp), %rdx /* old_pil_addr */ 314 movl %eax, %ecx /* original pil */ 315 leave /* unwind stack */ 316 jmp lock_set_spl_spin 317 SET_SIZE(lock_set_spl) 318 319#else 320 321 ENTRY(lock_set_spl) 322 movl 8(%esp), %eax /* get priority level */ 323 pushl %eax 324 call splr /* raise priority level */ 325 movl 8(%esp), %ecx /* ecx = lock addr */ 326 movl $-1, %edx 327 addl $4, %esp 328 xchgb %dl, (%ecx) /* try to set lock */ 329 testb %dl, %dl /* did we get the lock? ... */ 330 movl 12(%esp), %edx /* edx = olp pil addr (ZF unaffected) */ 331 jnz .lss_miss /* ... no, go to C for the hard case */ 332 movw %ax, (%edx) /* store old pil */ 333.lock_set_spl_lockstat_patch_point: 334 ret 335 movl %gs:CPU_THREAD, %edx /* edx = thread addr*/ 336 movl $LS_LOCK_SET_SPL_ACQUIRE, %eax 337 jmp lockstat_wrapper 338.lss_miss: 339 pushl %eax /* original pil */ 340 pushl %edx /* old_pil addr */ 341 pushl 16(%esp) /* new_pil */ 342 pushl %ecx /* lock addr */ 343 call lock_set_spl_spin 344 addl $16, %esp 345 ret 346 SET_SIZE(lock_set_spl) 347 348#endif /* !__amd64 */ 349 350#endif /* __lint */ 351 352/* 353 * void 354 * lock_init(lp) 355 */ 356 357#if defined(__lint) 358 359/* ARGSUSED */ 360void 361lock_init(lock_t *lp) 362{} 363 364#else /* __lint */ 365 366#if defined(__amd64) 367 368 ENTRY(lock_init) 369 movb $0, (%rdi) 370 ret 371 SET_SIZE(lock_init) 372 373#else 374 375 ENTRY(lock_init) 376 movl 4(%esp), %eax 377 movb $0, (%eax) 378 ret 379 SET_SIZE(lock_init) 380 381#endif /* !__amd64 */ 382 383#endif /* __lint */ 384 385/* 386 * void 387 * lock_set(lp) 388 */ 389 390#if defined(lint) || defined(__lint) 391 392/* ARGSUSED */ 393void 394lock_set(lock_t *lp) 395{} 396 397#else /* __lint */ 398 399#if defined(__amd64) 400 401 ENTRY(lock_set) 402 movb $-1, %dl 403 xchgb %dl, (%rdi) /* try to set lock */ 404 testb %dl, %dl /* did we get it? */ 405 jnz lock_set_spin /* no, go to C for the hard case */ 406.lock_set_lockstat_patch_point: 407 ret 408 movq %rdi, %rsi /* rsi = lock addr */ 409 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 410 movl $LS_LOCK_SET_ACQUIRE, %edi 411 jmp lockstat_wrapper 412 SET_SIZE(lock_set) 413 414#else 415 416 ENTRY(lock_set) 417 movl 4(%esp), %ecx /* ecx = lock addr */ 418 movl $-1, %edx 419 xchgb %dl, (%ecx) /* try to set lock */ 420 testb %dl, %dl /* did we get it? */ 421 jnz lock_set_spin /* no, go to C for the hard case */ 422.lock_set_lockstat_patch_point: 423 ret 424 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 425 movl $LS_LOCK_SET_ACQUIRE, %eax 426 jmp lockstat_wrapper 427 SET_SIZE(lock_set) 428 429#endif /* !__amd64 */ 430 431#endif /* __lint */ 432 433/* 434 * lock_clear_splx(lp, s) 435 */ 436 437#if defined(lint) || defined(__lint) 438 439/* ARGSUSED */ 440void 441lock_clear_splx(lock_t *lp, int s) 442{} 443 444#else /* __lint */ 445 446#if defined(__amd64) 447 448 ENTRY(lock_clear_splx) 449 movb $0, (%rdi) /* clear lock */ 450.lock_clear_splx_lockstat_patch_point: 451 jmp 0f 4520: 453 movl %esi, %edi /* arg for splx */ 454 jmp splx /* let splx do its thing */ 455.lock_clear_splx_lockstat: 456 pushq %rbp /* align stack properly */ 457 movq %rsp, %rbp 458 subq $16, %rsp /* space to save args across splx */ 459 movq %rdi, 8(%rsp) /* save lock ptr across splx call */ 460 movl %esi, %edi /* arg for splx */ 461 call splx /* lower the priority */ 462 movq 8(%rsp), %rsi /* rsi = lock ptr */ 463 leave /* unwind stack */ 464 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 465 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi 466 jmp lockstat_wrapper 467 SET_SIZE(lock_clear_splx) 468 469#else 470 471 ENTRY(lock_clear_splx) 472 movl 4(%esp), %eax /* eax = lock addr */ 473 movb $0, (%eax) /* clear lock */ 474.lock_clear_splx_lockstat_patch_point: 475 jmp 0f 4760: 477 movl 8(%esp), %edx /* edx = desired pil */ 478 movl %edx, 4(%esp) /* set spl arg up for splx */ 479 jmp splx /* let splx do it's thing */ 480.lock_clear_splx_lockstat: 481 movl 8(%esp), %edx /* edx = desired pil */ 482 pushl %ebp /* set up stack frame */ 483 movl %esp, %ebp 484 pushl %edx 485 call splx 486 leave /* unwind stack */ 487 movl 4(%esp), %ecx /* ecx = lock pointer */ 488 movl %gs:CPU_THREAD, %edx /* edx = thread addr */ 489 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %eax 490 jmp lockstat_wrapper 491 SET_SIZE(lock_clear_splx) 492 493#endif /* !__amd64 */ 494 495#if defined(__GNUC_AS__) 496#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \ 497 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2) 498 499#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \ 500 (.lock_clear_splx_lockstat_patch_point + 1) 501#else 502#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \ 503 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2] 504 505#define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \ 506 [.lock_clear_splx_lockstat_patch_point + 1] 507#endif 508 509#endif /* __lint */ 510 511/* 512 * mutex_enter() and mutex_exit(). 513 * 514 * These routines handle the simple cases of mutex_enter() (adaptive 515 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). 516 * If anything complicated is going on we punt to mutex_vector_enter(). 517 * 518 * mutex_tryenter() is similar to mutex_enter() but returns zero if 519 * the lock cannot be acquired, nonzero on success. 520 * 521 * If mutex_exit() gets preempted in the window between checking waiters 522 * and clearing the lock, we can miss wakeups. Disabling preemption 523 * in the mutex code is prohibitively expensive, so instead we detect 524 * mutex preemption by examining the trapped PC in the interrupt path. 525 * If we interrupt a thread in mutex_exit() that has not yet cleared 526 * the lock, cmnint() resets its PC back to the beginning of 527 * mutex_exit() so it will check again for waiters when it resumes. 528 * 529 * The lockstat code below is activated when the lockstat driver 530 * calls lockstat_hot_patch() to hot-patch the kernel mutex code. 531 * Note that we don't need to test lockstat_event_mask here -- we won't 532 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats. 533 */ 534#if defined(lint) || defined(__lint) 535 536/* ARGSUSED */ 537void 538mutex_enter(kmutex_t *lp) 539{} 540 541/* ARGSUSED */ 542int 543mutex_tryenter(kmutex_t *lp) 544{ return (0); } 545 546/* ARGSUSED */ 547int 548mutex_adaptive_tryenter(mutex_impl_t *lp) 549{ return (0); } 550 551/* ARGSUSED */ 552void 553mutex_exit(kmutex_t *lp) 554{} 555 556#else 557 558#if defined(__amd64) 559 560 ENTRY_NP(mutex_enter) 561 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 562 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 563 lock 564 cmpxchgq %rdx, (%rdi) 565 jnz mutex_vector_enter 566.mutex_enter_lockstat_patch_point: 567#if defined(OPTERON_WORKAROUND_6323525) 568.mutex_enter_6323525_patch_point: 569 ret /* nop space for lfence */ 570 nop 571 nop 572.mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */ 573 nop 574#else /* OPTERON_WORKAROUND_6323525 */ 575 ret 576#endif /* OPTERON_WORKAROUND_6323525 */ 577 movq %rdi, %rsi 578 movl $LS_MUTEX_ENTER_ACQUIRE, %edi 579/* 580 * expects %rdx=thread, %rsi=lock, %edi=lockstat event 581 */ 582 ALTENTRY(lockstat_wrapper) 583 incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */ 584 leaq lockstat_probemap(%rip), %rax 585 movl (%rax, %rdi, DTRACE_IDSIZE), %eax 586 testl %eax, %eax /* check for non-zero probe */ 587 jz 1f 588 pushq %rbp /* align stack properly */ 589 movq %rsp, %rbp 590 movl %eax, %edi 591 call *lockstat_probe 592 leave /* unwind stack */ 5931: 594 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 595 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 596 movl $1, %eax /* return success if tryenter */ 597 ret 598 SET_SIZE(lockstat_wrapper) 599 SET_SIZE(mutex_enter) 600 601/* 602 * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event 603 */ 604 ENTRY(lockstat_wrapper_arg) 605 incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */ 606 leaq lockstat_probemap(%rip), %rax 607 movl (%rax, %rdi, DTRACE_IDSIZE), %eax 608 testl %eax, %eax /* check for non-zero probe */ 609 jz 1f 610 pushq %rbp /* align stack properly */ 611 movq %rsp, %rbp 612 movl %eax, %edi 613 call *lockstat_probe 614 leave /* unwind stack */ 6151: 616 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 617 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 618 movl $1, %eax /* return success if tryenter */ 619 ret 620 SET_SIZE(lockstat_wrapper_arg) 621 622 623 ENTRY(mutex_tryenter) 624 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 625 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 626 lock 627 cmpxchgq %rdx, (%rdi) 628 jnz mutex_vector_tryenter 629 not %eax /* return success (nonzero) */ 630#if defined(OPTERON_WORKAROUND_6323525) 631.mutex_tryenter_lockstat_patch_point: 632.mutex_tryenter_6323525_patch_point: 633 ret /* nop space for lfence */ 634 nop 635 nop 636.mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */ 637 nop 638#else /* OPTERON_WORKAROUND_6323525 */ 639.mutex_tryenter_lockstat_patch_point: 640 ret 641#endif /* OPTERON_WORKAROUND_6323525 */ 642 movq %rdi, %rsi 643 movl $LS_MUTEX_ENTER_ACQUIRE, %edi 644 jmp lockstat_wrapper 645 SET_SIZE(mutex_tryenter) 646 647 ENTRY(mutex_adaptive_tryenter) 648 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 649 xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 650 lock 651 cmpxchgq %rdx, (%rdi) 652 jnz 0f 653 not %eax /* return success (nonzero) */ 654#if defined(OPTERON_WORKAROUND_6323525) 655.mutex_atryenter_6323525_patch_point: 656 ret /* nop space for lfence */ 657 nop 658 nop 659 nop 660#else /* OPTERON_WORKAROUND_6323525 */ 661 ret 662#endif /* OPTERON_WORKAROUND_6323525 */ 6630: 664 xorl %eax, %eax /* return failure */ 665 ret 666 SET_SIZE(mutex_adaptive_tryenter) 667 668 .globl mutex_exit_critical_start 669 670 ENTRY(mutex_exit) 671mutex_exit_critical_start: /* If interrupted, restart here */ 672 movq %gs:CPU_THREAD, %rdx 673 cmpq %rdx, (%rdi) 674 jne mutex_vector_exit /* wrong type or wrong owner */ 675 movq $0, (%rdi) /* clear owner AND lock */ 676.mutex_exit_critical_end: 677.mutex_exit_lockstat_patch_point: 678 ret 679 movq %rdi, %rsi 680 movl $LS_MUTEX_EXIT_RELEASE, %edi 681 jmp lockstat_wrapper 682 SET_SIZE(mutex_exit) 683 684 .globl mutex_exit_critical_size 685 .type mutex_exit_critical_size, @object 686 .align CPTRSIZE 687mutex_exit_critical_size: 688 .quad .mutex_exit_critical_end - mutex_exit_critical_start 689 SET_SIZE(mutex_exit_critical_size) 690 691#else 692 693 ENTRY_NP(mutex_enter) 694 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 695 movl 4(%esp), %ecx /* ecx = lock ptr */ 696 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 697 lock 698 cmpxchgl %edx, (%ecx) 699 jnz mutex_vector_enter 700#if defined(OPTERON_WORKAROUND_6323525) 701.mutex_enter_lockstat_patch_point: 702.mutex_enter_6323525_patch_point: 703 ret /* nop space for lfence */ 704 nop 705 nop 706.mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */ 707 nop 708#else /* OPTERON_WORKAROUND_6323525 */ 709.mutex_enter_lockstat_patch_point: 710 ret 711#endif /* OPTERON_WORKAROUND_6323525 */ 712 movl $LS_MUTEX_ENTER_ACQUIRE, %eax 713 ALTENTRY(lockstat_wrapper) /* expects edx=thread, ecx=lock, */ 714 /* eax=lockstat event */ 715 pushl %ebp /* buy a frame */ 716 movl %esp, %ebp 717 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */ 718 pushl %edx /* save thread pointer */ 719 movl $lockstat_probemap, %edx 720 movl (%edx, %eax, DTRACE_IDSIZE), %eax 721 testl %eax, %eax /* check for non-zero probe */ 722 jz 1f 723 pushl %ecx /* push lock */ 724 pushl %eax /* push probe ID */ 725 call *lockstat_probe 726 addl $8, %esp 7271: 728 popl %edx /* restore thread pointer */ 729 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */ 730 movl $1, %eax /* return success if tryenter */ 731 popl %ebp /* pop off frame */ 732 ret 733 SET_SIZE(lockstat_wrapper) 734 SET_SIZE(mutex_enter) 735 736 ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */ 737 /* eax=lockstat event, pushed arg */ 738 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */ 739 pushl %edx /* save thread pointer */ 740 movl $lockstat_probemap, %edx 741 movl (%edx, %eax, DTRACE_IDSIZE), %eax 742 testl %eax, %eax /* check for non-zero probe */ 743 jz 1f 744 pushl %ebp /* save %ebp */ 745 pushl 8(%esp) /* push arg1 */ 746 movl %ebp, 12(%esp) /* fake up the stack frame */ 747 movl %esp, %ebp /* fake up base pointer */ 748 addl $12, %ebp /* adjust faked base pointer */ 749 pushl %ecx /* push lock */ 750 pushl %eax /* push probe ID */ 751 call *lockstat_probe 752 addl $12, %esp /* adjust for arguments */ 753 popl %ebp /* pop frame */ 7541: 755 popl %edx /* restore thread pointer */ 756 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */ 757 movl $1, %eax /* return success if tryenter */ 758 addl $4, %esp /* pop argument */ 759 ret 760 SET_SIZE(lockstat_wrapper_arg) 761 762 763 ENTRY(mutex_tryenter) 764 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 765 movl 4(%esp), %ecx /* ecx = lock ptr */ 766 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 767 lock 768 cmpxchgl %edx, (%ecx) 769 jnz mutex_vector_tryenter 770 movl %ecx, %eax 771#if defined(OPTERON_WORKAROUND_6323525) 772.mutex_tryenter_lockstat_patch_point: 773.mutex_tryenter_6323525_patch_point: 774 ret /* nop space for lfence */ 775 nop 776 nop 777.mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */ 778 nop 779#else /* OPTERON_WORKAROUND_6323525 */ 780.mutex_tryenter_lockstat_patch_point: 781 ret 782#endif /* OPTERON_WORKAROUND_6323525 */ 783 movl $LS_MUTEX_ENTER_ACQUIRE, %eax 784 jmp lockstat_wrapper 785 SET_SIZE(mutex_tryenter) 786 787 ENTRY(mutex_adaptive_tryenter) 788 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 789 movl 4(%esp), %ecx /* ecx = lock ptr */ 790 xorl %eax, %eax /* eax = 0 (unheld adaptive) */ 791 lock 792 cmpxchgl %edx, (%ecx) 793 jnz 0f 794 movl %ecx, %eax 795#if defined(OPTERON_WORKAROUND_6323525) 796.mutex_atryenter_6323525_patch_point: 797 ret /* nop space for lfence */ 798 nop 799 nop 800 nop 801#else /* OPTERON_WORKAROUND_6323525 */ 802 ret 803#endif /* OPTERON_WORKAROUND_6323525 */ 8040: 805 xorl %eax, %eax 806 ret 807 SET_SIZE(mutex_adaptive_tryenter) 808 809 .globl mutex_exit_critical_size 810 .globl mutex_exit_critical_start 811 812 ENTRY(mutex_exit) 813mutex_exit_critical_start: /* If interrupted, restart here */ 814 movl %gs:CPU_THREAD, %edx 815 movl 4(%esp), %ecx 816 cmpl %edx, (%ecx) 817 jne mutex_vector_exit /* wrong type or wrong owner */ 818 movl $0, (%ecx) /* clear owner AND lock */ 819.mutex_exit_critical_end: 820.mutex_exit_lockstat_patch_point: 821 ret 822 movl $LS_MUTEX_EXIT_RELEASE, %eax 823 jmp lockstat_wrapper 824 SET_SIZE(mutex_exit) 825 826 .globl mutex_exit_critical_size 827 .type mutex_exit_critical_size, @object 828 .align CPTRSIZE 829mutex_exit_critical_size: 830 .long .mutex_exit_critical_end - mutex_exit_critical_start 831 SET_SIZE(mutex_exit_critical_size) 832 833#endif /* !__amd64 */ 834 835#endif /* __lint */ 836 837/* 838 * rw_enter() and rw_exit(). 839 * 840 * These routines handle the simple cases of rw_enter (write-locking an unheld 841 * lock or read-locking a lock that's neither write-locked nor write-wanted) 842 * and rw_exit (no waiters or not the last reader). If anything complicated 843 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. 844 */ 845#if defined(lint) || defined(__lint) 846 847/* ARGSUSED */ 848void 849rw_enter(krwlock_t *lp, krw_t rw) 850{} 851 852/* ARGSUSED */ 853void 854rw_exit(krwlock_t *lp) 855{} 856 857#else /* __lint */ 858 859#if defined(__amd64) 860 861 ENTRY(rw_enter) 862 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 863 cmpl $RW_WRITER, %esi 864 je .rw_write_enter 865 incl T_KPRI_REQ(%rdx) /* THREAD_KPRI_REQUEST() */ 866 movq (%rdi), %rax /* rax = old rw_wwwh value */ 867 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax 868 jnz rw_enter_sleep 869 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */ 870 lock 871 cmpxchgq %rdx, (%rdi) /* try to grab read lock */ 872 jnz rw_enter_sleep 873.rw_read_enter_lockstat_patch_point: 874 ret 875 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 876 movq %rdi, %rsi /* rsi = lock ptr */ 877 movl $LS_RW_ENTER_ACQUIRE, %edi 878 movl $RW_READER, %edx 879 jmp lockstat_wrapper_arg 880.rw_write_enter: 881 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */ 882 xorl %eax, %eax /* rax = unheld value */ 883 lock 884 cmpxchgq %rdx, (%rdi) /* try to grab write lock */ 885 jnz rw_enter_sleep 886 887#if defined(OPTERON_WORKAROUND_6323525) 888.rw_write_enter_lockstat_patch_point: 889.rw_write_enter_6323525_patch_point: 890 ret 891 nop 892 nop 893.rw_write_enter_lockstat_6323525_patch_point: 894 nop 895#else /* OPTERON_WORKAROUND_6323525 */ 896.rw_write_enter_lockstat_patch_point: 897 ret 898#endif /* OPTERON_WORKAROUND_6323525 */ 899 900 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 901 movq %rdi, %rsi /* rsi = lock ptr */ 902 movl $LS_RW_ENTER_ACQUIRE, %edi 903 movl $RW_WRITER, %edx 904 jmp lockstat_wrapper_arg 905 SET_SIZE(rw_enter) 906 907 ENTRY(rw_exit) 908 movq (%rdi), %rax /* rax = old rw_wwwh value */ 909 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */ 910 jne .rw_not_single_reader 911 xorl %edx, %edx /* rdx = new value (unheld) */ 912.rw_read_exit: 913 lock 914 cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 915 jnz rw_exit_wakeup 916 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 917 decl T_KPRI_REQ(%rcx) /* THREAD_KPRI_RELEASE() */ 918.rw_read_exit_lockstat_patch_point: 919 ret 920 movq %rdi, %rsi /* rsi = lock ptr */ 921 movl $LS_RW_EXIT_RELEASE, %edi 922 movl $RW_READER, %edx 923 jmp lockstat_wrapper_arg 924.rw_not_single_reader: 925 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */ 926 jnz .rw_write_exit 927 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */ 928 cmpl $RW_READ_LOCK, %edx 929 jge .rw_read_exit /* not last reader, safe to drop */ 930 jmp rw_exit_wakeup /* last reader with waiters */ 931.rw_write_exit: 932 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */ 933 xorl %edx, %edx /* rdx = new value (unheld) */ 934 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */ 935 lock 936 cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 937 jnz rw_exit_wakeup 938.rw_write_exit_lockstat_patch_point: 939 ret 940 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 941 movq %rdi, %rsi /* rsi - lock ptr */ 942 movl $LS_RW_EXIT_RELEASE, %edi 943 movl $RW_WRITER, %edx 944 jmp lockstat_wrapper_arg 945 SET_SIZE(rw_exit) 946 947#else 948 949 ENTRY(rw_enter) 950 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 951 movl 4(%esp), %ecx /* ecx = lock ptr */ 952 cmpl $RW_WRITER, 8(%esp) 953 je .rw_write_enter 954 incl T_KPRI_REQ(%edx) /* THREAD_KPRI_REQUEST() */ 955 movl (%ecx), %eax /* eax = old rw_wwwh value */ 956 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax 957 jnz rw_enter_sleep 958 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */ 959 lock 960 cmpxchgl %edx, (%ecx) /* try to grab read lock */ 961 jnz rw_enter_sleep 962.rw_read_enter_lockstat_patch_point: 963 ret 964 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 965 movl $LS_RW_ENTER_ACQUIRE, %eax 966 pushl $RW_READER 967 jmp lockstat_wrapper_arg 968.rw_write_enter: 969 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */ 970 xorl %eax, %eax /* eax = unheld value */ 971 lock 972 cmpxchgl %edx, (%ecx) /* try to grab write lock */ 973 jnz rw_enter_sleep 974 975#if defined(OPTERON_WORKAROUND_6323525) 976.rw_write_enter_lockstat_patch_point: 977.rw_write_enter_6323525_patch_point: 978 ret 979 nop 980 nop 981.rw_write_enter_lockstat_6323525_patch_point: 982 nop 983#else /* OPTERON_WORKAROUND_6323525 */ 984.rw_write_enter_lockstat_patch_point: 985 ret 986#endif /* OPTERON_WORKAROUND_6323525 */ 987 988 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 989 movl $LS_RW_ENTER_ACQUIRE, %eax 990 pushl $RW_WRITER 991 jmp lockstat_wrapper_arg 992 SET_SIZE(rw_enter) 993 994 ENTRY(rw_exit) 995 movl 4(%esp), %ecx /* ecx = lock ptr */ 996 movl (%ecx), %eax /* eax = old rw_wwwh value */ 997 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */ 998 jne .rw_not_single_reader 999 xorl %edx, %edx /* edx = new value (unheld) */ 1000.rw_read_exit: 1001 lock 1002 cmpxchgl %edx, (%ecx) /* try to drop read lock */ 1003 jnz rw_exit_wakeup 1004 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1005 decl T_KPRI_REQ(%edx) /* THREAD_KPRI_RELEASE() */ 1006.rw_read_exit_lockstat_patch_point: 1007 ret 1008 movl $LS_RW_EXIT_RELEASE, %eax 1009 pushl $RW_READER 1010 jmp lockstat_wrapper_arg 1011.rw_not_single_reader: 1012 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */ 1013 jnz .rw_write_exit 1014 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */ 1015 cmpl $RW_READ_LOCK, %edx 1016 jge .rw_read_exit /* not last reader, safe to drop */ 1017 jmp rw_exit_wakeup /* last reader with waiters */ 1018.rw_write_exit: 1019 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */ 1020 xorl %edx, %edx /* edx = new value (unheld) */ 1021 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */ 1022 lock 1023 cmpxchgl %edx, (%ecx) /* try to drop read lock */ 1024 jnz rw_exit_wakeup 1025.rw_write_exit_lockstat_patch_point: 1026 ret 1027 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ 1028 movl $LS_RW_EXIT_RELEASE, %eax 1029 pushl $RW_WRITER 1030 jmp lockstat_wrapper_arg 1031 SET_SIZE(rw_exit) 1032 1033#endif /* !__amd64 */ 1034 1035#endif /* __lint */ 1036 1037#if defined(OPTERON_WORKAROUND_6323525) 1038#if defined(lint) || defined(__lint) 1039 1040int workaround_6323525_patched; 1041 1042void 1043patch_workaround_6323525(void) 1044{} 1045 1046#else /* lint */ 1047 1048/* 1049 * If it is necessary to patch the lock enter routines with the lfence 1050 * workaround, workaround_6323525_patched is set to a non-zero value so that 1051 * the lockstat_hat_patch routine can patch to the new location of the 'ret' 1052 * instruction. 1053 */ 1054 DGDEF3(workaround_6323525_patched, 4, 4) 1055 .long 0 1056 1057#if defined(__amd64) 1058 1059#define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \ 1060 movq $size, %rbx; \ 1061 movq $dstaddr, %r13; \ 1062 addq %rbx, %r13; \ 1063 movq $srcaddr, %r12; \ 1064 addq %rbx, %r12; \ 10650: \ 1066 decq %r13; \ 1067 decq %r12; \ 1068 movzbl (%r12), %esi; \ 1069 movq $1, %rdx; \ 1070 movq %r13, %rdi; \ 1071 call hot_patch_kernel_text; \ 1072 decq %rbx; \ 1073 testq %rbx, %rbx; \ 1074 jg 0b; 1075 1076/* 1077 * patch_workaround_6323525: provide workaround for 6323525 1078 * 1079 * The workaround is to place a fencing instruction (lfence) between the 1080 * mutex operation and the subsequent read-modify-write instruction. 1081 * 1082 * This routine hot patches the lfence instruction on top of the space 1083 * reserved by nops in the lock enter routines. 1084 */ 1085 ENTRY_NP(patch_workaround_6323525) 1086 pushq %rbp 1087 movq %rsp, %rbp 1088 pushq %r12 1089 pushq %r13 1090 pushq %rbx 1091 1092 /* 1093 * lockstat_hot_patch() to use the alternate lockstat workaround 1094 * 6323525 patch points (points past the lfence instruction to the 1095 * new ret) when workaround_6323525_patched is set. 1096 */ 1097 movl $1, workaround_6323525_patched 1098 1099 /* 1100 * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter 1101 * routines. The 4 bytes are patched in reverse order so that the 1102 * the existing ret is overwritten last. This provides lock enter 1103 * sanity during the intermediate patching stages. 1104 */ 1105 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4) 1106 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4) 1107 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4) 1108 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4) 1109 1110 popq %rbx 1111 popq %r13 1112 popq %r12 1113 movq %rbp, %rsp 1114 popq %rbp 1115 ret 1116_lfence_insn: 1117 lfence 1118 ret 1119 SET_SIZE(patch_workaround_6323525) 1120 1121 1122#else /* __amd64 */ 1123 1124#define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \ 1125 movl $size, %ebx; \ 1126 movl $srcaddr, %esi; \ 1127 addl %ebx, %esi; \ 1128 movl $dstaddr, %edi; \ 1129 addl %ebx, %edi; \ 11300: \ 1131 decl %esi; \ 1132 decl %edi; \ 1133 pushl $1; \ 1134 movzbl (%esi), %eax; \ 1135 pushl %eax; \ 1136 pushl %edi; \ 1137 call hot_patch_kernel_text; \ 1138 addl $12, %esp; \ 1139 decl %ebx; \ 1140 testl %ebx, %ebx; \ 1141 jg 0b; 1142 1143 1144 /* see comments above */ 1145 ENTRY_NP(patch_workaround_6323525) 1146 pushl %ebp 1147 movl %esp, %ebp 1148 pushl %ebx 1149 pushl %esi 1150 pushl %edi 1151 1152 movl $1, workaround_6323525_patched 1153 1154 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4) 1155 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4) 1156 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4) 1157 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4) 1158 1159 popl %edi 1160 popl %esi 1161 popl %ebx 1162 movl %ebp, %esp 1163 popl %ebp 1164 ret 1165_lfence_insn: 1166 .byte 0xf, 0xae, 0xe8 / [lfence instruction] 1167 ret 1168 SET_SIZE(patch_workaround_6323525) 1169 1170#endif /* !__amd64 */ 1171#endif /* !lint */ 1172#endif /* OPTERON_WORKAROUND_6323525 */ 1173 1174 1175#if defined(lint) || defined(__lint) 1176 1177void 1178lockstat_hot_patch(void) 1179{} 1180 1181#else 1182 1183#if defined(__amd64) 1184 1185#define HOT_PATCH(addr, event, active_instr, normal_instr, len) \ 1186 movq $normal_instr, %rsi; \ 1187 movq $active_instr, %rdi; \ 1188 leaq lockstat_probemap(%rip), %rax; \ 1189 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \ 1190 testl %eax, %eax; \ 1191 jz 9f; \ 1192 movq %rdi, %rsi; \ 11939: \ 1194 movq $len, %rdx; \ 1195 movq $addr, %rdi; \ 1196 call hot_patch_kernel_text 1197 1198#else 1199 1200#define HOT_PATCH(addr, event, active_instr, normal_instr, len) \ 1201 movl $normal_instr, %ecx; \ 1202 movl $active_instr, %edx; \ 1203 movl $lockstat_probemap, %eax; \ 1204 movl _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \ 1205 testl %eax, %eax; \ 1206 jz . + 4; \ 1207 movl %edx, %ecx; \ 1208 pushl $len; \ 1209 pushl %ecx; \ 1210 pushl $addr; \ 1211 call hot_patch_kernel_text; \ 1212 addl $12, %esp; 1213 1214#endif /* !__amd64 */ 1215 1216 ENTRY(lockstat_hot_patch) 1217#if defined(__amd64) 1218 pushq %rbp /* align stack properly */ 1219 movq %rsp, %rbp 1220#endif /* __amd64 */ 1221 1222#if defined(OPTERON_WORKAROUND_6323525) 1223 cmpl $0, workaround_6323525_patched 1224 je 1f 1225 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point, 1226 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1227 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point, 1228 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1229 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point, 1230 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1231 jmp 2f 12321: 1233 HOT_PATCH(.mutex_enter_lockstat_patch_point, 1234 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1235 HOT_PATCH(.mutex_tryenter_lockstat_patch_point, 1236 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1237 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 1238 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 12392: 1240#else /* OPTERON_WORKAROUND_6323525 */ 1241 HOT_PATCH(.mutex_enter_lockstat_patch_point, 1242 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1243 HOT_PATCH(.mutex_tryenter_lockstat_patch_point, 1244 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1245 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 1246 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1247#endif /* !OPTERON_WORKAROUND_6323525 */ 1248 HOT_PATCH(.mutex_exit_lockstat_patch_point, 1249 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1250 HOT_PATCH(.rw_read_enter_lockstat_patch_point, 1251 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1252 HOT_PATCH(.rw_write_exit_lockstat_patch_point, 1253 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1254 HOT_PATCH(.rw_read_exit_lockstat_patch_point, 1255 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1) 1256 HOT_PATCH(.lock_set_lockstat_patch_point, 1257 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1258 HOT_PATCH(.lock_try_lockstat_patch_point, 1259 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1260 HOT_PATCH(.lock_clear_lockstat_patch_point, 1261 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1) 1262 HOT_PATCH(.lock_set_spl_lockstat_patch_point, 1263 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1) 1264 1265 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT, 1266 LS_LOCK_CLEAR_SPLX_RELEASE, 1267 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1); 1268#if defined(__amd64) 1269 leave /* unwind stack */ 1270#endif /* __amd64 */ 1271 ret 1272 SET_SIZE(lockstat_hot_patch) 1273 1274#endif /* __lint */ 1275 1276#if defined(lint) || defined(__lint) 1277 1278/* XX64 membar_*() should be inlines */ 1279 1280void 1281membar_enter(void) 1282{} 1283 1284void 1285membar_exit(void) 1286{} 1287 1288void 1289membar_producer(void) 1290{} 1291 1292void 1293membar_consumer(void) 1294{} 1295 1296#else /* __lint */ 1297 1298#if defined(__amd64) 1299 1300 ENTRY(membar_enter) 1301 ALTENTRY(membar_exit) 1302 mfence /* lighter weight than lock; xorq $0,(%rsp) */ 1303 ret 1304 SET_SIZE(membar_exit) 1305 SET_SIZE(membar_enter) 1306 1307 ENTRY(membar_producer) 1308 sfence 1309 ret 1310 SET_SIZE(membar_producer) 1311 1312 ENTRY(membar_consumer) 1313 lfence 1314 ret 1315 SET_SIZE(membar_consumer) 1316 1317#else 1318 1319 ENTRY(membar_enter) 1320 ALTENTRY(membar_exit) 1321 lock 1322 xorl $0, (%esp) 1323 ret 1324 SET_SIZE(membar_exit) 1325 SET_SIZE(membar_enter) 1326 1327/* 1328 * On machines that support sfence and lfence, these 1329 * memory barriers can be more precisely implemented 1330 * without causing the whole world to stop 1331 */ 1332 ENTRY(membar_producer) 1333 .globl _patch_sfence_ret 1334_patch_sfence_ret: /* c.f. membar #StoreStore */ 1335 lock 1336 xorl $0, (%esp) 1337 ret 1338 SET_SIZE(membar_producer) 1339 1340 ENTRY(membar_consumer) 1341 .globl _patch_lfence_ret 1342_patch_lfence_ret: /* c.f. membar #LoadLoad */ 1343 lock 1344 xorl $0, (%esp) 1345 ret 1346 SET_SIZE(membar_consumer) 1347 1348#endif /* !__amd64 */ 1349 1350#endif /* __lint */ 1351 1352/* 1353 * thread_onproc() 1354 * Set thread in onproc state for the specified CPU. 1355 * Also set the thread lock pointer to the CPU's onproc lock. 1356 * Since the new lock isn't held, the store ordering is important. 1357 * If not done in assembler, the compiler could reorder the stores. 1358 */ 1359#if defined(lint) || defined(__lint) 1360 1361void 1362thread_onproc(kthread_id_t t, cpu_t *cp) 1363{ 1364 t->t_state = TS_ONPROC; 1365 t->t_lockp = &cp->cpu_thread_lock; 1366} 1367 1368#else /* __lint */ 1369 1370#if defined(__amd64) 1371 1372 ENTRY(thread_onproc) 1373 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */ 1374 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */ 1375 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */ 1376 ret 1377 SET_SIZE(thread_onproc) 1378 1379#else 1380 1381 ENTRY(thread_onproc) 1382 movl 4(%esp), %eax 1383 movl 8(%esp), %ecx 1384 addl $CPU_THREAD_LOCK, %ecx /* pointer to disp_lock while running */ 1385 movl $ONPROC_THREAD, T_STATE(%eax) /* set state to TS_ONPROC */ 1386 movl %ecx, T_LOCKP(%eax) /* store new lock pointer */ 1387 ret 1388 SET_SIZE(thread_onproc) 1389 1390#endif /* !__amd64 */ 1391 1392#endif /* __lint */ 1393