1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28#if defined(lint) 29#include <sys/types.h> 30#include <sys/thread.h> 31#include <sys/cpuvar.h> 32#else /* lint */ 33#include "assym.h" 34#endif /* lint */ 35 36#include <sys/t_lock.h> 37#include <sys/mutex.h> 38#include <sys/mutex_impl.h> 39#include <sys/rwlock_impl.h> 40#include <sys/asm_linkage.h> 41#include <sys/machlock.h> 42#include <sys/machthread.h> 43#include <sys/lockstat.h> 44 45/* #define DEBUG */ 46 47#ifdef DEBUG 48#include <sys/machparam.h> 49#endif /* DEBUG */ 50 51/************************************************************************ 52 * ATOMIC OPERATIONS 53 */ 54 55/* 56 * uint8_t ldstub(uint8_t *cp) 57 * 58 * Store 0xFF at the specified location, and return its previous content. 59 */ 60 61#if defined(lint) 62uint8_t 63ldstub(uint8_t *cp) 64{ 65 uint8_t rv; 66 rv = *cp; 67 *cp = 0xFF; 68 return rv; 69} 70#else /* lint */ 71 72 ENTRY(ldstub) 73 retl 74 ldstub [%o0], %o0 75 SET_SIZE(ldstub) 76 77#endif /* lint */ 78 79/************************************************************************ 80 * MEMORY BARRIERS -- see atomic.h for full descriptions. 81 */ 82 83#if defined(lint) 84 85void 86membar_enter(void) 87{} 88 89void 90membar_exit(void) 91{} 92 93void 94membar_producer(void) 95{} 96 97void 98membar_consumer(void) 99{} 100 101#else /* lint */ 102 103#ifdef SF_ERRATA_51 104 .align 32 105 ENTRY(membar_return) 106 retl 107 nop 108 SET_SIZE(membar_return) 109#define MEMBAR_RETURN ba,pt %icc, membar_return 110#else 111#define MEMBAR_RETURN retl 112#endif 113 114 ENTRY(membar_enter) 115 MEMBAR_RETURN 116 membar #StoreLoad|#StoreStore 117 SET_SIZE(membar_enter) 118 119 ENTRY(membar_exit) 120 MEMBAR_RETURN 121 membar #LoadStore|#StoreStore 122 SET_SIZE(membar_exit) 123 124 ENTRY(membar_producer) 125 MEMBAR_RETURN 126 membar #StoreStore 127 SET_SIZE(membar_producer) 128 129 ENTRY(membar_consumer) 130 MEMBAR_RETURN 131 membar #LoadLoad 132 SET_SIZE(membar_consumer) 133 134#endif /* lint */ 135 136/************************************************************************ 137 * MINIMUM LOCKS 138 */ 139 140#if defined(lint) 141 142/* 143 * lock_try(lp), ulock_try(lp) 144 * - returns non-zero on success. 145 * - doesn't block interrupts so don't use this to spin on a lock. 146 * - uses "0xFF is busy, anything else is free" model. 147 * 148 * ulock_try() is for a lock in the user address space. 149 * For all V7/V8 sparc systems they are same since the kernel and 150 * user are mapped in a user' context. 151 * For V9 platforms the lock_try and ulock_try are different impl. 152 */ 153 154int 155lock_try(lock_t *lp) 156{ 157 return (0xFF ^ ldstub(lp)); 158} 159 160int 161lock_spin_try(lock_t *lp) 162{ 163 return (0xFF ^ ldstub(lp)); 164} 165 166void 167lock_set(lock_t *lp) 168{ 169 extern void lock_set_spin(lock_t *); 170 171 if (!lock_try(lp)) 172 lock_set_spin(lp); 173 membar_enter(); 174} 175 176void 177lock_clear(lock_t *lp) 178{ 179 membar_exit(); 180 *lp = 0; 181} 182 183int 184ulock_try(lock_t *lp) 185{ 186 return (0xFF ^ ldstub(lp)); 187} 188 189void 190ulock_clear(lock_t *lp) 191{ 192 membar_exit(); 193 *lp = 0; 194} 195 196#else /* lint */ 197 198 .align 32 199 ENTRY(lock_try) 200 ldstub [%o0], %o1 ! try to set lock, get value in %o1 201 brnz,pn %o1, 1f 202 membar #LoadLoad 203.lock_try_lockstat_patch_point: 204 retl 205 or %o0, 1, %o0 ! ensure lo32 != 0 2061: 207 retl 208 clr %o0 209 SET_SIZE(lock_try) 210 211 .align 32 212 ENTRY(lock_spin_try) 213 ldstub [%o0], %o1 ! try to set lock, get value in %o1 214 brnz,pn %o1, 1f 215 membar #LoadLoad 216 retl 217 or %o0, 1, %o0 ! ensure lo32 != 0 2181: 219 retl 220 clr %o0 221 SET_SIZE(lock_spin_try) 222 223 .align 32 224 ENTRY(lock_set) 225 ldstub [%o0], %o1 226 brnz,pn %o1, 1f ! go to C for the hard case 227 membar #LoadLoad 228.lock_set_lockstat_patch_point: 229 retl 230 nop 2311: 232 sethi %hi(lock_set_spin), %o2 ! load up for jump to C 233 jmp %o2 + %lo(lock_set_spin) 234 nop ! delay: do nothing 235 SET_SIZE(lock_set) 236 237 ENTRY(lock_clear) 238 membar #LoadStore|#StoreStore 239.lock_clear_lockstat_patch_point: 240 retl 241 clrb [%o0] 242 SET_SIZE(lock_clear) 243 244 .align 32 245 ENTRY(ulock_try) 246 ldstuba [%o0]ASI_USER, %o1 ! try to set lock, get value in %o1 247 xor %o1, 0xff, %o0 ! delay - return non-zero if success 248 retl 249 membar #LoadLoad 250 SET_SIZE(ulock_try) 251 252 ENTRY(ulock_clear) 253 membar #LoadStore|#StoreStore 254 retl 255 stba %g0, [%o0]ASI_USER ! clear lock 256 SET_SIZE(ulock_clear) 257 258#endif /* lint */ 259 260 261/* 262 * lock_set_spl(lp, new_pil, *old_pil_addr) 263 * Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr. 264 */ 265 266#if defined(lint) 267 268/* ARGSUSED */ 269void 270lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil_addr) 271{ 272 extern int splr(int); 273 extern void lock_set_spl_spin(lock_t *, int, u_short *, int); 274 int old_pil; 275 276 old_pil = splr(new_pil); 277 if (!lock_try(lp)) { 278 lock_set_spl_spin(lp, new_pil, old_pil_addr, old_pil); 279 } else { 280 *old_pil_addr = (u_short)old_pil; 281 membar_enter(); 282 } 283} 284 285#else /* lint */ 286 287 ENTRY(lock_set_spl) 288 rdpr %pil, %o3 ! %o3 = current pil 289 cmp %o3, %o1 ! is current pil high enough? 290 bl,a,pt %icc, 1f ! if not, write %pil in delay 291 wrpr %g0, %o1, %pil 2921: 293 ldstub [%o0], %o4 ! try the lock 294 brnz,pn %o4, 2f ! go to C for the miss case 295 membar #LoadLoad 296.lock_set_spl_lockstat_patch_point: 297 retl 298 sth %o3, [%o2] ! delay - save original pil 2992: 300 sethi %hi(lock_set_spl_spin), %o5 ! load up jmp to C 301 jmp %o5 + %lo(lock_set_spl_spin) ! jmp to lock_set_spl_spin 302 nop ! delay: do nothing 303 SET_SIZE(lock_set_spl) 304 305#endif /* lint */ 306 307/* 308 * lock_clear_splx(lp, s) 309 */ 310 311#if defined(lint) 312 313void 314lock_clear_splx(lock_t *lp, int s) 315{ 316 extern void splx(int); 317 318 lock_clear(lp); 319 splx(s); 320} 321 322#else /* lint */ 323 324 ENTRY(lock_clear_splx) 325 ldn [THREAD_REG + T_CPU], %o2 ! get CPU pointer 326 membar #LoadStore|#StoreStore 327 ld [%o2 + CPU_BASE_SPL], %o2 328 clrb [%o0] ! clear lock 329 cmp %o2, %o1 ! compare new to base 330 movl %xcc, %o1, %o2 ! use new pri if base is less 331.lock_clear_splx_lockstat_patch_point: 332 retl 333 wrpr %g0, %o2, %pil 334 SET_SIZE(lock_clear_splx) 335 336#endif /* lint */ 337 338/* 339 * mutex_enter() and mutex_exit(). 340 * 341 * These routines handle the simple cases of mutex_enter() (adaptive 342 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). 343 * If anything complicated is going on we punt to mutex_vector_enter(). 344 * 345 * mutex_tryenter() is similar to mutex_enter() but returns zero if 346 * the lock cannot be acquired, nonzero on success. 347 * 348 * If mutex_exit() gets preempted in the window between checking waiters 349 * and clearing the lock, we can miss wakeups. Disabling preemption 350 * in the mutex code is prohibitively expensive, so instead we detect 351 * mutex preemption by examining the trapped PC in the interrupt path. 352 * If we interrupt a thread in mutex_exit() that has not yet cleared 353 * the lock, pil_interrupt() resets its PC back to the beginning of 354 * mutex_exit() so it will check again for waiters when it resumes. 355 * 356 * The lockstat code below is activated when the lockstat driver 357 * calls lockstat_hot_patch() to hot-patch the kernel mutex code. 358 * Note that we don't need to test lockstat_event_mask here -- we won't 359 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats. 360 */ 361 362#if defined (lint) 363 364/* ARGSUSED */ 365void 366mutex_enter(kmutex_t *lp) 367{} 368 369/* ARGSUSED */ 370int 371mutex_tryenter(kmutex_t *lp) 372{ return (0); } 373 374/* ARGSUSED */ 375void 376mutex_exit(kmutex_t *lp) 377{} 378 379/* ARGSUSED */ 380void * 381mutex_owner_running(mutex_impl_t *lp) 382{ return (NULL); } 383 384#else 385 .align 32 386 ENTRY(mutex_enter) 387 mov THREAD_REG, %o1 388 casx [%o0], %g0, %o1 ! try to acquire as adaptive 389 brnz,pn %o1, 1f ! locked or wrong type 390 membar #LoadLoad 391.mutex_enter_lockstat_patch_point: 392 retl 393 nop 3941: 395 sethi %hi(mutex_vector_enter), %o2 ! load up for jump to C 396 jmp %o2 + %lo(mutex_vector_enter) 397 nop 398 SET_SIZE(mutex_enter) 399 400 ENTRY(mutex_tryenter) 401 mov THREAD_REG, %o1 402 casx [%o0], %g0, %o1 ! try to acquire as adaptive 403 brnz,pn %o1, 1f ! locked or wrong type continue 404 membar #LoadLoad 405.mutex_tryenter_lockstat_patch_point: 406 retl 407 or %o0, 1, %o0 ! ensure lo32 != 0 4081: 409 sethi %hi(mutex_vector_tryenter), %o2 ! hi bits 410 jmp %o2 + %lo(mutex_vector_tryenter) ! go to C 411 nop 412 SET_SIZE(mutex_tryenter) 413 414 ENTRY(mutex_adaptive_tryenter) 415 mov THREAD_REG, %o1 416 casx [%o0], %g0, %o1 ! try to acquire as adaptive 417 brnz,pn %o1, 0f ! locked or wrong type 418 membar #LoadLoad 419 retl 420 or %o0, 1, %o0 ! ensure lo32 != 0 4210: 422 retl 423 mov %g0, %o0 424 SET_SIZE(mutex_adaptive_tryenter) 425 426 ! these need to be together and cache aligned for performance. 427 .align 64 428 .global mutex_exit_critical_size 429 .global mutex_exit_critical_start 430 .global mutex_owner_running_critical_size 431 .global mutex_owner_running_critical_start 432 433mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start 434 435 .align 32 436 437 ENTRY(mutex_exit) 438mutex_exit_critical_start: ! If we are interrupted, restart here 439 ldn [%o0], %o1 ! get the owner field 440 membar #LoadStore|#StoreStore 441 cmp THREAD_REG, %o1 ! do we own lock with no waiters? 442 be,a,pt %ncc, 1f ! if so, drive on ... 443 stn %g0, [%o0] ! delay: clear lock if we owned it 444.mutex_exit_critical_end: ! for pil_interrupt() hook 445 ba,a,pt %xcc, mutex_vector_exit ! go to C for the hard cases 4461: 447.mutex_exit_lockstat_patch_point: 448 retl 449 nop 450 SET_SIZE(mutex_exit) 451 452mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start 453 454 .align 32 455 456 ENTRY(mutex_owner_running) 457mutex_owner_running_critical_start: ! If interrupted restart here 458 ldn [%o0], %o1 ! get the owner field 459 and %o1, MUTEX_THREAD, %o1 ! remove the waiters bit if any 460 brz,pn %o1, 1f ! if so, drive on ... 461 nop 462 ldn [%o1+T_CPU], %o2 ! get owner->t_cpu 463 ldn [%o2+CPU_THREAD], %o3 ! get owner->t_cpu->cpu_thread 464.mutex_owner_running_critical_end: ! for pil_interrupt() hook 465 cmp %o1, %o3 ! owner == running thread? 466 be,a,pt %xcc, 2f ! yes, go return cpu 467 nop 4681: 469 retl 470 mov %g0, %o0 ! return 0 (owner not running) 4712: 472 retl 473 mov %o2, %o0 ! owner running, return cpu 474 SET_SIZE(mutex_owner_running) 475 476#endif /* lint */ 477 478/* 479 * rw_enter() and rw_exit(). 480 * 481 * These routines handle the simple cases of rw_enter (write-locking an unheld 482 * lock or read-locking a lock that's neither write-locked nor write-wanted) 483 * and rw_exit (no waiters or not the last reader). If anything complicated 484 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. 485 */ 486#if defined(lint) 487 488/* ARGSUSED */ 489void 490rw_enter(krwlock_t *lp, krw_t rw) 491{} 492 493/* ARGSUSED */ 494void 495rw_exit(krwlock_t *lp) 496{} 497 498#else 499 500 .align 16 501 ENTRY(rw_enter) 502 cmp %o1, RW_WRITER ! entering as writer? 503 be,a,pn %icc, 2f ! if so, go do it ... 504 or THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner 505 ld [THREAD_REG + T_KPRI_REQ], %o3 ! begin THREAD_KPRI_REQUEST() 506 ldn [%o0], %o4 ! %o4 = old lock value 507 inc %o3 ! bump kpri 508 st %o3, [THREAD_REG + T_KPRI_REQ] ! store new kpri 5091: 510 andcc %o4, RW_WRITE_CLAIMED, %g0 ! write-locked or write-wanted? 511 bz,pt %xcc, 3f ! if so, prepare to block 512 add %o4, RW_READ_LOCK, %o5 ! delay: increment hold count 513 sethi %hi(rw_enter_sleep), %o2 ! load up jump 514 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep 515 nop ! delay: do nothing 5163: 517 casx [%o0], %o4, %o5 ! try to grab read lock 518 cmp %o4, %o5 ! did we get it? 519#ifdef sun4v 520 be,a,pt %xcc, 0f 521 membar #LoadLoad 522 sethi %hi(rw_enter_sleep), %o2 ! load up jump 523 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep 524 nop ! delay: do nothing 5250: 526#else /* sun4v */ 527 bne,pn %xcc, 1b ! if not, try again 528 mov %o5, %o4 ! delay: %o4 = old lock value 529 membar #LoadLoad 530#endif /* sun4v */ 531.rw_read_enter_lockstat_patch_point: 532 retl 533 nop 5342: 535 casx [%o0], %g0, %o5 ! try to grab write lock 536 brz,pt %o5, 4f ! branch around if we got it 537 membar #LoadLoad ! done regardless of where we go 538 sethi %hi(rw_enter_sleep), %o2 539 jmp %o2 + %lo(rw_enter_sleep) ! jump to rw_enter_sleep if not 540 nop ! delay: do nothing 5414: 542.rw_write_enter_lockstat_patch_point: 543 retl 544 nop 545 SET_SIZE(rw_enter) 546 547 .align 16 548 ENTRY(rw_exit) 549 ldn [%o0], %o4 ! %o4 = old lock value 550 membar #LoadStore|#StoreStore ! membar_exit() 551 subcc %o4, RW_READ_LOCK, %o5 ! %o5 = new lock value if reader 552 bnz,pn %xcc, 2f ! single reader, no waiters? 553 clr %o1 5541: 555 ld [THREAD_REG + T_KPRI_REQ], %g1 ! begin THREAD_KPRI_RELEASE() 556 srl %o4, RW_HOLD_COUNT_SHIFT, %o3 ! %o3 = hold count (lockstat) 557 casx [%o0], %o4, %o5 ! try to drop lock 558 cmp %o4, %o5 ! did we succeed? 559 bne,pn %xcc, rw_exit_wakeup ! if not, go to C 560 dec %g1 ! delay: drop kpri 561.rw_read_exit_lockstat_patch_point: 562 retl 563 st %g1, [THREAD_REG + T_KPRI_REQ] ! delay: store new kpri 5642: 565 andcc %o4, RW_WRITE_LOCKED, %g0 ! are we a writer? 566 bnz,a,pt %xcc, 3f 567 or THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner 568 cmp %o5, RW_READ_LOCK ! would lock still be held? 569 bge,pt %xcc, 1b ! if so, go ahead and drop it 570 nop 571 ba,pt %xcc, rw_exit_wakeup ! otherwise, wake waiters 572 nop 5733: 574 casx [%o0], %o4, %o1 ! try to drop write lock 575 cmp %o4, %o1 ! did we succeed? 576 bne,pn %xcc, rw_exit_wakeup ! if not, go to C 577 nop 578.rw_write_exit_lockstat_patch_point: 579 retl 580 nop 581 SET_SIZE(rw_exit) 582 583#endif 584 585#if defined(lint) 586 587void 588lockstat_hot_patch(void) 589{} 590 591#else 592 593#define RETL 0x81c3e008 594#define NOP 0x01000000 595#define BA 0x10800000 596 597#define DISP22 ((1 << 22) - 1) 598#define ANNUL 0x20000000 599 600#define HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs) \ 601 ba 1f; \ 602 rd %pc, %o0; \ 603 save %sp, -SA(MINFRAME), %sp; \ 604 set lockstat_probemap, %l1; \ 605 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ 606 brz,pn %o0, 0f; \ 607 ldub [THREAD_REG + T_LOCKSTAT], %l0; \ 608 add %l0, 1, %l2; \ 609 stub %l2, [THREAD_REG + T_LOCKSTAT]; \ 610 set lockstat_probe, %g1; \ 611 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ 612 brz,a,pn %o0, 0f; \ 613 stub %l0, [THREAD_REG + T_LOCKSTAT]; \ 614 ldn [%g1], %g2; \ 615 mov rs, %o2; \ 616 jmpl %g2, %o7; \ 617 mov %i0, %o1; \ 618 stub %l0, [THREAD_REG + T_LOCKSTAT]; \ 6190: ret; \ 620 restore %g0, 1, %o0; /* for mutex_tryenter / lock_try */ \ 6211: set addr, %o1; \ 622 sub %o0, %o1, %o0; \ 623 srl %o0, 2, %o0; \ 624 inc %o0; \ 625 set DISP22, %o1; \ 626 and %o1, %o0, %o0; \ 627 set BA, %o1; \ 628 or %o1, %o0, %o0; \ 629 sethi %hi(annul), %o2; \ 630 add %o0, %o2, %o2; \ 631 set addr, %o0; \ 632 set normal_instr, %o1; \ 633 ld [%i0 + (event * DTRACE_IDSIZE)], %o3; \ 634 tst %o3; \ 635 movnz %icc, %o2, %o1; \ 636 call hot_patch_kernel_text; \ 637 mov 4, %o2; \ 638 membar #Sync 639 640#define HOT_PATCH(addr, event, normal_instr) \ 641 HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1) 642 643#define HOT_PATCH_ARG(addr, event, normal_instr, arg) \ 644 HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg) 645 646#define HOT_PATCH_ANNULLED(addr, event, normal_instr) \ 647 HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1) 648 649 ENTRY(lockstat_hot_patch) 650 save %sp, -SA(MINFRAME), %sp 651 set lockstat_probemap, %i0 652 HOT_PATCH(.mutex_enter_lockstat_patch_point, 653 LS_MUTEX_ENTER_ACQUIRE, RETL) 654 HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point, 655 LS_MUTEX_TRYENTER_ACQUIRE, RETL) 656 HOT_PATCH(.mutex_exit_lockstat_patch_point, 657 LS_MUTEX_EXIT_RELEASE, RETL) 658 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 659 LS_RW_ENTER_ACQUIRE, RETL) 660 HOT_PATCH(.rw_read_enter_lockstat_patch_point, 661 LS_RW_ENTER_ACQUIRE, RETL) 662 HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point, 663 LS_RW_EXIT_RELEASE, RETL, RW_WRITER) 664 HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point, 665 LS_RW_EXIT_RELEASE, RETL, RW_READER) 666 HOT_PATCH(.lock_set_lockstat_patch_point, 667 LS_LOCK_SET_ACQUIRE, RETL) 668 HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point, 669 LS_LOCK_TRY_ACQUIRE, RETL) 670 HOT_PATCH(.lock_clear_lockstat_patch_point, 671 LS_LOCK_CLEAR_RELEASE, RETL) 672 HOT_PATCH(.lock_set_spl_lockstat_patch_point, 673 LS_LOCK_SET_SPL_ACQUIRE, RETL) 674 HOT_PATCH(.lock_clear_splx_lockstat_patch_point, 675 LS_LOCK_CLEAR_SPLX_RELEASE, RETL) 676 ret 677 restore 678 SET_SIZE(lockstat_hot_patch) 679 680#endif /* lint */ 681 682/* 683 * asm_mutex_spin_enter(mutex_t *) 684 * 685 * For use by assembly interrupt handler only. 686 * Does not change spl, since the interrupt handler is assumed to be 687 * running at high level already. 688 * Traps may be off, so cannot panic. 689 * Does not keep statistics on the lock. 690 * 691 * Entry: %l6 - points to mutex 692 * %l7 - address of call (returns to %l7+8) 693 * Uses: %l6, %l5 694 */ 695#ifndef lint 696 .align 16 697 ENTRY_NP(asm_mutex_spin_enter) 698 ldstub [%l6 + M_SPINLOCK], %l5 ! try to set lock, get value in %l5 6991: 700 tst %l5 701 bnz 3f ! lock already held - go spin 702 nop 7032: 704 jmp %l7 + 8 ! return 705 membar #LoadLoad 706 ! 707 ! Spin on lock without using an atomic operation to prevent the caches 708 ! from unnecessarily moving ownership of the line around. 709 ! 7103: 711 ldub [%l6 + M_SPINLOCK], %l5 7124: 713 tst %l5 714 bz,a 1b ! lock appears to be free, try again 715 ldstub [%l6 + M_SPINLOCK], %l5 ! delay slot - try to set lock 716 717 sethi %hi(panicstr) , %l5 718 ldn [%l5 + %lo(panicstr)], %l5 719 tst %l5 720 bnz 2b ! after panic, feign success 721 nop 722 b 4b 723 ldub [%l6 + M_SPINLOCK], %l5 ! delay - reload lock 724 SET_SIZE(asm_mutex_spin_enter) 725#endif /* lint */ 726 727/* 728 * asm_mutex_spin_exit(mutex_t *) 729 * 730 * For use by assembly interrupt handler only. 731 * Does not change spl, since the interrupt handler is assumed to be 732 * running at high level already. 733 * 734 * Entry: %l6 - points to mutex 735 * %l7 - address of call (returns to %l7+8) 736 * Uses: none 737 */ 738#ifndef lint 739 ENTRY_NP(asm_mutex_spin_exit) 740 membar #LoadStore|#StoreStore 741 jmp %l7 + 8 ! return 742 clrb [%l6 + M_SPINLOCK] ! delay - clear lock 743 SET_SIZE(asm_mutex_spin_exit) 744#endif /* lint */ 745 746/* 747 * thread_onproc() 748 * Set thread in onproc state for the specified CPU. 749 * Also set the thread lock pointer to the CPU's onproc lock. 750 * Since the new lock isn't held, the store ordering is important. 751 * If not done in assembler, the compiler could reorder the stores. 752 */ 753#if defined(lint) 754 755void 756thread_onproc(kthread_id_t t, cpu_t *cp) 757{ 758 t->t_state = TS_ONPROC; 759 t->t_lockp = &cp->cpu_thread_lock; 760} 761 762#else /* lint */ 763 764 ENTRY(thread_onproc) 765 set TS_ONPROC, %o2 ! TS_ONPROC state 766 st %o2, [%o0 + T_STATE] ! store state 767 add %o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running 768 retl ! return 769 stn %o3, [%o0 + T_LOCKP] ! delay - store new lock pointer 770 SET_SIZE(thread_onproc) 771 772#endif /* lint */ 773 774/* delay function used in some mutex code - just do 3 nop cas ops */ 775#if defined(lint) 776 777/* ARGSUSED */ 778void 779cas_delay(void *addr) 780{} 781#else /* lint */ 782 ENTRY(cas_delay) 783 casx [%o0], %g0, %g0 784 casx [%o0], %g0, %g0 785 retl 786 casx [%o0], %g0, %g0 787 SET_SIZE(cas_delay) 788#endif /* lint */ 789 790#if defined(lint) 791 792/* 793 * alternative delay function for some niagara processors. The rd 794 * instruction uses less resources than casx on those cpus. 795 */ 796/* ARGSUSED */ 797void 798rdccr_delay(void) 799{} 800#else /* lint */ 801 ENTRY(rdccr_delay) 802 rd %ccr, %g0 803 rd %ccr, %g0 804 retl 805 rd %ccr, %g0 806 SET_SIZE(rdccr_delay) 807#endif /* lint */ 808 809/* 810 * mutex_delay_default(void) 811 * Spins for approx a few hundred processor cycles and returns to caller. 812 */ 813#if defined(lint) 814 815void 816mutex_delay_default(void) 817{} 818 819#else /* lint */ 820 821 ENTRY(mutex_delay_default) 822 mov 72,%o0 8231: brgz %o0, 1b 824 dec %o0 825 retl 826 nop 827 SET_SIZE(mutex_delay_default) 828 829#endif /* lint */ 830