1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * 25 * Copyright 2020 Joyent, Inc. 26 */ 27 28#include "assym.h" 29 30#include <sys/t_lock.h> 31#include <sys/mutex.h> 32#include <sys/mutex_impl.h> 33#include <sys/rwlock_impl.h> 34#include <sys/asm_linkage.h> 35#include <sys/machlock.h> 36#include <sys/machthread.h> 37#include <sys/lockstat.h> 38 39/* #define DEBUG */ 40 41#ifdef DEBUG 42#include <sys/machparam.h> 43#endif /* DEBUG */ 44 45/************************************************************************ 46 * ATOMIC OPERATIONS 47 */ 48 49/* 50 * uint8_t ldstub(uint8_t *cp) 51 * 52 * Store 0xFF at the specified location, and return its previous content. 53 */ 54 55 ENTRY(ldstub) 56 retl 57 ldstub [%o0], %o0 58 SET_SIZE(ldstub) 59 60/************************************************************************ 61 * MEMORY BARRIERS -- see atomic.h for full descriptions. 62 */ 63 64#ifdef SF_ERRATA_51 65 .align 32 66 ENTRY(membar_return) 67 retl 68 nop 69 SET_SIZE(membar_return) 70#define MEMBAR_RETURN ba,pt %icc, membar_return 71#else 72#define MEMBAR_RETURN retl 73#endif 74 75 ENTRY(membar_enter) 76 MEMBAR_RETURN 77 membar #StoreLoad|#StoreStore 78 SET_SIZE(membar_enter) 79 80 ENTRY(membar_exit) 81 MEMBAR_RETURN 82 membar #LoadStore|#StoreStore 83 SET_SIZE(membar_exit) 84 85 ENTRY(membar_producer) 86 MEMBAR_RETURN 87 membar #StoreStore 88 SET_SIZE(membar_producer) 89 90 ENTRY(membar_consumer) 91 MEMBAR_RETURN 92 membar #LoadLoad 93 SET_SIZE(membar_consumer) 94 95/************************************************************************ 96 * MINIMUM LOCKS 97 */ 98 99/* 100 * lock_try(lp), ulock_try(lp) 101 * - returns non-zero on success. 102 * - doesn't block interrupts so don't use this to spin on a lock. 103 * - uses "0xFF is busy, anything else is free" model. 104 * 105 * ulock_try() is for a lock in the user address space. 106 */ 107 108 .align 32 109 ENTRY(lock_try) 110 ldstub [%o0], %o1 ! try to set lock, get value in %o1 111 brnz,pn %o1, 1f 112 membar #LoadLoad 113.lock_try_lockstat_patch_point: 114 retl 115 or %o0, 1, %o0 ! ensure lo32 != 0 1161: 117 retl 118 clr %o0 119 SET_SIZE(lock_try) 120 121 .align 32 122 ENTRY(lock_spin_try) 123 ldstub [%o0], %o1 ! try to set lock, get value in %o1 124 brnz,pn %o1, 1f 125 membar #LoadLoad 126 retl 127 or %o0, 1, %o0 ! ensure lo32 != 0 1281: 129 retl 130 clr %o0 131 SET_SIZE(lock_spin_try) 132 133 .align 32 134 ENTRY(lock_set) 135 ldstub [%o0], %o1 136 brnz,pn %o1, 1f ! go to C for the hard case 137 membar #LoadLoad 138.lock_set_lockstat_patch_point: 139 retl 140 nop 1411: 142 sethi %hi(lock_set_spin), %o2 ! load up for jump to C 143 jmp %o2 + %lo(lock_set_spin) 144 nop ! delay: do nothing 145 SET_SIZE(lock_set) 146 147 ENTRY(lock_clear) 148 membar #LoadStore|#StoreStore 149.lock_clear_lockstat_patch_point: 150 retl 151 clrb [%o0] 152 SET_SIZE(lock_clear) 153 154 .align 32 155 ENTRY(ulock_try) 156 ldstuba [%o0]ASI_USER, %o1 ! try to set lock, get value in %o1 157 xor %o1, 0xff, %o0 ! delay - return non-zero if success 158 retl 159 membar #LoadLoad 160 SET_SIZE(ulock_try) 161 162 ENTRY(ulock_clear) 163 membar #LoadStore|#StoreStore 164 retl 165 stba %g0, [%o0]ASI_USER ! clear lock 166 SET_SIZE(ulock_clear) 167 168 169/* 170 * lock_set_spl(lp, new_pil, *old_pil_addr) 171 * Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr. 172 */ 173 174 ENTRY(lock_set_spl) 175 rdpr %pil, %o3 ! %o3 = current pil 176 cmp %o3, %o1 ! is current pil high enough? 177 bl,a,pt %icc, 1f ! if not, write %pil in delay 178 wrpr %g0, %o1, %pil 1791: 180 ldstub [%o0], %o4 ! try the lock 181 brnz,pn %o4, 2f ! go to C for the miss case 182 membar #LoadLoad 183.lock_set_spl_lockstat_patch_point: 184 retl 185 sth %o3, [%o2] ! delay - save original pil 1862: 187 sethi %hi(lock_set_spl_spin), %o5 ! load up jmp to C 188 jmp %o5 + %lo(lock_set_spl_spin) ! jmp to lock_set_spl_spin 189 nop ! delay: do nothing 190 SET_SIZE(lock_set_spl) 191 192/* 193 * lock_clear_splx(lp, s) 194 */ 195 196 ENTRY(lock_clear_splx) 197 ldn [THREAD_REG + T_CPU], %o2 ! get CPU pointer 198 membar #LoadStore|#StoreStore 199 ld [%o2 + CPU_BASE_SPL], %o2 200 clrb [%o0] ! clear lock 201 cmp %o2, %o1 ! compare new to base 202 movl %xcc, %o1, %o2 ! use new pri if base is less 203.lock_clear_splx_lockstat_patch_point: 204 retl 205 wrpr %g0, %o2, %pil 206 SET_SIZE(lock_clear_splx) 207 208/* 209 * mutex_enter() and mutex_exit(). 210 * 211 * These routines handle the simple cases of mutex_enter() (adaptive 212 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). 213 * If anything complicated is going on we punt to mutex_vector_enter(). 214 * 215 * mutex_tryenter() is similar to mutex_enter() but returns zero if 216 * the lock cannot be acquired, nonzero on success. 217 * 218 * If mutex_exit() gets preempted in the window between checking waiters 219 * and clearing the lock, we can miss wakeups. Disabling preemption 220 * in the mutex code is prohibitively expensive, so instead we detect 221 * mutex preemption by examining the trapped PC in the interrupt path. 222 * If we interrupt a thread in mutex_exit() that has not yet cleared 223 * the lock, pil_interrupt() resets its PC back to the beginning of 224 * mutex_exit() so it will check again for waiters when it resumes. 225 * 226 * The lockstat code below is activated when the lockstat driver 227 * calls lockstat_hot_patch() to hot-patch the kernel mutex code. 228 * Note that we don't need to test lockstat_event_mask here -- we won't 229 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats. 230 */ 231 232 .align 32 233 ENTRY(mutex_enter) 234 mov THREAD_REG, %o1 235 casx [%o0], %g0, %o1 ! try to acquire as adaptive 236 brnz,pn %o1, 1f ! locked or wrong type 237 membar #LoadLoad 238.mutex_enter_lockstat_patch_point: 239 retl 240 nop 2411: 242 sethi %hi(mutex_vector_enter), %o2 ! load up for jump to C 243 jmp %o2 + %lo(mutex_vector_enter) 244 nop 245 SET_SIZE(mutex_enter) 246 247 ENTRY(mutex_tryenter) 248 mov THREAD_REG, %o1 249 casx [%o0], %g0, %o1 ! try to acquire as adaptive 250 brnz,pn %o1, 1f ! locked or wrong type continue 251 membar #LoadLoad 252.mutex_tryenter_lockstat_patch_point: 253 retl 254 or %o0, 1, %o0 ! ensure lo32 != 0 2551: 256 sethi %hi(mutex_vector_tryenter), %o2 ! hi bits 257 jmp %o2 + %lo(mutex_vector_tryenter) ! go to C 258 nop 259 SET_SIZE(mutex_tryenter) 260 261 ENTRY(mutex_adaptive_tryenter) 262 mov THREAD_REG, %o1 263 casx [%o0], %g0, %o1 ! try to acquire as adaptive 264 brnz,pn %o1, 0f ! locked or wrong type 265 membar #LoadLoad 266 retl 267 or %o0, 1, %o0 ! ensure lo32 != 0 2680: 269 retl 270 mov %g0, %o0 271 SET_SIZE(mutex_adaptive_tryenter) 272 273 ! these need to be together and cache aligned for performance. 274 .align 64 275 .global mutex_exit_critical_size 276 .global mutex_exit_critical_start 277 .global mutex_owner_running_critical_size 278 .global mutex_owner_running_critical_start 279 280mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start 281 282 .align 32 283 284 ENTRY(mutex_exit) 285mutex_exit_critical_start: ! If we are interrupted, restart here 286 ldn [%o0], %o1 ! get the owner field 287 membar #LoadStore|#StoreStore 288 cmp THREAD_REG, %o1 ! do we own lock with no waiters? 289 be,a,pt %ncc, 1f ! if so, drive on ... 290 stn %g0, [%o0] ! delay: clear lock if we owned it 291.mutex_exit_critical_end: ! for pil_interrupt() hook 292 ba,a,pt %xcc, mutex_vector_exit ! go to C for the hard cases 2931: 294.mutex_exit_lockstat_patch_point: 295 retl 296 nop 297 SET_SIZE(mutex_exit) 298 299mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start 300 301 .align 32 302 303 ENTRY(mutex_owner_running) 304mutex_owner_running_critical_start: ! If interrupted restart here 305 ldn [%o0], %o1 ! get the owner field 306 and %o1, MUTEX_THREAD, %o1 ! remove the waiters bit if any 307 brz,pn %o1, 1f ! if so, drive on ... 308 nop 309 ldn [%o1+T_CPU], %o2 ! get owner->t_cpu 310 ldn [%o2+CPU_THREAD], %o3 ! get owner->t_cpu->cpu_thread 311.mutex_owner_running_critical_end: ! for pil_interrupt() hook 312 cmp %o1, %o3 ! owner == running thread? 313 be,a,pt %xcc, 2f ! yes, go return cpu 314 nop 3151: 316 retl 317 mov %g0, %o0 ! return 0 (owner not running) 3182: 319 retl 320 mov %o2, %o0 ! owner running, return cpu 321 SET_SIZE(mutex_owner_running) 322 323/* 324 * rw_enter() and rw_exit(). 325 * 326 * These routines handle the simple cases of rw_enter (write-locking an unheld 327 * lock or read-locking a lock that's neither write-locked nor write-wanted) 328 * and rw_exit (no waiters or not the last reader). If anything complicated 329 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. 330 */ 331 332 .align 16 333 ENTRY(rw_enter) 334 cmp %o1, RW_WRITER ! entering as writer? 335 be,a,pn %icc, 2f ! if so, go do it ... 336 or THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner 337 ldn [%o0], %o4 ! %o4 = old lock value 3381: 339 andcc %o4, RW_WRITE_CLAIMED, %g0 ! write-locked or write-wanted? 340 bz,pt %xcc, 3f ! if so, prepare to block 341 add %o4, RW_READ_LOCK, %o5 ! delay: increment hold count 342 sethi %hi(rw_enter_sleep), %o2 ! load up jump 343 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep 344 nop ! delay: do nothing 3453: 346 casx [%o0], %o4, %o5 ! try to grab read lock 347 cmp %o4, %o5 ! did we get it? 348#ifdef sun4v 349 be,a,pt %xcc, 0f 350 membar #LoadLoad 351 sethi %hi(rw_enter_sleep), %o2 ! load up jump 352 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep 353 nop ! delay: do nothing 3540: 355#else /* sun4v */ 356 bne,pn %xcc, 1b ! if not, try again 357 mov %o5, %o4 ! delay: %o4 = old lock value 358 membar #LoadLoad 359#endif /* sun4v */ 360.rw_read_enter_lockstat_patch_point: 361 retl 362 nop 3632: 364 casx [%o0], %g0, %o5 ! try to grab write lock 365 brz,pt %o5, 4f ! branch around if we got it 366 membar #LoadLoad ! done regardless of where we go 367 sethi %hi(rw_enter_sleep), %o2 368 jmp %o2 + %lo(rw_enter_sleep) ! jump to rw_enter_sleep if not 369 nop ! delay: do nothing 3704: 371.rw_write_enter_lockstat_patch_point: 372 retl 373 nop 374 SET_SIZE(rw_enter) 375 376 .align 16 377 ENTRY(rw_exit) 378 ldn [%o0], %o4 ! %o4 = old lock value 379 membar #LoadStore|#StoreStore ! membar_exit() 380 subcc %o4, RW_READ_LOCK, %o5 ! %o5 = new lock value if reader 381 bnz,pn %xcc, 2f ! single reader, no waiters? 382 clr %o1 3831: 384 srl %o4, RW_HOLD_COUNT_SHIFT, %o3 ! %o3 = hold count (lockstat) 385 casx [%o0], %o4, %o5 ! try to drop lock 386 cmp %o4, %o5 ! did we succeed? 387 bne,pn %xcc, rw_exit_wakeup ! if not, go to C 388 nop ! delay: do nothing 389.rw_read_exit_lockstat_patch_point: 390 retl 391 nop ! delay: do nothing 3922: 393 andcc %o4, RW_WRITE_LOCKED, %g0 ! are we a writer? 394 bnz,a,pt %xcc, 3f 395 or THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner 396 cmp %o5, RW_READ_LOCK ! would lock still be held? 397 bge,pt %xcc, 1b ! if so, go ahead and drop it 398 nop 399 ba,pt %xcc, rw_exit_wakeup ! otherwise, wake waiters 400 nop 4013: 402 casx [%o0], %o4, %o1 ! try to drop write lock 403 cmp %o4, %o1 ! did we succeed? 404 bne,pn %xcc, rw_exit_wakeup ! if not, go to C 405 nop 406.rw_write_exit_lockstat_patch_point: 407 retl 408 nop 409 SET_SIZE(rw_exit) 410 411#define RETL 0x81c3e008 412#define NOP 0x01000000 413#define BA 0x10800000 414 415#define DISP22 ((1 << 22) - 1) 416#define ANNUL 0x20000000 417 418#define HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs) \ 419 ba 1f; \ 420 rd %pc, %o0; \ 421 save %sp, -SA(MINFRAME), %sp; \ 422 set lockstat_probemap, %l1; \ 423 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ 424 brz,pn %o0, 0f; \ 425 ldub [THREAD_REG + T_LOCKSTAT], %l0; \ 426 add %l0, 1, %l2; \ 427 stub %l2, [THREAD_REG + T_LOCKSTAT]; \ 428 set lockstat_probe, %g1; \ 429 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ 430 brz,a,pn %o0, 0f; \ 431 stub %l0, [THREAD_REG + T_LOCKSTAT]; \ 432 ldn [%g1], %g2; \ 433 mov rs, %o2; \ 434 jmpl %g2, %o7; \ 435 mov %i0, %o1; \ 436 stub %l0, [THREAD_REG + T_LOCKSTAT]; \ 4370: ret; \ 438 restore %g0, 1, %o0; /* for mutex_tryenter / lock_try */ \ 4391: set addr, %o1; \ 440 sub %o0, %o1, %o0; \ 441 srl %o0, 2, %o0; \ 442 inc %o0; \ 443 set DISP22, %o1; \ 444 and %o1, %o0, %o0; \ 445 set BA, %o1; \ 446 or %o1, %o0, %o0; \ 447 sethi %hi(annul), %o2; \ 448 add %o0, %o2, %o2; \ 449 set addr, %o0; \ 450 set normal_instr, %o1; \ 451 ld [%i0 + (event * DTRACE_IDSIZE)], %o3; \ 452 tst %o3; \ 453 movnz %icc, %o2, %o1; \ 454 call hot_patch_kernel_text; \ 455 mov 4, %o2; \ 456 membar #Sync 457 458#define HOT_PATCH(addr, event, normal_instr) \ 459 HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1) 460 461#define HOT_PATCH_ARG(addr, event, normal_instr, arg) \ 462 HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg) 463 464#define HOT_PATCH_ANNULLED(addr, event, normal_instr) \ 465 HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1) 466 467 ENTRY(lockstat_hot_patch) 468 save %sp, -SA(MINFRAME), %sp 469 set lockstat_probemap, %i0 470 HOT_PATCH(.mutex_enter_lockstat_patch_point, 471 LS_MUTEX_ENTER_ACQUIRE, RETL) 472 HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point, 473 LS_MUTEX_TRYENTER_ACQUIRE, RETL) 474 HOT_PATCH(.mutex_exit_lockstat_patch_point, 475 LS_MUTEX_EXIT_RELEASE, RETL) 476 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 477 LS_RW_ENTER_ACQUIRE, RETL) 478 HOT_PATCH(.rw_read_enter_lockstat_patch_point, 479 LS_RW_ENTER_ACQUIRE, RETL) 480 HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point, 481 LS_RW_EXIT_RELEASE, RETL, RW_WRITER) 482 HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point, 483 LS_RW_EXIT_RELEASE, RETL, RW_READER) 484 HOT_PATCH(.lock_set_lockstat_patch_point, 485 LS_LOCK_SET_ACQUIRE, RETL) 486 HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point, 487 LS_LOCK_TRY_ACQUIRE, RETL) 488 HOT_PATCH(.lock_clear_lockstat_patch_point, 489 LS_LOCK_CLEAR_RELEASE, RETL) 490 HOT_PATCH(.lock_set_spl_lockstat_patch_point, 491 LS_LOCK_SET_SPL_ACQUIRE, RETL) 492 HOT_PATCH(.lock_clear_splx_lockstat_patch_point, 493 LS_LOCK_CLEAR_SPLX_RELEASE, RETL) 494 ret 495 restore 496 SET_SIZE(lockstat_hot_patch) 497 498/* 499 * asm_mutex_spin_enter(mutex_t *) 500 * 501 * For use by assembly interrupt handler only. 502 * Does not change spl, since the interrupt handler is assumed to be 503 * running at high level already. 504 * Traps may be off, so cannot panic. 505 * Does not keep statistics on the lock. 506 * 507 * Entry: %l6 - points to mutex 508 * %l7 - address of call (returns to %l7+8) 509 * Uses: %l6, %l5 510 */ 511 .align 16 512 ENTRY_NP(asm_mutex_spin_enter) 513 ldstub [%l6 + M_SPINLOCK], %l5 ! try to set lock, get value in %l5 5141: 515 tst %l5 516 bnz 3f ! lock already held - go spin 517 nop 5182: 519 jmp %l7 + 8 ! return 520 membar #LoadLoad 521 ! 522 ! Spin on lock without using an atomic operation to prevent the caches 523 ! from unnecessarily moving ownership of the line around. 524 ! 5253: 526 ldub [%l6 + M_SPINLOCK], %l5 5274: 528 tst %l5 529 bz,a 1b ! lock appears to be free, try again 530 ldstub [%l6 + M_SPINLOCK], %l5 ! delay slot - try to set lock 531 532 sethi %hi(panicstr) , %l5 533 ldn [%l5 + %lo(panicstr)], %l5 534 tst %l5 535 bnz 2b ! after panic, feign success 536 nop 537 b 4b 538 ldub [%l6 + M_SPINLOCK], %l5 ! delay - reload lock 539 SET_SIZE(asm_mutex_spin_enter) 540 541/* 542 * asm_mutex_spin_exit(mutex_t *) 543 * 544 * For use by assembly interrupt handler only. 545 * Does not change spl, since the interrupt handler is assumed to be 546 * running at high level already. 547 * 548 * Entry: %l6 - points to mutex 549 * %l7 - address of call (returns to %l7+8) 550 * Uses: none 551 */ 552 ENTRY_NP(asm_mutex_spin_exit) 553 membar #LoadStore|#StoreStore 554 jmp %l7 + 8 ! return 555 clrb [%l6 + M_SPINLOCK] ! delay - clear lock 556 SET_SIZE(asm_mutex_spin_exit) 557 558/* 559 * thread_onproc() 560 * Set thread in onproc state for the specified CPU. 561 * Also set the thread lock pointer to the CPU's onproc lock. 562 * Since the new lock isn't held, the store ordering is important. 563 * If not done in assembler, the compiler could reorder the stores. 564 */ 565 566 ENTRY(thread_onproc) 567 set TS_ONPROC, %o2 ! TS_ONPROC state 568 st %o2, [%o0 + T_STATE] ! store state 569 add %o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running 570 retl ! return 571 stn %o3, [%o0 + T_LOCKP] ! delay - store new lock pointer 572 SET_SIZE(thread_onproc) 573 574/* delay function used in some mutex code - just do 3 nop cas ops */ 575 ENTRY(cas_delay) 576 casx [%o0], %g0, %g0 577 casx [%o0], %g0, %g0 578 retl 579 casx [%o0], %g0, %g0 580 SET_SIZE(cas_delay) 581 582/* 583 * alternative delay function for some niagara processors. The rd 584 * instruction uses less resources than casx on those cpus. 585 */ 586 ENTRY(rdccr_delay) 587 rd %ccr, %g0 588 rd %ccr, %g0 589 retl 590 rd %ccr, %g0 591 SET_SIZE(rdccr_delay) 592 593/* 594 * mutex_delay_default(void) 595 * Spins for approx a few hundred processor cycles and returns to caller. 596 */ 597 598 ENTRY(mutex_delay_default) 599 mov 72,%o0 6001: brgz %o0, 1b 601 dec %o0 602 retl 603 nop 604 SET_SIZE(mutex_delay_default) 605 606