1 /*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice(s), this list of conditions and the following disclaimer as 11 * the first lines of this file unmodified other than the possible 12 * addition of one or more copyright notices. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice(s), this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 18 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 27 * DAMAGE. 28 */ 29 30 /* 31 * Shared/exclusive locks. This implementation attempts to ensure 32 * deterministic lock granting behavior, so that slocks and xlocks are 33 * interleaved. 34 * 35 * Priority propagation will not generally raise the priority of lock holders, 36 * so should not be relied upon in combination with sx locks. 37 */ 38 39 #include "opt_ddb.h" 40 #include "opt_hwpmc_hooks.h" 41 #include "opt_no_adaptive_sx.h" 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/kdb.h> 49 #include <sys/kernel.h> 50 #include <sys/ktr.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/sched.h> 55 #include <sys/sleepqueue.h> 56 #include <sys/sx.h> 57 #include <sys/smp.h> 58 #include <sys/sysctl.h> 59 60 #if defined(SMP) && !defined(NO_ADAPTIVE_SX) 61 #include <machine/cpu.h> 62 #endif 63 64 #ifdef DDB 65 #include <ddb/ddb.h> 66 #endif 67 68 #if defined(SMP) && !defined(NO_ADAPTIVE_SX) 69 #define ADAPTIVE_SX 70 #endif 71 72 CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE); 73 74 #ifdef HWPMC_HOOKS 75 #include <sys/pmckern.h> 76 PMC_SOFT_DECLARE( , , lock, failed); 77 #endif 78 79 /* Handy macros for sleep queues. */ 80 #define SQ_EXCLUSIVE_QUEUE 0 81 #define SQ_SHARED_QUEUE 1 82 83 /* 84 * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file. We 85 * drop Giant anytime we have to sleep or if we adaptively spin. 86 */ 87 #define GIANT_DECLARE \ 88 int _giantcnt = 0; \ 89 WITNESS_SAVE_DECL(Giant) \ 90 91 #define GIANT_SAVE(work) do { \ 92 if (mtx_owned(&Giant)) { \ 93 work++; \ 94 WITNESS_SAVE(&Giant.lock_object, Giant); \ 95 while (mtx_owned(&Giant)) { \ 96 _giantcnt++; \ 97 mtx_unlock(&Giant); \ 98 } \ 99 } \ 100 } while (0) 101 102 #define GIANT_RESTORE() do { \ 103 if (_giantcnt > 0) { \ 104 mtx_assert(&Giant, MA_NOTOWNED); \ 105 while (_giantcnt--) \ 106 mtx_lock(&Giant); \ 107 WITNESS_RESTORE(&Giant.lock_object, Giant); \ 108 } \ 109 } while (0) 110 111 /* 112 * Returns true if an exclusive lock is recursed. It assumes 113 * curthread currently has an exclusive lock. 114 */ 115 #define sx_recursed(sx) ((sx)->sx_recurse != 0) 116 117 static void assert_sx(const struct lock_object *lock, int what); 118 #ifdef DDB 119 static void db_show_sx(const struct lock_object *lock); 120 #endif 121 static void lock_sx(struct lock_object *lock, uintptr_t how); 122 #ifdef KDTRACE_HOOKS 123 static int owner_sx(const struct lock_object *lock, struct thread **owner); 124 #endif 125 static uintptr_t unlock_sx(struct lock_object *lock); 126 127 struct lock_class lock_class_sx = { 128 .lc_name = "sx", 129 .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, 130 .lc_assert = assert_sx, 131 #ifdef DDB 132 .lc_ddb_show = db_show_sx, 133 #endif 134 .lc_lock = lock_sx, 135 .lc_unlock = unlock_sx, 136 #ifdef KDTRACE_HOOKS 137 .lc_owner = owner_sx, 138 #endif 139 }; 140 141 #ifndef INVARIANTS 142 #define _sx_assert(sx, what, file, line) 143 #endif 144 145 #ifdef ADAPTIVE_SX 146 static __read_frequently u_int asx_retries = 10; 147 static __read_frequently u_int asx_loops = 10000; 148 static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging"); 149 SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, ""); 150 SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, ""); 151 152 static struct lock_delay_config __read_frequently sx_delay; 153 154 SYSCTL_INT(_debug_sx, OID_AUTO, delay_base, CTLFLAG_RW, &sx_delay.base, 155 0, ""); 156 SYSCTL_INT(_debug_sx, OID_AUTO, delay_max, CTLFLAG_RW, &sx_delay.max, 157 0, ""); 158 159 LOCK_DELAY_SYSINIT_DEFAULT(sx_delay); 160 #endif 161 162 void 163 assert_sx(const struct lock_object *lock, int what) 164 { 165 166 sx_assert((const struct sx *)lock, what); 167 } 168 169 void 170 lock_sx(struct lock_object *lock, uintptr_t how) 171 { 172 struct sx *sx; 173 174 sx = (struct sx *)lock; 175 if (how) 176 sx_slock(sx); 177 else 178 sx_xlock(sx); 179 } 180 181 uintptr_t 182 unlock_sx(struct lock_object *lock) 183 { 184 struct sx *sx; 185 186 sx = (struct sx *)lock; 187 sx_assert(sx, SA_LOCKED | SA_NOTRECURSED); 188 if (sx_xlocked(sx)) { 189 sx_xunlock(sx); 190 return (0); 191 } else { 192 sx_sunlock(sx); 193 return (1); 194 } 195 } 196 197 #ifdef KDTRACE_HOOKS 198 int 199 owner_sx(const struct lock_object *lock, struct thread **owner) 200 { 201 const struct sx *sx; 202 uintptr_t x; 203 204 sx = (const struct sx *)lock; 205 x = sx->sx_lock; 206 *owner = NULL; 207 return ((x & SX_LOCK_SHARED) != 0 ? (SX_SHARERS(x) != 0) : 208 ((*owner = (struct thread *)SX_OWNER(x)) != NULL)); 209 } 210 #endif 211 212 void 213 sx_sysinit(void *arg) 214 { 215 struct sx_args *sargs = arg; 216 217 sx_init_flags(sargs->sa_sx, sargs->sa_desc, sargs->sa_flags); 218 } 219 220 void 221 sx_init_flags(struct sx *sx, const char *description, int opts) 222 { 223 int flags; 224 225 MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK | 226 SX_NOPROFILE | SX_NOADAPTIVE | SX_NEW)) == 0); 227 ASSERT_ATOMIC_LOAD_PTR(sx->sx_lock, 228 ("%s: sx_lock not aligned for %s: %p", __func__, description, 229 &sx->sx_lock)); 230 231 flags = LO_SLEEPABLE | LO_UPGRADABLE; 232 if (opts & SX_DUPOK) 233 flags |= LO_DUPOK; 234 if (opts & SX_NOPROFILE) 235 flags |= LO_NOPROFILE; 236 if (!(opts & SX_NOWITNESS)) 237 flags |= LO_WITNESS; 238 if (opts & SX_RECURSE) 239 flags |= LO_RECURSABLE; 240 if (opts & SX_QUIET) 241 flags |= LO_QUIET; 242 if (opts & SX_NEW) 243 flags |= LO_NEW; 244 245 flags |= opts & SX_NOADAPTIVE; 246 lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags); 247 sx->sx_lock = SX_LOCK_UNLOCKED; 248 sx->sx_recurse = 0; 249 } 250 251 void 252 sx_destroy(struct sx *sx) 253 { 254 255 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held")); 256 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed")); 257 sx->sx_lock = SX_LOCK_DESTROYED; 258 lock_destroy(&sx->lock_object); 259 } 260 261 int 262 sx_try_slock_(struct sx *sx, const char *file, int line) 263 { 264 uintptr_t x; 265 266 if (SCHEDULER_STOPPED()) 267 return (1); 268 269 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 270 ("sx_try_slock() by idle thread %p on sx %s @ %s:%d", 271 curthread, sx->lock_object.lo_name, file, line)); 272 273 x = sx->sx_lock; 274 for (;;) { 275 KASSERT(x != SX_LOCK_DESTROYED, 276 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 277 if (!(x & SX_LOCK_SHARED)) 278 break; 279 if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, x + SX_ONE_SHARER)) { 280 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 281 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); 282 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, 283 sx, 0, 0, file, line, LOCKSTAT_READER); 284 TD_LOCKS_INC(curthread); 285 return (1); 286 } 287 } 288 289 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 290 return (0); 291 } 292 293 int 294 _sx_xlock(struct sx *sx, int opts, const char *file, int line) 295 { 296 uintptr_t tid, x; 297 int error = 0; 298 299 KASSERT(kdb_active != 0 || SCHEDULER_STOPPED() || 300 !TD_IS_IDLETHREAD(curthread), 301 ("sx_xlock() by idle thread %p on sx %s @ %s:%d", 302 curthread, sx->lock_object.lo_name, file, line)); 303 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 304 ("sx_xlock() of destroyed sx @ %s:%d", file, line)); 305 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 306 line, NULL); 307 tid = (uintptr_t)curthread; 308 x = SX_LOCK_UNLOCKED; 309 if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid)) 310 error = _sx_xlock_hard(sx, x, tid, opts, file, line); 311 else 312 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx, 313 0, 0, file, line, LOCKSTAT_WRITER); 314 if (!error) { 315 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, 316 file, line); 317 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 318 TD_LOCKS_INC(curthread); 319 } 320 321 return (error); 322 } 323 324 int 325 sx_try_xlock_(struct sx *sx, const char *file, int line) 326 { 327 struct thread *td; 328 uintptr_t tid, x; 329 int rval; 330 bool recursed; 331 332 td = curthread; 333 tid = (uintptr_t)td; 334 if (SCHEDULER_STOPPED_TD(td)) 335 return (1); 336 337 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td), 338 ("sx_try_xlock() by idle thread %p on sx %s @ %s:%d", 339 curthread, sx->lock_object.lo_name, file, line)); 340 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 341 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); 342 343 rval = 1; 344 recursed = false; 345 x = SX_LOCK_UNLOCKED; 346 for (;;) { 347 if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid)) 348 break; 349 if (x == SX_LOCK_UNLOCKED) 350 continue; 351 if (x == tid && (sx->lock_object.lo_flags & LO_RECURSABLE)) { 352 sx->sx_recurse++; 353 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 354 break; 355 } 356 rval = 0; 357 break; 358 } 359 360 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 361 if (rval) { 362 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 363 file, line); 364 if (!recursed) 365 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, 366 sx, 0, 0, file, line, LOCKSTAT_WRITER); 367 TD_LOCKS_INC(curthread); 368 } 369 370 return (rval); 371 } 372 373 void 374 _sx_xunlock(struct sx *sx, const char *file, int line) 375 { 376 377 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 378 ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); 379 _sx_assert(sx, SA_XLOCKED, file, line); 380 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 381 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 382 line); 383 #if LOCK_DEBUG > 0 384 _sx_xunlock_hard(sx, (uintptr_t)curthread, file, line); 385 #else 386 __sx_xunlock(sx, curthread, file, line); 387 #endif 388 TD_LOCKS_DEC(curthread); 389 } 390 391 /* 392 * Try to do a non-blocking upgrade from a shared lock to an exclusive lock. 393 * This will only succeed if this thread holds a single shared lock. 394 * Return 1 if if the upgrade succeed, 0 otherwise. 395 */ 396 int 397 sx_try_upgrade_(struct sx *sx, const char *file, int line) 398 { 399 uintptr_t x; 400 int success; 401 402 if (SCHEDULER_STOPPED()) 403 return (1); 404 405 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 406 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); 407 _sx_assert(sx, SA_SLOCKED, file, line); 408 409 /* 410 * Try to switch from one shared lock to an exclusive lock. We need 411 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 412 * we will wake up the exclusive waiters when we drop the lock. 413 */ 414 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS; 415 success = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x, 416 (uintptr_t)curthread | x); 417 LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line); 418 if (success) { 419 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 420 file, line); 421 LOCKSTAT_RECORD0(sx__upgrade, sx); 422 } 423 return (success); 424 } 425 426 /* 427 * Downgrade an unrecursed exclusive lock into a single shared lock. 428 */ 429 void 430 sx_downgrade_(struct sx *sx, const char *file, int line) 431 { 432 uintptr_t x; 433 int wakeup_swapper; 434 435 if (SCHEDULER_STOPPED()) 436 return; 437 438 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 439 ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); 440 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); 441 #ifndef INVARIANTS 442 if (sx_recursed(sx)) 443 panic("downgrade of a recursed lock"); 444 #endif 445 446 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); 447 448 /* 449 * Try to switch from an exclusive lock with no shared waiters 450 * to one sharer with no shared waiters. If there are 451 * exclusive waiters, we don't need to lock the sleep queue so 452 * long as we preserve the flag. We do one quick try and if 453 * that fails we grab the sleepq lock to keep the flags from 454 * changing and do it the slow way. 455 * 456 * We have to lock the sleep queue if there are shared waiters 457 * so we can wake them up. 458 */ 459 x = sx->sx_lock; 460 if (!(x & SX_LOCK_SHARED_WAITERS) && 461 atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | 462 (x & SX_LOCK_EXCLUSIVE_WAITERS))) { 463 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 464 return; 465 } 466 467 /* 468 * Lock the sleep queue so we can read the waiters bits 469 * without any races and wakeup any shared waiters. 470 */ 471 sleepq_lock(&sx->lock_object); 472 473 /* 474 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single 475 * shared lock. If there are any shared waiters, wake them up. 476 */ 477 wakeup_swapper = 0; 478 x = sx->sx_lock; 479 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | 480 (x & SX_LOCK_EXCLUSIVE_WAITERS)); 481 if (x & SX_LOCK_SHARED_WAITERS) 482 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 483 0, SQ_SHARED_QUEUE); 484 sleepq_release(&sx->lock_object); 485 486 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 487 LOCKSTAT_RECORD0(sx__downgrade, sx); 488 489 if (wakeup_swapper) 490 kick_proc0(); 491 } 492 493 /* 494 * This function represents the so-called 'hard case' for sx_xlock 495 * operation. All 'easy case' failures are redirected to this. Note 496 * that ideally this would be a static function, but it needs to be 497 * accessible from at least sx.h. 498 */ 499 int 500 _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t tid, int opts, 501 const char *file, int line) 502 { 503 GIANT_DECLARE; 504 #ifdef ADAPTIVE_SX 505 volatile struct thread *owner; 506 u_int i, n, spintries = 0; 507 #endif 508 #ifdef LOCK_PROFILING 509 uint64_t waittime = 0; 510 int contested = 0; 511 #endif 512 int error = 0; 513 #if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) 514 struct lock_delay_arg lda; 515 #endif 516 #ifdef KDTRACE_HOOKS 517 u_int sleep_cnt = 0; 518 int64_t sleep_time = 0; 519 int64_t all_time = 0; 520 #endif 521 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 522 uintptr_t state; 523 #endif 524 int extra_work = 0; 525 526 if (SCHEDULER_STOPPED()) 527 return (0); 528 529 #if defined(ADAPTIVE_SX) 530 lock_delay_arg_init(&lda, &sx_delay); 531 #elif defined(KDTRACE_HOOKS) 532 lock_delay_arg_init(&lda, NULL); 533 #endif 534 535 if (__predict_false(x == SX_LOCK_UNLOCKED)) 536 x = SX_READ_VALUE(sx); 537 538 /* If we already hold an exclusive lock, then recurse. */ 539 if (__predict_false(lv_sx_owner(x) == (struct thread *)tid)) { 540 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, 541 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n", 542 sx->lock_object.lo_name, file, line)); 543 sx->sx_recurse++; 544 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 545 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 546 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 547 return (0); 548 } 549 550 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 551 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 552 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 553 554 #ifdef HWPMC_HOOKS 555 PMC_SOFT_CALL( , , lock, failed); 556 #endif 557 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 558 &waittime); 559 560 #ifdef LOCK_PROFILING 561 extra_work = 1; 562 state = x; 563 #elif defined(KDTRACE_HOOKS) 564 extra_work = lockstat_enabled; 565 if (__predict_false(extra_work)) { 566 all_time -= lockstat_nsecs(&sx->lock_object); 567 state = x; 568 } 569 #endif 570 571 for (;;) { 572 if (x == SX_LOCK_UNLOCKED) { 573 if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid)) 574 break; 575 continue; 576 } 577 #ifdef KDTRACE_HOOKS 578 lda.spin_cnt++; 579 #endif 580 #ifdef ADAPTIVE_SX 581 /* 582 * If the lock is write locked and the owner is 583 * running on another CPU, spin until the owner stops 584 * running or the state of the lock changes. 585 */ 586 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 587 if ((x & SX_LOCK_SHARED) == 0) { 588 owner = lv_sx_owner(x); 589 if (TD_IS_RUNNING(owner)) { 590 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 591 CTR3(KTR_LOCK, 592 "%s: spinning on %p held by %p", 593 __func__, sx, owner); 594 KTR_STATE1(KTR_SCHED, "thread", 595 sched_tdname(curthread), "spinning", 596 "lockname:\"%s\"", 597 sx->lock_object.lo_name); 598 GIANT_SAVE(extra_work); 599 do { 600 lock_delay(&lda); 601 x = SX_READ_VALUE(sx); 602 owner = lv_sx_owner(x); 603 } while (owner != NULL && 604 TD_IS_RUNNING(owner)); 605 KTR_STATE0(KTR_SCHED, "thread", 606 sched_tdname(curthread), "running"); 607 continue; 608 } 609 } else if (SX_SHARERS(x) && spintries < asx_retries) { 610 KTR_STATE1(KTR_SCHED, "thread", 611 sched_tdname(curthread), "spinning", 612 "lockname:\"%s\"", sx->lock_object.lo_name); 613 GIANT_SAVE(extra_work); 614 spintries++; 615 for (i = 0; i < asx_loops; i += n) { 616 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 617 CTR4(KTR_LOCK, 618 "%s: shared spinning on %p with %u and %u", 619 __func__, sx, spintries, i); 620 n = SX_SHARERS(x); 621 lock_delay_spin(n); 622 x = SX_READ_VALUE(sx); 623 if ((x & SX_LOCK_SHARED) == 0 || 624 SX_SHARERS(x) == 0) 625 break; 626 } 627 #ifdef KDTRACE_HOOKS 628 lda.spin_cnt += i; 629 #endif 630 KTR_STATE0(KTR_SCHED, "thread", 631 sched_tdname(curthread), "running"); 632 if (i != asx_loops) 633 continue; 634 } 635 } 636 #endif 637 638 sleepq_lock(&sx->lock_object); 639 x = SX_READ_VALUE(sx); 640 641 /* 642 * If the lock was released while spinning on the 643 * sleep queue chain lock, try again. 644 */ 645 if (x == SX_LOCK_UNLOCKED) { 646 sleepq_release(&sx->lock_object); 647 continue; 648 } 649 650 #ifdef ADAPTIVE_SX 651 /* 652 * The current lock owner might have started executing 653 * on another CPU (or the lock could have changed 654 * owners) while we were waiting on the sleep queue 655 * chain lock. If so, drop the sleep queue lock and try 656 * again. 657 */ 658 if (!(x & SX_LOCK_SHARED) && 659 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 660 owner = (struct thread *)SX_OWNER(x); 661 if (TD_IS_RUNNING(owner)) { 662 sleepq_release(&sx->lock_object); 663 continue; 664 } 665 } 666 #endif 667 668 /* 669 * If an exclusive lock was released with both shared 670 * and exclusive waiters and a shared waiter hasn't 671 * woken up and acquired the lock yet, sx_lock will be 672 * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS. 673 * If we see that value, try to acquire it once. Note 674 * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS 675 * as there are other exclusive waiters still. If we 676 * fail, restart the loop. 677 */ 678 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) { 679 if (atomic_cmpset_acq_ptr(&sx->sx_lock, 680 SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS, 681 tid | SX_LOCK_EXCLUSIVE_WAITERS)) { 682 sleepq_release(&sx->lock_object); 683 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 684 __func__, sx); 685 break; 686 } 687 sleepq_release(&sx->lock_object); 688 x = SX_READ_VALUE(sx); 689 continue; 690 } 691 692 /* 693 * Try to set the SX_LOCK_EXCLUSIVE_WAITERS. If we fail, 694 * than loop back and retry. 695 */ 696 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 697 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 698 x | SX_LOCK_EXCLUSIVE_WAITERS)) { 699 sleepq_release(&sx->lock_object); 700 x = SX_READ_VALUE(sx); 701 continue; 702 } 703 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 704 CTR2(KTR_LOCK, "%s: %p set excl waiters flag", 705 __func__, sx); 706 } 707 708 /* 709 * Since we have been unable to acquire the exclusive 710 * lock and the exclusive waiters flag is set, we have 711 * to sleep. 712 */ 713 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 714 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 715 __func__, sx); 716 717 #ifdef KDTRACE_HOOKS 718 sleep_time -= lockstat_nsecs(&sx->lock_object); 719 #endif 720 GIANT_SAVE(extra_work); 721 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 722 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 723 SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE); 724 if (!(opts & SX_INTERRUPTIBLE)) 725 sleepq_wait(&sx->lock_object, 0); 726 else 727 error = sleepq_wait_sig(&sx->lock_object, 0); 728 #ifdef KDTRACE_HOOKS 729 sleep_time += lockstat_nsecs(&sx->lock_object); 730 sleep_cnt++; 731 #endif 732 if (error) { 733 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 734 CTR2(KTR_LOCK, 735 "%s: interruptible sleep by %p suspended by signal", 736 __func__, sx); 737 break; 738 } 739 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 740 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 741 __func__, sx); 742 x = SX_READ_VALUE(sx); 743 } 744 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 745 if (__predict_true(!extra_work)) 746 return (error); 747 #endif 748 #ifdef KDTRACE_HOOKS 749 all_time += lockstat_nsecs(&sx->lock_object); 750 if (sleep_time) 751 LOCKSTAT_RECORD4(sx__block, sx, sleep_time, 752 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 753 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 754 if (lda.spin_cnt > sleep_cnt) 755 LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time, 756 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 757 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 758 #endif 759 if (!error) 760 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx, 761 contested, waittime, file, line, LOCKSTAT_WRITER); 762 GIANT_RESTORE(); 763 return (error); 764 } 765 766 /* 767 * This function represents the so-called 'hard case' for sx_xunlock 768 * operation. All 'easy case' failures are redirected to this. Note 769 * that ideally this would be a static function, but it needs to be 770 * accessible from at least sx.h. 771 */ 772 void 773 _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 774 { 775 uintptr_t x, setx; 776 int queue, wakeup_swapper; 777 778 if (SCHEDULER_STOPPED()) 779 return; 780 781 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 782 783 x = SX_READ_VALUE(sx); 784 if (x & SX_LOCK_RECURSED) { 785 /* The lock is recursed, unrecurse one level. */ 786 if ((--sx->sx_recurse) == 0) 787 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 788 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 789 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); 790 return; 791 } 792 793 LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_WRITER); 794 if (x == tid && 795 atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) 796 return; 797 798 MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | 799 SX_LOCK_EXCLUSIVE_WAITERS)); 800 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 801 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx); 802 803 sleepq_lock(&sx->lock_object); 804 x = SX_READ_VALUE(sx); 805 806 /* 807 * The wake up algorithm here is quite simple and probably not 808 * ideal. It gives precedence to shared waiters if they are 809 * present. For this condition, we have to preserve the 810 * state of the exclusive waiters flag. 811 * If interruptible sleeps left the shared queue empty avoid a 812 * starvation for the threads sleeping on the exclusive queue by giving 813 * them precedence and cleaning up the shared waiters bit anyway. 814 */ 815 setx = SX_LOCK_UNLOCKED; 816 queue = SQ_EXCLUSIVE_QUEUE; 817 if ((x & SX_LOCK_SHARED_WAITERS) != 0 && 818 sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) { 819 queue = SQ_SHARED_QUEUE; 820 setx |= (x & SX_LOCK_EXCLUSIVE_WAITERS); 821 } 822 atomic_store_rel_ptr(&sx->sx_lock, setx); 823 824 /* Wake up all the waiters for the specific queue. */ 825 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 826 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue", 827 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : 828 "exclusive"); 829 830 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, 831 queue); 832 sleepq_release(&sx->lock_object); 833 if (wakeup_swapper) 834 kick_proc0(); 835 } 836 837 static bool __always_inline 838 __sx_slock_try(struct sx *sx, uintptr_t *xp, const char *file, int line) 839 { 840 841 /* 842 * If no other thread has an exclusive lock then try to bump up 843 * the count of sharers. Since we have to preserve the state 844 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the 845 * shared lock loop back and retry. 846 */ 847 while (*xp & SX_LOCK_SHARED) { 848 MPASS(!(*xp & SX_LOCK_SHARED_WAITERS)); 849 if (atomic_fcmpset_acq_ptr(&sx->sx_lock, xp, 850 *xp + SX_ONE_SHARER)) { 851 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 852 CTR4(KTR_LOCK, "%s: %p succeed %p -> %p", 853 __func__, sx, (void *)*xp, 854 (void *)(*xp + SX_ONE_SHARER)); 855 return (true); 856 } 857 } 858 return (false); 859 } 860 861 static int __noinline 862 _sx_slock_hard(struct sx *sx, int opts, const char *file, int line, uintptr_t x) 863 { 864 GIANT_DECLARE; 865 #ifdef ADAPTIVE_SX 866 volatile struct thread *owner; 867 #endif 868 #ifdef LOCK_PROFILING 869 uint64_t waittime = 0; 870 int contested = 0; 871 #endif 872 int error = 0; 873 #if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) 874 struct lock_delay_arg lda; 875 #endif 876 #ifdef KDTRACE_HOOKS 877 u_int sleep_cnt = 0; 878 int64_t sleep_time = 0; 879 int64_t all_time = 0; 880 #endif 881 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 882 uintptr_t state; 883 #endif 884 int extra_work = 0; 885 886 if (SCHEDULER_STOPPED()) 887 return (0); 888 889 #if defined(ADAPTIVE_SX) 890 lock_delay_arg_init(&lda, &sx_delay); 891 #elif defined(KDTRACE_HOOKS) 892 lock_delay_arg_init(&lda, NULL); 893 #endif 894 895 #ifdef HWPMC_HOOKS 896 PMC_SOFT_CALL( , , lock, failed); 897 #endif 898 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 899 &waittime); 900 901 #ifdef LOCK_PROFILING 902 extra_work = 1; 903 state = x; 904 #elif defined(KDTRACE_HOOKS) 905 extra_work = lockstat_enabled; 906 if (__predict_false(extra_work)) { 907 all_time -= lockstat_nsecs(&sx->lock_object); 908 state = x; 909 } 910 #endif 911 912 /* 913 * As with rwlocks, we don't make any attempt to try to block 914 * shared locks once there is an exclusive waiter. 915 */ 916 for (;;) { 917 if (__sx_slock_try(sx, &x, file, line)) 918 break; 919 #ifdef KDTRACE_HOOKS 920 lda.spin_cnt++; 921 #endif 922 923 #ifdef ADAPTIVE_SX 924 /* 925 * If the owner is running on another CPU, spin until 926 * the owner stops running or the state of the lock 927 * changes. 928 */ 929 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 930 owner = lv_sx_owner(x); 931 if (TD_IS_RUNNING(owner)) { 932 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 933 CTR3(KTR_LOCK, 934 "%s: spinning on %p held by %p", 935 __func__, sx, owner); 936 KTR_STATE1(KTR_SCHED, "thread", 937 sched_tdname(curthread), "spinning", 938 "lockname:\"%s\"", sx->lock_object.lo_name); 939 GIANT_SAVE(extra_work); 940 do { 941 lock_delay(&lda); 942 x = SX_READ_VALUE(sx); 943 owner = lv_sx_owner(x); 944 } while (owner != NULL && TD_IS_RUNNING(owner)); 945 KTR_STATE0(KTR_SCHED, "thread", 946 sched_tdname(curthread), "running"); 947 continue; 948 } 949 } 950 #endif 951 952 /* 953 * Some other thread already has an exclusive lock, so 954 * start the process of blocking. 955 */ 956 sleepq_lock(&sx->lock_object); 957 x = SX_READ_VALUE(sx); 958 959 /* 960 * The lock could have been released while we spun. 961 * In this case loop back and retry. 962 */ 963 if (x & SX_LOCK_SHARED) { 964 sleepq_release(&sx->lock_object); 965 continue; 966 } 967 968 #ifdef ADAPTIVE_SX 969 /* 970 * If the owner is running on another CPU, spin until 971 * the owner stops running or the state of the lock 972 * changes. 973 */ 974 if (!(x & SX_LOCK_SHARED) && 975 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 976 owner = (struct thread *)SX_OWNER(x); 977 if (TD_IS_RUNNING(owner)) { 978 sleepq_release(&sx->lock_object); 979 x = SX_READ_VALUE(sx); 980 continue; 981 } 982 } 983 #endif 984 985 /* 986 * Try to set the SX_LOCK_SHARED_WAITERS flag. If we 987 * fail to set it drop the sleep queue lock and loop 988 * back. 989 */ 990 if (!(x & SX_LOCK_SHARED_WAITERS)) { 991 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 992 x | SX_LOCK_SHARED_WAITERS)) { 993 sleepq_release(&sx->lock_object); 994 x = SX_READ_VALUE(sx); 995 continue; 996 } 997 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 998 CTR2(KTR_LOCK, "%s: %p set shared waiters flag", 999 __func__, sx); 1000 } 1001 1002 /* 1003 * Since we have been unable to acquire the shared lock, 1004 * we have to sleep. 1005 */ 1006 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1007 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 1008 __func__, sx); 1009 1010 #ifdef KDTRACE_HOOKS 1011 sleep_time -= lockstat_nsecs(&sx->lock_object); 1012 #endif 1013 GIANT_SAVE(extra_work); 1014 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 1015 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 1016 SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE); 1017 if (!(opts & SX_INTERRUPTIBLE)) 1018 sleepq_wait(&sx->lock_object, 0); 1019 else 1020 error = sleepq_wait_sig(&sx->lock_object, 0); 1021 #ifdef KDTRACE_HOOKS 1022 sleep_time += lockstat_nsecs(&sx->lock_object); 1023 sleep_cnt++; 1024 #endif 1025 if (error) { 1026 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1027 CTR2(KTR_LOCK, 1028 "%s: interruptible sleep by %p suspended by signal", 1029 __func__, sx); 1030 break; 1031 } 1032 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1033 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 1034 __func__, sx); 1035 x = SX_READ_VALUE(sx); 1036 } 1037 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 1038 if (__predict_true(!extra_work)) 1039 return (error); 1040 #endif 1041 #ifdef KDTRACE_HOOKS 1042 all_time += lockstat_nsecs(&sx->lock_object); 1043 if (sleep_time) 1044 LOCKSTAT_RECORD4(sx__block, sx, sleep_time, 1045 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 1046 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 1047 if (lda.spin_cnt > sleep_cnt) 1048 LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time, 1049 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 1050 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 1051 #endif 1052 if (error == 0) { 1053 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx, 1054 contested, waittime, file, line, LOCKSTAT_READER); 1055 } 1056 GIANT_RESTORE(); 1057 return (error); 1058 } 1059 1060 int 1061 _sx_slock(struct sx *sx, int opts, const char *file, int line) 1062 { 1063 uintptr_t x; 1064 int error; 1065 1066 KASSERT(kdb_active != 0 || SCHEDULER_STOPPED() || 1067 !TD_IS_IDLETHREAD(curthread), 1068 ("sx_slock() by idle thread %p on sx %s @ %s:%d", 1069 curthread, sx->lock_object.lo_name, file, line)); 1070 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 1071 ("sx_slock() of destroyed sx @ %s:%d", file, line)); 1072 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); 1073 1074 error = 0; 1075 x = SX_READ_VALUE(sx); 1076 if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__acquire) || 1077 !__sx_slock_try(sx, &x, file, line))) 1078 error = _sx_slock_hard(sx, opts, file, line, x); 1079 if (error == 0) { 1080 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 1081 WITNESS_LOCK(&sx->lock_object, 0, file, line); 1082 TD_LOCKS_INC(curthread); 1083 } 1084 return (error); 1085 } 1086 1087 static bool __always_inline 1088 _sx_sunlock_try(struct sx *sx, uintptr_t *xp) 1089 { 1090 1091 for (;;) { 1092 /* 1093 * We should never have sharers while at least one thread 1094 * holds a shared lock. 1095 */ 1096 KASSERT(!(*xp & SX_LOCK_SHARED_WAITERS), 1097 ("%s: waiting sharers", __func__)); 1098 1099 /* 1100 * See if there is more than one shared lock held. If 1101 * so, just drop one and return. 1102 */ 1103 if (SX_SHARERS(*xp) > 1) { 1104 if (atomic_fcmpset_rel_ptr(&sx->sx_lock, xp, 1105 *xp - SX_ONE_SHARER)) { 1106 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1107 CTR4(KTR_LOCK, 1108 "%s: %p succeeded %p -> %p", 1109 __func__, sx, (void *)*xp, 1110 (void *)(*xp - SX_ONE_SHARER)); 1111 return (true); 1112 } 1113 continue; 1114 } 1115 1116 /* 1117 * If there aren't any waiters for an exclusive lock, 1118 * then try to drop it quickly. 1119 */ 1120 if (!(*xp & SX_LOCK_EXCLUSIVE_WAITERS)) { 1121 MPASS(*xp == SX_SHARERS_LOCK(1)); 1122 *xp = SX_SHARERS_LOCK(1); 1123 if (atomic_fcmpset_rel_ptr(&sx->sx_lock, 1124 xp, SX_LOCK_UNLOCKED)) { 1125 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1126 CTR2(KTR_LOCK, "%s: %p last succeeded", 1127 __func__, sx); 1128 return (true); 1129 } 1130 continue; 1131 } 1132 break; 1133 } 1134 return (false); 1135 } 1136 1137 static void __noinline 1138 _sx_sunlock_hard(struct sx *sx, uintptr_t x, const char *file, int line) 1139 { 1140 int wakeup_swapper; 1141 1142 if (SCHEDULER_STOPPED()) 1143 return; 1144 1145 LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_READER); 1146 1147 for (;;) { 1148 if (_sx_sunlock_try(sx, &x)) 1149 break; 1150 1151 /* 1152 * At this point, there should just be one sharer with 1153 * exclusive waiters. 1154 */ 1155 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS)); 1156 1157 sleepq_lock(&sx->lock_object); 1158 1159 /* 1160 * Wake up semantic here is quite simple: 1161 * Just wake up all the exclusive waiters. 1162 * Note that the state of the lock could have changed, 1163 * so if it fails loop back and retry. 1164 */ 1165 if (!atomic_cmpset_rel_ptr(&sx->sx_lock, 1166 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS, 1167 SX_LOCK_UNLOCKED)) { 1168 sleepq_release(&sx->lock_object); 1169 x = SX_READ_VALUE(sx); 1170 continue; 1171 } 1172 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1173 CTR2(KTR_LOCK, "%s: %p waking up all thread on" 1174 "exclusive queue", __func__, sx); 1175 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 1176 0, SQ_EXCLUSIVE_QUEUE); 1177 sleepq_release(&sx->lock_object); 1178 if (wakeup_swapper) 1179 kick_proc0(); 1180 break; 1181 } 1182 } 1183 1184 void 1185 _sx_sunlock(struct sx *sx, const char *file, int line) 1186 { 1187 uintptr_t x; 1188 1189 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 1190 ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); 1191 _sx_assert(sx, SA_SLOCKED, file, line); 1192 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 1193 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 1194 1195 x = SX_READ_VALUE(sx); 1196 if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__release) || 1197 !_sx_sunlock_try(sx, &x))) 1198 _sx_sunlock_hard(sx, x, file, line); 1199 1200 TD_LOCKS_DEC(curthread); 1201 } 1202 1203 #ifdef INVARIANT_SUPPORT 1204 #ifndef INVARIANTS 1205 #undef _sx_assert 1206 #endif 1207 1208 /* 1209 * In the non-WITNESS case, sx_assert() can only detect that at least 1210 * *some* thread owns an slock, but it cannot guarantee that *this* 1211 * thread owns an slock. 1212 */ 1213 void 1214 _sx_assert(const struct sx *sx, int what, const char *file, int line) 1215 { 1216 #ifndef WITNESS 1217 int slocked = 0; 1218 #endif 1219 1220 if (panicstr != NULL) 1221 return; 1222 switch (what) { 1223 case SA_SLOCKED: 1224 case SA_SLOCKED | SA_NOTRECURSED: 1225 case SA_SLOCKED | SA_RECURSED: 1226 #ifndef WITNESS 1227 slocked = 1; 1228 /* FALLTHROUGH */ 1229 #endif 1230 case SA_LOCKED: 1231 case SA_LOCKED | SA_NOTRECURSED: 1232 case SA_LOCKED | SA_RECURSED: 1233 #ifdef WITNESS 1234 witness_assert(&sx->lock_object, what, file, line); 1235 #else 1236 /* 1237 * If some other thread has an exclusive lock or we 1238 * have one and are asserting a shared lock, fail. 1239 * Also, if no one has a lock at all, fail. 1240 */ 1241 if (sx->sx_lock == SX_LOCK_UNLOCKED || 1242 (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked || 1243 sx_xholder(sx) != curthread))) 1244 panic("Lock %s not %slocked @ %s:%d\n", 1245 sx->lock_object.lo_name, slocked ? "share " : "", 1246 file, line); 1247 1248 if (!(sx->sx_lock & SX_LOCK_SHARED)) { 1249 if (sx_recursed(sx)) { 1250 if (what & SA_NOTRECURSED) 1251 panic("Lock %s recursed @ %s:%d\n", 1252 sx->lock_object.lo_name, file, 1253 line); 1254 } else if (what & SA_RECURSED) 1255 panic("Lock %s not recursed @ %s:%d\n", 1256 sx->lock_object.lo_name, file, line); 1257 } 1258 #endif 1259 break; 1260 case SA_XLOCKED: 1261 case SA_XLOCKED | SA_NOTRECURSED: 1262 case SA_XLOCKED | SA_RECURSED: 1263 if (sx_xholder(sx) != curthread) 1264 panic("Lock %s not exclusively locked @ %s:%d\n", 1265 sx->lock_object.lo_name, file, line); 1266 if (sx_recursed(sx)) { 1267 if (what & SA_NOTRECURSED) 1268 panic("Lock %s recursed @ %s:%d\n", 1269 sx->lock_object.lo_name, file, line); 1270 } else if (what & SA_RECURSED) 1271 panic("Lock %s not recursed @ %s:%d\n", 1272 sx->lock_object.lo_name, file, line); 1273 break; 1274 case SA_UNLOCKED: 1275 #ifdef WITNESS 1276 witness_assert(&sx->lock_object, what, file, line); 1277 #else 1278 /* 1279 * If we hold an exclusve lock fail. We can't 1280 * reliably check to see if we hold a shared lock or 1281 * not. 1282 */ 1283 if (sx_xholder(sx) == curthread) 1284 panic("Lock %s exclusively locked @ %s:%d\n", 1285 sx->lock_object.lo_name, file, line); 1286 #endif 1287 break; 1288 default: 1289 panic("Unknown sx lock assertion: %d @ %s:%d", what, file, 1290 line); 1291 } 1292 } 1293 #endif /* INVARIANT_SUPPORT */ 1294 1295 #ifdef DDB 1296 static void 1297 db_show_sx(const struct lock_object *lock) 1298 { 1299 struct thread *td; 1300 const struct sx *sx; 1301 1302 sx = (const struct sx *)lock; 1303 1304 db_printf(" state: "); 1305 if (sx->sx_lock == SX_LOCK_UNLOCKED) 1306 db_printf("UNLOCKED\n"); 1307 else if (sx->sx_lock == SX_LOCK_DESTROYED) { 1308 db_printf("DESTROYED\n"); 1309 return; 1310 } else if (sx->sx_lock & SX_LOCK_SHARED) 1311 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock)); 1312 else { 1313 td = sx_xholder(sx); 1314 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1315 td->td_tid, td->td_proc->p_pid, td->td_name); 1316 if (sx_recursed(sx)) 1317 db_printf(" recursed: %d\n", sx->sx_recurse); 1318 } 1319 1320 db_printf(" waiters: "); 1321 switch(sx->sx_lock & 1322 (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) { 1323 case SX_LOCK_SHARED_WAITERS: 1324 db_printf("shared\n"); 1325 break; 1326 case SX_LOCK_EXCLUSIVE_WAITERS: 1327 db_printf("exclusive\n"); 1328 break; 1329 case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS: 1330 db_printf("exclusive and shared\n"); 1331 break; 1332 default: 1333 db_printf("none\n"); 1334 } 1335 } 1336 1337 /* 1338 * Check to see if a thread that is blocked on a sleep queue is actually 1339 * blocked on an sx lock. If so, output some details and return true. 1340 * If the lock has an exclusive owner, return that in *ownerp. 1341 */ 1342 int 1343 sx_chain(struct thread *td, struct thread **ownerp) 1344 { 1345 struct sx *sx; 1346 1347 /* 1348 * Check to see if this thread is blocked on an sx lock. 1349 * First, we check the lock class. If that is ok, then we 1350 * compare the lock name against the wait message. 1351 */ 1352 sx = td->td_wchan; 1353 if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx || 1354 sx->lock_object.lo_name != td->td_wmesg) 1355 return (0); 1356 1357 /* We think we have an sx lock, so output some details. */ 1358 db_printf("blocked on sx \"%s\" ", td->td_wmesg); 1359 *ownerp = sx_xholder(sx); 1360 if (sx->sx_lock & SX_LOCK_SHARED) 1361 db_printf("SLOCK (count %ju)\n", 1362 (uintmax_t)SX_SHARERS(sx->sx_lock)); 1363 else 1364 db_printf("XLOCK\n"); 1365 return (1); 1366 } 1367 #endif 1368