1 /*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice(s), this list of conditions and the following disclaimer as 11 * the first lines of this file unmodified other than the possible 12 * addition of one or more copyright notices. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice(s), this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 18 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 27 * DAMAGE. 28 */ 29 30 /* 31 * Shared/exclusive locks. This implementation attempts to ensure 32 * deterministic lock granting behavior, so that slocks and xlocks are 33 * interleaved. 34 * 35 * Priority propagation will not generally raise the priority of lock holders, 36 * so should not be relied upon in combination with sx locks. 37 */ 38 39 #include "opt_adaptive_sx.h" 40 #include "opt_ddb.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/ktr.h> 47 #include <sys/lock.h> 48 #include <sys/lock_profile.h> 49 #include <sys/mutex.h> 50 #include <sys/proc.h> 51 #include <sys/sleepqueue.h> 52 #include <sys/sx.h> 53 #include <sys/systm.h> 54 55 #ifdef ADAPTIVE_SX 56 #include <machine/cpu.h> 57 #endif 58 59 #ifdef DDB 60 #include <ddb/ddb.h> 61 #endif 62 63 #if !defined(SMP) && defined(ADAPTIVE_SX) 64 #error "You must have SMP to enable the ADAPTIVE_SX option" 65 #endif 66 67 /* Handy macros for sleep queues. */ 68 #define SQ_EXCLUSIVE_QUEUE 0 69 #define SQ_SHARED_QUEUE 1 70 71 /* 72 * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file. We 73 * drop Giant anytime we have to sleep or if we adaptively spin. 74 */ 75 #define GIANT_DECLARE \ 76 int _giantcnt = 0; \ 77 WITNESS_SAVE_DECL(Giant) \ 78 79 #define GIANT_SAVE() do { \ 80 if (mtx_owned(&Giant)) { \ 81 WITNESS_SAVE(&Giant.lock_object, Giant); \ 82 while (mtx_owned(&Giant)) { \ 83 _giantcnt++; \ 84 mtx_unlock(&Giant); \ 85 } \ 86 } \ 87 } while (0) 88 89 #define GIANT_RESTORE() do { \ 90 if (_giantcnt > 0) { \ 91 mtx_assert(&Giant, MA_NOTOWNED); \ 92 while (_giantcnt--) \ 93 mtx_lock(&Giant); \ 94 WITNESS_RESTORE(&Giant.lock_object, Giant); \ 95 } \ 96 } while (0) 97 98 /* 99 * Returns true if an exclusive lock is recursed. It assumes 100 * curthread currently has an exclusive lock. 101 */ 102 #define sx_recursed(sx) ((sx)->sx_recurse != 0) 103 104 /* 105 * Return a pointer to the owning thread if the lock is exclusively 106 * locked. 107 */ 108 #define sx_xholder(sx) \ 109 ((sx)->sx_lock & SX_LOCK_SHARED ? NULL : \ 110 (struct thread *)SX_OWNER((sx)->sx_lock)) 111 112 #ifdef DDB 113 static void db_show_sx(struct lock_object *lock); 114 #endif 115 static void lock_sx(struct lock_object *lock, int how); 116 static int unlock_sx(struct lock_object *lock); 117 118 struct lock_class lock_class_sx = { 119 .lc_name = "sx", 120 .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, 121 #ifdef DDB 122 .lc_ddb_show = db_show_sx, 123 #endif 124 .lc_lock = lock_sx, 125 .lc_unlock = unlock_sx, 126 }; 127 128 #ifndef INVARIANTS 129 #define _sx_assert(sx, what, file, line) 130 #endif 131 132 void 133 lock_sx(struct lock_object *lock, int how) 134 { 135 struct sx *sx; 136 137 sx = (struct sx *)lock; 138 if (how) 139 sx_xlock(sx); 140 else 141 sx_slock(sx); 142 } 143 144 int 145 unlock_sx(struct lock_object *lock) 146 { 147 struct sx *sx; 148 149 sx = (struct sx *)lock; 150 sx_assert(sx, SX_LOCKED | SX_NOTRECURSED); 151 if (sx_xlocked(sx)) { 152 sx_xunlock(sx); 153 return (1); 154 } else { 155 sx_sunlock(sx); 156 return (0); 157 } 158 } 159 160 void 161 sx_sysinit(void *arg) 162 { 163 struct sx_args *sargs = arg; 164 165 sx_init(sargs->sa_sx, sargs->sa_desc); 166 } 167 168 void 169 sx_init_flags(struct sx *sx, const char *description, int opts) 170 { 171 int flags; 172 173 flags = LO_SLEEPABLE | LO_UPGRADABLE | LO_RECURSABLE; 174 if (opts & SX_DUPOK) 175 flags |= LO_DUPOK; 176 if (opts & SX_NOPROFILE) 177 flags |= LO_NOPROFILE; 178 if (!(opts & SX_NOWITNESS)) 179 flags |= LO_WITNESS; 180 if (opts & SX_QUIET) 181 flags |= LO_QUIET; 182 183 flags |= opts & SX_ADAPTIVESPIN; 184 sx->sx_lock = SX_LOCK_UNLOCKED; 185 sx->sx_recurse = 0; 186 lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags); 187 } 188 189 void 190 sx_destroy(struct sx *sx) 191 { 192 193 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held")); 194 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed")); 195 sx->sx_lock = SX_LOCK_DESTROYED; 196 lock_destroy(&sx->lock_object); 197 } 198 199 void 200 _sx_slock(struct sx *sx, const char *file, int line) 201 { 202 203 MPASS(curthread != NULL); 204 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 205 ("sx_slock() of destroyed sx @ %s:%d", file, line)); 206 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line); 207 __sx_slock(sx, file, line); 208 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 209 WITNESS_LOCK(&sx->lock_object, 0, file, line); 210 curthread->td_locks++; 211 } 212 213 int 214 _sx_try_slock(struct sx *sx, const char *file, int line) 215 { 216 uintptr_t x; 217 218 x = sx->sx_lock; 219 KASSERT(x != SX_LOCK_DESTROYED, 220 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 221 if ((x & SX_LOCK_SHARED) && atomic_cmpset_acq_ptr(&sx->sx_lock, x, 222 x + SX_ONE_SHARER)) { 223 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 224 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); 225 curthread->td_locks++; 226 return (1); 227 } 228 229 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 230 return (0); 231 } 232 233 void 234 _sx_xlock(struct sx *sx, const char *file, int line) 235 { 236 237 MPASS(curthread != NULL); 238 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 239 ("sx_xlock() of destroyed sx @ %s:%d", file, line)); 240 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 241 line); 242 __sx_xlock(sx, curthread, file, line); 243 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, file, line); 244 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 245 curthread->td_locks++; 246 } 247 248 int 249 _sx_try_xlock(struct sx *sx, const char *file, int line) 250 { 251 int rval; 252 253 MPASS(curthread != NULL); 254 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 255 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); 256 257 if (sx_xlocked(sx)) { 258 sx->sx_recurse++; 259 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 260 rval = 1; 261 } else 262 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, 263 (uintptr_t)curthread); 264 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 265 if (rval) { 266 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 267 file, line); 268 curthread->td_locks++; 269 } 270 271 return (rval); 272 } 273 274 void 275 _sx_sunlock(struct sx *sx, const char *file, int line) 276 { 277 278 MPASS(curthread != NULL); 279 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 280 ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); 281 _sx_assert(sx, SX_SLOCKED, file, line); 282 curthread->td_locks--; 283 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 284 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 285 if (SX_SHARERS(sx->sx_lock) == 0) 286 lock_profile_release_lock(&sx->lock_object); 287 __sx_sunlock(sx, file, line); 288 } 289 290 void 291 _sx_xunlock(struct sx *sx, const char *file, int line) 292 { 293 294 MPASS(curthread != NULL); 295 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 296 ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); 297 _sx_assert(sx, SX_XLOCKED, file, line); 298 curthread->td_locks--; 299 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 300 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 301 line); 302 if (!sx_recursed(sx)) 303 lock_profile_release_lock(&sx->lock_object); 304 __sx_xunlock(sx, curthread, file, line); 305 } 306 307 /* 308 * Try to do a non-blocking upgrade from a shared lock to an exclusive lock. 309 * This will only succeed if this thread holds a single shared lock. 310 * Return 1 if if the upgrade succeed, 0 otherwise. 311 */ 312 int 313 _sx_try_upgrade(struct sx *sx, const char *file, int line) 314 { 315 uintptr_t x; 316 int success; 317 318 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 319 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); 320 _sx_assert(sx, SX_SLOCKED, file, line); 321 322 /* 323 * Try to switch from one shared lock to an exclusive lock. We need 324 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 325 * we will wake up the exclusive waiters when we drop the lock. 326 */ 327 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS; 328 success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x, 329 (uintptr_t)curthread | x); 330 LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line); 331 if (success) 332 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 333 file, line); 334 return (success); 335 } 336 337 /* 338 * Downgrade an unrecursed exclusive lock into a single shared lock. 339 */ 340 void 341 _sx_downgrade(struct sx *sx, const char *file, int line) 342 { 343 uintptr_t x; 344 345 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 346 ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); 347 _sx_assert(sx, SX_XLOCKED | SX_NOTRECURSED, file, line); 348 #ifndef INVARIANTS 349 if (sx_recursed(sx)) 350 panic("downgrade of a recursed lock"); 351 #endif 352 353 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); 354 355 /* 356 * Try to switch from an exclusive lock with no shared waiters 357 * to one sharer with no shared waiters. If there are 358 * exclusive waiters, we don't need to lock the sleep queue so 359 * long as we preserve the flag. We do one quick try and if 360 * that fails we grab the sleepq lock to keep the flags from 361 * changing and do it the slow way. 362 * 363 * We have to lock the sleep queue if there are shared waiters 364 * so we can wake them up. 365 */ 366 x = sx->sx_lock; 367 if (!(x & SX_LOCK_SHARED_WAITERS) && 368 atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | 369 (x & SX_LOCK_EXCLUSIVE_WAITERS))) { 370 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 371 return; 372 } 373 374 /* 375 * Lock the sleep queue so we can read the waiters bits 376 * without any races and wakeup any shared waiters. 377 */ 378 sleepq_lock(&sx->lock_object); 379 380 /* 381 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single 382 * shared lock. If there are any shared waiters, wake them up. 383 */ 384 x = sx->sx_lock; 385 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | 386 (x & SX_LOCK_EXCLUSIVE_WAITERS)); 387 if (x & SX_LOCK_SHARED_WAITERS) 388 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, 389 SQ_SHARED_QUEUE); 390 else 391 sleepq_release(&sx->lock_object); 392 393 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 394 } 395 396 /* 397 * This function represents the so-called 'hard case' for sx_xlock 398 * operation. All 'easy case' failures are redirected to this. Note 399 * that ideally this would be a static function, but it needs to be 400 * accessible from at least sx.h. 401 */ 402 void 403 _sx_xlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 404 { 405 GIANT_DECLARE; 406 #ifdef ADAPTIVE_SX 407 volatile struct thread *owner; 408 #endif 409 uintptr_t x; 410 int contested = 0; 411 uint64_t waitstart = 0; 412 413 /* If we already hold an exclusive lock, then recurse. */ 414 if (sx_xlocked(sx)) { 415 sx->sx_recurse++; 416 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 417 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 418 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 419 return; 420 } 421 lock_profile_obtain_lock_failed(&(sx)->lock_object, 422 &contested, &waitstart); 423 424 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 425 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 426 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 427 428 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 429 #ifdef ADAPTIVE_SX 430 /* 431 * If the lock is write locked and the owner is 432 * running on another CPU, spin until the owner stops 433 * running or the state of the lock changes. 434 */ 435 x = sx->sx_lock; 436 if (!(x & SX_LOCK_SHARED) && 437 (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) { 438 x = SX_OWNER(x); 439 owner = (struct thread *)x; 440 if (TD_IS_RUNNING(owner)) { 441 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 442 CTR3(KTR_LOCK, 443 "%s: spinning on %p held by %p", 444 __func__, sx, owner); 445 GIANT_SAVE(); 446 while (SX_OWNER(sx->sx_lock) == x && 447 TD_IS_RUNNING(owner)) 448 cpu_spinwait(); 449 continue; 450 } 451 } 452 #endif 453 454 sleepq_lock(&sx->lock_object); 455 x = sx->sx_lock; 456 457 /* 458 * If the lock was released while spinning on the 459 * sleep queue chain lock, try again. 460 */ 461 if (x == SX_LOCK_UNLOCKED) { 462 sleepq_release(&sx->lock_object); 463 continue; 464 } 465 466 #ifdef ADAPTIVE_SX 467 /* 468 * The current lock owner might have started executing 469 * on another CPU (or the lock could have changed 470 * owners) while we were waiting on the sleep queue 471 * chain lock. If so, drop the sleep queue lock and try 472 * again. 473 */ 474 if (!(x & SX_LOCK_SHARED) && 475 (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) { 476 owner = (struct thread *)SX_OWNER(x); 477 if (TD_IS_RUNNING(owner)) { 478 sleepq_release(&sx->lock_object); 479 continue; 480 } 481 } 482 #endif 483 484 /* 485 * If an exclusive lock was released with both shared 486 * and exclusive waiters and a shared waiter hasn't 487 * woken up and acquired the lock yet, sx_lock will be 488 * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS. 489 * If we see that value, try to acquire it once. Note 490 * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS 491 * as there are other exclusive waiters still. If we 492 * fail, restart the loop. 493 */ 494 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) { 495 if (atomic_cmpset_acq_ptr(&sx->sx_lock, 496 SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS, 497 tid | SX_LOCK_EXCLUSIVE_WAITERS)) { 498 sleepq_release(&sx->lock_object); 499 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 500 __func__, sx); 501 break; 502 } 503 sleepq_release(&sx->lock_object); 504 continue; 505 } 506 507 /* 508 * Try to set the SX_LOCK_EXCLUSIVE_WAITERS. If we fail, 509 * than loop back and retry. 510 */ 511 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 512 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 513 x | SX_LOCK_EXCLUSIVE_WAITERS)) { 514 sleepq_release(&sx->lock_object); 515 continue; 516 } 517 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 518 CTR2(KTR_LOCK, "%s: %p set excl waiters flag", 519 __func__, sx); 520 } 521 522 /* 523 * Since we have been unable to acquire the exclusive 524 * lock and the exclusive waiters flag is set, we have 525 * to sleep. 526 */ 527 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 528 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 529 __func__, sx); 530 531 GIANT_SAVE(); 532 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 533 SLEEPQ_SX, SQ_EXCLUSIVE_QUEUE); 534 sleepq_wait(&sx->lock_object); 535 536 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 537 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 538 __func__, sx); 539 } 540 541 GIANT_RESTORE(); 542 lock_profile_obtain_lock_success(&(sx)->lock_object, contested, 543 waitstart, file, line); 544 } 545 546 /* 547 * This function represents the so-called 'hard case' for sx_xunlock 548 * operation. All 'easy case' failures are redirected to this. Note 549 * that ideally this would be a static function, but it needs to be 550 * accessible from at least sx.h. 551 */ 552 void 553 _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 554 { 555 uintptr_t x; 556 int queue; 557 558 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 559 560 /* If the lock is recursed, then unrecurse one level. */ 561 if (sx_xlocked(sx) && sx_recursed(sx)) { 562 if ((--sx->sx_recurse) == 0) 563 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 564 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 565 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); 566 return; 567 } 568 MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | 569 SX_LOCK_EXCLUSIVE_WAITERS)); 570 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 571 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx); 572 573 sleepq_lock(&sx->lock_object); 574 x = SX_LOCK_UNLOCKED; 575 576 /* 577 * The wake up algorithm here is quite simple and probably not 578 * ideal. It gives precedence to shared waiters if they are 579 * present. For this condition, we have to preserve the 580 * state of the exclusive waiters flag. 581 */ 582 if (sx->sx_lock & SX_LOCK_SHARED_WAITERS) { 583 queue = SQ_SHARED_QUEUE; 584 x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS); 585 } else 586 queue = SQ_EXCLUSIVE_QUEUE; 587 588 /* Wake up all the waiters for the specific queue. */ 589 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 590 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue", 591 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : 592 "exclusive"); 593 atomic_store_rel_ptr(&sx->sx_lock, x); 594 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, queue); 595 } 596 597 /* 598 * This function represents the so-called 'hard case' for sx_slock 599 * operation. All 'easy case' failures are redirected to this. Note 600 * that ideally this would be a static function, but it needs to be 601 * accessible from at least sx.h. 602 */ 603 void 604 _sx_slock_hard(struct sx *sx, const char *file, int line) 605 { 606 GIANT_DECLARE; 607 #ifdef ADAPTIVE_SX 608 volatile struct thread *owner; 609 #endif 610 uintptr_t x; 611 uint64_t waitstart = 0; 612 int contested = 0; 613 /* 614 * As with rwlocks, we don't make any attempt to try to block 615 * shared locks once there is an exclusive waiter. 616 */ 617 618 for (;;) { 619 x = sx->sx_lock; 620 621 /* 622 * If no other thread has an exclusive lock then try to bump up 623 * the count of sharers. Since we have to preserve the state 624 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the 625 * shared lock loop back and retry. 626 */ 627 if (x & SX_LOCK_SHARED) { 628 MPASS(!(x & SX_LOCK_SHARED_WAITERS)); 629 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, 630 x + SX_ONE_SHARER)) { 631 if (SX_SHARERS(x) == 0) 632 lock_profile_obtain_lock_success( 633 &sx->lock_object, contested, 634 waitstart, file, line); 635 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 636 CTR4(KTR_LOCK, 637 "%s: %p succeed %p -> %p", __func__, 638 sx, (void *)x, 639 (void *)(x + SX_ONE_SHARER)); 640 break; 641 } 642 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 643 &waitstart); 644 645 continue; 646 } 647 648 #ifdef ADAPTIVE_SX 649 /* 650 * If the owner is running on another CPU, spin until 651 * the owner stops running or the state of the lock 652 * changes. 653 */ 654 else if (sx->lock_object.lo_flags & SX_ADAPTIVESPIN) { 655 x = SX_OWNER(x); 656 owner = (struct thread *)x; 657 if (TD_IS_RUNNING(owner)) { 658 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 659 &waitstart); 660 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 661 CTR3(KTR_LOCK, 662 "%s: spinning on %p held by %p", 663 __func__, sx, owner); 664 GIANT_SAVE(); 665 while (SX_OWNER(sx->sx_lock) == x && 666 TD_IS_RUNNING(owner)) 667 cpu_spinwait(); 668 continue; 669 } 670 } 671 #endif 672 else 673 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 674 &waitstart); 675 676 /* 677 * Some other thread already has an exclusive lock, so 678 * start the process of blocking. 679 */ 680 sleepq_lock(&sx->lock_object); 681 x = sx->sx_lock; 682 683 /* 684 * The lock could have been released while we spun. 685 * In this case loop back and retry. 686 */ 687 if (x & SX_LOCK_SHARED) { 688 sleepq_release(&sx->lock_object); 689 continue; 690 } 691 692 #ifdef ADAPTIVE_SX 693 /* 694 * If the owner is running on another CPU, spin until 695 * the owner stops running or the state of the lock 696 * changes. 697 */ 698 if (!(x & SX_LOCK_SHARED) && 699 (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) { 700 owner = (struct thread *)SX_OWNER(x); 701 if (TD_IS_RUNNING(owner)) { 702 sleepq_release(&sx->lock_object); 703 continue; 704 } 705 } 706 #endif 707 708 /* 709 * Try to set the SX_LOCK_SHARED_WAITERS flag. If we 710 * fail to set it drop the sleep queue lock and loop 711 * back. 712 */ 713 if (!(x & SX_LOCK_SHARED_WAITERS)) { 714 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 715 x | SX_LOCK_SHARED_WAITERS)) { 716 sleepq_release(&sx->lock_object); 717 continue; 718 } 719 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 720 CTR2(KTR_LOCK, "%s: %p set shared waiters flag", 721 __func__, sx); 722 } 723 724 /* 725 * Since we have been unable to acquire the shared lock, 726 * we have to sleep. 727 */ 728 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 729 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 730 __func__, sx); 731 732 GIANT_SAVE(); 733 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 734 SLEEPQ_SX, SQ_SHARED_QUEUE); 735 sleepq_wait(&sx->lock_object); 736 737 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 738 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 739 __func__, sx); 740 } 741 742 GIANT_RESTORE(); 743 } 744 745 /* 746 * This function represents the so-called 'hard case' for sx_sunlock 747 * operation. All 'easy case' failures are redirected to this. Note 748 * that ideally this would be a static function, but it needs to be 749 * accessible from at least sx.h. 750 */ 751 void 752 _sx_sunlock_hard(struct sx *sx, const char *file, int line) 753 { 754 uintptr_t x; 755 756 for (;;) { 757 x = sx->sx_lock; 758 759 /* 760 * We should never have sharers while at least one thread 761 * holds a shared lock. 762 */ 763 KASSERT(!(x & SX_LOCK_SHARED_WAITERS), 764 ("%s: waiting sharers", __func__)); 765 766 /* 767 * See if there is more than one shared lock held. If 768 * so, just drop one and return. 769 */ 770 if (SX_SHARERS(x) > 1) { 771 if (atomic_cmpset_ptr(&sx->sx_lock, x, 772 x - SX_ONE_SHARER)) { 773 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 774 CTR4(KTR_LOCK, 775 "%s: %p succeeded %p -> %p", 776 __func__, sx, (void *)x, 777 (void *)(x - SX_ONE_SHARER)); 778 break; 779 } 780 continue; 781 } 782 783 /* 784 * If there aren't any waiters for an exclusive lock, 785 * then try to drop it quickly. 786 */ 787 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 788 MPASS(x == SX_SHARERS_LOCK(1)); 789 if (atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1), 790 SX_LOCK_UNLOCKED)) { 791 lock_profile_release_lock(&sx->lock_object); 792 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 793 CTR2(KTR_LOCK, "%s: %p last succeeded", 794 __func__, sx); 795 break; 796 } 797 continue; 798 } 799 800 /* 801 * At this point, there should just be one sharer with 802 * exclusive waiters. 803 */ 804 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS)); 805 806 lock_profile_release_lock(&sx->lock_object); 807 sleepq_lock(&sx->lock_object); 808 809 /* 810 * Wake up semantic here is quite simple: 811 * Just wake up all the exclusive waiters. 812 * Note that the state of the lock could have changed, 813 * so if it fails loop back and retry. 814 */ 815 if (!atomic_cmpset_ptr(&sx->sx_lock, 816 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS, 817 SX_LOCK_UNLOCKED)) { 818 sleepq_release(&sx->lock_object); 819 continue; 820 } 821 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 822 CTR2(KTR_LOCK, "%s: %p waking up all thread on" 823 "exclusive queue", __func__, sx); 824 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, 825 SQ_EXCLUSIVE_QUEUE); 826 break; 827 } 828 } 829 830 #ifdef INVARIANT_SUPPORT 831 #ifndef INVARIANTS 832 #undef _sx_assert 833 #endif 834 835 /* 836 * In the non-WITNESS case, sx_assert() can only detect that at least 837 * *some* thread owns an slock, but it cannot guarantee that *this* 838 * thread owns an slock. 839 */ 840 void 841 _sx_assert(struct sx *sx, int what, const char *file, int line) 842 { 843 #ifndef WITNESS 844 int slocked = 0; 845 #endif 846 847 if (panicstr != NULL) 848 return; 849 switch (what) { 850 case SX_SLOCKED: 851 case SX_SLOCKED | SX_NOTRECURSED: 852 case SX_SLOCKED | SX_RECURSED: 853 #ifndef WITNESS 854 slocked = 1; 855 /* FALLTHROUGH */ 856 #endif 857 case SX_LOCKED: 858 case SX_LOCKED | SX_NOTRECURSED: 859 case SX_LOCKED | SX_RECURSED: 860 #ifdef WITNESS 861 witness_assert(&sx->lock_object, what, file, line); 862 #else 863 /* 864 * If some other thread has an exclusive lock or we 865 * have one and are asserting a shared lock, fail. 866 * Also, if no one has a lock at all, fail. 867 */ 868 if (sx->sx_lock == SX_LOCK_UNLOCKED || 869 (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked || 870 sx_xholder(sx) != curthread))) 871 panic("Lock %s not %slocked @ %s:%d\n", 872 sx->lock_object.lo_name, slocked ? "share " : "", 873 file, line); 874 875 if (!(sx->sx_lock & SX_LOCK_SHARED)) { 876 if (sx_recursed(sx)) { 877 if (what & SX_NOTRECURSED) 878 panic("Lock %s recursed @ %s:%d\n", 879 sx->lock_object.lo_name, file, 880 line); 881 } else if (what & SX_RECURSED) 882 panic("Lock %s not recursed @ %s:%d\n", 883 sx->lock_object.lo_name, file, line); 884 } 885 #endif 886 break; 887 case SX_XLOCKED: 888 case SX_XLOCKED | SX_NOTRECURSED: 889 case SX_XLOCKED | SX_RECURSED: 890 if (sx_xholder(sx) != curthread) 891 panic("Lock %s not exclusively locked @ %s:%d\n", 892 sx->lock_object.lo_name, file, line); 893 if (sx_recursed(sx)) { 894 if (what & SX_NOTRECURSED) 895 panic("Lock %s recursed @ %s:%d\n", 896 sx->lock_object.lo_name, file, line); 897 } else if (what & SX_RECURSED) 898 panic("Lock %s not recursed @ %s:%d\n", 899 sx->lock_object.lo_name, file, line); 900 break; 901 case SX_UNLOCKED: 902 #ifdef WITNESS 903 witness_assert(&sx->lock_object, what, file, line); 904 #else 905 /* 906 * If we hold an exclusve lock fail. We can't 907 * reliably check to see if we hold a shared lock or 908 * not. 909 */ 910 if (sx_xholder(sx) == curthread) 911 panic("Lock %s exclusively locked @ %s:%d\n", 912 sx->lock_object.lo_name, file, line); 913 #endif 914 break; 915 default: 916 panic("Unknown sx lock assertion: %d @ %s:%d", what, file, 917 line); 918 } 919 } 920 #endif /* INVARIANT_SUPPORT */ 921 922 #ifdef DDB 923 static void 924 db_show_sx(struct lock_object *lock) 925 { 926 struct thread *td; 927 struct sx *sx; 928 929 sx = (struct sx *)lock; 930 931 db_printf(" state: "); 932 if (sx->sx_lock == SX_LOCK_UNLOCKED) 933 db_printf("UNLOCKED\n"); 934 else if (sx->sx_lock == SX_LOCK_DESTROYED) { 935 db_printf("DESTROYED\n"); 936 return; 937 } else if (sx->sx_lock & SX_LOCK_SHARED) 938 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock)); 939 else { 940 td = sx_xholder(sx); 941 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 942 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 943 if (sx_recursed(sx)) 944 db_printf(" recursed: %d\n", sx->sx_recurse); 945 } 946 947 db_printf(" waiters: "); 948 switch(sx->sx_lock & 949 (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) { 950 case SX_LOCK_SHARED_WAITERS: 951 db_printf("shared\n"); 952 break; 953 case SX_LOCK_EXCLUSIVE_WAITERS: 954 db_printf("exclusive\n"); 955 break; 956 case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS: 957 db_printf("exclusive and shared\n"); 958 break; 959 default: 960 db_printf("none\n"); 961 } 962 } 963 964 /* 965 * Check to see if a thread that is blocked on a sleep queue is actually 966 * blocked on an sx lock. If so, output some details and return true. 967 * If the lock has an exclusive owner, return that in *ownerp. 968 */ 969 int 970 sx_chain(struct thread *td, struct thread **ownerp) 971 { 972 struct sx *sx; 973 974 /* 975 * Check to see if this thread is blocked on an sx lock. 976 * First, we check the lock class. If that is ok, then we 977 * compare the lock name against the wait message. 978 */ 979 sx = td->td_wchan; 980 if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx || 981 sx->lock_object.lo_name != td->td_wmesg) 982 return (0); 983 984 /* We think we have an sx lock, so output some details. */ 985 db_printf("blocked on sx \"%s\" ", td->td_wmesg); 986 *ownerp = sx_xholder(sx); 987 if (sx->sx_lock & SX_LOCK_SHARED) 988 db_printf("SLOCK (count %ju)\n", 989 (uintmax_t)SX_SHARERS(sx->sx_lock)); 990 else 991 db_printf("XLOCK\n"); 992 return (1); 993 } 994 #endif 995