1 /*- 2 * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include "opt_ddb.h" 30 #include "opt_kdtrace.h" 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/ktr.h> 37 #include <sys/lock.h> 38 #include <sys/lock_profile.h> 39 #include <sys/lockmgr.h> 40 #include <sys/mutex.h> 41 #include <sys/proc.h> 42 #include <sys/sleepqueue.h> 43 #ifdef DEBUG_LOCKS 44 #include <sys/stack.h> 45 #endif 46 #include <sys/systm.h> 47 48 #include <machine/cpu.h> 49 50 #ifdef DDB 51 #include <ddb/ddb.h> 52 #endif 53 54 CTASSERT((LK_NOSHARE & LO_CLASSFLAGS) == LK_NOSHARE); 55 56 #define SQ_EXCLUSIVE_QUEUE 0 57 #define SQ_SHARED_QUEUE 1 58 59 #ifndef INVARIANTS 60 #define _lockmgr_assert(lk, what, file, line) 61 #define TD_LOCKS_INC(td) 62 #define TD_LOCKS_DEC(td) 63 #else 64 #define TD_LOCKS_INC(td) ((td)->td_locks++) 65 #define TD_LOCKS_DEC(td) ((td)->td_locks--) 66 #endif 67 #define TD_SLOCKS_INC(td) ((td)->td_lk_slocks++) 68 #define TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--) 69 70 #ifndef DEBUG_LOCKS 71 #define STACK_PRINT(lk) 72 #define STACK_SAVE(lk) 73 #define STACK_ZERO(lk) 74 #else 75 #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack) 76 #define STACK_SAVE(lk) stack_save(&(lk)->lk_stack) 77 #define STACK_ZERO(lk) stack_zero(&(lk)->lk_stack) 78 #endif 79 80 #define LOCK_LOG2(lk, string, arg1, arg2) \ 81 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \ 82 CTR2(KTR_LOCK, (string), (arg1), (arg2)) 83 #define LOCK_LOG3(lk, string, arg1, arg2, arg3) \ 84 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \ 85 CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3)) 86 87 #define GIANT_DECLARE \ 88 int _i = 0; \ 89 WITNESS_SAVE_DECL(Giant) 90 #define GIANT_RESTORE() do { \ 91 if (_i > 0) { \ 92 while (_i--) \ 93 mtx_lock(&Giant); \ 94 WITNESS_RESTORE(&Giant.lock_object, Giant); \ 95 } \ 96 } while (0) 97 #define GIANT_SAVE() do { \ 98 if (mtx_owned(&Giant)) { \ 99 WITNESS_SAVE(&Giant.lock_object, Giant); \ 100 while (mtx_owned(&Giant)) { \ 101 _i++; \ 102 mtx_unlock(&Giant); \ 103 } \ 104 } \ 105 } while (0) 106 107 #define LK_CAN_SHARE(x) \ 108 (((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 || \ 109 curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT))) 110 #define LK_TRYOP(x) \ 111 ((x) & LK_NOWAIT) 112 113 #define LK_CAN_WITNESS(x) \ 114 (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x)) 115 #define LK_TRYWIT(x) \ 116 (LK_TRYOP(x) ? LOP_TRYLOCK : 0) 117 118 #define lockmgr_disowned(lk) \ 119 (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC) 120 121 #define lockmgr_xlocked(lk) \ 122 (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread) 123 124 static void assert_lockmgr(struct lock_object *lock, int how); 125 #ifdef DDB 126 static void db_show_lockmgr(struct lock_object *lock); 127 #endif 128 static void lock_lockmgr(struct lock_object *lock, int how); 129 #ifdef KDTRACE_HOOKS 130 static int owner_lockmgr(struct lock_object *lock, struct thread **owner); 131 #endif 132 static int unlock_lockmgr(struct lock_object *lock); 133 134 struct lock_class lock_class_lockmgr = { 135 .lc_name = "lockmgr", 136 .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE, 137 .lc_assert = assert_lockmgr, 138 #ifdef DDB 139 .lc_ddb_show = db_show_lockmgr, 140 #endif 141 .lc_lock = lock_lockmgr, 142 .lc_unlock = unlock_lockmgr, 143 #ifdef KDTRACE_HOOKS 144 .lc_owner = owner_lockmgr, 145 #endif 146 }; 147 148 static __inline struct thread * 149 lockmgr_xholder(struct lock *lk) 150 { 151 uintptr_t x; 152 153 x = lk->lk_lock; 154 return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x)); 155 } 156 157 /* 158 * It assumes sleepq_lock held and returns with this one unheld. 159 * It also assumes the generic interlock is sane and previously checked. 160 * If LK_INTERLOCK is specified the interlock is not reacquired after the 161 * sleep. 162 */ 163 static __inline int 164 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk, 165 const char *wmesg, int pri, int timo, int queue) 166 { 167 GIANT_DECLARE; 168 struct lock_class *class; 169 int catch, error; 170 171 class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL; 172 catch = pri & PCATCH; 173 pri &= PRIMASK; 174 error = 0; 175 176 LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk, 177 (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared"); 178 179 if (flags & LK_INTERLOCK) 180 class->lc_unlock(ilk); 181 GIANT_SAVE(); 182 sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ? 183 SLEEPQ_INTERRUPTIBLE : 0), queue); 184 if ((flags & LK_TIMELOCK) && timo) 185 sleepq_set_timeout(&lk->lock_object, timo); 186 187 /* 188 * Decisional switch for real sleeping. 189 */ 190 if ((flags & LK_TIMELOCK) && timo && catch) 191 error = sleepq_timedwait_sig(&lk->lock_object, pri); 192 else if ((flags & LK_TIMELOCK) && timo) 193 error = sleepq_timedwait(&lk->lock_object, pri); 194 else if (catch) 195 error = sleepq_wait_sig(&lk->lock_object, pri); 196 else 197 sleepq_wait(&lk->lock_object, pri); 198 GIANT_RESTORE(); 199 if ((flags & LK_SLEEPFAIL) && error == 0) 200 error = ENOLCK; 201 202 return (error); 203 } 204 205 static __inline int 206 wakeupshlk(struct lock *lk, const char *file, int line) 207 { 208 uintptr_t v, x; 209 int queue, wakeup_swapper; 210 211 TD_LOCKS_DEC(curthread); 212 TD_SLOCKS_DEC(curthread); 213 WITNESS_UNLOCK(&lk->lock_object, 0, file, line); 214 LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line); 215 216 wakeup_swapper = 0; 217 for (;;) { 218 x = lk->lk_lock; 219 220 /* 221 * If there is more than one shared lock held, just drop one 222 * and return. 223 */ 224 if (LK_SHARERS(x) > 1) { 225 if (atomic_cmpset_ptr(&lk->lk_lock, x, 226 x - LK_ONE_SHARER)) 227 break; 228 continue; 229 } 230 231 /* 232 * If there are not waiters on the exclusive queue, drop the 233 * lock quickly. 234 */ 235 if ((x & LK_ALL_WAITERS) == 0) { 236 MPASS(x == LK_SHARERS_LOCK(1)); 237 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1), 238 LK_UNLOCKED)) 239 break; 240 continue; 241 } 242 243 /* 244 * We should have a sharer with waiters, so enter the hard 245 * path in order to handle wakeups correctly. 246 */ 247 sleepq_lock(&lk->lock_object); 248 x = lk->lk_lock & LK_ALL_WAITERS; 249 v = LK_UNLOCKED; 250 251 /* 252 * If the lock has exclusive waiters, give them preference in 253 * order to avoid deadlock with shared runners up. 254 */ 255 if (x & LK_EXCLUSIVE_WAITERS) { 256 queue = SQ_EXCLUSIVE_QUEUE; 257 v |= (x & LK_SHARED_WAITERS); 258 } else { 259 MPASS(x == LK_SHARED_WAITERS); 260 queue = SQ_SHARED_QUEUE; 261 } 262 263 if (!atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x, 264 v)) { 265 sleepq_release(&lk->lock_object); 266 continue; 267 } 268 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", 269 __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : 270 "exclusive"); 271 wakeup_swapper = sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 272 0, queue); 273 sleepq_release(&lk->lock_object); 274 break; 275 } 276 277 lock_profile_release_lock(&lk->lock_object); 278 return (wakeup_swapper); 279 } 280 281 static void 282 assert_lockmgr(struct lock_object *lock, int what) 283 { 284 285 panic("lockmgr locks do not support assertions"); 286 } 287 288 static void 289 lock_lockmgr(struct lock_object *lock, int how) 290 { 291 292 panic("lockmgr locks do not support sleep interlocking"); 293 } 294 295 static int 296 unlock_lockmgr(struct lock_object *lock) 297 { 298 299 panic("lockmgr locks do not support sleep interlocking"); 300 } 301 302 #ifdef KDTRACE_HOOKS 303 static int 304 owner_lockmgr(struct lock_object *lock, struct thread **owner) 305 { 306 307 panic("lockmgr locks do not support owner inquiring"); 308 } 309 #endif 310 311 void 312 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags) 313 { 314 int iflags; 315 316 MPASS((flags & ~LK_INIT_MASK) == 0); 317 318 iflags = LO_SLEEPABLE | LO_UPGRADABLE; 319 if (flags & LK_CANRECURSE) 320 iflags |= LO_RECURSABLE; 321 if ((flags & LK_NODUP) == 0) 322 iflags |= LO_DUPOK; 323 if (flags & LK_NOPROFILE) 324 iflags |= LO_NOPROFILE; 325 if ((flags & LK_NOWITNESS) == 0) 326 iflags |= LO_WITNESS; 327 if (flags & LK_QUIET) 328 iflags |= LO_QUIET; 329 iflags |= flags & LK_NOSHARE; 330 331 lk->lk_lock = LK_UNLOCKED; 332 lk->lk_recurse = 0; 333 lk->lk_timo = timo; 334 lk->lk_pri = pri; 335 lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags); 336 STACK_ZERO(lk); 337 } 338 339 void 340 lockdestroy(struct lock *lk) 341 { 342 343 KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held")); 344 KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed")); 345 lock_destroy(&lk->lock_object); 346 } 347 348 int 349 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, 350 const char *wmesg, int pri, int timo, const char *file, int line) 351 { 352 GIANT_DECLARE; 353 struct lock_class *class; 354 const char *iwmesg; 355 uintptr_t tid, v, x; 356 u_int op; 357 int error, ipri, itimo, queue, wakeup_swapper; 358 #ifdef LOCK_PROFILING 359 uint64_t waittime = 0; 360 int contested = 0; 361 #endif 362 363 error = 0; 364 tid = (uintptr_t)curthread; 365 op = (flags & LK_TYPE_MASK); 366 iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg; 367 ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri; 368 itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo; 369 370 MPASS((flags & ~LK_TOTAL_MASK) == 0); 371 KASSERT((op & (op - 1)) == 0, 372 ("%s: Invalid requested operation @ %s:%d", __func__, file, line)); 373 KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 || 374 (op != LK_DOWNGRADE && op != LK_RELEASE), 375 ("%s: Invalid flags in regard of the operation desired @ %s:%d", 376 __func__, file, line)); 377 KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL, 378 ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d", 379 __func__, file, line)); 380 381 class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL; 382 if (panicstr != NULL) { 383 if (flags & LK_INTERLOCK) 384 class->lc_unlock(ilk); 385 return (0); 386 } 387 388 if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE)) 389 op = LK_EXCLUSIVE; 390 391 wakeup_swapper = 0; 392 switch (op) { 393 case LK_SHARED: 394 if (LK_CAN_WITNESS(flags)) 395 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, 396 file, line, ilk); 397 for (;;) { 398 x = lk->lk_lock; 399 400 /* 401 * If no other thread has an exclusive lock, or 402 * no exclusive waiter is present, bump the count of 403 * sharers. Since we have to preserve the state of 404 * waiters, if we fail to acquire the shared lock 405 * loop back and retry. 406 */ 407 if (LK_CAN_SHARE(x)) { 408 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, 409 x + LK_ONE_SHARER)) 410 break; 411 continue; 412 } 413 lock_profile_obtain_lock_failed(&lk->lock_object, 414 &contested, &waittime); 415 416 /* 417 * If the lock is already held by curthread in 418 * exclusive way avoid a deadlock. 419 */ 420 if (LK_HOLDER(x) == tid) { 421 LOCK_LOG2(lk, 422 "%s: %p already held in exclusive mode", 423 __func__, lk); 424 error = EDEADLK; 425 break; 426 } 427 428 /* 429 * If the lock is expected to not sleep just give up 430 * and return. 431 */ 432 if (LK_TRYOP(flags)) { 433 LOCK_LOG2(lk, "%s: %p fails the try operation", 434 __func__, lk); 435 error = EBUSY; 436 break; 437 } 438 439 /* 440 * Acquire the sleepqueue chain lock because we 441 * probabilly will need to manipulate waiters flags. 442 */ 443 sleepq_lock(&lk->lock_object); 444 x = lk->lk_lock; 445 446 /* 447 * if the lock can be acquired in shared mode, try 448 * again. 449 */ 450 if (LK_CAN_SHARE(x)) { 451 sleepq_release(&lk->lock_object); 452 continue; 453 } 454 455 /* 456 * Try to set the LK_SHARED_WAITERS flag. If we fail, 457 * loop back and retry. 458 */ 459 if ((x & LK_SHARED_WAITERS) == 0) { 460 if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x, 461 x | LK_SHARED_WAITERS)) { 462 sleepq_release(&lk->lock_object); 463 continue; 464 } 465 LOCK_LOG2(lk, "%s: %p set shared waiters flag", 466 __func__, lk); 467 } 468 469 /* 470 * As far as we have been unable to acquire the 471 * shared lock and the shared waiters flag is set, 472 * we will sleep. 473 */ 474 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, 475 SQ_SHARED_QUEUE); 476 flags &= ~LK_INTERLOCK; 477 if (error) { 478 LOCK_LOG3(lk, 479 "%s: interrupted sleep for %p with %d", 480 __func__, lk, error); 481 break; 482 } 483 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 484 __func__, lk); 485 } 486 if (error == 0) { 487 lock_profile_obtain_lock_success(&lk->lock_object, 488 contested, waittime, file, line); 489 LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, 490 line); 491 WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, 492 line); 493 TD_LOCKS_INC(curthread); 494 TD_SLOCKS_INC(curthread); 495 STACK_SAVE(lk); 496 } 497 break; 498 case LK_UPGRADE: 499 _lockmgr_assert(lk, KA_SLOCKED, file, line); 500 x = lk->lk_lock & LK_ALL_WAITERS; 501 502 /* 503 * Try to switch from one shared lock to an exclusive one. 504 * We need to preserve waiters flags during the operation. 505 */ 506 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x, 507 tid | x)) { 508 LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, 509 line); 510 WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | 511 LK_TRYWIT(flags), file, line); 512 TD_SLOCKS_DEC(curthread); 513 break; 514 } 515 516 /* 517 * We have been unable to succeed in upgrading, so just 518 * give up the shared lock. 519 */ 520 wakeup_swapper |= wakeupshlk(lk, file, line); 521 522 /* FALLTHROUGH */ 523 case LK_EXCLUSIVE: 524 if (LK_CAN_WITNESS(flags)) 525 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER | 526 LOP_EXCLUSIVE, file, line, ilk); 527 528 /* 529 * If curthread already holds the lock and this one is 530 * allowed to recurse, simply recurse on it. 531 */ 532 if (lockmgr_xlocked(lk)) { 533 if ((flags & LK_CANRECURSE) == 0 && 534 (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) { 535 536 /* 537 * If the lock is expected to not panic just 538 * give up and return. 539 */ 540 if (LK_TRYOP(flags)) { 541 LOCK_LOG2(lk, 542 "%s: %p fails the try operation", 543 __func__, lk); 544 error = EBUSY; 545 break; 546 } 547 if (flags & LK_INTERLOCK) 548 class->lc_unlock(ilk); 549 panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n", 550 __func__, iwmesg, file, line); 551 } 552 lk->lk_recurse++; 553 LOCK_LOG2(lk, "%s: %p recursing", __func__, lk); 554 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, 555 lk->lk_recurse, file, line); 556 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | 557 LK_TRYWIT(flags), file, line); 558 TD_LOCKS_INC(curthread); 559 break; 560 } 561 562 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, 563 tid)) { 564 lock_profile_obtain_lock_failed(&lk->lock_object, 565 &contested, &waittime); 566 567 /* 568 * If the lock is expected to not sleep just give up 569 * and return. 570 */ 571 if (LK_TRYOP(flags)) { 572 LOCK_LOG2(lk, "%s: %p fails the try operation", 573 __func__, lk); 574 error = EBUSY; 575 break; 576 } 577 578 /* 579 * Acquire the sleepqueue chain lock because we 580 * probabilly will need to manipulate waiters flags. 581 */ 582 sleepq_lock(&lk->lock_object); 583 x = lk->lk_lock; 584 v = x & LK_ALL_WAITERS; 585 586 /* 587 * if the lock has been released while we spun on 588 * the sleepqueue chain lock just try again. 589 */ 590 if (x == LK_UNLOCKED) { 591 sleepq_release(&lk->lock_object); 592 continue; 593 } 594 595 /* 596 * The lock can be in the state where there is a 597 * pending queue of waiters, but still no owner. 598 * This happens when the lock is contested and an 599 * owner is going to claim the lock. 600 * If curthread is the one successfully acquiring it 601 * claim lock ownership and return, preserving waiters 602 * flags. 603 */ 604 if (x == (LK_UNLOCKED | v)) { 605 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, 606 tid | v)) { 607 sleepq_release(&lk->lock_object); 608 LOCK_LOG2(lk, 609 "%s: %p claimed by a new writer", 610 __func__, lk); 611 break; 612 } 613 sleepq_release(&lk->lock_object); 614 continue; 615 } 616 617 /* 618 * Try to set the LK_EXCLUSIVE_WAITERS flag. If we 619 * fail, loop back and retry. 620 */ 621 if ((x & LK_EXCLUSIVE_WAITERS) == 0) { 622 if (!atomic_cmpset_ptr(&lk->lk_lock, x, 623 x | LK_EXCLUSIVE_WAITERS)) { 624 sleepq_release(&lk->lock_object); 625 continue; 626 } 627 LOCK_LOG2(lk, "%s: %p set excl waiters flag", 628 __func__, lk); 629 } 630 631 /* 632 * As far as we have been unable to acquire the 633 * exclusive lock and the exclusive waiters flag 634 * is set, we will sleep. 635 */ 636 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, 637 SQ_EXCLUSIVE_QUEUE); 638 flags &= ~LK_INTERLOCK; 639 if (error) { 640 LOCK_LOG3(lk, 641 "%s: interrupted sleep for %p with %d", 642 __func__, lk, error); 643 break; 644 } 645 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 646 __func__, lk); 647 } 648 if (error == 0) { 649 lock_profile_obtain_lock_success(&lk->lock_object, 650 contested, waittime, file, line); 651 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, 652 lk->lk_recurse, file, line); 653 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | 654 LK_TRYWIT(flags), file, line); 655 TD_LOCKS_INC(curthread); 656 STACK_SAVE(lk); 657 } 658 break; 659 case LK_DOWNGRADE: 660 _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line); 661 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line); 662 WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line); 663 TD_SLOCKS_INC(curthread); 664 665 /* 666 * In order to preserve waiters flags, just spin. 667 */ 668 for (;;) { 669 x = lk->lk_lock & LK_ALL_WAITERS; 670 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, 671 LK_SHARERS_LOCK(1) | x)) 672 break; 673 cpu_spinwait(); 674 } 675 break; 676 case LK_RELEASE: 677 _lockmgr_assert(lk, KA_LOCKED, file, line); 678 x = lk->lk_lock; 679 680 if ((x & LK_SHARE) == 0) { 681 682 /* 683 * As first option, treact the lock as if it has not 684 * any waiter. 685 * Fix-up the tid var if the lock has been disowned. 686 */ 687 if (LK_HOLDER(x) == LK_KERNPROC) 688 tid = LK_KERNPROC; 689 else { 690 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, 691 file, line); 692 TD_LOCKS_DEC(curthread); 693 } 694 LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, 695 lk->lk_recurse, file, line); 696 697 /* 698 * The lock is held in exclusive mode. 699 * If the lock is recursed also, then unrecurse it. 700 */ 701 if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) { 702 LOCK_LOG2(lk, "%s: %p unrecursing", __func__, 703 lk); 704 lk->lk_recurse--; 705 break; 706 } 707 if (tid != LK_KERNPROC) 708 lock_profile_release_lock(&lk->lock_object); 709 710 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid, 711 LK_UNLOCKED)) 712 break; 713 714 sleepq_lock(&lk->lock_object); 715 x = lk->lk_lock & LK_ALL_WAITERS; 716 v = LK_UNLOCKED; 717 718 /* 719 * If the lock has exclusive waiters, give them 720 * preference in order to avoid deadlock with 721 * shared runners up. 722 */ 723 if (x & LK_EXCLUSIVE_WAITERS) { 724 queue = SQ_EXCLUSIVE_QUEUE; 725 v |= (x & LK_SHARED_WAITERS); 726 } else { 727 MPASS(x == LK_SHARED_WAITERS); 728 queue = SQ_SHARED_QUEUE; 729 } 730 731 LOCK_LOG3(lk, 732 "%s: %p waking up threads on the %s queue", 733 __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : 734 "exclusive"); 735 atomic_store_rel_ptr(&lk->lk_lock, v); 736 wakeup_swapper = sleepq_broadcast(&lk->lock_object, 737 SLEEPQ_LK, 0, queue); 738 sleepq_release(&lk->lock_object); 739 break; 740 } else 741 wakeup_swapper = wakeupshlk(lk, file, line); 742 break; 743 case LK_DRAIN: 744 if (LK_CAN_WITNESS(flags)) 745 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER | 746 LOP_EXCLUSIVE, file, line, ilk); 747 748 /* 749 * Trying to drain a lock we already own will result in a 750 * deadlock. 751 */ 752 if (lockmgr_xlocked(lk)) { 753 if (flags & LK_INTERLOCK) 754 class->lc_unlock(ilk); 755 panic("%s: draining %s with the lock held @ %s:%d\n", 756 __func__, iwmesg, file, line); 757 } 758 759 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { 760 lock_profile_obtain_lock_failed(&lk->lock_object, 761 &contested, &waittime); 762 763 /* 764 * If the lock is expected to not sleep just give up 765 * and return. 766 */ 767 if (LK_TRYOP(flags)) { 768 LOCK_LOG2(lk, "%s: %p fails the try operation", 769 __func__, lk); 770 error = EBUSY; 771 break; 772 } 773 774 /* 775 * Acquire the sleepqueue chain lock because we 776 * probabilly will need to manipulate waiters flags. 777 */ 778 sleepq_lock(&lk->lock_object); 779 x = lk->lk_lock; 780 v = x & LK_ALL_WAITERS; 781 782 /* 783 * if the lock has been released while we spun on 784 * the sleepqueue chain lock just try again. 785 */ 786 if (x == LK_UNLOCKED) { 787 sleepq_release(&lk->lock_object); 788 continue; 789 } 790 791 if (x == (LK_UNLOCKED | v)) { 792 v = x; 793 if (v & LK_EXCLUSIVE_WAITERS) { 794 queue = SQ_EXCLUSIVE_QUEUE; 795 v &= ~LK_EXCLUSIVE_WAITERS; 796 } else { 797 MPASS(v & LK_SHARED_WAITERS); 798 queue = SQ_SHARED_QUEUE; 799 v &= ~LK_SHARED_WAITERS; 800 } 801 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) { 802 sleepq_release(&lk->lock_object); 803 continue; 804 } 805 LOCK_LOG3(lk, 806 "%s: %p waking up all threads on the %s queue", 807 __func__, lk, queue == SQ_SHARED_QUEUE ? 808 "shared" : "exclusive"); 809 wakeup_swapper |= sleepq_broadcast( 810 &lk->lock_object, SLEEPQ_LK, 0, queue); 811 812 /* 813 * If shared waiters have been woken up we need 814 * to wait for one of them to acquire the lock 815 * before to set the exclusive waiters in 816 * order to avoid a deadlock. 817 */ 818 if (queue == SQ_SHARED_QUEUE) { 819 for (v = lk->lk_lock; 820 (v & LK_SHARE) && !LK_SHARERS(v); 821 v = lk->lk_lock) 822 cpu_spinwait(); 823 } 824 } 825 826 /* 827 * Try to set the LK_EXCLUSIVE_WAITERS flag. If we 828 * fail, loop back and retry. 829 */ 830 if ((x & LK_EXCLUSIVE_WAITERS) == 0) { 831 if (!atomic_cmpset_ptr(&lk->lk_lock, x, 832 x | LK_EXCLUSIVE_WAITERS)) { 833 sleepq_release(&lk->lock_object); 834 continue; 835 } 836 LOCK_LOG2(lk, "%s: %p set drain waiters flag", 837 __func__, lk); 838 } 839 840 /* 841 * As far as we have been unable to acquire the 842 * exclusive lock and the exclusive waiters flag 843 * is set, we will sleep. 844 */ 845 if (flags & LK_INTERLOCK) { 846 class->lc_unlock(ilk); 847 flags &= ~LK_INTERLOCK; 848 } 849 GIANT_SAVE(); 850 sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK, 851 SQ_EXCLUSIVE_QUEUE); 852 sleepq_wait(&lk->lock_object, ipri & PRIMASK); 853 GIANT_RESTORE(); 854 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 855 __func__, lk); 856 } 857 858 if (error == 0) { 859 lock_profile_obtain_lock_success(&lk->lock_object, 860 contested, waittime, file, line); 861 LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0, 862 lk->lk_recurse, file, line); 863 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | 864 LK_TRYWIT(flags), file, line); 865 TD_LOCKS_INC(curthread); 866 STACK_SAVE(lk); 867 } 868 break; 869 default: 870 if (flags & LK_INTERLOCK) 871 class->lc_unlock(ilk); 872 panic("%s: unknown lockmgr request 0x%x\n", __func__, op); 873 } 874 875 if (flags & LK_INTERLOCK) 876 class->lc_unlock(ilk); 877 if (wakeup_swapper) 878 kick_proc0(); 879 880 return (error); 881 } 882 883 void 884 _lockmgr_disown(struct lock *lk, const char *file, int line) 885 { 886 uintptr_t tid, x; 887 888 tid = (uintptr_t)curthread; 889 _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line); 890 891 /* 892 * If the owner is already LK_KERNPROC just skip the whole operation. 893 */ 894 if (LK_HOLDER(lk->lk_lock) != tid) 895 return; 896 lock_profile_release_lock(&lk->lock_object); 897 LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line); 898 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); 899 TD_LOCKS_DEC(curthread); 900 901 /* 902 * In order to preserve waiters flags, just spin. 903 */ 904 for (;;) { 905 x = lk->lk_lock & LK_ALL_WAITERS; 906 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, 907 LK_KERNPROC | x)) 908 return; 909 cpu_spinwait(); 910 } 911 } 912 913 void 914 lockmgr_printinfo(struct lock *lk) 915 { 916 struct thread *td; 917 uintptr_t x; 918 919 if (lk->lk_lock == LK_UNLOCKED) 920 printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name); 921 else if (lk->lk_lock & LK_SHARE) 922 printf("lock type %s: SHARED (count %ju)\n", 923 lk->lock_object.lo_name, 924 (uintmax_t)LK_SHARERS(lk->lk_lock)); 925 else { 926 td = lockmgr_xholder(lk); 927 printf("lock type %s: EXCL by thread %p (pid %d)\n", 928 lk->lock_object.lo_name, td, td->td_proc->p_pid); 929 } 930 931 x = lk->lk_lock; 932 if (x & LK_EXCLUSIVE_WAITERS) 933 printf(" with exclusive waiters pending\n"); 934 if (x & LK_SHARED_WAITERS) 935 printf(" with shared waiters pending\n"); 936 937 STACK_PRINT(lk); 938 } 939 940 int 941 lockstatus(struct lock *lk) 942 { 943 uintptr_t v, x; 944 int ret; 945 946 ret = LK_SHARED; 947 x = lk->lk_lock; 948 v = LK_HOLDER(x); 949 950 if ((x & LK_SHARE) == 0) { 951 if (v == (uintptr_t)curthread || v == LK_KERNPROC) 952 ret = LK_EXCLUSIVE; 953 else 954 ret = LK_EXCLOTHER; 955 } else if (x == LK_UNLOCKED) 956 ret = 0; 957 958 return (ret); 959 } 960 961 #ifdef INVARIANT_SUPPORT 962 #ifndef INVARIANTS 963 #undef _lockmgr_assert 964 #endif 965 966 void 967 _lockmgr_assert(struct lock *lk, int what, const char *file, int line) 968 { 969 int slocked = 0; 970 971 if (panicstr != NULL) 972 return; 973 switch (what) { 974 case KA_SLOCKED: 975 case KA_SLOCKED | KA_NOTRECURSED: 976 case KA_SLOCKED | KA_RECURSED: 977 slocked = 1; 978 case KA_LOCKED: 979 case KA_LOCKED | KA_NOTRECURSED: 980 case KA_LOCKED | KA_RECURSED: 981 #ifdef WITNESS 982 983 /* 984 * We cannot trust WITNESS if the lock is held in exclusive 985 * mode and a call to lockmgr_disown() happened. 986 * Workaround this skipping the check if the lock is held in 987 * exclusive mode even for the KA_LOCKED case. 988 */ 989 if (slocked || (lk->lk_lock & LK_SHARE)) { 990 witness_assert(&lk->lock_object, what, file, line); 991 break; 992 } 993 #endif 994 if (lk->lk_lock == LK_UNLOCKED || 995 ((lk->lk_lock & LK_SHARE) == 0 && (slocked || 996 (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))))) 997 panic("Lock %s not %slocked @ %s:%d\n", 998 lk->lock_object.lo_name, slocked ? "share" : "", 999 file, line); 1000 1001 if ((lk->lk_lock & LK_SHARE) == 0) { 1002 if (lockmgr_recursed(lk)) { 1003 if (what & KA_NOTRECURSED) 1004 panic("Lock %s recursed @ %s:%d\n", 1005 lk->lock_object.lo_name, file, 1006 line); 1007 } else if (what & KA_RECURSED) 1008 panic("Lock %s not recursed @ %s:%d\n", 1009 lk->lock_object.lo_name, file, line); 1010 } 1011 break; 1012 case KA_XLOCKED: 1013 case KA_XLOCKED | KA_NOTRECURSED: 1014 case KA_XLOCKED | KA_RECURSED: 1015 if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)) 1016 panic("Lock %s not exclusively locked @ %s:%d\n", 1017 lk->lock_object.lo_name, file, line); 1018 if (lockmgr_recursed(lk)) { 1019 if (what & KA_NOTRECURSED) 1020 panic("Lock %s recursed @ %s:%d\n", 1021 lk->lock_object.lo_name, file, line); 1022 } else if (what & KA_RECURSED) 1023 panic("Lock %s not recursed @ %s:%d\n", 1024 lk->lock_object.lo_name, file, line); 1025 break; 1026 case KA_UNLOCKED: 1027 if (lockmgr_xlocked(lk) || lockmgr_disowned(lk)) 1028 panic("Lock %s exclusively locked @ %s:%d\n", 1029 lk->lock_object.lo_name, file, line); 1030 break; 1031 default: 1032 panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file, 1033 line); 1034 } 1035 } 1036 #endif 1037 1038 #ifdef DDB 1039 int 1040 lockmgr_chain(struct thread *td, struct thread **ownerp) 1041 { 1042 struct lock *lk; 1043 1044 lk = td->td_wchan; 1045 1046 if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr) 1047 return (0); 1048 db_printf("blocked on lockmgr %s", lk->lock_object.lo_name); 1049 if (lk->lk_lock & LK_SHARE) 1050 db_printf("SHARED (count %ju)\n", 1051 (uintmax_t)LK_SHARERS(lk->lk_lock)); 1052 else 1053 db_printf("EXCL\n"); 1054 *ownerp = lockmgr_xholder(lk); 1055 1056 return (1); 1057 } 1058 1059 static void 1060 db_show_lockmgr(struct lock_object *lock) 1061 { 1062 struct thread *td; 1063 struct lock *lk; 1064 1065 lk = (struct lock *)lock; 1066 1067 db_printf(" state: "); 1068 if (lk->lk_lock == LK_UNLOCKED) 1069 db_printf("UNLOCKED\n"); 1070 else if (lk->lk_lock & LK_SHARE) 1071 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock)); 1072 else { 1073 td = lockmgr_xholder(lk); 1074 if (td == (struct thread *)LK_KERNPROC) 1075 db_printf("XLOCK: LK_KERNPROC\n"); 1076 else 1077 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1078 td->td_tid, td->td_proc->p_pid, 1079 td->td_proc->p_comm); 1080 if (lockmgr_recursed(lk)) 1081 db_printf(" recursed: %d\n", lk->lk_recurse); 1082 } 1083 db_printf(" waiters: "); 1084 switch (lk->lk_lock & LK_ALL_WAITERS) { 1085 case LK_SHARED_WAITERS: 1086 db_printf("shared\n"); 1087 break; 1088 case LK_EXCLUSIVE_WAITERS: 1089 db_printf("exclusive\n"); 1090 break; 1091 case LK_ALL_WAITERS: 1092 db_printf("shared and exclusive\n"); 1093 break; 1094 default: 1095 db_printf("none\n"); 1096 } 1097 } 1098 #endif 1099