1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2009 Konstantin Belousov <kib@FreeBSD.org> 5 * Copyright (c) 2023 The FreeBSD Foundation 6 * 7 * Portions of this software were developed by 8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 9 * the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice unmodified, this list of conditions, and the following 16 * disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 26 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/param.h> 34 #include <sys/kassert.h> 35 #include <sys/kernel.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/proc.h> 39 #include <sys/rangelock.h> 40 #include <sys/sleepqueue.h> 41 #include <sys/smr.h> 42 #include <sys/sysctl.h> 43 44 #include <vm/uma.h> 45 46 /* 47 * Immediately after initialization (subject to 'rangelock_cheat' 48 * below), and until a request comes that conflicts with granted ones 49 * based on type, rangelocks serve requests in the "cheating" mode. 50 * In this mode, a rangelock behaves like a sxlock, as if each request 51 * covered the whole range of the protected object. On receiving a 52 * conflicting request (any request while a write request is 53 * effective, or any write request while some read ones are 54 * effective), all requests granted in "cheating" mode are drained, 55 * and the rangelock then switches to effectively keeping track of the 56 * precise range of each new request. 57 * 58 * Normal sx implementation is not used to not bloat structures (most 59 * important, vnodes) which embeds rangelocks. 60 * 61 * The cheating greatly helps very common pattern where file is first 62 * written single-threaded, and then read by many processes. 63 * 64 * Lock is in cheat mode when RL_CHEAT_CHEATING bit is set in the 65 * lock->head. Special cookies are returned in this mode, and 66 * trylocks are same as normal locks but do not drain. 67 */ 68 69 static int rangelock_cheat = 1; 70 SYSCTL_INT(_debug, OID_AUTO, rangelock_cheat, CTLFLAG_RWTUN, 71 &rangelock_cheat, 0, 72 ""); 73 74 #define RL_CHEAT_MASK 0x7 75 #define RL_CHEAT_CHEATING 0x1 76 /* #define RL_CHEAT_RLOCKED 0x0 */ 77 #define RL_CHEAT_WLOCKED 0x2 78 #define RL_CHEAT_DRAINING 0x4 79 80 #define RL_CHEAT_READER 0x8 81 82 #define RL_RET_CHEAT_RLOCKED 0x1100 83 #define RL_RET_CHEAT_WLOCKED 0x2200 84 85 static bool 86 rangelock_cheat_lock(struct rangelock *lock, int locktype, bool trylock, 87 void **cookie) 88 { 89 uintptr_t v, x; 90 91 v = (uintptr_t)atomic_load_ptr(&lock->head); 92 if ((v & RL_CHEAT_CHEATING) == 0) 93 return (false); 94 if ((v & RL_CHEAT_DRAINING) != 0) { 95 drain: 96 if (trylock) { 97 *cookie = NULL; 98 return (true); 99 } 100 sleepq_lock(&lock->head); 101 drain1: 102 DROP_GIANT(); 103 for (;;) { 104 v = (uintptr_t)atomic_load_ptr(&lock->head); 105 if ((v & RL_CHEAT_DRAINING) == 0) 106 break; 107 sleepq_add(&lock->head, NULL, "ranged1", 0, 0); 108 sleepq_wait(&lock->head, PRI_USER); 109 sleepq_lock(&lock->head); 110 } 111 sleepq_release(&lock->head); 112 PICKUP_GIANT(); 113 return (false); 114 } 115 116 switch (locktype) { 117 case RL_LOCK_READ: 118 for (;;) { 119 if ((v & RL_CHEAT_WLOCKED) != 0) { 120 if (trylock) { 121 *cookie = NULL; 122 return (true); 123 } 124 x = v | RL_CHEAT_DRAINING; 125 sleepq_lock(&lock->head); 126 if (atomic_fcmpset_rel_ptr(&lock->head, &v, 127 x) != 0) 128 goto drain1; 129 sleepq_release(&lock->head); 130 /* Possibly forgive passed conflict */ 131 } else { 132 x = (v & ~RL_CHEAT_MASK) + RL_CHEAT_READER; 133 x |= RL_CHEAT_CHEATING; 134 if (atomic_fcmpset_acq_ptr(&lock->head, &v, 135 x) != 0) 136 break; 137 } 138 if ((v & RL_CHEAT_CHEATING) == 0) 139 return (false); 140 if ((v & RL_CHEAT_DRAINING) != 0) 141 goto drain; 142 } 143 *(uintptr_t *)cookie = RL_RET_CHEAT_RLOCKED; 144 break; 145 case RL_LOCK_WRITE: 146 for (;;) { 147 if ((v & ~RL_CHEAT_MASK) >= RL_CHEAT_READER || 148 (v & RL_CHEAT_WLOCKED) != 0) { 149 if (trylock) { 150 *cookie = NULL; 151 return (true); 152 } 153 x = v | RL_CHEAT_DRAINING; 154 sleepq_lock(&lock->head); 155 if (atomic_fcmpset_rel_ptr(&lock->head, &v, 156 x) != 0) 157 goto drain1; 158 sleepq_release(&lock->head); 159 /* Possibly forgive passed conflict */ 160 } else { 161 x = RL_CHEAT_WLOCKED | RL_CHEAT_CHEATING; 162 if (atomic_fcmpset_acq_ptr(&lock->head, &v, 163 x) != 0) 164 break; 165 } 166 if ((v & RL_CHEAT_CHEATING) == 0) 167 return (false); 168 if ((v & RL_CHEAT_DRAINING) != 0) 169 goto drain; 170 } 171 *(uintptr_t *)cookie = RL_RET_CHEAT_WLOCKED; 172 break; 173 default: 174 __assert_unreachable(); 175 break; 176 } 177 return (true); 178 } 179 180 static bool 181 rangelock_cheat_unlock(struct rangelock *lock, void *cookie) 182 { 183 uintptr_t v, x; 184 185 v = (uintptr_t)atomic_load_ptr(&lock->head); 186 if ((v & RL_CHEAT_CHEATING) == 0) 187 return (false); 188 189 MPASS((uintptr_t)cookie == RL_RET_CHEAT_WLOCKED || 190 (uintptr_t)cookie == RL_RET_CHEAT_RLOCKED); 191 192 switch ((uintptr_t)cookie) { 193 case RL_RET_CHEAT_RLOCKED: 194 for (;;) { 195 MPASS((v & ~RL_CHEAT_MASK) >= RL_CHEAT_READER); 196 MPASS((v & RL_CHEAT_WLOCKED) == 0); 197 x = (v & ~RL_CHEAT_MASK) - RL_CHEAT_READER; 198 if ((v & RL_CHEAT_DRAINING) != 0) { 199 if (x != 0) { 200 x |= RL_CHEAT_DRAINING | 201 RL_CHEAT_CHEATING; 202 if (atomic_fcmpset_rel_ptr(&lock->head, 203 &v, x) != 0) 204 break; 205 } else { 206 sleepq_lock(&lock->head); 207 if (atomic_fcmpset_rel_ptr(&lock->head, 208 &v, x) != 0) { 209 sleepq_broadcast( 210 &lock->head, 211 SLEEPQ_SLEEP, 0, 0); 212 sleepq_release(&lock->head); 213 break; 214 } 215 sleepq_release(&lock->head); 216 } 217 } else { 218 x |= RL_CHEAT_CHEATING; 219 if (atomic_fcmpset_rel_ptr(&lock->head, &v, 220 x) != 0) 221 break; 222 } 223 } 224 break; 225 case RL_RET_CHEAT_WLOCKED: 226 for (;;) { 227 MPASS((v & RL_CHEAT_WLOCKED) != 0); 228 if ((v & RL_CHEAT_DRAINING) != 0) { 229 sleepq_lock(&lock->head); 230 atomic_store_ptr(&lock->head, 0); 231 sleepq_broadcast(&lock->head, 232 SLEEPQ_SLEEP, 0, 0); 233 sleepq_release(&lock->head); 234 break; 235 } else { 236 if (atomic_fcmpset_ptr(&lock->head, &v, 237 RL_CHEAT_CHEATING) != 0) 238 break; 239 } 240 } 241 break; 242 default: 243 __assert_unreachable(); 244 break; 245 } 246 return (true); 247 } 248 249 static bool 250 rangelock_cheat_destroy(struct rangelock *lock) 251 { 252 uintptr_t v; 253 254 v = (uintptr_t)atomic_load_ptr(&lock->head); 255 if ((v & RL_CHEAT_CHEATING) == 0) 256 return (false); 257 MPASS(v == RL_CHEAT_CHEATING); 258 return (true); 259 } 260 261 /* 262 * Implementation of range locks based on the paper 263 * https://doi.org/10.1145/3342195.3387533 264 * arXiv:2006.12144v1 [cs.OS] 22 Jun 2020 265 * Scalable Range Locks for Scalable Address Spaces and Beyond 266 * by Alex Kogan, Dave Dice, and Shady Issa 267 */ 268 269 static struct rl_q_entry *rl_e_unmark(const struct rl_q_entry *e); 270 271 /* 272 * rl_q_next links all granted ranges in the lock. We cannot free an 273 * rl_q_entry while in the smr section, and cannot reuse rl_q_next 274 * linkage since other threads might follow it even after CAS removed 275 * the range. Use rl_q_free for local list of ranges to remove after 276 * the smr section is dropped. 277 */ 278 struct rl_q_entry { 279 struct rl_q_entry *rl_q_next; 280 struct rl_q_entry *rl_q_free; 281 off_t rl_q_start, rl_q_end; 282 int rl_q_flags; 283 #ifdef INVARIANTS 284 struct thread *rl_q_owner; 285 #endif 286 }; 287 288 static uma_zone_t rl_entry_zone; 289 static smr_t rl_smr; 290 291 static void 292 rangelock_sys_init(void) 293 { 294 rl_entry_zone = uma_zcreate("rl_entry", sizeof(struct rl_q_entry), 295 NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct rl_q_entry), 296 UMA_ZONE_SMR); 297 rl_smr = uma_zone_get_smr(rl_entry_zone); 298 } 299 SYSINIT(rl, SI_SUB_LOCK, SI_ORDER_ANY, rangelock_sys_init, NULL); 300 301 static struct rl_q_entry * 302 rlqentry_alloc(vm_ooffset_t start, vm_ooffset_t end, int flags) 303 { 304 struct rl_q_entry *e; 305 struct thread *td; 306 307 td = curthread; 308 if (td->td_rlqe != NULL) { 309 e = td->td_rlqe; 310 td->td_rlqe = NULL; 311 } else { 312 e = uma_zalloc_smr(rl_entry_zone, M_WAITOK); 313 } 314 e->rl_q_next = NULL; 315 e->rl_q_free = NULL; 316 e->rl_q_start = start; 317 e->rl_q_end = end; 318 e->rl_q_flags = flags; 319 #ifdef INVARIANTS 320 e->rl_q_owner = curthread; 321 #endif 322 return (e); 323 } 324 325 void 326 rangelock_entry_free(struct rl_q_entry *e) 327 { 328 uma_zfree_smr(rl_entry_zone, e); 329 } 330 331 void 332 rangelock_init(struct rangelock *lock) 333 { 334 lock->sleepers = false; 335 atomic_store_ptr(&lock->head, rangelock_cheat ? RL_CHEAT_CHEATING : 0); 336 } 337 338 void 339 rangelock_destroy(struct rangelock *lock) 340 { 341 struct rl_q_entry *e, *ep; 342 343 MPASS(!lock->sleepers); 344 if (rangelock_cheat_destroy(lock)) 345 return; 346 for (e = (struct rl_q_entry *)atomic_load_ptr(&lock->head); 347 e != NULL; e = rl_e_unmark(ep)) { 348 ep = atomic_load_ptr(&e->rl_q_next); 349 uma_zfree_smr(rl_entry_zone, e); 350 } 351 } 352 353 static bool 354 rl_e_is_marked(const struct rl_q_entry *e) 355 { 356 return (((uintptr_t)e & 1) != 0); 357 } 358 359 static struct rl_q_entry * 360 rl_e_unmark_unchecked(const struct rl_q_entry *e) 361 { 362 return ((struct rl_q_entry *)((uintptr_t)e & ~1)); 363 } 364 365 static struct rl_q_entry * 366 rl_e_unmark(const struct rl_q_entry *e) 367 { 368 MPASS(rl_e_is_marked(e)); 369 return (rl_e_unmark_unchecked(e)); 370 } 371 372 static void 373 rl_e_mark(struct rl_q_entry *e) 374 { 375 #if defined(INVARIANTS) && defined(__LP64__) 376 int r = atomic_testandset_long((uintptr_t *)&e->rl_q_next, 0); 377 MPASS(r == 0); 378 #else 379 atomic_set_ptr((uintptr_t *)&e->rl_q_next, 1); 380 #endif 381 } 382 383 static struct rl_q_entry * 384 rl_q_load(struct rl_q_entry **p) 385 { 386 return ((struct rl_q_entry *)atomic_load_acq_ptr((uintptr_t *)p)); 387 } 388 389 static bool 390 rl_e_is_rlock(const struct rl_q_entry *e) 391 { 392 return ((e->rl_q_flags & RL_LOCK_TYPE_MASK) == RL_LOCK_READ); 393 } 394 395 static void 396 rangelock_unlock_int(struct rangelock *lock, struct rl_q_entry *e) 397 { 398 bool sleepers; 399 400 MPASS(lock != NULL && e != NULL); 401 MPASS(!rl_e_is_marked(rl_q_load(&e->rl_q_next))); 402 MPASS(e->rl_q_owner == curthread); 403 404 rl_e_mark(e); 405 sleepers = lock->sleepers; 406 lock->sleepers = false; 407 if (sleepers) 408 sleepq_broadcast(&lock->sleepers, SLEEPQ_SLEEP, 0, 0); 409 } 410 411 void 412 rangelock_unlock(struct rangelock *lock, void *cookie) 413 { 414 if (rangelock_cheat_unlock(lock, cookie)) 415 return; 416 417 sleepq_lock(&lock->sleepers); 418 rangelock_unlock_int(lock, cookie); 419 sleepq_release(&lock->sleepers); 420 } 421 422 /* 423 * result: -1 if e1 before e2, or both locks are readers and e1 424 * starts before or at e2 425 * 1 if e1 after e2, or both locks are readers and e1 426 * starts after e2 427 * 0 if e1 and e2 overlap and at least one lock is writer 428 */ 429 static int 430 rl_e_compare(const struct rl_q_entry *e1, const struct rl_q_entry *e2) 431 { 432 bool rds; 433 434 if (e1 == NULL) 435 return (1); 436 if (e2->rl_q_start >= e1->rl_q_end) 437 return (-1); 438 rds = rl_e_is_rlock(e1) && rl_e_is_rlock(e2); 439 if (e2->rl_q_start >= e1->rl_q_start && rds) 440 return (-1); 441 if (e1->rl_q_start >= e2->rl_q_end) 442 return (1); 443 if (e1->rl_q_start >= e2->rl_q_start && rds) 444 return (1); 445 return (0); 446 } 447 448 static void 449 rl_insert_sleep(struct rangelock *lock) 450 { 451 smr_exit(rl_smr); 452 DROP_GIANT(); 453 lock->sleepers = true; 454 sleepq_add(&lock->sleepers, NULL, "rangelk", 0, 0); 455 sleepq_wait(&lock->sleepers, PRI_USER); 456 PICKUP_GIANT(); 457 smr_enter(rl_smr); 458 } 459 460 static bool 461 rl_q_cas(struct rl_q_entry **prev, struct rl_q_entry *old, 462 struct rl_q_entry *new) 463 { 464 return (atomic_cmpset_rel_ptr((uintptr_t *)prev, (uintptr_t)old, 465 (uintptr_t)new) != 0); 466 } 467 468 enum RL_INSERT_RES { 469 RL_TRYLOCK_FAILED, 470 RL_LOCK_SUCCESS, 471 RL_LOCK_RETRY, 472 }; 473 474 static enum RL_INSERT_RES 475 rl_r_validate(struct rangelock *lock, struct rl_q_entry *e, bool trylock, 476 struct rl_q_entry **free) 477 { 478 struct rl_q_entry *cur, *next, **prev; 479 480 prev = &e->rl_q_next; 481 cur = rl_q_load(prev); 482 MPASS(!rl_e_is_marked(cur)); /* nobody can unlock e yet */ 483 for (;;) { 484 if (cur == NULL || cur->rl_q_start > e->rl_q_end) 485 return (RL_LOCK_SUCCESS); 486 next = rl_q_load(&cur->rl_q_next); 487 if (rl_e_is_marked(next)) { 488 next = rl_e_unmark(next); 489 if (rl_q_cas(prev, cur, next)) { 490 cur->rl_q_free = *free; 491 *free = cur; 492 } 493 cur = next; 494 continue; 495 } 496 if (rl_e_is_rlock(cur)) { 497 prev = &cur->rl_q_next; 498 cur = rl_e_unmark_unchecked(rl_q_load(prev)); 499 continue; 500 } 501 if (!rl_e_is_marked(rl_q_load(&cur->rl_q_next))) { 502 sleepq_lock(&lock->sleepers); 503 if (rl_e_is_marked(rl_q_load(&cur->rl_q_next))) { 504 sleepq_release(&lock->sleepers); 505 continue; 506 } 507 rangelock_unlock_int(lock, e); 508 if (trylock) { 509 sleepq_release(&lock->sleepers); 510 return (RL_TRYLOCK_FAILED); 511 } 512 rl_insert_sleep(lock); 513 return (RL_LOCK_RETRY); 514 } 515 } 516 } 517 518 static enum RL_INSERT_RES 519 rl_w_validate(struct rangelock *lock, struct rl_q_entry *e, 520 bool trylock, struct rl_q_entry **free) 521 { 522 struct rl_q_entry *cur, *next, **prev; 523 524 prev = (struct rl_q_entry **)&lock->head; 525 cur = rl_q_load(prev); 526 MPASS(!rl_e_is_marked(cur)); /* head is not marked */ 527 for (;;) { 528 if (cur == e) 529 return (RL_LOCK_SUCCESS); 530 next = rl_q_load(&cur->rl_q_next); 531 if (rl_e_is_marked(next)) { 532 next = rl_e_unmark(next); 533 if (rl_q_cas(prev, cur, next)) { 534 cur->rl_q_next = *free; 535 *free = cur; 536 } 537 cur = next; 538 continue; 539 } 540 if (cur->rl_q_end <= e->rl_q_start) { 541 prev = &cur->rl_q_next; 542 cur = rl_e_unmark_unchecked(rl_q_load(prev)); 543 continue; 544 } 545 sleepq_lock(&lock->sleepers); 546 rangelock_unlock_int(lock, e); 547 if (trylock) { 548 sleepq_release(&lock->sleepers); 549 return (RL_TRYLOCK_FAILED); 550 } 551 rl_insert_sleep(lock); 552 return (RL_LOCK_RETRY); 553 } 554 } 555 556 static enum RL_INSERT_RES 557 rl_insert(struct rangelock *lock, struct rl_q_entry *e, bool trylock, 558 struct rl_q_entry **free) 559 { 560 struct rl_q_entry *cur, *next, **prev; 561 int r; 562 563 again: 564 prev = (struct rl_q_entry **)&lock->head; 565 cur = rl_q_load(prev); 566 if (cur == NULL && rl_q_cas(prev, NULL, e)) 567 return (RL_LOCK_SUCCESS); 568 569 for (;;) { 570 if (cur != NULL) { 571 if (rl_e_is_marked(cur)) 572 goto again; 573 574 next = rl_q_load(&cur->rl_q_next); 575 if (rl_e_is_marked(next)) { 576 next = rl_e_unmark(next); 577 if (rl_q_cas(prev, cur, next)) { 578 #ifdef INVARIANTS 579 cur->rl_q_owner = NULL; 580 #endif 581 cur->rl_q_free = *free; 582 *free = cur; 583 } 584 cur = next; 585 continue; 586 } 587 } 588 589 r = rl_e_compare(cur, e); 590 if (r == -1) { 591 prev = &cur->rl_q_next; 592 cur = rl_q_load(prev); 593 } else if (r == 0) { 594 sleepq_lock(&lock->sleepers); 595 if (__predict_false(rl_e_is_marked(rl_q_load( 596 &cur->rl_q_next)))) { 597 sleepq_release(&lock->sleepers); 598 continue; 599 } 600 if (trylock) { 601 sleepq_release(&lock->sleepers); 602 return (RL_TRYLOCK_FAILED); 603 } 604 rl_insert_sleep(lock); 605 /* e is still valid */ 606 goto again; 607 } else /* r == 1 */ { 608 e->rl_q_next = cur; 609 if (rl_q_cas(prev, cur, e)) { 610 atomic_thread_fence_acq(); 611 return (rl_e_is_rlock(e) ? 612 rl_r_validate(lock, e, trylock, free) : 613 rl_w_validate(lock, e, trylock, free)); 614 } 615 /* Reset rl_q_next in case we hit fast path. */ 616 e->rl_q_next = NULL; 617 cur = rl_q_load(prev); 618 } 619 } 620 } 621 622 static struct rl_q_entry * 623 rangelock_lock_int(struct rangelock *lock, bool trylock, vm_ooffset_t start, 624 vm_ooffset_t end, int locktype) 625 { 626 struct rl_q_entry *e, *free, *x, *xp; 627 struct thread *td; 628 void *cookie; 629 enum RL_INSERT_RES res; 630 631 if (rangelock_cheat_lock(lock, locktype, trylock, &cookie)) 632 return (cookie); 633 td = curthread; 634 for (res = RL_LOCK_RETRY; res == RL_LOCK_RETRY;) { 635 free = NULL; 636 e = rlqentry_alloc(start, end, locktype); 637 smr_enter(rl_smr); 638 res = rl_insert(lock, e, trylock, &free); 639 smr_exit(rl_smr); 640 if (res == RL_TRYLOCK_FAILED) { 641 MPASS(trylock); 642 e->rl_q_free = free; 643 free = e; 644 e = NULL; 645 } 646 for (x = free; x != NULL; x = xp) { 647 MPASS(!rl_e_is_marked(x)); 648 xp = x->rl_q_free; 649 MPASS(!rl_e_is_marked(xp)); 650 if (td->td_rlqe == NULL) { 651 smr_synchronize(rl_smr); 652 td->td_rlqe = x; 653 } else { 654 uma_zfree_smr(rl_entry_zone, x); 655 } 656 } 657 } 658 return (e); 659 } 660 661 void * 662 rangelock_rlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) 663 { 664 return (rangelock_lock_int(lock, false, start, end, RL_LOCK_READ)); 665 } 666 667 void * 668 rangelock_tryrlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) 669 { 670 return (rangelock_lock_int(lock, true, start, end, RL_LOCK_READ)); 671 } 672 673 void * 674 rangelock_wlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) 675 { 676 return (rangelock_lock_int(lock, false, start, end, RL_LOCK_WRITE)); 677 } 678 679 void * 680 rangelock_trywlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end) 681 { 682 return (rangelock_lock_int(lock, true, start, end, RL_LOCK_WRITE)); 683 } 684 685 #ifdef INVARIANT_SUPPORT 686 void 687 _rangelock_cookie_assert(void *cookie, int what, const char *file, int line) 688 { 689 } 690 #endif /* INVARIANT_SUPPORT */ 691 692 #include "opt_ddb.h" 693 #ifdef DDB 694 #include <ddb/ddb.h> 695 696 DB_SHOW_COMMAND(rangelock, db_show_rangelock) 697 { 698 struct rangelock *lock; 699 struct rl_q_entry *e, *x; 700 uintptr_t v; 701 702 if (!have_addr) { 703 db_printf("show rangelock addr\n"); 704 return; 705 } 706 707 lock = (struct rangelock *)addr; 708 db_printf("rangelock %p sleepers %d\n", lock, lock->sleepers); 709 v = lock->head; 710 if ((v & RL_CHEAT_CHEATING) != 0) { 711 db_printf(" cheating head %#jx\n", (uintmax_t)v); 712 return; 713 } 714 for (e = (struct rl_q_entry *)(lock->head);;) { 715 x = rl_e_is_marked(e) ? rl_e_unmark(e) : e; 716 if (x == NULL) 717 break; 718 db_printf(" entry %p marked %d %d start %#jx end %#jx " 719 "flags %x next %p", 720 e, rl_e_is_marked(e), rl_e_is_marked(x->rl_q_next), 721 x->rl_q_start, x->rl_q_end, x->rl_q_flags, x->rl_q_next); 722 #ifdef INVARIANTS 723 db_printf(" owner %p (%d)", x->rl_q_owner, 724 x->rl_q_owner != NULL ? x->rl_q_owner->td_tid : -1); 725 #endif 726 db_printf("\n"); 727 e = x->rl_q_next; 728 } 729 } 730 731 #endif /* DDB */ 732