xref: /freebsd/sys/kern/kern_rangelock.c (revision 024248c933c5741a21c17eda63092f330dd98337)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2009 Konstantin Belousov <kib@FreeBSD.org>
5  * Copyright (c) 2023 The FreeBSD Foundation
6  *
7  * Portions of this software were developed by
8  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
9  * the FreeBSD Foundation.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice unmodified, this list of conditions, and the following
16  *    disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
22  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
23  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
24  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
26  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
30  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include <sys/param.h>
34 #include <sys/kassert.h>
35 #include <sys/kernel.h>
36 #include <sys/lock.h>
37 #include <sys/mutex.h>
38 #include <sys/proc.h>
39 #include <sys/rangelock.h>
40 #include <sys/sleepqueue.h>
41 #include <sys/smr.h>
42 #include <sys/sysctl.h>
43 
44 #include <vm/uma.h>
45 
46 /*
47  * Immediately after initialization (subject to 'rangelock_cheat'
48  * below), and until a request comes that conflicts with granted ones
49  * based on type, rangelocks serve requests in the "cheating" mode.
50  * In this mode, a rangelock behaves like a sxlock, as if each request
51  * covered the whole range of the protected object.  On receiving a
52  * conflicting request (any request while a write request is
53  * effective, or any write request while some read ones are
54  * effective), all requests granted in "cheating" mode are drained,
55  * and the rangelock then switches to effectively keeping track of the
56  * precise range of each new request.
57  *
58  * Normal sx implementation is not used to not bloat structures (most
59  * important, vnodes) which embeds rangelocks.
60  *
61  * The cheating greatly helps very common pattern where file is first
62  * written single-threaded, and then read by many processes.
63  *
64  * Lock is in cheat mode when RL_CHEAT_CHEATING bit is set in the
65  * lock->head.  Special cookies are returned in this mode, and
66  * trylocks are same as normal locks but do not drain.
67  */
68 
69 static int rangelock_cheat = 1;
70 SYSCTL_INT(_debug, OID_AUTO, rangelock_cheat, CTLFLAG_RWTUN,
71     &rangelock_cheat, 0,
72     "");
73 
74 #define	RL_CHEAT_MASK		0x7
75 #define	RL_CHEAT_CHEATING	0x1
76 /* #define	RL_CHEAT_RLOCKED	0x0 */
77 #define	RL_CHEAT_WLOCKED	0x2
78 #define	RL_CHEAT_DRAINING	0x4
79 
80 #define	RL_CHEAT_READER		0x8
81 
82 #define	RL_RET_CHEAT_RLOCKED	0x1100
83 #define	RL_RET_CHEAT_WLOCKED	0x2200
84 
85 static bool
86 rangelock_cheat_lock(struct rangelock *lock, int locktype, bool trylock,
87     void **cookie)
88 {
89 	uintptr_t v, x;
90 
91 	v = (uintptr_t)atomic_load_ptr(&lock->head);
92 	if ((v & RL_CHEAT_CHEATING) == 0)
93 		return (false);
94 	if ((v & RL_CHEAT_DRAINING) != 0) {
95 drain:
96 		if (trylock) {
97 			*cookie = NULL;
98 			return (true);
99 		}
100 		sleepq_lock(&lock->head);
101 drain1:
102 		DROP_GIANT();
103 		for (;;) {
104 			v = (uintptr_t)atomic_load_ptr(&lock->head);
105 			if ((v & RL_CHEAT_DRAINING) == 0)
106 				break;
107 			sleepq_add(&lock->head, NULL, "ranged1", 0, 0);
108 			sleepq_wait(&lock->head, PRI_USER);
109 			sleepq_lock(&lock->head);
110 		}
111 		sleepq_release(&lock->head);
112 		PICKUP_GIANT();
113 		return (false);
114 	}
115 
116 	switch (locktype) {
117 	case RL_LOCK_READ:
118 		for (;;) {
119 			if ((v & RL_CHEAT_WLOCKED) != 0) {
120 				if (trylock) {
121 					*cookie = NULL;
122 					return (true);
123 				}
124 				x = v | RL_CHEAT_DRAINING;
125 				sleepq_lock(&lock->head);
126 				if (atomic_fcmpset_rel_ptr(&lock->head, &v,
127 				    x) != 0)
128 					goto drain1;
129 				sleepq_release(&lock->head);
130 				/* Possibly forgive passed conflict */
131 			} else {
132 				x = (v & ~RL_CHEAT_MASK) + RL_CHEAT_READER;
133 				x |= RL_CHEAT_CHEATING;
134 				if (atomic_fcmpset_acq_ptr(&lock->head, &v,
135 				    x) != 0)
136 					break;
137 			}
138 			if ((v & RL_CHEAT_CHEATING) == 0)
139 				return (false);
140 			if ((v & RL_CHEAT_DRAINING) != 0)
141 				goto drain;
142 		}
143 		*(uintptr_t *)cookie = RL_RET_CHEAT_RLOCKED;
144 		break;
145 	case RL_LOCK_WRITE:
146 		for (;;) {
147 			if ((v & ~RL_CHEAT_MASK) >= RL_CHEAT_READER ||
148 			    (v & RL_CHEAT_WLOCKED) != 0) {
149 				if (trylock) {
150 					*cookie = NULL;
151 					return (true);
152 				}
153 				x = v | RL_CHEAT_DRAINING;
154 				sleepq_lock(&lock->head);
155 				if (atomic_fcmpset_rel_ptr(&lock->head, &v,
156 				    x) != 0)
157 					goto drain1;
158 				sleepq_release(&lock->head);
159 				/* Possibly forgive passed conflict */
160 			} else {
161 				x = RL_CHEAT_WLOCKED | RL_CHEAT_CHEATING;
162 				if (atomic_fcmpset_acq_ptr(&lock->head, &v,
163 				    x) != 0)
164 					break;
165 			}
166 			if ((v & RL_CHEAT_CHEATING) == 0)
167 				return (false);
168 			if ((v & RL_CHEAT_DRAINING) != 0)
169 				goto drain;
170 		}
171 		*(uintptr_t *)cookie = RL_RET_CHEAT_WLOCKED;
172 		break;
173 	default:
174 		__assert_unreachable();
175 		break;
176 	}
177 	return (true);
178 }
179 
180 static bool
181 rangelock_cheat_unlock(struct rangelock *lock, void *cookie)
182 {
183 	uintptr_t v, x;
184 
185 	v = (uintptr_t)atomic_load_ptr(&lock->head);
186 	if ((v & RL_CHEAT_CHEATING) == 0)
187 		return (false);
188 
189 	MPASS((uintptr_t)cookie == RL_RET_CHEAT_WLOCKED ||
190 	    (uintptr_t)cookie == RL_RET_CHEAT_RLOCKED);
191 
192 	switch ((uintptr_t)cookie) {
193 	case RL_RET_CHEAT_RLOCKED:
194 		for (;;) {
195 			MPASS((v & ~RL_CHEAT_MASK) >= RL_CHEAT_READER);
196 			MPASS((v & RL_CHEAT_WLOCKED) == 0);
197 			x = (v & ~RL_CHEAT_MASK) - RL_CHEAT_READER;
198 			if ((v & RL_CHEAT_DRAINING) != 0) {
199 				if (x != 0) {
200 					x |= RL_CHEAT_DRAINING |
201 					    RL_CHEAT_CHEATING;
202 					if (atomic_fcmpset_rel_ptr(&lock->head,
203 					    &v, x) != 0)
204 						break;
205 				} else {
206 					sleepq_lock(&lock->head);
207 					if (atomic_fcmpset_rel_ptr(&lock->head,
208 					    &v, x) != 0) {
209 						sleepq_broadcast(
210 						    &lock->head,
211 						    SLEEPQ_SLEEP, 0, 0);
212 						sleepq_release(&lock->head);
213 						break;
214 					}
215 					sleepq_release(&lock->head);
216 				}
217 			} else {
218 				x |= RL_CHEAT_CHEATING;
219 				if (atomic_fcmpset_rel_ptr(&lock->head, &v,
220 				    x) != 0)
221 					break;
222 			}
223 		}
224 		break;
225 	case RL_RET_CHEAT_WLOCKED:
226 		for (;;) {
227 			MPASS((v & RL_CHEAT_WLOCKED) != 0);
228 			if ((v & RL_CHEAT_DRAINING) != 0) {
229 				sleepq_lock(&lock->head);
230 				atomic_store_ptr(&lock->head, 0);
231 				sleepq_broadcast(&lock->head,
232 				    SLEEPQ_SLEEP, 0, 0);
233 				sleepq_release(&lock->head);
234 				break;
235 			} else {
236 				if (atomic_fcmpset_ptr(&lock->head, &v,
237 				    RL_CHEAT_CHEATING) != 0)
238 					break;
239 			}
240 		}
241 		break;
242 	default:
243 		__assert_unreachable();
244 		break;
245 	}
246 	return (true);
247 }
248 
249 static bool
250 rangelock_cheat_destroy(struct rangelock *lock)
251 {
252 	uintptr_t v;
253 
254 	v = (uintptr_t)atomic_load_ptr(&lock->head);
255 	if ((v & RL_CHEAT_CHEATING) == 0)
256 		return (false);
257 	MPASS(v == RL_CHEAT_CHEATING);
258 	return (true);
259 }
260 
261 /*
262  * Implementation of range locks based on the paper
263  * https://doi.org/10.1145/3342195.3387533
264  * arXiv:2006.12144v1 [cs.OS] 22 Jun 2020
265  * Scalable Range Locks for Scalable Address Spaces and Beyond
266  * by Alex Kogan, Dave Dice, and Shady Issa
267  */
268 
269 static struct rl_q_entry *rl_e_unmark(const struct rl_q_entry *e);
270 
271 /*
272  * rl_q_next links all granted ranges in the lock.  We cannot free an
273  * rl_q_entry while in the smr section, and cannot reuse rl_q_next
274  * linkage since other threads might follow it even after CAS removed
275  * the range.  Use rl_q_free for local list of ranges to remove after
276  * the smr section is dropped.
277  */
278 struct rl_q_entry {
279 	struct rl_q_entry *rl_q_next;
280 	struct rl_q_entry *rl_q_free;
281 	off_t		rl_q_start, rl_q_end;
282 	int		rl_q_flags;
283 #ifdef INVARIANTS
284 	struct thread	*rl_q_owner;
285 #endif
286 };
287 
288 static uma_zone_t rl_entry_zone;
289 static smr_t rl_smr;
290 
291 static void
292 rangelock_sys_init(void)
293 {
294 	rl_entry_zone = uma_zcreate("rl_entry", sizeof(struct rl_q_entry),
295 	    NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct rl_q_entry),
296 	    UMA_ZONE_SMR);
297 	rl_smr = uma_zone_get_smr(rl_entry_zone);
298 }
299 SYSINIT(rl, SI_SUB_LOCK, SI_ORDER_ANY, rangelock_sys_init, NULL);
300 
301 static struct rl_q_entry *
302 rlqentry_alloc(vm_ooffset_t start, vm_ooffset_t end, int flags)
303 {
304 	struct rl_q_entry *e;
305 	struct thread *td;
306 
307 	td = curthread;
308 	if (td->td_rlqe != NULL) {
309 		e = td->td_rlqe;
310 		td->td_rlqe = NULL;
311 	} else {
312 		e = uma_zalloc_smr(rl_entry_zone, M_WAITOK);
313 	}
314 	e->rl_q_next = NULL;
315 	e->rl_q_free = NULL;
316 	e->rl_q_start = start;
317 	e->rl_q_end = end;
318 	e->rl_q_flags = flags;
319 #ifdef INVARIANTS
320 	e->rl_q_owner = curthread;
321 #endif
322 	return (e);
323 }
324 
325 void
326 rangelock_entry_free(struct rl_q_entry *e)
327 {
328 	uma_zfree_smr(rl_entry_zone, e);
329 }
330 
331 void
332 rangelock_init(struct rangelock *lock)
333 {
334 	lock->sleepers = false;
335 	atomic_store_ptr(&lock->head, rangelock_cheat ? RL_CHEAT_CHEATING : 0);
336 }
337 
338 void
339 rangelock_destroy(struct rangelock *lock)
340 {
341 	struct rl_q_entry *e, *ep;
342 
343 	MPASS(!lock->sleepers);
344 	if (rangelock_cheat_destroy(lock))
345 		return;
346 	for (e = (struct rl_q_entry *)atomic_load_ptr(&lock->head);
347 	    e != NULL; e = rl_e_unmark(ep)) {
348 		ep = atomic_load_ptr(&e->rl_q_next);
349 		uma_zfree_smr(rl_entry_zone, e);
350 	}
351 }
352 
353 static bool
354 rl_e_is_marked(const struct rl_q_entry *e)
355 {
356 	return (((uintptr_t)e & 1) != 0);
357 }
358 
359 static struct rl_q_entry *
360 rl_e_unmark_unchecked(const struct rl_q_entry *e)
361 {
362 	return ((struct rl_q_entry *)((uintptr_t)e & ~1));
363 }
364 
365 static struct rl_q_entry *
366 rl_e_unmark(const struct rl_q_entry *e)
367 {
368 	MPASS(rl_e_is_marked(e));
369 	return (rl_e_unmark_unchecked(e));
370 }
371 
372 static void
373 rl_e_mark(struct rl_q_entry *e)
374 {
375 #if defined(INVARIANTS) && defined(__LP64__)
376 	int r = atomic_testandset_long((uintptr_t *)&e->rl_q_next, 0);
377 	MPASS(r == 0);
378 #else
379 	atomic_set_ptr((uintptr_t *)&e->rl_q_next, 1);
380 #endif
381 }
382 
383 static struct rl_q_entry *
384 rl_q_load(struct rl_q_entry **p)
385 {
386 	return ((struct rl_q_entry *)atomic_load_acq_ptr((uintptr_t *)p));
387 }
388 
389 static bool
390 rl_e_is_rlock(const struct rl_q_entry *e)
391 {
392 	return ((e->rl_q_flags & RL_LOCK_TYPE_MASK) == RL_LOCK_READ);
393 }
394 
395 static void
396 rangelock_unlock_int(struct rangelock *lock, struct rl_q_entry *e)
397 {
398 	bool sleepers;
399 
400 	MPASS(lock != NULL && e != NULL);
401 	MPASS(!rl_e_is_marked(rl_q_load(&e->rl_q_next)));
402 	MPASS(e->rl_q_owner == curthread);
403 
404 	rl_e_mark(e);
405 	sleepers = lock->sleepers;
406 	lock->sleepers = false;
407 	if (sleepers)
408 		sleepq_broadcast(&lock->sleepers, SLEEPQ_SLEEP, 0, 0);
409 }
410 
411 void
412 rangelock_unlock(struct rangelock *lock, void *cookie)
413 {
414 	if (rangelock_cheat_unlock(lock, cookie))
415 		return;
416 
417 	sleepq_lock(&lock->sleepers);
418 	rangelock_unlock_int(lock, cookie);
419 	sleepq_release(&lock->sleepers);
420 }
421 
422 /*
423  * result: -1 if e1 before e2, or both locks are readers and e1
424  *		starts before or at e2
425  *          1 if e1 after e2, or both locks are readers and e1
426  *		starts after e2
427  *          0 if e1 and e2 overlap and at least one lock is writer
428  */
429 static int
430 rl_e_compare(const struct rl_q_entry *e1, const struct rl_q_entry *e2)
431 {
432 	bool rds;
433 
434 	if (e1 == NULL)
435 		return (1);
436 	if (e2->rl_q_start >= e1->rl_q_end)
437 		return (-1);
438 	rds = rl_e_is_rlock(e1) && rl_e_is_rlock(e2);
439 	if (e2->rl_q_start >= e1->rl_q_start && rds)
440 		return (-1);
441 	if (e1->rl_q_start >= e2->rl_q_end)
442 		return (1);
443 	if (e1->rl_q_start >= e2->rl_q_start && rds)
444 		return (1);
445 	return (0);
446 }
447 
448 static void
449 rl_insert_sleep(struct rangelock *lock)
450 {
451 	smr_exit(rl_smr);
452 	DROP_GIANT();
453 	lock->sleepers = true;
454 	sleepq_add(&lock->sleepers, NULL, "rangelk", 0, 0);
455 	sleepq_wait(&lock->sleepers, PRI_USER);
456 	PICKUP_GIANT();
457 	smr_enter(rl_smr);
458 }
459 
460 static bool
461 rl_q_cas(struct rl_q_entry **prev, struct rl_q_entry *old,
462     struct rl_q_entry *new)
463 {
464 	return (atomic_cmpset_rel_ptr((uintptr_t *)prev, (uintptr_t)old,
465 	    (uintptr_t)new) != 0);
466 }
467 
468 enum RL_INSERT_RES {
469 	RL_TRYLOCK_FAILED,
470 	RL_LOCK_SUCCESS,
471 	RL_LOCK_RETRY,
472 };
473 
474 static enum RL_INSERT_RES
475 rl_r_validate(struct rangelock *lock, struct rl_q_entry *e, bool trylock,
476     struct rl_q_entry **free)
477 {
478 	struct rl_q_entry *cur, *next, **prev;
479 
480 	prev = &e->rl_q_next;
481 	cur = rl_q_load(prev);
482 	MPASS(!rl_e_is_marked(cur));	/* nobody can unlock e yet */
483 	for (;;) {
484 		if (cur == NULL || cur->rl_q_start > e->rl_q_end)
485 			return (RL_LOCK_SUCCESS);
486 		next = rl_q_load(&cur->rl_q_next);
487 		if (rl_e_is_marked(next)) {
488 			next = rl_e_unmark(next);
489 			if (rl_q_cas(prev, cur, next)) {
490 				cur->rl_q_free = *free;
491 				*free = cur;
492 			}
493 			cur = next;
494 			continue;
495 		}
496 		if (rl_e_is_rlock(cur)) {
497 			prev = &cur->rl_q_next;
498 			cur = rl_e_unmark_unchecked(rl_q_load(prev));
499 			continue;
500 		}
501 		if (!rl_e_is_marked(rl_q_load(&cur->rl_q_next))) {
502 			sleepq_lock(&lock->sleepers);
503 			if (rl_e_is_marked(rl_q_load(&cur->rl_q_next))) {
504 				sleepq_release(&lock->sleepers);
505 				continue;
506 			}
507 			rangelock_unlock_int(lock, e);
508 			if (trylock) {
509 				sleepq_release(&lock->sleepers);
510 				return (RL_TRYLOCK_FAILED);
511 			}
512 			rl_insert_sleep(lock);
513 			return (RL_LOCK_RETRY);
514 		}
515 	}
516 }
517 
518 static enum RL_INSERT_RES
519 rl_w_validate(struct rangelock *lock, struct rl_q_entry *e,
520     bool trylock, struct rl_q_entry **free)
521 {
522 	struct rl_q_entry *cur, *next, **prev;
523 
524 	prev = (struct rl_q_entry **)&lock->head;
525 	cur = rl_q_load(prev);
526 	MPASS(!rl_e_is_marked(cur));	/* head is not marked */
527 	for (;;) {
528 		if (cur == e)
529 			return (RL_LOCK_SUCCESS);
530 		next = rl_q_load(&cur->rl_q_next);
531 		if (rl_e_is_marked(next)) {
532 			next = rl_e_unmark(next);
533 			if (rl_q_cas(prev, cur, next)) {
534 				cur->rl_q_next = *free;
535 				*free = cur;
536 			}
537 			cur = next;
538 			continue;
539 		}
540 		if (cur->rl_q_end <= e->rl_q_start) {
541 			prev = &cur->rl_q_next;
542 			cur = rl_e_unmark_unchecked(rl_q_load(prev));
543 			continue;
544 		}
545 		sleepq_lock(&lock->sleepers);
546 		rangelock_unlock_int(lock, e);
547 		if (trylock) {
548 			sleepq_release(&lock->sleepers);
549 			return (RL_TRYLOCK_FAILED);
550 		}
551 		rl_insert_sleep(lock);
552 		return (RL_LOCK_RETRY);
553 	}
554 }
555 
556 static enum RL_INSERT_RES
557 rl_insert(struct rangelock *lock, struct rl_q_entry *e, bool trylock,
558     struct rl_q_entry **free)
559 {
560 	struct rl_q_entry *cur, *next, **prev;
561 	int r;
562 
563 again:
564 	prev = (struct rl_q_entry **)&lock->head;
565 	cur = rl_q_load(prev);
566 	if (cur == NULL && rl_q_cas(prev, NULL, e))
567 		return (RL_LOCK_SUCCESS);
568 
569 	for (;;) {
570 		if (cur != NULL) {
571 			if (rl_e_is_marked(cur))
572 				goto again;
573 
574 			next = rl_q_load(&cur->rl_q_next);
575 			if (rl_e_is_marked(next)) {
576 				next = rl_e_unmark(next);
577 				if (rl_q_cas(prev, cur, next)) {
578 #ifdef INVARIANTS
579 					cur->rl_q_owner = NULL;
580 #endif
581 					cur->rl_q_free = *free;
582 					*free = cur;
583 				}
584 				cur = next;
585 				continue;
586 			}
587 		}
588 
589 		r = rl_e_compare(cur, e);
590 		if (r == -1) {
591 			prev = &cur->rl_q_next;
592 			cur = rl_q_load(prev);
593 		} else if (r == 0) {
594 			sleepq_lock(&lock->sleepers);
595 			if (__predict_false(rl_e_is_marked(rl_q_load(
596 			    &cur->rl_q_next)))) {
597 				sleepq_release(&lock->sleepers);
598 				continue;
599 			}
600 			if (trylock) {
601 				sleepq_release(&lock->sleepers);
602 				return (RL_TRYLOCK_FAILED);
603 			}
604 			rl_insert_sleep(lock);
605 			/* e is still valid */
606 			goto again;
607 		} else /* r == 1 */ {
608 			e->rl_q_next = cur;
609 			if (rl_q_cas(prev, cur, e)) {
610 				atomic_thread_fence_acq();
611 				return (rl_e_is_rlock(e) ?
612 				    rl_r_validate(lock, e, trylock, free) :
613 				    rl_w_validate(lock, e, trylock, free));
614 			}
615 			/* Reset rl_q_next in case we hit fast path. */
616 			e->rl_q_next = NULL;
617 			cur = rl_q_load(prev);
618 		}
619 	}
620 }
621 
622 static struct rl_q_entry *
623 rangelock_lock_int(struct rangelock *lock, bool trylock, vm_ooffset_t start,
624     vm_ooffset_t end, int locktype)
625 {
626 	struct rl_q_entry *e, *free, *x, *xp;
627 	struct thread *td;
628 	void *cookie;
629 	enum RL_INSERT_RES res;
630 
631 	if (rangelock_cheat_lock(lock, locktype, trylock, &cookie))
632 		return (cookie);
633 	td = curthread;
634 	for (res = RL_LOCK_RETRY; res == RL_LOCK_RETRY;) {
635 		free = NULL;
636 		e = rlqentry_alloc(start, end, locktype);
637 		smr_enter(rl_smr);
638 		res = rl_insert(lock, e, trylock, &free);
639 		smr_exit(rl_smr);
640 		if (res == RL_TRYLOCK_FAILED) {
641 			MPASS(trylock);
642 			e->rl_q_free = free;
643 			free = e;
644 			e = NULL;
645 		}
646 		for (x = free; x != NULL; x = xp) {
647 			MPASS(!rl_e_is_marked(x));
648 			xp = x->rl_q_free;
649 			MPASS(!rl_e_is_marked(xp));
650 			if (td->td_rlqe == NULL) {
651 				smr_synchronize(rl_smr);
652 				td->td_rlqe = x;
653 			} else {
654 				uma_zfree_smr(rl_entry_zone, x);
655 			}
656 		}
657 	}
658 	return (e);
659 }
660 
661 void *
662 rangelock_rlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)
663 {
664 	return (rangelock_lock_int(lock, false, start, end, RL_LOCK_READ));
665 }
666 
667 void *
668 rangelock_tryrlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)
669 {
670 	return (rangelock_lock_int(lock, true, start, end, RL_LOCK_READ));
671 }
672 
673 void *
674 rangelock_wlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)
675 {
676 	return (rangelock_lock_int(lock, false, start, end, RL_LOCK_WRITE));
677 }
678 
679 void *
680 rangelock_trywlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end)
681 {
682 	return (rangelock_lock_int(lock, true, start, end, RL_LOCK_WRITE));
683 }
684 
685 #ifdef INVARIANT_SUPPORT
686 void
687 _rangelock_cookie_assert(void *cookie, int what, const char *file, int line)
688 {
689 }
690 #endif	/* INVARIANT_SUPPORT */
691 
692 #include "opt_ddb.h"
693 #ifdef DDB
694 #include <ddb/ddb.h>
695 
696 DB_SHOW_COMMAND(rangelock, db_show_rangelock)
697 {
698 	struct rangelock *lock;
699 	struct rl_q_entry *e, *x;
700 	uintptr_t v;
701 
702 	if (!have_addr) {
703 		db_printf("show rangelock addr\n");
704 		return;
705 	}
706 
707 	lock = (struct rangelock *)addr;
708 	db_printf("rangelock %p sleepers %d\n", lock, lock->sleepers);
709 	v = lock->head;
710 	if ((v & RL_CHEAT_CHEATING) != 0) {
711 		db_printf("  cheating head %#jx\n", (uintmax_t)v);
712 		return;
713 	}
714 	for (e = (struct rl_q_entry *)(lock->head);;) {
715 		x = rl_e_is_marked(e) ? rl_e_unmark(e) : e;
716 		if (x == NULL)
717 			break;
718 		db_printf("  entry %p marked %d %d start %#jx end %#jx "
719 		    "flags %x next %p",
720 		    e, rl_e_is_marked(e), rl_e_is_marked(x->rl_q_next),
721 		    x->rl_q_start, x->rl_q_end, x->rl_q_flags, x->rl_q_next);
722 #ifdef INVARIANTS
723 		db_printf(" owner %p (%d)", x->rl_q_owner,
724 		    x->rl_q_owner != NULL ? x->rl_q_owner->td_tid : -1);
725 #endif
726 		db_printf("\n");
727 		e = x->rl_q_next;
728 	}
729 }
730 
731 #endif	/* DDB */
732