1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015, 2016 The FreeBSD Foundation
5 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
6 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
7 * All rights reserved.
8 *
9 * Portions of this software were developed by Konstantin Belousov
10 * under sponsorship from the FreeBSD Foundation.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice unmodified, this list of conditions, and the following
17 * disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <sys/cdefs.h>
35 #include "opt_umtx_profiling.h"
36
37 #include <sys/param.h>
38 #include <sys/kernel.h>
39 #include <sys/fcntl.h>
40 #include <sys/file.h>
41 #include <sys/filedesc.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mman.h>
46 #include <sys/mutex.h>
47 #include <sys/priv.h>
48 #include <sys/proc.h>
49 #include <sys/resource.h>
50 #include <sys/resourcevar.h>
51 #include <sys/rwlock.h>
52 #include <sys/sbuf.h>
53 #include <sys/sched.h>
54 #include <sys/smp.h>
55 #include <sys/sysctl.h>
56 #include <sys/systm.h>
57 #include <sys/sysproto.h>
58 #include <sys/syscallsubr.h>
59 #include <sys/taskqueue.h>
60 #include <sys/time.h>
61 #include <sys/eventhandler.h>
62 #include <sys/umtx.h>
63 #include <sys/umtxvar.h>
64
65 #include <security/mac/mac_framework.h>
66
67 #include <vm/vm.h>
68 #include <vm/vm_param.h>
69 #include <vm/pmap.h>
70 #include <vm/uma.h>
71 #include <vm/vm_map.h>
72 #include <vm/vm_object.h>
73
74 #include <machine/atomic.h>
75 #include <machine/cpu.h>
76
77 #include <compat/freebsd32/freebsd32.h>
78 #ifdef COMPAT_FREEBSD32
79 #include <compat/freebsd32/freebsd32_proto.h>
80 #endif
81
82 #define _UMUTEX_TRY 1
83 #define _UMUTEX_WAIT 2
84
85 #ifdef UMTX_PROFILING
86 #define UPROF_PERC_BIGGER(w, f, sw, sf) \
87 (((w) > (sw)) || ((w) == (sw) && (f) > (sf)))
88 #endif
89
90 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
91 #ifdef INVARIANTS
92 #define UMTXQ_ASSERT_LOCKED_BUSY(key) do { \
93 struct umtxq_chain *uc; \
94 \
95 uc = umtxq_getchain(key); \
96 mtx_assert(&uc->uc_lock, MA_OWNED); \
97 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); \
98 } while (0)
99 #else
100 #define UMTXQ_ASSERT_LOCKED_BUSY(key) do {} while (0)
101 #endif
102
103 /*
104 * Don't propagate time-sharing priority, there is a security reason,
105 * a user can simply introduce PI-mutex, let thread A lock the mutex,
106 * and let another thread B block on the mutex, because B is
107 * sleeping, its priority will be boosted, this causes A's priority to
108 * be boosted via priority propagating too and will never be lowered even
109 * if it is using 100%CPU, this is unfair to other processes.
110 */
111
112 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
113 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
114 PRI_MAX_TIMESHARE : (td)->td_user_pri)
115
116 #define GOLDEN_RATIO_32 1640531527U
117 #ifndef UMTX_CHAINS
118 #define UMTX_CHAINS 512
119 #endif
120 #define UMTX_SHIFTS (__WORD_BIT - 9)
121
122 #define GET_SHARE(flags) \
123 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
124
125 #define BUSY_SPINS 200
126
127 struct umtx_copyops {
128 int (*copyin_timeout)(const void *uaddr, struct timespec *tsp);
129 int (*copyin_umtx_time)(const void *uaddr, size_t size,
130 struct _umtx_time *tp);
131 int (*copyin_robust_lists)(const void *uaddr, size_t size,
132 struct umtx_robust_lists_params *rbp);
133 int (*copyout_timeout)(void *uaddr, size_t size,
134 struct timespec *tsp);
135 const size_t timespec_sz;
136 const size_t umtx_time_sz;
137 const bool compat32;
138 };
139
140 _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32");
141 _Static_assert(__offsetof(struct umutex, m_spare[0]) ==
142 __offsetof(struct umutex32, m_spare[0]), "m_spare32");
143
144 int umtx_shm_vnobj_persistent = 0;
145 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN,
146 &umtx_shm_vnobj_persistent, 0,
147 "False forces destruction of umtx attached to file, on last close");
148 static int umtx_max_rb = 1000;
149 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN,
150 &umtx_max_rb, 0,
151 "Maximum number of robust mutexes allowed for each thread");
152
153 static uma_zone_t umtx_pi_zone;
154 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS];
155 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
156 static int umtx_pi_allocated;
157
158 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
159 "umtx debug");
160 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
161 &umtx_pi_allocated, 0, "Allocated umtx_pi");
162 static int umtx_verbose_rb = 1;
163 SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN,
164 &umtx_verbose_rb, 0,
165 "");
166
167 #ifdef UMTX_PROFILING
168 static long max_length;
169 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length");
170 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
171 "umtx chain stats");
172 #endif
173
174 static inline void umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
175 const struct _umtx_time *umtxtime);
176
177 static void umtx_shm_init(void);
178 static void umtxq_sysinit(void *);
179 static void umtxq_hash(struct umtx_key *key);
180 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags,
181 bool rb);
182 static void umtx_thread_cleanup(struct thread *td);
183 SYSINIT(umtx, SI_SUB_EVENTHANDLER, SI_ORDER_LAST, umtxq_sysinit, NULL);
184
185 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
186
187 static struct mtx umtx_lock;
188
189 #ifdef UMTX_PROFILING
190 static void
umtx_init_profiling(void)191 umtx_init_profiling(void)
192 {
193 struct sysctl_oid *chain_oid;
194 char chain_name[10];
195 int i;
196
197 for (i = 0; i < UMTX_CHAINS; ++i) {
198 snprintf(chain_name, sizeof(chain_name), "%d", i);
199 chain_oid = SYSCTL_ADD_NODE(NULL,
200 SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO,
201 chain_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
202 "umtx hash stats");
203 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
204 "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL);
205 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
206 "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL);
207 }
208 }
209
210 static int
sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)211 sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)
212 {
213 char buf[512];
214 struct sbuf sb;
215 struct umtxq_chain *uc;
216 u_int fract, i, j, tot, whole;
217 u_int sf0, sf1, sf2, sf3, sf4;
218 u_int si0, si1, si2, si3, si4;
219 u_int sw0, sw1, sw2, sw3, sw4;
220
221 sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN);
222 for (i = 0; i < 2; i++) {
223 tot = 0;
224 for (j = 0; j < UMTX_CHAINS; ++j) {
225 uc = &umtxq_chains[i][j];
226 mtx_lock(&uc->uc_lock);
227 tot += uc->max_length;
228 mtx_unlock(&uc->uc_lock);
229 }
230 if (tot == 0)
231 sbuf_printf(&sb, "%u) Empty ", i);
232 else {
233 sf0 = sf1 = sf2 = sf3 = sf4 = 0;
234 si0 = si1 = si2 = si3 = si4 = 0;
235 sw0 = sw1 = sw2 = sw3 = sw4 = 0;
236 for (j = 0; j < UMTX_CHAINS; j++) {
237 uc = &umtxq_chains[i][j];
238 mtx_lock(&uc->uc_lock);
239 whole = uc->max_length * 100;
240 mtx_unlock(&uc->uc_lock);
241 fract = (whole % tot) * 100;
242 if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) {
243 sf0 = fract;
244 si0 = j;
245 sw0 = whole;
246 } else if (UPROF_PERC_BIGGER(whole, fract, sw1,
247 sf1)) {
248 sf1 = fract;
249 si1 = j;
250 sw1 = whole;
251 } else if (UPROF_PERC_BIGGER(whole, fract, sw2,
252 sf2)) {
253 sf2 = fract;
254 si2 = j;
255 sw2 = whole;
256 } else if (UPROF_PERC_BIGGER(whole, fract, sw3,
257 sf3)) {
258 sf3 = fract;
259 si3 = j;
260 sw3 = whole;
261 } else if (UPROF_PERC_BIGGER(whole, fract, sw4,
262 sf4)) {
263 sf4 = fract;
264 si4 = j;
265 sw4 = whole;
266 }
267 }
268 sbuf_printf(&sb, "queue %u:\n", i);
269 sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot,
270 sf0 / tot, si0);
271 sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot,
272 sf1 / tot, si1);
273 sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot,
274 sf2 / tot, si2);
275 sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot,
276 sf3 / tot, si3);
277 sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot,
278 sf4 / tot, si4);
279 }
280 }
281 sbuf_trim(&sb);
282 sbuf_finish(&sb);
283 sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
284 sbuf_delete(&sb);
285 return (0);
286 }
287
288 static int
sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)289 sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)
290 {
291 struct umtxq_chain *uc;
292 u_int i, j;
293 int clear, error;
294
295 clear = 0;
296 error = sysctl_handle_int(oidp, &clear, 0, req);
297 if (error != 0 || req->newptr == NULL)
298 return (error);
299
300 if (clear != 0) {
301 for (i = 0; i < 2; ++i) {
302 for (j = 0; j < UMTX_CHAINS; ++j) {
303 uc = &umtxq_chains[i][j];
304 mtx_lock(&uc->uc_lock);
305 uc->length = 0;
306 uc->max_length = 0;
307 mtx_unlock(&uc->uc_lock);
308 }
309 }
310 }
311 return (0);
312 }
313
314 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear,
315 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
316 sysctl_debug_umtx_chains_clear, "I",
317 "Clear umtx chains statistics");
318 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks,
319 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0,
320 sysctl_debug_umtx_chains_peaks, "A",
321 "Highest peaks in chains max length");
322 #endif
323
324 static void
umtxq_sysinit(void * arg __unused)325 umtxq_sysinit(void *arg __unused)
326 {
327 int i, j;
328
329 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
330 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
331 for (i = 0; i < 2; ++i) {
332 for (j = 0; j < UMTX_CHAINS; ++j) {
333 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
334 MTX_DEF | MTX_DUPOK);
335 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
336 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
337 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
338 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
339 umtxq_chains[i][j].uc_busy = 0;
340 umtxq_chains[i][j].uc_waiters = 0;
341 #ifdef UMTX_PROFILING
342 umtxq_chains[i][j].length = 0;
343 umtxq_chains[i][j].max_length = 0;
344 #endif
345 }
346 }
347 #ifdef UMTX_PROFILING
348 umtx_init_profiling();
349 #endif
350 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_DEF);
351 umtx_shm_init();
352 }
353
354 struct umtx_q *
umtxq_alloc(void)355 umtxq_alloc(void)
356 {
357 struct umtx_q *uq;
358
359 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
360 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
361 M_WAITOK | M_ZERO);
362 TAILQ_INIT(&uq->uq_spare_queue->head);
363 TAILQ_INIT(&uq->uq_pi_contested);
364 uq->uq_inherited_pri = PRI_MAX;
365 return (uq);
366 }
367
368 void
umtxq_free(struct umtx_q * uq)369 umtxq_free(struct umtx_q *uq)
370 {
371
372 MPASS(uq->uq_spare_queue != NULL);
373 free(uq->uq_spare_queue, M_UMTX);
374 free(uq, M_UMTX);
375 }
376
377 static inline void
umtxq_hash(struct umtx_key * key)378 umtxq_hash(struct umtx_key *key)
379 {
380 unsigned n;
381
382 n = (uintptr_t)key->info.both.a + key->info.both.b;
383 key->hash = ((n * GOLDEN_RATIO_32) >> UMTX_SHIFTS) % UMTX_CHAINS;
384 }
385
386 struct umtxq_chain *
umtxq_getchain(struct umtx_key * key)387 umtxq_getchain(struct umtx_key *key)
388 {
389
390 if (key->type <= TYPE_SEM)
391 return (&umtxq_chains[1][key->hash]);
392 return (&umtxq_chains[0][key->hash]);
393 }
394
395 /*
396 * Set chain to busy state when following operation
397 * may be blocked (kernel mutex can not be used).
398 */
399 void
umtxq_busy(struct umtx_key * key)400 umtxq_busy(struct umtx_key *key)
401 {
402 struct umtxq_chain *uc;
403
404 uc = umtxq_getchain(key);
405 mtx_assert(&uc->uc_lock, MA_OWNED);
406 if (uc->uc_busy) {
407 #ifdef SMP
408 if (smp_cpus > 1) {
409 int count = BUSY_SPINS;
410 if (count > 0) {
411 umtxq_unlock(key);
412 while (uc->uc_busy && --count > 0)
413 cpu_spinwait();
414 umtxq_lock(key);
415 }
416 }
417 #endif
418 while (uc->uc_busy) {
419 uc->uc_waiters++;
420 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
421 uc->uc_waiters--;
422 }
423 }
424 uc->uc_busy = 1;
425 }
426
427 /*
428 * Unbusy a chain.
429 */
430 void
umtxq_unbusy(struct umtx_key * key)431 umtxq_unbusy(struct umtx_key *key)
432 {
433 struct umtxq_chain *uc;
434
435 uc = umtxq_getchain(key);
436 mtx_assert(&uc->uc_lock, MA_OWNED);
437 KASSERT(uc->uc_busy != 0, ("not busy"));
438 uc->uc_busy = 0;
439 if (uc->uc_waiters)
440 wakeup_one(uc);
441 }
442
443 void
umtxq_busy_unlocked(struct umtx_key * key)444 umtxq_busy_unlocked(struct umtx_key *key)
445 {
446 umtxq_lock(key);
447 umtxq_busy(key);
448 umtxq_unlock(key);
449 }
450
451 void
umtxq_unbusy_unlocked(struct umtx_key * key)452 umtxq_unbusy_unlocked(struct umtx_key *key)
453 {
454 umtxq_lock(key);
455 umtxq_unbusy(key);
456 umtxq_unlock(key);
457 }
458
459 static struct umtxq_queue *
umtxq_queue_lookup(struct umtx_key * key,int q)460 umtxq_queue_lookup(struct umtx_key *key, int q)
461 {
462 struct umtxq_queue *uh;
463 struct umtxq_chain *uc;
464
465 uc = umtxq_getchain(key);
466 UMTXQ_LOCKED_ASSERT(uc);
467 LIST_FOREACH(uh, &uc->uc_queue[q], link) {
468 if (umtx_key_match(&uh->key, key))
469 return (uh);
470 }
471
472 return (NULL);
473 }
474
475 void
umtxq_insert_queue(struct umtx_q * uq,int q)476 umtxq_insert_queue(struct umtx_q *uq, int q)
477 {
478 struct umtxq_queue *uh;
479 struct umtxq_chain *uc;
480
481 uc = umtxq_getchain(&uq->uq_key);
482 UMTXQ_LOCKED_ASSERT(uc);
483 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
484 uh = umtxq_queue_lookup(&uq->uq_key, q);
485 if (uh != NULL) {
486 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
487 } else {
488 uh = uq->uq_spare_queue;
489 uh->key = uq->uq_key;
490 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
491 #ifdef UMTX_PROFILING
492 uc->length++;
493 if (uc->length > uc->max_length) {
494 uc->max_length = uc->length;
495 if (uc->max_length > max_length)
496 max_length = uc->max_length;
497 }
498 #endif
499 }
500 uq->uq_spare_queue = NULL;
501
502 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
503 uh->length++;
504 uq->uq_flags |= UQF_UMTXQ;
505 uq->uq_cur_queue = uh;
506 return;
507 }
508
509 void
umtxq_remove_queue(struct umtx_q * uq,int q)510 umtxq_remove_queue(struct umtx_q *uq, int q)
511 {
512 struct umtxq_chain *uc;
513 struct umtxq_queue *uh;
514
515 uc = umtxq_getchain(&uq->uq_key);
516 UMTXQ_LOCKED_ASSERT(uc);
517 if (uq->uq_flags & UQF_UMTXQ) {
518 uh = uq->uq_cur_queue;
519 TAILQ_REMOVE(&uh->head, uq, uq_link);
520 uh->length--;
521 uq->uq_flags &= ~UQF_UMTXQ;
522 if (TAILQ_EMPTY(&uh->head)) {
523 KASSERT(uh->length == 0,
524 ("inconsistent umtxq_queue length"));
525 #ifdef UMTX_PROFILING
526 uc->length--;
527 #endif
528 LIST_REMOVE(uh, link);
529 } else {
530 uh = LIST_FIRST(&uc->uc_spare_queue);
531 KASSERT(uh != NULL, ("uc_spare_queue is empty"));
532 LIST_REMOVE(uh, link);
533 }
534 uq->uq_spare_queue = uh;
535 uq->uq_cur_queue = NULL;
536 }
537 }
538
539 /*
540 * Check if there are multiple waiters
541 */
542 int
umtxq_count(struct umtx_key * key)543 umtxq_count(struct umtx_key *key)
544 {
545 struct umtxq_queue *uh;
546
547 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
548 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
549 if (uh != NULL)
550 return (uh->length);
551 return (0);
552 }
553
554 /*
555 * Check if there are multiple PI waiters and returns first
556 * waiter.
557 */
558 static int
umtxq_count_pi(struct umtx_key * key,struct umtx_q ** first)559 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
560 {
561 struct umtxq_queue *uh;
562
563 *first = NULL;
564 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
565 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
566 if (uh != NULL) {
567 *first = TAILQ_FIRST(&uh->head);
568 return (uh->length);
569 }
570 return (0);
571 }
572
573 /*
574 * Wake up threads waiting on an userland object by a bit mask.
575 */
576 int
umtxq_signal_mask(struct umtx_key * key,int n_wake,u_int bitset)577 umtxq_signal_mask(struct umtx_key *key, int n_wake, u_int bitset)
578 {
579 struct umtxq_queue *uh;
580 struct umtx_q *uq, *uq_temp;
581 int ret;
582
583 ret = 0;
584 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
585 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
586 if (uh == NULL)
587 return (0);
588 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
589 if ((uq->uq_bitset & bitset) == 0)
590 continue;
591 umtxq_remove_queue(uq, UMTX_SHARED_QUEUE);
592 wakeup_one(uq);
593 if (++ret >= n_wake)
594 break;
595 }
596 return (ret);
597 }
598
599 /*
600 * Wake up threads waiting on an userland object.
601 */
602
603 static int
umtxq_signal_queue(struct umtx_key * key,int n_wake,int q)604 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
605 {
606 struct umtxq_queue *uh;
607 struct umtx_q *uq;
608 int ret;
609
610 ret = 0;
611 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
612 uh = umtxq_queue_lookup(key, q);
613 if (uh != NULL) {
614 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
615 umtxq_remove_queue(uq, q);
616 wakeup(uq);
617 if (++ret >= n_wake)
618 return (ret);
619 }
620 }
621 return (ret);
622 }
623
624 /*
625 * Wake up specified thread.
626 */
627 static inline void
umtxq_signal_thread(struct umtx_q * uq)628 umtxq_signal_thread(struct umtx_q *uq)
629 {
630
631 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
632 umtxq_remove(uq);
633 wakeup(uq);
634 }
635
636 /*
637 * Wake up a maximum of n_wake threads that are waiting on an userland
638 * object identified by key. The remaining threads are removed from queue
639 * identified by key and added to the queue identified by key2 (requeued).
640 * The n_requeue specifies an upper limit on the number of threads that
641 * are requeued to the second queue.
642 */
643 int
umtxq_requeue(struct umtx_key * key,int n_wake,struct umtx_key * key2,int n_requeue)644 umtxq_requeue(struct umtx_key *key, int n_wake, struct umtx_key *key2,
645 int n_requeue)
646 {
647 struct umtxq_queue *uh;
648 struct umtx_q *uq, *uq_temp;
649 int ret;
650
651 ret = 0;
652 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
653 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key2));
654 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
655 if (uh == NULL)
656 return (0);
657 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
658 if (++ret <= n_wake) {
659 umtxq_remove(uq);
660 wakeup_one(uq);
661 } else {
662 umtxq_remove(uq);
663 uq->uq_key = *key2;
664 umtxq_insert(uq);
665 if (ret - n_wake == n_requeue)
666 break;
667 }
668 }
669 return (ret);
670 }
671
672 static inline int
tstohz(const struct timespec * tsp)673 tstohz(const struct timespec *tsp)
674 {
675 struct timeval tv;
676
677 TIMESPEC_TO_TIMEVAL(&tv, tsp);
678 return tvtohz(&tv);
679 }
680
681 void
umtx_abs_timeout_init(struct umtx_abs_timeout * timo,int clockid,int absolute,const struct timespec * timeout)682 umtx_abs_timeout_init(struct umtx_abs_timeout *timo, int clockid,
683 int absolute, const struct timespec *timeout)
684 {
685
686 timo->clockid = clockid;
687 if (!absolute) {
688 timo->is_abs_real = false;
689 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
690 timespecadd(&timo->cur, timeout, &timo->end);
691 } else {
692 timo->end = *timeout;
693 timo->is_abs_real = clockid == CLOCK_REALTIME ||
694 clockid == CLOCK_REALTIME_FAST ||
695 clockid == CLOCK_REALTIME_PRECISE ||
696 clockid == CLOCK_TAI ||
697 clockid == CLOCK_SECOND;
698 }
699 }
700
701 static void
umtx_abs_timeout_init2(struct umtx_abs_timeout * timo,const struct _umtx_time * umtxtime)702 umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
703 const struct _umtx_time *umtxtime)
704 {
705
706 umtx_abs_timeout_init(timo, umtxtime->_clockid,
707 (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout);
708 }
709
710 static void
umtx_abs_timeout_enforce_min(sbintime_t * sbt)711 umtx_abs_timeout_enforce_min(sbintime_t *sbt)
712 {
713 sbintime_t when, mint;
714
715 mint = curproc->p_umtx_min_timeout;
716 if (__predict_false(mint != 0)) {
717 when = sbinuptime() + mint;
718 if (*sbt < when)
719 *sbt = when;
720 }
721 }
722
723 static int
umtx_abs_timeout_getsbt(struct umtx_abs_timeout * timo,sbintime_t * sbt,int * flags)724 umtx_abs_timeout_getsbt(struct umtx_abs_timeout *timo, sbintime_t *sbt,
725 int *flags)
726 {
727 struct bintime bt, bbt;
728 struct timespec tts;
729 sbintime_t rem;
730
731 switch (timo->clockid) {
732
733 /* Clocks that can be converted into absolute time. */
734 case CLOCK_REALTIME:
735 case CLOCK_REALTIME_PRECISE:
736 case CLOCK_REALTIME_FAST:
737 case CLOCK_MONOTONIC:
738 case CLOCK_MONOTONIC_PRECISE:
739 case CLOCK_MONOTONIC_FAST:
740 case CLOCK_UPTIME:
741 case CLOCK_UPTIME_PRECISE:
742 case CLOCK_UPTIME_FAST:
743 case CLOCK_SECOND:
744 timespec2bintime(&timo->end, &bt);
745 switch (timo->clockid) {
746 case CLOCK_REALTIME:
747 case CLOCK_REALTIME_PRECISE:
748 case CLOCK_REALTIME_FAST:
749 case CLOCK_SECOND:
750 getboottimebin(&bbt);
751 bintime_sub(&bt, &bbt);
752 break;
753 }
754 if (bt.sec < 0)
755 return (ETIMEDOUT);
756 if (bt.sec >= (SBT_MAX >> 32)) {
757 *sbt = 0;
758 *flags = 0;
759 return (0);
760 }
761 *sbt = bttosbt(bt);
762 umtx_abs_timeout_enforce_min(sbt);
763
764 /*
765 * Check if the absolute time should be aligned to
766 * avoid firing multiple timer events in non-periodic
767 * timer mode.
768 */
769 switch (timo->clockid) {
770 case CLOCK_REALTIME_FAST:
771 case CLOCK_MONOTONIC_FAST:
772 case CLOCK_UPTIME_FAST:
773 rem = *sbt % tc_tick_sbt;
774 if (__predict_true(rem != 0))
775 *sbt += tc_tick_sbt - rem;
776 break;
777 case CLOCK_SECOND:
778 rem = *sbt % SBT_1S;
779 if (__predict_true(rem != 0))
780 *sbt += SBT_1S - rem;
781 break;
782 }
783 *flags = C_ABSOLUTE;
784 return (0);
785
786 /* Clocks that has to be periodically polled. */
787 case CLOCK_VIRTUAL:
788 case CLOCK_PROF:
789 case CLOCK_THREAD_CPUTIME_ID:
790 case CLOCK_PROCESS_CPUTIME_ID:
791 case CLOCK_TAI: /* Boot time is not necessarily stable in TAI */
792 default:
793 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
794 if (timespeccmp(&timo->end, &timo->cur, <=))
795 return (ETIMEDOUT);
796 timespecsub(&timo->end, &timo->cur, &tts);
797 *sbt = tick_sbt * tstohz(&tts);
798 *flags = C_HARDCLOCK;
799 return (0);
800 }
801 }
802
803 static uint32_t
umtx_unlock_val(uint32_t flags,bool rb)804 umtx_unlock_val(uint32_t flags, bool rb)
805 {
806
807 if (rb)
808 return (UMUTEX_RB_OWNERDEAD);
809 else if ((flags & UMUTEX_NONCONSISTENT) != 0)
810 return (UMUTEX_RB_NOTRECOV);
811 else
812 return (UMUTEX_UNOWNED);
813
814 }
815
816 /*
817 * Put thread into sleep state, before sleeping, check if
818 * thread was removed from umtx queue.
819 */
820 int
umtxq_sleep(struct umtx_q * uq,const char * wmesg,struct umtx_abs_timeout * timo)821 umtxq_sleep(struct umtx_q *uq, const char *wmesg,
822 struct umtx_abs_timeout *timo)
823 {
824 struct umtxq_chain *uc;
825 sbintime_t sbt = 0;
826 int error, flags = 0;
827
828 uc = umtxq_getchain(&uq->uq_key);
829 UMTXQ_LOCKED_ASSERT(uc);
830 for (;;) {
831 if (!(uq->uq_flags & UQF_UMTXQ)) {
832 error = 0;
833 break;
834 }
835 if (timo != NULL) {
836 if (timo->is_abs_real)
837 curthread->td_rtcgen =
838 atomic_load_acq_int(&rtc_generation);
839 error = umtx_abs_timeout_getsbt(timo, &sbt, &flags);
840 if (error != 0)
841 break;
842 }
843 error = msleep_sbt(uq, &uc->uc_lock, PCATCH | PDROP, wmesg,
844 sbt, 0, flags);
845 uc = umtxq_getchain(&uq->uq_key);
846 mtx_lock(&uc->uc_lock);
847 if (error == EINTR || error == ERESTART)
848 break;
849 if (error == EWOULDBLOCK && (flags & C_ABSOLUTE) != 0) {
850 error = ETIMEDOUT;
851 break;
852 }
853 }
854
855 curthread->td_rtcgen = 0;
856 return (error);
857 }
858
859 /*
860 * Convert userspace address into unique logical address.
861 */
862 int
umtx_key_get(const void * addr,int type,int share,struct umtx_key * key)863 umtx_key_get(const void *addr, int type, int share, struct umtx_key *key)
864 {
865 struct thread *td = curthread;
866 vm_map_t map;
867 vm_map_entry_t entry;
868 vm_pindex_t pindex;
869 vm_prot_t prot;
870 boolean_t wired;
871
872 key->type = type;
873 if (share == THREAD_SHARE) {
874 key->shared = 0;
875 key->info.private.vs = td->td_proc->p_vmspace;
876 key->info.private.addr = (uintptr_t)addr;
877 } else {
878 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
879 map = &td->td_proc->p_vmspace->vm_map;
880 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
881 &entry, &key->info.shared.object, &pindex, &prot,
882 &wired) != KERN_SUCCESS) {
883 return (EFAULT);
884 }
885
886 if ((share == PROCESS_SHARE) ||
887 (share == AUTO_SHARE &&
888 VM_INHERIT_SHARE == entry->inheritance)) {
889 key->shared = 1;
890 key->info.shared.offset = (vm_offset_t)addr -
891 entry->start + entry->offset;
892 vm_object_reference(key->info.shared.object);
893 } else {
894 key->shared = 0;
895 key->info.private.vs = td->td_proc->p_vmspace;
896 key->info.private.addr = (uintptr_t)addr;
897 }
898 vm_map_lookup_done(map, entry);
899 }
900
901 umtxq_hash(key);
902 return (0);
903 }
904
905 /*
906 * Release key.
907 */
908 void
umtx_key_release(struct umtx_key * key)909 umtx_key_release(struct umtx_key *key)
910 {
911 if (key->shared)
912 vm_object_deallocate(key->info.shared.object);
913 }
914
915 #ifdef COMPAT_FREEBSD10
916 /*
917 * Lock a umtx object.
918 */
919 static int
do_lock_umtx(struct thread * td,struct umtx * umtx,u_long id,const struct timespec * timeout)920 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
921 const struct timespec *timeout)
922 {
923 struct umtx_abs_timeout timo;
924 struct umtx_q *uq;
925 u_long owner;
926 u_long old;
927 int error = 0;
928
929 uq = td->td_umtxq;
930 if (timeout != NULL)
931 umtx_abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout);
932
933 /*
934 * Care must be exercised when dealing with umtx structure. It
935 * can fault on any access.
936 */
937 for (;;) {
938 /*
939 * Try the uncontested case. This should be done in userland.
940 */
941 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
942
943 /* The acquire succeeded. */
944 if (owner == UMTX_UNOWNED)
945 return (0);
946
947 /* The address was invalid. */
948 if (owner == -1)
949 return (EFAULT);
950
951 /* If no one owns it but it is contested try to acquire it. */
952 if (owner == UMTX_CONTESTED) {
953 owner = casuword(&umtx->u_owner,
954 UMTX_CONTESTED, id | UMTX_CONTESTED);
955
956 if (owner == UMTX_CONTESTED)
957 return (0);
958
959 /* The address was invalid. */
960 if (owner == -1)
961 return (EFAULT);
962
963 error = thread_check_susp(td, false);
964 if (error != 0)
965 break;
966
967 /* If this failed the lock has changed, restart. */
968 continue;
969 }
970
971 /*
972 * If we caught a signal, we have retried and now
973 * exit immediately.
974 */
975 if (error != 0)
976 break;
977
978 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
979 AUTO_SHARE, &uq->uq_key)) != 0)
980 return (error);
981
982 umtxq_lock(&uq->uq_key);
983 umtxq_busy(&uq->uq_key);
984 umtxq_insert(uq);
985 umtxq_unbusy(&uq->uq_key);
986 umtxq_unlock(&uq->uq_key);
987
988 /*
989 * Set the contested bit so that a release in user space
990 * knows to use the system call for unlock. If this fails
991 * either some one else has acquired the lock or it has been
992 * released.
993 */
994 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
995
996 /* The address was invalid. */
997 if (old == -1) {
998 umtxq_lock(&uq->uq_key);
999 umtxq_remove(uq);
1000 umtxq_unlock(&uq->uq_key);
1001 umtx_key_release(&uq->uq_key);
1002 return (EFAULT);
1003 }
1004
1005 /*
1006 * We set the contested bit, sleep. Otherwise the lock changed
1007 * and we need to retry or we lost a race to the thread
1008 * unlocking the umtx.
1009 */
1010 umtxq_lock(&uq->uq_key);
1011 if (old == owner)
1012 error = umtxq_sleep(uq, "umtx", timeout == NULL ? NULL :
1013 &timo);
1014 umtxq_remove(uq);
1015 umtxq_unlock(&uq->uq_key);
1016 umtx_key_release(&uq->uq_key);
1017
1018 if (error == 0)
1019 error = thread_check_susp(td, false);
1020 }
1021
1022 if (timeout == NULL) {
1023 /* Mutex locking is restarted if it is interrupted. */
1024 if (error == EINTR)
1025 error = ERESTART;
1026 } else {
1027 /* Timed-locking is not restarted. */
1028 if (error == ERESTART)
1029 error = EINTR;
1030 }
1031 return (error);
1032 }
1033
1034 /*
1035 * Unlock a umtx object.
1036 */
1037 static int
do_unlock_umtx(struct thread * td,struct umtx * umtx,u_long id)1038 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
1039 {
1040 struct umtx_key key;
1041 u_long owner;
1042 u_long old;
1043 int error;
1044 int count;
1045
1046 /*
1047 * Make sure we own this mtx.
1048 */
1049 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
1050 if (owner == -1)
1051 return (EFAULT);
1052
1053 if ((owner & ~UMTX_CONTESTED) != id)
1054 return (EPERM);
1055
1056 /* This should be done in userland */
1057 if ((owner & UMTX_CONTESTED) == 0) {
1058 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
1059 if (old == -1)
1060 return (EFAULT);
1061 if (old == owner)
1062 return (0);
1063 owner = old;
1064 }
1065
1066 /* We should only ever be in here for contested locks */
1067 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
1068 &key)) != 0)
1069 return (error);
1070
1071 umtxq_lock(&key);
1072 umtxq_busy(&key);
1073 count = umtxq_count(&key);
1074 umtxq_unlock(&key);
1075
1076 /*
1077 * When unlocking the umtx, it must be marked as unowned if
1078 * there is zero or one thread only waiting for it.
1079 * Otherwise, it must be marked as contested.
1080 */
1081 old = casuword(&umtx->u_owner, owner,
1082 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
1083 umtxq_lock(&key);
1084 umtxq_signal(&key,1);
1085 umtxq_unbusy(&key);
1086 umtxq_unlock(&key);
1087 umtx_key_release(&key);
1088 if (old == -1)
1089 return (EFAULT);
1090 if (old != owner)
1091 return (EINVAL);
1092 return (0);
1093 }
1094
1095 #ifdef COMPAT_FREEBSD32
1096
1097 /*
1098 * Lock a umtx object.
1099 */
1100 static int
do_lock_umtx32(struct thread * td,uint32_t * m,uint32_t id,const struct timespec * timeout)1101 do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id,
1102 const struct timespec *timeout)
1103 {
1104 struct umtx_abs_timeout timo;
1105 struct umtx_q *uq;
1106 uint32_t owner;
1107 uint32_t old;
1108 int error = 0;
1109
1110 uq = td->td_umtxq;
1111
1112 if (timeout != NULL)
1113 umtx_abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout);
1114
1115 /*
1116 * Care must be exercised when dealing with umtx structure. It
1117 * can fault on any access.
1118 */
1119 for (;;) {
1120 /*
1121 * Try the uncontested case. This should be done in userland.
1122 */
1123 owner = casuword32(m, UMUTEX_UNOWNED, id);
1124
1125 /* The acquire succeeded. */
1126 if (owner == UMUTEX_UNOWNED)
1127 return (0);
1128
1129 /* The address was invalid. */
1130 if (owner == -1)
1131 return (EFAULT);
1132
1133 /* If no one owns it but it is contested try to acquire it. */
1134 if (owner == UMUTEX_CONTESTED) {
1135 owner = casuword32(m,
1136 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1137 if (owner == UMUTEX_CONTESTED)
1138 return (0);
1139
1140 /* The address was invalid. */
1141 if (owner == -1)
1142 return (EFAULT);
1143
1144 error = thread_check_susp(td, false);
1145 if (error != 0)
1146 break;
1147
1148 /* If this failed the lock has changed, restart. */
1149 continue;
1150 }
1151
1152 /*
1153 * If we caught a signal, we have retried and now
1154 * exit immediately.
1155 */
1156 if (error != 0)
1157 return (error);
1158
1159 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
1160 AUTO_SHARE, &uq->uq_key)) != 0)
1161 return (error);
1162
1163 umtxq_lock(&uq->uq_key);
1164 umtxq_busy(&uq->uq_key);
1165 umtxq_insert(uq);
1166 umtxq_unbusy(&uq->uq_key);
1167 umtxq_unlock(&uq->uq_key);
1168
1169 /*
1170 * Set the contested bit so that a release in user space
1171 * knows to use the system call for unlock. If this fails
1172 * either some one else has acquired the lock or it has been
1173 * released.
1174 */
1175 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
1176
1177 /* The address was invalid. */
1178 if (old == -1) {
1179 umtxq_lock(&uq->uq_key);
1180 umtxq_remove(uq);
1181 umtxq_unlock(&uq->uq_key);
1182 umtx_key_release(&uq->uq_key);
1183 return (EFAULT);
1184 }
1185
1186 /*
1187 * We set the contested bit, sleep. Otherwise the lock changed
1188 * and we need to retry or we lost a race to the thread
1189 * unlocking the umtx.
1190 */
1191 umtxq_lock(&uq->uq_key);
1192 if (old == owner)
1193 error = umtxq_sleep(uq, "umtx", timeout == NULL ?
1194 NULL : &timo);
1195 umtxq_remove(uq);
1196 umtxq_unlock(&uq->uq_key);
1197 umtx_key_release(&uq->uq_key);
1198
1199 if (error == 0)
1200 error = thread_check_susp(td, false);
1201 }
1202
1203 if (timeout == NULL) {
1204 /* Mutex locking is restarted if it is interrupted. */
1205 if (error == EINTR)
1206 error = ERESTART;
1207 } else {
1208 /* Timed-locking is not restarted. */
1209 if (error == ERESTART)
1210 error = EINTR;
1211 }
1212 return (error);
1213 }
1214
1215 /*
1216 * Unlock a umtx object.
1217 */
1218 static int
do_unlock_umtx32(struct thread * td,uint32_t * m,uint32_t id)1219 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
1220 {
1221 struct umtx_key key;
1222 uint32_t owner;
1223 uint32_t old;
1224 int error;
1225 int count;
1226
1227 /*
1228 * Make sure we own this mtx.
1229 */
1230 owner = fuword32(m);
1231 if (owner == -1)
1232 return (EFAULT);
1233
1234 if ((owner & ~UMUTEX_CONTESTED) != id)
1235 return (EPERM);
1236
1237 /* This should be done in userland */
1238 if ((owner & UMUTEX_CONTESTED) == 0) {
1239 old = casuword32(m, owner, UMUTEX_UNOWNED);
1240 if (old == -1)
1241 return (EFAULT);
1242 if (old == owner)
1243 return (0);
1244 owner = old;
1245 }
1246
1247 /* We should only ever be in here for contested locks */
1248 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
1249 &key)) != 0)
1250 return (error);
1251
1252 umtxq_lock(&key);
1253 umtxq_busy(&key);
1254 count = umtxq_count(&key);
1255 umtxq_unlock(&key);
1256
1257 /*
1258 * When unlocking the umtx, it must be marked as unowned if
1259 * there is zero or one thread only waiting for it.
1260 * Otherwise, it must be marked as contested.
1261 */
1262 old = casuword32(m, owner,
1263 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1264 umtxq_lock(&key);
1265 umtxq_signal(&key,1);
1266 umtxq_unbusy(&key);
1267 umtxq_unlock(&key);
1268 umtx_key_release(&key);
1269 if (old == -1)
1270 return (EFAULT);
1271 if (old != owner)
1272 return (EINVAL);
1273 return (0);
1274 }
1275 #endif /* COMPAT_FREEBSD32 */
1276 #endif /* COMPAT_FREEBSD10 */
1277
1278 /*
1279 * Fetch and compare value, sleep on the address if value is not changed.
1280 */
1281 static int
do_wait(struct thread * td,void * addr,u_long id,struct _umtx_time * timeout,int compat32,int is_private)1282 do_wait(struct thread *td, void *addr, u_long id,
1283 struct _umtx_time *timeout, int compat32, int is_private)
1284 {
1285 struct umtx_abs_timeout timo;
1286 struct umtx_q *uq;
1287 u_long tmp;
1288 uint32_t tmp32;
1289 int error = 0;
1290
1291 uq = td->td_umtxq;
1292 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
1293 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1294 return (error);
1295
1296 if (timeout != NULL)
1297 umtx_abs_timeout_init2(&timo, timeout);
1298
1299 umtxq_lock(&uq->uq_key);
1300 umtxq_insert(uq);
1301 umtxq_unlock(&uq->uq_key);
1302 if (compat32 == 0) {
1303 error = fueword(addr, &tmp);
1304 if (error != 0)
1305 error = EFAULT;
1306 } else {
1307 error = fueword32(addr, &tmp32);
1308 if (error == 0)
1309 tmp = tmp32;
1310 else
1311 error = EFAULT;
1312 }
1313 umtxq_lock(&uq->uq_key);
1314 if (error == 0) {
1315 if (tmp == id)
1316 error = umtxq_sleep(uq, "uwait", timeout == NULL ?
1317 NULL : &timo);
1318 if ((uq->uq_flags & UQF_UMTXQ) == 0)
1319 error = 0;
1320 else
1321 umtxq_remove(uq);
1322 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
1323 umtxq_remove(uq);
1324 }
1325 umtxq_unlock(&uq->uq_key);
1326 umtx_key_release(&uq->uq_key);
1327 if (error == ERESTART)
1328 error = EINTR;
1329 return (error);
1330 }
1331
1332 /*
1333 * Wake up threads sleeping on the specified address.
1334 */
1335 int
kern_umtx_wake(struct thread * td,void * uaddr,int n_wake,int is_private)1336 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
1337 {
1338 struct umtx_key key;
1339 int ret;
1340
1341 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
1342 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
1343 return (ret);
1344 umtxq_lock(&key);
1345 umtxq_signal(&key, n_wake);
1346 umtxq_unlock(&key);
1347 umtx_key_release(&key);
1348 return (0);
1349 }
1350
1351 /*
1352 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1353 */
1354 static int
do_lock_normal(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int mode)1355 do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
1356 struct _umtx_time *timeout, int mode)
1357 {
1358 struct umtx_abs_timeout timo;
1359 struct umtx_q *uq;
1360 uint32_t owner, old, id;
1361 int error, rv;
1362
1363 id = td->td_tid;
1364 uq = td->td_umtxq;
1365 error = 0;
1366 if (timeout != NULL)
1367 umtx_abs_timeout_init2(&timo, timeout);
1368
1369 /*
1370 * Care must be exercised when dealing with umtx structure. It
1371 * can fault on any access.
1372 */
1373 for (;;) {
1374 rv = fueword32(&m->m_owner, &owner);
1375 if (rv == -1)
1376 return (EFAULT);
1377 if (mode == _UMUTEX_WAIT) {
1378 if (owner == UMUTEX_UNOWNED ||
1379 owner == UMUTEX_CONTESTED ||
1380 owner == UMUTEX_RB_OWNERDEAD ||
1381 owner == UMUTEX_RB_NOTRECOV)
1382 return (0);
1383 } else {
1384 /*
1385 * Robust mutex terminated. Kernel duty is to
1386 * return EOWNERDEAD to the userspace. The
1387 * umutex.m_flags UMUTEX_NONCONSISTENT is set
1388 * by the common userspace code.
1389 */
1390 if (owner == UMUTEX_RB_OWNERDEAD) {
1391 rv = casueword32(&m->m_owner,
1392 UMUTEX_RB_OWNERDEAD, &owner,
1393 id | UMUTEX_CONTESTED);
1394 if (rv == -1)
1395 return (EFAULT);
1396 if (rv == 0) {
1397 MPASS(owner == UMUTEX_RB_OWNERDEAD);
1398 return (EOWNERDEAD); /* success */
1399 }
1400 MPASS(rv == 1);
1401 rv = thread_check_susp(td, false);
1402 if (rv != 0)
1403 return (rv);
1404 continue;
1405 }
1406 if (owner == UMUTEX_RB_NOTRECOV)
1407 return (ENOTRECOVERABLE);
1408
1409 /*
1410 * Try the uncontested case. This should be
1411 * done in userland.
1412 */
1413 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
1414 &owner, id);
1415 /* The address was invalid. */
1416 if (rv == -1)
1417 return (EFAULT);
1418
1419 /* The acquire succeeded. */
1420 if (rv == 0) {
1421 MPASS(owner == UMUTEX_UNOWNED);
1422 return (0);
1423 }
1424
1425 /*
1426 * If no one owns it, but it is contested or
1427 * the CAS above failed spuriously (possible
1428 * on ll/sc architectures), try to acquire it.
1429 * Sleeping would be forever in the spurious
1430 * case: no owner exists to wake us.
1431 */
1432 MPASS(rv == 1);
1433 if (owner == UMUTEX_CONTESTED ||
1434 owner == UMUTEX_UNOWNED) {
1435 rv = casueword32(&m->m_owner, owner,
1436 &owner, id | UMUTEX_CONTESTED);
1437 /* The address was invalid. */
1438 if (rv == -1)
1439 return (EFAULT);
1440 if (rv == 0) {
1441 MPASS(owner == UMUTEX_CONTESTED ||
1442 owner == UMUTEX_UNOWNED);
1443 return (0);
1444 }
1445 if (rv == 1) {
1446 rv = thread_check_susp(td, false);
1447 if (rv != 0)
1448 return (rv);
1449 }
1450
1451 /*
1452 * If this failed the lock has
1453 * changed, restart.
1454 */
1455 continue;
1456 }
1457
1458 /* rv == 1 with a real owner, fall through to sleep. */
1459 rv = thread_check_susp(td, false);
1460 if (rv != 0)
1461 return (rv);
1462 }
1463
1464 if (mode == _UMUTEX_TRY)
1465 return (EBUSY);
1466
1467 /*
1468 * If we caught a signal, we have retried and now
1469 * exit immediately.
1470 */
1471 if (error != 0)
1472 return (error);
1473
1474 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1475 GET_SHARE(flags), &uq->uq_key)) != 0)
1476 return (error);
1477
1478 umtxq_lock(&uq->uq_key);
1479 umtxq_busy(&uq->uq_key);
1480 umtxq_insert(uq);
1481 umtxq_unlock(&uq->uq_key);
1482
1483 /*
1484 * Set the contested bit so that a release in user space
1485 * knows to use the system call for unlock. If this fails
1486 * either some one else has acquired the lock or it has been
1487 * released.
1488 */
1489 rv = casueword32(&m->m_owner, owner, &old,
1490 owner | UMUTEX_CONTESTED);
1491
1492 /* The address was invalid or casueword failed to store. */
1493 if (rv == -1 || rv == 1) {
1494 umtxq_lock(&uq->uq_key);
1495 umtxq_remove(uq);
1496 umtxq_unbusy(&uq->uq_key);
1497 umtxq_unlock(&uq->uq_key);
1498 umtx_key_release(&uq->uq_key);
1499 if (rv == -1)
1500 return (EFAULT);
1501 if (rv == 1) {
1502 rv = thread_check_susp(td, false);
1503 if (rv != 0)
1504 return (rv);
1505 }
1506 continue;
1507 }
1508
1509 /*
1510 * We set the contested bit, sleep. Otherwise the lock changed
1511 * and we need to retry or we lost a race to the thread
1512 * unlocking the umtx.
1513 */
1514 umtxq_lock(&uq->uq_key);
1515 umtxq_unbusy(&uq->uq_key);
1516 MPASS(old == owner);
1517 error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
1518 NULL : &timo);
1519 umtxq_remove(uq);
1520 umtxq_unlock(&uq->uq_key);
1521 umtx_key_release(&uq->uq_key);
1522
1523 if (error == 0)
1524 error = thread_check_susp(td, false);
1525 }
1526
1527 return (0);
1528 }
1529
1530 /*
1531 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1532 */
1533 static int
do_unlock_normal(struct thread * td,struct umutex * m,uint32_t flags,bool rb)1534 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
1535 {
1536 struct umtx_key key;
1537 uint32_t owner, old, id, newlock;
1538 int error, count;
1539
1540 id = td->td_tid;
1541
1542 again:
1543 /*
1544 * Make sure we own this mtx.
1545 */
1546 error = fueword32(&m->m_owner, &owner);
1547 if (error == -1)
1548 return (EFAULT);
1549
1550 if ((owner & ~UMUTEX_CONTESTED) != id)
1551 return (EPERM);
1552
1553 newlock = umtx_unlock_val(flags, rb);
1554 if ((owner & UMUTEX_CONTESTED) == 0) {
1555 error = casueword32(&m->m_owner, owner, &old, newlock);
1556 if (error == -1)
1557 return (EFAULT);
1558 if (error == 1) {
1559 error = thread_check_susp(td, false);
1560 if (error != 0)
1561 return (error);
1562 goto again;
1563 }
1564 MPASS(old == owner);
1565 return (0);
1566 }
1567
1568 /* We should only ever be in here for contested locks */
1569 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1570 &key)) != 0)
1571 return (error);
1572
1573 umtxq_lock(&key);
1574 umtxq_busy(&key);
1575 count = umtxq_count(&key);
1576 umtxq_unlock(&key);
1577
1578 /*
1579 * When unlocking the umtx, it must be marked as unowned if
1580 * there is zero or one thread only waiting for it.
1581 * Otherwise, it must be marked as contested.
1582 */
1583 if (count > 1)
1584 newlock |= UMUTEX_CONTESTED;
1585 error = casueword32(&m->m_owner, owner, &old, newlock);
1586 umtxq_lock(&key);
1587 umtxq_signal(&key, 1);
1588 umtxq_unbusy(&key);
1589 umtxq_unlock(&key);
1590 umtx_key_release(&key);
1591 if (error == -1)
1592 return (EFAULT);
1593 if (error == 1) {
1594 if (old != owner)
1595 return (EINVAL);
1596 error = thread_check_susp(td, false);
1597 if (error != 0)
1598 return (error);
1599 goto again;
1600 }
1601 return (0);
1602 }
1603
1604 /*
1605 * Check if the mutex is available and wake up a waiter,
1606 * only for simple mutex.
1607 */
1608 static int
do_wake_umutex(struct thread * td,struct umutex * m)1609 do_wake_umutex(struct thread *td, struct umutex *m)
1610 {
1611 struct umtx_key key;
1612 uint32_t owner;
1613 uint32_t flags;
1614 int error;
1615 int count;
1616
1617 again:
1618 error = fueword32(&m->m_owner, &owner);
1619 if (error == -1)
1620 return (EFAULT);
1621
1622 if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD &&
1623 owner != UMUTEX_RB_NOTRECOV)
1624 return (0);
1625
1626 error = fueword32(&m->m_flags, &flags);
1627 if (error == -1)
1628 return (EFAULT);
1629
1630 /* We should only ever be in here for contested locks */
1631 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1632 &key)) != 0)
1633 return (error);
1634
1635 umtxq_lock(&key);
1636 umtxq_busy(&key);
1637 count = umtxq_count(&key);
1638 umtxq_unlock(&key);
1639
1640 if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD &&
1641 owner != UMUTEX_RB_NOTRECOV) {
1642 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
1643 UMUTEX_UNOWNED);
1644 if (error == -1) {
1645 error = EFAULT;
1646 } else if (error == 1) {
1647 umtxq_lock(&key);
1648 umtxq_unbusy(&key);
1649 umtxq_unlock(&key);
1650 umtx_key_release(&key);
1651 error = thread_check_susp(td, false);
1652 if (error != 0)
1653 return (error);
1654 goto again;
1655 }
1656 }
1657
1658 umtxq_lock(&key);
1659 if (error == 0 && count != 0) {
1660 MPASS((owner & ~UMUTEX_CONTESTED) == 0 ||
1661 owner == UMUTEX_RB_OWNERDEAD ||
1662 owner == UMUTEX_RB_NOTRECOV);
1663 umtxq_signal(&key, 1);
1664 }
1665 umtxq_unbusy(&key);
1666 umtxq_unlock(&key);
1667 umtx_key_release(&key);
1668 return (error);
1669 }
1670
1671 /*
1672 * Check if the mutex has waiters and tries to fix contention bit.
1673 */
1674 static int
do_wake2_umutex(struct thread * td,struct umutex * m,uint32_t flags)1675 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags)
1676 {
1677 struct umtx_key key;
1678 uint32_t owner, old;
1679 int type;
1680 int error;
1681 int count;
1682
1683 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT |
1684 UMUTEX_ROBUST)) {
1685 case 0:
1686 case UMUTEX_ROBUST:
1687 type = TYPE_NORMAL_UMUTEX;
1688 break;
1689 case UMUTEX_PRIO_INHERIT:
1690 type = TYPE_PI_UMUTEX;
1691 break;
1692 case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST):
1693 type = TYPE_PI_ROBUST_UMUTEX;
1694 break;
1695 case UMUTEX_PRIO_PROTECT:
1696 type = TYPE_PP_UMUTEX;
1697 break;
1698 case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST):
1699 type = TYPE_PP_ROBUST_UMUTEX;
1700 break;
1701 default:
1702 return (EINVAL);
1703 }
1704 if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0)
1705 return (error);
1706
1707 owner = 0;
1708 umtxq_lock(&key);
1709 umtxq_busy(&key);
1710 count = umtxq_count(&key);
1711 umtxq_unlock(&key);
1712
1713 error = fueword32(&m->m_owner, &owner);
1714 if (error == -1)
1715 error = EFAULT;
1716
1717 /*
1718 * Only repair contention bit if there is a waiter, this means
1719 * the mutex is still being referenced by userland code,
1720 * otherwise don't update any memory.
1721 */
1722 while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 &&
1723 (count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) {
1724 error = casueword32(&m->m_owner, owner, &old,
1725 owner | UMUTEX_CONTESTED);
1726 if (error == -1) {
1727 error = EFAULT;
1728 break;
1729 }
1730 if (error == 0) {
1731 MPASS(old == owner);
1732 break;
1733 }
1734 owner = old;
1735 error = thread_check_susp(td, false);
1736 }
1737
1738 umtxq_lock(&key);
1739 if (error == EFAULT) {
1740 umtxq_signal(&key, INT_MAX);
1741 } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 ||
1742 owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV))
1743 umtxq_signal(&key, 1);
1744 umtxq_unbusy(&key);
1745 umtxq_unlock(&key);
1746 umtx_key_release(&key);
1747 return (error);
1748 }
1749
1750 struct umtx_pi *
umtx_pi_alloc(int flags)1751 umtx_pi_alloc(int flags)
1752 {
1753 struct umtx_pi *pi;
1754
1755 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1756 if (pi == NULL)
1757 return (NULL);
1758
1759 TAILQ_INIT(&pi->pi_blocked);
1760 atomic_add_int(&umtx_pi_allocated, 1);
1761 return (pi);
1762 }
1763
1764 void
umtx_pi_free(struct umtx_pi * pi)1765 umtx_pi_free(struct umtx_pi *pi)
1766 {
1767 uma_zfree(umtx_pi_zone, pi);
1768 atomic_add_int(&umtx_pi_allocated, -1);
1769 }
1770
1771 /*
1772 * Adjust the thread's position on a pi_state after its priority has been
1773 * changed.
1774 */
1775 static int
umtx_pi_adjust_thread(struct umtx_pi * pi,struct thread * td)1776 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1777 {
1778 struct umtx_q *uq, *uq1, *uq2;
1779 struct thread *td1;
1780
1781 mtx_assert(&umtx_lock, MA_OWNED);
1782 if (pi == NULL)
1783 return (0);
1784
1785 uq = td->td_umtxq;
1786
1787 /*
1788 * Check if the thread needs to be moved on the blocked chain.
1789 * It needs to be moved if either its priority is lower than
1790 * the previous thread or higher than the next thread.
1791 */
1792 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1793 uq2 = TAILQ_NEXT(uq, uq_lockq);
1794 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1795 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1796 /*
1797 * Remove thread from blocked chain and determine where
1798 * it should be moved to.
1799 */
1800 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1801 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1802 td1 = uq1->uq_thread;
1803 MPASS(td1->td_proc->p_magic == P_MAGIC);
1804 if (UPRI(td1) > UPRI(td))
1805 break;
1806 }
1807
1808 if (uq1 == NULL)
1809 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1810 else
1811 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1812 }
1813 return (1);
1814 }
1815
1816 static struct umtx_pi *
umtx_pi_next(struct umtx_pi * pi)1817 umtx_pi_next(struct umtx_pi *pi)
1818 {
1819 struct umtx_q *uq_owner;
1820
1821 if (pi->pi_owner == NULL)
1822 return (NULL);
1823 uq_owner = pi->pi_owner->td_umtxq;
1824 if (uq_owner == NULL)
1825 return (NULL);
1826 return (uq_owner->uq_pi_blocked);
1827 }
1828
1829 /*
1830 * Floyd's Cycle-Finding Algorithm.
1831 */
1832 static bool
umtx_pi_check_loop(struct umtx_pi * pi)1833 umtx_pi_check_loop(struct umtx_pi *pi)
1834 {
1835 struct umtx_pi *pi1; /* fast iterator */
1836
1837 mtx_assert(&umtx_lock, MA_OWNED);
1838 if (pi == NULL)
1839 return (false);
1840 pi1 = pi;
1841 for (;;) {
1842 pi = umtx_pi_next(pi);
1843 if (pi == NULL)
1844 break;
1845 pi1 = umtx_pi_next(pi1);
1846 if (pi1 == NULL)
1847 break;
1848 pi1 = umtx_pi_next(pi1);
1849 if (pi1 == NULL)
1850 break;
1851 if (pi == pi1)
1852 return (true);
1853 }
1854 return (false);
1855 }
1856
1857 /*
1858 * Propagate priority when a thread is blocked on POSIX
1859 * PI mutex.
1860 */
1861 static void
umtx_propagate_priority(struct thread * td)1862 umtx_propagate_priority(struct thread *td)
1863 {
1864 struct umtx_q *uq;
1865 struct umtx_pi *pi;
1866 int pri;
1867
1868 mtx_assert(&umtx_lock, MA_OWNED);
1869 pri = UPRI(td);
1870 uq = td->td_umtxq;
1871 pi = uq->uq_pi_blocked;
1872 if (pi == NULL)
1873 return;
1874 if (umtx_pi_check_loop(pi))
1875 return;
1876
1877 for (;;) {
1878 td = pi->pi_owner;
1879 if (td == NULL || td == curthread)
1880 return;
1881
1882 MPASS(td->td_proc != NULL);
1883 MPASS(td->td_proc->p_magic == P_MAGIC);
1884
1885 thread_lock(td);
1886 if (td->td_lend_user_pri > pri)
1887 sched_lend_user_prio(td, pri);
1888 else {
1889 thread_unlock(td);
1890 break;
1891 }
1892 thread_unlock(td);
1893
1894 /*
1895 * Pick up the lock that td is blocked on.
1896 */
1897 uq = td->td_umtxq;
1898 pi = uq->uq_pi_blocked;
1899 if (pi == NULL)
1900 break;
1901 /* Resort td on the list if needed. */
1902 umtx_pi_adjust_thread(pi, td);
1903 }
1904 }
1905
1906 /*
1907 * Unpropagate priority for a PI mutex when a thread blocked on
1908 * it is interrupted by signal or resumed by others.
1909 */
1910 static void
umtx_repropagate_priority(struct umtx_pi * pi)1911 umtx_repropagate_priority(struct umtx_pi *pi)
1912 {
1913 struct umtx_q *uq, *uq_owner;
1914 struct umtx_pi *pi2;
1915 int pri;
1916
1917 mtx_assert(&umtx_lock, MA_OWNED);
1918
1919 if (umtx_pi_check_loop(pi))
1920 return;
1921 while (pi != NULL && pi->pi_owner != NULL) {
1922 pri = PRI_MAX;
1923 uq_owner = pi->pi_owner->td_umtxq;
1924
1925 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1926 uq = TAILQ_FIRST(&pi2->pi_blocked);
1927 if (uq != NULL) {
1928 if (pri > UPRI(uq->uq_thread))
1929 pri = UPRI(uq->uq_thread);
1930 }
1931 }
1932
1933 if (pri > uq_owner->uq_inherited_pri)
1934 pri = uq_owner->uq_inherited_pri;
1935 thread_lock(pi->pi_owner);
1936 sched_lend_user_prio(pi->pi_owner, pri);
1937 thread_unlock(pi->pi_owner);
1938 if ((pi = uq_owner->uq_pi_blocked) != NULL)
1939 umtx_pi_adjust_thread(pi, uq_owner->uq_thread);
1940 }
1941 }
1942
1943 /*
1944 * Insert a PI mutex into owned list.
1945 */
1946 static void
umtx_pi_setowner(struct umtx_pi * pi,struct thread * owner)1947 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1948 {
1949 struct umtx_q *uq_owner;
1950
1951 uq_owner = owner->td_umtxq;
1952 mtx_assert(&umtx_lock, MA_OWNED);
1953 MPASS(pi->pi_owner == NULL);
1954 pi->pi_owner = owner;
1955 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1956 }
1957
1958 /*
1959 * Disown a PI mutex, and remove it from the owned list.
1960 */
1961 static void
umtx_pi_disown(struct umtx_pi * pi)1962 umtx_pi_disown(struct umtx_pi *pi)
1963 {
1964
1965 mtx_assert(&umtx_lock, MA_OWNED);
1966 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link);
1967 pi->pi_owner = NULL;
1968 }
1969
1970 /*
1971 * Claim ownership of a PI mutex.
1972 */
1973 int
umtx_pi_claim(struct umtx_pi * pi,struct thread * owner)1974 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1975 {
1976 struct umtx_q *uq;
1977 int pri;
1978
1979 mtx_lock(&umtx_lock);
1980 if (pi->pi_owner == owner) {
1981 mtx_unlock(&umtx_lock);
1982 return (0);
1983 }
1984
1985 if (pi->pi_owner != NULL) {
1986 /*
1987 * userland may have already messed the mutex, sigh.
1988 */
1989 mtx_unlock(&umtx_lock);
1990 return (EPERM);
1991 }
1992 umtx_pi_setowner(pi, owner);
1993 uq = TAILQ_FIRST(&pi->pi_blocked);
1994 if (uq != NULL) {
1995 pri = UPRI(uq->uq_thread);
1996 thread_lock(owner);
1997 if (pri < UPRI(owner))
1998 sched_lend_user_prio(owner, pri);
1999 thread_unlock(owner);
2000 }
2001 mtx_unlock(&umtx_lock);
2002 return (0);
2003 }
2004
2005 /*
2006 * Adjust a thread's order position in its blocked PI mutex,
2007 * this may result new priority propagating process.
2008 */
2009 void
umtx_pi_adjust(struct thread * td,u_char oldpri)2010 umtx_pi_adjust(struct thread *td, u_char oldpri)
2011 {
2012 struct umtx_q *uq;
2013 struct umtx_pi *pi;
2014
2015 uq = td->td_umtxq;
2016 mtx_lock(&umtx_lock);
2017 /*
2018 * Pick up the lock that td is blocked on.
2019 */
2020 pi = uq->uq_pi_blocked;
2021 if (pi != NULL) {
2022 umtx_pi_adjust_thread(pi, td);
2023 umtx_repropagate_priority(pi);
2024 }
2025 mtx_unlock(&umtx_lock);
2026 }
2027
2028 /*
2029 * Sleep on a PI mutex.
2030 */
2031 int
umtxq_sleep_pi(struct umtx_q * uq,struct umtx_pi * pi,uint32_t owner,const char * wmesg,struct umtx_abs_timeout * timo,bool shared)2032 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner,
2033 const char *wmesg, struct umtx_abs_timeout *timo, bool shared)
2034 {
2035 struct thread *td;
2036 struct umtx_q *uq1;
2037 int error, pri;
2038 #ifdef INVARIANTS
2039 struct umtxq_chain *uc;
2040
2041 uc = umtxq_getchain(&pi->pi_key);
2042 #endif
2043 error = 0;
2044 td = uq->uq_thread;
2045 KASSERT(td == curthread, ("inconsistent uq_thread"));
2046 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
2047 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy"));
2048 umtxq_insert(uq);
2049 mtx_lock(&umtx_lock);
2050 if (pi->pi_owner == NULL) {
2051 struct thread *ownertd;
2052
2053 mtx_unlock(&umtx_lock);
2054 ownertd = tdfind(owner, shared ? -1 : td->td_proc->p_pid);
2055 mtx_lock(&umtx_lock);
2056 if (ownertd != NULL) {
2057 /*
2058 * An exiting thread that has already called
2059 * umtx_thread_exit() must not be made the owner of a
2060 * shared mutex.
2061 */
2062 if ((ownertd->td_proc->p_flag & P_WEXIT) == 0 &&
2063 (ownertd->td_dbgflags & TDB_EXIT) == 0 &&
2064 pi->pi_owner == NULL)
2065 umtx_pi_setowner(pi, ownertd);
2066 PROC_UNLOCK(ownertd->td_proc);
2067 }
2068 }
2069
2070 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
2071 pri = UPRI(uq1->uq_thread);
2072 if (pri > UPRI(td))
2073 break;
2074 }
2075
2076 if (uq1 != NULL)
2077 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
2078 else
2079 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
2080
2081 uq->uq_pi_blocked = pi;
2082 thread_lock(td);
2083 td->td_flags |= TDF_UPIBLOCKED;
2084 thread_unlock(td);
2085 umtx_propagate_priority(td);
2086 mtx_unlock(&umtx_lock);
2087 umtxq_unbusy(&uq->uq_key);
2088
2089 error = umtxq_sleep(uq, wmesg, timo);
2090 umtxq_remove(uq);
2091
2092 mtx_lock(&umtx_lock);
2093 uq->uq_pi_blocked = NULL;
2094 thread_lock(td);
2095 td->td_flags &= ~TDF_UPIBLOCKED;
2096 thread_unlock(td);
2097 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
2098 umtx_repropagate_priority(pi);
2099 mtx_unlock(&umtx_lock);
2100 umtxq_unlock(&uq->uq_key);
2101
2102 return (error);
2103 }
2104
2105 /*
2106 * Add reference count for a PI mutex.
2107 */
2108 void
umtx_pi_ref(struct umtx_pi * pi)2109 umtx_pi_ref(struct umtx_pi *pi)
2110 {
2111
2112 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key));
2113 pi->pi_refcount++;
2114 }
2115
2116 /*
2117 * Decrease reference count for a PI mutex, if the counter
2118 * is decreased to zero, its memory space is freed.
2119 */
2120 void
umtx_pi_unref(struct umtx_pi * pi)2121 umtx_pi_unref(struct umtx_pi *pi)
2122 {
2123 struct umtxq_chain *uc;
2124
2125 uc = umtxq_getchain(&pi->pi_key);
2126 UMTXQ_LOCKED_ASSERT(uc);
2127 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
2128 if (--pi->pi_refcount == 0) {
2129 mtx_lock(&umtx_lock);
2130 if (pi->pi_owner != NULL)
2131 umtx_pi_disown(pi);
2132 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
2133 ("blocked queue not empty"));
2134 mtx_unlock(&umtx_lock);
2135 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
2136 umtx_pi_free(pi);
2137 }
2138 }
2139
2140 /*
2141 * Find a PI mutex in hash table.
2142 */
2143 struct umtx_pi *
umtx_pi_lookup(struct umtx_key * key)2144 umtx_pi_lookup(struct umtx_key *key)
2145 {
2146 struct umtxq_chain *uc;
2147 struct umtx_pi *pi;
2148
2149 uc = umtxq_getchain(key);
2150 UMTXQ_LOCKED_ASSERT(uc);
2151
2152 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
2153 if (umtx_key_match(&pi->pi_key, key)) {
2154 return (pi);
2155 }
2156 }
2157 return (NULL);
2158 }
2159
2160 /*
2161 * Insert a PI mutex into hash table.
2162 */
2163 void
umtx_pi_insert(struct umtx_pi * pi)2164 umtx_pi_insert(struct umtx_pi *pi)
2165 {
2166 struct umtxq_chain *uc;
2167
2168 uc = umtxq_getchain(&pi->pi_key);
2169 UMTXQ_LOCKED_ASSERT(uc);
2170 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
2171 }
2172
2173 /*
2174 * Drop a PI mutex and wakeup a top waiter.
2175 */
2176 int
umtx_pi_drop(struct thread * td,struct umtx_key * key,bool rb,int * count)2177 umtx_pi_drop(struct thread *td, struct umtx_key *key, bool rb, int *count)
2178 {
2179 struct umtx_q *uq_first, *uq_first2, *uq_me;
2180 struct umtx_pi *pi, *pi2;
2181 int pri;
2182
2183 UMTXQ_ASSERT_LOCKED_BUSY(key);
2184 *count = umtxq_count_pi(key, &uq_first);
2185 if (uq_first != NULL) {
2186 mtx_lock(&umtx_lock);
2187 pi = uq_first->uq_pi_blocked;
2188 KASSERT(pi != NULL, ("pi == NULL?"));
2189 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
2190 mtx_unlock(&umtx_lock);
2191 /* userland messed the mutex */
2192 return (EPERM);
2193 }
2194 uq_me = td->td_umtxq;
2195 if (pi->pi_owner == td)
2196 umtx_pi_disown(pi);
2197 /* get highest priority thread which is still sleeping. */
2198 uq_first = TAILQ_FIRST(&pi->pi_blocked);
2199 while (uq_first != NULL &&
2200 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
2201 uq_first = TAILQ_NEXT(uq_first, uq_lockq);
2202 }
2203 pri = PRI_MAX;
2204 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
2205 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
2206 if (uq_first2 != NULL) {
2207 if (pri > UPRI(uq_first2->uq_thread))
2208 pri = UPRI(uq_first2->uq_thread);
2209 }
2210 }
2211 thread_lock(td);
2212 sched_lend_user_prio(td, pri);
2213 thread_unlock(td);
2214 mtx_unlock(&umtx_lock);
2215 if (uq_first)
2216 umtxq_signal_thread(uq_first);
2217 } else {
2218 pi = umtx_pi_lookup(key);
2219 /*
2220 * A umtx_pi can exist if a signal or timeout removed the
2221 * last waiter from the umtxq, but there is still
2222 * a thread in do_lock_pi() holding the umtx_pi.
2223 */
2224 if (pi != NULL) {
2225 /*
2226 * The umtx_pi can be unowned, such as when a thread
2227 * has just entered do_lock_pi(), allocated the
2228 * umtx_pi, and unlocked the umtxq.
2229 * If the current thread owns it, it must disown it.
2230 */
2231 mtx_lock(&umtx_lock);
2232 if (pi->pi_owner == td)
2233 umtx_pi_disown(pi);
2234 mtx_unlock(&umtx_lock);
2235 }
2236 }
2237 return (0);
2238 }
2239
2240 /*
2241 * Lock a PI mutex.
2242 */
2243 static int
do_lock_pi(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int try)2244 do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
2245 struct _umtx_time *timeout, int try)
2246 {
2247 struct umtx_abs_timeout timo;
2248 struct umtx_q *uq;
2249 struct umtx_pi *pi, *new_pi;
2250 uint32_t id, old_owner, owner, old;
2251 int error, rv;
2252
2253 id = td->td_tid;
2254 uq = td->td_umtxq;
2255
2256 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2257 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
2258 &uq->uq_key)) != 0)
2259 return (error);
2260
2261 if (timeout != NULL)
2262 umtx_abs_timeout_init2(&timo, timeout);
2263
2264 umtxq_lock(&uq->uq_key);
2265 pi = umtx_pi_lookup(&uq->uq_key);
2266 if (pi == NULL) {
2267 new_pi = umtx_pi_alloc(M_NOWAIT);
2268 if (new_pi == NULL) {
2269 umtxq_unlock(&uq->uq_key);
2270 new_pi = umtx_pi_alloc(M_WAITOK);
2271 umtxq_lock(&uq->uq_key);
2272 pi = umtx_pi_lookup(&uq->uq_key);
2273 if (pi != NULL) {
2274 umtx_pi_free(new_pi);
2275 new_pi = NULL;
2276 }
2277 }
2278 if (new_pi != NULL) {
2279 new_pi->pi_key = uq->uq_key;
2280 umtx_pi_insert(new_pi);
2281 pi = new_pi;
2282 }
2283 }
2284 umtx_pi_ref(pi);
2285 umtxq_unlock(&uq->uq_key);
2286
2287 /*
2288 * Care must be exercised when dealing with umtx structure. It
2289 * can fault on any access.
2290 */
2291 for (;;) {
2292 /*
2293 * Try the uncontested case. This should be done in userland.
2294 */
2295 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
2296 /* The address was invalid. */
2297 if (rv == -1) {
2298 error = EFAULT;
2299 break;
2300 }
2301 /* The acquire succeeded. */
2302 if (rv == 0) {
2303 MPASS(owner == UMUTEX_UNOWNED);
2304 error = 0;
2305 break;
2306 }
2307
2308 if (owner == UMUTEX_RB_NOTRECOV) {
2309 error = ENOTRECOVERABLE;
2310 break;
2311 }
2312
2313 /*
2314 * Nobody owns it, but the acquire failed. This can happen
2315 * with ll/sc atomics.
2316 */
2317 if (owner == UMUTEX_UNOWNED) {
2318 error = thread_check_susp(td, true);
2319 if (error != 0)
2320 break;
2321 continue;
2322 }
2323
2324 /*
2325 * Avoid overwriting a possible error from sleep due
2326 * to the pending signal with suspension check result.
2327 */
2328 if (error == 0) {
2329 error = thread_check_susp(td, true);
2330 if (error != 0)
2331 break;
2332 }
2333
2334 /* If no one owns it but it is contested try to acquire it. */
2335 if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) {
2336 old_owner = owner;
2337 rv = casueword32(&m->m_owner, owner, &owner,
2338 id | UMUTEX_CONTESTED);
2339 /* The address was invalid. */
2340 if (rv == -1) {
2341 error = EFAULT;
2342 break;
2343 }
2344 if (rv == 1) {
2345 if (error == 0) {
2346 error = thread_check_susp(td, true);
2347 if (error != 0)
2348 break;
2349 }
2350
2351 /*
2352 * If this failed the lock could
2353 * changed, restart.
2354 */
2355 continue;
2356 }
2357
2358 MPASS(rv == 0);
2359 MPASS(owner == old_owner);
2360 umtxq_lock(&uq->uq_key);
2361 umtxq_busy(&uq->uq_key);
2362 error = umtx_pi_claim(pi, td);
2363 umtxq_unbusy(&uq->uq_key);
2364 umtxq_unlock(&uq->uq_key);
2365 if (error != 0) {
2366 /*
2367 * Since we're going to return an
2368 * error, restore the m_owner to its
2369 * previous, unowned state to avoid
2370 * compounding the problem.
2371 */
2372 (void)casuword32(&m->m_owner,
2373 id | UMUTEX_CONTESTED, old_owner);
2374 }
2375 if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD)
2376 error = EOWNERDEAD;
2377 break;
2378 }
2379
2380 if ((owner & ~UMUTEX_CONTESTED) == id) {
2381 error = EDEADLK;
2382 break;
2383 }
2384
2385 if (try != 0) {
2386 error = EBUSY;
2387 break;
2388 }
2389
2390 /*
2391 * If we caught a signal, we have retried and now
2392 * exit immediately.
2393 */
2394 if (error != 0)
2395 break;
2396
2397 umtxq_busy_unlocked(&uq->uq_key);
2398
2399 /*
2400 * Set the contested bit so that a release in user space
2401 * knows to use the system call for unlock. If this fails
2402 * either some one else has acquired the lock or it has been
2403 * released.
2404 */
2405 rv = casueword32(&m->m_owner, owner, &old, owner |
2406 UMUTEX_CONTESTED);
2407
2408 /* The address was invalid. */
2409 if (rv == -1) {
2410 umtxq_unbusy_unlocked(&uq->uq_key);
2411 error = EFAULT;
2412 break;
2413 }
2414 if (rv == 1) {
2415 umtxq_unbusy_unlocked(&uq->uq_key);
2416 error = thread_check_susp(td, true);
2417 if (error != 0)
2418 break;
2419
2420 /*
2421 * The lock changed and we need to retry or we
2422 * lost a race to the thread unlocking the
2423 * umtx. Note that the UMUTEX_RB_OWNERDEAD
2424 * value for owner is impossible there.
2425 */
2426 continue;
2427 }
2428
2429 umtxq_lock(&uq->uq_key);
2430
2431 /* We set the contested bit, sleep. */
2432 MPASS(old == owner);
2433 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
2434 "umtxpi", timeout == NULL ? NULL : &timo,
2435 (flags & USYNC_PROCESS_SHARED) != 0);
2436 if (error != 0)
2437 continue;
2438
2439 error = thread_check_susp(td, false);
2440 if (error != 0)
2441 break;
2442 }
2443
2444 umtxq_lock(&uq->uq_key);
2445 umtx_pi_unref(pi);
2446 umtxq_unlock(&uq->uq_key);
2447
2448 umtx_key_release(&uq->uq_key);
2449 return (error);
2450 }
2451
2452 /*
2453 * Unlock a PI mutex.
2454 */
2455 static int
do_unlock_pi(struct thread * td,struct umutex * m,uint32_t flags,bool rb)2456 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
2457 {
2458 struct umtx_key key;
2459 uint32_t id, new_owner, old, owner;
2460 int count, error;
2461
2462 id = td->td_tid;
2463
2464 usrloop:
2465 /*
2466 * Make sure we own this mtx.
2467 */
2468 error = fueword32(&m->m_owner, &owner);
2469 if (error == -1)
2470 return (EFAULT);
2471
2472 if ((owner & ~UMUTEX_CONTESTED) != id)
2473 return (EPERM);
2474
2475 new_owner = umtx_unlock_val(flags, rb);
2476
2477 /* This should be done in userland */
2478 if ((owner & UMUTEX_CONTESTED) == 0) {
2479 error = casueword32(&m->m_owner, owner, &old, new_owner);
2480 if (error == -1)
2481 return (EFAULT);
2482 if (error == 1) {
2483 error = thread_check_susp(td, true);
2484 if (error != 0)
2485 return (error);
2486 goto usrloop;
2487 }
2488 if (old == owner)
2489 return (0);
2490 owner = old;
2491 }
2492
2493 /* We should only ever be in here for contested locks */
2494 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2495 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
2496 &key)) != 0)
2497 return (error);
2498
2499 umtxq_lock(&key);
2500 umtxq_busy(&key);
2501 error = umtx_pi_drop(td, &key, rb, &count);
2502 if (error != 0) {
2503 umtxq_unbusy(&key);
2504 umtxq_unlock(&key);
2505 umtx_key_release(&key);
2506 /* userland messed the mutex */
2507 return (error);
2508 }
2509 umtxq_unlock(&key);
2510
2511 /*
2512 * When unlocking the umtx, it must be marked as unowned if
2513 * there is zero or one thread only waiting for it.
2514 * Otherwise, it must be marked as contested.
2515 */
2516
2517 if (count > 1)
2518 new_owner |= UMUTEX_CONTESTED;
2519 again:
2520 error = casueword32(&m->m_owner, owner, &old, new_owner);
2521 if (error == 1) {
2522 error = thread_check_susp(td, false);
2523 if (error == 0)
2524 goto again;
2525 }
2526 umtxq_unbusy_unlocked(&key);
2527 umtx_key_release(&key);
2528 if (error == -1)
2529 return (EFAULT);
2530 if (error == 0 && old != owner)
2531 return (EINVAL);
2532 return (error);
2533 }
2534
2535 /*
2536 * Lock a PP mutex.
2537 */
2538 static int
do_lock_pp(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int try)2539 do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
2540 struct _umtx_time *timeout, int try)
2541 {
2542 struct umtx_abs_timeout timo;
2543 struct umtx_q *uq, *uq2;
2544 struct umtx_pi *pi;
2545 uint32_t ceiling;
2546 uint32_t owner, id;
2547 int error, pri, old_inherited_pri, new_pri, rv;
2548 bool su;
2549
2550 id = td->td_tid;
2551 uq = td->td_umtxq;
2552 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2553 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2554 &uq->uq_key)) != 0)
2555 return (error);
2556
2557 if (timeout != NULL)
2558 umtx_abs_timeout_init2(&timo, timeout);
2559
2560 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2561 for (;;) {
2562 old_inherited_pri = uq->uq_inherited_pri;
2563 umtxq_busy_unlocked(&uq->uq_key);
2564
2565 rv = fueword32(&m->m_ceilings[0], &ceiling);
2566 if (rv == -1) {
2567 error = EFAULT;
2568 goto out;
2569 }
2570 ceiling = RTP_PRIO_MAX - ceiling;
2571 if (ceiling > RTP_PRIO_MAX) {
2572 error = EINVAL;
2573 goto out;
2574 }
2575 new_pri = PRI_MIN_REALTIME + ceiling;
2576
2577 if (td->td_base_user_pri < new_pri) {
2578 error = EINVAL;
2579 goto out;
2580 }
2581 if (su) {
2582 mtx_lock(&umtx_lock);
2583 if (new_pri < uq->uq_inherited_pri) {
2584 uq->uq_inherited_pri = new_pri;
2585 thread_lock(td);
2586 if (new_pri < UPRI(td))
2587 sched_lend_user_prio(td, new_pri);
2588 thread_unlock(td);
2589 }
2590 mtx_unlock(&umtx_lock);
2591 }
2592
2593 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2594 id | UMUTEX_CONTESTED);
2595 /* The address was invalid. */
2596 if (rv == -1) {
2597 error = EFAULT;
2598 break;
2599 }
2600 if (rv == 0) {
2601 MPASS(owner == UMUTEX_CONTESTED);
2602 error = 0;
2603 break;
2604 }
2605 /* rv == 1 */
2606 if (owner == UMUTEX_RB_OWNERDEAD) {
2607 rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
2608 &owner, id | UMUTEX_CONTESTED);
2609 if (rv == -1) {
2610 error = EFAULT;
2611 break;
2612 }
2613 if (rv == 0) {
2614 MPASS(owner == UMUTEX_RB_OWNERDEAD);
2615 error = EOWNERDEAD; /* success */
2616 break;
2617 }
2618
2619 /*
2620 * rv == 1, only check for suspension if we
2621 * did not already catched a signal. If we
2622 * get an error from the check, the same
2623 * condition is checked by the umtxq_sleep()
2624 * call below, so we should obliterate the
2625 * error to not skip the last loop iteration.
2626 */
2627 if (error == 0) {
2628 error = thread_check_susp(td, false);
2629 if (error == 0 && try == 0) {
2630 umtxq_unbusy_unlocked(&uq->uq_key);
2631 continue;
2632 }
2633 error = 0;
2634 }
2635 } else if (owner == UMUTEX_RB_NOTRECOV) {
2636 error = ENOTRECOVERABLE;
2637 } else if (owner == UMUTEX_CONTESTED) {
2638 /* Spurious failure, retry. */
2639 umtxq_unbusy_unlocked(&uq->uq_key);
2640 continue;
2641 }
2642
2643 if (try != 0)
2644 error = EBUSY;
2645
2646 /*
2647 * If we caught a signal, we have retried and now
2648 * exit immediately.
2649 */
2650 if (error != 0)
2651 break;
2652
2653 umtxq_lock(&uq->uq_key);
2654 umtxq_insert(uq);
2655 umtxq_unbusy(&uq->uq_key);
2656 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ?
2657 NULL : &timo);
2658 umtxq_remove(uq);
2659 umtxq_unlock(&uq->uq_key);
2660
2661 mtx_lock(&umtx_lock);
2662 uq->uq_inherited_pri = old_inherited_pri;
2663 pri = PRI_MAX;
2664 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2665 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2666 if (uq2 != NULL) {
2667 if (pri > UPRI(uq2->uq_thread))
2668 pri = UPRI(uq2->uq_thread);
2669 }
2670 }
2671 if (pri > uq->uq_inherited_pri)
2672 pri = uq->uq_inherited_pri;
2673 thread_lock(td);
2674 sched_lend_user_prio(td, pri);
2675 thread_unlock(td);
2676 mtx_unlock(&umtx_lock);
2677 }
2678
2679 if (error != 0 && error != EOWNERDEAD) {
2680 mtx_lock(&umtx_lock);
2681 uq->uq_inherited_pri = old_inherited_pri;
2682 pri = PRI_MAX;
2683 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2684 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2685 if (uq2 != NULL) {
2686 if (pri > UPRI(uq2->uq_thread))
2687 pri = UPRI(uq2->uq_thread);
2688 }
2689 }
2690 if (pri > uq->uq_inherited_pri)
2691 pri = uq->uq_inherited_pri;
2692 thread_lock(td);
2693 sched_lend_user_prio(td, pri);
2694 thread_unlock(td);
2695 mtx_unlock(&umtx_lock);
2696 }
2697
2698 out:
2699 umtxq_unbusy_unlocked(&uq->uq_key);
2700 umtx_key_release(&uq->uq_key);
2701 return (error);
2702 }
2703
2704 /*
2705 * Unlock a PP mutex.
2706 */
2707 static int
do_unlock_pp(struct thread * td,struct umutex * m,uint32_t flags,bool rb)2708 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
2709 {
2710 struct umtx_key key;
2711 struct umtx_q *uq, *uq2;
2712 struct umtx_pi *pi;
2713 uint32_t id, owner, rceiling;
2714 int error, pri, new_inherited_pri;
2715 bool su;
2716
2717 id = td->td_tid;
2718 uq = td->td_umtxq;
2719 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2720
2721 /*
2722 * Make sure we own this mtx.
2723 */
2724 error = fueword32(&m->m_owner, &owner);
2725 if (error == -1)
2726 return (EFAULT);
2727
2728 if ((owner & ~UMUTEX_CONTESTED) != id)
2729 return (EPERM);
2730
2731 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2732 if (error != 0)
2733 return (error);
2734
2735 if (rceiling == -1)
2736 new_inherited_pri = PRI_MAX;
2737 else {
2738 rceiling = RTP_PRIO_MAX - rceiling;
2739 if (rceiling > RTP_PRIO_MAX)
2740 return (EINVAL);
2741 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2742 }
2743
2744 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2745 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2746 &key)) != 0)
2747 return (error);
2748 umtxq_busy_unlocked(&key);
2749
2750 /*
2751 * For priority protected mutex, always set unlocked state
2752 * to UMUTEX_CONTESTED, so that userland always enters kernel
2753 * to lock the mutex, it is necessary because thread priority
2754 * has to be adjusted for such mutex.
2755 */
2756 error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) |
2757 UMUTEX_CONTESTED);
2758
2759 umtxq_lock(&key);
2760 if (error == 0)
2761 umtxq_signal(&key, 1);
2762 umtxq_unbusy(&key);
2763 umtxq_unlock(&key);
2764
2765 if (error == -1)
2766 error = EFAULT;
2767 else {
2768 mtx_lock(&umtx_lock);
2769 if (su || new_inherited_pri == PRI_MAX)
2770 uq->uq_inherited_pri = new_inherited_pri;
2771 pri = PRI_MAX;
2772 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2773 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2774 if (uq2 != NULL) {
2775 if (pri > UPRI(uq2->uq_thread))
2776 pri = UPRI(uq2->uq_thread);
2777 }
2778 }
2779 if (pri > uq->uq_inherited_pri)
2780 pri = uq->uq_inherited_pri;
2781 thread_lock(td);
2782 sched_lend_user_prio(td, pri);
2783 thread_unlock(td);
2784 mtx_unlock(&umtx_lock);
2785 }
2786 umtx_key_release(&key);
2787 return (error);
2788 }
2789
2790 static int
do_set_ceiling(struct thread * td,struct umutex * m,uint32_t ceiling,uint32_t * old_ceiling)2791 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2792 uint32_t *old_ceiling)
2793 {
2794 struct umtx_q *uq;
2795 uint32_t flags, id, owner, save_ceiling;
2796 int error, rv, rv1;
2797
2798 error = fueword32(&m->m_flags, &flags);
2799 if (error == -1)
2800 return (EFAULT);
2801 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2802 return (EINVAL);
2803 if (ceiling > RTP_PRIO_MAX)
2804 return (EINVAL);
2805 id = td->td_tid;
2806 uq = td->td_umtxq;
2807 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2808 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2809 &uq->uq_key)) != 0)
2810 return (error);
2811 for (;;) {
2812 umtxq_busy_unlocked(&uq->uq_key);
2813
2814 rv = fueword32(&m->m_ceilings[0], &save_ceiling);
2815 if (rv == -1) {
2816 error = EFAULT;
2817 break;
2818 }
2819
2820 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2821 id | UMUTEX_CONTESTED);
2822 if (rv == -1) {
2823 error = EFAULT;
2824 break;
2825 }
2826
2827 if (rv == 0) {
2828 MPASS(owner == UMUTEX_CONTESTED);
2829 rv = suword32(&m->m_ceilings[0], ceiling);
2830 rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED);
2831 error = (rv == 0 && rv1 == 0) ? 0: EFAULT;
2832 break;
2833 }
2834
2835 if ((owner & ~UMUTEX_CONTESTED) == id) {
2836 rv = suword32(&m->m_ceilings[0], ceiling);
2837 error = rv == 0 ? 0 : EFAULT;
2838 break;
2839 }
2840
2841 if (owner == UMUTEX_RB_OWNERDEAD) {
2842 error = EOWNERDEAD;
2843 break;
2844 } else if (owner == UMUTEX_RB_NOTRECOV) {
2845 error = ENOTRECOVERABLE;
2846 break;
2847 } else if (owner == UMUTEX_CONTESTED) {
2848 /* Spurious failure, retry. */
2849 umtxq_unbusy_unlocked(&uq->uq_key);
2850 continue;
2851 }
2852
2853 /*
2854 * If we caught a signal, we have retried and now
2855 * exit immediately.
2856 */
2857 if (error != 0)
2858 break;
2859
2860 /*
2861 * We set the contested bit, sleep. Otherwise the lock changed
2862 * and we need to retry or we lost a race to the thread
2863 * unlocking the umtx.
2864 */
2865 umtxq_lock(&uq->uq_key);
2866 umtxq_insert(uq);
2867 umtxq_unbusy(&uq->uq_key);
2868 error = umtxq_sleep(uq, "umtxpp", NULL);
2869 umtxq_remove(uq);
2870 umtxq_unlock(&uq->uq_key);
2871 }
2872 umtxq_lock(&uq->uq_key);
2873 if (error == 0)
2874 umtxq_signal(&uq->uq_key, INT_MAX);
2875 umtxq_unbusy(&uq->uq_key);
2876 umtxq_unlock(&uq->uq_key);
2877 umtx_key_release(&uq->uq_key);
2878 if (error == 0 && old_ceiling != NULL) {
2879 rv = suword32(old_ceiling, save_ceiling);
2880 error = rv == 0 ? 0 : EFAULT;
2881 }
2882 return (error);
2883 }
2884
2885 /*
2886 * Lock a userland POSIX mutex.
2887 */
2888 static int
do_lock_umutex(struct thread * td,struct umutex * m,struct _umtx_time * timeout,int mode)2889 do_lock_umutex(struct thread *td, struct umutex *m,
2890 struct _umtx_time *timeout, int mode)
2891 {
2892 uint32_t flags;
2893 int error;
2894
2895 error = fueword32(&m->m_flags, &flags);
2896 if (error == -1)
2897 return (EFAULT);
2898
2899 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2900 case 0:
2901 error = do_lock_normal(td, m, flags, timeout, mode);
2902 break;
2903 case UMUTEX_PRIO_INHERIT:
2904 error = do_lock_pi(td, m, flags, timeout, mode);
2905 break;
2906 case UMUTEX_PRIO_PROTECT:
2907 error = do_lock_pp(td, m, flags, timeout, mode);
2908 break;
2909 default:
2910 return (EINVAL);
2911 }
2912 if (timeout == NULL) {
2913 if (error == EINTR && mode != _UMUTEX_WAIT)
2914 error = ERESTART;
2915 } else {
2916 /* Timed-locking is not restarted. */
2917 if (error == ERESTART)
2918 error = EINTR;
2919 }
2920 return (error);
2921 }
2922
2923 /*
2924 * Unlock a userland POSIX mutex.
2925 */
2926 static int
do_unlock_umutex(struct thread * td,struct umutex * m,bool rb)2927 do_unlock_umutex(struct thread *td, struct umutex *m, bool rb)
2928 {
2929 uint32_t flags;
2930 int error;
2931
2932 error = fueword32(&m->m_flags, &flags);
2933 if (error == -1)
2934 return (EFAULT);
2935
2936 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2937 case 0:
2938 return (do_unlock_normal(td, m, flags, rb));
2939 case UMUTEX_PRIO_INHERIT:
2940 return (do_unlock_pi(td, m, flags, rb));
2941 case UMUTEX_PRIO_PROTECT:
2942 return (do_unlock_pp(td, m, flags, rb));
2943 }
2944
2945 return (EINVAL);
2946 }
2947
2948 static int
do_cv_wait(struct thread * td,struct ucond * cv,struct umutex * m,struct umtx_abs_timeout * timo,u_long wflags)2949 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2950 struct umtx_abs_timeout *timo, u_long wflags)
2951 {
2952 struct umtx_q *uq;
2953 uint32_t flags, hasw;
2954 int error;
2955
2956 uq = td->td_umtxq;
2957 error = fueword32(&cv->c_flags, &flags);
2958 if (error == -1)
2959 return (EFAULT);
2960 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2961 if (error != 0)
2962 return (error);
2963
2964 umtxq_lock(&uq->uq_key);
2965 umtxq_busy(&uq->uq_key);
2966 umtxq_insert(uq);
2967 umtxq_unlock(&uq->uq_key);
2968
2969 /*
2970 * Set c_has_waiters to 1 before releasing user mutex, also
2971 * don't modify cache line when unnecessary.
2972 */
2973 error = fueword32(&cv->c_has_waiters, &hasw);
2974 if (error == 0 && hasw == 0)
2975 error = suword32(&cv->c_has_waiters, 1);
2976 if (error != 0) {
2977 umtxq_lock(&uq->uq_key);
2978 umtxq_remove(uq);
2979 umtxq_unbusy(&uq->uq_key);
2980 error = EFAULT;
2981 goto out;
2982 }
2983
2984 umtxq_unbusy_unlocked(&uq->uq_key);
2985
2986 error = do_unlock_umutex(td, m, false);
2987
2988 umtxq_lock(&uq->uq_key);
2989 if (error == 0)
2990 error = umtxq_sleep(uq, "ucond", timo);
2991
2992 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2993 error = 0;
2994 else {
2995 /*
2996 * This must be timeout,interrupted by signal or
2997 * surprious wakeup, clear c_has_waiter flag when
2998 * necessary.
2999 */
3000 umtxq_busy(&uq->uq_key);
3001 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
3002 int oldlen = uq->uq_cur_queue->length;
3003 umtxq_remove(uq);
3004 if (oldlen == 1) {
3005 umtxq_unlock(&uq->uq_key);
3006 if (suword32(&cv->c_has_waiters, 0) != 0 &&
3007 error == 0)
3008 error = EFAULT;
3009 umtxq_lock(&uq->uq_key);
3010 }
3011 }
3012 umtxq_unbusy(&uq->uq_key);
3013 if (error == ERESTART)
3014 error = EINTR;
3015 }
3016 out:
3017 umtxq_unlock(&uq->uq_key);
3018 umtx_key_release(&uq->uq_key);
3019 return (error);
3020 }
3021
3022 /*
3023 * Signal a userland condition variable.
3024 */
3025 static int
do_cv_signal(struct thread * td,struct ucond * cv)3026 do_cv_signal(struct thread *td, struct ucond *cv)
3027 {
3028 struct umtx_key key;
3029 int error, cnt, nwake;
3030 uint32_t flags;
3031
3032 error = fueword32(&cv->c_flags, &flags);
3033 if (error == -1)
3034 return (EFAULT);
3035 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
3036 return (error);
3037 umtxq_lock(&key);
3038 umtxq_busy(&key);
3039 cnt = umtxq_count(&key);
3040 nwake = umtxq_signal(&key, 1);
3041 if (cnt <= nwake) {
3042 umtxq_unlock(&key);
3043 error = suword32(&cv->c_has_waiters, 0);
3044 if (error == -1)
3045 error = EFAULT;
3046 umtxq_lock(&key);
3047 }
3048 umtxq_unbusy(&key);
3049 umtxq_unlock(&key);
3050 umtx_key_release(&key);
3051 return (error);
3052 }
3053
3054 static int
do_cv_broadcast(struct thread * td,struct ucond * cv)3055 do_cv_broadcast(struct thread *td, struct ucond *cv)
3056 {
3057 struct umtx_key key;
3058 int error;
3059 uint32_t flags;
3060
3061 error = fueword32(&cv->c_flags, &flags);
3062 if (error == -1)
3063 return (EFAULT);
3064 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
3065 return (error);
3066
3067 umtxq_lock(&key);
3068 umtxq_busy(&key);
3069 umtxq_signal(&key, INT_MAX);
3070 umtxq_unlock(&key);
3071
3072 error = suword32(&cv->c_has_waiters, 0);
3073 if (error == -1)
3074 error = EFAULT;
3075
3076 umtxq_unbusy_unlocked(&key);
3077
3078 umtx_key_release(&key);
3079 return (error);
3080 }
3081
3082 static int
do_rw_rdlock(struct thread * td,struct urwlock * rwlock,long fflag,struct _umtx_time * timeout)3083 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag,
3084 struct _umtx_time *timeout)
3085 {
3086 struct umtx_abs_timeout timo;
3087 struct umtx_q *uq;
3088 uint32_t flags, wrflags;
3089 int32_t state, oldstate;
3090 int32_t blocked_readers;
3091 int error, error1, rv;
3092
3093 uq = td->td_umtxq;
3094 error = fueword32(&rwlock->rw_flags, &flags);
3095 if (error == -1)
3096 return (EFAULT);
3097 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3098 if (error != 0)
3099 return (error);
3100
3101 if (timeout != NULL)
3102 umtx_abs_timeout_init2(&timo, timeout);
3103
3104 wrflags = URWLOCK_WRITE_OWNER;
3105 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
3106 wrflags |= URWLOCK_WRITE_WAITERS;
3107
3108 for (;;) {
3109 rv = fueword32(&rwlock->rw_state, &state);
3110 if (rv == -1) {
3111 umtx_key_release(&uq->uq_key);
3112 return (EFAULT);
3113 }
3114
3115 /* try to lock it */
3116 while (!(state & wrflags)) {
3117 if (__predict_false(URWLOCK_READER_COUNT(state) ==
3118 URWLOCK_MAX_READERS)) {
3119 umtx_key_release(&uq->uq_key);
3120 return (EAGAIN);
3121 }
3122 rv = casueword32(&rwlock->rw_state, state,
3123 &oldstate, state + 1);
3124 if (rv == -1) {
3125 umtx_key_release(&uq->uq_key);
3126 return (EFAULT);
3127 }
3128 if (rv == 0) {
3129 MPASS(oldstate == state);
3130 umtx_key_release(&uq->uq_key);
3131 return (0);
3132 }
3133 error = thread_check_susp(td, true);
3134 if (error != 0)
3135 break;
3136 state = oldstate;
3137 }
3138
3139 if (error)
3140 break;
3141
3142 /* grab monitor lock */
3143 umtxq_busy_unlocked(&uq->uq_key);
3144
3145 /*
3146 * re-read the state, in case it changed between the try-lock above
3147 * and the check below
3148 */
3149 rv = fueword32(&rwlock->rw_state, &state);
3150 if (rv == -1)
3151 error = EFAULT;
3152
3153 /* set read contention bit */
3154 while (error == 0 && (state & wrflags) &&
3155 !(state & URWLOCK_READ_WAITERS)) {
3156 rv = casueword32(&rwlock->rw_state, state,
3157 &oldstate, state | URWLOCK_READ_WAITERS);
3158 if (rv == -1) {
3159 error = EFAULT;
3160 break;
3161 }
3162 if (rv == 0) {
3163 MPASS(oldstate == state);
3164 goto sleep;
3165 }
3166 state = oldstate;
3167 error = thread_check_susp(td, false);
3168 if (error != 0)
3169 break;
3170 }
3171 if (error != 0) {
3172 umtxq_unbusy_unlocked(&uq->uq_key);
3173 break;
3174 }
3175
3176 /* state is changed while setting flags, restart */
3177 if (!(state & wrflags)) {
3178 umtxq_unbusy_unlocked(&uq->uq_key);
3179 error = thread_check_susp(td, true);
3180 if (error != 0)
3181 break;
3182 continue;
3183 }
3184
3185 sleep:
3186 /*
3187 * Contention bit is set, before sleeping, increase
3188 * read waiter count.
3189 */
3190 rv = fueword32(&rwlock->rw_blocked_readers,
3191 &blocked_readers);
3192 if (rv == 0)
3193 rv = suword32(&rwlock->rw_blocked_readers,
3194 blocked_readers + 1);
3195 if (rv == -1) {
3196 umtxq_unbusy_unlocked(&uq->uq_key);
3197 error = EFAULT;
3198 break;
3199 }
3200
3201 while (state & wrflags) {
3202 umtxq_lock(&uq->uq_key);
3203 umtxq_insert(uq);
3204 umtxq_unbusy(&uq->uq_key);
3205
3206 error = umtxq_sleep(uq, "urdlck", timeout == NULL ?
3207 NULL : &timo);
3208
3209 umtxq_busy(&uq->uq_key);
3210 umtxq_remove(uq);
3211 umtxq_unlock(&uq->uq_key);
3212 if (error)
3213 break;
3214 rv = fueword32(&rwlock->rw_state, &state);
3215 if (rv == -1) {
3216 error = EFAULT;
3217 break;
3218 }
3219 }
3220
3221 /* decrease read waiter count, and may clear read contention bit */
3222 rv = fueword32(&rwlock->rw_blocked_readers,
3223 &blocked_readers);
3224 if (rv == 0)
3225 rv = suword32(&rwlock->rw_blocked_readers,
3226 blocked_readers - 1);
3227 if (rv == -1) {
3228 umtxq_unbusy_unlocked(&uq->uq_key);
3229 error = EFAULT;
3230 break;
3231 }
3232 if (blocked_readers == 1) {
3233 rv = fueword32(&rwlock->rw_state, &state);
3234 if (rv == -1) {
3235 umtxq_unbusy_unlocked(&uq->uq_key);
3236 error = EFAULT;
3237 break;
3238 }
3239 for (;;) {
3240 rv = casueword32(&rwlock->rw_state, state,
3241 &oldstate, state & ~URWLOCK_READ_WAITERS);
3242 if (rv == -1) {
3243 error = EFAULT;
3244 break;
3245 }
3246 if (rv == 0) {
3247 MPASS(oldstate == state);
3248 break;
3249 }
3250 state = oldstate;
3251 error1 = thread_check_susp(td, false);
3252 if (error1 != 0) {
3253 if (error == 0)
3254 error = error1;
3255 break;
3256 }
3257 }
3258 }
3259
3260 umtxq_unbusy_unlocked(&uq->uq_key);
3261 if (error != 0)
3262 break;
3263 }
3264 umtx_key_release(&uq->uq_key);
3265 if (error == ERESTART)
3266 error = EINTR;
3267 return (error);
3268 }
3269
3270 static int
do_rw_wrlock(struct thread * td,struct urwlock * rwlock,struct _umtx_time * timeout)3271 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout)
3272 {
3273 struct umtx_abs_timeout timo;
3274 struct umtx_q *uq;
3275 uint32_t flags;
3276 int32_t state, oldstate;
3277 int32_t blocked_writers;
3278 int32_t blocked_readers;
3279 int error, error1, rv;
3280
3281 uq = td->td_umtxq;
3282 error = fueword32(&rwlock->rw_flags, &flags);
3283 if (error == -1)
3284 return (EFAULT);
3285 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3286 if (error != 0)
3287 return (error);
3288
3289 if (timeout != NULL)
3290 umtx_abs_timeout_init2(&timo, timeout);
3291
3292 blocked_readers = 0;
3293 for (;;) {
3294 rv = fueword32(&rwlock->rw_state, &state);
3295 if (rv == -1) {
3296 umtx_key_release(&uq->uq_key);
3297 return (EFAULT);
3298 }
3299 while ((state & URWLOCK_WRITE_OWNER) == 0 &&
3300 URWLOCK_READER_COUNT(state) == 0) {
3301 rv = casueword32(&rwlock->rw_state, state,
3302 &oldstate, state | URWLOCK_WRITE_OWNER);
3303 if (rv == -1) {
3304 umtx_key_release(&uq->uq_key);
3305 return (EFAULT);
3306 }
3307 if (rv == 0) {
3308 MPASS(oldstate == state);
3309 umtx_key_release(&uq->uq_key);
3310 return (0);
3311 }
3312 state = oldstate;
3313 error = thread_check_susp(td, true);
3314 if (error != 0)
3315 break;
3316 }
3317
3318 if (error) {
3319 if ((state & (URWLOCK_WRITE_OWNER |
3320 URWLOCK_WRITE_WAITERS)) == 0 &&
3321 blocked_readers != 0) {
3322 umtxq_lock(&uq->uq_key);
3323 umtxq_busy(&uq->uq_key);
3324 umtxq_signal_queue(&uq->uq_key, INT_MAX,
3325 UMTX_SHARED_QUEUE);
3326 umtxq_unbusy(&uq->uq_key);
3327 umtxq_unlock(&uq->uq_key);
3328 }
3329
3330 break;
3331 }
3332
3333 /* grab monitor lock */
3334 umtxq_busy_unlocked(&uq->uq_key);
3335
3336 /*
3337 * Re-read the state, in case it changed between the
3338 * try-lock above and the check below.
3339 */
3340 rv = fueword32(&rwlock->rw_state, &state);
3341 if (rv == -1)
3342 error = EFAULT;
3343
3344 while (error == 0 && ((state & URWLOCK_WRITE_OWNER) ||
3345 URWLOCK_READER_COUNT(state) != 0) &&
3346 (state & URWLOCK_WRITE_WAITERS) == 0) {
3347 rv = casueword32(&rwlock->rw_state, state,
3348 &oldstate, state | URWLOCK_WRITE_WAITERS);
3349 if (rv == -1) {
3350 error = EFAULT;
3351 break;
3352 }
3353 if (rv == 0) {
3354 MPASS(oldstate == state);
3355 goto sleep;
3356 }
3357 state = oldstate;
3358 error = thread_check_susp(td, false);
3359 if (error != 0)
3360 break;
3361 }
3362 if (error != 0) {
3363 umtxq_unbusy_unlocked(&uq->uq_key);
3364 break;
3365 }
3366
3367 if ((state & URWLOCK_WRITE_OWNER) == 0 &&
3368 URWLOCK_READER_COUNT(state) == 0) {
3369 umtxq_unbusy_unlocked(&uq->uq_key);
3370 error = thread_check_susp(td, false);
3371 if (error != 0)
3372 break;
3373 continue;
3374 }
3375 sleep:
3376 rv = fueword32(&rwlock->rw_blocked_writers,
3377 &blocked_writers);
3378 if (rv == 0)
3379 rv = suword32(&rwlock->rw_blocked_writers,
3380 blocked_writers + 1);
3381 if (rv == -1) {
3382 umtxq_unbusy_unlocked(&uq->uq_key);
3383 error = EFAULT;
3384 break;
3385 }
3386
3387 while ((state & URWLOCK_WRITE_OWNER) ||
3388 URWLOCK_READER_COUNT(state) != 0) {
3389 umtxq_lock(&uq->uq_key);
3390 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3391 umtxq_unbusy(&uq->uq_key);
3392
3393 error = umtxq_sleep(uq, "uwrlck", timeout == NULL ?
3394 NULL : &timo);
3395
3396 umtxq_busy(&uq->uq_key);
3397 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3398 umtxq_unlock(&uq->uq_key);
3399 if (error)
3400 break;
3401 rv = fueword32(&rwlock->rw_state, &state);
3402 if (rv == -1) {
3403 error = EFAULT;
3404 break;
3405 }
3406 }
3407
3408 rv = fueword32(&rwlock->rw_blocked_writers,
3409 &blocked_writers);
3410 if (rv == 0)
3411 rv = suword32(&rwlock->rw_blocked_writers,
3412 blocked_writers - 1);
3413 if (rv == -1) {
3414 umtxq_unbusy_unlocked(&uq->uq_key);
3415 error = EFAULT;
3416 break;
3417 }
3418 if (blocked_writers == 1) {
3419 rv = fueword32(&rwlock->rw_state, &state);
3420 if (rv == -1) {
3421 umtxq_unbusy_unlocked(&uq->uq_key);
3422 error = EFAULT;
3423 break;
3424 }
3425 for (;;) {
3426 rv = casueword32(&rwlock->rw_state, state,
3427 &oldstate, state & ~URWLOCK_WRITE_WAITERS);
3428 if (rv == -1) {
3429 error = EFAULT;
3430 break;
3431 }
3432 if (rv == 0) {
3433 MPASS(oldstate == state);
3434 break;
3435 }
3436 state = oldstate;
3437 error1 = thread_check_susp(td, false);
3438 /*
3439 * We are leaving the URWLOCK_WRITE_WAITERS
3440 * behind, but this should not harm the
3441 * correctness.
3442 */
3443 if (error1 != 0) {
3444 if (error == 0)
3445 error = error1;
3446 break;
3447 }
3448 }
3449 rv = fueword32(&rwlock->rw_blocked_readers,
3450 &blocked_readers);
3451 if (rv == -1) {
3452 umtxq_unbusy_unlocked(&uq->uq_key);
3453 error = EFAULT;
3454 break;
3455 }
3456 } else
3457 blocked_readers = 0;
3458
3459 umtxq_unbusy_unlocked(&uq->uq_key);
3460 }
3461
3462 umtx_key_release(&uq->uq_key);
3463 if (error == ERESTART)
3464 error = EINTR;
3465 return (error);
3466 }
3467
3468 static int
do_rw_unlock(struct thread * td,struct urwlock * rwlock)3469 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
3470 {
3471 struct umtx_q *uq;
3472 uint32_t flags;
3473 int32_t state, oldstate;
3474 int error, rv, q, count;
3475
3476 uq = td->td_umtxq;
3477 error = fueword32(&rwlock->rw_flags, &flags);
3478 if (error == -1)
3479 return (EFAULT);
3480 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3481 if (error != 0)
3482 return (error);
3483
3484 error = fueword32(&rwlock->rw_state, &state);
3485 if (error == -1) {
3486 error = EFAULT;
3487 goto out;
3488 }
3489 if (state & URWLOCK_WRITE_OWNER) {
3490 for (;;) {
3491 rv = casueword32(&rwlock->rw_state, state,
3492 &oldstate, state & ~URWLOCK_WRITE_OWNER);
3493 if (rv == -1) {
3494 error = EFAULT;
3495 goto out;
3496 }
3497 if (rv == 1) {
3498 state = oldstate;
3499 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
3500 error = EPERM;
3501 goto out;
3502 }
3503 error = thread_check_susp(td, true);
3504 if (error != 0)
3505 goto out;
3506 } else
3507 break;
3508 }
3509 } else if (URWLOCK_READER_COUNT(state) != 0) {
3510 for (;;) {
3511 rv = casueword32(&rwlock->rw_state, state,
3512 &oldstate, state - 1);
3513 if (rv == -1) {
3514 error = EFAULT;
3515 goto out;
3516 }
3517 if (rv == 1) {
3518 state = oldstate;
3519 if (URWLOCK_READER_COUNT(oldstate) == 0) {
3520 error = EPERM;
3521 goto out;
3522 }
3523 error = thread_check_susp(td, true);
3524 if (error != 0)
3525 goto out;
3526 } else
3527 break;
3528 }
3529 } else {
3530 error = EPERM;
3531 goto out;
3532 }
3533
3534 count = 0;
3535
3536 if (!(flags & URWLOCK_PREFER_READER)) {
3537 if (state & URWLOCK_WRITE_WAITERS) {
3538 count = 1;
3539 q = UMTX_EXCLUSIVE_QUEUE;
3540 } else if (state & URWLOCK_READ_WAITERS) {
3541 count = INT_MAX;
3542 q = UMTX_SHARED_QUEUE;
3543 }
3544 } else {
3545 if (state & URWLOCK_READ_WAITERS) {
3546 count = INT_MAX;
3547 q = UMTX_SHARED_QUEUE;
3548 } else if (state & URWLOCK_WRITE_WAITERS) {
3549 count = 1;
3550 q = UMTX_EXCLUSIVE_QUEUE;
3551 }
3552 }
3553
3554 if (count) {
3555 umtxq_lock(&uq->uq_key);
3556 umtxq_busy(&uq->uq_key);
3557 umtxq_signal_queue(&uq->uq_key, count, q);
3558 umtxq_unbusy(&uq->uq_key);
3559 umtxq_unlock(&uq->uq_key);
3560 }
3561 out:
3562 umtx_key_release(&uq->uq_key);
3563 return (error);
3564 }
3565
3566 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
3567 static int
do_sem_wait(struct thread * td,struct _usem * sem,struct _umtx_time * timeout)3568 do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout)
3569 {
3570 struct umtx_abs_timeout timo;
3571 struct umtx_q *uq;
3572 uint32_t flags, count, count1;
3573 int error, rv, rv1;
3574
3575 uq = td->td_umtxq;
3576 error = fueword32(&sem->_flags, &flags);
3577 if (error == -1)
3578 return (EFAULT);
3579 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3580 if (error != 0)
3581 return (error);
3582
3583 if (timeout != NULL)
3584 umtx_abs_timeout_init2(&timo, timeout);
3585
3586 again:
3587 umtxq_lock(&uq->uq_key);
3588 umtxq_busy(&uq->uq_key);
3589 umtxq_insert(uq);
3590 umtxq_unlock(&uq->uq_key);
3591 rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
3592 if (rv != -1)
3593 rv1 = fueword32(&sem->_count, &count);
3594 if (rv == -1 || rv1 == -1 || count != 0 || (rv == 1 && count1 == 0)) {
3595 if (rv == 0)
3596 rv = suword32(&sem->_has_waiters, 0);
3597 umtxq_lock(&uq->uq_key);
3598 umtxq_unbusy(&uq->uq_key);
3599 umtxq_remove(uq);
3600 umtxq_unlock(&uq->uq_key);
3601 if (rv == -1 || rv1 == -1) {
3602 error = EFAULT;
3603 goto out;
3604 }
3605 if (count != 0) {
3606 error = 0;
3607 goto out;
3608 }
3609 MPASS(rv == 1 && count1 == 0);
3610 rv = thread_check_susp(td, true);
3611 if (rv == 0)
3612 goto again;
3613 error = rv;
3614 goto out;
3615 }
3616 umtxq_lock(&uq->uq_key);
3617 umtxq_unbusy(&uq->uq_key);
3618
3619 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3620
3621 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3622 error = 0;
3623 else {
3624 umtxq_remove(uq);
3625 /* A relative timeout cannot be restarted. */
3626 if (error == ERESTART && timeout != NULL &&
3627 (timeout->_flags & UMTX_ABSTIME) == 0)
3628 error = EINTR;
3629 }
3630 umtxq_unlock(&uq->uq_key);
3631 out:
3632 umtx_key_release(&uq->uq_key);
3633 return (error);
3634 }
3635
3636 /*
3637 * Signal a userland semaphore.
3638 */
3639 static int
do_sem_wake(struct thread * td,struct _usem * sem)3640 do_sem_wake(struct thread *td, struct _usem *sem)
3641 {
3642 struct umtx_key key;
3643 int error, cnt;
3644 uint32_t flags;
3645
3646 error = fueword32(&sem->_flags, &flags);
3647 if (error == -1)
3648 return (EFAULT);
3649 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3650 return (error);
3651 umtxq_lock(&key);
3652 umtxq_busy(&key);
3653 cnt = umtxq_count(&key);
3654 if (cnt > 0) {
3655 /*
3656 * Check if count is greater than 0, this means the memory is
3657 * still being referenced by user code, so we can safely
3658 * update _has_waiters flag.
3659 */
3660 if (cnt == 1) {
3661 umtxq_unlock(&key);
3662 error = suword32(&sem->_has_waiters, 0);
3663 umtxq_lock(&key);
3664 if (error == -1)
3665 error = EFAULT;
3666 }
3667 umtxq_signal(&key, 1);
3668 }
3669 umtxq_unbusy(&key);
3670 umtxq_unlock(&key);
3671 umtx_key_release(&key);
3672 return (error);
3673 }
3674 #endif
3675
3676 static int
do_sem2_wait(struct thread * td,struct _usem2 * sem,struct _umtx_time * timeout)3677 do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
3678 {
3679 struct umtx_abs_timeout timo;
3680 struct umtx_q *uq;
3681 uint32_t count, flags;
3682 int error, rv;
3683
3684 uq = td->td_umtxq;
3685 flags = fuword32(&sem->_flags);
3686 if (timeout != NULL)
3687 umtx_abs_timeout_init2(&timo, timeout);
3688
3689 again:
3690 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3691 if (error != 0)
3692 return (error);
3693 umtxq_lock(&uq->uq_key);
3694 umtxq_busy(&uq->uq_key);
3695 umtxq_insert(uq);
3696 umtxq_unlock(&uq->uq_key);
3697 rv = fueword32(&sem->_count, &count);
3698 if (rv == -1) {
3699 umtxq_lock(&uq->uq_key);
3700 umtxq_unbusy(&uq->uq_key);
3701 umtxq_remove(uq);
3702 umtxq_unlock(&uq->uq_key);
3703 umtx_key_release(&uq->uq_key);
3704 return (EFAULT);
3705 }
3706 for (;;) {
3707 if (USEM_COUNT(count) != 0) {
3708 umtxq_lock(&uq->uq_key);
3709 umtxq_unbusy(&uq->uq_key);
3710 umtxq_remove(uq);
3711 umtxq_unlock(&uq->uq_key);
3712 umtx_key_release(&uq->uq_key);
3713 return (0);
3714 }
3715 if (count == USEM_HAS_WAITERS)
3716 break;
3717 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
3718 if (rv == 0)
3719 break;
3720 umtxq_lock(&uq->uq_key);
3721 umtxq_unbusy(&uq->uq_key);
3722 umtxq_remove(uq);
3723 umtxq_unlock(&uq->uq_key);
3724 umtx_key_release(&uq->uq_key);
3725 if (rv == -1)
3726 return (EFAULT);
3727 rv = thread_check_susp(td, true);
3728 if (rv != 0)
3729 return (rv);
3730 goto again;
3731 }
3732 umtxq_lock(&uq->uq_key);
3733 umtxq_unbusy(&uq->uq_key);
3734
3735 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3736
3737 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3738 error = 0;
3739 else {
3740 umtxq_remove(uq);
3741 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) {
3742 /* A relative timeout cannot be restarted. */
3743 if (error == ERESTART)
3744 error = EINTR;
3745 if (error == EINTR) {
3746 kern_clock_gettime(curthread, timo.clockid,
3747 &timo.cur);
3748 timespecsub(&timo.end, &timo.cur,
3749 &timeout->_timeout);
3750 }
3751 }
3752 }
3753 umtxq_unlock(&uq->uq_key);
3754 umtx_key_release(&uq->uq_key);
3755 return (error);
3756 }
3757
3758 /*
3759 * Signal a userland semaphore.
3760 */
3761 static int
do_sem2_wake(struct thread * td,struct _usem2 * sem)3762 do_sem2_wake(struct thread *td, struct _usem2 *sem)
3763 {
3764 struct umtx_key key;
3765 int error, cnt, rv;
3766 uint32_t count, flags;
3767
3768 rv = fueword32(&sem->_flags, &flags);
3769 if (rv == -1)
3770 return (EFAULT);
3771 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3772 return (error);
3773 umtxq_lock(&key);
3774 umtxq_busy(&key);
3775 cnt = umtxq_count(&key);
3776 if (cnt > 0) {
3777 /*
3778 * If this was the last sleeping thread, clear the waiters
3779 * flag in _count.
3780 */
3781 if (cnt == 1) {
3782 umtxq_unlock(&key);
3783 rv = fueword32(&sem->_count, &count);
3784 while (rv != -1 && count & USEM_HAS_WAITERS) {
3785 rv = casueword32(&sem->_count, count, &count,
3786 count & ~USEM_HAS_WAITERS);
3787 if (rv == 1) {
3788 rv = thread_check_susp(td, false);
3789 if (rv != 0)
3790 break;
3791 }
3792 }
3793 if (rv == -1)
3794 error = EFAULT;
3795 else if (rv > 0) {
3796 error = rv;
3797 }
3798 umtxq_lock(&key);
3799 }
3800
3801 umtxq_signal(&key, 1);
3802 }
3803 umtxq_unbusy(&key);
3804 umtxq_unlock(&key);
3805 umtx_key_release(&key);
3806 return (error);
3807 }
3808
3809 #ifdef COMPAT_FREEBSD10
3810 int
freebsd10__umtx_lock(struct thread * td,struct freebsd10__umtx_lock_args * uap)3811 freebsd10__umtx_lock(struct thread *td, struct freebsd10__umtx_lock_args *uap)
3812 {
3813 return (do_lock_umtx(td, uap->umtx, td->td_tid, 0));
3814 }
3815
3816 int
freebsd10__umtx_unlock(struct thread * td,struct freebsd10__umtx_unlock_args * uap)3817 freebsd10__umtx_unlock(struct thread *td,
3818 struct freebsd10__umtx_unlock_args *uap)
3819 {
3820 return (do_unlock_umtx(td, uap->umtx, td->td_tid));
3821 }
3822 #endif
3823
3824 inline int
umtx_copyin_timeout(const void * uaddr,struct timespec * tsp)3825 umtx_copyin_timeout(const void *uaddr, struct timespec *tsp)
3826 {
3827 int error;
3828
3829 error = copyin(uaddr, tsp, sizeof(*tsp));
3830 if (error == 0) {
3831 if (!timespecvalid_interval(tsp))
3832 error = EINVAL;
3833 }
3834 return (error);
3835 }
3836
3837 static inline int
umtx_copyin_umtx_time(const void * uaddr,size_t size,struct _umtx_time * tp)3838 umtx_copyin_umtx_time(const void *uaddr, size_t size, struct _umtx_time *tp)
3839 {
3840 int error;
3841
3842 if (size <= sizeof(tp->_timeout)) {
3843 tp->_clockid = CLOCK_REALTIME;
3844 tp->_flags = 0;
3845 error = copyin(uaddr, &tp->_timeout, sizeof(tp->_timeout));
3846 } else
3847 error = copyin(uaddr, tp, sizeof(*tp));
3848 if (error != 0)
3849 return (error);
3850 if (!timespecvalid_interval(&tp->_timeout))
3851 return (EINVAL);
3852 return (0);
3853 }
3854
3855 static int
umtx_copyin_robust_lists(const void * uaddr,size_t size,struct umtx_robust_lists_params * rb)3856 umtx_copyin_robust_lists(const void *uaddr, size_t size,
3857 struct umtx_robust_lists_params *rb)
3858 {
3859
3860 if (size > sizeof(*rb))
3861 return (EINVAL);
3862 return (copyin(uaddr, rb, size));
3863 }
3864
3865 static int
umtx_copyout_timeout(void * uaddr,size_t sz,struct timespec * tsp)3866 umtx_copyout_timeout(void *uaddr, size_t sz, struct timespec *tsp)
3867 {
3868
3869 /*
3870 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
3871 * and we're only called if sz >= sizeof(timespec) as supplied in the
3872 * copyops.
3873 */
3874 KASSERT(sz >= sizeof(*tsp),
3875 ("umtx_copyops specifies incorrect sizes"));
3876
3877 return (copyout(tsp, uaddr, sizeof(*tsp)));
3878 }
3879
3880 #ifdef COMPAT_FREEBSD10
3881 static int
__umtx_op_lock_umtx(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3882 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap,
3883 const struct umtx_copyops *ops)
3884 {
3885 struct timespec *ts, timeout;
3886 int error;
3887
3888 /* Allow a null timespec (wait forever). */
3889 if (uap->uaddr2 == NULL)
3890 ts = NULL;
3891 else {
3892 error = ops->copyin_timeout(uap->uaddr2, &timeout);
3893 if (error != 0)
3894 return (error);
3895 ts = &timeout;
3896 }
3897 #ifdef COMPAT_FREEBSD32
3898 if (ops->compat32)
3899 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3900 #endif
3901 return (do_lock_umtx(td, uap->obj, uap->val, ts));
3902 }
3903
3904 static int
__umtx_op_unlock_umtx(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3905 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap,
3906 const struct umtx_copyops *ops)
3907 {
3908 #ifdef COMPAT_FREEBSD32
3909 if (ops->compat32)
3910 return (do_unlock_umtx32(td, uap->obj, uap->val));
3911 #endif
3912 return (do_unlock_umtx(td, uap->obj, uap->val));
3913 }
3914 #endif /* COMPAT_FREEBSD10 */
3915
3916 #if !defined(COMPAT_FREEBSD10)
3917 static int
__umtx_op_unimpl(struct thread * td __unused,struct _umtx_op_args * uap __unused,const struct umtx_copyops * ops __unused)3918 __umtx_op_unimpl(struct thread *td __unused, struct _umtx_op_args *uap __unused,
3919 const struct umtx_copyops *ops __unused)
3920 {
3921 return (EOPNOTSUPP);
3922 }
3923 #endif /* COMPAT_FREEBSD10 */
3924
3925 static int
__umtx_op_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3926 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap,
3927 const struct umtx_copyops *ops)
3928 {
3929 struct _umtx_time timeout, *tm_p;
3930 int error;
3931
3932 if (uap->uaddr2 == NULL)
3933 tm_p = NULL;
3934 else {
3935 error = ops->copyin_umtx_time(
3936 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3937 if (error != 0)
3938 return (error);
3939 tm_p = &timeout;
3940 }
3941 return (do_wait(td, uap->obj, uap->val, tm_p, ops->compat32, 0));
3942 }
3943
3944 static int
__umtx_op_wait_uint(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3945 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap,
3946 const struct umtx_copyops *ops)
3947 {
3948 struct _umtx_time timeout, *tm_p;
3949 int error;
3950
3951 if (uap->uaddr2 == NULL)
3952 tm_p = NULL;
3953 else {
3954 error = ops->copyin_umtx_time(
3955 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3956 if (error != 0)
3957 return (error);
3958 tm_p = &timeout;
3959 }
3960 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
3961 }
3962
3963 static int
__umtx_op_wait_uint_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3964 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap,
3965 const struct umtx_copyops *ops)
3966 {
3967 struct _umtx_time *tm_p, timeout;
3968 int error;
3969
3970 if (uap->uaddr2 == NULL)
3971 tm_p = NULL;
3972 else {
3973 error = ops->copyin_umtx_time(
3974 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3975 if (error != 0)
3976 return (error);
3977 tm_p = &timeout;
3978 }
3979 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
3980 }
3981
3982 static int
__umtx_op_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)3983 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap,
3984 const struct umtx_copyops *ops __unused)
3985 {
3986
3987 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3988 }
3989
3990 #define BATCH_SIZE 128
3991 static int
__umtx_op_nwake_private_native(struct thread * td,struct _umtx_op_args * uap)3992 __umtx_op_nwake_private_native(struct thread *td, struct _umtx_op_args *uap)
3993 {
3994 char *uaddrs[BATCH_SIZE], **upp;
3995 int count, error, i, pos, tocopy;
3996
3997 upp = (char **)uap->obj;
3998 error = 0;
3999 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4000 pos += tocopy) {
4001 tocopy = MIN(count, BATCH_SIZE);
4002 error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *));
4003 if (error != 0)
4004 break;
4005 for (i = 0; i < tocopy; ++i) {
4006 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1);
4007 }
4008 maybe_yield();
4009 }
4010 return (error);
4011 }
4012
4013 static int
__umtx_op_nwake_private_compat32(struct thread * td,struct _umtx_op_args * uap)4014 __umtx_op_nwake_private_compat32(struct thread *td, struct _umtx_op_args *uap)
4015 {
4016 uint32_t uaddrs[BATCH_SIZE], *upp;
4017 int count, error, i, pos, tocopy;
4018
4019 upp = (uint32_t *)uap->obj;
4020 error = 0;
4021 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4022 pos += tocopy) {
4023 tocopy = MIN(count, BATCH_SIZE);
4024 error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t));
4025 if (error != 0)
4026 break;
4027 for (i = 0; i < tocopy; ++i) {
4028 kern_umtx_wake(td, (void *)(uintptr_t)uaddrs[i],
4029 INT_MAX, 1);
4030 }
4031 maybe_yield();
4032 }
4033 return (error);
4034 }
4035
4036 static int
__umtx_op_nwake_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4037 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap,
4038 const struct umtx_copyops *ops)
4039 {
4040
4041 if (ops->compat32)
4042 return (__umtx_op_nwake_private_compat32(td, uap));
4043 return (__umtx_op_nwake_private_native(td, uap));
4044 }
4045
4046 static int
__umtx_op_wake_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4047 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap,
4048 const struct umtx_copyops *ops __unused)
4049 {
4050
4051 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
4052 }
4053
4054 static int
__umtx_op_lock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4055 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap,
4056 const struct umtx_copyops *ops)
4057 {
4058 struct _umtx_time *tm_p, timeout;
4059 int error;
4060
4061 /* Allow a null timespec (wait forever). */
4062 if (uap->uaddr2 == NULL)
4063 tm_p = NULL;
4064 else {
4065 error = ops->copyin_umtx_time(
4066 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4067 if (error != 0)
4068 return (error);
4069 tm_p = &timeout;
4070 }
4071 return (do_lock_umutex(td, uap->obj, tm_p, 0));
4072 }
4073
4074 static int
__umtx_op_trylock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4075 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap,
4076 const struct umtx_copyops *ops __unused)
4077 {
4078
4079 return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY));
4080 }
4081
4082 static int
__umtx_op_wait_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4083 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap,
4084 const struct umtx_copyops *ops)
4085 {
4086 struct _umtx_time *tm_p, timeout;
4087 int error;
4088
4089 /* Allow a null timespec (wait forever). */
4090 if (uap->uaddr2 == NULL)
4091 tm_p = NULL;
4092 else {
4093 error = ops->copyin_umtx_time(
4094 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4095 if (error != 0)
4096 return (error);
4097 tm_p = &timeout;
4098 }
4099 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
4100 }
4101
4102 static int
__umtx_op_wake_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4103 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap,
4104 const struct umtx_copyops *ops __unused)
4105 {
4106
4107 return (do_wake_umutex(td, uap->obj));
4108 }
4109
4110 static int
__umtx_op_unlock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4111 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap,
4112 const struct umtx_copyops *ops __unused)
4113 {
4114
4115 return (do_unlock_umutex(td, uap->obj, false));
4116 }
4117
4118 static int
__umtx_op_set_ceiling(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4119 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap,
4120 const struct umtx_copyops *ops __unused)
4121 {
4122
4123 return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1));
4124 }
4125
4126 static int
__umtx_op_cv_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4127 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap,
4128 const struct umtx_copyops *ops)
4129 {
4130 struct umtx_abs_timeout *timop, timo;
4131 struct timespec *ts, timeout;
4132 struct _umtx_time umtime;
4133 struct ucond *cv;
4134 u_long wflags;
4135 uint32_t clockid;
4136 int error;
4137
4138 cv = uap->obj;
4139 wflags = uap->val;
4140 if ((wflags & ~(CVWAIT_CHECK_UNPARKING | CVWAIT_ABSTIME |
4141 CVWAIT_CLOCKID | CVWAIT_UMTX_TIME)) != 0 ||
4142 ((wflags & (CVWAIT_ABSTIME | CVWAIT_CLOCKID)) != 0 &&
4143 (wflags & CVWAIT_UMTX_TIME) != 0))
4144 return (EINVAL);
4145
4146 if ((wflags & CVWAIT_UMTX_TIME) == 0) {
4147 /* Allow a null timespec (wait forever). */
4148 if (uap->uaddr2 == NULL) {
4149 ts = NULL;
4150 } else {
4151 error = ops->copyin_timeout(uap->uaddr2, &timeout);
4152 if (error != 0)
4153 return (error);
4154 ts = &timeout;
4155 }
4156 if ((wflags & CVWAIT_CLOCKID) != 0) {
4157 error = fueword32(&cv->c_clockid, &clockid);
4158 if (error == -1)
4159 return (EFAULT);
4160 } else {
4161 clockid = CLOCK_REALTIME;
4162 }
4163 if (ts != NULL) {
4164 umtx_abs_timeout_init(&timo, clockid,
4165 (wflags & CVWAIT_ABSTIME) != 0, ts);
4166 timop = &timo;
4167 } else {
4168 timop = NULL;
4169 }
4170 } else {
4171 if (uap->uaddr2 == NULL)
4172 return (EINVAL);
4173 error = ops->copyin_umtx_time(uap->uaddr2, ops->umtx_time_sz,
4174 &umtime);
4175 if (error != 0)
4176 return (error);
4177 timop = &timo;
4178 umtx_abs_timeout_init2(timop, &umtime);
4179 }
4180 /* only HW clock id will work. */
4181 if (timop != NULL && (timop->clockid < CLOCK_REALTIME ||
4182 timop->clockid >= CLOCK_THREAD_CPUTIME_ID) &&
4183 timop->clockid != CLOCK_TAI)
4184 return (EINVAL);
4185
4186 return (do_cv_wait(td, cv, uap->uaddr1, timop, wflags));
4187 }
4188
4189 static int
__umtx_op_cv_signal(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4190 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap,
4191 const struct umtx_copyops *ops __unused)
4192 {
4193
4194 return (do_cv_signal(td, uap->obj));
4195 }
4196
4197 static int
__umtx_op_cv_broadcast(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4198 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap,
4199 const struct umtx_copyops *ops __unused)
4200 {
4201
4202 return (do_cv_broadcast(td, uap->obj));
4203 }
4204
4205 static int
__umtx_op_rw_rdlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4206 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap,
4207 const struct umtx_copyops *ops)
4208 {
4209 struct _umtx_time timeout;
4210 int error;
4211
4212 /* Allow a null timespec (wait forever). */
4213 if (uap->uaddr2 == NULL) {
4214 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
4215 } else {
4216 error = ops->copyin_umtx_time(uap->uaddr2,
4217 (size_t)uap->uaddr1, &timeout);
4218 if (error != 0)
4219 return (error);
4220 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
4221 }
4222 return (error);
4223 }
4224
4225 static int
__umtx_op_rw_wrlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4226 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap,
4227 const struct umtx_copyops *ops)
4228 {
4229 struct _umtx_time timeout;
4230 int error;
4231
4232 /* Allow a null timespec (wait forever). */
4233 if (uap->uaddr2 == NULL) {
4234 error = do_rw_wrlock(td, uap->obj, 0);
4235 } else {
4236 error = ops->copyin_umtx_time(uap->uaddr2,
4237 (size_t)uap->uaddr1, &timeout);
4238 if (error != 0)
4239 return (error);
4240
4241 error = do_rw_wrlock(td, uap->obj, &timeout);
4242 }
4243 return (error);
4244 }
4245
4246 static int
__umtx_op_rw_unlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4247 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap,
4248 const struct umtx_copyops *ops __unused)
4249 {
4250
4251 return (do_rw_unlock(td, uap->obj));
4252 }
4253
4254 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4255 static int
__umtx_op_sem_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4256 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap,
4257 const struct umtx_copyops *ops)
4258 {
4259 struct _umtx_time *tm_p, timeout;
4260 int error;
4261
4262 /* Allow a null timespec (wait forever). */
4263 if (uap->uaddr2 == NULL)
4264 tm_p = NULL;
4265 else {
4266 error = ops->copyin_umtx_time(
4267 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4268 if (error != 0)
4269 return (error);
4270 tm_p = &timeout;
4271 }
4272 return (do_sem_wait(td, uap->obj, tm_p));
4273 }
4274
4275 static int
__umtx_op_sem_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4276 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap,
4277 const struct umtx_copyops *ops __unused)
4278 {
4279
4280 return (do_sem_wake(td, uap->obj));
4281 }
4282 #endif
4283
4284 static int
__umtx_op_wake2_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4285 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap,
4286 const struct umtx_copyops *ops __unused)
4287 {
4288
4289 return (do_wake2_umutex(td, uap->obj, uap->val));
4290 }
4291
4292 static int
__umtx_op_sem2_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4293 __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap,
4294 const struct umtx_copyops *ops)
4295 {
4296 struct _umtx_time *tm_p, timeout;
4297 size_t uasize;
4298 int error;
4299
4300 /* Allow a null timespec (wait forever). */
4301 if (uap->uaddr2 == NULL) {
4302 uasize = 0;
4303 tm_p = NULL;
4304 } else {
4305 uasize = (size_t)uap->uaddr1;
4306 error = ops->copyin_umtx_time(uap->uaddr2, uasize, &timeout);
4307 if (error != 0)
4308 return (error);
4309 tm_p = &timeout;
4310 }
4311 error = do_sem2_wait(td, uap->obj, tm_p);
4312 if (error == EINTR && uap->uaddr2 != NULL &&
4313 (timeout._flags & UMTX_ABSTIME) == 0 &&
4314 uasize >= ops->umtx_time_sz + ops->timespec_sz) {
4315 error = ops->copyout_timeout(
4316 (void *)((uintptr_t)uap->uaddr2 + ops->umtx_time_sz),
4317 uasize - ops->umtx_time_sz, &timeout._timeout);
4318 if (error == 0) {
4319 error = EINTR;
4320 }
4321 }
4322
4323 return (error);
4324 }
4325
4326 static int
__umtx_op_sem2_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4327 __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap,
4328 const struct umtx_copyops *ops __unused)
4329 {
4330
4331 return (do_sem2_wake(td, uap->obj));
4332 }
4333
4334 #define USHM_OBJ_UMTX(o) \
4335 ((struct umtx_shm_obj_list *)(&(o)->umtx_data))
4336
4337 #define USHMF_LINKED 0x0001
4338 struct umtx_shm_reg {
4339 TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link;
4340 LIST_ENTRY(umtx_shm_reg) ushm_obj_link;
4341 struct umtx_key ushm_key;
4342 struct ucred *ushm_cred;
4343 struct shmfd *ushm_obj;
4344 u_int ushm_refcnt;
4345 u_int ushm_flags;
4346 };
4347
4348 LIST_HEAD(umtx_shm_obj_list, umtx_shm_reg);
4349 TAILQ_HEAD(umtx_shm_reg_head, umtx_shm_reg);
4350
4351 static uma_zone_t umtx_shm_reg_zone;
4352 static struct umtx_shm_reg_head umtx_shm_registry[UMTX_CHAINS];
4353 static struct mtx umtx_shm_lock;
4354 static struct umtx_shm_reg_head umtx_shm_reg_delfree =
4355 TAILQ_HEAD_INITIALIZER(umtx_shm_reg_delfree);
4356
4357 static void umtx_shm_free_reg(struct umtx_shm_reg *reg);
4358
4359 static void
umtx_shm_reg_delfree_tq(void * context __unused,int pending __unused)4360 umtx_shm_reg_delfree_tq(void *context __unused, int pending __unused)
4361 {
4362 struct umtx_shm_reg_head d;
4363 struct umtx_shm_reg *reg, *reg1;
4364
4365 TAILQ_INIT(&d);
4366 mtx_lock(&umtx_shm_lock);
4367 TAILQ_CONCAT(&d, &umtx_shm_reg_delfree, ushm_reg_link);
4368 mtx_unlock(&umtx_shm_lock);
4369 TAILQ_FOREACH_SAFE(reg, &d, ushm_reg_link, reg1) {
4370 TAILQ_REMOVE(&d, reg, ushm_reg_link);
4371 umtx_shm_free_reg(reg);
4372 }
4373 }
4374
4375 static struct task umtx_shm_reg_delfree_task =
4376 TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL);
4377
4378 /*
4379 * Returns 0 if a SHM with the passed key is found in the registry, in which
4380 * case it is returned through 'oreg'. Otherwise, returns an error among ESRCH
4381 * (no corresponding SHM; ESRCH was chosen for compatibility, ENOENT would have
4382 * been preferable) or EOVERFLOW (there is a corresponding SHM, but reference
4383 * count would overflow, so can't return it), in which case '*oreg' is left
4384 * unchanged.
4385 */
4386 static int
umtx_shm_find_reg_locked(const struct umtx_key * key,struct umtx_shm_reg ** const oreg)4387 umtx_shm_find_reg_locked(const struct umtx_key *key,
4388 struct umtx_shm_reg **const oreg)
4389 {
4390 struct umtx_shm_reg *reg;
4391 struct umtx_shm_reg_head *reg_head;
4392
4393 KASSERT(key->shared, ("umtx_p_find_rg: private key"));
4394 mtx_assert(&umtx_shm_lock, MA_OWNED);
4395 reg_head = &umtx_shm_registry[key->hash];
4396 TAILQ_FOREACH(reg, reg_head, ushm_reg_link) {
4397 KASSERT(reg->ushm_key.shared,
4398 ("non-shared key on reg %p %d", reg, reg->ushm_key.shared));
4399 if (reg->ushm_key.info.shared.object ==
4400 key->info.shared.object &&
4401 reg->ushm_key.info.shared.offset ==
4402 key->info.shared.offset) {
4403 KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM"));
4404 KASSERT(reg->ushm_refcnt != 0,
4405 ("reg %p refcnt 0 onlist", reg));
4406 KASSERT((reg->ushm_flags & USHMF_LINKED) != 0,
4407 ("reg %p not linked", reg));
4408 /*
4409 * Don't let overflow happen, just deny a new reference
4410 * (this is additional protection against some reference
4411 * count leak, which is known not to be the case at the
4412 * time of this writing).
4413 */
4414 if (__predict_false(reg->ushm_refcnt == UINT_MAX))
4415 return (EOVERFLOW);
4416 reg->ushm_refcnt++;
4417 *oreg = reg;
4418 return (0);
4419 }
4420 }
4421 return (ESRCH);
4422 }
4423
4424 /*
4425 * Calls umtx_shm_find_reg_unlocked() under the 'umtx_shm_lock'.
4426 */
4427 static int
umtx_shm_find_reg(const struct umtx_key * key,struct umtx_shm_reg ** const oreg)4428 umtx_shm_find_reg(const struct umtx_key *key, struct umtx_shm_reg **const oreg)
4429 {
4430 int error;
4431
4432 mtx_lock(&umtx_shm_lock);
4433 error = umtx_shm_find_reg_locked(key, oreg);
4434 mtx_unlock(&umtx_shm_lock);
4435 return (error);
4436 }
4437
4438 static void
umtx_shm_free_reg(struct umtx_shm_reg * reg)4439 umtx_shm_free_reg(struct umtx_shm_reg *reg)
4440 {
4441
4442 chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0);
4443 crfree(reg->ushm_cred);
4444 shm_drop(reg->ushm_obj);
4445 uma_zfree(umtx_shm_reg_zone, reg);
4446 }
4447
4448 static bool
umtx_shm_unref_reg_locked(struct umtx_shm_reg * reg,bool linked_ref)4449 umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool linked_ref)
4450 {
4451 mtx_assert(&umtx_shm_lock, MA_OWNED);
4452 KASSERT(reg->ushm_refcnt != 0, ("ushm_reg %p refcnt 0", reg));
4453
4454 if (linked_ref) {
4455 if ((reg->ushm_flags & USHMF_LINKED) == 0)
4456 /*
4457 * The reference tied to USHMF_LINKED has already been
4458 * released concurrently.
4459 */
4460 return (false);
4461
4462 TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], reg,
4463 ushm_reg_link);
4464 LIST_REMOVE(reg, ushm_obj_link);
4465 reg->ushm_flags &= ~USHMF_LINKED;
4466 }
4467
4468 reg->ushm_refcnt--;
4469 return (reg->ushm_refcnt == 0);
4470 }
4471
4472 static void
umtx_shm_unref_reg(struct umtx_shm_reg * reg,bool linked_ref)4473 umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool linked_ref)
4474 {
4475 vm_object_t object;
4476 bool dofree;
4477
4478 if (linked_ref) {
4479 /*
4480 * Note: This may be executed multiple times on the same
4481 * shared-memory VM object in presence of concurrent callers
4482 * because 'umtx_shm_lock' is not held all along in umtx_shm()
4483 * and here.
4484 */
4485 object = reg->ushm_obj->shm_object;
4486 VM_OBJECT_WLOCK(object);
4487 vm_object_set_flag(object, OBJ_UMTXDEAD);
4488 VM_OBJECT_WUNLOCK(object);
4489 }
4490 mtx_lock(&umtx_shm_lock);
4491 dofree = umtx_shm_unref_reg_locked(reg, linked_ref);
4492 mtx_unlock(&umtx_shm_lock);
4493 if (dofree)
4494 umtx_shm_free_reg(reg);
4495 }
4496
4497 void
umtx_shm_object_init(vm_object_t object)4498 umtx_shm_object_init(vm_object_t object)
4499 {
4500
4501 LIST_INIT(USHM_OBJ_UMTX(object));
4502 }
4503
4504 void
umtx_shm_object_terminated(vm_object_t object)4505 umtx_shm_object_terminated(vm_object_t object)
4506 {
4507 struct umtx_shm_reg *reg, *reg1;
4508 bool dofree;
4509
4510 if (LIST_EMPTY(USHM_OBJ_UMTX(object)))
4511 return;
4512
4513 dofree = false;
4514 mtx_lock(&umtx_shm_lock);
4515 LIST_FOREACH_SAFE(reg, USHM_OBJ_UMTX(object), ushm_obj_link, reg1) {
4516 if (umtx_shm_unref_reg_locked(reg, true)) {
4517 TAILQ_INSERT_TAIL(&umtx_shm_reg_delfree, reg,
4518 ushm_reg_link);
4519 dofree = true;
4520 }
4521 }
4522 mtx_unlock(&umtx_shm_lock);
4523 if (dofree)
4524 taskqueue_enqueue(taskqueue_thread, &umtx_shm_reg_delfree_task);
4525 }
4526
4527 static int
umtx_shm_create_reg(struct thread * td,const struct umtx_key * key,struct umtx_shm_reg ** res)4528 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key,
4529 struct umtx_shm_reg **res)
4530 {
4531 struct shmfd *shm;
4532 struct umtx_shm_reg *reg, *reg1;
4533 struct ucred *cred;
4534 int error;
4535
4536 error = umtx_shm_find_reg(key, res);
4537 if (error != ESRCH) {
4538 /*
4539 * Either no error occured, and '*res' was filled, or EOVERFLOW
4540 * was returned, indicating a reference count limit, and we
4541 * won't create a duplicate registration. In both cases, we are
4542 * done.
4543 */
4544 return (error);
4545 }
4546 /* No entry, we will create one. */
4547
4548 cred = td->td_ucred;
4549 if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP)))
4550 return (ENOMEM);
4551 shm = shm_alloc(td->td_ucred, O_RDWR, false);
4552 if (shm == NULL) {
4553 chgumtxcnt(cred->cr_ruidinfo, -1, 0);
4554 return (ENOMEM);
4555 }
4556 reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO);
4557 bcopy(key, ®->ushm_key, sizeof(*key));
4558 reg->ushm_obj = shm;
4559 reg->ushm_cred = crhold(cred);
4560 error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE);
4561 if (error != 0) {
4562 umtx_shm_free_reg(reg);
4563 return (error);
4564 }
4565 mtx_lock(&umtx_shm_lock);
4566 /* Re-lookup as 'umtx_shm_lock' has been temporarily released. */
4567 error = umtx_shm_find_reg_locked(key, ®1);
4568 switch (error) {
4569 case 0:
4570 mtx_unlock(&umtx_shm_lock);
4571 umtx_shm_free_reg(reg);
4572 *res = reg1;
4573 return (0);
4574 case ESRCH:
4575 break;
4576 default:
4577 mtx_unlock(&umtx_shm_lock);
4578 umtx_shm_free_reg(reg);
4579 return (error);
4580 }
4581 TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link);
4582 LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg,
4583 ushm_obj_link);
4584 reg->ushm_flags = USHMF_LINKED;
4585 /*
4586 * This is one reference for the registry and the list of shared
4587 * mutexes referenced by the VM object containing the lock pointer, and
4588 * another for the caller, which it will free after use. So, one of
4589 * these is tied to the presence of USHMF_LINKED.
4590 */
4591 reg->ushm_refcnt = 2;
4592 mtx_unlock(&umtx_shm_lock);
4593 *res = reg;
4594 return (0);
4595 }
4596
4597 static int
umtx_shm_alive(struct thread * td,void * addr)4598 umtx_shm_alive(struct thread *td, void *addr)
4599 {
4600 vm_map_t map;
4601 vm_map_entry_t entry;
4602 vm_object_t object;
4603 vm_pindex_t pindex;
4604 vm_prot_t prot;
4605 int res, ret;
4606 boolean_t wired;
4607
4608 map = &td->td_proc->p_vmspace->vm_map;
4609 res = vm_map_lookup(&map, (uintptr_t)addr, VM_PROT_READ, &entry,
4610 &object, &pindex, &prot, &wired);
4611 if (res != KERN_SUCCESS)
4612 return (EFAULT);
4613 if (object == NULL)
4614 ret = EINVAL;
4615 else
4616 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0;
4617 vm_map_lookup_done(map, entry);
4618 return (ret);
4619 }
4620
4621 static void
umtx_shm_init(void)4622 umtx_shm_init(void)
4623 {
4624 int i;
4625
4626 umtx_shm_reg_zone = uma_zcreate("umtx_shm", sizeof(struct umtx_shm_reg),
4627 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
4628 mtx_init(&umtx_shm_lock, "umtxshm", NULL, MTX_DEF);
4629 for (i = 0; i < nitems(umtx_shm_registry); i++)
4630 TAILQ_INIT(&umtx_shm_registry[i]);
4631 }
4632
4633 static int
umtx_shm(struct thread * td,void * addr,u_int flags)4634 umtx_shm(struct thread *td, void *addr, u_int flags)
4635 {
4636 struct umtx_key key;
4637 struct umtx_shm_reg *reg;
4638 struct file *fp;
4639 int error, fd;
4640
4641 if (__bitcount(flags & (UMTX_SHM_CREAT | UMTX_SHM_LOOKUP |
4642 UMTX_SHM_DESTROY| UMTX_SHM_ALIVE)) != 1)
4643 return (EINVAL);
4644 if ((flags & UMTX_SHM_ALIVE) != 0)
4645 return (umtx_shm_alive(td, addr));
4646 error = umtx_key_get(addr, TYPE_SHM, PROCESS_SHARE, &key);
4647 if (error != 0)
4648 return (error);
4649 KASSERT(key.shared == 1, ("non-shared key"));
4650 error = (flags & UMTX_SHM_CREAT) != 0 ?
4651 umtx_shm_create_reg(td, &key, ®) :
4652 umtx_shm_find_reg(&key, ®);
4653 umtx_key_release(&key);
4654 if (error != 0)
4655 return (error);
4656 KASSERT(reg != NULL, ("no reg"));
4657 if ((flags & UMTX_SHM_DESTROY) != 0) {
4658 umtx_shm_unref_reg(reg, true);
4659 } else {
4660 /*
4661 * The current vmspace has the mapping, so it can be
4662 * converted into shm filedescriptor for current
4663 * thread.
4664 */
4665 error = falloc_caps(td, &fp, &fd, O_CLOEXEC, NULL);
4666 if (error == 0) {
4667 shm_hold(reg->ushm_obj);
4668 finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj,
4669 &shm_ops);
4670 td->td_retval[0] = fd;
4671 fdrop(fp, td);
4672 }
4673 }
4674 umtx_shm_unref_reg(reg, false);
4675 return (error);
4676 }
4677
4678 static int
__umtx_op_shm(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4679 __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap,
4680 const struct umtx_copyops *ops __unused)
4681 {
4682
4683 return (umtx_shm(td, uap->uaddr1, uap->val));
4684 }
4685
4686 static int
__umtx_op_robust_lists(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4687 __umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap,
4688 const struct umtx_copyops *ops)
4689 {
4690 struct umtx_robust_lists_params rb;
4691 int error;
4692
4693 if (ops->compat32) {
4694 if ((td->td_pflags2 & TDP2_COMPAT32RB) == 0 &&
4695 (td->td_rb_list != 0 || td->td_rbp_list != 0 ||
4696 td->td_rb_inact != 0))
4697 return (EBUSY);
4698 } else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) {
4699 return (EBUSY);
4700 }
4701
4702 bzero(&rb, sizeof(rb));
4703 error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb);
4704 if (error != 0)
4705 return (error);
4706
4707 if (ops->compat32)
4708 td->td_pflags2 |= TDP2_COMPAT32RB;
4709
4710 td->td_rb_list = rb.robust_list_offset;
4711 td->td_rbp_list = rb.robust_priv_list_offset;
4712 td->td_rb_inact = rb.robust_inact_offset;
4713 return (0);
4714 }
4715
4716 static int
__umtx_op_get_min_timeout(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4717 __umtx_op_get_min_timeout(struct thread *td, struct _umtx_op_args *uap,
4718 const struct umtx_copyops *ops)
4719 {
4720 long val;
4721 int error, val1;
4722
4723 val = sbttons(td->td_proc->p_umtx_min_timeout);
4724 if (ops->compat32) {
4725 val1 = (int)val;
4726 error = copyout(&val1, uap->uaddr1, sizeof(val1));
4727 } else {
4728 error = copyout(&val, uap->uaddr1, sizeof(val));
4729 }
4730 return (error);
4731 }
4732
4733 static int
__umtx_op_set_min_timeout(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4734 __umtx_op_set_min_timeout(struct thread *td, struct _umtx_op_args *uap,
4735 const struct umtx_copyops *ops)
4736 {
4737 if (uap->val < 0)
4738 return (EINVAL);
4739 td->td_proc->p_umtx_min_timeout = nstosbt(uap->val);
4740 return (0);
4741 }
4742
4743 #if defined(__i386__) || defined(__amd64__)
4744 /*
4745 * Provide the standard 32-bit definitions for x86, since native/compat32 use a
4746 * 32-bit time_t there. Other architectures just need the i386 definitions
4747 * along with their standard compat32.
4748 */
4749 struct timespecx32 {
4750 int64_t tv_sec;
4751 int32_t tv_nsec;
4752 };
4753
4754 struct umtx_timex32 {
4755 struct timespecx32 _timeout;
4756 uint32_t _flags;
4757 uint32_t _clockid;
4758 };
4759
4760 #ifndef __i386__
4761 #define timespeci386 timespec32
4762 #define umtx_timei386 umtx_time32
4763 #endif
4764 #else /* !__i386__ && !__amd64__ */
4765 /* 32-bit architectures can emulate i386, so define these almost everywhere. */
4766 struct timespeci386 {
4767 int32_t tv_sec;
4768 int32_t tv_nsec;
4769 };
4770
4771 struct umtx_timei386 {
4772 struct timespeci386 _timeout;
4773 uint32_t _flags;
4774 uint32_t _clockid;
4775 };
4776
4777 #if defined(__LP64__)
4778 #define timespecx32 timespec32
4779 #define umtx_timex32 umtx_time32
4780 #endif
4781 #endif
4782
4783 static int
umtx_copyin_robust_lists32(const void * uaddr,size_t size,struct umtx_robust_lists_params * rbp)4784 umtx_copyin_robust_lists32(const void *uaddr, size_t size,
4785 struct umtx_robust_lists_params *rbp)
4786 {
4787 struct umtx_robust_lists_params_compat32 rb32;
4788 int error;
4789
4790 if (size > sizeof(rb32))
4791 return (EINVAL);
4792 bzero(&rb32, sizeof(rb32));
4793 error = copyin(uaddr, &rb32, size);
4794 if (error != 0)
4795 return (error);
4796 CP(rb32, *rbp, robust_list_offset);
4797 CP(rb32, *rbp, robust_priv_list_offset);
4798 CP(rb32, *rbp, robust_inact_offset);
4799 return (0);
4800 }
4801
4802 #ifndef __i386__
4803 static inline int
umtx_copyin_timeouti386(const void * uaddr,struct timespec * tsp)4804 umtx_copyin_timeouti386(const void *uaddr, struct timespec *tsp)
4805 {
4806 struct timespeci386 ts32;
4807 int error;
4808
4809 error = copyin(uaddr, &ts32, sizeof(ts32));
4810 if (error == 0) {
4811 if (!timespecvalid_interval(&ts32))
4812 error = EINVAL;
4813 else {
4814 CP(ts32, *tsp, tv_sec);
4815 CP(ts32, *tsp, tv_nsec);
4816 }
4817 }
4818 return (error);
4819 }
4820
4821 static inline int
umtx_copyin_umtx_timei386(const void * uaddr,size_t size,struct _umtx_time * tp)4822 umtx_copyin_umtx_timei386(const void *uaddr, size_t size, struct _umtx_time *tp)
4823 {
4824 struct umtx_timei386 t32;
4825 int error;
4826
4827 t32._clockid = CLOCK_REALTIME;
4828 t32._flags = 0;
4829 if (size <= sizeof(t32._timeout))
4830 error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout));
4831 else
4832 error = copyin(uaddr, &t32, sizeof(t32));
4833 if (error != 0)
4834 return (error);
4835 if (!timespecvalid_interval(&t32._timeout))
4836 return (EINVAL);
4837 TS_CP(t32, *tp, _timeout);
4838 CP(t32, *tp, _flags);
4839 CP(t32, *tp, _clockid);
4840 return (0);
4841 }
4842
4843 static int
umtx_copyout_timeouti386(void * uaddr,size_t sz,struct timespec * tsp)4844 umtx_copyout_timeouti386(void *uaddr, size_t sz, struct timespec *tsp)
4845 {
4846 struct timespeci386 remain32 = {
4847 .tv_sec = tsp->tv_sec,
4848 .tv_nsec = tsp->tv_nsec,
4849 };
4850
4851 /*
4852 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4853 * and we're only called if sz >= sizeof(timespec) as supplied in the
4854 * copyops.
4855 */
4856 KASSERT(sz >= sizeof(remain32),
4857 ("umtx_copyops specifies incorrect sizes"));
4858
4859 return (copyout(&remain32, uaddr, sizeof(remain32)));
4860 }
4861 #endif /* !__i386__ */
4862
4863 #if defined(__i386__) || defined(__LP64__)
4864 static inline int
umtx_copyin_timeoutx32(const void * uaddr,struct timespec * tsp)4865 umtx_copyin_timeoutx32(const void *uaddr, struct timespec *tsp)
4866 {
4867 struct timespecx32 ts32;
4868 int error;
4869
4870 error = copyin(uaddr, &ts32, sizeof(ts32));
4871 if (error == 0) {
4872 if (!timespecvalid_interval(&ts32))
4873 error = EINVAL;
4874 else {
4875 CP(ts32, *tsp, tv_sec);
4876 CP(ts32, *tsp, tv_nsec);
4877 }
4878 }
4879 return (error);
4880 }
4881
4882 static inline int
umtx_copyin_umtx_timex32(const void * uaddr,size_t size,struct _umtx_time * tp)4883 umtx_copyin_umtx_timex32(const void *uaddr, size_t size, struct _umtx_time *tp)
4884 {
4885 struct umtx_timex32 t32;
4886 int error;
4887
4888 t32._clockid = CLOCK_REALTIME;
4889 t32._flags = 0;
4890 if (size <= sizeof(t32._timeout))
4891 error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout));
4892 else
4893 error = copyin(uaddr, &t32, sizeof(t32));
4894 if (error != 0)
4895 return (error);
4896 if (!timespecvalid_interval(&t32._timeout))
4897 return (EINVAL);
4898 TS_CP(t32, *tp, _timeout);
4899 CP(t32, *tp, _flags);
4900 CP(t32, *tp, _clockid);
4901 return (0);
4902 }
4903
4904 static int
umtx_copyout_timeoutx32(void * uaddr,size_t sz,struct timespec * tsp)4905 umtx_copyout_timeoutx32(void *uaddr, size_t sz, struct timespec *tsp)
4906 {
4907 struct timespecx32 remain32 = {
4908 .tv_sec = tsp->tv_sec,
4909 .tv_nsec = tsp->tv_nsec,
4910 };
4911
4912 /*
4913 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4914 * and we're only called if sz >= sizeof(timespec) as supplied in the
4915 * copyops.
4916 */
4917 KASSERT(sz >= sizeof(remain32),
4918 ("umtx_copyops specifies incorrect sizes"));
4919
4920 return (copyout(&remain32, uaddr, sizeof(remain32)));
4921 }
4922 #endif /* __i386__ || __LP64__ */
4923
4924 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap,
4925 const struct umtx_copyops *umtx_ops);
4926
4927 static const _umtx_op_func op_table[] = {
4928 #ifdef COMPAT_FREEBSD10
4929 [UMTX_OP_LOCK] = __umtx_op_lock_umtx,
4930 [UMTX_OP_UNLOCK] = __umtx_op_unlock_umtx,
4931 #else
4932 [UMTX_OP_LOCK] = __umtx_op_unimpl,
4933 [UMTX_OP_UNLOCK] = __umtx_op_unimpl,
4934 #endif
4935 [UMTX_OP_WAIT] = __umtx_op_wait,
4936 [UMTX_OP_WAKE] = __umtx_op_wake,
4937 [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex,
4938 [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex,
4939 [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex,
4940 [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling,
4941 [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait,
4942 [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal,
4943 [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast,
4944 [UMTX_OP_WAIT_UINT] = __umtx_op_wait_uint,
4945 [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock,
4946 [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock,
4947 [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock,
4948 [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private,
4949 [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private,
4950 [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex,
4951 [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex,
4952 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4953 [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait,
4954 [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake,
4955 #else
4956 [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl,
4957 [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl,
4958 #endif
4959 [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private,
4960 [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex,
4961 [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait,
4962 [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake,
4963 [UMTX_OP_SHM] = __umtx_op_shm,
4964 [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists,
4965 [UMTX_OP_GET_MIN_TIMEOUT] = __umtx_op_get_min_timeout,
4966 [UMTX_OP_SET_MIN_TIMEOUT] = __umtx_op_set_min_timeout,
4967 };
4968
4969 static const struct umtx_copyops umtx_native_ops = {
4970 .copyin_timeout = umtx_copyin_timeout,
4971 .copyin_umtx_time = umtx_copyin_umtx_time,
4972 .copyin_robust_lists = umtx_copyin_robust_lists,
4973 .copyout_timeout = umtx_copyout_timeout,
4974 .timespec_sz = sizeof(struct timespec),
4975 .umtx_time_sz = sizeof(struct _umtx_time),
4976 };
4977
4978 #ifndef __i386__
4979 static const struct umtx_copyops umtx_native_opsi386 = {
4980 .copyin_timeout = umtx_copyin_timeouti386,
4981 .copyin_umtx_time = umtx_copyin_umtx_timei386,
4982 .copyin_robust_lists = umtx_copyin_robust_lists32,
4983 .copyout_timeout = umtx_copyout_timeouti386,
4984 .timespec_sz = sizeof(struct timespeci386),
4985 .umtx_time_sz = sizeof(struct umtx_timei386),
4986 .compat32 = true,
4987 };
4988 #endif
4989
4990 #if defined(__i386__) || defined(__LP64__)
4991 /* i386 can emulate other 32-bit archs, too! */
4992 static const struct umtx_copyops umtx_native_opsx32 = {
4993 .copyin_timeout = umtx_copyin_timeoutx32,
4994 .copyin_umtx_time = umtx_copyin_umtx_timex32,
4995 .copyin_robust_lists = umtx_copyin_robust_lists32,
4996 .copyout_timeout = umtx_copyout_timeoutx32,
4997 .timespec_sz = sizeof(struct timespecx32),
4998 .umtx_time_sz = sizeof(struct umtx_timex32),
4999 .compat32 = true,
5000 };
5001
5002 #ifdef COMPAT_FREEBSD32
5003 #ifdef __amd64__
5004 #define umtx_native_ops32 umtx_native_opsi386
5005 #else
5006 #define umtx_native_ops32 umtx_native_opsx32
5007 #endif
5008 #endif /* COMPAT_FREEBSD32 */
5009 #endif /* __i386__ || __LP64__ */
5010
5011 #define UMTX_OP__FLAGS (UMTX_OP__32BIT | UMTX_OP__I386)
5012
5013 static int
kern__umtx_op(struct thread * td,void * obj,int op,unsigned long val,void * uaddr1,void * uaddr2,const struct umtx_copyops * ops)5014 kern__umtx_op(struct thread *td, void *obj, int op, unsigned long val,
5015 void *uaddr1, void *uaddr2, const struct umtx_copyops *ops)
5016 {
5017 struct _umtx_op_args uap = {
5018 .obj = obj,
5019 .op = op & ~UMTX_OP__FLAGS,
5020 .val = val,
5021 .uaddr1 = uaddr1,
5022 .uaddr2 = uaddr2
5023 };
5024
5025 if ((uap.op >= nitems(op_table)))
5026 return (EINVAL);
5027 return ((*op_table[uap.op])(td, &uap, ops));
5028 }
5029
5030 int
sys__umtx_op(struct thread * td,struct _umtx_op_args * uap)5031 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap)
5032 {
5033 static const struct umtx_copyops *umtx_ops;
5034
5035 umtx_ops = &umtx_native_ops;
5036 #ifdef __LP64__
5037 if ((uap->op & (UMTX_OP__32BIT | UMTX_OP__I386)) != 0) {
5038 if ((uap->op & UMTX_OP__I386) != 0)
5039 umtx_ops = &umtx_native_opsi386;
5040 else
5041 umtx_ops = &umtx_native_opsx32;
5042 }
5043 #elif !defined(__i386__)
5044 /* We consider UMTX_OP__32BIT a nop on !i386 ILP32. */
5045 if ((uap->op & UMTX_OP__I386) != 0)
5046 umtx_ops = &umtx_native_opsi386;
5047 #else
5048 /* Likewise, UMTX_OP__I386 is a nop on i386. */
5049 if ((uap->op & UMTX_OP__32BIT) != 0)
5050 umtx_ops = &umtx_native_opsx32;
5051 #endif
5052 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5053 uap->uaddr2, umtx_ops));
5054 }
5055
5056 #ifdef COMPAT_FREEBSD32
5057 #ifdef COMPAT_FREEBSD10
5058 int
freebsd10_freebsd32__umtx_lock(struct thread * td,struct freebsd10_freebsd32__umtx_lock_args * uap)5059 freebsd10_freebsd32__umtx_lock(struct thread *td,
5060 struct freebsd10_freebsd32__umtx_lock_args *uap)
5061 {
5062 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
5063 }
5064
5065 int
freebsd10_freebsd32__umtx_unlock(struct thread * td,struct freebsd10_freebsd32__umtx_unlock_args * uap)5066 freebsd10_freebsd32__umtx_unlock(struct thread *td,
5067 struct freebsd10_freebsd32__umtx_unlock_args *uap)
5068 {
5069 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
5070 }
5071 #endif /* COMPAT_FREEBSD10 */
5072
5073 int
freebsd32__umtx_op(struct thread * td,struct freebsd32__umtx_op_args * uap)5074 freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap)
5075 {
5076
5077 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5078 uap->uaddr2, &umtx_native_ops32));
5079 }
5080 #endif /* COMPAT_FREEBSD32 */
5081
5082 void
umtx_thread_init(struct thread * td)5083 umtx_thread_init(struct thread *td)
5084 {
5085
5086 td->td_umtxq = umtxq_alloc();
5087 td->td_umtxq->uq_thread = td;
5088 }
5089
5090 void
umtx_thread_fini(struct thread * td)5091 umtx_thread_fini(struct thread *td)
5092 {
5093
5094 umtxq_free(td->td_umtxq);
5095 }
5096
5097 /*
5098 * It will be called when new thread is created, e.g fork().
5099 */
5100 void
umtx_thread_alloc(struct thread * td)5101 umtx_thread_alloc(struct thread *td)
5102 {
5103 struct umtx_q *uq;
5104
5105 uq = td->td_umtxq;
5106 uq->uq_inherited_pri = PRI_MAX;
5107
5108 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
5109 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
5110 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
5111 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
5112 }
5113
5114 /*
5115 * exec() hook.
5116 *
5117 * Clear robust lists for all process' threads, not delaying the
5118 * cleanup to thread exit, since the relevant address space is
5119 * destroyed right now.
5120 */
5121 void
umtx_exec(struct proc * p)5122 umtx_exec(struct proc *p)
5123 {
5124 struct thread *td;
5125
5126 KASSERT(p == curproc, ("need curproc"));
5127 KASSERT((p->p_flag & P_HADTHREADS) == 0 ||
5128 (p->p_flag & P_STOPPED_SINGLE) != 0,
5129 ("curproc must be single-threaded"));
5130 /*
5131 * There is no need to lock the list as only this thread can be
5132 * running.
5133 */
5134 FOREACH_THREAD_IN_PROC(p, td) {
5135 KASSERT(td == curthread ||
5136 ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)),
5137 ("running thread %p %p", p, td));
5138 umtx_thread_cleanup(td);
5139 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0;
5140 }
5141
5142 p->p_umtx_min_timeout = 0;
5143 }
5144
5145 /*
5146 * thread exit hook.
5147 */
5148 void
umtx_thread_exit(struct thread * td)5149 umtx_thread_exit(struct thread *td)
5150 {
5151
5152 umtx_thread_cleanup(td);
5153 }
5154
5155 static int
umtx_read_uptr(struct thread * td,uintptr_t ptr,uintptr_t * res,bool compat32)5156 umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res, bool compat32)
5157 {
5158 u_long res1;
5159 uint32_t res32;
5160 int error;
5161
5162 if (compat32) {
5163 error = fueword32((void *)ptr, &res32);
5164 if (error == 0)
5165 res1 = res32;
5166 } else {
5167 error = fueword((void *)ptr, &res1);
5168 }
5169 if (error == 0)
5170 *res = res1;
5171 else
5172 error = EFAULT;
5173 return (error);
5174 }
5175
5176 static void
umtx_read_rb_list(struct thread * td,struct umutex * m,uintptr_t * rb_list,bool compat32)5177 umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list,
5178 bool compat32)
5179 {
5180 struct umutex32 m32;
5181
5182 if (compat32) {
5183 memcpy(&m32, m, sizeof(m32));
5184 *rb_list = m32.m_rb_lnk;
5185 } else {
5186 *rb_list = m->m_rb_lnk;
5187 }
5188 }
5189
5190 static int
umtx_handle_rb(struct thread * td,uintptr_t rbp,uintptr_t * rb_list,bool inact,bool compat32)5191 umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact,
5192 bool compat32)
5193 {
5194 struct umutex m;
5195 int error;
5196
5197 KASSERT(td->td_proc == curproc, ("need current vmspace"));
5198 error = copyin((void *)rbp, &m, sizeof(m));
5199 if (error != 0)
5200 return (error);
5201 if (rb_list != NULL)
5202 umtx_read_rb_list(td, &m, rb_list, compat32);
5203 if ((m.m_flags & UMUTEX_ROBUST) == 0)
5204 return (EINVAL);
5205 if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid)
5206 /* inact is cleared after unlock, allow the inconsistency */
5207 return (inact ? 0 : EINVAL);
5208 return (do_unlock_umutex(td, (struct umutex *)rbp, true));
5209 }
5210
5211 static void
umtx_cleanup_rb_list(struct thread * td,uintptr_t rb_list,uintptr_t * rb_inact,const char * name,bool compat32)5212 umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact,
5213 const char *name, bool compat32)
5214 {
5215 int error, i;
5216 uintptr_t rbp;
5217 bool inact;
5218
5219 if (rb_list == 0)
5220 return;
5221 error = umtx_read_uptr(td, rb_list, &rbp, compat32);
5222 for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) {
5223 if (rbp == *rb_inact) {
5224 inact = true;
5225 *rb_inact = 0;
5226 } else
5227 inact = false;
5228 error = umtx_handle_rb(td, rbp, &rbp, inact, compat32);
5229 }
5230 if (i == umtx_max_rb && umtx_verbose_rb) {
5231 uprintf("comm %s pid %d: reached umtx %smax rb %d\n",
5232 td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb);
5233 }
5234 if (error != 0 && umtx_verbose_rb) {
5235 uprintf("comm %s pid %d: handling %srb error %d\n",
5236 td->td_proc->p_comm, td->td_proc->p_pid, name, error);
5237 }
5238 }
5239
5240 /*
5241 * Clean up umtx data.
5242 */
5243 static void
umtx_thread_cleanup(struct thread * td)5244 umtx_thread_cleanup(struct thread *td)
5245 {
5246 struct umtx_q *uq;
5247 struct umtx_pi *pi;
5248 uintptr_t rb_inact;
5249 bool compat32;
5250
5251 /*
5252 * Disown pi mutexes.
5253 */
5254 uq = td->td_umtxq;
5255 if (uq != NULL) {
5256 if (uq->uq_inherited_pri != PRI_MAX ||
5257 !TAILQ_EMPTY(&uq->uq_pi_contested)) {
5258 mtx_lock(&umtx_lock);
5259 uq->uq_inherited_pri = PRI_MAX;
5260 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
5261 pi->pi_owner = NULL;
5262 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
5263 }
5264 mtx_unlock(&umtx_lock);
5265 }
5266 sched_lend_user_prio_cond(td, PRI_MAX);
5267 }
5268
5269 compat32 = (td->td_pflags2 & TDP2_COMPAT32RB) != 0;
5270 td->td_pflags2 &= ~TDP2_COMPAT32RB;
5271
5272 if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0)
5273 return;
5274
5275 /*
5276 * Handle terminated robust mutexes. Must be done after
5277 * robust pi disown, otherwise unlock could see unowned
5278 * entries.
5279 */
5280 rb_inact = td->td_rb_inact;
5281 if (rb_inact != 0)
5282 (void)umtx_read_uptr(td, rb_inact, &rb_inact, compat32);
5283 umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, "", compat32);
5284 umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv ", compat32);
5285 if (rb_inact != 0)
5286 (void)umtx_handle_rb(td, rb_inact, NULL, true, compat32);
5287 }
5288