1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015, 2016 The FreeBSD Foundation
5 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
6 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
7 * All rights reserved.
8 *
9 * Portions of this software were developed by Konstantin Belousov
10 * under sponsorship from the FreeBSD Foundation.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice unmodified, this list of conditions, and the following
17 * disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <sys/cdefs.h>
35 #include "opt_umtx_profiling.h"
36
37 #include <sys/param.h>
38 #include <sys/kernel.h>
39 #include <sys/fcntl.h>
40 #include <sys/file.h>
41 #include <sys/filedesc.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mman.h>
46 #include <sys/mutex.h>
47 #include <sys/priv.h>
48 #include <sys/proc.h>
49 #include <sys/resource.h>
50 #include <sys/resourcevar.h>
51 #include <sys/rwlock.h>
52 #include <sys/sbuf.h>
53 #include <sys/sched.h>
54 #include <sys/smp.h>
55 #include <sys/sysctl.h>
56 #include <sys/systm.h>
57 #include <sys/sysproto.h>
58 #include <sys/syscallsubr.h>
59 #include <sys/taskqueue.h>
60 #include <sys/time.h>
61 #include <sys/eventhandler.h>
62 #include <sys/umtx.h>
63 #include <sys/umtxvar.h>
64
65 #include <security/mac/mac_framework.h>
66
67 #include <vm/vm.h>
68 #include <vm/vm_param.h>
69 #include <vm/pmap.h>
70 #include <vm/uma.h>
71 #include <vm/vm_map.h>
72 #include <vm/vm_object.h>
73
74 #include <machine/atomic.h>
75 #include <machine/cpu.h>
76
77 #include <compat/freebsd32/freebsd32.h>
78 #ifdef COMPAT_FREEBSD32
79 #include <compat/freebsd32/freebsd32_proto.h>
80 #endif
81
82 #define _UMUTEX_TRY 1
83 #define _UMUTEX_WAIT 2
84
85 #ifdef UMTX_PROFILING
86 #define UPROF_PERC_BIGGER(w, f, sw, sf) \
87 (((w) > (sw)) || ((w) == (sw) && (f) > (sf)))
88 #endif
89
90 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
91 #ifdef INVARIANTS
92 #define UMTXQ_ASSERT_LOCKED_BUSY(key) do { \
93 struct umtxq_chain *uc; \
94 \
95 uc = umtxq_getchain(key); \
96 mtx_assert(&uc->uc_lock, MA_OWNED); \
97 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); \
98 } while (0)
99 #else
100 #define UMTXQ_ASSERT_LOCKED_BUSY(key) do {} while (0)
101 #endif
102
103 /*
104 * Don't propagate time-sharing priority, there is a security reason,
105 * a user can simply introduce PI-mutex, let thread A lock the mutex,
106 * and let another thread B block on the mutex, because B is
107 * sleeping, its priority will be boosted, this causes A's priority to
108 * be boosted via priority propagating too and will never be lowered even
109 * if it is using 100%CPU, this is unfair to other processes.
110 */
111
112 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
113 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
114 PRI_MAX_TIMESHARE : (td)->td_user_pri)
115
116 #define GOLDEN_RATIO_32 1640531527U
117 #ifndef UMTX_CHAINS
118 #define UMTX_CHAINS 512
119 #endif
120 #define UMTX_SHIFTS (__WORD_BIT - 9)
121
122 #define GET_SHARE(flags) \
123 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
124
125 #define BUSY_SPINS 200
126
127 struct umtx_copyops {
128 int (*copyin_timeout)(const void *uaddr, struct timespec *tsp);
129 int (*copyin_umtx_time)(const void *uaddr, size_t size,
130 struct _umtx_time *tp);
131 int (*copyin_robust_lists)(const void *uaddr, size_t size,
132 struct umtx_robust_lists_params *rbp);
133 int (*copyout_timeout)(void *uaddr, size_t size,
134 struct timespec *tsp);
135 const size_t timespec_sz;
136 const size_t umtx_time_sz;
137 const bool compat32;
138 };
139
140 _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32");
141 _Static_assert(__offsetof(struct umutex, m_spare[0]) ==
142 __offsetof(struct umutex32, m_spare[0]), "m_spare32");
143
144 int umtx_shm_vnobj_persistent = 0;
145 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN,
146 &umtx_shm_vnobj_persistent, 0,
147 "False forces destruction of umtx attached to file, on last close");
148 static int umtx_max_rb = 1000;
149 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN,
150 &umtx_max_rb, 0,
151 "Maximum number of robust mutexes allowed for each thread");
152
153 static uma_zone_t umtx_pi_zone;
154 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS];
155 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
156 static int umtx_pi_allocated;
157
158 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
159 "umtx debug");
160 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
161 &umtx_pi_allocated, 0, "Allocated umtx_pi");
162 static int umtx_verbose_rb = 1;
163 SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN,
164 &umtx_verbose_rb, 0,
165 "");
166
167 #ifdef UMTX_PROFILING
168 static long max_length;
169 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length");
170 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
171 "umtx chain stats");
172 #endif
173
174 static inline void umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
175 const struct _umtx_time *umtxtime);
176
177 static void umtx_shm_init(void);
178 static void umtxq_sysinit(void *);
179 static void umtxq_hash(struct umtx_key *key);
180 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags,
181 bool rb);
182 static void umtx_thread_cleanup(struct thread *td);
183 SYSINIT(umtx, SI_SUB_EVENTHANDLER, SI_ORDER_LAST, umtxq_sysinit, NULL);
184
185 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
186
187 static struct mtx umtx_lock;
188
189 #ifdef UMTX_PROFILING
190 static void
umtx_init_profiling(void)191 umtx_init_profiling(void)
192 {
193 struct sysctl_oid *chain_oid;
194 char chain_name[10];
195 int i;
196
197 for (i = 0; i < UMTX_CHAINS; ++i) {
198 snprintf(chain_name, sizeof(chain_name), "%d", i);
199 chain_oid = SYSCTL_ADD_NODE(NULL,
200 SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO,
201 chain_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
202 "umtx hash stats");
203 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
204 "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL);
205 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
206 "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL);
207 }
208 }
209
210 static int
sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)211 sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)
212 {
213 char buf[512];
214 struct sbuf sb;
215 struct umtxq_chain *uc;
216 u_int fract, i, j, tot, whole;
217 u_int sf0, sf1, sf2, sf3, sf4;
218 u_int si0, si1, si2, si3, si4;
219 u_int sw0, sw1, sw2, sw3, sw4;
220
221 sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN);
222 for (i = 0; i < 2; i++) {
223 tot = 0;
224 for (j = 0; j < UMTX_CHAINS; ++j) {
225 uc = &umtxq_chains[i][j];
226 mtx_lock(&uc->uc_lock);
227 tot += uc->max_length;
228 mtx_unlock(&uc->uc_lock);
229 }
230 if (tot == 0)
231 sbuf_printf(&sb, "%u) Empty ", i);
232 else {
233 sf0 = sf1 = sf2 = sf3 = sf4 = 0;
234 si0 = si1 = si2 = si3 = si4 = 0;
235 sw0 = sw1 = sw2 = sw3 = sw4 = 0;
236 for (j = 0; j < UMTX_CHAINS; j++) {
237 uc = &umtxq_chains[i][j];
238 mtx_lock(&uc->uc_lock);
239 whole = uc->max_length * 100;
240 mtx_unlock(&uc->uc_lock);
241 fract = (whole % tot) * 100;
242 if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) {
243 sf0 = fract;
244 si0 = j;
245 sw0 = whole;
246 } else if (UPROF_PERC_BIGGER(whole, fract, sw1,
247 sf1)) {
248 sf1 = fract;
249 si1 = j;
250 sw1 = whole;
251 } else if (UPROF_PERC_BIGGER(whole, fract, sw2,
252 sf2)) {
253 sf2 = fract;
254 si2 = j;
255 sw2 = whole;
256 } else if (UPROF_PERC_BIGGER(whole, fract, sw3,
257 sf3)) {
258 sf3 = fract;
259 si3 = j;
260 sw3 = whole;
261 } else if (UPROF_PERC_BIGGER(whole, fract, sw4,
262 sf4)) {
263 sf4 = fract;
264 si4 = j;
265 sw4 = whole;
266 }
267 }
268 sbuf_printf(&sb, "queue %u:\n", i);
269 sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot,
270 sf0 / tot, si0);
271 sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot,
272 sf1 / tot, si1);
273 sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot,
274 sf2 / tot, si2);
275 sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot,
276 sf3 / tot, si3);
277 sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot,
278 sf4 / tot, si4);
279 }
280 }
281 sbuf_trim(&sb);
282 sbuf_finish(&sb);
283 sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
284 sbuf_delete(&sb);
285 return (0);
286 }
287
288 static int
sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)289 sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)
290 {
291 struct umtxq_chain *uc;
292 u_int i, j;
293 int clear, error;
294
295 clear = 0;
296 error = sysctl_handle_int(oidp, &clear, 0, req);
297 if (error != 0 || req->newptr == NULL)
298 return (error);
299
300 if (clear != 0) {
301 for (i = 0; i < 2; ++i) {
302 for (j = 0; j < UMTX_CHAINS; ++j) {
303 uc = &umtxq_chains[i][j];
304 mtx_lock(&uc->uc_lock);
305 uc->length = 0;
306 uc->max_length = 0;
307 mtx_unlock(&uc->uc_lock);
308 }
309 }
310 }
311 return (0);
312 }
313
314 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear,
315 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
316 sysctl_debug_umtx_chains_clear, "I",
317 "Clear umtx chains statistics");
318 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks,
319 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0,
320 sysctl_debug_umtx_chains_peaks, "A",
321 "Highest peaks in chains max length");
322 #endif
323
324 static void
umtxq_sysinit(void * arg __unused)325 umtxq_sysinit(void *arg __unused)
326 {
327 int i, j;
328
329 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
330 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
331 for (i = 0; i < 2; ++i) {
332 for (j = 0; j < UMTX_CHAINS; ++j) {
333 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
334 MTX_DEF | MTX_DUPOK);
335 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
336 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
337 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
338 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
339 umtxq_chains[i][j].uc_busy = 0;
340 umtxq_chains[i][j].uc_waiters = 0;
341 #ifdef UMTX_PROFILING
342 umtxq_chains[i][j].length = 0;
343 umtxq_chains[i][j].max_length = 0;
344 #endif
345 }
346 }
347 #ifdef UMTX_PROFILING
348 umtx_init_profiling();
349 #endif
350 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_DEF);
351 umtx_shm_init();
352 }
353
354 struct umtx_q *
umtxq_alloc(void)355 umtxq_alloc(void)
356 {
357 struct umtx_q *uq;
358
359 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
360 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
361 M_WAITOK | M_ZERO);
362 TAILQ_INIT(&uq->uq_spare_queue->head);
363 TAILQ_INIT(&uq->uq_pi_contested);
364 uq->uq_inherited_pri = PRI_MAX;
365 return (uq);
366 }
367
368 void
umtxq_free(struct umtx_q * uq)369 umtxq_free(struct umtx_q *uq)
370 {
371
372 MPASS(uq->uq_spare_queue != NULL);
373 free(uq->uq_spare_queue, M_UMTX);
374 free(uq, M_UMTX);
375 }
376
377 static inline void
umtxq_hash(struct umtx_key * key)378 umtxq_hash(struct umtx_key *key)
379 {
380 unsigned n;
381
382 n = (uintptr_t)key->info.both.a + key->info.both.b;
383 key->hash = ((n * GOLDEN_RATIO_32) >> UMTX_SHIFTS) % UMTX_CHAINS;
384 }
385
386 struct umtxq_chain *
umtxq_getchain(struct umtx_key * key)387 umtxq_getchain(struct umtx_key *key)
388 {
389
390 if (key->type <= TYPE_SEM)
391 return (&umtxq_chains[1][key->hash]);
392 return (&umtxq_chains[0][key->hash]);
393 }
394
395 /*
396 * Set chain to busy state when following operation
397 * may be blocked (kernel mutex can not be used).
398 */
399 void
umtxq_busy(struct umtx_key * key)400 umtxq_busy(struct umtx_key *key)
401 {
402 struct umtxq_chain *uc;
403
404 uc = umtxq_getchain(key);
405 mtx_assert(&uc->uc_lock, MA_OWNED);
406 if (uc->uc_busy) {
407 #ifdef SMP
408 if (smp_cpus > 1) {
409 int count = BUSY_SPINS;
410 if (count > 0) {
411 umtxq_unlock(key);
412 while (uc->uc_busy && --count > 0)
413 cpu_spinwait();
414 umtxq_lock(key);
415 }
416 }
417 #endif
418 while (uc->uc_busy) {
419 uc->uc_waiters++;
420 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
421 uc->uc_waiters--;
422 }
423 }
424 uc->uc_busy = 1;
425 }
426
427 /*
428 * Unbusy a chain.
429 */
430 void
umtxq_unbusy(struct umtx_key * key)431 umtxq_unbusy(struct umtx_key *key)
432 {
433 struct umtxq_chain *uc;
434
435 uc = umtxq_getchain(key);
436 mtx_assert(&uc->uc_lock, MA_OWNED);
437 KASSERT(uc->uc_busy != 0, ("not busy"));
438 uc->uc_busy = 0;
439 if (uc->uc_waiters)
440 wakeup_one(uc);
441 }
442
443 void
umtxq_busy_unlocked(struct umtx_key * key)444 umtxq_busy_unlocked(struct umtx_key *key)
445 {
446 umtxq_lock(key);
447 umtxq_busy(key);
448 umtxq_unlock(key);
449 }
450
451 void
umtxq_unbusy_unlocked(struct umtx_key * key)452 umtxq_unbusy_unlocked(struct umtx_key *key)
453 {
454 umtxq_lock(key);
455 umtxq_unbusy(key);
456 umtxq_unlock(key);
457 }
458
459 static struct umtxq_queue *
umtxq_queue_lookup(struct umtx_key * key,int q)460 umtxq_queue_lookup(struct umtx_key *key, int q)
461 {
462 struct umtxq_queue *uh;
463 struct umtxq_chain *uc;
464
465 uc = umtxq_getchain(key);
466 UMTXQ_LOCKED_ASSERT(uc);
467 LIST_FOREACH(uh, &uc->uc_queue[q], link) {
468 if (umtx_key_match(&uh->key, key))
469 return (uh);
470 }
471
472 return (NULL);
473 }
474
475 void
umtxq_insert_queue(struct umtx_q * uq,int q)476 umtxq_insert_queue(struct umtx_q *uq, int q)
477 {
478 struct umtxq_queue *uh;
479 struct umtxq_chain *uc;
480
481 uc = umtxq_getchain(&uq->uq_key);
482 UMTXQ_LOCKED_ASSERT(uc);
483 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
484 uh = umtxq_queue_lookup(&uq->uq_key, q);
485 if (uh != NULL) {
486 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
487 } else {
488 uh = uq->uq_spare_queue;
489 uh->key = uq->uq_key;
490 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
491 #ifdef UMTX_PROFILING
492 uc->length++;
493 if (uc->length > uc->max_length) {
494 uc->max_length = uc->length;
495 if (uc->max_length > max_length)
496 max_length = uc->max_length;
497 }
498 #endif
499 }
500 uq->uq_spare_queue = NULL;
501
502 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
503 uh->length++;
504 uq->uq_flags |= UQF_UMTXQ;
505 uq->uq_cur_queue = uh;
506 return;
507 }
508
509 void
umtxq_remove_queue(struct umtx_q * uq,int q)510 umtxq_remove_queue(struct umtx_q *uq, int q)
511 {
512 struct umtxq_chain *uc;
513 struct umtxq_queue *uh;
514
515 uc = umtxq_getchain(&uq->uq_key);
516 UMTXQ_LOCKED_ASSERT(uc);
517 if (uq->uq_flags & UQF_UMTXQ) {
518 uh = uq->uq_cur_queue;
519 TAILQ_REMOVE(&uh->head, uq, uq_link);
520 uh->length--;
521 uq->uq_flags &= ~UQF_UMTXQ;
522 if (TAILQ_EMPTY(&uh->head)) {
523 KASSERT(uh->length == 0,
524 ("inconsistent umtxq_queue length"));
525 #ifdef UMTX_PROFILING
526 uc->length--;
527 #endif
528 LIST_REMOVE(uh, link);
529 } else {
530 uh = LIST_FIRST(&uc->uc_spare_queue);
531 KASSERT(uh != NULL, ("uc_spare_queue is empty"));
532 LIST_REMOVE(uh, link);
533 }
534 uq->uq_spare_queue = uh;
535 uq->uq_cur_queue = NULL;
536 }
537 }
538
539 /*
540 * Check if there are multiple waiters
541 */
542 int
umtxq_count(struct umtx_key * key)543 umtxq_count(struct umtx_key *key)
544 {
545 struct umtxq_queue *uh;
546
547 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
548 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
549 if (uh != NULL)
550 return (uh->length);
551 return (0);
552 }
553
554 /*
555 * Check if there are multiple PI waiters and returns first
556 * waiter.
557 */
558 static int
umtxq_count_pi(struct umtx_key * key,struct umtx_q ** first)559 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
560 {
561 struct umtxq_queue *uh;
562
563 *first = NULL;
564 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
565 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
566 if (uh != NULL) {
567 *first = TAILQ_FIRST(&uh->head);
568 return (uh->length);
569 }
570 return (0);
571 }
572
573 /*
574 * Wake up threads waiting on an userland object by a bit mask.
575 */
576 int
umtxq_signal_mask(struct umtx_key * key,int n_wake,u_int bitset)577 umtxq_signal_mask(struct umtx_key *key, int n_wake, u_int bitset)
578 {
579 struct umtxq_queue *uh;
580 struct umtx_q *uq, *uq_temp;
581 int ret;
582
583 ret = 0;
584 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
585 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
586 if (uh == NULL)
587 return (0);
588 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
589 if ((uq->uq_bitset & bitset) == 0)
590 continue;
591 umtxq_remove_queue(uq, UMTX_SHARED_QUEUE);
592 wakeup_one(uq);
593 if (++ret >= n_wake)
594 break;
595 }
596 return (ret);
597 }
598
599 /*
600 * Wake up threads waiting on an userland object.
601 */
602
603 static int
umtxq_signal_queue(struct umtx_key * key,int n_wake,int q)604 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
605 {
606 struct umtxq_queue *uh;
607 struct umtx_q *uq;
608 int ret;
609
610 ret = 0;
611 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
612 uh = umtxq_queue_lookup(key, q);
613 if (uh != NULL) {
614 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
615 umtxq_remove_queue(uq, q);
616 wakeup(uq);
617 if (++ret >= n_wake)
618 return (ret);
619 }
620 }
621 return (ret);
622 }
623
624 /*
625 * Wake up specified thread.
626 */
627 static inline void
umtxq_signal_thread(struct umtx_q * uq)628 umtxq_signal_thread(struct umtx_q *uq)
629 {
630
631 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
632 umtxq_remove(uq);
633 wakeup(uq);
634 }
635
636 /*
637 * Wake up a maximum of n_wake threads that are waiting on an userland
638 * object identified by key. The remaining threads are removed from queue
639 * identified by key and added to the queue identified by key2 (requeued).
640 * The n_requeue specifies an upper limit on the number of threads that
641 * are requeued to the second queue.
642 */
643 int
umtxq_requeue(struct umtx_key * key,int n_wake,struct umtx_key * key2,int n_requeue)644 umtxq_requeue(struct umtx_key *key, int n_wake, struct umtx_key *key2,
645 int n_requeue)
646 {
647 struct umtxq_queue *uh;
648 struct umtx_q *uq, *uq_temp;
649 int ret;
650
651 ret = 0;
652 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
653 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key2));
654 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
655 if (uh == NULL)
656 return (0);
657 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
658 if (++ret <= n_wake) {
659 umtxq_remove(uq);
660 wakeup_one(uq);
661 } else {
662 umtxq_remove(uq);
663 uq->uq_key = *key2;
664 umtxq_insert(uq);
665 if (ret - n_wake == n_requeue)
666 break;
667 }
668 }
669 return (ret);
670 }
671
672 static inline int
tstohz(const struct timespec * tsp)673 tstohz(const struct timespec *tsp)
674 {
675 struct timeval tv;
676
677 TIMESPEC_TO_TIMEVAL(&tv, tsp);
678 return tvtohz(&tv);
679 }
680
681 void
umtx_abs_timeout_init(struct umtx_abs_timeout * timo,int clockid,int absolute,const struct timespec * timeout)682 umtx_abs_timeout_init(struct umtx_abs_timeout *timo, int clockid,
683 int absolute, const struct timespec *timeout)
684 {
685
686 timo->clockid = clockid;
687 if (!absolute) {
688 timo->is_abs_real = false;
689 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
690 timespecadd(&timo->cur, timeout, &timo->end);
691 } else {
692 timo->end = *timeout;
693 timo->is_abs_real = clockid == CLOCK_REALTIME ||
694 clockid == CLOCK_REALTIME_FAST ||
695 clockid == CLOCK_REALTIME_PRECISE ||
696 clockid == CLOCK_TAI ||
697 clockid == CLOCK_SECOND;
698 }
699 }
700
701 static void
umtx_abs_timeout_init2(struct umtx_abs_timeout * timo,const struct _umtx_time * umtxtime)702 umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
703 const struct _umtx_time *umtxtime)
704 {
705
706 umtx_abs_timeout_init(timo, umtxtime->_clockid,
707 (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout);
708 }
709
710 static void
umtx_abs_timeout_enforce_min(sbintime_t * sbt)711 umtx_abs_timeout_enforce_min(sbintime_t *sbt)
712 {
713 sbintime_t when, mint;
714
715 mint = curproc->p_umtx_min_timeout;
716 if (__predict_false(mint != 0)) {
717 when = sbinuptime() + mint;
718 if (*sbt < when)
719 *sbt = when;
720 }
721 }
722
723 static int
umtx_abs_timeout_getsbt(struct umtx_abs_timeout * timo,sbintime_t * sbt,int * flags)724 umtx_abs_timeout_getsbt(struct umtx_abs_timeout *timo, sbintime_t *sbt,
725 int *flags)
726 {
727 struct bintime bt, bbt;
728 struct timespec tts;
729 sbintime_t rem;
730
731 switch (timo->clockid) {
732
733 /* Clocks that can be converted into absolute time. */
734 case CLOCK_REALTIME:
735 case CLOCK_REALTIME_PRECISE:
736 case CLOCK_REALTIME_FAST:
737 case CLOCK_MONOTONIC:
738 case CLOCK_MONOTONIC_PRECISE:
739 case CLOCK_MONOTONIC_FAST:
740 case CLOCK_UPTIME:
741 case CLOCK_UPTIME_PRECISE:
742 case CLOCK_UPTIME_FAST:
743 case CLOCK_SECOND:
744 timespec2bintime(&timo->end, &bt);
745 switch (timo->clockid) {
746 case CLOCK_REALTIME:
747 case CLOCK_REALTIME_PRECISE:
748 case CLOCK_REALTIME_FAST:
749 case CLOCK_SECOND:
750 getboottimebin(&bbt);
751 bintime_sub(&bt, &bbt);
752 break;
753 }
754 if (bt.sec < 0)
755 return (ETIMEDOUT);
756 if (bt.sec >= (SBT_MAX >> 32)) {
757 *sbt = 0;
758 *flags = 0;
759 return (0);
760 }
761 *sbt = bttosbt(bt);
762 umtx_abs_timeout_enforce_min(sbt);
763
764 /*
765 * Check if the absolute time should be aligned to
766 * avoid firing multiple timer events in non-periodic
767 * timer mode.
768 */
769 switch (timo->clockid) {
770 case CLOCK_REALTIME_FAST:
771 case CLOCK_MONOTONIC_FAST:
772 case CLOCK_UPTIME_FAST:
773 rem = *sbt % tc_tick_sbt;
774 if (__predict_true(rem != 0))
775 *sbt += tc_tick_sbt - rem;
776 break;
777 case CLOCK_SECOND:
778 rem = *sbt % SBT_1S;
779 if (__predict_true(rem != 0))
780 *sbt += SBT_1S - rem;
781 break;
782 }
783 *flags = C_ABSOLUTE;
784 return (0);
785
786 /* Clocks that has to be periodically polled. */
787 case CLOCK_VIRTUAL:
788 case CLOCK_PROF:
789 case CLOCK_THREAD_CPUTIME_ID:
790 case CLOCK_PROCESS_CPUTIME_ID:
791 case CLOCK_TAI: /* Boot time is not necessarily stable in TAI */
792 default:
793 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
794 if (timespeccmp(&timo->end, &timo->cur, <=))
795 return (ETIMEDOUT);
796 timespecsub(&timo->end, &timo->cur, &tts);
797 *sbt = tick_sbt * tstohz(&tts);
798 *flags = C_HARDCLOCK;
799 return (0);
800 }
801 }
802
803 static uint32_t
umtx_unlock_val(uint32_t flags,bool rb)804 umtx_unlock_val(uint32_t flags, bool rb)
805 {
806
807 if (rb)
808 return (UMUTEX_RB_OWNERDEAD);
809 else if ((flags & UMUTEX_NONCONSISTENT) != 0)
810 return (UMUTEX_RB_NOTRECOV);
811 else
812 return (UMUTEX_UNOWNED);
813
814 }
815
816 /*
817 * Put thread into sleep state, before sleeping, check if
818 * thread was removed from umtx queue.
819 */
820 int
umtxq_sleep(struct umtx_q * uq,const char * wmesg,struct umtx_abs_timeout * timo)821 umtxq_sleep(struct umtx_q *uq, const char *wmesg,
822 struct umtx_abs_timeout *timo)
823 {
824 struct umtxq_chain *uc;
825 sbintime_t sbt = 0;
826 int error, flags = 0;
827
828 uc = umtxq_getchain(&uq->uq_key);
829 UMTXQ_LOCKED_ASSERT(uc);
830 for (;;) {
831 if (!(uq->uq_flags & UQF_UMTXQ)) {
832 error = 0;
833 break;
834 }
835 if (timo != NULL) {
836 if (timo->is_abs_real)
837 curthread->td_rtcgen =
838 atomic_load_acq_int(&rtc_generation);
839 error = umtx_abs_timeout_getsbt(timo, &sbt, &flags);
840 if (error != 0)
841 break;
842 }
843 error = msleep_sbt(uq, &uc->uc_lock, PCATCH | PDROP, wmesg,
844 sbt, 0, flags);
845 uc = umtxq_getchain(&uq->uq_key);
846 mtx_lock(&uc->uc_lock);
847 if (error == EINTR || error == ERESTART)
848 break;
849 if (error == EWOULDBLOCK && (flags & C_ABSOLUTE) != 0) {
850 error = ETIMEDOUT;
851 break;
852 }
853 }
854
855 curthread->td_rtcgen = 0;
856 return (error);
857 }
858
859 /*
860 * Convert userspace address into unique logical address.
861 */
862 int
umtx_key_get(const void * addr,int type,int share,struct umtx_key * key)863 umtx_key_get(const void *addr, int type, int share, struct umtx_key *key)
864 {
865 struct thread *td = curthread;
866 vm_map_t map;
867 vm_map_entry_t entry;
868 vm_pindex_t pindex;
869 vm_prot_t prot;
870 boolean_t wired;
871
872 key->type = type;
873 if (share == THREAD_SHARE) {
874 key->shared = 0;
875 key->info.private.vs = td->td_proc->p_vmspace;
876 key->info.private.addr = (uintptr_t)addr;
877 } else {
878 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
879 map = &td->td_proc->p_vmspace->vm_map;
880 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
881 &entry, &key->info.shared.object, &pindex, &prot,
882 &wired) != KERN_SUCCESS) {
883 return (EFAULT);
884 }
885
886 if ((share == PROCESS_SHARE) ||
887 (share == AUTO_SHARE &&
888 VM_INHERIT_SHARE == entry->inheritance)) {
889 key->shared = 1;
890 key->info.shared.offset = (vm_offset_t)addr -
891 entry->start + entry->offset;
892 vm_object_reference(key->info.shared.object);
893 } else {
894 key->shared = 0;
895 key->info.private.vs = td->td_proc->p_vmspace;
896 key->info.private.addr = (uintptr_t)addr;
897 }
898 vm_map_lookup_done(map, entry);
899 }
900
901 umtxq_hash(key);
902 return (0);
903 }
904
905 /*
906 * Release key.
907 */
908 void
umtx_key_release(struct umtx_key * key)909 umtx_key_release(struct umtx_key *key)
910 {
911 if (key->shared)
912 vm_object_deallocate(key->info.shared.object);
913 }
914
915 #ifdef COMPAT_FREEBSD10
916 /*
917 * Lock a umtx object.
918 */
919 static int
do_lock_umtx(struct thread * td,struct umtx * umtx,u_long id,const struct timespec * timeout)920 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
921 const struct timespec *timeout)
922 {
923 struct umtx_abs_timeout timo;
924 struct umtx_q *uq;
925 u_long owner;
926 u_long old;
927 int error = 0;
928
929 uq = td->td_umtxq;
930 if (timeout != NULL)
931 umtx_abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout);
932
933 /*
934 * Care must be exercised when dealing with umtx structure. It
935 * can fault on any access.
936 */
937 for (;;) {
938 /*
939 * Try the uncontested case. This should be done in userland.
940 */
941 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
942
943 /* The acquire succeeded. */
944 if (owner == UMTX_UNOWNED)
945 return (0);
946
947 /* The address was invalid. */
948 if (owner == -1)
949 return (EFAULT);
950
951 /* If no one owns it but it is contested try to acquire it. */
952 if (owner == UMTX_CONTESTED) {
953 owner = casuword(&umtx->u_owner,
954 UMTX_CONTESTED, id | UMTX_CONTESTED);
955
956 if (owner == UMTX_CONTESTED)
957 return (0);
958
959 /* The address was invalid. */
960 if (owner == -1)
961 return (EFAULT);
962
963 error = thread_check_susp(td, false);
964 if (error != 0)
965 break;
966
967 /* If this failed the lock has changed, restart. */
968 continue;
969 }
970
971 /*
972 * If we caught a signal, we have retried and now
973 * exit immediately.
974 */
975 if (error != 0)
976 break;
977
978 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
979 AUTO_SHARE, &uq->uq_key)) != 0)
980 return (error);
981
982 umtxq_lock(&uq->uq_key);
983 umtxq_busy(&uq->uq_key);
984 umtxq_insert(uq);
985 umtxq_unbusy(&uq->uq_key);
986 umtxq_unlock(&uq->uq_key);
987
988 /*
989 * Set the contested bit so that a release in user space
990 * knows to use the system call for unlock. If this fails
991 * either some one else has acquired the lock or it has been
992 * released.
993 */
994 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
995
996 /* The address was invalid. */
997 if (old == -1) {
998 umtxq_lock(&uq->uq_key);
999 umtxq_remove(uq);
1000 umtxq_unlock(&uq->uq_key);
1001 umtx_key_release(&uq->uq_key);
1002 return (EFAULT);
1003 }
1004
1005 /*
1006 * We set the contested bit, sleep. Otherwise the lock changed
1007 * and we need to retry or we lost a race to the thread
1008 * unlocking the umtx.
1009 */
1010 umtxq_lock(&uq->uq_key);
1011 if (old == owner)
1012 error = umtxq_sleep(uq, "umtx", timeout == NULL ? NULL :
1013 &timo);
1014 umtxq_remove(uq);
1015 umtxq_unlock(&uq->uq_key);
1016 umtx_key_release(&uq->uq_key);
1017
1018 if (error == 0)
1019 error = thread_check_susp(td, false);
1020 }
1021
1022 if (timeout == NULL) {
1023 /* Mutex locking is restarted if it is interrupted. */
1024 if (error == EINTR)
1025 error = ERESTART;
1026 } else {
1027 /* Timed-locking is not restarted. */
1028 if (error == ERESTART)
1029 error = EINTR;
1030 }
1031 return (error);
1032 }
1033
1034 /*
1035 * Unlock a umtx object.
1036 */
1037 static int
do_unlock_umtx(struct thread * td,struct umtx * umtx,u_long id)1038 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
1039 {
1040 struct umtx_key key;
1041 u_long owner;
1042 u_long old;
1043 int error;
1044 int count;
1045
1046 /*
1047 * Make sure we own this mtx.
1048 */
1049 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
1050 if (owner == -1)
1051 return (EFAULT);
1052
1053 if ((owner & ~UMTX_CONTESTED) != id)
1054 return (EPERM);
1055
1056 /* This should be done in userland */
1057 if ((owner & UMTX_CONTESTED) == 0) {
1058 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
1059 if (old == -1)
1060 return (EFAULT);
1061 if (old == owner)
1062 return (0);
1063 owner = old;
1064 }
1065
1066 /* We should only ever be in here for contested locks */
1067 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
1068 &key)) != 0)
1069 return (error);
1070
1071 umtxq_lock(&key);
1072 umtxq_busy(&key);
1073 count = umtxq_count(&key);
1074 umtxq_unlock(&key);
1075
1076 /*
1077 * When unlocking the umtx, it must be marked as unowned if
1078 * there is zero or one thread only waiting for it.
1079 * Otherwise, it must be marked as contested.
1080 */
1081 old = casuword(&umtx->u_owner, owner,
1082 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
1083 umtxq_lock(&key);
1084 umtxq_signal(&key,1);
1085 umtxq_unbusy(&key);
1086 umtxq_unlock(&key);
1087 umtx_key_release(&key);
1088 if (old == -1)
1089 return (EFAULT);
1090 if (old != owner)
1091 return (EINVAL);
1092 return (0);
1093 }
1094
1095 #ifdef COMPAT_FREEBSD32
1096
1097 /*
1098 * Lock a umtx object.
1099 */
1100 static int
do_lock_umtx32(struct thread * td,uint32_t * m,uint32_t id,const struct timespec * timeout)1101 do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id,
1102 const struct timespec *timeout)
1103 {
1104 struct umtx_abs_timeout timo;
1105 struct umtx_q *uq;
1106 uint32_t owner;
1107 uint32_t old;
1108 int error = 0;
1109
1110 uq = td->td_umtxq;
1111
1112 if (timeout != NULL)
1113 umtx_abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout);
1114
1115 /*
1116 * Care must be exercised when dealing with umtx structure. It
1117 * can fault on any access.
1118 */
1119 for (;;) {
1120 /*
1121 * Try the uncontested case. This should be done in userland.
1122 */
1123 owner = casuword32(m, UMUTEX_UNOWNED, id);
1124
1125 /* The acquire succeeded. */
1126 if (owner == UMUTEX_UNOWNED)
1127 return (0);
1128
1129 /* The address was invalid. */
1130 if (owner == -1)
1131 return (EFAULT);
1132
1133 /* If no one owns it but it is contested try to acquire it. */
1134 if (owner == UMUTEX_CONTESTED) {
1135 owner = casuword32(m,
1136 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1137 if (owner == UMUTEX_CONTESTED)
1138 return (0);
1139
1140 /* The address was invalid. */
1141 if (owner == -1)
1142 return (EFAULT);
1143
1144 error = thread_check_susp(td, false);
1145 if (error != 0)
1146 break;
1147
1148 /* If this failed the lock has changed, restart. */
1149 continue;
1150 }
1151
1152 /*
1153 * If we caught a signal, we have retried and now
1154 * exit immediately.
1155 */
1156 if (error != 0)
1157 return (error);
1158
1159 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
1160 AUTO_SHARE, &uq->uq_key)) != 0)
1161 return (error);
1162
1163 umtxq_lock(&uq->uq_key);
1164 umtxq_busy(&uq->uq_key);
1165 umtxq_insert(uq);
1166 umtxq_unbusy(&uq->uq_key);
1167 umtxq_unlock(&uq->uq_key);
1168
1169 /*
1170 * Set the contested bit so that a release in user space
1171 * knows to use the system call for unlock. If this fails
1172 * either some one else has acquired the lock or it has been
1173 * released.
1174 */
1175 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
1176
1177 /* The address was invalid. */
1178 if (old == -1) {
1179 umtxq_lock(&uq->uq_key);
1180 umtxq_remove(uq);
1181 umtxq_unlock(&uq->uq_key);
1182 umtx_key_release(&uq->uq_key);
1183 return (EFAULT);
1184 }
1185
1186 /*
1187 * We set the contested bit, sleep. Otherwise the lock changed
1188 * and we need to retry or we lost a race to the thread
1189 * unlocking the umtx.
1190 */
1191 umtxq_lock(&uq->uq_key);
1192 if (old == owner)
1193 error = umtxq_sleep(uq, "umtx", timeout == NULL ?
1194 NULL : &timo);
1195 umtxq_remove(uq);
1196 umtxq_unlock(&uq->uq_key);
1197 umtx_key_release(&uq->uq_key);
1198
1199 if (error == 0)
1200 error = thread_check_susp(td, false);
1201 }
1202
1203 if (timeout == NULL) {
1204 /* Mutex locking is restarted if it is interrupted. */
1205 if (error == EINTR)
1206 error = ERESTART;
1207 } else {
1208 /* Timed-locking is not restarted. */
1209 if (error == ERESTART)
1210 error = EINTR;
1211 }
1212 return (error);
1213 }
1214
1215 /*
1216 * Unlock a umtx object.
1217 */
1218 static int
do_unlock_umtx32(struct thread * td,uint32_t * m,uint32_t id)1219 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
1220 {
1221 struct umtx_key key;
1222 uint32_t owner;
1223 uint32_t old;
1224 int error;
1225 int count;
1226
1227 /*
1228 * Make sure we own this mtx.
1229 */
1230 owner = fuword32(m);
1231 if (owner == -1)
1232 return (EFAULT);
1233
1234 if ((owner & ~UMUTEX_CONTESTED) != id)
1235 return (EPERM);
1236
1237 /* This should be done in userland */
1238 if ((owner & UMUTEX_CONTESTED) == 0) {
1239 old = casuword32(m, owner, UMUTEX_UNOWNED);
1240 if (old == -1)
1241 return (EFAULT);
1242 if (old == owner)
1243 return (0);
1244 owner = old;
1245 }
1246
1247 /* We should only ever be in here for contested locks */
1248 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
1249 &key)) != 0)
1250 return (error);
1251
1252 umtxq_lock(&key);
1253 umtxq_busy(&key);
1254 count = umtxq_count(&key);
1255 umtxq_unlock(&key);
1256
1257 /*
1258 * When unlocking the umtx, it must be marked as unowned if
1259 * there is zero or one thread only waiting for it.
1260 * Otherwise, it must be marked as contested.
1261 */
1262 old = casuword32(m, owner,
1263 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1264 umtxq_lock(&key);
1265 umtxq_signal(&key,1);
1266 umtxq_unbusy(&key);
1267 umtxq_unlock(&key);
1268 umtx_key_release(&key);
1269 if (old == -1)
1270 return (EFAULT);
1271 if (old != owner)
1272 return (EINVAL);
1273 return (0);
1274 }
1275 #endif /* COMPAT_FREEBSD32 */
1276 #endif /* COMPAT_FREEBSD10 */
1277
1278 /*
1279 * Fetch and compare value, sleep on the address if value is not changed.
1280 */
1281 static int
do_wait(struct thread * td,void * addr,u_long id,struct _umtx_time * timeout,int compat32,int is_private)1282 do_wait(struct thread *td, void *addr, u_long id,
1283 struct _umtx_time *timeout, int compat32, int is_private)
1284 {
1285 struct umtx_abs_timeout timo;
1286 struct umtx_q *uq;
1287 u_long tmp;
1288 uint32_t tmp32;
1289 int error = 0;
1290
1291 uq = td->td_umtxq;
1292 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
1293 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1294 return (error);
1295
1296 if (timeout != NULL)
1297 umtx_abs_timeout_init2(&timo, timeout);
1298
1299 umtxq_lock(&uq->uq_key);
1300 umtxq_insert(uq);
1301 umtxq_unlock(&uq->uq_key);
1302 if (compat32 == 0) {
1303 error = fueword(addr, &tmp);
1304 if (error != 0)
1305 error = EFAULT;
1306 } else {
1307 error = fueword32(addr, &tmp32);
1308 if (error == 0)
1309 tmp = tmp32;
1310 else
1311 error = EFAULT;
1312 }
1313 umtxq_lock(&uq->uq_key);
1314 if (error == 0) {
1315 if (tmp == id)
1316 error = umtxq_sleep(uq, "uwait", timeout == NULL ?
1317 NULL : &timo);
1318 if ((uq->uq_flags & UQF_UMTXQ) == 0)
1319 error = 0;
1320 else
1321 umtxq_remove(uq);
1322 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
1323 umtxq_remove(uq);
1324 }
1325 umtxq_unlock(&uq->uq_key);
1326 umtx_key_release(&uq->uq_key);
1327 if (error == ERESTART)
1328 error = EINTR;
1329 return (error);
1330 }
1331
1332 /*
1333 * Wake up threads sleeping on the specified address.
1334 */
1335 int
kern_umtx_wake(struct thread * td,void * uaddr,int n_wake,int is_private)1336 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
1337 {
1338 struct umtx_key key;
1339 int ret;
1340
1341 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
1342 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
1343 return (ret);
1344 umtxq_lock(&key);
1345 umtxq_signal(&key, n_wake);
1346 umtxq_unlock(&key);
1347 umtx_key_release(&key);
1348 return (0);
1349 }
1350
1351 /*
1352 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1353 */
1354 static int
do_lock_normal(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int mode)1355 do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
1356 struct _umtx_time *timeout, int mode)
1357 {
1358 struct umtx_abs_timeout timo;
1359 struct umtx_q *uq;
1360 uint32_t owner, old, id;
1361 int error, rv;
1362
1363 id = td->td_tid;
1364 uq = td->td_umtxq;
1365 error = 0;
1366 if (timeout != NULL)
1367 umtx_abs_timeout_init2(&timo, timeout);
1368
1369 /*
1370 * Care must be exercised when dealing with umtx structure. It
1371 * can fault on any access.
1372 */
1373 for (;;) {
1374 rv = fueword32(&m->m_owner, &owner);
1375 if (rv == -1)
1376 return (EFAULT);
1377 if (mode == _UMUTEX_WAIT) {
1378 if (owner == UMUTEX_UNOWNED ||
1379 owner == UMUTEX_CONTESTED ||
1380 owner == UMUTEX_RB_OWNERDEAD ||
1381 owner == UMUTEX_RB_NOTRECOV)
1382 return (0);
1383 } else {
1384 /*
1385 * Robust mutex terminated. Kernel duty is to
1386 * return EOWNERDEAD to the userspace. The
1387 * umutex.m_flags UMUTEX_NONCONSISTENT is set
1388 * by the common userspace code.
1389 */
1390 if (owner == UMUTEX_RB_OWNERDEAD) {
1391 rv = casueword32(&m->m_owner,
1392 UMUTEX_RB_OWNERDEAD, &owner,
1393 id | UMUTEX_CONTESTED);
1394 if (rv == -1)
1395 return (EFAULT);
1396 if (rv == 0) {
1397 MPASS(owner == UMUTEX_RB_OWNERDEAD);
1398 return (EOWNERDEAD); /* success */
1399 }
1400 MPASS(rv == 1);
1401 rv = thread_check_susp(td, false);
1402 if (rv != 0)
1403 return (rv);
1404 continue;
1405 }
1406 if (owner == UMUTEX_RB_NOTRECOV)
1407 return (ENOTRECOVERABLE);
1408
1409 /*
1410 * Try the uncontested case. This should be
1411 * done in userland.
1412 */
1413 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
1414 &owner, id);
1415 /* The address was invalid. */
1416 if (rv == -1)
1417 return (EFAULT);
1418
1419 /* The acquire succeeded. */
1420 if (rv == 0) {
1421 MPASS(owner == UMUTEX_UNOWNED);
1422 return (0);
1423 }
1424
1425 /*
1426 * If no one owns it but it is contested try
1427 * to acquire it.
1428 */
1429 MPASS(rv == 1);
1430 if (owner == UMUTEX_CONTESTED) {
1431 rv = casueword32(&m->m_owner,
1432 UMUTEX_CONTESTED, &owner,
1433 id | UMUTEX_CONTESTED);
1434 /* The address was invalid. */
1435 if (rv == -1)
1436 return (EFAULT);
1437 if (rv == 0) {
1438 MPASS(owner == UMUTEX_CONTESTED);
1439 return (0);
1440 }
1441 if (rv == 1) {
1442 rv = thread_check_susp(td, false);
1443 if (rv != 0)
1444 return (rv);
1445 }
1446
1447 /*
1448 * If this failed the lock has
1449 * changed, restart.
1450 */
1451 continue;
1452 }
1453
1454 /* rv == 1 but not contested, likely store failure */
1455 rv = thread_check_susp(td, false);
1456 if (rv != 0)
1457 return (rv);
1458 }
1459
1460 if (mode == _UMUTEX_TRY)
1461 return (EBUSY);
1462
1463 /*
1464 * If we caught a signal, we have retried and now
1465 * exit immediately.
1466 */
1467 if (error != 0)
1468 return (error);
1469
1470 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1471 GET_SHARE(flags), &uq->uq_key)) != 0)
1472 return (error);
1473
1474 umtxq_lock(&uq->uq_key);
1475 umtxq_busy(&uq->uq_key);
1476 umtxq_insert(uq);
1477 umtxq_unlock(&uq->uq_key);
1478
1479 /*
1480 * Set the contested bit so that a release in user space
1481 * knows to use the system call for unlock. If this fails
1482 * either some one else has acquired the lock or it has been
1483 * released.
1484 */
1485 rv = casueword32(&m->m_owner, owner, &old,
1486 owner | UMUTEX_CONTESTED);
1487
1488 /* The address was invalid or casueword failed to store. */
1489 if (rv == -1 || rv == 1) {
1490 umtxq_lock(&uq->uq_key);
1491 umtxq_remove(uq);
1492 umtxq_unbusy(&uq->uq_key);
1493 umtxq_unlock(&uq->uq_key);
1494 umtx_key_release(&uq->uq_key);
1495 if (rv == -1)
1496 return (EFAULT);
1497 if (rv == 1) {
1498 rv = thread_check_susp(td, false);
1499 if (rv != 0)
1500 return (rv);
1501 }
1502 continue;
1503 }
1504
1505 /*
1506 * We set the contested bit, sleep. Otherwise the lock changed
1507 * and we need to retry or we lost a race to the thread
1508 * unlocking the umtx.
1509 */
1510 umtxq_lock(&uq->uq_key);
1511 umtxq_unbusy(&uq->uq_key);
1512 MPASS(old == owner);
1513 error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
1514 NULL : &timo);
1515 umtxq_remove(uq);
1516 umtxq_unlock(&uq->uq_key);
1517 umtx_key_release(&uq->uq_key);
1518
1519 if (error == 0)
1520 error = thread_check_susp(td, false);
1521 }
1522
1523 return (0);
1524 }
1525
1526 /*
1527 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1528 */
1529 static int
do_unlock_normal(struct thread * td,struct umutex * m,uint32_t flags,bool rb)1530 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
1531 {
1532 struct umtx_key key;
1533 uint32_t owner, old, id, newlock;
1534 int error, count;
1535
1536 id = td->td_tid;
1537
1538 again:
1539 /*
1540 * Make sure we own this mtx.
1541 */
1542 error = fueword32(&m->m_owner, &owner);
1543 if (error == -1)
1544 return (EFAULT);
1545
1546 if ((owner & ~UMUTEX_CONTESTED) != id)
1547 return (EPERM);
1548
1549 newlock = umtx_unlock_val(flags, rb);
1550 if ((owner & UMUTEX_CONTESTED) == 0) {
1551 error = casueword32(&m->m_owner, owner, &old, newlock);
1552 if (error == -1)
1553 return (EFAULT);
1554 if (error == 1) {
1555 error = thread_check_susp(td, false);
1556 if (error != 0)
1557 return (error);
1558 goto again;
1559 }
1560 MPASS(old == owner);
1561 return (0);
1562 }
1563
1564 /* We should only ever be in here for contested locks */
1565 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1566 &key)) != 0)
1567 return (error);
1568
1569 umtxq_lock(&key);
1570 umtxq_busy(&key);
1571 count = umtxq_count(&key);
1572 umtxq_unlock(&key);
1573
1574 /*
1575 * When unlocking the umtx, it must be marked as unowned if
1576 * there is zero or one thread only waiting for it.
1577 * Otherwise, it must be marked as contested.
1578 */
1579 if (count > 1)
1580 newlock |= UMUTEX_CONTESTED;
1581 error = casueword32(&m->m_owner, owner, &old, newlock);
1582 umtxq_lock(&key);
1583 umtxq_signal(&key, 1);
1584 umtxq_unbusy(&key);
1585 umtxq_unlock(&key);
1586 umtx_key_release(&key);
1587 if (error == -1)
1588 return (EFAULT);
1589 if (error == 1) {
1590 if (old != owner)
1591 return (EINVAL);
1592 error = thread_check_susp(td, false);
1593 if (error != 0)
1594 return (error);
1595 goto again;
1596 }
1597 return (0);
1598 }
1599
1600 /*
1601 * Check if the mutex is available and wake up a waiter,
1602 * only for simple mutex.
1603 */
1604 static int
do_wake_umutex(struct thread * td,struct umutex * m)1605 do_wake_umutex(struct thread *td, struct umutex *m)
1606 {
1607 struct umtx_key key;
1608 uint32_t owner;
1609 uint32_t flags;
1610 int error;
1611 int count;
1612
1613 again:
1614 error = fueword32(&m->m_owner, &owner);
1615 if (error == -1)
1616 return (EFAULT);
1617
1618 if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD &&
1619 owner != UMUTEX_RB_NOTRECOV)
1620 return (0);
1621
1622 error = fueword32(&m->m_flags, &flags);
1623 if (error == -1)
1624 return (EFAULT);
1625
1626 /* We should only ever be in here for contested locks */
1627 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1628 &key)) != 0)
1629 return (error);
1630
1631 umtxq_lock(&key);
1632 umtxq_busy(&key);
1633 count = umtxq_count(&key);
1634 umtxq_unlock(&key);
1635
1636 if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD &&
1637 owner != UMUTEX_RB_NOTRECOV) {
1638 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
1639 UMUTEX_UNOWNED);
1640 if (error == -1) {
1641 error = EFAULT;
1642 } else if (error == 1) {
1643 umtxq_lock(&key);
1644 umtxq_unbusy(&key);
1645 umtxq_unlock(&key);
1646 umtx_key_release(&key);
1647 error = thread_check_susp(td, false);
1648 if (error != 0)
1649 return (error);
1650 goto again;
1651 }
1652 }
1653
1654 umtxq_lock(&key);
1655 if (error == 0 && count != 0) {
1656 MPASS((owner & ~UMUTEX_CONTESTED) == 0 ||
1657 owner == UMUTEX_RB_OWNERDEAD ||
1658 owner == UMUTEX_RB_NOTRECOV);
1659 umtxq_signal(&key, 1);
1660 }
1661 umtxq_unbusy(&key);
1662 umtxq_unlock(&key);
1663 umtx_key_release(&key);
1664 return (error);
1665 }
1666
1667 /*
1668 * Check if the mutex has waiters and tries to fix contention bit.
1669 */
1670 static int
do_wake2_umutex(struct thread * td,struct umutex * m,uint32_t flags)1671 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags)
1672 {
1673 struct umtx_key key;
1674 uint32_t owner, old;
1675 int type;
1676 int error;
1677 int count;
1678
1679 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT |
1680 UMUTEX_ROBUST)) {
1681 case 0:
1682 case UMUTEX_ROBUST:
1683 type = TYPE_NORMAL_UMUTEX;
1684 break;
1685 case UMUTEX_PRIO_INHERIT:
1686 type = TYPE_PI_UMUTEX;
1687 break;
1688 case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST):
1689 type = TYPE_PI_ROBUST_UMUTEX;
1690 break;
1691 case UMUTEX_PRIO_PROTECT:
1692 type = TYPE_PP_UMUTEX;
1693 break;
1694 case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST):
1695 type = TYPE_PP_ROBUST_UMUTEX;
1696 break;
1697 default:
1698 return (EINVAL);
1699 }
1700 if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0)
1701 return (error);
1702
1703 owner = 0;
1704 umtxq_lock(&key);
1705 umtxq_busy(&key);
1706 count = umtxq_count(&key);
1707 umtxq_unlock(&key);
1708
1709 error = fueword32(&m->m_owner, &owner);
1710 if (error == -1)
1711 error = EFAULT;
1712
1713 /*
1714 * Only repair contention bit if there is a waiter, this means
1715 * the mutex is still being referenced by userland code,
1716 * otherwise don't update any memory.
1717 */
1718 while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 &&
1719 (count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) {
1720 error = casueword32(&m->m_owner, owner, &old,
1721 owner | UMUTEX_CONTESTED);
1722 if (error == -1) {
1723 error = EFAULT;
1724 break;
1725 }
1726 if (error == 0) {
1727 MPASS(old == owner);
1728 break;
1729 }
1730 owner = old;
1731 error = thread_check_susp(td, false);
1732 }
1733
1734 umtxq_lock(&key);
1735 if (error == EFAULT) {
1736 umtxq_signal(&key, INT_MAX);
1737 } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 ||
1738 owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV))
1739 umtxq_signal(&key, 1);
1740 umtxq_unbusy(&key);
1741 umtxq_unlock(&key);
1742 umtx_key_release(&key);
1743 return (error);
1744 }
1745
1746 struct umtx_pi *
umtx_pi_alloc(int flags)1747 umtx_pi_alloc(int flags)
1748 {
1749 struct umtx_pi *pi;
1750
1751 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1752 if (pi == NULL)
1753 return (NULL);
1754
1755 TAILQ_INIT(&pi->pi_blocked);
1756 atomic_add_int(&umtx_pi_allocated, 1);
1757 return (pi);
1758 }
1759
1760 void
umtx_pi_free(struct umtx_pi * pi)1761 umtx_pi_free(struct umtx_pi *pi)
1762 {
1763 uma_zfree(umtx_pi_zone, pi);
1764 atomic_add_int(&umtx_pi_allocated, -1);
1765 }
1766
1767 /*
1768 * Adjust the thread's position on a pi_state after its priority has been
1769 * changed.
1770 */
1771 static int
umtx_pi_adjust_thread(struct umtx_pi * pi,struct thread * td)1772 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1773 {
1774 struct umtx_q *uq, *uq1, *uq2;
1775 struct thread *td1;
1776
1777 mtx_assert(&umtx_lock, MA_OWNED);
1778 if (pi == NULL)
1779 return (0);
1780
1781 uq = td->td_umtxq;
1782
1783 /*
1784 * Check if the thread needs to be moved on the blocked chain.
1785 * It needs to be moved if either its priority is lower than
1786 * the previous thread or higher than the next thread.
1787 */
1788 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1789 uq2 = TAILQ_NEXT(uq, uq_lockq);
1790 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1791 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1792 /*
1793 * Remove thread from blocked chain and determine where
1794 * it should be moved to.
1795 */
1796 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1797 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1798 td1 = uq1->uq_thread;
1799 MPASS(td1->td_proc->p_magic == P_MAGIC);
1800 if (UPRI(td1) > UPRI(td))
1801 break;
1802 }
1803
1804 if (uq1 == NULL)
1805 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1806 else
1807 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1808 }
1809 return (1);
1810 }
1811
1812 static struct umtx_pi *
umtx_pi_next(struct umtx_pi * pi)1813 umtx_pi_next(struct umtx_pi *pi)
1814 {
1815 struct umtx_q *uq_owner;
1816
1817 if (pi->pi_owner == NULL)
1818 return (NULL);
1819 uq_owner = pi->pi_owner->td_umtxq;
1820 if (uq_owner == NULL)
1821 return (NULL);
1822 return (uq_owner->uq_pi_blocked);
1823 }
1824
1825 /*
1826 * Floyd's Cycle-Finding Algorithm.
1827 */
1828 static bool
umtx_pi_check_loop(struct umtx_pi * pi)1829 umtx_pi_check_loop(struct umtx_pi *pi)
1830 {
1831 struct umtx_pi *pi1; /* fast iterator */
1832
1833 mtx_assert(&umtx_lock, MA_OWNED);
1834 if (pi == NULL)
1835 return (false);
1836 pi1 = pi;
1837 for (;;) {
1838 pi = umtx_pi_next(pi);
1839 if (pi == NULL)
1840 break;
1841 pi1 = umtx_pi_next(pi1);
1842 if (pi1 == NULL)
1843 break;
1844 pi1 = umtx_pi_next(pi1);
1845 if (pi1 == NULL)
1846 break;
1847 if (pi == pi1)
1848 return (true);
1849 }
1850 return (false);
1851 }
1852
1853 /*
1854 * Propagate priority when a thread is blocked on POSIX
1855 * PI mutex.
1856 */
1857 static void
umtx_propagate_priority(struct thread * td)1858 umtx_propagate_priority(struct thread *td)
1859 {
1860 struct umtx_q *uq;
1861 struct umtx_pi *pi;
1862 int pri;
1863
1864 mtx_assert(&umtx_lock, MA_OWNED);
1865 pri = UPRI(td);
1866 uq = td->td_umtxq;
1867 pi = uq->uq_pi_blocked;
1868 if (pi == NULL)
1869 return;
1870 if (umtx_pi_check_loop(pi))
1871 return;
1872
1873 for (;;) {
1874 td = pi->pi_owner;
1875 if (td == NULL || td == curthread)
1876 return;
1877
1878 MPASS(td->td_proc != NULL);
1879 MPASS(td->td_proc->p_magic == P_MAGIC);
1880
1881 thread_lock(td);
1882 if (td->td_lend_user_pri > pri)
1883 sched_lend_user_prio(td, pri);
1884 else {
1885 thread_unlock(td);
1886 break;
1887 }
1888 thread_unlock(td);
1889
1890 /*
1891 * Pick up the lock that td is blocked on.
1892 */
1893 uq = td->td_umtxq;
1894 pi = uq->uq_pi_blocked;
1895 if (pi == NULL)
1896 break;
1897 /* Resort td on the list if needed. */
1898 umtx_pi_adjust_thread(pi, td);
1899 }
1900 }
1901
1902 /*
1903 * Unpropagate priority for a PI mutex when a thread blocked on
1904 * it is interrupted by signal or resumed by others.
1905 */
1906 static void
umtx_repropagate_priority(struct umtx_pi * pi)1907 umtx_repropagate_priority(struct umtx_pi *pi)
1908 {
1909 struct umtx_q *uq, *uq_owner;
1910 struct umtx_pi *pi2;
1911 int pri;
1912
1913 mtx_assert(&umtx_lock, MA_OWNED);
1914
1915 if (umtx_pi_check_loop(pi))
1916 return;
1917 while (pi != NULL && pi->pi_owner != NULL) {
1918 pri = PRI_MAX;
1919 uq_owner = pi->pi_owner->td_umtxq;
1920
1921 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1922 uq = TAILQ_FIRST(&pi2->pi_blocked);
1923 if (uq != NULL) {
1924 if (pri > UPRI(uq->uq_thread))
1925 pri = UPRI(uq->uq_thread);
1926 }
1927 }
1928
1929 if (pri > uq_owner->uq_inherited_pri)
1930 pri = uq_owner->uq_inherited_pri;
1931 thread_lock(pi->pi_owner);
1932 sched_lend_user_prio(pi->pi_owner, pri);
1933 thread_unlock(pi->pi_owner);
1934 if ((pi = uq_owner->uq_pi_blocked) != NULL)
1935 umtx_pi_adjust_thread(pi, uq_owner->uq_thread);
1936 }
1937 }
1938
1939 /*
1940 * Insert a PI mutex into owned list.
1941 */
1942 static void
umtx_pi_setowner(struct umtx_pi * pi,struct thread * owner)1943 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1944 {
1945 struct umtx_q *uq_owner;
1946
1947 uq_owner = owner->td_umtxq;
1948 mtx_assert(&umtx_lock, MA_OWNED);
1949 MPASS(pi->pi_owner == NULL);
1950 pi->pi_owner = owner;
1951 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1952 }
1953
1954 /*
1955 * Disown a PI mutex, and remove it from the owned list.
1956 */
1957 static void
umtx_pi_disown(struct umtx_pi * pi)1958 umtx_pi_disown(struct umtx_pi *pi)
1959 {
1960
1961 mtx_assert(&umtx_lock, MA_OWNED);
1962 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link);
1963 pi->pi_owner = NULL;
1964 }
1965
1966 /*
1967 * Claim ownership of a PI mutex.
1968 */
1969 int
umtx_pi_claim(struct umtx_pi * pi,struct thread * owner)1970 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1971 {
1972 struct umtx_q *uq;
1973 int pri;
1974
1975 mtx_lock(&umtx_lock);
1976 if (pi->pi_owner == owner) {
1977 mtx_unlock(&umtx_lock);
1978 return (0);
1979 }
1980
1981 if (pi->pi_owner != NULL) {
1982 /*
1983 * userland may have already messed the mutex, sigh.
1984 */
1985 mtx_unlock(&umtx_lock);
1986 return (EPERM);
1987 }
1988 umtx_pi_setowner(pi, owner);
1989 uq = TAILQ_FIRST(&pi->pi_blocked);
1990 if (uq != NULL) {
1991 pri = UPRI(uq->uq_thread);
1992 thread_lock(owner);
1993 if (pri < UPRI(owner))
1994 sched_lend_user_prio(owner, pri);
1995 thread_unlock(owner);
1996 }
1997 mtx_unlock(&umtx_lock);
1998 return (0);
1999 }
2000
2001 /*
2002 * Adjust a thread's order position in its blocked PI mutex,
2003 * this may result new priority propagating process.
2004 */
2005 void
umtx_pi_adjust(struct thread * td,u_char oldpri)2006 umtx_pi_adjust(struct thread *td, u_char oldpri)
2007 {
2008 struct umtx_q *uq;
2009 struct umtx_pi *pi;
2010
2011 uq = td->td_umtxq;
2012 mtx_lock(&umtx_lock);
2013 /*
2014 * Pick up the lock that td is blocked on.
2015 */
2016 pi = uq->uq_pi_blocked;
2017 if (pi != NULL) {
2018 umtx_pi_adjust_thread(pi, td);
2019 umtx_repropagate_priority(pi);
2020 }
2021 mtx_unlock(&umtx_lock);
2022 }
2023
2024 /*
2025 * Sleep on a PI mutex.
2026 */
2027 int
umtxq_sleep_pi(struct umtx_q * uq,struct umtx_pi * pi,uint32_t owner,const char * wmesg,struct umtx_abs_timeout * timo,bool shared)2028 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner,
2029 const char *wmesg, struct umtx_abs_timeout *timo, bool shared)
2030 {
2031 struct thread *td;
2032 struct umtx_q *uq1;
2033 int error, pri;
2034 #ifdef INVARIANTS
2035 struct umtxq_chain *uc;
2036
2037 uc = umtxq_getchain(&pi->pi_key);
2038 #endif
2039 error = 0;
2040 td = uq->uq_thread;
2041 KASSERT(td == curthread, ("inconsistent uq_thread"));
2042 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
2043 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy"));
2044 umtxq_insert(uq);
2045 mtx_lock(&umtx_lock);
2046 if (pi->pi_owner == NULL) {
2047 struct thread *ownertd;
2048
2049 mtx_unlock(&umtx_lock);
2050 ownertd = tdfind(owner, shared ? -1 : td->td_proc->p_pid);
2051 mtx_lock(&umtx_lock);
2052 if (ownertd != NULL) {
2053 /*
2054 * An exiting thread that has already called
2055 * umtx_thread_exit() must not be made the owner of a
2056 * shared mutex.
2057 */
2058 if ((ownertd->td_proc->p_flag & P_WEXIT) == 0 &&
2059 (ownertd->td_dbgflags & TDB_EXIT) == 0 &&
2060 pi->pi_owner == NULL)
2061 umtx_pi_setowner(pi, ownertd);
2062 PROC_UNLOCK(ownertd->td_proc);
2063 }
2064 }
2065
2066 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
2067 pri = UPRI(uq1->uq_thread);
2068 if (pri > UPRI(td))
2069 break;
2070 }
2071
2072 if (uq1 != NULL)
2073 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
2074 else
2075 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
2076
2077 uq->uq_pi_blocked = pi;
2078 thread_lock(td);
2079 td->td_flags |= TDF_UPIBLOCKED;
2080 thread_unlock(td);
2081 umtx_propagate_priority(td);
2082 mtx_unlock(&umtx_lock);
2083 umtxq_unbusy(&uq->uq_key);
2084
2085 error = umtxq_sleep(uq, wmesg, timo);
2086 umtxq_remove(uq);
2087
2088 mtx_lock(&umtx_lock);
2089 uq->uq_pi_blocked = NULL;
2090 thread_lock(td);
2091 td->td_flags &= ~TDF_UPIBLOCKED;
2092 thread_unlock(td);
2093 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
2094 umtx_repropagate_priority(pi);
2095 mtx_unlock(&umtx_lock);
2096 umtxq_unlock(&uq->uq_key);
2097
2098 return (error);
2099 }
2100
2101 /*
2102 * Add reference count for a PI mutex.
2103 */
2104 void
umtx_pi_ref(struct umtx_pi * pi)2105 umtx_pi_ref(struct umtx_pi *pi)
2106 {
2107
2108 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key));
2109 pi->pi_refcount++;
2110 }
2111
2112 /*
2113 * Decrease reference count for a PI mutex, if the counter
2114 * is decreased to zero, its memory space is freed.
2115 */
2116 void
umtx_pi_unref(struct umtx_pi * pi)2117 umtx_pi_unref(struct umtx_pi *pi)
2118 {
2119 struct umtxq_chain *uc;
2120
2121 uc = umtxq_getchain(&pi->pi_key);
2122 UMTXQ_LOCKED_ASSERT(uc);
2123 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
2124 if (--pi->pi_refcount == 0) {
2125 mtx_lock(&umtx_lock);
2126 if (pi->pi_owner != NULL)
2127 umtx_pi_disown(pi);
2128 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
2129 ("blocked queue not empty"));
2130 mtx_unlock(&umtx_lock);
2131 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
2132 umtx_pi_free(pi);
2133 }
2134 }
2135
2136 /*
2137 * Find a PI mutex in hash table.
2138 */
2139 struct umtx_pi *
umtx_pi_lookup(struct umtx_key * key)2140 umtx_pi_lookup(struct umtx_key *key)
2141 {
2142 struct umtxq_chain *uc;
2143 struct umtx_pi *pi;
2144
2145 uc = umtxq_getchain(key);
2146 UMTXQ_LOCKED_ASSERT(uc);
2147
2148 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
2149 if (umtx_key_match(&pi->pi_key, key)) {
2150 return (pi);
2151 }
2152 }
2153 return (NULL);
2154 }
2155
2156 /*
2157 * Insert a PI mutex into hash table.
2158 */
2159 void
umtx_pi_insert(struct umtx_pi * pi)2160 umtx_pi_insert(struct umtx_pi *pi)
2161 {
2162 struct umtxq_chain *uc;
2163
2164 uc = umtxq_getchain(&pi->pi_key);
2165 UMTXQ_LOCKED_ASSERT(uc);
2166 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
2167 }
2168
2169 /*
2170 * Drop a PI mutex and wakeup a top waiter.
2171 */
2172 int
umtx_pi_drop(struct thread * td,struct umtx_key * key,bool rb,int * count)2173 umtx_pi_drop(struct thread *td, struct umtx_key *key, bool rb, int *count)
2174 {
2175 struct umtx_q *uq_first, *uq_first2, *uq_me;
2176 struct umtx_pi *pi, *pi2;
2177 int pri;
2178
2179 UMTXQ_ASSERT_LOCKED_BUSY(key);
2180 *count = umtxq_count_pi(key, &uq_first);
2181 if (uq_first != NULL) {
2182 mtx_lock(&umtx_lock);
2183 pi = uq_first->uq_pi_blocked;
2184 KASSERT(pi != NULL, ("pi == NULL?"));
2185 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
2186 mtx_unlock(&umtx_lock);
2187 /* userland messed the mutex */
2188 return (EPERM);
2189 }
2190 uq_me = td->td_umtxq;
2191 if (pi->pi_owner == td)
2192 umtx_pi_disown(pi);
2193 /* get highest priority thread which is still sleeping. */
2194 uq_first = TAILQ_FIRST(&pi->pi_blocked);
2195 while (uq_first != NULL &&
2196 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
2197 uq_first = TAILQ_NEXT(uq_first, uq_lockq);
2198 }
2199 pri = PRI_MAX;
2200 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
2201 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
2202 if (uq_first2 != NULL) {
2203 if (pri > UPRI(uq_first2->uq_thread))
2204 pri = UPRI(uq_first2->uq_thread);
2205 }
2206 }
2207 thread_lock(td);
2208 sched_lend_user_prio(td, pri);
2209 thread_unlock(td);
2210 mtx_unlock(&umtx_lock);
2211 if (uq_first)
2212 umtxq_signal_thread(uq_first);
2213 } else {
2214 pi = umtx_pi_lookup(key);
2215 /*
2216 * A umtx_pi can exist if a signal or timeout removed the
2217 * last waiter from the umtxq, but there is still
2218 * a thread in do_lock_pi() holding the umtx_pi.
2219 */
2220 if (pi != NULL) {
2221 /*
2222 * The umtx_pi can be unowned, such as when a thread
2223 * has just entered do_lock_pi(), allocated the
2224 * umtx_pi, and unlocked the umtxq.
2225 * If the current thread owns it, it must disown it.
2226 */
2227 mtx_lock(&umtx_lock);
2228 if (pi->pi_owner == td)
2229 umtx_pi_disown(pi);
2230 mtx_unlock(&umtx_lock);
2231 }
2232 }
2233 return (0);
2234 }
2235
2236 /*
2237 * Lock a PI mutex.
2238 */
2239 static int
do_lock_pi(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int try)2240 do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
2241 struct _umtx_time *timeout, int try)
2242 {
2243 struct umtx_abs_timeout timo;
2244 struct umtx_q *uq;
2245 struct umtx_pi *pi, *new_pi;
2246 uint32_t id, old_owner, owner, old;
2247 int error, rv;
2248
2249 id = td->td_tid;
2250 uq = td->td_umtxq;
2251
2252 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2253 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
2254 &uq->uq_key)) != 0)
2255 return (error);
2256
2257 if (timeout != NULL)
2258 umtx_abs_timeout_init2(&timo, timeout);
2259
2260 umtxq_lock(&uq->uq_key);
2261 pi = umtx_pi_lookup(&uq->uq_key);
2262 if (pi == NULL) {
2263 new_pi = umtx_pi_alloc(M_NOWAIT);
2264 if (new_pi == NULL) {
2265 umtxq_unlock(&uq->uq_key);
2266 new_pi = umtx_pi_alloc(M_WAITOK);
2267 umtxq_lock(&uq->uq_key);
2268 pi = umtx_pi_lookup(&uq->uq_key);
2269 if (pi != NULL) {
2270 umtx_pi_free(new_pi);
2271 new_pi = NULL;
2272 }
2273 }
2274 if (new_pi != NULL) {
2275 new_pi->pi_key = uq->uq_key;
2276 umtx_pi_insert(new_pi);
2277 pi = new_pi;
2278 }
2279 }
2280 umtx_pi_ref(pi);
2281 umtxq_unlock(&uq->uq_key);
2282
2283 /*
2284 * Care must be exercised when dealing with umtx structure. It
2285 * can fault on any access.
2286 */
2287 for (;;) {
2288 /*
2289 * Try the uncontested case. This should be done in userland.
2290 */
2291 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
2292 /* The address was invalid. */
2293 if (rv == -1) {
2294 error = EFAULT;
2295 break;
2296 }
2297 /* The acquire succeeded. */
2298 if (rv == 0) {
2299 MPASS(owner == UMUTEX_UNOWNED);
2300 error = 0;
2301 break;
2302 }
2303
2304 if (owner == UMUTEX_RB_NOTRECOV) {
2305 error = ENOTRECOVERABLE;
2306 break;
2307 }
2308
2309 /*
2310 * Nobody owns it, but the acquire failed. This can happen
2311 * with ll/sc atomics.
2312 */
2313 if (owner == UMUTEX_UNOWNED) {
2314 error = thread_check_susp(td, true);
2315 if (error != 0)
2316 break;
2317 continue;
2318 }
2319
2320 /*
2321 * Avoid overwriting a possible error from sleep due
2322 * to the pending signal with suspension check result.
2323 */
2324 if (error == 0) {
2325 error = thread_check_susp(td, true);
2326 if (error != 0)
2327 break;
2328 }
2329
2330 /* If no one owns it but it is contested try to acquire it. */
2331 if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) {
2332 old_owner = owner;
2333 rv = casueword32(&m->m_owner, owner, &owner,
2334 id | UMUTEX_CONTESTED);
2335 /* The address was invalid. */
2336 if (rv == -1) {
2337 error = EFAULT;
2338 break;
2339 }
2340 if (rv == 1) {
2341 if (error == 0) {
2342 error = thread_check_susp(td, true);
2343 if (error != 0)
2344 break;
2345 }
2346
2347 /*
2348 * If this failed the lock could
2349 * changed, restart.
2350 */
2351 continue;
2352 }
2353
2354 MPASS(rv == 0);
2355 MPASS(owner == old_owner);
2356 umtxq_lock(&uq->uq_key);
2357 umtxq_busy(&uq->uq_key);
2358 error = umtx_pi_claim(pi, td);
2359 umtxq_unbusy(&uq->uq_key);
2360 umtxq_unlock(&uq->uq_key);
2361 if (error != 0) {
2362 /*
2363 * Since we're going to return an
2364 * error, restore the m_owner to its
2365 * previous, unowned state to avoid
2366 * compounding the problem.
2367 */
2368 (void)casuword32(&m->m_owner,
2369 id | UMUTEX_CONTESTED, old_owner);
2370 }
2371 if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD)
2372 error = EOWNERDEAD;
2373 break;
2374 }
2375
2376 if ((owner & ~UMUTEX_CONTESTED) == id) {
2377 error = EDEADLK;
2378 break;
2379 }
2380
2381 if (try != 0) {
2382 error = EBUSY;
2383 break;
2384 }
2385
2386 /*
2387 * If we caught a signal, we have retried and now
2388 * exit immediately.
2389 */
2390 if (error != 0)
2391 break;
2392
2393 umtxq_busy_unlocked(&uq->uq_key);
2394
2395 /*
2396 * Set the contested bit so that a release in user space
2397 * knows to use the system call for unlock. If this fails
2398 * either some one else has acquired the lock or it has been
2399 * released.
2400 */
2401 rv = casueword32(&m->m_owner, owner, &old, owner |
2402 UMUTEX_CONTESTED);
2403
2404 /* The address was invalid. */
2405 if (rv == -1) {
2406 umtxq_unbusy_unlocked(&uq->uq_key);
2407 error = EFAULT;
2408 break;
2409 }
2410 if (rv == 1) {
2411 umtxq_unbusy_unlocked(&uq->uq_key);
2412 error = thread_check_susp(td, true);
2413 if (error != 0)
2414 break;
2415
2416 /*
2417 * The lock changed and we need to retry or we
2418 * lost a race to the thread unlocking the
2419 * umtx. Note that the UMUTEX_RB_OWNERDEAD
2420 * value for owner is impossible there.
2421 */
2422 continue;
2423 }
2424
2425 umtxq_lock(&uq->uq_key);
2426
2427 /* We set the contested bit, sleep. */
2428 MPASS(old == owner);
2429 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
2430 "umtxpi", timeout == NULL ? NULL : &timo,
2431 (flags & USYNC_PROCESS_SHARED) != 0);
2432 if (error != 0)
2433 continue;
2434
2435 error = thread_check_susp(td, false);
2436 if (error != 0)
2437 break;
2438 }
2439
2440 umtxq_lock(&uq->uq_key);
2441 umtx_pi_unref(pi);
2442 umtxq_unlock(&uq->uq_key);
2443
2444 umtx_key_release(&uq->uq_key);
2445 return (error);
2446 }
2447
2448 /*
2449 * Unlock a PI mutex.
2450 */
2451 static int
do_unlock_pi(struct thread * td,struct umutex * m,uint32_t flags,bool rb)2452 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
2453 {
2454 struct umtx_key key;
2455 uint32_t id, new_owner, old, owner;
2456 int count, error;
2457
2458 id = td->td_tid;
2459
2460 usrloop:
2461 /*
2462 * Make sure we own this mtx.
2463 */
2464 error = fueword32(&m->m_owner, &owner);
2465 if (error == -1)
2466 return (EFAULT);
2467
2468 if ((owner & ~UMUTEX_CONTESTED) != id)
2469 return (EPERM);
2470
2471 new_owner = umtx_unlock_val(flags, rb);
2472
2473 /* This should be done in userland */
2474 if ((owner & UMUTEX_CONTESTED) == 0) {
2475 error = casueword32(&m->m_owner, owner, &old, new_owner);
2476 if (error == -1)
2477 return (EFAULT);
2478 if (error == 1) {
2479 error = thread_check_susp(td, true);
2480 if (error != 0)
2481 return (error);
2482 goto usrloop;
2483 }
2484 if (old == owner)
2485 return (0);
2486 owner = old;
2487 }
2488
2489 /* We should only ever be in here for contested locks */
2490 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2491 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
2492 &key)) != 0)
2493 return (error);
2494
2495 umtxq_lock(&key);
2496 umtxq_busy(&key);
2497 error = umtx_pi_drop(td, &key, rb, &count);
2498 if (error != 0) {
2499 umtxq_unbusy(&key);
2500 umtxq_unlock(&key);
2501 umtx_key_release(&key);
2502 /* userland messed the mutex */
2503 return (error);
2504 }
2505 umtxq_unlock(&key);
2506
2507 /*
2508 * When unlocking the umtx, it must be marked as unowned if
2509 * there is zero or one thread only waiting for it.
2510 * Otherwise, it must be marked as contested.
2511 */
2512
2513 if (count > 1)
2514 new_owner |= UMUTEX_CONTESTED;
2515 again:
2516 error = casueword32(&m->m_owner, owner, &old, new_owner);
2517 if (error == 1) {
2518 error = thread_check_susp(td, false);
2519 if (error == 0)
2520 goto again;
2521 }
2522 umtxq_unbusy_unlocked(&key);
2523 umtx_key_release(&key);
2524 if (error == -1)
2525 return (EFAULT);
2526 if (error == 0 && old != owner)
2527 return (EINVAL);
2528 return (error);
2529 }
2530
2531 /*
2532 * Lock a PP mutex.
2533 */
2534 static int
do_lock_pp(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int try)2535 do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
2536 struct _umtx_time *timeout, int try)
2537 {
2538 struct umtx_abs_timeout timo;
2539 struct umtx_q *uq, *uq2;
2540 struct umtx_pi *pi;
2541 uint32_t ceiling;
2542 uint32_t owner, id;
2543 int error, pri, old_inherited_pri, new_pri, rv;
2544 bool su;
2545
2546 id = td->td_tid;
2547 uq = td->td_umtxq;
2548 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2549 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2550 &uq->uq_key)) != 0)
2551 return (error);
2552
2553 if (timeout != NULL)
2554 umtx_abs_timeout_init2(&timo, timeout);
2555
2556 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2557 for (;;) {
2558 old_inherited_pri = uq->uq_inherited_pri;
2559 umtxq_busy_unlocked(&uq->uq_key);
2560
2561 rv = fueword32(&m->m_ceilings[0], &ceiling);
2562 if (rv == -1) {
2563 error = EFAULT;
2564 goto out;
2565 }
2566 ceiling = RTP_PRIO_MAX - ceiling;
2567 if (ceiling > RTP_PRIO_MAX) {
2568 error = EINVAL;
2569 goto out;
2570 }
2571 new_pri = PRI_MIN_REALTIME + ceiling;
2572
2573 if (td->td_base_user_pri < new_pri) {
2574 error = EINVAL;
2575 goto out;
2576 }
2577 if (su) {
2578 mtx_lock(&umtx_lock);
2579 if (new_pri < uq->uq_inherited_pri) {
2580 uq->uq_inherited_pri = new_pri;
2581 thread_lock(td);
2582 if (new_pri < UPRI(td))
2583 sched_lend_user_prio(td, new_pri);
2584 thread_unlock(td);
2585 }
2586 mtx_unlock(&umtx_lock);
2587 }
2588
2589 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2590 id | UMUTEX_CONTESTED);
2591 /* The address was invalid. */
2592 if (rv == -1) {
2593 error = EFAULT;
2594 break;
2595 }
2596 if (rv == 0) {
2597 MPASS(owner == UMUTEX_CONTESTED);
2598 error = 0;
2599 break;
2600 }
2601 /* rv == 1 */
2602 if (owner == UMUTEX_RB_OWNERDEAD) {
2603 rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
2604 &owner, id | UMUTEX_CONTESTED);
2605 if (rv == -1) {
2606 error = EFAULT;
2607 break;
2608 }
2609 if (rv == 0) {
2610 MPASS(owner == UMUTEX_RB_OWNERDEAD);
2611 error = EOWNERDEAD; /* success */
2612 break;
2613 }
2614
2615 /*
2616 * rv == 1, only check for suspension if we
2617 * did not already catched a signal. If we
2618 * get an error from the check, the same
2619 * condition is checked by the umtxq_sleep()
2620 * call below, so we should obliterate the
2621 * error to not skip the last loop iteration.
2622 */
2623 if (error == 0) {
2624 error = thread_check_susp(td, false);
2625 if (error == 0 && try == 0) {
2626 umtxq_unbusy_unlocked(&uq->uq_key);
2627 continue;
2628 }
2629 error = 0;
2630 }
2631 } else if (owner == UMUTEX_RB_NOTRECOV) {
2632 error = ENOTRECOVERABLE;
2633 } else if (owner == UMUTEX_CONTESTED) {
2634 /* Spurious failure, retry. */
2635 umtxq_unbusy_unlocked(&uq->uq_key);
2636 continue;
2637 }
2638
2639 if (try != 0)
2640 error = EBUSY;
2641
2642 /*
2643 * If we caught a signal, we have retried and now
2644 * exit immediately.
2645 */
2646 if (error != 0)
2647 break;
2648
2649 umtxq_lock(&uq->uq_key);
2650 umtxq_insert(uq);
2651 umtxq_unbusy(&uq->uq_key);
2652 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ?
2653 NULL : &timo);
2654 umtxq_remove(uq);
2655 umtxq_unlock(&uq->uq_key);
2656
2657 mtx_lock(&umtx_lock);
2658 uq->uq_inherited_pri = old_inherited_pri;
2659 pri = PRI_MAX;
2660 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2661 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2662 if (uq2 != NULL) {
2663 if (pri > UPRI(uq2->uq_thread))
2664 pri = UPRI(uq2->uq_thread);
2665 }
2666 }
2667 if (pri > uq->uq_inherited_pri)
2668 pri = uq->uq_inherited_pri;
2669 thread_lock(td);
2670 sched_lend_user_prio(td, pri);
2671 thread_unlock(td);
2672 mtx_unlock(&umtx_lock);
2673 }
2674
2675 if (error != 0 && error != EOWNERDEAD) {
2676 mtx_lock(&umtx_lock);
2677 uq->uq_inherited_pri = old_inherited_pri;
2678 pri = PRI_MAX;
2679 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2680 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2681 if (uq2 != NULL) {
2682 if (pri > UPRI(uq2->uq_thread))
2683 pri = UPRI(uq2->uq_thread);
2684 }
2685 }
2686 if (pri > uq->uq_inherited_pri)
2687 pri = uq->uq_inherited_pri;
2688 thread_lock(td);
2689 sched_lend_user_prio(td, pri);
2690 thread_unlock(td);
2691 mtx_unlock(&umtx_lock);
2692 }
2693
2694 out:
2695 umtxq_unbusy_unlocked(&uq->uq_key);
2696 umtx_key_release(&uq->uq_key);
2697 return (error);
2698 }
2699
2700 /*
2701 * Unlock a PP mutex.
2702 */
2703 static int
do_unlock_pp(struct thread * td,struct umutex * m,uint32_t flags,bool rb)2704 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
2705 {
2706 struct umtx_key key;
2707 struct umtx_q *uq, *uq2;
2708 struct umtx_pi *pi;
2709 uint32_t id, owner, rceiling;
2710 int error, pri, new_inherited_pri;
2711 bool su;
2712
2713 id = td->td_tid;
2714 uq = td->td_umtxq;
2715 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2716
2717 /*
2718 * Make sure we own this mtx.
2719 */
2720 error = fueword32(&m->m_owner, &owner);
2721 if (error == -1)
2722 return (EFAULT);
2723
2724 if ((owner & ~UMUTEX_CONTESTED) != id)
2725 return (EPERM);
2726
2727 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2728 if (error != 0)
2729 return (error);
2730
2731 if (rceiling == -1)
2732 new_inherited_pri = PRI_MAX;
2733 else {
2734 rceiling = RTP_PRIO_MAX - rceiling;
2735 if (rceiling > RTP_PRIO_MAX)
2736 return (EINVAL);
2737 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2738 }
2739
2740 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2741 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2742 &key)) != 0)
2743 return (error);
2744 umtxq_busy_unlocked(&key);
2745
2746 /*
2747 * For priority protected mutex, always set unlocked state
2748 * to UMUTEX_CONTESTED, so that userland always enters kernel
2749 * to lock the mutex, it is necessary because thread priority
2750 * has to be adjusted for such mutex.
2751 */
2752 error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) |
2753 UMUTEX_CONTESTED);
2754
2755 umtxq_lock(&key);
2756 if (error == 0)
2757 umtxq_signal(&key, 1);
2758 umtxq_unbusy(&key);
2759 umtxq_unlock(&key);
2760
2761 if (error == -1)
2762 error = EFAULT;
2763 else {
2764 mtx_lock(&umtx_lock);
2765 if (su || new_inherited_pri == PRI_MAX)
2766 uq->uq_inherited_pri = new_inherited_pri;
2767 pri = PRI_MAX;
2768 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2769 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2770 if (uq2 != NULL) {
2771 if (pri > UPRI(uq2->uq_thread))
2772 pri = UPRI(uq2->uq_thread);
2773 }
2774 }
2775 if (pri > uq->uq_inherited_pri)
2776 pri = uq->uq_inherited_pri;
2777 thread_lock(td);
2778 sched_lend_user_prio(td, pri);
2779 thread_unlock(td);
2780 mtx_unlock(&umtx_lock);
2781 }
2782 umtx_key_release(&key);
2783 return (error);
2784 }
2785
2786 static int
do_set_ceiling(struct thread * td,struct umutex * m,uint32_t ceiling,uint32_t * old_ceiling)2787 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2788 uint32_t *old_ceiling)
2789 {
2790 struct umtx_q *uq;
2791 uint32_t flags, id, owner, save_ceiling;
2792 int error, rv, rv1;
2793
2794 error = fueword32(&m->m_flags, &flags);
2795 if (error == -1)
2796 return (EFAULT);
2797 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2798 return (EINVAL);
2799 if (ceiling > RTP_PRIO_MAX)
2800 return (EINVAL);
2801 id = td->td_tid;
2802 uq = td->td_umtxq;
2803 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2804 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2805 &uq->uq_key)) != 0)
2806 return (error);
2807 for (;;) {
2808 umtxq_busy_unlocked(&uq->uq_key);
2809
2810 rv = fueword32(&m->m_ceilings[0], &save_ceiling);
2811 if (rv == -1) {
2812 error = EFAULT;
2813 break;
2814 }
2815
2816 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2817 id | UMUTEX_CONTESTED);
2818 if (rv == -1) {
2819 error = EFAULT;
2820 break;
2821 }
2822
2823 if (rv == 0) {
2824 MPASS(owner == UMUTEX_CONTESTED);
2825 rv = suword32(&m->m_ceilings[0], ceiling);
2826 rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED);
2827 error = (rv == 0 && rv1 == 0) ? 0: EFAULT;
2828 break;
2829 }
2830
2831 if ((owner & ~UMUTEX_CONTESTED) == id) {
2832 rv = suword32(&m->m_ceilings[0], ceiling);
2833 error = rv == 0 ? 0 : EFAULT;
2834 break;
2835 }
2836
2837 if (owner == UMUTEX_RB_OWNERDEAD) {
2838 error = EOWNERDEAD;
2839 break;
2840 } else if (owner == UMUTEX_RB_NOTRECOV) {
2841 error = ENOTRECOVERABLE;
2842 break;
2843 } else if (owner == UMUTEX_CONTESTED) {
2844 /* Spurious failure, retry. */
2845 umtxq_unbusy_unlocked(&uq->uq_key);
2846 continue;
2847 }
2848
2849 /*
2850 * If we caught a signal, we have retried and now
2851 * exit immediately.
2852 */
2853 if (error != 0)
2854 break;
2855
2856 /*
2857 * We set the contested bit, sleep. Otherwise the lock changed
2858 * and we need to retry or we lost a race to the thread
2859 * unlocking the umtx.
2860 */
2861 umtxq_lock(&uq->uq_key);
2862 umtxq_insert(uq);
2863 umtxq_unbusy(&uq->uq_key);
2864 error = umtxq_sleep(uq, "umtxpp", NULL);
2865 umtxq_remove(uq);
2866 umtxq_unlock(&uq->uq_key);
2867 }
2868 umtxq_lock(&uq->uq_key);
2869 if (error == 0)
2870 umtxq_signal(&uq->uq_key, INT_MAX);
2871 umtxq_unbusy(&uq->uq_key);
2872 umtxq_unlock(&uq->uq_key);
2873 umtx_key_release(&uq->uq_key);
2874 if (error == 0 && old_ceiling != NULL) {
2875 rv = suword32(old_ceiling, save_ceiling);
2876 error = rv == 0 ? 0 : EFAULT;
2877 }
2878 return (error);
2879 }
2880
2881 /*
2882 * Lock a userland POSIX mutex.
2883 */
2884 static int
do_lock_umutex(struct thread * td,struct umutex * m,struct _umtx_time * timeout,int mode)2885 do_lock_umutex(struct thread *td, struct umutex *m,
2886 struct _umtx_time *timeout, int mode)
2887 {
2888 uint32_t flags;
2889 int error;
2890
2891 error = fueword32(&m->m_flags, &flags);
2892 if (error == -1)
2893 return (EFAULT);
2894
2895 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2896 case 0:
2897 error = do_lock_normal(td, m, flags, timeout, mode);
2898 break;
2899 case UMUTEX_PRIO_INHERIT:
2900 error = do_lock_pi(td, m, flags, timeout, mode);
2901 break;
2902 case UMUTEX_PRIO_PROTECT:
2903 error = do_lock_pp(td, m, flags, timeout, mode);
2904 break;
2905 default:
2906 return (EINVAL);
2907 }
2908 if (timeout == NULL) {
2909 if (error == EINTR && mode != _UMUTEX_WAIT)
2910 error = ERESTART;
2911 } else {
2912 /* Timed-locking is not restarted. */
2913 if (error == ERESTART)
2914 error = EINTR;
2915 }
2916 return (error);
2917 }
2918
2919 /*
2920 * Unlock a userland POSIX mutex.
2921 */
2922 static int
do_unlock_umutex(struct thread * td,struct umutex * m,bool rb)2923 do_unlock_umutex(struct thread *td, struct umutex *m, bool rb)
2924 {
2925 uint32_t flags;
2926 int error;
2927
2928 error = fueword32(&m->m_flags, &flags);
2929 if (error == -1)
2930 return (EFAULT);
2931
2932 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2933 case 0:
2934 return (do_unlock_normal(td, m, flags, rb));
2935 case UMUTEX_PRIO_INHERIT:
2936 return (do_unlock_pi(td, m, flags, rb));
2937 case UMUTEX_PRIO_PROTECT:
2938 return (do_unlock_pp(td, m, flags, rb));
2939 }
2940
2941 return (EINVAL);
2942 }
2943
2944 static int
do_cv_wait(struct thread * td,struct ucond * cv,struct umutex * m,struct umtx_abs_timeout * timo,u_long wflags)2945 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2946 struct umtx_abs_timeout *timo, u_long wflags)
2947 {
2948 struct umtx_q *uq;
2949 uint32_t flags, hasw;
2950 int error;
2951
2952 uq = td->td_umtxq;
2953 error = fueword32(&cv->c_flags, &flags);
2954 if (error == -1)
2955 return (EFAULT);
2956 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2957 if (error != 0)
2958 return (error);
2959
2960 umtxq_lock(&uq->uq_key);
2961 umtxq_busy(&uq->uq_key);
2962 umtxq_insert(uq);
2963 umtxq_unlock(&uq->uq_key);
2964
2965 /*
2966 * Set c_has_waiters to 1 before releasing user mutex, also
2967 * don't modify cache line when unnecessary.
2968 */
2969 error = fueword32(&cv->c_has_waiters, &hasw);
2970 if (error == 0 && hasw == 0)
2971 error = suword32(&cv->c_has_waiters, 1);
2972 if (error != 0) {
2973 umtxq_lock(&uq->uq_key);
2974 umtxq_remove(uq);
2975 umtxq_unbusy(&uq->uq_key);
2976 error = EFAULT;
2977 goto out;
2978 }
2979
2980 umtxq_unbusy_unlocked(&uq->uq_key);
2981
2982 error = do_unlock_umutex(td, m, false);
2983
2984 umtxq_lock(&uq->uq_key);
2985 if (error == 0)
2986 error = umtxq_sleep(uq, "ucond", timo);
2987
2988 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2989 error = 0;
2990 else {
2991 /*
2992 * This must be timeout,interrupted by signal or
2993 * surprious wakeup, clear c_has_waiter flag when
2994 * necessary.
2995 */
2996 umtxq_busy(&uq->uq_key);
2997 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
2998 int oldlen = uq->uq_cur_queue->length;
2999 umtxq_remove(uq);
3000 if (oldlen == 1) {
3001 umtxq_unlock(&uq->uq_key);
3002 if (suword32(&cv->c_has_waiters, 0) != 0 &&
3003 error == 0)
3004 error = EFAULT;
3005 umtxq_lock(&uq->uq_key);
3006 }
3007 }
3008 umtxq_unbusy(&uq->uq_key);
3009 if (error == ERESTART)
3010 error = EINTR;
3011 }
3012 out:
3013 umtxq_unlock(&uq->uq_key);
3014 umtx_key_release(&uq->uq_key);
3015 return (error);
3016 }
3017
3018 /*
3019 * Signal a userland condition variable.
3020 */
3021 static int
do_cv_signal(struct thread * td,struct ucond * cv)3022 do_cv_signal(struct thread *td, struct ucond *cv)
3023 {
3024 struct umtx_key key;
3025 int error, cnt, nwake;
3026 uint32_t flags;
3027
3028 error = fueword32(&cv->c_flags, &flags);
3029 if (error == -1)
3030 return (EFAULT);
3031 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
3032 return (error);
3033 umtxq_lock(&key);
3034 umtxq_busy(&key);
3035 cnt = umtxq_count(&key);
3036 nwake = umtxq_signal(&key, 1);
3037 if (cnt <= nwake) {
3038 umtxq_unlock(&key);
3039 error = suword32(&cv->c_has_waiters, 0);
3040 if (error == -1)
3041 error = EFAULT;
3042 umtxq_lock(&key);
3043 }
3044 umtxq_unbusy(&key);
3045 umtxq_unlock(&key);
3046 umtx_key_release(&key);
3047 return (error);
3048 }
3049
3050 static int
do_cv_broadcast(struct thread * td,struct ucond * cv)3051 do_cv_broadcast(struct thread *td, struct ucond *cv)
3052 {
3053 struct umtx_key key;
3054 int error;
3055 uint32_t flags;
3056
3057 error = fueword32(&cv->c_flags, &flags);
3058 if (error == -1)
3059 return (EFAULT);
3060 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
3061 return (error);
3062
3063 umtxq_lock(&key);
3064 umtxq_busy(&key);
3065 umtxq_signal(&key, INT_MAX);
3066 umtxq_unlock(&key);
3067
3068 error = suword32(&cv->c_has_waiters, 0);
3069 if (error == -1)
3070 error = EFAULT;
3071
3072 umtxq_unbusy_unlocked(&key);
3073
3074 umtx_key_release(&key);
3075 return (error);
3076 }
3077
3078 static int
do_rw_rdlock(struct thread * td,struct urwlock * rwlock,long fflag,struct _umtx_time * timeout)3079 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag,
3080 struct _umtx_time *timeout)
3081 {
3082 struct umtx_abs_timeout timo;
3083 struct umtx_q *uq;
3084 uint32_t flags, wrflags;
3085 int32_t state, oldstate;
3086 int32_t blocked_readers;
3087 int error, error1, rv;
3088
3089 uq = td->td_umtxq;
3090 error = fueword32(&rwlock->rw_flags, &flags);
3091 if (error == -1)
3092 return (EFAULT);
3093 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3094 if (error != 0)
3095 return (error);
3096
3097 if (timeout != NULL)
3098 umtx_abs_timeout_init2(&timo, timeout);
3099
3100 wrflags = URWLOCK_WRITE_OWNER;
3101 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
3102 wrflags |= URWLOCK_WRITE_WAITERS;
3103
3104 for (;;) {
3105 rv = fueword32(&rwlock->rw_state, &state);
3106 if (rv == -1) {
3107 umtx_key_release(&uq->uq_key);
3108 return (EFAULT);
3109 }
3110
3111 /* try to lock it */
3112 while (!(state & wrflags)) {
3113 if (__predict_false(URWLOCK_READER_COUNT(state) ==
3114 URWLOCK_MAX_READERS)) {
3115 umtx_key_release(&uq->uq_key);
3116 return (EAGAIN);
3117 }
3118 rv = casueword32(&rwlock->rw_state, state,
3119 &oldstate, state + 1);
3120 if (rv == -1) {
3121 umtx_key_release(&uq->uq_key);
3122 return (EFAULT);
3123 }
3124 if (rv == 0) {
3125 MPASS(oldstate == state);
3126 umtx_key_release(&uq->uq_key);
3127 return (0);
3128 }
3129 error = thread_check_susp(td, true);
3130 if (error != 0)
3131 break;
3132 state = oldstate;
3133 }
3134
3135 if (error)
3136 break;
3137
3138 /* grab monitor lock */
3139 umtxq_busy_unlocked(&uq->uq_key);
3140
3141 /*
3142 * re-read the state, in case it changed between the try-lock above
3143 * and the check below
3144 */
3145 rv = fueword32(&rwlock->rw_state, &state);
3146 if (rv == -1)
3147 error = EFAULT;
3148
3149 /* set read contention bit */
3150 while (error == 0 && (state & wrflags) &&
3151 !(state & URWLOCK_READ_WAITERS)) {
3152 rv = casueword32(&rwlock->rw_state, state,
3153 &oldstate, state | URWLOCK_READ_WAITERS);
3154 if (rv == -1) {
3155 error = EFAULT;
3156 break;
3157 }
3158 if (rv == 0) {
3159 MPASS(oldstate == state);
3160 goto sleep;
3161 }
3162 state = oldstate;
3163 error = thread_check_susp(td, false);
3164 if (error != 0)
3165 break;
3166 }
3167 if (error != 0) {
3168 umtxq_unbusy_unlocked(&uq->uq_key);
3169 break;
3170 }
3171
3172 /* state is changed while setting flags, restart */
3173 if (!(state & wrflags)) {
3174 umtxq_unbusy_unlocked(&uq->uq_key);
3175 error = thread_check_susp(td, true);
3176 if (error != 0)
3177 break;
3178 continue;
3179 }
3180
3181 sleep:
3182 /*
3183 * Contention bit is set, before sleeping, increase
3184 * read waiter count.
3185 */
3186 rv = fueword32(&rwlock->rw_blocked_readers,
3187 &blocked_readers);
3188 if (rv == 0)
3189 rv = suword32(&rwlock->rw_blocked_readers,
3190 blocked_readers + 1);
3191 if (rv == -1) {
3192 umtxq_unbusy_unlocked(&uq->uq_key);
3193 error = EFAULT;
3194 break;
3195 }
3196
3197 while (state & wrflags) {
3198 umtxq_lock(&uq->uq_key);
3199 umtxq_insert(uq);
3200 umtxq_unbusy(&uq->uq_key);
3201
3202 error = umtxq_sleep(uq, "urdlck", timeout == NULL ?
3203 NULL : &timo);
3204
3205 umtxq_busy(&uq->uq_key);
3206 umtxq_remove(uq);
3207 umtxq_unlock(&uq->uq_key);
3208 if (error)
3209 break;
3210 rv = fueword32(&rwlock->rw_state, &state);
3211 if (rv == -1) {
3212 error = EFAULT;
3213 break;
3214 }
3215 }
3216
3217 /* decrease read waiter count, and may clear read contention bit */
3218 rv = fueword32(&rwlock->rw_blocked_readers,
3219 &blocked_readers);
3220 if (rv == 0)
3221 rv = suword32(&rwlock->rw_blocked_readers,
3222 blocked_readers - 1);
3223 if (rv == -1) {
3224 umtxq_unbusy_unlocked(&uq->uq_key);
3225 error = EFAULT;
3226 break;
3227 }
3228 if (blocked_readers == 1) {
3229 rv = fueword32(&rwlock->rw_state, &state);
3230 if (rv == -1) {
3231 umtxq_unbusy_unlocked(&uq->uq_key);
3232 error = EFAULT;
3233 break;
3234 }
3235 for (;;) {
3236 rv = casueword32(&rwlock->rw_state, state,
3237 &oldstate, state & ~URWLOCK_READ_WAITERS);
3238 if (rv == -1) {
3239 error = EFAULT;
3240 break;
3241 }
3242 if (rv == 0) {
3243 MPASS(oldstate == state);
3244 break;
3245 }
3246 state = oldstate;
3247 error1 = thread_check_susp(td, false);
3248 if (error1 != 0) {
3249 if (error == 0)
3250 error = error1;
3251 break;
3252 }
3253 }
3254 }
3255
3256 umtxq_unbusy_unlocked(&uq->uq_key);
3257 if (error != 0)
3258 break;
3259 }
3260 umtx_key_release(&uq->uq_key);
3261 if (error == ERESTART)
3262 error = EINTR;
3263 return (error);
3264 }
3265
3266 static int
do_rw_wrlock(struct thread * td,struct urwlock * rwlock,struct _umtx_time * timeout)3267 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout)
3268 {
3269 struct umtx_abs_timeout timo;
3270 struct umtx_q *uq;
3271 uint32_t flags;
3272 int32_t state, oldstate;
3273 int32_t blocked_writers;
3274 int32_t blocked_readers;
3275 int error, error1, rv;
3276
3277 uq = td->td_umtxq;
3278 error = fueword32(&rwlock->rw_flags, &flags);
3279 if (error == -1)
3280 return (EFAULT);
3281 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3282 if (error != 0)
3283 return (error);
3284
3285 if (timeout != NULL)
3286 umtx_abs_timeout_init2(&timo, timeout);
3287
3288 blocked_readers = 0;
3289 for (;;) {
3290 rv = fueword32(&rwlock->rw_state, &state);
3291 if (rv == -1) {
3292 umtx_key_release(&uq->uq_key);
3293 return (EFAULT);
3294 }
3295 while ((state & URWLOCK_WRITE_OWNER) == 0 &&
3296 URWLOCK_READER_COUNT(state) == 0) {
3297 rv = casueword32(&rwlock->rw_state, state,
3298 &oldstate, state | URWLOCK_WRITE_OWNER);
3299 if (rv == -1) {
3300 umtx_key_release(&uq->uq_key);
3301 return (EFAULT);
3302 }
3303 if (rv == 0) {
3304 MPASS(oldstate == state);
3305 umtx_key_release(&uq->uq_key);
3306 return (0);
3307 }
3308 state = oldstate;
3309 error = thread_check_susp(td, true);
3310 if (error != 0)
3311 break;
3312 }
3313
3314 if (error) {
3315 if ((state & (URWLOCK_WRITE_OWNER |
3316 URWLOCK_WRITE_WAITERS)) == 0 &&
3317 blocked_readers != 0) {
3318 umtxq_lock(&uq->uq_key);
3319 umtxq_busy(&uq->uq_key);
3320 umtxq_signal_queue(&uq->uq_key, INT_MAX,
3321 UMTX_SHARED_QUEUE);
3322 umtxq_unbusy(&uq->uq_key);
3323 umtxq_unlock(&uq->uq_key);
3324 }
3325
3326 break;
3327 }
3328
3329 /* grab monitor lock */
3330 umtxq_busy_unlocked(&uq->uq_key);
3331
3332 /*
3333 * Re-read the state, in case it changed between the
3334 * try-lock above and the check below.
3335 */
3336 rv = fueword32(&rwlock->rw_state, &state);
3337 if (rv == -1)
3338 error = EFAULT;
3339
3340 while (error == 0 && ((state & URWLOCK_WRITE_OWNER) ||
3341 URWLOCK_READER_COUNT(state) != 0) &&
3342 (state & URWLOCK_WRITE_WAITERS) == 0) {
3343 rv = casueword32(&rwlock->rw_state, state,
3344 &oldstate, state | URWLOCK_WRITE_WAITERS);
3345 if (rv == -1) {
3346 error = EFAULT;
3347 break;
3348 }
3349 if (rv == 0) {
3350 MPASS(oldstate == state);
3351 goto sleep;
3352 }
3353 state = oldstate;
3354 error = thread_check_susp(td, false);
3355 if (error != 0)
3356 break;
3357 }
3358 if (error != 0) {
3359 umtxq_unbusy_unlocked(&uq->uq_key);
3360 break;
3361 }
3362
3363 if ((state & URWLOCK_WRITE_OWNER) == 0 &&
3364 URWLOCK_READER_COUNT(state) == 0) {
3365 umtxq_unbusy_unlocked(&uq->uq_key);
3366 error = thread_check_susp(td, false);
3367 if (error != 0)
3368 break;
3369 continue;
3370 }
3371 sleep:
3372 rv = fueword32(&rwlock->rw_blocked_writers,
3373 &blocked_writers);
3374 if (rv == 0)
3375 rv = suword32(&rwlock->rw_blocked_writers,
3376 blocked_writers + 1);
3377 if (rv == -1) {
3378 umtxq_unbusy_unlocked(&uq->uq_key);
3379 error = EFAULT;
3380 break;
3381 }
3382
3383 while ((state & URWLOCK_WRITE_OWNER) ||
3384 URWLOCK_READER_COUNT(state) != 0) {
3385 umtxq_lock(&uq->uq_key);
3386 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3387 umtxq_unbusy(&uq->uq_key);
3388
3389 error = umtxq_sleep(uq, "uwrlck", timeout == NULL ?
3390 NULL : &timo);
3391
3392 umtxq_busy(&uq->uq_key);
3393 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3394 umtxq_unlock(&uq->uq_key);
3395 if (error)
3396 break;
3397 rv = fueword32(&rwlock->rw_state, &state);
3398 if (rv == -1) {
3399 error = EFAULT;
3400 break;
3401 }
3402 }
3403
3404 rv = fueword32(&rwlock->rw_blocked_writers,
3405 &blocked_writers);
3406 if (rv == 0)
3407 rv = suword32(&rwlock->rw_blocked_writers,
3408 blocked_writers - 1);
3409 if (rv == -1) {
3410 umtxq_unbusy_unlocked(&uq->uq_key);
3411 error = EFAULT;
3412 break;
3413 }
3414 if (blocked_writers == 1) {
3415 rv = fueword32(&rwlock->rw_state, &state);
3416 if (rv == -1) {
3417 umtxq_unbusy_unlocked(&uq->uq_key);
3418 error = EFAULT;
3419 break;
3420 }
3421 for (;;) {
3422 rv = casueword32(&rwlock->rw_state, state,
3423 &oldstate, state & ~URWLOCK_WRITE_WAITERS);
3424 if (rv == -1) {
3425 error = EFAULT;
3426 break;
3427 }
3428 if (rv == 0) {
3429 MPASS(oldstate == state);
3430 break;
3431 }
3432 state = oldstate;
3433 error1 = thread_check_susp(td, false);
3434 /*
3435 * We are leaving the URWLOCK_WRITE_WAITERS
3436 * behind, but this should not harm the
3437 * correctness.
3438 */
3439 if (error1 != 0) {
3440 if (error == 0)
3441 error = error1;
3442 break;
3443 }
3444 }
3445 rv = fueword32(&rwlock->rw_blocked_readers,
3446 &blocked_readers);
3447 if (rv == -1) {
3448 umtxq_unbusy_unlocked(&uq->uq_key);
3449 error = EFAULT;
3450 break;
3451 }
3452 } else
3453 blocked_readers = 0;
3454
3455 umtxq_unbusy_unlocked(&uq->uq_key);
3456 }
3457
3458 umtx_key_release(&uq->uq_key);
3459 if (error == ERESTART)
3460 error = EINTR;
3461 return (error);
3462 }
3463
3464 static int
do_rw_unlock(struct thread * td,struct urwlock * rwlock)3465 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
3466 {
3467 struct umtx_q *uq;
3468 uint32_t flags;
3469 int32_t state, oldstate;
3470 int error, rv, q, count;
3471
3472 uq = td->td_umtxq;
3473 error = fueword32(&rwlock->rw_flags, &flags);
3474 if (error == -1)
3475 return (EFAULT);
3476 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3477 if (error != 0)
3478 return (error);
3479
3480 error = fueword32(&rwlock->rw_state, &state);
3481 if (error == -1) {
3482 error = EFAULT;
3483 goto out;
3484 }
3485 if (state & URWLOCK_WRITE_OWNER) {
3486 for (;;) {
3487 rv = casueword32(&rwlock->rw_state, state,
3488 &oldstate, state & ~URWLOCK_WRITE_OWNER);
3489 if (rv == -1) {
3490 error = EFAULT;
3491 goto out;
3492 }
3493 if (rv == 1) {
3494 state = oldstate;
3495 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
3496 error = EPERM;
3497 goto out;
3498 }
3499 error = thread_check_susp(td, true);
3500 if (error != 0)
3501 goto out;
3502 } else
3503 break;
3504 }
3505 } else if (URWLOCK_READER_COUNT(state) != 0) {
3506 for (;;) {
3507 rv = casueword32(&rwlock->rw_state, state,
3508 &oldstate, state - 1);
3509 if (rv == -1) {
3510 error = EFAULT;
3511 goto out;
3512 }
3513 if (rv == 1) {
3514 state = oldstate;
3515 if (URWLOCK_READER_COUNT(oldstate) == 0) {
3516 error = EPERM;
3517 goto out;
3518 }
3519 error = thread_check_susp(td, true);
3520 if (error != 0)
3521 goto out;
3522 } else
3523 break;
3524 }
3525 } else {
3526 error = EPERM;
3527 goto out;
3528 }
3529
3530 count = 0;
3531
3532 if (!(flags & URWLOCK_PREFER_READER)) {
3533 if (state & URWLOCK_WRITE_WAITERS) {
3534 count = 1;
3535 q = UMTX_EXCLUSIVE_QUEUE;
3536 } else if (state & URWLOCK_READ_WAITERS) {
3537 count = INT_MAX;
3538 q = UMTX_SHARED_QUEUE;
3539 }
3540 } else {
3541 if (state & URWLOCK_READ_WAITERS) {
3542 count = INT_MAX;
3543 q = UMTX_SHARED_QUEUE;
3544 } else if (state & URWLOCK_WRITE_WAITERS) {
3545 count = 1;
3546 q = UMTX_EXCLUSIVE_QUEUE;
3547 }
3548 }
3549
3550 if (count) {
3551 umtxq_lock(&uq->uq_key);
3552 umtxq_busy(&uq->uq_key);
3553 umtxq_signal_queue(&uq->uq_key, count, q);
3554 umtxq_unbusy(&uq->uq_key);
3555 umtxq_unlock(&uq->uq_key);
3556 }
3557 out:
3558 umtx_key_release(&uq->uq_key);
3559 return (error);
3560 }
3561
3562 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
3563 static int
do_sem_wait(struct thread * td,struct _usem * sem,struct _umtx_time * timeout)3564 do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout)
3565 {
3566 struct umtx_abs_timeout timo;
3567 struct umtx_q *uq;
3568 uint32_t flags, count, count1;
3569 int error, rv, rv1;
3570
3571 uq = td->td_umtxq;
3572 error = fueword32(&sem->_flags, &flags);
3573 if (error == -1)
3574 return (EFAULT);
3575 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3576 if (error != 0)
3577 return (error);
3578
3579 if (timeout != NULL)
3580 umtx_abs_timeout_init2(&timo, timeout);
3581
3582 again:
3583 umtxq_lock(&uq->uq_key);
3584 umtxq_busy(&uq->uq_key);
3585 umtxq_insert(uq);
3586 umtxq_unlock(&uq->uq_key);
3587 rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
3588 if (rv != -1)
3589 rv1 = fueword32(&sem->_count, &count);
3590 if (rv == -1 || rv1 == -1 || count != 0 || (rv == 1 && count1 == 0)) {
3591 if (rv == 0)
3592 rv = suword32(&sem->_has_waiters, 0);
3593 umtxq_lock(&uq->uq_key);
3594 umtxq_unbusy(&uq->uq_key);
3595 umtxq_remove(uq);
3596 umtxq_unlock(&uq->uq_key);
3597 if (rv == -1 || rv1 == -1) {
3598 error = EFAULT;
3599 goto out;
3600 }
3601 if (count != 0) {
3602 error = 0;
3603 goto out;
3604 }
3605 MPASS(rv == 1 && count1 == 0);
3606 rv = thread_check_susp(td, true);
3607 if (rv == 0)
3608 goto again;
3609 error = rv;
3610 goto out;
3611 }
3612 umtxq_lock(&uq->uq_key);
3613 umtxq_unbusy(&uq->uq_key);
3614
3615 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3616
3617 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3618 error = 0;
3619 else {
3620 umtxq_remove(uq);
3621 /* A relative timeout cannot be restarted. */
3622 if (error == ERESTART && timeout != NULL &&
3623 (timeout->_flags & UMTX_ABSTIME) == 0)
3624 error = EINTR;
3625 }
3626 umtxq_unlock(&uq->uq_key);
3627 out:
3628 umtx_key_release(&uq->uq_key);
3629 return (error);
3630 }
3631
3632 /*
3633 * Signal a userland semaphore.
3634 */
3635 static int
do_sem_wake(struct thread * td,struct _usem * sem)3636 do_sem_wake(struct thread *td, struct _usem *sem)
3637 {
3638 struct umtx_key key;
3639 int error, cnt;
3640 uint32_t flags;
3641
3642 error = fueword32(&sem->_flags, &flags);
3643 if (error == -1)
3644 return (EFAULT);
3645 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3646 return (error);
3647 umtxq_lock(&key);
3648 umtxq_busy(&key);
3649 cnt = umtxq_count(&key);
3650 if (cnt > 0) {
3651 /*
3652 * Check if count is greater than 0, this means the memory is
3653 * still being referenced by user code, so we can safely
3654 * update _has_waiters flag.
3655 */
3656 if (cnt == 1) {
3657 umtxq_unlock(&key);
3658 error = suword32(&sem->_has_waiters, 0);
3659 umtxq_lock(&key);
3660 if (error == -1)
3661 error = EFAULT;
3662 }
3663 umtxq_signal(&key, 1);
3664 }
3665 umtxq_unbusy(&key);
3666 umtxq_unlock(&key);
3667 umtx_key_release(&key);
3668 return (error);
3669 }
3670 #endif
3671
3672 static int
do_sem2_wait(struct thread * td,struct _usem2 * sem,struct _umtx_time * timeout)3673 do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
3674 {
3675 struct umtx_abs_timeout timo;
3676 struct umtx_q *uq;
3677 uint32_t count, flags;
3678 int error, rv;
3679
3680 uq = td->td_umtxq;
3681 flags = fuword32(&sem->_flags);
3682 if (timeout != NULL)
3683 umtx_abs_timeout_init2(&timo, timeout);
3684
3685 again:
3686 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3687 if (error != 0)
3688 return (error);
3689 umtxq_lock(&uq->uq_key);
3690 umtxq_busy(&uq->uq_key);
3691 umtxq_insert(uq);
3692 umtxq_unlock(&uq->uq_key);
3693 rv = fueword32(&sem->_count, &count);
3694 if (rv == -1) {
3695 umtxq_lock(&uq->uq_key);
3696 umtxq_unbusy(&uq->uq_key);
3697 umtxq_remove(uq);
3698 umtxq_unlock(&uq->uq_key);
3699 umtx_key_release(&uq->uq_key);
3700 return (EFAULT);
3701 }
3702 for (;;) {
3703 if (USEM_COUNT(count) != 0) {
3704 umtxq_lock(&uq->uq_key);
3705 umtxq_unbusy(&uq->uq_key);
3706 umtxq_remove(uq);
3707 umtxq_unlock(&uq->uq_key);
3708 umtx_key_release(&uq->uq_key);
3709 return (0);
3710 }
3711 if (count == USEM_HAS_WAITERS)
3712 break;
3713 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
3714 if (rv == 0)
3715 break;
3716 umtxq_lock(&uq->uq_key);
3717 umtxq_unbusy(&uq->uq_key);
3718 umtxq_remove(uq);
3719 umtxq_unlock(&uq->uq_key);
3720 umtx_key_release(&uq->uq_key);
3721 if (rv == -1)
3722 return (EFAULT);
3723 rv = thread_check_susp(td, true);
3724 if (rv != 0)
3725 return (rv);
3726 goto again;
3727 }
3728 umtxq_lock(&uq->uq_key);
3729 umtxq_unbusy(&uq->uq_key);
3730
3731 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3732
3733 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3734 error = 0;
3735 else {
3736 umtxq_remove(uq);
3737 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) {
3738 /* A relative timeout cannot be restarted. */
3739 if (error == ERESTART)
3740 error = EINTR;
3741 if (error == EINTR) {
3742 kern_clock_gettime(curthread, timo.clockid,
3743 &timo.cur);
3744 timespecsub(&timo.end, &timo.cur,
3745 &timeout->_timeout);
3746 }
3747 }
3748 }
3749 umtxq_unlock(&uq->uq_key);
3750 umtx_key_release(&uq->uq_key);
3751 return (error);
3752 }
3753
3754 /*
3755 * Signal a userland semaphore.
3756 */
3757 static int
do_sem2_wake(struct thread * td,struct _usem2 * sem)3758 do_sem2_wake(struct thread *td, struct _usem2 *sem)
3759 {
3760 struct umtx_key key;
3761 int error, cnt, rv;
3762 uint32_t count, flags;
3763
3764 rv = fueword32(&sem->_flags, &flags);
3765 if (rv == -1)
3766 return (EFAULT);
3767 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3768 return (error);
3769 umtxq_lock(&key);
3770 umtxq_busy(&key);
3771 cnt = umtxq_count(&key);
3772 if (cnt > 0) {
3773 /*
3774 * If this was the last sleeping thread, clear the waiters
3775 * flag in _count.
3776 */
3777 if (cnt == 1) {
3778 umtxq_unlock(&key);
3779 rv = fueword32(&sem->_count, &count);
3780 while (rv != -1 && count & USEM_HAS_WAITERS) {
3781 rv = casueword32(&sem->_count, count, &count,
3782 count & ~USEM_HAS_WAITERS);
3783 if (rv == 1) {
3784 rv = thread_check_susp(td, false);
3785 if (rv != 0)
3786 break;
3787 }
3788 }
3789 if (rv == -1)
3790 error = EFAULT;
3791 else if (rv > 0) {
3792 error = rv;
3793 }
3794 umtxq_lock(&key);
3795 }
3796
3797 umtxq_signal(&key, 1);
3798 }
3799 umtxq_unbusy(&key);
3800 umtxq_unlock(&key);
3801 umtx_key_release(&key);
3802 return (error);
3803 }
3804
3805 #ifdef COMPAT_FREEBSD10
3806 int
freebsd10__umtx_lock(struct thread * td,struct freebsd10__umtx_lock_args * uap)3807 freebsd10__umtx_lock(struct thread *td, struct freebsd10__umtx_lock_args *uap)
3808 {
3809 return (do_lock_umtx(td, uap->umtx, td->td_tid, 0));
3810 }
3811
3812 int
freebsd10__umtx_unlock(struct thread * td,struct freebsd10__umtx_unlock_args * uap)3813 freebsd10__umtx_unlock(struct thread *td,
3814 struct freebsd10__umtx_unlock_args *uap)
3815 {
3816 return (do_unlock_umtx(td, uap->umtx, td->td_tid));
3817 }
3818 #endif
3819
3820 inline int
umtx_copyin_timeout(const void * uaddr,struct timespec * tsp)3821 umtx_copyin_timeout(const void *uaddr, struct timespec *tsp)
3822 {
3823 int error;
3824
3825 error = copyin(uaddr, tsp, sizeof(*tsp));
3826 if (error == 0) {
3827 if (!timespecvalid_interval(tsp))
3828 error = EINVAL;
3829 }
3830 return (error);
3831 }
3832
3833 static inline int
umtx_copyin_umtx_time(const void * uaddr,size_t size,struct _umtx_time * tp)3834 umtx_copyin_umtx_time(const void *uaddr, size_t size, struct _umtx_time *tp)
3835 {
3836 int error;
3837
3838 if (size <= sizeof(tp->_timeout)) {
3839 tp->_clockid = CLOCK_REALTIME;
3840 tp->_flags = 0;
3841 error = copyin(uaddr, &tp->_timeout, sizeof(tp->_timeout));
3842 } else
3843 error = copyin(uaddr, tp, sizeof(*tp));
3844 if (error != 0)
3845 return (error);
3846 if (!timespecvalid_interval(&tp->_timeout))
3847 return (EINVAL);
3848 return (0);
3849 }
3850
3851 static int
umtx_copyin_robust_lists(const void * uaddr,size_t size,struct umtx_robust_lists_params * rb)3852 umtx_copyin_robust_lists(const void *uaddr, size_t size,
3853 struct umtx_robust_lists_params *rb)
3854 {
3855
3856 if (size > sizeof(*rb))
3857 return (EINVAL);
3858 return (copyin(uaddr, rb, size));
3859 }
3860
3861 static int
umtx_copyout_timeout(void * uaddr,size_t sz,struct timespec * tsp)3862 umtx_copyout_timeout(void *uaddr, size_t sz, struct timespec *tsp)
3863 {
3864
3865 /*
3866 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
3867 * and we're only called if sz >= sizeof(timespec) as supplied in the
3868 * copyops.
3869 */
3870 KASSERT(sz >= sizeof(*tsp),
3871 ("umtx_copyops specifies incorrect sizes"));
3872
3873 return (copyout(tsp, uaddr, sizeof(*tsp)));
3874 }
3875
3876 #ifdef COMPAT_FREEBSD10
3877 static int
__umtx_op_lock_umtx(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3878 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap,
3879 const struct umtx_copyops *ops)
3880 {
3881 struct timespec *ts, timeout;
3882 int error;
3883
3884 /* Allow a null timespec (wait forever). */
3885 if (uap->uaddr2 == NULL)
3886 ts = NULL;
3887 else {
3888 error = ops->copyin_timeout(uap->uaddr2, &timeout);
3889 if (error != 0)
3890 return (error);
3891 ts = &timeout;
3892 }
3893 #ifdef COMPAT_FREEBSD32
3894 if (ops->compat32)
3895 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3896 #endif
3897 return (do_lock_umtx(td, uap->obj, uap->val, ts));
3898 }
3899
3900 static int
__umtx_op_unlock_umtx(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3901 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap,
3902 const struct umtx_copyops *ops)
3903 {
3904 #ifdef COMPAT_FREEBSD32
3905 if (ops->compat32)
3906 return (do_unlock_umtx32(td, uap->obj, uap->val));
3907 #endif
3908 return (do_unlock_umtx(td, uap->obj, uap->val));
3909 }
3910 #endif /* COMPAT_FREEBSD10 */
3911
3912 #if !defined(COMPAT_FREEBSD10)
3913 static int
__umtx_op_unimpl(struct thread * td __unused,struct _umtx_op_args * uap __unused,const struct umtx_copyops * ops __unused)3914 __umtx_op_unimpl(struct thread *td __unused, struct _umtx_op_args *uap __unused,
3915 const struct umtx_copyops *ops __unused)
3916 {
3917 return (EOPNOTSUPP);
3918 }
3919 #endif /* COMPAT_FREEBSD10 */
3920
3921 static int
__umtx_op_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3922 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap,
3923 const struct umtx_copyops *ops)
3924 {
3925 struct _umtx_time timeout, *tm_p;
3926 int error;
3927
3928 if (uap->uaddr2 == NULL)
3929 tm_p = NULL;
3930 else {
3931 error = ops->copyin_umtx_time(
3932 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3933 if (error != 0)
3934 return (error);
3935 tm_p = &timeout;
3936 }
3937 return (do_wait(td, uap->obj, uap->val, tm_p, ops->compat32, 0));
3938 }
3939
3940 static int
__umtx_op_wait_uint(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3941 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap,
3942 const struct umtx_copyops *ops)
3943 {
3944 struct _umtx_time timeout, *tm_p;
3945 int error;
3946
3947 if (uap->uaddr2 == NULL)
3948 tm_p = NULL;
3949 else {
3950 error = ops->copyin_umtx_time(
3951 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3952 if (error != 0)
3953 return (error);
3954 tm_p = &timeout;
3955 }
3956 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
3957 }
3958
3959 static int
__umtx_op_wait_uint_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3960 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap,
3961 const struct umtx_copyops *ops)
3962 {
3963 struct _umtx_time *tm_p, timeout;
3964 int error;
3965
3966 if (uap->uaddr2 == NULL)
3967 tm_p = NULL;
3968 else {
3969 error = ops->copyin_umtx_time(
3970 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3971 if (error != 0)
3972 return (error);
3973 tm_p = &timeout;
3974 }
3975 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
3976 }
3977
3978 static int
__umtx_op_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)3979 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap,
3980 const struct umtx_copyops *ops __unused)
3981 {
3982
3983 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3984 }
3985
3986 #define BATCH_SIZE 128
3987 static int
__umtx_op_nwake_private_native(struct thread * td,struct _umtx_op_args * uap)3988 __umtx_op_nwake_private_native(struct thread *td, struct _umtx_op_args *uap)
3989 {
3990 char *uaddrs[BATCH_SIZE], **upp;
3991 int count, error, i, pos, tocopy;
3992
3993 upp = (char **)uap->obj;
3994 error = 0;
3995 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
3996 pos += tocopy) {
3997 tocopy = MIN(count, BATCH_SIZE);
3998 error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *));
3999 if (error != 0)
4000 break;
4001 for (i = 0; i < tocopy; ++i) {
4002 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1);
4003 }
4004 maybe_yield();
4005 }
4006 return (error);
4007 }
4008
4009 static int
__umtx_op_nwake_private_compat32(struct thread * td,struct _umtx_op_args * uap)4010 __umtx_op_nwake_private_compat32(struct thread *td, struct _umtx_op_args *uap)
4011 {
4012 uint32_t uaddrs[BATCH_SIZE], *upp;
4013 int count, error, i, pos, tocopy;
4014
4015 upp = (uint32_t *)uap->obj;
4016 error = 0;
4017 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4018 pos += tocopy) {
4019 tocopy = MIN(count, BATCH_SIZE);
4020 error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t));
4021 if (error != 0)
4022 break;
4023 for (i = 0; i < tocopy; ++i) {
4024 kern_umtx_wake(td, (void *)(uintptr_t)uaddrs[i],
4025 INT_MAX, 1);
4026 }
4027 maybe_yield();
4028 }
4029 return (error);
4030 }
4031
4032 static int
__umtx_op_nwake_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4033 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap,
4034 const struct umtx_copyops *ops)
4035 {
4036
4037 if (ops->compat32)
4038 return (__umtx_op_nwake_private_compat32(td, uap));
4039 return (__umtx_op_nwake_private_native(td, uap));
4040 }
4041
4042 static int
__umtx_op_wake_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4043 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap,
4044 const struct umtx_copyops *ops __unused)
4045 {
4046
4047 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
4048 }
4049
4050 static int
__umtx_op_lock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4051 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap,
4052 const struct umtx_copyops *ops)
4053 {
4054 struct _umtx_time *tm_p, timeout;
4055 int error;
4056
4057 /* Allow a null timespec (wait forever). */
4058 if (uap->uaddr2 == NULL)
4059 tm_p = NULL;
4060 else {
4061 error = ops->copyin_umtx_time(
4062 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4063 if (error != 0)
4064 return (error);
4065 tm_p = &timeout;
4066 }
4067 return (do_lock_umutex(td, uap->obj, tm_p, 0));
4068 }
4069
4070 static int
__umtx_op_trylock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4071 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap,
4072 const struct umtx_copyops *ops __unused)
4073 {
4074
4075 return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY));
4076 }
4077
4078 static int
__umtx_op_wait_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4079 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap,
4080 const struct umtx_copyops *ops)
4081 {
4082 struct _umtx_time *tm_p, timeout;
4083 int error;
4084
4085 /* Allow a null timespec (wait forever). */
4086 if (uap->uaddr2 == NULL)
4087 tm_p = NULL;
4088 else {
4089 error = ops->copyin_umtx_time(
4090 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4091 if (error != 0)
4092 return (error);
4093 tm_p = &timeout;
4094 }
4095 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
4096 }
4097
4098 static int
__umtx_op_wake_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4099 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap,
4100 const struct umtx_copyops *ops __unused)
4101 {
4102
4103 return (do_wake_umutex(td, uap->obj));
4104 }
4105
4106 static int
__umtx_op_unlock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4107 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap,
4108 const struct umtx_copyops *ops __unused)
4109 {
4110
4111 return (do_unlock_umutex(td, uap->obj, false));
4112 }
4113
4114 static int
__umtx_op_set_ceiling(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4115 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap,
4116 const struct umtx_copyops *ops __unused)
4117 {
4118
4119 return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1));
4120 }
4121
4122 static int
__umtx_op_cv_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4123 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap,
4124 const struct umtx_copyops *ops)
4125 {
4126 struct umtx_abs_timeout *timop, timo;
4127 struct timespec *ts, timeout;
4128 struct _umtx_time umtime;
4129 struct ucond *cv;
4130 u_long wflags;
4131 uint32_t clockid;
4132 int error;
4133
4134 cv = uap->obj;
4135 wflags = uap->val;
4136 if ((wflags & ~(CVWAIT_CHECK_UNPARKING | CVWAIT_ABSTIME |
4137 CVWAIT_CLOCKID | CVWAIT_UMTX_TIME)) != 0 ||
4138 ((wflags & (CVWAIT_ABSTIME | CVWAIT_CLOCKID)) != 0 &&
4139 (wflags & CVWAIT_UMTX_TIME) != 0))
4140 return (EINVAL);
4141
4142 if ((wflags & CVWAIT_UMTX_TIME) == 0) {
4143 /* Allow a null timespec (wait forever). */
4144 if (uap->uaddr2 == NULL) {
4145 ts = NULL;
4146 } else {
4147 error = ops->copyin_timeout(uap->uaddr2, &timeout);
4148 if (error != 0)
4149 return (error);
4150 ts = &timeout;
4151 }
4152 if ((wflags & CVWAIT_CLOCKID) != 0) {
4153 error = fueword32(&cv->c_clockid, &clockid);
4154 if (error == -1)
4155 return (EFAULT);
4156 } else {
4157 clockid = CLOCK_REALTIME;
4158 }
4159 if (ts != NULL) {
4160 umtx_abs_timeout_init(&timo, clockid,
4161 (wflags & CVWAIT_ABSTIME) != 0, ts);
4162 timop = &timo;
4163 } else {
4164 timop = NULL;
4165 }
4166 } else {
4167 if (uap->uaddr2 == NULL)
4168 return (EINVAL);
4169 error = ops->copyin_umtx_time(uap->uaddr2, ops->umtx_time_sz,
4170 &umtime);
4171 if (error != 0)
4172 return (error);
4173 timop = &timo;
4174 umtx_abs_timeout_init2(timop, &umtime);
4175 }
4176 /* only HW clock id will work. */
4177 if (timop != NULL && (timop->clockid < CLOCK_REALTIME ||
4178 timop->clockid >= CLOCK_THREAD_CPUTIME_ID) &&
4179 timop->clockid != CLOCK_TAI)
4180 return (EINVAL);
4181
4182 return (do_cv_wait(td, cv, uap->uaddr1, timop, wflags));
4183 }
4184
4185 static int
__umtx_op_cv_signal(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4186 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap,
4187 const struct umtx_copyops *ops __unused)
4188 {
4189
4190 return (do_cv_signal(td, uap->obj));
4191 }
4192
4193 static int
__umtx_op_cv_broadcast(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4194 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap,
4195 const struct umtx_copyops *ops __unused)
4196 {
4197
4198 return (do_cv_broadcast(td, uap->obj));
4199 }
4200
4201 static int
__umtx_op_rw_rdlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4202 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap,
4203 const struct umtx_copyops *ops)
4204 {
4205 struct _umtx_time timeout;
4206 int error;
4207
4208 /* Allow a null timespec (wait forever). */
4209 if (uap->uaddr2 == NULL) {
4210 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
4211 } else {
4212 error = ops->copyin_umtx_time(uap->uaddr2,
4213 (size_t)uap->uaddr1, &timeout);
4214 if (error != 0)
4215 return (error);
4216 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
4217 }
4218 return (error);
4219 }
4220
4221 static int
__umtx_op_rw_wrlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4222 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap,
4223 const struct umtx_copyops *ops)
4224 {
4225 struct _umtx_time timeout;
4226 int error;
4227
4228 /* Allow a null timespec (wait forever). */
4229 if (uap->uaddr2 == NULL) {
4230 error = do_rw_wrlock(td, uap->obj, 0);
4231 } else {
4232 error = ops->copyin_umtx_time(uap->uaddr2,
4233 (size_t)uap->uaddr1, &timeout);
4234 if (error != 0)
4235 return (error);
4236
4237 error = do_rw_wrlock(td, uap->obj, &timeout);
4238 }
4239 return (error);
4240 }
4241
4242 static int
__umtx_op_rw_unlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4243 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap,
4244 const struct umtx_copyops *ops __unused)
4245 {
4246
4247 return (do_rw_unlock(td, uap->obj));
4248 }
4249
4250 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4251 static int
__umtx_op_sem_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4252 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap,
4253 const struct umtx_copyops *ops)
4254 {
4255 struct _umtx_time *tm_p, timeout;
4256 int error;
4257
4258 /* Allow a null timespec (wait forever). */
4259 if (uap->uaddr2 == NULL)
4260 tm_p = NULL;
4261 else {
4262 error = ops->copyin_umtx_time(
4263 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4264 if (error != 0)
4265 return (error);
4266 tm_p = &timeout;
4267 }
4268 return (do_sem_wait(td, uap->obj, tm_p));
4269 }
4270
4271 static int
__umtx_op_sem_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4272 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap,
4273 const struct umtx_copyops *ops __unused)
4274 {
4275
4276 return (do_sem_wake(td, uap->obj));
4277 }
4278 #endif
4279
4280 static int
__umtx_op_wake2_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4281 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap,
4282 const struct umtx_copyops *ops __unused)
4283 {
4284
4285 return (do_wake2_umutex(td, uap->obj, uap->val));
4286 }
4287
4288 static int
__umtx_op_sem2_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4289 __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap,
4290 const struct umtx_copyops *ops)
4291 {
4292 struct _umtx_time *tm_p, timeout;
4293 size_t uasize;
4294 int error;
4295
4296 /* Allow a null timespec (wait forever). */
4297 if (uap->uaddr2 == NULL) {
4298 uasize = 0;
4299 tm_p = NULL;
4300 } else {
4301 uasize = (size_t)uap->uaddr1;
4302 error = ops->copyin_umtx_time(uap->uaddr2, uasize, &timeout);
4303 if (error != 0)
4304 return (error);
4305 tm_p = &timeout;
4306 }
4307 error = do_sem2_wait(td, uap->obj, tm_p);
4308 if (error == EINTR && uap->uaddr2 != NULL &&
4309 (timeout._flags & UMTX_ABSTIME) == 0 &&
4310 uasize >= ops->umtx_time_sz + ops->timespec_sz) {
4311 error = ops->copyout_timeout(
4312 (void *)((uintptr_t)uap->uaddr2 + ops->umtx_time_sz),
4313 uasize - ops->umtx_time_sz, &timeout._timeout);
4314 if (error == 0) {
4315 error = EINTR;
4316 }
4317 }
4318
4319 return (error);
4320 }
4321
4322 static int
__umtx_op_sem2_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4323 __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap,
4324 const struct umtx_copyops *ops __unused)
4325 {
4326
4327 return (do_sem2_wake(td, uap->obj));
4328 }
4329
4330 #define USHM_OBJ_UMTX(o) \
4331 ((struct umtx_shm_obj_list *)(&(o)->umtx_data))
4332
4333 #define USHMF_LINKED 0x0001
4334 struct umtx_shm_reg {
4335 TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link;
4336 LIST_ENTRY(umtx_shm_reg) ushm_obj_link;
4337 struct umtx_key ushm_key;
4338 struct ucred *ushm_cred;
4339 struct shmfd *ushm_obj;
4340 u_int ushm_refcnt;
4341 u_int ushm_flags;
4342 };
4343
4344 LIST_HEAD(umtx_shm_obj_list, umtx_shm_reg);
4345 TAILQ_HEAD(umtx_shm_reg_head, umtx_shm_reg);
4346
4347 static uma_zone_t umtx_shm_reg_zone;
4348 static struct umtx_shm_reg_head umtx_shm_registry[UMTX_CHAINS];
4349 static struct mtx umtx_shm_lock;
4350 static struct umtx_shm_reg_head umtx_shm_reg_delfree =
4351 TAILQ_HEAD_INITIALIZER(umtx_shm_reg_delfree);
4352
4353 static void umtx_shm_free_reg(struct umtx_shm_reg *reg);
4354
4355 static void
umtx_shm_reg_delfree_tq(void * context __unused,int pending __unused)4356 umtx_shm_reg_delfree_tq(void *context __unused, int pending __unused)
4357 {
4358 struct umtx_shm_reg_head d;
4359 struct umtx_shm_reg *reg, *reg1;
4360
4361 TAILQ_INIT(&d);
4362 mtx_lock(&umtx_shm_lock);
4363 TAILQ_CONCAT(&d, &umtx_shm_reg_delfree, ushm_reg_link);
4364 mtx_unlock(&umtx_shm_lock);
4365 TAILQ_FOREACH_SAFE(reg, &d, ushm_reg_link, reg1) {
4366 TAILQ_REMOVE(&d, reg, ushm_reg_link);
4367 umtx_shm_free_reg(reg);
4368 }
4369 }
4370
4371 static struct task umtx_shm_reg_delfree_task =
4372 TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL);
4373
4374 /*
4375 * Returns 0 if a SHM with the passed key is found in the registry, in which
4376 * case it is returned through 'oreg'. Otherwise, returns an error among ESRCH
4377 * (no corresponding SHM; ESRCH was chosen for compatibility, ENOENT would have
4378 * been preferable) or EOVERFLOW (there is a corresponding SHM, but reference
4379 * count would overflow, so can't return it), in which case '*oreg' is left
4380 * unchanged.
4381 */
4382 static int
umtx_shm_find_reg_locked(const struct umtx_key * key,struct umtx_shm_reg ** const oreg)4383 umtx_shm_find_reg_locked(const struct umtx_key *key,
4384 struct umtx_shm_reg **const oreg)
4385 {
4386 struct umtx_shm_reg *reg;
4387 struct umtx_shm_reg_head *reg_head;
4388
4389 KASSERT(key->shared, ("umtx_p_find_rg: private key"));
4390 mtx_assert(&umtx_shm_lock, MA_OWNED);
4391 reg_head = &umtx_shm_registry[key->hash];
4392 TAILQ_FOREACH(reg, reg_head, ushm_reg_link) {
4393 KASSERT(reg->ushm_key.shared,
4394 ("non-shared key on reg %p %d", reg, reg->ushm_key.shared));
4395 if (reg->ushm_key.info.shared.object ==
4396 key->info.shared.object &&
4397 reg->ushm_key.info.shared.offset ==
4398 key->info.shared.offset) {
4399 KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM"));
4400 KASSERT(reg->ushm_refcnt != 0,
4401 ("reg %p refcnt 0 onlist", reg));
4402 KASSERT((reg->ushm_flags & USHMF_LINKED) != 0,
4403 ("reg %p not linked", reg));
4404 /*
4405 * Don't let overflow happen, just deny a new reference
4406 * (this is additional protection against some reference
4407 * count leak, which is known not to be the case at the
4408 * time of this writing).
4409 */
4410 if (__predict_false(reg->ushm_refcnt == UINT_MAX))
4411 return (EOVERFLOW);
4412 reg->ushm_refcnt++;
4413 *oreg = reg;
4414 return (0);
4415 }
4416 }
4417 return (ESRCH);
4418 }
4419
4420 /*
4421 * Calls umtx_shm_find_reg_unlocked() under the 'umtx_shm_lock'.
4422 */
4423 static int
umtx_shm_find_reg(const struct umtx_key * key,struct umtx_shm_reg ** const oreg)4424 umtx_shm_find_reg(const struct umtx_key *key, struct umtx_shm_reg **const oreg)
4425 {
4426 int error;
4427
4428 mtx_lock(&umtx_shm_lock);
4429 error = umtx_shm_find_reg_locked(key, oreg);
4430 mtx_unlock(&umtx_shm_lock);
4431 return (error);
4432 }
4433
4434 static void
umtx_shm_free_reg(struct umtx_shm_reg * reg)4435 umtx_shm_free_reg(struct umtx_shm_reg *reg)
4436 {
4437
4438 chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0);
4439 crfree(reg->ushm_cred);
4440 shm_drop(reg->ushm_obj);
4441 uma_zfree(umtx_shm_reg_zone, reg);
4442 }
4443
4444 static bool
umtx_shm_unref_reg_locked(struct umtx_shm_reg * reg,bool linked_ref)4445 umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool linked_ref)
4446 {
4447 mtx_assert(&umtx_shm_lock, MA_OWNED);
4448 KASSERT(reg->ushm_refcnt != 0, ("ushm_reg %p refcnt 0", reg));
4449
4450 if (linked_ref) {
4451 if ((reg->ushm_flags & USHMF_LINKED) == 0)
4452 /*
4453 * The reference tied to USHMF_LINKED has already been
4454 * released concurrently.
4455 */
4456 return (false);
4457
4458 TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], reg,
4459 ushm_reg_link);
4460 LIST_REMOVE(reg, ushm_obj_link);
4461 reg->ushm_flags &= ~USHMF_LINKED;
4462 }
4463
4464 reg->ushm_refcnt--;
4465 return (reg->ushm_refcnt == 0);
4466 }
4467
4468 static void
umtx_shm_unref_reg(struct umtx_shm_reg * reg,bool linked_ref)4469 umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool linked_ref)
4470 {
4471 vm_object_t object;
4472 bool dofree;
4473
4474 if (linked_ref) {
4475 /*
4476 * Note: This may be executed multiple times on the same
4477 * shared-memory VM object in presence of concurrent callers
4478 * because 'umtx_shm_lock' is not held all along in umtx_shm()
4479 * and here.
4480 */
4481 object = reg->ushm_obj->shm_object;
4482 VM_OBJECT_WLOCK(object);
4483 vm_object_set_flag(object, OBJ_UMTXDEAD);
4484 VM_OBJECT_WUNLOCK(object);
4485 }
4486 mtx_lock(&umtx_shm_lock);
4487 dofree = umtx_shm_unref_reg_locked(reg, linked_ref);
4488 mtx_unlock(&umtx_shm_lock);
4489 if (dofree)
4490 umtx_shm_free_reg(reg);
4491 }
4492
4493 void
umtx_shm_object_init(vm_object_t object)4494 umtx_shm_object_init(vm_object_t object)
4495 {
4496
4497 LIST_INIT(USHM_OBJ_UMTX(object));
4498 }
4499
4500 void
umtx_shm_object_terminated(vm_object_t object)4501 umtx_shm_object_terminated(vm_object_t object)
4502 {
4503 struct umtx_shm_reg *reg, *reg1;
4504 bool dofree;
4505
4506 if (LIST_EMPTY(USHM_OBJ_UMTX(object)))
4507 return;
4508
4509 dofree = false;
4510 mtx_lock(&umtx_shm_lock);
4511 LIST_FOREACH_SAFE(reg, USHM_OBJ_UMTX(object), ushm_obj_link, reg1) {
4512 if (umtx_shm_unref_reg_locked(reg, true)) {
4513 TAILQ_INSERT_TAIL(&umtx_shm_reg_delfree, reg,
4514 ushm_reg_link);
4515 dofree = true;
4516 }
4517 }
4518 mtx_unlock(&umtx_shm_lock);
4519 if (dofree)
4520 taskqueue_enqueue(taskqueue_thread, &umtx_shm_reg_delfree_task);
4521 }
4522
4523 static int
umtx_shm_create_reg(struct thread * td,const struct umtx_key * key,struct umtx_shm_reg ** res)4524 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key,
4525 struct umtx_shm_reg **res)
4526 {
4527 struct shmfd *shm;
4528 struct umtx_shm_reg *reg, *reg1;
4529 struct ucred *cred;
4530 int error;
4531
4532 error = umtx_shm_find_reg(key, res);
4533 if (error != ESRCH) {
4534 /*
4535 * Either no error occured, and '*res' was filled, or EOVERFLOW
4536 * was returned, indicating a reference count limit, and we
4537 * won't create a duplicate registration. In both cases, we are
4538 * done.
4539 */
4540 return (error);
4541 }
4542 /* No entry, we will create one. */
4543
4544 cred = td->td_ucred;
4545 if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP)))
4546 return (ENOMEM);
4547 shm = shm_alloc(td->td_ucred, O_RDWR, false);
4548 if (shm == NULL) {
4549 chgumtxcnt(cred->cr_ruidinfo, -1, 0);
4550 return (ENOMEM);
4551 }
4552 reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO);
4553 bcopy(key, ®->ushm_key, sizeof(*key));
4554 reg->ushm_obj = shm;
4555 reg->ushm_cred = crhold(cred);
4556 error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE);
4557 if (error != 0) {
4558 umtx_shm_free_reg(reg);
4559 return (error);
4560 }
4561 mtx_lock(&umtx_shm_lock);
4562 /* Re-lookup as 'umtx_shm_lock' has been temporarily released. */
4563 error = umtx_shm_find_reg_locked(key, ®1);
4564 switch (error) {
4565 case 0:
4566 mtx_unlock(&umtx_shm_lock);
4567 umtx_shm_free_reg(reg);
4568 *res = reg1;
4569 return (0);
4570 case ESRCH:
4571 break;
4572 default:
4573 mtx_unlock(&umtx_shm_lock);
4574 umtx_shm_free_reg(reg);
4575 return (error);
4576 }
4577 TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link);
4578 LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg,
4579 ushm_obj_link);
4580 reg->ushm_flags = USHMF_LINKED;
4581 /*
4582 * This is one reference for the registry and the list of shared
4583 * mutexes referenced by the VM object containing the lock pointer, and
4584 * another for the caller, which it will free after use. So, one of
4585 * these is tied to the presence of USHMF_LINKED.
4586 */
4587 reg->ushm_refcnt = 2;
4588 mtx_unlock(&umtx_shm_lock);
4589 *res = reg;
4590 return (0);
4591 }
4592
4593 static int
umtx_shm_alive(struct thread * td,void * addr)4594 umtx_shm_alive(struct thread *td, void *addr)
4595 {
4596 vm_map_t map;
4597 vm_map_entry_t entry;
4598 vm_object_t object;
4599 vm_pindex_t pindex;
4600 vm_prot_t prot;
4601 int res, ret;
4602 boolean_t wired;
4603
4604 map = &td->td_proc->p_vmspace->vm_map;
4605 res = vm_map_lookup(&map, (uintptr_t)addr, VM_PROT_READ, &entry,
4606 &object, &pindex, &prot, &wired);
4607 if (res != KERN_SUCCESS)
4608 return (EFAULT);
4609 if (object == NULL)
4610 ret = EINVAL;
4611 else
4612 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0;
4613 vm_map_lookup_done(map, entry);
4614 return (ret);
4615 }
4616
4617 static void
umtx_shm_init(void)4618 umtx_shm_init(void)
4619 {
4620 int i;
4621
4622 umtx_shm_reg_zone = uma_zcreate("umtx_shm", sizeof(struct umtx_shm_reg),
4623 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
4624 mtx_init(&umtx_shm_lock, "umtxshm", NULL, MTX_DEF);
4625 for (i = 0; i < nitems(umtx_shm_registry); i++)
4626 TAILQ_INIT(&umtx_shm_registry[i]);
4627 }
4628
4629 static int
umtx_shm(struct thread * td,void * addr,u_int flags)4630 umtx_shm(struct thread *td, void *addr, u_int flags)
4631 {
4632 struct umtx_key key;
4633 struct umtx_shm_reg *reg;
4634 struct file *fp;
4635 int error, fd;
4636
4637 if (__bitcount(flags & (UMTX_SHM_CREAT | UMTX_SHM_LOOKUP |
4638 UMTX_SHM_DESTROY| UMTX_SHM_ALIVE)) != 1)
4639 return (EINVAL);
4640 if ((flags & UMTX_SHM_ALIVE) != 0)
4641 return (umtx_shm_alive(td, addr));
4642 error = umtx_key_get(addr, TYPE_SHM, PROCESS_SHARE, &key);
4643 if (error != 0)
4644 return (error);
4645 KASSERT(key.shared == 1, ("non-shared key"));
4646 error = (flags & UMTX_SHM_CREAT) != 0 ?
4647 umtx_shm_create_reg(td, &key, ®) :
4648 umtx_shm_find_reg(&key, ®);
4649 umtx_key_release(&key);
4650 if (error != 0)
4651 return (error);
4652 KASSERT(reg != NULL, ("no reg"));
4653 if ((flags & UMTX_SHM_DESTROY) != 0) {
4654 umtx_shm_unref_reg(reg, true);
4655 } else {
4656 /*
4657 * The current vmspace has the mapping, so it can be
4658 * converted into shm filedescriptor for current
4659 * thread.
4660 */
4661 error = falloc_caps(td, &fp, &fd, O_CLOEXEC, NULL);
4662 if (error == 0) {
4663 shm_hold(reg->ushm_obj);
4664 finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj,
4665 &shm_ops);
4666 td->td_retval[0] = fd;
4667 fdrop(fp, td);
4668 }
4669 }
4670 umtx_shm_unref_reg(reg, false);
4671 return (error);
4672 }
4673
4674 static int
__umtx_op_shm(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4675 __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap,
4676 const struct umtx_copyops *ops __unused)
4677 {
4678
4679 return (umtx_shm(td, uap->uaddr1, uap->val));
4680 }
4681
4682 static int
__umtx_op_robust_lists(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4683 __umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap,
4684 const struct umtx_copyops *ops)
4685 {
4686 struct umtx_robust_lists_params rb;
4687 int error;
4688
4689 if (ops->compat32) {
4690 if ((td->td_pflags2 & TDP2_COMPAT32RB) == 0 &&
4691 (td->td_rb_list != 0 || td->td_rbp_list != 0 ||
4692 td->td_rb_inact != 0))
4693 return (EBUSY);
4694 } else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) {
4695 return (EBUSY);
4696 }
4697
4698 bzero(&rb, sizeof(rb));
4699 error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb);
4700 if (error != 0)
4701 return (error);
4702
4703 if (ops->compat32)
4704 td->td_pflags2 |= TDP2_COMPAT32RB;
4705
4706 td->td_rb_list = rb.robust_list_offset;
4707 td->td_rbp_list = rb.robust_priv_list_offset;
4708 td->td_rb_inact = rb.robust_inact_offset;
4709 return (0);
4710 }
4711
4712 static int
__umtx_op_get_min_timeout(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4713 __umtx_op_get_min_timeout(struct thread *td, struct _umtx_op_args *uap,
4714 const struct umtx_copyops *ops)
4715 {
4716 long val;
4717 int error, val1;
4718
4719 val = sbttons(td->td_proc->p_umtx_min_timeout);
4720 if (ops->compat32) {
4721 val1 = (int)val;
4722 error = copyout(&val1, uap->uaddr1, sizeof(val1));
4723 } else {
4724 error = copyout(&val, uap->uaddr1, sizeof(val));
4725 }
4726 return (error);
4727 }
4728
4729 static int
__umtx_op_set_min_timeout(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4730 __umtx_op_set_min_timeout(struct thread *td, struct _umtx_op_args *uap,
4731 const struct umtx_copyops *ops)
4732 {
4733 if (uap->val < 0)
4734 return (EINVAL);
4735 td->td_proc->p_umtx_min_timeout = nstosbt(uap->val);
4736 return (0);
4737 }
4738
4739 #if defined(__i386__) || defined(__amd64__)
4740 /*
4741 * Provide the standard 32-bit definitions for x86, since native/compat32 use a
4742 * 32-bit time_t there. Other architectures just need the i386 definitions
4743 * along with their standard compat32.
4744 */
4745 struct timespecx32 {
4746 int64_t tv_sec;
4747 int32_t tv_nsec;
4748 };
4749
4750 struct umtx_timex32 {
4751 struct timespecx32 _timeout;
4752 uint32_t _flags;
4753 uint32_t _clockid;
4754 };
4755
4756 #ifndef __i386__
4757 #define timespeci386 timespec32
4758 #define umtx_timei386 umtx_time32
4759 #endif
4760 #else /* !__i386__ && !__amd64__ */
4761 /* 32-bit architectures can emulate i386, so define these almost everywhere. */
4762 struct timespeci386 {
4763 int32_t tv_sec;
4764 int32_t tv_nsec;
4765 };
4766
4767 struct umtx_timei386 {
4768 struct timespeci386 _timeout;
4769 uint32_t _flags;
4770 uint32_t _clockid;
4771 };
4772
4773 #if defined(__LP64__)
4774 #define timespecx32 timespec32
4775 #define umtx_timex32 umtx_time32
4776 #endif
4777 #endif
4778
4779 static int
umtx_copyin_robust_lists32(const void * uaddr,size_t size,struct umtx_robust_lists_params * rbp)4780 umtx_copyin_robust_lists32(const void *uaddr, size_t size,
4781 struct umtx_robust_lists_params *rbp)
4782 {
4783 struct umtx_robust_lists_params_compat32 rb32;
4784 int error;
4785
4786 if (size > sizeof(rb32))
4787 return (EINVAL);
4788 bzero(&rb32, sizeof(rb32));
4789 error = copyin(uaddr, &rb32, size);
4790 if (error != 0)
4791 return (error);
4792 CP(rb32, *rbp, robust_list_offset);
4793 CP(rb32, *rbp, robust_priv_list_offset);
4794 CP(rb32, *rbp, robust_inact_offset);
4795 return (0);
4796 }
4797
4798 #ifndef __i386__
4799 static inline int
umtx_copyin_timeouti386(const void * uaddr,struct timespec * tsp)4800 umtx_copyin_timeouti386(const void *uaddr, struct timespec *tsp)
4801 {
4802 struct timespeci386 ts32;
4803 int error;
4804
4805 error = copyin(uaddr, &ts32, sizeof(ts32));
4806 if (error == 0) {
4807 if (!timespecvalid_interval(&ts32))
4808 error = EINVAL;
4809 else {
4810 CP(ts32, *tsp, tv_sec);
4811 CP(ts32, *tsp, tv_nsec);
4812 }
4813 }
4814 return (error);
4815 }
4816
4817 static inline int
umtx_copyin_umtx_timei386(const void * uaddr,size_t size,struct _umtx_time * tp)4818 umtx_copyin_umtx_timei386(const void *uaddr, size_t size, struct _umtx_time *tp)
4819 {
4820 struct umtx_timei386 t32;
4821 int error;
4822
4823 t32._clockid = CLOCK_REALTIME;
4824 t32._flags = 0;
4825 if (size <= sizeof(t32._timeout))
4826 error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout));
4827 else
4828 error = copyin(uaddr, &t32, sizeof(t32));
4829 if (error != 0)
4830 return (error);
4831 if (!timespecvalid_interval(&t32._timeout))
4832 return (EINVAL);
4833 TS_CP(t32, *tp, _timeout);
4834 CP(t32, *tp, _flags);
4835 CP(t32, *tp, _clockid);
4836 return (0);
4837 }
4838
4839 static int
umtx_copyout_timeouti386(void * uaddr,size_t sz,struct timespec * tsp)4840 umtx_copyout_timeouti386(void *uaddr, size_t sz, struct timespec *tsp)
4841 {
4842 struct timespeci386 remain32 = {
4843 .tv_sec = tsp->tv_sec,
4844 .tv_nsec = tsp->tv_nsec,
4845 };
4846
4847 /*
4848 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4849 * and we're only called if sz >= sizeof(timespec) as supplied in the
4850 * copyops.
4851 */
4852 KASSERT(sz >= sizeof(remain32),
4853 ("umtx_copyops specifies incorrect sizes"));
4854
4855 return (copyout(&remain32, uaddr, sizeof(remain32)));
4856 }
4857 #endif /* !__i386__ */
4858
4859 #if defined(__i386__) || defined(__LP64__)
4860 static inline int
umtx_copyin_timeoutx32(const void * uaddr,struct timespec * tsp)4861 umtx_copyin_timeoutx32(const void *uaddr, struct timespec *tsp)
4862 {
4863 struct timespecx32 ts32;
4864 int error;
4865
4866 error = copyin(uaddr, &ts32, sizeof(ts32));
4867 if (error == 0) {
4868 if (!timespecvalid_interval(&ts32))
4869 error = EINVAL;
4870 else {
4871 CP(ts32, *tsp, tv_sec);
4872 CP(ts32, *tsp, tv_nsec);
4873 }
4874 }
4875 return (error);
4876 }
4877
4878 static inline int
umtx_copyin_umtx_timex32(const void * uaddr,size_t size,struct _umtx_time * tp)4879 umtx_copyin_umtx_timex32(const void *uaddr, size_t size, struct _umtx_time *tp)
4880 {
4881 struct umtx_timex32 t32;
4882 int error;
4883
4884 t32._clockid = CLOCK_REALTIME;
4885 t32._flags = 0;
4886 if (size <= sizeof(t32._timeout))
4887 error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout));
4888 else
4889 error = copyin(uaddr, &t32, sizeof(t32));
4890 if (error != 0)
4891 return (error);
4892 if (!timespecvalid_interval(&t32._timeout))
4893 return (EINVAL);
4894 TS_CP(t32, *tp, _timeout);
4895 CP(t32, *tp, _flags);
4896 CP(t32, *tp, _clockid);
4897 return (0);
4898 }
4899
4900 static int
umtx_copyout_timeoutx32(void * uaddr,size_t sz,struct timespec * tsp)4901 umtx_copyout_timeoutx32(void *uaddr, size_t sz, struct timespec *tsp)
4902 {
4903 struct timespecx32 remain32 = {
4904 .tv_sec = tsp->tv_sec,
4905 .tv_nsec = tsp->tv_nsec,
4906 };
4907
4908 /*
4909 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4910 * and we're only called if sz >= sizeof(timespec) as supplied in the
4911 * copyops.
4912 */
4913 KASSERT(sz >= sizeof(remain32),
4914 ("umtx_copyops specifies incorrect sizes"));
4915
4916 return (copyout(&remain32, uaddr, sizeof(remain32)));
4917 }
4918 #endif /* __i386__ || __LP64__ */
4919
4920 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap,
4921 const struct umtx_copyops *umtx_ops);
4922
4923 static const _umtx_op_func op_table[] = {
4924 #ifdef COMPAT_FREEBSD10
4925 [UMTX_OP_LOCK] = __umtx_op_lock_umtx,
4926 [UMTX_OP_UNLOCK] = __umtx_op_unlock_umtx,
4927 #else
4928 [UMTX_OP_LOCK] = __umtx_op_unimpl,
4929 [UMTX_OP_UNLOCK] = __umtx_op_unimpl,
4930 #endif
4931 [UMTX_OP_WAIT] = __umtx_op_wait,
4932 [UMTX_OP_WAKE] = __umtx_op_wake,
4933 [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex,
4934 [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex,
4935 [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex,
4936 [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling,
4937 [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait,
4938 [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal,
4939 [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast,
4940 [UMTX_OP_WAIT_UINT] = __umtx_op_wait_uint,
4941 [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock,
4942 [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock,
4943 [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock,
4944 [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private,
4945 [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private,
4946 [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex,
4947 [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex,
4948 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4949 [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait,
4950 [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake,
4951 #else
4952 [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl,
4953 [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl,
4954 #endif
4955 [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private,
4956 [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex,
4957 [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait,
4958 [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake,
4959 [UMTX_OP_SHM] = __umtx_op_shm,
4960 [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists,
4961 [UMTX_OP_GET_MIN_TIMEOUT] = __umtx_op_get_min_timeout,
4962 [UMTX_OP_SET_MIN_TIMEOUT] = __umtx_op_set_min_timeout,
4963 };
4964
4965 static const struct umtx_copyops umtx_native_ops = {
4966 .copyin_timeout = umtx_copyin_timeout,
4967 .copyin_umtx_time = umtx_copyin_umtx_time,
4968 .copyin_robust_lists = umtx_copyin_robust_lists,
4969 .copyout_timeout = umtx_copyout_timeout,
4970 .timespec_sz = sizeof(struct timespec),
4971 .umtx_time_sz = sizeof(struct _umtx_time),
4972 };
4973
4974 #ifndef __i386__
4975 static const struct umtx_copyops umtx_native_opsi386 = {
4976 .copyin_timeout = umtx_copyin_timeouti386,
4977 .copyin_umtx_time = umtx_copyin_umtx_timei386,
4978 .copyin_robust_lists = umtx_copyin_robust_lists32,
4979 .copyout_timeout = umtx_copyout_timeouti386,
4980 .timespec_sz = sizeof(struct timespeci386),
4981 .umtx_time_sz = sizeof(struct umtx_timei386),
4982 .compat32 = true,
4983 };
4984 #endif
4985
4986 #if defined(__i386__) || defined(__LP64__)
4987 /* i386 can emulate other 32-bit archs, too! */
4988 static const struct umtx_copyops umtx_native_opsx32 = {
4989 .copyin_timeout = umtx_copyin_timeoutx32,
4990 .copyin_umtx_time = umtx_copyin_umtx_timex32,
4991 .copyin_robust_lists = umtx_copyin_robust_lists32,
4992 .copyout_timeout = umtx_copyout_timeoutx32,
4993 .timespec_sz = sizeof(struct timespecx32),
4994 .umtx_time_sz = sizeof(struct umtx_timex32),
4995 .compat32 = true,
4996 };
4997
4998 #ifdef COMPAT_FREEBSD32
4999 #ifdef __amd64__
5000 #define umtx_native_ops32 umtx_native_opsi386
5001 #else
5002 #define umtx_native_ops32 umtx_native_opsx32
5003 #endif
5004 #endif /* COMPAT_FREEBSD32 */
5005 #endif /* __i386__ || __LP64__ */
5006
5007 #define UMTX_OP__FLAGS (UMTX_OP__32BIT | UMTX_OP__I386)
5008
5009 static int
kern__umtx_op(struct thread * td,void * obj,int op,unsigned long val,void * uaddr1,void * uaddr2,const struct umtx_copyops * ops)5010 kern__umtx_op(struct thread *td, void *obj, int op, unsigned long val,
5011 void *uaddr1, void *uaddr2, const struct umtx_copyops *ops)
5012 {
5013 struct _umtx_op_args uap = {
5014 .obj = obj,
5015 .op = op & ~UMTX_OP__FLAGS,
5016 .val = val,
5017 .uaddr1 = uaddr1,
5018 .uaddr2 = uaddr2
5019 };
5020
5021 if ((uap.op >= nitems(op_table)))
5022 return (EINVAL);
5023 return ((*op_table[uap.op])(td, &uap, ops));
5024 }
5025
5026 int
sys__umtx_op(struct thread * td,struct _umtx_op_args * uap)5027 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap)
5028 {
5029 static const struct umtx_copyops *umtx_ops;
5030
5031 umtx_ops = &umtx_native_ops;
5032 #ifdef __LP64__
5033 if ((uap->op & (UMTX_OP__32BIT | UMTX_OP__I386)) != 0) {
5034 if ((uap->op & UMTX_OP__I386) != 0)
5035 umtx_ops = &umtx_native_opsi386;
5036 else
5037 umtx_ops = &umtx_native_opsx32;
5038 }
5039 #elif !defined(__i386__)
5040 /* We consider UMTX_OP__32BIT a nop on !i386 ILP32. */
5041 if ((uap->op & UMTX_OP__I386) != 0)
5042 umtx_ops = &umtx_native_opsi386;
5043 #else
5044 /* Likewise, UMTX_OP__I386 is a nop on i386. */
5045 if ((uap->op & UMTX_OP__32BIT) != 0)
5046 umtx_ops = &umtx_native_opsx32;
5047 #endif
5048 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5049 uap->uaddr2, umtx_ops));
5050 }
5051
5052 #ifdef COMPAT_FREEBSD32
5053 #ifdef COMPAT_FREEBSD10
5054 int
freebsd10_freebsd32__umtx_lock(struct thread * td,struct freebsd10_freebsd32__umtx_lock_args * uap)5055 freebsd10_freebsd32__umtx_lock(struct thread *td,
5056 struct freebsd10_freebsd32__umtx_lock_args *uap)
5057 {
5058 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
5059 }
5060
5061 int
freebsd10_freebsd32__umtx_unlock(struct thread * td,struct freebsd10_freebsd32__umtx_unlock_args * uap)5062 freebsd10_freebsd32__umtx_unlock(struct thread *td,
5063 struct freebsd10_freebsd32__umtx_unlock_args *uap)
5064 {
5065 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
5066 }
5067 #endif /* COMPAT_FREEBSD10 */
5068
5069 int
freebsd32__umtx_op(struct thread * td,struct freebsd32__umtx_op_args * uap)5070 freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap)
5071 {
5072
5073 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5074 uap->uaddr2, &umtx_native_ops32));
5075 }
5076 #endif /* COMPAT_FREEBSD32 */
5077
5078 void
umtx_thread_init(struct thread * td)5079 umtx_thread_init(struct thread *td)
5080 {
5081
5082 td->td_umtxq = umtxq_alloc();
5083 td->td_umtxq->uq_thread = td;
5084 }
5085
5086 void
umtx_thread_fini(struct thread * td)5087 umtx_thread_fini(struct thread *td)
5088 {
5089
5090 umtxq_free(td->td_umtxq);
5091 }
5092
5093 /*
5094 * It will be called when new thread is created, e.g fork().
5095 */
5096 void
umtx_thread_alloc(struct thread * td)5097 umtx_thread_alloc(struct thread *td)
5098 {
5099 struct umtx_q *uq;
5100
5101 uq = td->td_umtxq;
5102 uq->uq_inherited_pri = PRI_MAX;
5103
5104 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
5105 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
5106 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
5107 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
5108 }
5109
5110 /*
5111 * exec() hook.
5112 *
5113 * Clear robust lists for all process' threads, not delaying the
5114 * cleanup to thread exit, since the relevant address space is
5115 * destroyed right now.
5116 */
5117 void
umtx_exec(struct proc * p)5118 umtx_exec(struct proc *p)
5119 {
5120 struct thread *td;
5121
5122 KASSERT(p == curproc, ("need curproc"));
5123 KASSERT((p->p_flag & P_HADTHREADS) == 0 ||
5124 (p->p_flag & P_STOPPED_SINGLE) != 0,
5125 ("curproc must be single-threaded"));
5126 /*
5127 * There is no need to lock the list as only this thread can be
5128 * running.
5129 */
5130 FOREACH_THREAD_IN_PROC(p, td) {
5131 KASSERT(td == curthread ||
5132 ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)),
5133 ("running thread %p %p", p, td));
5134 umtx_thread_cleanup(td);
5135 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0;
5136 }
5137
5138 p->p_umtx_min_timeout = 0;
5139 }
5140
5141 /*
5142 * thread exit hook.
5143 */
5144 void
umtx_thread_exit(struct thread * td)5145 umtx_thread_exit(struct thread *td)
5146 {
5147
5148 umtx_thread_cleanup(td);
5149 }
5150
5151 static int
umtx_read_uptr(struct thread * td,uintptr_t ptr,uintptr_t * res,bool compat32)5152 umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res, bool compat32)
5153 {
5154 u_long res1;
5155 uint32_t res32;
5156 int error;
5157
5158 if (compat32) {
5159 error = fueword32((void *)ptr, &res32);
5160 if (error == 0)
5161 res1 = res32;
5162 } else {
5163 error = fueword((void *)ptr, &res1);
5164 }
5165 if (error == 0)
5166 *res = res1;
5167 else
5168 error = EFAULT;
5169 return (error);
5170 }
5171
5172 static void
umtx_read_rb_list(struct thread * td,struct umutex * m,uintptr_t * rb_list,bool compat32)5173 umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list,
5174 bool compat32)
5175 {
5176 struct umutex32 m32;
5177
5178 if (compat32) {
5179 memcpy(&m32, m, sizeof(m32));
5180 *rb_list = m32.m_rb_lnk;
5181 } else {
5182 *rb_list = m->m_rb_lnk;
5183 }
5184 }
5185
5186 static int
umtx_handle_rb(struct thread * td,uintptr_t rbp,uintptr_t * rb_list,bool inact,bool compat32)5187 umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact,
5188 bool compat32)
5189 {
5190 struct umutex m;
5191 int error;
5192
5193 KASSERT(td->td_proc == curproc, ("need current vmspace"));
5194 error = copyin((void *)rbp, &m, sizeof(m));
5195 if (error != 0)
5196 return (error);
5197 if (rb_list != NULL)
5198 umtx_read_rb_list(td, &m, rb_list, compat32);
5199 if ((m.m_flags & UMUTEX_ROBUST) == 0)
5200 return (EINVAL);
5201 if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid)
5202 /* inact is cleared after unlock, allow the inconsistency */
5203 return (inact ? 0 : EINVAL);
5204 return (do_unlock_umutex(td, (struct umutex *)rbp, true));
5205 }
5206
5207 static void
umtx_cleanup_rb_list(struct thread * td,uintptr_t rb_list,uintptr_t * rb_inact,const char * name,bool compat32)5208 umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact,
5209 const char *name, bool compat32)
5210 {
5211 int error, i;
5212 uintptr_t rbp;
5213 bool inact;
5214
5215 if (rb_list == 0)
5216 return;
5217 error = umtx_read_uptr(td, rb_list, &rbp, compat32);
5218 for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) {
5219 if (rbp == *rb_inact) {
5220 inact = true;
5221 *rb_inact = 0;
5222 } else
5223 inact = false;
5224 error = umtx_handle_rb(td, rbp, &rbp, inact, compat32);
5225 }
5226 if (i == umtx_max_rb && umtx_verbose_rb) {
5227 uprintf("comm %s pid %d: reached umtx %smax rb %d\n",
5228 td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb);
5229 }
5230 if (error != 0 && umtx_verbose_rb) {
5231 uprintf("comm %s pid %d: handling %srb error %d\n",
5232 td->td_proc->p_comm, td->td_proc->p_pid, name, error);
5233 }
5234 }
5235
5236 /*
5237 * Clean up umtx data.
5238 */
5239 static void
umtx_thread_cleanup(struct thread * td)5240 umtx_thread_cleanup(struct thread *td)
5241 {
5242 struct umtx_q *uq;
5243 struct umtx_pi *pi;
5244 uintptr_t rb_inact;
5245 bool compat32;
5246
5247 /*
5248 * Disown pi mutexes.
5249 */
5250 uq = td->td_umtxq;
5251 if (uq != NULL) {
5252 if (uq->uq_inherited_pri != PRI_MAX ||
5253 !TAILQ_EMPTY(&uq->uq_pi_contested)) {
5254 mtx_lock(&umtx_lock);
5255 uq->uq_inherited_pri = PRI_MAX;
5256 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
5257 pi->pi_owner = NULL;
5258 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
5259 }
5260 mtx_unlock(&umtx_lock);
5261 }
5262 sched_lend_user_prio_cond(td, PRI_MAX);
5263 }
5264
5265 compat32 = (td->td_pflags2 & TDP2_COMPAT32RB) != 0;
5266 td->td_pflags2 &= ~TDP2_COMPAT32RB;
5267
5268 if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0)
5269 return;
5270
5271 /*
5272 * Handle terminated robust mutexes. Must be done after
5273 * robust pi disown, otherwise unlock could see unowned
5274 * entries.
5275 */
5276 rb_inact = td->td_rb_inact;
5277 if (rb_inact != 0)
5278 (void)umtx_read_uptr(td, rb_inact, &rb_inact, compat32);
5279 umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, "", compat32);
5280 umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv ", compat32);
5281 if (rb_inact != 0)
5282 (void)umtx_handle_rb(td, rb_inact, NULL, true, compat32);
5283 }
5284